# Based on RFC 9309 specification

# Test wildcard patterns
User-agent: WildcardBot
Disallow: /*.pdf$
Disallow: /admin/*.php
Disallow: /*/private/
Allow: /public/*.html

# Test end-of-path ($) patterns
User-agent: EndPathBot
Disallow: /fish$
Disallow: /temp$
Allow: /fishing

# Test complex patterns
User-agent: ComplexBot
Disallow: /
Allow: /$
Allow: /index.html$
Allow: /public/

/**
 * A factory class for managing and creating crawler clients based on URL patterns.
 * This class implements AutoCloseable to properly handle resource cleanup.
 *
 * <p>The factory maintains a map of regular expression patterns to crawler clients,
 * allowing for URL-based client selection. Clients can be added with specific patterns
 * and optionally at specific positions in the processing order.</p>
 *

    /**
     * Retrieves a list of URL patterns to include for a given session.
     *
     * @param sessionId the ID of the session for which to retrieve the include URL patterns
     * @return a list of compiled regular expression patterns representing the URLs to include
     */
    List<Pattern> getIncludeUrlPatternList(String sessionId);

    /**
     * Retrieves a list of URL patterns to be excluded for a given session.
     *

    }

    /**
     * Gets the list of include URL patterns for the specified session.
     *
     * @param sessionId The session ID.
     * @return The list of compiled include patterns.
     * @throws CrawlerSystemException if the patterns cannot be loaded.
     */
    @Override
    public List<Pattern> getIncludeUrlPatternList(final String sessionId) {
        try {

    /** Map of session IDs to include URL patterns for filtering URLs. */
    protected volatile Map<String, List<Pattern>> includeUrlPatternMap = new HashMap<>();

    /** Map of session IDs to exclude URL patterns for filtering URLs. */
    protected volatile Map<String, List<Pattern>> excludeUrlPatternMap = new HashMap<>();

    /**
     * Creates a new MemoryDataHelper instance.

     * @param regex the regular expression pattern
     */
    public void addRule(final String key, final String regex) {
        regexMap.put(key, Pattern.compile(regex));
    }

    /**
     * Adds a compiled regular expression rule for the specified field.
     * @param key the field name to match against
     * @param pattern the compiled regular expression pattern
     */

     * (java.lang.String)
     */
    /**
     * Retrieves a list of URL patterns to be excluded for a given session.
     * @param sessionId The ID of the session.
     * @return A list of compiled regular expression patterns.
     */
    @Override
    public List<Pattern> getExcludeUrlPatternList(final String sessionId) {
        return dataHelper.getExcludeUrlPatternList(sessionId);
    }

    public String urls;

    /**
     * URL patterns to include during crawling.
     */
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String includedUrls;

    /**
     * URL patterns to exclude during crawling.
     */
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String excludedUrls;

    /**
     * Document URL patterns to include in search index.
     */

     * <p>
     * Patterns can be exact class names or use wildcards with '*' at the end.
     * For example: "com.example.*" allows all classes in the com.example package.
     * </p>
     *
     * @param allowedPatterns the patterns of classes to allow
     * @return an ObjectInputFilter configured with the specified patterns
     */

        final List<Pattern> includeList = getUrlFilterService().getIncludeUrlPatternList(sessionId);
        final List<Pattern> excludeList = getUrlFilterService().getExcludeUrlPatternList(sessionId);

        if (!includeList.isEmpty()) {
            boolean match = false;
            for (final Pattern pattern : includeList) {
                final Matcher matcher = pattern.matcher(url);