* Constructs a new PathPattern from the given robots.txt path pattern.
         * @param pattern the path pattern string from robots.txt (may contain * and $)
         */
        public PathPattern(final String pattern) {
            this.pattern = pattern;
            this.regexPattern = compilePattern(pattern);
            this.priorityLength = calculatePriorityLength(pattern);
        }

        /**

        }
        if (client == null) {
            throw new CrawlerSystemException("CrawlerClient instance is null. Cannot register null client.");
        }
        clientMap.put(Pattern.compile(regex), client);
    }

    /**
     * Adds a client with a regular expression pattern at a specific position.
     * @param regex The regular expression to match URLs.
     * @param client The CrawlerClient instance.

    /** Format identifier for PDF date parsing */
    private static final String PDF_DATE = "pdf_date";

    /** Regular expression pattern for matching email addresses */
    private static final Pattern EMAIL_ADDRESS_PATTERN =
            Pattern.compile("[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}", Pattern.CASE_INSENSITIVE);

    /**
     * Cache for storing resource file modification timestamps to enable cache busting.

import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.exception.ExtractException;
import org.commonmark.ext.front.matter.YamlFrontMatterExtension;
import org.commonmark.ext.front.matter.YamlFrontMatterVisitor;
import org.commonmark.node.AbstractVisitor;
import org.commonmark.node.Heading;
import org.commonmark.node.Link;
import org.commonmark.node.Node;

     */
    private static final String EXCLUDE = "exclude";

    /**
     * Cache for include filters.
     */
    protected LoadingCache<String, List<Pattern>> includeFilterCache;

    /**
     * Cache for exclude filters.
     */
    protected LoadingCache<String, List<Pattern>> excludeFilterCache;

    /**
     * The expiration time for the filter cache in seconds.
     */

    </magic>
    <glob pattern="*.bin"/>
    <glob pattern="*.dms"/>
    <glob pattern="*.lha"/>
    <glob pattern="*.lrf"/>
    <glob pattern="*.lzh"/>
    <glob pattern="*.so"/>
    <glob pattern="*.dist"/>
    <glob pattern="*.distz"/>
    <glob pattern="*.pkg"/>
    <glob pattern="*.bpk"/>
    <glob pattern="*.dump"/>
    <glob pattern="*.elc"/>
    <glob pattern="*.deploy"/>

        // Setup mock returns
        List<Pattern> mockIncludePatterns = new ArrayList<>();
        mockIncludePatterns.add(Pattern.compile("https://example.com/.*"));
        mockIncludePatterns.add(Pattern.compile("https://test.com/.*"));
        mockIncludePatterns.add(Pattern.compile("https://sample.org/.*"));

        List<Pattern> mockExcludePatterns = new ArrayList<>();

    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */
    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */

                    logger.debug("Using exclude pattern: {}", pattern);
                }
            }
            if (paramMap.get(URL_EXCLUDE_PATTERN) instanceof final Pattern pattern) {
                if (pattern.matcher(url).matches()) {
                    if (logger.isDebugEnabled()) {
                        logger.debug("Skipping URL {} due to exclude pattern: {}", url, pattern);
                    }
                    return false;

            return false;
        }

        Pattern pattern = (Pattern) propMap.get(USER_CODE_PATTERN);
        if (pattern == null) {
            pattern = Pattern.compile(getUserCodePattern());
            propMap.put(USER_CODE_PATTERN, pattern);
        }
        return pattern.matcher(userCode).matches();
    }

    String getQueryCollapseInnerHitsSorts();