</magic>
    <glob pattern="*.bin"/>
    <glob pattern="*.dms"/>
    <glob pattern="*.lha"/>
    <glob pattern="*.lrf"/>
    <glob pattern="*.lzh"/>
    <glob pattern="*.so"/>
    <glob pattern="*.dist"/>
    <glob pattern="*.distz"/>
    <glob pattern="*.pkg"/>
    <glob pattern="*.bpk"/>
    <glob pattern="*.dump"/>
    <glob pattern="*.elc"/>
    <glob pattern="*.deploy"/>

    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */
    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */

    /**
     * Pattern for XML tags.
     */
    protected Pattern xmlTagPattern = Pattern.compile("<[^>]+>");

    /**
     * Returns the encoding pattern.
     * @return The encoding pattern.
     */
    @Override
    protected Pattern getEncodingPattern() {
        return xmlEncodingPattern;
    }

    /**

import org.opensearch.index.analysis.AbstractTokenFilterFactory;

public class PatternConcatenationFilterFactory extends AbstractTokenFilterFactory {

    private Pattern pattern1;

    private Pattern pattern2;

    public PatternConcatenationFilterFactory(final IndexSettings indexSettings, final Environment environment, final String name,
            final Settings settings) {
        super(indexSettings, name, settings);

     * @return the parsed charset name, or null if not found
     */
    protected String parseCharset(final String content) {
        final Pattern pattern = Pattern.compile("; *charset *= *([a-zA-Z0-9\\-_]+)", Pattern.CASE_INSENSITIVE);
        final Matcher matcher = pattern.matcher(content);
        if (matcher.find()) {
            return matcher.group(1);
        }
        return null;
    }

    /**

     * @param url URL
     * @return true if url is matched
     */
    boolean match(String url);

    /**
     * Add an url pattern as a target.
     *
     * @param urlPattern Regular expression that is crawled
     */
    void addInclude(String urlPattern);

    /**
     * Add an url pattern as a non-target.
     *
     * @param urlPattern Regular expression that is not crawled
     */

     */
    private CharUtil() {
    }

    /**
     * Checks if the given character is a valid URL character.
     *
     * Valid URL characters include:
     * - Lowercase letters (a-z)
     * - Uppercase letters (A-Z)
     * - Digits (0-9)
     * - Special characters: . - * _ : / + % = & ? # [ ] @ ~ ! $ ' ( ) , ;
     *
     * @param c the character to check

        sitemapsRule.addRule("url", Pattern.compile(".*sitemap.*"));
        assertTrue(sitemapsRule.match(responseData));
        InputStream is = responseData.getResponseBody();
        assertTrue(is instanceof InputStream);
        CloseableUtil.closeQuietly(responseData);
    }

    private void assertMatchFalse(ResponseData responseData) {
        sitemapsRule.addRule("url", Pattern.compile(".*sitemap.*"));

crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns
crawler.urlFilter.addInclude("https://example.com/.*");
crawler.urlFilter.addInclude(".*\\.pdf$");

// Exclude patterns  
crawler.urlFilter.addExclude(".*\\.js$");
crawler.urlFilter.addExclude(".*login.*");
```

## Supported Protocols and Formats

    /** Pattern for extracting charset from meta tags. */
    protected Pattern metaCharsetPattern = Pattern.compile("<meta.*content\\s*=\\s*['\"].*;\\s*charset=([\\w\\d\\-_]*)['\"]\\s*/?>",
            Pattern.MULTILINE | Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for HTML tags.
     */
    protected Pattern htmlTagPattern = Pattern.compile("<[^>]+>");

    /** Map of parser features. */