User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

    protected static final Pattern DISALLOW_RECORD = Pattern.compile("^disallow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing allow records. */
    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */

package org.codelibs.fess.crawler.entity;

import java.io.Serializable;

/**
 * The Sitemap interface represents a sitemap entity with location and last modification date.
 * It extends the Serializable interface to allow sitemap objects to be serialized.
 */
public interface Sitemap extends Serializable {

    /**
     * Retrieves the location (URL) of the sitemap.
     *
     * @return the location of the sitemap as a String.

import org.codelibs.fess.crawler.entity.ResponseData;

/**
 * Interface representing a client for a web crawler.
 * This client is responsible for executing requests and handling responses.
 * It extends {@link AutoCloseable} to allow for resource management.
 */
public interface CrawlerClient extends AutoCloseable {

    /**
     * Sets the initialization parameters for the crawler client.
     *

 *   to be executed when the computation is complete.
 *   The Promise class has then and error methods that allow you to register callbacks
 *   for successful and unsuccessful computations, respectively.
 * </p>
 *
 * <p>
 *   The Deferred class uses a CountDownLatch to allow you to wait for the computation to complete.
 *   The resolve and reject methods decrement the CountDownLatch, allowing the getResponse method

 *
 * <p>Key features:</p>
 * <ul>
 *   <li>Supports multiple user-agent directives with pattern matching</li>
 *   <li>Handles Allow and Disallow rules for path-based access control</li>
 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *

        final String clusterName = UUID.randomUUID().toString();
        runner.onBuild((number, settingsBuilder) -> {
            settingsBuilder.put("http.cors.enabled", true);
            settingsBuilder.put("http.cors.allow-origin", "*");
            settingsBuilder.put("discovery.type", "single-node");
        }).build(newConfigs().clusterName(clusterName).numOfNode(1));

        // wait for yellow status
        runner.ensureYellow();

        this.useRobotsTxtDisallows = useRobotsTxtDisallows;
    }

    /**
     * Sets whether to use robots.txt allow rules.
     *
     * @param useRobotsTxtAllows True to use allow rules, false otherwise
     */
    public void setUseRobotsTxtAllows(final boolean useRobotsTxtAllows) {
        this.useRobotsTxtAllows = useRobotsTxtAllows;
    }

    <!-- and a lower priority than pcap - TIKA-4174 -->
    <magic priority="55">
      <match minShouldMatch="2">
        <match value="user-agent:" type="stringignorecase" offset="0"/>
        <match value="allow:" type="stringignorecase" offset="0"/>
        <match value="disallow:" type="stringignorecase" offset="0"/>
        <match value="sitemap:" type="stringignorecase" offset="0"/>