protected static final Pattern DISALLOW_RECORD = Pattern.compile("^disallow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing allow records. */
    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */

 *   to be executed when the computation is complete.
 *   The Promise class has then and error methods that allow you to register callbacks
 *   for successful and unsuccessful computations, respectively.
 * </p>
 *
 * <p>
 *   The Deferred class uses a CountDownLatch to allow you to wait for the computation to complete.
 *   The resolve and reject methods decrement the CountDownLatch, allowing the getResponse method

 *
 * <p>Key features:</p>
 * <ul>
 *   <li>Supports multiple user-agent directives with pattern matching</li>
 *   <li>Handles Allow and Disallow rules for path-based access control</li>
 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *

        final String clusterName = UUID.randomUUID().toString();
        runner.onBuild((number, settingsBuilder) -> {
            settingsBuilder.put("http.cors.enabled", true);
            settingsBuilder.put("http.cors.allow-origin", "*");
            settingsBuilder.put("discovery.type", "single-node");
        }).build(newConfigs().clusterName(clusterName).numOfNode(1));

        // wait for yellow status
        runner.ensureYellow();

        this.useRobotsTxtDisallows = useRobotsTxtDisallows;
    }

    /**
     * Sets whether to use robots.txt allow rules.
     *
     * @param useRobotsTxtAllows True to use allow rules, false otherwise
     */
    public void setUseRobotsTxtAllows(final boolean useRobotsTxtAllows) {
        this.useRobotsTxtAllows = useRobotsTxtAllows;
    }