settingsBuilder.put("http.cors.enabled", true);
            settingsBuilder.put("discovery.type", "single-node");
        }).build(newConfigs().clusterName(clusterName).numOfNode(1));

        // wait for yellow status
        runner.ensureYellow();

        System.setProperty(FesenClient.HTTP_ADDRESS, "localhost:" + runner.node().settings().get("http.port", "9201"));

        super.setUp();
    }

    @Override

            settingsBuilder.put("http.cors.enabled", true);
            settingsBuilder.put("discovery.type", "single-node");
        }).build(newConfigs().clusterName(clusterName).numOfNode(1));

        // wait for yellow status
        runner.ensureYellow();

        System.setProperty(FesenClient.HTTP_ADDRESS, "localhost:" + runner.node().settings().get("http.port", "9201"));

        super.setUp();
    }

    @Override

            settingsBuilder.put("http.cors.enabled", true);
            settingsBuilder.put("http.cors.allow-origin", "*");
            settingsBuilder.put("discovery.type", "single-node");
        }).build(newConfigs().clusterName(clusterName).numOfNode(1));

        // wait for yellow status
        runner.ensureYellow();

     */
    public boolean connected() {
        return connected;
    }

    /**
     * Connects to the OpenSearch cluster.
     * Creates a new client and waits for cluster health to be yellow or better.
     */
    public void connect() {
        destroy();
        client = createClient();

        final ClusterHealthResponse healthResponse =

User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

</tr>
<!-- IF TEXT-ONLY bottom padding -->
<tr>
<td height="28" style="padding:0;margin:0;line-height:1px;font-size:1px;"></td>
</tr>
<!-- end bottom padding for a text-only tweet-->
<!-- IF USER DOES NOT FOLLOW ACCOUNT else -->
<tr>
<td colspan="2" style="padding:0;margin:0;line-height:1px;font-size:1px;">
<table border="0" cellpadding="0" cellspacing="0" align="left" style="padding:0;margin:0;line-height:1px;font-size:1px;">

package org.codelibs.fess.crawler.entity;

import java.io.Serializable;

/**
 * The Sitemap interface represents a sitemap entity with location and last modification date.
 * It extends the Serializable interface to allow sitemap objects to be serialized.
 */
public interface Sitemap extends Serializable {

    /**
     * Retrieves the location (URL) of the sitemap.
     *
     * @return the location of the sitemap as a String.

                HcHttpClient.constructRedirectLocation("http://example.com/", "テスト"));
        assertEquals("http://example.com/hello%20world", HcHttpClient.constructRedirectLocation("http://example.com/", "hello world"));
        assertEquals("http://user:******@****.***/path/file",
                HcHttpClient.constructRedirectLocation("http://user:******@****.***/path/", "file"));

    System.out.println("Processed: " + processed + " URLs");
    Thread.sleep(5000);
}
```

## Contributing

We welcome contributions to Fess Crawler! Please follow these guidelines:

1. **Fork** the repository
2. **Create** a feature branch (`git checkout -b feature/amazing-feature`)
3. **Commit** your changes (`git commit -m 'Add amazing feature'`)

    protected static final Pattern DISALLOW_RECORD = Pattern.compile("^disallow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing allow records. */
    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */