buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');
            buf.append("<loc>http://localhost:7070/index.html</loc>").append('\n');

			tgtName := strings.TrimPrefix(k, envVarPrefix)
			if tgtName != "" {
				if v, ok := filterMap[k]; ok {
					if strings.HasPrefix(envVarPrefix, v) {
						filterMap[k] = envVarPrefix
					}
				} else {
					filterMap[k] = envVarPrefix
				}
			}
		}
	}

	for k, v := range filterMap {
		seen.Add(strings.TrimPrefix(k, v))
	}

	seen.Remove(Default)

extractorFactory.addExtractor("text/html", tikaExtractor, 1);  // Fallback
```

### Helpers

**RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
**SitemapsHelper**: Sitemap XML parsing, index handling
**MimeTypeHelper**: MIME detection via Tika
**EncodingHelper**: Charset detection with BOM
**UrlConvertHelper**: URL normalization

---

## Development Workflow

      matrix:
        dbversion: [ 'v6.5.0' ]
        go: ['1.24', '1.25']
        platform: [ ubuntu-latest ]
    runs-on: ${{ matrix.platform }}

    steps:
      - name: Setup TiDB
        uses: Icemap/tidb-action@main
        with:
          port: 9940
          version: ${{matrix.dbversion}}

      - name: Set up Go 1.x
        uses: actions/setup-go@v4
        with:

    suite.addTest(filterSortedMapSuite());
    suite.addTest(filterNavigableMapSuite());
    return suite;
  }

  static TestSuite filterMapSuite() {
    TestSuite suite = new TestSuite("FilterMap");
    suite.addTest(
        MapTestSuiteBuilder.using(
                new TestStringMapGenerator() {
                  @Override

        // Test sitemaps
        String[] sitemaps = robotsTxt.getSitemaps();
        assertEquals(1, sitemaps.length);
        assertEquals("http://www.example.com/sitemap.xml", sitemaps[0]);
    }

    public void testParse_malformed() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots_malformed.txt");
        try {

                    rule.setResponseProcessor(container.getComponent("sitemapsResponseProcessor"));
                    rule.setRuleId("sitemapsRule");
                    rule.addRule("url", ".*sitemap.*");
                })//
                .<DefaultResponseProcessor> singleton("defaultResponseProcessor", DefaultResponseProcessor.class, processor -> {

    suite.addTest(filterSortedMapSuite());
    suite.addTest(filterNavigableMapSuite());
    return suite;
  }

  static TestSuite filterMapSuite() {
    TestSuite suite = new TestSuite("FilterMap");
    suite.addTest(
        MapTestSuiteBuilder.using(
                new TestStringMapGenerator() {
                  @Override

        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

        <match value="allow:" type="stringignorecase" offset="0"/>
        <match value="disallow:" type="stringignorecase" offset="0"/>
        <match value="sitemap:" type="stringignorecase" offset="0"/>
        <match value="\nuser-agent:" type="stringignorecase" offset="0:1000"/>
        <match value="\nallow:" type="stringignorecase" offset="0:1000"/>