extractorFactory.addExtractor("text/html", tikaExtractor, 1);  // Fallback
```

### Helpers

**RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
**SitemapsHelper**: Sitemap XML parsing, index handling
**MimeTypeHelper**: MIME detection via Tika
**EncodingHelper**: Charset detection with BOM
**UrlConvertHelper**: URL normalization

---

## Development Workflow

        assertFalse(robotsTxt.allows("/ddd", "Hoge Crawler"));

        String[] sitemaps = robotsTxt.getSitemaps();
        assertEquals(2, sitemaps.length);
        assertEquals("http://www.example.com/sitmap.xml", sitemaps[0]);
        assertEquals("http://www.example.net/sitmap.xml", sitemaps[1]);

    }

    public void testParse_disable() {

                    rule.setResponseProcessor(container.getComponent("sitemapsResponseProcessor"));
                    rule.setRuleId("sitemapsRule");
                    rule.addRule("url", ".*sitemap.*");
                })//
                .<DefaultResponseProcessor> singleton("defaultResponseProcessor", DefaultResponseProcessor.class, processor -> {

                    if (robotsTxt != null) {
                        final String[] sitemaps = robotsTxt.getSitemaps();
                        if (sitemaps.length > 0) {
                            crawlerContext.addSitemaps(sitemaps);
                        }

                        final RobotsTxt.Directive directive = robotsTxt.getMatchedDirective(userAgent);

sitemap2.xml http://www.example.com/sitemap1.xml.gz 2004-10-01T18:23:17+00:00 http://www.example.com/sitemap2.xml.gz 2005-01-01...

    private static final int ST_EXT = 5;

    private final byte[] in;
    private int inLen;

    /**
     * Creates a new MimeMap instance by loading MIME type mappings from the resource file.
     *
     * @throws IOException if there is an error reading the mime.map resource file
     */
    public MimeMap() throws IOException {
        int n;

        in = new byte[IN_SIZE];

sitemap1.xml http://www.example.com/ 2005-01-01 monthly 0.8 http://www.example.com/catalog?item=12&desc=vacation_hawaii weekly http://www.example.com/catalog?item=73&desc=vacation_new_zealand 2004-12-23 weekly http://www.example.com/catalog?item=74&desc=vacation_newfoundland 2004-12-23T18:00:15+00:00 0.3 http://www.example.com/catalog?item=83&desc=vacation_usa 2004-11-23...

Shinsuke Sugaya <******@****.***> 1444529815 +0900

Shinsuke Sugaya <******@****.***> 1444529815 +0900

            assertEquals("application/pdf", mimeMap.getMimeType("pdf"));
            assertEquals("application/msword", mimeMap.getMimeType("doc"));
            assertEquals("application/vnd.ms-excel", mimeMap.getMimeType("xls"));
            assertEquals("text/html", mimeMap.getMimeType("html"));
            assertEquals("text/html", mimeMap.getMimeType("htm"));
            assertEquals("image/jpeg", mimeMap.getMimeType("jpg"));