```

### Helpers

**RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
**SitemapsHelper**: Sitemap XML parsing, index handling
**MimeTypeHelper**: MIME detection via Tika
**EncodingHelper**: Charset detection with BOM
**UrlConvertHelper**: URL normalization

---

## Development Workflow

### Build Commands

```bash

        container.singleton("archiveStreamFactory", ArchiveStreamFactory.class)
                .singleton("compressorStreamFactory", CompressorStreamFactory.class)
                .singleton("mimeTypeHelper", MimeTypeHelperImpl.class)
                .singleton("tikaExtractor", TikaExtractor.class)
                .singleton("textExtractor", TextExtractor.class)
                .singleton("zipExtractor", ZipExtractor.class)

        logger.info("endpoint: {}", endpoint);

        StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("mimeTypeHelper", MimeTypeHelperImpl.class)//
                .singleton("storageClient", StorageClient.class);
        storageClient = container.getComponent("storageClient");
        Map<String, Object> params = new HashMap<>();

        logger.info("endpoint: {}", endpoint);

        StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("mimeTypeHelper", MimeTypeHelperImpl.class)//
                .singleton("s3Client", S3Client.class);
        s3Client = container.getComponent("s3Client");
        Map<String, Object> params = new HashMap<>();

        String endpoint = String.format("http://%s:%s", gcsServer.getHost(), mappedPort);
        logger.info("endpoint: {}", endpoint);

        StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("mimeTypeHelper", MimeTypeHelperImpl.class)//
                .singleton("gcsClient", GcsClient.class);
        gcsClient = container.getComponent("gcsClient");
        Map<String, Object> params = new HashMap<>();

    public FtpClient ftpClient;

    @Override
    protected void setUp() throws Exception {
        super.setUp();
        StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("mimeTypeHelper", MimeTypeHelperImpl.class)//
                .singleton("ftpClient", FtpClient.class);
        ftpClient = container.getComponent("ftpClient");
    }

                .singleton("encodingHelper", EncodingHelper.class)
                .singleton("contentLengthHelper", ContentLengthHelper.class)
                .singleton("mimeTypeHelper", MimeTypeHelperImpl.class)
                .<FileTransformer> singleton("fileTransformer", FileTransformer.class, transformer -> {
                    transformer.setName("fileTransformer");

    @Override
    protected void setUp() throws Exception {
        super.setUp();

        StandardCrawlerContainer container = new StandardCrawlerContainer();
        container.singleton("mimeTypeHelper", MimeTypeHelperImpl.class)
                .singleton("tikaExtractor", TikaExtractor.class)
                .<ExtractorFactory> singleton("extractorFactory", ExtractorFactory.class, factory -> {