/** Whether robots.txt processing is enabled. */
    protected boolean enabled = true;

    /**
     * Creates a new RobotsTxtHelper instance.
     */
    public RobotsTxtHelper() {
        // Default constructor
    }

    /**
     * Parses a robots.txt file from the given input stream using UTF-8 encoding.
     * @param stream the input stream to parse

public class RobotsTxtHelperTest extends PlainTestCase {
    public RobotsTxtHelper robotsTxtHelper;

    @Override
    protected void setUp() throws Exception {
        super.setUp();
        StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("robotsTxtHelper", RobotsTxtHelper.class);
        robotsTxtHelper = container.getComponent("robotsTxtHelper");
    }

    public void testParse() {

                .singleton("urlFilterService", UrlFilterServiceImpl.class)//
                .singleton("urlFilter", UrlFilterImpl.class)//
                .singleton("robotsTxtHelper", RobotsTxtHelper.class)//
                .singleton("httpClient", HcHttpClient.class);
        httpClient = container.getComponent("httpClient");
        urlFilter = container.getComponent("urlFilter");
    }

    /** Logger instance for this class */
    private static final Logger logger = LogManager.getLogger(HcHttpClient.class);

    /** Helper for processing robots.txt files */
    @Resource
    protected RobotsTxtHelper robotsTxtHelper;

    /** Helper for managing content length limits */
    @Resource
    protected ContentLengthHelper contentLengthHelper;

    /** Helper for determining MIME types */
    @Resource

                    transformer.setChildUrlRuleMap(childUrlRuleMap);
                })
                .singleton("dataHelper", MemoryDataHelper.class)
                .singleton("robotsTxtHelper", RobotsTxtHelper.class)
                .<CrawlerClientFactory> singleton("clientFactory", CrawlerClientFactory.class, factory -> {
                    factory.addClient("http:.*", container.getComponent("httpClient"));

**Registration**:
```java
extractorFactory.addExtractor("text/html", htmlExtractor, 2);  // Weight 2
extractorFactory.addExtractor("text/html", tikaExtractor, 1);  // Fallback
```

### Helpers

**RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
**SitemapsHelper**: Sitemap XML parsing, index handling
**MimeTypeHelper**: MIME detection via Tika
**EncodingHelper**: Charset detection with BOM