/** Whether robots.txt processing is enabled. */
    protected boolean enabled = true;

    /**
     * Creates a new RobotsTxtHelper instance.
     */
    public RobotsTxtHelper() {
        // Default constructor
    }

    /**
     * Parses a robots.txt file from the given input stream using UTF-8 encoding.
     * @param stream the input stream to parse

                .singleton("urlFilterService", UrlFilterServiceImpl.class)//
                .singleton("urlFilter", UrlFilterImpl.class)//
                .singleton("robotsTxtHelper", RobotsTxtHelper.class)//
                .singleton("httpClient", HcHttpClient.class);
        httpClient = container.getComponent("httpClient");
        urlFilter = container.getComponent("urlFilter");
    }

    /** Logger instance for this class */
    private static final Logger logger = LogManager.getLogger(HcHttpClient.class);

    /** Helper for processing robots.txt files */
    @Resource
    protected RobotsTxtHelper robotsTxtHelper;

    /** Helper for managing content length limits */
    @Resource
    protected ContentLengthHelper contentLengthHelper;

    /** Helper for determining MIME types */
    @Resource

                    transformer.setChildUrlRuleMap(childUrlRuleMap);
                })
                .singleton("dataHelper", MemoryDataHelper.class)
                .singleton("robotsTxtHelper", RobotsTxtHelper.class)
                .<CrawlerClientFactory> singleton("clientFactory", CrawlerClientFactory.class, factory -> {
                    factory.addClient("http:.*", container.getComponent("httpClient"));