.singleton("clientFactory", CrawlerClientFactory.class);
        clientFactory = container.getComponent("clientFactory");
        FaultTolerantClient httpClient = container.getComponent("httpClient");
        httpClient.setCrawlerClient(new HcHttpClient());
        clientFactory.addClient("http:.*", httpClient);
        clientFactory.addClient("https:.*", httpClient);

     */
    @Resource
    protected IntervalController intervalController;

    /**
     * Factory for creating crawler clients.
     */
    @Resource
    protected CrawlerClientFactory clientFactory;

    /**
     * Context object containing crawler state and configuration.
     */
    protected CrawlerContext crawlerContext;

    /**

        this.noWaitOnFolder = noWaitOnFolder;
    }

    /**
     * Sets the client factory.
     * @param clientFactory The client factory.
     */
    public void setClientFactory(final CrawlerClientFactory clientFactory) {
        this.clientFactory = clientFactory;
    }

    /**
     * Sets the crawler context.
     * @param crawlerContext The CrawlerContext instance.
     */

            }
            return null;
        })
                .filter(StringUtil::isNotBlank)
                .findFirst()//
                .map(s -> clientFactory.getClient(s + ":" + url))//
                .orElseGet(() -> clientFactory.getClient(url));
        if (logger.isDebugEnabled()) {
            logger.debug("CrawlerClient: {}", client.getClass().getCanonicalName());
        }
        return client;

                })
                .singleton("dataHelper", MemoryDataHelper.class)
                .singleton("robotsTxtHelper", RobotsTxtHelper.class)
                .<CrawlerClientFactory> singleton("clientFactory", CrawlerClientFactory.class, factory -> {
                    factory.addClient("http:.*", container.getComponent("httpClient"));
                    factory.addClient("file:.*", container.getComponent("fsClient"));