client.checkMaxContentLength(responseData);

        responseData.setContentLength(-1);
        client.checkMaxContentLength(responseData);

        responseData.setContentLength(1000L);
        client.checkMaxContentLength(responseData);

        responseData.setContentLength(1000000L);
        client.checkMaxContentLength(responseData);

        responseData.setContentLength(1000000000L);

            if (logger.isDebugEnabled()) {
                logger.debug("Interrupt {}", runningThread);
            }
            runningThread.interrupt();
            ThreadUtil.sleepQuietly(1000L);
            count++;
        }
    }

    /**
     * Stops the timeout target by setting the running flag to false.
     */
    public void stop() {
        if (logger.isDebugEnabled()) {

        for (int i = 0; i < 100; i++) {
            sb.append("word").append(i).append(" ");
        }
        String text = sb.toString();
        String field = "content";
        String lang = "en";

        List<AnalyzeToken> tokens = analyzer.analyze(text, field, lang);

        assertNotNull(tokens);
        assertEquals(100, tokens.size());
        for (int i = 0; i < 100; i++) {

        ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);
        assertTrue(processor.isSuccessful(responseData));

        responseData.setHttpStatusCode(100);
        assertFalse(processor.isSuccessful(responseData));
        responseData.setHttpStatusCode(304);
        assertFalse(processor.isSuccessful(responseData));
        responseData.setHttpStatusCode(404);

    public StorageClient storageClient;

    private GenericContainer minioServer;

    @Override
    protected void setUp() throws Exception {
        super.setUp();

        final int port = 9000;
        logger.info("Creating {}", IMAGE_NAME);
        minioServer = new GenericContainer<>(IMAGE_NAME)//
                .withEnv("MINIO_ACCESS_KEY", ACCESS_KEY)//
                .withEnv("MINIO_SECRET_KEY", SECRET_KEY)//

    protected List<CrawlerClientFactory> clientFactoryList = new LinkedList<>();

    /**
     * The maximum size of the client factory list.
     */
    protected int maxClientFactorySize = 10000;

    /**
     * Registers a CrawlerClientFactory with this creator.
     * All existing client mappings will be loaded into the new factory.
     * @param crawlerClientFactory The CrawlerClientFactory to register.
     */

    /** Delay in milliseconds before processing a URL */
    protected long delayMillisBeforeProcessing = 0L;

    /** Delay in milliseconds for waiting for new URLs */
    protected long delayMillisForWaitingNewUrl = 1000L;

    /**
     * Default constructor with default delay values.
     */
    public DefaultIntervalController() {
    }

    /**
     * Constructor with configurable delay parameters.
     *

        FtpClient client = new FtpClient() {
            @Override
            protected ResponseData getResponseData(final String uri, final boolean includeContent) {
                try {
                    Thread.sleep(2000);
                } catch (InterruptedException e) {
                    throw new CrawlingAccessException(e);
                }
                return null;
            }
        };

            task.setDaemon(isDaemonThread);
            task.start();
            task.join(timeout);
            if (!done.get()) {
                for (int i = 0; i < 100 && !done.get(); i++) {
                    task.interrupt();
                    ThreadUtil.sleep(100L);
                }
                throw new ExtractException("PDFBox process cannot finish in " + timeout + " sec.");
            }

    public void test_doGet_with4Exception_interval100() {
        final FaultTolerantClient client = new FaultTolerantClient();
        client.setRetryInterval(100);
        final TestClient testClient = new TestClient();
        testClient.exceptionCount = 4;
        testClient.interval = 100;
        final TestListener testListener = new TestListener();
        client.setCrawlerClient(testClient);
        client.setRequestListener(testListener);