while ((callback = doneCallbacks.poll()) != null) {
                executeCallbacks.add(callback);
            }
        }
        if (executeCallbacks.size() > 0) {
            try {
                executeCallbacks.stream().forEach(callback -> callback.accept(response));
            } catch (final Exception ignore) {}
        }
        latch.countDown();
    }

    /**

     */
    public RobotsTxt parse(final InputStream stream, final String charsetName) {
        if (!enabled) {
            return null;
        }

        try {
            @SuppressWarnings("resource")
            final BufferedReader reader = new BufferedReader(new InputStreamReader(new BOMInputStream(stream), charsetName));

            String line;

        for (int i = 0; i < threadCount; i++) {
            executor.submit(new Runnable() {
                @Override
                public void run() {
                    try {
                        startLatch.await();
                        for (int j = 0; j < operationsPerThread; j++) {
                            crawlerContext.incrementAndGetAccessCount();
                        }

     * @return true if the response data matches the sitemaps rule, false otherwise.
     */
    @Override
    public boolean match(final ResponseData responseData) {
        if (super.match(responseData)) {
            try (final InputStream is = responseData.getResponseBody()) {
                final SitemapsHelper sitemapsHelper = crawlerContainer.getComponent("sitemapsHelper");
                return sitemapsHelper.isValid(is);

        CrawlingParameterUtil.setCrawlerContext(crawlerContext);
        CrawlingParameterUtil.setUrlQueueService(urlQueueService);
        CrawlingParameterUtil.setDataService(dataService);
        try {
            while (crawlerContext.getStatus() != CrawlerStatus.DONE && isContinue(threadCheckCount)) {
                final UrlQueue<?> urlQueue = urlQueueService.poll(crawlerContext.sessionId);

                if (httpStatusCode < 400 || httpStatusCode == 401) {
                    parseTokenPage(tokenPattern, responseParams, entity);
                } else {
                    String content;
                    try {
                        content = new String(InputStreamUtil.getBytes(entity.getContent()), Constants.UTF_8_CHARSET);
                    } catch (final IOException e) {
                        content = e.getMessage();

    protected <T> BulkResponse doInsertAll(final List<T> list, final OpType opType) {
        try {
            return getClient().get(c -> {
                final BulkRequestBuilder bulkRequest = c.prepareBulk();
                for (final T target : list) {
                    final String id = getId(getSessionId(target), getUrl(target));
                    try (final XContentBuilder source = getXContentBuilder(target)) {

    }

    /**
     * Test creation with null component name
     */
    public void test_create_nullComponentName() {
        factory.setComponentName(null);

        try {
            factory.create();
            fail("Should throw exception for null component name");
        } catch (Exception e) {
            // Expected behavior
            assertTrue(true);
        }
    }

        crawlerContext.sessionId = sdf.format(new Date());
    }

    /**
     * Adds a URL to the crawling queue.
     * @param url The URL to add.
     */
    public void addUrl(final String url) {
        try {
            urlQueueService.add(crawlerContext.sessionId, url);
        } catch (final Exception e) {
            logger.warn("Failed to add url: " + url, e);
        }
        urlFilter.processUrl(url);
    }

    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {
            final BufferedInputStream bis = new BufferedInputStream(in);
            final String enc = getEncoding(bis);
            final String content = UNESCAPE_HTML4.translate(new String(InputStreamUtil.getBytes(bis), enc));