if (logger.isInfoEnabled()) {
            logger.info("[EXEC TIME] crawling time: {}ms", execTime);
        }

        indexUpdater.setFinishCrawling(true);
        try {
            indexUpdater.join();
        } catch (final InterruptedException e) {
            logger.warn("Interrupted index update.", e);
        }

    }

    /**
     * Sets the flag indicating whether crawling should be finished.
     *
     * @param finishCrawling true if crawling should be finished, false otherwise
     */
    public void setFinishCrawling(final boolean finishCrawling) {
        this.finishCrawling = finishCrawling;
    }

    /**
     * Gets the total number of documents processed.
     *
     * @return the total document count
     */