"bukankah", "bukanlah", "bukannya", "cuma", "percuma", "dahulu", "dalam", "dan", "dapat", "dari", "daripada", "dekat", "demi", "demikian", "demikianlah", "sedemikian", "dengan", "depan", "di", "dia", "dialah", "dini", "diri", "dirinya", "terdiri", "dong", "dulu", "enggak", "enggaknya", "entah", "entahlah", "terhadap", "terhadapnya", "hal", "hampir", "hanya", "hanyalah", "harus", "haruslah", "harusnya", "seharusnya", "hendak", "hendaklah", "hendaknya", "hingga", "sehingga", "ia", "ialah", "ibarat",...

    /**
     * Maximum number of URLs to access during crawling.
     */
    protected long maxAccessCount = Long.MAX_VALUE;

    /**
     * Interval time in milliseconds between crawling executions.
     */
    protected long crawlingExecutionInterval = Constants.DEFAULT_CRAWLING_EXECUTION_INTERVAL;

    /**
     * Thread priority for index updater operations.
     */

     * @param begin the timestamp when tracking began
     * @param done whether tracking was completed normally
     */
    protected void printStats(final Object keyObj, final StatsObject data, final long begin, final boolean done) {
        final StringBuilder buf = createStringBuffer(keyObj, begin);
        if (done) {
            buf.append('\t').append("done:").append(getCurrentTimeMillis() - begin);
        }
        data.entrySet()

        }

        public long getConnectionsCreated() {
            return connectionsCreated.get();
        }

        public long getSuccessCount() {
            return successCount.get();
        }

        public long getFailureCount() {
            return failureCount.get();
        }

        public long getFirstConnectionTime() {
            return firstConnectionTime;

        crawlerContext.setStatus(CrawlerStatus.RUNNING);
        assertEquals(CrawlerStatus.RUNNING, crawlerContext.getStatus());

        crawlerContext.setStatus(CrawlerStatus.DONE);
        assertEquals(CrawlerStatus.DONE, crawlerContext.getStatus());

        crawlerContext.setStatus(null);
        assertNull(crawlerContext.getStatus());
    }

    /**
     * Test urlFilter getter and setter
     */

     * Stops the crawling process.
     * Sets the crawler status to DONE and interrupts all crawler threads.
     */
    public void stop() {
        if (logger.isInfoEnabled()) {
            logger.info("Stopping crawler: sessionId={}, status={}", crawlerContext.sessionId, crawlerContext.getStatus());
        }
        crawlerContext.setStatus(CrawlerStatus.DONE);
        try {
            if (crawlerThreadGroup != null) {

            @Override
            public long createTime() {
                return 0;
            }

            @Override
            public long lastModified() {
                return 0;
            }

            @Override
            public long lastAccess() {
                return 0;
            }

            @Override
            public long length() {
                return 0;
            }

        final long execTime = systemHelper.getCurrentTimeAsLong() - startTime;
        crawlingInfoHelper.putToInfoMap(Constants.DATA_CRAWLING_EXEC_TIME, Long.toString(execTime));
        if (logger.isInfoEnabled()) {
            logger.info("[EXEC TIME] crawling time: {}ms", execTime);
        }

        crawlingInfoHelper.putToInfoMap(Constants.DATA_INDEX_EXEC_TIME, Long.toString(indexUpdateCallback.getExecuteTime()));

                } finally {
                    done.set(true);
                }
            }, Thread.currentThread().getName() + "-pdf");
            task.setDaemon(isDaemonThread);
            task.start();
            task.join(timeout);
            if (!done.get()) {
                for (int i = 0; i < 100 && !done.get(); i++) {
                    task.interrupt();

    // Polling and backoff constants (in ms)
    private static final long MAX_POLL_INTERVAL = 5000; // Normal polling: max 5 seconds
    private static final long MAX_ERROR_BACKOFF = 30000; // Error polling: max 30 seconds
    private static final long BASE_POLL_INTERVAL = 1000;
    private static final long BASE_ERROR_DELAY = 1000;
    private static final int MAX_FAILURE_LIMIT = 5; // Limit failures for backoff calculation