/**
     * Checks the crawler status and waits if the crawler is not running.
     * This method blocks until the crawler is running again.
     */
    public void checkCrawlerStatus() {
        while (!crawlerRunning) {
            ThreadUtil.sleepQuietly(crawlerWaitMillis);
        }
    }

    /**
     * Applies delay based on the configured interval rules.

        helper.setCrawlerWaitMillis(50);

        // Test when crawler is running
        assertTrue(helper.isCrawlerRunning());
        long start = System.currentTimeMillis();
        helper.checkCrawlerStatus();
        long end = System.currentTimeMillis();
        assertTrue(end - start < 100); // Should return quickly

        // Test when crawler is not running
        helper.setCrawlerRunning(false);