/**
     * Checks the crawler status and waits if the crawler is not running.
     * This method blocks until the crawler is running again.
     */
    public void checkCrawlerStatus() {
        while (!crawlerRunning) {
            ThreadUtil.sleepQuietly(crawlerWaitMillis);
        }
    }

    /**
     * Applies delay based on the configured interval rules.