}
        }
    }

    /**
     * Delays the crawling process before processing a URL.
     */
    protected abstract void delayBeforeProcessing();

    /**
     * Delays the crawling process after processing a URL.
     */
    protected abstract void delayAfterProcessing();

    /**
     * Delays the crawling process when there are no URLs in the queue.
     */

 * This class provides a default way to manage delays between crawler operations.
 * It allows setting delays before processing, after processing, when no URLs are in the queue,
 * and when waiting for new URLs.
 * The delays are configurable via constructor parameters.
 *
 */
public class DefaultIntervalController extends AbstractIntervalController {

    /** Delay in milliseconds after processing a URL */

 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.
 * If the time since the last request to the host is less than the configured delay,
 * the thread waits until the delay has elapsed.
 * This class is thread-safe.

    runner.close();
    runner.clean();
}
```

### Testing Best Practices

- Minimize `Thread.sleep()` - use only when absolutely necessary
- For timestamp tests: use minimal delays (50-100ms)
- Clean only test-specific indices (not `_all` unless needed)
- Use `runner.refresh()` after index operations
- Test with realistic multilingual content

### Key Test Classes

     * @param to the end time in HH:MM format
     * @param days comma-separated list of days (1=Sunday, 7=Saturday)
     * @param delay the delay in milliseconds to apply during this interval
     */
    public void addIntervalRule(final String from, final String to, final String days, final long delay) {
        ruleList.add(new IntervalRule(from, to, days, delay));
    }

    /**

    }

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.
     */

    /**
     * Container for managing crawler components.
     */
    @Resource
    protected CrawlerContainer crawlerContainer;

    /**
     * Controller for managing crawling intervals and delays.
     */
    @Resource
    protected IntervalController intervalController;

    /**
     * Factory for creating crawler clients.
     */
    @Resource

    /**
     * Updates the user's group and role membership information with lazy loading for parent groups.
     * Direct groups are retrieved synchronously, while parent groups are fetched asynchronously
     * to avoid login delays when users have many nested group memberships.
     * @param user The Entra ID user to update.
     */
    public void updateMemberOf(final EntraIdUser user) {
        if (logger.isDebugEnabled()) {

        // First access - should not delay
        final long start1 = System.nanoTime();
        controller.delayBeforeProcessing();
        final long elapsed1 = (System.nanoTime() - start1) / 1000000;
        assertTrue("First access should not delay", elapsed1 < 50);

        // Second access to same host - should delay
        q.setUrl("http://example.com/page2");

    }

    /**
     * Test multiple delay calls
     */
    public void test_multipleDelayCalls() {
        final TestIntervalController controller = new TestIntervalController();

        controller.delay(IntervalController.PRE_PROCESSING);
        controller.delay(IntervalController.POST_PROCESSING);
        controller.delay(IntervalController.NO_URL_IN_QUEUE);
        controller.delay(IntervalController.WAIT_NEW_URL);