}
    }

    /**
     * Applies delay based on the configured interval rules.
     * This method calculates the appropriate delay for the current time
     * and applies it by sleeping the current thread.
     */
    public void delayByRules() {
        final long delay = getDelay();
        if (delay > 0) {
            ThreadUtil.sleep(delay);
        }
    }

    /**

 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.
 * If the time since the last request to the host is less than the configured delay,
 * the thread waits until the delay has elapsed.
 * This class is thread-safe.

    }

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.
     */

        } finally {
            CloseableUtil.closeQuietly(in);
        }

        // Should not throw exception for invalid crawl-delay
        assertNotNull(robotsTxt);
        // Invalid crawl-delay should be ignored (default 0)
        assertEquals(0, robotsTxt.getCrawlDelay("TestBot"));
        // Other directives should still work
        assertFalse(robotsTxt.allows("/test/", "TestBot"));
    }

        // First access - should not delay
        final long start1 = System.nanoTime();
        controller.delayBeforeProcessing();
        final long elapsed1 = (System.nanoTime() - start1) / 1000000;
        assertTrue("First access should not delay", elapsed1 < 50);

        // Second access to same host - should delay
        q.setUrl("http://example.com/page2");

```

### CrawlerThread (`CrawlerThread.java`)

Worker thread for crawling.

**Flow**: Poll URL → Validate → Get client → Delay → Check last-modified → Execute → Process → Extract children → Queue children → Delay

### CrawlerClientFactory

Pattern-based client selection using `LinkedHashMap<Pattern, CrawlerClient>`.

**Standard Patterns**:
```java

    runner.close();
    runner.clean();
}
```

### Testing Best Practices

- Minimize `Thread.sleep()` - use only when absolutely necessary
- For timestamp tests: use minimal delays (50-100ms)
- Clean only test-specific indices (not `_all` unless needed)
- Use `runner.refresh()` after index operations
- Test with realistic multilingual content

### Key Test Classes

    /**
     * Container for managing crawler components.
     */
    @Resource
    protected CrawlerContainer crawlerContainer;

    /**
     * Controller for managing crawling intervals and delays.
     */
    @Resource
    protected IntervalController intervalController;

    /**
     * Factory for creating crawler clients.
     */
    @Resource

        crawlerContext.setStatus(CrawlerStatus.RUNNING);
        crawlerContext.maxThreadCheckCount = 1; // Will exit after 1 check

        crawlerThread.run();

        verify(intervalController, times(1)).delay(IntervalController.NO_URL_IN_QUEUE);
    }

    /**
     * Test isContinue with active threads still running.
     */
    public void test_isContinue_withActiveThreads() throws Exception {

                    logger.debug("Interrupted.", e);
                }
            }
        }
    }

    /**
     * Sets the crawling execution interval.
     * This interval controls the delay between starting new crawler threads
     * and the frequency of status checks during crawling operations.
     *
     * @param crawlingExecutionInterval interval in milliseconds
     */