} finally {
            CloseableUtil.closeQuietly(in);
        }

        // Should not throw exception for invalid crawl-delay
        assertNotNull(robotsTxt);
        // Invalid crawl-delay should be ignored (default 0)
        assertEquals(0, robotsTxt.getCrawlDelay("TestBot"));
        // Other directives should still work
        assertFalse(robotsTxt.allows("/test/", "TestBot"));
    }

        // First access - should not delay
        final long start1 = System.nanoTime();
        controller.delayBeforeProcessing();
        final long elapsed1 = (System.nanoTime() - start1) / 1000000;
        assertTrue("First access should not delay", elapsed1 < 50);

        // Second access to same host - should delay
        q.setUrl("http://example.com/page2");

    }

    /**
     * Gets recommended retry delay in milliseconds
     *
     * @return retry delay or 0 if no retry
     */
    public long getRetryDelay() {
        switch (severity) {
        case RECOVERABLE:
            return 100; // Immediate retry with small delay
        case TRANSIENT:
            return 5000; // Wait 5 seconds
        default:

     *
     * @return the registration timeout
     */
    long getWitnessRegistrationTimeout();

    /**
     * Gets the witness reconnect delay in milliseconds.
     *
     * @return the reconnect delay
     */
    long getWitnessReconnectDelay();

    /**
     * Gets whether witness service discovery is enabled.
     *
     * @return true if discovery is enabled
     */

            try {
                // Wait before retry (except first attempt)
                if (attempt > 0) {
                    long delay = retryDelay * (1L << attempt); // Exponential backoff
                    Thread.sleep(delay);
                    log.debug("Reconnection attempt {} for {}", attempt + 1, info.getPath());
                }

                // Perform the actual reconnection

    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */
    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */

            Thread.sleep(5); // Small delay to ensure different timestamps
            Smb2LeaseKey key2 = limitedManager.requestLease("/share/file2.txt", Smb2LeaseState.SMB2_LEASE_READ_CACHING);
            Thread.sleep(5); // Small delay to ensure different timestamps

    }

    private void scheduleRecovery(ChannelInfo channel, FailoverState state) {
        state.incrementRetry();

        // For test purposes, execute recovery immediately without delay
        // In production, this might use the delay from getNextRetryTime()
        executor.submit(() -> attemptRecovery(channel, state));
    }

```

### CrawlerThread (`CrawlerThread.java`)

Worker thread for crawling.

**Flow**: Poll URL → Validate → Get client → Delay → Check last-modified → Execute → Process → Extract children → Queue children → Delay

### CrawlerClientFactory

Pattern-based client selection using `LinkedHashMap<Pattern, CrawlerClient>`.

**Standard Patterns**:
```java

sina
här
ha
mot
alla
under
någon
eller
allt
mycket
sedan
ju
denna
själv
detta
åt
utan
varit
hur
ingen
mitt
ni
bli
blev
oss
din
dessa
några
deras
blir
mina
samma
vilken
er
sådan
vår
blivit
dess
inom
mellan
sådant
varför
varje
vilka
ditt
vem
vilket
sitta
sådana
vart
dina
vars
vårt