// Check for potential overflow before multiplication
                    long delay = (retryDelayMs > Long.MAX_VALUE / multiplier) ? MAX_RETRY_DELAY
                            : Math.min(MAX_RETRY_DELAY, retryDelayMs * multiplier);
                    Thread.sleep(delay);
                }

                // Attempt to reset the connection
                connection.reset();

        } finally {
            CloseableUtil.closeQuietly(in);
        }

        // Should not throw exception for invalid crawl-delay
        assertNotNull(robotsTxt);
        // Invalid crawl-delay should be ignored (default 0)
        assertEquals(0, robotsTxt.getCrawlDelay("TestBot"));
        // Other directives should still work
        assertFalse(robotsTxt.allows("/test/", "TestBot"));
    }

     *
     * @return the registration timeout
     */
    long getWitnessRegistrationTimeout();

    /**
     * Gets the witness reconnect delay in milliseconds.
     *
     * @return the reconnect delay
     */
    long getWitnessReconnectDelay();

    /**
     * Gets whether witness service discovery is enabled.
     *
     * @return true if discovery is enabled
     */

    }

    /**
     * Gets recommended retry delay in milliseconds
     *
     * @return retry delay or 0 if no retry
     */
    public long getRetryDelay() {
        switch (severity) {
        case RECOVERABLE:
            return 100; // Immediate retry with small delay
        case TRANSIENT:
            return 5000; // Wait 5 seconds
        default:

        // First access - should not delay
        final long start1 = System.nanoTime();
        controller.delayBeforeProcessing();
        final long elapsed1 = (System.nanoTime() - start1) / 1000000;
        assertTrue("First access should not delay", elapsed1 < 50);

        // Second access to same host - should delay
        q.setUrl("http://example.com/page2");

            Thread.sleep(5); // Small delay to ensure different timestamps
            Smb2LeaseKey key2 = limitedManager.requestLease("/share/file2.txt", Smb2LeaseState.SMB2_LEASE_READ_CACHING);
            Thread.sleep(5); // Small delay to ensure different timestamps

```

### CrawlerThread (`CrawlerThread.java`)

Worker thread for crawling.

**Flow**: Poll URL → Validate → Get client → Delay → Check last-modified → Execute → Process → Extract children → Queue children → Delay

### CrawlerClientFactory

Pattern-based client selection using `LinkedHashMap<Pattern, CrawlerClient>`.

**Standard Patterns**:
```java

            try {
                // Wait before retry (except first attempt)
                if (attempt > 0) {
                    long delay = retryDelay * (1L << attempt); // Exponential backoff
                    Thread.sleep(delay);
                    log.debug("Reconnection attempt {} for {}", attempt + 1, info.getPath());
                }

                // Perform the actual reconnection

    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */
    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */

    }

    private void scheduleRecovery(ChannelInfo channel, FailoverState state) {
        state.incrementRetry();

        // For test purposes, execute recovery immediately without delay
        // In production, this might use the delay from getNextRetryTime()
        executor.submit(() -> attemptRecovery(channel, state));
    }