Disallow: /admin/
InvalidDirective: some-value
unknown-field: test
Disallow /missing-colon
Disallow:    # empty value (should be treated as allowing all)
Allow: /public/
Crawl-delay: invalid-number
Crawl-delay: -10
Crawl-delay: 5.5
Crawl-delay:
Allow:    # empty value

# Case 3: Multiple colons in directive
User-agent: MultiColonBot
Disallow: http://example.com:8080/path
Allow: /path:with:colons

                    // Check for potential overflow before multiplication
                    long delay = (retryDelayMs > Long.MAX_VALUE / multiplier) ? MAX_RETRY_DELAY
                            : Math.min(MAX_RETRY_DELAY, retryDelayMs * multiplier);
                    Thread.sleep(delay);
                }

                // Attempt to reset the connection
                connection.reset();

            if (!exponentialBackoff) {
                return initialDelayMs;
            }

            long delay = initialDelayMs;
            for (int i = 1; i < attemptNumber; i++) {
                delay = (long) (delay * backoffMultiplier);
            }

            return Math.min(delay, maxDelayMs);
        }

        public int getMaxAttempts() {
            return maxAttempts;
        }

```

### CrawlerThread (`CrawlerThread.java`)

Worker thread for crawling.

**Flow**: Poll URL → Validate → Get client → Delay → Check last-modified → Execute → Process → Extract children → Queue children → Delay

### CrawlerClientFactory

Pattern-based client selection using `LinkedHashMap<Pattern, CrawlerClient>`.

**Standard Patterns**:
```java

            try {
                // Wait before retry (except first attempt)
                if (attempt > 0) {
                    long delay = retryDelay * (1L << attempt); // Exponential backoff
                    Thread.sleep(delay);
                    log.debug("Reconnection attempt {} for {}", attempt + 1, info.getPath());
                }

                // Perform the actual reconnection

        } finally {
            CloseableUtil.closeQuietly(in);
        }

        // Should not throw exception for invalid crawl-delay
        assertNotNull(robotsTxt);
        // Invalid crawl-delay should be ignored (default 0)
        assertEquals(0, robotsTxt.getCrawlDelay("TestBot"));
        // Other directives should still work
        assertFalse(robotsTxt.allows("/test/", "TestBot"));
    }

    }

    /**
     * Gets recommended retry delay in milliseconds
     *
     * @return retry delay or 0 if no retry
     */
    public long getRetryDelay() {
        switch (severity) {
        case RECOVERABLE:
            return 100; // Immediate retry with small delay
        case TRANSIENT:
            return 5000; // Wait 5 seconds
        default:

    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */
    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */

    }

    private void scheduleRecovery(ChannelInfo channel, FailoverState state) {
        state.incrementRetry();

        // For test purposes, execute recovery immediately without delay
        // In production, this might use the delay from getNextRetryTime()
        executor.submit(() -> attemptRecovery(channel, state));
    }

"sem", "mesmo", "aos", "seus", "quem", "nas", "me", "esse", "eles", "você", "essa", "num", "nem", "suas", "meu", "às", "minha", "numa", "pelos", "elas", "qual", "nós", "lhe", "deles", "essas", "esses", "pelas", "este", "dele", "tu", "te", "vocês", "vos", "lhes", "meus", "minhas", "teu", "tua", "teus", "tuas", "nosso", "nossa", "nossos", "nossas", "dela", "delas", "esta", "estes", "estas", "aquele", "aquela", "aqueles", "aquelas", "isto", "aquilo", "estou", "está", "estamos", "estão", "estive", "esteve",...