}
    }

    /**
     * Applies delay based on the configured interval rules.
     * This method calculates the appropriate delay for the current time
     * and applies it by sleeping the current thread.
     */
    public void delayByRules() {
        final long delay = getDelay();
        if (delay > 0) {
            ThreadUtil.sleep(delay);
        }
    }

    /**

labels.advance_search_filetype_powerpoint=MS PowerPoint
labels.advance_search_occt=Cerca in
labels.advance_search_occt_default=Intera pagina
labels.advance_search_occt_allintitle=Titolo della pagina
labels.advance_search_occt_allinurl=URL della pagina
labels.advance_search_sitesearch=Sito o dominio
labels.advance_search_timestamp=Data di aggiornamento
labels.advance_search_timestamp_default=Qualsiasi data

 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.
 * If the time since the last request to the host is less than the configured delay,
 * the thread waits until the delay has elapsed.
 * This class is thread-safe.

    }

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.
     */

        } finally {
            CloseableUtil.closeQuietly(in);
        }

        // Should not throw exception for invalid crawl-delay
        assertNotNull(robotsTxt);
        // Invalid crawl-delay should be ignored (default 0)
        assertEquals(0, robotsTxt.getCrawlDelay("TestBot"));
        // Other directives should still work
        assertFalse(robotsTxt.allows("/test/", "TestBot"));
    }

        // First access - should not delay
        final long start1 = System.nanoTime();
        controller.delayBeforeProcessing();
        final long elapsed1 = (System.nanoTime() - start1) / 1000000;
        assertTrue("First access should not delay", elapsed1 < 50);

        // Second access to same host - should delay
        q.setUrl("http://example.com/page2");

labels.crawling_info_CrawlerStartTime=Hora de inicio del rastreador
labels.crawling_info_CrawlerEndTime=Hora de finalización del rastreador
labels.crawling_info_CrawlerExecTime=Tiempo de ejecución del rastreador
labels.crawling_info_CrawlerStatus=Estado del rastreador
labels.crawling_info_WebFsCrawlExecTime=Tiempo de ejecución del rastreo (Web/Archivo)
labels.crawling_info_WebFsCrawlStartTime=Hora de inicio del rastreo (Web/Archivo)

```

### CrawlerThread (`CrawlerThread.java`)

Worker thread for crawling.

**Flow**: Poll URL → Validate → Get client → Delay → Check last-modified → Execute → Process → Extract children → Queue children → Delay

### CrawlerClientFactory

Pattern-based client selection using `LinkedHashMap<Pattern, CrawlerClient>`.

**Standard Patterns**:
```java

        assertFalse(CharUtil.isUrlChar('\u0000')); // null
        assertFalse(CharUtil.isUrlChar('\u0001')); // SOH
        assertFalse(CharUtil.isUrlChar('\u001F')); // US
        assertFalse(CharUtil.isUrlChar('\u007F')); // DEL
    }

    public void test_isUrlChar_extendedAscii() {
        // Test extended ASCII characters (not valid URL chars without encoding)
        assertFalse(CharUtil.isUrlChar('\u00A0')); // non-breaking space

        crawlerContext.setStatus(CrawlerStatus.RUNNING);
        crawlerContext.maxThreadCheckCount = 1; // Will exit after 1 check

        crawlerThread.run();

        verify(intervalController, times(1)).delay(IntervalController.NO_URL_IN_QUEUE);
    }

    /**
     * Test isContinue with active threads still running.
     */
    public void test_isContinue_withActiveThreads() throws Exception {