crawler.crawlerContext.setMaxThreadCheckCount(3);
        crawler.crawlerContext.setMaxAccessCount(maxCount);
        crawler.crawlerContext.setNumOfThread(numOfThread);
        crawler.urlFilter.addInclude(url + ".*");
        crawler.urlFilter.addExclude(url + "/dir1/.*");
        final String sessionId = crawler.execute();
        assertEquals(maxCount, dataService.getCount(sessionId));
        dataService.delete(sessionId);
    }

        crawler.crawlerContext.setMaxThreadCheckCount(3);
        crawler.crawlerContext.setMaxAccessCount(maxCount);
        crawler.crawlerContext.setNumOfThread(numOfThread);
        crawler.urlFilter.addInclude(url + ".*");
        crawler.urlFilter.addExclude(url + "/dir1/.*");
        final String sessionId = crawler.execute();
        assertEquals(maxCount, dataService.getCount(sessionId));
        dataService.delete(sessionId);
    }

        String sessionId = "test-session-006";
        urlFilter.init(sessionId);

        urlFilter.addExclude(".*\\.(css|js)$");
        urlFilter.addExclude(".*\\/admin\\/.*");
        urlFilter.addExclude(".*#.*");

        assertTrue(urlFilter.match("https://example.com/page.html"));
        assertFalse(urlFilter.match("https://example.com/style.css"));
        assertFalse(urlFilter.match("https://example.com/script.js"));

    /**
     * Returns the URL filter.
     * @return The UrlFilter instance.
     */
    public UrlFilter getUrlFilter() {
        return urlFilter;
    }

    /**
     * Sets the URL filter.
     * @param urlFilter The UrlFilter instance to set.
     */
    public void setUrlFilter(final UrlFilter urlFilter) {
        this.urlFilter = urlFilter;
    }

    /**
     * Returns the rule manager.

                .singleton("urlFilter", UrlFilterImpl.class)//
                .singleton("robotsTxtHelper", RobotsTxtHelper.class)//
                .singleton("httpClient", HcHttpClient.class);
        httpClient = container.getComponent("httpClient");
        urlFilter = container.getComponent("urlFilter");
    }

    public void test_doGet() {

crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns
crawler.urlFilter.addInclude("https://example.com/.*");
crawler.urlFilter.addInclude(".*\\.pdf$");

// Exclude patterns  
crawler.urlFilter.addExclude(".*\\.js$");
crawler.urlFilter.addExclude(".*login.*");
```

## Supported Protocols and Formats

### Protocols

    protected void setUp() throws Exception {
        super.setUp();
        crawlerContext = new CrawlerContext();
    }

    /**
     * Test implementation of UrlFilter for testing
     */
    private static class TestUrlFilter implements UrlFilter {
        @Override
        public void init(String sessionId) {
        }

        @Override
        public void addInclude(String urlPattern) {
        }

        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.urlFilter.match(d.getUrl()))
                .map(d -> {
                    final UrlQueue<?> uq = crawlerContainer.getComponent("urlQueue");
                    uq.setCreateTime(SystemUtil.currentTimeMillis());

        try {
            // clear url filter
            urlFilterService.delete(sid);
        } catch (final Exception e) {
            logger.warn("Failed to delete UrlFilter for {}", sid, e);
        }

        try {
            // clear queue
            urlQueueService.clearCache();
            urlQueueService.delete(sid);
        } catch (final Exception e) {