crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns
crawler.urlFilter.addInclude("https://example.com/.*");
crawler.urlFilter.addInclude(".*\\.pdf$");

// Exclude patterns  
crawler.urlFilter.addExclude(".*\\.js$");
crawler.urlFilter.addExclude(".*login.*");
```

## Supported Protocols and Formats

### Protocols

    protected void setUp() throws Exception {
        super.setUp();
        crawlerContext = new CrawlerContext();
    }

    /**
     * Test implementation of UrlFilter for testing
     */
    private static class TestUrlFilter implements UrlFilter {
        @Override
        public void init(String sessionId) {
        }

        @Override
        public void addInclude(String urlPattern) {
        }

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="urlFilter"
		class="org.codelibs.fess.crawler.filter.impl.UrlFilterImpl" instance="prototype">
	</component>

        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.urlFilter.match(d.getUrl()))
                .map(d -> {
                    final UrlQueue<?> uq = crawlerContainer.getComponent("urlQueue");
                    uq.setCreateTime(SystemUtil.currentTimeMillis());

        try {
            // clear url filter
            urlFilterService.delete(sid);
        } catch (final Exception e) {
            logger.warn("Failed to delete UrlFilter for {}", sid, e);
        }

        try {
            // clear queue
            urlQueueService.clearCache();
            urlQueueService.delete(sid);
        } catch (final Exception e) {

            public static final String KEEP_ORIGINAL_BODY = "keep.original.body";
            public static final String CLEANUP_ALL = "cleanup.all";
            public static final String CLEANUP_URL_FILTERS = "cleanup.urlFilters";
            public static final String JCIFS_PREFIX = "jcifs.";
            public static final String HTML_CANONICAL_XPATH = "html.canonical.xpath";