* <ol>
 *   <li>Initialization: Sets up the crawler context and initializes the URL filter.</li>
 *   <li>Thread Creation: Creates a group of crawler threads to perform the actual crawling.</li>
 *   <li>Execution: Starts the crawler threads and waits for them to complete.</li>
 *   <li>Cleanup: Deletes the crawled data and clears the URL filter.</li>
 * </ol>
 *

    }

    /**
     * Adds a tag to filter by.
     * @param tag The tag.
     */
    public void addTag(final String tag) {
        tags.add(tag);
    }

    /**
     * Adds a role to filter by.
     * @param role The role.
     */
    public void addRole(final String role) {
        roles.add(role);
    }

    /**
     * Adds a field to filter by.
     * @param field The field name.

```java
SuggestResponse response = suggester.suggest()
    .setQuery("search term")
    .setSize(10)                           // max results
    .setTags(new String[]{"category"})      // filter by tags
    .setRoles(new String[]{"user"})         // filter by roles  
    .setLanguages(new String[]{"en", "ja"}) // language preference
    .setFields(new String[]{"title"})       // search specific fields
    .execute()
    .getResponse();
```

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.filter;

import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.impl.DataServiceImpl;

    client.setConnectionTimeout(30000);
    client.setMaxConnections(100);
});

// Configure URL filtering
container.singleton("urlFilter", UrlFilterImpl.class, filter -> {
    filter.addInclude("https://example.com/.*");
    filter.addExclude(".*\\.(css|js|png|jpg|gif)$");
});

// Configure content extraction
container.singleton("tikaExtractor", TikaExtractor.class);

        if (crawlerContext.getMaxDepth() >= 0 && depth > crawlerContext.getMaxDepth()) {
            return;
        }

        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.urlFilter.match(d.getUrl()))
                .map(d -> {

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.filter.UrlFilter;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.helper.RobotsTxtHelper;
import org.codelibs.fess.crawler.helper.impl.MimeTypeHelperImpl;

                getAliasesResponse.getAliases()
                        .entrySet()
                        .forEach(x -> x.getValue()
                                .stream()
                                .filter(y -> updateAlias.equals(y.alias()))
                                .forEach(y -> updateIndices.add(x.getKey())));
            }
            if (updateIndices.size() != 1) {
                if (logger.isDebugEnabled()) {

    /**
     * Test urlFilter getter and setter
     */
    public void test_urlFilter() {
        assertNull(crawlerContext.getUrlFilter());

        TestUrlFilter filter = new TestUrlFilter();
        crawlerContext.setUrlFilter(filter);
        assertSame(filter, crawlerContext.getUrlFilter());

        crawlerContext.setUrlFilter(null);
        assertNull(crawlerContext.getUrlFilter());
    }

    /**

        SearchResponse response = getClient().get(c -> c.prepareSearch(index)
                .setScroll(new TimeValue(scrollTimeout))
                .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.termQuery(SESSION_ID, oldSessionId)))
                .setSize(scrollSize)
                .execute());
        String scrollId = response.getScrollId();
        try {
            while (scrollId != null) {