this.sessionId = sessionId;
    }

    /**
     * Returns the filter type.
     * @return The filter type.
     */
    public String getFilterType() {
        return filterType;
    }

    /**
     * Sets the filter type.
     * @param filterType The filter type.
     */
    public void setFilterType(final String filterType) {
        this.filterType = filterType;

        this.dataIndex = dataIndex;
    }

    /**
     * Returns the filter index name.
     * @return The filter index name.
     */
    public String getFilterIndex() {
        return filterIndex;
    }

    /**
     * Sets the filter index name.
     * @param filterIndex The filter index name.
     */
    public void setFilterIndex(final String filterIndex) {

    /**
     * Field name for filter type.
     */
    private static final String FILTER_TYPE = "filterType";

    /**
     * Filter type for include filters.
     */
    private static final String INCLUDE = "include";

    /**
     * Filter type for exclude filters.
     */
    private static final String EXCLUDE = "exclude";

    /**

 * <ol>
 *   <li>Initialization: Sets up the crawler context and initializes the URL filter.</li>
 *   <li>Thread Creation: Creates a group of crawler threads to perform the actual crawling.</li>
 *   <li>Execution: Starts the crawler threads and waits for them to complete.</li>
 *   <li>Cleanup: Deletes the crawled data and clears the URL filter.</li>
 * </ol>
 *

 * It also provides methods to initialize the filter with a session ID, clear the filter,
 * match a URL against the defined patterns, and process a URL to add include or exclude patterns based on predefined filtering patterns.
 *
 */
/**
 * This class is an implementation of a URL filter.
 */
public class UrlFilterImpl implements UrlFilter {

    /**

    public void setStatus(final CrawlerStatus status) {
        this.status = status;
    }

    /**
     * Returns the URL filter.
     * @return The UrlFilter.
     */
    public UrlFilter getUrlFilter() {
        return urlFilter;
    }

    /**
     * Sets the URL filter.
     * @param urlFilter The UrlFilter.
     */
    public void setUrlFilter(final UrlFilter urlFilter) {

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.filter;

import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.impl.DataServiceImpl;

    client.setConnectionTimeout(30000);
    client.setMaxConnections(100);
});

// Configure URL filtering
container.singleton("urlFilter", UrlFilterImpl.class, filter -> {
    filter.addInclude("https://example.com/.*");
    filter.addExclude(".*\\.(css|js|png|jpg|gif)$");
});

// Configure content extraction
container.singleton("tikaExtractor", TikaExtractor.class);

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.filter.UrlFilter;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.helper.RobotsTxtHelper;
import org.codelibs.fess.crawler.helper.impl.MimeTypeHelperImpl;

        SearchResponse response = getClient().get(c -> c.prepareSearch(index)
                .setScroll(new TimeValue(scrollTimeout))
                .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.termQuery(SESSION_ID, oldSessionId)))
                .setSize(scrollSize)
                .execute());
        String scrollId = response.getScrollId();
        try {
            while (scrollId != null) {