client.setConnectionTimeout(30000);
    client.setMaxConnections(100);
});

// Configure URL filtering
container.singleton("urlFilter", UrlFilterImpl.class, filter -> {
    filter.addInclude("https://example.com/.*");
    filter.addExclude(".*\\.(css|js|png|jpg|gif)$");
});

// Configure content extraction
container.singleton("tikaExtractor", TikaExtractor.class);

 * It also provides methods to initialize the filter with a session ID, clear the filter,
 * match a URL against the defined patterns, and process a URL to add include or exclude patterns based on predefined filtering patterns.
 *
 */
/**
 * This class is an implementation of a URL filter.
 */
public class UrlFilterImpl implements UrlFilter {

    /**

            subProjectToClassTimes
                .entries
                .filter { "UNKNOWN" != it.key }
                .filter { model.subprojects.getSubprojectByName(it.key) != null }
                .map {
                    SubprojectTestClassTime(
                        model.subprojects.getSubprojectByName(it.key)!!,
                        it.value.filter { tct ->
                            tct.sourceSet != "test"

 * <ol>
 *   <li>Initialization: Sets up the crawler context and initializes the URL filter.</li>
 *   <li>Thread Creation: Creates a group of crawler threads to perform the actual crawling.</li>
 *   <li>Execution: Starts the crawler threads and waits for them to complete.</li>
 *   <li>Cleanup: Deletes the crawled data and clears the URL filter.</li>
 * </ol>
 *

/**
 * The QueryLog class represents a log entry containing a query string and an optional filter query string.
 */
public class QueryLog {
    private final String q;
    private final String fq;

    /**
     * Constructor for QueryLog.
     * @param queryString The query string.
     * @param filterQueryString The filter query string (can be null).
     */

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.filter;

/**
 * UrlFilter checks if a given url is a target one.
 *
 * @author shinsuke
 *
 */
public interface UrlFilter {

    /**
     * Initialize a url filter by sessionId.
     *
     * @param sessionId Session ID
     */
    void init(String sessionId);

    /**

	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
    <include path="crawler/container.xml"/>
    <include path="crawler/client.xml"/>
    <include path="crawler/rule.xml"/>
    <include path="crawler/filter.xml"/>
    <include path="crawler/interval.xml"/>
    <include path="crawler/extractor.xml"/>
    <include path="crawler/mimetype.xml"/>
    <include path="crawler/encoding.xml"/>
    <include path="crawler/urlconverter.xml"/>

    /**
     * Test urlFilter getter and setter
     */
    public void test_urlFilter() {
        assertNull(crawlerContext.getUrlFilter());

        TestUrlFilter filter = new TestUrlFilter();
        crawlerContext.setUrlFilter(filter);
        assertSame(filter, crawlerContext.getUrlFilter());

        crawlerContext.setUrlFilter(null);
        assertNull(crawlerContext.getUrlFilter());
    }

    /**

    }

    /**
     * Adds a tag to filter by.
     * @param tag The tag.
     */
    public void addTag(final String tag) {
        tags.add(tag);
    }

    /**
     * Adds a role to filter by.
     * @param role The role.
     */
    public void addRole(final String role) {
        roles.add(role);
    }

    /**
     * Adds a field to filter by.
     * @param field The field name.

				"application/vnd.sun.xml.impress",
				"application/vnd.sun.xml.impress.template",
				"application/vnd.sun.xml.math",
				"application/vnd.sun.xml.writer",
				"application/vnd.sun.xml.writer.global",
				"application/vnd.sun.xml.writer.template",
				"application/vnd.sun.wadl+xml",
				"application/vnd.sus-calendar",
				"application/vnd.svd",
				"application/vnd.swiftview-ics",