private static final String FILTER_TYPE = "filterType";

    /**
     * Filter type for include filters.
     */
    private static final String INCLUDE = "include";

    /**
     * Filter type for exclude filters.
     */
    private static final String EXCLUDE = "exclude";

    /**
     * Cache for include filters.
     */
    protected LoadingCache<String, List<Pattern>> includeFilterCache;

    /**

 */
package org.codelibs.fess.crawler.service;

import java.util.List;
import java.util.regex.Pattern;

/**
 * Service interface for managing URL filters.
 * Provides methods to add and remove include/exclude URL filters,
 * as well as retrieve the patterns of these filters.
 */
public interface UrlFilterService {

    /**
     * Adds a URL to the include filter list for the specified session.
     *

import java.io.IOException;

import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

/**
 * OpenSearchUrlFilter is an entity for URL filters in OpenSearch.
 */
public class OpenSearchUrlFilter implements ToXContent {

    /**
     * Creates a new instance of OpenSearchUrlFilter.
     */
    public OpenSearchUrlFilter() {
        // NOP
    }

- **StorageClient**: Cloud storage integration

#### Content Processing Pipeline
- **Extractors**: Content extraction from various formats
- **Transformers**: Data transformation and enrichment
- **Filters**: URL filtering with regex patterns
- **Rules**: Content processing rules and validation

## Building and Testing

### Build Commands

```bash
# Build all modules
mvn clean install

        assertFalse(urlFilter.match("https://other.com/page"));

        // Clear the filter
        urlFilter.clear();

        // After clear, all URLs should match (no filters applied)
        assertTrue(urlFilter.match("https://example.com/page"));
        assertTrue(urlFilter.match("https://other.com/page"));
        assertTrue(urlFilter.match("https://any.com/image.jpg"));
    }

 *   <li>Extracting text from metadata if the main content extraction fails</li>
 *   <li>Reading content as plain text if all other methods fail</li>
 *   <li>Applying post-extraction filters</li>
 *   <li>Handling Tika exceptions, including zip bomb exceptions</li>
 * </ul>
 *
 * <p>
 * The class also supports configuration options such as:
 * </p>
 * <ul>
 *   <li>Output encoding</li>

 *   <li>Cleanup: Deletes the crawled data and clears the URL filter.</li>
 * </ol>
 *
 * <p>The crawler can be configured with various parameters, such as the number of threads,
 * the maximum depth of crawling, and URL filters.
 *
 * <p>Example usage:
 * <pre>
 *   Crawler crawler = new Crawler();
 *   crawler.addUrl("http://example.com/");
 *   crawler.execute();
 *   crawler.close();
 * </pre>
 */