private static final String FILTER_TYPE = "filterType";

    /**
     * Filter type for include filters.
     */
    private static final String INCLUDE = "include";

    /**
     * Filter type for exclude filters.
     */
    private static final String EXCLUDE = "exclude";

    /**
     * Cache for include filters.
     */
    protected LoadingCache<String, List<Pattern>> includeFilterCache;

    /**

 */
package org.codelibs.fess.crawler.service;

import java.util.List;
import java.util.regex.Pattern;

/**
 * Service interface for managing URL filters.
 * Provides methods to add and remove include/exclude URL filters,
 * as well as retrieve the patterns of these filters.
 */
public interface UrlFilterService {

    /**
     * Adds a URL to the include filter list for the specified session.
     *

import java.io.IOException;

import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

/**
 * OpenSearchUrlFilter is an entity for URL filters in OpenSearch.
 */
public class OpenSearchUrlFilter implements ToXContent {

    /**
     * Creates a new instance of OpenSearchUrlFilter.
     */
    public OpenSearchUrlFilter() {
        // NOP
    }

- **StorageClient**: Cloud storage integration

#### Content Processing Pipeline
- **Extractors**: Content extraction from various formats
- **Transformers**: Data transformation and enrichment
- **Filters**: URL filtering with regex patterns
- **Rules**: Content processing rules and validation

## Building and Testing

### Build Commands

```bash
# Build all modules
mvn clean install

        assertFalse(urlFilter.match("https://other.com/page"));

        // Clear the filter
        urlFilter.clear();

        // After clear, all URLs should match (no filters applied)
        assertTrue(urlFilter.match("https://example.com/page"));
        assertTrue(urlFilter.match("https://other.com/page"));
        assertTrue(urlFilter.match("https://any.com/image.jpg"));
    }

 *   <li>Extracting text from metadata if the main content extraction fails</li>
 *   <li>Reading content as plain text if all other methods fail</li>
 *   <li>Applying post-extraction filters</li>
 *   <li>Handling Tika exceptions, including zip bomb exceptions</li>
 * </ul>
 *
 * <p>
 * The class also supports configuration options such as:
 * </p>
 * <ul>
 *   <li>Output encoding</li>

 *   <li>Cleanup: Deletes the crawled data and clears the URL filter.</li>
 * </ol>
 *
 * <p>The crawler can be configured with various parameters, such as the number of threads,
 * the maximum depth of crawling, and URL filters.
 *
 * <p>Example usage:
 * <pre>
 *   Crawler crawler = new Crawler();
 *   crawler.addUrl("http://example.com/");
 *   crawler.execute();
 *   crawler.close();
 * </pre>
 */

this.each(function(c){n(this).wrapAll(b?a.call(this,c):a)})},unwrap:function(){return this.parent().each(function(){n.nodeName(this,"body")||n(this).replaceWith(this.childNodes)}).end()}}),n.expr.filters.hidden=function(a){return a.offsetWidth<=0&&a.offsetHeight<=0},n.expr.filters.visible=function(a){return!n.expr.filters.hidden(a)};var wc=/%20/g,xc=/\[\]$/,yc=/\r?\n/g,zc=/^(?:submit|button|image|reset|file)$/i,Ac=/^(?:input|select|textarea|keygen)/i;function Bc(a,b,c,d){var e;if(n.isArray(b))n.each(b,fun...

  </mime-type>

  <mime-type type="application/x-tika-msworks-spreadsheet">
    <glob pattern="*.xlr"/>
    <sub-class-of type="application/vnd.ms-excel"/>
    <!-- this has to be highter than the Excel match -->
    <magic priority="60">
      <match value="0xd0cf11e0a1b11ae1" type="string" offset="0:8">