}

    /**
     * Checks if the given character is a valid URL character.
     *
     * Valid URL characters include:
     * - Lowercase letters (a-z)
     * - Uppercase letters (A-Z)
     * - Digits (0-9)
     * - Special characters: . - * _ : / + % = & ? # [ ] @ ~ ! $ ' ( ) , ;
     *
     * @param c the character to check
     * @return {@code true} if the character is a valid URL character, {@code false} otherwise

    private final boolean digit;

    private final boolean letter;

    public CharTypeFilterFactory(final IndexSettings indexSettings, final Environment environment, final String name,
            final Settings settings) {
        super(indexSettings, name, settings);

        alphabetic = settings.getAsBoolean("alphabetic", true);
        digit = settings.getAsBoolean("digit", true);

import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * Helper class for managing content length limits based on MIME types.
 * It allows setting default and MIME type-specific maximum content lengths.
 * The class provides methods to add, retrieve, and manage these limits.
 */
public class ContentLengthHelper {
    /**
     * Constructs a new ContentLengthHelper.
     */

				"application/vnd.hzn-3d-crossword",
				"application/vnd.ibm.afplinedata",
				"application/vnd.ibm.electronic-media",
				"application/vnd.ibm.minipay",
				"application/vnd.ibm.modcap",
				"application/vnd.ibm.rights-management",
				"application/vnd.ibm.secure-container",
				"application/vnd.iccprofile",
				"application/vnd.igloader",
				"application/vnd.immervision-ivp",
				"application/vnd.immervision-ivu",

			<artifactId>tika-parser-crypto-module</artifactId>
			<version>${tika.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.tika</groupId>
			<artifactId>tika-parser-digest-commons</artifactId>
			<version>${tika.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.tika</groupId>
			<artifactId>tika-parser-font-module</artifactId>

      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf

    public static final String FS_FILE_GROUPS = "fsFileGroups";

    /** Character encoding for files */
    protected String charset = Constants.UTF_8;

    /** Helper for managing content length limits */
    @Resource
    protected ContentLengthHelper contentLengthHelper;

    /** Flag to track initialization status */
    protected AtomicBoolean isInit = new AtomicBoolean(false);

    /**

 * configured HTTP status codes. It also handles the storage of child URLs found in the
 * response data, respecting the maximum depth and access count limits.
 * </p>
 *
 */
public class DefaultResponseProcessor implements ResponseProcessor {
    /** Logger instance for this class */
    private static final Logger logger = LogManager.getLogger(DefaultResponseProcessor.class);

        }

        refresh();
    }

    /**
     * Generates a unique ID for a document based on session ID and URL.
     * Uses hashing for long IDs to ensure they fit within OpenSearch limits.
     *
     * @param sessionId The session ID.
     * @param url The URL.
     * @return The generated unique ID.
     */
    private String getId(final String sessionId, final String url) {

  </mime-type>
  <mime-type type="application/vnd.ibm.modcap">
    <glob pattern="*.afp"/>
    <glob pattern="*.listafp"/>
    <glob pattern="*.list3820"/>
  </mime-type>
  <mime-type type="application/vnd.ibm.rights-management">
    <glob pattern="*.irm"/>
  </mime-type>
  <mime-type type="application/vnd.ibm.secure-container">
    <glob pattern="*.sc"/>
  </mime-type>
  <mime-type type="application/vnd.iccprofile">