}

    /**
     * Checks if the given character is a valid URL character.
     *
     * Valid URL characters include:
     * - Lowercase letters (a-z)
     * - Uppercase letters (A-Z)
     * - Digits (0-9)
     * - Special characters: . - * _ : / + % = & ? # [ ] @ ~ ! $ ' ( ) , ;
     *
     * @param c the character to check
     * @return {@code true} if the character is a valid URL character, {@code false} otherwise

    private final boolean digit;

    private final boolean letter;

    public CharTypeFilterFactory(final IndexSettings indexSettings, final Environment environment, final String name,
            final Settings settings) {
        super(indexSettings, name, settings);

        alphabetic = settings.getAsBoolean("alphabetic", true);
        digit = settings.getAsBoolean("digit", true);

import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * Helper class for managing content length limits based on MIME types.
 * It allows setting default and MIME type-specific maximum content lengths.
 * The class provides methods to add, retrieve, and manage these limits.
 */
public class ContentLengthHelper {
    /**
     * Constructs a new ContentLengthHelper.
     */

    public static final String FS_FILE_GROUPS = "fsFileGroups";

    /** Character encoding for files */
    protected String charset = Constants.UTF_8;

    /** Helper for managing content length limits */
    @Resource
    protected ContentLengthHelper contentLengthHelper;

    /** Flag to track initialization status */
    protected AtomicBoolean isInit = new AtomicBoolean(false);

    /**

 * configured HTTP status codes. It also handles the storage of child URLs found in the
 * response data, respecting the maximum depth and access count limits.
 * </p>
 *
 */
public class DefaultResponseProcessor implements ResponseProcessor {
    /** Logger instance for this class */
    private static final Logger logger = LogManager.getLogger(DefaultResponseProcessor.class);

    /**
     * The character encoding to use for content. Defaults to UTF-8.
     */
    protected String charset = Constants.UTF_8;

    /**
     * Helper for managing content length validation and limits.
     */
    @Resource
    protected ContentLengthHelper contentLengthHelper;

    /**
     * Flag indicating whether the client has been initialized.
     */
    protected volatile boolean isInit = false;

    public FtpClient() {
        // Default constructor
    }

    /** Character encoding for FTP operations */
    protected String charset = Constants.UTF_8;

    /** Helper for managing content length limits */
    @Resource
    protected ContentLengthHelper contentLengthHelper;

    /**
     * The FTP authentication holder.
     */
    protected volatile FtpAuthenticationHolder ftpAuthenticationHolder;

        }

        refresh();
    }

    /**
     * Generates a unique ID for a document based on session ID and URL.
     * Uses hashing for long IDs to ensure they fit within OpenSearch limits.
     *
     * @param sessionId The session ID.
     * @param url The URL.
     * @return The generated unique ID.
     */
    private String getId(final String sessionId, final String url) {

    /** Helper for processing robots.txt files */
    @Resource
    protected RobotsTxtHelper robotsTxtHelper;

    /** Helper for managing content length limits */
    @Resource
    protected ContentLengthHelper contentLengthHelper;

    /** Helper for determining MIME types */
    @Resource
    protected MimeTypeHelper mimeTypeHelper;

  </mime-type>
  <mime-type type="application/vnd.ibm.modcap">
    <glob pattern="*.afp"/>
    <glob pattern="*.listafp"/>
    <glob pattern="*.list3820"/>
  </mime-type>
  <mime-type type="application/vnd.ibm.rights-management">
    <glob pattern="*.irm"/>
  </mime-type>
  <mime-type type="application/vnd.ibm.secure-container">
    <glob pattern="*.sc"/>
  </mime-type>
  <mime-type type="application/vnd.iccprofile">