other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.

      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.

     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.
     */
    public int getCrawlDelay(final String userAgent) {
        final Directive directive = getMatchedDirective(userAgent);
        if (directive == null) {
            return 0;
        }
        return directive.getCrawlDelay();
    }

    /**

                    if (value != null) {
                        if (!currentDirectiveList.isEmpty() && value.length() > 0) {
                            for (final Directive directive : currentDirectiveList) {
                                directive.addDisallow(value);
                            }
                        }
                    } else if ((value = getValue(ALLOW_RECORD, line)) != null) {

 *   <li>Normalize the filename to handle special characters.</li>
 *   <li>Configure whether to use the filename for MIME type detection.</li>
 *   <li>Configure whether to use the filename for MIME type detection when the stream is octet-stream.</li>
 * </ul>
 *
 * <p>
 * The MIME type detection is based on the {@code tika-mimetypes.xml} resource, which is loaded during initialization.
 * </p>
 *
 * <p>
 * Usage:
 * </p>

    }

    /**
     * Returns the preload size for charset detection.
     * @return The preload size for charset detection.
     */
    public int getPreloadSizeForCharset() {
        return preloadSizeForCharset;
    }

    /**
     * Sets the preload size for charset detection.
     * @param preloadSizeForCharset The preload size for charset detection to set.
     */

package org.codelibs.fess.crawler.helper;

import java.io.InputStream;
import java.util.Map;

/**
 * MimeTypeHelper provides methods to determine the content type of a given input stream or file.
 * It allows content type detection based on the stream's content and/or filename.
 */
public interface MimeTypeHelper {
    /**
     * Determines the content type of the given input stream and filename.
     * @param is the input stream to analyze

    /**
     * Checks if automatic UTF-8 encoding detection is enabled.
     *
     * @return True if auto-detection is enabled, false otherwise
     */
    public boolean isAutodetectEncoding() {
        return autodetectEncoding;
    }

    /**
     * Sets whether automatic UTF-8 encoding detection is enabled.
     *
     * @param autodetectEncoding True to enable auto-detection, false to disable
     */

                        }

                        final RobotsTxt.Directive directive = robotsTxt.getMatchedDirective(userAgent);
                        if (directive != null) {
                            if (useRobotsTxtDisallows) {
                                for (String urlPattern : directive.getDisallows()) {
                                    if (StringUtil.isNotBlank(urlPattern)) {

 * It also integrates with other Fess Crawler components, such as {@link ContentLengthHelper} and
 * {@link MimeTypeHelper}, to handle content length checks and MIME type detection.
 * </p>
 *
 * <p>
 * The class uses JCIFS properties to configure the SMB connection.
 * </p>
 *
 * <p>
 * Usage example:
 * </p>
 *
 * <pre>
 * {@code