// Default constructor
    }

    /**
     * Extracts the filename from the parameters.
     * @param in The input stream (not used).
     * @param params The parameters, expected to contain ExtractData.RESOURCE_NAME_KEY.
     * @return An ExtractData object containing the filename as content.
     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {

     * "java.protocol.handler.pkgs" to include the package "org.codelibs.fess.net.protocol".
     * If the property is not already set, it will be initialized with this package.
     * If the property is set but does not contain this package, the package will be appended.
     */
    default void initialize() {
        final StringBuilder buf = new StringBuilder(100);
        final String value = System.getProperty("java.protocol.handler.pkgs");

import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

 * It contains various attributes related to the crawler's state, configuration, and runtime data.
 * This class provides methods to access and modify these attributes, allowing for control and monitoring
 * of the crawler's behavior.
 *
 * <p>
 * The context includes information such as the session ID, active thread count, access count, crawler status,

     *
     * @param document The document to parse, represented as a map of field names to values.
     * @param fields The fields to extract from the document.
     * @param tagFieldNames The names of the fields that contain tags.
     * @param roleFieldName The name of the field that contains role information.
     * @param langFieldName The name of the field that contains language information.