<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,
 * allowed/disallowed paths, crawl delays, and sitemap URLs.
 *
 * <p>The robots.txt protocol is implemented according to the standard specification,

/third_party/etcd*
/default.etcd

# Also ignore protoc installed by hack/install-protoc.sh
/third_party/protoc*

# User cluster configs
.kubeconfig

.tags*

# Version file for dockerized build
.dockerized-kube-version-defs

# Web UI
/www/master/node_modules/
/www/master/npm-debug.log
/www/master/shared/config/development.json

# Karma output
/www/test_out

    # - - - - - - - - - -/

    # /- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # o schemaHtmlFileName: (NotRequired - Default 'schema-[project-name].html')
    #  The file name (not contain path) of SchemaHtml.
    #  Basically you don't need this.
    #  (For example, when you use Application Behavior, you need this)
    #
    #; schemaHtmlFileName = xxx.html
    # - - - - - - - - - -/

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *

    }

    /**
     * Parses a robots.txt file from the given input stream using UTF-8 encoding.
     * @param stream the input stream to parse
     * @return the parsed RobotsTxt object, or null if disabled
     */
    public RobotsTxt parse(final InputStream stream) {
        return parse(stream, Constants.UTF_8);
    }

    /**
     * Parses a robots.txt file from the given input stream using the specified character encoding.

 */
public abstract class SystemUtil {

    /**
     * Do not instantiate.
     */
    protected SystemUtil() {
    }

    /**
     * <code>file.encoding</code> system property. Example: UTF-8
     */
    public static final String FILE_ENCODING = System.getProperty("file.encoding");

    /**
     * <code>line.separator</code> system property. For example, on Mac OS X: <code>"\n"</code>
     */

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *

    public static final String RESOURCE_NAME_KEY = "resourceName";

    /** URL key for metadata */
    public static final String URL = "url";

    /** File passwords key for metadata */
    public static final String FILE_PASSWORDS = "file.passwords";

    /** Map containing metadata key-value pairs */
    protected Map<String, String[]> metadata = new HashMap<>();

    /** The extracted content text */