/** Map of user agent patterns to their corresponding directives. */
    protected final Map<Pattern, Directive> directiveMap = new LinkedHashMap<>();

    /** List of sitemap URLs found in the robots.txt file. */
    private final List<String> sitemapList = new ArrayList<>();

    /**
     * Creates a new RobotsTxt instance.
     */
    public RobotsTxt() {
        // Default constructor
    }

    }

    /**
     * Use an XPath string to select a single node.
     *
     * @param contextNode The node to start searching from.
     * @param expression A valid XPath string.
     * @return The first node found that matches the XPath, or null.
     *
     * @throws XPathExpressionException if an XPath expression error occurs.
     */

 */
package org.codelibs.fess.crawler.exception;

import java.util.Set;

import org.codelibs.fess.crawler.entity.RequestData;

/**
 * {@link ChildUrlsException} is thrown when child URLs are found during crawling.
 * It extends {@link CrawlerSystemException} and holds a set of {@link RequestData}
 * representing the child URLs that caused the exception.
 *
 */

        if (cachedXPathAPI == null) {
            cachedXPathAPI = new XPathAPI();
            xpathAPI.set(cachedXPathAPI);
        }
        return cachedXPathAPI;
    }

    /**
     * Stores child URLs found in the HTML content.
     *
     * @param responseData the response data containing the HTML content
     * @param resultData the result data to store child URLs in

    }

    /**
     * Retrieves an SMB authentication configuration that matches the given path.
     * @param path The path to match.
     * @return The matching SmbAuthentication object, or null if no match is found.
     */
    public SmbAuthentication get(final String path) {
        if (path == null) {
            return null;
        }

        for (final Map.Entry<String, SmbAuthentication> entry : authMap.entrySet()) {

     * that tries each extractor in order until one succeeds.
     *
     * @param key the key to look up
     * @return the extractor for the key, or null if not found
     */
    public Extractor getExtractor(final String key) {
        final Extractor[] extractors = extractorMap.get(key);
        if (extractors == null || extractors.length == 0) {
            return null;

     * @param <T> The type of the parameter.
     * @param key The key of the parameter.
     * @param defaultValue The default value if the parameter is not found.
     * @param clazz The class type to convert the parameter to.
     * @return The parameter value, or the default value if not found.
     */
    protected <T> T getInitParameter(final String key, final T defaultValue, final Class<T> clazz) {
        if (initParamMap != null) {

    }

    /**
     * Gets the values associated with a given key from the metadata.
     *
     * @param key the metadata key
     * @return the values associated with the key, or null if not found
     */
    public String[] getValues(final String key) {
        return metadata.get(key);
    }

    /**
     * Gets the set of all metadata keys.
     *
     * @return the set of metadata keys

     * @param clazz The class of the object to retrieve.
     * @param sessionId The session ID of the document.
     * @param url The URL of the document.
     * @return The retrieved object, or null if not found.
     * @throws OpenSearchAccessException if the retrieval fails.
     */
    protected <T> T get(final Class<T> clazz, final String sessionId, final String url) {
        final String id = getId(sessionId, url);

    }

    /**
     * Gets the received date from a message by parsing the received headers.
     *
     * @param message the message to get the received date from
     * @return the received date or null if not found
     * @throws MessagingException if message access fails
     */
    protected static Date getReceivedDate(final Message message) throws MessagingException {
        final Date today = new Date();