/**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.
     */

 *
 */
public class HtmlXpathExtractor extends AbstractXmlExtractor {
    /**
     * Regular expression pattern to match the charset attribute in the meta tag of HTML documents.
     * The pattern captures the charset value specified in the content attribute of the meta tag.
     * Example: <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     */

    }

    /**
     * Gets the extractor for the specified file extension.
     *
     * @param ext the file extension
     * @return the extractor for the extension, or null if not found
     */
    private Extractor getExtractor(final String ext) {
        return extractorMap.get(ext);
    }

    /**
     * Gets the output extension for the specified input extension.
     *

        responseBodyFile = responseBody;
        isTemporaryFile = isTemporary;
    }

    /**
     * Gets the character set of the response.
     *
     * @return the character set, or null if not specified
     */
    public String getCharSet() {
        return charSet;
    }

    /**
     * Sets the character set of the response.
     *
     * @param charSet the character set to set
     */

        }
    }

    /**
     * Deletes all URL queue entries for the specified session.
     *
     * @param sessionId The session ID.
     */
    @Override
    public void delete(final String sessionId) {
        deleteBySessionId(sessionId);
    }

    /**
     * Offers multiple URL queue entries for the specified session.
     * Only URLs that don't already exist will be added.
     *

    /**
     * Extracts a list of TermQuery objects from the given Query object that match the specified fields.
     *
     * @param query the Query object to extract TermQuery objects from
     * @param fields an array of field names to match against the TermQuery objects
     * @return a list of TermQuery objects that match the specified fields, or an empty list if no matches are found
     */

 *   <li>Extracting child URLs from the HTML content based on configured rules.</li>
 *   <li>Handling redirect URLs specified in the response headers.</li>
 * </ol>
 * <p>
 * The class also provides methods for configuring features and properties of the
 * underlying DOM parser, as well as defining rules for extracting child URLs
 * from specific HTML tags and attributes.
 * </p>
 *
 * <p>
 * <b>Configuration:</b>
 * </p>
 * <ul>

 * and processing them according to specific rules. The main functionality is encapsulated
 * within the nested {@link TextNormalizeContext} class.
 *
 * <p>The text normalization process includes:
 * <ul>
 *   <li>Treating ISO control characters and specified space characters as spaces.</li>
 *   <li>Appending alphanumeric characters (0-9, A-Z, a-z) to the buffer.</li>

            }
        }
        throw new CrawlingAccessException("Invalid path: " + path);
    }

    /**
     * Retrieves response data for the specified URI.
     * @param uri the URI to retrieve data for
     * @param includeContent whether to include the actual content in the response
     * @return the response data containing metadata and optionally content

        private String objectName;
        /** Cached object statistics response */
        private StatObjectResponse statObject;

        /**
         * Constructs a new StorageURLConnection for the specified URL.
         * This constructor parses the URL to extract bucket and object names.
         *
         * @param url The storage URL to connect to
         */
        protected StorageURLConnection(final URL url) {