import org.apache.logging.log4j.Logger;
import org.codelibs.core.lang.StringUtil;

/**
 * Utility class for text normalization and processing.
 *
 * This class provides methods to normalize text by reading characters from a provided Reader
 * and processing them according to specific rules. The main functionality is encapsulated
 * within the nested {@link TextNormalizeContext} class.
 *

import javax.xml.xpath.XPathFactory;
import javax.xml.xpath.XPathNodes;

import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.w3c.dom.Node;

/**
 * The XPathAPI class provides methods to evaluate XPath expressions and select nodes or node lists from an XML document.
 * It uses the javax.xml.xpath.XPath and javax.xml.xpath.XPathFactory classes to create and evaluate XPath expressions.
 *

/**
 * The {@code MemoryDataHelper} class provides a helper for managing crawler data in memory.
 * It stores URL queues, access results, and URL patterns for inclusion and exclusion.
 * This class is designed to be used in a single-threaded environment or with external synchronization.
 *
 * <p>It uses {@code Map} and {@code Queue} data structures to hold the data. The class provides

import org.codelibs.fess.crawler.exception.UnsupportedExtractException;

import jakarta.annotation.Resource;

/**
 * Factory class for managing and retrieving {@link Extractor} instances.
 * This class provides methods to add, retrieve, and manage extractors based on a key.
 * It also includes a builder for creating extractors.
 *
 * <p>
 * The factory maintains a map of keys to an array of {@link Extractor} objects.

import org.codelibs.fess.crawler.container.CrawlerContainer;

import jakarta.annotation.Resource;

/**
 * A factory for creating and managing pooled crawler objects.
 * This class extends {@link BasePooledObjectFactory} and provides
 * methods for creating, wrapping, and destroying crawler components
 * obtained from a {@link CrawlerContainer}.
 *
 * @param <T> the type of the pooled object
 */

 * This class holds information about a single Sitemap, including its location and last modification timestamp.
 * It implements the {@link Sitemap} interface.
 *
 * <p>
 * A Sitemap file provides search engines with a list of URLs available for crawling.
 * This class encapsulates the essential attributes of a Sitemap entry, allowing for efficient management
 * and processing of Sitemap data.
 * </p>
 *
 * <p>

 */
package org.codelibs.fess.crawler.client.http;

import org.apache.http.auth.AuthScheme;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;

/**
 * The Authentication interface provides methods to retrieve authentication details
 * required for HTTP client authentication.
 */
public interface Authentication {

    /**
     * Retrieves the authentication scope associated with this authentication.

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.processor.ResponseProcessor;

/**
 * NullResponseProcessor is a class that implements the ResponseProcessor interface.
 * It provides a no-op implementation for processing response data.
 * This class can be used when you want to skip the response processing step in a crawler.
 *
 * @author kuma
 */

/**
 * Implementation of the {@link UrlFilter} interface.
 * This class provides functionality to filter URLs based on include and exclude patterns.
 * It uses a {@link UrlFilterService} to manage the URL filtering rules.
 * The class supports caching of include and exclude patterns for scenarios where a session ID is not available.
 * It also provides methods to initialize the filter with a session ID, clear the filter,

 * using the Apache Tika library. It extends {@link PasswordBasedExtractor} to handle password-protected files.
 * </p>
 *
 * <p>
 * This class provides methods to extract text from an input stream, handling different scenarios such as:
 * </p>
 * <ul>
 *   <li>Normalizing text content</li>
 *   <li>Handling resource names and content types</li>