import org.codelibs.core.stream.StreamUtil;
import org.codelibs.fess.crawler.Constants;

/**
 * The FormScheme class implements the AuthScheme interface to provide
 * form-based authentication for HTTP clients. It handles the process of
 * obtaining a token and logging in using the provided credentials.
 *
 * <p>This class supports both GET and POST methods for token and login
 * requests. It also allows for the replacement of placeholders in URLs and

package org.codelibs.fess.crawler.entity;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;

/**
 * Represents a set of Sitemaps, which can be either a UrlSet or an Index.
 * This class provides methods to manage a list of Sitemap objects and determine the type of the SitemapSet.
 *
 */
public class SitemapSet implements Serializable {

    private static final long serialVersionUID = 1L;

 */
package org.codelibs.fess.crawler.helper;

import java.util.HashMap;
import java.util.Locale;
import java.util.Map;

import org.codelibs.core.lang.StringUtil;

/**
 * EncodingHelper provides utility methods for managing and normalizing character encodings.
 * It allows setting a default encoding, mapping various encoding names to preferred ones,

import java.io.Serializable;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents an HTTP request header with a name and value.
 * This class is used to encapsulate the header information for HTTP requests.
 * It provides methods to get and set the name and value of the header,
 * as well as a method to validate the header.
 *
 * <p>Example usage:</p>
 * <pre>

import java.util.LinkedHashMap;
import java.util.Map;

import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * Helper class for converting URLs based on a set of predefined rules.
 *
 * <p>This class provides functionality to convert URLs by replacing parts of the URL
 * based on a map of target strings and their corresponding replacements. It allows
 * adding new conversion rules, setting the entire conversion map, and converting

/**
 * The {@link CrawlerContext} class holds the context information for a crawler execution.
 * It contains various attributes related to the crawler's state, configuration, and runtime data.
 * This class provides methods to access and modify these attributes, allowing for control and monitoring
 * of the crawler's behavior.
 *
 * <p>

 * It uses XPath expressions to extract data from the XML and stores it in a ResultData object.
 * </p>
 *
 * <p>
 * This class provides several configuration options to customize the XML parsing process, such as:
 * </p>
 * <ul>
 *   <li>Namespace awareness</li>
 *   <li>Coalescing</li>
 *   <li>Entity expansion</li>

 * It extends the AbstractTransformer class.
 *
 * <p>
 * This transformer extracts the binary content from the response body of a web resource,
 * stores it as a byte array in the ResultData, and provides a method to retrieve this data
 * as a ByteArrayInputStream.
 * </p>
 *
 * <p>
 * The transform method takes a ResponseData object, checks if it has a response body,

 */
package org.codelibs.fess.crawler;

import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;

/**
 * Constants used in the fess-crawler.
 * This class provides a collection of constant values for HTTP methods, status codes,
 * transformer names, boolean values, character encodings, date/time formats, and XML features.
 * It is designed to avoid the instantiation.
 */

import org.xml.sax.helpers.DefaultHandler;

/**
 * Helper class for parsing and validating sitemaps.
 * It supports XML sitemaps, XML sitemap indexes, and text sitemaps,
 * and can handle GZIP compressed sitemaps.
 * The class provides methods to check if an input stream is a valid sitemap,
 * and to parse an input stream into a {@link SitemapSet} object.
 * It uses SAX parser for XML sitemaps and XML sitemap indexes,