*/
    Credentials getCredentials();

    /**
     * Retrieves the authentication scheme to be used for HTTP requests.
     *
     * @return the authentication scheme
     */
    /**
     * Retrieves the authentication scheme to be used for HTTP requests.
     *
     * @return the authentication scheme
     */
    AuthScheme getAuthScheme();

import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;

/**
 * Interface representing a client for a web crawler.
 * This client is responsible for executing requests and handling responses.
 * It extends {@link AutoCloseable} to allow for resource management.
 */
public interface CrawlerClient extends AutoCloseable {

    /**

 * that controls the interval between requests to the same host.
 * It uses a ConcurrentMap to store the last access time for each host.
 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.

/**
 * Represents a request data for crawling.
 * This class encapsulates the HTTP method, URL, and weight associated with a crawling request.
 */
public class RequestData {
    /**
     * HTTP methods supported for crawling requests.
     */
    public enum Method {
        /** HTTP GET method. */
        GET,
        /** HTTP POST method. */
        POST,
        /** HTTP HEAD method. */
        HEAD;
    }

import org.codelibs.fess.crawler.exception.MultipleCrawlingAccessException;

/**
 * A fault-tolerant wrapper for CrawlerClient that implements retry logic for failed requests.
 * This client will attempt to execute requests multiple times before giving up, with configurable
 * retry counts and intervals between attempts.
 *
 * <p>The client supports a RequestListener interface to monitor the request lifecycle and handle

import java.io.Serializable;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents an HTTP request header with a name and value.
 * This class is used to encapsulate the header information for HTTP requests.
 * It provides methods to get and set the name and value of the header,
 * as well as a method to validate the header.
 *
 * <p>Example usage:</p>
 * <pre>

 * form-based authentication for HTTP clients. It handles the process of
 * obtaining a token and logging in using the provided credentials.
 *
 * <p>This class supports both GET and POST methods for token and login
 * requests. It also allows for the replacement of placeholders in URLs and
 * parameters with actual credentials.
 *
 * <p>Usage example:
 * <pre>
 * {@code
 * Map<String, String> params = new HashMap<>();

import jakarta.annotation.Resource;

/**
 * HcHttpClient is an HTTP client implementation that extends AbstractCrawlerClient.
 * It provides various configurations and settings for making HTTP requests, including
 * connection timeouts, proxy settings, user agent, request headers, cookie management,
 * and SSL configurations. The client also supports robots.txt parsing and form-based
 * authentication schemes.
 *

        return directive.allows(path);
    }

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found

     */
    public void setConnTimeout(final long connTimeout) {
        this.connTimeout = connTimeout;
    }

    /**
     * Sets the search preference for search requests.
     *
     * @param searchPreference The search preference.
     */
    public void setSearchPreference(final String searchPreference) {
        this.searchPreference = searchPreference;
    }