## Overview

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * RobotsTxtException is an exception class that represents an error related to robots.txt processing during web crawling.
 * It extends CrawlerSystemException and provides constructors to create instances with a message and/or a cause.
 *
 */
public class RobotsTxtException extends CrawlerSystemException {

 * into a ResultData object, and for retrieving the binary data from an AccessResultData object.
 * It extends the AbstractTransformer class.
 *
 * <p>
 * This transformer extracts the binary content from the response body of a web resource,
 * stores it as a byte array in the ResultData, and provides a method to retrieve this data
 * as a ByteArrayInputStream.
 * </p>
 *
 * <p>

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval;

/**
 * The {@code IntervalController} interface defines methods for controlling
 * the delay intervals in a web crawler. It includes constants representing
 * different types of processing states and a method to introduce a delay
 * based on the type of processing.
 * <p>
 * Constants:
 * </p>
 * <ul>

import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.container.CrawlerContainer;

import jakarta.annotation.Resource;

/**
 * Creates and manages crawler clients for web crawling operations.
 * This class handles the registration and loading of crawler client factories and their associated clients.
 * <p>

     */
    public SitemapUrl() {
        super();
    }

    /**
     * URL of the page. This URL must begin with the protocol (such as http) and
     * end with a trailing slash, if your web server requires it. This value
     * must be less than 2,048 characters.
     */
    private String loc;

    /**
     * The date of last modification of the file. This date should be in W3C

import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * The Crawler class is the main class for web crawling. It manages the crawling process,
 * including adding URLs to the queue, filtering URLs, managing crawler threads,
 * and handling the overall crawling lifecycle.
 *

 *   <li>PROXY_CREDENTIALS_PROPERTY: Proxy credentials.</li>
 *   <li>USER_AGENT_PROPERTY: User agent string.</li>
 *   <li>ROBOTS_TXT_ENABLED_PROPERTY: Enable or disable robots.txt parsing.</li>
 *   <li>AUTHENTICATIONS_PROPERTY: Web authentications.</li>
 *   <li>REQUEST_HEADERS_PROPERTY: Custom request headers.</li>
 *   <li>REDIRECTS_ENABLED: Enable or disable HTTP redirects.</li>
 *   <li>COOKIES_PROPERTY: Cookie settings.</li>