* It allows setting delays before processing, after processing, when no URLs are in the queue,
 * and when waiting for new URLs.
 * The delays are configurable via constructor parameters.
 *
 */
public class DefaultIntervalController extends AbstractIntervalController {

    /** Delay in milliseconds after processing a URL */
    protected long delayMillisAfterProcessing = 0L;

 *
 * <p>
 * This class defines the contract for delaying the crawling process at various points, such as:
 * </p>
 * <ul>
 *   <li>Before processing a URL ({@link #delayBeforeProcessing()})</li>
 *   <li>After processing a URL ({@link #delayAfterProcessing()})</li>
 *   <li>When there are no URLs in the queue ({@link #delayAtNoUrlInQueue()})</li>

    /** Indicates a redirect location was found. */
    REDIRECT_LOCATION,
    /** Indicates processing a response. */
    PROCESS_RESPONSE,
    /** Indicates the crawling process has finished. */
    FINISHED_CRAWLING,
    /** Indicates processing child URLs due to an exception. */
    PROCESS_CHILD_URLS_BY_EXCEPTION,
    /** Indicates processing a child URL due to an exception. */
    PROCESS_CHILD_URL_BY_EXCEPTION,

 * the delay intervals in a web crawler. It includes constants representing
 * different types of processing states and a method to introduce a delay
 * based on the type of processing.
 * <p>
 * Constants:
 * </p>
 * <ul>
 *   <li>{@code PRE_PROCESSING} - Represents the pre-processing state.</li>
 *   <li>{@code POST_PROCESSING} - Represents the post-processing state.</li>

/**
 * NullResponseProcessor is a class that implements the ResponseProcessor interface.
 * It provides a no-op implementation for processing response data.
 * This class can be used when you want to skip the response processing step in a crawler.
 *
 * @author kuma
 */
public class NullResponseProcessor implements ResponseProcessor {

    /**
     * Creates a new NullResponseProcessor instance.

     */
    public static final String ISO_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";

    /**
     * The feature for secure processing in XML.
     */
    public static final String FEATURE_SECURE_PROCESSING = "http://javax.xml.XMLConstants/feature/secure-processing";

    /**
     * The feature for external general entities in XML.
     */

 * that controls the interval between requests to the same host.
 * It uses a ConcurrentMap to store the last access time for each host.
 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.

        }
        return line;
    }

    /**
     * Checks if robots.txt processing is enabled.
     * @return true if enabled, false otherwise
     */
    public boolean isEnabled() {
        return enabled;
    }

    /**
     * Sets whether robots.txt processing is enabled.
     * @param enabled true to enable, false to disable
     */

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * RobotsTxtException is an exception class that represents an error related to robots.txt processing during web crawling.
 * It extends CrawlerSystemException and provides constructors to create instances with a message and/or a cause.
 *
 */
public class RobotsTxtException extends CrawlerSystemException {

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * Exception thrown during sitemaps processing in the crawler.
 * This exception extends {@link org.codelibs.fess.crawler.exception.CrawlerSystemException}
 * and provides constructors for wrapping other exceptions or creating a new exception with a message.
 */