import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

 * It contains various attributes related to the crawler's state, configuration, and runtime data.
 * This class provides methods to access and modify these attributes, allowing for control and monitoring
 * of the crawler's behavior.
 *
 * <p>
 * The context includes information such as the session ID, active thread count, access count, crawler status,

     *        unknown.)
     */
    public CrawlerSystemException(final Throwable cause) {
        super(cause);
    }

    /**
     * Constructs a new CrawlerSystemException with the specified detail message and controls suppression and stack trace writing.
     *
     * @param message the detail message
     * @param enableSuppression whether or not suppression is enabled

import org.codelibs.fess.crawler.util.CrawlingParameterUtil;

/**
 * HostIntervalController is an implementation of {@link org.codelibs.fess.crawler.interval.IntervalController}
 * that controls the interval between requests to the same host.
 * It uses a ConcurrentMap to store the last access time for each host.
 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,