*
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */
    protected static final Pattern USER_AGENT_RECORD =
            Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */

 * passed to a {@link ChildUrlsException} to be processed by the crawler.
 * </p>
 *
 * <p>
 * The class also handles potential {@link IOException}s that may occur during
 * the parsing of the response body.
 * </p>
 */
public class SitemapsResponseProcessor implements ResponseProcessor {
    /** The crawler container for component lookup. */
    @Resource
    protected CrawlerContainer crawlerContainer;

     *
     * @param accessResultData the data containing the XML content to be parsed
     * @return a map containing the parsed data from the XML content
     * @throws CrawlerSystemException if an error occurs while parsing the XML content
     */
    public static Map<String, Object> getDataMap(final AccessResultData<?> accessResultData) {
        // create input source

import org.codelibs.fess.suggest.entity.SuggestItem;
import org.codelibs.fess.suggest.index.contents.querylog.QueryLog;
import org.codelibs.fess.suggest.normalizer.Normalizer;

/**
 * Interface for parsing content and creating suggest items.
 */
public interface ContentsParser {
    /**
     * Parses the given search words and creates a SuggestItem.
     *
     * @param words the array of search words

            logger.warn("Failed to parse the content by {}", path, e);
            return StringUtil.EMPTY_STRINGS;
        }

    }

    /**
     * Creates and configures a DOM parser for parsing HTML content.
     *
     * @return a configured DOMParser instance
     * @throws CrawlerSystemException if the parser configuration is invalid
     */
    protected DOMParser getDomParser() {

 * <ul>
 *   <li>{@link #expired()}: Handles the expiration of connections by closing expired and idle connections.</li>
 * </ul>
 *
 * <p>Logging:</p>
 * <ul>
 *   <li>Logs a warning if the clientConnectionManager is null.</li>
 *   <li>Logs a warning if an exception occurs during connection monitoring.</li>
 * </ul>
 *
 */
public class HcConnectionMonitorTarget implements TimeoutTarget {

     */
    public static final String DEBUG = "DEBUG";

    /**
     * Log level constant for info messages.
     */
    public static final String INFO = "INFO";

    /**
     * Log level constant for warning messages.
     */
    public static final String WARN = "WARN";

    /**
     * Log level constant for error messages.
     */
    public static final String ERROR = "ERROR";

        int count = 0;
        int prev = -1;
        Map<String, Object> source;
        while ((source = reader.read()) != null) {
            int current = Integer.parseInt(source.get("field2").toString());
            assertTrue(prev < current);
            prev = current;
            count++;
        }
        assertEquals(num, count);