*/
package org.codelibs.fess.crawler.rule.impl;

import java.util.ArrayList;
import java.util.List;

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.rule.Rule;
import org.codelibs.fess.crawler.rule.RuleManager;

/**
 * Implementation of the {@link RuleManager} interface.
 * Manages a list of rules for the crawler.

import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.extractor.Extractor;
import org.codelibs.fess.crawler.extractor.ExtractorFactory;

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />
	<include path="crawler/transformer.xml" />

	<component name="ruleManager" class="org.codelibs.fess.crawler.rule.impl.RuleManagerImpl" instance="prototype">
		<postConstruct name="addRule">
			<arg>sitemapsRule</arg>
		</postConstruct>
		<postConstruct name="addRule">

{"index":{"_index":"fess_config.scheduled_job","_id":"default_crawler"}}
{"name":"Default Crawler","target":"all","cronExpression":"0 0 * * *","scriptType":"groovy","scriptData":"return container.getComponent(\"crawlJob\").logLevel(\"info\").gcLogging().execute(executor);","jobLogging":true,"crawler":true,"available":true,"sortOrder":1,"createdBy":"system","createdTime":0,"updatedBy":"system","updatedTime":0}
{"index":{"_index":"fess_config.scheduled_job","_id":"suggest_indexer"}}

 */
package org.codelibs.fess.crawler.entity;

import java.io.Serializable;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.function.Function;

import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * This class represents the result data of a crawl.
 */
public class ResultData implements Serializable {

            <td>Fri Jan 13 13:53:34 UTC 2017</td>
            <td align="right">
                              &nbsp;
                          </td>
            <td></td>
          </tr>
                  <tr>
            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler-db-mysql/">fess-crawler-db-mysql/</a></td>

import org.codelibs.core.lang.SystemUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.entity.UrlQueue;

import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.AbstractCrawlerClient;
import org.codelibs.fess.crawler.client.AccessTimeoutTarget;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;

import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.AbstractCrawlerClient;
import org.codelibs.fess.crawler.client.AccessTimeoutTarget;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;

 */
package org.codelibs.fess.crawler;

import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;

import org.codelibs.core.collection.LruHashSet;
import org.codelibs.fess.crawler.filter.UrlFilter;
import org.codelibs.fess.crawler.interval.IntervalController;
import org.codelibs.fess.crawler.rule.RuleManager;

/**