*/
package org.codelibs.fess.crawler.rule.impl;

import java.io.InputStream;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.helper.SitemapsHelper;

/**

 */
package org.codelibs.fess.crawler.service.impl;

import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.AccessResultImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.DataService;
import org.dbflute.utflute.core.PlainTestCase;

/**

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="intervalController"
		class="org.codelibs.fess.crawler.interval.FessIntervalController"
		instance="prototype">
	</component>

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="urlFilter"
		class="org.codelibs.fess.crawler.filter.impl.UrlFilterImpl" instance="prototype">
	</component>

 */
package org.codelibs.fess.crawler.extractor.impl;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;

import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.dbflute.utflute.core.PlainTestCase;

        this.available = value;
    }

    public Boolean getCrawler() {
        checkSpecifiedProperty("crawler");
        return crawler;
    }

    public void setCrawler(Boolean value) {
        registerModifiedProperty("crawler");
        this.crawler = value;
    }

    public String getCreatedBy() {
        checkSpecifiedProperty("createdBy");

index.document.suggest.index=fess
# Index name for crawler documents.
index.document.crawler.index=fess_crawler
# Number of primary shards for crawler queue index.
index.document.crawler.queue.number_of_shards=10
# Number of primary shards for crawler data index.
index.document.crawler.data.number_of_shards=10
# Number of primary shards for crawler filter index.
index.document.crawler.filter.number_of_shards=10

    }

    /** Wait time in milliseconds when crawler is not running */
    protected long crawlerWaitMillis = 10000;

    /** List of interval rules for controlling crawler timing */
    protected List<IntervalRule> ruleList = new ArrayList<>();

    /**
     * Checks the crawler status and waits if the crawler is not running.
     * This method blocks until the crawler is running again.
     */

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.rule.impl;

import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.beans.util.CopyOptions;
import org.codelibs.fess.crawler.entity.ResponseData;

/**

        return resultBuf.toString();

    }

    /**
     * Gets the count of currently running crawler jobs.
     * This method queries the scheduled jobs to count active crawler processes.
     * Used to enforce maximum concurrent crawler limits.
     *
     * @return the number of currently running crawler jobs
     */
    protected int getRunningJobCount() {
        final AtomicInteger counter = new AtomicInteger(0);