crawler1.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler1.getCrawlerContext().setNumOfThread(numOfThread);

            Thread.sleep(100);

            final Crawler crawler2 = getComponent(Crawler.class);
            crawler2.setBackground(true);
            ((UrlFilterImpl) crawler2.urlFilter).setIncludeFilteringPattern("$1$2$3.*");
            crawler2.addUrl(url2);

<components namespace="fessCrawler">
    <include path="crawler/container.xml"/>
    <include path="crawler/client.xml"/>
    <include path="crawler/rule.xml"/>
    <include path="crawler/filter.xml"/>
    <include path="crawler/interval.xml"/>
    <include path="crawler/extractor.xml"/>
    <include path="crawler/mimetype.xml"/>
    <include path="crawler/encoding.xml"/>
    <include path="crawler/urlconverter.xml"/>
    <include path="crawler/log.xml"/>

    protected float weight = 1.0f;

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.fess.crawler.entity.UrlQueue#getId()
     */
    @Override
    public IDTYPE getId() {
        return id;
    }

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.fess.crawler.entity.UrlQueue#setId(IDTYPE)
     */
    @Override
    public void setId(final IDTYPE id) {
        this.id = id;

import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.ftp.FtpClient;
import org.codelibs.fess.crawler.client.http.HcHttpClient;
import org.codelibs.fess.crawler.client.smb.SmbClient;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;

<project name="deps" basedir=".">
	<property name="target.dir" value="${basedir}/target/deps" />
	<property name="webinf.dir" value="${basedir}/src/main/webapp/WEB-INF" />
	<property name="crawler.dir" value="${basedir}/src/main/webapp/WEB-INF/env/crawler" />
	<property name="suggest.dir" value="${basedir}/src/main/webapp/WEB-INF/env/suggest" />
	<property name="thumbnail.dir" value="${basedir}/src/main/webapp/WEB-INF/env/thumbnail" />

 */
package org.codelibs.fess.crawler.service.impl;

import java.util.List;
import java.util.Map;
import java.util.function.Consumer;

import org.apache.lucene.search.TotalHits;
import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResultData;
import org.codelibs.fess.crawler.exception.OpenSearchAccessException;

	<groupId>org.codelibs.fess</groupId>
	<artifactId>fess-crawler-parent</artifactId>
	<version>15.0.0-SNAPSHOT</version>
	<packaging>pom</packaging>
	<name>Fess Crawler Project</name>
	<description>Fess Crawler is Crawler Framework.</description>
	<url>https://github.com/codelibs/fess-crawler</url>
	<issueManagement>
		<system>GitHub</system>
		<url>https://github.com/codelibs/fess-crawler/issues</url>
	</issueManagement>
	<scm>

 */
package org.codelibs.fess.crawler.service.impl;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.codelibs.fess.crawler.entity.OpenSearchUrlFilter;

#smart.deploy.mode.location = maihama_env.properties: lasta_di.smart.deploy.mode

# package for smart deploy target e.g. org.docksidestage.app
smart.package1 = org.codelibs.fess.crawler

import org.codelibs.core.misc.Pair;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;