import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;
import org.codelibs.fess.crawler.service.UrlQueueService;
import org.codelibs.fess.crawler.transformer.impl.FileTransformer;
import org.codelibs.fess.crawler.util.CrawlerWebServer;
import org.codelibs.opensearch.runner.OpenSearchRunner;

		<dependency>
			<groupId>org.codelibs.fess</groupId>
			<artifactId>fess-crawler</artifactId>
			<version>${project.version}</version>
		</dependency>
		<dependency>
			<groupId>org.codelibs.fess</groupId>
			<artifactId>fess-crawler-lasta</artifactId>
			<version>${project.version}</version>
			<optional>true</optional>
		</dependency>
		<dependency>
			<groupId>org.codelibs</groupId>
			<artifactId>corelib</artifactId>

    protected float weight = 1.0f;

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.fess.crawler.entity.UrlQueue#getId()
     */
    @Override
    public IDTYPE getId() {
        return id;
    }

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.fess.crawler.entity.UrlQueue#setId(IDTYPE)
     */
    @Override
    public void setId(final IDTYPE id) {

	<property name="target.dir" value="${basedir}/target/modules" />

	<!-- Maven Repository -->
	<property name="maven.snapshot.repo.url" value="https://maven.codelibs.org/" />
	<property name="maven.release.repo.url" value="https://maven.codelibs.org/" />
	<property name="opensearch.version" value="2.17.1" />

	<target name="install.modules">
		<mkdir dir="${target.dir}" />
		<delete dir="${modules.dir}" />

 */
package org.codelibs.fess.crawler.service.impl;

import java.util.List;
import java.util.Map;
import java.util.function.Consumer;

import org.apache.lucene.search.TotalHits;
import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResultData;
import org.codelibs.fess.crawler.exception.OpenSearchAccessException;

		<mkdir dir="${plugins.dir}" />
		<!-- analysis-extension -->
		<antcall target="install.plugin">
			<param name="repo.url" value="${maven.release.repo.url}" />
			<param name="plugin.groupId" value="org/codelibs/opensearch" />
			<param name="plugin.name.prefix" value="opensearch-" />
			<param name="plugin.name" value="analysis-extension" />
			<param name="plugin.version" value="2.17.1" />

	<!-- Crawler -->
	<component name="crawler" class="org.codelibs.fess.crawler.Crawler" instance="prototype" >
	</component>

	<!-- Crawler Thread -->
	<component name="crawlerThread" class="org.codelibs.fess.crawler.CrawlerThread" instance="prototype" >
	</component>

	<!-- Entity -->
	<component name="accessResult"
		class="org.codelibs.fess.crawler.entity.OpenSearchAccessResult" instance="prototype">
	</component>

 */
package org.codelibs.fess.crawler.service.impl;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.codelibs.fess.crawler.entity.OpenSearchUrlFilter;

	<groupId>org.codelibs.fess</groupId>
	<artifactId>fess-crawler-parent</artifactId>
	<version>15.0.0-SNAPSHOT</version>
	<packaging>pom</packaging>
	<name>Fess Crawler Project</name>
	<description>Fess Crawler is Crawler Framework.</description>
	<url>https://github.com/codelibs/fess-crawler</url>
	<issueManagement>
		<system>GitHub</system>
		<url>https://github.com/codelibs/fess-crawler/issues</url>
	</issueManagement>

		</dependency>
		<dependency>
			<groupId>org.codelibs</groupId>
			<artifactId>corelib</artifactId>
			<version>${corelib.version}</version>
		</dependency>
		<dependency>
			<groupId>org.codelibs</groupId>
			<artifactId>sai</artifactId>
			<version>${sai.version}</version>
		</dependency>
		<dependency>
			<groupId>org.codelibs</groupId>
			<artifactId>xpp3</artifactId>