* governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.util;

import org.codelibs.fess.util.ComponentUtil;

/**
 * Fess-specific crawler configuration that extends OpenSearchCrawlerConfig.
 * This class provides configuration settings for the Fess crawler including
 * index names, shard counts, and replica counts for queue, data, and filter indices.
 */

	<modelVersion>4.0.0</modelVersion>
	<artifactId>fess-crawler-opensearch</artifactId>
	<packaging>jar</packaging>
	<name>Fess Crawler OpenSearch</name>
	<parent>
		<groupId>org.codelibs.fess</groupId>
		<artifactId>fess-crawler-parent</artifactId>
		<version>15.3.0-SNAPSHOT</version>
		<relativePath>../pom.xml</relativePath>
	</parent>
	<build>
		<plugins>

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="robotsTxtHelper" class="org.codelibs.fess.crawler.helper.RobotsTxtHelper"
		instance="prototype">
	</component>

 */
package org.codelibs.fess.crawler.client.fs;

import java.io.File;
import java.util.Date;
import java.util.Set;

import org.codelibs.core.io.InputStreamUtil;
import org.codelibs.core.io.ResourceUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.RequestData;

    protected DynamicProperties systemProperties;

    /** Service for managing web crawler configurations */
    @Resource
    protected WebConfigService webConfigService;

    /** Service for managing file crawler configurations */
    @Resource
    protected FileConfigService fileConfigService;

    /** Helper for managing crawler processes */
    @Resource
    protected ProcessHelper processHelper;

        if (artifactType != ArtifactType.CRAWLER) {
            return false;
        }

        if ("fess-crawler".equals(name)//
                || "fess-crawler-db".equals(name)//
                || "fess-crawler-db-h2".equals(name)//
                || "fess-crawler-db-mysql".equals(name)//
                || "fess-crawler-es".equals(name)//
                || "fess-crawler-opensearch".equals(name)//

import org.codelibs.core.lang.ThreadUtil;
import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.helper.impl.MimeTypeHelperImpl;

import java.util.regex.Pattern;

import org.apache.commons.io.input.BOMInputStream;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.RobotsTxt;
import org.codelibs.fess.crawler.entity.RobotsTxt.Directive;
import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=

import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.AbstractCrawlerClient;
import org.codelibs.fess.crawler.client.AccessTimeoutTarget;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;

	<modelVersion>4.0.0</modelVersion>
	<artifactId>fess-crawler-lasta</artifactId>
	<packaging>jar</packaging>
	<name>Fess Crawler for LastaFlute</name>
	<description>This is LastaFlute support.</description>
	<parent>
		<groupId>org.codelibs.fess</groupId>
		<artifactId>fess-crawler-parent</artifactId>
		<version>15.3.0-SNAPSHOT</version>
		<relativePath>../pom.xml</relativePath>
	</parent>