* governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.util;

import org.codelibs.fess.util.ComponentUtil;

/**
 * Fess-specific crawler configuration that extends OpenSearchCrawlerConfig.
 * This class provides configuration settings for the Fess crawler including
 * index names, shard counts, and replica counts for queue, data, and filter indices.
 */

labels.crawling_info_delete_all_cancel=Cancel
labels.crawling_info_thread_dump=Thread Dump
labels.crawling_info_CrawlerStartTime=Crawler Start Time
labels.crawling_info_CrawlerEndTime=Crawler End Time
labels.crawling_info_CrawlerExecTime=Crawler Execution Time
labels.crawling_info_CrawlerStatus=Crawler Status
labels.crawling_info_WebFsCrawlExecTime=Crawl Execution Time (Web/File)

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="intervalController"
		class="org.codelibs.fess.crawler.interval.FessIntervalController"
		instance="prototype">
	</component>

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN" 
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<!-- Crawler Thread -->
	<component name="crawlerThread" class="org.codelibs.fess.crawler.FessCrawlerThread" instance="prototype" >
	</component>

        }
        if (1500 <= count) {
            logger.info("Time out: Failed to start crawler)");
            fail(); // Time Out
        }

        logger.info("Crawler is running");
        count = 0;
        isRunning = true;
        sleepTime = 100; // Reset to 100ms for termination wait

        // Wait until the crawler terminates (with exponential backoff)
        while (count < 3000 && isRunning) {

import org.codelibs.core.lang.ThreadUtil;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;

        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-lasta"));
        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-parent"));
        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-playwright"));
        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-webdriver"));

<project name="deps" basedir=".">
	<property name="target.dir" value="${basedir}/target/deps" />
	<property name="webinf.dir" value="${basedir}/src/main/webapp/WEB-INF" />
	<property name="crawler.dir" value="${basedir}/src/main/webapp/WEB-INF/env/crawler" />
	<property name="suggest.dir" value="${basedir}/src/main/webapp/WEB-INF/env/suggest" />
	<property name="thumbnail.dir" value="${basedir}/src/main/webapp/WEB-INF/env/thumbnail" />

  -F "file=@documents.bulk"
```

### 4. Direct Crawler Configuration

Instead of importing documents, configure Fess to crawl your data sources:

**Web Crawling**:
- **Admin Path**: Crawler > Web
- **Supports**: HTTP/HTTPS websites
- **Features**: JavaScript rendering, authentication, custom headers

**File Crawling**:
- **Admin Path**: Crawler > File
- **Supports**: SMB, FTP, local file systems

        helper.setCrawlerWaitMillis(50);

        // Test when crawler is running
        assertTrue(helper.isCrawlerRunning());
        long start = System.currentTimeMillis();
        helper.checkCrawlerStatus();
        long end = System.currentTimeMillis();
        assertTrue(end - start < 100); // Should return quickly

        // Test when crawler is not running
        helper.setCrawlerRunning(false);