* </pre>
 */
public class Crawler {

    /**
     * Creates a new instance of Crawler.
     */
    public Crawler() {
        // Default constructor
    }

    /** Logger instance for this class. */
    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /** Thread name for web and file system crawling process. */

    OkHttpClient client = new OkHttpClient.Builder()
        .cache(cache)
        .callTimeout(5, TimeUnit.SECONDS)
        .build();

    Crawler crawler = new Crawler(client, queueLimit, hostLimit);
    crawler.queue.add(HttpUrl.get(args[1]));
    crawler.parallelDrainQueue(threadCount);
  }

        Crawler.Options options = new Crawler.Options();
        options.sessionId = "test-session";
        options.fileConfigIds = "file1";

        int result = crawler.doCrawl(options);
        assertEquals(Constants.EXIT_OK, result);
    }

    public void test_doCrawl_withDataConfigIds() {
        Crawler.Options options = new Crawler.Options();
        options.sessionId = "test-session";

 */
package org.codelibs.fess.crawler.service;

import java.util.List;
import java.util.Map;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;
import org.codelibs.fess.crawler.util.OpenSearchCrawlerConfig;

    }

    public void setCrawler_Avg(ConditionOptionCall<AvgAggregationBuilder> opLambda) {
        setCrawler_Avg("crawler", opLambda);
    }

    public void setCrawler_Avg(String name, ConditionOptionCall<AvgAggregationBuilder> opLambda) {
        AvgAggregationBuilder builder = regAvgA(name, "crawler");
        if (opLambda != null) {
            opLambda.callback(builder);
        }
    }

 */
package org.codelibs.fess.crawler.processor;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor;

    /** The key of the configuration. e.g. UTF-8 */
    String CRAWLER_DOCUMENT_SITE_ENCODING = "crawler.document.site.encoding";

    /** The key of the configuration. e.g. unknown */
    String CRAWLER_DOCUMENT_UNKNOWN_HOSTNAME = "crawler.document.unknown.hostname";

    /** The key of the configuration. e.g. false */

* [LastaFlute](https://github.com/lastaflute/lastaflute "LastaFlute"): Web Framework
* [Lasta Job](https://github.com/lastaflute/lasta-job "Lasta Job"): Job Scheduler
* [Fess Crawler](https://github.com/codelibs/fess-crawler "Fess Crawler"): Web Crawler

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.CrawlerStatus;
import org.codelibs.fess.crawler.interval.FessIntervalController;
import org.codelibs.fess.crawler.service.impl.OpenSearchDataService;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlFilterService;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;

        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-lasta"));
        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-parent"));
        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-playwright"));
        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-webdriver"));