*
 * <p>Example usage:
 * <pre>
 *   Crawler crawler = new Crawler();
 *   crawler.addUrl("http://example.com/");
 *   crawler.execute();
 *   crawler.close();
 * </pre>
 */
public class Crawler implements Runnable, AutoCloseable {

    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /**
     * Service for managing URL queues during crawling.
     */

 * </pre>
 */
public class Crawler {

    /**
     * Creates a new instance of Crawler.
     */
    public Crawler() {
        // Default constructor
    }

    /** Logger instance for this class. */
    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /** Thread name for web and file system crawling process. */

{"index":{"_index":"fess_config.scheduled_job","_id":"default_crawler"}}
{"name":"Default Crawler","target":"all","cronExpression":"0 0 * * *","scriptType":"groovy","scriptData":"return container.getComponent(\"crawlJob\").logLevel(\"info\").gcLogging().execute(executor);","jobLogging":true,"crawler":true,"available":true,"sortOrder":1,"createdBy":"system","createdTime":0,"updatedBy":"system","updatedTime":0}
{"index":{"_index":"fess_config.scheduled_job","_id":"suggest_indexer"}}

**DI Config**: `fess-crawler-lasta/src/main/resources/`
- `crawler.xml` (root), `crawler/client.xml`, `crawler/extractor.xml`, `crawler/rule.xml`, `crawler/transformer.xml`, `crawler/transformer_basic.xml`
- `crawler/mimetype.xml`, `crawler/encoding.xml`, `crawler/robotstxt.xml`, `crawler/sitemaps.xml`

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.container;

import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlQueueService;
import org.dbflute.utflute.lastadi.LastaDiTestCase;
import org.junit.jupiter.api.Test;

        Crawler.Options options = new Crawler.Options();
        options.sessionId = "test-session";
        options.fileConfigIds = "file1";

        int result = crawler.doCrawl(options);
        assertEquals(Constants.EXIT_OK, result);
    }

    @Test
    public void test_doCrawl_withDataConfigIds() {
        Crawler.Options options = new Crawler.Options();

        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-lasta"));
        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-parent"));
        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-playwright"));
        assertTrue(pluginHelper.isExcludedName(ArtifactType.CRAWLER, "fess-crawler-webdriver"));

import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;
import org.codelibs.fess.crawler.service.UrlQueueService;
import org.codelibs.fess.crawler.transformer.impl.FileTransformer;
import org.codelibs.fess.crawler.util.CrawlerWebServer;
import org.codelibs.opensearch.runner.OpenSearchRunner;
import org.dbflute.utflute.lastadi.LastaDiTestCase;

 */
package org.codelibs.fess.crawler;

import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

import org.codelibs.core.collection.LruHashSet;
import org.codelibs.fess.crawler.filter.UrlFilter;
import org.codelibs.fess.crawler.interval.IntervalController;
import org.codelibs.fess.crawler.rule.RuleManager;

/**

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.CrawlerStatus;
import org.codelibs.fess.crawler.interval.FessIntervalController;
import org.codelibs.fess.crawler.service.impl.OpenSearchDataService;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlFilterService;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;