*
 * <p>Example usage:
 * <pre>
 *   Crawler crawler = new Crawler();
 *   crawler.addUrl("http://example.com/");
 *   crawler.execute();
 *   crawler.close();
 * </pre>
 */
public class Crawler implements Runnable, AutoCloseable {

    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /**
     * Service for managing URL queues during crawling.
     */

 * </pre>
 */
public class Crawler {

    /**
     * Creates a new instance of Crawler.
     */
    public Crawler() {
        // Default constructor
    }

    /** Logger instance for this class. */
    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /** Thread name for web and file system crawling process. */

    OkHttpClient client = new OkHttpClient.Builder()
        .cache(cache)
        .callTimeout(5, TimeUnit.SECONDS)
        .build();

    Crawler crawler = new Crawler(client, queueLimit, hostLimit);
    crawler.queue.add(HttpUrl.get(args[1]));
    crawler.parallelDrainQueue(threadCount);
  }

            crawler1.addUrl(url1);
            crawler1.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler1.getCrawlerContext().setNumOfThread(numOfThread);

            final Crawler crawler2 = crawlerContainer.getComponent("crawler");
            crawler2.setSessionId(crawler2.getSessionId() + "2");
            crawler2.setBackground(true);

```java
// Create multiple crawler instances
Crawler crawler1 = container.getComponent("crawler");
crawler1.setSessionId("session1");
crawler1.addUrl("https://site1.com");

Crawler crawler2 = container.getComponent("crawler");  
crawler2.setSessionId("session2");
crawler2.addUrl("https://site2.com");

// Execute concurrently
crawler1.setBackground(true);
crawler2.setBackground(true);

            crawler1.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler1.getCrawlerContext().setNumOfThread(numOfThread);

            final Crawler crawler2 = getComponent(Crawler.class);
            crawler2.setBackground(true);
            ((UrlFilterImpl) crawler2.urlFilter).setIncludeFilteringPattern("$1$2$3.*");
            crawler2.addUrl(url2);
            crawler2.getCrawlerContext().setMaxAccessCount(maxCount);

            crawler1.addUrl(url1);
            crawler1.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler1.getCrawlerContext().setNumOfThread(numOfThread);

            final Crawler crawler2 = container.getComponent("crawler");
            crawler2.setSessionId(crawler2.getSessionId() + "2");
            crawler2.setBackground(true);

        Crawler.Options options = new Crawler.Options();
        options.sessionId = "test-session";
        options.fileConfigIds = "file1";

        int result = crawler.doCrawl(options);
        assertEquals(Constants.EXIT_OK, result);
    }

    public void test_doCrawl_withDataConfigIds() {
        Crawler.Options options = new Crawler.Options();
        options.sessionId = "test-session";

import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.entity.UrlQueueImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * Implementation of the {@link UrlQueueService} interface.
 * This class provides methods for managing a queue of URLs to be crawled,

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;