/**
     * Data service for checking access results.
     */
    @Resource
    protected OpenSearchDataService dataService;

    /**
     * Cache for session queues.
     */
    protected Map<String, QueueHolder> sessionCache = new ConcurrentHashMap<>();

    /**
     * The number of URLs to fetch when polling.
     */
    protected int pollingFetchSize = 1000;

    /**

            finishedSessionIdList.add(sessionId);
        }
    }

    /**
     * Deletes all data associated with a specific crawler session.
     * Removes URL filters, URL queues, and access result data for the session.
     *
     * @param sessionId the session ID whose data should be deleted
     */
    private void deleteBySessionId(final String sessionId) {
        try {

        // Execute
        service.generateUrlQueues(previousSessionId, sessionId);

        // Verify
        assertEquals(2, newQueue.size());

        // Verify the generated queues have correct properties
        List<UrlQueueImpl<Long>> queueList = new ArrayList<>(newQueue);
        boolean foundPage1 = false;
        boolean foundPage2 = false;

   * @param iterable the iterable to wrap
   * @return a view of the supplied iterable that wraps each generated iterator through {@link
   *     Iterators#consumingIterator(Iterator)}; for queues, an iterable that generates iterators
   *     that return and consume the queue's elements in queue order
   * @see Iterators#consumingIterator(Iterator)
   * @since 2.0
   */

 *   crawler.close();
 * </pre>
 */
public class Crawler implements Runnable, AutoCloseable {

    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /**
     * Service for managing URL queues during crawling.
     */
    @Resource
    protected UrlQueueService<UrlQueue<?>> urlQueueService;

    /**
     * Service for managing access result data.
     */
    @Resource

 *
 * <p>Key features include:</p>
 * <ul>
 * <li>Incremental crawling support with last-modified timestamp checking</li>
 * <li>Document expiration handling</li>
 * <li>Child URL extraction and queueing</li>
 * <li>Integration with Fess configuration and permission systems</li>
 * <li>Client selection based on URL patterns</li>
 * </ul>
 *
 * @see CrawlerThread
 * @see org.codelibs.fess.crawler.client.CrawlerClient

    assertFalse(segment.clearValueForTesting(key, hash, valueRef));
    segment.setWeakValueReferenceForTesting(entry, valueRef);
    assertTrue(segment.clearValueForTesting(key, hash, valueRef));
  }

  // reference queues

  public void testDrainKeyReferenceQueueOnWrite() {
    for (MapMaker maker : allWeakKeyStrengthMakers()) {
      MapMakerInternalMap<Object, Object, ?, ?> map = makeMap(maker.concurrencyLevel(1));

    assertFalse(segment.clearValueForTesting(key, hash, valueRef));
    segment.setWeakValueReferenceForTesting(entry, valueRef);
    assertTrue(segment.clearValueForTesting(key, hash, valueRef));
  }

  // reference queues

  public void testDrainKeyReferenceQueueOnWrite() {
    for (MapMaker maker : allWeakKeyStrengthMakers()) {
      MapMakerInternalMap<Object, Object, ?, ?> map = makeMap(maker.concurrencyLevel(1));

      it.next();
      it.remove();
    }
    segment.evictEntries(entry);
    assertThat(map).hasSize(maxSize);
    assertThat(map).isEqualTo(originalMap);
  }

  // reference queues

  public void testDrainKeyReferenceQueueOnWrite() {
    for (CacheBuilder<Object, Object> builder : allKeyValueStrengthMakers()) {
      LocalCache<Object, Object> map = makeLocalCache(builder.concurrencyLevel(1));

      it.next();
      it.remove();
    }
    segment.evictEntries(entry);
    assertThat(map).hasSize(maxSize);
    assertThat(map).isEqualTo(originalMap);
  }

  // reference queues

  public void testDrainKeyReferenceQueueOnWrite() {
    for (CacheBuilder<Object, Object> builder : allKeyValueStrengthMakers()) {
      LocalCache<Object, Object> map = makeLocalCache(builder.concurrencyLevel(1));