*
 * <p>The job supports selective crawling by specifying configuration IDs for different
 * types of crawlers (web, file, data). It manages the crawler process lifecycle,
 * handles timeout scenarios, and ensures proper cleanup of resources.</p>
 *
 * <p>Key features:</p>
 * <ul>
 *   <li>Concurrent crawler process management with configurable limits</li>
 *   <li>Selective crawling based on configuration IDs</li>

        this.leaseToPath = new ConcurrentHashMap<>();
        this.scheduler = Executors.newScheduledThreadPool(1);
        this.changeNotifier = new DirectoryChangeNotifier(this);

        // Schedule periodic cache cleanup
        scheduler.scheduleAtFixedRate(this::cleanupExpiredEntries, 60, 60, TimeUnit.SECONDS);
    }

    /**
     * Request a directory lease
     *
     * @param directoryPath directory path

class ConnectionPoolTest {
  private val routePlanner = FakeRoutePlanner()
  private val factory = routePlanner.factory
  private val peer = MockHttp2Peer()

  /** The fake task runner prevents the cleanup runnable from being started.  */
  private val addressA = factory.newAddress("a")
  private val routeA1 = factory.newRoute(addressA)
  private val addressB = factory.newAddress("b")

    /** Property key for crawling user agent configuration. */
    public static final String CRAWLING_USER_AGENT_PROPERTY = "crawling.user.agent";

    /** Property key for cleanup day interval configuration. */
    public static final String DAY_FOR_CLEANUP_PROPERTY = "day.for.cleanup";

    // ============================================================
    // Web API Property Keys
    // ============================================================

  - [Changelog since v1.31.13](#changelog-since-v13113)
  - [Changes by Kind](#changes-by-kind)
    - [Feature](#feature)
    - [Bug or Regression](#bug-or-regression)
    - [Other (Cleanup or Flake)](#other-cleanup-or-flake)
  - [Dependencies](#dependencies)
    - [Added](#added)
    - [Changed](#changed)
    - [Removed](#removed)
- [v1.31.13](#v13113)
  - [Downloads for v1.31.13](#downloads-for-v13113)

     */
    public HtmlTagBasedGenerator() {
        super();
    }

    /**
     * Destroys this thumbnail generator and releases any resources.
     * This implementation is empty as no cleanup is required.
     */
    @Override
    public void destroy() {
    }

    /**
     * Creates a thumbnail generation task for the specified document.
     *

  - [Changelog since v1.28.14](#changelog-since-v12814)
  - [Changes by Kind](#changes-by-kind)
    - [Feature](#feature)
    - [Bug or Regression](#bug-or-regression)
    - [Other (Cleanup or Flake)](#other-cleanup-or-flake)
  - [Dependencies](#dependencies)
    - [Added](#added)
    - [Changed](#changed)
    - [Removed](#removed)
- [v1.28.14](#v12814)
  - [Downloads for v1.28.14](#downloads-for-v12814)

  - [Changelog since v1.34.1](#changelog-since-v1341)
  - [Changes by Kind](#changes-by-kind-1)
    - [Feature](#feature-1)
    - [Bug or Regression](#bug-or-regression-1)
    - [Other (Cleanup or Flake)](#other-cleanup-or-flake)
  - [Dependencies](#dependencies-1)
    - [Added](#added-1)
    - [Changed](#changed-1)
    - [Removed](#removed-1)
- [v1.34.1](#v1341)
  - [Downloads for v1.34.1](#downloads-for-v1341)

        final String sessionId = crawler.getSessionId();

        // Add some test data
        crawler.addUrl("http://example.com/");

        // Perform cleanup - should not throw exception
        crawler.cleanup(sessionId);
    }

    public void test_stop() throws Exception {
        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        try {

                5, // max attempts per IP
                20, // max global attempts per minute
                Duration.ofSeconds(2), // short lockout for testing
                Duration.ofMinutes(1) // cleanup interval
        );
    }

    @AfterEach
    public void tearDown() {
        if (rateLimiter != null) {
            rateLimiter.close();
        }
    }

    @Test