assertFalse(robotsTxt.allows("/store", "PriorityBot"));
        assertFalse(robotsTxt.allows("/store/items", "PriorityBot"));
        assertTrue(robotsTxt.allows("/store/public", "PriorityBot")); // Allow is more specific
        assertTrue(robotsTxt.allows("/store/public/items", "PriorityBot"));
        assertFalse(robotsTxt.allows("/store/public/sale", "PriorityBot")); // Most specific disallow

 * that controls the interval between requests to the same host.
 * It uses a Guava Cache to store the last access time for each host.
 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.

                final long numOfDeleted = searchEngineClient.deleteByQuery(index, queryBuilder);
                logger.info("[{}] Deleted {} stale documents.", dataConfig.getName(), numOfDeleted);
            } catch (final Exception e) {
                logger.error("[{}] Failed to delete stale documents.", dataConfig.getName(), e);
            }
        }

        /**
         * Checks if the crawling thread has finished execution.

Execution context and configuration.

**Important Fields**:
```java
String sessionId                // Format: yyyyMMddHHmmssSSS
volatile CrawlerStatus status   // DONE, RUNNING
AtomicLong accessCount          // Thread-safe counter
int numOfThread = 10            // Crawler threads
int maxDepth = -1               // Max depth (-1 = unlimited)
long maxAccessCount = 0         // Max URLs (0 = unlimited)
```

        startLatch.countDown();

        // Wait for all threads to complete
        assertTrue("Threads did not complete in time", doneLatch.await(5, TimeUnit.SECONDS));

        // All threads should fail with the same IOException (project ID is blank)
        // But importantly, there should be no race condition errors
        assertEquals(threadCount, failureCount.get());
        assertEquals(0, successCount.get());

    }

    /**
     * Test isPathSafe with safe path
     *
     * @throws Exception
     */
    @Test
    public void testIsPathSafe_SafePath() throws Exception {
        final Path baseDir = tempFolder.getRoot().toPath();
        final Path safePath = baseDir.resolve("subdir/file.txt");

        assertTrue("Safe path should be allowed", FileUtil.isPathSafe(safePath, baseDir));
    }

    /**

    protected static final int DEFAULT_BUF_SIZE = 4096; // 4k

    /** Max Buffer Size */
    protected static final int MAX_BUF_SIZE = 10 * 1024 * 1024; // 10m

    /**
     * Validates that a given path is safe and does not attempt path traversal attacks.
     * <p>
     * This method checks if the resolved absolute path starts with the allowed base directory,

        // Mix of same and different hosts
        for (int i = 0; i < numTasks; i++) {
            final int index = i;
            tasks.add(new Callable<Integer>() {
                public Integer call() throws Exception {
                    final UrlQueue q = new UrlQueueImpl();
                    // Use modulo to create multiple accesses to same hosts

        startLatch.countDown();

        // Wait for all threads to complete
        assertTrue("Threads did not complete in time", doneLatch.await(5, TimeUnit.SECONDS));

        // All threads should fail with the same IOException (endpoint is blank)
        // But importantly, there should be no race condition errors
        assertEquals(threadCount, failureCount.get());
        assertEquals(0, successCount.get());

    protected long requestInterval = 500;

    /**
     * Sends a list of documents to the search engine for indexing.
     * This method handles thumbnail processing, deletes old documents with the same URL,
     * and performs bulk indexing operations with proper error handling.
     *
     * @param searchEngineClient the search engine client to use for indexing
     * @param docList the list of documents to be indexed