Disallow: /
Allow: /$
Allow: /index.html$
Allow: /public/

# Test priority rules (longer match wins)
User-agent: PriorityBot
Disallow: /store
Allow: /store/public
Disallow: /store/public/sale

# Test Allow vs Disallow with same length (Allow wins)
User-agent: SameLengthBot
Disallow: /page
Allow: /page

# Test multiple wildcards
User-agent: MultiWildcardBot
Disallow: /*.cgi*
Disallow: /*?*id=*

        assertFalse(robotsTxt.allows("/store", "PriorityBot"));
        assertFalse(robotsTxt.allows("/store/items", "PriorityBot"));
        assertTrue(robotsTxt.allows("/store/public", "PriorityBot")); // Allow is more specific
        assertTrue(robotsTxt.allows("/store/public/items", "PriorityBot"));
        assertFalse(robotsTxt.allows("/store/public/sale", "PriorityBot")); // Most specific disallow

 * that controls the interval between requests to the same host.
 * It uses a Guava Cache to store the last access time for each host.
 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.

        }
        if (salt == null || salt.length != SALT_SIZE) {
            throw new IllegalArgumentException("Invalid salt");
        }

        this.salt = salt.clone();
        this.masterKey = deriveKey(masterPassword, this.salt);

        // Clear the master password after use
        Arrays.fill(masterPassword, '\0');
    }

                final long numOfDeleted = searchEngineClient.deleteByQuery(index, queryBuilder);
                logger.info("[{}] Deleted {} stale documents.", dataConfig.getName(), numOfDeleted);
            } catch (final Exception e) {
                logger.error("[{}] Failed to delete stale documents.", dataConfig.getName(), e);
            }
        }

        /**
         * Checks if the crawling thread has finished execution.

        // Act & Assert
        // unwrap to interface itself should return same instance
        CredentialsInternal unwrapped = creds.unwrap(CredentialsInternal.class);
        assertNotNull(unwrapped, "unwrap to CredentialsInternal returns instance");
        assertSame(creds, unwrapped, "unwrap returns the same object instance");

    }

    /**
     * Test: Thread-safe listener invocation during connection
     * Verifies that listeners can be added while connect() is executing
     */
    @Test
    public void testConcurrentListenerInvocation() throws Exception {
        // This test verifies CopyOnWriteArrayList allows safe iteration
        assertTrue("OnConnectListener list should be thread-safe",

        @DisplayName("Should return salt correctly")
        void testGetSalt() {
            int[] hashAlgos = { 0x01 };
            byte[] salt = { 0x01, 0x02, 0x03, 0x04 };
            PreauthIntegrityNegotiateContext context = new PreauthIntegrityNegotiateContext(mockConfig, hashAlgos, salt);

            assertArrayEquals(salt, context.getSalt());
            // The implementation returns the same reference, not a defensive copy

Execution context and configuration.

**Important Fields**:
```java
String sessionId                // Format: yyyyMMddHHmmssSSS
volatile CrawlerStatus status   // DONE, RUNNING
AtomicLong accessCount          // Thread-safe counter
int numOfThread = 10            // Crawler threads
int maxDepth = -1               // Max depth (-1 = unlimited)
long maxAccessCount = 0         // Max URLs (0 = unlimited)
```

        startLatch.countDown();

        // Wait for all threads to complete
        assertTrue("Threads did not complete in time", doneLatch.await(5, TimeUnit.SECONDS));

        // All threads should fail with the same IOException (project ID is blank)
        // But importantly, there should be no race condition errors
        assertEquals(threadCount, failureCount.get());
        assertEquals(0, successCount.get());