Disallow: /
Allow: /$
Allow: /index.html$
Allow: /public/

# Test priority rules (longer match wins)
User-agent: PriorityBot
Disallow: /store
Allow: /store/public
Disallow: /store/public/sale

# Test Allow vs Disallow with same length (Allow wins)
User-agent: SameLengthBot
Disallow: /page
Allow: /page

# Test multiple wildcards
User-agent: MultiWildcardBot
Disallow: /*.cgi*
Disallow: /*?*id=*

        assertFalse(robotsTxt.allows("/store", "PriorityBot"));
        assertFalse(robotsTxt.allows("/store/items", "PriorityBot"));
        assertTrue(robotsTxt.allows("/store/public", "PriorityBot")); // Allow is more specific
        assertTrue(robotsTxt.allows("/store/public/items", "PriorityBot"));
        assertFalse(robotsTxt.allows("/store/public/sale", "PriorityBot")); // Most specific disallow

"oricât", "orice", "oricînd", "oricine", "oricît", "oricum", "oriunde", "până", "pe", "pentru", "peste", "pînă", "poate", "pot", "prea", "prima", "primul", "prin", "printr", "sa", "să", "săi", "sale", "sau", "său", "se", "şi", "sînt", "sîntem", "sînteţi", "spre", "sub", "sunt", "suntem", "sunteţi", "ta", "tăi", "tale", "tău", "te", "ţi", "ţie", "tine", "toată", "toate", "tot", "toţi", "totuşi", "tu", "un", "una", "unde", "undeva", "unei", "unele", "uneori", "unor", "vă", "vi", "voastră", "voastre", "voi",...

 * that controls the interval between requests to the same host.
 * It uses a Guava Cache to store the last access time for each host.
 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.

        }
        if (salt == null || salt.length != SALT_SIZE) {
            throw new IllegalArgumentException("Invalid salt");
        }

        this.salt = salt.clone();
        this.masterKey = deriveKey(masterPassword, this.salt);

        // Clear the master password after use
        Arrays.fill(masterPassword, '\0');
    }

                final long numOfDeleted = searchEngineClient.deleteByQuery(index, queryBuilder);
                logger.info("[{}] Deleted {} stale documents.", dataConfig.getName(), numOfDeleted);
            } catch (final Exception e) {
                logger.error("[{}] Failed to delete stale documents.", dataConfig.getName(), e);
            }
        }

        /**
         * Checks if the crawling thread has finished execution.

        // Act & Assert
        // unwrap to interface itself should return same instance
        CredentialsInternal unwrapped = creds.unwrap(CredentialsInternal.class);
        assertNotNull(unwrapped, "unwrap to CredentialsInternal returns instance");
        assertSame(creds, unwrapped, "unwrap returns the same object instance");

    }

    /**
     * Test: Thread-safe listener invocation during connection
     * Verifies that listeners can be added while connect() is executing
     */
    @Test
    public void testConcurrentListenerInvocation() throws Exception {
        // This test verifies CopyOnWriteArrayList allows safe iteration
        assertTrue("OnConnectListener list should be thread-safe",

        @DisplayName("Should return salt correctly")
        void testGetSalt() {
            int[] hashAlgos = { 0x01 };
            byte[] salt = { 0x01, 0x02, 0x03, 0x04 };
            PreauthIntegrityNegotiateContext context = new PreauthIntegrityNegotiateContext(mockConfig, hashAlgos, salt);

            assertArrayEquals(salt, context.getSalt());
            // The implementation returns the same reference, not a defensive copy

Execution context and configuration.

**Important Fields**:
```java
String sessionId                // Format: yyyyMMddHHmmssSSS
volatile CrawlerStatus status   // DONE, RUNNING
AtomicLong accessCount          // Thread-safe counter
int numOfThread = 10            // Crawler threads
int maxDepth = -1               // Max depth (-1 = unlimited)
long maxAccessCount = 0         // Max URLs (0 = unlimited)
```