}
                    } finally {
                        latch.countDown();
                    }
                }).start();
            }

            latch.await(5, java.util.concurrent.TimeUnit.SECONDS);
            assertEquals(5, successCount.get(), "All concurrent calls should succeed");
        }
    }

                    originalCause));
        }

        return CompletableFuture.supplyAsync(() -> {
            try {
                // Wait before retry (except first attempt)
                if (attempt > 0) {
                    long delay = retryDelay * (1L << attempt); // Exponential backoff
                    Thread.sleep(delay);

        }
        return directive.allows(path);
    }

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found

                    this.handle = this.pipe.openUnshared(this.uncPath, 0, this.access, this.sharing, SmbConstants.ATTR_NORMAL, 0);
                    return this.handle.acquire();
                }

                // Wait for named pipe availability - called when pipe is not immediately available
                if (this.uncPath.startsWith("\\pipe\\")) {

            settingsBuilder.put("http.cors.enabled", true);
            settingsBuilder.put("discovery.type", "single-node");
        }).build(newConfigs().clusterName(clusterName).numOfNode(1));

        // wait for yellow status
        runner.ensureYellow();

        System.setProperty(FesenClient.HTTP_ADDRESS, "localhost:" + runner.node().settings().get("http.port", "9201"));

        super.setUp();
    }

    @Override

            });
        }

        // Start all threads simultaneously
        for (Thread thread : threads) {
            thread.start();
        }

        // Wait for all threads to complete
        for (Thread thread : threads) {
            thread.join();
        }

        // All threads should see the initialized state
        for (boolean result : initResults) {

// Check crawling status
while (crawler.crawlerContext.getStatus() == CrawlerStatus.RUNNING) {
    Thread.sleep(1000);
    System.out.println("Crawling in progress...");
}

// Wait for completion
crawler.awaitTermination();
System.out.println("Crawling completed");
```

### Custom Content Extractor

```java
public class CustomExtractor extends AbstractExtractor {
    @Override

                        }
                    }

                    final long start = System.currentTimeMillis();
                    while (timeout > 0) {
                        response.wait(timeout);

                        /* JetDirect printer can respond to regular broadcast query
                         * with node status so we need to check to make sure that

         * Waits for the crawling thread to terminate within a specified time limit.
         * This method blocks until the thread completes or the timeout expires.
         *
         * @param mills maximum time to wait in milliseconds
         */
        public void awaitTermination(final long mills) {
            try {
                join(mills);
            } catch (final InterruptedException e) {

import org.dbflute.utflute.core.PlainTestCase;
import org.testcontainers.Testcontainers;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;

import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import io.minio.SetObjectTagsArgs;

/**