Client client = runner.client();
        SuggestSettings settings = suggester.settings();
        int num = 10000;

        addDocument(indexName, client, num);

        ESSourceReader reader = new ESSourceReader(client, settings, indexName);
        reader.setScrollSize(1000);
        int count = 0;
        Set<String> valueSet = Collections.synchronizedSet(new HashSet<>());
        Map<String, Object> source;

```java
// Set maximum number of URLs to crawl
crawler.crawlerContext.setMaxAccessCount(1000);

// Set number of crawler threads
crawler.crawlerContext.setNumOfThread(10);

// Set maximum crawl depth
crawler.crawlerContext.setMaxDepth(3);

// Set request interval (politeness)
crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns

        // Add URLs up to capacity (10000)
        for (int i = 0; i < 10000; i++) {
            urlSet.add("http://example" + i + ".com/robots.txt");
        }
        assertEquals(10000, urlSet.size());

        // Add one more should maintain size at 10000 (LRU eviction)
        urlSet.add("http://overflow.com/robots.txt");
        assertEquals(10000, urlSet.size());

        AtomicInteger numObInputDoc = new AtomicInteger(0);
        ESSourceReader reader = new ESSourceReader(client, suggester.settings(), indexName);
        reader.setScrollSize(1000);

        suggester.indexer().indexFromDocument(() -> reader, 1000, () -> ThreadUtil.sleep(100)).then(response -> {
            numObInputDoc.set(response.getNumberOfInputDocs());
            latch.countDown();
        }).error(t -> {
            t.printStackTrace();

        }

        final MimeTypeHelper mimeTypeHelper = getMimeTypeHelper();
        final ExtractorFactory extractorFactory = getExtractorFactory();
        final StringBuilder buf = new StringBuilder(1000);

        try (final ArchiveInputStream ais =
                archiveStreamFactory.createArchiveInputStream(in.markSupported() ? in : new BufferedInputStream(in))) {
            ZipArchiveEntry entry = null;

     * @return A text.
     */
    protected String getTextInternal(final InputStream in, final MimeTypeHelper mimeTypeHelper, final ExtractorFactory extractorFactory) {

        final StringBuilder buf = new StringBuilder(1000);

        ArchiveInputStream ais = null;

        try {
            ais = archiveStreamFactory.createArchiveInputStream("tar", in);
            TarArchiveEntry entry = null;
            long contentSize = 0;

        // Create multiple threads adding patterns
        Thread thread1 = new Thread(() -> {
            for (int i = 0; i < 100; i++) {
                urlFilter.addInclude("https://site" + i + ".com/.*");
            }
        });

        Thread thread2 = new Thread(() -> {
            for (int i = 0; i < 100; i++) {
                urlFilter.addExclude(".*\\.type" + i + "$");
            }
        });

        }

        final MimeTypeHelper mimeTypeHelper = getMimeTypeHelper();
        final ExtractorFactory extractorFactory = getExtractorFactory();
        final StringBuilder buf = new StringBuilder(1000);

        File tempFile = null;
        LhaFile lhaFile = null;
        try {
            tempFile = createTempFile("crawler-", ".lzh", null);
            try (FileOutputStream fos = new FileOutputStream(tempFile)) {

        } else {
            this.limitPercentage = Integer.parseInt(limitPercentage);
        }

        if (this.limitPercentage > 100) {
            this.limitPercentage = 100;
        } else if (this.limitPercentage < 1) {
            this.limitPercentage = 1;
        }
    }

    /**
     * Sets the limit number.

        private final Map<String, Object> dataMap = new HashMap<>();

        private String fieldName;

        private final StringBuilder buffer = new StringBuilder(1000);

        @Override
        public void startDocument() {
            dataMap.clear();
        }

        @Override