```java
// Set maximum number of URLs to crawl
crawler.crawlerContext.setMaxAccessCount(1000);

// Set number of crawler threads
crawler.crawlerContext.setNumOfThread(10);

// Set maximum crawl depth
crawler.crawlerContext.setMaxDepth(3);

// Set request interval (politeness)
crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns

        // Add URLs up to capacity (10000)
        for (int i = 0; i < 10000; i++) {
            urlSet.add("http://example" + i + ".com/robots.txt");
        }
        assertEquals(10000, urlSet.size());

        // Add one more should maintain size at 10000 (LRU eviction)
        urlSet.add("http://overflow.com/robots.txt");
        assertEquals(10000, urlSet.size());

        AtomicInteger numObInputDoc = new AtomicInteger(0);
        ESSourceReader reader = new ESSourceReader(client, suggester.settings(), indexName);
        reader.setScrollSize(1000);

        suggester.indexer().indexFromDocument(() -> reader, 1000, () -> ThreadUtil.sleep(100)).then(response -> {
            numObInputDoc.set(response.getNumberOfInputDocs());
            latch.countDown();
        }).error(t -> {
            t.printStackTrace();

        // Create multiple threads adding patterns
        Thread thread1 = new Thread(() -> {
            for (int i = 0; i < 100; i++) {
                urlFilter.addInclude("https://site" + i + ".com/.*");
            }
        });

        Thread thread2 = new Thread(() -> {
            for (int i = 0; i < 100; i++) {
                urlFilter.addExclude(".*\\.type" + i + "$");
            }
        });

        } else {
            this.limitPercentage = Integer.parseInt(limitPercentage);
        }

        if (this.limitPercentage > 100) {
            this.limitPercentage = 100;
        } else if (this.limitPercentage < 1) {
            this.limitPercentage = 1;
        }
    }

    /**
     * Sets the limit number.

 * <pre>{@code
 * Reader reader = new StringReader("Example text to normalize.");
 * String normalizedText = TextUtil.normalizeText(reader)
 *                                  .initialCapacity(5000)
 *                                  .maxAlphanumTermSize(100)
 *                                  .maxSymbolTermSize(50)
 *                                  .duplicateTermRemoved(true)
 *                                  .spaceChars(new int[] { ' ', '\u00a0' })

     * @return the extracted body text
     * @throws ExtractException if extraction fails
     */
    protected String getBodyText(final MimeMessage message) {
        final StringBuilder buf = new StringBuilder(1000);
        try {
            final Object content = message.getContent();
            if (content instanceof final Multipart multipart) {
                final int count = multipart.getCount();

    private final int maxAnalyzedContentLength;

    /**
     * Constructor.
     */
    public DefaultContentsParser() {
        maxAnalyzedContentLength = Integer.getInteger("fess.suggest.max.analyzed.content.length", 1000);
    }

    @Override
    public SuggestItem parseSearchWords(final String[] words, final String[][] readings, final String[] fields, final String[] tags,

            throw new CrawlingAccessException("Could not parse " + responseData.getUrl(), e);
        }
        final Document document = parser.getDocument();

        final StringBuilder buf = new StringBuilder(1000);
        buf.append(getResultDataHeader());
        for (final Map.Entry<String, String> entry : fieldRuleMap.entrySet()) {
            final String path = entry.getValue();
            try {

    protected long executionTimeout = 30L * 1000L; // 30sec

    /** The working directory for the command. */
    protected File workingDirectory = null;

    /** The encoding for the command's output. */
    protected String commandOutputEncoding = Charset.defaultCharset().displayName();

    /** The maximum number of lines to buffer from command output. */
    protected int maxOutputLine = 1000;