* @return the sitemap set
         */
        public SitemapSet getSitemapSet() {
            return sitemapSet;
        }

    }

    /**
     * Sets the preload size for checking file format.
     * @param preloadSize the preload size in bytes
     */
    public void setPreloadSize(final int preloadSize) {
        this.preloadSize = preloadSize;
    }

    }

    /**
     * Gets the preload size for charset detection.
     *
     * @return the preload size in bytes
     */
    public int getPreloadSizeForCharset() {
        return preloadSizeForCharset;
    }

    /**
     * Sets the preload size for charset detection.
     *
     * @param preloadSizeForCharset the preload size in bytes to set
     */

     */
    public void setOutputEncoding(final String outputEncoding) {
        this.outputEncoding = outputEncoding;
    }

    /**
     * Sets whether to read content as text if extraction fails.
     * @param readAsTextIfFailed If true, read a content as a text when an extraction fails.
     */
    public void setReadAsTextIfFailed(final boolean readAsTextIfFailed) {
        this.readAsTextIfFailed = readAsTextIfFailed;

 * </p>
 *
 * <p>
 * The reader supports limiting the number of documents read based on a percentage of the total documents
 * or a fixed number. It also allows filtering documents based on their size, using the {@code limitOfDocumentSize}
 * parameter.
 * </p>
 *
 * <p>
 * The reader uses a queue to buffer documents read from Elasticsearch, and it retries failed requests
 * up to a maximum number of times.
 * </p>
 *

            int alphanumSize = 0;
            int symbolSize = 0;
            int c;
            final Set<String> termCache = new HashSet<>(1000);
            try {
                while ((c = reader.read()) != -1) {
                    if (Character.isISOControl(c) || isSpaceChar(c)) {
                        if (duplicateTermRemoved) {
                            if (alphanumSize > 0) {

            try {
                QueryLog queryLog = queryLogReader.read();
                while (queryLog != null) {
                    if (Thread.currentThread().isInterrupted()) {
                        break;
                    }
                    queryLogs.add(queryLog);
                    queryLog = queryLogReader.read();

         * @return true if terminated, false otherwise.
         */
        public boolean isTeminated() {
            return teminated;
        }
    }

    /**
     * Thread to read and buffer output from an input stream.
     */
    protected static class InputStreamThread extends Thread {

        private BufferedReader br;

        private final List<String> list = new LinkedList<>();

    client.setMaxConnectionsPerRoute(20);   // Per-host connections
    client.setConnectionTimeout(30000);     // Connection timeout
    client.setSocketTimeout(60000);         // Read timeout
});
```

### Memory Management

```java
// Configure memory usage
crawler.crawlerContext.setMaxAccessCount(10000);     // Limit crawled URLs

                BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(dest))) {
            final byte[] buf = new byte[1024];
            int length;
            while (-1 < (length = in.read(buf))) {
                out.write(buf, 0, length);
                out.flush();
            }
        } catch (final IOException e) {
            throw new IORuntimeException(e);
        }
    }

        xpathAPICache =
                CacheBuilder.newBuilder().expireAfterAccess(cacheDuration, TimeUnit.MINUTES).build(new CacheLoader<String, XPathAPI>() {
                    @Override
                    public XPathAPI load(final String key) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("created XPathAPI by {}", key);
                        }
                        return new XPathAPI();