reader.setScrollSize(1000);
        int count = 0;
        Set<String> valueSet = Collections.synchronizedSet(new HashSet<>());
        Map<String, Object> source;
        while ((source = reader.read()) != null) {
            assertTrue(source.get("field1").toString().startsWith("test"));
            valueSet.add(source.get("field1").toString());
            count++;
        }
        assertEquals(num, count);

        return inputStream.read(b, off, len);
    }

    /**
     * Reads some number of bytes from the input stream and stores them into the buffer array b.
     *
     * @param b the buffer into which the data is read
     * @return the total number of bytes read into the buffer, or -1 if there is no more data
     * @throws IOException if an I/O error occurs
     */
    @Override
    public int read(final byte[] b) throws IOException {

     */
    public File getTemporaryFile() {
        return tempFile;
    }

    /*
     * (non-Javadoc)
     *
     * @see java.io.InputStream#read()
     */
    @Override
    public int read() throws IOException {
        return fileInputStream.read();
    }

    @Override
    public int available() throws IOException {
        return fileInputStream.available();
    }

    /**

/**
 * The {@code QueryLogReader} interface provides methods to read query logs and close the reader.
 * It extends the {@code Closeable} interface, ensuring that resources can be released when no longer needed.
 */
public interface QueryLogReader extends Closeable {
    /**
     * Reads a query log.
     * @return The query log.
     */
    QueryLog read();

    @Override
    void close();

 */
public interface DocumentReader extends Closeable {
    /**
     * Reads a document and returns its contents as a map.
     *
     * @return a map containing the document's data, or null if there are no more documents to read.
     */
    Map<String, Object> read();

    @Override
    void close();

/**
 * <p>
 * {@link ESSourceReader} reads documents from Elasticsearch using the scroll API.
 * It implements the {@link DocumentReader} interface to provide a way to iterate over documents
 * in a large index without loading all of them into memory at once.
 * </p>
 *
 * <p>
 * The reader supports limiting the number of documents read based on a percentage of the total documents

     */
    public void setOutputEncoding(final String outputEncoding) {
        this.outputEncoding = outputEncoding;
    }

    /**
     * Sets whether to read content as text if extraction fails.
     * @param readAsTextIfFailed If true, read a content as a text when an extraction fails.
     */
    public void setReadAsTextIfFailed(final boolean readAsTextIfFailed) {
        this.readAsTextIfFailed = readAsTextIfFailed;

        } catch (final Exception e) {
            CloseableUtil.closeQuietly(fos); // for deleting file
            FileUtil.deleteInBackground(tempFile); // clean up
            throw new CrawlingAccessException("Could not read a response body: " + responseData.getUrl(), e);
        } finally {
            CloseableUtil.closeQuietly(fos);
        }
        return tempFile;
    }

            int alphanumSize = 0;
            int symbolSize = 0;
            int c;
            final Set<String> termCache = new HashSet<>(1000);
            try {
                while ((c = reader.read()) != -1) {
                    if (Character.isISOControl(c) || isSpaceChar(c)) {
                        if (duplicateTermRemoved) {
                            if (alphanumSize > 0) {

        final BufferedInputStream bis = new BufferedInputStream(in);
        bis.mark(preloadSize);

        final byte[] bytes = new byte[preloadSize];
        try {
            if (bis.read(bytes) == -1) {
                return false;
            }

            final String preloadDate = new String(bytes, Constants.UTF_8);