reader.setScrollSize(1000);
        int count = 0;
        Set<String> valueSet = Collections.synchronizedSet(new HashSet<>());
        Map<String, Object> source;
        while ((source = reader.read()) != null) {
            assertTrue(source.get("field1").toString().startsWith("test"));
            valueSet.add(source.get("field1").toString());
            count++;
        }

     * @throws CurlException if an error occurs while accessing the content.
     */
    public String getContentAsString() {
        final byte[] bytes = new byte[4096];
        try (BufferedInputStream bis = new BufferedInputStream(getContentAsStream());
                ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
            int length = bis.read(bytes);
            while (length != -1) {
                if (length != 0) {

     * delayMillisBeforeProcessing parameter.
     *
     * @throws InterruptedRuntimeException if the thread is interrupted during the delay
     * @throws CrawlerSystemException if an error occurs while processing the URL
     */
    @Override
    protected void delayBeforeProcessing() {
        final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
        if (urlQueue == null) {
            return;

 *   <li>After processing a URL ({@link #delayAfterProcessing()})</li>
 *   <li>When there are no URLs in the queue ({@link #delayAtNoUrlInQueue()})</li>
 *   <li>While waiting for new URLs to be added to the queue ({@link #delayForWaitingNewUrl()})</li>
 * </ul>
 *
 * <p>
 * Subclasses are responsible for implementing the abstract methods to define the actual delay

            urlQueueList.add(urlQueue);
        }

        urlQueueService.offerAll(sessionId, urlQueueList);

        // Poll all items
        int count = 0;
        while (true) {
            final OpenSearchUrlQueue polled = urlQueueService.poll(sessionId);
            if (polled == null) {
                break;
            }
            count++;

     *
     * @param responseData the response data containing the response body
     * @return a temporary file containing the response body
     * @throws CrawlingAccessException if an error occurs while reading the response body
     */
    public static File createResponseBodyFile(final ResponseData responseData) {
        File tempFile = null;
        try (final InputStream is = responseData.getResponseBody()) {

        final Pattern attrPattern = Pattern.compile("\\s[^ ]+=\"([^\"]*)\"");
        while (matcher.find()) {
            final String tagStr = matcher.group();
            final Matcher attrMatcher = attrPattern.matcher(tagStr);
            final StringBuilder buf = new StringBuilder(100);
            while (attrMatcher.find()) {
                buf.append(attrMatcher.group(1)).append(' ');
            }

     * Checks if a bucket exists in the object storage.
     * @param name the name of the bucket to check
     * @return true if the bucket exists, false otherwise
     * @throws CrawlingAccessException if an error occurs while checking bucket existence
     */
    protected boolean bucketExists(final String name) {
        try {
            final BucketExistsArgs args = BucketExistsArgs.builder().bucket(name).build();

     */
    protected void parseJwtClaim(final String jwtClaim, final Map<String, Object> attributes) throws IOException {
        try (final JsonParser jsonParser = jsonFactory.createJsonParser(jwtClaim)) {
            while (jsonParser.nextToken() != JsonToken.END_OBJECT) {
                final String name = jsonParser.getCurrentName();
                if (name != null) {
                    jsonParser.nextToken();

 * the standard Fess file transformation process with support for various content types.
 *
 * <p>It extends AbstractFessFileTransformer to provide file-specific transformation
 * capabilities while using the appropriate extractor for each document type.</p>
 */
public class FessStandardTransformer extends AbstractFessFileTransformer {
    /**
     * Default constructor.
     */
    public FessStandardTransformer() {