/**
     * QueueHolder holds the waiting and crawling queues.
     */
    protected static class QueueHolder {
        /**
         * Constructs a new QueueHolder.
         */
        public QueueHolder() {
            // Default constructor
        }

        /**
         * The queue for URLs waiting to be crawled.
         */
        protected Queue<OpenSearchUrlQueue> waitingQueue = new ConcurrentLinkedQueue<>();

        }
        log(logHelper, LogType.FINISHED_THREAD, crawlerContext);
    }

    /**
     * Adds sitemaps from robots.txt to the crawling queue.
     * @param urlQueue The URL queue to add sitemaps to.
     */
    protected void addSitemapsFromRobotsTxt(final UrlQueue<?> urlQueue) {
        final String[] sitemaps = crawlerContext.removeSitemaps();
        if (sitemaps != null) {

 *   crawler.close();
 * </pre>
 */
public class Crawler implements Runnable, AutoCloseable {

    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /**
     * Service for managing URL queues during crawling.
     */
    @Resource
    protected UrlQueueService<UrlQueue<?>> urlQueueService;

    /**
     * Service for managing access result data.
     */
    @Resource

 * }
 * </pre>
 */
public class ESSourceReader implements DocumentReader {
    private static final Logger logger = LogManager.getLogger(ESSourceReader.class);

    /** Queue of documents. */
    protected final Queue<Map<String, Object>> queue = new ConcurrentLinkedQueue<>();
    /** Flag indicating if reading is finished. */
    protected final AtomicBoolean isFinished = new AtomicBoolean(false);
    /** Random number generator. */

        block = new StringBuilder();
        nextBlkStart = 0;
        queue = new PriorityQueue<>(100, new MyTokensComparator());
        this.synonyms = new ArrayList<>();
    }

    @Override
    public boolean incrementToken() throws IOException {
        while (true) {
            final MyToken nextToken = getNextUniqueToken(queue, prevToken);
            if (nextToken == null) {
                getNextBlock();

 *   <li>Getting content</li>
 *   <li>Handling redirects</li>
 *   <li>Processing responses</li>
 *   <li>Handling exceptions during crawling and child URL processing</li>
 *   <li>Handling cases where no URL is in the queue</li>
 *   <li>Handling cases where no response processor or rule is found</li>
 *   <li>Handling system errors</li>
 * </ul>
 *
 * <p>
 * Each logging method checks the log level before logging the message.

    /**
     * The FTP client configuration.
     */
    protected FTPClientConfig ftpClientConfig;

    /**
     * The queue of FTPClient instances.
     */
    protected final Queue<FTPClient> ftpClientQueue = new ConcurrentLinkedQueue<>();

    /**
     * The active external host.
     */
    protected String activeExternalHost;

    /**

                return true;
            }
        }
        return false;
    }

    /**
     * Processes the result data and stores it along with child URLs.
     *
     * @param urlQueue the URL queue entry
     * @param responseData the response data
     * @param resultData the result data
     */
    protected void processResult(final UrlQueue<?> urlQueue, final ResponseData responseData, final ResultData resultData) {