*   <li>Before processing a URL ({@link #delayBeforeProcessing()})</li>
 *   <li>After processing a URL ({@link #delayAfterProcessing()})</li>
 *   <li>When there are no URLs in the queue ({@link #delayAtNoUrlInQueue()})</li>
 *   <li>While waiting for new URLs to be added to the queue ({@link #delayForWaitingNewUrl()})</li>
 * </ul>
 *
 * <p>
 * Subclasses are responsible for implementing the abstract methods to define the actual delay

        for (UrlQueueImpl<Long> queue : queueList) {
            assertEquals(sessionId, queue.getSessionId());
            assertEquals(Integer.valueOf(0), queue.getDepth());
            assertNotNull(queue.getCreateTime());

            if ("https://example.com/page1".equals(queue.getUrl())) {
                foundPage1 = true;
                assertEquals("GET", queue.getMethod());

            // Default constructor
        }

        /**
         * The queue for URLs waiting to be crawled.
         */
        protected Queue<OpenSearchUrlQueue> waitingQueue = new ConcurrentLinkedQueue<>();

        /**
         * The queue for URLs currently being crawled.
         */
        protected Queue<OpenSearchUrlQueue> crawlingQueue = new ConcurrentLinkedQueue<>();
    }

    /**

    private static final int MAX_BUFFER_SIZE = 0x100000; // 1MB maximum

    // Use concurrent queue for lock-free operations
    private static final java.util.concurrent.ConcurrentLinkedQueue<byte[]> bufferQueue =
            new java.util.concurrent.ConcurrentLinkedQueue<>();

    // Track queue size with atomic counter for efficiency

    /** The metadata associated with the URL queue. */
    protected String metaData;

    /** The encoding of the URL queue. */
    protected String encoding;

    /** The parent URL of the current URL in the queue. */
    protected String parentUrl;

    /** The depth of the URL in the queue. */
    protected Integer depth;

    /** The last modified timestamp of the URL in the queue. */
    protected Long lastModified;

    }

    /**
     * Adds a URL to the crawling queue.
     * @param url The URL to add.
     */
    public void addUrl(final String url) {
        try {
            urlQueueService.add(crawlerContext.sessionId, url);
            if (logger.isDebugEnabled()) {
                logger.debug("Added URL to queue: url={}, sessionId={}", url, crawlerContext.sessionId);
            }

# https://github.com/apache/lucene-solr/blob/master/lucene/analysis/common/src/resources/org/apache/lucene/analysis/br/stopwords.txt
a
ainda
alem
ambas
ambos
antes
ao
aonde
aos
apos
aquele
aqueles
as
assim
com
como
contra
contudo
cuja
cujas
cujo
cujos
da
das
de
dela
dele
deles
demais
depois
desde
desta
deste
dispoe
dispoem
diversa
diversas
diversos

        assertTrue(fesenClient.prepareSearch("fess_crawler.queue")
                .setQuery(QueryBuilders.termQuery("sessionId", "id1"))
                .execute()
                .actionGet()
                .getHits()
                .getTotalHits()
                .value() > 0);
        assertTrue(fesenClient.prepareSearch("fess_crawler.queue")
                .setQuery(QueryBuilders.termQuery("sessionId", "id2"))

    }

    private RESPONSE response = null;

    private Throwable error = null;

    private final Promise promise = new Promise();

    private final Queue<Consumer<RESPONSE>> doneCallbacks = new LinkedBlockingQueue<>();

    private final Queue<Consumer<Throwable>> errorCallbacks = new LinkedBlockingQueue<>();

    private final CountDownLatch latch = new CountDownLatch(1);

    /**

    /**
     * Maximum size of the thumbnail generation task queue.
     */
    protected int thumbnailTaskQueueSize = 10000;

    /**
     * Number of tasks to process in bulk operations.
     */
    protected int thumbnailTaskBulkSize = 100;

    /**
     * Timeout in milliseconds for thumbnail task queue operations.
     */
    protected long thumbnailTaskQueueTimeout = 10 * 1000L;