Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 10 of 19 for UrlQueue (0.06 sec)

  1. fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueue.java

     */
    package org.codelibs.fess.crawler.entity;
    
    /**
     * The UrlQueue interface represents a queue of URLs to be processed by a web crawler.
     * It provides methods to get and set various properties of a URL queue entry.
     *
     * @param <IDTYPE> the type of the identifier for the URL queue entry
     */
    public interface UrlQueue<IDTYPE> {
    
        /**
         * Retrieves the unique identifier of the URL queue.
         *
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sat Mar 15 06:52:00 UTC 2025
    - 4.3K bytes
    - Viewed (0)
  2. fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java

         * Validates whether the URL queue entry is valid for crawling.
         * @param urlQueue The URL queue entry to validate.
         * @return true if valid, false otherwise.
         */
        protected boolean isValid(final UrlQueue<?> urlQueue) {
            if (urlQueue == null || StringUtil.isBlank(urlQueue.getUrl())
                    || crawlerContext.getMaxDepth() >= 0 && urlQueue.getDepth() > crawlerContext.getMaxDepth()) {
                return false;
            }
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Thu Aug 07 02:55:08 UTC 2025
    - 20.4K bytes
    - Viewed (0)
  3. fess-crawler/src/main/java/org/codelibs/fess/crawler/helper/impl/LogHelperImpl.java

         * @param objs the log objects (should contain UrlQueue and thread check count)
         */
        protected void processNoUrlInQueue(final Object... objs) {
            // CrawlerContext crawlerContext = (CrawlerContext) objs[0];
            final UrlQueue<?> urlQueue = (UrlQueue<?>) objs[1];
            final Integer threadCheckCount = (Integer) objs[2];
            if (logger.isDebugEnabled()) {
                if (urlQueue != null && urlQueue.getUrl() != null) {
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sun Jul 06 02:13:03 UTC 2025
    - 14K bytes
    - Viewed (0)
  4. fess-crawler/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java

                assertEquals(maxCount, dataService.getCount(sessionId2));
    
                UrlQueue urlQueue;
                while ((urlQueue = urlQueueService.poll(sessionId1)) != null) {
                    assertTrue(urlQueue.getUrl().startsWith(url1));
                }
                while ((urlQueue = urlQueueService.poll(sessionId2)) != null) {
                    assertTrue(urlQueue.getUrl().startsWith(url2));
                }
    
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sat Sep 06 04:15:37 UTC 2025
    - 19.1K bytes
    - Viewed (0)
  5. fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueService.java

                return;
            }
            final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();
            urlQueue.setSessionId(sessionId);
            urlQueue.setUrl(url);
            urlQueue.setCreateTime(System.currentTimeMillis());
            urlQueue.setLastModified(0L);
            urlQueue.setDepth(0);
            urlQueue.setMethod(Constants.GET_METHOD);
            insert(urlQueue);
        }
    
        /**
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Thu Aug 07 02:55:08 UTC 2025
    - 17K bytes
    - Viewed (1)
  6. fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java

    /**
     * Implementation of the {@link UrlQueue} interface.
     * This class represents a URL to be crawled, storing its ID, session ID,
     * HTTP method, URL, metadata, encoding, parent URL, depth, last modified time,
     * creation time, and weight.
     *
     * @param <IDTYPE> the type of the identifier for the URL queue entry
     */
    public class UrlQueueImpl<IDTYPE> implements UrlQueue<IDTYPE> {
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sun Jul 06 02:13:03 UTC 2025
    - 6.1K bytes
    - Viewed (0)
  7. fess-crawler-opensearch/src/test/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueServiceTest.java

        public void test_insert_update_deleteTx() {
            final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();
            urlQueue.setCreateTime(System.currentTimeMillis());
            urlQueue.setDepth(1);
            urlQueue.setMethod("GET");
            urlQueue.setSessionId("sessionId");
            urlQueue.setUrl("http://www.example.com/");
    
            urlQueueService.insert(urlQueue);
            assertTrue(fesenClient.prepareSearch("fess_crawler.queue")
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Thu Aug 07 02:55:08 UTC 2025
    - 5.8K bytes
    - Viewed (0)
  8. fess-crawler-lasta/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java

                assertEquals(maxCount, dataService.getCount(sessionId2));
    
                UrlQueue urlQueue;
                while ((urlQueue = urlQueueService.poll(sessionId1)) != null) {
                    assertTrue(urlQueue.getUrl().startsWith(url1));
                }
                while ((urlQueue = urlQueueService.poll(sessionId2)) != null) {
                    assertTrue(urlQueue.getUrl().startsWith(url2));
                }
    
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sat Sep 06 04:15:37 UTC 2025
    - 12.8K bytes
    - Viewed (0)
  9. fess-crawler/src/main/java/org/codelibs/fess/crawler/service/impl/UrlQueueServiceImpl.java

            synchronized (urlQueueList) {
                final UrlQueueImpl<Long> urlQueue = new UrlQueueImpl<>();
                urlQueue.setSessionId(sessionId);
                urlQueue.setMethod(Constants.GET_METHOD);
                urlQueue.setUrl(url);
                urlQueue.setDepth(0);
                urlQueue.setCreateTime(SystemUtil.currentTimeMillis());
                urlQueueList.add(urlQueue);
            }
        }
    
        /*
         * (non-Javadoc)
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sun Jul 06 02:13:03 UTC 2025
    - 9.3K bytes
    - Viewed (0)
  10. fess-crawler/src/main/java/org/codelibs/fess/crawler/service/UrlQueueService.java

    import java.util.List;
    
    import org.codelibs.fess.crawler.entity.UrlQueue;
    
    /**
     * Service interface for managing URL queues.
     * Provides methods for adding, retrieving, and managing URLs within a crawling session.
     *
     * @param <QUEUE> the type of URL queue
     */
    public interface UrlQueueService<QUEUE extends UrlQueue<?>> {
    
        /**
         * Updates the session ID.
         *
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sun Jul 06 02:13:03 UTC 2025
    - 2.6K bytes
    - Viewed (0)
Back to top