Search Options

Results per page
Sort
Preferred Languages
Advance

Results 11 - 20 of 26 for UrlQueue (0.05 sec)

  1. fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueService.java

                return;
            }
            final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();
            urlQueue.setSessionId(sessionId);
            urlQueue.setUrl(url);
            urlQueue.setCreateTime(System.currentTimeMillis());
            urlQueue.setLastModified(0L);
            urlQueue.setDepth(0);
            urlQueue.setMethod(Constants.GET_METHOD);
            insert(urlQueue);
        }
    
        @Override
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Nov 07 04:44:10 UTC 2024
    - 13.4K bytes
    - Viewed (0)
  2. fess-crawler/src/main/java/org/codelibs/fess/crawler/interval/impl/HostIntervalController.java

         * delayBeforeProcessing()
         */
        @Override
        protected void delayBeforeProcessing() {
            final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
            if (urlQueue == null) {
                return;
            }
    
            final String url = urlQueue.getUrl();
            if (StringUtil.isBlank(url) || url.startsWith("file:")) {
                // not target
                return;
            }
    
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 3K bytes
    - Viewed (0)
  3. src/main/java/org/codelibs/fess/crawler/transformer/FessTransformer.java

                final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
                if (urlQueue != null) {
                    final String parentUrl = urlQueue.getParentUrl();
                    if (StringUtil.isNotEmpty(parentUrl)) {
                        final String sessionId = urlQueue.getSessionId();
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Fri Oct 11 21:11:58 UTC 2024
    - 9.5K bytes
    - Viewed (0)
  4. fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java

        protected float weight = 1.0f;
    
        /*
         * (non-Javadoc)
         *
         * @see org.codelibs.fess.crawler.entity.UrlQueue#getId()
         */
        @Override
        public IDTYPE getId() {
            return id;
        }
    
        /*
         * (non-Javadoc)
         *
         * @see org.codelibs.fess.crawler.entity.UrlQueue#setId(IDTYPE)
         */
        @Override
        public void setId(final IDTYPE id) {
            this.id = id;
        }
    
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Mon Nov 04 07:44:18 UTC 2024
    - 5.2K bytes
    - Viewed (0)
  5. fess-crawler-opensearch/src/test/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueServiceTest.java

        public void test_insert_update_deleteTx() {
            final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();
            urlQueue.setCreateTime(System.currentTimeMillis());
            urlQueue.setDepth(1);
            urlQueue.setMethod("GET");
            urlQueue.setSessionId("sessionId");
            urlQueue.setUrl("http://www.example.com/");
    
            urlQueueService.insert(urlQueue);
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Nov 07 04:44:10 UTC 2024
    - 5.1K bytes
    - Viewed (0)
  6. src/main/java/org/codelibs/fess/helper/CrawlerStatsHelper.java

            return buf;
        }
    
        protected String getUrl(final Object keyObj) {
            if (keyObj instanceof final UrlQueue<?> urlQueue) {
                return escapeValue(urlQueue.getUrl());
            }
            if (keyObj instanceof final StatsKeyObject statsKey) {
                return escapeValue(statsKey.getUrl());
            }
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Thu Feb 22 01:53:18 UTC 2024
    - 9.9K bytes
    - Viewed (0)
  7. fess-crawler-lasta/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java

                assertEquals(maxCount, dataService.getCount(sessionId2));
    
                UrlQueue urlQueue;
                while ((urlQueue = urlQueueService.poll(sessionId1)) != null) {
                    assertTrue(urlQueue.getUrl().startsWith(url1));
                }
                while ((urlQueue = urlQueueService.poll(sessionId2)) != null) {
                    assertTrue(urlQueue.getUrl().startsWith(url2));
                }
    
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:47:32 UTC 2024
    - 11.9K bytes
    - Viewed (0)
  8. src/main/java/org/codelibs/fess/ds/callback/FileListIndexUpdateCallbackImpl.java

                final long maxAccessCount = getMaxAccessCount(paramMap, dataMap);
                long counter = 0;
                final Deque<String> urlQueue = new LinkedList<>();
                final Set<String> processedUrls = new HashSet<>();
                urlQueue.offer(url);
                while (!urlQueue.isEmpty() && (maxAccessCount < 0 || counter < maxAccessCount)) {
                    counter++;
                    final Map<String, Object> localDataMap =
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Thu Feb 22 01:37:57 UTC 2024
    - 16.8K bytes
    - Viewed (0)
  9. fess-crawler/src/main/java/org/codelibs/fess/crawler/service/UrlQueueService.java

    import java.util.List;
    
    import org.codelibs.fess.crawler.entity.UrlQueue;
    
    /**
     * @author shinsuke
     *
     */
    public interface UrlQueueService<QUEUE extends UrlQueue<?>> {
    
        void updateSessionId(String oldSessionId, String newSessionId);
    
        void add(String sessionId, String url);
    
        void insert(QUEUE urlQueue);
    
        void delete(String sessionId);
    
        void deleteAll();
    
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 1.3K bytes
    - Viewed (0)
  10. fess-crawler-lasta/src/main/resources/crawler.xml

    	</component>
    
    	<!-- Entity -->
    	<component name="accessResult" class="org.codelibs.fess.crawler.entity.AccessResultImpl" instance="prototype" >
    	</component>
    	<component name="urlQueue" class="org.codelibs.fess.crawler.entity.UrlQueueImpl" instance="prototype" >
    	</component>
    
    	<!-- Service -->
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Tue Nov 28 13:40:25 UTC 2017
    - 1.7K bytes
    - Viewed (0)
Back to top