Search Options

Results per page
Sort
Preferred Languages
Advance

Results 11 - 20 of 33 for urlquery (0.17 sec)

  1. fess-crawler-es/src/test/java/org/codelibs/fess/crawler/service/impl/EsUrlQueueServiceTest.java

        public void test_insert_update_deleteTx() {
            final EsUrlQueue urlQueue = new EsUrlQueue();
            urlQueue.setCreateTime(System.currentTimeMillis());
            urlQueue.setDepth(1);
            urlQueue.setMethod("GET");
            urlQueue.setSessionId("sessionId");
            urlQueue.setUrl("http://www.example.com/");
    
            urlQueueService.insert(urlQueue);
    Registered: Wed Jun 12 15:17:51 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 5.1K bytes
    - Viewed (0)
  2. fess-crawler/src/main/java/org/codelibs/fess/crawler/service/impl/UrlQueueServiceImpl.java

                final UrlQueueImpl<Long> urlQueue = new UrlQueueImpl<>();
                urlQueue.setSessionId(sessionId);
                urlQueue.setMethod(Constants.GET_METHOD);
                urlQueue.setUrl(url);
                urlQueue.setUrl(url);
                urlQueue.setDepth(0);
                urlQueue.setCreateTime(SystemUtil.currentTimeMillis());
                urlQueueList.add(urlQueue);
            }
        }
    
        /*
    Registered: Wed Jun 12 15:17:51 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 7.5K bytes
    - Viewed (0)
  3. fess-crawler/src/main/java/org/codelibs/fess/crawler/processor/impl/DefaultResponseProcessor.java

                    if (logger.isDebugEnabled()) {
                        logger.debug("Canceled urlQueue: {}", urlQueue);
                    }
                    // cancel crawling
                    crawlerContext.decrementAndGetAccessCount();
                    final List<UrlQueue<?>> newUrlQueueList = new ArrayList<>();
                    newUrlQueueList.add(urlQueue);
    Registered: Wed Jun 12 15:17:51 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 9K bytes
    - Viewed (0)
  4. fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java

        protected Long createTime;
    
        /*
         * (non-Javadoc)
         *
         * @see org.codelibs.fess.crawler.entity.UrlQueue#getId()
         */
        @Override
        public IDTYPE getId() {
            return id;
        }
    
        /*
         * (non-Javadoc)
         *
         * @see org.codelibs.fess.crawler.entity.UrlQueue#setId(IDTYPE)
         */
        @Override
        public void setId(final IDTYPE id) {
            this.id = id;
        }
    
    Registered: Wed Jun 12 15:17:51 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 5K bytes
    - Viewed (0)
  5. src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java

        protected void storeChildUrlsToQueue(final UrlQueue<?> urlQueue, final Set<RequestData> childUrlSet) {
            if (childUrlSet != null) {
                // add an url
                try {
                    storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()),
                            urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
    Registered: Wed Jun 12 13:08:18 UTC 2024
    - Last Modified: Thu Feb 22 01:37:57 UTC 2024
    - 14.6K bytes
    - Viewed (0)
  6. fess-crawler/src/main/java/org/codelibs/fess/crawler/util/CrawlingParameterUtil.java

            if (urlQueue == null) {
                URL_QUEUE_THREAD_LOCAL.remove();
            } else {
                URL_QUEUE_THREAD_LOCAL.set(urlQueue);
            }
        }
    
        public static CrawlerContext getCrawlerContext() {
            return ROBOT_CONTEXT_THREAD_LOCAL.get();
        }
    
        public static void setCrawlerContext(final CrawlerContext crawlerContext) {
            if (crawlerContext == null) {
    Registered: Wed Jun 12 15:17:51 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 2.9K bytes
    - Viewed (0)
  7. fess-crawler/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java

                assertEquals(maxCount, dataService.getCount(sessionId2));
    
                UrlQueue urlQueue;
                while ((urlQueue = urlQueueService.poll(sessionId1)) != null) {
                    assertTrue(urlQueue.getUrl().startsWith(url1));
                }
                while ((urlQueue = urlQueueService.poll(sessionId2)) != null) {
                    assertTrue(urlQueue.getUrl().startsWith(url2));
                }
    
    Registered: Wed Jun 12 15:17:51 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 18K bytes
    - Viewed (0)
  8. fess-crawler/src/main/java/org/codelibs/fess/crawler/service/UrlQueueService.java

    import java.util.List;
    
    import org.codelibs.fess.crawler.entity.UrlQueue;
    
    /**
     * @author shinsuke
     *
     */
    public interface UrlQueueService<QUEUE extends UrlQueue<?>> {
    
        void updateSessionId(String oldSessionId, String newSessionId);
    
        void add(String sessionId, String url);
    
        void insert(QUEUE urlQueue);
    
        void delete(String sessionId);
    
        void deleteAll();
    
    Registered: Wed Jun 12 15:17:51 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 1.3K bytes
    - Viewed (0)
  9. fess-crawler-es/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java

                UrlQueue urlQueue;
                while ((urlQueue = urlQueueService.poll(sessionId1)) != null) {
                    assertTrue(urlQueue.getUrl() + "=>" + url1, urlQueue.getUrl().startsWith(url1));
                }
                while ((urlQueue = urlQueueService.poll(sessionId2)) != null) {
                    assertTrue(urlQueue.getUrl() + "=>" + url2, urlQueue.getUrl().startsWith(url2));
                }
    
    Registered: Wed Jun 12 15:17:51 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 7.7K bytes
    - Viewed (0)
  10. fess-crawler/src/main/java/org/codelibs/fess/crawler/interval/impl/HostIntervalController.java

         * delayBeforeProcessing()
         */
        @Override
        protected void delayBeforeProcessing() {
            final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
            if (urlQueue == null) {
                return;
            }
    
            final String url = urlQueue.getUrl();
            if (StringUtil.isBlank(url) || url.startsWith("file:")) {
                // not target
                return;
            }
    
    Registered: Wed Jun 12 15:17:51 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 3K bytes
    - Viewed (0)
Back to top