Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 10 of 15 for UrlQueue (0.05 sec)

  1. fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueue.java

     * governing permissions and limitations under the License.
     */
    package org.codelibs.fess.crawler.entity;
    
    /**
     * @author shinsuke
     *
     */
    public interface UrlQueue<IDTYPE> {
    
        IDTYPE getId();
    
        void setId(IDTYPE id);
    
        String getSessionId();
    
        void setSessionId(String sessionId);
    
        String getMethod();
    
        void setMethod(String method);
    
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Mon Nov 04 07:44:18 UTC 2024
    - 1.4K bytes
    - Viewed (0)
  2. src/main/java/org/codelibs/fess/helper/CrawlerLogHelper.java

            if (objs.length > 1 && objs[1] instanceof final UrlQueue<?> urlQueue) {
                ComponentUtil.getCrawlerStatsHelper().begin(urlQueue);
            }
        }
    
        @Override
        protected void processCleanupCrawling(final Object... objs) {
            super.processCleanupCrawling(objs);
            if (objs.length > 1 && objs[1] instanceof final UrlQueue<?> urlQueue) {
                ComponentUtil.getCrawlerStatsHelper().done(urlQueue);
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Fri Oct 11 21:11:58 UTC 2024
    - 7.3K bytes
    - Viewed (0)
  3. fess-crawler/src/main/java/org/codelibs/fess/crawler/service/impl/UrlQueueServiceImpl.java

                final UrlQueueImpl<Long> urlQueue = new UrlQueueImpl<>();
                urlQueue.setSessionId(sessionId);
                urlQueue.setMethod(Constants.GET_METHOD);
                urlQueue.setUrl(url);
                urlQueue.setUrl(url);
                urlQueue.setDepth(0);
                urlQueue.setCreateTime(SystemUtil.currentTimeMillis());
                urlQueueList.add(urlQueue);
            }
        }
    
        /*
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:47:32 UTC 2024
    - 7.5K bytes
    - Viewed (0)
  4. fess-crawler/src/main/java/org/codelibs/fess/crawler/processor/impl/DefaultResponseProcessor.java

                    if (logger.isDebugEnabled()) {
                        logger.debug("Canceled urlQueue: {}", urlQueue);
                    }
                    // cancel crawling
                    crawlerContext.decrementAndGetAccessCount();
                    final List<UrlQueue<?>> newUrlQueueList = new ArrayList<>();
                    newUrlQueueList.add(urlQueue);
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:47:32 UTC 2024
    - 9K bytes
    - Viewed (0)
  5. fess-crawler/src/main/java/org/codelibs/fess/crawler/util/CrawlingParameterUtil.java

            if (urlQueue == null) {
                URL_QUEUE_THREAD_LOCAL.remove();
            } else {
                URL_QUEUE_THREAD_LOCAL.set(urlQueue);
            }
        }
    
        public static CrawlerContext getCrawlerContext() {
            return ROBOT_CONTEXT_THREAD_LOCAL.get();
        }
    
        public static void setCrawlerContext(final CrawlerContext crawlerContext) {
            if (crawlerContext == null) {
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 2.9K bytes
    - Viewed (0)
  6. fess-crawler-opensearch/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java

                UrlQueue urlQueue;
                while ((urlQueue = urlQueueService.poll(sessionId1)) != null) {
                    assertTrue(urlQueue.getUrl() + "=>" + url1, urlQueue.getUrl().startsWith(url1));
                }
                while ((urlQueue = urlQueueService.poll(sessionId2)) != null) {
                    assertTrue(urlQueue.getUrl() + "=>" + url2, urlQueue.getUrl().startsWith(url2));
                }
    
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Nov 07 04:44:10 UTC 2024
    - 7.7K bytes
    - Viewed (0)
  7. fess-crawler/src/main/java/org/codelibs/fess/crawler/interval/impl/HostIntervalController.java

         * delayBeforeProcessing()
         */
        @Override
        protected void delayBeforeProcessing() {
            final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
            if (urlQueue == null) {
                return;
            }
    
            final String url = urlQueue.getUrl();
            if (StringUtil.isBlank(url) || url.startsWith("file:")) {
                // not target
                return;
            }
    
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 3K bytes
    - Viewed (0)
  8. src/main/java/org/codelibs/fess/crawler/transformer/FessTransformer.java

                final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
                if (urlQueue != null) {
                    final String parentUrl = urlQueue.getParentUrl();
                    if (StringUtil.isNotEmpty(parentUrl)) {
                        final String sessionId = urlQueue.getSessionId();
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Fri Oct 11 21:11:58 UTC 2024
    - 9.5K bytes
    - Viewed (0)
  9. fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java

        protected float weight = 1.0f;
    
        /*
         * (non-Javadoc)
         *
         * @see org.codelibs.fess.crawler.entity.UrlQueue#getId()
         */
        @Override
        public IDTYPE getId() {
            return id;
        }
    
        /*
         * (non-Javadoc)
         *
         * @see org.codelibs.fess.crawler.entity.UrlQueue#setId(IDTYPE)
         */
        @Override
        public void setId(final IDTYPE id) {
            this.id = id;
        }
    
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Mon Nov 04 07:44:18 UTC 2024
    - 5.2K bytes
    - Viewed (0)
  10. fess-crawler-opensearch/src/test/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueServiceTest.java

        public void test_insert_update_deleteTx() {
            final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();
            urlQueue.setCreateTime(System.currentTimeMillis());
            urlQueue.setDepth(1);
            urlQueue.setMethod("GET");
            urlQueue.setSessionId("sessionId");
            urlQueue.setUrl("http://www.example.com/");
    
            urlQueueService.insert(urlQueue);
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Nov 07 04:44:10 UTC 2024
    - 5.1K bytes
    - Viewed (0)
Back to top