Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 10 of 11 for UrlQueue (0.2 sec)

  2. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/helper/impl/LogHelperImpl.java 

            final UrlQueue<?> urlQueue = (UrlQueue<?>) objs[1];
        final Throwable e = (Throwable) objs[2];
        logger.error("Crawling Exception at " + urlQueue.getUrl(), e);
    }

    protected void processCrawlingAccessException(final Object... objs) {
        // CrawlerContext crawlerContext = (CrawlerContext) objs[0];
        final UrlQueue<?> urlQueue = (UrlQueue<?>) objs[1];
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 10.3K bytes
    - Viewed (0)
  3. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java 

            }
    }

    protected boolean isValid(final UrlQueue<?> urlQueue) {
        if (urlQueue == null || StringUtil.isBlank(urlQueue.getUrl())
                || crawlerContext.getMaxDepth() >= 0 && urlQueue.getDepth() > crawlerContext.getMaxDepth()) {
            return false;
        }

        // url filter
        if (crawlerContext.urlFilter.match(urlQueue.getUrl())) {
            return true;
        }
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Sat Oct 12 01:41:37 UTC 2024
    - 15.5K bytes
    - Viewed (0)
  4. github.com/codelibs/fess-crawler

    fess-crawler/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java 

                assertEquals(maxCount, dataService.getCount(sessionId2));

            UrlQueue urlQueue;
            while ((urlQueue = urlQueueService.poll(sessionId1)) != null) {
                assertTrue(urlQueue.getUrl().startsWith(url1));
            }
            while ((urlQueue = urlQueueService.poll(sessionId2)) != null) {
                assertTrue(urlQueue.getUrl().startsWith(url2));
            }
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:36:27 UTC 2024
    - 18K bytes
    - Viewed (0)
  5. github.com/codelibs/fess

    src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java 

        protected void storeChildUrlsToQueue(final UrlQueue<?> urlQueue, final Set<RequestData> childUrlSet) {
        if (childUrlSet != null) {
            // add an url
            try {
                storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()),
                        urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Thu Feb 22 01:37:57 UTC 2024
    - 14.6K bytes
    - Viewed (0)
  6. github.com/codelibs/fess-crawler

    fess-crawler-opensearch/src/main/java/org/codelibs/fess/crawler/service/impl/OpenSearchUrlQueueService.java 

                return;
        }
        final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();
        urlQueue.setSessionId(sessionId);
        urlQueue.setUrl(url);
        urlQueue.setCreateTime(System.currentTimeMillis());
        urlQueue.setLastModified(0L);
        urlQueue.setDepth(0);
        urlQueue.setMethod(Constants.GET_METHOD);
        insert(urlQueue);
    }

    @Override
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Nov 07 04:44:10 UTC 2024
    - 13.4K bytes
    - Viewed (0)
  7. github.com/codelibs/fess

    src/main/java/org/codelibs/fess/helper/CrawlerStatsHelper.java 

            return buf;
    }

    protected String getUrl(final Object keyObj) {
        if (keyObj instanceof final UrlQueue<?> urlQueue) {
            return escapeValue(urlQueue.getUrl());
        }
        if (keyObj instanceof final StatsKeyObject statsKey) {
            return escapeValue(statsKey.getUrl());
        }
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Thu Feb 22 01:53:18 UTC 2024
    - 9.9K bytes
    - Viewed (0)
  8. github.com/codelibs/fess-crawler

    fess-crawler-lasta/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java 

                assertEquals(maxCount, dataService.getCount(sessionId2));

            UrlQueue urlQueue;
            while ((urlQueue = urlQueueService.poll(sessionId1)) != null) {
                assertTrue(urlQueue.getUrl().startsWith(url1));
            }
            while ((urlQueue = urlQueueService.poll(sessionId2)) != null) {
                assertTrue(urlQueue.getUrl().startsWith(url2));
            }
    Registered: Sun Nov 10 03:50:12 UTC 2024
    - Last Modified: Thu Feb 22 01:47:32 UTC 2024
    - 11.9K bytes
    - Viewed (0)
  9. github.com/codelibs/fess

    src/main/java/org/codelibs/fess/ds/callback/FileListIndexUpdateCallbackImpl.java 

                final long maxAccessCount = getMaxAccessCount(paramMap, dataMap);
            long counter = 0;
            final Deque<String> urlQueue = new LinkedList<>();
            final Set<String> processedUrls = new HashSet<>();
            urlQueue.offer(url);
            while (!urlQueue.isEmpty() && (maxAccessCount < 0 || counter < maxAccessCount)) {
                counter++;
                final Map<String, Object> localDataMap =
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Thu Feb 22 01:37:57 UTC 2024
    - 16.8K bytes
    - Viewed (0)
  10. github.com/codelibs/fess

    src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java 

            String urlEncoding;
        final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
        if (urlQueue != null && urlQueue.getEncoding() != null) {
            urlEncoding = urlQueue.getEncoding();
        } else {
            urlEncoding = responseData.getCharSet();
        }

        // cid
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Fri Oct 11 21:11:58 UTC 2024
    - 23.6K bytes
    - Viewed (0)
  11. github.com/codelibs/fess

    src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java 

            final Map<String, String> xpathConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.XPATH);

        String urlEncoding;
        final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
        if (urlQueue != null && urlQueue.getEncoding() != null) {
            urlEncoding = urlQueue.getEncoding();
        } else {
            urlEncoding = responseData.getCharSet();
        }

        // cid
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Thu Oct 24 13:01:38 UTC 2024
    - 42.9K bytes
    - Viewed (0)
Back to top