Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 6 of 6 for storeChildUrls (0.6 sec)

  1. fess-crawler/src/test/java/org/codelibs/fess/crawler/CrawlerThreadTest.java

            final java.lang.reflect.Method method = CrawlerThread.class.getDeclaredMethod("storeChildUrls", Set.class, String.class, int.class);
            method.setAccessible(true);
            method.invoke(crawlerThread, childUrlList, "http://example.com/", 2);
    
            verify(urlQueueService, times(1)).offerAll(anyString(), any());
        }
    
        /**
         * Test storeChildUrls with depth exceeding maxDepth.
         */
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Mon Nov 24 03:59:47 UTC 2025
    - 18.3K bytes
    - Viewed (0)
  2. fess-crawler/src/main/java/org/codelibs/fess/crawler/processor/impl/DefaultResponseProcessor.java

                    if (crawlerContext.getMaxDepth() < 0 || depth <= crawlerContext.getMaxDepth()) {
                        // add and filter urls
                        storeChildUrls(crawlerContext, resultData.getChildUrlSet(), urlQueue.getUrl(), depth, resultData.getEncoding());
                    }
                } else if (crawlerContext.getMaxDepth() < 0 || urlQueue.getDepth() <= crawlerContext.getMaxDepth()) {
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Thu Aug 07 02:55:08 UTC 2025
    - 12.5K bytes
    - Viewed (0)
  3. fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java

     *   <li>Processing the response using a {@link ResponseProcessor} associated with a {@link Rule}.</li>
     *   <li>Extracting child URLs and adding them to the queue using {@link #storeChildUrls(Set, String, int)}
     *       or {@link #storeChildUrl(String, String, float, int)}.</li>
     *   <li>Handling exceptions that may occur during the crawling process.</li>
     * </ol>
     *
     * <p>
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Thu Aug 07 02:55:08 UTC 2025
    - 20.4K bytes
    - Viewed (0)
  4. fess-crawler/src/main/java/org/codelibs/fess/crawler/transformer/impl/HtmlTransformer.java

            try {
                // data
                storeData(responseData, resultData);
    
                if (isHtml(responseData) && !responseData.isNoFollow()) {
                    // urls
                    storeChildUrls(responseData, resultData);
                }
            } finally {
                xpathAPI.remove();
            }
    
            final Object redirectUrlObj = responseData.getMetaDataMap().get(LOCATION_HEADER);
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Sat Nov 29 07:42:33 UTC 2025
    - 30.5K bytes
    - Viewed (0)
  5. src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java

         */
        protected void storeChildUrlsToQueue(final UrlQueue<?> urlQueue, final Set<RequestData> childUrlSet) {
            if (childUrlSet != null) {
                // add an url
                try {
                    storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()),
                            urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
    Registered: Sat Dec 20 09:19:18 UTC 2025
    - Last Modified: Thu Dec 11 09:47:03 UTC 2025
    - 19.5K bytes
    - Viewed (0)
  6. src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java

                    }
                    if (noindex) {
                        logger.info("META(robots=noindex): {}", responseData.getUrl());
                        storeChildUrls(responseData, resultData);
                        throw new ChildUrlsException(resultData.getChildUrlSet(), "#processMetaRobots");
                    }
                    if (nofollow) {
    Registered: Sat Dec 20 09:19:18 UTC 2025
    - Last Modified: Fri Dec 12 13:58:40 UTC 2025
    - 54.6K bytes
    - Viewed (0)
Back to top