- Sort Score
- Result 10 results
- Languages All
Results 1 - 6 of 6 for storeChildUrls (0.6 sec)
-
fess-crawler/src/test/java/org/codelibs/fess/crawler/CrawlerThreadTest.java
final java.lang.reflect.Method method = CrawlerThread.class.getDeclaredMethod("storeChildUrls", Set.class, String.class, int.class); method.setAccessible(true); method.invoke(crawlerThread, childUrlList, "http://example.com/", 2); verify(urlQueueService, times(1)).offerAll(anyString(), any()); } /** * Test storeChildUrls with depth exceeding maxDepth. */
Registered: Sat Dec 20 11:21:39 UTC 2025 - Last Modified: Mon Nov 24 03:59:47 UTC 2025 - 18.3K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/processor/impl/DefaultResponseProcessor.java
if (crawlerContext.getMaxDepth() < 0 || depth <= crawlerContext.getMaxDepth()) { // add and filter urls storeChildUrls(crawlerContext, resultData.getChildUrlSet(), urlQueue.getUrl(), depth, resultData.getEncoding()); } } else if (crawlerContext.getMaxDepth() < 0 || urlQueue.getDepth() <= crawlerContext.getMaxDepth()) {
Registered: Sat Dec 20 11:21:39 UTC 2025 - Last Modified: Thu Aug 07 02:55:08 UTC 2025 - 12.5K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java
* <li>Processing the response using a {@link ResponseProcessor} associated with a {@link Rule}.</li> * <li>Extracting child URLs and adding them to the queue using {@link #storeChildUrls(Set, String, int)} * or {@link #storeChildUrl(String, String, float, int)}.</li> * <li>Handling exceptions that may occur during the crawling process.</li> * </ol> * * <p>
Registered: Sat Dec 20 11:21:39 UTC 2025 - Last Modified: Thu Aug 07 02:55:08 UTC 2025 - 20.4K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/transformer/impl/HtmlTransformer.java
try { // data storeData(responseData, resultData); if (isHtml(responseData) && !responseData.isNoFollow()) { // urls storeChildUrls(responseData, resultData); } } finally { xpathAPI.remove(); } final Object redirectUrlObj = responseData.getMetaDataMap().get(LOCATION_HEADER);Registered: Sat Dec 20 11:21:39 UTC 2025 - Last Modified: Sat Nov 29 07:42:33 UTC 2025 - 30.5K bytes - Viewed (0) -
src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java
*/ protected void storeChildUrlsToQueue(final UrlQueue<?> urlQueue, final Set<RequestData> childUrlSet) { if (childUrlSet != null) { // add an url try { storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()), urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);Registered: Sat Dec 20 09:19:18 UTC 2025 - Last Modified: Thu Dec 11 09:47:03 UTC 2025 - 19.5K bytes - Viewed (0) -
src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java
} if (noindex) { logger.info("META(robots=noindex): {}", responseData.getUrl()); storeChildUrls(responseData, resultData); throw new ChildUrlsException(resultData.getChildUrlSet(), "#processMetaRobots"); } if (nofollow) {Registered: Sat Dec 20 09:19:18 UTC 2025 - Last Modified: Fri Dec 12 13:58:40 UTC 2025 - 54.6K bytes - Viewed (0)