- Sort Score
- Result 10 results
- Languages All
Results 11 - 20 of 37 for urlquery (0.2 sec)
-
fess-crawler/src/main/java/org/codelibs/fess/crawler/helper/impl/LogHelperImpl.java
final UrlQueue<?> urlQueue = (UrlQueue<?>) objs[1]; final Throwable e = (Throwable) objs[2]; logger.error("Crawling Exception at " + urlQueue.getUrl(), e); } protected void processCrawlingAccessException(final Object... objs) { // CrawlerContext crawlerContext = (CrawlerContext) objs[0]; final UrlQueue<?> urlQueue = (UrlQueue<?>) objs[1];
Registered: Wed Jun 12 15:17:51 UTC 2024 - Last Modified: Thu Feb 22 01:36:27 UTC 2024 - 10.3K bytes - Viewed (1) -
fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/EsUrlQueueService.java
return; } final EsUrlQueue urlQueue = new EsUrlQueue(); urlQueue.setSessionId(sessionId); urlQueue.setUrl(url); urlQueue.setCreateTime(System.currentTimeMillis()); urlQueue.setLastModified(0L); urlQueue.setDepth(0); urlQueue.setMethod(Constants.GET_METHOD); insert(urlQueue); } @Override
Registered: Wed Jun 12 15:17:51 UTC 2024 - Last Modified: Thu Feb 22 01:36:27 UTC 2024 - 12.9K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java
} } protected boolean isValid(final UrlQueue<?> urlQueue) { if ((urlQueue == null) || StringUtil.isBlank(urlQueue.getUrl()) || (crawlerContext.getMaxDepth() >= 0 && urlQueue.getDepth() > crawlerContext.getMaxDepth())) { return false; } // url filter if (crawlerContext.urlFilter.match(urlQueue.getUrl())) { return true; }
Registered: Wed Jun 12 15:17:51 UTC 2024 - Last Modified: Thu Feb 22 01:36:27 UTC 2024 - 15.5K bytes - Viewed (0) -
fess-crawler-es/src/test/java/org/codelibs/fess/crawler/service/impl/EsUrlQueueServiceTest.java
public void test_insert_update_deleteTx() { final EsUrlQueue urlQueue = new EsUrlQueue(); urlQueue.setCreateTime(System.currentTimeMillis()); urlQueue.setDepth(1); urlQueue.setMethod("GET"); urlQueue.setSessionId("sessionId"); urlQueue.setUrl("http://www.example.com/"); urlQueueService.insert(urlQueue);
Registered: Wed Jun 12 15:17:51 UTC 2024 - Last Modified: Thu Feb 22 01:36:27 UTC 2024 - 5.1K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/service/impl/UrlQueueServiceImpl.java
final UrlQueueImpl<Long> urlQueue = new UrlQueueImpl<>(); urlQueue.setSessionId(sessionId); urlQueue.setMethod(Constants.GET_METHOD); urlQueue.setUrl(url); urlQueue.setUrl(url); urlQueue.setDepth(0); urlQueue.setCreateTime(SystemUtil.currentTimeMillis()); urlQueueList.add(urlQueue); } } /*
Registered: Wed Jun 12 15:17:51 UTC 2024 - Last Modified: Thu Feb 22 01:36:27 UTC 2024 - 7.5K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/processor/impl/DefaultResponseProcessor.java
if (logger.isDebugEnabled()) { logger.debug("Canceled urlQueue: {}", urlQueue); } // cancel crawling crawlerContext.decrementAndGetAccessCount(); final List<UrlQueue<?>> newUrlQueueList = new ArrayList<>(); newUrlQueueList.add(urlQueue);
Registered: Wed Jun 12 15:17:51 UTC 2024 - Last Modified: Thu Feb 22 01:36:27 UTC 2024 - 9K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java
protected Long createTime; /* * (non-Javadoc) * * @see org.codelibs.fess.crawler.entity.UrlQueue#getId() */ @Override public IDTYPE getId() { return id; } /* * (non-Javadoc) * * @see org.codelibs.fess.crawler.entity.UrlQueue#setId(IDTYPE) */ @Override public void setId(final IDTYPE id) { this.id = id; }
Registered: Wed Jun 12 15:17:51 UTC 2024 - Last Modified: Thu Feb 22 01:36:27 UTC 2024 - 5K bytes - Viewed (0) -
src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java
protected void storeChildUrlsToQueue(final UrlQueue<?> urlQueue, final Set<RequestData> childUrlSet) { if (childUrlSet != null) { // add an url try { storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()), urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
Registered: Wed Jun 12 13:08:18 UTC 2024 - Last Modified: Thu Feb 22 01:37:57 UTC 2024 - 14.6K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/util/CrawlingParameterUtil.java
if (urlQueue == null) { URL_QUEUE_THREAD_LOCAL.remove(); } else { URL_QUEUE_THREAD_LOCAL.set(urlQueue); } } public static CrawlerContext getCrawlerContext() { return ROBOT_CONTEXT_THREAD_LOCAL.get(); } public static void setCrawlerContext(final CrawlerContext crawlerContext) { if (crawlerContext == null) {
Registered: Wed Jun 12 15:17:51 UTC 2024 - Last Modified: Thu Feb 22 01:36:27 UTC 2024 - 2.9K bytes - Viewed (0) -
fess-crawler/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java
assertEquals(maxCount, dataService.getCount(sessionId2)); UrlQueue urlQueue; while ((urlQueue = urlQueueService.poll(sessionId1)) != null) { assertTrue(urlQueue.getUrl().startsWith(url1)); } while ((urlQueue = urlQueueService.poll(sessionId2)) != null) { assertTrue(urlQueue.getUrl().startsWith(url2)); }
Registered: Wed Jun 12 15:17:51 UTC 2024 - Last Modified: Thu Feb 22 01:36:27 UTC 2024 - 18K bytes - Viewed (0)