- Sort Score
- Result 10 results
- Languages All
Results 1 - 10 of 16 for CrawlerContext (0.6 sec)
-
fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerContext.java
* </p> * * <p> * It also provides thread-local storage for sitemaps, allowing each thread to have its own set of sitemaps. * </p> */ public class CrawlerContext { /** * Constructs a new CrawlerContext. */ public CrawlerContext() { // Default constructor } /** * Session identifier for the crawling session. */ protected String sessionId; /**
Registered: Sun Sep 21 03:50:09 UTC 2025 - Last Modified: Sun Jul 06 02:13:03 UTC 2025 - 8.9K bytes - Viewed (0) -
src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java
// num of threads final CrawlerContext crawlerContext = crawler.getCrawlerContext(); final int numOfThread = webConfig.getNumOfThread() != null ? webConfig.getNumOfThread() : Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB; crawlerContext.setNumOfThread(numOfThread); // depth
Registered: Thu Sep 04 12:52:25 UTC 2025 - Last Modified: Thu Jul 17 08:28:31 UTC 2025 - 24.9K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/helper/impl/LogHelperImpl.java
// CrawlerContext crawlerContext = (CrawlerContext) objs[0]; } /** * Processes no URL in queue log events. * * @param objs the log objects (should contain UrlQueue and thread check count) */ protected void processNoUrlInQueue(final Object... objs) { // CrawlerContext crawlerContext = (CrawlerContext) objs[0];
Registered: Sun Sep 21 03:50:09 UTC 2025 - Last Modified: Sun Jul 06 02:13:03 UTC 2025 - 14K bytes - Viewed (0) -
fess-crawler/src/test/java/org/codelibs/fess/crawler/CrawlerContextTest.java
assertEquals(10, crawlerContext.getNumOfThread()); crawlerContext.setNumOfThread(5); assertEquals(5, crawlerContext.getNumOfThread()); crawlerContext.setNumOfThread(0); assertEquals(0, crawlerContext.getNumOfThread()); crawlerContext.setNumOfThread(-1); assertEquals(-1, crawlerContext.getNumOfThread()); crawlerContext.setNumOfThread(Integer.MAX_VALUE);
Registered: Sun Sep 21 03:50:09 UTC 2025 - Last Modified: Sat Sep 06 04:15:37 UTC 2025 - 25.6K bytes - Viewed (0) -
fess-crawler/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java
fileTransformer.setPath(file.getAbsolutePath()); crawler.addUrl(url); crawler.crawlerContext.setMaxThreadCheckCount(3); // crawler.crawlerContext.setMaxAccessCount(maxCount); crawler.crawlerContext.setNumOfThread(numOfThread); crawler.crawlerContext.setMaxDepth(1); crawler.urlFilter.addInclude(url + ".*"); final String sessionId = crawler.execute();
Registered: Sun Sep 21 03:50:09 UTC 2025 - Last Modified: Sat Sep 06 04:15:37 UTC 2025 - 19.1K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/CrawlerThread.java
this.clientFactory = clientFactory; } /** * Sets the crawler context. * @param crawlerContext The CrawlerContext instance. */ public void setCrawlerContext(final CrawlerContext crawlerContext) { this.crawlerContext = crawlerContext; }
Registered: Sun Sep 21 03:50:09 UTC 2025 - Last Modified: Thu Aug 07 02:55:08 UTC 2025 - 20.4K bytes - Viewed (0) -
src/main/java/org/codelibs/fess/helper/CrawlerLogHelper.java
@Override protected void processCrawlingException(final Object... objs) { try { final CrawlerContext crawlerContext = (CrawlerContext) objs[0]; final UrlQueue<?> urlQueue = (UrlQueue<?>) objs[1]; final Throwable e = (Throwable) objs[2]; storeFailureUrl(crawlerContext, urlQueue, e.getClass().getCanonicalName(), e); } catch (final ContainerNotAvailableException e) {
Registered: Thu Sep 04 12:52:25 UTC 2025 - Last Modified: Thu Jul 17 08:28:31 UTC 2025 - 8.2K bytes - Viewed (0) -
fess-crawler-lasta/src/test/java/org/codelibs/fess/crawler/CrawlerTest.java
assertNotSame(sessionId1, sessionId2); assertNotSame(crawler1.crawlerContext, crawler2.crawlerContext); // Wait for both crawlers to start with polling long startTime = System.currentTimeMillis(); while ((crawler1.crawlerContext.getStatus() != CrawlerStatus.RUNNING || crawler2.crawlerContext.getStatus() != CrawlerStatus.RUNNING) && System.currentTimeMillis() - startTime < 5000) {
Registered: Sun Sep 21 03:50:09 UTC 2025 - Last Modified: Sat Sep 06 04:15:37 UTC 2025 - 12.8K bytes - Viewed (0) -
fess-crawler/src/main/java/org/codelibs/fess/crawler/Crawler.java
// context crawlerContext.urlFilter = urlFilter; crawlerContext.ruleManager = ruleManager; crawlerContext.intervalController = intervalController; urlFilter.init(crawlerContext.sessionId); crawlerThreadGroup = new ThreadGroup("Crawler-" + crawlerContext.sessionId); final Thread[] threads = new Thread[crawlerContext.getNumOfThread()];
Registered: Sun Sep 21 03:50:09 UTC 2025 - Last Modified: Sun Jul 06 02:13:03 UTC 2025 - 14K bytes - Viewed (0) -
README.md
```java // Set maximum number of URLs to crawl crawler.crawlerContext.setMaxAccessCount(1000); // Set number of crawler threads crawler.crawlerContext.setNumOfThread(10); // Set maximum crawl depth crawler.crawlerContext.setMaxDepth(3); // Set request interval (politeness) crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second ``` ### URL Filtering ```java
Registered: Sun Sep 21 03:50:09 UTC 2025 - Last Modified: Sun Aug 31 05:32:52 UTC 2025 - 15.3K bytes - Viewed (0)