- Sort Score
- Result 10 results
- Languages All
Results 1 - 10 of 112 for Crawled (0.64 sec)
-
misc/linkcheck/linkcheck.go
defer mu.Unlock() if u, frag, ok := strings.Cut(url, "#"); ok { url = u if frag != "" { uf := urlFrag{url, frag} neededFrags[uf] = append(neededFrags[uf], sourceURL) } } if crawled[url] { return } crawled[url] = true wg.Add(1) go func() { urlq <- url }() } func addProblem(url, errmsg string) { msg := fmt.Sprintf("Error on %s: %s (from %s)", url, errmsg, linkSources[url])
Go - Registered: Tue Apr 30 11:13:12 GMT 2024 - Last Modified: Wed Oct 06 15:53:04 GMT 2021 - 3.9K bytes - Viewed (0) -
samples/crawler/src/main/java/okhttp3/sample/Crawler.java
System.out.println("Usage: Crawler <cache dir> <root>"); return; } int threadCount = 20; long cacheByteCount = 1024L * 1024L * 100L; Cache cache = new Cache(new File(args[0]), cacheByteCount); OkHttpClient client = new OkHttpClient.Builder() .cache(cache) .build(); Crawler crawler = new Crawler(client); crawler.queue.add(HttpUrl.get(args[1]));
Java - Registered: Fri May 03 11:42:14 GMT 2024 - Last Modified: Thu Aug 12 07:26:27 GMT 2021 - 4.6K bytes - Viewed (0) -
src/main/java/org/codelibs/fess/exec/Crawler.java
if (logger.isDebugEnabled()) { logger.debug("Crawler is stopped.", e); } else if (logger.isInfoEnabled()) { logger.info("Crawler is stopped."); } exitCode = Constants.EXIT_FAIL; } catch (final Throwable t) { logger.error("Crawler does not work correctly.", t); exitCode = Constants.EXIT_FAIL; } finally {
Java - Registered: Mon May 06 08:04:11 GMT 2024 - Last Modified: Thu Feb 22 01:37:57 GMT 2024 - 24.1K bytes - Viewed (2) -
src/main/resources/crawler_es+crawlerConfig.xml
"http://dbflute.org/meta/lastadi10.dtd"> <components namespace="fessCrawler"> <component name="crawlerConfig" class="org.codelibs.fess.crawler.util.FessCrawlerConfig"> <property name="queueIndex">"fess_crawler.queue"</property> <property name="queueShards">5</property> <property name="queueReplicas">1</property> <property name="dataIndex">"fess_crawler.data"</property> <property name="dataShards">5</property>
XML - Registered: Mon May 06 08:04:11 GMT 2024 - Last Modified: Fri Dec 03 13:06:40 GMT 2021 - 756 bytes - Viewed (0) -
src/main/resources/fess_label_de.properties
labels.crawling_info_delete_all_cancel=Abbrechen labels.crawling_info_thread_dump=Thread-Dump labels.crawling_info_CrawlerStartTime=Crawler Startzeit labels.crawling_info_CrawlerEndTime=Crawler Endzeit labels.crawling_info_CrawlerExecTime=Crawler Ausführungsdauer labels.crawling_info_CrawlerStatus=Crawler-Status labels.crawling_info_WebFsCrawlExecTime=Crawl Ausführungsdauer (Web/Dateisystem)
Properties - Registered: Mon May 06 08:04:11 GMT 2024 - Last Modified: Fri Mar 22 11:58:34 GMT 2024 - 42.8K bytes - Viewed (0) -
src/main/java/org/codelibs/fess/es/config/exentity/DataConfig.java
private static final String CRAWLER_WEB_PREFIX = "crawler.web."; private static final String CRAWLER_WEB_HEADER_PREFIX = CRAWLER_WEB_PREFIX + "header."; private static final String CRAWLER_WEB_AUTH = CRAWLER_WEB_PREFIX + "auth"; private static final String CRAWLER_USERAGENT = "crawler.useragent"; private static final String CRAWLER_PARAM_PREFIX = "crawler.param.";
Java - Registered: Mon May 06 08:04:11 GMT 2024 - Last Modified: Thu Feb 22 01:37:57 GMT 2024 - 18.6K bytes - Viewed (0) -
src/main/resources/fess_indices/fess_config.scheduled_job/scheduled_job.bulk
{"index":{"_index":"fess_config.scheduled_job","_id":"default_crawler"}} {"name":"Default Crawler","target":"all","cronExpression":"0 0 * * *","scriptType":"groovy","scriptData":"return container.getComponent(\"crawlJob\").logLevel(\"info\").gcLogging().execute(executor);","jobLogging":true,"crawler":true,"available":true,"sortOrder":1,"createdBy":"system","createdTime":0,"updatedBy":"system","updatedTime":0} {"index":{"_index":"fess_config.scheduled_job","_id":"suggest_indexer"}}
Plain Text - Registered: Mon May 06 08:04:11 GMT 2024 - Last Modified: Thu Dec 02 13:14:56 GMT 2021 - 4.2K bytes - Viewed (0) -
src/main/resources/fess_config.properties
crawler.ignore.robots.tags=false crawler.ignore.content.exception=true crawler.failure.url.status.codes=404 crawler.system.monitor.interval=60 crawler.hotthread.ignore_idle_threads=true crawler.hotthread.interval=500ms crawler.hotthread.snapshots=10 crawler.hotthread.threads=3 crawler.hotthread.timeout=30s crawler.hotthread.type=cpu crawler.metadata.content.excludes=resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*,X-TIKA.*
Properties - Registered: Mon May 06 08:04:11 GMT 2024 - Last Modified: Thu Apr 11 02:34:53 GMT 2024 - 30.6K bytes - Viewed (1) -
src/main/java/org/codelibs/fess/indexer/IndexUpdater.java
import org.codelibs.fess.Constants; import org.codelibs.fess.crawler.Crawler; import org.codelibs.fess.crawler.entity.AccessResult; import org.codelibs.fess.crawler.entity.AccessResultData; import org.codelibs.fess.crawler.entity.EsAccessResult; import org.codelibs.fess.crawler.entity.EsUrlQueue; import org.codelibs.fess.crawler.service.DataService; import org.codelibs.fess.crawler.service.UrlFilterService;
Java - Registered: Mon May 06 08:04:11 GMT 2024 - Last Modified: Thu Feb 22 01:37:57 GMT 2024 - 24.2K bytes - Viewed (0) -
src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java
import org.codelibs.fess.app.service.FailureUrlService; import org.codelibs.fess.crawler.builder.RequestDataBuilder; import org.codelibs.fess.crawler.client.CrawlerClient; import org.codelibs.fess.crawler.entity.RequestData; import org.codelibs.fess.crawler.entity.ResponseData; import org.codelibs.fess.crawler.entity.UrlQueue; import org.codelibs.fess.crawler.log.LogType; import org.codelibs.fess.es.client.SearchEngineClient;
Java - Registered: Mon May 06 08:04:11 GMT 2024 - Last Modified: Thu Feb 22 01:37:57 GMT 2024 - 14.6K bytes - Viewed (0)