defer mu.Unlock()
	if u, frag, ok := strings.Cut(url, "#"); ok {
		url = u
		if frag != "" {
			uf := urlFrag{url, frag}
			neededFrags[uf] = append(neededFrags[uf], sourceURL)
		}
	}
	if crawled[url] {
		return
	}
	crawled[url] = true

	wg.Add(1)
	go func() {
		urlq <- url
	}()
}

func addProblem(url, errmsg string) {
	msg := fmt.Sprintf("Error on %s: %s (from %s)", url, errmsg, linkSources[url])

      System.out.println("Usage: Crawler <cache dir> <root>");
      return;
    }

    int threadCount = 20;
    long cacheByteCount = 1024L * 1024L * 100L;

    Cache cache = new Cache(new File(args[0]), cacheByteCount);
    OkHttpClient client = new OkHttpClient.Builder()
        .cache(cache)
        .build();

    Crawler crawler = new Crawler(client);
    crawler.queue.add(HttpUrl.get(args[1]));

            if (logger.isDebugEnabled()) {
                logger.debug("Crawler is stopped.", e);
            } else if (logger.isInfoEnabled()) {
                logger.info("Crawler is stopped.");
            }
            exitCode = Constants.EXIT_FAIL;
        } catch (final Throwable t) {
            logger.error("Crawler does not work correctly.", t);
            exitCode = Constants.EXIT_FAIL;
        } finally {

/*
 [Crawler Notification]
 Crawler notification mail.
*/
subject: [FESS] Crawler completed: /*pmb.hostname*/
>>>
--- Server Info ---
Host Name: /*pmb.hostname:orElse('Unknown')*/
Job Name: /*pmb.jobname:orElse('Unknown')*/

--- Web/FileSystem Crawler ---
Start Time: /*pmb.webFsCrawlStartTime:orElse('-')*/
End Time:   /*pmb.webFsCrawlEndTime:orElse('-')*/
Exec Time:  /*pmb.webFsCrawlExecTime:orElse('-')*/ ms

--- Web/FileSystem Indexer ---

	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<component name="crawlerConfig" class="org.codelibs.fess.crawler.util.FessCrawlerConfig">
		<property name="queueIndex">"fess_crawler.queue"</property>
		<property name="queueShards">5</property>
		<property name="queueReplicas">1</property>
		<property name="dataIndex">"fess_crawler.data"</property>
		<property name="dataShards">5</property>

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN" 
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<!-- Crawler Thread -->
	<component name="crawlerThread" class="org.codelibs.fess.crawler.FessCrawlerThread" instance="prototype" >
	</component>

  This time Alice waited patiently until it chose to speak again.
In a minute or two the Caterpillar took the hookah out of its
mouth and yawned once or twice, and shook itself.  Then it got
down off the mushroom, and crawled away in the grass, merely
remarking as it went, `One side will make you grow taller, and
the other side will make you grow shorter.'

  `One side of WHAT?  The other side of WHAT?' thought Alice to
herself.

  This time Alice waited patiently until it chose to speak again.
In a minute or two the Caterpillar took the hookah out of its
mouth and yawned once or twice, and shook itself.  Then it got
down off the mushroom, and crawled away in the grass, merely
remarking as it went, `One side will make you grow taller, and
the other side will make you grow shorter.'

  `One side of WHAT?  The other side of WHAT?' thought Alice to
herself.

labels.crawling_info_delete_all_cancel=Abbrechen
labels.crawling_info_thread_dump=Thread-Dump
labels.crawling_info_CrawlerStartTime=Crawler Startzeit
labels.crawling_info_CrawlerEndTime=Crawler Endzeit
labels.crawling_info_CrawlerExecTime=Crawler Ausführungsdauer
labels.crawling_info_CrawlerStatus=Crawler-Status
labels.crawling_info_WebFsCrawlExecTime=Crawl Ausführungsdauer (Web/Dateisystem)

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.EsAccessResult;
import org.codelibs.fess.crawler.entity.EsUrlQueue;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;