and left off when they liked, so that it was not easy to know
when the race was over.  However, when they had been running half
an hour or so, and were quite dry again, the Dodo suddenly called
out `The race is over!' and they all crowded round it, panting,
and asking, `But who has won?'

  This question the Dodo could not answer without a great deal of
thought, and it sat for a long time with one finger pressed upon

labels.crawling_info_thread_dump=Thread-Dump
labels.crawling_info_CrawlerStartTime=Crawler Startzeit
labels.crawling_info_CrawlerEndTime=Crawler Endzeit
labels.crawling_info_CrawlerExecTime=Crawler Ausführungsdauer
labels.crawling_info_CrawlerStatus=Crawler-Status
labels.crawling_info_WebFsCrawlExecTime=Crawl Ausführungsdauer (Web/Dateisystem)
labels.crawling_info_WebFsCrawlStartTime=Crawl Startzeit (Web/Dateisystem)

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.fess.crawler.entity.Sitemap#getLoc()
     */
    @Override
    public String getLoc() {
        return loc;
    }

    public void setLoc(final String loc) {
        this.loc = loc;
    }

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.fess.crawler.entity.Sitemap#getLastmod()
     */
    @Override

import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.RobotsTxt;
import org.codelibs.fess.crawler.entity.RobotsTxt.Directive;
import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.CrawlerStatus;
import org.codelibs.fess.crawler.interval.FessIntervalController;
import org.codelibs.fess.crawler.service.impl.EsDataService;
import org.codelibs.fess.crawler.service.impl.EsUrlFilterService;
import org.codelibs.fess.crawler.service.impl.EsUrlQueueService;

* [LastaFlute](https://github.com/lastaflute/lastaflute "LastaFlute"): Web-Framework
* [Lasta Job](https://github.com/lastaflute/lasta-job "Lasta Job"): Job-Scheduler
* [Fess Crawler](https://github.com/codelibs/fess-crawler "Fess Crawler"): Web-Crawler

labels.crawling_info_thread_dump=Thread Dump
labels.crawling_info_CrawlerStartTime=Crawler start time
labels.crawling_info_CrawlerEndTime=Crawler end time
labels.crawling_info_CrawlerExecTime=Crawler exec time
labels.crawling_info_CrawlerStatus=Crawler status
labels.crawling_info_WebFsCrawlExecTime=Crawl exec time (Web/File system)
labels.crawling_info_WebFsCrawlStartTime=Crawl start time (Web/File system)

import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.AbstractCrawlerClient;
import org.codelibs.fess.crawler.client.AccessTimeoutTarget;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;

 */
package org.codelibs.fess.crawler.util;

import java.util.function.Consumer;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathEvaluationResult;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import javax.xml.xpath.XPathNodes;

import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.w3c.dom.Node;