labels.crawling_info_thread_dump=Vidage de thread
labels.crawling_info_CrawlerStartTime=Heure de début du robot d'exploration
labels.crawling_info_CrawlerEndTime=Heure de fin du robot d'exploration
labels.crawling_info_CrawlerExecTime=Temps d'exécution du robot d'exploration
labels.crawling_info_CrawlerStatus=Statut du robot d'exploration
labels.crawling_info_WebFsCrawlExecTime=Temps d'exécution de l'exploration (Web/Fichier)

    /** XPath expression for extracting robots content from meta tags */
    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    /** Robots tag value indicating no indexing or following */
    private static final String ROBOTS_TAG_NONE = "none";

    /** Robots tag value indicating no indexing */

            httpClientPropertyMap.put(name, value);
        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

            httpClientPropertyMap.put(name, value);
        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

                return false;
            };
        };

        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/");
        responseData.addMetaData("X-Robots-Tag", "noindex,nofollow");

        try {
            transformer.processXRobotsTag(responseData, new ResultData());
            fail();
        } catch (ChildUrlsException e) {

    peer.acceptFrame() // ACK
    peer.acceptFrame() // SYN_STREAM
    peer.acceptFrame() // DATA
    peer.sendFrame().headers(false, 3, headerEntries("a", "android"))
    peer.sendFrame().data(true, 3, Buffer().writeUtf8("robot"), 5)
    peer.acceptFrame() // PING
    peer.sendFrame().ping(true, Http2Connection.AWAIT_PING, 0) // PING
    peer.play()

    // Play it back.
    val connection = connect(peer)

# Pattern for environment variable keys in crawling data.
crawler.data.env.param.key.pattern=^FESS_ENV_.*
# Whether to ignore robots.txt during crawling.
crawler.ignore.robots.txt=false
# Whether to ignore robots meta tags during crawling.
crawler.ignore.robots.tags=false
# Whether to ignore content exceptions during crawling.
crawler.ignore.content.exception=true
# HTTP status codes considered as failure URLs.

	{"barfoobarfooyyyzzzyyyzzzyyyzzzyyyxxxzzzyyy", "x", 33},
	{"fofofofooofoboo", "oo", 7},
	{"fofofofofofoboo", "ob", 11},
	{"fofofofofofoboo", "boo", 12},
	{"fofofofofofoboo", "oboo", 11},
	{"fofofofofoooboo", "fooo", 8},
	{"fofofofofofoboo", "foboo", 10},
	{"fofofofofofoboo", "fofob", 8},
	{"fofofofofofofoffofoobarfoo", "foffof", 12},
	{"fofofofofoofofoffofoobarfoo", "foffof", 13},
	{"fofofofofofofoffofoobarfoo", "foffofo", 12},