String CRAWLER_DOCUMENT_HTML_CANONICAL_XPATH = "crawler.document.html.canonical.xpath";

    /** The key of the configuration. e.g. noscript,script,style,header,footer,aside,nav,a[rel=nofollow] */
    String CRAWLER_DOCUMENT_HTML_PRUNED_TAGS = "crawler.document.html.pruned.tags";

    /** The key of the configuration. e.g. 120 */

*\n   * **DEPRECATION**: This way to access PopperUtils is deprecated\n   * and will be removed in v2! Use the PopperUtils module directly instead.\n   * Due to the high instability of the methods contained in Utils, we can't\n   * guarantee them to follow semver. Use them at your own risk!\n   * @static\n   * @private\n   * @type {Object}\n   * @deprecated since version 1.8\n   * @member Utils\n   * @memberof Popper\n   */\n  static Utils = (typeof window !== 'undefined' ? window : global).PopperUtils;\n\n...

                    }
                    if (content.contains(ROBOTS_TAG_NOFOLLOW)) {
                        nofollow = true;
                    }
                }
                if (noindex && nofollow) {
                    logger.info("META(robots=noindex,nofollow): {}", responseData.getUrl());
                    throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots");
                }

            assertTrue(e.getChildUrlList().isEmpty());
        } catch (Exception e) {
            fail();
        }
    }

    public void test_processMetaRobots_nofollow() throws Exception {
        final String data = "<meta name=\"robots\" content=\"nofollow\" />";
        final Document document = getDocument(data);

        final FessXpathTransformer transformer = new FessXpathTransformer() {

        FessConfig fessConfig = new FessConfig.SimpleImpl() {
            @Override
            public String getCrawlerDocumentHtmlPrunedTags() {
                return "script,div#main,p.image,a[rel=nofollow],div[x-y=a-.:_0]";
            }
        };

        PrunedTag[] tags = fessConfig.getCrawlerDocumentHtmlPrunedTagsAsArray();
        assertTrue(matchesTag(tags[0], "<script></script>"));

crawler.document.html.digest.xpath=//META[@name='description']/@content
crawler.document.html.canonical.xpath=//LINK[@rel='canonical'][1]/@href
crawler.document.html.pruned.tags=noscript,script,style,header,footer,aside,nav,a[rel=nofollow]
crawler.document.html.max.digest.length=120
crawler.document.html.default.lang=
crawler.document.html.default.include.index.patterns=

            if (logger.isDebugEnabled()) {
                logger.debug("{} does not exists.", dir);
            }
            return;
        }
        try (Stream<Path> walk = Files.walk(dir, FileVisitOption.FOLLOW_LINKS)) {
            walk.sorted(Comparator.reverseOrder()).forEach(f -> {
                if (logger.isDebugEnabled()) {
                    logger.debug("Deleting {}", f);
                }
                try {