}
                    if (content.contains(ROBOTS_TAG_NOFOLLOW)) {
                        nofollow = true;
                    }
                }
                if (noindex && nofollow) {
                    logger.info("META(robots=noindex,nofollow): {}", responseData.getUrl());
                    throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots");
                }

            assertTrue(e.getChildUrlList().isEmpty());
        } catch (Exception e) {
            fail();
        }
    }

    public void test_processMetaRobots_nofollow() throws Exception {
        final String data = "<meta name=\"robots\" content=\"nofollow\" />";
        final Document document = getDocument(data);

        final FessXpathTransformer transformer = new FessXpathTransformer() {

        FessConfig fessConfig = new FessConfig.SimpleImpl() {
            @Override
            public String getCrawlerDocumentHtmlPrunedTags() {
                return "script,div#main,p.image,a[rel=nofollow],div[x-y=a-.:_0]";
            }
        };

        PrunedTag[] tags = fessConfig.getCrawlerDocumentHtmlPrunedTagsAsArray();
        assertTrue(matchesTag(tags[0], "<script></script>"));

    String CRAWLER_DOCUMENT_HTML_CANONICAL_XPATH = "crawler.document.html.canonical.xpath";

    /** The key of the configuration. e.g. noscript,script,style,header,footer,aside,nav,a[rel=nofollow] */
    String CRAWLER_DOCUMENT_HTML_PRUNED_TAGS = "crawler.document.html.pruned.tags";

    /** The key of the configuration. e.g. 120 */

*\n   * **DEPRECATION**: This way to access PopperUtils is deprecated\n   * and will be removed in v2! Use the PopperUtils module directly instead.\n   * Due to the high instability of the methods contained in Utils, we can't\n   * guarantee them to follow semver. Use them at your own risk!\n   * @static\n   * @private\n   * @type {Object}\n   * @deprecated since version 1.8\n   * @member Utils\n   * @memberof Popper\n   */\n  static Utils = (typeof window !== 'undefined' ? window : global).PopperUtils;\n\n...

crawler.document.html.digest.xpath=//META[@name='description']/@content
crawler.document.html.canonical.xpath=//LINK[@rel='canonical'][1]/@href
crawler.document.html.pruned.tags=noscript,script,style,header,footer,aside,nav,a[rel=nofollow]
crawler.document.html.max.digest.length=120
crawler.document.html.default.lang=
crawler.document.html.default.include.index.patterns=

            if (logger.isDebugEnabled()) {
                logger.debug("{} does not exists.", dir);
            }
            return;
        }
        try (Stream<Path> walk = Files.walk(dir, FileVisitOption.FOLLOW_LINKS)) {
            walk.sorted(Comparator.reverseOrder()).forEach(f -> {
                if (logger.isDebugEnabled()) {
                    logger.debug("Deleting {}", f);
                }
                try {

 * resources are files and directories however an <code>SmbFile</code>
 * may also refer to servers and workgroups. If the resource is a file or
 * directory the methods of <code>SmbFile</code> follow the behavior of
 * the well known {@link java.io.File} class. One fundamental difference
 * is the usage of a URL scheme [1] to specify the target file or
 * directory. SmbFile URLs have the following syntax:
 *

linked with the Library has the freedom and the wherewithal to run
that program using a modified version of the Library.

  The precise terms and conditions for copying, distribution and
modification follow.  Pay close attention to the difference between a
"work based on the library" and a "work that uses the library".  The
former contains code derived from the library, whereas the latter must

 * resources are files and directories however an <code>SmbFile</code>
 * may also refer to servers and workgroups. If the resource is a file or
 * directory the methods of <code>SmbFile</code> follow the behavior of
 * the well known {@link java.io.File} class. One fundamental difference
 * is the usage of a URL scheme [1] to specify the target file or
 * directory. SmbFile URLs have the following syntax:
 *