}
                    if (content.contains(ROBOTS_TAG_NOFOLLOW)) {
                        nofollow = true;
                    }
                }
                if (noindex && nofollow) {
                    logger.info("META(robots=noindex,nofollow): {}", responseData.getUrl());
                    throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots");
                }

crawler.document.html.canonical.xpath=//LINK[@rel='canonical'][1]/@href
# HTML tags to prune (remove) during document processing.
crawler.document.html.pruned.tags=noscript,script,style,header,footer,aside,nav,a[rel=nofollow]
# Maximum length of digest extracted from HTML documents.
crawler.document.html.max.digest.length=120
# Default language for HTML documents.
crawler.document.html.default.lang=

</tr>
<!-- IF TEXT-ONLY bottom padding -->
<tr>
<td height="28" style="padding:0;margin:0;line-height:1px;font-size:1px;"></td>
</tr>
<!-- end bottom padding for a text-only tweet-->
<!-- IF USER DOES NOT FOLLOW ACCOUNT else -->
<tr>
<td colspan="2" style="padding:0;margin:0;line-height:1px;font-size:1px;">
<table border="0" cellpadding="0" cellspacing="0" align="left" style="padding:0;margin:0;line-height:1px;font-size:1px;">