</tr>
</tbody>
</table>
<!--///////////////////// tweets /////////////////////////-->
<!--///////////////////// stand alone tweet /////////////////////////-->
<table class="collapse" align="center" width="500" style="width:500px;background-color:#ffffff;padding:0;margin:0;line-height:1px;font-size:1px;" cellpadding="0" cellspacing="0" border="0">

     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {
            throw new CrawlerSystemException("url is null or empty.");
        }

        if (robotsTxtHelper == null || !robotsTxtHelper.isEnabled()) {
            // not support robots.txt