<div>
<table width="100%" border="0" cellpadding="0" cellspacing="0" style="padding:0;margin:0;line-height:1px;font-size:1px;">
<!--<![endif]-->
<tbody>
<tr>

        try {
            processRobotsTxt(url);
        } catch (final CrawlingAccessException e) {
            if (logger.isInfoEnabled()) {
                final StringBuilder buf = new StringBuilder(100);
                buf.append(e.getMessage());
                if (e.getCause() != null) {
                    buf.append(e.getCause().getMessage());
                }
                logger.info(buf.toString());