try {
            processRobotsTxt(url);
        } catch (final CrawlingAccessException e) {
            if (logger.isInfoEnabled()) {
                final StringBuilder buf = new StringBuilder(100);
                buf.append(e.getMessage());
                if (e.getCause() != null) {
                    buf.append(e.getCause().getMessage());
                }
                logger.info(buf.toString());

         *
         * @param path The path to append to the URL
         * @return The complete FTP URL
         */
        public String toUrl(final String path) {
            final StringBuilder buf = new StringBuilder(100);
            buf.append("ftp://");
            buf.append(getHost());
            final int port = getPort();
            if (port != DEFAULT_FTP_PORT) {
                buf.append(':').append(port);
            }

    <alias type="image/ntf"/>
    <magic priority="50">
      <match value="NITF01.10" type="string" offset="0"/>
      <match value="NITF02.000" type="string" offset="0"/>
      <match value="NITF02.100" type="string" offset="0"/>
    </magic>
    <glob pattern="*.ntf"/>
    <glob pattern="*.nitf"/>
  </mime-type>

  <mime-type type="image/png">
    <acronym>PNG</acronym>