"application/commonground",
				"application/conference-info+xml",
				"application/cpl+xml",
				"application/csta+xml",
				"application/cstadata+xml",
				"application/cu-seeme",
				"application/cybercash",
				"application/davmount+xml",
				"application/dca-rft",
				"application/dec-dx",
				"application/dialog-info+xml",
				"application/dicom",
				"application/dita+xml",

                logger.debug("{} is already visited.", robotTxtUrl);
            }
            return;
        }

        if (logger.isInfoEnabled()) {
            logger.info("Checking URL: {}", robotTxtUrl);
        }
        // add url to a set
        crawlerContext.getRobotsTxtUrlSet().add(robotTxtUrl);

        final HttpGet httpGet = new HttpGet(robotTxtUrl);