// check url
        if (crawlerContext.getRobotsTxtUrlSet().contains(robotTxtUrl)) {
            if (logger.isDebugEnabled()) {
                logger.debug("{} is already visited.", robotTxtUrl);
            }
            return;
        }

        if (logger.isInfoEnabled()) {
            logger.info("Checking URL: {}", robotTxtUrl);
        }
        // add url to a set

				"application/vnd.oma.poc.optimized-progress-report+xml",
				"application/vnd.oma.xcap-directory+xml",
				"application/vnd.omads-email+xml",
				"application/vnd.omads-file+xml",
				"application/vnd.omads-folder+xml",
				"application/vnd.omaloc-supl-init",
				"application/vnd.openofficeorg.extension",
				"application/vnd.openxmlformats-officedocument.presentationml.presentation",