// check url
        if (crawlerContext.getRobotsTxtUrlSet().contains(robotTxtUrl)) {
            if (logger.isDebugEnabled()) {
                logger.debug("{} is already visited.", robotTxtUrl);
            }
            return;
        }

        if (logger.isInfoEnabled()) {
            logger.info("Checking URL: {}", robotTxtUrl);
        }
        // add url to a set