if (StringUtil.isBlank(url)) {
            throw new CrawlerSystemException("url is null or empty.");
        }

        if (robotsTxtHelper == null || !robotsTxtHelper.isEnabled()) {
            // not support robots.txt
            return;
        }

        // crawler context
        final CrawlerContext crawlerContext = CrawlingParameterUtil.getCrawlerContext();