}
        return line;
    }

    /**
     * Checks if robots.txt processing is enabled.
     * @return true if enabled, false otherwise
     */
    public boolean isEnabled() {
        return enabled;
    }

    /**
     * Sets whether robots.txt processing is enabled.
     * @param enabled true to enable, false to disable
     */

        if (StringUtil.isBlank(url)) {
            throw new CrawlerSystemException("url is null or empty.");
        }

        if (robotsTxtHelper == null || !robotsTxtHelper.isEnabled()) {
            // not support robots.txt
            return;
        }

        // crawler context
        final CrawlerContext crawlerContext = CrawlingParameterUtil.getCrawlerContext();