}

    /**
     * Sets whether to use robots.txt disallow rules.
     *
     * @param useRobotsTxtDisallows True to use disallow rules, false otherwise
     */
    public void setUseRobotsTxtDisallows(final boolean useRobotsTxtDisallows) {
        this.useRobotsTxtDisallows = useRobotsTxtDisallows;
    }

    /**
     * Sets whether to use robots.txt allow rules.
     *