}

    /**
     * Sets whether to use robots.txt allow rules.
     *
     * @param useRobotsTxtAllows True to use allow rules, false otherwise
     */
    public void setUseRobotsTxtAllows(final boolean useRobotsTxtAllows) {
        this.useRobotsTxtAllows = useRobotsTxtAllows;
    }

    /**
     * Sets the credentials provider for authentication.
     *