}

    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    public void setCredentialsProvider(final CredentialsProvider credentialsProvider) {
        this.credentialsProvider = credentialsProvider;
    }

    public void setAuthCache(final AuthCache authCache) {
        this.authCache = authCache;
    }

        this.useRobotsTxtAllows = useRobotsTxtAllows;
    }

    public void setCredentialsProvider(final CredentialsProvider credentialsProvider) {
        this.credentialsProvider = credentialsProvider;
    }

    public void setAuthCache(final AuthCache authCache) {
        this.authCache = authCache;
    }