/**
     * Sets whether to use robots.txt disallow rules.
     *
     * @param useRobotsTxtDisallows True to use disallow rules, false otherwise
     */
    public void setUseRobotsTxtDisallows(final boolean useRobotsTxtDisallows) {
        this.useRobotsTxtDisallows = useRobotsTxtDisallows;
    }

    /**
     * Sets whether to use robots.txt allow rules.
     *

				"application/onenote",
				"application/parityfec",
				"application/patch-ops-error+xml",
				"application/pgp-encrypted",
				"application/pgp-keys",
				"application/pgp-signature",
				"application/pics-rules",
				"application/pidf+xml",
				"application/pidf-diff+xml",
				"application/pkcs10",
				"application/pkcs7-mime",
				"application/pkcs7-signature",
				"application/pkix-cert",
				"application/pkix-crl",