/**
     * Sets whether to use robots.txt disallow rules.
     *
     * @param useRobotsTxtDisallows True to use disallow rules, false otherwise
     */
    public void setUseRobotsTxtDisallows(final boolean useRobotsTxtDisallows) {
        this.useRobotsTxtDisallows = useRobotsTxtDisallows;
    }

    /**
     * Sets whether to use robots.txt allow rules.
     *

				"application/onenote",
				"application/parityfec",
				"application/patch-ops-error+xml",
				"application/pgp-encrypted",
				"application/pgp-keys",
				"application/pgp-signature",
				"application/pics-rules",
				"application/pidf+xml",
				"application/pidf-diff+xml",
				"application/pkcs10",
				"application/pkcs7-mime",
				"application/pkcs7-signature",
				"application/pkix-cert",
				"application/pkix-crl",

      },
      "dutch_stemmer": {
        "type":       "stemmer",
        "language":   "dutch"
      },
      "dutch_override": {
        "type":       "stemmer_override",
        "rules": [
          "fiets=>fiets",
          "bromfiets=>bromfiets",
          "ei=>eier",
          "kind=>kinder"
        ]
      },
      "english_keywords": {
        "type":       "keyword_marker",