# Maximum size of alphanumeric terms in documents.
crawler.document.max.alphanum.term.size=20
# Maximum size of symbol terms in documents.
crawler.document.max.symbol.term.size=10
# Whether to remove duplicate terms in documents.
crawler.document.duplicate.term.removed=false
# Unicode space characters for document parsing.

        return buf.toString().trim();
    }

    /**
     * Replaces duplicate hosts in the URL using the duplicate host helper.
     *
     * @param url the URL to process
     * @return the URL with duplicate hosts replaced
     */
    protected String replaceDuplicateHost(final String url) {
        try {
            // remove duplicate host