if (duplicateTermRemoved && symbolSize > 0) {
                            isSpace = removeLastDuplication(buf, symbolSize, isSpace, termCache);
                        }
                        // alphanum
                        if (maxAlphanumTermSize >= 0) {
                            if (alphanumSize < maxAlphanumTermSize) {
                                buf.appendCodePoint(c);

                        continue;
                    }

                    TestAnalyzeToken token = new TestAnalyzeToken(word, position++, startOffset, endOffset, 1, // positionIncrement
                            "<ALPHANUM>", Collections.emptyMap());
                    tokens.add(token);
                    offset = endOffset;
                }
            }

            return tokens;
        }

        @Override

crawler.document.append.data=true
# Whether to append filename to documents.
crawler.document.append.filename=false
# Maximum size of alphanumeric terms in documents.
crawler.document.max.alphanum.term.size=20
# Maximum size of symbol terms in documents.
crawler.document.max.symbol.term.size=10
# Whether to remove duplicate terms in documents.
crawler.document.duplicate.term.removed=false

    String CRAWLER_DOCUMENT_APPEND_FILENAME = "crawler.document.append.filename";

    /** The key of the configuration. e.g. 20 */
    String CRAWLER_DOCUMENT_MAX_ALPHANUM_TERM_SIZE = "crawler.document.max.alphanum.term.size";

    /** The key of the configuration. e.g. 10 */
    String CRAWLER_DOCUMENT_MAX_SYMBOL_TERM_SIZE = "crawler.document.max.symbol.term.size";

    /** The key of the configuration. e.g. false */