import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.fst.FST;

// https://issues.apache.org/jira/browse/LUCENE-5252
public final class NGramSynonymTokenizer extends Tokenizer {

    public static final int DEFAULT_N_SIZE = 2;

    public static final String DEFAULT_DELIMITERS = " 　\t\n\r";

    static final int BUFFER_SIZE = 4096;

    private final int n;

        super(indexSettings, settings, name);
        ignoreCase = settings.getAsBoolean("ignore_case", true);
        n = settings.getAsInt("n", NGramSynonymTokenizer.DEFAULT_N_SIZE);
        delimiters = settings.get("delimiters", NGramSynonymTokenizer.DEFAULT_DELIMITERS);
        expand = settings.getAsBoolean("expand", true);

        settings.getAsBoolean("expand_ngram", false); // TODO remove