return i > 0 && unicode.IsDigit(ch)
}

func (t *Tokenizer) Text() string {
	switch t.tok {
	case LSH:
		return "<<"
	case RSH:
		return ">>"
	case ARR:
		return "->"
	case ROT:
		return "@>"
	}
	return t.s.TokenText()
}

func (t *Tokenizer) File() string {
	return t.base.Filename()
}

func (t *Tokenizer) Base() *src.PosBase {
	return t.base
}

        "tokenizer" : "standard"
      },
      "reading_term_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard"
      },
      "normalize_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "keyword",
        "char_filter" : ["mapping_char"],
        "filter" : ["lowercase"]
      },
      "contents_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard",

 * use a tokenizer to process the input.
 *
 * <p>
 * The class provides methods to initialize the converter, convert strings, and
 * check if a tokenizer is enabled. It also includes methods to create a token
 * stream and extract reading information from the stream's attributes, although
 * the tokenizer-related functionality is currently commented out.
 * </p>
 */

            @Override
            protected TokenStreamComponents createComponents(final String fieldName) {
                final Tokenizer tokenizer = new KeywordTokenizer();
                final TokenStream stream = ignoreCase ? new LowerCaseFilter(tokenizer) : tokenizer;
                return new TokenStreamComponents(tokenizer, stream);
            }
        };
    }

            if (handler == null) {
                final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                while (tokenizer.hasMoreTokens()) {
                    final String provider = tokenizer.nextToken().trim();
                    if (provider.equals("jcifs")) {
                        continue;
                    }

	})
}

func testBadInstParser(t *testing.T, goarch string, tests []badInstTest) {
	for i, test := range tests {
		arch, ctxt := setArch(goarch)
		tokenizer := lex.NewTokenizer("", strings.NewReader(test.input+"\n"), nil)
		parser := NewParser(ctxt, arch, tokenizer)

		err := tryParse(t, func() {
			parser.Parse()
		})

		switch {
		case err == nil:

            if (handler == null) {
                final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                while (tokenizer.hasMoreTokens()) {
                    final String provider = tokenizer.nextToken().trim();
                    if (provider.equals("jcifs.smb1")) {
                        continue;
                    }

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.opensearch.extension.analysis;

import org.apache.lucene.analysis.Tokenizer;
import org.opensearch.common.settings.Settings;
import org.opensearch.env.Environment;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.analysis.AbstractTokenizerFactory;

/**

import java.io.IOException;
import java.io.StringReader;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.util.CSVUtil;

                .put("analysis.analyzer.test_reading_analyzer.tokenizer", "standard")
                .put("analysis.analyzer.test_reading_term_analyzer.type", "custom")
                .put("analysis.analyzer.test_reading_term_analyzer.tokenizer", "standard")
                .put("analysis.analyzer.test_reading_analyzer_ja.type", "custom")
                .put("analysis.analyzer.test_reading_analyzer_ja.tokenizer", "keyword")