return i > 0 && unicode.IsDigit(ch)
}

func (t *Tokenizer) Text() string {
	switch t.tok {
	case LSH:
		return "<<"
	case RSH:
		return ">>"
	case ARR:
		return "->"
	case ROT:
		return "@>"
	}
	return t.s.TokenText()
}

func (t *Tokenizer) File() string {
	return t.base.Filename()
}

func (t *Tokenizer) Base() *src.PosBase {
	return t.base
}

    static {
        setup(defaultCtype);
    }

    /**
     * {@link Tokenizer}を作成します。
     *
     * @param str
     *            文字列。{@literal null}であってはいけません
     */
    public Tokenizer(final String str) {
        this(str, defaultCtype);
    }

    /**
     * {@link Tokenizer}を作成します。
     *
     * @param str
     *            文字列。{@literal null}であってはいけません
     * @param ctype

    public void testEOF() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("");
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
    }

    /**
     * @throws Exception
     */
    @Test
    public void testWhitespace() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("\t       \n");

	})
}

func testBadInstParser(t *testing.T, goarch string, tests []badInstTest) {
	for i, test := range tests {
		arch, ctxt := setArch(goarch)
		tokenizer := lex.NewTokenizer("", strings.NewReader(test.input+"\n"), nil)
		parser := NewParser(ctxt, arch, tokenizer)

		err := tryParse(t, func() {
			parser.Parse()
		})

		switch {
		case err == nil:

        },
        "simplified_chinese_tokenizer": {
            "type": "fess_simplified_chinese_tokenizer"
        },
        "vietnamese_tokenizer": {
            "type": "fess_vietnamese_tokenizer",
            "sentence_detector": false,
            "ambiguities_resolved": false
        },
        "unigram_tokenizer": {
          "type": "ngram",
          "min_gram": 1,

    checkNotNull(input);
    Tokenizer tokenizer = new Tokenizer(input);
    try {
      String type = tokenizer.consumeToken(TOKEN_MATCHER);
      consumeSeparator(tokenizer, '/');
      String subtype = tokenizer.consumeToken(TOKEN_MATCHER);
      ImmutableListMultimap.Builder<String, String> parameters = ImmutableListMultimap.builder();
      while (tokenizer.hasMore()) {
        consumeSeparator(tokenizer, ';');

                }
            }
            if (idx < tok.length) {
                qualifier = tok[idx++];
                fallback = isDigits(qualifier);
            }

            // string tokenizer won't detect these and ignores them
            if (part1.contains("..") || part1.startsWith(".") || part1.endsWith(".")) {
                fallback = true;
            }

            if (fallback) {

		if i > 0 {
			name, value = name[:i], name[i+1:]
		}
		tokens := Tokenize(name)
		if len(tokens) != 1 || tokens[0].ScanToken != scanner.Ident {
			fmt.Fprintf(os.Stderr, "asm: parsing -D: %q is not a valid identifier name\n", tokens[0])
			flags.Usage()
		}
		macros[name] = &Macro{
			name:   name,
			args:   nil,
			tokens: Tokenize(value),
		}
	}
	return macros
}

var panicOnError bool // For testing.

package asm

import (
	"strings"
	"testing"

	"cmd/asm/internal/lex"
)

func tokenize(s string) [][]lex.Token {
	res := [][]lex.Token{}
	if len(s) == 0 {
		return res
	}
	for _, o := range strings.Split(s, ",") {
		res = append(res, lex.Tokenize(o))
	}
	return res
}

func TestErroneous(t *testing.T) {

	type errtest struct {
		pseudo   string
		operands string

type Macro struct {
	name   string   // The #define name.
	args   []string // Formal arguments.
	tokens []Token  // Body of macro.
}

// Tokenize turns a string into a list of Tokens; used to parse the -D flag and in tests.
func Tokenize(str string) []Token {
	t := NewTokenizer("command line", strings.NewReader(str), nil)
	var tokens []Token
	for {
		tok := t.Next()
		if tok == scanner.EOF {
			break