package asm

import (
	"strings"
	"testing"

	"cmd/asm/internal/lex"
)

func tokenize(s string) [][]lex.Token {
	res := [][]lex.Token{}
	if len(s) == 0 {
		return res
	}
	for _, o := range strings.Split(s, ",") {
		res = append(res, lex.Tokenize(o))
	}
	return res
}

func TestErroneous(t *testing.T) {

	type errtest struct {
		pseudo   string
		operands string

}

func TestExpr(t *testing.T) {
	p := NewParser(nil, nil, nil) // Expression evaluation uses none of these fields of the parser.
	for i, test := range exprTests {
		p.start(lex.Tokenize(test.input))
		result := int64(p.expr())
		if result != test.output {
			t.Errorf("%d: %q evaluated to %d; expected %d", i, test.input, result, test.output)
		}
		tok := p.next()

type Macro struct {
	name   string   // The #define name.
	args   []string // Formal arguments.
	tokens []Token  // Body of macro.
}

// Tokenize turns a string into a list of Tokens; used to parse the -D flag and in tests.
func Tokenize(str string) []Token {
	t := NewTokenizer("command line", strings.NewReader(str), nil)
	var tokens []Token
	for {
		tok := t.Next()
		if tok == scanner.EOF {
			break

	return i > 0 && unicode.IsDigit(ch)
}

func (t *Tokenizer) Text() string {
	switch t.tok {
	case LSH:
		return "<<"
	case RSH:
		return ">>"
	case ARR:
		return "->"
	case ROT:
		return "@>"
	}
	return t.s.TokenText()
}

func (t *Tokenizer) File() string {
	return t.base.Filename()
}

func (t *Tokenizer) Base() *src.PosBase {
	return t.base
}

    static {
        setup(defaultCtype);
    }

    /**
     * Creates a {@link Tokenizer}.
     *
     * @param str
     *            The string. Must not be {@literal null}.
     */
    public Tokenizer(final String str) {
        this(str, defaultCtype);
    }

    /**
     * Creates a {@link Tokenizer}.
     *
     * @param str
     *            The string. Must not be {@literal null}.

            if (handler == null) {
                final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                while (tokenizer.hasMoreTokens()) {
                    final String provider = tokenizer.nextToken().trim();
                    if (provider.equals("jcifs")) {
                        continue;
                    }

            if (handler == null) {
                final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                while (tokenizer.hasMoreTokens()) {
                    final String provider = tokenizer.nextToken().trim();
                    if (provider.equals("jcifs.smb1")) {
                        continue;
                    }

    public void testEOF() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("");
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
    }

    /**
     * @throws Exception
     */
    @Test
    public void testWhitespace() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("\t       \n");

 * use a tokenizer to process the input.
 *
 * <p>
 * The class provides methods to initialize the converter, convert strings, and
 * check if a tokenizer is enabled. It also includes methods to create a token
 * stream and extract reading information from the stream's attributes, although
 * the tokenizer-related functionality is currently commented out.
 * </p>
 */

import java.io.IOException;
import java.io.StringReader;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.util.CSVUtil;