if i > 0 {
			name, value = name[:i], name[i+1:]
		}
		tokens := Tokenize(name)
		if len(tokens) != 1 || tokens[0].ScanToken != scanner.Ident {
			fmt.Fprintf(os.Stderr, "asm: parsing -D: %q is not a valid identifier name\n", tokens[0])
			flags.Usage()
		}
		macros[name] = &Macro{
			name:   name,
			args:   nil,
			tokens: Tokenize(value),
		}
	}
	return macros
}

var panicOnError bool // For testing.

	scanCalled   bool      // Scan has been called; buffer is in use.
	done         bool      // Scan has finished.
}

// SplitFunc is the signature of the split function used to tokenize the
// input. The arguments are an initial substring of the remaining unprocessed
// data and a flag, atEOF, that reports whether the [Reader] has no more data

            current.println("        method: $method")
            current.println("        path: \"$path\"")
            if (query != null) {
                for (String param: query.tokenize('&')) {
                    def (String name, String value) = param.tokenize('=')
                    if (value == null) {
                        value = ''
                    }
                    current.println("        $name: \"$value\"")

		}
	}()

	parse()

	return nil
}

func testBadOperandParser(t *testing.T, parser *Parser, tests []badOperandTest) {
	for _, test := range tests {
		err := tryParse(t, func() {
			parser.start(lex.Tokenize(test.input))
			addr := obj.Addr{}
			parser.operand(&addr)
		})

		switch {
		case err == nil:
			t.Errorf("fail at %s: got no errors; expected %s\n", test.input, test.error)

        final Tokenizer tokenizer;
        if (trimRootToken && hasDots) {
            tokenizer = new Tokenizer(expression);
            tokenizer.nextPropertyName();
            if (tokenizer.getPosition() == EOF) {
                return null;
            }
        } else {
            tokenizer = new Tokenizer("." + expression);
        }

    checkNotNull(input);
    Tokenizer tokenizer = new Tokenizer(input);
    try {
      String type = tokenizer.consumeToken(TOKEN_MATCHER);
      consumeSeparator(tokenizer, '/');
      String subtype = tokenizer.consumeToken(TOKEN_MATCHER);
      ImmutableListMultimap.Builder<String, String> parameters = ImmutableListMultimap.builder();
      while (tokenizer.hasMore()) {
        consumeSeparator(tokenizer, ';');

        // Verify token length
        assertEquals(tokenSize, SMBUtil.readInt2(buffer, bodyOffset + 14));

        if (tokenSize > 0) {
            // Verify token content
            int tokenOffset = bodyOffset + 24;
            while ((tokenOffset % 8) != 0) {
                tokenOffset++; // Account for pad8
            }
            byte[] actualToken = new byte[tokenSize];

t n=this.lexer.state.inRawBlock;return{type:"text",raw:t[0],text:t[0],escaped:n}}}};var x=class l{tokens;options;state;inlineQueue;tokenizer;constructor(e){this.tokens=[],this.tokens.links=Object.create(null),this.options=e||R,this.options.tokenizer=this.options.tokenizer||new w,this.tokenizer=this.options.tokenizer,this.tokenizer.options=this.options,this.tokenizer.lexer=this,this.inlineQueue=[],this.state={inLink:!1,inRawBlock:!1,top:!0};let t={other:m,block:B.normal,inline:z.normal};this.opti...

String escaped = JsonUtil.escape("Hello \"World\" with special chars");
String unescaped = JsonUtil.unescape(escaped);

// Text tokenization
Tokenizer tokenizer = new Tokenizer("field1,field2,field3", ",");
while (tokenizer.hasMoreTokens()) {
    String token = tokenizer.nextToken();
    // Process each token
}

// Decimal formatting
DecimalFormat format = DecimalFormatUtil.getDecimalFormat("###,###.00");
```

        List<AnalyzeToken> tokens = analyzer.analyzeAndReading(text, field, lang);

        assertNotNull(tokens);
        assertEquals(2, tokens.size()); // "Test123" and "ABC-456" (hyphen is not a split char in our tokenizer)
        assertEquals("Test123", tokens.get(0).getTerm());
        assertEquals("ABC-456", tokens.get(1).getTerm());
        assertEquals("TEST123", ((TestAnalyzeToken) tokens.get(0)).getReading());