var tokens []Token
	for _, tok := range macro.tokens {
		if tok.ScanToken != scanner.Ident {
			tokens = append(tokens, tok)
			continue
		}
		substitution := actuals[tok.text]
		if substitution == nil {
			tokens = append(tokens, tok)
			continue
		}
		tokens = append(tokens, substitution...)
	}
	in.Push(NewSlice(in.Base(), in.Line(), tokens))
}

		if s.end > s.start || s.err != nil {
			advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil)
			if err != nil {
				if err == ErrFinalToken {
					s.token = token
					s.done = true
					// When token is not nil, it means the scanning stops
					// with a trailing token, and thus the return value
					// should be true to indicate the existence of the token.
					return token != nil

	// Create a custom split function by wrapping the existing ScanWords function.
	split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		advance, token, err = bufio.ScanWords(data, atEOF)
		if err == nil && token != nil {
			_, err = strconv.ParseInt(string(token), 10, 32)
		}
		return
	}
	// Set the split function for the scanning operation.
	scanner.Split(split)
	// Validate the input

package lex

import (
	"text/scanner"

	"cmd/internal/src"
)

// A Slice reads from a slice of Tokens.
type Slice struct {
	tokens []Token
	base   *src.PosBase
	line   int
	pos    int
}

func NewSlice(base *src.PosBase, line int, tokens []Token) *Slice {
	return &Slice{
		tokens: tokens,
		base:   base,
		line:   line,
		pos:    -1, // Next will advance to zero.
	}
}

	"cmd/internal/objabi"
	"cmd/internal/src"
)

// A Tokenizer is a simple wrapping of text/scanner.Scanner, configured
// for our purposes and made a TokenReader. It forms the lowest level,
// turning text from readers into tokens.
type Tokenizer struct {
	tok  ScanToken
	s    *scanner.Scanner
	base *src.PosBase
	line int
	file *os.File // If non-nil, file descriptor to close.
}

	Close()
}

// A Token is a scan token plus its string value.
// A macro is stored as a sequence of Tokens with spaces stripped.
type Token struct {
	ScanToken
	text string
}

// Make returns a Token with the given rune (ScanToken) and text representation.
func Make(token ScanToken, text string) Token {
	// Substitute the substitutes for . and /.
	text = strings.ReplaceAll(text, "\u00B7", ".")

		t.Fatalf("scan failed: %v", scanner.Err())
	}
	if token := scanner.Text(); token != word {
		t.Fatalf("unexpected token: %v", token)
	}
}

// Test that empty tokens, including at end of line or end of file, are found by the scanner.
// Issue 8672: Could miss final empty token.

func commaSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
	for i := 0; i < len(data); i++ {

</p>

<h3 id="Tokens">Tokens</h3>

<p>
Tokens form the vocabulary of the Go language.
There are four classes: <i>identifiers</i>, <i>keywords</i>, <i>operators
and punctuation</i>, and <i>literals</i>.  <i>White space</i>, formed from
spaces (U+0020), horizontal tabs (U+0009),
carriage returns (U+000D), and newlines (U+000A),
is ignored except as it separates tokens

		})
	}
}

func makeBenchInputHard() []byte {
	tokens := [...]string{
		"<a>", "<p>", "<b>", "<strong>",
		"</a>", "</p>", "</b>", "</strong>",
		"hello", "world",
	}
	x := make([]byte, 0, 1<<20)
	for {
		i := rand.Intn(len(tokens))
		if len(x)+len(tokens[i]) >= 1<<20 {
			break
		}
		x = append(x, tokens[i]...)
	}
	return x
}

	}
}

// have reports whether the remaining tokens (including the current one) contain the specified token.
func (p *Parser) have(token lex.ScanToken) bool {
	for i := p.inputPos; i < len(p.input); i++ {
		if p.input[i].ScanToken == token {
			return true
		}
	}
	return false
}

// at reports whether the next tokens are as requested.
func (p *Parser) at(next ...lex.ScanToken) bool {