s := t.s
	for {
		t.tok = ScanToken(s.Scan())
		if t.tok != scanner.Comment {
			break
		}
		text := s.TokenText()
		t.line += strings.Count(text, "\n")
		if constraint.IsGoBuild(text) {
			t.tok = BuildComment
			break
		}
	}
	switch t.tok {
	case '\n':
		t.line++
	case '-':
		if s.Peek() == '>' {
			s.Next()
			t.tok = ARR
			return ARR
		}
	case '@':

		return true
	case *ast.Ident:
		// TODO: Handle types defined within function.
		for _, d := range p.Decl {
			gd, ok := d.(*ast.GenDecl)
			if !ok || gd.Tok != token.TYPE {
				continue
			}
			for _, spec := range gd.Specs {
				ts, ok := spec.(*ast.TypeSpec)
				if !ok {
					continue
				}
				if ts.Name.Name == t.Name {

pkg go/ast, type ArrayType struct, Len Expr
pkg go/ast, type AssignStmt struct
pkg go/ast, type AssignStmt struct, Lhs []Expr
pkg go/ast, type AssignStmt struct, Rhs []Expr
pkg go/ast, type AssignStmt struct, Tok token.Token
pkg go/ast, type AssignStmt struct, TokPos token.Pos
pkg go/ast, type BadDecl struct
pkg go/ast, type BadDecl struct, From token.Pos
pkg go/ast, type BadDecl struct, To token.Pos

	nesting := 0
	var tokens []Token
	for {
		tok := in.Stack.Next()
		if tok == scanner.EOF || tok == '\n' {
			in.Error("unterminated arg list invoking macro:", macro.name)
		}
		if nesting == 0 && (tok == ')' || tok == ',') {
			return tokens, tok
		}
		if tok == '(' {
			nesting++
		}
		if tok == ')' {
			nesting--
		}
		tokens = append(tokens, Make(tok, in.Stack.Text()))
	}
}

			if tok == '\n' || tok == ';' || (nesting == 0 && (tok == ',' || tok == ':')) {
				if tok == ':' {
					// Remember this location so we can swap the operands below.
					if colon >= 0 {
						p.errorf("invalid ':' in operand")
						return word, cond, operands, true
					}
					colon = len(operands)
				}
				break
			}
			if tok == '(' || tok == '[' {
				nesting++
			}
			if tok == ')' || tok == ']' {

}

// drain returns a single string representing the processed input tokens.
func drain(input *Input) string {
	var buf strings.Builder
	for {
		tok := input.Next()
		if tok == scanner.EOF {
			return buf.String()
		}
		if tok == '#' {
			continue
		}
		if buf.Len() > 0 {
			buf.WriteByte('.')
		}
		buf.WriteString(input.Text())
	}
}

type badLexTest struct {

func Tokenize(str string) []Token {
	t := NewTokenizer("command line", strings.NewReader(str), nil)
	var tokens []Token
	for {
		tok := t.Next()
		if tok == scanner.EOF {
			break
		}
		tokens = append(tokens, Make(tok, t.Text()))
	}
	return tokens

		result := int64(p.expr())
		if result != test.output {
			t.Errorf("%d: %q evaluated to %d; expected %d", i, test.input, result, test.output)
		}
		tok := p.next()
		if test.atEOF && tok.ScanToken != scanner.EOF {
			t.Errorf("%d: %q: at EOF got %s", i, test.input, tok)
		} else if !test.atEOF && tok.ScanToken == scanner.EOF {
			t.Errorf("%d: %q: expected not EOF but at EOF", i, test.input)
		}
	}
}

type badExprTest struct {

	refName := make(map[*ast.Expr]*Name)
	for _, r := range f.Ref {
		refName[r.Expr] = r.Name
	}
	for _, d := range f.AST.Decls {
		d, ok := d.(*ast.GenDecl)
		if !ok || d.Tok != token.TYPE {
			continue
		}
		for _, s := range d.Specs {
			s := s.(*ast.TypeSpec)
			n := refName[&s.Type]
			if n != nil && n.Mangle != "" {
				override[n.Mangle] = s.Name.Name
			}
		}
	}

	s.tr = append(s.tr, tr)
}

func (s *Stack) Next() ScanToken {
	tos := s.tr[len(s.tr)-1]
	tok := tos.Next()
	for tok == scanner.EOF && len(s.tr) > 1 {
		tos.Close()
		// Pop the topmost item from the stack and resume with the next one down.
		s.tr = s.tr[:len(s.tr)-1]
		tok = s.Next()
	}
	return tok
}

func (s *Stack) Text() string {
	return s.tr[len(s.tr)-1].Text()
}