}

func (t *Tokenizer) Text() string {
	switch t.tok {
	case LSH:
		return "<<"
	case RSH:
		return ">>"
	case ARR:
		return "->"
	case ROT:
		return "@>"
	}
	return t.s.TokenText()
}

func (t *Tokenizer) File() string {
	return t.base.Filename()
}

func (t *Tokenizer) Base() *src.PosBase {
	return t.base
}

func (t *Tokenizer) SetBase(base *src.PosBase) {
	t.base = base

pkg text/scanner, method (*Scanner) Peek() int32
pkg text/scanner, method (*Scanner) Pos() Position
pkg text/scanner, method (*Scanner) Scan() int32
pkg text/scanner, method (*Scanner) TokenText() string
pkg text/scanner, method (Position) String() string
pkg text/scanner, method (Scanner) String() string
pkg text/scanner, type Position struct
pkg text/scanner, type Position struct, Column int