var s scanner.Scanner
	s.Init(r)
	// Newline is like a semicolon; other space characters are fine.
	s.Whitespace = 1<<'\t' | 1<<'\r' | 1<<' '
	// Don't skip comments: we need to count newlines.
	s.Mode = scanner.ScanChars |
		scanner.ScanFloats |
		scanner.ScanIdents |
		scanner.ScanInts |
		scanner.ScanStrings |
		scanner.ScanComments
	s.Position.Filename = name
	s.IsIdentRune = isIdentRune
	return &Tokenizer{
		s:    &s,

	ScanFloats     = 1 << -Float // includes Ints and hexadecimal floats
	ScanChars      = 1 << -Char
	ScanStrings    = 1 << -String
	ScanRawStrings = 1 << -RawString
	ScanComments   = 1 << -Comment
	SkipComments   = 1 << -skipComment // if set with ScanComments, comments become white space
	GoTokens       = ScanIdents | ScanFloats | ScanChars | ScanStrings | ScanRawStrings | ScanComments | SkipComments
)

	// Don't test ScanInts and ScanNumbers since some parts of
	// the floats in the source look like (invalid) octal ints
	// and ScanNumbers may return either Int or Float.
	testScanSelectedMode(t, ScanChars, Char)
	testScanSelectedMode(t, ScanStrings, String)
	testScanSelectedMode(t, SkipComments, 0)
	testScanSelectedMode(t, ScanComments, Comment)
}

func TestScanCustomIdent(t *testing.T) {

		{"Position", Type, 0},
		{"Position.Column", Field, 0},
		{"Position.Filename", Field, 0},
		{"Position.Line", Field, 0},
		{"Position.Offset", Field, 0},
		{"RawString", Const, 0},
		{"ScanChars", Const, 0},
		{"ScanComments", Const, 0},
		{"ScanFloats", Const, 0},
		{"ScanIdents", Const, 0},
		{"ScanInts", Const, 0},
		{"ScanRawStrings", Const, 0},
		{"ScanStrings", Const, 0},

pkg text/scanner, const GoWhitespace ideal-int
pkg text/scanner, const Ident ideal-int
pkg text/scanner, const Int ideal-int
pkg text/scanner, const RawString ideal-int
pkg text/scanner, const ScanChars ideal-int
pkg text/scanner, const ScanComments ideal-int
pkg text/scanner, const ScanFloats ideal-int
pkg text/scanner, const ScanIdents ideal-int
pkg text/scanner, const ScanInts ideal-int

pkg text/scanner, const GoWhitespace = 4294977024
pkg text/scanner, const Ident = -2
pkg text/scanner, const Int = -3
pkg text/scanner, const RawString = -7
pkg text/scanner, const ScanChars = 32
pkg text/scanner, const ScanComments = 256
pkg text/scanner, const ScanFloats = 16
pkg text/scanner, const ScanIdents = 4
pkg text/scanner, const ScanInts = 8
pkg text/scanner, const ScanRawStrings = 128