func (t *Tokenizer) Base() *src.PosBase {
	return t.base
}

func (t *Tokenizer) SetBase(base *src.PosBase) {
	t.base = base
}

func (t *Tokenizer) Line() int {
	return t.line
}

func (t *Tokenizer) Col() int {
	return t.s.Pos().Column
}

func (t *Tokenizer) Next() ScanToken {
	s := t.s
	for {
		t.tok = ScanToken(s.Scan())
		if t.tok != scanner.Comment {
			break
		}
		text := s.TokenText()

	// Helper function to register a collector and return a gatherer for it.
	mustRegister := func(c ...prometheus.Collector) prometheus.Gatherer {
		subRegistry := prometheus.NewRegistry()
		for _, col := range c {
			subRegistry.MustRegister(col)
		}
		r.MustRegister(subRegistry)
		return subRegistry
	}

	// Register all collectors and create gatherers for them.

			results[field.DBName] = result
		} else if table, col := matchName(column); col != "" && (table == stmt.Table || table == "") {
			if col == "*" {
				for _, dbName := range stmt.Schema.DBNames {
					results[dbName] = result
				}
			} else {
				results[col] = result
			}
		} else {
			results[column] = result
		}
	}

	// select columns

// The argument list is nil for no parens on the definition; otherwise a list of
// formal argument names.
func (in *Input) macroDefinition(name string) ([]string, []Token) {
	prevCol := in.Stack.Col()
	tok := in.Stack.Next()
	if tok == '\n' || tok == scanner.EOF {
		return nil, nil // No definition for macro
	}
	var args []string
	// The C preprocessor treats
	//	#define A(x)
	// and

		),
		"THIS.\n",
	},
	{
		"nested #define with args",
		lines(
			"#define A #define B(x) x",
			"A",
			"B(THIS)",
		),
		"THIS.\n",
	},
	/* This one fails. See comment in Slice.Col.
	{
		"nested #define with args",
		lines(
			"#define A #define B (x) x",
			"A",
			"B(THIS)",
		),
		"x.\n",
	},
	*/
}

func TestLex(t *testing.T) {

caug9?de?ja?rg??v&da?ed?og!.&a&b?m?p??bp?c&a?s??e&c?p?s??fd?gm?ip?jr?la?ma?nr?o&g?r?t??p&a?s??r&p?r??s&e?m?r??tm???rs?t??xiv?z&hb?ls?o&c?f??yx????c!.&as?ca?de?if?o&c?g??ro???e&bew?ccos?e&b?n&igne?oip??rac??gni&arg?rheob??h&sok?t&aew?orb???itnorf?k&col?o&p?rb???l&aed?ffeahcs??mal?nes?pinuj?t&a&eht?rebsnegömrev??law?nec?s&nom?ubkcolb??upmoc??v&o&csid?rdnal??resbo??wulksretlow?ywal?zifp??f!.&aterg?bew&-no,etis321,?drp?e&c&itsuj-reissiuh?narf-ne-setsitned-sneigrurihc,?lipuog,rianiretev,?hny,i&cc?rga...

co.vi
co.za
co.zm
co.zw
coach
cockpit.fr-par.scw.cloud
cockpit.nl-ams.scw.cloud
cockpit.pl-waw.scw.cloud
cocotte.jp
codeberg.page
codes
codespot.com
coffee
cog.mi.us
col.ng
college
collegefan.org
cologne
com
com.ac
com.af
com.ag
com.ai
com.al
com.am
com.ar
com.au
com.aw
com.az
com.ba
com.bb
com.bh
com.bi

co.vi
co.za
co.zm
co.zw
coach
cockpit.fr-par.scw.cloud
cockpit.nl-ams.scw.cloud
cockpit.pl-waw.scw.cloud
cocotte.jp
codeberg.page
codes
codespot.com
coffee
cog.mi.us
col.ng
college
collegefan.org
cologne
com
com.ac
com.af
com.ag
com.ai
com.al
com.am
com.ar
com.au
com.aw
com.az
com.ba
com.bb
com.bh
com.bi

// Submitted by Atanunu Igbunuroghene <******@****.***>
orx.biz
biz.gl
biz.ng
co.biz.ng
dl.biz.ng
go.biz.ng
lg.biz.ng
on.biz.ng
col.ng
firm.ng
gen.ng
ltd.ng
ngo.ng
plc.ng

// HostFly : https://www.ie.ua
// Submitted by Bohdan Dub <******@****.***>
ie.ua