</p>

<h3 id="Letters_and_digits">Letters and digits</h3>

<p>
The underscore character <code>_</code> (U+005F) is considered a letter.
</p>
<pre class="ebnf">
letter        = unicode_letter | "_" .
decimal_digit = "0" … "9" .
binary_digit  = "0" | "1" .
octal_digit   = "0" … "7" .
hex_digit     = "0" … "9" | "A" … "F" | "a" … "f" .
</pre>

	s.IsIdentRune = isIdentRune
	return &Tokenizer{
		s:    &s,
		base: src.NewFileBase(name, objabi.AbsFile(objabi.WorkingDir(), name, *flags.TrimPath)),
		line: 1,
		file: file,
	}
}

// We want center dot (·) and division slash (∕) to work as identifier characters.
func isIdentRune(ch rune, i int) bool {
	if unicode.IsLetter(ch) {
		return true
	}
	switch ch {
	case '_': // Underscore; traditional.
		return true

America/Mexico_City America/Miquelon America/Moncton America/Monterrey America/Montevideo America/Montreal America/Montserrat America/Nassau America/New_York America/Nipigon America/Nome America/Noronha America/North_Dakota/Beulah America/North_Dakota/Center America/North_Dakota/New_Salem America/Nuuk America/Ojinaga America/Panama America/Pangnirtung America/Paramaribo America/Phoenix America/Port-au-Prince America/Port_of_Spain America/Porto_Acre America/Porto_Velho America/Puerto_Rico America/Punta_Arenas...

	// If we see an error at not-str-lit:xxx, the corresponding name is not a string literal.
	//
	// The specific input forms are chosen so that they are valid C syntax regardless of
	// whether name denotes a type or an expression.

	var b bytes.Buffer
	b.WriteString(builtinProlog)
	b.WriteString(f.Preamble)

	for i, n := range names {
		fmt.Fprintf(&b, "#line %d \"not-declared\"\n"+

</p>

<h3 id="Letters_and_digits">Letters and digits</h3>

<p>
The underscore character <code>_</code> (U+005F) is considered a lowercase letter.
</p>
<pre class="ebnf">
letter        = unicode_letter | "_" .
decimal_digit = "0" … "9" .
binary_digit  = "0" | "1" .
octal_digit   = "0" … "7" .
hex_digit     = "0" … "9" | "A" … "F" | "a" … "f" .
</pre>

	// final release step of cat next/*.txt >go1.X.txt.
	// If the files don't end in full lines, the concatenation goes awry.
	if strings.Contains(s, "\r") {
		log.Printf("%s: contains CRLFs", filename)
		exitCode = 1
	}
	if filepath.Base(filename) == "go1.4.txt" {
		// No use for blank lines in api files, except go1.4.txt
		// used them in a reasonable way and we should let it be.

package lex

import (
	"strings"
	"testing"
	"text/scanner"
)

type lexTest struct {
	name   string
	input  string
	output string
}

var lexTests = []lexTest{
	{
		"empty",
		"",
		"",
	},
	{
		"simple",
		"1 (a)",
		"1.(.a.)",
	},
	{
		"simple define",
		lines(
			"#define A 1234",
			"A",
		),
		"1234.\n",
	},
	{
		"define without value",