description: |
        Requesting we remove a module here only hides the generated documentation on pkg.go.dev.
        It does not affect the behaviour of proxy.golang.org or the go command.
        Instead we recommend using the retract directive which will be processed by all 3 of the above.

        If you have deleted your repo, please recreate it and publish a retraction.

func NewTokenizer(name string, r io.Reader, file *os.File) *Tokenizer {
	var s scanner.Scanner
	s.Init(r)
	// Newline is like a semicolon; other space characters are fine.
	s.Whitespace = 1<<'\t' | 1<<'\r' | 1<<' '
	// Don't skip comments: we need to count newlines.
	s.Mode = scanner.ScanChars |
		scanner.ScanFloats |
		scanner.ScanIdents |
		scanner.ScanInts |
		scanner.ScanStrings |
		scanner.ScanComments
	s.Position.Filename = name

	// We need to parse the output from this gcc command, so ensure that it
	// doesn't have any ANSI escape sequences in it. (TERM=dumb is
	// insufficient; if the user specifies CGO_CFLAGS=-fdiagnostics-color,
	// GCC will ignore TERM, and GCC can also be configured at compile-time
	// to ignore TERM.)
	stderr := p.gccErrors(b.Bytes(), "-fdiagnostics-color=never")
	if strings.Contains(stderr, "unrecognized command line option") {

## Tools {#tools}

### Go command {#go-command}

package main

import (
	"bufio"
	"bytes"
	"internal/testenv"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
)

// TestMain executes the test binary as the addr2line command if
// GO_ADDR2LINETEST_IS_ADDR2LINE is set, and runs the tests otherwise.
func TestMain(m *testing.M) {
	if os.Getenv("GO_ADDR2LINETEST_IS_ADDR2LINE") != "" {
		main()
		os.Exit(0)
	}

// converted to int they correspond to the rune size that was read.
type readOp int8

// Don't use iota for these, as the values need to correspond with the
// names and comments, which is easier to see when being explicit.
const (
	opRead      readOp = -1 // Any other read operation.
	opInvalid   readOp = 0  // Non-read operation.
	opReadRune1 readOp = 1  // Read rune of size 1.

			//   API surface. (If we are running within the "std" module,
			//   vendored dependencies appear as themselves instead of
			//   their "vendor/" standard-library copies.)
			// - 'go list std' does not include commands, which cannot be
			//   imported anyway.
			if ip := pkg.ImportPath; pkg.Standard && ip != "unsafe" && !strings.HasPrefix(ip, "vendor/") && !internalPkg.MatchString(ip) {
				stdPackages = append(stdPackages, ip)
			}

are also treated as invalid.
(Older toolchains ignore `//go:debug` directives entirely.)

The defaults that will be compiled into a main package
are reported by the command:

{{raw `
	go list -f '{{.DefaultGODEBUG}}' my/main/package
`}}

Only differences from the base Go toolchain defaults are reported.

When testing a package, `//go:debug` lines in the `*_test.go`

	// able to treat them as byte arrays, so the godefs
	// inputs in package syscall say
	//
	//	// +godefs map struct_in_addr [4]byte
	//	// +godefs map struct_in_addr6 [16]byte
	//
	for _, g := range f.Comments {
		for _, c := range g.List {
			i := strings.Index(c.Text, "+godefs map")
			if i < 0 {
				continue
			}
			s := strings.TrimSpace(c.Text[i+len("+godefs map"):])
			i = strings.Index(s, " ")
			if i < 0 {

func ExampleScanner_emptyFinalToken() {
	// Comma-separated list; last entry is empty.
	const input = "1,2,3,4,"
	scanner := bufio.NewScanner(strings.NewReader(input))
	// Define a split function that separates on commas.
	onComma := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		for i := 0; i < len(data); i++ {
			if data[i] == ',' {
				return i + 1, data[:i], nil
			}
		}
		if !atEOF {