// means that erroneous UTF-8 encodings translate to U+FFFD = "\xef\xbf\xbd".
// Because of the Scan interface, this makes it impossible for the client to
// distinguish correctly encoded replacement runes from encoding errors.
func ScanRunes(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// Fast path 1: ASCII.

		{"InvalidASCII", []byte("foo\xffbar")},
		{"InvalidNonASCII", []byte("日本語\xff日本語")},
	}
	replacement := []byte("\uFFFD")
	b.ResetTimer()
	for _, test := range tests {
		b.Run(test.name, func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				ToValidUTF8(test.input, replacement)
			}
		})
	}
}

func makeBenchInputHard() []byte {
	tokens := [...]string{

	// Invalid runes, including negative ones, should be written as the
	// replacement character.
	for _, r := range []rune{-1, utf8.MaxRune + 1} {
		var buf strings.Builder
		w := NewWriter(&buf)
		w.WriteRune(r)
		w.Flush()
		if s := buf.String(); s != "\uFFFD" {
			t.Errorf("WriteRune(%d) wrote %q, not replacement character", r, s)
		}
	}
}

func TestReadStringAllocs(t *testing.T) {

and the second value, of type <code>rune</code>, will be the value of
the corresponding code point.  If the iteration encounters an invalid
UTF-8 sequence, the second value will be <code>0xFFFD</code>,
the Unicode replacement character, and the next iteration will advance
a single byte in the string.
</li>

<li>
The iteration order over maps is not specified
and is not guaranteed to be the same from one iteration to the next.

			comment: "in the 世界",
			flags:   0x808,
		},
		{
			name:    "hi, こんにちわ",
			comment: "in the 世界",
			flags:   0x808,
		},
		{
			name:    "the replacement rune is �",
			comment: "the replacement rune is �",
			flags:   0x808,
		},
		{
			// Name is Japanese encoded in Shift JIS.
			name:    "\x93\xfa\x96{\x8c\xea.txt",
			comment: "in the 世界",

}

// mangle replaces references to C names in arg with the mangled names,
// rewriting calls when it finds them.
// It removes the corresponding references in f.Ref and f.Calls, so that we
// don't try to do the replacement again in rewriteRef or rewriteCall.
// If addPosition is true, add position info to the idents of C names in arg.
func (p *Package) mangle(f *File, arg *ast.Expr, addPosition bool) (ast.Expr, bool) {
	needsUnsafe := false

}

// ToValidUTF8 treats s as UTF-8-encoded bytes and returns a copy with each run of bytes
// representing invalid UTF-8 replaced with the bytes in replacement, which may be empty.
func ToValidUTF8(s, replacement []byte) []byte {
	b := make([]byte, 0, len(s)+len(replacement))
	invalid := false // previous byte was from an invalid UTF-8 sequence
	for i := 0; i < len(s); {
		c := s[i]
		if c < utf8.RuneSelf {
			i++

and the second value, of type <code>rune</code>, will be the value of
the corresponding code point. If the iteration encounters an invalid
UTF-8 sequence, the second value will be <code>0xFFFD</code>,
the Unicode replacement character, and the next iteration will advance
a single byte in the string.
</li>

<li>
The iteration order over maps is not specified
and is not guaranteed to be the same from one iteration to the next.