// for our purposes and made a TokenReader. It forms the lowest level,
// turning text from readers into tokens.
type Tokenizer struct {
	tok  ScanToken
	s    *scanner.Scanner
	base *src.PosBase
	line int
	file *os.File // If non-nil, file descriptor to close.
}

func NewTokenizer(name string, r io.Reader, file *os.File) *Tokenizer {
	var s scanner.Scanner
	s.Init(r)

		zb0001Len--
		zb0001Mask |= 0x1
	}
	// variable map header, size zb0001Len
	o = append(o, 0x80|uint8(zb0001Len))
	if zb0001Len == 0 {
		return
	}
	if (zb0001Mask & 0x1) == 0 { // if not omitted
		// string "V2Obj"
		o = append(o, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a)
		if z.ObjectV2 == nil {
			o = msgp.AppendNil(o)
		} else {
			// map header, size 1
			// string "DDir"

package cmd

// Code generated by github.com/tinylib/msgp DO NOT EDIT.

import (
	"github.com/tinylib/msgp/msgp"
)

// DecodeMsg implements msgp.Decodable
func (z *nsScannerOptions) DecodeMsg(dc *msgp.Reader) (err error) {
	var field []byte
	_ = field
	var zb0001 uint32
	zb0001, err = dc.ReadMapHeader()
	if err != nil {
		err = msgp.WrapError(err)
		return
	}
	for zb0001 > 0 {
		zb0001--
		field, err = dc.ReadMapKeyPtr()

	o = msgp.Require(b, z.Msgsize())
	// map header, size 4
	// string "APIVersion"
	o = append(o, 0x84, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
	o = msgp.AppendString(o, z.APIVersion)
	// string "Flags"
	o = append(o, 0xa5, 0x46, 0x6c, 0x61, 0x67, 0x73)
	// map header, size 3
	// string "Filter"
	o = append(o, 0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)

        final List<ProtwordsItem> itemList = new ArrayList<>();
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, Constants.UTF_8))) {
            long id = 0;
            String line = null;
            while ((line = reader.readLine()) != null) {
                if (line.length() == 0 || line.charAt(0) == '#') {
                    if (updater != null) {

        final List<StopwordsItem> itemList = new ArrayList<>();
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, Constants.UTF_8))) {
            long id = 0;
            String line = null;
            while ((line = reader.readLine()) != null) {
                if (line.length() == 0 || line.charAt(0) == '#') {
                    if (updater != null) {

        File file = new File(path);
        if (!file.exists()) {
            return Collections.emptyList();
        }
        try (FileReader reader = new FileReader(file)) {
            List<UpgradedProperty> upgradedProperties = new Gson().fromJson(reader, new TypeToken<List<UpgradedProperty>>() {}.getType());
            // FIXME There should be no duplicates, yet there are some
            return upgradedProperties.stream()

	}

	timeTierAction := auditTierActions(ctx, oi.TransitionedObject.Tier, length)
	reader, err := tgtClient.Get(ctx, oi.TransitionedObject.Name, remoteVersionID(oi.TransitionedObject.VersionID), gopts)
	if err != nil {
		return nil, err
	}
	closer := func() {
		timeTierAction(reader.Close())
	}
	return fn(reader, h, closer)
}

// RestoreRequestType represents type of restore.
type RestoreRequestType string

type DecoderError struct {
	msg       string // description of error
	context   string // additional error context
	pos       errPos
	atChar    byte
	readerErr error // underlying reader error, if any
}

// ReaderErr returns the underlying error.
func (e DecoderError) ReaderErr() error { return e.readerErr }

// Error returns a string representation of the error.
func (e DecoderError) Error() string {

        final List<StemmerOverrideItem> itemList = new ArrayList<>();
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, Constants.UTF_8))) {
            long id = 0;
            String line = null;
            while ((line = reader.readLine()) != null) {
                // Remove comments