s := t.s
	for {
		t.tok = ScanToken(s.Scan())
		if t.tok != scanner.Comment {
			break
		}
		text := s.TokenText()
		t.line += strings.Count(text, "\n")
		if constraint.IsGoBuild(text) {
			t.tok = BuildComment
			break
		}
	}
	switch t.tok {
	case '\n':
		t.line++
	case '-':
		if s.Peek() == '>' {
			s.Next()
			t.tok = ARR
			return ARR
		}
	case '@':

                if (minorVersion == null) {
                    fallback = true;
                }
            }
            if (idx < tok.length) {
                incrementalVersion = getNextIntegerToken(tok[idx++]);
                if (incrementalVersion == null) {
                    fallback = true;
                }
            }
            if (idx < tok.length) {
                qualifier = tok[idx];

	s.tr = append(s.tr, tr)
}

func (s *Stack) Next() ScanToken {
	tos := s.tr[len(s.tr)-1]
	tok := tos.Next()
	for tok == scanner.EOF && len(s.tr) > 1 {
		tos.Close()
		// Pop the topmost item from the stack and resume with the next one down.
		s.tr = s.tr[:len(s.tr)-1]
		tok = s.Next()
	}
	return tok
}

func (s *Stack) Text() string {
	return s.tr[len(s.tr)-1].Text()
}

		result := int64(p.expr())
		if result != test.output {
			t.Errorf("%d: %q evaluated to %d; expected %d", i, test.input, result, test.output)
		}
		tok := p.next()
		if test.atEOF && tok.ScanToken != scanner.EOF {
			t.Errorf("%d: %q: at EOF got %s", i, test.input, tok)
		} else if !test.atEOF && tok.ScanToken == scanner.EOF {
			t.Errorf("%d: %q: expected not EOF but at EOF", i, test.input)
		}
	}
}

type badExprTest struct {

}

// drain returns a single string representing the processed input tokens.
func drain(input *Input) string {
	var buf strings.Builder
	for {
		tok := input.Next()
		if tok == scanner.EOF {
			return buf.String()
		}
		if tok == '#' {
			continue
		}
		if buf.Len() > 0 {
			buf.WriteByte('.')
		}
		buf.WriteString(input.Text())
	}
}

type badLexTest struct {

     */
    static class TestSpnegoToken extends SpnegoToken {
        private boolean parsed = false;

        @Override
        public byte[] toByteArray() {
            byte[] tok = getMechanismToken();
            return tok == null ? new byte[0] : tok;
        }

        @Override
        protected void parse(byte[] token) throws IOException {
            if (token == null) {
                throw new IOException("token is null");

func Tokenize(str string) []Token {
	t := NewTokenizer("command line", strings.NewReader(str), nil)
	var tokens []Token
	for {
		tok := t.Next()
		if tok == scanner.EOF {
			break
		}
		tokens = append(tokens, Make(tok, t.Text()))
	}
	return tokens

                    byte[] tok = new byte[] { 0x04, 0x01 };
                    byte[] buf = new byte[2 + 2 + der.length + 4];
                    int p = 0;
                    buf[p++] = tok[0];
                    buf[p++] = tok[1];
                    buf[p++] = 0x00;
                    buf[p++] = (byte) der.length;

        final String p = props.getProperty(key);
        if (p != null) {
            final StringTokenizer tok = new StringTokenizer(p, delim);
            final int len = tok.countTokens();
            final InetAddress[] arr = new InetAddress[len];
            for (int i = 0; i < len; i++) {
                final String addr = tok.nextToken();
                try {
                    arr[i] = InetAddress.getByName(addr);