/**
     * @throws Exception
     */
    @Test
    public void pend_testDot() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("abc.hoge");
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_WORD));
        assertThat(tokenizer.getStringValue(), is("abc.hoge"));
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
    }

     */
    public static final int TT_EOF = -1;

    /**
     * Quoteをあらわします。
     */
    public static final int TT_QUOTE = '\'';

    /**
     * 単語をあらわします。
     */
    public static final int TT_WORD = -3;

    private static final int TT_NOTHING = -4;

    private static final int NEED_CHAR = Integer.MAX_VALUE;

    private static final int QUOTE = '\'';

    private static final byte CT_WHITESPACE = 1;