import static org.junit.Assert.assertThat;

import org.junit.Test;

/**
 * @author higa
 *
 */
public class TokenizerTest {

    /**
     * @throws Exception
     */
    @Test
    public void testEOF() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("");
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
    }

    /**