assertNotNull(tokens);
        assertEquals(3, tokens.size());
        assertEquals(0, tokens.get(0).getPosition());
        assertEquals(1, tokens.get(1).getPosition());
        assertEquals(2, tokens.get(2).getPosition());
    }

    @Test
    public void testAnalyzeTokenOffsets() {
        // Test that token offsets are correct
        String text = "one two three";