# Case 4: Extra whitespace
User-agent:    ExtraSpaceBot
Disallow:     /spaced/
   Allow:   /also-spaced/

# Case 5: Mixed case directives (should still work)
UsEr-AgEnT: MixedCaseBot
DiSaLlOw: /test1/
AlLoW: /test2/
CrAwL-dElAy: 2
SiTeMaP: http://example.com/sitemap.xml

# Case 6: Comments in various positions
User-agent: CommentBot # inline comment
Disallow: /path1/ # another comment
# Full line comment
Allow: /path2/

                new String[] { "tag1" }, new String[] { SuggestConstants.DEFAULT_ROLE }, null, SuggestItem.Kind.DOCUMENT);

        String[][] readings2 = new String[1][];
        readings2[0] = new String[] { "test2" };
        SuggestItem item2 = new SuggestItem(new String[] { "テスト2" }, readings2, new String[] { "content" }, 1, 0, -1,

                new ElevateWord("test1", 2.0f, Collections.singletonList("test1"), Collections.singletonList("content"), null, null);
        ElevateWord elevateWord2 =
                new ElevateWord("test2", 3.0f, Collections.singletonList("test2"), Collections.singletonList("content"), null, null);

        suggester.settings().elevateWord().add(elevateWord1);
        suggester.settings().elevateWord().add(elevateWord2);

        // Test MixedCaseBot - mixed case directives should work
        assertFalse(robotsTxt.allows("/test1/", "MixedCaseBot"));
        assertTrue(robotsTxt.allows("/test2/", "MixedCaseBot"));
        assertEquals(2, robotsTxt.getCrawlDelay("MixedCaseBot"));

        // Test CommentBot - inline comments should be stripped
        assertFalse(robotsTxt.allows("/path1/", "CommentBot"));