assertTrue(robotsTxt.allows("/also-spaced/", "ExtraSpaceBot"));

        // Test MixedCaseBot - mixed case directives should work
        assertFalse(robotsTxt.allows("/test1/", "MixedCaseBot"));
        assertTrue(robotsTxt.allows("/test2/", "MixedCaseBot"));
        assertEquals(2, robotsTxt.getCrawlDelay("MixedCaseBot"));

        // Test CommentBot - inline comments should be stripped

Allow: /path:with:colons

# Case 4: Extra whitespace
User-agent:    ExtraSpaceBot
Disallow:     /spaced/
   Allow:   /also-spaced/

# Case 5: Mixed case directives (should still work)
UsEr-AgEnT: MixedCaseBot
DiSaLlOw: /test1/
AlLoW: /test2/
CrAwL-dElAy: 2
SiTeMaP: http://example.com/sitemap.xml

# Case 6: Comments in various positions
User-agent: CommentBot # inline comment
Disallow: /path1/ # another comment