assertEquals(2, robotsTxt.getCrawlDelay("MixedCaseBot"));

        // Test CommentBot - inline comments should be stripped
        assertFalse(robotsTxt.allows("/path1/", "CommentBot"));
        assertTrue(robotsTxt.allows("/path2/", "CommentBot"));

        // Test EmptyLineBot - empty lines should not cause issues
        assertFalse(robotsTxt.allows("/test/", "EmptyLineBot"));

UsEr-AgEnT: MixedCaseBot
DiSaLlOw: /test1/
AlLoW: /test2/
CrAwL-dElAy: 2
SiTeMaP: http://example.com/sitemap.xml

# Case 6: Comments in various positions
User-agent: CommentBot # inline comment
Disallow: /path1/ # another comment
# Full line comment
Allow: /path2/

# Case 7: Empty lines and whitespace-only lines
User-agent: EmptyLineBot


Disallow: /test/

Allow: /public/