assertFalse(robotsTxt.allows("/valid2/", "TestBot"));
        assertTrue(robotsTxt.allows("/valid3/", "TestBot"));
        assertFalse(robotsTxt.allows("/valid4/", "TestBot"));
    }

    public void testParse_emptyValues() {
        String robotsTxtContent = "User-agent: TestBot\n" + "Disallow:\n" + "Allow:\n" + "Crawl-delay:\n";

        RobotsTxt robotsTxt;