assertNotNull(robotsTxt);
        // Everything should be allowed by default
        assertTrue(robotsTxt.allows("/anything", "AnyBot"));
    }

    public void testParse_malformedCrawlDelay() {
        String robotsTxtContent = "User-agent: TestBot\n" + "Crawl-delay: abc\n" + "Disallow: /test/\n";

        RobotsTxt robotsTxt;