// Allow should take precedence
        assertTrue(robotsTxt.allows("/admin/public/page.html", "MyBot"));
        assertFalse(robotsTxt.allows("/admin/secret.html", "MyBot"));
    }

    public void test_getCrawlDelayWithNoDirective() {
        // Test getCrawlDelay when no directive matches
        RobotsTxt robotsTxt = new RobotsTxt();

        assertEquals(0, robotsTxt.getCrawlDelay("MyBot"));
    }