assertFalse(robotsTxt.allows("/admin/secret.html", "MyBot"));
        assertTrue(robotsTxt.allows("/public/", "MyBot")); // Not disallowed
    }

    public void test_allowsWithAllowOverridingDisallow() {
        // Test that allow takes precedence over disallow
        RobotsTxt robotsTxt = new RobotsTxt();

        Directive directive = new Directive("MyBot");
        directive.addDisallow("/admin/");