assertFalse(robotsTxt.allows("/admin/secret.html", "MyBot"));
        assertTrue(robotsTxt.allows("/public/", "MyBot")); // Not disallowed
    }

    public void test_allowsWithAllowOverridingDisallow() {
        // Test that allow takes precedence over disallow
        RobotsTxt robotsTxt = new RobotsTxt();

        Directive directive = new Directive("MyBot");
        directive.addDisallow("/admin/");

StandardCrawlerContainer container = new StandardCrawlerContainer();

// Configure HTTP client with custom settings
container.singleton("httpClient", HcHttpClient.class, client -> {
    client.setUserAgent("MyBot/1.0");
    client.setConnectionTimeout(30000);
    client.setMaxConnections(100);
});

// Configure URL filtering
container.singleton("urlFilter", UrlFilterImpl.class, filter -> {