assertFalse(urlFilter.match("https://other.com/page.html"));
    }

    /**
     * Test match with no patterns configured
     */
    public void test_match_noPatterns() {
        String sessionId = "test-session-009";
        urlFilter.init(sessionId);

        // Without any patterns, all URLs should match
        assertTrue(urlFilter.match("https://example.com/"));

crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns
crawler.urlFilter.addInclude("https://example.com/.*");
crawler.urlFilter.addInclude(".*\\.pdf$");

// Exclude patterns  
crawler.urlFilter.addExclude(".*\\.js$");
crawler.urlFilter.addExclude(".*login.*");
```

## Supported Protocols and Formats

 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {
    private static final String ALL_BOTS = "*";

    /** Map of user agent patterns to their corresponding directives. */
    protected final Map<Pattern, Directive> directiveMap = new LinkedHashMap<>();

    /** List of sitemap URLs found in the robots.txt file. */

        ConditionalAbstractRule conditionalRule = new ConditionalAbstractRule();
        conditionalRule.crawlerContainer = container;
        conditionalRule.setRuleId("conditionalRule");

        // Set patterns
        conditionalRule.setUrlPattern("https?://.*\\.example\\.com/.*");
        conditionalRule.setMimeTypePattern("text/.*");

        // Test matching
        ResponseData responseData1 = new ResponseData();

/**
 * A container implementation that manages the lifecycle and dependency injection of components
 * in a crawler application. This container supports both singleton and prototype component
 * instantiation patterns.
 *
 * <p>The container provides mechanisms for:
 * <ul>
 *   <li>Registering and retrieving components by name</li>
 *   <li>Managing singleton instances with lifecycle hooks</li>