#### Content Processing Pipeline
- **Extractors**: Content extraction from various formats
- **Transformers**: Data transformation and enrichment
- **Filters**: URL filtering with regex patterns
- **Rules**: Content processing rules and validation

## Building and Testing

### Build Commands

```bash
# Build all modules
mvn clean install

# Build without tests
mvn clean install -DskipTests

        return null;
    }

    /**
     * Adds a directive to the robots.txt rules.
     * The user-agent pattern in the directive is converted to a regular expression pattern,
     * where '*' is replaced with '.*' for pattern matching, and stored case-insensitively.
     *
     * @param directive The directive to add to the robots.txt rules
     */
    public void addDirective(final Directive directive) {

        testRule.setRuleId("middle");
        testRule.register(1);

        List<Rule> rules = ruleManager.getRules();
        assertEquals(3, rules.size());
        assertEquals("rule1", rules.get(0).getRuleId());
        assertEquals("middle", rules.get(1).getRuleId());
        assertEquals("rule2", rules.get(2).getRuleId());
    }

    /**
     * Test register method with last index
     */

 *   <li>Extracting child URLs from the HTML content based on configured rules.</li>
 *   <li>Handling redirect URLs specified in the response headers.</li>
 * </ol>
 * <p>
 * The class also provides methods for configuring features and properties of the
 * underlying DOM parser, as well as defining rules for extracting child URLs
 * from specific HTML tags and attributes.
 * </p>
 *
 * <p>

        } catch (final ChildUrlsException e) {
            String[] urls = e.getChildUrlList().stream().map(r -> r.getUrl()).sorted().toArray(String[]::new);
            assertEquals(3, urls.length);
            assertEquals(baseUrl + "dir1/", urls[0]);
            assertEquals(baseUrl + "dir3/", urls[1]);
            assertEquals(baseUrl + "file1.txt", urls[2]);
        }
        try {
            smbClient.doGet(baseUrl + "dir1/");

        List<Rule> rules = ruleManager.getRules();
        assertEquals(4, rules.size());
        assertEquals("rule1", rules.get(0).getRuleId());
        assertEquals("rule2", rules.get(1).getRuleId());
        assertEquals("rule3", rules.get(2).getRuleId());
        assertEquals("rule4", rules.get(3).getRuleId());
    }

    /**
     * Test adding duplicate rules
     */

     */
    @Resource
    protected DataService<AccessResult<?>> dataService;

    /**
     * Filter for URLs to control which URLs are crawled.
     */
    @Resource
    protected UrlFilter urlFilter;

    /**
     * Manager for crawling rules and configurations.
     */
    @Resource
    protected RuleManager ruleManager;

    /**
     * Container for managing crawler components.

        StatefulTransformer transformer = new StatefulTransformer("statefulTransformer");

        // Process multiple URLs
        String[] urls = { "http://example1.com", "http://example2.com", "http://example3.com" };

        for (String url : urls) {
            ResponseData responseData = new ResponseData();
            responseData.setUrl(url);

        List<Rule> rules = new ArrayList<>();

        // Create rules with unique IDs
        for (int i = 0; i < 100; i++) {
            TestRule rule = new TestRule("rule_" + i, new TestResponseProcessor("processor_" + i), true);
            rules.add(rule);
        }

        // Verify all IDs are unique
        for (int i = 0; i < rules.size(); i++) {

        this.path = path;
    }

    /**
     * Gets the replacement string for question marks in URLs.
     *
     * @return the question mark replacement string
     */
    public String getQuestionStr() {
        return questionStr;
    }

    /**
     * Sets the replacement string for question marks in URLs.
     *
     * @param questionStr the question mark replacement string to set
     */