/**
     * Test adding invalid regex include pattern
     */
    public void test_addInclude_invalidRegex() {
        String sessionId = "test-session-004";
        urlFilter.init(sessionId);

        // Invalid regex pattern should be handled gracefully
        urlFilter.addInclude(".*[invalid");
        urlFilter.addInclude("https://valid.com/.*");

        // Valid pattern should still work

     * The user-agent pattern in the directive is converted to a regular expression pattern,
     * where '*' is replaced with '.*' for pattern matching, and stored case-insensitively.
     *
     * @param directive The directive to add to the robots.txt rules
     */
    public void addDirective(final Directive directive) {

     * @return the parsed charset name, or null if not found
     */
    protected String parseCharset(final String content) {
        final Pattern pattern = Pattern.compile("; *charset *= *([a-zA-Z0-9\\-_]+)", Pattern.CASE_INSENSITIVE);
        final Matcher matcher = pattern.matcher(content);
        if (matcher.find()) {
            return matcher.group(1);
        }
        return null;
    }

    /**

crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns
crawler.urlFilter.addInclude("https://example.com/.*");
crawler.urlFilter.addInclude(".*\\.pdf$");

// Exclude patterns  
crawler.urlFilter.addExclude(".*\\.js$");
crawler.urlFilter.addExclude(".*login.*");
```

## Supported Protocols and Formats

    }

    /**
     * Extracts the token value from the content using the given pattern.
     * @param tokenPattern The regex pattern.
     * @param content The content to search.
     * @return The extracted token value.
     */
    protected String getTokenValue(final String tokenPattern, final String content) {
        final Matcher matcher = Pattern.compile(tokenPattern).matcher(content);
        if (matcher.find()) {

    }

    /**
     * Gets the pattern for extracting charset from meta tags.
     *
     * @return the meta charset pattern
     */
    public Pattern getMetaCharsetPattern() {
        return metaCharsetPattern;
    }

    /**
     * Sets the pattern for extracting charset from meta tags.
     *
     * @param metaCharsetPattern the meta charset pattern to set
     */

        ConditionalAbstractRule conditionalRule = new ConditionalAbstractRule();
        conditionalRule.crawlerContainer = container;
        conditionalRule.setRuleId("conditionalRule");

        // Set patterns
        conditionalRule.setUrlPattern("https?://.*\\.example\\.com/.*");
        conditionalRule.setMimeTypePattern("text/.*");

        // Test matching
        ResponseData responseData1 = new ResponseData();

PopularWordsResponse popularWords = suggester.popularWords()
    .setSize(20)                    // top 20 words
    .setQuery("tech*")              // filter pattern
    .execute()
    .getResponse();

popularWords.getItems().forEach(item -> {
    System.out.println("Popular: " + item.getText() + 
                      " (freq: " + item.getDocFreq() + ")");
});
```

     */
    public XpathTransformer() {
        super();
    }

    private static final Logger logger = LogManager.getLogger(XpathTransformer.class);

    private static final Pattern SPACE_PATTERN = Pattern.compile("\\s+", Pattern.MULTILINE);

    /**
     * A map of field rules, where the key is the field name and the value is the XPath expression.
     */

import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.Base64;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Pattern;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.core.CoreLibConstants;
import org.codelibs.core.lang.StringUtil;