}

    /**
     * Strips invalid XML characters from the input string.
     *
     * This method removes characters that are not allowed in XML documents
     * according to the XML 1.0 specification. Valid characters include:
     * - Tab (0x9)
     * - Line feed (0xA)
     * - Carriage return (0xD)
     * - Any character between 0x20 and 0xD7FF
     * - Any character between 0xE000 and 0xFFFD

        assertEquals(100, crawlerContext.getRobotsTxtUrlSet().size());
        assertNotNull(crawlerContext.removeSitemaps());
    }

    /**
     * Test boundary values
     */
    public void test_boundaryValues() {
        // Test with maximum values
        crawlerContext.setActiveThreadCount(Integer.MAX_VALUE);
        assertEquals(Integer.MAX_VALUE, crawlerContext.getActiveThreadCount().intValue());

    transformer.setMaxContentSize(10 * 1024 * 1024); // 10MB limit
});
```

## Troubleshooting

### Common Issues

#### Connection Timeouts
```java
// Increase timeout values
client.setConnectionTimeout(60000);  // 60 seconds
client.setSocketTimeout(120000);     // 120 seconds
```

#### Memory Issues
```java
// Reduce concurrent threads and batch sizes

                                }
                            } catch (final NumberFormatException e) {
                                // ignore invalid crawl-delay values
                            }
                        }
                    } else if ((value = getValue(SITEMAP_RECORD, line)) != null && value.length() > 0) {
                        robotsTxt.addSitemap(value);

    public void setUp() {
        testAnalyzer = new TestSuggestAnalyzer();
        analyzer = testAnalyzer;
    }

    @Test
    public void testAnalyzeWithValidInput() {
        // Test normal analyze with valid input
        String text = "test text";
        String field = "content";
        String lang = "en";

        List<AnalyzeToken> tokens = analyzer.analyze(text, field, lang);

        assertNotNull(tokens);

 * only one of them (allRequired = false). It also supports a default rule that always matches.
 *
 * <p>
 * The class uses a map of field names to Pattern objects to store the regular expressions.
 * The match method extracts the values of the specified fields from the ResponseData and
 * applies the corresponding regular expressions.
 * </p>
 *
 * <p>
 * Example usage:
 * </p>
 *
 * <pre>
 * {@code
 * RegexRule rule = new RegexRule();

        assertNotNull(exception);
        assertEquals(message, exception.getMessage());
        assertSame(cause, exception.getCause());
    }

    /**
     * Test constructor with null message and valid cause
     */
    public void test_constructor_withNullMessageAndValidCause() {
        Exception cause = new IllegalStateException("State error");

        // Default constructor
    }

    @Override
    /**
     * Sets the initialization parameters for the underlying CrawlerClient.
     *
     * @param params a map of parameter names and values to be set
     */
    public void setInitParameterMap(final Map<String, Object> params) {
        client.setInitParameterMap(params);
    }

    /**
     * Executes the request with retry logic.

     */
    public Directive getDirective(final String userAgent) {
        if (userAgent == null) {
            return null;
        }
        for (final Directive directive : directiveMap.values()) {
            if (userAgent.equals(directive.getUserAgent())) {
                return directive;
            }
        }
        return null;
    }

    /**

        final CountDownLatch startLatch = new CountDownLatch(1);
        final CountDownLatch endLatch = new CountDownLatch(threadCount);
        final AtomicInteger errorCount = new AtomicInteger(0);

        // Set initial values
        testRule.setRuleId("concurrentRule");
        TestResponseProcessor processor = new TestResponseProcessor();
        testRule.setResponseProcessor(processor);

        for (int i = 0; i < threadCount; i++) {