assertTrue(crawlerContext.getRobotsTxtUrlSet().contains("http://localhost:7070/robots.txt"));
            assertFalse(urlFilter.match("http://localhost:7070/admin/"));
            assertFalse(urlFilter.match("http://localhost:7070/websvn/"));
        } finally {
            server.stop();
        }
    }

    public void test_processRobotsTxt_disabled() {

    }

    /**
     * Test match method implementation
     */
    public void test_match_implementation() {
        ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com");

        // Test default match (true)
        assertTrue(testRule.match(responseData));
        assertEquals(1, testRule.getMatchCallCount());

        // Without any patterns, all URLs should match
        assertTrue(urlFilter.match("https://example.com/"));
        assertTrue(urlFilter.match("https://test.com/page"));
        assertTrue(urlFilter.match("ftp://files.com/document.pdf"));
        assertTrue(urlFilter.match("file:///home/user/file.txt"));
    }

    /**
     * Test match with complex URL patterns
     */

                        } catch (final Exception e1) {
                            log(logHelper, LogType.CRAWLING_EXCEPTION, crawlerContext, urlQueue, e1);
                        }
                        if (noWaitOnFolder) {
                            continue;
                        }
                    } catch (final CrawlingAccessException e) {

            fail("Should throw IndexOutOfBoundsException for negative index");
        } catch (IndexOutOfBoundsException e) {
            // Expected
        }

        try {
            ruleManager.addRule(5, rule2);
            fail("Should throw IndexOutOfBoundsException for index > size");
        } catch (IndexOutOfBoundsException e) {
            // Expected
        }
    }

    /**

## Contributing

We welcome contributions to Fess Crawler! Please follow these guidelines:

1. **Fork** the repository
2. **Create** a feature branch (`git checkout -b feature/amazing-feature`)
3. **Commit** your changes (`git commit -m 'Add amazing feature'`)
4. **Push** to the branch (`git push origin feature/amazing-feature`)
5. **Open** a Pull Request

### Development Setup

```bash
# Clone the repository

        responseData.setUrl("http://example.com");

        assertTrue(rule.match(responseData));
        assertTrue(rule.match(responseData));
        assertTrue(rule.match(responseData));
        assertEquals(3, rule.getMatchCallCount());
    }

    /**
     * Test Rule with always false match
     */
    public void test_ruleNeverMatch() {

 * </p>
 *
 */
public class HtmlXpathExtractor extends AbstractXmlExtractor {
    /**
     * Regular expression pattern to match the charset attribute in the meta tag of HTML documents.
     * The pattern captures the charset value specified in the content attribute of the meta tag.

            return response;
        } catch (final OpenSearchStatusException e) {
            if (e.status() == RestStatus.CONFLICT) {
                throw new CrawlingAccessException("[" + e.status() + "] Failed to insert " + id, e);
            }
            throw new OpenSearchAccessException("[" + e.status() + "] Failed to insert " + id, e);
        } catch (final Exception e) {

                            Thread.sleep(10);
                        } catch (InterruptedException e) {
                            Thread.currentThread().interrupt();
                            break;
                        }
                    }
                    statusReadLatch.countDown();
                } catch (Exception e) {
                    statusReadLatch.countDown();
                }