} catch (final Exception e) {
                logger.warn("Failed to add exclude_urls on " + sessionId, e);
            }
            cachedExcludeSet.clear();
        }
    }

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.fess.crawler.filter.UrlFilter#match(java.lang.String)
     */
    @Override
    public boolean match(final String url) {

            assertTrue(crawlerContext.getRobotsTxtUrlSet().contains("http://localhost:7070/robots.txt"));
            assertFalse(urlFilter.match("http://localhost:7070/admin/"));
            assertFalse(urlFilter.match("http://localhost:7070/websvn/"));
        } finally {
            server.stop();
        }
    }

    public void test_processRobotsTxt_disabled() {

                creator.register(this);
            }
        } catch (final Exception e) {
            if (logger.isDebugEnabled()) {
                logger.debug("CrawlerClientCreator is unavailable.", e);
            }
        }
    }

    /**
     * Adds a client with a regular expression pattern.
     * @param regex The regular expression to match URLs.
     * @param client The CrawlerClient instance.
     */

    }

    /**
     * Test match method implementation
     */
    public void test_match_implementation() {
        ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com");

        // Test default match (true)
        assertTrue(testRule.match(responseData));
        assertEquals(1, testRule.getMatchCallCount());

        // Without any patterns, all URLs should match
        assertTrue(urlFilter.match("https://example.com/"));
        assertTrue(urlFilter.match("https://test.com/page"));
        assertTrue(urlFilter.match("ftp://files.com/document.pdf"));
        assertTrue(urlFilter.match("file:///home/user/file.txt"));
    }

    /**
     * Test match with complex URL patterns
     */

     */
    @Override
    public boolean match(final ResponseData responseData) {
        if (super.match(responseData)) {
            try (final InputStream is = responseData.getResponseBody()) {
                final SitemapsHelper sitemapsHelper = crawlerContainer.getComponent("sitemapsHelper");
                return sitemapsHelper.isValid(is);
            } catch (final CrawlingAccessException e) {
                throw e;

 *   <li>Dynamic passwords provided through extraction parameters</li>
 * </ul>
 *
 * <p>Passwords are matched against URLs or resource names using regular expression patterns.
 * The extractor first tries to match against the URL, then falls back to the resource name if available.
 *
 * @author shinsuke
 */
public abstract class PasswordBasedExtractor extends AbstractExtractor {

    /** Logger instance for this class. */

                        } catch (final Exception e1) {
                            log(logHelper, LogType.CRAWLING_EXCEPTION, crawlerContext, urlQueue, e1);
                        }
                        if (noWaitOnFolder) {
                            continue;
                        }
                    } catch (final CrawlingAccessException e) {

            fail("Should throw IndexOutOfBoundsException for negative index");
        } catch (IndexOutOfBoundsException e) {
            // Expected
        }

        try {
            ruleManager.addRule(5, rule2);
            fail("Should throw IndexOutOfBoundsException for index > size");
        } catch (IndexOutOfBoundsException e) {
            // Expected
        }
    }

    /**

        }
    }

    /**
     * Registers a client component with a regular expression.
     * The component will be loaded into all registered CrawlerClientFactories.
     * @param regex The regular expression to match URLs.
     * @param componentName The name of the component to register.
     */
    public synchronized void register(final String regex, final String componentName) {
        clientMap.put(regex, componentName);