}
        }
    }

    /**
     * Adds a client with a regular expression pattern.
     * @param regex The regular expression to match URLs.
     * @param client The CrawlerClient instance.
     */
    public void addClient(final String regex, final CrawlerClient client) {
        if (StringUtil.isBlank(regex)) {
            throw new CrawlerSystemException("A regular expression is null.");
        }

     * Adds a password for files matching the given regular expression pattern.
     * @param regex the regular expression pattern to match against URLs or resource names
     * @param password the password to use for matching files
     */
    public void addPassword(final String regex, final String password) {
        passwordMap.put(Pattern.compile(regex), password);
    }

    /**
     * Returns the password for the given parameters.

     * @param regex The regular expression to match URLs.
     * @param componentName The name of the component to register.
     */
    public synchronized void register(final String regex, final String componentName) {
        clientMap.put(regex, componentName);
        clientFactoryList.forEach(f -> load(f, regex, componentName));
    }

    /**

     * @param regex The regular expression for the client.
     * @param client The CrawlerClient instance.
     */
    @Override
    public void addClient(final String regex, final CrawlerClient client) {
        factory.addClient(regex, client);
    }

    /**
     * Adds a client to the wrapped factory at a specific position.
     * @param regex The regular expression for the client.

    /**
     * Adds a regular expression rule for the specified field.
     * @param key the field name to match against
     * @param regex the regular expression pattern
     */
    public void addRule(final String key, final String regex) {
        regexMap.put(key, Pattern.compile(regex));
    }

    /**
     * Adds a compiled regular expression rule for the specified field.

        assertFalse(urlFilter.match("https://other.com/page.html"));
    }

    /**
     * Test adding invalid regex include pattern
     */
    public void test_addInclude_invalidRegex() {
        String sessionId = "test-session-004";
        urlFilter.init(sessionId);

        // Invalid regex pattern should be handled gracefully
        urlFilter.addInclude(".*[invalid");

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.opensearch.extension.analysis;

import java.util.regex.Pattern;

import org.apache.lucene.analysis.TokenStream;
import org.codelibs.analysis.ja.PatternConcatenationFilter;
import org.opensearch.common.settings.Settings;
import org.opensearch.env.Environment;

 */
package org.codelibs.fess.crawler.filter.impl;

import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.container.CrawlerContainer;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.BiConsumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.input.BOMInputStream;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.RobotsTxt;