Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */
    protected static final Pattern DISALLOW_RECORD = Pattern.compile("^disallow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing allow records. */

    /**
     * Pattern for XML encoding.
     */
    protected Pattern xmlEncodingPattern =
            Pattern.compile("<\\?xml.*encoding\\s*=\\s*['\"]([\\w\\d\\-_]*)['\"]\\s*\\?>", Pattern.MULTILINE | Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for XML tags.
     */
    protected Pattern xmlTagPattern = Pattern.compile("<[^>]+>");

    /**
     * Returns the encoding pattern.
     * @return The encoding pattern.
     */

     * @param password the password to use for matching files
     */
    public void addPassword(final String regex, final String password) {
        passwordMap.put(Pattern.compile(regex), password);
    }

    /**
     * Returns the password for the given parameters.
     * @param params The parameters.
     * @return The password.
     */

     *
     * @see org.codelibs.fess.crawler.filter.UrlFilter#addExclude(java.lang.String)
     */
    @Override
    public void addExclude(final String urlPattern) {
        try {
            Pattern.compile(urlPattern);
        } catch (final Exception e) {
            if (logger.isWarnEnabled()) {
                logger.warn("Invalid exclude pattern: {}", urlPattern);
            }
            return;
        }

            if (pos == current) {
                newClientMap.put(Pattern.compile(regex), client);
                added = true;
            }
            newClientMap.put(entry.getKey(), entry.getValue());
            current++;
        }
        if (!added) {
            newClientMap.put(Pattern.compile(regex), client);
        }
        clientMap = newClientMap;
    }

    /**

     */
    public synchronized void addIncludeUrlPattern(final String sessionId, final String url) {
        final List<Pattern> patternList = getIncludeUrlPatternList(sessionId);
        patternList.add(Pattern.compile(url));
    }

    /**
     * Returns the list of include URL patterns for the specified session.
     * Creates a new list if one doesn't exist.
     * @param sessionId the session ID

    /** Pattern for extracting charset from meta tags. */
    protected Pattern metaCharsetPattern = Pattern.compile("<meta.*content\\s*=\\s*['\"].*;\\s*charset=([\\w\\d\\-_]*)['\"]\\s*/?>",
            Pattern.MULTILINE | Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for HTML tags.
     */
    protected Pattern htmlTagPattern = Pattern.compile("<[^>]+>");

    /** Map of parser features. */

                        return getList(OpenSearchUrlFilter.class, key, QueryBuilders.termQuery(FILTER_TYPE, type), null, maxLoadSize, null)
                                .stream()
                                .map(f -> Pattern.compile(f.getUrl()))
                                .collect(Collectors.toList());
                    }
                });
    }

    /**
     * Adds an include URL filter for the specified session.
     *

     * If you want to follow a html spec, use 512. */
    protected int preloadSizeForCharset = 2048;

    /**
     * Pattern for invalid URLs.
     */
    protected Pattern invalidUrlPattern = Pattern.compile("^\\s*javascript:|" //
            + "^\\s*mailto:|" //
            + "^\\s*irc:|" //
            + "^\\s*skype:|" //
            + "^\\s*about:|" + "^\\s*fscommand:|" //
            + "^\\s*aim:|" //

     * @param key the field name to match against
     * @param regex the regular expression pattern
     */
    public void addRule(final String key, final String regex) {
        regexMap.put(key, Pattern.compile(regex));
    }

    /**
     * Adds a compiled regular expression rule for the specified field.
     * @param key the field name to match against
     * @param pattern the compiled regular expression pattern