*   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

    public static final String EMPTY_STRING = "";

    /** The system property name for user dictionary encoding. */
    public static final String USER_DICT_ENCODING = "fess.user.dict.encoding";

    /** The system property name for user dictionary path. */
    public static final String USER_DICT_PATH = "fess.user.dict.path";

    /** The text separator. */
    public static final String TEXT_SEPARATOR = " ";

- **Asynchronous Operations**: Non-blocking suggestion requests with callback support
- **Index Management**: Automatic index creation, switching, and maintenance
- **Customizable Scoring**: User boost, document frequency, and query frequency weighting

## Technology Stack

- **Java**: 21+ (configured via parent POM)
- **OpenSearch**: Latest (provided scope)
- **Apache Lucene**: Query parsing and text analysis

        // Test Kind enum
        assertEquals("query", SuggestItem.Kind.QUERY.toString());
        assertEquals("document", SuggestItem.Kind.DOCUMENT.toString());
        assertEquals("user", SuggestItem.Kind.USER.toString());
    }

    @Test
    public void testGetUpdatedSource() {
        // Test getUpdatedSource method
        String[] text = { "test" };
        String[][] readings = { { "test" } };

    /**
     * The kind of suggest item.
     */
    public enum Kind {
        /** Document kind. */
        DOCUMENT("document"),
        /** Query kind. */
        QUERY("query"),
        /** User kind. */
        USER("user");

        private final String kind;

        /**
         * Constructor for Kind.
         * @param kind The kind string.
         */
        Kind(final String kind) {

    public void setAuthSchemeProviderMap(final Map<String, AuthSchemeProvider> authSchemeProviderMap) {
        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */
    protected static final Pattern USER_AGENT_RECORD =
            Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */

        String[] values = CSVUtil.parse(rule);
        if (dup.add(values[0]) == false) {
            throw new IllegalArgumentException("Found duplicate term [" + values[0] + "] in user dictionary.");
        }
        return rule;
    }

    public static UserDictionary getUserDictionary(Environment env, Settings settings) {

     */
    public void setCookieSpec(final String cookieSpec) {
        this.cookieSpec = cookieSpec;
    }

    /**
     * Sets the user agent string.
     *
     * @param userAgent The user agent string
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the proxy host name.

 * <li>{@link #SCORE} - The score field.</li>
 * <li>{@link #QUERY_FREQ} - The query frequency field.</li>
 * <li>{@link #DOC_FREQ} - The document frequency field.</li>
 * <li>{@link #USER_BOOST} - The user boost field.</li>
 * <li>{@link #KINDS} - The kinds field.</li>
 * <li>{@link #TIMESTAMP} - The timestamp field.</li>
 * <li>{@link #TAGS} - The tags field.</li>
 * <li>{@link #ROLES} - The roles field.</li>