*   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

- **Asynchronous Operations**: Non-blocking suggestion requests with callback support
- **Index Management**: Automatic index creation, switching, and maintenance
- **Customizable Scoring**: User boost, document frequency, and query frequency weighting

## Technology Stack

- **Java**: 21+ (configured via parent POM)
- **OpenSearch**: Latest (provided scope)
- **Apache Lucene**: Query parsing and text analysis

        // Test Kind enum
        assertEquals("query", SuggestItem.Kind.QUERY.toString());
        assertEquals("document", SuggestItem.Kind.DOCUMENT.toString());
        assertEquals("user", SuggestItem.Kind.USER.toString());
    }

    @Test
    public void testGetUpdatedSource() {
        // Test getUpdatedSource method
        String[] text = { "test" };
        String[][] readings = { { "test" } };

    /**
     * The kind of suggest item.
     */
    public enum Kind {
        /** Document kind. */
        DOCUMENT("document"),
        /** Query kind. */
        QUERY("query"),
        /** User kind. */
        USER("user");

        private final String kind;

        /**
         * Constructor for Kind.
         * @param kind The kind string.
         */
        Kind(final String kind) {

    public void setAuthSchemeProviderMap(final Map<String, AuthSchemeProvider> authSchemeProviderMap) {
        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

        assertTrue(urlFilter.match("https://test.com/page"));
        assertTrue(urlFilter.match("ftp://files.com/document.pdf"));
        assertTrue(urlFilter.match("file:///home/user/file.txt"));
    }

    /**
     * Test match with complex URL patterns
     */
    public void test_match_complexUrls() {
        String sessionId = "test-session-010";
        urlFilter.init(sessionId);

        boolean hasUserKind = false;
        for (SuggestItem.Kind kind : suggestItem.getKinds()) {
            if (kind == SuggestItem.Kind.USER) {
                hasUserKind = true;
                break;
            }
        }
        assertTrue("Should have USER kind", hasUserKind);
    }

    @Test
    public void testToSuggestItemWithEmptyReadings() {
        // Test toSuggestItem with empty readings

                .mustNot(QueryBuilders.matchPhraseQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()))
                .mustNot(QueryBuilders.matchPhraseQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString())));
        if (deleteResponse.hasError()) {
            throw new SuggestIndexException(deleteResponse.getErrors().get(0));
        }

        final List<SuggestItem> updateItems = new ArrayList<>();

        return charset;
    }

    /**
     * Gets the character encoding used for files.
     *
     * @return the character encoding
     */
    public String getCharset() {
        return charset;
    }

    /**
     * Sets the character encoding used for files.
     *
     * @param charset the character encoding to set
     */

 *   <li>passiveLocalHost: The local IP address to use for passive mode FTP.</li>
 *   <li>passiveNatWorkaround: Whether to use a NAT workaround in passive mode.</li>
 *   <li>reportActiveExternalHost: The external IP address to report in active mode.</li>
 *   <li>useEPSVwithIPv4: Whether to use EPSV with IPv4.</li>
 *   <li>isImplicit: Whether to use implicit SSL/TLS encryption.</li>