*   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

- **Asynchronous Operations**: Non-blocking suggestion requests with callback support
- **Index Management**: Automatic index creation, switching, and maintenance
- **Customizable Scoring**: User boost, document frequency, and query frequency weighting

## Technology Stack

- **Java**: 21+ (configured via parent POM)
- **OpenSearch**: Latest (provided scope)
- **Apache Lucene**: Query parsing and text analysis

        // Test Kind enum
        assertEquals("query", SuggestItem.Kind.QUERY.toString());
        assertEquals("document", SuggestItem.Kind.DOCUMENT.toString());
        assertEquals("user", SuggestItem.Kind.USER.toString());
    }

    @Test
    public void testGetUpdatedSource() {
        // Test getUpdatedSource method
        String[] text = { "test" };
        String[][] readings = { { "test" } };

    /**
     * The kind of suggest item.
     */
    public enum Kind {
        /** Document kind. */
        DOCUMENT("document"),
        /** Query kind. */
        QUERY("query"),
        /** User kind. */
        USER("user");

        private final String kind;

        /**
         * Constructor for Kind.
         * @param kind The kind string.
         */
        Kind(final String kind) {

    public void setAuthSchemeProviderMap(final Map<String, AuthSchemeProvider> authSchemeProviderMap) {
        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

        assertTrue(urlFilter.match("https://test.com/page"));
        assertTrue(urlFilter.match("ftp://files.com/document.pdf"));
        assertTrue(urlFilter.match("file:///home/user/file.txt"));
    }

    /**
     * Test match with complex URL patterns
     */
    public void test_match_complexUrls() {
        String sessionId = "test-session-010";
        urlFilter.init(sessionId);

        boolean hasUserKind = false;
        for (SuggestItem.Kind kind : suggestItem.getKinds()) {
            if (kind == SuggestItem.Kind.USER) {
                hasUserKind = true;
                break;
            }
        }
        assertTrue("Should have USER kind", hasUserKind);
    }

    @Test
    public void testToSuggestItemWithEmptyReadings() {
        // Test toSuggestItem with empty readings

                .mustNot(QueryBuilders.matchPhraseQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()))
                .mustNot(QueryBuilders.matchPhraseQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString())));
        if (deleteResponse.hasError()) {
            throw new SuggestIndexException(deleteResponse.getErrors().get(0));
        }

        final List<SuggestItem> updateItems = new ArrayList<>();

    /** Key for file attribute view in metadata */
    public static final String FILE_ATTRIBUTE_VIEW = "fileAttributeView";

    /** Key for file user in metadata */
    public static final String FS_FILE_USER = "fsFileUser";

    /** Key for file groups in metadata */
    public static final String FS_FILE_GROUPS = "fsFileGroups";

 * from FTP servers. The client supports various configurations, including authentication, timeouts,
 * passive/active modes, and encoding settings. It also handles FTP file attributes such as user,
 * group, and symbolic links.
 *
 * <p>
 * The class uses Apache Commons Net library for FTP communication. It maintains a queue of FTPClient
 * instances to improve performance by reusing connections.
 * </p>
 *