throw new CrawlingAccessException("[" + e.status() + "] Failed to insert " + id, e);
            }
            throw new OpenSearchAccessException("[" + e.status() + "] Failed to insert " + id, e);
        } catch (final Exception e) {
            throw new OpenSearchAccessException("Failed to insert " + id, e);
        }
    }

    /**

        urlQueue.setMethod(Constants.GET_METHOD);
        insert(urlQueue);
    }

    /**
     * Inserts a URL queue entry into the OpenSearch index.
     *
     * @param urlQueue The URL queue entry to insert.
     */
    @Override
    public void insert(final OpenSearchUrlQueue urlQueue) {
        try {
            super.insert(urlQueue, urlQueue.getId() == null ? OpType.CREATE : OpType.INDEX);

        urlFilter.addInclude("https?://[^/]+\\.example\\.com/.*");

        assertTrue(urlFilter.match("http://www.example.com/"));
        assertTrue(urlFilter.match("https://api.example.com/v1/users"));
        assertTrue(urlFilter.match("http://subdomain.example.com/page"));
        assertFalse(urlFilter.match("https://example.org/"));
        assertFalse(urlFilter.match("ftp://files.example.com/"));
    }

    /**

 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

        super.setUp();

        Path tempDir = Files.createTempDirectory("smbdata");
        Path publicDir = tempDir.resolve("public");
        Files.createDirectory(publicDir);
        Path usersDir = tempDir.resolve("users");
        Files.createDirectory(usersDir);
        Path testuser1Dir = tempDir.resolve("testuser1");
        Files.createDirectory(testuser1Dir);
        Path testuser2Dir = tempDir.resolve("testuser2");

        super.insert(accessResult, accessResult.getId() == null ? OpType.CREATE : OpType.INDEX);
    }

    /**
     * Updates an access result in the OpenSearch index.
     *
     * @param accessResult The access result to update.
     */
    @Override
    public void update(final OpenSearchAccessResult accessResult) {
        super.insert(accessResult, OpType.INDEX);
    }

    /**

    public void setAuthSchemeProviderMap(final Map<String, AuthSchemeProvider> authSchemeProviderMap) {
        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

        TestRule rule3 = new TestRule("rule3", true);

        ruleManager.addRule(rule1);
        ruleManager.addRule(rule3);
        ruleManager.addRule(1, rule2); // Insert in middle

        List<Rule> rules = ruleManager.getRules();
        assertEquals(3, rules.size());
        assertEquals("rule1", rules.get(0).getRuleId());
        assertEquals("rule2", rules.get(1).getRuleId());

        responseData1.setUrl("http://www.example.com/page");
        assertTrue(rule.match(responseData1));

        ResponseData responseData2 = new ResponseData();
        responseData2.setUrl("https://api.example.com/v1/users");
        assertTrue(rule.match(responseData2));

        ResponseData responseData3 = new ResponseData();
        responseData3.setUrl("http://other.com/page");
        assertFalse(rule.match(responseData3));
    }

    /** Key for file attribute view in metadata */
    public static final String FILE_ATTRIBUTE_VIEW = "fileAttributeView";

    /** Key for file user in metadata */
    public static final String FS_FILE_USER = "fsFileUser";

    /** Key for file groups in metadata */
    public static final String FS_FILE_GROUPS = "fsFileGroups";