List<String> urls = new ArrayList<>();
        urls.add("http://example.com/page1");
        urls.add("http://example.com/page2");
        urls.add("http://example.com/page1"); // Duplicate
        urls.add("http://example.com/page3");
        urls.add("http://example.com/page2"); // Duplicate

        Set<RequestData> result = crawlerThread.getAnchorSet(urls);

        assertEquals("http://example.com/page1", result.get(0));
        assertEquals("http://example.com/page2", result.get(1));
        assertEquals("http://example.com/page3", result.get(2));
    }

    @Test
    public void test_getAnchorList_duplicatesFromSameTagType() throws Exception {
        final String data =

      .isEqualTo("Page 2")
    val page1 = server.takeRequest()
    assertThat(page1.requestLine).isEqualTo("POST /page1 HTTP/1.1")
    assertThat(page1.body?.utf8()).isEqualTo("ABCD")
    val page2 = server.takeRequest()
    assertThat(page2.requestLine).isEqualTo("GET /page2 HTTP/1.1")
  }

  @Test
  fun redirectedPostStripsRequestBodyHeaders() {
    server.enqueue(

        });

        assertEquals(5, page1.size());

        // Get second page (5 items)
        final List<OpenSearchAccessResult> page2 = dataService.getAccessResultList(builder -> {
            builder.setQuery(QueryBuilders.termQuery("sessionId", sessionId));
            builder.setFrom(5);
            builder.setSize(5);
        });

        assertEquals(5, page2.size());

        dataService.delete(sessionId);

        // Get first page
        final Map<String, Object> searchBody = new HashMap<>();
        searchBody.put("size", 5);
        searchBody.put("page", 1);

        String response = checkMethodBase(searchBody).get(API_PATH + "/logs").asString();
        List<Map<String, Object>> page1 = JsonPath.from(response).getList("response.logs");

        logger.info("Page 1: {}", page1);

        // Get second page

        assertEquals(1, patterns.size());
        assertTrue(patterns.get(0).matcher("http://example.com/page1").matches());
        assertFalse(patterns.get(0).matcher("http://other.com/page1").matches());

        urlFilterService.delete(sessionId);
    }

    @Test
    public void test_addExcludeUrlFilter_singleTx() {

        // Get first page
        final Map<String, Object> searchBody = new HashMap<>();
        searchBody.put("size", 5);
        searchBody.put("page", 1);

        String response = checkMethodBase(searchBody).get(API_PATH + "/logs").asString();
        List<Map<String, Object>> page1 = JsonPath.from(response).getList("response.logs");

        logger.info("Page 1: {}", page1);

        // Get second page

    // then
    assertThat(response.body.string()).isEqualTo("Page 2")
    val page1 = server.takeRequest()
    assertThat(page1.requestLine).isEqualTo("PROPFIND /page1 HTTP/1.1")
    assertThat(page1.body?.utf8()).isEqualTo("Request Body")
    val page2 = server.takeRequest()
    assertThat(page2.requestLine).isEqualTo("PROPFIND /page2 HTTP/1.1")
    assertThat(page2.body?.utf8()).isEqualTo("Request Body")
  }

  @Test

        doc.put("url", "https://example.com/page.html");
        doc.put("title", "Test Page");
        doc.put("content", "Hello World");
        setupMockComponents(Collections.singletonList(doc));

        final String result = indexExportJob.execute();

        assertEquals("Exported 1 documents.", result);
        final Path expectedFile = tempDir.resolve("example.com/page.html");
        assertTrue(Files.exists(expectedFile));

        final List<ChatSource> sources = new ArrayList<>();
        final ChatSource source1 = new ChatSource();
        source1.setUrl("https://example.com/page1");
        source1.setTitle("Page 1");
        sources.add(source1);

        final Map<String, Object> response = chatApiManager.createSuccessResponse("session-123", "Response content", sources);