.gzip()
        .gzip()
        .build()
    }.also {
      assertThat(it).hasMessage("Content-Encoding already set: gzip")
    }
  }

  @Test
  fun curlGet() {
    val request =
      Request
        .Builder()
        .url("https://example.com")
        .header("Authorization", "Bearer abc123")
        .build()

    val curl = request.toCurl()

import java.util.logging.Level;
import java.util.logging.Logger;

import org.codelibs.curl.Curl.Method;
import org.junit.Test;

public class CurlTest {
    private static final Logger logger = Logger.getLogger(CurlTest.class.getName());

    @Test
    public void test_Get() {
        Curl.get("https://www.codelibs.org/").execute(response -> {

                + "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n" + "  <url>\n" + "    <lastmod>2025-01-01</lastmod>\n"
                + "    <changefreq>daily</changefreq>\n" + "    <priority>0.8</priority>\n" + "  </url>\n" + "  <url>\n"
                + "    <loc>http://www.example.com/valid.html</loc>\n" + "    <lastmod>2025-01-02</lastmod>\n" + "  </url>\n" + "</urlset>";

            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            assertEquals(1, urlSet.size());
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                assertTrue(url.contains("dir1/test3.txt"));
            }

     */
    public void test_robotsTxtUrlSet() {
        Set<String> urlSet = crawlerContext.getRobotsTxtUrlSet();
        assertNotNull(urlSet);
        assertTrue(urlSet instanceof LruHashSet);
        assertTrue(urlSet.isEmpty());

        // Add URLs to default set
        urlSet.add("http://example.com/robots.txt");
        urlSet.add("http://test.com/robots.txt");

            final int depth, final String encoding) {
        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.getUrlFilter().match(d.getUrl()))
                .map(d -> {

        try {
            fsClient.doGet("file://" + path);
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                if (url.indexOf(".svn") < 0) {

        }
        final Set<RequestData> urlSet = new HashSet<>(docList.size());
        for (final Map<String, Object> doc : docList) {
            final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
            if (StringUtil.isNotBlank(url)) {
                urlSet.add(RequestDataBuilder.newRequestData().get().url(url).build());
            }
        }

        try {
            if (bis.read(bytes) == -1) {
                return false;
            }

            final String preloadDate = new String(bytes, Constants.UTF_8);
            if (preloadDate.indexOf("<urlset") >= 0 || preloadDate.indexOf("<sitemapindex") >= 0 || preloadDate.startsWith("http://")
                    || preloadDate.startsWith("https://")) {
                // XML Sitemaps
                return true;
            }

            return;
        }

        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.urlFilter.match(d.getUrl()))
                .map(d -> {