val clientBuilder =
    OkHttpClient.Builder().connectionSpecs(listOf(ConnectionSpec.CLEARTEXT)).dns {
      throw UnknownHostException("shortcircuit")
    }

  val request = Request("http://google.com/robots.txt".toHttpUrl())

  @Test
  fun testHttpLoggingInterceptor() {
    val interceptor =
      HttpLoggingInterceptor().apply {
        level = HttpLoggingInterceptor.Level.BASIC
      }

      "https://www.allizom.org/robots.txt",
      "https://tls13.crypto.mozilla.org/",
      "https://tls.ctf.network/robots.txt",
      "https://rustls.jbp.io/",
      "https://h2o.examp1e.net",
      "https://mew.org/",
      "https://tls13.baishancloud.com/",
      "https://tls13.akamai.io/",
      "https://swifttls.org/",
      "https://www.googleapis.com/robots.txt",
      "https://graph.facebook.com/robots.txt",

            httpClientPropertyMap.put(name, value);
        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

    }

    OkHttpClient client = builder.build();

    sendRequest(client, "https://valid-isrgrootx1.letsencrypt.org/robots.txt");

    try {
      sendRequest(client, "https://google.com/robots.txt");
      if (androidMorEarlier) {
        // will pass with default CAs on N or later
        fail();
      }
    } catch (SSLHandshakeException sslhe) {

      .Builder()
      .addInterceptor(CompressionInterceptor(Zstd))
      .build()

  sendRequest("https://developers.facebook.com/docs/", client)
  sendRequest("https://www.facebook.com/robots.txt", client)
  sendRequest("https://www.instagram.com/robots.txt", client)
}

private fun sendRequest(
  url: String,
  client: OkHttpClient,
) {
  val req = Request.Builder().url(url).build()

  client.newCall(req).execute().use {

    client.newCall(request).execute().use {
      assertThat(it.protocol).isEqualTo(Protocol.HTTP_2)
      assertThat(it.handshake!!.tlsVersion).isEqualTo(TlsVersion.TLS_1_3)
    }
  }

  @Test
  @Disabled
  fun testGoogle() {
    assumeNetwork()

    val request = Request.Builder().url("https://google.com/robots.txt").build()

        // Add URLs to default set
        urlSet.add("http://example.com/robots.txt");
        urlSet.add("http://test.com/robots.txt");
        assertEquals(2, crawlerContext.getRobotsTxtUrlSet().size());

        // Set new set
        Set<String> newSet = new HashSet<>();
        newSet.add("http://new.com/robots.txt");
        crawlerContext.setRobotsTxtUrlSet(newSet);

      client
        .newBuilder()
        .eventListener(EventListener.NONE)
        .dns(dohDns)
        .build()

    dohEnabledClient.get("https://www.twitter.com/robots.txt")
    dohEnabledClient.get("https://www.facebook.com/robots.txt")
  }

  @Test
  fun testCustomTrustManager() {
    assumeNetwork()

    val trustManager =
      object : X509TrustManager {

    /** XPath expression for extracting robots content from meta tags */
    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    /** Robots tag value indicating no indexing or following */
    private static final String ROBOTS_TAG_NONE = "none";

    /** Robots tag value indicating no indexing */

    engine.shutdown()
    cacheDir.deleteRecursively()
  }

  @Test
  fun get() {
    val executor = Executors.newCachedThreadPool()

    val completableFuture = execute(engine, executor, "https://google.com/robots.txt")

    try {
      val response = completableFuture.get(10, TimeUnit.SECONDS)

      assertEquals(200, response.code)
      assertEquals("h3", response.negotiatedProtocol)