// Add URLs to default set
        urlSet.add("http://example.com/robots.txt");
        urlSet.add("http://test.com/robots.txt");
        assertEquals(2, crawlerContext.getRobotsTxtUrlSet().size());

        // Set new set
        Set<String> newSet = new HashSet<>();
        newSet.add("http://new.com/robots.txt");
        crawlerContext.setRobotsTxtUrlSet(newSet);

      .Builder()
      .addInterceptor(CompressionInterceptor(Zstd))
      .build()

  sendRequest("https://developers.facebook.com/docs/", client)
  sendRequest("https://www.facebook.com/robots.txt", client)
  sendRequest("https://www.instagram.com/robots.txt", client)
}

private fun sendRequest(
  url: String,
  client: OkHttpClient,
) {
  val req = Request.Builder().url(url).build()

  client.newCall(req).execute().use {

      client
        .newBuilder()
        .eventListener(EventListener.NONE)
        .dns(dohDns)
        .build()

    dohEnabledClient.get("https://www.twitter.com/robots.txt")
    dohEnabledClient.get("https://www.facebook.com/robots.txt")
  }

  @Test
  fun testCustomTrustManager() {
    assumeNetwork()

    val trustManager =
      object : X509TrustManager {

  val clientBuilder =
    OkHttpClient.Builder().connectionSpecs(listOf(ConnectionSpec.CLEARTEXT)).dns {
      throw UnknownHostException("shortcircuit")
    }

  val request = Request("http://google.com/robots.txt".toHttpUrl())

  @Test
  fun testHttpLoggingInterceptor() {
    val interceptor =
      HttpLoggingInterceptor().apply {
        level = HttpLoggingInterceptor.Level.BASIC
      }

    }

    /* DFS */

    /* Split DFS path like \fs1.example.com\root5\link2\foo\bar.txt into at
     * most 3 components (not including the first index which is always empty):
     * result[0] = ""
     * result[1] = "fs1.example.com"
     * result[2] = "root5"
     * result[3] = "link2\foo\bar.txt"
     */
    void dfsPathSplit(final String path, final String[] result) {

        robotsTxtHelper = container.getComponent("robotsTxtHelper");
    }

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

            httpClientPropertyMap.put(name, value);
        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

package okhttp3.internal.connection

import okhttp3.Route

/**
 * A denylist of failed routes to avoid when creating a new connection to a target address. This is
 * used so that OkHttp can learn from its mistakes: if there was a failure attempting to connect to
 * a specific IP address or proxy server, that failure is remembered and alternate routes are
 * preferred.
 */
class RouteDatabase {
  private val _failedRoutes = mutableSetOf<Route>()

  @Test
  fun testBasicAuth() {
    fakeDns["server"] = listOf(InetAddress.getLocalHost())

    val route = factory.newRoute()

    val request =
      Request
        .Builder()
        .url("https://server/robots.txt")
        .build()
    val response =
      Response
        .Builder()
        .request(request)
        .code(401)
        .header("WWW-Authenticate", "Basic realm=\"User Visible Realm\"")

    engine.shutdown()
    cacheDir.deleteRecursively()
  }

  @Test
  fun get() {
    val executor = Executors.newCachedThreadPool()

    val completableFuture = execute(engine, executor, "https://google.com/robots.txt")

    try {
      val response = completableFuture.get(10, TimeUnit.SECONDS)

      assertEquals(200, response.code)
      assertEquals("h3", response.negotiatedProtocol)