@Test
  fun testBasicAuth() {
    fakeDns["server"] = listOf(InetAddress.getLocalHost())

    val route = factory.newRoute()

    val request =
      Request
        .Builder()
        .url("https://server/robots.txt")
        .build()
    val response =
      Response
        .Builder()
        .request(request)
        .code(401)
        .header("WWW-Authenticate", "Basic realm=\"User Visible Realm\"")

Se você precisa de algo fácil de integrar com o FastAPI, mas que seja mais robusto, suportado por Redis, PostgreSQL ou outros, verifique o <a href="https://github.com/encode/broadcaster" class="external-link" target="_blank">encode/broadcaster</a>.

///

## Mais informações

class ApacheHttpClientTest {
  private var httpClient = HttpClients.createDefault()

  @After fun tearDown() {
    httpClient.close()
  }

  @Test fun get() {
    val request = HttpGet("https://google.com/robots.txt")

    httpClient.execute(request).use { response ->
      assertEquals(200, response.code)
      // TODO enable ALPN later
      assertEquals(HttpVersion.HTTP_1_1, response.version)
    }
  }

    client.newCall(request).execute().use {
      assertThat(it.protocol).isEqualTo(Protocol.HTTP_2)
      assertThat(it.handshake!!.tlsVersion).isEqualTo(TlsVersion.TLS_1_3)
    }
  }

  @Test
  @Disabled
  fun testGoogle() {
    assumeNetwork()

    val request = Request.Builder().url("https://google.com/robots.txt").build()

            tempDir.delete();
            tempDir.mkdirs();

            // robots.txt
            StringBuilder buf = new StringBuilder();
            buf.append("User-agent: *").append('\n');
            buf.append("Disallow: /admin/").append('\n');
            buf.append("Disallow: /websvn/").append('\n');
            final File robotTxtFile = new File(tempDir, "robots.txt");

                return false;
            };
        };

        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/");
        responseData.addMetaData("X-Robots-Tag", "noindex,nofollow");

        try {
            transformer.processXRobotsTag(responseData, new ResultData());
            fail();
        } catch (ChildUrlsException e) {

import okhttp3.HttpUrl.Companion.toHttpUrl
import org.junit.Assert.assertEquals
import org.junit.Test

class IdnaTest {
  @Test
  fun testHostnameFunction() {
    assertEquals("xn--n3h.net", "https://☃.net/robots.txt".toHttpUrl().host)
  }

    <magic priority="50">
      <match value="MPX,Microsoft Project for Windows," type="string" offset="0"/>
    </magic>
    <sub-class-of type="text/plain"/>
  </mime-type>
  <mime-type type="text/x-robots">
    <!-- robots.txt file -->
    <!-- draft: https://datatracker.ietf.org/doc/html/draft-koster-rep -->
    <!-- should have a higher priority than rfc822 - TIKA-3489 -->
    <!-- and a lower priority than pcap - TIKA-4174 -->

Pero ten en cuenta que, como todo se maneja en memoria, en una sola lista, solo funcionará mientras el proceso esté en ejecución, y solo funcionará con un solo proceso.

Si necesitas algo fácil de integrar con FastAPI pero que sea más robusto, soportado por Redis, PostgreSQL u otros, revisa <a href="https://github.com/encode/broadcaster" class="external-link" target="_blank">encode/broadcaster</a>.

///

## Más información

@RunWith(AndroidJUnit4.class)
public class OkHttpClientTest {
  @Test public void get() throws IOException {
    OkHttpClient client = new OkHttpClient();
    Request request = new Request.Builder()
        .url("https://google.com/robots.txt")
        .build();
    try (Response response = client.newCall(request).execute()) {
      assertEquals(200, response.code());
      assertEquals(Protocol.HTTP_2, response.protocol());
    }
  }