/** XPath expression for extracting robots content from meta tags */
    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    /** Robots tag value indicating no indexing or following */
    private static final String ROBOTS_TAG_NONE = "none";

    /** Robots tag value indicating no indexing */

:#9eeaf9;--bs-warning-border-subtle:#ffe69c;--bs-danger-border-subtle:#f1aeb5;--bs-light-border-subtle:#e9ecef;--bs-dark-border-subtle:#adb5bd;--bs-white-rgb:255,255,255;--bs-black-rgb:0,0,0;--bs-font-sans-serif:system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue","Noto Sans","Liberation Sans",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--bs-font-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;--bs-grad...

      Request
        .Builder()
        .url("https://graph.facebook.com/robots.txt?s=fb")
        .build()
    val twitterRequest =
      Request
        .Builder()
        .url("https://api.twitter.com/robots.txt?s=tw")
        .build()
    val googleRequest =
      Request
        .Builder()
        .url("https://www.google.com/robots.txt?s=g")
        .build()

    try {
      for (i in 1..2) {

  @Test
  fun testBasicAuth() {
    fakeDns["server"] = listOf(InetAddress.getLocalHost())

    val route = factory.newRoute()

    val request =
      Request
        .Builder()
        .url("https://server/robots.txt")
        .build()
    val response =
      Response
        .Builder()
        .request(request)
        .code(401)
        .header("WWW-Authenticate", "Basic realm=\"User Visible Realm\"")

# Test robots.txt for wildcard (*) and end-of-path ($) support
# Based on RFC 9309 specification

# Test wildcard patterns
User-agent: WildcardBot
Disallow: /*.pdf$
Disallow: /admin/*.php
Disallow: /*/private/
Allow: /public/*.html

# Test end-of-path ($) patterns
User-agent: EndPathBot
Disallow: /fish$
Disallow: /temp$
Allow: /fishing

# Test complex patterns
User-agent: ComplexBot
Disallow: /
Allow: /$

import okhttp3.HttpUrl.Companion.toHttpUrl
import org.junit.Assert.assertEquals
import org.junit.Test

class IdnaTest {
  @Test
  fun testHostnameFunction() {
    assertEquals("xn--n3h.net", "https://☃.net/robots.txt".toHttpUrl().host)
  }

 */
class PublicSuffixDatabaseTest {
  @Test
  fun testTopLevelDomain() {
    assertThat("https://www.google.com/robots.txt".toHttpUrl().topPrivateDomain()).isEqualTo("google.com")
  }

import okhttp3.HttpUrl;

import java.io.IOException;

public class Main {
  public static void main(String[] args) throws IOException {
    Call call = OkHttpCaller.callOkHttp(HttpUrl.get("https://square.com/robots.txt"));

    System.out.println(call.execute().body().string());
  }

    client.newCall(request).execute().use {
      assertThat(it.protocol).isEqualTo(Protocol.HTTP_2)
      assertThat(it.handshake!!.tlsVersion).isEqualTo(TlsVersion.TLS_1_3)
    }
  }

  @Test
  @Disabled
  fun testGoogle() {
    assumeNetwork()

    val request = Request.Builder().url("https://google.com/robots.txt").build()

    client.newCall(Request(url = server.url("/"))).execute().use {
      assertThat(it.body.string()).isEqualTo("abc")
    }
  }

  @Test
  fun testExternalSite() {
    client.newCall(Request(url = "https://google.com/robots.txt".toHttpUrl())).execute().use {
      assertThat(it.code).isEqualTo(200)
    }
  }