/** XPath expression for extracting robots content from meta tags */
    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    /** Robots tag value indicating no indexing or following */
    private static final String ROBOTS_TAG_NONE = "none";

    /** Robots tag value indicating no indexing */

    engine.shutdown()
    cacheDir.deleteRecursively()
  }

  @Test
  fun get() {
    val executor = Executors.newCachedThreadPool()

    val completableFuture = execute(engine, executor, "https://google.com/robots.txt")

    try {
      val response = completableFuture.get(10, TimeUnit.SECONDS)

      assertEquals(200, response.code)
      assertEquals("h3", response.negotiatedProtocol)

  @Test
  fun testBasicAuth() {
    fakeDns["server"] = listOf(InetAddress.getLocalHost())

    val route = factory.newRoute()

    val request =
      Request
        .Builder()
        .url("https://server/robots.txt")
        .build()
    val response =
      Response
        .Builder()
        .request(request)
        .code(401)
        .header("WWW-Authenticate", "Basic realm=\"User Visible Realm\"")

      Request
        .Builder()
        .url("https://graph.facebook.com/robots.txt?s=fb")
        .build()
    val twitterRequest =
      Request
        .Builder()
        .url("https://api.twitter.com/robots.txt?s=tw")
        .build()
    val googleRequest =
      Request
        .Builder()
        .url("https://www.google.com/robots.txt?s=g")
        .build()

    try {
      for (i in 1..2) {

    client.newCall(request).execute().use {
      assertThat(it.protocol).isEqualTo(Protocol.HTTP_2)
      assertThat(it.handshake!!.tlsVersion).isEqualTo(TlsVersion.TLS_1_3)
    }
  }

  @Test
  @Disabled
  fun testGoogle() {
    assumeNetwork()

    val request = Request.Builder().url("https://google.com/robots.txt").build()

# Test robots.txt for wildcard (*) and end-of-path ($) support
# Based on RFC 9309 specification

# Test wildcard patterns
User-agent: WildcardBot
Disallow: /*.pdf$
Disallow: /admin/*.php
Disallow: /*/private/
Allow: /public/*.html

# Test end-of-path ($) patterns
User-agent: EndPathBot
Disallow: /fish$
Disallow: /temp$
Allow: /fishing

# Test complex patterns
User-agent: ComplexBot
Disallow: /
Allow: /$

import okhttp3.HttpUrl.Companion.toHttpUrl
import org.junit.Assert.assertEquals
import org.junit.Test

class IdnaTest {
  @Test
  fun testHostnameFunction() {
    assertEquals("xn--n3h.net", "https://☃.net/robots.txt".toHttpUrl().host)
  }

            tempDir.delete();
            tempDir.mkdirs();

            // robots.txt
            StringBuilder buf = new StringBuilder();
            buf.append("User-agent: *").append('\n');
            buf.append("Disallow: /admin/").append('\n');
            buf.append("Disallow: /websvn/").append('\n');
            final File robotTxtFile = new File(tempDir, "robots.txt");

import okhttp3.HttpUrl;

import java.io.IOException;

public class Main {
  public static void main(String[] args) throws IOException {
    Call call = OkHttpCaller.callOkHttp(HttpUrl.get("https://square.com/robots.txt"));

    System.out.println(call.execute().body().string());
  }

        },
        {
            "year": 2011,
            "publisher": "Harper Collins",
            "type": "Paperback",
            "pages": 265
        }
    ]
}
{
    "title": "The Robots of Dawn",
    "authorInfo": {
        "name": "Isaac Asimov",
        "yearRange": [1920, 1992],
        "penName": "Paul French"
    },
    "genre": "Science fiction",
    "publicationHistory": [
        {