client =
      OkHttpClient
        .Builder()
        .sslSocketFactory(certificates.sslSocketFactory(), certificates.trustManager)
        .build()
  }

  fun run() {
    showUrl("https://squareup.com/robots.txt")
    showUrl("https://publicobject.com/helloworld.txt")
  }

  private fun showUrl(url: String) {
    val request = Builder().url(url).build()
    client
      .newCall(request)
      .execute()

          handshakeCertificates.trustManager,
        )
    }

    val client = clientBuilder.build()

    val request =
      Request
        .Builder()
        .url("https://valid-isrgrootx1.letsencrypt.org/robots.txt")
        .build()
    client.newCall(request).execute().use { response ->
      assertThat(response.code).isEqualTo(404)
      assertThat(response.protocol).isEqualTo(Protocol.HTTP_2)
    }
  }

    /** XPath expression for extracting robots content from meta tags */
    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    /** Robots tag value indicating no indexing or following */
    private static final String ROBOTS_TAG_NONE = "none";

    /** Robots tag value indicating no indexing */

- **Testing**: JUnit 4, UTFlute, Mockito 5.7.0
- **Storage**: In-memory (default), OpenSearch (optional)

### Protocols

- **HTTP/HTTPS**: Full crawling, cookies, auth, robots.txt
- **File**: Local/network file systems
- **FTP**: With authentication
- **SMB/CIFS**: Windows shares (SMB1/SMB2+)
- **Storage**: MinIO/S3-compatible

### Content Formats

import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,
 * allowed/disallowed paths, crawl delays, and sitemap URLs.
 *
 * <p>The robots.txt protocol is implemented according to the standard specification,

  }

  @Test
  fun normalizeSingleSelfSignedCertificate() {
    val root =
      HeldCertificate
        .Builder()
        .serialNumber(1L)
        .build()
    val cleaner = get(root.certificate)
    assertThat(cleaner.clean(list(root), "hostname")).isEqualTo(list(root))
  }

  @Test
  fun normalizeUnknownSelfSignedCertificate() {
    val root =
      HeldCertificate
        .Builder()
        .serialNumber(1L)

      "https://www.allizom.org/robots.txt",
      "https://tls13.crypto.mozilla.org/",
      "https://tls.ctf.network/robots.txt",
      "https://rustls.jbp.io/",
      "https://h2o.examp1e.net",
      "https://mew.org/",
      "https://tls13.baishancloud.com/",
      "https://tls13.akamai.io/",
      "https://swifttls.org/",
      "https://www.googleapis.com/robots.txt",
      "https://graph.facebook.com/robots.txt",

    }

    /**
     * Verify that the message + rootCause constructor stores the root.
     */
    @Test
    @DisplayName("Message + root cause stores root cause")
    void testStringAndThrowableConstructor() {
        Throwable root = mock(Throwable.class);
        SmbException ex = new SmbException("oops", root);
        assertSame(root, ex.getRootCause());
    }

    /**

    }

    /**
     * Returns the set of robots.txt URLs.
     * @return The set of robots.txt URLs.
     */
    public Set<String> getRobotsTxtUrlSet() {
        return robotsTxtUrlSet;
    }

    /**
     * Sets the set of robots.txt URLs.
     * @param robotsTxtUrlSet The set of robots.txt URLs.
     */
    public void setRobotsTxtUrlSet(final Set<String> robotsTxtUrlSet) {

        robotsTxtHelper = container.getComponent("robotsTxtHelper");
    }

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }