*   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

}

fun Requirements.requiresNotEc2Agent() {
    doesNotContain("teamcity.agent.name", "ec2")
    // US region agents have name "EC2-XXX"
    doesNotContain("teamcity.agent.name", "EC2")
}

/**
 * We have some "shared" host where a Linux build agent and a Windows build agent
 * both run on the same bare metal. Some builds require exclusive access to the

        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

class RequestCommonTest {
  @Test
  fun constructorNormal() {
    val url = "https://example.com/".toHttpUrl()
    val body = "hello".toRequestBody()
    val headers = headersOf("User-Agent", "RequestTest")
    val method = "PUT"
    val request =
      Request(
        url = url,
        headers = headers,
        method = method,
        body = body,
      )

      .assertLogEqual("--> GET $url http/1.1")
      .assertLogEqual("Host: $host")
      .assertLogEqual("Connection: Keep-Alive")
      .assertLogEqual("Accept-Encoding: gzip")
      .assertLogMatch(Regex("""User-Agent: okhttp/.+"""))
      .assertLogEqual("--> END GET")
      .assertLogMatch(Regex("""<-- 200 OK $url \(\d+ms\)"""))
      .assertLogEqual("Content-Length: 0")
      .assertLogEqual("<-- END HTTP")
      .assertNoMoreLogs()

    assertThat(response.request.header("User-Agent")).isNotNull()
    assertThat(response.request.header("User-Agent")).isEqualTo("user request")
    assertThat(response.networkResponse!!.request.header("User-Agent")).isEqualTo(
      "intercepted request",
    )
  }

  @Test
  fun applicationInterceptorThrowsRuntimeExceptionAsynchronous() {
    interceptorThrowsRuntimeExceptionAsynchronous(false)

import org.junit.jupiter.api.Test

class RequestTest {
  @Test
  fun constructor() {
    val url = "https://example.com/".toHttpUrl()
    val body = "hello".toRequestBody()
    val headers = headersOf("User-Agent", "RequestTest")
    val method = "PUT"
    val request =
      Request(
        url = url,
        headers = headers,
        method = method,
        body = body,
      )

    // ============================================================
    // User Agent Configuration
    // ============================================================

    /** Prefix for Fess crawler user agent string. */
    public static final String CRAWLING_USER_AGENT_PREFIX = "Mozilla/5.0 (compatible; Fess/";

    /** Suffix for Fess crawler user agent string with bot information URL. */

  stars: 2323
  owner_login: aminalaee
  owner_html_url: https://github.com/aminalaee
- name: nextpy
  html_url: https://github.com/dot-agent/nextpy
  stars: 2314
  owner_login: dot-agent
  owner_html_url: https://github.com/dot-agent
- name: mcp-context-forge
  html_url: https://github.com/IBM/mcp-context-forge
  stars: 2236
  owner_login: IBM
  owner_html_url: https://github.com/IBM

                + "<start_urls>https://example.com</start_urls>" + "</globalparams></config></eef>";

        parser.parse(new InputSource(new StringReader(xmlWithUserAgent)));

        parser.getWebConfig().ifPresent(config -> {
            assertEquals("custom-agent", config.getUserAgent());