User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

    assertFailsWith<NullPointerException> {
      (mapOf("User-Agent" to null) as Map<String, String>).toHeaders()
    }
  }

  @Test fun toMultimapGroupsHeaders() {
    val headers =
      Headers.headersOf(
        "cache-control",
        "no-cache",
        "cache-control",
        "no-store",
        "user-agent",
        "OkHttp",
      )
    val headerMap = headers.toMultimap()

 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */
    protected static final Pattern USER_AGENT_RECORD =
            Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */

        RobotsTxt robotsTxt;
        final InputStream in = new java.io.ByteArrayInputStream(robotsTxtContent.getBytes());
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

        // Directives before User-agent should be ignored

            params {
                // Disable jdk-provider-plugin, otherwise the JAVA_HOME will be overwritten
                // https://github.com/gradle/teamcity-jdk-provider-plugin/blob/main/teamcity-jdk-provider-plugin-agent/src/main/kotlin/org/gradle/teamcity_jdk_provider_plugin/JdkProviderAgentLifecycleListener.kt#L22
                param("JdkProviderEnabled", "false")
                // should be the same version we run TeamCity with

	}
}

// uaTransport - User-Agent  transport
type uaTransport struct {
	ua string
	rt http.RoundTripper
}

func (u *uaTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req2 := req.Clone(req.Context())
	req2.Header.Set("User-Agent", u.ua)
	return u.rt.RoundTrip(req2)
}

// WithUserAgent wraps an existing transport with custom User-Agent

import vcsroots.useAbsoluteVcs

object WarmupEc2Agent : BuildType({
    name = "Warmup EC2 Agent"
    id("Util_WarmupEc2Agent")

    vcs.useAbsoluteVcs(VersionedSettingsBranch.fromDslContext().vcsRootId())

    features {
        freeDiskSpace {
            // Lower the limit such that the agent work directories aren't cleaned during the AMI baking process
            requiredSpace = "100mb"
        }
    }

        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

from typing import Union

from fastapi import FastAPI, Header

app = FastAPI()


@app.get("/items/")
async def read_items(user_agent: Union[str, None] = Header(default=None)):

      .assertLogEqual("--> GET $url http/1.1")
      .assertLogEqual("Host: $host")
      .assertLogEqual("Connection: Keep-Alive")
      .assertLogEqual("Accept-Encoding: gzip")
      .assertLogMatch(Regex("""User-Agent: okhttp/.+"""))
      .assertLogEqual("--> END GET")
      .assertLogMatch(Regex("""<-- 200 OK $url \(\d+ms\)"""))
      .assertLogEqual("Content-Length: 0")
      .assertLogEqual("<-- END HTTP")
      .assertNoMoreLogs()