// Normalize user-agent to lowercase
                        final String userAgent = value.toLowerCase(Locale.ENGLISH);
                        Directive currentDirective = robotsTxt.getDirective(userAgent);
                        if (currentDirective == null) {
                            currentDirective = new Directive(userAgent);
                            robotsTxt.addDirective(currentDirective);
                        }

      },
      "configParameter": {
        "type": "keyword"
      },
      "depth": {
        "type": "integer"
      },
      "maxAccessCount": {
        "type": "long"
      },
      "userAgent": {
        "type": "keyword"
      },
      "numOfThread": {
        "type": "integer"
      },
      "intervalTime": {
        "type": "integer"
      },
      "timeToLive": {
        "type": "integer"

            doColumn("searchWord");
        }

        public void columnUser() {
            doColumn("user");
        }

        public void columnUserAgent() {
            doColumn("userAgent");
        }

        public void columnUserInfoId() {
            doColumn("userInfoId");
        }

        public void columnUserSessionId() {
            doColumn("userSessionId");
        }

		"EpochTime":        {strconv.FormatInt(currTime.Unix(), 10)},
		"SecureTransport":  {strconv.FormatBool(r.TLS != nil)},
		"SourceIp":         {handlers.GetSourceIPRaw(r)},
		"UserAgent":        {r.UserAgent()},
		"Referer":          {r.Referer()},
		"principaltype":    {principalType},
		"userid":           {username},
		"username":         {username},
		"versionid":        {vid},

    if (!isSpecialHeader(parts[0])) {
      request.header(parts[0], parts[1])
    }
  }
  referer?.let {
    request.header("Referer", it)
  }
  request.header("User-Agent", userAgent)

  return request.build()
}

private fun Main.mediaType(): MediaType? {
  val mimeType =
    headers?.let {
      for (header in it) {
        val parts = header.split(':', limit = 2)

    "statusCode": 200,
    "rx": 401,
    "tx": 0,
    "timeToResponse": "13309747ns",
    "timeToResponseInNS": "13309747"
  },
  "remotehost": "127.0.0.1",
  "requestID": "17CDC1F4D7E69123",
  "userAgent": "MinIO (linux; amd64) minio-go/v7.0.70 mc/RELEASE.2024-04-30T17-44-48Z",
  "requestPath": "/testbucket/hosts",
  "requestHost": "localhost:9000",
  "requestHeader": {
    "Accept-Encoding": "zstd,gzip",

    /** Buffer to accumulate character data between XML tags. */
    protected StringBuilder textBuf = new StringBuilder(1000);

    /** User agent string to be used for web crawling. */
    protected String userAgent = "gsa-crawler";

    /**
     * Default constructor for GsaConfigParser.
     */
    public GsaConfigParser() {
        super();
    }

    /**

    assertThat(request.url.toString()).isEqualTo("http://example.com/")
    assertThat(request.header("Referer")).isEqualTo("foo")
    assertThat(request.body).isNull()
  }

  @Test
  fun userAgent() {
    val request = fromArgs("-A", "foo", "http://example.com").createRequest()
    assertThat(request.method).isEqualTo("GET")
    assertThat(request.url.toString()).isEqualTo("http://example.com/")

        "false"
      ],
      "SourceIp": [
        "127.0.0.1"
      ],
      "User-Agent": [
        "MinIO (linux; amd64) minio-go/v7.0.24 mc/DEVELOPMENT.2022-04-20T23-07-53Z"
      ],
      "UserAgent": [
        "MinIO (linux; amd64) minio-go/v7.0.24 mc/DEVELOPMENT.2022-04-20T23-07-53Z"
      ],
      "X-Amz-Content-Sha256": [
        "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
      ],

            final SearchLog searchLog = queue.poll();
            if (searchLog != null) {
                final String userAgent = searchLog.getUserAgent();
                final boolean isBot =
                        userAgent != null && stream(botNames).get(stream -> stream.anyMatch(botName -> userAgent.indexOf(botName) >= 0));
                if (!isBot) {
                    searchLog.getUserInfo().ifPresent(userInfo -> {