if err = globalDNSConfig.Put(bucket); err != nil {
					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
						Force:      true,
						SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
					})
					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
					return
				}

				// Load updated bucket metadata into memory.
				globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)

            throw new CrawlerSystemException("HTTP URL is null or empty. Cannot process robots.txt without a valid URL.");
        }

        if (robotsTxtHelper == null || !robotsTxtHelper.isEnabled()) {
            // not support robots.txt
            return;
        }

        // crawler context
        final CrawlerContext crawlerContext = CrawlingParameterUtil.getCrawlerContext();

	if serviceAccount == "" {
		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
		return
	}

	if serviceAccount == siteReplicatorSvcAcc && globalSiteReplicationSys.isEnabled() {
		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
		return
	}
	// We do not care if service account is readable or not at this point,

	keys := make([]string, 0, 3)

	// Append credentials secret key itself
	keys = append(keys, u.Credentials.SecretKey)

	// Use site-replication credentials if found
	if globalSiteReplicationSys.isEnabled() {
		secretKey, err := getTokenSigningKey()
		if err != nil {
			return nil, err
		}
		keys = append(keys, secretKey)
	}