// These settings are needed to ensure that kafka client doesn't hang on brokers
	// refer https://github.com/IBM/sarama/issues/765#issuecomment-254333355
	sconfig.Producer.Retry.Max = 2
	sconfig.Producer.Retry.Backoff = (10 * time.Second)
	sconfig.Producer.Return.Successes = true
	sconfig.Producer.Return.Errors = true
	sconfig.Producer.RequiredAcks = 1
	sconfig.Producer.Timeout = (10 * time.Second)

func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
	if err.HTTPStatusCode == http.StatusServiceUnavailable {
		// Set retry-after header to indicate user-agents to retry request after 120secs.
		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
		w.Header().Set(xhttp.RetryAfter, "120")
	}

	switch err.Code {
	case "InvalidRegion":

				return
			}
		case "Notify":
			err = z.Notify.DecodeMsg(dc)
			if err != nil {
				err = msgp.WrapError(err, "Notify")
				return
			}
		case "Retry":
			err = z.Retry.DecodeMsg(dc)
			if err != nil {
				err = msgp.WrapError(err, "Retry")
				return
			}
		default:
			err = dc.Skip()
			if err != nil {
				err = msgp.WrapError(err)
				return
			}
		}
	}
	return
}

		default:
			// This is where we'd set it to -1 (at least
			// if body != NoBody) to mean unknown, but
			// that broke people during the Go 1.8 testing
			// period. People depend on it being 0 I
			// guess. Maybe retry later. See Issue 18117.
		}
		// For client requests, Request.ContentLength of 0
		// means either actually 0, or unknown. The only way
		// to explicitly say that the ContentLength is zero is

	// These settings are needed to ensure that kafka client doesn't hang on brokers
	// refer https://github.com/IBM/sarama/issues/765#issuecomment-254333355
	config.Producer.Retry.Max = 2
	config.Producer.Retry.Backoff = (1 * time.Second)
	config.Producer.Return.Successes = true
	config.Producer.Return.Errors = true
	config.Producer.RequiredAcks = 1
	config.Producer.Timeout = (5 * time.Second)

	for ambientPod, err = s.handlers.GetPodIfAmbient(addCmd.PodName, addCmd.PodNamespace); (ambientPod == nil) && (retries < maxStaleRetries); retries++ {
		if err != nil {
			return err
		}
		log.Warnf("got an event for pod %s in namespace %s not found in current pod cache, retry %d of %d",

			if ok {
				globalNotificationSys.ReloadPoolMeta(ctx)
			}
			z.poolMetaMutex.Unlock()
		}

		wk.Take()
		go func(setIdx int) {
			defer wk.Give()
			// We will perpetually retry listing if it fails, since we cannot
			// possibly give up in this matter
			for {
				if contextCanceled(ctx) {
					break
				}

				err := set.listObjectsToDecommission(ctx, bi,

		// If server is in shutdown.
		if atomic.LoadUint32(&srv.inShutdown) != 0 {
			// To indicate disable keep-alive, server is shutting down.
			w.Header().Set("Connection", "close")

			// Add 1 minute retry header, incase-client wants to honor it
			w.Header().Set(RetryAfter, "60")

			w.WriteHeader(http.StatusServiceUnavailable)
			w.Write([]byte(http.ErrServerClosed.Error()))
			return
		}

	cred := globalActiveCred
	if cred.AccessKey != accessKey {
		if !globalIAMSys.Initialized() {
			// Check if server has initialized, then only proceed
			// to check for IAM users otherwise its okay for clients
			// to retry with 503 errors when server is coming up.
			return auth.Credentials{}, false, ErrServerNotInitialized
		}

		// Check if the access key is part of users credentials.

		})
		if err != nil {
			if !configRetriableErrors(err) {
				logger.Fatal(err, "Unable to initialize backend")
			}
			retry := time.Duration(r.Float64() * float64(5*time.Second))
			storageLogIf(ctx, fmt.Errorf("Unable to initialize backend: %w, retrying in %s", err, retry))
			time.Sleep(retry)
			attempt++
			continue
		}
		break
	}

	return z, nil
}