// Create two worker tasks, using single host server defs.
  // A single host server def contains a client and the remote host.
  // Example:
  //
  //  Worker1:
  //  cluster { job { name: "client" tasks { key: 0 value: "localhost:14525" } }
  //            job { name: "worker" tasks { key: 1 value: "localhost:14523" } }
  //          } job_name: "worker" task_index: 1 protocol: "grpc"
  //

default   details-v1-698d88b-krdw7       10.244.2.55 ambient-worker2 None     HBONE
default   httpbin-7447985f87-t8hv7       10.244.1.40 ambient-worker  None     TCP
default   productpage-v1-675fc69cf-kkrm2 10.244.2.56 ambient-worker2 None     HBONE
default   ratings-v1-6484c4d9bb-8xc2r    10.244.2.57 ambient-worker2 None     HBONE
default   reviews-v1-5b5d6494f4-c7z5w    10.244.1.41 ambient-worker  None     HBONE

      "workloadName": "ratings-v1",
      "workloadType": "deployment",
      "canonicalName": "ratings",
      "canonicalRevision": "v1",
      "network": "network1",
      "node": "ambient-worker2",
      "nativeTunnel": true,
      "status": "Healthy",
      "clusterId": "Kubernetes"
    },
    "/10.244.2.58": {
      "workloadIps": [
        "10.244.2.58"
      ],
      "protocol": "HBONE",

  int port = tensorflow::testing::PickUnusedPortOrDie();
  // 3. Replace worker1 with a new worker in server_def_0 and server_def_1.
  ReplaceTaskInServerDef(&server_def_0, /*task_index=*/1, "localhost", port);
  ReplaceTaskInServerDef(&server_def_1, /*task_index=*/1, "localhost", port);
  // 4. Start a new task to replace worker1.
  server_def_0.set_task_index(1);
  worker_server1.release();

		workers = workers[:len(workers)-1]
		worker <- expiryOp(nil)
		es.stats.workers.Add(-1)
	}
	// Atomically replace workers.
	es.workers.Store(&workers)
}

// Worker handles 4 types of expiration tasks.
// 1. Expiry of objects, includes regular and transitioned objects
// 2. Expiry of noncurrent versions due to NewerNoncurrentVersions

	if (checkOld > 0 && len(p.workers) != checkOld) || n == len(p.workers) || n < 1 {
		// Either already satisfied or worker count changed while we waited for the lock.
		return
	}
	for len(p.workers) < n {
		input := make(chan ReplicationWorkerOperation, 10000)
		p.workers = append(p.workers, input)

		go p.AddWorker(input, &p.activeWorkers)
	}
	for len(p.workers) > n {
		worker := p.workers[len(p.workers)-1]

  /**
   * This counter prevents an ABA issue where a thread may successfully schedule the worker, the
   * worker runs and exhausts the queue, another thread enqueues a task and fails to schedule the
   * worker, and then the first thread's call to delegate.execute() returns. Without this counter,
   * it would observe the QUEUING state and set it to QUEUED, and the worker would never be
   * scheduled again for future submissions.
   */
  @GuardedBy("queue")

  /**
   * This counter prevents an ABA issue where a thread may successfully schedule the worker, the
   * worker runs and exhausts the queue, another thread enqueues a task and fails to schedule the
   * worker, and then the first thread's call to delegate.execute() returns. Without this counter,
   * it would observe the QUEUING state and set it to QUEUED, and the worker would never be
   * scheduled again for future submissions.
   */
  @GuardedBy("queue")

And if you need to have multiple workers, you can simply use the `--workers` command line option.

/// note | Technical Details

The Docker image was created when Uvicorn didn't support managing and restarting dead workers, so it was needed to use Gunicorn with Uvicorn, which added quite some complexity, just to have Gunicorn manage and restart the Uvicorn worker processes.

Here are some possible combinations and strategies:

* **Uvicorn** with `--workers`
    * One Uvicorn **process manager** would listen on the **IP** and **port**, and it would start **multiple Uvicorn worker processes**.
* **Kubernetes** and other distributed **container systems**