// Create two contexts.
  int32_t init_timeout_in_ms = 300000;
  TFE_Context* ctx_0 =
      CreateContext(serialized_server_def_0,
                    /*isolate_session_state=*/false, init_timeout_in_ms);
  TFE_Context* ctx_1 =
      CreateContext(serialized_server_def_1,
                    /*isolate_session_state=*/false, init_timeout_in_ms);

  // Remote device on `worker2`.

  }

  {
    std::unique_ptr<ParallelTensor> int_tensors =
        parallel_device.ScalarsFromSequence<int>({5, 6}, context.get(),
                                                 status.get());
    ASSERT_EQ(TF_GetCode(status.get()), TF_OK) << TF_Message(status.get());
    ExpectScalarEq<int>(int_tensors->tensor(0), 5);
    ExpectScalarEq<int>(int_tensors->tensor(1), 6);
  }
}

}  // namespace parallel_device

  auto s = RunModel(grad_model, ctx, inputs, absl::MakeSpan(outputs),
                    /*use_function=*/use_function);
  ASSERT_EQ(errors::OK, s.code()) << s.message();

  for (int i = 0; i < num_inputs; ++i) {
    if (!outputs[i]) continue;

    AbstractTensorHandlePtr numerical_grad;
    {
      AbstractTensorHandle* numerical_grad_raw;
      s = CalcNumericalGrad(ctx, model, inputs,

#include "tensorflow/core/protobuf/tensorflow_server.pb.h"

namespace {

using ::tensorflow::string;

void ReplaceTaskInServerDef(tensorflow::ServerDef* server_def, int task_index) {
  tensorflow::JobDef* job_def = server_def->mutable_cluster()->mutable_job(0);
  int port = tensorflow::testing::PickUnusedPortOrDie();
  job_def->mutable_tasks()->at(task_index) =
      tensorflow::strings::StrCat("localhost:", port);
}

 public:
  Status Compute(AbstractContext* ctx,
                 absl::Span<AbstractTensorHandle* const> grad_outputs,
                 absl::Span<AbstractTensorHandle*> grad_inputs) override {
    for (int i = 0; i < grad_outputs.size(); i++) {
      auto grad_input = grad_outputs[i];
      // TODO(srbs): Should we add a copy contructor to AbstractTensorHandle
      // that takes care of this similar to `Tensor`?

  TFE_Op* matmul = MatMulOp(ctx, h0_task1, h1_task1);
  TFE_OpSetDevice(matmul, remote_device_name, status);
  EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);

  TFE_TensorHandle* retvals[1];
  int num_retvals = 1;
  TFE_Execute(matmul, &retvals[0], &num_retvals, status);
  EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);

  TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);

                         {12, 15, 18, 12, 15, 18, 12, 15, 18},
                         {6, 6, 6, 15, 15, 15, 24, 24, 24},
                         {6, 15, 24, 6, 15, 24, 6, 15, 24}};

  for (int i{}; i < 4; ++i) {
    bool transpose_a = transpose_a_vals[i];
    bool transpose_b = transpose_b_vals[i];
    Model MatMulModel =
        [transpose_a, transpose_b](
            AbstractContext* ctx,

  TF_RETURN_IF_ERROR(
      env->GetSymbolFromLibrary(dso_handle, "TF_InitGraph", &dso_symbol));

  // Step 2: Call `TF_InitPlugin`
  auto init_fn = reinterpret_cast<TFInitGraphPluginFn>(dso_symbol);
  return InitGraphPlugin(init_fn);
}

absl::Status InitGraphPlugin(TFInitGraphPluginFn init_fn) {
  TP_OptimizerRegistrationParams params{
      TP_OPTIMIZER_REGISTRATION_PARAMS_STRUCT_SIZE};

                                     /*isolate_session_state=*/false,
                                     /*init_timeout_in_ms=*/0);
  TFE_Context* ctx_1 = CreateContext(serialized_server_def_1,
                                     /*isolate_session_state=*/false,
                                     /*init_timeout_in_ms=*/0);

  // Remote device on `worker1`.

    const TF_Graph* fn_body, const char* fn_name, int ninputs,
    const TF_Output* inputs, std::vector<OutputTensor>* input_tensors,
    std::unordered_map<const Node*, std::vector<int>>* input_nodes)
    TF_EXCLUSIVE_LOCKS_REQUIRED(fn_body->mu) {
  input_tensors->reserve(ninputs);
  for (int i = 0; i < ninputs; ++i) {
    Node* node = inputs[i].oper ? &inputs[i].oper->node : nullptr;
    int idx = inputs[i].index;