namespace gradients {
namespace {

class ReluGradientFunction : public GradientFunction {
 public:
  explicit ReluGradientFunction(vector<AbstractTensorHandle*> f_outputs)
      : forward_outputs_(f_outputs) {
    for (auto output : forward_outputs_) {
      if (output) {
        output->Ref();
      }
    }
  }

  Status Compute(AbstractContext* ctx,
                 absl::Span<AbstractTensorHandle* const> grad_outputs,

    t.reset(x_raw);
  }

  AbstractOperationPtr check_numerics_op(ctx->CreateOperation());
  ForwardOperation forward_op;
  Status s = Reset(check_numerics_op.get(), "CheckNumerics",
                   /*raw_device_name=*/nullptr, &forward_op);
  ASSERT_EQ(errors::OK, s.code()) << s.message();
  if (isa<TracingOperation>(check_numerics_op.get())) {

Status RunAndMaybeSum(AbstractContext* ctx, Model forward,
                      absl::Span<AbstractTensorHandle* const> inputs,
                      absl::Span<AbstractTensorHandle*> outputs,
                      bool use_function) {
  AbstractTensorHandle* model_outputs[1];

  // Run the model.
  TF_RETURN_IF_ERROR(
      RunModel(forward, ctx, inputs, model_outputs, use_function));

    device_threads_.emplace_back(new DeviceThread(
        devices[device_index].c_str(), is_async, in_flight_nodes_limit));
  }
}

// Necessary for a unique_ptr to a forward-declared type.
ParallelDevice::~ParallelDevice() = default;

std::unique_ptr<ParallelTensor> ParallelDevice::CopyToParallelDevice(
    TFE_Context* context, TFE_TensorHandle* tensor, TF_Status* status) const {

      ASSERT_EQ(errors::OK, status_.code()) << status_.message();
      immediate_execution_ctx_.reset(ctx_raw);
    }

    // Computing numerical gradients with TensorFloat-32 is numerically
    // unstable. Some forward pass tests also fail with TensorFloat-32 due to
    // low tolerances
    enable_tensor_float_32_execution(false);
  }

  AbstractContextPtr immediate_execution_ctx_;
  GradientRegistry registry_;
  Status status_;

    //   |      dz|          |
    //   |        |          |
    //   |     Const_3       |
    //   |                   |
    //   |        ^          |
    //   |       z|          |     // MatMul Forward Graph
    //   |        |          |
    //   |      MatMul       |
    //   |     /       \     |
    //   |    ^         ^    |
    //   |    |         |    |
    //   |---x|        y|----|

    forward_grads.resize(output_tensors.size());
    TF_RETURN_IF_ERROR(ForwardpropFromTape(
        op_type, output_tensors, backward_function_getter,
        backward_function_deleter, in_grads, absl::MakeSpan(forward_grads)));
  } else {
    TF_RETURN_IF_ERROR(
        (*forward_function)(in_grads, &forward_grads, use_batch_));
  }
  for (int i = 0; i < forward_grads.size(); ++i) {
    if (forward_grads[i] != nullptr) {

      ASSERT_NEAR(manuals[j], danalytical[j], abs_error);
    }
  }

  TF_DeleteTensor(analytical_tensor);
  delete[] danalytical;
}

Model BuildGradModel(Model forward, GradientRegistry registry) {
  return [forward_model = std::move(forward),
          grad_registry = std::move(registry)](
             AbstractContext* ctx,
             absl::Span<AbstractTensorHandle* const> inputs,

# This is the same as the official TensorFlow builds.
# See https://developer.nvidia.com/cuda-gpus#compute
# `compute_XY` enables PTX embedding in addition to SASS. PTX
# is forward compatible beyond the current compute capability major
# release while SASS is only forward compatible inside the current
# major release. Example: sm_80 kernels can run on sm_89 GPUs but
# not on sm_90 GPUs. compute_80 kernels though can also run on sm_90 GPUs.

             const char* raw_device_name, ForwardOperation* forward_op_) {
  forward_op_->op_name = op;
  forward_op_->attrs.Reset(op);
  return op_->Reset(op, raw_device_name);
}
Status AddInput(AbstractOperation* op_, AbstractTensorHandle* input,
                ForwardOperation* forward_op_) {
  TF_RETURN_IF_ERROR(op_->AddInput(input));
  forward_op_->inputs.push_back(input);
  return absl::OkStatus();
}