backwards. See PR
            [47925](https://github.com/tensorflow/tensorflow/pull/47925).
        *   `tf.image.crop_and_resize` gradient w.r.t. either `image` or
            `boxes`. See PR
            [48905](https://github.com/tensorflow/tensorflow/pull/48905).
        *   `tf.sparse.sparse_dense_matmul` forwards. See PR

    forward_grads.resize(output_tensors.size());
    TF_RETURN_IF_ERROR(ForwardpropFromTape(
        op_type, output_tensors, backward_function_getter,
        backward_function_deleter, in_grads, absl::MakeSpan(forward_grads)));
  } else {
    TF_RETURN_IF_ERROR(
        (*forward_function)(in_grads, &forward_grads, use_batch_));
  }
  for (int i = 0; i < forward_grads.size(); ++i) {
    if (forward_grads[i] != nullptr) {

      ASSERT_NEAR(manuals[j], danalytical[j], abs_error);
    }
  }

  TF_DeleteTensor(analytical_tensor);
  delete[] danalytical;
}

Model BuildGradModel(Model forward, GradientRegistry registry) {
  return [forward_model = std::move(forward),
          grad_registry = std::move(registry)](
             AbstractContext* ctx,
             absl::Span<AbstractTensorHandle* const> inputs,

      ASSERT_EQ(errors::OK, status_.code()) << status_.message();
      immediate_execution_ctx_.reset(ctx_raw);
    }

    // Computing numerical gradients with TensorFloat-32 is numerically
    // unstable. Some forward pass tests also fail with TensorFloat-32 due to
    // low tolerances
    enable_tensor_float_32_execution(false);
  }

  AbstractContextPtr immediate_execution_ctx_;
  GradientRegistry registry_;

# This is the same as the official TensorFlow builds.
# See https://developer.nvidia.com/cuda-gpus#compute
# `compute_XY` enables PTX embedding in addition to SASS. PTX
# is forward compatible beyond the current compute capability major
# release while SASS is only forward compatible inside the current
# major release. Example: sm_80 kernels can run on sm_89 GPUs but
# not on sm_90 GPUs. compute_80 kernels though can also run on sm_90 GPUs.

    t.reset(x_raw);
  }

  AbstractOperationPtr check_numerics_op(ctx->CreateOperation());
  ForwardOperation forward_op;
  Status s = Reset(check_numerics_op.get(), "CheckNumerics",
                   /*raw_device_name=*/nullptr, &forward_op);
  ASSERT_EQ(errors::OK, s.code()) << s.message();
  if (isa<TracingOperation>(check_numerics_op.get())) {

Status RunAndMaybeSum(AbstractContext* ctx, Model forward,
                      absl::Span<AbstractTensorHandle* const> inputs,
                      absl::Span<AbstractTensorHandle*> outputs,
                      bool use_function) {
  AbstractTensorHandle* model_outputs[1];

  // Run the model.
  TF_RETURN_IF_ERROR(
      RunModel(forward, ctx, inputs, model_outputs, use_function));

    device_threads_.emplace_back(new DeviceThread(
        devices[device_index].c_str(), is_async, in_flight_nodes_limit));
  }
}

// Necessary for a unique_ptr to a forward-declared type.
ParallelDevice::~ParallelDevice() = default;

std::unique_ptr<ParallelTensor> ParallelDevice::CopyToParallelDevice(
    TFE_Context* context, TFE_TensorHandle* tensor, TF_Status* status) const {

#include "tensorflow/core/framework/types.h"

namespace tensorflow {
namespace parallel_device {

using TensorHandlePtr = tensorflow::Safe_TFE_TensorHandlePtr;

class ParallelTensor;
class DeviceThread;

// Forwards operations to `devices`, maintaining ParallelTensor with components
// placed on each underlying device.
class ParallelDevice {
 public:
  // Eager async execution is only supported when remote eager is not in use