)

func logOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
	logger.LogOnceIf(ctx, logSubsys, err, id, errKind...)
}

// ErrTargetsOffline - Indicates single/multiple target failures.
var ErrTargetsOffline = errors.New("one or more targets are offline. Please use `mc admin info --json` to check the offline targets")

// TestSubSysLambdaTargets - tests notification targets of given subsystem

`PENDING` to `COMPLETED` or `FAILED` after replication attempt either succeeded or failed respectively. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle.

To perform bi-directional replication, repeat the above process on the target site - this time setting the source bucket as the replication target. It is recommended...

| `minio_bucket_replication_proxied_get_requests_failures` | Number of failures in GET requests proxied to replication target                          |
| `minio_bucket_replication_proxied_head_requests_failures` | Number of failures in HEAD requests proxied to replication target                          |

				objCountPerThread[i]++
				mu.Lock()
				uploadTimes = append(uploadTimes, response)
				mu.Unlock()
			}
		}(i)
	}
	wg.Wait()

	// We already saw write failures, no need to proceed into read's
	if retError != "" {
		return SpeedTestResult{
			Uploads:     totalBytesWritten,
			Downloads:   totalBytesRead,
			UploadTimes: uploadTimes,
			Error:       retError,
		}, nil

          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "minio_bucket_replication_proxied_head_requests_failures{job=\"$scrape_jobs\"}",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "thresholds": [],

			return entries, ctx.Err()
		}

		// If many failures, check the cache state.
		if retries > 10 {
			err := o.checkMetacacheState(ctx, rpc)
			if err != nil {
				return entries, fmt.Errorf("remote listing canceled: %w", err)
			}
			retries = 1
		}

		// All operations are performed without locks, so we must be careful and allow for failures.
		// Read metadata associated with the object from a disk.

          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,
          "expr": "minio_bucket_replication_proxied_head_requests_failures{job=\"$scrape_jobs\"}",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],

type DeleteBucketOptions struct {
	NoLock     bool             // does not lock the delete bucket call if set to 'true'
	NoRecreate bool             // do not recreate bucket on delete failures
	Force      bool             // Force deletion
	SRDeleteOp SRBucketDeleteOp // only when site replication is enabled
}

// BucketOptions provides options for ListBuckets and GetBucketInfo call.

All replication failures are picked up by the scanner which runs at a one minute frequency, each time scanning upto a sixteenth of the namespace. Object versions marked `PENDING` or `FAILED` are re-queued for replication.

func checkFailedUnlocks(locks []string, tolerance int) bool {
	unlocksFailed := 0
	for lockID := range locks {
		if isLocked(locks[lockID]) {
			unlocksFailed++
		}
	}

	// Unlock failures are higher than tolerance limit
	// for this instance of unlocker, we should let the
	// caller know that lock is not successfully released
	// yet.
	if len(locks)-tolerance == tolerance {