run: |
          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
          make test-versioning

      - name: Test Multipart upload with failures
        run: |
          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
          sudo sysctl net.ipv6.conf.default.disable_ipv6=0

MinIO in distributed mode lets you pool multiple drives (even on different machines) into a single object storage server. As drives are distributed across several nodes, distributed MinIO can withstand multiple node failures and yet ensure full data protection.

## Why distributed MinIO?

`PENDING` to `COMPLETED` or `FAILED` after replication attempt either succeeded or failed respectively. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle.

To perform bi-directional replication, repeat the above process on the target site - this time setting the source bucket as the replication target. It is recommended...

      },
      "annotations": {
        "description": "MinIO instance 127.0.0.1:9000 of job minio-job has tolerance <=0 for more than 5 minutes.",
        "summary": "Instance 127.0.0.1:9000 unable to tolerate node failures"
      },
      "startsAt": "2023-11-18T06:20:09.456Z",
      "endsAt": "0001-01-01T00:00:00Z",
      "generatorURL": "http://fedora-minio:9090/graph?g0.expr=minio_cluster_health_erasure_set_tolerance+%3C%3D+0&g0.tab=1",

          f'Here\'s a fragment of the log containing the failure:\n\n'
          f'[ ... TRUNCATED ... ]\n\n'
          f'{invocation_results["log_fragment"]}\n'
          f'\n[ ... TRUNCATED ... ]\n'
      )
      failure = ElemTree.SubElement(
          testcase, elem_name,
          message=f'Bazel invocation {invocation_id} failed.')
      failure.text = failure_msg
    else:

		},
		Totals: madmin.RStat{
			Count: float64(rt.SinceUptime.Count),
			Bytes: rt.SinceUptime.Bytes,
		},
		ErrCounts: errCounts,
	}
}

func (rt *RTimedMetrics) addsize(size int64, err error) {
	// failures seen since uptime
	atomic.AddInt64(&rt.SinceUptime.Bytes, size)
	atomic.AddInt64(&rt.SinceUptime.Count, 1)
	rt.LastMinute.addsize(size)
	rt.LastHour.addsize(size)

        }
    }

    /**
     * Builds a failure message from a bulk response containing failed operations.
     *
     * @param bulkResponse The bulk response containing potential failures.
     * @param ignoreAlreadyExists Whether to ignore already existing document failures.
     * @return The formatted failure message, or empty string if no failures to report.
     */

	// Total Size for objects that have not yet been replicated
	ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"`
	// Total size for objects that have witness one or more failures and will be retried
	ReplicationFailedSizeV1 uint64 `json:"objectsFailedReplicationTotalSize"`
	// Total size for objects that have been replicated to destination
	ReplicatedSizeV1 uint64 `json:"objectsReplicatedTotalSize"`

		Help:      "Number of failures in GET requests proxied to replication target",
		Type:      counterMetric,
	}
}

func getClusterReplProxiedHeadFailedOperationsMD(ns MetricNamespace) MetricDescription {
	return MetricDescription{
		Namespace: ns,
		Subsystem: replicationSubsystem,
		Name:      proxiedHeadRequestsFailures,
		Help:      "Number of failures in HEAD requests proxied to replication target",

	if err != nil {
		return err
	}

	*b = BatchJobNotification(tmp)
	b.line, b.col = val.Line, val.Column
	return nil
}

// BatchJobRetry stores retry configuration used in the event of failures.
type BatchJobRetry struct {
	line, col int
	Attempts  int           `yaml:"attempts" json:"attempts"` // number of retry attempts
	Delay     time.Duration `yaml:"delay" json:"delay"`       // delay between each retries