| `minio_cluster_health_erasure_set_online_drives`  | Count of online drives in the erasure set      |
| `minio_cluster_health_erasure_set_read_quorum`    | Get read quorum of the erasure set             |
| `minio_cluster_health_erasure_set_write_quorum`   | Get write quorum of the erasure set            |
| `minio_cluster_health_erasure_set_status`         | Get current health status of the erasure set   |

## Cluster Replication Metrics

	healMRFMetaFormat    = 1
	healMRFMetaVersionV1 = 1
)

// PartialOperation is a successful upload/delete of an object
// but not written in all disks (having quorum)
type PartialOperation struct {
	Bucket              string
	Object              string
	VersionID           string
	Versions            []byte
	SetIndex, PoolIndex int
	Queued              time.Time

	case <-ctx.Done():
		t.Fatal("Lock context canceled which is not expected")
	case <-timer.C:
	}

	// Should be safe operation in all cases
	dm.Unlock(t.Context())
}

// Test canceling context while quorum servers report lock not found
func TestFailedRefreshLock(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode.")
	}

	// Simulate Refresh response to return no locking found

- Any unrecognized disks - return failure

Some disks are offline and we have quorum.
-----------------
- Some unformatted - format all and return success,
  treat disks offline as corrupted.
- Any JBOD inconsistent - return failure
- Some are corrupt (missing format.json)
- Any unrecognized disks - return failure

No read quorum
-----------------
failure for all cases.

test-multipart: install-race ## test multipart
	@echo "Test multipart behavior when part files are missing"
	@(env bash $(PWD)/buildscripts/multipart-quorum-test.sh)

test-timeout: install-race ## test multipart
	@echo "Test server timeout"
	@(env bash $(PWD)/buildscripts/test-timeout.sh)

verify: install-race ## verify minio various setups

	listingQuorum := (set.setDriveCount + 1) / 2

	// How to resolve partial results.
	resolver := metadataResolutionParams{
		dirQuorum: listingQuorum, // make sure to capture all quorum ratios
		objQuorum: listingQuorum, // make sure to capture all quorum ratios
		bucket:    bucketName,
	}

	err := listPathRaw(ctx, listPathRawOptions{
		disks:          disks,
		bucket:         bucketName,
		recursive:      true,

	reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
	ctx, cancelCtx := context.WithCancel(logger.SetReqInfo(GlobalContext, reqInfo))

	hs := madmin.HealOpts{
		// Remove objects that do not have read-quorum
		Remove: healDeleteDangling,
	}

	return &healSequence{
		startTime:   UTCNow(),
		clientToken: bgHealingUUID,
		// run-background heal with reserved bucket
		bucket:   minioReservedBucket,
		settings: hs,

          "fullMetaSearch": false,
          "hide": false,
          "includeNullMetadata": true,
          "instant": false,
          "legendFormat": "Pool {{pool}} / Set {{set}} - Read Quorum",
          "range": true,
          "refId": "B",
          "useBackend": false
        },
        {
          "datasource": {
            "type": "prometheus",

				if err == nil {
					allAtEOF = false
				}
				errs[i] = err
			}(len(errs), set)
			errs = append(errs, nil)
		}
	}
	mu.Unlock()

	// Gather results to a single channel.
	// Quorum is one since we are merging across sets.
	err := mergeEntryChannels(ctx, inputs, results, 1)

	cancelList()
	wg.Wait()

	// we should return 'errs' from per disk
	if isAllNotFound(errs) {

	})
	return mg
}

func getClusterWriteQuorumMD() MetricDescription {
	return MetricDescription{
		Namespace: clusterMetricNamespace,
		Subsystem: "write",
		Name:      "quorum",
		Help:      "Maximum write quorum across all pools and sets",
		Type:      gaugeMetric,
	}
}

func getClusterHealthStatusMD() MetricDescription {
	return MetricDescription{
		Namespace: clusterMetricNamespace,