"hide": false,
          "instant": true,
          "legendFormat": ".",
          "range": false,
          "refId": "B"
        }
      ],
      "title": "Total Online/Offline Drives",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {

				logger.Info(fmt.Sprintf("Waiting for at least %d remote servers with valid configuration to be online", len(clnts)/2))
				if len(offlineEndpoints) > 0 {
					logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
				}
				if len(incorrectConfigs) > 0 {
					logger.Info(fmt.Sprintf("Following servers have mismatching configuration %s", incorrectConfigs))
				}

			_, ok := disks[d.Endpoint]
			if !ok {
				continue
			}
		}

		if d.State != madmin.DriveStateOk && d.State != madmin.DriveStateUnformatted {
			metrics[d.Endpoint] = madmin.DiskMetric{NDisks: 1, Offline: 1}
			continue
		}

		var dm madmin.DiskMetric
		dm.NDisks = 1
		if d.Healing {
			dm.Healing++
		}
		if d.Metrics != nil {
			dm.LifeTimeOps = make(map[string]uint64, len(d.Metrics.APICalls))

	wg      sync.WaitGroup
}

func newMRFState() mrfState {
	return mrfState{
		opCh: make(chan PartialOperation, mrfOpsQueueSize),
	}
}

// Add a partial S3 operation (put/delete) when one or more disks are offline.
func (m *mrfState) addPartialOp(op PartialOperation) {
	if m == nil {
		return
	}

	if atomic.LoadInt32(&m.closed) == 1 {
		return
	}

	m.wg.Add(1)
	defer m.wg.Done()

		return
	}
	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
		replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN)
		sendEvent(eventArgs{
			BucketName: dobj.Bucket,
			Object: ObjectInfo{
				Bucket:       dobj.Bucket,
				Name:         dobj.ObjectName,

	if err != nil {
		var drivesNotFound int
		for _, format := range formatConfigs {
			if format != nil {
				continue
			}
			drivesNotFound++
		}
		return nil, fmt.Errorf("%w (offline-drives=%d/%d)", err, drivesNotFound, len(formatConfigs))
	}

	if format.ID == "" {
		return nil, errors.New("deployment ID missing from disk format, unable to start the server")
	}

	return format, nil
}

	}

	inconsistent := 0
	for i, meta := range partsMetadata {
		if !meta.IsValid() {
			// Since for majority of the cases erasure.Index matches with erasure.Distribution we can
			// consider the offline disks as consistent.
			continue
		}
		if !meta.Deleted {
			if len(meta.Erasure.Distribution) != len(onlineDisks) {
				// Erasure distribution seems to have lesser

package cmd

import (
	"bytes"
	"crypto/rand"
	"io"
	"os"
	"testing"
)

var erasureHealTests = []struct {
	dataBlocks, disks int

	// number of offline disks is also number of staleDisks for
	// erasure reconstruction in this test
	offDisks int

	// bad disks are online disks which return errors
	badDisks, badStaleDisks int

	blocksize, size int64

   *       {@code consistentHash} will handle it. If, however, you are dividing traffic among
   *       servers {@code alpha}, {@code bravo}, and {@code charlie} and you occasionally need to
   *       take each of the servers offline, {@code consistentHash} will be a poor fit: It provides
   *       no way for you to specify which of the three buckets is disappearing. Thus, if your

		} else if bitrotHeal == 1 {
			return newBuf, errFileCorrupt
		}
		return newBuf, nil
	}

	// If we cannot decode, just return read quorum error.
	return nil, fmt.Errorf("%w (offline-disks=%d/%d)", errErasureReadQuorum, disksNotFound, len(p.readers))
}

// Decode reads from readers, reconstructs data if needed and writes the data to the writer.