In most setups this is sufficient to heal the content after drive replacements. Setting `max_sleep` to a *lower* value and setting `max_io` to a *higher* value would make heal go faster.

}

function perform_test() {
	start_minio_3_node 120 $2

	echo "Testing Distributed Erasure setup healing of drives"
	echo "Remove the contents of the disks belonging to '${1}' erasure set"

	rm -rf ${WORK_DIR}/${1}/*/

	set -x
	start_minio_3_node 120 $2

	rv=$(check_online)
	if [ "$rv" == "1" ]; then

	// if this header is set to "true"
	MinIOStorageClassDefaults = "x-minio-storage-class-defaults"

	// Reports number of drives currently healing
	MinIOHealingDrives = "x-minio-healing-drives"

	// Header indicates if the delete marker should be preserved by client
	MinIOSourceDeleteMarker = "x-minio-source-deletemarker"

	// Header indicates if the delete marker version needs to be purged.

	storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "")
	if err != nil {
		t.Fatalf("Unable to format drives for erasure, %s", err)
	}

	ep := PoolEndpoints{Endpoints: endpoints}

	parity, err := ecDrivesNoConfig(16)
	if err != nil {
		t.Fatalf("Unexpected error during EC drive config: %v", err)
	}
	if _, err := newErasureSets(ctx, ep, storageDisks, format, parity, 0); err != nil {

// first server to initialize them in distributed set to initialize them.
var errNotFirstDisk = errors.New("Not first drive")

// error returned by first disk waiting to initialize other servers.
var errFirstDiskWait = errors.New("Waiting on other drives")

// error returned for a negative actual size.
var errInvalidDecompressedSize = errors.New("Invalid Decompressed Size")

          "expr": "minio_node_drive_used_bytes{job=~\"$scrape_jobs\",server=\"$server\"}",
          "interval": "",
          "legendFormat": "Used [{{drive}}]",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "exemplar": true,

}

func (h *healSequence) countFailed(res madmin.HealResultItem) {
	h.mutex.Lock()
	defer h.mutex.Unlock()

	for _, d := range res.After.Drives {
		// For failed items we report the endpoint and drive state
		// This will help users take corrective actions for drives
		h.healFailedItemsMap[d.Endpoint+","+d.State]++
	}

	h.lastHealActivity = UTCNow()
}

	if err != nil {
		t.Fatal(err)
	}

	// Object was uploaded with 4 known bad drives, so we should still be able to lose 3 drives and still write to the object.
	erasureDisks = xl.getDisks()
	z.serverPools[0].erasureDisksMu.Lock()
	xl.getDisks = func() []StorageAPI {
		erasureDisks[7] = nil
		erasureDisks[8] = nil

          "editorMode": "builder",
          "expr": "minio_cluster_health_erasure_set_online_drives{job=~\"$scrape_jobs\"}",
          "fullMetaSearch": false,
          "hide": false,
          "includeNullMetadata": true,
          "instant": false,
          "legendFormat": "Pool {{pool}} / Set {{set}} - Online Drives",
          "range": true,
          "refId": "A",
          "useBackend": false
        },

		readsPerSec:       "Reads per second on a drive",
		writesPerSec:      "Writes per second on a drive",
		readsKBPerSec:     "Kilobytes read per second on a drive",
		writesKBPerSec:    "Kilobytes written per second on a drive",
		readsAwait:        "Average time for read requests to be served on a drive",
		writesAwait:       "Average time for write requests to be served on a drive",