echo "my content" | /tmp/mc pipe myminio/testbucket/file-$i
	done
}

function perform_test() {
	start_port=$2

	start_minio_3_node $start_port

	echo "Testing Distributed Erasure setup healing of drives"
	echo "Remove the contents of the disks belonging to '${1}' node"

	rm -rf ${WORK_DIR}/${1}/*/

	set -x
	start_minio_3_node $start_port

	check_heal ${1}
	rv=$?

To verify the above sample alert follow below steps

1. Start a distributed MinIO instance (4 nodes setup)
2. Start Prometheus server and AlertManager
3. Bring down couple of MinIO instances to bring down the Erasure Set tolerance to -1 and verify the same with `mc admin prometheus metrics ALIAS | grep minio_cluster_health_erasure_set_status`

NOTE:

- `timeToFirstByte` and `timeToResponse` will be expressed in Nanoseconds.
- Additionally in the case of the erasure coded setup `tags.objectLocation` provides per object details about
  - Pool number the object operation was performed on.
  - Set number the object operation was performed on.

## Configuration Directory

package cmd

import (
	"net/http"

	"github.com/minio/minio/internal/grid"
	"github.com/minio/mux"
)

// Composed function registering routers for only distributed Erasure setup.
func registerDistErasureRouters(router *mux.Router, endpointServerPools EndpointServerPools) {
	var (
		lockGrid   = globalLockGrid.Load()
		commonGrid = globalGrid.Load()
	)

- All versions, including delete-markers should be deleted before deleting a bucket.
- **Versioning feature is only available in erasure coded and distributed erasure coded setups**.

## How to configure versioning on a bucket

Each bucket created has a versioning configuration associated with it. By default bucket is unversioned as shown below

```

mc: Health data saved to dc-11-health_20200321053323.json.gz
```

The gzipped output contains debugging information for your system

## Decoding Metadata

Metadata is stored in `xl.meta` files for erasure coded objects. Each disk in the set containing the object has this file. The file format is a binary format and therefore requires tools to view values.

### Installing xl-meta

	IdleMode int32 // 0 => on, 1 => off

	// Alert upon this many excess object versions
	ExcessVersions int64 // 100

	// Alert upon this many excess sub-folders per folder in an erasure set.
	ExcessFolders int64 // 50000

	// MaxWait is maximum wait time between operations
	MaxWait time.Duration
	// Cycle is the time.Duration between each scanner cycles
	Cycle time.Duration
}

`bucket1.domain.com` will point to IP address set in `MINIO_PUBLIC_IPS`.

- This field is mandatory for standalone and erasure code MinIO server deployments, to enable federated mode.
- This field is optional for distributed deployments. If you don't set this field in a federated setup, we use the IP addresses of

			t.Fatalf("Test %d: expected %v but got %v for %v", i+1, tc.ondisk, actual, tc.meta)
		}
	}
}

func TestObjectIsRemote(t *testing.T) {
	fi := newFileInfo("object", 8, 8)
	fi.Erasure.Index = 1
	if !fi.IsValid() {
		t.Fatalf("unable to get xl meta")
	}

	testCases := []struct {
		meta   map[string]string
		remote bool
	}{
		{
			// restore in progress