Krishnan Parthasarathi <******@****.***> 1725619883 -0700

    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "Instance {{ $labels.server }} has lost quorum on pool {{ $labels.pool }} on set {{ $labels.set }}"
      description: "MinIO instance {{ $labels.server }} of job {{ $labels.job }} has lost quorum on pool {{ $labels.pool }} on set {{ $labels.set }} for more than 5 minutes."
```

## Verify the configuration and alerts

## Cluster probe

### Cluster-writeable probe

The reply is '200 OK' if cluster has write quorum if not it returns '503 Service Unavailable'.

```
curl http://minio1:9001/minio/health/cluster
HTTP/1.1 503 Service Unavailable
Accept-Ranges: bytes
Content-Length: 0
Server: MinIO
Vary: Origin
X-Amz-Bucket-Region: us-east-1
X-Minio-Write-Quorum: 3
X-Amz-Request-Id: 16239D6AB80EBECF
X-Xss-Protection: 1; mode=block

		"Overall write quorum across pools and sets")
	erasureSetOverallHealthMD = NewGaugeMD(erasureSetOverallHealth,
		"Overall health across pools and sets (1=healthy, 0=unhealthy)")
	erasureSetReadQuorumMD = NewGaugeMD(erasureSetReadQuorum,
		"Read quorum for the erasure set in a pool", poolIDL, setIDL)
	erasureSetWriteQuorumMD = NewGaugeMD(erasureSetWriteQuorum,
		"Write quorum for the erasure set in a pool", poolIDL, setIDL)

package cmd

import "errors"

// errErasureReadQuorum - did not meet read quorum.
var errErasureReadQuorum = errors.New("Read failed. Insufficient number of drives online")

// errErasureWriteQuorum - did not meet write quorum.
var errErasureWriteQuorum = errors.New("Write failed. Insufficient number of drives online")

// Encode reads from the reader, erasure-encodes the data and writes to the writers.
func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []io.Writer, buf []byte, quorum int) (total int64, err error) {
	writer := &multiWriter{
		writers:     writers,
		writeQuorum: quorum,
		errs:        make([]error, len(writers)),
	}

	for {
		var blocks [][]byte
		n, err := io.ReadFull(src, buf)
		if err != nil {

| Minimum number of drives per server when server count is 4      | 01            |
| Maximum number of drives per server                             | no-limit      |
| Read quorum                                                     | N/2           |
| Write quorum                                                    | N/2+1         |

## Limits of S3 API

	localDrives := cloneDrives(globalLocalDrivesMap)
	globalLocalDrivesMu.RUnlock()

	quorum := (len(localDrives) / 2)

	buckets = make([]BucketInfo, 0, 32)
	healBuckets := xsync.NewMapOf[string, VolInfo]()

	// lists all unique buckets across drives.
	if err := listAllBuckets(ctx, localDrives, healBuckets, quorum); err != nil {
		return nil, err
	}

	// include deleted buckets in listBuckets output

sleep 1m

docker system prune -f || true
docker volume prune -f || true
docker volume rm $(docker volume ls -q -f dangling=true) || true

# Stop two nodes, one of each pool, to check that all S3 calls work while quorum is still there
[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio2
[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio6

	if err != nil {
		log.Fatalln(err)
	}

	opts := madmin.HealOpts{
		Recursive: true,                  // recursively heal all objects at 'prefix'
		Remove:    true,                  // remove content that has lost quorum and not recoverable
		ScanMode:  madmin.HealNormalScan, // by default do not do 'deep' scanning
	}

	start, _, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, "", false, false)
	if err != nil {