// simple function which returns a message which gives the context of the test
	// and then followed by the actual error message.
	failTestStr := func(testType, failMsg string) string {
		return fmt.Sprintf("MinIO %s: %s fail for \"%s\": \n<Error> %s", instanceType, testType, testName, failMsg)
	}

	// httptest Recorder to capture all the response by the http handler.
	rec := httptest.NewRecorder()

| `minio_cluster_iam_plugin_authn_service_last_fail_seconds`      | `counter` | When plugin authentication is configured, returns time (in seconds) since the last failed request to the service         |        |

		uploadedParts = append(uploadedParts, minio.CompletePart{
			PartNumber: pInfo.PartNumber,
			ETag:       pInfo.ETag,
		})
	}

	// really big value but its okay on heavily loaded systems. This is just tail end timeout.
	cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute)
	defer ccancel()
	_, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{

	}
	if !currDeploymentIDsSet.IsEmpty() {
		// If current cluster is already SR enabled and no new site being added ,fail.
		if currDeploymentIDsSet.Equals(deploymentIDsSet) {
			return madmin.ReplicateAddStatus{}, errSRCannotJoin
		}
		if len(currDeploymentIDsSet.Intersection(deploymentIDsSet)) != len(currDeploymentIDsSet) {

const (
	lastSyncDurationMillis                 = "last_sync_duration_millis"
	pluginAuthnServiceFailedRequestsMinute = "plugin_authn_service_failed_requests_minute"
	pluginAuthnServiceLastFailSeconds      = "plugin_authn_service_last_fail_seconds"
	pluginAuthnServiceLastSuccSeconds      = "plugin_authn_service_last_succ_seconds"
	pluginAuthnServiceSuccAvgRttMsMinute   = "plugin_authn_service_succ_avg_rtt_ms_minute"

	// Storage operations.

	// Returns true if disk is online and its valid i.e valid format.json.
	// This has nothing to do with if the drive is hung or not responding.
	// For that individual storage API calls will fail properly. The purpose
	// of this function is to know if the "drive" has "format.json" or not
	// if it has a "format.json" then is it correct "format.json" or not.
	IsOnline() bool

failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)

## we do not need to fail here, since we are going to test
## upgrading to master, healing and being able to recover
## the last version.
if [ $failed_count_site1 -ne 0 ]; then
	echo "failed with multipart on site1 uploads ${failed_count_site1}"
fi

		}

		cctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		if prefix == "" {
			// if all objects are not deleted yet this call may fail.
			return clnt.RemoveBucket(cctx, bucket)
		}

		objectsCh := make(chan minio.ObjectInfo)

		// Send object names that are needed to be removed to objectsCh
		go func() {

		t.Errorf("expected: success-vol to be created")
	}

	// removing the path and simulating disk failure
	os.RemoveAll(path)
	// should fail with errDiskNotFound.
	if _, err = xlStorage.ListVols(context.Background()); err != errDiskNotFound {
		t.Errorf("Expected to fail with \"%s\", but instead failed with \"%s\"", errDiskNotFound, err)
	}
}

				offlineDrives++
				continue
			}
		}

		if offlineDrives >= (len(onlineDisks)+1)/2 {
			// if offline drives are more than 50% of the drives
			// we have no quorum, we shouldn't proceed just
			// fail at that point.
			return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
		}

		if parityDrives >= len(onlineDisks)/2 {
			parityDrives = len(onlineDisks) / 2
		}

		if parityOrig != parityDrives {