pluginAuthnServiceLastFailSeconds      = "plugin_authn_service_last_fail_seconds"
	pluginAuthnServiceLastSuccSeconds      = "plugin_authn_service_last_succ_seconds"
	pluginAuthnServiceSuccAvgRttMsMinute   = "plugin_authn_service_succ_avg_rtt_ms_minute"
	pluginAuthnServiceSuccMaxRttMsMinute   = "plugin_authn_service_succ_max_rtt_ms_minute"

	loadNodesUpDown := func() (v nodesOnline, err error) {
		v.Online, v.Offline = globalNotificationSys.GetPeerOnlineCount()
		return
	}
	return cachevalue.NewFromFunc(1*time.Minute,
		cachevalue.Opts{ReturnLastGood: true},
		loadNodesUpDown)
}

type driveIOStatMetrics struct {
	readsPerSec    float64
	readsKBPerSec  float64
	readsAwait     float64
	writesPerSec   float64

		WriteBufferSize:       64 << 10, // 64KiB moving up from 4KiB default
		ReadBufferSize:        64 << 10, // 64KiB moving up from 4KiB default
		IdleConnTimeout:       15 * time.Second,
		ResponseHeaderTimeout: 15 * time.Minute, // Conservative timeout is the default (for MinIO internode)
		TLSHandshakeTimeout:   10 * time.Second,
		TLSClientConfig:       &tlsClientConfig,
		ForceAttemptHTTP2:     s.EnableHTTP2,

	globalDomainIPs   set.StringSet // Root domain IP address(s) for a distributed MinIO deployment

	globalOperationTimeout       = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops
	globalDeleteOperationTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute)  // default time for delete ops

	globalBucketObjectLockSys *BucketObjectLockSys
	globalBucketQuotaSys      *BucketQuotaSys

| `minio_cluster_iam_plugin_authn_service_succ_avg_rtt_ms_minute` | `counter` | When plugin authentication is configured, returns average round-trip-time of successful requests in the last full minute |        |
| `minio_cluster_iam_plugin_authn_service_succ_max_rtt_ms_minute` | `counter` | When plugin authentication is configured, returns maximum round-trip-time of successful requests in the last full minute |        |

	writeTotal        MetricName = "write_total"
	total             MetricName = "total"
	freeInodes        MetricName = "free_inodes"

	lastMinFailedCount  MetricName = "last_minute_failed_count"
	lastMinFailedBytes  MetricName = "last_minute_failed_bytes"
	lastHourFailedCount MetricName = "last_hour_failed_count"
	lastHourFailedBytes MetricName = "last_hour_failed_bytes"
	totalFailedCount    MetricName = "total_failed_count"

| `minio_cluster_replication_last_minute_failed_bytes`       | Total number of bytes failed at least once to replicate in the last full minute.                         |
| `minio_cluster_replication_last_minute_failed_count`       | Total number of objects which failed replication in the last full minute.                                |

	})
	t.Run("testServerOutCongestion", func(t *testing.T) {
		defer timeout(1 * time.Minute)()
		testServerOutCongestion(t, local, remote)
		assertNoActive(t, connRemoteLocal)
		assertNoActive(t, connLocalToRemote)
	})
	t.Run("testServerInCongestion", func(t *testing.T) {
		defer timeout(1 * time.Minute)()
		testServerInCongestion(t, local, remote)
		assertNoActive(t, connRemoteLocal)

		Name:   "conn-read-deadline",
		Usage:  "custom connection READ deadline",
		Hidden: true,
		Value:  10 * time.Minute,
		EnvVar: "MINIO_CONN_READ_DEADLINE",
	},
	cli.DurationFlag{
		Name:   "conn-write-deadline",
		Usage:  "custom connection WRITE deadline",
		Hidden: true,
		Value:  10 * time.Minute,
		EnvVar: "MINIO_CONN_WRITE_DEADLINE",
	},
	cli.DurationFlag{
		Name:   "conn-user-timeout",

	"math"
	"net/http"
	"sync"
	"time"

	"github.com/minio/madmin-go/v3"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	resourceMetricsCollectionInterval = time.Minute
	resourceMetricsCacheInterval      = time.Minute

	// drive stats
	totalInodes    MetricName = "total_inodes"
	readsPerSec    MetricName = "reads_per_sec"
	writesPerSec   MetricName = "writes_per_sec"