"expr": "rate(minio_inter_node_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])",
          "interval": "",
          "legendFormat": "Internode Bytes Sent [{{server}}]",
          "refId": "B"
        }
      ],
      "title": "Internode Traffic",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"

```sh
mc admin trace myminio
```

To trace entire HTTP request

```sh
mc admin trace --verbose myminio
```

To trace entire HTTP request and also internode communication

```sh
mc admin trace --all --verbose myminio
```

## Subnet Health

// Make sure to bump the internode version at storage-rest-common.go
type RawFileInfo struct {
	// Content of entire xl.meta (may contain data depending on what was requested by the caller.
	Buf []byte `msg:"b,allownil"`
}

// FileInfo - represents file stat information.
// The above means that any added/deleted fields are incompatible.
// Make sure to bump the internode version at storage-rest-common.go

	bucketAPICollectorPath         collectorPath = "/bucket/api"
	bucketReplicationCollectorPath collectorPath = "/bucket/replication"

	systemNetworkInternodeCollectorPath collectorPath = "/system/network/internode"
	systemDriveCollectorPath            collectorPath = "/system/drive"
	systemMemoryCollectorPath           collectorPath = "/system/memory"
	systemCPUCollectorPath              collectorPath = "/system/cpu"

	deleteMultipartCleanupSleeper = newDynamicSleeper(5, 25*time.Millisecond, false)

	// Is MINIO_SYNC_BOOT set?
	globalEnableSyncBoot bool

	// Contains NIC interface name used for internode communication
	globalInternodeInterface     string
	globalInternodeInterfaceOnce sync.Once

	// Set last client perf extra time (get lock, and validate)
	globalLastClientPerfExtraTime int64

| `minio_inter_node_traffic_dial_avg_time`  | Average time of internodes TCP dial calls.              |
| `minio_inter_node_traffic_dial_errors`    | Total number of internode TCP dial timeouts and errors. |
| `minio_inter_node_traffic_errors_total`   | Total number of failed internode calls.                 |

	}

	var err error
	globalNodeAuthToken, err = authenticateNode(globalActiveCred.AccessKey, globalActiveCred.SecretKey)
	if err != nil {
		logger.Fatal(err, "Unable to generate internode credentials")
	}
}

// Initialize KMS global variable after valiadating and loading the configuration.
// It depends on KMS env variables and global cli flags.
func handleKMSConfig() {

		LookupHost:  globalDNSCache.LookupHost,
		DialTimeout: rest.DefaultTimeout,
		RootCAs:     globalRootCAs,
		TCPOptions:  globalTCPOptions,
	}
}

// NewInternodeHTTPTransport returns a transport for internode MinIO
// connections.
func NewInternodeHTTPTransport(maxIdleConnsPerHost int) func() http.RoundTripper {
	return xhttp.ConnSettings{
		LookupHost:       globalDNSCache.LookupHost,

	// FIXME: network calls made in this manner such as one goroutine per node,
	// can easily eat into the internode bandwidth. This function would be mostly
	// TX saturating, however there are situations where a RX might also saturate.
	// To avoid these problems we must split the work at scale. With 1000 node

		Name:      errorsTotal,
		Help:      "Total number of failed internode calls",
		Type:      counterMetric,
	}
}

func getInternodeTCPDialTimeout() MetricDescription {
	return MetricDescription{
		Namespace: interNodeMetricNamespace,
		Subsystem: trafficSubsystem,
		Name:      "dial_errors",
		Help:      "Total number of internode TCP dial timeouts and errors",
		Type:      counterMetric,
	}
}