k, v, ok := lc.GetOldest()
	if k != "" {
		t.Fatalf("should be empty")
	}
	if v != "" {
		t.Fatalf("should be empty")
	}
	if ok {
		t.Fatalf("should be false")
	}

	lc.Add("key1", "val1")

	time.Sleep(100 * time.Millisecond) // not enough to expire
	if lc.Len() != 1 {

		t.Errorf("Failure to decrease timeout appropriately")
	}
}

func TestDynamicTimeoutConcurrent(t *testing.T) {
	// Race test.
	timeout := newDynamicTimeout(time.Second, time.Millisecond)
	var wg sync.WaitGroup
	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
		wg.Add(1)
		rng := rand.New(rand.NewSource(int64(i)))
		go func() {
			defer wg.Done()
			for range 100 {
				for range 100 {

								t.Fatal("failed:", err, ok)
							}
						}
					}
					start := time.Now()
					l.expireOldLocks(time.Hour)
					t.Logf("Scan Took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap))
					if len(l.lockMap) != locks {
						t.Fatalf("objects deleted, want %d != got %d", locks, len(l.lockMap))
					}
					if len(l.lockUID) != locks*readers {

			t.Fatal("failed to acquire lock")
		}

		// lk2
		lk2ch := make(chan struct{})
		go func() {
			defer close(lk2ch)
			nsLk.lock(ctx, "volume", "path", "source", "opsID", false, 1*time.Millisecond)
		}()
		time.Sleep(1 * time.Millisecond) // wait for goroutine to advance; ref=2

		// Unlock the 1st lock; ref=1 after this line
		nsLk.unlock("volume", "path", false)

	for index := range buckets {
		g.Go(func() error {
			// Sleep and stagger to avoid blocked CPU and thundering
			// herd upon start up sequence.
			time.Sleep(25*time.Millisecond + time.Duration(rand.Int63n(int64(100*time.Millisecond))))

			_, _ = sys.objAPI.HealBucket(ctx, buckets[index], madmin.HealOpts{Recreate: true})
			meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[index])
			if err != nil {

package cmd

import (
	"testing"
	"time"
)

func TestTierMetrics(t *testing.T) {
	tier := "WARM-1"
	globalTierMetrics.Observe(tier, 200*time.Millisecond)
	expSuccess := 10
	expFailure := 5
	for range expSuccess {
		globalTierMetrics.logSuccess(tier)
	}
	for range expFailure {
		globalTierMetrics.logFailure(tier)
	}
	metrics := globalTierMetrics.Report()

}

func waitForLowIO(maxIO int, maxWait time.Duration, currentIO func() int) {
	// No need to wait run at full speed.
	if maxIO <= 0 {
		return
	}

	const waitTick = 100 * time.Millisecond

	tmpMaxWait := maxWait

	for currentIO() >= maxIO {
		if tmpMaxWait > 0 {
			if tmpMaxWait < waitTick {
				time.Sleep(tmpMaxWait)
				return
			}
			time.Sleep(waitTick)
			tmpMaxWait -= waitTick

func (rl ReplicationLatency) getUploadLatency() (ret map[string]uint64) {
	ret = make(map[string]uint64)
	avg := rl.UploadHistogram.GetAvgData()
	for k, v := range avg {
		// Convert nanoseconds to milliseconds
		ret[sizeTagToString(k)] = uint64(v.avg() / time.Millisecond)
	}
	return
}

// Update replication upload latency with a new value
func (rl *ReplicationLatency) update(size int64, duration time.Duration) {

		if len(leaked) == 0 {
			return
		}
		// wait a test again will deadline.
		if UTCNow().Before(deadline) {
			time.Sleep(leakDetectPauseTimeMs * time.Millisecond)
			continue
		}
		// after the deadline time report all the difference in the latest snapshot compared with the initial one.
		for _, g := range leaked {
			t.Errorf("Leaked goroutine: %v", g)
		}
		return
	}

	retries := 0
	rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))

	const (
		retryDelay    = 50 * time.Millisecond
		retryDelay250 = 250 * time.Millisecond
	)

	for {
		if contextCanceled(ctx) {
			return entries, ctx.Err()
		}

		// If many failures, check the cache state.
		if retries > 10 {