t.Logf("got traces %v from %s", traces, cluster)
									if err != nil {
										return fmt.Errorf("cannot get traces from zipkin: %v", err)
									}
									if !tracing.VerifyOtelEchoTraces(ctx, appNsInst.Name(), cluster.Name(), traces) {
										return errors.New("cannot find expected traces")
									}
									return nil
								}, retry.Delay(3*time.Second), retry.Timeout(80*time.Second))

			// find the root span of candidate trace and do recursive comparison
			if s.ParentSpanID == "" && CompareTrace(t, s, wtr) {
				return true
			}
		}
	}

	return false
}

func VerifyOtelEchoTraces(t framework.TestContext, namespace, clName string, traces []zipkin.Trace) bool {
	t.Helper()
	wtr := WantOtelTraceRoot(namespace, clName)
	for _, trace := range traces {