client/tso: add a TSO wait failed duration observer (#8763)

ref #8281

Add a TSO wait failed duration observer.

Signed-off-by: JmPotato <[email protected]>

Author: JmPotato

Diff for: client/metrics.go

@@ -139,7 +139,7 @@ func initMetrics(constLabels prometheus.Labels) {
 }
 
 var (
-	cmdDurationWait                     prometheus.Observer
+	cmdDurationTSOWait                  prometheus.Observer
 	cmdDurationTSO                      prometheus.Observer
 	cmdDurationTSOAsyncWait             prometheus.Observer
 	cmdDurationGetRegion                prometheus.Observer
@@ -166,6 +166,7 @@ var (
 	cmdDurationUpdateServiceSafePointV2 prometheus.Observer
 
 	cmdFailDurationGetRegion                  prometheus.Observer
+	cmdFailDurationTSOWait                    prometheus.Observer
 	cmdFailDurationTSO                        prometheus.Observer
 	cmdFailDurationGetAllMembers              prometheus.Observer
 	cmdFailDurationGetPrevRegion              prometheus.Observer
@@ -189,7 +190,7 @@ var (
 
 func initCmdDurations() {
 	// WithLabelValues is a heavy operation, define variable to avoid call it every time.
-	cmdDurationWait = cmdDuration.WithLabelValues("wait")
+	cmdDurationTSOWait = cmdDuration.WithLabelValues("wait")
 	cmdDurationTSO = cmdDuration.WithLabelValues("tso")
 	cmdDurationTSOAsyncWait = cmdDuration.WithLabelValues("tso_async_wait")
 	cmdDurationGetRegion = cmdDuration.WithLabelValues("get_region")
@@ -216,6 +217,7 @@ func initCmdDurations() {
 	cmdDurationUpdateServiceSafePointV2 = cmdDuration.WithLabelValues("update_service_safe_point_v2")
 
 	cmdFailDurationGetRegion = cmdFailedDuration.WithLabelValues("get_region")
+	cmdFailDurationTSOWait = cmdFailedDuration.WithLabelValues("wait")
 	cmdFailDurationTSO = cmdFailedDuration.WithLabelValues("tso")
 	cmdFailDurationGetAllMembers = cmdFailedDuration.WithLabelValues("get_member_info")
 	cmdFailDurationGetPrevRegion = cmdFailedDuration.WithLabelValues("get_prev_region")

Diff for: client/tso_request.go

@@ -74,13 +74,14 @@ func (req *tsoRequest) waitCtx(ctx context.Context) (physical int64, logical int
 		defer req.pool.Put(req)
 		defer trace.StartRegion(req.requestCtx, "pdclient.tsoReqDone").End()
 		err = errors.WithStack(err)
+		now := time.Now()
 		if err != nil {
-			cmdFailDurationTSO.Observe(time.Since(req.start).Seconds())
+			cmdFailDurationTSOWait.Observe(now.Sub(start).Seconds())
+			cmdFailDurationTSO.Observe(now.Sub(req.start).Seconds())
 			return 0, 0, err
 		}
 		physical, logical = req.physical, req.logical
-		now := time.Now()
-		cmdDurationWait.Observe(now.Sub(start).Seconds())
+		cmdDurationTSOWait.Observe(now.Sub(start).Seconds())
 		cmdDurationTSO.Observe(now.Sub(req.start).Seconds())
 		return
 	case <-ctx.Done():