*: integrate circuitbreaker for get region calls to PD

Signed-off-by: artem_danilov <[email protected]>

DEPS.bzl

@@ -7037,13 +7037,13 @@ def go_deps():
         name = "com_github_tikv_client_go_v2",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sha256 = "0b9c8af5f1dda40c386ee4412e5f56580bba391d787f9a91264a122ce1ea50ae",
-        strip_prefix = "github.com/tikv/client-go/[email protected].8-0.20241225040645-f2266d6bf259",
+        sha256 = "7b4e966e06ef0bb2e8677a80e323cc4bf5f838c30ff0024d1180ac8667da74a1",
+        strip_prefix = "github.com/Tema/client-go/[email protected].0-20250108010516-c325d48bed73",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241225040645-f2266d6bf259.zip",
-            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241225040645-f2266d6bf259.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241225040645-f2266d6bf259.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20241225040645-f2266d6bf259.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/Tema/client-go/v2/com_github_tema_client_go_v2-v2.0.0-20250108010516-c325d48bed73.zip",
+            "http://ats.apps.svc/gomod/github.com/Tema/client-go/v2/com_github_tema_client_go_v2-v2.0.0-20250108010516-c325d48bed73.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/Tema/client-go/v2/com_github_tema_client_go_v2-v2.0.0-20250108010516-c325d48bed73.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Tema/client-go/v2/com_github_tema_client_go_v2-v2.0.0-20250108010516-c325d48bed73.zip",
         ],
     )
     go_repository(

go.mod

@@ -330,3 +330,6 @@ replace (
 	sourcegraph.com/sourcegraph/appdash => github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0
 	sourcegraph.com/sourcegraph/appdash-data => github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
 )
+
+// pending https://github.com/tikv/client-go/pull/1543
+replace github.com/tikv/client-go/v2 => github.com/Tema/client-go/v2 v2.0.0-20250108010516-c325d48bed73

go.sum

@@ -80,6 +80,10 @@ github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+
 github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
 github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
+github.com/Tema/client-go/v2 v2.0.0-20250107045051-7dde728bc5e5 h1:+dH38cGhjLOJfUyrIFXZ7sObUB0JZ243jVoMLlFGR0I=
+github.com/Tema/client-go/v2 v2.0.0-20250107045051-7dde728bc5e5/go.mod h1:YMP2WbENjh0V0PaNY9LPRh7bLto2MmmuRE0zJ7IAZIA=
+github.com/Tema/client-go/v2 v2.0.0-20250108010516-c325d48bed73 h1:2IBYnKqJ5FZwHQUkJCUYTGD9fjP+lTkJz8ewJfSSw4E=
+github.com/Tema/client-go/v2 v2.0.0-20250108010516-c325d48bed73/go.mod h1:YMP2WbENjh0V0PaNY9LPRh7bLto2MmmuRE0zJ7IAZIA=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
@@ -830,8 +834,6 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJf
 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU=
 github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
 github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
-github.com/tikv/client-go/v2 v2.0.8-0.20241225040645-f2266d6bf259 h1:L+IptTYGAYldntJTyU/VIEcomRoeVDTHZnA9HMDrGF4=
-github.com/tikv/client-go/v2 v2.0.8-0.20241225040645-f2266d6bf259/go.mod h1:H+EqIN/3fOXhVgzZT91Bk1XF1kfvlBZCYcpMIC8FNbg=
 github.com/tikv/pd/client v0.0.0-20250107032658-5c4ab57d68de h1:fefo6U56UMca1CfwY/FVhfVcinyDgXgffQQp9lfMeLg=
 github.com/tikv/pd/client v0.0.0-20250107032658-5c4ab57d68de/go.mod h1:6xcCIcECSowarZ9+AqdURngHzS+t5w2x3qhwg1moj4o=
 github.com/timakin/bodyclose v0.0.0-20241017074812-ed6a65f985e3 h1:y4mJRFlM6fUyPhoXuFg/Yu02fg/nIPFMOY8tOqppoFg=

pkg/domain/BUILD.bazel

@@ -110,6 +110,7 @@ go_library(
         "@com_github_tikv_pd_client//:client",
         "@com_github_tikv_pd_client//http",
         "@com_github_tikv_pd_client//opt",
+        "@com_github_tikv_pd_client//pkg/circuitbreaker",
         "@com_github_tikv_pd_client//resource_group/controller",
         "@io_etcd_go_etcd_client_v3//:client",
         "@io_etcd_go_etcd_client_v3//concurrency",

pkg/domain/domain_sysvars.go

@@ -22,8 +22,10 @@ import (
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/meta"
 	"github.com/pingcap/tidb/pkg/sessionctx/variable"
+	"github.com/tikv/client-go/v2/tikv"
 	pd "github.com/tikv/pd/client"
 	"github.com/tikv/pd/client/opt"
+	"github.com/tikv/pd/client/pkg/circuitbreaker"
 )
 
 // initDomainSysVars() is called when a domain is initialized.
@@ -45,6 +47,8 @@ func (do *Domain) initDomainSysVars() {
 	variable.SetLowResolutionTSOUpdateInterval = do.setLowResolutionTSOUpdateInterval
 
 	variable.ChangeSchemaCacheSize = do.changeSchemaCacheSize
+
+	variable.ChangePDMetadataCircuitBreakerErrorRateThresholdPct = do.changePDMetadataCircuitBreakerErrorRateThresholdPct
 }
 
 // setStatsCacheCapacity sets statsCache cap
@@ -150,3 +154,9 @@ func (do *Domain) changeSchemaCacheSize(ctx context.Context, size uint64) error
 	do.infoCache.Data.SetCacheCapacity(size)
 	return nil
 }
+
+func (do *Domain) changePDMetadataCircuitBreakerErrorRateThresholdPct(errorRatePct uint32) {
+	tikv.ChangePdRegionMetaCircuitBreakerSettings(func(config *circuitbreaker.Settings) {
+		config.ErrorRateThresholdPct = errorRatePct
+	})
+}

pkg/sessionctx/variable/sysvar.go

@@ -3447,6 +3447,14 @@ var defaultSysVars = []*SysVar{
 			return (*SetPDClientDynamicOption.Load())(TiDBTSOClientRPCMode, val)
 		},
 	},
+	{Scope: ScopeGlobal, Name: TiDBCircuitBreakerPDMetadataErrorRateThresholdPct, Value: strconv.Itoa(DefTiDBCircuitBreakerPDMetaErrorRatePct), Type: TypeUnsigned, MinValue: 0, MaxValue: 100,
+		SetGlobal: func(_ context.Context, s *SessionVars, val string) error {
+			if ChangePDMetadataCircuitBreakerErrorRateThresholdPct != nil {
+				ChangePDMetadataCircuitBreakerErrorRateThresholdPct(uint32(tidbOptPositiveInt32(val, DefTiDBCircuitBreakerPDMetaErrorRatePct)))
+			}
+			return nil
+		},
+	},
 }
 
 // GlobalSystemVariableInitialValue gets the default value for a system variable including ones that are dynamically set (e.g. based on the store)

pkg/sessionctx/variable/sysvar_test.go

@@ -1742,6 +1742,30 @@ func TestTiDBSchemaCacheSize(t *testing.T) {
 	require.Error(t, err)
 }
 
+func TestTiDBCircuitBreakerPDMetadataErrorRateThresholdPct(t *testing.T) {
+	sv := GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdPct)
+	vars := NewSessionVars(nil)
+
+	// Too low, will get raised to the min value
+	val, err := sv.Validate(vars, "-1", ScopeGlobal)
+	require.NoError(t, err)
+	require.Equal(t, strconv.FormatInt(GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdPct).MinValue, 10), val)
+	warn := vars.StmtCtx.GetWarnings()[0].Err
+	require.Equal(t, "[variable:1292]Truncated incorrect tidb_cb_pd_metadata_error_rate_threshold_pct value: '-1'", warn.Error())
+
+	// Too high, will get lowered to the max value
+	val, err = sv.Validate(vars, "101", ScopeGlobal)
+	require.NoError(t, err)
+	require.Equal(t, strconv.FormatUint(GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdPct).MaxValue, 10), val)
+	warn = vars.StmtCtx.GetWarnings()[1].Err
+	require.Equal(t, "[variable:1292]Truncated incorrect tidb_cb_pd_metadata_error_rate_threshold_pct value: '101'", warn.Error())
+
+	// valid
+	val, err = sv.Validate(vars, "10", ScopeGlobal)
+	require.NoError(t, err)
+	require.Equal(t, "10", val)
+}
+
 func TestEnableWindowFunction(t *testing.T) {
 	vars := NewSessionVars(nil)
 	require.Equal(t, vars.EnableWindowFunction, DefEnableWindowFunction)

pkg/sessionctx/variable/tidb_vars.go

@@ -1218,6 +1218,9 @@ const (
 	// TiDBTSOClientRPCMode controls how the TSO client performs the TSO RPC requests. It internally controls the
 	// concurrency of the RPC. This variable provides an approach to tune the latency of getting timestamps from PD.
 	TiDBTSOClientRPCMode = "tidb_tso_client_rpc_mode"
+	// This variable is used to set percent of errors to trip the circuit breaker for get region calls to PD
+	// https://github.com/tikv/rfcs/blob/master/text/0115-circuit-breaker.md
+	TiDBCircuitBreakerPDMetadataErrorRateThresholdPct = "tidb_cb_pd_metadata_error_rate_threshold_pct"
 )
 
 // TiDB intentional limits
@@ -1571,6 +1574,7 @@ const (
 	DefOptEnableProjectionPushDown                    = true
 	DefTiDBEnableSharedLockPromotion                  = false
 	DefTiDBTSOClientRPCMode                           = TSOClientRPCModeDefault
+	DefTiDBCircuitBreakerPDMetaErrorRatePct           = 0
 )
 
 // Process global variables.
@@ -1730,6 +1734,8 @@ var (
 	EnableStatsOwner func() error = nil
 	// DisableStatsOwner is the func registered by stats to disable running stats in this instance.
 	DisableStatsOwner func() error = nil
+	// ChangePDMetadataCircuitBreakerErrorRateThresholdPct changes the error rate threshold of the PD metadata circuit breaker.
+	ChangePDMetadataCircuitBreakerErrorRateThresholdPct func(uint32) = nil
 )
 
 // Hooks functions for Cluster Resource Control.