From ff7151279d86e8bdf2d12b520fcea560b2aaebe9 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 8 Aug 2024 14:30:40 +0800 Subject: [PATCH 01/42] checker: use ttl cache (#8505) close tikv/pd#8500 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/grpc_service.go | 2 +- pkg/mock/mockcluster/mockcluster.go | 2 +- pkg/schedule/checker/checker_controller.go | 19 +++++++++---------- pkg/schedule/checker/replica_checker.go | 4 ++-- pkg/schedule/checker/replica_checker_test.go | 20 ++++++++++---------- pkg/schedule/checker/rule_checker.go | 4 ++-- pkg/schedule/checker/rule_checker_test.go | 4 ++-- pkg/schedule/handler/handler.go | 4 ++-- pkg/schedule/scatter/region_scatterer.go | 6 +++--- pkg/schedule/splitter/region_splitter.go | 6 +++--- server/cluster/cluster_worker.go | 2 +- server/cluster/scheduling_controller.go | 4 ++-- 12 files changed, 38 insertions(+), 39 deletions(-) diff --git a/pkg/mcs/scheduling/server/grpc_service.go b/pkg/mcs/scheduling/server/grpc_service.go index d068aa5c058..7eb2554f7f2 100644 --- a/pkg/mcs/scheduling/server/grpc_service.go +++ b/pkg/mcs/scheduling/server/grpc_service.go @@ -324,7 +324,7 @@ func (s *Service) AskBatchSplit(_ context.Context, request *schedulingpb.AskBatc // If region splits during the scheduling process, regions with abnormal // status may be left, and these regions need to be checked with higher // priority. - c.GetCoordinator().GetCheckerController().AddPendingProcessedRegions(recordRegions...) + c.GetCoordinator().GetCheckerController().AddPendingProcessedRegions(false, recordRegions...) return &schedulingpb.AskBatchSplitResponse{ Header: s.header(), diff --git a/pkg/mock/mockcluster/mockcluster.go b/pkg/mock/mockcluster/mockcluster.go index cb1efeb544b..bbd4fbb6811 100644 --- a/pkg/mock/mockcluster/mockcluster.go +++ b/pkg/mock/mockcluster/mockcluster.go @@ -845,7 +845,7 @@ func (mc *Cluster) SetStoreLabel(storeID uint64, labels map[string]string) { } // AddPendingProcessedRegions mock method -func (mc *Cluster) AddPendingProcessedRegions(ids ...uint64) { +func (mc *Cluster) AddPendingProcessedRegions(_ bool, ids ...uint64) { for _, id := range ids { mc.pendingProcessedRegions[id] = struct{}{} } diff --git a/pkg/schedule/checker/checker_controller.go b/pkg/schedule/checker/checker_controller.go index 4704996f7d9..200ab388e30 100644 --- a/pkg/schedule/checker/checker_controller.go +++ b/pkg/schedule/checker/checker_controller.go @@ -68,7 +68,7 @@ type Controller struct { mergeChecker *MergeChecker jointStateChecker *JointStateChecker priorityInspector *PriorityInspector - pendingProcessedRegions cache.Cache + pendingProcessedRegions *cache.TTLUint64 suspectKeyRanges *cache.TTLString // suspect key-range regions that may need fix // duration is the duration of the last patrol round. @@ -88,7 +88,7 @@ type Controller struct { // NewController create a new Controller. func NewController(ctx context.Context, cluster sche.CheckerCluster, conf config.CheckerConfigProvider, ruleManager *placement.RuleManager, labeler *labeler.RegionLabeler, opController *operator.Controller) *Controller { - pendingProcessedRegions := cache.NewDefaultCache(DefaultPendingRegionCacheSize) + pendingProcessedRegions := cache.NewIDTTL(ctx, time.Minute, 3*time.Minute) return &Controller{ ctx: ctx, cluster: cluster, @@ -311,7 +311,7 @@ func (c *Controller) tryAddOperators(region *core.RegionInfo) { c.opController.AddWaitingOperator(ops...) c.RemovePendingProcessedRegion(id) } else { - c.AddPendingProcessedRegions(id) + c.AddPendingProcessedRegions(true, id) } } @@ -327,16 +327,15 @@ func (c *Controller) GetRuleChecker() *RuleChecker { // GetPendingProcessedRegions returns the pending processed regions in the cache. func (c *Controller) GetPendingProcessedRegions() []uint64 { - pendingRegions := make([]uint64, 0) - for _, item := range c.pendingProcessedRegions.Elems() { - pendingRegions = append(pendingRegions, item.Key) - } - return pendingRegions + return c.pendingProcessedRegions.GetAllID() } // AddPendingProcessedRegions adds the pending processed region into the cache. -func (c *Controller) AddPendingProcessedRegions(ids ...uint64) { +func (c *Controller) AddPendingProcessedRegions(needCheckLen bool, ids ...uint64) { for _, id := range ids { + if needCheckLen && c.pendingProcessedRegions.Len() > DefaultPendingRegionCacheSize { + return + } c.pendingProcessedRegions.Put(id, nil) } } @@ -385,7 +384,7 @@ func (c *Controller) CheckSuspectRanges() { if lastRegion.GetEndKey() != nil && bytes.Compare(lastRegion.GetEndKey(), keyRange[1]) < 0 { c.AddSuspectKeyRange(lastRegion.GetEndKey(), keyRange[1]) } - c.AddPendingProcessedRegions(regionIDList...) + c.AddPendingProcessedRegions(false, regionIDList...) } } } diff --git a/pkg/schedule/checker/replica_checker.go b/pkg/schedule/checker/replica_checker.go index 6be5432125b..b0c42e88258 100644 --- a/pkg/schedule/checker/replica_checker.go +++ b/pkg/schedule/checker/replica_checker.go @@ -44,11 +44,11 @@ type ReplicaChecker struct { PauseController cluster sche.CheckerCluster conf config.CheckerConfigProvider - pendingProcessedRegions cache.Cache + pendingProcessedRegions *cache.TTLUint64 } // NewReplicaChecker creates a replica checker. -func NewReplicaChecker(cluster sche.CheckerCluster, conf config.CheckerConfigProvider, pendingProcessedRegions cache.Cache) *ReplicaChecker { +func NewReplicaChecker(cluster sche.CheckerCluster, conf config.CheckerConfigProvider, pendingProcessedRegions *cache.TTLUint64) *ReplicaChecker { return &ReplicaChecker{ cluster: cluster, conf: conf, diff --git a/pkg/schedule/checker/replica_checker_test.go b/pkg/schedule/checker/replica_checker_test.go index a9139ee9804..da04fb6d768 100644 --- a/pkg/schedule/checker/replica_checker_test.go +++ b/pkg/schedule/checker/replica_checker_test.go @@ -51,7 +51,7 @@ func (suite *replicaCheckerTestSuite) SetupTest() { suite.ctx, suite.cancel = context.WithCancel(context.Background()) suite.cluster = mockcluster.NewCluster(suite.ctx, cfg) suite.cluster.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - suite.rc = NewReplicaChecker(suite.cluster, suite.cluster.GetCheckerConfig(), cache.NewDefaultCache(10)) + suite.rc = NewReplicaChecker(suite.cluster, suite.cluster.GetCheckerConfig(), cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) stats := &pdpb.StoreStats{ Capacity: 100, Available: 100, @@ -213,7 +213,7 @@ func (suite *replicaCheckerTestSuite) TestBasic() { tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetMaxSnapshotCount(2) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) // Add stores 1,2,3,4. tc.AddRegionStore(1, 4) @@ -290,7 +290,7 @@ func (suite *replicaCheckerTestSuite) TestLostStore() { tc.AddRegionStore(1, 1) tc.AddRegionStore(2, 1) - rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) // now region peer in store 1,2,3.but we just have store 1,2 // This happens only in recovering the PD tc @@ -309,7 +309,7 @@ func (suite *replicaCheckerTestSuite) TestOffline() { tc.SetMaxReplicas(3) tc.SetLocationLabels([]string{"zone", "rack", "host"}) - rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) tc.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) tc.AddLabelsStore(2, 2, map[string]string{"zone": "z2", "rack": "r1", "host": "h1"}) tc.AddLabelsStore(3, 3, map[string]string{"zone": "z3", "rack": "r1", "host": "h1"}) @@ -361,7 +361,7 @@ func (suite *replicaCheckerTestSuite) TestDistinctScore() { tc.SetMaxReplicas(3) tc.SetLocationLabels([]string{"zone", "rack", "host"}) - rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) tc.AddLabelsStore(1, 9, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) tc.AddLabelsStore(2, 8, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) @@ -441,7 +441,7 @@ func (suite *replicaCheckerTestSuite) TestDistinctScore2() { tc.SetMaxReplicas(5) tc.SetLocationLabels([]string{"zone", "host"}) - rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) tc.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "host": "h1"}) tc.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "host": "h2"}) @@ -470,7 +470,7 @@ func (suite *replicaCheckerTestSuite) TestStorageThreshold() { tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetLocationLabels([]string{"zone"}) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) tc.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) tc.UpdateStorageRatio(1, 0.5, 0.5) @@ -506,7 +506,7 @@ func (suite *replicaCheckerTestSuite) TestOpts() { opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) tc.AddRegionStore(1, 100) tc.AddRegionStore(2, 100) @@ -539,7 +539,7 @@ func (suite *replicaCheckerTestSuite) TestFixDownPeer() { tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetLocationLabels([]string{"zone"}) - rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) tc.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) tc.AddLabelsStore(2, 1, map[string]string{"zone": "z1"}) @@ -571,7 +571,7 @@ func (suite *replicaCheckerTestSuite) TestFixOfflinePeer() { tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetLocationLabels([]string{"zone"}) - rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewDefaultCache(10)) + rc := NewReplicaChecker(tc, tc.GetCheckerConfig(), cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) tc.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) tc.AddLabelsStore(2, 1, map[string]string{"zone": "z1"}) diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index a8acb002951..e29cd2bc05b 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -52,14 +52,14 @@ type RuleChecker struct { PauseController cluster sche.CheckerCluster ruleManager *placement.RuleManager - pendingProcessedRegions cache.Cache + pendingProcessedRegions *cache.TTLUint64 pendingList cache.Cache switchWitnessCache *cache.TTLUint64 record *recorder } // NewRuleChecker creates a checker instance. -func NewRuleChecker(ctx context.Context, cluster sche.CheckerCluster, ruleManager *placement.RuleManager, pendingProcessedRegions cache.Cache) *RuleChecker { +func NewRuleChecker(ctx context.Context, cluster sche.CheckerCluster, ruleManager *placement.RuleManager, pendingProcessedRegions *cache.TTLUint64) *RuleChecker { return &RuleChecker{ cluster: cluster, ruleManager: ruleManager, diff --git a/pkg/schedule/checker/rule_checker_test.go b/pkg/schedule/checker/rule_checker_test.go index f99208a988b..b24a95e2ade 100644 --- a/pkg/schedule/checker/rule_checker_test.go +++ b/pkg/schedule/checker/rule_checker_test.go @@ -62,7 +62,7 @@ func (suite *ruleCheckerTestSuite) SetupTest() { suite.cluster.SetEnableWitness(true) suite.cluster.SetEnableUseJointConsensus(false) suite.ruleManager = suite.cluster.RuleManager - suite.rc = NewRuleChecker(suite.ctx, suite.cluster, suite.ruleManager, cache.NewDefaultCache(10)) + suite.rc = NewRuleChecker(suite.ctx, suite.cluster, suite.ruleManager, cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) } func (suite *ruleCheckerTestSuite) TearDownTest() { @@ -1955,7 +1955,7 @@ func (suite *ruleCheckerTestAdvancedSuite) SetupTest() { suite.cluster.SetEnableWitness(true) suite.cluster.SetEnableUseJointConsensus(true) suite.ruleManager = suite.cluster.RuleManager - suite.rc = NewRuleChecker(suite.ctx, suite.cluster, suite.ruleManager, cache.NewDefaultCache(10)) + suite.rc = NewRuleChecker(suite.ctx, suite.cluster, suite.ruleManager, cache.NewIDTTL(suite.ctx, time.Minute, 3*time.Minute)) } func (suite *ruleCheckerTestAdvancedSuite) TearDownTest() { diff --git a/pkg/schedule/handler/handler.go b/pkg/schedule/handler/handler.go index 9f9de274278..748a17b87ef 100644 --- a/pkg/schedule/handler/handler.go +++ b/pkg/schedule/handler/handler.go @@ -1124,7 +1124,7 @@ func (h *Handler) AccelerateRegionsScheduleInRange(rawStartKey, rawEndKey string for _, region := range regions { regionsIDList = append(regionsIDList, region.GetID()) } - co.GetCheckerController().AddPendingProcessedRegions(regionsIDList...) + co.GetCheckerController().AddPendingProcessedRegions(false, regionsIDList...) } return nil } @@ -1151,7 +1151,7 @@ func (h *Handler) AccelerateRegionsScheduleInRanges(startKeys [][]byte, endKeys for _, region := range regions { regionsIDList = append(regionsIDList, region.GetID()) } - co.GetCheckerController().AddPendingProcessedRegions(regionsIDList...) + co.GetCheckerController().AddPendingProcessedRegions(false, regionsIDList...) } return nil } diff --git a/pkg/schedule/scatter/region_scatterer.go b/pkg/schedule/scatter/region_scatterer.go index 100b9eb764d..efef5439fed 100644 --- a/pkg/schedule/scatter/region_scatterer.go +++ b/pkg/schedule/scatter/region_scatterer.go @@ -125,12 +125,12 @@ type RegionScatterer struct { ordinaryEngine engineContext specialEngines sync.Map opController *operator.Controller - addSuspectRegions func(regionIDs ...uint64) + addSuspectRegions func(bool, ...uint64) } // NewRegionScatterer creates a region scatterer. // RegionScatter is used for the `Lightning`, it will scatter the specified regions before import data. -func NewRegionScatterer(ctx context.Context, cluster sche.SharedCluster, opController *operator.Controller, addSuspectRegions func(regionIDs ...uint64)) *RegionScatterer { +func NewRegionScatterer(ctx context.Context, cluster sche.SharedCluster, opController *operator.Controller, addSuspectRegions func(bool, ...uint64)) *RegionScatterer { return &RegionScatterer{ ctx: ctx, name: regionScatterName, @@ -275,7 +275,7 @@ func (r *RegionScatterer) scatterRegions(regions map[uint64]*core.RegionInfo, fa // in a group level instead of cluster level. func (r *RegionScatterer) Scatter(region *core.RegionInfo, group string, skipStoreLimit bool) (*operator.Operator, error) { if !filter.IsRegionReplicated(r.cluster, region) { - r.addSuspectRegions(region.GetID()) + r.addSuspectRegions(false, region.GetID()) scatterSkipNotReplicatedCounter.Inc() log.Warn("region not replicated during scatter", zap.Uint64("region-id", region.GetID())) return nil, errors.Errorf("region %d is not fully replicated", region.GetID()) diff --git a/pkg/schedule/splitter/region_splitter.go b/pkg/schedule/splitter/region_splitter.go index 124ad935655..37b33dad480 100644 --- a/pkg/schedule/splitter/region_splitter.go +++ b/pkg/schedule/splitter/region_splitter.go @@ -58,11 +58,11 @@ func NewSplitRegionsHandler(cluster sche.ClusterInformer, oc *operator.Controlle type RegionSplitter struct { cluster sche.ClusterInformer handler SplitRegionsHandler - addSuspectRegions func(ids ...uint64) + addSuspectRegions func(bool, ...uint64) } // NewRegionSplitter return a region splitter -func NewRegionSplitter(cluster sche.ClusterInformer, handler SplitRegionsHandler, addSuspectRegions func(ids ...uint64)) *RegionSplitter { +func NewRegionSplitter(cluster sche.ClusterInformer, handler SplitRegionsHandler, addSuspectRegions func(bool, ...uint64)) *RegionSplitter { return &RegionSplitter{ cluster: cluster, handler: handler, @@ -173,7 +173,7 @@ func (r *RegionSplitter) groupKeysByRegion(keys [][]byte) map[uint64]*regionGrou func (r *RegionSplitter) checkRegionValid(region *core.RegionInfo) bool { if !filter.IsRegionReplicated(r.cluster, region) { - r.addSuspectRegions(region.GetID()) + r.addSuspectRegions(false, region.GetID()) return false } if region.GetLeader() == nil { diff --git a/server/cluster/cluster_worker.go b/server/cluster/cluster_worker.go index 2d9bb411995..941282c4aca 100644 --- a/server/cluster/cluster_worker.go +++ b/server/cluster/cluster_worker.go @@ -165,7 +165,7 @@ func (c *RaftCluster) HandleAskBatchSplit(request *pdpb.AskBatchSplitRequest) (* // If region splits during the scheduling process, regions with abnormal // status may be left, and these regions need to be checked with higher // priority. - c.AddPendingProcessedRegions(recordRegions...) + c.AddPendingProcessedRegions(false, recordRegions...) resp := &pdpb.AskBatchSplitResponse{Ids: splitIDs} diff --git a/server/cluster/scheduling_controller.go b/server/cluster/scheduling_controller.go index bd515669670..b4c29ceed46 100644 --- a/server/cluster/scheduling_controller.go +++ b/server/cluster/scheduling_controller.go @@ -404,10 +404,10 @@ func (sc *schedulingController) PauseOrResumeChecker(name string, t int64) error } // AddPendingProcessedRegions adds regions to suspect list. -func (sc *schedulingController) AddPendingProcessedRegions(regionIDs ...uint64) { +func (sc *schedulingController) AddPendingProcessedRegions(needCheckLen bool, regionIDs ...uint64) { sc.mu.RLock() defer sc.mu.RUnlock() - sc.coordinator.GetCheckerController().AddPendingProcessedRegions(regionIDs...) + sc.coordinator.GetCheckerController().AddPendingProcessedRegions(needCheckLen, regionIDs...) } // GetPendingProcessedRegions gets all suspect regions. From 56f082db674404aeb4495f8fa54721cfaf0fe5ea Mon Sep 17 00:00:00 2001 From: ShuNing Date: Thu, 8 Aug 2024 14:41:10 +0800 Subject: [PATCH 02/42] resource_manager: add metrics to show resource group configuration (#8478) close tikv/pd#8444 Signed-off-by: nolouch --- pkg/mcs/resourcemanager/server/manager.go | 5 ++++- pkg/mcs/resourcemanager/server/metrics.go | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go index 8588c22b9ae..58b8b5426a4 100644 --- a/pkg/mcs/resourcemanager/server/manager.go +++ b/pkg/mcs/resourcemanager/server/manager.go @@ -457,6 +457,7 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) { delete(maxPerSecTrackers, r.name) readRequestUnitMaxPerSecCost.DeleteLabelValues(r.name) writeRequestUnitMaxPerSecCost.DeleteLabelValues(r.name) + resourceGroupConfigGauge.DeletePartialMatch(prometheus.Labels{newResourceGroupNameLabel: r.name}) } } case <-availableRUTicker.C: @@ -476,8 +477,10 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) { ru = 0 } availableRUCounter.WithLabelValues(group.Name, group.Name).Set(ru) + resourceGroupConfigGauge.WithLabelValues(group.Name, priorityLabel).Set(float64(group.Priority)) + resourceGroupConfigGauge.WithLabelValues(group.Name, ruPerSecLabel).Set(float64(group.RUSettings.RU.Settings.FillRate)) + resourceGroupConfigGauge.WithLabelValues(group.Name, ruCapacityLabel).Set(float64(group.RUSettings.RU.Settings.BurstLimit)) } - case <-recordMaxTicker.C: // Record the sum of RRU and WRU every second. m.RLock() diff --git a/pkg/mcs/resourcemanager/server/metrics.go b/pkg/mcs/resourcemanager/server/metrics.go index 45c94e5c735..6ed4d8ff10a 100644 --- a/pkg/mcs/resourcemanager/server/metrics.go +++ b/pkg/mcs/resourcemanager/server/metrics.go @@ -29,6 +29,11 @@ const ( tiflashTypeLabel = "ap" defaultTypeLabel = "tp" newResourceGroupNameLabel = "resource_group" + + // Labels for the config. + ruPerSecLabel = "ru_per_sec" + ruCapacityLabel = "ru_capacity" + priorityLabel = "priority" ) var ( @@ -116,6 +121,14 @@ var ( Name: "available_ru", Help: "Counter of the available RU for all resource groups.", }, []string{resourceGroupNameLabel, newResourceGroupNameLabel}) + + resourceGroupConfigGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: serverSubsystem, + Name: "group_config", + Help: "Config of the resource group.", + }, []string{newResourceGroupNameLabel, typeLabel}) ) func init() { @@ -130,4 +143,5 @@ func init() { prometheus.MustRegister(availableRUCounter) prometheus.MustRegister(readRequestUnitMaxPerSecCost) prometheus.MustRegister(writeRequestUnitMaxPerSecCost) + prometheus.MustRegister(resourceGroupConfigGauge) } From 77419242cd4763f43cdb71ae82f0ae59c2eb06f6 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 8 Aug 2024 15:59:10 +0800 Subject: [PATCH 03/42] *: not use a fixed port number (#8506) ref tikv/pd#7969 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tools/pd-ut/alloc/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pd-ut/alloc/server.go b/tools/pd-ut/alloc/server.go index aced73467fb..a5643001142 100644 --- a/tools/pd-ut/alloc/server.go +++ b/tools/pd-ut/alloc/server.go @@ -28,7 +28,7 @@ import ( "go.uber.org/zap" ) -var statusAddress = flag.String("status-addr", "0.0.0.0:20180", "status address") +var statusAddress = flag.String("status-addr", "0.0.0.0:0", "status address") func RunHTTPServer() *http.Server { err := os.Setenv(tempurl.AllocURLFromUT, fmt.Sprintf("http://%s/alloc", *statusAddress)) From 8644ec6e75e218931a74687d104b674b6131f837 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Thu, 8 Aug 2024 19:47:10 +0800 Subject: [PATCH 04/42] *: instead scheduler type string of types.CheckerSchedulerType (#8485) ref tikv/pd#8379 - Remove the old scheduler type, such as BalanceLeaderType - Instead old scheduler type string of types.CheckerSchedulerType - Instead old scheduler name of types.CheckerSchedulerType in SchedulerRegisterMap Signed-off-by: okJiang <819421878@qq.com> --- pkg/mcs/scheduling/server/cluster.go | 5 +- pkg/mock/mockconfig/mockconfig.go | 3 +- pkg/schedule/config/config.go | 9 +- pkg/schedule/config/config_provider.go | 6 +- pkg/schedule/coordinator.go | 34 ++++-- .../schedulers/balance_benchmark_test.go | 14 +-- pkg/schedule/schedulers/balance_leader.go | 4 +- pkg/schedule/schedulers/balance_region.go | 4 +- pkg/schedule/schedulers/balance_test.go | 49 ++++---- pkg/schedule/schedulers/balance_witness.go | 4 +- .../schedulers/balance_witness_test.go | 3 +- pkg/schedule/schedulers/evict_leader.go | 2 - pkg/schedule/schedulers/evict_leader_test.go | 7 +- pkg/schedule/schedulers/evict_slow_store.go | 2 - .../schedulers/evict_slow_store_test.go | 4 +- pkg/schedule/schedulers/evict_slow_trend.go | 2 - .../schedulers/evict_slow_trend_test.go | 4 +- pkg/schedule/schedulers/grant_hot_region.go | 6 +- pkg/schedule/schedulers/grant_leader.go | 4 +- pkg/schedule/schedulers/hot_region.go | 4 +- .../schedulers/hot_region_rank_v2_test.go | 12 +- pkg/schedule/schedulers/hot_region_test.go | 94 +++++++------- pkg/schedule/schedulers/init.go | 86 +++++++------ pkg/schedule/schedulers/label.go | 2 - pkg/schedule/schedulers/random_merge.go | 4 +- pkg/schedule/schedulers/scatter_range.go | 2 - pkg/schedule/schedulers/scheduler.go | 33 +++-- pkg/schedule/schedulers/scheduler_test.go | 31 ++--- pkg/schedule/schedulers/shuffle_hot_region.go | 2 - pkg/schedule/schedulers/shuffle_leader.go | 4 +- pkg/schedule/schedulers/shuffle_region.go | 4 +- pkg/schedule/schedulers/split_bucket.go | 4 +- .../schedulers/transfer_witness_leader.go | 2 - .../transfer_witness_leader_test.go | 5 +- pkg/schedule/type/type.go | 115 +++++++++++------- plugin/scheduler_example/evict_leader.go | 6 +- server/api/scheduler.go | 2 +- server/cluster/cluster_test.go | 35 +++--- server/handler.go | 4 +- tests/server/api/scheduler_test.go | 6 +- 40 files changed, 335 insertions(+), 288 deletions(-) diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 955af4b9b4a..a5a3a709184 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -309,11 +309,12 @@ func (c *Cluster) updateScheduler() { ) // Create the newly added schedulers. for _, scheduler := range latestSchedulersConfig { + schedulerType := types.ConvertOldStrToType[scheduler.Type] s, err := schedulers.CreateScheduler( - scheduler.Type, + schedulerType, c.coordinator.GetOperatorController(), c.storage, - schedulers.ConfigSliceDecoder(scheduler.Type, scheduler.Args), + schedulers.ConfigSliceDecoder(schedulerType, scheduler.Args), schedulersController.RemoveScheduler, ) if err != nil { diff --git a/pkg/mock/mockconfig/mockconfig.go b/pkg/mock/mockconfig/mockconfig.go index 0516f9cd467..ccd1f98154d 100644 --- a/pkg/mock/mockconfig/mockconfig.go +++ b/pkg/mock/mockconfig/mockconfig.go @@ -16,6 +16,7 @@ package mockconfig import ( sc "github.com/tikv/pd/pkg/schedule/config" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/server/config" ) @@ -23,7 +24,7 @@ import ( func NewTestOptions() *config.PersistOptions { // register default schedulers in case config check fail. for _, d := range sc.DefaultSchedulers { - sc.RegisterScheduler(d.Type) + sc.RegisterScheduler(types.ConvertOldStrToType[d.Type]) } c := config.NewConfig() c.Adjust(nil, false) diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index d35f7ac6383..08bcc57ffe2 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core/storelimit" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" @@ -571,10 +572,10 @@ type SchedulerConfig struct { // If these schedulers are not in the persistent configuration, they // will be created automatically when reloading. var DefaultSchedulers = SchedulerConfigs{ - {Type: "balance-region"}, - {Type: "balance-leader"}, - {Type: "hot-region"}, - {Type: "evict-slow-store"}, + {Type: types.SchedulerTypeCompatibleMap[types.BalanceRegionScheduler]}, + {Type: types.SchedulerTypeCompatibleMap[types.BalanceLeaderScheduler]}, + {Type: types.SchedulerTypeCompatibleMap[types.BalanceHotRegionScheduler]}, + {Type: types.SchedulerTypeCompatibleMap[types.EvictSlowStoreScheduler]}, } // IsDefaultScheduler checks whether the scheduler is enabled by default. diff --git a/pkg/schedule/config/config_provider.go b/pkg/schedule/config/config_provider.go index 51ade0edb77..d7bc38a7c03 100644 --- a/pkg/schedule/config/config_provider.go +++ b/pkg/schedule/config/config_provider.go @@ -33,13 +33,13 @@ const RejectLeader = "reject-leader" var schedulerMap sync.Map // RegisterScheduler registers the scheduler type. -func RegisterScheduler(typ string) { +func RegisterScheduler(typ types.CheckerSchedulerType) { schedulerMap.Store(typ, struct{}{}) } // IsSchedulerRegistered checks if the named scheduler type is registered. -func IsSchedulerRegistered(name string) bool { - _, ok := schedulerMap.Load(name) +func IsSchedulerRegistered(typ types.CheckerSchedulerType) bool { + _, ok := schedulerMap.Load(typ) return ok } diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go index cb935801ce2..89c99ac90b8 100644 --- a/pkg/schedule/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -34,6 +34,7 @@ import ( "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/schedule/splitter" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/utils/logutil" @@ -187,7 +188,7 @@ func (c *Coordinator) driveSlowNodeScheduler() { // If the cluster was set up with `raft-kv2` engine, this cluster should // enable `evict-slow-trend` scheduler as default. if c.GetCluster().GetStoreConfig().IsRaftKV2() { - typ := schedulers.EvictSlowTrendType + typ := types.EvictSlowTrendScheduler args := []string{} s, err := schedulers.CreateScheduler(typ, c.opController, c.cluster.GetStorage(), schedulers.ConfigSliceDecoder(typ, args), c.schedulers.RemoveScheduler) @@ -275,7 +276,7 @@ func (c *Coordinator) InitSchedulers(needRun bool) { typ := schedulers.FindSchedulerTypeByName(name) var cfg sc.SchedulerConfig for _, c := range scheduleCfg.Schedulers { - if c.Type == typ { + if c.Type == types.SchedulerTypeCompatibleMap[typ] { cfg = c break } @@ -288,7 +289,8 @@ func (c *Coordinator) InitSchedulers(needRun bool) { log.Info("skip create scheduler with independent configuration", zap.String("scheduler-name", name), zap.String("scheduler-type", cfg.Type), zap.Strings("scheduler-args", cfg.Args)) continue } - s, err := schedulers.CreateScheduler(cfg.Type, c.opController, c.cluster.GetStorage(), schedulers.ConfigJSONDecoder([]byte(data)), c.schedulers.RemoveScheduler) + s, err := schedulers.CreateScheduler(types.ConvertOldStrToType[cfg.Type], c.opController, + c.cluster.GetStorage(), schedulers.ConfigJSONDecoder([]byte(data)), c.schedulers.RemoveScheduler) if err != nil { log.Error("can not create scheduler with independent configuration", zap.String("scheduler-name", name), zap.Strings("scheduler-args", cfg.Args), errs.ZapError(err)) continue @@ -296,12 +298,14 @@ func (c *Coordinator) InitSchedulers(needRun bool) { if needRun { log.Info("create scheduler with independent configuration", zap.String("scheduler-name", s.GetName())) if err = c.schedulers.AddScheduler(s); err != nil { - log.Error("can not add scheduler with independent configuration", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", cfg.Args), errs.ZapError(err)) + log.Error("can not add scheduler with independent configuration", + zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", cfg.Args), errs.ZapError(err)) } } else { log.Info("create scheduler handler with independent configuration", zap.String("scheduler-name", s.GetName())) if err = c.schedulers.AddSchedulerHandler(s); err != nil { - log.Error("can not add scheduler handler with independent configuration", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", cfg.Args), errs.ZapError(err)) + log.Error("can not add scheduler handler with independent configuration", + zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", cfg.Args), errs.ZapError(err)) } } } @@ -316,16 +320,22 @@ func (c *Coordinator) InitSchedulers(needRun bool) { continue } - s, err := schedulers.CreateScheduler(schedulerCfg.Type, c.opController, c.cluster.GetStorage(), schedulers.ConfigSliceDecoder(schedulerCfg.Type, schedulerCfg.Args), c.schedulers.RemoveScheduler) + tp := types.ConvertOldStrToType[schedulerCfg.Type] + s, err := schedulers.CreateScheduler(tp, c.opController, + c.cluster.GetStorage(), schedulers.ConfigSliceDecoder(tp, schedulerCfg.Args), c.schedulers.RemoveScheduler) if err != nil { - log.Error("can not create scheduler", zap.String("scheduler-type", schedulerCfg.Type), zap.Strings("scheduler-args", schedulerCfg.Args), errs.ZapError(err)) + log.Error("can not create scheduler", zap.Stringer("type", tp), zap.String("scheduler-type", schedulerCfg.Type), + zap.Strings("scheduler-args", schedulerCfg.Args), errs.ZapError(err)) continue } if needRun { - log.Info("create scheduler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", schedulerCfg.Args)) - if err = c.schedulers.AddScheduler(s, schedulerCfg.Args...); err != nil && !errors.ErrorEqual(err, errs.ErrSchedulerExisted.FastGenByArgs()) { - log.Error("can not add scheduler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", schedulerCfg.Args), errs.ZapError(err)) + log.Info("create scheduler", zap.String("scheduler-name", s.GetName()), + zap.Strings("scheduler-args", schedulerCfg.Args)) + if err = c.schedulers.AddScheduler(s, schedulerCfg.Args...); err != nil && + !errors.ErrorEqual(err, errs.ErrSchedulerExisted.FastGenByArgs()) { + log.Error("can not add scheduler", zap.String("scheduler-name", + s.GetName()), zap.Strings("scheduler-args", schedulerCfg.Args), errs.ZapError(err)) } else { // Only records the valid scheduler config. scheduleCfg.Schedulers[k] = schedulerCfg @@ -362,7 +372,7 @@ func (c *Coordinator) LoadPlugin(pluginPath string, ch chan string) { log.Error("GetFunction SchedulerType error", errs.ZapError(err)) return } - schedulerType := SchedulerType.(func() string) + schedulerType := SchedulerType.(func() types.CheckerSchedulerType) // get func: SchedulerArgs from plugin SchedulerArgs, err := c.pluginInterface.GetFunction(pluginPath, "SchedulerArgs") if err != nil { @@ -373,7 +383,7 @@ func (c *Coordinator) LoadPlugin(pluginPath string, ch chan string) { // create and add user scheduler s, err := schedulers.CreateScheduler(schedulerType(), c.opController, c.cluster.GetStorage(), schedulers.ConfigSliceDecoder(schedulerType(), schedulerArgs()), c.schedulers.RemoveScheduler) if err != nil { - log.Error("can not create scheduler", zap.String("scheduler-type", schedulerType()), errs.ZapError(err)) + log.Error("can not create scheduler", zap.Stringer("scheduler-type", schedulerType()), errs.ZapError(err)) return } log.Info("create scheduler", zap.String("scheduler-name", s.GetName())) diff --git a/pkg/schedule/schedulers/balance_benchmark_test.go b/pkg/schedule/schedulers/balance_benchmark_test.go index 2d7befd27af..4fb6a4fb781 100644 --- a/pkg/schedule/schedulers/balance_benchmark_test.go +++ b/pkg/schedule/schedulers/balance_benchmark_test.go @@ -155,7 +155,7 @@ func BenchmarkPlacementRule(b *testing.B) { re := assert.New(b) cancel, tc, oc := newBenchCluster(true, true, false) defer cancel() - sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}, []BalanceRegionCreateOption{WithBalanceRegionName(BalanceRegionType)}...) + sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}) b.ResetTimer() var ops []*operator.Operator var plans []plan.Plan @@ -171,7 +171,7 @@ func BenchmarkPlacementRule(b *testing.B) { func BenchmarkLabel(b *testing.B) { cancel, tc, oc := newBenchCluster(false, true, false) defer cancel() - sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}, []BalanceRegionCreateOption{WithBalanceRegionName(BalanceRegionType)}...) + sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}) b.ResetTimer() for i := 0; i < b.N; i++ { sc.Schedule(tc, false) @@ -181,7 +181,7 @@ func BenchmarkLabel(b *testing.B) { func BenchmarkNoLabel(b *testing.B) { cancel, tc, oc := newBenchCluster(false, false, false) defer cancel() - sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}, []BalanceRegionCreateOption{WithBalanceRegionName(BalanceRegionType)}...) + sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}) b.ResetTimer() for i := 0; i < b.N; i++ { sc.Schedule(tc, false) @@ -191,7 +191,7 @@ func BenchmarkNoLabel(b *testing.B) { func BenchmarkDiagnosticNoLabel1(b *testing.B) { cancel, tc, oc := newBenchCluster(false, false, false) defer cancel() - sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}, []BalanceRegionCreateOption{WithBalanceRegionName(BalanceRegionType)}...) + sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}) b.ResetTimer() for i := 0; i < b.N; i++ { sc.Schedule(tc, true) @@ -201,7 +201,7 @@ func BenchmarkDiagnosticNoLabel1(b *testing.B) { func BenchmarkDiagnosticNoLabel2(b *testing.B) { cancel, tc, oc := newBenchBigCluster(100, 100) defer cancel() - sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}, []BalanceRegionCreateOption{WithBalanceRegionName(BalanceRegionType)}...) + sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}) b.ResetTimer() for i := 0; i < b.N; i++ { sc.Schedule(tc, true) @@ -211,7 +211,7 @@ func BenchmarkDiagnosticNoLabel2(b *testing.B) { func BenchmarkNoLabel2(b *testing.B) { cancel, tc, oc := newBenchBigCluster(100, 100) defer cancel() - sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}, []BalanceRegionCreateOption{WithBalanceRegionName(BalanceRegionType)}...) + sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}) b.ResetTimer() for i := 0; i < b.N; i++ { sc.Schedule(tc, false) @@ -221,7 +221,7 @@ func BenchmarkNoLabel2(b *testing.B) { func BenchmarkTombStore(b *testing.B) { cancel, tc, oc := newBenchCluster(false, false, true) defer cancel() - sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}, []BalanceRegionCreateOption{WithBalanceRegionName(BalanceRegionType)}...) + sc := newBalanceRegionScheduler(oc, &balanceRegionSchedulerConfig{}) b.ResetTimer() for i := 0; i < b.N; i++ { sc.Schedule(tc, false) diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index f6c8dd5d1b6..05b429c825f 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -43,8 +43,6 @@ import ( const ( // BalanceLeaderName is balance leader scheduler name. BalanceLeaderName = "balance-leader-scheduler" - // BalanceLeaderType is balance leader scheduler type. - BalanceLeaderType = "balance-leader" // BalanceLeaderBatchSize is the default number of operators to transfer leaders by one scheduling. // Default value is 4 which is subjected by scheduler-max-waiting-operator and leader-schedule-limit // If you want to increase balance speed more, please increase above-mentioned param. @@ -536,7 +534,7 @@ func (l *balanceLeaderScheduler) createOperator(solver *solver, collector *plan. } solver.Step++ defer func() { solver.Step-- }() - op, err := operator.CreateTransferLeaderOperator(BalanceLeaderType, solver, solver.Region, solver.targetStoreID(), []uint64{}, operator.OpLeader) + op, err := operator.CreateTransferLeaderOperator(l.GetName(), solver, solver.Region, solver.targetStoreID(), []uint64{}, operator.OpLeader) if err != nil { log.Debug("fail to create balance leader operator", errs.ZapError(err)) if collector != nil { diff --git a/pkg/schedule/schedulers/balance_region.go b/pkg/schedule/schedulers/balance_region.go index 7c19187dd74..174b6af1c83 100644 --- a/pkg/schedule/schedulers/balance_region.go +++ b/pkg/schedule/schedulers/balance_region.go @@ -33,8 +33,6 @@ import ( const ( // BalanceRegionName is balance region scheduler name. BalanceRegionName = "balance-region-scheduler" - // BalanceRegionType is balance region scheduler type. - BalanceRegionType = "balance-region" ) type balanceRegionSchedulerConfig struct { @@ -245,7 +243,7 @@ func (s *balanceRegionScheduler) transferPeer(solver *solver, collector *plan.Co oldPeer := solver.Region.GetStorePeer(sourceID) newPeer := &metapb.Peer{StoreId: solver.Target.GetID(), Role: oldPeer.Role} solver.Step++ - op, err := operator.CreateMovePeerOperator(BalanceRegionType, solver, solver.Region, operator.OpRegion, oldPeer.GetStoreId(), newPeer) + op, err := operator.CreateMovePeerOperator(s.GetName(), solver, solver.Region, operator.OpRegion, oldPeer.GetStoreId(), newPeer) if err != nil { balanceRegionCreateOpFailCounter.Inc() if collector != nil { diff --git a/pkg/schedule/schedulers/balance_test.go b/pkg/schedule/schedulers/balance_test.go index 0cfaf510f1b..2f163742bbb 100644 --- a/pkg/schedule/schedulers/balance_test.go +++ b/pkg/schedule/schedulers/balance_test.go @@ -31,6 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" "github.com/tikv/pd/pkg/versioninfo" @@ -239,7 +240,7 @@ func TestBalanceLeaderSchedulerTestSuite(t *testing.T) { func (suite *balanceLeaderSchedulerTestSuite) SetupTest() { re := suite.Require() suite.cancel, suite.conf, suite.tc, suite.oc = prepareSchedulersTest() - lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) + lb, err := CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", ""})) re.NoError(err) suite.lb = lb } @@ -583,34 +584,34 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestSingleRangeBalance() { suite.tc.UpdateStoreLeaderWeight(3, 1) suite.tc.UpdateStoreLeaderWeight(4, 2) suite.tc.AddLeaderRegionWithRange(1, "a", "g", 1, 2, 3, 4) - lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) + lb, err := CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", ""})) re.NoError(err) ops, _ := lb.Schedule(suite.tc, false) re.NotEmpty(ops) re.Len(ops, 1) re.Len(ops[0].Counters, 1) re.Len(ops[0].FinishedCounters, 1) - lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"h", "n"})) + lb, err = CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"h", "n"})) re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) re.Empty(ops) - lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"b", "f"})) + lb, err = CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"b", "f"})) re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) re.Empty(ops) - lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", "a"})) + lb, err = CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", "a"})) re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) re.Empty(ops) - lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"g", ""})) + lb, err = CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"g", ""})) re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) re.Empty(ops) - lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", "f"})) + lb, err = CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", "f"})) re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) re.Empty(ops) - lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"b", ""})) + lb, err = CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"b", ""})) re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) re.Empty(ops) @@ -630,7 +631,7 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestMultiRangeBalance() { suite.tc.UpdateStoreLeaderWeight(3, 1) suite.tc.UpdateStoreLeaderWeight(4, 2) suite.tc.AddLeaderRegionWithRange(1, "a", "g", 1, 2, 3, 4) - lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", "g", "o", "t"})) + lb, err := CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", "g", "o", "t"})) re.NoError(err) ops, _ := lb.Schedule(suite.tc, false) re.Equal(uint64(1), ops[0].RegionID()) @@ -669,7 +670,7 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestBatchBalance() { suite.tc.AddLeaderRegionWithRange(uint64(102), "102a", "102z", 1, 2, 3) suite.tc.AddLeaderRegionWithRange(uint64(103), "103a", "103z", 4, 5, 6) - lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) + lb, err := CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", ""})) re.NoError(err) ops, _ := lb.Schedule(suite.tc, false) re.Len(ops, 2) @@ -761,7 +762,7 @@ func checkBalanceRegionSchedule1(re *require.Assertions, enablePlacementRules bo tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 1) - sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) // Add stores 1,2,3,4. tc.AddRegionStore(1, 6) @@ -816,7 +817,7 @@ func checkReplica3(re *require.Assertions, enablePlacementRules bool) { tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 3) - sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) // Store 1 has the largest region score, so the balance scheduler tries to replace peer in store 1. tc.AddLabelsStore(1, 16, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) @@ -890,7 +891,7 @@ func checkReplica5(re *require.Assertions, enablePlacementRules bool) { tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 5) - sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) tc.AddLabelsStore(1, 4, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) tc.AddLabelsStore(2, 5, map[string]string{"zone": "z2", "rack": "r1", "host": "h1"}) @@ -991,7 +992,7 @@ func checkBalanceRegionSchedule2(re *require.Assertions, enablePlacementRules bo core.SetApproximateKeys(200), ) - sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) tc.AddRegionStore(1, 11) @@ -1047,7 +1048,7 @@ func checkBalanceRegionStoreWeight(re *require.Assertions, enablePlacementRules tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 1) - sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) tc.AddRegionStore(1, 10) @@ -1082,7 +1083,7 @@ func checkBalanceRegionOpInfluence(re *require.Assertions, enablePlacementRules tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 1) - sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) // Add stores 1,2,3,4. tc.AddRegionStoreWithLeader(1, 2) @@ -1118,7 +1119,7 @@ func checkReplacePendingRegion(re *require.Assertions, enablePlacementRules bool tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) tc.SetEnablePlacementRules(enablePlacementRules) tc.SetMaxReplicasWithLabel(enablePlacementRules, 3) - sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) // Store 1 has the largest region score, so the balance scheduler try to replace peer in store 1. tc.AddLabelsStore(1, 16, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) @@ -1148,7 +1149,7 @@ func TestBalanceRegionShouldNotBalance(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) region := tc.MockRegionInfo(1, 0, []uint64{2, 3, 4}, nil, nil) tc.PutRegion(region) @@ -1161,7 +1162,7 @@ func TestBalanceRegionEmptyRegion(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - sb, err := CreateScheduler(BalanceRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceRegionType, []string{"", ""})) + sb, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) tc.AddRegionStore(1, 10) tc.AddRegionStore(2, 9) @@ -1207,7 +1208,7 @@ func checkRandomMergeSchedule(re *require.Assertions, enablePlacementRules bool) tc.SetMaxReplicasWithLabel(enablePlacementRules, 3) tc.SetMergeScheduleLimit(1) - mb, err := CreateScheduler(RandomMergeType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(RandomMergeType, []string{"", ""})) + mb, err := CreateScheduler(types.RandomMergeScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.RandomMergeScheduler, []string{"", ""})) re.NoError(err) tc.AddRegionStore(1, 4) @@ -1289,7 +1290,7 @@ func checkScatterRangeBalance(re *require.Assertions, enablePlacementRules bool) tc.UpdateStoreStatus(uint64(i)) } - hb, err := CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_50", "t"})) + hb, err := CreateScheduler(types.ScatterRangeScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.ScatterRangeScheduler, []string{"s_00", "s_50", "t"})) re.NoError(err) scheduleAndApplyOperator(tc, hb, 100) @@ -1363,7 +1364,7 @@ func checkBalanceLeaderLimit(re *require.Assertions, enablePlacementRules bool) // test not allow schedule leader tc.SetLeaderScheduleLimit(0) - hb, err := CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_50", "t"})) + hb, err := CreateScheduler(types.ScatterRangeScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.ScatterRangeScheduler, []string{"s_00", "s_50", "t"})) re.NoError(err) scheduleAndApplyOperator(tc, hb, 100) @@ -1387,7 +1388,7 @@ func TestConcurrencyUpdateConfig(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_50", "t"})) + hb, err := CreateScheduler(types.ScatterRangeScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.ScatterRangeScheduler, []string{"s_00", "s_50", "t"})) sche := hb.(*scatterRangeScheduler) re.NoError(err) ch := make(chan struct{}) @@ -1460,7 +1461,7 @@ func TestBalanceWhenRegionNotHeartbeat(t *testing.T) { tc.UpdateStoreStatus(uint64(i)) } - hb, err := CreateScheduler(ScatterRangeType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ScatterRangeType, []string{"s_00", "s_09", "t"})) + hb, err := CreateScheduler(types.ScatterRangeScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.ScatterRangeScheduler, []string{"s_00", "s_09", "t"})) re.NoError(err) scheduleAndApplyOperator(tc, hb, 100) diff --git a/pkg/schedule/schedulers/balance_witness.go b/pkg/schedule/schedulers/balance_witness.go index dbb0d012c72..0415ad03618 100644 --- a/pkg/schedule/schedulers/balance_witness.go +++ b/pkg/schedule/schedulers/balance_witness.go @@ -43,8 +43,6 @@ import ( const ( // BalanceWitnessName is balance witness scheduler name. BalanceWitnessName = "balance-witness-scheduler" - // BalanceWitnessType is balance witness scheduler type. - BalanceWitnessType = "balance-witness" // balanceWitnessBatchSize is the default number of operators to transfer witnesses by one scheduling. // Default value is 4 which is subjected by scheduler-max-waiting-operator and witness-schedule-limit // If you want to increase balance speed more, please increase above-mentioned param. @@ -357,7 +355,7 @@ func (b *balanceWitnessScheduler) createOperator(solver *solver, collector *plan } solver.Step++ defer func() { solver.Step-- }() - op, err := operator.CreateMoveWitnessOperator(BalanceWitnessType, solver, solver.Region, solver.sourceStoreID(), solver.targetStoreID()) + op, err := operator.CreateMoveWitnessOperator(b.GetName(), solver, solver.Region, solver.sourceStoreID(), solver.targetStoreID()) if err != nil { log.Debug("fail to create balance witness operator", errs.ZapError(err)) return nil diff --git a/pkg/schedule/schedulers/balance_witness_test.go b/pkg/schedule/schedulers/balance_witness_test.go index 2b6723a5172..d8715f71784 100644 --- a/pkg/schedule/schedulers/balance_witness_test.go +++ b/pkg/schedule/schedulers/balance_witness_test.go @@ -23,6 +23,7 @@ import ( "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/storage" ) @@ -50,7 +51,7 @@ func (suite *balanceWitnessSchedulerTestSuite) SetupTest() { Count: 4, }, }) - lb, err := CreateScheduler(BalanceWitnessType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceWitnessType, []string{"", ""}), nil) + lb, err := CreateScheduler(types.BalanceWitnessScheduler, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceWitnessScheduler, []string{"", ""}), nil) re.NoError(err) suite.lb = lb } diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index 7e5c4706043..d43f540a489 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -40,8 +40,6 @@ import ( const ( // EvictLeaderName is evict leader scheduler name. EvictLeaderName = "evict-leader-scheduler" - // EvictLeaderType is evict leader scheduler type. - EvictLeaderType = "evict-leader" // EvictLeaderBatchSize is the number of operators to transfer // leaders by one scheduling EvictLeaderBatchSize = 3 diff --git a/pkg/schedule/schedulers/evict_leader_test.go b/pkg/schedule/schedulers/evict_leader_test.go index 63f7cde3b15..eb97be516d7 100644 --- a/pkg/schedule/schedulers/evict_leader_test.go +++ b/pkg/schedule/schedulers/evict_leader_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/operator" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" ) @@ -41,7 +42,7 @@ func TestEvictLeader(t *testing.T) { tc.AddLeaderRegion(2, 2, 1) tc.AddLeaderRegion(3, 3, 1) - sl, err := CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(EvictLeaderType, []string{"1"}), func(string) error { return nil }) + sl, err := CreateScheduler(types.EvictLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.EvictLeaderScheduler, []string{"1"}), func(string) error { return nil }) re.NoError(err) re.True(sl.IsScheduleAllowed(tc)) ops, _ := sl.Schedule(tc, false) @@ -54,7 +55,7 @@ func TestEvictLeaderWithUnhealthyPeer(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sl, err := CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(EvictLeaderType, []string{"1"}), func(string) error { return nil }) + sl, err := CreateScheduler(types.EvictLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.EvictLeaderScheduler, []string{"1"}), func(string) error { return nil }) re.NoError(err) // Add stores 1, 2, 3 @@ -120,7 +121,7 @@ func TestBatchEvict(t *testing.T) { tc.AddLeaderRegion(6, 2, 1, 3) tc.AddLeaderRegion(7, 3, 1, 2) - sl, err := CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(EvictLeaderType, []string{"1"}), func(string) error { return nil }) + sl, err := CreateScheduler(types.EvictLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.EvictLeaderScheduler, []string{"1"}), func(string) error { return nil }) re.NoError(err) re.True(sl.IsScheduleAllowed(tc)) ops, _ := sl.Schedule(tc, false) diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index bc0590531af..d0fb963bd52 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -37,8 +37,6 @@ import ( const ( // EvictSlowStoreName is evict leader scheduler name. EvictSlowStoreName = "evict-slow-store-scheduler" - // EvictSlowStoreType is evict leader scheduler type. - EvictSlowStoreType = "evict-slow-store" slowStoreEvictThreshold = 100 slowStoreRecoverThreshold = 1 diff --git a/pkg/schedule/schedulers/evict_slow_store_test.go b/pkg/schedule/schedulers/evict_slow_store_test.go index 440ab85d08e..ad5b16e8ca3 100644 --- a/pkg/schedule/schedulers/evict_slow_store_test.go +++ b/pkg/schedule/schedulers/evict_slow_store_test.go @@ -58,9 +58,9 @@ func (suite *evictSlowStoreTestSuite) SetupTest() { storage := storage.NewStorageWithMemoryBackend() var err error - suite.es, err = CreateScheduler(EvictSlowStoreType, suite.oc, storage, ConfigSliceDecoder(EvictSlowStoreType, []string{}), nil) + suite.es, err = CreateScheduler(types.EvictSlowStoreScheduler, suite.oc, storage, ConfigSliceDecoder(types.EvictSlowStoreScheduler, []string{}), nil) re.NoError(err) - suite.bs, err = CreateScheduler(BalanceLeaderType, suite.oc, storage, ConfigSliceDecoder(BalanceLeaderType, []string{}), nil) + suite.bs, err = CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage, ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{}), nil) re.NoError(err) } diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index 5fa799c45b5..767a2ee40a0 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -38,8 +38,6 @@ import ( const ( // EvictSlowTrendName is evict leader by slow trend scheduler name. EvictSlowTrendName = "evict-slow-trend-scheduler" - // EvictSlowTrendType is evict leader by slow trend scheduler type. - EvictSlowTrendType = "evict-slow-trend" ) const ( diff --git a/pkg/schedule/schedulers/evict_slow_trend_test.go b/pkg/schedule/schedulers/evict_slow_trend_test.go index c01ae4959ba..10da5c91565 100644 --- a/pkg/schedule/schedulers/evict_slow_trend_test.go +++ b/pkg/schedule/schedulers/evict_slow_trend_test.go @@ -72,9 +72,9 @@ func (suite *evictSlowTrendTestSuite) SetupTest() { storage := storage.NewStorageWithMemoryBackend() var err error - suite.es, err = CreateScheduler(EvictSlowTrendType, suite.oc, storage, ConfigSliceDecoder(EvictSlowTrendType, []string{})) + suite.es, err = CreateScheduler(types.EvictSlowTrendScheduler, suite.oc, storage, ConfigSliceDecoder(types.EvictSlowTrendScheduler, []string{})) re.NoError(err) - suite.bs, err = CreateScheduler(BalanceLeaderType, suite.oc, storage, ConfigSliceDecoder(BalanceLeaderType, []string{})) + suite.bs, err = CreateScheduler(types.BalanceLeaderScheduler, suite.oc, storage, ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{})) re.NoError(err) } diff --git a/pkg/schedule/schedulers/grant_hot_region.go b/pkg/schedule/schedulers/grant_hot_region.go index 1e45096a881..38eadb10c41 100644 --- a/pkg/schedule/schedulers/grant_hot_region.go +++ b/pkg/schedule/schedulers/grant_hot_region.go @@ -44,8 +44,6 @@ import ( const ( // GrantHotRegionName is grant hot region scheduler name. GrantHotRegionName = "grant-hot-region-scheduler" - // GrantHotRegionType is grant hot region scheduler type. - GrantHotRegionType = "grant-hot-region" ) type grantHotRegionSchedulerConfig struct { @@ -337,9 +335,9 @@ func (s *grantHotRegionScheduler) transfer(cluster sche.SchedulerCluster, region dstStore := &metapb.Peer{StoreId: destStoreIDs[i]} if isLeader { - op, err = operator.CreateTransferLeaderOperator(GrantHotRegionType+"-leader", cluster, srcRegion, dstStore.StoreId, []uint64{}, operator.OpLeader) + op, err = operator.CreateTransferLeaderOperator(s.GetName()+"-leader", cluster, srcRegion, dstStore.StoreId, []uint64{}, operator.OpLeader) } else { - op, err = operator.CreateMovePeerOperator(GrantHotRegionType+"-move", cluster, srcRegion, operator.OpRegion|operator.OpLeader, srcStore.GetID(), dstStore) + op, err = operator.CreateMovePeerOperator(s.GetName()+"-move", cluster, srcRegion, operator.OpRegion|operator.OpLeader, srcStore.GetID(), dstStore) } op.SetPriorityLevel(constant.High) return diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index 1cf194c5f49..747a7ee6c0c 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -39,8 +39,6 @@ import ( const ( // GrantLeaderName is grant leader scheduler name. GrantLeaderName = "grant-leader-scheduler" - // GrantLeaderType is grant leader scheduler type. - GrantLeaderType = "grant-leader" ) type grantLeaderSchedulerConfig struct { @@ -239,7 +237,7 @@ func (s *grantLeaderScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ( continue } - op, err := operator.CreateForceTransferLeaderOperator(GrantLeaderType, cluster, region, id, operator.OpLeader) + op, err := operator.CreateForceTransferLeaderOperator(s.GetName(), cluster, region, id, operator.OpLeader) if err != nil { log.Debug("fail to create grant leader operator", errs.ZapError(err)) continue diff --git a/pkg/schedule/schedulers/hot_region.go b/pkg/schedule/schedulers/hot_region.go index ff837e67ad2..717c1413ac4 100644 --- a/pkg/schedule/schedulers/hot_region.go +++ b/pkg/schedule/schedulers/hot_region.go @@ -46,9 +46,7 @@ import ( const ( // HotRegionName is balance hot region scheduler name. - HotRegionName = "balance-hot-region-scheduler" - // HotRegionType is balance hot region scheduler type. - HotRegionType = "hot-region" + HotRegionName = "balance-hot-region-scheduler" splitHotReadBuckets = "split-hot-read-region" splitHotWriteBuckets = "split-hot-write-region" splitProgressiveRank = 5 diff --git a/pkg/schedule/schedulers/hot_region_rank_v2_test.go b/pkg/schedule/schedulers/hot_region_rank_v2_test.go index 029d47c3c51..f00e9dde787 100644 --- a/pkg/schedule/schedulers/hot_region_rank_v2_test.go +++ b/pkg/schedule/schedulers/hot_region_rank_v2_test.go @@ -32,7 +32,7 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sche, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) + sche, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) hb.types = []resourceType{writePeer} @@ -93,7 +93,7 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirst(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sche, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) + sche, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) hb.types = []resourceType{writePeer} @@ -145,7 +145,7 @@ func TestHotWriteRegionScheduleWithRevertRegionsDimFirstOnly(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sche, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) + sche, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) hb.types = []resourceType{writePeer} @@ -206,7 +206,7 @@ func TestHotReadRegionScheduleWithRevertRegionsDimSecond(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sche, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) + sche, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) hb.conf.setDstToleranceRatio(0.0) @@ -265,7 +265,7 @@ func TestSkipUniformStore(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) + hb, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb.(*hotScheduler).conf.setSrcToleranceRatio(1) hb.(*hotScheduler).conf.setDstToleranceRatio(1) @@ -419,7 +419,7 @@ func checkHotReadRegionScheduleWithSmallHotRegion(re *require.Assertions, highLo addOtherRegions func(*mockcluster.Cluster, *hotScheduler)) []*operator.Operator { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sche, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil, nil) + sche, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil, nil) re.NoError(err) hb := sche.(*hotScheduler) hb.conf.setSrcToleranceRatio(1) diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index fc7a6ae4417..7ee0883fb7d 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -29,6 +29,7 @@ import ( "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/statistics/utils" @@ -40,15 +41,20 @@ import ( "github.com/tikv/pd/pkg/versioninfo" ) +var ( + writeType = types.CheckerSchedulerType(utils.Write.String()) + readType = types.CheckerSchedulerType(utils.Read.String()) +) + func init() { // disable denoising in test. statistics.Denoising = false statisticsInterval = 0 - RegisterScheduler(utils.Write.String(), func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(writeType, func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) { cfg := initHotRegionScheduleConfig() return newHotWriteScheduler(opController, cfg), nil }) - RegisterScheduler(utils.Read.String(), func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(readType, func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) { return newHotReadScheduler(opController, initHotRegionScheduleConfig()), nil }) } @@ -81,7 +87,7 @@ func TestUpgrade(t *testing.T) { cancel, _, _, oc := prepareSchedulersTest() defer cancel() // new - sche, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(HotRegionType, nil)) + sche, err := CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceHotRegionScheduler, nil)) re.NoError(err) hb := sche.(*hotScheduler) re.Equal([]string{utils.QueryPriority, utils.BytePriority}, hb.conf.getReadPriorities()) @@ -89,7 +95,7 @@ func TestUpgrade(t *testing.T) { re.Equal([]string{utils.BytePriority, utils.KeyPriority}, hb.conf.getWritePeerPriorities()) re.Equal("v2", hb.conf.getRankFormulaVersion()) // upgrade from json(null) - sche, err = CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) + sche, err = CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) hb = sche.(*hotScheduler) re.Equal([]string{utils.QueryPriority, utils.BytePriority}, hb.conf.getReadPriorities()) @@ -98,7 +104,7 @@ func TestUpgrade(t *testing.T) { re.Equal("v2", hb.conf.getRankFormulaVersion()) // upgrade from < 5.2 config51 := `{"min-hot-byte-rate":100,"min-hot-key-rate":10,"min-hot-query-rate":10,"max-zombie-rounds":5,"max-peer-number":1000,"byte-rate-rank-step-ratio":0.05,"key-rate-rank-step-ratio":0.05,"query-rate-rank-step-ratio":0.05,"count-rank-step-ratio":0.01,"great-dec-ratio":0.95,"minor-dec-ratio":0.99,"src-tolerance-ratio":1.05,"dst-tolerance-ratio":1.05,"strict-picking-store":"true","enable-for-tiflash":"true"}` - sche, err = CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte(config51))) + sche, err = CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte(config51))) re.NoError(err) hb = sche.(*hotScheduler) re.Equal([]string{utils.BytePriority, utils.KeyPriority}, hb.conf.getReadPriorities()) @@ -107,7 +113,7 @@ func TestUpgrade(t *testing.T) { re.Equal("v1", hb.conf.getRankFormulaVersion()) // upgrade from < 6.4 config54 := `{"min-hot-byte-rate":100,"min-hot-key-rate":10,"min-hot-query-rate":10,"max-zombie-rounds":5,"max-peer-number":1000,"byte-rate-rank-step-ratio":0.05,"key-rate-rank-step-ratio":0.05,"query-rate-rank-step-ratio":0.05,"count-rank-step-ratio":0.01,"great-dec-ratio":0.95,"minor-dec-ratio":0.99,"src-tolerance-ratio":1.05,"dst-tolerance-ratio":1.05,"read-priorities":["query","byte"],"write-leader-priorities":["query","byte"],"write-peer-priorities":["byte","key"],"strict-picking-store":"true","enable-for-tiflash":"true","forbid-rw-type":"none"}` - sche, err = CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte(config54))) + sche, err = CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte(config54))) re.NoError(err) hb = sche.(*hotScheduler) re.Equal([]string{utils.QueryPriority, utils.BytePriority}, hb.conf.getReadPriorities()) @@ -133,7 +139,7 @@ func checkGCPendingOpInfos(re *require.Assertions, enablePlacementRules bool) { tc.PutStoreWithLabels(id) } - sche, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) + sche, err := CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) hb := sche.(*hotScheduler) @@ -203,7 +209,7 @@ func TestSplitIfRegionTooHot(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) b := &metapb.Buckets{ RegionId: 1, @@ -258,7 +264,7 @@ func TestSplitIfRegionTooHot(t *testing.T) { addRegionInfo(tc, utils.Write, []testRegionInfo{ {1, []uint64{1, 2, 3}, 4 * units.MiB, 0, 0}, }) - hb, _ = CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, _ = CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) hb.(*hotScheduler).types = []resourceType{writePeer} ops, _ = hb.Schedule(tc, false) re.Len(ops, 1) @@ -277,7 +283,7 @@ func TestSplitBucketsBySize(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() tc.SetRegionBucketEnabled(true) defer cancel() - hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) solve := newBalanceSolver(hb.(*hotScheduler), tc, utils.Read, transferLeader) solve.cur = &solution{} @@ -328,7 +334,7 @@ func TestSplitBucketsByLoad(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() tc.SetRegionBucketEnabled(true) defer cancel() - hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) solve := newBalanceSolver(hb.(*hotScheduler), tc, utils.Read, transferLeader) solve.cur = &solution{} @@ -397,7 +403,7 @@ func checkHotWriteRegionPlacement(re *require.Assertions, enablePlacementRules b tc.SetEnablePlacementRules(enablePlacementRules) labels := []string{"zone", "host"} tc.SetMaxReplicasWithLabel(enablePlacementRules, 3, labels...) - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).types = []resourceType{writePeer} hb.(*hotScheduler).conf.setHistorySampleDuration(0) @@ -454,7 +460,7 @@ func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlace tc.SetMaxReplicasWithLabel(enablePlacementRules, 3, labels...) tc.SetHotRegionCacheHitsThreshold(0) - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.setHistorySampleDuration(0) hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{utils.BytePriority, utils.KeyPriority} @@ -649,7 +655,7 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { }, }, })) - sche, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + sche, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb := sche.(*hotScheduler) hb.conf.setHistorySampleDuration(0) @@ -846,7 +852,7 @@ func TestHotWriteRegionScheduleWithQuery(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.setSrcToleranceRatio(1) hb.(*hotScheduler).conf.setDstToleranceRatio(1) @@ -881,7 +887,7 @@ func TestHotWriteRegionScheduleWithKeyRate(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).types = []resourceType{writePeer} hb.(*hotScheduler).conf.setDstToleranceRatio(1) @@ -939,7 +945,7 @@ func TestHotWriteRegionScheduleUnhealthyStore(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.setDstToleranceRatio(1) hb.(*hotScheduler).conf.setSrcToleranceRatio(1) @@ -984,7 +990,7 @@ func TestHotWriteRegionScheduleCheckHot(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.setDstToleranceRatio(1) hb.(*hotScheduler).conf.setSrcToleranceRatio(1) @@ -1016,7 +1022,7 @@ func TestHotWriteRegionScheduleWithLeader(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) hb.(*hotScheduler).types = []resourceType{writeLeader} hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{utils.KeyPriority, utils.BytePriority} hb.(*hotScheduler).conf.setHistorySampleDuration(0) @@ -1082,7 +1088,7 @@ func checkHotWriteRegionScheduleWithPendingInfluence(re *require.Assertions, dim pendingAmpFactor = 0.0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.RankFormulaVersion = "v1" hb.(*hotScheduler).conf.setHistorySampleDuration(0) @@ -1166,7 +1172,7 @@ func TestHotWriteRegionScheduleWithRuleEnabled(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetEnablePlacementRules(true) - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{utils.KeyPriority, utils.BytePriority} hb.(*hotScheduler).conf.setHistorySampleDuration(0) @@ -1246,7 +1252,7 @@ func TestHotReadRegionScheduleByteRateOnly(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) - scheduler, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + scheduler, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb := scheduler.(*hotScheduler) hb.conf.ReadPriorities = []string{utils.BytePriority, utils.KeyPriority} @@ -1368,7 +1374,7 @@ func TestHotReadRegionScheduleWithQuery(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.setSrcToleranceRatio(1) hb.(*hotScheduler).conf.setDstToleranceRatio(1) @@ -1401,7 +1407,7 @@ func TestHotReadRegionScheduleWithKeyRate(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.RankFormulaVersion = "v1" hb.(*hotScheduler).conf.setSrcToleranceRatio(1) @@ -1461,7 +1467,7 @@ func TestHotReadRegionScheduleWithPendingInfluence(t *testing.T) { func checkHotReadRegionScheduleWithPendingInfluence(re *require.Assertions, dim int) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) // For test hb.(*hotScheduler).conf.RankFormulaVersion = "v1" @@ -1573,7 +1579,7 @@ func TestHotReadWithEvictLeaderScheduler(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).conf.setSrcToleranceRatio(1) hb.(*hotScheduler).conf.setDstToleranceRatio(1) @@ -1877,7 +1883,7 @@ func checkHotCacheCheckRegionFlow(re *require.Assertions, testCase testHotCacheC tc.SetEnablePlacementRules(enablePlacementRules) labels := []string{"zone", "host"} tc.SetMaxReplicasWithLabel(enablePlacementRules, 3, labels...) - sche, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) + sche, err := CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) hb := sche.(*hotScheduler) heartbeat := tc.AddLeaderRegionWithWriteInfo @@ -1985,7 +1991,7 @@ func TestHotCacheSortHotPeer(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sche, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) + sche, err := CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) hb := sche.(*hotScheduler) leaderSolver := newBalanceSolver(hb, tc, utils.Read, transferLeader) @@ -2039,7 +2045,7 @@ func TestInfluenceByRWType(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).types = []resourceType{writePeer} hb.(*hotScheduler).conf.setDstToleranceRatio(1) @@ -2139,7 +2145,7 @@ func checkHotReadPeerSchedule(re *require.Assertions, enablePlacementRules bool) tc.PutStoreWithLabels(id) } - sche, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) + sche, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) hb := sche.(*hotScheduler) hb.conf.ReadPriorities = []string{utils.BytePriority, utils.KeyPriority} @@ -2159,7 +2165,7 @@ func TestHotScheduleWithPriority(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).types = []resourceType{writePeer} hb.(*hotScheduler).conf.setDstToleranceRatio(1.05) @@ -2197,7 +2203,7 @@ func TestHotScheduleWithPriority(t *testing.T) { clearPendingInfluence(hb.(*hotScheduler)) // assert read priority schedule - hb, err = CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err = CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) tc.UpdateStorageReadStats(5, 10*units.MiB*utils.StoreHeartBeatReportInterval, 10*units.MiB*utils.StoreHeartBeatReportInterval) tc.UpdateStorageReadStats(4, 10*units.MiB*utils.StoreHeartBeatReportInterval, 10*units.MiB*utils.StoreHeartBeatReportInterval) @@ -2218,7 +2224,7 @@ func TestHotScheduleWithPriority(t *testing.T) { re.Len(ops, 1) operatorutil.CheckTransferLeader(re, ops[0], operator.OpHotRegion, 1, 3) - hb, err = CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err = CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) hb.(*hotScheduler).types = []resourceType{writePeer} hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{utils.KeyPriority, utils.BytePriority} hb.(*hotScheduler).conf.RankFormulaVersion = "v1" @@ -2261,7 +2267,7 @@ func TestHotScheduleWithStddev(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).types = []resourceType{writePeer} hb.(*hotScheduler).conf.setDstToleranceRatio(1.0) @@ -2320,7 +2326,7 @@ func TestHotWriteLeaderScheduleWithPriority(t *testing.T) { pendingAmpFactor = 0.0 cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) hb.(*hotScheduler).types = []resourceType{writeLeader} hb.(*hotScheduler).conf.setDstToleranceRatio(1) @@ -2356,7 +2362,7 @@ func TestCompatibility(t *testing.T) { re := require.New(t) cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(writeType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) // default checkPriority(re, hb.(*hotScheduler), tc, [3][2]int{ @@ -2424,7 +2430,7 @@ func TestCompatibilityConfig(t *testing.T) { defer cancel() // From new or 3.x cluster, it will use new config - hb, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder("hot-region", nil)) + hb, err := CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceHotRegionScheduler, nil)) re.NoError(err) checkPriority(re, hb.(*hotScheduler), tc, [3][2]int{ {utils.QueryDim, utils.ByteDim}, @@ -2433,8 +2439,8 @@ func TestCompatibilityConfig(t *testing.T) { }) // Config file is not currently supported - hb, err = CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), - ConfigSliceDecoder("hot-region", []string{"read-priorities=byte,query"})) + hb, err = CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), + ConfigSliceDecoder(types.BalanceHotRegionScheduler, []string{"read-priorities=byte,query"})) re.NoError(err) checkPriority(re, hb.(*hotScheduler), tc, [3][2]int{ {utils.QueryDim, utils.ByteDim}, @@ -2461,7 +2467,7 @@ func TestCompatibilityConfig(t *testing.T) { re.NoError(err) err = storage.SaveSchedulerConfig(HotRegionName, data) re.NoError(err) - hb, err = CreateScheduler(HotRegionType, oc, storage, ConfigJSONDecoder(data)) + hb, err = CreateScheduler(types.BalanceHotRegionScheduler, oc, storage, ConfigJSONDecoder(data)) re.NoError(err) checkPriority(re, hb.(*hotScheduler), tc, [3][2]int{ {utils.ByteDim, utils.KeyDim}, @@ -2477,7 +2483,7 @@ func TestCompatibilityConfig(t *testing.T) { re.NoError(err) err = storage.SaveSchedulerConfig(HotRegionName, data) re.NoError(err) - hb, err = CreateScheduler(HotRegionType, oc, storage, ConfigJSONDecoder(data)) + hb, err = CreateScheduler(types.BalanceHotRegionScheduler, oc, storage, ConfigJSONDecoder(data)) re.NoError(err) checkPriority(re, hb.(*hotScheduler), tc, [3][2]int{ {utils.KeyDim, utils.QueryDim}, @@ -2591,7 +2597,7 @@ func TestMaxZombieDuration(t *testing.T) { re := require.New(t) cancel, _, _, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder("hot-region", nil)) + hb, err := CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceHotRegionScheduler, nil)) re.NoError(err) maxZombieDur := hb.(*hotScheduler).conf.getValidConf().MaxZombieRounds testCases := []maxZombieDurTestCase{ @@ -2644,7 +2650,7 @@ func TestExpect(t *testing.T) { re := require.New(t) cancel, _, _, oc := prepareSchedulersTest() defer cancel() - hb, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder("hot-region", nil)) + hb, err := CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceHotRegionScheduler, nil)) re.NoError(err) testCases := []struct { rankVersion string @@ -2952,7 +2958,7 @@ func TestEncodeConfig(t *testing.T) { re := require.New(t) cancel, _, _, oc := prepareSchedulersTest() defer cancel() - sche, err := CreateScheduler(HotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) + sche, err := CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigJSONDecoder([]byte("null"))) re.NoError(err) data, err := sche.EncodeConfig() re.NoError(err) diff --git a/pkg/schedule/schedulers/init.go b/pkg/schedule/schedulers/init.go index 5990aa2cda3..0e1917e8552 100644 --- a/pkg/schedule/schedulers/init.go +++ b/pkg/schedule/schedulers/init.go @@ -22,6 +22,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/schedule/operator" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/storage/endpoint" ) @@ -36,7 +37,7 @@ func Register() { func schedulersRegister() { // balance leader - RegisterSliceDecoderBuilder(BalanceLeaderType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.BalanceLeaderScheduler, func(args []string) ConfigDecoder { return func(v any) error { conf, ok := v.(*balanceLeaderSchedulerConfig) if !ok { @@ -52,7 +53,8 @@ func schedulersRegister() { } }) - RegisterScheduler(BalanceLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.BalanceLeaderScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &balanceLeaderSchedulerConfig{storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -64,7 +66,7 @@ func schedulersRegister() { }) // balance region - RegisterSliceDecoderBuilder(BalanceRegionType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.BalanceRegionScheduler, func(args []string) ConfigDecoder { return func(v any) error { conf, ok := v.(*balanceRegionSchedulerConfig) if !ok { @@ -79,7 +81,8 @@ func schedulersRegister() { } }) - RegisterScheduler(BalanceRegionType, func(opController *operator.Controller, _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.BalanceRegionScheduler, func(opController *operator.Controller, + _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &balanceRegionSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -88,7 +91,7 @@ func schedulersRegister() { }) // balance witness - RegisterSliceDecoderBuilder(BalanceWitnessType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.BalanceWitnessScheduler, func(args []string) ConfigDecoder { return func(v any) error { conf, ok := v.(*balanceWitnessSchedulerConfig) if !ok { @@ -104,7 +107,8 @@ func schedulersRegister() { } }) - RegisterScheduler(BalanceWitnessType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.BalanceWitnessScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &balanceWitnessSchedulerConfig{storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -116,7 +120,7 @@ func schedulersRegister() { }) // evict leader - RegisterSliceDecoderBuilder(EvictLeaderType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.EvictLeaderScheduler, func(args []string) ConfigDecoder { return func(v any) error { if len(args) != 1 { return errs.ErrSchedulerConfig.FastGenByArgs("id") @@ -141,7 +145,8 @@ func schedulersRegister() { } }) - RegisterScheduler(EvictLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.EvictLeaderScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &evictLeaderSchedulerConfig{StoreIDWithRanges: make(map[uint64][]core.KeyRange), storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -152,13 +157,14 @@ func schedulersRegister() { }) // evict slow store - RegisterSliceDecoderBuilder(EvictSlowStoreType, func([]string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.EvictSlowStoreScheduler, func([]string) ConfigDecoder { return func(any) error { return nil } }) - RegisterScheduler(EvictSlowStoreType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.EvictSlowStoreScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := initEvictSlowStoreSchedulerConfig(storage) if err := decoder(conf); err != nil { return nil, err @@ -168,7 +174,7 @@ func schedulersRegister() { }) // grant hot region - RegisterSliceDecoderBuilder(GrantHotRegionType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.GrantHotRegionScheduler, func(args []string) ConfigDecoder { return func(v any) error { if len(args) != 2 { return errs.ErrSchedulerConfig.FastGenByArgs("id") @@ -198,7 +204,8 @@ func schedulersRegister() { } }) - RegisterScheduler(GrantHotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.GrantHotRegionScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &grantHotRegionSchedulerConfig{StoreIDs: make([]uint64, 0), storage: storage} conf.cluster = opController.GetCluster() if err := decoder(conf); err != nil { @@ -208,13 +215,14 @@ func schedulersRegister() { }) // hot region - RegisterSliceDecoderBuilder(HotRegionType, func([]string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.BalanceHotRegionScheduler, func([]string) ConfigDecoder { return func(any) error { return nil } }) - RegisterScheduler(HotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.BalanceHotRegionScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := initHotRegionScheduleConfig() var data map[string]any if err := decoder(&data); err != nil { @@ -235,7 +243,7 @@ func schedulersRegister() { }) // grant leader - RegisterSliceDecoderBuilder(GrantLeaderType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.GrantLeaderScheduler, func(args []string) ConfigDecoder { return func(v any) error { if len(args) != 1 { return errs.ErrSchedulerConfig.FastGenByArgs("id") @@ -259,7 +267,8 @@ func schedulersRegister() { } }) - RegisterScheduler(GrantLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.GrantLeaderScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { conf := &grantLeaderSchedulerConfig{StoreIDWithRanges: make(map[uint64][]core.KeyRange), storage: storage} conf.cluster = opController.GetCluster() conf.removeSchedulerCb = removeSchedulerCb[0] @@ -270,7 +279,7 @@ func schedulersRegister() { }) // label - RegisterSliceDecoderBuilder(LabelType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.LabelScheduler, func(args []string) ConfigDecoder { return func(v any) error { conf, ok := v.(*labelSchedulerConfig) if !ok { @@ -285,7 +294,8 @@ func schedulersRegister() { } }) - RegisterScheduler(LabelType, func(opController *operator.Controller, _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.LabelScheduler, func(opController *operator.Controller, + _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &labelSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -294,7 +304,7 @@ func schedulersRegister() { }) // random merge - RegisterSliceDecoderBuilder(RandomMergeType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.RandomMergeScheduler, func(args []string) ConfigDecoder { return func(v any) error { conf, ok := v.(*randomMergeSchedulerConfig) if !ok { @@ -309,7 +319,8 @@ func schedulersRegister() { } }) - RegisterScheduler(RandomMergeType, func(opController *operator.Controller, _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.RandomMergeScheduler, func(opController *operator.Controller, + _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &randomMergeSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -319,7 +330,7 @@ func schedulersRegister() { // scatter range // args: [start-key, end-key, range-name]. - RegisterSliceDecoderBuilder(ScatterRangeType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.ScatterRangeScheduler, func(args []string) ConfigDecoder { return func(v any) error { if len(args) != 3 { return errs.ErrSchedulerConfig.FastGenByArgs("ranges and name") @@ -338,7 +349,8 @@ func schedulersRegister() { } }) - RegisterScheduler(ScatterRangeType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.ScatterRangeScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &scatterRangeSchedulerConfig{ storage: storage, } @@ -353,7 +365,7 @@ func schedulersRegister() { }) // shuffle hot region - RegisterSliceDecoderBuilder(ShuffleHotRegionType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.ShuffleHotRegionScheduler, func(args []string) ConfigDecoder { return func(v any) error { conf, ok := v.(*shuffleHotRegionSchedulerConfig) if !ok { @@ -371,7 +383,8 @@ func schedulersRegister() { } }) - RegisterScheduler(ShuffleHotRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.ShuffleHotRegionScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &shuffleHotRegionSchedulerConfig{Limit: uint64(1)} if err := decoder(conf); err != nil { return nil, err @@ -381,7 +394,7 @@ func schedulersRegister() { }) // shuffle leader - RegisterSliceDecoderBuilder(ShuffleLeaderType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.ShuffleLeaderScheduler, func(args []string) ConfigDecoder { return func(v any) error { conf, ok := v.(*shuffleLeaderSchedulerConfig) if !ok { @@ -397,7 +410,8 @@ func schedulersRegister() { } }) - RegisterScheduler(ShuffleLeaderType, func(opController *operator.Controller, _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.ShuffleLeaderScheduler, func(opController *operator.Controller, + _ endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &shuffleLeaderSchedulerConfig{} if err := decoder(conf); err != nil { return nil, err @@ -406,7 +420,7 @@ func schedulersRegister() { }) // shuffle region - RegisterSliceDecoderBuilder(ShuffleRegionType, func(args []string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.ShuffleRegionScheduler, func(args []string) ConfigDecoder { return func(v any) error { conf, ok := v.(*shuffleRegionSchedulerConfig) if !ok { @@ -422,7 +436,8 @@ func schedulersRegister() { } }) - RegisterScheduler(ShuffleRegionType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.ShuffleRegionScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &shuffleRegionSchedulerConfig{storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -431,13 +446,14 @@ func schedulersRegister() { }) // split bucket - RegisterSliceDecoderBuilder(SplitBucketType, func([]string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.SplitBucketScheduler, func([]string) ConfigDecoder { return func(any) error { return nil } }) - RegisterScheduler(SplitBucketType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.SplitBucketScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := initSplitBucketConfig() if err := decoder(conf); err != nil { return nil, err @@ -447,24 +463,26 @@ func schedulersRegister() { }) // transfer witness leader - RegisterSliceDecoderBuilder(TransferWitnessLeaderType, func([]string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.TransferWitnessLeaderScheduler, func([]string) ConfigDecoder { return func(any) error { return nil } }) - RegisterScheduler(TransferWitnessLeaderType, func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.TransferWitnessLeaderScheduler, func(opController *operator.Controller, + _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) { return newTransferWitnessLeaderScheduler(opController), nil }) // evict slow store by trend - RegisterSliceDecoderBuilder(EvictSlowTrendType, func([]string) ConfigDecoder { + RegisterSliceDecoderBuilder(types.EvictSlowTrendScheduler, func([]string) ConfigDecoder { return func(any) error { return nil } }) - RegisterScheduler(EvictSlowTrendType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { + RegisterScheduler(types.EvictSlowTrendScheduler, func(opController *operator.Controller, + storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := initEvictSlowTrendSchedulerConfig(storage) if err := decoder(conf); err != nil { return nil, err diff --git a/pkg/schedule/schedulers/label.go b/pkg/schedule/schedulers/label.go index f57d82b3149..8d4f42262ac 100644 --- a/pkg/schedule/schedulers/label.go +++ b/pkg/schedule/schedulers/label.go @@ -31,8 +31,6 @@ import ( const ( // LabelName is label scheduler name. LabelName = "label-scheduler" - // LabelType is label scheduler type. - LabelType = "label" ) type labelSchedulerConfig struct { diff --git a/pkg/schedule/schedulers/random_merge.go b/pkg/schedule/schedulers/random_merge.go index 751ab1eaa9d..676e5407e72 100644 --- a/pkg/schedule/schedulers/random_merge.go +++ b/pkg/schedule/schedulers/random_merge.go @@ -32,8 +32,6 @@ import ( const ( // RandomMergeName is random merge scheduler name. RandomMergeName = "random-merge-scheduler" - // RandomMergeType is random merge scheduler type. - RandomMergeType = "random-merge" ) type randomMergeSchedulerConfig struct { @@ -103,7 +101,7 @@ func (s *randomMergeScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ( return nil, nil } - ops, err := operator.CreateMergeRegionOperator(RandomMergeType, cluster, region, target, operator.OpMerge) + ops, err := operator.CreateMergeRegionOperator(s.GetName(), cluster, region, target, operator.OpMerge) if err != nil { log.Debug("fail to create merge region operator", errs.ZapError(err)) return nil, nil diff --git a/pkg/schedule/schedulers/scatter_range.go b/pkg/schedule/schedulers/scatter_range.go index 76a47dd973b..9c9606b29a9 100644 --- a/pkg/schedule/schedulers/scatter_range.go +++ b/pkg/schedule/schedulers/scatter_range.go @@ -33,8 +33,6 @@ import ( ) const ( - // ScatterRangeType is scatter range scheduler type - ScatterRangeType = "scatter-range" // ScatterRangeName is scatter range scheduler name ScatterRangeName = "scatter-range" ) diff --git a/pkg/schedule/schedulers/scheduler.go b/pkg/schedule/schedulers/scheduler.go index 894544d9617..7fce5d9c46e 100644 --- a/pkg/schedule/schedulers/scheduler.go +++ b/pkg/schedule/schedulers/scheduler.go @@ -99,11 +99,11 @@ func ConfigJSONDecoder(data []byte) ConfigDecoder { } // ConfigSliceDecoder the default decode for the config. -func ConfigSliceDecoder(name string, args []string) ConfigDecoder { - builder, ok := schedulerArgsToDecoder[name] +func ConfigSliceDecoder(typ types.CheckerSchedulerType, args []string) ConfigDecoder { + builder, ok := schedulerArgsToDecoder[typ] if !ok { return func(any) error { - return errors.Errorf("the config decoder do not register for %s", name) + return errors.Errorf("the config decoder do not register for %v", typ) } } return builder(args) @@ -113,31 +113,37 @@ func ConfigSliceDecoder(name string, args []string) ConfigDecoder { type CreateSchedulerFunc func(opController *operator.Controller, storage endpoint.ConfigStorage, dec ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) var ( - schedulerMap = make(map[string]CreateSchedulerFunc) - schedulerArgsToDecoder = make(map[string]ConfigSliceDecoderBuilder) + schedulerMap = make(map[types.CheckerSchedulerType]CreateSchedulerFunc) + schedulerArgsToDecoder = make(map[types.CheckerSchedulerType]ConfigSliceDecoderBuilder) ) // RegisterScheduler binds a scheduler creator. It should be called in init() // func of a package. -func RegisterScheduler(typ string, createFn CreateSchedulerFunc) { +func RegisterScheduler(typ types.CheckerSchedulerType, createFn CreateSchedulerFunc) { if _, ok := schedulerMap[typ]; ok { - log.Fatal("duplicated scheduler", zap.String("type", typ), errs.ZapError(errs.ErrSchedulerDuplicated)) + log.Fatal("duplicated scheduler", zap.Stringer("type", typ), errs.ZapError(errs.ErrSchedulerDuplicated)) } schedulerMap[typ] = createFn } // RegisterSliceDecoderBuilder convert arguments to config. It should be called in init() // func of package. -func RegisterSliceDecoderBuilder(typ string, builder ConfigSliceDecoderBuilder) { +func RegisterSliceDecoderBuilder(typ types.CheckerSchedulerType, builder ConfigSliceDecoderBuilder) { if _, ok := schedulerArgsToDecoder[typ]; ok { - log.Fatal("duplicated scheduler", zap.String("type", typ), errs.ZapError(errs.ErrSchedulerDuplicated)) + log.Fatal("duplicated scheduler", zap.Stringer("type", typ), errs.ZapError(errs.ErrSchedulerDuplicated)) } schedulerArgsToDecoder[typ] = builder config.RegisterScheduler(typ) } // CreateScheduler creates a scheduler with registered creator func. -func CreateScheduler(typ string, oc *operator.Controller, storage endpoint.ConfigStorage, dec ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { +func CreateScheduler( + typ types.CheckerSchedulerType, + oc *operator.Controller, + storage endpoint.ConfigStorage, + dec ConfigDecoder, + removeSchedulerCb ...func(string) error, +) (Scheduler, error) { fn, ok := schedulerMap[typ] if !ok { return nil, errs.ErrSchedulerCreateFuncNotRegistered.FastGenByArgs(typ) @@ -156,10 +162,11 @@ func SaveSchedulerConfig(storage endpoint.ConfigStorage, s Scheduler) error { } // FindSchedulerTypeByName finds the type of the specified name. -func FindSchedulerTypeByName(name string) string { - var typ string +func FindSchedulerTypeByName(name string) types.CheckerSchedulerType { + var typ types.CheckerSchedulerType for registeredType := range schedulerMap { - if strings.Contains(name, registeredType) { + if strings.Contains(name, registeredType.String()) { + // If the name matches multiple types, we should choose the longest one. if len(registeredType) > len(typ) { typ = registeredType } diff --git a/pkg/schedule/schedulers/scheduler_test.go b/pkg/schedule/schedulers/scheduler_test.go index 48040841c76..8dfe9f3616f 100644 --- a/pkg/schedule/schedulers/scheduler_test.go +++ b/pkg/schedule/schedulers/scheduler_test.go @@ -28,6 +28,7 @@ import ( "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" @@ -63,7 +64,7 @@ func TestShuffleLeader(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sl, err := CreateScheduler(ShuffleLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ShuffleLeaderType, []string{"", ""})) + sl, err := CreateScheduler(types.ShuffleLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.ShuffleLeaderScheduler, []string{"", ""})) re.NoError(err) ops, _ := sl.Schedule(tc, false) re.Empty(ops) @@ -101,7 +102,7 @@ func TestRejectLeader(t *testing.T) { tc.AddLeaderRegion(2, 2, 1, 3) // The label scheduler transfers leader out of store1. - sl, err := CreateScheduler(LabelType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(LabelType, []string{"", ""})) + sl, err := CreateScheduler(types.LabelScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.LabelScheduler, []string{"", ""})) re.NoError(err) ops, _ := sl.Schedule(tc, false) operatorutil.CheckTransferLeaderFrom(re, ops[0], operator.OpLeader, 1) @@ -113,13 +114,13 @@ func TestRejectLeader(t *testing.T) { // As store3 is disconnected, store1 rejects leader. Balancer will not create // any operators. - bs, err := CreateScheduler(BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) + bs, err := CreateScheduler(types.BalanceLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", ""})) re.NoError(err) ops, _ = bs.Schedule(tc, false) re.Empty(ops) // Can't evict leader from store2, neither. - el, err := CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(EvictLeaderType, []string{"2"}), func(string) error { return nil }) + el, err := CreateScheduler(types.EvictLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.EvictLeaderScheduler, []string{"2"}), func(string) error { return nil }) re.NoError(err) ops, _ = el.Schedule(tc, false) re.Empty(ops) @@ -145,7 +146,7 @@ func TestRemoveRejectLeader(t *testing.T) { defer cancel() tc.AddRegionStore(1, 0) tc.AddRegionStore(2, 1) - el, err := CreateScheduler(EvictLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(EvictLeaderType, []string{"1"}), func(string) error { return nil }) + el, err := CreateScheduler(types.EvictLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.EvictLeaderScheduler, []string{"1"}), func(string) error { return nil }) re.NoError(err) tc.DeleteStore(tc.GetStore(1)) _, err = el.(*evictLeaderScheduler).conf.removeStoreLocked(1) @@ -165,7 +166,7 @@ func checkBalance(re *require.Assertions, enablePlacementRules bool) { tc.SetEnablePlacementRules(enablePlacementRules) labels := []string{"zone", "host"} tc.SetMaxReplicasWithLabel(enablePlacementRules, 3, labels...) - hb, err := CreateScheduler(ShuffleHotRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder("shuffle-hot-region", []string{"", ""})) + hb, err := CreateScheduler(types.ShuffleHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.ShuffleHotRegionScheduler, []string{"", ""})) re.NoError(err) // Add stores 1, 2, 3, 4, 5, 6 with hot peer counts 3, 2, 2, 2, 0, 0. tc.AddLabelsStore(1, 3, map[string]string{"zone": "z1", "host": "h1"}) @@ -211,7 +212,7 @@ func TestHotRegionScheduleAbnormalReplica(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetHotRegionScheduleLimit(0) - hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil) + hb, err := CreateScheduler(readType, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) tc.AddRegionStore(1, 3) @@ -237,7 +238,7 @@ func TestShuffleRegion(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sl, err := CreateScheduler(ShuffleRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ShuffleRegionType, []string{"", ""})) + sl, err := CreateScheduler(types.ShuffleRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.ShuffleRegionScheduler, []string{"", ""})) re.NoError(err) re.True(sl.IsScheduleAllowed(tc)) ops, _ := sl.Schedule(tc, false) @@ -301,7 +302,7 @@ func TestShuffleRegionRole(t *testing.T) { }, peers[0]) tc.PutRegion(region) - sl, err := CreateScheduler(ShuffleRegionType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(ShuffleRegionType, []string{"", ""})) + sl, err := CreateScheduler(types.ShuffleRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.ShuffleRegionScheduler, []string{"", ""})) re.NoError(err) conf := sl.(*shuffleRegionScheduler).conf @@ -321,8 +322,8 @@ func TestSpecialUseHotRegion(t *testing.T) { defer cancel() storage := storage.NewStorageWithMemoryBackend() - cd := ConfigSliceDecoder(BalanceRegionType, []string{"", ""}) - bs, err := CreateScheduler(BalanceRegionType, oc, storage, cd) + cd := ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""}) + bs, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage, cd) re.NoError(err) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) @@ -359,7 +360,7 @@ func TestSpecialUseHotRegion(t *testing.T) { tc.AddLeaderRegionWithWriteInfo(3, 1, 512*units.KiB*utils.RegionHeartBeatReportInterval, 0, 0, utils.RegionHeartBeatReportInterval, []uint64{2, 3}) tc.AddLeaderRegionWithWriteInfo(4, 2, 512*units.KiB*utils.RegionHeartBeatReportInterval, 0, 0, utils.RegionHeartBeatReportInterval, []uint64{1, 3}) tc.AddLeaderRegionWithWriteInfo(5, 3, 512*units.KiB*utils.RegionHeartBeatReportInterval, 0, 0, utils.RegionHeartBeatReportInterval, []uint64{1, 2}) - hs, err := CreateScheduler(utils.Write.String(), oc, storage, cd) + hs, err := CreateScheduler(writeType, oc, storage, cd) re.NoError(err) for i := 0; i < 100; i++ { ops, _ = hs.Schedule(tc, false) @@ -376,8 +377,8 @@ func TestSpecialUseReserved(t *testing.T) { defer cancel() storage := storage.NewStorageWithMemoryBackend() - cd := ConfigSliceDecoder(BalanceRegionType, []string{"", ""}) - bs, err := CreateScheduler(BalanceRegionType, oc, storage, cd) + cd := ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""}) + bs, err := CreateScheduler(types.BalanceRegionScheduler, oc, storage, cd) re.NoError(err) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) @@ -410,7 +411,7 @@ func TestBalanceLeaderWithConflictRule(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() tc.SetEnablePlacementRules(true) - lb, err := CreateScheduler(BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) + lb, err := CreateScheduler(types.BalanceLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", ""})) re.NoError(err) tc.AddLeaderStore(1, 1) diff --git a/pkg/schedule/schedulers/shuffle_hot_region.go b/pkg/schedule/schedulers/shuffle_hot_region.go index 5bb5d269b63..71bc9107fdb 100644 --- a/pkg/schedule/schedulers/shuffle_hot_region.go +++ b/pkg/schedule/schedulers/shuffle_hot_region.go @@ -38,8 +38,6 @@ import ( const ( // ShuffleHotRegionName is shuffle hot region scheduler name. ShuffleHotRegionName = "shuffle-hot-region-scheduler" - // ShuffleHotRegionType is shuffle hot region scheduler type. - ShuffleHotRegionType = "shuffle-hot-region" ) type shuffleHotRegionSchedulerConfig struct { diff --git a/pkg/schedule/schedulers/shuffle_leader.go b/pkg/schedule/schedulers/shuffle_leader.go index 46e04efb23d..2cd6c231a11 100644 --- a/pkg/schedule/schedulers/shuffle_leader.go +++ b/pkg/schedule/schedulers/shuffle_leader.go @@ -29,8 +29,6 @@ import ( const ( // ShuffleLeaderName is shuffle leader scheduler name. ShuffleLeaderName = "shuffle-leader-scheduler" - // ShuffleLeaderType is shuffle leader scheduler type. - ShuffleLeaderType = "shuffle-leader" ) type shuffleLeaderSchedulerConfig struct { @@ -94,7 +92,7 @@ func (s *shuffleLeaderScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) shuffleLeaderNoFollowerCounter.Inc() return nil, nil } - op, err := operator.CreateTransferLeaderOperator(ShuffleLeaderType, cluster, region, targetStore.GetID(), []uint64{}, operator.OpAdmin) + op, err := operator.CreateTransferLeaderOperator(s.GetName(), cluster, region, targetStore.GetID(), []uint64{}, operator.OpAdmin) if err != nil { log.Debug("fail to create shuffle leader operator", errs.ZapError(err)) return nil, nil diff --git a/pkg/schedule/schedulers/shuffle_region.go b/pkg/schedule/schedulers/shuffle_region.go index ca759042e8f..c179efd32c1 100644 --- a/pkg/schedule/schedulers/shuffle_region.go +++ b/pkg/schedule/schedulers/shuffle_region.go @@ -30,8 +30,6 @@ import ( const ( // ShuffleRegionName is shuffle region scheduler name. ShuffleRegionName = "shuffle-region-scheduler" - // ShuffleRegionType is shuffle region scheduler type. - ShuffleRegionType = "shuffle-region" ) type shuffleRegionScheduler struct { @@ -109,7 +107,7 @@ func (s *shuffleRegionScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) return nil, nil } - op, err := operator.CreateMovePeerOperator(ShuffleRegionType, cluster, region, operator.OpRegion, oldPeer.GetStoreId(), newPeer) + op, err := operator.CreateMovePeerOperator(s.GetName(), cluster, region, operator.OpRegion, oldPeer.GetStoreId(), newPeer) if err != nil { shuffleRegionCreateOperatorFailCounter.Inc() return nil, nil diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index 7f33b996f1c..92edea1c82a 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -39,8 +39,6 @@ import ( const ( // SplitBucketName is the split bucket name. SplitBucketName = "split-bucket-scheduler" - // SplitBucketType is the spilt bucket type. - SplitBucketType = "split-bucket" // defaultHotDegree is the default hot region threshold. defaultHotDegree = 3 defaultSplitLimit = 10 @@ -271,7 +269,7 @@ func (s *splitBucketScheduler) splitBucket(plan *splitBucketPlan) []*operator.Op if bytes.Compare(region.GetEndKey(), splitBucket.EndKey) > 0 { splitKey = append(splitKey, splitBucket.EndKey) } - op, err := operator.CreateSplitRegionOperator(SplitBucketType, region, operator.OpSplit, + op, err := operator.CreateSplitRegionOperator(s.GetName(), region, operator.OpSplit, pdpb.CheckPolicy_USEKEY, splitKey) if err != nil { splitBucketCreateOperatorFailCounter.Inc() diff --git a/pkg/schedule/schedulers/transfer_witness_leader.go b/pkg/schedule/schedulers/transfer_witness_leader.go index c1c59620735..9e7bd9bccc1 100644 --- a/pkg/schedule/schedulers/transfer_witness_leader.go +++ b/pkg/schedule/schedulers/transfer_witness_leader.go @@ -30,8 +30,6 @@ import ( const ( // TransferWitnessLeaderName is transfer witness leader scheduler name. TransferWitnessLeaderName = "transfer-witness-leader-scheduler" - // TransferWitnessLeaderType is transfer witness leader scheduler type. - TransferWitnessLeaderType = "transfer-witness-leader" // TransferWitnessLeaderBatchSize is the number of operators to to transfer // leaders by one scheduling transferWitnessLeaderBatchSize = 3 diff --git a/pkg/schedule/schedulers/transfer_witness_leader_test.go b/pkg/schedule/schedulers/transfer_witness_leader_test.go index 1da968d8dc2..046b7aeb53f 100644 --- a/pkg/schedule/schedulers/transfer_witness_leader_test.go +++ b/pkg/schedule/schedulers/transfer_witness_leader_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/operator" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" ) @@ -38,7 +39,7 @@ func TestTransferWitnessLeader(t *testing.T) { // Add regions 1 with leader in stores 1 tc.AddLeaderRegion(1, 1, 2, 3) - sl, err := CreateScheduler(TransferWitnessLeaderType, oc, storage.NewStorageWithMemoryBackend(), nil) + sl, err := CreateScheduler(types.TransferWitnessLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) RecvRegionInfo(sl) <- tc.GetRegion(1) re.True(sl.IsScheduleAllowed(tc)) @@ -53,7 +54,7 @@ func TestTransferWitnessLeaderWithUnhealthyPeer(t *testing.T) { cancel, _, tc, oc := prepareSchedulersTest() defer cancel() - sl, err := CreateScheduler(TransferWitnessLeaderType, oc, storage.NewStorageWithMemoryBackend(), nil) + sl, err := CreateScheduler(types.TransferWitnessLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), nil) re.NoError(err) // Add stores 1, 2, 3 diff --git a/pkg/schedule/type/type.go b/pkg/schedule/type/type.go index 1f6211a9783..b7e0b26482e 100644 --- a/pkg/schedule/type/type.go +++ b/pkg/schedule/type/type.go @@ -73,48 +73,75 @@ const ( LabelScheduler CheckerSchedulerType = "label-scheduler" ) -// SchedulerTypeCompatibleMap exists for compatibility. -// -// It is used in the `PersistOptions` and `PersistConfig`. These two structs -// are persisted in the storage, so we need to keep the compatibility. -var SchedulerTypeCompatibleMap = map[CheckerSchedulerType]string{ - BalanceLeaderScheduler: "balance-leader", - BalanceRegionScheduler: "balance-region", - BalanceWitnessScheduler: "balance-witness", - EvictLeaderScheduler: "evict-leader", - EvictSlowStoreScheduler: "evict-slow-store", - EvictSlowTrendScheduler: "evict-slow-trend", - GrantLeaderScheduler: "grant-leader", - GrantHotRegionScheduler: "grant-hot-region", - BalanceHotRegionScheduler: "hot-region", - RandomMergeScheduler: "random-merge", - ScatterRangeScheduler: "scatter-range", - ShuffleHotRegionScheduler: "shuffle-hot-region", - ShuffleLeaderScheduler: "shuffle-leader", - ShuffleRegionScheduler: "shuffle-region", - SplitBucketScheduler: "split-bucket", - TransferWitnessLeaderScheduler: "transfer-witness-leader", - LabelScheduler: "label", -} +// TODO: SchedulerTypeCompatibleMap and ConvertOldStrToType should be removed after +// fixing this issue(https://github.com/tikv/pd/issues/8474). +var ( + // SchedulerTypeCompatibleMap exists for compatibility. + // + // It is used for `SchedulerConfig` in the `PersistOptions` and `PersistConfig`. + // These two structs are persisted in the storage, so we need to keep the compatibility. + SchedulerTypeCompatibleMap = map[CheckerSchedulerType]string{ + BalanceLeaderScheduler: "balance-leader", + BalanceRegionScheduler: "balance-region", + BalanceWitnessScheduler: "balance-witness", + EvictLeaderScheduler: "evict-leader", + EvictSlowStoreScheduler: "evict-slow-store", + EvictSlowTrendScheduler: "evict-slow-trend", + GrantLeaderScheduler: "grant-leader", + GrantHotRegionScheduler: "grant-hot-region", + BalanceHotRegionScheduler: "hot-region", + RandomMergeScheduler: "random-merge", + ScatterRangeScheduler: "scatter-range", + ShuffleHotRegionScheduler: "shuffle-hot-region", + ShuffleLeaderScheduler: "shuffle-leader", + ShuffleRegionScheduler: "shuffle-region", + SplitBucketScheduler: "split-bucket", + TransferWitnessLeaderScheduler: "transfer-witness-leader", + LabelScheduler: "label", + } -// SchedulerStr2Type is a map to convert the scheduler string to the CheckerSchedulerType. -var SchedulerStr2Type = map[string]CheckerSchedulerType{ - "balance-leader-scheduler": BalanceLeaderScheduler, - "balance-region-scheduler": BalanceRegionScheduler, - "balance-witness-scheduler": BalanceWitnessScheduler, - "evict-leader-scheduler": EvictLeaderScheduler, - "evict-slow-store-scheduler": EvictSlowStoreScheduler, - "evict-slow-trend-scheduler": EvictSlowTrendScheduler, - "grant-leader-scheduler": GrantLeaderScheduler, - "grant-hot-region-scheduler": GrantHotRegionScheduler, - "balance-hot-region-scheduler": BalanceHotRegionScheduler, - "random-merge-scheduler": RandomMergeScheduler, - // TODO: update to `scatter-range-scheduler` - "scatter-range": ScatterRangeScheduler, - "shuffle-hot-region-scheduler": ShuffleHotRegionScheduler, - "shuffle-leader-scheduler": ShuffleLeaderScheduler, - "shuffle-region-scheduler": ShuffleRegionScheduler, - "split-bucket-scheduler": SplitBucketScheduler, - "transfer-witness-leader-scheduler": TransferWitnessLeaderScheduler, - "label-scheduler": LabelScheduler, -} + // ConvertOldStrToType exists for compatibility. + // + // It is used to convert the old scheduler type to `CheckerSchedulerType`. + ConvertOldStrToType = map[string]CheckerSchedulerType{ + "balance-leader": BalanceLeaderScheduler, + "balance-region": BalanceRegionScheduler, + "balance-witness": BalanceWitnessScheduler, + "evict-leader": EvictLeaderScheduler, + "evict-slow-store": EvictSlowStoreScheduler, + "evict-slow-trend": EvictSlowTrendScheduler, + "grant-leader": GrantLeaderScheduler, + "grant-hot-region": GrantHotRegionScheduler, + "hot-region": BalanceHotRegionScheduler, + "random-merge": RandomMergeScheduler, + "scatter-range": ScatterRangeScheduler, + "shuffle-hot-region": ShuffleHotRegionScheduler, + "shuffle-leader": ShuffleLeaderScheduler, + "shuffle-region": ShuffleRegionScheduler, + "split-bucket": SplitBucketScheduler, + "transfer-witness-leader": TransferWitnessLeaderScheduler, + "label": LabelScheduler, + } + + // StringToSchedulerType is a map to convert the scheduler string to the CheckerSchedulerType. + StringToSchedulerType = map[string]CheckerSchedulerType{ + "balance-leader-scheduler": BalanceLeaderScheduler, + "balance-region-scheduler": BalanceRegionScheduler, + "balance-witness-scheduler": BalanceWitnessScheduler, + "evict-leader-scheduler": EvictLeaderScheduler, + "evict-slow-store-scheduler": EvictSlowStoreScheduler, + "evict-slow-trend-scheduler": EvictSlowTrendScheduler, + "grant-leader-scheduler": GrantLeaderScheduler, + "grant-hot-region-scheduler": GrantHotRegionScheduler, + "balance-hot-region-scheduler": BalanceHotRegionScheduler, + "random-merge-scheduler": RandomMergeScheduler, + // TODO: update to `scatter-range-scheduler` + "scatter-range": ScatterRangeScheduler, + "shuffle-hot-region-scheduler": ShuffleHotRegionScheduler, + "shuffle-leader-scheduler": ShuffleLeaderScheduler, + "shuffle-region-scheduler": ShuffleRegionScheduler, + "split-bucket-scheduler": SplitBucketScheduler, + "transfer-witness-leader-scheduler": TransferWitnessLeaderScheduler, + "label-scheduler": LabelScheduler, + } +) diff --git a/plugin/scheduler_example/evict_leader.go b/plugin/scheduler_example/evict_leader.go index 49156abc40c..c7842debdcb 100644 --- a/plugin/scheduler_example/evict_leader.go +++ b/plugin/scheduler_example/evict_leader.go @@ -48,7 +48,7 @@ const ( ) func init() { - schedulers.RegisterSliceDecoderBuilder(EvictLeaderType, func(args []string) schedulers.ConfigDecoder { + schedulers.RegisterSliceDecoderBuilder(userEvictLeaderScheduler, func(args []string) schedulers.ConfigDecoder { return func(v any) error { if len(args) != 1 { return errors.New("should specify the store-id") @@ -71,7 +71,7 @@ func init() { } }) - schedulers.RegisterScheduler(EvictLeaderType, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder schedulers.ConfigDecoder, _ ...func(string) error) (schedulers.Scheduler, error) { + schedulers.RegisterScheduler(userEvictLeaderScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder schedulers.ConfigDecoder, _ ...func(string) error) (schedulers.Scheduler, error) { conf := &evictLeaderSchedulerConfig{StoreIDWitRanges: make(map[uint64][]core.KeyRange), storage: storage} if err := decoder(conf); err != nil { return nil, err @@ -228,7 +228,7 @@ func (s *evictLeaderScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ( if target == nil { continue } - op, err := operator.CreateTransferLeaderOperator(EvictLeaderType, cluster, region, target.GetID(), []uint64{}, operator.OpLeader) + op, err := operator.CreateTransferLeaderOperator(s.GetName(), cluster, region, target.GetID(), []uint64{}, operator.OpLeader) if err != nil { log.Debug("fail to create evict leader operator", errs.ZapError(err)) continue diff --git a/server/api/scheduler.go b/server/api/scheduler.go index e8b9b54380c..306f67ae058 100644 --- a/server/api/scheduler.go +++ b/server/api/scheduler.go @@ -85,7 +85,7 @@ func (h *schedulerHandler) CreateScheduler(w http.ResponseWriter, r *http.Reques return } - tp, ok := types.SchedulerStr2Type[name] + tp, ok := types.StringToSchedulerType[name] if !ok { h.r.JSON(w, http.StatusBadRequest, "unknown scheduler") return diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index f33354b9668..3f01305b3f1 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -53,6 +53,7 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/schedulers" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" @@ -329,7 +330,7 @@ func TestSetOfflineWithReplica(t *testing.T) { func addEvictLeaderScheduler(cluster *RaftCluster, storeID uint64) (evictScheduler schedulers.Scheduler, err error) { args := []string{fmt.Sprintf("%d", storeID)} - evictScheduler, err = schedulers.CreateScheduler(schedulers.EvictLeaderType, cluster.GetOperatorController(), cluster.storage, schedulers.ConfigSliceDecoder(schedulers.EvictLeaderType, args), cluster.GetCoordinator().GetSchedulersController().RemoveScheduler) + evictScheduler, err = schedulers.CreateScheduler(types.EvictLeaderScheduler, cluster.GetOperatorController(), cluster.storage, schedulers.ConfigSliceDecoder(types.EvictLeaderScheduler, args), cluster.GetCoordinator().GetSchedulersController().RemoveScheduler) if err != nil { return } @@ -3096,7 +3097,7 @@ func TestAddScheduler(t *testing.T) { oc := co.GetOperatorController() // test ConfigJSONDecoder create - bl, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigJSONDecoder([]byte("{}"))) + bl, err := schedulers.CreateScheduler(types.BalanceLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigJSONDecoder([]byte("{}"))) re.NoError(err) conf, err := bl.EncodeConfig() re.NoError(err) @@ -3105,16 +3106,16 @@ func TestAddScheduler(t *testing.T) { re.NoError(err) batch := data["batch"].(float64) re.Equal(4, int(batch)) - gls, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"0"}), controller.RemoveScheduler) + gls, err := schedulers.CreateScheduler(types.GrantLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(types.GrantLeaderScheduler, []string{"0"}), controller.RemoveScheduler) re.NoError(err) re.Error(controller.AddScheduler(gls)) re.Error(controller.RemoveScheduler(gls.GetName())) - gls, err = schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"}), controller.RemoveScheduler) + gls, err = schedulers.CreateScheduler(types.GrantLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(types.GrantLeaderScheduler, []string{"1"}), controller.RemoveScheduler) re.NoError(err) re.NoError(controller.AddScheduler(gls)) - hb, err := schedulers.CreateScheduler(schedulers.HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigJSONDecoder([]byte("{}"))) + hb, err := schedulers.CreateScheduler(types.BalanceHotRegionScheduler, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigJSONDecoder([]byte("{}"))) re.NoError(err) conf, err = hb.EncodeConfig() re.NoError(err) @@ -3157,10 +3158,10 @@ func TestPersistScheduler(t *testing.T) { oc := co.GetOperatorController() storage := tc.RaftCluster.storage - gls1, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"}), controller.RemoveScheduler) + gls1, err := schedulers.CreateScheduler(types.GrantLeaderScheduler, oc, storage, schedulers.ConfigSliceDecoder(types.GrantLeaderScheduler, []string{"1"}), controller.RemoveScheduler) re.NoError(err) re.NoError(controller.AddScheduler(gls1, "1")) - evict, err := schedulers.CreateScheduler(schedulers.EvictLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.EvictLeaderType, []string{"2"}), controller.RemoveScheduler) + evict, err := schedulers.CreateScheduler(types.EvictLeaderScheduler, oc, storage, schedulers.ConfigSliceDecoder(types.EvictLeaderScheduler, []string{"2"}), controller.RemoveScheduler) re.NoError(err) re.NoError(controller.AddScheduler(evict, "2")) re.Len(controller.GetSchedulerNames(), defaultCount+2) @@ -3183,11 +3184,13 @@ func TestPersistScheduler(t *testing.T) { // whether the schedulers added or removed in dynamic way are recorded in opt _, newOpt, err := newTestScheduleConfig() re.NoError(err) - shuffle, err := schedulers.CreateScheduler(schedulers.ShuffleRegionType, oc, storage, schedulers.ConfigJSONDecoder([]byte("null"))) + shuffle, err := schedulers.CreateScheduler(types.ShuffleRegionScheduler, oc, storage, schedulers.ConfigJSONDecoder([]byte("null"))) re.NoError(err) re.NoError(controller.AddScheduler(shuffle)) // suppose we add a new default enable scheduler - sc.DefaultSchedulers = append(sc.DefaultSchedulers, sc.SchedulerConfig{Type: "shuffle-region"}) + sc.DefaultSchedulers = append(sc.DefaultSchedulers, sc.SchedulerConfig{ + Type: types.SchedulerTypeCompatibleMap[types.ShuffleRegionScheduler], + }) defer func() { sc.DefaultSchedulers = sc.DefaultSchedulers[:len(sc.DefaultSchedulers)-1] }() @@ -3219,10 +3222,10 @@ func TestPersistScheduler(t *testing.T) { co.Run() controller = co.GetSchedulersController() re.Len(controller.GetSchedulerNames(), 3) - bls, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) + bls, err := schedulers.CreateScheduler(types.BalanceLeaderScheduler, oc, storage, schedulers.ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", ""})) re.NoError(err) re.NoError(controller.AddScheduler(bls)) - brs, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) + brs, err := schedulers.CreateScheduler(types.BalanceRegionScheduler, oc, storage, schedulers.ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) re.NoError(controller.AddScheduler(brs)) re.Len(controller.GetSchedulerNames(), 5) @@ -3271,7 +3274,7 @@ func TestRemoveScheduler(t *testing.T) { oc := co.GetOperatorController() storage := tc.RaftCluster.storage - gls1, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"}), controller.RemoveScheduler) + gls1, err := schedulers.CreateScheduler(types.GrantLeaderScheduler, oc, storage, schedulers.ConfigSliceDecoder(types.GrantLeaderScheduler, []string{"1"}), controller.RemoveScheduler) re.NoError(err) re.NoError(controller.AddScheduler(gls1, "1")) re.Len(controller.GetSchedulerNames(), defaultCount+1) @@ -3457,7 +3460,7 @@ func TestStoreOverloaded(t *testing.T) { tc, co, cleanup := prepare(nil, nil, nil, re) defer cleanup() oc := co.GetOperatorController() - lb, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, tc.storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) + lb, err := schedulers.CreateScheduler(types.BalanceRegionScheduler, oc, tc.storage, schedulers.ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) opt := tc.GetOpts() re.NoError(tc.addRegionStore(4, 100)) @@ -3511,7 +3514,7 @@ func TestStoreOverloadedWithReplace(t *testing.T) { tc, co, cleanup := prepare(nil, nil, nil, re) defer cleanup() oc := co.GetOperatorController() - lb, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, tc.storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) + lb, err := schedulers.CreateScheduler(types.BalanceRegionScheduler, oc, tc.storage, schedulers.ConfigSliceDecoder(types.BalanceRegionScheduler, []string{"", ""})) re.NoError(err) re.NoError(tc.addRegionStore(4, 100)) @@ -3604,7 +3607,7 @@ func TestController(t *testing.T) { re.NoError(tc.addLeaderRegion(1, 1)) re.NoError(tc.addLeaderRegion(2, 2)) - scheduler, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) + scheduler, err := schedulers.CreateScheduler(types.BalanceLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", ""})) re.NoError(err) lb := &mockLimitScheduler{ Scheduler: scheduler, @@ -3690,7 +3693,7 @@ func TestInterval(t *testing.T) { tc, co, cleanup := prepare(nil, nil, nil, re) defer cleanup() - lb, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, co.GetOperatorController(), storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) + lb, err := schedulers.CreateScheduler(types.BalanceLeaderScheduler, co.GetOperatorController(), storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(types.BalanceLeaderScheduler, []string{"", ""})) re.NoError(err) sc := schedulers.NewScheduleController(tc.ctx, co.GetCluster(), co.GetOperatorController(), lb) diff --git a/server/handler.go b/server/handler.go index 34a78a93c3c..995f98a3324 100644 --- a/server/handler.go +++ b/server/handler.go @@ -188,8 +188,6 @@ func (h *Handler) GetAllRequestHistoryHotRegion(request *HistoryHotRegionsReques // AddScheduler adds a scheduler. func (h *Handler) AddScheduler(tp types.CheckerSchedulerType, args ...string) error { - // TODO: remove this map in subsequent PRs, because we need use new type in the `CreateScheduler`. - name := types.SchedulerTypeCompatibleMap[tp] c, err := h.GetRaftCluster() if err != nil { return err @@ -201,7 +199,7 @@ func (h *Handler) AddScheduler(tp types.CheckerSchedulerType, args ...string) er } else { removeSchedulerCb = c.GetCoordinator().GetSchedulersController().RemoveScheduler } - s, err := schedulers.CreateScheduler(name, c.GetOperatorController(), h.s.storage, schedulers.ConfigSliceDecoder(name, args), removeSchedulerCb) + s, err := schedulers.CreateScheduler(tp, c.GetOperatorController(), h.s.storage, schedulers.ConfigSliceDecoder(tp, args), removeSchedulerCb) if err != nil { return err } diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index 62b7db9d958..f3b8509d2e5 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" sc "github.com/tikv/pd/pkg/schedule/config" + types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" @@ -656,7 +657,10 @@ func (suite *scheduleTestSuite) checkDisable(cluster *tests.TestCluster) { re.NoError(err) originSchedulers := scheduleConfig.Schedulers - scheduleConfig.Schedulers = sc.SchedulerConfigs{sc.SchedulerConfig{Type: "shuffle-leader", Disable: true}} + scheduleConfig.Schedulers = sc.SchedulerConfigs{sc.SchedulerConfig{ + Type: types.SchedulerTypeCompatibleMap[types.ShuffleLeaderScheduler], + Disable: true, + }} body, err = json.Marshal(scheduleConfig) re.NoError(err) err = tu.CheckPostJSON(tests.TestDialClient, u, body, tu.StatusOK(re)) From 8bd348ea1121e44f8493811382afc8088baa6970 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 9 Aug 2024 10:27:40 +0800 Subject: [PATCH 05/42] tso: skip resetting leader when resetting tso allocator (#8495) ref tikv/pd#8477 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/keyspace/tso_keyspace_group.go | 3 +++ pkg/tso/allocator_manager.go | 10 +++++----- pkg/tso/global_allocator.go | 2 +- server/server.go | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go index a04d7392426..63a29d91dc0 100644 --- a/pkg/keyspace/tso_keyspace_group.go +++ b/pkg/keyspace/tso_keyspace_group.go @@ -181,6 +181,9 @@ func (m *GroupManager) allocNodesToAllKeyspaceGroups(ctx context.Context) { log.Info("the raftcluster is closed, stop to alloc nodes to all keyspace groups") return case <-ticker.C: + if m.GetNodesCount() == 0 { + continue + } } groups, err := m.store.LoadKeyspaceGroups(utils.DefaultKeyspaceGroupID, 0) if err != nil { diff --git a/pkg/tso/allocator_manager.go b/pkg/tso/allocator_manager.go index 62a4fb97a57..902a43541d6 100644 --- a/pkg/tso/allocator_manager.go +++ b/pkg/tso/allocator_manager.go @@ -660,7 +660,7 @@ func (am *AllocatorManager) campaignAllocatorLeader( // Start keepalive the Local TSO Allocator leadership and enable Local TSO service. ctx, cancel := context.WithCancel(loopCtx) defer cancel() - defer am.ResetAllocatorGroup(allocator.GetDCLocation()) + defer am.ResetAllocatorGroup(allocator.GetDCLocation(), false) // Maintain the Local TSO Allocator leader go allocator.KeepAllocatorLeader(ctx) @@ -785,7 +785,7 @@ func (am *AllocatorManager) updateAllocator(ag *allocatorGroup) { zap.String("dc-location", ag.dcLocation), zap.String("name", am.member.Name()), errs.ZapError(err)) - am.ResetAllocatorGroup(ag.dcLocation) + am.ResetAllocatorGroup(ag.dcLocation, false) return } } @@ -1038,7 +1038,7 @@ func (am *AllocatorManager) PriorityChecker() { log.Info("next leader key found, resign current leader", logutil.CondUint32("keyspace-group-id", am.kgID, am.kgID > 0), zap.Uint64("nextLeaderID", nextLeader)) - am.ResetAllocatorGroup(allocatorGroup.dcLocation) + am.ResetAllocatorGroup(allocatorGroup.dcLocation, false) } } } @@ -1132,13 +1132,13 @@ func (am *AllocatorManager) HandleRequest(ctx context.Context, dcLocation string // ResetAllocatorGroup will reset the allocator's leadership and TSO initialized in memory. // It usually should be called before re-triggering an Allocator leader campaign. -func (am *AllocatorManager) ResetAllocatorGroup(dcLocation string) { +func (am *AllocatorManager) ResetAllocatorGroup(dcLocation string, skipResetLeader bool) { am.mu.Lock() defer am.mu.Unlock() if allocatorGroup, exist := am.mu.allocatorGroups[dcLocation]; exist { allocatorGroup.allocator.Reset() // Reset if it still has the leadership. Otherwise the data race may occur because of the re-campaigning. - if allocatorGroup.leadership.Check() { + if !skipResetLeader && allocatorGroup.leadership.Check() { allocatorGroup.leadership.Reset() } } diff --git a/pkg/tso/global_allocator.go b/pkg/tso/global_allocator.go index f90dc5f26fe..cf82d58f884 100644 --- a/pkg/tso/global_allocator.go +++ b/pkg/tso/global_allocator.go @@ -617,7 +617,7 @@ func (gta *GlobalTSOAllocator) campaignLeader() { return } defer func() { - gta.am.ResetAllocatorGroup(GlobalDCLocation) + gta.am.ResetAllocatorGroup(GlobalDCLocation, false) }() tsoLabel := fmt.Sprintf("TSO Service Group %d", gta.getGroupID()) diff --git a/server/server.go b/server/server.go index 7e0ee36278f..c10d15e90b1 100644 --- a/server/server.go +++ b/server/server.go @@ -1746,7 +1746,7 @@ func (s *Server) campaignLeader() { return } defer func() { - s.tsoAllocatorManager.ResetAllocatorGroup(tso.GlobalDCLocation) + s.tsoAllocatorManager.ResetAllocatorGroup(tso.GlobalDCLocation, false) failpoint.Inject("updateAfterResetTSO", func() { if err = allocator.UpdateTSO(); !errorspkg.Is(err, errs.ErrUpdateTimestamp) { log.Panic("the tso update after reset should return ErrUpdateTimestamp as expected", zap.Error(err)) From 92adb24ef996c3196be52be2b4a2a43e3fb06b4b Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Fri, 9 Aug 2024 17:12:41 +0800 Subject: [PATCH 06/42] *: add some comments to exported function, part 3 of enable revive.exported (#8507) ref tikv/pd#8458 add comments for exported and update some exported-function to unexported Signed-off-by: okJiang <819421878@qq.com> --- client/client.go | 25 ++++++ client/meta_storage_client.go | 3 + client/mock_pd_service_discovery.go | 49 +++++++++--- client/pd_service_discovery.go | 1 + client/resource_manager_client.go | 10 ++- client/tso_stream.go | 4 + pkg/mcs/scheduling/server/apis/v1/api.go | 1 + pkg/schedule/checker/checker_controller.go | 1 + pkg/schedule/prepare_checker.go | 3 +- pkg/schedule/schedulers/balance_leader.go | 9 ++- .../schedulers/balance_leader_test.go | 2 +- pkg/schedule/schedulers/balance_witness.go | 8 +- pkg/schedule/schedulers/evict_leader.go | 4 +- pkg/schedule/schedulers/evict_leader_test.go | 8 +- pkg/schedule/schedulers/evict_slow_trend.go | 5 +- pkg/schedule/schedulers/grant_hot_region.go | 5 +- pkg/schedule/schedulers/grant_leader.go | 4 +- pkg/schedule/schedulers/shuffle_hot_region.go | 4 +- pkg/schedule/schedulers/split_bucket.go | 1 + .../unsafe_recovery_controller.go | 78 +++++++++---------- .../unsafe_recovery_controller_test.go | 2 +- server/api/admin.go | 16 ++-- server/api/config.go | 8 +- server/api/diagnostic.go | 2 +- server/api/metric.go | 4 +- server/api/middleware.go | 2 +- server/api/plugin_disable.go | 4 +- server/api/region.go | 20 ++--- server/api/router.go | 26 +++---- server/api/rule.go | 2 +- server/api/scheduler.go | 2 +- server/api/server_test.go | 6 +- server/forward.go | 2 +- server/grpc_service.go | 8 +- tests/cluster.go | 4 +- tests/config.go | 13 ++-- tests/testutil.go | 5 ++ 37 files changed, 218 insertions(+), 133 deletions(-) diff --git a/client/client.go b/client/client.go index aafe4aba77f..8ba03161227 100644 --- a/client/client.go +++ b/client/client.go @@ -645,6 +645,7 @@ func (c *client) setup() error { return nil } +// Close closes the client. func (c *client) Close() { c.cancel() c.wg.Wait() @@ -802,6 +803,7 @@ func (c *client) UpdateOption(option DynamicOption, value any) error { return nil } +// GetAllMembers gets the members Info from PD. func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { start := time.Now() defer func() { cmdDurationGetAllMembers.Observe(time.Since(start).Seconds()) }() @@ -848,10 +850,12 @@ func (c *client) getRegionAPIClientAndContext(ctx context.Context, allowFollower return serviceClient, serviceClient.BuildGRPCTargetContext(ctx, !allowFollower) } +// GetTSAsync implements the TSOClient interface. func (c *client) GetTSAsync(ctx context.Context) TSFuture { return c.GetLocalTSAsync(ctx, globalDCLocation) } +// GetLocalTSAsync implements the TSOClient interface. func (c *client) GetLocalTSAsync(ctx context.Context, dcLocation string) TSFuture { defer trace.StartRegion(ctx, "pdclient.GetLocalTSAsync").End() if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { @@ -902,16 +906,19 @@ func (c *client) dispatchTSORequestWithRetry(ctx context.Context, dcLocation str return req } +// GetTS implements the TSOClient interface. func (c *client) GetTS(ctx context.Context) (physical int64, logical int64, err error) { resp := c.GetTSAsync(ctx) return resp.Wait() } +// GetLocalTS implements the TSOClient interface. func (c *client) GetLocalTS(ctx context.Context, dcLocation string) (physical int64, logical int64, err error) { resp := c.GetLocalTSAsync(ctx, dcLocation) return resp.Wait() } +// GetMinTS implements the TSOClient interface. func (c *client) GetMinTS(ctx context.Context) (physical int64, logical int64, err error) { // Handle compatibility issue in case of PD/API server doesn't support GetMinTS API. serviceMode := c.getServiceMode() @@ -975,6 +982,7 @@ func handleRegionResponse(res *pdpb.GetRegionResponse) *Region { return r } +// GetRegionFromMember implements the RPCClient interface. func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string, _ ...GetRegionOption) (*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.GetRegionFromMember", opentracing.ChildOf(span.Context())) @@ -1013,6 +1021,7 @@ func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs return handleRegionResponse(resp), nil } +// GetRegion implements the RPCClient interface. func (c *client) GetRegion(ctx context.Context, key []byte, opts ...GetRegionOption) (*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.GetRegion", opentracing.ChildOf(span.Context())) @@ -1051,6 +1060,7 @@ func (c *client) GetRegion(ctx context.Context, key []byte, opts ...GetRegionOpt return handleRegionResponse(resp), nil } +// GetPrevRegion implements the RPCClient interface. func (c *client) GetPrevRegion(ctx context.Context, key []byte, opts ...GetRegionOption) (*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.GetPrevRegion", opentracing.ChildOf(span.Context())) @@ -1089,6 +1099,7 @@ func (c *client) GetPrevRegion(ctx context.Context, key []byte, opts ...GetRegio return handleRegionResponse(resp), nil } +// GetRegionByID implements the RPCClient interface. func (c *client) GetRegionByID(ctx context.Context, regionID uint64, opts ...GetRegionOption) (*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.GetRegionByID", opentracing.ChildOf(span.Context())) @@ -1127,6 +1138,7 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64, opts ...Get return handleRegionResponse(resp), nil } +// ScanRegions implements the RPCClient interface. func (c *client) ScanRegions(ctx context.Context, key, endKey []byte, limit int, opts ...GetRegionOption) ([]*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.ScanRegions", opentracing.ChildOf(span.Context())) @@ -1176,6 +1188,7 @@ func (c *client) ScanRegions(ctx context.Context, key, endKey []byte, limit int, return handleRegionsResponse(resp), nil } +// BatchScanRegions implements the RPCClient interface. func (c *client) BatchScanRegions(ctx context.Context, ranges []KeyRange, limit int, opts ...GetRegionOption) ([]*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.BatchScanRegions", opentracing.ChildOf(span.Context())) @@ -1274,6 +1287,7 @@ func handleRegionsResponse(resp *pdpb.ScanRegionsResponse) []*Region { return regions } +// GetStore implements the RPCClient interface. func (c *client) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.GetStore", opentracing.ChildOf(span.Context())) @@ -1312,6 +1326,7 @@ func handleStoreResponse(resp *pdpb.GetStoreResponse) (*metapb.Store, error) { return store, nil } +// GetAllStores implements the RPCClient interface. func (c *client) GetAllStores(ctx context.Context, opts ...GetStoreOption) ([]*metapb.Store, error) { // Applies options options := &GetStoreOp{} @@ -1345,6 +1360,7 @@ func (c *client) GetAllStores(ctx context.Context, opts ...GetStoreOption) ([]*m return resp.GetStores(), nil } +// UpdateGCSafePoint implements the RPCClient interface. func (c *client) UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint64, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.UpdateGCSafePoint", opentracing.ChildOf(span.Context())) @@ -1406,6 +1422,7 @@ func (c *client) UpdateServiceGCSafePoint(ctx context.Context, serviceID string, return resp.GetMinSafePoint(), nil } +// ScatterRegion implements the RPCClient interface. func (c *client) ScatterRegion(ctx context.Context, regionID uint64) error { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.ScatterRegion", opentracing.ChildOf(span.Context())) @@ -1440,6 +1457,7 @@ func (c *client) scatterRegionsWithGroup(ctx context.Context, regionID uint64, g return nil } +// ScatterRegions implements the RPCClient interface. func (c *client) ScatterRegions(ctx context.Context, regionsID []uint64, opts ...RegionsOption) (*pdpb.ScatterRegionResponse, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.ScatterRegions", opentracing.ChildOf(span.Context())) @@ -1448,6 +1466,7 @@ func (c *client) ScatterRegions(ctx context.Context, regionsID []uint64, opts .. return c.scatterRegionsWithOptions(ctx, regionsID, opts...) } +// SplitAndScatterRegions implements the RPCClient interface. func (c *client) SplitAndScatterRegions(ctx context.Context, splitKeys [][]byte, opts ...RegionsOption) (*pdpb.SplitAndScatterRegionsResponse, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.SplitAndScatterRegions", opentracing.ChildOf(span.Context())) @@ -1476,6 +1495,7 @@ func (c *client) SplitAndScatterRegions(ctx context.Context, splitKeys [][]byte, return protoClient.SplitAndScatterRegions(ctx, req) } +// GetOperator implements the RPCClient interface. func (c *client) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.GetOperator", opentracing.ChildOf(span.Context())) @@ -1575,6 +1595,7 @@ func trimHTTPPrefix(str string) string { return str } +// LoadGlobalConfig implements the RPCClient interface. func (c *client) LoadGlobalConfig(ctx context.Context, names []string, configPath string) ([]GlobalConfigItem, int64, error) { ctx, cancel := context.WithTimeout(ctx, c.option.timeout) defer cancel() @@ -1602,6 +1623,7 @@ func (c *client) LoadGlobalConfig(ctx context.Context, names []string, configPat return res, resp.GetRevision(), nil } +// StoreGlobalConfig implements the RPCClient interface. func (c *client) StoreGlobalConfig(ctx context.Context, configPath string, items []GlobalConfigItem) error { resArr := make([]*pdpb.GlobalConfigItem, len(items)) for i, it := range items { @@ -1620,6 +1642,7 @@ func (c *client) StoreGlobalConfig(ctx context.Context, configPath string, items return nil } +// WatchGlobalConfig implements the RPCClient interface. func (c *client) WatchGlobalConfig(ctx context.Context, configPath string, revision int64) (chan []GlobalConfigItem, error) { // TODO: Add retry mechanism // register watch components there @@ -1671,6 +1694,7 @@ func (c *client) WatchGlobalConfig(ctx context.Context, configPath string, revis return globalConfigWatcherCh, err } +// GetExternalTimestamp implements the RPCClient interface. func (c *client) GetExternalTimestamp(ctx context.Context) (uint64, error) { ctx, cancel := context.WithTimeout(ctx, c.option.timeout) defer cancel() @@ -1691,6 +1715,7 @@ func (c *client) GetExternalTimestamp(ctx context.Context) (uint64, error) { return resp.GetTimestamp(), nil } +// SetExternalTimestamp implements the RPCClient interface. func (c *client) SetExternalTimestamp(ctx context.Context, timestamp uint64) error { ctx, cancel := context.WithTimeout(ctx, c.option.timeout) defer cancel() diff --git a/client/meta_storage_client.go b/client/meta_storage_client.go index fe7e8a33e93..200491cf19a 100644 --- a/client/meta_storage_client.go +++ b/client/meta_storage_client.go @@ -104,6 +104,7 @@ func getPrefix(key []byte) []byte { return []byte{0} } +// Put implements the MetaStorageClient interface. func (c *client) Put(ctx context.Context, key, value []byte, opts ...OpOption) (*meta_storagepb.PutResponse, error) { options := &Op{} for _, opt := range opts { @@ -139,6 +140,7 @@ func (c *client) Put(ctx context.Context, key, value []byte, opts ...OpOption) ( return resp, nil } +// Get implements the MetaStorageClient interface. func (c *client) Get(ctx context.Context, key []byte, opts ...OpOption) (*meta_storagepb.GetResponse, error) { options := &Op{} for _, opt := range opts { @@ -177,6 +179,7 @@ func (c *client) Get(ctx context.Context, key []byte, opts ...OpOption) (*meta_s return resp, nil } +// Watch implements the MetaStorageClient interface. func (c *client) Watch(ctx context.Context, key []byte, opts ...OpOption) (chan []*meta_storagepb.Event, error) { eventCh := make(chan []*meta_storagepb.Event, 100) options := &Op{} diff --git a/client/mock_pd_service_discovery.go b/client/mock_pd_service_discovery.go index f1fabd0a1d2..16462b0b1e6 100644 --- a/client/mock_pd_service_discovery.go +++ b/client/mock_pd_service_discovery.go @@ -56,19 +56,46 @@ func (m *mockPDServiceDiscovery) GetAllServiceClients() []ServiceClient { return m.clients } -func (*mockPDServiceDiscovery) GetClusterID() uint64 { return 0 } -func (*mockPDServiceDiscovery) GetKeyspaceID() uint32 { return 0 } -func (*mockPDServiceDiscovery) GetKeyspaceGroupID() uint32 { return 0 } -func (*mockPDServiceDiscovery) GetServiceURLs() []string { return nil } +// GetClusterID implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) GetClusterID() uint64 { return 0 } + +// GetKeyspaceID implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) GetKeyspaceID() uint32 { return 0 } + +// GetKeyspaceGroupID implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) GetKeyspaceGroupID() uint32 { return 0 } + +// GetServiceURLs implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) GetServiceURLs() []string { return nil } + +// GetServingEndpointClientConn implements the ServiceDiscovery interface. func (*mockPDServiceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { return nil } -func (*mockPDServiceDiscovery) GetClientConns() *sync.Map { return nil } -func (*mockPDServiceDiscovery) GetServingURL() string { return "" } -func (*mockPDServiceDiscovery) GetBackupURLs() []string { return nil } -func (*mockPDServiceDiscovery) GetServiceClient() ServiceClient { return nil } + +// GetClientConns implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) GetClientConns() *sync.Map { return nil } + +// GetServingURL implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) GetServingURL() string { return "" } + +// GetBackupURLs implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) GetBackupURLs() []string { return nil } + +// GetServiceClient implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) GetServiceClient() ServiceClient { return nil } + +// GetOrCreateGRPCConn implements the ServiceDiscovery interface. func (*mockPDServiceDiscovery) GetOrCreateGRPCConn(string) (*grpc.ClientConn, error) { return nil, nil } -func (*mockPDServiceDiscovery) ScheduleCheckMemberChanged() {} -func (*mockPDServiceDiscovery) CheckMemberChanged() error { return nil } -func (*mockPDServiceDiscovery) AddServingURLSwitchedCallback(...func()) {} + +// ScheduleCheckMemberChanged implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) ScheduleCheckMemberChanged() {} + +// CheckMemberChanged implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) CheckMemberChanged() error { return nil } + +// AddServingURLSwitchedCallback implements the ServiceDiscovery interface. +func (*mockPDServiceDiscovery) AddServingURLSwitchedCallback(...func()) {} + +// AddServiceURLsSwitchedCallback implements the ServiceDiscovery interface. func (*mockPDServiceDiscovery) AddServiceURLsSwitchedCallback(...func()) {} diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index 9378ed278e0..e8f4c0d7707 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -482,6 +482,7 @@ func newPDServiceDiscovery( return pdsd } +// Init initializes the PD service discovery. func (c *pdServiceDiscovery) Init() error { if c.isInitialized { return nil diff --git a/client/resource_manager_client.go b/client/resource_manager_client.go index 19adbd199b0..45182c07d3a 100644 --- a/client/resource_manager_client.go +++ b/client/resource_manager_client.go @@ -108,6 +108,7 @@ func (c *client) ListResourceGroups(ctx context.Context, ops ...GetResourceGroup return resp.GetGroups(), nil } +// GetResourceGroup implements the ResourceManagerClient interface. func (c *client) GetResourceGroup(ctx context.Context, resourceGroupName string, ops ...GetResourceGroupOption) (*rmpb.ResourceGroup, error) { cc, err := c.resourceManagerClient() if err != nil { @@ -133,10 +134,12 @@ func (c *client) GetResourceGroup(ctx context.Context, resourceGroupName string, return resp.GetGroup(), nil } +// AddResourceGroup implements the ResourceManagerClient interface. func (c *client) AddResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) { return c.putResourceGroup(ctx, metaGroup, add) } +// ModifyResourceGroup implements the ResourceManagerClient interface. func (c *client) ModifyResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) { return c.putResourceGroup(ctx, metaGroup, modify) } @@ -167,6 +170,7 @@ func (c *client) putResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceG return resp.GetBody(), nil } +// DeleteResourceGroup implements the ResourceManagerClient interface. func (c *client) DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) { cc, err := c.resourceManagerClient() if err != nil { @@ -187,6 +191,7 @@ func (c *client) DeleteResourceGroup(ctx context.Context, resourceGroupName stri return resp.GetBody(), nil } +// LoadResourceGroups implements the ResourceManagerClient interface. func (c *client) LoadResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, int64, error) { resp, err := c.Get(ctx, GroupSettingsPathPrefixBytes, WithPrefix()) if err != nil { @@ -206,6 +211,7 @@ func (c *client) LoadResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, return groups, resp.Header.Revision, nil } +// AcquireTokenBuckets implements the ResourceManagerClient interface. func (c *client) AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) { req := &tokenRequest{ done: make(chan error, 1), @@ -214,7 +220,7 @@ func (c *client) AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBuc Request: request, } c.tokenDispatcher.tokenBatchController.tokenRequestCh <- req - grantedTokens, err := req.Wait() + grantedTokens, err := req.wait() if err != nil { return nil, err } @@ -229,7 +235,7 @@ type tokenRequest struct { TokenBuckets []*rmpb.TokenBucketResponse } -func (req *tokenRequest) Wait() (tokenBuckets []*rmpb.TokenBucketResponse, err error) { +func (req *tokenRequest) wait() (tokenBuckets []*rmpb.TokenBucketResponse, err error) { select { case err = <-req.done: err = errors.WithStack(err) diff --git a/client/tso_stream.go b/client/tso_stream.go index da9cab95ba0..76b6ae3c51c 100644 --- a/client/tso_stream.go +++ b/client/tso_stream.go @@ -116,6 +116,7 @@ type pdTSOStreamAdapter struct { stream pdpb.PD_TsoClient } +// Send implements the grpcTSOStreamAdapter interface. func (s pdTSOStreamAdapter) Send(clusterID uint64, _, _ uint32, dcLocation string, count int64) error { req := &pdpb.TsoRequest{ Header: &pdpb.RequestHeader{ @@ -127,6 +128,7 @@ func (s pdTSOStreamAdapter) Send(clusterID uint64, _, _ uint32, dcLocation strin return s.stream.Send(req) } +// Recv implements the grpcTSOStreamAdapter interface. func (s pdTSOStreamAdapter) Recv() (tsoRequestResult, error) { resp, err := s.stream.Recv() if err != nil { @@ -145,6 +147,7 @@ type tsoTSOStreamAdapter struct { stream tsopb.TSO_TsoClient } +// Send implements the grpcTSOStreamAdapter interface. func (s tsoTSOStreamAdapter) Send(clusterID uint64, keyspaceID, keyspaceGroupID uint32, dcLocation string, count int64) error { req := &tsopb.TsoRequest{ Header: &tsopb.RequestHeader{ @@ -158,6 +161,7 @@ func (s tsoTSOStreamAdapter) Send(clusterID uint64, keyspaceID, keyspaceGroupID return s.stream.Send(req) } +// Recv implements the grpcTSOStreamAdapter interface. func (s tsoTSOStreamAdapter) Recv() (tsoRequestResult, error) { resp, err := s.stream.Recv() if err != nil { diff --git a/pkg/mcs/scheduling/server/apis/v1/api.go b/pkg/mcs/scheduling/server/apis/v1/api.go index 39aa11927ca..ce1522d465f 100644 --- a/pkg/mcs/scheduling/server/apis/v1/api.go +++ b/pkg/mcs/scheduling/server/apis/v1/api.go @@ -79,6 +79,7 @@ type server struct { *scheserver.Server } +// GetCluster returns the cluster. func (s *server) GetCluster() sche.SchedulerCluster { return s.Server.GetCluster() } diff --git a/pkg/schedule/checker/checker_controller.go b/pkg/schedule/checker/checker_controller.go index 200ab388e30..17bf36cc92f 100644 --- a/pkg/schedule/checker/checker_controller.go +++ b/pkg/schedule/checker/checker_controller.go @@ -44,6 +44,7 @@ const ( // In order to avoid the patrolRegionScanLimit to be too big or too small, it will be limited to [128,8192]. // It takes about 10s to iterate 1,024,000 regions(with DefaultPatrolRegionInterval=10ms) where other steps are not considered. MinPatrolRegionScanLimit = 128 + // MaxPatrolScanRegionLimit is the max limit of regions to scan for a batch. MaxPatrolScanRegionLimit = 8192 patrolRegionPartition = 1024 ) diff --git a/pkg/schedule/prepare_checker.go b/pkg/schedule/prepare_checker.go index df7074b9073..187d68e3d9f 100644 --- a/pkg/schedule/prepare_checker.go +++ b/pkg/schedule/prepare_checker.go @@ -73,13 +73,14 @@ func (checker *prepareChecker) check(c *core.BasicCluster, collectWaitTime ...ti return true } +// IsPrepared returns whether the coordinator is prepared. func (checker *prepareChecker) IsPrepared() bool { checker.RLock() defer checker.RUnlock() return checker.prepared } -// for test purpose +// SetPrepared is for test purpose func (checker *prepareChecker) SetPrepared() { checker.Lock() defer checker.Unlock() diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index 05b429c825f..b434c7ad706 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -100,7 +100,7 @@ func (conf *balanceLeaderSchedulerConfig) validateLocked() bool { return conf.Batch >= 1 && conf.Batch <= 10 } -func (conf *balanceLeaderSchedulerConfig) Clone() *balanceLeaderSchedulerConfig { +func (conf *balanceLeaderSchedulerConfig) clone() *balanceLeaderSchedulerConfig { conf.RLock() defer conf.RUnlock() ranges := make([]core.KeyRange, len(conf.Ranges)) @@ -157,7 +157,7 @@ func (handler *balanceLeaderHandler) updateConfig(w http.ResponseWriter, r *http } func (handler *balanceLeaderHandler) listConfig(w http.ResponseWriter, _ *http.Request) { - conf := handler.config.Clone() + conf := handler.config.clone() handler.rd.JSON(w, http.StatusOK, conf) } @@ -190,6 +190,7 @@ func newBalanceLeaderScheduler(opController *operator.Controller, conf *balanceL return s } +// ServeHTTP implements the http.Handler interface. func (l *balanceLeaderScheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { l.handler.ServeHTTP(w, r) } @@ -204,12 +205,14 @@ func WithBalanceLeaderName(name string) BalanceLeaderCreateOption { } } +// EncodeConfig implements the Scheduler interface. func (l *balanceLeaderScheduler) EncodeConfig() ([]byte, error) { l.conf.RLock() defer l.conf.RUnlock() return EncodeConfig(l.conf) } +// ReloadConfig implements the Scheduler interface. func (l *balanceLeaderScheduler) ReloadConfig() error { l.conf.Lock() defer l.conf.Unlock() @@ -229,6 +232,7 @@ func (l *balanceLeaderScheduler) ReloadConfig() error { return nil } +// IsScheduleAllowed implements the Scheduler interface. func (l *balanceLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { allowed := l.OpController.OperatorCount(operator.OpLeader) < cluster.GetSchedulerConfig().GetLeaderScheduleLimit() if !allowed { @@ -329,6 +333,7 @@ func (cs *candidateStores) resortStoreWithPos(pos int) { } } +// Schedule implements the Scheduler interface. func (l *balanceLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { basePlan := plan.NewBalanceSchedulerPlan() var collector *plan.Collector diff --git a/pkg/schedule/schedulers/balance_leader_test.go b/pkg/schedule/schedulers/balance_leader_test.go index 44ad1f2c0a5..eb1d8a539ce 100644 --- a/pkg/schedule/schedulers/balance_leader_test.go +++ b/pkg/schedule/schedulers/balance_leader_test.go @@ -30,7 +30,7 @@ func TestBalanceLeaderSchedulerConfigClone(t *testing.T) { Ranges: keyRanges1, Batch: 10, } - conf2 := conf.Clone() + conf2 := conf.clone() re.Equal(conf.Batch, conf2.Batch) re.Equal(conf.Ranges, conf2.Ranges) diff --git a/pkg/schedule/schedulers/balance_witness.go b/pkg/schedule/schedulers/balance_witness.go index 0415ad03618..599af6df637 100644 --- a/pkg/schedule/schedulers/balance_witness.go +++ b/pkg/schedule/schedulers/balance_witness.go @@ -59,7 +59,7 @@ type balanceWitnessSchedulerConfig struct { Batch int `json:"batch"` } -func (conf *balanceWitnessSchedulerConfig) Update(data []byte) (int, any) { +func (conf *balanceWitnessSchedulerConfig) update(data []byte) (int, any) { conf.Lock() defer conf.Unlock() @@ -97,7 +97,7 @@ func (conf *balanceWitnessSchedulerConfig) validateLocked() bool { return conf.Batch >= 1 && conf.Batch <= 10 } -func (conf *balanceWitnessSchedulerConfig) Clone() *balanceWitnessSchedulerConfig { +func (conf *balanceWitnessSchedulerConfig) clone() *balanceWitnessSchedulerConfig { conf.RLock() defer conf.RUnlock() ranges := make([]core.KeyRange, len(conf.Ranges)) @@ -149,12 +149,12 @@ func newBalanceWitnessHandler(conf *balanceWitnessSchedulerConfig) http.Handler func (handler *balanceWitnessHandler) updateConfig(w http.ResponseWriter, r *http.Request) { data, _ := io.ReadAll(r.Body) r.Body.Close() - httpCode, v := handler.config.Update(data) + httpCode, v := handler.config.update(data) handler.rd.JSON(w, httpCode, v) } func (handler *balanceWitnessHandler) listConfig(w http.ResponseWriter, _ *http.Request) { - conf := handler.config.Clone() + conf := handler.config.clone() handler.rd.JSON(w, http.StatusOK, conf) } diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index d43f540a489..d4e26cb1b68 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -72,7 +72,7 @@ func (conf *evictLeaderSchedulerConfig) getBatch() int { return conf.Batch } -func (conf *evictLeaderSchedulerConfig) Clone() *evictLeaderSchedulerConfig { +func (conf *evictLeaderSchedulerConfig) clone() *evictLeaderSchedulerConfig { conf.RLock() defer conf.RUnlock() storeIDWithRanges := make(map[uint64][]core.KeyRange) @@ -460,7 +460,7 @@ func (handler *evictLeaderHandler) updateConfig(w http.ResponseWriter, r *http.R } func (handler *evictLeaderHandler) listConfig(w http.ResponseWriter, _ *http.Request) { - conf := handler.config.Clone() + conf := handler.config.clone() handler.rd.JSON(w, http.StatusOK, conf) } diff --git a/pkg/schedule/schedulers/evict_leader_test.go b/pkg/schedule/schedulers/evict_leader_test.go index eb97be516d7..692dda63437 100644 --- a/pkg/schedule/schedulers/evict_leader_test.go +++ b/pkg/schedule/schedulers/evict_leader_test.go @@ -89,19 +89,19 @@ func TestConfigClone(t *testing.T) { re := require.New(t) emptyConf := &evictLeaderSchedulerConfig{StoreIDWithRanges: make(map[uint64][]core.KeyRange)} - con2 := emptyConf.Clone() + con2 := emptyConf.clone() re.Empty(con2.getKeyRangesByID(1)) con2.StoreIDWithRanges[1], _ = getKeyRanges([]string{"a", "b", "c", "d"}) - con3 := con2.Clone() + con3 := con2.clone() re.Equal(len(con3.getRanges(1)), len(con2.getRanges(1))) con3.StoreIDWithRanges[1][0].StartKey = []byte("aaa") - con4 := con3.Clone() + con4 := con3.clone() re.True(bytes.Equal(con4.StoreIDWithRanges[1][0].StartKey, con3.StoreIDWithRanges[1][0].StartKey)) con4.Batch = 10 - con5 := con4.Clone() + con5 := con4.clone() re.Equal(con5.getBatch(), con4.getBatch()) } diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index 767a2ee40a0..206791900c6 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -76,7 +76,7 @@ func initEvictSlowTrendSchedulerConfig(storage endpoint.ConfigStorage) *evictSlo } } -func (conf *evictSlowTrendSchedulerConfig) Clone() *evictSlowTrendSchedulerConfig { +func (conf *evictSlowTrendSchedulerConfig) clone() *evictSlowTrendSchedulerConfig { conf.RLock() defer conf.RUnlock() return &evictSlowTrendSchedulerConfig{ @@ -266,7 +266,7 @@ func (handler *evictSlowTrendHandler) updateConfig(w http.ResponseWriter, r *htt } func (handler *evictSlowTrendHandler) listConfig(w http.ResponseWriter, _ *http.Request) { - conf := handler.config.Clone() + conf := handler.config.clone() handler.rd.JSON(w, http.StatusOK, conf) } @@ -276,6 +276,7 @@ type evictSlowTrendScheduler struct { handler http.Handler } +// GetNextInterval implements the Scheduler interface. func (s *evictSlowTrendScheduler) GetNextInterval(time.Duration) time.Duration { var growthType intervalGrowthType // If it already found a slow node as candidate, the next interval should be shorter diff --git a/pkg/schedule/schedulers/grant_hot_region.go b/pkg/schedule/schedulers/grant_hot_region.go index 38eadb10c41..15a520f95d0 100644 --- a/pkg/schedule/schedulers/grant_hot_region.go +++ b/pkg/schedule/schedulers/grant_hot_region.go @@ -90,7 +90,7 @@ func (conf *grantHotRegionSchedulerConfig) clone() *grantHotRegionSchedulerConfi } } -func (conf *grantHotRegionSchedulerConfig) Persist() error { +func (conf *grantHotRegionSchedulerConfig) persist() error { conf.RLock() defer conf.RUnlock() data, err := EncodeConfig(conf) @@ -215,7 +215,7 @@ func (handler *grantHotRegionHandler) updateConfig(w http.ResponseWriter, r *htt return } - if err = handler.config.Persist(); err != nil { + if err = handler.config.persist(); err != nil { handler.config.setStoreLeaderID(0) handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) return @@ -239,6 +239,7 @@ func newGrantHotRegionHandler(config *grantHotRegionSchedulerConfig) http.Handle return router } +// Schedule implements the Scheduler interface. func (s *grantHotRegionScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*operator.Operator, []plan.Plan) { grantHotRegionCounter.Inc() typ := s.randomType() diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index 747a7ee6c0c..134eddda880 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -49,7 +49,7 @@ type grantLeaderSchedulerConfig struct { removeSchedulerCb func(name string) error } -func (conf *grantLeaderSchedulerConfig) BuildWithArgs(args []string) error { +func (conf *grantLeaderSchedulerConfig) buildWithArgs(args []string) error { if len(args) != 1 { return errs.ErrSchedulerConfig.FastGenByArgs("id") } @@ -285,7 +285,7 @@ func (handler *grantLeaderHandler) updateConfig(w http.ResponseWriter, r *http.R args = append(args, handler.config.getRanges(id)...) } - err := handler.config.BuildWithArgs(args) + err := handler.config.buildWithArgs(args) if err != nil { handler.rd.JSON(w, http.StatusBadRequest, err.Error()) return diff --git a/pkg/schedule/schedulers/shuffle_hot_region.go b/pkg/schedule/schedulers/shuffle_hot_region.go index 71bc9107fdb..686322961cb 100644 --- a/pkg/schedule/schedulers/shuffle_hot_region.go +++ b/pkg/schedule/schedulers/shuffle_hot_region.go @@ -46,7 +46,7 @@ type shuffleHotRegionSchedulerConfig struct { Limit uint64 `json:"limit"` } -func (conf *shuffleHotRegionSchedulerConfig) Clone() *shuffleHotRegionSchedulerConfig { +func (conf *shuffleHotRegionSchedulerConfig) clone() *shuffleHotRegionSchedulerConfig { conf.RLock() defer conf.RUnlock() return &shuffleHotRegionSchedulerConfig{ @@ -238,7 +238,7 @@ func (handler *shuffleHotRegionHandler) updateConfig(w http.ResponseWriter, r *h } func (handler *shuffleHotRegionHandler) listConfig(w http.ResponseWriter, _ *http.Request) { - conf := handler.config.Clone() + conf := handler.config.clone() handler.rd.JSON(w, http.StatusOK, conf) } diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index 92edea1c82a..7b238890107 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -163,6 +163,7 @@ func newSplitBucketScheduler(opController *operator.Controller, conf *splitBucke return ret } +// ReloadConfig implement Scheduler interface. func (s *splitBucketScheduler) ReloadConfig() error { s.conf.Lock() defer s.conf.Unlock() diff --git a/pkg/unsaferecovery/unsafe_recovery_controller.go b/pkg/unsaferecovery/unsafe_recovery_controller.go index 89cd6e6393c..aa3cf192270 100644 --- a/pkg/unsaferecovery/unsafe_recovery_controller.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller.go @@ -751,31 +751,31 @@ type regionItem struct { // Less returns true if the region start key is less than the other. func (r *regionItem) Less(other *regionItem) bool { - left := r.Region().GetStartKey() - right := other.Region().GetStartKey() + left := r.region().GetStartKey() + right := other.region().GetStartKey() return bytes.Compare(left, right) < 0 } -func (r *regionItem) Contains(key []byte) bool { - start, end := r.Region().GetStartKey(), r.Region().GetEndKey() +func (r *regionItem) contains(key []byte) bool { + start, end := r.region().GetStartKey(), r.region().GetEndKey() return bytes.Compare(key, start) >= 0 && (len(end) == 0 || bytes.Compare(key, end) < 0) } -func (r *regionItem) Region() *metapb.Region { +func (r *regionItem) region() *metapb.Region { return r.report.GetRegionState().GetRegion() } -func (r *regionItem) IsInitialized() bool { - return len(r.Region().Peers) != 0 +func (r *regionItem) isInitialized() bool { + return len(r.region().Peers) != 0 } -func (r *regionItem) IsEpochStale(other *regionItem) bool { - re := r.Region().GetRegionEpoch() - oe := other.Region().GetRegionEpoch() +func (r *regionItem) isEpochStale(other *regionItem) bool { + re := r.region().GetRegionEpoch() + oe := other.region().GetRegionEpoch() return re.GetVersion() < oe.GetVersion() || (re.GetVersion() == oe.GetVersion() && re.GetConfVer() < oe.GetConfVer()) } -func (r *regionItem) IsRaftStale(origin *regionItem, u *Controller) bool { +func (r *regionItem) isRaftStale(origin *regionItem, u *Controller) bool { cmps := []func(a, b *regionItem) int{ func(a, b *regionItem) int { return int(a.report.GetRaftState().GetHardState().GetTerm()) - int(b.report.GetRaftState().GetHardState().GetTerm()) @@ -800,7 +800,7 @@ func (r *regionItem) IsRaftStale(origin *regionItem, u *Controller) bool { return 1 } // better use voter rather than learner - for _, peer := range a.Region().GetPeers() { + for _, peer := range a.region().GetPeers() { if peer.StoreId == a.storeID { if peer.Role == metapb.PeerRole_DemotingVoter || peer.Role == metapb.PeerRole_Learner { return -1 @@ -857,11 +857,11 @@ func (t *regionTree) getOverlaps(item *regionItem) []*regionItem { result = item } - end := item.Region().GetEndKey() + end := item.region().GetEndKey() var overlaps []*regionItem t.tree.AscendGreaterOrEqual(result, func(i *regionItem) bool { over := i - if len(end) > 0 && bytes.Compare(end, over.Region().GetStartKey()) <= 0 { + if len(end) > 0 && bytes.Compare(end, over.region().GetStartKey()) <= 0 { return false } overlaps = append(overlaps, over) @@ -878,7 +878,7 @@ func (t *regionTree) find(item *regionItem) *regionItem { return false }) - if result == nil || !result.Contains(item.Region().GetStartKey()) { + if result == nil || !result.contains(item.region().GetStartKey()) { return nil } @@ -891,15 +891,15 @@ func (t *regionTree) find(item *regionItem) *regionItem { func (t *regionTree) insert(item *regionItem) (bool, error) { overlaps := t.getOverlaps(item) - if t.contains(item.Region().GetId()) { + if t.contains(item.region().GetId()) { // it's ensured by the `buildUpFromReports` that only insert the latest peer of one region. - return false, errors.Errorf("region %v shouldn't be updated twice", item.Region().GetId()) + return false, errors.Errorf("region %v shouldn't be updated twice", item.region().GetId()) } for _, newer := range overlaps { - log.Info("unsafe recovery found overlap regions", logutil.ZapRedactStringer("newer-region-meta", core.RegionToHexMeta(newer.Region())), logutil.ZapRedactStringer("older-region-meta", core.RegionToHexMeta(item.Region()))) + log.Info("unsafe recovery found overlap regions", logutil.ZapRedactStringer("newer-region-meta", core.RegionToHexMeta(newer.region())), logutil.ZapRedactStringer("older-region-meta", core.RegionToHexMeta(item.region()))) // it's ensured by the `buildUpFromReports` that peers are inserted in epoch descending order. - if newer.IsEpochStale(item) { + if newer.isEpochStale(item) { return false, errors.Errorf("region %v's epoch shouldn't be staler than old ones %v", item, newer) } } @@ -907,7 +907,7 @@ func (t *regionTree) insert(item *regionItem) (bool, error) { return false, nil } - t.regions[item.Region().GetId()] = item + t.regions[item.region().GetId()] = item t.tree.ReplaceOrInsert(item) return true, nil } @@ -925,7 +925,7 @@ func (u *Controller) buildUpFromReports() (*regionTree, map[uint64][]*regionItem for storeID, storeReport := range u.storeReports { for _, peerReport := range storeReport.PeerReports { item := ®ionItem{report: peerReport, storeID: storeID} - peersMap[item.Region().GetId()] = append(peersMap[item.Region().GetId()], item) + peersMap[item.region().GetId()] = append(peersMap[item.region().GetId()], item) } } @@ -934,11 +934,11 @@ func (u *Controller) buildUpFromReports() (*regionTree, map[uint64][]*regionItem for _, peers := range peersMap { var latest *regionItem for _, peer := range peers { - if latest == nil || latest.IsEpochStale(peer) { + if latest == nil || latest.isEpochStale(peer) { latest = peer } } - if !latest.IsInitialized() { + if !latest.isInitialized() { // ignore the uninitialized peer continue } @@ -947,7 +947,7 @@ func (u *Controller) buildUpFromReports() (*regionTree, map[uint64][]*regionItem // sort in descending order of epoch sort.SliceStable(newestPeerReports, func(i, j int) bool { - return newestPeerReports[j].IsEpochStale(newestPeerReports[i]) + return newestPeerReports[j].isEpochStale(newestPeerReports[i]) }) newestRegionTree := newRegionTree() @@ -963,7 +963,7 @@ func (u *Controller) buildUpFromReports() (*regionTree, map[uint64][]*regionItem func (u *Controller) selectLeader(peersMap map[uint64][]*regionItem, region *metapb.Region) *regionItem { var leader *regionItem for _, peer := range peersMap[region.GetId()] { - if leader == nil || leader.IsRaftStale(peer, u) { + if leader == nil || leader.isRaftStale(peer, u) { leader = peer } } @@ -978,7 +978,7 @@ func (u *Controller) generateTombstoneTiFlashLearnerPlan(newestRegionTree *regio var err error newestRegionTree.tree.Ascend(func(item *regionItem) bool { - region := item.Region() + region := item.region() if !u.canElectLeader(region, false) { leader := u.selectLeader(peersMap, region) if leader == nil { @@ -1019,7 +1019,7 @@ func (u *Controller) generateForceLeaderPlan(newestRegionTree *regionTree, peers // considering the Failed stores newestRegionTree.tree.Ascend(func(item *regionItem) bool { report := item.report - region := item.Region() + region := item.region() if !u.canElectLeader(region, false) { if hasForceLeader(region) { // already is a force leader, skip @@ -1050,7 +1050,7 @@ func (u *Controller) generateForceLeaderPlan(newestRegionTree *regionTree, peers } if u.autoDetect { // For auto detect, the failedStores is empty. So need to add the detected Failed store to the list - for _, peer := range u.getFailedPeers(leader.Region()) { + for _, peer := range u.getFailedPeers(leader.region()) { found := false for _, store := range storeRecoveryPlan.ForceLeader.FailedStores { if store == peer.StoreId { @@ -1064,7 +1064,7 @@ func (u *Controller) generateForceLeaderPlan(newestRegionTree *regionTree, peers } } storeRecoveryPlan.ForceLeader.EnterForceLeaders = append(storeRecoveryPlan.ForceLeader.EnterForceLeaders, region.GetId()) - u.recordAffectedRegion(leader.Region()) + u.recordAffectedRegion(leader.region()) hasPlan = true } return true @@ -1104,7 +1104,7 @@ func (u *Controller) generateDemoteFailedVoterPlan(newestRegionTree *regionTree, // Check the regions in newest Region Tree to see if it can still elect leader // considering the Failed stores newestRegionTree.tree.Ascend(func(item *regionItem) bool { - region := item.Region() + region := item.region() if !u.canElectLeader(region, false) { leader := findForceLeader(peersMap, region) if leader == nil { @@ -1115,10 +1115,10 @@ func (u *Controller) generateDemoteFailedVoterPlan(newestRegionTree *regionTree, storeRecoveryPlan.Demotes = append(storeRecoveryPlan.Demotes, &pdpb.DemoteFailedVoters{ RegionId: region.GetId(), - FailedVoters: u.getFailedPeers(leader.Region()), + FailedVoters: u.getFailedPeers(leader.region()), }, ) - u.recordAffectedRegion(leader.Region()) + u.recordAffectedRegion(leader.region()) hasPlan = true } return true @@ -1181,7 +1181,7 @@ func (u *Controller) generateCreateEmptyRegionPlan(newestRegionTree *regionTree, lastEnd := []byte("") var lastStoreID uint64 newestRegionTree.tree.Ascend(func(item *regionItem) bool { - region := item.Region() + region := item.region() storeID := item.storeID if !bytes.Equal(region.StartKey, lastEnd) { if u.cluster.GetStore(storeID).IsTiFlash() { @@ -1200,16 +1200,16 @@ func (u *Controller) generateCreateEmptyRegionPlan(newestRegionTree *regionTree, // paranoid check: shouldn't overlap with any of the peers for _, peers := range peersMap { for _, peer := range peers { - if !peer.IsInitialized() { + if !peer.isInitialized() { continue } - if (bytes.Compare(newRegion.StartKey, peer.Region().StartKey) <= 0 && - (len(newRegion.EndKey) == 0 || bytes.Compare(peer.Region().StartKey, newRegion.EndKey) < 0)) || - ((len(peer.Region().EndKey) == 0 || bytes.Compare(newRegion.StartKey, peer.Region().EndKey) < 0) && - (len(newRegion.EndKey) == 0 || (len(peer.Region().EndKey) != 0 && bytes.Compare(peer.Region().EndKey, newRegion.EndKey) <= 0))) { + if (bytes.Compare(newRegion.StartKey, peer.region().StartKey) <= 0 && + (len(newRegion.EndKey) == 0 || bytes.Compare(peer.region().StartKey, newRegion.EndKey) < 0)) || + ((len(peer.region().EndKey) == 0 || bytes.Compare(newRegion.StartKey, peer.region().EndKey) < 0) && + (len(newRegion.EndKey) == 0 || (len(peer.region().EndKey) != 0 && bytes.Compare(peer.region().EndKey, newRegion.EndKey) <= 0))) { err = errors.Errorf( "Find overlap peer %v with newly created empty region %v", - logutil.RedactStringer(core.RegionToHexMeta(peer.Region())), + logutil.RedactStringer(core.RegionToHexMeta(peer.region())), logutil.RedactStringer(core.RegionToHexMeta(newRegion)), ) return false diff --git a/pkg/unsaferecovery/unsafe_recovery_controller_test.go b/pkg/unsaferecovery/unsafe_recovery_controller_test.go index cce38285212..4eeed08077c 100644 --- a/pkg/unsaferecovery/unsafe_recovery_controller_test.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller_test.go @@ -1934,7 +1934,7 @@ func TestSelectLeader(t *testing.T) { Id: 1, } leader := recoveryController.selectLeader(peersMap, region) - re.Equal(leader.Region().Id, c.leaderID, "case: %d", i) + re.Equal(leader.region().Id, c.leaderID, "case: %d", i) } } diff --git a/server/api/admin.go b/server/api/admin.go index 2184dc66aa6..dfaecbec755 100644 --- a/server/api/admin.go +++ b/server/api/admin.go @@ -62,7 +62,7 @@ func (h *adminHandler) DeleteRegionCache(w http.ResponseWriter, r *http.Request) } rc.RemoveRegionIfExist(regionID) if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { - err = h.DeleteRegionCacheInSchedulingServer(regionID) + err = h.deleteRegionCacheInSchedulingServer(regionID) } msg := "The region is removed from server cache." h.rd.JSON(w, http.StatusOK, h.buildMsg(msg, err)) @@ -102,7 +102,7 @@ func (h *adminHandler) DeleteRegionStorage(w http.ResponseWriter, r *http.Reques // Remove region from cache. rc.RemoveRegionIfExist(regionID) if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { - err = h.DeleteRegionCacheInSchedulingServer(regionID) + err = h.deleteRegionCacheInSchedulingServer(regionID) } msg := "The region is removed from server cache and region meta storage." h.rd.JSON(w, http.StatusOK, h.buildMsg(msg, err)) @@ -118,7 +118,7 @@ func (h *adminHandler) DeleteAllRegionCache(w http.ResponseWriter, r *http.Reque rc := getCluster(r) rc.ResetRegionCache() if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { - err = h.DeleteRegionCacheInSchedulingServer() + err = h.deleteRegionCacheInSchedulingServer() } msg := "All regions are removed from server cache." h.rd.JSON(w, http.StatusOK, h.buildMsg(msg, err)) @@ -148,7 +148,7 @@ func (h *adminHandler) SavePersistFile(w http.ResponseWriter, r *http.Request) { h.rd.Text(w, http.StatusOK, "") } -func (h *adminHandler) MarkSnapshotRecovering(w http.ResponseWriter, _ *http.Request) { +func (h *adminHandler) markSnapshotRecovering(w http.ResponseWriter, _ *http.Request) { if err := h.svr.MarkSnapshotRecovering(); err != nil { h.rd.Text(w, http.StatusInternalServerError, err.Error()) return @@ -156,7 +156,7 @@ func (h *adminHandler) MarkSnapshotRecovering(w http.ResponseWriter, _ *http.Req h.rd.Text(w, http.StatusOK, "") } -func (h *adminHandler) IsSnapshotRecovering(w http.ResponseWriter, r *http.Request) { +func (h *adminHandler) isSnapshotRecovering(w http.ResponseWriter, r *http.Request) { marked, err := h.svr.IsSnapshotRecovering(r.Context()) if err != nil { h.rd.Text(w, http.StatusInternalServerError, err.Error()) @@ -168,7 +168,7 @@ func (h *adminHandler) IsSnapshotRecovering(w http.ResponseWriter, r *http.Reque h.rd.JSON(w, http.StatusOK, &resStruct{Marked: marked}) } -func (h *adminHandler) UnmarkSnapshotRecovering(w http.ResponseWriter, r *http.Request) { +func (h *adminHandler) unmarkSnapshotRecovering(w http.ResponseWriter, r *http.Request) { if err := h.svr.UnmarkSnapshotRecovering(r.Context()); err != nil { h.rd.Text(w, http.StatusInternalServerError, err.Error()) return @@ -178,7 +178,7 @@ func (h *adminHandler) UnmarkSnapshotRecovering(w http.ResponseWriter, r *http.R // RecoverAllocID recover base alloc id // body should be in {"id": "123"} format -func (h *adminHandler) RecoverAllocID(w http.ResponseWriter, r *http.Request) { +func (h *adminHandler) recoverAllocID(w http.ResponseWriter, r *http.Request) { var input map[string]any if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { return @@ -215,7 +215,7 @@ func (h *adminHandler) RecoverAllocID(w http.ResponseWriter, r *http.Request) { h.rd.Text(w, http.StatusOK, "") } -func (h *adminHandler) DeleteRegionCacheInSchedulingServer(id ...uint64) error { +func (h *adminHandler) deleteRegionCacheInSchedulingServer(id ...uint64) error { addr, ok := h.svr.GetServicePrimaryAddr(h.svr.Context(), utils.SchedulingServiceName) if !ok { return errs.ErrNotFoundSchedulingAddr.FastGenByArgs() diff --git a/server/api/config.go b/server/api/config.go index d280439a988..9f568221d89 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -64,7 +64,7 @@ func (h *confHandler) GetConfig(w http.ResponseWriter, r *http.Request) { cfg := h.svr.GetConfig() if h.svr.IsServiceIndependent(utils.SchedulingServiceName) && r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { - schedulingServerConfig, err := h.GetSchedulingServerConfig() + schedulingServerConfig, err := h.getSchedulingServerConfig() if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return @@ -338,7 +338,7 @@ func getConfigMap(cfg map[string]any, key []string, value any) map[string]any { func (h *confHandler) GetScheduleConfig(w http.ResponseWriter, r *http.Request) { if h.svr.IsServiceIndependent(utils.SchedulingServiceName) && r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { - cfg, err := h.GetSchedulingServerConfig() + cfg, err := h.getSchedulingServerConfig() if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return @@ -412,7 +412,7 @@ func (h *confHandler) SetScheduleConfig(w http.ResponseWriter, r *http.Request) func (h *confHandler) GetReplicationConfig(w http.ResponseWriter, r *http.Request) { if h.svr.IsServiceIndependent(utils.SchedulingServiceName) && r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { - cfg, err := h.GetSchedulingServerConfig() + cfg, err := h.getSchedulingServerConfig() if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return @@ -562,7 +562,7 @@ func (h *confHandler) GetPDServerConfig(w http.ResponseWriter, _ *http.Request) h.rd.JSON(w, http.StatusOK, h.svr.GetPDServerConfig()) } -func (h *confHandler) GetSchedulingServerConfig() (*config.Config, error) { +func (h *confHandler) getSchedulingServerConfig() (*config.Config, error) { addr, ok := h.svr.GetServicePrimaryAddr(h.svr.Context(), utils.SchedulingServiceName) if !ok { return nil, errs.ErrNotFoundSchedulingAddr.FastGenByArgs() diff --git a/server/api/diagnostic.go b/server/api/diagnostic.go index 1a05b0d83b8..23016519dee 100644 --- a/server/api/diagnostic.go +++ b/server/api/diagnostic.go @@ -36,7 +36,7 @@ func newDiagnosticHandler(svr *server.Server, rd *render.Render) *diagnosticHand } } -func (h *diagnosticHandler) GetDiagnosticResult(w http.ResponseWriter, r *http.Request) { +func (h *diagnosticHandler) getDiagnosticResult(w http.ResponseWriter, r *http.Request) { name := mux.Vars(r)["name"] result, err := h.handler.GetDiagnosticResult(name) if err != nil { diff --git a/server/api/metric.go b/server/api/metric.go index 6a9dc4a7cde..a457895bddf 100644 --- a/server/api/metric.go +++ b/server/api/metric.go @@ -28,11 +28,11 @@ type queryMetric struct { s *server.Server } -func newQueryMetric(s *server.Server) *queryMetric { +func newqueryMetric(s *server.Server) *queryMetric { return &queryMetric{s: s} } -func (h *queryMetric) QueryMetric(w http.ResponseWriter, r *http.Request) { +func (h *queryMetric) queryMetric(w http.ResponseWriter, r *http.Request) { metricAddr := h.s.GetConfig().PDServerCfg.MetricStorage if metricAddr == "" { http.Error(w, "metric storage doesn't set", http.StatusInternalServerError) diff --git a/server/api/middleware.go b/server/api/middleware.go index 6536935592f..010889f08ce 100644 --- a/server/api/middleware.go +++ b/server/api/middleware.go @@ -88,7 +88,7 @@ func newClusterMiddleware(s *server.Server) clusterMiddleware { } } -func (m clusterMiddleware) Middleware(h http.Handler) http.Handler { +func (m clusterMiddleware) middleware(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rc := m.s.GetRaftCluster() if rc == nil { diff --git a/server/api/plugin_disable.go b/server/api/plugin_disable.go index 289a140a4d6..596cddac5d7 100644 --- a/server/api/plugin_disable.go +++ b/server/api/plugin_disable.go @@ -30,12 +30,12 @@ func newPluginHandler(*server.Handler, *render.Render) *pluginHandler { return &pluginHandler{} } -func (*pluginHandler) LoadPlugin(w http.ResponseWriter, _ *http.Request) { +func (*pluginHandler) loadPlugin(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotImplemented) w.Write([]byte("load plugin is disabled, please `PLUGIN=1 $(MAKE) pd-server` first")) } -func (*pluginHandler) UnloadPlugin(w http.ResponseWriter, _ *http.Request) { +func (*pluginHandler) unloadPlugin(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotImplemented) w.Write([]byte("unload plugin is disabled, please `PLUGIN=1 $(MAKE) pd-server` first")) } diff --git a/server/api/region.go b/server/api/region.go index c6bc3d9e699..ae25d659544 100644 --- a/server/api/region.go +++ b/server/api/region.go @@ -553,7 +553,7 @@ const ( // @Failure 400 {string} string "The input is invalid." // @Router /regions/writeflow [get] func (h *regionsHandler) GetTopWriteFlowRegions(w http.ResponseWriter, r *http.Request) { - h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetBytesWritten() < b.GetBytesWritten() }) + h.getTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetBytesWritten() < b.GetBytesWritten() }) } // @Tags region @@ -564,7 +564,7 @@ func (h *regionsHandler) GetTopWriteFlowRegions(w http.ResponseWriter, r *http.R // @Failure 400 {string} string "The input is invalid." // @Router /regions/writequery [get] func (h *regionsHandler) GetTopWriteQueryRegions(w http.ResponseWriter, r *http.Request) { - h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { + h.getTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetWriteQueryNum() < b.GetWriteQueryNum() }) } @@ -577,7 +577,7 @@ func (h *regionsHandler) GetTopWriteQueryRegions(w http.ResponseWriter, r *http. // @Failure 400 {string} string "The input is invalid." // @Router /regions/readflow [get] func (h *regionsHandler) GetTopReadFlowRegions(w http.ResponseWriter, r *http.Request) { - h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetBytesRead() < b.GetBytesRead() }) + h.getTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetBytesRead() < b.GetBytesRead() }) } // @Tags region @@ -588,7 +588,7 @@ func (h *regionsHandler) GetTopReadFlowRegions(w http.ResponseWriter, r *http.Re // @Failure 400 {string} string "The input is invalid." // @Router /regions/readquery [get] func (h *regionsHandler) GetTopReadQueryRegions(w http.ResponseWriter, r *http.Request) { - h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { + h.getTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetReadQueryNum() < b.GetReadQueryNum() }) } @@ -601,7 +601,7 @@ func (h *regionsHandler) GetTopReadQueryRegions(w http.ResponseWriter, r *http.R // @Failure 400 {string} string "The input is invalid." // @Router /regions/confver [get] func (h *regionsHandler) GetTopConfVerRegions(w http.ResponseWriter, r *http.Request) { - h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { + h.getTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetMeta().GetRegionEpoch().GetConfVer() < b.GetMeta().GetRegionEpoch().GetConfVer() }) } @@ -614,7 +614,7 @@ func (h *regionsHandler) GetTopConfVerRegions(w http.ResponseWriter, r *http.Req // @Failure 400 {string} string "The input is invalid." // @Router /regions/version [get] func (h *regionsHandler) GetTopVersionRegions(w http.ResponseWriter, r *http.Request) { - h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { + h.getTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetMeta().GetRegionEpoch().GetVersion() < b.GetMeta().GetRegionEpoch().GetVersion() }) } @@ -627,7 +627,7 @@ func (h *regionsHandler) GetTopVersionRegions(w http.ResponseWriter, r *http.Req // @Failure 400 {string} string "The input is invalid." // @Router /regions/size [get] func (h *regionsHandler) GetTopSizeRegions(w http.ResponseWriter, r *http.Request) { - h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { + h.getTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetApproximateSize() < b.GetApproximateSize() }) } @@ -640,7 +640,7 @@ func (h *regionsHandler) GetTopSizeRegions(w http.ResponseWriter, r *http.Reques // @Failure 400 {string} string "The input is invalid." // @Router /regions/keys [get] func (h *regionsHandler) GetTopKeysRegions(w http.ResponseWriter, r *http.Request) { - h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { + h.getTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetApproximateKeys() < b.GetApproximateKeys() }) } @@ -653,7 +653,7 @@ func (h *regionsHandler) GetTopKeysRegions(w http.ResponseWriter, r *http.Reques // @Failure 400 {string} string "The input is invalid." // @Router /regions/cpu [get] func (h *regionsHandler) GetTopCPURegions(w http.ResponseWriter, r *http.Request) { - h.GetTopNRegions(w, r, func(a, b *core.RegionInfo) bool { + h.getTopNRegions(w, r, func(a, b *core.RegionInfo) bool { return a.GetCPUUsage() < b.GetCPUUsage() }) } @@ -740,7 +740,7 @@ func (h *regionsHandler) AccelerateRegionsScheduleInRanges(w http.ResponseWriter h.rd.Text(w, http.StatusOK, msgBuilder.String()) } -func (h *regionsHandler) GetTopNRegions(w http.ResponseWriter, r *http.Request, less func(a, b *core.RegionInfo) bool) { +func (h *regionsHandler) getTopNRegions(w http.ResponseWriter, r *http.Request, less func(a, b *core.RegionInfo) bool) { rc := getCluster(r) limit, err := h.AdjustLimit(r.URL.Query().Get("limit")) if err != nil { diff --git a/server/api/router.go b/server/api/router.go index 7aef165b267..0e129706b43 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -126,7 +126,7 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { apiRouter := rootRouter.PathPrefix(apiPrefix).Subrouter() clusterRouter := apiRouter.NewRoute().Subrouter() - clusterRouter.Use(newClusterMiddleware(svr).Middleware) + clusterRouter.Use(newClusterMiddleware(svr).middleware) escapeRouter := clusterRouter.NewRoute().Subrouter().UseEncodedPath() @@ -149,11 +149,11 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(apiRouter, "/schedulers/{name}", schedulerHandler.PauseOrResumeScheduler, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) diagnosticHandler := newDiagnosticHandler(svr, rd) - registerFunc(clusterRouter, "/schedulers/diagnostic/{name}", diagnosticHandler.GetDiagnosticResult, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(clusterRouter, "/schedulers/diagnostic/{name}", diagnosticHandler.getDiagnosticResult, setMethods(http.MethodGet), setAuditBackend(prometheus)) schedulerConfigHandler := newSchedulerConfigHandler(svr, rd) - registerPrefix(apiRouter, "/scheduler-config", "HandleSchedulerConfig", schedulerConfigHandler.HandleSchedulerConfig, setMethods(http.MethodPost, http.MethodDelete, http.MethodPut, http.MethodPatch), setAuditBackend(localLog, prometheus)) - registerPrefix(apiRouter, "/scheduler-config", "GetSchedulerConfig", schedulerConfigHandler.HandleSchedulerConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerPrefix(apiRouter, "/scheduler-config", "HandleSchedulerConfig", schedulerConfigHandler.handleSchedulerConfig, setMethods(http.MethodPost, http.MethodDelete, http.MethodPut, http.MethodPatch), setAuditBackend(localLog, prometheus)) + registerPrefix(apiRouter, "/scheduler-config", "GetSchedulerConfig", schedulerConfigHandler.handleSchedulerConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) clusterHandler := newClusterHandler(svr, rd) registerFunc(apiRouter, "/cluster", clusterHandler.GetCluster, setMethods(http.MethodGet), setAuditBackend(prometheus)) @@ -177,7 +177,7 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { rulesHandler := newRulesHandler(svr, rd) ruleRouter := clusterRouter.NewRoute().Subrouter() - ruleRouter.Use(newRuleMiddleware(svr, rd).Middleware) + ruleRouter.Use(newRuleMiddleware(svr, rd).middleware) registerFunc(ruleRouter, "/config/rules", rulesHandler.GetAllRules, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(ruleRouter, "/config/rules", rulesHandler.SetAllRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) registerFunc(ruleRouter, "/config/rules/batch", rulesHandler.BatchRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) @@ -310,10 +310,10 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(clusterRouter, "/admin/storage/region/{id}", adminHandler.DeleteRegionStorage, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) registerFunc(clusterRouter, "/admin/cache/regions", adminHandler.DeleteAllRegionCache, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) registerFunc(apiRouter, "/admin/persist-file/{file_name}", adminHandler.SavePersistFile, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) - registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.IsSnapshotRecovering, setMethods(http.MethodGet), setAuditBackend(localLog, prometheus)) - registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.MarkSnapshotRecovering, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) - registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.UnmarkSnapshotRecovering, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) - registerFunc(apiRouter, "/admin/base-alloc-id", adminHandler.RecoverAllocID, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.isSnapshotRecovering, setMethods(http.MethodGet), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.markSnapshotRecovering, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/admin/cluster/markers/snapshot-recovering", adminHandler.unmarkSnapshotRecovering, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(apiRouter, "/admin/base-alloc-id", adminHandler.recoverAllocID, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) serviceMiddlewareHandler := newServiceMiddlewareHandler(svr, rd) registerFunc(apiRouter, "/service-middleware/config", serviceMiddlewareHandler.GetServiceMiddlewareConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) @@ -327,16 +327,16 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(clusterRouter, "/replication_mode/status", replicationModeHandler.GetReplicationModeStatus, setAuditBackend(prometheus)) pluginHandler := newPluginHandler(handler, rd) - registerFunc(apiRouter, "/plugin", pluginHandler.LoadPlugin, setMethods(http.MethodPost), setAuditBackend(prometheus)) - registerFunc(apiRouter, "/plugin", pluginHandler.UnloadPlugin, setMethods(http.MethodDelete), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/plugin", pluginHandler.loadPlugin, setMethods(http.MethodPost), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/plugin", pluginHandler.unloadPlugin, setMethods(http.MethodDelete), setAuditBackend(prometheus)) healthHandler := newHealthHandler(svr, rd) registerFunc(apiRouter, "/health", healthHandler.GetHealthStatus, setMethods(http.MethodGet), setAuditBackend(prometheus)) registerFunc(apiRouter, "/ping", healthHandler.Ping, setMethods(http.MethodGet), setAuditBackend(prometheus)) // metric query use to query metric data, the protocol is compatible with prometheus. - registerFunc(apiRouter, "/metric/query", newQueryMetric(svr).QueryMetric, setMethods(http.MethodGet, http.MethodPost), setAuditBackend(prometheus)) - registerFunc(apiRouter, "/metric/query_range", newQueryMetric(svr).QueryMetric, setMethods(http.MethodGet, http.MethodPost), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/metric/query", newqueryMetric(svr).queryMetric, setMethods(http.MethodGet, http.MethodPost), setAuditBackend(prometheus)) + registerFunc(apiRouter, "/metric/query_range", newqueryMetric(svr).queryMetric, setMethods(http.MethodGet, http.MethodPost), setAuditBackend(prometheus)) pprofHandler := newPprofHandler(svr, rd) // profile API diff --git a/server/api/rule.go b/server/api/rule.go index bdb3db2016d..27a95a08269 100644 --- a/server/api/rule.go +++ b/server/api/rule.go @@ -57,7 +57,7 @@ func newRuleMiddleware(s *server.Server, rd *render.Render) ruleMiddleware { } } -func (m ruleMiddleware) Middleware(h http.Handler) http.Handler { +func (m ruleMiddleware) middleware(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { manager, err := m.GetRuleManager() if err == errs.ErrPlacementDisabled { diff --git a/server/api/scheduler.go b/server/api/scheduler.go index 306f67ae058..1d502013558 100644 --- a/server/api/scheduler.go +++ b/server/api/scheduler.go @@ -259,7 +259,7 @@ func newSchedulerConfigHandler(svr *server.Server, rd *render.Render) *scheduler } } -func (h *schedulerConfigHandler) HandleSchedulerConfig(w http.ResponseWriter, r *http.Request) { +func (h *schedulerConfigHandler) handleSchedulerConfig(w http.ResponseWriter, r *http.Request) { handler := h.svr.GetHandler() sh, err := handler.GetSchedulerConfigHandler() if err == nil && sh != nil { diff --git a/server/api/server_test.go b/server/api/server_test.go index 7ebee9bcdfe..af41905ad86 100644 --- a/server/api/server_test.go +++ b/server/api/server_test.go @@ -242,7 +242,7 @@ func (suite *serviceTestSuite) TestServiceLabels() { apiutil.NewAccessPath("/pd/api/v1/leader/resign", "")) re.Equal("", serviceLabel) - accessPaths = suite.svr.GetServiceLabels("QueryMetric") + accessPaths = suite.svr.GetServiceLabels("queryMetric") re.Len(accessPaths, 4) sort.Slice(accessPaths, func(i, j int) bool { if accessPaths[i].Path == accessPaths[j].Path { @@ -260,10 +260,10 @@ func (suite *serviceTestSuite) TestServiceLabels() { re.Equal(http.MethodPost, accessPaths[3].Method) serviceLabel = suite.svr.GetAPIAccessServiceLabel( apiutil.NewAccessPath("/pd/api/v1/metric/query", http.MethodPost)) - re.Equal("QueryMetric", serviceLabel) + re.Equal("queryMetric", serviceLabel) serviceLabel = suite.svr.GetAPIAccessServiceLabel( apiutil.NewAccessPath("/pd/api/v1/metric/query", http.MethodGet)) - re.Equal("QueryMetric", serviceLabel) + re.Equal("queryMetric", serviceLabel) } func (suite *adminTestSuite) TestCleanPath() { diff --git a/server/forward.go b/server/forward.go index 5c49b871020..c407e545f6f 100644 --- a/server/forward.go +++ b/server/forward.go @@ -332,7 +332,7 @@ func forwardReportBucketClientToServer(forwardStream pdpb.PD_ReportBucketsClient errCh <- errors.WithStack(err) return } - if err := server.Send(resp); err != nil { + if err := server.send(resp); err != nil { errCh <- errors.WithStack(err) return } diff --git a/server/grpc_service.go b/server/grpc_service.go index fa9156e884e..fee5e0e3355 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -1024,7 +1024,7 @@ type bucketHeartbeatServer struct { closed int32 } -func (b *bucketHeartbeatServer) Send(bucket *pdpb.ReportBucketsResponse) error { +func (b *bucketHeartbeatServer) send(bucket *pdpb.ReportBucketsResponse) error { if atomic.LoadInt32(&b.closed) == 1 { return status.Errorf(codes.Canceled, "stream is closed") } @@ -1047,7 +1047,7 @@ func (b *bucketHeartbeatServer) Send(bucket *pdpb.ReportBucketsResponse) error { } } -func (b *bucketHeartbeatServer) Recv() (*pdpb.ReportBucketsRequest, error) { +func (b *bucketHeartbeatServer) recv() (*pdpb.ReportBucketsRequest, error) { if atomic.LoadInt32(&b.closed) == 1 { return nil, io.EOF } @@ -1083,7 +1083,7 @@ func (s *GrpcServer) ReportBuckets(stream pdpb.PD_ReportBucketsServer) error { } } for { - request, err := server.Recv() + request, err := server.recv() failpoint.Inject("grpcClientClosed", func() { err = status.Error(codes.Canceled, "grpc client closed") request = nil @@ -1132,7 +1132,7 @@ func (s *GrpcServer) ReportBuckets(stream pdpb.PD_ReportBucketsServer) error { resp := &pdpb.ReportBucketsResponse{ Header: s.notBootstrappedHeader(), } - err := server.Send(resp) + err := server.send(resp) return errors.WithStack(err) } if err := s.validateRequest(request.GetHeader()); err != nil { diff --git a/tests/cluster.go b/tests/cluster.go index c7368fe3c3a..2aa783590df 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -810,7 +810,7 @@ func (c *TestCluster) HandleReportBuckets(b *metapb.Buckets) error { // Join is used to add a new TestServer into the cluster. func (c *TestCluster) Join(ctx context.Context, opts ...ConfigOption) (*TestServer, error) { - conf, err := c.config.Join().Generate(opts...) + conf, err := c.config.join().Generate(opts...) if err != nil { return nil, err } @@ -824,7 +824,7 @@ func (c *TestCluster) Join(ctx context.Context, opts ...ConfigOption) (*TestServ // JoinAPIServer is used to add a new TestAPIServer into the cluster. func (c *TestCluster) JoinAPIServer(ctx context.Context, opts ...ConfigOption) (*TestServer, error) { - conf, err := c.config.Join().Generate(opts...) + conf, err := c.config.join().Generate(opts...) if err != nil { return nil, err } diff --git a/tests/config.go b/tests/config.go index a162a02009c..62dd5f08c1f 100644 --- a/tests/config.go +++ b/tests/config.go @@ -47,6 +47,7 @@ func newServerConfig(name string, cc *clusterConfig, join bool) *serverConfig { } } +// Generate generates a config for the server. func (c *serverConfig) Generate(opts ...ConfigOption) (*config.Config, error) { arguments := []string{ "--name=" + c.Name, @@ -57,9 +58,9 @@ func (c *serverConfig) Generate(opts ...ConfigOption) (*config.Config, error) { "--advertise-peer-urls=" + c.AdvertisePeerURLs, } if c.Join { - arguments = append(arguments, "--join="+c.ClusterConfig.GetJoinAddr()) + arguments = append(arguments, "--join="+c.ClusterConfig.getJoinAddr()) } else { - arguments = append(arguments, "--initial-cluster="+c.ClusterConfig.GetServerAddrs()) + arguments = append(arguments, "--initial-cluster="+c.ClusterConfig.getServerAddrs()) } flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) @@ -110,7 +111,7 @@ func newClusterConfig(n int) *clusterConfig { return &cc } -func (c *clusterConfig) Join() *serverConfig { +func (c *clusterConfig) join() *serverConfig { sc := newServerConfig(c.nextServerName(), c, true) c.JoinServers = append(c.JoinServers, sc) return sc @@ -120,7 +121,7 @@ func (c *clusterConfig) nextServerName() string { return fmt.Sprintf("pd%d", len(c.InitialServers)+len(c.JoinServers)+1) } -func (c *clusterConfig) GetServerAddrs() string { +func (c *clusterConfig) getServerAddrs() string { addrs := make([]string, 0, len(c.InitialServers)) for _, s := range c.InitialServers { addrs = append(addrs, fmt.Sprintf("%s=%s", s.Name, s.PeerURLs)) @@ -128,14 +129,16 @@ func (c *clusterConfig) GetServerAddrs() string { return strings.Join(addrs, ",") } -func (c *clusterConfig) GetJoinAddr() string { +func (c *clusterConfig) getJoinAddr() string { return c.InitialServers[0].PeerURLs } +// GetClientURL returns the client URL of the cluster. func (c *clusterConfig) GetClientURL() string { return c.InitialServers[0].ClientURLs } +// GetClientURLs returns all client URLs of the cluster. func (c *clusterConfig) GetClientURLs() []string { urls := make([]string, 0, len(c.InitialServers)) for _, svr := range c.InitialServers { diff --git a/tests/testutil.go b/tests/testutil.go index c895d206c05..2989ba7b010 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -50,6 +50,7 @@ import ( ) var ( + // TestDialClient is a http client for test. TestDialClient = &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, @@ -286,11 +287,15 @@ func MustReportBuckets(re *require.Assertions, cluster *TestCluster, regionID ui return buckets } +// SchedulerMode is used for test purpose. type SchedulerMode int const ( + // Both represents both PD mode and API mode. Both SchedulerMode = iota + // PDMode represents PD mode. PDMode + // APIMode represents API mode. APIMode ) From 6783814d50020270f4d95ff99616acf64c60aadd Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 9 Aug 2024 18:27:41 +0800 Subject: [PATCH 07/42] ms: make TestBootstrapDefaultKeyspaceGroup stable (#8510) close tikv/pd#8395 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/keyspace/tso_keyspace_group.go | 1 + tests/integrations/mcs/tso/server_test.go | 32 ++++++++++++----------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/pkg/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go index 63a29d91dc0..07acac58a3f 100644 --- a/pkg/keyspace/tso_keyspace_group.go +++ b/pkg/keyspace/tso_keyspace_group.go @@ -211,6 +211,7 @@ func (m *GroupManager) allocNodesToAllKeyspaceGroups(ctx context.Context) { log.Error("failed to alloc nodes for keyspace group", zap.Uint32("keyspace-group-id", group.ID), zap.Error(err)) continue } + log.Info("alloc nodes for keyspace group", zap.Uint32("keyspace-group-id", group.ID), zap.Any("nodes", nodes)) group.Members = nodes } } diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 598bfd53303..0d44e70414c 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -598,22 +598,24 @@ func (suite *CommonTestSuite) TestAdvertiseAddr() { func (suite *CommonTestSuite) TestBootstrapDefaultKeyspaceGroup() { re := suite.Require() - // check the default keyspace group + // check the default keyspace group and wait for alloc tso nodes for the default keyspace group check := func() { - resp, err := tests.TestDialClient.Get(suite.pdLeader.GetServer().GetConfig().AdvertiseClientUrls + "/pd/api/v2/tso/keyspace-groups") - re.NoError(err) - defer resp.Body.Close() - re.Equal(http.StatusOK, resp.StatusCode) - respString, err := io.ReadAll(resp.Body) - re.NoError(err) - var kgs []*endpoint.KeyspaceGroup - re.NoError(json.Unmarshal(respString, &kgs)) - re.Len(kgs, 1) - re.Equal(utils.DefaultKeyspaceGroupID, kgs[0].ID) - re.Equal(endpoint.Basic.String(), kgs[0].UserKind) - re.Empty(kgs[0].SplitState) - re.Empty(kgs[0].Members) - re.Empty(kgs[0].KeyspaceLookupTable) + testutil.Eventually(re, func() bool { + resp, err := tests.TestDialClient.Get(suite.pdLeader.GetServer().GetConfig().AdvertiseClientUrls + "/pd/api/v2/tso/keyspace-groups") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respString, err := io.ReadAll(resp.Body) + re.NoError(err) + var kgs []*endpoint.KeyspaceGroup + re.NoError(json.Unmarshal(respString, &kgs)) + re.Len(kgs, 1) + re.Equal(utils.DefaultKeyspaceGroupID, kgs[0].ID) + re.Equal(endpoint.Basic.String(), kgs[0].UserKind) + re.Empty(kgs[0].SplitState) + re.Empty(kgs[0].KeyspaceLookupTable) + return len(kgs[0].Members) == 1 + }) } check() From 6b812c6f03189d22f47c3af0e1866ea986504807 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 12 Aug 2024 11:24:01 +0800 Subject: [PATCH 08/42] *: unify the usage of independent service (#8508) ref tikv/pd#8477 Signed-off-by: Ryan Leung --- pkg/utils/apiutil/serverapi/middleware.go | 2 +- server/api/admin.go | 36 +++++++++------ server/api/config.go | 6 +-- server/cluster/cluster.go | 28 +++++++++--- server/grpc_service.go | 55 ++++++++++++----------- server/server.go | 9 ---- 6 files changed, 78 insertions(+), 58 deletions(-) diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 9af8d234b34..3b0d776dfd6 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -129,7 +129,7 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri for _, rule := range h.microserviceRedirectRules { // Now we only support checking the scheduling service whether it is independent if rule.targetServiceName == mcsutils.SchedulingServiceName { - if !h.s.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !h.s.GetRaftCluster().IsServiceIndependent(mcsutils.SchedulingServiceName) { continue } } diff --git a/server/api/admin.go b/server/api/admin.go index dfaecbec755..15d5d40412a 100644 --- a/server/api/admin.go +++ b/server/api/admin.go @@ -61,11 +61,14 @@ func (h *adminHandler) DeleteRegionCache(w http.ResponseWriter, r *http.Request) return } rc.RemoveRegionIfExist(regionID) - if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { + msg := "The region is removed from server cache." + if rc.IsServiceIndependent(utils.SchedulingServiceName) { err = h.deleteRegionCacheInSchedulingServer(regionID) + if err != nil { + msg = buildMsg(err) + } } - msg := "The region is removed from server cache." - h.rd.JSON(w, http.StatusOK, h.buildMsg(msg, err)) + h.rd.JSON(w, http.StatusOK, msg) } // @Tags admin @@ -101,11 +104,15 @@ func (h *adminHandler) DeleteRegionStorage(w http.ResponseWriter, r *http.Reques } // Remove region from cache. rc.RemoveRegionIfExist(regionID) - if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { + msg := "The region is removed from server cache and region meta storage." + if rc.IsServiceIndependent(utils.SchedulingServiceName) { err = h.deleteRegionCacheInSchedulingServer(regionID) + if err != nil { + msg = buildMsg(err) + } } - msg := "The region is removed from server cache and region meta storage." - h.rd.JSON(w, http.StatusOK, h.buildMsg(msg, err)) + + h.rd.JSON(w, http.StatusOK, msg) } // @Tags admin @@ -117,11 +124,15 @@ func (h *adminHandler) DeleteAllRegionCache(w http.ResponseWriter, r *http.Reque var err error rc := getCluster(r) rc.ResetRegionCache() - if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { + msg := "All regions are removed from server cache." + if rc.IsServiceIndependent(utils.SchedulingServiceName) { err = h.deleteRegionCacheInSchedulingServer() + if err != nil { + msg = buildMsg(err) + } } - msg := "All regions are removed from server cache." - h.rd.JSON(w, http.StatusOK, h.buildMsg(msg, err)) + + h.rd.JSON(w, http.StatusOK, msg) } // Intentionally no swagger mark as it is supposed to be only used in @@ -240,9 +251,6 @@ func (h *adminHandler) deleteRegionCacheInSchedulingServer(id ...uint64) error { return nil } -func (h *adminHandler) buildMsg(msg string, err error) string { - if h.svr.IsServiceIndependent(utils.SchedulingServiceName) && err != nil { - return fmt.Sprintf("This operation was executed in API server but needs to be re-executed on scheduling server due to the following error: %s", err.Error()) - } - return msg +func buildMsg(err error) string { + return fmt.Sprintf("This operation was executed in API server but needs to be re-executed on scheduling server due to the following error: %s", err.Error()) } diff --git a/server/api/config.go b/server/api/config.go index 9f568221d89..88cb17783bc 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -62,7 +62,7 @@ func newConfHandler(svr *server.Server, rd *render.Render) *confHandler { // @Router /config [get] func (h *confHandler) GetConfig(w http.ResponseWriter, r *http.Request) { cfg := h.svr.GetConfig() - if h.svr.IsServiceIndependent(utils.SchedulingServiceName) && + if h.svr.GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) && r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { schedulingServerConfig, err := h.getSchedulingServerConfig() if err != nil { @@ -336,7 +336,7 @@ func getConfigMap(cfg map[string]any, key []string, value any) map[string]any { // @Success 200 {object} sc.ScheduleConfig // @Router /config/schedule [get] func (h *confHandler) GetScheduleConfig(w http.ResponseWriter, r *http.Request) { - if h.svr.IsServiceIndependent(utils.SchedulingServiceName) && + if h.svr.GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) && r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { cfg, err := h.getSchedulingServerConfig() if err != nil { @@ -410,7 +410,7 @@ func (h *confHandler) SetScheduleConfig(w http.ResponseWriter, r *http.Request) // @Success 200 {object} sc.ReplicationConfig // @Router /config/replicate [get] func (h *confHandler) GetReplicationConfig(w http.ResponseWriter, r *http.Request) { - if h.svr.IsServiceIndependent(utils.SchedulingServiceName) && + if h.svr.GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) && r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { cfg, err := h.getSchedulingServerConfig() if err != nil { diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index c8013c63e2d..2150fe992ad 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -375,18 +375,18 @@ func (c *RaftCluster) checkServices() { servers, err := discovery.Discover(c.etcdClient, strconv.FormatUint(c.clusterID, 10), mcsutils.SchedulingServiceName) if c.opt.GetMicroServiceConfig().IsSchedulingFallbackEnabled() && (err != nil || len(servers) == 0) { c.startSchedulingJobs(c, c.hbstreams) - c.independentServices.Delete(mcsutils.SchedulingServiceName) + c.UnsetServiceIndependent(mcsutils.SchedulingServiceName) } else { if c.stopSchedulingJobs() || c.coordinator == nil { c.initCoordinator(c.ctx, c, c.hbstreams) } if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { - c.independentServices.Store(mcsutils.SchedulingServiceName, true) + c.SetServiceIndependent(mcsutils.SchedulingServiceName) } } } else { c.startSchedulingJobs(c, c.hbstreams) - c.independentServices.Delete(mcsutils.SchedulingServiceName) + c.UnsetServiceIndependent(mcsutils.SchedulingServiceName) } } @@ -2439,9 +2439,25 @@ func IsClientURL(addr string, etcdClient *clientv3.Client) bool { // IsServiceIndependent returns whether the service is independent. func (c *RaftCluster) IsServiceIndependent(name string) bool { - independent, exist := c.independentServices.Load(name) - if !exist { + if c == nil { return false } - return independent.(bool) + _, exist := c.independentServices.Load(name) + return exist +} + +// SetServiceIndependent sets the service to be independent. +func (c *RaftCluster) SetServiceIndependent(name string) { + if c == nil { + return + } + c.independentServices.Store(name, struct{}{}) +} + +// UnsetServiceIndependent unsets the service to be independent. +func (c *RaftCluster) UnsetServiceIndependent(name string) { + if c == nil { + return + } + c.independentServices.Delete(name) } diff --git a/server/grpc_service.go b/server/grpc_service.go index fee5e0e3355..21126addd24 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -951,7 +951,7 @@ func (s *GrpcServer) StoreHeartbeat(ctx context.Context, request *pdpb.StoreHear s.handleDamagedStore(request.GetStats()) storeHeartbeatHandleDuration.WithLabelValues(storeAddress, storeLabel).Observe(time.Since(start).Seconds()) - if s.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, _ := s.updateSchedulingClient(ctx) cli := forwardCli.getClient() if cli != nil { @@ -1307,7 +1307,7 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error regionHeartbeatHandleDuration.WithLabelValues(storeAddress, storeLabel).Observe(time.Since(start).Seconds()) regionHeartbeatCounter.WithLabelValues(storeAddress, storeLabel, "report", "ok").Inc() - if s.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(utils.SchedulingServiceName) { if forwardErrCh != nil { select { case err, ok := <-forwardErrCh: @@ -1786,7 +1786,13 @@ func (s *GrpcServer) AskBatchSplit(ctx context.Context, request *pdpb.AskBatchSp }, nil } } - if s.IsServiceIndependent(utils.SchedulingServiceName) { + + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.AskBatchSplitResponse{Header: s.notBootstrappedHeader()}, nil + } + + if rc.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.AskBatchSplitResponse{ @@ -1821,11 +1827,6 @@ func (s *GrpcServer) AskBatchSplit(ctx context.Context, request *pdpb.AskBatchSp return rsp.(*pdpb.AskBatchSplitResponse), err } - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.AskBatchSplitResponse{Header: s.notBootstrappedHeader()}, nil - } - if !versioninfo.IsFeatureSupported(rc.GetOpts().GetClusterVersion(), versioninfo.BatchSplit) { return &pdpb.AskBatchSplitResponse{Header: s.incompatibleVersion("batch_split")}, nil } @@ -2015,7 +2016,13 @@ func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterReg }, nil } } - if s.IsServiceIndependent(utils.SchedulingServiceName) { + + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.ScatterRegionResponse{Header: s.notBootstrappedHeader()}, nil + } + + if rc.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.ScatterRegionResponse{ @@ -2067,11 +2074,6 @@ func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterReg return rsp.(*pdpb.ScatterRegionResponse), err } - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.ScatterRegionResponse{Header: s.notBootstrappedHeader()}, nil - } - if len(request.GetRegionsId()) > 0 { percentage, err := scatterRegions(rc, request.GetRegionsId(), request.GetGroup(), int(request.GetRetryLimit()), request.GetSkipStoreLimit()) if err != nil { @@ -2292,7 +2294,13 @@ func (s *GrpcServer) GetOperator(ctx context.Context, request *pdpb.GetOperatorR }, nil } } - if s.IsServiceIndependent(utils.SchedulingServiceName) { + + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.GetOperatorResponse{Header: s.notBootstrappedHeader()}, nil + } + + if rc.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.GetOperatorResponse{ @@ -2327,11 +2335,6 @@ func (s *GrpcServer) GetOperator(ctx context.Context, request *pdpb.GetOperatorR return rsp.(*pdpb.GetOperatorResponse), err } - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.GetOperatorResponse{Header: s.notBootstrappedHeader()}, nil - } - opController := rc.GetOperatorController() requestID := request.GetRegionId() r := opController.GetOperatorStatus(requestID) @@ -2611,7 +2614,13 @@ func (s *GrpcServer) SplitRegions(ctx context.Context, request *pdpb.SplitRegion }, nil } } - if s.IsServiceIndependent(utils.SchedulingServiceName) { + + rc := s.GetRaftCluster() + if rc == nil { + return &pdpb.SplitRegionsResponse{Header: s.notBootstrappedHeader()}, nil + } + + if rc.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.SplitRegionsResponse{ @@ -2648,10 +2657,6 @@ func (s *GrpcServer) SplitRegions(ctx context.Context, request *pdpb.SplitRegion return rsp.(*pdpb.SplitRegionsResponse), err } - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.SplitRegionsResponse{Header: s.notBootstrappedHeader()}, nil - } finishedPercentage, newRegionIDs := rc.GetRegionSplitter().SplitRegions(ctx, request.GetSplitKeys(), int(request.GetRetryLimit())) return &pdpb.SplitRegionsResponse{ Header: s.header(), diff --git a/server/server.go b/server/server.go index c10d15e90b1..a08d31b2945 100644 --- a/server/server.go +++ b/server/server.go @@ -1465,15 +1465,6 @@ func (s *Server) GetRegions() []*core.RegionInfo { return nil } -// IsServiceIndependent returns if the service is enabled -func (s *Server) IsServiceIndependent(name string) bool { - rc := s.GetRaftCluster() - if rc != nil { - return rc.IsServiceIndependent(name) - } - return false -} - // GetServiceLabels returns ApiAccessPaths by given service label // TODO: this function will be used for updating api rate limit config func (s *Server) GetServiceLabels(serviceLabel string) []apiutil.AccessPath { From 3f32f547ff18219103efb0e36d24de4eacce6d67 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Mon, 12 Aug 2024 14:16:31 +0800 Subject: [PATCH 09/42] config: Deprecate `SchedulersPayload` (#8509) ref tikv/pd#8474 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/server.go | 8 -------- pkg/schedule/config/config.go | 4 ---- server/api/config.go | 1 - server/server.go | 10 ---------- tests/integrations/mcs/scheduling/api_test.go | 13 +------------ tools/pd-ctl/pdctl/command/config_command.go | 1 - tools/pd-ctl/tests/config/config_test.go | 1 - 7 files changed, 1 insertion(+), 37 deletions(-) diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index 50936325f45..eaf29b81fc0 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -540,14 +540,6 @@ func (s *Server) GetConfig() *config.Config { cfg.Schedule = *s.persistConfig.GetScheduleConfig().Clone() cfg.Replication = *s.persistConfig.GetReplicationConfig().Clone() cfg.ClusterVersion = *s.persistConfig.GetClusterVersion() - if s.storage == nil { - return cfg - } - sches, configs, err := s.storage.LoadAllSchedulerConfigs() - if err != nil { - return cfg - } - cfg.Schedule.SchedulersPayload = schedulers.ToPayload(sches, configs) return cfg } diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index 08bcc57ffe2..124bff0a704 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -278,9 +278,6 @@ type ScheduleConfig struct { // Schedulers support for loading customized schedulers Schedulers SchedulerConfigs `toml:"schedulers" json:"schedulers-v2"` // json v2 is for the sake of compatible upgrade - // Only used to display - SchedulersPayload map[string]any `toml:"schedulers-payload" json:"schedulers-payload"` - // Controls the time interval between write hot regions info into leveldb. HotRegionsWriteInterval typeutil.Duration `toml:"hot-regions-write-interval" json:"hot-regions-write-interval"` @@ -324,7 +321,6 @@ func (c *ScheduleConfig) Clone() *ScheduleConfig { cfg := *c cfg.StoreLimit = storeLimit cfg.Schedulers = schedulers - cfg.SchedulersPayload = nil return &cfg } diff --git a/server/api/config.go b/server/api/config.go index 88cb17783bc..c5681f9bbea 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -343,7 +343,6 @@ func (h *confHandler) GetScheduleConfig(w http.ResponseWriter, r *http.Request) h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - cfg.Schedule.SchedulersPayload = nil h.rd.JSON(w, http.StatusOK, cfg.Schedule) return } diff --git a/server/server.go b/server/server.go index a08d31b2945..1c42070cfe3 100644 --- a/server/server.go +++ b/server/server.go @@ -63,7 +63,6 @@ import ( sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/placement" - "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" @@ -982,14 +981,6 @@ func (s *Server) GetConfig() *config.Config { cfg.MicroService = *s.persistOptions.GetMicroServiceConfig().Clone() cfg.LabelProperty = s.persistOptions.GetLabelPropertyConfig().Clone() cfg.ClusterVersion = *s.persistOptions.GetClusterVersion() - if s.storage == nil { - return cfg - } - sches, configs, err := s.storage.LoadAllSchedulerConfigs() - if err != nil { - return cfg - } - cfg.Schedule.SchedulersPayload = schedulers.ToPayload(sches, configs) return cfg } @@ -1054,7 +1045,6 @@ func (s *Server) SetScheduleConfig(cfg sc.ScheduleConfig) error { return err } old := s.persistOptions.GetScheduleConfig() - cfg.SchedulersPayload = nil s.persistOptions.SetScheduleConfig(&cfg) if err := s.persistOptions.Persist(s.storage); err != nil { s.persistOptions.SetScheduleConfig(old) diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index eba94213d7e..4425369ed71 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -395,14 +395,6 @@ func (suite *apiTestSuite) checkConfig(cluster *tests.TestCluster) { re.Equal(cfg.Replication.MaxReplicas, s.GetConfig().Replication.MaxReplicas) re.Equal(cfg.Replication.LocationLabels, s.GetConfig().Replication.LocationLabels) re.Equal(cfg.DataDir, s.GetConfig().DataDir) - testutil.Eventually(re, func() bool { - // wait for all schedulers to be loaded in scheduling server. - return len(cfg.Schedule.SchedulersPayload) == 4 - }) - re.Contains(cfg.Schedule.SchedulersPayload, "balance-leader-scheduler") - re.Contains(cfg.Schedule.SchedulersPayload, "balance-region-scheduler") - re.Contains(cfg.Schedule.SchedulersPayload, "balance-hot-region-scheduler") - re.Contains(cfg.Schedule.SchedulersPayload, "evict-slow-store-scheduler") } func (suite *apiTestSuite) TestConfigForward() { @@ -423,10 +415,7 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { testutil.ReadGetJSON(re, tests.TestDialClient, urlPrefix, &cfg) re.Equal(cfg["schedule"].(map[string]any)["leader-schedule-limit"], float64(opts.GetLeaderScheduleLimit())) - re.Equal(cfg["replication"].(map[string]any)["max-replicas"], - float64(opts.GetReplicationConfig().MaxReplicas)) - schedulers := cfg["schedule"].(map[string]any)["schedulers-payload"].(map[string]any) - return len(schedulers) == 4 + return cfg["replication"].(map[string]any)["max-replicas"] == float64(opts.GetReplicationConfig().MaxReplicas) }) // Test to change config in api server diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index 0c3851350cc..6f88e06a0dd 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -255,7 +255,6 @@ func showConfigCommandFunc(cmd *cobra.Command, _ []string) { var hideConfig = []string{ "schedulers-v2", - "schedulers-payload", "store-limit", "enable-remove-down-replica", "enable-replace-offline-replica", diff --git a/tools/pd-ctl/tests/config/config_test.go b/tools/pd-ctl/tests/config/config_test.go index f3c261e1f49..0404796a81e 100644 --- a/tools/pd-ctl/tests/config/config_test.go +++ b/tools/pd-ctl/tests/config/config_test.go @@ -130,7 +130,6 @@ func (suite *configTestSuite) checkConfig(cluster *pdTests.TestCluster) { // hidden config scheduleConfig.Schedulers = nil - scheduleConfig.SchedulersPayload = nil scheduleConfig.StoreLimit = nil scheduleConfig.SchedulerMaxWaitingOperator = 0 scheduleConfig.EnableRemoveDownReplica = false From f3602e31640a55b61a3b14fb3989c4a0742a2416 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 12 Aug 2024 15:25:31 +0800 Subject: [PATCH 10/42] mcs: remove wait api service ready (#8476) ref tikv/pd#8477 Signed-off-by: Ryan Leung --- pkg/keyspace/keyspace.go | 24 ++-- pkg/keyspace/keyspace_test.go | 18 +-- pkg/keyspace/tso_keyspace_group.go | 38 +++--- pkg/keyspace/tso_keyspace_group_test.go | 14 +- pkg/keyspace/util.go | 6 +- pkg/keyspace/util_test.go | 6 +- pkg/mcs/discovery/discover.go | 6 +- pkg/mcs/discovery/key_path.go | 6 +- pkg/mcs/resourcemanager/server/config.go | 12 +- pkg/mcs/resourcemanager/server/server.go | 46 +++---- pkg/mcs/scheduling/server/config/config.go | 12 +- pkg/mcs/scheduling/server/server.go | 59 +++------ pkg/mcs/server/server.go | 11 +- pkg/mcs/tso/server/apis/v1/api.go | 3 +- pkg/mcs/tso/server/config.go | 14 +- pkg/mcs/tso/server/config_test.go | 4 +- pkg/mcs/tso/server/server.go | 61 +++------ pkg/mcs/utils/{ => constant}/constant.go | 8 +- pkg/mcs/utils/util.go | 124 ++++++++---------- pkg/member/participant.go | 8 +- pkg/schedule/hbstream/heartbeat_streams.go | 6 +- pkg/storage/endpoint/key_path.go | 30 ++--- pkg/tso/allocator_manager.go | 4 +- pkg/tso/global_allocator.go | 4 +- pkg/tso/keyspace_group_manager.go | 80 +++++------ pkg/tso/keyspace_group_manager_test.go | 86 ++++++------ pkg/utils/apiutil/serverapi/middleware.go | 6 +- pkg/utils/tsoutil/tso_request.go | 4 +- server/api/admin.go | 10 +- server/api/config.go | 10 +- server/api/server.go | 40 +++--- server/apiv2/handlers/tso_keyspace_group.go | 8 +- server/cluster/cluster.go | 32 ++--- server/cluster/cluster_worker.go | 6 +- server/forward.go | 14 +- server/grpc_service.go | 18 +-- server/handler.go | 8 +- server/server.go | 12 +- server/server_test.go | 4 +- tests/cluster.go | 4 +- tests/integrations/client/client_test.go | 4 +- tests/integrations/client/keyspace_test.go | 8 +- .../mcs/discovery/register_test.go | 17 +-- .../mcs/keyspace/tso_keyspace_group_test.go | 52 ++++---- tests/integrations/mcs/members/member_test.go | 6 +- tests/integrations/mcs/scheduling/api_test.go | 4 +- .../mcs/scheduling/server_test.go | 6 +- tests/integrations/mcs/tso/api_test.go | 8 +- .../mcs/tso/keyspace_group_manager_test.go | 48 +++---- tests/integrations/mcs/tso/server_test.go | 91 +++---------- tests/integrations/tso/client_test.go | 12 +- tests/server/apiv2/handlers/keyspace_test.go | 10 +- .../apiv2/handlers/tso_keyspace_group_test.go | 6 +- tests/testutil.go | 4 +- tests/tso_cluster.go | 6 +- tools/go.mod | 2 +- tools/pd-api-bench/config/config.go | 2 +- tools/pd-api-bench/main.go | 2 +- .../pdctl/command/keyspace_group_command.go | 4 +- .../tests/keyspace/keyspace_group_test.go | 20 +-- tools/pd-ctl/tests/keyspace/keyspace_test.go | 4 +- 61 files changed, 519 insertions(+), 663 deletions(-) rename pkg/mcs/utils/{ => constant}/constant.go (95%) diff --git a/pkg/keyspace/keyspace.go b/pkg/keyspace/keyspace.go index bcee8e75dea..08835e24baf 100644 --- a/pkg/keyspace/keyspace.go +++ b/pkg/keyspace/keyspace.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/id" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/slice" @@ -118,20 +118,20 @@ func NewKeyspaceManager( cluster: cluster, config: config, kgm: kgm, - nextPatrolStartID: utils.DefaultKeyspaceID, + nextPatrolStartID: constant.DefaultKeyspaceID, } } // Bootstrap saves default keyspace info. func (manager *Manager) Bootstrap() error { // Split Keyspace Region for default keyspace. - if err := manager.splitKeyspaceRegion(utils.DefaultKeyspaceID, false); err != nil { + if err := manager.splitKeyspaceRegion(constant.DefaultKeyspaceID, false); err != nil { return err } now := time.Now().Unix() defaultKeyspaceMeta := &keyspacepb.KeyspaceMeta{ - Id: utils.DefaultKeyspaceID, - Name: utils.DefaultKeyspaceName, + Id: constant.DefaultKeyspaceID, + Name: constant.DefaultKeyspaceName, State: keyspacepb.KeyspaceState_ENABLED, CreatedAt: now, StateChangedAt: now, @@ -543,7 +543,7 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) // It returns error if saving failed, operation not allowed, or if keyspace not exists. func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.KeyspaceState, now int64) (*keyspacepb.KeyspaceMeta, error) { // Changing the state of default keyspace is not allowed. - if name == utils.DefaultKeyspaceName { + if name == constant.DefaultKeyspaceName { log.Warn("[keyspace] failed to update keyspace config", zap.Error(ErrModifyDefaultKeyspace), ) @@ -595,7 +595,7 @@ func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.Key // It returns error if saving failed, operation not allowed, or if keyspace not exists. func (manager *Manager) UpdateKeyspaceStateByID(id uint32, newState keyspacepb.KeyspaceState, now int64) (*keyspacepb.KeyspaceMeta, error) { // Changing the state of default keyspace is not allowed. - if id == utils.DefaultKeyspaceID { + if id == constant.DefaultKeyspaceID { log.Warn("[keyspace] failed to update keyspace config", zap.Error(ErrModifyDefaultKeyspace), ) @@ -724,18 +724,18 @@ func (manager *Manager) PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID var defaultKeyspaceGroup *endpoint.KeyspaceGroup err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error { var err error - defaultKeyspaceGroup, err = manager.kgm.store.LoadKeyspaceGroup(txn, utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroup, err = manager.kgm.store.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) if err != nil { return err } if defaultKeyspaceGroup == nil { - return errors.Errorf("default keyspace group %d not found", utils.DefaultKeyspaceGroupID) + return errors.Errorf("default keyspace group %d not found", constant.DefaultKeyspaceGroupID) } if defaultKeyspaceGroup.IsSplitting() { - return ErrKeyspaceGroupInSplit(utils.DefaultKeyspaceGroupID) + return ErrKeyspaceGroupInSplit(constant.DefaultKeyspaceGroupID) } if defaultKeyspaceGroup.IsMerging() { - return ErrKeyspaceGroupInMerging(utils.DefaultKeyspaceGroupID) + return ErrKeyspaceGroupInMerging(constant.DefaultKeyspaceGroupID) } keyspaces, err := manager.store.LoadRangeKeyspace(txn, manager.nextPatrolStartID, etcdutil.MaxEtcdTxnOps) if err != nil { @@ -784,7 +784,7 @@ func (manager *Manager) PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID // Only save the keyspace group meta if any keyspace is assigned to it. assigned = true } - ks.Config[TSOKeyspaceGroupIDKey] = strconv.FormatUint(uint64(utils.DefaultKeyspaceGroupID), 10) + ks.Config[TSOKeyspaceGroupIDKey] = strconv.FormatUint(uint64(constant.DefaultKeyspaceGroupID), 10) err = manager.store.SaveKeyspaceMeta(txn, ks) if err != nil { log.Error("[keyspace] failed to save keyspace meta during patrol", diff --git a/pkg/keyspace/keyspace_test.go b/pkg/keyspace/keyspace_test.go index 4bd8dfd5474..b322def6bad 100644 --- a/pkg/keyspace/keyspace_test.go +++ b/pkg/keyspace/keyspace_test.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" @@ -187,7 +187,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceConfig() { re.Error(err) } // Changing config of DEFAULT keyspace is allowed. - updated, err := manager.UpdateKeyspaceConfig(utils.DefaultKeyspaceName, mutations) + updated, err := manager.UpdateKeyspaceConfig(constant.DefaultKeyspaceName, mutations) re.NoError(err) // remove auto filled fields delete(updated.Config, TSOKeyspaceGroupIDKey) @@ -227,7 +227,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ENABLED, newTime) re.Error(err) // Changing state of DEFAULT keyspace is not allowed. - _, err = manager.UpdateKeyspaceState(utils.DefaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, newTime) + _, err = manager.UpdateKeyspaceState(constant.DefaultKeyspaceName, keyspacepb.KeyspaceState_DISABLED, newTime) re.Error(err) } } @@ -392,7 +392,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignment() { }) re.NoError(err) // Check if the keyspace is not attached to the default group. - defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(constant.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(defaultKeyspaceGroup) re.NotContains(defaultKeyspaceGroup.Keyspaces, uint32(111)) @@ -400,7 +400,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignment() { err = suite.manager.PatrolKeyspaceAssignment(0, 0) re.NoError(err) // Check if the keyspace is attached to the default group. - defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(constant.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(defaultKeyspaceGroup) re.Contains(defaultKeyspaceGroup.Keyspaces, uint32(111)) @@ -421,7 +421,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentInBatch() { re.NoError(err) } // Check if all the keyspaces are not attached to the default group. - defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(constant.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(defaultKeyspaceGroup) for i := 1; i < etcdutil.MaxEtcdTxnOps*2+1; i++ { @@ -431,7 +431,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentInBatch() { err = suite.manager.PatrolKeyspaceAssignment(0, 0) re.NoError(err) // Check if all the keyspaces are attached to the default group. - defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(constant.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(defaultKeyspaceGroup) for i := 1; i < etcdutil.MaxEtcdTxnOps*2+1; i++ { @@ -454,7 +454,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentWithRange() { re.NoError(err) } // Check if all the keyspaces are not attached to the default group. - defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(constant.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(defaultKeyspaceGroup) for i := 1; i < etcdutil.MaxEtcdTxnOps*2+1; i++ { @@ -469,7 +469,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentWithRange() { err = suite.manager.PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID) re.NoError(err) // Check if only the keyspaces within the range are attached to the default group. - defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(constant.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(defaultKeyspaceGroup) for i := 1; i < etcdutil.MaxEtcdTxnOps*2+1; i++ { diff --git a/pkg/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go index 07acac58a3f..ffab89f4f0e 100644 --- a/pkg/keyspace/tso_keyspace_group.go +++ b/pkg/keyspace/tso_keyspace_group.go @@ -29,7 +29,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/balancer" "github.com/tikv/pd/pkg/mcs/discovery" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" @@ -90,7 +90,7 @@ func NewKeyspaceGroupManager( ctx, cancel := context.WithCancel(ctx) groups := make(map[endpoint.UserKind]*indexedHeap) for i := 0; i < int(endpoint.UserKindCount); i++ { - groups[endpoint.UserKind(i)] = newIndexedHeap(int(utils.MaxKeyspaceGroupCountInUse)) + groups[endpoint.UserKind(i)] = newIndexedHeap(int(constant.MaxKeyspaceGroupCountInUse)) } m := &GroupManager{ ctx: ctx, @@ -119,9 +119,9 @@ func (m *GroupManager) Bootstrap(ctx context.Context) error { // leave the replica/member list empty. The TSO service will assign the default keyspace group replica // to every tso node/pod by default. defaultKeyspaceGroup := &endpoint.KeyspaceGroup{ - ID: utils.DefaultKeyspaceGroupID, + ID: constant.DefaultKeyspaceGroupID, UserKind: endpoint.Basic.String(), - Keyspaces: []uint32{utils.DefaultKeyspaceID}, + Keyspaces: []uint32{constant.DefaultKeyspaceID}, } m.Lock() @@ -134,7 +134,7 @@ func (m *GroupManager) Bootstrap(ctx context.Context) error { } // Load all the keyspace groups from the storage and add to the respective userKind groups. - groups, err := m.store.LoadKeyspaceGroups(utils.DefaultKeyspaceGroupID, 0) + groups, err := m.store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) if err != nil { return err } @@ -185,7 +185,7 @@ func (m *GroupManager) allocNodesToAllKeyspaceGroups(ctx context.Context) { continue } } - groups, err := m.store.LoadKeyspaceGroups(utils.DefaultKeyspaceGroupID, 0) + groups, err := m.store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) if err != nil { log.Error("failed to load all keyspace groups", zap.Error(err)) continue @@ -205,8 +205,8 @@ func (m *GroupManager) allocNodesToAllKeyspaceGroups(ctx context.Context) { if numExistMembers != 0 && numExistMembers == len(group.Members) && numExistMembers == m.GetNodesCount() { continue } - if numExistMembers < utils.DefaultKeyspaceGroupReplicaCount { - nodes, err := m.AllocNodesForKeyspaceGroup(group.ID, existMembers, utils.DefaultKeyspaceGroupReplicaCount) + if numExistMembers < constant.DefaultKeyspaceGroupReplicaCount { + nodes, err := m.AllocNodesForKeyspaceGroup(group.ID, existMembers, constant.DefaultKeyspaceGroupReplicaCount) if err != nil { log.Error("failed to alloc nodes for keyspace group", zap.Uint32("keyspace-group-id", group.ID), zap.Error(err)) continue @@ -430,7 +430,7 @@ func (m *GroupManager) UpdateKeyspaceForGroup(userKind endpoint.UserKind, groupI failpoint.Inject("externalAllocNode", func(val failpoint.Value) { failpointOnce.Do(func() { addrs := val.(string) - _ = m.SetNodesForKeyspaceGroup(utils.DefaultKeyspaceGroupID, strings.Split(addrs, ",")) + _ = m.SetNodesForKeyspaceGroup(constant.DefaultKeyspaceGroupID, strings.Split(addrs, ",")) }) }) m.Lock() @@ -574,7 +574,7 @@ func (m *GroupManager) SplitKeyspaceGroupByID( return err } // Check if the source keyspace group has enough replicas. - if len(splitSourceKg.Members) < utils.DefaultKeyspaceGroupReplicaCount { + if len(splitSourceKg.Members) < constant.DefaultKeyspaceGroupReplicaCount { return ErrKeyspaceGroupNotEnoughReplicas } // Check if the new keyspace group already exists. @@ -634,7 +634,7 @@ func buildSplitKeyspaces( oldKeyspaceMap[keyspace] = struct{}{} } for _, keyspace := range new { - if keyspace == utils.DefaultKeyspaceID { + if keyspace == constant.DefaultKeyspaceID { return nil, nil, ErrModifyDefaultKeyspace } if _, ok := oldKeyspaceMap[keyspace]; !ok { @@ -670,7 +670,7 @@ func buildSplitKeyspaces( newKeyspaceMap = make(map[uint32]struct{}, newNum) ) for _, keyspace := range old { - if keyspace == utils.DefaultKeyspaceID { + if keyspace == constant.DefaultKeyspaceID { // The source keyspace group must be the default keyspace group and we always keep the default // keyspace in the default keyspace group. continue @@ -778,7 +778,7 @@ func (m *GroupManager) AllocNodesForKeyspaceGroup(id uint32, existMembers map[st for addr := range existMembers { nodes = append(nodes, endpoint.KeyspaceGroupMember{ Address: addr, - Priority: utils.DefaultKeyspaceGroupReplicaPriority, + Priority: constant.DefaultKeyspaceGroupReplicaPriority, }) } @@ -804,7 +804,7 @@ func (m *GroupManager) AllocNodesForKeyspaceGroup(id uint32, existMembers map[st existMembers[addr] = struct{}{} nodes = append(nodes, endpoint.KeyspaceGroupMember{ Address: addr, - Priority: utils.DefaultKeyspaceGroupReplicaPriority, + Priority: constant.DefaultKeyspaceGroupReplicaPriority, }) } kg.Members = nodes @@ -844,7 +844,7 @@ func (m *GroupManager) SetNodesForKeyspaceGroup(id uint32, nodes []string) error for _, node := range nodes { members = append(members, endpoint.KeyspaceGroupMember{ Address: node, - Priority: utils.DefaultKeyspaceGroupReplicaPriority, + Priority: constant.DefaultKeyspaceGroupReplicaPriority, }) } kg.Members = members @@ -923,7 +923,7 @@ func (m *GroupManager) MergeKeyspaceGroups(mergeTargetID uint32, mergeList []uin if (mergeListNum+1)*2 > etcdutil.MaxEtcdTxnOps { return ErrExceedMaxEtcdTxnOps } - if slice.Contains(mergeList, utils.DefaultKeyspaceGroupID) { + if slice.Contains(mergeList, constant.DefaultKeyspaceGroupID) { return ErrModifyDefaultKeyspaceGroup } var ( @@ -1071,7 +1071,7 @@ func (m *GroupManager) MergeAllIntoDefaultKeyspaceGroup() error { groupsToMerge = make([]uint32, 0, maxBatchSize) ) for idx, group := range groups.GetAll() { - if group.ID == utils.DefaultKeyspaceGroupID { + if group.ID == constant.DefaultKeyspaceGroupID { continue } groupsToMerge = append(groupsToMerge, group.ID) @@ -1085,7 +1085,7 @@ func (m *GroupManager) MergeAllIntoDefaultKeyspaceGroup() error { zap.Int("merged-group-num", mergedGroupNum), zap.Int("unmerged-group-num", unmergedGroupNum)) // Reach the batch size, merge them into the default keyspace group. - if err := m.MergeKeyspaceGroups(utils.DefaultKeyspaceGroupID, groupsToMerge); err != nil { + if err := m.MergeKeyspaceGroups(constant.DefaultKeyspaceGroupID, groupsToMerge); err != nil { log.Error("failed to merge all keyspace groups into the default one", zap.Int("index", idx), zap.Int("batch-size", len(groupsToMerge)), @@ -1112,7 +1112,7 @@ func (m *GroupManager) MergeAllIntoDefaultKeyspaceGroup() error { ticker.Stop() return nil case <-ticker.C: - kg, err := m.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + kg, err := m.GetKeyspaceGroupByID(constant.DefaultKeyspaceGroupID) if err != nil { log.Error("failed to check the default keyspace group merge state", zap.Int("index", idx), diff --git a/pkg/keyspace/tso_keyspace_group_test.go b/pkg/keyspace/tso_keyspace_group_test.go index b5df85c56a8..4dcb85b2939 100644 --- a/pkg/keyspace/tso_keyspace_group_test.go +++ b/pkg/keyspace/tso_keyspace_group_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/mock/mockconfig" "github.com/tikv/pd/pkg/mock/mockid" @@ -87,7 +87,7 @@ func (suite *keyspaceGroupTestSuite) TestKeyspaceGroupOperations() { re.NoError(err) re.Len(kgs, 2) // get the default keyspace group - kg, err := suite.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) + kg, err := suite.kgm.GetKeyspaceGroupByID(constant.DefaultKeyspaceGroupID) re.NoError(err) re.Equal(uint32(0), kg.ID) re.Equal(endpoint.Basic.String(), kg.UserKind) @@ -248,13 +248,13 @@ func (suite *keyspaceGroupTestSuite) TestKeyspaceGroupSplit() { ID: uint32(2), UserKind: endpoint.Standard.String(), Keyspaces: []uint32{111, 222, 333}, - Members: make([]endpoint.KeyspaceGroupMember, utils.DefaultKeyspaceGroupReplicaCount), + Members: make([]endpoint.KeyspaceGroupMember, constant.DefaultKeyspaceGroupReplicaCount), }, } err := suite.kgm.CreateKeyspaceGroups(keyspaceGroups) re.NoError(err) // split the default keyspace - err = suite.kgm.SplitKeyspaceGroupByID(0, 4, []uint32{utils.DefaultKeyspaceID}) + err = suite.kgm.SplitKeyspaceGroupByID(0, 4, []uint32{constant.DefaultKeyspaceID}) re.ErrorIs(err, ErrModifyDefaultKeyspace) // split the keyspace group 1 to 4 err = suite.kgm.SplitKeyspaceGroupByID(1, 4, []uint32{444}) @@ -341,7 +341,7 @@ func (suite *keyspaceGroupTestSuite) TestKeyspaceGroupSplitRange() { ID: uint32(2), UserKind: endpoint.Standard.String(), Keyspaces: []uint32{111, 333, 444, 555, 666}, - Members: make([]endpoint.KeyspaceGroupMember, utils.DefaultKeyspaceGroupReplicaCount), + Members: make([]endpoint.KeyspaceGroupMember, constant.DefaultKeyspaceGroupReplicaCount), }, } err := suite.kgm.CreateKeyspaceGroups(keyspaceGroups) @@ -388,7 +388,7 @@ func (suite *keyspaceGroupTestSuite) TestKeyspaceGroupMerge() { ID: uint32(1), UserKind: endpoint.Basic.String(), Keyspaces: []uint32{111, 222, 333}, - Members: make([]endpoint.KeyspaceGroupMember, utils.DefaultKeyspaceGroupReplicaCount), + Members: make([]endpoint.KeyspaceGroupMember, constant.DefaultKeyspaceGroupReplicaCount), }, { ID: uint32(3), @@ -453,7 +453,7 @@ func (suite *keyspaceGroupTestSuite) TestKeyspaceGroupMerge() { err = suite.kgm.MergeKeyspaceGroups(1, make([]uint32, etcdutil.MaxEtcdTxnOps/2)) re.ErrorIs(err, ErrExceedMaxEtcdTxnOps) // merge the default keyspace group - err = suite.kgm.MergeKeyspaceGroups(1, []uint32{utils.DefaultKeyspaceGroupID}) + err = suite.kgm.MergeKeyspaceGroups(1, []uint32{constant.DefaultKeyspaceGroupID}) re.ErrorIs(err, ErrModifyDefaultKeyspaceGroup) } diff --git a/pkg/keyspace/util.go b/pkg/keyspace/util.go index a3d9f6345e3..ac7d7b20398 100644 --- a/pkg/keyspace/util.go +++ b/pkg/keyspace/util.go @@ -24,7 +24,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/tikv/pd/pkg/codec" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/storage/endpoint" ) @@ -110,7 +110,7 @@ func validateID(id uint32) error { if id > spaceIDMax { return errors.Errorf("illegal keyspace id %d, larger than spaceID Max %d", id, spaceIDMax) } - if id == utils.DefaultKeyspaceID { + if id == constant.DefaultKeyspaceID { return errors.Errorf("illegal keyspace id %d, collides with default keyspace id", id) } return nil @@ -127,7 +127,7 @@ func validateName(name string) error { if !isValid { return errors.Errorf("illegal keyspace name %s, should contain only alphanumerical and underline", name) } - if name == utils.DefaultKeyspaceName { + if name == constant.DefaultKeyspaceName { return errors.Errorf("illegal keyspace name %s, collides with default keyspace name", name) } return nil diff --git a/pkg/keyspace/util_test.go b/pkg/keyspace/util_test.go index 48500fcd535..ab544b21a5d 100644 --- a/pkg/keyspace/util_test.go +++ b/pkg/keyspace/util_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/codec" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule/labeler" ) @@ -31,7 +31,7 @@ func TestValidateID(t *testing.T) { id uint32 hasErr bool }{ - {utils.DefaultKeyspaceID, true}, // Reserved id should result in error. + {constant.DefaultKeyspaceID, true}, // Reserved id should result in error. {100, false}, {spaceIDMax - 1, false}, {spaceIDMax, false}, @@ -49,7 +49,7 @@ func TestValidateName(t *testing.T) { name string hasErr bool }{ - {utils.DefaultKeyspaceName, true}, // Reserved name should result in error. + {constant.DefaultKeyspaceName, true}, // Reserved name should result in error. {"keyspaceName1", false}, {"keyspace_name_1", false}, {"10", false}, diff --git a/pkg/mcs/discovery/discover.go b/pkg/mcs/discovery/discover.go index 3e1d678cffb..c219cbc047f 100644 --- a/pkg/mcs/discovery/discover.go +++ b/pkg/mcs/discovery/discover.go @@ -20,7 +20,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" "go.etcd.io/etcd/clientv3" @@ -47,8 +47,8 @@ func Discover(cli *clientv3.Client, clusterID, serviceName string) ([]string, er // GetMSMembers returns all the members of the specified service name. func GetMSMembers(serviceName string, client *clientv3.Client) ([]ServiceRegistryEntry, error) { switch serviceName { - case utils.TSOServiceName, utils.SchedulingServiceName, utils.ResourceManagerServiceName: - clusterID, err := etcdutil.GetClusterID(client, utils.ClusterIDPath) + case constant.TSOServiceName, constant.SchedulingServiceName, constant.ResourceManagerServiceName: + clusterID, err := etcdutil.GetClusterID(client, constant.ClusterIDPath) if err != nil { return nil, err } diff --git a/pkg/mcs/discovery/key_path.go b/pkg/mcs/discovery/key_path.go index b7bf9d1cac3..76ca387d4b1 100644 --- a/pkg/mcs/discovery/key_path.go +++ b/pkg/mcs/discovery/key_path.go @@ -18,7 +18,7 @@ import ( "strconv" "strings" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" ) const ( @@ -27,12 +27,12 @@ const ( // RegistryPath returns the full path to store microservice addresses. func RegistryPath(clusterID, serviceName, serviceAddr string) string { - return strings.Join([]string{utils.MicroserviceRootPath, clusterID, serviceName, registryKey, serviceAddr}, "/") + return strings.Join([]string{constant.MicroserviceRootPath, clusterID, serviceName, registryKey, serviceAddr}, "/") } // ServicePath returns the path to store microservice addresses. func ServicePath(clusterID, serviceName string) string { - return strings.Join([]string{utils.MicroserviceRootPath, clusterID, serviceName, registryKey, ""}, "/") + return strings.Join([]string{constant.MicroserviceRootPath, clusterID, serviceName, registryKey, ""}, "/") } // TSOPath returns the path to store TSO addresses. diff --git a/pkg/mcs/resourcemanager/server/config.go b/pkg/mcs/resourcemanager/server/config.go index 03fc6718926..9a899c9dc07 100644 --- a/pkg/mcs/resourcemanager/server/config.go +++ b/pkg/mcs/resourcemanager/server/config.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/spf13/pflag" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/metricutil" @@ -242,7 +242,7 @@ func (c *Config) Adjust(meta *toml.MetaData) error { configutil.AdjustString(&c.AdvertiseListenAddr, c.ListenAddr) if !configMetaData.IsDefined("enable-grpc-gateway") { - c.EnableGRPCGateway = utils.DefaultEnableGRPCGateway + c.EnableGRPCGateway = constant.DefaultEnableGRPCGateway } c.adjustLog(configMetaData.Child("log")) @@ -251,17 +251,17 @@ func (c *Config) Adjust(meta *toml.MetaData) error { } c.Controller.Adjust(configMetaData.Child("controller")) - configutil.AdjustInt64(&c.LeaderLease, utils.DefaultLeaderLease) + configutil.AdjustInt64(&c.LeaderLease, constant.DefaultLeaderLease) return nil } func (c *Config) adjustLog(meta *configutil.ConfigMetaData) { if !meta.IsDefined("disable-error-verbose") { - c.Log.DisableErrorVerbose = utils.DefaultDisableErrorVerbose + c.Log.DisableErrorVerbose = constant.DefaultDisableErrorVerbose } - configutil.AdjustString(&c.Log.Format, utils.DefaultLogFormat) - configutil.AdjustString(&c.Log.Level, utils.DefaultLogLevel) + configutil.AdjustString(&c.Log.Format, constant.DefaultLogFormat) + configutil.AdjustString(&c.Log.Level, constant.DefaultLogLevel) } // GetName returns the Name diff --git a/pkg/mcs/resourcemanager/server/server.go b/pkg/mcs/resourcemanager/server/server.go index 19317d8202a..c8ccfab0333 100644 --- a/pkg/mcs/resourcemanager/server/server.go +++ b/pkg/mcs/resourcemanager/server/server.go @@ -20,7 +20,6 @@ import ( "os" "os/signal" "runtime" - "strconv" "sync" "sync/atomic" "syscall" @@ -28,7 +27,6 @@ import ( grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/diagnosticspb" "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/log" @@ -39,6 +37,7 @@ import ( "github.com/tikv/pd/pkg/mcs/discovery" "github.com/tikv/pd/pkg/mcs/server" "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" @@ -77,6 +76,8 @@ type Server struct { // primaryCallbacks will be called after the server becomes leader. primaryCallbacks []func(context.Context) error + // for service registry + serviceID *discovery.ServiceRegistryEntry serviceRegister *discovery.ServiceRegister } @@ -90,6 +91,11 @@ func (s *Server) GetAddr() string { return s.cfg.ListenAddr } +// GetAdvertiseListenAddr returns the advertise address of the server. +func (s *Server) GetAdvertiseListenAddr() string { + return s.cfg.AdvertiseListenAddr +} + // SetLogLevel sets log level. func (s *Server) SetLogLevel(level string) error { if !logutil.IsLevelLegal(level) { @@ -103,18 +109,14 @@ func (s *Server) SetLogLevel(level string) error { // Run runs the Resource Manager server. func (s *Server) Run() (err error) { - skipWaitAPIServiceReady := false - failpoint.Inject("skipWaitAPIServiceReady", func() { - skipWaitAPIServiceReady = true - }) - if !skipWaitAPIServiceReady { - if err := utils.WaitAPIServiceReady(s); err != nil { - return err - } + if err = utils.InitClient(s); err != nil { + return err } - if err := utils.InitClient(s); err != nil { + + if s.clusterID, s.serviceID, s.serviceRegister, err = utils.Register(s, constant.ResourceManagerServiceName); err != nil { return err } + return s.startServer() } @@ -189,7 +191,7 @@ func (s *Server) campaignLeader() { member.ServiceMemberGauge.WithLabelValues(serviceName).Set(1) log.Info("resource manager primary is ready to serve", zap.String("resource-manager-primary-name", s.participant.Name())) - leaderTicker := time.NewTicker(utils.LeaderTickInterval) + leaderTicker := time.NewTicker(constant.LeaderTickInterval) defer leaderTicker.Stop() for { @@ -294,10 +296,6 @@ func (s *Server) GetLeaderListenUrls() []string { } func (s *Server) startServer() (err error) { - if s.clusterID, err = utils.InitClusterID(s.Context(), s.GetClient()); err != nil { - return err - } - log.Info("init cluster id", zap.Uint64("cluster-id", s.clusterID)) // The independent Resource Manager service still reuses PD version info since PD and Resource Manager are just // different service modes provided by the same pd-server binary bs.ServerInfoGauge.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) @@ -306,13 +304,13 @@ func (s *Server) startServer() (err error) { uniqueName := s.cfg.GetAdvertiseListenAddr() uniqueID := memberutil.GenerateUniqueID(uniqueName) log.Info("joining primary election", zap.String("participant-name", uniqueName), zap.Uint64("participant-id", uniqueID)) - s.participant = member.NewParticipant(s.GetClient(), utils.ResourceManagerServiceName) + s.participant = member.NewParticipant(s.GetClient(), constant.ResourceManagerServiceName) p := &resource_manager.Participant{ Name: uniqueName, Id: uniqueID, // id is unique among all participants ListenUrls: []string{s.cfg.GetAdvertiseListenAddr()}, } - s.participant.InitInfo(p, endpoint.ResourceManagerSvcRootPath(s.clusterID), utils.PrimaryKey, "primary election") + s.participant.InitInfo(p, endpoint.ResourceManagerSvcRootPath(s.clusterID), constant.PrimaryKey, "primary election") s.service = &Service{ ctx: s.Context(), @@ -338,18 +336,6 @@ func (s *Server) startServer() (err error) { // We should make sure the calling sequence is right. s.startServerLoop() - // Server has started. - entry := &discovery.ServiceRegistryEntry{ServiceAddr: s.cfg.AdvertiseListenAddr, Name: s.Name()} - serializedEntry, err := entry.Serialize() - if err != nil { - return err - } - s.serviceRegister = discovery.NewServiceRegister(s.Context(), s.GetClient(), strconv.FormatUint(s.clusterID, 10), - utils.ResourceManagerServiceName, s.cfg.AdvertiseListenAddr, serializedEntry, discovery.DefaultLeaseInSeconds) - if err := s.serviceRegister.Register(); err != nil { - log.Error("failed to register the service", zap.String("service-name", utils.ResourceManagerServiceName), errs.ZapError(err)) - return err - } atomic.StoreInt64(&s.isRunning, 1) return nil } diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index c1fcad33ace..4b855d09899 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -34,7 +34,7 @@ import ( "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/core/storelimit" - "github.com/tikv/pd/pkg/mcs/utils" + mcsconstant "github.com/tikv/pd/pkg/mcs/utils/constant" sc "github.com/tikv/pd/pkg/schedule/config" types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/slice" @@ -144,7 +144,7 @@ func (c *Config) adjust(meta *toml.MetaData) error { configutil.AdjustString(&c.AdvertiseListenAddr, c.ListenAddr) if !configMetaData.IsDefined("enable-grpc-gateway") { - c.EnableGRPCGateway = utils.DefaultEnableGRPCGateway + c.EnableGRPCGateway = mcsconstant.DefaultEnableGRPCGateway } c.adjustLog(configMetaData.Child("log")) @@ -152,7 +152,7 @@ func (c *Config) adjust(meta *toml.MetaData) error { return err } - configutil.AdjustInt64(&c.LeaderLease, utils.DefaultLeaderLease) + configutil.AdjustInt64(&c.LeaderLease, mcsconstant.DefaultLeaderLease) if err := c.Schedule.Adjust(configMetaData.Child("schedule"), false); err != nil { return err @@ -162,10 +162,10 @@ func (c *Config) adjust(meta *toml.MetaData) error { func (c *Config) adjustLog(meta *configutil.ConfigMetaData) { if !meta.IsDefined("disable-error-verbose") { - c.Log.DisableErrorVerbose = utils.DefaultDisableErrorVerbose + c.Log.DisableErrorVerbose = mcsconstant.DefaultDisableErrorVerbose } - configutil.AdjustString(&c.Log.Format, utils.DefaultLogFormat) - configutil.AdjustString(&c.Log.Level, utils.DefaultLogLevel) + configutil.AdjustString(&c.Log.Format, mcsconstant.DefaultLogFormat) + configutil.AdjustString(&c.Log.Level, mcsconstant.DefaultLogLevel) } // GetName returns the Name diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index eaf29b81fc0..6ec4e444d6d 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -20,9 +20,7 @@ import ( "net/http" "os" "os/signal" - "path/filepath" "runtime" - "strconv" "sync" "sync/atomic" "syscall" @@ -47,6 +45,7 @@ import ( "github.com/tikv/pd/pkg/mcs/scheduling/server/rule" "github.com/tikv/pd/pkg/mcs/server" "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/schedule" sc "github.com/tikv/pd/pkg/schedule/config" @@ -123,6 +122,11 @@ func (s *Server) GetAddr() string { return s.cfg.ListenAddr } +// GetAdvertiseListenAddr returns the advertise address of the server. +func (s *Server) GetAdvertiseListenAddr() string { + return s.cfg.AdvertiseListenAddr +} + // GetBackendEndpoints returns the backend endpoints. func (s *Server) GetBackendEndpoints() string { return s.cfg.BackendEndpoints @@ -140,20 +144,15 @@ func (s *Server) SetLogLevel(level string) error { } // Run runs the scheduling server. -func (s *Server) Run() error { - skipWaitAPIServiceReady := false - failpoint.Inject("skipWaitAPIServiceReady", func() { - skipWaitAPIServiceReady = true - }) - if !skipWaitAPIServiceReady { - if err := utils.WaitAPIServiceReady(s); err != nil { - return err - } +func (s *Server) Run() (err error) { + if err = utils.InitClient(s); err != nil { + return err } - if err := utils.InitClient(s); err != nil { + if s.clusterID, s.serviceID, s.serviceRegister, err = utils.Register(s, constant.SchedulingServiceName); err != nil { return err } + return s.startServer() } @@ -290,7 +289,7 @@ func (s *Server) campaignLeader() { member.ServiceMemberGauge.WithLabelValues(serviceName).Set(1) log.Info("scheduling primary is ready to serve", zap.String("scheduling-primary-name", s.participant.Name())) - leaderTicker := time.NewTicker(utils.LeaderTickInterval) + leaderTicker := time.NewTicker(constant.LeaderTickInterval) defer leaderTicker.Stop() for { @@ -408,37 +407,20 @@ func (s *Server) GetLeaderListenUrls() []string { } func (s *Server) startServer() (err error) { - if s.clusterID, err = utils.InitClusterID(s.Context(), s.GetClient()); err != nil { - return err - } - log.Info("init cluster id", zap.Uint64("cluster-id", s.clusterID)) // The independent Scheduling service still reuses PD version info since PD and Scheduling are just // different service modes provided by the same pd-server binary bs.ServerInfoGauge.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) bs.ServerMaxProcsGauge.Set(float64(runtime.GOMAXPROCS(0))) - execPath, err := os.Executable() - deployPath := filepath.Dir(execPath) - if err != nil { - deployPath = "" - } - s.serviceID = &discovery.ServiceRegistryEntry{ - ServiceAddr: s.cfg.AdvertiseListenAddr, - Version: versioninfo.PDReleaseVersion, - GitHash: versioninfo.PDGitHash, - DeployPath: deployPath, - StartTimestamp: s.StartTimestamp(), - Name: s.Name(), - } uniqueName := s.cfg.GetAdvertiseListenAddr() uniqueID := memberutil.GenerateUniqueID(uniqueName) log.Info("joining primary election", zap.String("participant-name", uniqueName), zap.Uint64("participant-id", uniqueID)) - s.participant = member.NewParticipant(s.GetClient(), utils.SchedulingServiceName) + s.participant = member.NewParticipant(s.GetClient(), constant.SchedulingServiceName) p := &schedulingpb.Participant{ Name: uniqueName, Id: uniqueID, // id is unique among all participants ListenUrls: []string{s.cfg.GetAdvertiseListenAddr()}, } - s.participant.InitInfo(p, endpoint.SchedulingSvcRootPath(s.clusterID), utils.PrimaryKey, "primary election") + s.participant.InitInfo(p, endpoint.SchedulingSvcRootPath(s.clusterID), constant.PrimaryKey, "primary election") s.service = &Service{Server: s} s.AddServiceReadyCallback(s.startCluster) @@ -461,17 +443,6 @@ func (s *Server) startServer() (err error) { cb() } - // Server has started. - serializedEntry, err := s.serviceID.Serialize() - if err != nil { - return err - } - s.serviceRegister = discovery.NewServiceRegister(s.Context(), s.GetClient(), strconv.FormatUint(s.clusterID, 10), - utils.SchedulingServiceName, s.cfg.GetAdvertiseListenAddr(), serializedEntry, discovery.DefaultLeaseInSeconds) - if err := s.serviceRegister.Register(); err != nil { - log.Error("failed to register the service", zap.String("service-name", utils.SchedulingServiceName), errs.ZapError(err)) - return err - } atomic.StoreInt64(&s.isRunning, 1) return nil } @@ -483,7 +454,7 @@ func (s *Server) startCluster(context.Context) error { if err != nil { return err } - s.hbStreams = hbstream.NewHeartbeatStreams(s.Context(), s.clusterID, utils.SchedulingServiceName, s.basicCluster) + s.hbStreams = hbstream.NewHeartbeatStreams(s.Context(), s.clusterID, constant.SchedulingServiceName, s.basicCluster) s.cluster, err = NewCluster(s.Context(), s.persistConfig, s.storage, s.basicCluster, s.hbStreams, s.clusterID, s.checkMembershipCh) if err != nil { return err diff --git a/pkg/mcs/server/server.go b/pkg/mcs/server/server.go index d30d3337a46..31b6c2c76cb 100644 --- a/pkg/mcs/server/server.go +++ b/pkg/mcs/server/server.go @@ -24,7 +24,7 @@ import ( "time" "github.com/pingcap/log" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/grpcutil" "go.etcd.io/etcd/clientv3" "google.golang.org/grpc" @@ -100,6 +100,11 @@ func (bs *BaseServer) SetEtcdClient(etcdClient *clientv3.Client) { bs.etcdClient = etcdClient } +// GetEtcdClient returns the etcd client. +func (bs *BaseServer) GetEtcdClient() *clientv3.Client { + return bs.etcdClient +} + // SetHTTPClient sets the http client. func (bs *BaseServer) SetHTTPClient(httpClient *http.Client) { bs.httpClient = httpClient @@ -147,9 +152,9 @@ func (bs *BaseServer) InitListener(tlsCfg *grpcutil.TLSConfig, listenAddr string } if tlsConfig != nil { bs.secure = true - bs.muxListener, err = tls.Listen(utils.TCPNetworkStr, listenURL.Host, tlsConfig) + bs.muxListener, err = tls.Listen(constant.TCPNetworkStr, listenURL.Host, tlsConfig) } else { - bs.muxListener, err = net.Listen(utils.TCPNetworkStr, listenURL.Host) + bs.muxListener, err = net.Listen(constant.TCPNetworkStr, listenURL.Host) } return err } diff --git a/pkg/mcs/tso/server/apis/v1/api.go b/pkg/mcs/tso/server/apis/v1/api.go index 44f4b353d58..d5c4cd4ec48 100644 --- a/pkg/mcs/tso/server/apis/v1/api.go +++ b/pkg/mcs/tso/server/apis/v1/api.go @@ -28,6 +28,7 @@ import ( "github.com/tikv/pd/pkg/errs" tsoserver "github.com/tikv/pd/pkg/mcs/tso/server" "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/apiutil/multiservicesapi" @@ -211,7 +212,7 @@ func ResetTS(c *gin.Context) { // GetHealth returns the health status of the TSO service. func GetHealth(c *gin.Context) { svr := c.MustGet(multiservicesapi.ServiceContextKey).(*tsoserver.Service) - am, err := svr.GetKeyspaceGroupManager().GetAllocatorManager(utils.DefaultKeyspaceGroupID) + am, err := svr.GetKeyspaceGroupManager().GetAllocatorManager(constant.DefaultKeyspaceGroupID) if err != nil { c.String(http.StatusInternalServerError, err.Error()) return diff --git a/pkg/mcs/tso/server/config.go b/pkg/mcs/tso/server/config.go index 82ac777ad06..8a3fe1ca161 100644 --- a/pkg/mcs/tso/server/config.go +++ b/pkg/mcs/tso/server/config.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/spf13/pflag" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/pkg/utils/grpcutil" @@ -41,7 +41,7 @@ const ( defaultBackendEndpoints = "http://127.0.0.1:2379" defaultListenAddr = "http://127.0.0.1:3379" - defaultTSOSaveInterval = time.Duration(utils.DefaultLeaderLease) * time.Second + defaultTSOSaveInterval = time.Duration(constant.DefaultLeaderLease) * time.Second defaultTSOUpdatePhysicalInterval = 50 * time.Millisecond maxTSOUpdatePhysicalInterval = 10 * time.Second minTSOUpdatePhysicalInterval = 1 * time.Millisecond @@ -206,7 +206,7 @@ func (c *Config) Adjust(meta *toml.MetaData) error { configutil.AdjustString(&c.AdvertiseListenAddr, c.ListenAddr) configutil.AdjustDuration(&c.MaxResetTSGap, defaultMaxResetTSGap) - configutil.AdjustInt64(&c.LeaderLease, utils.DefaultLeaderLease) + configutil.AdjustInt64(&c.LeaderLease, constant.DefaultLeaderLease) configutil.AdjustDuration(&c.TSOSaveInterval, defaultTSOSaveInterval) configutil.AdjustDuration(&c.TSOUpdatePhysicalInterval, defaultTSOUpdatePhysicalInterval) @@ -221,7 +221,7 @@ func (c *Config) Adjust(meta *toml.MetaData) error { } if !configMetaData.IsDefined("enable-grpc-gateway") { - c.EnableGRPCGateway = utils.DefaultEnableGRPCGateway + c.EnableGRPCGateway = constant.DefaultEnableGRPCGateway } c.adjustLog(configMetaData.Child("log")) @@ -230,10 +230,10 @@ func (c *Config) Adjust(meta *toml.MetaData) error { func (c *Config) adjustLog(meta *configutil.ConfigMetaData) { if !meta.IsDefined("disable-error-verbose") { - c.Log.DisableErrorVerbose = utils.DefaultDisableErrorVerbose + c.Log.DisableErrorVerbose = constant.DefaultDisableErrorVerbose } - configutil.AdjustString(&c.Log.Format, utils.DefaultLogFormat) - configutil.AdjustString(&c.Log.Level, utils.DefaultLogLevel) + configutil.AdjustString(&c.Log.Format, constant.DefaultLogFormat) + configutil.AdjustString(&c.Log.Level, constant.DefaultLogLevel) } // Validate is used to validate if some configurations are right. diff --git a/pkg/mcs/tso/server/config_test.go b/pkg/mcs/tso/server/config_test.go index 2cb9c8e019a..2bd27a67492 100644 --- a/pkg/mcs/tso/server/config_test.go +++ b/pkg/mcs/tso/server/config_test.go @@ -21,7 +21,7 @@ import ( "github.com/BurntSushi/toml" "github.com/stretchr/testify/require" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" ) func TestConfigBasic(t *testing.T) { @@ -35,7 +35,7 @@ func TestConfigBasic(t *testing.T) { re.True(strings.HasPrefix(cfg.GetName(), defaultName)) re.Equal(defaultBackendEndpoints, cfg.BackendEndpoints) re.Equal(defaultListenAddr, cfg.ListenAddr) - re.Equal(utils.DefaultLeaderLease, cfg.LeaderLease) + re.Equal(constant.DefaultLeaderLease, cfg.LeaderLease) re.False(cfg.EnableLocalTSO) re.True(cfg.EnableGRPCGateway) re.Equal(defaultTSOSaveInterval, cfg.TSOSaveInterval.Duration) diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index a120cbc9868..0134a65dfd0 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -20,9 +20,7 @@ import ( "net/http" "os" "os/signal" - "path/filepath" "runtime" - "strconv" "sync" "sync/atomic" "syscall" @@ -30,7 +28,6 @@ import ( grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/diagnosticspb" "github.com/pingcap/kvproto/pkg/tsopb" "github.com/pingcap/log" @@ -41,6 +38,7 @@ import ( "github.com/tikv/pd/pkg/mcs/discovery" "github.com/tikv/pd/pkg/mcs/server" "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/systimemon" @@ -106,6 +104,11 @@ func (s *Server) GetAddr() string { return s.cfg.ListenAddr } +// GetAdvertiseListenAddr returns the advertise address of the server. +func (s *Server) GetAdvertiseListenAddr() string { + return s.cfg.AdvertiseListenAddr +} + // GetBackendEndpoints returns the backend endpoints. func (s *Server) GetBackendEndpoints() string { return s.cfg.BackendEndpoints @@ -143,24 +146,20 @@ func (s *Server) SetLogLevel(level string) error { } // Run runs the TSO server. -func (s *Server) Run() error { - skipWaitAPIServiceReady := false - failpoint.Inject("skipWaitAPIServiceReady", func() { - skipWaitAPIServiceReady = true - }) - if !skipWaitAPIServiceReady { - if err := utils.WaitAPIServiceReady(s); err != nil { - return err - } - } +func (s *Server) Run() (err error) { go systimemon.StartMonitor(s.Context(), time.Now, func() { log.Error("system time jumps backward", errs.ZapError(errs.ErrIncorrectSystemTime)) timeJumpBackCounter.Inc() }) - if err := utils.InitClient(s); err != nil { + if err = utils.InitClient(s); err != nil { return err } + + if s.clusterID, s.serviceID, s.serviceRegister, err = utils.Register(s, constant.TSOServiceName); err != nil { + return err + } + return s.startServer() } @@ -199,7 +198,7 @@ func (s *Server) Close() { // IsServing implements basicserver. It returns whether the server is the leader // if there is embedded etcd, or the primary otherwise. func (s *Server) IsServing() bool { - return s.IsKeyspaceServing(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) + return s.IsKeyspaceServing(constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) } // IsKeyspaceServing returns whether the server is the primary of the given keyspace. @@ -222,7 +221,7 @@ func (s *Server) IsKeyspaceServing(keyspaceID, keyspaceGroupID uint32) bool { // The entry at the index 0 is the primary's service endpoint. func (s *Server) GetLeaderListenUrls() []string { member, err := s.keyspaceGroupManager.GetElectionMember( - utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) + constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) if err != nil { log.Error("failed to get election member", errs.ZapError(err)) return nil @@ -355,11 +354,6 @@ func (s *Server) GetTLSConfig() *grpcutil.TLSConfig { } func (s *Server) startServer() (err error) { - if s.clusterID, err = utils.InitClusterID(s.Context(), s.GetClient()); err != nil { - return err - } - log.Info("init cluster id", zap.Uint64("cluster-id", s.clusterID)) - // It may lose accuracy if use float64 to store uint64. So we store the cluster id in label. metaDataGauge.WithLabelValues(fmt.Sprintf("cluster%d", s.clusterID)).Set(0) // The independent TSO service still reuses PD version info since PD and TSO are just @@ -371,19 +365,6 @@ func (s *Server) startServer() (err error) { s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(s.Context()) legacySvcRootPath := endpoint.LegacyRootPath(s.clusterID) tsoSvcRootPath := endpoint.TSOSvcRootPath(s.clusterID) - execPath, err := os.Executable() - deployPath := filepath.Dir(execPath) - if err != nil { - deployPath = "" - } - s.serviceID = &discovery.ServiceRegistryEntry{ - ServiceAddr: s.cfg.AdvertiseListenAddr, - Version: versioninfo.PDReleaseVersion, - GitHash: versioninfo.PDGitHash, - DeployPath: deployPath, - StartTimestamp: s.StartTimestamp(), - Name: s.Name(), - } s.keyspaceGroupManager = tso.NewKeyspaceGroupManager( s.serverLoopCtx, s.serviceID, s.GetClient(), s.GetHTTPClient(), s.cfg.AdvertiseListenAddr, s.clusterID, legacySvcRootPath, tsoSvcRootPath, s.cfg) @@ -410,18 +391,6 @@ func (s *Server) startServer() (err error) { cb() } - // Server has started. - serializedEntry, err := s.serviceID.Serialize() - if err != nil { - return err - } - s.serviceRegister = discovery.NewServiceRegister(s.Context(), s.GetClient(), strconv.FormatUint(s.clusterID, 10), - utils.TSOServiceName, s.cfg.AdvertiseListenAddr, serializedEntry, discovery.DefaultLeaseInSeconds) - if err := s.serviceRegister.Register(); err != nil { - log.Error("failed to register the service", zap.String("service-name", utils.TSOServiceName), errs.ZapError(err)) - return err - } - atomic.StoreInt64(&s.isRunning, 1) return nil } diff --git a/pkg/mcs/utils/constant.go b/pkg/mcs/utils/constant/constant.go similarity index 95% rename from pkg/mcs/utils/constant.go rename to pkg/mcs/utils/constant/constant.go index c6c882f5179..cd01c94f3e0 100644 --- a/pkg/mcs/utils/constant.go +++ b/pkg/mcs/utils/constant/constant.go @@ -12,14 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package utils +package constant import "time" const ( - // RetryIntervalWaitAPIService is the interval to retry. + // ClusterIDPath is the path to store cluster id + ClusterIDPath = "/pd/cluster_id" + // RetryInterval is the interval to retry. // Note: the interval must be less than the timeout of tidb and tikv, which is 2s by default in tikv. - RetryIntervalWaitAPIService = 500 * time.Millisecond + RetryInterval = 500 * time.Millisecond // TCPNetworkStr is the string of tcp network TCPNetworkStr = "tcp" diff --git a/pkg/mcs/utils/util.go b/pkg/mcs/utils/util.go index 55e6c54e9b9..fa76aadc1e8 100644 --- a/pkg/mcs/utils/util.go +++ b/pkg/mcs/utils/util.go @@ -19,19 +19,21 @@ import ( "net" "net/http" "os" + "path/filepath" + "strconv" "strings" "sync" "time" "github.com/gin-gonic/gin" - "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/diagnosticspb" - "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/soheilhy/cmux" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/discovery" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/apiutil/multiservicesapi" "github.com/tikv/pd/pkg/utils/etcdutil" @@ -45,30 +47,26 @@ import ( "google.golang.org/grpc/keepalive" ) -const ( - // maxRetryTimes is the max retry times for initializing the cluster ID. - maxRetryTimes = 5 - // ClusterIDPath is the path to store cluster id - ClusterIDPath = "/pd/cluster_id" - // retryInterval is the interval to retry. - retryInterval = time.Second -) - // InitClusterID initializes the cluster ID. func InitClusterID(ctx context.Context, client *clientv3.Client) (id uint64, err error) { - ticker := time.NewTicker(retryInterval) + ticker := time.NewTicker(constant.RetryInterval) defer ticker.Stop() - for i := 0; i < maxRetryTimes; i++ { - if clusterID, err := etcdutil.GetClusterID(client, ClusterIDPath); err == nil && clusterID != 0 { + retryTimes := 0 + for { + if clusterID, err := etcdutil.GetClusterID(client, constant.ClusterIDPath); err == nil && clusterID != 0 { return clusterID, nil } select { case <-ctx.Done(): return 0, err case <-ticker.C: + retryTimes++ + if retryTimes/500 > 0 { + log.Warn("etcd is not ready, retrying", errs.ZapError(err)) + retryTimes /= 500 + } } } - return 0, errors.Errorf("failed to init cluster ID after retrying %d times", maxRetryTimes) } // PromHandler is a handler to get prometheus metrics. @@ -95,6 +93,7 @@ func StatusHandler(c *gin.Context) { } type server interface { + GetAdvertiseListenAddr() string GetBackendEndpoints() string Context() context.Context GetTLSConfig() *grpcutil.TLSConfig @@ -107,6 +106,7 @@ type server interface { GetGRPCServer() *grpc.Server SetGRPCServer(*grpc.Server) SetHTTPServer(*http.Server) + GetEtcdClient() *clientv3.Client SetEtcdClient(*clientv3.Client) SetHTTPClient(*http.Client) IsSecure() bool @@ -114,59 +114,7 @@ type server interface { SetUpRestHandler() (http.Handler, apiutil.APIServiceGroup) diagnosticspb.DiagnosticsServer StartTimestamp() int64 -} - -// WaitAPIServiceReady waits for the api service ready. -func WaitAPIServiceReady(s server) error { - var ( - ready bool - err error - ) - ticker := time.NewTicker(RetryIntervalWaitAPIService) - defer ticker.Stop() - retryTimes := 0 - for { - ready, err = isAPIServiceReady(s) - if err == nil && ready { - return nil - } - select { - case <-s.Context().Done(): - return errors.New("context canceled while waiting api server ready") - case <-ticker.C: - retryTimes++ - if retryTimes/500 > 0 { - log.Warn("api server is not ready, retrying", errs.ZapError(err)) - retryTimes /= 500 - } - } - } -} - -func isAPIServiceReady(s server) (bool, error) { - urls := strings.Split(s.GetBackendEndpoints(), ",") - if len(urls) == 0 { - return false, errors.New("no backend endpoints") - } - cc, err := s.GetDelegateClient(s.Context(), s.GetTLSConfig(), urls[0]) - if err != nil { - return false, err - } - clusterInfo, err := pdpb.NewPDClient(cc).GetClusterInfo(s.Context(), &pdpb.GetClusterInfoRequest{}) - if err != nil { - return false, err - } - if clusterInfo.GetHeader().GetError() != nil { - return false, errors.Errorf(clusterInfo.GetHeader().GetError().String()) - } - modes := clusterInfo.ServiceModes - if len(modes) == 0 { - return false, errors.New("no service mode") - } - if modes[0] == pdpb.ServiceMode_API_SVC_MODE { - return true, nil - } - return false, nil + Name() string } // InitClient initializes the etcd and http clients. @@ -267,7 +215,7 @@ func StopHTTPServer(s server) { log.Info("stopping http server") defer log.Info("http server stopped") - ctx, cancel := context.WithTimeout(context.Background(), DefaultHTTPGracefulShutdownTimeout) + ctx, cancel := context.WithTimeout(context.Background(), constant.DefaultHTTPGracefulShutdownTimeout) defer cancel() // First, try to gracefully shutdown the http server @@ -305,7 +253,7 @@ func StopGRPCServer(s server) { return } - ctx, cancel := context.WithTimeout(context.Background(), DefaultGRPCGracefulStopTimeout) + ctx, cancel := context.WithTimeout(context.Background(), constant.DefaultGRPCGracefulStopTimeout) defer cancel() // First, try to gracefully shutdown the grpc server @@ -330,6 +278,42 @@ func StopGRPCServer(s server) { } } +// Register registers the service. +func Register(s server, serviceName string) (uint64, *discovery.ServiceRegistryEntry, *discovery.ServiceRegister, error) { + var ( + clusterID uint64 + err error + ) + if clusterID, err = InitClusterID(s.Context(), s.GetEtcdClient()); err != nil { + return 0, nil, nil, err + } + log.Info("init cluster id", zap.Uint64("cluster-id", clusterID)) + execPath, err := os.Executable() + deployPath := filepath.Dir(execPath) + if err != nil { + deployPath = "" + } + serviceID := &discovery.ServiceRegistryEntry{ + ServiceAddr: s.GetAdvertiseListenAddr(), + Version: versioninfo.PDReleaseVersion, + GitHash: versioninfo.PDGitHash, + DeployPath: deployPath, + StartTimestamp: s.StartTimestamp(), + Name: s.Name(), + } + serializedEntry, err := serviceID.Serialize() + if err != nil { + return 0, nil, nil, err + } + serviceRegister := discovery.NewServiceRegister(s.Context(), s.GetEtcdClient(), strconv.FormatUint(clusterID, 10), + serviceName, s.GetAdvertiseListenAddr(), serializedEntry, discovery.DefaultLeaseInSeconds) + if err := serviceRegister.Register(); err != nil { + log.Error("failed to register the service", zap.String("service-name", serviceName), errs.ZapError(err)) + return 0, nil, nil, err + } + return clusterID, serviceID, serviceRegister, nil +} + // Exit exits the program with the given code. func Exit(code int) { log.Sync() diff --git a/pkg/member/participant.go b/pkg/member/participant.go index 8a0ffadd31e..43a91195bff 100644 --- a/pkg/member/participant.go +++ b/pkg/member/participant.go @@ -28,7 +28,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/etcdutil" "go.etcd.io/etcd/clientv3" "go.uber.org/zap" @@ -377,11 +377,11 @@ func (m *Participant) SetCampaignChecker(checker leadershipCheckFunc) { // NewParticipantByService creates a new participant by service name. func NewParticipantByService(serviceName string) (p participant) { switch serviceName { - case utils.TSOServiceName: + case constant.TSOServiceName: p = &tsopb.Participant{} - case utils.SchedulingServiceName: + case constant.SchedulingServiceName: p = &schedulingpb.Participant{} - case utils.ResourceManagerServiceName: + case constant.ResourceManagerServiceName: p = &resource_manager.Participant{} } return p diff --git a/pkg/schedule/hbstream/heartbeat_streams.go b/pkg/schedule/hbstream/heartbeat_streams.go index 57a7521c0a7..d9bf3209bec 100644 --- a/pkg/schedule/hbstream/heartbeat_streams.go +++ b/pkg/schedule/hbstream/heartbeat_streams.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/logutil" "go.uber.org/zap" ) @@ -114,7 +114,7 @@ func (s *HeartbeatStreams) run() { var keepAlive core.RegionHeartbeatResponse switch s.typ { - case utils.SchedulingServiceName: + case constant.SchedulingServiceName: keepAlive = &schedulingpb.RegionHeartbeatResponse{Header: &schedulingpb.ResponseHeader{ClusterId: s.clusterID}} default: keepAlive = &pdpb.RegionHeartbeatResponse{Header: &pdpb.ResponseHeader{ClusterId: s.clusterID}} @@ -204,7 +204,7 @@ func (s *HeartbeatStreams) SendMsg(region *core.RegionInfo, op *Operation) { // TODO: use generic var resp core.RegionHeartbeatResponse switch s.typ { - case utils.SchedulingServiceName: + case constant.SchedulingServiceName: resp = &schedulingpb.RegionHeartbeatResponse{ Header: &schedulingpb.ResponseHeader{ClusterId: s.clusterID}, RegionId: region.GetID(), diff --git a/pkg/storage/endpoint/key_path.go b/pkg/storage/endpoint/key_path.go index dbcd9690419..8cc8f172e84 100644 --- a/pkg/storage/endpoint/key_path.go +++ b/pkg/storage/endpoint/key_path.go @@ -21,7 +21,7 @@ import ( "strconv" "strings" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" ) const ( @@ -55,12 +55,12 @@ const ( resourceGroupStatesPath = "states" controllerConfigPath = "controller" // tso storage endpoint has prefix `tso` - tsoServiceKey = utils.TSOServiceName + tsoServiceKey = constant.TSOServiceName globalTSOAllocatorEtcdPrefix = "gta" // TimestampKey is the key of timestamp oracle used for the suffix. TimestampKey = "timestamp" - tsoKeyspaceGroupPrefix = tsoServiceKey + "/" + utils.KeyspaceGroupsKey + tsoKeyspaceGroupPrefix = tsoServiceKey + "/" + constant.KeyspaceGroupsKey keyspaceGroupsMembershipKey = "membership" keyspaceGroupsElectionKey = "election" @@ -296,24 +296,24 @@ func GetCompiledKeyspaceGroupIDRegexp() *regexp.Regexp { // ResourceManagerSvcRootPath returns the root path of resource manager service. // Path: /ms/{cluster_id}/resource_manager func ResourceManagerSvcRootPath(clusterID uint64) string { - return svcRootPath(clusterID, utils.ResourceManagerServiceName) + return svcRootPath(clusterID, constant.ResourceManagerServiceName) } // SchedulingSvcRootPath returns the root path of scheduling service. // Path: /ms/{cluster_id}/scheduling func SchedulingSvcRootPath(clusterID uint64) string { - return svcRootPath(clusterID, utils.SchedulingServiceName) + return svcRootPath(clusterID, constant.SchedulingServiceName) } // TSOSvcRootPath returns the root path of tso service. // Path: /ms/{cluster_id}/tso func TSOSvcRootPath(clusterID uint64) string { - return svcRootPath(clusterID, utils.TSOServiceName) + return svcRootPath(clusterID, constant.TSOServiceName) } func svcRootPath(clusterID uint64, svcName string) string { c := strconv.FormatUint(clusterID, 10) - return path.Join(utils.MicroserviceRootPath, c, svcName) + return path.Join(constant.MicroserviceRootPath, c, svcName) } // LegacyRootPath returns the root path of legacy pd service. @@ -327,29 +327,29 @@ func LegacyRootPath(clusterID uint64) string { // non-default keyspace group: "/ms/{cluster_id}/tso/keyspace_groups/election/{group}/primary". func KeyspaceGroupPrimaryPath(rootPath string, keyspaceGroupID uint32) string { electionPath := KeyspaceGroupsElectionPath(rootPath, keyspaceGroupID) - return path.Join(electionPath, utils.PrimaryKey) + return path.Join(electionPath, constant.PrimaryKey) } // SchedulingPrimaryPath returns the path of scheduling primary. // Path: /ms/{cluster_id}/scheduling/primary func SchedulingPrimaryPath(clusterID uint64) string { - return path.Join(SchedulingSvcRootPath(clusterID), utils.PrimaryKey) + return path.Join(SchedulingSvcRootPath(clusterID), constant.PrimaryKey) } // KeyspaceGroupsElectionPath returns the path of keyspace groups election. // default keyspace group: "/ms/{cluster_id}/tso/00000". // non-default keyspace group: "/ms/{cluster_id}/tso/keyspace_groups/election/{group}". func KeyspaceGroupsElectionPath(rootPath string, keyspaceGroupID uint32) string { - if keyspaceGroupID == utils.DefaultKeyspaceGroupID { + if keyspaceGroupID == constant.DefaultKeyspaceGroupID { return path.Join(rootPath, "00000") } - return path.Join(rootPath, utils.KeyspaceGroupsKey, keyspaceGroupsElectionKey, fmt.Sprintf("%05d", keyspaceGroupID)) + return path.Join(rootPath, constant.KeyspaceGroupsKey, keyspaceGroupsElectionKey, fmt.Sprintf("%05d", keyspaceGroupID)) } // GetCompiledNonDefaultIDRegexp returns the compiled regular expression for matching non-default keyspace group id. func GetCompiledNonDefaultIDRegexp(clusterID uint64) *regexp.Regexp { rootPath := TSOSvcRootPath(clusterID) - pattern := strings.Join([]string{rootPath, utils.KeyspaceGroupsKey, keyspaceGroupsElectionKey, `(\d{5})`, utils.PrimaryKey + `$`}, "/") + pattern := strings.Join([]string{rootPath, constant.KeyspaceGroupsKey, keyspaceGroupsElectionKey, `(\d{5})`, constant.PrimaryKey + `$`}, "/") return regexp.MustCompile(pattern) } @@ -378,7 +378,7 @@ func buildPath(withSuffix bool, str ...string) string { // 2. for the non-default keyspace groups: // {group}/gta in /ms/{cluster_id}/tso/{group}/gta/timestamp func KeyspaceGroupGlobalTSPath(groupID uint32) string { - if groupID == utils.DefaultKeyspaceGroupID { + if groupID == constant.DefaultKeyspaceGroupID { return "" } return path.Join(fmt.Sprintf("%05d", groupID), globalTSOAllocatorEtcdPrefix) @@ -390,7 +390,7 @@ func KeyspaceGroupGlobalTSPath(groupID uint32) string { // 2. for the non-default keyspace groups: // {group}/lta/{dc-location} in /ms/{cluster_id}/tso/{group}/lta/{dc-location}/timestamp func KeyspaceGroupLocalTSPath(keyPrefix string, groupID uint32, dcLocation string) string { - if groupID == utils.DefaultKeyspaceGroupID { + if groupID == constant.DefaultKeyspaceGroupID { return path.Join(keyPrefix, dcLocation) } return path.Join(fmt.Sprintf("%05d", groupID), keyPrefix, dcLocation) @@ -409,7 +409,7 @@ func TimestampPath(tsPath string) string { func FullTimestampPath(clusterID uint64, groupID uint32) string { rootPath := TSOSvcRootPath(clusterID) tsPath := TimestampPath(KeyspaceGroupGlobalTSPath(groupID)) - if groupID == utils.DefaultKeyspaceGroupID { + if groupID == constant.DefaultKeyspaceGroupID { rootPath = LegacyRootPath(clusterID) } return path.Join(rootPath, tsPath) diff --git a/pkg/tso/allocator_manager.go b/pkg/tso/allocator_manager.go index 902a43541d6..9cd5ae9c743 100644 --- a/pkg/tso/allocator_manager.go +++ b/pkg/tso/allocator_manager.go @@ -31,7 +31,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/errs" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" @@ -683,7 +683,7 @@ func (am *AllocatorManager) campaignAllocatorLeader( } logger.Info("local tso allocator leader is ready to serve") - leaderTicker := time.NewTicker(mcsutils.LeaderTickInterval) + leaderTicker := time.NewTicker(constant.LeaderTickInterval) defer leaderTicker.Stop() for { diff --git a/pkg/tso/global_allocator.go b/pkg/tso/global_allocator.go index cf82d58f884..b745544ad05 100644 --- a/pkg/tso/global_allocator.go +++ b/pkg/tso/global_allocator.go @@ -28,7 +28,7 @@ import ( "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" "github.com/tikv/pd/pkg/errs" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" @@ -635,7 +635,7 @@ func (gta *GlobalTSOAllocator) campaignLeader() { logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), zap.String("tso-primary-name", gta.member.Name())) - leaderTicker := time.NewTicker(mcsutils.LeaderTickInterval) + leaderTicker := time.NewTicker(constant.LeaderTickInterval) defer leaderTicker.Stop() for { diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index 83a1369d2f2..1a24e571688 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -33,7 +33,7 @@ import ( "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/mcs/discovery" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" @@ -67,9 +67,9 @@ type state struct { // assigned with an allocator manager managing its global/local tso allocators. // Use a fixed size array to maximize the efficiency of concurrent access to // different keyspace groups for tso service. - ams [mcsutils.MaxKeyspaceGroupCountInUse]*AllocatorManager + ams [constant.MaxKeyspaceGroupCountInUse]*AllocatorManager // kgs stores the keyspace groups' membership/distribution meta. - kgs [mcsutils.MaxKeyspaceGroupCountInUse]*endpoint.KeyspaceGroup + kgs [constant.MaxKeyspaceGroupCountInUse]*endpoint.KeyspaceGroup // keyspaceLookupTable is a map from keyspace to the keyspace group to which it belongs. keyspaceLookupTable map[uint32]uint32 // splittingGroups is the cache of splitting keyspace group related information. @@ -256,19 +256,19 @@ func (s *state) getKeyspaceGroupMetaWithCheck( // The keyspace doesn't belong to any keyspace group but the keyspace has been assigned to a // keyspace group before, which means the keyspace group hasn't initialized yet. - if keyspaceGroupID != mcsutils.DefaultKeyspaceGroupID { + if keyspaceGroupID != constant.DefaultKeyspaceGroupID { return nil, nil, keyspaceGroupID, errs.ErrKeyspaceNotAssigned.FastGenByArgs(keyspaceID) } // For migrating the existing keyspaces which have no keyspace group assigned as configured // in the keyspace meta. All these keyspaces will be served by the default keyspace group. - if s.ams[mcsutils.DefaultKeyspaceGroupID] == nil { - return nil, nil, mcsutils.DefaultKeyspaceGroupID, + if s.ams[constant.DefaultKeyspaceGroupID] == nil { + return nil, nil, constant.DefaultKeyspaceGroupID, errs.ErrKeyspaceNotAssigned.FastGenByArgs(keyspaceID) } - return s.ams[mcsutils.DefaultKeyspaceGroupID], - s.kgs[mcsutils.DefaultKeyspaceGroupID], - mcsutils.DefaultKeyspaceGroupID, nil + return s.ams[constant.DefaultKeyspaceGroupID], + s.kgs[constant.DefaultKeyspaceGroupID], + constant.DefaultKeyspaceGroupID, nil } func (s *state) getNextPrimaryToReset( @@ -278,7 +278,7 @@ func (s *state) getNextPrimaryToReset( defer s.RUnlock() // Both s.ams and s.kgs are arrays with the fixed size defined by the const value MaxKeyspaceGroupCountInUse. - groupSize := int(mcsutils.MaxKeyspaceGroupCountInUse) + groupSize := int(constant.MaxKeyspaceGroupCountInUse) groupID %= groupSize for j := 0; j < groupSize; groupID, j = (groupID+1)%groupSize, j+1 { am := s.ams[groupID] @@ -403,10 +403,10 @@ func NewKeyspaceGroupManager( tsoSvcRootPath string, cfg ServiceConfig, ) *KeyspaceGroupManager { - if mcsutils.MaxKeyspaceGroupCountInUse > mcsutils.MaxKeyspaceGroupCount { + if constant.MaxKeyspaceGroupCountInUse > constant.MaxKeyspaceGroupCount { log.Fatal("MaxKeyspaceGroupCountInUse is larger than MaxKeyspaceGroupCount", - zap.Uint32("max-keyspace-group-count-in-use", mcsutils.MaxKeyspaceGroupCountInUse), - zap.Uint32("max-keyspace-group-count", mcsutils.MaxKeyspaceGroupCount)) + zap.Uint32("max-keyspace-group-count-in-use", constant.MaxKeyspaceGroupCountInUse), + zap.Uint32("max-keyspace-group-count", constant.MaxKeyspaceGroupCount)) } ctx, cancel := context.WithCancel(ctx) @@ -539,7 +539,7 @@ func (kgm *KeyspaceGroupManager) InitializeGroupWatchLoop() error { return errs.ErrJSONUnmarshal.Wrap(err) } kgm.updateKeyspaceGroup(group) - if group.ID == mcsutils.DefaultKeyspaceGroupID { + if group.ID == constant.DefaultKeyspaceGroupID { defaultKGConfigured = true } return nil @@ -589,12 +589,12 @@ func (kgm *KeyspaceGroupManager) InitializeGroupWatchLoop() error { if !defaultKGConfigured { log.Info("initializing default keyspace group") group := &endpoint.KeyspaceGroup{ - ID: mcsutils.DefaultKeyspaceGroupID, + ID: constant.DefaultKeyspaceGroupID, Members: []endpoint.KeyspaceGroupMember{{ Address: kgm.tsoServiceID.ServiceAddr, - Priority: mcsutils.DefaultKeyspaceGroupReplicaPriority, + Priority: constant.DefaultKeyspaceGroupReplicaPriority, }}, - Keyspaces: []uint32{mcsutils.DefaultKeyspaceID}, + Keyspaces: []uint32{constant.DefaultKeyspaceID}, } kgm.updateKeyspaceGroup(group) } @@ -680,11 +680,11 @@ func (kgm *KeyspaceGroupManager) updateKeyspaceGroup(group *endpoint.KeyspaceGro } // If the default keyspace group isn't assigned to any tso node/pod, assign it to everyone. - if group.ID == mcsutils.DefaultKeyspaceGroupID && len(group.Members) == 0 { + if group.ID == constant.DefaultKeyspaceGroupID && len(group.Members) == 0 { // TODO: fill members with all tso nodes/pods. group.Members = []endpoint.KeyspaceGroupMember{{ Address: kgm.tsoServiceID.ServiceAddr, - Priority: mcsutils.DefaultKeyspaceGroupReplicaPriority, + Priority: constant.DefaultKeyspaceGroupReplicaPriority, }} } @@ -730,13 +730,13 @@ func (kgm *KeyspaceGroupManager) updateKeyspaceGroup(group *endpoint.KeyspaceGro zap.String("participant-name", uniqueName), zap.Uint64("participant-id", uniqueID)) // Initialize the participant info to join the primary election. - participant := member.NewParticipant(kgm.etcdClient, mcsutils.TSOServiceName) + participant := member.NewParticipant(kgm.etcdClient, constant.TSOServiceName) p := &tsopb.Participant{ Name: uniqueName, Id: uniqueID, // id is unique among all participants ListenUrls: []string{kgm.cfg.GetAdvertiseListenAddr()}, } - participant.InitInfo(p, endpoint.KeyspaceGroupsElectionPath(kgm.tsoSvcRootPath, group.ID), mcsutils.PrimaryKey, "keyspace group primary election") + participant.InitInfo(p, endpoint.KeyspaceGroupsElectionPath(kgm.tsoSvcRootPath, group.ID), constant.PrimaryKey, "keyspace group primary election") // If the keyspace group is in split, we should ensure that the primary elected by the new keyspace group // is always on the same TSO Server node as the primary of the old keyspace group, and this constraint cannot // be broken until the entire split process is completed. @@ -760,7 +760,7 @@ func (kgm *KeyspaceGroupManager) updateKeyspaceGroup(group *endpoint.KeyspaceGro tsRootPath string storage *endpoint.StorageEndpoint ) - if group.ID == mcsutils.DefaultKeyspaceGroupID { + if group.ID == constant.DefaultKeyspaceGroupID { tsRootPath = kgm.legacySvcRootPath storage = kgm.legacySvcStorage } else { @@ -806,12 +806,12 @@ func validateSplit( // could not be modified during the split process, so we can only check the // member count of the source group here. memberCount := len(sourceGroup.Members) - if memberCount < mcsutils.DefaultKeyspaceGroupReplicaCount { + if memberCount < constant.DefaultKeyspaceGroupReplicaCount { log.Error("the split source keyspace group does not have enough members", zap.Uint32("target", targetGroup.ID), zap.Uint32("source", splitSourceID), zap.Int("member-count", memberCount), - zap.Int("replica-count", mcsutils.DefaultKeyspaceGroupReplicaCount)) + zap.Int("replica-count", constant.DefaultKeyspaceGroupReplicaCount)) return false } return true @@ -891,20 +891,20 @@ func (kgm *KeyspaceGroupManager) updateKeyspaceGroupMembership( j++ } } - if groupID == mcsutils.DefaultKeyspaceGroupID { - if _, ok := newGroup.KeyspaceLookupTable[mcsutils.DefaultKeyspaceID]; !ok { + if groupID == constant.DefaultKeyspaceGroupID { + if _, ok := newGroup.KeyspaceLookupTable[constant.DefaultKeyspaceID]; !ok { log.Warn("default keyspace is not in default keyspace group. add it back") - kgm.keyspaceLookupTable[mcsutils.DefaultKeyspaceID] = groupID - newGroup.KeyspaceLookupTable[mcsutils.DefaultKeyspaceID] = struct{}{} + kgm.keyspaceLookupTable[constant.DefaultKeyspaceID] = groupID + newGroup.KeyspaceLookupTable[constant.DefaultKeyspaceID] = struct{}{} newGroup.Keyspaces = make([]uint32, 1+len(newKeyspaces)) - newGroup.Keyspaces[0] = mcsutils.DefaultKeyspaceID + newGroup.Keyspaces[0] = constant.DefaultKeyspaceID copy(newGroup.Keyspaces[1:], newKeyspaces) } } else { - if _, ok := newGroup.KeyspaceLookupTable[mcsutils.DefaultKeyspaceID]; ok { + if _, ok := newGroup.KeyspaceLookupTable[constant.DefaultKeyspaceID]; ok { log.Warn("default keyspace is in non-default keyspace group. remove it") - kgm.keyspaceLookupTable[mcsutils.DefaultKeyspaceID] = mcsutils.DefaultKeyspaceGroupID - delete(newGroup.KeyspaceLookupTable, mcsutils.DefaultKeyspaceID) + kgm.keyspaceLookupTable[constant.DefaultKeyspaceID] = constant.DefaultKeyspaceGroupID + delete(newGroup.KeyspaceLookupTable, constant.DefaultKeyspaceID) newGroup.Keyspaces = newKeyspaces[1:] } } @@ -935,16 +935,16 @@ func (kgm *KeyspaceGroupManager) updateKeyspaceGroupMembership( func (kgm *KeyspaceGroupManager) deleteKeyspaceGroup(groupID uint32) { log.Info("delete keyspace group", zap.Uint32("keyspace-group-id", groupID)) - if groupID == mcsutils.DefaultKeyspaceGroupID { + if groupID == constant.DefaultKeyspaceGroupID { log.Info("removed default keyspace group meta config from the storage. " + "now every tso node/pod will initialize it") group := &endpoint.KeyspaceGroup{ - ID: mcsutils.DefaultKeyspaceGroupID, + ID: constant.DefaultKeyspaceGroupID, Members: []endpoint.KeyspaceGroupMember{{ Address: kgm.tsoServiceID.ServiceAddr, - Priority: mcsutils.DefaultKeyspaceGroupReplicaPriority, + Priority: constant.DefaultKeyspaceGroupReplicaPriority, }}, - Keyspaces: []uint32{mcsutils.DefaultKeyspaceID}, + Keyspaces: []uint32{constant.DefaultKeyspaceID}, } kgm.updateKeyspaceGroup(group) return @@ -1011,7 +1011,7 @@ func (kgm *KeyspaceGroupManager) FindGroupByKeyspaceID( keyspaceID uint32, ) (*AllocatorManager, *endpoint.KeyspaceGroup, uint32, error) { curAM, curKeyspaceGroup, curKeyspaceGroupID, err := - kgm.getKeyspaceGroupMetaWithCheck(keyspaceID, mcsutils.DefaultKeyspaceGroupID) + kgm.getKeyspaceGroupMetaWithCheck(keyspaceID, constant.DefaultKeyspaceGroupID) if err != nil { return nil, nil, curKeyspaceGroupID, err } @@ -1087,11 +1087,11 @@ func (kgm *KeyspaceGroupManager) HandleTSORequest( } func checkKeySpaceGroupID(id uint32) error { - if id < mcsutils.MaxKeyspaceGroupCountInUse { + if id < constant.MaxKeyspaceGroupCountInUse { return nil } return errs.ErrKeyspaceGroupIDInvalid.FastGenByArgs( - fmt.Sprintf("%d shouldn't >= %d", id, mcsutils.MaxKeyspaceGroupCountInUse)) + fmt.Sprintf("%d shouldn't >= %d", id, constant.MaxKeyspaceGroupCountInUse)) } // GetMinTS returns the minimum timestamp across all keyspace groups served by this TSO server/pod. @@ -1505,7 +1505,7 @@ func (kgm *KeyspaceGroupManager) deletedGroupCleaner() { } for _, groupID := range kgm.getDeletedGroups() { // Do not clean the default keyspace group data. - if groupID == mcsutils.DefaultKeyspaceGroupID { + if groupID == constant.DefaultKeyspaceGroupID { continue } empty = false diff --git a/pkg/tso/keyspace_group_manager_test.go b/pkg/tso/keyspace_group_manager_test.go index fc057409c2a..b891aeb246d 100644 --- a/pkg/tso/keyspace_group_manager_test.go +++ b/pkg/tso/keyspace_group_manager_test.go @@ -32,7 +32,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/mcs/discovery" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/syncutil" @@ -85,10 +85,10 @@ func (suite *keyspaceGroupManagerTestSuite) createConfig() *TestServiceConfig { BackendEndpoints: suite.backendEndpoints, ListenAddr: addr, AdvertiseListenAddr: addr, - LeaderLease: mcsutils.DefaultLeaderLease, + LeaderLease: constant.DefaultLeaderLease, LocalTSOEnabled: false, TSOUpdatePhysicalInterval: 50 * time.Millisecond, - TSOSaveInterval: time.Duration(mcsutils.DefaultLeaderLease) * time.Second, + TSOSaveInterval: time.Duration(constant.DefaultLeaderLease) * time.Second, MaxResetTSGap: time.Hour * 24, TLSConfig: nil, } @@ -131,15 +131,15 @@ func (suite *keyspaceGroupManagerTestSuite) TestDeletedGroupCleanup() { re.NotContains(mgr.deletedGroups, 1) mgr.RUnlock() // Try to delete the default keyspace group. - suite.applyEtcdEvents(re, rootPath, []*etcdEvent{generateKeyspaceGroupDeleteEvent(mcsutils.DefaultKeyspaceGroupID)}) + suite.applyEtcdEvents(re, rootPath, []*etcdEvent{generateKeyspaceGroupDeleteEvent(constant.DefaultKeyspaceGroupID)}) // Default keyspace group should NOT be deleted. mgr.RLock() - re.NotNil(mgr.ams[mcsutils.DefaultKeyspaceGroupID]) - re.NotNil(mgr.kgs[mcsutils.DefaultKeyspaceGroupID]) - re.NotContains(mgr.deletedGroups, mcsutils.DefaultKeyspaceGroupID) + re.NotNil(mgr.ams[constant.DefaultKeyspaceGroupID]) + re.NotNil(mgr.kgs[constant.DefaultKeyspaceGroupID]) + re.NotContains(mgr.deletedGroups, constant.DefaultKeyspaceGroupID) mgr.RUnlock() // Default keyspace group TSO key should NOT be deleted. - ts, err := mgr.legacySvcStorage.LoadTimestamp(endpoint.KeyspaceGroupGlobalTSPath(mcsutils.DefaultKeyspaceGroupID)) + ts, err := mgr.legacySvcStorage.LoadTimestamp(endpoint.KeyspaceGroupGlobalTSPath(constant.DefaultKeyspaceGroupID)) re.NoError(err) re.NotEmpty(ts) @@ -156,7 +156,7 @@ func (suite *keyspaceGroupManagerTestSuite) TestNewKeyspaceGroupManager() { clusterIDStr := strconv.FormatUint(clusterID, 10) legacySvcRootPath := path.Join("/pd", clusterIDStr) - tsoSvcRootPath := path.Join(mcsutils.MicroserviceRootPath, clusterIDStr, "tso") + tsoSvcRootPath := path.Join(constant.MicroserviceRootPath, clusterIDStr, "tso") electionNamePrefix := "tso-server-" + clusterIDStr kgm := NewKeyspaceGroupManager( @@ -173,14 +173,14 @@ func (suite *keyspaceGroupManagerTestSuite) TestNewKeyspaceGroupManager() { re.Equal(tsoSvcRootPath, kgm.tsoSvcRootPath) re.Equal(suite.cfg, kgm.cfg) - am, err := kgm.GetAllocatorManager(mcsutils.DefaultKeyspaceGroupID) + am, err := kgm.GetAllocatorManager(constant.DefaultKeyspaceGroupID) re.NoError(err) re.False(am.enableLocalTSO) - re.Equal(mcsutils.DefaultKeyspaceGroupID, am.kgID) - re.Equal(mcsutils.DefaultLeaderLease, am.leaderLease) + re.Equal(constant.DefaultKeyspaceGroupID, am.kgID) + re.Equal(constant.DefaultLeaderLease, am.leaderLease) re.Equal(time.Hour*24, am.maxResetTSGap()) re.Equal(legacySvcRootPath, am.rootPath) - re.Equal(time.Duration(mcsutils.DefaultLeaderLease)*time.Second, am.saveInterval) + re.Equal(time.Duration(constant.DefaultLeaderLease)*time.Second, am.saveInterval) re.Equal(time.Duration(50)*time.Millisecond, am.updatePhysicalInterval) } @@ -436,38 +436,38 @@ func (suite *keyspaceGroupManagerTestSuite) TestGetKeyspaceGroupMetaWithCheck() re.NoError(err) // Should be able to get AM for the default/null keyspace and keyspace 1, 2 in keyspace group 0. - am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(mcsutils.DefaultKeyspaceID, 0) + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(constant.DefaultKeyspaceID, 0) re.NoError(err) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.Equal(constant.DefaultKeyspaceGroupID, kgid) re.NotNil(am) re.NotNil(kg) - am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(mcsutils.NullKeyspaceID, 0) + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(constant.NullKeyspaceID, 0) re.NoError(err) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.Equal(constant.DefaultKeyspaceGroupID, kgid) re.NotNil(am) re.NotNil(kg) am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(1, 0) re.NoError(err) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.Equal(constant.DefaultKeyspaceGroupID, kgid) re.NotNil(am) re.NotNil(kg) am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(2, 0) re.NoError(err) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.Equal(constant.DefaultKeyspaceGroupID, kgid) re.NotNil(am) re.NotNil(kg) // Should still succeed even keyspace 3 isn't explicitly assigned to any // keyspace group. It will be assigned to the default keyspace group. am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(3, 0) re.NoError(err) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.Equal(constant.DefaultKeyspaceGroupID, kgid) re.NotNil(am) re.NotNil(kg) // Should succeed and get the meta of keyspace group 0, because keyspace 0 // belongs to group 0, though the specified group 1 doesn't exist. - am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(mcsutils.DefaultKeyspaceID, 1) + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(constant.DefaultKeyspaceID, 1) re.NoError(err) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.Equal(constant.DefaultKeyspaceGroupID, kgid) re.NotNil(am) re.NotNil(kg) // Should fail because keyspace 3 isn't explicitly assigned to any keyspace @@ -501,8 +501,8 @@ func (suite *keyspaceGroupManagerTestSuite) TestDefaultMembershipRestriction() { // Create keyspace group 0 which contains keyspace 0, 1, 2. addKeyspaceGroupAssignment( - suite.ctx, suite.etcdClient, mcsutils.DefaultKeyspaceGroupID, rootPath, - []string{svcAddr}, []int{0}, []uint32{mcsutils.DefaultKeyspaceID, 1, 2}) + suite.ctx, suite.etcdClient, constant.DefaultKeyspaceGroupID, rootPath, + []string{svcAddr}, []int{0}, []uint32{constant.DefaultKeyspaceID, 1, 2}) // Create keyspace group 3 which contains keyspace 3, 4. addKeyspaceGroupAssignment( suite.ctx, suite.etcdClient, uint32(3), mgr.legacySvcRootPath, @@ -513,18 +513,18 @@ func (suite *keyspaceGroupManagerTestSuite) TestDefaultMembershipRestriction() { // Should be able to get AM for keyspace 0 in keyspace group 0. am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck( - mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) re.NoError(err) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.Equal(constant.DefaultKeyspaceGroupID, kgid) re.NotNil(am) re.NotNil(kg) event = generateKeyspaceGroupPutEvent( - mcsutils.DefaultKeyspaceGroupID, []uint32{1, 2}, []string{svcAddr}) + constant.DefaultKeyspaceGroupID, []uint32{1, 2}, []string{svcAddr}) err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) re.NoError(err) event = generateKeyspaceGroupPutEvent( - 3, []uint32{mcsutils.DefaultKeyspaceID, 3, 4}, []string{svcAddr}) + 3, []uint32{constant.DefaultKeyspaceID, 3, 4}, []string{svcAddr}) err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) re.NoError(err) @@ -533,15 +533,15 @@ func (suite *keyspaceGroupManagerTestSuite) TestDefaultMembershipRestriction() { time.Sleep(1 * time.Second) // Should still be able to get AM for keyspace 0 in keyspace group 0. am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck( - mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) re.NoError(err) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.Equal(constant.DefaultKeyspaceGroupID, kgid) re.NotNil(am) re.NotNil(kg) // Should succeed and return the keyspace group meta from the default keyspace group - am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(mcsutils.DefaultKeyspaceID, 3) + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(constant.DefaultKeyspaceID, 3) re.NoError(err) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.Equal(constant.DefaultKeyspaceGroupID, kgid) re.NotNil(am) re.NotNil(kg) } @@ -574,8 +574,8 @@ func (suite *keyspaceGroupManagerTestSuite) TestKeyspaceMovementConsistency() { // Create keyspace group 0 which contains keyspace 0, 1, 2. addKeyspaceGroupAssignment( - suite.ctx, suite.etcdClient, mcsutils.DefaultKeyspaceGroupID, - rootPath, []string{svcAddr}, []int{0}, []uint32{mcsutils.DefaultKeyspaceID, 10, 20}) + suite.ctx, suite.etcdClient, constant.DefaultKeyspaceGroupID, + rootPath, []string{svcAddr}, []int{0}, []uint32{constant.DefaultKeyspaceID, 10, 20}) // Create keyspace group 1 which contains keyspace 3, 4. addKeyspaceGroupAssignment( suite.ctx, suite.etcdClient, uint32(1), rootPath, @@ -585,9 +585,9 @@ func (suite *keyspaceGroupManagerTestSuite) TestKeyspaceMovementConsistency() { re.NoError(err) // Should be able to get AM for keyspace 10 in keyspace group 0. - am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(10, mcsutils.DefaultKeyspaceGroupID) + am, kg, kgid, err = mgr.getKeyspaceGroupMetaWithCheck(10, constant.DefaultKeyspaceGroupID) re.NoError(err) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kgid) + re.Equal(constant.DefaultKeyspaceGroupID, kgid) re.NotNil(am) re.NotNil(kg) @@ -603,7 +603,7 @@ func (suite *keyspaceGroupManagerTestSuite) TestKeyspaceMovementConsistency() { }, testutil.WithWaitFor(3*time.Second), testutil.WithTickInterval(50*time.Millisecond)) event = generateKeyspaceGroupPutEvent( - mcsutils.DefaultKeyspaceGroupID, []uint32{mcsutils.DefaultKeyspaceID, 20}, []string{svcAddr}) + constant.DefaultKeyspaceGroupID, []uint32{constant.DefaultKeyspaceID, 20}, []string{svcAddr}) err = putKeyspaceGroupToEtcd(suite.ctx, suite.etcdClient, rootPath, event.ksg) re.NoError(err) @@ -773,7 +773,7 @@ func (suite *keyspaceGroupManagerTestSuite) runTestLoadKeyspaceGroupsAssignment( // If no keyspace group is assigned to this host/pod, the default keyspace group should be initialized. if numberOfKeyspaceGroupsToAdd <= 0 { - expectedGroupIDs = append(expectedGroupIDs, mcsutils.DefaultKeyspaceGroupID) + expectedGroupIDs = append(expectedGroupIDs, constant.DefaultKeyspaceGroupID) } // Verify the keyspace group assignment. @@ -799,7 +799,7 @@ func (suite *keyspaceGroupManagerTestSuite) newKeyspaceGroupManager( tsoServiceID := &discovery.ServiceRegistryEntry{ServiceAddr: cfg.GetAdvertiseListenAddr()} clusterIDStr := strconv.FormatUint(clusterID, 10) legacySvcRootPath := path.Join("/pd", clusterIDStr) - tsoSvcRootPath := path.Join(mcsutils.MicroserviceRootPath, clusterIDStr, "tso") + tsoSvcRootPath := path.Join(constant.MicroserviceRootPath, clusterIDStr, "tso") electionNamePrefix := "kgm-test-" + cfg.GetAdvertiseListenAddr() kgm := NewKeyspaceGroupManager( @@ -1043,7 +1043,7 @@ func (suite *keyspaceGroupManagerTestSuite) TestPrimaryPriorityChange() { }() var err error - defaultPriority := mcsutils.DefaultKeyspaceGroupReplicaPriority + defaultPriority := constant.DefaultKeyspaceGroupReplicaPriority clusterID := rand.Uint64() clusterIDStr := strconv.FormatUint(clusterID, 10) @@ -1061,7 +1061,7 @@ func (suite *keyspaceGroupManagerTestSuite) TestPrimaryPriorityChange() { }() // Create three keyspace groups on two TSO servers with default replica priority. - ids := []uint32{0, mcsutils.MaxKeyspaceGroupCountInUse / 2, mcsutils.MaxKeyspaceGroupCountInUse - 1} + ids := []uint32{0, constant.MaxKeyspaceGroupCountInUse / 2, constant.MaxKeyspaceGroupCountInUse - 1} for _, id := range ids { addKeyspaceGroupAssignment( suite.ctx, suite.etcdClient, id, rootPath, @@ -1154,14 +1154,14 @@ func (suite *keyspaceGroupManagerTestSuite) registerTSOServer( serviceID := &discovery.ServiceRegistryEntry{ServiceAddr: cfg.GetAdvertiseListenAddr()} serializedEntry, err := serviceID.Serialize() re.NoError(err) - serviceKey := discovery.RegistryPath(clusterID, mcsutils.TSOServiceName, svcAddr) + serviceKey := discovery.RegistryPath(clusterID, constant.TSOServiceName, svcAddr) _, err = suite.etcdClient.Put(suite.ctx, serviceKey, serializedEntry) return err } // Deregister TSO server. func (suite *keyspaceGroupManagerTestSuite) deregisterTSOServer(clusterID, svcAddr string) error { - serviceKey := discovery.RegistryPath(clusterID, mcsutils.TSOServiceName, svcAddr) + serviceKey := discovery.RegistryPath(clusterID, constant.TSOServiceName, svcAddr) if _, err := suite.etcdClient.Delete(suite.ctx, serviceKey); err != nil { return err } diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 3b0d776dfd6..23723d9b254 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" @@ -128,8 +128,8 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri r.URL.Path = strings.TrimRight(r.URL.Path, "/") for _, rule := range h.microserviceRedirectRules { // Now we only support checking the scheduling service whether it is independent - if rule.targetServiceName == mcsutils.SchedulingServiceName { - if !h.s.GetRaftCluster().IsServiceIndependent(mcsutils.SchedulingServiceName) { + if rule.targetServiceName == constant.SchedulingServiceName { + if !h.s.GetRaftCluster().IsServiceIndependent(constant.SchedulingServiceName) { continue } } diff --git a/pkg/utils/tsoutil/tso_request.go b/pkg/utils/tsoutil/tso_request.go index ffcaf2c7330..3405debad9d 100644 --- a/pkg/utils/tsoutil/tso_request.go +++ b/pkg/utils/tsoutil/tso_request.go @@ -17,7 +17,7 @@ package tsoutil import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/kvproto/pkg/tsopb" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "google.golang.org/grpc" ) @@ -141,7 +141,7 @@ func (r *PDProtoRequest) getCount() uint32 { // count defines the count of timestamps to retrieve. func (r *PDProtoRequest) process(forwardStream stream, count uint32) (tsoResp, error) { return forwardStream.process(r.request.GetHeader().GetClusterId(), count, - utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID, r.request.GetDcLocation()) + constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID, r.request.GetDcLocation()) } // postProcess sends the response back to the sender of the request diff --git a/server/api/admin.go b/server/api/admin.go index 15d5d40412a..24b9feaea75 100644 --- a/server/api/admin.go +++ b/server/api/admin.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" "github.com/unrolled/render" @@ -62,7 +62,7 @@ func (h *adminHandler) DeleteRegionCache(w http.ResponseWriter, r *http.Request) } rc.RemoveRegionIfExist(regionID) msg := "The region is removed from server cache." - if rc.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(constant.SchedulingServiceName) { err = h.deleteRegionCacheInSchedulingServer(regionID) if err != nil { msg = buildMsg(err) @@ -105,7 +105,7 @@ func (h *adminHandler) DeleteRegionStorage(w http.ResponseWriter, r *http.Reques // Remove region from cache. rc.RemoveRegionIfExist(regionID) msg := "The region is removed from server cache and region meta storage." - if rc.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(constant.SchedulingServiceName) { err = h.deleteRegionCacheInSchedulingServer(regionID) if err != nil { msg = buildMsg(err) @@ -125,7 +125,7 @@ func (h *adminHandler) DeleteAllRegionCache(w http.ResponseWriter, r *http.Reque rc := getCluster(r) rc.ResetRegionCache() msg := "All regions are removed from server cache." - if rc.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(constant.SchedulingServiceName) { err = h.deleteRegionCacheInSchedulingServer() if err != nil { msg = buildMsg(err) @@ -227,7 +227,7 @@ func (h *adminHandler) recoverAllocID(w http.ResponseWriter, r *http.Request) { } func (h *adminHandler) deleteRegionCacheInSchedulingServer(id ...uint64) error { - addr, ok := h.svr.GetServicePrimaryAddr(h.svr.Context(), utils.SchedulingServiceName) + addr, ok := h.svr.GetServicePrimaryAddr(h.svr.Context(), constant.SchedulingServiceName) if !ok { return errs.ErrNotFoundSchedulingAddr.FastGenByArgs() } diff --git a/server/api/config.go b/server/api/config.go index c5681f9bbea..7b011957d22 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -28,7 +28,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/jsonutil" @@ -62,7 +62,7 @@ func newConfHandler(svr *server.Server, rd *render.Render) *confHandler { // @Router /config [get] func (h *confHandler) GetConfig(w http.ResponseWriter, r *http.Request) { cfg := h.svr.GetConfig() - if h.svr.GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) && + if h.svr.GetRaftCluster().IsServiceIndependent(constant.SchedulingServiceName) && r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { schedulingServerConfig, err := h.getSchedulingServerConfig() if err != nil { @@ -336,7 +336,7 @@ func getConfigMap(cfg map[string]any, key []string, value any) map[string]any { // @Success 200 {object} sc.ScheduleConfig // @Router /config/schedule [get] func (h *confHandler) GetScheduleConfig(w http.ResponseWriter, r *http.Request) { - if h.svr.GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) && + if h.svr.GetRaftCluster().IsServiceIndependent(constant.SchedulingServiceName) && r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { cfg, err := h.getSchedulingServerConfig() if err != nil { @@ -409,7 +409,7 @@ func (h *confHandler) SetScheduleConfig(w http.ResponseWriter, r *http.Request) // @Success 200 {object} sc.ReplicationConfig // @Router /config/replicate [get] func (h *confHandler) GetReplicationConfig(w http.ResponseWriter, r *http.Request) { - if h.svr.GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) && + if h.svr.GetRaftCluster().IsServiceIndependent(constant.SchedulingServiceName) && r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { cfg, err := h.getSchedulingServerConfig() if err != nil { @@ -562,7 +562,7 @@ func (h *confHandler) GetPDServerConfig(w http.ResponseWriter, _ *http.Request) } func (h *confHandler) getSchedulingServerConfig() (*config.Config, error) { - addr, ok := h.svr.GetServicePrimaryAddr(h.svr.Context(), utils.SchedulingServiceName) + addr, ok := h.svr.GetServicePrimaryAddr(h.svr.Context(), constant.SchedulingServiceName) if !ok { return nil, errs.ErrNotFoundSchedulingAddr.FastGenByArgs() } diff --git a/server/api/server.go b/server/api/server.go index 7b7066c4f77..8a58669de3b 100644 --- a/server/api/server.go +++ b/server/api/server.go @@ -22,7 +22,7 @@ import ( "github.com/gorilla/mux" scheapi "github.com/tikv/pd/pkg/mcs/scheduling/server/apis/v1" tsoapi "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" - mcs "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/apiutil/serverapi" "github.com/tikv/pd/server" @@ -70,22 +70,22 @@ func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.AP serverapi.MicroserviceRedirectRule( prefix+"/admin/reset-ts", tsoapi.APIPathPrefix+"/admin/reset-ts", - mcs.TSOServiceName, + constant.TSOServiceName, []string{http.MethodPost}), serverapi.MicroserviceRedirectRule( prefix+"/operators", scheapi.APIPathPrefix+"/operators", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodPost, http.MethodGet, http.MethodDelete}), serverapi.MicroserviceRedirectRule( prefix+"/checker", // Note: this is a typo in the original code scheapi.APIPathPrefix+"/checkers", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodPost, http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/region/id", scheapi.APIPathPrefix+"/config/regions", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}, func(r *http.Request) bool { // The original code uses the path "/region/id" to get the region id. @@ -95,79 +95,79 @@ func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.AP serverapi.MicroserviceRedirectRule( prefix+"/regions/accelerate-schedule", scheapi.APIPathPrefix+"/regions/accelerate-schedule", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodPost}), serverapi.MicroserviceRedirectRule( prefix+"/regions/scatter", scheapi.APIPathPrefix+"/regions/scatter", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodPost}), serverapi.MicroserviceRedirectRule( prefix+"/regions/split", scheapi.APIPathPrefix+"/regions/split", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodPost}), serverapi.MicroserviceRedirectRule( prefix+"/regions/replicated", scheapi.APIPathPrefix+"/regions/replicated", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/config/region-label/rules", scheapi.APIPathPrefix+"/config/region-label/rules", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/config/region-label/rule/", // Note: this is a typo in the original code scheapi.APIPathPrefix+"/config/region-label/rules", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/hotspot", scheapi.APIPathPrefix+"/hotspot", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/config/rules", scheapi.APIPathPrefix+"/config/rules", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/config/rule/", scheapi.APIPathPrefix+"/config/rule", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/config/rule_group/", scheapi.APIPathPrefix+"/config/rule_groups", // Note: this is a typo in the original code - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/config/rule_groups", scheapi.APIPathPrefix+"/config/rule_groups", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/config/placement-rule", scheapi.APIPathPrefix+"/config/placement-rule", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), // because the writing of all the meta information of the scheduling service is in the API server, // we should not post and delete the scheduler directly in the scheduling service. serverapi.MicroserviceRedirectRule( prefix+"/schedulers", scheapi.APIPathPrefix+"/schedulers", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/scheduler-config", scheapi.APIPathPrefix+"/schedulers/config", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/schedulers/", // Note: this means "/schedulers/{name}", which is to be used to pause or resume the scheduler scheapi.APIPathPrefix+"/schedulers", - mcs.SchedulingServiceName, + constant.SchedulingServiceName, []string{http.MethodPost}), ), negroni.Wrap(r)), diff --git a/server/apiv2/handlers/tso_keyspace_group.go b/server/apiv2/handlers/tso_keyspace_group.go index 6dafc98e603..e99e8cf55a4 100644 --- a/server/apiv2/handlers/tso_keyspace_group.go +++ b/server/apiv2/handlers/tso_keyspace_group.go @@ -22,7 +22,7 @@ import ( "github.com/gin-gonic/gin" "github.com/pingcap/errors" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/syncutil" @@ -237,7 +237,7 @@ func SplitKeyspaceGroupByID(c *gin.Context) { c.AbortWithStatusJSON(http.StatusBadRequest, "invalid empty keyspaces") return } - if splitParams.StartKeyspaceID < utils.DefaultKeyspaceID || + if splitParams.StartKeyspaceID < constant.DefaultKeyspaceID || splitParams.StartKeyspaceID > splitParams.EndKeyspaceID { c.AbortWithStatusJSON(http.StatusBadRequest, "invalid start/end keyspace id") return @@ -400,7 +400,7 @@ func AllocNodesForKeyspaceGroup(c *gin.Context) { c.AbortWithStatusJSON(http.StatusBadRequest, errs.ErrBindJSON.Wrap(err).GenWithStackByCause()) return } - if manager.GetNodesCount() < allocParams.Replica || allocParams.Replica < utils.DefaultKeyspaceGroupReplicaCount { + if manager.GetNodesCount() < allocParams.Replica || allocParams.Replica < constant.DefaultKeyspaceGroupReplicaCount { c.AbortWithStatusJSON(http.StatusBadRequest, "invalid replica, should be in [2, nodes_num]") return } @@ -553,5 +553,5 @@ func parseNodeAddress(c *gin.Context) (string, error) { } func isValid(id uint32) bool { - return id >= utils.DefaultKeyspaceGroupID && id <= utils.MaxKeyspaceGroupCountInUse + return id >= constant.DefaultKeyspaceGroupID && id <= constant.MaxKeyspaceGroupCountInUse } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 2150fe992ad..a2a365bb4c1 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -42,7 +42,7 @@ import ( "github.com/tikv/pd/pkg/id" "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/mcs/discovery" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/memory" "github.com/tikv/pd/pkg/progress" "github.com/tikv/pd/pkg/ratelimit" @@ -327,7 +327,7 @@ func (c *RaftCluster) Start(s Server) error { return err } - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !c.IsServiceIndependent(constant.SchedulingServiceName) { for _, store := range c.GetStores() { storeID := store.GetID() c.slowStat.ObserveSlowStoreStatus(storeID, store.IsSlow()) @@ -372,21 +372,21 @@ func (c *RaftCluster) Start(s Server) error { func (c *RaftCluster) checkServices() { if c.isAPIServiceMode { - servers, err := discovery.Discover(c.etcdClient, strconv.FormatUint(c.clusterID, 10), mcsutils.SchedulingServiceName) + servers, err := discovery.Discover(c.etcdClient, strconv.FormatUint(c.clusterID, 10), constant.SchedulingServiceName) if c.opt.GetMicroServiceConfig().IsSchedulingFallbackEnabled() && (err != nil || len(servers) == 0) { c.startSchedulingJobs(c, c.hbstreams) - c.UnsetServiceIndependent(mcsutils.SchedulingServiceName) + c.UnsetServiceIndependent(constant.SchedulingServiceName) } else { if c.stopSchedulingJobs() || c.coordinator == nil { c.initCoordinator(c.ctx, c, c.hbstreams) } - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { - c.SetServiceIndependent(mcsutils.SchedulingServiceName) + if !c.IsServiceIndependent(constant.SchedulingServiceName) { + c.SetServiceIndependent(constant.SchedulingServiceName) } } } else { c.startSchedulingJobs(c, c.hbstreams) - c.UnsetServiceIndependent(mcsutils.SchedulingServiceName) + c.UnsetServiceIndependent(constant.SchedulingServiceName) } } @@ -757,7 +757,7 @@ func (c *RaftCluster) Stop() { } c.running = false c.cancel() - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !c.IsServiceIndependent(constant.SchedulingServiceName) { c.stopSchedulingJobs() } c.heartbeatRunner.Stop() @@ -889,7 +889,7 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest nowTime := time.Now() var newStore *core.StoreInfo // If this cluster has slow stores, we should awaken hibernated regions in other stores. - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !c.IsServiceIndependent(constant.SchedulingServiceName) { if needAwaken, slowStoreIDs := c.NeedAwakenAllRegionsInStore(storeID); needAwaken { log.Info("forcely awaken hibernated regions", zap.Uint64("store-id", storeID), zap.Uint64s("slow-stores", slowStoreIDs)) newStore = store.Clone(core.SetStoreStats(stats), core.SetLastHeartbeatTS(nowTime), core.SetLastAwakenTime(nowTime), opt) @@ -924,7 +924,7 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest regions map[uint64]*core.RegionInfo interval uint64 ) - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !c.IsServiceIndependent(constant.SchedulingServiceName) { c.hotStat.Observe(storeID, newStore.GetStoreStats()) c.hotStat.FilterUnhealthyStore(c) c.slowStat.ObserveSlowStoreStatus(storeID, newStore.IsSlow()) @@ -985,7 +985,7 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest e := int64(dur)*2 - int64(stat.GetTotalDurationSec()) store.Feedback(float64(e)) } - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !c.IsServiceIndependent(constant.SchedulingServiceName) { // Here we will compare the reported regions with the previous hot peers to decide if it is still hot. collectUnReportedPeerTask := func(cache *statistics.HotPeerCache) { stats := cache.CheckColdPeer(storeID, regions, interval) @@ -1041,7 +1041,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio region.Inherit(origin, c.GetStoreConfig().IsEnableRegionBucket()) - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !c.IsServiceIndependent(constant.SchedulingServiceName) { cluster.HandleStatsAsync(c, region) } tracer.OnAsyncHotStatsFinished() @@ -1106,7 +1106,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio ) tracer.OnUpdateSubTreeFinished() - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !c.IsServiceIndependent(constant.SchedulingServiceName) { ctx.MiscRunner.RunTask( regionID, ratelimit.HandleOverlaps, @@ -1449,7 +1449,7 @@ func (c *RaftCluster) BuryStore(storeID uint64, forceBury bool) error { c.resetProgress(storeID, addr) storeIDStr := strconv.FormatUint(storeID, 10) statistics.ResetStoreStatistics(addr, storeIDStr) - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !c.IsServiceIndependent(constant.SchedulingServiceName) { c.removeStoreStatistics(storeID) } } @@ -1585,7 +1585,7 @@ func (c *RaftCluster) setStore(store *core.StoreInfo) error { } } c.PutStore(store) - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !c.IsServiceIndependent(constant.SchedulingServiceName) { c.updateStoreStatistics(store.GetID(), store.IsSlow()) } return nil @@ -1635,7 +1635,7 @@ func (c *RaftCluster) checkStores() { zap.Int("region-count", c.GetTotalRegionCount()), errs.ZapError(err)) } - } else if c.IsPrepared() || (c.IsServiceIndependent(mcsutils.SchedulingServiceName) && c.isStorePrepared()) { + } else if c.IsPrepared() || (c.IsServiceIndependent(constant.SchedulingServiceName) && c.isStorePrepared()) { threshold := c.getThreshold(stores, store) regionSize := float64(store.GetRegionSize()) log.Debug("store serving threshold", zap.Uint64("store-id", storeID), zap.Float64("threshold", threshold), zap.Float64("region-size", regionSize)) diff --git a/server/cluster/cluster_worker.go b/server/cluster/cluster_worker.go index 941282c4aca..c14ceff2153 100644 --- a/server/cluster/cluster_worker.go +++ b/server/cluster/cluster_worker.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/statistics/buckets" @@ -62,7 +62,7 @@ func (c *RaftCluster) HandleRegionHeartbeat(region *core.RegionInfo) error { } tracer.OnAllStageFinished() - if c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if c.IsServiceIndependent(constant.SchedulingServiceName) { return nil } c.coordinator.GetOperatorController().Dispatch(region, operator.DispatchFromHeartBeat, c.coordinator.RecordOpStepWithTTL) @@ -257,7 +257,7 @@ func (c *RaftCluster) HandleReportBuckets(b *metapb.Buckets) error { if err := c.processReportBuckets(b); err != nil { return err } - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if !c.IsServiceIndependent(constant.SchedulingServiceName) { c.hotStat.CheckAsync(buckets.NewCheckPeerTask(b)) } return nil diff --git a/server/forward.go b/server/forward.go index c407e545f6f..e13f5d45f71 100644 --- a/server/forward.go +++ b/server/forward.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/kvproto/pkg/tsopb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/logutil" @@ -47,8 +47,8 @@ func forwardTSORequest( Header: &tsopb.RequestHeader{ ClusterId: request.GetHeader().GetClusterId(), SenderId: request.GetHeader().GetSenderId(), - KeyspaceId: utils.DefaultKeyspaceID, - KeyspaceGroupId: utils.DefaultKeyspaceGroupID, + KeyspaceId: constant.DefaultKeyspaceID, + KeyspaceGroupId: constant.DefaultKeyspaceGroupID, }, Count: request.GetCount(), DcLocation: request.GetDcLocation(), @@ -134,7 +134,7 @@ func (s *GrpcServer) forwardTSO(stream pdpb.PD_TsoServer) error { return status.Errorf(codes.Unknown, err.Error()) } - forwardedHost, ok := s.GetServicePrimaryAddr(stream.Context(), utils.TSOServiceName) + forwardedHost, ok := s.GetServicePrimaryAddr(stream.Context(), constant.TSOServiceName) if !ok || len(forwardedHost) == 0 { tsoStreamErr = errors.WithStack(ErrNotFoundTSOAddr) return tsoStreamErr @@ -408,8 +408,8 @@ func (s *GrpcServer) getGlobalTSO(ctx context.Context) (pdpb.Timestamp, error) { request := &tsopb.TsoRequest{ Header: &tsopb.RequestHeader{ ClusterId: s.ClusterID(), - KeyspaceId: utils.DefaultKeyspaceID, - KeyspaceGroupId: utils.DefaultKeyspaceGroupID, + KeyspaceId: constant.DefaultKeyspaceID, + KeyspaceGroupId: constant.DefaultKeyspaceGroupID, }, Count: 1, } @@ -439,7 +439,7 @@ func (s *GrpcServer) getGlobalTSO(ctx context.Context) (pdpb.Timestamp, error) { if i > 0 { time.Sleep(retryIntervalRequestTSOServer) } - forwardedHost, ok = s.GetServicePrimaryAddr(ctx, utils.TSOServiceName) + forwardedHost, ok = s.GetServicePrimaryAddr(ctx, constant.TSOServiceName) if !ok || forwardedHost == "" { return pdpb.Timestamp{}, ErrNotFoundTSOAddr } diff --git a/server/grpc_service.go b/server/grpc_service.go index 21126addd24..0f241986a9b 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -37,7 +37,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/tso" @@ -951,7 +951,7 @@ func (s *GrpcServer) StoreHeartbeat(ctx context.Context, request *pdpb.StoreHear s.handleDamagedStore(request.GetStats()) storeHeartbeatHandleDuration.WithLabelValues(storeAddress, storeLabel).Observe(time.Since(start).Seconds()) - if rc.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(constant.SchedulingServiceName) { forwardCli, _ := s.updateSchedulingClient(ctx) cli := forwardCli.getClient() if cli != nil { @@ -987,7 +987,7 @@ func (s *GrpcServer) StoreHeartbeat(ctx context.Context, request *pdpb.StoreHear // 2. forwardedHost is not empty and forwardedHost is equal to pre, return pre // 3. the rest of cases, update forwardedHost and return new client func (s *GrpcServer) updateSchedulingClient(ctx context.Context) (*schedulingClient, error) { - forwardedHost, _ := s.GetServicePrimaryAddr(ctx, utils.SchedulingServiceName) + forwardedHost, _ := s.GetServicePrimaryAddr(ctx, constant.SchedulingServiceName) if forwardedHost == "" { return nil, ErrNotFoundSchedulingAddr } @@ -1307,7 +1307,7 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error regionHeartbeatHandleDuration.WithLabelValues(storeAddress, storeLabel).Observe(time.Since(start).Seconds()) regionHeartbeatCounter.WithLabelValues(storeAddress, storeLabel, "report", "ok").Inc() - if rc.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(constant.SchedulingServiceName) { if forwardErrCh != nil { select { case err, ok := <-forwardErrCh: @@ -1321,7 +1321,7 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error default: } } - forwardedSchedulingHost, ok := s.GetServicePrimaryAddr(stream.Context(), utils.SchedulingServiceName) + forwardedSchedulingHost, ok := s.GetServicePrimaryAddr(stream.Context(), constant.SchedulingServiceName) if !ok || len(forwardedSchedulingHost) == 0 { log.Debug("failed to find scheduling service primary address") if cancel != nil { @@ -1792,7 +1792,7 @@ func (s *GrpcServer) AskBatchSplit(ctx context.Context, request *pdpb.AskBatchSp return &pdpb.AskBatchSplitResponse{Header: s.notBootstrappedHeader()}, nil } - if rc.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(constant.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.AskBatchSplitResponse{ @@ -2022,7 +2022,7 @@ func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterReg return &pdpb.ScatterRegionResponse{Header: s.notBootstrappedHeader()}, nil } - if rc.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(constant.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.ScatterRegionResponse{ @@ -2300,7 +2300,7 @@ func (s *GrpcServer) GetOperator(ctx context.Context, request *pdpb.GetOperatorR return &pdpb.GetOperatorResponse{Header: s.notBootstrappedHeader()}, nil } - if rc.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(constant.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.GetOperatorResponse{ @@ -2620,7 +2620,7 @@ func (s *GrpcServer) SplitRegions(ctx context.Context, request *pdpb.SplitRegion return &pdpb.SplitRegionsResponse{Header: s.notBootstrappedHeader()}, nil } - if rc.IsServiceIndependent(utils.SchedulingServiceName) { + if rc.IsServiceIndependent(constant.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.SplitRegionsResponse{ diff --git a/server/handler.go b/server/handler.go index 995f98a3324..e3e4184f177 100644 --- a/server/handler.go +++ b/server/handler.go @@ -28,7 +28,7 @@ import ( "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/errs" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule" sc "github.com/tikv/pd/pkg/schedule/config" sche "github.com/tikv/pd/pkg/schedule/core" @@ -194,7 +194,7 @@ func (h *Handler) AddScheduler(tp types.CheckerSchedulerType, args ...string) er } var removeSchedulerCb func(string) error - if c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if c.IsServiceIndependent(constant.SchedulingServiceName) { removeSchedulerCb = c.GetCoordinator().GetSchedulersController().RemoveSchedulerHandler } else { removeSchedulerCb = c.GetCoordinator().GetSchedulersController().RemoveScheduler @@ -204,7 +204,7 @@ func (h *Handler) AddScheduler(tp types.CheckerSchedulerType, args ...string) er return err } log.Info("create scheduler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", args)) - if c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if c.IsServiceIndependent(constant.SchedulingServiceName) { if err = c.AddSchedulerHandler(s, args...); err != nil { log.Error("can not add scheduler handler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", args), errs.ZapError(err)) return err @@ -231,7 +231,7 @@ func (h *Handler) RemoveScheduler(name string) error { if err != nil { return err } - if c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + if c.IsServiceIndependent(constant.SchedulingServiceName) { if err = c.RemoveSchedulerHandler(name); err != nil { log.Error("can not remove scheduler handler", zap.String("scheduler-name", name), errs.ZapError(err)) } else { diff --git a/server/server.go b/server/server.go index 1c42070cfe3..6e0179798e8 100644 --- a/server/server.go +++ b/server/server.go @@ -56,7 +56,7 @@ import ( rm_server "github.com/tikv/pd/pkg/mcs/resourcemanager/server" _ "github.com/tikv/pd/pkg/mcs/resourcemanager/server/apis/v1" // init API group _ "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" // init tso API group - mcs "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/replication" @@ -475,7 +475,7 @@ func (s *Server) startServer(ctx context.Context) error { s.tsoDispatcher = tsoutil.NewTSODispatcher(tsoProxyHandleDuration, tsoProxyBatchSize) s.tsoProtoFactory = &tsoutil.TSOProtoFactory{} s.pdProtoFactory = &tsoutil.PDProtoFactory{} - s.tsoAllocatorManager = tso.NewAllocatorManager(s.ctx, mcs.DefaultKeyspaceGroupID, s.member, s.rootPath, s.storage, s, false) + s.tsoAllocatorManager = tso.NewAllocatorManager(s.ctx, constant.DefaultKeyspaceGroupID, s.member, s.rootPath, s.storage, s, false) // When disabled the Local TSO, we should clean up the Local TSO Allocator's meta info written in etcd if it exists. if !s.cfg.EnableLocalTSO { if err = s.tsoAllocatorManager.CleanUpDCLocation(); err != nil { @@ -1789,7 +1789,7 @@ func (s *Server) campaignLeader() { CheckPDVersionWithClusterVersion(s.persistOptions) log.Info(fmt.Sprintf("%s leader is ready to serve", s.mode), zap.String("leader-name", s.Name())) - leaderTicker := time.NewTicker(mcs.LeaderTickInterval) + leaderTicker := time.NewTicker(constant.LeaderTickInterval) defer leaderTicker.Stop() for { @@ -2018,15 +2018,15 @@ func (s *Server) SetServicePrimaryAddr(serviceName, addr string) { } func (s *Server) initTSOPrimaryWatcher() { - serviceName := mcs.TSOServiceName + serviceName := constant.TSOServiceName tsoRootPath := endpoint.TSOSvcRootPath(s.ClusterID()) - tsoServicePrimaryKey := endpoint.KeyspaceGroupPrimaryPath(tsoRootPath, mcs.DefaultKeyspaceGroupID) + tsoServicePrimaryKey := endpoint.KeyspaceGroupPrimaryPath(tsoRootPath, constant.DefaultKeyspaceGroupID) s.tsoPrimaryWatcher = s.initServicePrimaryWatcher(serviceName, tsoServicePrimaryKey) s.tsoPrimaryWatcher.StartWatchLoop() } func (s *Server) initSchedulingPrimaryWatcher() { - serviceName := mcs.SchedulingServiceName + serviceName := constant.SchedulingServiceName primaryKey := endpoint.SchedulingPrimaryPath(s.ClusterID()) s.schedulingPrimaryWatcher = s.initServicePrimaryWatcher(serviceName, primaryKey) s.schedulingPrimaryWatcher.StartWatchLoop() diff --git a/server/server_test.go b/server/server_test.go index 410afda448d..80771721a60 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/assertutil" "github.com/tikv/pd/pkg/utils/etcdutil" @@ -264,7 +264,7 @@ func TestAPIService(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() mockHandler := CreateMockHandler(re, "127.0.0.1") - svr, err := CreateServer(ctx, cfg, []string{utils.APIServiceName}, mockHandler) + svr, err := CreateServer(ctx, cfg, []string{constant.APIServiceName}, mockHandler) re.NoError(err) defer svr.Close() err = svr.Run() diff --git a/tests/cluster.go b/tests/cluster.go index 2aa783590df..ad796226e31 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -34,7 +34,7 @@ import ( "github.com/tikv/pd/pkg/id" "github.com/tikv/pd/pkg/keyspace" scheduling "github.com/tikv/pd/pkg/mcs/scheduling/server" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/swaggerserver" "github.com/tikv/pd/pkg/tso" @@ -84,7 +84,7 @@ func NewTestServer(ctx context.Context, cfg *config.Config) (*TestServer, error) // NewTestAPIServer creates a new TestServer. func NewTestAPIServer(ctx context.Context, cfg *config.Config) (*TestServer, error) { - return createTestServer(ctx, cfg, []string{utils.APIServiceName}) + return createTestServer(ctx, cfg, []string{constant.APIServiceName}) } func createTestServer(ctx context.Context, cfg *config.Config, services []string) (*TestServer, error) { diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 4138b775d7c..79eec0accde 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -44,7 +44,7 @@ import ( "github.com/tikv/pd/client/retry" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/tso" @@ -407,7 +407,7 @@ func TestTSOFollowerProxyWithTSOService(t *testing.T) { tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, backendEndpoints) re.NoError(err) defer tsoCluster.Destroy() - cli := mcs.SetupClientWithKeyspaceID(ctx, re, utils.DefaultKeyspaceID, strings.Split(backendEndpoints, ",")) + cli := mcs.SetupClientWithKeyspaceID(ctx, re, constant.DefaultKeyspaceID, strings.Split(backendEndpoints, ",")) re.NotNil(cli) defer cli.Close() // TSO service does not support the follower proxy, so enabling it should fail. diff --git a/tests/integrations/client/keyspace_test.go b/tests/integrations/client/keyspace_test.go index df396808d8e..573302f0695 100644 --- a/tests/integrations/client/keyspace_test.go +++ b/tests/integrations/client/keyspace_test.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/keyspace" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/server" ) @@ -65,10 +65,10 @@ func (suite *clientTestSuite) TestLoadKeyspace() { _, err := suite.client.LoadKeyspace(suite.ctx, "non-existing keyspace") re.Error(err) // Loading default keyspace should be successful. - keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, utils.DefaultKeyspaceName) + keyspaceDefault, err := suite.client.LoadKeyspace(suite.ctx, constant.DefaultKeyspaceName) re.NoError(err) - re.Equal(utils.DefaultKeyspaceID, keyspaceDefault.GetId()) - re.Equal(utils.DefaultKeyspaceName, keyspaceDefault.GetName()) + re.Equal(constant.DefaultKeyspaceID, keyspaceDefault.GetId()) + re.Equal(constant.DefaultKeyspaceName, keyspaceDefault.GetName()) } func (suite *clientTestSuite) TestGetAllKeyspaces() { diff --git a/tests/integrations/mcs/discovery/register_test.go b/tests/integrations/mcs/discovery/register_test.go index baf3442226a..ce019b29da8 100644 --- a/tests/integrations/mcs/discovery/register_test.go +++ b/tests/integrations/mcs/discovery/register_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/suite" bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/mcs/discovery" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" @@ -72,14 +72,9 @@ func (suite *serverRegisterTestSuite) TearDownSuite() { } func (suite *serverRegisterTestSuite) TestServerRegister() { - // test register, primary and unregister when start tso and resource-manager with only one server for i := 0; i < 3; i++ { - suite.checkServerRegister(utils.TSOServiceName) + suite.checkServerRegister(constant.TSOServiceName) } - // TODO: uncomment after resource-manager is ready - // for i := 0; i < 3; i++ { - // suite.checkServerRegister(utils.ResourceManagerServiceName) - // } } func (suite *serverRegisterTestSuite) checkServerRegister(serviceName string) { @@ -113,9 +108,7 @@ func (suite *serverRegisterTestSuite) checkServerRegister(serviceName string) { } func (suite *serverRegisterTestSuite) TestServerPrimaryChange() { - suite.checkServerPrimaryChange(utils.TSOServiceName, 3) - // TODO: uncomment after resource-manager is ready - // suite.checkServerPrimaryChange(utils.ResourceManagerServiceName, 3) + suite.checkServerPrimaryChange(constant.TSOServiceName, 3) } func (suite *serverRegisterTestSuite) checkServerPrimaryChange(serviceName string, serverNum int) { @@ -161,9 +154,9 @@ func (suite *serverRegisterTestSuite) checkServerPrimaryChange(serviceName strin func (suite *serverRegisterTestSuite) addServer(serviceName string) (bs.Server, func()) { re := suite.Require() switch serviceName { - case utils.TSOServiceName: + case constant.TSOServiceName: return tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) - case utils.ResourceManagerServiceName: + case constant.ResourceManagerServiceName: return tests.StartSingleResourceManagerTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) default: return nil, nil diff --git a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go index 7c95b99bcc7..49f1566de93 100644 --- a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go +++ b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go @@ -28,7 +28,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" bs "github.com/tikv/pd/pkg/basicserver" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" @@ -88,7 +88,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodesUpdate() { cleanup() } }() - for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount+1; i++ { + for i := 0; i < constant.DefaultKeyspaceGroupReplicaCount+1; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s @@ -108,11 +108,11 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodesUpdate() { // alloc nodes for the keyspace group. id := 1 params := &handlers.AllocNodesForKeyspaceGroupParams{ - Replica: utils.DefaultKeyspaceGroupReplicaCount, + Replica: constant.DefaultKeyspaceGroupReplicaCount, } got, code := suite.tryAllocNodesForKeyspaceGroup(re, id, params) re.Equal(http.StatusOK, code) - re.Len(got, utils.DefaultKeyspaceGroupReplicaCount) + re.Len(got, constant.DefaultKeyspaceGroupReplicaCount) oldMembers := make(map[string]struct{}) for _, member := range got { re.Contains(nodes, member.Address) @@ -120,7 +120,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodesUpdate() { } // alloc node update to 3. - params.Replica = utils.DefaultKeyspaceGroupReplicaCount + 1 + params.Replica = constant.DefaultKeyspaceGroupReplicaCount + 1 got, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) re.Equal(http.StatusOK, code) re.Len(got, params.Replica) @@ -144,7 +144,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocReplica() { cleanup() } }() - for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { + for i := 0; i < constant.DefaultKeyspaceGroupReplicaCount; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s @@ -160,14 +160,14 @@ func (suite *keyspaceGroupTestSuite) TestAllocReplica() { // replica is less than default replica. params = &handlers.AllocNodesForKeyspaceGroupParams{ - Replica: utils.DefaultKeyspaceGroupReplicaCount - 1, + Replica: constant.DefaultKeyspaceGroupReplicaCount - 1, } _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) re.Equal(http.StatusBadRequest, code) // there is no any keyspace group. params = &handlers.AllocNodesForKeyspaceGroupParams{ - Replica: utils.DefaultKeyspaceGroupReplicaCount, + Replica: constant.DefaultKeyspaceGroupReplicaCount, } _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) re.Equal(http.StatusBadRequest, code) @@ -182,7 +182,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocReplica() { code = suite.tryCreateKeyspaceGroup(re, kgs) re.Equal(http.StatusOK, code) params = &handlers.AllocNodesForKeyspaceGroupParams{ - Replica: utils.DefaultKeyspaceGroupReplicaCount, + Replica: constant.DefaultKeyspaceGroupReplicaCount, } got, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) re.Equal(http.StatusOK, code) @@ -192,7 +192,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocReplica() { // the keyspace group is exist, but the replica is more than the num of nodes. params = &handlers.AllocNodesForKeyspaceGroupParams{ - Replica: utils.DefaultKeyspaceGroupReplicaCount + 1, + Replica: constant.DefaultKeyspaceGroupReplicaCount + 1, } _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) re.Equal(http.StatusBadRequest, code) @@ -203,7 +203,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocReplica() { nodes[s2.GetAddr()] = s2 tests.WaitForPrimaryServing(re, nodes) params = &handlers.AllocNodesForKeyspaceGroupParams{ - Replica: utils.DefaultKeyspaceGroupReplicaCount + 1, + Replica: constant.DefaultKeyspaceGroupReplicaCount + 1, } got, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) re.Equal(http.StatusOK, code) @@ -213,14 +213,14 @@ func (suite *keyspaceGroupTestSuite) TestAllocReplica() { // the keyspace group is exist, the new replica is equal to the old replica. params = &handlers.AllocNodesForKeyspaceGroupParams{ - Replica: utils.DefaultKeyspaceGroupReplicaCount + 1, + Replica: constant.DefaultKeyspaceGroupReplicaCount + 1, } _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) re.Equal(http.StatusBadRequest, code) // the keyspace group is exist, the new replica is less than the old replica. params = &handlers.AllocNodesForKeyspaceGroupParams{ - Replica: utils.DefaultKeyspaceGroupReplicaCount, + Replica: constant.DefaultKeyspaceGroupReplicaCount, } _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) re.Equal(http.StatusBadRequest, code) @@ -228,7 +228,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocReplica() { // the keyspace group is not exist. id = 2 params = &handlers.AllocNodesForKeyspaceGroupParams{ - Replica: utils.DefaultKeyspaceGroupReplicaCount, + Replica: constant.DefaultKeyspaceGroupReplicaCount, } _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) re.Equal(http.StatusBadRequest, code) @@ -244,7 +244,7 @@ func (suite *keyspaceGroupTestSuite) TestSetNodes() { cleanup() } }() - for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { + for i := 0; i < constant.DefaultKeyspaceGroupReplicaCount; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s @@ -311,7 +311,7 @@ func (suite *keyspaceGroupTestSuite) TestDefaultKeyspaceGroup() { cleanup() } }() - for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { + for i := 0; i < constant.DefaultKeyspaceGroupReplicaCount; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s @@ -322,13 +322,13 @@ func (suite *keyspaceGroupTestSuite) TestDefaultKeyspaceGroup() { var kg *endpoint.KeyspaceGroup var code int testutil.Eventually(re, func() bool { - kg, code = suite.tryGetKeyspaceGroup(re, utils.DefaultKeyspaceGroupID) + kg, code = suite.tryGetKeyspaceGroup(re, constant.DefaultKeyspaceGroupID) return code == http.StatusOK && kg != nil }, testutil.WithWaitFor(time.Second*1)) - re.Equal(utils.DefaultKeyspaceGroupID, kg.ID) + re.Equal(constant.DefaultKeyspaceGroupID, kg.ID) // the allocNodesToAllKeyspaceGroups loop will run every 100ms. testutil.Eventually(re, func() bool { - return len(kg.Members) == utils.DefaultKeyspaceGroupReplicaCount + return len(kg.Members) == constant.DefaultKeyspaceGroupReplicaCount }) for _, member := range kg.Members { re.Contains(nodes, member.Address) @@ -345,7 +345,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodes() { cleanup() } }() - for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount+1; i++ { + for i := 0; i < constant.DefaultKeyspaceGroupReplicaCount+1; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s @@ -365,8 +365,8 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodes() { // alloc nodes for the keyspace group var kg *endpoint.KeyspaceGroup testutil.Eventually(re, func() bool { - kg, code = suite.tryGetKeyspaceGroup(re, utils.DefaultKeyspaceGroupID) - return code == http.StatusOK && kg != nil && len(kg.Members) == utils.DefaultKeyspaceGroupReplicaCount + kg, code = suite.tryGetKeyspaceGroup(re, constant.DefaultKeyspaceGroupID) + return code == http.StatusOK && kg != nil && len(kg.Members) == constant.DefaultKeyspaceGroupReplicaCount }) stopNode := kg.Members[0].Address // close one of members @@ -374,13 +374,13 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodes() { // the member list will be updated testutil.Eventually(re, func() bool { - kg, code = suite.tryGetKeyspaceGroup(re, utils.DefaultKeyspaceGroupID) + kg, code = suite.tryGetKeyspaceGroup(re, constant.DefaultKeyspaceGroupID) for _, member := range kg.Members { if member.Address == stopNode { return false } } - return code == http.StatusOK && kg != nil && len(kg.Members) == utils.DefaultKeyspaceGroupReplicaCount + return code == http.StatusOK && kg != nil && len(kg.Members) == constant.DefaultKeyspaceGroupReplicaCount }) } @@ -407,7 +407,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocOneNode() { // alloc nodes for the keyspace group var kg *endpoint.KeyspaceGroup testutil.Eventually(re, func() bool { - kg, code = suite.tryGetKeyspaceGroup(re, utils.DefaultKeyspaceGroupID) + kg, code = suite.tryGetKeyspaceGroup(re, constant.DefaultKeyspaceGroupID) return code == http.StatusOK && kg != nil && len(kg.Members) == 1 }) stopNode := kg.Members[0].Address @@ -423,7 +423,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocOneNode() { // the member list will be updated testutil.Eventually(re, func() bool { - kg, code = suite.tryGetKeyspaceGroup(re, utils.DefaultKeyspaceGroupID) + kg, code = suite.tryGetKeyspaceGroup(re, constant.DefaultKeyspaceGroupID) if len(kg.Members) != 0 && kg.Members[0].Address == stopNode { return false } diff --git a/tests/integrations/mcs/members/member_test.go b/tests/integrations/mcs/members/member_test.go index d650d1ded4f..79b269e2e8a 100644 --- a/tests/integrations/mcs/members/member_test.go +++ b/tests/integrations/mcs/members/member_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/suite" pdClient "github.com/tikv/pd/client/http" bs "github.com/tikv/pd/pkg/basicserver" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" @@ -57,7 +57,7 @@ func (suite *memberTestSuite) SetupTest() { // TSO nodes := make(map[string]bs.Server) - for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { + for i := 0; i < constant.DefaultKeyspaceGroupReplicaCount; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) nodes[s.GetAddr()] = s suite.cleanupFunc = append(suite.cleanupFunc, func() { @@ -96,7 +96,7 @@ func (suite *memberTestSuite) TestMembers() { re := suite.Require() members, err := suite.pdClient.GetMicroServiceMembers(suite.ctx, "tso") re.NoError(err) - re.Len(members, utils.DefaultKeyspaceGroupReplicaCount) + re.Len(members, constant.DefaultKeyspaceGroupReplicaCount) members, err = suite.pdClient.GetMicroServiceMembers(suite.ctx, "scheduling") re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 4425369ed71..68f276f9e22 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -16,7 +16,7 @@ import ( "github.com/tikv/pd/pkg/core" _ "github.com/tikv/pd/pkg/mcs/scheduling/server/apis/v1" "github.com/tikv/pd/pkg/mcs/scheduling/server/config" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule/handler" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" @@ -110,7 +110,7 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { var respSlice []string var resp map[string]any testutil.Eventually(re, func() bool { - return leader.GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) + return leader.GetRaftCluster().IsServiceIndependent(constant.SchedulingServiceName) }) // Test operators diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 35dc78d0bee..ab6bb93c60c 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core/storelimit" - mcs "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/utils/testutil" @@ -141,7 +141,7 @@ func (suite *serverTestSuite) TestPrimaryChange() { primary := tc.GetPrimaryServer() oldPrimaryAddr := primary.GetAddr() testutil.Eventually(re, func() bool { - watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName) + watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, constant.SchedulingServiceName) return ok && oldPrimaryAddr == watchedAddr && len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 4 }) @@ -152,7 +152,7 @@ func (suite *serverTestSuite) TestPrimaryChange() { newPrimaryAddr := primary.GetAddr() re.NotEqual(oldPrimaryAddr, newPrimaryAddr) testutil.Eventually(re, func() bool { - watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName) + watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, constant.SchedulingServiceName) return ok && newPrimaryAddr == watchedAddr && len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 4 }) diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index a07ad53f906..758185b5e5b 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/suite" tso "github.com/tikv/pd/pkg/mcs/tso/server" apis "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/testutil" @@ -86,11 +86,11 @@ func (suite *tsoAPITestSuite) TestGetKeyspaceGroupMembers() { re.NotNil(primary) members := mustGetKeyspaceGroupMembers(re, primary) re.Len(members, 1) - defaultGroupMember := members[mcsutils.DefaultKeyspaceGroupID] + defaultGroupMember := members[constant.DefaultKeyspaceGroupID] re.NotNil(defaultGroupMember) - re.Equal(mcsutils.DefaultKeyspaceGroupID, defaultGroupMember.Group.ID) + re.Equal(constant.DefaultKeyspaceGroupID, defaultGroupMember.Group.ID) re.True(defaultGroupMember.IsPrimary) - primaryMember, err := primary.GetMember(mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + primaryMember, err := primary.GetMember(constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) re.NoError(err) re.Equal(primaryMember.GetLeaderID(), defaultGroupMember.PrimaryID) } diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index 2acbbcc3b42..cdd4dc106b9 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -31,7 +31,7 @@ import ( clierrs "github.com/tikv/pd/client/errs" "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/errs" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/storage/endpoint" @@ -106,7 +106,7 @@ func cleanupKeyspaceGroups(re *require.Assertions, server *tests.TestServer) { keyspaceGroups := handlersutil.MustLoadKeyspaceGroups(re, server, "0", "0") for _, group := range keyspaceGroups { // Do not delete default keyspace group. - if group.ID == mcsutils.DefaultKeyspaceGroupID { + if group.ID == constant.DefaultKeyspaceGroupID { continue } handlersutil.MustDeleteKeyspaceGroup(re, server, group.ID) @@ -121,8 +121,8 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestKeyspacesServedByDefaultKeysp for _, keyspaceID := range []uint32{0, 1, 2} { served := false for _, server := range suite.tsoCluster.GetServers() { - if server.IsKeyspaceServing(keyspaceID, mcsutils.DefaultKeyspaceGroupID) { - tam, err := server.GetTSOAllocatorManager(mcsutils.DefaultKeyspaceGroupID) + if server.IsKeyspaceServing(keyspaceID, constant.DefaultKeyspaceGroupID) { + tam, err := server.GetTSOAllocatorManager(constant.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(tam) served = true @@ -140,10 +140,10 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestKeyspacesServedByDefaultKeysp // won't be served at this time. Default keyspace will be served by default keyspace group // all the time. for _, server := range suite.tsoCluster.GetServers() { - server.IsKeyspaceServing(mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + server.IsKeyspaceServing(constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) for _, keyspaceGroupID := range []uint32{1, 2, 3} { - server.IsKeyspaceServing(mcsutils.DefaultKeyspaceID, keyspaceGroupID) - server.IsKeyspaceServing(mcsutils.DefaultKeyspaceID, keyspaceGroupID) + server.IsKeyspaceServing(constant.DefaultKeyspaceID, keyspaceGroupID) + server.IsKeyspaceServing(constant.DefaultKeyspaceID, keyspaceGroupID) for _, keyspaceID := range []uint32{1, 2, 3} { if server.IsKeyspaceServing(keyspaceID, keyspaceGroupID) { tam, err := server.GetTSOAllocatorManager(keyspaceGroupID) @@ -500,7 +500,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMembers() { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) // wait for finishing alloc nodes - waitFinishAllocNodes(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID) + waitFinishAllocNodes(re, suite.pdLeaderServer, constant.DefaultKeyspaceGroupID) testConfig := map[string]string{ "config": "1", "tso_keyspace_group_id": "0", @@ -510,7 +510,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMembers() { Name: "test_keyspace", Config: testConfig, }) - waitFinishAllocNodes(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID) + waitFinishAllocNodes(re, suite.pdLeaderServer, constant.DefaultKeyspaceGroupID) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) } @@ -519,7 +519,7 @@ func waitFinishAllocNodes(re *require.Assertions, server *tests.TestServer, grou testutil.Eventually(re, func() bool { kg := handlersutil.MustLoadKeyspaceGroupByID(re, server, groupID) re.Equal(groupID, kg.ID) - return len(kg.Members) == mcsutils.DefaultKeyspaceGroupReplicaCount + return len(kg.Members) == constant.DefaultKeyspaceGroupReplicaCount }) } @@ -560,7 +560,7 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { return err == nil }) - waitFinishSplit(re, leaderServer, 0, 1, []uint32{mcsutils.DefaultKeyspaceID, 1}, []uint32{2}) + waitFinishSplit(re, leaderServer, 0, 1, []uint32{constant.DefaultKeyspaceID, 1}, []uint32{2}) // Then split keyspace group 0 to 2 with keyspace 1. testutil.Eventually(re, func() bool { @@ -568,7 +568,7 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { return err == nil }) - waitFinishSplit(re, leaderServer, 0, 2, []uint32{mcsutils.DefaultKeyspaceID}, []uint32{1}) + waitFinishSplit(re, leaderServer, 0, 2, []uint32{constant.DefaultKeyspaceID}, []uint32{1}) // Check the keyspace group 0 is split to 1 and 2. kg0 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 0) @@ -619,12 +619,12 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMerge() { err = suite.tsoCluster.GetPrimaryServer(222, firstID).ResetTS(tsoutil.GenerateTS(&ts), false, true, firstID) re.NoError(err) // Merge the keyspace group `firstID` and `secondID` to the default keyspace group. - handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ + handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, constant.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ MergeList: []uint32{firstID, secondID}, }) // Check the keyspace group `firstID` and `secondID` are merged to the default keyspace group. - kg := handlersutil.MustLoadKeyspaceGroupByID(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID) - re.Equal(mcsutils.DefaultKeyspaceGroupID, kg.ID) + kg := handlersutil.MustLoadKeyspaceGroupByID(re, suite.pdLeaderServer, constant.DefaultKeyspaceGroupID) + re.Equal(constant.DefaultKeyspaceGroupID, kg.ID) for _, keyspaceID := range []uint32{111, 222, 333} { re.Contains(kg.Keyspaces, keyspaceID) } @@ -632,7 +632,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMerge() { // Check the merged TSO from the default keyspace group is greater than the TSO from the keyspace group`firstID`. var mergedTS pdpb.Timestamp testutil.Eventually(re, func() bool { - mergedTS, err = suite.requestTSO(re, 333, mcsutils.DefaultKeyspaceGroupID) + mergedTS, err = suite.requestTSO(re, 333, constant.DefaultKeyspaceGroupID) if err != nil { re.ErrorIs(err, errs.ErrKeyspaceGroupIsMerging) } @@ -662,11 +662,11 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMergeClient() // Request the TSO for keyspace 222 concurrently via client. cancel := suite.dispatchClient(re, 222, id) // Merge the keyspace group 1 to the default keyspace group. - handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ + handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, constant.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ MergeList: []uint32{id}, }) // Wait for the default keyspace group to finish the merge. - waitFinishMerge(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, []uint32{111, 222, 333}) + waitFinishMerge(re, suite.pdLeaderServer, constant.DefaultKeyspaceGroupID, []uint32{111, 222, 333}) // Stop the client. cancel() } @@ -695,7 +695,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMergeBeforeIn re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/failedToSaveTimestamp", `return(true)`)) // Request the TSO for the default keyspace concurrently via client. id := suite.allocID() - cancel := suite.dispatchClient(re, mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + cancel := suite.dispatchClient(re, constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) // Create the keyspace group 1 with keyspaces [111, 222, 333]. handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ KeyspaceGroups: []*endpoint.KeyspaceGroup{ @@ -708,11 +708,11 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMergeBeforeIn }, }) // Merge the keyspace group `id` to the default keyspace group. - handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ + handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, constant.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ MergeList: []uint32{id}, }) // Wait for the default keyspace group to finish the merge. - waitFinishMerge(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, []uint32{111, 222, 333}) + waitFinishMerge(re, suite.pdLeaderServer, constant.DefaultKeyspaceGroupID, []uint32{111, 222, 333}) // Stop the client. cancel() re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/failedToSaveTimestamp")) @@ -757,7 +757,7 @@ func TestGetTSOImmediately(t *testing.T) { return err == nil }) - waitFinishSplit(re, leaderServer, 0, 1, []uint32{mcsutils.DefaultKeyspaceID, 1}, []uint32{2}) + waitFinishSplit(re, leaderServer, 0, 1, []uint32{constant.DefaultKeyspaceID, 1}, []uint32{2}) kg0 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 0) kg1 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 1) @@ -823,11 +823,11 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestKeyspaceGroupMergeIntoDefault re.NotNil(svr) } // Merge all the keyspace groups into the default keyspace group. - handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ + handlersutil.MustMergeKeyspaceGroup(re, suite.pdLeaderServer, constant.DefaultKeyspaceGroupID, &handlers.MergeKeyspaceGroupsParams{ MergeAllIntoDefault: true, }) // Wait for all the keyspace groups to be merged. - waitFinishMerge(re, suite.pdLeaderServer, mcsutils.DefaultKeyspaceGroupID, keyspaces) + waitFinishMerge(re, suite.pdLeaderServer, constant.DefaultKeyspaceGroupID, keyspaces) // Check if all the keyspace groups are merged. groups = handlersutil.MustLoadKeyspaceGroups(re, suite.pdLeaderServer, "0", "0") re.Len(groups, 1) diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 0d44e70414c..980f0db6f12 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -34,7 +34,7 @@ import ( "github.com/tikv/pd/pkg/mcs/discovery" tso "github.com/tikv/pd/pkg/mcs/tso/server" tsoapi "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/tempurl" @@ -144,17 +144,15 @@ func (suite *tsoServerTestSuite) TestParticipantStartWithAdvertiseListenAddr() { return s.IsServing() }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) - member, err := s.GetMember(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) + member, err := s.GetMember(constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) re.NoError(err) - re.Equal(fmt.Sprintf("%s-%05d", cfg.AdvertiseListenAddr, utils.DefaultKeyspaceGroupID), member.Name()) + re.Equal(fmt.Sprintf("%s-%05d", cfg.AdvertiseListenAddr, constant.DefaultKeyspaceGroupID), member.Name()) } func TestTSOPath(t *testing.T) { re := require.New(t) checkTSOPath(re, true /*isAPIServiceMode*/) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/tso/server/skipWaitAPIServiceReady", "return(true)")) checkTSOPath(re, false /*isAPIServiceMode*/) - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/tso/server/skipWaitAPIServiceReady")) } func checkTSOPath(re *require.Assertions, isAPIServiceMode bool) { @@ -212,57 +210,6 @@ func getEtcdTimestampKeyNum(re *require.Assertions, client *clientv3.Client) int return count } -func TestWaitAPIServiceReady(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - startCluster := func(isAPIServiceMode bool) (cluster *tests.TestCluster, backendEndpoints string) { - var err error - if isAPIServiceMode { - cluster, err = tests.NewTestAPICluster(ctx, 1) - } else { - cluster, err = tests.NewTestCluster(ctx, 1) - } - re.NoError(err) - err = cluster.RunInitialServers() - re.NoError(err) - leaderName := cluster.WaitLeader() - re.NotEmpty(leaderName) - pdLeader := cluster.GetServer(leaderName) - return cluster, pdLeader.GetAddr() - } - - // tso server cannot be started because the pd server is not ready as api service. - cluster, backendEndpoints := startCluster(false /*isAPIServiceMode*/) - sctx, scancel := context.WithTimeout(ctx, time.Second*10) - defer scancel() - s, _, err := tests.StartSingleTSOTestServerWithoutCheck(sctx, re, backendEndpoints, tempurl.Alloc()) - re.Error(err) - re.Nil(s) - cluster.Destroy() - - // tso server can be started because the pd server is ready as api service. - cluster, backendEndpoints = startCluster(true /*isAPIServiceMode*/) - sctx, scancel = context.WithTimeout(ctx, time.Second*10) - defer scancel() - s, cleanup, err := tests.StartSingleTSOTestServerWithoutCheck(sctx, re, backendEndpoints, tempurl.Alloc()) - re.NoError(err) - defer cluster.Destroy() - defer cleanup() - - for i := 0; i < 12; i++ { - select { - case <-time.After(time.Second): - case <-sctx.Done(): - return - } - if s != nil && s.IsServing() { - break - } - } -} - type APIServerForward struct { re *require.Assertions ctx context.Context @@ -305,10 +252,10 @@ func (suite *APIServerForward) ShutDown() { etcdClient := suite.pdLeader.GetEtcdClient() clusterID := strconv.FormatUint(suite.pdLeader.GetClusterID(), 10) - endpoints, err := discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) + endpoints, err := discovery.Discover(etcdClient, clusterID, constant.TSOServiceName) re.NoError(err) if len(endpoints) != 0 { - endpoints, err = discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) + endpoints, err = discovery.Discover(etcdClient, clusterID, constant.TSOServiceName) re.NoError(err) re.Empty(endpoints) } @@ -341,15 +288,15 @@ func TestForwardTSOWhenPrimaryChanged(t *testing.T) { tc.WaitForDefaultPrimaryServing(re) // can use the tso-related interface with old primary - oldPrimary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) + oldPrimary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, constant.TSOServiceName) re.True(exist) suite.checkAvailableTSO(re) // can use the tso-related interface with new primary tc.DestroyServer(oldPrimary) - time.Sleep(time.Duration(utils.DefaultLeaderLease) * time.Second) // wait for leader lease timeout + time.Sleep(time.Duration(constant.DefaultLeaderLease) * time.Second) // wait for leader lease timeout tc.WaitForDefaultPrimaryServing(re) - primary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) + primary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, constant.TSOServiceName) re.True(exist) re.NotEqual(oldPrimary, primary) suite.checkAvailableTSO(re) @@ -363,8 +310,8 @@ func TestForwardTSOWhenPrimaryChanged(t *testing.T) { } } tc.WaitForDefaultPrimaryServing(re) - time.Sleep(time.Duration(utils.DefaultLeaderLease) * time.Second) // wait for leader lease timeout - primary, exist = suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) + time.Sleep(time.Duration(constant.DefaultLeaderLease) * time.Second) // wait for leader lease timeout + primary, exist = suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, constant.TSOServiceName) re.True(exist) re.Equal(oldPrimary, primary) suite.checkAvailableTSO(re) @@ -381,7 +328,7 @@ func TestResignTSOPrimaryForward(t *testing.T) { tc.WaitForDefaultPrimaryServing(re) for j := 0; j < 10; j++ { - tc.ResignPrimary(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) + tc.ResignPrimary(constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) tc.WaitForDefaultPrimaryServing(re) var err error for i := 0; i < 3; i++ { // try 3 times @@ -465,7 +412,7 @@ func (suite *APIServerForward) checkForwardTSOUnexpectedToFollower(checkTSO func // get follower's address servers := tc.GetServers() - oldPrimary := tc.GetPrimaryServer(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID).GetAddr() + oldPrimary := tc.GetPrimaryServer(constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID).GetAddr() var follower string for addr := range servers { if addr != oldPrimary { @@ -476,8 +423,8 @@ func (suite *APIServerForward) checkForwardTSOUnexpectedToFollower(checkTSO func re.NotEmpty(follower) // write follower's address to cache to simulate cache is not updated. - suite.pdLeader.GetServer().SetServicePrimaryAddr(utils.TSOServiceName, follower) - errorAddr, ok := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) + suite.pdLeader.GetServer().SetServicePrimaryAddr(constant.TSOServiceName, follower) + errorAddr, ok := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, constant.TSOServiceName) re.True(ok) re.Equal(follower, errorAddr) @@ -486,7 +433,7 @@ func (suite *APIServerForward) checkForwardTSOUnexpectedToFollower(checkTSO func // test tso request will success after cache is updated suite.checkAvailableTSO(re) - newPrimary, exist2 := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) + newPrimary, exist2 := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, constant.TSOServiceName) re.True(exist2) re.NotEqual(errorAddr, newPrimary) re.Equal(oldPrimary, newPrimary) @@ -569,7 +516,7 @@ func (suite *CommonTestSuite) SetupSuite() { suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 1, suite.backendEndpoints) re.NoError(err) suite.tsoCluster.WaitForDefaultPrimaryServing(re) - suite.tsoDefaultPrimaryServer = suite.tsoCluster.GetPrimaryServer(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) + suite.tsoDefaultPrimaryServer = suite.tsoCluster.GetPrimaryServer(constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) } func (suite *CommonTestSuite) TearDownSuite() { @@ -577,10 +524,10 @@ func (suite *CommonTestSuite) TearDownSuite() { suite.tsoCluster.Destroy() etcdClient := suite.pdLeader.GetEtcdClient() clusterID := strconv.FormatUint(suite.pdLeader.GetClusterID(), 10) - endpoints, err := discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) + endpoints, err := discovery.Discover(etcdClient, clusterID, constant.TSOServiceName) re.NoError(err) if len(endpoints) != 0 { - endpoints, err = discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) + endpoints, err = discovery.Discover(etcdClient, clusterID, constant.TSOServiceName) re.NoError(err) re.Empty(endpoints) } @@ -610,7 +557,7 @@ func (suite *CommonTestSuite) TestBootstrapDefaultKeyspaceGroup() { var kgs []*endpoint.KeyspaceGroup re.NoError(json.Unmarshal(respString, &kgs)) re.Len(kgs, 1) - re.Equal(utils.DefaultKeyspaceGroupID, kgs[0].ID) + re.Equal(constant.DefaultKeyspaceGroupID, kgs[0].ID) re.Equal(endpoint.Basic.String(), kgs[0].UserKind) re.Empty(kgs[0].SplitState) re.Empty(kgs[0].KeyspaceLookupTable) diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index bb721e6c5dd..f5f33240d18 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -31,7 +31,7 @@ import ( pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/testutil" bs "github.com/tikv/pd/pkg/basicserver" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/tempurl" @@ -108,8 +108,8 @@ func (suite *tsoClientTestSuite) SetupSuite() { re.NoError(err) innerClient, ok := client.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) re.True(ok) - re.Equal(mcsutils.NullKeyspaceID, innerClient.GetServiceDiscovery().GetKeyspaceID()) - re.Equal(mcsutils.DefaultKeyspaceGroupID, innerClient.GetServiceDiscovery().GetKeyspaceGroupID()) + re.Equal(constant.NullKeyspaceID, innerClient.GetServiceDiscovery().GetKeyspaceID()) + re.Equal(constant.DefaultKeyspaceGroupID, innerClient.GetServiceDiscovery().GetKeyspaceGroupID()) mcs.WaitForTSOServiceAvailable(suite.ctx, re, client) suite.clients = make([]pd.Client, 0) suite.clients = append(suite.clients, client) @@ -121,7 +121,7 @@ func (suite *tsoClientTestSuite) SetupSuite() { keyspaceGroupID uint32 keyspaceIDs []uint32 }{ - {0, []uint32{mcsutils.DefaultKeyspaceID, 10}}, + {0, []uint32{constant.DefaultKeyspaceID, 10}}, {1, []uint32{1, 11}}, {2, []uint32{2}}, } @@ -387,7 +387,7 @@ func (suite *tsoClientTestSuite) TestRandomResignLeader() { for _, keyspaceID := range keyspaceIDs { go func(keyspaceID uint32) { defer wg.Done() - err := suite.tsoCluster.ResignPrimary(keyspaceID, mcsutils.DefaultKeyspaceGroupID) + err := suite.tsoCluster.ResignPrimary(keyspaceID, constant.DefaultKeyspaceGroupID) re.NoError(err) suite.tsoCluster.WaitForPrimaryServing(re, keyspaceID, 0) }(keyspaceID) @@ -480,11 +480,9 @@ func TestMixedTSODeployment(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval", "return(true)")) re.NoError(failpoint.Enable("github.com/tikv/pd/client/skipUpdateServiceMode", "return(true)")) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/tso/server/skipWaitAPIServiceReady", "return(true)")) defer func() { re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval")) re.NoError(failpoint.Disable("github.com/tikv/pd/client/skipUpdateServiceMode")) - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/tso/server/skipWaitAPIServiceReady")) }() ctx, cancel := context.WithCancel(context.Background()) diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index 535f01cc33e..18466ca0da7 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/apiv2/handlers" "github.com/tikv/pd/tests" @@ -73,8 +73,8 @@ func (suite *keyspaceTestSuite) TestCreateLoadKeyspace() { loaded := mustLoadKeyspaces(re, suite.server, created.Name) re.Equal(created, loaded) } - defaultKeyspace := mustLoadKeyspaces(re, suite.server, utils.DefaultKeyspaceName) - re.Equal(utils.DefaultKeyspaceName, defaultKeyspace.Name) + defaultKeyspace := mustLoadKeyspaces(re, suite.server, constant.DefaultKeyspaceName) + re.Equal(constant.DefaultKeyspaceName, defaultKeyspace.Name) re.Equal(keyspacepb.KeyspaceState_ENABLED, defaultKeyspace.State) } @@ -125,7 +125,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { re.Equal(keyspacepb.KeyspaceState_TOMBSTONE, tombstone.State) } // Changing default keyspace's state is NOT allowed. - success, _ := sendUpdateStateRequest(re, suite.server, utils.DefaultKeyspaceName, &handlers.UpdateStateParam{State: "disabled"}) + success, _ := sendUpdateStateRequest(re, suite.server, constant.DefaultKeyspaceName, &handlers.UpdateStateParam{State: "disabled"}) re.False(success) } @@ -139,7 +139,7 @@ func (suite *keyspaceTestSuite) TestLoadRangeKeyspace() { for i, created := range keyspaces { re.Equal(created, loadResponse.Keyspaces[i+1].KeyspaceMeta) } - re.Equal(utils.DefaultKeyspaceName, loadResponse.Keyspaces[0].Name) + re.Equal(constant.DefaultKeyspaceName, loadResponse.Keyspaces[0].Name) re.Equal(keyspacepb.KeyspaceState_ENABLED, loadResponse.Keyspaces[0].State) } diff --git a/tests/server/apiv2/handlers/tso_keyspace_group_test.go b/tests/server/apiv2/handlers/tso_keyspace_group_test.go index 2bf2db715fa..91a07ccd6b1 100644 --- a/tests/server/apiv2/handlers/tso_keyspace_group_test.go +++ b/tests/server/apiv2/handlers/tso_keyspace_group_test.go @@ -20,7 +20,7 @@ import ( "testing" "github.com/stretchr/testify/suite" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/server/apiv2/handlers" "github.com/tikv/pd/tests" @@ -97,7 +97,7 @@ func (suite *keyspaceGroupTestSuite) TestCreateKeyspaceGroups() { // invalid ID. kgs = &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ { - ID: utils.MaxKeyspaceGroupCount + 1, + ID: constant.MaxKeyspaceGroupCount + 1, UserKind: endpoint.Standard.String(), }, }} @@ -138,7 +138,7 @@ func (suite *keyspaceGroupTestSuite) TestSplitKeyspaceGroup() { ID: uint32(1), UserKind: endpoint.Standard.String(), Keyspaces: []uint32{111, 222, 333}, - Members: make([]endpoint.KeyspaceGroupMember, utils.DefaultKeyspaceGroupReplicaCount), + Members: make([]endpoint.KeyspaceGroupMember, constant.DefaultKeyspaceGroupReplicaCount), }, }} diff --git a/tests/testutil.go b/tests/testutil.go index 2989ba7b010..51b4a410f8a 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -40,7 +40,7 @@ import ( scheduling "github.com/tikv/pd/pkg/mcs/scheduling/server" sc "github.com/tikv/pd/pkg/mcs/scheduling/server/config" tso "github.com/tikv/pd/pkg/mcs/tso/server" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/testutil" @@ -422,7 +422,7 @@ func (s *SchedulingTestEnvironment) startCluster(m SchedulerMode) { cluster.SetSchedulingCluster(tc) time.Sleep(200 * time.Millisecond) // wait for scheduling cluster to update member testutil.Eventually(re, func() bool { - return cluster.GetLeaderServer().GetServer().GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) + return cluster.GetLeaderServer().GetServer().GetRaftCluster().IsServiceIndependent(constant.SchedulingServiceName) }) s.clusters[APIMode] = cluster } diff --git a/tests/tso_cluster.go b/tests/tso_cluster.go index e1fdb6d69ca..50495bc4abc 100644 --- a/tests/tso_cluster.go +++ b/tests/tso_cluster.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" tso "github.com/tikv/pd/pkg/mcs/tso/server" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" @@ -180,7 +180,7 @@ func (tc *TestTSOCluster) WaitForPrimaryServing(re *require.Assertions, keyspace // WaitForDefaultPrimaryServing waits for one of servers being elected to be the primary/leader of the default keyspace. func (tc *TestTSOCluster) WaitForDefaultPrimaryServing(re *require.Assertions) *tso.Server { - return tc.WaitForPrimaryServing(re, mcsutils.DefaultKeyspaceID, mcsutils.DefaultKeyspaceGroupID) + return tc.WaitForPrimaryServing(re, constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) } // GetServer returns the TSO server by the given address. @@ -203,7 +203,7 @@ func (tc *TestTSOCluster) GetKeyspaceGroupMember() (members []endpoint.KeyspaceG for _, server := range tc.servers { members = append(members, endpoint.KeyspaceGroupMember{ Address: server.GetAddr(), - Priority: mcsutils.DefaultKeyspaceGroupReplicaPriority, + Priority: constant.DefaultKeyspaceGroupReplicaPriority, }) } return diff --git a/tools/go.mod b/tools/go.mod index aef95dfa660..7152d25fed9 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -24,7 +24,6 @@ require ( github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 github.com/pingcap/kvproto v0.0.0-20240716095229-5f7ffec83ea7 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 - github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/prometheus/common v0.51.1 github.com/spf13/cobra v1.8.0 @@ -130,6 +129,7 @@ require ( github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/prometheus/client_model v0.6.0 // indirect diff --git a/tools/pd-api-bench/config/config.go b/tools/pd-api-bench/config/config.go index 0320665c29b..d493479348d 100644 --- a/tools/pd-api-bench/config/config.go +++ b/tools/pd-api-bench/config/config.go @@ -15,8 +15,8 @@ package config import ( + "github.com/pingcap/errors" "github.com/pingcap/log" - "github.com/pkg/errors" flag "github.com/spf13/pflag" "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/tools/pd-api-bench/cases" diff --git a/tools/pd-api-bench/main.go b/tools/pd-api-bench/main.go index 78bec1e1d01..cb472fb5973 100644 --- a/tools/pd-api-bench/main.go +++ b/tools/pd-api-bench/main.go @@ -29,8 +29,8 @@ import ( "github.com/gin-contrib/gzip" "github.com/gin-contrib/pprof" "github.com/gin-gonic/gin" + "github.com/pingcap/errors" "github.com/pingcap/log" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" flag "github.com/spf13/pflag" pd "github.com/tikv/pd/client" diff --git a/tools/pd-ctl/pdctl/command/keyspace_group_command.go b/tools/pd-ctl/pdctl/command/keyspace_group_command.go index 9c3a45f4744..f315417e555 100644 --- a/tools/pd-ctl/pdctl/command/keyspace_group_command.go +++ b/tools/pd-ctl/pdctl/command/keyspace_group_command.go @@ -23,7 +23,7 @@ import ( "strings" "github.com/spf13/cobra" - mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" ) @@ -266,7 +266,7 @@ func mergeKeyspaceGroupCommandFunc(cmd *cobra.Command, args []string) { return } targetGroupID = uint32(target) - if targetGroupID != mcsutils.DefaultKeyspaceGroupID { + if targetGroupID != constant.DefaultKeyspaceGroupID { cmd.Println("Unable to merge all keyspace groups into a non-default keyspace group") return } diff --git a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go index 1e3763d5d6e..2acb38af47e 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go @@ -24,7 +24,7 @@ import ( "github.com/pingcap/failpoint" "github.com/stretchr/testify/require" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/apiv2/handlers" @@ -51,22 +51,22 @@ func TestKeyspaceGroup(t *testing.T) { cmd := ctl.GetRootCmd() // Show keyspace group information. - defaultKeyspaceGroupID := fmt.Sprintf("%d", utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroupID := fmt.Sprintf("%d", constant.DefaultKeyspaceGroupID) args := []string{"-u", pdAddr, "keyspace-group"} output, err := tests.ExecuteCommand(cmd, append(args, defaultKeyspaceGroupID)...) re.NoError(err) var keyspaceGroup endpoint.KeyspaceGroup err = json.Unmarshal(output, &keyspaceGroup) re.NoError(err) - re.Equal(utils.DefaultKeyspaceGroupID, keyspaceGroup.ID) - re.Contains(keyspaceGroup.Keyspaces, utils.DefaultKeyspaceID) + re.Equal(constant.DefaultKeyspaceGroupID, keyspaceGroup.ID) + re.Contains(keyspaceGroup.Keyspaces, constant.DefaultKeyspaceID) // Split keyspace group. handlersutil.MustCreateKeyspaceGroup(re, leaderServer, &handlers.CreateKeyspaceGroupParams{ KeyspaceGroups: []*endpoint.KeyspaceGroup{ { ID: 1, UserKind: endpoint.Standard.String(), - Members: make([]endpoint.KeyspaceGroupMember, utils.DefaultKeyspaceGroupReplicaCount), + Members: make([]endpoint.KeyspaceGroupMember, constant.DefaultKeyspaceGroupReplicaCount), Keyspaces: []uint32{111, 222, 333}, }, }, @@ -171,7 +171,7 @@ func TestExternalAllocNodeWhenStart(t *testing.T) { re.NoError(leaderServer.BootstrapCluster()) // check keyspace group information. - defaultKeyspaceGroupID := fmt.Sprintf("%d", utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroupID := fmt.Sprintf("%d", constant.DefaultKeyspaceGroupID) args := []string{"-u", pdAddr, "keyspace-group"} testutil.Eventually(re, func() bool { output, err := tests.ExecuteCommand(cmd, append(args, defaultKeyspaceGroupID)...) @@ -215,7 +215,7 @@ func TestSetNodeAndPriorityKeyspaceGroup(t *testing.T) { re.NoError(leaderServer.BootstrapCluster()) // set-node keyspace group. - defaultKeyspaceGroupID := fmt.Sprintf("%d", utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroupID := fmt.Sprintf("%d", constant.DefaultKeyspaceGroupID) testutil.Eventually(re, func() bool { args := []string{"-u", pdAddr, "keyspace-group", "set-node", defaultKeyspaceGroupID, tsoAddrs[0], tsoAddrs[1]} output, err := tests.ExecuteCommand(cmd, args...) @@ -244,7 +244,7 @@ func TestSetNodeAndPriorityKeyspaceGroup(t *testing.T) { var keyspaceGroup endpoint.KeyspaceGroup err = json.Unmarshal(output, &keyspaceGroup) re.NoError(err) - re.Equal(utils.DefaultKeyspaceGroupID, keyspaceGroup.ID) + re.Equal(constant.DefaultKeyspaceGroupID, keyspaceGroup.ID) re.Len(keyspaceGroup.Members, 2) for _, member := range keyspaceGroup.Members { re.Contains(tsoAddrs, member.Address) @@ -527,7 +527,7 @@ func TestShowKeyspaceGroupPrimary(t *testing.T) { tc.WaitLeader() leaderServer := tc.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) - defaultKeyspaceGroupID := fmt.Sprintf("%d", utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroupID := fmt.Sprintf("%d", constant.DefaultKeyspaceGroupID) // check keyspace group 0 information. var keyspaceGroup endpoint.KeyspaceGroup @@ -537,7 +537,7 @@ func TestShowKeyspaceGroupPrimary(t *testing.T) { re.NoError(err) err = json.Unmarshal(output, &keyspaceGroup) re.NoError(err) - re.Equal(utils.DefaultKeyspaceGroupID, keyspaceGroup.ID) + re.Equal(constant.DefaultKeyspaceGroupID, keyspaceGroup.ID) return len(keyspaceGroup.Members) == 2 }) for _, member := range keyspaceGroup.Members { diff --git a/tools/pd-ctl/tests/keyspace/keyspace_test.go b/tools/pd-ctl/tests/keyspace/keyspace_test.go index 54c25fc2099..b0cb1a2293e 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_test.go @@ -27,7 +27,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/keyspace" - "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/testutil" api "github.com/tikv/pd/server/apiv2/handlers" "github.com/tikv/pd/server/config" @@ -64,7 +64,7 @@ func TestKeyspace(t *testing.T) { tc.WaitLeader() leaderServer := tc.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) - defaultKeyspaceGroupID := fmt.Sprintf("%d", utils.DefaultKeyspaceGroupID) + defaultKeyspaceGroupID := fmt.Sprintf("%d", constant.DefaultKeyspaceGroupID) var k api.KeyspaceMeta keyspaceName := "keyspace_1" From 917f24a73359ab8cf35e8e8d0687a01a32ba55ad Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 13 Aug 2024 11:43:31 +0800 Subject: [PATCH 11/42] client: use defer for cancel (#8520) ref tikv/pd#4399 Signed-off-by: Ryan Leung --- client/client.go | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/client/client.go b/client/client.go index 8ba03161227..d5cf7cf28d0 100644 --- a/client/client.go +++ b/client/client.go @@ -809,14 +809,13 @@ func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { defer func() { cmdDurationGetAllMembers.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() req := &pdpb.GetMembersRequest{Header: c.requestHeader()} protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return nil, errs.ErrClientGetProtoClient } resp, err := protoClient.GetMembers(ctx, req) - cancel() if err = c.respForErr(cmdFailDurationGetAllMembers, start, err, resp.GetHeader()); err != nil { return nil, err } @@ -934,17 +933,16 @@ func (c *client) GetMinTS(ctx context.Context) (physical int64, logical int64, e return 0, 0, errs.ErrClientGetMinTSO.FastGenByArgs("undefined service mode") } ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() // Call GetMinTS API to get the minimal TS from the API leader. protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return 0, 0, errs.ErrClientGetProtoClient } resp, err := protoClient.GetMinTS(ctx, &pdpb.GetMinTSRequest{ Header: c.requestHeader(), }) - cancel() if err != nil { if strings.Contains(err.Error(), "Unimplemented") { // If the method is not supported, we fallback to GetTS. @@ -1297,17 +1295,16 @@ func (c *client) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, e defer func() { cmdDurationGetStore.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() req := &pdpb.GetStoreRequest{ Header: c.requestHeader(), StoreId: storeID, } protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return nil, errs.ErrClientGetProtoClient } resp, err := protoClient.GetStore(ctx, req) - cancel() if err = c.respForErr(cmdFailedDurationGetStore, start, err, resp.GetHeader()); err != nil { return nil, err @@ -1342,17 +1339,16 @@ func (c *client) GetAllStores(ctx context.Context, opts ...GetStoreOption) ([]*m defer func() { cmdDurationGetAllStores.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() req := &pdpb.GetAllStoresRequest{ Header: c.requestHeader(), ExcludeTombstoneStores: options.excludeTombstone, } protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return nil, errs.ErrClientGetProtoClient } resp, err := protoClient.GetAllStores(ctx, req) - cancel() if err = c.respForErr(cmdFailedDurationGetAllStores, start, err, resp.GetHeader()); err != nil { return nil, err @@ -1370,17 +1366,16 @@ func (c *client) UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint6 defer func() { cmdDurationUpdateGCSafePoint.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() req := &pdpb.UpdateGCSafePointRequest{ Header: c.requestHeader(), SafePoint: safePoint, } protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return 0, errs.ErrClientGetProtoClient } resp, err := protoClient.UpdateGCSafePoint(ctx, req) - cancel() if err = c.respForErr(cmdFailedDurationUpdateGCSafePoint, start, err, resp.GetHeader()); err != nil { return 0, err @@ -1402,6 +1397,7 @@ func (c *client) UpdateServiceGCSafePoint(ctx context.Context, serviceID string, defer func() { cmdDurationUpdateServiceGCSafePoint.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() req := &pdpb.UpdateServiceGCSafePointRequest{ Header: c.requestHeader(), ServiceId: []byte(serviceID), @@ -1410,11 +1406,9 @@ func (c *client) UpdateServiceGCSafePoint(ctx context.Context, serviceID string, } protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return 0, errs.ErrClientGetProtoClient } resp, err := protoClient.UpdateServiceGCSafePoint(ctx, req) - cancel() if err = c.respForErr(cmdFailedDurationUpdateServiceGCSafePoint, start, err, resp.GetHeader()); err != nil { return 0, err @@ -1436,6 +1430,7 @@ func (c *client) scatterRegionsWithGroup(ctx context.Context, regionID uint64, g defer func() { cmdDurationScatterRegion.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() req := &pdpb.ScatterRegionRequest{ Header: c.requestHeader(), RegionId: regionID, @@ -1443,11 +1438,9 @@ func (c *client) scatterRegionsWithGroup(ctx context.Context, regionID uint64, g } protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return errs.ErrClientGetProtoClient } resp, err := protoClient.ScatterRegion(ctx, req) - cancel() if err != nil { return err } @@ -1489,7 +1482,6 @@ func (c *client) SplitAndScatterRegions(ctx context.Context, splitKeys [][]byte, protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return nil, errs.ErrClientGetProtoClient } return protoClient.SplitAndScatterRegions(ctx, req) @@ -1512,7 +1504,6 @@ func (c *client) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOpe } protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return nil, errs.ErrClientGetProtoClient } return protoClient.GetOperator(ctx, req) @@ -1539,7 +1530,6 @@ func (c *client) SplitRegions(ctx context.Context, splitKeys [][]byte, opts ...R } protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return nil, errs.ErrClientGetProtoClient } return protoClient.SplitRegions(ctx, req) @@ -1559,6 +1549,7 @@ func (c *client) scatterRegionsWithOptions(ctx context.Context, regionsID []uint opt(options) } ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() req := &pdpb.ScatterRegionRequest{ Header: c.requestHeader(), Group: options.group, @@ -1569,11 +1560,9 @@ func (c *client) scatterRegionsWithOptions(ctx context.Context, regionsID []uint protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { - cancel() return nil, errs.ErrClientGetProtoClient } resp, err := protoClient.ScatterRegion(ctx, req) - cancel() if err != nil { return nil, err From 1c1cd99ba73eaf1fd80a03310221b37ec48cf7d1 Mon Sep 17 00:00:00 2001 From: Hu# Date: Tue, 13 Aug 2024 12:58:31 +0800 Subject: [PATCH 12/42] pdms: support primary/transfer api for scheduling and tso (#8157) ref tikv/pd#5766, close tikv/pd#7995 Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/election/leadership.go | 53 +++- pkg/election/leadership_test.go | 4 +- pkg/election/lease.go | 24 +- pkg/election/lease_test.go | 19 +- pkg/mcs/scheduling/server/apis/v1/api.go | 36 +++ pkg/mcs/scheduling/server/server.go | 33 ++ pkg/mcs/scheduling/server/testutil.go | 2 + pkg/mcs/tso/server/apis/v1/api.go | 49 +++ pkg/mcs/utils/expected_primary.go | 182 +++++++++++ pkg/member/member.go | 4 +- pkg/member/participant.go | 16 + pkg/tso/global_allocator.go | 49 ++- tests/integrations/mcs/members/member_test.go | 283 +++++++++++++++++- 13 files changed, 705 insertions(+), 49 deletions(-) create mode 100644 pkg/mcs/utils/expected_primary.go diff --git a/pkg/election/leadership.go b/pkg/election/leadership.go index f252eabe072..1361d685b57 100644 --- a/pkg/election/leadership.go +++ b/pkg/election/leadership.go @@ -71,6 +71,9 @@ type Leadership struct { // campaignTimes is used to record the campaign times of the leader within `campaignTimesRecordTimeout`. // It is ordered by time to prevent the leader from campaigning too frequently. campaignTimes []time.Time + // primaryWatch is for the primary watch only, + // which is used to reuse `Watch` interface in `Leadership`. + primaryWatch atomic.Bool } // NewLeadership creates a new Leadership. @@ -84,17 +87,18 @@ func NewLeadership(client *clientv3.Client, leaderKey, purpose string) *Leadersh return leadership } -// getLease gets the lease of leadership, only if leadership is valid, +// GetLease gets the lease of leadership, only if leadership is valid, // i.e. the owner is a true leader, the lease is not nil. -func (ls *Leadership) getLease() *lease { +func (ls *Leadership) GetLease() *Lease { l := ls.lease.Load() if l == nil { return nil } - return l.(*lease) + return l.(*Lease) } -func (ls *Leadership) setLease(lease *lease) { +// SetLease sets the lease of leadership. +func (ls *Leadership) SetLease(lease *Lease) { ls.lease.Store(lease) } @@ -114,6 +118,16 @@ func (ls *Leadership) GetLeaderKey() string { return ls.leaderKey } +// SetPrimaryWatch sets the primary watch flag. +func (ls *Leadership) SetPrimaryWatch(val bool) { + ls.primaryWatch.Store(val) +} + +// IsPrimary gets the primary watch flag. +func (ls *Leadership) IsPrimary() bool { + return ls.primaryWatch.Load() +} + // GetCampaignTimesNum is used to get the campaign times of the leader within `campaignTimesRecordTimeout`. // Need to make sure `AddCampaignTimes` is called before this function. func (ls *Leadership) GetCampaignTimesNum() int { @@ -152,18 +166,19 @@ func (ls *Leadership) AddCampaignTimes() { func (ls *Leadership) Campaign(leaseTimeout int64, leaderData string, cmps ...clientv3.Cmp) error { ls.leaderValue = leaderData // Create a new lease to campaign - newLease := &lease{ - Purpose: ls.purpose, - client: ls.client, - lease: clientv3.NewLease(ls.client), - } - ls.setLease(newLease) + newLease := NewLease(ls.client, ls.purpose) + ls.SetLease(newLease) failpoint.Inject("skipGrantLeader", func(val failpoint.Value) { + name, ok := val.(string) + if len(name) == 0 { + // return directly when not set the name + failpoint.Return(errors.Errorf("failed to grant lease")) + } var member pdpb.Member _ = member.Unmarshal([]byte(leaderData)) - name, ok := val.(string) if ok && member.Name == name { + // only return when the name is set and the name is equal to the leader name failpoint.Return(errors.Errorf("failed to grant lease")) } }) @@ -200,12 +215,12 @@ func (ls *Leadership) Keep(ctx context.Context) { ls.keepAliveCancelFuncLock.Lock() ls.keepAliveCtx, ls.keepAliveCancelFunc = context.WithCancel(ctx) ls.keepAliveCancelFuncLock.Unlock() - go ls.getLease().KeepAlive(ls.keepAliveCtx) + go ls.GetLease().KeepAlive(ls.keepAliveCtx) } // Check returns whether the leadership is still available. func (ls *Leadership) Check() bool { - return ls != nil && ls.getLease() != nil && !ls.getLease().IsExpired() + return ls != nil && ls.GetLease() != nil && !ls.GetLease().IsExpired() } // LeaderTxn returns txn() with a leader comparison to guarantee that @@ -376,6 +391,13 @@ func (ls *Leadership) Watch(serverCtx context.Context, revision int64) { zap.Int64("revision", wresp.Header.Revision), zap.String("leader-key", ls.leaderKey), zap.String("purpose", ls.purpose)) return } + // ONLY `{service}/primary/transfer` API update primary will meet this condition. + if ev.Type == mvccpb.PUT && ls.IsPrimary() { + log.Info("current leadership is updated", zap.Int64("revision", wresp.Header.Revision), + zap.String("leader-key", ls.leaderKey), zap.ByteString("cur-value", ev.Kv.Value), + zap.String("purpose", ls.purpose)) + return + } } revision = wresp.Header.Revision + 1 } @@ -385,7 +407,7 @@ func (ls *Leadership) Watch(serverCtx context.Context, revision int64) { // Reset does some defer jobs such as closing lease, resetting lease etc. func (ls *Leadership) Reset() { - if ls == nil || ls.getLease() == nil { + if ls == nil || ls.GetLease() == nil { return } ls.keepAliveCancelFuncLock.Lock() @@ -393,5 +415,6 @@ func (ls *Leadership) Reset() { ls.keepAliveCancelFunc() } ls.keepAliveCancelFuncLock.Unlock() - ls.getLease().Close() + ls.GetLease().Close() + ls.SetPrimaryWatch(false) } diff --git a/pkg/election/leadership_test.go b/pkg/election/leadership_test.go index 40f0bcbee23..e16c5842542 100644 --- a/pkg/election/leadership_test.go +++ b/pkg/election/leadership_test.go @@ -100,9 +100,9 @@ func TestLeadership(t *testing.T) { leadership2.Keep(ctx) // Check the lease. - lease1 := leadership1.getLease() + lease1 := leadership1.GetLease() re.NotNil(lease1) - lease2 := leadership2.getLease() + lease2 := leadership2.GetLease() re.NotNil(lease2) re.True(lease1.IsExpired()) diff --git a/pkg/election/lease.go b/pkg/election/lease.go index 45d702def5e..c2e9eb97117 100644 --- a/pkg/election/lease.go +++ b/pkg/election/lease.go @@ -34,9 +34,9 @@ const ( slowRequestTime = etcdutil.DefaultSlowRequestTime ) -// lease is used as the low-level mechanism for campaigning and renewing elected leadership. +// Lease is used as the low-level mechanism for campaigning and renewing elected leadership. // The way to gain and maintain leadership is to update and keep the lease alive continuously. -type lease struct { +type Lease struct { // purpose is used to show what this election for Purpose string // etcd client and lease @@ -48,8 +48,17 @@ type lease struct { expireTime atomic.Value } +// NewLease creates a new Lease instance. +func NewLease(client *clientv3.Client, purpose string) *Lease { + return &Lease{ + Purpose: purpose, + client: client, + lease: clientv3.NewLease(client), + } +} + // Grant uses `lease.Grant` to initialize the lease and expireTime. -func (l *lease) Grant(leaseTimeout int64) error { +func (l *Lease) Grant(leaseTimeout int64) error { if l == nil { return errs.ErrEtcdGrantLease.GenWithStackByCause("lease is nil") } @@ -71,7 +80,7 @@ func (l *lease) Grant(leaseTimeout int64) error { } // Close releases the lease. -func (l *lease) Close() error { +func (l *Lease) Close() error { if l == nil { return nil } @@ -92,7 +101,7 @@ func (l *lease) Close() error { // IsExpired checks if the lease is expired. If it returns true, // current leader should step down and try to re-elect again. -func (l *lease) IsExpired() bool { +func (l *Lease) IsExpired() bool { if l == nil || l.expireTime.Load() == nil { return true } @@ -100,7 +109,7 @@ func (l *lease) IsExpired() bool { } // KeepAlive auto renews the lease and update expireTime. -func (l *lease) KeepAlive(ctx context.Context) { +func (l *Lease) KeepAlive(ctx context.Context) { defer logutil.LogPanic() if l == nil { @@ -109,6 +118,7 @@ func (l *lease) KeepAlive(ctx context.Context) { ctx, cancel := context.WithCancel(ctx) defer cancel() timeCh := l.keepAliveWorker(ctx, l.leaseTimeout/3) + defer log.Info("lease keep alive stopped", zap.String("purpose", l.Purpose)) var maxExpire time.Time timer := time.NewTimer(l.leaseTimeout) @@ -146,7 +156,7 @@ func (l *lease) KeepAlive(ctx context.Context) { } // Periodically call `lease.KeepAliveOnce` and post back latest received expire time into the channel. -func (l *lease) keepAliveWorker(ctx context.Context, interval time.Duration) <-chan time.Time { +func (l *Lease) keepAliveWorker(ctx context.Context, interval time.Duration) <-chan time.Time { ch := make(chan time.Time) go func() { diff --git a/pkg/election/lease_test.go b/pkg/election/lease_test.go index 3d8515eadb2..3a02de97239 100644 --- a/pkg/election/lease_test.go +++ b/pkg/election/lease_test.go @@ -21,7 +21,6 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" ) func TestLease(t *testing.T) { @@ -30,16 +29,8 @@ func TestLease(t *testing.T) { defer clean() // Create the lease. - lease1 := &lease{ - Purpose: "test_lease_1", - client: client, - lease: clientv3.NewLease(client), - } - lease2 := &lease{ - Purpose: "test_lease_2", - client: client, - lease: clientv3.NewLease(client), - } + lease1 := NewLease(client, "test_lease_1") + lease2 := NewLease(client, "test_lease_2") re.True(lease1.IsExpired()) re.True(lease2.IsExpired()) re.NoError(lease1.Close()) @@ -95,11 +86,7 @@ func TestLeaseKeepAlive(t *testing.T) { defer clean() // Create the lease. - lease := &lease{ - Purpose: "test_lease", - client: client, - lease: clientv3.NewLease(client), - } + lease := NewLease(client, "test_lease") re.NoError(lease.Grant(defaultLeaseTimeout)) ch := lease.keepAliveWorker(context.Background(), 2*time.Second) diff --git a/pkg/mcs/scheduling/server/apis/v1/api.go b/pkg/mcs/scheduling/server/apis/v1/api.go index ce1522d465f..8b9427a8896 100644 --- a/pkg/mcs/scheduling/server/apis/v1/api.go +++ b/pkg/mcs/scheduling/server/apis/v1/api.go @@ -32,6 +32,7 @@ import ( "github.com/tikv/pd/pkg/errs" scheserver "github.com/tikv/pd/pkg/mcs/scheduling/server" mcsutils "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/response" sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/handler" @@ -120,6 +121,7 @@ func NewService(srv *scheserver.Service) *Service { s.RegisterHotspotRouter() s.RegisterRegionsRouter() s.RegisterStoresRouter() + s.RegisterPrimaryRouter() return s } @@ -226,6 +228,12 @@ func (s *Service) RegisterConfigRouter() { regions.GET("/:id/labels", getRegionLabels) } +// RegisterPrimaryRouter registers the router of the primary handler. +func (s *Service) RegisterPrimaryRouter() { + router := s.root.Group("primary") + router.POST("transfer", transferPrimary) +} + // @Tags admin // @Summary Change the log level. // @Produce json @@ -1478,3 +1486,31 @@ func getRegionByID(c *gin.Context) { } c.Data(http.StatusOK, "application/json", b) } + +// TransferPrimary transfers the primary member to `new_primary`. +// @Tags primary +// @Summary Transfer the primary member to `new_primary`. +// @Produce json +// @Param new_primary body string false "new primary name" +// @Success 200 string string +// @Router /primary/transfer [post] +func transferPrimary(c *gin.Context) { + svr := c.MustGet(multiservicesapi.ServiceContextKey).(*scheserver.Server) + var input map[string]string + if err := c.BindJSON(&input); err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + + newPrimary := "" + if v, ok := input["new_primary"]; ok { + newPrimary = v + } + + if err := mcsutils.TransferPrimary(svr.GetClient(), svr.GetParticipant().GetExpectedPrimaryLease(), + constant.SchedulingServiceName, svr.Name(), newPrimary, 0); err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, "success") +} diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index 6ec4e444d6d..e1753cf2972 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -21,6 +21,7 @@ import ( "os" "os/signal" "runtime" + "strings" "sync" "sync/atomic" "syscall" @@ -132,6 +133,11 @@ func (s *Server) GetBackendEndpoints() string { return s.cfg.BackendEndpoints } +// GetParticipant returns the participant. +func (s *Server) GetParticipant() *member.Participant { + return s.participant +} + // SetLogLevel sets log level. func (s *Server) SetLogLevel(level string) error { if !logutil.IsLevelLegal(level) { @@ -242,6 +248,20 @@ func (s *Server) primaryElectionLoop() { log.Info("the scheduling primary has changed, try to re-campaign a primary") } + // To make sure the expected primary(if existed) and new primary are on the same server. + expectedPrimary := utils.GetExpectedPrimaryFlag(s.GetClient(), s.participant.GetLeaderPath()) + // skip campaign the primary if the expected primary is not empty and not equal to the current memberValue. + // expected primary ONLY SET BY `{service}/primary/transfer` API. + if len(expectedPrimary) > 0 && !strings.Contains(s.participant.MemberValue(), expectedPrimary) { + log.Info("skip campaigning of scheduling primary and check later", + zap.String("server-name", s.Name()), + zap.String("expected-primary-id", expectedPrimary), + zap.Uint64("member-id", s.participant.ID()), + zap.String("cur-member-value", s.participant.MemberValue())) + time.Sleep(200 * time.Millisecond) + continue + } + s.campaignLeader() } } @@ -285,7 +305,17 @@ func (s *Server) campaignLeader() { cb() } }() + // check expected primary and watch the primary. + exitPrimary := make(chan struct{}) + lease, err := utils.KeepExpectedPrimaryAlive(ctx, s.GetClient(), exitPrimary, + s.cfg.LeaderLease, s.participant.GetLeaderPath(), s.participant.MemberValue(), constant.SchedulingServiceName) + if err != nil { + log.Error("prepare scheduling primary watch error", errs.ZapError(err)) + return + } + s.participant.SetExpectedPrimaryLease(lease) s.participant.EnableLeader() + member.ServiceMemberGauge.WithLabelValues(serviceName).Set(1) log.Info("scheduling primary is ready to serve", zap.String("scheduling-primary-name", s.participant.Name())) @@ -303,6 +333,9 @@ func (s *Server) campaignLeader() { // Server is closed and it should return nil. log.Info("server is closed") return + case <-exitPrimary: + log.Info("no longer be primary because primary have been updated, the scheduling primary will step down") + return } } } diff --git a/pkg/mcs/scheduling/server/testutil.go b/pkg/mcs/scheduling/server/testutil.go index aba88945434..312365c81ab 100644 --- a/pkg/mcs/scheduling/server/testutil.go +++ b/pkg/mcs/scheduling/server/testutil.go @@ -49,12 +49,14 @@ func NewTestServer(ctx context.Context, re *require.Assertions, cfg *config.Conf // GenerateConfig generates a new config with the given options. func GenerateConfig(c *config.Config) (*config.Config, error) { arguments := []string{ + "--name=" + c.Name, "--listen-addr=" + c.ListenAddr, "--advertise-listen-addr=" + c.AdvertiseListenAddr, "--backend-endpoints=" + c.BackendEndpoints, } flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + flagSet.StringP("name", "", "", "human-readable name for this scheduling member") flagSet.BoolP("version", "V", false, "print version information and exit") flagSet.StringP("config", "", "", "config file") flagSet.StringP("backend-endpoints", "", "", "url for etcd client") diff --git a/pkg/mcs/tso/server/apis/v1/api.go b/pkg/mcs/tso/server/apis/v1/api.go index d5c4cd4ec48..19b3a1be612 100644 --- a/pkg/mcs/tso/server/apis/v1/api.go +++ b/pkg/mcs/tso/server/apis/v1/api.go @@ -30,6 +30,7 @@ import ( "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/apiutil/multiservicesapi" "github.com/tikv/pd/pkg/utils/logutil" @@ -98,6 +99,7 @@ func NewService(srv *tsoserver.Service) *Service { s.RegisterKeyspaceGroupRouter() s.RegisterHealthRouter() s.RegisterConfigRouter() + s.RegisterPrimaryRouter() return s } @@ -126,6 +128,12 @@ func (s *Service) RegisterConfigRouter() { router.GET("", getConfig) } +// RegisterPrimaryRouter registers the router of the primary handler. +func (s *Service) RegisterPrimaryRouter() { + router := s.root.Group("primary") + router.POST("transfer", transferPrimary) +} + func changeLogLevel(c *gin.Context) { svr := c.MustGet(multiservicesapi.ServiceContextKey).(*tsoserver.Service) var level string @@ -266,3 +274,44 @@ func getConfig(c *gin.Context) { svr := c.MustGet(multiservicesapi.ServiceContextKey).(*tsoserver.Service) c.IndentedJSON(http.StatusOK, svr.GetConfig()) } + +// TransferPrimary transfers the primary member to `new_primary`. +// @Tags primary +// @Summary Transfer the primary member to `new_primary`. +// @Produce json +// @Param new_primary body string false "new primary name" +// @Success 200 string string +// @Router /primary/transfer [post] +func transferPrimary(c *gin.Context) { + svr := c.MustGet(multiservicesapi.ServiceContextKey).(*tsoserver.Service) + var input map[string]string + if err := c.BindJSON(&input); err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + + // We only support default keyspace group now. + newPrimary, keyspaceGroupID := "", constant.DefaultKeyspaceGroupID + if v, ok := input["new_primary"]; ok { + newPrimary = v + } + + allocator, err := svr.GetTSOAllocatorManager(keyspaceGroupID) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + + globalAllocator, err := allocator.GetAllocator(tso.GlobalDCLocation) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + + if err := utils.TransferPrimary(svr.GetClient(), globalAllocator.(*tso.GlobalTSOAllocator).GetExpectedPrimaryLease(), + constant.TSOServiceName, svr.Name(), newPrimary, keyspaceGroupID); err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, "success") +} diff --git a/pkg/mcs/utils/expected_primary.go b/pkg/mcs/utils/expected_primary.go new file mode 100644 index 00000000000..d44b2eae436 --- /dev/null +++ b/pkg/mcs/utils/expected_primary.go @@ -0,0 +1,182 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "fmt" + "math/rand" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/election" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/discovery" + "github.com/tikv/pd/pkg/mcs/utils/constant" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/etcdutil" + "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" +) + +// expectedPrimaryFlag is the flag to indicate the expected primary. +// 1. When the primary was campaigned successfully, it will set the `expected_primary` flag. +// 2. Using `{service}/primary/transfer` API will revoke the previous lease and set a new `expected_primary` flag. +// This flag used to help new primary to campaign successfully while other secondaries can skip the campaign. +const expectedPrimaryFlag = "expected_primary" + +// expectedPrimaryPath formats the primary path with the expected primary flag. +func expectedPrimaryPath(primaryPath string) string { + return fmt.Sprintf("%s/%s", primaryPath, expectedPrimaryFlag) +} + +// GetExpectedPrimaryFlag gets the expected primary flag. +func GetExpectedPrimaryFlag(client *clientv3.Client, primaryPath string) string { + path := expectedPrimaryPath(primaryPath) + primary, err := etcdutil.GetValue(client, path) + if err != nil { + log.Error("get expected primary flag error", errs.ZapError(err), zap.String("primary-path", path)) + return "" + } + + return string(primary) +} + +// markExpectedPrimaryFlag marks the expected primary flag when the primary is specified. +func markExpectedPrimaryFlag(client *clientv3.Client, primaryPath string, leaderRaw string, leaseID clientv3.LeaseID) (int64, error) { + path := expectedPrimaryPath(primaryPath) + log.Info("set expected primary flag", zap.String("primary-path", path), zap.String("leader-raw", leaderRaw)) + // write a flag to indicate the expected primary. + resp, err := kv.NewSlowLogTxn(client). + Then(clientv3.OpPut(expectedPrimaryPath(primaryPath), leaderRaw, clientv3.WithLease(leaseID))). + Commit() + if err != nil || !resp.Succeeded { + log.Error("mark expected primary error", errs.ZapError(err), zap.String("primary-path", path)) + return 0, err + } + return resp.Header.Revision, nil +} + +// KeepExpectedPrimaryAlive keeps the expected primary alive. +// We use lease to keep `expected primary` healthy. +// ONLY reset by the following conditions: +// - changed by `{service}/primary/transfer` API. +// - leader lease expired. +// ONLY primary called this function. +func KeepExpectedPrimaryAlive(ctx context.Context, cli *clientv3.Client, exitPrimary chan<- struct{}, + leaseTimeout int64, leaderPath, memberValue, service string) (*election.Lease, error) { + log.Info("primary start to watch the expected primary", zap.String("service", service), zap.String("primary-value", memberValue)) + service = fmt.Sprintf("%s expected primary", service) + lease := election.NewLease(cli, service) + if err := lease.Grant(leaseTimeout); err != nil { + return nil, err + } + + revision, err := markExpectedPrimaryFlag(cli, leaderPath, memberValue, lease.ID.Load().(clientv3.LeaseID)) + if err != nil { + log.Error("mark expected primary error", errs.ZapError(err)) + return nil, err + } + // Keep alive the current expected primary leadership to indicate that the server is still alive. + // Watch the expected primary path to check whether the expected primary has changed by `{service}/primary/transfer` API. + expectedPrimary := election.NewLeadership(cli, expectedPrimaryPath(leaderPath), service) + expectedPrimary.SetLease(lease) + expectedPrimary.Keep(ctx) + + go watchExpectedPrimary(ctx, expectedPrimary, revision+1, exitPrimary) + return lease, nil +} + +// watchExpectedPrimary watches `{service}/primary/transfer` API whether changed the expected primary. +func watchExpectedPrimary(ctx context.Context, + expectedPrimary *election.Leadership, revision int64, exitPrimary chan<- struct{}) { + expectedPrimary.SetPrimaryWatch(true) + // ONLY exited watch by the following conditions: + // - changed by `{service}/primary/transfer` API. + // - leader lease expired. + expectedPrimary.Watch(ctx, revision) + expectedPrimary.Reset() + defer log.Info("primary exit the primary watch loop") + select { + case <-ctx.Done(): + return + case exitPrimary <- struct{}{}: + return + } +} + +// TransferPrimary transfers the primary of the specified service. +// keyspaceGroupID is optional, only used for TSO service. +func TransferPrimary(client *clientv3.Client, lease *election.Lease, serviceName, + oldPrimary, newPrimary string, keyspaceGroupID uint32) error { + if lease == nil { + return errors.New("current lease is nil, please check leadership") + } + log.Info("try to transfer primary", zap.String("service", serviceName), zap.String("from", oldPrimary), zap.String("to", newPrimary)) + entries, err := discovery.GetMSMembers(serviceName, client) + if err != nil { + return err + } + + // Do nothing when I am the only member of cluster. + if len(entries) == 1 { + return errors.Errorf("no valid secondary to transfer primary, the only member is %s", entries[0].Name) + } + + var primaryIDs []string + for _, member := range entries { + if (newPrimary == "" && member.Name != oldPrimary) || (newPrimary != "" && member.Name == newPrimary) { + primaryIDs = append(primaryIDs, member.ServiceAddr) + } + } + if len(primaryIDs) == 0 { + return errors.Errorf("no valid secondary to transfer primary, from %s to %s", oldPrimary, newPrimary) + } + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + nextPrimaryID := r.Intn(len(primaryIDs)) + + clusterID, err := etcdutil.GetClusterID(client, constant.ClusterIDPath) + if err != nil { + return errors.Errorf("failed to get cluster ID: %v", err) + } + + // update expected primary flag + grantResp, err := client.Grant(client.Ctx(), constant.DefaultLeaderLease) + if err != nil { + return errors.Errorf("failed to grant lease for expected primary, err: %v", err) + } + + // revoke current primary's lease to ensure keepalive goroutine of primary exits. + if err := lease.Close(); err != nil { + return errors.Errorf("failed to revoke current primary's lease: %v", err) + } + + var primaryPath string + switch serviceName { + case constant.SchedulingServiceName: + primaryPath = endpoint.SchedulingPrimaryPath(clusterID) + case constant.TSOServiceName: + tsoRootPath := endpoint.TSOSvcRootPath(clusterID) + primaryPath = endpoint.KeyspaceGroupPrimaryPath(tsoRootPath, keyspaceGroupID) + } + _, err = markExpectedPrimaryFlag(client, primaryPath, primaryIDs[nextPrimaryID], grantResp.ID) + if err != nil { + return errors.Errorf("failed to mark expected primary flag for %s, err: %v", serviceName, err) + } + return nil +} diff --git a/pkg/member/member.go b/pkg/member/member.go index 99fc5457b71..9c61a832b5f 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -331,8 +331,8 @@ func (m *EmbeddedEtcdMember) GetEtcdLeader() uint64 { } // IsSameLeader checks whether a server is the leader itself. -func (m *EmbeddedEtcdMember) IsSameLeader(leader *pdpb.Member) bool { - return leader.GetMemberId() == m.ID() +func (m *EmbeddedEtcdMember) IsSameLeader(leader any) bool { + return leader.(*pdpb.Member).GetMemberId() == m.ID() } // InitMemberInfo initializes the member info. diff --git a/pkg/member/participant.go b/pkg/member/participant.go index 43a91195bff..5e35d127bf7 100644 --- a/pkg/member/participant.go +++ b/pkg/member/participant.go @@ -67,6 +67,8 @@ type Participant struct { campaignChecker atomic.Value // Store as leadershipCheckFunc // lastLeaderUpdatedTime is the last time when the leader is updated. lastLeaderUpdatedTime atomic.Value + // expectedPrimaryLease is the expected lease for the primary. + expectedPrimaryLease atomic.Value // stored as *election.Lease } // NewParticipant create a new Participant. @@ -374,6 +376,20 @@ func (m *Participant) SetCampaignChecker(checker leadershipCheckFunc) { m.campaignChecker.Store(checker) } +// SetExpectedPrimaryLease sets the expected lease for the primary. +func (m *Participant) SetExpectedPrimaryLease(lease *election.Lease) { + m.expectedPrimaryLease.Store(lease) +} + +// GetExpectedPrimaryLease gets the expected lease for the primary. +func (m *Participant) GetExpectedPrimaryLease() *election.Lease { + l := m.expectedPrimaryLease.Load() + if l == nil { + return nil + } + return l.(*election.Lease) +} + // NewParticipantByService creates a new participant by service name. func NewParticipantByService(serviceName string) (p participant) { switch serviceName { diff --git a/pkg/tso/global_allocator.go b/pkg/tso/global_allocator.go index b745544ad05..38511ee2913 100644 --- a/pkg/tso/global_allocator.go +++ b/pkg/tso/global_allocator.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "runtime/trace" + "strings" "sync" "sync/atomic" "time" @@ -27,7 +28,9 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" + "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/errs" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/slice" @@ -78,8 +81,10 @@ type GlobalTSOAllocator struct { // for global TSO synchronization am *AllocatorManager // for election use - member ElectionMember - timestampOracle *timestampOracle + member ElectionMember + // expectedPrimaryLease is used to store the expected primary lease. + expectedPrimaryLease atomic.Value // store as *election.LeaderLease + timestampOracle *timestampOracle // syncRTT is the RTT duration a SyncMaxTS RPC call will cost, // which is used to estimate the MaxTS in a Global TSO generation // to reduce the gRPC network IO latency. @@ -560,6 +565,20 @@ func (gta *GlobalTSOAllocator) primaryElectionLoop() { logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0)) } + // To make sure the expected primary(if existed) and new primary are on the same server. + expectedPrimary := mcsutils.GetExpectedPrimaryFlag(gta.member.Client(), gta.member.GetLeaderPath()) + // skip campaign the primary if the expected primary is not empty and not equal to the current memberValue. + // expected primary ONLY SET BY `{service}/primary/transfer` API. + if len(expectedPrimary) > 0 && !strings.Contains(gta.member.MemberValue(), expectedPrimary) { + log.Info("skip campaigning of tso primary and check later", + zap.String("server-name", gta.member.Name()), + zap.String("expected-primary-id", expectedPrimary), + zap.Uint64("member-id", gta.member.ID()), + zap.String("cur-memberValue", gta.member.MemberValue())) + time.Sleep(200 * time.Millisecond) + continue + } + gta.campaignLeader() } } @@ -596,7 +615,7 @@ func (gta *GlobalTSOAllocator) campaignLeader() { gta.member.ResetLeader() }) - // maintain the the leadership, after this, TSO can be service. + // maintain the leadership, after this, TSO can be service. gta.member.KeepLeader(ctx) log.Info("campaign tso primary ok", logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), @@ -620,8 +639,18 @@ func (gta *GlobalTSOAllocator) campaignLeader() { gta.am.ResetAllocatorGroup(GlobalDCLocation, false) }() - tsoLabel := fmt.Sprintf("TSO Service Group %d", gta.getGroupID()) + // check expected primary and watch the primary. + exitPrimary := make(chan struct{}) + lease, err := mcsutils.KeepExpectedPrimaryAlive(ctx, gta.member.Client(), exitPrimary, + gta.am.leaderLease, gta.member.GetLeaderPath(), gta.member.MemberValue(), constant.TSOServiceName) + if err != nil { + log.Error("prepare tso primary watch error", errs.ZapError(err)) + return + } + gta.expectedPrimaryLease.Store(lease) gta.member.EnableLeader() + + tsoLabel := fmt.Sprintf("TSO Service Group %d", gta.getGroupID()) member.ServiceMemberGauge.WithLabelValues(tsoLabel).Set(1) defer resetLeaderOnce.Do(func() { cancel() @@ -651,10 +680,22 @@ func (gta *GlobalTSOAllocator) campaignLeader() { log.Info("exit leader campaign", logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0)) return + case <-exitPrimary: + log.Info("no longer be primary because primary have been updated, the TSO primary will step down") + return } } } +// GetExpectedPrimaryLease returns the expected primary lease. +func (gta *GlobalTSOAllocator) GetExpectedPrimaryLease() *election.Lease { + l := gta.expectedPrimaryLease.Load() + if l == nil { + return nil + } + return l.(*election.Lease) +} + func (gta *GlobalTSOAllocator) getMetrics() *tsoMetrics { return gta.timestampOracle.metrics } diff --git a/tests/integrations/mcs/members/member_test.go b/tests/integrations/mcs/members/member_test.go index 79b269e2e8a..dffa6305d0b 100644 --- a/tests/integrations/mcs/members/member_test.go +++ b/tests/integrations/mcs/members/member_test.go @@ -15,13 +15,18 @@ package members_test import ( + "bytes" "context" + "encoding/json" + "fmt" + "net/http" "testing" + "time" + "github.com/pingcap/failpoint" "github.com/stretchr/testify/suite" pdClient "github.com/tikv/pd/client/http" bs "github.com/tikv/pd/pkg/basicserver" - "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" @@ -35,6 +40,9 @@ type memberTestSuite struct { server *tests.TestServer backendEndpoints string pdClient pdClient.Client + + tsoNodes map[string]bs.Server + schedulingNodes map[string]bs.Server } func TestMemberTestSuite(t *testing.T) { @@ -57,7 +65,7 @@ func (suite *memberTestSuite) SetupTest() { // TSO nodes := make(map[string]bs.Server) - for i := 0; i < constant.DefaultKeyspaceGroupReplicaCount; i++ { + for i := 0; i < 3; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) nodes[s.GetAddr()] = s suite.cleanupFunc = append(suite.cleanupFunc, func() { @@ -65,6 +73,7 @@ func (suite *memberTestSuite) SetupTest() { }) } tests.WaitForPrimaryServing(re, nodes) + suite.tsoNodes = nodes // Scheduling nodes = make(map[string]bs.Server) @@ -76,6 +85,7 @@ func (suite *memberTestSuite) SetupTest() { }) } tests.WaitForPrimaryServing(re, nodes) + suite.schedulingNodes = nodes suite.cleanupFunc = append(suite.cleanupFunc, func() { cancel() @@ -96,7 +106,7 @@ func (suite *memberTestSuite) TestMembers() { re := suite.Require() members, err := suite.pdClient.GetMicroServiceMembers(suite.ctx, "tso") re.NoError(err) - re.Len(members, constant.DefaultKeyspaceGroupReplicaCount) + re.Len(members, 3) members, err = suite.pdClient.GetMicroServiceMembers(suite.ctx, "scheduling") re.NoError(err) @@ -113,3 +123,270 @@ func (suite *memberTestSuite) TestPrimary() { re.NoError(err) re.NotEmpty(primary) } + +func (suite *memberTestSuite) TestCampaignPrimaryWhileServerClose() { + re := suite.Require() + primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, "tso") + re.NoError(err) + re.NotEmpty(primary) + + supportedServices := []string{"tso", "scheduling"} + for _, service := range supportedServices { + var nodes map[string]bs.Server + switch service { + case "tso": + nodes = suite.tsoNodes + case "scheduling": + nodes = suite.schedulingNodes + } + + primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + + // Close old and new primary to mock campaign primary + for _, member := range nodes { + if member.GetAddr() != primary { + nodes[member.Name()].Close() + break + } + } + nodes[primary].Close() + tests.WaitForPrimaryServing(re, nodes) + + // primary should be different with before + onlyPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + re.NotEqual(primary, onlyPrimary) + } +} + +func (suite *memberTestSuite) TestTransferPrimary() { + re := suite.Require() + supportedServices := []string{"tso", "scheduling"} + for _, service := range supportedServices { + var nodes map[string]bs.Server + switch service { + case "tso": + nodes = suite.tsoNodes + case "scheduling": + nodes = suite.schedulingNodes + } + + // Test resign primary by random + primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + + newPrimaryData := make(map[string]any) + newPrimaryData["new_primary"] = "" + data, _ := json.Marshal(newPrimaryData) + resp, err := tests.TestDialClient.Post(fmt.Sprintf("%s/%s/api/v1/primary/transfer", primary, service), + "application/json", bytes.NewBuffer(data)) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) + resp.Body.Close() + + testutil.Eventually(re, func() bool { + for _, member := range nodes { + if member.GetAddr() != primary && member.IsServing() { + return true + } + } + return false + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + primary, err = suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + + // Test transfer primary to a specific node + var newPrimary string + for _, member := range nodes { + if member.GetAddr() != primary { + newPrimary = member.Name() + break + } + } + newPrimaryData["new_primary"] = newPrimary + data, _ = json.Marshal(newPrimaryData) + resp, err = tests.TestDialClient.Post(fmt.Sprintf("%s/%s/api/v1/primary/transfer", primary, service), + "application/json", bytes.NewBuffer(data)) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) + resp.Body.Close() + + testutil.Eventually(re, func() bool { + return nodes[newPrimary].IsServing() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + primary, err = suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + re.Equal(primary, newPrimary) + + // Test transfer primary to a non-exist node + newPrimary = "http://" + newPrimaryData["new_primary"] = newPrimary + data, _ = json.Marshal(newPrimaryData) + resp, err = tests.TestDialClient.Post(fmt.Sprintf("%s/%s/api/v1/primary/transfer", primary, service), + "application/json", bytes.NewBuffer(data)) + re.NoError(err) + re.Equal(http.StatusInternalServerError, resp.StatusCode) + resp.Body.Close() + } +} + +func (suite *memberTestSuite) TestCampaignPrimaryAfterTransfer() { + re := suite.Require() + supportedServices := []string{"tso", "scheduling"} + for _, service := range supportedServices { + var nodes map[string]bs.Server + switch service { + case "tso": + nodes = suite.tsoNodes + case "scheduling": + nodes = suite.schedulingNodes + } + + primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + + // Test transfer primary to a specific node + var newPrimary string + for _, member := range nodes { + if member.GetAddr() != primary { + newPrimary = member.Name() + break + } + } + newPrimaryData := make(map[string]any) + newPrimaryData["new_primary"] = newPrimary + data, _ := json.Marshal(newPrimaryData) + resp, err := tests.TestDialClient.Post(fmt.Sprintf("%s/%s/api/v1/primary/transfer", primary, service), + "application/json", bytes.NewBuffer(data)) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) + resp.Body.Close() + + tests.WaitForPrimaryServing(re, nodes) + newPrimary, err = suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + re.NotEqual(primary, newPrimary) + + // Close old and new primary to mock campaign primary + nodes[primary].Close() + nodes[newPrimary].Close() + tests.WaitForPrimaryServing(re, nodes) + // Primary should be different with before + onlyPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + re.NotEqual(primary, onlyPrimary) + re.NotEqual(newPrimary, onlyPrimary) + } +} + +func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpired() { + re := suite.Require() + primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, "tso") + re.NoError(err) + re.NotEmpty(primary) + + supportedServices := []string{"tso", "scheduling"} + for _, service := range supportedServices { + var nodes map[string]bs.Server + switch service { + case "tso": + nodes = suite.tsoNodes + case "scheduling": + nodes = suite.schedulingNodes + } + + primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + + // Test transfer primary to a specific node + var newPrimary string + for _, member := range nodes { + if member.GetAddr() != primary { + newPrimary = member.Name() + break + } + } + // Mock the new primary can not grant leader which means the lease will expire + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/election/skipGrantLeader", `return()`)) + newPrimaryData := make(map[string]any) + newPrimaryData["new_primary"] = newPrimary + data, _ := json.Marshal(newPrimaryData) + resp, err := tests.TestDialClient.Post(fmt.Sprintf("%s/%s/api/v1/primary/transfer", primary, service), + "application/json", bytes.NewBuffer(data)) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) + resp.Body.Close() + + // Wait for the old primary exit and new primary campaign + // cannot check newPrimary isServing when skipGrantLeader is enabled + testutil.Eventually(re, func() bool { + return !nodes[primary].IsServing() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + // TODO: Add campaign times check in mcs to avoid frequent campaign + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/election/skipGrantLeader")) + // Can still work after lease expired + tests.WaitForPrimaryServing(re, nodes) + } +} + +// TestTransferPrimaryWhileLeaseExpiredAndServerDown tests transfer primary while lease expired and server down +func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpiredAndServerDown() { + re := suite.Require() + primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, "tso") + re.NoError(err) + re.NotEmpty(primary) + + supportedServices := []string{"tso", "scheduling"} + for _, service := range supportedServices { + var nodes map[string]bs.Server + switch service { + case "tso": + nodes = suite.tsoNodes + case "scheduling": + nodes = suite.schedulingNodes + } + + primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + + // Test transfer primary to a specific node + var newPrimary string + for _, member := range nodes { + if member.GetAddr() != primary { + newPrimary = member.Name() + break + } + } + // Mock the new primary can not grant leader which means the lease will expire + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/election/skipGrantLeader", `return()`)) + newPrimaryData := make(map[string]any) + newPrimaryData["new_primary"] = "" + data, _ := json.Marshal(newPrimaryData) + resp, err := tests.TestDialClient.Post(fmt.Sprintf("%s/%s/api/v1/primary/transfer", primary, service), + "application/json", bytes.NewBuffer(data)) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) + resp.Body.Close() + + // Wait for the old primary exit and new primary campaign + // cannot check newPrimary isServing when skipGrantLeader is enabled + testutil.Eventually(re, func() bool { + return !nodes[primary].IsServing() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + + // TODO: Add campaign times check in mcs to avoid frequent campaign + // for now, close the current primary to mock the server down + nodes[newPrimary].Close() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/election/skipGrantLeader")) + + tests.WaitForPrimaryServing(re, nodes) + // Primary should be different with before + onlyPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + re.NoError(err) + re.NotEqual(newPrimary, onlyPrimary) + } +} From 3b3757202353cbff2bc09d53842bd1ba1e65a617 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Wed, 14 Aug 2024 11:46:37 +0800 Subject: [PATCH 13/42] scheduler: extract storage operations to schedulerConfig interface (#8515) ref tikv/pd#8379, close tikv/pd#8514 Signed-off-by: okJiang <819421878@qq.com> --- pkg/schedule/schedulers/balance_leader.go | 26 ++---- pkg/schedule/schedulers/balance_witness.go | 26 ++---- pkg/schedule/schedulers/config.go | 60 ++++++++++++ pkg/schedule/schedulers/config_test.go | 50 ++++++++++ pkg/schedule/schedulers/evict_leader.go | 33 ++----- pkg/schedule/schedulers/evict_slow_store.go | 36 ++------ .../schedulers/evict_slow_store_test.go | 14 +-- pkg/schedule/schedulers/evict_slow_trend.go | 38 +++----- .../schedulers/evict_slow_trend_test.go | 13 +-- pkg/schedule/schedulers/grant_hot_region.go | 19 +--- pkg/schedule/schedulers/grant_leader.go | 19 +--- pkg/schedule/schedulers/hot_region.go | 10 +- pkg/schedule/schedulers/hot_region_config.go | 15 +-- pkg/schedule/schedulers/init.go | 91 +++++++++++++------ pkg/schedule/schedulers/scatter_range.go | 20 +--- pkg/schedule/schedulers/shuffle_hot_region.go | 26 ++---- pkg/schedule/schedulers/shuffle_leader.go | 7 +- pkg/schedule/schedulers/shuffle_region.go | 9 +- .../schedulers/shuffle_region_config.go | 13 +-- pkg/schedule/schedulers/split_bucket.go | 27 ++---- 20 files changed, 254 insertions(+), 298 deletions(-) create mode 100644 pkg/schedule/schedulers/config.go create mode 100644 pkg/schedule/schedulers/config_test.go diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index b434c7ad706..ef254ee6950 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -32,7 +32,6 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" types "github.com/tikv/pd/pkg/schedule/type" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/reflectutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" @@ -56,8 +55,9 @@ const ( type balanceLeaderSchedulerConfig struct { syncutil.RWMutex - storage endpoint.ConfigStorage - Ranges []core.KeyRange `json:"ranges"` + schedulerConfig + + Ranges []core.KeyRange `json:"ranges"` // Batch is used to generate multiple operators by one scheduling Batch int `json:"batch"` } @@ -79,7 +79,7 @@ func (conf *balanceLeaderSchedulerConfig) update(data []byte) (int, any) { } return http.StatusBadRequest, "invalid batch size which should be an integer between 1 and 10" } - if err := conf.persistLocked(); err != nil { + if err := conf.save(); err != nil { log.Warn("failed to save balance-leader-scheduler config", errs.ZapError(err)) } log.Info("balance-leader-scheduler config is updated", zap.ByteString("old", oldConfig), zap.ByteString("new", newConfig)) @@ -111,14 +111,6 @@ func (conf *balanceLeaderSchedulerConfig) clone() *balanceLeaderSchedulerConfig } } -func (conf *balanceLeaderSchedulerConfig) persistLocked() error { - data, err := EncodeConfig(conf) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(BalanceLeaderName, data) -} - func (conf *balanceLeaderSchedulerConfig) getBatch() int { conf.RLock() defer conf.RUnlock() @@ -216,15 +208,9 @@ func (l *balanceLeaderScheduler) EncodeConfig() ([]byte, error) { func (l *balanceLeaderScheduler) ReloadConfig() error { l.conf.Lock() defer l.conf.Unlock() - cfgData, err := l.conf.storage.LoadSchedulerConfig(l.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } + newCfg := &balanceLeaderSchedulerConfig{} - if err = DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := l.conf.load(newCfg); err != nil { return err } l.conf.Ranges = newCfg.Ranges diff --git a/pkg/schedule/schedulers/balance_witness.go b/pkg/schedule/schedulers/balance_witness.go index 599af6df637..450c43647cf 100644 --- a/pkg/schedule/schedulers/balance_witness.go +++ b/pkg/schedule/schedulers/balance_witness.go @@ -33,7 +33,6 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" types "github.com/tikv/pd/pkg/schedule/type" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/reflectutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -53,8 +52,9 @@ const ( type balanceWitnessSchedulerConfig struct { syncutil.RWMutex - storage endpoint.ConfigStorage - Ranges []core.KeyRange `json:"ranges"` + schedulerConfig + + Ranges []core.KeyRange `json:"ranges"` // Batch is used to generate multiple operators by one scheduling Batch int `json:"batch"` } @@ -76,7 +76,7 @@ func (conf *balanceWitnessSchedulerConfig) update(data []byte) (int, any) { } return http.StatusBadRequest, "invalid batch size which should be an integer between 1 and 10" } - if err := conf.persistLocked(); err != nil { + if err := conf.save(); err != nil { log.Warn("failed to persist config", zap.Error(err)) } log.Info("balance-witness-scheduler config is updated", zap.ByteString("old", oldc), zap.ByteString("new", newc)) @@ -108,14 +108,6 @@ func (conf *balanceWitnessSchedulerConfig) clone() *balanceWitnessSchedulerConfi } } -func (conf *balanceWitnessSchedulerConfig) persistLocked() error { - data, err := EncodeConfig(conf) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(BalanceWitnessName, data) -} - func (conf *balanceWitnessSchedulerConfig) getBatch() int { conf.RLock() defer conf.RUnlock() @@ -215,15 +207,9 @@ func (b *balanceWitnessScheduler) EncodeConfig() ([]byte, error) { func (b *balanceWitnessScheduler) ReloadConfig() error { b.conf.Lock() defer b.conf.Unlock() - cfgData, err := b.conf.storage.LoadSchedulerConfig(b.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } + newCfg := &balanceWitnessSchedulerConfig{} - if err = DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := b.conf.load(newCfg); err != nil { return err } b.conf.Ranges = newCfg.Ranges diff --git a/pkg/schedule/schedulers/config.go b/pkg/schedule/schedulers/config.go new file mode 100644 index 00000000000..0c7caf686c3 --- /dev/null +++ b/pkg/schedule/schedulers/config.go @@ -0,0 +1,60 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schedulers + +import ( + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/tikv/pd/pkg/storage/endpoint" +) + +type schedulerConfig interface { + save() error + load(any) error + init(name string, storage endpoint.ConfigStorage, data any) +} + +type baseSchedulerConfig struct { + name string + storage endpoint.ConfigStorage + + // data is the config of the scheduler. + data any +} + +func (b *baseSchedulerConfig) init(name string, storage endpoint.ConfigStorage, data any) { + b.name = name + b.storage = storage + b.data = data +} + +func (b *baseSchedulerConfig) save() error { + data, err := EncodeConfig(b.data) + failpoint.Inject("persistFail", func() { + err = errors.New("fail to persist") + }) + if err != nil { + return err + } + return b.storage.SaveSchedulerConfig(b.name, data) +} + +func (b *baseSchedulerConfig) load(v any) error { + data, err := b.storage.LoadSchedulerConfig(b.name) + if err != nil { + return err + } + return DecodeConfig([]byte(data), v) +} diff --git a/pkg/schedule/schedulers/config_test.go b/pkg/schedule/schedulers/config_test.go new file mode 100644 index 00000000000..31858bd7c10 --- /dev/null +++ b/pkg/schedule/schedulers/config_test.go @@ -0,0 +1,50 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schedulers + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/storage" +) + +func TestSchedulerConfig(t *testing.T) { + s := storage.NewStorageWithMemoryBackend() + + type testConfig struct { + schedulerConfig + Value string `json:"value"` + } + + cfg := &testConfig{ + schedulerConfig: &baseSchedulerConfig{}, + } + cfg.init("test", s, cfg) + + cfg.Value = "test" + require.NoError(t, cfg.save()) + newTc := &testConfig{} + require.NoError(t, cfg.load(newTc)) + require.Equal(t, cfg.Value, newTc.Value) + + // config with another name cannot loaded the previous config + cfg2 := &testConfig{ + schedulerConfig: &baseSchedulerConfig{}, + } + cfg2.init("test2", s, cfg2) + // report error because the config is empty and cannot be decoded + require.Error(t, cfg2.load(newTc)) +} diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index d4e26cb1b68..85f861f0082 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -20,7 +20,6 @@ import ( "github.com/gorilla/mux" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" @@ -30,7 +29,6 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" types "github.com/tikv/pd/pkg/schedule/type" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -48,7 +46,8 @@ const ( type evictLeaderSchedulerConfig struct { syncutil.RWMutex - storage endpoint.ConfigStorage + schedulerConfig + StoreIDWithRanges map[uint64][]core.KeyRange `json:"store-id-ranges"` // Batch is used to generate multiple operators by one scheduling Batch int `json:"batch"` @@ -85,17 +84,6 @@ func (conf *evictLeaderSchedulerConfig) clone() *evictLeaderSchedulerConfig { } } -func (conf *evictLeaderSchedulerConfig) persistLocked() error { - data, err := EncodeConfig(conf) - failpoint.Inject("persistFail", func() { - err = errors.New("fail to persist") - }) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(types.EvictLeaderScheduler.String(), data) -} - func (conf *evictLeaderSchedulerConfig) getRanges(id uint64) []string { conf.RLock() defer conf.RUnlock() @@ -145,18 +133,11 @@ func (conf *evictLeaderSchedulerConfig) encodeConfig() ([]byte, error) { return EncodeConfig(conf) } -func (conf *evictLeaderSchedulerConfig) reloadConfig(name string) error { +func (conf *evictLeaderSchedulerConfig) reloadConfig() error { conf.Lock() defer conf.Unlock() - cfgData, err := conf.storage.LoadSchedulerConfig(name) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } newCfg := &evictLeaderSchedulerConfig{} - if err = DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := conf.load(newCfg); err != nil { return err } pauseAndResumeLeaderTransfer(conf.cluster, conf.StoreIDWithRanges, newCfg.StoreIDWithRanges) @@ -203,7 +184,7 @@ func (conf *evictLeaderSchedulerConfig) update(id uint64, newRanges []core.KeyRa conf.StoreIDWithRanges[id] = newRanges } conf.Batch = batch - err := conf.persistLocked() + err := conf.save() if err != nil && id != 0 { _, _ = conf.removeStoreLocked(id) } @@ -220,7 +201,7 @@ func (conf *evictLeaderSchedulerConfig) delete(id uint64) (any, error) { } keyRanges := conf.StoreIDWithRanges[id] - err = conf.persistLocked() + err = conf.save() if err != nil { conf.resetStoreLocked(id, keyRanges) conf.Unlock() @@ -275,7 +256,7 @@ func (s *evictLeaderScheduler) EncodeConfig() ([]byte, error) { // ReloadConfig reloads the config from the storage. func (s *evictLeaderScheduler) ReloadConfig() error { - return s.conf.reloadConfig(s.GetName()) + return s.conf.reloadConfig() } // PrepareConfig implements the Scheduler interface. diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index d0fb963bd52..de581f597bb 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -27,7 +27,6 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" types "github.com/tikv/pd/pkg/schedule/type" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -44,8 +43,9 @@ const ( type evictSlowStoreSchedulerConfig struct { syncutil.RWMutex + schedulerConfig + cluster *core.BasicCluster - storage endpoint.ConfigStorage // Last timestamp of the chosen slow store for eviction. lastSlowStoreCaptureTS time.Time // Duration gap for recovering the candidate, unit: s. @@ -53,9 +53,9 @@ type evictSlowStoreSchedulerConfig struct { EvictedStores []uint64 `json:"evict-stores"` } -func initEvictSlowStoreSchedulerConfig(storage endpoint.ConfigStorage) *evictSlowStoreSchedulerConfig { +func initEvictSlowStoreSchedulerConfig() *evictSlowStoreSchedulerConfig { return &evictSlowStoreSchedulerConfig{ - storage: storage, + schedulerConfig: &baseSchedulerConfig{}, lastSlowStoreCaptureTS: time.Time{}, RecoveryDurationGap: defaultRecoveryDurationGap, EvictedStores: make([]uint64, 0), @@ -70,17 +70,6 @@ func (conf *evictSlowStoreSchedulerConfig) clone() *evictSlowStoreSchedulerConfi } } -func (conf *evictSlowStoreSchedulerConfig) persistLocked() error { - data, err := EncodeConfig(conf) - failpoint.Inject("persistFail", func() { - err = errors.New("fail to persist") - }) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(types.EvictSlowStoreScheduler.String(), data) -} - func (conf *evictSlowStoreSchedulerConfig) getStores() []uint64 { conf.RLock() defer conf.RUnlock() @@ -121,7 +110,7 @@ func (conf *evictSlowStoreSchedulerConfig) setStoreAndPersist(id uint64) error { defer conf.Unlock() conf.EvictedStores = []uint64{id} conf.lastSlowStoreCaptureTS = time.Now() - return conf.persistLocked() + return conf.save() } func (conf *evictSlowStoreSchedulerConfig) clearAndPersist() (oldID uint64, err error) { @@ -131,7 +120,7 @@ func (conf *evictSlowStoreSchedulerConfig) clearAndPersist() (oldID uint64, err if oldID > 0 { conf.EvictedStores = []uint64{} conf.lastSlowStoreCaptureTS = time.Time{} - err = conf.persistLocked() + err = conf.save() } return } @@ -162,12 +151,13 @@ func (handler *evictSlowStoreHandler) updateConfig(w http.ResponseWriter, r *htt handler.rd.JSON(w, http.StatusInternalServerError, errors.New("invalid argument for 'recovery-duration'").Error()) return } + handler.config.Lock() defer handler.config.Unlock() prevRecoveryDurationGap := handler.config.RecoveryDurationGap recoveryDurationGap := uint64(recoveryDurationGapFloat) handler.config.RecoveryDurationGap = recoveryDurationGap - if err := handler.config.persistLocked(); err != nil { + if err := handler.config.save(); err != nil { handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) handler.config.RecoveryDurationGap = prevRecoveryDurationGap return @@ -201,15 +191,9 @@ func (s *evictSlowStoreScheduler) EncodeConfig() ([]byte, error) { func (s *evictSlowStoreScheduler) ReloadConfig() error { s.conf.Lock() defer s.conf.Unlock() - cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } + newCfg := &evictSlowStoreSchedulerConfig{} - if err = DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := s.conf.load(newCfg); err != nil { return err } old := make(map[uint64]struct{}) diff --git a/pkg/schedule/schedulers/evict_slow_store_test.go b/pkg/schedule/schedulers/evict_slow_store_test.go index ad5b16e8ca3..406a08b9c99 100644 --- a/pkg/schedule/schedulers/evict_slow_store_test.go +++ b/pkg/schedule/schedulers/evict_slow_store_test.go @@ -16,8 +16,6 @@ package schedulers import ( "context" - "encoding/json" - "strings" "testing" "github.com/pingcap/failpoint" @@ -103,18 +101,10 @@ func (suite *evictSlowStoreTestSuite) TestEvictSlowStore() { re.Zero(es2.conf.evictStore()) // check the value from storage. - sches, vs, err := es2.conf.storage.LoadAllSchedulerConfigs() - re.NoError(err) - valueStr := "" - for id, sche := range sches { - if strings.EqualFold(sche, EvictSlowStoreName) { - valueStr = vs[id] - } - } - var persistValue evictSlowStoreSchedulerConfig - err = json.Unmarshal([]byte(valueStr), &persistValue) + err := es2.conf.load(&persistValue) re.NoError(err) + re.Equal(es2.conf.EvictedStores, persistValue.EvictedStores) re.Zero(persistValue.evictStore()) re.True(persistValue.readyForRecovery()) diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index 206791900c6..427787016a2 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -28,7 +28,6 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" types "github.com/tikv/pd/pkg/schedule/type" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -54,8 +53,9 @@ type slowCandidate struct { type evictSlowTrendSchedulerConfig struct { syncutil.RWMutex + schedulerConfig + cluster *core.BasicCluster - storage endpoint.ConfigStorage // Candidate for eviction in current tick. evictCandidate slowCandidate // Last chosen candidate for eviction. @@ -66,9 +66,9 @@ type evictSlowTrendSchedulerConfig struct { EvictedStores []uint64 `json:"evict-by-trend-stores"` } -func initEvictSlowTrendSchedulerConfig(storage endpoint.ConfigStorage) *evictSlowTrendSchedulerConfig { +func initEvictSlowTrendSchedulerConfig() *evictSlowTrendSchedulerConfig { return &evictSlowTrendSchedulerConfig{ - storage: storage, + schedulerConfig: &baseSchedulerConfig{}, evictCandidate: slowCandidate{}, lastEvictCandidate: slowCandidate{}, RecoveryDurationGap: defaultRecoveryDurationGap, @@ -84,17 +84,6 @@ func (conf *evictSlowTrendSchedulerConfig) clone() *evictSlowTrendSchedulerConfi } } -func (conf *evictSlowTrendSchedulerConfig) persistLocked() error { - data, err := EncodeConfig(conf) - failpoint.Inject("persistFail", func() { - err = errors.New("fail to persist") - }) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(types.EvictSlowTrendScheduler.String(), data) -} - func (conf *evictSlowTrendSchedulerConfig) getStores() []uint64 { conf.RLock() defer conf.RUnlock() @@ -205,7 +194,7 @@ func (conf *evictSlowTrendSchedulerConfig) setStoreAndPersist(id uint64) error { conf.Lock() defer conf.Unlock() conf.EvictedStores = []uint64{id} - return conf.persistLocked() + return conf.save() } func (conf *evictSlowTrendSchedulerConfig) clearAndPersist(cluster sche.SchedulerCluster) (oldID uint64, err error) { @@ -222,7 +211,7 @@ func (conf *evictSlowTrendSchedulerConfig) clearAndPersist(cluster sche.Schedule conf.Lock() defer conf.Unlock() conf.EvictedStores = []uint64{} - return oldID, conf.persistLocked() + return oldID, conf.save() } type evictSlowTrendHandler struct { @@ -256,7 +245,7 @@ func (handler *evictSlowTrendHandler) updateConfig(w http.ResponseWriter, r *htt prevRecoveryDurationGap := handler.config.RecoveryDurationGap recoveryDurationGap := uint64(recoveryDurationGapFloat) handler.config.RecoveryDurationGap = recoveryDurationGap - if err := handler.config.persistLocked(); err != nil { + if err := handler.config.save(); err != nil { handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) handler.config.RecoveryDurationGap = prevRecoveryDurationGap return @@ -304,17 +293,11 @@ func (s *evictSlowTrendScheduler) EncodeConfig() ([]byte, error) { func (s *evictSlowTrendScheduler) ReloadConfig() error { s.conf.Lock() defer s.conf.Unlock() - cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } newCfg := &evictSlowTrendSchedulerConfig{} - if err = DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := s.conf.load(newCfg); err != nil { return err } + old := make(map[uint64]struct{}) for _, id := range s.conf.EvictedStores { old[id] = struct{}{} @@ -456,11 +439,12 @@ func (s *evictSlowTrendScheduler) Schedule(cluster sche.SchedulerCluster, _ bool func newEvictSlowTrendScheduler(opController *operator.Controller, conf *evictSlowTrendSchedulerConfig) Scheduler { handler := newEvictSlowTrendHandler(conf) - return &evictSlowTrendScheduler{ + sche := &evictSlowTrendScheduler{ BaseScheduler: NewBaseScheduler(opController, types.EvictSlowTrendScheduler), conf: conf, handler: handler, } + return sche } func chooseEvictCandidate(cluster sche.SchedulerCluster, lastEvictCandidate *slowCandidate) (slowStore *core.StoreInfo) { diff --git a/pkg/schedule/schedulers/evict_slow_trend_test.go b/pkg/schedule/schedulers/evict_slow_trend_test.go index 10da5c91565..02cb65021eb 100644 --- a/pkg/schedule/schedulers/evict_slow_trend_test.go +++ b/pkg/schedule/schedulers/evict_slow_trend_test.go @@ -16,8 +16,6 @@ package schedulers import ( "context" - "encoding/json" - "strings" "testing" "time" @@ -186,17 +184,8 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() { re.Zero(es2.conf.evictedStore()) // check the value from storage. - sches, vs, err := es2.conf.storage.LoadAllSchedulerConfigs() - re.NoError(err) - valueStr := "" - for id, sche := range sches { - if strings.EqualFold(sche, EvictSlowTrendName) { - valueStr = vs[id] - } - } - var persistValue evictSlowTrendSchedulerConfig - err = json.Unmarshal([]byte(valueStr), &persistValue) + err := es2.conf.load(&persistValue) re.NoError(err) re.Equal(es2.conf.EvictedStores, persistValue.EvictedStores) re.Zero(persistValue.evictedStore()) diff --git a/pkg/schedule/schedulers/grant_hot_region.go b/pkg/schedule/schedulers/grant_hot_region.go index 15a520f95d0..a441f41062a 100644 --- a/pkg/schedule/schedulers/grant_hot_region.go +++ b/pkg/schedule/schedulers/grant_hot_region.go @@ -34,7 +34,6 @@ import ( "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -48,7 +47,8 @@ const ( type grantHotRegionSchedulerConfig struct { syncutil.RWMutex - storage endpoint.ConfigStorage + schedulerConfig + cluster *core.BasicCluster StoreIDs []uint64 `json:"store-id"` StoreLeaderID uint64 `json:"store-leader-id"` @@ -93,11 +93,7 @@ func (conf *grantHotRegionSchedulerConfig) clone() *grantHotRegionSchedulerConfi func (conf *grantHotRegionSchedulerConfig) persist() error { conf.RLock() defer conf.RUnlock() - data, err := EncodeConfig(conf) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(types.GrantHotRegionScheduler.String(), data) + return conf.save() } func (conf *grantHotRegionSchedulerConfig) has(storeID uint64) bool { @@ -146,15 +142,8 @@ func (s *grantHotRegionScheduler) EncodeConfig() ([]byte, error) { func (s *grantHotRegionScheduler) ReloadConfig() error { s.conf.Lock() defer s.conf.Unlock() - cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } newCfg := &grantHotRegionSchedulerConfig{} - if err := DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := s.conf.load(newCfg); err != nil { return err } s.conf.StoreIDs = newCfg.StoreIDs diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index 134eddda880..d70c0b19d67 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -29,7 +29,6 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" types "github.com/tikv/pd/pkg/schedule/type" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -43,7 +42,8 @@ const ( type grantLeaderSchedulerConfig struct { syncutil.RWMutex - storage endpoint.ConfigStorage + schedulerConfig + StoreIDWithRanges map[uint64][]core.KeyRange `json:"store-id-ranges"` cluster *core.BasicCluster removeSchedulerCb func(name string) error @@ -83,11 +83,7 @@ func (conf *grantLeaderSchedulerConfig) clone() *grantLeaderSchedulerConfig { func (conf *grantLeaderSchedulerConfig) persist() error { conf.RLock() defer conf.RUnlock() - data, err := EncodeConfig(conf) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(types.GrantLeaderScheduler.String(), data) + return conf.save() } func (conf *grantLeaderSchedulerConfig) getRanges(id uint64) []string { @@ -176,15 +172,8 @@ func (s *grantLeaderScheduler) EncodeConfig() ([]byte, error) { func (s *grantLeaderScheduler) ReloadConfig() error { s.conf.Lock() defer s.conf.Unlock() - cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } newCfg := &grantLeaderSchedulerConfig{} - if err = DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := s.conf.load(newCfg); err != nil { return err } pauseAndResumeLeaderTransfer(s.conf.cluster, s.conf.StoreIDWithRanges, newCfg.StoreIDWithRanges) diff --git a/pkg/schedule/schedulers/hot_region.go b/pkg/schedule/schedulers/hot_region.go index 717c1413ac4..ab595ec9058 100644 --- a/pkg/schedule/schedulers/hot_region.go +++ b/pkg/schedule/schedulers/hot_region.go @@ -222,15 +222,9 @@ func (h *hotScheduler) EncodeConfig() ([]byte, error) { func (h *hotScheduler) ReloadConfig() error { h.conf.Lock() defer h.conf.Unlock() - cfgData, err := h.conf.storage.LoadSchedulerConfig(h.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } + newCfg := &hotRegionSchedulerConfig{} - if err := DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := h.conf.load(newCfg); err != nil { return err } h.conf.MinHotByteRate = newCfg.MinHotByteRate diff --git a/pkg/schedule/schedulers/hot_region_config.go b/pkg/schedule/schedulers/hot_region_config.go index 83121254cc0..0424a582bf4 100644 --- a/pkg/schedule/schedulers/hot_region_config.go +++ b/pkg/schedule/schedulers/hot_region_config.go @@ -28,7 +28,6 @@ import ( "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/reflectutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" @@ -59,6 +58,7 @@ var compatiblePrioritiesConfig = prioritiesConfig{ // params about hot region. func initHotRegionScheduleConfig() *hotRegionSchedulerConfig { cfg := &hotRegionSchedulerConfig{ + schedulerConfig: &baseSchedulerConfig{}, MinHotByteRate: 100, MinHotKeyRate: 10, MinHotQueryRate: 10, @@ -114,7 +114,8 @@ func (conf *hotRegionSchedulerConfig) getValidConf() *hotRegionSchedulerConfig { type hotRegionSchedulerConfig struct { syncutil.RWMutex - storage endpoint.ConfigStorage + schedulerConfig + lastQuerySupported bool MinHotByteRate float64 `json:"min-hot-byte-rate"` @@ -455,7 +456,7 @@ func (conf *hotRegionSchedulerConfig) handleSetConfig(w http.ResponseWriter, r * } newc, _ := json.Marshal(conf) if !bytes.Equal(oldc, newc) { - if err := conf.persistLocked(); err != nil { + if err := conf.save(); err != nil { log.Warn("failed to persist config", zap.Error(err)) } log.Info("hot-region-scheduler config is updated", zap.String("old", string(oldc)), zap.String("new", string(newc))) @@ -477,14 +478,6 @@ func (conf *hotRegionSchedulerConfig) handleSetConfig(w http.ResponseWriter, r * rd.Text(w, http.StatusBadRequest, "Config item is not found.") } -func (conf *hotRegionSchedulerConfig) persistLocked() error { - data, err := EncodeConfig(conf) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(HotRegionName, data) -} - func (conf *hotRegionSchedulerConfig) checkQuerySupport(cluster sche.SchedulerCluster) bool { version := cluster.GetSchedulerConfig().GetClusterVersion() querySupport := versioninfo.IsFeatureSupported(version, versioninfo.HotScheduleWithQuery) diff --git a/pkg/schedule/schedulers/init.go b/pkg/schedule/schedulers/init.go index 0e1917e8552..16f78284cf8 100644 --- a/pkg/schedule/schedulers/init.go +++ b/pkg/schedule/schedulers/init.go @@ -55,14 +55,18 @@ func schedulersRegister() { RegisterScheduler(types.BalanceLeaderScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { - conf := &balanceLeaderSchedulerConfig{storage: storage} + conf := &balanceLeaderSchedulerConfig{ + schedulerConfig: &baseSchedulerConfig{}, + } if err := decoder(conf); err != nil { return nil, err } if conf.Batch == 0 { conf.Batch = BalanceLeaderBatchSize } - return newBalanceLeaderScheduler(opController, conf), nil + sche := newBalanceLeaderScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // balance region @@ -109,14 +113,18 @@ func schedulersRegister() { RegisterScheduler(types.BalanceWitnessScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { - conf := &balanceWitnessSchedulerConfig{storage: storage} + conf := &balanceWitnessSchedulerConfig{ + schedulerConfig: &baseSchedulerConfig{}, + } if err := decoder(conf); err != nil { return nil, err } if conf.Batch == 0 { conf.Batch = balanceWitnessBatchSize } - return newBalanceWitnessScheduler(opController, conf), nil + sche := newBalanceWitnessScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // evict leader @@ -147,13 +155,18 @@ func schedulersRegister() { RegisterScheduler(types.EvictLeaderScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { - conf := &evictLeaderSchedulerConfig{StoreIDWithRanges: make(map[uint64][]core.KeyRange), storage: storage} + conf := &evictLeaderSchedulerConfig{ + schedulerConfig: &baseSchedulerConfig{}, + StoreIDWithRanges: make(map[uint64][]core.KeyRange), + } if err := decoder(conf); err != nil { return nil, err } conf.cluster = opController.GetCluster() conf.removeSchedulerCb = removeSchedulerCb[0] - return newEvictLeaderScheduler(opController, conf), nil + sche := newEvictLeaderScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // evict slow store @@ -165,12 +178,14 @@ func schedulersRegister() { RegisterScheduler(types.EvictSlowStoreScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { - conf := initEvictSlowStoreSchedulerConfig(storage) + conf := initEvictSlowStoreSchedulerConfig() if err := decoder(conf); err != nil { return nil, err } conf.cluster = opController.GetCluster() - return newEvictSlowStoreScheduler(opController, conf), nil + sche := newEvictSlowStoreScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // grant hot region @@ -206,12 +221,17 @@ func schedulersRegister() { RegisterScheduler(types.GrantHotRegionScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { - conf := &grantHotRegionSchedulerConfig{StoreIDs: make([]uint64, 0), storage: storage} + conf := &grantHotRegionSchedulerConfig{ + schedulerConfig: &baseSchedulerConfig{}, + StoreIDs: make([]uint64, 0), + } conf.cluster = opController.GetCluster() if err := decoder(conf); err != nil { return nil, err } - return newGrantHotRegionScheduler(opController, conf), nil + sche := newGrantHotRegionScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // hot region @@ -238,8 +258,9 @@ func schedulersRegister() { return nil, err } } - conf.storage = storage - return newHotScheduler(opController, conf), nil + sche := newHotScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // grant leader @@ -269,13 +290,18 @@ func schedulersRegister() { RegisterScheduler(types.GrantLeaderScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, removeSchedulerCb ...func(string) error) (Scheduler, error) { - conf := &grantLeaderSchedulerConfig{StoreIDWithRanges: make(map[uint64][]core.KeyRange), storage: storage} + conf := &grantLeaderSchedulerConfig{ + schedulerConfig: &baseSchedulerConfig{}, + StoreIDWithRanges: make(map[uint64][]core.KeyRange), + } conf.cluster = opController.GetCluster() conf.removeSchedulerCb = removeSchedulerCb[0] if err := decoder(conf); err != nil { return nil, err } - return newGrantLeaderScheduler(opController, conf), nil + sche := newGrantLeaderScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // label @@ -352,7 +378,7 @@ func schedulersRegister() { RegisterScheduler(types.ScatterRangeScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { conf := &scatterRangeSchedulerConfig{ - storage: storage, + schedulerConfig: &baseSchedulerConfig{}, } if err := decoder(conf); err != nil { return nil, err @@ -361,7 +387,9 @@ func schedulersRegister() { if len(rangeName) == 0 { return nil, errs.ErrSchedulerConfig.FastGenByArgs("range name") } - return newScatterRangeScheduler(opController, conf), nil + sche := newScatterRangeScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // shuffle hot region @@ -385,12 +413,16 @@ func schedulersRegister() { RegisterScheduler(types.ShuffleHotRegionScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { - conf := &shuffleHotRegionSchedulerConfig{Limit: uint64(1)} + conf := &shuffleHotRegionSchedulerConfig{ + schedulerConfig: &baseSchedulerConfig{}, + Limit: uint64(1), + } if err := decoder(conf); err != nil { return nil, err } - conf.storage = storage - return newShuffleHotRegionScheduler(opController, conf), nil + sche := newShuffleHotRegionScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // shuffle leader @@ -405,7 +437,6 @@ func schedulersRegister() { return err } conf.Ranges = ranges - conf.Name = ShuffleLeaderName return nil } }) @@ -438,11 +469,15 @@ func schedulersRegister() { RegisterScheduler(types.ShuffleRegionScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { - conf := &shuffleRegionSchedulerConfig{storage: storage} + conf := &shuffleRegionSchedulerConfig{ + schedulerConfig: &baseSchedulerConfig{}, + } if err := decoder(conf); err != nil { return nil, err } - return newShuffleRegionScheduler(opController, conf), nil + sche := newShuffleRegionScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // split bucket @@ -458,8 +493,9 @@ func schedulersRegister() { if err := decoder(conf); err != nil { return nil, err } - conf.storage = storage - return newSplitBucketScheduler(opController, conf), nil + sche := newSplitBucketScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) // transfer witness leader @@ -483,10 +519,13 @@ func schedulersRegister() { RegisterScheduler(types.EvictSlowTrendScheduler, func(opController *operator.Controller, storage endpoint.ConfigStorage, decoder ConfigDecoder, _ ...func(string) error) (Scheduler, error) { - conf := initEvictSlowTrendSchedulerConfig(storage) + conf := initEvictSlowTrendSchedulerConfig() if err := decoder(conf); err != nil { return nil, err } - return newEvictSlowTrendScheduler(opController, conf), nil + + sche := newEvictSlowTrendScheduler(opController, conf) + conf.init(sche.GetName(), storage, conf) + return sche, nil }) } diff --git a/pkg/schedule/schedulers/scatter_range.go b/pkg/schedule/schedulers/scatter_range.go index 9c9606b29a9..e86785fcc19 100644 --- a/pkg/schedule/schedulers/scatter_range.go +++ b/pkg/schedule/schedulers/scatter_range.go @@ -26,7 +26,6 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" types "github.com/tikv/pd/pkg/schedule/type" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -39,7 +38,8 @@ const ( type scatterRangeSchedulerConfig struct { syncutil.RWMutex - storage endpoint.ConfigStorage + schedulerConfig + RangeName string `json:"range-name"` StartKey string `json:"start-key"` EndKey string `json:"end-key"` @@ -69,14 +69,9 @@ func (conf *scatterRangeSchedulerConfig) clone() *scatterRangeSchedulerConfig { } func (conf *scatterRangeSchedulerConfig) persist() error { - name := conf.getSchedulerName() conf.RLock() defer conf.RUnlock() - data, err := EncodeConfig(conf) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(name, data) + return conf.save() } func (conf *scatterRangeSchedulerConfig) getRangeName() string { @@ -153,15 +148,8 @@ func (l *scatterRangeScheduler) EncodeConfig() ([]byte, error) { func (l *scatterRangeScheduler) ReloadConfig() error { l.config.Lock() defer l.config.Unlock() - cfgData, err := l.config.storage.LoadSchedulerConfig(l.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } newCfg := &scatterRangeSchedulerConfig{} - if err := DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := l.config.load(newCfg); err != nil { return err } l.config.RangeName = newCfg.RangeName diff --git a/pkg/schedule/schedulers/shuffle_hot_region.go b/pkg/schedule/schedulers/shuffle_hot_region.go index 686322961cb..f8544fff48d 100644 --- a/pkg/schedule/schedulers/shuffle_hot_region.go +++ b/pkg/schedule/schedulers/shuffle_hot_region.go @@ -28,7 +28,6 @@ import ( "github.com/tikv/pd/pkg/schedule/plan" types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/statistics" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -42,8 +41,9 @@ const ( type shuffleHotRegionSchedulerConfig struct { syncutil.RWMutex - storage endpoint.ConfigStorage - Limit uint64 `json:"limit"` + schedulerConfig + + Limit uint64 `json:"limit"` } func (conf *shuffleHotRegionSchedulerConfig) clone() *shuffleHotRegionSchedulerConfig { @@ -54,14 +54,6 @@ func (conf *shuffleHotRegionSchedulerConfig) clone() *shuffleHotRegionSchedulerC } } -func (conf *shuffleHotRegionSchedulerConfig) persistLocked() error { - data, err := EncodeConfig(conf) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(types.ShuffleHotRegionScheduler.String(), data) -} - func (conf *shuffleHotRegionSchedulerConfig) getLimit() uint64 { conf.RLock() defer conf.RUnlock() @@ -106,15 +98,8 @@ func (s *shuffleHotRegionScheduler) EncodeConfig() ([]byte, error) { func (s *shuffleHotRegionScheduler) ReloadConfig() error { s.conf.Lock() defer s.conf.Unlock() - cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } newCfg := &shuffleHotRegionSchedulerConfig{} - if err = DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := s.conf.load(newCfg); err != nil { return err } s.conf.Limit = newCfg.Limit @@ -224,11 +209,12 @@ func (handler *shuffleHotRegionHandler) updateConfig(w http.ResponseWriter, r *h handler.rd.JSON(w, http.StatusBadRequest, "invalid limit") return } + handler.config.Lock() defer handler.config.Unlock() previous := handler.config.Limit handler.config.Limit = uint64(limit) - err := handler.config.persistLocked() + err := handler.config.save() if err != nil { handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) handler.config.Limit = previous diff --git a/pkg/schedule/schedulers/shuffle_leader.go b/pkg/schedule/schedulers/shuffle_leader.go index 2cd6c231a11..4270613667b 100644 --- a/pkg/schedule/schedulers/shuffle_leader.go +++ b/pkg/schedule/schedulers/shuffle_leader.go @@ -32,7 +32,6 @@ const ( ) type shuffleLeaderSchedulerConfig struct { - Name string `json:"name"` Ranges []core.KeyRange `json:"ranges"` // TODO: When we prepare to use Ranges, we will need to implement the ReloadConfig function for this scheduler. } @@ -46,11 +45,11 @@ type shuffleLeaderScheduler struct { // newShuffleLeaderScheduler creates an admin scheduler that shuffles leaders // between stores. func newShuffleLeaderScheduler(opController *operator.Controller, conf *shuffleLeaderSchedulerConfig) Scheduler { + base := NewBaseScheduler(opController, types.ShuffleLeaderScheduler) filters := []filter.Filter{ - &filter.StoreStateFilter{ActionScope: conf.Name, TransferLeader: true, OperatorLevel: constant.Low}, - filter.NewSpecialUseFilter(conf.Name), + &filter.StoreStateFilter{ActionScope: base.GetName(), TransferLeader: true, OperatorLevel: constant.Low}, + filter.NewSpecialUseFilter(base.GetName()), } - base := NewBaseScheduler(opController, types.ShuffleLeaderScheduler) return &shuffleLeaderScheduler{ BaseScheduler: base, conf: conf, diff --git a/pkg/schedule/schedulers/shuffle_region.go b/pkg/schedule/schedulers/shuffle_region.go index c179efd32c1..5d4c49e0fcc 100644 --- a/pkg/schedule/schedulers/shuffle_region.go +++ b/pkg/schedule/schedulers/shuffle_region.go @@ -67,15 +67,8 @@ func (s *shuffleRegionScheduler) EncodeConfig() ([]byte, error) { func (s *shuffleRegionScheduler) ReloadConfig() error { s.conf.Lock() defer s.conf.Unlock() - cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } newCfg := &shuffleRegionSchedulerConfig{} - if err := DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := s.conf.load(newCfg); err != nil { return err } s.conf.Roles = newCfg.Roles diff --git a/pkg/schedule/schedulers/shuffle_region_config.go b/pkg/schedule/schedulers/shuffle_region_config.go index fbf53cfeb4d..2e3394a58df 100644 --- a/pkg/schedule/schedulers/shuffle_region_config.go +++ b/pkg/schedule/schedulers/shuffle_region_config.go @@ -21,7 +21,6 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/slice" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -37,7 +36,7 @@ var allRoles = []string{roleLeader, roleFollower, roleLearner} type shuffleRegionSchedulerConfig struct { syncutil.RWMutex - storage endpoint.ConfigStorage + schedulerConfig Ranges []core.KeyRange `json:"ranges"` Roles []string `json:"roles"` // can include `leader`, `follower`, `learner`. @@ -100,18 +99,10 @@ func (conf *shuffleRegionSchedulerConfig) handleSetRoles(w http.ResponseWriter, defer conf.Unlock() old := conf.Roles conf.Roles = roles - if err := conf.persist(); err != nil { + if err := conf.save(); err != nil { conf.Roles = old // revert rd.Text(w, http.StatusInternalServerError, err.Error()) return } rd.Text(w, http.StatusOK, "Config is updated.") } - -func (conf *shuffleRegionSchedulerConfig) persist() error { - data, err := EncodeConfig(conf) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(ShuffleRegionName, data) -} diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index 7b238890107..0d8fa614aef 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -30,7 +30,6 @@ import ( "github.com/tikv/pd/pkg/schedule/plan" types "github.com/tikv/pd/pkg/schedule/type" "github.com/tikv/pd/pkg/statistics/buckets" - "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/reflectutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -46,14 +45,15 @@ const ( func initSplitBucketConfig() *splitBucketSchedulerConfig { return &splitBucketSchedulerConfig{ - Degree: defaultHotDegree, - SplitLimit: defaultSplitLimit, + schedulerConfig: &baseSchedulerConfig{}, + Degree: defaultHotDegree, + SplitLimit: defaultSplitLimit, } } type splitBucketSchedulerConfig struct { syncutil.RWMutex - storage endpoint.ConfigStorage + schedulerConfig Degree int `json:"degree"` SplitLimit uint64 `json:"split-limit"` } @@ -66,14 +66,6 @@ func (conf *splitBucketSchedulerConfig) clone() *splitBucketSchedulerConfig { } } -func (conf *splitBucketSchedulerConfig) persistLocked() error { - data, err := EncodeConfig(conf) - if err != nil { - return err - } - return conf.storage.SaveSchedulerConfig(SplitBucketName, data) -} - func (conf *splitBucketSchedulerConfig) getDegree() int { conf.RLock() defer conf.RUnlock() @@ -120,7 +112,7 @@ func (h *splitBucketHandler) updateConfig(w http.ResponseWriter, r *http.Request } newc, _ := json.Marshal(h.conf) if !bytes.Equal(oldc, newc) { - if err := h.conf.persistLocked(); err != nil { + if err := h.conf.save(); err != nil { log.Warn("failed to save config", errs.ZapError(err)) } rd.Text(w, http.StatusOK, "Config is updated.") @@ -167,15 +159,8 @@ func newSplitBucketScheduler(opController *operator.Controller, conf *splitBucke func (s *splitBucketScheduler) ReloadConfig() error { s.conf.Lock() defer s.conf.Unlock() - cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) - if err != nil { - return err - } - if len(cfgData) == 0 { - return nil - } newCfg := &splitBucketSchedulerConfig{} - if err := DecodeConfig([]byte(cfgData), newCfg); err != nil { + if err := s.conf.load(newCfg); err != nil { return err } s.conf.SplitLimit = newCfg.SplitLimit From 3117f994e0d5ce2e56089c1e15fd5e653bd506d0 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Wed, 14 Aug 2024 12:22:36 +0800 Subject: [PATCH 14/42] tso: fix an invalid err != nil check (#8523) close tikv/pd#8524 Use the continue label to retry the whole merge loop correctly. Signed-off-by: JmPotato --- pkg/tso/keyspace_group_manager.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index 1a24e571688..6db935cdce8 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -1302,6 +1302,7 @@ func (kgm *KeyspaceGroupManager) mergingChecker(ctx context.Context, mergeTarget mergeMap[id] = struct{}{} } +mergeLoop: for { select { case <-ctx.Done(): @@ -1373,15 +1374,13 @@ func (kgm *KeyspaceGroupManager) mergingChecker(ctx context.Context, mergeTarget zap.Uint32("merge-id", id), zap.Time("ts", ts), zap.Error(err)) - break + // Retry from the beginning of the loop. + continue mergeLoop } if ts.After(mergedTS) { mergedTS = ts } } - if err != nil { - continue - } // Update the newly merged TSO if the merged TSO is not zero. if mergedTS != typeutil.ZeroTime { log.Info("start to set the newly merged TSO", From cc2900c42e1dfd3f3eda210e1e51715cac491053 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 15 Aug 2024 13:56:40 +0800 Subject: [PATCH 15/42] *: upgrade etcd to v3.5.15 (#8441) close tikv/pd#3877, ref tikv/pd#5134 Signed-off-by: lhy1024 --- go.mod | 30 ++++++-- go.sum | 77 +++++++++++++++---- pkg/autoscaling/calculation.go | 2 +- pkg/autoscaling/types.go | 2 +- pkg/basicserver/basic_server.go | 2 +- pkg/election/leadership.go | 4 +- pkg/election/leadership_test.go | 4 +- pkg/election/lease.go | 2 +- pkg/encryption/key_manager.go | 4 +- pkg/encryption/key_manager_test.go | 2 +- pkg/id/id.go | 2 +- pkg/keyspace/tso_keyspace_group.go | 4 +- pkg/mcs/discovery/discover.go | 2 +- pkg/mcs/discovery/register.go | 2 +- pkg/mcs/discovery/register_test.go | 4 +- pkg/mcs/metastorage/server/grpc_service.go | 2 +- pkg/mcs/metastorage/server/manager.go | 2 +- pkg/mcs/scheduling/server/config/watcher.go | 4 +- pkg/mcs/scheduling/server/meta/watcher.go | 4 +- pkg/mcs/scheduling/server/rule/watcher.go | 4 +- .../scheduling/server/rule/watcher_test.go | 4 +- pkg/mcs/server/server.go | 2 +- pkg/mcs/utils/expected_primary.go | 2 +- pkg/mcs/utils/util.go | 6 +- pkg/member/member.go | 4 +- pkg/member/participant.go | 2 +- pkg/storage/endpoint/config.go | 2 +- pkg/storage/endpoint/gc_safe_point.go | 2 +- pkg/storage/endpoint/keyspace.go | 2 +- pkg/storage/endpoint/safepoint_v2.go | 2 +- pkg/storage/endpoint/tso.go | 2 +- pkg/storage/endpoint/tso_keyspace_group.go | 2 +- pkg/storage/endpoint/util.go | 2 +- pkg/storage/etcd_backend.go | 2 +- pkg/storage/kv/etcd_kv.go | 2 +- pkg/storage/kv/kv_test.go | 2 +- pkg/storage/storage.go | 2 +- pkg/storage/storage_test.go | 2 +- pkg/tso/allocator_manager.go | 2 +- pkg/tso/keyspace_group_manager.go | 4 +- pkg/tso/keyspace_group_manager_test.go | 4 +- pkg/tso/local_allocator.go | 2 +- pkg/tso/tso.go | 2 +- pkg/utils/etcdutil/etcdutil.go | 26 ++++--- pkg/utils/etcdutil/etcdutil_test.go | 36 ++++++--- pkg/utils/etcdutil/health_checker.go | 2 +- pkg/utils/etcdutil/testutil.go | 15 +++- pkg/utils/grpcutil/grpcutil.go | 2 +- server/cluster/cluster.go | 2 +- server/config/config.go | 7 +- server/config/persist_options.go | 2 +- server/gc_service.go | 2 +- server/grpc_service.go | 2 +- server/join/join.go | 4 +- server/keyspace_service.go | 4 +- server/server.go | 14 ++-- server/server_test.go | 6 +- server/testutil.go | 2 +- tests/cluster.go | 2 +- tests/integrations/client/client_test.go | 2 +- tests/integrations/client/client_tls_test.go | 2 +- tests/integrations/client/gc_client_test.go | 2 +- tests/integrations/go.mod | 29 +++++-- tests/integrations/go.sum | 77 +++++++++++++++---- .../mcs/scheduling/server_test.go | 3 + tests/integrations/mcs/tso/server_test.go | 2 +- tests/server/tso/manager_test.go | 2 +- tools/go.mod | 29 +++++-- tools/go.sum | 77 +++++++++++++++---- tools/pd-api-bench/cases/cases.go | 2 +- tools/pd-api-bench/cases/controller.go | 2 +- tools/pd-api-bench/main.go | 2 +- tools/pd-backup/main.go | 4 +- tools/pd-backup/pdbackup/backup.go | 2 +- tools/pd-backup/pdbackup/backup_test.go | 4 +- tools/pd-backup/tests/backup_test.go | 2 +- tools/pd-ctl/pdctl/command/global.go | 2 +- tools/pd-ctl/tests/health/health_test.go | 2 +- tools/pd-ctl/tests/store/store_test.go | 2 +- tools/pd-heartbeat-bench/main.go | 2 +- tools/pd-heartbeat-bench/metrics/util.go | 2 +- tools/pd-recover/main.go | 4 +- tools/pd-simulator/simulator/drive.go | 2 +- tools/regions-dump/main.go | 4 +- tools/stores-dump/main.go | 4 +- 85 files changed, 417 insertions(+), 195 deletions(-) diff --git a/go.mod b/go.mod index aee8cc7afad..9dc5fa9f249 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20240716095229-5f7ffec83ea7 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 - github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b + github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c github.com/prometheus/client_golang v1.19.0 github.com/prometheus/common v0.51.1 github.com/sasha-s/go-deadlock v0.2.0 @@ -52,7 +52,10 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 github.com/unrolled/render v1.0.1 github.com/urfave/negroni v0.3.0 - go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca + go.etcd.io/etcd/api/v3 v3.5.15 + go.etcd.io/etcd/client/pkg/v3 v3.5.15 + go.etcd.io/etcd/client/v3 v3.5.15 + go.etcd.io/etcd/server/v3 v3.5.15 go.uber.org/atomic v1.10.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 @@ -84,11 +87,10 @@ require ( github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect github.com/bytedance/sonic v1.9.1 // indirect - github.com/cenkalti/backoff/v4 v4.0.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dnephin/pflag v1.0.7 // indirect @@ -99,6 +101,8 @@ require ( github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect @@ -112,6 +116,7 @@ require ( github.com/goccy/go-graphviz v0.1.3 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -121,6 +126,7 @@ require ( github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 // indirect github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -169,8 +175,18 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - // Fix panic in unit test with go >= 1.14, ref: etcd-io/bbolt#201 https://github.com/etcd-io/bbolt/pull/201 - go.etcd.io/bbolt v1.3.9 // indirect + go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/etcd/client/v2 v2.305.15 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.15 // indirect + go.etcd.io/etcd/raft/v3 v3.5.15 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/sdk v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 diff --git a/go.sum b/go.sum index 3bbddb05476..e3bc1ed175a 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/AlekSi/gocov-xml v1.0.0 h1:4QctJBgXEkbzeKz6PJy6bt3JSPNSN4I2mITYW+eKUoQ= github.com/AlekSi/gocov-xml v1.0.0/go.mod h1:J0qYeZ6tDg4oZubW9mAAgxlqw39PDfoEkzB3HXSbEuA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -67,8 +72,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= -github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= -github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -80,14 +85,14 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb h1:GIzvVQ9UkUlOhSDlqmrQAAAUd6R3E+caIisNEyWXvNE= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -111,6 +116,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= @@ -138,6 +145,11 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -175,6 +187,7 @@ github.com/goccy/go-graphviz v0.1.3/go.mod h1:pMYpbAqJT10V8dzV1JN/g/wUlG/0imKPzn github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -182,6 +195,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= @@ -189,8 +204,8 @@ github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EO github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -235,6 +250,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -377,8 +394,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b h1:MKgJ9yCQxD5ewLERuoiiD9XVOHuqZ2WRZnB20yMiKyo= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c h1:iiVpATdlLLr0NEgKZu+ZnBuYR4J8IgjNE1hNMjMOkYY= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -513,10 +530,38 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca h1:LCc0GAhfJ+qDqnUbE7ybQ0mTz1dNRn2iiM6e183p/5E= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca/go.mod h1:1AyK+XVcIwjbjw5EYrhT+IiMYSgRZTohGb2ceZ0/US8= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= +go.etcd.io/etcd/client/v2 v2.305.15 h1:VG2xbf8Vz1KJh65Ar2V5eDmfkp1bpzkSEHlhJM3usp8= +go.etcd.io/etcd/client/v2 v2.305.15/go.mod h1:Ad5dRjPVb/n5yXgAWQ/hXzuXXkBk0Y658ocuXYaUU48= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= +go.etcd.io/etcd/pkg/v3 v3.5.15 h1:/Iu6Sr3iYaAjy++8sIDoZW9/EfhcwLZwd4FOZX2mMOU= +go.etcd.io/etcd/pkg/v3 v3.5.15/go.mod h1:e3Acf298sPFmTCGTrnGvkClEw9RYIyPtNzi1XM8rets= +go.etcd.io/etcd/raft/v3 v3.5.15 h1:jOA2HJF7zb3wy8H/pL13e8geWqkEa/kUs0waUggZC0I= +go.etcd.io/etcd/raft/v3 v3.5.15/go.mod h1:k3r7P4seEiUcgxOPLp+mloJWV3Q4QLPGNvy/OgC8OtM= +go.etcd.io/etcd/server/v3 v3.5.15 h1:x35jrWnZgsRwMsFsUJIUdT1bvzIz1B+29HjMfRYVN/E= +go.etcd.io/etcd/server/v3 v3.5.15/go.mod h1:l9jX9oa/iuArjqz0RNX/TDbc70dLXxRZo/nmPucrpFo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/sdk v1.20.0 h1:5Jf6imeFZlZtKv9Qbo6qt2ZkmWtdWx/wzcCbNUlAWGM= +go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/pkg/autoscaling/calculation.go b/pkg/autoscaling/calculation.go index 8c8783dd618..b155c44bf68 100644 --- a/pkg/autoscaling/calculation.go +++ b/pkg/autoscaling/calculation.go @@ -29,7 +29,7 @@ import ( "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/autoscaling/types.go b/pkg/autoscaling/types.go index 5499d20f510..53c614312b2 100644 --- a/pkg/autoscaling/types.go +++ b/pkg/autoscaling/types.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // Strategy within a HTTP request provides rules and resources to help make decision for auto scaling. diff --git a/pkg/basicserver/basic_server.go b/pkg/basicserver/basic_server.go index 28ba3ad08de..4b7e4eeab50 100644 --- a/pkg/basicserver/basic_server.go +++ b/pkg/basicserver/basic_server.go @@ -18,7 +18,7 @@ import ( "context" "net/http" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // Server defines the common basic behaviors of a server diff --git a/pkg/election/leadership.go b/pkg/election/leadership.go index 1361d685b57..ec64a003c53 100644 --- a/pkg/election/leadership.go +++ b/pkg/election/leadership.go @@ -28,8 +28,8 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/syncutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/election/leadership_test.go b/pkg/election/leadership_test.go index e16c5842542..1846d671a61 100644 --- a/pkg/election/leadership_test.go +++ b/pkg/election/leadership_test.go @@ -25,8 +25,8 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" ) const defaultLeaseTimeout = 1 diff --git a/pkg/election/lease.go b/pkg/election/lease.go index c2e9eb97117..21bd43018b5 100644 --- a/pkg/election/lease.go +++ b/pkg/election/lease.go @@ -24,7 +24,7 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/encryption/key_manager.go b/pkg/encryption/key_manager.go index 28a5f04d0f7..5a029847a9d 100644 --- a/pkg/encryption/key_manager.go +++ b/pkg/encryption/key_manager.go @@ -26,8 +26,8 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/syncutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/encryption/key_manager_test.go b/pkg/encryption/key_manager_test.go index 26453eeb5b3..e827a714a3a 100644 --- a/pkg/encryption/key_manager_test.go +++ b/pkg/encryption/key_manager_test.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // #nosec G101 diff --git a/pkg/id/id.go b/pkg/id/id.go index ea4a2a54341..0225dedd4f1 100644 --- a/pkg/id/id.go +++ b/pkg/id/id.go @@ -24,7 +24,7 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go index ffab89f4f0e..5d7137ac3c2 100644 --- a/pkg/keyspace/tso_keyspace_group.go +++ b/pkg/keyspace/tso_keyspace_group.go @@ -37,8 +37,8 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/discovery/discover.go b/pkg/mcs/discovery/discover.go index c219cbc047f..119ce7957d9 100644 --- a/pkg/mcs/discovery/discover.go +++ b/pkg/mcs/discovery/discover.go @@ -23,7 +23,7 @@ import ( "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/discovery/register.go b/pkg/mcs/discovery/register.go index 36fb19f2673..ab27387ab91 100644 --- a/pkg/mcs/discovery/register.go +++ b/pkg/mcs/discovery/register.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/logutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/discovery/register_test.go b/pkg/mcs/discovery/register_test.go index 707c251e5fb..3ea0148d75f 100644 --- a/pkg/mcs/discovery/register_test.go +++ b/pkg/mcs/discovery/register_test.go @@ -24,8 +24,8 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" ) func TestRegister(t *testing.T) { diff --git a/pkg/mcs/metastorage/server/grpc_service.go b/pkg/mcs/metastorage/server/grpc_service.go index c3ecfa572a1..4f43114ea29 100644 --- a/pkg/mcs/metastorage/server/grpc_service.go +++ b/pkg/mcs/metastorage/server/grpc_service.go @@ -24,7 +24,7 @@ import ( bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/mcs/registry" "github.com/tikv/pd/pkg/utils/apiutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/pkg/mcs/metastorage/server/manager.go b/pkg/mcs/metastorage/server/manager.go index bd082f48202..17fee85c08b 100644 --- a/pkg/mcs/metastorage/server/manager.go +++ b/pkg/mcs/metastorage/server/manager.go @@ -19,7 +19,7 @@ import ( bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/scheduling/server/config/watcher.go b/pkg/mcs/scheduling/server/config/watcher.go index d1ca99bd36d..46201ba3cd3 100644 --- a/pkg/mcs/scheduling/server/config/watcher.go +++ b/pkg/mcs/scheduling/server/config/watcher.go @@ -29,8 +29,8 @@ import ( "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/scheduling/server/meta/watcher.go b/pkg/mcs/scheduling/server/meta/watcher.go index 2daa6766d75..42e9cf054ec 100644 --- a/pkg/mcs/scheduling/server/meta/watcher.go +++ b/pkg/mcs/scheduling/server/meta/watcher.go @@ -26,8 +26,8 @@ import ( "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/scheduling/server/rule/watcher.go b/pkg/mcs/scheduling/server/rule/watcher.go index ea90b9d4e49..118da18376c 100644 --- a/pkg/mcs/scheduling/server/rule/watcher.go +++ b/pkg/mcs/scheduling/server/rule/watcher.go @@ -26,8 +26,8 @@ import ( "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/scheduling/server/rule/watcher_test.go b/pkg/mcs/scheduling/server/rule/watcher_test.go index 37fce0a0ded..11b721b4e9e 100644 --- a/pkg/mcs/scheduling/server/rule/watcher_test.go +++ b/pkg/mcs/scheduling/server/rule/watcher_test.go @@ -28,8 +28,8 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" ) const ( diff --git a/pkg/mcs/server/server.go b/pkg/mcs/server/server.go index 31b6c2c76cb..fef05a85012 100644 --- a/pkg/mcs/server/server.go +++ b/pkg/mcs/server/server.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/grpcutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "google.golang.org/grpc" ) diff --git a/pkg/mcs/utils/expected_primary.go b/pkg/mcs/utils/expected_primary.go index d44b2eae436..c65d0a1cc5e 100644 --- a/pkg/mcs/utils/expected_primary.go +++ b/pkg/mcs/utils/expected_primary.go @@ -29,7 +29,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/mcs/utils/util.go b/pkg/mcs/utils/util.go index fa76aadc1e8..dfcfc0e312d 100644 --- a/pkg/mcs/utils/util.go +++ b/pkg/mcs/utils/util.go @@ -40,8 +40,8 @@ import ( "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/versioninfo" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/types" + etcdtypes "go.etcd.io/etcd/client/pkg/v3/types" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" @@ -123,7 +123,7 @@ func InitClient(s server) error { if err != nil { return err } - backendUrls, err := types.NewURLs(strings.Split(s.GetBackendEndpoints(), ",")) + backendUrls, err := etcdtypes.NewURLs(strings.Split(s.GetBackendEndpoints(), ",")) if err != nil { return err } diff --git a/pkg/member/member.go b/pkg/member/member.go index 9c61a832b5f..8bf50e6d310 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -34,8 +34,8 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/zap" ) diff --git a/pkg/member/participant.go b/pkg/member/participant.go index 5e35d127bf7..599e56387d6 100644 --- a/pkg/member/participant.go +++ b/pkg/member/participant.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/storage/endpoint/config.go b/pkg/storage/endpoint/config.go index 820778e36ff..71ae8cb5248 100644 --- a/pkg/storage/endpoint/config.go +++ b/pkg/storage/endpoint/config.go @@ -19,7 +19,7 @@ import ( "strings" "github.com/tikv/pd/pkg/errs" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // ConfigStorage defines the storage operations on the config. diff --git a/pkg/storage/endpoint/gc_safe_point.go b/pkg/storage/endpoint/gc_safe_point.go index 85b29e0b47e..8d59d827fa4 100644 --- a/pkg/storage/endpoint/gc_safe_point.go +++ b/pkg/storage/endpoint/gc_safe_point.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // ServiceSafePoint is the safepoint for a specific service diff --git a/pkg/storage/endpoint/keyspace.go b/pkg/storage/endpoint/keyspace.go index 30540e49a2e..b892250c463 100644 --- a/pkg/storage/endpoint/keyspace.go +++ b/pkg/storage/endpoint/keyspace.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/storage/kv" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) const ( diff --git a/pkg/storage/endpoint/safepoint_v2.go b/pkg/storage/endpoint/safepoint_v2.go index 8d690d07261..2072f9c12ec 100644 --- a/pkg/storage/endpoint/safepoint_v2.go +++ b/pkg/storage/endpoint/safepoint_v2.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/storage/endpoint/tso.go b/pkg/storage/endpoint/tso.go index f0aa9d8cde0..30aceb88fa8 100644 --- a/pkg/storage/endpoint/tso.go +++ b/pkg/storage/endpoint/tso.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/storage/endpoint/tso_keyspace_group.go b/pkg/storage/endpoint/tso_keyspace_group.go index 5b6a7481176..9c7b12fff5f 100644 --- a/pkg/storage/endpoint/tso_keyspace_group.go +++ b/pkg/storage/endpoint/tso_keyspace_group.go @@ -21,7 +21,7 @@ import ( "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // UserKind represents the user kind. diff --git a/pkg/storage/endpoint/util.go b/pkg/storage/endpoint/util.go index 06535490206..39aa6240f5a 100644 --- a/pkg/storage/endpoint/util.go +++ b/pkg/storage/endpoint/util.go @@ -23,7 +23,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) func (se *StorageEndpoint) loadProto(key string, msg proto.Message) (bool, error) { diff --git a/pkg/storage/etcd_backend.go b/pkg/storage/etcd_backend.go index f354c1bb070..e9af5fc67f6 100644 --- a/pkg/storage/etcd_backend.go +++ b/pkg/storage/etcd_backend.go @@ -17,7 +17,7 @@ package storage import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // etcdBackend is a storage backend that stores data in etcd, diff --git a/pkg/storage/kv/etcd_kv.go b/pkg/storage/kv/etcd_kv.go index e2eb8c979eb..5945990c51b 100644 --- a/pkg/storage/kv/etcd_kv.go +++ b/pkg/storage/kv/etcd_kv.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/storage/kv/kv_test.go b/pkg/storage/kv/kv_test.go index 93359934da1..3bc3120af95 100644 --- a/pkg/storage/kv/kv_test.go +++ b/pkg/storage/kv/kv_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) func TestEtcd(t *testing.T) { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 5e006133d22..dce2f1712b8 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -24,7 +24,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/syncutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // Storage is the interface for the backend storage of the PD. diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 460489ecd10..d5f27e34714 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -30,7 +30,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/storage/endpoint" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) func TestBasic(t *testing.T) { diff --git a/pkg/tso/allocator_manager.go b/pkg/tso/allocator_manager.go index 9cd5ae9c743..1f5bce04583 100644 --- a/pkg/tso/allocator_manager.go +++ b/pkg/tso/allocator_manager.go @@ -40,7 +40,7 @@ import ( "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" ) diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index 6db935cdce8..0e62fc0346c 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -45,8 +45,8 @@ import ( "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/tso/keyspace_group_manager_test.go b/pkg/tso/keyspace_group_manager_test.go index b891aeb246d..fb4ac92a59c 100644 --- a/pkg/tso/keyspace_group_manager_test.go +++ b/pkg/tso/keyspace_group_manager_test.go @@ -40,8 +40,8 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/goleak" ) diff --git a/pkg/tso/local_allocator.go b/pkg/tso/local_allocator.go index e9019bf2bf3..3bbaa661dd6 100644 --- a/pkg/tso/local_allocator.go +++ b/pkg/tso/local_allocator.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/tso/tso.go b/pkg/tso/tso.go index bcb3169e73c..d04710ac378 100644 --- a/pkg/tso/tso.go +++ b/pkg/tso/tso.go @@ -31,7 +31,7 @@ import ( "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/utils/etcdutil/etcdutil.go b/pkg/utils/etcdutil/etcdutil.go index 2b89c8b4da6..3eb1afabeac 100644 --- a/pkg/utils/etcdutil/etcdutil.go +++ b/pkg/utils/etcdutil/etcdutil.go @@ -33,11 +33,12 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/etcdserver" - "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" - "go.etcd.io/etcd/mvcc/mvccpb" - "go.etcd.io/etcd/pkg/types" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/mvccpb" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + etcdtypes "go.etcd.io/etcd/client/pkg/v3/types" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/etcdserver" "go.uber.org/zap" "google.golang.org/grpc/codes" ) @@ -74,7 +75,7 @@ const ( // CheckClusterID checks etcd cluster ID, returns an error if mismatch. // This function will never block even quorum is not satisfied. -func CheckClusterID(localClusterID types.ID, um types.URLsMap, tlsConfig *tls.Config) error { +func CheckClusterID(localClusterID etcdtypes.ID, um etcdtypes.URLsMap, tlsConfig *tls.Config) error { if len(um) == 0 { return nil } @@ -88,7 +89,7 @@ func CheckClusterID(localClusterID types.ID, um types.URLsMap, tlsConfig *tls.Co trp := &http.Transport{ TLSClientConfig: tlsConfig, } - remoteCluster, gerr := etcdserver.GetClusterFromRemotePeers(nil, []string{u}, trp, true) + remoteCluster, gerr := etcdserver.GetClusterFromRemotePeers(nil, []string{u}, trp) trp.CloseIdleConnections() if gerr != nil { // Do not return error, because other members may be not ready. @@ -119,12 +120,17 @@ func ListEtcdMembers(ctx context.Context, client *clientv3.Client) (*clientv3.Me time.Sleep(time.Duration(d) * time.Second) }) newCtx, cancel := context.WithTimeout(ctx, DefaultRequestTimeout) - listResp, err := client.MemberList(newCtx) + // After the etcd server is upgraded to v3.5.x, the MemberList API will return the member list in a linearizable way by default. + // It is introduced by https://github.com/etcd-io/etcd/pull/11639 + // If Linearizable is set to false, the member list will be returned with server's local data. + // If Linearizable is set to true, it is served with linearizable guarantee. If the server is disconnected from quorum, `MemberList` call will fail. + c := clientv3.RetryClusterClient(client) + resp, err := c.MemberList(newCtx, &etcdserverpb.MemberListRequest{Linearizable: false}) cancel() if err != nil { - return listResp, errs.ErrEtcdMemberList.Wrap(err).GenWithStackByCause() + return (*clientv3.MemberListResponse)(resp), errs.ErrEtcdMemberList.Wrap(err).GenWithStackByCause() } - return listResp, nil + return (*clientv3.MemberListResponse)(resp), nil } // RemoveEtcdMember removes a member by the given id. diff --git a/pkg/utils/etcdutil/etcdutil_test.go b/pkg/utils/etcdutil/etcdutil_test.go index c402081fa2f..99f71ffde05 100644 --- a/pkg/utils/etcdutil/etcdutil_test.go +++ b/pkg/utils/etcdutil/etcdutil_test.go @@ -34,11 +34,11 @@ import ( "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/etcdserver/etcdserverpb" - "go.etcd.io/etcd/mvcc/mvccpb" - "go.etcd.io/etcd/pkg/types" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/mvccpb" + etcdtypes "go.etcd.io/etcd/client/pkg/v3/types" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/goleak" ) @@ -46,6 +46,16 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, testutil.LeakOptions...) } +func TestAddMember(t *testing.T) { + re := require.New(t) + servers, client1, clean := NewTestEtcdCluster(t, 1) + defer clean() + etcd1, cfg1 := servers[0], servers[0].Config() + etcd2 := MustAddEtcdMember(t, &cfg1, client1) + defer etcd2.Close() + checkMembers(re, client1, []*embed.Etcd{etcd1, etcd2}) +} + func TestMemberHelpers(t *testing.T) { re := require.New(t) servers, client1, clean := NewTestEtcdCluster(t, 1) @@ -65,7 +75,7 @@ func TestMemberHelpers(t *testing.T) { checkMembers(re, client1, []*embed.Etcd{etcd1, etcd2}) // Test CheckClusterID - urlsMap, err := types.NewURLsMap(etcd2.Config().InitialCluster) + urlsMap, err := etcdtypes.NewURLsMap(etcd2.Config().InitialCluster) re.NoError(err) err = CheckClusterID(etcd1.Server.Cluster().ID(), urlsMap, &tls.Config{MinVersion: tls.VersionTLS12}) re.NoError(err) @@ -172,7 +182,6 @@ func TestEtcdClientSync(t *testing.T) { servers, client1, clean := NewTestEtcdCluster(t, 1) defer clean() etcd1, cfg1 := servers[0], servers[0].Config() - defer etcd1.Close() // Add a new member. etcd2 := MustAddEtcdMember(t, &cfg1, client1) @@ -303,7 +312,7 @@ func checkEtcdWithHangLeader(t *testing.T) error { go proxyWithDiscard(ctx, re, cfg1.ListenClientUrls[0].String(), proxyAddr, &enableDiscard) // Create an etcd client with etcd1 as endpoint. - urls, err := types.NewURLs([]string{proxyAddr}) + urls, err := etcdtypes.NewURLs([]string{proxyAddr}) re.NoError(err) client1, err := CreateEtcdClient(nil, urls) re.NoError(err) @@ -771,7 +780,16 @@ func (suite *loopWatcherTestSuite) startEtcd(re *require.Assertions) { suite.etcd = etcd1 <-etcd1.Server.ReadyNotify() suite.cleans = append(suite.cleans, func() { - suite.etcd.Close() + if suite.etcd.Server != nil { + select { + case _, ok := <-suite.etcd.Err(): + if !ok { + return + } + default: + } + suite.etcd.Close() + } }) } diff --git a/pkg/utils/etcdutil/health_checker.go b/pkg/utils/etcdutil/health_checker.go index c5ece5a0804..a933279ff2b 100644 --- a/pkg/utils/etcdutil/health_checker.go +++ b/pkg/utils/etcdutil/health_checker.go @@ -26,7 +26,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/pkg/utils/etcdutil/testutil.go b/pkg/utils/etcdutil/testutil.go index 13c10260a40..e67ae415736 100644 --- a/pkg/utils/etcdutil/testutil.go +++ b/pkg/utils/etcdutil/testutil.go @@ -24,9 +24,9 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/etcdserver/etcdserverpb" + "go.etcd.io/etcd/api/v3/etcdserverpb" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" ) // NewTestSingleConfig is used to create a etcd config for the unit test purpose. @@ -86,7 +86,14 @@ func NewTestEtcdCluster(t *testing.T, count int) (servers []*embed.Etcd, etcdCli clean = func() { etcdClient.Close() for _, server := range servers { - if server != nil { + if server.Server != nil { + select { + case _, ok := <-server.Err(): + if !ok { + return + } + default: + } server.Close() } } diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index 5633533ae4a..5f852ff7359 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/logutil" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/backoff" diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index a2a365bb4c1..9e12b158f42 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -65,7 +65,7 @@ import ( "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/server/config/config.go b/server/config/config.go index 81da5f6d9db..8c4f6eaacc8 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -39,8 +39,8 @@ import ( "github.com/tikv/pd/pkg/utils/metricutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/zap" ) @@ -739,6 +739,9 @@ func (c *Config) GenEmbedEtcdConfig() (*embed.Config, error) { cfg.PeerTLSInfo.TrustedCAFile = c.Security.CAPath cfg.PeerTLSInfo.CertFile = c.Security.CertPath cfg.PeerTLSInfo.KeyFile = c.Security.KeyPath + // TODO: After https://github.com/etcd-io/etcd/pull/18015, AllowedCN is Deprecated. + // It will be replaced by AllowedCNs in the future to support multi cn. + // nolint:staticcheck cfg.PeerTLSInfo.AllowedCN = allowedCN cfg.ForceNewCluster = c.ForceNewCluster cfg.ZapLoggerBuilder = embed.NewZapCoreLoggerBuilder(c.Logger, c.Logger.Core(), c.LogProps.Syncer) diff --git a/server/config/persist_options.go b/server/config/persist_options.go index b6963a6645a..19f0ef0d475 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -38,7 +38,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/server/gc_service.go b/server/gc_service.go index 8c967d542ee..d31e047b1ea 100644 --- a/server/gc_service.go +++ b/server/gc_service.go @@ -28,7 +28,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/tsoutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" ) diff --git a/server/grpc_service.go b/server/grpc_service.go index 0f241986a9b..f7a0ea9443e 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -47,7 +47,7 @@ import ( "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/cluster" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/multierr" "go.uber.org/zap" "google.golang.org/grpc" diff --git a/server/join/join.go b/server/join/join.go index 8da90b7201b..bdc2704cd47 100644 --- a/server/join/join.go +++ b/server/join/join.go @@ -27,8 +27,8 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/zap" ) diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 44973a83061..afc5b41b1b6 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -25,8 +25,8 @@ import ( "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" ) // KeyspaceServer wraps GrpcServer to provide keyspace service. diff --git a/server/server.go b/server/server.go index 6e0179798e8..205d220180f 100644 --- a/server/server.go +++ b/server/server.go @@ -80,10 +80,10 @@ import ( "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/mvcc/mvccpb" - "go.etcd.io/etcd/pkg/types" + "go.etcd.io/etcd/api/v3/mvccpb" + etcdtypes "go.etcd.io/etcd/client/pkg/v3/types" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -331,7 +331,7 @@ func (s *Server) startEtcd(ctx context.Context) error { } // Check cluster ID - urlMap, err := types.NewURLsMap(s.cfg.InitialCluster) + urlMap, err := etcdtypes.NewURLsMap(s.cfg.InitialCluster) if err != nil { return errs.ErrEtcdURLMap.Wrap(err).GenWithStackByCause() } @@ -1667,8 +1667,8 @@ func (s *Server) leaderLoop() { log.Info("the pd leader is lost for a long time, try to re-campaign a pd leader with resign etcd leader", zap.Duration("timeout", randomTimeout), zap.Time("last-updated", lastUpdated), - zap.String("current-leader-member-id", types.ID(etcdLeader).String()), - zap.String("transferee-member-id", types.ID(s.member.ID()).String()), + zap.String("current-leader-member-id", etcdtypes.ID(etcdLeader).String()), + zap.String("transferee-member-id", etcdtypes.ID(s.member.ID()).String()), ) if err := s.member.MoveEtcdLeader(s.ctx, etcdLeader, s.member.ID()); err != nil { log.Error("failed to move etcd leader", errs.ZapError(err)) diff --git a/server/server_test.go b/server/server_test.go index 80771721a60..50c116083e0 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -30,8 +30,8 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/embed" - "go.etcd.io/etcd/pkg/types" + etcdtypes "go.etcd.io/etcd/client/pkg/v3/types" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/goleak" ) @@ -321,7 +321,7 @@ func TestCheckClusterID(t *testing.T) { etcd, err := embed.StartEtcd(svr.etcdCfg) re.NoError(err) - urlsMap, err := types.NewURLsMap(svr.cfg.InitialCluster) + urlsMap, err := etcdtypes.NewURLsMap(svr.cfg.InitialCluster) re.NoError(err) tlsConfig, err := svr.cfg.Security.ToTLSConfig() re.NoError(err) diff --git a/server/testutil.go b/server/testutil.go index be6b2bbebb0..7b159d7d94b 100644 --- a/server/testutil.go +++ b/server/testutil.go @@ -33,7 +33,7 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/embed" + "go.etcd.io/etcd/server/v3/embed" ) // NewTestServer creates a pd server for testing. diff --git a/tests/cluster.go b/tests/cluster.go index ad796226e31..f9cbb99ce25 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -47,7 +47,7 @@ import ( "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" "github.com/tikv/pd/server/join" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // TestServer states. diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 79eec0accde..9517444d017 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -56,7 +56,7 @@ import ( "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" "github.com/tikv/pd/tests/integrations/mcs" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/goleak" ) diff --git a/tests/integrations/client/client_tls_test.go b/tests/integrations/client/client_tls_test.go index 091fea2a4c8..b915d77b3b0 100644 --- a/tests/integrations/client/client_tls_test.go +++ b/tests/integrations/client/client_tls_test.go @@ -32,7 +32,7 @@ import ( "github.com/tikv/pd/pkg/utils/netutil" "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" "google.golang.org/grpc" ) diff --git a/tests/integrations/client/gc_client_test.go b/tests/integrations/client/gc_client_test.go index 0913579f47e..8494b8edbcf 100644 --- a/tests/integrations/client/gc_client_test.go +++ b/tests/integrations/client/gc_client_test.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/utils/assertutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" ) diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index f56b1cdc32a..d7305379e9f 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -21,7 +21,8 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 - go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca + go.etcd.io/etcd/client/pkg/v3 v3.5.15 + go.etcd.io/etcd/client/v3 v3.5.15 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.62.1 @@ -57,13 +58,12 @@ require ( github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect - github.com/cenkalti/backoff/v4 v4.0.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudfoundry/gosigar v1.3.6 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect @@ -74,6 +74,8 @@ require ( github.com/gin-contrib/pprof v1.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.9.1 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect @@ -87,6 +89,7 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -98,6 +101,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 // indirect github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -125,7 +129,7 @@ require ( github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -159,7 +163,20 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.etcd.io/bbolt v1.3.9 // indirect + go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/etcd/api/v3 v3.5.15 // indirect + go.etcd.io/etcd/client/v2 v2.305.15 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.15 // indirect + go.etcd.io/etcd/raft/v3 v3.5.15 // indirect + go.etcd.io/etcd/server/v3 v3.5.15 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/sdk v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index e23f71d41ce..914ce1ff35c 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -1,5 +1,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -65,8 +70,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= -github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= -github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -80,14 +85,14 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudfoundry/gosigar v1.3.6 h1:gIc08FbB3QPb+nAQhINIK/qhf5REKkY0FTGgRGXkcVc= github.com/cloudfoundry/gosigar v1.3.6/go.mod h1:lNWstu5g5gw59O09Y+wsMNFzBSnU8a0u+Sfx4dq360E= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb h1:GIzvVQ9UkUlOhSDlqmrQAAAUd6R3E+caIisNEyWXvNE= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -108,6 +113,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -132,6 +139,11 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -169,6 +181,7 @@ github.com/goccy/go-graphviz v0.1.3/go.mod h1:pMYpbAqJT10V8dzV1JN/g/wUlG/0imKPzn github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -177,6 +190,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= @@ -184,8 +199,8 @@ github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EO github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -228,6 +243,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -374,8 +391,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b h1:MKgJ9yCQxD5ewLERuoiiD9XVOHuqZ2WRZnB20yMiKyo= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c h1:iiVpATdlLLr0NEgKZu+ZnBuYR4J8IgjNE1hNMjMOkYY= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -504,10 +521,38 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca h1:LCc0GAhfJ+qDqnUbE7ybQ0mTz1dNRn2iiM6e183p/5E= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca/go.mod h1:1AyK+XVcIwjbjw5EYrhT+IiMYSgRZTohGb2ceZ0/US8= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= +go.etcd.io/etcd/client/v2 v2.305.15 h1:VG2xbf8Vz1KJh65Ar2V5eDmfkp1bpzkSEHlhJM3usp8= +go.etcd.io/etcd/client/v2 v2.305.15/go.mod h1:Ad5dRjPVb/n5yXgAWQ/hXzuXXkBk0Y658ocuXYaUU48= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= +go.etcd.io/etcd/pkg/v3 v3.5.15 h1:/Iu6Sr3iYaAjy++8sIDoZW9/EfhcwLZwd4FOZX2mMOU= +go.etcd.io/etcd/pkg/v3 v3.5.15/go.mod h1:e3Acf298sPFmTCGTrnGvkClEw9RYIyPtNzi1XM8rets= +go.etcd.io/etcd/raft/v3 v3.5.15 h1:jOA2HJF7zb3wy8H/pL13e8geWqkEa/kUs0waUggZC0I= +go.etcd.io/etcd/raft/v3 v3.5.15/go.mod h1:k3r7P4seEiUcgxOPLp+mloJWV3Q4QLPGNvy/OgC8OtM= +go.etcd.io/etcd/server/v3 v3.5.15 h1:x35jrWnZgsRwMsFsUJIUdT1bvzIz1B+29HjMfRYVN/E= +go.etcd.io/etcd/server/v3 v3.5.15/go.mod h1:l9jX9oa/iuArjqz0RNX/TDbc70dLXxRZo/nmPucrpFo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/sdk v1.20.0 h1:5Jf6imeFZlZtKv9Qbo6qt2ZkmWtdWx/wzcCbNUlAWGM= +go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index ab6bb93c60c..5c08892a972 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -675,11 +675,14 @@ func (suite *multipleServerTestSuite) TestReElectLeader() { newLeaderName := suite.cluster.WaitLeader() re.NotEqual(originLeaderName, newLeaderName) + suite.pdLeader = suite.cluster.GetServer(newLeaderName) suite.pdLeader.ResignLeader() newLeaderName = suite.cluster.WaitLeader() re.Equal(originLeaderName, newLeaderName) + suite.pdLeader = suite.cluster.GetServer(newLeaderName) rc = suite.pdLeader.GetServer().GetRaftCluster() + re.NotNil(rc) rc.IsPrepared() } diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 980f0db6f12..7ab92026197 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -42,7 +42,7 @@ import ( "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/tests" "github.com/tikv/pd/tests/integrations/mcs" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/goleak" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" diff --git a/tests/server/tso/manager_test.go b/tests/server/tso/manager_test.go index c4f5aa5e3e3..1feb74e6643 100644 --- a/tests/server/tso/manager_test.go +++ b/tests/server/tso/manager_test.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) // TestClusterDCLocations will write different dc-locations to each server diff --git a/tools/go.mod b/tools/go.mod index 7152d25fed9..b1a3c1e3543 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -31,7 +31,10 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 - go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca + go.etcd.io/etcd/client/pkg/v3 v3.5.15 + go.etcd.io/etcd/client/v3 v3.5.15 + go.etcd.io/etcd/pkg/v3 v3.5.15 + go.etcd.io/etcd/server/v3 v3.5.15 go.uber.org/automaxprocs v1.5.3 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 @@ -65,17 +68,18 @@ require ( github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect github.com/bytedance/sonic v1.9.1 // indirect - github.com/cenkalti/backoff/v4 v4.0.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect @@ -90,6 +94,7 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -101,6 +106,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 // indirect github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -127,7 +133,7 @@ require ( github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -159,7 +165,18 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.etcd.io/bbolt v1.3.9 // indirect + go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/etcd/api/v3 v3.5.15 // indirect + go.etcd.io/etcd/client/v2 v2.305.15 // indirect + go.etcd.io/etcd/raft/v3 v3.5.15 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/sdk v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect diff --git a/tools/go.sum b/tools/go.sum index d9bc29aebff..0933f7ceda7 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -1,5 +1,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -63,8 +68,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= -github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= -github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -79,14 +84,14 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWs github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb h1:GIzvVQ9UkUlOhSDlqmrQAAAUd6R3E+caIisNEyWXvNE= -github.com/coreos/pkg v0.0.0-20240122114842-bbd7aa9bf6fb/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -107,6 +112,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -131,6 +138,11 @@ github.com/go-echarts/go-echarts v1.0.0 h1:n181E4iXwj4zrU9VYmdM2m8dyhERt2w9k9YhH github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164yO4/wedFbcWyo= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -168,6 +180,7 @@ github.com/goccy/go-graphviz v0.1.3/go.mod h1:pMYpbAqJT10V8dzV1JN/g/wUlG/0imKPzn github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -175,6 +188,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= @@ -182,8 +197,8 @@ github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EO github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -226,6 +241,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69 h1:7xsUJsB2NrdcttQPa7JLEaGzvdbk7KvfrjgHZXOQRo0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -371,8 +388,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b h1:MKgJ9yCQxD5ewLERuoiiD9XVOHuqZ2WRZnB20yMiKyo= -github.com/pingcap/tidb-dashboard v0.0.0-20240718034516-e6e78c7c120b/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c h1:iiVpATdlLLr0NEgKZu+ZnBuYR4J8IgjNE1hNMjMOkYY= +github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -504,10 +521,38 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca h1:LCc0GAhfJ+qDqnUbE7ybQ0mTz1dNRn2iiM6e183p/5E= -go.etcd.io/etcd v0.5.0-alpha.5.0.20240320135013-950cd5fbe6ca/go.mod h1:1AyK+XVcIwjbjw5EYrhT+IiMYSgRZTohGb2ceZ0/US8= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= +go.etcd.io/etcd/client/v2 v2.305.15 h1:VG2xbf8Vz1KJh65Ar2V5eDmfkp1bpzkSEHlhJM3usp8= +go.etcd.io/etcd/client/v2 v2.305.15/go.mod h1:Ad5dRjPVb/n5yXgAWQ/hXzuXXkBk0Y658ocuXYaUU48= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= +go.etcd.io/etcd/pkg/v3 v3.5.15 h1:/Iu6Sr3iYaAjy++8sIDoZW9/EfhcwLZwd4FOZX2mMOU= +go.etcd.io/etcd/pkg/v3 v3.5.15/go.mod h1:e3Acf298sPFmTCGTrnGvkClEw9RYIyPtNzi1XM8rets= +go.etcd.io/etcd/raft/v3 v3.5.15 h1:jOA2HJF7zb3wy8H/pL13e8geWqkEa/kUs0waUggZC0I= +go.etcd.io/etcd/raft/v3 v3.5.15/go.mod h1:k3r7P4seEiUcgxOPLp+mloJWV3Q4QLPGNvy/OgC8OtM= +go.etcd.io/etcd/server/v3 v3.5.15 h1:x35jrWnZgsRwMsFsUJIUdT1bvzIz1B+29HjMfRYVN/E= +go.etcd.io/etcd/server/v3 v3.5.15/go.mod h1:l9jX9oa/iuArjqz0RNX/TDbc70dLXxRZo/nmPucrpFo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/sdk v1.20.0 h1:5Jf6imeFZlZtKv9Qbo6qt2ZkmWtdWx/wzcCbNUlAWGM= +go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/tools/pd-api-bench/cases/cases.go b/tools/pd-api-bench/cases/cases.go index 6ffa429fdb7..d81e7e0fa9d 100644 --- a/tools/pd-api-bench/cases/cases.go +++ b/tools/pd-api-bench/cases/cases.go @@ -24,7 +24,7 @@ import ( "github.com/pingcap/log" pd "github.com/tikv/pd/client" pdHttp "github.com/tikv/pd/client/http" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/tools/pd-api-bench/cases/controller.go b/tools/pd-api-bench/cases/controller.go index dc48b5280cf..a4e20f25758 100644 --- a/tools/pd-api-bench/cases/controller.go +++ b/tools/pd-api-bench/cases/controller.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/log" pd "github.com/tikv/pd/client" pdHttp "github.com/tikv/pd/client/http" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/tools/pd-api-bench/main.go b/tools/pd-api-bench/main.go index cb472fb5973..f642ede0069 100644 --- a/tools/pd-api-bench/main.go +++ b/tools/pd-api-bench/main.go @@ -40,7 +40,7 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/tools/pd-api-bench/cases" "github.com/tikv/pd/tools/pd-api-bench/config" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" diff --git a/tools/pd-backup/main.go b/tools/pd-backup/main.go index c424113e451..c0ee07afd6e 100644 --- a/tools/pd-backup/main.go +++ b/tools/pd-backup/main.go @@ -22,8 +22,8 @@ import ( "time" "github.com/tikv/pd/tools/pd-backup/pdbackup" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" ) var ( diff --git a/tools/pd-backup/pdbackup/backup.go b/tools/pd-backup/pdbackup/backup.go index f0b17e48267..7eea1fb4b0b 100644 --- a/tools/pd-backup/pdbackup/backup.go +++ b/tools/pd-backup/pdbackup/backup.go @@ -29,7 +29,7 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) const ( diff --git a/tools/pd-backup/pdbackup/backup_test.go b/tools/pd-backup/pdbackup/backup_test.go index b67873baf8e..f11cfef8e42 100644 --- a/tools/pd-backup/pdbackup/backup_test.go +++ b/tools/pd-backup/pdbackup/backup_test.go @@ -22,8 +22,8 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server/config" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/embed" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" "go.uber.org/goleak" ) diff --git a/tools/pd-backup/tests/backup_test.go b/tools/pd-backup/tests/backup_test.go index 83a465e94fe..2a55a790849 100644 --- a/tools/pd-backup/tests/backup_test.go +++ b/tools/pd-backup/tests/backup_test.go @@ -25,7 +25,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/tests" "github.com/tikv/pd/tools/pd-backup/pdbackup" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" ) func TestBackup(t *testing.T) { diff --git a/tools/pd-ctl/pdctl/command/global.go b/tools/pd-ctl/pdctl/command/global.go index 7438345127e..c5c633de217 100644 --- a/tools/pd-ctl/pdctl/command/global.go +++ b/tools/pd-ctl/pdctl/command/global.go @@ -29,7 +29,7 @@ import ( "github.com/spf13/cobra" pd "github.com/tikv/pd/client/http" "github.com/tikv/pd/pkg/utils/apiutil" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" ) const ( diff --git a/tools/pd-ctl/tests/health/health_test.go b/tools/pd-ctl/tests/health/health_test.go index 1d6cf884ccf..be9d5027988 100644 --- a/tools/pd-ctl/tests/health/health_test.go +++ b/tools/pd-ctl/tests/health/health_test.go @@ -31,7 +31,7 @@ import ( pdTests "github.com/tikv/pd/tests" ctl "github.com/tikv/pd/tools/pd-ctl/pdctl" "github.com/tikv/pd/tools/pd-ctl/tests" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" ) func TestHealth(t *testing.T) { diff --git a/tools/pd-ctl/tests/store/store_test.go b/tools/pd-ctl/tests/store/store_test.go index ae35839837e..983c86b242a 100644 --- a/tools/pd-ctl/tests/store/store_test.go +++ b/tools/pd-ctl/tests/store/store_test.go @@ -35,7 +35,7 @@ import ( pdTests "github.com/tikv/pd/tests" ctl "github.com/tikv/pd/tools/pd-ctl/pdctl" "github.com/tikv/pd/tools/pd-ctl/tests" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" ) func TestStoreLimitV2(t *testing.T) { diff --git a/tools/pd-heartbeat-bench/main.go b/tools/pd-heartbeat-bench/main.go index 9d71be1129b..cfa0495c31c 100644 --- a/tools/pd-heartbeat-bench/main.go +++ b/tools/pd-heartbeat-bench/main.go @@ -48,7 +48,7 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/tools/pd-heartbeat-bench/config" "github.com/tikv/pd/tools/pd-heartbeat-bench/metrics" - "go.etcd.io/etcd/pkg/report" + "go.etcd.io/etcd/pkg/v3/report" "go.uber.org/zap" ) diff --git a/tools/pd-heartbeat-bench/metrics/util.go b/tools/pd-heartbeat-bench/metrics/util.go index bf5010e73da..e409cf10815 100644 --- a/tools/pd-heartbeat-bench/metrics/util.go +++ b/tools/pd-heartbeat-bench/metrics/util.go @@ -26,7 +26,7 @@ import ( "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" - "go.etcd.io/etcd/pkg/report" + "go.etcd.io/etcd/pkg/v3/report" "go.uber.org/zap" ) diff --git a/tools/pd-recover/main.go b/tools/pd-recover/main.go index 9b5d08013db..3423d06bb9f 100644 --- a/tools/pd-recover/main.go +++ b/tools/pd-recover/main.go @@ -28,8 +28,8 @@ import ( "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" ) var ( diff --git a/tools/pd-simulator/simulator/drive.go b/tools/pd-simulator/simulator/drive.go index 87b779559ca..e602ba2df65 100644 --- a/tools/pd-simulator/simulator/drive.go +++ b/tools/pd-simulator/simulator/drive.go @@ -38,7 +38,7 @@ import ( "github.com/tikv/pd/tools/pd-simulator/simulator/config" "github.com/tikv/pd/tools/pd-simulator/simulator/info" "github.com/tikv/pd/tools/pd-simulator/simulator/simutil" - "go.etcd.io/etcd/clientv3" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) diff --git a/tools/regions-dump/main.go b/tools/regions-dump/main.go index dddbcbba854..5ae4241cdc0 100644 --- a/tools/regions-dump/main.go +++ b/tools/regions-dump/main.go @@ -29,8 +29,8 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" ) var ( diff --git a/tools/stores-dump/main.go b/tools/stores-dump/main.go index b3d9b00be9f..0409244772f 100644 --- a/tools/stores-dump/main.go +++ b/tools/stores-dump/main.go @@ -28,8 +28,8 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/utils/etcdutil" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" ) var ( From 1b8fc6a950dbe658a0051c7f8228367b85bc9180 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 15 Aug 2024 14:17:40 +0800 Subject: [PATCH 16/42] *: add linters to organize import dependency (#8532) ref tikv/pd#4322 Signed-off-by: Ryan Leung --- .gitignore | 1 + .golangci.yml | 11 +++++++++++ client/go.mod | 2 +- client/http/client_test.go | 7 ++++--- client/resource_group/controller/controller.go | 3 +-- go.mod | 2 +- go.sum | 3 +-- pkg/gctuner/finalizer.go | 3 +-- pkg/gctuner/memory_limit_tuner.go | 16 ++++++---------- pkg/memory/var.go | 13 ++----------- tools/pd-simulator/simulator/cases/hot_write.go | 1 + tools/pd-simulator/simulator/simutil/key_test.go | 4 ++-- 12 files changed, 32 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index dc2efa5004f..a9d942fe706 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ coverage go.work* embedded_assets_handler.go *.log +*.bin diff --git a/.golangci.yml b/.golangci.yml index bc1ba393f39..e09011c45be 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -15,6 +15,9 @@ linters: - gofmt - revive - errcheck + - exportloopref + - goimports + - depguard linters-settings: gocritic: # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty @@ -205,6 +208,14 @@ linters-settings: - (net/http.ResponseWriter).Write - github.com/pingcap/log.Sync - (github.com/tikv/pd/pkg/ratelimit.Runner).RunTask + depguard: + rules: + denied-deps: + deny: + - pkg: go.uber.org/atomic + desc: "Use 'sync/atomic' instead of 'go.uber.org/atomic'" + - pkg: github.com/pkg/errors + desc: "Use 'github.com/pingcap/errors' instead of 'github.com/pkg/errors'" issues: exclude-rules: - path: (_test\.go|pkg/mock/.*\.go|tests/.*\.go) diff --git a/client/go.mod b/client/go.mod index 8dc706a4540..eb537f93ec8 100644 --- a/client/go.mod +++ b/client/go.mod @@ -14,7 +14,6 @@ require ( github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.8.2 - go.uber.org/atomic v1.10.0 go.uber.org/goleak v1.1.11 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 @@ -34,6 +33,7 @@ require ( github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/stretchr/objx v0.5.0 // indirect + go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/client/http/client_test.go b/client/http/client_test.go index 8769fa53f9a..5d63a34df54 100644 --- a/client/http/client_test.go +++ b/client/http/client_test.go @@ -18,13 +18,13 @@ import ( "context" "net/http" "strings" + "sync/atomic" "testing" "time" "github.com/stretchr/testify/require" "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/retry" - "go.uber.org/atomic" ) func TestPDAllowFollowerHandleHeader(t *testing.T) { @@ -53,7 +53,8 @@ func TestPDAllowFollowerHandleHeader(t *testing.T) { func TestWithCallerID(t *testing.T) { re := require.New(t) checked := 0 - expectedVal := atomic.NewString(defaultCallerID) + var expectedVal atomic.Value + expectedVal.Store(defaultCallerID) httpClient := NewHTTPClientWithRequestChecker(func(req *http.Request) error { val := req.Header.Get(xCallerIDKey) // Exclude the request sent by the inner client. @@ -68,7 +69,7 @@ func TestWithCallerID(t *testing.T) { defer c.Close() c.GetRegions(context.Background()) expectedVal.Store("test") - c.WithCallerID(expectedVal.Load()).GetRegions(context.Background()) + c.WithCallerID(expectedVal.Load().(string)).GetRegions(context.Background()) re.Equal(2, checked) } diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index 8e32b156a61..e5ad1f81fa5 100644 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -32,7 +32,6 @@ import ( "github.com/prometheus/client_golang/prometheus" pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/errs" - atomicutil "go.uber.org/atomic" "go.uber.org/zap" "golang.org/x/exp/slices" ) @@ -57,7 +56,7 @@ const ( lowToken selectType = 1 ) -var enableControllerTraceLog = atomicutil.NewBool(false) +var enableControllerTraceLog atomic.Bool func logControllerTrace(msg string, fields ...zap.Field) { if enableControllerTraceLog.Load() { diff --git a/go.mod b/go.mod index 9dc5fa9f249..79db4f3940c 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,6 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.15 go.etcd.io/etcd/client/v3 v3.5.15 go.etcd.io/etcd/server/v3 v3.5.15 - go.uber.org/atomic v1.10.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 @@ -187,6 +186,7 @@ require ( go.opentelemetry.io/otel/sdk v1.20.0 // indirect go.opentelemetry.io/otel/trace v1.20.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect + go.uber.org/atomic v1.9.0 // indirect go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 diff --git a/go.sum b/go.sum index e3bc1ed175a..e57fdcbce9c 100644 --- a/go.sum +++ b/go.sum @@ -566,9 +566,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.9.0 h1:pJTDXKEhRqBI8W7rU7kwT5EgyRZuSMVSFcZolOvKK9U= go.uber.org/dig v1.9.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/fx v1.12.0 h1:+1+3Cz9M0dFMPy9SW9XUIUHye8bnPUm7q7DroNGWYG4= diff --git a/pkg/gctuner/finalizer.go b/pkg/gctuner/finalizer.go index b9a73fa7ce5..a1424980b66 100644 --- a/pkg/gctuner/finalizer.go +++ b/pkg/gctuner/finalizer.go @@ -16,8 +16,7 @@ package gctuner import ( "runtime" - - "go.uber.org/atomic" + "sync/atomic" ) type finalizerCallback func() diff --git a/pkg/gctuner/memory_limit_tuner.go b/pkg/gctuner/memory_limit_tuner.go index 77a8f9db4c6..8a852b191d8 100644 --- a/pkg/gctuner/memory_limit_tuner.go +++ b/pkg/gctuner/memory_limit_tuner.go @@ -17,6 +17,7 @@ package gctuner import ( "math" "runtime/debug" + "sync/atomic" "time" "github.com/pingcap/failpoint" @@ -24,7 +25,6 @@ import ( util "github.com/tikv/pd/pkg/gogc" "github.com/tikv/pd/pkg/memory" "github.com/tikv/pd/pkg/utils/logutil" - atomicutil "go.uber.org/atomic" "go.uber.org/zap" ) @@ -35,10 +35,10 @@ var GlobalMemoryLimitTuner = &memoryLimitTuner{} // So we can change memory limit dynamically to avoid frequent GC when memory usage is greater than the limit. type memoryLimitTuner struct { finalizer *finalizer - isTuning atomicutil.Bool - percentage atomicutil.Float64 - waitingReset atomicutil.Bool - nextGCTriggeredByMemoryLimit atomicutil.Bool + isTuning atomic.Bool + percentage atomic.Value + waitingReset atomic.Bool + nextGCTriggeredByMemoryLimit atomic.Bool } // fallbackPercentage indicates the fallback memory limit percentage when turning. @@ -74,8 +74,6 @@ func (t *memoryLimitTuner) tuning() { if t.nextGCTriggeredByMemoryLimit.Load() && t.waitingReset.CompareAndSwap(false, true) { go func() { defer logutil.LogPanic() - memory.MemoryLimitGCLast.Store(time.Now()) - memory.MemoryLimitGCTotal.Add(1) setMemoryLimit(t.calcMemoryLimit(fallbackPercentage)) resetInterval := 1 * time.Minute // Wait 1 minute and set back, to avoid frequent GC failpoint.Inject("testMemoryLimitTuner", func(val failpoint.Value) { @@ -89,12 +87,10 @@ func (t *memoryLimitTuner) tuning() { continue } }() - memory.TriggerMemoryLimitGC.Store(true) } t.nextGCTriggeredByMemoryLimit.Store(true) } else { t.nextGCTriggeredByMemoryLimit.Store(false) - memory.TriggerMemoryLimitGC.Store(false) } } @@ -117,7 +113,7 @@ func (t *memoryLimitTuner) SetPercentage(percentage float64) { // GetPercentage get the percentage from memory limit tuner. func (t *memoryLimitTuner) GetPercentage() float64 { - return t.percentage.Load() + return t.percentage.Load().(float64) } // UpdateMemoryLimit updates the memory limit. diff --git a/pkg/memory/var.go b/pkg/memory/var.go index fd218183937..1ecc0aa3238 100644 --- a/pkg/memory/var.go +++ b/pkg/memory/var.go @@ -15,19 +15,10 @@ package memory import ( - "time" - - atomicutil "go.uber.org/atomic" + atomic "sync/atomic" ) // Process global variables for memory limit. var ( - ServerMemoryLimitOriginText = atomicutil.NewString("0") - ServerMemoryLimit = atomicutil.NewUint64(0) - ServerMemoryLimitSessMinSize = atomicutil.NewUint64(128 << 20) - - QueryForceDisk = atomicutil.NewInt64(0) - TriggerMemoryLimitGC = atomicutil.NewBool(false) - MemoryLimitGCLast = atomicutil.NewTime(time.Time{}) - MemoryLimitGCTotal = atomicutil.NewInt64(0) + ServerMemoryLimit atomic.Uint64 ) diff --git a/tools/pd-simulator/simulator/cases/hot_write.go b/tools/pd-simulator/simulator/cases/hot_write.go index 8f08264590d..54e944e540d 100644 --- a/tools/pd-simulator/simulator/cases/hot_write.go +++ b/tools/pd-simulator/simulator/cases/hot_write.go @@ -16,6 +16,7 @@ package cases import ( "fmt" + "github.com/docker/go-units" "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core" diff --git a/tools/pd-simulator/simulator/simutil/key_test.go b/tools/pd-simulator/simulator/simutil/key_test.go index ae00887afa8..9db7c597751 100644 --- a/tools/pd-simulator/simulator/simutil/key_test.go +++ b/tools/pd-simulator/simulator/simutil/key_test.go @@ -15,12 +15,12 @@ package simutil import ( - "github.com/pingcap/kvproto/pkg/metapb" - "github.com/tikv/pd/pkg/core" "testing" + "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/codec" + "github.com/tikv/pd/pkg/core" ) func TestGenerateTableKeys(t *testing.T) { From 4b0878ed38c46eca398ee4857520c8c06c92ec5f Mon Sep 17 00:00:00 2001 From: Hu# Date: Fri, 16 Aug 2024 12:01:11 +0800 Subject: [PATCH 17/42] tests/api: add wait leader for api (#8540) close tikv/pd#8513 tests/api: add wait leader for api Signed-off-by: husharp --- tests/server/api/api_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index 31769d91ffc..8d48221784a 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -129,6 +129,13 @@ func (suite *middlewareTestSuite) SetupSuite() { suite.cluster = cluster } +func (suite *middlewareTestSuite) SetupTest() { + re := suite.Require() + re.NotEmpty(suite.cluster.WaitLeader()) + leader := suite.cluster.GetLeaderServer() + re.NotNil(leader) +} + func (suite *middlewareTestSuite) TearDownSuite() { re := suite.Require() re.NoError(failpoint.Disable("github.com/tikv/pd/server/api/enableFailpointAPI")) From 10cbdcf0c551c3e337ebc1b885b1f2ccd874b163 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 16 Aug 2024 14:50:41 +0800 Subject: [PATCH 18/42] config: disable EnableV2 of etcd (#8536) close tikv/pd#8535 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/api/member_test.go | 25 +++++++++++-------------- server/config/config.go | 1 - 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/server/api/member_test.go b/server/api/member_test.go index d3318081e62..ad4812249c7 100644 --- a/server/api/member_test.go +++ b/server/api/member_test.go @@ -15,9 +15,8 @@ package api import ( - "bytes" + "context" "encoding/json" - "fmt" "io" "math/rand" "net/http" @@ -31,6 +30,7 @@ import ( "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" + clientv3 "go.etcd.io/etcd/client/v3" ) type memberTestSuite struct { @@ -124,11 +124,10 @@ func (suite *memberTestSuite) TestChangeLeaderPeerUrls() { var got pdpb.Member re.NoError(json.Unmarshal(buf, &got)) - id := got.GetMemberId() peerUrls := got.GetPeerUrls() newPeerUrls := []string{"http://127.0.0.1:1111"} - suite.changeLeaderPeerUrls(leader, id, newPeerUrls) + suite.changeLeaderPeerUrls(leader, newPeerUrls) addr = suite.cfgs[rand.Intn(len(suite.cfgs))].ClientUrls + apiPrefix + "/api/v1/members" resp, err = testDialClient.Get(addr) re.NoError(err) @@ -141,21 +140,19 @@ func (suite *memberTestSuite) TestChangeLeaderPeerUrls() { re.Equal(newPeerUrls, got1["etcd_leader"].GetPeerUrls()) // reset - suite.changeLeaderPeerUrls(leader, id, peerUrls) + suite.changeLeaderPeerUrls(leader, peerUrls) } -func (suite *memberTestSuite) changeLeaderPeerUrls(leader *pdpb.Member, id uint64, urls []string) { +func (suite *memberTestSuite) changeLeaderPeerUrls(leader *pdpb.Member, urls []string) { re := suite.Require() - data := map[string][]string{"peerURLs": urls} - postData, err := json.Marshal(data) - re.NoError(err) - req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("%s/v2/members/%s", leader.GetClientUrls()[0], fmt.Sprintf("%x", id)), bytes.NewBuffer(postData)) + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: leader.GetClientUrls(), + }) re.NoError(err) - req.Header.Set("Content-Type", "application/json") - resp, err := testDialClient.Do(req) + _, err = cli.MemberUpdate(context.Background(), leader.GetMemberId(), urls) re.NoError(err) - re.Equal(204, resp.StatusCode) - resp.Body.Close() + cli.Close() } func (suite *memberTestSuite) TestResignMyself() { diff --git a/server/config/config.go b/server/config/config.go index 8c4f6eaacc8..c1791be1fd9 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -746,7 +746,6 @@ func (c *Config) GenEmbedEtcdConfig() (*embed.Config, error) { cfg.ForceNewCluster = c.ForceNewCluster cfg.ZapLoggerBuilder = embed.NewZapCoreLoggerBuilder(c.Logger, c.Logger.Core(), c.LogProps.Syncer) cfg.EnableGRPCGateway = c.EnableGRPCGateway - cfg.EnableV2 = true cfg.Logger = "zap" var err error From a7663514c5ebfc78a99bacc84e030c9b7530a10b Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 19 Aug 2024 17:55:41 +0800 Subject: [PATCH 19/42] *: fix some typos (#8544) ref tikv/pd#4399 Signed-off-by: lhy1024 --- client/pd_service_discovery.go | 4 ++-- conf/config.toml | 2 +- pkg/autoscaling/calculation.go | 2 +- pkg/gc/safepoint_test.go | 16 ++++++++-------- pkg/member/election_leader.go | 2 +- pkg/movingaverage/weight_allocator.go | 2 +- pkg/replication/replication_mode_test.go | 2 +- pkg/schedule/checker/rule_checker.go | 4 ++-- pkg/schedule/operator/operator.go | 2 +- .../operator/operator_controller_test.go | 2 +- pkg/schedule/plan/plan.go | 18 +++++++++--------- pkg/schedule/scatter/region_scatterer.go | 2 +- pkg/schedule/schedulers/evict_slow_store.go | 2 +- pkg/schedule/schedulers/evict_slow_trend.go | 2 +- pkg/schedule/schedulers/metrics.go | 2 +- pkg/schedule/schedulers/utils.go | 2 +- scripts/dashboard-version | 2 +- scripts/update-dashboard.sh | 8 ++++---- server/api/middleware.go | 2 +- server/cluster/cluster_test.go | 4 ++-- tests/server/apiv2/handlers/keyspace_test.go | 4 ++-- tests/server/cluster/cluster_test.go | 2 +- tools/pd-api-bench/cases/controller.go | 12 ++++++------ tools/pd-ctl/pdctl/command/config_command.go | 18 +++++++++--------- tools/pd-simulator/README.md | 2 +- 25 files changed, 60 insertions(+), 60 deletions(-) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index e8f4c0d7707..c34a5bebac6 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -157,7 +157,7 @@ type pdServiceClient struct { } // NOTE: In the current implementation, the URL passed in is bound to have a scheme, -// because it is processed in `newPDServiceDiscovery`, and the url returned by etcd member owns the sheme. +// because it is processed in `newPDServiceDiscovery`, and the url returned by etcd member owns the scheme. // When testing, the URL is also bound to have a scheme. func newPDServiceClient(url, leaderURL string, conn *grpc.ClientConn, isLeader bool) ServiceClient { cli := &pdServiceClient{ @@ -1074,7 +1074,7 @@ func (c *pdServiceDiscovery) updateServiceClient(members []*pdpb.Member, leader leaderURL := pickMatchedURL(leader.GetClientUrls(), c.tlsCfg) leaderChanged, err := c.switchLeader(leaderURL) followerChanged := c.updateFollowers(members, leader.GetMemberId(), leaderURL) - // don't need to recreate balancer if no changess. + // don't need to recreate balancer if no changes. if !followerChanged && !leaderChanged { return err } diff --git a/conf/config.toml b/conf/config.toml index 0c4acf5fd8c..f2feacf30f7 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -206,7 +206,7 @@ [keyspace] ## pre-alloc is used to pre-allocate keyspaces during pd bootstrap. -## Its value should be a list of strings, denotting the name of the keyspaces. +## Its value should be a list of strings, denoting the name of the keyspaces. ## Example: ## pre-alloc = ["admin", "user1", "user2"] # pre-alloc = [] diff --git a/pkg/autoscaling/calculation.go b/pkg/autoscaling/calculation.go index b155c44bf68..43aa2972ed8 100644 --- a/pkg/autoscaling/calculation.go +++ b/pkg/autoscaling/calculation.go @@ -431,7 +431,7 @@ func findBestGroupToScaleOut(strategy *Strategy, groups []*Plan, component Compo }, } - // TODO: we can provide different senerios by using options and remove this kind of special judgement. + // TODO: we can provide different scenarios by using options and remove this kind of special judgement. if component == TiKV { group.Labels[filter.SpecialUseKey] = filter.SpecialUseHotRegion } diff --git a/pkg/gc/safepoint_test.go b/pkg/gc/safepoint_test.go index 39cd3660b2b..bc1e551594c 100644 --- a/pkg/gc/safepoint_test.go +++ b/pkg/gc/safepoint_test.go @@ -85,7 +85,7 @@ func TestGCSafePointUpdateCurrently(t *testing.T) { func TestServiceGCSafePointUpdate(t *testing.T) { re := require.New(t) manager := NewSafePointManager(newGCStorage(), config.PDServerConfig{}) - gcworkerServiceID := "gc_worker" + gcWorkerServiceID := "gc_worker" cdcServiceID := "cdc" brServiceID := "br" cdcServiceSafePoint := uint64(10) @@ -101,7 +101,7 @@ func TestServiceGCSafePointUpdate(t *testing.T) { re.NoError(err) re.True(updated) // the service will init the service safepoint to 0(<10 for cdc) for gc_worker. - re.Equal(gcworkerServiceID, min.ServiceID) + re.Equal(gcWorkerServiceID, min.ServiceID) }() // update the safepoint for br to 15 should success @@ -111,24 +111,24 @@ func TestServiceGCSafePointUpdate(t *testing.T) { re.NoError(err) re.True(updated) // the service will init the service safepoint to 0(<10 for cdc) for gc_worker. - re.Equal(gcworkerServiceID, min.ServiceID) + re.Equal(gcWorkerServiceID, min.ServiceID) }() - // update safepoint to 8 for gc_woker should be success + // update safepoint to 8 for gc_worker should be success go func() { defer wg.Done() // update with valid ttl for gc_worker should be success. - min, updated, _ := manager.UpdateServiceGCSafePoint(gcworkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) + min, updated, _ := manager.UpdateServiceGCSafePoint(gcWorkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) re.True(updated) // the current min safepoint should be 8 for gc_worker(cdc 10) re.Equal(gcWorkerSafePoint, min.SafePoint) - re.Equal(gcworkerServiceID, min.ServiceID) + re.Equal(gcWorkerServiceID, min.ServiceID) }() go func() { defer wg.Done() // update safepoint of gc_worker's service with ttl not infinity should be failed. - _, updated, err := manager.UpdateServiceGCSafePoint(gcworkerServiceID, 10000, 10, time.Now()) + _, updated, err := manager.UpdateServiceGCSafePoint(gcWorkerServiceID, 10000, 10, time.Now()) re.Error(err) re.False(updated) }() @@ -145,7 +145,7 @@ func TestServiceGCSafePointUpdate(t *testing.T) { wg.Wait() // update safepoint to 15(>10 for cdc) for gc_worker gcWorkerSafePoint = uint64(15) - min, updated, err := manager.UpdateServiceGCSafePoint(gcworkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) + min, updated, err := manager.UpdateServiceGCSafePoint(gcWorkerServiceID, gcWorkerSafePoint, math.MaxInt64, time.Now()) re.NoError(err) re.True(updated) re.Equal(cdcServiceID, min.ServiceID) diff --git a/pkg/member/election_leader.go b/pkg/member/election_leader.go index 24520bfbe64..81afc5dbd0a 100644 --- a/pkg/member/election_leader.go +++ b/pkg/member/election_leader.go @@ -21,7 +21,7 @@ import ( ) // ElectionLeader defines the common interface of the leader, which is the pdpb.Member -// for in PD/API service or the tsopb.Participant in the microserives. +// for in PD/API service or the tsopb.Participant in the micro services. type ElectionLeader interface { // GetListenUrls returns the listen urls GetListenUrls() []string diff --git a/pkg/movingaverage/weight_allocator.go b/pkg/movingaverage/weight_allocator.go index f63ce377e08..e0427d84645 100644 --- a/pkg/movingaverage/weight_allocator.go +++ b/pkg/movingaverage/weight_allocator.go @@ -20,7 +20,7 @@ package movingaverage // WeightAllocator will divide these items into some segments whose number named as segNum which should great than 0. // And the items at first segment will be assigned more weight that is `segNum` times that of item at last segment. // If you want assign same weights, just input segNum as 1. -// If length is 10 and segNum is 3, it will make the weight arrry as [3,3,3,3,2,2,2,1,1,1], +// If length is 10 and segNum is 3, it will make the weight array as [3,3,3,3,2,2,2,1,1,1], // and then uniform it : [3,3,3,3,2,2,2,1,1,1]/sum(arr)=arr/21, // And the final weight is [0.143,0.143,0.143,0.143,0.095,0.095,0.095,0.047,0.047,0.047]; // If length is 10 and segNum is 1, the weight is [0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1]; diff --git a/pkg/replication/replication_mode_test.go b/pkg/replication/replication_mode_test.go index bbaada98924..243d7f7d8f1 100644 --- a/pkg/replication/replication_mode_test.go +++ b/pkg/replication/replication_mode_test.go @@ -409,7 +409,7 @@ func TestReplicateState(t *testing.T) { rep.tickReplicateStatus() assertLastData(t, replicator.lastData[1], "sync", stateID, nil) - // repliate state to new member + // replicate state to new member replicator.memberIDs = append(replicator.memberIDs, 2, 3) rep.tickReplicateStatus() assertLastData(t, replicator.lastData[2], "sync", stateID, nil) diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index e29cd2bc05b..23018ad7096 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -265,7 +265,7 @@ func (c *RuleChecker) replaceUnexpectedRulePeer(region *core.RegionInfo, rf *pla minCount := uint64(math.MaxUint64) for _, p := range region.GetPeers() { count := c.record.getOfflineLeaderCount(p.GetStoreId()) - checkPeerhealth := func() bool { + checkPeerHealth := func() bool { if p.GetId() == peer.GetId() { return true } @@ -274,7 +274,7 @@ func (c *RuleChecker) replaceUnexpectedRulePeer(region *core.RegionInfo, rf *pla } return c.allowLeader(fit, p) } - if minCount > count && checkPeerhealth() { + if minCount > count && checkPeerHealth() { minCount = count newLeader = p } diff --git a/pkg/schedule/operator/operator.go b/pkg/schedule/operator/operator.go index 4d57d4fc6c7..f89f6606412 100644 --- a/pkg/schedule/operator/operator.go +++ b/pkg/schedule/operator/operator.go @@ -45,7 +45,7 @@ var ( EpochNotMatch CancelReasonType = "epoch not match" // AlreadyExist is the cancel reason when the operator is running. AlreadyExist CancelReasonType = "already exist" - // AdminStop is the cancel reason when the operator is stopped by adminer. + // AdminStop is the cancel reason when the operator is stopped by admin. AdminStop CancelReasonType = "admin stop" // NotInRunningState is the cancel reason when the operator is not in running state. NotInRunningState CancelReasonType = "not in running state" diff --git a/pkg/schedule/operator/operator_controller_test.go b/pkg/schedule/operator/operator_controller_test.go index 3894df7e5e7..16ba899db1d 100644 --- a/pkg/schedule/operator/operator_controller_test.go +++ b/pkg/schedule/operator/operator_controller_test.go @@ -523,7 +523,7 @@ func (suite *operatorControllerTestSuite) TestCheckOperatorLightly() { re.Nil(r) re.Equal(reason, RegionNotFound) - // check failed because of verions of region epoch changed + // check failed because of versions of region epoch changed cluster.PutRegion(target) source.GetMeta().RegionEpoch = &metapb.RegionEpoch{ConfVer: 0, Version: 1} r, reason = controller.checkOperatorLightly(ops[0]) diff --git a/pkg/schedule/plan/plan.go b/pkg/schedule/plan/plan.go index 8a389b9b9e8..5eb8a345914 100644 --- a/pkg/schedule/plan/plan.go +++ b/pkg/schedule/plan/plan.go @@ -30,18 +30,18 @@ type Plan interface { SetStatus(*Status) } -// Summary is used to analyse plan simply. +// Summary is used to analyze plan simply. // It will return the status of store. type Summary func([]Plan) (map[uint64]Status, bool, error) -// Collector is a plan collector +// Collector is a plan collector. type Collector struct { basePlan Plan unschedulablePlans []Plan schedulablePlans []Plan } -// NewCollector returns a new Collector +// NewCollector returns a new Collector. func NewCollector(plan Plan) *Collector { return &Collector{ basePlan: plan, @@ -50,7 +50,7 @@ func NewCollector(plan Plan) *Collector { } } -// Collect is used to collect a new Plan and save it into PlanCollector +// Collect is used to collect a new Plan and save it into PlanCollector. func (c *Collector) Collect(opts ...Option) { if c == nil { return @@ -63,7 +63,7 @@ func (c *Collector) Collect(opts ...Option) { } } -// GetPlans returns all plans and the first part plans are schedulable +// GetPlans returns all plans and the first part plans are schedulable. func (c *Collector) GetPlans() []Plan { if c == nil { return nil @@ -71,24 +71,24 @@ func (c *Collector) GetPlans() []Plan { return append(c.schedulablePlans, c.unschedulablePlans...) } -// Option is to do some action for plan +// Option is to do some action for plan. type Option func(plan Plan) -// SetStatus is used to set status for plan +// SetStatus is used to set status for plan. func SetStatus(status *Status) Option { return func(plan Plan) { plan.SetStatus(status) } } -// SetResource is used to generate Resource for plan +// SetResource is used to generate Resource for plan. func SetResource(resource any) Option { return func(plan Plan) { plan.SetResource(resource) } } -// SetResourceWithStep is used to generate Resource for plan +// SetResourceWithStep is used to generate Resource for plan. func SetResourceWithStep(resource any, step int) Option { return func(plan Plan) { plan.SetResourceWithStep(resource, step) diff --git a/pkg/schedule/scatter/region_scatterer.go b/pkg/schedule/scatter/region_scatterer.go index efef5439fed..71b2cd346c7 100644 --- a/pkg/schedule/scatter/region_scatterer.go +++ b/pkg/schedule/scatter/region_scatterer.go @@ -437,7 +437,7 @@ func isSameDistribution(region *core.RegionInfo, targetPeers map[uint64]*metapb. // selectNewPeer return the new peer which pick the fewest picked count. // it keeps the origin peer if the origin store's pick count is equal the fewest pick. -// it can be diveded into three steps: +// it can be divided into three steps: // 1. found the max pick count and the min pick count. // 2. if max pick count equals min pick count, it means all store picked count are some, return the origin peer. // 3. otherwise, select the store which pick count is the min pick count and pass all filter. diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index de581f597bb..8f9c8841e04 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -94,7 +94,7 @@ func (conf *evictSlowStoreSchedulerConfig) evictStore() uint64 { return conf.getStores()[0] } -// readyForRecovery checks whether the last cpatured candidate is ready for recovery. +// readyForRecovery checks whether the last captured candidate is ready for recovery. func (conf *evictSlowStoreSchedulerConfig) readyForRecovery() bool { conf.RLock() defer conf.RUnlock() diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index 427787016a2..b63b4b4a2e0 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -147,7 +147,7 @@ func (conf *evictSlowTrendSchedulerConfig) lastCandidateCapturedSecs() uint64 { return DurationSinceAsSecs(conf.lastEvictCandidate.captureTS) } -// readyForRecovery checks whether the last cpatured candidate is ready for recovery. +// readyForRecovery checks whether the last captured candidate is ready for recovery. func (conf *evictSlowTrendSchedulerConfig) readyForRecovery() bool { conf.RLock() defer conf.RUnlock() diff --git a/pkg/schedule/schedulers/metrics.go b/pkg/schedule/schedulers/metrics.go index 42170e43818..90b0b0c6bb2 100644 --- a/pkg/schedule/schedulers/metrics.go +++ b/pkg/schedule/schedulers/metrics.go @@ -110,7 +110,7 @@ var ( Namespace: "pd", Subsystem: "scheduler", Name: "store_slow_trend_evicted_status", - Help: "Store evited by slow trend status for schedule", + Help: "Store evicted by slow trend status for schedule", }, []string{"address", "store"}) storeSlowTrendActionStatusGauge = prometheus.NewGaugeVec( diff --git a/pkg/schedule/schedulers/utils.go b/pkg/schedule/schedulers/utils.go index 1e911cf7b06..7cbfe714aa9 100644 --- a/pkg/schedule/schedulers/utils.go +++ b/pkg/schedule/schedulers/utils.go @@ -286,7 +286,7 @@ func sliceLoadCmp(cmps ...storeLoadCmp) storeLoadCmp { } // stLdRankCmp returns a cmp that compares the two loads with discretized data. -// For example, if the rank function discretice data by step 10 , the load 11 and 19 will be considered equal. +// For example, if the rank function discretize data by step 10 , the load 11 and 19 will be considered equal. func stLdRankCmp(dim func(ld *statistics.StoreLoad) float64, rank func(value float64) int64) storeLoadCmp { return func(ld1, ld2 *statistics.StoreLoad) int { return rankCmp(dim(ld1), dim(ld2), rank) diff --git a/scripts/dashboard-version b/scripts/dashboard-version index 42ff906db4e..d8d960683ae 100644 --- a/scripts/dashboard-version +++ b/scripts/dashboard-version @@ -1,3 +1,3 @@ # This file is updated by running scripts/update-dashboard.sh -# Don't edit it manullay +# Don't edit it manually 8.3.0-e6e78c7c diff --git a/scripts/update-dashboard.sh b/scripts/update-dashboard.sh index 7411960f766..6eff7d5f827 100755 --- a/scripts/update-dashboard.sh +++ b/scripts/update-dashboard.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" BASE_DIR="$(dirname "$CUR_DIR")" DASHBOARD_VERSION_FILE="$BASE_DIR/scripts/dashboard-version" # old version @@ -23,9 +23,9 @@ if [ "$#" -ge 1 ]; then # so that we don't need to modify the embed-dashboard-ui.sh logic TO_FILE_VERSION=${DASHBOARD_VERSION#v} - echo "# This file is updated by running scripts/update-dashboard.sh" > $DASHBOARD_VERSION_FILE - echo "# Don't edit it manullay" >> $DASHBOARD_VERSION_FILE - echo $TO_FILE_VERSION >> $DASHBOARD_VERSION_FILE + echo "# This file is updated by running scripts/update-dashboard.sh" >$DASHBOARD_VERSION_FILE + echo "# Don't edit it manually" >>$DASHBOARD_VERSION_FILE + echo $TO_FILE_VERSION >>$DASHBOARD_VERSION_FILE fi echo "+ Update dashboard version to $DASHBOARD_VERSION" diff --git a/server/api/middleware.go b/server/api/middleware.go index 010889f08ce..fd0d81412ea 100644 --- a/server/api/middleware.go +++ b/server/api/middleware.go @@ -46,7 +46,7 @@ func (s *serviceMiddlewareBuilder) createHandler(next func(http.ResponseWriter, return negroni.New(append(s.handlers, negroni.WrapFunc(next))...) } -// requestInfoMiddleware is used to gather info from requsetInfo +// requestInfoMiddleware is used to gather info from requestInfo type requestInfoMiddleware struct { svr *server.Server } diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 3f01305b3f1..cffefbba444 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -952,8 +952,8 @@ func TestRegionHeartbeat(t *testing.T) { re.NoError(cluster.processRegionHeartbeat(ctx, overlapRegion)) tracer.OnAllStageFinished() re.Condition(func() bool { - fileds := tracer.LogFields() - return slice.AllOf(fileds, func(i int) bool { return fileds[i].Integer > 0 }) + fields := tracer.LogFields() + return slice.AllOf(fields, func(i int) bool { return fields[i].Integer > 0 }) }, "should have stats") region = &metapb.Region{} ok, err = storage.LoadRegion(regions[n-1].GetID(), region) diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index 18466ca0da7..f3aa55bbe43 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -109,7 +109,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { success, disabledAgain := sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "disabled"}) re.True(success) re.Equal(disabled, disabledAgain) - // Tombstoning a DISABLED keyspace should not be allowed. + // Tombstone a DISABLED keyspace should not be allowed. success, _ = sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "tombstone"}) re.False(success) // Archiving a DISABLED keyspace should be allowed. @@ -119,7 +119,7 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { // Enabling an ARCHIVED keyspace is not allowed. success, _ = sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "enabled"}) re.False(success) - // Tombstoning an ARCHIVED keyspace is allowed. + // Tombstone an ARCHIVED keyspace is allowed. success, tombstone := sendUpdateStateRequest(re, suite.server, created.Name, &handlers.UpdateStateParam{State: "tombstone"}) re.True(success) re.Equal(keyspacepb.KeyspaceState_TOMBSTONE, tombstone.State) diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index c95aa50cb3d..27d3300b293 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -827,7 +827,7 @@ func TestSetScheduleOpt(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // TODO: enable placementrules + // TODO: enable placementRules tc, err := tests.NewTestCluster(ctx, 1, func(cfg *config.Config, _ string) { cfg.Replication.EnablePlacementRules = false }) defer tc.Destroy() re.NoError(err) diff --git a/tools/pd-api-bench/cases/controller.go b/tools/pd-api-bench/cases/controller.go index a4e20f25758..e19c79c0f3e 100644 --- a/tools/pd-api-bench/cases/controller.go +++ b/tools/pd-api-bench/cases/controller.go @@ -231,10 +231,10 @@ func (c *httpController) run() { case <-ticker.C: err := c.Do(c.ctx, hCli) if err != nil { - log.Error("meet erorr when doing HTTP request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing HTTP request", zap.String("case", c.Name()), zap.Error(err)) } case <-c.ctx.Done(): - log.Info("Got signal to exit running HTTP case") + log.Info("got signal to exit running HTTP case") return } } @@ -300,10 +300,10 @@ func (c *gRPCController) run() { case <-ticker.C: err := c.Unary(c.ctx, cli) if err != nil { - log.Error("meet erorr when doing gRPC request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing gRPC request", zap.String("case", c.Name()), zap.Error(err)) } case <-c.ctx.Done(): - log.Info("Got signal to exit running gRPC case") + log.Info("got signal to exit running gRPC case") return } } @@ -374,10 +374,10 @@ func (c *etcdController) run() { case <-ticker.C: err := c.Unary(c.ctx, cli) if err != nil { - log.Error("meet erorr when doing etcd request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing etcd request", zap.String("case", c.Name()), zap.Error(err)) } case <-c.ctx.Done(): - log.Info("Got signal to exit running etcd case") + log.Info("got signal to exit running etcd case") return } } diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index 6f88e06a0dd..4efaae3fa13 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -77,7 +77,7 @@ func NewShowConfigCommand() *cobra.Command { sc.AddCommand(NewShowClusterVersionCommand()) sc.AddCommand(newShowReplicationModeCommand()) sc.AddCommand(NewShowServerConfigCommand()) - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") return sc } @@ -88,7 +88,7 @@ func NewShowAllConfigCommand() *cobra.Command { Short: "show all config of PD", Run: showAllConfigCommandFunc, } - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") return sc } @@ -99,7 +99,7 @@ func NewShowScheduleConfigCommand() *cobra.Command { Short: "show schedule config of PD", Run: showScheduleConfigCommandFunc, } - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") return sc } @@ -110,7 +110,7 @@ func NewShowReplicationConfigCommand() *cobra.Command { Short: "show replication config of PD", Run: showReplicationConfigCommandFunc, } - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") return sc } @@ -447,7 +447,7 @@ func NewPlacementRulesCommand() *cobra.Command { show.Flags().String("id", "", "rule id") show.Flags().String("region", "", "region id") show.Flags().Bool("detail", false, "detailed match info for region") - show.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + show.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") load := &cobra.Command{ Use: "load", Short: "load placement rules to a file", @@ -457,7 +457,7 @@ func NewPlacementRulesCommand() *cobra.Command { load.Flags().String("id", "", "rule id") load.Flags().String("region", "", "region id") load.Flags().String("out", "rules.json", "the filename contains rules") - load.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + load.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") save := &cobra.Command{ Use: "save", Short: "save rules from file", @@ -473,7 +473,7 @@ func NewPlacementRulesCommand() *cobra.Command { Short: "show rule group configuration(s)", Run: showRuleGroupFunc, } - ruleGroupShow.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + ruleGroupShow.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") ruleGroupSet := &cobra.Command{ Use: "set ", Short: "update rule group configuration", @@ -496,7 +496,7 @@ func NewPlacementRulesCommand() *cobra.Command { Run: getRuleBundle, } ruleBundleGet.Flags().String("out", "", "the output file") - ruleBundleGet.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + ruleBundleGet.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") ruleBundleSet := &cobra.Command{ Use: "set", Short: "set rule group config and its rules from file", @@ -515,7 +515,7 @@ func NewPlacementRulesCommand() *cobra.Command { Run: loadRuleBundle, } ruleBundleLoad.Flags().String("out", "rules.json", "the output file") - ruleBundleLoad.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micor service") + ruleBundleLoad.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") ruleBundleSave := &cobra.Command{ Use: "save", Short: "save all group configs and rules from file", diff --git a/tools/pd-simulator/README.md b/tools/pd-simulator/README.md index 107f6c40f64..d0519bfedff 100644 --- a/tools/pd-simulator/README.md +++ b/tools/pd-simulator/README.md @@ -44,7 +44,7 @@ Run a specific case with an external PD: ./pd-simulator -pd="http://127.0.0.1:2379" -case="casename" ``` -Run with tiup playgroudn : +Run with tiup playground: ```shell tiup playground nightly --host 127.0.0.1 --kv.binpath ./pd-simulator --kv=1 --db=0 --kv.config=./tikv.conf ``` From 6bf980de258c8e13c032fa023504ba30929d7f48 Mon Sep 17 00:00:00 2001 From: Hu# Date: Tue, 20 Aug 2024 15:51:12 +0800 Subject: [PATCH 20/42] tests/tso: Create independent clients to prevent interfering with other tests. (#8546) ref tikv/pd#8103 tests/tso: Create independent clients to prevent interfering with other tests. Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/tso/client_test.go | 53 ++++++++++++++------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index f5f33240d18..a669e093200 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -103,17 +103,7 @@ func (suite *tsoClientTestSuite) SetupSuite() { suite.backendEndpoints = suite.pdLeaderServer.GetAddr() suite.keyspaceIDs = make([]uint32, 0) - if suite.legacy { - client, err := pd.NewClientWithContext(suite.ctx, suite.getBackendEndpoints(), pd.SecurityOption{}, pd.WithForwardingOption(true)) - re.NoError(err) - innerClient, ok := client.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) - re.True(ok) - re.Equal(constant.NullKeyspaceID, innerClient.GetServiceDiscovery().GetKeyspaceID()) - re.Equal(constant.DefaultKeyspaceGroupID, innerClient.GetServiceDiscovery().GetKeyspaceGroupID()) - mcs.WaitForTSOServiceAvailable(suite.ctx, re, client) - suite.clients = make([]pd.Client, 0) - suite.clients = append(suite.clients, client) - } else { + if !suite.legacy { suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 3, suite.backendEndpoints) re.NoError(err) @@ -148,7 +138,23 @@ func (suite *tsoClientTestSuite) SetupSuite() { }, }) } + } +} +// Create independent clients to prevent interfering with other tests. +func (suite *tsoClientTestSuite) SetupTest() { + re := suite.Require() + if suite.legacy { + client, err := pd.NewClientWithContext(suite.ctx, suite.getBackendEndpoints(), pd.SecurityOption{}, pd.WithForwardingOption(true)) + re.NoError(err) + innerClient, ok := client.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) + re.True(ok) + re.Equal(constant.NullKeyspaceID, innerClient.GetServiceDiscovery().GetKeyspaceID()) + re.Equal(constant.DefaultKeyspaceGroupID, innerClient.GetServiceDiscovery().GetKeyspaceGroupID()) + mcs.WaitForTSOServiceAvailable(suite.ctx, re, client) + suite.clients = make([]pd.Client, 0) + suite.clients = append(suite.clients, client) + } else { suite.waitForAllKeyspaceGroupsInServing(re) } } @@ -183,15 +189,18 @@ func (suite *tsoClientTestSuite) waitForAllKeyspaceGroupsInServing(re *require.A re.Equal(len(suite.keyspaceIDs), len(suite.clients)) } +func (suite *tsoClientTestSuite) TearDownTest() { + for _, client := range suite.clients { + client.Close() + } +} + func (suite *tsoClientTestSuite) TearDownSuite() { suite.cancel() if !suite.legacy { suite.tsoCluster.Destroy() } suite.cluster.Destroy() - for _, client := range suite.clients { - client.Close() - } } func (suite *tsoClientTestSuite) TestGetTS() { @@ -432,21 +441,13 @@ func (suite *tsoClientTestSuite) TestGetTSWhileResettingTSOClient() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/client/delayDispatchTSORequest", "return(true)")) var ( - clients []pd.Client stopSignal atomic.Bool wg sync.WaitGroup ) - // Create independent clients to prevent interfering with other tests. - if suite.legacy { - client, err := pd.NewClientWithContext(suite.ctx, suite.getBackendEndpoints(), pd.SecurityOption{}, pd.WithForwardingOption(true)) - re.NoError(err) - clients = []pd.Client{client} - } else { - clients = mcs.WaitForMultiKeyspacesTSOAvailable(suite.ctx, re, suite.keyspaceIDs, suite.getBackendEndpoints()) - } - wg.Add(tsoRequestConcurrencyNumber * len(clients)) + + wg.Add(tsoRequestConcurrencyNumber * len(suite.clients)) for i := 0; i < tsoRequestConcurrencyNumber; i++ { - for _, client := range clients { + for _, client := range suite.clients { go func(client pd.Client) { defer wg.Done() var lastTS uint64 @@ -465,7 +466,7 @@ func (suite *tsoClientTestSuite) TestGetTSWhileResettingTSOClient() { } // Reset the TSO clients while requesting TSO concurrently. for i := 0; i < tsoRequestConcurrencyNumber; i++ { - for _, client := range clients { + for _, client := range suite.clients { client.(interface{ ResetTSOClient() }).ResetTSOClient() } } From 2e8f20fef9ce497f9a2d9ed2be3174ecf16fd17e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=8D=E7=99=BD?= <251098199@qq.com> Date: Wed, 21 Aug 2024 11:21:11 +0800 Subject: [PATCH 21/42] server: refactor params of `askSplit` call (#8537) close tikv/pd#5014 Signed-off-by: qingfeng777 <251098199@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/grpc_service.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/server/grpc_service.go b/server/grpc_service.go index f7a0ea9443e..d3db45c8364 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -1756,10 +1756,7 @@ func (s *GrpcServer) AskSplit(ctx context.Context, request *pdpb.AskSplitRequest "missing region for split"), }, nil } - req := &pdpb.AskSplitRequest{ - Region: request.Region, - } - split, err := rc.HandleAskSplit(req) + split, err := rc.HandleAskSplit(request) if err != nil { return &pdpb.AskSplitResponse{ Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), @@ -1836,11 +1833,7 @@ func (s *GrpcServer) AskBatchSplit(ctx context.Context, request *pdpb.AskBatchSp "missing region for split"), }, nil } - req := &pdpb.AskBatchSplitRequest{ - Region: request.Region, - SplitCount: request.SplitCount, - } - split, err := rc.HandleAskBatchSplit(req) + split, err := rc.HandleAskBatchSplit(request) if err != nil { return &pdpb.AskBatchSplitResponse{ Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), From 9c14c63a06a6230eaf1a59fe0aa97342015da1f0 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 21 Aug 2024 12:07:42 +0800 Subject: [PATCH 22/42] mcs: fix potential data race in scheduling server (#8539) close tikv/pd#8538 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/server.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index e1753cf2972..9ea369aae9e 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -55,6 +55,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/memberutil" @@ -193,7 +194,7 @@ func (s *Server) updateAPIServerMemberLoop() { if !s.IsServing() { continue } - members, err := s.GetClient().MemberList(ctx) + members, err := etcdutil.ListEtcdMembers(ctx, s.GetClient()) if err != nil { log.Warn("failed to list members", errs.ZapError(err)) continue @@ -212,6 +213,11 @@ func (s *Server) updateAPIServerMemberLoop() { cc, err := s.GetDelegateClient(ctx, s.GetTLSConfig(), ep.ClientURLs[0]) if err != nil { log.Info("failed to get delegate client", errs.ZapError(err)) + continue + } + if !s.IsServing() { + // double check + break } if s.cluster.SwitchAPIServerLeader(pdpb.NewPDClient(cc)) { if status.Leader != curLeader { From ce1d0e800f34130e56677861d1570a47c7aa2a87 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Wed, 21 Aug 2024 13:24:11 +0800 Subject: [PATCH 23/42] scheduler: remove old scheduler name (#8516) ref tikv/pd#8379 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/cluster.go | 2 +- pkg/mcs/scheduling/server/config/config.go | 2 +- pkg/mock/mockconfig/mockconfig.go | 2 +- pkg/schedule/checker/merge_checker.go | 2 +- pkg/schedule/checker/replica_checker.go | 2 +- pkg/schedule/checker/rule_checker.go | 2 +- pkg/schedule/config/config.go | 2 +- pkg/schedule/config/config_provider.go | 2 +- pkg/schedule/coordinator.go | 4 +- pkg/schedule/filter/counter_test.go | 2 +- pkg/schedule/handler/handler.go | 4 +- pkg/schedule/operator/metrics.go | 2 +- pkg/schedule/schedulers/balance_leader.go | 4 +- pkg/schedule/schedulers/balance_region.go | 7 +-- pkg/schedule/schedulers/balance_test.go | 2 +- pkg/schedule/schedulers/balance_witness.go | 4 +- .../schedulers/balance_witness_test.go | 2 +- pkg/schedule/schedulers/base_scheduler.go | 2 +- .../schedulers/diagnostic_recorder.go | 36 ++++++++------ pkg/schedule/schedulers/evict_leader.go | 6 +-- pkg/schedule/schedulers/evict_leader_test.go | 2 +- pkg/schedule/schedulers/evict_slow_store.go | 5 +- .../schedulers/evict_slow_store_test.go | 2 +- pkg/schedule/schedulers/evict_slow_trend.go | 7 +-- .../schedulers/evict_slow_trend_test.go | 2 +- pkg/schedule/schedulers/grant_hot_region.go | 7 +-- pkg/schedule/schedulers/grant_leader.go | 9 +--- pkg/schedule/schedulers/hot_region.go | 6 +-- pkg/schedule/schedulers/hot_region_test.go | 8 ++-- pkg/schedule/schedulers/init.go | 2 +- pkg/schedule/schedulers/label.go | 9 +--- pkg/schedule/schedulers/metrics.go | 2 +- pkg/schedule/schedulers/random_merge.go | 7 +-- pkg/schedule/schedulers/scatter_range.go | 7 +-- pkg/schedule/schedulers/scheduler.go | 2 +- .../schedulers/scheduler_controller.go | 5 +- pkg/schedule/schedulers/scheduler_test.go | 2 +- pkg/schedule/schedulers/shuffle_hot_region.go | 7 +-- pkg/schedule/schedulers/shuffle_leader.go | 7 +-- pkg/schedule/schedulers/shuffle_region.go | 13 ++--- pkg/schedule/schedulers/split_bucket.go | 4 +- .../schedulers/transfer_witness_leader.go | 4 +- .../transfer_witness_leader_test.go | 2 +- pkg/schedule/{type => types}/type.go | 0 plugin/scheduler_example/evict_leader.go | 2 +- server/api/diagnostic_test.go | 13 ++--- server/api/scheduler.go | 11 ++--- server/cluster/cluster_test.go | 48 +++++++++---------- server/cluster/scheduling_controller.go | 3 +- server/config/persist_options.go | 2 +- server/handler.go | 2 +- .../mcs/scheduling/config_test.go | 15 +++--- .../mcs/scheduling/server_test.go | 35 +++++++------- .../realcluster/scheduler_test.go | 26 +++++----- tests/server/api/scheduler_test.go | 2 +- tests/server/cluster/cluster_test.go | 11 +++-- 56 files changed, 164 insertions(+), 218 deletions(-) rename pkg/schedule/{type => types}/type.go (100%) diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index a5a3a709184..31120ae6082 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -27,7 +27,7 @@ import ( "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/schedule/splitter" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index 4b855d09899..4d7cee91f4a 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -36,7 +36,7 @@ import ( "github.com/tikv/pd/pkg/core/storelimit" mcsconstant "github.com/tikv/pd/pkg/mcs/utils/constant" sc "github.com/tikv/pd/pkg/schedule/config" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/configutil" diff --git a/pkg/mock/mockconfig/mockconfig.go b/pkg/mock/mockconfig/mockconfig.go index ccd1f98154d..6e595f982c1 100644 --- a/pkg/mock/mockconfig/mockconfig.go +++ b/pkg/mock/mockconfig/mockconfig.go @@ -16,7 +16,7 @@ package mockconfig import ( sc "github.com/tikv/pd/pkg/schedule/config" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/server/config" ) diff --git a/pkg/schedule/checker/merge_checker.go b/pkg/schedule/checker/merge_checker.go index 65189d35c1d..d5a39da83ae 100644 --- a/pkg/schedule/checker/merge_checker.go +++ b/pkg/schedule/checker/merge_checker.go @@ -31,7 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/logutil" ) diff --git a/pkg/schedule/checker/replica_checker.go b/pkg/schedule/checker/replica_checker.go index b0c42e88258..a21e19b3d66 100644 --- a/pkg/schedule/checker/replica_checker.go +++ b/pkg/schedule/checker/replica_checker.go @@ -26,7 +26,7 @@ import ( "github.com/tikv/pd/pkg/schedule/config" sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "go.uber.org/zap" ) diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index 23018ad7096..82807441bf8 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -31,7 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/versioninfo" "go.uber.org/zap" ) diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index 124bff0a704..344569d6460 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -20,7 +20,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core/storelimit" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/configutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" diff --git a/pkg/schedule/config/config_provider.go b/pkg/schedule/config/config_provider.go index d7bc38a7c03..95bcad5add0 100644 --- a/pkg/schedule/config/config_provider.go +++ b/pkg/schedule/config/config_provider.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/core/storelimit" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage/endpoint" ) diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go index 89c99ac90b8..2a31045129e 100644 --- a/pkg/schedule/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -34,7 +34,7 @@ import ( "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/schedule/splitter" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/utils/logutil" @@ -182,7 +182,7 @@ func (c *Coordinator) driveSlowNodeScheduler() { case <-ticker.C: { // If enabled, exit. - if exists, _ := c.schedulers.IsSchedulerExisted(schedulers.EvictSlowTrendName); exists { + if exists, _ := c.schedulers.IsSchedulerExisted(types.EvictSlowTrendScheduler.String()); exists { return } // If the cluster was set up with `raft-kv2` engine, this cluster should diff --git a/pkg/schedule/filter/counter_test.go b/pkg/schedule/filter/counter_test.go index 5b8d5144412..7c7acc5e9a5 100644 --- a/pkg/schedule/filter/counter_test.go +++ b/pkg/schedule/filter/counter_test.go @@ -18,7 +18,7 @@ import ( "testing" "github.com/stretchr/testify/require" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" ) func TestString(t *testing.T) { diff --git a/pkg/schedule/handler/handler.go b/pkg/schedule/handler/handler.go index 748a17b87ef..a8540b4b5f4 100644 --- a/pkg/schedule/handler/handler.go +++ b/pkg/schedule/handler/handler.go @@ -38,6 +38,7 @@ import ( "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/statistics/utils" @@ -884,7 +885,8 @@ func (h *Handler) GetSchedulerByStatus(status string, needTS bool) (any, error) // GetDiagnosticResult returns the diagnostic results of the specified scheduler. func (h *Handler) GetDiagnosticResult(name string) (*schedulers.DiagnosticResult, error) { - if _, ok := schedulers.DiagnosableSummaryFunc[name]; !ok { + tp := types.StringToSchedulerType[name] + if _, ok := schedulers.DiagnosableSummaryFunc[tp]; !ok { return nil, errs.ErrSchedulerUndiagnosable.FastGenByArgs(name) } co := h.GetCoordinator() diff --git a/pkg/schedule/operator/metrics.go b/pkg/schedule/operator/metrics.go index 74f9ddad0c7..47a165500e9 100644 --- a/pkg/schedule/operator/metrics.go +++ b/pkg/schedule/operator/metrics.go @@ -16,7 +16,7 @@ package operator import ( "github.com/prometheus/client_golang/prometheus" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" ) var ( diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index ef254ee6950..60dbee79dc4 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -31,7 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/reflectutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/utils/typeutil" @@ -40,8 +40,6 @@ import ( ) const ( - // BalanceLeaderName is balance leader scheduler name. - BalanceLeaderName = "balance-leader-scheduler" // BalanceLeaderBatchSize is the default number of operators to transfer leaders by one scheduling. // Default value is 4 which is subjected by scheduler-max-waiting-operator and leader-schedule-limit // If you want to increase balance speed more, please increase above-mentioned param. diff --git a/pkg/schedule/schedulers/balance_region.go b/pkg/schedule/schedulers/balance_region.go index 174b6af1c83..1ebe65d732c 100644 --- a/pkg/schedule/schedulers/balance_region.go +++ b/pkg/schedule/schedulers/balance_region.go @@ -26,15 +26,10 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "go.uber.org/zap" ) -const ( - // BalanceRegionName is balance region scheduler name. - BalanceRegionName = "balance-region-scheduler" -) - type balanceRegionSchedulerConfig struct { Ranges []core.KeyRange `json:"ranges"` // TODO: When we prepare to use Ranges, we will need to implement the ReloadConfig function for this scheduler. diff --git a/pkg/schedule/schedulers/balance_test.go b/pkg/schedule/schedulers/balance_test.go index 2f163742bbb..e9ae771d18b 100644 --- a/pkg/schedule/schedulers/balance_test.go +++ b/pkg/schedule/schedulers/balance_test.go @@ -31,7 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" "github.com/tikv/pd/pkg/versioninfo" diff --git a/pkg/schedule/schedulers/balance_witness.go b/pkg/schedule/schedulers/balance_witness.go index 450c43647cf..47f23d470cc 100644 --- a/pkg/schedule/schedulers/balance_witness.go +++ b/pkg/schedule/schedulers/balance_witness.go @@ -32,7 +32,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/reflectutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -40,8 +40,6 @@ import ( ) const ( - // BalanceWitnessName is balance witness scheduler name. - BalanceWitnessName = "balance-witness-scheduler" // balanceWitnessBatchSize is the default number of operators to transfer witnesses by one scheduling. // Default value is 4 which is subjected by scheduler-max-waiting-operator and witness-schedule-limit // If you want to increase balance speed more, please increase above-mentioned param. diff --git a/pkg/schedule/schedulers/balance_witness_test.go b/pkg/schedule/schedulers/balance_witness_test.go index d8715f71784..03fcac77ccc 100644 --- a/pkg/schedule/schedulers/balance_witness_test.go +++ b/pkg/schedule/schedulers/balance_witness_test.go @@ -23,7 +23,7 @@ import ( "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" ) diff --git a/pkg/schedule/schedulers/base_scheduler.go b/pkg/schedule/schedulers/base_scheduler.go index b3dae9856e6..6e160effea7 100644 --- a/pkg/schedule/schedulers/base_scheduler.go +++ b/pkg/schedule/schedulers/base_scheduler.go @@ -23,7 +23,7 @@ import ( "github.com/tikv/pd/pkg/errs" sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/typeutil" ) diff --git a/pkg/schedule/schedulers/diagnostic_recorder.go b/pkg/schedule/schedulers/diagnostic_recorder.go index df57dbebe71..cd99262ee48 100644 --- a/pkg/schedule/schedulers/diagnostic_recorder.go +++ b/pkg/schedule/schedulers/diagnostic_recorder.go @@ -23,6 +23,7 @@ import ( sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" + "github.com/tikv/pd/pkg/schedule/types" ) const ( @@ -46,27 +47,27 @@ const ( // DiagnosableSummaryFunc includes all implementations of plan.Summary. // And it also includes all schedulers which pd support to diagnose. -var DiagnosableSummaryFunc = map[string]plan.Summary{ - BalanceRegionName: plan.BalancePlanSummary, - BalanceLeaderName: plan.BalancePlanSummary, +var DiagnosableSummaryFunc = map[types.CheckerSchedulerType]plan.Summary{ + types.BalanceRegionScheduler: plan.BalancePlanSummary, + types.BalanceLeaderScheduler: plan.BalancePlanSummary, } // DiagnosticRecorder is used to manage diagnostic for one scheduler. type DiagnosticRecorder struct { - schedulerName string + schedulerType types.CheckerSchedulerType config sc.SchedulerConfigProvider summaryFunc plan.Summary results *cache.FIFO } // NewDiagnosticRecorder creates a new DiagnosticRecorder. -func NewDiagnosticRecorder(name string, config sc.SchedulerConfigProvider) *DiagnosticRecorder { - summaryFunc, ok := DiagnosableSummaryFunc[name] +func NewDiagnosticRecorder(tp types.CheckerSchedulerType, config sc.SchedulerConfigProvider) *DiagnosticRecorder { + summaryFunc, ok := DiagnosableSummaryFunc[tp] if !ok { return nil } return &DiagnosticRecorder{ - schedulerName: name, + schedulerType: tp, config: config, summaryFunc: summaryFunc, results: cache.NewFIFO(maxDiagnosticResultNum), @@ -131,11 +132,11 @@ func (d *DiagnosticRecorder) GetLastResult() *DiagnosticResult { } } else if firstStatus == Pending { // This is used to handle pending status because of reach limit in `IsScheduleAllowed` - resStr = fmt.Sprintf("%s reach limit", d.schedulerName) + resStr = fmt.Sprintf("%v reach limit", d.schedulerType) } } return &DiagnosticResult{ - Name: d.schedulerName, + Name: d.schedulerType.String(), Status: firstStatus, Summary: resStr, Timestamp: uint64(time.Now().Unix()), @@ -147,7 +148,11 @@ func (d *DiagnosticRecorder) SetResultFromStatus(status string) { if d == nil { return } - result := &DiagnosticResult{Name: d.schedulerName, Timestamp: uint64(time.Now().Unix()), Status: status} + result := &DiagnosticResult{ + Name: d.schedulerType.String(), + Timestamp: uint64(time.Now().Unix()), + Status: status, + } d.results.Put(result.Timestamp, result) } @@ -161,11 +166,14 @@ func (d *DiagnosticRecorder) SetResultFromPlans(ops []*operator.Operator, plans } func (d *DiagnosticRecorder) analyze(ops []*operator.Operator, plans []plan.Plan, ts uint64) *DiagnosticResult { - res := &DiagnosticResult{Name: d.schedulerName, Timestamp: ts, Status: Normal} - name := d.schedulerName + res := &DiagnosticResult{ + Name: d.schedulerType.String(), + Timestamp: ts, + Status: Normal, + } // TODO: support more schedulers and checkers - switch name { - case BalanceRegionName, BalanceLeaderName: + switch d.schedulerType { + case types.BalanceRegionScheduler, types.BalanceLeaderScheduler: if len(ops) != 0 { res.Status = Scheduling return res diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index 85f861f0082..a7d656a3e42 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -28,7 +28,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -36,8 +36,6 @@ import ( ) const ( - // EvictLeaderName is evict leader scheduler name. - EvictLeaderName = "evict-leader-scheduler" // EvictLeaderBatchSize is the number of operators to transfer // leaders by one scheduling EvictLeaderBatchSize = 3 @@ -212,7 +210,7 @@ func (conf *evictLeaderSchedulerConfig) delete(id uint64) (any, error) { return resp, nil } conf.Unlock() - if err := conf.removeSchedulerCb(EvictLeaderName); err != nil { + if err := conf.removeSchedulerCb(types.EvictLeaderScheduler.String()); err != nil { if !errors.ErrorEqual(err, errs.ErrSchedulerNotFound.FastGenByArgs()) { conf.resetStore(id, keyRanges) } diff --git a/pkg/schedule/schedulers/evict_leader_test.go b/pkg/schedule/schedulers/evict_leader_test.go index 692dda63437..b2c84ec68b7 100644 --- a/pkg/schedule/schedulers/evict_leader_test.go +++ b/pkg/schedule/schedulers/evict_leader_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" ) diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index 8f9c8841e04..d23fc2f8ff8 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -26,7 +26,7 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" @@ -34,9 +34,6 @@ import ( ) const ( - // EvictSlowStoreName is evict leader scheduler name. - EvictSlowStoreName = "evict-slow-store-scheduler" - slowStoreEvictThreshold = 100 slowStoreRecoverThreshold = 1 ) diff --git a/pkg/schedule/schedulers/evict_slow_store_test.go b/pkg/schedule/schedulers/evict_slow_store_test.go index 406a08b9c99..79651fb5b5c 100644 --- a/pkg/schedule/schedulers/evict_slow_store_test.go +++ b/pkg/schedule/schedulers/evict_slow_store_test.go @@ -23,7 +23,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" ) diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index b63b4b4a2e0..8fd76bdccd4 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -27,18 +27,13 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" "go.uber.org/zap" ) -const ( - // EvictSlowTrendName is evict leader by slow trend scheduler name. - EvictSlowTrendName = "evict-slow-trend-scheduler" -) - const ( alterEpsilon = 1e-9 minReCheckDurationGap = 120 // default gap for re-check the slow node, unit: s diff --git a/pkg/schedule/schedulers/evict_slow_trend_test.go b/pkg/schedule/schedulers/evict_slow_trend_test.go index 02cb65021eb..4be6eeb58d9 100644 --- a/pkg/schedule/schedulers/evict_slow_trend_test.go +++ b/pkg/schedule/schedulers/evict_slow_trend_test.go @@ -25,7 +25,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" ) diff --git a/pkg/schedule/schedulers/grant_hot_region.go b/pkg/schedule/schedulers/grant_hot_region.go index a441f41062a..18402c14437 100644 --- a/pkg/schedule/schedulers/grant_hot_region.go +++ b/pkg/schedule/schedulers/grant_hot_region.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" @@ -40,11 +40,6 @@ import ( "go.uber.org/zap" ) -const ( - // GrantHotRegionName is grant hot region scheduler name. - GrantHotRegionName = "grant-hot-region-scheduler" -) - type grantHotRegionSchedulerConfig struct { syncutil.RWMutex schedulerConfig diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index d70c0b19d67..5dbb6eef5f6 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -28,18 +28,13 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" "go.uber.org/zap" ) -const ( - // GrantLeaderName is grant leader scheduler name. - GrantLeaderName = "grant-leader-scheduler" -) - type grantLeaderSchedulerConfig struct { syncutil.RWMutex schedulerConfig @@ -312,7 +307,7 @@ func (handler *grantLeaderHandler) deleteConfig(w http.ResponseWriter, r *http.R return } if last { - if err := handler.config.removeSchedulerCb(GrantLeaderName); err != nil { + if err := handler.config.removeSchedulerCb(types.GrantLeaderScheduler.String()); err != nil { if errors.ErrorEqual(err, errs.ErrSchedulerNotFound.FastGenByArgs()) { handler.rd.JSON(w, http.StatusNotFound, err.Error()) } else { diff --git a/pkg/schedule/schedulers/hot_region.go b/pkg/schedule/schedulers/hot_region.go index ab595ec9058..6506698b75c 100644 --- a/pkg/schedule/schedulers/hot_region.go +++ b/pkg/schedule/schedulers/hot_region.go @@ -34,7 +34,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" @@ -45,8 +45,6 @@ import ( ) const ( - // HotRegionName is balance hot region scheduler name. - HotRegionName = "balance-hot-region-scheduler" splitHotReadBuckets = "split-hot-read-region" splitHotWriteBuckets = "split-hot-write-region" splitProgressiveRank = 5 @@ -191,7 +189,6 @@ func (h *baseHotScheduler) randomType() resourceType { } type hotScheduler struct { - name string *baseHotScheduler syncutil.RWMutex // config of hot scheduler @@ -203,7 +200,6 @@ func newHotScheduler(opController *operator.Controller, conf *hotRegionScheduler base := newBaseHotScheduler(opController, conf.getHistorySampleDuration(), conf.getHistorySampleInterval()) ret := &hotScheduler{ - name: HotRegionName, baseHotScheduler: base, conf: conf, } diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index 7ee0883fb7d..a4b3225312d 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -29,7 +29,7 @@ import ( "github.com/tikv/pd/pkg/mock/mockcluster" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/statistics/utils" @@ -61,14 +61,12 @@ func init() { func newHotReadScheduler(opController *operator.Controller, conf *hotRegionSchedulerConfig) *hotScheduler { ret := newHotScheduler(opController, conf) - ret.name = "" ret.types = []resourceType{readLeader, readPeer} return ret } func newHotWriteScheduler(opController *operator.Controller, conf *hotRegionSchedulerConfig) *hotScheduler { ret := newHotScheduler(opController, conf) - ret.name = "" ret.types = []resourceType{writeLeader, writePeer} return ret } @@ -2465,7 +2463,7 @@ func TestCompatibilityConfig(t *testing.T) { "dst-tolerance-ratio": 1.05, }) re.NoError(err) - err = storage.SaveSchedulerConfig(HotRegionName, data) + err = storage.SaveSchedulerConfig(types.BalanceHotRegionScheduler.String(), data) re.NoError(err) hb, err = CreateScheduler(types.BalanceHotRegionScheduler, oc, storage, ConfigJSONDecoder(data)) re.NoError(err) @@ -2481,7 +2479,7 @@ func TestCompatibilityConfig(t *testing.T) { cfg.WriteLeaderPriorities = []string{"query", "key"} data, err = EncodeConfig(cfg) re.NoError(err) - err = storage.SaveSchedulerConfig(HotRegionName, data) + err = storage.SaveSchedulerConfig(types.BalanceHotRegionScheduler.String(), data) re.NoError(err) hb, err = CreateScheduler(types.BalanceHotRegionScheduler, oc, storage, ConfigJSONDecoder(data)) re.NoError(err) diff --git a/pkg/schedule/schedulers/init.go b/pkg/schedule/schedulers/init.go index 16f78284cf8..e3101d6788b 100644 --- a/pkg/schedule/schedulers/init.go +++ b/pkg/schedule/schedulers/init.go @@ -22,7 +22,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage/endpoint" ) diff --git a/pkg/schedule/schedulers/label.go b/pkg/schedule/schedulers/label.go index 8d4f42262ac..d1a06a5c4ff 100644 --- a/pkg/schedule/schedulers/label.go +++ b/pkg/schedule/schedulers/label.go @@ -24,15 +24,10 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "go.uber.org/zap" ) -const ( - // LabelName is label scheduler name. - LabelName = "label-scheduler" -) - type labelSchedulerConfig struct { Ranges []core.KeyRange `json:"ranges"` // TODO: When we prepare to use Ranges, we will need to implement the ReloadConfig function for this scheduler. @@ -95,7 +90,7 @@ func (s *labelScheduler) Schedule(cluster sche.SchedulerCluster, _ bool) ([]*ope f := filter.NewExcludedFilter(s.GetName(), nil, excludeStores) target := filter.NewCandidates(cluster.GetFollowerStores(region)). - FilterTarget(cluster.GetSchedulerConfig(), nil, nil, &filter.StoreStateFilter{ActionScope: LabelName, TransferLeader: true, OperatorLevel: constant.Medium}, f). + FilterTarget(cluster.GetSchedulerConfig(), nil, nil, &filter.StoreStateFilter{ActionScope: s.GetName(), TransferLeader: true, OperatorLevel: constant.Medium}, f). RandomPick() if target == nil { log.Debug("label scheduler no target found for region", zap.Uint64("region-id", region.GetID())) diff --git a/pkg/schedule/schedulers/metrics.go b/pkg/schedule/schedulers/metrics.go index 90b0b0c6bb2..a518a167af7 100644 --- a/pkg/schedule/schedulers/metrics.go +++ b/pkg/schedule/schedulers/metrics.go @@ -16,7 +16,7 @@ package schedulers import ( "github.com/prometheus/client_golang/prometheus" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" ) var ( diff --git a/pkg/schedule/schedulers/random_merge.go b/pkg/schedule/schedulers/random_merge.go index 676e5407e72..f6660472f57 100644 --- a/pkg/schedule/schedulers/random_merge.go +++ b/pkg/schedule/schedulers/random_merge.go @@ -26,12 +26,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" -) - -const ( - // RandomMergeName is random merge scheduler name. - RandomMergeName = "random-merge-scheduler" + "github.com/tikv/pd/pkg/schedule/types" ) type randomMergeSchedulerConfig struct { diff --git a/pkg/schedule/schedulers/scatter_range.go b/pkg/schedule/schedulers/scatter_range.go index e86785fcc19..37f00d2df6e 100644 --- a/pkg/schedule/schedulers/scatter_range.go +++ b/pkg/schedule/schedulers/scatter_range.go @@ -25,17 +25,12 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" ) -const ( - // ScatterRangeName is scatter range scheduler name - ScatterRangeName = "scatter-range" -) - type scatterRangeSchedulerConfig struct { syncutil.RWMutex schedulerConfig diff --git a/pkg/schedule/schedulers/scheduler.go b/pkg/schedule/schedulers/scheduler.go index 7fce5d9c46e..27be70680af 100644 --- a/pkg/schedule/schedulers/scheduler.go +++ b/pkg/schedule/schedulers/scheduler.go @@ -27,7 +27,7 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage/endpoint" "go.uber.org/zap" ) diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go index c29b75a371a..5e1082acee3 100644 --- a/pkg/schedule/schedulers/scheduler_controller.go +++ b/pkg/schedule/schedulers/scheduler_controller.go @@ -29,6 +29,7 @@ import ( "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" @@ -400,7 +401,7 @@ func (c *Controller) GetPausedSchedulerDelayUntil(name string) (int64, error) { func (c *Controller) CheckTransferWitnessLeader(region *core.RegionInfo) { if core.NeedTransferWitnessLeader(region) { c.RLock() - s, ok := c.schedulers[TransferWitnessLeaderName] + s, ok := c.schedulers[types.TransferWitnessLeaderScheduler.String()] c.RUnlock() if ok { select { @@ -440,7 +441,7 @@ func NewScheduleController(ctx context.Context, cluster sche.SchedulerCluster, o nextInterval: s.GetMinInterval(), ctx: ctx, cancel: cancel, - diagnosticRecorder: NewDiagnosticRecorder(s.GetName(), cluster.GetSchedulerConfig()), + diagnosticRecorder: NewDiagnosticRecorder(s.GetType(), cluster.GetSchedulerConfig()), } } diff --git a/pkg/schedule/schedulers/scheduler_test.go b/pkg/schedule/schedulers/scheduler_test.go index 8dfe9f3616f..ba734230ea5 100644 --- a/pkg/schedule/schedulers/scheduler_test.go +++ b/pkg/schedule/schedulers/scheduler_test.go @@ -28,7 +28,7 @@ import ( "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" diff --git a/pkg/schedule/schedulers/shuffle_hot_region.go b/pkg/schedule/schedulers/shuffle_hot_region.go index f8544fff48d..b8818dc48da 100644 --- a/pkg/schedule/schedulers/shuffle_hot_region.go +++ b/pkg/schedule/schedulers/shuffle_hot_region.go @@ -26,7 +26,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" @@ -34,11 +34,6 @@ import ( "go.uber.org/zap" ) -const ( - // ShuffleHotRegionName is shuffle hot region scheduler name. - ShuffleHotRegionName = "shuffle-hot-region-scheduler" -) - type shuffleHotRegionSchedulerConfig struct { syncutil.RWMutex schedulerConfig diff --git a/pkg/schedule/schedulers/shuffle_leader.go b/pkg/schedule/schedulers/shuffle_leader.go index 4270613667b..842a26d9b12 100644 --- a/pkg/schedule/schedulers/shuffle_leader.go +++ b/pkg/schedule/schedulers/shuffle_leader.go @@ -23,12 +23,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" -) - -const ( - // ShuffleLeaderName is shuffle leader scheduler name. - ShuffleLeaderName = "shuffle-leader-scheduler" + "github.com/tikv/pd/pkg/schedule/types" ) type shuffleLeaderSchedulerConfig struct { diff --git a/pkg/schedule/schedulers/shuffle_region.go b/pkg/schedule/schedulers/shuffle_region.go index 5d4c49e0fcc..a4c247d3363 100644 --- a/pkg/schedule/schedulers/shuffle_region.go +++ b/pkg/schedule/schedulers/shuffle_region.go @@ -24,12 +24,7 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" -) - -const ( - // ShuffleRegionName is shuffle region scheduler name. - ShuffleRegionName = "shuffle-region-scheduler" + "github.com/tikv/pd/pkg/schedule/types" ) type shuffleRegionScheduler struct { @@ -41,11 +36,11 @@ type shuffleRegionScheduler struct { // newShuffleRegionScheduler creates an admin scheduler that shuffles regions // between stores. func newShuffleRegionScheduler(opController *operator.Controller, conf *shuffleRegionSchedulerConfig) Scheduler { + base := NewBaseScheduler(opController, types.ShuffleRegionScheduler) filters := []filter.Filter{ - &filter.StoreStateFilter{ActionScope: ShuffleRegionName, MoveRegion: true, OperatorLevel: constant.Low}, - filter.NewSpecialUseFilter(ShuffleRegionName), + &filter.StoreStateFilter{ActionScope: base.GetName(), MoveRegion: true, OperatorLevel: constant.Low}, + filter.NewSpecialUseFilter(base.GetName()), } - base := NewBaseScheduler(opController, types.ShuffleRegionScheduler) return &shuffleRegionScheduler{ BaseScheduler: base, conf: conf, diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index 0d8fa614aef..a0881ae1a34 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -28,7 +28,7 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/utils/reflectutil" "github.com/tikv/pd/pkg/utils/syncutil" @@ -36,8 +36,6 @@ import ( ) const ( - // SplitBucketName is the split bucket name. - SplitBucketName = "split-bucket-scheduler" // defaultHotDegree is the default hot region threshold. defaultHotDegree = 3 defaultSplitLimit = 10 diff --git a/pkg/schedule/schedulers/transfer_witness_leader.go b/pkg/schedule/schedulers/transfer_witness_leader.go index 9e7bd9bccc1..2ef0fc6a4f2 100644 --- a/pkg/schedule/schedulers/transfer_witness_leader.go +++ b/pkg/schedule/schedulers/transfer_witness_leader.go @@ -24,12 +24,10 @@ import ( "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" ) const ( - // TransferWitnessLeaderName is transfer witness leader scheduler name. - TransferWitnessLeaderName = "transfer-witness-leader-scheduler" // TransferWitnessLeaderBatchSize is the number of operators to to transfer // leaders by one scheduling transferWitnessLeaderBatchSize = 3 diff --git a/pkg/schedule/schedulers/transfer_witness_leader_test.go b/pkg/schedule/schedulers/transfer_witness_leader_test.go index 046b7aeb53f..b100e0a9535 100644 --- a/pkg/schedule/schedulers/transfer_witness_leader_test.go +++ b/pkg/schedule/schedulers/transfer_witness_leader_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/operator" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" ) diff --git a/pkg/schedule/type/type.go b/pkg/schedule/types/type.go similarity index 100% rename from pkg/schedule/type/type.go rename to pkg/schedule/types/type.go diff --git a/plugin/scheduler_example/evict_leader.go b/plugin/scheduler_example/evict_leader.go index c7842debdcb..c20cfd41814 100644 --- a/plugin/scheduler_example/evict_leader.go +++ b/plugin/scheduler_example/evict_leader.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/schedule/schedulers" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/syncutil" diff --git a/server/api/diagnostic_test.go b/server/api/diagnostic_test.go index 8c4089a8710..c85d9a45369 100644 --- a/server/api/diagnostic_test.go +++ b/server/api/diagnostic_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" @@ -95,17 +96,17 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { re.NoError(tu.ReadGetJSON(re, testDialClient, addr, cfg)) re.True(cfg.Schedule.EnableDiagnostic) - balanceRegionURL := suite.urlPrefix + "/" + schedulers.BalanceRegionName + balanceRegionURL := suite.urlPrefix + "/" + types.BalanceRegionScheduler.String() result := &schedulers.DiagnosticResult{} err = tu.ReadGetJSON(re, testDialClient, balanceRegionURL, result) re.NoError(err) re.Equal("disabled", result.Status) - evictLeaderURL := suite.urlPrefix + "/" + schedulers.EvictLeaderName + evictLeaderURL := suite.urlPrefix + "/" + types.EvictLeaderScheduler.String() re.NoError(tu.CheckGetJSON(testDialClient, evictLeaderURL, nil, tu.StatusNotOK(re))) input := make(map[string]any) - input["name"] = schedulers.BalanceRegionName + input["name"] = types.BalanceRegionScheduler.String() body, err := json.Marshal(input) re.NoError(err) err = tu.CheckPostJSON(testDialClient, suite.schedulerPrefix, body, tu.StatusOK(re)) @@ -116,14 +117,14 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { input["delay"] = 30 pauseArgs, err := json.Marshal(input) re.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.schedulerPrefix+"/"+schedulers.BalanceRegionName, pauseArgs, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, suite.schedulerPrefix+"/"+types.BalanceRegionScheduler.String(), pauseArgs, tu.StatusOK(re)) re.NoError(err) suite.checkStatus("paused", balanceRegionURL) input["delay"] = 0 pauseArgs, err = json.Marshal(input) re.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.schedulerPrefix+"/"+schedulers.BalanceRegionName, pauseArgs, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, suite.schedulerPrefix+"/"+types.BalanceRegionScheduler.String(), pauseArgs, tu.StatusOK(re)) re.NoError(err) suite.checkStatus("pending", balanceRegionURL) @@ -132,7 +133,7 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { fmt.Println("after put region") suite.checkStatus("normal", balanceRegionURL) - deleteURL := fmt.Sprintf("%s/%s", suite.schedulerPrefix, schedulers.BalanceRegionName) + deleteURL := fmt.Sprintf("%s/%s", suite.schedulerPrefix, types.BalanceRegionScheduler.String()) err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) re.NoError(err) suite.checkStatus("disabled", balanceRegionURL) diff --git a/server/api/scheduler.go b/server/api/scheduler.go index 1d502013558..c96e4c123de 100644 --- a/server/api/scheduler.go +++ b/server/api/scheduler.go @@ -24,8 +24,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule/schedulers" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" "github.com/unrolled/render" @@ -177,11 +176,11 @@ func (h *schedulerHandler) CreateScheduler(w http.ResponseWriter, r *http.Reques func (h *schedulerHandler) DeleteScheduler(w http.ResponseWriter, r *http.Request) { name := mux.Vars(r)["name"] switch { - case strings.HasPrefix(name, schedulers.EvictLeaderName) && name != schedulers.EvictLeaderName: - h.redirectSchedulerDelete(w, name, schedulers.EvictLeaderName) + case strings.HasPrefix(name, types.EvictLeaderScheduler.String()) && name != types.EvictLeaderScheduler.String(): + h.redirectSchedulerDelete(w, name, types.EvictLeaderScheduler.String()) return - case strings.HasPrefix(name, schedulers.GrantLeaderName) && name != schedulers.GrantLeaderName: - h.redirectSchedulerDelete(w, name, schedulers.GrantLeaderName) + case strings.HasPrefix(name, types.GrantLeaderScheduler.String()) && name != types.GrantLeaderScheduler.String(): + h.redirectSchedulerDelete(w, name, types.GrantLeaderScheduler.String()) return default: if err := h.RemoveScheduler(name); err != nil { diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index cffefbba444..9a9420988a1 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -53,7 +53,7 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/schedulers" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" @@ -366,7 +366,7 @@ func TestSetOfflineStoreWithEvictLeader(t *testing.T) { err = cluster.RemoveStore(3, false) re.Error(err) re.Contains(err.Error(), string(errs.ErrNoStoreForRegionLeader.RFCCode())) - re.NoError(cluster.RemoveScheduler(schedulers.EvictLeaderName)) + re.NoError(cluster.RemoveScheduler(types.EvictLeaderScheduler.String())) re.NoError(cluster.RemoveStore(3, false)) } @@ -2453,10 +2453,10 @@ func TestDispatch(t *testing.T) { waitOperator(re, co, 1) controller := co.GetSchedulersController() operatorutil.CheckTransferPeer(re, co.GetOperatorController().GetOperator(1), operator.OpKind(0), 4, 1) - re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) + re.NoError(controller.RemoveScheduler(types.BalanceRegionScheduler.String())) waitOperator(re, co, 2) operatorutil.CheckTransferLeader(re, co.GetOperatorController().GetOperator(2), operator.OpKind(0), 4, 2) - re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) + re.NoError(controller.RemoveScheduler(types.BalanceLeaderScheduler.String())) stream := mockhbstream.NewHeartbeatStream() @@ -3075,10 +3075,10 @@ func TestAddScheduler(t *testing.T) { defer cleanup() controller := co.GetSchedulersController() re.Len(controller.GetSchedulerNames(), len(sc.DefaultSchedulers)) - re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) - re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) + re.NoError(controller.RemoveScheduler(types.BalanceLeaderScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceHotRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.EvictSlowStoreScheduler.String())) re.Empty(controller.GetSchedulerNames()) stream := mockhbstream.NewHeartbeatStream() @@ -3170,10 +3170,10 @@ func TestPersistScheduler(t *testing.T) { re.Len(sches, defaultCount+2) // remove all default schedulers - re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) - re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) + re.NoError(controller.RemoveScheduler(types.BalanceLeaderScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceHotRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.EvictSlowStoreScheduler.String())) // only remains 2 items with independent config. re.Len(controller.GetSchedulerNames(), 2) re.NoError(co.GetCluster().GetSchedulerConfig().Persist(storage)) @@ -3233,7 +3233,7 @@ func TestPersistScheduler(t *testing.T) { // the scheduler option should contain 9 items // the `hot scheduler` are disabled re.Len(co.GetCluster().GetSchedulerConfig().(*config.PersistOptions).GetSchedulers(), defaultCount+3) - re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName)) + re.NoError(controller.RemoveScheduler(types.GrantLeaderScheduler.String())) // the scheduler that is not enable by default will be completely deleted re.Len(co.GetCluster().GetSchedulerConfig().(*config.PersistOptions).GetSchedulers(), defaultCount+2) re.Len(controller.GetSchedulerNames(), 4) @@ -3250,7 +3250,7 @@ func TestPersistScheduler(t *testing.T) { co.Run() controller = co.GetSchedulersController() re.Len(controller.GetSchedulerNames(), 4) - re.NoError(controller.RemoveScheduler(schedulers.EvictLeaderName)) + re.NoError(controller.RemoveScheduler(types.EvictLeaderScheduler.String())) re.Len(controller.GetSchedulerNames(), 3) } @@ -3283,11 +3283,11 @@ func TestRemoveScheduler(t *testing.T) { re.Len(sches, defaultCount+1) // remove all schedulers - re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) - re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) - re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName)) - re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) + re.NoError(controller.RemoveScheduler(types.BalanceLeaderScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.BalanceHotRegionScheduler.String())) + re.NoError(controller.RemoveScheduler(types.GrantLeaderScheduler.String())) + re.NoError(controller.RemoveScheduler(types.EvictSlowStoreScheduler.String())) // all removed sches, _, err = storage.LoadAllSchedulerConfigs() re.NoError(err) @@ -3360,15 +3360,15 @@ func TestPauseScheduler(t *testing.T) { controller := co.GetSchedulersController() _, err := controller.IsSchedulerAllowed("test") re.Error(err) - controller.PauseOrResumeScheduler(schedulers.BalanceLeaderName, 60) - paused, _ := controller.IsSchedulerPaused(schedulers.BalanceLeaderName) + controller.PauseOrResumeScheduler(types.BalanceLeaderScheduler.String(), 60) + paused, _ := controller.IsSchedulerPaused(types.BalanceLeaderScheduler.String()) re.True(paused) - pausedAt, err := controller.GetPausedSchedulerDelayAt(schedulers.BalanceLeaderName) + pausedAt, err := controller.GetPausedSchedulerDelayAt(types.BalanceLeaderScheduler.String()) re.NoError(err) - resumeAt, err := controller.GetPausedSchedulerDelayUntil(schedulers.BalanceLeaderName) + resumeAt, err := controller.GetPausedSchedulerDelayUntil(types.BalanceLeaderScheduler.String()) re.NoError(err) re.Equal(int64(60), resumeAt-pausedAt) - allowed, _ := controller.IsSchedulerAllowed(schedulers.BalanceLeaderName) + allowed, _ := controller.IsSchedulerAllowed(types.BalanceLeaderScheduler.String()) re.False(allowed) } diff --git a/server/cluster/scheduling_controller.go b/server/cluster/scheduling_controller.go index b4c29ceed46..20a36a6817d 100644 --- a/server/cluster/scheduling_controller.go +++ b/server/cluster/scheduling_controller.go @@ -33,6 +33,7 @@ import ( "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/schedule/splitter" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/statistics/utils" @@ -455,7 +456,7 @@ func (sc *schedulingController) getEvictLeaderStores() (evictStores []uint64) { if sc.coordinator == nil { return nil } - handler, ok := sc.coordinator.GetSchedulersController().GetSchedulerHandlers()[schedulers.EvictLeaderName] + handler, ok := sc.coordinator.GetSchedulersController().GetSchedulerHandlers()[types.EvictLeaderScheduler.String()] if !ok { return } diff --git a/server/config/persist_options.go b/server/config/persist_options.go index 19f0ef0d475..807e9699a25 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -33,7 +33,7 @@ import ( "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/core/storelimit" sc "github.com/tikv/pd/pkg/schedule/config" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" diff --git a/server/handler.go b/server/handler.go index e3e4184f177..1464c768aed 100644 --- a/server/handler.go +++ b/server/handler.go @@ -34,7 +34,7 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/handler" "github.com/tikv/pd/pkg/schedule/schedulers" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/storage" diff --git a/tests/integrations/mcs/scheduling/config_test.go b/tests/integrations/mcs/scheduling/config_test.go index 168d3a8742c..fcc9a78b0f3 100644 --- a/tests/integrations/mcs/scheduling/config_test.go +++ b/tests/integrations/mcs/scheduling/config_test.go @@ -28,6 +28,7 @@ import ( "github.com/tikv/pd/pkg/mcs/scheduling/server/config" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" @@ -165,14 +166,14 @@ func (suite *configTestSuite) TestSchedulerConfigWatch() { }) re.Equal(namesFromAPIServer, namesFromSchedulingServer) // Add a new scheduler. - api.MustAddScheduler(re, suite.pdLeaderServer.GetAddr(), schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.pdLeaderServer.GetAddr(), types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) // Check the new scheduler's config. testutil.Eventually(re, func() bool { namesFromSchedulingServer, _, err = storage.LoadAllSchedulerConfigs() re.NoError(err) - return slice.Contains(namesFromSchedulingServer, schedulers.EvictLeaderName) + return slice.Contains(namesFromSchedulingServer, types.EvictLeaderScheduler.String()) }) assertEvictLeaderStoreIDs(re, storage, []uint64{1}) // Update the scheduler by adding a store. @@ -187,20 +188,20 @@ func (suite *configTestSuite) TestSchedulerConfigWatch() { }, ) re.NoError(err) - api.MustAddScheduler(re, suite.pdLeaderServer.GetAddr(), schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.pdLeaderServer.GetAddr(), types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 2, }) assertEvictLeaderStoreIDs(re, storage, []uint64{1, 2}) // Update the scheduler by removing a store. - api.MustDeleteScheduler(re, suite.pdLeaderServer.GetAddr(), fmt.Sprintf("%s-%d", schedulers.EvictLeaderName, 1)) + api.MustDeleteScheduler(re, suite.pdLeaderServer.GetAddr(), fmt.Sprintf("%s-%d", types.EvictLeaderScheduler.String(), 1)) assertEvictLeaderStoreIDs(re, storage, []uint64{2}) // Delete the scheduler. - api.MustDeleteScheduler(re, suite.pdLeaderServer.GetAddr(), schedulers.EvictLeaderName) + api.MustDeleteScheduler(re, suite.pdLeaderServer.GetAddr(), types.EvictLeaderScheduler.String()) // Check the removed scheduler's config. testutil.Eventually(re, func() bool { namesFromSchedulingServer, _, err = storage.LoadAllSchedulerConfigs() re.NoError(err) - return !slice.Contains(namesFromSchedulingServer, schedulers.EvictLeaderName) + return !slice.Contains(namesFromSchedulingServer, types.EvictLeaderScheduler.String()) }) watcher.Close() } @@ -212,7 +213,7 @@ func assertEvictLeaderStoreIDs( StoreIDWithRanges map[uint64][]core.KeyRange `json:"store-id-ranges"` } testutil.Eventually(re, func() bool { - cfg, err := storage.LoadSchedulerConfig(schedulers.EvictLeaderName) + cfg, err := storage.LoadSchedulerConfig(types.EvictLeaderScheduler.String()) re.NoError(err) err = schedulers.DecodeConfig([]byte(cfg), &evictLeaderCfg) re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 5c08892a972..57306640394 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -32,6 +32,7 @@ import ( "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/tests" @@ -305,7 +306,7 @@ func (suite *serverTestSuite) TestSchedulerSync() { schedulersController := tc.GetPrimaryServer().GetCluster().GetCoordinator().GetSchedulersController() checkEvictLeaderSchedulerExist(re, schedulersController, false) // Add a new evict-leader-scheduler through the API server. - api.MustAddScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) // Check if the evict-leader-scheduler is added. @@ -323,54 +324,54 @@ func (suite *serverTestSuite) TestSchedulerSync() { }, ) re.NoError(err) - api.MustAddScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 2, }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) // Delete a store_id from the evict-leader-scheduler through the API server. - api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", schedulers.EvictLeaderName, 1)) + api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", types.EvictLeaderScheduler.String(), 1)) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{2}) // Add a store_id to the evict-leader-scheduler through the API server by the scheduler handler. - api.MustCallSchedulerConfigAPI(re, http.MethodPost, suite.backendEndpoints, schedulers.EvictLeaderName, []string{"config"}, map[string]any{ - "name": schedulers.EvictLeaderName, + api.MustCallSchedulerConfigAPI(re, http.MethodPost, suite.backendEndpoints, types.EvictLeaderScheduler.String(), []string{"config"}, map[string]any{ + "name": types.EvictLeaderScheduler.String(), "store_id": 1, }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) // Delete a store_id from the evict-leader-scheduler through the API server by the scheduler handler. - api.MustCallSchedulerConfigAPI(re, http.MethodDelete, suite.backendEndpoints, schedulers.EvictLeaderName, []string{"delete", "2"}, nil) + api.MustCallSchedulerConfigAPI(re, http.MethodDelete, suite.backendEndpoints, types.EvictLeaderScheduler.String(), []string{"delete", "2"}, nil) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1}) // If the last store is deleted, the scheduler should be removed. - api.MustCallSchedulerConfigAPI(re, http.MethodDelete, suite.backendEndpoints, schedulers.EvictLeaderName, []string{"delete", "1"}, nil) + api.MustCallSchedulerConfigAPI(re, http.MethodDelete, suite.backendEndpoints, types.EvictLeaderScheduler.String(), []string{"delete", "1"}, nil) // Check if the scheduler is removed. checkEvictLeaderSchedulerExist(re, schedulersController, false) // Delete the evict-leader-scheduler through the API server by removing the last store_id. - api.MustAddScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1}) - api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", schedulers.EvictLeaderName, 1)) + api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", types.EvictLeaderScheduler.String(), 1)) checkEvictLeaderSchedulerExist(re, schedulersController, false) // Delete the evict-leader-scheduler through the API server. - api.MustAddScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1}) - api.MustDeleteScheduler(re, suite.backendEndpoints, schedulers.EvictLeaderName) + api.MustDeleteScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String()) checkEvictLeaderSchedulerExist(re, schedulersController, false) // The default scheduler could not be deleted, it could only be disabled. defaultSchedulerNames := []string{ - schedulers.BalanceLeaderName, - schedulers.BalanceRegionName, - schedulers.HotRegionName, + types.BalanceLeaderScheduler.String(), + types.BalanceRegionScheduler.String(), + types.BalanceHotRegionScheduler.String(), } checkDisabled := func(name string, shouldDisabled bool) { re.NotNil(schedulersController.GetScheduler(name), name) @@ -395,14 +396,14 @@ func (suite *serverTestSuite) TestSchedulerSync() { func checkEvictLeaderSchedulerExist(re *require.Assertions, sc *schedulers.Controller, exist bool) { testutil.Eventually(re, func() bool { if !exist { - return sc.GetScheduler(schedulers.EvictLeaderName) == nil + return sc.GetScheduler(types.EvictLeaderScheduler.String()) == nil } - return sc.GetScheduler(schedulers.EvictLeaderName) != nil + return sc.GetScheduler(types.EvictLeaderScheduler.String()) != nil }) } func checkEvictLeaderStoreIDs(re *require.Assertions, sc *schedulers.Controller, expected []uint64) { - handler, ok := sc.GetSchedulerHandlers()[schedulers.EvictLeaderName] + handler, ok := sc.GetSchedulerHandlers()[types.EvictLeaderScheduler.String()] re.True(ok) h, ok := handler.(interface { EvictStoreIDs() []uint64 diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go index 3b75e6c8c88..98a18158114 100644 --- a/tests/integrations/realcluster/scheduler_test.go +++ b/tests/integrations/realcluster/scheduler_test.go @@ -25,7 +25,7 @@ import ( pd "github.com/tikv/pd/client/http" "github.com/tikv/pd/client/testutil" "github.com/tikv/pd/pkg/schedule/labeler" - "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" ) // https://github.com/tikv/pd/issues/6988#issuecomment-1694924611 @@ -47,9 +47,9 @@ func TestTransferLeader(t *testing.T) { } // record scheduler - re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.EvictLeaderName, 1)) + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.EvictLeaderScheduler.String(), 1)) defer func() { - re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName)) + re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String())) }() res, err := pdHTTPCli.GetSchedulers(ctx) re.NoError(err) @@ -89,16 +89,16 @@ func TestRegionLabelDenyScheduler(t *testing.T) { re.NotEmpty(regions.Regions) region1 := regions.Regions[0] - err = pdHTTPCli.DeleteScheduler(ctx, schedulers.BalanceLeaderName) + err = pdHTTPCli.DeleteScheduler(ctx, types.BalanceLeaderScheduler.String()) if err == nil { defer func() { - pdHTTPCli.CreateScheduler(ctx, schedulers.BalanceLeaderName, 0) + pdHTTPCli.CreateScheduler(ctx, types.BalanceLeaderScheduler.String(), 0) }() } - re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.GrantLeaderName, uint64(region1.Leader.StoreID))) + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.GrantLeaderScheduler.String(), uint64(region1.Leader.StoreID))) defer func() { - pdHTTPCli.DeleteScheduler(ctx, schedulers.GrantLeaderName) + pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String()) }() // wait leader transfer @@ -135,10 +135,10 @@ func TestRegionLabelDenyScheduler(t *testing.T) { re.Equal(labelRule.RuleType, labelRules[1].RuleType) // enable evict leader scheduler, and check it works - re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.GrantLeaderName)) - re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.EvictLeaderName, uint64(region1.Leader.StoreID))) + re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String())) + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.EvictLeaderScheduler.String(), uint64(region1.Leader.StoreID))) defer func() { - pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName) + pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String()) }() testutil.Eventually(re, func() bool { regions, err := pdHTTPCli.GetRegions(ctx) @@ -151,10 +151,10 @@ func TestRegionLabelDenyScheduler(t *testing.T) { return true }, testutil.WithWaitFor(time.Minute)) - re.NoError(pdHTTPCli.DeleteScheduler(ctx, schedulers.EvictLeaderName)) - re.NoError(pdHTTPCli.CreateScheduler(ctx, schedulers.GrantLeaderName, uint64(region1.Leader.StoreID))) + re.NoError(pdHTTPCli.DeleteScheduler(ctx, types.EvictLeaderScheduler.String())) + re.NoError(pdHTTPCli.CreateScheduler(ctx, types.GrantLeaderScheduler.String(), uint64(region1.Leader.StoreID))) defer func() { - pdHTTPCli.DeleteScheduler(ctx, schedulers.GrantLeaderName) + pdHTTPCli.DeleteScheduler(ctx, types.GrantLeaderScheduler.String()) }() testutil.Eventually(re, func() bool { regions, err := pdHTTPCli.GetRegions(ctx) diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index f3b8509d2e5..330a69eca63 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" sc "github.com/tikv/pd/pkg/schedule/config" - types "github.com/tikv/pd/pkg/schedule/type" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 27d3300b293..375edf027db 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -40,6 +40,7 @@ import ( sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/syncer" @@ -1410,10 +1411,10 @@ func TestTransferLeaderForScheduler(t *testing.T) { re.True(leaderServer.GetRaftCluster().IsPrepared()) schedsNum := len(rc.GetCoordinator().GetSchedulersController().GetSchedulerNames()) // Add evict leader scheduler - api.MustAddScheduler(re, leaderServer.GetAddr(), schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, leaderServer.GetAddr(), types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) - api.MustAddScheduler(re, leaderServer.GetAddr(), schedulers.EvictLeaderName, map[string]any{ + api.MustAddScheduler(re, leaderServer.GetAddr(), types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 2, }) // Check scheduler updated. @@ -1468,14 +1469,14 @@ func TestTransferLeaderForScheduler(t *testing.T) { func checkEvictLeaderSchedulerExist(re *require.Assertions, sc *schedulers.Controller, exist bool) { testutil.Eventually(re, func() bool { if !exist { - return sc.GetScheduler(schedulers.EvictLeaderName) == nil + return sc.GetScheduler(types.EvictLeaderScheduler.String()) == nil } - return sc.GetScheduler(schedulers.EvictLeaderName) != nil + return sc.GetScheduler(types.EvictLeaderScheduler.String()) != nil }) } func checkEvictLeaderStoreIDs(re *require.Assertions, sc *schedulers.Controller, expected []uint64) { - handler, ok := sc.GetSchedulerHandlers()[schedulers.EvictLeaderName] + handler, ok := sc.GetSchedulerHandlers()[types.EvictLeaderScheduler.String()] re.True(ok) h, ok := handler.(interface { EvictStoreIDs() []uint64 From 6c30bbf6eda61747d96f8ae090fba5d1e298c4a6 Mon Sep 17 00:00:00 2001 From: Jack Lyu <63168620+JackL9u@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:31:41 +0800 Subject: [PATCH 24/42] ctl: replace gc_safepoint call with PD HTTP SDK (#8504) ref tikv/pd#7300 replace gc_safepoint call with PD HTTP SDK Signed-off-by: Boyang Lyu Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/http/api.go | 6 + client/http/interface.go | 30 ++++ client/http/request_info.go | 2 + client/http/types.go | 16 ++ pkg/storage/endpoint/gc_safe_point.go | 1 + server/api/service_gc_safepoint.go | 1 + tests/integrations/client/http_client_test.go | 82 +++++++++++ .../pdctl/command/gc_safepoint_command.go | 34 +---- .../pd-ctl/tests/safepoint/safepoint_test.go | 138 ++++++++++++++++++ 9 files changed, 284 insertions(+), 26 deletions(-) create mode 100644 tools/pd-ctl/tests/safepoint/safepoint_test.go diff --git a/client/http/api.go b/client/http/api.go index 3376a48770d..d1bce99f4f9 100644 --- a/client/http/api.go +++ b/client/http/api.go @@ -79,6 +79,7 @@ const ( Status = "/pd/api/v1/status" Version = "/pd/api/v1/version" operators = "/pd/api/v1/operators" + safepoint = "/pd/api/v1/gc/safepoint" // Micro Service microServicePrefix = "/pd/api/v2/ms" // Keyspace @@ -215,3 +216,8 @@ func GetUpdateKeyspaceConfigURL(keyspaceName string) string { func GetKeyspaceMetaByNameURL(keyspaceName string) string { return fmt.Sprintf(GetKeyspaceMetaByName, keyspaceName) } + +// GetDeleteSafePointURI returns the URI for delete safepoint service +func GetDeleteSafePointURI(serviceID string) string { + return fmt.Sprintf("%s/%s", safepoint, serviceID) +} diff --git a/client/http/interface.go b/client/http/interface.go index cd9fc22702e..f5cd1a38211 100644 --- a/client/http/interface.go +++ b/client/http/interface.go @@ -100,6 +100,8 @@ type Client interface { /* Other interfaces */ GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) GetPDVersion(context.Context) (string, error) + GetGCSafePoint(context.Context) (ListServiceGCSafepoint, error) + DeleteGCSafePoint(context.Context, string) (string, error) /* Micro Service interfaces */ GetMicroServiceMembers(context.Context, string) ([]MicroServiceMember, error) GetMicroServicePrimary(context.Context, string) (string, error) @@ -1024,3 +1026,31 @@ func (c *client) GetKeyspaceMetaByName(ctx context.Context, keyspaceName string) } return &keyspaceMetaPB, nil } + +// GetGCSafePoint gets the GC safe point list. +func (c *client) GetGCSafePoint(ctx context.Context) (ListServiceGCSafepoint, error) { + var gcSafePoint ListServiceGCSafepoint + err := c.request(ctx, newRequestInfo(). + WithName(GetGCSafePointName). + WithURI(safepoint). + WithMethod(http.MethodGet). + WithResp(&gcSafePoint)) + if err != nil { + return gcSafePoint, err + } + return gcSafePoint, nil +} + +// DeleteGCSafePoint deletes a GC safe point with the given service ID. +func (c *client) DeleteGCSafePoint(ctx context.Context, serviceID string) (string, error) { + var msg string + err := c.request(ctx, newRequestInfo(). + WithName(DeleteGCSafePointName). + WithURI(GetDeleteSafePointURI(serviceID)). + WithMethod(http.MethodDelete). + WithResp(&msg)) + if err != nil { + return msg, err + } + return msg, nil +} diff --git a/client/http/request_info.go b/client/http/request_info.go index 783220bcc60..94f71c6186e 100644 --- a/client/http/request_info.go +++ b/client/http/request_info.go @@ -85,6 +85,8 @@ const ( deleteOperators = "DeleteOperators" UpdateKeyspaceGCManagementTypeName = "UpdateKeyspaceGCManagementType" GetKeyspaceMetaByNameName = "GetKeyspaceMetaByName" + GetGCSafePointName = "GetGCSafePoint" + DeleteGCSafePointName = "DeleteGCSafePoint" ) type requestInfo struct { diff --git a/client/http/types.go b/client/http/types.go index 55f9b65caad..4bc60978a0e 100644 --- a/client/http/types.go +++ b/client/http/types.go @@ -25,6 +25,22 @@ import ( pd "github.com/tikv/pd/client" ) +// ServiceSafePoint is the safepoint for a specific service +// NOTE: This type is in sync with pd/pkg/storage/endpoint/gc_safe_point.go +type ServiceSafePoint struct { + ServiceID string `json:"service_id"` + ExpiredAt int64 `json:"expired_at"` + SafePoint uint64 `json:"safe_point"` +} + +// ListServiceGCSafepoint is the response for list service GC safepoint. +// NOTE: This type is in sync with pd/server/api/service_gc_safepoint.go +type ListServiceGCSafepoint struct { + ServiceGCSafepoints []*ServiceSafePoint `json:"service_gc_safe_points"` + MinServiceGcSafepoint uint64 `json:"min_service_gc_safe_point,omitempty"` + GCSafePoint uint64 `json:"gc_safe_point"` +} + // ClusterState saves some cluster state information. // NOTE: This type sync with https://github.com/tikv/pd/blob/5eae459c01a797cbd0c416054c6f0cad16b8740a/server/cluster/cluster.go#L173 type ClusterState struct { diff --git a/pkg/storage/endpoint/gc_safe_point.go b/pkg/storage/endpoint/gc_safe_point.go index 8d59d827fa4..7b0b0bf86a7 100644 --- a/pkg/storage/endpoint/gc_safe_point.go +++ b/pkg/storage/endpoint/gc_safe_point.go @@ -28,6 +28,7 @@ import ( // ServiceSafePoint is the safepoint for a specific service // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +// This type is in sync with `client/http/types.go`. type ServiceSafePoint struct { ServiceID string `json:"service_id"` ExpiredAt int64 `json:"expired_at"` diff --git a/server/api/service_gc_safepoint.go b/server/api/service_gc_safepoint.go index d6bb153eb6f..ca29f9c352f 100644 --- a/server/api/service_gc_safepoint.go +++ b/server/api/service_gc_safepoint.go @@ -38,6 +38,7 @@ func newServiceGCSafepointHandler(svr *server.Server, rd *render.Render) *servic // ListServiceGCSafepoint is the response for list service GC safepoint. // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +// This type is in sync with `pd/client/http/types.go`. type ListServiceGCSafepoint struct { ServiceGCSafepoints []*endpoint.ServiceSafePoint `json:"service_gc_safe_points"` MinServiceGcSafepoint uint64 `json:"min_service_gc_safe_point,omitempty"` diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index fd8b65f01ba..fe0962012e6 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -37,9 +37,11 @@ import ( sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/pkg/versioninfo" + "github.com/tikv/pd/server/api" "github.com/tikv/pd/tests" ) @@ -835,3 +837,83 @@ func (suite *httpClientTestSuite) TestRetryOnLeaderChange() { cancel() wg.Wait() } + +func (suite *httpClientTestSuite) TestGetGCSafePoint() { + re := suite.Require() + client := suite.client + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + + // adding some safepoints to the server + list := &api.ListServiceGCSafepoint{ + ServiceGCSafepoints: []*endpoint.ServiceSafePoint{ + { + ServiceID: "AAA", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 1, + }, + { + ServiceID: "BBB", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 2, + }, + { + ServiceID: "CCC", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 3, + }, + }, + GCSafePoint: 1, + MinServiceGcSafepoint: 1, + } + + storage := suite.cluster.GetLeaderServer().GetServer().GetStorage() + for _, ssp := range list.ServiceGCSafepoints { + err := storage.SaveServiceGCSafePoint(ssp) + re.NoError(err) + } + storage.SaveGCSafePoint(1) + + // get the safepoints and start testing + l, err := client.GetGCSafePoint(ctx) + re.NoError(err) + + re.Equal(uint64(1), l.GCSafePoint) + re.Equal(uint64(1), l.MinServiceGcSafepoint) + re.Len(l.ServiceGCSafepoints, 3) + + // sort the gc safepoints based on order of ServiceID + sort.Slice(l.ServiceGCSafepoints, func(i, j int) bool { + return l.ServiceGCSafepoints[i].ServiceID < l.ServiceGCSafepoints[j].ServiceID + }) + + for i, val := range l.ServiceGCSafepoints { + re.Equal(list.ServiceGCSafepoints[i].ServiceID, val.ServiceID) + re.Equal(list.ServiceGCSafepoints[i].SafePoint, val.SafePoint) + } + + // delete the safepoints + for i := 0; i < 3; i++ { + msg, err := client.DeleteGCSafePoint(ctx, list.ServiceGCSafepoints[i].ServiceID) + re.NoError(err) + re.Equal("Delete service GC safepoint successfully.", msg) + } + + // check that the safepoitns are indeed deleted + l, err = client.GetGCSafePoint(ctx) + re.NoError(err) + + re.Equal(uint64(1), l.GCSafePoint) + re.Equal(uint64(0), l.MinServiceGcSafepoint) + re.Empty(l.ServiceGCSafepoints) + + // try delete gc_worker, should get an error + _, err = client.DeleteGCSafePoint(ctx, "gc_worker") + re.Error(err) + + // try delete some non-exist safepoints, should return normally + var msg string + msg, err = client.DeleteGCSafePoint(ctx, "non_exist") + re.NoError(err) + re.Equal("Delete service GC safepoint successfully.", msg) +} diff --git a/tools/pd-ctl/pdctl/command/gc_safepoint_command.go b/tools/pd-ctl/pdctl/command/gc_safepoint_command.go index f4a6b6fcfd0..9a07d92937f 100644 --- a/tools/pd-ctl/pdctl/command/gc_safepoint_command.go +++ b/tools/pd-ctl/pdctl/command/gc_safepoint_command.go @@ -15,24 +15,18 @@ package command import ( - "encoding/json" - "net/http" "sort" "github.com/spf13/cobra" - "github.com/tikv/pd/server/api" -) - -var ( - serviceGCSafepointPrefix = "pd/api/v1/gc/safepoint" ) // NewServiceGCSafepointCommand return a service gc safepoint subcommand of rootCmd func NewServiceGCSafepointCommand() *cobra.Command { l := &cobra.Command{ - Use: "service-gc-safepoint", - Short: "show all service gc safepoint", - Run: showSSPs, + Use: "service-gc-safepoint", + Short: "show all service gc safepoint", + PersistentPreRunE: requirePDClient, + Run: showSSPs, } l.AddCommand(NewDeleteServiceGCSafepointCommand()) return l @@ -50,25 +44,15 @@ func NewDeleteServiceGCSafepointCommand() *cobra.Command { } func showSSPs(cmd *cobra.Command, _ []string) { - r, err := doRequest(cmd, serviceGCSafepointPrefix, http.MethodGet, http.Header{}) + safepoint, err := PDCli.GetGCSafePoint(cmd.Context()) if err != nil { cmd.Printf("Failed to get service GC safepoint: %s\n", err) return } - var safepoint api.ListServiceGCSafepoint - if err := json.Unmarshal([]byte(r), &safepoint); err != nil { - cmd.Printf("Failed to unmarshal service GC safepoint: %s\n", err) - return - } sort.Slice(safepoint.ServiceGCSafepoints, func(i, j int) bool { return safepoint.ServiceGCSafepoints[i].SafePoint < safepoint.ServiceGCSafepoints[j].SafePoint }) - data, err := json.MarshalIndent(safepoint, "", " ") - if err != nil { - cmd.Printf("Failed to marshal service GC safepoint: %s\n", err) - return - } - cmd.Println(string(data)) + jsonPrint(cmd, safepoint) } func deleteSSP(cmd *cobra.Command, args []string) { @@ -76,12 +60,10 @@ func deleteSSP(cmd *cobra.Command, args []string) { cmd.Usage() return } - serviceID := args[0] - deleteURL := serviceGCSafepointPrefix + "/" + serviceID - r, err := doRequest(cmd, deleteURL, http.MethodDelete, http.Header{}) + r, err := PDCli.DeleteGCSafePoint(cmd.Context(), args[0]) if err != nil { cmd.Printf("Failed to delete service GC safepoint: %s\n", err) return } - cmd.Println(r) + jsonPrint(cmd, r) } diff --git a/tools/pd-ctl/tests/safepoint/safepoint_test.go b/tools/pd-ctl/tests/safepoint/safepoint_test.go new file mode 100644 index 00000000000..5551cce1fff --- /dev/null +++ b/tools/pd-ctl/tests/safepoint/safepoint_test.go @@ -0,0 +1,138 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package safepoint_test + +import ( + "context" + "encoding/json" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/server/api" + pdTests "github.com/tikv/pd/tests" + ctl "github.com/tikv/pd/tools/pd-ctl/pdctl" + "github.com/tikv/pd/tools/pd-ctl/tests" +) + +func TestSafepoint(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tc, err := pdTests.NewTestCluster(ctx, 3) + re.NoError(err) + defer tc.Destroy() + err = tc.RunInitialServers() + re.NoError(err) + tc.WaitLeader() + leaderServer := tc.GetLeaderServer() + re.NoError(leaderServer.BootstrapCluster()) + pdAddr := tc.GetConfig().GetClientURL() + cmd := ctl.GetRootCmd() + + // add some gc_safepoint to the server + list := &api.ListServiceGCSafepoint{ + ServiceGCSafepoints: []*endpoint.ServiceSafePoint{ + { + ServiceID: "AAA", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 1, + }, + { + ServiceID: "BBB", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 2, + }, + { + ServiceID: "CCC", + ExpiredAt: time.Now().Unix() + 10, + SafePoint: 3, + }, + }, + GCSafePoint: 1, + MinServiceGcSafepoint: 1, + } + + storage := leaderServer.GetServer().GetStorage() + for _, ssp := range list.ServiceGCSafepoints { + err := storage.SaveServiceGCSafePoint(ssp) + re.NoError(err) + } + storage.SaveGCSafePoint(1) + + // get the safepoints + args := []string{"-u", pdAddr, "service-gc-safepoint"} + output, err := tests.ExecuteCommand(cmd, args...) + re.NoError(err) + + // create an container to hold the received values + var l api.ListServiceGCSafepoint + re.NoError(json.Unmarshal(output, &l)) + + // test if the points are what we expected + re.Equal(uint64(1), l.GCSafePoint) + re.Equal(uint64(1), l.MinServiceGcSafepoint) + re.Len(l.ServiceGCSafepoints, 3) + + // sort the gc safepoints based on order of ServiceID + sort.Slice(l.ServiceGCSafepoints, func(i, j int) bool { + return l.ServiceGCSafepoints[i].ServiceID < l.ServiceGCSafepoints[j].ServiceID + }) + + for i, val := range l.ServiceGCSafepoints { + re.Equal(list.ServiceGCSafepoints[i].ServiceID, val.ServiceID) + re.Equal(list.ServiceGCSafepoints[i].SafePoint, val.SafePoint) + } + + // delete the safepoints + for i := 0; i < 3; i++ { + args = []string{"-u", pdAddr, "service-gc-safepoint", "delete", list.ServiceGCSafepoints[i].ServiceID} + output, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + var msg string + re.NoError(json.Unmarshal(output, &msg)) + re.Equal("Delete service GC safepoint successfully.", msg) + } + + // do a second round of get safepoints to ensure that the safe points are indeed deleted + args = []string{"-u", pdAddr, "service-gc-safepoint"} + output, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + + var ll api.ListServiceGCSafepoint + re.NoError(json.Unmarshal(output, &ll)) + + re.Equal(uint64(1), ll.GCSafePoint) + re.Equal(uint64(0), ll.MinServiceGcSafepoint) + re.Empty(ll.ServiceGCSafepoints) + + // try delete the "gc_worker", should get an error message + args = []string{"-u", pdAddr, "service-gc-safepoint", "delete", "gc_worker"} + output, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + + // output should be an error message + re.Equal("Failed to delete service GC safepoint: request pd http api failed with status: '500 Internal Server Error', body: '\"cannot remove service safe point of gc_worker\"'\n", string(output)) + + // try delete a non-exist safepoint, should return normally + args = []string{"-u", pdAddr, "service-gc-safepoint", "delete", "non_exist"} + output, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + var msg string + re.NoError(json.Unmarshal(output, &msg)) + re.Equal("Delete service GC safepoint successfully.", msg) +} From cbb5a5bfd549d5a80891ce88e40aba3d499f3422 Mon Sep 17 00:00:00 2001 From: Hu# Date: Wed, 21 Aug 2024 15:45:12 +0800 Subject: [PATCH 25/42] ms: change the default name of the microservice to lowercase (#8551) close tikv/pd#8550 ms: change the default name of the microservice to lowercase Signed-off-by: husharp --- pkg/mcs/resourcemanager/server/config.go | 2 +- pkg/mcs/scheduling/server/config/config.go | 2 +- pkg/mcs/tso/server/config.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/mcs/resourcemanager/server/config.go b/pkg/mcs/resourcemanager/server/config.go index 9a899c9dc07..47f07486c1d 100644 --- a/pkg/mcs/resourcemanager/server/config.go +++ b/pkg/mcs/resourcemanager/server/config.go @@ -35,7 +35,7 @@ import ( ) const ( - defaultName = "Resource Manager" + defaultName = "resource manager" defaultBackendEndpoints = "http://127.0.0.1:2379" defaultListenAddr = "http://127.0.0.1:3379" diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index 4d7cee91f4a..cca2651d99e 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -47,7 +47,7 @@ import ( ) const ( - defaultName = "Scheduling" + defaultName = "scheduling" defaultBackendEndpoints = "http://127.0.0.1:2379" defaultListenAddr = "http://127.0.0.1:3379" ) diff --git a/pkg/mcs/tso/server/config.go b/pkg/mcs/tso/server/config.go index 8a3fe1ca161..209a9deb949 100644 --- a/pkg/mcs/tso/server/config.go +++ b/pkg/mcs/tso/server/config.go @@ -37,7 +37,7 @@ import ( const ( defaultMaxResetTSGap = 24 * time.Hour - defaultName = "TSO" + defaultName = "tso" defaultBackendEndpoints = "http://127.0.0.1:2379" defaultListenAddr = "http://127.0.0.1:3379" From b132ea64d4f5fb5a53fa3fa18e5ed7c796e4d4ae Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 22 Aug 2024 10:58:44 +0800 Subject: [PATCH 26/42] api: remove duplicate `/debug/pprof/trace` register (#8554) close tikv/pd#8553 Signed-off-by: lhy1024 --- server/util.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/util.go b/server/util.go index 83455e2a6fe..b80a07ab28a 100644 --- a/server/util.go +++ b/server/util.go @@ -17,7 +17,6 @@ package server import ( "context" "net/http" - "net/http/pprof" "path/filepath" "strings" @@ -133,10 +132,6 @@ func combineBuilderServerHTTPService(ctx context.Context, svr *Server, serviceBu apiService.UseHandler(router) userHandlers[pdAPIPrefix] = apiService - - // fix issue https://github.com/tikv/pd/issues/7253 - // FIXME: remove me after upgrade - userHandlers["/debug/pprof/trace"] = http.HandlerFunc(pprof.Trace) return userHandlers, nil } From c963e778efb19ca189765116699e97619fb05020 Mon Sep 17 00:00:00 2001 From: Hu# Date: Thu, 22 Aug 2024 11:09:12 +0800 Subject: [PATCH 27/42] tests/resource_control: enable trace log to help diagnose (#8545) ref tikv/pd#8512 enable trace log to help diagnose Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/resourcemanager/server/config.go | 3 ++- .../resourcemanager/resource_manager_test.go | 17 +++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/mcs/resourcemanager/server/config.go b/pkg/mcs/resourcemanager/server/config.go index 47f07486c1d..360f3b169b9 100644 --- a/pkg/mcs/resourcemanager/server/config.go +++ b/pkg/mcs/resourcemanager/server/config.go @@ -127,8 +127,9 @@ func (rmc *ControllerConfig) Adjust(meta *configutil.ConfigMetaData) { if !meta.IsDefined("ltb-token-rpc-max-delay") { configutil.AdjustDuration(&rmc.LTBTokenRPCMaxDelay, defaultLTBTokenRPCMaxDelay) } - failpoint.Inject("enableDegradedMode", func() { + failpoint.Inject("enableDegradedModeAndTraceLog", func() { configutil.AdjustDuration(&rmc.DegradedModeWaitDuration, time.Second) + configutil.AdjustBool(&rmc.EnableControllerTraceLog, true) }) } diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index 10b1a0b4520..4b2e2040ab0 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/failpoint" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/log" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" pd "github.com/tikv/pd/client" @@ -37,6 +38,7 @@ import ( "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/tests" "go.uber.org/goleak" + "go.uber.org/zap" // Register Service _ "github.com/tikv/pd/pkg/mcs/registry" @@ -64,7 +66,7 @@ func (suite *resourceManagerClientTestSuite) SetupSuite() { var err error re := suite.Require() - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedMode", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedModeAndTraceLog", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) suite.ctx, suite.clean = context.WithCancel(context.Background()) @@ -151,7 +153,7 @@ func (suite *resourceManagerClientTestSuite) TearDownSuite() { suite.client.Close() suite.cluster.Destroy() suite.clean() - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedMode")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedModeAndTraceLog")) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) } @@ -410,11 +412,9 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { testCases := []struct { resourceGroupName string tcs []tokenConsumptionPerSecond - len int }{ { resourceGroupName: rg.Name, - len: 8, tcs: []tokenConsumptionPerSecond{ {rruTokensAtATime: 50, wruTokensAtATime: 20, times: 100, waitDuration: 0}, {rruTokensAtATime: 50, wruTokensAtATime: 100, times: 100, waitDuration: 0}, @@ -434,7 +434,7 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { v := false <-tricker.C for _, cas := range testCases { - if i >= cas.len { + if i >= len(cas.tcs) { continue } v = true @@ -452,8 +452,9 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { sum += time.Since(startTime) rgsController.OnResponse(cas.resourceGroupName, rreq, rres) rgsController.OnResponse(cas.resourceGroupName, wreq, wres) - time.Sleep(1000 * time.Microsecond) + time.Sleep(time.Millisecond) } + log.Info("finished test case", zap.Int("index", i), zap.Duration("sum", sum), zap.Duration("waitDuration", cas.tcs[i].waitDuration)) re.LessOrEqual(sum, buffDuration+cas.tcs[i].waitDuration) } i++ @@ -1479,12 +1480,12 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupControllerConfigCh readBaseCost := 1.5 defaultCfg := controller.DefaultConfig() expectCfg := server.ControllerConfig{ - // failpoint enableDegradedMode will setup and set it be 1s. + // failpoint enableDegradedModeAndTraceLog will set it be 1s and enable trace log. DegradedModeWaitDuration: typeutil.NewDuration(time.Second), + EnableControllerTraceLog: true, LTBMaxWaitDuration: typeutil.Duration(defaultCfg.LTBMaxWaitDuration), LTBTokenRPCMaxDelay: typeutil.Duration(defaultCfg.LTBTokenRPCMaxDelay), RequestUnit: server.RequestUnitConfig(defaultCfg.RequestUnit), - EnableControllerTraceLog: defaultCfg.EnableControllerTraceLog, } expectRUCfg := controller.GenerateRUConfig(defaultCfg) expectRUCfg.DegradedModeWaitDuration = time.Second From 32829f140db8a7b783668b8350e29d42c05c7aa4 Mon Sep 17 00:00:00 2001 From: Hu# Date: Thu, 22 Aug 2024 11:49:42 +0800 Subject: [PATCH 28/42] tests: make TestAllocIDAfterLeaderChange more stable (#8558) close tikv/pd#8557 need to wait leader Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/mcs/scheduling/server_test.go | 1 + tests/server/join/join_test.go | 2 ++ tests/server/watch/leader_watch_test.go | 1 + 3 files changed, 4 insertions(+) diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 57306640394..4a19321ce57 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -106,6 +106,7 @@ func (suite *serverTestSuite) TestAllocIDAfterLeaderChange() { pd2, err := suite.cluster.Join(suite.ctx) re.NoError(err) err = pd2.Run() + re.NotEmpty(suite.cluster.WaitLeader()) re.NoError(err) tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) re.NoError(err) diff --git a/tests/server/join/join_test.go b/tests/server/join/join_test.go index 1eaa59a4e2a..5d087caf5e4 100644 --- a/tests/server/join/join_test.go +++ b/tests/server/join/join_test.go @@ -58,6 +58,7 @@ func TestSimpleJoin(t *testing.T) { re.NoError(err) _, err = os.Stat(filepath.Join(pd2.GetConfig().DataDir, "join")) re.False(os.IsNotExist(err)) + re.NotEmpty(cluster.WaitLeader()) members, err = etcdutil.ListEtcdMembers(ctx, client) re.NoError(err) re.Len(members.Members, 2) @@ -73,6 +74,7 @@ func TestSimpleJoin(t *testing.T) { re.NoError(err) _, err = os.Stat(filepath.Join(pd3.GetConfig().DataDir, "join")) re.False(os.IsNotExist(err)) + re.NotEmpty(cluster.WaitLeader()) members, err = etcdutil.ListEtcdMembers(ctx, client) re.NoError(err) re.Len(members.Members, 3) diff --git a/tests/server/watch/leader_watch_test.go b/tests/server/watch/leader_watch_test.go index 863cf7a0c08..39c54b5c500 100644 --- a/tests/server/watch/leader_watch_test.go +++ b/tests/server/watch/leader_watch_test.go @@ -57,6 +57,7 @@ func TestWatcher(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayWatcher", `pause`)) err = pd3.Run() re.NoError(err) + re.NotEmpty(cluster.WaitLeader()) time.Sleep(200 * time.Millisecond) re.Equal(pd1.GetConfig().Name, pd3.GetLeader().GetName()) err = pd1.Stop() From 8e45f9e956436cd0f3924ccadebacbb7fa2def3a Mon Sep 17 00:00:00 2001 From: Hu# Date: Thu, 22 Aug 2024 12:01:43 +0800 Subject: [PATCH 29/42] tests: unify frequency check time to make test stable (#8556) close tikv/pd#8555 tests: unify frequency check time to make test stable Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/member/member.go | 8 +------- tests/integrations/client/client_test.go | 8 ++------ tests/integrations/client/http_client_test.go | 4 ++++ tests/integrations/mcs/scheduling/server_test.go | 4 ++-- tests/server/cluster/cluster_test.go | 8 ++++---- tests/server/id/id_test.go | 4 ++-- tests/server/region_syncer/region_syncer_test.go | 4 ++-- 7 files changed, 17 insertions(+), 23 deletions(-) diff --git a/pkg/member/member.go b/pkg/member/member.go index 8bf50e6d310..2dc8be52031 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -187,13 +187,7 @@ func (m *EmbeddedEtcdMember) CampaignLeader(ctx context.Context, leaseTimeout in failpoint.Return(m.leadership.Campaign(leaseTimeout, m.MemberValue())) }) - checkTimes := campaignLeaderFrequencyTimes - failpoint.Inject("changeFrequencyTimes", func(val failpoint.Value) { - if v, ok := val.(int); ok { - checkTimes = v - } - }) - if m.leadership.GetCampaignTimesNum() > checkTimes { + if m.leadership.GetCampaignTimesNum() > campaignLeaderFrequencyTimes { if err := m.ResignEtcdLeader(ctx, m.Name(), ""); err != nil { return err } diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 9517444d017..3c8bba1fafd 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -166,19 +166,15 @@ func TestClientLeaderChange(t *testing.T) { func TestLeaderTransferAndMoveCluster(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() cluster, err := tests.NewTestCluster(ctx, 3) re.NoError(err) defer cluster.Destroy() - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) - defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) - }() endpoints := runServer(re, cluster) cli := setupCli(ctx, re, endpoints) diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index fe0962012e6..68643d5f0c8 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" @@ -71,6 +72,7 @@ func TestHTTPClientTestSuiteWithServiceDiscovery(t *testing.T) { func (suite *httpClientTestSuite) SetupSuite() { re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) suite.ctx, suite.cancelFunc = context.WithCancel(context.Background()) cluster, err := tests.NewTestCluster(suite.ctx, 2) @@ -125,6 +127,8 @@ func (suite *httpClientTestSuite) SetupSuite() { } func (suite *httpClientTestSuite) TearDownSuite() { + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) suite.cancelFunc() suite.client.Close() suite.cluster.Destroy() diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 4a19321ce57..085b87afe86 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -654,9 +654,9 @@ func (suite *multipleServerTestSuite) TearDownSuite() { func (suite *multipleServerTestSuite) TestReElectLeader() { re := suite.Require() - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) re.NoError(err) diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 375edf027db..a1a7f68646e 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -187,9 +187,9 @@ func TestDamagedRegion(t *testing.T) { func TestRegionStatistics(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1650,9 +1650,9 @@ func TestMinResolvedTS(t *testing.T) { // See https://github.com/tikv/pd/issues/4941 func TestTransferLeaderBack(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/tests/server/id/id_test.go b/tests/server/id/id_test.go index c7dee0d2924..8b0e7ec60b7 100644 --- a/tests/server/id/id_test.go +++ b/tests/server/id/id_test.go @@ -108,9 +108,9 @@ func TestCommand(t *testing.T) { func TestMonotonicID(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/tests/server/region_syncer/region_syncer_test.go b/tests/server/region_syncer/region_syncer_test.go index 6a5c1ea361c..80f5186f904 100644 --- a/tests/server/region_syncer/region_syncer_test.go +++ b/tests/server/region_syncer/region_syncer_test.go @@ -255,9 +255,9 @@ func TestPrepareChecker(t *testing.T) { // ref: https://github.com/tikv/pd/issues/6988 func TestPrepareCheckerWithTransferLeader(t *testing.T) { re := require.New(t) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/changeFrequencyTimes", "return(10)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/changeFrequencyTimes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 84138e4b876908274f68cedbb4c5e4c57472ee34 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Thu, 22 Aug 2024 15:27:13 +0800 Subject: [PATCH 30/42] heartbeat: reduce unnecessary `RunTask` (#8559) ref tikv/pd#7897 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/cluster/cluster.go | 32 +++++++++++++--------------- pkg/mcs/scheduling/server/cluster.go | 21 ++++++++++-------- server/cluster/cluster.go | 26 +++++++++++----------- 3 files changed, 41 insertions(+), 38 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index ddba8f89fb6..b1abe4677df 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -74,22 +74,20 @@ func HandleOverlaps(ctx context.Context, c Cluster, overlaps []*core.RegionInfo) } // Collect collects the cluster information. -func Collect(ctx context.Context, c Cluster, region *core.RegionInfo, hasRegionStats bool) { - if hasRegionStats { - // get region again from root tree. make sure the observed region is the latest. - bc := c.GetBasicCluster() - if bc == nil { - return - } - region = bc.GetRegion(region.GetID()) - if region == nil { - return - } - select { - case <-ctx.Done(): - return - default: - } - c.GetRegionStats().Observe(region, c.GetBasicCluster().GetRegionStores(region)) +func Collect(ctx context.Context, c Cluster, region *core.RegionInfo) { + // get region again from root tree. make sure the observed region is the latest. + bc := c.GetBasicCluster() + if bc == nil { + return + } + region = bc.GetRegion(region.GetID()) + if region == nil { + return + } + select { + case <-ctx.Done(): + return + default: } + c.GetRegionStats().Observe(region, c.GetBasicCluster().GetRegionStores(region)) } diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 31120ae6082..0dcb26a1a1f 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -631,7 +631,7 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c regionID, ratelimit.ObserveRegionStatsAsync, func(ctx context.Context) { - cluster.Collect(ctx, c, region, hasRegionStats) + cluster.Collect(ctx, c, region) }, ) } @@ -679,14 +679,17 @@ func (c *Cluster) processRegionHeartbeat(ctx *core.MetaProcessContext, region *c ) } tracer.OnSaveCacheFinished() - // handle region stats - ctx.TaskRunner.RunTask( - regionID, - ratelimit.CollectRegionStatsAsync, - func(ctx context.Context) { - cluster.Collect(ctx, c, region, hasRegionStats) - }, - ) + if hasRegionStats { + // handle region stats + ctx.TaskRunner.RunTask( + regionID, + ratelimit.CollectRegionStatsAsync, + func(ctx context.Context) { + cluster.Collect(ctx, c, region) + }, + ) + } + tracer.OnCollectRegionStatsFinished() return nil } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 9e12b158f42..9e85a0f1d84 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1062,7 +1062,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio regionID, ratelimit.ObserveRegionStatsAsync, func(ctx context.Context) { - cluster.Collect(ctx, c, region, hasRegionStats) + cluster.Collect(ctx, c, region) }, ) } @@ -1119,17 +1119,19 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio } tracer.OnSaveCacheFinished() - // handle region stats - ctx.MiscRunner.RunTask( - regionID, - ratelimit.CollectRegionStatsAsync, - func(ctx context.Context) { - // TODO: Due to the accuracy requirements of the API "/regions/check/xxx", - // region stats needs to be collected in API mode. - // We need to think of a better way to reduce this part of the cost in the future. - cluster.Collect(ctx, c, region, hasRegionStats) - }, - ) + if hasRegionStats { + // handle region stats + ctx.MiscRunner.RunTask( + regionID, + ratelimit.CollectRegionStatsAsync, + func(ctx context.Context) { + // TODO: Due to the accuracy requirements of the API "/regions/check/xxx", + // region stats needs to be collected in API mode. + // We need to think of a better way to reduce this part of the cost in the future. + cluster.Collect(ctx, c, region) + }, + ) + } tracer.OnCollectRegionStatsFinished() if c.storage != nil { From 9e7a1b975d4ca63baffb23de80396cb324f64095 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Thu, 22 Aug 2024 15:52:42 +0800 Subject: [PATCH 31/42] *: make TestStores stable (#8522) close tikv/pd#7954 add a failpoint to make sure the offline store do not convert to tombstone Signed-off-by: okJiang <819421878@qq.com> --- server/cluster/cluster.go | 7 +++++-- tests/integrations/mcs/scheduling/api_test.go | 10 +++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 9e85a0f1d84..fdeffd15e9c 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1669,9 +1669,12 @@ func (c *RaftCluster) checkStores() { if c.IsPrepared() { c.updateProgress(id, store.GetAddress(), removingAction, float64(regionSize), float64(regionSize), false /* dec */) } - regionCount := c.GetStoreRegionCount(id) // If the store is empty, it can be buried. - if regionCount == 0 { + needBury := c.GetStoreRegionCount(id) == 0 + failpoint.Inject("doNotBuryStore", func(_ failpoint.Value) { + needBury = false + }) + if needBury { if err := c.BuryStore(id, false); err != nil { log.Error("bury store failed", zap.Stringer("store", offlineStore), diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 68f276f9e22..d2027a616cb 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -629,13 +629,13 @@ func (suite *apiTestSuite) checkStores(cluster *tests.TestCluster) { Version: "2.0.0", }, } - // prevent the offline store from changing to tombstone - tests.MustPutRegion(re, cluster, 3, 6, []byte("a"), []byte("b")) + + re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/doNotBuryStore", `return(true)`)) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/doNotBuryStore")) + }() for _, store := range stores { tests.MustPutStore(re, cluster, store) - if store.GetId() == 6 { - cluster.GetLeaderServer().GetRaftCluster().GetBasicCluster().UpdateStoreStatus(6) - } } // Test /stores apiServerAddr := cluster.GetLeaderServer().GetAddr() From 61a85e52ecb851008953bb4efb4939681f2b1ade Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Thu, 22 Aug 2024 16:52:42 +0800 Subject: [PATCH 32/42] *: enable revive.exported linter (#8563) close tikv/pd#8458 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- .golangci.yml | 3 +- tools/pd-api-bench/cases/cases.go | 68 ++++++++-------- tools/pd-api-bench/cases/controller.go | 64 +++++++-------- tools/pd-ctl/pdctl/command/global.go | 1 + tools/pd-heartbeat-bench/metrics/util.go | 19 +++-- tools/pd-simulator/simulator/client.go | 78 +++++++++++-------- tools/pd-simulator/simulator/config/config.go | 2 + tools/pd-simulator/simulator/drive.go | 17 ++-- tools/pd-simulator/simulator/event.go | 4 +- tools/pd-simulator/simulator/raft.go | 2 +- tools/pd-tso-bench/main.go | 10 +-- tools/pd-ut/alloc/server.go | 1 + 12 files changed, 145 insertions(+), 124 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index e09011c45be..e5b5fe1ab56 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -226,5 +226,4 @@ issues: linters: - errcheck include: - # remove the comment after the path is ready - # - EXC0012 + - EXC0012 diff --git a/tools/pd-api-bench/cases/cases.go b/tools/pd-api-bench/cases/cases.go index d81e7e0fa9d..f863d3248c6 100644 --- a/tools/pd-api-bench/cases/cases.go +++ b/tools/pd-api-bench/cases/cases.go @@ -80,12 +80,12 @@ func (c *Config) Clone() *Config { // Case is the interface for all cases. type Case interface { - Name() string - SetQPS(int64) - GetQPS() int64 - SetBurst(int64) - GetBurst() int64 - GetConfig() *Config + getName() string + setQPS(int64) + getQPS() int64 + setBurst(int64) + getBurst() int64 + getConfig() *Config } type baseCase struct { @@ -93,35 +93,35 @@ type baseCase struct { cfg *Config } -func (c *baseCase) Name() string { +func (c *baseCase) getName() string { return c.name } -func (c *baseCase) SetQPS(qps int64) { +func (c *baseCase) setQPS(qps int64) { c.cfg.QPS = qps } -func (c *baseCase) GetQPS() int64 { +func (c *baseCase) getQPS() int64 { return c.cfg.QPS } -func (c *baseCase) SetBurst(burst int64) { +func (c *baseCase) setBurst(burst int64) { c.cfg.Burst = burst } -func (c *baseCase) GetBurst() int64 { +func (c *baseCase) getBurst() int64 { return c.cfg.Burst } -func (c *baseCase) GetConfig() *Config { +func (c *baseCase) getConfig() *Config { return c.cfg.Clone() } // EtcdCase is the interface for all etcd api cases. type EtcdCase interface { Case - Init(context.Context, *clientv3.Client) error - Unary(context.Context, *clientv3.Client) error + init(context.Context, *clientv3.Client) error + unary(context.Context, *clientv3.Client) error } // EtcdCreateFn is function type to create EtcdCase. @@ -138,7 +138,7 @@ var EtcdCaseFnMap = map[string]EtcdCreateFn{ // GRPCCase is the interface for all gRPC cases. type GRPCCase interface { Case - Unary(context.Context, pd.Client) error + unary(context.Context, pd.Client) error } // GRPCCreateFn is function type to create GRPCCase. @@ -159,7 +159,7 @@ var GRPCCaseFnMap = map[string]GRPCCreateFn{ // HTTPCase is the interface for all HTTP cases. type HTTPCase interface { Case - Do(context.Context, pdHttp.Client) error + do(context.Context, pdHttp.Client) error } // HTTPCreateFn is function type to create HTTPCase. @@ -186,7 +186,7 @@ func newMinResolvedTS() func() HTTPCase { } } -func (c *minResolvedTS) Do(ctx context.Context, cli pdHttp.Client) error { +func (c *minResolvedTS) do(ctx context.Context, cli pdHttp.Client) error { minResolvedTS, storesMinResolvedTS, err := cli.GetMinResolvedTSByStoresIDs(ctx, storesID) if Debug { log.Info("do HTTP case", zap.String("case", c.name), zap.Uint64("min-resolved-ts", minResolvedTS), zap.Any("store-min-resolved-ts", storesMinResolvedTS), zap.Error(err)) @@ -214,7 +214,7 @@ func newRegionStats() func() HTTPCase { } } -func (c *regionsStats) Do(ctx context.Context, cli pdHttp.Client) error { +func (c *regionsStats) do(ctx context.Context, cli pdHttp.Client) error { upperBound := totalRegion / c.regionSample if upperBound < 1 { upperBound = 1 @@ -248,7 +248,7 @@ func newUpdateGCSafePoint() func() GRPCCase { } } -func (*updateGCSafePoint) Unary(ctx context.Context, cli pd.Client) error { +func (*updateGCSafePoint) unary(ctx context.Context, cli pd.Client) error { s := time.Now().Unix() _, err := cli.UpdateGCSafePoint(ctx, uint64(s)) if err != nil { @@ -272,7 +272,7 @@ func newUpdateServiceGCSafePoint() func() GRPCCase { } } -func (*updateServiceGCSafePoint) Unary(ctx context.Context, cli pd.Client) error { +func (*updateServiceGCSafePoint) unary(ctx context.Context, cli pd.Client) error { s := time.Now().Unix() id := rand.Int63n(100) + 1 _, err := cli.UpdateServiceGCSafePoint(ctx, strconv.FormatInt(id, 10), id, uint64(s)) @@ -297,7 +297,7 @@ func newGetRegion() func() GRPCCase { } } -func (*getRegion) Unary(ctx context.Context, cli pd.Client) error { +func (*getRegion) unary(ctx context.Context, cli pd.Client) error { id := rand.Intn(totalRegion)*4 + 1 _, err := cli.GetRegion(ctx, generateKeyForSimulator(id)) if err != nil { @@ -321,7 +321,7 @@ func newGetRegionEnableFollower() func() GRPCCase { } } -func (*getRegionEnableFollower) Unary(ctx context.Context, cli pd.Client) error { +func (*getRegionEnableFollower) unary(ctx context.Context, cli pd.Client) error { id := rand.Intn(totalRegion)*4 + 1 _, err := cli.GetRegion(ctx, generateKeyForSimulator(id), pd.WithAllowFollowerHandle()) if err != nil { @@ -347,7 +347,7 @@ func newScanRegions() func() GRPCCase { } } -func (c *scanRegions) Unary(ctx context.Context, cli pd.Client) error { +func (c *scanRegions) unary(ctx context.Context, cli pd.Client) error { upperBound := totalRegion / c.regionSample random := rand.Intn(upperBound) startID := c.regionSample*random*4 + 1 @@ -375,7 +375,7 @@ func newTso() func() GRPCCase { } } -func (*tso) Unary(ctx context.Context, cli pd.Client) error { +func (*tso) unary(ctx context.Context, cli pd.Client) error { _, _, err := cli.GetTS(ctx) if err != nil { return err @@ -398,7 +398,7 @@ func newGetStore() func() GRPCCase { } } -func (*getStore) Unary(ctx context.Context, cli pd.Client) error { +func (*getStore) unary(ctx context.Context, cli pd.Client) error { storeIdx := rand.Intn(totalStore) _, err := cli.GetStore(ctx, storesID[storeIdx]) if err != nil { @@ -422,7 +422,7 @@ func newGetStores() func() GRPCCase { } } -func (*getStores) Unary(ctx context.Context, cli pd.Client) error { +func (*getStores) unary(ctx context.Context, cli pd.Client) error { _, err := cli.GetAllStores(ctx) if err != nil { return err @@ -451,7 +451,7 @@ func newGetKV() func() EtcdCase { } } -func (*getKV) Init(ctx context.Context, cli *clientv3.Client) error { +func (*getKV) init(ctx context.Context, cli *clientv3.Client) error { for i := 0; i < 100; i++ { _, err := cli.Put(ctx, fmt.Sprintf("/test/0001/%4d", i), fmt.Sprintf("%4d", i)) if err != nil { @@ -461,7 +461,7 @@ func (*getKV) Init(ctx context.Context, cli *clientv3.Client) error { return nil } -func (*getKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*getKV) unary(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Get(ctx, "/test/0001", clientv3.WithPrefix()) return err } @@ -481,9 +481,9 @@ func newPutKV() func() EtcdCase { } } -func (*putKV) Init(context.Context, *clientv3.Client) error { return nil } +func (*putKV) init(context.Context, *clientv3.Client) error { return nil } -func (*putKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*putKV) unary(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Put(ctx, "/test/0001/0000", "test") return err } @@ -503,9 +503,9 @@ func newDeleteKV() func() EtcdCase { } } -func (*deleteKV) Init(context.Context, *clientv3.Client) error { return nil } +func (*deleteKV) init(context.Context, *clientv3.Client) error { return nil } -func (*deleteKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*deleteKV) unary(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Delete(ctx, "/test/0001/0000") return err } @@ -525,9 +525,9 @@ func newTxnKV() func() EtcdCase { } } -func (*txnKV) Init(context.Context, *clientv3.Client) error { return nil } +func (*txnKV) init(context.Context, *clientv3.Client) error { return nil } -func (*txnKV) Unary(ctx context.Context, cli *clientv3.Client) error { +func (*txnKV) unary(ctx context.Context, cli *clientv3.Client) error { txn := cli.Txn(ctx) txn = txn.If(clientv3.Compare(clientv3.Value("/test/0001/0000"), "=", "test")) txn = txn.Then(clientv3.OpPut("/test/0001/0000", "test2")) diff --git a/tools/pd-api-bench/cases/controller.go b/tools/pd-api-bench/cases/controller.go index e19c79c0f3e..a77474db3a7 100644 --- a/tools/pd-api-bench/cases/controller.go +++ b/tools/pd-api-bench/cases/controller.go @@ -62,7 +62,7 @@ func (c *Coordinator) GetHTTPCase(name string) (*Config, error) { c.mu.RLock() defer c.mu.RUnlock() if controller, ok := c.http[name]; ok { - return controller.GetConfig(), nil + return controller.getConfig(), nil } return nil, errors.Errorf("case %v does not exist", name) } @@ -72,7 +72,7 @@ func (c *Coordinator) GetGRPCCase(name string) (*Config, error) { c.mu.RLock() defer c.mu.RUnlock() if controller, ok := c.grpc[name]; ok { - return controller.GetConfig(), nil + return controller.getConfig(), nil } return nil, errors.Errorf("case %v does not exist", name) } @@ -82,7 +82,7 @@ func (c *Coordinator) GetEtcdCase(name string) (*Config, error) { c.mu.RLock() defer c.mu.RUnlock() if controller, ok := c.etcd[name]; ok { - return controller.GetConfig(), nil + return controller.getConfig(), nil } return nil, errors.Errorf("case %v does not exist", name) } @@ -93,7 +93,7 @@ func (c *Coordinator) GetAllHTTPCases() map[string]*Config { defer c.mu.RUnlock() ret := make(map[string]*Config) for name, c := range c.http { - ret[name] = c.GetConfig() + ret[name] = c.getConfig() } return ret } @@ -104,7 +104,7 @@ func (c *Coordinator) GetAllGRPCCases() map[string]*Config { defer c.mu.RUnlock() ret := make(map[string]*Config) for name, c := range c.grpc { - ret[name] = c.GetConfig() + ret[name] = c.getConfig() } return ret } @@ -115,7 +115,7 @@ func (c *Coordinator) GetAllEtcdCases() map[string]*Config { defer c.mu.RUnlock() ret := make(map[string]*Config) for name, c := range c.etcd { - ret[name] = c.GetConfig() + ret[name] = c.getConfig() } return ret } @@ -131,9 +131,9 @@ func (c *Coordinator) SetHTTPCase(name string, cfg *Config) error { c.http[name] = controller } controller.stop() - controller.SetQPS(cfg.QPS) + controller.setQPS(cfg.QPS) if cfg.Burst > 0 { - controller.SetBurst(cfg.Burst) + controller.setBurst(cfg.Burst) } controller.run() } else { @@ -153,9 +153,9 @@ func (c *Coordinator) SetGRPCCase(name string, cfg *Config) error { c.grpc[name] = controller } controller.stop() - controller.SetQPS(cfg.QPS) + controller.setQPS(cfg.QPS) if cfg.Burst > 0 { - controller.SetBurst(cfg.Burst) + controller.setBurst(cfg.Burst) } controller.run() } else { @@ -175,9 +175,9 @@ func (c *Coordinator) SetEtcdCase(name string, cfg *Config) error { c.etcd[name] = controller } controller.stop() - controller.SetQPS(cfg.QPS) + controller.setQPS(cfg.QPS) if cfg.Burst > 0 { - controller.SetBurst(cfg.Burst) + controller.setBurst(cfg.Burst) } controller.run() } else { @@ -207,15 +207,15 @@ func newHTTPController(ctx context.Context, clis []pdHttp.Client, fn HTTPCreateF // run tries to run the HTTP api bench. func (c *httpController) run() { - if c.GetQPS() <= 0 || c.cancel != nil { + if c.getQPS() <= 0 || c.cancel != nil { return } c.ctx, c.cancel = context.WithCancel(c.pctx) - qps := c.GetQPS() - burst := c.GetBurst() + qps := c.getQPS() + burst := c.getBurst() cliNum := int64(len(c.clients)) tt := time.Duration(base*burst*cliNum/qps) * time.Microsecond - log.Info("begin to run http case", zap.String("case", c.Name()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) + log.Info("begin to run http case", zap.String("case", c.getName()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) for _, hCli := range c.clients { c.wg.Add(1) go func(hCli pdHttp.Client) { @@ -229,9 +229,9 @@ func (c *httpController) run() { for { select { case <-ticker.C: - err := c.Do(c.ctx, hCli) + err := c.do(c.ctx, hCli) if err != nil { - log.Error("meet error when doing HTTP request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing HTTP request", zap.String("case", c.getName()), zap.Error(err)) } case <-c.ctx.Done(): log.Info("got signal to exit running HTTP case") @@ -276,15 +276,15 @@ func newGRPCController(ctx context.Context, clis []pd.Client, fn GRPCCreateFn) * // run tries to run the gRPC api bench. func (c *gRPCController) run() { - if c.GetQPS() <= 0 || c.cancel != nil { + if c.getQPS() <= 0 || c.cancel != nil { return } c.ctx, c.cancel = context.WithCancel(c.pctx) - qps := c.GetQPS() - burst := c.GetBurst() + qps := c.getQPS() + burst := c.getBurst() cliNum := int64(len(c.clients)) tt := time.Duration(base*burst*cliNum/qps) * time.Microsecond - log.Info("begin to run gRPC case", zap.String("case", c.Name()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) + log.Info("begin to run gRPC case", zap.String("case", c.getName()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) for _, cli := range c.clients { c.wg.Add(1) go func(cli pd.Client) { @@ -298,9 +298,9 @@ func (c *gRPCController) run() { for { select { case <-ticker.C: - err := c.Unary(c.ctx, cli) + err := c.unary(c.ctx, cli) if err != nil { - log.Error("meet error when doing gRPC request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing gRPC request", zap.String("case", c.getName()), zap.Error(err)) } case <-c.ctx.Done(): log.Info("got signal to exit running gRPC case") @@ -345,18 +345,18 @@ func newEtcdController(ctx context.Context, clis []*clientv3.Client, fn EtcdCrea // run tries to run the gRPC api bench. func (c *etcdController) run() { - if c.GetQPS() <= 0 || c.cancel != nil { + if c.getQPS() <= 0 || c.cancel != nil { return } c.ctx, c.cancel = context.WithCancel(c.pctx) - qps := c.GetQPS() - burst := c.GetBurst() + qps := c.getQPS() + burst := c.getBurst() cliNum := int64(len(c.clients)) tt := time.Duration(base*burst*cliNum/qps) * time.Microsecond - log.Info("begin to run etcd case", zap.String("case", c.Name()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) - err := c.Init(c.ctx, c.clients[0]) + log.Info("begin to run etcd case", zap.String("case", c.getName()), zap.Int64("qps", qps), zap.Int64("burst", burst), zap.Duration("interval", tt)) + err := c.init(c.ctx, c.clients[0]) if err != nil { - log.Error("init error", zap.String("case", c.Name()), zap.Error(err)) + log.Error("init error", zap.String("case", c.getName()), zap.Error(err)) return } for _, cli := range c.clients { @@ -372,9 +372,9 @@ func (c *etcdController) run() { for { select { case <-ticker.C: - err := c.Unary(c.ctx, cli) + err := c.unary(c.ctx, cli) if err != nil { - log.Error("meet error when doing etcd request", zap.String("case", c.Name()), zap.Error(err)) + log.Error("meet error when doing etcd request", zap.String("case", c.getName()), zap.Error(err)) } case <-c.ctx.Done(): log.Info("got signal to exit running etcd case") diff --git a/tools/pd-ctl/pdctl/command/global.go b/tools/pd-ctl/pdctl/command/global.go index c5c633de217..c7e0cfd691d 100644 --- a/tools/pd-ctl/pdctl/command/global.go +++ b/tools/pd-ctl/pdctl/command/global.go @@ -33,6 +33,7 @@ import ( ) const ( + // PDControlCallerID is used to set the caller ID for PD client PDControlCallerID = "pd-ctl" clusterPrefix = "pd/api/v1/cluster" ) diff --git a/tools/pd-heartbeat-bench/metrics/util.go b/tools/pd-heartbeat-bench/metrics/util.go index e409cf10815..f747e5b507f 100644 --- a/tools/pd-heartbeat-bench/metrics/util.go +++ b/tools/pd-heartbeat-bench/metrics/util.go @@ -32,12 +32,12 @@ import ( var ( prometheusCli api.Client - finalMetrics2Collect []Metric + finalMetrics2Collect []metric avgRegionStats report.Stats avgStoreTime float64 collectRound = 1.0 - metrics2Collect = []Metric{ + metrics2Collect = []metric{ {promSQL: cpuMetric, name: "max cpu usage(%)", max: true}, {promSQL: memoryMetric, name: "max memory usage(G)", max: true}, {promSQL: goRoutineMetric, name: "max go routines", max: true}, @@ -69,7 +69,7 @@ var ( } ) -type Metric struct { +type metric struct { promSQL string name string value float64 @@ -77,9 +77,10 @@ type Metric struct { max bool } +// InitMetric2Collect initializes the metrics to collect func InitMetric2Collect(endpoint string) (withMetric bool) { for _, name := range breakdownNames { - metrics2Collect = append(metrics2Collect, Metric{ + metrics2Collect = append(metrics2Collect, metric{ promSQL: hbBreakdownMetricByName(name), name: name, }) @@ -94,7 +95,7 @@ func InitMetric2Collect(endpoint string) (withMetric bool) { log.Error("parse prometheus url error", zap.Error(err)) return false } - prometheusCli, err = NewPrometheusClient(*cu) + prometheusCli, err = newPrometheusClient(*cu) if err != nil { log.Error("create prometheus client error", zap.Error(err)) return false @@ -108,7 +109,7 @@ func InitMetric2Collect(endpoint string) (withMetric bool) { return true } -func NewPrometheusClient(prometheusURL url.URL) (api.Client, error) { +func newPrometheusClient(prometheusURL url.URL) (api.Client, error) { client, err := api.NewClient(api.Config{ Address: prometheusURL.String(), }) @@ -122,6 +123,7 @@ func NewPrometheusClient(prometheusURL url.URL) (api.Client, error) { // WarmUpRound wait for the first round to warm up const WarmUpRound = 1 +// CollectMetrics collects the metrics func CollectMetrics(curRound int, wait time.Duration) { if curRound < WarmUpRound { return @@ -183,7 +185,7 @@ func getMetric(cli api.Client, query string, ts time.Time) ([]float64, error) { return value, nil } -func formatMetrics(ms []Metric) string { +func formatMetrics(ms []metric) string { res := "" for _, m := range ms { res += "[" + m.name + "]" + " " + fmt.Sprintf("%.10f", m.value) + " " @@ -191,6 +193,7 @@ func formatMetrics(ms []Metric) string { return res } +// CollectRegionAndStoreStats collects the region and store stats func CollectRegionAndStoreStats(regionStats *report.Stats, storeTime *float64) { if regionStats != nil && storeTime != nil { collect(*regionStats, *storeTime) @@ -211,6 +214,7 @@ func collect(regionStats report.Stats, storeTime float64) { avgStoreTime = average(avgStoreTime, storeTime) } +// OutputConclusion outputs the final conclusion func OutputConclusion() { logFields := RegionFields(avgRegionStats, zap.Float64("avg store time", avgStoreTime), @@ -219,6 +223,7 @@ func OutputConclusion() { log.Info("final metrics collected", logFields...) } +// RegionFields returns the fields for region stats func RegionFields(stats report.Stats, fields ...zap.Field) []zap.Field { return append([]zap.Field{ zap.String("total", fmt.Sprintf("%.4fs", stats.Total.Seconds())), diff --git a/tools/pd-simulator/simulator/client.go b/tools/pd-simulator/simulator/client.go index 8acf3ccd9ab..224d54b05d6 100644 --- a/tools/pd-simulator/simulator/client.go +++ b/tools/pd-simulator/simulator/client.go @@ -40,13 +40,13 @@ import ( // Client is a PD (Placement Driver) client. // It should not be used after calling Close(). type Client interface { - AllocID(context.Context) (uint64, error) + allocID(context.Context) (uint64, error) PutStore(context.Context, *metapb.Store) error StoreHeartbeat(context.Context, *pdpb.StoreStats) error RegionHeartbeat(context.Context, *core.RegionInfo) error - HeartbeatStreamLoop() - ChangeConn(*grpc.ClientConn) error + heartbeatStreamLoop() + changeConn(*grpc.ClientConn) error Close() } @@ -61,15 +61,17 @@ const ( var ( // errFailInitClusterID is returned when failed to load clusterID from all supplied PD addresses. errFailInitClusterID = errors.New("[pd] failed to get cluster id") - PDHTTPClient pdHttp.Client - SD pd.ServiceDiscovery - ClusterID atomic.Uint64 + // PDHTTPClient is a client for PD HTTP API. + PDHTTPClient pdHttp.Client + // SD is a service discovery for PD. + SD pd.ServiceDiscovery + clusterID atomic.Uint64 ) // requestHeader returns a header for fixed ClusterID. func requestHeader() *pdpb.RequestHeader { return &pdpb.RequestHeader{ - ClusterId: ClusterID.Load(), + ClusterId: clusterID.Load(), } } @@ -110,7 +112,7 @@ func createConn(url string) (*grpc.ClientConn, error) { return cc, nil } -func (c *client) ChangeConn(cc *grpc.ClientConn) error { +func (c *client) changeConn(cc *grpc.ClientConn) error { c.clientConn = cc simutil.Logger.Info("change pd client with endpoints", zap.String("tag", c.tag), zap.String("pd-address", cc.Target())) return nil @@ -143,7 +145,7 @@ func (c *client) createHeartbeatStream() (pdpb.PD_RegionHeartbeatClient, context return stream, ctx, cancel } -func (c *client) HeartbeatStreamLoop() { +func (c *client) heartbeatStreamLoop() { c.wg.Add(1) defer c.wg.Done() for { @@ -173,10 +175,10 @@ func (c *client) HeartbeatStreamLoop() { if client := SD.GetServiceClient(); client != nil { _, conn, err := getLeaderURL(ctx, client.GetClientConn()) if err != nil { - simutil.Logger.Error("[HeartbeatStreamLoop] failed to get leader URL", zap.Error(err)) + simutil.Logger.Error("[heartbeatStreamLoop] failed to get leader URL", zap.Error(err)) continue } - if err = c.ChangeConn(conn); err == nil { + if err = c.changeConn(conn); err == nil { break } } @@ -234,6 +236,7 @@ func (c *client) reportRegionHeartbeat(ctx context.Context, stream pdpb.PD_Regio } } +// Close closes the client. func (c *client) Close() { if c.cancel == nil { simutil.Logger.Info("pd client has been closed", zap.String("tag", c.tag)) @@ -248,7 +251,7 @@ func (c *client) Close() { } } -func (c *client) AllocID(ctx context.Context) (uint64, error) { +func (c *client) allocID(ctx context.Context) (uint64, error) { ctx, cancel := context.WithTimeout(ctx, pdTimeout) resp, err := c.pdClient().AllocID(ctx, &pdpb.AllocIDRequest{ Header: requestHeader(), @@ -263,6 +266,7 @@ func (c *client) AllocID(ctx context.Context) (uint64, error) { return resp.GetId(), nil } +// PutStore sends PutStore to PD. func (c *client) PutStore(ctx context.Context, store *metapb.Store) error { ctx, cancel := context.WithTimeout(ctx, pdTimeout) newStore := typeutil.DeepClone(store, core.StoreFactory) @@ -281,6 +285,7 @@ func (c *client) PutStore(ctx context.Context, store *metapb.Store) error { return nil } +// StoreHeartbeat sends a StoreHeartbeat to PD. func (c *client) StoreHeartbeat(ctx context.Context, newStats *pdpb.StoreStats) error { ctx, cancel := context.WithTimeout(ctx, pdTimeout) resp, err := c.pdClient().StoreHeartbeat(ctx, &pdpb.StoreHeartbeatRequest{ @@ -298,17 +303,18 @@ func (c *client) StoreHeartbeat(ctx context.Context, newStats *pdpb.StoreStats) return nil } +// RegionHeartbeat sends a RegionHeartbeat to PD. func (c *client) RegionHeartbeat(_ context.Context, region *core.RegionInfo) error { c.reportRegionHeartbeatCh <- region return nil } -type RetryClient struct { +type retryClient struct { client Client retryCount int } -func NewRetryClient(node *Node) *RetryClient { +func newRetryClient(node *Node) *retryClient { // Init PD client and putting it into node. tag := fmt.Sprintf("store %d", node.Store.Id) var ( @@ -331,8 +337,8 @@ func NewRetryClient(node *Node) *RetryClient { } node.client = client - // Init RetryClient - retryClient := &RetryClient{ + // Init retryClient + retryClient := &retryClient{ client: client, retryCount: retryTimes, } @@ -342,12 +348,12 @@ func NewRetryClient(node *Node) *RetryClient { }) // start heartbeat stream node.receiveRegionHeartbeatCh = receiveRegionHeartbeatCh - go client.HeartbeatStreamLoop() + go client.heartbeatStreamLoop() return retryClient } -func (rc *RetryClient) requestWithRetry(f func() (any, error)) (any, error) { +func (rc *retryClient) requestWithRetry(f func() (any, error)) (any, error) { // execute the function directly if res, err := f(); err == nil { return res, nil @@ -362,7 +368,7 @@ func (rc *RetryClient) requestWithRetry(f func() (any, error)) (any, error) { simutil.Logger.Error("[retry] failed to get leader URL", zap.Error(err)) return nil, err } - if err = rc.client.ChangeConn(conn); err != nil { + if err = rc.client.changeConn(conn); err != nil { simutil.Logger.Error("failed to change connection", zap.Error(err)) return nil, err } @@ -381,8 +387,8 @@ func getLeaderURL(ctx context.Context, conn *grpc.ClientConn) (string, *grpc.Cli if members.GetHeader().GetError() != nil { return "", nil, errors.New(members.GetHeader().GetError().String()) } - ClusterID.Store(members.GetHeader().GetClusterId()) - if ClusterID.Load() == 0 { + clusterID.Store(members.GetHeader().GetClusterId()) + if clusterID.Load() == 0 { return "", nil, errors.New("cluster id is 0") } if members.GetLeader() == nil { @@ -393,9 +399,9 @@ func getLeaderURL(ctx context.Context, conn *grpc.ClientConn) (string, *grpc.Cli return leaderURL, conn, err } -func (rc *RetryClient) AllocID(ctx context.Context) (uint64, error) { +func (rc *retryClient) allocID(ctx context.Context) (uint64, error) { res, err := rc.requestWithRetry(func() (any, error) { - id, err := rc.client.AllocID(ctx) + id, err := rc.client.allocID(ctx) return id, err }) if err != nil { @@ -404,7 +410,8 @@ func (rc *RetryClient) AllocID(ctx context.Context) (uint64, error) { return res.(uint64), nil } -func (rc *RetryClient) PutStore(ctx context.Context, store *metapb.Store) error { +// PutStore sends PutStore to PD. +func (rc *retryClient) PutStore(ctx context.Context, store *metapb.Store) error { _, err := rc.requestWithRetry(func() (any, error) { err := rc.client.PutStore(ctx, store) return nil, err @@ -412,7 +419,8 @@ func (rc *RetryClient) PutStore(ctx context.Context, store *metapb.Store) error return err } -func (rc *RetryClient) StoreHeartbeat(ctx context.Context, newStats *pdpb.StoreStats) error { +// StoreHeartbeat sends a StoreHeartbeat to PD. +func (rc *retryClient) StoreHeartbeat(ctx context.Context, newStats *pdpb.StoreStats) error { _, err := rc.requestWithRetry(func() (any, error) { err := rc.client.StoreHeartbeat(ctx, newStats) return nil, err @@ -420,7 +428,8 @@ func (rc *RetryClient) StoreHeartbeat(ctx context.Context, newStats *pdpb.StoreS return err } -func (rc *RetryClient) RegionHeartbeat(ctx context.Context, region *core.RegionInfo) error { +// RegionHeartbeat sends a RegionHeartbeat to PD. +func (rc *retryClient) RegionHeartbeat(ctx context.Context, region *core.RegionInfo) error { _, err := rc.requestWithRetry(func() (any, error) { err := rc.client.RegionHeartbeat(ctx, region) return nil, err @@ -428,15 +437,16 @@ func (rc *RetryClient) RegionHeartbeat(ctx context.Context, region *core.RegionI return err } -func (*RetryClient) ChangeConn(_ *grpc.ClientConn) error { +func (*retryClient) changeConn(_ *grpc.ClientConn) error { panic("unImplement") } -func (rc *RetryClient) HeartbeatStreamLoop() { - rc.client.HeartbeatStreamLoop() +func (rc *retryClient) heartbeatStreamLoop() { + rc.client.heartbeatStreamLoop() } -func (rc *RetryClient) Close() { +// Close closes the client. +func (rc *retryClient) Close() { rc.client.Close() } @@ -465,10 +475,10 @@ retry: break retry } } - if ClusterID.Load() == 0 { + if clusterID.Load() == 0 { return "", nil, errors.WithStack(errFailInitClusterID) } - simutil.Logger.Info("get cluster id successfully", zap.Uint64("cluster-id", ClusterID.Load())) + simutil.Logger.Info("get cluster id successfully", zap.Uint64("cluster-id", clusterID.Load())) // Check if the cluster is already bootstrapped. ctx, cancel := context.WithTimeout(ctx, pdTimeout) @@ -513,7 +523,6 @@ retry: } /* PDHTTPClient is a client for PD HTTP API, these are the functions that are used in the simulator */ - func PutPDConfig(config *sc.PDConfig) error { if len(config.PlacementRules) > 0 { ruleOps := make([]*pdHttp.RuleOp, 0) @@ -541,8 +550,9 @@ func PutPDConfig(config *sc.PDConfig) error { return nil } +// ChooseToHaltPDSchedule is used to choose whether to halt the PD schedule func ChooseToHaltPDSchedule(halt bool) { - HaltSchedule.Store(halt) + haltSchedule.Store(halt) PDHTTPClient.SetConfig(context.Background(), map[string]any{ "schedule.halt-scheduling": strconv.FormatBool(halt), }) diff --git a/tools/pd-simulator/simulator/config/config.go b/tools/pd-simulator/simulator/config/config.go index 4d182a2a03f..32c483ceba7 100644 --- a/tools/pd-simulator/simulator/config/config.go +++ b/tools/pd-simulator/simulator/config/config.go @@ -130,6 +130,8 @@ func (sc *SimConfig) Adjust(meta *toml.MetaData) error { return sc.ServerConfig.Adjust(meta, false) } + +// Speed returns the tick speed of the simulator. func (sc *SimConfig) Speed() uint64 { return uint64(time.Second / sc.SimTickInterval.Duration) } diff --git a/tools/pd-simulator/simulator/drive.go b/tools/pd-simulator/simulator/drive.go index e602ba2df65..947d0664755 100644 --- a/tools/pd-simulator/simulator/drive.go +++ b/tools/pd-simulator/simulator/drive.go @@ -135,7 +135,7 @@ func (d *Driver) allocID() error { return err } ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) - rootPath := path.Join("/pd", strconv.FormatUint(ClusterID.Load(), 10)) + rootPath := path.Join("/pd", strconv.FormatUint(clusterID.Load(), 10)) allocIDPath := path.Join(rootPath, "alloc_id") _, err = etcdClient.Put(ctx, allocIDPath, string(typeutil.Uint64ToBytes(maxID+1000))) if err != nil { @@ -171,7 +171,7 @@ func (d *Driver) updateNodesClient() error { PDHTTPClient = pdHttp.NewClientWithServiceDiscovery("pd-simulator", SD) for _, node := range d.conn.Nodes { - node.client = NewRetryClient(node) + node.client = newRetryClient(node) } return nil } @@ -191,6 +191,7 @@ func (d *Driver) Tick() { }() } +// StepRegions steps regions. func (d *Driver) StepRegions(ctx context.Context) { for { select { @@ -209,6 +210,7 @@ func (d *Driver) StepRegions(ctx context.Context) { } } +// StoresHeartbeat sends heartbeat to all stores. func (d *Driver) StoresHeartbeat(ctx context.Context) { config := d.raftEngine.storeConfig storeInterval := uint64(config.RaftStore.StoreHeartBeatInterval.Duration / config.SimTickInterval.Duration) @@ -229,6 +231,7 @@ func (d *Driver) StoresHeartbeat(ctx context.Context) { } } +// RegionsHeartbeat sends heartbeat to all regions. func (d *Driver) RegionsHeartbeat(ctx context.Context) { // ensure only wait for the first time heartbeat done firstReport := true @@ -281,9 +284,9 @@ func (d *Driver) RegionsHeartbeat(ctx context.Context) { } } - // Only set HaltSchedule to false when the leader count is 80% of the total region count. + // Only set haltSchedule to false when the leader count is 80% of the total region count. // using firstReport to avoid the haltSchedule set to true manually. - if HaltSchedule.Load() && firstReport { + if haltSchedule.Load() && firstReport { storeInterval := uint64(config.RaftStore.StoreHeartBeatInterval.Duration / config.SimTickInterval.Duration) ticker := time.NewTicker(time.Duration(storeInterval)) for range ticker.C { @@ -294,7 +297,7 @@ func (d *Driver) RegionsHeartbeat(ctx context.Context) { leaderCount += store.Status.LeaderCount } // Add halt schedule check to avoid the situation that the leader count is always less than 80%. - if leaderCount > int64(float64(d.simConfig.TotalRegion)*0.8) || !HaltSchedule.Load() { + if leaderCount > int64(float64(d.simConfig.TotalRegion)*0.8) || !haltSchedule.Load() { ChooseToHaltPDSchedule(false) firstReport = false ticker.Stop() @@ -310,11 +313,11 @@ func (d *Driver) RegionsHeartbeat(ctx context.Context) { } } -var HaltSchedule atomic.Bool +var haltSchedule atomic.Bool // Check checks if the simulation is completed. func (d *Driver) Check() bool { - if !HaltSchedule.Load() { + if !haltSchedule.Load() { return false } var stats []info.StoreStats diff --git a/tools/pd-simulator/simulator/event.go b/tools/pd-simulator/simulator/event.go index 86da86ed20d..d22f35756ef 100644 --- a/tools/pd-simulator/simulator/event.go +++ b/tools/pd-simulator/simulator/event.go @@ -177,7 +177,7 @@ type AddNode struct{} func (*AddNode) Run(raft *RaftEngine, _ int64) bool { config := raft.storeConfig nodes := raft.conn.getNodes() - id, err := nodes[0].client.AllocID(context.TODO()) + id, err := nodes[0].client.allocID(context.TODO()) if err != nil { simutil.Logger.Error("alloc node id failed", zap.Error(err)) return false @@ -196,7 +196,7 @@ func (*AddNode) Run(raft *RaftEngine, _ int64) bool { raft.conn.Nodes[s.ID] = n n.raftEngine = raft - n.client = NewRetryClient(n) + n.client = newRetryClient(n) err = n.Start() if err != nil { diff --git a/tools/pd-simulator/simulator/raft.go b/tools/pd-simulator/simulator/raft.go index 9a219d32f9f..7f3bf78622f 100644 --- a/tools/pd-simulator/simulator/raft.go +++ b/tools/pd-simulator/simulator/raft.go @@ -294,6 +294,6 @@ func (r *RaftEngine) allocID(storeID uint64) (uint64, error) { if !ok { return 0, errors.Errorf("node %d not found", storeID) } - id, err := node.client.AllocID(context.Background()) + id, err := node.client.allocID(context.Background()) return id, errors.WithStack(err) } diff --git a/tools/pd-tso-bench/main.go b/tools/pd-tso-bench/main.go index 1e59dfd2a2a..96ff2a51d0a 100644 --- a/tools/pd-tso-bench/main.go +++ b/tools/pd-tso-bench/main.go @@ -176,7 +176,7 @@ func showStats(ctx context.Context, durCh chan time.Duration) { case <-ticker.C: // runtime.GC() if *verbose { - fmt.Println(s.Counter()) + fmt.Println(s.counter()) } total.merge(s) s = newStats() @@ -184,8 +184,8 @@ func showStats(ctx context.Context, durCh chan time.Duration) { s.update(d) case <-statCtx.Done(): fmt.Println("\nTotal:") - fmt.Println(total.Counter()) - fmt.Println(total.Percentage()) + fmt.Println(total.counter()) + fmt.Println(total.percentage()) // Calculate the percentiles by using the tDigest algorithm. fmt.Printf("P0.5: %.4fms, P0.8: %.4fms, P0.9: %.4fms, P0.99: %.4fms\n\n", latencyTDigest.Quantile(0.5), latencyTDigest.Quantile(0.8), latencyTDigest.Quantile(0.9), latencyTDigest.Quantile(0.99)) if *verbose { @@ -329,7 +329,7 @@ func (s *stats) merge(other *stats) { s.oneThousandCnt += other.oneThousandCnt } -func (s *stats) Counter() string { +func (s *stats) counter() string { return fmt.Sprintf( "count: %d, max: %.4fms, min: %.4fms, avg: %.4fms\n<1ms: %d, >1ms: %d, >2ms: %d, >5ms: %d, >10ms: %d, >30ms: %d, >50ms: %d, >100ms: %d, >200ms: %d, >400ms: %d, >800ms: %d, >1s: %d", s.count, float64(s.maxDur.Nanoseconds())/float64(time.Millisecond), float64(s.minDur.Nanoseconds())/float64(time.Millisecond), float64(s.totalDur.Nanoseconds())/float64(s.count)/float64(time.Millisecond), @@ -337,7 +337,7 @@ func (s *stats) Counter() string { s.eightHundredCnt, s.oneThousandCnt) } -func (s *stats) Percentage() string { +func (s *stats) percentage() string { return fmt.Sprintf( "count: %d, <1ms: %2.2f%%, >1ms: %2.2f%%, >2ms: %2.2f%%, >5ms: %2.2f%%, >10ms: %2.2f%%, >30ms: %2.2f%%, >50ms: %2.2f%%, >100ms: %2.2f%%, >200ms: %2.2f%%, >400ms: %2.2f%%, >800ms: %2.2f%%, >1s: %2.2f%%", s.count, s.calculate(s.submilliCnt), s.calculate(s.milliCnt), s.calculate(s.twoMilliCnt), s.calculate(s.fiveMilliCnt), s.calculate(s.tenMSCnt), s.calculate(s.thirtyCnt), s.calculate(s.fiftyCnt), diff --git a/tools/pd-ut/alloc/server.go b/tools/pd-ut/alloc/server.go index a5643001142..ffa3bce0aa5 100644 --- a/tools/pd-ut/alloc/server.go +++ b/tools/pd-ut/alloc/server.go @@ -30,6 +30,7 @@ import ( var statusAddress = flag.String("status-addr", "0.0.0.0:0", "status address") +// RunHTTPServer runs a HTTP server to provide alloc address. func RunHTTPServer() *http.Server { err := os.Setenv(tempurl.AllocURLFromUT, fmt.Sprintf("http://%s/alloc", *statusAddress)) if err != nil { From 6998fb539aa95d3f96f7ef6516a84474970b8920 Mon Sep 17 00:00:00 2001 From: Hu# Date: Mon, 26 Aug 2024 16:09:15 +0800 Subject: [PATCH 33/42] mcs/transfer: Added checks for available tso nodes (#8530) close tikv/pd#8529 Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/apis/v1/api.go | 2 +- pkg/mcs/tso/server/apis/v1/api.go | 8 ++- pkg/mcs/utils/expected_primary.go | 6 +- tests/integrations/mcs/members/member_test.go | 68 +++++++++++++++---- 4 files changed, 68 insertions(+), 16 deletions(-) diff --git a/pkg/mcs/scheduling/server/apis/v1/api.go b/pkg/mcs/scheduling/server/apis/v1/api.go index 8b9427a8896..c374456a6eb 100644 --- a/pkg/mcs/scheduling/server/apis/v1/api.go +++ b/pkg/mcs/scheduling/server/apis/v1/api.go @@ -1508,7 +1508,7 @@ func transferPrimary(c *gin.Context) { } if err := mcsutils.TransferPrimary(svr.GetClient(), svr.GetParticipant().GetExpectedPrimaryLease(), - constant.SchedulingServiceName, svr.Name(), newPrimary, 0); err != nil { + constant.SchedulingServiceName, svr.Name(), newPrimary, 0, nil); err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return } diff --git a/pkg/mcs/tso/server/apis/v1/api.go b/pkg/mcs/tso/server/apis/v1/api.go index 19b3a1be612..68a654c7315 100644 --- a/pkg/mcs/tso/server/apis/v1/api.go +++ b/pkg/mcs/tso/server/apis/v1/api.go @@ -307,9 +307,15 @@ func transferPrimary(c *gin.Context) { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return } + // only members of specific group are valid primary candidates. + group := svr.GetKeyspaceGroupManager().GetKeyspaceGroups()[keyspaceGroupID] + memberMap := make(map[string]bool, len(group.Members)) + for _, member := range group.Members { + memberMap[member.Address] = true + } if err := utils.TransferPrimary(svr.GetClient(), globalAllocator.(*tso.GlobalTSOAllocator).GetExpectedPrimaryLease(), - constant.TSOServiceName, svr.Name(), newPrimary, keyspaceGroupID); err != nil { + constant.TSOServiceName, svr.Name(), newPrimary, keyspaceGroupID, memberMap); err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return } diff --git a/pkg/mcs/utils/expected_primary.go b/pkg/mcs/utils/expected_primary.go index c65d0a1cc5e..102bb8d785c 100644 --- a/pkg/mcs/utils/expected_primary.go +++ b/pkg/mcs/utils/expected_primary.go @@ -122,7 +122,7 @@ func watchExpectedPrimary(ctx context.Context, // TransferPrimary transfers the primary of the specified service. // keyspaceGroupID is optional, only used for TSO service. func TransferPrimary(client *clientv3.Client, lease *election.Lease, serviceName, - oldPrimary, newPrimary string, keyspaceGroupID uint32) error { + oldPrimary, newPrimary string, keyspaceGroupID uint32, tsoMembersMap map[string]bool) error { if lease == nil { return errors.New("current lease is nil, please check leadership") } @@ -139,6 +139,10 @@ func TransferPrimary(client *clientv3.Client, lease *election.Lease, serviceName var primaryIDs []string for _, member := range entries { + // only members of specific group are valid primary candidates for TSO service. + if tsoMembersMap != nil && !tsoMembersMap[member.ServiceAddr] { + continue + } if (newPrimary == "" && member.Name != oldPrimary) || (newPrimary != "" && member.Name == newPrimary) { primaryIDs = append(primaryIDs, member.ServiceAddr) } diff --git a/tests/integrations/mcs/members/member_test.go b/tests/integrations/mcs/members/member_test.go index dffa6305d0b..4e1e6534416 100644 --- a/tests/integrations/mcs/members/member_test.go +++ b/tests/integrations/mcs/members/member_test.go @@ -19,14 +19,19 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "testing" "time" "github.com/pingcap/failpoint" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" pdClient "github.com/tikv/pd/client/http" bs "github.com/tikv/pd/pkg/basicserver" + tso "github.com/tikv/pd/pkg/mcs/tso/server" + "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" @@ -41,6 +46,9 @@ type memberTestSuite struct { backendEndpoints string pdClient pdClient.Client + // We only test `DefaultKeyspaceGroupID` here. + // tsoAvailMembers is used to check the tso members which in the DefaultKeyspaceGroupID. + tsoAvailMembers map[string]bool tsoNodes map[string]bs.Server schedulingNodes map[string]bs.Server } @@ -51,6 +59,7 @@ func TestMemberTestSuite(t *testing.T) { func (suite *memberTestSuite) SetupTest() { re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) ctx, cancel := context.WithCancel(context.Background()) suite.ctx = ctx cluster, err := tests.NewTestAPICluster(suite.ctx, 1) @@ -65,6 +74,7 @@ func (suite *memberTestSuite) SetupTest() { // TSO nodes := make(map[string]bs.Server) + // mock 3 tso nodes, which is more than the default replica count(DefaultKeyspaceGroupReplicaCount). for i := 0; i < 3; i++ { s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) nodes[s.GetAddr()] = s @@ -72,8 +82,16 @@ func (suite *memberTestSuite) SetupTest() { cleanup() }) } - tests.WaitForPrimaryServing(re, nodes) + primary := tests.WaitForPrimaryServing(re, nodes) + members := mustGetKeyspaceGroupMembers(re, nodes[primary].(*tso.Server)) + // Get the tso nodes suite.tsoNodes = nodes + // We only test `DefaultKeyspaceGroupID` here. + // tsoAvailMembers is used to check the tso members which in the DefaultKeyspaceGroupID. + suite.tsoAvailMembers = make(map[string]bool) + for _, member := range members[constant.DefaultKeyspaceGroupID].Group.Members { + suite.tsoAvailMembers[member.Address] = true + } // Scheduling nodes = make(map[string]bs.Server) @@ -100,6 +118,8 @@ func (suite *memberTestSuite) TearDownTest() { suite.pdClient.Close() } suite.cluster.Destroy() + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) } func (suite *memberTestSuite) TestMembers() { @@ -124,7 +144,7 @@ func (suite *memberTestSuite) TestPrimary() { re.NotEmpty(primary) } -func (suite *memberTestSuite) TestCampaignPrimaryWhileServerClose() { +func (suite *memberTestSuite) TestPrimaryWorkWhileOtherServerClose() { re := suite.Require() primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, "tso") re.NoError(err) @@ -143,20 +163,18 @@ func (suite *memberTestSuite) TestCampaignPrimaryWhileServerClose() { primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) re.NoError(err) - // Close old and new primary to mock campaign primary + // Close non-primary node. for _, member := range nodes { if member.GetAddr() != primary { nodes[member.Name()].Close() - break } } - nodes[primary].Close() tests.WaitForPrimaryServing(re, nodes) - // primary should be different with before - onlyPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + // primary should be same with before. + curPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) re.NoError(err) - re.NotEqual(primary, onlyPrimary) + re.Equal(primary, curPrimary) } } @@ -200,6 +218,9 @@ func (suite *memberTestSuite) TestTransferPrimary() { // Test transfer primary to a specific node var newPrimary string for _, member := range nodes { + if service == "tso" && !suite.tsoAvailMembers[member.GetAddr()] { + continue + } if member.GetAddr() != primary { newPrimary = member.Name() break @@ -251,6 +272,9 @@ func (suite *memberTestSuite) TestCampaignPrimaryAfterTransfer() { // Test transfer primary to a specific node var newPrimary string for _, member := range nodes { + if service == "tso" && !suite.tsoAvailMembers[member.GetAddr()] { + continue + } if member.GetAddr() != primary { newPrimary = member.Name() break @@ -270,15 +294,13 @@ func (suite *memberTestSuite) TestCampaignPrimaryAfterTransfer() { re.NoError(err) re.NotEqual(primary, newPrimary) - // Close old and new primary to mock campaign primary - nodes[primary].Close() + // Close primary to push other nodes campaign primary nodes[newPrimary].Close() tests.WaitForPrimaryServing(re, nodes) // Primary should be different with before - onlyPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + anotherPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) re.NoError(err) - re.NotEqual(primary, onlyPrimary) - re.NotEqual(newPrimary, onlyPrimary) + re.NotEqual(newPrimary, anotherPrimary) } } @@ -304,6 +326,9 @@ func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpired() { // Test transfer primary to a specific node var newPrimary string for _, member := range nodes { + if service == "tso" && !suite.tsoAvailMembers[member.GetAddr()] { + continue + } if member.GetAddr() != primary { newPrimary = member.Name() break @@ -356,6 +381,9 @@ func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpiredAndServerDown( // Test transfer primary to a specific node var newPrimary string for _, member := range nodes { + if service == "tso" && !suite.tsoAvailMembers[member.GetAddr()] { + continue + } if member.GetAddr() != primary { newPrimary = member.Name() break @@ -390,3 +418,17 @@ func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpiredAndServerDown( re.NotEqual(newPrimary, onlyPrimary) } } + +func mustGetKeyspaceGroupMembers(re *require.Assertions, server *tso.Server) map[uint32]*apis.KeyspaceGroupMember { + httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+"/tso/api/v1/keyspace-groups/members", nil) + re.NoError(err) + httpResp, err := tests.TestDialClient.Do(httpReq) + re.NoError(err) + defer httpResp.Body.Close() + data, err := io.ReadAll(httpResp.Body) + re.NoError(err) + re.Equal(http.StatusOK, httpResp.StatusCode, string(data)) + var resp map[uint32]*apis.KeyspaceGroupMember + re.NoError(json.Unmarshal(data, &resp)) + return resp +} From 09e272f5d932bffae5b5c9e8f908994a829b3b4c Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 27 Aug 2024 11:43:46 +0800 Subject: [PATCH 34/42] *: remove the previous joined pd (#8562) close tikv/pd#7954 Signed-off-by: Ryan Leung --- tests/cluster.go | 5 +++++ tests/integrations/mcs/scheduling/api_test.go | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/tests/cluster.go b/tests/cluster.go index f9cbb99ce25..5cd90d8e03e 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -611,6 +611,11 @@ func (c *TestCluster) StopAll() error { return nil } +// DeleteServer is used to delete a server. +func (c *TestCluster) DeleteServer(name string) { + delete(c.servers, name) +} + // GetServer returns a server with a given name. func (c *TestCluster) GetServer(name string) *TestServer { return c.servers[name] diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index d2027a616cb..6335d104e3e 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -513,6 +513,20 @@ func (suite *apiTestSuite) checkFollowerForward(cluster *tests.TestCluster) { defer cancel() follower, err := cluster.JoinAPIServer(ctx) re.NoError(err) + defer func() { + leader := cluster.GetLeaderServer() + cli := leader.GetEtcdClient() + testutil.Eventually(re, func() bool { + _, err = cli.MemberRemove(context.Background(), follower.GetServer().GetMember().ID()) + return err == nil + }) + testutil.Eventually(re, func() bool { + res, err := cli.MemberList(context.Background()) + return err == nil && len(res.Members) == 1 + }) + cluster.DeleteServer(follower.GetConfig().Name) + follower.Destroy() + }() re.NoError(follower.Run()) re.NotEmpty(cluster.WaitLeader()) From 571c59bbcb490cbd0a1a0bef682cfa8a863732cf Mon Sep 17 00:00:00 2001 From: Hu# Date: Mon, 2 Sep 2024 13:47:07 +0800 Subject: [PATCH 35/42] mcs/metastorage: remove the register restful API (#8583) ref tikv/pd#5836 Signed-off-by: husharp --- pkg/mcs/metastorage/server/grpc_service.go | 6 +++--- tools/pd-api-bench/README.md | 4 ++-- tools/pd-simulator/simulator/drive.go | 2 -- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/mcs/metastorage/server/grpc_service.go b/pkg/mcs/metastorage/server/grpc_service.go index 4f43114ea29..c0037353437 100644 --- a/pkg/mcs/metastorage/server/grpc_service.go +++ b/pkg/mcs/metastorage/server/grpc_service.go @@ -71,9 +71,9 @@ func (s *Service) RegisterGRPCService(g *grpc.Server) { } // RegisterRESTHandler registers the service to REST server. -func (s *Service) RegisterRESTHandler(userDefineHandlers map[string]http.Handler) error { - handler, group := SetUpRestHandler(s) - return apiutil.RegisterUserDefinedHandlers(userDefineHandlers, &group, handler) +func (*Service) RegisterRESTHandler(_ map[string]http.Handler) error { + // restful API is not implemented yet. + return nil } func (s *Service) checkServing() error { diff --git a/tools/pd-api-bench/README.md b/tools/pd-api-bench/README.md index 019dbd5e6a3..76d568b3787 100644 --- a/tools/pd-api-bench/README.md +++ b/tools/pd-api-bench/README.md @@ -66,7 +66,7 @@ The api bench cases we support are as follows: You can run shell as follows. ```shell -go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug true +go run main.go --http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 --client 1 --debug true ``` ### TLS @@ -76,7 +76,7 @@ You can use the following command to generate a certificate for testing TLS: ```shell mkdir cert ./cert_opt.sh generate cert -go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug true -cacert ./cert/ca.pem -cert ./cert/pd-server.pem -key ./cert/pd-server-key.pem +go run main.go --http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 --client 1 --debug true --cacert ./cert/ca.pem --cert ./cert/pd-server.pem --key ./cert/pd-server-key.pem ./cert_opt.sh cleanup cert rm -rf cert ``` diff --git a/tools/pd-simulator/simulator/drive.go b/tools/pd-simulator/simulator/drive.go index 947d0664755..22d4175ecc6 100644 --- a/tools/pd-simulator/simulator/drive.go +++ b/tools/pd-simulator/simulator/drive.go @@ -266,7 +266,6 @@ func (d *Driver) RegionsHeartbeat(ctx context.Context) { healthyNodes[n.Store.GetId()] = true } } - report := 0 for _, region := range regions { hibernatePercent := d.simConfig.HibernatePercent // using rand(0,100) to meet hibernatePercent @@ -280,7 +279,6 @@ func (d *Driver) RegionsHeartbeat(ctx context.Context) { continue } nodesChannel[storeID] <- region.Clone() - report++ } } From 8571ca4c3bb8161209c9854c75e2910b57191a60 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 2 Sep 2024 18:12:37 +0800 Subject: [PATCH 36/42] *: use reset for ticker (#8585) ref tikv/pd#7969 Signed-off-by: Ryan Leung --- client/resource_group/controller/controller.go | 9 +++------ pkg/keyspace/tso_keyspace_group.go | 3 +-- pkg/mcs/resourcemanager/server/manager.go | 3 +-- pkg/mcs/scheduling/server/server.go | 3 +-- pkg/schedule/coordinator.go | 2 +- pkg/tso/allocator_manager.go | 3 +-- server/cluster/cluster.go | 9 +++------ server/cluster/scheduling_controller.go | 3 +-- tools/pd-api-bench/cases/controller.go | 6 +++--- tools/pd-heartbeat-bench/main.go | 6 +++--- 10 files changed, 18 insertions(+), 29 deletions(-) diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index e5ad1f81fa5..6cb7995a42e 100644 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -249,16 +249,13 @@ func (c *ResourceGroupsController) Start(ctx context.Context) { defer emergencyTokenAcquisitionTicker.Stop() failpoint.Inject("fastCleanup", func() { - cleanupTicker.Stop() - cleanupTicker = time.NewTicker(100 * time.Millisecond) + cleanupTicker.Reset(100 * time.Millisecond) // because of checking `gc.run.consumption` in cleanupTicker, // so should also change the stateUpdateTicker. - stateUpdateTicker.Stop() - stateUpdateTicker = time.NewTicker(200 * time.Millisecond) + stateUpdateTicker.Reset(200 * time.Millisecond) }) failpoint.Inject("acceleratedReportingPeriod", func() { - stateUpdateTicker.Stop() - stateUpdateTicker = time.NewTicker(time.Millisecond * 100) + stateUpdateTicker.Reset(time.Millisecond * 100) }) _, metaRevision, err := c.provider.LoadResourceGroups(ctx) diff --git a/pkg/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go index 5d7137ac3c2..f1ed6002a8c 100644 --- a/pkg/keyspace/tso_keyspace_group.go +++ b/pkg/keyspace/tso_keyspace_group.go @@ -162,8 +162,7 @@ func (m *GroupManager) allocNodesToAllKeyspaceGroups(ctx context.Context) { defer m.wg.Done() ticker := time.NewTicker(allocNodesToKeyspaceGroupsInterval) failpoint.Inject("acceleratedAllocNodes", func() { - ticker.Stop() - ticker = time.NewTicker(time.Millisecond * 100) + ticker.Reset(time.Millisecond * 100) }) defer ticker.Stop() log.Info("start to alloc nodes to all keyspace groups") diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go index 58b8b5426a4..7a07714e07a 100644 --- a/pkg/mcs/resourcemanager/server/manager.go +++ b/pkg/mcs/resourcemanager/server/manager.go @@ -324,8 +324,7 @@ func (m *Manager) GetResourceGroupList(withStats bool) []*ResourceGroup { func (m *Manager) persistLoop(ctx context.Context) { ticker := time.NewTicker(time.Minute) failpoint.Inject("fastPersist", func() { - ticker.Stop() - ticker = time.NewTicker(100 * time.Millisecond) + ticker.Reset(100 * time.Millisecond) }) defer ticker.Stop() for { diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index 9ea369aae9e..e3a6d9bd648 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -178,8 +178,7 @@ func (s *Server) updateAPIServerMemberLoop() { defer cancel() ticker := time.NewTicker(memberUpdateInterval) failpoint.Inject("fastUpdateMember", func() { - ticker.Stop() - ticker = time.NewTicker(100 * time.Millisecond) + ticker.Reset(100 * time.Millisecond) }) defer ticker.Stop() var curLeader uint64 diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go index 2a31045129e..2736c687fdb 100644 --- a/pkg/schedule/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -217,7 +217,7 @@ func (c *Coordinator) RunUntilStop(collectWaitTime ...time.Duration) { func (c *Coordinator) Run(collectWaitTime ...time.Duration) { ticker := time.NewTicker(runSchedulerCheckInterval) failpoint.Inject("changeCoordinatorTicker", func() { - ticker = time.NewTicker(100 * time.Millisecond) + ticker.Reset(100 * time.Millisecond) }) defer ticker.Stop() log.Info("coordinator starts to collect cluster information") diff --git a/pkg/tso/allocator_manager.go b/pkg/tso/allocator_manager.go index 1f5bce04583..56ee8313d57 100644 --- a/pkg/tso/allocator_manager.go +++ b/pkg/tso/allocator_manager.go @@ -715,8 +715,7 @@ func (am *AllocatorManager) AllocatorDaemon(ctx context.Context) { } tsTicker := time.NewTicker(am.updatePhysicalInterval) failpoint.Inject("fastUpdatePhysicalInterval", func() { - tsTicker.Stop() - tsTicker = time.NewTicker(time.Millisecond) + tsTicker.Reset(time.Millisecond) }) defer tsTicker.Stop() checkerTicker := time.NewTicker(PriorityCheck) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index fdeffd15e9c..ef6d45203d8 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -396,8 +396,7 @@ func (c *RaftCluster) runServiceCheckJob() { ticker := time.NewTicker(serviceCheckInterval) failpoint.Inject("highFrequencyClusterJobs", func() { - ticker.Stop() - ticker = time.NewTicker(time.Millisecond) + ticker.Reset(time.Millisecond) }) defer ticker.Stop() @@ -675,8 +674,7 @@ func (c *RaftCluster) runMetricsCollectionJob() { ticker := time.NewTicker(metricsCollectionJobInterval) failpoint.Inject("highFrequencyClusterJobs", func() { - ticker.Stop() - ticker = time.NewTicker(time.Millisecond) + ticker.Reset(time.Millisecond) }) defer ticker.Stop() @@ -699,8 +697,7 @@ func (c *RaftCluster) runNodeStateCheckJob() { ticker := time.NewTicker(nodeStateCheckJobInterval) failpoint.Inject("highFrequencyClusterJobs", func() { - ticker.Stop() - ticker = time.NewTicker(2 * time.Second) + ticker.Reset(2 * time.Second) }) defer ticker.Stop() diff --git a/server/cluster/scheduling_controller.go b/server/cluster/scheduling_controller.go index 20a36a6817d..5d617700804 100644 --- a/server/cluster/scheduling_controller.go +++ b/server/cluster/scheduling_controller.go @@ -150,8 +150,7 @@ func (sc *schedulingController) runSchedulingMetricsCollectionJob() { ticker := time.NewTicker(metricsCollectionJobInterval) failpoint.Inject("highFrequencyClusterJobs", func() { - ticker.Stop() - ticker = time.NewTicker(time.Millisecond) + ticker.Reset(time.Millisecond) }) defer ticker.Stop() diff --git a/tools/pd-api-bench/cases/controller.go b/tools/pd-api-bench/cases/controller.go index a77474db3a7..75c3c25f7ab 100644 --- a/tools/pd-api-bench/cases/controller.go +++ b/tools/pd-api-bench/cases/controller.go @@ -224,7 +224,7 @@ func (c *httpController) run() { for i := int64(0); i < burst; i++ { go func() { defer c.wg.Done() - var ticker = time.NewTicker(tt) + ticker := time.NewTicker(tt) defer ticker.Stop() for { select { @@ -293,7 +293,7 @@ func (c *gRPCController) run() { for i := int64(0); i < burst; i++ { go func() { defer c.wg.Done() - var ticker = time.NewTicker(tt) + ticker := time.NewTicker(tt) defer ticker.Stop() for { select { @@ -367,7 +367,7 @@ func (c *etcdController) run() { for i := int64(0); i < burst; i++ { go func() { defer c.wg.Done() - var ticker = time.NewTicker(tt) + ticker := time.NewTicker(tt) defer ticker.Stop() for { select { diff --git a/tools/pd-heartbeat-bench/main.go b/tools/pd-heartbeat-bench/main.go index cfa0495c31c..e77f0797bef 100644 --- a/tools/pd-heartbeat-bench/main.go +++ b/tools/pd-heartbeat-bench/main.go @@ -167,7 +167,7 @@ func putStores(ctx context.Context, cfg *config.Config, cli pdpb.PDClient, store log.Fatal("failed to put store", zap.Uint64("store-id", i), zap.String("err", resp.GetHeader().GetError().String())) } go func(ctx context.Context, storeID uint64) { - var heartbeatTicker = time.NewTicker(10 * time.Second) + heartbeatTicker := time.NewTicker(10 * time.Second) defer heartbeatTicker.Stop() for { select { @@ -526,9 +526,9 @@ func main() { header := &pdpb.RequestHeader{ ClusterId: clusterID, } - var heartbeatTicker = time.NewTicker(regionReportInterval * time.Second) + heartbeatTicker := time.NewTicker(regionReportInterval * time.Second) defer heartbeatTicker.Stop() - var resolvedTSTicker = time.NewTicker(time.Second) + resolvedTSTicker := time.NewTicker(time.Second) defer resolvedTSTicker.Stop() withMetric := metrics.InitMetric2Collect(cfg.MetricsAddr) for { From cdc86a829f70cfc507aa53b242c1b3e3be17a694 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 3 Sep 2024 15:49:00 +0800 Subject: [PATCH 37/42] *: make `TestBatchEvict` stable (#8590) close tikv/pd#8560 Signed-off-by: Ryan Leung --- pkg/schedule/schedulers/evict_leader_test.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/schedule/schedulers/evict_leader_test.go b/pkg/schedule/schedulers/evict_leader_test.go index b2c84ec68b7..ecd80813c0f 100644 --- a/pkg/schedule/schedulers/evict_leader_test.go +++ b/pkg/schedule/schedulers/evict_leader_test.go @@ -26,6 +26,7 @@ import ( "github.com/tikv/pd/pkg/schedule/types" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/operatorutil" + "github.com/tikv/pd/pkg/utils/testutil" ) func TestEvictLeader(t *testing.T) { @@ -115,7 +116,7 @@ func TestBatchEvict(t *testing.T) { tc.AddLeaderStore(2, 0) tc.AddLeaderStore(3, 0) // the random might be the same, so we add 1000 regions to make sure the batch is full - for i := 1; i <= 1000; i++ { + for i := 1; i <= 10000; i++ { tc.AddLeaderRegion(uint64(i), 1, 2, 3) } tc.AddLeaderRegion(6, 2, 1, 3) @@ -124,9 +125,13 @@ func TestBatchEvict(t *testing.T) { sl, err := CreateScheduler(types.EvictLeaderScheduler, oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(types.EvictLeaderScheduler, []string{"1"}), func(string) error { return nil }) re.NoError(err) re.True(sl.IsScheduleAllowed(tc)) - ops, _ := sl.Schedule(tc, false) - re.Len(ops, 3) + testutil.Eventually(re, func() bool { + ops, _ := sl.Schedule(tc, false) + return len(ops) == 3 + }) sl.(*evictLeaderScheduler).conf.Batch = 5 - ops, _ = sl.Schedule(tc, false) - re.Len(ops, 5) + testutil.Eventually(re, func() bool { + ops, _ := sl.Schedule(tc, false) + return len(ops) == 5 + }) } From 52a53c9a17abb8f589d80ae837bfea8f9e21d51f Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 4 Sep 2024 10:37:40 +0800 Subject: [PATCH 38/42] security: support multiple CN for TLS connections (#8518) close tikv/pd#5134 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/tlsutil/tlsconfig.go | 104 ++++++-------- pkg/utils/grpcutil/grpcutil.go | 22 +-- server/config/config.go | 15 +- tests/integrations/client/cert_opt.sh | 13 +- tests/integrations/client/client_tls_test.go | 140 +++++++++++++------ 5 files changed, 160 insertions(+), 134 deletions(-) diff --git a/client/tlsutil/tlsconfig.go b/client/tlsutil/tlsconfig.go index 5bf03dc4afc..88d797d3b3a 100644 --- a/client/tlsutil/tlsconfig.go +++ b/client/tlsutil/tlsconfig.go @@ -43,20 +43,20 @@ import ( "github.com/tikv/pd/client/errs" ) -// TLSInfo stores tls configuration to connect to etcd. -type TLSInfo struct { - CertFile string - KeyFile string - TrustedCAFile string - InsecureSkipVerify bool +// tlsInfo stores tls configuration to connect to etcd. +type tlsInfo struct { + certFile string + keyFile string + trustedCAFile string + insecureSkipVerify bool - // ServerName ensures the cert matches the given host in case of discovery / virtual hosting - ServerName string + // serverName ensures the cert matches the given host in case of discovery / virtual hosting + serverName string - // CipherSuites is a list of supported cipher suites. + // cipherSuites is a list of supported cipher suites. // If empty, Go auto-populates it by default. // Note that cipher suites are prioritized in the given order. - CipherSuites []uint16 + cipherSuites []uint16 selfCert bool @@ -64,24 +64,24 @@ type TLSInfo struct { // should be left nil. In that case, tls.X509KeyPair will be used. parseFunc func([]byte, []byte) (tls.Certificate, error) - // AllowedCN is a CN which must be provided by a client. - AllowedCN string + // allowedCNs is a list of CNs which must be provided by a client. + allowedCNs []string } -// ClientConfig generates a tls.Config object for use by an HTTP client. -func (info TLSInfo) ClientConfig() (*tls.Config, error) { +// clientConfig generates a tls.Config object for use by an HTTP client. +func (info tlsInfo) clientConfig() (*tls.Config, error) { var cfg *tls.Config var err error - if !info.Empty() { + if !info.empty() { cfg, err = info.baseConfig() if err != nil { return nil, err } } else { - cfg = &tls.Config{ServerName: info.ServerName} + cfg = &tls.Config{ServerName: info.serverName} } - cfg.InsecureSkipVerify = info.InsecureSkipVerify + cfg.InsecureSkipVerify = info.insecureSkipVerify CAFiles := info.cafiles() if len(CAFiles) > 0 { @@ -97,36 +97,38 @@ func (info TLSInfo) ClientConfig() (*tls.Config, error) { return cfg, nil } -// Empty returns if the TLSInfo is unset. -func (info TLSInfo) Empty() bool { - return info.CertFile == "" && info.KeyFile == "" +// empty returns if the TLSInfo is unset. +func (info tlsInfo) empty() bool { + return info.certFile == "" && info.keyFile == "" } -func (info TLSInfo) baseConfig() (*tls.Config, error) { - if info.KeyFile == "" || info.CertFile == "" { - return nil, fmt.Errorf("KeyFile and CertFile must both be present[key: %v, cert: %v]", info.KeyFile, info.CertFile) +func (info tlsInfo) baseConfig() (*tls.Config, error) { + if info.keyFile == "" || info.certFile == "" { + return nil, fmt.Errorf("KeyFile and CertFile must both be present[key: %v, cert: %v]", info.keyFile, info.certFile) } - _, err := NewCert(info.CertFile, info.KeyFile, info.parseFunc) + _, err := NewCert(info.certFile, info.keyFile, info.parseFunc) if err != nil { return nil, err } cfg := &tls.Config{ MinVersion: tls.VersionTLS12, - ServerName: info.ServerName, + ServerName: info.serverName, } - if len(info.CipherSuites) > 0 { - cfg.CipherSuites = info.CipherSuites + if len(info.cipherSuites) > 0 { + cfg.CipherSuites = info.cipherSuites } - if info.AllowedCN != "" { + if len(info.allowedCNs) > 0 { cfg.VerifyPeerCertificate = func(_ [][]byte, verifiedChains [][]*x509.Certificate) error { for _, chains := range verifiedChains { if len(chains) != 0 { - if info.AllowedCN == chains[0].Subject.CommonName { - return nil + for _, allowedCN := range info.allowedCNs { + if allowedCN == chains[0].Subject.CommonName { + return nil + } } } } @@ -137,19 +139,19 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) { // this only reloads certs when there's a client request // TODO: support server-side refresh (e.g. inotify, SIGHUP), caching cfg.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { - return NewCert(info.CertFile, info.KeyFile, info.parseFunc) + return NewCert(info.certFile, info.keyFile, info.parseFunc) } cfg.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { - return NewCert(info.CertFile, info.KeyFile, info.parseFunc) + return NewCert(info.certFile, info.keyFile, info.parseFunc) } return cfg, nil } // cafiles returns a list of CA file paths. -func (info TLSInfo) cafiles() []string { +func (info tlsInfo) cafiles() []string { cs := make([]string, 0) - if info.TrustedCAFile != "" { - cs = append(cs, info.TrustedCAFile) + if info.trustedCAFile != "" { + cs = append(cs, info.trustedCAFile) } return cs } @@ -162,8 +164,8 @@ type TLSConfig struct { CertPath string `toml:"cert-path" json:"cert-path"` // KeyPath is the path of file that contains X509 key in PEM format. KeyPath string `toml:"key-path" json:"key-path"` - // CertAllowedCN is a CN which must be provided by a client - CertAllowedCN []string `toml:"cert-allowed-cn" json:"cert-allowed-cn"` + // CertAllowedCNs is the list of CN which must be provided by a client + CertAllowedCNs []string `toml:"cert-allowed-cn" json:"cert-allowed-cn"` SSLCABytes []byte SSLCertBytes []byte @@ -194,33 +196,17 @@ func (s TLSConfig) ToTLSConfig() (*tls.Config, error) { if len(s.CertPath) == 0 && len(s.KeyPath) == 0 { return nil, nil } - allowedCN, err := s.GetOneAllowedCN() - if err != nil { - return nil, err - } - tlsInfo := TLSInfo{ - CertFile: s.CertPath, - KeyFile: s.KeyPath, - TrustedCAFile: s.CAPath, - AllowedCN: allowedCN, + tlsInfo := tlsInfo{ + certFile: s.CertPath, + keyFile: s.KeyPath, + trustedCAFile: s.CAPath, + allowedCNs: s.CertAllowedCNs, } - tlsConfig, err := tlsInfo.ClientConfig() + tlsConfig, err := tlsInfo.clientConfig() if err != nil { return nil, errs.ErrEtcdTLSConfig.Wrap(err).GenWithStackByCause() } return tlsConfig, nil } - -// GetOneAllowedCN only gets the first one CN. -func (s TLSConfig) GetOneAllowedCN() (string, error) { - switch len(s.CertAllowedCN) { - case 1: - return s.CertAllowedCN[0], nil - case 0: - return "", nil - default: - return "", errs.ErrSecurityConfig.FastGenByArgs("only supports one CN") - } -} diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index 5f852ff7359..1d1e6478036 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -52,8 +52,8 @@ type TLSConfig struct { CertPath string `toml:"cert-path" json:"cert-path"` // KeyPath is the path of file that contains X509 key in PEM format. KeyPath string `toml:"key-path" json:"key-path"` - // CertAllowedCN is a CN which must be provided by a client - CertAllowedCN []string `toml:"cert-allowed-cn" json:"cert-allowed-cn"` + // CertAllowedCNs is the list of CN which must be provided by a client + CertAllowedCNs []string `toml:"cert-allowed-cn" json:"cert-allowed-cn"` SSLCABytes []byte SSLCertBytes []byte @@ -65,16 +65,12 @@ func (s TLSConfig) ToTLSInfo() (*transport.TLSInfo, error) { if len(s.CertPath) == 0 && len(s.KeyPath) == 0 { return nil, nil } - allowedCN, err := s.GetOneAllowedCN() - if err != nil { - return nil, err - } return &transport.TLSInfo{ CertFile: s.CertPath, KeyFile: s.KeyPath, TrustedCAFile: s.CAPath, - AllowedCN: allowedCN, + AllowedCNs: s.CertAllowedCNs, }, nil } @@ -114,18 +110,6 @@ func (s TLSConfig) ToTLSConfig() (*tls.Config, error) { return tlsConfig, nil } -// GetOneAllowedCN only gets the first one CN. -func (s TLSConfig) GetOneAllowedCN() (string, error) { - switch len(s.CertAllowedCN) { - case 1: - return s.CertAllowedCN[0], nil - case 0: - return "", nil - default: - return "", errs.ErrSecurityConfig.FastGenByArgs("only supports one CN") - } -} - // GetClientConn returns a gRPC client connection. // creates a client connection to the given target. By default, it's // a non-blocking dial (the function won't wait for connections to be diff --git a/server/config/config.go b/server/config/config.go index c1791be1fd9..135ea70b03c 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -726,23 +726,20 @@ func (c *Config) GenEmbedEtcdConfig() (*embed.Config, error) { cfg.QuotaBackendBytes = int64(c.QuotaBackendBytes) cfg.MaxRequestBytes = c.MaxRequestBytes - allowedCN, serr := c.Security.GetOneAllowedCN() - if serr != nil { - return nil, serr - } cfg.ClientTLSInfo.ClientCertAuth = len(c.Security.CAPath) != 0 cfg.ClientTLSInfo.TrustedCAFile = c.Security.CAPath cfg.ClientTLSInfo.CertFile = c.Security.CertPath cfg.ClientTLSInfo.KeyFile = c.Security.KeyPath - // Client no need to set the CN. (cfg.ClientTLSInfo.AllowedCN = allowedCN) + // Keep compatibility with https://github.com/tikv/pd/pull/2305 + // Only check client cert when there are multiple CNs. + if len(c.Security.CertAllowedCNs) > 1 { + cfg.ClientTLSInfo.AllowedCNs = c.Security.CertAllowedCNs + } cfg.PeerTLSInfo.ClientCertAuth = len(c.Security.CAPath) != 0 cfg.PeerTLSInfo.TrustedCAFile = c.Security.CAPath cfg.PeerTLSInfo.CertFile = c.Security.CertPath cfg.PeerTLSInfo.KeyFile = c.Security.KeyPath - // TODO: After https://github.com/etcd-io/etcd/pull/18015, AllowedCN is Deprecated. - // It will be replaced by AllowedCNs in the future to support multi cn. - // nolint:staticcheck - cfg.PeerTLSInfo.AllowedCN = allowedCN + cfg.PeerTLSInfo.AllowedCNs = c.Security.CertAllowedCNs cfg.ForceNewCluster = c.ForceNewCluster cfg.ZapLoggerBuilder = embed.NewZapCoreLoggerBuilder(c.Logger, c.Logger.Core(), c.LogProps.Syncer) cfg.EnableGRPCGateway = c.EnableGRPCGateway diff --git a/tests/integrations/client/cert_opt.sh b/tests/integrations/client/cert_opt.sh index 3984e67f3ab..02f72249db7 100755 --- a/tests/integrations/client/cert_opt.sh +++ b/tests/integrations/client/cert_opt.sh @@ -3,7 +3,7 @@ cert_dir="$2" function generate_certs() { if [[ ! -z "$cert_dir" ]]; then - cd "$cert_dir" || exit 255 # Change to the specified directory + cd "$cert_dir" || exit 255 # Change to the specified directory fi if ! [[ "$0" =~ "cert_opt.sh" ]]; then @@ -21,10 +21,10 @@ function generate_certs() { openssl req -new -x509 -key ca-key.pem -out ca.pem -days 1 -subj "/CN=ca" # pd-server openssl genpkey -algorithm RSA -out pd-server-key.pem - openssl req -new -key pd-server-key.pem -out pd-server.csr -subj "/CN=pd-server" + openssl req -new -key pd-server-key.pem -out pd-server.csr -subj "/CN=pd-server" # Add IP address as a SAN - echo "subjectAltName = IP:127.0.0.1" > extfile.cnf + echo "subjectAltName = IP:127.0.0.1" >extfile.cnf openssl x509 -req -in pd-server.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out pd-server.pem -days 1 -extfile extfile.cnf # Clean up the temporary extension file @@ -34,11 +34,16 @@ function generate_certs() { openssl genpkey -algorithm RSA -out client-key.pem openssl req -new -key client-key.pem -out client.csr -subj "/CN=client" openssl x509 -req -in client.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out client.pem -days 1 + + # client2 + openssl genpkey -algorithm RSA -out tidb-client-key.pem + openssl req -new -key tidb-client-key.pem -out tidb-client.csr -subj "/CN=tidb" + openssl x509 -req -in tidb-client.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out tidb-client.pem -days 1 } function cleanup_certs() { if [[ ! -z "$cert_dir" ]]; then - cd "$cert_dir" || exit 255 # Change to the specified directory + cd "$cert_dir" || exit 255 # Change to the specified directory fi rm -f ca.pem ca-key.pem ca.srl diff --git a/tests/integrations/client/client_tls_test.go b/tests/integrations/client/client_tls_test.go index b915d77b3b0..75706c3d902 100644 --- a/tests/integrations/client/client_tls_test.go +++ b/tests/integrations/client/client_tls_test.go @@ -36,54 +36,24 @@ import ( "google.golang.org/grpc" ) -var ( - certPath = strings.Join([]string{".", "cert"}, string(filepath.Separator)) - certExpiredPath = strings.Join([]string{".", "cert-expired"}, string(filepath.Separator)) - certScript = strings.Join([]string{".", "cert_opt.sh"}, string(filepath.Separator)) - testTLSInfo = transport.TLSInfo{ - KeyFile: strings.Join([]string{".", "cert", "pd-server-key.pem"}, string(filepath.Separator)), - CertFile: strings.Join([]string{".", "cert", "pd-server.pem"}, string(filepath.Separator)), - TrustedCAFile: strings.Join([]string{".", "cert", "ca.pem"}, string(filepath.Separator)), - } - - testClientTLSInfo = transport.TLSInfo{ - KeyFile: strings.Join([]string{".", "cert", "client-key.pem"}, string(filepath.Separator)), - CertFile: strings.Join([]string{".", "cert", "client.pem"}, string(filepath.Separator)), - TrustedCAFile: strings.Join([]string{".", "cert", "ca.pem"}, string(filepath.Separator)), - } - - testTLSInfoExpired = transport.TLSInfo{ - KeyFile: strings.Join([]string{".", "cert-expired", "pd-server-key.pem"}, string(filepath.Separator)), - CertFile: strings.Join([]string{".", "cert-expired", "pd-server.pem"}, string(filepath.Separator)), - TrustedCAFile: strings.Join([]string{".", "cert-expired", "ca.pem"}, string(filepath.Separator)), - } -) +var certScript = strings.Join([]string{".", "cert_opt.sh"}, string(filepath.Separator)) // TestTLSReloadAtomicReplace ensures server reloads expired/valid certs // when all certs are atomically replaced by directory renaming. // And expects server to reject client requests, and vice versa. func TestTLSReloadAtomicReplace(t *testing.T) { - // generate certs - for _, path := range []string{certPath, certExpiredPath} { - if err := os.Mkdir(path, 0755); err != nil { - t.Fatal(err) - } - if err := exec.Command(certScript, "generate", path).Run(); err != nil { - t.Fatal(err) - } - } - defer func() { - for _, path := range []string{certPath, certExpiredPath} { - if err := exec.Command(certScript, "cleanup", path).Run(); err != nil { - t.Fatal(err) - } - if err := os.RemoveAll(path); err != nil { - t.Fatal(err) - } - } - }() - re := require.New(t) + + certPath := strings.Join([]string{".", "cert"}, string(filepath.Separator)) + certExpiredPath := strings.Join([]string{".", "cert-expired"}, string(filepath.Separator)) + cleanFunc := generateCerts(re, certPath) + defer cleanFunc() + cleanFunc = generateCerts(re, certExpiredPath) + defer cleanFunc() + testTLSInfo := buildTLSInfo(certPath, "pd-server") + testTLSInfoExpired := buildTLSInfo(certExpiredPath, "pd-server") + testClientTLSInfo := buildTLSInfo(certPath, "client") + ctx, cancel := context.WithCancel(context.Background()) defer cancel() tmpDir := t.TempDir() @@ -120,12 +90,35 @@ func TestTLSReloadAtomicReplace(t *testing.T) { err = os.Rename(certsDirExp, certsDir) re.NoError(err) } - testTLSReload(ctx, re, cloneFunc, replaceFunc, revertFunc) + testTLSReload(ctx, re, testClientTLSInfo, cloneFunc, replaceFunc, revertFunc) +} + +func buildTLSInfo(path, name string) transport.TLSInfo { + return transport.TLSInfo{ + KeyFile: strings.Join([]string{path, name + "-key.pem"}, string(filepath.Separator)), + CertFile: strings.Join([]string{path, name + ".pem"}, string(filepath.Separator)), + TrustedCAFile: strings.Join([]string{path, "ca.pem"}, string(filepath.Separator)), + } +} + +func generateCerts(re *require.Assertions, path string) func() { + err := os.Mkdir(path, 0755) + re.NoError(err) + err = exec.Command(certScript, "generate", path).Run() + re.NoError(err) + + return func() { + err := exec.Command(certScript, "cleanup", path).Run() + re.NoError(err) + err = os.RemoveAll(path) + re.NoError(err) + } } func testTLSReload( ctx context.Context, re *require.Assertions, + testClientTLSInfo transport.TLSInfo, cloneFunc func() transport.TLSInfo, replaceFunc func(), revertFunc func()) { @@ -275,3 +268,64 @@ func copyFile(src, dst string) error { } return w.Sync() } + +func TestMultiCN(t *testing.T) { + re := require.New(t) + + certPath := strings.Join([]string{".", "cert-multi-cn"}, string(filepath.Separator)) + cleanFunc := generateCerts(re, certPath) + defer cleanFunc() + testTLSInfo := buildTLSInfo(certPath, "pd-server") + testClientTLSInfo := buildTLSInfo(certPath, "client") + testTiDBClientTLSInfo := buildTLSInfo(certPath, "tidb-client") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + clus, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { + conf.Security.TLSConfig = grpcutil.TLSConfig{ + KeyPath: testTLSInfo.KeyFile, + CertPath: testTLSInfo.CertFile, + CAPath: testTLSInfo.TrustedCAFile, + CertAllowedCNs: []string{"tidb", "pd-server"}, + } + conf.AdvertiseClientUrls = strings.ReplaceAll(conf.AdvertiseClientUrls, "http", "https") + conf.ClientUrls = strings.ReplaceAll(conf.ClientUrls, "http", "https") + conf.AdvertisePeerUrls = strings.ReplaceAll(conf.AdvertisePeerUrls, "http", "https") + conf.PeerUrls = strings.ReplaceAll(conf.PeerUrls, "http", "https") + conf.InitialCluster = strings.ReplaceAll(conf.InitialCluster, "http", "https") + }) + re.NoError(err) + defer clus.Destroy() + err = clus.RunInitialServers() + re.NoError(err) + clus.WaitLeader() + + testServers := clus.GetServers() + endpoints := make([]string, 0, len(testServers)) + for _, s := range testServers { + endpoints = append(endpoints, s.GetConfig().AdvertiseClientUrls) + } + + // cn TiDB is allowed + re.NoError(testAllowedCN(ctx, endpoints, testTiDBClientTLSInfo)) + + // cn client is not allowed + re.Error(testAllowedCN(ctx, endpoints, testClientTLSInfo)) +} + +func testAllowedCN(ctx context.Context, endpoints []string, tls transport.TLSInfo) error { + ctx1, cancel1 := context.WithTimeout(ctx, 3*time.Second) + defer cancel1() + cli, err := pd.NewClientWithContext(ctx1, endpoints, pd.SecurityOption{ + CAPath: tls.TrustedCAFile, + CertPath: tls.CertFile, + KeyPath: tls.KeyFile, + }, pd.WithGRPCDialOptions(grpc.WithBlock())) + if err != nil { + return err + } + defer cli.Close() + _, err = cli.GetAllMembers(ctx1) + return err +} From 750d475e6b13dd45bda373f4ad718e16f9d18adb Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 4 Sep 2024 12:10:58 +0800 Subject: [PATCH 39/42] *: make `TestRegionCheck` stable (#8591) close tikv/pd#8519 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/api/region_test.go | 51 +++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/server/api/region_test.go b/server/api/region_test.go index fb32496c16a..482208261e3 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -126,9 +126,13 @@ func (suite *regionTestSuite) TestRegionCheck() { url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "down-peer") r2 := &response.RegionsInfo{} - re.NoError(tu.ReadGetJSON(re, testDialClient, url, r2)) - r2.Adjust() - re.Equal(&response.RegionsInfo{Count: 1, Regions: []response.RegionInfo{*response.NewAPIRegionInfo(r)}}, r2) + tu.Eventually(re, func() bool { + if err := tu.ReadGetJSON(re, testDialClient, url, r2); err != nil { + return false + } + r2.Adjust() + return suite.Equal(&response.RegionsInfo{Count: 1, Regions: []response.RegionInfo{*response.NewAPIRegionInfo(r)}}, r2) + }) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "pending-peer") r3 := &response.RegionsInfo{} @@ -146,25 +150,37 @@ func (suite *regionTestSuite) TestRegionCheck() { mustRegionHeartbeat(re, suite.svr, r) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "empty-region") r5 := &response.RegionsInfo{} - re.NoError(tu.ReadGetJSON(re, testDialClient, url, r5)) - r5.Adjust() - re.Equal(&response.RegionsInfo{Count: 1, Regions: []response.RegionInfo{*response.NewAPIRegionInfo(r)}}, r5) + tu.Eventually(re, func() bool { + if err := tu.ReadGetJSON(re, testDialClient, url, r5); err != nil { + return false + } + r5.Adjust() + return suite.Equal(&response.RegionsInfo{Count: 1, Regions: []response.RegionInfo{*response.NewAPIRegionInfo(r)}}, r5) + }) r = r.Clone(core.SetApproximateSize(1)) mustRegionHeartbeat(re, suite.svr, r) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "hist-size") r6 := make([]*histItem, 1) - re.NoError(tu.ReadGetJSON(re, testDialClient, url, &r6)) - histSizes := []*histItem{{Start: 1, End: 1, Count: 1}} - re.Equal(histSizes, r6) + tu.Eventually(re, func() bool { + if err := tu.ReadGetJSON(re, testDialClient, url, &r6); err != nil { + return false + } + histSizes := []*histItem{{Start: 1, End: 1, Count: 1}} + return suite.Equal(histSizes, r6) + }) r = r.Clone(core.SetApproximateKeys(1000)) mustRegionHeartbeat(re, suite.svr, r) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "hist-keys") r7 := make([]*histItem, 1) - re.NoError(tu.ReadGetJSON(re, testDialClient, url, &r7)) - histKeys := []*histItem{{Start: 1000, End: 1999, Count: 1}} - re.Equal(histKeys, r7) + tu.Eventually(re, func() bool { + if err := tu.ReadGetJSON(re, testDialClient, url, &r7); err != nil { + return false + } + histKeys := []*histItem{{Start: 1000, End: 1999, Count: 1}} + return suite.Equal(histKeys, r7) + }) // ref https://github.com/tikv/pd/issues/3558, we should change size to pass `NeedUpdate` for observing. r = r.Clone(core.SetApproximateKeys(0)) @@ -172,10 +188,13 @@ func (suite *regionTestSuite) TestRegionCheck() { mustRegionHeartbeat(re, suite.svr, r) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "offline-peer") r8 := &response.RegionsInfo{} - re.NoError(tu.ReadGetJSON(re, testDialClient, url, r8)) - r4.Adjust() - re.Equal(1, r8.Count) - re.Equal(r.GetID(), r8.Regions[0].ID) + tu.Eventually(re, func() bool { + if err := tu.ReadGetJSON(re, testDialClient, url, r8); err != nil { + return false + } + r4.Adjust() + return suite.Equal(r.GetID(), r8.Regions[0].ID) && r8.Count == 1 + }) } func (suite *regionTestSuite) TestRegions() { From 193fe6cac7eea06282b1803aee327abf283979ef Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 4 Sep 2024 16:23:56 +0800 Subject: [PATCH 40/42] *: make `TestLeaderPriority` stable (#8592) close tikv/pd#8447 Signed-off-by: Ryan Leung --- tests/server/member/member_test.go | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index 6cd9eacddf1..2d0c9098153 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -161,21 +161,33 @@ func TestLeaderPriority(t *testing.T) { re.NotEmpty(cluster.WaitLeader()) - leader1, err := cluster.GetServer("pd1").GetEtcdLeader() + leader, err := cluster.GetServer("pd1").GetEtcdLeader() re.NoError(err) - server1 := cluster.GetServer(leader1) - addr := server1.GetConfig().ClientUrls + server := cluster.GetServer(leader) + addr := server.GetConfig().ClientUrls // PD leader should sync with etcd leader. testutil.Eventually(re, func() bool { - return cluster.GetLeader() == leader1 + leader, err := cluster.GetServer("pd1").GetEtcdLeader() + if err != nil { + return false + } + return cluster.GetLeader() == leader }) // Bind a lower priority to current leader. - post(t, re, addr+"/pd/api/v1/members/name/"+leader1, `{"leader-priority": -1}`) + post(t, re, addr+"/pd/api/v1/members/name/"+leader, `{"leader-priority": -1}`) + // Wait etcd leader change. - leader2 := waitEtcdLeaderChange(re, server1, leader1) + waitEtcdLeaderChange(re, server, leader) // PD leader should sync with etcd leader again. testutil.Eventually(re, func() bool { - return cluster.GetLeader() == leader2 + etcdLeader, err := server.GetEtcdLeader() + if err != nil { + return false + } + if cluster.GetLeader() == etcdLeader { + return true + } + return false }) } From a4ae781d07dcc75db681cb475c6785d830e410ac Mon Sep 17 00:00:00 2001 From: Sparkle <1284531+baurine@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:08:52 +0800 Subject: [PATCH 41/42] chore(dashboard): update TiDB Dashboard to v8.4.0-6a0d342d [master] (#8581) ref tikv/pd#4257 Signed-off-by: mornyx Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> Co-authored-by: mornyx --- conf/config.toml | 3 +++ go.mod | 2 +- go.sum | 4 ++-- pkg/dashboard/adapter/config.go | 1 + scripts/dashboard-version | 2 +- server/config/config.go | 15 ++++++++------- tests/integrations/go.mod | 2 +- tests/integrations/go.sum | 4 ++-- tools/go.mod | 2 +- tools/go.sum | 4 ++-- 10 files changed, 22 insertions(+), 17 deletions(-) diff --git a/conf/config.toml b/conf/config.toml index f2feacf30f7..a4adc9c7d66 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -204,6 +204,9 @@ ## When enabled, usage data will be sent to PingCAP for improving user experience. # enable-telemetry = false +## When enabled, configuring a custom prometheus address through Dashboard will not be allowed. +# disable-custom-prom-addr = false + [keyspace] ## pre-alloc is used to pre-allocate keyspaces during pd bootstrap. ## Its value should be a list of strings, denoting the name of the keyspaces. diff --git a/go.mod b/go.mod index 79db4f3940c..44e56de85b3 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20240716095229-5f7ffec83ea7 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 - github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c + github.com/pingcap/tidb-dashboard v0.0.0-20240830080600-3d0c3db0d55c github.com/prometheus/client_golang v1.19.0 github.com/prometheus/common v0.51.1 github.com/sasha-s/go-deadlock v0.2.0 diff --git a/go.sum b/go.sum index e57fdcbce9c..e40bcbfef73 100644 --- a/go.sum +++ b/go.sum @@ -394,8 +394,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c h1:iiVpATdlLLr0NEgKZu+ZnBuYR4J8IgjNE1hNMjMOkYY= -github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= +github.com/pingcap/tidb-dashboard v0.0.0-20240830080600-3d0c3db0d55c h1:+0Tl9izTX2X5cy4wO4cbvjjsg8LqKpHdH5bEMv7pTb4= +github.com/pingcap/tidb-dashboard v0.0.0-20240830080600-3d0c3db0d55c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/pkg/dashboard/adapter/config.go b/pkg/dashboard/adapter/config.go index 348b146c854..cf79690e73b 100644 --- a/pkg/dashboard/adapter/config.go +++ b/pkg/dashboard/adapter/config.go @@ -35,6 +35,7 @@ func GenDashboardConfig(srv *server.Server) (*config.Config, error) { dashboardCfg.PublicPathPrefix = cfg.Dashboard.PublicPathPrefix dashboardCfg.EnableTelemetry = cfg.Dashboard.EnableTelemetry dashboardCfg.EnableExperimental = cfg.Dashboard.EnableExperimental + dashboardCfg.DisableCustomPromAddr = cfg.Dashboard.DisableCustomPromAddr if dashboardCfg.ClusterTLSConfig, err = cfg.Security.ToTLSConfig(); err != nil { return nil, err } diff --git a/scripts/dashboard-version b/scripts/dashboard-version index d8d960683ae..1f6c34ca2fd 100644 --- a/scripts/dashboard-version +++ b/scripts/dashboard-version @@ -1,3 +1,3 @@ # This file is updated by running scripts/update-dashboard.sh # Don't edit it manually -8.3.0-e6e78c7c +8.4.0-3d0c3db0 diff --git a/server/config/config.go b/server/config/config.go index 135ea70b03c..d8bd086225c 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -771,13 +771,14 @@ func (c *Config) GenEmbedEtcdConfig() (*embed.Config, error) { // DashboardConfig is the configuration for tidb-dashboard. type DashboardConfig struct { - TiDBCAPath string `toml:"tidb-cacert-path" json:"tidb-cacert-path"` - TiDBCertPath string `toml:"tidb-cert-path" json:"tidb-cert-path"` - TiDBKeyPath string `toml:"tidb-key-path" json:"tidb-key-path"` - PublicPathPrefix string `toml:"public-path-prefix" json:"public-path-prefix"` - InternalProxy bool `toml:"internal-proxy" json:"internal-proxy"` - EnableTelemetry bool `toml:"enable-telemetry" json:"enable-telemetry"` - EnableExperimental bool `toml:"enable-experimental" json:"enable-experimental"` + TiDBCAPath string `toml:"tidb-cacert-path" json:"tidb-cacert-path"` + TiDBCertPath string `toml:"tidb-cert-path" json:"tidb-cert-path"` + TiDBKeyPath string `toml:"tidb-key-path" json:"tidb-key-path"` + PublicPathPrefix string `toml:"public-path-prefix" json:"public-path-prefix"` + InternalProxy bool `toml:"internal-proxy" json:"internal-proxy"` + EnableTelemetry bool `toml:"enable-telemetry" json:"enable-telemetry"` + EnableExperimental bool `toml:"enable-experimental" json:"enable-experimental"` + DisableCustomPromAddr bool `toml:"disable-custom-prom-addr" json:"disable-custom-prom-addr"` } // ToTiDBTLSConfig generates tls config for connecting to TiDB, used by tidb-dashboard. diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index d7305379e9f..f25723f7d25 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -129,7 +129,7 @@ require ( github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20240830080600-3d0c3db0d55c // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index 914ce1ff35c..91e1ba6481e 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -391,8 +391,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c h1:iiVpATdlLLr0NEgKZu+ZnBuYR4J8IgjNE1hNMjMOkYY= -github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= +github.com/pingcap/tidb-dashboard v0.0.0-20240830080600-3d0c3db0d55c h1:+0Tl9izTX2X5cy4wO4cbvjjsg8LqKpHdH5bEMv7pTb4= +github.com/pingcap/tidb-dashboard v0.0.0-20240830080600-3d0c3db0d55c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/tools/go.mod b/tools/go.mod index b1a3c1e3543..d3f8657df3d 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -133,7 +133,7 @@ require ( github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20240830080600-3d0c3db0d55c // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tools/go.sum b/tools/go.sum index 0933f7ceda7..da9d2c19967 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -388,8 +388,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c h1:iiVpATdlLLr0NEgKZu+ZnBuYR4J8IgjNE1hNMjMOkYY= -github.com/pingcap/tidb-dashboard v0.0.0-20240815045040-4d89bc193a0c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= +github.com/pingcap/tidb-dashboard v0.0.0-20240830080600-3d0c3db0d55c h1:+0Tl9izTX2X5cy4wO4cbvjjsg8LqKpHdH5bEMv7pTb4= +github.com/pingcap/tidb-dashboard v0.0.0-20240830080600-3d0c3db0d55c/go.mod h1:AT9vfeojwr/GGCHTURXtA8yZBE9AW8LdIo02/eYdfHU= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= From 0bb9b1333226eb24eccf40ab1ae62835cfcf2f55 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 6 Sep 2024 13:54:51 +0800 Subject: [PATCH 42/42] *: make `TestForwardTSOUnexpectedToFollower` stable (#8595) close tikv/pd#8397, close tikv/pd#8589 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/grpc_service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/grpc_service.go b/server/grpc_service.go index d3db45c8364..216611038e5 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -57,7 +57,7 @@ import ( const ( heartbeatSendTimeout = 5 * time.Second - maxRetryTimesRequestTSOServer = 3 + maxRetryTimesRequestTSOServer = 6 retryIntervalRequestTSOServer = 500 * time.Millisecond getMinTSFromTSOServerTimeout = 1 * time.Second defaultGRPCDialTimeout = 3 * time.Second