diff --git a/config/retry/config.go b/config/retry/config.go
index 59a4c5fe50..7431ad8f9f 100644
--- a/config/retry/config.go
+++ b/config/retry/config.go
@@ -177,7 +177,7 @@ var (
 	BoTiFlashRPC       = NewConfig("tiflashRPC", &metrics.BackoffHistogramRPC, NewBackoffFnCfg(100, 2000, EqualJitter), tikverr.ErrTiFlashServerTimeout)
 	BoTxnLock          = NewConfig("txnLock", &metrics.BackoffHistogramLock, NewBackoffFnCfg(100, 3000, EqualJitter), tikverr.ErrResolveLockTimeout)
 	BoPDRPC            = NewConfig("pdRPC", &metrics.BackoffHistogramPD, NewBackoffFnCfg(500, 3000, EqualJitter), tikverr.NewErrPDServerTimeout(""))
-	BoPDRegionMetadata = NewConfigWithRetryLimit("pdRegionMetadata", &metrics.BackoffHistogramPD, NewBackoffFnCfg(500, 3000, EqualJitter), NewRetryRateLimiter(10, 10), tikverr.NewErrPDServerTimeout(""))
+	BoPDRegionMetadata = NewConfigWithRetryLimit("pdRegionMetadata", &metrics.BackoffHistogramPD, NewBackoffFnCfg(500, 3000, EqualJitter), NewRetryRateLimiter(100, 1), tikverr.NewErrPDServerTimeout(""))
 	// change base time to 2ms, because it may recover soon.
 	BoRegionMiss               = NewConfig("regionMiss", &metrics.BackoffHistogramRegionMiss, NewBackoffFnCfg(2, 500, NoJitter), tikverr.ErrRegionUnavailable)
 	BoRegionScheduling         = NewConfig("regionScheduling", &metrics.BackoffHistogramRegionScheduling, NewBackoffFnCfg(2, 500, NoJitter), tikverr.ErrRegionUnavailable)
diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go
index 01dfa385ca..cc9d54e242 100644
--- a/internal/locate/region_cache.go
+++ b/internal/locate/region_cache.go
@@ -2059,7 +2059,7 @@ func (c *RegionCache) loadRegion(bo *retry.Backoffer, key []byte, isEndKey bool,
 	opts = append(opts, pd.WithBuckets())
 	for {
 		if backoffErr != nil {
-			err := bo.Backoff(retry.BoPDRPC, backoffErr)
+			err := bo.Backoff(retry.BoPDRegionMetadata, backoffErr)
 			if err != nil {
 				return nil, errors.WithStack(err)
 			}
@@ -2077,6 +2077,10 @@ func (c *RegionCache) loadRegion(bo *retry.Backoffer, key []byte, isEndKey bool,
 			metrics.RegionCacheCounterWithGetCacheMissError.Inc()
 		} else {
 			metrics.RegionCacheCounterWithGetCacheMissOK.Inc()
+			if backoffErr == nil {
+				// refill retry allowance only for the original call
+				bo.OnSuccess(retry.BoPDRegionMetadata)
+			}
 		}
 		if err != nil {
 			if apicodec.IsDecodeError(err) {
@@ -2112,7 +2116,7 @@ func (c *RegionCache) loadRegionByID(bo *retry.Backoffer, regionID uint64) (*Reg
 	var backoffErr error
 	for {
 		if backoffErr != nil {
-			err := bo.Backoff(retry.BoPDRPC, backoffErr)
+			err := bo.Backoff(retry.BoPDRegionMetadata, backoffErr)
 			if err != nil {
 				return nil, errors.WithStack(err)
 			}
@@ -2124,6 +2128,10 @@ func (c *RegionCache) loadRegionByID(bo *retry.Backoffer, regionID uint64) (*Reg
 			metrics.RegionCacheCounterWithGetRegionByIDError.Inc()
 		} else {
 			metrics.RegionCacheCounterWithGetRegionByIDOK.Inc()
+			if backoffErr == nil {
+				// refill retry allowance only for the original call
+				bo.OnSuccess(retry.BoPDRegionMetadata)
+			}
 		}
 		if err != nil {
 			if apicodec.IsDecodeError(err) {