fix wrong backoffer for pd client

Signed-off-by: iosmanthus <[email protected]>
tikv · Sep 2, 2022 · c615180 · c615180
1 parent 1b5edcd
commit c615180
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 8 deletions.
diff --git a/src/main/java/org/tikv/common/AbstractGRPCClient.java b/src/main/java/org/tikv/common/AbstractGRPCClient.java
@@ -193,13 +193,17 @@ private boolean doCheckHealth(BackOffer backOffer, String addressStr, HostMappin
         HealthCheckResponse resp = stub.check(req);
         return resp.getStatus() == HealthCheckResponse.ServingStatus.SERVING;
       } catch (Exception e) {
-        logger.warn("check health failed.", e);
+        logger.warn("check health failed, addr: {}, caused by: {}", addressStr, e.getMessage());
         backOffer.doBackOff(BackOffFuncType.BoCheckHealth, e);
       }
     }
   }
 
   protected boolean checkHealth(BackOffer backOffer, String addressStr, HostMapping hostMapping) {
-    return doCheckHealth(backOffer, addressStr, hostMapping);
+    try {
+      return doCheckHealth(backOffer, addressStr, hostMapping);
+    } catch (Exception e) {
+      return false;
+    }
   }
 }
diff --git a/src/main/java/org/tikv/common/PDClient.java b/src/main/java/org/tikv/common/PDClient.java
@@ -462,14 +462,19 @@ private GetMembersResponse doGetMembers(BackOffer backOffer, URI uri) {
         }
         return resp;
       } catch (Exception e) {
-        logger.warn("failed to get member from pd server.", e);
+        logger.warn(
+            "failed to get member from pd server from {}, caused by: {}", uri, e.getMessage());
         backOffer.doBackOff(BackOffFuncType.BoPDRPC, e);
       }
     }
   }
 
   private GetMembersResponse getMembers(BackOffer backOffer, URI uri) {
-    return doGetMembers(backOffer, uri);
+    try {
+      return doGetMembers(backOffer, uri);
+    } catch (Exception e) {
+      return null;
+    }
   }
 
   // return whether the leader has changed to target address `leaderUrlStr`.
@@ -524,13 +529,16 @@ synchronized boolean createFollowerClientWrapper(
   public void tryUpdateLeaderOrForwardFollower() {
     if (updateLeaderNotify.compareAndSet(false, true)) {
       try {
-        BackOffer backOffer = defaultBackOffer();
         updateLeaderService.submit(
             () -> {
               try {
-                updateLeaderOrForwardFollower(backOffer);
+                updateLeaderOrForwardFollower();
+              } catch (Exception e) {
+                logger.info("update leader or forward follower failed", e);
+                throw e;
               } finally {
                 updateLeaderNotify.set(false);
+                logger.info("updating leader finish");
               }
             });
       } catch (RejectedExecutionException e) {
@@ -540,11 +548,13 @@ public void tryUpdateLeaderOrForwardFollower() {
     }
   }
 
-  private synchronized void updateLeaderOrForwardFollower(BackOffer backOffer) {
+  private synchronized void updateLeaderOrForwardFollower() {
+    logger.warn("updating leader or forward follower");
     if (System.currentTimeMillis() - lastUpdateLeaderTime < MIN_TRY_UPDATE_DURATION) {
       return;
     }
     for (URI url : this.pdAddrs) {
+      BackOffer backOffer = this.probeBackOffer();
       // since resp is null, we need update leader's address by walking through all pd server.
       GetMembersResponse resp = getMembers(backOffer, url);
       if (resp == null) {
@@ -602,8 +612,9 @@ && createFollowerClientWrapper(backOffer, followerUrlStr, leaderUrlStr)) {
   }
 
   public void tryUpdateLeader() {
+    logger.info("try update leader");
     for (URI url : this.pdAddrs) {
-      BackOffer backOffer = defaultBackOffer();
+      BackOffer backOffer = this.probeBackOffer();
       // since resp is null, we need update leader's address by walking through all pd server.
       GetMembersResponse resp = getMembers(backOffer, url);
       if (resp == null) {
@@ -856,4 +867,9 @@ public RequestKeyCodec getCodec() {
   private static BackOffer defaultBackOffer() {
     return ConcreteBackOffer.newCustomBackOff(BackOffer.PD_INFO_BACKOFF);
   }
+
+  private BackOffer probeBackOffer() {
+    int maxSleep = (int) getTimeout() * 2;
+    return ConcreteBackOffer.newCustomBackOff(maxSleep);
+  }
 }