Merge pull request #337 from rushikeshjadhav/linstor-retry-UpToDate-check

tests/storage/linstor: Add retry logic to _get_diskful_hosts to handle transient failures

Author: stormi

tests/storage/linstor/test_linstor_sr.py

@@ -138,17 +138,30 @@ def test_linstor_missing(self, linstor_sr, host):
 # --- Test diskless resources --------------------------------------------------
 
 def _get_diskful_hosts(host, controller_option, volume_name):
-    # Find host where volume is diskless
-    # | {volume_name} | {host} | 7017 | Unused | Ok    |   UpToDate | 2023-10-24 18:52:05 |
-    lines = host.ssh([
-        "linstor", controller_option, "resource", "list",
-        "|", "grep", volume_name, "|", "grep", "UpToDate"
-    ]).splitlines()
-    diskfuls = []
-    for line in lines:
-        hostname = line.split('|')[2].strip()
-        diskfuls += hostname
-    return diskfuls
+    # TODO: If any resource is in a temporary creation state or unknown, then need to wait intelligently.
+    attempt = 0
+    retries = 3
+    sleep_sec = 5
+
+    while attempt < retries:
+        try:
+            # Find host where volume is UpToDate
+            # | {volume_name} | {host} | 7017 | Unused | Ok    |   UpToDate | 2023-10-24 18:52:05 |
+            lines = host.ssh([
+                "linstor", controller_option, "resource", "list",
+                "|", "grep", volume_name, "|", "grep", "UpToDate"
+            ]).splitlines()
+            diskfuls = []
+            for line in lines:
+                hostname = line.split('|')[2].strip()
+                diskfuls += hostname
+            return diskfuls
+        except SSHCommandFailed as e:
+            logging.error("SSH Command Failed (attempt %d/%d): %s", attempt + 1, retries, e)
+            attempt += 1
+            if attempt >= retries:
+                raise
+            time.sleep(sleep_sec)
 
 def _ensure_resource_remain_diskless(host, controller_option, volume_name, diskless):
     diskfuls = _get_diskful_hosts(host, controller_option, volume_name)