Try #598:

scripts/helm/install.sh

@@ -23,7 +23,6 @@ repo_add() {
 TIMEOUT="5m"
 WAIT=
 DRY_RUN=""
-CHART=
 SCRIPT_DIR="$(dirname "$0")"
 CHART_DIR="$SCRIPT_DIR"/../../chart
 CHART_SOURCE=$CHART_DIR
@@ -138,7 +137,7 @@ if [ -n "$HOSTED" ]; then
 fi
 
 if [ "$(helm ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then
-  already_exists_log= "Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE"
+  already_exists_log="Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE"
   if [ -n "$FAIL_IF_INSTALLED" ]; then
     die "ERROR: $already_exists_log" 1
   fi

scripts/python/test.sh

@@ -41,8 +41,6 @@ while test $# -gt 0; do
     exit 0
     ;;
   *)
-    print_help
-    log_fatal "unexpected argument '$arg'" 1
     ;;
   esac
   shift

tests/bdd/common/environment.py

@@ -4,14 +4,17 @@
 logger = logging.getLogger(__name__)
 
 
-def get_env(variable: str):
-    try:
-        value = os.getenv(variable)
-        if len(value) == 0:
-            raise ValueError("Env {variable} is empty")
-        logger.info(f"Found env {variable}={value}")
-        return value
+def get_env(variable: str, warn=True):
+    value = os.getenv(variable)
+    if value is None:
+        if warn:
+            logger.warning(f"The env {variable} does not exist")
+        return None
 
-    except Exception as e:
-        logger.error(f"Failed to get env {variable}: {e}")
+    if len(value) == 0:
+        if warn:
+            logger.warning(f"The env {variable} is an empty string")
         return None
+
+    logger.info(f"Found env {variable}={value}")
+    return value

tests/bdd/common/helm.py

@@ -126,12 +126,12 @@ def latest_chart_so_far(version=None):
 
 class ChartSource(Enum):
     HOSTED = [
-        "/bin/bash",
+        "bash",
         "-c",
         os.path.join(root_dir(), "scripts/helm/install.sh") + " --hosted-chart --wait",
     ]
     LOCAL = [
-        "/bin/bash",
+        "bash",
         "-c",
         os.path.join(root_dir(), "scripts/helm/install.sh") + " --dep-update --wait",
     ]
@@ -176,31 +176,33 @@ def get_metadata_mayastor(self):
                 f"Error: command '{command}' failed with exit code {e.returncode}"
             )
             logger.error(f"Error Output: {e.stderr}")
-            return None
+            raise e
 
         except Exception as e:
             logger.error(f"An unexpected error occurred: {e}")
-            return None
+            raise e
 
-    def list(self):
+    def get_deployed(self, release: str):
         """
-        Lists the deployed Helm releases in the specified namespace.
+        Get the deployed Helm release in the specified namespace as json
 
         Executes the 'helm ls' command to retrieve a list of deployed releases.
 
         Returns:
             str: A newline-separated string of deployed release names, or None if an error occurs.
         """
+        args = [
+            helm_bin,
+            "ls",
+            "-n",
+            self.namespace,
+            "--deployed",
+            f"--filter=^{release}$",
+            "-o=json",
+        ]
         try:
             result = subprocess.run(
-                [
-                    helm_bin,
-                    "ls",
-                    "-n",
-                    self.namespace,
-                    "--deployed",
-                    "--short",
-                ],
+                args,
                 capture_output=True,
                 check=True,
                 text=True,
@@ -209,28 +211,23 @@ def list(self):
 
         except subprocess.CalledProcessError as e:
             logger.error(
-                f"Error: command 'helm ls -n {self.namespace} --deployed --short' failed with exit code {e.returncode}"
+                f"command '{args}' failed with exit code {e.returncode}"
             )
             logger.error(f"Error Output: {e.stderr}")
-            return None
+            raise e
 
         except Exception as e:
             logger.error(f"An unexpected error occurred: {e}")
-            return None
-
-    def release_is_deployed(self, release_name: str):
-        releases = self.list()
-        if releases is not None:
-            for release in releases:
-                if release == release_name:
-                    return True
-        return False
+            raise e
 
     def install_mayastor(self, source: ChartSource, version=None):
-        if self.release_is_deployed("mayastor"):
-            logger.error(
-                f"WARN: Helm release 'mayastor' already exists in the 'mayastor' namespace."
+        output_json = json.loads(self.get_deployed("mayastor"))
+        if len(output_json) == 1:
+            version = output_json[0]["app_version"]
+            logger.warning(
+                f"Helm release 'mayastor' already exists in the 'mayastor' namespace @ v{version}."
             )
+            assert output_json[0]["app_version"] == version
             return
 
         install_command = []
@@ -261,11 +258,11 @@ def install_mayastor(self, source: ChartSource, version=None):
                 f"Error: command {install_command} failed with exit code {e.returncode}"
             )
             logger.error(f"Error Output: {e.stderr}")
-            return None
+            raise e
 
         except Exception as e:
             logger.error(f"An unexpected error occurred: {e}")
-            return None
+            raise e
 
 
 def generate_test_tag():

tests/bdd/common/kubectl_mayastor.py

@@ -12,14 +12,18 @@ def get_bin_path():
     bins = get_env("TEST_DIR")
     if bins:
         return os.path.join(bins, "kubectl-plugin/bin/kubectl-mayastor")
-    logging.warning(f"Environmental variable 'BIN' is not set")
-    return which("kubectl-mayastor")
+    bin = which("kubectl-mayastor")
+    if bin is None:
+        msg = f"Failed to find kubectl-mayastor binary"
+        logging.error(msg)
+        raise Exception(msg)
+    return bin
 
 
 def kubectl_mayastor(args: list[str]):
     command = [get_bin_path()]
     command.extend(args)
-    logger.info(f"Running kubectl-mayastor command: {command}")
+    logger.info(f"Running kubectl-mayastor: {command}")
 
     try:
         result = subprocess.run(
@@ -34,8 +38,8 @@ def kubectl_mayastor(args: list[str]):
     except subprocess.CalledProcessError as e:
         logger.error(f"Error: command '{command}' failed with exit code {e.returncode}")
         logger.error(f"Error Output: {e.stderr}")
-        return None
+        raise e
 
     except Exception as e:
         logger.error(f"An unexpected error occurred: {e}")
-        return None
+        raise e

tests/bdd/common/repo.py

@@ -11,9 +11,8 @@ def root_dir():
 
 
 def run_script(script: str):
-    script = os.path.join(root_dir(), script)
-    logger.info(f"Running script '{script}'")
-    command = ["/bin/bash", "-c", script]
+    command = os.path.join(root_dir(), script)
+    logger.info(f"Running script '{command}'")
     try:
         result = subprocess.run(
             command,
@@ -28,8 +27,8 @@ def run_script(script: str):
     except subprocess.CalledProcessError as e:
         logger.error(f"Error: command {command} failed with exit code {e.returncode}")
         logger.error(f"Error Output: {e.stderr}")
-        return None
+        raise e
 
     except Exception as e:
         logger.error(f"An unexpected error occurred: {e}")
-        return None
+        raise e

tests/bdd/features/test_upgrade.py

@@ -2,6 +2,8 @@
 
 import logging
 
+import pytest
+
 from common.environment import get_env
 from common.helm import ChartSource, HelmReleaseClient, latest_chart_so_far
 from common.kubectl_mayastor import kubectl_mayastor
@@ -23,13 +25,13 @@ def test_upgrade_command_is_issued():
 @given("an installed mayastor helm chart")
 def an_installed_mayastor_helm_chart():
     """an installed mayastor helm chart."""
-    assert helm.install_mayastor(ChartSource.HOSTED, latest_chart_so_far()) is not None
+    helm.install_mayastor(ChartSource.HOSTED, latest_chart_so_far())
 
 
 @when("a kubectl mayastor upgrade command is issued")
 def a_kubectl_mayastor_upgrade_command_is_issued():
     """a kubectl mayastor upgrade command is issued."""
-    assert kubectl_mayastor(["upgrade"]) is not None
+    kubectl_mayastor(["upgrade"])
 
 
 @then("the installed chart should be upgraded to the kubectl mayastor plugin's version")
@@ -42,15 +44,23 @@ def the_installed_chart_should_be_upgraded_to_the_kubectl_mayastor_plugins_versi
     upgrade_target_version = upgrade_target_version.lstrip("v")
     logger.info(f"Value of upgrade_target_version={upgrade_target_version}")
 
+    def log_it():
+        log = (pytest.attempts % 10) == 0
+        pytest.attempts += 1
+        return log
+
     @retry(
         stop_max_attempt_number=450,
         wait_fixed=2000,
     )
     def helm_upgrade_succeeded():
-        logger.info("Checking if helm upgrade succeeded...")
+        log = log_it()
+        if log:
+            logger.info("Checking if helm upgrade succeeded...")
         metadata = helm.get_metadata_mayastor()
-        logger.debug(f"helm get metadata output={metadata}")
-        logger.debug(f"upgrade_target_version={upgrade_target_version}")
+        if log:
+            logger.debug(f"helm get metadata output={metadata}")
+            logger.debug(f"upgrade_target_version={upgrade_target_version}")
         if metadata:
             assert metadata["version"] == upgrade_target_version
             return
@@ -60,25 +70,40 @@ def helm_upgrade_succeeded():
         stop_max_attempt_number=600,
         wait_fixed=2000,
     )
-    def data_plane_upgrade_succeeded():
-        logger.info("Checking if data-plane upgrade succeeded...")
+    def data_plane_upgrade_succeeded(target_tag):
+        log = log_it()
+        if log:
+            logger.info("Checking if data-plane upgrade succeeded...")
         config.load_kube_config()
         v1 = client.CoreV1Api()
         label_selector = "app=io-engine"
         pods = v1.list_namespaced_pod(
             namespace="mayastor", label_selector=label_selector
         )
-        switch = True
-        for pod in pods.items:
+        io_engines = list(
+            filter(
+                lambda pod: any(container.name == 'io-engine' for container in pod.spec.containers),
+                pods.items
+            )
+        )
+        if len(io_engines) == 0:
+            return
+
+        all_done = True
+        for pod in io_engines:
             for i, container in enumerate(pod.spec.containers):
                 if container.name == "io-engine":
-                    logger.info(
-                        f"pod.metadata.name={pod.metadata.name}, pod.spec.containers[{i}].image={container.image}"
-                    )
-                    switch = switch and container.image.endswith(":develop")
-                    logger.info(f"Value of 'switch' after the AND={switch}")
+                    if not container.image.endswith(f":{target_tag}"):
+                        all_done = False
+                    if log:
+                        logger.info(
+                            f"pod.metadata.name={pod.metadata.name}, pod.spec.containers[{i}].image={container.image}"
+                        )
                     break
-        assert switch
+        assert all_done is True
 
+    pytest.attempts = 0
     helm_upgrade_succeeded()
-    data_plane_upgrade_succeeded()
+    pytest.attempts = 0
+    # todo: should be release-$v on release branches
+    data_plane_upgrade_succeeded("develop")