refactor(pytest): make test more robust and easier to debug

.github/workflows/k8s-ci.yml

@@ -4,7 +4,7 @@ on:
 
 jobs:
   k8s-ci:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps:
       - name: Bind mount /dev/sda1 to /nix
         run: |
@@ -18,30 +18,27 @@ jobs:
         with:
           kvm: true
       - uses: DeterminateSystems/magic-nix-cache-action@v6
-      - name: Pre-populate nix-shell
+      - name: Setup Nix Path
         run: |
           export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
           echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
-          nix-shell ./scripts/k8s/shell.nix --run "echo"
-      - name: Build binaries and images
-        id: build
-        run: |
-          TAG=$(nix-shell ./shell.nix --run './scripts/python/generate-test-tag.sh')
-          TEST_DIR=$(realpath $(mktemp -d ./test-dir-XXXXXX))
-          nix-shell ./shell.nix --run "./scripts/python/tag-chart.sh $TAG"
-          RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" ./scripts/release.sh --tag $TAG --build-binary-out $TEST_DIR --no-static-linking --skip-publish --debug
-          echo "tag=$TAG" >> $GITHUB_OUTPUT
-          echo "bin=$TEST_DIR" >> $GITHUB_OUTPUT
+      - name: Pre-populate K8s nix-shell
+        run: nix-shell ./scripts/k8s/shell.nix --run "echo"
+      - name: Pre-populate helm nix-shell
+        run: nix-shell ./scripts/helm/shell.nix --run "echo"
+      - name: Pre-populate pytest nix-shell
+        run: nix-shell ./scripts/python/shell.nix --run "echo"
       - name: BootStrap k8s cluster
-        run: |
-          nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label"
-      - name: Load images to Kind cluster
-        run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/load-images-to-kind.sh --tag ${{ steps.build.outputs.tag }} --trim-debug-suffix"
+        run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label"
+      - name: Prepare vnext images and binary
+        run: nix-shell ./scripts/python/shell.nix --run "./scripts/python/upgrade-test-helper.sh --build --chart-tag --chart ./chart"
+      - name: Load images to kind cluster
+        run: nix-shell ./scripts/python/shell.nix --run "./scripts/python/upgrade-test-helper.sh --load"
       - name: Run Pytests
         run: |
-          export UPGRADE_TARGET_VERSION=${{ steps.build.outputs.tag }}
-          export TEST_DIR=${{ steps.build.outputs.bin }}
-          nix-shell ./shell.nix --run "./scripts/python/test.sh"
+          export REUSE_CLUSTER=1
+          export CHART_VNEXT_SKIP=1
+          nix-shell ./scripts/python/shell.nix --run "./scripts/python/test.sh"
       - name: The job has failed
         if: ${{ failure() }}
         run: |
@@ -50,7 +47,7 @@ jobs:
           nix-shell ./scripts/k8s/shell.nix --run "kubectl -n mayastor logs -l app=upgrade --all-containers=true"
 
   k8s-ci-vm:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@v11
@@ -61,8 +58,8 @@ jobs:
         run: |
           export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
           echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
-          nix-shell ./scripts/k8s/shell.nix --run "echo"
+          nix-shell ./scripts/helm/shell.nix --run "echo"
       - name: Test on VM
         run: |
-          nix-shell ./scripts/k8s/shell.nix --run "cd chart; helm dependency update"
+          nix-shell ./scripts/helm/shell.nix --run "cd chart; helm dependency update"
           nix-build ./tests/helm/test.nix --option sandbox false

.github/workflows/release-chart.yml

@@ -55,7 +55,13 @@ jobs:
         run: |
           tag="${{ github.ref_name }}"
           BASE_REF="${{ github.event.base_ref }}"
-          BRANCH="${BASE_REF#refs/heads/}"
+          if [ -n "$BASE_REF" ]; then
+            BRANCH="${BASE_REF#refs/heads/}"
+          else
+            BRANCH="$(nix-shell --pure --run "./scripts/helm/find-released-branch.sh "$tag"" ./scripts/helm/shell.nix)"
+          fi
+          echo "BASE_BRANCH=$BRANCH" >> $GITHUB_ENV
+
           nix-shell --pure --run "./scripts/helm/publish-chart-yaml.sh --released "$tag" --released-branch "$BRANCH"" ./scripts/helm/shell.nix
           nix-shell --pure --run "SKIP_GIT=1 ./scripts/helm/generate-readme.sh" ./scripts/helm/shell.nix
       - name: Create Pull Request
@@ -73,7 +79,7 @@ jobs:
           signoff: true
           delete-branch: true
           branch-suffix: "random"
-          base: ${{ github.event.base_ref }}
+          base: ${{ env.BASE_BRANCH }}
           token: ${{ secrets.ORG_CI_GITHUB }}
       - name: Approve Pull Request by CI Bot
         if: ${{ steps.cpr.outputs.pull-request-number }}

.gitignore

@@ -24,4 +24,6 @@ __pycache__
 # Pytest assets
 /test-dir-*
 tests/bdd/venv
-pytest.log
+pytest.log
+#/tests/bdd/chart-vnext/
+/chart/kubectl-plugin

chart/.helmignore

@@ -24,3 +24,4 @@
 README.md.tmpl
 # Nix Shell
 *.nix
+kubectl-plugin/

chart/shell.nix

@@ -1,16 +1,10 @@
-{}:
-let
-  sources = import ../nix/sources.nix;
-  pkgs = import sources.nixpkgs {
-    overlays = [ (_: _: { inherit sources; }) (import ../nix/overlay.nix { }) ];
-  };
-in
-with pkgs;
-let
-in
-mkShell {
+{ pkgs ? import (import ../nix/sources.nix).nixpkgs {
+    overlays = [ (_: _: { inherit (import ../nix/sources.nix); }) (import ../nix/overlay.nix { }) ];
+  }
+}:
+pkgs.mkShell {
   name = "helm-scripts-shell";
-  buildInputs = [
+  buildInputs = with pkgs; [
     coreutils
     git
     helm-docs

nix/pkgs/images/default.nix

@@ -26,7 +26,7 @@ let
       } // config;
     };
   build-exporter-image = { buildType }: {
-    io-engine = build-extensions-image rec{
+    io-engine = build-extensions-image rec {
       inherit buildType;
       package = extensions.${buildType}.metrics.exporter.io-engine;
       pname = package.pname;
@@ -49,6 +49,9 @@ let
     patchShebangs build/scripts/helm/publish-chart-yaml.sh
     patchShebangs build/scripts/helm/generate-consolidated-values.sh
     patchShebangs build/scripts/utils/log.sh
+    if [ -L build/chart/kubectl-plugin ]; then
+      rm build/chart/kubectl-plugin
+    fi
 
     # if tag is not semver just keep whatever is checked-in
     # todo: handle this properly?

pytest.ini

@@ -5,3 +5,4 @@ log_cli_format = %(asctime)s [%(levelname)s] %(message)s
 log_cli_date_format = %Y-%m-%d %H:%M:%S
 log_file = pytest.log
 log_file_level = DEBUG
+addopts = --log-disable kubernetes.client.rest

scripts/helm/find-released-branch.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+SCRIPTDIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")"
+ROOTDIR="$SCRIPTDIR/../.."
+
+source "$ROOTDIR/scripts/utils/log.sh"
+
+set -euo pipefail
+
+# Check if the given branch matches the tag
+# Example:
+# release/2.7 matches tag v2.7.2, v2.7.2-rc.4, etc..
+# release/2.7 does not match tag v2.6.0, etc...
+tag_matches_branch() {
+  local tag="${1#v}"
+  local release_branch="$2"
+
+  branch_version="${release_branch#release/}"
+  if ! [[ "$branch_version" =~ ^[0-9]+.[0-9]+$ ]]; then
+    return 1
+  fi
+
+  if ! [[ "$tag" = "$branch_version"* ]]; then
+    return 1
+  fi
+}
+
+# For the given tag, find the branch which is compatible
+# See tag_matches_branch for more information.
+find_released_branch() {
+  local tag="$1"
+  local branches=$(git branch -r --contains "$TAG" --points-at "$TAG" --format "%(refname:short)" 2>/dev/null)
+  local branch=""
+
+  for release_branch in $branches; do
+    release_branch=${release_branch#origin/}
+    if tag_matches_branch "$TAG" "$release_branch"; then
+      if [ -n "$branch" ]; then
+        log_fatal "Multiple branches matched!"
+      fi
+      branch="$release_branch"
+    fi
+  done
+
+  echo "$branch"
+}
+
+TAG="$1"
+BRANCH="$(find_released_branch "$TAG")"
+
+if [ -z "$BRANCH" ]; then
+  log_fatal "Failed to find matching released branch for tag '$TAG'"
+fi
+
+echo "$BRANCH"

scripts/helm/install.sh

@@ -7,23 +7,23 @@ repo_add() {
   local -r preferred_name=$2
 
   local repo
-  if [ "$(helm repo ls -o yaml | yq "contains([{\"url\": \"$url\"}])")" = "true" ]; then
-    repo=$(helm repo ls -o yaml | yq ".[] | select(.url == \"$url\") | .name")
+  if [ -z "$DRY_RUN" ] && [ "$($HELM repo ls -o yaml | yq "contains([{\"url\": \"$url\"}])")" = "true" ]; then
+    repo=$($HELM repo ls -o yaml | yq ".[] | select(.url == \"$url\") | .name")
   else
-    helm repo add "$preferred_name" "$url" > /dev/null
+    $HELM repo add "$preferred_name" "$url" > /dev/null
     repo=$preferred_name
   fi
 
-  helm repo update > /dev/null || true
+  $HELM repo update > /dev/null || true
 
   echo "$repo"
 }
 
 
 TIMEOUT="5m"
 WAIT=
-DRY_RUN=""
-CHART=
+DRY_RUN=
+HELM_DRY_RUN=""
 SCRIPT_DIR="$(dirname "$0")"
 CHART_DIR="$SCRIPT_DIR"/../../chart
 CHART_SOURCE=$CHART_DIR
@@ -35,6 +35,8 @@ HOSTED=
 VERSION=
 REGISTRY=
 DEFAULT_REGISTRY="https://openebs.github.io/mayastor-extensions"
+HELM="helm"
+KUBECTL="kubectl"
 
 help() {
   cat <<EOF
@@ -44,7 +46,8 @@ Options:
   -h, --help                     Display this text.
   --timeout  <timeout>           How long to wait for helm to complete install (Default: $TIMEOUT).
   --wait                         Wait for helm to complete install.
-  --dry-run                      Install helm with --dry-run.
+  --dry-run                      Don't run any commands, output them only.
+  --helm-dry-run                 Install helm with --dry-run.
   --dep-update                   Run helm dependency update.
   --fail-if-installed            Fail with a status code 1 if the helm release '$RELEASE_NAME' already exists in the $K8S_NAMESPACE namespace.
   --hosted-chart                 Install a hosted chart instead of the local chart.
@@ -80,8 +83,13 @@ while [ "$#" -gt 0 ]; do
     --wait)
       WAIT="yes"
       shift;;
+    --helm-dry-run)
+      HELM_DRY_RUN=" --dry-run"
+      shift;;
     --dry-run)
-      DRY_RUN=" --dry-run"
+      DRY_RUN="yes"
+      HELM="echo $HELM"
+      KUBECTL="echo $KUBECTL"
       shift;;
     --dep-update)
       DEP_UPDATE="y"
@@ -137,21 +145,21 @@ if [ -n "$HOSTED" ]; then
   DEP_UPDATE_ARG=
 fi
 
-if [ "$(helm ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then
-  already_exists_log= "Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE"
+if [ -z "$DRY_RUN" ] && [ "$($HELM ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then
+  already_exists_log="Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE"
   if [ -n "$FAIL_IF_INSTALLED" ]; then
     die "ERROR: $already_exists_log" 1
   fi
   echo "$already_exists_log"
 else
   echo "Installing Mayastor Chart"
   set -x
-  helm install "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \
+  $HELM install "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \
        --set="etcd.livenessProbe.initialDelaySeconds=5,etcd.readinessProbe.initialDelaySeconds=5,etcd.replicaCount=1" \
        --set="obs.callhome.enabled=true,obs.callhome.sendReport=false,localpv-provisioner.analytics.enabled=false" \
        --set="eventing.enabled=false" \
-       $DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG $VERSION_ARG
+       $HELM_DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG $VERSION_ARG
   set +x
 fi
 
-kubectl get pods -n "$K8S_NAMESPACE" -o wide
+$KUBECTL get pods -n "$K8S_NAMESPACE" -o wide

scripts/k8s/deployer.sh

@@ -16,6 +16,7 @@ KUBECTL="kubectl"
 DOCKER="docker"
 HUGE_PAGES=1800
 LABEL=
+CLEANUP="false"
 SUDO=${SUDO:-"sudo"}
 
 help() {
@@ -31,6 +32,7 @@ Options:
   --dry-run                         Don't do anything, just output steps.
   --hugepages     <num>             Add <num> 2MiB hugepages (Default: $HUGE_PAGES).
   --label                           Label worker nodes with the io-engine selector.
+  --cleanup                         Prior to starting, stops the running instance of the deployer.
 
 Command:
   start                             Start the k8s cluster.
@@ -90,6 +92,9 @@ while [ "$#" -gt 0 ]; do
         --label)
           LABEL="true"
           shift;;
+        --cleanup)
+          CLEANUP="true"
+          shift;;
         --hugepages)
           shift
           test $# -lt 1 && die "Missing hugepage number"
@@ -115,12 +120,14 @@ if [ -z "$COMMAND" ]; then
   die "No command specified!"
 fi
 
-if [ "$COMMAND" = "stop" ]; then
-  if command -v nvme 2>dev/null; then
+if [ "$COMMAND" = "stop" ] || [ "$CLEANUP" = "true" ]; then
+  if command -v nvme &>/dev/null; then
     $SUDO nvme disconnect-all
   fi
   $KIND delete cluster
-  exit 0
+  if [ "$COMMAND" = "stop" ]; then
+    exit 0
+  fi
 fi
 
 "$SCRIPT_DIR"/setup-io-prereq.sh --hugepages "$HUGE_PAGES" --nvme-tcp $DRY_RUN

scripts/k8s/setup-io-prereq.sh

@@ -35,7 +35,13 @@ die() {
 }
 
 setup_hugepages() {
-  $SYSCTL -w vm.nr_hugepages="$1"
+  local wanted="$1"
+  $SYSCTL -w vm.nr_hugepages="$wanted"
+  local actual
+  actual=$($SYSCTL -b vm.nr_hugepages)
+  if [ "$actual" != "$wanted" ]; then
+    die "Wanted $wanted hugepages, but only allocated $actual hugepages"
+  fi
 }
 
 modprobe_nvme_tcp() {
@@ -99,7 +105,7 @@ while [ "$#" -gt 0 ]; do
 done
 
 if [ -n "$HUGE_PAGES" ]; then
-  pages=$(sysctl -b vm.nr_hugepages)
+  pages=$($SYSCTL -b vm.nr_hugepages)
 
   if [ "$HUGE_PAGES" -gt "$pages" ]; then
     setup_hugepages "$HUGE_PAGES"

scripts/k8s/shell.nix

@@ -1,8 +1,8 @@
+{ pkgs ? import (import ../../nix/sources.nix).nixpkgs {
+    overlays = [ (_: _: { inherit (import ../../nix/sources.nix); }) (import ../../nix/overlay.nix { }) ];
+  }
+}:
 let
-  sources = import ../../nix/sources.nix;
-  pkgs = import sources.nixpkgs {
-    overlays = [ (_: _: { inherit sources; }) (import ../../nix/overlay.nix { }) ];
-  };
   inPureNixShell = builtins.getEnv "IN_NIX_SHELL" == "pure";
 in
 pkgs.mkShell {

scripts/python/shell.nix

@@ -0,0 +1 @@
+../../tests/bdd/shell.nix