Skip to content

Commit

Permalink
test(bdd): add vnext builtin the bdd tests
Browse files Browse the repository at this point in the history
Allows the bdd tests to run and setup the vnext chart on their own
We can still pre-deploy the kind cluster on CI for time tracking

Signed-off-by: Tiago Castro <[email protected]>
  • Loading branch information
tiagolobocastro committed Jan 14, 2025
1 parent 6bdd64a commit 3fe353c
Show file tree
Hide file tree
Showing 14 changed files with 433 additions and 213 deletions.
22 changes: 7 additions & 15 deletions .github/workflows/k8s-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,16 @@ jobs:
run: nix-shell ./scripts/helm/shell.nix --run "echo"
- name: Pre-populate pytest nix-shell
run: nix-shell ./scripts/python/shell.nix --run "echo"
- name: Build binaries and images
id: build
run: |
TAG=$(nix-shell ./scripts/helm/shell.nix --run './scripts/python/generate-test-tag.sh')
TEST_DIR=$(realpath $(mktemp -d ./test-dir-XXXXXX))
nix-shell ./scripts/helm/shell.nix --run "./scripts/python/tag-chart.sh $TAG"
RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" ./scripts/release.sh --tag $TAG --build-binary-out $TEST_DIR --no-static-linking --skip-publish --debug
echo "tag=$TAG" >> $GITHUB_OUTPUT
echo "bin=$TEST_DIR" >> $GITHUB_OUTPUT
- name: BootStrap k8s cluster
run: |
nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label"
- name: Load images to Kind cluster
run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/load-images-to-kind.sh --tag ${{ steps.build.outputs.tag }} --trim-debug-suffix"
run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label"
- name: Prepare vnext images and binary
run: nix-shell ./scripts/python/shell.nix --run "./scripts/python/upgrade-test-helper.sh --build --chart-tag --chart ./chart"
- name: Load images to kind cluster
run: nix-shell ./scripts/python/shell.nix --run "./scripts/python/upgrade-test-helper.sh --load"
- name: Run Pytests
run: |
export UPGRADE_TARGET_VERSION=${{ steps.build.outputs.tag }}
export TEST_DIR=${{ steps.build.outputs.bin }}
export REUSE_CLUSTER=1
export CHART_VNEXT_SKIP=1
nix-shell ./scripts/python/shell.nix --run "./scripts/python/test.sh"
- name: The job has failed
if: ${{ failure() }}
Expand Down
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,6 @@ __pycache__
# Pytest assets
/test-dir-*
tests/bdd/venv
pytest.log
pytest.log
#/tests/bdd/chart-vnext/
/chart/kubectl-plugin
1 change: 1 addition & 0 deletions chart/.helmignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,4 @@
README.md.tmpl
# Nix Shell
*.nix
kubectl-plugin/
5 changes: 4 additions & 1 deletion nix/pkgs/images/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ let
} // config;
};
build-exporter-image = { buildType }: {
io-engine = build-extensions-image rec{
io-engine = build-extensions-image rec {
inherit buildType;
package = extensions.${buildType}.metrics.exporter.io-engine;
pname = package.pname;
Expand All @@ -49,6 +49,9 @@ let
patchShebangs build/scripts/helm/publish-chart-yaml.sh
patchShebangs build/scripts/helm/generate-consolidated-values.sh
patchShebangs build/scripts/utils/log.sh
if [ -L build/chart/kubectl-plugin ]; then
rm build/chart/kubectl-plugin
fi
# if tag is not semver just keep whatever is checked-in
# todo: handle this properly?
Expand Down
13 changes: 10 additions & 3 deletions scripts/k8s/deployer.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ KUBECTL="kubectl"
DOCKER="docker"
HUGE_PAGES=1800
LABEL=
CLEANUP="false"
SUDO=${SUDO:-"sudo"}

help() {
Expand All @@ -31,6 +32,7 @@ Options:
--dry-run Don't do anything, just output steps.
--hugepages <num> Add <num> 2MiB hugepages (Default: $HUGE_PAGES).
--label Label worker nodes with the io-engine selector.
--cleanup Prior to starting, stops the running instance of the deployer.
Command:
start Start the k8s cluster.
Expand Down Expand Up @@ -90,6 +92,9 @@ while [ "$#" -gt 0 ]; do
--label)
LABEL="true"
shift;;
--cleanup)
CLEANUP="true"
shift;;
--hugepages)
shift
test $# -lt 1 && die "Missing hugepage number"
Expand All @@ -115,12 +120,14 @@ if [ -z "$COMMAND" ]; then
die "No command specified!"
fi

if [ "$COMMAND" = "stop" ]; then
if command -v nvme 2>dev/null; then
if [ "$COMMAND" = "stop" ] || [ "$CLEANUP" = "true" ]; then
if command -v nvme &>/dev/null; then
$SUDO nvme disconnect-all
fi
$KIND delete cluster
exit 0
if [ "$COMMAND" = "stop" ]; then
exit 0
fi
fi

"$SCRIPT_DIR"/setup-io-prereq.sh --hugepages "$HUGE_PAGES" --nvme-tcp $DRY_RUN
Expand Down
2 changes: 1 addition & 1 deletion scripts/python/tag-chart.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ fi

CHART_VERSION=${1#v}
IMAGE_TAG="v$CHART_VERSION"
CHART_DIR="$ROOT_DIR/chart"
CHART_DIR=${CHART_DIR:-"$ROOT_DIR/chart"}
# TODO: tests should work with its own copy of the chart. Shouldn't modify the chart.
# chart/Chart.yaml
yq_ibl "
Expand Down
137 changes: 137 additions & 0 deletions scripts/python/upgrade-test-helper.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
#!/usr/bin/env bash

# "Fork" the local chart into a separate location and build the container images and the plugin binary
# with a given TAG (or figures out the tag using $SCRIPT_DIR/build-upgrade-images.sh).
# This is used by the upgrade test

SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")"
ROOT_DIR="$SCRIPT_DIR/../.."

# Imports
source "$ROOT_DIR"/scripts/utils/log.sh

set -euo pipefail

TAG=
CHART_VNEXT=
CHART="$ROOT_DIR/chart"
CHART_FORK="false"
CHART_TAG="false"
IMAGE_BUILD="false"
IMAGE_LOAD="false"

cleanup_handler() {
ERROR=$?
trap - INT QUIT TERM HUP EXIT
if [ "$CHART_TAG" = "true" ]; then
if command -v git &>/dev/null; then
git restore "$CHART_VNEXT" || :
fi
fi
if [ $ERROR != 0 ]; then
exit $ERROR
fi
}

# Print usage options for this script.
print_help() {
cat <<EOF
Usage: $(basename "${0}") [OPTIONS]
Options:
-h, --help Display this text.
--tag <tag> The tag for the vnext chart.
Default: Automatically figured out.
--chart <path> The path where the chart will be copied to and modified for the vnext tag.
Default: \$workspace_root/tests/python/chart_vnext.
--fork Forks the vnext chart from "$CHART" into CHART_VNEXT.
--chart-tag Tag the vnext chart.
--build Builds the container images and the plugin binary.
--load Loads the images into the kind cluster.
Examples:
$(basename "${0}") --fork
The kubectl-mayastor binary will be built at CHART_VNEXT/kubectl-plugin/bin/kubectl-mayastor
EOF
}

# Parse args.
while test $# -gt 0; do
arg="$1"
case "$arg" in
--tag)
shift
TAG="$1"
;;
--chart)
shift
CHART_VNEXT="$1"
;;
--fork)
CHART_FORK="true"
;;
--chart-tag)
CHART_TAG="true"
;;
--build)
IMAGE_BUILD="true"
;;
--load)
IMAGE_LOAD="true"
;;
-h* | --help*)
print_help
exit 0
;;
*)
print_help
log_fatal "unexpected argument '$arg'" 1
;;
esac
shift
done

if [ "$(kubectl config current-context)" != "kind-kind" ]; then
log_fatal "Only Supported on Kind Clusters!"
fi

if [ -z "$TAG" ]; then
TAG="$("$SCRIPT_DIR"/generate-test-tag.sh)"
fi

if [ -z "$CHART_VNEXT" ]; then
CHART_VNEXT="$ROOT_DIR/tests/bdd/chart-vnext"
fi
KUBECTL_MAYASTOR="$CHART_VNEXT/kubectl-plugin/bin/kubectl-mayastor"

# Ensure the chart vnext is created, copied from the original
if [ "$CHART_FORK" = "true" ]; then
mkdir -p "$CHART_VNEXT"
rm -r "${CHART_VNEXT:?}"/*
cp -r "$CHART/." "${CHART_VNEXT:?}"
fi

if [ "$CHART_TAG" = "true" ]; then
# Tag the vnext chart
CHART_DIR="$CHART_VNEXT" "$SCRIPT_DIR"/tag-chart.sh "$TAG"
trap cleanup_handler INT QUIT TERM HUP EXIT
fi

# Build the vnext images and kubectl-binary (in debug mode)
if [ "$IMAGE_BUILD" = "true" ]; then
RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" "$ROOT_DIR"/scripts/release.sh --tag "$TAG" --build-binary-out "$CHART_VNEXT" --no-static-linking --skip-publish --debug

# Ensure binary is on the correct version
PLUGIN_VERSION="$($KUBECTL_MAYASTOR --version)"
if [[ ! "$PLUGIN_VERSION" =~ ^Kubectl\ Plugin\ \(kubectl-mayastor\).*\($TAG\+0\)$ ]]; then
log_fatal "The built kubectl-plugin reports version $PLUGIN_VERSION but we want $TAG"
fi
fi

cleanup_handler

# Load the images into the kind cluster
if [ "$IMAGE_LOAD" = "true" ]; then
"$ROOT_DIR"/scripts/k8s/load-images-to-kind.sh --tag "$TAG" --trim-debug-suffix
fi
7 changes: 7 additions & 0 deletions scripts/release.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,11 @@ BUILD_BINARIES="kubectl-plugin"
PROJECT="extensions"
. "$SOURCE_REL"

# Sadly helm ignore does not work on symlinks: https://github.com/helm/helm/issues/13284
# So we must cleanup to ensure the upgrade image is built correctly
CHART_DIR="$(dirname "$0")/../chart"
if [ -L "$CHART_DIR"/kubectl-plugin ]; then
rm "$CHART_DIR"/kubectl-plugin
fi

common_run $@
70 changes: 70 additions & 0 deletions tests/bdd/common/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import logging
import os
import subprocess

logger = logging.getLogger(__name__)


def root_dir():
file_path = os.path.abspath(__file__)
return file_path.split("tests/bdd")[0]


def chart_vnext():
vnext = os.getenv("CHART_VNEXT")
if vnext is not None:
return vnext
# return os.path.join(root_dir(), "./tests/bdd/chart-vnext")
return os.path.join(root_dir(), "./chart")


def chart_vnext_skip():
skip = os.getenv("CHART_VNEXT_SKIP")
if skip is not None and skip.lower() in ("yes", "true", "1", "0"):
return True
return False


def run(
command: str,
args: list[str] = None,
absolute=False,
capture_output=True,
log_run=True,
**kwargs,
):
if absolute:
command = [command]
else:
command = [os.path.join(root_dir(), command)]
if args is not None:
command.extend(args)
if log_run:
logger.info(f"Running '{command}'")
else:
logger.debug(f"Running '{command}'")
try:
result = subprocess.run(
command, capture_output=capture_output, check=True, text=True, **kwargs
)
logger.debug(
f"Command '{command}' completed with:\nStdErr Output: {result.stderr}\nStdOut Output: {result.stdout}"
)
return result.stdout.strip()

except subprocess.CalledProcessError as e:
logger.error(
f"Command '{command}' failed with exit code {e.returncode}\nStdErr Output: {e.stderr}\nStdOut Output: {e.stdout}"
)
raise e

except Exception as e:
logger.error(f"An unexpected error occurred: {e}")
raise e


def env_cleanup():
clean = os.getenv("CLEAN")
if clean is not None and clean.lower() in ("no", "false", "f", "0"):
return False
return True
Loading

0 comments on commit 3fe353c

Please sign in to comment.