From 52e1555f765efb7184eecaeef64456b7e2cd54af Mon Sep 17 00:00:00 2001 From: Siddhesh Ghadi Date: Mon, 21 Aug 2023 12:31:41 +0530 Subject: [PATCH 1/2] Run CI e2e tests on operator installed via OLM - Update GHA configuration to setup a k3d local image registry - Add olm-install.sh script to install operator via OLM - Enable e2e tests from tests/olm directory Signed-off-by: Siddhesh Ghadi --- .github/workflows/ci-build.yaml | 22 +++++--- hack/olm-install.sh | 95 +++++++++++++++++++++++++++++++++ hack/test.sh | 2 +- 3 files changed, 111 insertions(+), 8 deletions(-) create mode 100755 hack/olm-install.sh diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index 28700d9fc..f8ae0b1dc 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -48,6 +48,10 @@ jobs: matrix: k3s-version: [ v1.27.1 ] # k3s-version: [v1.20.2, v1.19.2, v1.18.9, v1.17.11, v1.16.15] + env: + K3D_REGISTRY: registry.localhost + K3D_REGISTRY_DOMAIN: k3d-registry.localhost + K3D_REGISTRY_PORT: 5000 steps: - name: Download kuttl plugin env: @@ -67,7 +71,10 @@ jobs: set -x curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash sudo mkdir -p $HOME/.kube && sudo chown -R runner $HOME/.kube - k3d cluster create --servers 3 --image rancher/k3s:${{ matrix.k3s-version }}-k3s1 + # setup registry & patch hosts file for registry domain resolution + k3d registry create ${K3D_REGISTRY} --port ${K3D_REGISTRY_PORT} + echo "127.0.0.1 ${K3D_REGISTRY_DOMAIN}" | sudo tee -a /etc/hosts + k3d cluster create --registry-use "${K3D_REGISTRY_DOMAIN}:${K3D_REGISTRY_PORT}" --servers 3 --image rancher/k3s:${{ matrix.k3s-version }}-k3s1 kubectl version k3d version - name: Checkout code @@ -90,18 +97,19 @@ jobs: - name: Download Go dependencies run: | go mod download - - name: Run the operator locally + - name: Run the operator via OLM env: ARGOCD_CLUSTER_CONFIG_NAMESPACES: argocd-e2e-cluster-config - run: | - set -o pipefail - make install generate fmt vet - # Use tee to flush output to the log. Other solutions like stdbuf don't work, not sure why. - REDIS_CONFIG_PATH="build/redis" GRAFANA_CONFIG_PATH="grafana" go run ./main.go 2>&1 | tee /tmp/e2e-operator-run.log & + REGISTRY: ${{ env.K3D_REGISTRY_DOMAIN }}:${{ env.K3D_REGISTRY_PORT }} + VERSION: "0.8.0" + run: ./hack/olm-install.sh - name: Run tests run: | set -o pipefail bash hack/test.sh 2>&1 | tee /tmp/e2e-test.log + - name: Save operator logs + if: ${{ failure() }} + run: kubectl logs -n operators deployment.apps/argocd-operator-controller-manager > /tmp/e2e-operator-run.log - name: Upload operator logs uses: actions/upload-artifact@v2 with: diff --git a/hack/olm-install.sh b/hack/olm-install.sh new file mode 100755 index 000000000..84b14ff30 --- /dev/null +++ b/hack/olm-install.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +set -e + +# This script installs operator via OLM for testing. +# Note: This script will update local configurations files. +# Ensure you revert changes before committing if running locally. + +check_command() { + command=$1 + if ! command -v $command &> /dev/null; then + echo "$command is not installed. Please install it before proceeding." + exit 1 + fi +} + +check_deps() { + echo "Checking dependencies..." + check_command kubectl + check_command operator-sdk + check_command make + echo "All required dependencies are installed." +} + +wait_for_deployment() { + NAMESPACE=$1 + DEPLOYMENT_NAME=$2 + REPLICAS=${3:-1} + MAX_WAIT_SECONDS=300 + + echo "Waiting for $DEPLOYMENT_NAME deployment replicas to be ready..." + + # Loop until replicas are ready or timeout is reached + WAITED_SECONDS=0 + while [ $WAITED_SECONDS -lt $MAX_WAIT_SECONDS ]; do + READY_REPLICAS=$(kubectl get deployment -n $NAMESPACE $DEPLOYMENT_NAME -o jsonpath='{.status.readyReplicas}') || true + if [ "$READY_REPLICAS" -eq "$REPLICAS" ]; then + echo "All $REPLICAS replicas are ready!" + break + else + echo "Waiting for replicas to be ready. Current ready replicas: $READY_REPLICAS" + sleep 10 + WAITED_SECONDS=$((WAITED_SECONDS + 10)) + fi + done + + if [ $WAITED_SECONDS -ge $MAX_WAIT_SECONDS ]; then + echo "Timed out waiting for replicas to be ready." + exit 1 + fi +} + +install_olm() { + if kubectl get deployment/olm-operator -n olm >/dev/null 2>&1; then + echo "OLM is installed on the cluster." + else + echo "Installing OLM...." + operator-sdk olm install || true + fi + wait_for_deployment "olm" "olm-operator" +} + +# env setup +REGISTRY="${REGISTRY:-k3d-registry.localhost:5000}" +ORG="${ORG:-local}" + +export IMAGE_TAG_BASE="$REGISTRY/$ORG/argocd-operator" +export VERSION="${VERSION:-0.8.0}" + +check_deps +install_olm + +# build and push operator image +make generate manifests +make docker-build docker-push + +# build and push bundle image +make bundle +make bundle-build bundle-push + +# build and push catalog image +make catalog-build catalog-push + +# install catalog +CATALOG_IMAGE="$IMAGE_TAG_BASE-catalog:v$VERSION" +sed "s|image:.*|image: $CATALOG_IMAGE|" deploy/catalog_source.yaml | kubectl apply -n operators -f - + +# create subscription +sed "s/sourceNamespace:.*/sourceNamespace: operators/" deploy/subscription.yaml | kubectl apply -n operators -f - + +wait_for_deployment "operators" "argocd-operator-controller-manager" + + +# cleanup +#operator-sdk cleanup argocd-operator -n operators \ No newline at end of file diff --git a/hack/test.sh b/hack/test.sh index 45c8627f5..533e8ea02 100755 --- a/hack/test.sh +++ b/hack/test.sh @@ -18,4 +18,4 @@ set -e -kubectl kuttl test ./tests/k8s ./tests/ha --config ./tests/kuttl-tests.yaml \ No newline at end of file +kubectl kuttl test ./tests/k8s ./tests/ha ./tests/olm --config ./tests/kuttl-tests.yaml \ No newline at end of file From 5a32c65129561ae7893d817b72080309bc328bef Mon Sep 17 00:00:00 2001 From: Siddhesh Ghadi Date: Mon, 21 Aug 2023 13:03:26 +0530 Subject: [PATCH 2/2] Install operator sdk Signed-off-by: Siddhesh Ghadi --- .github/workflows/ci-build.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index f8ae0b1dc..45479c04f 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -77,6 +77,24 @@ jobs: k3d cluster create --registry-use "${K3D_REGISTRY_DOMAIN}:${K3D_REGISTRY_PORT}" --servers 3 --image rancher/k3s:${{ matrix.k3s-version }}-k3s1 kubectl version k3d version + - name: Download and install Operator SDK + env: + SDK_VERSION: 1.11.0 + SDK_CHECKSUM: 'b1f6fb02c619cfcdb46edf41cbeb4d66f627fd8bba122edaeeb06718965299eb' + run: | + set -ue + set -o pipefail + curl -sLf --retry 3 \ + -o /tmp/operator-sdk_linux_amd64 \ + https://github.com/operator-framework/operator-sdk/releases/download/v1.11.0/operator-sdk_linux_amd64 + calculated=$(sha256sum /tmp/operator-sdk_linux_amd64 | awk '{print $1}') + if test "${calculated}" != "${SDK_CHECKSUM}"; then + echo "FAILED TO VALIDATE CHECKSUM" >&2 + echo "Download is: ${calculated}" + echo "Should: ${SDK_CHECKSUM}" + exit 1 + fi + sudo install -m 0755 /tmp/operator-sdk_linux_amd64 /usr/local/bin/operator-sdk - name: Checkout code uses: actions/checkout@v2 - name: Setup Golang