From 47890c1337a599a1bc0689f91c773212e797fef6 Mon Sep 17 00:00:00 2001 From: Shubhada Date: Fri, 20 Sep 2024 15:36:23 -0700 Subject: [PATCH] Integrate Docker for E2E testing with Makefile and Azure pipeline orchestration --- .pipelines/ci.yml | 88 ++++++++++++++++++- .../templates/e2e-pipeline-template.yml | 44 ++++++++++ .../templates/template-az-cli-login.yml | 10 ++- Dockerfile.vpn | 11 +++ Makefile | 45 ++++++++++ docker-compose.yml | 56 ++++++++++++ 6 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 .pipelines/templates/e2e-pipeline-template.yml create mode 100644 Dockerfile.vpn create mode 100644 docker-compose.yml diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml index cc2fed8cdaa..4ee0c851f35 100644 --- a/.pipelines/ci.yml +++ b/.pipelines/ci.yml @@ -1,4 +1,4 @@ -# Azure DevOps Pipeline running CI +# Azure DevOps Pipeline running CI and E2E trigger: branches: @@ -101,3 +101,89 @@ stages: acrFQDN: 'arosvcdev.azurecr.io' repository: 'aro' pushLatest: true + + - job: Lint_Az_ARO_Extension + pool: + name: 1es-aro-ci-pool + variables: + HOME: $(Agent.BuildDirectory) + steps: + - template: ./templates/template-checkout.yml + - script: | + set -xe + export AZDEV_CONFIG_DIR=$(Agent.BuildDirectory)/azdev-config + make test-python + [[ -z "$(git status -s)" ]] + target: python + + # New E2E Stage with Docker Compose + - stage: E2E + dependsOn: Containerized + jobs: + - job: Run_E2E_Tests + pool: + name: 1es-aro-ci-pool + steps: + # Checkout the code + - template: ./templates/template-checkout.yml + + # Install Docker Compose and pull the RP image + - template: ./templates/e2e-pipeline-template.yml + parameters: + rpImageACR: 'arosvcdev.azurecr.io' + acrCredentialsJSON: $(acr-credentials) + + # Install OpenVPN (the command will depend on the agent OS) + - script: | + sudo apt-get update && sudo apt-get install -y openvpn || \ + sudo tdnf install -y openvpn || \ + sudo yum install -y openvpn + displayName: Install OpenVPN + + # AZ CLI Login using the existing secret as in the old pipeline + - template: ./templates/template-az-cli-login.yml + parameters: + azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn) + + - script: | + az account set -s $AZURE_SUBSCRIPTION_ID + + # Download secrets tarball containing the VPN certificates and config + az storage blob download --account-name $(SECRET_SA_ACCOUNT_NAME) --container-name secrets --name vpn-secrets.tar.gz --file vpn-secrets.tar.gz + + # Extract the secrets tarball + sudo mkdir -p /etc/openvpn + sudo tar -xzf vpn-secrets.tar.gz -C /etc/openvpn + + # Ensure the extracted files contain the required certificates + ls /etc/openvpn + displayName: Download and Extract VPN Secrets + + # Setup Azure and source secrets/env + - script: | + export RP_IMAGE_ACR=arosvcdev.azurecr.io + export VERSION=${BUILD_BUILDID} + export E2E_FLAGS="--flag1 --flag2" + export E2E_LABEL="test-label" + export E2E_DELETE_CLUSTER="false" + echo "RP_IMAGE_ACR=$RP_IMAGE_ACR" > .env + echo "VERSION=$VERSION" >> .env + echo "E2E_FLAGS=$E2E_FLAGS" >> .env + echo "E2E_LABEL=$E2E_LABEL" >> .env + + cat .env + sudo openvpn --config /etc/openvpn/vpn.conf & + docker-compose --env-file .env -f docker-compose.yml up -d + displayName: Start OpenVPN and Run Docker Compose for E2E Services + + # Log the output from the e2e container in case of failure + - script: | + docker-compose logs e2e + displayName: Log E2E Test Output + condition: failed() + + # Clean up Docker Compose + - script: | + docker-compose down + displayName: Cleanup Docker Compose + condition: always() diff --git a/.pipelines/templates/e2e-pipeline-template.yml b/.pipelines/templates/e2e-pipeline-template.yml new file mode 100644 index 00000000000..c74062f01e5 --- /dev/null +++ b/.pipelines/templates/e2e-pipeline-template.yml @@ -0,0 +1,44 @@ +# ./templates/e2e-pipeline-template.yml +parameters: + - name: rpImageACR + type: string + - name: acrCredentialsJSON + type: string + +steps: + # Authenticate to ACR and Install Docker Compose + - task: AzureCLI@2 + displayName: 'Authenticate to ACR and Install Docker Compose' + inputs: + azureSubscription: 'ado-pipeline-dev-image-push' # Replace with your service connection + scriptType: bash + scriptLocation: 'inlineScript' + inlineScript: | + set -xe + # Ensure RP_IMAGE_ACR is correctly passed as a parameter + if [ -z "${{ parameters.rpImageACR }}" ]; then + echo "Error: RP_IMAGE_ACR is not set" + exit 1 + fi + + ACR_FQDN="${{ parameters.rpImageACR }}" + REGISTRY_NAME=$(echo $ACR_FQDN | cut -d'.' -f1) + + # Install Docker Compose + sudo apt-get update + sudo apt-get install -y docker-compose + + # Login to ACR + az acr login --name $REGISTRY_NAME + + # Pull the RP Docker image + - script: | + if [ -z "${{ parameters.rpImageACR }}" ]; then + echo "Error: RP_IMAGE_ACR is not set" + exit 1 + fi + + export RP_IMAGE_ACR=${{ parameters.rpImageACR }} + export VERSION=$(Build.BuildId) + docker pull ${RP_IMAGE_ACR}/aro:${VERSION} + displayName: Pull RP Docker Image diff --git a/.pipelines/templates/template-az-cli-login.yml b/.pipelines/templates/template-az-cli-login.yml index 14c7ea043fd..126cb902cec 100644 --- a/.pipelines/templates/template-az-cli-login.yml +++ b/.pipelines/templates/template-az-cli-login.yml @@ -7,7 +7,13 @@ steps: set -e trap 'rm -f devops-spn.json' EXIT - base64 -d >devops-spn.json <<<${{ parameters.azureDevOpsJSONSPN }} + echo "${{ parameters.azureDevOpsJSONSPN }}" | base64 -d > devops-spn.json - az login --service-principal -u "$(jq -r .clientId /dev/null + az login --service-principal \ + -u "$(jq -r .clientId /dev/null + + # Cleanup + rm -f devops-spn.json displayName: 🗝 AZ Login diff --git a/Dockerfile.vpn b/Dockerfile.vpn new file mode 100644 index 00000000000..d57e3739e1c --- /dev/null +++ b/Dockerfile.vpn @@ -0,0 +1,11 @@ +# Use a Microsoft-approved image +FROM mcr.microsoft.com/azure-cli:2.61.0 AS base + +# Install OpenVPN +RUN apk add --no-cache openvpn || tdnf install -y openvpn || dnf install -y openvpn + +# Create the config directory and generate a basic vpn.conf file +RUN mkdir -p /etc/openvpn && echo "client\nremote vpn-server-address 1194\nproto udp\ndev tun\nresolv-retry infinite\nnobind\npersist-key\npersist-tun\nca ca.crt\ncert client.crt\nkey client.key\ncomp-lzo\nverb 3" > /etc/openvpn/vpn.conf + +# Run OpenVPN when the container starts +CMD ["openvpn", "--config", "/etc/openvpn/vpn.conf"] diff --git a/Makefile b/Makefile index 99c2cdeeda2..e3233f815a7 100644 --- a/Makefile +++ b/Makefile @@ -379,6 +379,7 @@ LOCAL_ARO_PORTAL_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-portal-build LOCAL_ARO_RP_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-build LOCAL_AZ_EXT_ARO_IMAGE ?= azext-aro LOCAL_TUNNEL_IMAGE ?= aro-tunnel +LOCAL_VPN_IMAGE ?= vpn_image ############################################################################### # Targets @@ -539,3 +540,47 @@ run-rp: ci-rp podman-secrets --secret proxy-client.crt,target=/app/secrets/proxy-client.crt \ --secret proxy.crt,target=/app/secrets/proxy.crt \ $(LOCAL_ARO_RP_IMAGE):$(VERSION) rp + +# Run selenium using Docker +.PHONY: run-selenium +run-selenium: + docker run -d --name selenium-container selenium/standalone-chrome + +# Run RP using Docker +.PHONY: run-rp-docker +run-rp: run-selenium + docker run -d --name rp-container $(ARO_IMAGE_BASE):$(VERSION) + +# Run E2E Tests using Docker +.PHONY: run-e2e +run-e2e: e2e.test + docker-compose run --rm e2e /usr/local/bin/e2e.test $(E2E_FLAGS) --ginkgo.label-filter=$(E2E_LABEL) + +# Clean up containers after E2E tests +.PHONY: e2e-cluster-clean +e2e-cluster-clean: + docker stop selenium-container rp-container e2e-container || true + docker rm selenium-container rp-container e2e-container || true + +# Build the VPN Docker image +.PHONY: build-vpn +build-vpn: + @echo "Building VPN image with VERSION: $(VERSION)" + docker build . $(DOCKER_BUILD_CI_ARGS) \ + -f Dockerfile.vpn \ + -t $(LOCAL_VPN_IMAGE):$(VERSION) + +# Push the VPN image to ACR +.PHONY: push-vpn +push-vpn: build-vpn + @echo "Pushing VPN image to ACR: $(RP_IMAGE_ACR)" + @echo "VERSION is: $(VERSION)" + if [ -z "$(RP_IMAGE_ACR)" ]; then \ + echo "Error: RP_IMAGE_ACR is not set"; \ + exit 1; \ + fi + # Tag the VPN image with the ACR registry and version + docker tag $(LOCAL_VPN_IMAGE):$(VERSION) $(RP_IMAGE_ACR)/vpn_image:$(VERSION) + # Push the VPN image to ACR + docker push $(RP_IMAGE_ACR)/vpn_image:$(VERSION) + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000000..93a8eb2eb27 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3.8' + +services: + vpn: + # Use an image with OpenVPN installed (or you can use the previous image and install OpenVPN at runtime) + image: kylemanna/openvpn + container_name: vpn-container + privileged: true # Required for OpenVPN + network_mode: host # Use host network + volumes: + - /dev/shm:/dev/shm # Shared memory mount + - /etc/openvpn:/etc/openvpn # Mount the extracted VPN secrets + devices: + - /dev/net/tun # Required for VPN to access tunnel + command: ["openvpn", "--config", "/etc/openvpn/vpn.conf"] # Start OpenVPN with existing config + + selenium: + image: selenium/standalone-chrome + container_name: selenium-container + network_mode: host # Use host network (no need for port mappings) + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4444"] + interval: 30s + timeout: 10s + retries: 3 + + rp: + image: ${RP_IMAGE_ACR}/aro:${VERSION} + container_name: rp-container + network_mode: host # Use host network + depends_on: + vpn: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8443/healthz"] + interval: 30s + timeout: 10s + retries: 3 + + e2e: + image: ${RP_IMAGE_ACR}/aro:${VERSION} + container_name: e2e-container + network_mode: host # Use host network + depends_on: + rp: + condition: service_healthy + environment: + - CI=true + - E2E_FLAGS=${E2E_FLAGS} + - E2E_LABEL=${E2E_LABEL} + entrypoint: ["/usr/local/bin/e2e.test"] + command: ["${E2E_FLAGS}", "--ginkgo.label-filter=${E2E_LABEL}"] + +networks: + default: + external: true