From 0e4d0999620d1db5594ba7bf54bf0f9f23194fef Mon Sep 17 00:00:00 2001 From: Shubhada Date: Tue, 1 Oct 2024 14:50:20 -0700 Subject: [PATCH] fixed the vpn issue --- .pipelines/ci.yml | 214 ++++++++++-------- .../templates/e2e-pipeline-template.yml | 15 +- Makefile | 1 - docker-compose.yml | 35 ++- 4 files changed, 162 insertions(+), 103 deletions(-) diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml index a3542de08a5..938f1a414c6 100644 --- a/.pipelines/ci.yml +++ b/.pipelines/ci.yml @@ -21,152 +21,168 @@ variables: value: "arosvcdev.azurecr.io/azext-aro" - name: LOCAL_VPN_IMAGE value: "arosvcdev.azurecr.io/vpn" + - name: LOCAL_E2E_IMAGE + value: "arosvcdev.azurecr.io/e2e" - name: TAG value: $(Build.BuildId) - name: VERSION value: $(Build.BuildId) -jobs: - - job: Build_Test_And_Push_Az_ARO_Extension - pool: - name: 1es-aro-ci-pool - steps: - - template: ./templates/template-checkout.yml - - # Build and test the Az ARO Extension - - script: | - set -xe - DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(VERSION) - displayName: 🛠 Build & Test Az ARO Extension - - # Push the image to ACR - - template: ./templates/template-acr-push.yml - parameters: - acrFQDN: 'arosvcdev.azurecr.io' - repository: 'azext-aro' - pushLatest: true - - - job: Build_And_Test_RP_And_Portal - pool: - name: 1es-aro-ci-pool - steps: - - template: ./templates/template-checkout.yml - - # Build and test RP and Portal - - script: | - set -xe - DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(VERSION) - displayName: 🛠 Build & Test RP and Portal - - # Publish test results - - task: PublishTestResults@2 - displayName: 📊 Publish tests results - inputs: - testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml - condition: succeededOrFailed() - - # Publish code coverage results - - task: PublishCodeCoverageResults@2 - displayName: 📈 Publish code coverage - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml - failIfCoverageEmpty: false - condition: succeededOrFailed() +stages: + - stage: Containerized + jobs: + - job: Build_Test_And_Push_Az_ARO_Extension + pool: + name: 1es-aro-ci-pool + steps: + - template: ./templates/template-checkout.yml + + # Build and test the Az ARO Extension + - script: | + set -xe + DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(VERSION) + displayName: 🛠 Build & Test Az ARO Extension # Push the image to ACR - template: ./templates/template-acr-push.yml parameters: acrFQDN: 'arosvcdev.azurecr.io' - repository: 'aro' + repository: 'azext-aro' pushLatest: true - - job: Lint_Az_ARO_Extension + - job: Build_And_Test_RP_And_Portal pool: name: 1es-aro-ci-pool - variables: - HOME: $(Agent.BuildDirectory) steps: - template: ./templates/template-checkout.yml + + # Build and test RP and Portal - script: | set -xe - export AZDEV_CONFIG_DIR=$(Agent.BuildDirectory)/azdev-config - make test-python - [[ -z "$(git status -s)" ]] - target: python + DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(VERSION) + displayName: 🛠 Build & Test RP and Portal + + # Publish test results + - task: PublishTestResults@2 + displayName: 📊 Publish tests results + inputs: + testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml + condition: succeededOrFailed() + + # Publish code coverage results + - task: PublishCodeCoverageResults@2 + displayName: 📈 Publish code coverage + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml + failIfCoverageEmpty: false + condition: succeededOrFailed() - # New E2E Stage with Docker Compose - - stage: E2E + # Push the image to ACR + - template: ./templates/template-acr-push.yml + parameters: + acrFQDN: 'arosvcdev.azurecr.io' + repository: 'aro' + pushLatest: true + + - stage: E2E # E2E Stage using Docker Compose dependsOn: Containerized jobs: - job: Run_E2E_Tests pool: name: 1es-aro-ci-pool steps: - # Checkout the code + # Step 1: Checkout the code - template: ./templates/template-checkout.yml - # Install Docker Compose and pull the RP image - - template: ./templates/e2e-pipeline-template.yml - parameters: - rpImageACR: 'arosvcdev.azurecr.io' - acrCredentialsJSON: $(acr-credentials) - - # Install OpenVPN (the command will depend on the agent OS) + # Step 2: Install Docker, Docker Compose, and dependencies - script: | - set -xe - sudo apt-get update && sudo apt-get install -y openvpn || \ - sudo tdnf install -y openvpn || \ - sudo yum install -y openvpn - displayName: Install OpenVPN + sudo apt-get update + sudo apt-get install -y docker.io docker-compose-plugin make openvpn + sudo systemctl start docker + sudo systemctl enable docker + displayName: Install Docker, Docker Compose, and OpenVPN - # AZ CLI Login using the existing secret as in the old pipeline + # Step 3: AZ CLI Login - template: ./templates/template-az-cli-login.yml parameters: azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn) + # Step 4: Download and extract VPN secrets - script: | - set -xe az account set -s $AZURE_SUBSCRIPTION_ID - - # Download secrets tarball containing the VPN certificates and config - az storage blob download --account-name $(SECRET_SA_ACCOUNT_NAME) --container-name secrets --name vpn-secrets.tar.gz --file vpn-secrets.tar.gz - - # Extract the secrets tarball + SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets + . secrets/env sudo mkdir -p /etc/openvpn - sudo tar -xzf vpn-secrets.tar.gz -C /etc/openvpn - - # Ensure the extracted files contain the required certificates - ls /etc/openvpn - displayName: Download and Extract VPN Secrets + sudo cp ./secrets/* /etc/openvpn + displayName: Download and Setup VPN Secrets - # Setup Azure and source secrets/env + # Step 5: Get Kubeconfig for AKS Cluster + - script: | + export KEYVAULT_PREFIX=$(KEYVAULT_PREFIX) + hack/get-admin-kubeconfig.sh /subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$CLUSTER/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER > admin.kubeconfig + export KUBECONFIG=$(pwd)/admin.kubeconfig + echo "KUBECONFIG=$(pwd)/admin.kubeconfig" >> .env + cat $KUBECONFIG + displayName: Get Kubeconfig for AKS Cluster + + # Step 6: Deploy Hive Cluster + - script: | + export KUBECONFIG=$(pwd)/admin.kubeconfig + if [ -f "hack/hive/hive-dev-install.sh" ]; then + /hack/get-admin-kubeconfig.sh + else + echo "Hive installation script not found" + fi + displayName: Deploy Hive Cluster + + # Step 7: Set Up Environment Variables for Docker Compose - script: | - set -xe export RP_IMAGE_ACR=arosvcdev.azurecr.io export VERSION=${BUILD_BUILDID} - export E2E_FLAGS="--flag1 --flag2" - export E2E_LABEL="test-label" - export E2E_DELETE_CLUSTER="false" + export E2E_IMAGE=${RP_IMAGE_ACR}/e2e:${VERSION} + export E2E_LABEL='!smoke&&!regressiontest' echo "RP_IMAGE_ACR=$RP_IMAGE_ACR" > .env echo "VERSION=$VERSION" >> .env - echo "E2E_FLAGS=$E2E_FLAGS" >> .env + echo "E2E_IMAGE=$E2E_IMAGE" >> .env echo "E2E_LABEL=$E2E_LABEL" >> .env - + echo "KUBECONFIG=$(pwd)/admin.kubeconfig" >> .env + . secrets/env cat .env - sudo openvpn --config /etc/openvpn/vpn.conf & - docker-compose --env-file .env -f docker-compose.yml up -d - displayName: Start OpenVPN and Run Docker Compose for E2E Services + displayName: Set Up Environment Variables - # Log the output from the e2e container in case of failure + # Step 8: Start Services using Docker Compose - script: | - set -xe - docker-compose logs e2e - displayName: Log E2E Test Output + docker compose --env-file .env -f docker-compose.yml up -d + docker ps + displayName: Start Services using Docker Compose + + # Step 9: Execute E2E Tests + - script: | + docker compose up run-e2e --abort-on-container-exit + displayName: Execute E2E Tests + + # Step 10: Log the output from the services in case of failure + - script: | + docker compose logs vpn + docker compose logs selenium + docker compose logs rp + docker compose logs run-e2e + displayName: Log Service Output condition: failed() - # Clean up Docker Compose + # Step 11: Clean up Docker Compose services - script: | - docker-compose down - displayName: Cleanup Docker Compose + docker compose down + rm -f admin.kubeconfig + displayName: Cleanup Docker Compose Services and Kubeconfig condition: always() + + # Step 12: Clean Up Database (Optional) + - script: | + az cosmosdb sql database delete --name "$DATABASE_NAME" --yes --account-name "$DATABASE_ACCOUNT_NAME" --resource-group "$RESOURCEGROUP" + displayName: Clean Up Database + condition: always() + + # Step 13: AZ CLI Logout + - template: ./templates/template-az-cli-logout.yml diff --git a/.pipelines/templates/e2e-pipeline-template.yml b/.pipelines/templates/e2e-pipeline-template.yml index c74062f01e5..627772b698b 100644 --- a/.pipelines/templates/e2e-pipeline-template.yml +++ b/.pipelines/templates/e2e-pipeline-template.yml @@ -6,7 +6,7 @@ parameters: type: string steps: - # Authenticate to ACR and Install Docker Compose + # Step 1: Authenticate to ACR and Install Docker Compose - task: AzureCLI@2 displayName: 'Authenticate to ACR and Install Docker Compose' inputs: @@ -31,7 +31,7 @@ steps: # Login to ACR az acr login --name $REGISTRY_NAME - # Pull the RP Docker image + # Step 2: Pull the RP Docker image - script: | if [ -z "${{ parameters.rpImageACR }}" ]; then echo "Error: RP_IMAGE_ACR is not set" @@ -42,3 +42,14 @@ steps: export VERSION=$(Build.BuildId) docker pull ${RP_IMAGE_ACR}/aro:${VERSION} displayName: Pull RP Docker Image + + # Step 3: Pull the E2E Docker image (add this step) + - script: | + if [ -z "${{ parameters.rpImageACR }}" ]; then + echo "Error: RP_IMAGE_ACR is not set" + exit 1 + fi + + export E2E_IMAGE=${{ parameters.rpImageACR }}/e2e:${VERSION} + docker pull ${E2E_IMAGE} + displayName: Pull E2E Docker Image diff --git a/Makefile b/Makefile index 0a0f10e5d8e..a99bd9f0396 100644 --- a/Makefile +++ b/Makefile @@ -379,7 +379,6 @@ LOCAL_ARO_PORTAL_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-portal-build LOCAL_ARO_RP_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-build LOCAL_AZ_EXT_ARO_IMAGE ?= azext-aro LOCAL_TUNNEL_IMAGE ?= aro-tunnel -LOCAL_VPN_IMAGE ?= vpn_image ############################################################################### # Targets diff --git a/docker-compose.yml b/docker-compose.yml index 77d5964c7e6..b24aa290e17 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -117,7 +117,7 @@ services: container_name: aro-portal depends_on: rp: - condition: service_healthy + condition: service_started environment: - RP_MODE - AZURE_SUBSCRIPTION_ID @@ -153,6 +153,39 @@ services: timeout: 10s retries: 3 + run-e2e: + image: ${LOCAL_ARO_RP_IMAGE}:${VERSION} + container_name: run-e2e + network_mode: host + depends_on: + - vpn + - rp + - portal + - selenium + environment: + - KUBECONFIG=/app/admin.kubeconfig + - RP_MODE=development + - LOCATION=${LOCATION} + - CLUSTER=${CLUSTER} + - AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID} + - AZURE_TENANT_ID=${AZURE_TENANT_ID} + - AZURE_CLIENT_ID=${AZURE_CLIENT_ID} + - AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET} + - DATABASE_ACCOUNT_NAME=${DATABASE_ACCOUNT_NAME} + - DATABASE_NAME=${DATABASE_NAME} + - RESOURCEGROUP=${RESOURCEGROUP} + - PROXY_HOSTNAME=${PROXY_HOSTNAME} + volumes: + - ./admin.kubeconfig:/app/admin.kubeconfig:z + entrypoint: "/usr/local/bin/e2e.test" + command: ["-test.v", "--ginkgo.v", "--ginkgo.timeout=180m", "--ginkgo.flake-attempts=2", "--ginkgo.junit-report=/tmp/e2e-report.xml", "--ginkgo.label-filter=${E2E_LABEL}"] + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/healthz || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + restart: on-failure:3 + secrets: proxy-client-key: file: ./secrets/proxy-client.key