RP E2E Test Execution

.pipelines/ci.yml

@@ -1,4 +1,4 @@
-# Azure DevOps Pipeline running CI
+# Azure DevOps Pipeline running CI and E2E
 
 trigger:
   branches:
@@ -21,62 +21,190 @@ variables:
     value: "arosvcdev.azurecr.io/azext-aro"
   - name: LOCAL_VPN_IMAGE
     value: "arosvcdev.azurecr.io/vpn"
+  - name: LOCAL_E2E_IMAGE
+    value: "arosvcdev.azurecr.io/e2e"
   - name: TAG
     value: $(Build.BuildId)
   - name: VERSION
     value: $(Build.BuildId)
 
-jobs:
-  - job: Build_Test_And_Push_Az_ARO_Extension
-    pool:
-      name: 1es-aro-ci-pool
-    steps:
-      - template: ./templates/template-checkout.yml
-
-      # Build and test the Az ARO Extension
-      - script: |
-          set -xe
-          DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(VERSION)
-        displayName: 🛠 Build & Test Az ARO Extension
-
-      # Push the image to ACR
-      - template: ./templates/template-acr-push.yml
-        parameters:
-          acrFQDN: 'arosvcdev.azurecr.io'
-          repository: 'azext-aro'
-          pushLatest: true
-
-  - job: Build_And_Test_RP_And_Portal
-    pool:
-      name: 1es-aro-ci-pool
-    steps:
-      - template: ./templates/template-checkout.yml
-
-      # Build and test RP and Portal
-      - script: |
-          set -xe
-          DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(VERSION)
-        displayName: 🛠 Build & Test RP and Portal
-
-      # Publish test results
-      - task: PublishTestResults@2
-        displayName: 📊 Publish tests results
-        inputs:
-          testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml
-        condition: succeededOrFailed()
-
-      # Publish code coverage results
-      - task: PublishCodeCoverageResults@2
-        displayName: 📈 Publish code coverage
-        inputs:
-          codeCoverageTool: Cobertura
-          summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml
-          failIfCoverageEmpty: false
-        condition: succeededOrFailed()
-
-      # Push the image to ACR
-      - template: ./templates/template-acr-push.yml
-        parameters:
-          acrFQDN: 'arosvcdev.azurecr.io'
-          repository: 'aro'
-          pushLatest: true
+stages:
+  - stage: Containerized
+    jobs:
+      - job: Build_Test_And_Push_Az_ARO_Extension
+        pool:
+          name: 1es-aro-ci-pool
+        steps:
+          - template: ./templates/template-checkout.yml
+
+          # Build and test the Az ARO Extension
+          - script: |
+              set -xe
+              DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(VERSION)
+            displayName: 🛠 Build & Test Az ARO Extension
+
+          # Push the image to ACR
+          - template: ./templates/template-acr-push.yml
+            parameters:
+              acrFQDN: 'arosvcdev.azurecr.io'
+              repository: 'azext-aro'
+              pushLatest: true
+
+      - job: Build_And_Test_RP_And_Portal
+        pool:
+          name: 1es-aro-ci-pool
+        steps:
+          - template: ./templates/template-checkout.yml
+
+          # Build and test RP and Portal
+          - script: |
+              set -xe
+              DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(VERSION)
+            displayName: 🛠 Build & Test RP and Portal
+
+          # Publish test results
+          - task: PublishTestResults@2
+            displayName: 📊 Publish tests results
+            inputs:
+              testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml
+            condition: succeededOrFailed()
+
+          # Publish code coverage results
+          - task: PublishCodeCoverageResults@2
+            displayName: 📈 Publish code coverage
+            inputs:
+              codeCoverageTool: Cobertura
+              summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml
+              failIfCoverageEmpty: false
+            condition: succeededOrFailed()
+
+          # Push the image to ACR
+          - template: ./templates/template-acr-push.yml
+            parameters:
+              acrFQDN: 'arosvcdev.azurecr.io'
+              repository: 'aro'
+              pushLatest: true
+
+  - stage: E2E  # E2E Stage using Docker Compose
+    dependsOn: Containerized
+    jobs:
+      - job: Run_E2E_Tests
+        pool:
+          name: 1es-aro-ci-pool
+        steps:
+          # Step 1: Checkout the code
+          - template: ./templates/template-checkout.yml
+
+          # Step 2: Install Docker, Docker Compose, and dependencies
+          - script: |
+              sudo apt-get update
+              sudo apt-get install -y docker.io docker-compose-plugin make
+              sudo systemctl start docker
+              sudo systemctl enable docker
+            displayName: Install Docker and Docker Compose
+
+          # Step 3: AZ CLI Login
+          - template: ./templates/template-az-cli-login.yml
+            parameters:
+              azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn)
+
+          # Step 4: Download and extract VPN secrets
+          - script: |
+              az account set -s $AZURE_SUBSCRIPTION_ID
+              SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets
+              . secrets/env
+              ls -R ./secrets
+              cat ./secrets/env
+              export KEYVAULT_PREFIX="${RESOURCEGROUP}-dev"
+              echo "KEYVAULT_PREFIX is set to: $KEYVAULT_PREFIX"
+              export AZURE_SUBSCRIPTION_ID="$AZURE_SUBSCRIPTION_ID"
+            displayName: Download and Setup VPN Secrets
+
+          # Step 5: Get Kubeconfig for AKS Cluster with corrected Key Vault configuration
+          - script: |
+              echo "Setting up Key Vault access..."
+              # Dynamically construct the Key Vault URL
+              export KEYVAULT_PREFIX="${RESOURCEGROUP}-dev"
+              KEYVAULT_URL="https://${KEYVAULT_PREFIX}.vault.azure.net"
+              echo "Using Key Vault URL: $KEYVAULT_URL"
+
+              # Verify if the URL is reachable
+              nslookup ${KEYVAULT_PREFIX}.vault.azure.net || {
+                echo "Failed to resolve Key Vault DNS. Please check the Key Vault configuration."
+                exit 1
+              }
+
+              # Retrieve the kubeconfig
+              hack/get-admin-kubeconfig.sh /subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$CLUSTER/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER > admin.kubeconfig
+
+              if [ -f admin.kubeconfig ]; then
+                echo "Kubeconfig retrieved successfully."
+                echo "KUBECONFIG=$(pwd)/admin.kubeconfig" >> .env
+                cat admin.kubeconfig
+              else
+                echo "Failed to retrieve Kubeconfig."
+                exit 1
+              fi
+            displayName: Get Kubeconfig for AKS Cluster
+
+          # Step 6: Deploy Hive Operator
+          - script: |
+              export KUBECONFIG=$(pwd)/admin.kubeconfig
+              if [ -f "hack/hive/hive-dev-install.sh" ]; then
+                ./hack/hive/hive-dev-install.sh
+              else
+                echo "Hive installation script not found"
+                exit 1
+              fi
+            displayName: Deploy Hive Operator
+
+          # Step 7: Set Up Environment Variables for Docker Compose
+          - script: |
+              export RP_IMAGE_ACR=arosvcdev.azurecr.io
+              export VERSION=${BUILD_BUILDID}
+              export E2E_IMAGE=${RP_IMAGE_ACR}/e2e:${VERSION}
+              export E2E_LABEL='!smoke&&!regressiontest'
+              echo "RP_IMAGE_ACR=$RP_IMAGE_ACR" > .env
+              echo "VERSION=$VERSION" >> .env
+              echo "E2E_IMAGE=$E2E_IMAGE" >> .env
+              echo "E2E_LABEL=$E2E_LABEL" >> .env
+              echo "KUBECONFIG=$(pwd)/admin.kubeconfig" >> .env
+              . secrets/env
+              cat .env
+            displayName: Set Up Environment Variables
+
+          # Step 8: Start Services using Docker Compose
+          - script: |
+              docker compose --env-file .env -f docker-compose.yml up -d
+              docker ps
+            displayName: Start Services using Docker Compose
+
+          # Step 9: Execute E2E Tests
+          - script: |
+              docker compose up run-e2e --abort-on-container-exit
+            displayName: Execute E2E Tests
+
+          # Step 10: Log the output from the services in case of failure
+          - script: |
+              docker compose logs vpn
+              docker compose logs selenium
+              docker compose logs rp
+              docker compose logs run-e2e
+            displayName: Log Service Output
+            condition: failed()
+
+          # Step 11: Clean up Docker Compose services
+          - script: |
+              docker compose down
+              rm -f admin.kubeconfig
+            displayName: Cleanup Docker Compose Services and Kubeconfig
+            condition: always()
+
+          # Step 12: Clean Up Database
+          - script: |
+              az cosmosdb sql database delete --name "$DATABASE_NAME" --yes --account-name "$DATABASE_ACCOUNT_NAME" --resource-group "$RESOURCEGROUP"
+            displayName: Clean Up Database
+            condition: always()
+
+          # Step 13: AZ CLI Logout
+          - template: ./templates/template-az-cli-logout.yml

.pipelines/templates/e2e-pipeline-template.yml

@@ -0,0 +1,55 @@
+# ./templates/e2e-pipeline-template.yml
+parameters:
+  - name: rpImageACR
+    type: string
+  - name: acrCredentialsJSON
+    type: string
+
+steps:
+  # Step 1: Authenticate to ACR and Install Docker Compose
+  - task: AzureCLI@2
+    displayName: 'Authenticate to ACR and Install Docker Compose'
+    inputs:
+      azureSubscription: 'ado-pipeline-dev-image-push'  # Replace with your service connection
+      scriptType: bash
+      scriptLocation: 'inlineScript'
+      inlineScript: |
+        set -xe
+        # Ensure RP_IMAGE_ACR is correctly passed as a parameter
+        if [ -z "${{ parameters.rpImageACR }}" ]; then
+          echo "Error: RP_IMAGE_ACR is not set"
+          exit 1
+        fi
+
+        ACR_FQDN="${{ parameters.rpImageACR }}"
+        REGISTRY_NAME=$(echo $ACR_FQDN | cut -d'.' -f1)
+
+        # Install Docker Compose
+        sudo apt-get update
+        sudo apt-get install -y docker-compose
+
+        # Login to ACR
+        az acr login --name $REGISTRY_NAME
+
+  # Step 2: Pull the RP Docker image
+  - script: |
+      if [ -z "${{ parameters.rpImageACR }}" ]; then
+          echo "Error: RP_IMAGE_ACR is not set"
+          exit 1
+      fi
+
+      export RP_IMAGE_ACR=${{ parameters.rpImageACR }}
+      export VERSION=$(Build.BuildId)
+      docker pull ${RP_IMAGE_ACR}/aro:${VERSION}
+    displayName: Pull RP Docker Image
+
+  # Step 3: Pull the E2E Docker image (add this step)
+  - script: |
+      if [ -z "${{ parameters.rpImageACR }}" ]; then
+          echo "Error: RP_IMAGE_ACR is not set"
+          exit 1
+      fi
+
+      export E2E_IMAGE=${{ parameters.rpImageACR }}/e2e:${VERSION}
+      docker pull ${E2E_IMAGE}
+    displayName: Pull E2E Docker Image

.pipelines/templates/template-az-cli-login.yml

@@ -7,7 +7,13 @@ steps:
       set -e
 
       trap 'rm -f devops-spn.json' EXIT
-      base64 -d >devops-spn.json <<<${{ parameters.azureDevOpsJSONSPN }}
+      echo "${{ parameters.azureDevOpsJSONSPN }}" | base64 -d > devops-spn.json
 
-      az login --service-principal -u "$(jq -r .clientId <devops-spn.json)" -p "$(jq -r .clientSecret <devops-spn.json)" -t "$(jq -r .tenantId <devops-spn.json)" --allow-no-subscriptions >/dev/null
+      az login --service-principal \
+        -u "$(jq -r .clientId <devops-spn.json)" \
+        -p "$(jq -r .clientSecret <devops-spn.json)" \
+        -t "$(jq -r .tenantId <devops-spn.json)" --allow-no-subscriptions >/dev/null
+
+      # Cleanup
+      rm -f devops-spn.json
     displayName: 🗝 AZ Login

docker-compose.yml

@@ -117,7 +117,7 @@ services:
     container_name: aro-portal
     depends_on:
       rp:
-        condition: service_healthy
+        condition: service_started
     environment:
       - RP_MODE
       - AZURE_SUBSCRIPTION_ID
@@ -153,6 +153,39 @@ services:
       timeout: 10s
       retries: 3
 
+  run-e2e:
+    image: ${LOCAL_ARO_RP_IMAGE}:${VERSION}
+    container_name: run-e2e
+    network_mode: host
+    depends_on:
+      - vpn
+      - rp
+      - portal
+      - selenium
+    environment:
+      - KUBECONFIG=/app/admin.kubeconfig
+      - RP_MODE=development
+      - LOCATION=${LOCATION}
+      - CLUSTER=${CLUSTER}
+      - AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID}
+      - AZURE_TENANT_ID=${AZURE_TENANT_ID}
+      - AZURE_CLIENT_ID=${AZURE_CLIENT_ID}
+      - AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}
+      - DATABASE_ACCOUNT_NAME=${DATABASE_ACCOUNT_NAME}
+      - DATABASE_NAME=${DATABASE_NAME}
+      - RESOURCEGROUP=${RESOURCEGROUP}
+      - PROXY_HOSTNAME=${PROXY_HOSTNAME}
+    volumes:
+      - ./admin.kubeconfig:/app/admin.kubeconfig:z
+    entrypoint: "/usr/local/bin/e2e.test"
+    command: ["-test.v", "--ginkgo.v", "--ginkgo.timeout=180m", "--ginkgo.flake-attempts=2", "--ginkgo.junit-report=/tmp/e2e-report.xml", "--ginkgo.label-filter=${E2E_LABEL}"]
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:8080/healthz || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    restart: on-failure:3
+
 secrets:
   proxy-client-key:
     file: ./secrets/proxy-client.key