From c88a74ec34d21e9b2baa605b6985a4d785a0b577 Mon Sep 17 00:00:00 2001 From: Shubhada Date: Tue, 1 Oct 2024 14:50:20 -0700 Subject: [PATCH] fixed the vpn issue --- .pipelines/ci.yml | 86 +++++++++++++++++++++++++++++----------------- docker-compose.yml | 19 ++++++---- 2 files changed, 67 insertions(+), 38 deletions(-) diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml index 2e96b29cac0..3229432b742 100644 --- a/.pipelines/ci.yml +++ b/.pipelines/ci.yml @@ -116,7 +116,7 @@ stages: [[ -z "$(git status -s)" ]] target: python - # New E2E Stage with Docker Compose + # E2E Stage using Docker Compose - stage: E2E dependsOn: Containerized jobs: @@ -124,70 +124,92 @@ stages: pool: name: 1es-aro-ci-pool steps: - # Checkout the code + # Step 1: Checkout the code - template: ./templates/template-checkout.yml - # Install Docker Compose and pull the RP image + # Step 2: Pull the RP image from ACR - template: ./templates/e2e-pipeline-template.yml parameters: rpImageACR: 'arosvcdev.azurecr.io' acrCredentialsJSON: $(acr-credentials) - # Install OpenVPN (the command will depend on the agent OS) + # Step 3: Install Docker Compose and Docker - script: | - set -xe - sudo apt-get update && sudo apt-get install -y openvpn || \ - sudo tdnf install -y openvpn || \ - sudo yum install -y openvpn - displayName: Install OpenVPN + sudo apt-get update && sudo apt-get install -y openvpn docker.io docker-compose make + sudo systemctl start docker + sudo systemctl enable docker + displayName: Install OpenVPN, Docker, and Docker Compose - # AZ CLI Login using the existing secret as in the old pipeline + # Step 4: AZ CLI Login using the existing secret as in the old pipeline - template: ./templates/template-az-cli-login.yml parameters: azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn) + # Step 5: Download and extract VPN secrets with `make secrets` - script: | - set -xe az account set -s $AZURE_SUBSCRIPTION_ID - - # Download secrets tarball containing the VPN certificates and config - az storage blob download --account-name $(SECRET_SA_ACCOUNT_NAME) --container-name secrets --name vpn-secrets.tar.gz --file vpn-secrets.tar.gz - - # Extract the secrets tarball + + #command to download the secrets tarball and set the environment + SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets + . secrets/env sudo mkdir -p /etc/openvpn - sudo tar -xzf vpn-secrets.tar.gz -C /etc/openvpn - - # Ensure the extracted files contain the required certificates + tar -xzf secrets.tar.gz -C ./secrets + echo "Listing contents of extracted secrets:" + ls -R ./secrets + # Copy VPN secrets (certificates, config) to /etc/openvpn + sudo cp ./secrets/vpn-ca.crt /etc/openvpn/ca.crt + sudo cp ./secrets/vpn-client.crt /etc/openvpn/client.crt + sudo cp ./secrets/vpn-client.key /etc/openvpn/client.key + sudo cp ./secrets/vpn-eastus.ovpn /etc/openvpn/vpn.conf + ls /etc/openvpn - displayName: Download and Extract VPN Secrets + displayName: Download and Setup VPN Secrets - # Setup Azure and source secrets/env + # Step 6: Setup environment variables for Docker Compose - script: | - set -xe export RP_IMAGE_ACR=arosvcdev.azurecr.io export VERSION=${BUILD_BUILDID} export E2E_FLAGS="--flag1 --flag2" export E2E_LABEL="test-label" - export E2E_DELETE_CLUSTER="false" echo "RP_IMAGE_ACR=$RP_IMAGE_ACR" > .env echo "VERSION=$VERSION" >> .env echo "E2E_FLAGS=$E2E_FLAGS" >> .env echo "E2E_LABEL=$E2E_LABEL" >> .env - cat .env - sudo openvpn --config /etc/openvpn/vpn.conf & + displayName: Set Up Environment Variables + + # Step 7: Start services using Docker Compose + - script: | docker-compose --env-file .env -f docker-compose.yml up -d - displayName: Start OpenVPN and Run Docker Compose for E2E Services + docker ps + displayName: Start Services using Docker Compose - # Log the output from the e2e container in case of failure + # Step 8: Execute E2E tests - script: | - set -xe - docker-compose logs e2e - displayName: Log E2E Test Output + docker-compose exec e2e /usr/local/bin/e2e.test ${E2E_FLAGS} --ginkgo.label-filter=${E2E_LABEL} + displayName: Execute E2E Tests + + # Step 9: Log the output from the services in case of failure + - script: | + docker-compose logs vpn + docker-compose logs selenium + docker-compose logs rp + displayName: Log Service Output condition: failed() - # Clean up Docker Compose + # Step 10: Clean up Docker Compose services - script: | docker-compose down - displayName: Cleanup Docker Compose + displayName: Cleanup Docker Compose Services condition: always() + + # Step 11: Cleanup Azure resources + - script: | + export CI=true + . ./hack/e2e/run-rp-and-e2e.sh + clean_e2e_db + displayName: Cleanup (Azure) + condition: always() + + # Step 12: AZ CLI Logout + - template: ./templates/template-az-cli-logout.yml diff --git a/docker-compose.yml b/docker-compose.yml index 93a8eb2eb27..78d9885bdb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,7 @@ version: '3.8' services: vpn: - # Use an image with OpenVPN installed (or you can use the previous image and install OpenVPN at runtime) - image: kylemanna/openvpn + image: debian:latest container_name: vpn-container privileged: true # Required for OpenVPN network_mode: host # Use host network @@ -12,12 +11,20 @@ services: - /etc/openvpn:/etc/openvpn # Mount the extracted VPN secrets devices: - /dev/net/tun # Required for VPN to access tunnel - command: ["openvpn", "--config", "/etc/openvpn/vpn.conf"] # Start OpenVPN with existing config + command: > + bash -c " + apt-get update && + openvpn --version && openvpn --config /etc/openvpn/vpn.conf + healthcheck: + test: ["CMD", "pgrep", "openvpn"] + interval: 30s + timeout: 10s + retries: 3 selenium: image: selenium/standalone-chrome container_name: selenium-container - network_mode: host # Use host network (no need for port mappings) + network_mode: host healthcheck: test: ["CMD", "curl", "-f", "http://localhost:4444"] interval: 30s @@ -27,7 +34,7 @@ services: rp: image: ${RP_IMAGE_ACR}/aro:${VERSION} container_name: rp-container - network_mode: host # Use host network + network_mode: host depends_on: vpn: condition: service_healthy @@ -40,7 +47,7 @@ services: e2e: image: ${RP_IMAGE_ACR}/aro:${VERSION} container_name: e2e-container - network_mode: host # Use host network + network_mode: host depends_on: rp: condition: service_healthy