fixed the vpn issue

Azure · Oct 4, 2024 · d3f4b1e · d3f4b1e
1 parent 6d0bc90
commit d3f4b1e
Show file tree

Hide file tree

Showing 5 changed files with 216 additions and 158 deletions.
diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml
@@ -11,8 +11,6 @@ trigger:
     include:
       - v2*
 
-# PR triggers are overridden in the ADO UI
-
 resources:
   containers:
     - container: golang
@@ -28,147 +26,153 @@ resources:
 variables:
   - template: vars.yml
 
-jobs:
-  - job: Build_Test_And_Push_Az_ARO_Extension
-    pool:
-      name: 1es-aro-ci-pool
-    steps:
-      - template: ./templates/template-checkout.yml
-
-      # Build and test the Az ARO Extension
-      - script: |
-          set -xe
-          DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(Build.BuildId)
-        displayName: 🛠 Build & Test Az ARO Extension
-
-      # Push the image to ACR
-      - template: ./templates/template-acr-push.yml
-        parameters:
-          acrFQDN: 'arosvcdev.azurecr.io'
-          repository: 'azext-aro'
-          pushLatest: true
-
-  - job: Build_And_Test_RP_And_Portal
-    pool:
-      name: 1es-aro-ci-pool
-    steps:
-      - template: ./templates/template-checkout.yml
-
-      # Build and test RP and Portal
-      - script: |
-          set -xe
-          DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(Build.BuildId)
-        displayName: 🛠 Build & Test RP and Portal
-
-      # Publish test results
-      - task: PublishTestResults@2
-        displayName: 📊 Publish tests results
-        inputs:
-          testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml
-        condition: succeededOrFailed()
-
-      # Publish code coverage results
-      - task: PublishCodeCoverageResults@2
-        displayName: 📈 Publish code coverage
-        inputs:
-          codeCoverageTool: Cobertura
-          summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml
-          failIfCoverageEmpty: false
-        condition: succeededOrFailed()
+stages:
+  - stage: Containerized
+    jobs:
+      - job: Build_Test_And_Push_Az_ARO_Extension
+        pool:
+          name: 1es-aro-ci-pool
+        steps:
+          - template: ./templates/template-checkout.yml
+
+          # Build and test the Az ARO Extension
+          - script: |
+              set -xe
+              DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(Build.BuildId)
+            displayName: 🛠 Build & Test Az ARO Extension
 
           # Push the image to ACR
           - template: ./templates/template-acr-push.yml
             parameters:
               acrFQDN: 'arosvcdev.azurecr.io'
-              repository: 'aro'
+              repository: 'azext-aro'
               pushLatest: true
 
-      - job: Lint_Az_ARO_Extension
+      - job: Build_And_Test_RP_And_Portal
         pool:
           name: 1es-aro-ci-pool
-        variables:
-          HOME: $(Agent.BuildDirectory)
         steps:
           - template: ./templates/template-checkout.yml
+
+          # Build and test RP and Portal
           - script: |
               set -xe
-              export AZDEV_CONFIG_DIR=$(Agent.BuildDirectory)/azdev-config
-              make test-python
-              [[ -z "$(git status -s)" ]]
-            target: python
+              DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(Build.BuildId)
+            displayName: 🛠 Build & Test RP and Portal
+
+          # Publish test results
+          - task: PublishTestResults@2
+            displayName: 📊 Publish tests results
+            inputs:
+              testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml
+            condition: succeededOrFailed()
+
+          # Publish code coverage results
+          - task: PublishCodeCoverageResults@2
+            displayName: 📈 Publish code coverage
+            inputs:
+              codeCoverageTool: Cobertura
+              summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml
+              failIfCoverageEmpty: false
+            condition: succeededOrFailed()
 
-  # New E2E Stage with Docker Compose
-  - stage: E2E
+          # Push the image to ACR
+          - template: ./templates/template-acr-push.yml
+            parameters:
+              acrFQDN: 'arosvcdev.azurecr.io'
+              repository: 'aro'
+              pushLatest: true
+
+  - stage: E2E  # E2E Stage using Docker Compose
     dependsOn: Containerized
     jobs:
       - job: Run_E2E_Tests
         pool:
           name: 1es-aro-ci-pool
         steps:
-          # Checkout the code
+          # Step 1: Checkout the code
           - template: ./templates/template-checkout.yml
 
-          # Install Docker Compose and pull the RP image
+          # Step 2: Pull the RP image from ACR
           - template: ./templates/e2e-pipeline-template.yml
             parameters:
               rpImageACR: 'arosvcdev.azurecr.io'
               acrCredentialsJSON: $(acr-credentials)
 
-          # Install OpenVPN (the command will depend on the agent OS)
+          # Step 3: Install Docker Compose and Docker
           - script: |
-                set -xe
-                sudo apt-get update && sudo apt-get install -y openvpn || \
-                sudo tdnf install -y openvpn || \
-                sudo yum install -y openvpn
-            displayName: Install OpenVPN
-
-          # AZ CLI Login using the existing secret as in the old pipeline
+              sudo apt-get update
+              sudo apt-get install -y docker.io docker-compose-plugin make
+              sudo apt-get install -y openvpn
+              sudo systemctl start docker
+              sudo systemctl enable docker
+            displayName: Install OpenVPN, Docker, and Docker Compose
+
+          # Step 4: AZ CLI Login using the existing secret as in the old pipeline
           - template: ./templates/template-az-cli-login.yml
             parameters:
               azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn)
 
+          # Step 5: Download and extract VPN secrets with `make secrets`
           - script: |
-              set -xe
               az account set -s $AZURE_SUBSCRIPTION_ID
-
-              # Download secrets tarball containing the VPN certificates and config
-              az storage blob download --account-name $(SECRET_SA_ACCOUNT_NAME) --container-name secrets --name vpn-secrets.tar.gz --file vpn-secrets.tar.gz
-
-              # Extract the secrets tarball
+              #command to download the secrets tarball and set the environment
+              SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets
+              . secrets/env
               sudo mkdir -p /etc/openvpn
-              sudo tar -xzf vpn-secrets.tar.gz -C /etc/openvpn
+              ls -R ./secrets
+              sudo cp ./secrets/* /etc/openvpn
+              ls -R /etc/openvpn
 
-              # Ensure the extracted files contain the required certificates
-              ls /etc/openvpn
-            displayName: Download and Extract VPN Secrets
+            displayName: Download and Setup VPN Secrets
 
-          # Setup Azure and source secrets/env
+          # Step 6: Setup environment variables for Docker Compose
           - script: |
-              set -xe
               export RP_IMAGE_ACR=arosvcdev.azurecr.io
               export VERSION=${BUILD_BUILDID}
               export E2E_FLAGS="--flag1 --flag2"
               export E2E_LABEL="test-label"
-              export E2E_DELETE_CLUSTER="false"
               echo "RP_IMAGE_ACR=$RP_IMAGE_ACR" > .env
               echo "VERSION=$VERSION" >> .env
               echo "E2E_FLAGS=$E2E_FLAGS" >> .env
               echo "E2E_LABEL=$E2E_LABEL" >> .env
-
+              . secrets/env
               cat .env
-              sudo openvpn --config /etc/openvpn/vpn.conf &
-              docker-compose --env-file .env -f docker-compose.yml up -d
-            displayName: Start OpenVPN and Run Docker Compose for E2E Services
+            displayName: Set Up Environment Variables
 
-          # Log the output from the e2e container in case of failure
+          # Step 7: Start services using Docker Compose
           - script: |
-              set -xe
-              docker-compose logs e2e
-            displayName: Log E2E Test Output
+              docker compose --env-file .env -f docker-compose.yml up -d
+              docker ps
+            displayName: Start Services using Docker Compose
+
+          # Step 8: Execute E2E tests
+          - script: |
+              docker compose exec e2e-container make test-e2e
+            displayName: Execute E2E Tests
+
+          # Step 9: Log the output from the services in case of failure
+          - script: |
+              docker compose logs vpn
+              docker compose logs selenium
+              docker compose logs rp
+              docker compose logs e2e
+            displayName: Log Service Output
             condition: failed()
 
-          # Clean up Docker Compose
+          # Step 10: Clean up Docker Compose services
           - script: |
-              docker-compose down
-            displayName: Cleanup Docker Compose
+              docker compose down
+            displayName: Cleanup Docker Compose Services
             condition: always()
+
+          # Step 11: Cleanup Azure resources
+          - script: |
+              export CI=true
+              . ./hack/e2e/run-rp-and-e2e.sh
+              clean_e2e_db
+            displayName: Cleanup (Azure)
+            condition: always()
+
+          # Step 12: AZ CLI Logout
+          - template: ./templates/template-az-cli-logout.yml
diff --git a/Dockerfile.vpn b/Dockerfile.vpn
@@ -2,10 +2,7 @@
 FROM mcr.microsoft.com/azure-cli:2.61.0 AS base
 
 # Install OpenVPN
+USER root
 RUN apk add --no-cache openvpn || tdnf install -y openvpn || dnf install -y openvpn
 
-# Create the config directory and generate a basic vpn.conf file
-RUN mkdir -p /etc/openvpn && echo "client\nremote vpn-server-address 1194\nproto udp\ndev tun\nresolv-retry infinite\nnobind\npersist-key\npersist-tun\nca ca.crt\ncert client.crt\nkey client.key\ncomp-lzo\nverb 3" > /etc/openvpn/vpn.conf
-
-# Run OpenVPN when the container starts
-CMD ["openvpn", "--config", "/etc/openvpn/vpn.conf"]
+ENTRYPOINT openvpn
diff --git a/Dockerfile.vpn.dockerignore b/Dockerfile.vpn.dockerignore
@@ -0,0 +1,2 @@
+# ignore everything
+*
diff --git a/Makefile b/Makefile
@@ -379,7 +379,6 @@ LOCAL_ARO_PORTAL_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-portal-build
 LOCAL_ARO_RP_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-build
 LOCAL_AZ_EXT_ARO_IMAGE ?= azext-aro
 LOCAL_TUNNEL_IMAGE ?= aro-tunnel
-LOCAL_VPN_IMAGE ?= vpn_image
 
 ###############################################################################
 # Targets
@@ -416,6 +415,7 @@ ci-rp: fix-macos-vendor
 		--build-arg ARO_VERSION=$(VERSION) \
 		-t $(LOCAL_ARO_RP_IMAGE):$(VERSION)
 
+	export REGISTRY
 	# Extract test coverage files from build to local filesystem
 	docker create --name extract_cover_out $(LOCAL_ARO_RP_BUILD_IMAGE):$(VERSION); \
 	docker cp extract_cover_out:/app/report.xml ./report.xml; \
@@ -541,46 +541,12 @@ run-rp: ci-rp podman-secrets
 		--secret proxy.crt,target=/app/secrets/proxy.crt \
 		$(LOCAL_ARO_RP_IMAGE):$(VERSION) rp
 
-# Run selenium using Docker
-.PHONY: run-selenium
-run-selenium:
-	docker run -d --name selenium-container selenium/standalone-chrome
-
-# Run RP using Docker
-.PHONY: run-rp-docker
-run-rp: run-selenium
-	docker run -d --name rp-container $(ARO_IMAGE_BASE):$(VERSION)
-
-# Run E2E Tests using Docker
-.PHONY: run-e2e
-run-e2e: e2e.test
-	docker-compose run --rm e2e /usr/local/bin/e2e.test $(E2E_FLAGS) --ginkgo.label-filter=$(E2E_LABEL)
-
-# Clean up containers after E2E tests
-.PHONY: e2e-cluster-clean
-e2e-cluster-clean:
-	docker stop selenium-container rp-container e2e-container || true
-	docker rm selenium-container rp-container e2e-container || true
-
-# Build the VPN Docker image
-.PHONY: build-vpn
-build-vpn:
-	@echo "Building VPN image with VERSION: $(VERSION)"
-	docker build . $(DOCKER_BUILD_CI_ARGS) \
-		-f Dockerfile.vpn \
-		-t $(LOCAL_VPN_IMAGE):$(VERSION)
-
-# Push the VPN image to ACR
-.PHONY: push-vpn
-push-vpn: build-vpn
-	@echo "Pushing VPN image to ACR: $(RP_IMAGE_ACR)"
-	@echo "VERSION is: $(VERSION)"
-	if [ -z "$(RP_IMAGE_ACR)" ]; then \
-	    echo "Error: RP_IMAGE_ACR is not set"; \
-	    exit 1; \
-	fi
-	# Tag the VPN image with the ACR registry and version
-	docker tag $(LOCAL_VPN_IMAGE):$(VERSION) $(RP_IMAGE_ACR)/vpn_image:$(VERSION)
-	# Push the VPN image to ACR
-	docker push $(RP_IMAGE_ACR)/vpn_image:$(VERSION)
+.PHONY: vpn
+vpn:
+	docker-compose build vpn
 
+.PHONY: run-vpn
+run-vpn:
+	docker-compose rm -sf vpn
+	docker-compose up vpn
+	docker-compose logs --follow vpn