Skip to content

Commit

Permalink
fixed the vpn issue
Browse files Browse the repository at this point in the history
  • Loading branch information
@shubhadapaithankar
shubhadapaithankar committed Oct 11, 2024
1 parent c3b0bd6 commit 1d29b6f
Show file tree
Hide file tree
Showing 4 changed files with 186 additions and 105 deletions.
240 changes: 139 additions & 101 deletions .pipelines/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,152 +21,190 @@ variables:
value: "arosvcdev.azurecr.io/azext-aro"
- name: LOCAL_VPN_IMAGE
value: "arosvcdev.azurecr.io/vpn"
- name: LOCAL_E2E_IMAGE
value: "arosvcdev.azurecr.io/e2e"
- name: TAG
value: $(Build.BuildId)
- name: VERSION
value: $(Build.BuildId)

jobs:
- job: Build_Test_And_Push_Az_ARO_Extension
pool:
name: 1es-aro-ci-pool
steps:
- template: ./templates/template-checkout.yml

# Build and test the Az ARO Extension
- script: |
set -xe
DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(VERSION)
displayName: 🛠 Build & Test Az ARO Extension
# Push the image to ACR
- template: ./templates/template-acr-push.yml
parameters:
acrFQDN: 'arosvcdev.azurecr.io'
repository: 'azext-aro'
pushLatest: true

- job: Build_And_Test_RP_And_Portal
pool:
name: 1es-aro-ci-pool
steps:
- template: ./templates/template-checkout.yml

# Build and test RP and Portal
- script: |
set -xe
DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(VERSION)
displayName: 🛠 Build & Test RP and Portal
# Publish test results
- task: PublishTestResults@2
displayName: 📊 Publish tests results
inputs:
testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml
condition: succeededOrFailed()

# Publish code coverage results
- task: PublishCodeCoverageResults@2
displayName: 📈 Publish code coverage
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml
failIfCoverageEmpty: false
condition: succeededOrFailed()
stages:
- stage: Containerized
jobs:
- job: Build_Test_And_Push_Az_ARO_Extension
pool:
name: 1es-aro-ci-pool
steps:
- template: ./templates/template-checkout.yml

# Build and test the Az ARO Extension
- script: |
set -xe
DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(VERSION)
displayName: 🛠 Build & Test Az ARO Extension
# Push the image to ACR
- template: ./templates/template-acr-push.yml
parameters:
acrFQDN: 'arosvcdev.azurecr.io'
repository: 'aro'
repository: 'azext-aro'
pushLatest: true

- job: Lint_Az_ARO_Extension
- job: Build_And_Test_RP_And_Portal
pool:
name: 1es-aro-ci-pool
variables:
HOME: $(Agent.BuildDirectory)
steps:
- template: ./templates/template-checkout.yml

# Build and test RP and Portal
- script: |
set -xe
export AZDEV_CONFIG_DIR=$(Agent.BuildDirectory)/azdev-config
make test-python
[[ -z "$(git status -s)" ]]
target: python
DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(VERSION)
displayName: 🛠 Build & Test RP and Portal
# Publish test results
- task: PublishTestResults@2
displayName: 📊 Publish tests results
inputs:
testResultsFiles: $(System.DefaultWorkingDirectory)/report.xml
condition: succeededOrFailed()

# Publish code coverage results
- task: PublishCodeCoverageResults@2
displayName: 📈 Publish code coverage
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage.xml
failIfCoverageEmpty: false
condition: succeededOrFailed()

# New E2E Stage with Docker Compose
- stage: E2E
# Push the image to ACR
- template: ./templates/template-acr-push.yml
parameters:
acrFQDN: 'arosvcdev.azurecr.io'
repository: 'aro'
pushLatest: true

- stage: E2E # E2E Stage using Docker Compose
dependsOn: Containerized
jobs:
- job: Run_E2E_Tests
pool:
name: 1es-aro-ci-pool
steps:
# Checkout the code
# Step 1: Checkout the code
- template: ./templates/template-checkout.yml

# Install Docker Compose and pull the RP image
- template: ./templates/e2e-pipeline-template.yml
parameters:
rpImageACR: 'arosvcdev.azurecr.io'
acrCredentialsJSON: $(acr-credentials)

# Install OpenVPN (the command will depend on the agent OS)
# Step 2: Install Docker, Docker Compose, and dependencies
- script: |
set -xe
sudo apt-get update && sudo apt-get install -y openvpn || \
sudo tdnf install -y openvpn || \
sudo yum install -y openvpn
displayName: Install OpenVPN
sudo apt-get update
sudo apt-get install -y docker.io docker-compose-plugin make
sudo systemctl start docker
sudo systemctl enable docker
displayName: Install Docker and Docker Compose
# AZ CLI Login using the existing secret as in the old pipeline
# Step 3: AZ CLI Login
- template: ./templates/template-az-cli-login.yml
parameters:
azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn)

# Step 4: Download and extract VPN secrets
- script: |
set -xe
az account set -s $AZURE_SUBSCRIPTION_ID
# Download secrets tarball containing the VPN certificates and config
az storage blob download --account-name $(SECRET_SA_ACCOUNT_NAME) --container-name secrets --name vpn-secrets.tar.gz --file vpn-secrets.tar.gz
# Extract the secrets tarball
sudo mkdir -p /etc/openvpn
sudo tar -xzf vpn-secrets.tar.gz -C /etc/openvpn
# Ensure the extracted files contain the required certificates
ls /etc/openvpn
displayName: Download and Extract VPN Secrets
# Setup Azure and source secrets/env
SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets
. secrets/env
ls -R ./secrets
cat ./secrets/env
export KEYVAULT_PREFIX="${RESOURCEGROUP}-dev"
echo "KEYVAULT_PREFIX is set to: $KEYVAULT_PREFIX"
export AZURE_SUBSCRIPTION_ID="$AZURE_SUBSCRIPTION_ID"
displayName: Download and Setup VPN Secrets
# Step 5: Get Kubeconfig for AKS Cluster with corrected Key Vault configuration
- script: |
echo "Setting up Key Vault access..."
# Dynamically construct the Key Vault URL
export KEYVAULT_PREFIX="${RESOURCEGROUP}-dev"
KEYVAULT_URL="https://${KEYVAULT_PREFIX}.vault.azure.net"
echo "Using Key Vault URL: $KEYVAULT_URL"
# Verify if the URL is reachable
nslookup ${KEYVAULT_PREFIX}.vault.azure.net || {
echo "Failed to resolve Key Vault DNS. Please check the Key Vault configuration."
exit 1
}
# Retrieve the kubeconfig
hack/get-admin-kubeconfig.sh /subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$CLUSTER/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER > admin.kubeconfig
if [ -f admin.kubeconfig ]; then
echo "Kubeconfig retrieved successfully."
echo "KUBECONFIG=$(pwd)/admin.kubeconfig" >> .env
cat admin.kubeconfig
else
echo "Failed to retrieve Kubeconfig."
exit 1
fi
displayName: Get Kubeconfig for AKS Cluster
# Step 6: Deploy Hive Operator
- script: |
export KUBECONFIG=$(pwd)/admin.kubeconfig
if [ -f "hack/hive/hive-dev-install.sh" ]; then
./hack/hive/hive-dev-install.sh
else
echo "Hive installation script not found"
exit 1
fi
displayName: Deploy Hive Operator
# Step 7: Set Up Environment Variables for Docker Compose
- script: |
set -xe
export RP_IMAGE_ACR=arosvcdev.azurecr.io
export VERSION=${BUILD_BUILDID}
export E2E_FLAGS="--flag1 --flag2"
export E2E_LABEL="test-label"
export E2E_DELETE_CLUSTER="false"
export E2E_IMAGE=${RP_IMAGE_ACR}/e2e:${VERSION}
export E2E_LABEL='!smoke&&!regressiontest'
echo "RP_IMAGE_ACR=$RP_IMAGE_ACR" > .env
echo "VERSION=$VERSION" >> .env
echo "E2E_FLAGS=$E2E_FLAGS" >> .env
echo "E2E_IMAGE=$E2E_IMAGE" >> .env
echo "E2E_LABEL=$E2E_LABEL" >> .env
echo "KUBECONFIG=$(pwd)/admin.kubeconfig" >> .env
. secrets/env
cat .env
sudo openvpn --config /etc/openvpn/vpn.conf &
docker-compose --env-file .env -f docker-compose.yml up -d
displayName: Start OpenVPN and Run Docker Compose for E2E Services
displayName: Set Up Environment Variables
# Log the output from the e2e container in case of failure
# Step 8: Start Services using Docker Compose
- script: |
set -xe
docker-compose logs e2e
displayName: Log E2E Test Output
docker compose --env-file .env -f docker-compose.yml up -d
docker ps
displayName: Start Services using Docker Compose
# Step 9: Execute E2E Tests
- script: |
docker compose up run-e2e --abort-on-container-exit
displayName: Execute E2E Tests
# Step 10: Log the output from the services in case of failure
- script: |
docker compose logs vpn
docker compose logs selenium
docker compose logs rp
docker compose logs run-e2e
displayName: Log Service Output
condition: failed()
# Clean up Docker Compose
# Step 11: Clean up Docker Compose services
- script: |
docker-compose down
displayName: Cleanup Docker Compose
docker compose down
rm -f admin.kubeconfig
displayName: Cleanup Docker Compose Services and Kubeconfig
condition: always()
# Step 12: Clean Up Database
- script: |
az cosmosdb sql database delete --name "$DATABASE_NAME" --yes --account-name "$DATABASE_ACCOUNT_NAME" --resource-group "$RESOURCEGROUP"
displayName: Clean Up Database
condition: always()
# Step 13: AZ CLI Logout
- template: ./templates/template-az-cli-logout.yml
15 changes: 13 additions & 2 deletions .pipelines/templates/e2e-pipeline-template.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ parameters:
type: string

steps:
# Authenticate to ACR and Install Docker Compose
# Step 1: Authenticate to ACR and Install Docker Compose
- task: AzureCLI@2
displayName: 'Authenticate to ACR and Install Docker Compose'
inputs:
Expand All @@ -31,7 +31,7 @@ steps:
# Login to ACR
az acr login --name $REGISTRY_NAME
# Pull the RP Docker image
# Step 2: Pull the RP Docker image
- script: |
if [ -z "${{ parameters.rpImageACR }}" ]; then
echo "Error: RP_IMAGE_ACR is not set"
Expand All @@ -42,3 +42,14 @@ steps:
export VERSION=$(Build.BuildId)
docker pull ${RP_IMAGE_ACR}/aro:${VERSION}
displayName: Pull RP Docker Image
# Step 3: Pull the E2E Docker image (add this step)
- script: |
if [ -z "${{ parameters.rpImageACR }}" ]; then
echo "Error: RP_IMAGE_ACR is not set"
exit 1
fi
export E2E_IMAGE=${{ parameters.rpImageACR }}/e2e:${VERSION}
docker pull ${E2E_IMAGE}
displayName: Pull E2E Docker Image
1 change: 0 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,6 @@ LOCAL_ARO_PORTAL_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-portal-build
LOCAL_ARO_RP_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-build
LOCAL_AZ_EXT_ARO_IMAGE ?= azext-aro
LOCAL_TUNNEL_IMAGE ?= aro-tunnel
LOCAL_VPN_IMAGE ?= vpn_image

###############################################################################
# Targets
Expand Down
35 changes: 34 additions & 1 deletion docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ services:
container_name: aro-portal
depends_on:
rp:
condition: service_healthy
condition: service_started
environment:
- RP_MODE
- AZURE_SUBSCRIPTION_ID
Expand Down Expand Up @@ -153,6 +153,39 @@ services:
timeout: 10s
retries: 3

run-e2e:
image: ${LOCAL_ARO_RP_IMAGE}:${VERSION}
container_name: run-e2e
network_mode: host
depends_on:
- vpn
- rp
- portal
- selenium
environment:
- KUBECONFIG=/app/admin.kubeconfig
- RP_MODE=development
- LOCATION=${LOCATION}
- CLUSTER=${CLUSTER}
- AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID}
- AZURE_TENANT_ID=${AZURE_TENANT_ID}
- AZURE_CLIENT_ID=${AZURE_CLIENT_ID}
- AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}
- DATABASE_ACCOUNT_NAME=${DATABASE_ACCOUNT_NAME}
- DATABASE_NAME=${DATABASE_NAME}
- RESOURCEGROUP=${RESOURCEGROUP}
- PROXY_HOSTNAME=${PROXY_HOSTNAME}
volumes:
- ./admin.kubeconfig:/app/admin.kubeconfig:z
entrypoint: "/usr/local/bin/e2e.test"
command: ["-test.v", "--ginkgo.v", "--ginkgo.timeout=180m", "--ginkgo.flake-attempts=2", "--ginkgo.junit-report=/tmp/e2e-report.xml", "--ginkgo.label-filter=${E2E_LABEL}"]
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:8080/healthz || exit 1"]
interval: 30s
timeout: 10s
retries: 3
restart: on-failure:3

secrets:
proxy-client-key:
file: ./secrets/proxy-client.key
Expand Down

0 comments on commit 1d29b6f

Please sign in to comment.