From 5631766a6a63d8cd4c743f61e6e9380cc5174ba1 Mon Sep 17 00:00:00 2001 From: Shubhada Date: Thu, 3 Oct 2024 16:45:07 -0700 Subject: [PATCH 1/9] migrate VPN service to Docker Compos --- Dockerfile.vpn | 8 ++++++++ Dockerfile.vpn.dockerignore | 2 ++ Makefile | 10 ++++++++++ docker-compose.yml | 20 ++++++++++++++++++++ 4 files changed, 40 insertions(+) create mode 100644 Dockerfile.vpn create mode 100644 Dockerfile.vpn.dockerignore create mode 100644 docker-compose.yml diff --git a/Dockerfile.vpn b/Dockerfile.vpn new file mode 100644 index 00000000000..12bc93c3195 --- /dev/null +++ b/Dockerfile.vpn @@ -0,0 +1,8 @@ +# Use a Microsoft-approved image +FROM mcr.microsoft.com/azure-cli:2.61.0 AS base + +# Install OpenVPN +USER root +RUN apk add --no-cache openvpn || tdnf install -y openvpn || dnf install -y openvpn + +ENTRYPOINT openvpn diff --git a/Dockerfile.vpn.dockerignore b/Dockerfile.vpn.dockerignore new file mode 100644 index 00000000000..4ee7adaddb8 --- /dev/null +++ b/Dockerfile.vpn.dockerignore @@ -0,0 +1,2 @@ +# ignore everything +* diff --git a/Makefile b/Makefile index 0bffd9b7fb1..a39b7a540b8 100644 --- a/Makefile +++ b/Makefile @@ -540,3 +540,13 @@ run-rp: ci-rp podman-secrets --secret proxy-client.crt,target=/app/secrets/proxy-client.crt \ --secret proxy.crt,target=/app/secrets/proxy.crt \ $(LOCAL_ARO_RP_IMAGE):$(VERSION) rp + +.PHONY: vpn +vpn: + docker compose build vpn + +.PHONY: run-vpn +run-vpn: + docker compose rm -sf vpn + docker compose up vpn + docker compose logs --follow vpn diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000000..fc1e12eb6a5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +version: '3.8' + +services: + vpn: + image: vpn:${VERSION} + build: + context: . + dockerfile: Dockerfile.vpn + container_name: vpn + privileged: true + volumes: + - ${PWD}/secrets:/secrets:z + entrypoint: "openvpn" + command: ["/secrets/vpn-eastus.ovpn"] + healthcheck: + test: ["CMD", "pidof", "openvpn"] + start_period: 20s + interval: 20s + timeout: 3s + retries: 3 From 476e9280b915de6e83babd902e7d021187a1ebe4 Mon Sep 17 00:00:00 2001 From: Shubhada Date: Fri, 4 Oct 2024 10:15:38 -0700 Subject: [PATCH 2/9] Added rp in pipline and added proper image tagging/pushing --- .pipelines/ci.yml | 30 ++++--- .pipelines/templates/template-acr-push.yml | 32 +++++-- Makefile | 26 +++--- docker-compose.yml | 98 +++++++++++++++++++++- 4 files changed, 147 insertions(+), 39 deletions(-) diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml index 7ff841d9b50..a689a28fcf1 100644 --- a/.pipelines/ci.yml +++ b/.pipelines/ci.yml @@ -11,22 +11,20 @@ trigger: include: - v2* -# PR triggers are overridden in the ADO UI - -resources: - containers: - - container: golang - image: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 - options: --user=0 - - container: python - image: registry.access.redhat.com/ubi8/python-39:latest - options: --user=0 - - container: ubi8 - image: registry.access.redhat.com/ubi8/toolbox:8.8 - options: --user=0 --privileged -v /dev/shm:/dev/shm --device /dev/net/tun --name vpn - variables: - template: vars.yml + - name: REGISTRY + value: registry.access.redhat.com + - name: LOCAL_ARO_RP_IMAGE + value: "arosvcdev.azurecr.io/aro" + - name: LOCAL_ARO_AZEXT_IMAGE + value: "arosvcdev.azurecr.io/azext-aro" + - name: LOCAL_VPN_IMAGE + value: "arosvcdev.azurecr.io/vpn" + - name: TAG + value: $(Build.BuildId) + - name: VERSION + value: $(Build.BuildId) jobs: - job: Build_Test_And_Push_Az_ARO_Extension @@ -38,7 +36,7 @@ jobs: # Build and test the Az ARO Extension - script: | set -xe - DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(Build.BuildId) + DOCKER_BUILD_CI_ARGS="--load" make ci-azext-aro VERSION=$(VERSION) displayName: 🛠 Build & Test Az ARO Extension # Push the image to ACR @@ -57,7 +55,7 @@ jobs: # Build and test RP and Portal - script: | set -xe - DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(Build.BuildId) + DOCKER_BUILD_CI_ARGS="--load" make ci-rp VERSION=$(VERSION) displayName: 🛠 Build & Test RP and Portal # Publish test results diff --git a/.pipelines/templates/template-acr-push.yml b/.pipelines/templates/template-acr-push.yml index aadf07132a2..39b869c0139 100644 --- a/.pipelines/templates/template-acr-push.yml +++ b/.pipelines/templates/template-acr-push.yml @@ -20,6 +20,23 @@ steps: scriptLocation: 'inlineScript' inlineScript: | set -xe + + # Update packages and install docker.io and docker-compose-plugin + echo "Installing Docker and Docker Compose Plugin..." + sudo apt-get update + sudo apt-get install -y ca-certificates curl gnupg + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo tee /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo \"$VERSION_CODENAME\") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + sudo systemctl start docker + sudo systemctl enable docker + echo "Authenticating to Azure and ACR..." ACR_FQDN="${{ parameters.acrFQDN }}" REGISTRY_NAME=$(echo $ACR_FQDN | cut -d'.' -f1) @@ -32,34 +49,37 @@ steps: docker images # Ensure the image is available locally before tagging the build ID - IMAGE_NAME="${{ parameters.repository }}:$(Build.BuildId)" + IMAGE_NAME="${ACR_FQDN}/${{ parameters.repository }}:$(VERSION)" # Use full image name + echo "Checking for image $IMAGE_NAME..." if [[ "$(docker images -q $IMAGE_NAME 2> /dev/null)" == "" ]]; then echo "Error: Image $IMAGE_NAME not found. Exiting." exit 1 fi # Ensure the image is available locally before tagging 'latest' - IMAGE_LATEST="${{ parameters.repository }}:latest" + IMAGE_LATEST="${ACR_FQDN}/${{ parameters.repository }}:latest" + echo "Checking for image $IMAGE_LATEST..." if [[ "$(docker images -q $IMAGE_LATEST 2> /dev/null)" == "" ]]; then - echo "Warning: Image $IMAGE_LATEST not found. Skipping latest tag." + echo "Warning: Image $IMAGE_LATEST not found. Skipping 'latest' tag." SKIP_LATEST=true else + echo "Image $IMAGE_LATEST found. Proceeding with 'latest' tag." SKIP_LATEST=false fi # Tag the image with the ACR repository for the build ID echo "Tagging image with build ID..." - docker tag $IMAGE_NAME ${ACR_FQDN}/${{ parameters.repository }}:$(Build.BuildId) + docker tag $IMAGE_NAME ${ACR_FQDN}/${{ parameters.repository }}:$(VERSION) # If the latest image exists, tag it as well if [ "$SKIP_LATEST" == "false" ]; then echo "Tagging image with 'latest'..." - docker tag $IMAGE_LATEST ${ACR_FQDN}/${{ parameters.repository }}:latest + docker tag $IMAGE_NAME ${ACR_FQDN}/${{ parameters.repository }}:latest fi # Push the Docker image to ACR with build ID echo "Pushing image with build ID to ACR..." - docker push ${ACR_FQDN}/${{ parameters.repository }}:$(Build.BuildId) + docker push ${ACR_FQDN}/${{ parameters.repository }}:$(VERSION) # Optionally push the image as 'latest' if [ "${{ parameters.pushLatest }}" == "true" ] && [ "$SKIP_LATEST" == "false" ]; then diff --git a/Makefile b/Makefile index a39b7a540b8..e62cf99b543 100644 --- a/Makefile +++ b/Makefile @@ -374,7 +374,6 @@ DOCKER_BUILD_CI_ARGS ?= # Image names that will be found in the local podman image registry after build # (tags are always VERSION). -LOCAL_ARO_RP_IMAGE ?= aro LOCAL_ARO_PORTAL_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-portal-build LOCAL_ARO_RP_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-build LOCAL_AZ_EXT_ARO_IMAGE ?= azext-aro @@ -397,30 +396,29 @@ ci-clean: podman $(PODMAN_REMOTE_ARGS) \ image prune --all --filter="label=aro-*=true" +.PHONY: version +version: + echo "RP $(LOCAL_ARO_RP_IMAGE):$(VERSION)" + .PHONY: ci-rp ci-rp: fix-macos-vendor - docker build . $(DOCKER_BUILD_CI_ARGS) \ + docker build . ${DOCKER_BUILD_CI_ARGS} \ -f Dockerfile.ci-rp \ --ulimit=nofile=4096:4096 \ - --build-arg REGISTRY=$(REGISTRY) \ - --build-arg ARO_VERSION=$(VERSION) \ - --no-cache=$(NO_CACHE) \ + --build-arg REGISTRY=${REGISTRY} \ + --build-arg ARO_VERSION=${VERSION} \ + --no-cache=${NO_CACHE} \ --target=builder \ - -t $(LOCAL_ARO_RP_BUILD_IMAGE):$(VERSION) + -t ${LOCAL_ARO_RP_BUILD_IMAGE}:${VERSION} - docker build . $(DOCKER_BUILD_CI_ARGS) \ - -f Dockerfile.ci-rp \ - --ulimit=nofile=4096:4096 \ - --build-arg REGISTRY=$(REGISTRY) \ - --build-arg ARO_VERSION=$(VERSION) \ - -t $(LOCAL_ARO_RP_IMAGE):$(VERSION) + docker compose build rp # Extract test coverage files from build to local filesystem - docker create --name extract_cover_out $(LOCAL_ARO_RP_BUILD_IMAGE):$(VERSION); \ + docker create --name extract_cover_out ${LOCAL_ARO_RP_BUILD_IMAGE}:${VERSION}; \ docker cp extract_cover_out:/app/report.xml ./report.xml; \ docker cp extract_cover_out:/app/coverage.xml ./coverage.xml; \ docker rm extract_cover_out; - + .PHONY: ci-tunnel ci-tunnel: fix-macos-vendor diff --git a/docker-compose.yml b/docker-compose.yml index fc1e12eb6a5..03d922a3b71 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,6 @@ -version: '3.8' - services: vpn: - image: vpn:${VERSION} + image: ${LOCAL_VPN_IMAGE}:${VERSION} build: context: . dockerfile: Dockerfile.vpn @@ -10,6 +8,8 @@ services: privileged: true volumes: - ${PWD}/secrets:/secrets:z + devices: + - /dev/net/tun # required to modify VPN on host entrypoint: "openvpn" command: ["/secrets/vpn-eastus.ovpn"] healthcheck: @@ -18,3 +18,95 @@ services: interval: 20s timeout: 3s retries: 3 + + rp: + image: ${LOCAL_ARO_RP_IMAGE}:${VERSION} + build: + context: . + dockerfile: Dockerfile.ci-rp + args: + - REGISTRY=${REGISTRY} + - ARO_VERSION=${VERSION} + ulimits: + nofile: + soft: 4096 + hard: 4096 + container_name: aro-rp + depends_on: + vpn: + condition: service_healthy + command: ["rp"] + secrets: + - source: proxy-client-key + target: /app/secrets/proxy-client.key + - source: proxy-client-crt + target: /app/secrets/proxy-client.crt + - source: proxy-crt + target: /app/secrets/proxy.crt + - source: hive-kubeconfig + target: /app/secrets/aks.kubeconfig + environment: + # inherit from host + - ADMIN_OBJECT_ID + - ARO_IMAGE + - AZURE_ARM_CLIENT_ID + - AZURE_CLIENT_ID + - AZURE_CLIENT_SECRET + - AZURE_DBTOKEN_CLIENT_ID + - AZURE_ENVIRONMENT + - AZURE_FP_CLIENT_ID + - AZURE_FP_SERVICE_PRINCIPAL_ID + - AZURE_GATEWAY_CLIENT_ID + - AZURE_GATEWAY_CLIENT_SECRET + - AZURE_GATEWAY_SERVICE_PRINCIPAL_ID + - AZURE_PORTAL_ACCESS_GROUP_IDS + - AZURE_PORTAL_CLIENT_ID + - AZURE_RP_CLIENT_ID + - AZURE_RP_CLIENT_SECRET + - AZURE_SERVICE_PRINCIPAL_ID + - AZURE_SUBSCRIPTION_ID + - AZURE_TENANT_ID + - DATABASE_ACCOUNT_NAME + - DATABASE_NAME + - DOMAIN_NAME + - KEYVAULT_PREFIX + - LOCATION + - MOCK_MSI_CERT + - MOCK_MSI_CLIENT_ID + - MOCK_MSI_TENANT_ID + - OIDC_STORAGE_ACCOUNT_NAME + - PARENT_DOMAIN_NAME + - PARENT_DOMAIN_RESOURCEGROUP + - PROXY_HOSTNAME + - PULL_SECRET + - RESOURCEGROUP + - SECRET_SA_ACCOUNT_NAME + - STORAGE_ACCOUNT_DOMAIN + + # override + - ARO_ADOPT_BY_HIVE=true + - ARO_CHECKOUT_PATH=/app + - ARO_INSTALL_VIA_HIVE=true + - HIVE_KUBE_CONFIG_PATH=/app/secrets/aks.kubeconfig + - KUBECONFIG=/app/secrets/aks.kubeconfig + - RP_MODE=development + expose: + - "8443" + ports: + - "127.0.0.1:8443:8443" + healthcheck: + test: ["CMD", "curl", "-k", "http://localhost:8443/healthz"] + interval: 30s + timeout: 30s + retries: 3 + restart: on-failure:3 + +secrets: + proxy-client-key: + file: ./secrets/proxy-client.key + proxy-client-crt: + file: ./secrets/proxy-client.crt + proxy-crt: + file: ./secrets/proxy.crt + hive-kubeconfig: + file: ./aks.kubeconfig From d343005e39996272970f722cda4bb483207b56fc Mon Sep 17 00:00:00 2001 From: Shubhada Date: Fri, 4 Oct 2024 13:12:52 -0700 Subject: [PATCH 3/9] Ensures that the Docker image is correctly retagged with the repository prefix before it is pushed to the ACR. --- .pipelines/templates/template-acr-push.yml | 38 ++++++++++------------ Makefile | 4 --- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/.pipelines/templates/template-acr-push.yml b/.pipelines/templates/template-acr-push.yml index 39b869c0139..265d7fb51be 100644 --- a/.pipelines/templates/template-acr-push.yml +++ b/.pipelines/templates/template-acr-push.yml @@ -11,7 +11,6 @@ parameters: default: false steps: - # Authenticate to ACR and push the image - task: AzureCLI@2 displayName: 'Authenticate to Azure and Push Docker Image' inputs: @@ -21,7 +20,7 @@ steps: inlineScript: | set -xe - # Update packages and install docker.io and docker-compose-plugin + # Install Docker dependencies echo "Installing Docker and Docker Compose Plugin..." sudo apt-get update sudo apt-get install -y ca-certificates curl gnupg @@ -37,6 +36,7 @@ steps: sudo systemctl start docker sudo systemctl enable docker + # Authenticate to Azure and ACR echo "Authenticating to Azure and ACR..." ACR_FQDN="${{ parameters.acrFQDN }}" REGISTRY_NAME=$(echo $ACR_FQDN | cut -d'.' -f1) @@ -48,12 +48,17 @@ steps: echo "Listing Docker images..." docker images - # Ensure the image is available locally before tagging the build ID - IMAGE_NAME="${ACR_FQDN}/${{ parameters.repository }}:$(VERSION)" # Use full image name - echo "Checking for image $IMAGE_NAME..." - if [[ "$(docker images -q $IMAGE_NAME 2> /dev/null)" == "" ]]; then - echo "Error: Image $IMAGE_NAME not found. Exiting." + # Check if the image is already tagged with the full repository path + IMAGE_NAME="${ACR_FQDN}/${{ parameters.repository }}:$(VERSION)" + LOCAL_IMAGE="${{ parameters.repository }}:$(VERSION)" + + echo "Checking for image $LOCAL_IMAGE..." + if [[ "$(docker images -q $LOCAL_IMAGE 2> /dev/null)" == "" ]]; then + echo "Error: Image $LOCAL_IMAGE not found. Exiting." exit 1 + else + echo "Image found. Retagging with full repository path..." + docker tag $LOCAL_IMAGE $IMAGE_NAME fi # Ensure the image is available locally before tagging 'latest' @@ -67,22 +72,13 @@ steps: SKIP_LATEST=false fi - # Tag the image with the ACR repository for the build ID - echo "Tagging image with build ID..." - docker tag $IMAGE_NAME ${ACR_FQDN}/${{ parameters.repository }}:$(VERSION) - - # If the latest image exists, tag it as well - if [ "$SKIP_LATEST" == "false" ]; then - echo "Tagging image with 'latest'..." - docker tag $IMAGE_NAME ${ACR_FQDN}/${{ parameters.repository }}:latest - fi - - # Push the Docker image to ACR with build ID + # Push the Docker image to ACR with the build ID echo "Pushing image with build ID to ACR..." - docker push ${ACR_FQDN}/${{ parameters.repository }}:$(VERSION) + docker push $IMAGE_NAME # Optionally push the image as 'latest' if [ "${{ parameters.pushLatest }}" == "true" ] && [ "$SKIP_LATEST" == "false" ]; then - echo "Pushing 'latest' tag to ACR..." - docker push ${ACR_FQDN}/${{ parameters.repository }}:latest + echo "Tagging image with 'latest' and pushing..." + docker tag $IMAGE_NAME $IMAGE_LATEST + docker push $IMAGE_LATEST fi diff --git a/Makefile b/Makefile index e62cf99b543..b3658cf0902 100644 --- a/Makefile +++ b/Makefile @@ -396,10 +396,6 @@ ci-clean: podman $(PODMAN_REMOTE_ARGS) \ image prune --all --filter="label=aro-*=true" -.PHONY: version -version: - echo "RP $(LOCAL_ARO_RP_IMAGE):$(VERSION)" - .PHONY: ci-rp ci-rp: fix-macos-vendor docker build . ${DOCKER_BUILD_CI_ARGS} \ From 8db13ea0a50e5d06f10d0344f8028bbdc93ebd14 Mon Sep 17 00:00:00 2001 From: Shubhada Date: Fri, 4 Oct 2024 13:46:16 -0700 Subject: [PATCH 4/9] Replaced the whole run-rp with docker compose --- .pipelines/templates/template-acr-push.yml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.pipelines/templates/template-acr-push.yml b/.pipelines/templates/template-acr-push.yml index 265d7fb51be..8a37d3bdd97 100644 --- a/.pipelines/templates/template-acr-push.yml +++ b/.pipelines/templates/template-acr-push.yml @@ -48,17 +48,25 @@ steps: echo "Listing Docker images..." docker images - # Check if the image is already tagged with the full repository path + # Define both the full repository image name and the local name IMAGE_NAME="${ACR_FQDN}/${{ parameters.repository }}:$(VERSION)" LOCAL_IMAGE="${{ parameters.repository }}:$(VERSION)" - echo "Checking for image $LOCAL_IMAGE..." - if [[ "$(docker images -q $LOCAL_IMAGE 2> /dev/null)" == "" ]]; then - echo "Error: Image $LOCAL_IMAGE not found. Exiting." - exit 1 + # Check if the image exists locally with the full repository tag + echo "Checking for image $IMAGE_NAME..." + if [[ "$(docker images -q $IMAGE_NAME 2> /dev/null)" == "" ]]; then + # If the full repository tagged image does not exist, check for the local image + echo "Full repository image not found. Checking for local image $LOCAL_IMAGE..." + if [[ "$(docker images -q $LOCAL_IMAGE 2> /dev/null)" == "" ]]; then + echo "Error: Neither $IMAGE_NAME nor $LOCAL_IMAGE found. Exiting." + exit 1 + else + # Retag the local image with the full repository path + echo "Local image $LOCAL_IMAGE found. Retagging with full repository path..." + docker tag $LOCAL_IMAGE $IMAGE_NAME + fi else - echo "Image found. Retagging with full repository path..." - docker tag $LOCAL_IMAGE $IMAGE_NAME + echo "Image $IMAGE_NAME found. Proceeding to push..." fi # Ensure the image is available locally before tagging 'latest' From 19df15ebc4241fef9694e186910a3f03e75d1e6b Mon Sep 17 00:00:00 2001 From: Shubhada Date: Fri, 4 Oct 2024 14:30:07 -0700 Subject: [PATCH 5/9] Added env veriables for docker compose run --- Makefile | 58 ++++------------------------------------------------- env.example | 6 ++++++ 2 files changed, 10 insertions(+), 54 deletions(-) diff --git a/Makefile b/Makefile index b3658cf0902..b11b21ad382 100644 --- a/Makefile +++ b/Makefile @@ -374,6 +374,7 @@ DOCKER_BUILD_CI_ARGS ?= # Image names that will be found in the local podman image registry after build # (tags are always VERSION). +LOCAL_ARO_RP_IMAGE ?= aro LOCAL_ARO_PORTAL_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-portal-build LOCAL_ARO_RP_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-build LOCAL_AZ_EXT_ARO_IMAGE ?= azext-aro @@ -480,60 +481,9 @@ run-portal: ci-rp podman-secrets # run-rp executes the RP locally as similarly as possible to production. That # includes the use of Hive, meaning you need a VPN connection. .PHONY: run-rp -run-rp: ci-rp podman-secrets - podman $(PODMAN_REMOTE_ARGS) \ - run \ - --name aro-rp \ - --rm \ - -p 127.0.0.1:8443:8443 \ - -w /app \ - -e ARO_IMAGE \ - -e RP_MODE="development" \ - -e PROXY_HOSTNAME \ - -e DOMAIN_NAME \ - -e AZURE_RP_CLIENT_ID \ - -e AZURE_FP_CLIENT_ID \ - -e AZURE_SUBSCRIPTION_ID \ - -e AZURE_TENANT_ID \ - -e AZURE_RP_CLIENT_SECRET \ - -e LOCATION \ - -e RESOURCEGROUP \ - -e AZURE_ARM_CLIENT_ID \ - -e AZURE_FP_SERVICE_PRINCIPAL_ID \ - -e AZURE_DBTOKEN_CLIENT_ID \ - -e AZURE_PORTAL_CLIENT_ID \ - -e AZURE_PORTAL_ACCESS_GROUP_IDS \ - -e AZURE_CLIENT_ID \ - -e AZURE_SERVICE_PRINCIPAL_ID \ - -e AZURE_CLIENT_SECRET \ - -e AZURE_GATEWAY_CLIENT_ID \ - -e AZURE_GATEWAY_SERVICE_PRINCIPAL_ID \ - -e AZURE_GATEWAY_CLIENT_SECRET \ - -e DATABASE_NAME \ - -e PULL_SECRET \ - -e SECRET_SA_ACCOUNT_NAME \ - -e DATABASE_ACCOUNT_NAME \ - -e KEYVAULT_PREFIX \ - -e ADMIN_OBJECT_ID \ - -e PARENT_DOMAIN_NAME \ - -e PARENT_DOMAIN_RESOURCEGROUP \ - -e AZURE_ENVIRONMENT \ - -e STORAGE_ACCOUNT_DOMAIN \ - -e OIDC_STORAGE_ACCOUNT_NAME \ - -e KUBECONFIG="/app/secrets/aks.kubeconfig" \ - -e HIVE_KUBE_CONFIG_PATH="/app/secrets/aks.kubeconfig" \ - -e ARO_CHECKOUT_PATH="/app" \ - -e ARO_INSTALL_VIA_HIVE="true" \ - -e ARO_ADOPT_BY_HIVE="true" \ - -e MOCK_MSI_TENANT_ID \ - -e MOCK_MSI_CLIENT_ID \ - -e MOCK_MSI_OBJECT_ID \ - -e MOCK_MSI_CERT \ - --secret aks.kubeconfig,target=/app/secrets/aks.kubeconfig \ - --secret proxy-client.key,target=/app/secrets/proxy-client.key \ - --secret proxy-client.crt,target=/app/secrets/proxy-client.crt \ - --secret proxy.crt,target=/app/secrets/proxy.crt \ - $(LOCAL_ARO_RP_IMAGE):$(VERSION) rp +run-rp: + docker compose rm -sf rp + docker compose up rp .PHONY: vpn vpn: diff --git a/env.example b/env.example index 96e92defd3d..b8e5dd69c51 100644 --- a/env.example +++ b/env.example @@ -18,4 +18,10 @@ export MOCK_MSI_CERT="replace_with_value_output_by_hack/devtools/msi.sh" export MOCK_MSI_TENANT_ID="replace_with_value_output_by_hack/devtools/msi.sh" export PLATFORM_WORKLOAD_IDENTITY_ROLE_SETS="replace_with_value_output_by_hack/devtools/msi.sh" +# you will need this to run-rp , vpn and ci-rp using Docker compose +export REGISTRY=registry.access.redhat.com +export VERSION=latest +export TAG=latest +export LOCAL_VPN_IMAGE=arosvcdev.azurecr.io/vpn + . secrets/env From f219bc2f3a399456290bd137b48f313ed532adde Mon Sep 17 00:00:00 2001 From: Shubhada Date: Mon, 7 Oct 2024 12:39:39 -0700 Subject: [PATCH 6/9] Migrated of portal to compose --- Makefile | 28 ++-------------------------- docker-compose.yml | 41 +++++++++++++++++++++++++++++++++++++++++ env.example | 1 + 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index b11b21ad382..494d65a152d 100644 --- a/Makefile +++ b/Makefile @@ -451,32 +451,8 @@ podman-secrets: aks.kubeconfig podman $(PODMAN_REMOTE_ARGS) secret create proxy.crt ./secrets/proxy.crt .PHONY: run-portal -run-portal: ci-rp podman-secrets - podman $(PODMAN_REMOTE_ARGS) \ - run \ - --name aro-portal \ - --rm \ - -p 127.0.0.1:8444:8444 \ - -p 127.0.0.1:2222:2222 \ - --cap-drop net_raw \ - -e RP_MODE \ - -e AZURE_SUBSCRIPTION_ID \ - -e AZURE_TENANT_ID \ - -e LOCATION \ - -e RESOURCEGROUP \ - -e AZURE_PORTAL_CLIENT_ID \ - -e AZURE_PORTAL_ELEVATED_GROUP_IDS \ - -e AZURE_PORTAL_ACCESS_GROUP_IDS \ - -e AZURE_RP_CLIENT_SECRET \ - -e AZURE_RP_CLIENT_ID \ - -e KEYVAULT_PREFIX \ - -e DATABASE_ACCOUNT_NAME \ - -e DATABASE_NAME \ - -e NO_NPM=1 \ - --secret proxy-client.key,target=/app/secrets/proxy-client.key \ - --secret proxy-client.crt,target=/app/secrets/proxy-client.crt \ - --secret proxy.crt,target=/app/secrets/proxy.crt \ - $(LOCAL_ARO_RP_IMAGE):$(VERSION) portal +run-portal: + docker compose up portal # run-rp executes the RP locally as similarly as possible to production. That # includes the use of Hive, meaning you need a VPN connection. diff --git a/docker-compose.yml b/docker-compose.yml index 03d922a3b71..370a58f91da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,6 +101,47 @@ services: retries: 3 restart: on-failure:3 + portal: + image: ${LOCAL_ARO_RP_IMAGE}:${VERSION} + container_name: aro-portal + depends_on: + rp: + condition: service_started + environment: + - RP_MODE + - AZURE_SUBSCRIPTION_ID + - AZURE_TENANT_ID + - LOCATION + - RESOURCEGROUP + - AZURE_PORTAL_CLIENT_ID + - AZURE_PORTAL_ELEVATED_GROUP_IDS + - AZURE_PORTAL_ACCESS_GROUP_IDS + - AZURE_RP_CLIENT_SECRET + - AZURE_RP_CLIENT_ID + - KEYVAULT_PREFIX + - DATABASE_ACCOUNT_NAME + - DATABASE_NAME + - NO_NPM=1 + ports: + - "127.0.0.1:8444:8444" + - "127.0.0.1:2222:2222" + secrets: + - source: proxy-client-key + target: /app/secrets/proxy-client.key + - source: proxy-client-crt + target: /app/secrets/proxy-client.crt + - source: proxy-crt + target: /app/secrets/proxy.crt + cap_drop: + - NET_RAW + command: ["portal"] + restart: on-failure:3 + healthcheck: + test: ["CMD", "curl", "-k", "http://localhost:8444/healthz"] + interval: 30s + timeout: 10s + retries: 3 + secrets: proxy-client-key: file: ./secrets/proxy-client.key diff --git a/env.example b/env.example index b8e5dd69c51..c4a8add102e 100644 --- a/env.example +++ b/env.example @@ -20,6 +20,7 @@ export PLATFORM_WORKLOAD_IDENTITY_ROLE_SETS="replace_with_value_output_by_hack/d # you will need this to run-rp , vpn and ci-rp using Docker compose export REGISTRY=registry.access.redhat.com +export LOCAL_ARO_RP_IMAGE=localhost/aro export VERSION=latest export TAG=latest export LOCAL_VPN_IMAGE=arosvcdev.azurecr.io/vpn From e7c56172f9fd64e8d75d1e42b87f7870adf06d64 Mon Sep 17 00:00:00 2001 From: Shubhada Date: Wed, 9 Oct 2024 10:04:47 -0700 Subject: [PATCH 7/9] refactor: simplify setup, use host network for VPN, and improve service health checks --- Makefile | 9 --------- docker-compose.yml | 3 ++- env.example | 4 ++-- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 494d65a152d..fafd8319a17 100644 --- a/Makefile +++ b/Makefile @@ -461,12 +461,3 @@ run-rp: docker compose rm -sf rp docker compose up rp -.PHONY: vpn -vpn: - docker compose build vpn - -.PHONY: run-vpn -run-vpn: - docker compose rm -sf vpn - docker compose up vpn - docker compose logs --follow vpn diff --git a/docker-compose.yml b/docker-compose.yml index 370a58f91da..7d88a092198 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: dockerfile: Dockerfile.vpn container_name: vpn privileged: true + network_mode: host volumes: - ${PWD}/secrets:/secrets:z devices: @@ -106,7 +107,7 @@ services: container_name: aro-portal depends_on: rp: - condition: service_started + condition: service_healthy environment: - RP_MODE - AZURE_SUBSCRIPTION_ID diff --git a/env.example b/env.example index c4a8add102e..b2a4135faaa 100644 --- a/env.example +++ b/env.example @@ -20,9 +20,9 @@ export PLATFORM_WORKLOAD_IDENTITY_ROLE_SETS="replace_with_value_output_by_hack/d # you will need this to run-rp , vpn and ci-rp using Docker compose export REGISTRY=registry.access.redhat.com -export LOCAL_ARO_RP_IMAGE=localhost/aro +export LOCAL_ARO_RP_IMAGE=aro export VERSION=latest export TAG=latest -export LOCAL_VPN_IMAGE=arosvcdev.azurecr.io/vpn +export LOCAL_VPN_IMAGE=vpn . secrets/env From f4a2a40661287169a471e1840582c0b351eabccc Mon Sep 17 00:00:00 2001 From: Shubhada Date: Wed, 9 Oct 2024 11:49:25 -0700 Subject: [PATCH 8/9] The target has been removed as Docker Compose now fully manages the secrets --- Makefile | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/Makefile b/Makefile index fafd8319a17..fe6c828b389 100644 --- a/Makefile +++ b/Makefile @@ -415,7 +415,6 @@ ci-rp: fix-macos-vendor docker cp extract_cover_out:/app/report.xml ./report.xml; \ docker cp extract_cover_out:/app/coverage.xml ./coverage.xml; \ docker rm extract_cover_out; - .PHONY: ci-tunnel ci-tunnel: fix-macos-vendor @@ -436,20 +435,6 @@ ifeq ($(shell uname -s),Darwin) mv ./vendor/github.com/Microsoft ./vendor/github.com/temp-microsoft && mv ./vendor/github.com/temp-microsoft ./vendor/github.com/microsoft || true endif -.PHONY: podman-secrets -podman-secrets: aks.kubeconfig - podman $(PODMAN_REMOTE_ARGS) secret rm --ignore aks.kubeconfig - podman $(PODMAN_REMOTE_ARGS) secret create aks.kubeconfig ./aks.kubeconfig - - podman $(PODMAN_REMOTE_ARGS) secret rm --ignore proxy-client.key - podman $(PODMAN_REMOTE_ARGS) secret create proxy-client.key ./secrets/proxy-client.key - - podman $(PODMAN_REMOTE_ARGS) secret rm --ignore proxy-client.crt - podman $(PODMAN_REMOTE_ARGS) secret create proxy-client.crt ./secrets/proxy-client.crt - - podman $(PODMAN_REMOTE_ARGS) secret rm --ignore proxy.crt - podman $(PODMAN_REMOTE_ARGS) secret create proxy.crt ./secrets/proxy.crt - .PHONY: run-portal run-portal: docker compose up portal From 0249c984827f9ade7925b4ff8dd4910b54b3b9a7 Mon Sep 17 00:00:00 2001 From: Shubhada Date: Wed, 9 Oct 2024 12:23:19 -0700 Subject: [PATCH 9/9] Added aks.kubeconfig as a dependency for the run-rp target --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index fe6c828b389..36c95bff746 100644 --- a/Makefile +++ b/Makefile @@ -442,7 +442,6 @@ run-portal: # run-rp executes the RP locally as similarly as possible to production. That # includes the use of Hive, meaning you need a VPN connection. .PHONY: run-rp -run-rp: +run-rp: aks.kubeconfig docker compose rm -sf rp docker compose up rp -