fix(toil): simplify Argo deployment and debuggability (#1503)

Author: tommartensen

.github/workflows/PR.yaml

@@ -106,7 +106,7 @@ jobs:
 
       - name: Deploy infra to dev cluster
         run: |
-          ENVIRONMENT=development TEST_MODE=true make install-argo clean-argo-config install-monitoring helm-deploy
+          ENVIRONMENT=development TEST_MODE=true make helm-deploy
           sleep 10 # wait for old pods to disappear so the svc port-forward doesn't connect to them
           kubectl -n infra port-forward svc/infra-server-service 8443:8443 > /dev/null 2>&1 &
           sleep 10

.github/workflows/deploy.yaml

@@ -71,7 +71,7 @@ jobs:
           gcloud container clusters get-credentials infra-${{ inputs.environment }} \
             --project "${PROJECT}" \
             --region us-west2
-          ENVIRONMENT=${{ inputs.environment }} make install-argo clean-argo-config install-monitoring helm-deploy
+          ENVIRONMENT=${{ inputs.environment }} make helm-deploy
 
       - name: Notify infra channel about new version
         uses: slackapi/slack-github-action@v2.0.0

DEPLOYMENT.md

@@ -94,17 +94,6 @@ correct tooling installed with:
 
 Use the `deploy` Github action to update development or production environments with a new release.
 
-### Argo Deployment
-
-To install Argo workflow server, run:
-
-`ENVIRONMENT=<development,production> make install-argo`
-
-NOTE: This is a separate step and not a dependant chart for example to avoid too frequent Argo deployments.
-
-Also note that if you plan to deploy `infra-server` to this cluster, you will need to remove the default Argo workflow controller ConfigMap with the `clean-argo-config` Make target.
-This is required until [ROX-20269](https://issues.redhat.com/browse/ROX-20269) is resolved.
-
 ### Manual deployment
 
 To render a copy of the charts (for inspection), run:
@@ -156,3 +145,13 @@ The infra server logs are captured automatically by GCP.
 - [Logs Explorer: Production](https://cloudlogging.app.goo.gl/KqgSyE2mSq83M5Xs9)
 
 Adding `jsonPayload."log-type"="audit"` to the query will filter for audit logs.
+
+## Inspecting live workflows
+
+You can view the UI of the Argo server by forwarding its port:
+
+```bash
+kubectl port-forward -n argo svc/infra-server-argo-workflows-server 2746
+```
+
+and access [http://localhost:2746](http://localhost:2746).

Makefile

@@ -223,21 +223,27 @@ endif
 		exit 1; \
 	fi
 
+.PHONY: helm-dependency-update
+helm-dependency-update:
+	@helm dependency update chart/infra-server
+
+create-namespaces:
+	@kubectl create namespace argo >/dev/null 2>&1 || echo "namespace/argo already exists"; exit 0
+	@kubectl create namespace monitoring >/dev/null 2>&1 || echo "namespace/monitoring already exists"; exit 0
+
 ## Render template
 .PHONY: helm-template
-helm-template: pre-check
+helm-template: pre-check helm-dependency-update create-namespaces
 	@./scripts/deploy/helm.sh template $(VERSION) $(ENVIRONMENT) $(SECRET_VERSION)
 
 ## Deploy
 .PHONY: helm-deploy
-helm-deploy: pre-check
+helm-deploy: pre-check helm-dependency-update create-namespaces
 	@./scripts/deploy/helm.sh deploy $(VERSION) $(ENVIRONMENT) $(SECRET_VERSION)
-	# Pick up any eventual changes to the workflow controller configmap
-	@make bounce-argo-pods
 
 ## Diff
 .PHONY: helm-diff
-helm-diff: pre-check
+helm-diff: pre-check helm-dependency-update create-namespaces
 	@./scripts/deploy/helm.sh diff $(VERSION) $(ENVIRONMENT) $(SECRET_VERSION)
 
 ## Bounce pods
@@ -276,34 +282,6 @@ secrets-edit:
 secrets-revert:
 	@./scripts/deploy/secrets.sh revert $(ENVIRONMENT) $(SECRET_VERSION)
 
-##################
-## Dependencies ##
-##################
-.PHONY: install-argo
-install-argo: pre-check
-	helm repo add argo https://argoproj.github.io/argo-helm
-	helm upgrade \
-		argo-workflows \
-		argo/argo-workflows \
-		--version 0.16.9 \
-		--install \
-		--create-namespace \
-		--namespace argo
-
-.PHONY: clean-argo-config
-clean-argo-config: pre-check
-	kubectl delete configmap argo-workflows-workflow-controller-configmap -n argo || true
-
-.PHONY: install-monitoring
-install-monitoring: pre-check
-	helm dependency update chart/infra-monitoring
-	helm upgrade prometheus-stack chart/infra-monitoring \
-		--install \
-		--namespace monitoring \
-		--create-namespace \
-		--values chart/infra-monitoring/values.yaml \
-		--wait
-
 ###############
 ## Debugging ##
 ###############

README.md

@@ -32,7 +32,7 @@ You may also point to a different infra server with the `--endpoint` flag.
 To debug the server, you need to fulfil the prerequisites first.
 
 1. Have an authenticated `gcloud` CLI and GNU `sed` installed.
-1. Have your `KUBECONFIG` point to a cluster where Argo Workflows and the ConfigMaps and Secrets for infra are deployed. This is most easily achieved by connecting to a PR cluster or deploying infra with `ENVIRONMENT=<DEVELOPMENT,PRODUCTION> make install-argo clean-argo-config helm-deploy` to a new or local cluster. This cluster will only be used to run workflows.
+1. Have your `KUBECONFIG` point to a cluster where Argo Workflows and the ConfigMaps and Secrets for infra are deployed. This is most easily achieved by connecting to a PR cluster or deploying infra with `ENVIRONMENT=<DEVELOPMENT,PRODUCTION> make helm-deploy` to a new or local cluster. This cluster will only be used to run workflows.
 1. Run `make prepare-local-server-debugging` to set the contents of the `configuration` directory and compile the UI + CLI (for downloads).
 
 Then, you can use the "Debug Server" launch configuration.

chart/infra-monitoring/.gitignore

@@ -0,0 +1,2 @@
+# Ignore the dependency chart archives
+charts/

chart/infra-monitoring/.helmignore

@@ -10,3 +10,10 @@ annotations:
   acsDemoVersion: 4.6.2
   automationFlavorsVersion: 0.10.43
   ocpCredentialsMode: Passthrough
+dependencies:
+- name: argo-workflows
+  version: "0.45.9"
+  repository: "https://argoproj.github.io/argo-helm"
+- name: kube-prometheus
+  version: 11.1.1
+  repository: https://charts.bitnami.com/bitnami

chart/infra-monitoring/Chart.yaml

@@ -0,0 +1,26 @@
+argo-workflows:
+  namespaceOverride: argo
+  server:
+    authModes:
+      - server
+
+  controller:
+    # Default values that will apply to all Workflows from this controller, unless overridden on the Workflow-level
+    workflowDefaults:
+      metadata:
+        annotations:
+          argo: workflows
+      spec:
+        ttlStrategy:
+          # Keep the workflow pods & logs available for 30 days
+          secondsAfterCompletion: 2592000
+          secondsAfterSuccess: 2592000
+          secondsAfterFailure: 2592000
+
+  artifactRepository:
+    archiveLogs: true
+    gcs:
+      bucket: rhacs-infra-artifacts
+      serviceAccountKeySecret:
+        name: gcs-credentials
+        key: credentials.json

chart/infra-monitoring/requirements.lock

@@ -1,4 +1,5 @@
 kube-prometheus:
+  namespaceOverride: monitoring
   operator:
     resources:
       limits:

chart/infra-server/.gitignore

@@ -0,0 +1,9 @@
+dependencies:
+- name: argo-workflows
+  repository: https://argoproj.github.io/argo-helm
+  version: 0.45.9
+- name: kube-prometheus
+  repository: https://charts.bitnami.com/bitnami
+  version: 11.1.1
+digest: sha256:46322f064751933585c0985dad77996ed8ee216df0fed19ea9c40f7935f67ae7
+generated: "2025-03-05T10:30:52.962223+01:00"

chart/infra-server/Chart.yaml

@@ -1,45 +1,9 @@
 ---
-
-apiVersion: v1
-kind: ConfigMap
-
-metadata:
-  name: argo-workflows-workflow-controller-configmap
-  namespace: argo
-
-data:
-  config: |
-    artifactRepository:
-      archiveLogs: true
-      gcs:
-        bucket: rhacs-infra-artifacts
-        serviceAccountKeySecret:
-          name: gcs-credentials
-          key: credentials.json
-
-    # Default values that will apply to all Workflows from this controller, unless overridden on the Workflow-level
-    workflowDefaults:
-      metadata:
-        annotations:
-          argo: workflows
-      spec:
-        ttlStrategy:
-          # Keep the workflow pods & logs available for 30 days
-          secondsAfterCompletion: 2592000
-          secondsAfterSuccess: 2592000
-          secondsAfterFailure: 2592000
-
----
-
 apiVersion: v1
 kind: Secret
-
 metadata:
   name: gcs-credentials
   namespace: default
-
 data:
   credentials.json: |-
     {{ required ".Values.google_credentials_json is undefined" .Values.google_credentials_json }}
-
----

chart/infra-server/argo-values.yaml

@@ -7,6 +7,9 @@ metadata:
 spec:
   endpoints:
   - port: metrics
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
   selector:
     matchLabels:
       app: workflow-controller

chart/infra-monitoring/values.yaml chart/infra-server/monitoring-values.yaml

@@ -51,7 +51,7 @@ bin/infractl -k -e localhost:8443 whoami
 :rocket: If you only modify configuration (chart/infra-server/configuration) or templates (chart/infra-server/{static,templates}), you can get a faster update with:
 
 \`\`\`
-make install-local
+make helm-deploy
 \`\`\`
 
 ### Logs

chart/infra-server/requirements.lock

@@ -34,6 +34,8 @@ template() {
         --create-namespace \
         --dry-run \
         --namespace "${RELEASE_NAMESPACE}" \
+        --values chart/infra-server/argo-values.yaml \
+        --values chart/infra-server/monitoring-values.yaml \
         --set tag="${TAG}" \
         --set environment="${ENVIRONMENT}" \
         --set testMode="${TEST_MODE}" \
@@ -57,6 +59,8 @@ deploy() {
         --timeout 5m \
         --wait \
         --namespace "${RELEASE_NAMESPACE}" \
+        --values chart/infra-server/argo-values.yaml \
+        --values chart/infra-server/monitoring-values.yaml \
         --set tag="${TAG}" \
         --set environment="${ENVIRONMENT}" \
         --set testMode="${TEST_MODE}" \
@@ -80,6 +84,8 @@ diff() {
         --create-namespace \
         --dry-run \
         --namespace "${RELEASE_NAMESPACE}" \
+        --values chart/infra-server/argo-values.yaml \
+        --values chart/infra-server/monitoring-values.yaml \
         --set tag="${TAG}" \
         --set environment="${ENVIRONMENT}" \
         --set testMode="${TEST_MODE}" \