From d95760ebb6d285af3b050909a41a3691fb903b93 Mon Sep 17 00:00:00 2001 From: Daniele De Lorenzi <2905124+dark-vex@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:36:36 +0200 Subject: [PATCH 1/3] ci(validate-oc-ampere): add e2e cluster validation workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit oc-ampere had zero live e2e coverage (only offline flate/kubeconform schema checks) despite being a real production Flux Operator cluster. Ports the same top-level-resource loop (Mechanism 1) and fail-closed per-app loop (Mechanism 2) already merged for kubenuc/k8s-vms-daniele (#1790-#1795), extended with a new fourth resolution tier: a dirname()-based fallback that parses the root apps/kustomization.yaml for apps with no per-app deploy.yaml of their own. 2 of 4 real apps deploy for real: flux-operator (real-path fallback; verified via `helm template` with empty values that its chart installs only a Deployment/RBAC/CRDs with no hooks and stays idle until a FluxInstance CR exists — flux-instance.yaml itself stays skip-listed in this job, which bootstraps Flux via plain `flux install` instead) and ngx-webhook (new tier-4 resolver; no credentials, no outbound calls, NodePort-only). system-upgrade-controller and teleport-agent are excluded for the same live-external-side-effect reasons as k3s-rabbit's. Production oc-ampere's compute is genuinely ARM64 (terraform/oci/k8s-armchair, VM.Standard.A1.Flex); this e2e run validates manifest/reconciliation correctness on the (undocumented, presumed x86_64) self-hosted runner, not ARM64 image availability — stated plainly in the workflow's header comment. The comment-only touch to apps/ngx-webhook/manifests/deploy.yml exercises the tier-4 resolver on this workflow's first real CI run. Signed-off-by: Daniele De Lorenzi <2905124+dark-vex@users.noreply.github.com> --- .github/workflows/validate-oc-ampere.yml | 477 ++++++++++++++++++ .../apps/ngx-webhook/manifests/deploy.yml | 5 + 2 files changed, 482 insertions(+) create mode 100644 .github/workflows/validate-oc-ampere.yml diff --git a/.github/workflows/validate-oc-ampere.yml b/.github/workflows/validate-oc-ampere.yml new file mode 100644 index 000000000..321331fde --- /dev/null +++ b/.github/workflows/validate-oc-ampere.yml @@ -0,0 +1,477 @@ +name: oc-ampere-e2e + +on: + pull_request: + branches: [main] + paths: + - 'clusters/oc-ampere/**' + +permissions: + contents: read + +jobs: + detect-changes: + runs-on: self-hosted + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + outputs: + deploy_apps: ${{ steps.detect.outputs.deploy_apps }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + + - name: Detect changed apps + id: detect + run: | + # No infra baseline — oc-ampere has no apps that need to be + # present before other apps can reconcile. + INFRA_APPS="" + + # Detect changed app directories in oc-ampere. The grep requires + # a further '/' after 'apps/' so a changed file sitting directly + # in apps/ (e.g. apps/kustomization.yaml itself) is never + # mistaken for an app name — cut -d'/' -f1 would otherwise turn a + # bare 'apps/kustomization.yaml' diff into the literal bogus app + # name 'kustomization.yaml'. This bit kubenuc's and + # k8s-vms-daniele's e2e workflows twice before it was made + # structural there (see validate-kubenuc.yml); baking the fix in + # from day one here instead of waiting to rediscover it. + CHANGED=$(git diff --name-only origin/main...HEAD \ + | grep -E 'clusters/oc-ampere/apps/.+/' \ + | sed 's|clusters/oc-ampere/apps/||' \ + | cut -d'/' -f1 \ + | sort -u \ + | tr '\n' ' ') + + ALL_APPS=$(echo "$INFRA_APPS $CHANGED" | tr ' ' '\n' | sort -u | tr '\n' ' ' | xargs) + echo "Deploying apps: $ALL_APPS" + echo "deploy_apps=$ALL_APPS" >> "$GITHUB_OUTPUT" + + cluster-test: + needs: detect-changes + runs-on: self-hosted + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + timeout-minutes: 60 + env: + DEPLOY_APPS: ${{ needs.detect-changes.outputs.deploy_apps }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: '>=3.11' + + - name: Setup Flux + uses: fluxcd/flux2/action@16602fa989daa99762f1c6d1186ae2ad1c735815 # v2.9.3 + with: + version: 2.9.3 + + # NOTE — architecture mismatch: production oc-ampere's compute is + # confirmed genuinely ARM64 (terraform/oci/k8s-armchair/main.tf, + # `shape = "VM.Standard.A1.Flex"`). This job's self-hosted runner's + # own architecture is not documented anywhere in this repo — not + # confirmed x86_64 by any repo file, but there's no evidence of + # ARM64 self-hosted capacity either. This run validates manifest + # correctness and reconciliation health, NOT ARM64 image + # availability. Images not verified to have an arm64 variant by this + # workflow: nginxinc/nginx-unprivileged:1.31-alpine (ngx-webhook), + # ghcr.io/controlplaneio-fluxcd/flux-operator:v0.57.0 (flux-operator). + - name: Setup k3s + run: | + set -euo pipefail + + # Install k3s + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik --write-kubeconfig-mode=644" INSTALL_K3S_VERSION="v1.33.3+k3s1" sh - + mkdir -p ~/.kube + sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config + sudo chown $(id -u):$(id -g) ~/.kube/config + + # Helper: wait for systemd unit to become active + wait_for_unit() { + local unit="$1" + local timeout="${2:-120}" # seconds + local interval=5 + local waited=0 + + if ! command -v systemctl >/dev/null 2>&1; then + echo "systemctl not present on runner; cannot check systemd unit $unit" + return 2 + fi + + echo "Checking systemd unit: $unit" + + if ! sudo systemctl list-unit-files | grep -q "^${unit}\."; then + echo "Unit ${unit} not found in systemd unit-files. Continuing to poll in case it appears..." + fi + + while [ $waited -lt "$timeout" ]; do + if sudo systemctl is-active --quiet "$unit"; then + echo "Unit ${unit} is active" + if sudo systemctl is-failed --quiet "$unit"; then + echo "Unit ${unit} is in failed state" + return 3 + fi + return 0 + fi + + echo "[$((waited + interval))/${timeout}] unit ${unit} not active yet. status:" + sudo systemctl status "$unit" --no-pager -n 3 || true + + sleep $interval + waited=$((waited + interval)) + done + + echo "Timed out waiting for ${unit} to become active (timeout=${timeout}s)." + echo "Last 200 journal lines for ${unit}:" + sudo journalctl -u "$unit" --no-pager -n 200 || true + return 1 + } + + wait_for_unit k3s 180 + + if command -v systemctl >/dev/null 2>&1; then + if sudo systemctl is-enabled --quiet k3s; then + echo "k3s unit is enabled" + else + echo "k3s unit is not enabled (continuing; service may still be running)" + fi + fi + + echo "Waiting for Kubernetes node to be Ready..." + max_attempts=24 + attempt=1 + while [ $attempt -le $max_attempts ]; do + if kubectl get nodes --no-headers 2>/dev/null | awk '{print $2}' | grep -q '^Ready$'; then + echo "Kubernetes node is Ready" + break + fi + echo "Attempt $attempt/$max_attempts: node not Ready yet; sleeping 10s" + sleep 10 + attempt=$((attempt + 1)) + done + + if [ $attempt -gt $max_attempts ]; then + echo "Nodes did not become Ready in expected time. Dumping logs for debugging:" + kubectl get pods -A || true + sudo journalctl -u k3s --no-pager -n 200 || true + kubectl describe nodes || true + exit 1 + fi + + kubectl cluster-info || true + kubectl get nodes -o wide || true + + - name: Configure k3s registry mirrors + run: | + sudo mkdir -p /etc/rancher/k3s + sudo tee /etc/rancher/k3s/registries.yaml <<'EOF' + mirrors: + "docker.io": + endpoint: + - "https://mirror.gcr.io" + - "https://harbor.ddlns.net/v2/dolibarr/" + "registry-1.docker.io": + endpoint: + - "https://mirror.gcr.io" + - "https://harbor.ddlns.net/v2/dolibarr/" + EOF + sudo systemctl restart k3s + kubectl wait --for=condition=Ready nodes --all --timeout=120s + + - name: Install Flux in Kubernetes + run: flux install --timeout=5m + + - name: Verify Flux installation + run: | + kubectl -n flux-system wait --for=condition=ready pod --all --timeout=5m + kubectl -n flux-system get deploy + kubectl get crds | grep -E "(fluxcd|toolkit)" || true + kubectl get crd helmreleases.helm.toolkit.fluxcd.io || true + + - name: Setup GitRepository source + run: | + CURRENT_BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" + echo "Using branch: $CURRENT_BRANCH" + + flux create source git flux-system \ + --url=${{ github.event.repository.html_url }} \ + --branch="${CURRENT_BRANCH}" \ + --interval=5m \ + --username=${GITHUB_ACTOR} \ + --password=${{ secrets.GITHUB_TOKEN }} \ + --timeout=10m + + kubectl wait --for=condition=ready gitrepository/flux-system -n flux-system --timeout=10m + + - name: Deploy charts (HelmRepository sources) + run: | + # oc-ampere has 21 HelmRepository/OCIRepository sources under + # charts/ (only flux-operator and teleport-charts are actually + # consumed by a non-excluded app today) — this is the first e2e + # run anywhere in this repo to reconcile all of them at once. + # If this step times out, that volume — not a workflow bug — is + # the first thing to check. + flux create kustomization charts \ + --source=flux-system \ + --path=./clusters/oc-ampere/charts \ + --interval=5m \ + --prune=true \ + --timeout=10m \ + --wait + + kubectl -n flux-system wait kustomization/charts --for=condition=ready --timeout=10m + + - name: Deploy generic top-level cluster resources + run: | + set -euo pipefail + + # clusters/oc-ampere/kustomization.yaml's flat resources: list is + # the real production entry point. Rather than hand-enumerating + # each top-level resource here, parse it and apply everything not + # already handled by a dedicated step above. Same mechanism as + # validate-kubenuc.yml/validate-k8s-vms.yml — see those for the + # gap this closes. + # + # flux-instance.yaml MUST stay in SKIP_LIST here, unlike + # kubenuc/k8s-vms-daniele where it's skipped for the same generic + # reason but with lower stakes: oc-ampere's real production + # flux-operator (deployed for real below, via the per-app loop, + # since it's not in EXCLUDED_APPS) has nothing to reconcile if no + # FluxInstance CR exists — this job bootstraps Flux itself via + # plain `flux install`. If flux-instance.yaml were ever applied + # for real here, the freshly-installed Flux Operator would take + # over reconciling its own FluxInstance CR and fight the `flux + # install`-installed components mid-run. + # + # cluster-vars.yaml also stays in SKIP_LIST: nothing non-excluded + # in this workflow currently depends on it (see clusters/CLAUDE.md + # and project_k3s_rabbit_oc_ampere memory); re-add it here only + # once a non-excluded app gains a real + # dependsOn/postBuild.substituteFrom on cluster-vars. + CLUSTER=oc-ampere + KUST_FILE="clusters/${CLUSTER}/kustomization.yaml" + SKIP_LIST="apps.yaml charts.yaml cluster-vars.yaml flux-instance.yaml" + + RAW_LINES=$(sed -n '/^resources:/,/^[^ -]/p' "$KUST_FILE" | tr -d '\r' | grep -E '^[[:space:]]*-[[:space:]]' || true) + RAW_COUNT=$(printf '%s\n' "$RAW_LINES" | grep -c . || true) + + if [ "$RAW_COUNT" -eq 0 ]; then + echo "ERROR: found zero 'resources:' entries in $KUST_FILE — the resources: block format may have changed; update this parser, don't silently skip it" + exit 1 + fi + + RESOURCES=() + while IFS= read -r LINE; do + [ -z "$LINE" ] && continue + if [[ "$LINE" =~ ^[[:space:]]*-[[:space:]]+([A-Za-z0-9._-]+\.ya?ml)[[:space:]]*$ ]]; then + RESOURCES+=("${BASH_REMATCH[1]}") + else + echo "ERROR: could not parse a 'resources:' entry in $KUST_FILE: '$LINE'" + echo "This step fails closed on any format it doesn't recognize — update the parser, don't ignore it." + exit 1 + fi + done <<< "$RAW_LINES" + + if [ "${#RESOURCES[@]}" -ne "$RAW_COUNT" ]; then + echo "ERROR: parsed ${#RESOURCES[@]} resource entries but found $RAW_COUNT raw lines under resources: in $KUST_FILE" + exit 1 + fi + + for RESOURCE in "${RESOURCES[@]}"; do + SKIP=false + for S in $SKIP_LIST; do + [ "$RESOURCE" = "$S" ] && SKIP=true + done + if [ "$SKIP" = true ]; then + echo "Skipping $RESOURCE (handled by a dedicated step, or — for flux-instance.yaml/cluster-vars.yaml — not applied by this workflow, see comment above)" + continue + fi + + RESOURCE_PATH="clusters/${CLUSTER}/${RESOURCE}" + if [ ! -f "$RESOURCE_PATH" ]; then + echo "ERROR: resources entry '$RESOURCE' in $KUST_FILE does not resolve to exactly one existing file at $RESOURCE_PATH" + exit 1 + fi + + echo "=== Applying top-level resource: $RESOURCE ===" + OBJ=$(kubectl apply -f "$RESOURCE_PATH" -o name) + mapfile -t OBJS <<< "$OBJ" + for O in "${OBJS[@]}"; do + [ -z "$O" ] && continue + echo "Waiting for $O to be ready" + kubectl -n flux-system wait "$O" --for=condition=ready --timeout=5m + done + done + + # oc-ampere has 4 real apps: flux-operator and ngx-webhook deploy for + # real below (2/4); system-upgrade-controller and teleport-agent are + # excluded for the same reasons as k3s-rabbit's. ngx-webhook is the + # app the new tier-4 resolver (below) exists to cover — it has no + # deploy.yaml of its own, only a bare manifests/deploy.yml entry in + # the root apps/kustomization.yaml. + # + # The conditions that actually gate this job's exit code: Mechanism + # 1's per-object `kubectl wait` above, `charts`' `--wait` above, and + # the blanket `kubectl -n flux-system wait kustomization --all` at the + # end of this step. The "Check for failed reconciliations" step + # further down does NOT gate the job — its jq pipelines all end in + # `|| true`. + - name: Deploy infra baseline and changed apps + run: | + set -euo pipefail + echo "Apps to deploy: $DEPLOY_APPS" + + CLUSTER=oc-ampere + + # EXCLUDED_APPS: apps that would otherwise resolve via the -test + # mirror or the real clusters/oc-ampere/apps/$APP fallback below, + # but reconciling them for real in this ephemeral e2e cluster has + # a live external side-effect. Checked FIRST, before every + # resolution tier, so an excluded app can never slip through. + # Every entry MUST have a real, verified reason. See + # clusters/CLAUDE.md and the project_k3s_rabbit_oc_ampere memory. + declare -A EXCLUDED_APPS=( + [system-upgrade-controller]="real upgrade.cattle.io Plan CRDs with cordon: true against the real update.k3s.io channel — distinct from the safe top-level system-upgrade-controller.yaml Kustomization already applied generically earlier in this workflow; do not remove this exclusion just because that top-level one is safe" + [teleport-agent]="would register kubeClusterName: oc-ampere against live production Teleport infra — same risk class as k3s-rabbit's teleport-agent, regardless of its join-token secrets Kustomization currently being commented out in the file" + ) + + for APP in $DEPLOY_APPS; do + if [[ -v EXCLUDED_APPS[$APP] ]]; then + echo "Skipping $APP — excluded: ${EXCLUDED_APPS[$APP]}" + continue + fi + + # Tier 2: -test mirror directory. Structurally kept for + # consistency with kubenuc's/k8s-vms-daniele's e2e workflows, + # but this is dead code on oc-ampere today — no + # oc-ampere-test cluster exists. Documented here so a future + # mirror addition doesn't require re-deriving the resolution + # order from scratch. + TEST_APP_PATH="clusters/${CLUSTER}-test/apps/$APP" + if [ -d "$TEST_APP_PATH" ]; then + echo "=== Creating kustomization for: $APP (${CLUSTER}-test mirror) ===" + flux create kustomization "app-${APP}" \ + --source=flux-system \ + --path="./$TEST_APP_PATH" \ + --interval=5m \ + --prune=true \ + --wait --timeout=10m + continue + fi + + # Tier 3: real per-app deploy.y*ml. No dependsOn beyond + # charts/cluster-vars (already Ready by this point) exists + # among today's real-path fallback apps on this cluster — a + # future app that dependsOn something else must be checked by + # hand before relying on this fallback for it. + REAL_DEPLOY_PATH="" + for CANDIDATE in "clusters/${CLUSTER}/apps/$APP/deploy.yaml" "clusters/${CLUSTER}/apps/$APP/deploy.yml"; do + if [ -f "$CANDIDATE" ]; then + REAL_DEPLOY_PATH="$CANDIDATE" + break + fi + done + + if [ -n "$REAL_DEPLOY_PATH" ]; then + echo "=== Applying $APP's real production Kustomization: $REAL_DEPLOY_PATH ===" + kubectl apply -f "$REAL_DEPLOY_PATH" + continue + fi + + # Tier 4: raw-resource-in-root apps/kustomization.yaml + # fallback. Needed today for ngx-webhook, which has no + # deploy.yaml of its own — only a bare + # ngx-webhook/manifests/deploy.yml entry in the root + # apps/kustomization.yaml. Resolve to dirname() of the matched + # entry (its own immediate containing directory), not a + # first-path-segment-then-assume-parent heuristic — that + # looser heuristic only works today by accident of kustomize's + # recursive auto-discovery on this repo's current single-file + # layout, and dirname() stays correct regardless of future + # layout changes. Fail closed on more than one match, and on a + # bare-directory match (no per-app deploy.y*ml AND not a + # single resolvable file path) — this tier does not attempt to + # resolve an app with its own wrapper kustomization.yaml. + APPS_KUST_FILE="clusters/${CLUSTER}/apps/kustomization.yaml" + RAW_APP_LINES=$(sed -n '/^resources:/,/^[^ -]/p' "$APPS_KUST_FILE" | tr -d '\r' | grep -E '^[[:space:]]*-[[:space:]]' || true) + MATCHES=() + while IFS= read -r LINE; do + [ -z "$LINE" ] && continue + if [[ "$LINE" =~ ^[[:space:]]*-[[:space:]]+([A-Za-z0-9._/-]+)[[:space:]]*$ ]]; then + ENTRY="${BASH_REMATCH[1]}" + FIRST_SEGMENT="${ENTRY%%/*}" + if [ "$FIRST_SEGMENT" = "$APP" ]; then + MATCHES+=("$ENTRY") + fi + else + echo "ERROR: could not parse a 'resources:' entry in $APPS_KUST_FILE: '$LINE'" + exit 1 + fi + done <<< "$RAW_APP_LINES" + + if [ "${#MATCHES[@]}" -gt 1 ]; then + echo "ERROR: '$APP' matched more than one entry in $APPS_KUST_FILE: ${MATCHES[*]} — refusing to guess an apply order" + exit 1 + fi + + if [ "${#MATCHES[@]}" -eq 1 ]; then + ENTRY="${MATCHES[0]}" + if [[ "$ENTRY" != */* ]]; then + echo "ERROR: '$APP' resolved to a bare directory entry ('$ENTRY') in $APPS_KUST_FILE with no per-app deploy.y*ml. Tier 4 only handles raw file-path entries (e.g. ngx-webhook/manifests/deploy.yml) — a bare-directory app entry with its own wrapper kustomization.yaml needs a hand-reviewed resolution, not this generic fallback." + exit 1 + fi + RESOLVED_DIR="clusters/${CLUSTER}/apps/$(dirname "$ENTRY")" + echo "=== Creating tier-4 kustomization for: $APP (resolved via $APPS_KUST_FILE -> $RESOLVED_DIR) ===" + flux create kustomization "app-${APP}" \ + --source=flux-system \ + --path="./${RESOLVED_DIR}" \ + --interval=5m \ + --prune=true \ + --wait --timeout=10m + continue + fi + + echo "ERROR: '$APP' is in DEPLOY_APPS but resolves to none of: an EXCLUDED_APPS entry, a ${CLUSTER}-test mirror ($TEST_APP_PATH), a real clusters/${CLUSTER}/apps/$APP/deploy.y*ml, or an entry in $APPS_KUST_FILE" + echo "Fix this by adding a ${CLUSTER}-test mirror, adding a documented EXCLUDED_APPS entry above, or checking for a typo in the app name." + exit 1 + done + + echo "=== Waiting for kustomizations ===" + kubectl -n flux-system wait kustomization --all --for=condition=ready --timeout=30m + + echo "=== Waiting for HelmReleases ===" + kubectl get hr -A --no-headers 2>/dev/null | awk '{print $1, $2}' | while read ns name; do + echo "Waiting for HR: $name in $ns" + kubectl -n "$ns" wait hr "$name" --for=condition=ready --timeout=30m || true + done + + - name: Check resource deployment status + run: | + flux get kustomizations -A + flux get helmreleases -A + + - name: Check for failed reconciliations + run: | + kubectl get kustomizations -A -o json | jq -r '.items[] | select(.status.conditions[]? | select(.type=="Ready" and .status=="False")) | "\(.metadata.namespace)/\(.metadata.name)"' || true + kubectl get helmreleases -A -o json | jq -r '.items[] | select(.status.conditions[]? | select(.type=="Ready" and .status=="False")) | "\(.metadata.namespace)/\(.metadata.name)"' || true + + - name: Debug failure + if: failure() + run: | + flux get all -A + kubectl get pods -A | grep -v Running || true + kubectl get events -A --sort-by='.lastTimestamp' | tail -30 || true + + - name: Cleanup k3s + if: always() + run: | + /usr/local/bin/k3s-uninstall.sh || true + rm -rf ~/.kube/config || true + sudo rm -rf /etc/rancher/k3s || true + sudo rm -rf /var/lib/rancher/k3s || true + sudo rm -rf /run/k3s || true diff --git a/clusters/oc-ampere/apps/ngx-webhook/manifests/deploy.yml b/clusters/oc-ampere/apps/ngx-webhook/manifests/deploy.yml index 9f6fd9796..c2f19fd50 100644 --- a/clusters/oc-ampere/apps/ngx-webhook/manifests/deploy.yml +++ b/clusters/oc-ampere/apps/ngx-webhook/manifests/deploy.yml @@ -1,3 +1,8 @@ +# ngx-webhook has no per-app deploy.yaml — it's a bare +# manifests/deploy.yml entry in the root apps/kustomization.yaml, which is +# exactly why validate-oc-ampere.yml's e2e workflow needs a tier-4 +# resolver (dirname()-based) to reach it. This comment-only touch exists +# solely to exercise that resolver in the workflow's first real CI run. --- apiVersion: v1 kind: Namespace From 66d5dfd45014a24690b2545da8c75dcb97a441d1 Mon Sep 17 00:00:00 2001 From: Daniele De Lorenzi <2905124+dark-vex@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:51:33 +0200 Subject: [PATCH 2/3] fix(validate-oc-ampere): exercise flux-operator's real deploy in this PR's own CI run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An independent Codex review of this PR (before merge) found that DEPLOY_APPS is derived purely from `git diff` against main, and this PR's only touched app path was ngx-webhook/ — so flux-operator, despite not being in EXCLUDED_APPS and despite the workflow's own header comment claiming both apps "deploy for real," never actually entered the deploy loop on this PR's own CI run. The workflow's design was correct (a future PR touching flux-operator/ would exercise it), but this PR's verification run would have proven nothing about it. Add the same kind of comment-only touch already used for ngx-webhook, this time under apps/flux-operator/, so DEPLOY_APPS contains both apps and this PR's real CI run actually exercises flux-operator's real-path deploy — the first time anywhere in this repo. Also add a missing -f existence check to the tier-4 resolver (flagged in the same review): it previously only checked for a '/' in the matched apps/kustomization.yaml entry, not that the entry actually resolves to a real file, before computing dirname() — a future directory-style entry with a '/' in its name would have been misclassified as a file path rather than failing at the resolver boundary. Signed-off-by: Daniele De Lorenzi <2905124+dark-vex@users.noreply.github.com> --- .github/workflows/validate-oc-ampere.yml | 4 ++++ .../oc-ampere/apps/flux-operator/manifests/release.yml | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/validate-oc-ampere.yml b/.github/workflows/validate-oc-ampere.yml index 321331fde..f8b9e7e23 100644 --- a/.github/workflows/validate-oc-ampere.yml +++ b/.github/workflows/validate-oc-ampere.yml @@ -425,6 +425,10 @@ jobs: echo "ERROR: '$APP' resolved to a bare directory entry ('$ENTRY') in $APPS_KUST_FILE with no per-app deploy.y*ml. Tier 4 only handles raw file-path entries (e.g. ngx-webhook/manifests/deploy.yml) — a bare-directory app entry with its own wrapper kustomization.yaml needs a hand-reviewed resolution, not this generic fallback." exit 1 fi + if [ ! -f "clusters/${CLUSTER}/apps/${ENTRY}" ]; then + echo "ERROR: '$APP' resolved to '$ENTRY' in $APPS_KUST_FILE, but clusters/${CLUSTER}/apps/${ENTRY} is not a real file. Tier 4 only handles raw file-path entries — refusing to guess a directory from a non-existent path." + exit 1 + fi RESOLVED_DIR="clusters/${CLUSTER}/apps/$(dirname "$ENTRY")" echo "=== Creating tier-4 kustomization for: $APP (resolved via $APPS_KUST_FILE -> $RESOLVED_DIR) ===" flux create kustomization "app-${APP}" \ diff --git a/clusters/oc-ampere/apps/flux-operator/manifests/release.yml b/clusters/oc-ampere/apps/flux-operator/manifests/release.yml index 40a7d142b..04e7d2026 100644 --- a/clusters/oc-ampere/apps/flux-operator/manifests/release.yml +++ b/clusters/oc-ampere/apps/flux-operator/manifests/release.yml @@ -1,3 +1,11 @@ +# flux-operator is NOT in EXCLUDED_APPS in validate-oc-ampere.yml's e2e +# workflow — it deploys via the real deploy.y*ml fallback tier. This +# comment-only touch exists solely to put flux-operator into DEPLOY_APPS +# (derived purely from `git diff` against main) so this PR's own CI run +# actually exercises that real deploy for the first time in this repo, +# alongside ngx-webhook's tier-4 resolver. Without this touch, DEPLOY_APPS +# would only contain ngx-webhook and this PR's run would prove nothing +# about flux-operator's real-path deployment. --- apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease From a817d22bee05a391a2ace26a34c09503c1327298 Mon Sep 17 00:00:00 2001 From: Daniele De Lorenzi <2905124+dark-vex@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:14:11 +0200 Subject: [PATCH 3/3] fix(validate-oc-ampere): don't fail the job on zero HelmReleases Preventive fix for the same bug that failed the companion validate-k3s-rabbit.yml workflow's real CI run: "Check resource deployment status" calls `flux get helmreleases -A` with no `|| true`, and flux's CLI exits non-zero when it finds zero matching objects. This PR's own run passed because flux-operator's real HelmRelease happens to exist, but any future oc-ampere PR touching only excluded apps (system-upgrade-controller, teleport-agent) would deploy zero HelmReleases and hit this every time. Add `|| true` to both `flux get` calls, consistent with the "Check for failed reconciliations" step just below it. Signed-off-by: Daniele De Lorenzi <2905124+dark-vex@users.noreply.github.com> --- .github/workflows/validate-oc-ampere.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-oc-ampere.yml b/.github/workflows/validate-oc-ampere.yml index f8b9e7e23..abcb09a98 100644 --- a/.github/workflows/validate-oc-ampere.yml +++ b/.github/workflows/validate-oc-ampere.yml @@ -454,10 +454,17 @@ jobs: kubectl -n "$ns" wait hr "$name" --for=condition=ready --timeout=30m || true done + # Purely informational — `flux get` exits non-zero when it finds zero + # matching objects. This run happened to have a real HelmRelease + # (flux-operator's), but a future oc-ampere PR touching only excluded + # apps (system-upgrade-controller, teleport-agent) would deploy zero + # HelmReleases and fail this step every time without `|| true` — + # confirmed live on the companion k3s-rabbit workflow, which hits + # this every run by design (3/3 real apps excluded there). - name: Check resource deployment status run: | - flux get kustomizations -A - flux get helmreleases -A + flux get kustomizations -A || true + flux get helmreleases -A || true - name: Check for failed reconciliations run: |