fix(ci): decline unreadable CA pems; stop the probe test masking crashes

Two review findings, and the first exposed a real defect in the probes.

The test discarded stderr and forced success, so a probe that CRASHED was
indistinguishable from one that deliberately disabled the cache, and every
disabled-case assertion would pass on a broken script. It now captures the exit
status separately and fails on any non-zero exit, printing the probe's stderr.

That immediately surfaced a bug: the probes guarded the TLS path with
-z BAZEL_REMOTE_CA_PEM, which only catches an unset variable. A variable set to
a path that does not exist fell through to the  below and failed the job
instead of cleanly declining the cache. The guard is now -r, covering unset and
unreadable alike, and the message reports the offending path.

Added the two missing cases: TLS with a readable CA pem (the success path that
emits --tls_certificate, previously uncovered, asserted to produce both grpcs://
and the certificate flag so a silent downgrade to plaintext cannot pass), and
TLS with an unreadable CA path. Six probes, six cases.

Every assertion is backed by a negative control, each verified to fail when the
defect is reintroduced: dead default host restored, CA guard reverted to -z,
TLS downgraded to plaintext, and a probe that exits non-zero.

Co-authored-by: Balaji Ganesan <bganesan@nvidia.com>

Author: balajinvda

src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe

@@ -24,16 +24,18 @@ else
   host="${NVCF_BAZEL_REMOTE_HOST}"
   port="${NVCF_BAZEL_REMOTE_PORT:-443}"
   tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
-  if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
+  if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
     # TLS requires a CA pem. grpcurl can probe with `-insecure`, but
     # Bazel's own gRPC client cannot: with `grpcs://` and no
     # `--tls_certificate` it tries the system trust store, which does
     # not contain the cache's self-signed cert, and fails mid-build
     # with "General OpenSslEngine problem". Skip the cache cleanly
     # instead of greenlighting an unusable URL. Hit on
     # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
-    # variable) to enable.
-    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
+    # variable) to enable. The test is -r, not -z: a variable set to a
+    # path that does not exist would otherwise reach the `cp` below and
+    # fail the job instead of cleanly declining the cache.
+    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
   elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then
     # Env var is set but the file is missing or unreadable. Treat it
     # the same as unset and degrade to local-only rather than failing

src/compute-plane-services/nvca/scripts/.bazel-remote-probe

@@ -22,16 +22,18 @@ else
   host="${NVCF_BAZEL_REMOTE_HOST}"
   port="${NVCF_BAZEL_REMOTE_PORT:-443}"
   tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
-  if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
+  if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
     # TLS requires a CA pem. grpcurl can probe with `-insecure`, but
     # Bazel's own gRPC client cannot: with `grpcs://` and no
     # `--tls_certificate` it tries the system trust store, which does
     # not contain the cache's self-signed cert, and fails mid-build
     # with "General OpenSslEngine problem". Skip the cache cleanly
     # instead of greenlighting an unusable URL. Hit on
     # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
-    # variable) to enable.
-    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
+    # variable) to enable. The test is -r, not -z: a variable set to a
+    # path that does not exist would otherwise reach the `cp` below and
+    # fail the job instead of cleanly declining the cache.
+    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
   else
     probe_args=(-d '{"instance_name":""}')
     cache_url="grpc://${host}:${port}"

src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe

@@ -24,16 +24,18 @@ else
   host="${NVCF_BAZEL_REMOTE_HOST}"
   port="${NVCF_BAZEL_REMOTE_PORT:-443}"
   tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
-  if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
+  if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
     # TLS requires a CA pem. grpcurl can probe with `-insecure`, but
     # Bazel's own gRPC client cannot: with `grpcs://` and no
     # `--tls_certificate` it tries the system trust store, which does
     # not contain the cache's self-signed cert, and fails mid-build
     # with "General OpenSslEngine problem". Skip the cache cleanly
     # instead of greenlighting an unusable URL. Hit on
     # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
-    # variable) to enable.
-    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
+    # variable) to enable. The test is -r, not -z: a variable set to a
+    # path that does not exist would otherwise reach the `cp` below and
+    # fail the job instead of cleanly declining the cache.
+    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
   elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then
     # Env var is set but the file is missing or unreadable. Treat it
     # the same as unset and degrade to local-only rather than failing

src/control-plane-services/helm-reval/scripts/.bazel-remote-probe

@@ -22,16 +22,18 @@ else
   host="${NVCF_BAZEL_REMOTE_HOST}"
   port="${NVCF_BAZEL_REMOTE_PORT:-443}"
   tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
-  if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
+  if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
     # TLS requires a CA pem. grpcurl can probe with `-insecure`, but
     # Bazel's own gRPC client cannot: with `grpcs://` and no
     # `--tls_certificate` it tries the system trust store, which does
     # not contain the cache's self-signed cert, and fails mid-build
     # with "General OpenSslEngine problem". Skip the cache cleanly
     # instead of greenlighting an unusable URL. Hit on
     # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
-    # variable) to enable.
-    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
+    # variable) to enable. The test is -r, not -z: a variable set to a
+    # path that does not exist would otherwise reach the `cp` below and
+    # fail the job instead of cleanly declining the cache.
+    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
   else
     probe_args=(-d '{"instance_name":""}')
     cache_url="grpc://${host}:${port}"

src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe

@@ -24,16 +24,18 @@ else
   host="${NVCF_BAZEL_REMOTE_HOST}"
   port="${NVCF_BAZEL_REMOTE_PORT:-443}"
   tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
-  if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
+  if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
     # TLS requires a CA pem. grpcurl can probe with `-insecure`, but
     # Bazel's own gRPC client cannot: with `grpcs://` and no
     # `--tls_certificate` it tries the system trust store, which does
     # not contain the cache's self-signed cert, and fails mid-build
     # with "General OpenSslEngine problem". Skip the cache cleanly
     # instead of greenlighting an unusable URL. Hit on
     # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
-    # variable) to enable.
-    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
+    # variable) to enable. The test is -r, not -z: a variable set to a
+    # path that does not exist would otherwise reach the `cp` below and
+    # fail the job instead of cleanly declining the cache.
+    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
   elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then
     # Env var is set but the file is missing or unreadable. Treat it
     # the same as unset and degrade to local-only rather than failing

src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe

@@ -24,16 +24,18 @@ else
   host="${NVCF_BAZEL_REMOTE_HOST}"
   port="${NVCF_BAZEL_REMOTE_PORT:-443}"
   tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
-  if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
+  if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
     # TLS requires a CA pem. grpcurl can probe with `-insecure`, but
     # Bazel's own gRPC client cannot: with `grpcs://` and no
     # `--tls_certificate` it tries the system trust store, which does
     # not contain the cache's self-signed cert, and fails mid-build
     # with "General OpenSslEngine problem". Skip the cache cleanly
     # instead of greenlighting an unusable URL. Hit on
     # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
-    # variable) to enable.
-    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
+    # variable) to enable. The test is -r, not -z: a variable set to a
+    # path that does not exist would otherwise reach the `cp` below and
+    # fail the job instead of cleanly declining the cache.
+    echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
   elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then
     # Env var is set but the file is missing or unreadable. Treat it
     # the same as unset and degrade to local-only rather than failing

tools/ci/test-bazel-remote-probe

@@ -23,21 +23,56 @@ echo '{"cacheCapabilities":{"supportedCompressors":["IDENTITY","ZSTD"]}}'
 STUB
 chmod +x "${stub_dir}/grpcurl"
 
+# A readable (contents irrelevant) CA pem for the TLS success path.
+ca_pem="${stub_dir}/ca.pem"
+printf -- '-----BEGIN CERTIFICATE-----\nstub\n-----END CERTIFICATE-----\n' > "${ca_pem}"
+
 fail=0
 
 check() { # name, expectation (enabled|disabled), probe, env assignments...
   local name="$1" want="$2" probe="$3"; shift 3
-  local out
+  local out status svc
+  svc="$(basename "$(dirname "$(dirname "${probe}")")")"
+
+  # Capture stdout and the exit status separately. Discarding stderr and
+  # forcing success would let a probe that CRASHES look identical to one that
+  # deliberately disables the cache, so every disabled-case assertion would
+  # pass on a broken script.
+  set +e
   out="$(
     env -i PATH="${stub_dir}:/usr/bin:/bin" HOME="${stub_dir}" \
         CI_PROJECT_DIR="${stub_dir}" "$@" \
-        bash -c 'BAZEL_REMOTE_FLAGS=""; . "$0"; printf "%s" "${BAZEL_REMOTE_FLAGS}"' "${probe}" 2>/dev/null
-  )" || true
+        bash -c 'set -e; BAZEL_REMOTE_FLAGS=""; . "$0"; printf "%s" "${BAZEL_REMOTE_FLAGS}"' \
+        "${probe}" 2>"${stub_dir}/stderr.txt"
+  )"
+  status=$?
+  set -e
+
+  if [[ "${status}" -ne 0 ]]; then
+    printf '  FAIL %-26s %-11s probe exited %s (expected a clean %s decision)\n' \
+      "${svc}" "${name}" "${status}" "${want}" >&2
+    sed 's/^/         /' "${stub_dir}/stderr.txt" >&2 || true
+    fail=1
+    return
+  fi
+
   local got="disabled"
   [[ "${out}" == *--remote_cache=* ]] && got="enabled"
   if [[ "${got}" != "${want}" ]]; then
-    printf '  FAIL %-34s %-9s want=%s got=%s\n' "$(basename "$(dirname "$(dirname "${probe}")")")" "${name}" "${want}" "${got}" >&2
+    printf '  FAIL %-26s %-11s want=%s got=%s\n' "${svc}" "${name}" "${want}" "${got}" >&2
     fail=1
+    return
+  fi
+
+  # For the TLS-enabled case, the cache URL must be grpcs:// and the probe must
+  # emit --tls_certificate. Without this the case would pass on a probe that
+  # silently downgraded to plaintext.
+  if [[ "${name}" == "tls-with-ca" ]]; then
+    if [[ "${out}" != *--tls_certificate=* || "${out}" != *grpcs://* ]]; then
+      printf '  FAIL %-26s %-11s expected grpcs:// and --tls_certificate, got: %s\n' \
+        "${svc}" "${name}" "${out}" >&2
+      fail=1
+    fi
   fi
 }
 
@@ -72,13 +107,29 @@ for probe in "${probes[@]}"; do
     NVCF_BAZEL_REMOTE_PORT=8980 NVCF_BAZEL_REMOTE_TLS=0
 
   # TLS requested but no CA pem: must decline rather than emit an unusable URL.
+  # Bazel's gRPC client cannot verify a private cache cert from the system
+  # trust store, so a grpcs:// URL without --tls_certificate fails mid-build.
   check "tls-no-ca" disabled "${probe}" \
     NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \
     NVCF_BAZEL_REMOTE_PORT=443 NVCF_BAZEL_REMOTE_TLS=1
+
+  # TLS requested WITH a readable CA pem: the success path that actually emits
+  # the certificate flag. Without this case nothing covers the grpcs:// branch.
+  check "tls-with-ca" enabled "${probe}" \
+    NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \
+    NVCF_BAZEL_REMOTE_PORT=443 NVCF_BAZEL_REMOTE_TLS=1 \
+    BAZEL_REMOTE_CA_PEM="${ca_pem}"
+
+  # TLS requested with an UNREADABLE CA path: must decline, not emit a flag
+  # pointing at a file that does not exist.
+  check "tls-bad-ca" disabled "${probe}" \
+    NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \
+    NVCF_BAZEL_REMOTE_PORT=443 NVCF_BAZEL_REMOTE_TLS=1 \
+    BAZEL_REMOTE_CA_PEM="${stub_dir}/definitely-missing.pem"
 done
 
 if [[ "${fail}" -ne 0 ]]; then
   echo "bazel-remote-probe behavioral tests failed" >&2
   exit 1
 fi
-echo "bazel-remote-probe: ${#probes[@]} probes x 4 cases OK"
+echo "bazel-remote-probe: ${#probes[@]} probes x 6 cases OK"