Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 21 additions & 5 deletions .github/workflows/bazel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -430,14 +430,27 @@ jobs:
if [ -n "$CACHE_TOKEN" ] && [ -n "$CACHE_ENDPOINT" ]; then
printf '%s\n' "${{ vars.BAZEL_REMOTE_CACHE_CA }}" > "$RUNNER_TEMP/cache-ca.pem"
echo "CACHE_READY=1" >> "$GITHUB_ENV"
# Upload results only from main pushes; every other trigger
# (PRs) is read-only so it can never poison the cache.
if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then
# Upload from merge-queue runs and main pushes; PRs stay
# read-only so they can never poison the cache.
#
# merge_group is the primary writer. Its tree is exactly what is
# about to become main, and each queue entry gets its own ref, so
# these runs are never cancelled by the next merge. Main-push runs
# all share the concurrency group bazel-refs/heads/main with
# cancel-in-progress, so a second merge landing minutes later kills
# the first one's warm-write: that is how the byoo collector, whose
# own PR existed to make it cacheable, ended up never warming the
# cache. Main pushes still upload, as a backstop for anything that
# reaches main without going through the queue.
if [ "$GITHUB_EVENT_NAME" = "merge_group" ]; then
upload=true
elif [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then
upload=true
fi
echo "CACHE_UPLOAD=$upload" >> "$GITHUB_ENV"
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
# Make the mode visible in the log so a missing warm-write is never
# a silent guess: read-only PRs show upload=false, main pushes true.
# a silent guess: read-only PRs show upload=false, merge-queue runs
# and main pushes true.
if [ "$upload" = "true" ]; then
echo "remote cache ready: read-write (warming; upload=true)"
else
Expand Down Expand Up @@ -808,7 +821,10 @@ jobs:
printf '%s\n' "$CACHE_CA" > "$RUNNER_TEMP/cache-ca.pem"
echo "CACHE_READY=1" >> "$GITHUB_ENV"
upload=false
if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then upload=true; fi
# See the bazel job's gate above for why merge_group is the
# primary writer rather than main pushes.
if [ "$GITHUB_EVENT_NAME" = "merge_group" ]; then upload=true
elif [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then upload=true; fi
echo "CACHE_UPLOAD=$upload" >> "$GITHUB_ENV"
echo "remote cache ready (upload=$upload)"
else
Expand Down
2 changes: 1 addition & 1 deletion src/compute-plane-services/byoo-otel-collector/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ host `go` that Bazel does not track, CI binds the toolchain into the
action key with `--action_env=BYOO_GO_TOOLCHAIN`, so a Go bump in the CI
image cannot serve binaries built by the previous compiler. The wrapper
binary, in contrast, is a regular `go_binary` and benefits from full
Bazel hermeticity + remote-cache reuse.
Bazel hermeticity + nvcfbarn remote-cache reuse.

A containerized Go application that provides a complete observability solution by orchestrating three functional components: it generates OpenTelemetry Collector configurations, extracts and manages secrets from ESS (Encrypted Secret Store), and runs a custom-built OpenTelemetry Collector binary.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,37 +2,40 @@
# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the
# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the
# right --remote_cache + --tls_certificate flags, or empty if the cache is
# unreachable / disabled. Source this file (do NOT execute it) so the
# exported var is visible in the parent shell.
#
# Driven by four CI/CD variables; all default to safe values that fall back
# to local-only cleanly:
# NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1).
# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com).
# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default:
# the previous default host was decommissioned).
# NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar).
# NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1).
# BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert.
# Required when TLS=1.

BAZEL_REMOTE_FLAGS=""
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then
echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled"
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then
echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only"
else
host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}"
host="${NVCF_BAZEL_REMOTE_HOST}"
port="${NVCF_BAZEL_REMOTE_PORT:-443}"
tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
# TLS requires a CA pem. grpcurl can probe with `-insecure`, but
# Bazel's own gRPC client cannot: with `grpcs://` and no
# `--tls_certificate` it tries the system trust store, which does
# not contain the nvcfbarn self-signed cert, and fails mid-build
# not contain the cache's self-signed cert, and fails mid-build
Comment thread
balajinvda marked this conversation as resolved.
# with "General OpenSslEngine problem". Skip the cache cleanly
# instead of greenlighting an unusable URL. Hit on
# dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
# variable) to enable.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
# variable) to enable. The test is -r, not -z: a variable set to a
# path that does not exist would otherwise reach the `cp` below and
# fail the job instead of cleanly declining the cache.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then
# Env var is set but the file is missing or unreadable. Treat it
# the same as unset and degrade to local-only rather than failing
Expand Down
21 changes: 12 additions & 9 deletions src/compute-plane-services/nvca/scripts/.bazel-remote-probe
Original file line number Diff line number Diff line change
@@ -1,36 +1,39 @@
#!/usr/bin/env bash
#
# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the
# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the
# right --remote_cache + --tls_certificate flags, or empty if the cache is
# unreachable / disabled. Source this file (do NOT execute it) so the
# exported var is visible in the parent shell.
#
# Driven by four CI/CD variables; all default to safe values that fall back
# to local-only cleanly:
# NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1).
# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com).
# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default:
# the previous default host was decommissioned).
# NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar).
# NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1).
# BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert.
# Required when TLS=1.

BAZEL_REMOTE_FLAGS=""
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then
echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled"
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then
echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only"
else
host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}"
host="${NVCF_BAZEL_REMOTE_HOST}"
port="${NVCF_BAZEL_REMOTE_PORT:-443}"
tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
# TLS requires a CA pem. grpcurl can probe with `-insecure`, but
# Bazel's own gRPC client cannot: with `grpcs://` and no
# `--tls_certificate` it tries the system trust store, which does
# not contain the nvcfbarn self-signed cert, and fails mid-build
# not contain the cache's self-signed cert, and fails mid-build
# with "General OpenSslEngine problem". Skip the cache cleanly
# instead of greenlighting an unusable URL. Hit on
# dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
# variable) to enable.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
# variable) to enable. The test is -r, not -z: a variable set to a
# path that does not exist would otherwise reach the `cp` below and
# fail the job instead of cleanly declining the cache.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
else
probe_args=(-d '{"instance_name":""}')
cache_url="grpc://${host}:${port}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,37 +2,40 @@
# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the
# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the
# right --remote_cache + --tls_certificate flags, or empty if the cache is
# unreachable / disabled. Source this file (do NOT execute it) so the
# exported var is visible in the parent shell.
#
# Driven by four CI/CD variables; all default to safe values that fall back
# to local-only cleanly:
# NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1).
# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com).
# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default:
# the previous default host was decommissioned).
# NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar).
# NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1).
# BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert.
# Required when TLS=1.

BAZEL_REMOTE_FLAGS=""
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then
echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled"
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then
echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only"
else
host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}"
host="${NVCF_BAZEL_REMOTE_HOST}"
port="${NVCF_BAZEL_REMOTE_PORT:-443}"
tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
# TLS requires a CA pem. grpcurl can probe with `-insecure`, but
# Bazel's own gRPC client cannot: with `grpcs://` and no
# `--tls_certificate` it tries the system trust store, which does
# not contain the nvcfbarn self-signed cert, and fails mid-build
# not contain the cache's self-signed cert, and fails mid-build
# with "General OpenSslEngine problem". Skip the cache cleanly
# instead of greenlighting an unusable URL. Hit on
# dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
# variable) to enable.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
# variable) to enable. The test is -r, not -z: a variable set to a
# path that does not exist would otherwise reach the `cp` below and
# fail the job instead of cleanly declining the cache.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then
# Env var is set but the file is missing or unreadable. Treat it
# the same as unset and degrade to local-only rather than failing
Expand Down
Original file line number Diff line number Diff line change
@@ -1,36 +1,39 @@
#!/usr/bin/env bash
#
# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the
# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the
# right --remote_cache + --tls_certificate flags, or empty if the cache is
# unreachable / disabled. Source this file (do NOT execute it) so the
# exported var is visible in the parent shell.
#
# Driven by four CI/CD variables; all default to safe values that fall back
# to local-only cleanly:
# NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1).
# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com).
# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default:
# the previous default host was decommissioned).
# NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar).
# NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1).
# BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert.
# Required when TLS=1.

BAZEL_REMOTE_FLAGS=""
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then
echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled"
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then
echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only"
else
host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}"
host="${NVCF_BAZEL_REMOTE_HOST}"
port="${NVCF_BAZEL_REMOTE_PORT:-443}"
tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
# TLS requires a CA pem. grpcurl can probe with `-insecure`, but
# Bazel's own gRPC client cannot: with `grpcs://` and no
# `--tls_certificate` it tries the system trust store, which does
# not contain the nvcfbarn self-signed cert, and fails mid-build
# not contain the cache's self-signed cert, and fails mid-build
# with "General OpenSslEngine problem". Skip the cache cleanly
# instead of greenlighting an unusable URL. Hit on
# dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
# variable) to enable.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
# variable) to enable. The test is -r, not -z: a variable set to a
# path that does not exist would otherwise reach the `cp` below and
# fail the job instead of cleanly declining the cache.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
else
probe_args=(-d '{"instance_name":""}')
cache_url="grpc://${host}:${port}"
Expand Down
10 changes: 6 additions & 4 deletions src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,12 @@ go_test(
local = True,
# external: testcontainers spins up the localstack Docker container,
# so the test's effective inputs (Docker daemon, localstack image
# tag) live outside Bazel's hermetic action hash. `local = True`
# forces local-only execution but does NOT bypass result caching;
# the `external` tag is what stops nvcfbarn from pinning a green
# forever. requires-docker is kept so callers can filter out
# tag) live outside Bazel's hermetic action hash. `local = True` forces
# local, unsandboxed execution and, measured on Bazel 9.1.1, also stops
# the result being reused from the disk/remote cache. It does not disable
# every cache: the `external` tag is what prevents Bazel reusing a
# previously cached test result. requires-docker is kept so callers
# can filter out
# docker-needing tests with --test_tag_filters=-requires-docker.
tags = [
"external",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,37 +2,40 @@
# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the
# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the
# right --remote_cache + --tls_certificate flags, or empty if the cache is
# unreachable / disabled. Source this file (do NOT execute it) so the
# exported var is visible in the parent shell.
#
# Driven by four CI/CD variables; all default to safe values that fall back
# to local-only cleanly:
# NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1).
# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com).
# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default:
# the previous default host was decommissioned).
# NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar).
# NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1).
# BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert.
# Required when TLS=1.

BAZEL_REMOTE_FLAGS=""
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then
echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled"
if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then
echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only"
else
host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}"
host="${NVCF_BAZEL_REMOTE_HOST}"
port="${NVCF_BAZEL_REMOTE_PORT:-443}"
tls="${NVCF_BAZEL_REMOTE_TLS:-1}"
if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then
if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then
# TLS requires a CA pem. grpcurl can probe with `-insecure`, but
# Bazel's own gRPC client cannot: with `grpcs://` and no
# `--tls_certificate` it tries the system trust store, which does
# not contain the nvcfbarn self-signed cert, and fails mid-build
# not contain the cache's self-signed cert, and fails mid-build
# with "General OpenSslEngine problem". Skip the cache cleanly
# instead of greenlighting an unusable URL. Hit on
# dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI
# variable) to enable.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable."
# variable) to enable. The test is -r, not -z: a variable set to a
# path that does not exist would otherwise reach the `cp` below and
# fail the job instead of cleanly declining the cache.
echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable."
elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then
# Env var is set but the file is missing or unreadable. Treat it
# the same as unset and degrade to local-only rather than failing
Expand Down
Loading
Loading