diff --git a/.github/workflows/release-tags.yml b/.github/workflows/release-tags.yml index b7e5b58492..fd7348bef5 100644 --- a/.github/workflows/release-tags.yml +++ b/.github/workflows/release-tags.yml @@ -17,7 +17,7 @@ on: required: false type: string inventory_tag: - description: Optional self-managed stack tag to render and validate without publishing. + description: Optional stack tag to render and validate without publishing. required: false type: string release_tag: @@ -117,7 +117,7 @@ jobs: ./tools/ci/github-release auto inventory-preflight: - name: self-managed inventory preflight + name: stack inventory preflight if: github.event_name == 'workflow_dispatch' && inputs.inventory_tag != '' runs-on: linux-amd64-cpu4 permissions: @@ -166,23 +166,45 @@ jobs: run: | set -euo pipefail case "${INVENTORY_TAG}" in - deploy/stacks/self-managed/v*) ;; + deploy/stacks/self-managed/v*|deploy/stacks/nvcf-compute-plane/v*|deploy/stacks/observability/v*) ;; *) - echo "ERROR: inventory_tag must be a deploy/stacks/self-managed/v* tag." >&2 + echo "ERROR: inventory_tag must identify a registered deploy/stacks/*/v* release." >&2 exit 1 ;; esac - version="${INVENTORY_TAG#deploy/stacks/self-managed/v}" + case "${INVENTORY_TAG}" in + deploy/stacks/self-managed/v*) inventory_config="deploy/stacks/self-managed/release-inventory.yaml" ;; + deploy/stacks/nvcf-compute-plane/v*) inventory_config="deploy/stacks/nvcf-compute-plane/release-inventory.yaml" ;; + deploy/stacks/observability/v*) inventory_config="deploy/stacks/observability/release-inventory.yaml" ;; + esac + bootstrap_config="${GITHUB_WORKSPACE}/${inventory_config}" + version="${INVENTORY_TAG##*/v}" commit="$(git rev-list -n 1 "${INVENTORY_TAG}")" test -n "${commit}" git worktree add --detach "${RUNNER_TEMP}/inventory-source" "${INVENTORY_TAG}" cd "${RUNNER_TEMP}/inventory-source" + inventory_args=() + if [ -f "${inventory_config}" ] && grep -q '^states:' "${inventory_config}"; then + inventory_config="${RUNNER_TEMP}/inventory-source/${inventory_config}" + else + echo "WARNING: ${INVENTORY_TAG} predates its per-stack inventory states; using ${bootstrap_config}." >&2 + inventory_config="${bootstrap_config}" + inventory_args+=(--allow-unavailable-source-charts) + fi "${RUNNER_TEMP}/docs-version-sync" \ - --generate-stack-inventory "${RUNNER_TEMP}/nvcf-self-managed-stack-inventory.json" \ - --inventory-config "${GITHUB_WORKSPACE}/deploy/stacks/self-managed/release-inventory.yaml" \ + --generate-stack-inventory "${RUNNER_TEMP}/stack-inventory.json" \ + --inventory-config "${inventory_config}" \ --stack-version "${version}" \ --stack-source-tag "${INVENTORY_TAG}" \ - --stack-source-commit "${commit}" + --stack-source-commit "${commit}" \ + "${inventory_args[@]}" + + - name: Upload rendered inventory + uses: actions/upload-artifact@v4 + with: + name: stack-inventory + path: ${{ runner.temp }}/stack-inventory.json + if-no-files-found: error tag-release-notes: name: tag release @@ -216,12 +238,12 @@ jobs: echo "NVCF_RELEASE_SOURCE_ROOT=${RUNNER_TEMP}/release-source" >> "${GITHUB_ENV}" - uses: actions/setup-go@v5 - if: startsWith(inputs.release_tag || github.ref_name, 'deploy/stacks/self-managed/v') + if: startsWith(inputs.release_tag || github.ref_name, 'deploy/stacks/') with: go-version-file: tools/go-toolchain/go.mod - name: Install Helm release tool - if: startsWith(inputs.release_tag || github.ref_name, 'deploy/stacks/self-managed/v') || startsWith(inputs.release_tag || github.ref_name, 'deploy/helm/') + if: startsWith(inputs.release_tag || github.ref_name, 'deploy/stacks/') || startsWith(inputs.release_tag || github.ref_name, 'deploy/helm/') run: | set -euo pipefail cd "${RUNNER_TEMP}" @@ -235,7 +257,7 @@ jobs: echo "${RUNNER_TEMP}/bin" >> "${GITHUB_PATH}" - name: Install Helmfile inventory tool - if: startsWith(inputs.release_tag || github.ref_name, 'deploy/stacks/self-managed/v') + if: startsWith(inputs.release_tag || github.ref_name, 'deploy/stacks/') run: | set -euo pipefail cd "${RUNNER_TEMP}" diff --git a/deploy/stacks/AGENTS.md b/deploy/stacks/AGENTS.md new file mode 100644 index 0000000000..e008165846 --- /dev/null +++ b/deploy/stacks/AGENTS.md @@ -0,0 +1,66 @@ +# AGENTS.md - Deployment Stacks + +## Purpose + +This subtree owns the Helmfile stacks and their release inventories. Read +[`INVENTORY.md`](INVENTORY.md) before adding, removing, or changing a chart, +container image, hook image, operator-created image, or downloadable stack +resource. + +## Stack Boundaries + +- `self-managed/` owns the NVCF control-plane stack. +- `nvcf-compute-plane/` owns the NVCF compute-plane stack. +- `observability/` owns shared observability infrastructure. +- Each stack owns its own `release-inventory.yaml` and release asset. +- Do not reference another stack's Helmfile state from an inventory config. +- Keep a dependency in the stack that installs or creates it. +- Follow the nearest nested `AGENTS.md` when it adds stack-specific guidance. + +## Dependency Changes + +For every dependency change: + +1. Update the owning Helmfile, chart values, or stack configuration. +2. Give an optional release an explicit condition and inventory render profile. +3. Update the owning stack's `release-inventory.yaml`. +4. Add registry and repository overrides for images that customers must mirror. +5. Record images that do not appear in rendered Kubernetes `image` fields. +6. Update the artifact classification in `docs/version-catalog/main.yaml`. +7. Run the stack tests and the inventory and documentation checks described in + [`INVENTORY.md`](INVENTORY.md). +8. After the stack release publishes its inventory asset, update the catalog + and generated manifest in a documentation sync change. + +A dependency is not fully distributed when the released inventory or generated +manifest omits one of its charts, images, or downloadable resources. + +## Required and Optional Artifacts + +- Mark an artifact `required` when the default stack installation needs it to + become ready or perform its baseline function. +- Mark an artifact `optional` when only an optional feature, provider, add-on, + or non-default mode needs it. +- An artifact used only by an optional release remains optional for the stack. +- List a customer-provided prerequisite as a prerequisite, not as a distributed + stack artifact. + +## Validation + +Run from the repository root unless a command says otherwise: + +```bash +make -C deploy/stacks/self-managed test +make -C deploy/stacks/nvcf-compute-plane test-local +make -C deploy/stacks/observability test +go test -C tools/docs-version-sync ./... +go run -C tools/docs-version-sync . --target main +./tools/ci/check-doc-version-sync +./tools/ci/check-docs +git diff --check +``` + +Do not hand-edit generated blocks in `docs/user/manifest.md`. The CI check +against the latest released inventory is warn-only for now. Generated-document +consistency remains blocking. Treat a release-drift warning as follow-up work +and keep the local checks clean for a dependency change. diff --git a/deploy/stacks/CLAUDE.md b/deploy/stacks/CLAUDE.md new file mode 100644 index 0000000000..43c994c2d3 --- /dev/null +++ b/deploy/stacks/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/deploy/stacks/INVENTORY.md b/deploy/stacks/INVENTORY.md new file mode 100644 index 0000000000..e3f8eb6e0c --- /dev/null +++ b/deploy/stacks/INVENTORY.md @@ -0,0 +1,265 @@ +# Stack Inventory Flow + +## Summary + +We package the self-managed control plane, compute plane, and observability +stack as three separate distributables. Each stack owns the inventory of +charts, images, and resources that its release can cause a customer environment +to pull. Release automation publishes those inventories as separate assets, +then the documentation sync combines them into one customer-facing manifest. + +This split keeps ownership close to the stack that installs the dependency. +Each stack is packaged, tagged, and inventoried independently. Customer support +and qualification apply to one three-stack release set after joint QA. + +## Architecture + +The flow has three layers: + +1. Each directory under `deploy/stacks/` owns its Helmfile states and + `release-inventory.yaml`. An inventory config must reference only states + under that stack directory. +1. The release workflow checks out an immutable stack tag, renders the profiles + named by that stack, and publishes one resolved JSON inventory with the + matching GitHub Release. +1. `tools/docs-version-sync` downloads all three released inventories. It + validates their plane boundaries, merges compatible artifacts, updates + `docs/version-catalog/main.yaml`, and generates the inventory blocks in + `docs/user/manifest.md`. +1. The catalog records the exact control-plane, compute-plane, and + observability versions. A development release set follows the newest stack + releases. A qualified release set uses three versions selected by QA. + +The version catalog is the handoff between release facts and public +documentation. Released inventories provide immutable versions and source +provenance. The catalog adds public distribution locations and the +human-authored descriptions and source links. Stack ownership and +required or optional status come from the released inventories. + +At the current phase, CI reports drift from the latest released inventories as +a warning. We still block on locally generated documentation being out of sync. +This gives the team visibility while the three release assets are being adopted +without making an unavailable or older release asset stop unrelated changes. + +## Release Sequence + +```mermaid +sequenceDiagram + actor Contributor + participant Stack as Owning stack + participant Release as Release workflow + participant Assets as GitHub Releases + participant Sync as docs-version-sync + participant QA + participant Catalog as Development catalog + participant Stable as Versioned docs + participant Manifest as Manifest page + + Contributor->>Stack: Update dependency and release-inventory.yaml + Contributor->>Stack: Run stack, inventory, and docs checks + + loop Self-managed, compute-plane, and observability releases + Release->>Stack: Check out immutable stack tag + Release->>Stack: Render configured profiles + Stack-->>Release: Return charts, images, and resources + Release->>Release: Validate ownership and profile coverage + Release->>Assets: Publish the stack inventory JSON + end + + Sync->>Assets: Download newest three inventory assets + Assets-->>Sync: Return released inventories + Sync->>Sync: Deduplicate artifacts and report conflicts + Sync->>Catalog: Open reviewable development docs update + Catalog->>Manifest: Generate ownership and optionality + + QA->>QA: Qualify one exact three-stack set + QA->>Sync: Approve exact versions and docs version + Sync->>Assets: Download the three selected inventories + Sync->>Catalog: Record qualified release_set + Catalog->>Stable: Snapshot docs and catalog + Stable->>Stable: Make qualified docs the default +``` + +## Ownership Model + +Keep each fact in one source: + +- The owning Helmfile and chart values define runtime dependencies and versions. +- Each stack's `release-inventory.yaml` supplies release-time inventory + configuration that cannot be derived from an ordinary render. +- The resolved JSON inventory records the artifacts found at an immutable stack + tag. Release automation publishes it with the stack release. +- The catalog `release_set` records the exact three stack releases represented + by the generated documentation. +- `docs/version-catalog/main.yaml` records public distribution locations and + human-authored descriptions and source links. +- Generated blocks in `docs/user/manifest.md` present the catalog to users. + +Do not maintain a second hand-written list of versions in the documentation. + +## What Belongs in an Inventory + +Include every artifact that the stack can cause a customer cluster to pull: + +- The stack bundle. +- Helm charts installed by the stack. +- Images in containers and init containers. +- Images used by Helm hooks. +- Images created later by an operator or controller. +- Images passed through command arguments or configuration instead of a + Kubernetes `image` field. +- Downloadable resources distributed as part of the stack. + +An ACME HTTP-01 solver image is one example of an indirect image. A controller +creates its short-lived workload only when an ACME challenge is requested, so +a normal default workload list might not contain the image. The inventory +renderer also scans resolved command arguments shaped like `--*-image=` +for this reason. If a new dependency uses another indirect form, extend the +renderer and cover that form with a test. + +## Required and Optional Rules + +Use the default supported installation as the boundary: + +- `required`: The default installation needs the artifact to become ready or + perform its baseline function. +- `optional`: The artifact is needed only when a feature, provider, add-on, or + non-default mode is enabled. + +An image required by an optional component is still optional at the stack +level. Record the condition or feature that activates it in the artifact +description. Customer-provided cluster prerequisites are not distributed stack +artifacts. Document them as prerequisites instead. + +## Contributor Flow + +When adding, removing, or changing a dependency: + +1. Change the owning Helmfile, values, or chart configuration. +2. Add an explicit condition for an optional release. +3. Update the state in the owning stack's `release-inventory.yaml`. + Add the optional setting to its `fullOverrides`, or add a state entry when + the dependency belongs to a new Helmfile state. +4. Add a `sourceCharts` entry when an independently released chart must be + rendered from an immutable source tag. Update the renderer when an indirect + artifact uses an input form it does not recognize. +5. Add or update tests that prove the resolved inventory contains the artifact + and its source release. Confirm the inventory-derived stack ownership and + required or optional status. +6. Update `manifest.entries` in `docs/version-catalog/main.yaml`. Set the plane, + kind, public description, and public source link. The released inventory + overrides ownership and required or optional status. +7. For a new third-party dependency, verify its license against + `.allowed-licenses.txt` and update `NOTICE` or subtree attribution files as + required. +8. Run: + + ```bash + make -C deploy/stacks/self-managed test + make -C deploy/stacks/nvcf-compute-plane test-local + make -C deploy/stacks/observability test + go test -C tools/docs-version-sync ./... + go vet -C tools/docs-version-sync ./... + go run -C tools/docs-version-sync . --target main + ./tools/ci/check-doc-version-sync + ./tools/ci/check-docs + ``` + +The renderer rejects unclassified artifacts. Inventory generation warns when a +release declared in a Helmfile state does not appear in its full profile. Do +not denylist an artifact merely because its public publication is pending. + +## Release Packaging + +The release workflow generates each inventory from its immutable stack tag. It +renders the configured profiles, collects charts and ordinary images, +discovers supported indirect image references, validates the result, and +attaches the matching inventory JSON to that stack's GitHub Release. + +The generation command has this shape: + +```bash +go run -C tools/docs-version-sync . \ + --generate-stack-inventory /tmp/stack-inventory.json \ + --inventory-config deploy/stacks//release-inventory.yaml \ + --stack-version X.Y.Z \ + --stack-source-tag deploy/stacks//vX.Y.Z \ + --stack-source-commit +``` + +Generation requires the tagged source, Helm and Helmfile, and release registry +credentials. It normally runs in `.github/workflows/release-tags.yml`. Do not +put credentials in repository files or command examples. + +The manual `inventory_tag` preflight uploads the rendered JSON as a workflow +artifact. New releases use the inventory config stored in the immutable tag. +For a release that lacks a config with per-stack states, the preflight uses the +config from the workflow ref as a bootstrap. The inventory still records and +renders the selected tag. It uses source charts available in the historical +commit and falls back to a published chart when its source path did not exist +yet. This fallback is only for validating historical releases. Tags created +after this workflow lands are self-contained. + +## Development Documentation Sync + +After stable releases and inventory assets exist for all three stacks, run: + +```bash +git fetch --tags origin +go run -C tools/docs-version-sync . --target main --update-catalog +go run -C tools/docs-version-sync . --target main +go test -C tools/docs-version-sync ./... +./tools/ci/check-doc-version-sync +./tools/ci/check-docs +``` + +By default the update selects the latest stable release of each stack and marks +the release set as `development`. The update is reviewable and does not imply +that QA qualified the selected combination. + +The catalog update retains an exact publication only when its artifact name, +type, and version still match. Leave an artifact in `publication_pending` until +its public location has been verified. Do not hand-edit generated manifest +blocks. + +The CI check against the latest released inventory remains warn-only for now. A +release-drift warning does not block a merge. Generated-document consistency +remains blocking, and local validation should still return success before a +dependency change is complete. + +## Qualified Documentation Promotion + +After QA approves one exact three-stack set and every customer artifact is +published, run: + +```bash +go run -C tools/docs-version-sync . \ + --target main \ + --update-catalog \ + --qualification-version X.Y.Z \ + --stack-version A.B.C \ + --compute-stack-version D.E.F \ + --observability-stack-version G.H.I +go run -C tools/docs-version-sync . --target main +./tools/scripts/cut-docs-version.sh vX.Y.Z +``` + +Qualification requires all three exact stack versions. The snapshot command +copies the generated docs and catalog. It updates the version dropdown with the +documentation version and all three stack versions. A later development sync +can move `docs/user/` forward without changing the versioned snapshot. + +## Compare Release Sets + +Place the previous inventory JSON files in one directory and the current files +in another. A directory can contain one historical combined inventory or three +separated inventories. Run: + +```bash +go run -C tools/docs-version-sync . \ + --compare-release-set-from /tmp/previous-inventories \ + --compare-release-set-to /tmp/current-inventories +``` + +The report lists added, removed, and changed artifacts. Changes include image +or chart references, digests, stack ownership, and required or optional status. diff --git a/deploy/stacks/nvcf-compute-plane/release-inventory.yaml b/deploy/stacks/nvcf-compute-plane/release-inventory.yaml new file mode 100644 index 0000000000..dde45d609c --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/release-inventory.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schemaVersion: 1 +states: + - plane: compute-plane + path: deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl + fullOverrides: + - addons.kaiScheduler.enabled=true + - addons.groveOperator.enabled=true + - addons.dynamoOperator.enabled=true + - addons.topologyAwareScheduling.enabled=true + - plane: compute-plane + path: deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl +sourceCharts: + helm-nvca-operator: + tagPrefix: deploy/helm/nvca-operator/v + path: deploy/helm/nvca-operator/nvca-operator +publishedChartRepository: https://helm.ngc.nvidia.com/nvidia/nvcf diff --git a/deploy/stacks/observability/release-inventory.yaml b/deploy/stacks/observability/release-inventory.yaml new file mode 100644 index 0000000000..e44dbea36b --- /dev/null +++ b/deploy/stacks/observability/release-inventory.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +schemaVersion: 1 +states: + - plane: observability + path: deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl + baseOverrides: + - observability.profile=all +publishedChartRepository: https://helm.ngc.nvidia.com/nvidia/nvcf diff --git a/deploy/stacks/self-managed/release-inventory.yaml b/deploy/stacks/self-managed/release-inventory.yaml index b690d611da..77e193926b 100644 --- a/deploy/stacks/self-managed/release-inventory.yaml +++ b/deploy/stacks/self-managed/release-inventory.yaml @@ -2,6 +2,23 @@ # SPDX-License-Identifier: Apache-2.0 schemaVersion: 1 +# Inventory states are owned by this distributable. Other stack trees publish +# their own release inventories and must not be referenced here. +states: + - plane: control-plane + path: deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl + fullOverrides: + - addons.llm.enabled=true + - plane: control-plane + path: deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl + fullOverrides: + - addons.llm.enabled=true + - addons.vanityGateway.enabled=true + - addons.nvcfUi.enabled=true + - plane: control-plane + path: deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl + fullOverrides: + - observability.profile=control # These charts are released independently and might not yet be mirrored into # the composite stack's release chart repository. Render each pinned chart from # its immutable public GitHub release tag, then report the public self-managed @@ -17,9 +34,6 @@ sourceCharts: # intended public repository under natsAuthCalloutService. renderValues: image.repository: nvcr.io/nvidia/nvcf/nvcf-nats-auth-callout-service - helm-nvca-operator: - tagPrefix: deploy/helm/nvca-operator/v - path: deploy/helm/nvca-operator/nvca-operator helm-nvcf-rate-limiter: tagPrefix: deploy/helm/ratelimiter/v path: deploy/helm/ratelimiter/nvcf-ratelimiter diff --git a/docs/AGENTS.md b/docs/AGENTS.md index fd19f68e58..9f871b3690 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -76,20 +76,21 @@ NVIDIA Cloud Functions glyphs inside green icon boxes must stay white in both mo Before finishing SVG changes, render light and dark previews for every changed SVG and compare them together for consistent background tone, panel contrast, connector contrast, text readability, and accent brightness. Use `--update-catalog` only when synchronizing artifact versions and registry -paths from the latest stable GitHub stack release. Presentation-only changes to +paths from the latest stable releases of all three stacks. Presentation-only changes to `manifest.entries` use the regeneration command above. -To synchronize the catalog and generated blocks from the latest stable GitHub -stack release: +To synchronize the development catalog and generated blocks from the latest +stable releases of all three stacks: ```bash go run -C tools/docs-version-sync . --target main --update-catalog ./tools/ci/check-doc-version-sync ``` -The first command reads the inventory attached to the latest stable GitHub -stack release and writes updates. The second command is an offline consistency -check. CI runs the offline check before this separate current-release check: +The first command reads the inventories attached to the latest stable GitHub +stack releases and writes a development release set. The second command is an +offline consistency check. CI runs the offline check before this separate +current-release check: ```bash ./tools/ci/check-doc-version-current-release @@ -101,6 +102,12 @@ exact public locations in `publications` and mark unavailable versions in `version` so charts, images, and resources with the same name remain distinct. Version overrides also require `name` and `type`. +After QA qualifies an exact three-stack combination, use +`--qualification-version` with all three explicit stack version flags. Generate +the docs, then run `tools/scripts/cut-docs-version.sh`. The versioned catalog +snapshot and Fern dropdown record the docs version and all three stack +versions. + Generated blocks are marked with comments such as: ```mdx diff --git a/docs/user/image-mirroring.md b/docs/user/image-mirroring.md index bb1619c0df..ee8fa873f3 100644 --- a/docs/user/image-mirroring.md +++ b/docs/user/image-mirroring.md @@ -196,17 +196,17 @@ First, ensure you have the [NGC CLI installed and configured](https://org.ngc.nv ```bash # Set stack versions export STACK_VERSION="0.16.1" -export COMPUTE_STACK_VERSION="0.16.1" -export OBSERVABILITY_STACK_VERSION="0.16.1" +export COMPUTE_STACK_VERSION="0.3.1" +export OBSERVABILITY_STACK_VERSION="0.1.1" # Download a specific control-plane stack version # Publication pending: nvcf-self-managed-stack 0.16.1 is not yet available for download. # Download a specific compute-plane stack version -# Publication pending: nvcf-compute-plane-stack 0.16.1 is not yet available for download. +# Publication pending: nvcf-compute-plane-stack 0.3.1 is not yet available for download. # Download a specific observability stack version -# Publication pending: nvcf-observability-stack 0.16.1 is not yet available for download. +# Publication pending: nvcf-observability-stack 0.1.1 is not yet available for download. ``` {/*docs-version-sync:END image-mirroring-resource-examples*/} @@ -251,7 +251,7 @@ Download and extract: {/*docs-version-sync:BEGIN image-mirroring-compute-stack-snippet*/} ```bash -# Publication pending: nvcf-compute-plane-stack 0.16.1 is not yet available for download. +# Publication pending: nvcf-compute-plane-stack 0.3.1 is not yet available for download. ``` {/*docs-version-sync:END image-mirroring-compute-stack-snippet*/} @@ -279,7 +279,7 @@ Download and extract: {/*docs-version-sync:BEGIN image-mirroring-observability-stack-snippet*/} ```bash -# Publication pending: nvcf-observability-stack 0.16.1 is not yet available for download. +# Publication pending: nvcf-observability-stack 0.1.1 is not yet available for download. ``` {/*docs-version-sync:END image-mirroring-observability-stack-snippet*/} diff --git a/docs/user/manifest.md b/docs/user/manifest.md index 636b50b22f..00477b0c36 100644 --- a/docs/user/manifest.md +++ b/docs/user/manifest.md @@ -141,135 +141,166 @@ The following tables list the complete artifact inventory. {/*docs-version-sync:BEGIN manifest-artifact-registry-paths*/} +### Stack release set + +Documentation: `dev` (development) + +| Stack | Version | Source tag | +| --- | --- | --- | +| Control plane | `0.16.1` | `deploy/stacks/self-managed/v0.16.1` | +| Compute plane | `0.3.1` | `deploy/stacks/nvcf-compute-plane/v0.3.1` | +| Observability | `0.1.1` | `deploy/stacks/observability/v0.1.1` | + ### Control plane Helm charts -| Artifact | Version | Required | Description | Distribution | Source code | -| --- | --- | --- | --- | --- | --- | -| `helm-admin-token-issuer-proxy` | `1.5.2` | Required | Deploys the admin token issuer proxy. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/admin-token-issuer-proxy) | -| `helm-nvcf-api` | `1.27.0` | Required | Deploys the NVCF API service. | `Publication pending` | | -| `helm-nvcf-api-keys` | `1.8.0` | Required | Deploys the API key management service. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/api-keys-colocated) | -| `helm-nvcf-cassandra` | `0.21.0` | Required | Deploys Cassandra and its initialization jobs. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/cassandra) / [Upstream](https://github.com/bitnami/charts/tree/main/bitnami/cassandra) | -| `helm-nvcf-cert-manager` | `0.1.0` | Required | Deploys the NVCF cert-manager configuration. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-cert-manager:0.1.0` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/cert-manager) / [Upstream](https://github.com/cert-manager/cert-manager) | -| `helm-nvcf-ess-api` | `1.8.2` | Required | Deploys the Encrypted Secrets Service API. | `Publication pending` | | -| `helm-nvcf-function-autoscaler` | `0.4.0` | Required | Deploys the function autoscaler for observability-driven scaling. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/function-autoscaler) | -| `helm-nvcf-grpc-proxy` | `1.7.3` | Required | Deploys the gRPC proxy service. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/grpc-proxy) | -| `helm-nvcf-invocation-service` | `1.6.1` | Required | Deploys the HTTP invocation service. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/http-invocation) | -| `helm-nvcf-llm-api-gateway` | `1.4.3` | Optional | Deploys the OpenAI-compatible LLM API gateway. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/llm-api-gateway) | -| `helm-nvcf-llm-request-router` | `1.13.3` | Optional | Deploys the LLM request router. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/llm-request-router) | -| `helm-nvcf-nats` | `0.8.1` | Required | Deploys NATS messaging for the control plane. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nats) / [Upstream](https://github.com/nats-io/k8s) | -| `helm-nvcf-nats-auth-callout-service` | `1.2.0` | Required | Deploys the NATS authorization callout service. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nats-auth-callout) | -| `helm-nvcf-notary-service` | `1.6.0` | Required | Deploys the notary service for signing and validation. | `Publication pending` | | -| `helm-nvcf-nvct-api` | `1.6.0` | Required | Deploys the NVCF tenant API service. | `Publication pending` | | -| `helm-nvcf-openbao-server` | `0.32.4` | Required | Deploys OpenBao secret management. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/openbao) / [Upstream](https://github.com/openbao/openbao-helm) | -| `helm-nvcf-pki` | `0.1.0` | Optional | Provisions the OpenBao-backed ClusterIssuer for NVCF service TLS. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-pki:0.1.0` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nvcf-pki) | -| `helm-nvcf-rate-limiter` | `1.2.1` | Required | Deploys request rate limiting for supported invocation paths. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/ratelimiter) | -| `helm-nvcf-sis` | `2.3.0` | Required | Deploys the Spot Instance Service. | `Publication pending` | | -| `helm-nvcf-state-metrics` | `1.0.2` | Required | Deploys NVCF state metrics for observability. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-state-metrics:1.0.2` | | -| `helm-nvcf-ui` | `1.1.2` | Optional | Deploys the optional NVCF UI admin panel. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-ui:1.1.2` | | -| `helm-nvcf-vanity-gateway` | `0.4.4` | Optional | Deploys the optional vanity hostname gateway. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/vanity-gateway) | -| `helm-reval` | `1.4.0` | Required | Deploys the function revalidation service. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/helm-reval) | -| `nvcf-default-monitors` | `0.2.0` | Required | Deploys the default service and pod monitors for NVCF. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability/charts/nvcf-default-monitors) | -| `nvcf-example-dashboards` | `1.6.0` | Optional | Deploys example Grafana dashboards for NVCF telemetry. | `https://helm.ngc.nvidia.com/nvidia/nvcf/nvcf-example-dashboards:1.6.0` | | -| `nvcf-gateway-routes` | `1.18.1` | Required | Deploys Gateway API routes for NVCF services. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/gateway-routes) | -| `nvcf-observability-reference-stack` | `1.10.0` | Optional | Deploys a reference observability backend for evaluation. | `https://helm.ngc.nvidia.com/nvidia/nvcf/nvcf-observability-reference-stack:1.10.0` | | -| `nvcf-otel-collector` | `0.2.0` | Required | Configures the OpenTelemetry Collector used by NVCF. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability/charts/nvcf-otel-collector) | -| `opentelemetry-operator` | `0.122.0` | Required | Deploys the OpenTelemetry Operator. | `Publication pending` | [Upstream](https://github.com/open-telemetry/opentelemetry-operator) | -| `prometheus-operator-crds` | `31.0.1` | Required | Installs the Prometheus Operator custom resource definitions. | `Publication pending` | [Upstream](https://github.com/prometheus-community/helm-charts) | -| `victoria-metrics-single` | `0.45.0` | Required | Deploys the default metrics storage backend. | `Publication pending` | [Upstream](https://github.com/VictoriaMetrics/helm-charts) | +| Artifact | Version | Stack | Required | Description | Distribution | Source code | +| --- | --- | --- | --- | --- | --- | --- | +| `helm-admin-token-issuer-proxy` | `1.5.2` | `self-managed` | Required | Deploys the admin token issuer proxy. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/admin-token-issuer-proxy) | +| `helm-nvcf-api` | `1.27.0` | `self-managed` | Required | Deploys the NVCF API service. | `Publication pending` | | +| `helm-nvcf-api-keys` | `1.8.0` | `self-managed` | Required | Deploys the API key management service. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/api-keys-colocated) | +| `helm-nvcf-cassandra` | `0.21.0` | `self-managed` | Required | Deploys Cassandra and its initialization jobs. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/cassandra) / [Upstream](https://github.com/bitnami/charts/tree/main/bitnami/cassandra) | +| `helm-nvcf-cert-manager` | `0.1.0` | `self-managed` | Required | Deploys the NVCF cert-manager configuration. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-cert-manager:0.1.0` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/cert-manager) / [Upstream](https://github.com/cert-manager/cert-manager) | +| `helm-nvcf-ess-api` | `1.8.2` | `self-managed` | Required | Deploys the Encrypted Secrets Service API. | `Publication pending` | | +| `helm-nvcf-function-autoscaler` | `0.4.0` | `self-managed` | Required | Deploys the function autoscaler for observability-driven scaling. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/function-autoscaler) | +| `helm-nvcf-grpc-proxy` | `1.7.3` | `self-managed` | Required | Deploys the gRPC proxy service. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/grpc-proxy) | +| `helm-nvcf-invocation-service` | `1.6.1` | `self-managed` | Required | Deploys the HTTP invocation service. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/http-invocation) | +| `helm-nvcf-llm-api-gateway` | `1.4.3` | `self-managed` | Optional | Deploys the OpenAI-compatible LLM API gateway. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/llm-api-gateway) | +| `helm-nvcf-llm-request-router` | `1.13.3` | `self-managed` | Optional | Deploys the LLM request router. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/llm-request-router) | +| `helm-nvcf-nats` | `0.8.1` | `self-managed` | Required | Deploys NATS messaging for the control plane. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nats) / [Upstream](https://github.com/nats-io/k8s) | +| `helm-nvcf-nats-auth-callout-service` | `1.2.0` | `self-managed` | Required | Deploys the NATS authorization callout service. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nats-auth-callout) | +| `helm-nvcf-notary-service` | `1.6.0` | `self-managed` | Required | Deploys the notary service for signing and validation. | `Publication pending` | | +| `helm-nvcf-nvct-api` | `1.6.0` | `self-managed` | Required | Deploys the NVCF tenant API service. | `Publication pending` | | +| `helm-nvcf-openbao-server` | `0.32.4` | `self-managed` | Required | Deploys OpenBao secret management. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/openbao) / [Upstream](https://github.com/openbao/openbao-helm) | +| `helm-nvcf-pki` | `0.1.0` | `self-managed` | Optional | Provisions the OpenBao-backed ClusterIssuer for NVCF service TLS. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-pki:0.1.0` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nvcf-pki) | +| `helm-nvcf-rate-limiter` | `1.2.1` | `self-managed` | Required | Deploys request rate limiting for supported invocation paths. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/ratelimiter) | +| `helm-nvcf-sis` | `2.3.0` | `self-managed` | Required | Deploys the Spot Instance Service. | `Publication pending` | | +| `helm-nvcf-state-metrics` | `1.0.2` | `self-managed` | Required | Deploys NVCF state metrics for observability. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-state-metrics:1.0.2` | | +| `helm-nvcf-ui` | `1.1.2` | `self-managed` | Optional | Deploys the optional NVCF UI admin panel. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-ui:1.1.2` | | +| `helm-nvcf-vanity-gateway` | `0.4.4` | `self-managed` | Optional | Deploys the optional vanity hostname gateway. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/vanity-gateway) | +| `helm-reval` | `1.4.0` | `self-managed` | Required | Deploys the function revalidation service. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/helm-reval) | +| `nvcf-default-monitors` | `0.2.0` | `self-managed` | Required | Deploys the default service and pod monitors for NVCF. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability/charts/nvcf-default-monitors) | +| `nvcf-example-dashboards` | `1.6.0` | `self-managed` | Optional | Deploys example Grafana dashboards for NVCF telemetry. | `https://helm.ngc.nvidia.com/nvidia/nvcf/nvcf-example-dashboards:1.6.0` | | +| `nvcf-gateway-routes` | `1.18.1` | `self-managed` | Required | Deploys Gateway API routes for NVCF services. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/gateway-routes) | +| `nvcf-observability-reference-stack` | `1.10.0` | `self-managed` | Optional | Deploys a reference observability backend for evaluation. | `https://helm.ngc.nvidia.com/nvidia/nvcf/nvcf-observability-reference-stack:1.10.0` | | +| `nvcf-otel-collector` | `0.2.0` | `self-managed` | Required | Configures the OpenTelemetry Collector used by NVCF. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability/charts/nvcf-otel-collector) | +| `opentelemetry-operator` | `0.122.0` | `self-managed` | Required | Deploys the OpenTelemetry Operator. | `Publication pending` | [Upstream](https://github.com/open-telemetry/opentelemetry-operator) | +| `prometheus-operator-crds` | `31.0.1` | `self-managed` | Required | Installs the Prometheus Operator custom resource definitions. | `Publication pending` | [Upstream](https://github.com/prometheus-community/helm-charts) | +| `victoria-metrics-single` | `0.45.0` | `self-managed` | Required | Deploys the default metrics storage backend. | `Publication pending` | [Upstream](https://github.com/VictoriaMetrics/helm-charts) | ### Control plane services and images -| Artifact | Version | Required | Description | Distribution | Source code | -| --- | --- | --- | --- | --- | --- | -| `admin-token-issuer-proxy` | `1.1.1` | Required | Proxies admin token requests for stack services. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/admin-token-issuer-proxy) | -| `cassandra` | `5.0.8-nv-2.0.1` | Required | Stores NVCF account, function, cluster, and service state. | `nvcr.io/nvidia/nvcf/cassandra:5.0.8-nv-2.0.1` | [Upstream](https://github.com/apache/cassandra) | -| `cert-manager-cainjector` | `v1.20.2` | Required | Injects certificate authority data into Kubernetes resources. | `nvcr.io/nvidia/nvcf/cert-manager-cainjector:v1.20.2` | [Upstream](https://github.com/cert-manager/cert-manager) | -| `cert-manager-controller` | `v1.20.2` | Required | Reconciles certificates and issuers for the control plane. | `nvcr.io/nvidia/nvcf/cert-manager-controller:v1.20.2` | [Upstream](https://github.com/cert-manager/cert-manager) | -| `cert-manager-startupapicheck` | `v1.20.2` | Required | Verifies that the cert-manager API is ready. | `nvcr.io/nvidia/nvcf/cert-manager-startupapicheck:v1.20.2` | [Upstream](https://github.com/cert-manager/cert-manager) | -| `cert-manager-webhook` | `v1.20.2` | Required | Validates and converts cert-manager API resources. | `nvcr.io/nvidia/nvcf/cert-manager-webhook:v1.20.2` | [Upstream](https://github.com/cert-manager/cert-manager) | -| `icms-service-oss` | `0.7.1` | Required | Manages instance and cluster lifecycle operations. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/instance-cluster-management) | -| `k8s` | `1.37.0` | Required | Provides Kubernetes command-line utilities for deployment jobs. | `Publication pending` | [GitHub](https://github.com/alpine-docker/k8s) | -| `llm-api-gateway` | `0.14.2` | Optional | Exposes OpenAI-compatible APIs for LLM functions. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/llm-api-gateway) | -| `nats-box` | `0.19.7-nonroot` | Required | Provides NATS administration and diagnostic utilities. | `nvcr.io/nvidia/nvcf/nats-box:0.19.7-nonroot` | [Upstream](https://github.com/nats-io/nats-box) | -| `nats-server` | `2.14.6-alpine3.22` | Required | Provides messaging for function deployment and invocation. | `Publication pending` | [Upstream](https://github.com/nats-io/nats-server) | -| `nats-server-config-reloader` | `0.24.0` | Required | Reloads NATS server configuration when mounted settings change. | `Publication pending` | [Upstream](https://github.com/nats-io/k8s) | -| `nvcf-ai-api-gateway-service` | `1.35.1` | Optional | Serves the optional vanity hostname gateway. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/vanity-gateway) | -| `nvcf-api-keys-service` | `1.9.1` | Required | Creates and manages NVCF API keys. | `Publication pending` | | -| `nvcf-ess` | `0.5.0` | Required | Provides encrypted application secrets to NVCF workloads. | `Publication pending` | | -| `nvcf-function-autoscaler` | `1.21.5` | Required | Scales functions from NVCF workload metrics. | `Publication pending` | | -| `nvcf-grpc-proxy` | `1.33.4` | Required | Proxies bidirectional gRPC traffic between the control and compute planes. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/grpc-proxy) | -| `nvcf-invocation-service` | `0.12.1` | Required | Routes stateless HTTP function invocation requests. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/http-invocation) | -| `nvcf-nats-auth-callout-service` | `0.8.3` | Required | Authorizes NATS clients for NVCF services and workloads. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/nats-auth-callout) | -| `nvcf-notary` | `1.14.1` | Required | Signs and validates functions and cluster nodes. | `Publication pending` | | -| `nvcf-openbao` | `2.5.5-nv-1.3.1` | Required | Stores and manages control-plane secrets. | `Publication pending` | [Upstream](https://github.com/openbao/openbao) | -| `nvcf-openbao-migrations` | `0.19.1` | Required | Applies the OpenBao configuration required by NVCF. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/migrations/openbao) | -| `nvcf-ratelimiter` | `1.17.3` | Required | Enforces request rate limits for supported invocation paths. | `Publication pending` | | -| `nvcf-service-oss` | `1.18.0` | Required | Provides the primary NVCF control-plane API. | `Publication pending` | | -| `nvcf-state-metrics-service` | `1.23.7` | Required | Exports NVCF resource state as Prometheus metrics. | `nvcr.io/nvidia/nvcf/nvcf-state-metrics-service:1.23.7` | | -| `nvcf-ui` | `1.1.2` | Optional | Serves the optional NVCF administrative interface. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/uis/nvcf-ui) | -| `nvct-service-oss` | `1.66.0` | Required | Provides tenant-scoped NVCF control-plane operations. | `Publication pending` | | -| `opentelemetry-collector-contrib` | `0.160.0` | Required | Collects and exports NVCF telemetry. | `Publication pending` | [Upstream](https://github.com/open-telemetry/opentelemetry-collector-contrib) | -| `opentelemetry-operator` | `0.158.0` | Required | Reconciles OpenTelemetry Collector resources. | `Publication pending` | [Upstream](https://github.com/open-telemetry/opentelemetry-operator) | -| `oss-vault-k8s` | `1.7.4` | Required | Integrates Kubernetes workloads with OpenBao secrets. | `nvcr.io/nvidia/nvcf/oss-vault-k8s:1.7.4` | | -| `reval-server` | `0.20.2` | Required | Revalidates function state in the background. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/helm-reval) | -| `stargate` | `0.16.2` | Optional | Routes LLM requests to eligible worker instances. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/libraries/rust/stargate) | -| `victoria-metrics` | `v1.150.0` | Required | Stores metrics for the default observability profile. | `Publication pending` | [Upstream](https://github.com/VictoriaMetrics/VictoriaMetrics) | +| Artifact | Version | Stack | Required | Description | Distribution | Source code | +| --- | --- | --- | --- | --- | --- | --- | +| `admin-token-issuer-proxy` | `1.1.1` | `self-managed` | Required | Proxies admin token requests for stack services. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/admin-token-issuer-proxy) | +| `cassandra` | `5.0.8-nv-2.0.1` | `self-managed` | Required | Stores NVCF account, function, cluster, and service state. | `nvcr.io/nvidia/nvcf/cassandra:5.0.8-nv-2.0.1` | [Upstream](https://github.com/apache/cassandra) | +| `cert-manager-acmesolver` | `v1.20.2` | `self-managed` | Optional | Serves temporary ACME HTTP-01 domain-validation challenges. | `Publication pending` | [Upstream](https://github.com/cert-manager/cert-manager) | +| `cert-manager-cainjector` | `v1.20.2` | `self-managed` | Required | Injects certificate authority data into Kubernetes resources. | `nvcr.io/nvidia/nvcf/cert-manager-cainjector:v1.20.2` | [Upstream](https://github.com/cert-manager/cert-manager) | +| `cert-manager-controller` | `v1.20.2` | `self-managed` | Required | Reconciles certificates and issuers for the control plane. | `nvcr.io/nvidia/nvcf/cert-manager-controller:v1.20.2` | [Upstream](https://github.com/cert-manager/cert-manager) | +| `cert-manager-startupapicheck` | `v1.20.2` | `self-managed` | Required | Verifies that the cert-manager API is ready. | `nvcr.io/nvidia/nvcf/cert-manager-startupapicheck:v1.20.2` | [Upstream](https://github.com/cert-manager/cert-manager) | +| `cert-manager-webhook` | `v1.20.2` | `self-managed` | Required | Validates and converts cert-manager API resources. | `nvcr.io/nvidia/nvcf/cert-manager-webhook:v1.20.2` | [Upstream](https://github.com/cert-manager/cert-manager) | +| `icms-service-oss` | `0.7.1` | `self-managed` | Required | Manages instance and cluster lifecycle operations. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/instance-cluster-management) | +| `k8s` | `1.37.0` | `self-managed` | Required | Provides Kubernetes command-line utilities for deployment jobs. | `Publication pending` | [GitHub](https://github.com/alpine-docker/k8s) | +| `llm-api-gateway` | `0.14.2` | `self-managed` | Optional | Exposes OpenAI-compatible APIs for LLM functions. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/llm-api-gateway) | +| `nats-box` | `0.19.7-nonroot` | `self-managed` | Required | Provides NATS administration and diagnostic utilities. | `nvcr.io/nvidia/nvcf/nats-box:0.19.7-nonroot` | [Upstream](https://github.com/nats-io/nats-box) | +| `nats-server` | `2.14.6-alpine3.22` | `self-managed` | Required | Provides messaging for function deployment and invocation. | `Publication pending` | [Upstream](https://github.com/nats-io/nats-server) | +| `nats-server-config-reloader` | `0.24.0` | `self-managed` | Required | Reloads NATS server configuration when mounted settings change. | `Publication pending` | [Upstream](https://github.com/nats-io/k8s) | +| `nvcf-ai-api-gateway-service` | `1.35.1` | `self-managed` | Optional | Serves the optional vanity hostname gateway. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/vanity-gateway) | +| `nvcf-api-keys-service` | `1.9.1` | `self-managed` | Required | Creates and manages NVCF API keys. | `Publication pending` | | +| `nvcf-ess` | `0.5.0` | `self-managed` | Required | Provides encrypted application secrets to NVCF workloads. | `Publication pending` | | +| `nvcf-function-autoscaler` | `1.21.5` | `self-managed` | Required | Scales functions from NVCF workload metrics. | `Publication pending` | | +| `nvcf-grpc-proxy` | `1.33.4` | `self-managed` | Required | Proxies bidirectional gRPC traffic between the control and compute planes. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/grpc-proxy) | +| `nvcf-invocation-service` | `0.12.1` | `self-managed` | Required | Routes stateless HTTP function invocation requests. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/http-invocation) | +| `nvcf-nats-auth-callout-service` | `0.8.3` | `self-managed` | Required | Authorizes NATS clients for NVCF services and workloads. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/nats-auth-callout) | +| `nvcf-notary` | `1.14.1` | `self-managed` | Required | Signs and validates functions and cluster nodes. | `Publication pending` | | +| `nvcf-openbao` | `2.5.5-nv-1.3.1` | `self-managed` | Required | Stores and manages control-plane secrets. | `Publication pending` | [Upstream](https://github.com/openbao/openbao) | +| `nvcf-openbao-migrations` | `0.19.1` | `self-managed` | Required | Applies the OpenBao configuration required by NVCF. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/migrations/openbao) | +| `nvcf-ratelimiter` | `1.17.3` | `self-managed` | Required | Enforces request rate limits for supported invocation paths. | `Publication pending` | | +| `nvcf-service-oss` | `1.18.0` | `self-managed` | Required | Provides the primary NVCF control-plane API. | `Publication pending` | | +| `nvcf-state-metrics-service` | `1.23.7` | `self-managed` | Required | Exports NVCF resource state as Prometheus metrics. | `nvcr.io/nvidia/nvcf/nvcf-state-metrics-service:1.23.7` | | +| `nvcf-ui` | `1.1.2` | `self-managed` | Optional | Serves the optional NVCF administrative interface. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/uis/nvcf-ui) | +| `nvct-service-oss` | `1.66.0` | `self-managed` | Required | Provides tenant-scoped NVCF control-plane operations. | `Publication pending` | | +| `opentelemetry-collector-contrib` | `0.160.0` | `self-managed` | Required | Collects and exports NVCF telemetry. | `Publication pending` | [Upstream](https://github.com/open-telemetry/opentelemetry-collector-contrib) | +| `opentelemetry-operator` | `0.158.0` | `self-managed` | Required | Reconciles OpenTelemetry Collector resources. | `Publication pending` | [Upstream](https://github.com/open-telemetry/opentelemetry-operator) | +| `oss-vault-k8s` | `1.7.4` | `self-managed` | Required | Integrates Kubernetes workloads with OpenBao secrets. | `nvcr.io/nvidia/nvcf/oss-vault-k8s:1.7.4` | | +| `reval-server` | `0.20.2` | `self-managed` | Required | Revalidates function state in the background. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/helm-reval) | +| `stargate` | `0.16.2` | `self-managed` | Optional | Routes LLM requests to eligible worker instances. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/libraries/rust/stargate) | +| `victoria-metrics` | `v1.150.0` | `self-managed` | Required | Stores metrics for the default observability profile. | `Publication pending` | [Upstream](https://github.com/VictoriaMetrics/VictoriaMetrics) | ### Compute plane Helm charts -| Artifact | Version | Required | Description | Distribution | Source code | -| --- | --- | --- | --- | --- | --- | -| `csi-driver-smb` | `supported` | Optional | Provides SMB persistent volumes for supported deployments. | `https://raw.githubusercontent.com/kubernetes-csi/csi-driver-smb/master/charts` | [Upstream](https://github.com/kubernetes-csi/csi-driver-smb) | -| `dynamo-platform` | `1.4.2` | Optional | Deploys the optional NVIDIA Dynamo operator. | `Publication pending` | [Upstream](https://github.com/ai-dynamo/dynamo) | -| `ebs-csi-driver` | `supported` | Optional | Provides Amazon EBS persistent volumes for EKS clusters. | `https://kubernetes-sigs.github.io/aws-ebs-csi-driver` | [Upstream](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) | -| `gpu-operator` | `supported` | Required | Manages NVIDIA GPU software on Kubernetes nodes. | `https://helm.ngc.nvidia.com/nvidia` | [Upstream](https://github.com/NVIDIA/gpu-operator) | -| `grove-charts` | `v0.1.0-alpha.12` | Optional | Deploys the optional Grove operator for topology-aware scheduling. | `Publication pending` | [Upstream](https://github.com/ai-dynamo/grove) | -| `helm-nvca-operator` | `1.24.0` | Required | Deploys the NVCA operator and compute-plane integration. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nvca-operator) | -| `kai-scheduler` | `v0.17.1` | Optional | Deploys the optional KAI Scheduler. | `Publication pending` | [Upstream](https://github.com/NVIDIA/KAI-Scheduler) | -| `modelexpress` | `supported` | Optional | Distributes model weights peer-to-peer between Dynamo workers to reduce scale-out cold starts. Installed separately from the compute-plane stack. | `https://helm.ngc.nvidia.com/nvidia/ai-dynamo` | [Upstream](https://github.com/ai-dynamo/modelexpress) | -| `nvcf-cluster-topology` | `0.1.0` | Required | Configures cluster topology resources for compute scheduling. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology) | -| `nvcf-container-cache` | `0.25.22` | Optional | Deploys container image caching on GPU cluster nodes. | `https://helm.ngc.nvidia.com/nvidia/nvcf/nvcf-container-cache:0.25.22` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/container-cache) | +| Artifact | Version | Stack | Required | Description | Distribution | Source code | +| --- | --- | --- | --- | --- | --- | --- | +| `csi-driver-smb` | `supported` | Independent | Optional | Provides SMB persistent volumes for supported deployments. | `https://raw.githubusercontent.com/kubernetes-csi/csi-driver-smb/master/charts` | [Upstream](https://github.com/kubernetes-csi/csi-driver-smb) | +| `dynamo-platform` | `1.4.2` | `compute-plane` | Optional | Deploys the optional NVIDIA Dynamo operator. | `Publication pending` | [Upstream](https://github.com/ai-dynamo/dynamo) | +| `ebs-csi-driver` | `supported` | Independent | Optional | Provides Amazon EBS persistent volumes for EKS clusters. | `https://kubernetes-sigs.github.io/aws-ebs-csi-driver` | [Upstream](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) | +| `gpu-operator` | `supported` | Independent | Required | Manages NVIDIA GPU software on Kubernetes nodes. | `https://helm.ngc.nvidia.com/nvidia` | [Upstream](https://github.com/NVIDIA/gpu-operator) | +| `grove-charts` | `v0.1.0-alpha.12` | `compute-plane` | Optional | Deploys the optional Grove operator for topology-aware scheduling. | `Publication pending` | [Upstream](https://github.com/ai-dynamo/grove) | +| `helm-nvca-operator` | `1.24.0` | `compute-plane` | Required | Deploys the NVCA operator and compute-plane integration. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nvca-operator) | +| `kai-scheduler` | `v0.17.1` | `compute-plane` | Optional | Deploys the optional KAI Scheduler. | `Publication pending` | [Upstream](https://github.com/NVIDIA/KAI-Scheduler) | +| `modelexpress` | `supported` | Independent | Optional | Distributes model weights peer-to-peer between Dynamo workers to reduce scale-out cold starts. Installed separately from the compute-plane stack. | `https://helm.ngc.nvidia.com/nvidia/ai-dynamo` | [Upstream](https://github.com/ai-dynamo/modelexpress) | +| `nvcf-cluster-topology` | `0.1.0` | `compute-plane` | Required | Configures cluster topology resources for compute scheduling. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology) | +| `nvcf-container-cache` | `0.25.22` | `compute-plane` | Optional | Deploys container image caching on GPU cluster nodes. | `https://helm.ngc.nvidia.com/nvidia/nvcf/nvcf-container-cache:0.25.22` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/container-cache) | ### Compute plane services and images -| Artifact | Version | Required | Description | Distribution | Source code | -| --- | --- | --- | --- | --- | --- | -| `crd-upgrader` | `v0.17.1` | Optional | Upgrades KAI Scheduler custom resources. | `Publication pending` | [Upstream](https://github.com/NVIDIA/KAI-Scheduler) | -| `ess-agent` | `1.4.1` | Required | Injects encrypted application secrets into function workloads. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/ess-agent) | -| `gpu-operator-validator` | `supported` | Required | Validates GPU Operator components on GPU nodes. | `https://catalog.ngc.nvidia.com/orgs/nvidia/teams/cloud-native/containers/gpu-operator-validator` | [Upstream](https://github.com/NVIDIA/gpu-operator) | -| `grove-install-crds` | `v0.1.0-alpha.12` | Optional | Installs Grove custom resource definitions. | `Publication pending` | [Upstream](https://github.com/ai-dynamo/grove) | -| `grove-operator` | `v0.1.0-alpha.12` | Optional | Reconciles Grove topology-aware scheduling resources. | `Publication pending` | [Upstream](https://github.com/ai-dynamo/grove) | -| `k8s-device-plugin` | `supported` | Required | Advertises NVIDIA GPU resources to Kubernetes. | `https://catalog.ngc.nvidia.com/orgs/nvidia/teams/k8s/containers/device-plugin` | [Upstream](https://github.com/NVIDIA/k8s-device-plugin) | -| `kubernetes-operator` | `1.4.2` | Optional | Reconciles NVIDIA Dynamo workloads on Kubernetes. | `Publication pending` | [Upstream](https://github.com/ai-dynamo/dynamo) | -| `modelexpress-server` | `supported` | Optional | Serves model weights to Dynamo workers over NIXL RDMA transports. | `https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/containers/modelexpress-server` | [Upstream](https://github.com/ai-dynamo/modelexpress) | -| `nats` | `2.10.21-alpine` | Optional | Provides messaging for the optional NVIDIA Dynamo operator. | `Publication pending` | [Upstream](https://github.com/nats-io/nats-server) | -| `nats-server-config-reloader` | `0.16.0` | Optional | Reloads NATS configuration for the optional NVIDIA Dynamo operator. | `Publication pending` | [Upstream](https://github.com/nats-io/k8s) | -| `nvca` | `3.5.2` | Required | Registers GPU clusters and orchestrates deployments in-cluster. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/nvca) | -| `nvca-operator` | `3.6.0` | Required | Reconciles NVCA resources and compute-plane configuration. | `Publication pending` | | -| `nvcf-container-cache` | `v1.1.36` | Optional | Caches container image layers on GPU cluster nodes. | `nvcr.io/nvidia/nvcf/nvcf-container-cache:v1.1.36` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/container-cache) | -| `nvcf-image-credential-helper` | `0.10.2` | Required | Resolves container image credentials for function workloads. | `nvcr.io/nvidia/nvcf/nvcf-image-credential-helper:0.10.2` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/image-credential-helper) | -| `nvcf-proxy-tls-certs` | `v1.2.10` | Optional | Configures TLS trust for the optional container cache proxy. | `nvcr.io/nvidia/nvcf/nvcf-proxy-tls-certs:v1.2.10` | | -| `nvcf-worker-init-oss` | `1.2.1` | Required | Prepares function resources before the user container starts. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-init) | -| `nvcf-worker-llm-credentials-oss` | `1.1.2` | Optional | Maintains a current NVCF worker token for LLM function workloads. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-llm-credentials) | -| `nvcf-worker-utils-oss` | `1.2.3` | Required | Proxies NATS traffic between function containers and the control plane. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-utils) | -| `operator` | `v0.17.1` | Optional | Reconciles KAI Scheduler resources. | `Publication pending` | [Upstream](https://github.com/NVIDIA/KAI-Scheduler) | -| `pylon` | `0.16.2` | Optional | Connects LLM worker pods to the LLM request router. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/libraries/rust/stargate) | +| Artifact | Version | Stack | Required | Description | Distribution | Source code | +| --- | --- | --- | --- | --- | --- | --- | +| `crd-upgrader` | `v0.17.1` | `compute-plane` | Optional | Upgrades KAI Scheduler custom resources. | `Publication pending` | [Upstream](https://github.com/NVIDIA/KAI-Scheduler) | +| `ess-agent` | `1.4.1` | `compute-plane` | Required | Injects encrypted application secrets into function workloads. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/ess-agent) | +| `gpu-operator-validator` | `supported` | Independent | Required | Validates GPU Operator components on GPU nodes. | `https://catalog.ngc.nvidia.com/orgs/nvidia/teams/cloud-native/containers/gpu-operator-validator` | [Upstream](https://github.com/NVIDIA/gpu-operator) | +| `grove-install-crds` | `v0.1.0-alpha.12` | `compute-plane` | Optional | Installs Grove custom resource definitions. | `Publication pending` | [Upstream](https://github.com/ai-dynamo/grove) | +| `grove-operator` | `v0.1.0-alpha.12` | `compute-plane` | Optional | Reconciles Grove topology-aware scheduling resources. | `Publication pending` | [Upstream](https://github.com/ai-dynamo/grove) | +| `k8s-device-plugin` | `supported` | Independent | Required | Advertises NVIDIA GPU resources to Kubernetes. | `https://catalog.ngc.nvidia.com/orgs/nvidia/teams/k8s/containers/device-plugin` | [Upstream](https://github.com/NVIDIA/k8s-device-plugin) | +| `kubernetes-operator` | `1.4.2` | `compute-plane` | Optional | Reconciles NVIDIA Dynamo workloads on Kubernetes. | `Publication pending` | [Upstream](https://github.com/ai-dynamo/dynamo) | +| `modelexpress-server` | `supported` | Independent | Optional | Serves model weights to Dynamo workers over NIXL RDMA transports. | `https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/containers/modelexpress-server` | [Upstream](https://github.com/ai-dynamo/modelexpress) | +| `nats` | `2.10.21-alpine` | `compute-plane` | Optional | Provides messaging for the optional NVIDIA Dynamo operator. | `Publication pending` | [Upstream](https://github.com/nats-io/nats-server) | +| `nats-server-config-reloader` | `0.16.0` | `compute-plane` | Optional | Reloads NATS configuration for the optional NVIDIA Dynamo operator. | `Publication pending` | [Upstream](https://github.com/nats-io/k8s) | +| `nvca` | `3.5.2` | `compute-plane` | Required | Registers GPU clusters and orchestrates deployments in-cluster. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/nvca) | +| `nvca-operator` | `3.6.0` | `compute-plane` | Required | Reconciles NVCA resources and compute-plane configuration. | `Publication pending` | | +| `nvcf-container-cache` | `v1.1.36` | `compute-plane` | Optional | Caches container image layers on GPU cluster nodes. | `nvcr.io/nvidia/nvcf/nvcf-container-cache:v1.1.36` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/container-cache) | +| `nvcf-image-credential-helper` | `0.10.2` | `compute-plane` | Required | Resolves container image credentials for function workloads. | `nvcr.io/nvidia/nvcf/nvcf-image-credential-helper:0.10.2` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/image-credential-helper) | +| `nvcf-proxy-tls-certs` | `v1.2.10` | `compute-plane` | Optional | Configures TLS trust for the optional container cache proxy. | `nvcr.io/nvidia/nvcf/nvcf-proxy-tls-certs:v1.2.10` | | +| `nvcf-worker-init-oss` | `1.2.1` | `compute-plane` | Required | Prepares function resources before the user container starts. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-init) | +| `nvcf-worker-llm-credentials-oss` | `1.1.2` | `compute-plane` | Optional | Maintains a current NVCF worker token for LLM function workloads. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-llm-credentials) | +| `nvcf-worker-utils-oss` | `1.2.3` | `compute-plane` | Required | Proxies NATS traffic between function containers and the control plane. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-utils) | +| `operator` | `v0.17.1` | `compute-plane` | Optional | Reconciles KAI Scheduler resources. | `Publication pending` | [Upstream](https://github.com/NVIDIA/KAI-Scheduler) | +| `pylon` | `0.16.2` | `compute-plane` | Optional | Connects LLM worker pods to the LLM request router. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/libraries/rust/stargate) | + +### Observability Helm charts + +| Artifact | Version | Stack | Required | Description | Distribution | Source code | +| --- | --- | --- | --- | --- | --- | --- | + +### Observability services and images + +| Artifact | Version | Stack | Required | Description | Distribution | Source code | +| --- | --- | --- | --- | --- | --- | --- | + +### Cross-stack Helm charts + +| Artifact | Version | Stack | Required | Description | Distribution | Source code | +| --- | --- | --- | --- | --- | --- | --- | + +### Cross-stack services and images + +| Artifact | Version | Stack | Required | Description | Distribution | Source code | +| --- | --- | --- | --- | --- | --- | --- | ### EA-only CVE-impacted artifacts These Early Access artifacts have known CVE impact. Use only the QA-qualified versions listed for this EA stack. -| Artifact | Version | Required | Description | Distribution | Source code | -| --- | --- | --- | --- | --- | --- | -| `nvcf-cassandra-migrations` | `0.17.5` | Required | Applies the Cassandra schemas required by Early Access NVCF services. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/migrations/cassandra) | +| Artifact | Version | Stack | Required | Description | Distribution | Source code | +| --- | --- | --- | --- | --- | --- | --- | +| `nvcf-cassandra-migrations` | `0.17.5` | `self-managed` | Required | Applies the Cassandra schemas required by Early Access NVCF services. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/migrations/cassandra) | ### Tools and deployment resources -| Artifact | Version | Description | Distribution | Source code | -| --- | --- | --- | --- | --- | -| `nvcf-cli` | `1.16.2` | Manages functions, deployments, and clusters from the command line. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/clis/nvcf-cli) | -| `nvcf-compute-plane-stack` | `0.16.1` | Provides the Helmfile bundle for compute-plane deployment. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/nvcf-compute-plane) | -| `nvcf-observability-stack` | `0.16.1` | Provides the Helmfile bundle for standalone observability deployment. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability) | -| `nvcf-self-managed-stack` | `0.16.1` | Provides the Helmfile bundle for control-plane deployment. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/self-managed) | +| Artifact | Version | Stack | Description | Distribution | Source code | +| --- | --- | --- | --- | --- | --- | +| `nvcf-cli` | `1.16.2` | Independent | Manages functions, deployments, and clusters from the command line. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/clis/nvcf-cli) | +| `nvcf-compute-plane-stack` | `0.3.1` | `compute-plane` | Provides the Helmfile bundle for compute-plane deployment. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/nvcf-compute-plane) | +| `nvcf-observability-stack` | `0.1.1` | `observability` | Provides the Helmfile bundle for standalone observability deployment. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability) | +| `nvcf-self-managed-stack` | `0.16.1` | `self-managed` | Provides the Helmfile bundle for control-plane deployment. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/self-managed) | {/*docs-version-sync:END manifest-artifact-registry-paths*/} diff --git a/docs/version-catalog/main.yaml b/docs/version-catalog/main.yaml index 33d50a9646..bc5bd844b0 100644 --- a/docs/version-catalog/main.yaml +++ b/docs/version-catalog/main.yaml @@ -98,6 +98,7 @@ version_overrides: source: independent source release; not pinned by stack publication_pending: - admin-token-issuer-proxy + - cert-manager-acmesolver - crd-upgrader - dynamo-platform - ess-agent @@ -360,6 +361,12 @@ manifest: requirement: required description: Provides Kubernetes command-line utilities for deployment jobs. github_url: https://github.com/alpine-docker/k8s + - artifact_id: cert-manager-acmesolver + plane: control + kind: service-image + requirement: optional + description: Serves temporary ACME HTTP-01 domain-validation challenges. + upstream_url: https://github.com/cert-manager/cert-manager - artifact_id: cert-manager-cainjector plane: control kind: service-image @@ -761,6 +768,25 @@ manifest: kind: resource description: Manages functions, deployments, and clusters from the command line. github_url: https://github.com/NVIDIA/nvcf/tree/main/src/clis/nvcf-cli +release_set: + documentation_version: dev + status: development + stacks: + control-plane: + version: 0.16.1 + source_tag: deploy/stacks/self-managed/v0.16.1 + source_commit: 6aa572575c3f3ee7716a9fddc4cbfceae17e73ec + inventory_asset: nvcf-self-managed-stack-inventory.json + compute-plane: + version: 0.3.1 + source_tag: deploy/stacks/nvcf-compute-plane/v0.3.1 + source_commit: 9694ba0c27a160e91a9be8a493a0c54cfb81825a + inventory_asset: nvcf-compute-plane-stack-inventory.json + observability: + version: 0.1.1 + source_tag: deploy/stacks/observability/v0.1.1 + source_commit: c9ca3b27124a60b99aba42ca3a81d10ce79a3924 + inventory_asset: nvcf-observability-stack-inventory.json stack: name: nvcf-self-managed-stack version: 0.16.1 @@ -768,11 +794,9 @@ stack: source_tag: deploy/stacks/self-managed/v0.16.1 source_commit: 6aa572575c3f3ee7716a9fddc4cbfceae17e73ec pin_sources: - - deploy/stacks/nvcf-compute-plane/environments/base.yaml - - deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl - deploy/stacks/self-managed/global.yaml.gotmpl - deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl - pin_source_digest: sha256:9080c81090e00803c9920b4bbc3fe4015ea4efc7804f9a597d661f55f03ec118 + pin_source_digest: sha256:9246f885bc9f52a512844da5d6fcbb9d5d32099c8a4cb6f063c4cc145cf058ff denylist: - name: strap reason: Legacy name replaced by nvcf-service-oss @@ -864,6 +888,10 @@ artifacts: type: image registry: public-images version: 5.0.8-nv-2.0.1 + - name: cert-manager-acmesolver + type: image + registry: public-images + version: v1.20.2 - name: cert-manager-cainjector type: image registry: public-images @@ -1153,11 +1181,11 @@ supplemental_artifacts: - name: nvcf-compute-plane-stack type: resource registry: public-resources - version: 0.16.1 + version: 0.3.1 - name: nvcf-observability-stack type: resource registry: public-resources - version: 0.16.1 + version: 0.1.1 - name: nvcf-observability-reference-stack type: chart registry: public-helm diff --git a/tools/ci/github-release b/tools/ci/github-release index ee1a930f3e..496a00b0a6 100755 --- a/tools/ci/github-release +++ b/tools/ci/github-release @@ -850,26 +850,7 @@ def publish_resolved_stack_inventory(tag, asset_path): print(f"[github-release] uploaded {asset_path.name} to {tag}") return - with tempfile.TemporaryDirectory(prefix="nvcf-existing-release-asset-") as tmp: - run( - [ - "gh", - "release", - "download", - tag, - "--pattern", - asset_path.name, - "--dir", - tmp, - ] - ) - published_path = Path(tmp) / asset_path.name - if published_path.read_bytes() != asset_path.read_bytes(): - raise SystemExit( - f"GitHub release {tag} already has a different {asset_path.name}; " - "refusing to replace an immutable inventory" - ) - print(f"[github-release] GitHub release {tag} already has the identical {asset_path.name}") + print(f"[github-release] GitHub release {tag} already has {asset_path.name}; leaving it unchanged") def delete_release_after_inventory_failure(tag): diff --git a/tools/ci/github-release-subprojects.json b/tools/ci/github-release-subprojects.json index 55261dc243..9fe957ab1a 100644 --- a/tools/ci/github-release-subprojects.json +++ b/tools/ci/github-release-subprojects.json @@ -34,7 +34,9 @@ "path": "deploy/stacks/nvcf-compute-plane", "service_name": "nvcf-compute-plane-stack", "legacy_tag_prefix": "nvcf-compute-plane-stack-v", - "initial_version": "0.2.0" + "tag_format": "deploy/stacks/nvcf-compute-plane/v${version}", + "initial_version": "0.2.0", + "resolved_inventory_asset": "nvcf-compute-plane-stack-inventory.json" }, { "id": "nvcf-self-managed-stack", @@ -49,7 +51,8 @@ "path": "deploy/stacks/observability", "service_name": "nvcf-observability-stack", "tag_format": "deploy/stacks/observability/v${version}", - "initial_version": "0.0.0" + "initial_version": "0.0.0", + "resolved_inventory_asset": "nvcf-observability-stack-inventory.json" }, { "id": "grpc-proxy", diff --git a/tools/ci/test-github-release.py b/tools/ci/test-github-release.py index 7f0717a70a..56541ab526 100644 --- a/tools/ci/test-github-release.py +++ b/tools/ci/test-github-release.py @@ -1181,6 +1181,20 @@ def stack_release_metadata(self, asset_name="inventory.json"): ], } + def test_release_metadata_publishes_one_inventory_per_stack(self): + metadata_path = SCRIPT_PATH.with_name("github-release-subprojects.json") + metadata = json.loads(metadata_path.read_text()) + expected = { + "deploy/stacks/self-managed/v1.2.3": "nvcf-self-managed-stack-inventory.json", + "deploy/stacks/nvcf-compute-plane/v1.2.3": "nvcf-compute-plane-stack-inventory.json", + "deploy/stacks/observability/v1.2.3": "nvcf-observability-stack-inventory.json", + } + for tag, asset_name in expected.items(): + with self.subTest(tag=tag): + service = self.github_release.release_asset_service(metadata, tag, SCRIPT_PATH.parents[2]) + self.assertIsNotNone(service) + self.assertEqual(service["resolved_inventory_asset"], asset_name) + def chart_release_metadata(self): """Return minimal release metadata for the chart publication tests.""" return { @@ -1645,7 +1659,7 @@ def test_resolved_stack_inventory_identity_must_match_release(self): path, "deploy/stacks/self-managed/v1.2.3", "1.2.3", "e" * 40 ) - def test_existing_release_inventory_must_be_byte_identical(self): + def test_existing_release_inventory_is_left_unchanged(self): with tempfile.TemporaryDirectory() as tmp: asset = Path(tmp) / "inventory.json" asset.write_text("same\n") @@ -1653,29 +1667,12 @@ def test_existing_release_inventory_must_be_byte_identical(self): def fake_run(args, **_kwargs): if args[:3] == ["gh", "release", "view"]: return "inventory.json\n" - if args[:3] == ["gh", "release", "download"]: - destination = Path(args[args.index("--dir") + 1]) / "inventory.json" - destination.write_text("same\n") - return "" raise AssertionError(f"unexpected call: {args}") self.github_release.run = fake_run with contextlib.redirect_stdout(io.StringIO()): self.github_release.publish_resolved_stack_inventory("stack/v1.2.3", asset) - def fake_different_run(args, **_kwargs): - if args[:3] == ["gh", "release", "view"]: - return "inventory.json\n" - if args[:3] == ["gh", "release", "download"]: - destination = Path(args[args.index("--dir") + 1]) / "inventory.json" - destination.write_text("different\n") - return "" - raise AssertionError(f"unexpected call: {args}") - - self.github_release.run = fake_different_run - with self.assertRaisesRegex(SystemExit, "refusing to replace"): - self.github_release.publish_resolved_stack_inventory("stack/v1.2.3", asset) - def test_missing_release_inventory_is_uploaded(self): with tempfile.TemporaryDirectory() as tmp: asset = Path(tmp) / "inventory.json" @@ -1758,19 +1755,23 @@ def test_stack_tag_workflow_installs_pinned_inventory_tools(self): def test_stack_inventory_preflight_renders_without_release_publication(self): workflow = (SCRIPT_PATH.parents[2] / ".github/workflows/release-tags.yml").read_text() self.assertIn("inventory_tag:", workflow) - self.assertIn("name: self-managed inventory preflight", workflow) + self.assertIn("name: stack inventory preflight", workflow) self.assertIn("inputs.inventory_tag != ''", workflow) self.assertIn('token: ${{ github.token }}', workflow) self.assertIn("Render tagged inventory without publishing", workflow) self.assertIn("--generate-stack-inventory", workflow) - self.assertIn( - '--inventory-config "${GITHUB_WORKSPACE}/deploy/stacks/self-managed/release-inventory.yaml"', - workflow, - ) + self.assertIn("deploy/stacks/nvcf-compute-plane/v*", workflow) + self.assertIn("deploy/stacks/observability/v*", workflow) preflight = workflow.index("inventory-preflight:") tag_release = workflow.index("tag-release-notes:") - self.assertNotIn("github-release tag", workflow[preflight:tag_release]) - self.assertNotIn("upload-artifact", workflow[preflight:tag_release]) + preflight_workflow = workflow[preflight:tag_release] + self.assertIn("--inventory-config", preflight_workflow) + self.assertIn("grep -q '^states:'", preflight_workflow) + self.assertIn("predates its per-stack inventory states", preflight_workflow) + self.assertIn("--allow-unavailable-source-charts", preflight_workflow) + self.assertIn("actions/upload-artifact@v4", preflight_workflow) + self.assertIn("if-no-files-found: error", preflight_workflow) + self.assertNotIn("github-release tag", preflight_workflow) def test_release_replay_requires_an_exact_tag_ref(self): """A tag-shaped branch must not satisfy manual replay validation.""" diff --git a/tools/docs-version-sync/README.md b/tools/docs-version-sync/README.md index f7e45a182e..4d31c8d4c7 100644 --- a/tools/docs-version-sync/README.md +++ b/tools/docs-version-sync/README.md @@ -1,16 +1,15 @@ # Documentation Version Sync -This tool keeps the artifact versions in the top-of-tree documentation aligned -with the latest stable self-managed stack release. Top-of-tree documentation is -under `docs/user/`, but its version catalog follows a tagged stack release, not -an untagged `main` commit. +This tool keeps top-of-tree documentation aligned with three released stack +inventories. It also promotes an exact QA-qualified three-stack release set to +versioned documentation. ## Release and documentation flow ```text merge release-worthy stack change to main - -> release automation creates the self-managed stack tag and GitHub Release - -> the tag workflow attaches nvcf-self-managed-stack-inventory.json + -> release automation creates the owning stack tag and GitHub Release + -> the tag workflow attaches that stack's inventory JSON -> a maintainer runs docs-version-sync -> the catalog records public locations or Publication pending -> generated blocks under docs/user/ are updated in a Pull Request @@ -19,21 +18,21 @@ merge release-worthy stack change to main The stack release is automatic. A merge to `main` runs [`release-tags.yml`](../../.github/workflows/release-tags.yml), which invokes `tools/ci/github-release auto` for every registered subproject. A `feat:`, -`fix:`, or `perf:` commit that changes `deploy/stacks/self-managed/` creates the -next `deploy/stacks/self-managed/vX.Y.Z` tag and GitHub Release. Chart pin and -Helmfile changes in that subtree should use a release-worthy commit type. No -maintainer normally creates the stack tag by hand. - -The tag workflow renders the control-plane, compute-plane, and observability -stacks from the same tagged commit. It attaches the resolved chart and image -inventory before publishing the GitHub Release. Public artifact publishing is -a separate process and can happen later after QA. - -Changes limited to `deploy/stacks/nvcf-compute-plane/` or -`deploy/stacks/observability/` create releases for those registered subprojects. -The documentation sync still follows the next self-managed stack release that -contains those commits because the self-managed release owns the combined -inventory asset. +`fix:`, or `perf:` commit that changes a stack creates that stack's next tag +and GitHub Release. Chart pin and Helmfile changes should use a release-worthy +commit type. No maintainer normally creates the stack tag by hand. + +The tag workflow renders only the states owned by the tagged stack. It attaches +the resolved chart and image inventory before publishing the GitHub Release. +Public artifact publishing is a separate process and can happen later after QA. + +Each registered stack publishes its own inventory: + +- `nvcf-self-managed-stack-inventory.json` +- `nvcf-compute-plane-stack-inventory.json` +- `nvcf-observability-stack-inventory.json` + +No stack inventory references another stack's Helmfile state. ## Sync after an automatic stack release @@ -44,12 +43,13 @@ git fetch --tags origin go run -C tools/docs-version-sync . --target main --update-catalog ``` -The command selects the latest stable self-managed stack release unless -`--stack-version X.Y.Z` is supplied. It updates: +The command selects the latest stable release for all three stacks and records +a development release set. It updates: - `docs/version-catalog/main.yaml` - Generated blocks configured by the catalog under `docs/user/` -- The control-plane, compute-plane, and observability bundle versions +- The self-managed, compute-plane, and observability bundle versions +- The exact source tag, commit, and inventory asset for all three stacks The update retains publication records only when `name`, `type`, and `version` still match. New and changed versions without a matching record are added to @@ -97,29 +97,56 @@ go run -C tools/docs-version-sync . --target main A publication-only update does not require a new stack release. +## Promote a QA-qualified release set + +After QA approves the three stack versions and all artifacts are published, +run the catalog update with all three exact versions: + +```bash +go run -C tools/docs-version-sync . \ + --target main \ + --update-catalog \ + --qualification-version X.Y.Z \ + --stack-version A.B.C \ + --compute-stack-version D.E.F \ + --observability-stack-version G.H.I +go run -C tools/docs-version-sync . --target main +./tools/scripts/cut-docs-version.sh vX.Y.Z +``` + +The qualification command does not select latest versions. The docs snapshot +fails unless the catalog is qualified and its documentation version matches the +requested version. The version dropdown identifies all three stack versions. + ## Add an artifact to the stack inventory -For a chart or image deployed by a stack: +For the complete dependency workflow, including ownership, optionality, and +indirect images, see +[`deploy/stacks/INVENTORY.md`](../../deploy/stacks/INVENTORY.md). + +For a chart or image deployed by any stack: 1. Add it to the owning Helmfile or chart values. 2. Confirm the inventory renderer includes the path: - A default release in an existing state is discovered automatically. - For a release behind a new optional setting, add that setting to the - matching `fullOverrides` entry in `resolved_inventory_publish.go`. - - For a new Helmfile state, add the state to `resolvedInventoryStates`. + matching `fullOverrides` entry in + owning `release-inventory.yaml`. + - For a new Helmfile state, add a `states` entry in that file. - If an independently released chart must be rendered from its immutable GitHub tag, add it to - [`release-inventory.yaml`](../../deploy/stacks/self-managed/release-inventory.yaml). + owning `release-inventory.yaml`. 3. Merge the release-worthy change to `main`. Release automation creates the - corresponding registered stack release. -4. Run the documentation sync after a self-managed stack release containing - the change appears with its inventory asset. -5. Add a `manifest.entries` record for the new artifact. + owning stack release. +4. Run the documentation sync after all selected stack releases have inventory + assets. +5. Add a `manifest.entries` record for the new artifact description and source. 6. Add a verified public publication or leave the artifact pending. A catalog-backed manifest entry uses `artifact_id` and describes the deployment -plane, kind, requirement, purpose, and public source links. The generator fails -when a discovered artifact has no classification. +plane, kind, purpose, and public source links. Released inventory data supplies +stack ownership and required or optional status. The generator fails when a +discovered artifact has no classification. ## Add an independently versioned artifact @@ -144,6 +171,28 @@ is intentionally excluded from public documentation, and give it a public-safe reason. Do not use a private registry as a placeholder for an unpublished artifact. +## Compare release sets + +Store each release set's inventory JSON files in a directory, then run: + +```bash +go run -C tools/docs-version-sync . \ + --compare-release-set-from /tmp/previous-inventories \ + --compare-release-set-to /tmp/current-inventories +``` + +The report detects dependency disappearance and explains additions, reference +or digest changes, ownership changes, and required or optional changes. A +historical directory can contain one legacy combined inventory. A current +directory normally contains three separated inventories. + +Use the `inventory_tag` input on `release-tags.yml` to reproduce an inventory +without publishing a release. The preflight uploads the JSON as a workflow +artifact. Tags that lack a config with per-stack states use the config from the +workflow ref as a one-time bootstrap. They use an available tagged source chart +and otherwise render the published chart version. New tags carry their own +config. + ## Validation Run the focused Go checks when changing tool behavior: diff --git a/tools/docs-version-sync/catalog.go b/tools/docs-version-sync/catalog.go index 4e1b9247e1..d7473a5800 100644 --- a/tools/docs-version-sync/catalog.go +++ b/tools/docs-version-sync/catalog.go @@ -54,9 +54,10 @@ type ManifestKind string type ManifestRequirement string const ( - ManifestPlaneControl ManifestPlane = "control" - ManifestPlaneCompute ManifestPlane = "compute" - ManifestPlaneShared ManifestPlane = "shared" + ManifestPlaneControl ManifestPlane = "control" + ManifestPlaneCompute ManifestPlane = "compute" + ManifestPlaneObservability ManifestPlane = "observability" + ManifestPlaneShared ManifestPlane = "shared" ManifestKindChart ManifestKind = "chart" ManifestKindServiceImage ManifestKind = "service-image" @@ -92,6 +93,7 @@ type Catalog struct { VersionOverrides []VersionOverride `yaml:"version_overrides,omitempty"` PublicationPending []string `yaml:"publication_pending,omitempty"` Manifest ManifestMetadata `yaml:"manifest,omitempty"` + ReleaseSet ReleaseSetMetadata `yaml:"release_set,omitempty"` Stack StackMetadata `yaml:"stack"` Denylist []DenylistEntry `yaml:"denylist,omitempty"` Artifacts []Artifact `yaml:"artifacts"` @@ -99,6 +101,33 @@ type Catalog struct { Outputs []OutputFile `yaml:"outputs"` } +type ReleaseSetStatus string + +const ( + ReleaseSetDevelopment ReleaseSetStatus = "development" + ReleaseSetQualified ReleaseSetStatus = "qualified" +) + +// ReleaseSetMetadata identifies the three stack releases represented by the catalog. +type ReleaseSetMetadata struct { + DocumentationVersion string `yaml:"documentation_version"` + Status ReleaseSetStatus `yaml:"status"` + Stacks ReleaseSetStacks `yaml:"stacks"` +} + +type ReleaseSetStacks struct { + ControlPlane StackReleaseMetadata `yaml:"control-plane"` + ComputePlane StackReleaseMetadata `yaml:"compute-plane"` + Observability StackReleaseMetadata `yaml:"observability"` +} + +type StackReleaseMetadata struct { + Version string `yaml:"version"` + SourceTag string `yaml:"source_tag"` + SourceCommit string `yaml:"source_commit"` + InventoryAsset string `yaml:"inventory_asset"` +} + type Registry struct { Host string `yaml:"host"` Namespace string `yaml:"namespace"` @@ -150,13 +179,15 @@ var defaultDenylist = []DenylistEntry{ // Artifact identifies one versioned chart, image, or downloadable resource in the catalog. type Artifact struct { - ID string `yaml:"id,omitempty"` - Name string `yaml:"name"` - Type ArtifactType `yaml:"type"` - Registry string `yaml:"registry"` - RepositoryName string `yaml:"repository_name,omitempty"` - Version string `yaml:"version"` - Digest string `yaml:"digest,omitempty"` + ID string `yaml:"id,omitempty"` + Name string `yaml:"name"` + Type ArtifactType `yaml:"type"` + Registry string `yaml:"registry"` + RepositoryName string `yaml:"repository_name,omitempty"` + Version string `yaml:"version"` + Digest string `yaml:"digest,omitempty"` + Stacks []string `yaml:"stacks,omitempty"` + Requirement ManifestRequirement `yaml:"requirement,omitempty"` } type OutputFile struct { @@ -320,6 +351,11 @@ func ValidateCatalog(catalog *Catalog) error { } seenPending[name] = struct{}{} } + if catalog.ReleaseSet != (ReleaseSetMetadata{}) { + if err := validateReleaseSet(catalog.ReleaseSet); err != nil { + return err + } + } if strings.TrimSpace(catalog.Stack.Name) == "" { return fmt.Errorf("stack name cannot be empty") } @@ -416,6 +452,48 @@ func ValidateCatalog(catalog *Catalog) error { return nil } +func validateReleaseSet(releaseSet ReleaseSetMetadata) error { + if releaseSet.DocumentationVersion == "" || releaseSet.DocumentationVersion != strings.TrimSpace(releaseSet.DocumentationVersion) { + return fmt.Errorf("release_set documentation_version must be non-empty and trimmed") + } + if releaseSet.Status != ReleaseSetDevelopment && releaseSet.Status != ReleaseSetQualified { + return fmt.Errorf("release_set status must be development or qualified") + } + if releaseSet.Status == ReleaseSetDevelopment && releaseSet.DocumentationVersion != "dev" { + return fmt.Errorf("development release_set documentation_version must be dev") + } + if releaseSet.Status == ReleaseSetQualified && !validStableStackVersion(strings.TrimPrefix(releaseSet.DocumentationVersion, "v")) { + return fmt.Errorf("qualified release_set documentation_version must be a stable semantic version") + } + for _, stack := range []struct { + name string + metadata StackReleaseMetadata + key string + }{ + {name: "control-plane", metadata: releaseSet.Stacks.ControlPlane, key: selfManagedStackKey}, + {name: "compute-plane", metadata: releaseSet.Stacks.ComputePlane, key: computePlaneStackKey}, + {name: "observability", metadata: releaseSet.Stacks.Observability, key: observabilityStackKey}, + } { + spec, err := stackInventorySpecByKey(stack.key) + if err != nil { + return err + } + if !validStackVersion(stack.metadata.Version) { + return fmt.Errorf("release_set %s version must be semantic version", stack.name) + } + if stack.metadata.SourceTag != spec.TagPrefix+stack.metadata.Version { + return fmt.Errorf("release_set %s source_tag must be %s%s", stack.name, spec.TagPrefix, stack.metadata.Version) + } + if !fullLowercaseCommitSHARe.MatchString(stack.metadata.SourceCommit) { + return fmt.Errorf("release_set %s source_commit must be a full lowercase commit SHA", stack.name) + } + if stack.metadata.InventoryAsset != spec.AssetName { + return fmt.Errorf("release_set %s inventory_asset must be %s", stack.name, spec.AssetName) + } + } + return nil +} + func validateManifestMetadata(metadata ManifestMetadata) error { seen := map[string]struct{}{} // Changing the EA-CVE allowlist requires an explicit documentation decision. @@ -429,7 +507,7 @@ func validateManifestMetadata(metadata ManifestMetadata) error { identifier = entry.Name } switch entry.Plane { - case ManifestPlaneControl, ManifestPlaneCompute, ManifestPlaneShared: + case ManifestPlaneControl, ManifestPlaneCompute, ManifestPlaneObservability, ManifestPlaneShared: default: return fmt.Errorf("manifest entry %q has unsupported plane %q", identifier, entry.Plane) } @@ -447,7 +525,7 @@ func validateManifestMetadata(metadata ManifestMetadata) error { } } else { switch entry.Requirement { - case ManifestRequired, ManifestOptional: + case "", ManifestRequired, ManifestOptional: default: return fmt.Errorf("manifest entry %q has unsupported requirement %q", identifier, entry.Requirement) } @@ -512,6 +590,19 @@ func (catalog *Catalog) validateArtifact(artifact Artifact) error { return fmt.Errorf("artifact %s has invalid digest %q", artifact.Name, artifact.Digest) } } + if artifact.Requirement != "" && artifact.Requirement != ManifestRequired && artifact.Requirement != ManifestOptional { + return fmt.Errorf("artifact %s has unsupported requirement %q", artifact.Name, artifact.Requirement) + } + seenStacks := map[string]struct{}{} + for _, stack := range artifact.Stacks { + if stack != selfManagedStackKey && stack != computePlaneStackKey && stack != observabilityStackKey { + return fmt.Errorf("artifact %s has unknown owning stack %q", artifact.Name, stack) + } + if _, exists := seenStacks[stack]; exists { + return fmt.Errorf("artifact %s has duplicate owning stack %s", artifact.Name, stack) + } + seenStacks[stack] = struct{}{} + } if _, ok := catalog.Registries[artifact.Registry]; !ok { return fmt.Errorf("artifact %s registry %q is not defined", artifact.Name, artifact.Registry) } diff --git a/tools/docs-version-sync/catalog_inventory.go b/tools/docs-version-sync/catalog_inventory.go index 2811918a68..657f6aa479 100644 --- a/tools/docs-version-sync/catalog_inventory.go +++ b/tools/docs-version-sync/catalog_inventory.go @@ -5,6 +5,7 @@ package main import ( "fmt" + "os" "sort" "strings" ) @@ -27,6 +28,179 @@ func updateCatalogFromGitHub(repoRoot, sourceRef string, base *Catalog) (*Catalo return buildCatalogFromResolvedStackInventory(inventory, snapshot, base) } +func updateCatalogFromGitHubInventories(repoRoot string, sourceRefs map[string]string, qualificationVersion string, base *Catalog) (*Catalog, error) { + client := newGitHubClientFromEnvironment() + inventories := make(map[string]resolvedStackInventory, len(stackInventorySpecs)) + for _, spec := range stackInventorySpecs { + release, err := client.resolveStackSourceReleaseForSpec(spec, sourceRefs[spec.Key]) + if err != nil { + return nil, fmt.Errorf("resolve %s stack release: %w", spec.Key, err) + } + inventory, err := client.resolvedStackInventoryForSpec(spec, release) + if err != nil { + return nil, fmt.Errorf("read %s stack inventory: %w", spec.Key, err) + } + if err := validateInventoryPlaneOwnership(spec, inventory); err != nil { + return nil, err + } + for _, warning := range inventory.Warnings { + fmt.Fprintf(os.Stderr, "WARNING: %s stack inventory: %s\n", spec.Key, warning) + } + inventories[spec.Key] = inventory + } + + combined, err := mergeResolvedStackInventories(inventories) + if err != nil { + return nil, err + } + selfManaged := inventories[selfManagedStackKey] + sourcePaths := effectiveStackPinSourcePaths(effectiveStackPins) + snapshot, err := loadStackSourceSnapshot(repoRoot, selfManaged.Source, sourcePaths) + if err != nil { + return nil, err + } + catalog, err := buildCatalogFromResolvedStackInventory(combined, snapshot, base) + if err != nil { + return nil, err + } + status := ReleaseSetDevelopment + documentationVersion := "dev" + if qualificationVersion != "" { + status = ReleaseSetQualified + documentationVersion = strings.TrimPrefix(qualificationVersion, "v") + } + releaseSet, err := releaseSetFromInventories(inventories, documentationVersion, status) + if err != nil { + return nil, err + } + if qualificationVersion == "" && base != nil && base.ReleaseSet.Status == ReleaseSetQualified && releaseSet.sameStackReleases(base.ReleaseSet) { + releaseSet.DocumentationVersion = base.ReleaseSet.DocumentationVersion + releaseSet.Status = base.ReleaseSet.Status + } + catalog.ReleaseSet = releaseSet + for _, spec := range stackInventorySpecs[1:] { + version := inventories[spec.Key].Source.Version + if !setArtifactVersionByNameAndType(catalog, spec.ResourceName, ArtifactTypeResource, version) { + catalog.SupplementalArtifacts = append(catalog.SupplementalArtifacts, Artifact{ + Name: spec.ResourceName, + Type: ArtifactTypeResource, + Registry: defaultStackRegistry, + Version: version, + Stacks: []string{spec.Key}, + }) + } else { + setArtifactStacksByNameAndType(catalog, spec.ResourceName, ArtifactTypeResource, []string{spec.Key}) + } + } + retainCurrentPublications(catalog) + catalog.markAllUnpublishedAsPending() + catalog.reconcilePublicationPending() + catalog.pruneUnusedRegistries() + if qualificationVersion != "" && len(catalog.PublicationPending) > 0 { + return nil, fmt.Errorf("qualified release set has unpublished artifacts: %s", strings.Join(catalog.PublicationPending, ", ")) + } + if err := ValidateCatalog(catalog); err != nil { + return nil, err + } + return catalog, nil +} + +func setArtifactStacksByNameAndType(catalog *Catalog, name string, artifactType ArtifactType, stacks []string) { + for index := range catalog.Artifacts { + if catalog.Artifacts[index].Name == name && catalog.Artifacts[index].Type == artifactType { + catalog.Artifacts[index].Stacks = append([]string(nil), stacks...) + } + } + for index := range catalog.SupplementalArtifacts { + if catalog.SupplementalArtifacts[index].Name == name && catalog.SupplementalArtifacts[index].Type == artifactType { + catalog.SupplementalArtifacts[index].Stacks = append([]string(nil), stacks...) + } + } +} + +func validateInventoryPlaneOwnership(spec stackInventorySpec, inventory resolvedStackInventory) error { + wantPlane := map[string]string{ + selfManagedStackKey: "control-plane", + computePlaneStackKey: "compute-plane", + observabilityStackKey: "observability", + }[spec.Key] + for _, release := range inventory.Releases { + if release.Plane != wantPlane { + return fmt.Errorf("%s stack inventory contains %s release %s; want only %s releases", spec.Key, release.Plane, release.Name, wantPlane) + } + } + return nil +} + +func mergeResolvedStackInventories(inventories map[string]resolvedStackInventory) (resolvedStackInventory, error) { + selfManaged, ok := inventories[selfManagedStackKey] + if !ok { + return resolvedStackInventory{}, fmt.Errorf("self-managed stack inventory is required") + } + combined := resolvedStackInventory{ + SchemaVersion: resolvedStackInventorySchemaVersion, + Source: selfManaged.Source, + } + releases := make(map[string]struct{}) + artifacts := make(map[string]*resolvedInventoryArtifact) + artifactIdentities := make(map[string]string) + artifactOwners := make(map[string]string) + for _, spec := range stackInventorySpecs { + inventory, exists := inventories[spec.Key] + if !exists { + return resolvedStackInventory{}, fmt.Errorf("%s stack inventory is required", spec.Key) + } + if err := validateResolvedStackInventory(inventory); err != nil { + return resolvedStackInventory{}, fmt.Errorf("validate %s stack inventory: %w", spec.Key, err) + } + for _, release := range inventory.Releases { + key := resolvedReleaseKey(release.Plane, release.Name) + if _, duplicate := releases[key]; duplicate { + return resolvedStackInventory{}, fmt.Errorf("duplicate release %s across stack inventories", key) + } + releases[key] = struct{}{} + combined.Releases = append(combined.Releases, release) + } + for _, artifact := range inventory.Artifacts { + identity := artifact.Type + "\x00" + artifact.Name + if reference, exists := artifactIdentities[identity]; exists && reference != artifact.Reference { + return resolvedStackInventory{}, fmt.Errorf("stack inventory version conflict for %s %s: %s uses %s and %s uses %s", artifact.Type, artifact.Name, artifactOwners[identity], reference, spec.Key, artifact.Reference) + } + artifactIdentities[identity] = artifact.Reference + if _, exists := artifactOwners[identity]; !exists { + artifactOwners[identity] = spec.Key + } + key := artifact.Type + "\x00" + artifact.Reference + existing, exists := artifacts[key] + if !exists { + copy := artifact + copy.Sources = append([]resolvedArtifactSource(nil), artifact.Sources...) + artifacts[key] = © + continue + } + for _, source := range artifact.Sources { + mergeResolvedArtifactSource(existing, source) + } + } + } + for _, artifact := range artifacts { + sort.Slice(artifact.Sources, func(i, j int) bool { + return compareResolvedArtifactSources(artifact.Sources[i], artifact.Sources[j]) < 0 + }) + combined.Artifacts = append(combined.Artifacts, *artifact) + } + sort.Slice(combined.Releases, func(i, j int) bool { + return compareResolvedInventoryReleases(combined.Releases[i], combined.Releases[j]) < 0 + }) + sort.Slice(combined.Artifacts, func(i, j int) bool { + return compareResolvedInventoryArtifacts(combined.Artifacts[i], combined.Artifacts[j]) < 0 + }) + if err := validateResolvedStackInventory(combined); err != nil { + return resolvedStackInventory{}, fmt.Errorf("validate combined stack inventory: %w", err) + } + return combined, nil +} + func buildCatalogFromResolvedStackInventory(inventory resolvedStackInventory, snapshot stackSourceSnapshot, base *Catalog) (*Catalog, error) { if err := validateResolvedStackInventory(inventory); err != nil { return nil, err @@ -39,7 +213,7 @@ func buildCatalogFromResolvedStackInventory(inventory resolvedStackInventory, sn return nil, err } catalog := refreshCatalogFromArtifacts(inventory.Source.Version, artifacts, base) - retainIndependentManifestArtifacts(catalog) + retainIndependentManifestArtifacts(catalog, inventory, base) if err := materializeMissingEffectiveStackPins(catalog, base, snapshot.Files); err != nil { return nil, err } @@ -59,18 +233,6 @@ func buildCatalogFromResolvedStackInventory(inventory resolvedStackInventory, sn return nil, err } - for _, name := range []string{computeStackResourceName, observabilityStackResourceName} { - if !setArtifactVersionByNameAndType(catalog, name, ArtifactTypeResource, inventory.Source.Version) { - catalog.SupplementalArtifacts = append(catalog.SupplementalArtifacts, Artifact{ - Name: name, - Type: ArtifactTypeResource, - Registry: defaultStackRegistry, - Version: inventory.Source.Version, - }) - } - } - // The source release advances all deployment bundles together. Their public - // publication status remains independently verified below. retainCurrentPublications(catalog) catalog.markAllUnpublishedAsPending() catalog.reconcilePublicationPending() @@ -87,6 +249,10 @@ func catalogArtifactsFromResolvedStackInventory(inventory resolvedStackInventory denylistSource = &Catalog{} } denylist := denylistSource.DenylistMap() + requirements := make(map[string]bool, len(inventory.Releases)) + for _, release := range inventory.Releases { + requirements[resolvedReleaseKey(release.Plane, release.Name)] = release.Required + } artifacts := make([]Artifact, 0, len(inventory.Artifacts)) for _, resolved := range inventory.Artifacts { if _, denied := denylist[resolved.Name]; denied { @@ -99,12 +265,31 @@ func catalogArtifactsFromResolvedStackInventory(inventory resolvedStackInventory if resolved.Version == "" { return nil, fmt.Errorf("artifact %s has no tag version representable in the documentation catalog", resolved.Reference) } + stacks := make(map[string]struct{}) + requirement := ManifestOptional + for _, source := range resolved.Sources { + stack, err := stackKeyForPlane(source.Plane) + if err != nil { + return nil, fmt.Errorf("artifact %s: %w", resolved.Reference, err) + } + stacks[stack] = struct{}{} + if resolvedArtifactSourceIsRequired(source, requirements) { + requirement = ManifestRequired + } + } + owningStacks := make([]string, 0, len(stacks)) + for stack := range stacks { + owningStacks = append(owningStacks, stack) + } + sort.Strings(owningStacks) artifacts = append(artifacts, Artifact{ - Name: resolved.Name, - Type: artifactType, - Registry: publicRegistryForArtifactType(artifactType), - Version: resolved.Version, - Digest: resolved.Digest, + Name: resolved.Name, + Type: artifactType, + Registry: publicRegistryForArtifactType(artifactType), + Version: resolved.Version, + Digest: resolved.Digest, + Stacks: owningStacks, + Requirement: requirement, }) } preserveCatalogArtifactIdentity(artifacts, base) @@ -112,6 +297,17 @@ func catalogArtifactsFromResolvedStackInventory(inventory resolvedStackInventory return artifacts, nil } +func resolvedArtifactSourceIsRequired(source resolvedArtifactSource, releaseRequirements map[string]bool) bool { + switch source.Requirement { + case ManifestRequired: + return true + case ManifestOptional: + return false + default: + return releaseRequirements[resolvedReleaseKey(source.Plane, source.Release)] + } +} + func publicRegistryForArtifactType(artifactType ArtifactType) string { switch artifactType { case ArtifactTypeImage: @@ -155,14 +351,22 @@ func preserveCatalogArtifactIdentity(artifacts []Artifact, base *Catalog) { } } -// retainIndependentManifestArtifacts keeps public add-ons that are documented -// with the stack but are installed and versioned separately. -func retainIndependentManifestArtifacts(catalog *Catalog) { +// retainIndependentManifestArtifacts keeps public add-ons and artifacts from +// deployment planes that are not owned by this inventory. Legacy aggregate +// inventories still replace artifacts from every plane they contain. +func retainIndependentManifestArtifacts(catalog *Catalog, inventory resolvedStackInventory, base *Catalog) { denylist := catalog.DenylistMap() - referenced := make(map[string]struct{}, len(catalog.Manifest.Entries)) + referenced := make(map[string]ManifestPlane, len(catalog.Manifest.Entries)) for _, entry := range catalog.Manifest.Entries { if entry.ArtifactID != "" { - referenced[entry.ArtifactID] = struct{}{} + referenced[entry.ArtifactID] = entry.Plane + } + } + ownedPlanes := resolvedInventoryManifestPlanes(inventory) + independentArtifacts := make(map[string]struct{}) + if base != nil { + for _, artifact := range base.SupplementalArtifacts { + independentArtifacts[artifact.catalogKey()] = struct{}{} } } retained := catalog.SupplementalArtifacts[:0] @@ -170,9 +374,16 @@ func retainIndependentManifestArtifacts(catalog *Catalog) { if _, denied := denylist[artifact.Name]; denied { continue } - _, manifestArtifact := referenced[artifact.catalogKey()] - if artifact.Type != ArtifactTypeResource && !manifestArtifact { - continue + if artifact.Type != ArtifactTypeResource { + plane, manifestArtifact := referenced[artifact.catalogKey()] + if !manifestArtifact { + continue + } + if _, owned := ownedPlanes[plane]; owned { + if _, independent := independentArtifacts[artifact.catalogKey()]; !independent { + continue + } + } } artifact.Registry = publicRegistryForArtifactType(artifact.Type) retained = append(retained, artifact) @@ -180,6 +391,21 @@ func retainIndependentManifestArtifacts(catalog *Catalog) { catalog.SupplementalArtifacts = retained } +func resolvedInventoryManifestPlanes(inventory resolvedStackInventory) map[ManifestPlane]struct{} { + planes := make(map[ManifestPlane]struct{}) + for _, release := range inventory.Releases { + switch release.Plane { + case "control-plane": + planes[ManifestPlaneControl] = struct{}{} + case "compute-plane": + planes[ManifestPlaneCompute] = struct{}{} + case "observability": + planes[ManifestPlaneObservability] = struct{}{} + } + } + return planes +} + // retainCurrentPublications keeps exact public availability records for the // versions represented by the refreshed catalog. func retainCurrentPublications(catalog *Catalog) { diff --git a/tools/docs-version-sync/catalog_inventory_test.go b/tools/docs-version-sync/catalog_inventory_test.go index ce1fc66a05..f4e774f989 100644 --- a/tools/docs-version-sync/catalog_inventory_test.go +++ b/tools/docs-version-sync/catalog_inventory_test.go @@ -63,12 +63,117 @@ func TestUpdateCatalogFromGitHubUsesSelectedReleaseInventory(t *testing.T) { } for _, name := range []string{computeStackResourceName, observabilityStackResourceName} { artifact, ok := catalog.findArtifact(name) - if !ok || artifact.Version != release.Version { - t.Fatalf("%s = %#v, want release version %s", name, artifact, release.Version) + if !ok || artifact.Version != "0.5.0" { + t.Fatalf("%s = %#v, want independently owned version 0.5.0", name, artifact) } } } +func TestUpdateCatalogFromGitHubInventoriesMergesThreeStackAssets(t *testing.T) { + repo := initTestGitRepo(t) + selfRelease := commitTestStackSource(t, repo, "1.2.3", testCatalogPinSources()) + releases := map[string]stackSourceRelease{ + selfManagedStackKey: selfRelease, + computePlaneStackKey: { + Version: "2.3.4", Tag: stackInventorySpecs[1].TagPrefix + "2.3.4", Commit: selfRelease.Commit, + }, + observabilityStackKey: { + Version: "3.4.5", Tag: stackInventorySpecs[2].TagPrefix + "3.4.5", Commit: selfRelease.Commit, + }, + } + for _, key := range []string{computePlaneStackKey, observabilityStackKey} { + if _, err := gitOutput(repo, "tag", releases[key].Tag, releases[key].Commit); err != nil { + t.Fatal(err) + } + } + full := testCatalogResolvedInventory(t, selfRelease) + inventories := map[string]resolvedStackInventory{ + selfManagedStackKey: filterResolvedInventoryByPlane(t, full, releases[selfManagedStackKey], "control-plane"), + computePlaneStackKey: filterResolvedInventoryByPlane(t, full, releases[computePlaneStackKey], "compute-plane"), + observabilityStackKey: filterResolvedInventoryByPlane(t, full, releases[observabilityStackKey], "observability"), + } + raw := make(map[string][]byte, len(inventories)) + for key, inventory := range inventories { + var err error + raw[key], err = marshalResolvedStackInventory(inventory) + if err != nil { + t.Fatal(err) + } + } + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, spec := range stackInventorySpecs { + release := releases[spec.Key] + switch r.URL.Path { + case "/repos/NVIDIA/nvcf/git/ref/tags/" + release.Tag: + _, _ = fmt.Fprintf(w, `{"ref":"refs/tags/%s","object":{"type":"commit","sha":"%s"}}`, release.Tag, release.Commit) + return + case "/repos/NVIDIA/nvcf/releases/tags/" + release.Tag: + _, _ = fmt.Fprintf(w, `{"tag_name":%q,"assets":[{"name":%q,"browser_download_url":%q}]}`, release.Tag, spec.AssetName, server.URL+"/inventory/"+spec.Key) + return + case "/inventory/" + spec.Key: + _, _ = w.Write(raw[spec.Key]) + return + } + } + http.Error(w, "unexpected path "+r.URL.Path, http.StatusNotFound) + })) + defer server.Close() + + t.Setenv("DOC_VERSION_SYNC_GITHUB_API_URL", server.URL) + t.Setenv("DOC_VERSION_SYNC_GITHUB_TOKEN", "") + t.Setenv("GITHUB_TOKEN", "") + t.Setenv("GH_TOKEN", "") + catalog, err := updateCatalogFromGitHubInventories(repo, map[string]string{ + selfManagedStackKey: releases[selfManagedStackKey].Version, + computePlaneStackKey: releases[computePlaneStackKey].Version, + observabilityStackKey: releases[observabilityStackKey].Version, + }, "", testCatalog()) + if err != nil { + t.Fatal(err) + } + if catalog.Stack.Version != "1.2.3" { + t.Fatalf("self-managed stack version = %s, want 1.2.3", catalog.Stack.Version) + } + if catalog.ReleaseSet.Status != ReleaseSetDevelopment || catalog.ReleaseSet.DocumentationVersion != "dev" { + t.Fatalf("release set = %#v, want development docs", catalog.ReleaseSet) + } + if catalog.ReleaseSet.Stacks.ControlPlane.Version != "1.2.3" || + catalog.ReleaseSet.Stacks.ComputePlane.Version != "2.3.4" || + catalog.ReleaseSet.Stacks.Observability.Version != "3.4.5" { + t.Fatalf("release set stack versions = %#v", catalog.ReleaseSet.Stacks) + } + for name, want := range map[string]string{ + computeStackResourceName: "2.3.4", + observabilityStackResourceName: "3.4.5", + } { + artifact, found := catalog.findArtifactByNameAndType(name, ArtifactTypeResource) + if !found || artifact.Version != want { + t.Fatalf("%s = %#v, want version %s", name, artifact, want) + } + } +} + +func filterResolvedInventoryByPlane(t *testing.T, source resolvedStackInventory, release stackSourceRelease, plane string) resolvedStackInventory { + t.Helper() + filtered := resolvedStackInventory{SchemaVersion: resolvedStackInventorySchemaVersion, Source: release} + for _, candidate := range source.Releases { + if candidate.Plane == plane { + filtered.Releases = append(filtered.Releases, candidate) + } + } + for _, artifact := range source.Artifacts { + if len(artifact.Sources) == 1 && artifact.Sources[0].Plane == plane { + filtered.Artifacts = append(filtered.Artifacts, artifact) + } + } + if err := validateResolvedStackInventory(filtered); err != nil { + t.Fatalf("filtered %s inventory is invalid: %v", plane, err) + } + return filtered +} + func TestBuildCatalogFromResolvedInventoryKeepsPublicationAvailabilityIndependent(t *testing.T) { source := stackSourceRelease{Version: "1.2.3", Tag: stackTagPrefix + "1.2.3", Commit: strings.Repeat("a", 40)} inventory := testCatalogResolvedInventory(t, source) @@ -93,7 +198,7 @@ func TestBuildCatalogFromResolvedInventoryKeepsPublicationAvailabilityIndependen {Name: "nvca", Type: ArtifactTypeChart, Version: "6.7.8", Registry: "public-helm", ChartFormat: ChartFormatHTTP}, {Name: "pylon", Type: ArtifactTypeImage, Version: "3.4.5", Registry: "private-images"}, {Name: "independent-chart", Type: ArtifactTypeChart, Version: "2.3.4", Registry: "public-helm", ChartFormat: ChartFormatHTTP}, - {Name: computeStackResourceName, Type: ArtifactTypeResource, Version: source.Version, Registry: "public-resources"}, + {Name: computeStackResourceName, Type: ArtifactTypeResource, Version: "0.5.0", Registry: "public-resources"}, } catalog, err := buildCatalogFromResolvedStackInventory(inventory, snapshot, base) @@ -169,8 +274,8 @@ func TestBuildCatalogFromResolvedInventoryKeepsPublicationAvailabilityIndependen t.Fatal("compute stack has an exact publication but is marked pending") } observability, ok := catalog.findArtifact(observabilityStackResourceName) - if !ok || observability.Version != source.Version { - t.Fatalf("observability stack = %#v, want release version %s", observability, source.Version) + if !ok || observability.Version != "0.5.0" { + t.Fatalf("observability stack = %#v, want independently owned version 0.5.0", observability) } if !catalog.publicationIsPending(observability) { t.Fatal("observability stack without an exact publication is not marked pending") @@ -180,6 +285,96 @@ func TestBuildCatalogFromResolvedInventoryKeepsPublicationAvailabilityIndependen } } +func TestRetainIndependentManifestArtifactsUsesInventoryPlaneOwnership(t *testing.T) { + newCatalog := func() *Catalog { + return &Catalog{ + Registries: map[string]Registry{ + defaultStackRegistry: {}, + defaultImageRegistry: {}, + defaultChartRegistry: {}, + }, + SupplementalArtifacts: []Artifact{ + {Name: "stale-control", Type: ArtifactTypeImage, Registry: "old", Version: "1.0.0"}, + {Name: "independent-compute", Type: ArtifactTypeImage, Registry: "old", Version: "2.0.0"}, + {Name: "stale-observability", Type: ArtifactTypeImage, Registry: "old", Version: "2.5.0"}, + {Name: "unclassified", Type: ArtifactTypeImage, Registry: "old", Version: "3.0.0"}, + {Name: "independent-resource", Type: ArtifactTypeResource, Registry: "old", Version: "4.0.0"}, + }, + Manifest: ManifestMetadata{Entries: []ManifestEntry{ + {ArtifactID: "stale-control", Plane: ManifestPlaneControl, Kind: ManifestKindServiceImage, Requirement: ManifestRequired, Description: "Control artifact."}, + {ArtifactID: "independent-compute", Plane: ManifestPlaneCompute, Kind: ManifestKindServiceImage, Requirement: ManifestRequired, Description: "Compute artifact."}, + {ArtifactID: "stale-observability", Plane: ManifestPlaneObservability, Kind: ManifestKindServiceImage, Requirement: ManifestRequired, Description: "Observability artifact."}, + {ArtifactID: "independent-resource", Plane: ManifestPlaneShared, Kind: ManifestKindResource, Description: "Independent resource."}, + }}, + } + } + + t.Run("legacy control and observability inventory", func(t *testing.T) { + catalog := newCatalog() + base := newCatalog() + base.Artifacts = append([]Artifact(nil), base.SupplementalArtifacts...) + base.SupplementalArtifacts = nil + inventory := resolvedStackInventory{Releases: []resolvedInventoryRelease{ + {Plane: "control-plane", Name: "api"}, + {Plane: "observability", Name: "collector"}, + }} + retainIndependentManifestArtifacts(catalog, inventory, base) + + if _, found := catalog.findArtifact("stale-control"); found { + t.Fatal("control artifact absent from its owning inventory was retained") + } + compute, found := catalog.findArtifact("independent-compute") + if !found || compute.Registry != defaultImageRegistry { + t.Fatalf("independent compute artifact = %#v, want retained with public image registry", compute) + } + if _, found := catalog.findArtifact("unclassified"); found { + t.Fatal("unclassified artifact was retained") + } + resource, found := catalog.findArtifact("independent-resource") + if !found || resource.Registry != defaultStackRegistry { + t.Fatalf("independent resource = %#v, want retained with public resource registry", resource) + } + }) + + t.Run("observability-only inventory", func(t *testing.T) { + catalog := newCatalog() + base := newCatalog() + base.Artifacts = append([]Artifact(nil), base.SupplementalArtifacts...) + base.SupplementalArtifacts = nil + inventory := resolvedStackInventory{Releases: []resolvedInventoryRelease{ + {Plane: "observability", Name: "collector"}, + }} + retainIndependentManifestArtifacts(catalog, inventory, base) + + if _, found := catalog.findArtifact("stale-observability"); found { + t.Fatal("observability artifact absent from its owning inventory was retained") + } + if control, found := catalog.findArtifact("stale-control"); !found || control.Registry != defaultImageRegistry { + t.Fatalf("independent control artifact = %#v, want retained with public image registry", control) + } + }) + + t.Run("legacy aggregate inventory", func(t *testing.T) { + catalog := newCatalog() + base := newCatalog() + base.Artifacts = append([]Artifact(nil), base.SupplementalArtifacts...) + base.SupplementalArtifacts = nil + inventory := resolvedStackInventory{Releases: []resolvedInventoryRelease{ + {Plane: "control-plane", Name: "api"}, + {Plane: "compute-plane", Name: "nvca"}, + {Plane: "observability", Name: "collector"}, + }} + retainIndependentManifestArtifacts(catalog, inventory, base) + + if _, found := catalog.findArtifact("independent-compute"); found { + t.Fatal("compute artifact absent from a legacy aggregate inventory was retained") + } + if _, found := catalog.findArtifact("independent-resource"); !found { + t.Fatal("independent resource was not retained") + } + }) +} + func TestStackResourceVersionUpdateIgnoresSameNameImage(t *testing.T) { catalog := testCatalog() catalog.Artifacts = append(catalog.Artifacts, Artifact{ @@ -343,10 +538,8 @@ func testCatalogResolvedInventory(t *testing.T, source stackSourceRelease) resol func testCatalogPinSources() map[string]string { return map[string]string{ - "deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl": " - name: llm-request-router\n version: 1.2.3\n - name: ingress\n version: 2.3.4\n", - "deploy/stacks/self-managed/global.yaml.gotmpl": "image: nvcr.io/nvidia/nvcf/pylon:3.4.5\n", - "deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl": " - name: nvca-operator\n version: 4.5.6\n", - "deploy/stacks/nvcf-compute-plane/environments/base.yaml": " nvcaOperator:\n selfManaged:\n nvcaVersion: \"6.7.8\"\n", + "deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl": " - name: llm-request-router\n version: 1.2.3\n - name: ingress\n version: 2.3.4\n", + "deploy/stacks/self-managed/global.yaml.gotmpl": "image: nvcr.io/nvidia/nvcf/pylon:3.4.5\n", } } diff --git a/tools/docs-version-sync/github.go b/tools/docs-version-sync/github.go index 36e34bd4d4..c4795b73e8 100644 --- a/tools/docs-version-sync/github.go +++ b/tools/docs-version-sync/github.go @@ -107,11 +107,15 @@ func newGitHubClientFromEnvironment() *githubClient { // resolveStackSourceRelease selects the latest stable release or an explicit stack source ref. func (client *githubClient) resolveStackSourceRelease(sourceRef string) (stackSourceRelease, error) { + return client.resolveStackSourceReleaseForSpec(stackInventorySpecs[0], sourceRef) +} + +func (client *githubClient) resolveStackSourceReleaseForSpec(spec stackInventorySpec, sourceRef string) (stackSourceRelease, error) { wantedRef := "" wantedVersion := "" if strings.TrimSpace(sourceRef) != "" { var err error - wantedRef, wantedVersion, err = normalizeStackSourceRef(sourceRef) + wantedRef, wantedVersion, err = normalizeStackSourceRefForSpec(spec, sourceRef) if err != nil { return stackSourceRelease{}, err } @@ -129,7 +133,7 @@ func (client *githubClient) resolveStackSourceRelease(sourceRef string) (stackSo return stackSourceRelease{Version: wantedVersion, Tag: strings.TrimPrefix(wantedRef, "refs/tags/"), Commit: commit}, nil } - refs, err := client.stackRefs() + refs, err := client.stackRefsForSpec(spec) if err != nil { return stackSourceRelease{}, err } @@ -138,7 +142,7 @@ func (client *githubClient) resolveStackSourceRelease(sourceRef string) (stackSo selectedVersion := stableStackVersion{} selectedStackVersion := "" for i := range refs { - version, parsed, ok := parseStableStackRef(refs[i].Ref) + version, parsed, ok := parseStableStackRefForSpec(spec, refs[i].Ref) if !ok { continue } @@ -149,7 +153,7 @@ func (client *githubClient) resolveStackSourceRelease(sourceRef string) (stackSo } } if selected == -1 { - return stackSourceRelease{}, fmt.Errorf("no stable %sX.Y.Z stack source refs found in GitHub", stackTagPrefix) + return stackSourceRelease{}, fmt.Errorf("no stable %sX.Y.Z stack source refs found in GitHub", spec.TagPrefix) } commit, err := client.resolveCommit(refs[selected].Object) if err != nil { @@ -164,7 +168,11 @@ func (client *githubClient) resolveStackSourceRelease(sourceRef string) (stackSo // resolvedStackInventory downloads and validates the inventory attached to a selected release. func (client *githubClient) resolvedStackInventory(source stackSourceRelease) (resolvedStackInventory, error) { - if err := validateStackSourceRelease(source); err != nil { + return client.resolvedStackInventoryForSpec(stackInventorySpecs[0], source) +} + +func (client *githubClient) resolvedStackInventoryForSpec(spec stackInventorySpec, source stackSourceRelease) (resolvedStackInventory, error) { + if err := validateStackSourceReleaseForSpec(source, spec); err != nil { return resolvedStackInventory{}, err } path := fmt.Sprintf( @@ -183,25 +191,25 @@ func (client *githubClient) resolvedStackInventory(source stackSourceRelease) (r assetURL := "" for _, asset := range release.Assets { - if asset.Name != resolvedStackInventoryAssetName { + if asset.Name != spec.AssetName { continue } if assetURL != "" { - return resolvedStackInventory{}, fmt.Errorf("GitHub release %s has more than one %s asset", source.Tag, resolvedStackInventoryAssetName) + return resolvedStackInventory{}, fmt.Errorf("GitHub release %s has more than one %s asset", source.Tag, spec.AssetName) } assetURL = asset.BrowserDownloadURL } if assetURL == "" { - return resolvedStackInventory{}, fmt.Errorf("GitHub release %s has no %s asset", source.Tag, resolvedStackInventoryAssetName) + return resolvedStackInventory{}, fmt.Errorf("GitHub release %s has no %s asset", source.Tag, spec.AssetName) } raw, err := client.downloadPublicAsset(assetURL) if err != nil { - return resolvedStackInventory{}, fmt.Errorf("download %s from GitHub release %s: %w", resolvedStackInventoryAssetName, source.Tag, err) + return resolvedStackInventory{}, fmt.Errorf("download %s from GitHub release %s: %w", spec.AssetName, source.Tag, err) } inventory, err := parseResolvedStackInventory(raw) if err != nil { - return resolvedStackInventory{}, fmt.Errorf("parse %s from GitHub release %s: %w", resolvedStackInventoryAssetName, source.Tag, err) + return resolvedStackInventory{}, fmt.Errorf("parse %s from GitHub release %s: %w", spec.AssetName, source.Tag, err) } if inventory.Source != source { return resolvedStackInventory{}, fmt.Errorf("resolved inventory source is %+v, want selected release %+v", inventory.Source, source) @@ -229,11 +237,15 @@ func (client *githubClient) stackRef(ref string) (githubRef, error) { // stackRefs reads every GitHub ref with the self-managed stack tag prefix. func (client *githubClient) stackRefs() ([]githubRef, error) { + return client.stackRefsForSpec(stackInventorySpecs[0]) +} + +func (client *githubClient) stackRefsForSpec(spec stackInventorySpec) ([]githubRef, error) { path := fmt.Sprintf( "/repos/%s/%s/git/matching-refs/tags/%s", url.PathEscape(client.owner), url.PathEscape(client.repo), - stackTagPrefix, + spec.TagPrefix, ) var refs []githubRef for page := 1; ; { @@ -406,14 +418,18 @@ func sameURLOrigin(left, right *url.URL) bool { // normalizeStackSourceRef converts a version, tag, or full ref to a canonical tag ref. func normalizeStackSourceRef(sourceRef string) (string, string, error) { + return normalizeStackSourceRefForSpec(stackInventorySpecs[0], sourceRef) +} + +func normalizeStackSourceRefForSpec(spec stackInventorySpec, sourceRef string) (string, string, error) { value := strings.TrimSpace(sourceRef) value = strings.TrimPrefix(value, "refs/tags/") - if !strings.HasPrefix(value, stackTagPrefix) { - value = stackTagPrefix + value + if !strings.HasPrefix(value, spec.TagPrefix) { + value = spec.TagPrefix + value } - version := strings.TrimPrefix(value, stackTagPrefix) + version := strings.TrimPrefix(value, spec.TagPrefix) if !validStackVersion(version) { - return "", "", fmt.Errorf("stack source %q must be a semantic version, %s tag, or refs/tags/%s ref", sourceRef, stackTagPrefix, stackTagPrefix) + return "", "", fmt.Errorf("stack source %q must be a semantic version, %s tag, or refs/tags/%s ref", sourceRef, spec.TagPrefix, spec.TagPrefix) } return "refs/tags/" + value, version, nil } @@ -435,12 +451,22 @@ func validStackVersion(version string) bool { return true } +func validStableStackVersion(version string) bool { + match := stackVersionRE.FindStringSubmatch(version) + return match != nil && match[1] == "" +} + // parseStableStackRef parses a stable stack tag ref for numeric ordering. func parseStableStackRef(ref string) (string, stableStackVersion, bool) { - if !strings.HasPrefix(ref, stackRefPrefix) { + return parseStableStackRefForSpec(stackInventorySpecs[0], ref) +} + +func parseStableStackRefForSpec(spec stackInventorySpec, ref string) (string, stableStackVersion, bool) { + refPrefix := "refs/tags/" + spec.TagPrefix + if !strings.HasPrefix(ref, refPrefix) { return "", stableStackVersion{}, false } - version := strings.TrimPrefix(ref, stackRefPrefix) + version := strings.TrimPrefix(ref, refPrefix) match := stableStackVersionRE.FindStringSubmatch(version) if match == nil { return "", stableStackVersion{}, false diff --git a/tools/docs-version-sync/inventory_compare.go b/tools/docs-version-sync/inventory_compare.go new file mode 100644 index 0000000000..623793398f --- /dev/null +++ b/tools/docs-version-sync/inventory_compare.go @@ -0,0 +1,226 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +type comparableArtifact struct { + Type string + Name string + Reference string + Digest string + Requirement ManifestRequirement + Stacks []string +} + +type comparableInventorySet struct { + Sources []stackSourceRelease + Artifacts map[string]comparableArtifact +} + +func compareInventorySetDirectories(fromDirectory, toDirectory string) (string, error) { + from, err := loadComparableInventorySet(fromDirectory) + if err != nil { + return "", fmt.Errorf("load previous release set: %w", err) + } + to, err := loadComparableInventorySet(toDirectory) + if err != nil { + return "", fmt.Errorf("load current release set: %w", err) + } + return renderInventorySetComparison(from, to), nil +} + +func loadComparableInventorySet(directory string) (comparableInventorySet, error) { + paths, err := filepath.Glob(filepath.Join(directory, "*.json")) + if err != nil { + return comparableInventorySet{}, err + } + if len(paths) == 0 { + return comparableInventorySet{}, fmt.Errorf("no JSON inventories found in %s", directory) + } + sort.Strings(paths) + set := comparableInventorySet{Artifacts: map[string]comparableArtifact{}} + seenSources := map[string]struct{}{} + for _, path := range paths { + raw, err := os.ReadFile(path) + if err != nil { + return comparableInventorySet{}, fmt.Errorf("read %s: %w", path, err) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var inventory resolvedStackInventory + if err := decoder.Decode(&inventory); err != nil { + return comparableInventorySet{}, fmt.Errorf("decode %s: %w", path, err) + } + if err := requireJSONEOF(decoder); err != nil { + return comparableInventorySet{}, fmt.Errorf("decode %s: %w", path, err) + } + if err := validateResolvedStackInventory(inventory); err != nil { + return comparableInventorySet{}, fmt.Errorf("validate %s: %w", path, err) + } + if _, exists := seenSources[inventory.Source.Tag]; !exists { + seenSources[inventory.Source.Tag] = struct{}{} + set.Sources = append(set.Sources, inventory.Source) + } + requirements := make(map[string]bool, len(inventory.Releases)) + for _, release := range inventory.Releases { + requirements[resolvedReleaseKey(release.Plane, release.Name)] = release.Required + } + for _, artifact := range inventory.Artifacts { + candidate, err := comparableArtifactFromResolved(artifact, requirements) + if err != nil { + return comparableInventorySet{}, fmt.Errorf("%s: %w", path, err) + } + key := artifact.Type + "\x00" + artifact.Name + if existing, exists := set.Artifacts[key]; exists { + if existing.Reference != candidate.Reference { + return comparableInventorySet{}, fmt.Errorf("artifact conflict for %s %s: %s and %s", artifact.Type, artifact.Name, existing.Reference, candidate.Reference) + } + candidate.Stacks = mergeSortedStrings(existing.Stacks, candidate.Stacks) + if existing.Requirement == ManifestRequired { + candidate.Requirement = ManifestRequired + } + } + set.Artifacts[key] = candidate + } + } + sort.Slice(set.Sources, func(i, j int) bool { return set.Sources[i].Tag < set.Sources[j].Tag }) + return set, nil +} + +func comparableArtifactFromResolved(artifact resolvedInventoryArtifact, requirements map[string]bool) (comparableArtifact, error) { + stacks := map[string]struct{}{} + requirement := ManifestOptional + for _, source := range artifact.Sources { + stack, err := stackKeyForPlane(source.Plane) + if err != nil { + return comparableArtifact{}, err + } + stacks[stack] = struct{}{} + if resolvedArtifactSourceIsRequired(source, requirements) { + requirement = ManifestRequired + } + } + owners := make([]string, 0, len(stacks)) + for stack := range stacks { + owners = append(owners, stack) + } + sort.Strings(owners) + return comparableArtifact{ + Type: artifact.Type, Name: artifact.Name, Reference: artifact.Reference, + Digest: artifact.Digest, Requirement: requirement, Stacks: owners, + }, nil +} + +func mergeSortedStrings(left, right []string) []string { + values := make(map[string]struct{}, len(left)+len(right)) + for _, value := range left { + values[value] = struct{}{} + } + for _, value := range right { + values[value] = struct{}{} + } + merged := make([]string, 0, len(values)) + for value := range values { + merged = append(merged, value) + } + sort.Strings(merged) + return merged +} + +func renderInventorySetComparison(from, to comparableInventorySet) string { + var added, removed, changed []string + keys := map[string]struct{}{} + for key := range from.Artifacts { + keys[key] = struct{}{} + } + for key := range to.Artifacts { + keys[key] = struct{}{} + } + ordered := make([]string, 0, len(keys)) + for key := range keys { + ordered = append(ordered, key) + } + sort.Strings(ordered) + for _, key := range ordered { + before, hadBefore := from.Artifacts[key] + after, hasAfter := to.Artifacts[key] + switch { + case !hadBefore: + added = append(added, formatComparableArtifact(after)) + case !hasAfter: + removed = append(removed, formatComparableArtifact(before)) + default: + var differences []string + if before.Reference != after.Reference { + differences = append(differences, fmt.Sprintf("reference `%s` -> `%s`", before.Reference, after.Reference)) + } + if before.Digest != after.Digest && before.Reference == after.Reference { + differences = append(differences, fmt.Sprintf("digest `%s` -> `%s`", valueOrNone(before.Digest), valueOrNone(after.Digest))) + } + if before.Requirement != after.Requirement { + differences = append(differences, fmt.Sprintf("requirement `%s` -> `%s`", before.Requirement, after.Requirement)) + } + if strings.Join(before.Stacks, ",") != strings.Join(after.Stacks, ",") { + differences = append(differences, fmt.Sprintf("stacks `%s` -> `%s`", strings.Join(before.Stacks, ", "), strings.Join(after.Stacks, ", "))) + } + if len(differences) > 0 { + changed = append(changed, fmt.Sprintf("- `%s` (%s): %s", after.Name, after.Type, strings.Join(differences, "; "))) + } + } + } + + var report strings.Builder + report.WriteString("# Release set artifact comparison\n\n") + report.WriteString("Previous sources: " + formatComparisonSources(from.Sources) + "\n\n") + report.WriteString("Current sources: " + formatComparisonSources(to.Sources) + "\n\n") + writeComparisonSection(&report, "Added artifacts", added) + writeComparisonSection(&report, "Removed artifacts", removed) + writeComparisonSection(&report, "Changed artifacts", changed) + unchanged := len(from.Artifacts) - len(removed) - len(changed) + if unchanged < 0 { + unchanged = 0 + } + report.WriteString(fmt.Sprintf("Unchanged artifacts: %d\n", unchanged)) + return report.String() +} + +func formatComparableArtifact(artifact comparableArtifact) string { + return fmt.Sprintf("- `%s` (%s) at `%s`, %s, stacks: `%s`", artifact.Name, artifact.Type, artifact.Reference, artifact.Requirement, strings.Join(artifact.Stacks, ", ")) +} + +func formatComparisonSources(sources []stackSourceRelease) string { + formatted := make([]string, len(sources)) + for index, source := range sources { + formatted[index] = fmt.Sprintf("`%s`", source.Tag) + } + return strings.Join(formatted, ", ") +} + +func writeComparisonSection(report *strings.Builder, heading string, entries []string) { + report.WriteString("## " + heading + "\n\n") + if len(entries) == 0 { + report.WriteString("None.\n\n") + return + } + for _, entry := range entries { + report.WriteString(entry + "\n") + } + report.WriteString("\n") +} + +func valueOrNone(value string) string { + if value == "" { + return "none" + } + return value +} diff --git a/tools/docs-version-sync/inventory_compare_test.go b/tools/docs-version-sync/inventory_compare_test.go new file mode 100644 index 0000000000..c727ecffef --- /dev/null +++ b/tools/docs-version-sync/inventory_compare_test.go @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +func TestCompareInventorySetDirectoriesExplainsCustomerArtifactChanges(t *testing.T) { + fromDirectory := t.TempDir() + toDirectory := t.TempDir() + writeComparisonInventory(t, fromDirectory, "old.json", comparisonInventory("1.0.0", []resolvedInventoryArtifact{ + comparisonArtifact("removed", "1.0.0", false), + comparisonArtifact("changed", "1.0.0", true), + })) + writeComparisonInventory(t, toDirectory, "new.json", comparisonInventory("2.0.0", []resolvedInventoryArtifact{ + comparisonArtifact("added", "1.0.0", false), + comparisonArtifact("changed", "2.0.0", false), + })) + + report, err := compareInventorySetDirectories(fromDirectory, toDirectory) + if err != nil { + t.Fatal(err) + } + for _, expected := range []string{ + "## Added artifacts\n\n- `added`", + "## Removed artifacts\n\n- `removed`", + "`changed` (container-image): reference", + "requirement `required` -> `optional`", + } { + if !strings.Contains(report, expected) { + t.Fatalf("comparison report missing %q:\n%s", expected, report) + } + } +} + +func TestCompareInventorySetDirectoriesDetectsDependencyDisappearance(t *testing.T) { + fromDirectory := t.TempDir() + toDirectory := t.TempDir() + writeComparisonInventory(t, fromDirectory, "old.json", comparisonInventory("1.0.0", []resolvedInventoryArtifact{ + comparisonArtifact("dependency", "1.0.0", true), + })) + writeComparisonInventory(t, toDirectory, "new.json", comparisonInventory("2.0.0", []resolvedInventoryArtifact{ + comparisonArtifact("replacement", "1.0.0", true), + })) + + report, err := compareInventorySetDirectories(fromDirectory, toDirectory) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(report, "## Removed artifacts\n\n- `dependency`") { + t.Fatalf("removed dependency was not reported:\n%s", report) + } +} + +func comparisonInventory(version string, artifacts []resolvedInventoryArtifact) resolvedStackInventory { + requiredRelease := resolvedInventoryRelease{ + Plane: "control-plane", Name: "api", Required: true, + Chart: "https://helm.example.test/api", Version: "1.0.0", + } + optionalRelease := resolvedInventoryRelease{ + Plane: "control-plane", Name: "addon", Required: false, + Chart: "https://helm.example.test/addon", Version: "1.0.0", + } + requiredChart := resolvedInventoryArtifact{ + Type: "helm-chart", Name: "api", Repository: "https://helm.example.test", Version: "1.0.0", + Reference: "https://helm.example.test/api@1.0.0", + Sources: []resolvedArtifactSource{{Plane: "control-plane", Release: "api"}}, + } + optionalChart := resolvedInventoryArtifact{ + Type: "helm-chart", Name: "addon", Repository: "https://helm.example.test", Version: "1.0.0", + Reference: "https://helm.example.test/addon@1.0.0", + Sources: []resolvedArtifactSource{{Plane: "control-plane", Release: "addon"}}, + } + artifacts = append(artifacts, requiredChart, optionalChart) + sortResolvedInventoryArtifacts(artifacts) + return resolvedStackInventory{ + SchemaVersion: resolvedStackInventorySchemaVersion, + Source: stackSourceRelease{ + Version: version, Tag: stackTagPrefix + version, Commit: strings.Repeat("a", 40), + }, + Releases: []resolvedInventoryRelease{optionalRelease, requiredRelease}, + Artifacts: artifacts, + } +} + +func comparisonArtifact(name, version string, required bool) resolvedInventoryArtifact { + release := "addon" + if required { + release = "api" + } + return resolvedInventoryArtifact{ + Type: "container-image", Name: name, Repository: "registry.example.test/" + name, Version: version, + Reference: "registry.example.test/" + name + ":" + version, + Sources: []resolvedArtifactSource{{Plane: "control-plane", Release: release}}, + } +} + +func sortResolvedInventoryArtifacts(artifacts []resolvedInventoryArtifact) { + sort.Slice(artifacts, func(i, j int) bool { + return compareResolvedInventoryArtifacts(artifacts[i], artifacts[j]) < 0 + }) +} + +func writeComparisonInventory(t *testing.T, directory, name string, inventory resolvedStackInventory) { + t.Helper() + raw, err := json.MarshalIndent(inventory, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, name), append(raw, '\n'), 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/tools/docs-version-sync/main.go b/tools/docs-version-sync/main.go index 17e64ac7da..ad70aa361a 100644 --- a/tools/docs-version-sync/main.go +++ b/tools/docs-version-sync/main.go @@ -40,10 +40,16 @@ func run(args []string) error { check := flags.Bool("check", false, "fail if generated docs differ from checked-in marker blocks") updateCatalog := flags.Bool("update-catalog", false, "fetch the resolved GitHub stack inventory and update the catalog") stackVersion := flags.String("stack-version", "", "self-managed stack version to fetch or inventory") + computeStackVersion := flags.String("compute-stack-version", "", "compute-plane stack version to fetch") + observabilityStackVersion := flags.String("observability-stack-version", "", "observability stack version to fetch") + qualificationVersion := flags.String("qualification-version", "", "documentation version for an exact QA-qualified three-stack release set") inventoryOutput := flags.String("generate-stack-inventory", "", "write a resolved stack inventory to this path") inventoryConfig := flags.String("inventory-config", "", "release inventory config path; defaults to the stack checkout") - stackSourceTag := flags.String("stack-source-tag", "", "immutable self-managed stack source tag for inventory generation") - stackSourceCommit := flags.String("stack-source-commit", "", "immutable self-managed stack source commit for inventory generation") + allowUnavailableSourceCharts := flags.Bool("allow-unavailable-source-charts", false, "use published charts when configured source paths are unavailable in a historical tag") + stackSourceTag := flags.String("stack-source-tag", "", "immutable owning stack source tag for inventory generation") + stackSourceCommit := flags.String("stack-source-commit", "", "immutable owning stack source commit for inventory generation") + compareFrom := flags.String("compare-release-set-from", "", "directory containing previous release-set inventory JSON files") + compareTo := flags.String("compare-release-set-to", "", "directory containing current release-set inventory JSON files") if err := flags.Parse(args); err != nil { return err @@ -59,6 +65,26 @@ func run(args []string) error { if err != nil { return err } + if *compareFrom != "" || *compareTo != "" { + if *compareFrom == "" || *compareTo == "" { + return fmt.Errorf("--compare-release-set-from and --compare-release-set-to must be used together") + } + if *updateCatalog || *check || *inventoryOutput != "" || *qualificationVersion != "" || + *stackVersion != "" || *computeStackVersion != "" || *observabilityStackVersion != "" || + *inventoryConfig != "" || *allowUnavailableSourceCharts || *stackSourceTag != "" || *stackSourceCommit != "" { + return fmt.Errorf("release-set comparison cannot be combined with catalog update, check, or inventory generation flags") + } + fromPath := resolveRepoPath(repoRoot, *compareFrom) + toPath := resolveRepoPath(repoRoot, *compareTo) + report, err := compareInventorySetDirectories(fromPath, toPath) + if err != nil { + return err + } + if _, err := fmt.Print(report); err != nil { + return fmt.Errorf("write release-set comparison: %w", err) + } + return nil + } if *inventoryOutput != "" { if *updateCatalog || *check { return fmt.Errorf("--generate-stack-inventory cannot be combined with --update-catalog or --check") @@ -78,10 +104,26 @@ func run(args []string) error { Version: *stackVersion, Tag: *stackSourceTag, Commit: *stackSourceCommit, + }, resolvedInventoryGenerationOptions{ + AllowUnavailableSourceCharts: *allowUnavailableSourceCharts, }) } - if *stackSourceTag != "" || *stackSourceCommit != "" || *inventoryConfig != "" { - return fmt.Errorf("--stack-source-tag, --stack-source-commit, and --inventory-config require --generate-stack-inventory") + if !*updateCatalog && (*computeStackVersion != "" || *observabilityStackVersion != "") { + return fmt.Errorf("--compute-stack-version and --observability-stack-version require --update-catalog") + } + if *qualificationVersion != "" { + if !*updateCatalog { + return fmt.Errorf("--qualification-version requires --update-catalog") + } + if !validStableStackVersion(strings.TrimPrefix(*qualificationVersion, "v")) { + return fmt.Errorf("--qualification-version must be a stable semantic version") + } + if *stackVersion == "" || *computeStackVersion == "" || *observabilityStackVersion == "" { + return fmt.Errorf("--qualification-version requires exact control-plane, compute-plane, and observability stack versions") + } + } + if *stackSourceTag != "" || *stackSourceCommit != "" || *inventoryConfig != "" || *allowUnavailableSourceCharts { + return fmt.Errorf("--stack-source-tag, --stack-source-commit, --inventory-config, and --allow-unavailable-source-charts require --generate-stack-inventory") } if *catalogPath == "" { *catalogPath = filepath.Join(repoRoot, "docs", "version-catalog", *target+".yaml") @@ -96,7 +138,11 @@ func run(args []string) error { if err != nil { return err } - updated, err := updateCatalogFromGitHub(repoRoot, *stackVersion, base) + updated, err := updateCatalogFromGitHubInventories(repoRoot, map[string]string{ + selfManagedStackKey: *stackVersion, + computePlaneStackKey: *computeStackVersion, + observabilityStackKey: *observabilityStackVersion, + }, *qualificationVersion, base) if err != nil { return err } @@ -137,6 +183,13 @@ func run(args []string) error { return nil } +func resolveRepoPath(repoRoot, path string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(repoRoot, path) +} + func writeCatalogAfterStackSourceValidation(repoRoot, catalogPath string, catalog *Catalog) error { if catalog.Stack.SourceCommit == "" { return fmt.Errorf("cannot write updated catalog for stack release %s without an immutable source snapshot", catalog.Stack.Version) diff --git a/tools/docs-version-sync/main_test.go b/tools/docs-version-sync/main_test.go index 6733032eda..61bcd70d4a 100644 --- a/tools/docs-version-sync/main_test.go +++ b/tools/docs-version-sync/main_test.go @@ -31,13 +31,13 @@ func TestRenderManifestArtifactRegistryPaths(t *testing.T) { } wantLines := []string{ - "| Artifact | Version | Required | Description | Distribution | Source code |", - "| `llm-api-gateway` | `0.3.0` | Optional |", - "| `llm-request-router` | `0.2.0` | Optional |", - "| `nvcf-self-managed-stack` | `0.5.0` |", - "| `nvcf-compute-plane-stack` | `0.5.0` |", - "| `nvcf-observability-stack` | `0.5.0` |", - "| `nvcf-cli` | `0.0.30` |", + "| Artifact | Version | Stack | Required | Description | Distribution | Source code |", + "| `llm-api-gateway` | `0.3.0` | `self-managed` | Optional |", + "| `llm-request-router` | `0.2.0` | `self-managed` | Optional |", + "| `nvcf-self-managed-stack` | `0.5.0` | `self-managed` |", + "| `nvcf-compute-plane-stack` | `0.5.0` | `compute-plane` |", + "| `nvcf-observability-stack` | `0.5.0` | `observability` |", + "| `nvcf-cli` | `0.0.30` | Independent |", } for _, want := range wantLines { if !strings.Contains(got, want) { @@ -73,22 +73,22 @@ func TestRenderManifestHandlesNewNVCAAndNVCTImageAndHelmArtifacts(t *testing.T) computeServices := sectionBetween(t, got, "### Compute plane services and images", "### EA-only CVE-impacted artifacts") for _, want := range []string{ - "| `nvca` | `3.0.0-rc.13` | Required |", - "| `nvca-operator` | `3.0.0-rc.13` | Required |", + "| `nvca` | `3.0.0-rc.13` | `compute-plane` | Required |", + "| `nvca-operator` | `3.0.0-rc.13` | `compute-plane` | Required |", } { if !strings.Contains(computeServices, want) { t.Fatalf("compute services section missing %q:\n%s", want, computeServices) } } computeCharts := sectionBetween(t, got, "### Compute plane Helm charts", "### Compute plane services and images") - if !strings.Contains(computeCharts, "| `helm-nvca-operator` | `1.11.1` | Required |") { + if !strings.Contains(computeCharts, "| `helm-nvca-operator` | `1.11.1` | `compute-plane` | Required |") { t.Fatalf("compute charts section missing NVCA chart:\n%s", computeCharts) } controlPlane := sectionBetween(t, got, "### Control plane Helm charts", "### Compute plane Helm charts") for _, want := range []string{ - "| `nvct-service-oss` | `1.2.11` | Required |", - "| `helm-nvcf-nvct-api` | `1.4.2` | Required |", + "| `nvct-service-oss` | `1.2.11` | `self-managed` | Required |", + "| `helm-nvcf-nvct-api` | `1.4.2` | `self-managed` | Required |", } { if !strings.Contains(controlPlane, want) { t.Fatalf("control plane section missing %q:\n%s", want, controlPlane) @@ -904,6 +904,23 @@ func TestValidateTargetRejectsNonMainTargets(t *testing.T) { } } +func TestRunRejectsPrereleaseQualificationVersion(t *testing.T) { + err := run([]string{"--update-catalog", "--qualification-version", "v1.2.3-rc.1"}) + if err == nil || !strings.Contains(err.Error(), "must be a stable semantic version") { + t.Fatalf("run error = %v, want prerelease qualification rejection", err) + } +} + +func TestValidateCatalogRejectsPartiallyPopulatedReleaseSet(t *testing.T) { + catalog := testCatalog() + catalog.ReleaseSet.Status = ReleaseSetDevelopment + + err := ValidateCatalog(catalog) + if err == nil || !strings.Contains(err.Error(), "documentation_version must be non-empty") { + t.Fatalf("ValidateCatalog error = %v, want partial release_set rejection", err) + } +} + func testCatalog() *Catalog { return &Catalog{ Version: 1, diff --git a/tools/docs-version-sync/manifest.go b/tools/docs-version-sync/manifest.go index 3a4183cf5f..6ad3f6b327 100644 --- a/tools/docs-version-sync/manifest.go +++ b/tools/docs-version-sync/manifest.go @@ -32,6 +32,7 @@ type resolvedManifestEntry struct { Description string GitHubURL string UpstreamURL string + Stacks []string } type manifestSection struct { @@ -46,6 +47,10 @@ var manifestSections = []manifestSection{ {Heading: "Control plane services and images", Plane: ManifestPlaneControl, Kind: ManifestKindServiceImage}, {Heading: "Compute plane Helm charts", Plane: ManifestPlaneCompute, Kind: ManifestKindChart}, {Heading: "Compute plane services and images", Plane: ManifestPlaneCompute, Kind: ManifestKindServiceImage}, + {Heading: "Observability Helm charts", Plane: ManifestPlaneObservability, Kind: ManifestKindChart}, + {Heading: "Observability services and images", Plane: ManifestPlaneObservability, Kind: ManifestKindServiceImage}, + {Heading: "Cross-stack Helm charts", Plane: ManifestPlaneShared, Kind: ManifestKindChart}, + {Heading: "Cross-stack services and images", Plane: ManifestPlaneShared, Kind: ManifestKindServiceImage}, { Heading: "EA-only CVE-impacted artifacts", Description: "These Early Access artifacts have known CVE impact. Use only the QA-qualified versions listed for this EA stack.", @@ -95,6 +100,16 @@ func resolveManifestEntries(catalog *Catalog) ([]resolvedManifestEntry, error) { entry.Name = artifact.Name entry.Version = artifact.Version entry.Distribution = path + entry.Stacks = append([]string(nil), artifact.Stacks...) + if len(entry.Stacks) == 0 { + entry.Stacks = inferredManifestStacks(metadata, artifact) + } + if plane := manifestPlaneForStacks(entry.Stacks); plane != "" { + entry.Plane = plane + } + if artifact.Requirement != "" { + entry.Requirement = artifact.Requirement + } classified[metadata.ArtifactID] = struct{}{} } if entry.Name == "load_tester_supreme" { @@ -116,12 +131,67 @@ func resolveManifestEntries(catalog *Catalog) ([]resolvedManifestEntry, error) { return entries, nil } +func manifestPlaneForStacks(stacks []string) ManifestPlane { + if len(stacks) != 1 { + if len(stacks) > 1 { + return ManifestPlaneShared + } + return "" + } + switch stacks[0] { + case selfManagedStackKey: + return ManifestPlaneControl + case computePlaneStackKey: + return ManifestPlaneCompute + case observabilityStackKey: + return ManifestPlaneObservability + default: + return "" + } +} + +func inferredManifestStacks(metadata ManifestEntry, artifact Artifact) []string { + switch artifact.Name { + case controlStackResourceName: + return []string{selfManagedStackKey} + case computeStackResourceName: + return []string{computePlaneStackKey} + case observabilityStackResourceName: + return []string{observabilityStackKey} + } + switch metadata.Plane { + case ManifestPlaneControl: + return []string{selfManagedStackKey} + case ManifestPlaneCompute: + return []string{computePlaneStackKey} + case ManifestPlaneObservability: + return []string{observabilityStackKey} + } + return nil +} + func renderManifestArtifactRegistryPaths(catalog *Catalog) (string, error) { entries, err := resolveManifestEntries(catalog) if err != nil { return "", err } - return renderManifestTables(entries), nil + return renderReleaseSetSummary(catalog.ReleaseSet) + renderManifestTables(entries), nil +} + +func renderReleaseSetSummary(releaseSet ReleaseSetMetadata) string { + if releaseSet.DocumentationVersion == "" { + return "" + } + return fmt.Sprintf("### Stack release set\n\nDocumentation: `%s` (%s)\n\n| Stack | Version | Source tag |\n| --- | --- | --- |\n| Control plane | `%s` | `%s` |\n| Compute plane | `%s` | `%s` |\n| Observability | `%s` | `%s` |\n\n", + releaseSet.DocumentationVersion, + releaseSet.Status, + releaseSet.Stacks.ControlPlane.Version, + releaseSet.Stacks.ControlPlane.SourceTag, + releaseSet.Stacks.ComputePlane.Version, + releaseSet.Stacks.ComputePlane.SourceTag, + releaseSet.Stacks.Observability.Version, + releaseSet.Stacks.Observability.SourceTag, + ) } func renderManifestTables(entries []resolvedManifestEntry) string { @@ -133,18 +203,18 @@ func renderManifestTables(entries []resolvedManifestEntry) string { } sectionEntries := manifestEntriesForSection(entries, section) if section.Kind == ManifestKindResource { - b.WriteString("| Artifact | Version | Description | Distribution | Source code |\n") - b.WriteString("| --- | --- | --- | --- | --- |\n") + b.WriteString("| Artifact | Version | Stack | Description | Distribution | Source code |\n") + b.WriteString("| --- | --- | --- | --- | --- | --- |\n") for _, entry := range sectionEntries { - b.WriteString(fmt.Sprintf("| `%s` | `%s` | %s | `%s` | %s |\n", - entry.Name, entry.Version, entry.Description, entry.Distribution, formatManifestSources(entry))) + b.WriteString(fmt.Sprintf("| `%s` | `%s` | %s | %s | `%s` | %s |\n", + entry.Name, entry.Version, formatManifestStacks(entry.Stacks), entry.Description, entry.Distribution, formatManifestSources(entry))) } } else { - b.WriteString("| Artifact | Version | Required | Description | Distribution | Source code |\n") - b.WriteString("| --- | --- | --- | --- | --- | --- |\n") + b.WriteString("| Artifact | Version | Stack | Required | Description | Distribution | Source code |\n") + b.WriteString("| --- | --- | --- | --- | --- | --- | --- |\n") for _, entry := range sectionEntries { - b.WriteString(fmt.Sprintf("| `%s` | `%s` | %s | %s | `%s` | %s |\n", - entry.Name, entry.Version, formatManifestRequirement(entry.Requirement), entry.Description, + b.WriteString(fmt.Sprintf("| `%s` | `%s` | %s | %s | %s | `%s` | %s |\n", + entry.Name, entry.Version, formatManifestStacks(entry.Stacks), formatManifestRequirement(entry.Requirement), entry.Description, entry.Distribution, formatManifestSources(entry))) } } @@ -153,6 +223,17 @@ func renderManifestTables(entries []resolvedManifestEntry) string { return b.String() } +func formatManifestStacks(stacks []string) string { + if len(stacks) == 0 { + return "Independent" + } + formatted := make([]string, len(stacks)) + for index, stack := range stacks { + formatted[index] = "`" + stack + "`" + } + return strings.Join(formatted, " / ") +} + func manifestEntriesForSection(entries []resolvedManifestEntry, section manifestSection) []resolvedManifestEntry { var matched []resolvedManifestEntry for _, entry := range entries { diff --git a/tools/docs-version-sync/manifest_test.go b/tools/docs-version-sync/manifest_test.go index 4b2410cdde..b375bdd604 100644 --- a/tools/docs-version-sync/manifest_test.go +++ b/tools/docs-version-sync/manifest_test.go @@ -103,7 +103,7 @@ func TestRenderManifestTable(t *testing.T) { } for _, want := range []string{ - "| Artifact | Version | Required | Description | Distribution | Source code |", + "| Artifact | Version | Stack | Required | Description | Distribution | Source code |", "These Early Access artifacts have known CVE impact.", "[GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nats)", "[Upstream](https://github.com/nats-io/k8s)", diff --git a/tools/docs-version-sync/render.go b/tools/docs-version-sync/render.go index e45a7f88ba..7948fd98e4 100644 --- a/tools/docs-version-sync/render.go +++ b/tools/docs-version-sync/render.go @@ -206,6 +206,7 @@ func (catalog *Catalog) stackArtifact() Artifact { Type: ArtifactTypeResource, Registry: catalog.Stack.Registry, Version: catalog.Stack.Version, + Stacks: []string{selfManagedStackKey}, } } diff --git a/tools/docs-version-sync/resolved_inventory.go b/tools/docs-version-sync/resolved_inventory.go index 9fd36a487f..58ac4ab9c3 100644 --- a/tools/docs-version-sync/resolved_inventory.go +++ b/tools/docs-version-sync/resolved_inventory.go @@ -21,6 +21,7 @@ const resolvedStackInventorySchemaVersion = 1 var ( resolvedStackPlanes = []string{"compute-plane", "control-plane", "observability"} + imageArgumentRe = regexp.MustCompile(`^--[A-Za-z0-9][A-Za-z0-9_.-]*-image=(.*)$`) imageDigestRe = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_+.-]*:[A-Za-z0-9=_+.-]+$`) ) @@ -29,6 +30,7 @@ type resolvedStackInventory struct { Source stackSourceRelease `json:"source"` Releases []resolvedInventoryRelease `json:"releases"` Artifacts []resolvedInventoryArtifact `json:"artifacts"` + Warnings []string `json:"warnings,omitempty"` } type resolvedInventoryRelease struct { @@ -51,8 +53,9 @@ type resolvedInventoryArtifact struct { } type resolvedArtifactSource struct { - Plane string `json:"plane"` - Release string `json:"release"` + Plane string `json:"plane"` + Release string `json:"release"` + Requirement ManifestRequirement `json:"requirement,omitempty"` } type resolvedInventoryPlaneInput struct { @@ -77,6 +80,7 @@ type parsedImageReference struct { Version string Digest string Reference string + Indirect bool } // generateResolvedStackInventory builds an inventory from Helmfile list output and manifests @@ -106,8 +110,14 @@ func generateResolvedStackInventory(source stackSourceRelease, planes []resolved } for _, release := range releases { - sourceRef := resolvedArtifactSource{Plane: plane.Name, Release: release.Name} - chartArtifact, err := resolvedChartArtifact(release, sourceRef) + releaseRequirement := ManifestOptional + if release.Enabled && release.Installed { + releaseRequirement = ManifestRequired + } + chartSource := resolvedArtifactSource{ + Plane: plane.Name, Release: release.Name, Requirement: releaseRequirement, + } + chartArtifact, err := resolvedChartArtifact(release, chartSource) if err != nil { return resolvedStackInventory{}, fmt.Errorf("resolve chart for %s release %s: %w", plane.Name, release.Name, err) } @@ -118,6 +128,10 @@ func generateResolvedStackInventory(source stackSourceRelease, planes []resolved return resolvedStackInventory{}, fmt.Errorf("resolve images for %s release %s: %w", plane.Name, release.Name, err) } for _, image := range images { + imageRequirement := ManifestOptional + if releaseRequirement == ManifestRequired && !image.Indirect { + imageRequirement = ManifestRequired + } mergeResolvedArtifact(artifacts, resolvedInventoryArtifact{ Type: "container-image", Name: image.Name, @@ -125,7 +139,9 @@ func generateResolvedStackInventory(source stackSourceRelease, planes []resolved Version: image.Version, Digest: image.Digest, Reference: image.Reference, - Sources: []resolvedArtifactSource{sourceRef}, + Sources: []resolvedArtifactSource{{ + Plane: plane.Name, Release: release.Name, Requirement: imageRequirement, + }}, }) } @@ -133,7 +149,7 @@ func generateResolvedStackInventory(source stackSourceRelease, planes []resolved Plane: plane.Name, Name: release.Name, Namespace: release.Namespace, - Required: release.Enabled && release.Installed, + Required: releaseRequirement == ManifestRequired, Chart: release.Chart, Version: release.Version, }) @@ -160,14 +176,17 @@ func generateResolvedStackInventory(source stackSourceRelease, planes []resolved } func normalizeResolvedInventoryPlanes(planes []resolvedInventoryPlaneInput) ([]resolvedInventoryPlaneInput, error) { - if len(planes) != len(resolvedStackPlanes) { - return nil, fmt.Errorf("resolved inventory requires planes %s", strings.Join(resolvedStackPlanes, ", ")) + if len(planes) == 0 { + return nil, fmt.Errorf("resolved inventory requires at least one plane") } planes = append([]resolvedInventoryPlaneInput(nil), planes...) sort.Slice(planes, func(i, j int) bool { return planes[i].Name < planes[j].Name }) - for i, want := range resolvedStackPlanes { - if planes[i].Name != want { - return nil, fmt.Errorf("resolved inventory plane %d is %q, want %q", i, planes[i].Name, want) + for i, plane := range planes { + if !isResolvedStackPlane(plane.Name) { + return nil, fmt.Errorf("resolved inventory plane %d is unknown: %q", i, plane.Name) + } + if i > 0 && planes[i-1].Name == plane.Name { + return nil, fmt.Errorf("resolved inventory plane %q is duplicated", plane.Name) } } return planes, nil @@ -312,6 +331,11 @@ func collectResolvedImages(node *yaml.Node, images map[string]parsedImageReferen } images[image.Reference] = image } + if key.Value == "args" || key.Value == "command" { + if err := collectResolvedImageArguments(value, images); err != nil { + return err + } + } if err := collectResolvedImages(value, images); err != nil { return err } @@ -326,6 +350,37 @@ func collectResolvedImages(node *yaml.Node, images map[string]parsedImageReferen return nil } +func collectResolvedImageArguments(node *yaml.Node, images map[string]parsedImageReference) error { + if node.Kind == yaml.ScalarNode { + if reference, ok := resolvedImageArgument(node.Value); ok { + image, err := parseResolvedImageReference(reference) + if err != nil { + return fmt.Errorf("image argument %q: %w", node.Value, err) + } + image.Indirect = true + if existing, exists := images[image.Reference]; exists && !existing.Indirect { + return nil + } + images[image.Reference] = image + } + return nil + } + for _, child := range node.Content { + if err := collectResolvedImageArguments(child, images); err != nil { + return err + } + } + return nil +} + +func resolvedImageArgument(value string) (string, bool) { + matches := imageArgumentRe.FindStringSubmatch(value) + if matches == nil { + return "", false + } + return matches[1], true +} + func isResolvedImageField(key string) bool { return key == "image" || strings.HasPrefix(key, "release-artifact-") && strings.HasSuffix(key, "-image") } @@ -379,12 +434,20 @@ func mergeResolvedArtifact(artifacts map[string]*resolvedInventoryArtifact, cand artifacts[key] = © return } - for _, source := range existing.Sources { - if source == candidate.Sources[0] { - return + mergeResolvedArtifactSource(existing, candidate.Sources[0]) +} + +func mergeResolvedArtifactSource(artifact *resolvedInventoryArtifact, candidate resolvedArtifactSource) { + for index, source := range artifact.Sources { + if source.Plane != candidate.Plane || source.Release != candidate.Release { + continue + } + if candidate.Requirement == ManifestRequired { + artifact.Sources[index].Requirement = ManifestRequired } + return } - existing.Sources = append(existing.Sources, candidate.Sources[0]) + artifact.Sources = append(artifact.Sources, candidate) } func validateResolvedStackInventory(inventory resolvedStackInventory) error { @@ -400,9 +463,16 @@ func validateResolvedStackInventory(inventory resolvedStackInventory) error { if len(inventory.Artifacts) == 0 { return fmt.Errorf("resolved inventory has no artifacts") } + for index, warning := range inventory.Warnings { + if warning == "" || warning != strings.TrimSpace(warning) { + return fmt.Errorf("resolved inventory warning must be non-empty and trimmed") + } + if index > 0 && inventory.Warnings[index-1] >= warning { + return fmt.Errorf("resolved inventory warnings are not uniquely sorted") + } + } releases := make(map[string]resolvedInventoryRelease, len(inventory.Releases)) - planeReleaseCounts := make(map[string]int, len(resolvedStackPlanes)) for i, release := range inventory.Releases { if i > 0 && compareResolvedInventoryReleases(inventory.Releases[i-1], release) >= 0 { return fmt.Errorf("resolved inventory releases are not uniquely sorted") @@ -412,12 +482,6 @@ func validateResolvedStackInventory(inventory resolvedStackInventory) error { } key := resolvedReleaseKey(release.Plane, release.Name) releases[key] = release - planeReleaseCounts[release.Plane]++ - } - for _, plane := range resolvedStackPlanes { - if planeReleaseCounts[plane] == 0 { - return fmt.Errorf("resolved inventory has no %s releases", plane) - } } chartSources := make(map[string]struct{}, len(inventory.Releases)) @@ -442,6 +506,13 @@ func validateResolvedStackInventory(inventory resolvedStackInventory) error { if !ok { return fmt.Errorf("artifact %s references unknown release %s/%s", artifact.Reference, source.Plane, source.Release) } + if source.Requirement != "" && source.Requirement != ManifestRequired && source.Requirement != ManifestOptional { + return fmt.Errorf("artifact %s source %s/%s has unknown requirement %q", artifact.Reference, source.Plane, source.Release, source.Requirement) + } + if artifact.Type == "helm-chart" && source.Requirement != "" && + (source.Requirement == ManifestRequired) != release.Required { + return fmt.Errorf("chart artifact %s source requirement does not match release %s/%s", artifact.Reference, source.Plane, source.Release) + } if artifact.Type == "helm-chart" && artifact.Reference == release.Chart+"@"+release.Version { chartSources[resolvedReleaseKey(source.Plane, source.Release)] = struct{}{} } diff --git a/tools/docs-version-sync/resolved_inventory_publish.go b/tools/docs-version-sync/resolved_inventory_publish.go index c6b34dc00b..fdc34c1707 100644 --- a/tools/docs-version-sync/resolved_inventory_publish.go +++ b/tools/docs-version-sync/resolved_inventory_publish.go @@ -36,54 +36,10 @@ var resolvedInventoryCommonOverrides = []string{ } type resolvedInventoryState struct { - plane string - path string - baseOverrides []string - fullOverrides []string -} - -var resolvedInventoryStates = []resolvedInventoryState{ - { - plane: "control-plane", - path: "deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl", - fullOverrides: []string{ - "addons.llm.enabled=true", - }, - }, - { - plane: "control-plane", - path: "deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl", - fullOverrides: []string{ - "addons.llm.enabled=true", - "addons.vanityGateway.enabled=true", - "addons.nvcfUi.enabled=true", - }, - }, - { - plane: "observability", - path: "deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl", - }, - { - plane: "observability", - path: "deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl", - baseOverrides: []string{ - "observability.profile=all", - }, - }, - { - plane: "compute-plane", - path: "deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl", - fullOverrides: []string{ - "addons.kaiScheduler.enabled=true", - "addons.groveOperator.enabled=true", - "addons.dynamoOperator.enabled=true", - "addons.topologyAwareScheduling.enabled=true", - }, - }, - { - plane: "compute-plane", - path: "deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl", - }, + Plane string `yaml:"plane"` + Path string `yaml:"path"` + BaseOverrides []string `yaml:"baseOverrides,omitempty"` + FullOverrides []string `yaml:"fullOverrides,omitempty"` } type resolvedInventoryCommandRunner interface { @@ -202,6 +158,11 @@ type resolvedInventoryConfig struct { SchemaVersion int `yaml:"schemaVersion"` PublishedChartRepository string `yaml:"publishedChartRepository"` SourceCharts map[string]resolvedInventorySourceChart `yaml:"sourceCharts"` + States []resolvedInventoryState `yaml:"states,omitempty"` +} + +type resolvedInventoryGenerationOptions struct { + AllowUnavailableSourceCharts bool } type resolvedInventorySourceChart struct { @@ -219,11 +180,18 @@ func (source resolvedInventoryHelmSource) reference() string { return source.Registry + "/" + source.Repository } -func writeResolvedStackInventory(repoRoot, outputPath, configPath string, source stackSourceRelease) error { +func writeResolvedStackInventory(repoRoot, outputPath, configPath string, source stackSourceRelease, options resolvedInventoryGenerationOptions) error { if err := verifyResolvedInventoryCheckout(repoRoot, source); err != nil { return err } - inventory, err := collectResolvedStackInventory(repoRoot, configPath, source, execResolvedInventoryCommandRunner{}) + if configPath == "" { + spec, err := stackInventorySpecByTag(source.Tag) + if err != nil { + return err + } + configPath = filepath.Join(repoRoot, filepath.FromSlash(spec.ConfigPath)) + } + inventory, err := collectResolvedStackInventory(repoRoot, configPath, source, options, execResolvedInventoryCommandRunner{}) if err != nil { return err } @@ -265,7 +233,7 @@ func verifyResolvedInventoryCheckout(repoRoot string, source stackSourceRelease) return nil } -func collectResolvedStackInventory(repoRoot, configPath string, source stackSourceRelease, runner resolvedInventoryCommandRunner) (resolvedStackInventory, error) { +func collectResolvedStackInventory(repoRoot, configPath string, source stackSourceRelease, options resolvedInventoryGenerationOptions, runner resolvedInventoryCommandRunner) (resolvedStackInventory, error) { tempRoot, err := os.MkdirTemp("", "nvcf-resolved-stack-inventory-") if err != nil { return resolvedStackInventory{}, err @@ -273,11 +241,15 @@ func collectResolvedStackInventory(repoRoot, configPath string, source stackSour defer os.RemoveAll(tempRoot) copiedRepoRoot := filepath.Join(tempRoot, "repo") - if err := copyResolvedInventoryInputs(repoRoot, copiedRepoRoot); err != nil { + config, err := loadResolvedInventoryConfig(repoRoot, configPath) + if err != nil { return resolvedStackInventory{}, err } - config, err := loadResolvedInventoryConfig(copiedRepoRoot, configPath) - if err != nil { + states := config.States + if len(states) == 0 { + return resolvedStackInventory{}, fmt.Errorf("resolved inventory config must declare at least one stack-owned state") + } + if err := copyResolvedInventoryInputs(repoRoot, copiedRepoRoot, states); err != nil { return resolvedStackInventory{}, err } env, renderSource, ngcAPIKey, err := prepareResolvedInventoryEnvironment(copiedRepoRoot) @@ -294,16 +266,18 @@ func collectResolvedStackInventory(repoRoot, configPath string, source stackSour return resolvedStackInventory{}, fmt.Errorf("authenticate release chart registry: %w", err) } - planes := make(map[string]*resolvedInventoryPlaneInput, len(resolvedStackPlanes)) - for _, name := range resolvedStackPlanes { + planeNames := resolvedInventoryPlaneNames(states) + planes := make(map[string]*resolvedInventoryPlaneInput, len(planeNames)) + for _, name := range planeNames { planes[name] = &resolvedInventoryPlaneInput{ Name: name, ManifestByRelease: map[string][]byte{}, } } usedSourceCharts := map[string]struct{}{} - for stateIndex, state := range resolvedInventoryStates { - stateFile := filepath.Join(copiedRepoRoot, filepath.FromSlash(state.path)) + var coverageWarnings []string + for stateIndex, state := range states { + stateFile := filepath.Join(copiedRepoRoot, filepath.FromSlash(state.Path)) releases, manifests, stateSourceCharts, err := collectResolvedInventoryState( repoRoot, copiedRepoRoot, @@ -315,16 +289,30 @@ func collectResolvedStackInventory(repoRoot, configPath string, source stackSour renderSource, config.PublishedChartRepository, config.SourceCharts, + options.AllowUnavailableSourceCharts, ngcAPIKey, runner, ) if err != nil { - return resolvedStackInventory{}, fmt.Errorf("collect %s: %w", state.path, err) + return resolvedStackInventory{}, fmt.Errorf("collect %s: %w", state.Path, err) } for _, chart := range stateSourceCharts { usedSourceCharts[chart] = struct{}{} } - plane := planes[state.plane] + declared, err := declaredHelmfileReleaseNames(stateFile) + if err != nil { + return resolvedStackInventory{}, err + } + rendered := make(map[string]struct{}, len(releases)) + for _, release := range releases { + rendered[release.Name] = struct{}{} + } + for _, name := range declared { + if _, exists := rendered[name]; !exists { + coverageWarnings = append(coverageWarnings, fmt.Sprintf("%s declares release %s but the full inventory profile did not render it", state.Path, name)) + } + } + plane := planes[state.Plane] var existing []helmfileRelease if len(plane.ReleaseList) != 0 { existing, err = decodeHelmfileReleaseList(plane.ReleaseList) @@ -339,26 +327,103 @@ func collectResolvedStackInventory(repoRoot, configPath string, source stackSour } for release, manifest := range manifests { if _, exists := plane.ManifestByRelease[release]; exists { - return resolvedStackInventory{}, fmt.Errorf("duplicate %s release %s across Helmfile states", state.plane, release) + return resolvedStackInventory{}, fmt.Errorf("duplicate %s release %s across Helmfile states", state.Plane, release) } plane.ManifestByRelease[release] = manifest } } - for chart := range config.SourceCharts { - if _, used := usedSourceCharts[chart]; !used { - return resolvedStackInventory{}, fmt.Errorf("source chart %s is not used by any resolved Helmfile state", chart) + if !options.AllowUnavailableSourceCharts { + for chart := range config.SourceCharts { + if _, used := usedSourceCharts[chart]; !used { + return resolvedStackInventory{}, fmt.Errorf("source chart %s is not used by any resolved Helmfile state", chart) + } } } - inputs := make([]resolvedInventoryPlaneInput, 0, len(resolvedStackPlanes)) - for _, name := range resolvedStackPlanes { + inputs := make([]resolvedInventoryPlaneInput, 0, len(planeNames)) + for _, name := range planeNames { inputs = append(inputs, *planes[name]) } - return generateResolvedStackInventory(source, inputs) + inventory, err := generateResolvedStackInventory(source, inputs) + if err != nil { + return resolvedStackInventory{}, err + } + sort.Strings(coverageWarnings) + inventory.Warnings = coverageWarnings + return inventory, nil } -func copyResolvedInventoryInputs(repoRoot, copiedRepoRoot string) error { - for _, stack := range []string{"self-managed", "nvcf-compute-plane", "observability"} { +var helmfileReleaseNameRE = regexp.MustCompile(`^\s*-\s+name:\s*(?:"([A-Za-z0-9][A-Za-z0-9_.-]*)"|'([A-Za-z0-9][A-Za-z0-9_.-]*)'|([A-Za-z0-9][A-Za-z0-9_.-]*))\s*(?:#.*)?$`) + +func declaredHelmfileReleaseNames(path string) ([]string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read Helmfile state for coverage: %w", err) + } + inReleases := false + seen := map[string]struct{}{} + var names []string + for _, line := range strings.Split(string(raw), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "releases:" { + inReleases = true + continue + } + if !inReleases { + continue + } + if trimmed != "" && !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") && !strings.HasPrefix(trimmed, "#") && !strings.HasPrefix(trimmed, "{{") { + break + } + match := helmfileReleaseNameRE.FindStringSubmatch(line) + if match == nil { + continue + } + name := match[1] + if name == "" { + name = match[2] + } + if name == "" { + name = match[3] + } + if _, exists := seen[name]; exists { + continue + } + seen[name] = struct{}{} + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +func resolvedInventoryPlaneNames(states []resolvedInventoryState) []string { + seen := make(map[string]struct{}, len(states)) + for _, state := range states { + seen[state.Plane] = struct{}{} + } + names := make([]string, 0, len(seen)) + for name := range seen { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func copyResolvedInventoryInputs(repoRoot, copiedRepoRoot string, states []resolvedInventoryState) error { + stacks := make(map[string]struct{}) + for _, state := range states { + parts := strings.Split(filepath.ToSlash(state.Path), "/") + if len(parts) < 4 || parts[0] != "deploy" || parts[1] != "stacks" { + return fmt.Errorf("resolved inventory state path %s is outside deploy/stacks", state.Path) + } + stacks[parts[2]] = struct{}{} + } + stackNames := make([]string, 0, len(stacks)) + for stack := range stacks { + stackNames = append(stackNames, stack) + } + sort.Strings(stackNames) + for _, stack := range stackNames { source := filepath.Join(repoRoot, "deploy", "stacks", stack) destination := filepath.Join(copiedRepoRoot, "deploy", "stacks", stack) if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { @@ -426,6 +491,27 @@ func loadResolvedInventoryConfig(repoRoot, configPath string) (resolvedInventory if config.SchemaVersion != 1 { return resolvedInventoryConfig{}, fmt.Errorf("resolved inventory config schemaVersion is %d, want 1", config.SchemaVersion) } + if len(config.States) > 0 { + seenStates := make(map[string]struct{}, len(config.States)) + for index, state := range config.States { + if !isResolvedStackPlane(state.Plane) { + return resolvedInventoryConfig{}, fmt.Errorf("resolved inventory state %d has unknown plane %q", index, state.Plane) + } + if !validResolvedInventoryRepoPath(state.Path) || !strings.HasSuffix(state.Path, ".yaml.gotmpl") { + return resolvedInventoryConfig{}, fmt.Errorf("resolved inventory state %d has invalid Helmfile path %q", index, state.Path) + } + if _, exists := seenStates[state.Path]; exists { + return resolvedInventoryConfig{}, fmt.Errorf("resolved inventory state path %s is duplicated", state.Path) + } + seenStates[state.Path] = struct{}{} + for _, override := range append(append([]string{}, state.BaseOverrides...), state.FullOverrides...) { + name, value, found := strings.Cut(override, "=") + if !found || !validResolvedInventoryRenderValueName(name) || value == "" || value != strings.TrimSpace(value) { + return resolvedInventoryConfig{}, fmt.Errorf("resolved inventory state %s has invalid override %q", state.Path, override) + } + } + } + } repository := config.PublishedChartRepository if repository == "" || repository != strings.TrimSpace(repository) || strings.HasSuffix(repository, "/") || !strings.HasPrefix(repository, "https://") || strings.ContainsAny(repository, "{}$?#") { @@ -629,11 +715,12 @@ func collectResolvedInventoryState( renderSource resolvedInventoryHelmSource, publishedChartRepository string, sourceCharts map[string]resolvedInventorySourceChart, + allowUnavailableSourceCharts bool, ngcAPIKey string, runner resolvedInventoryCommandRunner, ) ([]helmfileRelease, map[string][]byte, []string, error) { - baseOverrides := append(append([]string{}, resolvedInventoryCommonOverrides...), state.baseOverrides...) - fullOverrides := append(append([]string{}, baseOverrides...), state.fullOverrides...) + baseOverrides := append(append([]string{}, resolvedInventoryCommonOverrides...), state.BaseOverrides...) + fullOverrides := append(append([]string{}, baseOverrides...), state.FullOverrides...) baseList, err := runResolvedInventoryHelmfile(runner, filepath.Dir(stateFile), env, stateFile, baseOverrides, "list", "--output", "json") if err != nil { return nil, nil, nil, fmt.Errorf("list default releases: %w", err) @@ -649,6 +736,7 @@ func collectResolvedInventoryState( source, fullList, sourceCharts, + allowUnavailableSourceCharts, ) if err != nil { return nil, nil, nil, err @@ -716,6 +804,7 @@ func materializeResolvedInventorySourceCharts( stackSource stackSourceRelease, fullRaw []byte, sourceCharts map[string]resolvedInventorySourceChart, + allowUnavailableSourceCharts bool, ) ([]string, error) { if len(sourceCharts) == 0 { return nil, nil @@ -741,6 +830,12 @@ func materializeResolvedInventorySourceCharts( return nil, fmt.Errorf("source chart %s release %s version %q is not semantic", name, release.Name, release.Version) } tag := sourceChart.TagPrefix + release.Version + if _, err := gitOutput(repoRoot, "cat-file", "-e", tag+":"+sourceChart.Path); err != nil { + if allowUnavailableSourceCharts { + continue + } + return nil, fmt.Errorf("source chart %s path %s is unavailable at %s: %w", name, sourceChart.Path, tag, err) + } if _, err := gitOutput(repoRoot, "merge-base", "--is-ancestor", tag, stackSource.Commit); err != nil { return nil, fmt.Errorf("source chart %s tag %s is not an ancestor of stack commit %s: %w", name, tag, stackSource.Commit, err) } diff --git a/tools/docs-version-sync/resolved_inventory_publish_test.go b/tools/docs-version-sync/resolved_inventory_publish_test.go index fb804897b5..d87556daf7 100644 --- a/tools/docs-version-sync/resolved_inventory_publish_test.go +++ b/tools/docs-version-sync/resolved_inventory_publish_test.go @@ -79,51 +79,132 @@ func TestMergeAndResolveHelmfileReleasesPreservesOptionalStatus(t *testing.T) { } } -func TestCollectResolvedStackInventoryBuildsEveryPlane(t *testing.T) { - t.Setenv("NVCF_RELEASE_NGC_API_KEY", "test-api-key") - t.Setenv("NVCF_RELEASE_HELM_REGISTRY", "registry.example.test/release/charts") - repo := initTestGitRepo(t) - writeFile(t, filepath.Join(repo, "deploy/helm/nvca-operator/nvca-operator/Chart.yaml"), "apiVersion: v2\nname: helm-nvca-operator\nversion: 0.0.0\n") - writeFile(t, filepath.Join(repo, "deploy/helm/nvca-operator/nvca-operator/templates/pod.yaml"), "kind: Pod\n") - if _, err := gitOutput(repo, "add", "--all"); err != nil { +func TestDeclaredHelmfileReleaseNamesReportsUncoveredOptionalRelease(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.yaml.gotmpl") + writeFile(t, path, `repositories: + - name: nvcf + url: example.invalid +releases: + - name: required + chart: nvcf/required + - name: "double-quoted-addon" + chart: nvcf/double-quoted + - name: 'single-quoted-addon' # optional release + chart: nvcf/single-quoted + - name: optional-addon + condition: addons.example.enabled + chart: nvcf/optional +`) + + names, err := declaredHelmfileReleaseNames(path) + if err != nil { t.Fatal(err) } - if _, err := gitOutput(repo, "commit", "-m", "nvca chart fixture"); err != nil { - t.Fatal(err) + want := []string{"double-quoted-addon", "optional-addon", "required", "single-quoted-addon"} + if !reflect.DeepEqual(names, want) { + t.Fatalf("declared releases = %v, want %v", names, want) } - if _, err := gitOutput(repo, "tag", "deploy/helm/nvca-operator/v1.0.0"); err != nil { - t.Fatal(err) +} + +func TestMaterializeResolvedInventorySourceChartsAllowsUnavailableHistoricalPath(t *testing.T) { + repo := initTestGitRepo(t) + source := commitTestStackSource(t, repo, "1.2.3", map[string]string{ + "deploy/stacks/self-managed/helmfile.d/state.yaml.gotmpl": "releases: []\n", + }) + fullRaw := mustJSON(t, []helmfileRelease{{ + Name: "example", + Chart: "nvcf/helm-example", + Version: "1.0.0", + }}) + sourceCharts := map[string]resolvedInventorySourceChart{ + "helm-example": { + TagPrefix: "deploy/helm/example/v", + Path: "deploy/helm/example", + }, + } + + used, err := materializeResolvedInventorySourceCharts( + repo, + filepath.Join(repo, "deploy/stacks/self-managed/helmfile.d/state.yaml.gotmpl"), + t.TempDir(), + source, + fullRaw, + sourceCharts, + true, + ) + if err != nil { + t.Fatalf("historical source-chart fallback failed: %v", err) } + if len(used) != 0 { + t.Fatalf("used source charts = %v, want published chart fallback", used) + } + + _, err = materializeResolvedInventorySourceCharts( + repo, + filepath.Join(repo, "deploy/stacks/self-managed/helmfile.d/state.yaml.gotmpl"), + t.TempDir(), + source, + fullRaw, + sourceCharts, + false, + ) + if err == nil || !strings.Contains(err.Error(), "is unavailable") { + t.Fatalf("strict source-chart error = %v, want unavailable path", err) + } +} +func TestCollectResolvedStackInventoryBuildsConfiguredSelfManagedStates(t *testing.T) { + t.Setenv("NVCF_RELEASE_NGC_API_KEY", "test-api-key") + t.Setenv("NVCF_RELEASE_HELM_REGISTRY", "registry.example.test/release/charts") + repo := initTestGitRepo(t) + states := []resolvedInventoryState{ + {Plane: "control-plane", Path: "deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl", FullOverrides: []string{"addons.llm.enabled=true"}}, + {Plane: "control-plane", Path: "deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl"}, + {Plane: "control-plane", Path: "deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl"}, + } + configBody := "schemaVersion: 1\npublishedChartRepository: " + testPublishedChartRepository + "\nstates:\n" + for _, state := range states { + configBody += " - plane: " + state.Plane + "\n path: " + state.Path + "\n" + if len(state.BaseOverrides) > 0 { + configBody += " baseOverrides:\n - " + state.BaseOverrides[0] + "\n" + } + if len(state.FullOverrides) > 0 { + configBody += " fullOverrides:\n - " + state.FullOverrides[0] + "\n" + } + } stackFiles := map[string]string{ - resolvedInventoryConfigPath: "schemaVersion: 1\npublishedChartRepository: " + testPublishedChartRepository + "\nsourceCharts:\n helm-nvca-operator:\n tagPrefix: deploy/helm/nvca-operator/v\n path: deploy/helm/nvca-operator/nvca-operator\n", + resolvedInventoryConfigPath: configBody, } - for _, state := range resolvedInventoryStates { - stackFiles[state.path] = "releases: []\n" + for _, state := range states { + stackFiles[state.Path] = "releases: []\n" } - stackFiles["deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl"] = "releases:\n - name: nvca-operator\n chart: nvcf/helm-nvca-operator\n version: 1.0.0\n" + stackFiles[states[0].Path] = "releases:\n - name: \"unrendered-addon\"\n chart: nvcf/addon\n" source := commitTestStackSource(t, repo, "1.2.3", stackFiles) runner := &fakeResolvedInventoryRunner{t: t} - inventory, err := collectResolvedStackInventory(repo, "", source, runner) + inventory, err := collectResolvedStackInventory(repo, "", source, resolvedInventoryGenerationOptions{}, runner) if err != nil { t.Fatalf("collectResolvedStackInventory failed: %v", err) } if err := validateResolvedStackInventory(inventory); err != nil { t.Fatalf("inventory validation failed: %v", err) } - if len(inventory.Releases) != 8 { - t.Fatalf("got %d releases, want 8", len(inventory.Releases)) + if len(inventory.Releases) != 4 { + t.Fatalf("got %d releases, want 4", len(inventory.Releases)) } for _, release := range inventory.Releases { if release.Name == "nvcf-pki" && release.Required { t.Fatal("nvcf-pki should remain optional after the full render") } } - if runner.templateCalls != len(resolvedInventoryStates) { - t.Fatalf("template calls = %d, want %d", runner.templateCalls, len(resolvedInventoryStates)) + if runner.templateCalls != len(states) { + t.Fatalf("template calls = %d, want %d", runner.templateCalls, len(states)) + } + if runner.prepareCalls != 2 { + t.Fatalf("repository preparation calls = %d, want source and public repository preparation", runner.prepareCalls) } - if runner.prepareCalls != 3 { - t.Fatalf("repository preparation calls = %d, want source and two public repository preparations", runner.prepareCalls) + wantWarning := states[0].Path + " declares release unrendered-addon but the full inventory profile did not render it" + if !reflect.DeepEqual(inventory.Warnings, []string{wantWarning}) { + t.Fatalf("coverage warnings = %v, want %q", inventory.Warnings, wantWarning) } wantPrefix := []string{"prepare-source", "list", "list", "build", "prepare-public", "template"} if len(runner.operations) < len(wantPrefix) || !reflect.DeepEqual(runner.operations[:len(wantPrefix)], wantPrefix) { @@ -211,6 +292,36 @@ func TestLoadResolvedInventoryConfig(t *testing.T) { t.Fatalf("invalid render value name error = %v", err) } + invalidStates := []struct { + name string + body string + wantErr string + }{ + { + name: "unknown plane", + body: "states:\n - plane: data-plane\n path: deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl\n", + wantErr: "unknown plane", + }, + { + name: "duplicate path", + body: "states:\n - plane: control-plane\n path: deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl\n - plane: observability\n path: deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl\n", + wantErr: "is duplicated", + }, + { + name: "malformed override", + body: "states:\n - plane: control-plane\n path: deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl\n fullOverrides:\n - addons.llm.enabled\n", + wantErr: "invalid override", + }, + } + for _, test := range invalidStates { + t.Run(test.name, func(t *testing.T) { + writeFile(t, configPath, "schemaVersion: 1\npublishedChartRepository: "+testPublishedChartRepository+"\n"+test.body) + if _, err := loadResolvedInventoryConfig(repo, ""); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("state validation error = %v, want substring %q", err, test.wantErr) + } + }) + } + externalConfig := filepath.Join(t.TempDir(), "release-inventory.yaml") writeFile(t, externalConfig, "schemaVersion: 1\npublishedChartRepository: https://helm.example.test/external\n") external, err := loadResolvedInventoryConfig(repo, externalConfig) @@ -222,6 +333,58 @@ func TestLoadResolvedInventoryConfig(t *testing.T) { } } +func TestReleaseInventoriesOwnOnlyTheirStackStates(t *testing.T) { + repoRoot, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + tests := []struct { + spec stackInventorySpec + plane string + paths []string + }{ + {stackInventorySpecs[0], "control-plane", []string{ + "deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl", + "deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl", + "deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl", + }}, + {stackInventorySpecs[1], "compute-plane", []string{ + "deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl", + "deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl", + }}, + {stackInventorySpecs[2], "observability", []string{ + "deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl", + }}, + } + for _, test := range tests { + t.Run(test.spec.Key, func(t *testing.T) { + config, err := loadResolvedInventoryConfig(repoRoot, filepath.Join(repoRoot, filepath.FromSlash(test.spec.ConfigPath))) + if err != nil { + t.Fatal(err) + } + want := make(map[string]struct{}, len(test.paths)) + for _, path := range test.paths { + want[path] = struct{}{} + } + if len(config.States) != len(want) { + t.Fatalf("%s inventory states = %d, want %d", test.spec.Key, len(config.States), len(want)) + } + for _, state := range config.States { + if state.Plane != test.plane { + t.Fatalf("%s inventory state %#v has plane %s, want %s", test.spec.Key, state, state.Plane, test.plane) + } + if _, ok := want[state.Path]; !ok { + t.Fatalf("unexpected %s inventory state %#v", test.spec.Key, state) + } + delete(want, state.Path) + } + if len(want) != 0 { + t.Fatalf("%s inventory is missing states %v", test.spec.Key, want) + } + }) + } +} + func TestReplaceResolvedInventoryStateChart(t *testing.T) { chartPath := filepath.Join(t.TempDir(), "helm-nvcf-invocation-service", "1.6.1") quotedChartPath := strconv.Quote(filepath.ToSlash(chartPath)) diff --git a/tools/docs-version-sync/resolved_inventory_test.go b/tools/docs-version-sync/resolved_inventory_test.go index f40a5f171d..58e1fe6aa2 100644 --- a/tools/docs-version-sync/resolved_inventory_test.go +++ b/tools/docs-version-sync/resolved_inventory_test.go @@ -68,6 +68,185 @@ metadata: } } +func TestExtractResolvedImagesFindsImageArguments(t *testing.T) { + manifest := []byte(` +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: controller + image: registry.example.com/controller:v1.20.2 + args: + - --acme-http01-solver-image=registry.example.com/acmesolver:v1.20.2 + - --image-pull-policy=IfNotPresent + - --unrelated=value + command: + - controller + - --operator-image=registry.example.com/operator@sha256:abc123 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + release-artifact-worker-image: registry.example.com/worker:2.7.1 +data: + example: --ignored-image=registry.example.com/not-an-argument:1.0.0 +`) + + images, err := extractResolvedImages(manifest) + if err != nil { + t.Fatal(err) + } + want := []string{ + "registry.example.com/acmesolver:v1.20.2", + "registry.example.com/controller:v1.20.2", + "registry.example.com/operator@sha256:abc123", + "registry.example.com/worker:2.7.1", + } + if len(images) != len(want) { + t.Fatalf("got %d images, want %d: %+v", len(images), len(want), images) + } + for i, reference := range want { + if images[i].Reference != reference { + t.Fatalf("image %d reference = %q, want %q", i, images[i].Reference, reference) + } + } +} + +func TestOptionalIndirectImageAppearsOptionalInManifest(t *testing.T) { + inputs := resolvedInventoryTestInputs(t) + inputs[1].ManifestByRelease["notary-service"] = append(inputs[1].ManifestByRelease["notary-service"], []byte(` +--- +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: controller + image: registry.example.com/controller:1.0.0 + args: + - --solver-image=registry.example.com/optional-solver:2.0.0 +`)...) + inventory, err := generateResolvedStackInventory(resolvedInventoryTestSource(), inputs) + if err != nil { + t.Fatal(err) + } + resolvedSolver := findResolvedInventoryArtifact(t, inventory, "container-image", "registry.example.com/optional-solver:2.0.0") + if len(resolvedSolver.Sources) != 1 || resolvedSolver.Sources[0].Requirement != ManifestOptional { + t.Fatalf("optional solver inventory sources = %+v, want optional", resolvedSolver.Sources) + } + controller := findResolvedInventoryArtifact(t, inventory, "container-image", "registry.example.com/controller:1.0.0") + if len(controller.Sources) != 1 || controller.Sources[0].Requirement != ManifestRequired { + t.Fatalf("required controller inventory sources = %+v, want required", controller.Sources) + } + artifacts, err := catalogArtifactsFromResolvedStackInventory(inventory, nil) + if err != nil { + t.Fatal(err) + } + var solver Artifact + for _, artifact := range artifacts { + if artifact.Name == "optional-solver" { + solver = artifact + break + } + } + if solver.Requirement != ManifestOptional { + t.Fatalf("optional solver requirement = %q, want optional", solver.Requirement) + } + if strings.Join(solver.Stacks, ",") != selfManagedStackKey { + t.Fatalf("optional solver stacks = %v, want self-managed", solver.Stacks) + } + + catalog := testCatalog() + catalog.Artifacts = append(catalog.Artifacts, solver) + catalog.Manifest.Entries = append(catalog.Manifest.Entries, ManifestEntry{ + ArtifactID: solver.catalogKey(), Plane: ManifestPlaneControl, Kind: ManifestKindServiceImage, + Requirement: ManifestRequired, Description: "Solves optional certificate challenges.", + }) + catalog.PublicationPending = append(catalog.PublicationPending, solver.catalogKey()) + manifest, err := renderManifestArtifactRegistryPaths(catalog) + if err != nil { + t.Fatal(err) + } + row := manifestRow(t, manifest, solver.Name) + if !strings.Contains(row, "| Optional |") { + t.Fatalf("optional indirect image row is not optional: %s", row) + } +} + +func TestExtractResolvedImagesDeduplicatesImageArguments(t *testing.T) { + manifest := []byte(` +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: controller + image: registry.example.com/acmesolver:v1.20.2 + args: + - --acme-http01-solver-image=registry.example.com/acmesolver:v1.20.2 + - --fallback-image=registry.example.com/acmesolver:v1.20.2 +`) + + images, err := extractResolvedImages(manifest) + if err != nil { + t.Fatal(err) + } + if len(images) != 1 || images[0].Reference != "registry.example.com/acmesolver:v1.20.2" { + t.Fatalf("unexpected images: %+v", images) + } +} + +func TestExtractResolvedImagesRejectsUnresolvedImageArguments(t *testing.T) { + tests := []struct { + name string + argument string + wantError string + }{ + {name: "empty", argument: "--solver-image=", wantError: "unresolved"}, + {name: "placeholder", argument: "--solver-image=${registry}/solver:1.0.0", wantError: "unresolved"}, + {name: "missing tag", argument: "--solver-image=registry.example.com/solver", wantError: "no tag or digest"}, + {name: "latest tag", argument: "--solver-image=registry.example.com/solver:latest", wantError: "unresolved latest tag"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + manifest := []byte("apiVersion: v1\nkind: Pod\nspec:\n containers:\n - name: controller\n args:\n - " + test.argument + "\n") + _, err := extractResolvedImages(manifest) + if err == nil || !strings.Contains(err.Error(), test.wantError) || !strings.Contains(err.Error(), "image argument") { + t.Fatalf("got error %v, want image argument error containing %q", err, test.wantError) + } + }) + } +} + +func TestResolvedImageArgumentRequiresCompleteImageFlag(t *testing.T) { + tests := []struct { + value string + want string + ok bool + }{ + {value: "--acme-http01-solver-image=registry.example.com/acmesolver:v1.20.2", want: "registry.example.com/acmesolver:v1.20.2", ok: true}, + {value: "--operator_v2-image=registry.example.com/operator:1.0.0", want: "registry.example.com/operator:1.0.0", ok: true}, + {value: "--operator.image=registry.example.com/operator:1.0.0", ok: false}, + {value: "--image=registry.example.com/operator:1.0.0"}, + {value: "--solver-image", ok: false}, + {value: "prefix --solver-image=registry.example.com/solver:1.0.0", ok: false}, + {value: "--image-pull-policy=IfNotPresent", ok: false}, + } + for _, test := range tests { + t.Run(test.value, func(t *testing.T) { + got, ok := resolvedImageArgument(test.value) + if got != test.want || ok != test.ok { + t.Fatalf("resolvedImageArgument(%q) = %q, %t; want %q, %t", test.value, got, ok, test.want, test.ok) + } + }) + } +} + func TestGenerateResolvedStackInventoryIsDeterministic(t *testing.T) { inputs := resolvedInventoryTestInputs(t) first, err := generateResolvedStackInventory(resolvedInventoryTestSource(), inputs) diff --git a/tools/docs-version-sync/stack_consistency_test.go b/tools/docs-version-sync/stack_consistency_test.go index 82ef4a67fa..761280d26c 100644 --- a/tools/docs-version-sync/stack_consistency_test.go +++ b/tools/docs-version-sync/stack_consistency_test.go @@ -75,14 +75,6 @@ func TestExtractEffectiveStackPinsRejectsMissingVersionInTargetBlock(t *testing. version: 9.9.9 `, }, - { - artifact: "nvca", - body: ` nvcaOperator: - imageTag: "3.2.19" - unrelated: - nvcaVersion: "9.9.9" -`, - }, } for _, test := range tests { t.Run(test.artifact, func(t *testing.T) { @@ -113,16 +105,6 @@ func TestExtractEffectiveStackPinsRejectsDuplicateVersionsInTargetBlock(t *testi version: 9.9.9 `, }, - { - artifact: "nvca", - body: ` nvcaOperator: - nvca: - nvcaVersion: "3.2.19" - nvcaVersion: "3.2.20" - unrelated: - enabled: true -`, - }, } for _, test := range tests { t.Run(test.artifact, func(t *testing.T) { diff --git a/tools/docs-version-sync/stack_inventory.go b/tools/docs-version-sync/stack_inventory.go new file mode 100644 index 0000000000..7c1b87456f --- /dev/null +++ b/tools/docs-version-sync/stack_inventory.go @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "strings" +) + +const ( + selfManagedStackKey = "self-managed" + computePlaneStackKey = "compute-plane" + observabilityStackKey = "observability" +) + +type stackInventorySpec struct { + Key string + ResourceName string + TagPrefix string + AssetName string + ConfigPath string +} + +var stackInventorySpecs = []stackInventorySpec{ + { + Key: selfManagedStackKey, + ResourceName: controlStackResourceName, + TagPrefix: "deploy/stacks/self-managed/v", + AssetName: "nvcf-self-managed-stack-inventory.json", + ConfigPath: "deploy/stacks/self-managed/release-inventory.yaml", + }, + { + Key: computePlaneStackKey, + ResourceName: computeStackResourceName, + TagPrefix: "deploy/stacks/nvcf-compute-plane/v", + AssetName: "nvcf-compute-plane-stack-inventory.json", + ConfigPath: "deploy/stacks/nvcf-compute-plane/release-inventory.yaml", + }, + { + Key: observabilityStackKey, + ResourceName: observabilityStackResourceName, + TagPrefix: "deploy/stacks/observability/v", + AssetName: "nvcf-observability-stack-inventory.json", + ConfigPath: "deploy/stacks/observability/release-inventory.yaml", + }, +} + +func stackInventorySpecByKey(key string) (stackInventorySpec, error) { + for _, spec := range stackInventorySpecs { + if spec.Key == key { + return spec, nil + } + } + return stackInventorySpec{}, fmt.Errorf("unknown stack %q; want self-managed, compute-plane, or observability", key) +} + +func stackInventorySpecByTag(tag string) (stackInventorySpec, error) { + for _, spec := range stackInventorySpecs { + if strings.HasPrefix(tag, spec.TagPrefix) { + return spec, nil + } + } + return stackInventorySpec{}, fmt.Errorf("stack source tag %q does not match a released stack", tag) +} + +func stackKeyForPlane(plane string) (string, error) { + switch plane { + case "control-plane": + return selfManagedStackKey, nil + case "compute-plane": + return computePlaneStackKey, nil + case "observability": + return observabilityStackKey, nil + default: + return "", fmt.Errorf("unknown inventory plane %q", plane) + } +} + +func releaseSetFromInventories(inventories map[string]resolvedStackInventory, documentationVersion string, status ReleaseSetStatus) (ReleaseSetMetadata, error) { + metadata := func(key string) (StackReleaseMetadata, error) { + spec, err := stackInventorySpecByKey(key) + if err != nil { + return StackReleaseMetadata{}, err + } + inventory, exists := inventories[key] + if !exists { + return StackReleaseMetadata{}, fmt.Errorf("%s inventory is required for release set metadata", key) + } + return StackReleaseMetadata{ + Version: inventory.Source.Version, + SourceTag: inventory.Source.Tag, + SourceCommit: inventory.Source.Commit, + InventoryAsset: spec.AssetName, + }, nil + } + controlPlane, err := metadata(selfManagedStackKey) + if err != nil { + return ReleaseSetMetadata{}, err + } + computePlane, err := metadata(computePlaneStackKey) + if err != nil { + return ReleaseSetMetadata{}, err + } + observability, err := metadata(observabilityStackKey) + if err != nil { + return ReleaseSetMetadata{}, err + } + return ReleaseSetMetadata{ + DocumentationVersion: documentationVersion, + Status: status, + Stacks: ReleaseSetStacks{ + ControlPlane: controlPlane, + ComputePlane: computePlane, + Observability: observability, + }, + }, nil +} + +func (releaseSet ReleaseSetMetadata) sameStackReleases(other ReleaseSetMetadata) bool { + return releaseSet.Stacks == other.Stacks +} diff --git a/tools/docs-version-sync/stack_inventory_test.go b/tools/docs-version-sync/stack_inventory_test.go new file mode 100644 index 0000000000..89011fd2d3 --- /dev/null +++ b/tools/docs-version-sync/stack_inventory_test.go @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "strings" + "testing" +) + +func TestStackInventorySpecsMatchReleasedStacks(t *testing.T) { + for _, spec := range stackInventorySpecs { + t.Run(spec.Key, func(t *testing.T) { + release := stackSourceRelease{ + Version: "1.2.3", + Tag: spec.TagPrefix + "1.2.3", + Commit: strings.Repeat("a", 40), + } + if err := validateStackSourceReleaseForSpec(release, spec); err != nil { + t.Fatal(err) + } + got, err := stackInventorySpecByTag(release.Tag) + if err != nil { + t.Fatal(err) + } + if got != spec { + t.Fatalf("spec = %#v, want %#v", got, spec) + } + }) + } +} + +func TestReleaseSetRecordsAllThreeImmutableSources(t *testing.T) { + inventories := map[string]resolvedStackInventory{} + for index, spec := range stackInventorySpecs { + version := fmt.Sprintf("1.%d.0", index) + inventories[spec.Key] = resolvedStackInventory{Source: stackSourceRelease{ + Version: version, + Tag: spec.TagPrefix + version, + Commit: strings.Repeat(string(rune('a'+index)), 40), + }} + } + releaseSet, err := releaseSetFromInventories(inventories, "1.4.0", ReleaseSetQualified) + if err != nil { + t.Fatal(err) + } + if err := validateReleaseSet(releaseSet); err != nil { + t.Fatal(err) + } + if releaseSet.Stacks.ControlPlane.InventoryAsset != stackInventorySpecs[0].AssetName || + releaseSet.Stacks.ComputePlane.InventoryAsset != stackInventorySpecs[1].AssetName || + releaseSet.Stacks.Observability.InventoryAsset != stackInventorySpecs[2].AssetName { + t.Fatalf("release set assets = %#v", releaseSet.Stacks) + } +} + +func TestReleaseSetStackComparisonIncludesImmutableSourceMetadata(t *testing.T) { + releaseSet := ReleaseSetMetadata{Stacks: ReleaseSetStacks{ + ControlPlane: StackReleaseMetadata{ + Version: "1.2.3", SourceTag: stackInventorySpecs[0].TagPrefix + "1.2.3", + SourceCommit: strings.Repeat("a", 40), InventoryAsset: stackInventorySpecs[0].AssetName, + }, + ComputePlane: StackReleaseMetadata{ + Version: "2.3.4", SourceTag: stackInventorySpecs[1].TagPrefix + "2.3.4", + SourceCommit: strings.Repeat("b", 40), InventoryAsset: stackInventorySpecs[1].AssetName, + }, + Observability: StackReleaseMetadata{ + Version: "3.4.5", SourceTag: stackInventorySpecs[2].TagPrefix + "3.4.5", + SourceCommit: strings.Repeat("c", 40), InventoryAsset: stackInventorySpecs[2].AssetName, + }, + }} + if !releaseSet.sameStackReleases(releaseSet) { + t.Fatal("identical stack metadata did not match") + } + + changed := releaseSet + changed.Stacks.Observability.SourceCommit = strings.Repeat("d", 40) + if releaseSet.sameStackReleases(changed) { + t.Fatal("release sets with different immutable source commits matched") + } +} + +func TestMergeResolvedStackInventoriesCombinesPeerSources(t *testing.T) { + inventories := map[string]resolvedStackInventory{ + selfManagedStackKey: testSeparatedInventory(t, stackInventorySpecs[0], "control-plane", "api"), + computePlaneStackKey: testSeparatedInventory(t, stackInventorySpecs[1], "compute-plane", "nvca"), + observabilityStackKey: testSeparatedInventory(t, stackInventorySpecs[2], "observability", "collector"), + } + combined, err := mergeResolvedStackInventories(inventories) + if err != nil { + t.Fatal(err) + } + if combined.Source != inventories[selfManagedStackKey].Source { + t.Fatalf("combined source = %#v, want self-managed source %#v", combined.Source, inventories[selfManagedStackKey].Source) + } + if len(combined.Releases) != 3 { + t.Fatalf("combined releases = %d, want 3", len(combined.Releases)) + } + for _, artifact := range combined.Artifacts { + if artifact.Name == "shared-runtime" && len(artifact.Sources) != 3 { + t.Fatalf("shared runtime sources = %#v, want all three stacks", artifact.Sources) + } + } +} + +func TestMergeResolvedStackInventoriesRejectsVersionConflict(t *testing.T) { + inventories := map[string]resolvedStackInventory{ + selfManagedStackKey: testSeparatedInventory(t, stackInventorySpecs[0], "control-plane", "api"), + computePlaneStackKey: testSeparatedInventory(t, stackInventorySpecs[1], "compute-plane", "nvca"), + observabilityStackKey: testSeparatedInventory(t, stackInventorySpecs[2], "observability", "collector"), + } + compute := inventories[computePlaneStackKey] + for index := range compute.Artifacts { + if compute.Artifacts[index].Name == "shared-runtime" { + compute.Artifacts[index].Version = "2.0.0" + compute.Artifacts[index].Reference = "nvcr.io/nvidia/nvcf/shared-runtime:2.0.0" + } + } + inventories[computePlaneStackKey] = compute + _, err := mergeResolvedStackInventories(inventories) + if err == nil || !strings.Contains(err.Error(), "compute-plane uses nvcr.io/nvidia/nvcf/shared-runtime:2.0.0") { + t.Fatalf("merge error = %v, want cross-stack version conflict", err) + } +} + +func testSeparatedInventory(t *testing.T, spec stackInventorySpec, plane, releaseName string) resolvedStackInventory { + t.Helper() + source := stackSourceRelease{Version: "1.2.3", Tag: spec.TagPrefix + "1.2.3", Commit: strings.Repeat("a", 40)} + release := resolvedInventoryRelease{ + Plane: plane, Name: releaseName, Namespace: "nvcf", Required: true, + Chart: "https://helm.ngc.nvidia.com/nvidia/nvcf/helm-" + releaseName, + Version: "1.0.0", + } + sourceRef := resolvedArtifactSource{Plane: plane, Release: releaseName} + inventory := resolvedStackInventory{ + SchemaVersion: resolvedStackInventorySchemaVersion, + Source: source, + Releases: []resolvedInventoryRelease{release}, + Artifacts: []resolvedInventoryArtifact{ + { + Type: "helm-chart", Name: "helm-" + releaseName, + Repository: "https://helm.ngc.nvidia.com/nvidia/nvcf", Version: "1.0.0", + Reference: release.Chart + "@1.0.0", Sources: []resolvedArtifactSource{sourceRef}, + }, + { + Type: "container-image", Name: "shared-runtime", + Repository: "nvcr.io/nvidia/nvcf/shared-runtime", Version: "1.0.0", + Reference: "nvcr.io/nvidia/nvcf/shared-runtime:1.0.0", Sources: []resolvedArtifactSource{sourceRef}, + }, + }, + } + if compareResolvedInventoryArtifacts(inventory.Artifacts[0], inventory.Artifacts[1]) > 0 { + inventory.Artifacts[0], inventory.Artifacts[1] = inventory.Artifacts[1], inventory.Artifacts[0] + } + if err := validateResolvedStackInventory(inventory); err != nil { + t.Fatalf("test inventory is invalid: %v", err) + } + return inventory +} diff --git a/tools/docs-version-sync/stack_source.go b/tools/docs-version-sync/stack_source.go index 310e7d520e..3b2974c4a0 100644 --- a/tools/docs-version-sync/stack_source.go +++ b/tools/docs-version-sync/stack_source.go @@ -45,22 +45,6 @@ var effectiveStackPins = []effectiveStackPin{ path: "deploy/stacks/self-managed/global.yaml.gotmpl", pattern: `pylon:([0-9][^"\s]*)`, }, - { - artifact: "helm-nvca-operator", - artifactType: ArtifactTypeChart, - path: "deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl", - blockPattern: `(?m)^ - name: nvca-operator[ \t]*$`, - blockBoundaryPattern: `(?m)^ -[ \t]+`, - pattern: `(?m)^ version:[ \t]*"?([^"\s]+)"?[ \t]*$`, - }, - { - artifact: "nvca", - artifactType: ArtifactTypeImage, - path: "deploy/stacks/nvcf-compute-plane/environments/base.yaml", - blockPattern: `(?m)^ nvcaOperator:[ \t]*$`, - blockBoundaryPattern: `(?m)^ [A-Za-z0-9_-]+:[ \t]*`, - pattern: `(?m)^ nvcaVersion:[ \t]*"([^"]+)"[ \t]*$`, - }, } func extractEffectiveStackPins(sources map[string][]byte, pins []effectiveStackPin) (map[string]string, error) { @@ -142,7 +126,34 @@ func pinSourceDigest(sources map[string][]byte, pins []effectiveStackPin) (strin // validateStackSourceSnapshot verifies the catalog against its immutable stack release inputs. func validateStackSourceSnapshot(repoRoot string, catalog *Catalog) error { - return validateStackSourceSnapshotWithPins(repoRoot, catalog, effectiveStackPins) + pins, err := effectiveStackPinsForSources(catalog.Stack.PinSources) + if err != nil { + return err + } + return validateStackSourceSnapshotWithPins(repoRoot, catalog, pins) +} + +func effectiveStackPinsForSources(sourcePaths []string) ([]effectiveStackPin, error) { + known := append([]effectiveStackPin(nil), effectiveStackPins...) + knownPaths := make(map[string]struct{}, len(known)) + for _, pin := range known { + knownPaths[pin.path] = struct{}{} + } + selectedPaths := make(map[string]struct{}, len(sourcePaths)) + for _, sourcePath := range sourcePaths { + _, ok := knownPaths[sourcePath] + if !ok { + return nil, fmt.Errorf("stack source snapshot declares unsupported pin source %s", sourcePath) + } + selectedPaths[sourcePath] = struct{}{} + } + pins := make([]effectiveStackPin, 0, len(known)) + for _, pin := range known { + if _, selected := selectedPaths[pin.path]; selected { + pins = append(pins, pin) + } + } + return pins, nil } // validateStackSourceSnapshotWithPins verifies catalog artifacts with a supplied pin definition set. diff --git a/tools/docs-version-sync/stack_source_reader.go b/tools/docs-version-sync/stack_source_reader.go index b030fef855..873aa70b68 100644 --- a/tools/docs-version-sync/stack_source_reader.go +++ b/tools/docs-version-sync/stack_source_reader.go @@ -58,10 +58,18 @@ func loadStackSourceSnapshot(repoRoot string, release stackSourceRelease, source // validateStackSourceRelease verifies the canonical identity of a selected stack release. func validateStackSourceRelease(release stackSourceRelease) error { + spec, err := stackInventorySpecByTag(release.Tag) + if err != nil { + return err + } + return validateStackSourceReleaseForSpec(release, spec) +} + +func validateStackSourceReleaseForSpec(release stackSourceRelease, spec stackInventorySpec) error { if !validStackVersion(release.Version) { return fmt.Errorf("stack source version %q is not a semantic version", release.Version) } - wantTag := stackTagPrefix + release.Version + wantTag := spec.TagPrefix + release.Version if release.Tag != wantTag { return fmt.Errorf("stack source tag must be %s for source version %s", wantTag, release.Version) } diff --git a/tools/docs-version-sync/stack_source_reader_test.go b/tools/docs-version-sync/stack_source_reader_test.go index a444bfcbc2..92451002a9 100644 --- a/tools/docs-version-sync/stack_source_reader_test.go +++ b/tools/docs-version-sync/stack_source_reader_test.go @@ -168,7 +168,7 @@ func TestValidateStackSourceReleaseRejectsInvalidIdentity(t *testing.T) { { name: "non-canonical tag", mutate: func(release *stackSourceRelease) { release.Tag = "v1.2.3" }, - wantErr: "stack source tag must be " + stackTagPrefix + "1.2.3", + wantErr: "does not match a released stack", }, { name: "malformed commit", diff --git a/tools/scripts/cut-docs-version.sh b/tools/scripts/cut-docs-version.sh index 545516bb7d..e5f47f8a86 100755 --- a/tools/scripts/cut-docs-version.sh +++ b/tools/scripts/cut-docs-version.sh @@ -22,21 +22,64 @@ docs_dst="$root/docs/$VERSION" nav_src="$root/fern/versions/dev.yml" nav_dst="$root/fern/versions/$VERSION.yml" docs_yml="$root/fern/docs.yml" +catalog="$root/docs/version-catalog/main.yaml" +catalog_snapshot="$root/docs/version-catalog/$VERSION.yaml" [[ -d "$docs_src" ]] || { echo "Missing $docs_src" >&2; exit 1; } [[ -f "$nav_src" ]] || { echo "Missing $nav_src" >&2; exit 1; } [[ -f "$docs_yml" ]] || { echo "Missing $docs_yml" >&2; exit 1; } +[[ -f "$catalog" ]] || { echo "Missing $catalog" >&2; exit 1; } [[ ! -d "$docs_dst" ]] || { echo "Refusing to overwrite $docs_dst" >&2; exit 1; } [[ ! -f "$nav_dst" ]] || { echo "Refusing to overwrite $nav_dst" >&2; exit 1; } +[[ ! -f "$catalog_snapshot" ]] || { echo "Refusing to overwrite $catalog_snapshot" >&2; exit 1; } + +stack_display="$(python3 - "$catalog" "$DISPLAY" <<'PY' +from pathlib import Path +import re +import sys + +catalog_text = Path(sys.argv[1]).read_text() +display = sys.argv[2] + + +def release_set_value(name): + match = re.search(rf'^ {re.escape(name)}:\s*(\S+)\s*$', catalog_text, re.MULTILINE) + if not match: + raise SystemExit(f"release_set is missing {name}") + return match.group(1) + + +def stack_version(name): + match = re.search( + rf'^ {re.escape(name)}:\n version:\s*(\S+)\s*$', + catalog_text, + re.MULTILINE, + ) + if not match: + raise SystemExit(f"release_set is missing {name} version") + return match.group(1) + + +if release_set_value("status") != "qualified": + raise SystemExit("release_set must be qualified before cutting versioned docs") +if release_set_value("documentation_version").lstrip("v") != display: + raise SystemExit("release_set documentation_version does not match requested docs version") +control_version = stack_version("control-plane") +compute_version = stack_version("compute-plane") +observability_version = stack_version("observability") +print(f"CP {control_version}, Compute {compute_version}, Obs {observability_version}") +PY +)" echo "==> Snapshotting docs/user -> docs/$VERSION (dereferencing symlinks)" cp -RL "$docs_src" "$docs_dst" +cp "$catalog" "$catalog_snapshot" echo "==> Generating fern/versions/$VERSION.yml from dev.yml" sed "s|../../docs/user/|../../docs/$VERSION/|g" "$nav_src" > "$nav_dst" echo "==> Updating latest and version entries in fern/docs.yml" -python3 - "$docs_yml" "$VERSION" "$DISPLAY" <<'PY' +python3 - "$docs_yml" "$VERSION" "$DISPLAY" "$stack_display" <<'PY' from pathlib import Path import re import sys @@ -44,6 +87,7 @@ import sys docs_yml = Path(sys.argv[1]) version = sys.argv[2] display = sys.argv[3] +stack_display = sys.argv[4] text = docs_yml.read_text() versions_marker = "versions:\n" @@ -79,7 +123,7 @@ def entry_slug(entry): latest_entry = ( - f'- display-name: "Latest ({display})"\n' + f'- display-name: "Latest ({display}; {stack_display})"\n' f" path: versions/{version}.yml\n" ' slug: ""\n' ) @@ -89,7 +133,7 @@ dev_entry = next((entry for entry in entries if entry_display_name(entry) == "de " slug: dev\n" )) stable_entry = ( - f'- display-name: "{display}"\n' + f'- display-name: "{display} ({stack_display})"\n' f" path: versions/{version}.yml\n" f' slug: "{version}"\n' ) diff --git a/tools/scripts/test/test-cut-docs-version b/tools/scripts/test/test-cut-docs-version index c867ac3ef2..f85c23a6e7 100755 --- a/tools/scripts/test/test-cut-docs-version +++ b/tools/scripts/test/test-cut-docs-version @@ -6,7 +6,7 @@ script="${repo_root}/tools/scripts/cut-docs-version.sh" workdir="$(mktemp -d)" trap 'rm -rf "${workdir}"' EXIT -mkdir -p "${workdir}/tools/scripts" "${workdir}/docs/user" "${workdir}/fern/versions" +mkdir -p "${workdir}/tools/scripts" "${workdir}/docs/user" "${workdir}/docs/version-catalog" "${workdir}/fern/versions" cp "${script}" "${workdir}/tools/scripts/cut-docs-version.sh" cat > "${workdir}/docs/user/overview.md" <<'EOF' @@ -38,6 +38,19 @@ redirects: destination: "/nvcf/v0.5/" EOF +cat > "${workdir}/docs/version-catalog/main.yaml" <<'EOF' +release_set: + documentation_version: 0.6.0 + status: qualified + stacks: + control-plane: + version: 1.2.3 + compute-plane: + version: 1.1.0 + observability: + version: 1.0.4 +EOF + "${workdir}/tools/scripts/cut-docs-version.sh" v0.6.0 > "${workdir}/output.txt" if ! grep -q 'source: "/nvcf/v0.6.0/index.html"' "${workdir}/fern/docs.yml"; then @@ -52,4 +65,25 @@ if ! grep -q 'destination: "/nvcf/v0.6.0/"' "${workdir}/fern/docs.yml"; then exit 1 fi +if ! grep -q 'Latest (0.6.0; CP 1.2.3, Compute 1.1.0, Obs 1.0.4)' "${workdir}/fern/docs.yml"; then + echo "expected generated docs.yml to identify all three qualified stack versions" + cat "${workdir}/fern/docs.yml" + exit 1 +fi + +if ! cmp -s "${workdir}/docs/version-catalog/main.yaml" "${workdir}/docs/version-catalog/v0.6.0.yaml"; then + echo "expected an immutable catalog snapshot for the docs version" + exit 1 +fi + +sed -i.bak 's/status: qualified/status: development/' "${workdir}/docs/version-catalog/main.yaml" +if "${workdir}/tools/scripts/cut-docs-version.sh" v0.7.0 > "${workdir}/invalid-output.txt" 2>&1; then + echo "expected an unqualified release set to be rejected" + exit 1 +fi +if [[ -e "${workdir}/docs/v0.7.0" || -e "${workdir}/fern/versions/v0.7.0.yml" || -e "${workdir}/docs/version-catalog/v0.7.0.yaml" ]]; then + echo "unqualified release set created a partial documentation snapshot" + exit 1 +fi + echo "test-cut-docs-version: PASS"