fix(ci): track database image releases

.github/workflows/chart-version-bump.yml

@@ -62,11 +62,13 @@ jobs:
           # fails any workflow that pins one.
           go-version-file: tools/go-toolchain/go.mod
 
-      - name: Test the bumper
-        # The bumper rewrites version fields in shipped charts, so its tests run
-        # here rather than somewhere that might not be reached. A test that
-        # gates nothing is not a test.
-        run: go test -C tools/chart-version-bumper ./...
+      - name: Test the chart release tools
+        # The bumper rewrites version fields in shipped charts, and the edge
+        # audit decodes the same ownership declarations. Exercise both schemas
+        # before either tool acts on a release.
+        run: |
+          go test -C tools/chart-version-bumper ./...
+          go test -C tools/chart-service-edge ./...
 
       - name: Test the commit type helper
         # It decides the semver step of the chart release this bump causes.
@@ -242,7 +244,7 @@ jobs:
           body="$(printf '%s\n' \
             "Opened by \`.github/workflows/chart-version-bump.yml\` when \`${TAG}\` was published." \
             "" \
-            "The released tag carries the version, so this is a direct update rather than a lookup of the newest published image." \
+            "The released tag identifies the version and source tree, so this is a direct update rather than a lookup of the newest published image." \
             "" \
             "Merging this does not move the self-managed stack. A chart version reaches the stack only once the chart itself is released, and publishing that chart release is what triggers \`stack-pin-bump.yml\`." \
             "" \

tools/chart-service-edge/main.go

@@ -33,6 +33,11 @@
 //
 //	"deploys": ["<service id>", ...]
 //
+// A multi-image chart instead uses an object with values_paths to identify the
+// exact fields owned by each service. It can use values_files for repeated pins
+// in additional values files and sets app_version when that service also owns
+// Chart.yaml's appVersion.
+//
 // listing the release-metadata ids of the services whose images it ships. A
 // chart that ships no first-party image (an upstream dependency, or resources
 // only) declares "deploys": [] to say so deliberately.

tools/chart-service-edge/main_test.go

@@ -213,3 +213,11 @@ func TestEmptyStringDeployEntryIsRejected(t *testing.T) {
 		t.Fatal("an empty string-form deploys entry must fail to decode")
 	}
 }
+
+func TestObjectDeployRequiresValuesPaths(t *testing.T) {
+	var m Metadata
+	err := json.Unmarshal([]byte(`{"services":[{"id":"c","path":"deploy/helm/c","deploys":[{"service":"svc"}]}]}`), &m)
+	if err == nil {
+		t.Fatal("an object deploy without values_paths must fail to decode")
+	}
+}

tools/chart-service-edge/metadata.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -34,12 +35,22 @@ type Entry struct {
 // decide which values.yaml line belongs to it. A chart with more than one
 // first-party image cannot use that evidence, so a deploy entry may instead
 // be an object naming the exact values.yaml paths (dotted, for example
-// "otelCollector.imageTag") that carry that service's tag. This tool only
-// audits which service ids are declared, so it does not care which shape a
-// given entry takes; UnmarshalJSON exists so decoding either shape succeeds.
+// "otelCollector.imageTag") that carry that service's tag. The object may set
+// values_files for repeated pins in other values files and app_version when the
+// same service owns the chart's appVersion. This tool only audits which service
+// ids are declared, so UnmarshalJSON exists primarily to make both shapes
+// available to that audit.
 type Deploy struct {
 	Service     string
 	ValuesPaths []string
+	ValuesFiles []ValuesFile
+	AppVersion  bool
+}
+
+// ValuesFile names an additional values file and the service-owned paths in it.
+type ValuesFile struct {
+	File  string   `json:"file"`
+	Paths []string `json:"paths"`
 }
 
 func (d *Deploy) UnmarshalJSON(b []byte) error {
@@ -52,16 +63,30 @@ func (d *Deploy) UnmarshalJSON(b []byte) error {
 		return nil
 	}
 	var obj struct {
-		Service     string   `json:"service"`
-		ValuesPaths []string `json:"values_paths"`
+		Service     string       `json:"service"`
+		ValuesPaths []string     `json:"values_paths"`
+		ValuesFiles []ValuesFile `json:"values_files"`
+		AppVersion  bool         `json:"app_version"`
 	}
 	if err := json.Unmarshal(b, &obj); err != nil {
 		return fmt.Errorf("deploys entry: %w", err)
 	}
 	if obj.Service == "" {
 		return fmt.Errorf("deploys entry missing \"service\"")
 	}
-	*d = Deploy{Service: obj.Service, ValuesPaths: obj.ValuesPaths}
+	if len(obj.ValuesPaths) == 0 && len(obj.ValuesFiles) == 0 {
+		return fmt.Errorf("object deploys entry for %s requires values_paths or values_files", obj.Service)
+	}
+	for _, valuesFile := range obj.ValuesFiles {
+		clean := filepath.Clean(valuesFile.File)
+		if filepath.IsAbs(valuesFile.File) || clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
+			return fmt.Errorf("deploys entry for %s: values_files file %q must be relative to the chart path", obj.Service, valuesFile.File)
+		}
+		if len(valuesFile.Paths) == 0 {
+			return fmt.Errorf("deploys entry for %s: values_files file %q requires paths", obj.Service, valuesFile.File)
+		}
+	}
+	*d = Deploy{Service: obj.Service, ValuesPaths: obj.ValuesPaths, ValuesFiles: obj.ValuesFiles, AppVersion: obj.AppVersion}
 	return nil
 }
 

tools/chart-version-bumper/artifact.go

@@ -0,0 +1,144 @@
+// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+const (
+	releasePlaceholder  = "${release}"
+	upstreamPlaceholder = "${upstream}"
+)
+
+// ArtifactVersion describes how a service release version becomes the version
+// embedded in the artifact it publishes. Most services need no declaration:
+// their artifact version is the release version. Wrapper images can combine an
+// upstream version from the released source tree with their release version.
+type ArtifactVersion struct {
+	SourceFile    string `json:"source_file"`
+	SourcePattern string `json:"source_pattern"`
+	Format        string `json:"format"`
+}
+
+// Release identifies a published service version and the artifact version a
+// chart must consume. Version remains the release version so chart semver
+// follows the released wrapper, not an upstream version embedded in its tag.
+type Release struct {
+	ServiceID       string
+	Version         string
+	ArtifactVersion string
+	entry           Entry
+}
+
+// ReleaseForTag resolves a release tag and, when declared, derives its artifact
+// version from the exact source tree the tag names.
+func (m *Metadata) ReleaseForTag(root, tag string) (Release, error) {
+	serviceID, version, err := m.ServiceForTag(tag)
+	if err != nil {
+		return Release{}, err
+	}
+	var entry Entry
+	for _, candidate := range m.Services {
+		if candidate.ID == serviceID {
+			entry = candidate
+			break
+		}
+	}
+	release := Release{ServiceID: serviceID, Version: version, ArtifactVersion: version, entry: entry}
+	if entry.ArtifactVersion == nil {
+		return release, nil
+	}
+	artifact, err := entry.ArtifactVersion.render(root, tag, entry.Path, version)
+	if err != nil {
+		return Release{}, fmt.Errorf("resolve artifact version for %s: %w", serviceID, err)
+	}
+	release.ArtifactVersion = artifact
+	return release, nil
+}
+
+func (a ArtifactVersion) render(root, tag, servicePath, version string) (string, error) {
+	if err := a.validate(); err != nil {
+		return "", err
+	}
+	upstream := ""
+	if strings.Contains(a.Format, upstreamPlaceholder) {
+		cleanSource := filepath.Clean(a.SourceFile)
+		if filepath.IsAbs(a.SourceFile) || cleanSource == "." || cleanSource == ".." || strings.HasPrefix(cleanSource, ".."+string(filepath.Separator)) {
+			return "", fmt.Errorf("source_file %q must name a relative file inside the service path", a.SourceFile)
+		}
+		path := filepath.Join(servicePath, cleanSource)
+		content, err := exec.Command("git", "-C", root, "show", tag+":"+filepath.ToSlash(path)).CombinedOutput()
+		if err != nil {
+			return "", fmt.Errorf("read %s from tag %s: %w: %s", path, tag, err, strings.TrimSpace(string(content)))
+		}
+		re := regexp.MustCompile(a.SourcePattern)
+		matches := re.FindAllSubmatch(content, -1)
+		index := re.SubexpIndex("upstream")
+		if len(matches) != 1 || index < 0 || index >= len(matches[0]) || len(matches[0][index]) == 0 {
+			return "", fmt.Errorf("source_pattern must match %s exactly once with a non-empty upstream capture", path)
+		}
+		upstream = string(matches[0][index])
+	}
+	artifact := strings.Replace(a.Format, releasePlaceholder, version, 1)
+	artifact = strings.Replace(artifact, upstreamPlaceholder, upstream, 1)
+	return artifact, nil
+}
+
+func (a ArtifactVersion) validate() error {
+	if strings.Count(a.Format, releasePlaceholder) != 1 {
+		return fmt.Errorf("format must contain %s exactly once", releasePlaceholder)
+	}
+	usesUpstream := strings.Contains(a.Format, upstreamPlaceholder)
+	if strings.Count(a.Format, upstreamPlaceholder) > 1 {
+		return fmt.Errorf("format may contain %s at most once", upstreamPlaceholder)
+	}
+	if usesUpstream != (a.SourceFile != "" && a.SourcePattern != "") {
+		return fmt.Errorf("source_file and source_pattern are required exactly when format uses %s", upstreamPlaceholder)
+	}
+	if strings.Contains(strings.ReplaceAll(strings.ReplaceAll(a.Format, releasePlaceholder, ""), upstreamPlaceholder, ""), "${") {
+		return fmt.Errorf("format contains an unknown placeholder")
+	}
+	if a.SourcePattern != "" {
+		re, err := regexp.Compile(a.SourcePattern)
+		if err != nil {
+			return fmt.Errorf("compile source_pattern: %w", err)
+		}
+		if re.SubexpIndex("upstream") < 0 {
+			return fmt.Errorf("source_pattern must define a named upstream capture")
+		}
+	}
+	return nil
+}
+
+// releaseVersionFromArtifact recovers the service release version from a
+// chart's current artifact tag. This keeps the workflow's major/minor/patch
+// decision based on the wrapper release even when the tag also embeds an
+// upstream version.
+func (a ArtifactVersion) releaseVersionFromArtifact(artifact string) (string, error) {
+	if err := a.validate(); err != nil {
+		return "", err
+	}
+	pattern := regexp.QuoteMeta(a.Format)
+	pattern = strings.Replace(pattern, regexp.QuoteMeta(upstreamPlaceholder), `.+?`, 1)
+	pattern = strings.Replace(pattern, regexp.QuoteMeta(releasePlaceholder), `(?P<release>.+?)`, 1)
+	re := regexp.MustCompile("^" + pattern + "$")
+	match := re.FindStringSubmatch(artifact)
+	index := re.SubexpIndex("release")
+	if match == nil || index < 0 || match[index] == "" {
+		return "", fmt.Errorf("artifact version %q does not match format %q", artifact, a.Format)
+	}
+	return match[index], nil
+}
+
+func (r Release) currentReleaseVersion(artifact string) (string, error) {
+	if r.entry.ArtifactVersion == nil {
+		return artifact, nil
+	}
+	return r.entry.ArtifactVersion.releaseVersionFromArtifact(artifact)
+}