Skip to content

feat: add build provenance tracking to engine images #796

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
k-rister merged 3 commits into from
Apr 16, 2026
Merged

feat: add build provenance tracking to engine images #796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
af69b94
feat: add build provenance tracking to engine images
k-rister Apr 15, 2026
42f0e30
fix: remove fedora userenv compat entry from workshop.json
k-rister Apr 15, 2026
ab3f5ea
fix: embed provenance file and annotations in all build stages
k-rister Apr 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions rickshaw-run
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2751,6 +2751,92 @@ debug_log (sprintf "image_ids (before):\n" . Dumper \%image_ids);
}
$source_input{'utility-dirs'} = \%utility_dirs;

# Build provenance data for engine image tracking
{
my %provenance;

# Build date
my $build_date = `date -u +%Y-%m-%dT%H:%M:%SZ`;
chomp $build_date;
$provenance{'build-date'} = $build_date;

# Source controller identification
my $hostname = `hostname -f 2>/dev/null || hostname`;
chomp $hostname;
my $ip = `ip -4 route get 1.0.0.0 2>/dev/null | grep -oP 'src \\K\\S+' || echo "unknown"`;
chomp $ip;
$provenance{'source'} = {
'hostname' => $hostname,
'ip' => $ip,
};

# Rickshaw client version (controller-side)
my $client_commit = `git -C $rickshaw_project_dir rev-parse HEAD 2>/dev/null`;
chomp $client_commit;
my %rickshaw_client = (
'commit' => $client_commit,
'dirty' => 'false',
);
my $client_dirty_rc = system("git -C $rickshaw_project_dir diff --quiet 2>/dev/null");
if ($client_dirty_rc != 0) {
$rickshaw_client{'dirty'} = 'true';
my $client_diff_hash = `git -C $rickshaw_project_dir diff 2>/dev/null | sha256sum | awk '{print \$1}'`;
chomp $client_diff_hash;
$rickshaw_client{'diff-hash'} = $client_diff_hash;
my $client_diff = `git -C $rickshaw_project_dir diff 2>/dev/null`;
$rickshaw_client{'diff'} = $client_diff;
}
$provenance{'rickshaw-client'} = \%rickshaw_client;

# Collect repo provenance (rickshaw excluded — tracked separately as client/server)
my %repos_provenance;
my %repo_paths = (
'toolbox' => $ENV{'TOOLBOX_HOME'},
'roadblock' => $run{'roadblock-dir'},
'workshop' => $run{'workshop-dir'},
);
# Add benchmark repos
foreach my $bench (keys %bench_dirs) {
$repo_paths{$bench} = $bench_dirs{$bench};
}
# Add utility repos
foreach my $util (keys %utility_dirs) {
$repo_paths{$util} = $utility_dirs{$util};
}

foreach my $name (sort keys %repo_paths) {
my $path = $repo_paths{$name};
next unless defined $path && -d $path;

my $commit = `git -C $path rev-parse HEAD 2>/dev/null`;
chomp $commit;
if ($commit eq '') {
log_print "WARNING: Could not get commit hash for '$name' at '$path'\n";
next;
}

my %repo_info = (
'commit' => $commit,
'dirty' => 'false',
);

my $dirty_rc = system("git -C $path diff --quiet 2>/dev/null");
if ($dirty_rc != 0) {
$repo_info{'dirty'} = 'true';
my $diff_hash = `git -C $path diff 2>/dev/null | sha256sum | awk '{print \$1}'`;
chomp $diff_hash;
$repo_info{'diff-hash'} = $diff_hash;
my $diff = `git -C $path diff 2>/dev/null`;
$repo_info{'diff'} = $diff;
}

$repos_provenance{$name} = \%repo_info;
}

$provenance{'repos'} = \%repos_provenance;
$source_input{'provenance'} = \%provenance;
}

my $source_input_file = $config_dir . "/image-source-input.json";
my $source_output_file = $config_dir . "/image-source-output.json";
if (put_json_file($source_input_file, \%source_input, $source_images_input_schema_file) > 0) {
Expand Down
4 changes: 4 additions & 0 deletions rickshaw-source-images-client
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,10 @@ def _build_api_request(input_data):
else:
raise SystemExit("ERROR: toolbox workshop.json not found: %s" % toolbox_workshop)

# --- Provenance ---
if "provenance" in input_data:
api_req["provenance"] = input_data["provenance"]

return api_req


Expand Down
87 changes: 86 additions & 1 deletion schema/source-images-input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
"workshop",
"roadblock-dir",
"cs-conf-file",
"use-workshop"
"use-workshop",
"provenance"
],
"additionalProperties": false,
"properties": {
Expand Down Expand Up @@ -141,6 +142,90 @@
"external-userenvs-dir": {
"description": "Optional path to a directory containing external userenv definitions.",
"type": "string"
},
"provenance": {
"description": "Build provenance tracking information for the engine image.",
"type": "object",
"required": ["build-date", "source", "rickshaw-client", "repos"],
"properties": {
"build-date": {
"type": "string",
"description": "ISO 8601 timestamp of when the build was initiated."
},
"source": {
"type": "object",
"description": "Identifying information about the controller that initiated the build.",
"required": ["hostname", "ip"],
"properties": {
"hostname": {
"type": "string",
"description": "Hostname of the controller."
},
"ip": {
"type": "string",
"description": "IP address of the controller."
}
},
"additionalProperties": false
},
"rickshaw-client": {
"type": "object",
"description": "Provenance for the controller-side rickshaw installation.",
"required": ["commit", "dirty"],
"properties": {
"commit": {
"type": "string",
"description": "Git commit hash of the rickshaw repo on the controller."
},
"dirty": {
"type": "string",
"enum": ["true", "false"],
"description": "Whether the repo has uncommitted changes."
},
"diff-hash": {
"type": "string",
"description": "SHA-256 hash of git diff output. Only present when dirty."
},
"diff": {
"type": "string",
"description": "Full git diff output. Only present when dirty."
}
},
"additionalProperties": false
},
"repos": {
"type": "object",
"description": "Map of repo names to their provenance data.",
"patternProperties": {
"^.+$": {
"type": "object",
"required": ["commit", "dirty"],
"properties": {
"commit": {
"type": "string",
"description": "Git commit hash (HEAD)."
},
"dirty": {
"type": "string",
"enum": ["true", "false"],
"description": "Whether the repo has uncommitted changes."
},
"diff-hash": {
"type": "string",
"description": "SHA-256 hash of git diff output. Only present when dirty is true."
},
"diff": {
"type": "string",
"description": "Full git diff output. Only present when dirty is true."
}
},
"additionalProperties": false
}
},
"additionalProperties": false
}
},
"additionalProperties": false
}
},
"$defs": {
Expand Down
141 changes: 134 additions & 7 deletions source-images-service/source_images_service/core/image_sourcer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,15 @@
logger = logging.getLogger(__name__)


def _write_cs_conf(path: Path, quay_label: str | None = None) -> None:
def _write_cs_conf(
path: Path,
quay_label: str | None = None,
annotations: list[str] | None = None,
) -> None:
"""Write the container-spec config JSON.

Called twice per stage: once without quay label (for hash calculation),
once with (for the actual build).
Called twice per stage: once without quay label/annotations (for hash
calculation), once with (for the actual build).
"""
cs_conf: dict[str, Any] = {
"workshop": {"schema": {"version": "2020.04.30"}},
Expand All @@ -52,6 +56,8 @@ def _write_cs_conf(path: Path, quay_label: str | None = None) -> None:
}
if quay_label is not None:
cs_conf["config"]["labels"] = [quay_label]
if annotations:
cs_conf["config"]["annotations"] = annotations

path.write_text(json.dumps(cs_conf, indent=2) + "\n")

Expand Down Expand Up @@ -282,6 +288,123 @@ def _source_container_image(
userenv, benchmark, workspace, request.utilities, request.use_workshop
)

# Generate provenance data for embedding in the final image
provenance_annotations: list[str] = []
provenance_req_file: Path | None = None

prov = request.provenance
provenance_annotations.append(
f"crucible.provenance.hostname={prov.source.hostname}"
)
provenance_annotations.append(
f"crucible.provenance.ip={prov.source.ip}"
)
rickshaw_client_str = prov.rickshaw_client.commit
if prov.rickshaw_client.dirty == "true":
rickshaw_client_str += f":dirty:{prov.rickshaw_client.diff_hash or 'unknown'}"
provenance_annotations.append(
f"crucible.provenance.rickshaw-client={rickshaw_client_str}"
)

# Get server-side rickshaw commit and dirty state for provenance
# Use the service's own installation path (not the workspace, which is materialized files)
server_commit = "unknown"
server_dirty = False
server_diff_hash = None
try:
import subprocess
service_dir = Path(__file__).resolve().parent.parent.parent.parent
result = subprocess.run(
["git", "-C", str(service_dir), "rev-parse", "HEAD"],
capture_output=True, text=True
)
if result.returncode == 0:
server_commit = result.stdout.strip()
dirty_result = subprocess.run(
["git", "-C", str(service_dir), "diff", "--quiet"],
capture_output=True
)
if dirty_result.returncode != 0:
server_dirty = True
hash_result = subprocess.run(
["git", "-C", str(service_dir), "diff"],
capture_output=True
)
if hash_result.returncode == 0:
import hashlib
server_diff_hash = hashlib.sha256(hash_result.stdout).hexdigest()
except Exception:
pass

rickshaw_server_str = server_commit
if server_dirty:
rickshaw_server_str += f":dirty:{server_diff_hash or 'unknown'}"
provenance_annotations.append(
f"crucible.provenance.rickshaw-server={rickshaw_server_str}"
)
for repo_name, repo_info in sorted(prov.repos.items()):
value = repo_info.commit
if repo_info.dirty == "true":
value += f":dirty:{repo_info.diff_hash or 'unknown'}"
provenance_annotations.append(
f"crucible.provenance.{repo_name}={value}"
)

# Build provenance JSON for embedding as a file in the image
provenance_data: dict[str, Any] = {
"build-date": prov.build_date,
"source": {
"hostname": prov.source.hostname,
"ip": prov.source.ip,
},
"rickshaw-client": {
"commit": prov.rickshaw_client.commit,
"dirty": prov.rickshaw_client.dirty,
"diff-hash": prov.rickshaw_client.diff_hash,
},
"rickshaw-server": {
"commit": server_commit,
"dirty": "true" if server_dirty else "false",
"diff-hash": server_diff_hash,
},
"repos": {},
}
for repo_name, repo_info in sorted(prov.repos.items()):
entry: dict[str, Any] = {
"commit": repo_info.commit,
"dirty": repo_info.dirty,
}
if repo_info.dirty == "true":
if repo_info.diff_hash:
entry["diff-hash"] = repo_info.diff_hash
if repo_info.diff:
entry["diff"] = repo_info.diff
provenance_data["repos"][repo_name] = entry

provenance_json_file = workspace.config / "build-provenance.json"
provenance_json_file.write_text(json.dumps(provenance_data, indent=2) + "\n")

# Generate a workshop requirement to copy provenance into the image
provenance_req = {
"workshop": {"schema": {"version": "2020.03.02"}},
"userenvs": [{"name": "default", "requirements": ["build-provenance"]}],
"requirements": [{
"name": "build-provenance",
"type": "files",
"files_info": {
"files": [{
"src": str(provenance_json_file),
"dst": "/etc/crucible/build-provenance.json",
}]
},
}],
}
provenance_req_file = workspace.config / "provenance-requirements.json"
provenance_req_file.write_text(json.dumps(provenance_req, indent=2) + "\n")
provenance_req_arg = f" --requirements {provenance_req_file}"

job.append_log(f"\tProvenance: {len(prov.repos)} repos tracked")

# Build staged workshop_args list
workshop_args: list[dict[str, str]] = []
userenv_arg = ""
Expand All @@ -291,14 +414,14 @@ def _source_container_image(
while len(requirements) > 0:
if count == 0:
userenv_arg = f" --userenv {userenv_file_path}"
req_arg = ""
req_arg = provenance_req_arg
update_policy = origin.get("update-policy", "")
if update_policy == "never":
skip_update = "true"
else:
skip_update = "false"
else:
req_arg = requirements.pop(0)
req_arg = requirements.pop(0) + provenance_req_arg
skip_update = "true"

# Write cs-conf.json without quay label (for hash calculation)
Expand All @@ -316,10 +439,14 @@ def _source_container_image(
job=job,
)

# Rewrite cs-conf.json with quay label
# Rewrite cs-conf.json with quay label (and annotations on last stage)
quay_exp = request.registries.get("public", reg)
exp_length = quay_exp.quay_expiration_length or ""
_write_cs_conf(cs_conf_file, quay_label=f"quay.expires-after={exp_length}")
_write_cs_conf(
cs_conf_file,
quay_label=f"quay.expires-after={exp_length}",
annotations=provenance_annotations,
)

workshop_args.append({
"userenv": userenv_arg,
Expand Down
Loading
Loading