ci: shard cargo-bench by package onto small dedicated agents

Splits the nightly cargo-bench step into 9 parallel Buildkite jobs, each
sharded by package rather than by target so a shard builds and runs one
package's full dependency closure at the ancestor and at HEAD. This lets
the step run on small dedicated 8cpu/32gb machines instead of a single
48-core box, and shards without an assigned package exit as no-ops. The
mzcompose allowlist in the plugin hook now matches the label by prefix
since parallel jobs append an index suffix to BUILDKITE_LABEL.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Author: antiguru

ci/nightly/pipeline.template.yml

@@ -304,6 +304,7 @@ steps:
         # Annotation-only while the noise floor on this hardware is being
         # calibrated. Remove after 2026-09-18 so regressions fail the build.
         soft_fail: true
+        parallelism: 9
         inputs:
           - Cargo.lock
           - Cargo.toml
@@ -314,9 +315,10 @@ steps:
               composition: cargo-bench
               ci-builder: stable
         agents:
-          # Dedicated hardware for stable timings. 48 cores because the step
-          # builds the bench targets twice in the bench profile.
-          queue: hetzner-x86-64-dedi-48cpu-192gb
+          # Dedicated hardware for stable timings, one shard per package with
+          # bench targets, so `parallelism` should track that package count.
+          # Extra shards exit as no-ops, fewer shards double up packages.
+          queue: hetzner-x86-64-dedi-8cpu-32gb
 
       - id: workload-replay
         topics: [iceberg, kafka, mysql, postgres, sql-server, ssh-tunnel]

ci/plugins/mzcompose/hooks/command

@@ -400,7 +400,6 @@ cleanup() {
 
   if [ ! -s services.log ] \
     && [ "$BUILDKITE_LABEL" != ":rust: cargo-fuzz" ] \
-    && [ "$BUILDKITE_LABEL" != ":rust: Cargo bench against ancestor" ] \
     && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of persist" ] \
     && [ "$BUILDKITE_LABEL" != "Long single-node Maelstrom coverage of persist" ] \
     && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of txn-wal" ] \
@@ -413,7 +412,8 @@ cleanup() {
     && [ "$BUILDKITE_LABEL" != ":rust: Miri test (full)" ] \
     && [[ ! "$BUILDKITE_LABEL" =~ Terraform\ .* ]] \
     && [[ ! "$BUILDKITE_LABEL" =~ Orchestratord\ .* ]] \
-    && [[ ! "$BUILDKITE_LABEL" =~ Cluster\ spec\ sheet.* ]]; then
+    && [[ ! "$BUILDKITE_LABEL" =~ Cluster\ spec\ sheet.* ]] \
+    && [[ ! "$BUILDKITE_LABEL" =~ ^:rust:\ Cargo\ bench\ against\ ancestor ]]; then
       echo "+++ services.log is empty, failing"
       if [[ $CI_ANNOTATE_ERRORS_RESULT -ne 0 ]]; then
         # Keep the retryable code (128 for GHCR/DockerHub trouble, 199 for a

test/cargo-bench/mzcompose.py

@@ -297,6 +297,16 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
     if not head_targets:
         raise RuntimeError("no bench targets found")
 
+    packages = sorted({t.package for t in head_targets})
+    # Sharding by package rather than by target keeps each shard to one
+    # dependency closure per checkout.
+    shard_packages = buildkite.shard_list(packages, lambda p: p)
+    if not shard_packages:
+        print("--- No bench packages assigned to this shard")
+        return
+    print(f"--- Bench packages in this shard: {', '.join(shard_packages)}")
+    head_targets = [t for t in head_targets if t.package in shard_packages]
+
     criterion_home = target_dir() / "criterion-compare"
     # Stale baselines from an earlier run would silently become the
     # comparison target, so start from an empty directory every time.
@@ -351,7 +361,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
                 env, CARGO_TARGET_DIR=str(target_dir() / "cargo-bench-ancestor")
             )
             ancestor_targets, ancestor_manifests = load_targets(
-                worktree, args.package, ancestor_env
+                worktree, shard_packages, ancestor_env
             )
             # Building ancestor before HEAD lets both build phases run at
             # full core parallelism back to back, then the run phase below
@@ -397,7 +407,9 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
     failed = report.has_regressions or bool(current_failures)
     if buildkite.is_in_buildkite():
         buildkite.add_annotation(
-            "error" if failed else "info", "Cargo bench results", markdown
+            "error" if failed else "info",
+            f"Cargo bench results ({', '.join(shard_packages)})",
+            markdown,
         )
         try:
             spawn.runv(