build(nix): link a test archive's binaries in a rustc of their own

With links bounded, the memory left at the peak is mostly rustc: a
link waiting for its slot is a rustc process that has finished
compiling and is waiting on its linker, with its whole compilation
session -- a gigabyte or two -- still resident. Cargo's `-j` keeps that
to one per job slot, but the slots are spent holding memory nobody
uses.

rustc can stop after writing its objects (`-Zno-link`, which leaves an
`.rlink` describing the link) and do the link in a separate invocation
(`-Zlink-only`). nix/scripts/rustc-split-link.sh, a `RUSTC_WRAPPER`, does
that for the crates that link (binaries and test harnesses), so the
process waiting on a slot is a small one. Everything else passes
straight through. It is set where links are bounded, the
`-Clinker-plugin-lto` test archives; `RUSTC_WRAPPER` is not part of
cargo's fingerprint, so the dependency artifacts built without it are
reused (the archive compiled the same 69 crates either way).

Building `tests.all` at `profile=release`, two link slots, 8 cores:
peak 13.2-14.1 -> 11.3 GiB, mean 6.5-6.8 -> 4.2 GiB, 611-616 -> 620 s.
The split archive's suite passed (3074 of 3075; the one failure, a
Unix socket path longer than SUN_LEN under a long worktree path, fails
the same way without the split).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Daniel Noland <daniel@githedgehog.com>

Author: daniel-noland

default.nix

@@ -906,6 +906,13 @@ let
     runtimeEnv.LINK_CC = "${pkgs.pkgsBuildHost.llvmPackages'.clang}/bin/${cc}";
     text = builtins.readFile ./nix/scripts/link-slots.sh;
   };
+  # A `RUSTC_WRAPPER` that compiles the crates that link and links them in separate rustc processes,
+  # so that one waiting for a link slot is not holding a compilation session (see the script).
+  rustc-split-link = pkgs.pkgsBuildHost.writeShellApplication {
+    name = "rustc-split-link";
+    runtimeInputs = [ pkgs.pkgsBuildHost.coreutils ];
+    text = builtins.readFile ./nix/scripts/rustc-split-link.sh;
+  };
   objcopy = if is-cross-compile then "${ctarget}-objcopy" else "objcopy";
   package-list = builtins.fromJSON (
     builtins.readFile (
@@ -1419,11 +1426,18 @@ let
       #
       # One slot makes the links the critical path for little memory; two keep the peak bounded
       # without that.
+      #
+      # A link waiting for its slot still has its rustc resident, compilation session and all, so
+      # link in a separate rustc (nix/scripts/rustc-split-link.sh). With two slots on 8 cores that
+      # took the peak from 13.2-14.1 to 11.3 GiB and the mean from 6.5-6.8 to 4.2 GiB, at the same
+      # 620 s. `RUSTC_WRAPPER` stays out of cargo's fingerprint, so the dependency artifacts built
+      # without it are reused.
       lto-link-slots =
         pkgs.lib.optionalAttrs (builtins.elem "-Clinker-plugin-lto" profile-tests'.RUSTFLAGS)
           {
             LINK_SLOTS = "2";
             LINK_THINLTO_JOBS = "4";
+            RUSTC_WRAPPER = "${rustc-split-link}/bin/rustc-split-link";
           };
     in
     pkgs.callPackage invoke {

nix/scripts/rustc-split-link.sh

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Open Network Fabric Authors
+
+# The `RUSTC_WRAPPER` of a test archive whose links are bounded (see link-slots.sh).
+#
+# A link waiting for its slot is a rustc process that has finished compiling and is waiting on its
+# linker, with its whole compilation session -- a gigabyte or two -- still resident. So for the
+# crates that link (binaries and test harnesses), compile with `-Zno-link`, which writes the objects
+# and an `.rlink` describing the link and then exits, freeing all of that; then link in a second,
+# small rustc with `-Zlink-only`, given the same arguments with the `.rlink` in place of the source.
+# Libraries, build scripts and cargo's own probes (`rustc -vV`) pass straight through. The `-Z`
+# flags need `RUSTC_BOOTSTRAP=1`, which these builds set.
+
+rustc="$1"
+shift
+
+links=false
+name=
+out_dir=
+extra=
+src=
+prev=
+for arg in "$@"; do
+  case "${prev}" in
+    --crate-name) name="${arg}" ;;
+    --out-dir) out_dir="${arg}" ;;
+    --crate-type) if [ "${arg}" = bin ]; then links=true; fi ;;
+    -C) case "${arg}" in extra-filename=*) extra="${arg#extra-filename=}" ;; esac ;;
+  esac
+  case "${arg}" in
+    --test | --crate-type=bin) links=true ;;
+    -Cextra-filename=*) extra="${arg#-Cextra-filename=}" ;;
+    --out-dir=*) out_dir="${arg#--out-dir=}" ;;
+    *.rs) if [ -z "${src}" ] && [ -f "${arg}" ]; then src="${arg}"; fi ;;
+  esac
+  prev="${arg}"
+done
+# Build scripts link too, but natively, without LTO, and cargo runs them as soon as they are built.
+case "${name}" in build_script_*) links=false ;; esac
+
+if ! "${links}" || [ -z "${src}" ] || [ -z "${out_dir}" ] || [ -z "${name}" ]; then
+  exec "${rustc}" "$@"
+fi
+
+"${rustc}" "$@" -Zno-link
+
+rlink="${out_dir}/${name}${extra}.rlink"
+if [ ! -f "${rlink}" ]; then
+  printf 'rustc-split-link: -Zno-link left no %s\n' "${rlink}" >&2
+  exit 1
+fi
+
+args=()
+for arg in "$@"; do
+  if [ "${arg}" = "${src}" ]; then
+    args+=("${rlink}")
+  else
+    args+=("${arg}")
+  fi
+done
+status=0
+"${rustc}" "${args[@]}" -Zlink-only || status=$?
+rm -f "${rlink}"
+exit "${status}"