build(nix): bound how many ThinLTO links a test archive runs at once

Under cross-language ThinLTO each test binary's link optimizes the
whole program, Rust and DPDK together, and holds gigabytes while it
does. Cargo cannot see that: `-j` caps rustc processes, any of which
may be linking, so how many links overlap is left to chance -- and an
unlucky overlap is what keeps running `check/release` out of its
20 GiB. Building `tests.all` at `profile=release` with unbounded links
on 10 cores peaked at 36.2 GiB with nine ld.lld processes resident at
once. Turning down `--thinlto-jobs` only shrinks every link to make
that overlap survivable, and the links keep growing with the code.

Bound the overlap instead. rustc now links through
nix/scripts/link-slots.sh, which, when `LINK_SLOTS` is set, takes one
of that many flock slots before running the real linker and holds it
until the linker exits; `LINK_THINLTO_JOBS` gives each bounded link
more backend threads. Unset, it is transparent, and only test archives
whose profile uses `-Clinker-plugin-lto` (`release`) set them: two
slots, four threads each. They are environment rather than flags, so
they stay out of cargo's fingerprint.

Measured with line-table debug info on 8 cores, three runs each:

    2 slots x 4 threads   611-616 s   peak 13.2-14.1 GiB
    1 slot  x 8 threads   913-935 s   peak 10.4-12.1 GiB

One slot puts the links on the critical path for little memory.

A link waiting for a slot still has its rustc parent resident, with
its whole compilation session; the next commit releases that.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Daniel Noland <daniel@githedgehog.com>

Author: daniel-noland

default.nix

@@ -889,6 +889,23 @@ let
   is-cross-compile = pkgs.stdenv'.buildPlatform.rust.rustcTarget != ctarget;
   cc = if is-cross-compile then "${ctarget}-clang" else "clang";
   strip = if is-cross-compile then "${ctarget}-strip" else "strip";
+  # What rustc links with: the cross or native clang, behind nix/scripts/link-slots.sh, which bounds
+  # how many links run at once where a build sets `LINK_SLOTS` (see `test-builder`) and is
+  # transparent everywhere else.
+  #
+  # The `-clang` is for rustc, which infers how to drive a linker from its name. Under a name it
+  # does not recognize it falls back to the target's default, which on x86_64-unknown-linux-gnu and
+  # aarch64-unknown-linux-gnu adds rustc's own self-contained lld (`-fuse-ld=lld` and its `gcc-ld`
+  # directory). Ending in `-clang` gets the link line it gives clang itself, on every target.
+  link-slots = pkgs.pkgsBuildHost.writeShellApplication {
+    name = "link-slots-clang";
+    runtimeInputs = with pkgs.pkgsBuildHost; [
+      coreutils
+      util-linux
+    ];
+    runtimeEnv.LINK_CC = "${pkgs.pkgsBuildHost.llvmPackages'.clang}/bin/${cc}";
+    text = builtins.readFile ./nix/scripts/link-slots.sh;
+  };
   objcopy = if is-cross-compile then "${ctarget}-objcopy" else "objcopy";
   package-list = builtins.fromJSON (
     builtins.readFile (
@@ -1021,7 +1038,7 @@ let
               builtins.concatStringsSep " " (
                 profile.RUSTFLAGS
                 ++ [
-                  "-Clinker=${pkgs.pkgsBuildHost.llvmPackages'.clang}/bin/${cc}"
+                  "-Clinker=${link-slots}/bin/link-slots-clang"
                   "-Clink-arg=--ld-path=${pkgs.pkgsBuildHost.llvmPackages'.lld}/bin/ld.lld"
                   "-Clink-arg=-L${sysroot}/lib"
                   # Keep debug paths stable across revisions. Source readers
@@ -1391,6 +1408,23 @@ let
         cargo-features == [ ]
         && !(builtins.elem "coverage" instrumentations)
         && (package == null || package == "dataplane-init");
+      # Under cross-language ThinLTO every test binary's link optimizes the whole program, and
+      # cargo cannot see how many of those overlap. Bound them (nix/scripts/link-slots.sh): two at a
+      # time, four backend threads each. Measured building `tests.all` at `profile=release` with
+      # line-table debug info on 8 cores, three runs each:
+      #
+      #     links unbounded (10 cores, full debug info)  514 s   36.2 GiB peak
+      #     2 slots x 4 threads                          614 s   13.2-14.1 GiB
+      #     1 slot  x 8 threads                          924 s   10.4-12.1 GiB
+      #
+      # One slot makes the links the critical path for little memory; two keep the peak bounded
+      # without that.
+      lto-link-slots =
+        pkgs.lib.optionalAttrs (builtins.elem "-Clinker-plugin-lto" profile-tests'.RUSTFLAGS)
+          {
+            LINK_SLOTS = "2";
+            LINK_THINLTO_JOBS = "4";
+          };
     in
     pkgs.callPackage invoke {
       builder = craneLib.mkCargoDerivation;
@@ -1401,6 +1435,7 @@ let
       // pkgs.lib.optionalAttrs wants-gateway {
         DATAPLANE_TEST_GATEWAY_ROOTFS = "${gateway-rootfs}";
       }
+      // lto-link-slots
       // {
         buildPhaseCargoCommand =
           (builtins.concatStringsSep " " (

nix/profiles.nix

@@ -76,9 +76,9 @@ let
     "-Cdwarf-version=5"
     "-Csymbol-mangling-version=v0"
     "-Clink-arg=-Wl,--as-needed,--gc-sections" # FRR builds don't like this, but rust does fine
-    # The backend threads of one ThinLTO link, each holding modules of its own. CI runs up to
-    # `lto_cores` such links at once in a 20 GiB cgroup, so the peak is roughly the product; 4 left
-    # too little headroom.
+    # The backend threads of one ThinLTO link, each holding modules of its own, for links nothing
+    # bounds: up to `lto_cores` of them can overlap in CI's 20 GiB cgroup. A test archive bounds its
+    # links instead and gives each more (`LINK_SLOTS` and `LINK_THINLTO_JOBS` in default.nix).
     "-Clink-arg=-Wl,--thinlto-jobs=2"
   ]
   ++ (

nix/scripts/link-slots.sh

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Open Network Fabric Authors
+
+# The linker rustc runs (`-Clinker`), in front of the real one, `LINK_CC`.
+#
+# Every ThinLTO link runs an ld.lld that optimizes the whole program, Rust and the C dependencies
+# together, and holds gigabytes while it does. Cargo cannot see that: `-j` caps how many rustc
+# processes run, and any of them may be linking, so how many links overlap is left to chance -- and
+# an unlucky overlap is what runs a build out of memory. With `LINK_SLOTS` set, a link first takes
+# one of that many slots and keeps it until the linker exits; the others wait their turn.
+#
+# `LINK_THINLTO_JOBS` sets each link's ThinLTO backend threads: with the links bounded, the ones
+# that run can use more.
+#
+# Unset (or `LINK_SLOTS=0`), this is transparent. Both are environment rather than flags so that
+# they stay out of cargo's fingerprint: builds with and without them share dependency artifacts.
+
+extra=()
+if [ -n "${LINK_THINLTO_JOBS:-}" ]; then
+  # lld takes the last `--thinlto-jobs`, so this overrides the profile's.
+  extra+=("-Wl,--thinlto-jobs=${LINK_THINLTO_JOBS}")
+fi
+
+if [ "${LINK_SLOTS:-0}" -gt 0 ]; then
+  # One directory per build: the nix sandbox gives each its own `NIX_BUILD_TOP`.
+  dir="${NIX_BUILD_TOP:-${TMPDIR:-/tmp}}/link-slots"
+  mkdir -p "${dir}"
+  while true; do
+    for ((slot = 1; slot <= LINK_SLOTS; slot++)); do
+      exec {fd}> "${dir}/${slot}"
+      if flock --nonblock "${fd}"; then
+        # The lock is held by the open descriptor, which the linker inherits through `exec`, so
+        # the slot is released when the linker exits, however it exits.
+        exec "${LINK_CC}" "$@" "${extra[@]}"
+      fi
+      exec {fd}>&-
+    done
+    sleep 0.2
+  done
+fi
+
+exec "${LINK_CC}" "$@" "${extra[@]}"