build(frr): skip the libgccjit build on x86_64 glibc

`fancy.libatomic` is one file copied out of a full libgccjit build:
gcc from source, the longest step in an image build.  No FRR object
on x86_64 glibc references libatomic (every atomic is inlined), yet
FRR linked it, named it in its buildInputs and kept it through
nuke-refs, so gcc was built anyway.  Worse, `fancy.libxcrypt` is
overridden per sanitizer, which gives libgccjit a new hash in every
sanitizer flavour: an ASan or TSan build rebuilds gcc.

Stop linking libatomic on x86_64 glibc and pass it to FRR only where
it is linked.  That takes two cuts: our own -latomic in LDFLAGS, and
the -latomic configure appends to LIBS when the build toolchain has
one (which would otherwise leave a DT_NEEDED nothing in the closure
provides).  `--as-needed` cannot do this for us: libtool hoists the
`-l` flags of a shared-library link ahead of every `-Wl,` flag.  A
postInstall check fails the build if any output still names
libatomic.  aarch64 and musl are unchanged.

The x86_64 glibc FRR build graph (default, release and ASan) loses
libgccjit and libatomic entirely.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Daniel Noland <daniel@githedgehog.com>

Author: daniel-noland

nix/overlays/frr.nix

@@ -10,6 +10,9 @@
 }:
 final: prev:
 let
+  # No FRR object references libatomic on x86_64 glibc; unverified on aarch64 and musl, which
+  # keep linking it.
+  drop-libatomic = final.stdenv.hostPlatform.isx86_64 && libc == "gnu";
   dep =
     pkg:
     (pkg.override { stdenv = final.stdenv'; }).overrideAttrs (orig: {
@@ -36,6 +39,9 @@ let
         // {
           stdenv = final.stdenv';
           inherit frrSrc vtysh-extensions;
+          # `fancy.libatomic` is copied out of a full libgccjit (gcc) build; do not ask for it
+          # where FRR does not link libatomic.
+          libatomic = if drop-libatomic then null else final.fancy.libatomic;
         }
       )).overrideAttrs
         (orig: {
@@ -63,8 +69,14 @@ let
             #     backs the cross-musl clang stdenv (note: stdenv, not
             #     stdenv' -- the prime is the clang stdenv whose cc.cc is
             #     clang and contains no libatomic at all).
+            #
+            # On x86_64 glibc no FRR object references libatomic at all (every atomic is inlined),
+            # so it is not linked there.  `--as-needed` cannot do this for us: libtool hoists the
+            # `-l` flags of a shared-library link ahead of every `-Wl,` flag.
             + (
-              if libc == "musl" then
+              if drop-libatomic then
+                ""
+              else if libc == "musl" then
                 " -L${final.stdenv.cc.cc.lib}/${final.stdenv.hostPlatform.config}/lib -latomic "
               else if libc == "gnu" then
                 " -L${final.fancy.libatomic}/lib -latomic "
@@ -83,6 +95,22 @@ let
             # this overrides the base package's --enable-static-bin.
             "--disable-static-bin"
           ];
+          # Where configure finds a libatomic (the build toolchain has one), it appends -latomic to
+          # LIBS, which ends every link and would leave a DT_NEEDED nothing in the closure provides.
+          postConfigure =
+            (orig.postConfigure or "")
+            + final.lib.optionalString drop-libatomic ''
+              sed -E -i '/^LIBS = /s/ -latomic\b//g' Makefile
+              grep -E '^LIBS = ' Makefile
+            '';
+          postInstall =
+            (orig.postInstall or "")
+            + final.lib.optionalString drop-libatomic ''
+              if grep -rl 'libatomic\.so' "$out"; then
+                echo "FRR still needs libatomic, which is not in its closure" >&2
+                exit 1
+              fi
+            '';
           # `buildPackages.nukeReferences` rather than `prev.nukeReferences`:
           # the `nuke-refs` script substitutes a perl path at build time, and
           # under a cross pkgset `prev.nukeReferences` picks up target-arch
@@ -104,7 +132,7 @@ let
                 -e ${final.fancy.readline} \
                 -e ${final.fancy.libxcrypt} \
                 -e ${final.fancy.json_c} \
-                ${if libc == "gnu" then "-e ${final.fancy.libatomic}" else ""} \
+                ${if libc == "gnu" && !drop-libatomic then "-e ${final.fancy.libatomic}" else ""} \
                 ${if libc == "musl" then "-e ${final.stdenv.cc.cc.lib}" else ""} \
                 '{}' +;
           '';

nix/pkgs/frr/default.nix

@@ -24,7 +24,8 @@
   pcre2,
   readline,
   rtrlib,
-  libatomic,
+  # null where FRR does not link libatomic at all (see `drop-libatomic` in nix/overlays/frr.nix)
+  libatomic ? null,
 
   # other general options besides snmp support
   numMultipath ? 8,
@@ -127,13 +128,14 @@ stdenv.mkDerivation (finalAttrs: {
     python3Minimal
     readline
   ]
-  # libatomic.so.1 is a DT_NEEDED of libfrr.so and of every daemon.  On glibc
-  # it comes from `fancy.libatomic`, a re-export of the single file FRR needs
-  # out of libgccjit (see the note on that attribute in nix/overlays/frr.nix).
-  # On musl FRR pulls libatomic from the cross-musl gcc-libs output via
-  # `stdenv.cc.cc.lib` in the overlay instead, so naming it here would only
-  # bloat the build closure without contributing any symbol.
-  ++ lib.optionals stdenv.hostPlatform.isGnu [ libatomic ]
+  # Where FRR links libatomic, libatomic.so.1 is a DT_NEEDED of libfrr.so and of
+  # every daemon.  On glibc it comes from `fancy.libatomic`, a re-export of the
+  # single file FRR needs out of libgccjit (see the note on that attribute in
+  # nix/overlays/frr.nix).  On musl FRR pulls libatomic from the cross-musl
+  # gcc-libs output via `stdenv.cc.cc.lib` in the overlay instead, so naming it
+  # here would only bloat the build closure without contributing any symbol.
+  # On x86_64 glibc it is not linked at all, and naming it would still build gcc.
+  ++ lib.optionals (stdenv.hostPlatform.isGnu && libatomic != null) [ libatomic ]
   ++ lib.optionals bgpRpki [ rtrlib ];
 
   # cross-compiling: clippy is compiled with the build host toolchain, split it out to ease