Consume hydra-queue-runner/builder module from upstream repo

build/hydra-queue-runner.nix

@@ -0,0 +1,80 @@
+{
+  config,
+  inputs,
+  lib,
+  ...
+}:
+
+let
+  machines = [
+    "elated-minsky"
+    "goofy-hopcroft"
+    "hopeful-rivest"
+    "sleepy-brown"
+    "eager-heisenberg"
+    "enormous-catfish"
+    "growing-jennet"
+    "intense-heron"
+    "kind-lumiere"
+    "maximum-snail"
+    "sweeping-filly"
+  ];
+in
+{
+  imports = [
+    inputs.hydra-queue-runner.nixosModules.queue-runner
+  ];
+
+  age.secrets = lib.listToAttrs (
+    map (
+      machine:
+      lib.nameValuePair "${machine}-queue-runner-token" {
+        file = ./secrets/${machine}-queue-runner-token.age;
+      }
+    ) machines
+  );
+
+  services.nginx = {
+    enable = true;
+    virtualHosts."queue-runner.hydra.nixos.org" = {
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/".extraConfig = ''
+        # This is necessary so that grpc connections do not get closed early
+        # see https://stackoverflow.com/a/67805465
+        client_body_timeout 31536000s;
+        client_max_body_size 0;
+
+        grpc_pass grpc://${config.services.queue-runner-dev.grpc.address}:${toString config.services.queue-runner-dev.grpc.port};
+
+        grpc_read_timeout 31536000s; # 1 year in seconds
+        grpc_send_timeout 31536000s; # 1 year in seconds
+        grpc_socket_keepalive on;
+
+        grpc_set_header Host $host;
+        grpc_set_header X-Real-IP $remote_addr;
+        grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        grpc_set_header X-Forwarded-Proto $scheme;
+      '';
+    };
+  };
+
+  services.queue-runner-dev = {
+    enable = true;
+    settings = {
+      dbUrl = "postgres://[email protected]:5432/hydra";
+      machineFreeFn = "DynamicWithMaxJobLimit";
+      stepSortFn = "WithRdeps";
+      # dispatchTriggerTimerInS?
+      queueTriggerTimerInS = 60;
+      concurrentUploadLimit = 48;
+      maxConcurrentDownloads = 48;
+      remoteStoreAddr = [
+        "s3://nix-cache?secret-key=/var/lib/hydra/queue-runner/keys/cache.nixos.org-1/secret&write-nar-listing=1&compression=zstd&compression-level=19&ls-compression=zstd&log-compression=zstd&index-debug-info=true"
+      ];
+      rootsDir = "/nix/var/nix/gcroots/hydra";
+      tokenListPath = map (machine: config.age.secrets."${machine}-queue-runner-token".path) machines;
+    };
+  };
+}

build/hydra.nix

@@ -104,6 +104,8 @@ in
     evaluator_workers = 16
     evaluator_max_memory_size = 8192
 
+    queue_runner_endpoint = http://${config.services.queue-runner-dev.rest.address}:${toString config.services.queue-runner-dev.rest.port}
+
     max_concurrent_evals = 1
 
     # increase the number of active compress slots (CPU is 48*2 on mimas)

build/mimas/default.nix

@@ -3,6 +3,7 @@
     ../common.nix
     ../hydra.nix
     ../hydra-proxy.nix
+    ../hydra-queue-runner.nix
     ./boot.nix
     ./firewall.nix
     ./network.nix

build/secrets.nix

@@ -17,6 +17,54 @@ let
     rfc39-record-push = [ machines.pluto ];
     tarball-mirror-aws-credentials = [ machines.pluto ];
     zrepl-ssh-key = [ machines.titan ];
+
+    # builders/
+    elated-minsky-queue-runner-token = with machines; [
+      mimas
+      elated-minsky
+    ];
+    goofy-hopcroft-queue-runner-token = with machines; [
+      mimas
+      goofy-hopcroft
+    ];
+    hopeful-rivest-queue-runner-token = with machines; [
+      mimas
+      hopeful-rivest
+    ];
+    sleepy-brown-queue-runner-token = with machines; [
+      mimas
+      sleepy-brown
+    ];
+
+    # macs/
+    eager-heisenberg-queue-runner-token = with machines; [
+      mimas
+      eager-heisenberg
+    ];
+    enormous-catfish-queue-runner-token = with machines; [
+      mimas
+      enormous-catfish
+    ];
+    growing-jennet-queue-runner-token = with machines; [
+      mimas
+      growing-jennet
+    ];
+    intense-heron-queue-runner-token = with machines; [
+      mimas
+      intense-heron
+    ];
+    kind-lumiere-queue-runner-token = with machines; [
+      mimas
+      kind-lumiere
+    ];
+    maximum-snail-queue-runner-token = with machines; [
+      mimas
+      maximum-snail
+    ];
+    sweeping-filly-queue-runner-token = with machines; [
+      mimas
+      sweeping-filly
+    ];
   };
 in
 builtins.listToAttrs (

build/secrets/eager-heisenberg-queue-runner-token.age

@@ -0,0 +1,23 @@
+age-encryption.org/v1
+-> ssh-ed25519 cKT5Kw d2hBbAiEI7iLoP1c7WgXkJXnqfsy3GWPy23NZcHrb3A
+dIEVrctp2Ryu92cSBILUE+qeeLz0raQ1nTLGAPaZec4
+-> ssh-ed25519 NJQh8Q nThSL+PZmkUrXssS5YXqS1x4InMJMJKBma7/UpZcb3E
+WIVRniPt17W/GkOySUO/tFk0wlecxIMMZtcgV4caG0M
+-> ssh-ed25519 Gr9EaQ MTnHof1JOu4d5vObVatnKyhi20Da0K0v5TSyxhk7gwI
+YXIYyvGWR2cf6GJb7VL4aiu0gxKLyK1PyGhgw2vLJz8
+-> ssh-ed25519 3ENwVg rIi+Y4H0U+wkaO4zmIEbDd2Bd7tQnesw4yW+klqqQBM
+vd1c2lP+A5cyk2bfUoO09oPo49SnGzlXf95FrxuxRlA
+-> ssh-rsa MuWD+w
+moxeHv57SfIBrPVMvLiWZhh1qJHIii5maadnQZl8JUqjSDFpnPX4hXNIvwrqBau7
+Xn2X3tncgQ2Vp33757YembRDSOU7X06QASaRitxFrbHJu4iRIYwcyWoHbYn6jhPc
+9yK39sMNliHgZXDq2c0+DThV/PpvZd8yuVlP2oI5FqjlITjiFnTnJf+3c+uquc6v
+mxEwWUnrA8dSJD7RzcshW7swHu3FeC+MValEuiIQJaDlMUa211DhTGgtpSebuFrg
+Nlx+ZqS2k8LO2qAFyCemoMRMwod7VsCqtid6PxdEuwd8O0v7wfVafu0z+LCGMZoy
+SxKlCaVvDQJSzkAcj7EHvA
+-> ssh-ed25519 92bXiA bH6FYqVLVNbMBleHCALYbv7nykoIHcvaWlIvQnbyNRg
+joPDIXaqdMccBWdXvsvV9/ZlOVbE6pmrOFQ+WgUno68
+-> ssh-ed25519 Y121Gw kWm5O/sfXSAYRFsFWgKgWR3dUSKo2OFN5I0npz2x+TI
+wfbOq5meojODlRi3RZ+uFNokSPYLZNndB9nhp31wMTo
+--- /EhbVaVRVAyPOjTpmhTcRSh3kuyT/KoEkedwitZpTNk
+T,­h›vÙ	`÷Ÿ·HA´eGŠç_Ï jû<ÍO©¬7ü{’둍ˣ
+‰Å4ŸÐhãÊH1ên	QˆÚÜQHŸuÓ[YÿײNÑ)¿™UëeC"Ä7Q¿§1^vj]ò®Ááz^ìû³ûlí."uØ«+9„þQ”

build/secrets/goofy-hopcroft-queue-runner-token.age

@@ -0,0 +1,22 @@
+age-encryption.org/v1
+-> ssh-ed25519 cKT5Kw hA/K9EJyGfAbGbokosZGVEJqasHjE2bgr2EpEN4O/iQ
+7GaeyhJHezMSytl+75UzkiLvbxMpWSKoYb7aEH/D1qU
+-> ssh-ed25519 h7xPTg oBM3m/s0x5ue87LfgCOpyTfs0R0N4dmKwa7oW/R+nCQ
+HTxdFwkGtkCficUjMSe1bE95fv5gwMEvIlaNPb+LJvM
+-> ssh-ed25519 Gr9EaQ GdbCzg5bOJlVsTebVEE+y6StuiH1kZRG07D/bt1zuww
+EZqucrVkaX6ZTGJT0aiHmp4o9Z3IUIk82Df1Z2YkU5s
+-> ssh-ed25519 3ENwVg Ky1YIXGrt+UX5y745wePV1pulUHrr1yXzFRd+MHEITc
+BmWr551rvrtWl2PxD/+qYodybA0xA6Z/1Noza0te+Vo
+-> ssh-rsa MuWD+w
+RjaIoseiPazdSz75+ly66RqY0IhyQPBtltWLgGEYzhTkmzpnQNcUVpwgiPSzbt5X
+y7o+o+QPaHeds5suS42ZzUPahhLp1v5ehVaMXvsmqxkOZfODLxF3GGoFj4SG/YjJ
+aDd+bagUql7HX0cZRp51LpnitzOxayd8qeUZg51mqFi8uWV1DBSYrFdcVHBNeGuQ
+AbdUl9tqFtYilqcBJhCJOsKsiUsrX2bC6ZP8A6Pmt3gl8UR8nJLhD5TwQH6FCxDO
+iKbY21BwiKH8CJhQTNix6uwmTOwlX9mp8N6UNmqWuXB/3F4NmpyubnUvG9t0QGVl
+EsS5dlQ04JG/WrWDQpOR/w
+-> ssh-ed25519 92bXiA 7EaMly7GPo9fPETY606UO9in6bhbkQhgRxsO2u5Bgws
+IzeyNKnkYt8lwTk1TRxLooJJJmPFxIYZJAoDHm1Oqtg
+-> ssh-ed25519 Y121Gw 3tlRc4oDBLx1/Dn/KwnyUzg/odwMGLaFDksNB5RTqCk
+TJhtG/2/0PL7k84hQyAFEvLAFyZYP1W8erUpCANG7Mw
+--- mKpJ626SlxFTL7kt2BJOna043kiReyoMA8hl604J2hc
+H²&ÅéµÛ=LÐ÷Gž›*t2I¦‡îÁ˜SÌ_›ñûX(ýk£ÒÛ(Í4NàYtœrJ^K0b&—?å¨#ÕœÂS=1æ
jÉ°ƒŒVQÞ¨Fx±óQÕF³H¡g´â4o
sÖ	͇ªlÆ~±tg;!ï€%O

build/secrets/hopeful-rivest-queue-runner-token.age

@@ -0,0 +1,22 @@
+age-encryption.org/v1
+-> ssh-ed25519 cKT5Kw jz7oaOXlftKuXEIeFcFXacn0gcDuQhGkZRLmf0QTPXQ
+Br67PR4rBrZaKbP/X8X4vFkPq8L5IiNicvfXBvuaVdw
+-> ssh-ed25519 BaUP3w 8o3MNSWRhtrCgaqdQsBfmmg3LCAD9khNCXNlTAgegzE
+c137Ep8omrJBRcnqbRMwVB87CyB66u07qj5Xjor8hSY
+-> ssh-ed25519 Gr9EaQ tEa19teKlX3ZXJBOmBnOLU9GwnkDlfSdUzxaAMsY+3Y
+gWS3dYhg6psO0WNCD+s0kjqzapOnU4hQgWrcKh0iDbk
+-> ssh-ed25519 3ENwVg LiSqdv8ukjIjACQwk6203kkNotG+oRgGTkqsITRNjiU
+jOnUs9E5Tcu9eEnR8WXW277LZ+tRNyqM4b3Hg8EGu/8
+-> ssh-rsa MuWD+w
+enx5oiARoCPhm1D/MIdgIh2kjZFx4rxszCmW0j7RaS0SXDPu79c1QENwgemQdvLY
+uwX6teB+LkkWdcA6AFqY2FclopBRZq15OQuMoztBjwGPUIlk8H8OHrusViDJuGNm
+zdWsL4htncmTUWaX31V1ZX/v+KFl2Zp5Mmpn8x4C21wm5d42SOd5VRnw/OlziJGX
+gUG2DqLpoKzXDG9SAsKfk417Akfb8RtlVza6/tb57hThi9EsORK+BnTsUt6r6H84
+NvTuqnOJJFOEWqeRz1UjLij/gI10LQvcxCzhXC/SqkG7FaMXQ92WAZ5hH7AePSEE
+I/OlAU2wPj+GmPFePPODSA
+-> ssh-ed25519 92bXiA nYLjnIjeF+TmJbVdCtdqK042xnYDpF4naM1u7up31SI
+yVhUbve1xiySx+dqRcWdJQOYB2TRGdALa0l4hu1UnbM
+-> ssh-ed25519 Y121Gw kxYp6X5VV1QRwo1HrTUCbdBHgKMjkI2AUnUnqGe3dCE
+Rl2LfKLy9BQi47ktXCm+T7G6sbkBsuYaoxt5oTH2uPI
+--- X3Fr2TVxWyEW1hm8h7eKwGJHJg3BjywJddTp5OLolF4
+0vO®—smó̈ IKÍú'}"΃úªS¶*¼ß²|OÞçEü$õÑxÃ×W¾¦‹nå®ú;3ô˜”ƒtp%ôí­\XèG4lÓöBYÂë¨yœÿmìÃÇ® >†“´æ´k×ð+Šªëݲ±»ÎR¹om)³þ`

build/secrets/kind-lumiere-queue-runner-token.age

@@ -0,0 +1,23 @@
+age-encryption.org/v1
+-> ssh-ed25519 cKT5Kw 8g2rqFnJ23pFpD4PniCDMPiueSroGH2yShkpHtPvZDc
+ZyYcqRHGP4H4ElRs3rNAOzJ7In3MnVT8/2NcLHga8Ho
+-> ssh-ed25519 jPdm4A k+8PUnPBFILqbb0Ikf2DMJEYVsLPwDtjYgQ6dVyNenc
+e1mhAEQhzVsnznBJRsMEp3gYOO00Gmf4BCvHsXpFELU
+-> ssh-ed25519 Gr9EaQ P0yT0M8e8ihKqossmqnIJc6074NXZ8KJmVL03BN7eV0
+GHWdPlIDCMFf7Pca4GXfRnhZ2NJAmM0doPsMThY+iVQ
+-> ssh-ed25519 3ENwVg UzvZZ0rFG3KaPQ6G6Oq4U/EQ3RRmPxyo6xF0tgadDDs
+vPUm8mpqVeiBGpxGUTnYACn7tOQDcuFP3E2gWLToyXY
+-> ssh-rsa MuWD+w
+qSOhRpEjjuMyt+nRRC8Yd1fInXTReZqLCp6GZoRnYbO69a1AIQwU1HU5CtAHbVFe
+8dIerlh4deN/T6wW3EvxM5hAA5co7kV68t3fgHGyQBdVGJvPuQRWaduSv21O/wbv
+epmGODM9YwFfnPMDHXqTzt+NYEJIJoUVpH1YTTfeZDyoRza2gJ5hoSPFXtomVHL4
+lO1+wcldYuELgY8bCeZpFP0kPmK7STYTa7LZxEF/yjqM2ZXhS6qOTV2+yRZhSKEy
+RizOnW0ePWrCSIVvxIr4+sGlKW5cwAqeatxiPZz7/3RFSxHBG9RC/ZZEmaZUF9Er
+cjILgCnk3lZJDnmpU6/+JA
+-> ssh-ed25519 92bXiA 4jz8lFxCSjJBJKWZTtxYruYiuQuJytQ8utDYZccQwFY
+zdLlneAU2P7zjDCC6tWVjySgJctB4Y5VXwEkvzqjhoU
+-> ssh-ed25519 Y121Gw Bhy7yX2r7RWBeS/K0bMVwXbvzYVAW88pzOHVtTKKIVQ
+Q9wuHdoI4SRXmjSA7iUUljjcO6dzPublR79rvPSlTlg
+--- 2DnKmT2R9XL5DR6z7+amRi5Y/8GphgkifpngTogcU/A
+âˆ½,n‡¦Ý”"¨Ö%¶bÐKDµ ÊƒW‘v¾
ôºæ‚6ˆÜ#ÜK4(Jfí*|ÚùãßN¸(ªÌÍÉèÚçâ:Æg¨Yð}ZuR1.©œEÉtñÿPôkMÉ
+ë;k¬³è©œs滁c†~ë|éȶ

build/secrets/sleepy-brown-queue-runner-token.age

@@ -0,0 +1,23 @@
+age-encryption.org/v1
+-> ssh-ed25519 cKT5Kw r8aZ+OCr9AE4h0zattrGpFPwBcnb28/Mj7vNC5EEHDE
+SaN75cMS6o0bcuIzeKF8siNu0P7rvJN4DLnL0R07t3M
+-> ssh-ed25519 le38mA 0syXJIHthuMy1Y6LbrfQX1QcADyJMOfmFbwzf3cQlHM
+X9HHBlfYBG64Awu+TZaA463Om18A7kSu7pMYwIDkehk
+-> ssh-ed25519 Gr9EaQ Wqex4/CIJTL+sm5GAlb0Du8mIjDz3QmvO7veYAQ+nmo
+o//67CmR5wPgSzLuF4exx4mW+FstyQunBqeDgs9HUk8
+-> ssh-ed25519 3ENwVg 5XF6k6rMk59p53Hw6nSak8iajZ7XzLJ5jOQ7aPwkdng
++YUOjq/VopumkLhVshF4GdzkjqO1aNMrfkx3TZaPtaA
+-> ssh-rsa MuWD+w
+gsSEjSCIFzKTsOXvJay3Ij9OpefMoAGL7AjXW1mQ4TvCVWO5M7gqYLrlgANKwMGK
+sm9tpNtncFn7hC7G3YWBOU/InMIQ/qlgL5jhRBhZpou/DKMtDA+IDVZJYvSQMcT1
+9467zxSpFtnjrmzW/6cnX3jjLlTRCc4AupoS1pMIeJ2gwZBNiCklS+QGPQTQiG/O
+oF1nA0h/08pCbrLHIwilhFmekDzg99EesiZ3Hbqc7+kz8kbaIV9iUqFsRvV1Dwzm
+K6wIQXf5nhcCkt/SAFSS/ZwwHOr19B0OR3t6L4dYMa+bl/LxW0yXYzvMo4rp07Mn
+oXFd+BuBEwzHI1x8wrTmUQ
+-> ssh-ed25519 92bXiA +t2D5pUYWeTRPTT7vrNYZirRUWKQO0gw5RB3o+CV0yk
+b5DsQ3FUMO14U7NB7H4G9ngpw5gfPTrYXIKa7yy5Wq4
+-> ssh-ed25519 Y121Gw X0D49VhFJ2kZqJATUmuKhJfQ6TIAZCkWDl2u6dqnQSk
+O0JtjZWXrS/NY/FXYB14kM3MpuoAaTd2Bf1oWw7REc4
+--- a+IPhlc1ru44iR5eHXGVe0X2fqgcSj03Lk1lyB3sZZg
+ˆ¿*‡]HHX![‡(+Fñ;8ûši¨O¦çÔU‘’Å&J'67˜þ=›I²aý1ÆÐSž6
+.ËÁep!4‚´d²ÉîuLú*‰ýßÇDWG˜<ëbG~`Þ£Ùڭ˃D£€]¹c§¯¡å¥m’|#\Ymôä;

builders/common/hydra-queue-builder.nix

@@ -0,0 +1,22 @@
+{
+  config,
+  inputs,
+  ...
+}:
+
+{
+  imports = [
+    inputs.hydra-queue-runner.nixosModules.queue-builder
+  ];
+
+  age.secrets."queue-runner-token" = {
+    file = ../../build/secrets/${config.networking.hostName}-queue-runner-token.age;
+  };
+
+  services.queue-builder-dev = {
+    enable = true;
+    queueRunnerAddr = "https://queue-runner.hydra.nixos.org";
+    authorizationFile = config.age.secrets."queue-runner-token".path;
+    maxJobs = config.nix.settings.max-jobs;
+  };
+}

builders/flake-module.nix

@@ -7,13 +7,17 @@
         inputs.nixpkgs.lib.nixosSystem {
           inherit system;
 
+          specialArgs = { inherit inputs; };
+
           modules = [
+            inputs.agenix.nixosModules.age
             inputs.disko.nixosModules.disko
 
             ./common/hardening.nix
             ./common/network.nix
             ./common/nix.nix
             ./common/node-exporter.nix
+            ./common/hydra-queue-builder.nix
             ./common/system.nix
             ./common/tools.nix
             ./common/update.nix
@@ -37,4 +41,14 @@
       # Ampere Q80-30 (80C), 128 GB DDR4 RAM, 2x960GB PCIe4 NVME
       hopeful-rivest = mkNixOS "aarch64-linux" ./instances/hopeful-rivest.nix;
     };
+
+  perSystem =
+    { pkgs, inputs', ... }:
+    {
+      devShells.builders = pkgs.mkShell {
+        buildInputs = [
+          inputs'.agenix.packages.agenix
+        ];
+      };
+    };
 }

dns/nixos.org.js

@@ -72,6 +72,7 @@ D("nixos.org",
 	A("mimas", "157.90.104.34"),
 	AAAA("mimas", "2a01:4f8:2220:11c8::1"),
 	CNAME("hydra", "mimas"),
+	CNAME("queue-runner.hydra", "mimas"),
 
 	A("pluto", "37.27.99.100"),
 	AAAA("pluto", "2a01:4f9:3070:15e0::1"),