diff --git a/build/hydra-queue-runner.nix b/build/hydra-queue-runner.nix new file mode 100644 index 00000000..61fe897e --- /dev/null +++ b/build/hydra-queue-runner.nix @@ -0,0 +1,80 @@ +{ + config, + inputs, + lib, + ... +}: + +let + machines = [ + "elated-minsky" + "goofy-hopcroft" + "hopeful-rivest" + "sleepy-brown" + "eager-heisenberg" + "enormous-catfish" + "growing-jennet" + "intense-heron" + "kind-lumiere" + "maximum-snail" + "sweeping-filly" + ]; +in +{ + imports = [ + inputs.hydra-queue-runner.nixosModules.queue-runner + ]; + + age.secrets = lib.listToAttrs ( + map ( + machine: + lib.nameValuePair "${machine}-queue-runner-token" { + file = ./secrets/${machine}-queue-runner-token.age; + } + ) machines + ); + + services.nginx = { + enable = true; + virtualHosts."queue-runner.hydra.nixos.org" = { + enableACME = true; + forceSSL = true; + + locations."/".extraConfig = '' + # This is necessary so that grpc connections do not get closed early + # see https://stackoverflow.com/a/67805465 + client_body_timeout 31536000s; + client_max_body_size 0; + + grpc_pass grpc://${config.services.queue-runner-dev.grpc.address}:${toString config.services.queue-runner-dev.grpc.port}; + + grpc_read_timeout 31536000s; # 1 year in seconds + grpc_send_timeout 31536000s; # 1 year in seconds + grpc_socket_keepalive on; + + grpc_set_header Host $host; + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + grpc_set_header X-Forwarded-Proto $scheme; + ''; + }; + }; + + services.queue-runner-dev = { + enable = true; + settings = { + dbUrl = "postgres://hydra@10.0.40.3:5432/hydra"; + machineFreeFn = "DynamicWithMaxJobLimit"; + stepSortFn = "WithRdeps"; + # dispatchTriggerTimerInS? + queueTriggerTimerInS = 60; + concurrentUploadLimit = 48; + maxConcurrentDownloads = 48; + remoteStoreAddr = [ + "s3://nix-cache?secret-key=/var/lib/hydra/queue-runner/keys/cache.nixos.org-1/secret&write-nar-listing=1&compression=zstd&compression-level=19&ls-compression=zstd&log-compression=zstd&index-debug-info=true" + ]; + rootsDir = "/nix/var/nix/gcroots/hydra"; + tokenListPath = map (machine: config.age.secrets."${machine}-queue-runner-token".path) machines; + }; + }; +} diff --git a/build/hydra.nix b/build/hydra.nix index c7fcbbb6..b2c79bb7 100644 --- a/build/hydra.nix +++ b/build/hydra.nix @@ -104,6 +104,8 @@ in evaluator_workers = 16 evaluator_max_memory_size = 8192 + queue_runner_endpoint = http://${config.services.queue-runner-dev.rest.address}:${toString config.services.queue-runner-dev.rest.port} + max_concurrent_evals = 1 # increase the number of active compress slots (CPU is 48*2 on mimas) diff --git a/build/mimas/default.nix b/build/mimas/default.nix index 4694e7f5..dd6df4b7 100644 --- a/build/mimas/default.nix +++ b/build/mimas/default.nix @@ -3,6 +3,7 @@ ../common.nix ../hydra.nix ../hydra-proxy.nix + ../hydra-queue-runner.nix ./boot.nix ./firewall.nix ./network.nix diff --git a/build/secrets.nix b/build/secrets.nix index db079967..c79a11b8 100644 --- a/build/secrets.nix +++ b/build/secrets.nix @@ -17,6 +17,54 @@ let rfc39-record-push = [ machines.pluto ]; tarball-mirror-aws-credentials = [ machines.pluto ]; zrepl-ssh-key = [ machines.titan ]; + + # builders/ + elated-minsky-queue-runner-token = with machines; [ + mimas + elated-minsky + ]; + goofy-hopcroft-queue-runner-token = with machines; [ + mimas + goofy-hopcroft + ]; + hopeful-rivest-queue-runner-token = with machines; [ + mimas + hopeful-rivest + ]; + sleepy-brown-queue-runner-token = with machines; [ + mimas + sleepy-brown + ]; + + # macs/ + eager-heisenberg-queue-runner-token = with machines; [ + mimas + eager-heisenberg + ]; + enormous-catfish-queue-runner-token = with machines; [ + mimas + enormous-catfish + ]; + growing-jennet-queue-runner-token = with machines; [ + mimas + growing-jennet + ]; + intense-heron-queue-runner-token = with machines; [ + mimas + intense-heron + ]; + kind-lumiere-queue-runner-token = with machines; [ + mimas + kind-lumiere + ]; + maximum-snail-queue-runner-token = with machines; [ + mimas + maximum-snail + ]; + sweeping-filly-queue-runner-token = with machines; [ + mimas + sweeping-filly + ]; }; in builtins.listToAttrs ( diff --git a/build/secrets/eager-heisenberg-queue-runner-token.age b/build/secrets/eager-heisenberg-queue-runner-token.age new file mode 100644 index 00000000..92e6547b --- /dev/null +++ b/build/secrets/eager-heisenberg-queue-runner-token.age @@ -0,0 +1,23 @@ +age-encryption.org/v1 +-> ssh-ed25519 cKT5Kw d2hBbAiEI7iLoP1c7WgXkJXnqfsy3GWPy23NZcHrb3A +dIEVrctp2Ryu92cSBILUE+qeeLz0raQ1nTLGAPaZec4 +-> ssh-ed25519 NJQh8Q nThSL+PZmkUrXssS5YXqS1x4InMJMJKBma7/UpZcb3E +WIVRniPt17W/GkOySUO/tFk0wlecxIMMZtcgV4caG0M +-> ssh-ed25519 Gr9EaQ MTnHof1JOu4d5vObVatnKyhi20Da0K0v5TSyxhk7gwI +YXIYyvGWR2cf6GJb7VL4aiu0gxKLyK1PyGhgw2vLJz8 +-> ssh-ed25519 3ENwVg rIi+Y4H0U+wkaO4zmIEbDd2Bd7tQnesw4yW+klqqQBM +vd1c2lP+A5cyk2bfUoO09oPo49SnGzlXf95FrxuxRlA +-> ssh-rsa MuWD+w +moxeHv57SfIBrPVMvLiWZhh1qJHIii5maadnQZl8JUqjSDFpnPX4hXNIvwrqBau7 +Xn2X3tncgQ2Vp33757YembRDSOU7X06QASaRitxFrbHJu4iRIYwcyWoHbYn6jhPc +9yK39sMNliHgZXDq2c0+DThV/PpvZd8yuVlP2oI5FqjlITjiFnTnJf+3c+uquc6v +mxEwWUnrA8dSJD7RzcshW7swHu3FeC+MValEuiIQJaDlMUa211DhTGgtpSebuFrg +Nlx+ZqS2k8LO2qAFyCemoMRMwod7VsCqtid6PxdEuwd8O0v7wfVafu0z+LCGMZoy +SxKlCaVvDQJSzkAcj7EHvA +-> ssh-ed25519 92bXiA bH6FYqVLVNbMBleHCALYbv7nykoIHcvaWlIvQnbyNRg +joPDIXaqdMccBWdXvsvV9/ZlOVbE6pmrOFQ+WgUno68 +-> ssh-ed25519 Y121Gw kWm5O/sfXSAYRFsFWgKgWR3dUSKo2OFN5I0npz2x+TI +wfbOq5meojODlRi3RZ+uFNokSPYLZNndB9nhp31wMTo +--- /EhbVaVRVAyPOjTpmhTcRSh3kuyT/KoEkedwitZpTNk +T,hv `HAe G_ j ssh-ed25519 cKT5Kw hA/K9EJyGfAbGbokosZGVEJqasHjE2bgr2EpEN4O/iQ +7GaeyhJHezMSytl+75UzkiLvbxMpWSKoYb7aEH/D1qU +-> ssh-ed25519 h7xPTg oBM3m/s0x5ue87LfgCOpyTfs0R0N4dmKwa7oW/R+nCQ +HTxdFwkGtkCficUjMSe1bE95fv5gwMEvIlaNPb+LJvM +-> ssh-ed25519 Gr9EaQ GdbCzg5bOJlVsTebVEE+y6StuiH1kZRG07D/bt1zuww +EZqucrVkaX6ZTGJT0aiHmp4o9Z3IUIk82Df1Z2YkU5s +-> ssh-ed25519 3ENwVg Ky1YIXGrt+UX5y745wePV1pulUHrr1yXzFRd+MHEITc +BmWr551rvrtWl2PxD/+qYodybA0xA6Z/1Noza0te+Vo +-> ssh-rsa MuWD+w +RjaIoseiPazdSz75+ly66RqY0IhyQPBtltWLgGEYzhTkmzpnQNcUVpwgiPSzbt5X +y7o+o+QPaHeds5suS42ZzUPahhLp1v5ehVaMXvsmqxkOZfODLxF3GGoFj4SG/YjJ +aDd+bagUql7HX0cZRp51LpnitzOxayd8qeUZg51mqFi8uWV1DBSYrFdcVHBNeGuQ +AbdUl9tqFtYilqcBJhCJOsKsiUsrX2bC6ZP8A6Pmt3gl8UR8nJLhD5TwQH6FCxDO +iKbY21BwiKH8CJhQTNix6uwmTOwlX9mp8N6UNmqWuXB/3F4NmpyubnUvG9t0QGVl +EsS5dlQ04JG/WrWDQpOR/w +-> ssh-ed25519 92bXiA 7EaMly7GPo9fPETY606UO9in6bhbkQhgRxsO2u5Bgws +IzeyNKnkYt8lwTk1TRxLooJJJmPFxIYZJAoDHm1Oqtg +-> ssh-ed25519 Y121Gw 3tlRc4oDBLx1/Dn/KwnyUzg/odwMGLaFDksNB5RTqCk +TJhtG/2/0PL7k84hQyAFEvLAFyZYP1W8erUpCANG7Mw +--- mKpJ626SlxFTL7kt2BJOna043kiReyoMA8hl604J2hc +H&=LG*t2I˜S_X(k(4NYtrJ^K0b&?#՜S=1jɰVQިFxQFHg4os ͇l~tg;!%O \ No newline at end of file diff --git a/build/secrets/growing-jennet-queue-runner-token.age b/build/secrets/growing-jennet-queue-runner-token.age new file mode 100644 index 00000000..89ccb996 Binary files /dev/null and b/build/secrets/growing-jennet-queue-runner-token.age differ diff --git a/build/secrets/hopeful-rivest-queue-runner-token.age b/build/secrets/hopeful-rivest-queue-runner-token.age new file mode 100644 index 00000000..78c05957 --- /dev/null +++ b/build/secrets/hopeful-rivest-queue-runner-token.age @@ -0,0 +1,22 @@ +age-encryption.org/v1 +-> ssh-ed25519 cKT5Kw jz7oaOXlftKuXEIeFcFXacn0gcDuQhGkZRLmf0QTPXQ +Br67PR4rBrZaKbP/X8X4vFkPq8L5IiNicvfXBvuaVdw +-> ssh-ed25519 BaUP3w 8o3MNSWRhtrCgaqdQsBfmmg3LCAD9khNCXNlTAgegzE +c137Ep8omrJBRcnqbRMwVB87CyB66u07qj5Xjor8hSY +-> ssh-ed25519 Gr9EaQ tEa19teKlX3ZXJBOmBnOLU9GwnkDlfSdUzxaAMsY+3Y +gWS3dYhg6psO0WNCD+s0kjqzapOnU4hQgWrcKh0iDbk +-> ssh-ed25519 3ENwVg LiSqdv8ukjIjACQwk6203kkNotG+oRgGTkqsITRNjiU +jOnUs9E5Tcu9eEnR8WXW277LZ+tRNyqM4b3Hg8EGu/8 +-> ssh-rsa MuWD+w +enx5oiARoCPhm1D/MIdgIh2kjZFx4rxszCmW0j7RaS0SXDPu79c1QENwgemQdvLY +uwX6teB+LkkWdcA6AFqY2FclopBRZq15OQuMoztBjwGPUIlk8H8OHrusViDJuGNm +zdWsL4htncmTUWaX31V1ZX/v+KFl2Zp5Mmpn8x4C21wm5d42SOd5VRnw/OlziJGX +gUG2DqLpoKzXDG9SAsKfk417Akfb8RtlVza6/tb57hThi9EsORK+BnTsUt6r6H84 +NvTuqnOJJFOEWqeRz1UjLij/gI10LQvcxCzhXC/SqkG7FaMXQ92WAZ5hH7AePSEE +I/OlAU2wPj+GmPFePPODSA +-> ssh-ed25519 92bXiA nYLjnIjeF+TmJbVdCtdqK042xnYDpF4naM1u7up31SI +yVhUbve1xiySx+dqRcWdJQOYB2TRGdALa0l4hu1UnbM +-> ssh-ed25519 Y121Gw kxYp6X5VV1QRwo1HrTUCbdBHgKMjkI2AUnUnqGe3dCE +Rl2LfKLy9BQi47ktXCm+T7G6sbkBsuYaoxt5oTH2uPI +--- X3Fr2TVxWyEW1hm8h7eKwGJHJg3BjywJddTp5OLolF4 +0vOsm̈IK'}"΃S*߲|OE$xWn;3tp%\XG4lBYymǮ > k+ݲRom)` \ No newline at end of file diff --git a/build/secrets/intense-heron-queue-runner-token.age b/build/secrets/intense-heron-queue-runner-token.age new file mode 100644 index 00000000..d67fe54e Binary files /dev/null and b/build/secrets/intense-heron-queue-runner-token.age differ diff --git a/build/secrets/kind-lumiere-queue-runner-token.age b/build/secrets/kind-lumiere-queue-runner-token.age new file mode 100644 index 00000000..1db82fff --- /dev/null +++ b/build/secrets/kind-lumiere-queue-runner-token.age @@ -0,0 +1,23 @@ +age-encryption.org/v1 +-> ssh-ed25519 cKT5Kw 8g2rqFnJ23pFpD4PniCDMPiueSroGH2yShkpHtPvZDc +ZyYcqRHGP4H4ElRs3rNAOzJ7In3MnVT8/2NcLHga8Ho +-> ssh-ed25519 jPdm4A k+8PUnPBFILqbb0Ikf2DMJEYVsLPwDtjYgQ6dVyNenc +e1mhAEQhzVsnznBJRsMEp3gYOO00Gmf4BCvHsXpFELU +-> ssh-ed25519 Gr9EaQ P0yT0M8e8ihKqossmqnIJc6074NXZ8KJmVL03BN7eV0 +GHWdPlIDCMFf7Pca4GXfRnhZ2NJAmM0doPsMThY+iVQ +-> ssh-ed25519 3ENwVg UzvZZ0rFG3KaPQ6G6Oq4U/EQ3RRmPxyo6xF0tgadDDs +vPUm8mpqVeiBGpxGUTnYACn7tOQDcuFP3E2gWLToyXY +-> ssh-rsa MuWD+w +qSOhRpEjjuMyt+nRRC8Yd1fInXTReZqLCp6GZoRnYbO69a1AIQwU1HU5CtAHbVFe +8dIerlh4deN/T6wW3EvxM5hAA5co7kV68t3fgHGyQBdVGJvPuQRWaduSv21O/wbv +epmGODM9YwFfnPMDHXqTzt+NYEJIJoUVpH1YTTfeZDyoRza2gJ5hoSPFXtomVHL4 +lO1+wcldYuELgY8bCeZpFP0kPmK7STYTa7LZxEF/yjqM2ZXhS6qOTV2+yRZhSKEy +RizOnW0ePWrCSIVvxIr4+sGlKW5cwAqeatxiPZz7/3RFSxHBG9RC/ZZEmaZUF9Er +cjILgCnk3lZJDnmpU6/+JA +-> ssh-ed25519 92bXiA 4jz8lFxCSjJBJKWZTtxYruYiuQuJytQ8utDYZccQwFY +zdLlneAU2P7zjDCC6tWVjySgJctB4Y5VXwEkvzqjhoU +-> ssh-ed25519 Y121Gw Bhy7yX2r7RWBeS/K0bMVwXbvzYVAW88pzOHVtTKKIVQ +Q9wuHdoI4SRXmjSA7iUUljjcO6dzPublR79rvPSlTlg +--- 2DnKmT2R9XL5DR6z7+amRi5Y/8GphgkifpngTogcU/A +,nݔ"%bKDʃWv6#K4(Jf*|N(:gY}ZuR1.EtPkM +;k詜s滁c~|ȶ \ No newline at end of file diff --git a/build/secrets/maximum-snail-queue-runner-token.age b/build/secrets/maximum-snail-queue-runner-token.age new file mode 100644 index 00000000..1f130edb Binary files /dev/null and b/build/secrets/maximum-snail-queue-runner-token.age differ diff --git a/build/secrets/sleepy-brown-queue-runner-token.age b/build/secrets/sleepy-brown-queue-runner-token.age new file mode 100644 index 00000000..6d24aebb --- /dev/null +++ b/build/secrets/sleepy-brown-queue-runner-token.age @@ -0,0 +1,23 @@ +age-encryption.org/v1 +-> ssh-ed25519 cKT5Kw r8aZ+OCr9AE4h0zattrGpFPwBcnb28/Mj7vNC5EEHDE +SaN75cMS6o0bcuIzeKF8siNu0P7rvJN4DLnL0R07t3M +-> ssh-ed25519 le38mA 0syXJIHthuMy1Y6LbrfQX1QcADyJMOfmFbwzf3cQlHM +X9HHBlfYBG64Awu+TZaA463Om18A7kSu7pMYwIDkehk +-> ssh-ed25519 Gr9EaQ Wqex4/CIJTL+sm5GAlb0Du8mIjDz3QmvO7veYAQ+nmo +o//67CmR5wPgSzLuF4exx4mW+FstyQunBqeDgs9HUk8 +-> ssh-ed25519 3ENwVg 5XF6k6rMk59p53Hw6nSak8iajZ7XzLJ5jOQ7aPwkdng ++YUOjq/VopumkLhVshF4GdzkjqO1aNMrfkx3TZaPtaA +-> ssh-rsa MuWD+w +gsSEjSCIFzKTsOXvJay3Ij9OpefMoAGL7AjXW1mQ4TvCVWO5M7gqYLrlgANKwMGK +sm9tpNtncFn7hC7G3YWBOU/InMIQ/qlgL5jhRBhZpou/DKMtDA+IDVZJYvSQMcT1 +9467zxSpFtnjrmzW/6cnX3jjLlTRCc4AupoS1pMIeJ2gwZBNiCklS+QGPQTQiG/O +oF1nA0h/08pCbrLHIwilhFmekDzg99EesiZ3Hbqc7+kz8kbaIV9iUqFsRvV1Dwzm +K6wIQXf5nhcCkt/SAFSS/ZwwHOr19B0OR3t6L4dYMa+bl/LxW0yXYzvMo4rp07Mn +oXFd+BuBEwzHI1x8wrTmUQ +-> ssh-ed25519 92bXiA +t2D5pUYWeTRPTT7vrNYZirRUWKQO0gw5RB3o+CV0yk +b5DsQ3FUMO14U7NB7H4G9ngpw5gfPTrYXIKa7yy5Wq4 +-> ssh-ed25519 Y121Gw X0D49VhFJ2kZqJATUmuKhJfQ6TIAZCkWDl2u6dqnQSk +O0JtjZWXrS/NY/FXYB14kM3MpuoAaTd2Bf1oWw7REc4 +--- a+IPhlc1ru44iR5eHXGVe0X2fqgcSj03Lk1lyB3sZZg +*]HHX![(+F;8 iOU&J'67=Ia1S6 +.ep!4‚duL*DWG