feat: init hydra-queue-runner-v2

dns/nixos.org.js

@@ -96,6 +96,7 @@ D("nixos.org",
 	// hydra staging area
 	A("staging-hydra", "157.180.25.203"),
 	AAAA("staging-hydra", "2a01:4f9:c012:d5d3::1"),
+	CNAME("queue-runner.staging-hydra", "staging-hydra"),
 
 	// services infra
 	A("caliban", "65.109.26.213"),

non-critical-infra/hosts/staging-hydra/ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBnTCCAU+gAwIBAgIUQpxYsPwAyTY70yYO9fcCmCaZreIwBQYDK2VwMEMxCzAJ
+BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
+cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
+WjBDMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExHjAcBgNVBAMM
+FWh5ZHJhLXF1ZXVlLXJ1bm5lci1jYTAqMAUGAytlcAMhAM+Mc/XSTXwJeWPxrpqo
+SPT5Xwi8/j85VO6TsfBlXFt4o1MwUTAdBgNVHQ4EFgQU0wQG6BxTKtYwlywuyD0a
+Vr/1r4gwHwYDVR0jBBgwFoAU0wQG6BxTKtYwlywuyD0aVr/1r4gwDwYDVR0TAQH/
+BAUwAwEB/zAFBgMrZXADQQA3BRP2+TkkDQPnPy6MQyDCxqfEeV6OQjtspSvCO0UL
+GWmfvzrlUQytwTFTPfVzaErbyVPbeYU5y8rmRoGPNSoI
+-----END CERTIFICATE-----

non-critical-infra/hosts/staging-hydra/client.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBlDCCAUagAwIBAgIUfUYjDOaJML1lIMkAMvLjnSTscYwwBQYDK2VwMEMxCzAJ
+BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
+cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
+WjBLMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExJjAkBgNVBAMM
+HWh5ZHJhLXF1ZXVlLWJ1aWxkZXItbG9jYWxob3N0MCowBQYDK2VwAyEAJqCvjdwY
+0l4z0cbwecONx3DfhpPo+SBymv7H2OQt4aejQjBAMB0GA1UdDgQWBBQw54bw+CrK
+O53J4qzB4Cknzf7CzTAfBgNVHSMEGDAWgBTTBAboHFMq1jCXLC7IPRpWv/WviDAF
+BgMrZXADQQB+XUCZQo0aX6IyEQDrds1d3SWF1rFBv5MjtZS25TYpnQQPgt9iiD6Q
+dtCwSVqyAgCG4XlkYBpU4AKVKqyMsSAC
+-----END CERTIFICATE-----

non-critical-infra/hosts/staging-hydra/default.nix

@@ -5,6 +5,8 @@
     inputs.srvos.nixosModules.server
     inputs.srvos.nixosModules.hardware-hetzner-cloud-arm
     ../../modules/common.nix
+    ../../modules/hydra-queue-runner-v2.nix
+    ../../modules/hydra-queue-builder-v2.nix
     ./hydra-proxy.nix
     ./hydra.nix
     inputs.hydra.nixosModules.hydra

non-critical-infra/hosts/staging-hydra/genca.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+set -x
+
+hosts="localhost ofborg-eval02 ofborg-eval03 ofborg-eval04 ofborg-build01 ofborg-build02 ofborg-build03 ofborg-build04 ofborg-build05"
+
+C="DE"
+O="NixOS Infra"
+
+newDir="$(date '+%Y-%m-%dT%H:%M')"
+mkdir "${newDir}"
+cd "${newDir}" || exit
+
+openssl genpkey -algorithm Ed25519 -out ca.key
+openssl req -x509 -new -nodes -key ca.key -sha256 -days 18250 -out ca.crt \
+  -subj "/C=${C}/O=${O}/CN=hydra-queue-runner-ca"
+
+cat <<EOF >server.cnf
+[req]
+prompt             = no
+x509_extensions    = v3_req
+req_extensions     = v3_req
+default_md         = sha256
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+C  = ${C}
+O  = ${O}
+CN = queue-runner.staging-hydra.nixos.org
+
+[v3_req]
+basicConstraints = CA:FALSE
+keyUsage         = nonRepudiation, digitalSignature, keyEncipherment, keyAgreement
+extendedKeyUsage = critical, serverAuth
+subjectAltName   = @alt_names
+
+[alt_names]
+DNS.1 = queue-runner.staging-hydra.nixos.org
+EOF
+
+openssl genpkey -algorithm Ed25519 -out server.key
+openssl req -new -key server.key -out server.csr -config server.cnf
+openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 18250 -sha256 -extfile server.cnf -extensions v3_req
+
+for host in ${hosts}; do
+  openssl genpkey -algorithm Ed25519 -out "client-${host}.key"
+  openssl req -new -key "client-${host}.key" -out "client-${host}.csr" \
+    -subj "/C=${C}/O=${O}/CN=hydra-queue-builder-${host}"
+  openssl x509 -req -in "client-${host}.csr" -CA ca.crt -CAkey ca.key -CAcreateserial -out "client-${host}.crt" -days 18250 -sha256
+done
+
+rm -rf -- *.csr *.srl
+rm server.cnf
+
+cd - || exit

non-critical-infra/hosts/staging-hydra/hydra.nix

@@ -24,8 +24,11 @@ in
     dates = "03,09,15,21:15";
   };
 
-  # gc outputs as well, since they are served from the cache
-  nix.settings.gc-keep-outputs = lib.mkForce false;
+  nix.settings = {
+    # gc outputs as well, since they are served from the cache
+    gc-keep-outputs = lib.mkForce false;
+    allowed-users = [ "hydra-www" ];
+  };
 
   # Don't rate-limit the journal.
   services.journald.rateLimitBurst = 0;
@@ -44,67 +47,136 @@ in
     };
   };
 
-  services.hydra-dev = {
-    enable = true;
-    package = pkgs.hydra;
-    buildMachinesFiles = [
-      (pkgs.writeText "local" ''
-        localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - -
-      '')
-    ];
-    logo = ../../../build/hydra-logo.png;
-    hydraURL = "https://hydra.nixos.org";
-    notificationSender = "[email protected]";
-    smtpHost = "localhost";
-    useSubstitutes = true;
-    extraConfig = ''
-      max_servers 30
+  services = {
+    hydra-dev = {
+      enable = true;
+      package = pkgs.hydra;
+      buildMachinesFiles = [
+        (pkgs.writeText "local" ''
+          localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - -
+        '')
+      ];
+      logo = ../../../build/hydra-logo.png;
+      hydraURL = "https://hydra.nixos.org";
+      notificationSender = "[email protected]";
+      smtpHost = "localhost";
+      useSubstitutes = true;
+      extraConfig = ''
+        max_servers 30
 
-      store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br
-      server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache}
-      binary_cache_public_uri = https://cache-staging.nixos.org
+        store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br
+        server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache}
+        binary_cache_public_uri = https://cache-staging.nixos.org
 
-      <Plugin::Session>
-        cache_size = 32m
-      </Plugin::Session>
+        <Plugin::Session>
+          cache_size = 32m
+        </Plugin::Session>
 
-      # patchelf:master:3
-      xxx-jobset-repeats = nixos:reproducibility:1
+        # patchelf:master:3
+        xxx-jobset-repeats = nixos:reproducibility:1
 
-      upload_logs_to_binary_cache = true
-      compress_build_logs = false  # conflicts with upload_logs_to_binary_cache
+        upload_logs_to_binary_cache = true
+        compress_build_logs = false  # conflicts with upload_logs_to_binary_cache
 
-      log_prefix = https://cache.nixos.org/
+        log_prefix = https://cache.nixos.org/
 
-      evaluator_workers = 1
-      evaluator_max_memory_size = 4096
+        evaluator_workers = 1
+        evaluator_max_memory_size = 4096
 
-      max_concurrent_evals = 1
+        queue_runner_endpoint = http://localhost:8080
 
-      # increase the number of active compress slots (CPU is 48*2 on mimas)
-      max_local_worker_threads = 144
+        max_concurrent_evals = 1
 
-      max_unsupported_time = 86400
+        # increase the number of active compress slots (CPU is 48*2 on mimas)
+        max_local_worker_threads = 144
 
-      allow_import_from_derivation = false
+        max_unsupported_time = 86400
 
-      max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB
-      max_db_connections = 350
+        allow_import_from_derivation = false
 
-      queue_runner_metrics_address = [::]:9198
+        max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB
+        max_db_connections = 350
 
-      <hydra_notify>
-        <prometheus>
-          listen_address = 0.0.0.0
-          port = 9199
-        </prometheus>
-      </hydra_notify>
-    '';
+        queue_runner_metrics_address = [::]:9198
+
+        <hydra_notify>
+          <prometheus>
+            listen_address = 0.0.0.0
+            port = 9199
+          </prometheus>
+        </hydra_notify>
+      '';
+    };
+
+    hydra-queue-runner-v2 = {
+      enable = true;
+      settings.remoteStoreAddr = [
+        "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
+      ];
+    };
+
+    hydra-queue-builder-v2 = {
+      enable = true;
+      queueRunnerAddr = "https://queue-runner.staging-hydra.nixos.org";
+      mtls = {
+        serverRootCaCertPath = "${./ca.crt}";
+        clientCertPath = "${./client.crt}";
+        clientKeyPath = config.sops.secrets."queue-runner-client.key".path;
+        domainName = "queue-runner.staging-hydra.nixos.org";
+      };
+    };
+
+    nginx = {
+      enable = true;
+      virtualHosts."queue-runner.staging-hydra.nixos.org" = {
+        extraConfig = ''
+          ssl_client_certificate ${./ca.crt};
+          ssl_verify_depth 2;
+          ssl_verify_client on;
+        '';
+
+        sslCertificate = ./server.crt;
+        sslCertificateKey = config.sops.secrets."queue-runner-server.key".path;
+        onlySSL = true;
+
+        locations."/".extraConfig = ''
+          # This is necessary so that grpc connections do not get closed early
+          # see https://stackoverflow.com/a/67805465
+          client_body_timeout 31536000s;
+
+          grpc_pass grpc://[::1]:50051;
+
+          grpc_read_timeout 31536000s; # 1 year in seconds
+          grpc_send_timeout 31536000s; # 1 year in seconds
+          grpc_socket_keepalive on;
+
+          grpc_set_header Host $host;
+          grpc_set_header X-Real-IP $remote_addr;
+          grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          grpc_set_header X-Forwarded-Proto $scheme;
+
+          grpc_set_header X-Client-DN $ssl_client_s_dn;
+          grpc_set_header X-Client-Cert $ssl_client_escaped_cert;
+        '';
+      };
+    };
   };
 
-  sops.secrets.hydra-users = {
-    sopsFile = ../../secrets/hydra-users.staging-hydra;
-    format = "binary";
+  sops.secrets = {
+    "queue-runner-server.key" = {
+      sopsFile = ../../secrets/queue-runner-server.key.staging-hydra;
+      format = "binary";
+      owner = config.systemd.services.hydra-queue-runner.serviceConfig.User;
+    };
+    "queue-runner-client.key" = {
+      sopsFile = ../../secrets/queue-runner-client.key.staging-hydra;
+      format = "binary";
+      owner = config.systemd.services.hydra-queue-runner.serviceConfig.User;
+    };
+    hydra-users = {
+      sopsFile = ../../secrets/hydra-users.staging-hydra;
+      format = "binary";
+    };
   };
 
   systemd = {
@@ -117,6 +189,8 @@ in
     services = {
       hydra-notify.enable = false;
       hydra-queue-runner = {
+        enable = false;
+
         # restarting the scheduler is very expensive
         restartIfChanged = false;
         serviceConfig = {

non-critical-infra/hosts/staging-hydra/server.crt

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB/jCCAbCgAwIBAgIUfUYjDOaJML1lIMkAMvLjnSTscYswBQYDK2VwMEMxCzAJ
+BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
+cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
+WjBSMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExLTArBgNVBAMM
+JHF1ZXVlLXJ1bm5lci5zdGFnaW5nLWh5ZHJhLm5peG9zLm9yZzAqMAUGAytlcAMh
+ANVnDi5rY0Ar4hPbqRJqS+Nw7b5GTg0QxL2DM7l1xTqHo4GkMIGhMAkGA1UdEwQC
+MAAwCwYDVR0PBAQDAgPoMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMBMC8GA1UdEQQo
+MCaCJHF1ZXVlLXJ1bm5lci5zdGFnaW5nLWh5ZHJhLm5peG9zLm9yZzAdBgNVHQ4E
+FgQU4ArR8rzVAt6dFkSXiMUlYYAzbwUwHwYDVR0jBBgwFoAU0wQG6BxTKtYwlywu
+yD0aVr/1r4gwBQYDK2VwA0EAScS72oaQ8PcYpH26FuRGnKaWe4e7fQ5RmKBUyC+5
+CiYIWu4D7fNGYJ15szCfh4nJIuyB0eXBv1ddAGAQMVdhDw==
+-----END CERTIFICATE-----