diff --git a/dns/nixos.org.js b/dns/nixos.org.js index e40bcf3d..e114f3cc 100644 --- a/dns/nixos.org.js +++ b/dns/nixos.org.js @@ -96,6 +96,7 @@ D("nixos.org", // hydra staging area A("staging-hydra", "157.180.25.203"), AAAA("staging-hydra", "2a01:4f9:c012:d5d3::1"), + CNAME("queue-runner.staging-hydra", "staging-hydra"), // services infra A("caliban", "65.109.26.213"), diff --git a/non-critical-infra/hosts/staging-hydra/ca.crt b/non-critical-infra/hosts/staging-hydra/ca.crt new file mode 100644 index 00000000..3337c6fc --- /dev/null +++ b/non-critical-infra/hosts/staging-hydra/ca.crt @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBnTCCAU+gAwIBAgIUQpxYsPwAyTY70yYO9fcCmCaZreIwBQYDK2VwMEMxCzAJ +BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt +cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3 +WjBDMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExHjAcBgNVBAMM +FWh5ZHJhLXF1ZXVlLXJ1bm5lci1jYTAqMAUGAytlcAMhAM+Mc/XSTXwJeWPxrpqo +SPT5Xwi8/j85VO6TsfBlXFt4o1MwUTAdBgNVHQ4EFgQU0wQG6BxTKtYwlywuyD0a +Vr/1r4gwHwYDVR0jBBgwFoAU0wQG6BxTKtYwlywuyD0aVr/1r4gwDwYDVR0TAQH/ +BAUwAwEB/zAFBgMrZXADQQA3BRP2+TkkDQPnPy6MQyDCxqfEeV6OQjtspSvCO0UL +GWmfvzrlUQytwTFTPfVzaErbyVPbeYU5y8rmRoGPNSoI +-----END CERTIFICATE----- diff --git a/non-critical-infra/hosts/staging-hydra/client.crt b/non-critical-infra/hosts/staging-hydra/client.crt new file mode 100644 index 00000000..2d0682ca --- /dev/null +++ b/non-critical-infra/hosts/staging-hydra/client.crt @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBlDCCAUagAwIBAgIUfUYjDOaJML1lIMkAMvLjnSTscYwwBQYDK2VwMEMxCzAJ +BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt +cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3 +WjBLMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExJjAkBgNVBAMM +HWh5ZHJhLXF1ZXVlLWJ1aWxkZXItbG9jYWxob3N0MCowBQYDK2VwAyEAJqCvjdwY +0l4z0cbwecONx3DfhpPo+SBymv7H2OQt4aejQjBAMB0GA1UdDgQWBBQw54bw+CrK +O53J4qzB4Cknzf7CzTAfBgNVHSMEGDAWgBTTBAboHFMq1jCXLC7IPRpWv/WviDAF +BgMrZXADQQB+XUCZQo0aX6IyEQDrds1d3SWF1rFBv5MjtZS25TYpnQQPgt9iiD6Q +dtCwSVqyAgCG4XlkYBpU4AKVKqyMsSAC +-----END CERTIFICATE----- diff --git a/non-critical-infra/hosts/staging-hydra/default.nix b/non-critical-infra/hosts/staging-hydra/default.nix index cfedb458..618bd3ab 100644 --- a/non-critical-infra/hosts/staging-hydra/default.nix +++ b/non-critical-infra/hosts/staging-hydra/default.nix @@ -5,6 +5,8 @@ inputs.srvos.nixosModules.server inputs.srvos.nixosModules.hardware-hetzner-cloud-arm ../../modules/common.nix + ../../modules/hydra-queue-runner-v2.nix + ../../modules/hydra-queue-builder-v2.nix ./hydra-proxy.nix ./hydra.nix inputs.hydra.nixosModules.hydra diff --git a/non-critical-infra/hosts/staging-hydra/genca.sh b/non-critical-infra/hosts/staging-hydra/genca.sh new file mode 100755 index 00000000..c8c3a8eb --- /dev/null +++ b/non-critical-infra/hosts/staging-hydra/genca.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -x + +hosts="localhost ofborg-eval02 ofborg-eval03 ofborg-eval04 ofborg-build01 ofborg-build02 ofborg-build03 ofborg-build04 ofborg-build05" + +C="DE" +O="NixOS Infra" + +newDir="$(date '+%Y-%m-%dT%H:%M')" +mkdir "${newDir}" +cd "${newDir}" || exit + +openssl genpkey -algorithm Ed25519 -out ca.key +openssl req -x509 -new -nodes -key ca.key -sha256 -days 18250 -out ca.crt \ + -subj "/C=${C}/O=${O}/CN=hydra-queue-runner-ca" + +cat <server.cnf +[req] +prompt = no +x509_extensions = v3_req +req_extensions = v3_req +default_md = sha256 +distinguished_name = req_distinguished_name + +[req_distinguished_name] +C = ${C} +O = ${O} +CN = queue-runner.staging-hydra.nixos.org + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment, keyAgreement +extendedKeyUsage = critical, serverAuth +subjectAltName = @alt_names + +[alt_names] +DNS.1 = queue-runner.staging-hydra.nixos.org +EOF + +openssl genpkey -algorithm Ed25519 -out server.key +openssl req -new -key server.key -out server.csr -config server.cnf +openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 18250 -sha256 -extfile server.cnf -extensions v3_req + +for host in ${hosts}; do + openssl genpkey -algorithm Ed25519 -out "client-${host}.key" + openssl req -new -key "client-${host}.key" -out "client-${host}.csr" \ + -subj "/C=${C}/O=${O}/CN=hydra-queue-builder-${host}" + openssl x509 -req -in "client-${host}.csr" -CA ca.crt -CAkey ca.key -CAcreateserial -out "client-${host}.crt" -days 18250 -sha256 +done + +rm -rf -- *.csr *.srl +rm server.cnf + +cd - || exit diff --git a/non-critical-infra/hosts/staging-hydra/hydra.nix b/non-critical-infra/hosts/staging-hydra/hydra.nix index 0b2d27ae..3c5be205 100644 --- a/non-critical-infra/hosts/staging-hydra/hydra.nix +++ b/non-critical-infra/hosts/staging-hydra/hydra.nix @@ -24,8 +24,11 @@ in dates = "03,09,15,21:15"; }; - # gc outputs as well, since they are served from the cache - nix.settings.gc-keep-outputs = lib.mkForce false; + nix.settings = { + # gc outputs as well, since they are served from the cache + gc-keep-outputs = lib.mkForce false; + allowed-users = [ "hydra-www" ]; + }; # Don't rate-limit the journal. services.journald.rateLimitBurst = 0; @@ -44,67 +47,136 @@ in }; }; - services.hydra-dev = { - enable = true; - package = pkgs.hydra; - buildMachinesFiles = [ - (pkgs.writeText "local" '' - localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - - - '') - ]; - logo = ../../../build/hydra-logo.png; - hydraURL = "https://hydra.nixos.org"; - notificationSender = "edolstra@gmail.com"; - smtpHost = "localhost"; - useSubstitutes = true; - extraConfig = '' - max_servers 30 + services = { + hydra-dev = { + enable = true; + package = pkgs.hydra; + buildMachinesFiles = [ + (pkgs.writeText "local" '' + localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - - + '') + ]; + logo = ../../../build/hydra-logo.png; + hydraURL = "https://hydra.nixos.org"; + notificationSender = "edolstra@gmail.com"; + smtpHost = "localhost"; + useSubstitutes = true; + extraConfig = '' + max_servers 30 - store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br - server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache} - binary_cache_public_uri = https://cache-staging.nixos.org + store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br + server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache} + binary_cache_public_uri = https://cache-staging.nixos.org - - cache_size = 32m - + + cache_size = 32m + - # patchelf:master:3 - xxx-jobset-repeats = nixos:reproducibility:1 + # patchelf:master:3 + xxx-jobset-repeats = nixos:reproducibility:1 - upload_logs_to_binary_cache = true - compress_build_logs = false # conflicts with upload_logs_to_binary_cache + upload_logs_to_binary_cache = true + compress_build_logs = false # conflicts with upload_logs_to_binary_cache - log_prefix = https://cache.nixos.org/ + log_prefix = https://cache.nixos.org/ - evaluator_workers = 1 - evaluator_max_memory_size = 4096 + evaluator_workers = 1 + evaluator_max_memory_size = 4096 - max_concurrent_evals = 1 + queue_runner_endpoint = http://localhost:8080 - # increase the number of active compress slots (CPU is 48*2 on mimas) - max_local_worker_threads = 144 + max_concurrent_evals = 1 - max_unsupported_time = 86400 + # increase the number of active compress slots (CPU is 48*2 on mimas) + max_local_worker_threads = 144 - allow_import_from_derivation = false + max_unsupported_time = 86400 - max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB - max_db_connections = 350 + allow_import_from_derivation = false - queue_runner_metrics_address = [::]:9198 + max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB + max_db_connections = 350 - - - listen_address = 0.0.0.0 - port = 9199 - - - ''; + queue_runner_metrics_address = [::]:9198 + + + + listen_address = 0.0.0.0 + port = 9199 + + + ''; + }; + + hydra-queue-runner-v2 = { + enable = true; + settings.remoteStoreAddr = [ + "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br" + ]; + }; + + hydra-queue-builder-v2 = { + enable = true; + queueRunnerAddr = "https://queue-runner.staging-hydra.nixos.org"; + mtls = { + serverRootCaCertPath = "${./ca.crt}"; + clientCertPath = "${./client.crt}"; + clientKeyPath = config.sops.secrets."queue-runner-client.key".path; + domainName = "queue-runner.staging-hydra.nixos.org"; + }; + }; + + nginx = { + enable = true; + virtualHosts."queue-runner.staging-hydra.nixos.org" = { + extraConfig = '' + ssl_client_certificate ${./ca.crt}; + ssl_verify_depth 2; + ssl_verify_client on; + ''; + + sslCertificate = ./server.crt; + sslCertificateKey = config.sops.secrets."queue-runner-server.key".path; + onlySSL = true; + + locations."/".extraConfig = '' + # This is necessary so that grpc connections do not get closed early + # see https://stackoverflow.com/a/67805465 + client_body_timeout 31536000s; + + grpc_pass grpc://[::1]:50051; + + grpc_read_timeout 31536000s; # 1 year in seconds + grpc_send_timeout 31536000s; # 1 year in seconds + grpc_socket_keepalive on; + + grpc_set_header Host $host; + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + grpc_set_header X-Forwarded-Proto $scheme; + + grpc_set_header X-Client-DN $ssl_client_s_dn; + grpc_set_header X-Client-Cert $ssl_client_escaped_cert; + ''; + }; + }; }; - sops.secrets.hydra-users = { - sopsFile = ../../secrets/hydra-users.staging-hydra; - format = "binary"; + sops.secrets = { + "queue-runner-server.key" = { + sopsFile = ../../secrets/queue-runner-server.key.staging-hydra; + format = "binary"; + owner = config.systemd.services.hydra-queue-runner.serviceConfig.User; + }; + "queue-runner-client.key" = { + sopsFile = ../../secrets/queue-runner-client.key.staging-hydra; + format = "binary"; + owner = config.systemd.services.hydra-queue-runner.serviceConfig.User; + }; + hydra-users = { + sopsFile = ../../secrets/hydra-users.staging-hydra; + format = "binary"; + }; }; systemd = { @@ -117,6 +189,8 @@ in services = { hydra-notify.enable = false; hydra-queue-runner = { + enable = false; + # restarting the scheduler is very expensive restartIfChanged = false; serviceConfig = { diff --git a/non-critical-infra/hosts/staging-hydra/server.crt b/non-critical-infra/hosts/staging-hydra/server.crt new file mode 100644 index 00000000..4a9f614e --- /dev/null +++ b/non-critical-infra/hosts/staging-hydra/server.crt @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB/jCCAbCgAwIBAgIUfUYjDOaJML1lIMkAMvLjnSTscYswBQYDK2VwMEMxCzAJ +BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt +cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3 +WjBSMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExLTArBgNVBAMM +JHF1ZXVlLXJ1bm5lci5zdGFnaW5nLWh5ZHJhLm5peG9zLm9yZzAqMAUGAytlcAMh +ANVnDi5rY0Ar4hPbqRJqS+Nw7b5GTg0QxL2DM7l1xTqHo4GkMIGhMAkGA1UdEwQC +MAAwCwYDVR0PBAQDAgPoMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMBMC8GA1UdEQQo +MCaCJHF1ZXVlLXJ1bm5lci5zdGFnaW5nLWh5ZHJhLm5peG9zLm9yZzAdBgNVHQ4E +FgQU4ArR8rzVAt6dFkSXiMUlYYAzbwUwHwYDVR0jBBgwFoAU0wQG6BxTKtYwlywu +yD0aVr/1r4gwBQYDK2VwA0EAScS72oaQ8PcYpH26FuRGnKaWe4e7fQ5RmKBUyC+5 +CiYIWu4D7fNGYJ15szCfh4nJIuyB0eXBv1ddAGAQMVdhDw== +-----END CERTIFICATE----- diff --git a/non-critical-infra/modules/hydra-queue-builder-v2.nix b/non-critical-infra/modules/hydra-queue-builder-v2.nix new file mode 100644 index 00000000..ad08fdc6 --- /dev/null +++ b/non-critical-infra/modules/hydra-queue-builder-v2.nix @@ -0,0 +1,275 @@ +{ + config, + pkgs, + lib, + ... +}: +let + cfg = config.services.hydra-queue-builder-v2; +in +{ + options = { + services.hydra-queue-builder-v2 = { + enable = lib.mkEnableOption "QueueBuilder"; + + queueRunnerAddr = lib.mkOption { + description = "Queue Runner address to the grpc server"; + type = lib.types.singleLineStr; + }; + + pingInterval = lib.mkOption { + description = "Interval in which pings are send to the runner"; + type = lib.types.ints.positive; + default = 10; + }; + + speedFactor = lib.mkOption { + description = "Additional Speed factor for this machine"; + type = lib.types.oneOf [ + lib.types.ints.positive + lib.types.float + ]; + default = 1; + }; + + maxJobs = lib.mkOption { + description = "Maximum allowed of jobs. This only is used if the queue runner uses this metrics for determining free machines."; + type = lib.types.ints.positive; + default = 4; + }; + + tmpAvailThreshold = lib.mkOption { + description = "Threshold in percent free for /tmp before jobs are no longer scheduled on the machine"; + type = lib.types.float; + default = 10.0; + }; + + storeAvailThreshold = lib.mkOption { + description = "Threshold in percent free for /nix/store before jobs are no longer scheduled on the machine"; + type = lib.types.float; + default = 10.0; + }; + + load1Threshold = lib.mkOption { + description = "Maximum Load1 threshold before we stop scheduling jobs on that node. Only used if PSI is not available."; + type = lib.types.float; + default = 8.0; + }; + + cpuPsiThreshold = lib.mkOption { + description = "Maximum CPU PSI in the last 10s before we stop scheduling jobs on that node"; + type = lib.types.float; + default = 75.0; + }; + + memPsiThreshold = lib.mkOption { + description = "Maximum Memory PSI in the last 10s before we stop scheduling jobs on that node"; + type = lib.types.float; + default = 80.0; + }; + + ioPsiThreshold = lib.mkOption { + description = "Maximum IO PSI in the last 10s before we stop scheduling jobs on that node. If null then this pressure check is disabled."; + type = lib.types.nullOr lib.types.float; + default = null; + }; + + systems = lib.mkOption { + description = "List of supported systems. If none are passed, system and extra-platforms are read from nix."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + supportedFeatures = lib.mkOption { + description = "Pass supported features to the builder. If none are passed, system features will be used."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + mandatoryFeatures = lib.mkOption { + description = "Pass mandatory features to the builder."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + useSubstitutes = lib.mkOption { + description = "Use substitution for paths"; + type = lib.types.bool; + default = true; + }; + + mtls = lib.mkOption { + description = "mtls options"; + default = null; + type = lib.types.nullOr ( + lib.types.submodule { + options = { + serverRootCaCertPath = lib.mkOption { + description = "Server root ca certificate path"; + type = lib.types.path; + }; + clientCertPath = lib.mkOption { + description = "Client certificate path"; + type = lib.types.path; + }; + clientKeyPath = lib.mkOption { + description = "Client key path"; + type = lib.types.path; + }; + domainName = lib.mkOption { + description = "Domain name for mtls"; + type = lib.types.singleLineStr; + }; + }; + } + ); + }; + + package = lib.mkOption { + type = lib.types.package; + default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).builder; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.hydra-queue-builder-v2 = { + description = "hydra-queue-builder-v2 main service"; + + requires = [ "nix-daemon.socket" ]; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = { + NIX_REMOTE = "daemon"; + LIBEV_FLAGS = "4"; # go ahead and mandate epoll(2) + RUST_BACKTRACE = "1"; + + # Note: it's important to set this for nix-store, because it wants to use + # $HOME in order to use a temporary cache dir. bizarre failures will occur + # otherwise + HOME = "/run/hydra-queue-builder-v2"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + + ExecStart = lib.escapeShellArgs ( + [ + (lib.getExe cfg.package) + "--gateway-endpoint" + cfg.queueRunnerAddr + "--ping-interval" + cfg.pingInterval + "--speed-factor" + cfg.speedFactor + "--max-jobs" + cfg.maxJobs + "--tmp-avail-threshold" + cfg.tmpAvailThreshold + "--store-avail-threshold" + cfg.storeAvailThreshold + "--load1-threshold" + cfg.load1Threshold + "--cpu-psi-threshold" + cfg.cpuPsiThreshold + "--mem-psi-threshold" + cfg.memPsiThreshold + ] + ++ lib.optionals (cfg.ioPsiThreshold != null) [ + "--io-psi-threshold" + cfg.ioPsiThreshold + ] + ++ (builtins.concatMap (v: [ + "--systems" + v + ]) cfg.systems) + ++ (builtins.concatMap (v: [ + "--supported-features" + v + ]) cfg.supportedFeatures) + ++ (builtins.concatMap (v: [ + "--mandatory-features" + v + ]) cfg.mandatoryFeatures) + ++ lib.optionals (cfg.useSubstitutes != null) [ + "--use-substitutes" + ] + ++ lib.optionals (cfg.mtls != null) [ + "--server-root-ca-cert-path" + cfg.mtls.serverRootCaCertPath + "--client-cert-path" + cfg.mtls.clientCertPath + "--client-key-path" + cfg.mtls.clientKeyPath + "--domain-name" + cfg.mtls.domainName + ] + ); + + User = "hydra-queue-builder"; + Group = "hydra"; + + ReadWritePaths = [ + "/nix/var/nix/gcroots/" + "/nix/var/nix/daemon-socket/socket" + ]; + ReadOnlyPaths = [ "/nix/" ]; + RuntimeDirectory = "hydra-queue-builder-v2"; + + PrivateNetwork = false; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + RemoveIPC = true; + UMask = "0077"; + + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + ProtectKernelLogs = true; + ProtectClock = true; + + RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6"; + + LockPersonality = true; + ProtectHostname = true; + RestrictRealtime = true; + MemoryDenyWriteExecute = true; + PrivateUsers = true; + RestrictNamespaces = true; + }; + }; + systemd.tmpfiles.rules = [ + "d /nix/var/nix/gcroots/per-user/hydra-queue-builder 0755 hydra-queue-builder hydra -" + ]; + nix.settings = { + allowed-users = [ "hydra-queue-builder" ]; + experimental-features = [ "nix-command" ]; + trusted-users = [ "hydra-queue-builder" ]; + }; + + users = { + groups.hydra = { }; + users.hydra-queue-builder = { + group = "hydra"; + isSystemUser = true; + }; + }; + }; +} diff --git a/non-critical-infra/modules/hydra-queue-runner-v2.nix b/non-critical-infra/modules/hydra-queue-runner-v2.nix new file mode 100644 index 00000000..0f1347a6 --- /dev/null +++ b/non-critical-infra/modules/hydra-queue-runner-v2.nix @@ -0,0 +1,300 @@ +{ + config, + pkgs, + lib, + ... +}: +let + cfg = config.services.hydra-queue-runner-v2; + + format = pkgs.formats.toml { }; +in +{ + options = { + services.hydra-queue-runner-v2 = { + enable = lib.mkEnableOption "QueueRunner"; + + settings = lib.mkOption { + description = "Reloadable settings for queue runner"; + type = lib.types.submodule { + options = { + hydraDataDir = lib.mkOption { + description = "Hydra data directory"; + type = lib.types.path; + default = "/var/lib/hydra"; + }; + dbUrl = lib.mkOption { + description = "Postgresql database url"; + type = lib.types.singleLineStr; + default = "postgres://hydra@%2Frun%2Fpostgresql:5432/hydra"; + }; + maxDbConnections = lib.mkOption { + description = "Postgresql maximum db connections"; + type = lib.types.ints.positive; + default = 128; + }; + machineSortFn = lib.mkOption { + description = "Function name for sorting machines"; + type = lib.types.enum [ + "SpeedFactorOnly" + "CpuCoreCountWithSpeedFactor" + "BogomipsWithSpeedFactor" + ]; + default = "SpeedFactorOnly"; + }; + machineFreeFn = lib.mkOption { + description = "Function name for determining \"idle\" machines"; + type = lib.types.enum [ + "Dynamic" + "DynamicWithMaxJobLimit" + "Static" + ]; + default = "Static"; + }; + dispatchTriggerTimerInS = lib.mkOption { + description = "Timer for triggering dispatch in an interval in seconds. Setting this to a value <= 0 will disable this timer and only trigger the dispatcher if queue changes happend."; + type = lib.types.int; + default = 120; + }; + queueTriggerTimerInS = lib.mkOption { + description = "Timer for triggering queue in an interval in seconds. Setting this to a value <= 0 will disable this timer and only trigger via pg notifications."; + type = lib.types.int; + default = -1; + }; + remoteStoreAddr = lib.mkOption { + description = "Remote store address"; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + useSubstitutes = lib.mkOption { + description = "Use substitution for paths"; + type = lib.types.bool; + default = false; + }; + rootsDir = lib.mkOption { + description = "Gcroots directory, defaults to /nix/var/nix/gcroots/per-user/$LOGNAME/hydra-roots"; + type = lib.types.nullOr lib.types.path; + default = null; + }; + maxRetries = lib.mkOption { + description = "Number of maximum amount of retries for a build step."; + type = lib.types.ints.positive; + default = 5; + }; + retryInterval = lib.mkOption { + description = "Interval in which retires should be able to be attempted again."; + type = lib.types.ints.positive; + default = 60; + }; + retryBackoff = lib.mkOption { + description = "Additional backoff on top of the retry interval."; + type = lib.types.float; + default = 3.0; + }; + maxUnsupportedTimeInS = lib.mkOption { + description = "Time until unsupported steps are aborted."; + type = lib.types.ints.unsigned; + default = 120; + }; + stopQueueRunAfterInS = lib.mkOption { + description = "Seconds after which the queue run should be interupted early. Setting this to a value <= 0 will disable this feature and the queue run will never exit early."; + type = lib.types.int; + default = 60; + }; + }; + }; + default = { }; + }; + + grpc = lib.mkOption { + description = "grpc options"; + default = { }; + type = lib.types.submodule { + options = { + address = lib.mkOption { + type = lib.types.singleLineStr; + default = "[::1]"; + description = "The IP address the grpc listener should bound to"; + }; + + port = lib.mkOption { + description = "Which grpc port this app should listen on"; + type = lib.types.port; + default = 50051; + }; + }; + }; + }; + + rest = lib.mkOption { + description = "rest options"; + default = { }; + type = lib.types.submodule { + options = { + address = lib.mkOption { + type = lib.types.singleLineStr; + default = "[::1]"; + description = "The IP address the rest listener should bound to"; + }; + + port = lib.mkOption { + description = "Which rest port this app should listen on"; + type = lib.types.port; + default = 8080; + }; + }; + }; + }; + + mtls = lib.mkOption { + description = "mtls options"; + default = null; + type = lib.types.nullOr ( + lib.types.submodule { + options = { + serverCertPath = lib.mkOption { + description = "Server certificate path"; + type = lib.types.path; + }; + serverKeyPath = lib.mkOption { + description = "Server key path"; + type = lib.types.path; + }; + clientCaCertPath = lib.mkOption { + description = "Client ca certificate path"; + type = lib.types.path; + }; + }; + } + ); + }; + package = lib.mkOption { + type = lib.types.package; + default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).runner; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.hydra-queue-runner-v2 = { + description = "hydra queue-runner-v2 main service"; + + requires = [ "nix-daemon.socket" ]; + after = [ + "network.target" + "postgresql.service" + ]; + wantedBy = [ "multi-user.target" ]; + reloadTriggers = [ config.environment.etc."hydra/queue-runner.toml".source ]; + + environment = { + NIX_REMOTE = "daemon"; + LIBEV_FLAGS = "4"; # go ahead and mandate epoll(2) + RUST_BACKTRACE = "1"; + + # Note: it's important to set this for nix-store, because it wants to use + # $HOME in order to use a temporary cache dir. bizarre failures will occur + # otherwise + HOME = "/var/lib/hydra/queue-runner"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + + ExecStart = lib.escapeShellArgs ( + [ + (lib.getExe cfg.package) + "--rest-bind" + "${cfg.rest.address}:${toString cfg.rest.port}" + "--grpc-bind" + "${cfg.grpc.address}:${toString cfg.grpc.port}" + "--config-path" + "/etc/hydra/queue-runner.toml" + ] + ++ lib.optionals (cfg.mtls != null) [ + "--server-cert-path" + cfg.mtls.serverCertPath + "--server-key-path" + cfg.mtls.serverKeyPath + "--client-ca-cert-path" + cfg.mtls.clientCaCertPath + ] + ); + ExecReload = "${pkgs.util-linux}/bin/kill -HUP $MAINPID"; + + User = "hydra-queue-runner"; + Group = "hydra"; + + StateDirectory = [ "hydra/queue-runner" ]; + StateDirectoryMode = "0700"; + ReadWritePaths = [ + "/nix/var/nix/gcroots/" + "/run/postgresql/.s.PGSQL.${toString config.services.postgresql.port}" + "/nix/var/nix/daemon-socket/socket" + "/var/lib/hydra/build-logs/" + ]; + ReadOnlyPaths = [ "/nix/" ]; + WorkingDirectory = "/var/lib/hydra/queue-runner"; + + PrivateNetwork = false; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + RemoveIPC = true; + UMask = "0022"; + + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + ProtectKernelLogs = true; + ProtectClock = true; + + RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6"; + + LockPersonality = true; + ProtectHostname = true; + RestrictRealtime = true; + MemoryDenyWriteExecute = true; + PrivateUsers = true; + RestrictNamespaces = true; + }; + }; + + environment.etc."hydra/queue-runner.toml".source = format.generate "queue-runner.toml" ( + lib.filterAttrsRecursive (_: v: v != null) cfg.settings + ); + systemd.tmpfiles.rules = [ + "d /nix/var/nix/gcroots/per-user/hydra-queue-runner 0755 hydra-queue-runner hydra -" + "d /var/lib/hydra/build-logs/ 0755 hydra-queue-runner hydra -" + ]; + + nix.settings = { + allowed-users = [ "hydra-queue-runner" ]; + experimental-features = [ "nix-command" ]; + trusted-users = [ "hydra-queue-runner" ]; + }; + + users = { + groups.hydra = { }; + users.hydra-queue-runner = { + group = "hydra"; + isSystemUser = true; + }; + }; + }; +} diff --git a/non-critical-infra/packages/hydra-queue-runner/default.nix b/non-critical-infra/packages/hydra-queue-runner/default.nix new file mode 100644 index 00000000..b6543357 --- /dev/null +++ b/non-critical-infra/packages/hydra-queue-runner/default.nix @@ -0,0 +1,104 @@ +{ + rustPackages, + fetchFromGitHub, + pkg-config, + openssl, + zlib, + protobuf, + lib, + makeWrapper, + nixVersions, + nlohmann_json, + libsodium, + boost, +}: +let + version = "unstable-2025-08-07"; + src = fetchFromGitHub { + owner = "helsinki-systems"; + repo = "hydra-queue-runner"; + rev = "54b3c9351d2ae10be5c4d1b97cc0f86300cd70ca"; + hash = "sha256-gR2DzWkTykM9GdW3Nf/V8eRv68fl3aO+NW0zNPFSRT0="; + }; + useFetchCargoVendor = true; + cargoHash = "sha256-oNUMmFfts4rjBX0k5mzsxpYA2JqgsRu1nMRFf/2rZa8="; + nativeBuildInputs = [ + pkg-config + protobuf + makeWrapper + ]; + buildInputs = [ + openssl + zlib + protobuf + + nixVersions.nix_2_29 + nlohmann_json + libsodium + boost + ]; + meta = { + description = "Hydra Queue-Runner implemented in rust"; + homepage = "https://github.com/helsinki-systems/hydra-queue-runner"; + license = [ lib.licenses.gpl3 ]; + maintainers = [ lib.maintainers.conni2461 ]; + platforms = lib.platforms.all; + }; +in +{ + runner = rustPackages.rustPlatform.buildRustPackage (finalAttrs: { + pname = "hydra-queue-runner"; + inherit version src; + __structuredAttrs = true; + strictDeps = true; + + inherit + useFetchCargoVendor + cargoHash + nativeBuildInputs + buildInputs + ; + + cargoBuildFlags = [ + "-p" + "queue-runner" + ]; + cargoTestFlags = finalAttrs.cargoBuildFlags; + + postInstall = '' + wrapProgram $out/bin/queue-runner --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]} + ''; + + meta = meta // { + mainProgram = "queue-runner"; + }; + }); + + builder = rustPackages.rustPlatform.buildRustPackage (finalAttrs: { + pname = "hydra-queue-builder"; + inherit src version; + __structuredAttrs = true; + strictDeps = true; + + inherit + useFetchCargoVendor + cargoHash + nativeBuildInputs + buildInputs + ; + + cargoBuildFlags = [ + "-p" + "builder" + ]; + cargoTestFlags = finalAttrs.cargoBuildFlags; + + postInstall = '' + wrapProgram $out/bin/builder --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]} + ''; + + meta = meta // { + mainProgram = "builder"; + }; + }); +} diff --git a/non-critical-infra/secrets/queue-runner-ca.key.staging-hydra b/non-critical-infra/secrets/queue-runner-ca.key.staging-hydra new file mode 100644 index 00000000..1244e803 --- /dev/null +++ b/non-critical-infra/secrets/queue-runner-ca.key.staging-hydra @@ -0,0 +1,35 @@ +{ + "data": "ENC[AES256_GCM,data:pe1lPpTo2O6CjOwyW3Go673wo4kJJ8O7XGk3M89pcdaoHPr0gdvvsSg8tG98qng4E5vfYnQpzXNFPBjmv9DvFkm3LDqOSmJsNw6Vp4LAIpkxQmuehGjDnsu2WxWC/JlG2Qm+2FS2saxZrfaeHEiZDvPEm1rN3oA=,iv:rrKEmM0PLIS+ur+cjW5tBR3UqOftOi6FTaqrymr6OIg=,tag:zGZtHCxldMKeWoNICBe3+A==,type:str]", + "sops": { + "age": [ + { + "recipient": "age1xj4dl6xdl5ztmetp9axa0epjj922hu6a2gut3rrs5rdc5xv85yjsq5ggpx", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBETllNUWc0YUlEUW9KdmlE\nSnZNdW9kekxxUzJ0ZzBXTkxMWEpOcFhQK25RCi9iejVhTUk4cnQ5VU1rcTBiTGNw\nMXNnS1ducEhTWk14c3d2V1pWdzFQUm8KLS0tIGNXOU9rQmtVRExXZWpYTTlvbk9i\nYlZjUDVyRWZwSHlsWDhzbWZCdXJ2N3MK71/bfgXtdfcS/bJpf7pq0IViBpRPDzc6\n4Hi+E+aZodq6167TrYT26LV5pSKYER+dIvk02FSBvOxCDvk6u6NyCQ==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGWDdPUWk1R2JCMW1pNDRI\nZFp3czA4YUdrekFMLzlFTHJtM083bDBSZUNNCjEwcUlSazBoUnZMM3BsWit1eTB4\nS3VEaFVad1ZRZDBXYk1vaHFKNklPZDQKLS0tIC9ucGpzd3NiRjEvN0dtQUtZZ3pj\nVE1zbHZvQ0R3T0tyL3dnZGU5dDVnMlUKjlH1sgor3H+zHzwAV9cZbdiGbcQo27pf\n7shoRdXedex44lm8+LlYQ9YfkyhZs3mR9/++0JsvijR7o7Lrb9w1gw==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1nnm255ah9wa4gpsaq0v023a75lnmlcxszt9lc6az3mtwzxgrucfq45rp7h", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGSm8xaFBWTU5mdS9weFRG\nbnVkamVVMG1BbDgrZ3pXOVdWMmhJM3d2V2dnClozd0h1elNYSWN5SWZOK2s2WTI1\nTVROTFBCOWM5anJ4endoOWs4cmFjSlEKLS0tIGE4TURtcjFyclg1NEFnRlhiMnZ0\ncWQ3RHIyUnZUSEE5Q1hWaG9kR3k2eDAKJ5JgJzDBTv+6lb29mWhe9B9VtrncXGhG\nQbFqF9+U4Q23A/gHa+FBAqT1dzc7U+IE5WILmQa7wJg29W6A0aVt/A==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1j3mkgedmeru63vwww6m44zfw09tg8yw6xdzstaq7ejfkvgcau40qwakm8x", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3TWRBemFSWk0yR2tJcXNM\nZzNBR3dTODRMa2JraGROTmphTHpjREdGQ2dFCkZudFF4Y05aOEZ2M3doR0NhY1V5\ndmtRV2s5ZVN4NEVmT2ZuNGdmQjhDY1kKLS0tICswZHpsc2RJMGZPc1hnd3FjZGRo\nVTFvdWFPeTlJa0p0RTdBaXlJb3hncDQKmO1/apc3mqJ4CyAWf3eVNlXdUhZCIRZF\nTK0HBh7xg8S/HkPJiBcMCdF/7slQeOLL3lzfeBjnAQ+mg8hoI2YCmA==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1jrh8yyq3swjru09s75s4mspu0mphh7h6z54z946raa9wx3pcdegq0x8t4h", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlMjNQeGdheUVuMmNWSVN2\nekd5SVVkVTNnMVdmRnUwSTFFaDcrSE90bGxZCmJZRFp5NkFYUFdnOVJ4Y1NQZ1pk\nTHlja29waUxlWlF3alJwSXMyY3VDcGsKLS0tIE9LZWJnb1hRcTJBQlA2Z2NWSFhB\ndE9xSEUzVEI2UHVuQUw0cnZ5S052bEUKimqgYE85vdiTS3YV08bD/I3IpM1sPXcD\nEroNwJH8+boHOGZODLQUoezjHkajnIoo/wjK1gELEHlvShAlzNPQuQ==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age17ez23w2kpxl0gcdx4ehcglkcfcfnv4qz0gq2n8ylxwx4yrtjpvjqxfasua", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuZXVvSEJ5RFZBRGZCUFNS\nazVxcEpGTmNBaVd2OUVwRnU3Nm9zaUNPOG5zCmFwYkdlUWEweWptVjMxSDY4MERF\nbTNqTmUwSytWRjRiWFFHWmJKZE0waUkKLS0tIDZaNTZVRG5SWjF1Vm9QWm9ERDZ1\nZFBTSkRJVytKaVBlYkZibTRNSmwxZlUKdr9dALaKngENjcRYUa8KRKKfngc/BMqh\ngQrqhE2BxCKffGxsyuyqIozSm+nEw5ihQviWgWU78/XLraAWnubfRg==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2025-07-31T12:09:59Z", + "mac": "ENC[AES256_GCM,data:EdssbCd38+X5Sr/qPSewtFgGyT0WFvR+0/ZkzpKGtj/7yS/Q9w0BoyZXbwuMF4R+n2aJ+NTmxPjCz9Ald0ENnXKXJcEHlKtTf9z2h+ft6rrFsBVBlrp3KfHY16EwcfBprvbbCkU6Bvxv5UGFGfJsw5l3jpL45LQQ7z528HiQz6c=,iv:02k7g9Fck1XFBor/TAHZGpgIiH2zPRDHsdRt4rjApeU=,tag:uJJ4J6znugGBQtARL6vwRg==,type:str]", + "unencrypted_suffix": "_unencrypted", + "version": "3.10.2" + } +} diff --git a/non-critical-infra/secrets/queue-runner-client.key.staging-hydra b/non-critical-infra/secrets/queue-runner-client.key.staging-hydra new file mode 100644 index 00000000..e32c6090 --- /dev/null +++ b/non-critical-infra/secrets/queue-runner-client.key.staging-hydra @@ -0,0 +1,35 @@ +{ + "data": "ENC[AES256_GCM,data:mER5O7YO36svhuSk8u5FLgBUkRaFs+k4sBccPqx0JZ5BWPhWDKQwlEKHeNuuMRPwbhtTEoKXFHN7PdcOKG3jtDNZuNuOvupNxeF8By47XpBFRJm7W9BhPdlN4UA8qoDdlWMZLBFjNSSGhV6+wGL/HGao/hpWDn0=,iv:JmAHNp6G4kVjhsLv5BgizEUatt+NbIyz8wi0YVdtN50=,tag:yAN1D0DMvOs0g4715HD6PQ==,type:str]", + "sops": { + "age": [ + { + "recipient": "age1xj4dl6xdl5ztmetp9axa0epjj922hu6a2gut3rrs5rdc5xv85yjsq5ggpx", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiUEkwL0xvV2trSUFUN1Ny\nNnBLOHhvaVFuNWxuQnE2Yys3MVAycjRCM0QwCmI1M0haRzJhUEw3aURhRmtWUG1C\nMWhpZHJiSnM4RXJvZ09WQWxxN3QyS28KLS0tIFo5Nm15U2hPSkxRUUZpRWdZTVJO\nV2p0QVlVbHV6am1LTTVoQzRLWkloUnMK6mK+TeQKok/XNH+wmGa1N1lVX3ME562o\nEev3kaAGWqz6f4bMacbCkgXmhOEryFiDCgok/TMQ6hSGa3COFXrzmQ==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPcCtST1pMdk1vNFBLRlJN\nd2IvTmUwcGQ2cm5aaXgvQTdoMkFFbnJYVm00CjJmYS95Z2FrMUdrcDJwWDFYWXY0\nNGZkSkZMRDA3NUxjVG1NdExKTjNVM1kKLS0tIEN6eHJlazlRa3NIaUUvSWtTdTYw\nZXpmcXJpY1pwUzkva2hYM1R3THhIM00KiwOB/Twye2mzFzLUiF01DTaqb0OLybSF\njRYHaZ5KWZ756c2DEnnB8TbQsN2JDpfe8pYW7tdgyfqNKaYV/l8nxQ==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1nnm255ah9wa4gpsaq0v023a75lnmlcxszt9lc6az3mtwzxgrucfq45rp7h", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6Q1htRVFycERhRHRSanFt\nTEhaTnIvMG1iYy9MZ0pLLzZqaVN0bVB1ZERNCjhkaVRXaXZoQjl6aUZYWnlvMGdG\nSDh1M09MU2I3aUF2YVgxUTVHZHYxazAKLS0tIFJtUzk1Wk9xazNUK3BJWTJMbGx6\nTzh5SlVBUkFIckQ3b0VYQWlyaGhuMEUKg+bdoltiqfMwrzytk4kwRKqUjS3XyUuN\ngX3ksj9ODZDvbwNXhgBLXkXpmefwUt0k2TJL/fTr4tqTU6hchkg7aA==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1j3mkgedmeru63vwww6m44zfw09tg8yw6xdzstaq7ejfkvgcau40qwakm8x", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqS1pTY0Rzci94Ykd3Wm1t\nVVl1TWlQc2xjdTBBbjZCWGF3bWZXaTlNaWxjCjJ0OFd3bWRTYjF5cnRaSy8zeFFF\nSWd0bkpSbnVyNTZOMU0yR1dVcUtlQlkKLS0tICt2QkdsUDcyZ1J2YjRJRkIyYW5u\naDBPWUEwNkJ5dkxqaGhCcWlNZVNXOEkK6SEQCqKZ0OsWKs6vmG18ikaeSkS7Zo9+\npUc4e/QWsRDOd/frLWzKIo8JujckNpMbbalQfykW7s0xCvGnZ1NYfQ==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1jrh8yyq3swjru09s75s4mspu0mphh7h6z54z946raa9wx3pcdegq0x8t4h", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwdXJ2bHIrdGdCSlRLL3Za\nVWZCUWRXajVmS2JCMkQ2NmZqMkVHcHAyTFc0Cngyd0QzbU9VMEVrOUxsSGVsWWNy\neGRNK3JleDFXN2p0dFYrYktVbzcrSGMKLS0tIFQ4WFc2RTFFNkx3ajUvQWJqWUpV\nUDBiOTFKQ0hIOWsxS2xYM1ptVDBwcGcK41AK30TtmKw9DCPCMnAZXl/C26t3NBNK\n3Sh4wL7yyhKmRUNutq5cnAw8vjzhRsgjJkZGmOuv5qN3s7nFTGdC5A==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age17ez23w2kpxl0gcdx4ehcglkcfcfnv4qz0gq2n8ylxwx4yrtjpvjqxfasua", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNFB6OW1sOHpseEdJSll4\neDlDN0xKQnJPMUFQcnpHaHFURUg4NmhNT3k0CmFJV2xSMnhocnlMQWI5ZThLK2R4\nZE1mRHBGbGM4UmxvalJ5TWpOUW1JR2cKLS0tIEd6UFNFc0E0QVI2bFBRUlFsWHh6\nLy9WYWpTK3E2ZjJEb01rNC84UG1DUTAKv1T3UPMBBYGHdmKCsSCUtBg/7SoQ7RjO\n0PyeOnGlvUUxj4H/z2sPRxuKj2QiqIiGqdSqOoGXNmp3omrQHtVfxg==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2025-07-31T12:14:02Z", + "mac": "ENC[AES256_GCM,data:KWe6LDUZtpGR8btMzp4Arb4b3rooS+pGOOs0W0gZYLovh4RdvMSdOHbslIZt3HzNvOWDNz3BM77jdNiSctZpLK6sXxzXB0JH7dq3H9+7cPRGyEIUYRvysjmFF1HHEw0isVkV7mPHNAQBL7y716Gw5plAfyEgCqpknbnQpKMIGmI=,iv:rRfVfr30XfMuAgkeNWq6x3l4LVFArC3X2sPz8dCamfo=,tag:lKKy+PvsuwG99mEu0/OZ7w==,type:str]", + "unencrypted_suffix": "_unencrypted", + "version": "3.10.2" + } +} diff --git a/non-critical-infra/secrets/queue-runner-server.key.staging-hydra b/non-critical-infra/secrets/queue-runner-server.key.staging-hydra new file mode 100644 index 00000000..c54a2489 --- /dev/null +++ b/non-critical-infra/secrets/queue-runner-server.key.staging-hydra @@ -0,0 +1,35 @@ +{ + "data": "ENC[AES256_GCM,data:forqylxAxHHWDqqTG8nxgqnbuyCTtRiEZ+0VA6qYRhlRY7KAYmTM4GK3wDa8dZPBi0e+LXG5aaigyc9FFsl7hHKdtQo84OPHyJOSCZyBYjMg9qlBsy/USqxftzfMjMAxVk/ibAi5LDrNl9Jh/7w74BBtbdF3qiE=,iv:5YbKjIdd+EfVpMGzTzP0VDy0Wev22bRc9IO5e3StpA4=,tag:/jsQItUIschI6B0rs7dwQg==,type:str]", + "sops": { + "age": [ + { + "recipient": "age1xj4dl6xdl5ztmetp9axa0epjj922hu6a2gut3rrs5rdc5xv85yjsq5ggpx", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuajRobWMzdHhqSVUydlVZ\nanZKY2REQWJ5KzgxNmtwZ0d4Mm1mTWFYYTM4CjhIVG1COTZkbWp3SkxtS3VDeTRB\ndGcxY0hqUk1DeU9CQldkcTUwMys1OTAKLS0tIFBibTB6bWY2ZjRxL0JFUDZmYTJ2\nMG5IclV2T2twa0NhOGtrdXZQeXhqV1UK/9RyMKw8K3AOe8PntM+5BMSSGuWuYdt4\nkDMjS3NH7Ab1tM3O/4b6lxyW2nheBXnhhqbq5tfqOVafpoIqgjZgxg==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArd3ZqdUhGREtLRlpla1Js\nK2JRRG1pYWx2NVNCT3F5dGlBc1EwOVo1Mm44CkNkSXQyK0VYZGYrMW05ZnhmbXFC\nOFpKSVVKZTE5WHkvN0hzVjRqbGV0eGMKLS0tIHJBNXNsSkcyd1EwckRTT2FFNDRk\nK1A5Q0dsZC80NWk3dE0rK2JKWG9GdjgK6exWpfYFkhSVZkU80rnLkHKqkZ6hrAGA\nNhiohmDfflTAurqEKzxPsVMiSWlmVEpKykzMrsE150evlZbj7YB9rw==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1nnm255ah9wa4gpsaq0v023a75lnmlcxszt9lc6az3mtwzxgrucfq45rp7h", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQUjBtZFhER05McTJqNk1V\neVJIMEJLcnBoMVAxazduckhJWm13Y2drcUFVCm9vRlI2ZjJDbDU0Qk5icmI2dVRn\ndUlvbWFmMjZob0ZQclhFOHJ5T25vcXcKLS0tIGNSU3NNOTNFcmZRY3BUWHRYdUpz\nT2g0TlhnSjVuWlpCYTRWN1FUSUE2V2sKxiobZEtFJDOTmxjRcfTS7HwIXOZuySXt\nOxkJoFfhaFRFOIqP6Ai8/9MtYP2UypRzdTBLP6T8iLYbT0jeM2h93Q==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1j3mkgedmeru63vwww6m44zfw09tg8yw6xdzstaq7ejfkvgcau40qwakm8x", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzTlBRb2lsV1ZYZ1htWUJ3\nS3dlQ1QrWXc3QVZFTWlxdzVGSXlNWFZ1aEdFCkpCSFB3WUlKa3ZUeGQ0SzFmZ0xP\nWGYwZk5MM3prMUUwbWovQzk1dGtvS3cKLS0tIC9jRlhLL3BtVyt4WS9rQisvc0Jo\ndjBHMVhGZlRlM3hsKzh2aU90ZzVhZTQKI1QZ9IhpNFC1hO8pl+2kjALVbTKnQjl7\nwsiwg7DUH8j16Cqe+BKqPLwV6wUZNcSgCc4b4kAdCeIme3Qc9VPofQ==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1jrh8yyq3swjru09s75s4mspu0mphh7h6z54z946raa9wx3pcdegq0x8t4h", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBOXc1QklpenBjTGh3NFNS\nZW5vZFdDNHJPWGsrN0MzSU0yaHVxR0tkOEU0CnIwTGxuUUZxTUd4b0YxZ0h6QUVa\nV1FocWJKdlpLaUVJZUNwTi8zTmlDV2cKLS0tIHYzNVYzZHgyVnlLcmJtMmk1aWwy\nTDZBSkJLY001MGxYNFQ1a3hSdG4vaEUKOdJSdnxDBansg5boUQp9xLGFMkQzKDf9\n9D7W+cwqSQEUpSTofP5elJKOsg/tdpiZaiSck8/4EHhQt2/qncgbAg==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age17ez23w2kpxl0gcdx4ehcglkcfcfnv4qz0gq2n8ylxwx4yrtjpvjqxfasua", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjcmJBbFJHbUJzYnpZWVBQ\nZjNtVTVmaUM2RzRYM1JoL2MrSmp4RFFMWkRFClBTcDdwaEMyMStNcEsrMzcwa2xG\nU083YnUvNStuNXNsOHR0TzB6NWROeVEKLS0tIFc5V0sxN1dUcksrdTFNdUZ4cyt2\nRy9rbFY2dEFZRlZmaitjUVZYZFZpK00KdLhQzJZLO4BMiWaJuXCxn/NDG5EJa/ee\nuNJj+AfwOzLk3M8rAapwAR7O0b9PL0gcU6oDybAnBlMo+UzdH8X9aQ==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2025-07-31T12:10:09Z", + "mac": "ENC[AES256_GCM,data:QRLZ7S6CoHaYQ+m0npUo2ISRkke4JvYe/cN5nBM7ngIdKQ+yyLK1d8ucG3YwB6HaVZXDM4F0BAKkwl3VXTDujKk24BQhb4G50EL91UTcaQQ4kIOdaSVh17/D6cXV1Han8hSrxrsXQ9vvyR9EmCVbjZMIP+TyyTbdAkFfHDch2Fk=,iv:Wm7rVVZRUjUeb/SU2GF5TnOcNawf8rsndlReEUtrGeg=,tag:PsL1RufQyx+7wayK0IOSlg==,type:str]", + "unencrypted_suffix": "_unencrypted", + "version": "3.10.2" + } +}