diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..bff29e6e1 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags = ["--cfg", "tokio_unstable"] diff --git a/.gitignore b/.gitignore index ea9c29856..ca435337b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ result result-* outputs +/target +/config.toml diff --git a/.sqlx/query-049c2b15e5806241473754264be493889dfa874d73dfe257a2b8554b6543acaf.json b/.sqlx/query-049c2b15e5806241473754264be493889dfa874d73dfe257a2b8554b6543acaf.json new file mode 100644 index 000000000..0553d8b7e --- /dev/null +++ b/.sqlx/query-049c2b15e5806241473754264be493889dfa874d73dfe257a2b8554b6543acaf.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT drvPath FROM BuildSteps WHERE build = $1 AND stepnr = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "drvpath", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int4", + "Int4" + ] + }, + "nullable": [ + true + ] + }, + "hash": "049c2b15e5806241473754264be493889dfa874d73dfe257a2b8554b6543acaf" +} diff --git a/.sqlx/query-05e4170e17728e552240a2474d97bdd750d7adc15ea27c0d7b10ba5330ca4f9e.json b/.sqlx/query-05e4170e17728e552240a2474d97bdd750d7adc15ea27c0d7b10ba5330ca4f9e.json new file mode 100644 index 000000000..c9e4406c9 --- /dev/null +++ b/.sqlx/query-05e4170e17728e552240a2474d97bdd750d7adc15ea27c0d7b10ba5330ca4f9e.json @@ -0,0 +1,80 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n builds.id,\n builds.jobset_id,\n jobsets.project as project,\n jobsets.name as jobset,\n job,\n drvPath,\n maxsilent,\n timeout,\n timestamp,\n globalPriority,\n priority\n FROM builds\n INNER JOIN jobsets ON builds.jobset_id = jobsets.id\n WHERE finished = 0 ORDER BY globalPriority desc, schedulingshares, random();", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "jobset_id", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "project", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "jobset", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "job", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "drvpath", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "maxsilent", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "timestamp", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "globalpriority", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "priority", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true, + false, + false, + false + ] + }, + "hash": "05e4170e17728e552240a2474d97bdd750d7adc15ea27c0d7b10ba5330ca4f9e" +} diff --git a/.sqlx/query-0626225fd580d1b49375835b17e129af1b84c9db615bdd224c5a36ed60344414.json b/.sqlx/query-0626225fd580d1b49375835b17e129af1b84c9db615bdd224c5a36ed60344414.json new file mode 100644 index 000000000..c7d7bea25 --- /dev/null +++ b/.sqlx/query-0626225fd580d1b49375835b17e129af1b84c9db615bdd224c5a36ed60344414.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE builds SET\n finished = 1,\n buildStatus = $2,\n startTime = $3,\n stopTime = $4,\n isCachedBuild = $5,\n notificationPendingSince = $4\n WHERE\n id = $1 AND finished = 0", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Int4", + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "0626225fd580d1b49375835b17e129af1b84c9db615bdd224c5a36ed60344414" +} diff --git a/.sqlx/query-0b0f985664863d2e7883f7460fc451de6f773b2fd0348a471e8e03571dc80492.json b/.sqlx/query-0b0f985664863d2e7883f7460fc451de6f773b2fd0348a471e8e03571dc80492.json new file mode 100644 index 000000000..0d11ec905 --- /dev/null +++ b/.sqlx/query-0b0f985664863d2e7883f7460fc451de6f773b2fd0348a471e8e03571dc80492.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n project,\n name,\n schedulingshares\n FROM jobsets", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "project", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "schedulingshares", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "0b0f985664863d2e7883f7460fc451de6f773b2fd0348a471e8e03571dc80492" +} diff --git a/.sqlx/query-0f15524bd358e02a14c66f9058c866e937d5a124040f77ac62c20371c7421ff5.json b/.sqlx/query-0f15524bd358e02a14c66f9058c866e937d5a124040f77ac62c20371c7421ff5.json new file mode 100644 index 000000000..d26b58ab2 --- /dev/null +++ b/.sqlx/query-0f15524bd358e02a14c66f9058c866e937d5a124040f77ac62c20371c7421ff5.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE builds SET\n finished = 1,\n buildStatus = $2,\n startTime = $3,\n stopTime = $4,\n size = $5,\n closureSize = $6,\n releaseName = $7,\n isCachedBuild = $8,\n notificationPendingSince = $4\n WHERE\n id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Int4", + "Int4", + "Int8", + "Int8", + "Text", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "0f15524bd358e02a14c66f9058c866e937d5a124040f77ac62c20371c7421ff5" +} diff --git a/.sqlx/query-1a5a0676ca03a1c4659f05b45e1a32719dac5fa8bbb879afd3c0e1e106178d59.json b/.sqlx/query-1a5a0676ca03a1c4659f05b45e1a32719dac5fa8bbb879afd3c0e1e106178d59.json new file mode 100644 index 000000000..61555ed82 --- /dev/null +++ b/.sqlx/query-1a5a0676ca03a1c4659f05b45e1a32719dac5fa8bbb879afd3c0e1e106178d59.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT MAX(s.build) FROM buildsteps s\n JOIN BuildStepOutputs o ON s.build = o.build\n WHERE startTime != 0\n AND stopTime != 0\n AND status = 1\n AND drvPath = $1\n AND name = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1a5a0676ca03a1c4659f05b45e1a32719dac5fa8bbb879afd3c0e1e106178d59" +} diff --git a/.sqlx/query-2a6f7b562d13337e240588d25734f9ea9c9601b2d010aa93038872c1538d0816.json b/.sqlx/query-2a6f7b562d13337e240588d25734f9ea9c9601b2d010aa93038872c1538d0816.json new file mode 100644 index 000000000..00c433582 --- /dev/null +++ b/.sqlx/query-2a6f7b562d13337e240588d25734f9ea9c9601b2d010aa93038872c1538d0816.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE builds SET finished = 1, buildStatus = $2, startTime = $3, stopTime = $3 where id = $1 and finished = 0", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "2a6f7b562d13337e240588d25734f9ea9c9601b2d010aa93038872c1538d0816" +} diff --git a/.sqlx/query-455497955081a6040cc5e95c6d96dae26d2400d878eb0eeb01d39487ff362480.json b/.sqlx/query-455497955081a6040cc5e95c6d96dae26d2400d878eb0eeb01d39487ff362480.json new file mode 100644 index 000000000..6765e5e49 --- /dev/null +++ b/.sqlx/query-455497955081a6040cc5e95c6d96dae26d2400d878eb0eeb01d39487ff362480.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM builds WHERE id = $1 AND finished = 0", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "455497955081a6040cc5e95c6d96dae26d2400d878eb0eeb01d39487ff362480" +} diff --git a/.sqlx/query-60433164aae6d537f4b9056a5f9cf46af129639d2cc4cdc3262293846c088906.json b/.sqlx/query-60433164aae6d537f4b9056a5f9cf46af129639d2cc4cdc3262293846c088906.json new file mode 100644 index 000000000..53fa792e0 --- /dev/null +++ b/.sqlx/query-60433164aae6d537f4b9056a5f9cf46af129639d2cc4cdc3262293846c088906.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE buildsteps SET busy = 0, status = $1, stopTime = $2 WHERE busy != 0;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "60433164aae6d537f4b9056a5f9cf46af129639d2cc4cdc3262293846c088906" +} diff --git a/.sqlx/query-68ff1a22bdbfe552106035ff2f7fd349df6c1da958374a4842124581c7654d5c.json b/.sqlx/query-68ff1a22bdbfe552106035ff2f7fd349df6c1da958374a4842124581c7654d5c.json new file mode 100644 index 000000000..06c4a46c8 --- /dev/null +++ b/.sqlx/query-68ff1a22bdbfe552106035ff2f7fd349df6c1da958374a4842124581c7654d5c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE buildoutputs SET path = $3 WHERE build = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "68ff1a22bdbfe552106035ff2f7fd349df6c1da958374a4842124581c7654d5c" +} diff --git a/.sqlx/query-694cba4e8063ef370358e1c5828b9450c3c06b8449eeee76f2a3a5131a6e3006.json b/.sqlx/query-694cba4e8063ef370358e1c5828b9450c3c06b8449eeee76f2a3a5131a6e3006.json new file mode 100644 index 000000000..1f2bcfa96 --- /dev/null +++ b/.sqlx/query-694cba4e8063ef370358e1c5828b9450c3c06b8449eeee76f2a3a5131a6e3006.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO systemstatus (\n what, status\n ) VALUES (\n 'queue-runner', $1\n ) ON CONFLICT (what) DO UPDATE SET status = EXCLUDED.status;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Json" + ] + }, + "nullable": [] + }, + "hash": "694cba4e8063ef370358e1c5828b9450c3c06b8449eeee76f2a3a5131a6e3006" +} diff --git a/.sqlx/query-6a79a1188d2334feb26b255c1d57430873aa1b192cacdc98ebf7a8e48665d7a4.json b/.sqlx/query-6a79a1188d2334feb26b255c1d57430873aa1b192cacdc98ebf7a8e48665d7a4.json new file mode 100644 index 000000000..06ea27bc4 --- /dev/null +++ b/.sqlx/query-6a79a1188d2334feb26b255c1d57430873aa1b192cacdc98ebf7a8e48665d7a4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE buildsteps SET\n busy = 0,\n status = $1,\n errorMsg = $4,\n startTime = $5,\n stopTime = $6,\n machine = $7,\n overhead = $8,\n timesBuilt = $9,\n isNonDeterministic = $10\n WHERE\n build = $2 AND stepnr = $3\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Int4", + "Text", + "Int4", + "Int4", + "Text", + "Int4", + "Int4", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "6a79a1188d2334feb26b255c1d57430873aa1b192cacdc98ebf7a8e48665d7a4" +} diff --git a/.sqlx/query-7b0ede437aa972b2588415a5bfe8ff545285d457dc809601c9c780c955e16ff5.json b/.sqlx/query-7b0ede437aa972b2588415a5bfe8ff545285d457dc809601c9c780c955e16ff5.json new file mode 100644 index 000000000..a04662e97 --- /dev/null +++ b/.sqlx/query-7b0ede437aa972b2588415a5bfe8ff545285d457dc809601c9c780c955e16ff5.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n type,\n subtype,\n fileSize,\n sha256hash,\n path,\n name,\n defaultPath\n FROM buildproducts\n WHERE build = $1 ORDER BY productnr;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "type", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "subtype", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "filesize", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "sha256hash", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "path", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "name", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "defaultpath", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + false, + true, + true, + true, + false, + true + ] + }, + "hash": "7b0ede437aa972b2588415a5bfe8ff545285d457dc809601c9c780c955e16ff5" +} diff --git a/.sqlx/query-83b10946d5b6a044491b73af2c8add459c04deb6b69e755840ff43e859fcd7a9.json b/.sqlx/query-83b10946d5b6a044491b73af2c8add459c04deb6b69e755840ff43e859fcd7a9.json new file mode 100644 index 000000000..ed37db39f --- /dev/null +++ b/.sqlx/query-83b10946d5b6a044491b73af2c8add459c04deb6b69e755840ff43e859fcd7a9.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n name, unit, value\n FROM buildmetrics\n WHERE build = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "unit", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "83b10946d5b6a044491b73af2c8add459c04deb6b69e755840ff43e859fcd7a9" +} diff --git a/.sqlx/query-84173580db29e288f8e83dbae7ab5636bdcab6b0243168dde7cb7f81d7129fac.json b/.sqlx/query-84173580db29e288f8e83dbae7ab5636bdcab6b0243168dde7cb7f81d7129fac.json new file mode 100644 index 000000000..1803bad92 --- /dev/null +++ b/.sqlx/query-84173580db29e288f8e83dbae7ab5636bdcab6b0243168dde7cb7f81d7129fac.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO builds (\n finished,\n timestamp,\n jobset_id,\n job,\n nixname,\n drvpath,\n system,\n maxsilent,\n timeout,\n ischannel,\n iscurrent,\n priority,\n globalpriority,\n keep\n ) VALUES (\n 0,\n EXTRACT(EPOCH FROM NOW())::INT4,\n $1,\n 'debug',\n 'debug',\n $2,\n $3,\n 7200,\n 36000,\n 0,\n 0,\n 100,\n 0,\n 0);", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "84173580db29e288f8e83dbae7ab5636bdcab6b0243168dde7cb7f81d7129fac" +} diff --git a/.sqlx/query-877e109257b264b87d8f0b90e91dcd7a913bfb4d216dd190eacd63f5a5f5cd45.json b/.sqlx/query-877e109257b264b87d8f0b90e91dcd7a913bfb4d216dd190eacd63f5a5f5cd45.json new file mode 100644 index 000000000..c14f22e80 --- /dev/null +++ b/.sqlx/query-877e109257b264b87d8f0b90e91dcd7a913bfb4d216dd190eacd63f5a5f5cd45.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT MAX(build) FROM buildsteps WHERE drvPath = $1 and startTime != 0 and stopTime != 0 and status = 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "877e109257b264b87d8f0b90e91dcd7a913bfb4d216dd190eacd63f5a5f5cd45" +} diff --git a/.sqlx/query-8d6f5c4a866eca834ec76d1b7068f4091f65139067f38b16b047ba2561c0cfee.json b/.sqlx/query-8d6f5c4a866eca834ec76d1b7068f4091f65139067f38b16b047ba2561c0cfee.json new file mode 100644 index 000000000..2789d301e --- /dev/null +++ b/.sqlx/query-8d6f5c4a866eca834ec76d1b7068f4091f65139067f38b16b047ba2561c0cfee.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE builds SET\n finished = 1,\n buildStatus = $2,\n startTime = $3,\n stopTime = $3,\n isCachedBuild = 1,\n notificationPendingSince = $3\n WHERE\n id = $1 AND finished = 0", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "8d6f5c4a866eca834ec76d1b7068f4091f65139067f38b16b047ba2561c0cfee" +} diff --git a/.sqlx/query-9ac00b06367900308f94877d24cc7db38d6e03e9208206195f52214648ddc2d2.json b/.sqlx/query-9ac00b06367900308f94877d24cc7db38d6e03e9208206195f52214648ddc2d2.json new file mode 100644 index 000000000..f597331b7 --- /dev/null +++ b/.sqlx/query-9ac00b06367900308f94877d24cc7db38d6e03e9208206195f52214648ddc2d2.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n id,\n globalPriority\n FROM builds\n WHERE finished = 0;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "globalpriority", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "9ac00b06367900308f94877d24cc7db38d6e03e9208206195f52214648ddc2d2" +} diff --git a/.sqlx/query-a1a314a6add9d6b0f8e0930bf3422165a892887680443545bbfcb2b94925706a.json b/.sqlx/query-a1a314a6add9d6b0f8e0930bf3422165a892887680443545bbfcb2b94925706a.json new file mode 100644 index 000000000..850901218 --- /dev/null +++ b/.sqlx/query-a1a314a6add9d6b0f8e0930bf3422165a892887680443545bbfcb2b94925706a.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE buildstepoutputs SET path = $4 WHERE build = $1 AND stepnr = $2 AND name = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a1a314a6add9d6b0f8e0930bf3422165a892887680443545bbfcb2b94925706a" +} diff --git a/.sqlx/query-a6cedfa152e75af251a255584fca2d67fdd2bf0bb258f4320da7c00bdf3c7680.json b/.sqlx/query-a6cedfa152e75af251a255584fca2d67fdd2bf0bb258f4320da7c00bdf3c7680.json new file mode 100644 index 000000000..38ee720f3 --- /dev/null +++ b/.sqlx/query-a6cedfa152e75af251a255584fca2d67fdd2bf0bb258f4320da7c00bdf3c7680.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO failedpaths (\n path\n ) VALUES (\n $1\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "a6cedfa152e75af251a255584fca2d67fdd2bf0bb258f4320da7c00bdf3c7680" +} diff --git a/.sqlx/query-a75df630ab35feb02b610618f538a1b3f5d45e09ad11c2c7a6113a505895c792.json b/.sqlx/query-a75df630ab35feb02b610618f538a1b3f5d45e09ad11c2c7a6113a505895c792.json new file mode 100644 index 000000000..d5319a0ac --- /dev/null +++ b/.sqlx/query-a75df630ab35feb02b610618f538a1b3f5d45e09ad11c2c7a6113a505895c792.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT MAX(stepnr) FROM buildsteps WHERE build = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a75df630ab35feb02b610618f538a1b3f5d45e09ad11c2c7a6113a505895c792" +} diff --git a/.sqlx/query-a9d877539b53ae4a71066f39613b2d02ed7a72e8ca184d2935a07f498a8992a7.json b/.sqlx/query-a9d877539b53ae4a71066f39613b2d02ed7a72e8ca184d2935a07f498a8992a7.json new file mode 100644 index 000000000..dc22ca5be --- /dev/null +++ b/.sqlx/query-a9d877539b53ae4a71066f39613b2d02ed7a72e8ca184d2935a07f498a8992a7.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE buildsteps SET busy = $1 WHERE build = $2 AND stepnr = $3 AND busy != 0 AND status IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "a9d877539b53ae4a71066f39613b2d02ed7a72e8ca184d2935a07f498a8992a7" +} diff --git a/.sqlx/query-aba8c9ff906e7abdbc49f2a4fb4a15a05ea5fab14dfa676cfb6a5a9d84297438.json b/.sqlx/query-aba8c9ff906e7abdbc49f2a4fb4a15a05ea5fab14dfa676cfb6a5a9d84297438.json new file mode 100644 index 000000000..74ea6e8d8 --- /dev/null +++ b/.sqlx/query-aba8c9ff906e7abdbc49f2a4fb4a15a05ea5fab14dfa676cfb6a5a9d84297438.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT s.startTime, s.stopTime FROM buildsteps s join builds b on build = id\n WHERE\n s.startTime IS NOT NULL AND\n to_timestamp(s.stopTime) > (NOW() - (interval '1 second' * $1)) AND\n jobset_id = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "starttime", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "stoptime", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Float8", + "Int4" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "aba8c9ff906e7abdbc49f2a4fb4a15a05ea5fab14dfa676cfb6a5a9d84297438" +} diff --git a/.sqlx/query-af02dbe27d340f0ee8ed4803d3a1ecf1fa7268064dba9130bbd833f1f484e2a7.json b/.sqlx/query-af02dbe27d340f0ee8ed4803d3a1ecf1fa7268064dba9130bbd833f1f484e2a7.json new file mode 100644 index 000000000..b7363a23b --- /dev/null +++ b/.sqlx/query-af02dbe27d340f0ee8ed4803d3a1ecf1fa7268064dba9130bbd833f1f484e2a7.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT status FROM systemstatus WHERE what = 'queue-runner';", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status", + "type_info": "Json" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "af02dbe27d340f0ee8ed4803d3a1ecf1fa7268064dba9130bbd833f1f484e2a7" +} diff --git a/.sqlx/query-b0f2aab8d3aae78a8c695d0a9433b19cb5ba6806d74c252593b76243e1c8c853.json b/.sqlx/query-b0f2aab8d3aae78a8c695d0a9433b19cb5ba6806d74c252593b76243e1c8c853.json new file mode 100644 index 000000000..b3a602296 --- /dev/null +++ b/.sqlx/query-b0f2aab8d3aae78a8c695d0a9433b19cb5ba6806d74c252593b76243e1c8c853.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM failedpaths where path = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "b0f2aab8d3aae78a8c695d0a9433b19cb5ba6806d74c252593b76243e1c8c853" +} diff --git a/.sqlx/query-b600c93d66b45b9859173e0e0f7f36b29a641f881e3e61f728b39ddf068dddde.json b/.sqlx/query-b600c93d66b45b9859173e0e0f7f36b29a641f881e3e61f728b39ddf068dddde.json new file mode 100644 index 000000000..0f6f3ba6e --- /dev/null +++ b/.sqlx/query-b600c93d66b45b9859173e0e0f7f36b29a641f881e3e61f728b39ddf068dddde.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM buildproducts WHERE build = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "b600c93d66b45b9859173e0e0f7f36b29a641f881e3e61f728b39ddf068dddde" +} diff --git a/.sqlx/query-d155253f103fb9c24374199bd58a827071e69bf33e427bfa040a83d07b909e31.json b/.sqlx/query-d155253f103fb9c24374199bd58a827071e69bf33e427bfa040a83d07b909e31.json new file mode 100644 index 000000000..03f62654b --- /dev/null +++ b/.sqlx/query-d155253f103fb9c24374199bd58a827071e69bf33e427bfa040a83d07b909e31.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT schedulingshares FROM jobsets WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "schedulingshares", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d155253f103fb9c24374199bd58a827071e69bf33e427bfa040a83d07b909e31" +} diff --git a/.sqlx/query-d1a70032949d4c470a20ba89589ce6f36c7fd576ed02d868f4185dd9f886a30b.json b/.sqlx/query-d1a70032949d4c470a20ba89589ce6f36c7fd576ed02d868f4185dd9f886a30b.json new file mode 100644 index 000000000..ab1332f4a --- /dev/null +++ b/.sqlx/query-d1a70032949d4c470a20ba89589ce6f36c7fd576ed02d868f4185dd9f886a30b.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO buildmetrics (\n build,\n name,\n unit,\n value,\n project,\n jobset,\n job,\n timestamp\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Text", + "Text", + "Float8", + "Text", + "Text", + "Text", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "d1a70032949d4c470a20ba89589ce6f36c7fd576ed02d868f4185dd9f886a30b" +} diff --git a/.sqlx/query-e0d7d4bdb291d4d403aa37627d8b1cdc91d6c3515d2ae530fe14830950eecffb.json b/.sqlx/query-e0d7d4bdb291d4d403aa37627d8b1cdc91d6c3515d2ae530fe14830950eecffb.json new file mode 100644 index 000000000..9a73e57fd --- /dev/null +++ b/.sqlx/query-e0d7d4bdb291d4d403aa37627d8b1cdc91d6c3515d2ae530fe14830950eecffb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT MAX(s.build) FROM buildsteps s\n JOIN BuildStepOutputs o ON s.build = o.build\n WHERE startTime != 0\n AND stopTime != 0\n AND status = 1\n AND path = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e0d7d4bdb291d4d403aa37627d8b1cdc91d6c3515d2ae530fe14830950eecffb" +} diff --git a/.sqlx/query-e418dcd190f68642391dc8793fc30ad500dc5d513fce55a901e1d4a400323cc8.json b/.sqlx/query-e418dcd190f68642391dc8793fc30ad500dc5d513fce55a901e1d4a400323cc8.json new file mode 100644 index 000000000..c670f2083 --- /dev/null +++ b/.sqlx/query-e418dcd190f68642391dc8793fc30ad500dc5d513fce55a901e1d4a400323cc8.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n id, buildStatus, releaseName, closureSize, size\n FROM builds b\n JOIN buildoutputs o on b.id = o.build\n WHERE finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "buildstatus", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "releasename", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "closuresize", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "e418dcd190f68642391dc8793fc30ad500dc5d513fce55a901e1d4a400323cc8" +} diff --git a/.sqlx/query-ec762ac14489cf15630e71703f7f1baf0e4fcf0a5bf41293343c626ce1b64f1a.json b/.sqlx/query-ec762ac14489cf15630e71703f7f1baf0e4fcf0a5bf41293343c626ce1b64f1a.json new file mode 100644 index 000000000..9aeb1026e --- /dev/null +++ b/.sqlx/query-ec762ac14489cf15630e71703f7f1baf0e4fcf0a5bf41293343c626ce1b64f1a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM buildmetrics WHERE build = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "ec762ac14489cf15630e71703f7f1baf0e4fcf0a5bf41293343c626ce1b64f1a" +} diff --git a/.sqlx/query-ed7c8532b3c113ceb008675379f500a950610c46d6edafa7d9dca960398d8e45.json b/.sqlx/query-ed7c8532b3c113ceb008675379f500a950610c46d6edafa7d9dca960398d8e45.json new file mode 100644 index 000000000..9eaae642e --- /dev/null +++ b/.sqlx/query-ed7c8532b3c113ceb008675379f500a950610c46d6edafa7d9dca960398d8e45.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO buildproducts (\n build,\n productnr,\n type,\n subtype,\n fileSize,\n sha256hash,\n path,\n name,\n defaultPath\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Text", + "Text", + "Int8", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ed7c8532b3c113ceb008675379f500a950610c46d6edafa7d9dca960398d8e45" +} diff --git a/.sqlx/query-f99df144ca85c041624011f32dcfa0cce129a17edc95fad2444552bee8d6779d.json b/.sqlx/query-f99df144ca85c041624011f32dcfa0cce129a17edc95fad2444552bee8d6779d.json new file mode 100644 index 000000000..383766a7d --- /dev/null +++ b/.sqlx/query-f99df144ca85c041624011f32dcfa0cce129a17edc95fad2444552bee8d6779d.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO buildsteps (\n build,\n stepnr,\n type,\n drvPath,\n busy,\n startTime,\n stopTime,\n system,\n status,\n propagatedFrom,\n errorMsg,\n machine\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12\n )\n ON CONFLICT DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Int4", + "Text", + "Int4", + "Int4", + "Int4", + "Text", + "Int4", + "Int4", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f99df144ca85c041624011f32dcfa0cce129a17edc95fad2444552bee8d6779d" +} diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000..8ab488298 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,4855 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "arc-swap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d03449bb8ca2cc2ef70869af31463d1ae5ccc8fa3e334b307203fbf815207e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-compression" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "atomic_float" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628d228f918ac3b82fe590352cc719d30664a0c13ca3a60266fe02c7132d480a" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "binary-cache" +version = "0.1.0" +dependencies = [ + "async-compression", + "backon", + "bytes", + "configparser", + "foldhash 0.2.0", + "fs-err", + "futures", + "hashbrown 0.16.1", + "hydra-tracing", + "moka", + "nix-utils", + "object_store", + "reqwest", + "secrecy", + "serde", + "serde_json", + "sha2", + "smallvec", + "tempfile", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +dependencies = [ + "serde_core", +] + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "borsh" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", +] + +[[package]] +name = "builder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-stream", + "backon", + "binary-cache", + "clap", + "fs-err", + "futures", + "gethostname", + "hashbrown 0.16.1", + "hydra-tracing", + "hyper-util", + "nix", + "nix-utils", + "parking_lot", + "procfs-core 0.18.0", + "prost", + "sd-notify", + "sha2", + "shared", + "sysinfo", + "thiserror", + "tikv-jemallocator", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tonic-prost-build", + "tower", + "tracing", + "uuid", +] + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "byte-unit" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c6d47a4e2961fb8721bcfc54feae6455f2f64e7054f9bc67e875f0e77f4c58d" +dependencies = [ + "rust_decimal", + "schemars", + "serde", + "utf8-width", +] + +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cc" +version = "1.2.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd4932aefd12402b36c60956a4fe0035421f544799057659ff86f923657aada3" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link 0.2.1", +] + +[[package]] +name = "clap" +version = "4.5.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "clap_lex" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" + +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "compression-codecs" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +dependencies = [ + "brotli", + "bzip2", + "compression-core", + "liblzma", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "configparser" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57e3272f0190c3f1584272d613719ba5fc7df7f4942fe542e63d949cf3a649b" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cxx" +version = "1.0.192" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbda285ba6e5866529faf76352bdf73801d9b44a6308d7cd58ca2379f378e994" +dependencies = [ + "cc", + "cxx-build", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash 0.2.0", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.192" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9efde466c5d532d57efd92f861da3bdb7f61e369128ce8b4c3fe0c9de4fa4d" +dependencies = [ + "cc", + "codespan-reporting", + "indexmap", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.114", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.192" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3efb93799095bccd4f763ca07997dc39a69e5e61ab52d2c407d4988d21ce144d" +dependencies = [ + "clap", + "codespan-reporting", + "indexmap", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.192" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3092010228026e143b32a4463ed9fa8f86dca266af4bf5f3b2a26e113dbe4e45" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.192" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31d72ebfcd351ae404fb00ff378dfc9571827a00722c9e735c9181aec320ba0a" +dependencies = [ + "indexmap", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "db" +version = "0.1.0" +dependencies = [ + "anyhow", + "futures", + "hashbrown 0.16.1", + "jiff", + "serde_json", + "sqlx", + "tracing", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f449e6c6c08c865631d4890cfacf252b3d396c9bcc83adb6623cdb02a8336c41" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flate2" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs-err" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf68cef89750956493a66a10f512b9e58d9db21f2a573c079c0bdf1207a54a7" +dependencies = [ + "autocfg", + "tokio", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix 1.1.3", + "windows-link 0.2.1", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hydra-tracing" +version = "0.1.0" +dependencies = [ + "anyhow", + "http", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "tonic", + "tracing", + "tracing-log", + "tracing-opentelemetry", + "tracing-subscriber", +] + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.5", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core 0.62.2", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jiff" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys 0.61.2", +] + +[[package]] +name = "jiff-static" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + +[[package]] +name = "libc" +version = "0.2.180" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" + +[[package]] +name = "liblzma" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" +dependencies = [ + "liblzma-sys", + "num_cpus", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "libredox" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +dependencies = [ + "bitflags", + "libc", + "redox_syscall 0.7.0", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" +dependencies = [ + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "listenfd" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87bc54a4629b4294d0b3ef041b64c40c611097a677d9dc07b2c67739fe39dba" +dependencies = [ + "libc", + "uuid", + "winapi", +] + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nix-diff" +version = "0.1.0" +source = "git+https://github.com/mic92/nix-diff-rs.git?rev=7b79a68963fd7fcb48a57071b52bc90bd8d60609#7b79a68963fd7fcb48a57071b52bc90bd8d60609" +dependencies = [ + "anyhow", + "memchr", + "similar", + "tempfile", + "tikv-jemallocator", + "tinyjson", +] + +[[package]] +name = "nix-utils" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "cxx", + "cxx-build", + "fs-err", + "futures", + "hashbrown 0.16.1", + "nix-diff", + "pkg-config", + "serde", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "ntapi" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "object_store" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d180d5469872facb82dec7e233ff850d615330ea3044a7813550b22ffa4f05ce" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.9.2", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391" + +[[package]] +name = "opentelemetry" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", + "tracing", +] + +[[package]] +name = "opentelemetry-http" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" +dependencies = [ + "http", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest", + "thiserror", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", + "tonic-prost", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" + +[[package]] +name = "opentelemetry_sdk" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "rand 0.9.2", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link 0.2.1", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.114", +] + +[[package]] +name = "proc-macro-crate" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "procfs" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" +dependencies = [ + "bitflags", + "hex", + "procfs-core 0.17.0", + "rustix 0.38.44", +] + +[[package]] +name = "procfs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" +dependencies = [ + "bitflags", + "chrono", + "flate2", + "procfs-core 0.18.0", + "rustix 1.1.3", +] + +[[package]] +name = "procfs-core" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" +dependencies = [ + "bitflags", + "hex", +] + +[[package]] +name = "procfs-core" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405" +dependencies = [ + "bitflags", + "chrono", + "hex", +] + +[[package]] +name = "prometheus" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "procfs 0.17.0", + "thiserror", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.114", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] + +[[package]] +name = "queue-runner" +version = "0.1.0" +dependencies = [ + "anyhow", + "arc-swap", + "atomic_float", + "backon", + "binary-cache", + "byte-unit", + "bytes", + "clap", + "db", + "fs-err", + "futures", + "futures-util", + "h2", + "hashbrown 0.16.1", + "http-body-util", + "hydra-tracing", + "hyper", + "hyper-util", + "jiff", + "listenfd", + "nix-utils", + "parking_lot", + "procfs 0.18.0", + "prometheus", + "prost", + "sd-notify", + "secrecy", + "serde", + "serde_json", + "sha2", + "shared", + "smallvec", + "thiserror", + "tikv-jemallocator", + "tokio", + "tokio-stream", + "toml", + "tonic", + "tonic-health", + "tonic-prost", + "tonic-prost-build", + "tonic-reflection", + "tower", + "tower-http", + "tracing", + "uuid", +] + +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.5", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rkyv" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust_decimal" +version = "1.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.5", + "rkyv", + "serde", + "serde_json", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scratch" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" + +[[package]] +name = "sd-notify" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4" +dependencies = [ + "libc", +] + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "serde", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shared" +version = "0.1.0" +dependencies = [ + "anyhow", + "fs-err", + "nix-utils", + "regex", + "sha2", + "tokio", + "tracing", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +dependencies = [ + "bstr", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.114", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.114", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror", + "tracing", + "url", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "sysinfo" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix 1.1.3", + "windows-sys 0.61.2", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tinyjson" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab95735ea2c8fd51154d01e39cf13912a78071c2d89abc49a7ef102a7dd725a" + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.9.11+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.23.10+spec-1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.0.6+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.0.6+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" + +[[package]] +name = "tonic" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", + "webpki-roots 1.0.5", + "zstd", +] + +[[package]] +name = "tonic-build" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c40aaccc9f9eccf2cd82ebc111adc13030d23e887244bc9cfa5d1d636049de3" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "tonic-health" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a82868bf299e0a1d2e8dce0dc33a46c02d6f045b2c1f1d6cc8dc3d0bf1812ef" +dependencies = [ + "prost", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", +] + +[[package]] +name = "tonic-prost" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a16cba4043dc3ff43fcb3f96b4c5c154c64cbd18ca8dce2ab2c6a451d058a2" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.114", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tonic-reflection" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34da53e8387581d66db16ff01f98a70b426b091fdf76856e289d5c1bd386ed7b" +dependencies = [ + "prost", + "prost-types", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +dependencies = [ + "js-sys", + "opentelemetry", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.114", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.5", +] + +[[package]] +name = "webpki-roots" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core 0.61.2", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link 0.2.1", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "zmij" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac93432f5b761b22864c774aac244fa5c0fd877678a4c37ebf6cf42208f9c9ec" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..731ed49ce --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,85 @@ +[workspace] + +resolver = "2" +members = ["src/builder", "src/queue-runner", "src/crates/*"] + +[workspace.package] +version = "0.1.0" +rust-version = "1.91.0" + +[workspace.dependencies] +anyhow = "1.0.98" +arc-swap = "1.7" +async-compression = { version = "0.4", default-features = false } +async-stream = "0.3" +atomic_float = "1.1.0" +backon = "1.5.2" +byte-unit = "5.1.6" +bytes = "1" +clap = "4" +configparser = "3.1" +cxx = "1" +cxx-build = "1" +digest = "0.10.7" +foldhash = "0.2" +fs-err = "3.0" +futures = "0.3" +futures-util = "0.3" +gethostname = "1" +h2 = "0.4" +hashbrown = "0.16" +http = "1.1" +http-body-util = "0.1" +hyper = { version = "1", features = ["full"] } +hyper-util = "0.1.10" +jiff = { version = "0.2", features = ["serde"] } +listenfd = "1" +moka = { version = "0.12", features = ["future"] } +nix = { version = "0.30", default-features = false } +nix-diff = { git = "https://github.com/mic92/nix-diff-rs.git", rev = "7b79a68963fd7fcb48a57071b52bc90bd8d60609" } +object_store = "0.13.0" +opentelemetry = "0.31.0" +opentelemetry-http = "0.31.0" +opentelemetry-otlp = "0.31.0" +opentelemetry-semantic-conventions = "0.31.0" +opentelemetry_sdk = "0.31.0" +parking_lot = "0.12.4" +pkg-config = "0.3" +procfs = "0.18" +procfs-core = "0.18" +prometheus = { version = "0.14.0", default-features = false } +prost = "0.14" +regex = "1" +reqwest = { version = "0.12", default-features = false, features = ["http2", "charset", "rustls-tls-webpki-roots", "system-proxy"] } +sd-notify = "0.4.5" +secrecy = "0.10" +serde = "1.0" +serde_json = "1.0" +sha2 = "0.10" +smallvec = "1.15" +sqlx = "0.8" +sysinfo = "0.37.0" +thiserror = "2.0" +tikv-jemallocator = "0.6" +tokio = { version = "1.34", features = ["full"] } +tokio-stream = "0.1" +tokio-util = "0.7" +toml = "0.9.0" +tonic = "0.14" +tonic-health = { version = "0.14", default-features = false } +tonic-prost = "0.14" +tonic-prost-build = "0.14" +tonic-reflection = "0.14" +tower = "0.5" +tower-http = "0.6" +tracing = "0.1" +tracing-log = "0.2.0" +tracing-opentelemetry = "0.32.0" +tracing-subscriber = "0.3.18" +url = "2.5.4" +uuid = "1.16" + +[profile.release] +lto = true +codegen-units = 1 +opt-level = 3 diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 000000000..d3e369eea --- /dev/null +++ b/clippy.toml @@ -0,0 +1,56 @@ +ignore-interior-mutability = ["bytes::Bytes", "queue_runner::build::Step"] +disallowed-methods = [ + { path = "std::fs::canonicalize", replacement = "fs_err::canonicalize" }, + { path = "std::fs::copy", replacement = "fs_err::copy" }, + { path = "std::fs::create_dir", replacement = "fs_err::create_dir" }, + { path = "std::fs::create_dir_all", replacement = "fs_err::create_dir_all" }, + { path = "std::fs::exists", replacement = "fs_err::exists" }, + { path = "std::fs::hard_link", replacement = "fs_err::hard_link" }, + { path = "std::fs::metadata", replacement = "fs_err::metadata" }, + { path = "std::fs::read", replacement = "fs_err::read" }, + { path = "std::fs::read_dir", replacement = "fs_err::read_dir" }, + { path = "std::fs::read_link", replacement = "fs_err::read_link" }, + { path = "std::fs::read_to_string", replacement = "fs_err::read_to_string" }, + { path = "std::fs::remove_dir", replacement = "fs_err::remove_dir" }, + { path = "std::fs::remove_dir_all", replacement = "fs_err::remove_dir_all" }, + { path = "std::fs::remove_file", replacement = "fs_err::remove_file" }, + { path = "std::fs::rename", replacement = "fs_err::rename" }, + { path = "std::fs::set_permissions", replacement = "fs_err::set_permissions" }, + { path = "std::fs::symlink_metadata", replacement = "fs_err::symlink_metadata" }, + { path = "std::fs::write", replacement = "fs_err::write" }, + { path = "std::fs::File::open", replacement = "fs_err::File::open" }, + { path = "std::fs::File::create", replacement = "fs_err::File::create" }, + { path = "std::fs::File::create_new", replacement = "fs_err::File::create_new" }, + { path = "std::fs::OpenOptions::new", replacement = "fs_err::OpenOptions::new" }, + + { path = "std::os::unix::fs::symlink", replacement = "fs_err::os::unix::symlink" }, + + { path = "std::path::Path::try_exists", reason = "Use fs_err::path::PathExt methods" }, + { path = "std::path::Path::metadata", reason = "Use fs_err::path::PathExt methods" }, + { path = "std::path::Path::symlink_metadata", reason = "Use fs_err::path::PathExt methods" }, + { path = "std::path::Path::canonicalize", reason = "Use fs_err::path::PathExt methods" }, + { path = "std::path::Path::read_link", reason = "Use fs_err::path::PathExt methods" }, + { path = "std::path::Path::read_dir", reason = "Use fs_err::path::PathExt methods" }, + + { path = "tokio::fs::canonicalize", replacement = "fs_err::tokio::canonicalize" }, + { path = "tokio::fs::copy", replacement = "fs_err::tokio::copy" }, + { path = "tokio::fs::create_dir", replacement = "fs_err::tokio::create_dir" }, + { path = "tokio::fs::create_dir_all", replacement = "fs_err::tokio::create_dir_all" }, + { path = "tokio::fs::hard_link", replacement = "fs_err::tokio::hard_link" }, + { path = "tokio::fs::metadata", replacement = "fs_err::tokio::metadata" }, + { path = "tokio::fs::read", replacement = "fs_err::tokio::read" }, + { path = "tokio::fs::read_dir", replacement = "fs_err::tokio::read_dir" }, + { path = "tokio::fs::read_link", replacement = "fs_err::tokio::read_link" }, + { path = "tokio::fs::read_to_string", replacement = "fs_err::tokio::read_to_string" }, + { path = "tokio::fs::remove_dir", replacement = "fs_err::tokio::remove_dir" }, + { path = "tokio::fs::remove_dir_all", replacement = "fs_err::tokio::remove_dir_all" }, + { path = "tokio::fs::remove_file", replacement = "fs_err::tokio::remove_file" }, + { path = "tokio::fs::rename", replacement = "fs_err::tokio::rename" }, + { path = "tokio::fs::set_permissions", replacement = "fs_err::tokio::set_permissions" }, + { path = "tokio::fs::symlink_metadata", replacement = "fs_err::tokio::symlink_metadata" }, + { path = "tokio::fs::write", replacement = "fs_err::tokio::write" }, + { path = "tokio::fs::File::open", replacement = "fs_err::tokio::File::open" }, + { path = "tokio::fs::File::create", replacement = "fs_err::tokio::File::create" }, + { path = "tokio::fs::File::create_new", replacement = "fs_err::tokio::File::create_new" }, + { path = "tokio::fs::OpenOptions::new", replacement = "fs_err::tokio::OpenOptions::new" }, +] diff --git a/meson.build b/meson.build index 923794e6e..a620dd63b 100644 --- a/meson.build +++ b/meson.build @@ -20,6 +20,9 @@ prom_cpp_pull_dep = dependency('prometheus-cpp-pull', required: true) mdbook = find_program('mdbook', native: true) perl = find_program('perl', native: true) +cargo = find_program('cargo', native: true, required: true) +rustc = find_program('rustc', native: true, required: true) + subdir('doc/manual') subdir('nixos-modules') subdir('src') diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 000000000..9e2742ab2 --- /dev/null +++ b/meson_options.txt @@ -0,0 +1 @@ +option('otel', type: 'feature', value: 'disabled', description: 'Enable OpenTelemetry support for Rust components') diff --git a/nixos-modules/darwin-builder-module.nix b/nixos-modules/darwin-builder-module.nix new file mode 100644 index 000000000..5d3d8b982 --- /dev/null +++ b/nixos-modules/darwin-builder-module.nix @@ -0,0 +1,254 @@ +{ + config, + pkgs, + lib, + ... +}: +let + cfg = config.services.queue-builder-dev; + user = config.users.users.hydra-queue-builder; +in +{ + options = { + services.queue-builder-dev = { + enable = lib.mkEnableOption "QueueBuilder"; + + queueRunnerAddr = lib.mkOption { + description = "Queue Runner address to the grpc server"; + type = lib.types.singleLineStr; + }; + + pingInterval = lib.mkOption { + description = "Interval in which pings are send to the runner"; + type = lib.types.ints.positive; + default = 10; + }; + + speedFactor = lib.mkOption { + description = "Additional Speed factor for this machine"; + type = lib.types.oneOf [ + lib.types.ints.positive + lib.types.float + ]; + default = 1; + }; + + maxJobs = lib.mkOption { + description = "Maximum allowed of jobs. This only is used if the queue runner uses this metrics for determining free machines."; + type = lib.types.ints.positive; + default = 4; + }; + + tmpAvailThreshold = lib.mkOption { + description = "Threshold in percent for /tmp before jobs are no longer scheduled on the machine"; + type = lib.types.float; + default = 10.0; + }; + + storeAvailThreshold = lib.mkOption { + description = "Threshold in percent for /nix/store before jobs are no longer scheduled on the machine"; + type = lib.types.float; + default = 10.0; + }; + + load1Threshold = lib.mkOption { + description = "Maximum Load1 threshold before we stop scheduling jobs on that node. Only used if PSI is not available."; + type = lib.types.float; + default = 8.0; + }; + + cpuPsiThreshold = lib.mkOption { + description = "Maximum CPU PSI in the last 10s before we stop scheduling jobs on that node"; + type = lib.types.float; + default = 75.0; + }; + + memPsiThreshold = lib.mkOption { + description = "Maximum Memory PSI in the last 10s before we stop scheduling jobs on that node"; + type = lib.types.float; + default = 80.0; + }; + + ioPsiThreshold = lib.mkOption { + description = "Maximum IO PSI in the last 10s before we stop scheduling jobs on that node. If null then this pressure check is disabled."; + type = lib.types.nullOr lib.types.float; + default = null; + }; + + systems = lib.mkOption { + description = "List of supported systems. If none are passed, system and extra-platforms are read from nix."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + supportedFeatures = lib.mkOption { + description = "Pass supported features to the builder. If none are passed, system features will be used."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + mandatoryFeatures = lib.mkOption { + description = "Pass mandatory features to the builder."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + useSubstitutes = lib.mkOption { + description = "Use substitution for paths"; + type = lib.types.bool; + default = true; + }; + + authorizationFile = lib.mkOption { + description = "Path to token authorization file if token auth should be used."; + type = lib.types.nullOr lib.types.path; + default = null; + }; + + mtls = lib.mkOption { + description = "mtls options"; + default = null; + type = lib.types.nullOr ( + lib.types.submodule { + options = { + serverRootCaCertPath = lib.mkOption { + description = "Server root ca certificate path"; + type = lib.types.path; + }; + clientCertPath = lib.mkOption { + description = "Client certificate path"; + type = lib.types.path; + }; + clientKeyPath = lib.mkOption { + description = "Client key path"; + type = lib.types.path; + }; + domainName = lib.mkOption { + description = "Domain name for mtls"; + type = lib.types.singleLineStr; + }; + }; + } + ); + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.callPackage ./. { }; + }; + + logFile = lib.mkOption { + type = lib.types.path; + default = "/var/log/hydra-queue-builder.log"; + description = "The logfile to use for the hydra-queue-builder service."; + }; + }; + }; + + config = lib.mkIf cfg.enable { + launchd.daemons.queue-builder-dev = { + script = '' + exec ${ + lib.escapeShellArgs ( + [ + "${cfg.package}/bin/builder" + "--gateway-endpoint" + cfg.queueRunnerAddr + "--ping-interval" + cfg.pingInterval + "--speed-factor" + cfg.speedFactor + "--max-jobs" + cfg.maxJobs + "--tmp-avail-threshold" + cfg.tmpAvailThreshold + "--store-avail-threshold" + cfg.storeAvailThreshold + "--load1-threshold" + cfg.load1Threshold + "--cpu-psi-threshold" + cfg.cpuPsiThreshold + "--mem-psi-threshold" + cfg.memPsiThreshold + ] + ++ lib.optionals (cfg.ioPsiThreshold != null) [ + "--io-psi-threshold" + cfg.ioPsiThreshold + ] + ++ (builtins.concatMap (v: [ + "--systems" + v + ]) cfg.systems) + ++ (builtins.concatMap (v: [ + "--supported-features" + v + ]) cfg.supportedFeatures) + ++ (builtins.concatMap (v: [ + "--mandatory-features" + v + ]) cfg.mandatoryFeatures) + ++ lib.optionals (cfg.useSubstitutes != null) [ + "--use-substitutes" + ] + ++ lib.optionals (cfg.authorizationFile != null) [ + "--authorization-file" + cfg.authorizationFile + ] + ++ lib.optionals (cfg.mtls != null) [ + "--server-root-ca-cert-path" + cfg.mtls.serverRootCaCertPath + "--client-cert-path" + cfg.mtls.clientCertPath + "--client-key-path" + cfg.mtls.clientKeyPath + "--domain-name" + cfg.mtls.domainName + ] + ) + } + ''; + + environment = { + RUST_BACKTRACE = "1"; + NIX_SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + }; + + serviceConfig = { + KeepAlive = true; + StandardErrorPath = cfg.logFile; + StandardOutPath = cfg.logFile; + + GroupName = "hydra"; + UserName = "hydra-queue-builder"; + WorkingDirectory = user.home; + }; + }; + users = { + users.hydra-queue-builder = { + uid = lib.mkDefault 535; + gid = lib.mkDefault config.users.groups.hydra.gid; + home = lib.mkDefault "/var/lib/hydra-queue-builder"; + shell = "/bin/bash"; + description = "hydra-queue-builder service user"; + }; + knownUsers = [ "hydra-queue-builder" ]; + groups.hydra = { + gid = lib.mkDefault 535; + description = "Nix group for hydra-queue-builder service"; + }; + knownGroups = [ "hydra" ]; + }; + + # FIXME: create logfiles automatically if defined. + system.activationScripts.preActivation.text = '' + mkdir -p '${user.home}' + touch '${cfg.logFile}' + chown ${toString user.uid}:${toString user.gid} '${user.home}' '${cfg.logFile}' + + # create gcroots + mkdir -p /nix/var/nix/gcroots/per-user/hydra-queue-builder + chown ${toString user.uid}:${toString user.gid} /nix/var/nix/gcroots/per-user/hydra-queue-builder + chmod 0755 /nix/var/nix/gcroots/per-user/hydra-queue-builder + ''; + }; +} diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix index 336869c5b..b1c814a11 100644 --- a/nixos-modules/default.nix +++ b/nixos-modules/default.nix @@ -3,7 +3,11 @@ { hydra = { pkgs, lib,... }: { _file = ./default.nix; - imports = [ ./hydra.nix ]; + imports = [ + ./hydra.nix + ./linux-builder-module.nix + ./queue-runner-module.nix + ]; services.hydra-dev.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.hydra; }; diff --git a/nixos-modules/linux-builder-module.nix b/nixos-modules/linux-builder-module.nix new file mode 100644 index 000000000..ff00b9fdd --- /dev/null +++ b/nixos-modules/linux-builder-module.nix @@ -0,0 +1,289 @@ +{ + config, + pkgs, + lib, + ... +}: +let + cfg = config.services.queue-builder-dev; +in +{ + options = { + services.queue-builder-dev = { + enable = lib.mkEnableOption "QueueBuilder"; + + queueRunnerAddr = lib.mkOption { + description = "Queue Runner address to the grpc server"; + type = lib.types.singleLineStr; + }; + + pingInterval = lib.mkOption { + description = "Interval in which pings are send to the runner"; + type = lib.types.ints.positive; + default = 10; + }; + + speedFactor = lib.mkOption { + description = "Additional Speed factor for this machine"; + type = lib.types.oneOf [ + lib.types.ints.positive + lib.types.float + ]; + default = 1; + }; + + maxJobs = lib.mkOption { + description = "Maximum allowed of jobs. This only is used if the queue runner uses this metrics for determining free machines."; + type = lib.types.ints.positive; + default = 4; + }; + + buildDirAvailThreshold = lib.mkOption { + description = "Threshold in percent for nix build dir before jobs are no longer scheduled on the machine"; + type = lib.types.float; + default = 10.0; + }; + + storeAvailThreshold = lib.mkOption { + description = "Threshold in percent for /nix/store before jobs are no longer scheduled on the machine"; + type = lib.types.float; + default = 10.0; + }; + + load1Threshold = lib.mkOption { + description = "Maximum Load1 threshold before we stop scheduling jobs on that node. Only used if PSI is not available."; + type = lib.types.float; + default = 8.0; + }; + + cpuPsiThreshold = lib.mkOption { + description = "Maximum CPU PSI in the last 10s before we stop scheduling jobs on that node"; + type = lib.types.float; + default = 75.0; + }; + + memPsiThreshold = lib.mkOption { + description = "Maximum Memory PSI in the last 10s before we stop scheduling jobs on that node"; + type = lib.types.float; + default = 80.0; + }; + + ioPsiThreshold = lib.mkOption { + description = "Maximum IO PSI in the last 10s before we stop scheduling jobs on that node. If null then this pressure check is disabled."; + type = lib.types.nullOr lib.types.float; + default = null; + }; + + systems = lib.mkOption { + description = "List of supported systems. If none are passed, system and extra-platforms are read from nix."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + supportedFeatures = lib.mkOption { + description = "Pass supported features to the builder. If none are passed, system features will be used."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + mandatoryFeatures = lib.mkOption { + description = "Pass mandatory features to the builder."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + useSubstitutes = lib.mkOption { + description = "Use substitution for paths"; + type = lib.types.bool; + default = true; + }; + + authorizationFile = lib.mkOption { + description = "Path to token authorization file if token auth should be used."; + type = lib.types.nullOr lib.types.path; + default = null; + }; + + mtls = lib.mkOption { + description = "mtls options"; + default = null; + type = lib.types.nullOr ( + lib.types.submodule { + options = { + serverRootCaCertPath = lib.mkOption { + description = "Server root ca certificate path"; + type = lib.types.path; + }; + clientCertPath = lib.mkOption { + description = "Client certificate path"; + type = lib.types.path; + }; + clientKeyPath = lib.mkOption { + description = "Client key path"; + type = lib.types.path; + }; + domainName = lib.mkOption { + description = "Domain name for mtls"; + type = lib.types.singleLineStr; + }; + }; + } + ); + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.callPackage ./. { }; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.queue-builder-dev = { + description = "queue-builder main service"; + + requires = [ "nix-daemon.socket" ]; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = { + NIX_REMOTE = "daemon"; + LIBEV_FLAGS = "4"; # go ahead and mandate epoll(2) + RUST_BACKTRACE = "1"; + + # Note: it's important to set this for nix-store, because it wants to use + # $HOME in order to use a temporary cache dir. bizarre failures will occur + # otherwise + HOME = "/run/queue-builder"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + + ExecStart = lib.escapeShellArgs ( + [ + "${cfg.package}/bin/builder" + "--gateway-endpoint" + cfg.queueRunnerAddr + "--ping-interval" + cfg.pingInterval + "--speed-factor" + cfg.speedFactor + "--max-jobs" + cfg.maxJobs + "--build-dir-avail-threshold" + cfg.buildDirAvailThreshold + "--store-avail-threshold" + cfg.storeAvailThreshold + "--load1-threshold" + cfg.load1Threshold + "--cpu-psi-threshold" + cfg.cpuPsiThreshold + "--mem-psi-threshold" + cfg.memPsiThreshold + ] + ++ lib.optionals (cfg.ioPsiThreshold != null) [ + "--io-psi-threshold" + cfg.ioPsiThreshold + ] + ++ (builtins.concatMap (v: [ + "--systems" + v + ]) cfg.systems) + ++ (builtins.concatMap (v: [ + "--supported-features" + v + ]) cfg.supportedFeatures) + ++ (builtins.concatMap (v: [ + "--mandatory-features" + v + ]) cfg.mandatoryFeatures) + ++ lib.optionals (cfg.useSubstitutes != null) [ + "--use-substitutes" + ] + ++ lib.optionals (cfg.authorizationFile != null) [ + "--authorization-file" + cfg.authorizationFile + ] + ++ lib.optionals (cfg.mtls != null) [ + "--server-root-ca-cert-path" + cfg.mtls.serverRootCaCertPath + "--client-cert-path" + cfg.mtls.clientCertPath + "--client-key-path" + cfg.mtls.clientKeyPath + "--domain-name" + cfg.mtls.domainName + ] + ); + + User = "hydra-queue-builder"; + Group = "hydra"; + + PrivateNetwork = false; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + + ReadWritePaths = [ + "/nix/var/nix/gcroots/" + "/nix/var/nix/daemon-socket/socket" + ]; + ReadOnlyPaths = [ "/nix/" ]; + RuntimeDirectory = "queue-builder"; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + RemoveIPC = true; + UMask = "0077"; + + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + ProtectKernelLogs = true; + ProtectClock = true; + + RestrictAddressFamilies = ""; + + LockPersonality = true; + ProtectHostname = true; + RestrictRealtime = true; + MemoryDenyWriteExecute = true; + PrivateUsers = true; + RestrictNamespaces = true; + }; + }; + + systemd.tmpfiles.rules = [ + "d /nix/var/nix/gcroots/per-user/hydra-queue-builder 0755 hydra-queue-builder hydra -" + ]; + nix = { + settings = { + allowed-users = [ "hydra-queue-builder" ]; + trusted-users = [ "hydra-queue-builder" ]; + experimental-features = [ "nix-command" ]; + }; + }; + + users = { + groups.hydra = { }; + users.hydra-queue-builder = { + group = "hydra"; + isSystemUser = true; + }; + }; + + }; +} diff --git a/nixos-modules/queue-runner-module.nix b/nixos-modules/queue-runner-module.nix new file mode 100644 index 000000000..1464c4a80 --- /dev/null +++ b/nixos-modules/queue-runner-module.nix @@ -0,0 +1,336 @@ +{ + config, + pkgs, + lib, + ... +}: +let + cfg = config.services.queue-runner-dev; + + format = pkgs.formats.toml { }; +in +{ + options = { + services.queue-runner-dev = { + enable = lib.mkEnableOption "QueueRunner"; + + settings = lib.mkOption { + description = "Reloadable settings for queue runner"; + type = lib.types.submodule { + options = { + hydraDataDir = lib.mkOption { + description = "Hydra data directory"; + type = lib.types.path; + default = "/var/lib/hydra"; + }; + dbUrl = lib.mkOption { + description = "Postgresql database url"; + type = lib.types.singleLineStr; + default = "postgres://hydra@%2Frun%2Fpostgresql:5432/hydra"; + }; + maxDbConnections = lib.mkOption { + description = "Postgresql maximum db connections"; + type = lib.types.ints.positive; + default = 128; + }; + machineSortFn = lib.mkOption { + description = "Function name for sorting machines"; + type = lib.types.enum [ + "SpeedFactorOnly" + "CpuCoreCountWithSpeedFactor" + "BogomipsWithSpeedFactor" + ]; + default = "SpeedFactorOnly"; + }; + machineFreeFn = lib.mkOption { + description = "Function name for determining \"idle\" machines"; + type = lib.types.enum [ + "Dynamic" + "DynamicWithMaxJobLimit" + "Static" + ]; + default = "Static"; + }; + stepSortFn = lib.mkOption { + description = "Function name for sorting steps/jobs"; + type = lib.types.enum [ + "Legacy" + "WithRdeps" + ]; + default = "WithRdeps"; + }; + dispatchTriggerTimerInS = lib.mkOption { + description = "Timer for triggering dispatch in an interval in seconds. Setting this to a value <= 0 will disable this timer and only trigger the dispatcher if queue changes happend."; + type = lib.types.int; + default = 120; + }; + queueTriggerTimerInS = lib.mkOption { + description = "Timer for triggering queue in an interval in seconds. Setting this to a value <= 0 will disable this timer and only trigger via pg notifications."; + type = lib.types.int; + default = -1; + }; + remoteStoreAddr = lib.mkOption { + description = "Remote store address"; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + useSubstitutes = lib.mkOption { + description = "Use substitution for paths"; + type = lib.types.bool; + default = false; + }; + rootsDir = lib.mkOption { + description = "Gcroots directory, defaults to /nix/var/nix/gcroots/per-user/$LOGNAME/hydra-roots"; + type = lib.types.nullOr lib.types.path; + default = null; + }; + maxRetries = lib.mkOption { + description = "Number of maximum amount of retries for a build step."; + type = lib.types.ints.positive; + default = 5; + }; + retryInterval = lib.mkOption { + description = "Interval in which retires should be able to be attempted again."; + type = lib.types.ints.positive; + default = 60; + }; + retryBackoff = lib.mkOption { + description = "Additional backoff on top of the retry interval."; + type = lib.types.float; + default = 3.0; + }; + maxUnsupportedTimeInS = lib.mkOption { + description = "Time until unsupported steps are aborted."; + type = lib.types.ints.unsigned; + default = 120; + }; + stopQueueRunAfterInS = lib.mkOption { + description = "Seconds after which the queue run should be interupted early. Setting this to a value <= 0 will disable this feature and the queue run will never exit early."; + type = lib.types.int; + default = 60; + }; + maxConcurrentDownloads = lib.mkOption { + description = "Max count of concurrent downloads per build. Increasing this will increase memory usage of the queue runner."; + type = lib.types.ints.positive; + default = 5; + }; + concurrentUploadLimit = lib.mkOption { + description = "Concurrent limit for uploading to s3."; + type = lib.types.ints.positive; + default = 5; + }; + tokenListPath = lib.mkOption { + description = "Path to a list of allowed authentication tokens."; + type = lib.types.nullOr lib.types.path; + default = null; + }; + enableFodChecker = lib.mkOption { + description = "This will enable the FOD checker. It will collect FOD in a separate queue and scheudle these builds to a separate machine with the mandatory feature FOD."; + type = lib.types.bool; + default = false; + }; + usePresignedUploads = lib.mkOption { + description = '' + If enabled the queue runner will no longer upload to s3 but rather the builder will do the uploads. + This also requires a s3 remote store, as well as substitution on the builders. + You can use forcedSubstituters setting to specify the required substituter on the builders. + ''; + type = lib.types.bool; + default = false; + }; + forcedSubstituters = lib.mkOption { + description = "Force a list of substituters per builder. Builder will no longer be accepted if they don't have `useSubstitutes` with the substituters listed here."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + }; + }; + default = { }; + }; + + grpc = lib.mkOption { + description = "grpc options"; + default = { }; + type = lib.types.submodule { + options = { + address = lib.mkOption { + type = lib.types.singleLineStr; + default = "[::1]"; + description = "The IP address the grpc listener should bound to"; + }; + + port = lib.mkOption { + description = "Which grpc port this app should listen on"; + type = lib.types.port; + default = 50051; + }; + }; + }; + }; + + rest = lib.mkOption { + description = "rest options"; + default = { }; + type = lib.types.submodule { + options = { + address = lib.mkOption { + type = lib.types.singleLineStr; + default = "[::1]"; + description = "The IP address the rest listener should bound to"; + }; + + port = lib.mkOption { + description = "Which rest port this app should listen on"; + type = lib.types.port; + default = 8080; + }; + }; + }; + }; + + mtls = lib.mkOption { + description = "mtls options"; + default = null; + type = lib.types.nullOr ( + lib.types.submodule { + options = { + serverCertPath = lib.mkOption { + description = "Server certificate path"; + type = lib.types.path; + }; + serverKeyPath = lib.mkOption { + description = "Server key path"; + type = lib.types.path; + }; + clientCaCertPath = lib.mkOption { + description = "Client ca certificate path"; + type = lib.types.path; + }; + }; + } + ); + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.callPackage ./. { }; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.queue-runner-dev = { + description = "queue-runner main service"; + + requires = [ "nix-daemon.socket" ]; + after = [ + "network.target" + "postgresql.service" + ]; + wantedBy = [ "multi-user.target" ]; + reloadTriggers = [ config.environment.etc."hydra/queue-runner.toml".source ]; + + environment = { + NIX_REMOTE = "daemon"; + LIBEV_FLAGS = "4"; # go ahead and mandate epoll(2) + RUST_BACKTRACE = "1"; + + # Note: it's important to set this for nix-store, because it wants to use + # $HOME in order to use a temporary cache dir. bizarre failures will occur + # otherwise + HOME = "/run/queue-runner"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + + ExecStart = lib.escapeShellArgs ( + [ + "${cfg.package}/bin/queue-runner" + "--rest-bind" + "${cfg.rest.address}:${toString cfg.rest.port}" + "--grpc-bind" + "${cfg.grpc.address}:${toString cfg.grpc.port}" + "--config-path" + "/etc/hydra/queue-runner.toml" + ] + ++ lib.optionals (cfg.mtls != null) [ + "--server-cert-path" + cfg.mtls.serverCertPath + "--server-key-path" + cfg.mtls.serverKeyPath + "--client-ca-cert-path" + cfg.mtls.clientCaCertPath + ] + ); + ExecReload = "${pkgs.util-linux}/bin/kill -HUP $MAINPID"; + + User = "hydra-queue-runner"; + Group = "hydra"; + + PrivateNetwork = false; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + StateDirectory = [ "hydra/queue-runner" ]; + StateDirectoryMode = "0700"; + ReadWritePaths = [ + "/nix/var/nix/gcroots/" + "/run/postgresql/.s.PGSQL.${toString config.services.postgresql.settings.port}" + "/nix/var/nix/daemon-socket/socket" + "/var/lib/hydra/build-logs/" + ]; + ReadOnlyPaths = [ "/nix/" ]; + RuntimeDirectory = "queue-runner"; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + RemoveIPC = true; + UMask = "0022"; + + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + ProtectKernelLogs = true; + ProtectClock = true; + + RestrictAddressFamilies = ""; + + LockPersonality = true; + ProtectHostname = true; + RestrictRealtime = true; + MemoryDenyWriteExecute = true; + PrivateUsers = true; + RestrictNamespaces = true; + }; + }; + + environment.etc."hydra/queue-runner.toml".source = format.generate "queue-runner.toml" ( + lib.filterAttrsRecursive (_: v: v != null) cfg.settings + ); + systemd.tmpfiles.rules = [ + "d /nix/var/nix/gcroots/per-user/hydra-queue-runner 0755 hydra-queue-runner hydra -" + "d /var/lib/hydra/build-logs/ 0755 hydra-queue-runner hydra -" + ]; + + users = { + groups.hydra = { }; + users.hydra-queue-runner = { + group = "hydra"; + isSystemUser = true; + }; + }; + }; +} diff --git a/package.nix b/package.nix index 3d25ab2d9..b50052858 100644 --- a/package.nix +++ b/package.nix @@ -54,6 +54,11 @@ , rpm , dpkg , cdrkit + +, rustPackages +, libsodium +, zlib +, protobuf }: let @@ -135,27 +140,39 @@ let version = "${builtins.readFile ./version.txt}.${builtins.substring 0 8 (rawSrc.lastModifiedDate or "19700101")}.${rawSrc.shortRev or "DIRTY"}"; in -stdenv.mkDerivation (finalAttrs: { +stdenv.mkDerivation { pname = "hydra"; inherit version; src = fileset.toSource { root = ./.; - fileset = fileset.unions ([ + fileset = fileset.unions [ + ./.cargo + ./.perlcriticrc + ./.sqlx + ./Cargo.lock + ./Cargo.toml ./doc ./meson.build + ./meson_options.txt ./nixos-modules ./src ./t ./version.txt - ./.perlcriticrc - ]); + ]; }; outputs = [ "out" "doc" ]; strictDeps = true; + cargoDeps = rustPackages.rustPlatform.importCargoLock { + lockFile = ./Cargo.lock; + outputHashes = { + "nix-diff-0.1.0" = "sha256-heUqcAnGmMogyVXskXc4FMORb8ZaK6vUX+mMOpbfSUw="; + }; + }; + nativeBuildInputs = [ makeWrapper meson @@ -167,6 +184,12 @@ stdenv.mkDerivation (finalAttrs: { perlDeps perl unzip + protobuf + rustPackages.cargo + rustPackages.clippy + rustPackages.rustPlatform.cargoSetupHook + rustPackages.rustc + rustPackages.rustfmt ]; buildInputs = [ @@ -181,6 +204,9 @@ stdenv.mkDerivation (finalAttrs: { boost nlohmann_json prometheus-cpp + libsodium + zlib + protobuf ]; nativeCheckInputs = [ @@ -229,6 +255,7 @@ stdenv.mkDerivation (finalAttrs: { ); OPENLDAP_ROOT = openldap; + NIX_CFLAGS_COMPILE = "-Wno-error"; mesonBuildType = "release"; @@ -239,7 +266,7 @@ stdenv.mkDerivation (finalAttrs: { shellHook = '' pushd $(git rev-parse --show-toplevel) >/dev/null - PATH=$(pwd)/build/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/build/src/hydra-queue-runner:$PATH + PATH=$(pwd)/build/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/build/src:$PATH PERL5LIB=$(pwd)/src/lib:$PERL5LIB export HYDRA_HOME="$(pwd)/src/" mkdir -p .hydra-data @@ -282,4 +309,4 @@ stdenv.mkDerivation (finalAttrs: { inherit perlDeps; nix = nixComponents.nix-cli; }; -}) +} diff --git a/src/builder/Cargo.toml b/src/builder/Cargo.toml new file mode 100644 index 000000000..4c325a348 --- /dev/null +++ b/src/builder/Cargo.toml @@ -0,0 +1,52 @@ +[package] +name = "builder" +version.workspace = true +edition = "2024" +license = "GPL-3.0" +rust-version.workspace = true + +[dependencies] +tracing.workspace = true +sd-notify.workspace = true + +anyhow.workspace = true +thiserror.workspace = true +clap = { workspace = true, features = ["derive"] } +uuid = { workspace = true, features = ["v4"] } +hashbrown.workspace = true +parking_lot.workspace = true +fs-err = { workspace = true, features = ["tokio"] } + +tokio = { workspace = true, features = ["full"] } +tokio-stream.workspace = true +backon.workspace = true +futures.workspace = true +prost.workspace = true +tonic = { workspace = true, features = ["zstd", "tls-webpki-roots"] } +tonic-prost.workspace = true +tower.workspace = true +hyper-util.workspace = true +async-stream.workspace = true + +gethostname.workspace = true +procfs-core.workspace = true +nix = { workspace = true, features = ["fs"] } + +nix-utils = { path = "../crates/nix-utils" } +shared = { path = "../crates/shared" } +hydra-tracing = { path = "../crates/tracing", features = ["tonic"] } +binary-cache = { path = "../crates/binary-cache" } + +[target.'cfg(target_os = "macos")'.dependencies] +sysinfo.workspace = true + +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemallocator.workspace = true + +[build-dependencies] +tonic-prost-build.workspace = true +sha2.workspace = true +fs-err = { workspace = true } + +[features] +otel = ["hydra-tracing/otel"] diff --git a/src/builder/build.rs b/src/builder/build.rs new file mode 100644 index 000000000..3f3f8fff2 --- /dev/null +++ b/src/builder/build.rs @@ -0,0 +1,30 @@ +use sha2::Digest; +use std::{env, path::PathBuf}; + +fn main() -> Result<(), Box> { + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + + let workspace_version = env::var("CARGO_PKG_VERSION")?; + + let proto_path = "../proto/v1/streaming.proto"; + let proto_content = fs_err::read_to_string(proto_path)?; + let mut hasher = sha2::Sha256::new(); + hasher.update(proto_content.as_bytes()); + let proto_hash = format!("{:x}", hasher.finalize()); + let version = format!("{}-{}", workspace_version, &proto_hash[..8]); + + fs_err::write( + out_dir.join("proto_version.rs"), + format!( + r#"// Generated during build - do not edit +pub const PROTO_API_VERSION: &str = "{version}"; +"# + ), + )?; + + tonic_prost_build::configure() + .build_server(false) + .file_descriptor_set_path(out_dir.join("streaming_descriptor.bin")) + .compile_protos(&["../proto/v1/streaming.proto"], &["../proto"])?; + Ok(()) +} diff --git a/src/builder/src/config.rs b/src/builder/src/config.rs new file mode 100644 index 000000000..04792270e --- /dev/null +++ b/src/builder/src/config.rs @@ -0,0 +1,165 @@ +use clap::Parser; + +#[derive(Parser, Debug)] +#[clap( + author, + version, + about, + long_about = None, +)] +pub struct Cli { + /// Gateway endpoint + #[clap(short, long, default_value = "http://[::1]:50051")] + pub gateway_endpoint: String, + + /// Ping interval in seconds + #[clap(short, long, default_value_t = 10)] + pub ping_interval: u64, + + /// Speed factor that is used when joining the queue-runner + #[clap(short, long, default_value_t = 1.0)] + pub speed_factor: f32, + + /// Maximum number of allowed jobs + #[clap(long, default_value_t = 4)] + pub max_jobs: u32, + + /// build dir available storage percentage Threshold + #[clap(long, default_value_t = 10.)] + pub build_dir_avail_threshold: f32, + + /// prefix/store available storage percentage Threshold + #[clap(long, default_value_t = 10.)] + pub store_avail_threshold: f32, + + /// Load1 Threshold + #[clap(long, default_value_t = 8.)] + pub load1_threshold: f32, + + /// CPU Pressure Threshold + #[clap(long, default_value_t = 75.)] + pub cpu_psi_threshold: f32, + + /// Memory Pressure Threshold + #[clap(long, default_value_t = 80.)] + pub mem_psi_threshold: f32, + + /// IO Pressure Threshold, null disables this pressure check + #[clap(long)] + pub io_psi_threshold: Option, + + /// Path to Server root ca cert + #[clap(long)] + pub server_root_ca_cert_path: Option, + + /// Path to Client cert + #[clap(long)] + pub client_cert_path: Option, + + /// Path to Client key + #[clap(long)] + pub client_key_path: Option, + + /// Domain name for mtls + #[clap(long)] + pub domain_name: Option, + + /// List of supported systems, defaults to systems from nix and extra-platforms + #[clap(long, default_value = None)] + pub systems: Option>, + + /// List of supported features, defaults to configured system features + #[clap(long, default_value = None)] + pub supported_features: Option>, + + /// List of mandatory features + #[clap(long, default_value = None)] + pub mandatory_features: Option>, + + /// Use substitution over pulling inputs via queue runner + #[clap(long, default_value_t = false)] + pub use_substitutes: bool, + + /// File to Authorization token, can be use as an alternative to mTLS + #[clap(long)] + pub authorization_file: Option, +} + +impl Default for Cli { + fn default() -> Self { + Self::new() + } +} + +impl Cli { + #[must_use] + pub fn new() -> Self { + Self::parse() + } + + #[must_use] + pub const fn mtls_enabled(&self) -> bool { + self.server_root_ca_cert_path.is_some() + && self.client_cert_path.is_some() + && self.client_key_path.is_some() + && self.domain_name.is_some() + } + + #[must_use] + pub const fn mtls_configured_correctly(&self) -> bool { + self.mtls_enabled() + || (self.server_root_ca_cert_path.is_none() + && self.client_cert_path.is_none() + && self.client_key_path.is_none() + && self.domain_name.is_none()) + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_mtls( + &self, + ) -> anyhow::Result<( + tonic::transport::Certificate, + tonic::transport::Identity, + String, + )> { + let server_root_ca_cert_path = self + .server_root_ca_cert_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("server_root_ca_cert_path not provided"))?; + let client_cert_path = self + .client_cert_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("client_cert_path not provided"))?; + let client_key_path = self + .client_key_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("client_key_path not provided"))?; + let domain_name = self + .domain_name + .as_deref() + .ok_or_else(|| anyhow::anyhow!("domain_name not provided"))?; + + let server_root_ca_cert = fs_err::tokio::read_to_string(server_root_ca_cert_path).await?; + let server_root_ca_cert = tonic::transport::Certificate::from_pem(server_root_ca_cert); + + let client_cert = fs_err::tokio::read_to_string(client_cert_path).await?; + let client_key = fs_err::tokio::read_to_string(client_key_path).await?; + let client_identity = tonic::transport::Identity::from_pem(client_cert, client_key); + + Ok((server_root_ca_cert, client_identity, domain_name.to_owned())) + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_authorization_token(&self) -> anyhow::Result> { + if let Some(path) = &self.authorization_file { + Ok(Some( + fs_err::tokio::read_to_string(path) + .await? + .trim() + .to_string(), + )) + } else { + Ok(None) + } + } +} diff --git a/src/builder/src/grpc.rs b/src/builder/src/grpc.rs new file mode 100644 index 000000000..2fe9d02df --- /dev/null +++ b/src/builder/src/grpc.rs @@ -0,0 +1,268 @@ +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use anyhow::Context as _; +use tonic::{Request, service::interceptor::InterceptedService, transport::Channel}; + +use runner_v1::{ + BuilderRequest, VersionCheckRequest, builder_request, runner_request, + runner_service_client::RunnerServiceClient, +}; + +pub mod runner_v1 { + // We need to allow pedantic here because of generated code + #![allow(clippy::pedantic)] + + tonic::include_proto!("runner.v1"); +} + +#[derive(Debug, Clone)] +pub enum BuilderInterceptor { + Token { + token: tonic::metadata::MetadataValue, + }, + Noop, +} + +impl tonic::service::Interceptor for BuilderInterceptor { + fn call(&mut self, request: tonic::Request<()>) -> Result, tonic::Status> { + let mut request = hydra_tracing::propagate::send_trace(request).map_err(|e| *e)?; + + if let Self::Token { token } = self { + request + .metadata_mut() + .insert("authorization", token.clone()); + } + + Ok(request) + } +} + +pub type BuilderClient = RunnerServiceClient>; + +impl BuilderClient { + #[tracing::instrument(skip(self, store_paths), err)] + pub async fn request_presigned_urls( + &mut self, + build_id: &str, + machine_id: &str, + store_paths: Vec<(nix_utils::StorePath, String, Vec)>, + ) -> anyhow::Result> { + use runner_v1::{PresignedNarRequest, PresignedUrlRequest}; + + let request = store_paths + .into_iter() + .map(|(path, nar_hash, build_ids)| PresignedNarRequest { + store_path: path.base_name().to_owned(), + nar_hash, + debug_info_build_ids: build_ids, + }) + .collect::>(); + + let response = self + .request_presigned_url(PresignedUrlRequest { + build_id: build_id.to_owned(), + machine_id: machine_id.to_owned(), + request, + }) + .await + .context("Failed to request presigned URLs")?; + + Ok(response.into_inner().inner) + } +} + +#[tracing::instrument(err)] +pub async fn init_client(cli: &crate::config::Cli) -> anyhow::Result { + if !cli.mtls_configured_correctly() { + tracing::error!( + "mtls configured inproperly, please pass all options: \ + server_root_ca_cert_path, client_cert_path, client_key_path and domain_name!" + ); + return Err(anyhow::anyhow!("Configuration issue")); + } + + tracing::info!("connecting to {}", cli.gateway_endpoint); + let channel = if cli.mtls_enabled() { + tracing::info!("mtls is enabled"); + let (server_root_ca_cert, client_identity, domain_name) = cli + .get_mtls() + .await + .context("Failed to get_mtls Certificate and Identity")?; + let tls = tonic::transport::ClientTlsConfig::new() + .domain_name(domain_name) + .ca_certificate(server_root_ca_cert) + .identity(client_identity); + + tonic::transport::Channel::builder(cli.gateway_endpoint.parse()?) + .tls_config(tls) + .context("Failed to attach tls config")? + .connect() + .await + .context("Failed to establish connection with Channel")? + } else if let Some(path) = cli.gateway_endpoint.strip_prefix("unix://") { + let path = path.to_owned(); + tonic::transport::Endpoint::try_from("http://[::]:50051")? + .connect_with_connector(tower::service_fn(move |_: tonic::transport::Uri| { + let path = path.clone(); + async move { + Ok::<_, std::io::Error>(hyper_util::rt::TokioIo::new( + tokio::net::UnixStream::connect(&path).await?, + )) + } + })) + .await + .context("Failed to establish unix socket connection with Channel")? + } else { + tonic::transport::Channel::builder(cli.gateway_endpoint.parse()?) + .connect() + .await + .context("Failed to establish connection with Channel")? + }; + + let interceptor = if let Some(t) = cli.get_authorization_token().await? { + BuilderInterceptor::Token { + token: format!("Bearer {t}").parse()?, + } + } else { + BuilderInterceptor::Noop + }; + + Ok(RunnerServiceClient::with_interceptor(channel, interceptor) + .send_compressed(tonic::codec::CompressionEncoding::Zstd) + .accept_compressed(tonic::codec::CompressionEncoding::Zstd) + .max_decoding_message_size(50 * 1024 * 1024) + .max_encoding_message_size(50 * 1024 * 1024)) +} + +#[tracing::instrument(skip(state), err)] +async fn handle_request( + state: Arc, + request: runner_request::Message, +) -> anyhow::Result<()> { + match request { + runner_request::Message::Join(m) => { + state + .max_concurrent_downloads + .store(m.max_concurrent_downloads, Ordering::Relaxed); + } + runner_request::Message::ConfigUpdate(m) => { + state + .max_concurrent_downloads + .store(m.max_concurrent_downloads, Ordering::Relaxed); + } + runner_request::Message::Ping(_) => (), + runner_request::Message::Build(m) => { + state.schedule_build(m)?; + } + runner_request::Message::Abort(m) => { + state.abort_build(&m)?; + } + } + Ok(()) +} + +#[tracing::instrument(skip(state), err)] +async fn check_version_compatibility(state: Arc) -> anyhow::Result<()> { + let mut client = state.client.clone(); + + let response = client + .check_version(Request::new(VersionCheckRequest { + version: crate::state::PROTO_API_VERSION.to_string(), + machine_id: state.id.to_string(), + hostname: state.hostname.clone(), + })) + .await?; + let response = response.into_inner(); + + if !response.compatible { + return Err(anyhow::anyhow!( + "API version mismatch: client has {}, server has {}", + crate::state::PROTO_API_VERSION, + response.server_version, + )); + } + + tracing::info!( + "Version check passed: client={}, server={}", + crate::state::PROTO_API_VERSION, + response.server_version + ); + Ok(()) +} + +#[tracing::instrument(skip(state), err)] +pub async fn start_bidirectional_stream(state: Arc) -> anyhow::Result<()> { + use tokio_stream::StreamExt as _; + + check_version_compatibility(state.clone()).await?; + + let join_msg = state.get_join_message().await?; + let state2 = state.clone(); + let ping_stream = async_stream::stream! { + yield BuilderRequest { + message: Some(builder_request::Message::Join(join_msg)) + }; + + let mut interval = tokio::time::interval(std::time::Duration::from_secs(state.config.ping_interval)); + loop { + interval.tick().await; + + let ping = match state.get_ping_message() { + Ok(v) => builder_request::Message::Ping(v), + Err(e) => { + tracing::error!("Failed to construct ping message: {e}"); + continue + }, + }; + tracing::debug!("sending ping: {ping:?}"); + + yield BuilderRequest { + message: Some(ping) + }; + } + }; + + let response = state2 + .client + .clone() + .open_tunnel(Request::new(ping_stream)) + .await; + + let mut stream = match response { + Ok(response) => response.into_inner(), + Err(e) => { + let error_str = e.to_string(); + if error_str.contains("API version mismatch") { + return Err(anyhow::anyhow!("API version mismatch: {error_str}")); + } + return Err(e.into()); + } + }; + + let mut consecutive_failure_count = 0; + while let Some(item) = stream.next().await { + match item.map(|v| v.message) { + Ok(Some(v)) => { + consecutive_failure_count = 0; + if let Err(err) = handle_request(state2.clone(), v).await { + tracing::error!("Failed to correctly handle request: {err}"); + } + } + Ok(None) => { + consecutive_failure_count = 0; + } + Err(e) => { + consecutive_failure_count += 1; + tracing::error!("stream message delivery failed: {e}"); + if consecutive_failure_count == 10 { + return Err(anyhow::anyhow!( + "Failed to communicate {consecutive_failure_count} times over the channel. \ + Terminating the application." + )); + } + } + } + } + Ok(()) +} diff --git a/src/builder/src/lib.rs b/src/builder/src/lib.rs new file mode 100644 index 000000000..967c1e6a2 --- /dev/null +++ b/src/builder/src/lib.rs @@ -0,0 +1,12 @@ +#![deny(clippy::all)] +#![deny(clippy::pedantic)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::expect_used)] +#![allow(clippy::missing_errors_doc)] + +pub mod config; +pub mod grpc; +pub mod metrics; +pub mod state; +pub mod system; +pub mod types; diff --git a/src/builder/src/main.rs b/src/builder/src/main.rs new file mode 100644 index 000000000..e4842d208 --- /dev/null +++ b/src/builder/src/main.rs @@ -0,0 +1,85 @@ +#![deny(clippy::all)] +#![deny(clippy::pedantic)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::expect_used)] + +mod config; +mod grpc; +mod metrics; +mod state; +mod system; +mod types; + +#[cfg(not(target_env = "msvc"))] +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +async fn stop_application( + state: &std::sync::Arc, + abort_handle: &tokio::task::AbortHandle, +) { + let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping]); + tracing::info!("Enabling halt"); + state.enable_halt(); + tracing::info!("Aborting all active builds"); + state.abort_all_active_builds(); + tracing::info!("Closing connection with queue-runner"); + abort_handle.abort(); + tracing::info!("Cleaning up gcroots"); + let _ = state.clear_gcroots().await; +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let _tracing_guard = hydra_tracing::init()?; + nix_utils::init_nix(); + + let cli = config::Cli::new(); + + let state = state::State::new(&cli).await?; + let task = tokio::spawn({ + let state = state.clone(); + async move { crate::grpc::start_bidirectional_stream(state.clone()).await } + }); + + let _notify = sd_notify::notify( + false, + &[ + sd_notify::NotifyState::Status("Running"), + sd_notify::NotifyState::Ready, + ], + ); + + let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?; + let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + + let abort_handle = task.abort_handle(); + + tokio::select! { + _ = sigint.recv() => { + tracing::info!("Received sigint - shutting down gracefully"); + stop_application(&state, &abort_handle).await; + } + _ = sigterm.recv() => { + tracing::info!("Received sigterm - shutting down gracefully"); + stop_application(&state, &abort_handle).await; + } + r = task => { + let _ = state.clear_gcroots().await; + match r { + Ok(Ok(())) => (), + Ok(Err(e)) => { + let error_str = e.to_string(); + if error_str.contains("API version mismatch") { + tracing::error!("ERROR: {error_str}"); + std::process::exit(65); // EX_DATAERR + } else { + return Err(e); + } + } + Err(e) => return Err(e.into()), + } + } + }; + Ok(()) +} diff --git a/src/builder/src/metrics.rs b/src/builder/src/metrics.rs new file mode 100644 index 000000000..5a492db75 --- /dev/null +++ b/src/builder/src/metrics.rs @@ -0,0 +1,51 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Debug, Default)] +#[allow(clippy::struct_field_names)] +pub struct Metrics { + pub substituting_path_count: AtomicU64, + + pub uploading_path_count: AtomicU64, + pub downloading_path_count: AtomicU64, +} + +impl Metrics { + pub fn add_substituting_path(&self, v: u64) { + self.substituting_path_count.fetch_add(v, Ordering::Relaxed); + } + + pub fn sub_substituting_path(&self, v: u64) { + self.substituting_path_count.fetch_sub(v, Ordering::Relaxed); + } + + #[must_use] + pub fn get_substituting_path_count(&self) -> u64 { + self.substituting_path_count.load(Ordering::Relaxed) + } + + pub fn add_uploading_path(&self, v: u64) { + self.uploading_path_count.fetch_add(v, Ordering::Relaxed); + } + + pub fn sub_uploading_path(&self, v: u64) { + self.uploading_path_count.fetch_sub(v, Ordering::Relaxed); + } + + #[must_use] + pub fn get_uploading_path_count(&self) -> u64 { + self.uploading_path_count.load(Ordering::Relaxed) + } + + pub fn add_downloading_path(&self, v: u64) { + self.downloading_path_count.fetch_add(v, Ordering::Relaxed); + } + + pub fn sub_downloading_path(&self, v: u64) { + self.downloading_path_count.fetch_sub(v, Ordering::Relaxed); + } + + #[must_use] + pub fn get_downloading_path_count(&self) -> u64 { + self.downloading_path_count.load(Ordering::Relaxed) + } +} diff --git a/src/builder/src/state.rs b/src/builder/src/state.rs new file mode 100644 index 000000000..4c885fdf4 --- /dev/null +++ b/src/builder/src/state.rs @@ -0,0 +1,1131 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::time::Instant; + +use anyhow::Context as _; +use backon::RetryableWithContext as _; +use futures::TryFutureExt as _; +use hashbrown::HashMap; +use tonic::Request; +use tracing::Instrument as _; + +use crate::grpc::{BuilderClient, runner_v1}; +use crate::types::BuildTimings; +use binary_cache::{Compression, PresignedUpload, PresignedUploadClient}; +use nix_utils::BaseStore as _; +use runner_v1::{ + AbortMessage, BuildMessage, BuildMetric, BuildProduct, BuildResultInfo, BuildResultState, + FetchRequisitesRequest, JoinMessage, LogChunk, NarData, NixSupport, Output, OutputNameOnly, + OutputWithPath, PingMessage, PressureState, StepStatus, StepUpdate, StorePaths, output, +}; + +include!(concat!(env!("OUT_DIR"), "/proto_version.rs")); + +const RETRY_MIN_DELAY: tokio::time::Duration = tokio::time::Duration::from_secs(3); +const RETRY_MAX_DELAY: tokio::time::Duration = tokio::time::Duration::from_secs(90); + +fn retry_strategy() -> backon::ExponentialBuilder { + backon::ExponentialBuilder::default() + .with_jitter() + .with_min_delay(RETRY_MIN_DELAY) + .with_max_delay(RETRY_MAX_DELAY) +} + +#[derive(thiserror::Error, Debug)] +pub enum JobFailure { + #[error("Build failure: `{0}`")] + Build(anyhow::Error), + #[error("Preparing failure: `{0}`")] + Preparing(anyhow::Error), + #[error("Import failure: `{0}`")] + Import(anyhow::Error), + #[error("Upload failure: `{0}`")] + Upload(anyhow::Error), + #[error("Post processing failure: `{0}`")] + PostProcessing(anyhow::Error), +} + +impl From for BuildResultState { + fn from(item: JobFailure) -> Self { + match item { + JobFailure::Build(_) => Self::BuildFailure, + JobFailure::Preparing(_) => Self::PreparingFailure, + JobFailure::Import(_) => Self::ImportFailure, + JobFailure::Upload(_) => Self::UploadFailure, + JobFailure::PostProcessing(_) => Self::PostProcessingFailure, + } + } +} + +#[derive(Debug)] +pub struct BuildInfo { + drv_path: nix_utils::StorePath, + handle: tokio::task::JoinHandle<()>, + was_cancelled: Arc, +} + +impl BuildInfo { + fn abort(&self) { + self.was_cancelled.store(true, Ordering::SeqCst); + self.handle.abort(); + } +} + +#[derive(Debug)] +pub struct Config { + pub ping_interval: u64, + pub speed_factor: f32, + pub max_jobs: u32, + pub build_dir_avail_threshold: f32, + pub store_avail_threshold: f32, + pub load1_threshold: f32, + pub cpu_psi_threshold: f32, + pub mem_psi_threshold: f32, + pub io_psi_threshold: Option, + pub gcroots: std::path::PathBuf, + pub systems: Vec, + pub supported_features: Vec, + pub mandatory_features: Vec, + pub cgroups: bool, + pub use_substitutes: bool, +} + +#[derive(Debug)] +pub struct State { + pub id: uuid::Uuid, + pub hostname: String, + pub config: Config, + pub max_concurrent_downloads: AtomicU32, + + active_builds: parking_lot::RwLock>>, + pub client: BuilderClient, + pub halt: AtomicBool, + pub metrics: Arc, + upload_client: PresignedUploadClient, +} + +#[derive(Debug)] +struct Gcroot { + root: std::path::PathBuf, +} + +impl Gcroot { + pub fn new(path: std::path::PathBuf) -> std::io::Result { + fs_err::create_dir_all(&path)?; + Ok(Self { root: path }) + } +} + +impl std::fmt::Display for Gcroot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + write!(f, "{}", self.root.display()) + } +} + +impl Drop for Gcroot { + fn drop(&mut self) { + if self.root.exists() { + let _ = fs_err::remove_dir_all(&self.root); + } + } +} + +impl State { + #[tracing::instrument(err)] + pub async fn new(cli: &super::config::Cli) -> anyhow::Result> { + nix_utils::set_verbosity(1); + + let logname = std::env::var("LOGNAME").context("LOGNAME not set")?; + let nix_state_dir = + std::env::var("NIX_STATE_DIR").unwrap_or_else(|_| "/nix/var/nix/".to_owned()); + let gcroots = std::path::PathBuf::from(nix_state_dir) + .join("gcroots/per-user") + .join(logname) + .join("hydra-roots"); + fs_err::tokio::create_dir_all(&gcroots).await?; + + let state = Arc::new(Self { + id: uuid::Uuid::new_v4(), + hostname: gethostname::gethostname().into_string().map_err(|v| { + anyhow::anyhow!( + "Couldn't convert hostname to string! OsString={}", + v.display() + ) + })?, + active_builds: parking_lot::RwLock::new(HashMap::with_capacity(10)), + config: Config { + ping_interval: cli.ping_interval, + speed_factor: cli.speed_factor, + max_jobs: cli.max_jobs, + build_dir_avail_threshold: cli.build_dir_avail_threshold, + store_avail_threshold: cli.store_avail_threshold, + load1_threshold: cli.load1_threshold, + cpu_psi_threshold: cli.cpu_psi_threshold, + mem_psi_threshold: cli.mem_psi_threshold, + io_psi_threshold: cli.io_psi_threshold, + gcroots, + systems: cli.systems.as_ref().map_or_else( + || { + let mut out = Vec::with_capacity(8); + out.push(nix_utils::get_this_system()); + out.extend(nix_utils::get_extra_platforms()); + out + }, + Clone::clone, + ), + supported_features: cli + .supported_features + .as_ref() + .map_or_else(nix_utils::get_system_features, Clone::clone), + mandatory_features: cli.mandatory_features.clone().unwrap_or_default(), + cgroups: nix_utils::get_use_cgroups(), + use_substitutes: cli.use_substitutes, + }, + max_concurrent_downloads: 5.into(), + client: crate::grpc::init_client(cli).await?, + halt: false.into(), + metrics: Arc::new(crate::metrics::Metrics::default()), + upload_client: PresignedUploadClient::new(), + }); + tracing::info!("Builder systems={:?}", state.config.systems); + tracing::info!( + "Builder supported_features={:?}", + state.config.supported_features + ); + tracing::info!( + "Builder mandatory_features={:?}", + state.config.mandatory_features + ); + tracing::info!("Builder use_cgroups={:?}", state.config.cgroups); + + Ok(state) + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_join_message(&self) -> anyhow::Result { + let sys = crate::system::BaseSystemInfo::new()?; + + Ok(JoinMessage { + machine_id: self.id.to_string(), + systems: self.config.systems.clone(), + hostname: self.hostname.clone(), + cpu_count: u32::try_from(sys.cpu_count)?, + bogomips: sys.bogomips, + speed_factor: self.config.speed_factor, + max_jobs: self.config.max_jobs, + build_dir_avail_threshold: self.config.build_dir_avail_threshold, + store_avail_threshold: self.config.store_avail_threshold, + load1_threshold: self.config.load1_threshold, + cpu_psi_threshold: self.config.cpu_psi_threshold, + mem_psi_threshold: self.config.mem_psi_threshold, + io_psi_threshold: self.config.io_psi_threshold, + total_mem: sys.total_memory, + supported_features: self.config.supported_features.clone(), + mandatory_features: self.config.mandatory_features.clone(), + cgroups: self.config.cgroups, + substituters: nix_utils::get_substituters(), + use_substitutes: self.config.use_substitutes, + nix_version: nix_utils::get_nix_version(), + }) + } + + #[tracing::instrument(skip(self), err)] + pub fn get_ping_message(&self) -> anyhow::Result { + let sysinfo = crate::system::SystemLoad::new(&nix_utils::get_build_dir())?; + + Ok(PingMessage { + machine_id: self.id.to_string(), + load1: sysinfo.load_avg_1, + load5: sysinfo.load_avg_5, + load15: sysinfo.load_avg_15, + mem_usage: sysinfo.mem_usage, + pressure: sysinfo.pressure.map(|p| PressureState { + cpu_some: p.cpu_some.map(Into::into), + mem_some: p.mem_some.map(Into::into), + mem_full: p.mem_full.map(Into::into), + io_some: p.io_some.map(Into::into), + io_full: p.io_full.map(Into::into), + irq_full: p.irq_full.map(Into::into), + }), + build_dir_free_percent: sysinfo.build_dir_free_percent, + store_free_percent: sysinfo.store_free_percent, + current_substituting_path_count: self.metrics.get_substituting_path_count(), + current_uploading_path_count: self.metrics.get_uploading_path_count(), + current_downloading_path_count: self.metrics.get_downloading_path_count(), + }) + } + + #[tracing::instrument(skip(self, m), fields(drv=%m.drv))] + pub fn schedule_build(self: Arc, m: BuildMessage) -> anyhow::Result<()> { + if self.halt.load(Ordering::SeqCst) { + tracing::warn!("State is set to halt, will no longer accept new builds!"); + return Err(anyhow::anyhow!("State set to halt.")); + } + + let drv = nix_utils::StorePath::new(&m.drv); + if self.contains_build(&drv) { + return Ok(()); + } + tracing::info!("Building {drv}"); + let build_id = uuid::Uuid::parse_str(&m.build_id)?; + + let was_cancelled = Arc::new(AtomicBool::new(false)); + let task_handle = tokio::spawn({ + let self_ = self.clone(); + let drv = drv.clone(); + let was_cancelled = was_cancelled.clone(); + async move { + let mut timings = BuildTimings::default(); + match Box::pin(self_.process_build(m, &mut timings)).await { + Ok(()) => { + tracing::info!("Successfully completed build process for {drv}"); + self_.remove_build(build_id); + } + Err(e) => { + if was_cancelled.load(Ordering::SeqCst) { + tracing::error!( + "Build of {drv} was cancelled {e}, not reporting Error" + ); + return; + } + + tracing::error!("Build of {drv} failed with {e}"); + self_.remove_build(build_id); + let failed_build = BuildResultInfo { + build_id: build_id.to_string(), + machine_id: self_.id.to_string(), + import_time_ms: u64::try_from(timings.import_elapsed.as_millis()) + .unwrap_or_default(), + build_time_ms: u64::try_from(timings.build_elapsed.as_millis()) + .unwrap_or_default(), + upload_time_ms: u64::try_from(timings.upload_elapsed.as_millis()) + .unwrap_or_default(), + result_state: BuildResultState::from(e) as i32, + nix_support: None, + outputs: vec![], + }; + + if let (_, Err(e)) = (|tuple: (BuilderClient, BuildResultInfo)| async { + let (mut client, body) = tuple; + let res = client.complete_build(body.clone()).await; + ((client, body), res) + }) + .retry(retry_strategy()) + .sleep(tokio::time::sleep) + .context((self_.client.clone(), failed_build)) + .notify(|err: &tonic::Status, dur: core::time::Duration| { + tracing::error!("Failed to submit build failure info: err={err}, retrying in={dur:?}"); + }) + .await + { + tracing::error!("Failed to submit build failure info: {e}"); + } + } + } + } + }); + + self.insert_new_build( + build_id, + BuildInfo { + drv_path: drv, + handle: task_handle, + was_cancelled, + }, + ); + Ok(()) + } + + fn contains_build(&self, drv: &nix_utils::StorePath) -> bool { + let active = self.active_builds.read(); + active.values().any(|b| b.drv_path == *drv) + } + + fn insert_new_build(&self, build_id: uuid::Uuid, b: BuildInfo) { + { + let mut active = self.active_builds.write(); + active.insert(build_id, Arc::new(b)); + } + self.publish_builds_to_sd_notify(); + } + + fn remove_build(&self, build_id: uuid::Uuid) -> Option> { + let b = { + let mut active = self.active_builds.write(); + active.remove(&build_id) + }; + self.publish_builds_to_sd_notify(); + b + } + + #[tracing::instrument(skip(self, m), fields(build_id=%m.build_id))] + pub fn abort_build(&self, m: &AbortMessage) -> anyhow::Result<()> { + tracing::info!("Try cancelling build"); + let build_id = uuid::Uuid::parse_str(&m.build_id)?; + if let Some(b) = self.remove_build(build_id) { + b.abort(); + } + Ok(()) + } + + pub fn abort_all_active_builds(&self) { + let mut active = self.active_builds.write(); + for b in active.values() { + b.abort(); + } + active.clear(); + } + + #[tracing::instrument(skip(self, m), fields(drv=%m.drv), err)] + #[allow(clippy::too_many_lines)] + async fn process_build( + &self, + m: BuildMessage, + timings: &mut BuildTimings, + ) -> Result<(), JobFailure> { + // we dont use anyhow here because we manually need to write the correct build status + // to the queue runner. + use tokio_stream::StreamExt as _; + + let store = nix_utils::LocalStore::init(); + + let machine_id = self.id; + let drv = nix_utils::StorePath::new(&m.drv); + let resolved_drv = m + .resolved_drv + .as_ref() + .map(|v| nix_utils::StorePath::new(v)); + + let before_import = Instant::now(); + let gcroot_prefix = uuid::Uuid::new_v4().to_string(); + let gcroot = self + .get_gcroot(&gcroot_prefix) + .map_err(|e| JobFailure::Preparing(e.into()))?; + + let mut client = self.client.clone(); + let _ = client // we ignore the error here, as this step status has no prio + .build_step_update(StepUpdate { + build_id: m.build_id.clone(), + machine_id: machine_id.to_string(), + step_status: StepStatus::SeningInputs as i32, + }) + .await; + let requisites = client + .fetch_drv_requisites(FetchRequisitesRequest { + path: resolved_drv.as_ref().unwrap_or(&drv).base_name().to_owned(), + include_outputs: false, + }) + .await + .map_err(|e| JobFailure::Import(e.into()))? + .into_inner() + .requisites; + + import_requisites( + &mut client, + store.clone(), + self.metrics.clone(), + &gcroot, + resolved_drv.as_ref().unwrap_or(&drv), + requisites + .into_iter() + .map(|s| nix_utils::StorePath::new(&s)), + usize::try_from(self.max_concurrent_downloads.load(Ordering::Relaxed)).unwrap_or(5), + self.config.use_substitutes, + ) + .await + .map_err(JobFailure::Import)?; + timings.import_elapsed = before_import.elapsed(); + + // Resolved drv and drv output paths are the same + let drv_info = nix_utils::query_drv(&store, &drv) + .await + .map_err(|e| JobFailure::Import(e.into()))? + .ok_or(JobFailure::Import(anyhow::anyhow!("drv not found")))?; + + let _ = client // we ignore the error here, as this step status has no prio + .build_step_update(StepUpdate { + build_id: m.build_id.clone(), + machine_id: machine_id.to_string(), + step_status: StepStatus::Building as i32, + }) + .await; + let before_build = Instant::now(); + let (mut child, mut log_output) = nix_utils::realise_drv( + &store, + resolved_drv.as_ref().unwrap_or(&drv), + &nix_utils::BuildOptions::complete(m.max_log_size, m.max_silent_time, m.build_timeout), + true, + ) + .await + .map_err(|e| JobFailure::Build(e.into()))?; + let drv2 = drv.clone(); + let log_stream = async_stream::stream! { + while let Some(chunk) = log_output.next().await { + match chunk { + Ok(chunk) => yield LogChunk { + drv: drv2.base_name().to_owned(), + data: format!("{chunk}\n").into(), + }, + Err(e) => { + tracing::error!("Failed to write log chunk to queue-runner: {e}"); + break + } + } + } + }; + client + .build_log(Request::new(log_stream)) + .await + .map_err(|e| JobFailure::Build(e.into()))?; + let output_paths = drv_info + .outputs + .iter() + .filter_map(|o| o.path.clone()) + .collect::>(); + nix_utils::validate_statuscode( + child + .wait() + .await + .map_err(|e| JobFailure::Build(e.into()))?, + ) + .map_err(|e| JobFailure::Build(e.into()))?; + for o in &output_paths { + nix_utils::add_root(&store, &gcroot.root, o); + } + + timings.build_elapsed = before_build.elapsed(); + tracing::info!("Finished building {drv}"); + + let _ = client // we ignore the error here, as this step status has no prio + .build_step_update(StepUpdate { + build_id: m.build_id.clone(), + machine_id: machine_id.to_string(), + step_status: StepStatus::ReceivingOutputs as i32, + }) + .await; + + let before_upload = Instant::now(); + self.upload_nars( + store.clone(), + output_paths, + &m.build_id, + &machine_id.to_string(), + m.presigned_url_opts, + ) + .await + .map_err(JobFailure::Upload)?; + timings.upload_elapsed = before_upload.elapsed(); + + let _ = client // we ignore the error here, as this step status has no prio + .build_step_update(StepUpdate { + build_id: m.build_id.clone(), + machine_id: machine_id.to_string(), + step_status: StepStatus::PostProcessing as i32, + }) + .await; + let build_results = Box::pin(new_success_build_result_info( + store.clone(), + machine_id, + &drv, + drv_info, + *timings, + m.build_id.clone(), + )) + .await + .map_err(JobFailure::PostProcessing)?; + + // This part is stupid, if writing doesnt work, we try to write a failure, maybe that works. + // We retry to ensure that this almost never happens. + (|tuple: (BuilderClient, BuildResultInfo)| async { + let (mut client, body) = tuple; + let res = client.complete_build(body.clone()).await; + ((client, body), res) + }) + .retry(retry_strategy()) + .sleep(tokio::time::sleep) + .context((client.clone(), build_results)) + .notify(|err: &tonic::Status, dur: core::time::Duration| { + tracing::error!("Failed to submit build success info: err={err}, retrying in={dur:?}"); + }) + .await + .1 + .map_err(|e| { + tracing::error!("Failed to submit build success info. Will fail build: err={e}"); + JobFailure::PostProcessing(e.into()) + })?; + Ok(()) + } + + #[tracing::instrument(skip(self), err)] + fn get_gcroot(&self, prefix: &str) -> std::io::Result { + Gcroot::new(self.config.gcroots.join(prefix)) + } + + #[tracing::instrument(skip(self))] + fn publish_builds_to_sd_notify(&self) { + let active = { + let builds = self.active_builds.read(); + builds + .values() + .map(|b| b.drv_path.base_name().to_owned()) + .collect::>() + }; + + let _notify = sd_notify::notify( + false, + &[ + sd_notify::NotifyState::Status(&if active.is_empty() { + "Building 0 drvs".into() + } else { + format!("Building {} drvs: {}", active.len(), active.join(", ")) + }), + sd_notify::NotifyState::Ready, + ], + ); + } + + #[tracing::instrument(skip(self), err)] + pub async fn clear_gcroots(&self) -> std::io::Result<()> { + fs_err::tokio::remove_dir_all(&self.config.gcroots).await?; + fs_err::tokio::create_dir_all(&self.config.gcroots).await?; + Ok(()) + } + + pub fn enable_halt(&self) { + self.halt.store(true, Ordering::SeqCst); + } + + #[tracing::instrument(skip(self, store, nars), err)] + async fn upload_nars( + &self, + store: nix_utils::LocalStore, + nars: Vec, + build_id: &str, + machine_id: &str, + presigned_url_opts: Option, + ) -> anyhow::Result<()> { + if let Some(opts) = presigned_url_opts { + upload_nars_presigned( + self.client.clone(), + self.upload_client.clone(), + store, + &nars, + opts, + build_id, + machine_id, + ) + .await + } else { + upload_nars_regular(self.client.clone(), store, self.metrics.clone(), nars).await + } + } +} + +#[tracing::instrument(skip(store), fields(%gcroot, %path))] +async fn filter_missing( + store: &nix_utils::LocalStore, + gcroot: &Gcroot, + path: nix_utils::StorePath, +) -> Option { + if store.is_valid_path(&path).await { + nix_utils::add_root(store, &gcroot.root, &path); + None + } else { + Some(path) + } +} + +async fn substitute_paths( + store: &nix_utils::LocalStore, + paths: &[nix_utils::StorePath], +) -> anyhow::Result<()> { + for p in paths { + store.ensure_path(p).await?; + } + Ok(()) +} + +#[tracing::instrument(skip(client, store, metrics), fields(%gcroot), err)] +async fn import_paths( + mut client: BuilderClient, + store: nix_utils::LocalStore, + metrics: Arc, + gcroot: &Gcroot, + paths: Vec, + filter: bool, + use_substitutes: bool, +) -> anyhow::Result<()> { + use futures::StreamExt as _; + + let paths = if filter { + futures::StreamExt::map(tokio_stream::iter(paths), |p| { + filter_missing(&store, gcroot, p) + }) + .buffered(10) + .filter_map(|o| async { o }) + .collect::>() + .await + } else { + paths + }; + let paths = if use_substitutes { + // we can ignore the error + metrics.add_substituting_path(paths.len() as u64); + let _ = substitute_paths(&store, &paths).await; + metrics.sub_substituting_path(paths.len() as u64); + let paths = futures::StreamExt::map(tokio_stream::iter(paths), |p| { + filter_missing(&store, gcroot, p) + }) + .buffered(10) + .filter_map(|o| async { o }) + .collect::>() + .await; + if paths.is_empty() { + return Ok(()); + } + paths + } else { + paths + }; + + tracing::debug!("Start importing paths"); + let stream = client + .stream_files(StorePaths { + paths: paths.iter().map(|p| p.base_name().to_owned()).collect(), + }) + .await? + .into_inner(); + + metrics.add_downloading_path(paths.len() as u64); + let import_result = store + .import_paths( + tokio_stream::StreamExt::map(stream, |s| { + s.map(|v| v.chunk.into()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::UnexpectedEof, e)) + }), + false, + ) + .await; + metrics.sub_downloading_path(paths.len() as u64); + import_result?; + tracing::debug!("Finished importing paths"); + + for p in paths { + nix_utils::add_root(&store, &gcroot.root, &p); + } + Ok(()) +} + +#[tracing::instrument(skip(client, store, metrics, requisites), fields(%gcroot, %drv), err)] +#[allow(clippy::too_many_arguments)] +async fn import_requisites>( + client: &mut BuilderClient, + store: nix_utils::LocalStore, + metrics: Arc, + gcroot: &Gcroot, + drv: &nix_utils::StorePath, + requisites: T, + max_concurrent_downloads: usize, + use_substitutes: bool, +) -> anyhow::Result<()> { + use futures::stream::StreamExt as _; + + let requisites = futures::StreamExt::map(tokio_stream::iter(requisites), |p| { + filter_missing(&store, gcroot, p) + }) + .buffered(50) + .filter_map(|o| async { o }) + .collect::>() + .await; + + let (input_drvs, input_srcs): (Vec<_>, Vec<_>) = requisites + .into_iter() + .partition(nix_utils::StorePath::is_drv); + + for srcs in input_srcs.chunks(max_concurrent_downloads) { + import_paths( + client.clone(), + store.clone(), + metrics.clone(), + gcroot, + srcs.to_vec(), + true, + use_substitutes, + ) + .await?; + } + + for drvs in input_drvs.chunks(max_concurrent_downloads) { + import_paths( + client.clone(), + store.clone(), + metrics.clone(), + gcroot, + drvs.to_vec(), + true, + false, // never use substitute for drvs + ) + .await?; + } + + let full_requisites = client + .clone() + .fetch_drv_requisites(FetchRequisitesRequest { + path: drv.base_name().to_owned(), + include_outputs: true, + }) + .await? + .into_inner() + .requisites + .into_iter() + .map(|s| nix_utils::StorePath::new(&s)) + .collect::>(); + let full_requisites = futures::StreamExt::map(tokio_stream::iter(full_requisites), |p| { + filter_missing(&store, gcroot, p) + }) + .buffered(50) + .filter_map(|o| async { o }) + .collect::>() + .await; + + for other in full_requisites.chunks(max_concurrent_downloads) { + // we can skip filtering here as we already done that + import_paths( + client.clone(), + store.clone(), + metrics.clone(), + gcroot, + other.to_vec(), + false, + use_substitutes, + ) + .await?; + } + + Ok(()) +} + +#[tracing::instrument(skip(client, store, metrics), err)] +async fn upload_nars_regular( + mut client: BuilderClient, + store: nix_utils::LocalStore, + metrics: Arc, + nars: Vec, +) -> anyhow::Result<()> { + let nars = { + use futures::stream::StreamExt as _; + + futures::StreamExt::map(tokio_stream::iter(nars), |p| { + let mut client = client.clone(); + async move { + if client + .has_path(runner_v1::StorePath { + path: p.base_name().to_owned(), + }) + .await + .is_ok_and(|r| r.into_inner().has_path) + { + None + } else { + Some(p) + } + } + }) + .buffered(10) + .filter_map(|o| async { o }) + .collect::>() + .await + }; + if nars.is_empty() { + return Ok(()); + } + + tracing::info!("Start uploading paths to queue runner directly"); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); + let before_upload = Instant::now(); + let nars_len = nars.len() as u64; + + metrics.add_uploading_path(nars_len); + let closure = move |data: &[u8]| { + let data = Vec::from(data); + tx.send(NarData { chunk: data }).is_ok() + }; + let a = client + .build_result(tokio_stream::wrappers::UnboundedReceiverStream::new(rx)) + .map_err(Into::::into); + + let b = tokio::task::spawn_blocking(move || { + async move { + store.export_paths(&nars, closure)?; + tracing::debug!("Finished exporting paths"); + Ok::<(), anyhow::Error>(()) + } + .in_current_span() + }) + .await? + .map_err(Into::::into); + futures::future::try_join(a, b).await?; + tracing::info!( + "Finished uploading paths to queue runner directly. elapsed={:?}", + before_upload.elapsed() + ); + + metrics.sub_uploading_path(nars_len); + Ok(()) +} + +#[tracing::instrument(skip(client, store), err)] +async fn upload_nars_presigned( + mut client: BuilderClient, + upload_client: PresignedUploadClient, + store: nix_utils::LocalStore, + output_paths: &[nix_utils::StorePath], + opts: crate::grpc::runner_v1::PresignedUploadOpts, + build_id: &str, + machine_id: &str, +) -> anyhow::Result<()> { + use futures::stream::StreamExt as _; + + tracing::info!("Start uploading paths using presigned urls"); + let before_upload = Instant::now(); + + let paths_to_upload = store + .query_requisites(&output_paths.iter().collect::>(), true) + .await + .unwrap_or_default(); + let paths_to_upload_ref = paths_to_upload.iter().collect::>(); + let path_infos = Arc::new(store.query_path_infos(&paths_to_upload_ref).await); + + let mut nars = Vec::with_capacity(paths_to_upload.len()); + let mut stream = tokio_stream::iter(paths_to_upload.clone()) + .map(|path| { + let store = store.clone(); + let path_infos = path_infos.clone(); + async move { + let debug_info_ids = if opts.upload_debug_info { + binary_cache::get_debug_info_build_ids(&store, &path).await? + } else { + Vec::new() + }; + let Some(narhash) = path_infos.get(&path).map(|i| i.nar_hash.clone()) else { + return Ok(None); + }; + Ok::<_, anyhow::Error>(Some((path, narhash, debug_info_ids))) + } + }) + .buffered(10); + + while let Some(v) = tokio_stream::StreamExt::next(&mut stream).await { + if let Some(v) = v? { + nars.push(v); + } + } + + if nars.len() != paths_to_upload.len() { + return Err(anyhow::anyhow!( + "Mismatch between paths_to_upload ({}) and paths_with_narhash ({})", + paths_to_upload.len(), + nars.len(), + )); + } + + let presigned_responses = client + .request_presigned_urls(build_id, machine_id, nars) + .await?; + + if presigned_responses.len() != paths_to_upload.len() { + return Err(anyhow::anyhow!( + "Mismatch between requested NARs ({}) and presigned URLs ({})", + paths_to_upload.len(), + presigned_responses.len() + )); + } + + for presigned_response in presigned_responses { + upload_single_nar_presigned( + &store, + &nix_utils::StorePath::new(&presigned_response.store_path), + build_id, + machine_id, + &presigned_response, + &mut client, + &upload_client, + ) + .await?; + } + + tracing::info!( + "Finished uploading paths using presigned urls. elapsed={:?}", + before_upload.elapsed() + ); + Ok(()) +} + +#[tracing::instrument(skip(store, nar_path, presigned_response), err)] +async fn upload_single_nar_presigned( + store: &nix_utils::LocalStore, + nar_path: &nix_utils::StorePath, + build_id: &str, + machine_id: &str, + presigned_response: &runner_v1::PresignedNarResponse, + client: &mut BuilderClient, + upload_client: &PresignedUploadClient, +) -> anyhow::Result<()> { + let narinfo = binary_cache::path_to_narinfo(store, nar_path).await?; + let nar_upload = presigned_response + .nar_upload + .as_ref() + .ok_or_else(|| anyhow::anyhow!("nar_upload information is missing"))?; + + let presigned_request = binary_cache::PresignedUploadResponse { + nar_url: presigned_response.nar_url.clone(), + nar_upload: binary_cache::PresignedUpload::new( + nar_upload.path.clone(), + nar_upload.url.clone(), + nar_upload.compression.parse().unwrap_or(Compression::None), + nar_upload.compression_level, + ), + ls_upload: presigned_response.ls_upload.as_ref().map(|ls| { + PresignedUpload::new( + ls.path.clone(), + ls.url.clone(), + ls.compression.parse().unwrap_or(Compression::None), + ls.compression_level, + ) + }), + debug_info_upload: presigned_response + .debug_info_upload + .iter() + .map(|p| { + PresignedUpload::new( + p.path.clone(), + p.url.clone(), + p.compression.parse().unwrap_or(Compression::None), + p.compression_level, + ) + }) + .collect(), + }; + + let updated_narinfo = upload_client + .process_presigned_request(store, narinfo, presigned_request) + .await?; + + tracing::debug!( + "Successfully uploaded presigned NAR for {} to {}", + nar_path, + updated_narinfo.url + ); + + if let (Some(file_hash), Some(file_size)) = ( + updated_narinfo.file_hash.as_ref(), + updated_narinfo.file_size, + ) { + let completion_msg = runner_v1::PresignedUploadComplete { + build_id: build_id.to_owned(), + machine_id: machine_id.to_owned(), + store_path: nar_path.base_name().to_owned(), + url: updated_narinfo.url.clone(), + compression: updated_narinfo.compression.as_str().to_owned(), + file_hash: file_hash.clone(), + file_size, + nar_hash: updated_narinfo.nar_hash, + nar_size: updated_narinfo.nar_size, + references: updated_narinfo + .references + .iter() + .map(|p| p.base_name().to_owned()) + .collect(), + deriver: updated_narinfo.deriver.map(|p| p.base_name().to_owned()), + ca: updated_narinfo.ca, + }; + + client + .notify_presigned_upload_complete(completion_msg) + .await?; + } + + Ok(()) +} + +#[tracing::instrument(skip(store, drv_info), fields(%drv), ret(level = tracing::Level::DEBUG), err)] +async fn new_success_build_result_info( + store: nix_utils::LocalStore, + machine_id: uuid::Uuid, + drv: &nix_utils::StorePath, + drv_info: nix_utils::Derivation, + timings: BuildTimings, + build_id: String, +) -> anyhow::Result { + let outputs = &drv_info + .outputs + .iter() + .filter_map(|o| o.path.as_ref()) + .collect::>(); + let pathinfos = store.query_path_infos(outputs).await; + let nix_support = Box::pin(shared::parse_nix_support_from_outputs( + &store, + &drv_info.outputs, + )) + .await?; + + let mut build_outputs = vec![]; + for o in drv_info.outputs { + build_outputs.push(Output { + output: Some(match o.path { + Some(p) => { + if let Some(info) = pathinfos.get(&p) { + output::Output::Withpath(OutputWithPath { + name: o.name, + closure_size: store.compute_closure_size(&p).await, + path: p.into_base_name(), + nar_size: info.nar_size, + nar_hash: info.nar_hash.clone(), + }) + } else { + output::Output::Nameonly(OutputNameOnly { name: o.name }) + } + } + None => output::Output::Nameonly(OutputNameOnly { name: o.name }), + }), + }); + } + + Ok(BuildResultInfo { + build_id, + machine_id: machine_id.to_string(), + import_time_ms: u64::try_from(timings.import_elapsed.as_millis())?, + build_time_ms: u64::try_from(timings.build_elapsed.as_millis())?, + upload_time_ms: u64::try_from(timings.upload_elapsed.as_millis())?, + result_state: BuildResultState::Success as i32, + outputs: build_outputs, + nix_support: Some(NixSupport { + metrics: nix_support + .metrics + .into_iter() + .map(|m| BuildMetric { + path: m.path, + name: m.name, + unit: m.unit, + value: m.value, + }) + .collect(), + failed: nix_support.failed, + hydra_release_name: nix_support.hydra_release_name, + products: nix_support + .products + .into_iter() + .map(|p| BuildProduct { + path: p.path, + default_path: p.default_path, + r#type: p.r#type, + subtype: p.subtype, + name: p.name, + is_regular: p.is_regular, + sha256hash: p.sha256hash, + file_size: p.file_size, + }) + .collect(), + }), + }) +} diff --git a/src/builder/src/system.rs b/src/builder/src/system.rs new file mode 100644 index 000000000..b03b14982 --- /dev/null +++ b/src/builder/src/system.rs @@ -0,0 +1,224 @@ +use hashbrown::HashMap; +use procfs_core::FromRead as _; + +pub struct BaseSystemInfo { + pub cpu_count: usize, + pub bogomips: f32, + pub total_memory: u64, +} + +impl BaseSystemInfo { + #[cfg(target_os = "linux")] + #[tracing::instrument(err)] + pub fn new() -> anyhow::Result { + let cpuinfo = procfs_core::CpuInfo::from_file("/proc/cpuinfo")?; + let meminfo = procfs_core::Meminfo::from_file("/proc/meminfo")?; + let bogomips = cpuinfo + .fields + .get("bogomips") + .and_then(|v| v.parse::().ok()) + .unwrap_or(0.0); + + Ok(Self { + cpu_count: cpuinfo.num_cores(), + bogomips, + total_memory: meminfo.mem_total, + }) + } + + #[cfg(target_os = "macos")] + #[tracing::instrument(err)] + pub fn new() -> anyhow::Result { + let mut sys = sysinfo::System::new_all(); + sys.refresh_memory(); + sys.refresh_cpu_all(); + + Ok(Self { + cpu_count: sys.cpus().len(), + bogomips: 0.0, + total_memory: sys.total_memory(), + }) + } +} + +pub struct Pressure { + pub avg10: f32, + pub avg60: f32, + pub avg300: f32, + pub total: u64, +} + +#[cfg(target_os = "linux")] +impl Pressure { + const fn new(record: &procfs_core::PressureRecord) -> Self { + Self { + avg10: record.avg10, + avg60: record.avg60, + avg300: record.avg300, + total: record.total, + } + } +} + +impl From for crate::grpc::runner_v1::Pressure { + fn from(val: Pressure) -> Self { + Self { + avg10: val.avg10, + avg60: val.avg60, + avg300: val.avg300, + total: val.total, + } + } +} + +pub struct PressureState { + pub cpu_some: Option, + pub mem_some: Option, + pub mem_full: Option, + pub io_some: Option, + pub io_full: Option, + pub irq_full: Option, +} + +// TODO: remove once https://github.com/eminence/procfs/issues/351 is resolved +// Next 3 Functions are copied from https://github.com/eminence/procfs/blob/v0.17.0/procfs-core/src/pressure.rs#L93 +// LICENSE is Apache2.0/MIT +#[cfg(target_os = "linux")] +fn get_f32(map: &HashMap<&str, &str>, value: &str) -> procfs_core::ProcResult { + map.get(value).map_or_else( + || Err(procfs_core::ProcError::Incomplete(None)), + |v| { + v.parse::() + .map_err(|_| procfs_core::ProcError::Incomplete(None)) + }, + ) +} + +#[cfg(target_os = "linux")] +fn get_total(map: &HashMap<&str, &str>) -> procfs_core::ProcResult { + map.get("total").map_or_else( + || Err(procfs_core::ProcError::Incomplete(None)), + |v| { + v.parse::() + .map_err(|_| procfs_core::ProcError::Incomplete(None)) + }, + ) +} + +#[cfg(target_os = "linux")] +fn parse_pressure_record(line: &str) -> procfs_core::ProcResult { + let mut parsed = HashMap::with_capacity(4); + + if !line.starts_with("some") && !line.starts_with("full") { + return Err(procfs_core::ProcError::Incomplete(None)); + } + + let values = &line[5..]; + + for kv_str in values.split_whitespace() { + let kv_split = kv_str.split('='); + let vec: Vec<&str> = kv_split.collect(); + if vec.len() == 2 { + parsed.insert(vec[0], vec[1]); + } + } + + Ok(procfs_core::PressureRecord { + avg10: get_f32(&parsed, "avg10")?, + avg60: get_f32(&parsed, "avg60")?, + avg300: get_f32(&parsed, "avg300")?, + total: get_total(&parsed)?, + }) +} + +#[cfg(target_os = "linux")] +impl PressureState { + #[must_use] + pub fn new() -> Option { + if !fs_err::exists("/proc/pressure").unwrap_or_default() { + return None; + } + + let cpu_psi = procfs_core::CpuPressure::from_file("proc/pressure/cpu").ok(); + let mem_psi = procfs_core::MemoryPressure::from_file("/proc/pressure/memory").ok(); + let io_psi = procfs_core::IoPressure::from_file("/proc/pressure/io").ok(); + let irq_psi_full = fs_err::read_to_string("/proc/pressure/irq") + .ok() + .and_then(|v| parse_pressure_record(&v).ok()); + + Some(Self { + cpu_some: cpu_psi.map(|v| Pressure::new(&v.some)), + mem_some: mem_psi.as_ref().map(|v| Pressure::new(&v.some)), + mem_full: mem_psi.map(|v| Pressure::new(&v.full)), + io_some: io_psi.as_ref().map(|v| Pressure::new(&v.some)), + io_full: io_psi.map(|v| Pressure::new(&v.full)), + irq_full: irq_psi_full.map(|v| Pressure::new(&v)), + }) + } +} + +pub struct SystemLoad { + pub load_avg_1: f32, + pub load_avg_5: f32, + pub load_avg_15: f32, + + pub mem_usage: u64, + pub pressure: Option, + + pub build_dir_free_percent: f64, + pub store_free_percent: f64, +} + +#[tracing::instrument(err)] +pub fn get_mount_free_percent(dest: &str) -> anyhow::Result { + let stat = nix::sys::statvfs::statvfs(dest)?; + + let total_bytes = (stat.blocks() as u64) * stat.block_size(); + let free_bytes = (stat.blocks_available() as u64) * stat.block_size(); + #[allow(clippy::cast_precision_loss)] + Ok(free_bytes as f64 / total_bytes as f64 * 100.0) +} + +impl SystemLoad { + #[cfg(target_os = "linux")] + #[tracing::instrument(err)] + pub fn new(build_dir: &str) -> anyhow::Result { + let meminfo = procfs_core::Meminfo::from_file("/proc/meminfo")?; + let load = procfs_core::LoadAverage::from_file("/proc/loadavg")?; + + // TODO: prefix + let nix_store_dir = + std::env::var("NIX_STORE_DIR").unwrap_or_else(|_| "/nix/store".to_owned()); + + Ok(Self { + load_avg_1: load.one, + load_avg_5: load.five, + load_avg_15: load.fifteen, + mem_usage: meminfo.mem_total - meminfo.mem_available.unwrap_or(0), + pressure: PressureState::new(), + build_dir_free_percent: get_mount_free_percent(build_dir).unwrap_or(0.), + store_free_percent: get_mount_free_percent(&nix_store_dir).unwrap_or(0.), + }) + } + + #[cfg(target_os = "macos")] + #[tracing::instrument(err)] + pub fn new(build_dir: &str) -> anyhow::Result { + let mut sys = sysinfo::System::new_all(); + sys.refresh_memory(); + let load = sysinfo::System::load_average(); + + // TODO: prefix + let nix_store_dir = std::env::var("NIX_STORE_DIR").unwrap_or("/nix/store".to_owned()); + + Ok(Self { + load_avg_1: load.one as f32, + load_avg_5: load.five as f32, + load_avg_15: load.fifteen as f32, + mem_usage: sys.used_memory(), + pressure: None, + build_dir_free_percent: get_mount_free_percent(build_dir).unwrap_or(0.), + store_free_percent: get_mount_free_percent(&nix_store_dir).unwrap_or(0.), + }) + } +} diff --git a/src/builder/src/types.rs b/src/builder/src/types.rs new file mode 100644 index 000000000..1e1d4bfe5 --- /dev/null +++ b/src/builder/src/types.rs @@ -0,0 +1,7 @@ +#[allow(clippy::struct_field_names)] +#[derive(Debug, Default, Clone, Copy)] +pub struct BuildTimings { + pub import_elapsed: std::time::Duration, + pub build_elapsed: std::time::Duration, + pub upload_elapsed: std::time::Duration, +} diff --git a/src/crates/binary-cache/Cargo.toml b/src/crates/binary-cache/Cargo.toml new file mode 100644 index 000000000..20e155197 --- /dev/null +++ b/src/crates/binary-cache/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "binary-cache" +version = "0.1.0" +edition = "2024" +license = "LGPL-2.1-only" +rust-version.workspace = true + +[dependencies] +tracing.workspace = true +bytes.workspace = true +thiserror.workspace = true +url.workspace = true + +object_store = { workspace = true, features = ["aws"] } +reqwest.workspace = true +tokio = { workspace = true, features = ["full"] } +tokio-stream = { workspace = true, features = ["io-util"] } +tokio-util = { workspace = true, features = ["io", "io-util"] } +fs-err = { workspace = true, features = ["tokio"] } +futures.workspace = true +async-compression = { workspace = true, features = [ + "tokio", + "xz", + "xz-parallel", + "bzip2", + "brotli", + "zstd", +] } +backon.workspace = true +sha2.workspace = true +secrecy.workspace = true +configparser.workspace = true +smallvec.workspace = true +foldhash.workspace = true +hashbrown.workspace = true +moka.workspace = true +serde.workspace = true +serde_json.workspace = true + +nix-utils = { path = "../nix-utils" } + +[dev-dependencies] +hydra-tracing = { path = "../tracing" } +tempfile = "3.23.0" diff --git a/src/crates/binary-cache/examples/download_file.rs b/src/crates/binary-cache/examples/download_file.rs new file mode 100644 index 000000000..416be17fe --- /dev/null +++ b/src/crates/binary-cache/examples/download_file.rs @@ -0,0 +1,42 @@ +use binary_cache::S3BinaryCacheClient; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let _tracing_guard = hydra_tracing::init()?; + let client = S3BinaryCacheClient::new( + "s3://store?region=unknown&endpoint=http://localhost:9000&scheme=http&write-nar-listing=1&ls-compression=br&log-compression=br&profile=local_nix_store".parse()?, + ) + .await?; + tracing::info!("{:#?}", client.cfg); + + let has_info = client + .has_narinfo(&nix_utils::StorePath::new( + "lmn7lwydprqibdkghw7wgcn21yhllz13-glibc-2.40-66", + )) + .await?; + tracing::info!("has narinfo? {has_info}"); + + let narinfo = client + .download_narinfo(&nix_utils::StorePath::new( + "lmn7lwydprqibdkghw7wgcn21yhllz13-glibc-2.40-66", + )) + .await?; + tracing::info!("narinfo:\n{narinfo:?}"); + + let nardata = client.download_nar(&narinfo.unwrap().url).await?; + tracing::info!("nardata len: {}", nardata.unwrap().len()); + + let stats = client.s3_stats(); + tracing::info!( + "stats: put={}, put_bytes={}, put_time_ms={}, get={}, get_bytes={}, get_time_ms={}, head={}", + stats.put, + stats.put_bytes, + stats.put_time_ms, + stats.get, + stats.get_bytes, + stats.get_time_ms, + stats.head + ); + + Ok(()) +} diff --git a/src/crates/binary-cache/examples/simple_presigned.rs b/src/crates/binary-cache/examples/simple_presigned.rs new file mode 100644 index 000000000..e663a3244 --- /dev/null +++ b/src/crates/binary-cache/examples/simple_presigned.rs @@ -0,0 +1,69 @@ +use futures::stream::StreamExt as _; + +use binary_cache::{PresignedUploadClient, S3BinaryCacheClient, path_to_narinfo}; +use nix_utils::BaseStore as _; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let now = std::time::Instant::now(); + + let _tracing_guard = hydra_tracing::init()?; + let store = nix_utils::LocalStore::init(); + let client = S3BinaryCacheClient::new( + format!( + "s3://store2?region=unknown&endpoint=http://localhost:9000&scheme=http&write-nar-listing=1&write-debug-info=1&compression=zstd&ls-compression=br&log-compression=br&secret-key={}/../../example-secret-key&profile=local_nix_store", + env!("CARGO_MANIFEST_DIR") + ).parse()?, + ) + .await?; + tracing::info!("{:#?}", client.cfg); + let upload_client = PresignedUploadClient::new(); + + let paths_to_copy = store + .query_requisites( + &[&nix_utils::StorePath::new( + "/nix/store/m1r53pnnm6hnjwyjmxska24y8amvlpjp-hello-2.12.1", + )], + true, + ) + .await + .unwrap_or_default(); + + let mut stream = tokio_stream::iter(paths_to_copy) + .map(|p| { + let client = client.clone(); + let upload_client = upload_client.clone(); + let store = store.clone(); + async move { + let narinfo = path_to_narinfo(&store, &p).await?; + + let presigned_request = client + .generate_nar_upload_presigned_url( + &narinfo.store_path, + &narinfo.nar_hash, + binary_cache::get_debug_info_build_ids(&store, &p).await?, + ) + .await?; + + let narinfo = upload_client + .process_presigned_request(&store, narinfo, presigned_request) + .await?; + + client + .upload_narinfo_after_presigned_upload(&store, narinfo) + .await?; + Ok::<(), Box>(()) + } + }) + .buffered(10); + + while let Some(v) = tokio_stream::StreamExt::next(&mut stream).await { + v?; + } + + tracing::info!("Client Metrics: {:#?}", upload_client.metrics()); + tracing::info!("Main Metrics: {:#?}", client.s3_stats()); + tracing::info!("Elapsed: {:#?}", now.elapsed()); + + Ok(()) +} diff --git a/src/crates/binary-cache/examples/upload_file.rs b/src/crates/binary-cache/examples/upload_file.rs new file mode 100644 index 000000000..4508573e3 --- /dev/null +++ b/src/crates/binary-cache/examples/upload_file.rs @@ -0,0 +1,35 @@ +use binary_cache::S3BinaryCacheClient; +use nix_utils::BaseStore as _; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let now = std::time::Instant::now(); + + let _tracing_guard = hydra_tracing::init()?; + let local = nix_utils::LocalStore::init(); + let client = S3BinaryCacheClient::new( + format!( + "s3://store2?region=unknown&endpoint=http://localhost:9000&scheme=http&write-nar-listing=1&compression=zstd&ls-compression=br&log-compression=br&secret-key={}/../../example-secret-key&profile=local_nix_store", + env!("CARGO_MANIFEST_DIR") + ).parse()?, + ) + .await?; + tracing::info!("{:#?}", client.cfg); + + let paths_to_copy = local + .query_requisites( + &[&nix_utils::StorePath::new( + "m1r53pnnm6hnjwyjmxska24y8amvlpjp-hello-2.12.1", + )], + true, + ) + .await + .unwrap_or_default(); + + client.copy_paths(&local, paths_to_copy, true).await?; + + tracing::info!("stats: {:#?}", client.s3_stats()); + tracing::info!("Elapsed: {:#?}", now.elapsed()); + + Ok(()) +} diff --git a/src/crates/binary-cache/examples/upload_logs.rs b/src/crates/binary-cache/examples/upload_logs.rs new file mode 100644 index 000000000..1084b640e --- /dev/null +++ b/src/crates/binary-cache/examples/upload_logs.rs @@ -0,0 +1,31 @@ +use binary_cache::S3BinaryCacheClient; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let _tracing_guard = hydra_tracing::init()?; + let client = S3BinaryCacheClient::new( + "s3://store?region=unknown&endpoint=http://localhost:9000&scheme=http&write-nar-listing=1&ls-compression=br&log-compression=br&profile=local_nix_store".parse()?, + ) + .await?; + tracing::info!("{:#?}", client.cfg); + + let file = fs_err::tokio::File::open("/tmp/asdf").await.unwrap(); + let reader = Box::new(tokio::io::BufReader::new(file)); + client + .upsert_file_stream("log/test2.drv", reader, "text/plain; charset=utf-8") + .await?; + + let stats = client.s3_stats(); + tracing::info!( + "stats: put={}, put_bytes={}, put_time_ms={}, get={}, get_bytes={}, get_time_ms={}, head={}", + stats.put, + stats.put_bytes, + stats.put_time_ms, + stats.get, + stats.get_bytes, + stats.get_time_ms, + stats.head + ); + + Ok(()) +} diff --git a/src/crates/binary-cache/examples/upload_realisation.rs b/src/crates/binary-cache/examples/upload_realisation.rs new file mode 100644 index 000000000..a43560d3c --- /dev/null +++ b/src/crates/binary-cache/examples/upload_realisation.rs @@ -0,0 +1,44 @@ +use binary_cache::S3BinaryCacheClient; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let _tracing_guard = hydra_tracing::init()?; + let local = nix_utils::LocalStore::init(); + let client = S3BinaryCacheClient::new( + format!( + "s3://store2?region=unknown&endpoint=http://localhost:9000&scheme=http&write-nar-listing=1&compression=zstd&ls-compression=br&log-compression=br&secret-key={}/../../example-secret-key&profile=local_nix_store", + env!("CARGO_MANIFEST_DIR") + ).parse()?, + ) + .await?; + tracing::info!("{:#?}", client.cfg); + + let id = nix_utils::DrvOutput { + drv_hash: "sha256:6e46b9cf4fecaeab4b3c0578f4ab99e89d2f93535878c4ac69b5d5c4eb3a3db9" + .to_string(), + output_name: "debug".to_string(), + }; + tracing::info!( + "has realisation before: {}", + client.has_realisation(&id).await? + ); + client.copy_realisation(&local, &id, true).await?; + tracing::info!( + "has realisation after: {}", + client.has_realisation(&id).await? + ); + + let stats = client.s3_stats(); + tracing::info!( + "stats: put={}, put_bytes={}, put_time_ms={}, get={}, get_bytes={}, get_time_ms={}, head={}", + stats.put, + stats.put_bytes, + stats.put_time_ms, + stats.get, + stats.get_bytes, + stats.get_time_ms, + stats.head + ); + + Ok(()) +} diff --git a/src/crates/binary-cache/src/cfg.rs b/src/crates/binary-cache/src/cfg.rs new file mode 100644 index 000000000..2a7690bba --- /dev/null +++ b/src/crates/binary-cache/src/cfg.rs @@ -0,0 +1,1092 @@ +use hashbrown::HashMap; +use smallvec::SmallVec; + +use crate::Compression; + +const MIN_PRESIGNED_URL_EXPIRY_SECS: u64 = 60; +const MAX_PRESIGNED_URL_EXPIRY_SECS: u64 = 24 * 60 * 60; + +#[derive(Debug, Clone)] +#[allow(clippy::struct_excessive_bools)] +pub struct S3CacheConfig { + pub client_config: S3ClientConfig, + + pub compression: Compression, + pub write_nar_listing: bool, + pub write_debug_info: bool, + pub write_realisation: bool, + pub secret_key_files: SmallVec<[std::path::PathBuf; 4]>, + pub parallel_compression: bool, + pub compression_level: Option, + + pub narinfo_compression: Compression, + pub ls_compression: Compression, + pub log_compression: Compression, + pub buffer_size: usize, + + pub presigned_url_expiry: std::time::Duration, +} + +impl S3CacheConfig { + #[must_use] + pub fn new(client_config: S3ClientConfig) -> Self { + Self { + client_config, + compression: Compression::Xz, + write_nar_listing: false, + write_debug_info: false, + write_realisation: false, + secret_key_files: SmallVec::default(), + parallel_compression: false, + compression_level: Option::default(), + narinfo_compression: Compression::None, + ls_compression: Compression::None, + log_compression: Compression::None, + buffer_size: 8 * 1024 * 1024, + presigned_url_expiry: std::time::Duration::from_secs(3600), + } + } + + #[must_use] + pub const fn with_compression(mut self, compression: Option) -> Self { + if let Some(compression) = compression { + self.compression = compression; + } + self + } + + #[must_use] + pub fn with_write_nar_listing(mut self, write_nar_listing: Option<&str>) -> Self { + if let Some(write_nar_listing) = write_nar_listing { + let s = write_nar_listing.trim().to_ascii_lowercase(); + self.write_nar_listing = s.as_str() == "1" || s.as_str() == "true"; + } + self + } + + #[must_use] + pub fn with_write_debug_info(mut self, write_debug_info: Option<&str>) -> Self { + if let Some(write_debug_info) = write_debug_info { + let s = write_debug_info.trim().to_ascii_lowercase(); + self.write_debug_info = s.as_str() == "1" || s.as_str() == "true"; + } + self + } + + #[must_use] + pub fn with_write_realisation(mut self, write_realisation: Option<&str>) -> Self { + if let Some(write_realisation) = write_realisation { + let s = write_realisation.trim().to_ascii_lowercase(); + self.write_realisation = s.as_str() == "1" || s.as_str() == "true"; + } + self + } + + #[must_use] + pub fn add_secret_key_files(mut self, secret_keys: &[std::path::PathBuf]) -> Self { + for sk in secret_keys { + self.secret_key_files.push(sk.into()); + } + self + } + + #[must_use] + pub fn with_parallel_compression(mut self, parallel_compression: Option<&str>) -> Self { + if let Some(parallel_compression) = parallel_compression { + let s = parallel_compression.trim().to_ascii_lowercase(); + self.parallel_compression = s.as_str() == "1" || s.as_str() == "true"; + } + self + } + + #[must_use] + pub const fn with_compression_level(mut self, compression_level: Option) -> Self { + if let Some(compression_level) = compression_level { + self.compression_level = Some(compression_level); + } + self + } + + #[must_use] + pub const fn with_narinfo_compression(mut self, compression: Option) -> Self { + if let Some(compression) = compression { + self.narinfo_compression = compression; + } + self + } + + #[must_use] + pub const fn with_ls_compression(mut self, compression: Option) -> Self { + if let Some(compression) = compression { + self.ls_compression = compression; + } + self + } + + #[must_use] + pub const fn with_log_compression(mut self, compression: Option) -> Self { + if let Some(compression) = compression { + self.log_compression = compression; + } + self + } + + #[must_use] + pub const fn with_buffer_size(mut self, buffer_size: Option) -> Self { + if let Some(buffer_size) = buffer_size { + self.buffer_size = buffer_size; + } + self + } + + pub fn with_presigned_url_expiry( + mut self, + expiry_secs: Option, + ) -> Result { + if let Some(expiry_secs) = expiry_secs { + if !(MIN_PRESIGNED_URL_EXPIRY_SECS..=MAX_PRESIGNED_URL_EXPIRY_SECS) + .contains(&expiry_secs) + { + return Err(UrlParseError::InvalidPresignedUrlExpiry( + expiry_secs, + MIN_PRESIGNED_URL_EXPIRY_SECS, + MAX_PRESIGNED_URL_EXPIRY_SECS, + )); + } + self.presigned_url_expiry = std::time::Duration::from_secs(expiry_secs); + } + Ok(self) + } + + pub(crate) const fn get_compression_level(&self) -> async_compression::Level { + if let Some(l) = self.compression_level { + async_compression::Level::Precise(l) + } else { + async_compression::Level::Default + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum UrlParseError { + #[error("Uri parse error: {0}")] + UriParseError(#[from] url::ParseError), + #[error("Int parse error: {0}")] + IntParseError(#[from] std::num::ParseIntError), + #[error("Invalid S3Scheme: {0}")] + S3SchemeParseError(String), + #[error("Invalid Compression: {0}")] + CompressionParseError(String), + #[error("Bad schema: {0}")] + BadSchema(String), + #[error("Bucket not defined")] + NoBucket, + #[error("Invalid presigned URL expiry: {0}. Must be between {1} and {2} seconds")] + InvalidPresignedUrlExpiry(u64, u64, u64), +} + +impl std::str::FromStr for S3CacheConfig { + type Err = UrlParseError; + + #[allow(clippy::too_many_lines)] + fn from_str(s: &str) -> Result { + let uri = url::Url::parse(&s.trim().to_ascii_lowercase())?; + if uri.scheme() != "s3" { + return Err(UrlParseError::BadSchema(uri.scheme().to_owned())); + } + let bucket = uri.authority(); + if bucket.is_empty() { + return Err(UrlParseError::NoBucket); + } + let query = uri.query_pairs().into_owned().collect::>(); + let cfg = S3ClientConfig::new(bucket.to_owned()) + .with_region(query.get("region").map(std::string::String::as_str)) + .with_scheme( + query + .get("scheme") + .map(|x| x.parse::()) + .transpose() + .map_err(UrlParseError::S3SchemeParseError)?, + ) + .with_endpoint(query.get("endpoint").map(std::string::String::as_str)) + .with_profile(query.get("profile").map(std::string::String::as_str)); + + Self::new(cfg) + .with_compression( + query + .get("compression") + .map(|x| x.parse::()) + .transpose() + .map_err(UrlParseError::CompressionParseError)?, + ) + .with_write_nar_listing( + query + .get("write-nar-listing") + .map(std::string::String::as_str), + ) + .with_write_debug_info( + query + .get("write-debug-info") + .map(std::string::String::as_str), + ) + .add_secret_key_files( + &query + .get("secret-key") + .map(|s| if s.is_empty() { vec![] } else { vec![s.into()] }) + .unwrap_or_default(), + ) + .add_secret_key_files( + &query + .get("secret-keys") + .map(|s| { + s.split(',') + .filter(|s| !s.is_empty()) + .map(Into::into) + .collect::>() + }) + .unwrap_or_default(), + ) + .with_parallel_compression( + query + .get("parallel-compression") + .map(std::string::String::as_str), + ) + .with_compression_level( + query + .get("compression-level") + .map(|x| x.parse::()) + .transpose()?, + ) + .with_narinfo_compression( + query + .get("narinfo-compression") + .map(|x| x.parse::()) + .transpose() + .map_err(UrlParseError::CompressionParseError)?, + ) + .with_ls_compression( + query + .get("ls-compression") + .map(|x| x.parse::()) + .transpose() + .map_err(UrlParseError::CompressionParseError)?, + ) + .with_log_compression( + query + .get("log-compression") + .map(|x| x.parse::()) + .transpose() + .map_err(UrlParseError::CompressionParseError)?, + ) + .with_buffer_size( + query + .get("buffer-size") + .map(|x| x.parse::()) + .transpose()?, + ) + .with_presigned_url_expiry( + query + .get("presigned-url-expiry") + .map(|x| x.parse::()) + .transpose()?, + ) + } +} + +#[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] +pub enum S3Scheme { + HTTP, + #[default] + HTTPS, +} + +impl std::str::FromStr for S3Scheme { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "http" => Ok(Self::HTTP), + "https" => Ok(Self::HTTPS), + v => Err(v.to_owned()), + } + } +} + +#[derive(Debug, Clone)] +pub struct S3ClientConfig { + pub region: String, + pub scheme: S3Scheme, + pub endpoint: Option, + pub bucket: String, + pub profile: Option, + pub(crate) credentials: Option, +} + +impl S3ClientConfig { + #[must_use] + pub fn new(bucket: String) -> Self { + Self { + region: "us-east-1".into(), + scheme: S3Scheme::default(), + endpoint: None, + bucket, + profile: None, + credentials: None, + } + } + + #[must_use] + pub fn with_region(mut self, region: Option<&str>) -> Self { + if let Some(region) = region { + self.region = region.into(); + } + self + } + + #[must_use] + pub const fn with_scheme(mut self, scheme: Option) -> Self { + if let Some(scheme) = scheme { + self.scheme = scheme; + } + self + } + + #[must_use] + pub fn with_endpoint(mut self, endpoint: Option<&str>) -> Self { + self.endpoint = endpoint.map(ToOwned::to_owned); + self + } + + #[must_use] + pub fn with_profile(mut self, profile: Option<&str>) -> Self { + self.profile = profile.map(ToOwned::to_owned); + self + } + + #[must_use] + pub fn with_credentials(mut self, credentials: Option) -> Self { + self.credentials = credentials; + self + } +} + +#[derive(Debug, Clone)] +pub struct S3CredentialsConfig { + pub access_key_id: String, + pub secret_access_key: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum ConfigReadError { + #[error("Env var not found: {0}")] + EnvVarNotFound(#[from] std::env::VarError), + #[error("Read error: {0}")] + ReadError(String), + #[error("Profile missing: {0}")] + ProfileMissing(String), + #[error("Value missing: {0}")] + ValueMissing(&'static str), +} + +pub fn read_aws_credentials_file(profile: &str) -> Result<(String, String), ConfigReadError> { + let home_dir = std::env::var("HOME").or_else(|_| std::env::var("USERPROFILE"))?; + let credentials_path = format!("{home_dir}/.aws/credentials"); + + let mut config = configparser::ini::Ini::new(); + let config_map = config + .load(&credentials_path) + .map_err(ConfigReadError::ReadError)?; + parse_aws_credentials_file(&config_map, profile) +} + +fn parse_aws_credentials_file( + config_map: &std::collections::HashMap< + String, + std::collections::HashMap>, + >, + profile: &str, +) -> Result<(String, String), ConfigReadError> { + let profile_map = if let Some(profile_map) = config_map.get(profile) { + profile_map + } else if let Some(profile_map) = config_map.get(&format!("profile {profile}")) { + profile_map + } else { + let mut r_section_map = None; + for (section_name, section_map) in config_map { + let trimmed_section = section_name.trim(); + if trimmed_section == profile || trimmed_section == format!("profile {profile}") { + r_section_map = Some(section_map); + break; + } + } + if let Some(section_map) = r_section_map { + section_map + } else { + return Err(ConfigReadError::ProfileMissing(profile.into())); + } + }; + + let access_key = profile_map + .get("aws_access_key_id") + .and_then(ToOwned::to_owned) + .ok_or(ConfigReadError::ValueMissing("aws_access_key_id"))?; + let secret_key = profile_map + .get("aws_secret_access_key") + .and_then(ToOwned::to_owned) + .ok_or(ConfigReadError::ValueMissing("aws_secret_access_key"))?; + + Ok((access_key, secret_key)) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use super::{ + Compression, ConfigReadError, S3CacheConfig, S3ClientConfig, S3CredentialsConfig, S3Scheme, + UrlParseError, parse_aws_credentials_file, + }; + use std::str::FromStr as _; + + #[test] + fn test_parsing_default_profile_works() { + let mut config = configparser::ini::Ini::new(); + let config_map = config + .read( + r" +# AWS credentials file format: +# ~/.aws/credentials +[default] +aws_access_key_id = AKIAIOSFODNN7EXAMPLE +aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +[production] +aws_access_key_id = AKIAI44QH8DHBEXAMPLE +aws_secret_access_key = je7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY" + .into(), + ) + .unwrap(); + + let (access_key, secret_key) = parse_aws_credentials_file(&config_map, "default").unwrap(); + assert_eq!(access_key, "AKIAIOSFODNN7EXAMPLE"); + assert_eq!(secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + } + + #[test] + fn test_parsing_profile_with_spaces_and_comments() { + let mut config = configparser::ini::Ini::new(); + let config_map = config + .read( + r" +# This is a comment +# AWS credentials file with various formatting +[default] +# Another comment before the key +aws_access_key_id = AKIAIOSFODNN7EXAMPLE +aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +# Profile with spaces in name +[profile test] +aws_access_key_id = AKIAI44QH8DHBEXAMPLE +aws_secret_access_key = je7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY + +# Profile with extra whitespace +[ staging ] +aws_access_key_id = AKIAI44QH8DHBSTAGING +aws_secret_access_key = je7MtGbClwBF/2Zp9Utk/h3yCo8nvbSTAGINGKEY" + .into(), + ) + .unwrap(); + + println!( + "Available profiles: {:?}", + config_map.keys().collect::>() + ); + + let (access_key, secret_key) = parse_aws_credentials_file(&config_map, "default").unwrap(); + assert_eq!(access_key, "AKIAIOSFODNN7EXAMPLE"); + assert_eq!(secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + + let (access_key, secret_key) = parse_aws_credentials_file(&config_map, "test").unwrap(); + assert_eq!(access_key, "AKIAI44QH8DHBEXAMPLE"); + assert_eq!(secret_key, "je7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY"); + + let (access_key, secret_key) = parse_aws_credentials_file(&config_map, "staging").unwrap(); + assert_eq!(access_key, "AKIAI44QH8DHBSTAGING"); + assert_eq!(secret_key, "je7MtGbClwBF/2Zp9Utk/h3yCo8nvbSTAGINGKEY"); + } + + #[test] + fn test_missing_profile_returns_error() { + let mut config = configparser::ini::Ini::new(); + let config_map = config + .read( + r" +[default] +aws_access_key_id = AKIAIOSFODNN7EXAMPLE +aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + .into(), + ) + .unwrap(); + + let result = parse_aws_credentials_file(&config_map, "nonexistent"); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + ConfigReadError::ProfileMissing(_) + )); + } + + #[test] + fn test_missing_access_key_returns_error() { + let mut config = configparser::ini::Ini::new(); + let config_map = config + .read( + r" +[default] +aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + .into(), + ) + .unwrap(); + + let result = parse_aws_credentials_file(&config_map, "default"); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + ConfigReadError::ValueMissing("aws_access_key_id") + )); + } + + #[test] + fn test_missing_secret_key_returns_error() { + let mut config = configparser::ini::Ini::new(); + let config_map = config + .read( + r" +[default] +aws_access_key_id = AKIAIOSFODNN7EXAMPLE" + .into(), + ) + .unwrap(); + + let result = parse_aws_credentials_file(&config_map, "default"); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + ConfigReadError::ValueMissing("aws_secret_access_key") + )); + } + + #[test] + fn test_empty_credentials_file_returns_error() { + let mut config = configparser::ini::Ini::new(); + let config_map = config.read(String::new()).unwrap(); + + let result = parse_aws_credentials_file(&config_map, "default"); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + ConfigReadError::ProfileMissing(_) + )); + } + + #[test] + fn test_profile_with_special_characters() { + let mut config = configparser::ini::Ini::new(); + let config_map = config + .read( + r" +[default] +aws_access_key_id = AKIAIOSFODNN7EXAMPLE +aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +[my-test_profile] +aws_access_key_id = AKIAI44QH8DHBTEST +aws_secret_access_key = je7MtGbClwBF/2Zp9Utk/h3yCo8nvbTESTKEY + +[profile_123] +aws_access_key_id = AKIAI44QH8DHB123 +aws_secret_access_key = je7MtGbClwBF/2Zp9Utk/h3yCo8nvb123KEY" + .into(), + ) + .unwrap(); + + let (access_key, secret_key) = + parse_aws_credentials_file(&config_map, "my-test_profile").unwrap(); + assert_eq!(access_key, "AKIAI44QH8DHBTEST"); + assert_eq!(secret_key, "je7MtGbClwBF/2Zp9Utk/h3yCo8nvbTESTKEY"); + + let (access_key, secret_key) = + parse_aws_credentials_file(&config_map, "profile_123").unwrap(); + assert_eq!(access_key, "AKIAI44QH8DHB123"); + assert_eq!(secret_key, "je7MtGbClwBF/2Zp9Utk/h3yCo8nvb123KEY"); + } + + #[test] + fn test_presigned_url_expiry_validation() { + let valid_cases = vec!["60", "3600", "86400"]; // 1min, 1hr, 1day + for expiry in valid_cases { + let config_str = format!("s3://test-bucket?presigned-url-expiry={expiry}"); + let result = S3CacheConfig::from_str(&config_str); + assert!(result.is_ok(), "Should accept expiry: {expiry}"); + } + + let invalid_cases = vec!["0", "30", "604801"]; // too small, too small, too large + for expiry in invalid_cases { + let config_str = format!("s3://test-bucket?presigned-url-expiry={expiry}"); + let result = S3CacheConfig::from_str(&config_str); + assert!(result.is_err(), "Should reject expiry: {expiry}"); + if let Err(UrlParseError::InvalidPresignedUrlExpiry(value, min, max)) = result { + assert_eq!(value, expiry.parse::().unwrap()); + assert_eq!(min, 60); + assert_eq!(max, 86_400); + } else { + panic!("Expected InvalidPresignedUrlExpiry error"); + } + } + } + + #[test] + fn test_s3_cache_config_new_default_values() { + let client_config = S3ClientConfig::new("test-bucket".to_string()); + let config = S3CacheConfig::new(client_config); + + assert_eq!(config.compression, Compression::Xz); + assert!(!config.write_nar_listing); + assert!(!config.write_debug_info); + assert!(!config.write_realisation); + assert!(config.secret_key_files.is_empty()); + assert!(!config.parallel_compression); + assert_eq!(config.compression_level, None); + assert_eq!(config.narinfo_compression, Compression::None); + assert_eq!(config.ls_compression, Compression::None); + assert_eq!(config.log_compression, Compression::None); + assert_eq!(config.buffer_size, 8 * 1024 * 1024); + assert_eq!( + config.presigned_url_expiry, + std::time::Duration::from_secs(3600) + ); + } + + #[test] + fn test_s3_cache_config_builder_methods() { + let client_config = S3ClientConfig::new("test-bucket".to_string()); + + let config = + S3CacheConfig::new(client_config.clone()).with_compression(Some(Compression::Bzip2)); + assert_eq!(config.compression, Compression::Bzip2); + + let config = S3CacheConfig::new(client_config.clone()).with_compression(None); + assert_eq!(config.compression, Compression::Xz); + + let config = S3CacheConfig::new(client_config.clone()).with_write_nar_listing(Some("true")); + assert!(config.write_nar_listing); + + let config = S3CacheConfig::new(client_config.clone()).with_write_nar_listing(Some("1")); + assert!(config.write_nar_listing); + + let config = + S3CacheConfig::new(client_config.clone()).with_write_nar_listing(Some("false")); + assert!(!config.write_nar_listing); + + let config = S3CacheConfig::new(client_config.clone()).with_write_nar_listing(Some("0")); + assert!(!config.write_nar_listing); + + let config = S3CacheConfig::new(client_config.clone()).with_write_nar_listing(None); + assert!(!config.write_nar_listing); + + let config = S3CacheConfig::new(client_config.clone()).with_write_debug_info(Some("TRUE")); + assert!(config.write_debug_info); + + let config = + S3CacheConfig::new(client_config.clone()).with_write_debug_info(Some(" True ")); + assert!(config.write_debug_info); + + let config = S3CacheConfig::new(client_config.clone()).with_write_debug_info(None); + assert!(!config.write_debug_info); + + let config = S3CacheConfig::new(client_config.clone()).with_write_realisation(Some("TRUE")); + assert!(config.write_realisation); + + let config = + S3CacheConfig::new(client_config.clone()).with_write_realisation(Some(" True ")); + assert!(config.write_realisation); + + let config = S3CacheConfig::new(client_config.clone()).with_write_realisation(None); + assert!(!config.write_realisation); + + let secret_keys = vec![ + std::path::PathBuf::from("/path/to/key1"), + std::path::PathBuf::from("/path/to/key2"), + ]; + let config = S3CacheConfig::new(client_config.clone()).add_secret_key_files(&secret_keys); + assert_eq!(config.secret_key_files.len(), 2); + assert_eq!( + config.secret_key_files[0], + std::path::PathBuf::from("/path/to/key1") + ); + assert_eq!( + config.secret_key_files[1], + std::path::PathBuf::from("/path/to/key2") + ); + + let config = S3CacheConfig::new(client_config.clone()).with_parallel_compression(Some("1")); + assert!(config.parallel_compression); + + let config = + S3CacheConfig::new(client_config.clone()).with_parallel_compression(Some("false")); + assert!(!config.parallel_compression); + + let config = S3CacheConfig::new(client_config.clone()).with_compression_level(Some(9)); + assert_eq!(config.compression_level, Some(9)); + assert!(matches!( + config.get_compression_level(), + async_compression::Level::Precise(9) + )); + + let config = S3CacheConfig::new(client_config.clone()).with_compression_level(None); + assert_eq!(config.compression_level, None); + assert!(matches!( + config.get_compression_level(), + async_compression::Level::Default + )); + + let config = S3CacheConfig::new(client_config.clone()) + .with_narinfo_compression(Some(Compression::Zstd)); + assert_eq!(config.narinfo_compression, Compression::Zstd); + + let config = S3CacheConfig::new(client_config.clone()) + .with_ls_compression(Some(Compression::Brotli)); + assert_eq!(config.ls_compression, Compression::Brotli); + + let config = + S3CacheConfig::new(client_config.clone()).with_log_compression(Some(Compression::Xz)); + assert_eq!(config.log_compression, Compression::Xz); + + let config = + S3CacheConfig::new(client_config.clone()).with_buffer_size(Some(16 * 1024 * 1024)); + assert_eq!(config.buffer_size, 16 * 1024 * 1024); + + let config = S3CacheConfig::new(client_config.clone()) + .with_presigned_url_expiry(Some(7200)) + .unwrap(); + assert_eq!( + config.presigned_url_expiry, + std::time::Duration::from_secs(7200) + ); + + let config = S3CacheConfig::new(client_config) + .with_presigned_url_expiry(None) + .unwrap(); + assert_eq!( + config.presigned_url_expiry, + std::time::Duration::from_secs(3600) + ); + } + + #[test] + fn test_s3_cache_config_from_str_basic() { + let config = S3CacheConfig::from_str("s3://my-bucket").unwrap(); + assert_eq!(config.client_config.bucket, "my-bucket"); + assert_eq!(config.client_config.region, "us-east-1"); + assert_eq!(config.client_config.scheme, S3Scheme::HTTPS); + assert!(config.client_config.endpoint.is_none()); + assert!(config.client_config.profile.is_none()); + } + + #[test] + fn test_s3_cache_config_from_str_with_parameters() { + let config_str = "s3://test-bucket?region=eu-west-1&scheme=http&endpoint=custom.example.com&profile=myprofile&compression=zstd&write-nar-listing=true&write-debug-info=1¶llel-compression=true&compression-level=9&narinfo-compression=bz2&ls-compression=br&log-compression=xz&buffer-size=16777216&presigned-url-expiry=7200"; + + let config = S3CacheConfig::from_str(config_str).unwrap(); + + assert_eq!(config.client_config.bucket, "test-bucket"); + assert_eq!(config.client_config.region, "eu-west-1"); + assert_eq!(config.client_config.scheme, S3Scheme::HTTP); + assert_eq!( + config.client_config.endpoint, + Some("custom.example.com".to_string()) + ); + assert_eq!(config.client_config.profile, Some("myprofile".to_string())); + assert_eq!(config.compression, Compression::Zstd); + assert!(config.write_nar_listing); + assert!(config.write_debug_info); + assert!(config.parallel_compression); + assert_eq!(config.compression_level, Some(9)); + assert_eq!(config.narinfo_compression, Compression::Bzip2); + assert_eq!(config.ls_compression, Compression::Brotli); + assert_eq!(config.log_compression, Compression::Xz); + assert_eq!(config.buffer_size, 16_777_216); + assert_eq!( + config.presigned_url_expiry, + std::time::Duration::from_secs(7200) + ); + } + + #[test] + fn test_s3_cache_config_from_str_with_secret_keys() { + let config_str = "s3://test-bucket?secret-key=/path/to/key1"; + let config = S3CacheConfig::from_str(config_str).unwrap(); + assert_eq!(config.secret_key_files.len(), 1); + assert_eq!( + config.secret_key_files[0], + std::path::PathBuf::from("/path/to/key1") + ); + + let config_str = "s3://test-bucket?secret-keys=/path/to/key1,/path/to/key2,/path/to/key3"; + let config = S3CacheConfig::from_str(config_str).unwrap(); + assert_eq!(config.secret_key_files.len(), 3); + assert_eq!( + config.secret_key_files[0], + std::path::PathBuf::from("/path/to/key1") + ); + assert_eq!( + config.secret_key_files[1], + std::path::PathBuf::from("/path/to/key2") + ); + assert_eq!( + config.secret_key_files[2], + std::path::PathBuf::from("/path/to/key3") + ); + + let config_str = "s3://test-bucket?secret-key="; + let config = S3CacheConfig::from_str(config_str).unwrap(); + assert!(config.secret_key_files.is_empty()); + + let config_str = "s3://test-bucket?secret-keys="; + let config = S3CacheConfig::from_str(config_str).unwrap(); + assert!(config.secret_key_files.is_empty()); + + let config_str = "s3://test-bucket?secret-keys=/path/to/key1,,/path/to/key3"; + let config = S3CacheConfig::from_str(config_str).unwrap(); + assert_eq!(config.secret_key_files.len(), 2); + assert_eq!( + config.secret_key_files[0], + std::path::PathBuf::from("/path/to/key1") + ); + assert_eq!( + config.secret_key_files[1], + std::path::PathBuf::from("/path/to/key3") + ); + } + + #[test] + fn test_s3_cache_config_from_str_case_insensitive() { + let config_str = + "s3://test-bucket?write-nar-listing=TRUE&write-debug-info=False¶llel-compression=1"; + let config = S3CacheConfig::from_str(config_str).unwrap(); + assert!(config.write_nar_listing); + assert!(!config.write_debug_info); + assert!(config.parallel_compression); + + let config_str = "s3://test-bucket?compression=XZ&narinfo-compression=BZ2&ls-compression=BR&log-compression=ZSTD"; + let config = S3CacheConfig::from_str(config_str).unwrap(); + assert_eq!(config.compression, Compression::Xz); + assert_eq!(config.narinfo_compression, Compression::Bzip2); + assert_eq!(config.ls_compression, Compression::Brotli); + assert_eq!(config.log_compression, Compression::Zstd); + } + + #[test] + fn test_s3_cache_config_from_str_errors() { + let result = S3CacheConfig::from_str("http://test-bucket"); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), UrlParseError::BadSchema(_))); + + let result = S3CacheConfig::from_str("s3://"); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), UrlParseError::NoBucket)); + + let result = S3CacheConfig::from_str("s3://test-bucket?compression=invalid"); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + UrlParseError::CompressionParseError(_) + )); + + let result = S3CacheConfig::from_str("s3://test-bucket?scheme=invalid"); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + UrlParseError::S3SchemeParseError(_) + )); + + let result = S3CacheConfig::from_str("s3://test-bucket?compression-level=invalid"); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + UrlParseError::IntParseError(_) + )); + + let result = S3CacheConfig::from_str("s3://test-bucket?buffer-size=invalid"); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + UrlParseError::IntParseError(_) + )); + + let result = S3CacheConfig::from_str("s3://test-bucket?presigned-url-expiry=invalid"); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + UrlParseError::IntParseError(_) + )); + } + + #[test] + fn test_s3_scheme_from_str() { + assert_eq!(S3Scheme::from_str("http").unwrap(), S3Scheme::HTTP); + assert_eq!(S3Scheme::from_str("HTTP").unwrap(), S3Scheme::HTTP); + assert_eq!(S3Scheme::from_str("Http").unwrap(), S3Scheme::HTTP); + assert_eq!(S3Scheme::from_str(" http ").unwrap(), S3Scheme::HTTP); + + assert_eq!(S3Scheme::from_str("https").unwrap(), S3Scheme::HTTPS); + assert_eq!(S3Scheme::from_str("HTTPS").unwrap(), S3Scheme::HTTPS); + assert_eq!(S3Scheme::from_str("Https").unwrap(), S3Scheme::HTTPS); + assert_eq!(S3Scheme::from_str(" https ").unwrap(), S3Scheme::HTTPS); + + assert!(S3Scheme::from_str("ftp").is_err()); + assert!(S3Scheme::from_str("").is_err()); + assert!(S3Scheme::from_str("invalid").is_err()); + } + + #[test] + fn test_s3_client_config_builder() { + let mut config = S3ClientConfig::new("test-bucket".to_string()); + + assert_eq!(config.bucket, "test-bucket"); + assert_eq!(config.region, "us-east-1"); + assert_eq!(config.scheme, S3Scheme::HTTPS); + assert!(config.endpoint.is_none()); + assert!(config.profile.is_none()); + assert!(config.credentials.is_none()); + + config = config + .with_region(Some("eu-west-1")) + .with_scheme(Some(S3Scheme::HTTP)) + .with_endpoint(Some("custom.example.com")) + .with_profile(Some("myprofile")) + .with_credentials(Some(S3CredentialsConfig { + access_key_id: "test-key".to_string(), + secret_access_key: "test-secret".to_string(), + })); + + assert_eq!(config.region, "eu-west-1"); + assert_eq!(config.scheme, S3Scheme::HTTP); + assert_eq!(config.endpoint, Some("custom.example.com".to_string())); + assert_eq!(config.profile, Some("myprofile".to_string())); + assert!(config.credentials.is_some()); + + let credentials = config.credentials.as_ref().unwrap(); + assert_eq!(credentials.access_key_id, "test-key"); + assert_eq!(credentials.secret_access_key, "test-secret"); + + let config = config + .with_region(None) + .with_scheme(None) + .with_endpoint(None) + .with_profile(None) + .with_credentials(None); + + assert_eq!(config.region, "eu-west-1"); + assert_eq!(config.scheme, S3Scheme::HTTP); + assert_eq!(config.endpoint, None); + assert_eq!(config.profile, None); + assert!(config.credentials.is_none()); + } + + #[test] + fn test_s3_client_config_new() { + let config = S3ClientConfig::new("my-bucket".to_string()); + assert_eq!(config.bucket, "my-bucket"); + assert_eq!(config.region, "us-east-1"); + assert_eq!(config.scheme, S3Scheme::HTTPS); + assert!(config.endpoint.is_none()); + assert!(config.profile.is_none()); + assert!(config.credentials.is_none()); + } + + #[test] + fn test_s3_cache_config_chaining() { + let client_config = S3ClientConfig::new("test-bucket".to_string()); + + let config = S3CacheConfig::new(client_config) + .with_compression(Some(Compression::Zstd)) + .with_write_nar_listing(Some("true")) + .with_write_debug_info(Some("1")) + .with_write_realisation(Some("1")) + .with_parallel_compression(Some("true")) + .with_compression_level(Some(6)) + .with_narinfo_compression(Some(Compression::Bzip2)) + .with_ls_compression(Some(Compression::Brotli)) + .with_log_compression(Some(Compression::Xz)) + .with_buffer_size(Some(16 * 1024 * 1024)) + .with_presigned_url_expiry(Some(7200)) + .unwrap() + .add_secret_key_files(&[ + std::path::PathBuf::from("/path/to/key1"), + std::path::PathBuf::from("/path/to/key2"), + ]); + + assert_eq!(config.compression, Compression::Zstd); + assert!(config.write_nar_listing); + assert!(config.write_debug_info); + assert!(config.write_realisation); + assert!(config.parallel_compression); + assert_eq!(config.compression_level, Some(6)); + assert_eq!(config.narinfo_compression, Compression::Bzip2); + assert_eq!(config.ls_compression, Compression::Brotli); + assert_eq!(config.log_compression, Compression::Xz); + assert_eq!(config.buffer_size, 16 * 1024 * 1024); + assert_eq!( + config.presigned_url_expiry, + std::time::Duration::from_secs(7200) + ); + assert_eq!(config.secret_key_files.len(), 2); + } + + #[test] + fn test_s3_cache_config_from_str_with_whitespace() { + let config_str = " s3://test-bucket?region=eu-west-1&compression=xz "; + let config = S3CacheConfig::from_str(config_str).unwrap(); + + assert_eq!(config.client_config.bucket, "test-bucket"); + assert_eq!(config.client_config.region, "eu-west-1"); + assert_eq!(config.compression, Compression::Xz); + } + + #[test] + fn test_s3_cache_config_from_str_empty_query_params() { + let config_str = "s3://test-bucket?"; + let config = S3CacheConfig::from_str(config_str).unwrap(); + + assert_eq!(config.client_config.bucket, "test-bucket"); + assert_eq!(config.client_config.region, "us-east-1"); + assert_eq!(config.compression, Compression::Xz); + } + + #[test] + fn test_s3_cache_config_presigned_url_expiry_boundaries() { + let config_str = "s3://test-bucket?presigned-url-expiry=60"; + let config = S3CacheConfig::from_str(config_str).unwrap(); + assert_eq!( + config.presigned_url_expiry, + std::time::Duration::from_secs(60) + ); + + let config_str = "s3://test-bucket?presigned-url-expiry=86400"; + let config = S3CacheConfig::from_str(config_str).unwrap(); + assert_eq!( + config.presigned_url_expiry, + std::time::Duration::from_secs(86400) + ); + + let config_str = "s3://test-bucket?presigned-url-expiry=59"; + let result = S3CacheConfig::from_str(config_str); + assert!(result.is_err()); + + let config_str = "s3://test-bucket?presigned-url-expiry=86401"; + let result = S3CacheConfig::from_str(config_str); + assert!(result.is_err()); + } +} diff --git a/src/crates/binary-cache/src/compression.rs b/src/crates/binary-cache/src/compression.rs new file mode 100644 index 000000000..6a52e2c53 --- /dev/null +++ b/src/crates/binary-cache/src/compression.rs @@ -0,0 +1,86 @@ +use async_compression::{ + Level, + tokio::bufread::{BrotliEncoder, BzEncoder, XzEncoder, ZstdEncoder}, +}; + +pub type CompressorFn = + Box Box + Send>; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Compression { + None, + Xz, + Bzip2, + Brotli, + Zstd, +} + +impl Compression { + #[must_use] + pub const fn ext(self) -> &'static str { + match self { + Self::None => "nar", + Self::Xz => "nar.xz", + Self::Bzip2 => "nar.bz2", + Self::Brotli => "nar.br", + Self::Zstd => "nar.zst", + } + } + + #[must_use] + pub const fn content_type(self) -> &'static str { + "application/x-nix-nar" + } + + #[must_use] + pub const fn content_encoding(self) -> &'static str { + "" + } + + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Xz => "xz", + Self::Bzip2 => "bz2", + Self::Brotli => "br", + Self::Zstd => "zstd", + } + } + + #[must_use] + pub fn get_compression_fn( + self, + level: Level, + parallel: bool, + ) -> CompressorFn { + match self { + Self::None => Box::new(|c| Box::new(c)), + Self::Xz => { + if parallel && let Some(cores) = std::num::NonZero::new(4) { + Box::new(move |s| Box::new(XzEncoder::parallel(s, level, cores))) + } else { + Box::new(move |s| Box::new(XzEncoder::with_quality(s, level))) + } + } + Self::Bzip2 => Box::new(move |s| Box::new(BzEncoder::with_quality(s, level))), + Self::Brotli => Box::new(move |s| Box::new(BrotliEncoder::with_quality(s, level))), + Self::Zstd => Box::new(move |s| Box::new(ZstdEncoder::with_quality(s, level))), + } + } +} + +impl std::str::FromStr for Compression { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "none" => Ok(Self::None), + "xz" => Ok(Self::Xz), + "bz2" => Ok(Self::Bzip2), + "br" => Ok(Self::Brotli), + "zstd" | "zst" => Ok(Self::Zstd), + o => Err(o.to_string()), + } + } +} diff --git a/src/crates/binary-cache/src/debug_info.rs b/src/crates/binary-cache/src/debug_info.rs new file mode 100644 index 000000000..d3f7ca4e2 --- /dev/null +++ b/src/crates/binary-cache/src/debug_info.rs @@ -0,0 +1,492 @@ +//! Debug info processing functionality for binary cache. +//! +//! This module handles the extraction and processing of debug information +//! from NIX store paths that contain debug symbols in the standard +//! `lib/debug/.build-id` directory structure. + +use nix_utils::BaseStore as _; + +use crate::CacheError; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct DebugInfoLink { + pub(crate) archive: String, + pub(crate) member: String, +} + +/// Processes debug info for a given store path using a custom full path. +/// This is useful for testing with custom store prefixes. +pub async fn process_debug_info( + nar_url: &str, + store: &nix_utils::LocalStore, + store_path: &nix_utils::StorePath, + client: C, +) -> Result<(), CacheError> +where + C: DebugInfoClient + Clone + Send + Sync + 'static, +{ + use futures::stream::StreamExt as _; + + let full_path = store.print_store_path(store_path); + let build_id_path = std::path::Path::new(&full_path).join("lib/debug/.build-id"); + + if !build_id_path.exists() { + tracing::debug!("No lib/debug/.build-id directory found in {}", store_path); + return Ok(()); + } + + let debug_files = find_debug_files(&build_id_path).await?; + let mut stream = tokio_stream::iter(debug_files) + .map(|(build_id, debug_path)| { + let client = client.clone(); + async move { + client + .create_debug_info_link(nar_url, build_id, debug_path) + .await + } + }) + .buffered(25); + while let Some(result) = tokio_stream::StreamExt::next(&mut stream).await { + result?; + } + Ok(()) +} + +pub async fn get_debug_info_build_ids( + store: &nix_utils::LocalStore, + store_path: &nix_utils::StorePath, +) -> Result, CacheError> { + let full_path = store.print_store_path(store_path); + let build_id_path = std::path::Path::new(&full_path).join("lib/debug/.build-id"); + + if !build_id_path.exists() { + tracing::debug!("No lib/debug/.build-id directory found in {}", store_path); + return Ok(vec![]); + } + + Ok(find_debug_files(&build_id_path) + .await? + .into_iter() + .map(|(id, _)| id) + .collect()) +} + +/// Finds debug files by scanning the build-id directory structure. +async fn find_debug_files( + build_id_path: &std::path::Path, +) -> Result, CacheError> { + let mut debug_files = Vec::new(); + + let mut entries = fs_err::tokio::read_dir(build_id_path) + .await + .map_err(CacheError::Io)?; + + while let Some(entry) = entries.next_entry().await.map_err(CacheError::Io)? { + let s1 = entry.file_name(); + let s1_str = s1.to_string_lossy(); + + // Check if it's a 2-character hex directory + if s1_str.len() != 2 + || !s1_str.chars().all(|c| c.is_ascii_hexdigit()) + || !entry.file_type().await.map_err(CacheError::Io)?.is_dir() + { + continue; + } + + let subdir_path = build_id_path.join(&s1); + let mut subdir_entries = fs_err::tokio::read_dir(&subdir_path) + .await + .map_err(CacheError::Io)?; + + while let Some(sub_entry) = subdir_entries.next_entry().await.map_err(CacheError::Io)? { + let s2 = sub_entry.file_name(); + let s2_str = s2.to_string_lossy(); + + // Check if it's a 38-character hex file ending with .debug + if s2_str.len() == 44 // 38 chars + .debug (6 chars) = 44 + && s2_str.ends_with(".debug") + && s2_str[..38].chars().all(|c| c.is_ascii_hexdigit()) + && sub_entry.file_type().await.map_err(CacheError::Io)?.is_file() + { + let build_id = format!("{s1_str}{}", &s2_str[..38]); + let debug_path = format!("lib/debug/.build-id/{s1_str}/{s2_str}"); + debug_files.push((build_id, debug_path)); + } + } + } + + Ok(debug_files) +} + +pub trait DebugInfoClient { + async fn create_debug_info_link( + &self, + nar_url: &str, + build_id: String, + debug_path: String, + ) -> Result<(), CacheError>; +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use super::*; + + #[derive(Debug, Clone)] + struct MockClient { + created_links: std::sync::Arc>>, + } + + impl MockClient { + fn new() -> Self { + Self { + created_links: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), + } + } + + fn get_created_links(&self) -> Vec { + self.created_links.lock().unwrap().clone() + } + } + + impl DebugInfoClient for MockClient { + async fn create_debug_info_link( + &self, + nar_url: &str, + _build_id: String, + debug_path: String, + ) -> Result<(), CacheError> { + let link = DebugInfoLink { + archive: format!("../{nar_url}"), + member: debug_path, + }; + + self.created_links.lock().unwrap().push(link); + Ok(()) + } + } + + #[tokio::test] + async fn test_find_debug_files() { + let temp_dir = tempfile::tempdir().unwrap().keep(); + let build_id_dir = temp_dir.join("test_build_id"); + + fs_err::tokio::create_dir_all(&build_id_dir).await.unwrap(); + + let ab_dir = build_id_dir.join("ab"); + fs_err::tokio::create_dir(&ab_dir).await.unwrap(); + fs_err::tokio::write( + &ab_dir.join("cdef1234567890123456789012345678901234.debug"), + "test debug content", + ) + .await + .unwrap(); + + let cd_dir = build_id_dir.join("cd"); + fs_err::tokio::create_dir(&cd_dir).await.unwrap(); + fs_err::tokio::write( + &cd_dir.join("ef567890123456789012345678901234567890.debug"), + "test debug content 2", + ) + .await + .unwrap(); + + let mut debug_files = find_debug_files(&build_id_dir).await.unwrap(); + debug_files.sort_by(|(a, _), (b, _)| a.cmp(b)); + + assert_eq!(debug_files.len(), 2); + assert_eq!( + debug_files[0], + ( + "abcdef1234567890123456789012345678901234".to_string(), + "lib/debug/.build-id/ab/cdef1234567890123456789012345678901234.debug".to_string() + ) + ); + assert_eq!( + debug_files[1], + ( + "cdef567890123456789012345678901234567890".to_string(), + "lib/debug/.build-id/cd/ef567890123456789012345678901234567890.debug".to_string() + ) + ); + + fs_err::tokio::remove_dir_all(&build_id_dir).await.unwrap(); + } + + #[tokio::test] + async fn test_process_debug_info_integration() { + let mock_client = MockClient::new(); + + let temp_dir = tempfile::tempdir().unwrap().keep(); + let store_prefix = temp_dir.join("nix/store"); + let store_path_str = "1234567890123456789012345678901234-debug-output"; + let store_path = nix_utils::StorePath::new(store_path_str); + let full_path = store_prefix.join(store_path_str); + + fs_err::tokio::create_dir_all(&full_path).await.unwrap(); + let build_id_dir = full_path.join("lib/debug/.build-id"); + fs_err::tokio::create_dir_all(&build_id_dir).await.unwrap(); + + let ab_dir = build_id_dir.join("ab"); + fs_err::tokio::create_dir(&ab_dir).await.unwrap(); + let debug_file = ab_dir.join("cdef1234567890123456789012345678901234.debug"); + fs_err::tokio::write(&debug_file, "test debug content") + .await + .unwrap(); + + let mut local = nix_utils::LocalStore::init(); + local.unsafe_set_store_path_prefix(store_prefix.as_path().to_string_lossy().to_string()); + + process_debug_info("test.nar", &local, &store_path, mock_client.clone()) + .await + .unwrap(); + + let created_links = mock_client.get_created_links(); + assert_eq!(created_links.len(), 1); + assert_eq!(created_links[0].archive, "../test.nar"); + assert_eq!( + created_links[0].member, + "lib/debug/.build-id/ab/cdef1234567890123456789012345678901234.debug" + ); + + fs_err::tokio::remove_dir_all(&full_path).await.unwrap(); + } + + #[tokio::test] + async fn test_find_debug_files_empty_directory() { + let temp_dir = tempfile::tempdir().unwrap().keep(); + let build_id_dir = temp_dir.join("empty_build_id"); + + fs_err::tokio::create_dir_all(&build_id_dir).await.unwrap(); + + let debug_files = find_debug_files(&build_id_dir).await.unwrap(); + assert_eq!(debug_files.len(), 0); + + fs_err::tokio::remove_dir_all(&build_id_dir).await.unwrap(); + } + + #[tokio::test] + async fn test_find_debug_files_invalid_structure() { + let temp_dir = tempfile::tempdir().unwrap().keep(); + let build_id_dir = temp_dir.join("invalid_build_id"); + + fs_err::tokio::create_dir_all(&build_id_dir).await.unwrap(); + + fs_err::tokio::create_dir(&build_id_dir.join("invalid")) + .await + .unwrap(); + fs_err::tokio::create_dir(&build_id_dir.join("xyz")) + .await + .unwrap(); + fs_err::tokio::create_dir(&build_id_dir.join("123")) + .await + .unwrap(); + + let valid_dir = build_id_dir.join("ab"); + fs_err::tokio::create_dir(&valid_dir).await.unwrap(); + + fs_err::tokio::write(&valid_dir.join("invalid.txt"), "content") + .await + .unwrap(); + fs_err::tokio::write(&valid_dir.join("cdef.debug"), "content") + .await + .unwrap(); + fs_err::tokio::write( + &valid_dir.join("cdef12345678901234567890123456789012345.debug"), + "content", + ) + .await + .unwrap(); + fs_err::tokio::write( + &valid_dir.join("cdef1234567890123456789012345678901234.txt"), + "content", + ) + .await + .unwrap(); + fs_err::tokio::write( + &valid_dir.join("xyz1234567890123456789012345678901234.debug"), + "content", + ) + .await + .unwrap(); + + let debug_files = find_debug_files(&build_id_dir).await.unwrap(); + assert_eq!(debug_files.len(), 0); + + fs_err::tokio::remove_dir_all(&build_id_dir).await.unwrap(); + } + + #[tokio::test] + async fn test_find_debug_files_mixed_valid_invalid() { + let temp_dir = tempfile::tempdir().unwrap().keep(); + let build_id_dir = temp_dir.join("mixed_build_id"); + + fs_err::tokio::create_dir_all(&build_id_dir).await.unwrap(); + + let ab_dir = build_id_dir.join("ab"); + fs_err::tokio::create_dir(&ab_dir).await.unwrap(); + fs_err::tokio::write( + &ab_dir.join("cdef1234567890123456789012345678901234.debug"), + "valid debug content", + ) + .await + .unwrap(); + + fs_err::tokio::create_dir(&build_id_dir.join("invalid")) + .await + .unwrap(); + fs_err::tokio::write( + &build_id_dir.join("invalid").join("somefile.debug"), + "content", + ) + .await + .unwrap(); + + let cd_dir = build_id_dir.join("cd"); + fs_err::tokio::create_dir(&cd_dir).await.unwrap(); + fs_err::tokio::write( + &cd_dir.join("ef567890123456789012345678901234567890.debug"), + "another valid debug content", + ) + .await + .unwrap(); + fs_err::tokio::write(&cd_dir.join("invalid.txt"), "invalid content") + .await + .unwrap(); + + let debug_files = find_debug_files(&build_id_dir).await.unwrap(); + assert_eq!(debug_files.len(), 2); + + let build_ids: Vec = debug_files.iter().map(|(id, _)| id.clone()).collect(); + assert!(build_ids.contains(&"abcdef1234567890123456789012345678901234".to_string())); + assert!(build_ids.contains(&"cdef567890123456789012345678901234567890".to_string())); + + fs_err::tokio::remove_dir_all(&build_id_dir).await.unwrap(); + } + + #[tokio::test] + async fn test_process_debug_info_no_build_id_dir() { + let mock_client = MockClient::new(); + + let temp_dir = tempfile::tempdir().unwrap().keep(); + let store_prefix = temp_dir.join("nix/store"); + let store_path_str = "1234567890123456789012345678901234-no-debug"; + let store_path = nix_utils::StorePath::new(store_path_str); + let full_path = temp_dir.join("nix/store").join(store_path_str); + + fs_err::tokio::create_dir_all(&full_path).await.unwrap(); + + let mut local = nix_utils::LocalStore::init(); + local.unsafe_set_store_path_prefix(store_prefix.as_path().to_string_lossy().to_string()); + + process_debug_info("test.nar", &local, &store_path, mock_client.clone()) + .await + .unwrap(); + + let created_links = mock_client.get_created_links(); + assert_eq!(created_links.len(), 0); + + fs_err::tokio::remove_dir_all(&full_path).await.unwrap(); + } + + #[tokio::test] + async fn test_process_debug_info_empty_build_id_dir() { + let mock_client = MockClient::new(); + + let temp_dir = tempfile::tempdir().unwrap().keep(); + let store_prefix = temp_dir.join("nix/store"); + let store_path_str = "1234567890123456789012345678901234-empty-debug"; + let store_path = nix_utils::StorePath::new(store_path_str); + let full_path = temp_dir.join("nix/store").join(store_path_str); + + fs_err::tokio::create_dir_all(&full_path).await.unwrap(); + let build_id_dir = full_path.join("lib/debug/.build-id"); + fs_err::tokio::create_dir_all(&build_id_dir).await.unwrap(); + + let mut local = nix_utils::LocalStore::init(); + local.unsafe_set_store_path_prefix(store_prefix.as_path().to_string_lossy().to_string()); + + process_debug_info("test.nar", &local, &store_path, mock_client.clone()) + .await + .unwrap(); + + let created_links = mock_client.get_created_links(); + assert_eq!(created_links.len(), 0); + + fs_err::tokio::remove_dir_all(&full_path).await.unwrap(); + } + + #[tokio::test] + async fn test_process_debug_info_multiple_files() { + let mock_client = MockClient::new(); + + let temp_dir = tempfile::tempdir().unwrap().keep(); + let store_prefix = temp_dir.join("nix/store"); + let store_path_str = "1234567890123456789012345678901234-multi-debug"; + let store_path = nix_utils::StorePath::new(store_path_str); + let full_path = temp_dir.join("nix/store").join(store_path_str); + + fs_err::tokio::create_dir_all(&full_path).await.unwrap(); + let build_id_dir = full_path.join("lib/debug/.build-id"); + fs_err::tokio::create_dir_all(&build_id_dir).await.unwrap(); + + let subdirs = [ + ("ab", "cdef1234567890123456789012345678901234"), + ("cd", "ef567890123456789012345678901234567890"), + ("12", "34567890123456789012345678901234567890"), + ]; + + for (subdir, filename) in &subdirs { + let dir = build_id_dir.join(subdir); + fs_err::tokio::create_dir(&dir).await.unwrap(); + fs_err::tokio::write(&dir.join(format!("{filename}.debug")), "debug content") + .await + .unwrap(); + } + + let mut local = nix_utils::LocalStore::init(); + local.unsafe_set_store_path_prefix(store_prefix.as_path().to_string_lossy().to_string()); + + process_debug_info("multi.nar", &local, &store_path, mock_client.clone()) + .await + .unwrap(); + + let created_links = mock_client.get_created_links(); + assert_eq!(created_links.len(), 3); + + for link in &created_links { + assert_eq!(link.archive, "../multi.nar"); + } + + let members: Vec = created_links.iter().map(|l| l.member.clone()).collect(); + assert!(members.contains( + &"lib/debug/.build-id/ab/cdef1234567890123456789012345678901234.debug".to_string() + )); + assert!(members.contains( + &"lib/debug/.build-id/cd/ef567890123456789012345678901234567890.debug".to_string() + )); + assert!(members.contains( + &"lib/debug/.build-id/12/34567890123456789012345678901234567890.debug".to_string() + )); + + fs_err::tokio::remove_dir_all(&full_path).await.unwrap(); + } + + #[tokio::test] + async fn test_debug_info_link_serialization() { + let link = DebugInfoLink { + archive: "../test.nar".to_string(), + member: "lib/debug/.build-id/ab/cdef1234567890123456789012345678901234.debug" + .to_string(), + }; + + let json = serde_json::to_string(&link).unwrap(); + let expected = r#"{"archive":"../test.nar","member":"lib/debug/.build-id/ab/cdef1234567890123456789012345678901234.debug"}"#; + assert_eq!(json, expected); + + let deserialized: DebugInfoLink = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.archive, link.archive); + assert_eq!(deserialized.member, link.member); + } +} diff --git a/src/crates/binary-cache/src/lib.rs b/src/crates/binary-cache/src/lib.rs new file mode 100644 index 000000000..5f77555ef --- /dev/null +++ b/src/crates/binary-cache/src/lib.rs @@ -0,0 +1,992 @@ +#![deny(clippy::all)] +#![deny(clippy::pedantic)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::expect_used)] +#![allow(clippy::missing_errors_doc)] + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use bytes::Bytes; +use moka::future::Cache; +use object_store::{ObjectStore as _, ObjectStoreExt as _, signer::Signer as _}; +use secrecy::ExposeSecret; +use smallvec::SmallVec; + +use nix_utils::BaseStore as _; +use nix_utils::RealisationOperations as _; + +mod cfg; +mod compression; +mod debug_info; +mod narinfo; +mod presigned; +mod streaming_hash; + +pub use crate::cfg::{S3CacheConfig, S3ClientConfig, S3CredentialsConfig, S3Scheme}; +pub use crate::compression::Compression; +pub use crate::debug_info::get_debug_info_build_ids; +pub use crate::narinfo::NarInfo; +use crate::narinfo::NarInfoError; +pub use crate::presigned::{ + PresignedUpload, PresignedUploadClient, PresignedUploadMetrics, PresignedUploadResponse, + PresignedUploadResult, +}; +pub use async_compression::Level as CompressionLevel; + +pub async fn path_to_narinfo( + store: &nix_utils::LocalStore, + path: &nix_utils::StorePath, +) -> Result { + let Some(path_info) = store.query_path_info(path).await else { + return Err(CacheError::PathNotFound { + path: path.to_string(), + }); + }; + let narinfo = NarInfo::simple(path, path_info, Compression::None); + let queried_references = store + .query_path_infos(&narinfo.references.iter().collect::>()) + .await; + for r in &narinfo.references { + if !queried_references.contains_key(r) { + return Err(CacheError::ReferenceVerifyError( + narinfo.store_path, + r.to_owned(), + )); + } + } + Ok(narinfo) +} + +#[derive(Debug, Default)] +struct AtomicS3Stats { + put: AtomicU64, + put_bytes: AtomicU64, + put_time_ms: AtomicU64, + get: AtomicU64, + get_bytes: AtomicU64, + get_time_ms: AtomicU64, + head: AtomicU64, +} + +#[derive(Debug, Default)] +pub struct S3Stats { + pub put: u64, + pub put_bytes: u64, + pub put_time_ms: u64, + pub get: u64, + pub get_bytes: u64, + pub get_time_ms: u64, + pub head: u64, +} + +impl S3Stats { + #[must_use] + pub fn put_speed(&self) -> f64 { + #[allow(clippy::cast_precision_loss)] + if self.put_time_ms > 0 { + self.put_bytes as f64 / self.put_time_ms as f64 * 1000.0 / (1024.0 * 1024.0) + } else { + 0.0 + } + } + + #[must_use] + pub fn get_speed(&self) -> f64 { + #[allow(clippy::cast_precision_loss)] + if self.get_time_ms > 0 { + self.get_bytes as f64 / self.get_time_ms as f64 * 1000.0 / (1024.0 * 1024.0) + } else { + 0.0 + } + } + + #[must_use] + pub fn cost_dollar_approx(&self) -> f64 { + #[allow(clippy::cast_precision_loss)] + (self.get_bytes as f64 / (1024.0 * 1024.0 * 1024.0)).mul_add( + 0.09, + ((self.get as f64 + self.head as f64) / 10000.0) + .mul_add(0.004, self.put as f64 / 1000.0 * 0.005), + ) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum CacheError { + #[error("Object store error: {0}")] + ObjectStore(#[from] object_store::Error), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("serde_json error: {0}")] + Serde(#[from] serde_json::Error), + #[error("Signing error: {0}")] + Signing(String), + #[error(transparent)] + NarInfoParseError(#[from] NarInfoError), + #[error(transparent)] + NixStoreError(#[from] nix_utils::Error), + #[error("cannot add '{0}' to the binary cache because the reference '{1}' is not valid")] + ReferenceVerifyError(nix_utils::StorePath, nix_utils::StorePath), + #[error("Hash error: {0}")] + HashingError(#[from] crate::streaming_hash::Error), + #[error("Render error: {0}")] + RenderError(#[from] std::fmt::Error), + #[error("HTTP request failed: {0}")] + HttpRequestError(#[from] reqwest::Error), + #[error("Upload failed for {path}: {reason}")] + UploadError { path: String, reason: String }, + #[error("Presigned URL generation failed for {path}: {reason}")] + PresignedUrlError { path: String, reason: String }, + #[error("Request cloning failed")] + RequestCloneError, + #[error("Path not found: {path}")] + PathNotFound { path: String }, + #[error("Configuration error: {message}")] + ConfigurationError { message: String }, +} + +#[derive(Clone)] +pub struct S3BinaryCacheClient { + s3: object_store::aws::AmazonS3, + pub cfg: cfg::S3CacheConfig, + s3_stats: Arc, + signing_keys: SmallVec<[secrecy::SecretString; 4]>, + narinfo_cache: Cache, +} + +#[tracing::instrument(skip(stream, chunk), err)] +async fn read_chunk_async( + stream: &mut S, + mut chunk: bytes::BytesMut, +) -> std::io::Result { + use tokio::io::AsyncReadExt as _; + + while chunk.len() < chunk.capacity() { + let read = stream.read_buf(&mut chunk).await?; + + if read == 0 { + break; + } + } + + Ok(chunk.freeze()) +} + +#[tracing::instrument(skip(upload_item, first_chunk, stream), err)] +async fn run_multipart_upload( + upload_item: &mut Box, + first_chunk: Bytes, + mut stream: &mut (dyn tokio::io::AsyncRead + Unpin + Send), + buffer_size: usize, +) -> Result { + let mut part_number = 1; + let mut first_chunk_opt = Some(first_chunk); + let mut file_size = 0; + + loop { + let chunk = if part_number == 1 + && let Some(first_chunk) = first_chunk_opt.take() + { + first_chunk + } else { + let buf = bytes::BytesMut::with_capacity(buffer_size); + read_chunk_async(&mut stream, buf).await? + }; + file_size += chunk.len(); + + if chunk.is_empty() { + break; + } + + tracing::debug!("Uploading part {} with size {}", part_number, chunk.len()); + upload_item.put_part(chunk.into()).await?; + part_number += 1; + } + + tracing::debug!( + "Completing multipart upload with {} parts, total size: {}", + part_number, + file_size + ); + upload_item.complete().await?; + Ok(file_size) +} + +impl S3BinaryCacheClient { + #[tracing::instrument(skip(cfg), err)] + fn construct_client( + cfg: &cfg::S3ClientConfig, + ) -> Result { + let mut builder = object_store::aws::AmazonS3Builder::from_env() + .with_region(&cfg.region) + .with_bucket_name(&cfg.bucket) + .with_imdsv1_fallback(); + + if let Some(credentials) = &cfg.credentials { + builder = builder + .with_access_key_id(&credentials.access_key_id) + .with_secret_access_key(&credentials.secret_access_key); + } else if std::env::var("AWS_ACCESS_KEY_ID").ok().is_none() + && std::env::var("AWS_SECRET_ACCESS_KEY").ok().is_none() + { + let profile = cfg.profile.as_deref().unwrap_or("default"); + if let Ok((access_key, secret_key)) = crate::cfg::read_aws_credentials_file(profile) { + tracing::info!( + "Using AWS credentials from credentials file for profile: {profile}", + ); + builder = builder + .with_access_key_id(&access_key) + .with_secret_access_key(&secret_key); + } else { + tracing::warn!( + "AWS credentials not found in environment variables or credentials file for profile: {profile}", + ); + } + } + + if let Some(endpoint) = &cfg.endpoint { + builder = builder.with_endpoint(endpoint); + builder = builder.with_virtual_hosted_style_request(false); + } + + if cfg.scheme == cfg::S3Scheme::HTTP { + builder = builder.with_allow_http(true); + } + + builder.build() + } + + #[tracing::instrument(skip(cfg), err)] + pub async fn new(cfg: cfg::S3CacheConfig) -> Result { + let mut signing_keys = SmallVec::default(); + for p in &cfg.secret_key_files { + signing_keys.push(secrecy::SecretString::new( + fs_err::tokio::read_to_string(p).await?.into(), + )); + } + + Ok(Self { + s3: Self::construct_client(&cfg.client_config)?, + cfg, + s3_stats: Arc::new(AtomicS3Stats::default()), + signing_keys, + narinfo_cache: Cache::builder() + .initial_capacity(1000) + .max_capacity(65536) + .build_with_hasher(foldhash::fast::RandomState::default()), + }) + } + + #[must_use] + pub fn s3_stats(&self) -> S3Stats { + S3Stats { + put: self.s3_stats.put.load(Ordering::Relaxed), + put_bytes: self.s3_stats.put_bytes.load(Ordering::Relaxed), + put_time_ms: self.s3_stats.put_time_ms.load(Ordering::Relaxed), + get: self.s3_stats.get.load(Ordering::Relaxed), + get_bytes: self.s3_stats.get_bytes.load(Ordering::Relaxed), + get_time_ms: self.s3_stats.get_time_ms.load(Ordering::Relaxed), + head: self.s3_stats.head.load(Ordering::Relaxed), + } + } + + #[tracing::instrument(skip(self), err)] + pub async fn head_object(&self, key: &str) -> Result { + let res = self.s3.head(&object_store::path::Path::from(key)).await; + self.s3_stats.head.fetch_add(1, Ordering::Relaxed); + match res { + Ok(_) => Ok(true), + Err(object_store::Error::NotFound { .. }) => Ok(false), + Err(e) => Err(CacheError::ObjectStore(e)), + } + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_object(&self, key: &str) -> Result, CacheError> { + let start = Instant::now(); + let get_result = match self.s3.get(&object_store::path::Path::from(key)).await { + Ok(v) => v, + Err(object_store::Error::NotFound { .. }) => return Ok(None), + Err(e) => return Err(CacheError::ObjectStore(e)), + }; + let bs = get_result.bytes().await?; + let elapsed = u64::try_from(start.elapsed().as_millis()).unwrap_or_default(); + + self.s3_stats.get.fetch_add(1, Ordering::Relaxed); + self.s3_stats.get_bytes.fetch_add( + u64::try_from(bs.len()).unwrap_or(u64::MAX), + Ordering::Relaxed, + ); + self.s3_stats + .get_time_ms + .fetch_add(elapsed, Ordering::Relaxed); + + Ok(Some(bs)) + } + + #[tracing::instrument(skip(self, content, content_type), err)] + pub async fn upsert_file( + &self, + name: &str, + content: String, + content_type: &str, + ) -> Result<(), CacheError> { + let stream = Box::new(std::io::Cursor::new(Bytes::from(content))); + self.upsert_file_stream(name, stream, content_type).await + } + + #[tracing::instrument(skip(self, stream, content_type), err)] + pub async fn upsert_file_stream( + &self, + name: &str, + mut stream: Box, + content_type: &str, + ) -> Result<(), CacheError> { + if name.starts_with("log/") { + let compressor = self.cfg.log_compression.get_compression_fn( + self.cfg.get_compression_level(), + self.cfg.parallel_compression, + ); + let mut stream = compressor(stream); + self.upload_file( + name, + &mut stream, + content_type, + self.cfg.log_compression.content_encoding(), + ) + .await + } else if std::path::Path::new(name) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("ls")) + { + let compressor = self.cfg.ls_compression.get_compression_fn( + self.cfg.get_compression_level(), + self.cfg.parallel_compression, + ); + let mut stream = compressor(stream); + self.upload_file( + name, + &mut stream, + content_type, + self.cfg.ls_compression.content_encoding(), + ) + .await + } else if std::path::Path::new(name) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("narinfo")) + { + let compressor = self.cfg.narinfo_compression.get_compression_fn( + self.cfg.get_compression_level(), + self.cfg.parallel_compression, + ); + let mut stream = compressor(stream); + self.upload_file( + name, + &mut stream, + content_type, + self.cfg.narinfo_compression.content_encoding(), + ) + .await + } else { + self.upload_file(name, &mut stream, content_type, "").await + } + } + + #[tracing::instrument(skip(self, stream, content_type), err)] + async fn upload_file( + &self, + name: &str, + mut stream: &mut (dyn tokio::io::AsyncRead + Unpin + Send), + content_type: &str, + content_encoding: &str, + ) -> Result<(), CacheError> { + let start = Instant::now(); + let buf = bytes::BytesMut::with_capacity(self.cfg.buffer_size); + let first_chunk = read_chunk_async(&mut stream, buf).await?; + let first_chunk_len = first_chunk.len(); + + if first_chunk_len < self.cfg.buffer_size { + self.s3 + .put_opts( + &object_store::path::Path::from(name), + object_store::PutPayload::from_bytes(first_chunk.clone()), + object_store::PutOptions { + attributes: { + let mut attrs = object_store::Attributes::new(); + attrs.insert( + object_store::Attribute::ContentType, + content_type.to_owned().into(), + ); + if !content_encoding.is_empty() { + attrs.insert( + object_store::Attribute::ContentEncoding, + content_encoding.to_owned().into(), + ); + } + attrs + }, + ..Default::default() + }, + ) + .await?; + + tracing::debug!("put_object for small file -> done, size: {first_chunk_len}",); + + let elapsed = u64::try_from(start.elapsed().as_millis()).unwrap_or_default(); + self.s3_stats + .put_time_ms + .fetch_add(elapsed, Ordering::Relaxed); + self.s3_stats.put.fetch_add(1, Ordering::Relaxed); + self.s3_stats + .put_bytes + .fetch_add(first_chunk_len as u64, Ordering::Relaxed); + + return Ok(()); + } + + tracing::debug!( + "Starting multipart upload for large file, first chunk size: {}", + first_chunk_len + ); + let mut multipart_upload = self + .s3 + .put_multipart_opts( + &object_store::path::Path::from(name), + object_store::PutMultipartOptions { + attributes: { + let mut attrs = object_store::Attributes::new(); + attrs.insert( + object_store::Attribute::ContentType, + content_type.to_owned().into(), + ); + if !content_encoding.is_empty() { + attrs.insert( + object_store::Attribute::ContentEncoding, + content_encoding.to_owned().into(), + ); + } + attrs + }, + ..Default::default() + }, + ) + .await?; + match run_multipart_upload( + &mut multipart_upload, + first_chunk, + stream, + self.cfg.buffer_size, + ) + .await + { + Ok(file_size) => { + let elapsed = u64::try_from(start.elapsed().as_millis()).unwrap_or_default(); + self.s3_stats + .put_time_ms + .fetch_add(elapsed, Ordering::Relaxed); + self.s3_stats.put.fetch_add(1, Ordering::Relaxed); + self.s3_stats + .put_bytes + .fetch_add(file_size as u64, Ordering::Relaxed); + } + Err(e) => { + tracing::warn!("Upload was interrupted - Aborting multipart upload: {e}"); + + if let Err(e) = multipart_upload.abort().await { + tracing::warn!("Failed to abort multipart upload: {e}"); + } + } + } + + Ok(()) + } + + #[tracing::instrument(skip(self, listing), err)] + async fn upload_listing(&self, path: &str, listing: String) -> Result<(), CacheError> { + self.upsert_file(path, listing, "application/json").await?; + Ok(()) + } + + #[tracing::instrument(skip(self, store, narinfo), err)] + async fn upload_narinfo( + &self, + store: &nix_utils::LocalStore, + narinfo: NarInfo, + ) -> Result { + let base = narinfo.store_path.hash_part(); + let info_key = format!("{base}.narinfo"); + self.upsert_file(&info_key, narinfo.render(store)?, "text/x-nix-narinfo") + .await?; + Ok(info_key) + } + + #[tracing::instrument(skip(self, store), fields(%path), err)] + async fn path_to_narinfo( + &self, + store: &nix_utils::LocalStore, + path: &nix_utils::StorePath, + ) -> Result { + let Some(path_info) = store.query_path_info(path).await else { + return Err(CacheError::PathNotFound { + path: path.to_string(), + }); + }; + let narinfo = NarInfo::new( + store, + path, + path_info, + self.cfg.compression, + &self.signing_keys, + ); + let queried_references = store + .query_path_infos(&narinfo.references.iter().collect::>()) + .await; + for r in &narinfo.references { + if !queried_references.contains_key(r) { + return Err(CacheError::ReferenceVerifyError( + narinfo.store_path, + r.to_owned(), + )); + } + } + Ok(narinfo) + } + + #[tracing::instrument(skip(self, store), err)] + pub async fn copy_path( + &self, + store: &nix_utils::LocalStore, + path: &nix_utils::StorePath, + repair: bool, + ) -> Result<(), CacheError> { + if !repair && self.has_narinfo(path).await? { + return Ok(()); + } + + let mut narinfo = self.path_to_narinfo(store, path).await?; + if self.cfg.write_nar_listing { + let ls = store.list_nar(&narinfo.store_path, true).await?; + self.upload_listing(&narinfo.get_ls_path(), ls).await?; + } + + if self.cfg.write_debug_info { + debug_info::process_debug_info(&narinfo.url, store, &narinfo.store_path, self.clone()) + .await?; + } + + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::>(); + let closure = move |data: &[u8]| { + let data = Bytes::copy_from_slice(data); + tx.send(Ok(data)).is_ok() + }; + + tokio::task::spawn({ + let path = narinfo.store_path.clone(); + let store = store.clone(); + async move { + let _ = store.nar_from_path(&path, closure); + } + }); + let stream = tokio_util::io::StreamReader::new( + tokio_stream::wrappers::UnboundedReceiverStream::new(rx), + ); + let compressor = narinfo.compression.get_compression_fn( + self.cfg.get_compression_level(), + self.cfg.parallel_compression, + ); + let compressed_stream = compressor(stream); + let (mut hashing_reader, _) = crate::streaming_hash::HashingReader::new(compressed_stream); + self.upload_file( + &narinfo.url, + &mut hashing_reader, + narinfo.compression.content_type(), + narinfo.compression.content_encoding(), + ) + .await?; + + let (file_hash, file_size) = hashing_reader.finalize()?; + + if let Ok(file_hash) = nix_utils::convert_hash( + &format!("{file_hash:x}"), + Some(nix_utils::HashAlgorithm::SHA256), + nix_utils::HashFormat::Nix32, + ) { + narinfo.file_hash = Some(format!("sha256:{file_hash}")); + narinfo.file_size = Some(file_size as u64); + } + + if self.cfg.write_realisation + && let Some(deriver) = narinfo.deriver.as_ref() + && let Ok(hashes) = store.static_output_hashes(deriver).await + { + for (output_name, drv_hash) in hashes { + self.copy_realisation( + store, + &nix_utils::DrvOutput { + drv_hash, + output_name, + }, + repair, + ) + .await?; + } + } + + self.upload_narinfo(store, narinfo).await?; + + Ok(()) + } + + #[tracing::instrument(skip(self, store, paths), err)] + pub async fn copy_paths( + &self, + store: &nix_utils::LocalStore, + paths: Vec, + repair: bool, + ) -> Result<(), CacheError> { + use futures::stream::StreamExt as _; + + let mut stream = tokio_stream::iter(paths) + .map(|p| async move { + tracing::debug!("copying path {p} to s3 binary cache."); + self.copy_path(store, &p, repair).await + }) + .buffered(10); + + while let Some(v) = tokio_stream::StreamExt::next(&mut stream).await { + v?; + } + + Ok(()) + } + + #[tracing::instrument(skip(self, store, id), err)] + pub async fn copy_realisation( + &self, + store: &nix_utils::LocalStore, + id: &nix_utils::DrvOutput, + repair: bool, + ) -> Result<(), CacheError> { + if !repair && self.has_realisation(id).await? { + return Ok(()); + } + + let mut raw_realisation = store.query_raw_realisation(&id.drv_hash, &id.output_name)?; + if !self.signing_keys.is_empty() { + for s in &self.signing_keys { + raw_realisation.sign(s.expose_secret())?; + } + } + + self.upsert_file( + &format!("realisations/{id}.doi"), + raw_realisation.as_json(), + "application/json", + ) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, realisation), err)] + pub async fn upload_realisation( + &self, + mut realisation: nix_utils::FfiRealisation, + repair: bool, + ) -> Result<(), CacheError> { + let id = realisation.get_id(); + if !repair && self.has_realisation(&id).await? { + return Ok(()); + } + + realisation.clear_signatures(); + if !self.signing_keys.is_empty() { + for s in &self.signing_keys { + realisation.sign(s.expose_secret())?; + } + } + + self.upsert_file( + &format!("realisations/{id}.doi"), + realisation.as_json(), + "application/json", + ) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self), err)] + pub async fn download_narinfo( + &self, + store_path: &nix_utils::StorePath, + ) -> Result, CacheError> { + if let Some(narinfo) = self.narinfo_cache.get(store_path).await { + return Ok(Some(narinfo)); + } + + match self + .get_object(&format!("{}.narinfo", store_path.hash_part())) + .await? + { + Some(v) => { + let narinfo: NarInfo = String::from_utf8_lossy(&v).parse()?; + self.narinfo_cache + .insert(store_path.to_owned(), narinfo.clone()) + .await; + Ok(Some(narinfo)) + } + None => Ok(None), + } + } + + #[tracing::instrument(skip(self), err)] + pub async fn download_nar(&self, nar_url: &str) -> Result, CacheError> { + self.get_object(nar_url).await + } + + #[tracing::instrument(skip(self), err)] + pub async fn has_narinfo(&self, store_path: &nix_utils::StorePath) -> Result { + if self.narinfo_cache.contains_key(store_path) { + return Ok(true); + } + Ok(self.download_narinfo(store_path).await?.is_some()) + } + + #[tracing::instrument(skip(self), err)] + pub async fn download_realisation( + &self, + id: &nix_utils::DrvOutput, + ) -> Result, CacheError> { + (self.get_object(&format!("realisations/{id}.doi")).await?).map_or_else( + || Ok(None), + |v| Ok(Some(String::from_utf8_lossy(&v).to_string())), + ) + } + + #[tracing::instrument(skip(self), err)] + pub async fn has_realisation(&self, id: &nix_utils::DrvOutput) -> Result { + Ok(self.download_realisation(id).await?.is_some()) + } + + #[tracing::instrument(skip(self, paths))] + pub async fn query_missing_paths( + &self, + paths: Vec, + ) -> Vec { + use futures::stream::StreamExt as _; + + tokio_stream::iter(paths) + .map(|p| async move { + if self.has_narinfo(&p).await.unwrap_or_default() { + None + } else { + Some(p) + } + }) + .buffered(50) + .filter_map(|p| async { p }) + .collect() + .await + } + + #[tracing::instrument(skip(self, outputs))] + pub async fn query_missing_remote_outputs( + &self, + outputs: Vec, + ) -> Vec { + use futures::stream::StreamExt as _; + + tokio_stream::iter(outputs) + .map(|o| async move { + let Some(path) = &o.path else { + return None; + }; + if self.has_narinfo(path).await.unwrap_or_default() { + None + } else { + Some(o) + } + }) + .buffered(50) + .filter_map(|o| async { o }) + .collect() + .await + } + + #[tracing::instrument(skip(self), err)] + pub async fn generate_nar_upload_presigned_url( + &self, + path: &nix_utils::StorePath, + nix32_nar_hash: &str, + debug_info_build_ids: Vec, + ) -> Result { + let nar_hash_url = nix32_nar_hash + .strip_prefix("sha256:") + .map_or_else(|| path.hash_part(), |h| h); + + let nar_url = format!("nar/{}.{}", nar_hash_url, self.cfg.compression.ext()); + let url = self + .s3 + .signed_url( + reqwest::Method::PUT, + &object_store::path::Path::from(nar_url.as_str()), + self.cfg.presigned_url_expiry, + ) + .await + .map_err(|e| CacheError::PresignedUrlError { + path: path.to_string(), + reason: format!("Failed to generate presigned URL for NAR: {e}"), + })?; + let ls_upload = if self.cfg.write_nar_listing { + let s3_file_path = format!("{}.ls", path.hash_part()); + Some(PresignedUpload { + url: self + .s3 + .signed_url( + reqwest::Method::PUT, + &object_store::path::Path::from(s3_file_path.as_str()), + self.cfg.presigned_url_expiry, + ) + .await + .map_err(|e| CacheError::PresignedUrlError { + path: s3_file_path.clone(), + reason: format!("Failed to generate presigned URL for listing: {e}"), + })? + .to_string(), + path: s3_file_path, + compression: self.cfg.ls_compression, + compression_level: self.cfg.get_compression_level(), + }) + } else { + None + }; + let debug_info_upload = if self.cfg.write_debug_info && !debug_info_build_ids.is_empty() { + use futures::stream::StreamExt as _; + + let mut o = Vec::with_capacity(debug_info_build_ids.len()); + let mut stream = tokio_stream::iter(debug_info_build_ids) + .map(|build_id| async move { + let s3_file_path = format!("debuginfo/{build_id}"); + // if this request fails, we assume default, which will then override the file + if self.head_object(&s3_file_path).await.unwrap_or_default() { + Ok(None) + } else { + Ok::<_, crate::CacheError>(Some(PresignedUpload { + url: self + .s3 + .signed_url( + reqwest::Method::PUT, + &object_store::path::Path::from(s3_file_path.as_str()), + self.cfg.presigned_url_expiry, + ) + .await? + .to_string(), + path: s3_file_path, + compression: Compression::None, + compression_level: async_compression::Level::Default, + })) + } + }) + .buffered(10); + + while let Some(v) = tokio_stream::StreamExt::next(&mut stream).await { + if let Some(v) = v? { + o.push(v); + } + } + o + } else { + vec![] + }; + + Ok(PresignedUploadResponse { + nar_url: nar_url.clone(), // we could deduplicate this, but its not that big of a deal + nar_upload: PresignedUpload { + path: nar_url, + url: url.to_string(), + compression: self.cfg.compression, + compression_level: self.cfg.get_compression_level(), + }, + ls_upload, + debug_info_upload, + }) + } + + #[tracing::instrument(skip(self, store, narinfo), err)] + pub async fn upload_narinfo_after_presigned_upload( + &self, + store: &nix_utils::LocalStore, + narinfo: NarInfo, + ) -> Result { + if self.cfg.write_nar_listing { + self.head_object(&narinfo.get_ls_path()) + .await? + .then_some(()) + .ok_or(CacheError::PathNotFound { + path: narinfo.get_ls_path(), + })?; + } + self.head_object(&narinfo.url) + .await? + .then_some(()) + .ok_or(CacheError::PathNotFound { + path: narinfo.url.clone(), + })?; + + let narinfo = narinfo.clear_sigs_and_sign(store, &self.signing_keys); + // TODO: we also need to integarte realisation into this! + self.upload_narinfo(store, narinfo).await + } +} + +impl crate::debug_info::DebugInfoClient for S3BinaryCacheClient { + /// Creates debug info links for build IDs found in NAR files. + /// + /// This function processes debug information from NIX store paths that contain + /// debug symbols in the standard `lib/debug/.build-id` directory structure. + /// It creates JSON links that allow debuggers to find debug symbols by build ID. + /// + /// The directory structure expected is: + /// lib/debug/.build-id/ab/cdef1234567890123456789012345678901234.debug + /// where 'ab' are the first 2 hex characters of the build ID and the rest + /// are the remaining 38 characters. + /// + /// Each debug info link contains: + /// ```json + /// { + /// "archive": "../nar-url", + /// "member": "lib/debug/.build-id/ab/cdef1234567890123456789012345678901234.debug" + /// } + /// ``` + #[tracing::instrument(skip(self, nar_url, build_id, debug_path), err)] + async fn create_debug_info_link( + &self, + nar_url: &str, + build_id: String, + debug_path: String, + ) -> Result<(), CacheError> { + let key = format!("debuginfo/{build_id}"); + + if self.head_object(&key).await? { + tracing::debug!("Debuginfo link {} already exists, skipping", key); + return Ok(()); + } + + let json_content = crate::debug_info::DebugInfoLink { + archive: format!("../{nar_url}"), + member: debug_path, + }; + + tracing::debug!("Creating debuginfo link from '{}' to '{}'", key, nar_url); + + self.upsert_file( + &key, + serde_json::to_string(&json_content)?, + "application/json", + ) + .await?; + + Ok(()) + } +} diff --git a/src/crates/binary-cache/src/narinfo.rs b/src/crates/binary-cache/src/narinfo.rs new file mode 100644 index 000000000..344318442 --- /dev/null +++ b/src/crates/binary-cache/src/narinfo.rs @@ -0,0 +1,346 @@ +use std::fmt::Write as _; + +use secrecy::ExposeSecret as _; + +use crate::Compression; + +use nix_utils::BaseStore as _; + +#[derive(Debug, Clone)] +pub struct NarInfo { + pub store_path: nix_utils::StorePath, + pub url: String, + pub compression: Compression, + pub file_hash: Option, + pub file_size: Option, + pub nar_hash: String, + pub nar_size: u64, + pub references: Vec, + pub deriver: Option, + pub ca: Option, + pub sigs: Vec, +} + +impl NarInfo { + #[must_use] + pub fn new( + store: &nix_utils::LocalStore, + path: &nix_utils::StorePath, + path_info: nix_utils::PathInfo, + compression: Compression, + signing_keys: &[secrecy::SecretString], + ) -> Self { + let nar_hash = if path_info.nar_hash.len() == 71 { + nix_utils::convert_hash( + &path_info.nar_hash[7..], + Some(nix_utils::HashAlgorithm::SHA256), + nix_utils::HashFormat::Nix32, + ) + .unwrap_or(path_info.nar_hash) + } else { + path_info.nar_hash + }; + let nar_hash_url = nar_hash + .strip_prefix("sha256:") + .map_or_else(|| path.hash_part(), |h| h); + + let narinfo = Self { + store_path: path.clone(), + url: format!("nar/{}.{}", nar_hash_url, compression.ext()), + compression, + file_hash: None, + file_size: None, + nar_hash, + nar_size: path_info.nar_size, + references: path_info.refs, + deriver: path_info.deriver, + ca: path_info.ca.clone(), + sigs: vec![], + }; + + let mut narinfo = narinfo.clear_sigs_and_sign(store, signing_keys); + if narinfo.sigs.is_empty() && !path_info.sigs.is_empty() { + narinfo.sigs = path_info.sigs; + } + + narinfo + } + + #[must_use] + pub fn simple( + path: &nix_utils::StorePath, + path_info: nix_utils::PathInfo, + compression: Compression, + ) -> Self { + let nar_hash = if path_info.nar_hash.len() == 71 { + nix_utils::convert_hash( + &path_info.nar_hash[7..], + Some(nix_utils::HashAlgorithm::SHA256), + nix_utils::HashFormat::Nix32, + ) + .unwrap_or(path_info.nar_hash) + } else { + path_info.nar_hash + }; + let nar_hash_url = nar_hash + .strip_prefix("sha256:") + .map_or_else(|| path.hash_part(), |h| h); + + Self { + store_path: path.clone(), + url: format!("nar/{}.{}", nar_hash_url, compression.ext()), + compression, + file_hash: None, + file_size: None, + nar_hash, + nar_size: path_info.nar_size, + references: path_info.refs, + deriver: path_info.deriver, + ca: path_info.ca, + sigs: vec![], + } + } + + #[must_use] + pub fn clear_sigs_and_sign( + mut self, + store: &nix_utils::LocalStore, + signing_keys: &[secrecy::SecretString], + ) -> Self { + self.sigs.clear(); // if we call this sign, we dont trust the signatures + if !signing_keys.is_empty() + && let Some(fp) = self.fingerprint_path(store) + { + for s in signing_keys { + self.sigs + .push(nix_utils::sign_string(s.expose_secret(), &fp)); + } + } + self + } + + #[must_use] + fn fingerprint_path(&self, store: &nix_utils::LocalStore) -> Option { + let root_store_dir = nix_utils::get_store_dir(); + let abs_path = store.print_store_path(&self.store_path); + + if abs_path[0..root_store_dir.len()] != root_store_dir || &self.nar_hash[0..7] != "sha256:" + { + return None; + } + + if self.nar_hash.len() != 59 { + return None; + } + + let refs = self + .references + .iter() + .map(|r| store.print_store_path(r)) + .collect::>(); + for r in &refs { + if r[0..root_store_dir.len()] != root_store_dir { + return None; + } + } + + Some(format!( + "1;{};{};{};{}", + abs_path, + self.nar_hash, + self.nar_size, + refs.join(",") + )) + } + + #[must_use] + pub fn get_ls_path(&self) -> String { + format!("{}.ls", self.store_path.hash_part()) + } + + pub fn render(&self, store: &nix_utils::LocalStore) -> Result { + let mut o = String::with_capacity(200); + writeln!(o, "StorePath: {}", store.print_store_path(&self.store_path))?; + writeln!(o, "URL: {}", self.url)?; + writeln!(o, "Compression: {}", self.compression.as_str())?; + if let Some(h) = &self.file_hash { + writeln!(o, "FileHash: {h}")?; + } + if let Some(s) = self.file_size { + writeln!(o, "FileSize: {s}")?; + } + writeln!(o, "NarHash: {}", self.nar_hash)?; + writeln!(o, "NarSize: {}", self.nar_size)?; + + writeln!( + o, + "References: {}", + self.references + .iter() + .map(nix_utils::StorePath::base_name) + .collect::>() + .join(" ") + )?; + + if let Some(d) = &self.deriver { + writeln!(o, "Deriver: {}", d.base_name())?; + } + if let Some(ca) = &self.ca { + writeln!(o, "CA: {ca}")?; + } + + for sig in &self.sigs { + writeln!(o, "Sig: {sig}")?; + } + Ok(o) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum NarInfoError { + #[error("missing required field: {0}")] + MissingField(&'static str), + #[error("invalid value for {field}: {value}")] + InvalidField { field: String, value: String }, + #[error("parse error on line {line}: {reason}")] + Line { line: usize, reason: String }, + #[error("integer parse error for {field}: {err}")] + Int { + field: &'static str, + err: std::num::ParseIntError, + }, +} + +impl std::str::FromStr for NarInfo { + type Err = NarInfoError; + + #[tracing::instrument(skip(input), err)] + #[allow(clippy::too_many_lines)] + fn from_str(input: &str) -> Result { + let mut out = Self { + store_path: nix_utils::StorePath::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bla"), + url: String::new(), + compression: Compression::None, + file_hash: None, + file_size: None, + nar_hash: String::new(), + nar_size: 0, + references: vec![], + deriver: None, + ca: None, + sigs: vec![], + }; + + // Temporaries to know what was present + let mut have_store_path = false; + let mut have_url = false; + let mut have_compression = false; + let mut have_nar_hash = false; + let mut have_nar_size = false; + + for (idx, raw_line) in input.lines().enumerate() { + let line_no = idx + 1; + let line = raw_line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + + let Some((k, v)) = line.split_once(':') else { + return Err(NarInfoError::Line { + line: line_no, + reason: "expected `Key: value`".into(), + }); + }; + let key = k.trim(); + let val = v + .strip_prefix(' ') + .map_or(v, |stripped| stripped) + .trim_end(); + + match key { + "StorePath" => { + out.store_path = nix_utils::StorePath::new(val); + have_store_path = true; + } + "URL" => { + out.url = val.to_string(); + have_url = true; + } + "Compression" => { + out.compression = val.parse().map_err(|e| NarInfoError::InvalidField { + field: "Compression".into(), + value: e, + })?; + have_compression = true; + } + "FileHash" => { + out.file_hash = Some(val.to_string()); + } + "FileSize" => { + out.file_size = Some(val.parse::().map_err(|e| NarInfoError::Int { + field: "FileSize", + err: e, + })?); + } + "NarHash" => { + out.nar_hash = val.to_string(); + have_nar_hash = true; + } + "NarSize" => { + out.nar_size = val.parse::().map_err(|e| NarInfoError::Int { + field: "NarSize", + err: e, + })?; + have_nar_size = true; + } + "References" => { + let refs = val + .split_whitespace() + .filter(|s| !s.is_empty()) + .map(nix_utils::StorePath::new) + .collect::>(); + out.references = refs; + } + "Deriver" => { + out.deriver = if val.is_empty() { + None + } else { + Some(nix_utils::StorePath::new(val)) + }; + } + "CA" => { + out.ca = if val.is_empty() { + None + } else { + Some(val.to_string()) + }; + } + "Sig" => { + if !val.is_empty() { + out.sigs.push(val.to_string()); + } + } + _ => {} + } + } + + // Validate requireds + if !have_store_path { + return Err(NarInfoError::MissingField("StorePath")); + } + if !have_url { + return Err(NarInfoError::MissingField("URL")); + } + if !have_compression { + return Err(NarInfoError::MissingField("Compression")); + } + if !have_nar_hash { + return Err(NarInfoError::MissingField("NarHash")); + } + if !have_nar_size { + return Err(NarInfoError::MissingField("NarSize")); + } + + Ok(out) + } +} diff --git a/src/crates/binary-cache/src/presigned.rs b/src/crates/binary-cache/src/presigned.rs new file mode 100644 index 000000000..bfe5d792e --- /dev/null +++ b/src/crates/binary-cache/src/presigned.rs @@ -0,0 +1,375 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use backon::Retryable; + +use bytes::Bytes; +use nix_utils::{BaseStore as _, LocalStore}; + +use tokio_util::io::StreamReader; + +use crate::{CacheError, Compression, streaming_hash::HashingReader}; + +const RETRY_MIN_DELAY_SECS: u64 = 1; +const RETRY_MAX_DELAY_SECS: u64 = 30; +const RETRY_MAX_ATTEMPTS: usize = 3; + +#[derive(Debug, Clone)] +pub struct PresignedUpload { + pub path: String, + pub url: String, + pub compression: Compression, + pub compression_level: async_compression::Level, +} + +impl PresignedUpload { + #[must_use] + pub fn new( + path: String, + url: String, + compression: Compression, + compression_level: i32, + ) -> Self { + Self { + path, + url, + compression, + compression_level: match compression_level { + 1 => async_compression::Level::Fastest, + l if (2..=8).contains(&l) => async_compression::Level::Precise(l), + 9 => async_compression::Level::Best, + _ => async_compression::Level::Default, + }, + } + } + + #[must_use] + pub const fn get_compression_level_as_i32(&self) -> i32 { + match self.compression_level { + async_compression::Level::Fastest => 1, + async_compression::Level::Precise(n) => n, + async_compression::Level::Best => 9, + async_compression::Level::Default | _ => 0, + } + } +} + +#[derive(Debug, Clone)] +pub struct PresignedUploadResponse { + pub nar_url: String, + pub nar_upload: PresignedUpload, + pub ls_upload: Option, + pub debug_info_upload: Vec, +} + +#[derive(Debug, Default)] +pub struct AtomicPresignedUploadMetrics { + pub put: AtomicU64, + pub put_bytes: AtomicU64, + pub put_time_ms: AtomicU64, +} + +#[derive(Debug, Default)] +pub struct PresignedUploadMetrics { + pub put: u64, + pub put_bytes: u64, + pub put_time_ms: u64, +} + +#[derive(Debug, Clone)] +pub struct PresignedUploadResult { + pub file_hash: String, + pub file_size: u64, +} + +#[derive(Debug, Clone)] +pub struct PresignedUploadClient { + client: reqwest::Client, + metrics: std::sync::Arc, +} + +impl PresignedUploadClient { + #[must_use] + pub fn new() -> Self { + Self { + client: reqwest::Client::new(), + metrics: std::sync::Arc::new(AtomicPresignedUploadMetrics::default()), + } + } + + #[must_use] + pub fn metrics(&self) -> PresignedUploadMetrics { + PresignedUploadMetrics { + put: self.metrics.put.load(Ordering::Relaxed), + put_bytes: self.metrics.put_bytes.load(Ordering::Relaxed), + put_time_ms: self.metrics.put_time_ms.load(Ordering::Relaxed), + } + } + + #[tracing::instrument(skip(self, store, narinfo, req), err)] + pub async fn process_presigned_request( + &self, + store: &LocalStore, + mut narinfo: crate::NarInfo, + req: PresignedUploadResponse, + ) -> Result { + narinfo.url = req.nar_url; + narinfo.compression = req.nar_upload.compression; + + if let Some(ls_upload) = req.ls_upload { + let _ = self + .upload_ls(store, &narinfo.store_path, &ls_upload) + .await?; + } + + if !req.debug_info_upload.is_empty() { + let debug_info_client = PresignedDebugInfoUpload { + client: self.clone(), + debug_info_urls: std::sync::Arc::new(req.debug_info_upload), + }; + crate::debug_info::process_debug_info( + &narinfo.url, + store, + &narinfo.store_path, + debug_info_client.clone(), + ) + .await?; + } + + let upload_res = self + .upload_nar(store, &narinfo.store_path, &req.nar_upload) + .await?; + narinfo.file_hash = Some(upload_res.file_hash); + narinfo.file_size = Some(upload_res.file_size); + + Ok(narinfo) + } + + #[tracing::instrument(skip(self, store, store_path), err)] + async fn upload_nar( + &self, + store: &LocalStore, + store_path: &nix_utils::StorePath, + upload: &PresignedUpload, + ) -> Result { + let start = std::time::Instant::now(); + + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::>(); + let (result_tx, result_rx) = tokio::sync::oneshot::channel::>(); + + let closure = { + let tx = tx.clone(); + move |data: &[u8]| { + let data = Bytes::copy_from_slice(data); + tx.send(Ok(data)).is_ok() + } + }; + + tokio::task::spawn({ + let path = store_path.clone(); + let store = store.clone(); + async move { + let result = store + .nar_from_path(&path, closure) + .map_err(|e| format!("NAR reading failed: {e}")); + let _ = result_tx.send(result); + } + }); + + drop(tx); + let stream = StreamReader::new(tokio_stream::wrappers::UnboundedReceiverStream::new(rx)); + let compressor = upload + .compression + .get_compression_fn(upload.compression_level, false); + let compressed_stream = compressor(stream); + let (hashing_reader, _) = HashingReader::new(compressed_stream); + + let upload_result = self.upload_any(upload, hashing_reader, start, None).await; + + match result_rx.await { + Ok(Ok(())) => upload_result, + Ok(Err(e)) => Err(CacheError::UploadError { + path: upload.path.clone(), + reason: e, + }), + Err(_) => Err(CacheError::UploadError { + path: upload.path.clone(), + reason: "NAR reading task was cancelled or panicked".to_string(), + }), + } + } + + #[tracing::instrument(skip(self, store, store_path), err)] + async fn upload_ls( + &self, + store: &LocalStore, + store_path: &nix_utils::StorePath, + upload: &PresignedUpload, + ) -> Result { + let start = std::time::Instant::now(); + + let ls = store.list_nar(store_path, true).await?; + let stream = Box::new(std::io::Cursor::new(Bytes::from(ls))); + let compressor = upload + .compression + .get_compression_fn(upload.compression_level, false); + let compressed_stream = compressor(stream); + let (hashing_reader, _) = HashingReader::new(compressed_stream); + + self.upload_any(upload, hashing_reader, start, Some("application/json")) + .await + } + + #[tracing::instrument(skip(self, content), err)] + async fn upload_json( + &self, + content: String, + upload: &PresignedUpload, + ) -> Result { + let start = std::time::Instant::now(); + + let stream = Box::new(std::io::Cursor::new(Bytes::from(content))); + let compressor = upload + .compression + .get_compression_fn(upload.compression_level, false); + let compressed_stream = compressor(stream); + let (hashing_reader, _) = HashingReader::new(compressed_stream); + + self.upload_any(upload, hashing_reader, start, Some("application/json")) + .await + } + + #[tracing::instrument(skip(self, start, reader), err)] + async fn upload_any( + &self, + upload: &PresignedUpload, + mut reader: HashingReader>, + start: std::time::Instant, + content_type: Option<&str>, + ) -> Result { + use tokio::io::AsyncReadExt as _; + + let mut request = self.client.put(&upload.url); + if let Some(content_type) = content_type { + request = request.header("Content-Type", content_type); + } else { + request = request.header("Content-Type", upload.compression.content_type()); + } + if !upload.compression.content_encoding().is_empty() { + request = request.header("Content-Encoding", upload.compression.content_encoding()); + } + + // TODO: We need multipart signed urls to fix this! + // object_store currently doesnt have support for this. + let mut buffer = Vec::new(); + reader.read_to_end(&mut buffer).await?; + + let _response = (|| async { + Ok::<_, CacheError>( + request + .try_clone() + .ok_or_else(|| CacheError::RequestCloneError)? + .body(buffer.clone()) + .send() + .await? + .error_for_status()?, + ) + }) + .retry( + &backon::ExponentialBuilder::default() + .with_min_delay(std::time::Duration::from_secs(RETRY_MIN_DELAY_SECS)) + .with_max_delay(std::time::Duration::from_secs(RETRY_MAX_DELAY_SECS)) + .with_max_times(RETRY_MAX_ATTEMPTS), + ) + .await?; + + let elapsed = u64::try_from(start.elapsed().as_millis()).unwrap_or_default(); + + let (file_hash, file_size) = reader.finalize()?; + + let file_hash = nix_utils::convert_hash( + &format!("{file_hash:x}"), + Some(nix_utils::HashAlgorithm::SHA256), + nix_utils::HashFormat::Nix32, + ) + .map_or_else( + |_| format!("sha256:{file_hash:x}"), + |converted_hash| format!("sha256:{converted_hash}"), + ); + + // Update metrics + self.metrics + .put_bytes + .fetch_add(file_size as u64, Ordering::Relaxed); + self.metrics + .put_time_ms + .fetch_add(elapsed, Ordering::Relaxed); + self.metrics.put.fetch_add(1, Ordering::Relaxed); + + Ok(PresignedUploadResult { + file_hash, + file_size: file_size as u64, + }) + } +} + +impl Default for PresignedUploadClient { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Clone)] +struct PresignedDebugInfoUpload { + client: PresignedUploadClient, + debug_info_urls: std::sync::Arc>, +} + +impl crate::debug_info::DebugInfoClient for PresignedDebugInfoUpload { + /// Creates debug info links for build IDs found in NAR files. + /// + /// This function processes debug information from NIX store paths that contain + /// debug symbols in the standard `lib/debug/.build-id` directory structure. + /// It creates JSON links that allow debuggers to find debug symbols by build ID. + /// + /// The directory structure expected is: + /// lib/debug/.build-id/ab/cdef1234567890123456789012345678901234.debug + /// where 'ab' are the first 2 hex characters of the build ID and the rest + /// are the remaining 38 characters. + /// + /// Each debug info link contains: + /// ```json + /// { + /// "archive": "../nar-url", + /// "member": "lib/debug/.build-id/ab/cdef1234567890123456789012345678901234.debug" + /// } + /// ``` + #[tracing::instrument(skip(self, nar_url, build_id, debug_path), err)] + async fn create_debug_info_link( + &self, + nar_url: &str, + build_id: String, + debug_path: String, + ) -> Result<(), CacheError> { + let key = format!("debuginfo/{build_id}"); + let upload = self + .debug_info_urls + .iter() + .find(|presigned| presigned.path == key) + .ok_or(CacheError::UploadError { + path: key.clone(), + reason: format!("Presigned URL not found for build ID: {build_id}"), + })?; + + let json_content = crate::debug_info::DebugInfoLink { + archive: format!("../{nar_url}"), + member: debug_path, + }; + + tracing::debug!("Creating debuginfo link from '{}' to '{}'", key, nar_url); + self.client + .upload_json(serde_json::to_string(&json_content)?, upload) + .await?; + + Ok(()) + } +} diff --git a/src/crates/binary-cache/src/streaming_hash.rs b/src/crates/binary-cache/src/streaming_hash.rs new file mode 100644 index 000000000..22e659af6 --- /dev/null +++ b/src/crates/binary-cache/src/streaming_hash.rs @@ -0,0 +1,317 @@ +use sha2::{Digest as _, Sha256}; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use tokio::io::{AsyncRead, ReadBuf}; +use tokio::sync::OnceCell; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Cannot finalize: stream has not been completed")] + NotCompleted, +} + +#[derive(Debug, Clone)] +pub struct HashResult { + inner: Arc, usize)>>, +} + +impl HashResult { + fn new() -> Self { + Self { + inner: Arc::new(OnceCell::new()), + } + } + + pub fn get(&self) -> Result<(sha2::digest::Output, usize), Error> { + Ok(self.inner.get().ok_or(Error::NotCompleted)?.to_owned()) + } +} + +#[derive(Debug)] +pub struct HashingReader { + inner: R, + hasher: Option, + size: usize, + hash_result: HashResult, +} + +impl Unpin for HashingReader where R: Unpin {} + +impl HashingReader { + pub fn new(reader: R) -> (Self, HashResult) { + let res = HashResult::new(); + ( + Self { + inner: reader, + hasher: Some(Sha256::new()), + size: 0, + hash_result: res.clone(), + }, + res, + ) + } + + pub fn finalize(&self) -> Result<(sha2::digest::Output, usize), Error> { + self.hash_result.get() + } +} + +impl AsyncRead for HashingReader { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let filled_len = buf.filled().len(); + match Pin::new(&mut self.inner).poll_read(cx, buf) { + Poll::Ready(Ok(())) => { + let new_data = &buf.filled()[filled_len..]; + if !new_data.is_empty() { + if let Some(hasher) = self.hasher.as_mut() { + hasher.update(new_data); + } + self.size += new_data.len(); + } + + if new_data.is_empty() + && let Some(hasher) = self.hasher.take() + { + let _ = self.hash_result.inner.set((hasher.finalize(), self.size)); + } + Poll::Ready(Ok(())) + } + Poll::Ready(Err(e)) => Poll::Ready(Err(e)), + Poll::Pending => Poll::Pending, + } + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use super::*; + use bytes::Bytes; + use sha2::{Digest, Sha256}; + use std::io::Cursor; + use tokio::io::AsyncReadExt; + + #[tokio::test] + async fn test_hashing_reader_empty_data() { + let data = b""; + let cursor = Cursor::new(data.to_vec()); + let (mut hashing_reader, _) = HashingReader::new(cursor); + + let mut buffer = Vec::new(); + let bytes_read = hashing_reader.read_to_end(&mut buffer).await.unwrap(); + + assert_eq!(bytes_read, 0); + assert_eq!(buffer, b""); + + let (hash, size) = hashing_reader.finalize().unwrap(); + assert_eq!(size, 0); + + // Verify against direct computation + let mut direct_hasher = Sha256::new(); + direct_hasher.update(data); + let expected_hash = direct_hasher.finalize(); + assert_eq!(hash, expected_hash); + } + + #[tokio::test] + async fn test_hashing_reader_small_data() { + let data = b"Hello, world!"; + let cursor = Cursor::new(data.to_vec()); + let (mut hashing_reader, _) = HashingReader::new(cursor); + + let mut buffer = Vec::new(); + let bytes_read = hashing_reader.read_to_end(&mut buffer).await.unwrap(); + + assert_eq!(bytes_read, data.len()); + assert_eq!(buffer, data); + + let (hash, size) = hashing_reader.finalize().unwrap(); + assert_eq!(size, data.len()); + + // Verify against direct computation + let mut direct_hasher = Sha256::new(); + direct_hasher.update(data); + let expected_hash = direct_hasher.finalize(); + assert_eq!(hash, expected_hash); + } + + #[tokio::test] + async fn test_hashing_reader_large_data() { + // Create 1MB of test data + let data = vec![0x42u8; 1024 * 1024]; + let cursor = Cursor::new(data.clone()); + let (mut hashing_reader, _) = HashingReader::new(cursor); + + let mut buffer = Vec::new(); + let bytes_read = hashing_reader.read_to_end(&mut buffer).await.unwrap(); + + assert_eq!(bytes_read, data.len()); + assert_eq!(buffer, data); + + let (hash, size) = hashing_reader.finalize().unwrap(); + assert_eq!(size, data.len()); + + // Verify against direct computation + let mut direct_hasher = Sha256::new(); + direct_hasher.update(&data); + let expected_hash = direct_hasher.finalize(); + assert_eq!(hash, expected_hash); + } + + #[tokio::test] + async fn test_hashing_reader_partial_reads() { + let data = b"The quick brown fox jumps over the lazy dog"; + let cursor = Cursor::new(data.to_vec()); + let (mut hashing_reader, _) = HashingReader::new(cursor); + + let mut buffer = [0u8; 10]; + let mut total_read = 0; + let mut all_data = Vec::new(); + + // Read in chunks + loop { + let bytes_read = hashing_reader.read(&mut buffer).await.unwrap(); + if bytes_read == 0 { + break; + } + total_read += bytes_read; + all_data.extend_from_slice(&buffer[..bytes_read]); + } + + assert_eq!(total_read, data.len()); + assert_eq!(all_data, data); + + let (hash, size) = hashing_reader.finalize().unwrap(); + assert_eq!(size, data.len()); + + // Verify against direct computation + let mut direct_hasher = Sha256::new(); + direct_hasher.update(data); + let expected_hash = direct_hasher.finalize(); + assert_eq!(hash, expected_hash); + } + + #[tokio::test] + async fn test_hashing_reader_exact_buffer_size() { + let data = b"Exactly 20 bytes!!"; + let cursor = Cursor::new(data.to_vec()); + let (mut hashing_reader, _) = HashingReader::new(cursor); + + let mut buffer = [0u8; 20]; + let bytes_read = hashing_reader.read(&mut buffer).await.unwrap(); + + assert_eq!(bytes_read, data.len()); + assert_eq!(&buffer[..bytes_read], data); + + // Try to read more - should get EOF + let mut buffer2 = [0u8; 10]; + let bytes_read2 = hashing_reader.read(&mut buffer2).await.unwrap(); + assert_eq!(bytes_read2, 0); + + let (hash, size) = hashing_reader.finalize().unwrap(); + assert_eq!(size, data.len()); + + // Verify against direct computation + let mut direct_hasher = Sha256::new(); + direct_hasher.update(data); + let expected_hash = direct_hasher.finalize(); + assert_eq!(hash, expected_hash); + } + + #[tokio::test] + async fn test_hashing_reader_different_data_patterns() { + let test_cases = vec![ + vec![0u8; 100], // All zeros + vec![0xFFu8; 100], // All 0xFF + (0..=255).collect::>(), // All byte values + vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10], // Small sequence + ]; + + for data in test_cases { + let cursor = Cursor::new(data.clone()); + let (mut hashing_reader, _) = HashingReader::new(cursor); + + let mut buffer = Vec::new(); + hashing_reader.read_to_end(&mut buffer).await.unwrap(); + + let (hash, size) = hashing_reader.finalize().unwrap(); + assert_eq!(size, data.len()); + assert_eq!(buffer, data); + + // Verify against direct computation + let mut direct_hasher = Sha256::new(); + direct_hasher.update(&data); + let expected_hash = direct_hasher.finalize(); + assert_eq!(hash, expected_hash); + } + } + + #[tokio::test] + async fn test_hashing_reader_with_async_stream() { + use tokio_stream::wrappers::ReceiverStream; + + let (tx, rx) = tokio::sync::mpsc::channel::>(10); + let data = b"Async stream test data"; + + // Spawn a task to send data + let data_clone = data.to_vec(); + tokio::spawn(async move { + for chunk in data_clone.chunks(5) { + tx.send(Ok(Bytes::copy_from_slice(chunk))).await.unwrap(); + } + }); + + let stream = ReceiverStream::new(rx); + let reader = tokio_util::io::StreamReader::new(stream); + let (mut hashing_reader, _) = HashingReader::new(reader); + + let mut buffer = Vec::new(); + hashing_reader.read_to_end(&mut buffer).await.unwrap(); + + assert_eq!(buffer, data); + + let (hash, size) = hashing_reader.finalize().unwrap(); + assert_eq!(size, data.len()); + + // Verify against direct computation + let mut direct_hasher = Sha256::new(); + direct_hasher.update(data); + let expected_hash = direct_hasher.finalize(); + assert_eq!(hash, expected_hash); + } + + #[tokio::test] + async fn test_hashing_reader_finalize_before_completion() { + let data = b"Hello, world!"; + let cursor = Cursor::new(data.to_vec()); + let (hashing_reader, _) = HashingReader::new(cursor); + + // Try to finalize before reading any data + let result = hashing_reader.finalize(); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), Error::NotCompleted)); + } + + #[tokio::test] + async fn test_hashing_reader_finalize_partial_read() { + let data = b"Hello, world!"; + let cursor = Cursor::new(data.to_vec()); + let (mut hashing_reader, _) = HashingReader::new(cursor); + + // Read only part of the data + let mut buffer = [0u8; 5]; + let bytes_read = hashing_reader.read(&mut buffer).await.unwrap(); + assert_eq!(bytes_read, 5); + + // Try to finalize before reading all data + let result = hashing_reader.finalize(); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), Error::NotCompleted)); + } +} diff --git a/src/crates/db/Cargo.toml b/src/crates/db/Cargo.toml new file mode 100644 index 000000000..f2e3f0dd1 --- /dev/null +++ b/src/crates/db/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "db" +version = "0.1.0" +edition = "2024" +license = "GPL-3.0" +rust-version.workspace = true + +[dependencies] +tracing.workspace = true +anyhow.workspace = true +futures.workspace = true +hashbrown.workspace = true + +serde_json.workspace = true +jiff.workspace = true + +sqlx = { workspace = true, features = [ + "runtime-tokio", + "tls-rustls-ring-webpki", + "postgres", +] } diff --git a/src/crates/db/src/connection.rs b/src/crates/db/src/connection.rs new file mode 100644 index 000000000..ec49dd20e --- /dev/null +++ b/src/crates/db/src/connection.rs @@ -0,0 +1,988 @@ +use sqlx::Acquire; + +use super::models::{ + Build, BuildSmall, BuildStatus, BuildSteps, InsertBuildMetric, InsertBuildProduct, + InsertBuildStep, InsertBuildStepOutput, Jobset, UpdateBuild, UpdateBuildStep, + UpdateBuildStepInFinish, +}; + +pub struct Connection { + conn: sqlx::pool::PoolConnection, +} + +pub struct Transaction<'a> { + tx: sqlx::PgTransaction<'a>, +} + +impl Connection { + #[must_use] + pub const fn new(conn: sqlx::pool::PoolConnection) -> Self { + Self { conn } + } + + #[tracing::instrument(skip(self), err)] + pub async fn begin_transaction(&mut self) -> sqlx::Result> { + let tx = self.conn.begin().await?; + Ok(Transaction { tx }) + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_not_finished_builds_fast(&mut self) -> sqlx::Result> { + sqlx::query_as!( + BuildSmall, + r#" + SELECT + id, + globalPriority + FROM builds + WHERE finished = 0;"# + ) + .fetch_all(&mut *self.conn) + .await + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_not_finished_builds(&mut self) -> sqlx::Result> { + sqlx::query_as!( + Build, + r#" + SELECT + builds.id, + builds.jobset_id, + jobsets.project as project, + jobsets.name as jobset, + job, + drvPath, + maxsilent, + timeout, + timestamp, + globalPriority, + priority + FROM builds + INNER JOIN jobsets ON builds.jobset_id = jobsets.id + WHERE finished = 0 ORDER BY globalPriority desc, schedulingshares, random();"# + ) + .fetch_all(&mut *self.conn) + .await + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_jobsets(&mut self) -> sqlx::Result> { + sqlx::query_as!( + Jobset, + r#" + SELECT + project, + name, + schedulingshares + FROM jobsets"# + ) + .fetch_all(&mut *self.conn) + .await + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_jobset_scheduling_shares( + &mut self, + jobset_id: i32, + ) -> sqlx::Result> { + Ok(sqlx::query!( + "SELECT schedulingshares FROM jobsets WHERE id = $1", + jobset_id, + ) + .fetch_optional(&mut *self.conn) + .await? + .map(|v| v.schedulingshares)) + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_jobset_build_steps( + &mut self, + jobset_id: i32, + scheduling_window: i64, + ) -> sqlx::Result> { + #[allow(clippy::cast_precision_loss)] + sqlx::query_as!( + BuildSteps, + r#" + SELECT s.startTime, s.stopTime FROM buildsteps s join builds b on build = id + WHERE + s.startTime IS NOT NULL AND + to_timestamp(s.stopTime) > (NOW() - (interval '1 second' * $1)) AND + jobset_id = $2 + "#, + Some((scheduling_window * 10) as f64), + jobset_id, + ) + .fetch_all(&mut *self.conn) + .await + } + + #[tracing::instrument(skip(self), err)] + pub async fn abort_build(&mut self, build_id: i32) -> sqlx::Result<()> { + #[allow(clippy::cast_possible_truncation)] + sqlx::query!( + "UPDATE builds SET finished = 1, buildStatus = $2, startTime = $3, stopTime = $3 where id = $1 and finished = 0", + build_id, + BuildStatus::Aborted as i32, + // TODO migrate to 64bit timestamp + jiff::Timestamp::now().as_second() as i32, + ) + .execute(&mut *self.conn) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, paths), err)] + pub async fn check_if_paths_failed(&mut self, paths: &[String]) -> sqlx::Result { + Ok( + !sqlx::query!("SELECT path FROM failedpaths where path = ANY($1)", paths) + .fetch_all(&mut *self.conn) + .await? + .is_empty(), + ) + } + + #[tracing::instrument(skip(self), err)] + pub async fn clear_busy(&mut self, stop_time: i32) -> sqlx::Result<()> { + sqlx::query!( + "UPDATE buildsteps SET busy = 0, status = $1, stopTime = $2 WHERE busy != 0;", + BuildStatus::Aborted as i32, + Some(stop_time), + ) + .execute(&mut *self.conn) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, step), err)] + pub async fn update_build_step(&mut self, step: UpdateBuildStep) -> sqlx::Result<()> { + sqlx::query!( + "UPDATE buildsteps SET busy = $1 WHERE build = $2 AND stepnr = $3 AND busy != 0 AND status IS NULL", + step.status as i32, + step.build_id, + step.step_nr, + ) + .execute(&mut *self.conn) + .await?; + Ok(()) + } + + pub async fn insert_debug_build( + &mut self, + jobset_id: i32, + drv_path: &str, + system: &str, + ) -> sqlx::Result<()> { + sqlx::query!( + r#"INSERT INTO builds ( + finished, + timestamp, + jobset_id, + job, + nixname, + drvpath, + system, + maxsilent, + timeout, + ischannel, + iscurrent, + priority, + globalpriority, + keep + ) VALUES ( + 0, + EXTRACT(EPOCH FROM NOW())::INT4, + $1, + 'debug', + 'debug', + $2, + $3, + 7200, + 36000, + 0, + 0, + 100, + 0, + 0);"#, + jobset_id, + drv_path, + system, + ) + .execute(&mut *self.conn) + .await?; + Ok(()) + } + + pub async fn get_build_output_for_path( + &mut self, + out_path: &str, + ) -> sqlx::Result> { + sqlx::query_as!( + super::models::BuildOutput, + r#" + SELECT + id, buildStatus, releaseName, closureSize, size + FROM builds b + JOIN buildoutputs o on b.id = o.build + WHERE finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1;"#, + out_path, + ) + .fetch_optional(&mut *self.conn) + .await + } + + pub async fn get_build_products_for_build_id( + &mut self, + build_id: i32, + ) -> sqlx::Result> { + sqlx::query_as!( + super::models::OwnedBuildProduct, + r#" + SELECT + type, + subtype, + fileSize, + sha256hash, + path, + name, + defaultPath + FROM buildproducts + WHERE build = $1 ORDER BY productnr;"#, + build_id + ) + .fetch_all(&mut *self.conn) + .await + } + + pub async fn get_build_metrics_for_build_id( + &mut self, + build_id: i32, + ) -> sqlx::Result> { + sqlx::query_as!( + crate::models::OwnedBuildMetric, + r#" + SELECT + name, unit, value + FROM buildmetrics + WHERE build = $1;"#, + build_id + ) + .fetch_all(&mut *self.conn) + .await + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_status(&mut self) -> sqlx::Result> { + Ok( + sqlx::query!("SELECT status FROM systemstatus WHERE what = 'queue-runner';",) + .fetch_optional(&mut *self.conn) + .await? + .map(|v| v.status), + ) + } +} + +impl Transaction<'_> { + #[tracing::instrument(skip(self), err)] + pub async fn commit(self) -> sqlx::Result<()> { + self.tx.commit().await + } + + #[tracing::instrument(skip(self, v), err)] + pub async fn update_build(&mut self, build_id: i32, v: UpdateBuild<'_>) -> sqlx::Result<()> { + sqlx::query!( + r#" + UPDATE builds SET + finished = 1, + buildStatus = $2, + startTime = $3, + stopTime = $4, + size = $5, + closureSize = $6, + releaseName = $7, + isCachedBuild = $8, + notificationPendingSince = $4 + WHERE + id = $1"#, + build_id, + v.status as i32, + v.start_time, + v.stop_time, + v.size, + v.closure_size, + v.release_name, + i32::from(v.is_cached_build), + ) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, status, start_time, stop_time, is_cached_build), err)] + pub async fn update_build_after_failure( + &mut self, + build_id: i32, + status: BuildStatus, + start_time: i32, + stop_time: i32, + is_cached_build: bool, + ) -> sqlx::Result<()> { + sqlx::query!( + r#" + UPDATE builds SET + finished = 1, + buildStatus = $2, + startTime = $3, + stopTime = $4, + isCachedBuild = $5, + notificationPendingSince = $4 + WHERE + id = $1 AND finished = 0"#, + build_id, + status as i32, + start_time, + stop_time, + i32::from(is_cached_build), + ) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, status), err)] + pub async fn update_build_after_previous_failure( + &mut self, + build_id: i32, + status: BuildStatus, + ) -> sqlx::Result<()> { + #[allow(clippy::cast_possible_truncation)] + sqlx::query!( + r#" + UPDATE builds SET + finished = 1, + buildStatus = $2, + startTime = $3, + stopTime = $3, + isCachedBuild = 1, + notificationPendingSince = $3 + WHERE + id = $1 AND finished = 0"#, + build_id, + status as i32, + // TODO migrate to 64bit timestamp + jiff::Timestamp::now().as_second() as i32, + ) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, name, path), err)] + pub async fn update_build_output( + &mut self, + build_id: i32, + name: &str, + path: &str, + ) -> sqlx::Result<()> { + // TODO: support inserting multiple at the same time + sqlx::query!( + "UPDATE buildoutputs SET path = $3 WHERE build = $1 AND name = $2", + build_id, + name, + path, + ) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_last_build_step_id(&mut self, path: &str) -> sqlx::Result> { + Ok(sqlx::query!("SELECT MAX(build) FROM buildsteps WHERE drvPath = $1 and startTime != 0 and stopTime != 0 and status = 1", path) + .fetch_optional(&mut *self.tx) + .await? + .and_then(|v| v.max)) + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_last_build_step_id_for_output_path( + &mut self, + path: &str, + ) -> sqlx::Result> { + Ok(sqlx::query!( + r#" + SELECT MAX(s.build) FROM buildsteps s + JOIN BuildStepOutputs o ON s.build = o.build + WHERE startTime != 0 + AND stopTime != 0 + AND status = 1 + AND path = $1 + "#, + path, + ) + .fetch_optional(&mut *self.tx) + .await? + .and_then(|v| v.max)) + } + + #[tracing::instrument(skip(self, drv_path, name), err)] + pub async fn get_last_build_step_id_for_output_with_drv( + &mut self, + drv_path: &str, + name: &str, + ) -> sqlx::Result> { + Ok(sqlx::query!( + r#" + SELECT MAX(s.build) FROM buildsteps s + JOIN BuildStepOutputs o ON s.build = o.build + WHERE startTime != 0 + AND stopTime != 0 + AND status = 1 + AND drvPath = $1 + AND name = $2 + "#, + drv_path, + name, + ) + .fetch_optional(&mut *self.tx) + .await? + .and_then(|v| v.max)) + } + + #[tracing::instrument(skip(self), err)] + pub async fn alloc_build_step(&mut self, build_id: i32) -> sqlx::Result { + Ok(sqlx::query!( + "SELECT MAX(stepnr) FROM buildsteps WHERE build = $1", + build_id + ) + .fetch_optional(&mut *self.tx) + .await? + .and_then(|v| v.max) + .map_or(1, |v| v + 1)) + } + + #[tracing::instrument(skip(self, step), err)] + pub async fn insert_build_step(&mut self, step: InsertBuildStep<'_>) -> sqlx::Result { + let success = sqlx::query!( + r#" + INSERT INTO buildsteps ( + build, + stepnr, + type, + drvPath, + busy, + startTime, + stopTime, + system, + status, + propagatedFrom, + errorMsg, + machine + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 + ) + ON CONFLICT DO NOTHING + "#, + step.build_id, + step.step_nr, + step.r#type as i32, + step.drv_path, + i32::from(step.busy), + step.start_time, + step.stop_time, + step.platform, + if step.status == BuildStatus::Busy { + None + } else { + Some(step.status as i32) + }, + step.propagated_from, + step.error_msg, + step.machine, + ) + .execute(&mut *self.tx) + .await? + .rows_affected() + != 0; + Ok(success) + } + + #[tracing::instrument(skip(self, outputs), err)] + pub async fn insert_build_step_outputs( + &mut self, + outputs: &[InsertBuildStepOutput], + ) -> sqlx::Result<()> { + if outputs.is_empty() { + return Ok(()); + } + + let mut query_builder = + sqlx::QueryBuilder::new("INSERT INTO buildstepoutputs (build, stepnr, name, path) "); + + query_builder.push_values(outputs, |mut b, output| { + b.push_bind(output.build_id) + .push_bind(output.step_nr) + .push_bind(&output.name) + .push_bind(&output.path); + }); + let query = query_builder.build(); + query.execute(&mut *self.tx).await?; + Ok(()) + } + + #[tracing::instrument(skip(self, name, path), err)] + pub async fn update_build_step_output( + &mut self, + build_id: i32, + step_nr: i32, + name: &str, + path: &str, + ) -> sqlx::Result<()> { + // TODO: support inserting multiple at the same time + sqlx::query!( + "UPDATE buildstepoutputs SET path = $4 WHERE build = $1 AND stepnr = $2 AND name = $3", + build_id, + step_nr, + name, + path, + ) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, res), err)] + pub async fn update_build_step_in_finish( + &mut self, + res: UpdateBuildStepInFinish<'_>, + ) -> sqlx::Result<()> { + sqlx::query!( + r#" + UPDATE buildsteps SET + busy = 0, + status = $1, + errorMsg = $4, + startTime = $5, + stopTime = $6, + machine = $7, + overhead = $8, + timesBuilt = $9, + isNonDeterministic = $10 + WHERE + build = $2 AND stepnr = $3 + "#, + res.status as i32, + res.build_id, + res.step_nr, + res.error_msg, + res.start_time, + res.stop_time, + res.machine, + res.overhead, + res.times_built, + res.is_non_deterministic, + ) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, build_id, step_nr), err)] + pub async fn get_drv_path_from_build_step( + &mut self, + build_id: i32, + step_nr: i32, + ) -> sqlx::Result> { + Ok(sqlx::query!( + "SELECT drvPath FROM BuildSteps WHERE build = $1 AND stepnr = $2", + build_id, + step_nr + ) + .fetch_optional(&mut *self.tx) + .await? + .and_then(|v| v.drvpath)) + } + + #[tracing::instrument(skip(self, build_id), err)] + pub async fn check_if_build_is_not_finished(&mut self, build_id: i32) -> sqlx::Result { + Ok(sqlx::query!( + "SELECT id FROM builds WHERE id = $1 AND finished = 0", + build_id, + ) + .fetch_optional(&mut *self.tx) + .await? + .is_some()) + } + + #[tracing::instrument(skip(self, p), err)] + pub async fn insert_build_product(&mut self, p: InsertBuildProduct<'_>) -> sqlx::Result<()> { + sqlx::query!( + r#" + INSERT INTO buildproducts ( + build, + productnr, + type, + subtype, + fileSize, + sha256hash, + path, + name, + defaultPath + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9 + ) + "#, + p.build_id, + p.product_nr, + p.r#type, + p.subtype, + p.file_size, + p.sha256hash, + p.path, + p.name, + p.default_path, + ) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, build_id), err)] + pub async fn delete_build_products_by_build_id(&mut self, build_id: i32) -> sqlx::Result<()> { + sqlx::query!("DELETE FROM buildproducts WHERE build = $1", build_id) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, metric), err)] + pub async fn insert_build_metric(&mut self, metric: InsertBuildMetric<'_>) -> sqlx::Result<()> { + sqlx::query!( + r#" + INSERT INTO buildmetrics ( + build, + name, + unit, + value, + project, + jobset, + job, + timestamp + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8 + ) + "#, + metric.build_id, + metric.name, + metric.unit, + metric.value, + metric.project, + metric.jobset, + metric.job, + metric.timestamp, + ) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, build_id), err)] + pub async fn delete_build_metrics_by_build_id(&mut self, build_id: i32) -> sqlx::Result<()> { + sqlx::query!("DELETE FROM buildmetrics WHERE build = $1", build_id) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, path), err)] + pub async fn insert_failed_paths(&mut self, path: &str) -> sqlx::Result<()> { + sqlx::query!( + r#" + INSERT INTO failedpaths ( + path + ) VALUES ( + $1 + ) + "#, + path, + ) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + #[tracing::instrument( + skip( + self, + start_time, + build_id, + platform, + machine, + status, + error_msg, + propagated_from + ), + err + )] + pub async fn create_build_step( + &mut self, + start_time: Option, + build_id: crate::models::BuildID, + drv_path: &str, + platform: Option<&str>, + machine: String, + status: crate::models::BuildStatus, + error_msg: Option, + propagated_from: Option, + outputs: Vec<(String, Option)>, + ) -> sqlx::Result { + let step_nr = loop { + let step_nr = self.alloc_build_step(build_id).await?; + if self + .insert_build_step(crate::models::InsertBuildStep { + build_id, + step_nr, + r#type: crate::models::BuildType::Build, + drv_path, + status, + busy: status == crate::models::BuildStatus::Busy, + start_time, + stop_time: if status == crate::models::BuildStatus::Busy { + None + } else { + start_time + }, + platform, + propagated_from, + error_msg: error_msg.as_deref(), + machine: &machine, + }) + .await? + { + break step_nr; + } + }; + + self.insert_build_step_outputs( + &outputs + .into_iter() + .map(|(name, path)| crate::models::InsertBuildStepOutput { + build_id, + step_nr, + name, + path, + }) + .collect::>(), + ) + .await?; + + if status == crate::models::BuildStatus::Busy { + self.notify_step_started(build_id, step_nr).await?; + } + + Ok(step_nr) + } + + #[tracing::instrument( + skip(self, start_time, stop_time, build_id, drv_path, output,), + err, + ret + )] + pub async fn create_substitution_step( + &mut self, + start_time: i32, + stop_time: i32, + build_id: crate::models::BuildID, + drv_path: &str, + output: (String, Option), + ) -> anyhow::Result { + let step_nr = loop { + let step_nr = self.alloc_build_step(build_id).await?; + if self + .insert_build_step(crate::models::InsertBuildStep { + build_id, + step_nr, + r#type: crate::models::BuildType::Substitution, + drv_path, + status: crate::models::BuildStatus::Success, + busy: false, + start_time: Some(start_time), + stop_time: Some(stop_time), + platform: None, + propagated_from: None, + error_msg: None, + machine: "", + }) + .await? + { + break step_nr; + } + }; + + self.insert_build_step_outputs(&[crate::models::InsertBuildStepOutput { + build_id, + step_nr, + name: output.0, + path: output.1, + }]) + .await?; + + Ok(step_nr) + } + + #[tracing::instrument(skip(self, build, is_cached_build, start_time, stop_time,), err)] + pub async fn mark_succeeded_build( + &mut self, + build: crate::models::MarkBuildSuccessData<'_>, + is_cached_build: bool, + start_time: i32, + stop_time: i32, + ) -> anyhow::Result<()> { + if build.finished_in_db { + return Ok(()); + } + + if !self.check_if_build_is_not_finished(build.id).await? { + return Ok(()); + } + + self.update_build( + build.id, + crate::models::UpdateBuild { + status: if build.failed { + crate::models::BuildStatus::FailedWithOutput + } else { + crate::models::BuildStatus::Success + }, + start_time, + stop_time, + size: i64::try_from(build.size)?, + closure_size: i64::try_from(build.closure_size)?, + release_name: build.release_name, + is_cached_build, + }, + ) + .await?; + + for (name, path) in &build.outputs { + self.update_build_output(build.id, name, path).await?; + } + + self.delete_build_products_by_build_id(build.id).await?; + + for (nr, p) in build.products.iter().enumerate() { + self.insert_build_product(crate::models::InsertBuildProduct { + build_id: build.id, + product_nr: i32::try_from(nr + 1)?, + r#type: p.r#type, + subtype: p.subtype, + file_size: p.filesize, + sha256hash: p.sha256hash, + path: p.path.as_deref().unwrap_or_default(), + name: p.name, + default_path: p.defaultpath.unwrap_or_default(), + }) + .await?; + } + + self.delete_build_metrics_by_build_id(build.id).await?; + for m in &build.metrics { + self.insert_build_metric(crate::models::InsertBuildMetric { + build_id: build.id, + name: m.1.name, + unit: m.1.unit, + value: m.1.value, + project: build.project_name, + jobset: build.jobset_name, + job: build.name, + timestamp: i32::try_from(build.timestamp)?, // TODO + }) + .await?; + } + Ok(()) + } + + #[tracing::instrument(skip(self, status), err)] + pub async fn upsert_status(&mut self, status: &serde_json::Value) -> sqlx::Result<()> { + sqlx::query!( + r#"INSERT INTO systemstatus ( + what, status + ) VALUES ( + 'queue-runner', $1 + ) ON CONFLICT (what) DO UPDATE SET status = EXCLUDED.status;"#, + Some(status), + ) + .execute(&mut *self.tx) + .await?; + Ok(()) + } +} + +impl Transaction<'_> { + #[tracing::instrument(skip(self), err)] + async fn notify_any(&mut self, channel: &str, msg: &str) -> sqlx::Result<()> { + sqlx::query( + r"SELECT pg_notify(chan, payload) from (values ($1, $2)) notifies(chan, payload)", + ) + .bind(channel) + .bind(msg) + .execute(&mut *self.tx) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self), err)] + pub async fn notify_builds_added(&mut self) -> sqlx::Result<()> { + self.notify_any("builds_added", "?").await?; + Ok(()) + } + + #[tracing::instrument(skip(self, build_id, dependent_ids,), err)] + pub async fn notify_build_finished( + &mut self, + build_id: i32, + dependent_ids: &[i32], + ) -> sqlx::Result<()> { + let mut q = vec![build_id.to_string()]; + q.extend(dependent_ids.iter().map(ToString::to_string)); + + self.notify_any("build_finished", &q.join("\t")).await?; + Ok(()) + } + + #[tracing::instrument(skip(self, build_id, step_nr,), err)] + pub async fn notify_step_started(&mut self, build_id: i32, step_nr: i32) -> sqlx::Result<()> { + self.notify_any("step_started", &format!("{build_id}\t{step_nr}")) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self, build_id, step_nr, log_file,), err)] + pub async fn notify_step_finished( + &mut self, + build_id: i32, + step_nr: i32, + log_file: &str, + ) -> sqlx::Result<()> { + self.notify_any( + "step_finished", + &format!("{build_id}\t{step_nr}\t{log_file}"), + ) + .await?; + Ok(()) + } + + #[tracing::instrument(skip(self), err)] + pub async fn notify_dump_status(&mut self) -> sqlx::Result<()> { + self.notify_any("dump_status", "").await?; + Ok(()) + } + + #[tracing::instrument(skip(self), err)] + pub async fn notify_status_dumped(&mut self) -> sqlx::Result<()> { + self.notify_any("status_dumped", "").await?; + Ok(()) + } +} diff --git a/src/crates/db/src/lib.rs b/src/crates/db/src/lib.rs new file mode 100644 index 000000000..8fe5f5d8e --- /dev/null +++ b/src/crates/db/src/lib.rs @@ -0,0 +1,54 @@ +#![deny(clippy::all)] +#![deny(clippy::pedantic)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::expect_used)] +#![allow(clippy::missing_errors_doc)] + +mod connection; +pub mod models; + +use std::str::FromStr as _; + +pub use connection::{Connection, Transaction}; +pub use sqlx::Error; + +#[derive(Clone)] +pub struct Database { + pool: sqlx::PgPool, +} + +impl Database { + pub async fn new(url: &str, max_connections: u32) -> Result { + Ok(Self { + pool: sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .connect(url) + .await?, + }) + } + + pub async fn get(&self) -> Result { + let conn = self.pool.acquire().await?; + Ok(Connection::new(conn)) + } + + #[tracing::instrument(skip(self, url), err)] + pub fn reconfigure_pool(&self, url: &str) -> anyhow::Result<()> { + // TODO: ability to change max_connections by dropping the pool and recreating it + self.pool + .set_connect_options(sqlx::postgres::PgConnectOptions::from_str(url)?); + Ok(()) + } + + pub async fn listener( + &self, + channels: Vec<&str>, + ) -> Result< + impl futures::Stream> + Unpin, + sqlx::Error, + > { + let mut listener = sqlx::postgres::PgListener::connect_with(&self.pool).await?; + listener.listen_all(channels).await?; + Ok(listener.into_stream()) + } +} diff --git a/src/crates/db/src/models.rs b/src/crates/db/src/models.rs new file mode 100644 index 000000000..99c41965c --- /dev/null +++ b/src/crates/db/src/models.rs @@ -0,0 +1,222 @@ +use hashbrown::HashMap; + +pub type BuildID = i32; + +#[repr(i32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BuildStatus { + Success = 0, + Failed = 1, + DepFailed = 2, // builds only + Aborted = 3, + Cancelled = 4, + FailedWithOutput = 6, // builds only + TimedOut = 7, + CachedFailure = 8, // steps only + Unsupported = 9, + LogLimitExceeded = 10, + NarSizeLimitExceeded = 11, + NotDeterministic = 12, + Busy = 100, // not stored +} + +impl BuildStatus { + #[must_use] + pub const fn from_i32(v: i32) -> Option { + match v { + 0 => Some(Self::Success), + 1 => Some(Self::Failed), + 2 => Some(Self::DepFailed), + 3 => Some(Self::Aborted), + 4 => Some(Self::Cancelled), + 6 => Some(Self::FailedWithOutput), + 7 => Some(Self::TimedOut), + 8 => Some(Self::CachedFailure), + 9 => Some(Self::Unsupported), + 10 => Some(Self::LogLimitExceeded), + 11 => Some(Self::NarSizeLimitExceeded), + 12 => Some(Self::NotDeterministic), + 100 => Some(Self::Busy), + _ => None, + } + } +} + +#[repr(i32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StepStatus { + Preparing = 1, + Connecting = 10, + SendingInputs = 20, + Building = 30, + WaitingForLocalSlot = 35, + ReceivingOutputs = 40, + PostProcessing = 50, +} + +pub struct Jobset { + pub project: String, + pub name: String, + pub schedulingshares: i32, +} + +pub struct BuildSmall { + pub id: BuildID, + pub globalpriority: i32, +} + +pub struct Build { + pub id: BuildID, + pub jobset_id: i32, + pub project: String, + pub jobset: String, + pub job: String, + pub drvpath: String, + pub maxsilent: Option, // maxsilent integer default 3600 + pub timeout: Option, // timeout integer default 36000 + pub timestamp: i64, + pub globalpriority: i32, + pub priority: i32, +} + +pub struct BuildSteps { + pub starttime: Option, + pub stoptime: Option, +} + +#[repr(i32)] +pub enum BuildType { + Build = 0, + Substitution = 1, +} + +pub struct UpdateBuild<'a> { + pub status: BuildStatus, + pub start_time: i32, + pub stop_time: i32, + pub size: i64, + pub closure_size: i64, + pub release_name: Option<&'a str>, + pub is_cached_build: bool, +} + +pub struct InsertBuildStep<'a> { + pub build_id: BuildID, + pub step_nr: i32, + pub r#type: BuildType, + pub drv_path: &'a str, + pub status: BuildStatus, + pub busy: bool, + pub start_time: Option, + pub stop_time: Option, + pub platform: Option<&'a str>, + pub propagated_from: Option, + pub error_msg: Option<&'a str>, + pub machine: &'a str, +} + +pub struct InsertBuildStepOutput { + pub build_id: BuildID, + pub step_nr: i32, + pub name: String, + pub path: Option, +} + +pub struct UpdateBuildStep { + pub build_id: BuildID, + pub step_nr: i32, + pub status: StepStatus, +} + +pub struct UpdateBuildStepInFinish<'a> { + pub build_id: BuildID, + pub step_nr: i32, + pub status: BuildStatus, + pub error_msg: Option<&'a str>, + pub start_time: i32, + pub stop_time: i32, + pub machine: Option<&'a str>, + pub overhead: Option, + pub times_built: Option, + pub is_non_deterministic: Option, +} + +pub struct InsertBuildProduct<'a> { + pub build_id: BuildID, + pub product_nr: i32, + pub r#type: &'a str, + pub subtype: &'a str, + pub file_size: Option, + pub sha256hash: Option<&'a str>, + pub path: &'a str, + pub name: &'a str, + pub default_path: &'a str, +} + +pub struct InsertBuildMetric<'a> { + pub build_id: BuildID, + pub name: &'a str, + pub unit: Option<&'a str>, + pub value: f64, + pub project: &'a str, + pub jobset: &'a str, + pub job: &'a str, + pub timestamp: i32, +} + +pub struct BuildOutput { + pub id: i32, + pub buildstatus: Option, + pub releasename: Option, + pub closuresize: Option, + pub size: Option, +} + +pub struct OwnedBuildProduct { + pub r#type: String, + pub subtype: String, + pub filesize: Option, + pub sha256hash: Option, + pub path: Option, + pub name: String, + pub defaultpath: Option, +} + +pub struct BuildProduct<'a> { + pub r#type: &'a str, + pub subtype: &'a str, + pub filesize: Option, + pub sha256hash: Option<&'a str>, + pub path: Option, + pub name: &'a str, + pub defaultpath: Option<&'a str>, +} + +pub struct OwnedBuildMetric { + pub name: String, + pub unit: Option, + pub value: f64, +} + +pub struct BuildMetric<'a> { + pub name: &'a str, + pub unit: Option<&'a str>, + pub value: f64, +} + +pub struct MarkBuildSuccessData<'a> { + pub id: BuildID, + pub name: &'a str, + pub project_name: &'a str, + pub jobset_name: &'a str, + pub finished_in_db: bool, + pub timestamp: i64, + + pub failed: bool, + pub closure_size: u64, + pub size: u64, + pub release_name: Option<&'a str>, + pub outputs: HashMap, + pub products: Vec>, + pub metrics: HashMap<&'a str, BuildMetric<'a>>, +} diff --git a/src/crates/nix-utils/Cargo.toml b/src/crates/nix-utils/Cargo.toml new file mode 100644 index 000000000..cce483b7b --- /dev/null +++ b/src/crates/nix-utils/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "nix-utils" +version = "0.1.0" +edition = "2024" +license = "LGPL-2.1-only" +rust-version.workspace = true + +[dependencies] +tracing.workspace = true +serde = { workspace = true, features = ["derive"] } +thiserror.workspace = true +anyhow.workspace = true +tokio = { workspace = true, features = ["full"] } +tokio-stream = { workspace = true, features = ["io-util"] } +tokio-util = { workspace = true, features = ["io", "io-util"] } +fs-err = { workspace = true, features = ["tokio"] } +futures.workspace = true +hashbrown.workspace = true +url.workspace = true +bytes.workspace = true +smallvec.workspace = true + +cxx.workspace = true + +nix-diff.workspace = true + +[build-dependencies] +cxx-build.workspace = true +pkg-config.workspace = true diff --git a/src/crates/nix-utils/LICENSE b/src/crates/nix-utils/LICENSE new file mode 100644 index 000000000..5ab7695ab --- /dev/null +++ b/src/crates/nix-utils/LICENSE @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/src/crates/nix-utils/build.rs b/src/crates/nix-utils/build.rs new file mode 100644 index 000000000..14128601f --- /dev/null +++ b/src/crates/nix-utils/build.rs @@ -0,0 +1,30 @@ +fn main() { + if std::env::var("DOCS_RS").is_ok() { + return; + } + + println!("cargo:rerun-if-changed=include/"); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=src/nix.cpp"); + println!("cargo:rerun-if-changed=src/lib.rs"); + println!("cargo:rerun-if-changed=src/hash.rs"); + println!("cargo:rerun-if-changed=src/realisation.rs"); + println!("cargo:rerun-if-changed=src/cxx/"); + + let library = pkg_config::probe_library("nix-main").unwrap(); + pkg_config::probe_library("nix-store").unwrap(); + pkg_config::probe_library("nix-util").unwrap(); + pkg_config::probe_library("libsodium").unwrap(); + + cxx_build::bridges(["src/lib.rs", "src/hash.rs", "src/realisation.rs"]) + .files([ + "src/nix.cpp", + "src/cxx/utils.cpp", + "src/cxx/hash.cpp", + "src/cxx/realisation.cpp", + ]) + .flag("-std=c++23") + .flag("-O2") + .includes(library.include_paths) + .compile("nix_utils"); +} diff --git a/src/crates/nix-utils/examples/convert_hash.rs b/src/crates/nix-utils/examples/convert_hash.rs new file mode 100644 index 000000000..217755527 --- /dev/null +++ b/src/crates/nix-utils/examples/convert_hash.rs @@ -0,0 +1,11 @@ +use nix_utils::{HashAlgorithm, HashFormat, convert_hash}; + +fn main() { + let x = convert_hash( + "1a4be2fe6b5246aa4ac8987a8a4af34c42a8dd7d08b46ab48516bcc1befbcd83", + Some(HashAlgorithm::SHA256), + HashFormat::SRI, + ) + .unwrap(); + println!("{x}"); +} diff --git a/src/crates/nix-utils/examples/copy_path.rs b/src/crates/nix-utils/examples/copy_path.rs new file mode 100644 index 000000000..2f4512dea --- /dev/null +++ b/src/crates/nix-utils/examples/copy_path.rs @@ -0,0 +1,27 @@ +use nix_utils::{self, copy_paths}; + +// requires env vars: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + +#[tokio::main] +async fn main() { + let local = nix_utils::LocalStore::init(); + let remote = + nix_utils::RemoteStore::init("s3://store?region=unknown&endpoint=http://localhost:9000"); + nix_utils::set_verbosity(1); + + let res = copy_paths( + local.as_base_store(), + remote.as_base_store(), + &[nix_utils::StorePath::new( + "1r5zv195y7b7b5q2daf5p82s2m6r4rg4-CVE-2024-56406.patch", + )], + false, + false, + false, + ) + .await; + println!("copy res={res:?}"); + + let stats = remote.get_s3_stats().unwrap(); + println!("stats {stats:?}"); +} diff --git a/src/crates/nix-utils/examples/drv_parse.rs b/src/crates/nix-utils/examples/drv_parse.rs new file mode 100644 index 000000000..bd3d43a26 --- /dev/null +++ b/src/crates/nix-utils/examples/drv_parse.rs @@ -0,0 +1,12 @@ +#[tokio::main] +async fn main() { + let store = nix_utils::LocalStore::init(); + let drv = nix_utils::query_drv( + &store, + &nix_utils::StorePath::new("5g60vyp4cbgwl12pav5apyi571smp62s-hello-2.12.2.drv"), + ) + .await + .unwrap(); + + println!("{drv:?}"); +} diff --git a/src/crates/nix-utils/examples/export_file.rs b/src/crates/nix-utils/examples/export_file.rs new file mode 100644 index 000000000..d0dc9bfa8 --- /dev/null +++ b/src/crates/nix-utils/examples/export_file.rs @@ -0,0 +1,34 @@ +use nix_utils::{self, BaseStore as _}; + +#[tokio::main] +async fn main() { + let store = nix_utils::LocalStore::init(); + + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::>(); + let closure = move |data: &[u8]| { + let data = Vec::from(data); + tx.send(data).is_ok() + }; + + let x = tokio::spawn(async move { + while let Some(x) = rx.recv().await { + print!("{}", String::from_utf8_lossy(&x)); + } + }); + + tokio::task::spawn_blocking(move || async move { + store + .export_paths( + &[nix_utils::StorePath::new( + "5g60vyp4cbgwl12pav5apyi571smp62s-hello-2.12.2.drv", + )], + closure, + ) + .unwrap(); + }) + .await + .unwrap() + .await; + + x.await.unwrap(); +} diff --git a/src/crates/nix-utils/examples/get_settings.rs b/src/crates/nix-utils/examples/get_settings.rs new file mode 100644 index 000000000..638f983ad --- /dev/null +++ b/src/crates/nix-utils/examples/get_settings.rs @@ -0,0 +1,12 @@ +fn main() { + let _store = nix_utils::LocalStore::init(); + println!("Nix prefix: {}", nix_utils::get_nix_prefix()); + println!("Store dir: {}", nix_utils::get_store_dir()); + println!("Log dir: {}", nix_utils::get_log_dir()); + println!("State dir: {}", nix_utils::get_state_dir()); + println!("System: {}", nix_utils::get_this_system()); + println!("Extra Platforms: {:?}", nix_utils::get_extra_platforms()); + println!("System features: {:?}", nix_utils::get_system_features()); + println!("Substituters: {:?}", nix_utils::get_substituters()); + println!("Use cgroups: {}", nix_utils::get_use_cgroups()); +} diff --git a/src/crates/nix-utils/examples/import_file_fd.rs b/src/crates/nix-utils/examples/import_file_fd.rs new file mode 100644 index 000000000..6cbe73081 --- /dev/null +++ b/src/crates/nix-utils/examples/import_file_fd.rs @@ -0,0 +1,35 @@ +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + +use nix_utils::{self, BaseStore as _}; + +#[tokio::main] +async fn main() { + let store = nix_utils::LocalStore::init(); + + let file = fs_err::tokio::File::open("/tmp/test.nar").await.unwrap(); + let mut reader = tokio::io::BufReader::new(file); + + println!("Importing test.nar == 5g60vyp4cbgwl12pav5apyi571smp62s-hello-2.12.2.drv"); + let (mut rx, tx) = tokio::net::unix::pipe::pipe().unwrap(); + + tokio::spawn(async move { + let mut buf: [u8; 1] = [0; 1]; + loop { + let s = reader.read(&mut buf).await.unwrap(); + if s == 0 { + break; + } + let _ = rx.write(&buf).await.unwrap(); + } + let _ = rx.shutdown().await; + drop(rx); + }); + tokio::task::spawn_blocking(move || async move { + store + .import_paths_with_fd(tx.into_blocking_fd().unwrap(), false) + .unwrap(); + }) + .await + .unwrap() + .await; +} diff --git a/src/crates/nix-utils/examples/import_file_stream.rs b/src/crates/nix-utils/examples/import_file_stream.rs new file mode 100644 index 000000000..8e00c28a5 --- /dev/null +++ b/src/crates/nix-utils/examples/import_file_stream.rs @@ -0,0 +1,11 @@ +use nix_utils::{self, BaseStore as _}; + +#[tokio::main] +async fn main() { + let store = nix_utils::LocalStore::init(); + + let file = fs_err::tokio::File::open("/tmp/test3.nar").await.unwrap(); + let stream = tokio_util::io::ReaderStream::new(file); + + store.import_paths(stream, false).await.unwrap(); +} diff --git a/src/crates/nix-utils/examples/is_valid_path.rs b/src/crates/nix-utils/examples/is_valid_path.rs new file mode 100644 index 000000000..6bac0c0bf --- /dev/null +++ b/src/crates/nix-utils/examples/is_valid_path.rs @@ -0,0 +1,13 @@ +use nix_utils::{self, BaseStore as _}; + +#[tokio::main] +async fn main() { + let store = nix_utils::LocalStore::init(); + let nix_prefix = nix_utils::get_nix_prefix(); + println!( + "storepath={nix_prefix} valid={}", + store + .is_valid_path(&nix_utils::StorePath::new(&nix_prefix)) + .await + ); +} diff --git a/src/crates/nix-utils/examples/list_nar.rs b/src/crates/nix-utils/examples/list_nar.rs new file mode 100644 index 000000000..f13c39e64 --- /dev/null +++ b/src/crates/nix-utils/examples/list_nar.rs @@ -0,0 +1,15 @@ +use nix_utils::BaseStore as _; + +#[tokio::main] +async fn main() { + let store = nix_utils::LocalStore::init(); + + let ls = store + .list_nar( + &nix_utils::StorePath::new("sqw9kyl8zrfnkklb3vp6gji9jw9qfgb5-hello-2.12.2"), + true, + ) + .await + .unwrap(); + println!("{ls:?}"); +} diff --git a/src/crates/nix-utils/examples/path_infos.rs b/src/crates/nix-utils/examples/path_infos.rs new file mode 100644 index 000000000..5f7901fd1 --- /dev/null +++ b/src/crates/nix-utils/examples/path_infos.rs @@ -0,0 +1,28 @@ +use nix_utils::BaseStore as _; + +#[tokio::main] +async fn main() { + let local = nix_utils::LocalStore::init(); + + let p1 = nix_utils::StorePath::new("ihl4ya67glh9815v1lanyqph0p7hdzfb-hdf5-cpp-1.14.6-bin"); + let p2 = nix_utils::StorePath::new("sgv5w811jvvxpjgmyw1n6l8hwfilha7x-hdf5-cpp-1.14.6-dev"); + let p3 = nix_utils::StorePath::new("vb6yrzk31ng8s6nzs4y4jq6qsjab3gxv-hdf5-cpp-1.14.6"); + + let infos = local.query_path_infos(&[&p1, &p2, &p3]).await; + + println!("{infos:?}"); + println!( + "closure_size {p1}: {}", + local.compute_closure_size(&p1).await + ); + println!( + "closure_size {p2}: {}", + local.compute_closure_size(&p2).await + ); + println!( + "closure_size {p3}: {}", + local.compute_closure_size(&p3).await + ); + + println!("stats: {:?}", local.get_store_stats()); +} diff --git a/src/crates/nix-utils/examples/query_requisites.rs b/src/crates/nix-utils/examples/query_requisites.rs new file mode 100644 index 000000000..157fe06cb --- /dev/null +++ b/src/crates/nix-utils/examples/query_requisites.rs @@ -0,0 +1,20 @@ +use nix_utils::BaseStore as _; + +#[tokio::main] +async fn main() { + let store = nix_utils::LocalStore::init(); + + let drv = nix_utils::StorePath::new("5g60vyp4cbgwl12pav5apyi571smp62s-hello-2.12.2.drv"); + let ps = store.query_requisites(&[&drv], false).await.unwrap(); + for p in ps { + println!("{}", store.print_store_path(&p)); + } + + println!(); + println!(); + + let ps = store.query_requisites(&[&drv], true).await.unwrap(); + for p in ps { + println!("{}", store.print_store_path(&p)); + } +} diff --git a/src/crates/nix-utils/examples/realisation_hashes.rs b/src/crates/nix-utils/examples/realisation_hashes.rs new file mode 100644 index 000000000..d36626458 --- /dev/null +++ b/src/crates/nix-utils/examples/realisation_hashes.rs @@ -0,0 +1,14 @@ +use nix_utils::BaseStore as _; + +#[tokio::main] +async fn main() { + let local = nix_utils::LocalStore::init(); + let hashes = local + .static_output_hashes(&nix_utils::StorePath::new( + "g6i53wpfisscqqj8d2hf3z83rzb9jklg-bash-5.2p37.drv", + )) + .await + .unwrap(); + + println!("hashes: {:?}", hashes); +} diff --git a/src/crates/nix-utils/examples/realisation_parse.rs b/src/crates/nix-utils/examples/realisation_parse.rs new file mode 100644 index 000000000..c901b5ca2 --- /dev/null +++ b/src/crates/nix-utils/examples/realisation_parse.rs @@ -0,0 +1,30 @@ +use fs_err::tokio::read_to_string; +use nix_utils::RealisationOperations as _; + +#[tokio::main] +async fn main() { + let json_str = "{\"dependentRealisations\":{}, \"id\": \"sha256:6e46b9cf4fecaeab4b3c0578f4ab99e89d2f93535878c4ac69b5d5c4eb3a3db9!debug\", \"outPath\": \"5cdp7ncqc47j3ylzqc2lpphgks78p02s-bash-5.2p37-debug\", \"signatures\":[] }"; + + let local = nix_utils::LocalStore::init(); + let mut realisation = local.parse_realisation(json_str).unwrap(); + + println!("id: {}", realisation.get_id()); + println!("json: {}", realisation.as_json()); + println!("fingerprint: {}", realisation.fingerprint()); + println!( + "struct: {:?}", + realisation.as_rust(local.as_base_store()).unwrap() + ); + + realisation + .sign( + &read_to_string(format!( + "{}/../../example-secret-key", + env!("CARGO_MANIFEST_DIR") + )) + .await + .unwrap(), + ) + .unwrap(); + println!("json signed: {:?}", realisation.as_json()); +} diff --git a/src/crates/nix-utils/examples/realisation_query.rs b/src/crates/nix-utils/examples/realisation_query.rs new file mode 100644 index 000000000..c5f5ec1fd --- /dev/null +++ b/src/crates/nix-utils/examples/realisation_query.rs @@ -0,0 +1,32 @@ +use fs_err::tokio::read_to_string; +use nix_utils::RealisationOperations as _; + +#[tokio::main] +async fn main() { + let local = nix_utils::LocalStore::init(); + let mut realisation = local + .query_raw_realisation( + "sha256:6e46b9cf4fecaeab4b3c0578f4ab99e89d2f93535878c4ac69b5d5c4eb3a3db9", + "debug", + ) + .unwrap(); + + println!("json: {}", realisation.as_json()); + println!("fingerprint: {}", realisation.fingerprint()); + println!( + "struct: {:?}", + realisation.as_rust(local.as_base_store()).unwrap() + ); + + realisation + .sign( + &read_to_string(format!( + "{}/../../example-secret-key", + env!("CARGO_MANIFEST_DIR") + )) + .await + .unwrap(), + ) + .unwrap(); + println!("json signed: {:?}", realisation.as_json()); +} diff --git a/src/crates/nix-utils/examples/stream_test.rs b/src/crates/nix-utils/examples/stream_test.rs new file mode 100644 index 000000000..07d6c6e50 --- /dev/null +++ b/src/crates/nix-utils/examples/stream_test.rs @@ -0,0 +1,29 @@ +use bytes::Bytes; +use tokio::io::AsyncReadExt; +use tokio_util::io::StreamReader; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Create a stream from an iterator. + let stream = tokio_stream::iter(vec![ + tokio::io::Result::Ok(Bytes::from_static(&[0, 1, 2, 3])), + tokio::io::Result::Ok(Bytes::from_static(&[4, 5, 6, 7])), + tokio::io::Result::Ok(Bytes::from_static(&[8, 9, 10, 11])), + ]); + + // Convert it to an AsyncRead. + let mut read = StreamReader::new(stream); + + // Read five bytes from the stream. + let mut buf = [0; 2]; + + loop { + let read = read.read(&mut buf).await?; + if read == 0 { + break; + } + println!("{buf:?}"); + } + + Ok(()) +} diff --git a/src/crates/nix-utils/examples/upsert_file.rs b/src/crates/nix-utils/examples/upsert_file.rs new file mode 100644 index 000000000..eb7290fe4 --- /dev/null +++ b/src/crates/nix-utils/examples/upsert_file.rs @@ -0,0 +1,22 @@ +// requires env vars: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + +#[tokio::main] +async fn main() { + let store = + nix_utils::RemoteStore::init("s3://store?region=unknown&endpoint=http://localhost:9000"); + nix_utils::set_verbosity(1); + let res = store + .upsert_file( + "log/z4zxibgvmk4ikarbbpwjql21wjmdvy85-dbus-1.drv".to_string(), + std::path::PathBuf::from( + concat!(env!("CARGO_MANIFEST_DIR"), "/examples/upsert_file.rs").to_string(), + ), + "text/plain; charset=utf-8", + ) + .await; + + println!("upsert res={res:?}",); + + let stats = store.get_s3_stats().unwrap(); + println!("stats {stats:?}",); +} diff --git a/src/crates/nix-utils/include/hash.h b/src/crates/nix-utils/include/hash.h new file mode 100644 index 000000000..f93261245 --- /dev/null +++ b/src/crates/nix-utils/include/hash.h @@ -0,0 +1,10 @@ +#pragma once + +#include "rust/cxx.h" + +#include "nix-utils/src/hash.rs.h" + +namespace nix_utils::hash { +rust::String convert_hash(rust::Str s, OptionalHashAlgorithm algo, + HashFormat out_format); +} // namespace nix_utils::hash diff --git a/src/crates/nix-utils/include/nix.h b/src/crates/nix-utils/include/nix.h new file mode 100644 index 000000000..01c415939 --- /dev/null +++ b/src/crates/nix-utils/include/nix.h @@ -0,0 +1,79 @@ +#pragma once + +#include "rust/cxx.h" +#include +#include + +namespace nix_utils { +class StoreWrapper { +public: + StoreWrapper(nix::ref _store); + + nix::ref _store; +}; +} // namespace nix_utils + +// we need to include this after StoreWrapper +#include "nix-utils/src/lib.rs.h" + +namespace nix_utils { +void init_nix(); +std::shared_ptr init(rust::Str uri); + +rust::String get_nix_prefix(); +rust::String get_store_dir(); +rust::String get_build_dir(); +rust::String get_log_dir(); +rust::String get_state_dir(); +rust::String get_nix_version(); +rust::String get_this_system(); +rust::Vec get_extra_platforms(); +rust::Vec get_system_features(); +rust::Vec get_substituters(); + +bool get_use_cgroups(); +void set_verbosity(int32_t level); +rust::String sign_string(rust::Str secret_key, rust::Str msg); + +bool is_valid_path(const StoreWrapper &wrapper, rust::Str path); +InternalPathInfo query_path_info(const StoreWrapper &wrapper, rust::Str path); +void clear_path_info_cache(const StoreWrapper &wrapper); +uint64_t compute_closure_size(const StoreWrapper &wrapper, rust::Str path); +rust::Vec compute_fs_closure(const StoreWrapper &wrapper, + rust::Str path, bool flip_direction, + bool include_outputs, + bool include_derivers); +rust::Vec +compute_fs_closures(const StoreWrapper &wrapper, + rust::Slice paths, bool flip_direction, + bool include_outputs, bool include_derivers, bool toposort); +void upsert_file(const StoreWrapper &wrapper, rust::Str path, rust::Str data, + rust::Str mime_type); +StoreStats get_store_stats(const StoreWrapper &wrapper); +S3Stats get_s3_stats(const StoreWrapper &wrapper); +void copy_paths(const StoreWrapper &src_store, const StoreWrapper &dst_store, + rust::Slice paths, bool repair, + bool check_sigs, bool substitute); + +void import_paths( + const StoreWrapper &wrapper, bool check_sigs, size_t runtime, size_t reader, + rust::Fn, size_t, size_t, size_t)> callback, + size_t user_data); +void import_paths_with_fd(const StoreWrapper &wrapper, bool check_sigs, + int32_t fd); +void export_paths(const StoreWrapper &src_store, + rust::Slice paths, + rust::Fn, size_t)> callback, + size_t userdata); +void nar_from_path(const StoreWrapper &src_store, rust::Str path, + rust::Fn, size_t)> callback, + size_t userdata); + +rust::String list_nar(const StoreWrapper &wrapper, rust::Str path, + bool recursive); + +void ensure_path(const StoreWrapper &wrapper, rust::Str path); +rust::String try_resolve_drv(const StoreWrapper &wrapper, rust::Str path); +rust::Vec static_output_hashes(const StoreWrapper &wrapper, + rust::Str output_path); +} // namespace nix_utils diff --git a/src/crates/nix-utils/include/realisation.h b/src/crates/nix-utils/include/realisation.h new file mode 100644 index 000000000..bf21a9e15 --- /dev/null +++ b/src/crates/nix-utils/include/realisation.h @@ -0,0 +1,40 @@ +#pragma once + +#include "rust/cxx.h" +#include + +#include +#include "nix-utils/include/nix.h" +#include "nix/store/realisation.hh" + +namespace nix_utils { +struct SharedRealisation; +struct DrvOutput; + +class InternalRealisation { +public: + InternalRealisation(nix::ref _realisation); + + rust::String as_json() const; + SharedRealisation to_rust(const nix_utils::StoreWrapper &wrapper) const; + DrvOutput get_drv_output() const; + + rust::String fingerprint() const; + void sign(rust::Str secret_key); + void clear_signatures(); + + void write_to_disk_cache(const nix_utils::StoreWrapper &wrapper) const; + +private: + nix::ref _realisation; +}; +} // namespace nix_utils + +#include "nix-utils/src/realisation.rs.h" + +namespace nix_utils { +std::shared_ptr +query_raw_realisation(const nix_utils::StoreWrapper &wrapper, + rust::Str output_id); +std::shared_ptr parse_realisation(rust::Str json_string); +} // namespace nix_utils diff --git a/src/crates/nix-utils/include/utils.h b/src/crates/nix-utils/include/utils.h new file mode 100644 index 000000000..8669bce1e --- /dev/null +++ b/src/crates/nix-utils/include/utils.h @@ -0,0 +1,14 @@ +#pragma once + +#include "rust/cxx.h" +#include + +#define AS_VIEW(rstr) std::string_view(rstr.data(), rstr.length()) +#define AS_STRING(rstr) std::string(rstr.data(), rstr.length()) + +rust::String extract_opt_path(const nix::Store &store, + const std::optional &v); +rust::Vec extract_path_set(const nix::Store &store, + const nix::StorePathSet &set); +rust::Vec extract_paths(const nix::Store &store, + const nix::StorePaths &set); diff --git a/src/crates/nix-utils/src/cxx/hash.cpp b/src/crates/nix-utils/src/cxx/hash.cpp new file mode 100644 index 000000000..63ad83be5 --- /dev/null +++ b/src/crates/nix-utils/src/cxx/hash.cpp @@ -0,0 +1,46 @@ +#include "nix-utils/include/hash.h" +#include "nix-utils/include/utils.h" + +#include + +namespace nix_utils::hash { +static inline std::optional +convert_algo(OptionalHashAlgorithm algo) { + switch (algo) { + case OptionalHashAlgorithm::MD5: + return nix::HashAlgorithm::MD5; + case OptionalHashAlgorithm::SHA1: + return nix::HashAlgorithm::SHA1; + case OptionalHashAlgorithm::SHA256: + return nix::HashAlgorithm::SHA256; + case OptionalHashAlgorithm::SHA512: + return nix::HashAlgorithm::SHA512; + case OptionalHashAlgorithm::BLAKE3: + return nix::HashAlgorithm::BLAKE3; + case OptionalHashAlgorithm::None: + default: + return std::nullopt; + }; +} + +static inline nix::HashFormat convert_format(HashFormat format) { + switch (format) { + case HashFormat::Base64: + return nix::HashFormat::Base64; + case HashFormat::Nix32: + return nix::HashFormat::Nix32; + case HashFormat::Base16: + return nix::HashFormat::Base16; + case HashFormat::SRI: + default: + return nix::HashFormat::SRI; + }; +} + +rust::String convert_hash(rust::Str s, OptionalHashAlgorithm algo, + HashFormat out_format) { + auto h = nix::Hash::parseAny(AS_VIEW(s), convert_algo(algo)); + auto f = convert_format(out_format); + return h.to_string(f, f == nix::HashFormat::SRI); +} +} // namespace nix_utils::hash diff --git a/src/crates/nix-utils/src/cxx/realisation.cpp b/src/crates/nix-utils/src/cxx/realisation.cpp new file mode 100644 index 000000000..242c5cd1f --- /dev/null +++ b/src/crates/nix-utils/src/cxx/realisation.cpp @@ -0,0 +1,101 @@ +#include "nix-utils/include/realisation.h" +#include "nix-utils/include/utils.h" + +#include "nix/store/nar-info-disk-cache.hh" +#include "nix/store/store-api.hh" +#include "nix/util/json-utils.hh" + +namespace nix_utils { +InternalRealisation::InternalRealisation( + nix::ref _realisation) + : _realisation(_realisation) {} + +rust::String InternalRealisation::as_json() const { + return nlohmann::json(*_realisation).dump(); +} + +SharedRealisation +InternalRealisation::to_rust(const nix_utils::StoreWrapper &wrapper) const { + auto store = wrapper._store; + + rust::Vec signatures; + signatures.reserve(_realisation->signatures.size()); + for (const std::string &sig : _realisation->signatures) { + signatures.push_back(sig); + } + + rust::Vec dependent; + dependent.reserve(_realisation->dependentRealisations.size()); + for (auto const &[drv_output, store_path] : + _realisation->dependentRealisations) { + dependent.push_back(DrvOutputPathTuple{ + DrvOutput{ + drv_output.strHash(), + drv_output.outputName, + }, + store->printStorePath(store_path), + }); + } + + return SharedRealisation{ + DrvOutput{ + _realisation->id.strHash(), + _realisation->id.outputName, + }, + store->printStorePath(_realisation->outPath), + signatures, + dependent, + }; +} + +DrvOutput InternalRealisation::get_drv_output() const { + return DrvOutput{_realisation->id.strHash(), _realisation->id.outputName}; +} + +rust::String InternalRealisation::fingerprint() const { + return _realisation->fingerprint(); +} + +void InternalRealisation::sign(rust::Str secret_key) { + nix::SecretKey s(AS_VIEW(secret_key)); + nix::LocalSigner signer(std::move(s)); + _realisation->sign(signer); +} + +void InternalRealisation::clear_signatures() { + _realisation->signatures.clear(); +} + +void InternalRealisation::write_to_disk_cache( + const nix_utils::StoreWrapper &wrapper) const { + auto disk_cache = nix::getNarInfoDiskCache(); + + if (disk_cache.get_ptr()) { + auto store = wrapper._store; + disk_cache->upsertRealisation(store->config.getHumanReadableURI(), + *_realisation.get_ptr()); + } +} + +std::shared_ptr +query_raw_realisation(const nix_utils::StoreWrapper &wrapper, + rust::Str output_id) { + auto store = wrapper._store; + auto realisation = + store->queryRealisation(nix::DrvOutput::parse(AS_STRING(output_id))); + if (!realisation) { + throw nix::Error("output_id '%s' isn't found", output_id); + } + + return std::make_unique( + nix::make_ref((nix::Realisation &)*realisation)); +} + +std::shared_ptr parse_realisation(rust::Str json_string) { + nlohmann::json encoded = nlohmann::json::parse(json_string); + nix::Realisation realisation = + nlohmann::adl_serializer::from_json(encoded); + return std::make_unique( + nix::make_ref(realisation)); +} +} // namespace nix_utils diff --git a/src/crates/nix-utils/src/cxx/utils.cpp b/src/crates/nix-utils/src/cxx/utils.cpp new file mode 100644 index 000000000..cc73bfb04 --- /dev/null +++ b/src/crates/nix-utils/src/cxx/utils.cpp @@ -0,0 +1,29 @@ +#include "nix-utils/include/utils.h" + +#include + +rust::String extract_opt_path(const nix::Store &store, + const std::optional &v) { + // TODO(conni2461): Replace with option + return v ? store.printStorePath(*v) : ""; +} + +rust::Vec extract_path_set(const nix::Store &store, + const nix::StorePathSet &set) { + rust::Vec data; + data.reserve(set.size()); + for (const nix::StorePath &path : set) { + data.emplace_back(store.printStorePath(path)); + } + return data; +} + +rust::Vec extract_paths(const nix::Store &store, + const nix::StorePaths &set) { + rust::Vec data; + data.reserve(set.size()); + for (const nix::StorePath &path : set) { + data.emplace_back(store.printStorePath(path)); + } + return data; +} diff --git a/src/crates/nix-utils/src/drv.rs b/src/crates/nix-utils/src/drv.rs new file mode 100644 index 000000000..62b9678dc --- /dev/null +++ b/src/crates/nix-utils/src/drv.rs @@ -0,0 +1,224 @@ +use hashbrown::HashMap; +use smallvec::SmallVec; + +use crate::BaseStore as _; +use crate::StorePath; + +#[derive(Debug, Clone)] +pub struct Output { + pub name: String, + pub path: Option, + pub hash: Option, + pub hash_algo: Option, +} + +#[derive(Debug, Clone)] +pub struct CAOutput { + pub name: String, + pub path: StorePath, + pub hash: String, + pub hash_algo: String, +} + +impl CAOutput { + pub fn get_sri_hash(&self) -> Result { + let algo = self.hash_algo.strip_prefix("r:").unwrap_or(&self.hash_algo); + Ok(super::convert_hash( + &self.hash, + Some(algo.parse()?), + super::HashFormat::SRI, + )?) + } +} + +#[derive(Debug, Clone)] +pub struct DerivationEnv { + inner: HashMap, +} + +impl DerivationEnv { + #[must_use] + pub const fn new(v: HashMap) -> Self { + Self { inner: v } + } + + #[must_use] + pub fn get(&self, k: &str) -> Option<&str> { + self.inner + .get(k) + .and_then(|v| if v.is_empty() { None } else { Some(v.as_str()) }) + } + + #[must_use] + pub fn get_name(&self) -> Option<&str> { + self.get("name") + } + + #[must_use] + pub fn get_required_system_features(&self) -> Vec<&str> { + self.get("requiredSystemFeatures") + .unwrap_or_default() + .split(' ') + .filter(|v| !v.is_empty()) + .collect() + } + + #[must_use] + pub fn get_output_hash(&self) -> Option<&str> { + self.get("outputHash") + } + + #[must_use] + pub fn get_output_hash_algo(&self) -> Option<&str> { + self.get("outputHashAlgo") + } + + #[must_use] + pub fn get_output_hash_mode(&self) -> Option<&str> { + self.get("outputHashMode") + } +} + +#[derive(Debug, Clone)] +pub struct Derivation { + pub env: DerivationEnv, + pub input_drvs: SmallVec<[String; 8]>, + pub outputs: SmallVec<[Output; 6]>, + pub name: StorePath, + pub system: String, +} + +impl Derivation { + fn new(path: &StorePath, v: nix_diff::types::Derivation) -> Self { + Self { + env: DerivationEnv::new( + v.env + .into_iter() + .filter_map(|(k, v)| { + Some((String::from_utf8(k).ok()?, String::from_utf8(v).ok()?)) + }) + .collect(), + ), + input_drvs: v + .input_derivations + .into_keys() + .filter_map(|v| String::from_utf8(v).ok()) + .collect(), + outputs: v + .outputs + .into_iter() + .filter_map(|(k, v)| { + Some(Output { + name: String::from_utf8(k).ok()?, + path: if v.path.is_empty() { + None + } else { + String::from_utf8(v.path).ok().map(|p| StorePath::new(&p)) + }, + hash: v + .hash + .map(String::from_utf8) + .transpose() + .ok()? + .and_then(|v| if v.is_empty() { None } else { Some(v) }), + hash_algo: v + .hash_algorithm + .map(String::from_utf8) + .transpose() + .ok()? + .and_then(|v| if v.is_empty() { None } else { Some(v) }), + }) + }) + .collect(), + name: path.clone(), + system: String::from_utf8(v.platform).unwrap_or_default(), + } + } + + #[must_use] + pub fn is_ca(&self) -> bool { + self.outputs + .iter() + .any(|o| o.hash.is_some() && o.hash_algo.is_some()) + } + + #[must_use] + pub fn get_ca_output(&self) -> Option { + self.outputs.iter().find_map(|o| { + Some(CAOutput { + path: o.path.clone()?, + hash: o.hash.clone()?, + hash_algo: o.hash_algo.clone()?, + name: o.name.clone(), + }) + }) + } +} + +fn parse_drv(drv_path: &StorePath, input: &str) -> Result { + Ok(Derivation::new( + drv_path, + nix_diff::parser::parse_derivation_string(input)?, + )) +} + +#[tracing::instrument(skip(store), fields(%drv), err)] +pub async fn query_drv( + store: &crate::LocalStore, + drv: &StorePath, +) -> Result, crate::Error> { + if !drv.is_drv() { + return Ok(None); + } + + let full_path = store.print_store_path(drv); + if !fs_err::tokio::try_exists(&full_path).await? { + return Ok(None); + } + + let input = fs_err::tokio::read_to_string(&full_path).await?; + Ok(Some(parse_drv(drv, &input)?)) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use crate::{StorePath, drv::parse_drv}; + + #[test] + fn test_ca_derivation() { + let drv_str = r#"Derive([("out","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-linux-6.16.tar.xz","sha256","1a4be2fe6b5246aa4ac8987a8a4af34c42a8dd7d08b46ab48516bcc1befbcd83")],[],[],"builtin","builtin:fetchurl",[],[("builder","builtin:fetchurl"),("executable",""),("impureEnvVars","http_proxy https_proxy ftp_proxy all_proxy no_proxy"),("name","linux-6.16.tar.xz"),("out","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-linux-6.16.tar.xz"),("outputHash","sha256-Gkvi/mtSRqpKyJh6ikrzTEKo3X0ItGq0hRa8wb77zYM="),("outputHashAlgo",""),("outputHashMode","flat"),("preferLocalBuild","1"),("system","builtin"),("unpack",""),("url","https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.16.tar.xz"),("urls","https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.16.tar.xz")])"#; + let drv_path = StorePath::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-linux-6.16.tar.xz.drv"); + let drv = parse_drv(&drv_path, drv_str).unwrap(); + assert_eq!(drv.name, drv_path); + assert_eq!(drv.env.get_name(), Some("linux-6.16.tar.xz")); + assert_eq!( + drv.env.get_output_hash(), + Some("sha256-Gkvi/mtSRqpKyJh6ikrzTEKo3X0ItGq0hRa8wb77zYM=") + ); + assert_eq!(drv.env.get_output_hash_algo(), None); + assert_eq!(drv.env.get_output_hash_mode(), Some("flat")); + assert!(drv.is_ca()); + let o = drv.get_ca_output().unwrap(); + assert_eq!( + o.path, + StorePath::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-linux-6.16.tar.xz") + ); + assert_eq!(o.name, String::from("out")); + assert_eq!( + o.hash, + String::from("1a4be2fe6b5246aa4ac8987a8a4af34c42a8dd7d08b46ab48516bcc1befbcd83") + ); + assert_eq!(o.hash_algo, String::from("sha256")); + } + + #[test] + fn test_no_ca_derivation() { + let drv_str = r#"Derive([("info","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-gnused-4.9-info","",""),("out","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-gnused-4.9","","")],[("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaawbootstrap-tools.drv",["out"]),("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bootstrap-stage4-stdenv-linux.drv",["out"]),("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-update-autotools-gnu-config-scripts-hook.drv",["out"]),("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-perl-5.40.0.drv",["out"]),("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-sed-4.9.tar.xz.drv",["out"])],["/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-source-stdenv.sh","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-default-builder.sh"],"x86_64-linux","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bootstrap-tools/bin/bash",["-e","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-source-stdenv.sh","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-default-builder.sh"],[("NIX_MAIN_PROGRAM","sed"),("__structuredAttrs",""),("buildInputs",""),("builder","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bootstrap-tools/bin/bash"),("cmakeFlags",""),("configureFlags",""),("depsBuildBuild",""),("depsBuildBuildPropagated",""),("depsBuildTarget",""),("depsBuildTargetPropagated",""),("depsHostHost",""),("depsHostHostPropagated",""),("depsTargetTarget",""),("depsTargetTargetPropagated",""),("doCheck",""),("doInstallCheck",""),("info","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-gnused-4.9-info"),("mesonFlags",""),("name","gnused-4.9"),("nativeBuildInputs","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-update-autotools-gnu-config-scripts-hook /nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-perl-5.40.0"),("out","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-gnused-4.9"),("outputs","out info"),("patches",""),("pname","gnused"),("preConfigure","patchShebangs ./build-aux/help2man"),("propagatedBuildInputs",""),("propagatedNativeBuildInputs",""),("src","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-sed-4.9.tar.xz"),("stdenv","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bootstrap-stage4-stdenv-linux"),("strictDeps",""),("system","x86_64-linux"),("version","4.9")])"#; + let drv_path = StorePath::new("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-gnused-4.9.drv"); + let drv = parse_drv(&drv_path, drv_str).unwrap(); + assert_eq!(drv.name, drv_path); + assert!(!drv.is_ca()); + } +} diff --git a/src/crates/nix-utils/src/hash.rs b/src/crates/nix-utils/src/hash.rs new file mode 100644 index 000000000..fd64c4f45 --- /dev/null +++ b/src/crates/nix-utils/src/hash.rs @@ -0,0 +1,96 @@ +#[cxx::bridge(namespace = "nix_utils::hash")] +mod ffi { + enum HashFormat { + Base64, + Nix32, + Base16, + SRI, + } + + enum OptionalHashAlgorithm { + None, + MD5, + SHA1, + SHA256, + SHA512, + BLAKE3, + } + + unsafe extern "C++" { + include!("nix-utils/include/hash.h"); + + fn convert_hash( + s: &str, + algo: OptionalHashAlgorithm, + out_format: HashFormat, + ) -> Result; + } +} + +pub enum HashFormat { + Base64, + Nix32, + Base16, + SRI, +} + +impl From for ffi::HashFormat { + fn from(value: HashFormat) -> Self { + match value { + HashFormat::Base64 => Self::Base64, + HashFormat::Nix32 => Self::Nix32, + HashFormat::Base16 => Self::Base16, + HashFormat::SRI => Self::SRI, + } + } +} + +pub enum HashAlgorithm { + MD5, + SHA1, + SHA256, + SHA512, + BLAKE3, +} + +impl From> for ffi::OptionalHashAlgorithm { + fn from(value: Option) -> Self { + value.map_or(Self::None, |v| match v { + HashAlgorithm::MD5 => Self::MD5, + HashAlgorithm::SHA1 => Self::SHA1, + HashAlgorithm::SHA256 => Self::SHA256, + HashAlgorithm::SHA512 => Self::SHA512, + HashAlgorithm::BLAKE3 => Self::BLAKE3, + }) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum ParseError { + #[error("Invalid Algorithm passed: {0}")] + InvalidAlgorithm(String), +} + +impl std::str::FromStr for HashAlgorithm { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + match s { + "md5" => Ok(Self::MD5), + "sha1" => Ok(Self::SHA1), + "sha256" => Ok(Self::SHA256), + "sha512" => Ok(Self::SHA512), + "blake3" => Ok(Self::BLAKE3), + _ => Err(ParseError::InvalidAlgorithm(s.into())), + } + } +} + +#[inline] +pub fn convert_hash( + s: &str, + algo: Option, + out_format: HashFormat, +) -> Result { + ffi::convert_hash(s, algo.into(), out_format.into()) +} diff --git a/src/crates/nix-utils/src/lib.rs b/src/crates/nix-utils/src/lib.rs new file mode 100644 index 000000000..296472f93 --- /dev/null +++ b/src/crates/nix-utils/src/lib.rs @@ -0,0 +1,1255 @@ +#![deny(clippy::all)] +#![deny(clippy::pedantic)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::expect_used)] +#![allow(clippy::missing_errors_doc)] + +mod drv; +mod hash; +mod realisation; +mod realise; +mod store_path; + +use hashbrown::HashMap; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("std io error: `{0}`")] + Io(#[from] std::io::Error), + + #[error("tokio join error: `{0}`")] + TokioJoin(#[from] tokio::task::JoinError), + + #[error("utf8 error: `{0}`")] + Utf8(#[from] std::str::Utf8Error), + + #[error("Failed to get tokio stdout stream")] + Stream, + + #[error("Command failed with `{0}`")] + Exit(std::process::ExitStatus), + + #[error("Exception was thrown `{0}`")] + Exception(#[from] cxx::Exception), + + #[error("anyhow error: `{0}`")] + Anyhow(#[from] anyhow::Error), + + #[error("hash parse error: `{0}`")] + HashParseError(#[from] hash::ParseError), +} + +pub use drv::{Derivation, DerivationEnv, Output as DerivationOutput, query_drv}; +pub use hash::{HashAlgorithm, HashFormat, convert_hash}; +pub use realisation::{DrvOutput, FfiRealisation, Realisation, RealisationOperations}; +pub use realise::{BuildOptions, realise_drv, realise_drvs}; +pub use store_path::StorePath; + +pub fn validate_statuscode(status: std::process::ExitStatus) -> Result<(), Error> { + if status.success() { + Ok(()) + } else { + Err(Error::Exit(status)) + } +} + +pub fn add_root(store: &LocalStore, root_dir: &std::path::Path, store_path: &StorePath) { + let path = root_dir.join(store_path.base_name()); + // force create symlink + if fs_err::exists(&path).unwrap_or_default() { + let _ = fs_err::remove_file(&path); + } + if !fs_err::exists(&path).unwrap_or_default() { + let _ = fs_err::os::unix::fs::symlink(store.print_store_path(store_path), path); + } +} + +#[cxx::bridge(namespace = "nix_utils")] +mod ffi { + #[derive(Debug)] + struct InternalPathInfo { + deriver: String, + nar_hash: String, + registration_time: i64, + nar_size: u64, + refs: Vec, + sigs: Vec, + ca: String, + } + + #[derive(Debug)] + struct StoreStats { + nar_info_read: u64, + nar_info_read_averted: u64, + nar_info_missing: u64, + nar_info_write: u64, + path_info_cache_size: u64, + nar_read: u64, + nar_read_bytes: u64, + nar_read_compressed_bytes: u64, + nar_write: u64, + nar_write_averted: u64, + nar_write_bytes: u64, + nar_write_compressed_bytes: u64, + nar_write_compression_time_ms: u64, + } + + #[derive(Debug)] + struct S3Stats { + put: u64, + put_bytes: u64, + put_time_ms: u64, + get: u64, + get_bytes: u64, + get_time_ms: u64, + head: u64, + } + + #[derive(Debug)] + struct DerivationHash { + output_name: String, + drv_hash: String, + } + + unsafe extern "C++" { + include!("nix-utils/include/nix.h"); + + type StoreWrapper; + + fn init_nix(); + fn init(uri: &str) -> SharedPtr; + + fn get_nix_prefix() -> String; + fn get_store_dir() -> String; + fn get_build_dir() -> String; + fn get_log_dir() -> String; + fn get_state_dir() -> String; + fn get_nix_version() -> String; + fn get_this_system() -> String; + fn get_extra_platforms() -> Vec; + fn get_system_features() -> Vec; + fn get_substituters() -> Vec; + + fn get_use_cgroups() -> bool; + fn set_verbosity(level: i32); + fn sign_string(secret_key: &str, msg: &str) -> String; + + fn is_valid_path(store: &StoreWrapper, path: &str) -> Result; + fn query_path_info(store: &StoreWrapper, path: &str) -> Result; + fn compute_closure_size(store: &StoreWrapper, path: &str) -> Result; + fn clear_path_info_cache(store: &StoreWrapper) -> Result<()>; + #[allow(clippy::fn_params_excessive_bools)] + fn compute_fs_closure( + store: &StoreWrapper, + path: &str, + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + ) -> Result>; + #[allow(clippy::fn_params_excessive_bools)] + fn compute_fs_closures( + store: &StoreWrapper, + paths: &[&str], + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + toposort: bool, + ) -> Result>; + fn upsert_file(store: &StoreWrapper, path: &str, data: &str, mime_type: &str) + -> Result<()>; + fn get_store_stats(store: &StoreWrapper) -> Result; + fn get_s3_stats(store: &StoreWrapper) -> Result; + fn copy_paths( + src_store: &StoreWrapper, + dst_store: &StoreWrapper, + paths: &[&str], + repair: bool, + check_sigs: bool, + substitute: bool, + ) -> Result<()>; + + fn import_paths( + store: &StoreWrapper, + check_sigs: bool, + runtime: usize, + reader: usize, + callback: unsafe extern "C" fn( + data: &mut [u8], + runtime: usize, + reader: usize, + user_data: usize, + ) -> usize, + user_data: usize, + ) -> Result<()>; + fn import_paths_with_fd(store: &StoreWrapper, check_sigs: bool, fd: i32) -> Result<()>; + fn export_paths( + store: &StoreWrapper, + paths: &[&str], + callback: unsafe extern "C" fn(data: &[u8], user_data: usize) -> bool, + user_data: usize, + ) -> Result<()>; + fn nar_from_path( + store: &StoreWrapper, + paths: &str, + callback: unsafe extern "C" fn(data: &[u8], user_data: usize) -> bool, + user_data: usize, + ) -> Result<()>; + + fn list_nar(store: &StoreWrapper, path: &str, recursive: bool) -> Result; + + fn ensure_path(store: &StoreWrapper, path: &str) -> Result<()>; + fn try_resolve_drv(store: &StoreWrapper, path: &str) -> Result; + fn static_output_hashes( + store: &StoreWrapper, + drv_path: &str, + ) -> Result>; + } +} + +pub use ffi::{S3Stats, StoreStats}; + +impl StoreStats { + #[must_use] + pub fn nar_compression_savings(&self) -> f64 { + #[allow(clippy::cast_precision_loss)] + if self.nar_write_bytes > 0 { + 1.0 - (self.nar_write_compressed_bytes as f64 / self.nar_write_bytes as f64) + } else { + 0.0 + } + } + #[must_use] + pub fn nar_compression_speed(&self) -> f64 { + #[allow(clippy::cast_precision_loss)] + if self.nar_write_compression_time_ms > 0 { + self.nar_write_bytes as f64 / self.nar_write_compression_time_ms as f64 * 1000.0 + / (1024.0 * 1024.0) + } else { + 0.0 + } + } +} + +#[inline] +#[must_use] +pub fn is_subpath(base: &std::path::Path, path: &std::path::Path) -> bool { + path.starts_with(base) +} + +#[inline] +pub fn init_nix() { + ffi::init_nix(); +} + +#[inline] +#[must_use] +pub fn get_nix_prefix() -> String { + ffi::get_nix_prefix() +} + +#[inline] +#[must_use] +pub fn get_store_dir() -> String { + ffi::get_store_dir() +} + +#[inline] +#[must_use] +pub fn get_build_dir() -> String { + ffi::get_build_dir() +} + +#[inline] +#[must_use] +pub fn get_log_dir() -> String { + ffi::get_log_dir() +} + +#[inline] +#[must_use] +pub fn get_state_dir() -> String { + ffi::get_state_dir() +} + +#[inline] +#[must_use] +pub fn get_nix_version() -> String { + ffi::get_nix_version() +} + +#[inline] +#[must_use] +pub fn get_this_system() -> String { + ffi::get_this_system() +} + +#[inline] +#[must_use] +pub fn get_extra_platforms() -> Vec { + ffi::get_extra_platforms() +} + +#[inline] +#[must_use] +pub fn get_system_features() -> Vec { + ffi::get_system_features() +} + +#[inline] +#[must_use] +pub fn get_substituters() -> Vec { + ffi::get_substituters() +} + +#[inline] +#[must_use] +pub fn get_use_cgroups() -> bool { + ffi::get_use_cgroups() +} + +#[inline] +/// Set the loglevel. +pub fn set_verbosity(level: i32) { + ffi::set_verbosity(level); +} + +#[inline] +#[must_use] +pub fn sign_string(secret_key: &str, msg: &str) -> String { + ffi::sign_string(secret_key, msg) +} + +pub(crate) async fn asyncify(f: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + match tokio::task::spawn_blocking(f).await { + Ok(res) => Ok(res?), + Err(_) => Err(std::io::Error::other("background task failed"))?, + } +} + +#[inline] +pub async fn copy_paths( + src: &BaseStoreImpl, + dst: &BaseStoreImpl, + paths: &[StorePath], + repair: bool, + check_sigs: bool, + substitute: bool, +) -> Result<(), Error> { + let paths = paths + .iter() + .map(|p| src.print_store_path(p)) + .collect::>(); + + let src = src.wrapper.clone(); + let dst = dst.wrapper.clone(); + + asyncify(move || { + let slice = paths + .iter() + .map(std::string::String::as_str) + .collect::>(); + ffi::copy_paths(&src, &dst, &slice, repair, check_sigs, substitute) + }) + .await +} + +#[derive(Debug)] +pub struct PathInfo { + pub deriver: Option, + pub nar_hash: String, + pub registration_time: i64, + pub nar_size: u64, + pub refs: Vec, + pub sigs: Vec, + pub ca: Option, +} + +impl From for PathInfo { + fn from(val: crate::ffi::InternalPathInfo) -> Self { + Self { + deriver: if val.deriver.is_empty() { + None + } else { + Some(StorePath::new(&val.deriver)) + }, + nar_hash: val.nar_hash, + registration_time: val.registration_time, + nar_size: val.nar_size, + refs: val.refs.iter().map(|v| StorePath::new(v)).collect(), + sigs: val.sigs, + ca: if val.ca.is_empty() { + None + } else { + Some(val.ca) + }, + } + } +} + +pub trait BaseStore { + #[must_use] + /// Check whether a path is valid. + fn is_valid_path(&self, path: &StorePath) -> impl std::future::Future; + + fn query_path_info( + &self, + path: &StorePath, + ) -> impl std::future::Future>; + fn query_path_infos( + &self, + paths: &[&StorePath], + ) -> impl std::future::Future>; + fn compute_closure_size(&self, path: &StorePath) -> impl std::future::Future; + + fn clear_path_info_cache(&self); + + #[allow(clippy::fn_params_excessive_bools)] + fn compute_fs_closure( + &self, + path: &str, + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + ) -> Result, cxx::Exception>; + + #[allow(clippy::fn_params_excessive_bools)] + fn compute_fs_closures( + &self, + paths: &[&StorePath], + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + toposort: bool, + ) -> impl std::future::Future, Error>>; + + fn query_requisites( + &self, + drvs: &[&StorePath], + include_outputs: bool, + ) -> impl std::future::Future, crate::Error>>; + + fn get_store_stats(&self) -> Result; + + /// Import paths from nar + fn import_paths( + &self, + stream: S, + check_sigs: bool, + ) -> impl std::future::Future> + where + S: tokio_stream::Stream> + + Send + + Unpin + + 'static; + + /// Import paths from nar + fn import_paths_with_fd(&self, fd: Fd, check_sigs: bool) -> Result<(), cxx::Exception> + where + Fd: std::os::fd::AsFd + std::os::fd::AsRawFd; + + /// Export a store path in NAR format. The data is passed in chunks to callback + fn export_paths(&self, paths: &[StorePath], callback: F) -> Result<(), cxx::Exception> + where + F: FnMut(&[u8]) -> bool; + + /// Export a store path in NAR format. The data is passed in chunks to callback + fn nar_from_path(&self, path: &StorePath, callback: F) -> Result<(), cxx::Exception> + where + F: FnMut(&[u8]) -> bool; + + fn list_nar( + &self, + path: &StorePath, + recursive: bool, + ) -> impl std::future::Future>; + + fn ensure_path(&self, path: &StorePath) + -> impl std::future::Future>; + fn try_resolve_drv( + &self, + path: &StorePath, + ) -> impl std::future::Future>; + fn static_output_hashes( + &self, + drv_path: &StorePath, + ) -> impl std::future::Future, crate::Error>>; + + #[must_use] + fn print_store_path(&self, path: &StorePath) -> String; +} + +unsafe impl Send for crate::ffi::StoreWrapper {} +unsafe impl Sync for crate::ffi::StoreWrapper {} + +#[derive(Clone)] +pub struct BaseStoreImpl { + wrapper: cxx::SharedPtr, + store_path_prefix: String, +} + +impl BaseStoreImpl { + fn new(store: cxx::SharedPtr) -> Self { + Self { + wrapper: store, + store_path_prefix: get_store_dir(), + } + } +} + +fn import_paths_trampoline( + data: &mut [u8], + runtime: usize, + reader: usize, + userdata: usize, +) -> usize +where + F: FnMut( + &tokio::runtime::Runtime, + &mut Box>, + &mut [u8], + ) -> usize, + S: futures::stream::Stream>, + E: Into, +{ + let runtime = + unsafe { &*(runtime as *mut std::ffi::c_void).cast::>() }; + let reader = unsafe { + &mut *(reader as *mut std::ffi::c_void) + .cast::>>() + }; + let closure = unsafe { &mut *(userdata as *mut std::ffi::c_void).cast::() }; + closure(runtime, reader, data) +} + +fn export_paths_trampoline(data: &[u8], userdata: usize) -> bool +where + F: FnMut(&[u8]) -> bool, +{ + let closure = unsafe { &mut *(userdata as *mut std::ffi::c_void).cast::() }; + closure(data) +} + +impl BaseStore for BaseStoreImpl { + #[inline] + async fn is_valid_path(&self, path: &StorePath) -> bool { + let store = self.wrapper.clone(); + let path = self.print_store_path(path); + asyncify(move || ffi::is_valid_path(&store, &path)) + .await + .unwrap_or(false) + } + + #[inline] + async fn query_path_info(&self, path: &StorePath) -> Option { + let store = self.wrapper.clone(); + let path = self.print_store_path(path); + asyncify(move || Ok(ffi::query_path_info(&store, &path).ok().map(Into::into))) + .await + .ok() + .flatten() + } + + #[inline] + async fn query_path_infos(&self, paths: &[&StorePath]) -> HashMap { + let paths = paths.iter().map(|v| (*v).to_owned()).collect::>(); + + asyncify({ + let self_ = self.clone(); + move || { + let mut res = HashMap::with_capacity(paths.len()); + for p in paths { + let full_path = self_.print_store_path(&p); + if let Some(info) = ffi::query_path_info(&self_.wrapper, &full_path) + .ok() + .map(Into::into) + { + res.insert(p, info); + } + } + Ok(res) + } + }) + .await + .unwrap_or_default() + } + + #[inline] + async fn compute_closure_size(&self, path: &StorePath) -> u64 { + let store = self.wrapper.clone(); + let path = self.print_store_path(path); + asyncify(move || ffi::compute_closure_size(&store, &path)) + .await + .unwrap_or_default() + } + + #[inline] + fn clear_path_info_cache(&self) { + let _ = ffi::clear_path_info_cache(&self.wrapper); + } + + #[inline] + #[tracing::instrument(skip(self), err)] + fn compute_fs_closure( + &self, + path: &str, + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + ) -> Result, cxx::Exception> { + ffi::compute_fs_closure( + &self.wrapper, + path, + flip_direction, + include_outputs, + include_derivers, + ) + } + + #[inline] + #[tracing::instrument(skip(self), err)] + async fn compute_fs_closures( + &self, + paths: &[&StorePath], + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + toposort: bool, + ) -> Result, Error> { + let store = self.wrapper.clone(); + let paths = paths + .iter() + .map(|v| self.print_store_path(v)) + .collect::>(); + + asyncify(move || { + let slice = paths + .iter() + .map(std::string::String::as_str) + .collect::>(); + Ok(ffi::compute_fs_closures( + &store, + &slice, + flip_direction, + include_outputs, + include_derivers, + toposort, + )? + .into_iter() + .map(|v| StorePath::new(&v)) + .collect()) + }) + .await + } + + async fn query_requisites( + &self, + drvs: &[&StorePath], + include_outputs: bool, + ) -> Result, Error> { + let mut out = self + .compute_fs_closures(drvs, false, include_outputs, false, true) + .await?; + out.reverse(); + Ok(out) + } + + fn get_store_stats(&self) -> Result { + ffi::get_store_stats(&self.wrapper) + } + + #[inline] + #[tracing::instrument(skip(self, stream), err)] + async fn import_paths(&self, stream: S, check_sigs: bool) -> Result<(), Error> + where + S: tokio_stream::Stream> + + Send + + Unpin + + 'static, + { + use tokio::io::AsyncReadExt as _; + + let callback = |runtime: &tokio::runtime::Runtime, + reader: &mut Box>, + data: &mut [u8]| { + runtime.block_on(async { reader.read(data).await.unwrap_or(0) }) + }; + + let reader = Box::new(tokio_util::io::StreamReader::new(stream)); + let store = self.clone(); + tokio::task::spawn_blocking(move || { + store.import_paths_with_cb(callback, reader, check_sigs) + }) + .await??; + Ok(()) + } + + #[inline] + #[tracing::instrument(skip(self, fd), err)] + fn import_paths_with_fd(&self, fd: Fd, check_sigs: bool) -> Result<(), cxx::Exception> + where + Fd: std::os::fd::AsFd + std::os::fd::AsRawFd, + { + ffi::import_paths_with_fd(&self.wrapper, check_sigs, fd.as_raw_fd()) + } + + #[inline] + #[tracing::instrument(skip(self, paths, callback), err)] + fn export_paths(&self, paths: &[StorePath], callback: F) -> Result<(), cxx::Exception> + where + F: FnMut(&[u8]) -> bool, + { + let paths = paths + .iter() + .map(|v| self.print_store_path(v)) + .collect::>(); + let slice = paths + .iter() + .map(std::string::String::as_str) + .collect::>(); + ffi::export_paths( + &self.wrapper, + &slice, + export_paths_trampoline::, + std::ptr::addr_of!(callback).cast::() as usize, + ) + } + + #[inline] + #[tracing::instrument(skip(self, path, callback), err)] + fn nar_from_path(&self, path: &StorePath, callback: F) -> Result<(), cxx::Exception> + where + F: FnMut(&[u8]) -> bool, + { + let path = self.print_store_path(path); + ffi::nar_from_path( + &self.wrapper, + &path, + export_paths_trampoline::, + std::ptr::addr_of!(callback).cast::() as usize, + ) + } + + #[inline] + #[tracing::instrument(skip(self), err)] + async fn list_nar(&self, path: &StorePath, recursive: bool) -> Result { + let store = self.wrapper.clone(); + let path = self.print_store_path(path); + asyncify(move || ffi::list_nar(&store, &path, recursive)).await + } + + #[inline] + async fn ensure_path(&self, path: &StorePath) -> Result<(), Error> { + let store = self.wrapper.clone(); + let path = self.print_store_path(path); + asyncify(move || { + ffi::ensure_path(&store, &path)?; + Ok(()) + }) + .await + } + + #[inline] + async fn try_resolve_drv(&self, path: &StorePath) -> Option { + let store = self.wrapper.clone(); + let path = self.print_store_path(path); + asyncify(move || { + let v = ffi::try_resolve_drv(&store, &path)?; + Ok(v.is_empty().then_some(v).map(|v| StorePath::new(&v))) + }) + .await + .ok() + .flatten() + } + + #[inline] + async fn static_output_hashes( + &self, + drv_path: &StorePath, + ) -> Result, crate::Error> { + let store = self.wrapper.clone(); + let drv_path = self.print_store_path(drv_path); + asyncify(move || { + let o = ffi::static_output_hashes(&store, &drv_path)?; + Ok(o.into_iter().map(|v| (v.output_name, v.drv_hash)).collect()) + }) + .await + } + + #[inline] + fn print_store_path(&self, path: &StorePath) -> String { + format!("{}/{}", self.store_path_prefix, path.base_name()) + } +} + +impl BaseStoreImpl { + #[inline] + #[tracing::instrument(skip(self, callback, reader), err)] + fn import_paths_with_cb( + &self, + callback: F, + reader: Box>, + check_sigs: bool, + ) -> Result<(), Error> + where + F: FnMut( + &tokio::runtime::Runtime, + &mut Box>, + &mut [u8], + ) -> usize, + S: futures::stream::Stream>, + E: Into, + { + let runtime = Box::new(tokio::runtime::Runtime::new()?); + ffi::import_paths( + &self.wrapper, + check_sigs, + std::ptr::addr_of!(runtime).cast::() as usize, + std::ptr::addr_of!(reader).cast::() as usize, + import_paths_trampoline::, + std::ptr::addr_of!(callback).cast::() as usize, + )?; + drop(reader); + drop(runtime); + Ok(()) + } +} + +#[derive(Clone)] +pub struct LocalStore { + base: BaseStoreImpl, +} + +impl LocalStore { + #[inline] + /// Initialise a new store + #[must_use] + pub fn init() -> Self { + let base = BaseStoreImpl::new(ffi::init("")); + Self { base } + } + + #[must_use] + pub const fn as_base_store(&self) -> &BaseStoreImpl { + &self.base + } + + #[tracing::instrument(skip(self, outputs))] + pub async fn query_missing_outputs( + &self, + outputs: Vec, + ) -> Vec { + use futures::stream::StreamExt as _; + + tokio_stream::iter(outputs) + .map(|o| async move { + let Some(path) = &o.path else { + return None; + }; + if self.is_valid_path(path).await { + None + } else { + Some(o) + } + }) + .buffered(50) + .filter_map(|o| async { o }) + .collect() + .await + } + + #[must_use] + pub const fn get_store_path_prefix(&self) -> &str { + self.base.store_path_prefix.as_str() + } + + pub fn unsafe_set_store_path_prefix(&mut self, prefix: String) { + self.base.store_path_prefix = prefix; + } +} + +impl BaseStore for LocalStore { + #[inline] + async fn is_valid_path(&self, path: &StorePath) -> bool { + self.base.is_valid_path(path).await + } + + #[inline] + async fn query_path_info(&self, path: &StorePath) -> Option { + self.base.query_path_info(path).await + } + + #[inline] + async fn query_path_infos(&self, paths: &[&StorePath]) -> HashMap { + self.base.query_path_infos(paths).await + } + + #[inline] + async fn compute_closure_size(&self, path: &StorePath) -> u64 { + self.base.compute_closure_size(path).await + } + + #[inline] + fn clear_path_info_cache(&self) { + self.base.clear_path_info_cache(); + } + + #[inline] + #[tracing::instrument(skip(self), err)] + fn compute_fs_closure( + &self, + path: &str, + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + ) -> Result, cxx::Exception> { + self.base + .compute_fs_closure(path, flip_direction, include_outputs, include_derivers) + } + + #[inline] + #[tracing::instrument(skip(self), err)] + async fn compute_fs_closures( + &self, + paths: &[&StorePath], + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + toposort: bool, + ) -> Result, Error> { + self.base + .compute_fs_closures( + paths, + flip_direction, + include_outputs, + include_derivers, + toposort, + ) + .await + } + + #[inline] + #[tracing::instrument(skip(self), err)] + async fn query_requisites( + &self, + drvs: &[&StorePath], + include_outputs: bool, + ) -> Result, Error> { + self.base.query_requisites(drvs, include_outputs).await + } + + #[inline] + fn get_store_stats(&self) -> Result { + self.base.get_store_stats() + } + + #[inline] + #[tracing::instrument(skip(self, stream), err)] + async fn import_paths(&self, stream: S, check_sigs: bool) -> Result<(), Error> + where + S: tokio_stream::Stream> + + Send + + Unpin + + 'static, + { + self.base.import_paths::(stream, check_sigs).await + } + + #[inline] + #[tracing::instrument(skip(self, fd), err)] + fn import_paths_with_fd(&self, fd: Fd, check_sigs: bool) -> Result<(), cxx::Exception> + where + Fd: std::os::fd::AsFd + std::os::fd::AsRawFd, + { + self.base.import_paths_with_fd(fd, check_sigs) + } + + #[inline] + #[tracing::instrument(skip(self, paths, callback), err)] + fn export_paths(&self, paths: &[StorePath], callback: F) -> Result<(), cxx::Exception> + where + F: FnMut(&[u8]) -> bool, + { + self.base.export_paths(paths, callback) + } + + #[inline] + #[tracing::instrument(skip(self, path, callback), err)] + fn nar_from_path(&self, path: &StorePath, callback: F) -> Result<(), cxx::Exception> + where + F: FnMut(&[u8]) -> bool, + { + self.base.nar_from_path(path, callback) + } + + #[inline] + #[tracing::instrument(skip(self), err)] + async fn list_nar(&self, path: &StorePath, recursive: bool) -> Result { + self.base.list_nar(path, recursive).await + } + + #[inline] + async fn ensure_path(&self, path: &StorePath) -> Result<(), Error> { + self.base.ensure_path(path).await + } + + #[inline] + async fn try_resolve_drv(&self, path: &StorePath) -> Option { + self.base.try_resolve_drv(path).await + } + + #[inline] + async fn static_output_hashes( + &self, + drv_path: &StorePath, + ) -> Result, crate::Error> { + self.base.static_output_hashes(drv_path).await + } + + #[inline] + fn print_store_path(&self, path: &StorePath) -> String { + self.base.print_store_path(path) + } +} + +#[derive(Clone)] +pub struct RemoteStore { + base: BaseStoreImpl, + + pub uri: String, + pub base_uri: String, +} + +impl RemoteStore { + #[inline] + /// Initialise a new store with uri + #[must_use] + pub fn init(uri: &str) -> Self { + let base_uri = url::Url::parse(uri) + .ok() + .and_then(|v| v.host_str().map(ToOwned::to_owned)) + .unwrap_or_default(); + + Self { + base: BaseStoreImpl::new(ffi::init(uri)), + uri: uri.into(), + base_uri, + } + } + + #[must_use] + pub const fn as_base_store(&self) -> &BaseStoreImpl { + &self.base + } + + #[inline] + pub async fn upsert_file( + &self, + path: String, + local_path: std::path::PathBuf, + mime_type: &'static str, + ) -> Result<(), Error> { + let store = self.base.wrapper.clone(); + asyncify(move || { + if let Ok(data) = fs_err::read_to_string(local_path) { + ffi::upsert_file(&store, &path, &data, mime_type)?; + } + Ok(()) + }) + .await + } + + #[inline] + pub fn get_s3_stats(&self) -> Result { + ffi::get_s3_stats(&self.base.wrapper) + } + + #[tracing::instrument(skip(self, paths))] + pub async fn query_missing_paths(&self, paths: Vec) -> Vec { + use futures::stream::StreamExt as _; + + tokio_stream::iter(paths) + .map(|p| async move { + if self.is_valid_path(&p).await { + None + } else { + Some(p) + } + }) + .buffered(50) + .filter_map(|p| async { p }) + .collect() + .await + } + + #[tracing::instrument(skip(self, outputs))] + pub async fn query_missing_remote_outputs( + &self, + outputs: Vec, + ) -> Vec { + use futures::stream::StreamExt as _; + + tokio_stream::iter(outputs) + .map(|o| async move { + let Some(path) = &o.path else { + return None; + }; + if self.is_valid_path(path).await { + None + } else { + Some(o) + } + }) + .buffered(50) + .filter_map(|o| async { o }) + .collect() + .await + } +} + +impl BaseStore for RemoteStore { + #[inline] + async fn is_valid_path(&self, path: &StorePath) -> bool { + self.base.is_valid_path(path).await + } + + #[inline] + async fn query_path_info(&self, path: &StorePath) -> Option { + self.base.query_path_info(path).await + } + + #[inline] + async fn query_path_infos(&self, paths: &[&StorePath]) -> HashMap { + self.base.query_path_infos(paths).await + } + + #[inline] + async fn compute_closure_size(&self, path: &StorePath) -> u64 { + self.base.compute_closure_size(path).await + } + + #[inline] + fn clear_path_info_cache(&self) { + self.base.clear_path_info_cache(); + } + + #[inline] + #[tracing::instrument(skip(self), err)] + fn compute_fs_closure( + &self, + path: &str, + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + ) -> Result, cxx::Exception> { + self.base + .compute_fs_closure(path, flip_direction, include_outputs, include_derivers) + } + + #[inline] + #[tracing::instrument(skip(self), err)] + async fn compute_fs_closures( + &self, + paths: &[&StorePath], + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + toposort: bool, + ) -> Result, Error> { + self.base + .compute_fs_closures( + paths, + flip_direction, + include_outputs, + include_derivers, + toposort, + ) + .await + } + + #[inline] + #[tracing::instrument(skip(self), err)] + async fn query_requisites( + &self, + drvs: &[&StorePath], + include_outputs: bool, + ) -> Result, Error> { + self.base.query_requisites(drvs, include_outputs).await + } + + #[inline] + fn get_store_stats(&self) -> Result { + self.base.get_store_stats() + } + + #[inline] + #[tracing::instrument(skip(self, stream), err)] + async fn import_paths(&self, stream: S, check_sigs: bool) -> Result<(), Error> + where + S: tokio_stream::Stream> + + Send + + Unpin + + 'static, + { + self.base.import_paths::(stream, check_sigs).await + } + + #[inline] + #[tracing::instrument(skip(self, fd), err)] + fn import_paths_with_fd(&self, fd: Fd, check_sigs: bool) -> Result<(), cxx::Exception> + where + Fd: std::os::fd::AsFd + std::os::fd::AsRawFd, + { + self.base.import_paths_with_fd(fd, check_sigs) + } + + #[inline] + #[tracing::instrument(skip(self, paths, callback), err)] + fn export_paths(&self, paths: &[StorePath], callback: F) -> Result<(), cxx::Exception> + where + F: FnMut(&[u8]) -> bool, + { + self.base.export_paths(paths, callback) + } + + #[inline] + #[tracing::instrument(skip(self, path, callback), err)] + fn nar_from_path(&self, path: &StorePath, callback: F) -> Result<(), cxx::Exception> + where + F: FnMut(&[u8]) -> bool, + { + self.base.nar_from_path(path, callback) + } + + #[inline] + #[tracing::instrument(skip(self), err)] + async fn list_nar(&self, path: &StorePath, recursive: bool) -> Result { + self.base.list_nar(path, recursive).await + } + + #[inline] + async fn ensure_path(&self, path: &StorePath) -> Result<(), Error> { + self.base.ensure_path(path).await + } + + #[inline] + async fn try_resolve_drv(&self, path: &StorePath) -> Option { + self.base.try_resolve_drv(path).await + } + + #[inline] + async fn static_output_hashes( + &self, + drv_path: &StorePath, + ) -> Result, crate::Error> { + self.base.static_output_hashes(drv_path).await + } + + #[inline] + fn print_store_path(&self, path: &StorePath) -> String { + self.base.print_store_path(path) + } +} diff --git a/src/crates/nix-utils/src/nix.cpp b/src/crates/nix-utils/src/nix.cpp new file mode 100644 index 000000000..44c98cdab --- /dev/null +++ b/src/crates/nix-utils/src/nix.cpp @@ -0,0 +1,371 @@ +#include "nix-utils/include/nix.h" +#include "nix-utils/include/utils.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include "nix/store/export-import.hh" +#include +#include +#include +#include +#include + +#include + +static std::atomic initializedNix = false; +static std::mutex nixInitMtx; + +namespace nix_utils { +void init_nix() { + if (!initializedNix) { + // We need this mutex here. if we have multiple threads that want to do + // init_nix at the same time. + // We need to ensure that initNix is finished on all threads before setting + // initializedNix. + // We also need to ensure that initNix not runs multiple times at the same + // time + std::lock_guard lock(nixInitMtx); + if (!initializedNix) { + nix::initNix(); + initializedNix = true; + } + } +} + +StoreWrapper::StoreWrapper(nix::ref _store) : _store(_store) {} + +std::shared_ptr init(rust::Str uri) { + init_nix(); + if (uri.empty()) { + nix::ref _store = nix::openStore(); + return std::make_shared(_store); + } else { + nix::ref _store = nix::openStore(AS_STRING(uri)); + return std::make_shared(_store); + } +} + +rust::String get_nix_prefix() { return nix::settings.nixPrefix; } +rust::String get_store_dir() { + init_nix(); + return nix::settings.nixStore; +} +rust::String get_build_dir() { + return nix::settings.buildDir.get().has_value() + ? *nix::settings.buildDir.get() + : nix::settings.nixStateDir + "/builds"; +} +rust::String get_log_dir() { return nix::settings.nixLogDir; } +rust::String get_state_dir() { return nix::settings.nixStateDir; } +rust::String get_nix_version() { return nix::nixVersion; } +rust::String get_this_system() { return nix::settings.thisSystem.get(); } +rust::Vec get_extra_platforms() { + auto set = nix::settings.extraPlatforms.get(); + rust::Vec data; + data.reserve(set.size()); + for (const auto &val : set) { + data.emplace_back(val); + } + return data; +} +rust::Vec get_system_features() { + auto set = nix::settings.systemFeatures.get(); + rust::Vec data; + data.reserve(set.size()); + for (const auto &val : set) { + data.emplace_back(val); + } + return data; +} +rust::Vec get_substituters() { + auto strs = nix::settings.substituters.get(); + rust::Vec data; + data.reserve(strs.size()); + for (const auto &val : strs) { + data.emplace_back(val); + } + return data; +} + +bool get_use_cgroups() { +#ifdef __linux__ + return nix::settings.useCgroups; +#endif + return false; +} +void set_verbosity(int32_t level) { nix::verbosity = (nix::Verbosity)level; } + +rust::String sign_string(rust::Str secret_key, rust::Str msg) { + return nix::SecretKey(AS_VIEW(secret_key)).signDetached(AS_VIEW(msg)); +} + +bool is_valid_path(const StoreWrapper &wrapper, rust::Str path) { + auto store = wrapper._store; + return store->isValidPath(store->parseStorePath(AS_VIEW(path))); +} + +InternalPathInfo query_path_info(const StoreWrapper &wrapper, rust::Str path) { + auto store = wrapper._store; + auto info = store->queryPathInfo(store->parseStorePath(AS_VIEW(path))); + + std::string narhash = info->narHash.to_string(nix::HashFormat::Nix32, true); + + rust::Vec refs = extract_path_set(*store, info->references); + + rust::Vec sigs; + sigs.reserve(info->sigs.size()); + for (const std::string &sig : info->sigs) { + sigs.push_back(sig); + } + + // TODO(conni2461): Replace "" with option + return InternalPathInfo{ + extract_opt_path(*store, info->deriver), + narhash, + info->registrationTime, + info->narSize, + refs, + sigs, + info->ca ? nix::renderContentAddress(*info->ca) : "", + }; +} + +uint64_t compute_closure_size(const StoreWrapper &wrapper, rust::Str path) { + auto store = wrapper._store; + nix::StorePathSet closure; + store->computeFSClosure(store->parseStorePath(AS_VIEW(path)), closure, false, + false); + + uint64_t totalNarSize = 0; + for (auto &p : closure) { + totalNarSize += store->queryPathInfo(p)->narSize; + } + return totalNarSize; +} + +void clear_path_info_cache(const StoreWrapper &wrapper) { + auto store = wrapper._store; + store->clearPathInfoCache(); +} + +rust::Vec compute_fs_closure(const StoreWrapper &wrapper, + rust::Str path, bool flip_direction, + bool include_outputs, + bool include_derivers) { + auto store = wrapper._store; + nix::StorePathSet path_set; + store->computeFSClosure(store->parseStorePath(AS_VIEW(path)), path_set, + flip_direction, include_outputs, include_derivers); + return extract_path_set(*store, path_set); +} + +rust::Vec compute_fs_closures(const StoreWrapper &wrapper, + rust::Slice paths, + bool flip_direction, + bool include_outputs, + bool include_derivers, + bool toposort) { + auto store = wrapper._store; + nix::StorePathSet path_set; + for (auto &path : paths) { + store->computeFSClosure(store->parseStorePath(AS_VIEW(path)), path_set, + flip_direction, include_outputs, include_derivers); + } + if (toposort) { + auto sorted = store->topoSortPaths(path_set); + return extract_paths(*store, sorted); + } else { + return extract_path_set(*store, path_set); + } +} + +void upsert_file(const StoreWrapper &wrapper, rust::Str path, rust::Str data, + rust::Str mime_type) { + auto store = wrapper._store.dynamic_pointer_cast(); + if (!store) { + throw nix::Error("Not a binary chache store"); + } + store->upsertFile(AS_STRING(path), AS_STRING(data), AS_STRING(mime_type)); +} + +StoreStats get_store_stats(const StoreWrapper &wrapper) { + auto store = wrapper._store; + auto &stats = store->getStats(); + return StoreStats{ + stats.narInfoRead.load(), + stats.narInfoReadAverted.load(), + stats.narInfoMissing.load(), + stats.narInfoWrite.load(), + stats.pathInfoCacheSize.load(), + stats.narRead.load(), + stats.narReadBytes.load(), + stats.narReadCompressedBytes.load(), + stats.narWrite.load(), + stats.narWriteAverted.load(), + stats.narWriteBytes.load(), + stats.narWriteCompressedBytes.load(), + stats.narWriteCompressionTimeMs.load(), + }; +} + +S3Stats get_s3_stats(const StoreWrapper &wrapper) { + auto store = wrapper._store; + auto s3Store = dynamic_cast(&*store); + if (!s3Store) { + throw nix::Error("Not a s3 binary chache store"); + } + auto &stats = s3Store->getS3Stats(); + return S3Stats{ + stats.put.load(), stats.putBytes.load(), stats.putTimeMs.load(), + stats.get.load(), stats.getBytes.load(), stats.getTimeMs.load(), + stats.head.load(), + }; +} + +void copy_paths(const StoreWrapper &src_store, const StoreWrapper &dst_store, + rust::Slice paths, bool repair, + bool check_sigs, bool substitute) { + nix::StorePathSet path_set; + for (auto &path : paths) { + path_set.insert(src_store._store->parseStorePath(AS_VIEW(path))); + } + nix::copyPaths(*src_store._store, *dst_store._store, path_set, + repair ? nix::Repair : nix::NoRepair, + check_sigs ? nix::CheckSigs : nix::NoCheckSigs, + substitute ? nix::Substitute : nix::NoSubstitute); +} + +void import_paths( + const StoreWrapper &wrapper, bool check_sigs, size_t runtime, size_t reader, + rust::Fn, size_t, size_t, size_t)> callback, + size_t user_data) { + nix::LambdaSource source([=](char *out, size_t out_len) { + auto data = rust::Slice((uint8_t *)out, out_len); + size_t ret = (*callback)(data, runtime, reader, user_data); + if (!ret) { + throw nix::EndOfFile("End of stream reached"); + } + return ret; + }); + + auto store = wrapper._store; + auto paths = nix::importPaths(*store, source, + check_sigs ? nix::CheckSigs : nix::NoCheckSigs); +} + +void import_paths_with_fd(const StoreWrapper &wrapper, bool check_sigs, + int32_t fd) { + nix::FdSource source(fd); + + auto store = wrapper._store; + nix::importPaths(*store, source, + check_sigs ? nix::CheckSigs : nix::NoCheckSigs); +} + +class StopExport : public std::exception { +public: + const char *what() { return "Stop exporting nar"; } +}; + +void export_paths(const StoreWrapper &wrapper, + rust::Slice paths, + rust::Fn, size_t)> callback, + size_t user_data) { + nix::LambdaSink sink([=](std::string_view v) { + auto data = rust::Slice((const uint8_t *)v.data(), v.size()); + bool ret = (*callback)(data, user_data); + if (!ret) { + throw StopExport(); + } + }); + + auto store = wrapper._store; + nix::StorePathSet path_set; + for (auto &path : paths) { + path_set.insert(store->followLinksToStorePath(AS_VIEW(path))); + } + try { + nix::exportPaths(*store, path_set, sink); + } catch (StopExport &e) { + // Intentionally do nothing. We're only using the exception as a + // short-circuiting mechanism. + } +} + +void nar_from_path(const StoreWrapper &wrapper, rust::Str path, + rust::Fn, size_t)> callback, + size_t user_data) { + nix::LambdaSink sink([=](std::string_view v) { + auto data = rust::Slice((const uint8_t *)v.data(), v.size()); + bool ret = (*callback)(data, user_data); + if (!ret) { + throw StopExport(); + } + }); + + auto store = wrapper._store; + try { + store->narFromPath(store->followLinksToStorePath(AS_VIEW(path)), sink); + } catch (StopExport &e) { + // Intentionally do nothing. We're only using the exception as a + // short-circuiting mechanism. + } +} + +rust::String list_nar(const StoreWrapper &wrapper, rust::Str path, + bool recursive) { + auto store = wrapper._store; + auto [store_path, rest] = store->toStorePath(AS_VIEW(path)); + + nlohmann::json j = { + {"version", 1}, + {"root", nix::listNar(store->getFSAccessor(), + nix::CanonPath{store_path.to_string()} / + nix::CanonPath{rest}, + recursive)}, + }; + + return j.dump(); +} + +void ensure_path(const StoreWrapper &wrapper, rust::Str path) { + auto store = wrapper._store; + store->ensurePath(store->followLinksToStorePath(AS_VIEW(path))); +} + +rust::String try_resolve_drv(const StoreWrapper &wrapper, rust::Str path) { + auto store = wrapper._store; + + auto drv = store->readDerivation(store->parseStorePath(AS_VIEW(path))); + auto resolved = drv.tryResolve(*store); + if (!resolved) { + return ""; + } + + auto resolved_path = writeDerivation(*store, *resolved, nix::NoRepair, false); + // TODO: return drv not drv path + return extract_opt_path(*store, resolved_path); +} + +rust::Vec static_output_hashes(const StoreWrapper &wrapper, + rust::Str drv_path) { + auto store = wrapper._store; + + auto drvHashes = staticOutputHashes( + *store, store->readDerivation(store->parseStorePath(AS_VIEW(drv_path)))); + rust::Vec data; + data.reserve(drvHashes.size()); + for (auto &[name, hash] : drvHashes) { + data.emplace_back( + DerivationHash{name, hash.to_string(nix::HashFormat::Base16, true)}); + } + return data; +} +} // namespace nix_utils diff --git a/src/crates/nix-utils/src/realisation.rs b/src/crates/nix-utils/src/realisation.rs new file mode 100644 index 000000000..556fab062 --- /dev/null +++ b/src/crates/nix-utils/src/realisation.rs @@ -0,0 +1,179 @@ +use hashbrown::HashMap; + +#[cxx::bridge(namespace = "nix_utils")] +mod ffi { + #[derive(Debug)] + struct DrvOutput { + drv_hash: String, + output_name: String, + } + + #[derive(Debug)] + struct DrvOutputPathTuple { + id: DrvOutput, + path: String, + } + + #[derive(Debug)] + struct SharedRealisation { + id: DrvOutput, + out_path: String, + signatures: Vec, + dependent_realisations: Vec, + } + + unsafe extern "C++" { + include!("nix-utils/include/realisation.h"); + + type StoreWrapper = crate::ffi::StoreWrapper; + type InternalRealisation; + + fn as_json(self: &InternalRealisation) -> String; + fn to_rust(self: &InternalRealisation, store: &StoreWrapper) -> Result; + fn get_drv_output(self: &InternalRealisation) -> DrvOutput; + + fn fingerprint(self: &InternalRealisation) -> String; + fn sign(self: Pin<&mut InternalRealisation>, secret_key: &str) -> Result<()>; + fn clear_signatures(self: Pin<&mut InternalRealisation>); + + fn write_to_disk_cache(self: &InternalRealisation, store: &StoreWrapper) -> Result<()>; + + fn query_raw_realisation( + store: &StoreWrapper, + output_id: &str, + ) -> Result>; + + fn parse_realisation(json_string: &str) -> Result>; + } +} + +#[derive(Clone)] +pub struct FfiRealisation { + inner: cxx::SharedPtr, +} +unsafe impl Send for FfiRealisation {} +unsafe impl Sync for FfiRealisation {} + +impl FfiRealisation { + #[must_use] + pub fn as_json(&self) -> String { + self.inner.as_json() + } + + pub fn as_rust(&self, store: &crate::BaseStoreImpl) -> Result { + Ok(self.inner.to_rust(&store.wrapper)?.into()) + } + + #[must_use] + pub fn get_id(&self) -> DrvOutput { + self.inner.get_drv_output().into() + } + + #[must_use] + pub fn fingerprint(&self) -> String { + self.inner.fingerprint() + } + + pub fn sign(&mut self, secret_key: &str) -> Result<(), crate::Error> { + unsafe { self.inner.pin_mut_unchecked() }.sign(secret_key)?; + Ok(()) + } + + pub fn clear_signatures(&mut self) { + unsafe { self.inner.pin_mut_unchecked() }.clear_signatures(); + } + + pub fn write_to_disk_cache(&self, store: &crate::BaseStoreImpl) -> Result<(), crate::Error> { + self.inner.write_to_disk_cache(&store.wrapper)?; + Ok(()) + } +} + +#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub struct DrvOutput { + pub drv_hash: String, + pub output_name: String, +} + +impl From for DrvOutput { + fn from(value: ffi::DrvOutput) -> Self { + Self { + drv_hash: value.drv_hash, + output_name: value.output_name, + } + } +} + +impl std::fmt::Display for DrvOutput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + write!(f, "{}!{}", self.drv_hash, self.output_name) + } +} + +#[derive(Debug, Clone)] +pub struct Realisation { + pub id: DrvOutput, + pub out_path: crate::StorePath, + pub signatures: Vec, + pub dependent_realisations: HashMap, +} + +impl From for Realisation { + fn from(value: ffi::SharedRealisation) -> Self { + Self { + id: value.id.into(), + out_path: crate::StorePath::new(&value.out_path), + signatures: value.signatures, + dependent_realisations: value + .dependent_realisations + .into_iter() + .map(|v| (v.id.into(), crate::StorePath::new(&v.path))) + .collect(), + } + } +} + +pub trait RealisationOperations { + fn query_raw_realisation( + &self, + output_hash: &str, + output_name: &str, + ) -> Result; + + fn parse_realisation(&self, json_string: &str) -> Result; +} + +impl RealisationOperations for crate::BaseStoreImpl { + fn query_raw_realisation( + &self, + output_hash: &str, + output_name: &str, + ) -> Result { + Ok(FfiRealisation { + inner: ffi::query_raw_realisation( + &self.wrapper, + &format!("{output_hash}!{output_name}"), + )?, + }) + } + + fn parse_realisation(&self, json_string: &str) -> Result { + Ok(FfiRealisation { + inner: ffi::parse_realisation(json_string)?, + }) + } +} + +impl RealisationOperations for crate::LocalStore { + fn query_raw_realisation( + &self, + output_hash: &str, + output_name: &str, + ) -> Result { + self.base.query_raw_realisation(output_hash, output_name) + } + + fn parse_realisation(&self, json_string: &str) -> Result { + self.base.parse_realisation(json_string) + } +} diff --git a/src/crates/nix-utils/src/realise.rs b/src/crates/nix-utils/src/realise.rs new file mode 100644 index 000000000..e811d34bc --- /dev/null +++ b/src/crates/nix-utils/src/realise.rs @@ -0,0 +1,141 @@ +use tokio::io::{AsyncBufReadExt as _, BufReader}; +use tokio_stream::wrappers::LinesStream; + +use crate::BaseStore as _; +use crate::StorePath; + +#[derive(Debug, Clone)] +pub struct BuildOptions { + max_log_size: u64, + max_silent_time: i32, + build_timeout: i32, + check: bool, +} + +impl BuildOptions { + #[must_use] + pub fn new(max_log_size: Option) -> Self { + Self { + max_log_size: max_log_size.unwrap_or(64u64 << 20), + max_silent_time: 0, + build_timeout: 0, + check: false, + } + } + + #[must_use] + pub const fn complete(max_log_size: u64, max_silent_time: i32, build_timeout: i32) -> Self { + Self { + max_log_size, + max_silent_time, + build_timeout, + check: false, + } + } + + pub const fn set_max_silent_time(&mut self, max_silent_time: i32) { + self.max_silent_time = max_silent_time; + } + + pub const fn set_build_timeout(&mut self, build_timeout: i32) { + self.build_timeout = build_timeout; + } + + #[must_use] + pub const fn get_max_log_size(&self) -> u64 { + self.max_log_size + } + + #[must_use] + pub const fn get_max_silent_time(&self) -> i32 { + self.max_silent_time + } + + #[must_use] + pub const fn get_build_timeout(&self) -> i32 { + self.build_timeout + } + + #[must_use] + pub const fn enable_check_build(mut self) -> Self { + self.check = true; + self + } +} + +#[allow(clippy::type_complexity)] +#[tracing::instrument(skip(store, opts, drvs), err)] +pub async fn realise_drvs( + store: &crate::LocalStore, + drvs: &[&StorePath], + opts: &BuildOptions, + kill_on_drop: bool, +) -> Result< + ( + tokio::process::Child, + tokio_stream::adapters::Merge< + LinesStream>, + LinesStream>, + >, + ), + crate::Error, +> { + use tokio_stream::StreamExt; + + let mut child = tokio::process::Command::new("nix-store") + .args([ + "-r", + "--quiet", // we want to always set this + "--no-gc-warning", // we want to always set this + "--max-silent-time", + &opts.max_silent_time.to_string(), + "--timeout", + &opts.build_timeout.to_string(), + "--option", + "max-build-log-size", + &opts.max_log_size.to_string(), + "--option", + "fallback", + "true", + "--option", + "substitute", + "false", + "--option", + "builders", + "", + ]) + .args(if opts.check { vec!["--check"] } else { vec![] }) + .args(drvs.iter().map(|v| store.print_store_path(v))) + .kill_on_drop(kill_on_drop) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + + let stdout = child.stdout.take().ok_or(crate::Error::Stream)?; + let stderr = child.stderr.take().ok_or(crate::Error::Stream)?; + + let stdout = LinesStream::new(BufReader::new(stdout).lines()); + let stderr = LinesStream::new(BufReader::new(stderr).lines()); + + Ok((child, StreamExt::merge(stdout, stderr))) +} + +#[allow(clippy::type_complexity)] +#[tracing::instrument(skip(store, opts), fields(%drv), err)] +pub async fn realise_drv( + store: &crate::LocalStore, + drv: &StorePath, + opts: &BuildOptions, + kill_on_drop: bool, +) -> Result< + ( + tokio::process::Child, + tokio_stream::adapters::Merge< + LinesStream>, + LinesStream>, + >, + ), + crate::Error, +> { + realise_drvs(store, &[drv], opts, kill_on_drop).await +} diff --git a/src/crates/nix-utils/src/store_path.rs b/src/crates/nix-utils/src/store_path.rs new file mode 100644 index 000000000..48b57012e --- /dev/null +++ b/src/crates/nix-utils/src/store_path.rs @@ -0,0 +1,135 @@ +pub const HASH_LEN: usize = 32; + +#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub struct StorePath { + base_name: String, +} + +impl StorePath { + #[must_use] + pub fn new(p: &str) -> Self { + p.strip_prefix("/nix/store/").map_or_else( + || { + debug_assert!(p.len() > HASH_LEN + 1); + Self { + base_name: p.to_string(), + } + }, + |postfix| { + debug_assert!(postfix.len() > HASH_LEN + 1); + Self { + base_name: postfix.to_string(), + } + }, + ) + } + + #[must_use] + pub fn into_base_name(self) -> String { + self.base_name + } + + #[must_use] + pub fn base_name(&self) -> &str { + &self.base_name + } + + #[must_use] + pub fn name(&self) -> &str { + &self.base_name[HASH_LEN + 1..] + } + + #[must_use] + pub fn hash_part(&self) -> &str { + &self.base_name[..HASH_LEN] + } + + #[must_use] + pub fn is_drv(&self) -> bool { + std::path::Path::new(&self.base_name) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("drv")) + } +} + +impl serde::Serialize for StorePath { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.base_name()) + } +} + +impl std::fmt::Display for StorePath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + write!(f, "{}", self.base_name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_store_path_creation() { + let path_str = "abc123def45678901234567890123456-package-name"; + let store_path = StorePath::new(path_str); + + assert_eq!(store_path.base_name(), path_str); + assert_eq!(store_path.name(), "package-name"); + assert_eq!(store_path.hash_part(), "abc123def45678901234567890123456"); + } + + #[test] + fn test_store_path_with_prefix() { + let store_path = StorePath::new("abc123def45678901234567890123456-package-name"); + + assert_eq!( + store_path.base_name(), + "abc123def45678901234567890123456-package-name" + ); + assert_eq!(store_path.name(), "package-name"); + assert_eq!(store_path.hash_part(), "abc123def45678901234567890123456"); + } + + #[test] + fn test_store_path_is_drv() { + let drv_path = StorePath::new("abc123def45678901234567890123456-package.drv"); + let regular_path = StorePath::new("abc123def45678901234567890123456-package"); + + assert!(drv_path.is_drv()); + assert!(!regular_path.is_drv()); + } + + #[test] + fn test_store_path_display() { + let path_str = "abc123def45678901234567890123456-package-name"; + let store_path = StorePath::new(path_str); + + assert_eq!(format!("{store_path}"), path_str); + } + + // #[test] + // TODO: we cant write tests accessing ffi: https://github.com/dtolnay/cxx/issues/1318 + // fn test_local_store_print_store_path() { + // let store = crate::LocalStore::init(); + // let path_str = "abc123def45678901234567890123456-package-name"; + // let store_path = StorePath::new(path_str); + // + // let printed_path = store.print_store_path(&store_path); + // let expected_prefix = crate::get_store_dir(); + // let expected_path = format!("{}/{}", expected_prefix, path_str); + // + // assert_eq!(printed_path, expected_path); + // } + + #[test] + fn test_store_path_into_base_name() { + let path_str = "abc123def45678901234567890123456-package-name"; + let store_path = StorePath::new(path_str); + + let base_name = store_path.into_base_name(); + assert_eq!(base_name, path_str); + } +} diff --git a/src/crates/shared/Cargo.toml b/src/crates/shared/Cargo.toml new file mode 100644 index 000000000..98fc8795b --- /dev/null +++ b/src/crates/shared/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "shared" +version = "0.1.0" +edition = "2024" +license = "GPL-3.0" +rust-version.workspace = true + +[dependencies] +tracing.workspace = true +anyhow.workspace = true +tokio = { workspace = true, features = ["full"] } +fs-err = { workspace = true, features = ["tokio"] } +regex.workspace = true +sha2.workspace = true + +nix-utils = { path = "../nix-utils" } diff --git a/src/crates/shared/src/lib.rs b/src/crates/shared/src/lib.rs new file mode 100644 index 000000000..ff617e708 --- /dev/null +++ b/src/crates/shared/src/lib.rs @@ -0,0 +1,450 @@ +#![deny(clippy::all)] +#![deny(clippy::pedantic)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::expect_used)] +#![allow(clippy::missing_errors_doc)] + +use std::{os::unix::fs::MetadataExt as _, sync::LazyLock}; + +use sha2::{Digest as _, Sha256}; +use tokio::io::{AsyncBufReadExt as _, AsyncReadExt as _, BufReader}; + +use nix_utils::{BaseStore as _, StorePath}; + +#[allow(clippy::expect_used)] +static VALIDATE_METRICS_NAME: LazyLock = + LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9._-]+").expect("Failed to compile regex")); +#[allow(clippy::expect_used)] +static VALIDATE_METRICS_UNIT: LazyLock = + LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9._%-]+").expect("Failed to compile regex")); +#[allow(clippy::expect_used)] +static VALIDATE_RELEASE_NAME: LazyLock = + LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9.@:_-]+").expect("Failed to compile regex")); +#[allow(clippy::expect_used)] +static VALIDATE_PRODUCT_NAME: LazyLock = + LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9.@:_ -]*").expect("Failed to compile regex")); +#[allow(clippy::expect_used)] +static BUILD_PRODUCT_PARSER: LazyLock = LazyLock::new(|| { + regex::Regex::new( + r#"([a-zA-Z0-9_-]+)\s+([a-zA-Z0-9_-]+)\s+(\"[^\"]+\"|[^\"\s<>]+)(\s+([^\"\s<>]+))?"#, + ) + .expect("Failed to compile regex") +}); + +#[derive(Debug)] +pub struct BuildProduct { + pub path: String, + pub default_path: String, + + pub r#type: String, + pub subtype: String, + pub name: String, + + pub is_regular: bool, + + pub sha256hash: Option, + pub file_size: Option, +} + +#[derive(Debug)] +pub struct BuildMetric { + pub path: String, + pub name: String, + pub unit: Option, + pub value: f64, +} + +#[derive(Debug)] +pub struct NixSupport { + pub failed: bool, + pub hydra_release_name: Option, + pub metrics: Vec, + pub products: Vec, +} + +#[derive(Debug, Clone)] +struct FileMetadata { + is_regular: bool, + size: u64, +} + +trait FsOperations { + fn is_inside_store(&self, path: &std::path::Path) -> bool; + fn get_metadata( + &self, + path: impl AsRef + std::fmt::Debug, + ) -> impl std::future::Future>; + fn get_file_hash( + &self, + path: impl Into + std::fmt::Debug, + ) -> impl std::future::Future>; +} + +#[derive(Debug, Clone)] +struct FilesystemOperations { + nix_store_dir: std::sync::Arc, +} + +impl FilesystemOperations { + fn new() -> Self { + Self { + nix_store_dir: std::sync::Arc::new(nix_utils::get_store_dir()), + } + } +} + +impl FsOperations for FilesystemOperations { + fn is_inside_store(&self, path: &std::path::Path) -> bool { + nix_utils::is_subpath(std::path::Path::new(self.nix_store_dir.as_str()), path) + } + + #[tracing::instrument(skip(self), err)] + async fn get_metadata( + &self, + path: impl AsRef + std::fmt::Debug, + ) -> Result { + let m = fs_err::tokio::metadata(path).await?; + Ok(FileMetadata { + is_regular: m.is_file(), + size: m.size(), + }) + } + + #[tracing::instrument(skip(self), err)] + async fn get_file_hash( + &self, + path: impl Into + std::fmt::Debug, + ) -> tokio::io::Result { + let file = fs_err::tokio::File::open(path).await?; + let mut reader = BufReader::new(file); + + let mut hasher = Sha256::new(); + let mut buf = [0u8; 16 * 1024]; + + loop { + let n = reader.read(&mut buf).await?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + + Ok(format!("{:x}", hasher.finalize())) + } +} + +fn parse_release_name(content: &str) -> Option { + let content = content.trim(); + if !content.is_empty() && VALIDATE_RELEASE_NAME.is_match(content) { + Some(content.to_owned()) + } else { + None + } +} + +fn parse_metric( + store: &nix_utils::LocalStore, + line: &str, + output: &StorePath, +) -> Option { + let fields: Vec = line.split_whitespace().map(ToOwned::to_owned).collect(); + if fields.len() < 2 || !VALIDATE_METRICS_NAME.is_match(&fields[0]) { + return None; + } + + Some(BuildMetric { + path: store.print_store_path(output), + name: fields[0].clone(), + value: fields[1].parse::().unwrap_or(0.0), + unit: if fields.len() >= 3 && VALIDATE_METRICS_UNIT.is_match(&fields[2]) { + Some(fields[2].clone()) + } else { + None + }, + }) +} + +async fn parse_build_product( + store: &nix_utils::LocalStore, + handle: Op, + output: &StorePath, + line: &str, +) -> Option +where + Op: FsOperations + Send + Sync + 'static, +{ + let captures = BUILD_PRODUCT_PARSER.captures(line)?; + + let s = captures[3].to_string(); + let path = if s.starts_with('"') && s.ends_with('"') { + s[1..s.len() - 1].to_string() + } else { + s + }; + + if path.is_empty() || !path.starts_with('/') { + return None; + } + if !handle.is_inside_store(std::path::Path::new(&path)) { + return None; + } + let metadata = handle.get_metadata(&path).await.ok()?; + + let name = { + let name = if path == store.print_store_path(output) { + String::new() + } else { + std::path::Path::new(&path) + .file_name() + .and_then(|f| f.to_str()) + .map(ToOwned::to_owned) + .unwrap_or_default() + }; + + if VALIDATE_PRODUCT_NAME.is_match(&name) { + name + } else { + String::new() + } + }; + + let sha256hash = if metadata.is_regular { + handle.get_file_hash(&path).await.ok() + } else { + None + }; + + Some(BuildProduct { + r#type: captures[1].to_string(), + subtype: captures[2].to_string(), + path, + default_path: captures + .get(5) + .map(|m| m.as_str().to_string()) + .unwrap_or_default(), + name, + is_regular: metadata.is_regular, + file_size: if metadata.is_regular { + Some(metadata.size) + } else { + None + }, + sha256hash, + }) +} + +#[tracing::instrument(skip(store), err)] +pub async fn parse_nix_support_from_outputs( + store: &nix_utils::LocalStore, + derivation_outputs: &[nix_utils::DerivationOutput], +) -> anyhow::Result { + let mut metrics = Vec::new(); + let mut failed = false; + let mut hydra_release_name = None; + + let outputs = derivation_outputs + .iter() + .filter_map(|o| o.path.as_ref()) + .collect::>(); + for output in &outputs { + let output_full_path = store.print_store_path(output); + let file_path = std::path::Path::new(&output_full_path).join("nix-support/hydra-metrics"); + let Ok(file) = fs_err::tokio::File::open(&file_path).await else { + continue; + }; + + let reader = BufReader::new(file); + let mut lines = reader.lines(); + + while let Some(line) = lines.next_line().await? { + if let Some(m) = parse_metric(store, &line, output) { + metrics.push(m); + } + } + } + + for output in &outputs { + let file_path = + std::path::Path::new(&store.print_store_path(output)).join("nix-support/failed"); + if fs_err::tokio::try_exists(file_path) + .await + .unwrap_or_default() + { + failed = true; + break; + } + } + + for output in &outputs { + let file_path = std::path::Path::new(&store.print_store_path(output)) + .join("nix-support/hydra-release-name"); + if let Ok(v) = fs_err::tokio::read_to_string(file_path).await + && let Some(v) = parse_release_name(&v) + { + hydra_release_name = Some(v); + break; + } + } + + let mut explicit_products = false; + let mut products = Vec::new(); + for output in &outputs { + let output_full_path = store.print_store_path(output); + let file_path = + std::path::Path::new(&output_full_path).join("nix-support/hydra-build-products"); + let Ok(file) = fs_err::tokio::File::open(&file_path).await else { + continue; + }; + + explicit_products = true; + + let reader = BufReader::new(file); + let mut lines = reader.lines(); + let fsop = FilesystemOperations::new(); + while let Some(line) = lines.next_line().await? { + if let Some(o) = Box::pin(parse_build_product(store, fsop.clone(), output, &line)).await + { + products.push(o); + } + } + } + + if !explicit_products { + for o in derivation_outputs { + let Some(path) = &o.path else { + continue; + }; + let full_path = store.print_store_path(path); + let Ok(metadata) = fs_err::tokio::metadata(&full_path).await else { + continue; + }; + if metadata.is_dir() { + products.push(BuildProduct { + r#type: "nix-build".to_string(), + subtype: if o.name == "out" { + String::new() + } else { + o.name.clone() + }, + path: full_path, + name: path.name().to_string(), + default_path: String::new(), + is_regular: false, + file_size: None, + sha256hash: None, + }); + } + } + } + + Ok(NixSupport { + failed, + hydra_release_name, + metrics, + products, + }) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use super::*; + + #[derive(Debug, Clone)] + struct DummyFsOperations { + valid_file: bool, + metadata: FileMetadata, + file_hash: String, + } + + impl FsOperations for DummyFsOperations { + fn is_inside_store(&self, _: &std::path::Path) -> bool { + self.valid_file + } + + async fn get_metadata( + &self, + _: impl AsRef + std::fmt::Debug, + ) -> Result { + Ok(self.metadata.clone()) + } + + async fn get_file_hash( + &self, + _: impl Into + std::fmt::Debug, + ) -> Result { + Ok(self.file_hash.clone()) + } + } + + #[tokio::test] + async fn test_build_products() { + let store = nix_utils::LocalStore::init(); + let output = nix_utils::StorePath::new("ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso"); + let line = format!( + "file iso {}/iso/custom.iso", + store.print_store_path(&output) + ); + let fsop = DummyFsOperations { + valid_file: true, + metadata: FileMetadata { + is_regular: true, + size: 12345, + }, + file_hash: "4306152c73d2a7a01dbac16ba48f45fa4ae5b746a1d282638524ae2ae93af210".into(), + }; + let build_product = parse_build_product(&store, fsop, &output, &line) + .await + .unwrap(); + assert!(build_product.is_regular); + assert_eq!( + build_product.path, + "/nix/store/ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso/iso/custom.iso" + ); + assert_eq!(build_product.name, "custom.iso"); + assert_eq!(build_product.file_size, Some(12345)); + assert_eq!( + build_product.sha256hash, + Some("4306152c73d2a7a01dbac16ba48f45fa4ae5b746a1d282638524ae2ae93af210".into()) + ); + } + + #[test] + fn test_parse_invalid_metric() { + let output = nix_utils::StorePath::new("ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso"); + let line = "nix-env.qaCount"; + let store = nix_utils::LocalStore::init(); + let m = parse_metric(&store, line, &output); + assert!(m.is_none()); + } + + #[test] + fn test_parse_metric_without_unit() { + let output = nix_utils::StorePath::new("ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso"); + let line = "nix-env.qaCount 4"; + let store = nix_utils::LocalStore::init(); + let m = parse_metric(&store, line, &output).unwrap(); + assert_eq!(m.name, "nix-env.qaCount"); + assert!((m.value - 4.0_f64).abs() < f64::EPSILON); + assert_eq!(m.unit, None); + } + + #[test] + fn test_parse_metric_with_unit() { + let output = nix_utils::StorePath::new("ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso"); + let line = "xzy.time 123.321 s"; + let store = nix_utils::LocalStore::init(); + let m = parse_metric(&store, line, &output).unwrap(); + assert_eq!(m.name, "xzy.time"); + assert!((m.value - 123.321_f64).abs() < f64::EPSILON); + assert_eq!(m.unit, Some("s".into())); + } + + #[test] + fn test_parse_release_name() { + let line = "nixos-25.11pre708350"; + let o = parse_release_name(line); + assert_eq!(o, Some("nixos-25.11pre708350".into())); + } +} diff --git a/src/crates/tracing/Cargo.toml b/src/crates/tracing/Cargo.toml new file mode 100644 index 000000000..8f42a7c54 --- /dev/null +++ b/src/crates/tracing/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "hydra-tracing" +version = "0.1.0" +edition = "2024" +license = "GPL-3.0" +rust-version.workspace = true + +[dependencies] +anyhow.workspace = true +tracing.workspace = true +tracing-log.workspace = true +tracing-subscriber = { workspace = true, features = ["registry", "env-filter"] } + +tonic = { workspace = true, optional = true } +http = { workspace = true, optional = true } + +opentelemetry = { workspace = true, optional = true } +opentelemetry_sdk = { workspace = true, features = [ + "rt-tokio", +], optional = true } +opentelemetry-otlp = { workspace = true, optional = true, features = [ + "grpc-tonic", +] } +opentelemetry-http = { workspace = true, optional = true } +opentelemetry-semantic-conventions = { workspace = true, optional = true } +tracing-opentelemetry = { workspace = true, optional = true } + +[features] +otel = [ + "opentelemetry", + "opentelemetry_sdk", + "opentelemetry-otlp", + "opentelemetry-http", + "opentelemetry-semantic-conventions", + "tracing-opentelemetry", +] +tonic = ["dep:tonic", "http"] diff --git a/src/crates/tracing/src/lib.rs b/src/crates/tracing/src/lib.rs new file mode 100644 index 000000000..c18c8a8b5 --- /dev/null +++ b/src/crates/tracing/src/lib.rs @@ -0,0 +1,91 @@ +#![deny(clippy::all)] +#![deny(clippy::pedantic)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::expect_used)] +#![allow(clippy::missing_errors_doc)] + +pub use tracing_subscriber::filter::EnvFilter; +use tracing_subscriber::layer::SubscriberExt as _; + +#[cfg(feature = "otel")] +use opentelemetry::trace::TracerProvider as _; + +#[cfg(feature = "tonic")] +pub mod propagate; + +#[cfg(feature = "otel")] +fn resource() -> opentelemetry_sdk::Resource { + opentelemetry_sdk::Resource::builder() + .with_service_name(env!("CARGO_PKG_NAME")) + .with_schema_url( + [opentelemetry::KeyValue::new( + opentelemetry_semantic_conventions::attribute::SERVICE_VERSION, + env!("CARGO_PKG_VERSION"), + )], + opentelemetry_semantic_conventions::SCHEMA_URL, + ) + .build() +} + +pub struct TracingGuard { + #[cfg(feature = "otel")] + tracer_provider: opentelemetry_sdk::trace::SdkTracerProvider, + + reload_handle: tracing_subscriber::reload::Handle, +} + +impl TracingGuard { + pub fn change_log_level(&self, new_filter: EnvFilter) { + let _ = self.reload_handle.modify(|filter| *filter = new_filter); + } +} + +impl Drop for TracingGuard { + fn drop(&mut self) { + #[cfg(feature = "otel")] + if let Err(err) = self.tracer_provider.shutdown() { + eprintln!("{err:?}"); + } + } +} + +#[cfg(feature = "otel")] +fn init_tracer_provider() -> anyhow::Result { + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .build()?; + + Ok(opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_resource(resource()) + .with_batch_exporter(exporter) + .build()) +} + +pub fn init() -> anyhow::Result { + tracing_log::LogTracer::init()?; + let (log_env_filter, reload_handle) = tracing_subscriber::reload::Layer::new( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ); + let fmt_layer = tracing_subscriber::fmt::layer().compact(); + let subscriber = tracing_subscriber::Registry::default() + .with(log_env_filter) + .with(fmt_layer); + + #[cfg(feature = "otel")] + { + let provider = init_tracer_provider()?; + let tracer = provider.tracer(env!("CARGO_PKG_NAME")); + let subscriber = subscriber.with(tracing_opentelemetry::OpenTelemetryLayer::new(tracer)); + tracing::subscriber::set_global_default(subscriber)?; + Ok(TracingGuard { + tracer_provider: provider, + reload_handle, + }) + } + + #[cfg(not(feature = "otel"))] + { + tracing::subscriber::set_global_default(subscriber)?; + Ok(TracingGuard { reload_handle }) + } +} diff --git a/src/crates/tracing/src/propagate.rs b/src/crates/tracing/src/propagate.rs new file mode 100644 index 000000000..72fa60c5f --- /dev/null +++ b/src/crates/tracing/src/propagate.rs @@ -0,0 +1,52 @@ +// Based on https://heikoseeberger.de/2023-08-28-dist-tracing-3/ + +#[cfg(feature = "otel")] +use opentelemetry::{global, propagation::Injector}; +#[cfg(feature = "otel")] +use tracing_opentelemetry::OpenTelemetrySpanExt; + +pub fn accept_trace(request: http::Request) -> http::Request { + #[cfg(feature = "otel")] + { + let parent_context = global::get_text_map_propagator(|propagator| { + propagator.extract(&opentelemetry_http::HeaderExtractor(request.headers())) + }); + let _ = tracing::Span::current().set_parent(parent_context); + } + request +} + +#[cfg(feature = "otel")] +struct MetadataInjector<'a>(&'a mut tonic::metadata::MetadataMap); + +#[cfg(feature = "otel")] +impl Injector for MetadataInjector<'_> { + fn set(&mut self, key: &str, value: String) { + use tonic::metadata::{MetadataKey, MetadataValue}; + use tracing::warn; + + match MetadataKey::from_bytes(key.as_bytes()) { + Ok(key) => match MetadataValue::try_from(&value) { + Ok(value) => { + self.0.insert(key, value); + } + Err(error) => warn!(value, error = format!("{error:#}"), "parse metadata value"), + }, + Err(error) => warn!(key, error = format!("{error:#}"), "parse metadata key"), + } + } +} + +#[allow(unused_mut)] +pub fn send_trace( + mut request: tonic::Request, +) -> Result, Box> { + #[cfg(feature = "otel")] + { + global::get_text_map_propagator(|propagator| { + let context = tracing::Span::current().context(); + propagator.inject_context(&context, &mut MetadataInjector(request.metadata_mut())); + }); + } + Ok(request) +} diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc deleted file mode 100644 index 4111a8fb4..000000000 --- a/src/hydra-queue-runner/build-remote.cc +++ /dev/null @@ -1,646 +0,0 @@ -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include "state.hh" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace nix; - -bool ::Machine::isLocalhost() const -{ - return storeUri.params.empty() && std::visit(overloaded { - [](const StoreReference::Auto &) { - return true; - }, - [](const StoreReference::Specified & s) { - return - (s.scheme == "local" || s.scheme == "unix") || - ((s.scheme == "ssh" || s.scheme == "ssh-ng") && - s.authority == "localhost"); - }, - }, storeUri.variant); -} - -namespace nix::build_remote { - -static std::unique_ptr openConnection( - ::Machine::ptr machine, SSHMaster & master) -{ - Strings command = {"nix-store", "--serve", "--write"}; - if (machine->isLocalhost()) { - command.push_back("--builders"); - command.push_back(""); - } else { - auto remoteStore = machine->storeUri.params.find("remote-store"); - if (remoteStore != machine->storeUri.params.end()) { - command.push_back("--store"); - command.push_back(escapeShellArgAlways(remoteStore->second)); - } - } - - auto ret = master.startCommand(std::move(command), { - "-a", "-oBatchMode=yes", "-oConnectTimeout=60", "-oTCPKeepAlive=yes" - }); - - // XXX: determine the actual max value we can use from /proc. - - // FIXME: Should this be upstreamed into `startCommand` in Nix? - - int pipesize = 1024 * 1024; - - fcntl(ret->in.get(), F_SETPIPE_SZ, &pipesize); - fcntl(ret->out.get(), F_SETPIPE_SZ, &pipesize); - - return ret; -} - - -static void copyClosureTo( - ::Machine::Connection & conn, - Store & destStore, - const StorePathSet & paths, - SubstituteFlag useSubstitutes = NoSubstitute) -{ - StorePathSet closure; - destStore.computeFSClosure(paths, closure); - - /* Send the "query valid paths" command with the "lock" option - enabled. This prevents a race where the remote host - garbage-collect paths that are already there. Optionally, ask - the remote host to substitute missing paths. */ - // FIXME: substitute output pollutes our build log - /* Get back the set of paths that are already valid on the remote - host. */ - auto present = conn.queryValidPaths( - destStore, true, closure, useSubstitutes); - - if (present.size() == closure.size()) return; - - auto sorted = destStore.topoSortPaths(closure); - - StorePathSet missing; - for (auto i = sorted.rbegin(); i != sorted.rend(); ++i) - if (!present.count(*i)) missing.insert(*i); - - printMsg(lvlDebug, "sending %d missing paths", missing.size()); - - std::unique_lock sendLock(conn.machine->state->sendLock, - std::chrono::seconds(600)); - - conn.to << ServeProto::Command::ImportPaths; - exportPaths(destStore, missing, conn.to); - conn.to.flush(); - - if (readInt(conn.from) != 1) - throw Error("remote machine failed to import closure"); -} - - -// FIXME: use Store::topoSortPaths(). -static StorePaths reverseTopoSortPaths(const std::map & paths) -{ - StorePaths sorted; - StorePathSet visited; - - std::function dfsVisit; - - dfsVisit = [&](const StorePath & path) { - if (!visited.insert(path).second) return; - - auto info = paths.find(path); - auto references = info == paths.end() ? StorePathSet() : info->second.references; - - for (auto & i : references) - /* Don't traverse into paths that don't exist. That can - happen due to substitutes for non-existent paths. */ - if (i != path && paths.count(i)) - dfsVisit(i); - - sorted.push_back(path); - }; - - for (auto & i : paths) - dfsVisit(i.first); - - return sorted; -} - -static std::pair openLogFile(const std::string & logDir, const StorePath & drvPath) -{ - std::string base(drvPath.to_string()); - auto logFile = logDir + "/" + std::string(base, 0, 2) + "/" + std::string(base, 2); - - createDirs(dirOf(logFile)); - - AutoCloseFD logFD = open(logFile.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0666); - if (!logFD) throw SysError("creating log file ‘%s’", logFile); - - return {std::move(logFile), std::move(logFD)}; -} - -static BasicDerivation sendInputs( - State & state, - Step & step, - Store & localStore, - Store & destStore, - ::Machine::Connection & conn, - unsigned int & overhead, - counter & nrStepsWaiting, - counter & nrStepsCopyingTo -) -{ - /* Replace the input derivations by their output paths to send a - minimal closure to the builder. - - `tryResolve` currently does *not* rewrite input addresses, so it - is safe to do this in all cases. (It should probably have a mode - to do that, however, but we would not use it here.) - */ - BasicDerivation basicDrv = ({ - auto maybeBasicDrv = step.drv->tryResolve(destStore, &localStore); - if (!maybeBasicDrv) - throw Error( - "the derivation '%s' can’t be resolved. It’s probably " - "missing some outputs", - localStore.printStorePath(step.drvPath)); - *maybeBasicDrv; - }); - - /* Ensure that the inputs exist in the destination store. This is - a no-op for regular stores, but for the binary cache store, - this will copy the inputs to the binary cache from the local - store. */ - if (&localStore != &destStore) { - copyClosure(localStore, destStore, - step.drv->inputSrcs, - NoRepair, NoCheckSigs, NoSubstitute); - } - - { - auto mc1 = std::make_shared>(nrStepsWaiting); - mc1.reset(); - MaintainCount mc2(nrStepsCopyingTo); - - printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’", - localStore.printStorePath(step.drvPath), conn.machine->storeUri.render()); - - auto now1 = std::chrono::steady_clock::now(); - - /* Copy the input closure. */ - if (conn.machine->isLocalhost()) { - StorePathSet closure; - destStore.computeFSClosure(basicDrv.inputSrcs, closure); - copyPaths(destStore, localStore, closure, NoRepair, NoCheckSigs, NoSubstitute); - } else { - copyClosureTo(conn, destStore, basicDrv.inputSrcs, Substitute); - } - - auto now2 = std::chrono::steady_clock::now(); - - overhead += std::chrono::duration_cast(now2 - now1).count(); - } - - return basicDrv; -} - -static BuildResult performBuild( - ::Machine::Connection & conn, - Store & localStore, - StorePath drvPath, - const BasicDerivation & drv, - const ServeProto::BuildOptions & options, - counter & nrStepsBuilding -) -{ - conn.putBuildDerivationRequest(localStore, drvPath, drv, options); - - BuildResult result; - - time_t startTime, stopTime; - - startTime = time(0); - { - MaintainCount mc(nrStepsBuilding); - result = ServeProto::Serialise::read(localStore, conn); - } - stopTime = time(0); - - if (!result.startTime) { - // If the builder gave `startTime = 0`, use our measurements - // instead of the builder's. - // - // Note: this represents the duration of a single round, rather - // than all rounds. - result.startTime = startTime; - result.stopTime = stopTime; - } - - // If the protocol was too old to give us `builtOutputs`, initialize - // it manually by introspecting the derivation. - if (GET_PROTOCOL_MINOR(conn.remoteVersion) < 6) - { - // If the remote is too old to handle CA derivations, we can’t get this - // far anyways - assert(drv.type().hasKnownOutputPaths()); - DerivationOutputsAndOptPaths drvOutputs = drv.outputsAndOptPaths(localStore); - // Since this a `BasicDerivation`, `staticOutputHashes` will not - // do any real work. - auto outputHashes = staticOutputHashes(localStore, drv); - if (auto * successP = result.tryGetSuccess()) { - for (auto & [outputName, output] : drvOutputs) { - auto outputPath = output.second; - // We’ve just asserted that the output paths of the derivation - // were known - assert(outputPath); - auto outputHash = outputHashes.at(outputName); - auto drvOutput = DrvOutput { outputHash, outputName }; - successP->builtOutputs.insert_or_assign( - std::move(outputName), - Realisation { drvOutput, *outputPath }); - } - } - } - - return result; -} - -static void copyPathFromRemote( - ::Machine::Connection & conn, - NarMemberDatas & narMembers, - Store & localStore, - Store & destStore, - const ValidPathInfo & info -) -{ - /* Receive the NAR from the remote and add it to the - destination store. Meanwhile, extract all the info from the - NAR that getBuildOutput() needs. */ - auto source2 = sinkToSource([&](Sink & sink) - { - /* Note: we should only send the command to dump the store - path to the remote if the NAR is actually going to get read - by the destination store, which won't happen if this path - is already valid on the destination store. Since this - lambda function only gets executed if someone tries to read - from source2, we will send the command from here rather - than outside the lambda. */ - conn.to << ServeProto::Command::DumpStorePath << localStore.printStorePath(info.path); - conn.to.flush(); - - TeeSource tee(conn.from, sink); - extractNarData(tee, localStore.printStorePath(info.path), narMembers); - }); - - destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); -} - -static void copyPathsFromRemote( - ::Machine::Connection & conn, - NarMemberDatas & narMembers, - Store & localStore, - Store & destStore, - const std::map & infos -) -{ - auto pathsSorted = reverseTopoSortPaths(infos); - - for (auto & path : pathsSorted) { - auto & info = infos.find(path)->second; - copyPathFromRemote( - conn, narMembers, localStore, destStore, - ValidPathInfo { path, info }); - } - -} - -} - -/* using namespace nix::build_remote; */ - -void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult) -{ - startTime = buildResult.startTime; - stopTime = buildResult.stopTime; - timesBuilt = buildResult.timesBuilt; - - std::visit(overloaded{ - [&](const BuildResult::Success & success) { - stepStatus = bsSuccess; - switch (success.status) { - case BuildResult::Success::Built: - break; - case BuildResult::Success::Substituted: - case BuildResult::Success::AlreadyValid: - case BuildResult::Success::ResolvesToAlreadyValid: - isCached = true; - break; - default: - assert(false); - } - }, - [&](const BuildResult::Failure & failure) { - errorMsg = failure.errorMsg; - isNonDeterministic = failure.isNonDeterministic; - switch (failure.status) { - case BuildResult::Failure::PermanentFailure: - stepStatus = bsFailed; - canCache = true; - errorMsg = ""; - break; - case BuildResult::Failure::InputRejected: - case BuildResult::Failure::OutputRejected: - stepStatus = bsFailed; - canCache = true; - break; - case BuildResult::Failure::TransientFailure: - stepStatus = bsFailed; - canRetry = true; - errorMsg = ""; - break; - case BuildResult::Failure::TimedOut: - stepStatus = bsTimedOut; - errorMsg = ""; - break; - case BuildResult::Failure::MiscFailure: - stepStatus = bsAborted; - canRetry = true; - break; - case BuildResult::Failure::LogLimitExceeded: - stepStatus = bsLogLimitExceeded; - break; - case BuildResult::Failure::NotDeterministic: - stepStatus = bsNotDeterministic; - canRetry = false; - canCache = true; - break; - case BuildResult::Failure::CachedFailure: - case BuildResult::Failure::DependencyFailed: - case BuildResult::Failure::NoSubstituters: - case BuildResult::Failure::HashMismatch: - stepStatus = bsAborted; - break; - default: - assert(false); - } - }, - }, buildResult.inner); -} - -/* Utility guard object to auto-release a semaphore on destruction. */ -template -class SemaphoreReleaser { -public: - SemaphoreReleaser(T* s) : sem(s) {} - ~SemaphoreReleaser() { sem->release(); } - -private: - T* sem; -}; - -void State::buildRemote(ref destStore, - std::unique_ptr reservation, - ::Machine::ptr machine, Step::ptr step, - const ServeProto::BuildOptions & buildOptions, - RemoteResult & result, std::shared_ptr activeStep, - std::function updateStep, - NarMemberDatas & narMembers) -{ - assert(BuildResult::Failure::TimedOut == 8); - - auto [logFile, logFD] = build_remote::openLogFile(logDir, step->drvPath); - AutoDelete logFileDel(logFile, false); - result.logFile = logFile; - - try { - - updateStep(ssConnecting); - - auto storeRef = machine->completeStoreReference(); - - auto * pSpecified = std::get_if(&storeRef.variant); - if (!pSpecified || pSpecified->scheme != "ssh") { - throw Error("Currently, only (legacy-)ssh stores are supported!"); - } - - LegacySSHStoreConfig storeConfig { - pSpecified->scheme, - pSpecified->authority, - storeRef.params - }; - - auto master = storeConfig.createSSHMaster( - false, // no SSH master yet - logFD.get()); - - // FIXME: rewrite to use Store. - auto child = build_remote::openConnection(machine, master); - - { - auto activeStepState(activeStep->state_.lock()); - if (activeStepState->cancelled) throw Error("step cancelled"); - activeStepState->pid = child->sshPid; - } - - Finally clearPid([&]() { - auto activeStepState(activeStep->state_.lock()); - activeStepState->pid = -1; - - /* FIXME: there is a slight race here with step - cancellation in State::processQueueChange(), which - could call kill() on this pid after we've done waitpid() - on it. With pid wrap-around, there is a tiny - possibility that we end up killing another - process. Meh. */ - }); - - ::Machine::Connection conn { - { - .to = child->in.get(), - .from = child->out.get(), - /* Handshake. */ - .remoteVersion = 0xdadbeef, // FIXME avoid dummy initialize - }, - /*.machine =*/ machine, - }; - - Finally updateStats([&]() { - bytesReceived += conn.from.read; - bytesSent += conn.to.written; - }); - - constexpr ServeProto::Version our_version = 0x206; - - try { - conn.remoteVersion = decltype(conn)::handshake( - conn.to, - conn.from, - our_version, - machine->storeUri.render()); - } catch (EndOfFile & e) { - child->sshPid.wait(); - std::string s = chomp(readFile(result.logFile)); - throw Error("cannot connect to ‘%1%’: %2%", machine->storeUri.render(), s); - } - - { - auto info(machine->state->connectInfo.lock()); - info->consecutiveFailures = 0; - } - - /* Gather the inputs. If the remote side is Nix <= 1.9, we have to - copy the entire closure of ‘drvPath’, as well as the required - outputs of the input derivations. On Nix > 1.9, we only need to - copy the immediate sources of the derivation and the required - outputs of the input derivations. */ - updateStep(ssSendingInputs); - BasicDerivation resolvedDrv = build_remote::sendInputs(*this, *step, *localStore, *destStore, conn, result.overhead, nrStepsWaiting, nrStepsCopyingTo); - - logFileDel.cancel(); - - /* Truncate the log to get rid of messages about substitutions - etc. on the remote system. */ - if (lseek(logFD.get(), SEEK_SET, 0) != 0) - throw SysError("seeking to the start of log file ‘%s’", result.logFile); - - if (ftruncate(logFD.get(), 0) == -1) - throw SysError("truncating log file ‘%s’", result.logFile); - - logFD = -1; - - /* Do the build. */ - printMsg(lvlDebug, "building ‘%s’ on ‘%s’", - localStore->printStorePath(step->drvPath), - machine->storeUri.render()); - - updateStep(ssBuilding); - - auto buildResult = build_remote::performBuild( - conn, - *localStore, - step->drvPath, - resolvedDrv, - buildOptions, - nrStepsBuilding - ); - - result.updateWithBuildResult(buildResult); - - if (result.stepStatus != bsSuccess) return; - - result.errorMsg = ""; - - /* If the path was substituted or already valid, then we didn't - get a build log. */ - if (result.isCached) { - printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’", - localStore->printStorePath(step->drvPath), machine->storeUri.render()); - unlink(result.logFile.c_str()); - result.logFile = ""; - } - - /* Throttle CPU-bound work. Opportunistically skip updating the current - * step, since this requires a DB roundtrip. */ - if (!localWorkThrottler.try_acquire()) { - MaintainCount mc(nrStepsWaitingForDownloadSlot); - updateStep(ssWaitingForLocalSlot); - localWorkThrottler.acquire(); - } - SemaphoreReleaser releaser(&localWorkThrottler); - - /* Once we've started copying outputs, release the machine reservation - * so further builds can happen. We do not release the machine earlier - * to avoid situations where the queue runner is bottlenecked on - * copying outputs and we end up building too many things that we - * haven't been able to allow copy slots for. */ - reservation.reset(); - wakeDispatcher(); - - StorePathSet outputs; - if (auto * successP = buildResult.tryGetSuccess()) - for (auto & [_, realisation] : successP->builtOutputs) - outputs.insert(realisation.outPath); - - /* Copy the output paths. */ - if (!machine->isLocalhost() || localStore != std::shared_ptr(destStore)) { - updateStep(ssReceivingOutputs); - - MaintainCount mc(nrStepsCopyingFrom); - - auto now1 = std::chrono::steady_clock::now(); - - auto infos = conn.queryPathInfos(*localStore, outputs); - - size_t totalNarSize = 0; - for (auto & [_, info] : infos) totalNarSize += info.narSize; - - if (totalNarSize > maxOutputSize) { - result.stepStatus = bsNarSizeLimitExceeded; - return; - } - - /* Copy each path. */ - printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)", - localStore->printStorePath(step->drvPath), machine->storeUri.render(), totalNarSize); - - build_remote::copyPathsFromRemote(conn, narMembers, *localStore, *destStore, infos); - auto now2 = std::chrono::steady_clock::now(); - - result.overhead += std::chrono::duration_cast(now2 - now1).count(); - } - - /* Register the outputs of the newly built drv */ - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - auto outputHashes = staticOutputHashes(*localStore, *step->drv); - if (auto * successP = buildResult.tryGetSuccess()) { - for (auto & [outputName, realisation] : successP->builtOutputs) { - // Register the resolved drv output - destStore->registerDrvOutput(realisation); - - // Also register the unresolved one - auto unresolvedRealisation = realisation; - unresolvedRealisation.signatures.clear(); - unresolvedRealisation.id.drvHash = outputHashes.at(outputName); - destStore->registerDrvOutput(unresolvedRealisation); - } - } - } - - /* Shut down the connection. */ - child->in = -1; - child->sshPid.wait(); - - } catch (Error & e) { - /* Disable this machine until a certain period of time has - passed. This period increases on every consecutive - failure. However, don't count failures that occurred soon - after the last one (to take into account steps started in - parallel). */ - auto info(machine->state->connectInfo.lock()); - auto now = std::chrono::system_clock::now(); - if (info->consecutiveFailures == 0 || info->lastFailure < now - std::chrono::seconds(30)) { - info->consecutiveFailures = std::min(info->consecutiveFailures + 1, (unsigned int) 4); - info->lastFailure = now; - int delta = retryInterval * std::pow(retryBackoff, info->consecutiveFailures - 1) + (rand() % 30); - printMsg(lvlInfo, "will disable machine ‘%1%’ for %2%s", machine->storeUri.render(), delta); - info->disabledUntil = now + std::chrono::seconds(delta); - } - throw; - } -} diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc deleted file mode 100644 index aa98acbb5..000000000 --- a/src/hydra-queue-runner/build-result.cc +++ /dev/null @@ -1,163 +0,0 @@ -#include "hydra-build-result.hh" -#include -#include -#include - -#include - -using namespace nix; - - -BuildOutput getBuildOutput( - nix::ref store, - NarMemberDatas & narMembers, - const OutputPathMap derivationOutputs) -{ - BuildOutput res; - - /* Compute the closure size. */ - StorePathSet outputs; - StorePathSet closure; - for (auto& [outputName, outputPath] : derivationOutputs) { - store->computeFSClosure(outputPath, closure); - outputs.insert(outputPath); - res.outputs.insert({outputName, outputPath}); - } - for (auto & path : closure) { - auto info = store->queryPathInfo(path); - res.closureSize += info->narSize; - if (outputs.count(path)) res.size += info->narSize; - } - - /* Fetch missing data. Usually buildRemote() will have extracted - this data from the incoming NARs. */ - for (auto & output : outputs) { - auto outputS = store->printStorePath(output); - if (!narMembers.count(outputS)) { - printInfo("fetching NAR contents of '%s'...", outputS); - auto source = sinkToSource([&](Sink & sink) - { - store->narFromPath(output, sink); - }); - extractNarData(*source, outputS, narMembers); - } - } - - /* Get build products. */ - bool explicitProducts = false; - - std::regex regex( - "([a-zA-Z0-9_-]+)" // type (e.g. "doc") - "[[:space:]]+" - "([a-zA-Z0-9_-]+)" // subtype (e.g. "readme") - "[[:space:]]+" - "(\"[^\"]+\"|[^[:space:]<>\"]+)" // path (may be quoted) - "([[:space:]]+([^[:space:]<>]+))?" // entry point - , std::regex::extended); - - for (auto & output : outputs) { - auto outputS = store->printStorePath(output); - - if (narMembers.count(outputS + "/nix-support/failed")) - res.failed = true; - - auto productsFile = narMembers.find(outputS + "/nix-support/hydra-build-products"); - if (productsFile == narMembers.end() || - productsFile->second.type != SourceAccessor::Type::tRegular) - continue; - assert(productsFile->second.contents); - - explicitProducts = true; - - for (auto & line : tokenizeString(productsFile->second.contents.value(), "\n")) { - BuildProduct product; - - std::smatch match; - if (!std::regex_match(line, match, regex)) continue; - - product.type = match[1]; - product.subtype = match[2]; - std::string s(match[3]); - product.path = s[0] == '"' && s.back() == '"' ? std::string(s, 1, s.size() - 2) : s; - product.defaultPath = match[5]; - - /* Ensure that the path exists and points into the Nix - store. */ - // FIXME: should we disallow products referring to other - // store paths, or that are outside the input closure? - if (product.path == "" || product.path[0] != '/') continue; - product.path = canonPath(product.path); - if (!store->isInStore(product.path)) continue; - - auto file = narMembers.find(product.path); - if (file == narMembers.end()) continue; - - product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path); - if (!std::regex_match(product.name, std::regex("[a-zA-Z0-9.@:_ -]*"))) - product.name = ""; - - if (file->second.type == SourceAccessor::Type::tRegular) { - product.isRegular = true; - product.fileSize = file->second.fileSize.value(); - product.sha256hash = file->second.sha256.value(); - } - - res.products.push_back(product); - } - } - - /* If no build products were explicitly declared, then add all - outputs as a product of type "nix-build". */ - if (!explicitProducts) { - for (auto & [name, output] : derivationOutputs) { - BuildProduct product; - product.path = store->printStorePath(output); - product.type = "nix-build"; - product.subtype = name == "out" ? "" : name; - product.name = output.name(); - - auto file = narMembers.find(product.path); - assert(file != narMembers.end()); - if (file->second.type == SourceAccessor::Type::tDirectory) - res.products.push_back(product); - } - } - - /* Get the release name from $output/nix-support/hydra-release-name. */ - for (auto & output : outputs) { - auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-release-name"); - if (file == narMembers.end() || - file->second.type != SourceAccessor::Type::tRegular) - continue; - auto contents = trim(file->second.contents.value()); - if (std::regex_match(contents, std::regex("[a-zA-Z0-9.@:_-]+"))) - res.releaseName = contents; - } - - /* Get metrics. */ - for (auto & output : outputs) { - auto file = narMembers.find(store->printStorePath(output) + "/nix-support/hydra-metrics"); - if (file == narMembers.end() || - file->second.type != SourceAccessor::Type::tRegular) - continue; - for (auto & line : tokenizeString(file->second.contents.value(), "\n")) { - auto fields = tokenizeString>(line); - if (fields.size() < 2) continue; - if (!std::regex_match(fields[0], std::regex("[a-zA-Z0-9._-]+"))) - continue; - BuildMetric metric; - metric.name = fields[0]; - try { - metric.value = std::stod(fields[1]); - } catch (...) { - continue; // skip this metric - } - metric.unit = fields.size() >= 3 ? fields[2] : ""; - if (!std::regex_match(metric.unit, std::regex("[a-zA-Z0-9._%-]+"))) - metric.unit = ""; - res.metrics[metric.name] = metric; - } - } - - return res; -} diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc deleted file mode 100644 index 85f1c8d3f..000000000 --- a/src/hydra-queue-runner/builder.cc +++ /dev/null @@ -1,506 +0,0 @@ -#include - -#include "state.hh" -#include "hydra-build-result.hh" -#include -#include - -using namespace nix; - - -void setThreadName(const std::string & name) -{ -#ifdef __linux__ - pthread_setname_np(pthread_self(), std::string(name, 0, 15).c_str()); -#endif -} - - -void State::builder(std::unique_ptr reservation) -{ - setThreadName("bld~" + std::string(reservation->step->drvPath.to_string())); - - StepResult res = sRetry; - - nrStepsStarted++; - - Step::wptr wstep = reservation->step; - - { - auto activeStep = std::make_shared(); - activeStep->step = reservation->step; - activeSteps_.lock()->insert(activeStep); - - Finally removeActiveStep([&]() { - activeSteps_.lock()->erase(activeStep); - }); - - std::string machine = reservation->machine->storeUri.render(); - - try { - auto destStore = getDestStore(); - // Might release the reservation. - res = doBuildStep(destStore, std::move(reservation), activeStep); - } catch (std::exception & e) { - printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s", - localStore->printStorePath(activeStep->step->drvPath), - machine, - e.what()); - } - } - - /* If there was a temporary failure, retry the step after an - exponentially increasing interval. */ - Step::ptr step = wstep.lock(); - if (res != sDone && step) { - - if (res == sRetry) { - auto step_(step->state.lock()); - step_->tries++; - nrRetries++; - if (step_->tries > maxNrRetries) maxNrRetries = step_->tries; // yeah yeah, not atomic - int delta = retryInterval * std::pow(retryBackoff, step_->tries - 1) + (rand() % 10); - printMsg(lvlInfo, "will retry ‘%s’ after %ss", localStore->printStorePath(step->drvPath), delta); - step_->after = std::chrono::system_clock::now() + std::chrono::seconds(delta); - } - - makeRunnable(step); - } -} - - -State::StepResult State::doBuildStep(nix::ref destStore, - std::unique_ptr reservation, - std::shared_ptr activeStep) -{ - auto step(reservation->step); - auto machine(reservation->machine); - - { - auto step_(step->state.lock()); - assert(step_->created); - assert(!step->finished); - } - - /* There can be any number of builds in the database that depend - on this derivation. Arbitrarily pick one (though preferring a - build of which this is the top-level derivation) for the - purpose of creating build steps. We could create a build step - record for every build, but that could be very expensive - (e.g. a stdenv derivation can be a dependency of tens of - thousands of builds), so we don't. - - We don't keep a Build::ptr here to allow - State::processQueueChange() to detect whether a step can be - cancelled (namely if there are no more Builds referring to - it). */ - BuildID buildId; - std::optional buildDrvPath; - // Other fields set below - nix::ServeProto::BuildOptions buildOptions { - .maxLogSize = maxLogSize, - .nrRepeats = step->isDeterministic ? 1u : 0u, - .enforceDeterminism = step->isDeterministic, - .keepFailed = false, - }; - - auto conn(dbPool.get()); - - { - std::set dependents; - std::set steps; - getDependents(step, dependents, steps); - - if (dependents.empty()) { - /* Apparently all builds that depend on this derivation - are gone (e.g. cancelled). So don't bother. This is - very unlikely to happen, because normally Steps are - only kept alive by being reachable from a - Build. However, it's possible that a new Build just - created a reference to this step. So to handle that - possibility, we retry this step (putting it back in - the runnable queue). If there are really no strong - pointers to the step, it will be deleted. */ - printMsg(lvlInfo, "maybe cancelling build step ‘%s’", localStore->printStorePath(step->drvPath)); - return sMaybeCancelled; - } - - Build::ptr build; - - for (auto build2 : dependents) { - if (build2->drvPath == step->drvPath) { - build = build2; - pqxx::work txn(*conn); - notifyBuildStarted(txn, build->id); - txn.commit(); - } - { - auto i = jobsetRepeats.find(std::make_pair(build2->projectName, build2->jobsetName)); - if (i != jobsetRepeats.end()) - buildOptions.nrRepeats = std::max(buildOptions.nrRepeats, i->second); - } - } - if (!build) build = *dependents.begin(); - - buildId = build->id; - buildDrvPath = build->drvPath; - buildOptions.maxSilentTime = build->maxSilentTime; - buildOptions.buildTimeout = build->buildTimeout; - - printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", - localStore->printStorePath(step->drvPath), buildOptions.nrRepeats + 1, machine->storeUri.render(), buildId, (dependents.size() - 1)); - } - - if (!buildOneDone) - buildOneDone = buildId == buildOne && step->drvPath == *buildDrvPath; - - RemoteResult result; - BuildOutput res; - unsigned int stepNr = 0; - bool stepFinished = false; - - Finally clearStep([&]() { - if (stepNr && !stepFinished) { - printError("marking step %d of build %d as orphaned", stepNr, buildId); - auto orphanedSteps_(orphanedSteps.lock()); - orphanedSteps_->emplace(buildId, stepNr); - } - - if (stepNr) { - /* Upload the log file to the binary cache. FIXME: should - be done on a worker thread. */ - try { - auto store = destStore.dynamic_pointer_cast(); - if (uploadLogsToBinaryCache && store && pathExists(result.logFile)) { - store->upsertFile("log/" + std::string(step->drvPath.to_string()), readFile(result.logFile), "text/plain; charset=utf-8"); - unlink(result.logFile.c_str()); - } - } catch (...) { - ignoreExceptionInDestructor(); - } - } - }); - - time_t stepStartTime = result.startTime = time(0); - - /* If any of the outputs have previously failed, then don't bother - building again. */ - if (checkCachedFailure(step, *conn)) - result.stepStatus = bsCachedFailure; - else { - - /* Create a build step record indicating that we started - building. */ - { - auto mc = startDbUpdate(); - pqxx::work txn(*conn); - stepNr = createBuildStep(txn, result.startTime, buildId, step, machine->storeUri.render(), bsBusy); - txn.commit(); - } - - auto updateStep = [&](StepState stepState) { - pqxx::work txn(*conn); - updateBuildStep(txn, buildId, stepNr, stepState); - txn.commit(); - }; - - /* Do the build. */ - NarMemberDatas narMembers; - - try { - /* FIXME: referring builds may have conflicting timeouts. */ - buildRemote(destStore, std::move(reservation), machine, step, buildOptions, result, activeStep, updateStep, narMembers); - } catch (Error & e) { - if (activeStep->state_.lock()->cancelled) { - printInfo("marking step %d of build %d as cancelled", stepNr, buildId); - result.stepStatus = bsCancelled; - result.canRetry = false; - } else { - result.stepStatus = bsAborted; - result.errorMsg = e.msg(); - result.canRetry = true; - } - } - - if (result.stepStatus == bsSuccess) { - updateStep(ssPostProcessing); - res = getBuildOutput(destStore, narMembers, destStore->queryDerivationOutputMap(step->drvPath, &*localStore)); - } - } - - time_t stepStopTime = time(0); - if (!result.stopTime) result.stopTime = stepStopTime; - - /* For standard failures, we don't care about the error - message. */ - if (result.stepStatus != bsAborted) - result.errorMsg = ""; - - /* Account the time we spent building this step by dividing it - among the jobsets that depend on it. */ - { - auto step_(step->state.lock()); - if (!step_->jobsets.empty()) { - // FIXME: loss of precision. - time_t charge = (result.stopTime - result.startTime) / step_->jobsets.size(); - for (auto & jobset : step_->jobsets) - jobset->addStep(result.startTime, charge); - } - } - - /* Finish the step in the database. */ - if (stepNr) { - pqxx::work txn(*conn); - finishBuildStep(txn, result, buildId, stepNr, machine->storeUri.render()); - txn.commit(); - } - - /* The step had a hopefully temporary failure (e.g. network - issue). Retry a number of times. */ - if (result.canRetry) { - printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s", - localStore->printStorePath(step->drvPath), machine->storeUri.render(), result.errorMsg); - assert(stepNr); - bool retry; - { - auto step_(step->state.lock()); - retry = step_->tries + 1 < maxTries; - } - if (retry) { - auto mc = startDbUpdate(); - stepFinished = true; - if (buildOneDone) exit(1); - return sRetry; - } - } - - if (result.stepStatus == bsSuccess) { - - assert(stepNr); - - for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) { - if (!optOutputPath) - throw Error( - "Missing output %s for derivation %d which was supposed to have succeeded", - outputName, localStore->printStorePath(step->drvPath)); - addRoot(*optOutputPath); - } - - /* Register success in the database for all Build objects that - have this step as the top-level step. Since the queue - monitor thread may be creating new referring Builds - concurrently, and updating the database may fail, we do - this in a loop, marking all known builds, repeating until - there are no unmarked builds. - */ - - std::vector buildIDs; - - while (true) { - - /* Get the builds that have this one as the top-level. */ - std::vector direct; - { - auto steps_(steps.lock()); - auto step_(step->state.lock()); - - for (auto & b_ : step_->builds) { - auto b = b_.lock(); - if (b && !b->finishedInDB) direct.push_back(b); - } - - /* If there are no builds left to update in the DB, - then we're done (except for calling - finishBuildStep()). Delete the step from - ‘steps’. Since we've been holding the ‘steps’ lock, - no new referrers can have been added in the - meantime or be added afterwards. */ - if (direct.empty()) { - printMsg(lvlDebug, "finishing build step ‘%s’", - localStore->printStorePath(step->drvPath)); - steps_->erase(step->drvPath); - } - } - - /* Update the database. */ - { - auto mc = startDbUpdate(); - - pqxx::work txn(*conn); - - for (auto & b : direct) { - printInfo("marking build %1% as succeeded", b->id); - markSucceededBuild(txn, b, res, buildId != b->id || result.isCached, - result.startTime, result.stopTime); - } - - txn.commit(); - } - - stepFinished = true; - - if (direct.empty()) break; - - /* Remove the direct dependencies from ‘builds’. This will - cause them to be destroyed. */ - for (auto & b : direct) { - auto builds_(builds.lock()); - b->finishedInDB = true; - builds_->erase(b->id); - buildIDs.push_back(b->id); - } - } - - /* Send notification about the builds that have this step as - the top-level. */ - { - pqxx::work txn(*conn); - for (auto id : buildIDs) - notifyBuildFinished(txn, id, {}); - txn.commit(); - } - - /* Wake up any dependent steps that have no other - dependencies. */ - { - auto step_(step->state.lock()); - for (auto & rdepWeak : step_->rdeps) { - auto rdep = rdepWeak.lock(); - if (!rdep) continue; - - bool runnable = false; - { - auto rdep_(rdep->state.lock()); - rdep_->deps.erase(step); - /* Note: if the step has not finished - initialisation yet, it will be made runnable in - createStep(), if appropriate. */ - if (rdep_->deps.empty() && rdep_->created) runnable = true; - } - - if (runnable) makeRunnable(rdep); - } - } - - } else - failStep(*conn, step, buildId, result, machine, stepFinished); - - // FIXME: keep stats about aborted steps? - nrStepsDone++; - totalStepTime += stepStopTime - stepStartTime; - totalStepBuildTime += result.stopTime - result.startTime; - machine->state->nrStepsDone++; - machine->state->totalStepTime += stepStopTime - stepStartTime; - machine->state->totalStepBuildTime += result.stopTime - result.startTime; - - if (buildOneDone) exit(0); // testing hack; FIXME: this won't run plugins - - return sDone; -} - - -void State::failStep( - Connection & conn, - Step::ptr step, - BuildID buildId, - const RemoteResult & result, - ::Machine::ptr machine, - bool & stepFinished) -{ - /* Register failure in the database for all Build objects that - directly or indirectly depend on this step. */ - - std::vector dependentIDs; - - while (true) { - /* Get the builds and steps that depend on this step. */ - std::set indirect; - { - auto steps_(steps.lock()); - std::set steps; - getDependents(step, indirect, steps); - - /* If there are no builds left, delete all referring - steps from ‘steps’. As for the success case, we can - be certain no new referrers can be added. */ - if (indirect.empty()) { - for (auto & s : steps) { - printMsg(lvlDebug, "finishing build step ‘%s’", - localStore->printStorePath(s->drvPath)); - steps_->erase(s->drvPath); - } - } - } - - if (indirect.empty() && stepFinished) break; - - /* Update the database. */ - { - auto mc = startDbUpdate(); - - pqxx::work txn(conn); - - /* Create failed build steps for every build that - depends on this, except when this step is cached - and is the top-level of that build (since then it's - redundant with the build's isCachedBuild field). */ - for (auto & build : indirect) { - if ((result.stepStatus == bsCachedFailure && build->drvPath == step->drvPath) || - ((result.stepStatus != bsCachedFailure && result.stepStatus != bsUnsupported) && buildId == build->id) || - build->finishedInDB) - continue; - createBuildStep(txn, - 0, build->id, step, machine ? machine->storeUri.render() : "", - result.stepStatus, result.errorMsg, buildId == build->id ? 0 : buildId); - } - - /* Mark all builds that depend on this derivation as failed. */ - for (auto & build : indirect) { - if (build->finishedInDB) continue; - printError("marking build %1% as failed", build->id); - txn.exec("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5, notificationPendingSince = $4 where id = $1 and finished = 0", - pqxx::params{build->id, - (int) (build->drvPath != step->drvPath && result.buildStatus() == bsFailed ? bsDepFailed : result.buildStatus()), - result.startTime, - result.stopTime, - result.stepStatus == bsCachedFailure ? 1 : 0}).no_rows(); - nrBuildsDone++; - } - - /* Remember failed paths in the database so that they - won't be built again. */ - if (result.stepStatus != bsCachedFailure && result.canCache) - for (auto & i : step->drv->outputsAndOptPaths(*localStore)) - if (i.second.second) - txn.exec("insert into FailedPaths values ($1)", pqxx::params{localStore->printStorePath(*i.second.second)}).no_rows(); - - txn.commit(); - } - - stepFinished = true; - - /* Remove the indirect dependencies from ‘builds’. This - will cause them to be destroyed. */ - for (auto & b : indirect) { - auto builds_(builds.lock()); - b->finishedInDB = true; - builds_->erase(b->id); - dependentIDs.push_back(b->id); - if (!buildOneDone && buildOne == b->id) buildOneDone = true; - } - } - - /* Send notification about this build and its dependents. */ - { - pqxx::work txn(conn); - notifyBuildFinished(txn, buildId, dependentIDs); - txn.commit(); - } -} - - -void State::addRoot(const StorePath & storePath) -{ - auto root = rootsDir + "/" + std::string(storePath.to_string()); - if (!pathExists(root)) writeFile(root, ""); -} diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc deleted file mode 100644 index ada25dc62..000000000 --- a/src/hydra-queue-runner/dispatcher.cc +++ /dev/null @@ -1,478 +0,0 @@ -#include -#include -#include -#include -#include - -#include "state.hh" - -using namespace nix; - - -void State::makeRunnable(Step::ptr step) -{ - printMsg(lvlChatty, "step ‘%s’ is now runnable", localStore->printStorePath(step->drvPath)); - - { - auto step_(step->state.lock()); - assert(step_->created); - assert(!step->finished); - assert(step_->deps.empty()); - step_->runnableSince = std::chrono::system_clock::now(); - } - - { - auto runnable_(runnable.lock()); - runnable_->push_back(step); - } - - wakeDispatcher(); -} - - -void State::dispatcher() -{ - printMsg(lvlDebug, "Waiting for the machines parsing to have completed at least once"); - machinesReadyLock.lock(); - - while (true) { - try { - printMsg(lvlDebug, "dispatcher woken up"); - nrDispatcherWakeups++; - - auto t_before_work = std::chrono::steady_clock::now(); - - auto sleepUntil = doDispatch(); - - auto t_after_work = std::chrono::steady_clock::now(); - - prom.dispatcher_time_spent_running.Increment( - std::chrono::duration_cast(t_after_work - t_before_work).count()); - dispatchTimeMs += std::chrono::duration_cast(t_after_work - t_before_work).count(); - - /* Sleep until we're woken up (either because a runnable build - is added, or because a build finishes). */ - { - auto dispatcherWakeup_(dispatcherWakeup.lock()); - if (!*dispatcherWakeup_) { - debug("dispatcher sleeping for %1%s", - std::chrono::duration_cast(sleepUntil - std::chrono::system_clock::now()).count()); - dispatcherWakeup_.wait_until(dispatcherWakeupCV, sleepUntil); - } - *dispatcherWakeup_ = false; - } - - auto t_after_sleep = std::chrono::steady_clock::now(); - prom.dispatcher_time_spent_waiting.Increment( - std::chrono::duration_cast(t_after_sleep - t_after_work).count()); - - } catch (std::exception & e) { - printError("dispatcher: %s", e.what()); - sleep(1); - } - - } - - printMsg(lvlError, "dispatcher exits"); -} - - -system_time State::doDispatch() -{ - /* Prune old historical build step info from the jobsets. */ - { - auto jobsets_(jobsets.lock()); - for (auto & jobset : *jobsets_) { - auto s1 = jobset.second->shareUsed(); - jobset.second->pruneSteps(); - auto s2 = jobset.second->shareUsed(); - if (s1 != s2) - debug("pruned scheduling window of ‘%1%:%2%’ from %3% to %4%", - jobset.first.first, jobset.first.second, s1, s2); - } - } - - system_time now = std::chrono::system_clock::now(); - - /* Start steps until we're out of steps or slots. */ - auto sleepUntil = system_time::max(); - bool keepGoing; - - /* Sort the runnable steps by priority. Priority is establised - as follows (in order of precedence): - - - The global priority of the builds that depend on the - step. This allows admins to bump a build to the front of - the queue. - - - The lowest used scheduling share of the jobsets depending - on the step. - - - The local priority of the build, as set via the build's - meta.schedulingPriority field. Note that this is not - quite correct: the local priority should only be used to - establish priority between builds in the same jobset, but - here it's used between steps in different jobsets if they - happen to have the same lowest used scheduling share. But - that's not very likely. - - - The lowest ID of the builds depending on the step; - i.e. older builds take priority over new ones. - - FIXME: O(n lg n); obviously, it would be better to keep a - runnable queue sorted by priority. */ - struct StepInfo - { - Step::ptr step; - bool alreadyScheduled = false; - - /* The lowest share used of any jobset depending on this - step. */ - double lowestShareUsed = 1e9; - - /* Info copied from step->state to ensure that the - comparator is a partial ordering (see MachineInfo). */ - int highestGlobalPriority; - int highestLocalPriority; - size_t numRequiredSystemFeatures; - size_t numRevDeps; - BuildID lowestBuildID; - - StepInfo(Step::ptr step, Step::State & step_) : step(step) - { - for (auto & jobset : step_.jobsets) - lowestShareUsed = std::min(lowestShareUsed, jobset->shareUsed()); - highestGlobalPriority = step_.highestGlobalPriority; - highestLocalPriority = step_.highestLocalPriority; - numRequiredSystemFeatures = step->requiredSystemFeatures.size(); - numRevDeps = step_.rdeps.size(); - lowestBuildID = step_.lowestBuildID; - } - }; - - std::vector runnableSorted; - - struct RunnablePerType - { - unsigned int count{0}; - std::chrono::seconds waitTime{0}; - }; - - std::unordered_map runnablePerType; - - { - auto runnable_(runnable.lock()); - runnableSorted.reserve(runnable_->size()); - for (auto i = runnable_->begin(); i != runnable_->end(); ) { - auto step = i->lock(); - - /* Remove dead steps. */ - if (!step) { - i = runnable_->erase(i); - continue; - } - - ++i; - - auto & r = runnablePerType[step->systemType]; - r.count++; - - /* Skip previously failed steps that aren't ready - to be retried. */ - auto step_(step->state.lock()); - r.waitTime += std::chrono::duration_cast(now - step_->runnableSince); - if (step_->tries > 0 && step_->after > now) { - if (step_->after < sleepUntil) - sleepUntil = step_->after; - continue; - } - - runnableSorted.emplace_back(step, *step_); - } - } - - sort(runnableSorted.begin(), runnableSorted.end(), - [](const StepInfo & a, const StepInfo & b) - { - return - a.highestGlobalPriority != b.highestGlobalPriority ? a.highestGlobalPriority > b.highestGlobalPriority : - a.lowestShareUsed != b.lowestShareUsed ? a.lowestShareUsed < b.lowestShareUsed : - a.highestLocalPriority != b.highestLocalPriority ? a.highestLocalPriority > b.highestLocalPriority : - a.numRequiredSystemFeatures != b.numRequiredSystemFeatures ? a.numRequiredSystemFeatures > b.numRequiredSystemFeatures : - a.numRevDeps != b.numRevDeps ? a.numRevDeps > b.numRevDeps : - a.lowestBuildID < b.lowestBuildID; - }); - - do { - now = std::chrono::system_clock::now(); - - /* Copy the currentJobs field of each machine. This is - necessary to ensure that the sort comparator below is - an ordering. std::sort() can segfault if it isn't. Also - filter out temporarily disabled machines. */ - struct MachineInfo - { - ::Machine::ptr machine; - unsigned long currentJobs; - }; - std::vector machinesSorted; - { - auto machines_(machines.lock()); - for (auto & m : *machines_) { - auto info(m.second->state->connectInfo.lock()); - if (!m.second->enabled) continue; - if (info->consecutiveFailures && info->disabledUntil > now) { - if (info->disabledUntil < sleepUntil) - sleepUntil = info->disabledUntil; - continue; - } - machinesSorted.push_back({m.second, m.second->state->currentJobs}); - } - } - - /* Sort the machines by a combination of speed factor and - available slots. Prioritise the available machines as - follows: - - - First by load divided by speed factor, rounded to the - nearest integer. This causes fast machines to be - preferred over slow machines with similar loads. - - - Then by speed factor. - - - Finally by load. */ - sort(machinesSorted.begin(), machinesSorted.end(), - [](const MachineInfo & a, const MachineInfo & b) -> bool - { - float ta = std::round(a.currentJobs / a.machine->speedFactor); - float tb = std::round(b.currentJobs / b.machine->speedFactor); - return - ta != tb ? ta < tb : - a.machine->speedFactor != b.machine->speedFactor ? a.machine->speedFactor > b.machine->speedFactor : - a.currentJobs > b.currentJobs; - }); - - /* Find a machine with a free slot and find a step to run - on it. Once we find such a pair, we restart the outer - loop because the machine sorting will have changed. */ - keepGoing = false; - - for (auto & mi : machinesSorted) { - if (mi.machine->state->currentJobs >= mi.machine->maxJobs) continue; - - for (auto & stepInfo : runnableSorted) { - if (stepInfo.alreadyScheduled) continue; - - auto & step(stepInfo.step); - - /* Can this machine do this step? */ - if (!mi.machine->supportsStep(step)) { - debug("machine '%s' does not support step '%s' (system type '%s')", - mi.machine->storeUri.render(), localStore->printStorePath(step->drvPath), step->drv->platform); - continue; - } - - /* Let's do this step. Remove it from the runnable - list. FIXME: O(n). */ - { - auto runnable_(runnable.lock()); - bool removed = false; - for (auto i = runnable_->begin(); i != runnable_->end(); ) - if (i->lock() == step) { - i = runnable_->erase(i); - removed = true; - break; - } else ++i; - assert(removed); - auto & r = runnablePerType[step->systemType]; - assert(r.count); - r.count--; - } - - stepInfo.alreadyScheduled = true; - - /* Make a slot reservation and start a thread to - do the build. */ - auto builderThread = std::thread(&State::builder, this, - std::make_unique(*this, step, mi.machine)); - builderThread.detach(); // FIXME? - - keepGoing = true; - break; - } - - if (keepGoing) break; - } - - /* Update the stats for the auto-scaler. */ - { - auto machineTypes_(machineTypes.lock()); - - for (auto & i : *machineTypes_) - i.second.runnable = 0; - - for (auto & i : runnablePerType) { - auto & j = (*machineTypes_)[i.first]; - j.runnable = i.second.count; - j.waitTime = i.second.waitTime; - } - } - - lastDispatcherCheck = std::chrono::system_clock::to_time_t(now); - - } while (keepGoing); - - abortUnsupported(); - - return sleepUntil; -} - - -void State::wakeDispatcher() -{ - { - auto dispatcherWakeup_(dispatcherWakeup.lock()); - *dispatcherWakeup_ = true; - } - dispatcherWakeupCV.notify_one(); -} - - -void State::abortUnsupported() -{ - /* Make a copy of 'runnable' and 'machines' so we don't block them - very long. */ - auto runnable2 = *runnable.lock(); - auto machines2 = *machines.lock(); - - system_time now = std::chrono::system_clock::now(); - auto now2 = time(0); - - std::unordered_set aborted; - - size_t count = 0; - - for (auto & wstep : runnable2) { - auto step(wstep.lock()); - if (!step) continue; - - bool supported = false; - for (auto & machine : machines2) { - if (machine.second->supportsStep(step)) { - step->state.lock()->lastSupported = now; - supported = true; - break; - } - } - - if (!supported) - count++; - - if (!supported - && std::chrono::duration_cast(now - step->state.lock()->lastSupported).count() >= maxUnsupportedTime) - { - printError("aborting unsupported build step '%s' (type '%s')", - localStore->printStorePath(step->drvPath), - step->systemType); - - aborted.insert(step); - - auto conn(dbPool.get()); - - std::set dependents; - std::set steps; - getDependents(step, dependents, steps); - - /* Maybe the step got cancelled. */ - if (dependents.empty()) continue; - - /* Find the build that has this step as the top-level (if - any). */ - Build::ptr build; - for (auto build2 : dependents) { - if (build2->drvPath == step->drvPath) - build = build2; - } - if (!build) build = *dependents.begin(); - - bool stepFinished = false; - - failStep( - *conn, step, build->id, - RemoteResult { - .stepStatus = bsUnsupported, - .errorMsg = fmt("unsupported system type '%s'", - step->systemType), - .startTime = now2, - .stopTime = now2, - }, - nullptr, stepFinished); - - if (buildOneDone) exit(1); - } - } - - /* Clean up 'runnable'. */ - { - auto runnable_(runnable.lock()); - for (auto i = runnable_->begin(); i != runnable_->end(); ) { - if (aborted.count(i->lock())) - i = runnable_->erase(i); - else - ++i; - } - } - - nrUnsupportedSteps = count; -} - - -void Jobset::addStep(time_t startTime, time_t duration) -{ - auto steps_(steps.lock()); - (*steps_)[startTime] = duration; - seconds += duration; -} - - -void Jobset::pruneSteps() -{ - time_t now = time(0); - auto steps_(steps.lock()); - while (!steps_->empty()) { - auto i = steps_->begin(); - if (i->first > now - schedulingWindow) break; - seconds -= i->second; - steps_->erase(i); - } -} - - -State::MachineReservation::MachineReservation(State & state, Step::ptr step, ::Machine::ptr machine) - : state(state), step(step), machine(machine) -{ - machine->state->currentJobs++; - - { - auto machineTypes_(state.machineTypes.lock()); - (*machineTypes_)[step->systemType].running++; - } -} - - -State::MachineReservation::~MachineReservation() -{ - auto prev = machine->state->currentJobs--; - assert(prev); - if (prev == 1) - machine->state->idleSince = time(0); - - { - auto machineTypes_(state.machineTypes.lock()); - auto & machineType = (*machineTypes_)[step->systemType]; - assert(machineType.running); - machineType.running--; - if (machineType.running == 0) - machineType.lastActive = std::chrono::system_clock::now(); - } -} diff --git a/src/hydra-queue-runner/hydra-build-result.hh b/src/hydra-queue-runner/hydra-build-result.hh deleted file mode 100644 index 654bf1be9..000000000 --- a/src/hydra-queue-runner/hydra-build-result.hh +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include "nar-extractor.hh" - -struct BuildProduct -{ - nix::Path path, defaultPath; - std::string type, subtype, name; - bool isRegular = false; - std::optional sha256hash; - std::optional fileSize; - BuildProduct() { } -}; - -struct BuildMetric -{ - std::string name, unit; - double value; -}; - -struct BuildOutput -{ - /* Whether this build has failed with output, i.e., the build - finished with exit code 0 but produced a file - $out/nix-support/failed. */ - bool failed = false; - - std::string releaseName; - - uint64_t closureSize = 0, size = 0; - - std::list products; - - std::map outputs; - - std::map metrics; -}; - -BuildOutput getBuildOutput( - nix::ref store, - NarMemberDatas & narMembers, - const nix::OutputPathMap derivationOutputs); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc deleted file mode 100644 index 7cdc04b3c..000000000 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ /dev/null @@ -1,968 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include - -#include -#include "state.hh" -#include "hydra-build-result.hh" -#include -#include - -#include -#include "hydra-config.hh" -#include -#include - -using namespace nix; -using nlohmann::json; - - -std::string getEnvOrDie(const std::string & key) -{ - auto value = getEnv(key); - if (!value) throw Error("environment variable '%s' is not set", key); - return *value; -} - -State::PromMetrics::PromMetrics() - : registry(std::make_shared()) - , queue_checks_started( - prometheus::BuildCounter() - .Name("hydraqueuerunner_queue_checks_started_total") - .Help("Number of times State::getQueuedBuilds() was started") - .Register(*registry) - .Add({}) - ) - , queue_build_loads( - prometheus::BuildCounter() - .Name("hydraqueuerunner_queue_build_loads_total") - .Help("Number of builds loaded") - .Register(*registry) - .Add({}) - ) - , queue_steps_created( - prometheus::BuildCounter() - .Name("hydraqueuerunner_queue_steps_created_total") - .Help("Number of steps created") - .Register(*registry) - .Add({}) - ) - , queue_checks_early_exits( - prometheus::BuildCounter() - .Name("hydraqueuerunner_queue_checks_early_exits_total") - .Help("Number of times State::getQueuedBuilds() yielded to potential bumps") - .Register(*registry) - .Add({}) - ) - , queue_checks_finished( - prometheus::BuildCounter() - .Name("hydraqueuerunner_queue_checks_finished_total") - .Help("Number of times State::getQueuedBuilds() was completed") - .Register(*registry) - .Add({}) - ) - , dispatcher_time_spent_running( - prometheus::BuildCounter() - .Name("hydraqueuerunner_dispatcher_time_spent_running") - .Help("Time (in micros) spent running the dispatcher") - .Register(*registry) - .Add({}) - ) - , dispatcher_time_spent_waiting( - prometheus::BuildCounter() - .Name("hydraqueuerunner_dispatcher_time_spent_waiting") - .Help("Time (in micros) spent waiting for the dispatcher to obtain work") - .Register(*registry) - .Add({}) - ) - , queue_monitor_time_spent_running( - prometheus::BuildCounter() - .Name("hydraqueuerunner_queue_monitor_time_spent_running") - .Help("Time (in micros) spent running the queue monitor") - .Register(*registry) - .Add({}) - ) - , queue_monitor_time_spent_waiting( - prometheus::BuildCounter() - .Name("hydraqueuerunner_queue_monitor_time_spent_waiting") - .Help("Time (in micros) spent waiting for the queue monitor to obtain work") - .Register(*registry) - .Add({}) - ) -{ - -} - -State::State(std::optional metricsAddrOpt) - : config(std::make_unique()) - , maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0)) - , dbPool(config->getIntOption("max_db_connections", 128)) - , localWorkThrottler(config->getIntOption("max_local_worker_threads", std::min(maxSupportedLocalWorkers, std::max(4u, std::thread::hardware_concurrency()) - 2))) - , maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30)) - , maxLogSize(config->getIntOption("max_log_size", 64ULL << 20)) - , uploadLogsToBinaryCache(config->getBoolOption("upload_logs_to_binary_cache", false)) - , rootsDir(config->getStrOption("gc_roots_dir", fmt("%s/gcroots/per-user/%s/hydra-roots", settings.nixStateDir, getEnvOrDie("LOGNAME")))) - , metricsAddr(config->getStrOption("queue_runner_metrics_address", std::string{"127.0.0.1:9198"})) -{ - hydraData = getEnvOrDie("HYDRA_DATA"); - - logDir = canonPath(hydraData + "/build-logs"); - - if (metricsAddrOpt.has_value()) { - metricsAddr = metricsAddrOpt.value(); - } - - /* handle deprecated store specification */ - if (config->getStrOption("store_mode") != "") - throw Error("store_mode in hydra.conf is deprecated, please use store_uri"); - if (config->getStrOption("binary_cache_dir") != "") - printMsg(lvlError, "hydra.conf: binary_cache_dir is deprecated and ignored. use store_uri=file:// instead"); - if (config->getStrOption("binary_cache_s3_bucket") != "") - printMsg(lvlError, "hydra.conf: binary_cache_s3_bucket is deprecated and ignored. use store_uri=s3:// instead"); - if (config->getStrOption("binary_cache_secret_key_file") != "") - printMsg(lvlError, "hydra.conf: binary_cache_secret_key_file is deprecated and ignored. use store_uri=...?secret-key= instead"); - - createDirs(rootsDir); -} - - -nix::MaintainCount State::startDbUpdate() -{ - if (nrActiveDbUpdates > 6) - printError("warning: %d concurrent database updates; PostgreSQL may be stalled", nrActiveDbUpdates.load()); - return MaintainCount(nrActiveDbUpdates); -} - - -ref State::getDestStore() -{ - return ref(_destStore); -} - - -void State::parseMachines(const std::string & contents) -{ - Machines newMachines, oldMachines; - { - auto machines_(machines.lock()); - oldMachines = *machines_; - } - - for (auto && machine_ : nix::Machine::parseConfig({}, contents)) { - auto machine = std::make_shared<::Machine>(std::move(machine_)); - - /* Re-use the State object of the previous machine with the - same name. */ - auto i = oldMachines.find(machine->storeUri.variant); - if (i == oldMachines.end()) - printMsg(lvlChatty, "adding new machine ‘%1%’", machine->storeUri.render()); - else - printMsg(lvlChatty, "updating machine ‘%1%’", machine->storeUri.render()); - machine->state = i == oldMachines.end() - ? std::make_shared<::Machine::State>() - : i->second->state; - newMachines[machine->storeUri.variant] = machine; - } - - for (auto & m : oldMachines) - if (newMachines.find(m.first) == newMachines.end()) { - if (m.second->enabled) - printInfo("removing machine ‘%1%’", m.second->storeUri.render()); - /* Add a disabled ::Machine object to make sure stats are - maintained. */ - auto machine = std::make_shared<::Machine>(*(m.second)); - machine->enabled = false; - newMachines[m.first] = machine; - } - - static bool warned = false; - if (newMachines.empty() && !warned) { - printError("warning: no build machines are defined"); - warned = true; - } - - auto machines_(machines.lock()); - *machines_ = newMachines; - - wakeDispatcher(); -} - - -void State::monitorMachinesFile() -{ - std::string defaultMachinesFile = "/etc/nix/machines"; - auto machinesFiles = tokenizeString>( - getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":"); - - if (machinesFiles.empty()) { - parseMachines("localhost " + - (settings.thisSystem == "x86_64-linux" ? "x86_64-linux,i686-linux" : settings.thisSystem.get()) - + " - " + std::to_string(settings.maxBuildJobs) + " 1 " - + concatStringsSep(",", StoreConfig::getDefaultSystemFeatures())); - machinesReadyLock.unlock(); - return; - } - - std::vector fileStats; - fileStats.resize(machinesFiles.size()); - for (unsigned int n = 0; n < machinesFiles.size(); ++n) { - auto & st(fileStats[n]); - st.st_ino = st.st_mtime = 0; - } - - auto readMachinesFiles = [&]() { - - /* Check if any of the machines files changed. */ - bool anyChanged = false; - for (unsigned int n = 0; n < machinesFiles.size(); ++n) { - Path machinesFile = machinesFiles[n]; - struct stat st; - if (stat(machinesFile.c_str(), &st) != 0) { - if (errno != ENOENT) - throw SysError("getting stats about ‘%s’", machinesFile); - st.st_ino = st.st_mtime = 0; - } - auto & old(fileStats[n]); - if (old.st_ino != st.st_ino || old.st_mtime != st.st_mtime) - anyChanged = true; - old = st; - } - - if (!anyChanged) return; - - debug("reloading machines files"); - - std::string contents; - for (auto & machinesFile : machinesFiles) { - try { - contents += readFile(machinesFile); - contents += '\n'; - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - } - - parseMachines(contents); - }; - - auto firstParse = true; - - while (true) { - try { - readMachinesFiles(); - if (firstParse) { - machinesReadyLock.unlock(); - firstParse = false; - } - // FIXME: use inotify. - sleep(30); - } catch (std::exception & e) { - printMsg(lvlError, "reloading machines file: %s", e.what()); - sleep(5); - } - } -} - - -void State::clearBusy(Connection & conn, time_t stopTime) -{ - pqxx::work txn(conn); - txn.exec("update BuildSteps set busy = 0, status = $1, stopTime = $2 where busy != 0", - pqxx::params{(int) bsAborted, - stopTime != 0 ? std::make_optional(stopTime) : std::nullopt}).no_rows(); - txn.commit(); -} - - -unsigned int State::allocBuildStep(pqxx::work & txn, BuildID buildId) -{ - auto res = txn.exec("select max(stepnr) from BuildSteps where build = $1", buildId).one_row(); - return res[0].is_null() ? 1 : res[0].as() + 1; -} - - -unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID buildId, Step::ptr step, - const std::string & machine, BuildStatus status, const std::string & errorMsg, BuildID propagatedFrom) -{ - restart: - auto stepNr = allocBuildStep(txn, buildId); - - auto r = txn.exec("insert into BuildSteps (build, stepnr, type, drvPath, busy, startTime, system, status, propagatedFrom, errorMsg, stopTime, machine) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) on conflict do nothing", - pqxx::params{buildId, - stepNr, - 0, // == build - localStore->printStorePath(step->drvPath), - status == bsBusy ? 1 : 0, - startTime != 0 ? std::make_optional(startTime) : std::nullopt, - step->drv->platform, - status != bsBusy ? std::make_optional((int) status) : std::nullopt, - propagatedFrom != 0 ? std::make_optional(propagatedFrom) : std::nullopt, // internal::params - errorMsg != "" ? std::make_optional(errorMsg) : std::nullopt, - startTime != 0 && status != bsBusy ? std::make_optional(startTime) : std::nullopt, - machine}); - - if (r.affected_rows() == 0) goto restart; - - for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) - txn.exec("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", - pqxx::params{buildId, stepNr, name, - output - ? std::optional { localStore->printStorePath(*output)} - : std::nullopt}).no_rows(); - - if (status == bsBusy) - txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); - - return stepNr; -} - - -void State::updateBuildStep(pqxx::work & txn, BuildID buildId, unsigned int stepNr, StepState stepState) -{ - if (txn.exec("update BuildSteps set busy = $1 where build = $2 and stepnr = $3 and busy != 0 and status is null", - pqxx::params{(int) stepState, - buildId, - stepNr}).affected_rows() != 1) - throw Error("step %d of build %d is in an unexpected state", stepNr, buildId); -} - - -void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result, - BuildID buildId, unsigned int stepNr, const std::string & machine) -{ - assert(result.startTime); - assert(result.stopTime); - txn.exec("update BuildSteps set busy = 0, status = $1, errorMsg = $4, startTime = $5, stopTime = $6, machine = $7, overhead = $8, timesBuilt = $9, isNonDeterministic = $10 where build = $2 and stepnr = $3", - pqxx::params{(int) result.stepStatus, buildId, stepNr, - result.errorMsg != "" ? std::make_optional(result.errorMsg) : std::nullopt, - result.startTime, result.stopTime, - machine != "" ? std::make_optional(machine) : std::nullopt, - result.overhead != 0 ? std::make_optional(result.overhead) : std::nullopt, - result.timesBuilt > 0 ? std::make_optional(result.timesBuilt) : std::nullopt, - result.timesBuilt > 1 ? std::make_optional(result.isNonDeterministic) : std::nullopt}).no_rows(); - assert(result.logFile.find('\t') == std::string::npos); - txn.exec(fmt("notify step_finished, '%d\t%d\t%s'", - buildId, stepNr, result.logFile)); - - if (result.stepStatus == bsSuccess) { - // Update the corresponding `BuildStepOutputs` row to add the output path - auto res = txn.exec("select drvPath from BuildSteps where build = $1 and stepnr = $2", pqxx::params{buildId, stepNr}).one_row(); - assert(res.size()); - StorePath drvPath = localStore->parseStorePath(res[0].as()); - // If we've finished building, all the paths should be known - for (auto & [name, output] : getDestStore()->queryDerivationOutputMap(drvPath, &*localStore)) - txn.exec("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3", - pqxx::params{buildId, stepNr, name, localStore->printStorePath(output)}).no_rows(); - } -} - - -int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const StorePath & storePath) -{ - restart: - auto stepNr = allocBuildStep(txn, build->id); - - auto r = txn.exec("insert into BuildSteps (build, stepnr, type, drvPath, busy, status, startTime, stopTime) values ($1, $2, $3, $4, $5, $6, $7, $8) on conflict do nothing", - pqxx::params{build->id, - stepNr, - 1, // == substitution - (localStore->printStorePath(drvPath)), - 0, - 0, - startTime, - stopTime}); - - if (r.affected_rows() == 0) goto restart; - - txn.exec("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", - pqxx::params{build->id, stepNr, outputName, - localStore->printStorePath(storePath)}).no_rows(); - - return stepNr; -} - - -/* Get the steps and unfinished builds that depend on the given step. */ -void getDependents(Step::ptr step, std::set & builds, std::set & steps) -{ - std::function visit; - - visit = [&](Step::ptr step) { - if (steps.count(step)) return; - steps.insert(step); - - std::vector rdeps; - - { - auto step_(step->state.lock()); - - for (auto & build : step_->builds) { - auto build_ = build.lock(); - if (build_ && !build_->finishedInDB) builds.insert(build_); - } - - /* Make a copy of rdeps so that we don't hold the lock for - very long. */ - rdeps = step_->rdeps; - } - - for (auto & rdep : rdeps) { - auto rdep_ = rdep.lock(); - if (rdep_) visit(rdep_); - } - }; - - visit(step); -} - - -void visitDependencies(std::function visitor, Step::ptr start) -{ - std::set queued; - std::queue todo; - todo.push(start); - - while (!todo.empty()) { - auto step = todo.front(); - todo.pop(); - - visitor(step); - - auto state(step->state.lock()); - for (auto & dep : state->deps) - if (queued.find(dep) == queued.end()) { - queued.insert(dep); - todo.push(dep); - } - } -} - - -void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, - const BuildOutput & res, bool isCachedBuild, time_t startTime, time_t stopTime) -{ - if (build->finishedInDB) return; - - if (txn.exec("select 1 from Builds where id = $1 and finished = 0", pqxx::params{build->id}).empty()) return; - - txn.exec("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, size = $5, closureSize = $6, releaseName = $7, isCachedBuild = $8, notificationPendingSince = $4 where id = $1", - pqxx::params{build->id, - (int) (res.failed ? bsFailedWithOutput : bsSuccess), - startTime, - stopTime, - res.size, - res.closureSize, - res.releaseName != "" ? std::make_optional(res.releaseName) : std::nullopt, - isCachedBuild ? 1 : 0}).no_rows(); - - for (auto & [outputName, outputPath] : res.outputs) { - txn.exec("update BuildOutputs set path = $3 where build = $1 and name = $2", - pqxx::params{build->id, - outputName, - localStore->printStorePath(outputPath)} - ).no_rows(); - } - - txn.exec("delete from BuildProducts where build = $1", pqxx::params{build->id}).no_rows(); - - unsigned int productNr = 1; - for (auto & product : res.products) { - txn.exec("insert into BuildProducts (build, productnr, type, subtype, fileSize, sha256hash, path, name, defaultPath) values ($1, $2, $3, $4, $5, $6, $7, $8, $9)", - pqxx::params{build->id, - productNr++, - product.type, - product.subtype, - product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt, - product.sha256hash ? std::make_optional(product.sha256hash->to_string(HashFormat::Base16, false)) : std::nullopt, - product.path, - product.name, - product.defaultPath}).no_rows(); - } - - txn.exec("delete from BuildMetrics where build = $1", pqxx::params{build->id}).no_rows(); - - for (auto & metric : res.metrics) { - txn.exec("insert into BuildMetrics (build, name, unit, value, project, jobset, job, timestamp) values ($1, $2, $3, $4, $5, $6, $7, $8)", - pqxx::params{build->id, - metric.second.name, - metric.second.unit != "" ? std::make_optional(metric.second.unit) : std::nullopt, - metric.second.value, - build->projectName, - build->jobsetName, - build->jobName, - build->timestamp}).no_rows(); - } - - nrBuildsDone++; -} - - -bool State::checkCachedFailure(Step::ptr step, Connection & conn) -{ - pqxx::work txn(conn); - for (auto & i : step->drv->outputsAndOptPaths(*localStore)) - if (i.second.second) - if (!txn.exec("select 1 from FailedPaths where path = $1", pqxx::params{localStore->printStorePath(*i.second.second)}).empty()) - return true; - return false; -} - - -void State::notifyBuildStarted(pqxx::work & txn, BuildID buildId) -{ - txn.exec(fmt("notify build_started, '%s'", buildId)); -} - - -void State::notifyBuildFinished(pqxx::work & txn, BuildID buildId, - const std::vector & dependentIds) -{ - auto payload = fmt("%d", buildId); - for (auto & d : dependentIds) - payload += fmt("\t%d", d); - // FIXME: apparently parameterized() doesn't support NOTIFY. - txn.exec(fmt("notify build_finished, '%s'", payload)); -} - - -std::shared_ptr State::acquireGlobalLock() -{ - Path lockPath = hydraData + "/queue-runner/lock"; - - createDirs(dirOf(lockPath)); - - auto lock = std::make_shared(); - if (!lock->lockPaths(PathSet({lockPath}), "", false)) return 0; - - return lock; -} - - -void State::dumpStatus(Connection & conn) -{ - time_t now = time(0); - json statusJson = { - {"status", "up"}, - {"time", time(0)}, - {"uptime", now - startedAt}, - {"pid", getpid()}, - - {"nrQueuedBuilds", builds.lock()->size()}, - {"nrActiveSteps", activeSteps_.lock()->size()}, - {"nrStepsBuilding", nrStepsBuilding.load()}, - {"nrStepsCopyingTo", nrStepsCopyingTo.load()}, - {"nrStepsWaitingForDownloadSlot", nrStepsWaitingForDownloadSlot.load()}, - {"nrStepsCopyingFrom", nrStepsCopyingFrom.load()}, - {"nrStepsWaiting", nrStepsWaiting.load()}, - {"nrUnsupportedSteps", nrUnsupportedSteps.load()}, - {"bytesSent", bytesSent.load()}, - {"bytesReceived", bytesReceived.load()}, - {"nrBuildsRead", nrBuildsRead.load()}, - {"buildReadTimeMs", buildReadTimeMs.load()}, - {"buildReadTimeAvgMs", nrBuildsRead == 0 ? 0.0 : (float) buildReadTimeMs / nrBuildsRead}, - {"nrBuildsDone", nrBuildsDone.load()}, - {"nrStepsStarted", nrStepsStarted.load()}, - {"nrStepsDone", nrStepsDone.load()}, - {"nrRetries", nrRetries.load()}, - {"maxNrRetries", maxNrRetries.load()}, - {"nrQueueWakeups", nrQueueWakeups.load()}, - {"nrDispatcherWakeups", nrDispatcherWakeups.load()}, - {"dispatchTimeMs", dispatchTimeMs.load()}, - {"dispatchTimeAvgMs", nrDispatcherWakeups == 0 ? 0.0 : (float) dispatchTimeMs / nrDispatcherWakeups}, - {"nrDbConnections", dbPool.count()}, - {"nrActiveDbUpdates", nrActiveDbUpdates.load()}, - }; - { - { - auto steps_(steps.lock()); - for (auto i = steps_->begin(); i != steps_->end(); ) - if (i->second.lock()) ++i; else i = steps_->erase(i); - statusJson["nrUnfinishedSteps"] = steps_->size(); - } - { - auto runnable_(runnable.lock()); - for (auto i = runnable_->begin(); i != runnable_->end(); ) - if (i->lock()) ++i; else i = runnable_->erase(i); - statusJson["nrRunnableSteps"] = runnable_->size(); - } - if (nrStepsDone) { - statusJson["totalStepTime"] = totalStepTime.load(); - statusJson["totalStepBuildTime"] = totalStepBuildTime.load(); - statusJson["avgStepTime"] = (float) totalStepTime / nrStepsDone; - statusJson["avgStepBuildTime"] = (float) totalStepBuildTime / nrStepsDone; - } - - { - auto machines_json = json::object(); - auto machines_(machines.lock()); - for (auto & i : *machines_) { - auto & m(i.second); - auto & s(m->state); - auto info(m->state->connectInfo.lock()); - - json machine = { - {"enabled", m->enabled}, - {"systemTypes", m->systemTypes}, - {"supportedFeatures", m->supportedFeatures}, - {"mandatoryFeatures", m->mandatoryFeatures}, - {"nrStepsDone", s->nrStepsDone.load()}, - {"currentJobs", s->currentJobs.load()}, - {"disabledUntil", std::chrono::system_clock::to_time_t(info->disabledUntil)}, - {"lastFailure", std::chrono::system_clock::to_time_t(info->lastFailure)}, - {"consecutiveFailures", info->consecutiveFailures}, - }; - - if (s->currentJobs == 0) - machine["idleSince"] = s->idleSince.load(); - if (m->state->nrStepsDone) { - machine["totalStepTime"] = s->totalStepTime.load(); - machine["totalStepBuildTime"] = s->totalStepBuildTime.load(); - machine["avgStepTime"] = (float) s->totalStepTime / s->nrStepsDone; - machine["avgStepBuildTime"] = (float) s->totalStepBuildTime / s->nrStepsDone; - } - machines_json[m->storeUri.render()] = machine; - } - statusJson["machines"] = machines_json; - } - - { - auto jobsets_json = json::object(); - auto jobsets_(jobsets.lock()); - for (auto & jobset : *jobsets_) { - jobsets_json[jobset.first.first + ":" + jobset.first.second] = { - {"shareUsed", jobset.second->shareUsed()}, - {"seconds", jobset.second->getSeconds()}, - }; - } - statusJson["jobsets"] = jobsets_json; - } - - { - auto machineTypesJson = json::object(); - auto machineTypes_(machineTypes.lock()); - for (auto & i : *machineTypes_) { - auto machineTypeJson = machineTypesJson[i.first] = { - {"runnable", i.second.runnable}, - {"running", i.second.running}, - }; - if (i.second.runnable > 0) - machineTypeJson["waitTime"] = i.second.waitTime.count() + - i.second.runnable * (time(0) - lastDispatcherCheck); - if (i.second.running == 0) - machineTypeJson["lastActive"] = std::chrono::system_clock::to_time_t(i.second.lastActive); - } - statusJson["machineTypes"] = machineTypesJson; - } - - auto store = getDestStore(); - - auto & stats = store->getStats(); - statusJson["store"] = { - {"narInfoRead", stats.narInfoRead.load()}, - {"narInfoReadAverted", stats.narInfoReadAverted.load()}, - {"narInfoMissing", stats.narInfoMissing.load()}, - {"narInfoWrite", stats.narInfoWrite.load()}, - {"narInfoCacheSize", stats.pathInfoCacheSize.load()}, - {"narRead", stats.narRead.load()}, - {"narReadBytes", stats.narReadBytes.load()}, - {"narReadCompressedBytes", stats.narReadCompressedBytes.load()}, - {"narWrite", stats.narWrite.load()}, - {"narWriteAverted", stats.narWriteAverted.load()}, - {"narWriteBytes", stats.narWriteBytes.load()}, - {"narWriteCompressedBytes", stats.narWriteCompressedBytes.load()}, - {"narWriteCompressionTimeMs", stats.narWriteCompressionTimeMs.load()}, - {"narCompressionSavings", - stats.narWriteBytes - ? 1.0 - (double) stats.narWriteCompressedBytes / stats.narWriteBytes - : 0.0}, - {"narCompressionSpeed", // MiB/s - stats.narWriteCompressionTimeMs - ? (double) stats.narWriteBytes / stats.narWriteCompressionTimeMs * 1000.0 / (1024.0 * 1024.0) - : 0.0}, - }; - -#if NIX_WITH_S3_SUPPORT - auto s3Store = dynamic_cast(&*store); - if (s3Store) { - auto & s3Stats = s3Store->getS3Stats(); - auto jsonS3 = statusJson["s3"] = { - {"put", s3Stats.put.load()}, - {"putBytes", s3Stats.putBytes.load()}, - {"putTimeMs", s3Stats.putTimeMs.load()}, - {"putSpeed", - s3Stats.putTimeMs - ? (double) s3Stats.putBytes / s3Stats.putTimeMs * 1000.0 / (1024.0 * 1024.0) - : 0.0}, - {"get", s3Stats.get.load()}, - {"getBytes", s3Stats.getBytes.load()}, - {"getTimeMs", s3Stats.getTimeMs.load()}, - {"getSpeed", - s3Stats.getTimeMs - ? (double) s3Stats.getBytes / s3Stats.getTimeMs * 1000.0 / (1024.0 * 1024.0) - : 0.0}, - {"head", s3Stats.head.load()}, - {"costDollarApprox", - (s3Stats.get + s3Stats.head) / 10000.0 * 0.004 - + s3Stats.put / 1000.0 * 0.005 + - + s3Stats.getBytes / (1024.0 * 1024.0 * 1024.0) * 0.09}, - }; - } -#endif - } - - { - auto mc = startDbUpdate(); - pqxx::work txn(conn); - // FIXME: use PostgreSQL 9.5 upsert. - txn.exec("delete from SystemStatus where what = 'queue-runner'").no_rows(); - txn.exec("insert into SystemStatus values ('queue-runner', $1)", pqxx::params{statusJson.dump()}).no_rows(); - txn.exec("notify status_dumped"); - txn.commit(); - } -} - - -void State::showStatus() -{ - auto conn(dbPool.get()); - receiver statusDumped(*conn, "status_dumped"); - - std::string status; - bool barf = false; - - /* Get the last JSON status dump from the database. */ - { - pqxx::work txn(*conn); - auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'"); - if (res.size()) status = res[0][0].as(); - } - - if (status != "") { - - /* If the status is not empty, then the queue runner is - running. Ask it to update the status dump. */ - { - pqxx::work txn(*conn); - txn.exec("notify dump_status"); - txn.commit(); - } - - /* Wait until it has done so. */ - barf = conn->await_notification(5, 0) == 0; - - /* Get the new status. */ - { - pqxx::work txn(*conn); - auto res = txn.exec("select status from SystemStatus where what = 'queue-runner'"); - if (res.size()) status = res[0][0].as(); - } - - } - - if (status == "") status = R"({"status":"down"})"; - - std::cout << status << "\n"; - - if (barf) - throw Error("queue runner did not respond; status information may be wrong"); -} - - -void State::unlock() -{ - auto lock = acquireGlobalLock(); - if (!lock) - throw Error("hydra-queue-runner is currently running"); - - auto conn(dbPool.get()); - - clearBusy(*conn, 0); - - { - pqxx::work txn(*conn); - txn.exec("delete from SystemStatus where what = 'queue-runner'").no_rows(); - txn.commit(); - } -} - - -void State::run(BuildID buildOne) -{ - /* Can't be bothered to shut down cleanly. Goodbye! */ - auto callback = createInterruptCallback([&]() { std::_Exit(0); }); - - startedAt = time(0); - this->buildOne = buildOne; - - auto lock = acquireGlobalLock(); - if (!lock) - throw Error("hydra-queue-runner is already running"); - - std::cout << "Starting the Prometheus exporter on " << metricsAddr << std::endl; - - /* Set up simple exporter, to show that we're still alive. */ - prometheus::Exposer promExposer{metricsAddr}; - auto exposerPort = promExposer.GetListeningPorts().front(); - - promExposer.RegisterCollectable(prom.registry); - - std::cout << "Started the Prometheus exporter, listening on " - << metricsAddr << "/metrics (port " << exposerPort << ")" - << std::endl; - - Store::Config::Params localParams; - localParams["max-connections"] = "16"; - localParams["max-connection-age"] = "600"; - localStore = openStore(getEnv("NIX_REMOTE").value_or(""), localParams); - - auto storeUri = config->getStrOption("store_uri"); - _destStore = storeUri == "" ? localStore : openStore(storeUri); - - useSubstitutes = config->getBoolOption("use-substitutes", false); - - // FIXME: hacky mechanism for configuring determinism checks. - for (auto & s : tokenizeString(config->getStrOption("xxx-jobset-repeats"))) { - auto s2 = tokenizeString>(s, ":"); - if (s2.size() != 3) throw Error("bad value in xxx-jobset-repeats"); - jobsetRepeats.emplace(std::make_pair(s2[0], s2[1]), std::stoi(s2[2])); - } - - { - auto conn(dbPool.get()); - clearBusy(*conn, 0); - dumpStatus(*conn); - } - - machinesReadyLock.lock(); - std::thread(&State::monitorMachinesFile, this).detach(); - - std::thread(&State::queueMonitor, this).detach(); - - std::thread(&State::dispatcher, this).detach(); - - /* Periodically clean up orphaned busy steps in the database. */ - std::thread([&]() { - while (true) { - sleep(180); - - std::set> steps; - { - auto orphanedSteps_(orphanedSteps.lock()); - if (orphanedSteps_->empty()) continue; - steps = *orphanedSteps_; - orphanedSteps_->clear(); - } - - try { - auto conn(dbPool.get()); - pqxx::work txn(*conn); - for (auto & step : steps) { - printMsg(lvlError, "cleaning orphaned step %d of build %d", step.second, step.first); - txn.exec("update BuildSteps set busy = 0, status = $1 where build = $2 and stepnr = $3 and busy != 0", - pqxx::params{(int) bsAborted, - step.first, - step.second}).no_rows(); - } - txn.commit(); - } catch (std::exception & e) { - printMsg(lvlError, "cleanup thread: %s", e.what()); - auto orphanedSteps_(orphanedSteps.lock()); - orphanedSteps_->insert(steps.begin(), steps.end()); - } - } - }).detach(); - - /* Make sure that old daemon connections are closed even when - we're not doing much. */ - std::thread([&]() { - while (true) { - sleep(10); - try { - if (auto remoteStore = getDestStore().dynamic_pointer_cast()) - remoteStore->flushBadConnections(); - } catch (std::exception & e) { - printMsg(lvlError, "connection flush thread: %s", e.what()); - } - } - }).detach(); - - /* Monitor the database for status dump requests (e.g. from - ‘hydra-queue-runner --status’). */ - while (true) { - try { - auto conn(dbPool.get()); - try { - receiver dumpStatus_(*conn, "dump_status"); - while (true) { - conn->await_notification(); - dumpStatus(*conn); - } - } catch (pqxx::broken_connection & connEx) { - printMsg(lvlError, "main thread: %s", connEx.what()); - printMsg(lvlError, "main thread: Reconnecting in 10s"); - conn.markBad(); - sleep(10); - } - } catch (std::exception & e) { - printMsg(lvlError, "main thread: %s", e.what()); - sleep(10); // probably a DB problem, so don't retry right away - } - } -} - - -int main(int argc, char * * argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); - - signal(SIGINT, SIG_DFL); - signal(SIGTERM, SIG_DFL); - signal(SIGHUP, SIG_DFL); - - // FIXME: do this in the child environment in openConnection(). - unsetenv("IN_SYSTEMD"); - - bool unlock = false; - bool status = false; - BuildID buildOne = 0; - std::optional metricsAddrOpt = std::nullopt; - - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--unlock") - unlock = true; - else if (*arg == "--status") - status = true; - else if (*arg == "--build-one") { - if (auto b = string2Int(getArg(*arg, arg, end))) - buildOne = *b; - else - throw Error("‘--build-one’ requires a build ID"); - } else if (*arg == "--prometheus-address") { - metricsAddrOpt = getArg(*arg, arg, end); - } else - return false; - return true; - }); - - settings.verboseBuild = true; - - State state{metricsAddrOpt}; - if (status) - state.showStatus(); - else if (unlock) - state.unlock(); - else - state.run(buildOne); - }); -} diff --git a/src/hydra-queue-runner/meson.build b/src/hydra-queue-runner/meson.build deleted file mode 100644 index 27dad2c06..000000000 --- a/src/hydra-queue-runner/meson.build +++ /dev/null @@ -1,24 +0,0 @@ -srcs = files( - 'builder.cc', - 'build-remote.cc', - 'build-result.cc', - 'dispatcher.cc', - 'hydra-queue-runner.cc', - 'nar-extractor.cc', - 'queue-monitor.cc', -) - -hydra_queue_runner = executable('hydra-queue-runner', - 'hydra-queue-runner.cc', - srcs, - dependencies: [ - libhydra_dep, - nix_util_dep, - nix_store_dep, - nix_main_dep, - pqxx_dep, - prom_cpp_core_dep, - prom_cpp_pull_dep, - ], - install: true, -) diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc deleted file mode 100644 index 3bf06ef31..000000000 --- a/src/hydra-queue-runner/nar-extractor.cc +++ /dev/null @@ -1,103 +0,0 @@ -#include "nar-extractor.hh" - -#include - -#include - -using namespace nix; - - -struct NarMemberConstructor : CreateRegularFileSink -{ - NarMemberData & curMember; - - HashSink hashSink = HashSink { HashAlgorithm::SHA256 }; - - std::optional expectedSize; - - NarMemberConstructor(NarMemberData & curMember) - : curMember(curMember) - { } - - void isExecutable() override - { - } - - void preallocateContents(uint64_t size) override - { - expectedSize = size; - } - - void operator () (std::string_view data) override - { - assert(expectedSize); - *curMember.fileSize += data.size(); - hashSink(data); - if (curMember.contents) { - curMember.contents->append(data); - } - assert(curMember.fileSize <= expectedSize); - if (curMember.fileSize == expectedSize) { - auto [hash, len] = hashSink.finish(); - assert(curMember.fileSize == len); - curMember.sha256 = hash; - } - } -}; - -struct Extractor : FileSystemObjectSink -{ - std::unordered_set filesToKeep { - "/nix-support/hydra-build-products", - "/nix-support/hydra-release-name", - "/nix-support/hydra-metrics", - }; - - NarMemberDatas & members; - std::filesystem::path prefix; - - Path toKey(const CanonPath & path) - { - std::filesystem::path p = prefix; - // Conditional to avoid trailing slash - if (!path.isRoot()) p /= path.rel(); - return p; - } - - Extractor(NarMemberDatas & members, const Path & prefix) - : members(members), prefix(prefix) - { } - - void createDirectory(const CanonPath & path) override - { - members.insert_or_assign(toKey(path), NarMemberData { .type = SourceAccessor::Type::tDirectory }); - } - - void createRegularFile(const CanonPath & path, std::function func) override - { - NarMemberConstructor nmc { - members.insert_or_assign(toKey(path), NarMemberData { - .type = SourceAccessor::Type::tRegular, - .fileSize = 0, - .contents = filesToKeep.count(path.abs()) ? std::optional("") : std::nullopt, - }).first->second, - }; - func(nmc); - } - - void createSymlink(const CanonPath & path, const std::string & target) override - { - members.insert_or_assign(toKey(path), NarMemberData { .type = SourceAccessor::Type::tSymlink }); - } -}; - - -void extractNarData( - Source & source, - const Path & prefix, - NarMemberDatas & members) -{ - Extractor extractor(members, prefix); - parseDump(extractor, source); - // Note: this point may not be reached if we're in a coroutine. -} diff --git a/src/hydra-queue-runner/nar-extractor.hh b/src/hydra-queue-runner/nar-extractor.hh deleted file mode 100644 index 0060efe29..000000000 --- a/src/hydra-queue-runner/nar-extractor.hh +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -struct NarMemberData -{ - nix::SourceAccessor::Type type; - std::optional fileSize; - std::optional contents; - std::optional sha256; -}; - -typedef std::map NarMemberDatas; - -/* Read a NAR from a source and get to some info about every file - inside the NAR. */ -void extractNarData( - nix::Source & source, - const nix::Path & prefix, - NarMemberDatas & members); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc deleted file mode 100644 index 7630230c4..000000000 --- a/src/hydra-queue-runner/queue-monitor.cc +++ /dev/null @@ -1,758 +0,0 @@ -#include "state.hh" -#include "hydra-build-result.hh" -#include -#include -#include - -#include -#include - -using namespace nix; - - -void State::queueMonitor() -{ - while (true) { - auto conn(dbPool.get()); - try { - queueMonitorLoop(*conn); - } catch (pqxx::broken_connection & e) { - printMsg(lvlError, "queue monitor: %s", e.what()); - printMsg(lvlError, "queue monitor: Reconnecting in 10s"); - conn.markBad(); - sleep(10); - } catch (std::exception & e) { - printError("queue monitor: %s", e.what()); - sleep(10); // probably a DB problem, so don't retry right away - } - } -} - - -void State::queueMonitorLoop(Connection & conn) -{ - receiver buildsAdded(conn, "builds_added"); - receiver buildsRestarted(conn, "builds_restarted"); - receiver buildsCancelled(conn, "builds_cancelled"); - receiver buildsDeleted(conn, "builds_deleted"); - receiver buildsBumped(conn, "builds_bumped"); - receiver jobsetSharesChanged(conn, "jobset_shares_changed"); - - auto destStore = getDestStore(); - - bool quit = false; - while (!quit) { - auto t_before_work = std::chrono::steady_clock::now(); - - localStore->clearPathInfoCache(); - - bool done = getQueuedBuilds(conn, destStore); - - if (buildOne && buildOneDone) quit = true; - - auto t_after_work = std::chrono::steady_clock::now(); - - prom.queue_monitor_time_spent_running.Increment( - std::chrono::duration_cast(t_after_work - t_before_work).count()); - - /* Sleep until we get notification from the database about an - event. */ - if (done && !quit) { - conn.await_notification(); - nrQueueWakeups++; - } else - conn.get_notifs(); - - if (auto lowestId = buildsAdded.get()) { - printMsg(lvlTalkative, "got notification: new builds added to the queue"); - } - if (buildsRestarted.get()) { - printMsg(lvlTalkative, "got notification: builds restarted"); - } - if (buildsCancelled.get() || buildsDeleted.get() || buildsBumped.get()) { - printMsg(lvlTalkative, "got notification: builds cancelled or bumped"); - processQueueChange(conn); - } - if (jobsetSharesChanged.get()) { - printMsg(lvlTalkative, "got notification: jobset shares changed"); - processJobsetSharesChange(conn); - } - - auto t_after_sleep = std::chrono::steady_clock::now(); - prom.queue_monitor_time_spent_waiting.Increment( - std::chrono::duration_cast(t_after_sleep - t_after_work).count()); - } - - exit(0); -} - - -struct PreviousFailure : public std::exception { - Step::ptr step; - PreviousFailure(Step::ptr step) : step(step) { } -}; - - -bool State::getQueuedBuilds(Connection & conn, - ref destStore) -{ - prom.queue_checks_started.Increment(); - - printInfo("checking the queue for builds..."); - - /* Grab the queued builds from the database, but don't process - them yet (since we don't want a long-running transaction). */ - std::vector newIDs; - std::unordered_map newBuildsByID; - std::multimap newBuildsByPath; - - { - pqxx::work txn(conn); - - auto res = txn.exec("select builds.id, builds.jobset_id, jobsets.project as project, " - "jobsets.name as jobset, job, drvPath, maxsilent, timeout, timestamp, " - "globalPriority, priority from Builds " - "inner join jobsets on builds.jobset_id = jobsets.id " - "where finished = 0 order by globalPriority desc, random()"); - - for (auto const & row : res) { - auto builds_(builds.lock()); - BuildID id = row["id"].as(); - if (buildOne && id != buildOne) continue; - if (builds_->count(id)) continue; - - auto build = std::make_shared( - localStore->parseStorePath(row["drvPath"].as())); - build->id = id; - build->jobsetId = row["jobset_id"].as(); - build->projectName = row["project"].as(); - build->jobsetName = row["jobset"].as(); - build->jobName = row["job"].as(); - build->maxSilentTime = row["maxsilent"].as(); - build->buildTimeout = row["timeout"].as(); - build->timestamp = row["timestamp"].as(); - build->globalPriority = row["globalPriority"].as(); - build->localPriority = row["priority"].as(); - build->jobset = createJobset(txn, build->projectName, build->jobsetName, build->jobsetId); - - newIDs.push_back(id); - newBuildsByID[id] = build; - newBuildsByPath.emplace(std::make_pair(build->drvPath, id)); - } - } - - std::set newRunnable; - unsigned int nrAdded; - std::function createBuild; - std::set finishedDrvs; - - createBuild = [&](Build::ptr build) { - prom.queue_build_loads.Increment(); - printMsg(lvlTalkative, "loading build %1% (%2%)", build->id, build->fullJobName()); - nrAdded++; - newBuildsByID.erase(build->id); - - if (!localStore->isValidPath(build->drvPath)) { - /* Derivation has been GC'ed prematurely. */ - printError("aborting GC'ed build %1%", build->id); - if (!build->finishedInDB) { - auto mc = startDbUpdate(); - pqxx::work txn(conn); - txn.exec("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3 where id = $1 and finished = 0", - pqxx::params{build->id, - (int) bsAborted, - time(0)}).no_rows(); - txn.commit(); - build->finishedInDB = true; - nrBuildsDone++; - } - return; - } - - std::set newSteps; - Step::ptr step; - - /* Create steps for this derivation and its dependencies. */ - try { - step = createStep(destStore, conn, build, build->drvPath, - build, 0, finishedDrvs, newSteps, newRunnable); - } catch (PreviousFailure & ex) { - - /* Some step previously failed, so mark the build as - failed right away. */ - if (!buildOneDone && build->id == buildOne) buildOneDone = true; - printMsg(lvlError, "marking build %d as cached failure due to ‘%s’", - build->id, localStore->printStorePath(ex.step->drvPath)); - if (!build->finishedInDB) { - auto mc = startDbUpdate(); - pqxx::work txn(conn); - - /* Find the previous build step record, first by - derivation path, then by output path. */ - BuildID propagatedFrom = 0; - - auto res = txn.exec("select max(build) from BuildSteps where drvPath = $1 and startTime != 0 and stopTime != 0 and status = 1", - pqxx::params{localStore->printStorePath(ex.step->drvPath)}).one_row(); - if (!res[0].is_null()) propagatedFrom = res[0].as(); - - if (!propagatedFrom) { - for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) { - constexpr std::string_view common = "select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where startTime != 0 and stopTime != 0 and status = 1"; - auto res = optOutputPath - ? txn.exec( - std::string { common } + " and path = $1", - pqxx::params{localStore->printStorePath(*optOutputPath)}) - : txn.exec( - std::string { common } + " and drvPath = $1 and name = $2", - pqxx::params{localStore->printStorePath(ex.step->drvPath), outputName}); - if (!res[0][0].is_null()) { - propagatedFrom = res[0][0].as(); - break; - } - } - } - - createBuildStep(txn, 0, build->id, ex.step, "", bsCachedFailure, "", propagatedFrom); - txn.exec("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $3, isCachedBuild = 1, notificationPendingSince = $3 " - "where id = $1 and finished = 0", - pqxx::params{build->id, - (int) (ex.step->drvPath == build->drvPath ? bsFailed : bsDepFailed), - time(0)}).no_rows(); - notifyBuildFinished(txn, build->id, {}); - txn.commit(); - build->finishedInDB = true; - nrBuildsDone++; - } - - return; - } - - /* Some of the new steps may be the top level of builds that - we haven't processed yet. So do them now. This ensures that - if build A depends on build B with top-level step X, then X - will be "accounted" to B in doBuildStep(). */ - for (auto & r : newSteps) { - auto i = newBuildsByPath.find(r->drvPath); - if (i == newBuildsByPath.end()) continue; - auto j = newBuildsByID.find(i->second); - if (j == newBuildsByID.end()) continue; - createBuild(j->second); - } - - /* If we didn't get a step, it means the step's outputs are - all valid. So we mark this as a finished, cached build. */ - if (!step) { - BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath); - - for (auto & i : destStore->queryDerivationOutputMap(build->drvPath, &*localStore)) - addRoot(i.second); - - { - auto mc = startDbUpdate(); - pqxx::work txn(conn); - time_t now = time(0); - if (!buildOneDone && build->id == buildOne) buildOneDone = true; - printMsg(lvlInfo, "marking build %1% as succeeded (cached)", build->id); - markSucceededBuild(txn, build, res, true, now, now); - notifyBuildFinished(txn, build->id, {}); - txn.commit(); - } - - build->finishedInDB = true; - - return; - } - - /* Note: if we exit this scope prior to this, the build and - all newly created steps are destroyed. */ - - { - auto builds_(builds.lock()); - if (!build->finishedInDB) // FIXME: can this happen? - (*builds_)[build->id] = build; - build->toplevel = step; - } - - build->propagatePriorities(); - - printMsg(lvlChatty, "added build %1% (top-level step %2%, %3% new steps)", - build->id, localStore->printStorePath(step->drvPath), newSteps.size()); - }; - - /* Now instantiate build steps for each new build. The builder - threads can start building the runnable build steps right away, - even while we're still processing other new builds. */ - system_time start = std::chrono::system_clock::now(); - - for (auto id : newIDs) { - auto i = newBuildsByID.find(id); - if (i == newBuildsByID.end()) continue; - auto build = i->second; - - auto now1 = std::chrono::steady_clock::now(); - - newRunnable.clear(); - nrAdded = 0; - try { - createBuild(build); - } catch (Error & e) { - e.addTrace({}, HintFmt("while loading build %d: ", build->id)); - throw; - } - - auto now2 = std::chrono::steady_clock::now(); - - buildReadTimeMs += std::chrono::duration_cast(now2 - now1).count(); - - /* Add the new runnable build steps to ‘runnable’ and wake up - the builder threads. */ - printMsg(lvlChatty, "got %1% new runnable steps from %2% new builds", newRunnable.size(), nrAdded); - for (auto & r : newRunnable) - makeRunnable(r); - - if (buildOne && newRunnable.size() == 0) buildOneDone = true; - - nrBuildsRead += nrAdded; - - /* Stop after a certain time to allow priority bumps to be - processed. */ - if (std::chrono::system_clock::now() > start + std::chrono::seconds(60)) { - prom.queue_checks_early_exits.Increment(); - break; - } - } - - prom.queue_checks_finished.Increment(); - return newBuildsByID.empty(); -} - - -void Build::propagatePriorities() -{ - /* Update the highest global priority and lowest build ID fields - of each dependency. This is used by the dispatcher to start - steps in order of descending global priority and ascending - build ID. */ - visitDependencies([&](const Step::ptr & step) { - auto step_(step->state.lock()); - step_->highestGlobalPriority = std::max(step_->highestGlobalPriority, globalPriority); - step_->highestLocalPriority = std::max(step_->highestLocalPriority, localPriority); - step_->lowestBuildID = std::min(step_->lowestBuildID, id); - step_->jobsets.insert(jobset); - }, toplevel); -} - - -void State::processQueueChange(Connection & conn) -{ - /* Get the current set of queued builds. */ - std::map currentIds; - { - pqxx::work txn(conn); - auto res = txn.exec("select id, globalPriority from Builds where finished = 0"); - for (auto const & row : res) - currentIds[row["id"].as()] = row["globalPriority"].as(); - } - - { - auto builds_(builds.lock()); - - for (auto i = builds_->begin(); i != builds_->end(); ) { - auto b = currentIds.find(i->first); - if (b == currentIds.end()) { - printInfo("discarding cancelled build %1%", i->first); - i = builds_->erase(i); - // FIXME: ideally we would interrupt active build steps here. - continue; - } - if (i->second->globalPriority < b->second) { - printInfo("priority of build %1% increased", i->first); - i->second->globalPriority = b->second; - i->second->propagatePriorities(); - } - ++i; - } - } - - { - auto activeSteps(activeSteps_.lock()); - for (auto & activeStep : *activeSteps) { - std::set dependents; - std::set steps; - getDependents(activeStep->step, dependents, steps); - if (!dependents.empty()) continue; - - { - auto activeStepState(activeStep->state_.lock()); - if (activeStepState->cancelled) continue; - activeStepState->cancelled = true; - if (activeStepState->pid != -1) { - printInfo("killing builder process %d of build step ‘%s’", - activeStepState->pid, - localStore->printStorePath(activeStep->step->drvPath)); - if (kill(activeStepState->pid, SIGINT) == -1) - printError("error killing build step ‘%s’: %s", - localStore->printStorePath(activeStep->step->drvPath), - strerror(errno)); - } - } - } - } -} - - -std::map> State::getMissingRemotePaths( - ref destStore, - const std::map> & paths) -{ - Sync>> missing_; - ThreadPool tp; - - for (auto & [output, maybeOutputPath] : paths) { - if (!maybeOutputPath) { - auto missing(missing_.lock()); - missing->insert({output, maybeOutputPath}); - } else { - tp.enqueue([&] { - if (!destStore->isValidPath(*maybeOutputPath)) { - auto missing(missing_.lock()); - missing->insert({output, maybeOutputPath}); - } - }); - } - } - - tp.process(); - - auto missing(missing_.lock()); - return *missing; -} - - -Step::ptr State::createStep(ref destStore, - Connection & conn, Build::ptr build, const StorePath & drvPath, - Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, - std::set & newSteps, std::set & newRunnable) -{ - if (finishedDrvs.find(drvPath) != finishedDrvs.end()) return 0; - - /* Check if the requested step already exists. If not, create a - new step. In any case, make the step reachable from - referringBuild or referringStep. This is done atomically (with - ‘steps’ locked), to ensure that this step can never become - reachable from a new build after doBuildStep has removed it - from ‘steps’. */ - Step::ptr step; - bool isNew = false; - { - auto steps_(steps.lock()); - - /* See if the step already exists in ‘steps’ and is not - stale. */ - auto prev = steps_->find(drvPath); - if (prev != steps_->end()) { - step = prev->second.lock(); - /* Since ‘step’ is a strong pointer, the referred Step - object won't be deleted after this. */ - if (!step) steps_->erase(drvPath); // remove stale entry - } - - /* If it doesn't exist, create it. */ - if (!step) { - step = std::make_shared(drvPath); - isNew = true; - } - - auto step_(step->state.lock()); - - assert(step_->created != isNew); - - if (referringBuild) - step_->builds.push_back(referringBuild); - - if (referringStep) - step_->rdeps.push_back(referringStep); - - steps_->insert_or_assign(drvPath, step); - } - - if (!isNew) return step; - - prom.queue_steps_created.Increment(); - - printMsg(lvlDebug, "considering derivation ‘%1%’", localStore->printStorePath(drvPath)); - - /* Initialize the step. Note that the step may be visible in - ‘steps’ before this point, but that doesn't matter because - it's not runnable yet, and other threads won't make it - runnable while step->created == false. */ - step->drv = std::make_unique(localStore->readDerivation(drvPath)); - { - try { - step->drvOptions = std::make_unique( - DerivationOptions::fromStructuredAttrs( - step->drv->env, - step->drv->structuredAttrs ? &*step->drv->structuredAttrs : nullptr)); - } catch (Error & e) { - e.addTrace({}, "while parsing derivation '%s'", localStore->printStorePath(drvPath)); - throw; - } - } - - step->preferLocalBuild = step->drvOptions->willBuildLocally(*localStore, *step->drv); - step->isDeterministic = getOr(step->drv->env, "isDetermistic", "0") == "1"; - - step->systemType = step->drv->platform; - { - StringSet features = step->requiredSystemFeatures = step->drvOptions->getRequiredSystemFeatures(*step->drv); - if (step->preferLocalBuild) - features.insert("local"); - if (!features.empty()) { - step->systemType += ":"; - step->systemType += concatStringsSep(",", features); - } - } - - /* If this derivation failed previously, give up. */ - if (checkCachedFailure(step, conn)) - throw PreviousFailure{step}; - - /* Are all outputs valid? */ - auto outputHashes = staticOutputHashes(*localStore, *(step->drv)); - std::map> paths; - for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) { - auto outputHash = outputHashes.at(outputName); - paths.insert({{outputHash, outputName}, maybeOutputPath}); - } - - auto missing = getMissingRemotePaths(destStore, paths); - bool valid = missing.empty(); - - /* Try to copy the missing paths from the local store or from - substitutes. */ - if (!missing.empty()) { - - size_t avail = 0; - for (auto & [i, pathOpt] : missing) { - // If we don't know the output path from the destination - // store, see if the local store can tell us. - if (/* localStore != destStore && */ !pathOpt && experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - if (auto maybeRealisation = localStore->queryRealisation(i)) - pathOpt = maybeRealisation->outPath; - - if (!pathOpt) { - // No hope of getting the store object if we don't know - // the path. - continue; - } - auto & path = *pathOpt; - - if (/* localStore != destStore && */ localStore->isValidPath(path)) - avail++; - else if (useSubstitutes) { - SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({{path, {}}}, infos); - if (infos.size() == 1) - avail++; - } - } - - if (missing.size() == avail) { - valid = true; - for (auto & [i, pathOpt] : missing) { - // If we found everything, then we should know the path - // to every missing store object now. - assert(pathOpt); - auto & path = *pathOpt; - - try { - time_t startTime = time(0); - - if (localStore->isValidPath(path)) - printInfo("copying output ‘%1%’ of ‘%2%’ from local store", - localStore->printStorePath(path), - localStore->printStorePath(drvPath)); - else { - printInfo("substituting output ‘%1%’ of ‘%2%’", - localStore->printStorePath(path), - localStore->printStorePath(drvPath)); - localStore->ensurePath(path); - // FIXME: should copy directly from substituter to destStore. - } - - copyClosure(*localStore, *destStore, - StorePathSet { path }, - NoRepair, CheckSigs, NoSubstitute); - - time_t stopTime = time(0); - - { - auto mc = startDbUpdate(); - pqxx::work txn(conn); - createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", path); - txn.commit(); - } - - } catch (Error & e) { - printError("while copying/substituting output ‘%s’ of ‘%s’: %s", - localStore->printStorePath(path), - localStore->printStorePath(drvPath), - e.what()); - valid = false; - break; - } - } - } - } - - // FIXME: check whether all outputs are in the binary cache. - if (valid) { - finishedDrvs.insert(drvPath); - return 0; - } - - /* No, we need to build. */ - printMsg(lvlDebug, "creating build step ‘%1%’", localStore->printStorePath(drvPath)); - - /* Create steps for the dependencies. */ - for (auto & i : step->drv->inputDrvs.map) { - auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable); - if (dep) { - auto step_(step->state.lock()); - step_->deps.insert(dep); - } - } - - /* If the step has no (remaining) dependencies, make it - runnable. */ - { - auto step_(step->state.lock()); - assert(!step_->created); - step_->created = true; - if (step_->deps.empty()) - newRunnable.insert(step); - } - - newSteps.insert(step); - - return step; -} - - -Jobset::ptr State::createJobset(pqxx::work & txn, - const std::string & projectName, const std::string & jobsetName, const JobsetID jobsetID) -{ - auto p = std::make_pair(projectName, jobsetName); - - { - auto jobsets_(jobsets.lock()); - auto i = jobsets_->find(p); - if (i != jobsets_->end()) return i->second; - } - - auto res = txn.exec("select schedulingShares from Jobsets where id = $1", - pqxx::params{jobsetID}).one_row(); - - auto shares = res["schedulingShares"].as(); - - auto jobset = std::make_shared(); - jobset->setShares(shares); - - /* Load the build steps from the last 24 hours. */ - auto res2 = txn.exec("select s.startTime, s.stopTime from BuildSteps s join Builds b on build = id " - "where s.startTime is not null and s.stopTime > $1 and jobset_id = $2", - pqxx::params{time(0) - Jobset::schedulingWindow * 10, - jobsetID}); - for (auto const & row : res2) { - time_t startTime = row["startTime"].as(); - time_t stopTime = row["stopTime"].as(); - jobset->addStep(startTime, stopTime - startTime); - } - - auto jobsets_(jobsets.lock()); - // Can't happen because only this thread adds to "jobsets". - assert(jobsets_->find(p) == jobsets_->end()); - (*jobsets_)[p] = jobset; - return jobset; -} - - -void State::processJobsetSharesChange(Connection & conn) -{ - /* Get the current set of jobsets. */ - pqxx::work txn(conn); - auto res = txn.exec("select project, name, schedulingShares from Jobsets"); - for (auto const & row : res) { - auto jobsets_(jobsets.lock()); - auto i = jobsets_->find(std::make_pair(row["project"].as(), row["name"].as())); - if (i == jobsets_->end()) continue; - i->second->setShares(row["schedulingShares"].as()); - } -} - - -BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref destStore, const nix::StorePath & drvPath) -{ - auto derivationOutputs = destStore->queryDerivationOutputMap(drvPath, &*localStore); - - { - pqxx::work txn(conn); - - for (auto & [name, output] : derivationOutputs) { - auto r = txn.exec("select id, buildStatus, releaseName, closureSize, size from Builds b " - "join BuildOutputs o on b.id = o.build " - "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1", - pqxx::params{localStore->printStorePath(output)}); - if (r.empty()) continue; - BuildID id = r[0][0].as(); - - printInfo("reusing build %d", id); - - BuildOutput res; - res.failed = r[0][1].as() == bsFailedWithOutput; - res.releaseName = r[0][2].is_null() ? "" : r[0][2].as(); - res.closureSize = r[0][3].is_null() ? 0 : r[0][3].as(); - res.size = r[0][4].is_null() ? 0 : r[0][4].as(); - - auto products = txn.exec("select type, subtype, fileSize, sha256hash, path, name, defaultPath from BuildProducts where build = $1 order by productnr", - pqxx::params{id}); - - for (auto row : products) { - BuildProduct product; - product.type = row[0].as(); - product.subtype = row[1].as(); - if (row[2].is_null()) - product.isRegular = false; - else { - product.isRegular = true; - product.fileSize = row[2].as(); - } - if (!row[3].is_null()) - product.sha256hash = Hash::parseAny(row[3].as(), HashAlgorithm::SHA256); - if (!row[4].is_null()) - product.path = row[4].as(); - product.name = row[5].as(); - if (!row[6].is_null()) - product.defaultPath = row[6].as(); - res.products.emplace_back(product); - } - - auto metrics = txn.exec("select name, unit, value from BuildMetrics where build = $1", - pqxx::params{id}); - - for (auto row : metrics) { - BuildMetric metric; - metric.name = row[0].as(); - metric.unit = row[1].is_null() ? "" : row[1].as(); - metric.value = row[2].as(); - res.metrics.emplace(metric.name, metric); - } - - return res; - } - - } - - NarMemberDatas narMembers; - return getBuildOutput(destStore, narMembers, derivationOutputs); -} diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh deleted file mode 100644 index eee042e21..000000000 --- a/src/hydra-queue-runner/state.hh +++ /dev/null @@ -1,597 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "db.hh" - -#include -#include -#include -#include -#include -#include -#include -#include "nar-extractor.hh" -#include -#include -#include -#include -#include - - -typedef unsigned int BuildID; - -typedef unsigned int JobsetID; - -typedef std::chrono::time_point system_time; - -typedef std::atomic counter; - - -typedef enum { - bsSuccess = 0, - bsFailed = 1, - bsDepFailed = 2, // builds only - bsAborted = 3, - bsCancelled = 4, - bsFailedWithOutput = 6, // builds only - bsTimedOut = 7, - bsCachedFailure = 8, // steps only - bsUnsupported = 9, - bsLogLimitExceeded = 10, - bsNarSizeLimitExceeded = 11, - bsNotDeterministic = 12, - bsBusy = 100, // not stored -} BuildStatus; - - -typedef enum { - ssPreparing = 1, - ssConnecting = 10, - ssSendingInputs = 20, - ssBuilding = 30, - ssWaitingForLocalSlot = 35, - ssReceivingOutputs = 40, - ssPostProcessing = 50, -} StepState; - - -struct RemoteResult -{ - BuildStatus stepStatus = bsAborted; - bool canRetry = false; // for bsAborted - bool isCached = false; // for bsSucceed - bool canCache = false; // for bsFailed - std::string errorMsg; // for bsAborted - - unsigned int timesBuilt = 0; - bool isNonDeterministic = false; - - time_t startTime = 0, stopTime = 0; - unsigned int overhead = 0; - nix::Path logFile; - - BuildStatus buildStatus() const - { - return stepStatus == bsCachedFailure ? bsFailed : stepStatus; - } - - void updateWithBuildResult(const nix::BuildResult &); -}; - - -struct Step; -struct BuildOutput; - - -class Jobset -{ -public: - - typedef std::shared_ptr ptr; - typedef std::weak_ptr wptr; - - static const time_t schedulingWindow = 24 * 60 * 60; - -private: - - std::atomic seconds{0}; - std::atomic shares{1}; - - /* The start time and duration of the most recent build steps. */ - nix::Sync> steps; - -public: - - double shareUsed() - { - return (double) seconds / shares; - } - - void setShares(int shares_) - { - assert(shares_ > 0); - shares = shares_; - } - - time_t getSeconds() { return seconds; } - - void addStep(time_t startTime, time_t duration); - - void pruneSteps(); -}; - - -struct Build -{ - typedef std::shared_ptr ptr; - typedef std::weak_ptr wptr; - - BuildID id; - nix::StorePath drvPath; - std::map outputs; - JobsetID jobsetId; - std::string projectName, jobsetName, jobName; - time_t timestamp; - unsigned int maxSilentTime, buildTimeout; - int localPriority, globalPriority; - - std::shared_ptr toplevel; - - Jobset::ptr jobset; - - std::atomic_bool finishedInDB{false}; - - Build(nix::StorePath && drvPath) : drvPath(std::move(drvPath)) - { } - - std::string fullJobName() - { - return projectName + ":" + jobsetName + ":" + jobName; - } - - void propagatePriorities(); -}; - - -struct Step -{ - typedef std::shared_ptr ptr; - typedef std::weak_ptr wptr; - - nix::StorePath drvPath; - std::unique_ptr drv; - std::unique_ptr drvOptions; - nix::StringSet requiredSystemFeatures; - bool preferLocalBuild; - bool isDeterministic; - std::string systemType; // concatenation of drv.platform and requiredSystemFeatures - - struct State - { - /* Whether the step has finished initialisation. */ - bool created = false; - - /* The build steps on which this step depends. */ - std::set deps; - - /* The build steps that depend on this step. */ - std::vector rdeps; - - /* Builds that have this step as the top-level derivation. */ - std::vector builds; - - /* Jobsets to which this step belongs. Used for determining - scheduling priority. */ - std::set jobsets; - - /* Number of times we've tried this step. */ - unsigned int tries = 0; - - /* Point in time after which the step can be retried. */ - system_time after; - - /* The highest global priority of any build depending on this - step. */ - int highestGlobalPriority{0}; - - /* The highest local priority of any build depending on this - step. */ - int highestLocalPriority{0}; - - /* The lowest ID of any build depending on this step. */ - BuildID lowestBuildID{std::numeric_limits::max()}; - - /* The time at which this step became runnable. */ - system_time runnableSince; - - /* The time that we last saw a machine that supports this - step. */ - system_time lastSupported = std::chrono::system_clock::now(); - }; - - std::atomic_bool finished{false}; // debugging - - nix::Sync state; - - Step(const nix::StorePath & drvPath) : drvPath(drvPath) - { } - - ~Step() - { - //printMsg(lvlError, format("destroying step %1%") % drvPath); - } -}; - - -void getDependents(Step::ptr step, std::set & builds, std::set & steps); - -/* Call ‘visitor’ for a step and all its dependencies. */ -void visitDependencies(std::function visitor, Step::ptr step); - - -struct Machine : nix::Machine -{ - typedef std::shared_ptr ptr; - - struct State { - typedef std::shared_ptr ptr; - counter currentJobs{0}; - counter nrStepsDone{0}; - counter totalStepTime{0}; // total time for steps, including closure copying - counter totalStepBuildTime{0}; // total build time for steps - std::atomic idleSince{0}; - - struct ConnectInfo - { - system_time lastFailure, disabledUntil; - unsigned int consecutiveFailures; - }; - nix::Sync connectInfo; - - /* Mutex to prevent multiple threads from sending data to the - same machine (which would be inefficient). */ - std::timed_mutex sendLock; - }; - - State::ptr state; - - bool supportsStep(Step::ptr step) - { - /* Check that this machine is of the type required by the - step. */ - if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) - return false; - - /* Check that the step requires all mandatory features of this - machine. (Thus, a machine with the mandatory "benchmark" - feature will *only* execute steps that require - "benchmark".) The "preferLocalBuild" bit of a step is - mapped to the "local" feature; thus machines that have - "local" as a mandatory feature will only do - preferLocalBuild steps. */ - for (auto & f : mandatoryFeatures) - if (!step->requiredSystemFeatures.count(f) - && !(f == "local" && step->preferLocalBuild)) - return false; - - /* Check that the machine supports all features required by - the step. */ - for (auto & f : step->requiredSystemFeatures) - if (!supportedFeatures.count(f)) return false; - - return true; - } - - bool isLocalhost() const; - - // A connection to a machine - struct Connection : nix::ServeProto::BasicClientConnection { - // Backpointer to the machine - ptr machine; - }; -}; - - -class HydraConfig; - - -class State -{ -private: - - std::unique_ptr config; - - // FIXME: Make configurable. - const unsigned int maxTries = 5; - const unsigned int retryInterval = 60; // seconds - const float retryBackoff = 3.0; - const unsigned int maxParallelCopyClosure = 4; - - /* Time in seconds before unsupported build steps are aborted. */ - const unsigned int maxUnsupportedTime = 0; - - nix::Path hydraData, logDir; - - bool useSubstitutes = false; - - /* The queued builds. */ - typedef std::map Builds; - nix::Sync builds; - - /* The jobsets. */ - typedef std::map, Jobset::ptr> Jobsets; - nix::Sync jobsets; - - /* All active or pending build steps (i.e. dependencies of the - queued builds). Note that these are weak pointers. Steps are - kept alive by being reachable from Builds or by being in - progress. */ - typedef std::map Steps; - nix::Sync steps; - - /* Build steps that have no unbuilt dependencies. */ - typedef std::list Runnable; - nix::Sync runnable; - - /* CV for waking up the dispatcher. */ - nix::Sync dispatcherWakeup; - std::condition_variable dispatcherWakeupCV; - - /* PostgreSQL connection pool. */ - nix::Pool dbPool; - - /* The build machines. */ - std::mutex machinesReadyLock; - typedef std::map Machines; - nix::Sync machines; // FIXME: use atomic_shared_ptr - - /* Throttler for CPU-bound local work. */ - static constexpr unsigned int maxSupportedLocalWorkers = 1024; - std::counting_semaphore localWorkThrottler; - - /* Various stats. */ - time_t startedAt; - counter nrBuildsRead{0}; - counter buildReadTimeMs{0}; - counter nrBuildsDone{0}; - counter nrStepsStarted{0}; - counter nrStepsDone{0}; - counter nrStepsBuilding{0}; - counter nrStepsCopyingTo{0}; - counter nrStepsWaitingForDownloadSlot{0}; - counter nrStepsCopyingFrom{0}; - counter nrStepsWaiting{0}; - counter nrUnsupportedSteps{0}; - counter nrRetries{0}; - counter maxNrRetries{0}; - counter totalStepTime{0}; // total time for steps, including closure copying - counter totalStepBuildTime{0}; // total build time for steps - counter nrQueueWakeups{0}; - counter nrDispatcherWakeups{0}; - counter dispatchTimeMs{0}; - counter bytesSent{0}; - counter bytesReceived{0}; - counter nrActiveDbUpdates{0}; - - /* Specific build to do for --build-one (testing only). */ - BuildID buildOne; - bool buildOneDone = false; - - /* Statistics per machine type for the Hydra auto-scaler. */ - struct MachineType - { - unsigned int runnable{0}, running{0}; - system_time lastActive; - std::chrono::seconds waitTime; // time runnable steps have been waiting - }; - - nix::Sync> machineTypes; - - struct MachineReservation - { - State & state; - Step::ptr step; - Machine::ptr machine; - MachineReservation(State & state, Step::ptr step, Machine::ptr machine); - ~MachineReservation(); - }; - - struct ActiveStep - { - Step::ptr step; - - struct State - { - pid_t pid = -1; - bool cancelled = false; - }; - - nix::Sync state_; - }; - - nix::Sync>> activeSteps_; - - std::atomic lastDispatcherCheck{0}; - - std::shared_ptr localStore; - std::shared_ptr _destStore; - - size_t maxOutputSize; - size_t maxLogSize; - - /* Steps that were busy while we encounted a PostgreSQL - error. These need to be cleared at a later time to prevent them - from showing up as busy until the queue runner is restarted. */ - nix::Sync>> orphanedSteps; - - /* How often the build steps of a jobset should be repeated in - order to detect non-determinism. */ - std::map, size_t> jobsetRepeats; - - bool uploadLogsToBinaryCache; - - /* Where to store GC roots. Defaults to - /nix/var/nix/gcroots/per-user/$USER/hydra-roots, overridable - via gc_roots_dir. */ - nix::Path rootsDir; - - std::string metricsAddr; - - struct PromMetrics - { - std::shared_ptr registry; - - prometheus::Counter& queue_checks_started; - prometheus::Counter& queue_build_loads; - prometheus::Counter& queue_steps_created; - prometheus::Counter& queue_checks_early_exits; - prometheus::Counter& queue_checks_finished; - - prometheus::Counter& dispatcher_time_spent_running; - prometheus::Counter& dispatcher_time_spent_waiting; - - prometheus::Counter& queue_monitor_time_spent_running; - prometheus::Counter& queue_monitor_time_spent_waiting; - - PromMetrics(); - }; - PromMetrics prom; - -public: - State(std::optional metricsAddrOpt); - -private: - - nix::MaintainCount startDbUpdate(); - - /* Return a store object to store build results. */ - nix::ref getDestStore(); - - void clearBusy(Connection & conn, time_t stopTime); - - void parseMachines(const std::string & contents); - - /* Thread to reload /etc/nix/machines periodically. */ - void monitorMachinesFile(); - - unsigned int allocBuildStep(pqxx::work & txn, BuildID buildId); - - unsigned int createBuildStep(pqxx::work & txn, time_t startTime, BuildID buildId, Step::ptr step, - const std::string & machine, BuildStatus status, const std::string & errorMsg = "", - BuildID propagatedFrom = 0); - - void updateBuildStep(pqxx::work & txn, BuildID buildId, unsigned int stepNr, StepState stepState); - - void finishBuildStep(pqxx::work & txn, const RemoteResult & result, BuildID buildId, unsigned int stepNr, - const std::string & machine); - - int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const nix::StorePath & drvPath, const nix::Derivation drv, const std::string & outputName, const nix::StorePath & storePath); - - void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status); - - void queueMonitor(); - - void queueMonitorLoop(Connection & conn); - - /* Check the queue for new builds. */ - bool getQueuedBuilds(Connection & conn, nix::ref destStore); - - /* Handle cancellation, deletion and priority bumps. */ - void processQueueChange(Connection & conn); - - BuildOutput getBuildOutputCached(Connection & conn, nix::ref destStore, - const nix::StorePath & drvPath); - - /* Returns paths missing from the remote store. Paths are processed in - * parallel to work around the possible latency of remote stores. */ - std::map> getMissingRemotePaths( - nix::ref destStore, - const std::map> & paths); - - Step::ptr createStep(nix::ref store, - Connection & conn, Build::ptr build, const nix::StorePath & drvPath, - Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, - std::set & newSteps, std::set & newRunnable); - - void failStep( - Connection & conn, - Step::ptr step, - BuildID buildId, - const RemoteResult & result, - Machine::ptr machine, - bool & stepFinished); - - Jobset::ptr createJobset(pqxx::work & txn, - const std::string & projectName, const std::string & jobsetName, const JobsetID); - - void processJobsetSharesChange(Connection & conn); - - void makeRunnable(Step::ptr step); - - /* The thread that selects and starts runnable builds. */ - void dispatcher(); - - system_time doDispatch(); - - void wakeDispatcher(); - - void abortUnsupported(); - - void builder(std::unique_ptr reservation); - - /* Perform the given build step. Return true if the step is to be - retried. */ - enum StepResult { sDone, sRetry, sMaybeCancelled }; - StepResult doBuildStep(nix::ref destStore, - std::unique_ptr reservation, - std::shared_ptr activeStep); - - void buildRemote(nix::ref destStore, - std::unique_ptr reservation, - Machine::ptr machine, Step::ptr step, - const nix::ServeProto::BuildOptions & buildOptions, - RemoteResult & result, std::shared_ptr activeStep, - std::function updateStep, - NarMemberDatas & narMembers); - - void markSucceededBuild(pqxx::work & txn, Build::ptr build, - const BuildOutput & res, bool isCachedBuild, time_t startTime, time_t stopTime); - - bool checkCachedFailure(Step::ptr step, Connection & conn); - - void notifyBuildStarted(pqxx::work & txn, BuildID buildId); - - void notifyBuildFinished(pqxx::work & txn, BuildID buildId, - const std::vector & dependentIds); - - /* Acquire the global queue runner lock, or null if somebody else - has it. */ - std::shared_ptr acquireGlobalLock(); - - void dumpStatus(Connection & conn); - - void addRoot(const nix::StorePath & storePath); - - void runMetricsExporter(); - -public: - - void showStatus(); - - void unlock(); - - void run(BuildID buildOne = 0); -}; diff --git a/src/meson.build b/src/meson.build index c2ffc0755..9bf5222f0 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,7 +1,6 @@ # Native code subdir('libhydra') subdir('hydra-evaluator') -subdir('hydra-queue-runner') hydra_libexecdir = get_option('libexecdir') / 'hydra' @@ -75,3 +74,60 @@ install_subdir('script', install_mode: 'rwxr-xr-x', strip_directory: true, ) + +# Build rust workspace +cargo_toml = files('../Cargo.toml') +workspace_root = '..' + +otel_feature = get_option('otel').enabled() + +build_type = get_option('buildtype') +if build_type == 'debug' + cargo_profile = 'debug' + cargo_build_args = ['build'] +else + cargo_profile = 'release' + cargo_build_args = ['build', '--release'] +endif + +if otel_feature + cargo_build_args += ['--features', 'otel'] +endif + +rust_build = custom_target( + 'rust-workspace-build', + input: cargo_toml, + output: ['build-stamp'], + command: [ + cargo, + cargo_build_args, + '--manifest-path', '@INPUT@', + '--workspace', + '--message-format=json-render-diagnostics' + ], + build_by_default: true, + install: false, + depends: [], +) + +rust_binaries = { + 'queue-runner': 'hydra-queue-runner-v2', + 'builder': 'hydra-builder-v2', +} + +rust_outputs = [] +foreach src_bin, dest_bin : rust_binaries + rust_outputs += custom_target( + 'rust-' + src_bin, + input: rust_build, + output: dest_bin, + command: [ + 'cp', + workspace_root / 'target' / cargo_profile / src_bin, + '@OUTPUT@' + ], + install: true, + install_dir: get_option('bindir'), + depends: rust_build, + ) +endforeach diff --git a/src/proto/v1/streaming.proto b/src/proto/v1/streaming.proto new file mode 100644 index 000000000..18b64d233 --- /dev/null +++ b/src/proto/v1/streaming.proto @@ -0,0 +1,298 @@ +syntax = "proto3"; + +package runner.v1; + +option java_multiple_files = true; +option java_outer_classname = "RunnerProto"; +option java_package = "io.grpc.hydra.runner"; + +service RunnerService { + rpc CheckVersion(VersionCheckRequest) returns (VersionCheckResponse) {} + rpc OpenTunnel(stream BuilderRequest) returns (stream RunnerRequest) {} + rpc BuildLog(stream LogChunk) returns (Empty) {} + rpc BuildResult(stream NarData) returns (Empty) {} + rpc BuildStepUpdate(StepUpdate) returns (Empty) {} + rpc CompleteBuild(BuildResultInfo) returns (Empty) {} + rpc FetchDrvRequisites(FetchRequisitesRequest) returns (DrvRequisitesMessage) {} + rpc HasPath(StorePath) returns (HasPathResponse) {} + rpc StreamFile(StorePath) returns (stream NarData) {} + rpc StreamFiles(StorePaths) returns (stream NarData) {} + rpc RequestPresignedUrl(PresignedUrlRequest) returns (PresignedUrlResponse) {} + rpc NotifyPresignedUploadComplete(PresignedUploadComplete) returns (Empty) {} +} + +message Empty {} + +message VersionCheckRequest { + string version = 1; + string machine_id = 2; + string hostname = 3; +} + +message VersionCheckResponse { + bool compatible = 1; + string server_version = 2; +} + +message BuilderRequest { + oneof message { + JoinMessage join = 1; + PingMessage ping = 2; + } +} + +message JoinMessage { + string machine_id = 1; // UUIDv4 + repeated string systems = 2; + string hostname = 3; + uint32 cpu_count = 4; + float bogomips = 5; + float speed_factor = 6; + uint32 max_jobs = 7; + float build_dir_avail_threshold = 8; + float store_avail_threshold = 9; + float load1_threshold = 10; + float cpu_psi_threshold = 11; + float mem_psi_threshold = 12; + optional float io_psi_threshold = 13; + uint64 total_mem = 14; + repeated string supported_features = 15; + repeated string mandatory_features = 16; + bool cgroups = 17; + repeated string substituters = 18; + bool use_substitutes = 19; + string nix_version = 20; +} + +message Pressure { + float avg10 = 1; + float avg60 = 2; + float avg300 = 3; + uint64 total = 4; +} + +message PressureState { + Pressure cpu_some = 1; + Pressure mem_some = 2; + Pressure mem_full = 3; + Pressure io_some = 4; + Pressure io_full = 5; + Pressure irq_full = 6; +} + +message PingMessage { + string machine_id = 1; // UUIDv4 + float load1 = 2; + float load5 = 3; + float load15 = 4; + uint64 mem_usage = 5; + optional PressureState pressure = 6; + double build_dir_free_percent = 7; + double store_free_percent = 8; + uint64 current_substituting_path_count = 9; + uint64 current_uploading_path_count = 10; + uint64 current_downloading_path_count = 11; +} + +message SimplePingMessage { + string message = 1; +} + +message RunnerRequest { + oneof message { + JoinResponse join = 1; + ConfigUpdate config_update = 2; + SimplePingMessage ping = 3; + BuildMessage build = 4; + AbortMessage abort = 5; + } +} + +message JoinResponse { + string machine_id = 1; // UUIDv4 + uint32 max_concurrent_downloads = 2; +} + +message ConfigUpdate { + uint32 max_concurrent_downloads = 1; +} + +message PresignedUploadOpts { + bool upload_debug_info = 1; +} + +message BuildMessage { + string build_id = 1; // UUIDv4 + string drv = 2; + optional string resolved_drv = 3; + uint64 max_log_size = 4; + int32 max_silent_time = 5; + int32 build_timeout = 6; + optional PresignedUploadOpts presigned_url_opts = 7; // None if presigned url upload is disabled + // bool is_deterministic = 5; + // bool enforce_determinism = 6; +} + +message DrvRequisitesMessage { + repeated string requisites = 1; +} + +message AbortMessage { + string build_id = 1; // UUIDv4 +} + +message LogChunk { + string drv = 1; + bytes data = 2; +} + +message FetchRequisitesRequest { + string path = 1; + bool include_outputs = 2; +} + +message StorePath { + string path = 1; +} + +message StorePaths { + repeated string paths = 1; +} + +message HasPathResponse { + bool has_path = 1; +} + +message NarData { + bytes chunk = 1; +} + +message OutputNameOnly { + string name = 1; +} + +message OutputWithPath { + string name = 1; + string path = 2; + uint64 closure_size = 3; + uint64 nar_size = 4; + string nar_hash = 5; +} + +message Output { + oneof output { + OutputNameOnly nameonly = 1; + OutputWithPath withpath = 2; + } +} + +message BuildMetric { + string path = 1; + string name = 2; + optional string unit = 3; + double value = 4; +} + +message BuildProduct { + string path = 1; + string default_path = 2; + + string type = 3; + string subtype = 4; + string name = 5; + + bool is_regular = 6; + + optional string sha256hash = 7; + optional uint64 file_size = 8; +} + +message NixSupport { + bool failed = 1; + optional string hydra_release_name = 2; + repeated BuildMetric metrics = 3; + repeated BuildProduct products = 4; +} + +enum StepStatus { + Preparing = 0; + Connecting = 1; + SeningInputs = 2; + Building = 3; + WaitingForLocalSlot = 4; + ReceivingOutputs = 5; + PostProcessing = 6; +} + +message StepUpdate { + // UUID to uniquely identify the build request + string build_id = 1; + string machine_id = 2; + StepStatus step_status = 3; +} + +enum BuildResultState { + BuildFailure = 0; + Success = 1; + PreparingFailure = 2; + ImportFailure = 3; + UploadFailure = 4; + PostProcessingFailure = 5; +} + +message BuildResultInfo { + string build_id = 1; // UUIDv4 + string machine_id = 2; // UUIDv4 + uint64 import_time_ms = 3; + uint64 build_time_ms = 4; + uint64 upload_time_ms = 5; + BuildResultState result_state = 6; + NixSupport nix_support = 7; + repeated Output outputs = 8; +} + +message PresignedNarRequest { + string store_path = 1; + string nar_hash = 2; + repeated string debug_info_build_ids = 3; +} + +message PresignedUrlRequest { + string build_id = 1; // UUIDv4 + string machine_id = 2; // UUIDv4 + repeated PresignedNarRequest request = 3; +} + +message PresignedUpload { + string path = 1; + string url = 2; + string compression = 3; + int32 compression_level = 4; +} + +message PresignedNarResponse { + string store_path = 1; + string nar_url = 2; + PresignedUpload nar_upload = 3; + optional PresignedUpload ls_upload = 4; + repeated PresignedUpload debug_info_upload = 5; +} + +message PresignedUrlResponse { + repeated PresignedNarResponse inner = 1; +} + +message PresignedUploadComplete { + string build_id = 1; // UUIDv4 + string machine_id = 2; // UUIDv4 + string store_path = 3; + string url = 4; + string compression = 5; + string file_hash = 6; + uint64 file_size = 7; + string nar_hash = 8; + uint64 nar_size = 9; + repeated string references = 10; + optional string deriver = 11; + optional string ca = 12; +} diff --git a/src/queue-runner/Cargo.toml b/src/queue-runner/Cargo.toml new file mode 100644 index 000000000..7734cece7 --- /dev/null +++ b/src/queue-runner/Cargo.toml @@ -0,0 +1,68 @@ +[package] +name = "queue-runner" +version.workspace = true +edition = "2024" +license = "GPL-3.0" +rust-version.workspace = true + +[dependencies] +tracing.workspace = true +sd-notify.workspace = true + +toml.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +secrecy = { workspace = true, features = ["serde"] } +smallvec = { workspace = true, features = ["serde"] } +hashbrown = { workspace = true, features = ["serde"] } +arc-swap.workspace = true +parking_lot.workspace = true + +thiserror.workspace = true +anyhow.workspace = true +backon.workspace = true +clap = { workspace = true, features = ["derive", "env"] } +uuid = { workspace = true, features = ["v4", "serde"] } +atomic_float.workspace = true +fs-err = { workspace = true, features = ["tokio"] } + +tokio = { workspace = true, features = ["full"] } +futures.workspace = true +futures-util.workspace = true +byte-unit.workspace = true + +tokio-stream.workspace = true +prost.workspace = true +tonic = { workspace = true, features = ["zstd", "tls-webpki-roots"] } +tonic-health.workspace = true +tonic-reflection.workspace = true +tonic-prost.workspace = true +listenfd.workspace = true +h2.workspace = true + +hyper = { workspace = true, features = ["full"] } +tower = { workspace = true, features = ["util"] } +tower-http = { workspace = true, features = ["trace"] } +http-body-util.workspace = true +hyper-util.workspace = true +bytes.workspace = true +jiff.workspace = true +prometheus = { workspace = true, features = ["process"] } +procfs.workspace = true + +db = { path = "../crates/db" } +nix-utils = { path = "../crates/nix-utils" } +binary-cache = { path = "../crates/binary-cache" } +shared = { path = "../crates/shared" } +hydra-tracing = { path = "../crates/tracing", features = ["tonic"] } + +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemallocator.workspace = true + +[build-dependencies] +tonic-prost-build.workspace = true +sha2.workspace = true +fs-err = { workspace = true } + +[features] +otel = ["hydra-tracing/otel"] diff --git a/src/queue-runner/build.rs b/src/queue-runner/build.rs new file mode 100644 index 000000000..aa17d2f5e --- /dev/null +++ b/src/queue-runner/build.rs @@ -0,0 +1,30 @@ +use sha2::Digest; +use std::{env, path::PathBuf}; + +fn main() -> Result<(), Box> { + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + + let workspace_version = env::var("CARGO_PKG_VERSION")?; + + let proto_path = "../proto/v1/streaming.proto"; + let proto_content = fs_err::read_to_string(proto_path)?; + let mut hasher = sha2::Sha256::new(); + hasher.update(proto_content.as_bytes()); + let proto_hash = format!("{:x}", hasher.finalize()); + let version = format!("{}-{}", workspace_version, &proto_hash[..8]); + + // Generate version module + fs_err::write( + out_dir.join("proto_version.rs"), + format!( + r#"// Generated during build - do not edit +pub const PROTO_API_VERSION: &str = "{version}"; +"# + ), + )?; + + tonic_prost_build::configure() + .file_descriptor_set_path(out_dir.join("streaming_descriptor.bin")) + .compile_protos(&["../proto/v1/streaming.proto"], &["../proto"])?; + Ok(()) +} diff --git a/src/queue-runner/examples/collect-fods.rs b/src/queue-runner/examples/collect-fods.rs new file mode 100644 index 000000000..d331e2982 --- /dev/null +++ b/src/queue-runner/examples/collect-fods.rs @@ -0,0 +1,19 @@ +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let p = nix_utils::StorePath::new("dzgpbp0vp7lj7lgj26rjgmnjicq2wf4k-hello-2.12.2.drv"); + let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(4); + + let store = nix_utils::LocalStore::init(); + let fod = std::sync::Arc::new(queue_runner::state::FodChecker::new(Some(tx))); + fod.clone().start_traverse_loop(store); + fod.to_traverse(&p); + fod.trigger_traverse(); + let _ = rx.recv().await; + fod.process( + async move |path: nix_utils::StorePath, _: nix_utils::Derivation| { + println!("{path}"); + }, + ) + .await; + Ok(()) +} diff --git a/src/queue-runner/src/config.rs b/src/queue-runner/src/config.rs new file mode 100644 index 000000000..38d21df71 --- /dev/null +++ b/src/queue-runner/src/config.rs @@ -0,0 +1,588 @@ +use std::{net::SocketAddr, sync::Arc}; + +use anyhow::Context as _; +use clap::Parser; + +#[derive(Debug, Clone)] +pub enum BindSocket { + Tcp(SocketAddr), + Unix(std::path::PathBuf), + ListenFd, +} + +impl std::str::FromStr for BindSocket { + type Err = String; + + fn from_str(s: &str) -> Result { + s.parse::() + .map(BindSocket::Tcp) + .or_else(|_| { + if s == "-" { + Ok(Self::ListenFd) + } else { + Ok(Self::Unix(s.into())) + } + }) + } +} + +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +pub struct Cli { + /// Query the queue runner status + #[clap(long)] + pub status: bool, + + /// REST server bind + #[clap(short, long, default_value = "[::1]:8080")] + pub rest_bind: SocketAddr, + + /// GRPC server bind, either a `SocketAddr`, a Path for a Unix Socket or `-` to use `ListenFD` (systemd socket activation) + #[clap(short, long, default_value = "[::1]:50051")] + pub grpc_bind: BindSocket, + + /// Config path + #[clap(short, long, default_value = "config.toml")] + pub config_path: String, + + /// Path to Server cert + #[clap(long)] + pub server_cert_path: Option, + + /// Path to Server key + #[clap(long)] + pub server_key_path: Option, + + /// Path to Client ca cert + #[clap(long)] + pub client_ca_cert_path: Option, +} + +impl Default for Cli { + fn default() -> Self { + Self::new() + } +} + +impl Cli { + #[must_use] + pub fn new() -> Self { + Self::parse() + } + + #[must_use] + pub const fn mtls_enabled(&self) -> bool { + self.server_cert_path.is_some() + && self.server_key_path.is_some() + && self.client_ca_cert_path.is_some() + } + + #[must_use] + pub const fn mtls_configured_correctly(&self) -> bool { + self.mtls_enabled() + || (self.server_cert_path.is_none() + && self.server_key_path.is_none() + && self.client_ca_cert_path.is_none()) + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_mtls( + &self, + ) -> anyhow::Result<(tonic::transport::Certificate, tonic::transport::Identity)> { + let server_cert_path = self + .server_cert_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("server_cert_path not provided"))?; + let server_key_path = self + .server_key_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("server_key_path not provided"))?; + + let client_ca_cert_path = self + .client_ca_cert_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("client_ca_cert_path not provided"))?; + let client_ca_cert = fs_err::tokio::read_to_string(client_ca_cert_path).await?; + let client_ca_cert = tonic::transport::Certificate::from_pem(client_ca_cert); + + let server_cert = fs_err::tokio::read_to_string(server_cert_path).await?; + let server_key = fs_err::tokio::read_to_string(server_key_path).await?; + let server_identity = tonic::transport::Identity::from_pem(server_cert, server_key); + Ok((client_ca_cert, server_identity)) + } +} + +fn default_data_dir() -> std::path::PathBuf { + "/tmp/hydra".into() +} + +fn default_pg_socket_url() -> secrecy::SecretString { + "postgres://hydra@%2Frun%2Fpostgresql:5432/hydra".into() +} + +const fn default_max_db_connections() -> u32 { + 128 +} + +const fn default_dispatch_trigger_timer_in_s() -> i64 { + 120 +} + +const fn default_queue_trigger_timer_in_s() -> i64 { + -1 +} + +const fn default_max_tries() -> u32 { + 5 +} + +const fn default_retry_interval() -> u32 { + 60 +} + +const fn default_retry_backoff() -> f32 { + 3.0 +} + +const fn default_max_unsupported_time_in_s() -> i64 { + 120 +} + +const fn default_stop_queue_run_after_in_s() -> i64 { + 60 +} + +const fn default_max_concurrent_downloads() -> u32 { + 5 +} + +const fn default_concurrent_upload_limit() -> usize { + 5 +} + +const fn default_enable_fod_checker() -> bool { + false +} + +#[derive(Debug, Default, serde::Deserialize, Copy, Clone, PartialEq, Eq)] +pub enum MachineSortFn { + SpeedFactorOnly, + CpuCoreCountWithSpeedFactor, + #[default] + BogomipsWithSpeedFactor, +} + +#[derive(Debug, Default, serde::Deserialize, Copy, Clone, PartialEq, Eq)] +pub enum MachineFreeFn { + Dynamic, + DynamicWithMaxJobLimit, + #[default] + Static, +} + +#[derive(Debug, Default, serde::Deserialize, Copy, Clone, PartialEq, Eq)] +pub enum StepSortFn { + Legacy, + #[default] + WithRdeps, +} + +/// Main configuration of the application +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +struct AppConfig { + #[serde(default = "default_data_dir")] + hydra_data_dir: std::path::PathBuf, + + #[serde(default = "default_pg_socket_url")] + db_url: secrecy::SecretString, + + #[serde(default = "default_max_db_connections")] + max_db_connections: u32, + + #[serde(default)] + machine_sort_fn: MachineSortFn, + + #[serde(default)] + machine_free_fn: MachineFreeFn, + + #[serde(default)] + step_sort_fn: StepSortFn, + + // setting this to -1, will disable the timer + #[serde(default = "default_dispatch_trigger_timer_in_s")] + dispatch_trigger_timer_in_s: i64, + + // setting this to -1, will disable the timer + #[serde(default = "default_queue_trigger_timer_in_s")] + queue_trigger_timer_in_s: i64, + + #[serde(default)] + remote_store_addr: Vec, + + #[serde(default)] + use_substitutes: bool, + + roots_dir: Option, + + #[serde(default = "default_max_tries")] + max_retries: u32, + + #[serde(default = "default_retry_interval")] + retry_interval: u32, + + #[serde(default = "default_retry_backoff")] + retry_backoff: f32, + + #[serde(default = "default_max_unsupported_time_in_s")] + max_unsupported_time_in_s: i64, + + #[serde(default = "default_stop_queue_run_after_in_s")] + stop_queue_run_after_in_s: i64, + + #[serde(default = "default_max_concurrent_downloads")] + max_concurrent_downloads: u32, + + #[serde(default = "default_concurrent_upload_limit")] + concurrent_upload_limit: usize, + + token_list_path: Option, + + #[serde(default = "default_enable_fod_checker")] + enable_fod_checker: bool, + + #[serde(default)] + use_presigned_uploads: bool, + + #[serde(default)] + forced_substituters: Vec, +} + +/// Prepared configuration of the application +#[derive(Debug)] +pub struct PreparedApp { + #[allow(dead_code)] + hydra_data_dir: std::path::PathBuf, + hydra_log_dir: std::path::PathBuf, + lockfile: std::path::PathBuf, + pub db_url: secrecy::SecretString, + max_db_connections: u32, + pub machine_sort_fn: MachineSortFn, + machine_free_fn: MachineFreeFn, + pub step_sort_fn: StepSortFn, + dispatch_trigger_timer: Option, + queue_trigger_timer: Option, + pub remote_store_addr: Vec, + use_substitutes: bool, + roots_dir: std::path::PathBuf, + max_retries: u32, + retry_interval: f32, + retry_backoff: f32, + max_unsupported_time: jiff::SignedDuration, + stop_queue_run_after: Option, + pub max_concurrent_downloads: u32, + concurrent_upload_limit: usize, + token_list: Option>, + pub enable_fod_checker: bool, + pub use_presigned_uploads: bool, + pub forced_substituters: Vec, +} + +impl TryFrom for PreparedApp { + type Error = anyhow::Error; + + fn try_from(val: AppConfig) -> Result { + let remote_store_addr = val + .remote_store_addr + .into_iter() + .filter(|v| { + v.starts_with("file://") + || v.starts_with("s3://") + || v.starts_with("ssh://") + || v.starts_with('/') + }) + .collect(); + + let logname = std::env::var("LOGNAME").context("LOGNAME env var missing")?; + let nix_state_dir = + std::env::var("NIX_STATE_DIR").unwrap_or_else(|_| "/nix/var/nix/".to_owned()); + let roots_dir = val.roots_dir.map_or_else( + || { + std::path::PathBuf::from(nix_state_dir) + .join("gcroots/per-user") + .join(logname) + .join("hydra-roots") + }, + |roots_dir| roots_dir, + ); + fs_err::create_dir_all(&roots_dir)?; + + let hydra_log_dir = val.hydra_data_dir.join("build-logs"); + let lockfile = val.hydra_data_dir.join("queue-runner/lock"); + + let token_list = val.token_list_path.and_then(|p| { + fs_err::read_to_string(p) + .map(|s| s.lines().map(|t| t.trim().to_string()).collect()) + .ok() + }); + + Ok(Self { + hydra_data_dir: val.hydra_data_dir, + hydra_log_dir, + lockfile, + db_url: val.db_url, + max_db_connections: val.max_db_connections, + machine_sort_fn: val.machine_sort_fn, + machine_free_fn: val.machine_free_fn, + step_sort_fn: val.step_sort_fn, + dispatch_trigger_timer: u64::try_from(val.dispatch_trigger_timer_in_s) + .ok() + .and_then(|v| { + if v == 0 { + None + } else { + Some(tokio::time::Duration::from_secs(v)) + } + }), + queue_trigger_timer: u64::try_from(val.queue_trigger_timer_in_s) + .ok() + .and_then(|v| { + if v == 0 { + None + } else { + Some(tokio::time::Duration::from_secs(v)) + } + }), + remote_store_addr, + use_substitutes: val.use_substitutes, + roots_dir, + max_retries: val.max_retries, + #[allow(clippy::cast_precision_loss)] + retry_interval: val.retry_interval as f32, + retry_backoff: val.retry_backoff, + max_unsupported_time: jiff::SignedDuration::from_secs(val.max_unsupported_time_in_s), + stop_queue_run_after: if val.stop_queue_run_after_in_s <= 0 { + None + } else { + Some(jiff::SignedDuration::from_secs( + val.stop_queue_run_after_in_s, + )) + }, + max_concurrent_downloads: val.max_concurrent_downloads, + concurrent_upload_limit: val.concurrent_upload_limit, + token_list, + enable_fod_checker: val.enable_fod_checker, + use_presigned_uploads: val.use_presigned_uploads, + forced_substituters: val.forced_substituters, + }) + } +} + +/// Loads the config from specified path +#[tracing::instrument(err)] +fn load_config(filepath: &str) -> anyhow::Result { + tracing::info!("Trying to loading file: {filepath}"); + let toml: AppConfig = if let Ok(content) = fs_err::read_to_string(filepath) { + toml::from_str(&content) + .with_context(|| format!("Failed to toml load from '{filepath}'"))? + } else { + tracing::warn!("no config file found! Using default config"); + toml::from_str("").context("Failed to parse empty string as config")? + }; + tracing::info!("Loaded config: {toml:?}"); + + toml.try_into().context("Failed to prepare configuration") +} + +#[derive(Debug, Clone)] +pub struct App { + inner: Arc>, +} + +impl App { + #[tracing::instrument(err)] + pub fn init(filepath: &str) -> anyhow::Result { + Ok(Self { + inner: Arc::new(arc_swap::ArcSwap::from(Arc::new(load_config(filepath)?))), + }) + } + + fn swap_inner(&self, new_val: PreparedApp) { + self.inner.store(Arc::new(new_val)); + } + + #[must_use] + pub fn get_hydra_log_dir(&self) -> std::path::PathBuf { + let inner = self.inner.load(); + inner.hydra_log_dir.clone() + } + + #[must_use] + pub fn get_lockfile(&self) -> std::path::PathBuf { + let inner = self.inner.load(); + inner.lockfile.clone() + } + + #[must_use] + pub fn get_db_url(&self) -> secrecy::SecretString { + let inner = self.inner.load(); + inner.db_url.clone() + } + + #[must_use] + pub fn get_max_db_connections(&self) -> u32 { + let inner = self.inner.load(); + inner.max_db_connections + } + + #[must_use] + pub fn get_machine_sort_fn(&self) -> MachineSortFn { + let inner = self.inner.load(); + inner.machine_sort_fn + } + + #[must_use] + pub fn get_machine_free_fn(&self) -> MachineFreeFn { + let inner = self.inner.load(); + inner.machine_free_fn + } + + #[must_use] + pub fn get_step_sort_fn(&self) -> StepSortFn { + let inner = self.inner.load(); + inner.step_sort_fn + } + + #[must_use] + pub fn use_presigned_uploads(&self) -> bool { + let inner = self.inner.load(); + inner.use_presigned_uploads + } + + #[must_use] + pub fn get_dispatch_trigger_timer(&self) -> Option { + let inner = self.inner.load(); + inner.dispatch_trigger_timer + } + + #[must_use] + pub fn get_queue_trigger_timer(&self) -> Option { + let inner = self.inner.load(); + inner.queue_trigger_timer + } + + #[must_use] + pub fn get_remote_store_addrs(&self) -> Vec { + let inner = self.inner.load(); + inner.remote_store_addr.clone() + } + + #[must_use] + pub fn get_use_substitutes(&self) -> bool { + let inner = self.inner.load(); + inner.use_substitutes + } + + #[must_use] + pub fn get_roots_dir(&self) -> std::path::PathBuf { + let inner = self.inner.load(); + inner.roots_dir.clone() + } + + #[must_use] + pub fn get_retry(&self) -> (u32, f32, f32) { + let inner = self.inner.load(); + (inner.max_retries, inner.retry_interval, inner.retry_backoff) + } + + #[must_use] + pub fn get_max_unsupported_time(&self) -> jiff::SignedDuration { + let inner = self.inner.load(); + inner.max_unsupported_time + } + + #[must_use] + pub fn get_stop_queue_run_after(&self) -> Option { + let inner = self.inner.load(); + inner.stop_queue_run_after + } + + #[must_use] + pub fn get_max_concurrent_downloads(&self) -> u32 { + let inner = self.inner.load(); + inner.max_concurrent_downloads + } + + #[must_use] + pub fn get_concurrent_upload_limit(&self) -> usize { + let inner = self.inner.load(); + inner.concurrent_upload_limit + } + + #[must_use] + pub fn has_token_list(&self) -> bool { + let inner = self.inner.load(); + inner.token_list.is_some() + } + + #[must_use] + pub fn check_if_contains_token(&self, token: &str) -> bool { + let inner = self.inner.load(); + inner + .token_list + .as_ref() + .is_some_and(|l| l.iter().any(|t| t == token)) + } + + #[must_use] + pub fn get_enable_fod_checker(&self) -> bool { + let inner = self.inner.load(); + inner.enable_fod_checker + } + + #[must_use] + pub fn get_forced_substituters(&self) -> Vec { + let inner = self.inner.load(); + inner.forced_substituters.clone() + } +} + +pub async fn reload(current_config: &App, filepath: &str, state: &Arc) { + let new_config = match load_config(filepath) { + Ok(c) => c, + Err(e) => { + tracing::warn!("Failed to load new config: {e}"); + let _notify = sd_notify::notify( + false, + &[ + sd_notify::NotifyState::Status("Reload failed"), + sd_notify::NotifyState::Errno(1), + ], + ); + + return; + } + }; + + if let Err(e) = state.reload_config_callback(&new_config).await { + tracing::error!("Config reload failed with {e}"); + let _notify = sd_notify::notify( + false, + &[ + sd_notify::NotifyState::Status("Configuration reloaded failed - Running"), + sd_notify::NotifyState::Errno(1), + ], + ); + return; + } + + current_config.swap_inner(new_config); + let _notify = sd_notify::notify( + false, + &[ + sd_notify::NotifyState::Status("Configuration reloaded - Running"), + sd_notify::NotifyState::Ready, + ], + ); +} diff --git a/src/queue-runner/src/io/build.rs b/src/queue-runner/src/io/build.rs new file mode 100644 index 000000000..0a53432be --- /dev/null +++ b/src/queue-runner/src/io/build.rs @@ -0,0 +1,40 @@ +use std::sync::atomic::Ordering; + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Build { + id: db::models::BuildID, + drv_path: nix_utils::StorePath, + jobset_id: crate::state::JobsetID, + name: String, + timestamp: jiff::Timestamp, + max_silent_time: i32, + timeout: i32, + local_priority: i32, + global_priority: i32, + finished_in_db: bool, +} + +impl From> for Build { + fn from(item: std::sync::Arc) -> Self { + Self { + id: item.id, + drv_path: item.drv_path.clone(), + jobset_id: item.jobset_id, + name: item.name.clone(), + timestamp: item.timestamp, + max_silent_time: item.max_silent_time, + timeout: item.timeout, + local_priority: item.local_priority, + global_priority: item.global_priority.load(Ordering::Relaxed), + finished_in_db: item.get_finished_in_db(), + } + } +} + +#[derive(Debug, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BuildPayload { + pub drv: String, + pub jobset_id: i32, +} diff --git a/src/queue-runner/src/io/jobset.rs b/src/queue-runner/src/io/jobset.rs new file mode 100644 index 000000000..94f15a531 --- /dev/null +++ b/src/queue-runner/src/io/jobset.rs @@ -0,0 +1,22 @@ +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Jobset { + id: crate::state::JobsetID, + project_name: String, + name: String, + + pub seconds: i64, + pub shares: u32, +} + +impl From> for Jobset { + fn from(item: std::sync::Arc) -> Self { + Self { + id: item.id, + project_name: item.project_name.clone(), + name: item.name.clone(), + seconds: item.get_seconds(), + shares: item.get_shares(), + } + } +} diff --git a/src/queue-runner/src/io/machine.rs b/src/queue-runner/src/io/machine.rs new file mode 100644 index 000000000..b23697eec --- /dev/null +++ b/src/queue-runner/src/io/machine.rs @@ -0,0 +1,213 @@ +use std::sync::{Arc, atomic::Ordering}; + +use smallvec::SmallVec; + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Pressure { + avg10: f32, + avg60: f32, + avg300: f32, + total: u64, +} + +impl From for Pressure { + fn from(v: crate::state::Pressure) -> Self { + Self { + avg10: v.avg10, + avg60: v.avg60, + avg300: v.avg300, + total: v.total, + } + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PressureState { + cpu_some: Option, + mem_some: Option, + mem_full: Option, + io_some: Option, + io_full: Option, + irq_full: Option, +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MachineStats { + current_jobs: u64, + nr_steps_done: u64, + avg_step_time_ms: u64, + avg_step_import_time_ms: u64, + avg_step_build_time_ms: u64, + avg_step_upload_time_ms: u64, + total_step_time_ms: u64, + total_step_import_time_ms: u64, + total_step_build_time_ms: u64, + total_step_upload_time_ms: u64, + idle_since: i64, + + last_failure: i64, + disabled_until: i64, + consecutive_failures: u64, + last_ping: i64, + since_last_ping: i64, + + load1: f32, + load5: f32, + load15: f32, + mem_usage: u64, + pressure: Option, + build_dir_free_percent: f64, + store_free_percent: f64, + current_uploading_path_count: u64, + current_downloading_path_count: u64, + + jobs_in_last_30s_start: i64, + jobs_in_last_30s_count: u64, + pub failed_builds: u64, + pub succeeded_builds: u64, +} + +impl MachineStats { + fn from(item: &std::sync::Arc, now: i64) -> Self { + let last_ping = item.get_last_ping(); + + let nr_steps_done = item.get_nr_steps_done(); + let total_step_time_ms = item.get_total_step_time_ms(); + let total_step_import_time_ms = item.get_total_step_import_time_ms(); + let total_step_build_time_ms = item.get_total_step_build_time_ms(); + let total_step_upload_time_ms = item.get_total_step_upload_time_ms(); + let ( + avg_step_time_ms, + avg_step_import_time_ms, + avg_step_build_time_ms, + avg_step_upload_time_ms, + ) = if nr_steps_done > 0 { + ( + total_step_time_ms / nr_steps_done, + total_step_import_time_ms / nr_steps_done, + total_step_build_time_ms / nr_steps_done, + total_step_upload_time_ms / nr_steps_done, + ) + } else { + (0, 0, 0, 0) + }; + + Self { + current_jobs: item.get_current_jobs(), + nr_steps_done, + avg_step_time_ms, + avg_step_import_time_ms, + avg_step_build_time_ms, + avg_step_upload_time_ms, + total_step_time_ms, + total_step_import_time_ms, + total_step_build_time_ms, + total_step_upload_time_ms, + idle_since: item.get_idle_since(), + last_failure: item.get_last_failure(), + disabled_until: item.get_disabled_until(), + consecutive_failures: item.get_consecutive_failures(), + last_ping, + since_last_ping: now - last_ping, + load1: item.get_load1(), + load5: item.get_load5(), + load15: item.get_load15(), + mem_usage: item.get_mem_usage(), + pressure: item.pressure.load().as_ref().map(|p| PressureState { + cpu_some: p.cpu_some.map(Into::into), + mem_some: p.mem_some.map(Into::into), + mem_full: p.mem_full.map(Into::into), + io_some: p.io_some.map(Into::into), + io_full: p.io_full.map(Into::into), + irq_full: p.irq_full.map(Into::into), + }), + build_dir_free_percent: item.get_build_dir_free_percent(), + store_free_percent: item.get_store_free_percent(), + current_uploading_path_count: item.get_current_uploading_path_count(), + current_downloading_path_count: item.get_current_downloading_count(), + jobs_in_last_30s_start: item.jobs_in_last_30s_start.load(Ordering::Relaxed), + jobs_in_last_30s_count: item.jobs_in_last_30s_count.load(Ordering::Relaxed), + failed_builds: item.get_failed_builds(), + succeeded_builds: item.get_succeeded_builds(), + } + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +#[allow(clippy::struct_excessive_bools)] +pub struct Machine { + systems: SmallVec<[crate::state::System; 4]>, + hostname: String, + uptime: f64, + cpu_count: u32, + bogomips: f32, + speed_factor: f32, + max_jobs: u32, + build_dir_avail_threshold: f64, + store_avail_threshold: f64, + load1_threshold: f32, + cpu_psi_threshold: f32, + mem_psi_threshold: f32, + io_psi_threshold: Option, + score: f32, + total_mem: u64, + supported_features: SmallVec<[String; 8]>, + mandatory_features: SmallVec<[String; 4]>, + cgroups: bool, + substituters: SmallVec<[String; 4]>, + use_substitutes: bool, + nix_version: String, + stats: MachineStats, + jobs: Vec, + + has_capacity: bool, + has_dynamic_capacity: bool, + has_static_capacity: bool, +} + +impl Machine { + #[must_use] + pub fn from_state( + item: &Arc, + sort_fn: crate::config::MachineSortFn, + free_fn: crate::config::MachineFreeFn, + ) -> Self { + let jobs = { item.jobs.read().iter().map(|j| j.path.clone()).collect() }; + let time = jiff::Timestamp::now(); + Self { + systems: item.systems.clone(), + uptime: (time - item.joined_at) + .total(jiff::Unit::Second) + .unwrap_or_default(), + hostname: item.hostname.clone(), + cpu_count: item.cpu_count, + bogomips: item.bogomips, + speed_factor: item.speed_factor, + max_jobs: item.max_jobs, + build_dir_avail_threshold: item.build_dir_avail_threshold, + store_avail_threshold: item.store_avail_threshold, + load1_threshold: item.load1_threshold, + cpu_psi_threshold: item.cpu_psi_threshold, + mem_psi_threshold: item.mem_psi_threshold, + io_psi_threshold: item.io_psi_threshold, + score: item.score(sort_fn), + total_mem: item.total_mem, + supported_features: item.supported_features.clone(), + mandatory_features: item.mandatory_features.clone(), + cgroups: item.cgroups, + substituters: item.substituters.clone(), + use_substitutes: item.use_substitutes, + nix_version: item.nix_version.clone(), + + stats: MachineStats::from(&item.stats, time.as_second()), + jobs, + has_capacity: item.has_capacity(free_fn), + has_dynamic_capacity: item.has_dynamic_capacity(), + has_static_capacity: item.has_static_capacity(), + } + } +} diff --git a/src/queue-runner/src/io/mod.rs b/src/queue-runner/src/io/mod.rs new file mode 100644 index 000000000..ee8de828a --- /dev/null +++ b/src/queue-runner/src/io/mod.rs @@ -0,0 +1,30 @@ +pub mod build; +pub mod jobset; +pub mod machine; +pub mod queue_runner; +pub mod response_types; +pub mod stats; +pub mod step; +pub mod step_info; + +pub use build::{Build, BuildPayload}; +pub use jobset::Jobset; +pub use machine::{Machine, MachineStats}; +pub use queue_runner::QueueRunnerStats; +pub use response_types::{ + BuildsResponse, DumpResponse, JobsetsResponse, MachinesResponse, QueueResponse, + StepInfoResponse, StepsResponse, +}; +pub use stats::{BuildQueueStats, CgroupStats, CpuStats, IoStats, MemoryStats, Process}; +pub use step::Step; +pub use step_info::StepInfo; + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Empty {} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Error { + pub error: String, +} diff --git a/src/queue-runner/src/io/queue_runner.rs b/src/queue-runner/src/io/queue_runner.rs new file mode 100644 index 000000000..ce751b101 --- /dev/null +++ b/src/queue-runner/src/io/queue_runner.rs @@ -0,0 +1,146 @@ +use std::sync::Arc; + +use hashbrown::HashMap; + +use super::{BuildQueueStats, Process}; + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct QueueRunnerStats { + status: &'static str, + time: jiff::Timestamp, + uptime: f64, + proc: Option, + supported_features: Vec, + + build_count: usize, + jobset_count: usize, + step_count: usize, + runnable_count: usize, + queue_stats: HashMap, + + queue_checks_started: u64, + queue_build_loads: u64, + queue_steps_created: u64, + queue_checks_early_exits: u64, + queue_checks_finished: u64, + + dispatcher_time_spent_running: u64, + dispatcher_time_spent_waiting: u64, + + queue_monitor_time_spent_running: u64, + queue_monitor_time_spent_waiting: u64, + + nr_builds_read: u64, + build_read_time_ms: u64, + nr_builds_unfinished: i64, + nr_builds_done: u64, + nr_builds_succeeded: u64, + nr_builds_failed: u64, + nr_steps_started: u64, + nr_steps_done: u64, + nr_steps_building: i64, + nr_steps_waiting: i64, + nr_steps_runnable: i64, + nr_steps_unfinished: i64, + nr_unsupported_steps: i64, + nr_unsupported_steps_aborted: u64, + nr_substitutes_started: u64, + nr_substitutes_failed: u64, + nr_substitutes_succeeded: u64, + nr_retries: u64, + max_nr_retries: i64, + avg_step_time_ms: i64, + avg_step_import_time_ms: i64, + avg_step_build_time_ms: i64, + avg_step_upload_time_ms: i64, + total_step_time_ms: u64, + total_step_import_time_ms: u64, + total_step_build_time_ms: u64, + total_step_upload_time_ms: u64, + nr_queue_wakeups: u64, + nr_dispatcher_wakeups: u64, + dispatch_time_ms: u64, + machines_total: i64, + machines_in_use: i64, +} + +impl QueueRunnerStats { + pub async fn new(state: Arc) -> Self { + let build_count = state.builds.len(); + let jobset_count = state.jobsets.len(); + let step_count = state.steps.len(); + let runnable_count = state.steps.len_runnable(); + let queue_stats = { + state + .queues + .get_stats_per_queue() + .await + .into_iter() + .map(|(system, stats)| (system, stats.into())) + .collect() + }; + + state.metrics.refresh_dynamic_metrics(&state).await; + + let time = jiff::Timestamp::now(); + Self { + status: "up", + time, + uptime: (time - state.started_at) + .total(jiff::Unit::Second) + .unwrap_or_default(), + proc: Process::new(), + supported_features: state.machines.get_supported_features(), + build_count, + jobset_count, + step_count, + runnable_count, + queue_stats, + queue_checks_started: state.metrics.queue_checks_started.get(), + queue_build_loads: state.metrics.queue_build_loads.get(), + queue_steps_created: state.metrics.queue_steps_created.get(), + queue_checks_early_exits: state.metrics.queue_checks_early_exits.get(), + queue_checks_finished: state.metrics.queue_checks_finished.get(), + + dispatcher_time_spent_running: state.metrics.dispatcher_time_spent_running.get(), + dispatcher_time_spent_waiting: state.metrics.dispatcher_time_spent_waiting.get(), + + queue_monitor_time_spent_running: state.metrics.queue_monitor_time_spent_running.get(), + queue_monitor_time_spent_waiting: state.metrics.queue_monitor_time_spent_waiting.get(), + + nr_builds_read: state.metrics.nr_builds_read.get(), + build_read_time_ms: state.metrics.build_read_time_ms.get(), + nr_builds_unfinished: state.metrics.nr_builds_unfinished.get(), + nr_builds_done: state.metrics.nr_builds_done.get(), + nr_builds_succeeded: state.metrics.nr_builds_succeeded.get(), + nr_builds_failed: state.metrics.nr_builds_failed.get(), + nr_steps_started: state.metrics.nr_steps_started.get(), + nr_steps_done: state.metrics.nr_steps_done.get(), + nr_steps_building: state.metrics.nr_steps_building.get(), + nr_steps_waiting: state.metrics.nr_steps_waiting.get(), + nr_steps_runnable: state.metrics.nr_steps_runnable.get(), + nr_steps_unfinished: state.metrics.nr_steps_unfinished.get(), + nr_unsupported_steps: state.metrics.nr_unsupported_steps.get(), + nr_unsupported_steps_aborted: state.metrics.nr_unsupported_steps_aborted.get(), + nr_substitutes_started: state.metrics.nr_substitutes_started.get(), + nr_substitutes_failed: state.metrics.nr_substitutes_failed.get(), + nr_substitutes_succeeded: state.metrics.nr_substitutes_succeeded.get(), + nr_retries: state.metrics.nr_retries.get(), + max_nr_retries: state.metrics.max_nr_retries.get(), + avg_step_time_ms: state.metrics.avg_step_time_ms.get(), + avg_step_import_time_ms: state.metrics.avg_step_import_time_ms.get(), + avg_step_build_time_ms: state.metrics.avg_step_build_time_ms.get(), + avg_step_upload_time_ms: state.metrics.avg_step_upload_time_ms.get(), + total_step_time_ms: state.metrics.total_step_time_ms.get(), + total_step_import_time_ms: state.metrics.total_step_import_time_ms.get(), + total_step_build_time_ms: state.metrics.total_step_build_time_ms.get(), + total_step_upload_time_ms: state.metrics.total_step_upload_time_ms.get(), + nr_queue_wakeups: state.metrics.nr_queue_wakeups.get(), + nr_dispatcher_wakeups: state.metrics.nr_dispatcher_wakeups.get(), + dispatch_time_ms: state.metrics.dispatch_time_ms.get(), + machines_total: state.metrics.machines_total.get(), + machines_in_use: state.metrics.machines_in_use.get(), + } + } +} diff --git a/src/queue-runner/src/io/response_types.rs b/src/queue-runner/src/io/response_types.rs new file mode 100644 index 000000000..ada81c2d7 --- /dev/null +++ b/src/queue-runner/src/io/response_types.rs @@ -0,0 +1,146 @@ +use hashbrown::HashMap; + +use nix_utils::BaseStore as _; + +use super::{ + Build, Jobset, Machine, QueueRunnerStats, Step, StepInfo, stats::S3Stats, stats::StoreStats, +}; + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MachinesResponse { + machines: HashMap, + machines_count: usize, +} + +impl MachinesResponse { + #[must_use] + pub fn new(machines: HashMap) -> Self { + Self { + machines_count: machines.len(), + machines, + } + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DumpResponse { + queue_runner: QueueRunnerStats, + machines: HashMap, + jobsets: HashMap, + store: Option, + s3: HashMap, +} + +impl DumpResponse { + #[must_use] + pub fn new( + queue_runner: QueueRunnerStats, + machines: HashMap, + jobsets: HashMap, + local_store: &nix_utils::LocalStore, + remote_stores: &[binary_cache::S3BinaryCacheClient], + ) -> Self { + let store = local_store + .get_store_stats() + .map_or(None, |s| Some(StoreStats::new(&s))); + + Self { + queue_runner, + machines, + jobsets, + store, + s3: remote_stores + .iter() + .map(|s| { + ( + s.cfg.client_config.bucket.clone(), + S3Stats::new(&s.s3_stats()), + ) + }) + .collect(), + } + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct JobsetsResponse { + jobsets: HashMap, + jobset_count: usize, +} + +impl JobsetsResponse { + #[must_use] + pub fn new(jobsets: HashMap) -> Self { + Self { + jobset_count: jobsets.len(), + jobsets, + } + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BuildsResponse { + builds: Vec, + build_count: usize, +} + +impl BuildsResponse { + #[must_use] + pub const fn new(builds: Vec) -> Self { + Self { + build_count: builds.len(), + builds, + } + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StepsResponse { + steps: Vec, + step_count: usize, +} + +impl StepsResponse { + #[must_use] + pub const fn new(steps: Vec) -> Self { + Self { + step_count: steps.len(), + steps, + } + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct QueueResponse { + queues: HashMap>, +} + +impl QueueResponse { + #[must_use] + pub const fn new(queues: HashMap>) -> Self { + Self { queues } + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StepInfoResponse { + steps: Vec, + step_count: usize, +} + +impl StepInfoResponse { + #[must_use] + pub const fn new(steps: Vec) -> Self { + Self { + step_count: steps.len(), + steps, + } + } +} diff --git a/src/queue-runner/src/io/stats.rs b/src/queue-runner/src/io/stats.rs new file mode 100644 index 000000000..242082986 --- /dev/null +++ b/src/queue-runner/src/io/stats.rs @@ -0,0 +1,287 @@ +use anyhow::Context as _; + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BuildQueueStats { + active_runnable: u64, + total_runnable: u64, + nr_runnable_waiting: u64, + nr_runnable_disabled: u64, + avg_runnable_time: u64, + wait_time_ms: u64, +} + +impl From for BuildQueueStats { + fn from(v: crate::state::BuildQueueStats) -> Self { + Self { + active_runnable: v.active_runnable, + total_runnable: v.total_runnable, + nr_runnable_waiting: v.nr_runnable_waiting, + nr_runnable_disabled: v.nr_runnable_disabled, + avg_runnable_time: v.avg_runnable_time, + wait_time_ms: v.wait_time, + } + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +#[allow(clippy::struct_field_names)] +pub struct MemoryStats { + current_bytes: u64, + peak_bytes: u64, + swap_current_bytes: u64, + zswap_current_bytes: u64, +} + +impl MemoryStats { + #[tracing::instrument(err)] + fn new(cgroups_path: &std::path::Path) -> anyhow::Result { + Ok(Self { + current_bytes: fs_err::read_to_string(cgroups_path.join("memory.current"))? + .trim() + .parse() + .context("memory current parsing failed")?, + peak_bytes: fs_err::read_to_string(cgroups_path.join("memory.peak"))? + .trim() + .parse() + .context("memory peak parsing failed")?, + swap_current_bytes: fs_err::read_to_string(cgroups_path.join("memory.swap.current"))? + .trim() + .parse() + .context("swap parsing failed")?, + zswap_current_bytes: fs_err::read_to_string(cgroups_path.join("memory.zswap.current"))? + .trim() + .parse() + .context("zswap parsing failed")?, + }) + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct IoStats { + total_read_bytes: u64, + total_write_bytes: u64, +} + +impl IoStats { + #[tracing::instrument(err)] + fn new(cgroups_path: &std::path::Path) -> anyhow::Result { + let mut total_read_bytes: u64 = 0; + let mut total_write_bytes: u64 = 0; + + let contents = fs_err::read_to_string(cgroups_path.join("io.stat"))?; + for line in contents.lines() { + for part in line.split_whitespace() { + if part.starts_with("rbytes=") { + total_read_bytes += part + .split('=') + .nth(1) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + } else if part.starts_with("wbytes=") { + total_write_bytes += part + .split('=') + .nth(1) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + } + } + } + + Ok(Self { + total_read_bytes, + total_write_bytes, + }) + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +#[allow(clippy::struct_field_names)] +pub struct CpuStats { + usage_usec: u128, + user_usec: u128, + system_usec: u128, +} + +impl CpuStats { + #[tracing::instrument(err)] + fn new(cgroups_path: &std::path::Path) -> anyhow::Result { + let contents = fs_err::read_to_string(cgroups_path.join("cpu.stat"))?; + + let mut usage_usec: u128 = 0; + let mut user_usec: u128 = 0; + let mut system_usec: u128 = 0; + + for line in contents.lines() { + if line.starts_with("usage_usec") { + usage_usec = line + .split_whitespace() + .nth(1) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + } else if line.starts_with("user_usec") { + user_usec = line + .split_whitespace() + .nth(1) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + } else if line.starts_with("system_usec") { + system_usec = line + .split_whitespace() + .nth(1) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + } + } + Ok(Self { + usage_usec, + user_usec, + system_usec, + }) + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CgroupStats { + memory: MemoryStats, + io: IoStats, + cpu: CpuStats, +} + +impl CgroupStats { + #[tracing::instrument(err)] + fn new(me: &procfs::process::Process) -> anyhow::Result { + let cgroups_pathname = format!( + "/sys/fs/cgroup/{}", + me.cgroups()? + .0 + .first() + .ok_or_else(|| anyhow::anyhow!("cgroup information is missing in process."))? + .pathname + ); + let cgroups_path = std::path::Path::new(&cgroups_pathname); + if !cgroups_path.exists() { + return Err(anyhow::anyhow!("cgroups directory does not exists.")); + } + + Ok(Self { + memory: MemoryStats::new(cgroups_path)?, + io: IoStats::new(cgroups_path)?, + cpu: CpuStats::new(cgroups_path)?, + }) + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Process { + pid: i32, + vsize_bytes: u64, + rss_bytes: u64, + shared_bytes: u64, + cgroup: Option, +} + +impl Process { + pub fn new() -> Option { + let me = procfs::process::Process::myself().ok()?; + let page_size = procfs::page_size(); + let statm = me.statm().ok()?; + let vsize = statm.size * page_size; + let rss = statm.resident * page_size; + let shared = statm.shared * page_size; + Some(Self { + pid: me.pid, + vsize_bytes: vsize, + rss_bytes: rss, + shared_bytes: shared, + cgroup: match CgroupStats::new(&me) { + Ok(v) => Some(v), + Err(e) => { + tracing::error!("failed to cgroups stats: {e}"); + None + } + }, + }) + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StoreStats { + nar_info_read: u64, + nar_info_read_averted: u64, + nar_info_missing: u64, + nar_info_write: u64, + path_info_cache_size: u64, + nar_read: u64, + nar_read_bytes: u64, + nar_read_compressed_bytes: u64, + nar_write: u64, + nar_write_averted: u64, + nar_write_bytes: u64, + nar_write_compressed_bytes: u64, + nar_write_compression_time_ms: u64, + nar_compression_savings: f64, + nar_compression_speed: f64, +} + +impl StoreStats { + #[must_use] + pub fn new(v: &nix_utils::StoreStats) -> Self { + Self { + nar_info_read: v.nar_info_read, + nar_info_read_averted: v.nar_info_read_averted, + nar_info_missing: v.nar_info_missing, + nar_info_write: v.nar_info_write, + path_info_cache_size: v.path_info_cache_size, + nar_read: v.nar_read, + nar_read_bytes: v.nar_read_bytes, + nar_read_compressed_bytes: v.nar_read_compressed_bytes, + nar_write: v.nar_write, + nar_write_averted: v.nar_write_averted, + nar_write_bytes: v.nar_write_bytes, + nar_write_compressed_bytes: v.nar_write_compressed_bytes, + nar_write_compression_time_ms: v.nar_write_compression_time_ms, + nar_compression_savings: v.nar_compression_savings(), + nar_compression_speed: v.nar_compression_speed(), + } + } +} + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct S3Stats { + put: u64, + put_bytes: u64, + put_time_ms: u64, + put_speed: f64, + get: u64, + get_bytes: u64, + get_time_ms: u64, + get_speed: f64, + head: u64, + cost_dollar_approx: f64, +} + +impl S3Stats { + #[must_use] + pub fn new(v: &binary_cache::S3Stats) -> Self { + Self { + put: v.put, + put_bytes: v.put_bytes, + put_time_ms: v.put_time_ms, + put_speed: v.put_speed(), + get: v.get, + get_bytes: v.get_bytes, + get_time_ms: v.get_time_ms, + get_speed: v.get_speed(), + head: v.head, + cost_dollar_approx: v.cost_dollar_approx(), + } + } +} diff --git a/src/queue-runner/src/io/step.rs b/src/queue-runner/src/io/step.rs new file mode 100644 index 000000000..276db54ae --- /dev/null +++ b/src/queue-runner/src/io/step.rs @@ -0,0 +1,42 @@ +use std::sync::atomic::Ordering; + +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +#[allow(clippy::struct_excessive_bools)] +pub struct Step { + drv_path: nix_utils::StorePath, + runnable: bool, + finished: bool, + previous_failure: bool, + + created: bool, + tries: u32, + highest_global_priority: i32, + highest_local_priority: i32, + + lowest_build_id: db::models::BuildID, + deps_count: u64, +} + +impl From> for Step { + fn from(item: std::sync::Arc) -> Self { + Self { + drv_path: item.get_drv_path().clone(), + runnable: item.get_runnable(), + finished: item.get_finished(), + previous_failure: item.get_previous_failure(), + created: item.atomic_state.get_created(), + tries: item.atomic_state.tries.load(Ordering::Relaxed), + highest_global_priority: item + .atomic_state + .highest_global_priority + .load(Ordering::Relaxed), + highest_local_priority: item + .atomic_state + .highest_local_priority + .load(Ordering::Relaxed), + lowest_build_id: item.atomic_state.lowest_build_id.load(Ordering::Relaxed), + deps_count: item.get_deps_size(), + } + } +} diff --git a/src/queue-runner/src/io/step_info.rs b/src/queue-runner/src/io/step_info.rs new file mode 100644 index 000000000..8598a9adf --- /dev/null +++ b/src/queue-runner/src/io/step_info.rs @@ -0,0 +1,38 @@ +use std::sync::atomic::Ordering; + +#[allow(clippy::struct_excessive_bools)] +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct StepInfo { + drv_path: nix_utils::StorePath, + already_scheduled: bool, + runnable: bool, + finished: bool, + cancelled: bool, + runnable_since: jiff::Timestamp, + + tries: u32, + + lowest_share_used: f64, + highest_global_priority: i32, + highest_local_priority: i32, + lowest_build_id: db::models::BuildID, +} + +impl From> for StepInfo { + fn from(item: std::sync::Arc) -> Self { + Self { + drv_path: item.step.get_drv_path().clone(), + already_scheduled: item.get_already_scheduled(), + runnable: item.step.get_runnable(), + finished: item.step.get_finished(), + cancelled: item.get_cancelled(), + runnable_since: item.runnable_since, + tries: item.step.atomic_state.tries.load(Ordering::Relaxed), + lowest_share_used: item.get_lowest_share_used(), + highest_global_priority: item.get_highest_global_priority(), + highest_local_priority: item.get_highest_local_priority(), + lowest_build_id: item.get_lowest_build_id(), + } + } +} diff --git a/src/queue-runner/src/lib.rs b/src/queue-runner/src/lib.rs new file mode 100644 index 000000000..1a1242ec0 --- /dev/null +++ b/src/queue-runner/src/lib.rs @@ -0,0 +1,12 @@ +#![deny(clippy::all)] +#![deny(clippy::pedantic)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::expect_used)] +#![allow(clippy::missing_errors_doc)] +#![recursion_limit = "256"] + +pub mod config; +pub mod io; +pub mod server; +pub mod state; +pub mod utils; diff --git a/src/queue-runner/src/lock_file.rs b/src/queue-runner/src/lock_file.rs new file mode 100644 index 000000000..e40ec36b6 --- /dev/null +++ b/src/queue-runner/src/lock_file.rs @@ -0,0 +1,23 @@ +pub(crate) struct LockFile { + path: std::path::PathBuf, + file: fs_err::File, +} + +impl LockFile { + pub(crate) fn acquire(path: impl Into) -> std::io::Result { + let path = path.into(); + if let Some(parent) = path.parent() { + fs_err::create_dir_all(parent)?; + } + let file = fs_err::File::create(&path)?; + file.try_lock()?; + Ok(Self { path, file }) + } +} + +impl Drop for LockFile { + fn drop(&mut self) { + let _ = self.file.unlock(); + let _ = fs_err::remove_file(&self.path); + } +} diff --git a/src/queue-runner/src/main.rs b/src/queue-runner/src/main.rs new file mode 100644 index 000000000..6fa67db89 --- /dev/null +++ b/src/queue-runner/src/main.rs @@ -0,0 +1,159 @@ +#![deny(clippy::all)] +#![deny(clippy::pedantic)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::expect_used)] +#![allow(clippy::missing_errors_doc)] +#![recursion_limit = "256"] + +pub mod config; +pub mod io; +pub mod lock_file; +pub mod server; +pub mod state; +pub mod utils; + +use anyhow::Context as _; + +use state::State; + +#[cfg(not(target_env = "msvc"))] +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +fn start_task_loops(state: &std::sync::Arc) -> Vec { + tracing::info!("QueueRunner starting task loops"); + + let mut service_list = vec![ + spawn_config_reloader(state.clone(), state.config.clone(), &state.cli.config_path), + state.clone().start_queue_monitor_loop(), + state.clone().start_dispatch_loop(), + state.clone().start_dump_status_loop(), + state.clone().start_uploader_queue(), + ]; + if let Some(fod_checker) = &state.fod_checker { + service_list.push(fod_checker.clone().start_traverse_loop(state.store.clone())); + } + + service_list +} + +fn spawn_config_reloader( + state: std::sync::Arc, + current_config: config::App, + filepath: &str, +) -> tokio::task::AbortHandle { + let filepath = filepath.to_owned(); + let task = tokio::spawn(async move { + loop { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup()) { + Ok(mut s) => { + let _ = s.recv().await; + tracing::info!("Reloading..."); + config::reload(¤t_config, &filepath, &state).await; + } + Err(e) => { + tracing::error!("Failed to create signal listener for SIGHUP: {e}"); + break; + } + } + } + }); + task.abort_handle() +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let tracing_guard = hydra_tracing::init()?; + + #[cfg(debug_assertions)] + { + // If we have a debug build we want to crash on a panic, because we use some debug_asserts, + // and that helps validating those! + let default_panic = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + default_panic(info); + std::process::exit(1); + })); + } + + nix_utils::init_nix(); + let state = State::new(&tracing_guard).await?; + if state.cli.status { + state.get_status_from_main_process().await?; + return Ok(()); + } + + let lockfile_path = state.config.get_lockfile(); + let _lock = lock_file::LockFile::acquire(&lockfile_path) + .context("Another instance is already running.")?; + + if !state.cli.mtls_configured_correctly() { + tracing::error!( + "mtls configured inproperly, please pass all options: server_cert_path, server_key_path and client_ca_cert_path!" + ); + return Err(anyhow::anyhow!("Configuration issue")); + } + + let task_abort_handles = start_task_loops(&state); + tracing::info!( + "QueueRunner listening on grpc: {:?} and rest: {}", + state.cli.grpc_bind, + state.cli.rest_bind + ); + let srv1 = server::grpc::Server::run(state.cli.grpc_bind.clone(), state.clone()); + let srv2 = server::http::Server::run(state.cli.rest_bind, state.clone()); + + let task = tokio::spawn(async move { + match futures_util::future::join(srv1, srv2).await { + (Ok(()), Ok(())) => Ok(()), + (Ok(()), Err(e)) => Err(anyhow::anyhow!("hyper error while awaiting handle: {e}")), + (Err(e), Ok(())) => Err(anyhow::anyhow!("tonic error while awaiting handle: {e}")), + (Err(e1), Err(e2)) => Err(anyhow::anyhow!( + "tonic and hyper error while awaiting handle: {e1} | {e2}" + )), + } + }); + + let _notify = sd_notify::notify( + false, + &[ + sd_notify::NotifyState::Status("Running"), + sd_notify::NotifyState::Ready, + ], + ); + + let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?; + let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + + let abort_handle = task.abort_handle(); + tokio::select! { + _ = sigint.recv() => { + tracing::info!("Received sigint - shutting down gracefully"); + let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping]); + abort_handle.abort(); + for h in task_abort_handles { + h.abort(); + } + // removing all machines will also mark all currently running jobs as cancelled + state.remove_all_machines().await; + let _ = state.clear_busy().await; + Ok(()) + } + _ = sigterm.recv() => { + tracing::info!("Received sigterm - shutting down gracefully"); + let _ = sd_notify::notify(false, &[sd_notify::NotifyState::Stopping]); + abort_handle.abort(); + for h in task_abort_handles { + h.abort(); + } + // removing all machines will also mark all currently running jobs as cancelled + state.remove_all_machines().await; + let _ = state.clear_busy().await; + Ok(()) + } + r = task => { + r??; + Ok(()) + } + } +} diff --git a/src/queue-runner/src/server/grpc.rs b/src/queue-runner/src/server/grpc.rs new file mode 100644 index 000000000..afb5754ef --- /dev/null +++ b/src/queue-runner/src/server/grpc.rs @@ -0,0 +1,763 @@ +use std::sync::Arc; + +use anyhow::Context as _; +use tokio::{io::AsyncWriteExt as _, sync::mpsc}; +use tonic::service::interceptor::InterceptedService; +use tower::ServiceBuilder; +use tracing::Instrument as _; + +use crate::{ + config::BindSocket, + server::grpc::runner_v1::{BuildResultState, StepUpdate}, + state::{Machine, MachineMessage, State}, +}; +use nix_utils::BaseStore as _; + +include!(concat!(env!("OUT_DIR"), "/proto_version.rs")); +use runner_v1::runner_service_server::{RunnerService, RunnerServiceServer}; +use runner_v1::{ + BuildResultInfo, BuilderRequest, FetchRequisitesRequest, JoinResponse, LogChunk, NarData, + PresignedUploadComplete, PresignedUrlRequest, PresignedUrlResponse, RunnerRequest, + SimplePingMessage, StorePath, StorePaths, VersionCheckRequest, VersionCheckResponse, + builder_request, +}; + +type BuilderResult = Result, tonic::Status>; +type OpenTunnelResponseStream = + std::pin::Pin> + Send>>; +type StreamFileResponseStream = + std::pin::Pin> + Send>>; + +// there is no reason to make this configurable, it only exists so we ensure the channel is not +// closed. we dont use this to write any actual information. +const BACKWARDS_PING_INTERVAL: u64 = 30; + +pub mod runner_v1 { + // We need to allow pedantic here because of generated code + #![allow(clippy::pedantic)] + + tonic::include_proto!("runner.v1"); + + pub(crate) const FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("streaming_descriptor"); + + impl From for db::models::StepStatus { + fn from(item: StepStatus) -> Self { + match item { + StepStatus::Preparing => Self::Preparing, + StepStatus::Connecting => Self::Connecting, + StepStatus::SeningInputs => Self::SendingInputs, + StepStatus::Building => Self::Building, + StepStatus::WaitingForLocalSlot => Self::WaitingForLocalSlot, + StepStatus::ReceivingOutputs => Self::ReceivingOutputs, + StepStatus::PostProcessing => Self::PostProcessing, + } + } + } +} + +fn match_for_io_error(err_status: &tonic::Status) -> Option<&std::io::Error> { + let mut err: &(dyn std::error::Error + 'static) = err_status; + + loop { + if let Some(io_err) = err.downcast_ref::() { + return Some(io_err); + } + + // h2::Error do not expose std::io::Error with `source()` + // https://github.com/hyperium/h2/pull/462 + if let Some(h2_err) = err.downcast_ref::() + && let Some(io_err) = h2_err.get_io() + { + return Some(io_err); + } + + err = err.source()?; + } +} + +#[tracing::instrument(skip(state, msg))] +fn handle_message(state: &Arc, msg: builder_request::Message) { + match msg { + // at this point in time, builder already joined, so this message can be ignored + builder_request::Message::Join(_) => (), + builder_request::Message::Ping(msg) => { + tracing::debug!("new ping: {msg:?}"); + let Ok(machine_id) = uuid::Uuid::parse_str(&msg.machine_id) else { + return; + }; + if let Some(m) = state.machines.get_machine_by_id(machine_id) { + m.stats.store_ping(&msg); + } + } + } +} + +#[derive(Debug, Clone)] +pub struct CheckAuthInterceptor { + config: crate::config::App, +} + +impl tonic::service::Interceptor for CheckAuthInterceptor { + fn call(&mut self, req: tonic::Request<()>) -> Result, tonic::Status> { + if self.config.has_token_list() { + match req.metadata().get("authorization") { + Some(t) + if self.config.check_if_contains_token( + t.to_str() + .map_err(|_| tonic::Status::unauthenticated("No valid auth token"))? + .strip_prefix("Bearer ") + .ok_or_else(|| tonic::Status::unauthenticated("No valid auth token"))?, + ) => + { + Ok(req) + } + _ => Err(tonic::Status::unauthenticated("No valid auth token")), + } + } else { + Ok(req) + } + } +} + +pub struct Server { + state: Arc, +} + +impl Server { + #[tracing::instrument(skip(state), err)] + pub async fn run(addr: BindSocket, state: Arc) -> anyhow::Result<()> { + let service = RunnerServiceServer::new(Self { + state: state.clone(), + }) + .send_compressed(tonic::codec::CompressionEncoding::Zstd) + .accept_compressed(tonic::codec::CompressionEncoding::Zstd) + .max_decoding_message_size(50 * 1024 * 1024) + .max_encoding_message_size(50 * 1024 * 1024); + let intercepted_service = InterceptedService::new( + service, + CheckAuthInterceptor { + config: state.config.clone(), + }, + ); + + let mut server = tonic::transport::Server::builder().layer( + ServiceBuilder::new() + .layer( + tower_http::trace::TraceLayer::new_for_grpc().make_span_with( + tower_http::trace::DefaultMakeSpan::new() + .level(tracing::Level::INFO) + .include_headers(false), + ), + ) + .map_request(hydra_tracing::propagate::accept_trace), + ); + + if state.cli.mtls_enabled() { + tracing::info!("Using mtls"); + let (client_ca_cert, server_identity) = state + .cli + .get_mtls() + .await + .context("Failed to get_mtls Certificate and Identity")?; + + let tls = tonic::transport::ServerTlsConfig::new() + .identity(server_identity) + .client_ca_root(client_ca_cert); + server = server.tls_config(tls)?; + } + let reflection_service = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set(runner_v1::FILE_DESCRIPTOR_SET) + .build_v1()?; + + let (_health_reporter, health_service) = tonic_health::server::health_reporter(); + let server = server + .add_service(health_service) + .add_service(reflection_service) + .add_service(intercepted_service); + + match addr { + BindSocket::Tcp(s) => server.serve(s).await?, + BindSocket::Unix(p) => { + let uds = tokio::net::UnixListener::bind(p)?; + let uds_stream = tokio_stream::wrappers::UnixListenerStream::new(uds); + server.serve_with_incoming(uds_stream).await?; + } + BindSocket::ListenFd => { + let listener = listenfd::ListenFd::from_env() + .take_unix_listener(0)? + .ok_or_else(|| anyhow::anyhow!("No listenfd found in env"))?; + listener.set_nonblocking(true)?; + let listener = tokio_stream::wrappers::UnixListenerStream::new( + tokio::net::UnixListener::from_std(listener)?, + ); + + server.serve_with_incoming(listener).await?; + } + } + + Ok(()) + } +} + +#[tonic::async_trait] +impl RunnerService for Server { + type OpenTunnelStream = OpenTunnelResponseStream; + type StreamFileStream = StreamFileResponseStream; + type StreamFilesStream = StreamFileResponseStream; + + #[tracing::instrument(skip(self, req), err)] + async fn check_version( + &self, + req: tonic::Request, + ) -> BuilderResult { + let req = req.into_inner(); + let server_version = PROTO_API_VERSION; + + if req.version == server_version { + tracing::info!( + "Version check passed: machine_id={}, hostname={}, client={}, server={}", + req.machine_id, + req.hostname, + req.version, + server_version + ); + Ok(tonic::Response::new(VersionCheckResponse { + compatible: true, + server_version: server_version.to_string(), + })) + } else { + tracing::warn!( + "Version check failed: machine_id={}, hostname={}, client={}, server={}", + req.machine_id, + req.hostname, + req.version, + server_version + ); + Ok(tonic::Response::new(VersionCheckResponse { + compatible: false, + server_version: server_version.to_string(), + })) + } + } + + #[tracing::instrument(skip(self, req), err)] + async fn open_tunnel( + &self, + req: tonic::Request>, + ) -> BuilderResult { + use tokio_stream::StreamExt as _; + + let mut stream = req.into_inner(); + let (input_tx, mut input_rx) = mpsc::channel::(128); + let use_presigned_uploads = self.state.config.use_presigned_uploads(); + let forced_substituters = self.state.config.get_forced_substituters(); + let machine = match stream.next().await { + Some(Ok(m)) => match m.message { + Some(runner_v1::builder_request::Message::Join(v)) => { + match Machine::new(v, input_tx, use_presigned_uploads, &forced_substituters) { + Ok(m) => Some(m), + Err(e) => { + tracing::error!("Rejecting new machine creation: {e}"); + return Err(tonic::Status::invalid_argument("Machine is not valid")); + } + } + } + _ => None, + }, + Some(Err(e)) => { + tracing::error!("Bad message in stream: {e}"); + None + } + _ => None, + }; + let Some(machine) = machine else { + return Err(tonic::Status::invalid_argument("No Ping message was sent")); + }; + + let state = self.state.clone(); + let machine_id = state.insert_machine(machine.clone()).await; + tracing::info!("Registered new machine: machine_id={machine_id} machine={machine}",); + + let (output_tx, output_rx) = mpsc::channel(128); + if let Err(e) = output_tx + .send(Ok(RunnerRequest { + message: Some(runner_v1::runner_request::Message::Join(JoinResponse { + machine_id: machine_id.to_string(), + max_concurrent_downloads: state.config.get_max_concurrent_downloads(), + })), + })) + .await + { + tracing::error!("Failed to send join response machine_id={machine_id} e={e}"); + return Err(tonic::Status::internal("Failed to send join Response.")); + } + + let mut ping_interval = + tokio::time::interval(std::time::Duration::from_secs(BACKWARDS_PING_INTERVAL)); + tokio::spawn(async move { + loop { + tokio::select! { + _ = ping_interval.tick() => { + let msg = RunnerRequest { + message: Some(runner_v1::runner_request::Message::Ping(SimplePingMessage { + message: "ping".into(), + })) + }; + if let Err(e) = output_tx.send(Ok(msg)).await { + tracing::error!("Failed to send message to machine={machine_id} e={e}"); + state.remove_machine(machine_id).await; + break + } + }, + msg = input_rx.recv() => { + if let Some(msg) = msg { + if let Err(e) = output_tx.send(Ok(msg.into_request())).await { + tracing::error!("Failed to send message to machine={machine_id} e={e}"); + state.remove_machine(machine_id).await; + break + } + } else { + state.remove_machine(machine_id).await; + break + } + }, + msg = stream.next() => match msg.map(|v| v.map(|v| v.message)) { + Some(Ok(Some(msg))) => handle_message(&state, msg), + Some(Ok(None)) => (), // empty meesage can be ignored + Some(Err(err)) => { + if let Some(io_err) = match_for_io_error(&err) + && io_err.kind() == std::io::ErrorKind::BrokenPipe { + tracing::error!("client disconnected: broken pipe: machine={machine_id} hostname={}", machine.hostname); + state.remove_machine(machine_id).await; + break; + } + + match output_tx.send(Err(err)).await { + Ok(()) => (), + Err(_err) => { + state.remove_machine(machine_id).await; + break + } + } + }, + None => { + state.remove_machine(machine_id).await; + break + } + } + } + } + }); + + Ok(tonic::Response::new( + Box::pin(tokio_stream::wrappers::ReceiverStream::new(output_rx)) + as Self::OpenTunnelStream, + )) + } + + #[tracing::instrument(skip(self, req), err)] + async fn build_log( + &self, + req: tonic::Request>, + ) -> BuilderResult { + use tokio_stream::StreamExt as _; + + let mut stream = req.into_inner(); + let state = self.state.clone(); + + let mut out_file: Option = None; + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + + if let Some(ref mut file) = out_file { + file.write_all(&chunk.data).await?; + } else { + let mut file = state + .new_log_file(&nix_utils::StorePath::new(&chunk.drv)) + .await + .map_err(|_| tonic::Status::internal("Failed to create log file."))?; + file.write_all(&chunk.data).await?; + out_file = Some(file); + } + } + + Ok(tonic::Response::new(runner_v1::Empty {})) + } + + #[tracing::instrument(skip(self, req), err)] + async fn build_result( + &self, + req: tonic::Request>, + ) -> BuilderResult { + let stream = req.into_inner(); + + // We leak memory if we use the store from state, so we open and close a new + // connection for each import. This sucks but using the state.store will result in the path + // not being closed! + { + let store = nix_utils::LocalStore::init(); + store + .import_paths( + tokio_stream::StreamExt::map(stream, |s| { + s.map(|v| v.chunk.into()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::UnexpectedEof, e)) + }), + false, + ) + .await + } + .map_err(|_| tonic::Status::internal("Failed to import path."))?; + Ok(tonic::Response::new(runner_v1::Empty {})) + } + + #[tracing::instrument(skip(self), err)] + async fn build_step_update( + &self, + req: tonic::Request, + ) -> BuilderResult { + let state = self.state.clone(); + + let req = req.into_inner(); + let build_id = uuid::Uuid::parse_str(&req.build_id).map_err(|e| { + tracing::error!("Failed to parse build_id into uuid: {e}"); + tonic::Status::invalid_argument("build_id is not a valid uuid.") + })?; + let machine_id = uuid::Uuid::parse_str(&req.machine_id).map_err(|e| { + tracing::error!("Failed to parse machine_id into uuid: {e}"); + tonic::Status::invalid_argument("machine_id is not a valid uuid.") + })?; + let step_status = db::models::StepStatus::from(req.step_status()); + + tokio::spawn({ + async move { + if let Err(e) = state + .update_build_step( + build_id, + machine_id, + step_status, + ) + .await + { + tracing::error!( + "Failed to update build step with build_id={build_id:?} step_status={step_status:?}: {e}" + ); + } + }.in_current_span() + }); + + Ok(tonic::Response::new(runner_v1::Empty {})) + } + + #[tracing::instrument(skip(self, req), fields(machine_id=req.get_ref().machine_id, build_id=req.get_ref().build_id), err)] + async fn complete_build( + &self, + req: tonic::Request, + ) -> BuilderResult { + let state = self.state.clone(); + + let req = req.into_inner(); + let build_id = uuid::Uuid::parse_str(&req.build_id).map_err(|e| { + tracing::error!("Failed to parse build_id into uuid: {e}"); + tonic::Status::invalid_argument("build_id is not a valid uuid.") + })?; + let machine_id = uuid::Uuid::parse_str(&req.machine_id).map_err(|e| { + tracing::error!("Failed to parse machine_id into uuid: {e}"); + tonic::Status::invalid_argument("machine_id is not a valid uuid.") + })?; + + tokio::spawn({ + async move { + if req.result_state() == BuildResultState::Success { + let build_output = crate::state::BuildOutput::from(req); + if let Err(e) = state + .succeed_step_by_uuid(build_id, machine_id, build_output) + .await + { + tracing::error!( + "Failed to mark step with build_id={build_id} as done: {e}" + ); + } + } else if let Err(e) = state + .fail_step_by_uuid( + build_id, + machine_id, + req.result_state().into(), + crate::state::BuildTimings::new( + req.import_time_ms, + req.build_time_ms, + req.upload_time_ms, + ), + ) + .await + { + tracing::error!("Failed to fail step with build_id={build_id}: {e}"); + } + } + .in_current_span() + }); + + Ok(tonic::Response::new(runner_v1::Empty {})) + } + + #[tracing::instrument(skip(self, req), err)] + async fn fetch_drv_requisites( + &self, + req: tonic::Request, + ) -> BuilderResult { + let state = self.state.clone(); + let req = req.into_inner(); + let drv = nix_utils::StorePath::new(&req.path); + + let requisites = state + .store + .query_requisites(&[&drv], req.include_outputs) + .await + .map_err(|e| { + tracing::error!("failed to toposort drv e={e}"); + tonic::Status::internal("failed to toposort drv.") + })? + .into_iter() + .map(nix_utils::StorePath::into_base_name) + .collect(); + + Ok(tonic::Response::new(runner_v1::DrvRequisitesMessage { + requisites, + })) + } + + #[tracing::instrument(skip(self, req), err)] + async fn has_path( + &self, + req: tonic::Request, + ) -> BuilderResult { + let path = nix_utils::StorePath::new(&req.into_inner().path); + let state = self.state.clone(); + let has_path = state.store.is_valid_path(&path).await; + + Ok(tonic::Response::new(runner_v1::HasPathResponse { + has_path, + })) + } + + #[tracing::instrument(skip(self, req), err)] + async fn stream_file( + &self, + req: tonic::Request, + ) -> BuilderResult { + let path = nix_utils::StorePath::new(&req.into_inner().path); + let store = nix_utils::LocalStore::init(); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::>(); + + let closure = move |data: &[u8]| { + let data = Vec::from(data); + tx.send(Ok(NarData { chunk: data })).is_ok() + }; + + tokio::task::spawn(async move { + let _ = store.export_paths(&[path], closure); + }); + + Ok(tonic::Response::new( + Box::pin(tokio_stream::wrappers::UnboundedReceiverStream::new(rx)) + as Self::StreamFileStream, + )) + } + + #[tracing::instrument(skip(self, req), err)] + async fn stream_files( + &self, + req: tonic::Request, + ) -> BuilderResult { + let req = req.into_inner(); + let paths = req + .paths + .into_iter() + .map(|p| nix_utils::StorePath::new(&p)) + .collect::>(); + + let store = nix_utils::LocalStore::init(); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::>(); + + let closure = move |data: &[u8]| { + let data = Vec::from(data); + tx.send(Ok(NarData { chunk: data })).is_ok() + }; + + tokio::task::spawn(async move { + let _ = store.export_paths(&paths, closure.clone()); + }); + + Ok(tonic::Response::new( + Box::pin(tokio_stream::wrappers::UnboundedReceiverStream::new(rx)) + as Self::StreamFilesStream, + )) + } + + #[tracing::instrument( + skip(self, req), + fields( + build_id=req.get_ref().build_id, + machine_id=req.get_ref().machine_id + ), + err + )] + async fn request_presigned_url( + &self, + req: tonic::Request, + ) -> BuilderResult { + let _state = self.state.clone(); + let req = req.into_inner(); + + let _build_id = uuid::Uuid::parse_str(&req.build_id).map_err(|e| { + tracing::error!("Failed to parse build_id into uuid: {e}"); + tonic::Status::invalid_argument("build_id is not a valid uuid.") + })?; + let _machine_id = uuid::Uuid::parse_str(&req.machine_id).map_err(|e| { + tracing::error!("Failed to parse machine_id into uuid: {e}"); + tonic::Status::invalid_argument("machine_id is not a valid uuid.") + })?; + + let remote_store = { + let remote_stores = _state.remote_stores.read(); + remote_stores + .first() + .cloned() + .ok_or_else(|| tonic::Status::failed_precondition("No remote store configured"))? + }; + + let mut responses = Vec::new(); + for presigned_request in req.request { + let store_path = nix_utils::StorePath::new(&presigned_request.store_path); + + let presigned_response = remote_store + .generate_nar_upload_presigned_url( + &store_path, + &presigned_request.nar_hash, + presigned_request.debug_info_build_ids, + ) + .await + .map_err(|e| { + tracing::error!("Failed to generate presigned URL for {}: {e}", store_path); + tonic::Status::internal("Failed to generate presigned URL") + })?; + + responses.push(runner_v1::PresignedNarResponse { + store_path: store_path.base_name().to_owned(), + nar_url: presigned_response.nar_url, + nar_upload: Some(runner_v1::PresignedUpload { + compression_level: presigned_response.nar_upload.get_compression_level_as_i32(), + url: presigned_response.nar_upload.url, + path: presigned_response.nar_upload.path, + compression: presigned_response + .nar_upload + .compression + .as_str() + .to_owned(), + }), + ls_upload: presigned_response + .ls_upload + .map(|ls| runner_v1::PresignedUpload { + compression_level: ls.get_compression_level_as_i32(), + url: ls.url, + path: ls.path, + compression: ls.compression.as_str().to_owned(), + }), + debug_info_upload: presigned_response + .debug_info_upload + .into_iter() + .map(|p| runner_v1::PresignedUpload { + compression_level: p.get_compression_level_as_i32(), + url: p.url, + path: p.path, + compression: p.compression.as_str().to_owned(), + }) + .collect(), + }); + } + + tracing::debug!("Generated {} presigned URLs", responses.len()); + Ok(tonic::Response::new(PresignedUrlResponse { + inner: responses, + })) + } + + #[tracing::instrument( + skip(self, req), + fields( + build_id=req.get_ref().build_id, + machine_id=req.get_ref().machine_id, + store_path=req.get_ref().store_path + ), + err, + )] + async fn notify_presigned_upload_complete( + &self, + req: tonic::Request, + ) -> BuilderResult { + let state = self.state.clone(); + let req = req.into_inner(); + + let build_id = uuid::Uuid::parse_str(&req.build_id).map_err(|e| { + tracing::error!("Failed to parse build_id into uuid: {e}"); + tonic::Status::invalid_argument("build_id is not a valid uuid.") + })?; + let machine_id = uuid::Uuid::parse_str(&req.machine_id).map_err(|e| { + tracing::error!("Failed to parse machine_id into uuid: {e}"); + tonic::Status::invalid_argument("machine_id is not a valid uuid.") + })?; + + let machine = state + .machines + .get_machine_by_id(machine_id) + .ok_or_else(|| tonic::Status::not_found("Machine not found"))?; + let _job = machine + .get_job_drv_for_build_id(build_id) + .ok_or_else(|| tonic::Status::not_found("Job not found for this build_id"))?; + + let remote_store = { + let remote_stores = state.remote_stores.read(); + remote_stores + .first() + .cloned() + .ok_or_else(|| tonic::Status::failed_precondition("No remote store configured"))? + }; + + let narinfo = binary_cache::NarInfo { + store_path: nix_utils::StorePath::new(&req.store_path), + url: req.url.clone(), + compression: remote_store.cfg.compression, + file_hash: Some(req.file_hash), + file_size: Some(req.file_size), + nar_hash: req.nar_hash, + nar_size: req.nar_size, + references: req + .references + .into_iter() + .map(|p| nix_utils::StorePath::new(&p)) + .collect(), + deriver: req.deriver.map(|p| nix_utils::StorePath::new(&p)), + ca: req.ca, + sigs: vec![], + }; + let store_path = narinfo.store_path.clone(); + + let narinfo_url = remote_store + .upload_narinfo_after_presigned_upload(&self.state.store, narinfo) + .await + .map_err(|e| { + tracing::error!("Failed to upload narinfo for {}: {e}", store_path); + tonic::Status::internal("Failed to upload narinfo") + })?; + + tracing::debug!( + "Presigned upload completed and narinfo uploaded for path: {}, url: {}, size: {} bytes, narinfo: {}", + store_path, + req.url, + req.file_size, + narinfo_url + ); + + Ok(tonic::Response::new(runner_v1::Empty {})) + } +} diff --git a/src/queue-runner/src/server/http.rs b/src/queue-runner/src/server/http.rs new file mode 100644 index 000000000..15f3e0d8f --- /dev/null +++ b/src/queue-runner/src/server/http.rs @@ -0,0 +1,346 @@ +use std::{net::SocketAddr, sync::Arc}; + +use crate::state::State; +use bytes::Bytes; +use http_body_util::{BodyExt as _, Full, combinators::BoxBody}; +use tracing::Instrument as _; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("uuid error: `{0}`")] + Uuid(#[from] uuid::Error), + + #[error("serde json error: `{0}`")] + SerdeJson(#[from] serde_json::Error), + + #[error("hyper http error: `{0}`")] + HyperHttp(#[from] hyper::http::Error), + + #[error("hyper error: `{0}`")] + Hyper(#[from] hyper::Error), + + #[error("std io error: `{0}`")] + Io(#[from] std::io::Error), + + #[error("anyhow error: `{0}`")] + Anyhow(#[from] anyhow::Error), + + #[error("db error: `{0}`")] + Sqlx(#[from] db::Error), + + #[error("Not found")] + NotFound, + + #[error("Fatal")] + Fatal, +} + +impl Error { + #[must_use] + pub const fn get_status(&self) -> hyper::StatusCode { + match *self { + Self::Uuid(_) + | Self::SerdeJson(_) + | Self::HyperHttp(_) + | Self::Hyper(_) + | Self::Io(_) + | Self::Anyhow(_) + | Self::Sqlx(_) + | Self::Fatal => hyper::StatusCode::INTERNAL_SERVER_ERROR, + Self::NotFound => hyper::StatusCode::NOT_FOUND, + } + } + + #[must_use] + pub fn get_body(&self) -> crate::io::Error { + crate::io::Error { + error: self.to_string(), + } + } +} + +fn full>(chunk: T) -> BoxBody { + Full::new(chunk.into()) + .map_err(|never| match never {}) + .boxed() +} + +fn construct_json_response( + status: hyper::StatusCode, + data: &U, +) -> Result>, Error> { + Ok(hyper::Response::builder() + .status(status) + .header(hyper::header::CONTENT_TYPE, "application/json") + .body(full(serde_json::to_string(data)?))?) +} + +type Response = hyper::Response>; + +fn construct_json_ok_response(data: &U) -> Result { + construct_json_response(hyper::StatusCode::OK, data) +} + +pub struct Server {} +impl Server { + pub async fn run(addr: SocketAddr, state: Arc) -> Result<(), Error> { + async move { + let listener = tokio::net::TcpListener::bind(&addr).await?; + let server_span = tracing::span!(tracing::Level::TRACE, "http_server", %addr); + + loop { + let (stream, _) = listener.accept().await?; + let io = hyper_util::rt::TokioIo::new(stream); + + let state = state.clone(); + tokio::task::spawn({ + let server_span = server_span.clone(); + async move { + if let Err(err) = hyper::server::conn::http1::Builder::new() + .serve_connection( + io, + hyper::service::service_fn(move |req| router(req, state.clone())), + ) + .instrument(server_span.clone()) + .await + { + tracing::error!("Error serving connection: {err:?}"); + } + } + }); + } + } + .await + } +} + +async fn router( + req: hyper::Request, + state: Arc, +) -> Result { + let span = tracing::span!( + tracing::Level::INFO, + "request", + method = ?req.method(), + uri = ?req.uri(), + headers = ?req.headers() + ); + async move { + let r = match (req.method(), req.uri().path()) { + (&hyper::Method::GET, "/status" | "/status/") => handler::status::get(state).await, + (&hyper::Method::GET, "/status/machines" | "/status/machines/") => { + handler::status::machines(state).await + } + (&hyper::Method::GET, "/status/jobsets" | "/status/jobsets/") => { + handler::status::jobsets(state) + } + (&hyper::Method::GET, "/status/builds" | "/status/builds/") => { + handler::status::builds(state) + } + (&hyper::Method::GET, "/status/steps" | "/status/steps/") => { + handler::status::steps(state) + } + (&hyper::Method::GET, "/status/runnable" | "/status/runnable/") => { + handler::status::runnable(state) + } + (&hyper::Method::GET, "/status/queues" | "/status/queues/") => { + handler::status::queues(state).await + } + (&hyper::Method::GET, "/status/queues/jobs" | "/status/queues/jobs/") => { + handler::status::queue_jobs(state).await + } + (&hyper::Method::GET, "/status/queues/scheduled" | "/status/queues/scheduled/") => { + handler::status::queue_scheduled(state).await + } + (&hyper::Method::POST, "/dump_status" | "/dump_status/") => { + handler::dump_status::post(state).await + } + (&hyper::Method::PUT, "/build" | "/build/") => handler::build::put(req, state).await, + (&hyper::Method::GET, "/metrics" | "/metrics/") => handler::metrics::get(state).await, + _ => Err(Error::NotFound), + }; + if let Err(r) = r.as_ref() { + construct_json_response(r.get_status(), &r.get_body()) + } else { + r + } + } + .instrument(span) + .await +} + +mod handler { + pub mod status { + use super::super::{Error, Response, construct_json_ok_response}; + use crate::{io, state::State}; + + #[tracing::instrument(skip(state), err)] + pub async fn get(state: std::sync::Arc) -> Result { + let queue_stats = io::QueueRunnerStats::new(state.clone()).await; + let sort_fn = state.config.get_machine_sort_fn(); + let free_fn = state.config.get_machine_free_fn(); + let machines = state + .machines + .get_all_machines() + .into_iter() + .map(|m| { + ( + m.hostname.clone(), + crate::io::Machine::from_state(&m, sort_fn, free_fn), + ) + }) + .collect(); + let jobsets = state.jobsets.clone_as_io(); + let remote_stores = { + let stores = state.remote_stores.read(); + stores.clone() + }; + construct_json_ok_response(&io::DumpResponse::new( + queue_stats, + machines, + jobsets, + &state.store, + &remote_stores, + )) + } + + #[tracing::instrument(skip(state), err)] + pub async fn machines(state: std::sync::Arc) -> Result { + let sort_fn = state.config.get_machine_sort_fn(); + let free_fn = state.config.get_machine_free_fn(); + let machines = state + .machines + .get_all_machines() + .into_iter() + .map(|m| { + ( + m.hostname.clone(), + crate::io::Machine::from_state(&m, sort_fn, free_fn), + ) + }) + .collect(); + construct_json_ok_response(&io::MachinesResponse::new(machines)) + } + + #[tracing::instrument(skip(state), err)] + pub fn jobsets(state: std::sync::Arc) -> Result { + let jobsets = state.jobsets.clone_as_io(); + construct_json_ok_response(&io::JobsetsResponse::new(jobsets)) + } + + #[tracing::instrument(skip(state), err)] + pub fn builds(state: std::sync::Arc) -> Result { + let builds = state.builds.clone_as_io(); + construct_json_ok_response(&io::BuildsResponse::new(builds)) + } + + #[tracing::instrument(skip(state), err)] + pub fn steps(state: std::sync::Arc) -> Result { + let steps = state.steps.clone_as_io(); + construct_json_ok_response(&io::StepsResponse::new(steps)) + } + + #[tracing::instrument(skip(state), err)] + pub fn runnable(state: std::sync::Arc) -> Result { + let steps = state.steps.clone_runnable_as_io(); + construct_json_ok_response(&io::StepsResponse::new(steps)) + } + + #[tracing::instrument(skip(state), err)] + pub async fn queues(state: std::sync::Arc) -> Result { + let queues = state + .queues + .clone_inner() + .await + .into_iter() + .map(|(s, q)| { + ( + s, + q.clone_inner() + .into_iter() + .filter_map(|v| v.upgrade().map(Into::into)) + .collect(), + ) + }) + .collect(); + construct_json_ok_response(&io::QueueResponse::new(queues)) + } + + #[tracing::instrument(skip(state), err)] + pub async fn queue_jobs(state: std::sync::Arc) -> Result { + let stepinfos = state + .queues + .get_jobs() + .await + .into_iter() + .map(Into::into) + .collect(); + construct_json_ok_response(&io::StepInfoResponse::new(stepinfos)) + } + + #[tracing::instrument(skip(state), err)] + pub async fn queue_scheduled(state: std::sync::Arc) -> Result { + let stepinfos = state + .queues + .get_scheduled() + .await + .into_iter() + .map(Into::into) + .collect(); + construct_json_ok_response(&io::StepInfoResponse::new(stepinfos)) + } + } + + pub mod dump_status { + use super::super::{Error, Response, construct_json_ok_response}; + use crate::{io, state::State}; + + #[tracing::instrument(skip(state), err)] + pub async fn post(state: std::sync::Arc) -> Result { + let mut db = state.db.get().await?; + let mut tx = db.begin_transaction().await?; + tx.notify_dump_status().await?; + tx.commit().await?; + construct_json_ok_response(&io::Empty {}) + } + } + + pub mod build { + use bytes::Buf as _; + use http_body_util::BodyExt as _; + + use super::super::{Error, Response, construct_json_ok_response}; + use crate::{io, state::State}; + + #[tracing::instrument(skip(req, state), err)] + pub async fn put( + req: hyper::Request, + state: std::sync::Arc, + ) -> Result { + let whole_body = req.collect().await?.aggregate(); + let data: io::BuildPayload = serde_json::from_reader(whole_body.reader())?; + + state + .queue_one_build(data.jobset_id, &nix_utils::StorePath::new(&data.drv)) + .await?; + construct_json_ok_response(&io::Empty {}) + } + } + + pub mod metrics { + use super::super::{Error, Response, full}; + use crate::state::State; + + #[tracing::instrument(skip(state), err)] + pub async fn get(state: std::sync::Arc) -> Result { + let metrics = state.metrics.gather_metrics(&state).await?; + Ok(hyper::Response::builder() + .status(hyper::StatusCode::OK) + .header( + hyper::header::CONTENT_TYPE, + "text/plain; version=0.0.4; charset=utf-8", + ) + .body(full(metrics))?) + } + } +} diff --git a/src/queue-runner/src/server/mod.rs b/src/queue-runner/src/server/mod.rs new file mode 100644 index 000000000..8eb789485 --- /dev/null +++ b/src/queue-runner/src/server/mod.rs @@ -0,0 +1,2 @@ +pub mod grpc; +pub mod http; diff --git a/src/queue-runner/src/state/atomic.rs b/src/queue-runner/src/state/atomic.rs new file mode 100644 index 000000000..ec6d5d61c --- /dev/null +++ b/src/queue-runner/src/state/atomic.rs @@ -0,0 +1,36 @@ +use jiff::Timestamp; +use std::sync::atomic::{AtomicI32, AtomicI64, Ordering}; + +#[derive(Debug)] +pub struct AtomicDateTime { + seconds: AtomicI64, + nanoseconds: AtomicI32, +} + +impl Default for AtomicDateTime { + fn default() -> Self { + Self::new(Timestamp::now()) + } +} + +impl AtomicDateTime { + #[must_use] + pub fn new(dt: Timestamp) -> Self { + Self { + seconds: AtomicI64::new(dt.as_second()), + nanoseconds: AtomicI32::new(dt.subsec_nanosecond()), + } + } + + pub fn load(&self) -> Timestamp { + let seconds = self.seconds.load(Ordering::Relaxed); + let nanoseconds = self.nanoseconds.load(Ordering::Relaxed); + Timestamp::new(seconds, nanoseconds).unwrap_or_default() + } + + pub fn store(&self, dt: Timestamp) { + self.seconds.store(dt.as_second(), Ordering::Relaxed); + self.nanoseconds + .store(dt.subsec_nanosecond(), Ordering::Relaxed); + } +} diff --git a/src/queue-runner/src/state/build.rs b/src/queue-runner/src/state/build.rs new file mode 100644 index 000000000..cf6c9d1b6 --- /dev/null +++ b/src/queue-runner/src/state/build.rs @@ -0,0 +1,694 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; + +use hashbrown::{HashMap, HashSet}; + +use super::{Jobset, JobsetID, Step}; +use db::models::{BuildID, BuildStatus}; +use nix_utils::BaseStore as _; + +pub type AtomicBuildID = AtomicI32; + +#[derive(Debug)] +pub struct Build { + pub id: BuildID, + pub drv_path: nix_utils::StorePath, + pub outputs: HashMap, + pub jobset_id: JobsetID, + pub name: String, + pub timestamp: jiff::Timestamp, + pub max_silent_time: i32, + pub timeout: i32, + pub local_priority: i32, + pub global_priority: AtomicI32, + + toplevel: arc_swap::ArcSwapOption, + pub jobset: Arc, + + finished_in_db: AtomicBool, +} + +impl PartialEq for Build { + fn eq(&self, other: &Self) -> bool { + self.drv_path == other.drv_path + } +} + +impl Eq for Build {} + +impl std::hash::Hash for Build { + fn hash(&self, state: &mut H) { + // ensure that drv_path is never mutable + // as we set Build as ignore-interior-mutability + self.drv_path.hash(state); + } +} + +impl Build { + #[must_use] + pub fn new_debug(drv_path: &nix_utils::StorePath) -> Arc { + Arc::new(Self { + id: BuildID::MAX, + drv_path: drv_path.to_owned(), + outputs: HashMap::with_capacity(6), + jobset_id: JobsetID::MAX, + name: "debug".into(), + timestamp: jiff::Timestamp::now(), + max_silent_time: i32::MAX, + timeout: i32::MAX, + local_priority: 1000, + global_priority: 1000.into(), + toplevel: arc_swap::ArcSwapOption::from(None), + jobset: Arc::new(Jobset::new(JobsetID::MAX, "debug", "debug")), + finished_in_db: false.into(), + }) + } + + #[tracing::instrument(skip(v, jobset), err)] + pub fn new(v: db::models::Build, jobset: Arc) -> anyhow::Result> { + Ok(Arc::new(Self { + id: v.id, + drv_path: nix_utils::StorePath::new(&v.drvpath), + outputs: HashMap::with_capacity(6), + jobset_id: v.jobset_id, + name: v.job, + timestamp: jiff::Timestamp::from_second(v.timestamp)?, + max_silent_time: v.maxsilent.unwrap_or(3600), + timeout: v.timeout.unwrap_or(36000), + local_priority: v.priority, + global_priority: v.globalpriority.into(), + toplevel: arc_swap::ArcSwapOption::from(None), + jobset, + finished_in_db: false.into(), + })) + } + + #[inline] + pub fn full_job_name(&self) -> String { + format!( + "{}:{}:{}", + self.jobset.project_name, self.jobset.name, self.name + ) + } + + #[inline] + pub fn get_finished_in_db(&self) -> bool { + self.finished_in_db.load(Ordering::SeqCst) + } + + #[inline] + pub fn set_finished_in_db(&self, v: bool) { + self.finished_in_db.store(v, Ordering::SeqCst); + } + + #[inline] + pub fn set_toplevel_step(&self, step: Arc) { + self.toplevel.store(Some(step)); + } + + pub fn propagate_priorities(&self) { + let mut queued = HashSet::new(); + let mut todo = std::collections::VecDeque::new(); + { + let toplevel = self.toplevel.load(); + if let Some(toplevel) = toplevel.as_ref() { + todo.push_back(toplevel.clone()); + } + } + + while let Some(step) = todo.pop_front() { + step.atomic_state.highest_global_priority.store( + std::cmp::max( + step.atomic_state + .highest_global_priority + .load(Ordering::Relaxed), + self.global_priority.load(Ordering::Relaxed), + ), + Ordering::Relaxed, + ); + step.atomic_state.highest_local_priority.store( + std::cmp::max( + step.atomic_state + .highest_local_priority + .load(Ordering::Relaxed), + self.local_priority, + ), + Ordering::Relaxed, + ); + step.atomic_state.lowest_build_id.store( + std::cmp::min( + step.atomic_state.lowest_build_id.load(Ordering::Relaxed), + self.id, + ), + Ordering::Relaxed, + ); + step.add_jobset(self.jobset.clone()); + for dep in step.get_all_deps_not_queued(&queued) { + queued.insert(dep.clone()); + todo.push_back(dep); + } + } + } +} + +#[derive(Debug)] +pub enum BuildResultState { + Success, + BuildFailure, + PreparingFailure, + ImportFailure, + UploadFailure, + PostProcessingFailure, + Aborted, + Cancelled, +} + +impl From for BuildResultState { + fn from(v: crate::server::grpc::runner_v1::BuildResultState) -> Self { + match v { + crate::server::grpc::runner_v1::BuildResultState::BuildFailure => Self::BuildFailure, + crate::server::grpc::runner_v1::BuildResultState::Success => Self::Success, + crate::server::grpc::runner_v1::BuildResultState::PreparingFailure => { + Self::PreparingFailure + } + crate::server::grpc::runner_v1::BuildResultState::ImportFailure => Self::ImportFailure, + crate::server::grpc::runner_v1::BuildResultState::UploadFailure => Self::UploadFailure, + crate::server::grpc::runner_v1::BuildResultState::PostProcessingFailure => { + Self::PostProcessingFailure + } + } + } +} + +#[allow(clippy::struct_excessive_bools)] +#[derive(Debug, Clone)] +pub struct RemoteBuild { + pub step_status: BuildStatus, + pub can_retry: bool, // for bsAborted + pub is_cached: bool, // for bsSucceed + pub can_cache: bool, // for bsFailed + pub error_msg: Option, // for bsAborted + + times_built: i32, + is_non_deterministic: bool, + + start_time: Option, + stop_time: Option, + + overhead: i32, + pub log_file: String, +} + +impl Default for RemoteBuild { + fn default() -> Self { + Self::new() + } +} + +impl RemoteBuild { + #[must_use] + pub const fn new() -> Self { + Self { + step_status: BuildStatus::Aborted, + can_retry: false, + is_cached: false, + can_cache: false, + error_msg: None, + times_built: 0, + is_non_deterministic: false, + start_time: None, + stop_time: None, + overhead: 0, + log_file: String::new(), + } + } + + #[must_use] + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + pub fn get_total_step_time_ms(&self) -> u64 { + if let (Some(start_time), Some(stop_time)) = (self.start_time, self.stop_time) { + (stop_time - start_time) + .total(jiff::Unit::Millisecond) + .unwrap_or_default() + .abs() as u64 + } else { + 0 + } + } + + pub const fn update_with_result_state(&mut self, state: &BuildResultState) { + match state { + BuildResultState::BuildFailure => { + self.can_retry = false; + } + BuildResultState::Success => (), + BuildResultState::PreparingFailure + | BuildResultState::ImportFailure + | BuildResultState::UploadFailure + | BuildResultState::PostProcessingFailure => { + self.can_retry = true; + } + BuildResultState::Aborted => { + self.can_retry = true; + self.step_status = BuildStatus::Aborted; + } + BuildResultState::Cancelled => { + self.can_retry = true; + self.step_status = BuildStatus::Cancelled; + } + } + } + + pub const fn set_start_and_stop(&mut self, v: jiff::Timestamp) { + self.start_time = Some(v); + self.stop_time = Some(v); + } + + pub fn set_start_time_now(&mut self) { + self.start_time = Some(jiff::Timestamp::now()); + } + + pub fn set_stop_time_now(&mut self) { + self.stop_time = Some(jiff::Timestamp::now()); + } + + #[must_use] + pub const fn has_start_time(&self) -> bool { + self.start_time.is_some() + } + + pub fn get_start_time_as_i32(&self) -> Result { + // TODO: migrate to 64 bit timestamps + i32::try_from( + self.start_time + .map(jiff::Timestamp::as_second) + .unwrap_or_default(), + ) + } + + #[must_use] + pub const fn has_stop_time(&self) -> bool { + self.stop_time.is_some() + } + + pub fn get_stop_time_as_i32(&self) -> Result { + // TODO: migrate to 64 bit timestamps + i32::try_from( + self.stop_time + .map(jiff::Timestamp::as_second) + .unwrap_or_default(), + ) + } + + #[must_use] + pub const fn get_overhead(&self) -> Option { + if self.overhead != 0 { + Some(self.overhead) + } else { + None + } + } + + #[must_use] + pub const fn get_times_built(&self) -> Option { + if self.times_built != 0 { + Some(self.times_built) + } else { + None + } + } + + #[must_use] + pub const fn get_is_non_deterministic(&self) -> Option { + if self.times_built != 0 { + Some(self.is_non_deterministic) + } else { + None + } + } + + pub fn set_overhead(&mut self, v: u128) -> Result<(), std::num::TryFromIntError> { + self.overhead = i32::try_from(v)?; + Ok(()) + } +} + +pub struct BuildProduct { + pub path: Option, + pub default_path: Option, + + pub r#type: String, + pub subtype: String, + pub name: String, + + pub is_regular: bool, + + pub sha256hash: Option, + pub file_size: Option, +} + +impl From for BuildProduct { + fn from(v: db::models::OwnedBuildProduct) -> Self { + Self { + path: v.path.map(|v| nix_utils::StorePath::new(&v)), + default_path: v.defaultpath, + r#type: v.r#type, + subtype: v.subtype, + name: v.name, + is_regular: v.filesize.is_some(), + sha256hash: v.sha256hash, + #[allow(clippy::cast_sign_loss)] + file_size: v.filesize.map(|v| v as u64), + } + } +} + +impl From for BuildProduct { + fn from(v: crate::server::grpc::runner_v1::BuildProduct) -> Self { + Self { + path: Some(nix_utils::StorePath::new(&v.path)), + default_path: Some(v.default_path), + r#type: v.r#type, + subtype: v.subtype, + name: v.name, + is_regular: v.is_regular, + sha256hash: v.sha256hash, + file_size: v.file_size, + } + } +} + +impl From for BuildProduct { + fn from(v: shared::BuildProduct) -> Self { + Self { + path: Some(nix_utils::StorePath::new(&v.path)), + default_path: Some(v.default_path), + r#type: v.r#type, + subtype: v.subtype, + name: v.name, + is_regular: v.is_regular, + sha256hash: v.sha256hash, + file_size: v.file_size, + } + } +} + +pub struct BuildMetric { + pub name: String, + pub unit: Option, + pub value: f64, +} + +impl From for BuildMetric { + fn from(v: db::models::OwnedBuildMetric) -> Self { + Self { + name: v.name, + unit: v.unit, + value: v.value, + } + } +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct BuildTimings { + pub import_elapsed: std::time::Duration, + pub build_elapsed: std::time::Duration, + pub upload_elapsed: std::time::Duration, +} + +impl BuildTimings { + #[must_use] + pub const fn new(import_time_ms: u64, build_time_ms: u64, upload_time_ms: u64) -> Self { + Self { + import_elapsed: std::time::Duration::from_millis(import_time_ms), + build_elapsed: std::time::Duration::from_millis(build_time_ms), + upload_elapsed: std::time::Duration::from_millis(upload_time_ms), + } + } + + #[must_use] + pub const fn get_overhead(&self) -> u128 { + self.import_elapsed.as_millis() + self.upload_elapsed.as_millis() + } +} + +pub struct BuildOutput { + pub failed: bool, + pub timings: BuildTimings, + pub release_name: Option, + + pub closure_size: u64, + pub size: u64, + + pub products: Vec, + pub outputs: HashMap, + pub metrics: HashMap, +} + +impl TryFrom for BuildOutput { + type Error = anyhow::Error; + + fn try_from(v: db::models::BuildOutput) -> anyhow::Result { + let build_status = BuildStatus::from_i32( + v.buildstatus + .ok_or_else(|| anyhow::anyhow!("buildstatus missing"))?, + ) + .ok_or_else(|| anyhow::anyhow!("buildstatus did not map"))?; + Ok(Self { + failed: build_status != BuildStatus::Success, + timings: BuildTimings::default(), + release_name: v.releasename, + #[allow(clippy::cast_sign_loss)] + closure_size: v.closuresize.unwrap_or_default() as u64, + #[allow(clippy::cast_sign_loss)] + size: v.size.unwrap_or_default() as u64, + products: vec![], + outputs: HashMap::with_capacity(6), + metrics: HashMap::with_capacity(10), + }) + } +} + +impl From for BuildOutput { + fn from(v: crate::server::grpc::runner_v1::BuildResultInfo) -> Self { + let mut outputs = HashMap::with_capacity(6); + let mut closure_size = 0; + let mut nar_size = 0; + + for o in v.outputs { + match o.output { + Some(crate::server::grpc::runner_v1::output::Output::Nameonly(_)) => { + // We dont care about outputs that dont have a path, + } + Some(crate::server::grpc::runner_v1::output::Output::Withpath(o)) => { + outputs.insert(o.name, nix_utils::StorePath::new(&o.path)); + closure_size += o.closure_size; + nar_size += o.nar_size; + } + None => (), + } + } + let (failed, release_name, products, metrics) = if let Some(nix_support) = v.nix_support { + ( + nix_support.failed, + nix_support.hydra_release_name, + nix_support.products, + nix_support.metrics, + ) + } else { + (false, None, vec![], vec![]) + }; + + Self { + failed, + timings: BuildTimings::new(v.import_time_ms, v.build_time_ms, v.upload_time_ms), + release_name, + closure_size, + size: nar_size, + products: products.into_iter().map(Into::into).collect(), + outputs, + metrics: metrics + .into_iter() + .map(|v| { + ( + v.path, + BuildMetric { + name: v.name, + unit: v.unit, + value: v.value, + }, + ) + }) + .collect(), + } + } +} + +impl BuildOutput { + #[tracing::instrument(skip(store, outputs), err)] + pub async fn new( + store: &nix_utils::LocalStore, + outputs: Vec, + ) -> anyhow::Result { + let flat_outputs = outputs + .iter() + .filter_map(|o| o.path.as_ref()) + .collect::>(); + let pathinfos = store.query_path_infos(&flat_outputs).await; + let nix_support = Box::pin(shared::parse_nix_support_from_outputs(store, &outputs)).await?; + + let mut outputs_map = HashMap::with_capacity(outputs.len()); + let mut closure_size = 0; + let mut nar_size = 0; + + for o in outputs { + if let Some(path) = o.path + && let Some(info) = pathinfos.get(&path) + { + closure_size += store.compute_closure_size(&path).await; + nar_size += info.nar_size; + outputs_map.insert(o.name, path); + } + } + + Ok(Self { + failed: nix_support.failed, + timings: BuildTimings::default(), + release_name: nix_support.hydra_release_name, + closure_size, + size: nar_size, + products: nix_support.products.into_iter().map(Into::into).collect(), + outputs: outputs_map, + metrics: nix_support + .metrics + .into_iter() + .map(|v| { + ( + v.path, + BuildMetric { + name: v.name, + unit: v.unit, + value: v.value, + }, + ) + }) + .collect(), + }) + } +} + +pub fn get_mark_build_sccuess_data<'a>( + store: &nix_utils::LocalStore, + b: &'a Arc, + res: &'a crate::state::BuildOutput, +) -> db::models::MarkBuildSuccessData<'a> { + db::models::MarkBuildSuccessData { + id: b.id, + name: &b.name, + project_name: &b.jobset.project_name, + jobset_name: &b.jobset.name, + finished_in_db: b.get_finished_in_db(), + timestamp: b.timestamp.as_second(), + failed: res.failed, + closure_size: res.closure_size, + size: res.size, + release_name: res.release_name.as_deref(), + outputs: res + .outputs + .iter() + .map(|(name, path)| (name.clone(), store.print_store_path(path))) + .collect(), + products: res + .products + .iter() + .map(|v| db::models::BuildProduct { + r#type: &v.r#type, + subtype: &v.subtype, + filesize: v.file_size.and_then(|v| i64::try_from(v).ok()), + sha256hash: v.sha256hash.as_deref(), + path: v.path.as_ref().map(|p| store.print_store_path(p)), + name: &v.name, + defaultpath: v.default_path.as_deref(), + }) + .collect(), + metrics: res + .metrics + .iter() + .map(|(name, m)| { + ( + name.as_str(), + db::models::BuildMetric { + name: &m.name, + unit: m.unit.as_deref(), + value: m.value, + }, + ) + }) + .collect(), + } +} + +#[derive(Clone)] +pub struct Builds { + inner: Arc>>>, +} + +impl Default for Builds { + fn default() -> Self { + Self::new() + } +} + +impl Builds { + #[must_use] + pub fn new() -> Self { + Self { + inner: Arc::new(parking_lot::RwLock::new(HashMap::with_capacity(1000))), + } + } + + #[must_use] + pub fn len(&self) -> usize { + self.inner.read().len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.inner.read().is_empty() + } + + #[must_use] + pub fn clone_as_io(&self) -> Vec { + let builds = self.inner.read(); + builds.values().map(|v| v.clone().into()).collect() + } + + pub fn update_priorities(&self, curr_ids: &HashMap) { + let mut builds = self.inner.write(); + builds.retain(|k, _| curr_ids.contains_key(k)); + for (id, build) in builds.iter() { + let Some(new_priority) = curr_ids.get(id) else { + // we should never get into this case because of the retain above + continue; + }; + + if build.global_priority.load(Ordering::Relaxed) < *new_priority { + tracing::info!("priority of build {id} increased"); + build + .global_priority + .store(*new_priority, Ordering::Relaxed); + build.propagate_priorities(); + } + } + } + + pub fn insert_new_build(&self, build: Arc) { + let mut builds = self.inner.write(); + builds.insert(build.id, build); + } + + pub fn remove_by_id(&self, id: BuildID) { + let mut builds = self.inner.write(); + builds.remove(&id); + } +} diff --git a/src/queue-runner/src/state/fod_checker.rs b/src/queue-runner/src/state/fod_checker.rs new file mode 100644 index 000000000..cb2dd6b85 --- /dev/null +++ b/src/queue-runner/src/state/fod_checker.rs @@ -0,0 +1,180 @@ +use std::sync::Arc; + +use hashbrown::{HashMap, HashSet}; +use nix_utils::{Derivation, LocalStore, StorePath}; + +pub struct FodChecker { + ca_derivations: parking_lot::RwLock>, + to_traverse: parking_lot::RwLock>, + + notify_traverse: tokio::sync::Notify, + traverse_done_notifier: Option>, +} + +async fn collect_ca_derivations( + store: &LocalStore, + drv: &StorePath, + processed: Arc>>, +) -> HashMap { + use futures::StreamExt as _; + + { + let p = processed.read(); + if p.contains(drv) { + return HashMap::new(); + } + } + { + let mut p = processed.write(); + p.insert(drv.clone()); + } + + let Some(parsed) = nix_utils::query_drv(store, drv).await.ok().flatten() else { + return HashMap::new(); + }; + + let is_ca = parsed.is_ca(); + let mut out = if parsed.input_drvs.is_empty() { + HashMap::new() + } else { + futures::StreamExt::map(tokio_stream::iter(parsed.input_drvs.clone()), |i| { + let processed = processed.clone(); + async move { + let i = StorePath::new(&i); + Box::pin(collect_ca_derivations(store, &i, processed)).await + } + }) + .buffered(10) + .flat_map(futures::stream::iter) + .collect::>() + .await + }; + if is_ca { + out.insert(drv.clone(), parsed); + } + + out +} + +impl FodChecker { + #[must_use] + pub fn new(traverse_done_notifier: Option>) -> Self { + Self { + ca_derivations: parking_lot::RwLock::new(HashMap::with_capacity(1000)), + to_traverse: parking_lot::RwLock::new(HashSet::new()), + + notify_traverse: tokio::sync::Notify::new(), + traverse_done_notifier, + } + } + + pub(super) fn add_ca_drv_parsed(&self, drv: &StorePath, parsed: &nix_utils::Derivation) { + if parsed.is_ca() { + let mut ca = self.ca_derivations.write(); + ca.insert(drv.clone(), parsed.clone()); + } + } + + pub fn to_traverse(&self, drv: &StorePath) { + let mut tt = self.to_traverse.write(); + tt.insert(drv.clone()); + } + + async fn traverse(&self, store: &nix_utils::LocalStore) { + use futures::StreamExt as _; + + let drvs = { + let mut tt = self.to_traverse.write(); + let v: Vec<_> = tt.iter().map(Clone::clone).collect(); + tt.clear(); + v + }; + + let processed = Arc::new(parking_lot::RwLock::new(HashSet::::new())); + let out = futures::StreamExt::map(tokio_stream::iter(drvs), |i| { + let processed = processed.clone(); + async move { Box::pin(collect_ca_derivations(store, &i, processed)).await } + }) + .buffered(10) + .flat_map(futures::stream::iter) + .collect::>() + .await; + + { + let mut ca_derivations = self.ca_derivations.write(); + ca_derivations.extend(out); + } + tracing::info!("ca count: {}", self.ca_derivations.read().len()); + } + + #[tracing::instrument(skip(self))] + pub fn trigger_traverse(&self) { + self.notify_traverse.notify_one(); + } + + #[tracing::instrument(skip(self, store))] + async fn traverse_loop(&self, store: nix_utils::LocalStore) { + loop { + tokio::select! { + () = self.notify_traverse.notified() => {}, + () = tokio::time::sleep(tokio::time::Duration::from_secs(60)) => {}, + }; + self.traverse(&store).await; + if let Some(tx) = &self.traverse_done_notifier { + let _ = tx.send(()).await; + } + } + } + + pub fn start_traverse_loop( + self: Arc, + store: nix_utils::LocalStore, + ) -> tokio::task::AbortHandle { + let task = tokio::task::spawn(async move { + Box::pin(self.traverse_loop(store)).await; + }); + task.abort_handle() + } + + pub async fn process(&self, processor: F) -> i64 + where + F: AsyncFn(StorePath, Derivation) -> (), + { + let drvs = { + let mut drvs = self.ca_derivations.write(); + let cloned = drvs.clone(); + drvs.clear(); + cloned + }; + + let mut c = 0; + for (path, drv) in drvs { + processor(path, drv).await; + c += 1; + } + + c + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use crate::state::fod_checker::FodChecker; + use nix_utils::BaseStore as _; + + #[ignore = "Requires a valid drv in the nix-store"] + #[tokio::test] + async fn test_traverse() { + let store = nix_utils::LocalStore::init(); + let hello_drv = + nix_utils::StorePath::new("rl5m4zxd24mkysmpbp4j9ak6q7ia6vj8-hello-2.12.2.drv"); + store.ensure_path(&hello_drv).await.unwrap(); + + let fod = FodChecker::new(None); + fod.to_traverse(&hello_drv); + fod.traverse(&store).await; + assert_eq!(fod.ca_derivations.read().len(), 59); + } +} diff --git a/src/queue-runner/src/state/jobset.rs b/src/queue-runner/src/state/jobset.rs new file mode 100644 index 000000000..33a669f99 --- /dev/null +++ b/src/queue-runner/src/state/jobset.rs @@ -0,0 +1,221 @@ +use std::collections::BTreeMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicI64, AtomicU32, Ordering}; + +use hashbrown::HashMap; + +pub type JobsetID = i32; +pub const SCHEDULING_WINDOW: i64 = 24 * 60 * 60; + +#[derive(Debug)] +pub struct Jobset { + pub id: JobsetID, + pub project_name: String, + pub name: String, + + seconds: AtomicI64, + shares: AtomicU32, + // The start time and duration of the most recent build steps. + steps: parking_lot::RwLock>, +} + +impl PartialEq for Jobset { + fn eq(&self, other: &Self) -> bool { + self.id == other.id && self.project_name == other.project_name && self.name == other.name + } +} + +impl Eq for Jobset {} + +impl std::hash::Hash for Jobset { + fn hash(&self, state: &mut H) { + self.id.hash(state); + self.project_name.hash(state); + self.name.hash(state); + } +} + +impl Jobset { + pub fn new>(id: JobsetID, project_name: S, name: S) -> Self { + Self { + id, + project_name: project_name.into(), + name: name.into(), + seconds: 0.into(), + shares: 0.into(), + steps: parking_lot::RwLock::new(BTreeMap::new()), + } + } + + pub fn full_name(&self) -> String { + format!("{}:{}", self.project_name, self.name) + } + + pub fn share_used(&self) -> f64 { + let seconds = self.seconds.load(Ordering::Relaxed); + let shares = self.shares.load(Ordering::Relaxed); + + // we dont care about the precision here + #[allow(clippy::cast_precision_loss)] + ((seconds as f64) / f64::from(shares)) + } + + pub fn set_shares(&self, shares: i32) -> anyhow::Result<()> { + debug_assert!(shares > 0); + self.shares.store(shares.try_into()?, Ordering::Relaxed); + Ok(()) + } + + pub fn get_shares(&self) -> u32 { + self.shares.load(Ordering::Relaxed) + } + + pub fn get_seconds(&self) -> i64 { + self.seconds.load(Ordering::Relaxed) + } + + pub fn add_step(&self, start_time: i64, duration: i64) { + self.steps.write().insert(start_time, duration); + self.seconds.fetch_add(duration, Ordering::Relaxed); + } + + pub fn prune_steps(&self) { + let now = jiff::Timestamp::now().as_second(); + let mut steps = self.steps.write(); + + loop { + let Some(first) = steps.first_entry() else { + break; + }; + let start_time = *first.key(); + + if start_time > now - SCHEDULING_WINDOW { + break; + } + self.seconds.fetch_sub(*first.get(), Ordering::Relaxed); + steps.remove(&start_time); + } + } +} + +// Projectname, Jobsetname +type JobsetName = (String, String); + +#[derive(Clone)] +pub struct Jobsets { + inner: Arc>>>, +} + +impl Default for Jobsets { + fn default() -> Self { + Self::new() + } +} + +impl Jobsets { + #[must_use] + pub fn new() -> Self { + Self { + inner: Arc::new(parking_lot::RwLock::new(HashMap::with_capacity(100))), + } + } + + #[must_use] + pub fn len(&self) -> usize { + self.inner.read().len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.inner.read().is_empty() + } + + #[must_use] + pub fn clone_as_io(&self) -> HashMap { + let jobsets = self.inner.read(); + jobsets + .values() + .map(|v| (v.full_name(), v.clone().into())) + .collect() + } + + #[tracing::instrument(skip(self))] + pub fn prune(&self) { + let jobsets = self.inner.read(); + for ((project_name, jobset_name), jobset) in jobsets.iter() { + let s1 = jobset.share_used(); + jobset.prune_steps(); + let s2 = jobset.share_used(); + if (s1 - s2).abs() > f64::EPSILON { + tracing::debug!( + "pruned scheduling window of '{project_name}:{jobset_name}' from {s1} to {s2}" + ); + } + } + } + + #[tracing::instrument(skip(self, conn), err)] + pub async fn create( + &self, + conn: &mut db::Connection, + jobset_id: i32, + project_name: &str, + jobset_name: &str, + ) -> anyhow::Result> { + let key = (project_name.to_owned(), jobset_name.to_owned()); + { + let jobsets = self.inner.read(); + if let Some(jobset) = jobsets.get(&key) { + return Ok(jobset.clone()); + } + } + + let shares = conn + .get_jobset_scheduling_shares(jobset_id) + .await? + .ok_or_else(|| anyhow::anyhow!("Scheduling Shares not found for jobset not found."))?; + let jobset = Jobset::new(jobset_id, project_name, jobset_name); + jobset.set_shares(shares)?; + + for step in conn + .get_jobset_build_steps(jobset_id, SCHEDULING_WINDOW) + .await? + { + let Some(starttime) = step.starttime else { + continue; + }; + let Some(stoptime) = step.stoptime else { + continue; + }; + jobset.add_step(i64::from(starttime), i64::from(stoptime - starttime)); + } + + let jobset = Arc::new(jobset); + { + let mut jobsets = self.inner.write(); + jobsets.insert(key, jobset.clone()); + } + + Ok(jobset) + } + + #[tracing::instrument(skip(self, conn), err)] + pub async fn handle_change(&self, conn: &mut db::Connection) -> anyhow::Result<()> { + let curr_jobsets_in_db = conn.get_jobsets().await?; + + let jobsets = self.inner.read(); + for row in curr_jobsets_in_db { + if let Some(i) = jobsets.get(&(row.project.clone(), row.name.clone())) + && let Err(e) = i.set_shares(row.schedulingshares) + { + tracing::error!( + "Failed to update jobset scheduling shares. project_name={} jobset_name={} e={}", + row.project, + row.name, + e, + ); + } + } + Ok(()) + } +} diff --git a/src/queue-runner/src/state/machine.rs b/src/queue-runner/src/state/machine.rs new file mode 100644 index 000000000..8db3fec96 --- /dev/null +++ b/src/queue-runner/src/state/machine.rs @@ -0,0 +1,901 @@ +use std::sync::{Arc, atomic::Ordering}; + +use hashbrown::{HashMap, HashSet}; +use smallvec::SmallVec; +use tokio::sync::mpsc; + +use db::models::BuildID; + +use super::{RemoteBuild, System}; +use crate::config::{MachineFreeFn, MachineSortFn}; +use crate::server::grpc::runner_v1::{AbortMessage, BuildMessage, JoinMessage, runner_request}; + +#[derive(Debug, Clone, Copy)] +pub struct Pressure { + pub avg10: f32, + pub avg60: f32, + pub avg300: f32, + pub total: u64, +} + +impl From for Pressure { + fn from(v: crate::server::grpc::runner_v1::Pressure) -> Self { + Self { + avg10: v.avg10, + avg60: v.avg60, + avg300: v.avg300, + total: v.total, + } + } +} + +#[derive(Debug)] +pub struct PressureState { + pub cpu_some: Option, + pub mem_some: Option, + pub mem_full: Option, + pub io_some: Option, + pub io_full: Option, + pub irq_full: Option, +} + +#[derive(Debug)] +pub struct Stats { + current_jobs: std::sync::atomic::AtomicU64, + nr_steps_done: std::sync::atomic::AtomicU64, + failed_builds: std::sync::atomic::AtomicU64, + succeeded_builds: std::sync::atomic::AtomicU64, + + total_step_time_ms: std::sync::atomic::AtomicU64, + total_step_import_time_ms: std::sync::atomic::AtomicU64, + total_step_build_time_ms: std::sync::atomic::AtomicU64, + total_step_upload_time_ms: std::sync::atomic::AtomicU64, + idle_since: std::sync::atomic::AtomicI64, + + last_failure: std::sync::atomic::AtomicI64, + disabled_until: std::sync::atomic::AtomicI64, + consecutive_failures: std::sync::atomic::AtomicU64, + last_ping: std::sync::atomic::AtomicI64, + + load1: atomic_float::AtomicF32, + load5: atomic_float::AtomicF32, + load15: atomic_float::AtomicF32, + mem_usage: std::sync::atomic::AtomicU64, + pub pressure: arc_swap::ArcSwapOption, + build_dir_free_percent: atomic_float::AtomicF64, + store_free_percent: atomic_float::AtomicF64, + current_uploading_path_count: std::sync::atomic::AtomicU64, + current_downloading_count: std::sync::atomic::AtomicU64, + + pub jobs_in_last_30s_start: std::sync::atomic::AtomicI64, + pub jobs_in_last_30s_count: std::sync::atomic::AtomicU64, +} + +impl Default for Stats { + fn default() -> Self { + Self::new() + } +} + +impl Stats { + #[must_use] + pub fn new() -> Self { + Self { + current_jobs: 0.into(), + nr_steps_done: 0.into(), + failed_builds: 0.into(), + succeeded_builds: 0.into(), + + total_step_time_ms: 0.into(), + total_step_import_time_ms: 0.into(), + total_step_build_time_ms: 0.into(), + total_step_upload_time_ms: 0.into(), + idle_since: (jiff::Timestamp::now().as_second()).into(), + last_failure: 0.into(), + disabled_until: 0.into(), + consecutive_failures: 0.into(), + last_ping: 0.into(), + + load1: 0.0.into(), + load5: 0.0.into(), + load15: 0.0.into(), + mem_usage: 0.into(), + + pressure: arc_swap::ArcSwapOption::from(None), + build_dir_free_percent: 0.0.into(), + store_free_percent: 0.0.into(), + current_uploading_path_count: 0.into(), + current_downloading_count: 0.into(), + + jobs_in_last_30s_start: 0.into(), + jobs_in_last_30s_count: 0.into(), + } + } + + pub fn store_current_jobs(&self, c: u64) { + if c == 0 && self.idle_since.load(Ordering::Relaxed) == 0 { + self.idle_since + .store(jiff::Timestamp::now().as_second(), Ordering::Relaxed); + } else { + self.idle_since.store(0, Ordering::Relaxed); + } + + self.current_jobs.store(c, Ordering::Relaxed); + } + + pub fn get_current_jobs(&self) -> u64 { + self.current_jobs.load(Ordering::Relaxed) + } + + pub fn get_nr_steps_done(&self) -> u64 { + self.nr_steps_done.load(Ordering::Relaxed) + } + + pub fn incr_nr_steps_done(&self) { + self.nr_steps_done.fetch_add(1, Ordering::Relaxed); + } + + pub fn get_total_step_time_ms(&self) -> u64 { + self.total_step_time_ms.load(Ordering::Relaxed) + } + + pub fn get_total_step_import_time_ms(&self) -> u64 { + self.total_step_import_time_ms.load(Ordering::Relaxed) + } + + fn add_to_total_step_import_time_ms(&self, v: u128) { + if let Ok(v) = u64::try_from(v) { + self.total_step_import_time_ms + .fetch_add(v, Ordering::Relaxed); + } + } + + pub fn get_total_step_build_time_ms(&self) -> u64 { + self.total_step_build_time_ms.load(Ordering::Relaxed) + } + + pub fn add_to_total_step_build_time_ms(&self, v: u128) { + if let Ok(v) = u64::try_from(v) { + self.total_step_build_time_ms + .fetch_add(v, Ordering::Relaxed); + } + } + + pub fn get_total_step_upload_time_ms(&self) -> u64 { + self.total_step_upload_time_ms.load(Ordering::Relaxed) + } + + fn add_to_total_step_upload_time_ms(&self, v: u128) { + if let Ok(v) = u64::try_from(v) { + self.total_step_upload_time_ms + .fetch_add(v, Ordering::Relaxed); + } + } + + pub fn get_idle_since(&self) -> i64 { + self.idle_since.load(Ordering::Relaxed) + } + + pub fn get_last_failure(&self) -> i64 { + self.last_failure.load(Ordering::Relaxed) + } + + pub fn get_disabled_until(&self) -> i64 { + self.disabled_until.load(Ordering::Relaxed) + } + + pub fn get_consecutive_failures(&self) -> u64 { + self.consecutive_failures.load(Ordering::Relaxed) + } + + pub fn get_failed_builds(&self) -> u64 { + self.failed_builds.load(Ordering::Relaxed) + } + + pub fn get_succeeded_builds(&self) -> u64 { + self.succeeded_builds.load(Ordering::Relaxed) + } + + pub fn track_build_success(&self, timings: super::build::BuildTimings, total_step_time: u64) { + self.succeeded_builds.fetch_add(1, Ordering::Relaxed); + self.add_to_total_step_import_time_ms(timings.import_elapsed.as_millis()); + self.add_to_total_step_build_time_ms(timings.build_elapsed.as_millis()); + self.add_to_total_step_upload_time_ms(timings.upload_elapsed.as_millis()); + self.total_step_time_ms + .fetch_add(total_step_time, Ordering::Relaxed); + self.consecutive_failures.store(0, Ordering::Relaxed); + } + + pub fn track_build_failure(&self, timings: super::build::BuildTimings, total_step_time: u64) { + self.failed_builds.fetch_add(1, Ordering::Relaxed); + self.add_to_total_step_import_time_ms(timings.import_elapsed.as_millis()); + self.add_to_total_step_build_time_ms(timings.build_elapsed.as_millis()); + self.add_to_total_step_upload_time_ms(timings.upload_elapsed.as_millis()); + self.total_step_time_ms + .fetch_add(total_step_time, Ordering::Relaxed); + self.last_failure + .store(jiff::Timestamp::now().as_second(), Ordering::Relaxed); + self.consecutive_failures.fetch_add(1, Ordering::Relaxed); + } + + pub fn get_last_ping(&self) -> i64 { + self.last_ping.load(Ordering::Relaxed) + } + + pub fn store_ping(&self, msg: &crate::server::grpc::runner_v1::PingMessage) { + self.last_ping + .store(jiff::Timestamp::now().as_second(), Ordering::Relaxed); + + self.load1.store(msg.load1, Ordering::Relaxed); + self.load5.store(msg.load5, Ordering::Relaxed); + self.load15.store(msg.load15, Ordering::Relaxed); + self.mem_usage.store(msg.mem_usage, Ordering::Relaxed); + + if let Some(p) = msg.pressure { + self.pressure.store(Some(Arc::new(PressureState { + cpu_some: p.cpu_some.map(Into::into), + mem_some: p.mem_some.map(Into::into), + mem_full: p.mem_full.map(Into::into), + io_some: p.io_some.map(Into::into), + io_full: p.io_full.map(Into::into), + irq_full: p.irq_full.map(Into::into), + }))); + } + + self.build_dir_free_percent + .store(msg.build_dir_free_percent, Ordering::Relaxed); + self.store_free_percent + .store(msg.store_free_percent, Ordering::Relaxed); + + self.current_uploading_path_count + .store(msg.current_uploading_path_count, Ordering::Relaxed); + self.current_downloading_count + .store(msg.current_downloading_path_count, Ordering::Relaxed); + } + + pub fn get_load1(&self) -> f32 { + self.load1.load(Ordering::Relaxed) + } + + pub fn get_load5(&self) -> f32 { + self.load5.load(Ordering::Relaxed) + } + + pub fn get_load15(&self) -> f32 { + self.load15.load(Ordering::Relaxed) + } + + pub fn get_mem_usage(&self) -> u64 { + self.mem_usage.load(Ordering::Relaxed) + } + + pub fn get_build_dir_free_percent(&self) -> f64 { + self.build_dir_free_percent.load(Ordering::Relaxed) + } + + pub fn get_store_free_percent(&self) -> f64 { + self.store_free_percent.load(Ordering::Relaxed) + } + + pub fn get_current_uploading_path_count(&self) -> u64 { + self.current_uploading_path_count.load(Ordering::Relaxed) + } + + pub fn get_current_downloading_count(&self) -> u64 { + self.current_downloading_count.load(Ordering::Relaxed) + } +} + +struct MachinesInner { + by_uuid: HashMap>, + // by_system is always sorted, as we insert sorted based on cpu score + by_system: HashMap>>, +} + +impl MachinesInner { + fn sort(&mut self, sort_fn: MachineSortFn) { + for machines in self.by_system.values_mut() { + machines.sort_by(|a, b| { + let r = a.score(sort_fn).total_cmp(&b.score(sort_fn)).reverse(); + if r.is_eq() { + // if score is the same then we do a tiebreaker on current jobs + a.stats.get_current_jobs().cmp(&b.stats.get_current_jobs()) + } else { + r + } + }); + } + } +} + +pub struct Machines { + inner: parking_lot::RwLock, + supported_features: parking_lot::RwLock>, +} + +impl Default for Machines { + fn default() -> Self { + Self::new() + } +} + +impl Machines { + pub fn new() -> Self { + Self { + inner: parking_lot::RwLock::new(MachinesInner { + by_uuid: HashMap::with_capacity(10), + by_system: HashMap::with_capacity(10), + }), + supported_features: parking_lot::RwLock::new(HashSet::new()), + } + } + + pub fn sort(&self, sort_fn: MachineSortFn) { + let mut inner = self.inner.write(); + inner.sort(sort_fn); + } + + pub fn get_supported_features(&self) -> Vec { + let supported_features = self.supported_features.read(); + supported_features.iter().cloned().collect() + } + + fn reconstruct_supported_features(&self) { + let all_supported_features = { + let inner = self.inner.read(); + inner + .by_uuid + .values() + .flat_map(|m| m.supported_features.clone()) + .collect::>() + }; + + { + let mut supported_features = self.supported_features.write(); + *supported_features = all_supported_features; + } + } + + #[tracing::instrument(skip(self, machine, sort_fn))] + pub fn insert_machine(&self, machine: Machine, sort_fn: MachineSortFn) -> uuid::Uuid { + let machine_id = machine.id; + { + let mut inner = self.inner.write(); + let machine = Arc::new(machine); + + inner.by_uuid.insert(machine_id, machine.clone()); + { + for system in &machine.systems { + let v = inner.by_system.entry(system.clone()).or_default(); + v.push(machine.clone()); + } + } + inner.sort(sort_fn); + } + self.reconstruct_supported_features(); + machine_id + } + + #[tracing::instrument(skip(self, machine_id))] + pub fn remove_machine(&self, machine_id: uuid::Uuid) -> Option> { + let m = { + let mut inner = self.inner.write(); + inner.by_uuid.remove(&machine_id).map_or_else( + || None, + |m| { + for system in &m.systems { + if let Some(v) = inner.by_system.get_mut(system) { + v.retain(|o| o.id != machine_id); + } + } + Some(m) + }, + ) + }; + self.reconstruct_supported_features(); + m + } + + #[tracing::instrument(skip(self, machine_id))] + pub fn get_machine_by_id(&self, machine_id: uuid::Uuid) -> Option> { + let inner = self.inner.read(); + inner.by_uuid.get(&machine_id).cloned() + } + + pub fn support_step(&self, s: &Arc) -> bool { + let Some(system) = s.get_system() else { + return false; + }; + self.get_machine_for_system(&system, &s.get_required_features(), None) + .is_some() + } + + #[tracing::instrument(skip(self, system))] + pub fn get_machine_for_system( + &self, + system: &str, + required_features: &[String], + free_fn: Option, + ) -> Option> { + // dup of machines.support_step + let inner = self.inner.read(); + if system == "builtin" { + inner + .by_uuid + .values() + .find(|m| { + free_fn.is_none_or(|free_fn| m.has_capacity(free_fn)) + && m.mandatory_features + .iter() + .all(|s| required_features.contains(s)) + && m.supports_all_features(required_features) + }) + .cloned() + } else { + inner.by_system.get(system).and_then(|machines| { + machines + .iter() + .find(|m| { + free_fn.is_none_or(|free_fn| m.has_capacity(free_fn)) + && m.mandatory_features + .iter() + .all(|s| required_features.contains(s)) + && m.supports_all_features(required_features) + }) + .cloned() + }) + } + } + + #[tracing::instrument(skip(self))] + pub fn get_all_machines(&self) -> Vec> { + let inner = self.inner.read(); + inner.by_uuid.values().cloned().collect() + } + + #[tracing::instrument(skip(self))] + pub fn get_machine_count(&self) -> usize { + self.inner.read().by_uuid.len() + } + + #[tracing::instrument(skip(self))] + pub fn get_machine_count_in_use(&self) -> usize { + self.inner + .read() + .by_uuid + .iter() + .filter(|(_, v)| v.stats.get_current_jobs() > 0) + .count() + } + + #[tracing::instrument(skip(self))] + pub async fn publish_new_config(&self, cfg: ConfigUpdate) { + let machines = { + self.inner + .read() + .by_uuid + .values() + .cloned() + .collect::>() + }; + + for m in machines { + let _ = m.publish_config_update(cfg).await; + } + } +} + +#[derive(Debug, Clone)] +pub struct Job { + pub internal_build_id: uuid::Uuid, + pub path: nix_utils::StorePath, + pub resolved_drv: Option, + pub build_id: BuildID, + pub step_nr: i32, + pub result: RemoteBuild, +} + +impl Job { + pub fn new( + build_id: BuildID, + path: nix_utils::StorePath, + resolved_drv: Option, + ) -> Self { + Self { + internal_build_id: uuid::Uuid::new_v4(), + path, + resolved_drv, + build_id, + step_nr: 0, + result: RemoteBuild::new(), + } + } +} + +pub struct PresignedUrlOpts { + pub upload_debug_info: bool, +} + +impl From for crate::server::grpc::runner_v1::PresignedUploadOpts { + fn from(value: PresignedUrlOpts) -> Self { + Self { + upload_debug_info: value.upload_debug_info, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ConfigUpdate { + pub max_concurrent_downloads: u32, +} + +pub enum Message { + ConfigUpdate(ConfigUpdate), + BuildMessage { + build_id: uuid::Uuid, + drv: nix_utils::StorePath, + resolved_drv: Option, + max_log_size: u64, + max_silent_time: i32, + build_timeout: i32, + presigned_url_opts: Option, + }, + AbortMessage { + build_id: uuid::Uuid, + }, +} + +impl Message { + pub fn into_request(self) -> crate::server::grpc::runner_v1::RunnerRequest { + let msg = match self { + Self::ConfigUpdate(m) => runner_request::Message::ConfigUpdate( + crate::server::grpc::runner_v1::ConfigUpdate { + max_concurrent_downloads: m.max_concurrent_downloads, + }, + ), + Self::BuildMessage { + build_id, + drv, + resolved_drv, + max_log_size, + max_silent_time, + build_timeout, + presigned_url_opts, + } => runner_request::Message::Build(BuildMessage { + build_id: build_id.to_string(), + drv: drv.into_base_name(), + resolved_drv: resolved_drv.map(nix_utils::StorePath::into_base_name), + max_log_size, + max_silent_time, + build_timeout, + presigned_url_opts: presigned_url_opts.map(Into::into), + }), + Self::AbortMessage { build_id } => runner_request::Message::Abort(AbortMessage { + build_id: build_id.to_string(), + }), + }; + + crate::server::grpc::runner_v1::RunnerRequest { message: Some(msg) } + } +} + +#[derive(Debug, Clone)] +pub struct Machine { + pub id: uuid::Uuid, + pub systems: SmallVec<[System; 4]>, + pub hostname: String, + pub cpu_count: u32, + pub bogomips: f32, + pub speed_factor: f32, + pub max_jobs: u32, + pub build_dir_avail_threshold: f64, + pub store_avail_threshold: f64, + pub load1_threshold: f32, + pub cpu_psi_threshold: f32, + pub mem_psi_threshold: f32, // If None, dont consider this value + pub io_psi_threshold: Option, // If None, dont consider this value + pub total_mem: u64, + pub supported_features: SmallVec<[String; 8]>, + pub mandatory_features: SmallVec<[String; 4]>, + pub cgroups: bool, + pub substituters: SmallVec<[String; 4]>, + pub use_substitutes: bool, + pub nix_version: String, + pub joined_at: jiff::Timestamp, + + msg_queue: mpsc::Sender, + pub stats: Arc, + pub jobs: Arc>>, +} + +impl std::fmt::Display for Machine { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "Machine: [systems={:?} hostname={} cpu_count={} bogomips={:.2} speed_factor={:.2} max_jobs={} total_mem={:.2} supported_features={:?} cgroups={} joined_at={}]", + self.systems, + self.hostname, + self.cpu_count, + self.bogomips, + self.speed_factor, + self.max_jobs, + byte_unit::Byte::from_u64(self.total_mem).get_adjusted_unit(byte_unit::Unit::GB), + self.supported_features, + self.cgroups, + self.joined_at, + ) + } +} + +impl Machine { + #[tracing::instrument(skip(tx), err)] + pub fn new( + msg: JoinMessage, + tx: mpsc::Sender, + use_presigned_uploads: bool, + forced_substituters: &[String], + ) -> anyhow::Result { + if use_presigned_uploads && !forced_substituters.is_empty() { + if !msg.use_substitutes { + return Err(anyhow::anyhow!( + "Forced_substituters is configured but builder doesnt use substituters. This is an issue because presigned uploads are enabled", + )); + } + + for forced_sub in forced_substituters { + if !msg.substituters.contains(forced_sub) { + return Err(anyhow::anyhow!( + "Builder missing required substituter '{}'. Available: {:?}", + forced_sub, + msg.substituters + )); + } + } + } + + Ok(Self { + id: msg.machine_id.parse()?, + systems: msg.systems.into(), + hostname: msg.hostname, + cpu_count: msg.cpu_count, + bogomips: msg.bogomips, + speed_factor: msg.speed_factor, + max_jobs: msg.max_jobs, + build_dir_avail_threshold: msg.build_dir_avail_threshold.into(), + store_avail_threshold: msg.store_avail_threshold.into(), + load1_threshold: msg.load1_threshold, + cpu_psi_threshold: msg.cpu_psi_threshold, + mem_psi_threshold: msg.mem_psi_threshold, + io_psi_threshold: msg.io_psi_threshold, + total_mem: msg.total_mem, + supported_features: msg.supported_features.into(), + mandatory_features: msg.mandatory_features.into(), + cgroups: msg.cgroups, + substituters: msg.substituters.into(), + use_substitutes: msg.use_substitutes, + nix_version: msg.nix_version, + + msg_queue: tx, + joined_at: jiff::Timestamp::now(), + stats: Arc::new(Stats::new()), + jobs: Arc::new(parking_lot::RwLock::new(Vec::new())), + }) + } + + #[tracing::instrument( + skip(self, job, opts, presigned_url_opts), + fields(build_id=job.build_id, step_nr=job.step_nr), + err, + )] + pub async fn build_drv( + &self, + job: Job, + opts: &nix_utils::BuildOptions, + presigned_url_opts: Option, + ) -> anyhow::Result<()> { + let drv = job.path.clone(); + self.msg_queue + .send(Message::BuildMessage { + build_id: job.internal_build_id, + drv, + resolved_drv: job.resolved_drv.clone(), + max_log_size: opts.get_max_log_size(), + max_silent_time: opts.get_max_silent_time(), + build_timeout: opts.get_build_timeout(), + presigned_url_opts, + }) + .await?; + + if self.stats.jobs_in_last_30s_count.load(Ordering::Relaxed) == 0 { + self.stats + .jobs_in_last_30s_start + .store(jiff::Timestamp::now().as_second(), Ordering::Relaxed); + } + + self.insert_job(job); + self.stats + .jobs_in_last_30s_count + .fetch_add(1, Ordering::Relaxed); + + Ok(()) + } + + #[tracing::instrument(skip(self), fields(build_id=%build_id), err)] + pub async fn abort_build(&self, build_id: uuid::Uuid) -> anyhow::Result<()> { + self.msg_queue + .send(Message::AbortMessage { build_id }) + .await?; + + // dont remove job from machine now, we will do that when the job is set to failed/cancelled + Ok(()) + } + + #[tracing::instrument(skip(self), err)] + pub async fn publish_config_update(&self, change: ConfigUpdate) -> anyhow::Result<()> { + self.msg_queue.send(Message::ConfigUpdate(change)).await?; + Ok(()) + } + + #[must_use] + pub fn has_dynamic_capacity(&self) -> bool { + let pressure = self.stats.pressure.load(); + + if let Some(cpu_some) = pressure.as_ref().and_then(|v| v.cpu_some) { + if cpu_some.avg10 > self.cpu_psi_threshold { + return false; + } + if let Some(mem_full) = pressure.as_ref().and_then(|v| v.mem_full) + && mem_full.avg10 > self.mem_psi_threshold + { + return false; + } + if let Some(threshold) = self.io_psi_threshold + && let Some(io_full) = pressure.as_ref().and_then(|v| v.io_full) + && io_full.avg10 > threshold + { + return false; + } + } else if self.stats.get_load1() > self.load1_threshold { + return false; + } + + true + } + + #[must_use] + pub fn has_static_capacity(&self) -> bool { + self.stats.get_current_jobs() < u64::from(self.max_jobs) + } + + #[must_use] + pub fn has_capacity(&self, free_fn: MachineFreeFn) -> bool { + let now = jiff::Timestamp::now().as_second(); + let jobs_in_last_30s_start = self.stats.jobs_in_last_30s_start.load(Ordering::Relaxed); + let jobs_in_last_30s_count = self.stats.jobs_in_last_30s_count.load(Ordering::Relaxed); + + // ensure that we dont submit more than 4 jobs in 30s + if now <= (jobs_in_last_30s_start + 30) + && jobs_in_last_30s_count >= 4 + // ensure that we havent already finished some of them, because then its fine again + && self.stats.get_current_jobs() >= 4 + { + return false; + } else if now > (jobs_in_last_30s_start + 30) { + // reset count + self.stats + .jobs_in_last_30s_start + .store(0, Ordering::Relaxed); + self.stats + .jobs_in_last_30s_count + .store(0, Ordering::Relaxed); + } + + if self.stats.get_build_dir_free_percent() < self.build_dir_avail_threshold { + return false; + } + + if self.stats.get_store_free_percent() < self.store_avail_threshold { + return false; + } + + match free_fn { + MachineFreeFn::Dynamic => self.has_dynamic_capacity(), + MachineFreeFn::DynamicWithMaxJobLimit => { + self.has_dynamic_capacity() && self.has_static_capacity() + } + MachineFreeFn::Static => self.has_static_capacity(), + } + } + + #[must_use] + pub fn supports_all_features(&self, features: &[String]) -> bool { + // TODO: mandetory features + features.iter().all(|f| self.supported_features.contains(f)) + } + + #[must_use] + pub fn score(&self, sort_fn: MachineSortFn) -> f32 { + match sort_fn { + MachineSortFn::SpeedFactorOnly => self.speed_factor, + MachineSortFn::CpuCoreCountWithSpeedFactor => + { + #[allow(clippy::cast_precision_loss)] + (self.speed_factor * (self.cpu_count as f32)) + } + MachineSortFn::BogomipsWithSpeedFactor => { + let bogomips = if self.bogomips > 1. { + self.bogomips + } else { + 1.0 + }; + #[allow(clippy::cast_precision_loss)] + (self.speed_factor * bogomips * (self.cpu_count as f32)) + } + } + } + + #[tracing::instrument(skip(self), fields(%drv))] + pub fn get_build_id_and_step_nr(&self, drv: &nix_utils::StorePath) -> Option<(i32, i32)> { + let jobs = self.jobs.read(); + jobs.iter() + .find(|j| &j.path == drv) + .map(|j| (j.build_id, j.step_nr)) + } + + #[tracing::instrument(skip(self), fields(%build_id))] + pub fn get_build_id_and_step_nr_by_uuid(&self, build_id: uuid::Uuid) -> Option<(i32, i32)> { + let jobs = self.jobs.read(); + jobs.iter() + .find(|j| j.internal_build_id == build_id) + .map(|j| (j.build_id, j.step_nr)) + } + + #[tracing::instrument(skip(self), fields(%build_id))] + pub fn get_job_drv_for_build_id(&self, build_id: uuid::Uuid) -> Option { + let jobs = self.jobs.read(); + jobs.iter() + .find(|j| j.internal_build_id == build_id) + .map(|v| v.path.clone()) + } + + #[tracing::instrument(skip(self), fields(%drv))] + pub fn get_internal_build_id_for_drv(&self, drv: &nix_utils::StorePath) -> Option { + let jobs = self.jobs.read(); + jobs.iter() + .find(|j| &j.path == drv) + .map(|v| v.internal_build_id) + } + + #[tracing::instrument(skip(self, job))] + fn insert_job(&self, job: Job) { + let mut jobs = self.jobs.write(); + jobs.push(job); + self.stats.store_current_jobs(jobs.len() as u64); + } + + #[tracing::instrument(skip(self), fields(%drv))] + pub fn remove_job(&self, drv: &nix_utils::StorePath) -> Option { + let job = { + let mut jobs = self.jobs.write(); + let job = jobs.iter().find(|j| &j.path == drv).cloned(); + jobs.retain(|j| &j.path != drv); + self.stats.incr_nr_steps_done(); + self.stats.store_current_jobs(jobs.len() as u64); + job + }; + + { + // if build finished fast we can subtract 1 here + let now = jiff::Timestamp::now().as_second(); + let jobs_in_last_30s_start = self.stats.jobs_in_last_30s_start.load(Ordering::Relaxed); + + if now <= (jobs_in_last_30s_start + 30) { + self.stats + .jobs_in_last_30s_count + .fetch_sub(1, Ordering::Relaxed); + } + } + + job + } +} diff --git a/src/queue-runner/src/state/metrics.rs b/src/queue-runner/src/state/metrics.rs new file mode 100644 index 000000000..1b0e01723 --- /dev/null +++ b/src/queue-runner/src/state/metrics.rs @@ -0,0 +1,1204 @@ +use std::sync::Arc; + +use prometheus::Encoder as _; + +use nix_utils::BaseStore as _; + +#[derive(Debug)] +pub struct PromMetrics { + pub queue_runner_current_time_seconds: prometheus::IntGauge, // hydraqueuerunner_current_time_seconds + pub queue_runner_uptime_seconds: prometheus::IntGauge, // hydraqueuerunner_uptime_seconds + + pub queue_checks_started: prometheus::IntCounter, + pub queue_build_loads: prometheus::IntCounter, + pub queue_steps_created: prometheus::IntCounter, + pub queue_checks_early_exits: prometheus::IntCounter, + pub queue_checks_finished: prometheus::IntCounter, + + pub dispatcher_time_spent_running: prometheus::IntCounter, + pub dispatcher_time_spent_waiting: prometheus::IntCounter, + + pub queue_monitor_time_spent_running: prometheus::IntCounter, + pub queue_monitor_time_spent_waiting: prometheus::IntCounter, + + pub nr_builds_read: prometheus::IntCounter, // hydraqueuerunner_builds_read + pub build_read_time_ms: prometheus::IntCounter, // hydraqueuerunner_builds_read_time_ms + pub nr_builds_unfinished: prometheus::IntGauge, // hydraqueuerunner_builds_unfinished + pub nr_builds_done: prometheus::IntCounter, // hydraqueuerunner_builds_finished + pub nr_builds_succeeded: prometheus::IntCounter, // hydraqueuerunner_builds_succeeded + pub nr_builds_failed: prometheus::IntCounter, // hydraqueuerunner_builds_failed + pub nr_steps_started: prometheus::IntCounter, // hydraqueuerunner_steps_started + pub nr_steps_done: prometheus::IntCounter, // hydraqueuerunner_steps_finished + pub nr_steps_building: prometheus::IntGauge, // hydraqueuerunner_steps_building + pub nr_steps_waiting: prometheus::IntGauge, // hydraqueuerunner_steps_waiting + pub nr_steps_runnable: prometheus::IntGauge, // hydraqueuerunner_steps_runnable + pub nr_steps_unfinished: prometheus::IntGauge, // hydraqueuerunner_steps_unfinished + pub nr_unsupported_steps: prometheus::IntGauge, // hydraqueuerunner_steps_unsupported + pub nr_unsupported_steps_aborted: prometheus::IntCounter, // hydraqueuerunner_steps_unsupported_aborted + pub nr_substitutes_started: prometheus::IntCounter, // hydraqueuerunner_substitutes_started + pub nr_substitutes_failed: prometheus::IntCounter, // hydraqueuerunner_substitutes_failed + pub nr_substitutes_succeeded: prometheus::IntCounter, // hydraqueuerunner_substitutes_succeeded + pub nr_retries: prometheus::IntCounter, // hydraqueuerunner_steps_retries + pub max_nr_retries: prometheus::IntGauge, // hydraqueuerunner_steps_max_retries + pub nr_steps_copying_to: prometheus::IntGauge, // hydraqueuerunner_steps_copying_to + pub nr_steps_copying_from: prometheus::IntGauge, // hydraqueuerunner_steps_copying_from + pub avg_step_time_ms: prometheus::IntGauge, // hydraqueuerunner_steps_avg_total_time_ms + pub avg_step_import_time_ms: prometheus::IntGauge, // hydraqueuerunner_steps_avg_import_time_ms + pub avg_step_build_time_ms: prometheus::IntGauge, // hydraqueuerunner_steps_avg_build_time_ms + pub avg_step_upload_time_ms: prometheus::IntGauge, // hydraqueuerunner_steps_avg_upload_time_ms + pub total_step_time_ms: prometheus::IntCounter, // hydraqueuerunner_steps_total_time_ms + pub total_step_import_time_ms: prometheus::IntCounter, // hydraqueuerunner_steps_total_import_time_ms + pub total_step_build_time_ms: prometheus::IntCounter, // hydraqueuerunner_steps_total_build_time_ms + pub total_step_upload_time_ms: prometheus::IntCounter, // hydraqueuerunner_steps_total_upload_time_ms + pub nr_queue_wakeups: prometheus::IntCounter, //hydraqueuerunner_monitor_checks + pub nr_dispatcher_wakeups: prometheus::IntCounter, // hydraqueuerunner_dispatch_wakeup + pub dispatch_time_ms: prometheus::IntCounter, // hydraqueuerunner_dispatch_time_ms + pub machines_total: prometheus::IntGauge, // hydraqueuerunner_machines_total + pub machines_in_use: prometheus::IntGauge, // hydraqueuerunner_machines_in_use + + // Per-machine-type metrics + pub runnable_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_runnable + pub running_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_running + pub waiting_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_waiting + pub disabled_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_disabled + pub avg_runnable_time_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_avg_runnable_time + pub wait_time_per_machine_type: prometheus::IntGaugeVec, // hydraqueuerunner_machine_type_wait_time + + // Per-machine metrics + pub machine_current_jobs: prometheus::IntGaugeVec, // hydraqueuerunner_machine_current_jobs + pub machine_steps_done: prometheus::IntGaugeVec, // hydraqueuerunner_machine_steps_done + pub machine_total_step_time_ms: prometheus::IntGaugeVec, // hydraqueuerunner_machine_total_step_time_ms + pub machine_total_step_import_time_ms: prometheus::IntGaugeVec, // hydraqueuerunner_machine_total_step_import_time_ms + pub machine_total_step_build_time_ms: prometheus::IntGaugeVec, // hydraqueuerunner_machine_total_step_build_time_ms + pub machine_total_step_upload_time_ms: prometheus::IntGaugeVec, // hydraqueuerunner_machine_total_step_upload_time_ms + pub machine_consecutive_failures: prometheus::IntGaugeVec, // hydraqueuerunner_machine_consecutive_failures + pub machine_last_ping_timestamp: prometheus::IntGaugeVec, // hydraqueuerunner_machine_last_ping_timestamp + pub machine_idle_since_timestamp: prometheus::IntGaugeVec, // hydraqueuerunner_machine_idle_since_timestamp + + // Store metrics (single store) + pub store_nar_info_read: prometheus::IntGauge, // hydraqueuerunner_store_nar_info_read + pub store_nar_info_read_averted: prometheus::IntGauge, // hydraqueuerunner_store_nar_info_read_averted + pub store_nar_info_missing: prometheus::IntGauge, // hydraqueuerunner_store_nar_info_missing + pub store_nar_info_write: prometheus::IntGauge, // hydraqueuerunner_store_nar_info_write + pub store_path_info_cache_size: prometheus::IntGauge, // hydraqueuerunner_store_path_info_cache_size + pub store_nar_read: prometheus::IntGauge, // hydraqueuerunner_store_nar_read + pub store_nar_read_bytes: prometheus::IntGauge, // hydraqueuerunner_store_nar_read_bytes + pub store_nar_read_compressed_bytes: prometheus::IntGauge, // hydraqueuerunner_store_nar_read_compressed_bytes + pub store_nar_write: prometheus::IntGauge, // hydraqueuerunner_store_nar_write + pub store_nar_write_averted: prometheus::IntGauge, // hydraqueuerunner_store_nar_write_averted + pub store_nar_write_bytes: prometheus::IntGauge, // hydraqueuerunner_store_nar_write_bytes + pub store_nar_write_compressed_bytes: prometheus::IntGauge, // hydraqueuerunner_store_nar_write_compressed_bytes + pub store_nar_write_compression_time_ms: prometheus::IntGauge, // hydraqueuerunner_store_nar_write_compression_time_ms + pub store_nar_compression_savings: prometheus::Gauge, // hydraqueuerunner_store_nar_compression_savings + pub store_nar_compression_speed: prometheus::Gauge, // hydraqueuerunner_store_nar_compression_speed + + // S3 metrics (multiple backends) + pub s3_put: prometheus::IntGaugeVec, // hydraqueuerunner_s3_put + pub s3_put_bytes: prometheus::IntGaugeVec, // hydraqueuerunner_s3_put_bytes + pub s3_put_time_ms: prometheus::IntGaugeVec, // hydraqueuerunner_s3_put_time_ms + pub s3_put_speed: prometheus::GaugeVec, // hydraqueuerunner_s3_put_speed + pub s3_get: prometheus::IntGaugeVec, // hydraqueuerunner_s3_get + pub s3_get_bytes: prometheus::IntGaugeVec, // hydraqueuerunner_s3_get_bytes + pub s3_get_time_ms: prometheus::IntGaugeVec, // hydraqueuerunner_s3_get_time_ms + pub s3_get_speed: prometheus::GaugeVec, // hydraqueuerunner_s3_get_speed + pub s3_head: prometheus::IntGaugeVec, // hydraqueuerunner_s3_head + pub s3_cost_dollar_approx: prometheus::GaugeVec, // hydraqueuerunner_s3_cost_dollar_approx + + // Build dependency and complexity metrics + pub build_input_drvs_histogram: prometheus::HistogramVec, // hydraqueuerunner_build_input_drvs_seconds + pub build_closure_size_bytes_histogram: prometheus::HistogramVec, // hydraqueuerunner_build_closure_size_bytes + + // Queue performance metrics + pub queue_sort_duration_ms_total: prometheus::IntCounter, // hydraqueuerunner_sort_duration_ms_total + pub queue_job_wait_time_histogram: prometheus::HistogramVec, // hydraqueuerunner_job_wait_time_seconds + pub queue_aborted_jobs_total: prometheus::IntCounter, // hydraqueuerunner_aborted_jobs_total + + // Jobset metrics + pub jobset_share_used: prometheus::IntGaugeVec, // hydraqueuerunner_jobset_share_used + pub jobset_seconds: prometheus::IntGaugeVec, // hydraqueuerunner_jobset_seconds +} + +impl PromMetrics { + #[allow(clippy::too_many_lines)] + #[tracing::instrument(err)] + pub fn new() -> anyhow::Result { + let queue_checks_started = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_queue_checks_started_total", + "Number of times State::get_queued_builds() was started", + ))?; + let queue_build_loads = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_queue_build_loads_total", + "Number of builds loaded", + ))?; + let queue_steps_created = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_queue_steps_created_total", + "Number of steps created", + ))?; + let queue_checks_early_exits = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_queue_checks_early_exits_total", + "Number of times State::get_queued_builds() yielded to potential bumps", + ))?; + let queue_checks_finished = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_queue_checks_finished_total", + "Number of times State::get_queued_builds() was completed", + ))?; + let dispatcher_time_spent_running = + prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_dispatcher_time_spent_running_total", + "Time (in micros) spent running the dispatcher", + ))?; + let dispatcher_time_spent_waiting = + prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_dispatcher_time_spent_waiting_total", + "Time (in micros) spent waiting for the dispatcher to obtain work", + ))?; + let queue_monitor_time_spent_running = + prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_monitor_time_spent_running_total", + "Time (in micros) spent running the queue monitor", + ))?; + let queue_monitor_time_spent_waiting = + prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_monitor_time_spent_waiting_total", + "Time (in micros) spent waiting for the queue monitor to obtain work", + ))?; + + let nr_builds_read = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_builds_read", + "Number of builds read from database", + ))?; + let build_read_time_ms = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_builds_read_time_ms", + "Time in milliseconds spent reading builds from database", + ))?; + let nr_builds_unfinished = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_builds_unfinished", + "Number of unfinished builds in the queue", + ))?; + let nr_builds_done = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_builds_finished", + "Number of finished builds in the queue", + ))?; + let nr_builds_succeeded = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_builds_succeeded", + "Number of successful builds in the queue", + ))?; + let nr_builds_failed = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_builds_failed", + "Number of failed builds in the queue", + ))?; + let nr_steps_started = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_started", + "Number of build steps that have been started", + ))?; + let nr_steps_done = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_finished", + "Number of build steps that have been completed", + ))?; + let nr_steps_building = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_building", + "Number of build steps currently being built", + ))?; + let nr_steps_waiting = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_waiting", + "Number of build steps waiting to be built", + ))?; + let nr_steps_runnable = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_runnable", + "Number of build steps that are ready to run", + ))?; + let nr_steps_unfinished = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_unfinished", + "Number of unfinished build steps", + ))?; + let nr_unsupported_steps = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_unsupported", + "Number of unsupported build steps", + ))?; + let nr_unsupported_steps_aborted = + prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_unsupported_aborted", + "Number of unsupported build steps that were aborted", + ))?; + let nr_substitutes_started = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_substitutes_started", + "Number of substitute downloads that have been started", + ))?; + let nr_substitutes_failed = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_substitutes_failed", + "Number of substitute downloads that have failed", + ))?; + let nr_substitutes_succeeded = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_substitutes_succeeded", + "Number of substitute downloads that have succeeded", + ))?; + let nr_retries = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_retries", + "Number of retries for build steps", + ))?; + let max_nr_retries = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_max_retries", + "Maximum number of retries allowed for build steps", + ))?; + let nr_steps_copying_to = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_copying_to", + "Number of build steps currently copying inputs to machines", + ))?; + let nr_steps_copying_from = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_copying_from", + "Number of build steps currently copying outputs from machines", + ))?; + let avg_step_time_ms = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_avg_total_time_ms", + "Average time in milliseconds for build steps to complete", + ))?; + let avg_step_import_time_ms = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_avg_import_time_ms", + "Average time in milliseconds for importing build steps", + ))?; + let avg_step_build_time_ms = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_avg_build_time_ms", + "Average time in milliseconds for building build steps", + ))?; + let avg_step_upload_time_ms = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_avg_upload_time_ms", + "Average time in milliseconds for uploading build steps", + ))?; + let total_step_time_ms = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_total_time_ms", + "Total time in milliseconds spent on all build steps", + ))?; + let total_step_import_time_ms = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_total_import_time_ms", + "Total time in milliseconds spent importing all build steps", + ))?; + let total_step_build_time_ms = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_total_build_time_ms", + "Total time in milliseconds spent building all build steps", + ))?; + let total_step_upload_time_ms = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_steps_total_upload_time_ms", + "Total time in milliseconds spent uploading all build steps", + ))?; + let nr_queue_wakeups = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_monitor_checks", + "Number of times the queue monitor has been woken up", + ))?; + let nr_dispatcher_wakeups = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_dispatch_wakeup", + "Number of times the dispatcher has been woken up", + ))?; + let dispatch_time_ms = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_dispatch_time_ms", + "Time in milliseconds spent dispatching build steps", + ))?; + let machines_total = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_machines_total", + "Total number of machines available for building", + ))?; + let machines_in_use = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_machines_in_use", + "Number of machines currently in use for building", + ))?; + + // Per-machine-type metrics + let runnable_per_machine_type = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_type_runnable", + "Number of runnable build steps per machine type", + ), + &["machine_type"], + )?; + let running_per_machine_type = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_type_running", + "Number of running build steps per machine type", + ), + &["machine_type"], + )?; + let waiting_per_machine_type = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_type_waiting", + "Number of waiting build steps per machine type", + ), + &["machine_type"], + )?; + let disabled_per_machine_type = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_type_disabled", + "Number of disabled build steps per machine type", + ), + &["machine_type"], + )?; + let avg_runnable_time_per_machine_type = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_type_avg_runnable_time", + "Average runnable time for build steps per machine type", + ), + &["machine_type"], + )?; + let wait_time_per_machine_type = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_type_wait_time", + "Wait time for build steps per machine type", + ), + &["machine_type"], + )?; + + // Per-machine metrics + let machine_current_jobs = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_current_jobs", + "Number of currently running jobs on each machine", + ), + &["hostname"], + )?; + let machine_steps_done = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_steps_done", + "Total number of steps completed by each machine", + ), + &["hostname"], + )?; + let machine_total_step_time_ms = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_total_step_time_ms", + "Total time in milliseconds spent on all steps by each machine", + ), + &["hostname"], + )?; + let machine_total_step_import_time_ms = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_total_step_import_time_ms", + "Total time in milliseconds spent importing steps by each machine", + ), + &["hostname"], + )?; + let machine_total_step_build_time_ms = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_total_step_build_time_ms", + "Total time in milliseconds spent building steps by each machine", + ), + &["hostname"], + )?; + let machine_total_step_upload_time_ms = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_total_step_upload_time_ms", + "Total time in milliseconds spent uploading steps by each machine", + ), + &["hostname"], + )?; + let machine_consecutive_failures = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_consecutive_failures", + "Number of consecutive failures for each machine", + ), + &["hostname"], + )?; + let machine_last_ping_timestamp = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_last_ping_timestamp", + "Unix timestamp of the last ping received from each machine", + ), + &["hostname"], + )?; + let machine_idle_since_timestamp = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_machine_idle_since_timestamp", + "Unix timestamp since when each machine has been idle (0 if currently busy)", + ), + &["hostname"], + )?; + + // Store metrics (single store) + let store_nar_info_read = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_info_read", + "Number of NAR info reads from store", + ))?; + let store_nar_info_read_averted = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_info_read_averted", + "Number of NAR info reads averted from store", + ))?; + let store_nar_info_missing = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_info_missing", + "Number of missing NAR info in store", + ))?; + let store_nar_info_write = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_info_write", + "Number of NAR info writes to store", + ))?; + let store_path_info_cache_size = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_path_info_cache_size", + "Size of path info cache in store", + ))?; + let store_nar_read = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_read", + "Number of NAR reads from store", + ))?; + let store_nar_read_bytes = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_read_bytes", + "Number of bytes read from NARs in store", + ))?; + let store_nar_read_compressed_bytes = + prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_read_compressed_bytes", + "Number of compressed bytes read from NARs in store", + ))?; + let store_nar_write = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_write", + "Number of NAR writes to store", + ))?; + let store_nar_write_averted = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_write_averted", + "Number of NAR writes averted to store", + ))?; + let store_nar_write_bytes = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_write_bytes", + "Number of bytes written to NARs in store", + ))?; + let store_nar_write_compressed_bytes = + prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_write_compressed_bytes", + "Number of compressed bytes written to NARs in store", + ))?; + let store_nar_write_compression_time_ms = + prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_write_compression_time_ms", + "Time in milliseconds spent compressing NARs in store", + ))?; + let store_nar_compression_savings = prometheus::Gauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_compression_savings", + "Compression savings ratio for NARs in store", + ))?; + let store_nar_compression_speed = prometheus::Gauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_store_nar_compression_speed", + "Compression speed for NARs in store", + ))?; + + // S3 metrics (multiple backends) + let s3_put = prometheus::IntGaugeVec::new( + prometheus::Opts::new("hydraqueuerunner_s3_put", "Number of S3 put operations"), + &["remote_store"], + )?; + let s3_put_bytes = prometheus::IntGaugeVec::new( + prometheus::Opts::new("hydraqueuerunner_s3_put_bytes", "Number of bytes put to S3"), + &["remote_store"], + )?; + let s3_put_time_ms = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_s3_put_time_ms", + "Time in milliseconds spent on S3 put operations", + ), + &["remote_store"], + )?; + let s3_put_speed = prometheus::GaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_s3_put_speed", + "Speed of S3 put operations", + ), + &["remote_store"], + )?; + let s3_get = prometheus::IntGaugeVec::new( + prometheus::Opts::new("hydraqueuerunner_s3_get", "Number of S3 get operations"), + &["remote_store"], + )?; + let s3_get_bytes = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_s3_get_bytes", + "Number of bytes gotten from S3", + ), + &["remote_store"], + )?; + let s3_get_time_ms = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_s3_get_time_ms", + "Time in milliseconds spent on S3 get operations", + ), + &["remote_store"], + )?; + let s3_get_speed = prometheus::GaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_s3_get_speed", + "Speed of S3 get operations", + ), + &["remote_store"], + )?; + let s3_head = prometheus::IntGaugeVec::new( + prometheus::Opts::new("hydraqueuerunner_s3_head", "Number of S3 head operations"), + &["remote_store"], + )?; + let s3_cost_dollar_approx = prometheus::GaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_s3_cost_dollar_approx", + "Approximate cost in dollars for S3 operations", + ), + &["remote_store"], + )?; + + // Build dependency and complexity metrics + let build_input_drvs_histogram = prometheus::HistogramVec::new( + prometheus::HistogramOpts::new( + "hydraqueuerunner_build_input_drvs_seconds", + "Distribution of number of input derivations per build", + ) + .buckets(vec![ + 0.0, + 1.0, + 5.0, + 10.0, + 25.0, + 50.0, + 100.0, + 250.0, + 500.0, + f64::INFINITY, + ]), + &["machine_type"], + )?; + let build_closure_size_bytes_histogram = prometheus::HistogramVec::new( + prometheus::HistogramOpts::new( + "hydraqueuerunner_build_closure_size_bytes", + "Distribution of build closure sizes in bytes", + ) + .buckets(vec![ + 1000.0, + 10_000.0, + 100_000.0, + 1_000_000.0, + 10_000_000.0, + 100_000_000.0, + 1_000_000_000.0, + f64::INFINITY, + ]), + &["machine_type"], + )?; + + // Queue performance metrics + let queue_sort_duration_ms_total = + prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_sort_duration_ms_total", + "Total time in milliseconds spent sorting jobs in queues", + ))?; + let queue_job_wait_time_histogram = prometheus::HistogramVec::new( + prometheus::HistogramOpts::new( + "hydraqueuerunner_job_wait_time_seconds", + "Distribution of time jobs wait in queue before being scheduled", + ) + .buckets(vec![ + 1.0, + 10.0, + 60.0, + 300.0, + 900.0, + 3600.0, + 7200.0, + 86400.0, + f64::INFINITY, + ]), + &["machine_type"], + )?; + + let queue_aborted_jobs_total = prometheus::IntCounter::with_opts(prometheus::Opts::new( + "hydraqueuerunner_aborted_jobs_total", + "Total number of jobs that were aborted", + ))?; + + // Jobset metrics + let jobset_share_used = prometheus::IntGaugeVec::new( + prometheus::Opts::new("hydraqueuerunner_jobset_share_used", "Share used by jobset"), + &["jobset_name"], + )?; + let jobset_seconds = prometheus::IntGaugeVec::new( + prometheus::Opts::new( + "hydraqueuerunner_jobset_seconds", + "Seconds allocated to jobset", + ), + &["jobset_name"], + )?; + + // Queue runner time metrics + let queue_runner_current_time_seconds = + prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_current_time_seconds", + "Current Unix timestamp in seconds", + ))?; + let queue_runner_uptime_seconds = prometheus::IntGauge::with_opts(prometheus::Opts::new( + "hydraqueuerunner_uptime_seconds", + "Queue runner uptime in seconds", + ))?; + + let r = prometheus::default_registry(); + r.register(Box::new(queue_checks_started.clone()))?; + r.register(Box::new(queue_build_loads.clone()))?; + r.register(Box::new(queue_steps_created.clone()))?; + r.register(Box::new(queue_checks_early_exits.clone()))?; + r.register(Box::new(queue_checks_finished.clone()))?; + r.register(Box::new(dispatcher_time_spent_running.clone()))?; + r.register(Box::new(dispatcher_time_spent_waiting.clone()))?; + r.register(Box::new(queue_monitor_time_spent_running.clone()))?; + r.register(Box::new(queue_monitor_time_spent_waiting.clone()))?; + r.register(Box::new(nr_builds_read.clone()))?; + r.register(Box::new(build_read_time_ms.clone()))?; + r.register(Box::new(nr_builds_unfinished.clone()))?; + r.register(Box::new(nr_builds_done.clone()))?; + r.register(Box::new(nr_builds_succeeded.clone()))?; + r.register(Box::new(nr_builds_failed.clone()))?; + r.register(Box::new(nr_steps_started.clone()))?; + r.register(Box::new(nr_steps_done.clone()))?; + r.register(Box::new(nr_steps_building.clone()))?; + r.register(Box::new(nr_steps_waiting.clone()))?; + r.register(Box::new(nr_steps_runnable.clone()))?; + r.register(Box::new(nr_steps_unfinished.clone()))?; + r.register(Box::new(nr_unsupported_steps.clone()))?; + r.register(Box::new(nr_unsupported_steps_aborted.clone()))?; + r.register(Box::new(nr_substitutes_started.clone()))?; + r.register(Box::new(nr_substitutes_failed.clone()))?; + r.register(Box::new(nr_substitutes_succeeded.clone()))?; + r.register(Box::new(nr_retries.clone()))?; + r.register(Box::new(max_nr_retries.clone()))?; + r.register(Box::new(nr_steps_copying_to.clone()))?; + r.register(Box::new(nr_steps_copying_from.clone()))?; + r.register(Box::new(avg_step_time_ms.clone()))?; + r.register(Box::new(avg_step_import_time_ms.clone()))?; + r.register(Box::new(avg_step_build_time_ms.clone()))?; + r.register(Box::new(avg_step_upload_time_ms.clone()))?; + r.register(Box::new(total_step_time_ms.clone()))?; + r.register(Box::new(total_step_import_time_ms.clone()))?; + r.register(Box::new(total_step_build_time_ms.clone()))?; + r.register(Box::new(total_step_upload_time_ms.clone()))?; + r.register(Box::new(nr_queue_wakeups.clone()))?; + r.register(Box::new(nr_dispatcher_wakeups.clone()))?; + r.register(Box::new(dispatch_time_ms.clone()))?; + r.register(Box::new(machines_total.clone()))?; + r.register(Box::new(machines_in_use.clone()))?; + r.register(Box::new(runnable_per_machine_type.clone()))?; + r.register(Box::new(running_per_machine_type.clone()))?; + r.register(Box::new(waiting_per_machine_type.clone()))?; + r.register(Box::new(disabled_per_machine_type.clone()))?; + r.register(Box::new(avg_runnable_time_per_machine_type.clone()))?; + r.register(Box::new(wait_time_per_machine_type.clone()))?; + r.register(Box::new(machine_current_jobs.clone()))?; + r.register(Box::new(machine_steps_done.clone()))?; + r.register(Box::new(machine_total_step_time_ms.clone()))?; + r.register(Box::new(machine_total_step_import_time_ms.clone()))?; + r.register(Box::new(machine_total_step_build_time_ms.clone()))?; + r.register(Box::new(machine_total_step_upload_time_ms.clone()))?; + r.register(Box::new(machine_consecutive_failures.clone()))?; + r.register(Box::new(machine_last_ping_timestamp.clone()))?; + r.register(Box::new(machine_idle_since_timestamp.clone()))?; + + // Store metrics + r.register(Box::new(store_nar_info_read.clone()))?; + r.register(Box::new(store_nar_info_read_averted.clone()))?; + r.register(Box::new(store_nar_info_missing.clone()))?; + r.register(Box::new(store_nar_info_write.clone()))?; + r.register(Box::new(store_path_info_cache_size.clone()))?; + r.register(Box::new(store_nar_read.clone()))?; + r.register(Box::new(store_nar_read_bytes.clone()))?; + r.register(Box::new(store_nar_read_compressed_bytes.clone()))?; + r.register(Box::new(store_nar_write.clone()))?; + r.register(Box::new(store_nar_write_averted.clone()))?; + r.register(Box::new(store_nar_write_bytes.clone()))?; + r.register(Box::new(store_nar_write_compressed_bytes.clone()))?; + r.register(Box::new(store_nar_write_compression_time_ms.clone()))?; + r.register(Box::new(store_nar_compression_savings.clone()))?; + r.register(Box::new(store_nar_compression_speed.clone()))?; + + // S3 metrics + r.register(Box::new(s3_put.clone()))?; + r.register(Box::new(s3_put_bytes.clone()))?; + r.register(Box::new(s3_put_time_ms.clone()))?; + r.register(Box::new(s3_put_speed.clone()))?; + r.register(Box::new(s3_get.clone()))?; + r.register(Box::new(s3_get_bytes.clone()))?; + r.register(Box::new(s3_get_time_ms.clone()))?; + r.register(Box::new(s3_get_speed.clone()))?; + r.register(Box::new(s3_head.clone()))?; + r.register(Box::new(s3_cost_dollar_approx.clone()))?; + + // Build dependency and complexity metrics + r.register(Box::new(build_input_drvs_histogram.clone()))?; + r.register(Box::new(build_closure_size_bytes_histogram.clone()))?; + + // Queue performance metrics + r.register(Box::new(queue_sort_duration_ms_total.clone()))?; + r.register(Box::new(queue_job_wait_time_histogram.clone()))?; + r.register(Box::new(queue_aborted_jobs_total.clone()))?; + + // Jobset metrics + r.register(Box::new(jobset_share_used.clone()))?; + r.register(Box::new(jobset_seconds.clone()))?; + + // Queue runner time metrics + r.register(Box::new(queue_runner_current_time_seconds.clone()))?; + r.register(Box::new(queue_runner_uptime_seconds.clone()))?; + + Ok(Self { + queue_runner_current_time_seconds, + queue_runner_uptime_seconds, + queue_checks_started, + queue_build_loads, + queue_steps_created, + queue_checks_early_exits, + queue_checks_finished, + dispatcher_time_spent_running, + dispatcher_time_spent_waiting, + queue_monitor_time_spent_running, + queue_monitor_time_spent_waiting, + nr_builds_read, + build_read_time_ms, + nr_builds_unfinished, + nr_builds_done, + nr_builds_succeeded, + nr_builds_failed, + nr_steps_started, + nr_steps_done, + nr_steps_building, + nr_steps_waiting, + nr_steps_runnable, + nr_steps_unfinished, + nr_unsupported_steps, + nr_unsupported_steps_aborted, + nr_substitutes_started, + nr_substitutes_failed, + nr_substitutes_succeeded, + nr_retries, + max_nr_retries, + nr_steps_copying_to, + nr_steps_copying_from, + avg_step_time_ms, + avg_step_import_time_ms, + avg_step_build_time_ms, + avg_step_upload_time_ms, + total_step_time_ms, + total_step_import_time_ms, + total_step_build_time_ms, + total_step_upload_time_ms, + nr_queue_wakeups, + nr_dispatcher_wakeups, + dispatch_time_ms, + machines_total, + machines_in_use, + runnable_per_machine_type, + running_per_machine_type, + waiting_per_machine_type, + disabled_per_machine_type, + avg_runnable_time_per_machine_type, + wait_time_per_machine_type, + machine_current_jobs, + machine_steps_done, + machine_total_step_time_ms, + machine_total_step_import_time_ms, + machine_total_step_build_time_ms, + machine_total_step_upload_time_ms, + machine_consecutive_failures, + machine_last_ping_timestamp, + machine_idle_since_timestamp, + + // Store metrics + store_nar_info_read, + store_nar_info_read_averted, + store_nar_info_missing, + store_nar_info_write, + store_path_info_cache_size, + store_nar_read, + store_nar_read_bytes, + store_nar_read_compressed_bytes, + store_nar_write, + store_nar_write_averted, + store_nar_write_bytes, + store_nar_write_compressed_bytes, + store_nar_write_compression_time_ms, + store_nar_compression_savings, + store_nar_compression_speed, + + // S3 metrics + s3_put, + s3_put_bytes, + s3_put_time_ms, + s3_put_speed, + s3_get, + s3_get_bytes, + s3_get_time_ms, + s3_get_speed, + s3_head, + s3_cost_dollar_approx, + + // Build dependency and complexity metrics + build_input_drvs_histogram, + build_closure_size_bytes_histogram, + + // Queue performance metrics + queue_sort_duration_ms_total, + queue_job_wait_time_histogram, + queue_aborted_jobs_total, + + // Jobset metrics + jobset_share_used, + jobset_seconds, + }) + } + + pub async fn refresh_dynamic_metrics(&self, state: &Arc) { + let nr_steps_done = self.nr_steps_done.get(); + if nr_steps_done > 0 { + let avg_time = self.total_step_time_ms.get() / nr_steps_done; + let avg_import_time = self.total_step_import_time_ms.get() / nr_steps_done; + let avg_build_time = self.total_step_build_time_ms.get() / nr_steps_done; + let avg_upload_time = self.total_step_upload_time_ms.get() / nr_steps_done; + + if let Ok(v) = i64::try_from(avg_time) { + self.avg_step_time_ms.set(v); + } + if let Ok(v) = i64::try_from(avg_import_time) { + self.avg_step_import_time_ms.set(v); + } + if let Ok(v) = i64::try_from(avg_upload_time) { + self.avg_step_upload_time_ms.set(v); + } + if let Ok(v) = i64::try_from(avg_build_time) { + self.avg_step_build_time_ms.set(v); + } + } + + if let Ok(v) = i64::try_from(state.builds.len()) { + self.nr_builds_unfinished.set(v); + } + if let Ok(v) = i64::try_from(state.steps.len()) { + self.nr_steps_unfinished.set(v); + } + if let Ok(v) = i64::try_from(state.steps.len_runnable()) { + self.nr_steps_runnable.set(v); + } + if let Ok(v) = i64::try_from(state.machines.get_machine_count()) { + self.machines_total.set(v); + } + if let Ok(v) = i64::try_from(state.machines.get_machine_count_in_use()) { + self.machines_in_use.set(v); + } + + self.refresh_per_machine_type_metrics(state).await; + self.refresh_per_machine_metrics(state); + self.refresh_store_metrics(state); + self.refresh_s3_metrics(state); + self.refresh_transfer_metrics(state); + self.refresh_jobset_metrics(state); + self.refresh_time_metrics(state); + } + + async fn refresh_per_machine_type_metrics(&self, state: &Arc) { + self.runnable_per_machine_type.reset(); + self.running_per_machine_type.reset(); + self.waiting_per_machine_type.reset(); + self.disabled_per_machine_type.reset(); + self.avg_runnable_time_per_machine_type.reset(); + self.wait_time_per_machine_type.reset(); + for (t, s) in state.queues.get_stats_per_queue().await { + if let Ok(v) = i64::try_from(s.total_runnable) { + self.runnable_per_machine_type + .with_label_values(std::slice::from_ref(&t)) + .set(v); + } + if let Ok(v) = i64::try_from(s.active_runnable) { + self.running_per_machine_type + .with_label_values(&[&t]) + .set(v); + } + if let Ok(v) = i64::try_from(s.nr_runnable_waiting) { + self.waiting_per_machine_type + .with_label_values(&[&t]) + .set(v); + } + if let Ok(v) = i64::try_from(s.nr_runnable_disabled) { + self.disabled_per_machine_type + .with_label_values(&[&t]) + .set(v); + } + if let Ok(v) = i64::try_from(s.avg_runnable_time) { + self.avg_runnable_time_per_machine_type + .with_label_values(&[&t]) + .set(v); + } + if let Ok(v) = i64::try_from(s.wait_time) { + self.wait_time_per_machine_type + .with_label_values(&[&t]) + .set(v); + } + } + } + + fn refresh_per_machine_metrics(&self, state: &Arc) { + self.machine_current_jobs.reset(); + self.machine_steps_done.reset(); + self.machine_total_step_time_ms.reset(); + self.machine_total_step_import_time_ms.reset(); + self.machine_total_step_build_time_ms.reset(); + self.machine_total_step_upload_time_ms.reset(); + self.machine_consecutive_failures.reset(); + self.machine_last_ping_timestamp.reset(); + self.machine_idle_since_timestamp.reset(); + + for machine in state.machines.get_all_machines() { + let hostname = &machine.hostname; + + let labels = &[hostname]; + + if let Ok(v) = i64::try_from(machine.stats.get_current_jobs()) { + self.machine_current_jobs.with_label_values(labels).set(v); + } + + if let Ok(v) = i64::try_from(machine.stats.get_nr_steps_done()) { + self.machine_steps_done.with_label_values(labels).set(v); + } + if let Ok(v) = i64::try_from(machine.stats.get_total_step_time_ms()) { + self.machine_total_step_time_ms + .with_label_values(labels) + .set(v); + } + if let Ok(v) = i64::try_from(machine.stats.get_total_step_import_time_ms()) { + self.machine_total_step_import_time_ms + .with_label_values(labels) + .set(v); + } + if let Ok(v) = i64::try_from(machine.stats.get_total_step_build_time_ms()) { + self.machine_total_step_build_time_ms + .with_label_values(labels) + .set(v); + } + if let Ok(v) = i64::try_from(machine.stats.get_total_step_upload_time_ms()) { + self.machine_total_step_upload_time_ms + .with_label_values(labels) + .set(v); + } + + if let Ok(v) = i64::try_from(machine.stats.get_consecutive_failures()) { + self.machine_consecutive_failures + .with_label_values(labels) + .set(v); + } + self.machine_last_ping_timestamp + .with_label_values(labels) + .set(machine.stats.get_last_ping()); + self.machine_idle_since_timestamp + .with_label_values(labels) + .set(machine.stats.get_idle_since()); + } + } + + fn refresh_store_metrics(&self, state: &Arc) { + if let Ok(store_stats) = state.store.get_store_stats() { + if let Ok(v) = i64::try_from(store_stats.nar_info_read) { + self.store_nar_info_read.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_info_read_averted) { + self.store_nar_info_read_averted.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_info_missing) { + self.store_nar_info_missing.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_info_write) { + self.store_nar_info_write.set(v); + } + if let Ok(v) = i64::try_from(store_stats.path_info_cache_size) { + self.store_path_info_cache_size.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_read) { + self.store_nar_read.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_read_bytes) { + self.store_nar_read_bytes.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_read_compressed_bytes) { + self.store_nar_read_compressed_bytes.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_write) { + self.store_nar_write.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_write_averted) { + self.store_nar_write_averted.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_write_bytes) { + self.store_nar_write_bytes.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_write_compressed_bytes) { + self.store_nar_write_compressed_bytes.set(v); + } + if let Ok(v) = i64::try_from(store_stats.nar_write_compression_time_ms) { + self.store_nar_write_compression_time_ms.set(v); + } + self.store_nar_compression_savings + .set(store_stats.nar_compression_savings()); + self.store_nar_compression_speed + .set(store_stats.nar_compression_speed()); + } + } + + fn refresh_s3_metrics(&self, state: &Arc) { + self.s3_put.reset(); + self.s3_put_bytes.reset(); + self.s3_put_time_ms.reset(); + self.s3_put_speed.reset(); + self.s3_get.reset(); + self.s3_get_bytes.reset(); + self.s3_get_time_ms.reset(); + self.s3_get_speed.reset(); + self.s3_head.reset(); + self.s3_cost_dollar_approx.reset(); + + let s3_backends = state.remote_stores.read(); + for remote_store in s3_backends.iter() { + let backend_name = &remote_store.cfg.client_config.bucket; + let s3_stats = remote_store.s3_stats(); + let labels = &[backend_name.as_str()]; + + if let Ok(v) = i64::try_from(s3_stats.put) { + self.s3_put.with_label_values(labels).set(v); + } + if let Ok(v) = i64::try_from(s3_stats.put_bytes) { + self.s3_put_bytes.with_label_values(labels).set(v); + } + if let Ok(v) = i64::try_from(s3_stats.put_time_ms) { + self.s3_put_time_ms.with_label_values(labels).set(v); + } + self.s3_put_speed + .with_label_values(labels) + .set(s3_stats.put_speed()); + if let Ok(v) = i64::try_from(s3_stats.get) { + self.s3_get.with_label_values(labels).set(v); + } + if let Ok(v) = i64::try_from(s3_stats.get_bytes) { + self.s3_get_bytes.with_label_values(labels).set(v); + } + if let Ok(v) = i64::try_from(s3_stats.get_time_ms) { + self.s3_get_time_ms.with_label_values(labels).set(v); + } + self.s3_get_speed + .with_label_values(labels) + .set(s3_stats.get_speed()); + if let Ok(v) = i64::try_from(s3_stats.head) { + self.s3_head.with_label_values(labels).set(v); + } + self.s3_cost_dollar_approx + .with_label_values(labels) + .set(s3_stats.cost_dollar_approx()); + } + } + + fn refresh_transfer_metrics(&self, state: &Arc) { + let mut total_uploading_path_count = 0u64; + let mut total_downloading_path_count = 0u64; + + for machine in state.machines.get_all_machines() { + total_uploading_path_count += machine.stats.get_current_uploading_path_count(); + total_downloading_path_count += machine.stats.get_current_downloading_count(); + } + + if let Ok(v) = i64::try_from(total_uploading_path_count) { + self.nr_steps_copying_to.set(v); + } + if let Ok(v) = i64::try_from(total_downloading_path_count) { + self.nr_steps_copying_from.set(v); + } + } + + fn refresh_jobset_metrics(&self, state: &Arc) { + self.jobset_share_used.reset(); + self.jobset_seconds.reset(); + + let jobsets = state.jobsets.clone_as_io(); + for (full_jobset_name, jobset) in &jobsets { + let labels = &[full_jobset_name.as_str()]; + + let v = i64::try_from(u64::from(jobset.shares)).unwrap_or(0); + self.jobset_share_used.with_label_values(labels).set(v); + + self.jobset_seconds + .with_label_values(labels) + .set(jobset.seconds); + } + } + + fn refresh_time_metrics(&self, state: &Arc) { + let now = jiff::Timestamp::now(); + + self.queue_runner_current_time_seconds.set(now.as_second()); + #[allow(clippy::cast_possible_truncation)] + self.queue_runner_uptime_seconds.set( + (now - state.started_at) + .total(jiff::Unit::Second) + .unwrap_or_default() as i64, + ); + } + + #[tracing::instrument(skip(self, state), err)] + pub async fn gather_metrics(&self, state: &Arc) -> anyhow::Result> { + self.refresh_dynamic_metrics(state).await; + + let mut buffer = Vec::new(); + let encoder = prometheus::TextEncoder::new(); + let metric_families = prometheus::gather(); + encoder.encode(&metric_families, &mut buffer)?; + + Ok(buffer) + } + + fn add_to_total_step_time_ms(&self, v: u64) { + self.total_step_time_ms.inc_by(v); + } + + fn add_to_total_step_import_time_ms(&self, v: u128) { + if let Ok(v) = u64::try_from(v) { + self.total_step_import_time_ms.inc_by(v); + } + } + + fn add_to_total_step_build_time_ms(&self, v: u128) { + if let Ok(v) = u64::try_from(v) { + self.total_step_build_time_ms.inc_by(v); + } + } + + fn add_to_total_step_upload_time_ms(&self, v: u128) { + if let Ok(v) = u64::try_from(v) { + self.total_step_upload_time_ms.inc_by(v); + } + } + + pub fn observe_build_input_drvs(&self, count: f64, machine_type: &str) { + self.build_input_drvs_histogram + .with_label_values(&[machine_type]) + .observe(count); + } + + pub fn observe_build_closure_size(&self, size_bytes: f64, machine_type: &str) { + self.build_closure_size_bytes_histogram + .with_label_values(&[machine_type]) + .observe(size_bytes); + } + + pub fn observe_job_wait_time(&self, wait_seconds: f64, machine_type: &str) { + self.queue_job_wait_time_histogram + .with_label_values(&[machine_type]) + .observe(wait_seconds); + } + + pub fn track_build_success(&self, timings: super::build::BuildTimings, total_step_time: u64) { + self.nr_builds_succeeded.inc(); + self.nr_steps_done.inc(); + self.nr_steps_building.sub(1); + self.add_to_total_step_import_time_ms(timings.import_elapsed.as_millis()); + self.add_to_total_step_build_time_ms(timings.build_elapsed.as_millis()); + self.add_to_total_step_upload_time_ms(timings.upload_elapsed.as_millis()); + self.add_to_total_step_time_ms(total_step_time); + } + + pub fn track_build_failure(&self, timings: super::build::BuildTimings, total_step_time: u64) { + self.nr_steps_done.inc(); + self.nr_steps_building.sub(1); + self.nr_builds_failed.inc(); + self.add_to_total_step_import_time_ms(timings.import_elapsed.as_millis()); + self.add_to_total_step_build_time_ms(timings.build_elapsed.as_millis()); + self.add_to_total_step_upload_time_ms(timings.upload_elapsed.as_millis()); + self.add_to_total_step_time_ms(total_step_time); + } +} diff --git a/src/queue-runner/src/state/mod.rs b/src/queue-runner/src/state/mod.rs new file mode 100644 index 000000000..74349aa4b --- /dev/null +++ b/src/queue-runner/src/state/mod.rs @@ -0,0 +1,1958 @@ +mod atomic; +mod build; +mod fod_checker; +mod jobset; +mod machine; +mod metrics; +mod queue; +mod step; +mod step_info; +mod uploader; + +pub use atomic::AtomicDateTime; +pub use build::{Build, BuildOutput, BuildResultState, BuildTimings, Builds, RemoteBuild}; +pub use jobset::{Jobset, JobsetID, Jobsets}; +pub use machine::{Machine, Message as MachineMessage, Pressure, Stats as MachineStats}; +pub use queue::{BuildQueueStats, Queues}; +pub use step::{Step, Steps}; +pub use step_info::StepInfo; + +use std::sync::Arc; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::time::Instant; + +use futures::TryStreamExt as _; +use hashbrown::{HashMap, HashSet}; +use secrecy::ExposeSecret as _; + +use db::models::{BuildID, BuildStatus}; +use nix_utils::BaseStore as _; + +use crate::config::{App, Cli}; +use crate::state::build::get_mark_build_sccuess_data; +pub use crate::state::fod_checker::FodChecker; +use crate::state::machine::Machines; +use crate::utils::finish_build_step; + +pub type System = String; + +enum CreateStepResult { + None, + Valid(Arc), + PreviousFailure(Arc), +} + +enum RealiseStepResult { + None, + Valid(Arc), + MaybeCancelled, + CachedFailure, +} + +pub struct State { + pub store: nix_utils::LocalStore, + pub remote_stores: parking_lot::RwLock>, + pub config: App, + pub cli: Cli, + pub db: db::Database, + + pub machines: Machines, + + pub log_dir: std::path::PathBuf, + + pub builds: Builds, + pub jobsets: Jobsets, + pub steps: Steps, + pub queues: Queues, + + pub fod_checker: Option>, + + pub started_at: jiff::Timestamp, + + pub metrics: metrics::PromMetrics, + pub notify_dispatch: tokio::sync::Notify, + pub uploader: uploader::Uploader, +} + +impl State { + #[tracing::instrument(skip(tracing_guard), err)] + pub async fn new(tracing_guard: &hydra_tracing::TracingGuard) -> anyhow::Result> { + let store = nix_utils::LocalStore::init(); + nix_utils::set_verbosity(1); + let cli = Cli::new(); + if cli.status { + tracing_guard.change_log_level(hydra_tracing::EnvFilter::new("error")); + } + + let config = App::init(&cli.config_path)?; + let log_dir = config.get_hydra_log_dir(); + let db = db::Database::new( + config.get_db_url().expose_secret(), + config.get_max_db_connections(), + ) + .await?; + + let _ = fs_err::tokio::create_dir_all(&log_dir).await; + + let mut remote_stores = vec![]; + for uri in config.get_remote_store_addrs() { + remote_stores.push(binary_cache::S3BinaryCacheClient::new(uri.parse()?).await?); + } + + Ok(Arc::new(Self { + store, + remote_stores: parking_lot::RwLock::new(remote_stores), + cli, + db, + machines: Machines::new(), + log_dir, + builds: Builds::new(), + jobsets: Jobsets::new(), + steps: Steps::new(), + queues: Queues::new(), + fod_checker: if config.get_enable_fod_checker() { + Some(Arc::new(FodChecker::new(None))) + } else { + None + }, + started_at: jiff::Timestamp::now(), + metrics: metrics::PromMetrics::new()?, + notify_dispatch: tokio::sync::Notify::new(), + uploader: uploader::Uploader::new(), + config, + })) + } + + #[tracing::instrument(skip(self, new_config), err)] + pub async fn reload_config_callback( + &self, + new_config: &crate::config::PreparedApp, + ) -> anyhow::Result<()> { + // IF this gets more complex we need a way to trap the state and revert. + // right now it doesnt matter because only reconfigure_pool can fail and this is the first + // thing we do. + + let curr_db_url = self.config.get_db_url(); + let curr_machine_sort_fn = self.config.get_machine_sort_fn(); + let curr_step_sort_fn = self.config.get_step_sort_fn(); + let curr_remote_stores = self.config.get_remote_store_addrs(); + let curr_enable_fod_checker = self.config.get_enable_fod_checker(); + let mut new_remote_stores = vec![]; + if curr_remote_stores != new_config.remote_store_addr { + for uri in &new_config.remote_store_addr { + new_remote_stores.push(binary_cache::S3BinaryCacheClient::new(uri.parse()?).await?); + } + } + + if curr_db_url.expose_secret() != new_config.db_url.expose_secret() { + self.db + .reconfigure_pool(new_config.db_url.expose_secret())?; + } + if curr_machine_sort_fn != new_config.machine_sort_fn { + self.machines.sort(new_config.machine_sort_fn); + } + if curr_step_sort_fn != new_config.step_sort_fn { + self.queues.sort_queues(curr_step_sort_fn).await; + } + if curr_remote_stores != new_config.remote_store_addr { + let mut remote_stores = self.remote_stores.write(); + *remote_stores = new_remote_stores; + } + + if curr_enable_fod_checker != new_config.enable_fod_checker { + tracing::warn!( + "Changing the value of enable_fod_checker currently requires a restart!" + ); + } + + self.machines + .publish_new_config(machine::ConfigUpdate { + max_concurrent_downloads: new_config.max_concurrent_downloads, + }) + .await; + + Ok(()) + } + + #[tracing::instrument(skip(self, machine))] + pub async fn insert_machine(&self, machine: Machine) -> uuid::Uuid { + if !machine.systems.is_empty() { + self.queues + .ensure_queues_for_systems(&machine.systems) + .await; + } + + let machine_id = self + .machines + .insert_machine(machine, self.config.get_machine_sort_fn()); + self.trigger_dispatch(); + machine_id + } + + #[tracing::instrument(skip(self))] + pub async fn remove_machine(&self, machine_id: uuid::Uuid) { + if let Some(m) = self.machines.remove_machine(machine_id) { + let jobs = { + let jobs = m.jobs.read(); + jobs.clone() + }; + for job in &jobs { + if let Err(e) = self + .fail_step( + machine_id, + &job.path, + // we fail this with preparing because we kinda want to restart all jobs if + // a machine is removed + BuildResultState::PreparingFailure, + BuildTimings::default(), + ) + .await + { + tracing::error!( + "Failed to fail step machine_id={machine_id} drv={} e={e}", + job.path + ); + } + } + } + } + + pub async fn remove_all_machines(&self) { + for m in self.machines.get_all_machines() { + self.remove_machine(m.id).await; + } + } + + #[tracing::instrument(skip(self), err)] + pub async fn clear_busy(&self) -> anyhow::Result<()> { + let mut db = self.db.get().await?; + db.clear_busy(0).await?; + Ok(()) + } + + #[tracing::instrument(skip(self, constraint), err)] + #[allow(clippy::too_many_lines)] + async fn realise_drv_on_valid_machine( + self: Arc, + constraint: queue::JobConstraint, + ) -> anyhow::Result { + let free_fn = self.config.get_machine_free_fn(); + + let Some((machine, step_info)) = constraint.resolve(&self.machines, free_fn) else { + return Ok(RealiseStepResult::None); + }; + let drv = step_info.step.get_drv_path(); + let mut build_options = nix_utils::BuildOptions::new(None); + + let build_id = { + let mut dependents = HashSet::new(); + let mut steps = HashSet::new(); + step_info.step.get_dependents(&mut dependents, &mut steps); + + if dependents.is_empty() { + // Apparently all builds that depend on this derivation are gone (e.g. cancelled). So + // don't bother. This is very unlikely to happen, because normally Steps are only kept + // alive by being reachable from a Build. However, it's possible that a new Build just + // created a reference to this step. So to handle that possibility, we retry this step + // (putting it back in the runnable queue). If there are really no strong pointers to + // the step, it will be deleted. + tracing::info!("maybe cancelling build step {drv}"); + return Ok(RealiseStepResult::MaybeCancelled); + } + + let Some(build) = dependents + .iter() + .find(|b| &b.drv_path == drv) + .or_else(|| dependents.iter().next()) + else { + // this should never happen, as we checked is_empty above and fallback is just any build + return Ok(RealiseStepResult::MaybeCancelled); + }; + + // We want the biggest timeout otherwise we could build a step like llvm with a timeout + // of 180 because a nixostest with a timeout got scheduled and needs this step + let biggest_max_silent_time = dependents.iter().map(|x| x.max_silent_time).max(); + let biggest_build_timeout = dependents.iter().map(|x| x.timeout).max(); + + build_options + .set_max_silent_time(biggest_max_silent_time.unwrap_or(build.max_silent_time)); + build_options.set_build_timeout(biggest_build_timeout.unwrap_or(build.timeout)); + build.id + }; + + let mut job = machine::Job::new( + build_id, + drv.to_owned(), + step_info.resolved_drv_path.clone(), + ); + job.result.set_start_time_now(); + if self.check_cached_failure(step_info.step.clone()).await { + job.result.step_status = BuildStatus::CachedFailure; + self.inner_fail_job(drv, None, job, step_info.step.clone()) + .await?; + return Ok(RealiseStepResult::CachedFailure); + } + + self.construct_log_file_path(drv) + .await? + .to_str() + .ok_or_else(|| anyhow::anyhow!("failed to construct log path string."))? + .clone_into(&mut job.result.log_file); + let step_nr = { + let mut db = self.db.get().await?; + let mut tx = db.begin_transaction().await?; + + let step_nr = tx + .create_build_step( + Some(job.result.get_start_time_as_i32()?), + build_id, + &self.store.print_store_path(step_info.step.get_drv_path()), + step_info.step.get_system().as_deref(), + machine.hostname.clone(), + BuildStatus::Busy, + None, + None, + step_info + .step + .get_outputs() + .unwrap_or_default() + .into_iter() + .map(|o| (o.name, o.path.map(|s| self.store.print_store_path(&s)))) + .collect(), + ) + .await?; + tx.commit().await?; + step_nr + }; + job.step_nr = step_nr; + + tracing::info!( + "Submitting build drv={drv} on machine={} hostname={} build_id={build_id} step_nr={step_nr}", + machine.id, + machine.hostname + ); + self.db + .get() + .await? + .update_build_step(db::models::UpdateBuildStep { + build_id, + step_nr, + status: db::models::StepStatus::Connecting, + }) + .await?; + machine + .build_drv( + job, + &build_options, + // TODO: cleanup + if self.config.use_presigned_uploads() { + let remote_stores = self.remote_stores.read(); + remote_stores + .first() + .map(|s| crate::state::machine::PresignedUrlOpts { + upload_debug_info: s.cfg.write_debug_info, + }) + } else { + None + }, + ) + .await?; + self.metrics.nr_steps_started.inc(); + self.metrics.nr_steps_building.add(1); + Ok(RealiseStepResult::Valid(machine)) + } + + #[tracing::instrument(skip(self), fields(%drv), err)] + async fn construct_log_file_path( + &self, + drv: &nix_utils::StorePath, + ) -> anyhow::Result { + let mut log_file = self.log_dir.clone(); + let (dir, file) = drv.base_name().split_at(2); + log_file.push(format!("{dir}/")); + let _ = fs_err::tokio::create_dir_all(&log_file).await; // create dir + log_file.push(file); + Ok(log_file) + } + + #[tracing::instrument(skip(self), fields(%drv), err)] + pub async fn new_log_file( + &self, + drv: &nix_utils::StorePath, + ) -> anyhow::Result { + let log_file = self.construct_log_file_path(drv).await?; + tracing::debug!("opening {log_file:?}"); + + Ok(fs_err::tokio::File::options() + .create(true) + .truncate(true) + .write(true) + .read(false) + .mode(0o666) + .open(log_file) + .await?) + } + + #[tracing::instrument(skip(self, new_ids, new_builds_by_id, new_builds_by_path))] + async fn process_new_builds( + &self, + new_ids: Vec, + new_builds_by_id: Arc>>>, + new_builds_by_path: HashMap>, + ) { + let finished_drvs = Arc::new(parking_lot::RwLock::new( + HashSet::::new(), + )); + + let starttime = jiff::Timestamp::now(); + for id in new_ids { + let Some(build) = new_builds_by_id.read().get(&id).cloned() else { + continue; + }; + + let new_runnable = Arc::new(parking_lot::RwLock::new(HashSet::>::new())); + let nr_added: Arc = Arc::new(0.into()); + let now = Instant::now(); + + Box::pin(self.create_build( + build, + nr_added.clone(), + new_builds_by_id.clone(), + &new_builds_by_path, + finished_drvs.clone(), + new_runnable.clone(), + )) + .await; + + // we should never run into this issue + #[allow(clippy::cast_possible_truncation)] + self.metrics + .build_read_time_ms + .inc_by(now.elapsed().as_millis() as u64); + + { + let new_runnable = new_runnable.read(); + tracing::info!( + "got {} new runnable steps from {} new builds", + new_runnable.len(), + nr_added.load(Ordering::Relaxed) + ); + for r in new_runnable.iter() { + r.make_runnable(); + } + } + if let Ok(added_u64) = u64::try_from(nr_added.load(Ordering::Relaxed)) { + self.metrics.nr_builds_read.inc_by(added_u64); + } + let stop_queue_run_after = self.config.get_stop_queue_run_after(); + + if let Some(stop_queue_run_after) = stop_queue_run_after + && jiff::Timestamp::now() > (starttime + stop_queue_run_after) + { + self.metrics.queue_checks_early_exits.inc(); + break; + } + } + + // This is here to ensure that we dont have any deps to finished steps + // This can happen because step creation is async and is_new can return a step that is + // still undecided if its finished or not. + self.steps.make_rdeps_runnable(); + + // we can just always trigger dispatch as we might have a free machine and its cheap + self.metrics.queue_checks_finished.inc(); + self.trigger_dispatch(); + if let Some(fod_checker) = &self.fod_checker { + fod_checker.trigger_traverse(); + } + } + + #[tracing::instrument(skip(self), err)] + async fn process_queue_change(&self) -> anyhow::Result<()> { + let mut db = self.db.get().await?; + let curr_ids: HashMap<_, _> = db + .get_not_finished_builds_fast() + .await? + .into_iter() + .map(|b| (b.id, b.globalpriority)) + .collect(); + self.builds.update_priorities(&curr_ids); + + let cancelled_steps = self.queues.kill_active_steps().await; + for (drv_path, machine_id) in cancelled_steps { + if let Err(e) = self + .fail_step( + machine_id, + &drv_path, + BuildResultState::Cancelled, + BuildTimings::default(), + ) + .await + { + tracing::error!( + "Failed to abort step machine_id={machine_id} drv={drv_path} e={e}", + ); + } + } + Ok(()) + } + + #[tracing::instrument(skip(self), fields(%drv_path))] + pub async fn queue_one_build( + &self, + jobset_id: i32, + drv_path: &nix_utils::StorePath, + ) -> anyhow::Result<()> { + let mut db = self.db.get().await?; + let drv = nix_utils::query_drv(&self.store, drv_path) + .await? + .ok_or_else(|| anyhow::anyhow!("drv not found"))?; + db.insert_debug_build( + jobset_id, + &self.store.print_store_path(drv_path), + &drv.system, + ) + .await?; + + let mut tx = db.begin_transaction().await?; + tx.notify_builds_added().await?; + tx.commit().await?; + Ok(()) + } + + #[tracing::instrument(skip(self), err)] + pub async fn get_queued_builds(&self) -> anyhow::Result<()> { + self.metrics.queue_checks_started.inc(); + + let mut new_ids = Vec::::with_capacity(1000); + let mut new_builds_by_id = HashMap::>::with_capacity(1000); + let mut new_builds_by_path = + HashMap::>::with_capacity(1000); + + { + let mut conn = self.db.get().await?; + for b in conn.get_not_finished_builds().await? { + let jobset = self + .jobsets + .create(&mut conn, b.jobset_id, &b.project, &b.jobset) + .await?; + let build = Build::new(b, jobset)?; + new_ids.push(build.id); + new_builds_by_id.insert(build.id, build.clone()); + new_builds_by_path + .entry(build.drv_path.clone()) + .or_insert_with(HashSet::new) + .insert(build.id); + } + } + tracing::debug!("new_ids: {new_ids:?}"); + tracing::debug!("new_builds_by_id: {new_builds_by_id:?}"); + tracing::debug!("new_builds_by_path: {new_builds_by_path:?}"); + + let new_builds_by_id = Arc::new(parking_lot::RwLock::new(new_builds_by_id)); + Box::pin(self.process_new_builds(new_ids, new_builds_by_id, new_builds_by_path)).await; + Ok(()) + } + + #[tracing::instrument(skip(self))] + pub fn start_queue_monitor_loop(self: Arc) -> tokio::task::AbortHandle { + let task = tokio::task::spawn({ + async move { + if let Err(e) = Box::pin(self.queue_monitor_loop()).await { + tracing::error!("Failed to spawn queue monitor loop. e={e}"); + } + } + }); + task.abort_handle() + } + + #[tracing::instrument(skip(self), err)] + async fn queue_monitor_loop(&self) -> anyhow::Result<()> { + let mut listener = self + .db + .listener(vec![ + "builds_added", + "builds_restarted", + "builds_cancelled", + "builds_deleted", + "builds_bumped", + "jobset_shares_changed", + ]) + .await?; + + loop { + let before_work = Instant::now(); + self.store.clear_path_info_cache(); + if let Err(e) = self.get_queued_builds().await { + tracing::error!("get_queue_builds failed inside queue monitor loop: {e}"); + continue; + } + + #[allow(clippy::cast_possible_truncation)] + self.metrics + .queue_monitor_time_spent_running + .inc_by(before_work.elapsed().as_micros() as u64); + + let before_sleep = Instant::now(); + let queue_trigger_timer = self.config.get_queue_trigger_timer(); + let notification = if let Some(timer) = queue_trigger_timer { + tokio::select! { + () = tokio::time::sleep(timer) => {"timer_reached".into()}, + v = listener.try_next() => match v { + Ok(Some(v)) => v.channel().to_owned(), + Ok(None) => continue, + Err(e) => { + tracing::warn!("PgListener failed with e={e}"); + continue; + } + }, + } + } else { + match listener.try_next().await { + Ok(Some(v)) => v.channel().to_owned(), + Ok(None) => continue, + Err(e) => { + tracing::warn!("PgListener failed with e={e}"); + continue; + } + } + }; + self.metrics.nr_queue_wakeups.inc(); + tracing::trace!("New notification from PgListener. notification={notification:?}"); + + match notification.as_ref() { + "builds_added" => { + tracing::debug!("got notification: new builds added to the queue"); + } + "builds_restarted" => tracing::debug!("got notification: builds restarted"), + "builds_cancelled" | "builds_deleted" | "builds_bumped" => { + tracing::info!("got notification: builds cancelled or bumped"); + if let Err(e) = self.process_queue_change().await { + tracing::error!("Failed to process queue change. e={e}"); + } + } + "jobset_shares_changed" => { + tracing::info!("got notification: jobset shares changed"); + match self.db.get().await { + Ok(mut conn) => { + if let Err(e) = self.jobsets.handle_change(&mut conn).await { + tracing::error!("Failed to handle jobset change. e={e}"); + } + } + Err(e) => { + tracing::error!( + "Failed to get db connection for event 'jobset_shares_changed'. e={e}" + ); + } + } + } + _ => (), + } + + #[allow(clippy::cast_possible_truncation)] + self.metrics + .queue_monitor_time_spent_waiting + .inc_by(before_sleep.elapsed().as_micros() as u64); + } + } + + #[tracing::instrument(skip(self))] + pub fn start_dispatch_loop(self: Arc) -> tokio::task::AbortHandle { + let task = tokio::task::spawn({ + async move { + loop { + let before_sleep = Instant::now(); + let dispatch_trigger_timer = self.config.get_dispatch_trigger_timer(); + if let Some(timer) = dispatch_trigger_timer { + tokio::select! { + () = self.notify_dispatch.notified() => {}, + () = tokio::time::sleep(timer) => {}, + }; + } else { + self.notify_dispatch.notified().await; + } + tracing::info!("starting dispatch"); + + #[allow(clippy::cast_possible_truncation)] + self.metrics + .dispatcher_time_spent_waiting + .inc_by(before_sleep.elapsed().as_micros() as u64); + + self.metrics.nr_dispatcher_wakeups.inc(); + let before_work = Instant::now(); + self.clone().do_dispatch_once().await; + + let elapsed = before_work.elapsed(); + + #[allow(clippy::cast_possible_truncation)] + self.metrics + .dispatcher_time_spent_running + .inc_by(elapsed.as_micros() as u64); + + #[allow(clippy::cast_possible_truncation)] + self.metrics + .dispatch_time_ms + .inc_by(elapsed.as_millis() as u64); + } + } + }); + task.abort_handle() + } + + #[tracing::instrument(skip(self), err)] + async fn dump_status_loop(self: Arc) -> anyhow::Result<()> { + let mut listener = self.db.listener(vec!["dump_status"]).await?; + + let state = self.clone(); + loop { + let _ = match listener.try_next().await { + Ok(Some(v)) => v, + Ok(None) => continue, + Err(e) => { + tracing::warn!("PgListener failed with e={e}"); + continue; + } + }; + + let state = state.clone(); + let queue_stats = crate::io::QueueRunnerStats::new(state.clone()).await; + let sort_fn = state.config.get_machine_sort_fn(); + let free_fn = state.config.get_machine_free_fn(); + let machines = state + .machines + .get_all_machines() + .into_iter() + .map(|m| { + ( + m.hostname.clone(), + crate::io::Machine::from_state(&m, sort_fn, free_fn), + ) + }) + .collect(); + let jobsets = self.jobsets.clone_as_io(); + let remote_stores = { + let stores = state.remote_stores.read(); + stores.clone() + }; + let dump_status = crate::io::DumpResponse::new( + queue_stats, + machines, + jobsets, + &state.store, + &remote_stores, + ); + { + let Ok(mut db) = self.db.get().await else { + continue; + }; + let Ok(mut tx) = db.begin_transaction().await else { + continue; + }; + let dump_status = match serde_json::to_value(dump_status) { + Ok(v) => v, + Err(e) => { + tracing::error!("Failed to update status in database: {e}"); + continue; + } + }; + if let Err(e) = tx.upsert_status(&dump_status).await { + tracing::error!("Failed to update status in database: {e}"); + continue; + } + if let Err(e) = tx.notify_status_dumped().await { + tracing::error!("Failed to update status in database: {e}"); + continue; + } + if let Err(e) = tx.commit().await { + tracing::error!("Failed to update status in database: {e}"); + } + } + } + } + + #[tracing::instrument(skip(self))] + pub fn start_dump_status_loop(self: Arc) -> tokio::task::AbortHandle { + let task = tokio::task::spawn({ + async move { + if let Err(e) = self.dump_status_loop().await { + tracing::error!("Failed to spawn queue monitor loop. e={e}"); + } + } + }); + task.abort_handle() + } + + #[tracing::instrument(skip(self))] + pub fn start_uploader_queue(self: Arc) -> tokio::task::AbortHandle { + let task = tokio::task::spawn({ + async move { + loop { + let local_store = self.store.clone(); + let remote_stores = { + let r = self.remote_stores.read(); + r.clone() + }; + let limit = self.config.get_concurrent_upload_limit(); + if limit < 2 { + self.uploader.upload_once(local_store, remote_stores).await; + } else { + self.uploader + .upload_many(local_store, remote_stores, limit) + .await; + } + } + } + }); + task.abort_handle() + } + + #[tracing::instrument(skip(self))] + pub async fn get_status_from_main_process(self: Arc) -> anyhow::Result<()> { + let mut db = self.db.get().await?; + + let mut listener = self.db.listener(vec!["status_dumped"]).await?; + { + let mut tx = db.begin_transaction().await?; + tx.notify_dump_status().await?; + tx.commit().await?; + } + + let _ = match listener.try_next().await { + Ok(Some(v)) => v, + Ok(None) => return Ok(()), + Err(e) => { + tracing::warn!("PgListener failed with e={e}"); + return Ok(()); + } + }; + if let Some(status) = db.get_status().await? { + // we want a println! here so it can be consumed by other tools + println!("{}", serde_json::to_string_pretty(&status)?); + } + + Ok(()) + } + + #[tracing::instrument(skip(self))] + pub fn trigger_dispatch(&self) { + self.notify_dispatch.notify_one(); + } + + #[tracing::instrument(skip(self))] + async fn do_dispatch_once(self: Arc) { + // Prune old historical build step info from the jobsets. + self.jobsets.prune(); + let new_runnable = self.steps.clone_runnable(); + + let now = jiff::Timestamp::now(); + let mut new_queues = HashMap::>::with_capacity(10); + for r in new_runnable { + let Some(system) = r.get_system() else { + continue; + }; + if r.atomic_state.tries.load(Ordering::Relaxed) > 0 { + continue; + } + let step_info = StepInfo::new(&self.store, r.clone()).await; + + new_queues + .entry(system) + .or_insert_with(|| Vec::with_capacity(100)) + .push(step_info); + } + + for (system, jobs) in new_queues { + self.queues + .insert_new_jobs( + system, + jobs, + &now, + self.config.get_step_sort_fn(), + &self.metrics, + ) + .await; + } + self.queues.remove_all_weak_pointer().await; + + let nr_steps_waiting_all_queues = self + .queues + .process( + { + let state = self.clone(); + async move |constraint: queue::JobConstraint| { + Box::pin(state.clone().realise_drv_on_valid_machine(constraint)).await + } + }, + &self.metrics, + ) + .await; + self.metrics + .nr_steps_waiting + .set(nr_steps_waiting_all_queues); + + self.abort_unsupported().await; + } + + #[tracing::instrument(skip(self, step_status), fields(%build_id, %machine_id), err)] + pub async fn update_build_step( + &self, + build_id: uuid::Uuid, + machine_id: uuid::Uuid, + step_status: db::models::StepStatus, + ) -> anyhow::Result<()> { + let build_id_and_step_nr = self.machines.get_machine_by_id(machine_id).and_then(|m| { + tracing::debug!( + "get job from machine by build_id: build_id={build_id} m={}", + m.id + ); + m.get_build_id_and_step_nr_by_uuid(build_id) + }); + + let Some((build_id, step_nr)) = build_id_and_step_nr else { + tracing::warn!( + "Failed to find job with build_id and step_nr for build_id={build_id:?} machine_id={machine_id:?}." + ); + return Ok(()); + }; + self.db + .get() + .await? + .update_build_step(db::models::UpdateBuildStep { + build_id, + step_nr, + status: step_status, + }) + .await?; + Ok(()) + } + + #[allow(clippy::too_many_lines)] + #[tracing::instrument(skip(self, output), fields(%machine_id, %drv_path), err)] + pub async fn succeed_step( + &self, + machine_id: uuid::Uuid, + drv_path: &nix_utils::StorePath, + output: BuildOutput, + ) -> anyhow::Result<()> { + tracing::info!("marking job as done: drv_path={drv_path}"); + let item = self + .queues + .remove_job_from_scheduled(drv_path) + .await + .ok_or_else(|| anyhow::anyhow!("Step is missing in queues.scheduled"))?; + + item.step_info.step.set_finished(true); + tracing::debug!( + "removing job from machine: drv_path={drv_path} m={}", + item.machine.id + ); + let mut job = item + .machine + .remove_job(drv_path) + .ok_or_else(|| anyhow::anyhow!("Job is missing in machine.jobs m={}", item.machine,))?; + self.queues + .remove_job(&item.step_info, &item.build_queue) + .await; + + job.result.step_status = BuildStatus::Success; + job.result.set_stop_time_now(); + job.result.set_overhead(output.timings.get_overhead())?; + + let total_step_time = job.result.get_total_step_time_ms(); + item.machine + .stats + .track_build_success(output.timings, total_step_time); + self.metrics + .track_build_success(output.timings, total_step_time); + + finish_build_step( + &self.db, + &self.store, + job.build_id, + job.step_nr, + &job.result, + Some(item.machine.hostname.clone()), + ) + .await?; + + // TODO: redo gc roots, we only need to root until we are done with that build + for (_, path) in &output.outputs { + self.add_root(path); + } + + let has_stores = { + let r = self.remote_stores.read(); + !r.is_empty() + }; + if has_stores { + // Only upload outputs if presigned uploads are NOT enabled + // When presigned uploads are enabled, builder handles NAR uploads directly + let outputs_to_upload = if self.config.use_presigned_uploads() { + vec![] + } else { + output + .outputs + .values() + .map(Clone::clone) + .collect::>() + }; + + if let Err(e) = self.uploader.schedule_upload( + outputs_to_upload, + format!("log/{}", job.path.base_name()), + job.result.log_file.clone(), + ) { + tracing::error!( + "Failed to schedule upload for build {} outputs: {}", + job.build_id, + e + ); + } + } + + let direct = item.step_info.step.get_direct_builds(); + if direct.is_empty() { + self.steps.remove(item.step_info.step.get_drv_path()); + } + + { + let mut db = self.db.get().await?; + let mut tx = db.begin_transaction().await?; + let start_time = job.result.get_start_time_as_i32()?; + let stop_time = job.result.get_stop_time_as_i32()?; + for b in &direct { + let is_cached = job.build_id != b.id || job.result.is_cached; + tx.mark_succeeded_build( + get_mark_build_sccuess_data(&self.store, b, &output), + is_cached, + start_time, + stop_time, + ) + .await?; + self.metrics.nr_builds_done.inc(); + } + + tx.commit().await?; + } + + // Remove the direct dependencies from 'builds'. This will cause them to be + // destroyed. + for b in &direct { + b.set_finished_in_db(true); + self.builds.remove_by_id(b.id); + } + + { + let mut db = self.db.get().await?; + let mut tx = db.begin_transaction().await?; + for b in direct { + tx.notify_build_finished(b.id, &[]).await?; + } + + tx.commit().await?; + } + + item.step_info.step.make_rdeps_runnable(); + + // always trigger dispatch, as we now might have a free machine again + self.trigger_dispatch(); + + Ok(()) + } + + #[tracing::instrument(skip(self), fields(%machine_id, %drv_path), err)] + pub async fn fail_step( + &self, + machine_id: uuid::Uuid, + drv_path: &nix_utils::StorePath, + state: BuildResultState, + timings: BuildTimings, + ) -> anyhow::Result<()> { + tracing::info!("removing job from running in system queue: drv_path={drv_path}"); + let item = self + .queues + .remove_job_from_scheduled(drv_path) + .await + .ok_or_else(|| anyhow::anyhow!("Step is missing in queues.scheduled"))?; + + item.step_info.step.set_finished(false); + + tracing::debug!( + "removing job from machine: drv_path={drv_path} m={}", + item.machine.id + ); + let mut job = item + .machine + .remove_job(drv_path) + .ok_or_else(|| anyhow::anyhow!("Job is missing in machine.jobs m={}", item.machine))?; + + job.result.step_status = BuildStatus::Failed; + // this can override step_status to something more specific + job.result.update_with_result_state(&state); + job.result.set_stop_time_now(); + job.result.set_overhead(timings.get_overhead())?; + + let total_step_time = job.result.get_total_step_time_ms(); + item.machine + .stats + .track_build_failure(timings, total_step_time); + self.metrics.track_build_failure(timings, total_step_time); + + let (max_retries, retry_interval, retry_backoff) = self.config.get_retry(); + + if job.result.can_retry { + item.step_info + .step + .atomic_state + .tries + .fetch_add(1, Ordering::Relaxed); + let tries = item + .step_info + .step + .atomic_state + .tries + .load(Ordering::Relaxed); + if tries < max_retries { + self.metrics.nr_retries.inc(); + #[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)] + let delta = (retry_interval * retry_backoff.powf((tries - 1) as f32)) as i64; + tracing::info!("will retry '{drv_path}' after {delta}s"); + item.step_info + .step + .set_after(jiff::Timestamp::now() + jiff::SignedDuration::from_secs(delta)); + if i64::from(tries) > self.metrics.max_nr_retries.get() { + self.metrics.max_nr_retries.set(i64::from(tries)); + } + + item.step_info.set_already_scheduled(false); + + finish_build_step( + &self.db, + &self.store, + job.build_id, + job.step_nr, + &job.result, + Some(item.machine.hostname.clone()), + ) + .await?; + self.trigger_dispatch(); + return Ok(()); + } + } + + // remove job from queues, aka actually fail the job + self.queues + .remove_job(&item.step_info, &item.build_queue) + .await; + + self.inner_fail_job( + drv_path, + Some(item.machine), + job, + item.step_info.step.clone(), + ) + .await + } + + #[tracing::instrument(skip(self, output), fields(%machine_id, build_id=%build_id), err)] + pub async fn succeed_step_by_uuid( + &self, + build_id: uuid::Uuid, + machine_id: uuid::Uuid, + output: BuildOutput, + ) -> anyhow::Result<()> { + let machine = self + .machines + .get_machine_by_id(machine_id) + .ok_or_else(|| anyhow::anyhow!("Machine with machine_id not found"))?; + let drv_path = machine + .get_job_drv_for_build_id(build_id) + .ok_or_else(|| anyhow::anyhow!("Job with build_id not found"))?; + + self.succeed_step(machine_id, &drv_path, output).await + } + + #[tracing::instrument(skip(self), fields(%machine_id, build_id=%build_id), err)] + pub async fn fail_step_by_uuid( + &self, + build_id: uuid::Uuid, + machine_id: uuid::Uuid, + state: BuildResultState, + timings: BuildTimings, + ) -> anyhow::Result<()> { + let machine = self + .machines + .get_machine_by_id(machine_id) + .ok_or_else(|| anyhow::anyhow!("Machine with machine_id not found"))?; + let drv_path = machine + .get_job_drv_for_build_id(build_id) + .ok_or_else(|| anyhow::anyhow!("Job with build_id not found"))?; + + self.fail_step(machine_id, &drv_path, state, timings).await + } + + #[allow(clippy::too_many_lines)] + #[tracing::instrument(skip(self, machine, job, step), fields(%drv_path), err)] + async fn inner_fail_job( + &self, + drv_path: &nix_utils::StorePath, + machine: Option>, + mut job: machine::Job, + step: Arc, + ) -> anyhow::Result<()> { + if !job.result.has_stop_time() { + job.result.set_stop_time_now(); + } + + if job.step_nr != 0 { + finish_build_step( + &self.db, + &self.store, + job.build_id, + job.step_nr, + &job.result, + machine.as_ref().map(|m| m.hostname.clone()), + ) + .await?; + } + + let mut dependent_ids = Vec::new(); + let mut step_finished = false; + loop { + let indirect = self.get_all_indirect_builds(&step); + if indirect.is_empty() && step_finished { + break; + } + + // Create failed build steps for every build that depends on this, except when this + // step is cached and is the top-level of that build (since then it's redundant with + // the build's isCachedBuild field). + { + let mut db = self.db.get().await?; + let mut tx = db.begin_transaction().await?; + for b in &indirect { + if (job.result.step_status == BuildStatus::CachedFailure + && &b.drv_path == step.get_drv_path()) + || ((job.result.step_status != BuildStatus::CachedFailure + && job.result.step_status != BuildStatus::Unsupported) + && job.build_id == b.id) + || b.get_finished_in_db() + { + continue; + } + + tx.create_build_step( + None, + b.id, + &self.store.print_store_path(step.get_drv_path()), + step.get_system().as_deref(), + machine + .as_deref() + .map(|m| m.hostname.clone()) + .unwrap_or_default(), + job.result.step_status, + job.result.error_msg.clone(), + if job.build_id == b.id { + None + } else { + Some(job.build_id) + }, + step.get_outputs() + .unwrap_or_default() + .into_iter() + .map(|o| (o.name, o.path.map(|s| self.store.print_store_path(&s)))) + .collect(), + ) + .await?; + } + + // Mark all builds that depend on this derivation as failed. + for b in &indirect { + if b.get_finished_in_db() { + continue; + } + + tracing::info!("marking build {} as failed", b.id); + let start_time = job.result.get_start_time_as_i32()?; + let stop_time = job.result.get_stop_time_as_i32()?; + tx.update_build_after_failure( + b.id, + if &b.drv_path != step.get_drv_path() + && job.result.step_status == BuildStatus::Failed + { + BuildStatus::DepFailed + } else { + job.result.step_status + }, + start_time, + stop_time, + job.result.step_status == BuildStatus::CachedFailure, + ) + .await?; + self.metrics.nr_builds_done.inc(); + } + + // Remember failed paths in the database so that they won't be built again. + if job.result.step_status != BuildStatus::CachedFailure && job.result.can_cache { + for o in step.get_outputs().unwrap_or_default() { + let Some(p) = o.path else { continue }; + tx.insert_failed_paths(&self.store.print_store_path(&p)) + .await?; + } + } + + tx.commit().await?; + } + + step_finished = true; + + // Remove the indirect dependencies from 'builds'. This will cause them to be + // destroyed. + for b in indirect { + b.set_finished_in_db(true); + self.builds.remove_by_id(b.id); + dependent_ids.push(b.id); + } + } + { + let mut db = self.db.get().await?; + let mut tx = db.begin_transaction().await?; + tx.notify_build_finished(job.build_id, &dependent_ids) + .await?; + tx.commit().await?; + } + + // trigger dispatch, as we now have a free mashine again + self.trigger_dispatch(); + + Ok(()) + } + + #[tracing::instrument(skip(self, step))] + fn get_all_indirect_builds(&self, step: &Arc) -> HashSet> { + let mut indirect = HashSet::new(); + let mut steps = HashSet::new(); + step.get_dependents(&mut indirect, &mut steps); + + // If there are no builds left, delete all referring + // steps from ‘steps’. As for the success case, we can + // be certain no new referrers can be added. + if indirect.is_empty() { + for s in steps { + let drv = s.get_drv_path(); + tracing::debug!("finishing build step '{drv}'"); + self.steps.remove(drv); + } + } + + indirect + } + + #[tracing::instrument(skip(self, build, step), err)] + async fn handle_previous_failure( + &self, + build: Arc, + step: Arc, + ) -> anyhow::Result<()> { + // Some step previously failed, so mark the build as failed right away. + tracing::warn!( + "marking build {} as cached failure due to '{}'", + build.id, + step.get_drv_path() + ); + if build.get_finished_in_db() { + return Ok(()); + } + + // if !build.finished_in_db + let mut conn = self.db.get().await?; + let mut tx = conn.begin_transaction().await?; + + // Find the previous build step record, first by derivation path, then by output + // path. + let mut propagated_from = tx + .get_last_build_step_id(&self.store.print_store_path(step.get_drv_path())) + .await? + .unwrap_or_default(); + + if propagated_from == 0 { + // we can access step.drv here because the value is always set if + // PreviousFailure is returned, so this should never yield None + + let outputs = step.get_outputs().unwrap_or_default(); + for o in outputs { + let res = if let Some(path) = &o.path { + tx.get_last_build_step_id_for_output_path(&self.store.print_store_path(path)) + .await + } else { + tx.get_last_build_step_id_for_output_with_drv( + &self.store.print_store_path(step.get_drv_path()), + &o.name, + ) + .await + }; + if let Ok(Some(res)) = res { + propagated_from = res; + break; + } + } + } + + tx.create_build_step( + None, + build.id, + &self.store.print_store_path(step.get_drv_path()), + step.get_system().as_deref(), + String::new(), + BuildStatus::CachedFailure, + None, + Some(propagated_from), + step.get_outputs() + .unwrap_or_default() + .into_iter() + .map(|o| (o.name, o.path.map(|s| self.store.print_store_path(&s)))) + .collect(), + ) + .await?; + tx.update_build_after_previous_failure( + build.id, + if step.get_drv_path() == &build.drv_path { + BuildStatus::Failed + } else { + BuildStatus::DepFailed + }, + ) + .await?; + + let _ = tx.notify_build_finished(build.id, &[]).await; + tx.commit().await?; + + build.set_finished_in_db(true); + self.metrics.nr_builds_done.inc(); + Ok(()) + } + + #[allow(clippy::too_many_lines)] + #[tracing::instrument(skip( + self, + build, + nr_added, + new_builds_by_id, + new_builds_by_path, + finished_drvs, + new_runnable + ), fields(build_id=build.id))] + async fn create_build( + &self, + build: Arc, + nr_added: Arc, + new_builds_by_id: Arc>>>, + new_builds_by_path: &HashMap>, + finished_drvs: Arc>>, + new_runnable: Arc>>>, + ) { + self.metrics.queue_build_loads.inc(); + tracing::info!("loading build {} ({})", build.id, build.full_job_name()); + nr_added.fetch_add(1, Ordering::Relaxed); + { + let mut new_builds_by_id = new_builds_by_id.write(); + new_builds_by_id.remove(&build.id); + } + + if !self.store.is_valid_path(&build.drv_path).await { + tracing::error!("aborting GC'ed build {}", build.id); + if !build.get_finished_in_db() { + match self.db.get().await { + Ok(mut conn) => { + if let Err(e) = conn.abort_build(build.id).await { + tracing::error!("Failed to abort the build={} e={}", build.id, e); + } + } + Err(e) => tracing::error!( + "Failed to get database connection so we can abort the build={} e={}", + build.id, + e + ), + } + } + + build.set_finished_in_db(true); + self.metrics.nr_builds_done.inc(); + return; + } + + // Create steps for this derivation and its dependencies. + let new_steps = Arc::new(parking_lot::RwLock::new(HashSet::>::new())); + let step = match self + .create_step( + // conn, + build.clone(), + build.drv_path.clone(), + Some(build.clone()), + None, + finished_drvs.clone(), + new_steps.clone(), + new_runnable.clone(), + ) + .await + { + CreateStepResult::None => None, + CreateStepResult::Valid(dep) => Some(dep), + CreateStepResult::PreviousFailure(step) => { + if let Err(e) = self.handle_previous_failure(build, step).await { + tracing::error!("Failed to handle previous failure: {e}"); + } + return; + } + }; + + { + use futures::stream::StreamExt as _; + + let builds = { + let new_steps = new_steps.read(); + new_steps + .iter() + .filter_map(|r| Some(new_builds_by_path.get(r.get_drv_path())?.clone())) + .flatten() + .collect::>() + }; + let mut stream = futures::StreamExt::map(tokio_stream::iter(builds), |b| { + let nr_added = nr_added.clone(); + let new_builds_by_id = new_builds_by_id.clone(); + let finished_drvs = finished_drvs.clone(); + let new_runnable = new_runnable.clone(); + async move { + let j = { + if let Some(j) = new_builds_by_id.read().get(&b) { + j.clone() + } else { + return; + } + }; + + Box::pin(self.create_build( + j, + nr_added, + new_builds_by_id, + new_builds_by_path, + finished_drvs, + new_runnable, + )) + .await; + } + }) + .buffered(10); + while tokio_stream::StreamExt::next(&mut stream).await.is_some() {} + } + + if let Some(step) = step { + if !build.get_finished_in_db() { + self.builds.insert_new_build(build.clone()); + } + + build.set_toplevel_step(step.clone()); + build.propagate_priorities(); + + tracing::info!( + "added build {} (top-level step {}, {} new steps)", + build.id, + step.get_drv_path(), + new_steps.read().len() + ); + } else { + // If we didn't get a step, it means the step's outputs are + // all valid. So we mark this as a finished, cached build. + if let Err(e) = self.handle_cached_build(build).await { + tracing::error!("failed to handle cached build: {e}"); + } + } + } + + #[allow(clippy::too_many_lines, clippy::too_many_arguments)] + #[tracing::instrument(skip( + self, + build, + referring_build, + referring_step, + finished_drvs, + new_steps, + new_runnable + ), fields(build_id=build.id, %drv_path))] + async fn create_step( + &self, + build: Arc, + drv_path: nix_utils::StorePath, + referring_build: Option>, + referring_step: Option>, + finished_drvs: Arc>>, + new_steps: Arc>>>, + new_runnable: Arc>>>, + ) -> CreateStepResult { + use futures::stream::StreamExt as _; + + { + let finished_drvs = finished_drvs.read(); + if finished_drvs.contains(&drv_path) { + return CreateStepResult::None; + } + } + + let (step, is_new) = + self.steps + .create(&drv_path, referring_build.as_ref(), referring_step.as_ref()); + if !is_new { + return CreateStepResult::Valid(step); + } + self.metrics.queue_steps_created.inc(); + tracing::debug!("considering derivation '{drv_path}'"); + + let Some(drv) = nix_utils::query_drv(&self.store, &drv_path) + .await + .ok() + .flatten() + else { + return CreateStepResult::None; + }; + if let Some(fod_checker) = &self.fod_checker { + fod_checker.add_ca_drv_parsed(&drv_path, &drv); + } + + let system_type = drv.system.as_str(); + #[allow(clippy::cast_precision_loss)] + self.metrics + .observe_build_input_drvs(drv.input_drvs.len() as f64, system_type); + + let use_substitutes = self.config.get_use_substitutes(); + // TODO: check all remote stores + let remote_store = { + let r = self.remote_stores.read(); + r.first().cloned() + }; + let missing_outputs = if let Some(ref remote_store) = remote_store { + let mut missing = remote_store + .query_missing_remote_outputs(drv.outputs.to_vec()) + .await; + if !missing.is_empty() + && self + .store + .query_missing_outputs(drv.outputs.to_vec()) + .await + .is_empty() + { + // we have all paths locally, so we can just upload them to the remote_store + if let Ok(log_file) = self.construct_log_file_path(&drv_path).await { + let missing_paths: Vec = + missing.iter().filter_map(|v| v.path.clone()).collect(); + if let Err(e) = self.uploader.schedule_upload( + missing_paths, + format!("log/{}", drv_path.base_name()), + log_file.to_string_lossy().to_string(), + ) { + tracing::error!("Failed to schedule upload for derivation {drv_path}: {e}"); + } else { + missing.clear(); + } + } + } + missing + } else { + self.store.query_missing_outputs(drv.outputs.to_vec()).await + }; + + step.set_drv(drv); + + if self.check_cached_failure(step.clone()).await { + step.set_previous_failure(true); + return CreateStepResult::PreviousFailure(step); + } + + tracing::debug!("missing outputs: {missing_outputs:?}"); + let finished = if !missing_outputs.is_empty() && use_substitutes { + use futures::stream::StreamExt as _; + + let mut substituted = 0; + let missing_outputs_len = missing_outputs.len(); + + let mut stream = futures::StreamExt::map(tokio_stream::iter(missing_outputs), |o| { + self.metrics.nr_substitutes_started.inc(); + crate::utils::substitute_output( + self.db.clone(), + self.store.clone(), + o, + build.id, + &drv_path, + remote_store.as_ref(), + ) + }) + .buffer_unordered(10); + while let Some(v) = tokio_stream::StreamExt::next(&mut stream).await { + match v { + Ok(v) if v => { + self.metrics.nr_substitutes_succeeded.inc(); + substituted += 1; + } + Ok(_) => { + self.metrics.nr_substitutes_failed.inc(); + } + Err(e) => { + self.metrics.nr_substitutes_failed.inc(); + tracing::warn!("Failed to substitute path: {e}"); + } + } + } + substituted == missing_outputs_len + } else { + missing_outputs.is_empty() + }; + + if finished { + if let Some(fod_checker) = &self.fod_checker { + fod_checker.to_traverse(&drv_path); + } + + finished_drvs.write().insert(drv_path.clone()); + step.set_finished(true); + return CreateStepResult::None; + } + + tracing::debug!("creating build step '{drv_path}"); + let Some(input_drvs) = step.get_input_drvs() else { + // this should never happen because we always a a drv set at this point in time + return CreateStepResult::None; + }; + + let step2 = step.clone(); + let mut stream = futures::StreamExt::map(tokio_stream::iter(input_drvs), |i| { + let build = build.clone(); + let step = step2.clone(); + let finished_drvs = finished_drvs.clone(); + let new_steps = new_steps.clone(); + let new_runnable = new_runnable.clone(); + async move { + let path = nix_utils::StorePath::new(&i); + Box::pin(self.create_step( + // conn, + build, + path, + None, + Some(step), + finished_drvs, + new_steps, + new_runnable, + )) + .await + } + }) + .buffered(25); + while let Some(v) = tokio_stream::StreamExt::next(&mut stream).await { + match v { + CreateStepResult::None => (), + CreateStepResult::Valid(dep) => { + if !dep.get_finished() && !dep.get_previous_failure() { + // finished can be true if a step was returned, that already exists in + // self.steps and is currently being processed for completion + step.add_dep(dep); + } + } + CreateStepResult::PreviousFailure(step) => { + return CreateStepResult::PreviousFailure(step); + } + } + } + + { + step.atomic_state.set_created(true); + if step.get_deps_size() == 0 { + let mut new_runnable = new_runnable.write(); + new_runnable.insert(step.clone()); + } + } + + { + let mut new_steps = new_steps.write(); + new_steps.insert(step.clone()); + } + CreateStepResult::Valid(step) + } + + #[tracing::instrument(skip(self, step), ret, level = "debug")] + async fn check_cached_failure(&self, step: Arc) -> bool { + let Some(drv_outputs) = step.get_outputs() else { + return false; + }; + + let Ok(mut conn) = self.db.get().await else { + return false; + }; + + conn.check_if_paths_failed( + &drv_outputs + .iter() + .filter_map(|o| o.path.as_ref().map(|p| self.store.print_store_path(p))) + .collect::>(), + ) + .await + .unwrap_or_default() + } + + #[tracing::instrument(skip(self, build), fields(build_id=build.id), err)] + async fn handle_cached_build(&self, build: Arc) -> anyhow::Result<()> { + let res = self.get_build_output_cached(&build.drv_path).await?; + + for (_, path) in &res.outputs { + self.add_root(path); + } + + { + let mut db = self.db.get().await?; + let mut tx = db.begin_transaction().await?; + + tracing::info!("marking build {} as succeeded (cached)", build.id); + let now = jiff::Timestamp::now().as_second(); + tx.mark_succeeded_build( + get_mark_build_sccuess_data(&self.store, &build, &res), + true, + i32::try_from(now)?, // TODO + i32::try_from(now)?, // TODO + ) + .await?; + self.metrics.nr_builds_done.inc(); + + tx.notify_build_finished(build.id, &[]).await?; + tx.commit().await?; + } + build.set_finished_in_db(true); + + Ok(()) + } + + #[tracing::instrument(skip(self), err)] + async fn get_build_output_cached( + &self, + drv_path: &nix_utils::StorePath, + ) -> anyhow::Result { + let drv = nix_utils::query_drv(&self.store, drv_path) + .await? + .ok_or_else(|| anyhow::anyhow!("Derivation not found"))?; + + { + let mut db = self.db.get().await?; + for o in &drv.outputs { + let Some(out_path) = &o.path else { + continue; + }; + let Some(db_build_output) = db + .get_build_output_for_path(&self.store.print_store_path(out_path)) + .await? + else { + continue; + }; + let build_id = db_build_output.id; + let Ok(mut res): anyhow::Result = db_build_output.try_into() else { + continue; + }; + + res.products = db + .get_build_products_for_build_id(build_id) + .await? + .into_iter() + .map(Into::into) + .collect(); + res.metrics = db + .get_build_metrics_for_build_id(build_id) + .await? + .into_iter() + .map(|v| (v.name.clone(), v.into())) + .collect(); + + return Ok(res); + } + } + + let build_output = BuildOutput::new(&self.store, drv.outputs.to_vec()).await?; + + #[allow(clippy::cast_precision_loss)] + self.metrics + .observe_build_closure_size(build_output.closure_size as f64, &drv.system); + + Ok(build_output) + } + + fn add_root(&self, drv_path: &nix_utils::StorePath) { + let roots_dir = self.config.get_roots_dir(); + nix_utils::add_root(&self.store, &roots_dir, drv_path); + } + + async fn abort_unsupported(&self) { + let runnable = self.steps.clone_runnable(); + let now = jiff::Timestamp::now(); + + let mut aborted = HashSet::new(); + let mut count = 0; + + let max_unsupported_time = self.config.get_max_unsupported_time(); + for step in &runnable { + let supported = self.machines.support_step(step); + if supported { + step.set_last_supported_now(); + continue; + } + + count += 1; + if (now - step.get_last_supported()) + .total(jiff::Unit::Second) + .unwrap_or_default() + < max_unsupported_time.as_secs_f64() + { + continue; + } + + let drv = step.get_drv_path(); + let system = step.get_system(); + tracing::error!("aborting unsupported build step '{drv}' (type '{system:?}')",); + + aborted.insert(step.clone()); + + let mut dependents = HashSet::new(); + let mut steps = HashSet::new(); + step.get_dependents(&mut dependents, &mut steps); + // Maybe the step got cancelled. + if dependents.is_empty() { + continue; + } + + // Find the build that has this step as the top-level (if any). + let Some(build) = dependents + .iter() + .find(|b| &b.drv_path == drv) + .or_else(|| dependents.iter().next()) + else { + // this should never happen, as we checked is_empty above and fallback is just any build + continue; + }; + + let mut job = machine::Job::new(build.id, drv.to_owned(), None); + job.result.set_start_and_stop(now); + job.result.step_status = BuildStatus::Unsupported; + job.result.error_msg = Some(format!( + "unsupported system type '{}'", + system.unwrap_or(String::new()) + )); + if let Err(e) = self.inner_fail_job(drv, None, job, step.clone()).await { + tracing::error!("Failed to fail step drv={drv} e={e}"); + } + } + + { + for step in &aborted { + self.queues.remove_job_by_path(step.get_drv_path()).await; + } + self.queues.remove_all_weak_pointer().await; + } + self.metrics.nr_unsupported_steps.set(count); + self.metrics + .nr_unsupported_steps_aborted + .inc_by(aborted.len() as u64); + } +} diff --git a/src/queue-runner/src/state/queue.rs b/src/queue-runner/src/state/queue.rs new file mode 100644 index 000000000..46ba9e77f --- /dev/null +++ b/src/queue-runner/src/state/queue.rs @@ -0,0 +1,668 @@ +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Weak}; + +use hashbrown::{HashMap, HashSet}; +use smallvec::SmallVec; + +use super::{StepInfo, System}; +use crate::config::StepSortFn; + +#[derive(Debug)] +pub struct BuildQueue { + // Note: ensure that this stays private + jobs: parking_lot::RwLock>>, + + active_runnable: AtomicU64, + total_runnable: AtomicU64, + nr_runnable_waiting: AtomicU64, + nr_runnable_disabled: AtomicU64, + avg_runnable_time: AtomicU64, + wait_time_ms: AtomicU64, +} + +#[derive(Debug)] +pub struct BuildQueueStats { + pub active_runnable: u64, + pub total_runnable: u64, + pub nr_runnable_waiting: u64, + pub nr_runnable_disabled: u64, + pub avg_runnable_time: u64, + pub wait_time: u64, +} + +impl BuildQueue { + fn new() -> Self { + Self { + jobs: parking_lot::RwLock::new(Vec::new()), + active_runnable: 0.into(), + total_runnable: 0.into(), + nr_runnable_waiting: 0.into(), + nr_runnable_disabled: 0.into(), + avg_runnable_time: 0.into(), + wait_time_ms: 0.into(), + } + } + + pub fn set_nr_runnable_waiting(&self, v: u64) { + self.nr_runnable_waiting.store(v, Ordering::Relaxed); + } + + pub fn set_nr_runnable_disabled(&self, v: u64) { + self.nr_runnable_disabled.store(v, Ordering::Relaxed); + } + + fn incr_active(&self) { + self.active_runnable.fetch_add(1, Ordering::Relaxed); + } + + fn decr_active(&self) { + self.active_runnable.fetch_sub(1, Ordering::Relaxed); + } + + #[tracing::instrument(skip(self, jobs))] + fn insert_new_jobs( + &self, + jobs: Vec>, + now: &jiff::Timestamp, + sort_fn: StepSortFn, + ) -> u64 { + let mut current_jobs = self.jobs.write(); + let mut wait_time_ms = 0u64; + + for j in jobs { + if let Some(owned) = j.upgrade() { + // this ensures we only ever have each step once + // so ensure that current_jobs is never written anywhere else + // this should never continue as jobs, should already exclude duplicates + if current_jobs + .iter() + .filter_map(std::sync::Weak::upgrade) + .any(|v| v.step.get_drv_path() == owned.step.get_drv_path()) + { + continue; + } + + // runnable since is always > now + wait_time_ms += u64::try_from(now.duration_since(owned.runnable_since).as_millis()) + .unwrap_or_default(); + current_jobs.push(j); + } + } + self.wait_time_ms.fetch_add(wait_time_ms, Ordering::Relaxed); + + // only keep valid pointers + drop(current_jobs); + self.scrube_jobs(); + self.sort_jobs(sort_fn) + } + + #[tracing::instrument(skip(self))] + pub fn sort_jobs(&self, sort_fn: StepSortFn) -> u64 { + let start_time = std::time::Instant::now(); + let cmp_fn = match sort_fn { + StepSortFn::Legacy => StepInfo::legacy_compare, + StepSortFn::WithRdeps => StepInfo::compare_with_rdeps, + }; + + { + let mut current_jobs = self.jobs.write(); + for job in current_jobs.iter_mut() { + let Some(job) = job.upgrade() else { continue }; + job.update_internal_stats(); + } + + current_jobs.sort_by(|a, b| { + let a = a.upgrade(); + let b = b.upgrade(); + match (a, b) { + (Some(a), Some(b)) => cmp_fn(a.as_ref(), b.as_ref()), + (Some(_), None) => std::cmp::Ordering::Greater, + (None, Some(_)) => std::cmp::Ordering::Less, + (None, None) => std::cmp::Ordering::Equal, + } + }); + } + u64::try_from(start_time.elapsed().as_millis()).unwrap_or_default() + } + + #[tracing::instrument(skip(self))] + pub fn scrube_jobs(&self) { + let mut current_jobs = self.jobs.write(); + current_jobs.retain(|v| v.upgrade().is_some()); + self.total_runnable + .store(current_jobs.len() as u64, Ordering::Relaxed); + } + + pub fn clone_inner(&self) -> Vec> { + (*self.jobs.read()).clone() + } + + pub fn get_stats(&self) -> BuildQueueStats { + BuildQueueStats { + active_runnable: self.active_runnable.load(Ordering::Relaxed), + total_runnable: self.total_runnable.load(Ordering::Relaxed), + nr_runnable_waiting: self.nr_runnable_waiting.load(Ordering::Relaxed), + nr_runnable_disabled: self.nr_runnable_disabled.load(Ordering::Relaxed), + avg_runnable_time: self.avg_runnable_time.load(Ordering::Relaxed), + wait_time: self.wait_time_ms.load(Ordering::Relaxed), + } + } +} + +#[derive(Clone)] +pub struct ScheduledItem { + pub step_info: Arc, + pub build_queue: Arc, + pub machine: Arc, +} + +impl ScheduledItem { + const fn new( + step_info: Arc, + build_queue: Arc, + machine: Arc, + ) -> Self { + Self { + step_info, + build_queue, + machine, + } + } +} + +pub struct InnerQueues { + // flat list of all step infos in queues, owning those steps inner queue dont own them + jobs: HashMap>, + inner: HashMap>, + #[allow(clippy::type_complexity)] + scheduled: parking_lot::RwLock>, +} + +impl Default for InnerQueues { + fn default() -> Self { + Self::new() + } +} + +impl InnerQueues { + fn new() -> Self { + Self { + jobs: HashMap::with_capacity(1000), + inner: HashMap::with_capacity(4), + scheduled: parking_lot::RwLock::new(HashMap::with_capacity(100)), + } + } + + #[tracing::instrument(skip(self, jobs))] + fn insert_new_jobs + std::fmt::Debug>( + &mut self, + system: S, + jobs: Vec, + now: &jiff::Timestamp, + sort_fn: StepSortFn, + ) -> u64 { + let mut submit_jobs: Vec> = Vec::new(); + for j in jobs { + let j = Arc::new(j); + // we need to check that get_finished is not true! + // the reason for this is that while a job is currently being proccessed for finished + // it can be resubmitted into the queues. + // to ensure that this does not block everything we need to ensure that it doesnt land + // here. + if !self.jobs.contains_key(j.step.get_drv_path()) && !j.step.get_finished() { + self.jobs + .insert(j.step.get_drv_path().to_owned(), j.clone()); + submit_jobs.push(Arc::downgrade(&j)); + } + } + + let queue = self + .inner + .entry(system.into()) + .or_insert_with(|| Arc::new(BuildQueue::new())); + // queues are sorted afterwards + queue.insert_new_jobs(submit_jobs, now, sort_fn) + } + + #[tracing::instrument(skip(self))] + fn ensure_queues_for_systems(&mut self, systems: &[System]) { + for system in systems { + self.inner + .entry(system.clone()) + .or_insert_with(|| Arc::new(BuildQueue::new())); + } + } + + #[tracing::instrument(skip(self))] + fn remove_all_weak_pointer(&self) { + for queue in self.inner.values() { + queue.scrube_jobs(); + } + } + + fn clone_inner(&self) -> HashMap> { + self.inner.clone() + } + + #[tracing::instrument(skip(self, step, queue))] + fn add_job_to_scheduled( + &self, + step: &Arc, + queue: &Arc, + machine: Arc, + ) { + self.scheduled.write().insert( + step.step.get_drv_path().to_owned(), + ScheduledItem::new(step.clone(), queue.clone(), machine), + ); + step.set_already_scheduled(true); + queue.incr_active(); + } + + #[tracing::instrument(skip(self), fields(%drv))] + fn remove_job_from_scheduled(&self, drv: &nix_utils::StorePath) -> Option { + let item = self.scheduled.write().remove(drv)?; + item.step_info.set_already_scheduled(false); + item.build_queue.decr_active(); + Some(item) + } + + fn remove_job_by_path(&mut self, drv: &nix_utils::StorePath) { + if self.jobs.remove(drv).is_none() { + tracing::error!("Failed to remove stepinfo drv={drv} from jobs!"); + } + } + + #[tracing::instrument(skip(self, stepinfo, queue))] + fn remove_job(&mut self, stepinfo: &Arc, queue: &Arc) { + if self.jobs.remove(stepinfo.step.get_drv_path()).is_none() { + tracing::error!( + "Failed to remove stepinfo drv={} from jobs!", + stepinfo.step.get_drv_path(), + ); + } + // active should be removed + queue.scrube_jobs(); + } + + #[tracing::instrument(skip(self))] + async fn kill_active_steps(&self) -> Vec<(nix_utils::StorePath, uuid::Uuid)> { + tracing::info!("Kill all active steps"); + let active = { + let scheduled = self.scheduled.read(); + scheduled.clone() + }; + + let mut cancelled_steps = vec![]; + for (drv_path, item) in &active { + if item.step_info.get_cancelled() { + continue; + } + + let mut dependents = HashSet::new(); + let mut steps = HashSet::new(); + item.step_info + .step + .get_dependents(&mut dependents, &mut steps); + if !dependents.is_empty() { + continue; + } + + { + tracing::info!("Cancelling step drv={drv_path}"); + item.step_info.set_cancelled(true); + + if let Some(internal_build_id) = + item.machine.get_internal_build_id_for_drv(drv_path) + { + if let Err(e) = item.machine.abort_build(internal_build_id).await { + tracing::error!( + "Failed to abort build drv_path={drv_path} build_id={internal_build_id} e={e}", + ); + continue; + } + } else { + tracing::warn!("No active build_id found for drv_path={drv_path}",); + continue; + } + + cancelled_steps.push((drv_path.to_owned(), item.machine.id)); + } + } + cancelled_steps + } + + #[tracing::instrument(skip(self))] + fn get_stats_per_queue(&self) -> HashMap { + self.inner + .iter() + .map(|(k, v)| (k.clone(), v.get_stats())) + .collect() + } + + fn get_jobs(&self) -> Vec> { + self.jobs.values().map(Clone::clone).collect() + } + + fn get_scheduled(&self) -> Vec> { + let s = self.scheduled.read(); + s.iter().map(|(_, item)| item.step_info.clone()).collect() + } + + pub fn sort_queues(&self, sort_fn: StepSortFn) { + for q in self.inner.values() { + q.sort_jobs(sort_fn); + } + } +} + +pub struct JobConstraint { + job: Arc, + system: System, + queue_features: SmallVec<[String; 4]>, +} + +impl JobConstraint { + pub const fn new( + job: Arc, + system: System, + queue_features: SmallVec<[String; 4]>, + ) -> Self { + Self { + job, + system, + queue_features, + } + } + + pub fn resolve( + self, + machines: &crate::state::Machines, + free_fn: crate::config::MachineFreeFn, + ) -> Option<(Arc, Arc)> { + let step_features = self.job.step.get_required_features(); + let merged_features = if self.queue_features.is_empty() { + step_features + } else { + [step_features.as_slice(), self.queue_features.as_slice()].concat() + }; + if let Some(machine) = + machines.get_machine_for_system(&self.system, &merged_features, Some(free_fn)) + { + Some((machine, self.job)) + } else { + let drv = self.job.step.get_drv_path(); + tracing::debug!("No free machine found for system={} drv={drv}", self.system); + None + } + } +} + +#[derive(Clone)] +pub struct Queues { + inner: Arc>, +} + +impl Default for Queues { + fn default() -> Self { + Self::new() + } +} + +impl Queues { + #[must_use] + pub fn new() -> Self { + Self { + inner: Arc::new(tokio::sync::RwLock::new(InnerQueues::new())), + } + } + + #[tracing::instrument(skip(self, jobs))] + pub async fn insert_new_jobs + std::fmt::Debug>( + &self, + system: S, + jobs: Vec, + now: &jiff::Timestamp, + sort_fn: StepSortFn, + metrics: &super::metrics::PromMetrics, + ) { + let sort_duration = self + .inner + .write() + .await + .insert_new_jobs(system, jobs, now, sort_fn); + metrics.queue_sort_duration_ms_total.inc_by(sort_duration); + } + + #[tracing::instrument(skip(self))] + pub async fn remove_all_weak_pointer(&self) { + let rq = self.inner.write().await; + rq.remove_all_weak_pointer(); + } + + #[tracing::instrument(skip(self))] + pub async fn ensure_queues_for_systems(&self, systems: &[System]) { + let mut wq = self.inner.write().await; + wq.ensure_queues_for_systems(systems); + } + + pub(super) async fn process( + &self, + processor: F, + metrics: &super::metrics::PromMetrics, + ) -> i64 + where + F: AsyncFn(JobConstraint) -> anyhow::Result, + { + let now = jiff::Timestamp::now(); + let mut nr_steps_waiting_all_queues = 0; + let queues = self.clone_inner().await; + for (system, queue) in queues { + let mut nr_disabled = 0; + let mut nr_waiting = 0; + for job in queue.clone_inner() { + let Some(job) = job.upgrade() else { + continue; + }; + if job.get_already_scheduled() { + tracing::debug!( + "Can't schedule job because job is already scheduled system={system} drv={}", + job.step.get_drv_path() + ); + continue; + } + if job.step.get_finished() { + tracing::debug!( + "Can't schedule job because job is already finished system={system} drv={}", + job.step.get_drv_path() + ); + continue; + } + let after = job.step.get_after(); + if after > now { + nr_disabled += 1; + tracing::debug!( + "Can't schedule job because job is not yet ready system={system} drv={} after={after}", + job.step.get_drv_path(), + ); + continue; + } + let constraint = JobConstraint::new(job.clone(), system.clone(), SmallVec::new()); + match processor(constraint).await { + Ok(crate::state::RealiseStepResult::Valid(m)) => { + let wait_seconds = now.duration_since(job.runnable_since).as_secs_f64(); + metrics.observe_job_wait_time(wait_seconds, &system); + + self.add_job_to_scheduled(&job, &queue, m).await; + } + Ok(crate::state::RealiseStepResult::None) => { + tracing::debug!( + "Waiting for job to schedule because no builder is ready system={system} drv={}", + job.step.get_drv_path(), + ); + nr_waiting += 1; + nr_steps_waiting_all_queues += 1; + } + Ok( + crate::state::RealiseStepResult::MaybeCancelled + | crate::state::RealiseStepResult::CachedFailure, + ) => { + // If this is maybe cancelled (and the cancellation is correct) it is + // enough to remove it from jobs which will then reduce the ref count + // to 0 as it has no dependents. + // If its a cached failure we need to also remove it from jobs, we + // already wrote cached failure into the db, at this point in time + self.remove_job(&job, &queue).await; + + metrics.queue_aborted_jobs_total.inc(); + } + Err(e) => { + tracing::warn!( + "Failed to realise drv on valid machine, will be skipped: drv={} e={e}", + job.step.get_drv_path(), + ); + } + } + queue.set_nr_runnable_waiting(nr_waiting); + queue.set_nr_runnable_disabled(nr_disabled); + } + } + nr_steps_waiting_all_queues + } + + pub async fn clone_inner(&self) -> HashMap> { + self.inner.read().await.clone_inner() + } + + #[tracing::instrument(skip(self, step, queue))] + pub async fn add_job_to_scheduled( + &self, + step: &Arc, + queue: &Arc, + machine: Arc, + ) { + let rq = self.inner.read().await; + rq.add_job_to_scheduled(step, queue, machine); + } + + #[tracing::instrument(skip(self), fields(%drv))] + pub async fn remove_job_from_scheduled( + &self, + drv: &nix_utils::StorePath, + ) -> Option { + let rq = self.inner.read().await; + rq.remove_job_from_scheduled(drv) + } + + pub async fn remove_job_by_path(&self, drv: &nix_utils::StorePath) { + let mut wq = self.inner.write().await; + wq.remove_job_by_path(drv); + } + + #[tracing::instrument(skip(self, stepinfo, queue))] + pub async fn remove_job(&self, stepinfo: &Arc, queue: &Arc) { + let mut wq = self.inner.write().await; + wq.remove_job(stepinfo, queue); + } + + #[tracing::instrument(skip(self))] + pub async fn kill_active_steps(&self) -> Vec<(nix_utils::StorePath, uuid::Uuid)> { + let rq = self.inner.read().await; + rq.kill_active_steps().await + } + + #[tracing::instrument(skip(self))] + pub async fn get_stats_per_queue(&self) -> HashMap { + self.inner.read().await.get_stats_per_queue() + } + + pub async fn get_jobs(&self) -> Vec> { + let rq = self.inner.read().await; + rq.get_jobs() + } + + pub async fn get_scheduled(&self) -> Vec> { + let rq = self.inner.read().await; + rq.get_scheduled() + } + + pub async fn sort_queues(&self, sort_fn: StepSortFn) { + let rq = self.inner.read().await; + rq.sort_queues(sort_fn); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::state::System; + + #[tokio::test] + async fn test_ensure_queues_for_systems() { + let queues = Queues::new(); + let systems = vec!["system1".to_string(), "system2".to_string()]; + + // Ensure queues for systems + queues.ensure_queues_for_systems(&systems).await; + + // Check that queues were created + let inner = queues.inner.read().await; + assert!(inner.inner.contains_key("system1")); + assert!(inner.inner.contains_key("system2")); + assert_eq!(inner.inner.len(), 2); + } + + #[tokio::test] + async fn test_ensure_queues_for_systems_empty() { + let queues = Queues::new(); + let systems: Vec = vec![]; + + // Ensure queues for empty systems list + queues.ensure_queues_for_systems(&systems).await; + + // Check that no queues were created + let inner = queues.inner.read().await; + assert_eq!(inner.inner.len(), 0); + } + + #[tokio::test] + async fn test_ensure_queues_for_systems_duplicate() { + let queues = Queues::new(); + let systems1 = vec!["system1".to_string(), "system2".to_string()]; + let systems2 = vec!["system2".to_string(), "system3".to_string()]; + + // Ensure queues for first set of systems + queues.ensure_queues_for_systems(&systems1).await; + + // Ensure queues for second set of systems (with overlap) + queues.ensure_queues_for_systems(&systems2).await; + + // Check that all queues were created but no duplicates + let inner = queues.inner.read().await; + assert!(inner.inner.contains_key("system1")); + assert!(inner.inner.contains_key("system2")); + assert!(inner.inner.contains_key("system3")); + assert_eq!(inner.inner.len(), 3); + } + + #[tokio::test] + async fn test_insert_machine_creates_queues_integration() { + // Test the integration concept - what happens when insert_machine is called + let systems = vec!["x86_64-linux".to_string(), "aarch64-linux".to_string()]; + let queues = Queues::new(); + + // Before: no queues + let inner_before = queues.inner.read().await; + assert_eq!(inner_before.inner.len(), 0); + drop(inner_before); + + // Call ensure_queues_for_systems (what insert_machine does) + queues.ensure_queues_for_systems(&systems).await; + + // After: queues should exist for all systems + let inner_after = queues.inner.read().await; + assert_eq!(inner_after.inner.len(), 2); + assert!(inner_after.inner.contains_key("x86_64-linux")); + assert!(inner_after.inner.contains_key("aarch64-linux")); + } +} diff --git a/src/queue-runner/src/state/step.rs b/src/queue-runner/src/state/step.rs new file mode 100644 index 000000000..c5478369b --- /dev/null +++ b/src/queue-runner/src/state/step.rs @@ -0,0 +1,501 @@ +use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Weak}; + +use hashbrown::{HashMap, HashSet}; + +use super::{Build, Jobset}; +use db::models::BuildID; + +#[derive(Debug)] +pub struct StepAtomicState { + created: AtomicBool, // Whether the step has finished initialisation. + pub tries: AtomicU32, // Number of times we've tried this step. + pub highest_global_priority: AtomicI32, // The highest global priority of any build depending on this step. + pub highest_local_priority: AtomicI32, // The highest local priority of any build depending on this step. + + pub lowest_build_id: super::build::AtomicBuildID, // The lowest ID of any build depending on this step. + + pub after: super::AtomicDateTime, // Point in time after which the step can be retried. + pub runnable_since: super::AtomicDateTime, // The time at which this step became runnable. + pub last_supported: super::AtomicDateTime, // The time that we last saw a machine that supports this step + + pub deps_len: AtomicU64, + pub rdeps_len: AtomicU64, +} + +impl StepAtomicState { + pub fn new(after: jiff::Timestamp, runnable_since: jiff::Timestamp) -> Self { + Self { + created: false.into(), + tries: 0.into(), + highest_global_priority: 0.into(), + highest_local_priority: 0.into(), + lowest_build_id: BuildID::MAX.into(), + after: super::AtomicDateTime::new(after), + runnable_since: super::AtomicDateTime::new(runnable_since), + // Set the default of last_supported to runnable_since. + // This fixes an issue that a step is marked as unsupported immediatly, if we currently + // dont have a machine that supports system/all features. + // So we still follow max_unsupported_time + last_supported: super::AtomicDateTime::new(runnable_since), + deps_len: 0.into(), + rdeps_len: 0.into(), + } + } + + #[inline] + pub fn get_created(&self) -> bool { + self.created.load(Ordering::SeqCst) + } + + #[inline] + pub fn set_created(&self, v: bool) { + self.created.store(v, Ordering::SeqCst); + } +} + +#[derive(Debug)] +pub struct StepState { + deps: HashSet>, // The build steps on which this step depends. + rdeps: Vec>, // The build steps that depend on this step. + builds: Vec>, // Builds that have this step as the top-level derivation. + jobsets: HashSet>, // Jobsets to which this step belongs. Used for determining scheduling priority. +} + +impl Default for StepState { + fn default() -> Self { + Self::new() + } +} + +impl StepState { + pub fn new() -> Self { + Self { + deps: HashSet::new(), + rdeps: Vec::new(), + builds: Vec::new(), + jobsets: HashSet::new(), + } + } +} + +#[derive(Debug)] +pub struct Step { + drv_path: nix_utils::StorePath, + drv: arc_swap::ArcSwapOption, + + runnable: AtomicBool, + finished: AtomicBool, + previous_failure: AtomicBool, + pub atomic_state: StepAtomicState, + state: parking_lot::RwLock, +} + +impl PartialEq for Step { + fn eq(&self, other: &Self) -> bool { + self.drv_path == other.drv_path + } +} + +impl Eq for Step {} + +impl std::hash::Hash for Step { + fn hash(&self, state: &mut H) { + // ensure that drv_path is never mutable + // as we set Step as ignore-interior-mutability + self.drv_path.hash(state); + } +} + +impl Step { + #[must_use] + pub fn new(drv_path: nix_utils::StorePath) -> Arc { + Arc::new(Self { + drv_path, + drv: arc_swap::ArcSwapOption::from(None), + runnable: false.into(), + finished: false.into(), + previous_failure: false.into(), + atomic_state: StepAtomicState::new( + jiff::Timestamp::UNIX_EPOCH, + jiff::Timestamp::UNIX_EPOCH, + ), + state: parking_lot::RwLock::new(StepState::new()), + }) + } + + #[inline] + pub const fn get_drv_path(&self) -> &nix_utils::StorePath { + &self.drv_path + } + + #[inline] + pub fn get_finished(&self) -> bool { + self.finished.load(Ordering::SeqCst) + } + + #[inline] + pub fn set_finished(&self, v: bool) { + self.finished.store(v, Ordering::SeqCst); + } + + #[inline] + pub fn get_previous_failure(&self) -> bool { + self.previous_failure.load(Ordering::SeqCst) + } + + #[inline] + pub fn set_previous_failure(&self, v: bool) { + self.previous_failure.store(v, Ordering::SeqCst); + } + + #[inline] + pub fn get_runnable(&self) -> bool { + self.runnable.load(Ordering::SeqCst) + } + + pub fn set_drv(&self, drv: nix_utils::Derivation) { + self.drv.store(Some(Arc::new(drv))); + } + + pub fn get_system(&self) -> Option { + let drv = self.drv.load_full(); + drv.as_ref().map(|drv| drv.system.clone()) + } + + pub fn get_input_drvs(&self) -> Option> { + let drv = self.drv.load_full(); + drv.as_ref().map(|drv| drv.input_drvs.to_vec()) + } + + pub fn get_after(&self) -> jiff::Timestamp { + self.atomic_state.after.load() + } + + pub fn set_after(&self, v: jiff::Timestamp) { + self.atomic_state.after.store(v); + } + + pub fn get_runnable_since(&self) -> jiff::Timestamp { + self.atomic_state.runnable_since.load() + } + + pub fn get_last_supported(&self) -> jiff::Timestamp { + self.atomic_state.last_supported.load() + } + + pub fn set_last_supported_now(&self) { + self.atomic_state + .last_supported + .store(jiff::Timestamp::now()); + } + + pub fn get_outputs(&self) -> Option> { + let drv = self.drv.load_full(); + drv.as_ref().map(|drv| drv.outputs.to_vec()) + } + + pub fn get_required_features(&self) -> Vec { + let drv = self.drv.load_full(); + drv.as_ref() + .map(|drv| { + drv.env + .get_required_system_features() + .into_iter() + .map(ToOwned::to_owned) + .collect() + }) + .unwrap_or_default() + } + + #[tracing::instrument(skip(self, builds, steps))] + pub fn get_dependents( + self: &Arc, + builds: &mut HashSet>, + steps: &mut HashSet>, + ) { + if steps.contains(self) { + return; + } + steps.insert(self.clone()); + + let rdeps = { + let state = self.state.read(); + for b in &state.builds { + let Some(b) = b.upgrade() else { continue }; + + if !b.get_finished_in_db() { + builds.insert(b); + } + } + state.rdeps.clone() + }; + + for rdep in rdeps { + let Some(rdep) = rdep.upgrade() else { continue }; + rdep.get_dependents(builds, steps); + } + } + + pub fn get_deps_size(&self) -> u64 { + self.atomic_state.deps_len.load(Ordering::Relaxed) + } + + pub fn make_rdeps_runnable(&self) { + if !self.get_finished() { + return; + } + + let mut state = self.state.write(); + state.rdeps.retain(|rdep| { + let Some(rdep) = rdep.upgrade() else { + return false; + }; + + let mut runnable = false; + { + let mut rdep_state = rdep.state.write(); + rdep_state + .deps + .retain(|s| s.get_drv_path() != self.get_drv_path()); + rdep.atomic_state + .deps_len + .store(rdep_state.deps.len() as u64, Ordering::Relaxed); + if rdep_state.deps.is_empty() && rdep.atomic_state.get_created() { + runnable = true; + } + } + + if runnable { + rdep.make_runnable(); + } + true + }); + self.atomic_state + .rdeps_len + .store(state.rdeps.len() as u64, Ordering::Relaxed); + } + + #[tracing::instrument(skip(self))] + pub fn make_runnable(&self) { + debug_assert!(self.atomic_state.created.load(Ordering::SeqCst)); + debug_assert!(!self.get_finished()); + + #[cfg(debug_assertions)] + { + let state = self.state.read(); + debug_assert!(state.deps.is_empty()); + } + + // only ever mark as runnable once + if !self.runnable.load(Ordering::SeqCst) { + tracing::info!("step '{}' is now runnable", self.get_drv_path()); + + self.runnable.store(true, Ordering::SeqCst); + let now = jiff::Timestamp::now(); + self.atomic_state.runnable_since.store(now); + // we also say now, is the last time that we supported this step. + // This ensure that we actually wait for max_unsupported_time until we mark it as + // unsupported. See also [`StepAtomicState::new`] + self.atomic_state.last_supported.store(now); + } + } + + pub fn get_lowest_share_used(&self) -> f64 { + let state = self.state.read(); + + state + .jobsets + .iter() + .map(|v| v.share_used()) + .min_by(f64::total_cmp) + .unwrap_or(1e9) + } + + pub fn add_jobset(&self, jobset: Arc) { + let mut state = self.state.write(); + state.jobsets.insert(jobset); + } + + pub fn add_dep(&self, dep: Arc) { + let mut state = self.state.write(); + state.deps.insert(dep); + self.atomic_state + .deps_len + .store(state.deps.len() as u64, Ordering::Relaxed); + } + + pub fn add_referring_data( + &self, + referring_build: Option<&Arc>, + referring_step: Option<&Arc>, + ) { + if referring_build.is_none() && referring_step.is_none() { + return; + } + + let mut state = self.state.write(); + if let Some(referring_build) = referring_build { + state.builds.push(Arc::downgrade(referring_build)); + } + if let Some(referring_step) = referring_step { + state.rdeps.push(Arc::downgrade(referring_step)); + self.atomic_state + .rdeps_len + .store(state.rdeps.len() as u64, Ordering::Relaxed); + } + } + + pub fn get_direct_builds(&self) -> Vec> { + let mut direct = Vec::new(); + let state = self.state.read(); + for b in &state.builds { + let Some(b) = b.upgrade() else { + continue; + }; + if !b.get_finished_in_db() { + direct.push(b); + } + } + + direct + } + + pub fn get_all_deps_not_queued(&self, queued: &HashSet>) -> Vec> { + let state = self.state.read(); + state + .deps + .iter() + .filter(|dep| !queued.contains(*dep)) + .map(Clone::clone) + .collect() + } +} + +#[derive(Clone)] +pub struct Steps { + inner: Arc>>>, +} + +impl Default for Steps { + fn default() -> Self { + Self::new() + } +} + +impl Steps { + #[must_use] + pub fn new() -> Self { + Self { + inner: Arc::new(parking_lot::RwLock::new(HashMap::with_capacity(10000))), + } + } + + #[must_use] + pub fn len(&self) -> usize { + let mut steps = self.inner.write(); + steps.retain(|_, s| s.upgrade().is_some()); + steps.len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + let mut steps = self.inner.write(); + steps.retain(|_, s| s.upgrade().is_some()); + steps.is_empty() + } + + #[must_use] + pub fn len_runnable(&self) -> usize { + let mut steps = self.inner.write(); + steps.retain(|_, s| s.upgrade().is_some()); + steps + .iter() + .filter_map(|(_, s)| s.upgrade().map(|v| v.get_runnable())) + .filter(|v| *v) + .count() + } + + #[must_use] + pub fn clone_as_io(&self) -> Vec { + let steps = self.inner.read(); + steps + .values() + .filter_map(std::sync::Weak::upgrade) + .map(Into::into) + .collect() + } + + #[must_use] + pub fn clone_runnable_as_io(&self) -> Vec { + let steps = self.inner.read(); + steps + .values() + .filter_map(std::sync::Weak::upgrade) + .filter(|v| v.get_runnable()) + .map(Into::into) + .collect() + } + + #[must_use] + pub fn clone_runnable(&self) -> Vec> { + let mut steps = self.inner.write(); + let mut new_runnable = Vec::with_capacity(steps.len()); + steps.retain(|_, r| { + let Some(step) = r.upgrade() else { + return false; + }; + if step.get_runnable() { + new_runnable.push(step); + } + true + }); + new_runnable + } + + pub fn make_rdeps_runnable(&self) { + let steps = self.inner.read(); + for (_, s) in steps.iter() { + let Some(s) = s.upgrade() else { + continue; + }; + if s.get_finished() && !s.get_previous_failure() { + s.make_rdeps_runnable(); + } + // TODO: if previous failure we should propably also remove from deps + } + } + + #[must_use] + pub fn create( + &self, + drv_path: &nix_utils::StorePath, + referring_build: Option<&Arc>, + referring_step: Option<&Arc>, + ) -> (Arc, bool) { + let mut is_new = false; + let mut steps = self.inner.write(); + let step = if let Some(step) = steps.get(drv_path) { + step.upgrade().map_or_else( + || { + steps.remove(drv_path); + is_new = true; + Step::new(drv_path.to_owned()) + }, + |step| step, + ) + } else { + is_new = true; + Step::new(drv_path.to_owned()) + }; + + step.add_referring_data(referring_build, referring_step); + steps.insert(drv_path.to_owned(), Arc::downgrade(&step)); + (step, is_new) + } + + pub fn remove(&self, drv_path: &nix_utils::StorePath) { + let mut steps = self.inner.write(); + steps.remove(drv_path); + } +} diff --git a/src/queue-runner/src/state/step_info.rs b/src/queue-runner/src/state/step_info.rs new file mode 100644 index 000000000..c523bfee0 --- /dev/null +++ b/src/queue-runner/src/state/step_info.rs @@ -0,0 +1,305 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use db::models::BuildID; +use nix_utils::BaseStore as _; + +use super::Step; + +pub struct StepInfo { + pub step: Arc, + pub resolved_drv_path: Option, + already_scheduled: AtomicBool, + cancelled: AtomicBool, + pub runnable_since: jiff::Timestamp, + lowest_share_used: atomic_float::AtomicF64, +} + +impl StepInfo { + pub async fn new(store: &nix_utils::LocalStore, step: Arc) -> Self { + Self { + resolved_drv_path: store.try_resolve_drv(step.get_drv_path()).await, + already_scheduled: false.into(), + cancelled: false.into(), + runnable_since: step.get_runnable_since(), + lowest_share_used: step.get_lowest_share_used().into(), + step, + } + } + + pub fn update_internal_stats(&self) { + self.lowest_share_used + .store(self.step.get_lowest_share_used(), Ordering::Relaxed); + } + + pub fn get_lowest_share_used(&self) -> f64 { + self.lowest_share_used.load(Ordering::Relaxed) + } + + pub fn get_highest_global_priority(&self) -> i32 { + self.step + .atomic_state + .highest_global_priority + .load(Ordering::Relaxed) + } + + pub fn get_highest_local_priority(&self) -> i32 { + self.step + .atomic_state + .highest_local_priority + .load(Ordering::Relaxed) + } + + pub fn get_lowest_build_id(&self) -> BuildID { + self.step + .atomic_state + .lowest_build_id + .load(Ordering::Relaxed) + } + + pub fn get_already_scheduled(&self) -> bool { + self.already_scheduled.load(Ordering::SeqCst) + } + + pub fn set_already_scheduled(&self, v: bool) { + self.already_scheduled.store(v, Ordering::SeqCst); + } + + pub fn set_cancelled(&self, v: bool) { + self.cancelled.store(v, Ordering::SeqCst); + } + + pub fn get_cancelled(&self) -> bool { + self.cancelled.load(Ordering::SeqCst) + } + + pub(super) fn legacy_compare(&self, other: &Self) -> std::cmp::Ordering { + #[allow(irrefutable_let_patterns)] + (if let c1 = self + .get_highest_global_priority() + .cmp(&other.get_highest_global_priority()) + && c1 != std::cmp::Ordering::Equal + { + c1 + } else if let c2 = other + .get_lowest_share_used() + .total_cmp(&self.get_lowest_share_used()) + && c2 != std::cmp::Ordering::Equal + { + c2 + } else if let c3 = self + .get_highest_local_priority() + .cmp(&other.get_highest_local_priority()) + && c3 != std::cmp::Ordering::Equal + { + c3 + } else { + other.get_lowest_build_id().cmp(&self.get_lowest_build_id()) + }) + .reverse() + } + + pub(super) fn compare_with_rdeps(&self, other: &Self) -> std::cmp::Ordering { + #[allow(irrefutable_let_patterns)] + (if let c1 = self + .get_highest_global_priority() + .cmp(&other.get_highest_global_priority()) + && c1 != std::cmp::Ordering::Equal + { + c1 + } else if let c2 = other + .get_lowest_share_used() + .total_cmp(&self.get_lowest_share_used()) + && c2 != std::cmp::Ordering::Equal + { + c2 + } else if let c3 = self + .step + .atomic_state + .rdeps_len + .load(Ordering::Relaxed) + .cmp(&other.step.atomic_state.rdeps_len.load(Ordering::Relaxed)) + && c3 != std::cmp::Ordering::Equal + { + c3 + } else if let c4 = self + .get_highest_local_priority() + .cmp(&other.get_highest_local_priority()) + && c4 != std::cmp::Ordering::Equal + { + c4 + } else { + other.get_lowest_build_id().cmp(&self.get_lowest_build_id()) + }) + .reverse() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use db::models::BuildID; + + fn create_test_step( + highest_global_priority: i32, + highest_local_priority: i32, + lowest_build_id: BuildID, + lowest_share_used: f64, + rdeps_len: u64, + ) -> StepInfo { + let step = Step::new(nix_utils::StorePath::new( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-test.drv", + )); + + step.atomic_state.highest_global_priority.store( + highest_global_priority, + std::sync::atomic::Ordering::Relaxed, + ); + step.atomic_state + .highest_local_priority + .store(highest_local_priority, std::sync::atomic::Ordering::Relaxed); + step.atomic_state + .lowest_build_id + .store(lowest_build_id, std::sync::atomic::Ordering::Relaxed); + step.atomic_state + .rdeps_len + .store(rdeps_len, std::sync::atomic::Ordering::Relaxed); + + StepInfo { + step, + resolved_drv_path: None, + already_scheduled: false.into(), + cancelled: false.into(), + runnable_since: jiff::Timestamp::now(), + lowest_share_used: lowest_share_used.into(), + } + } + + #[test] + fn test_legacy_compare_global_priority() { + let step1 = create_test_step(10, 1, 1, 1.0, 0); + let step2 = create_test_step(5, 1, 2, 1.0, 0); + + assert_eq!(step1.legacy_compare(&step2), std::cmp::Ordering::Less); + assert_eq!(step2.legacy_compare(&step1), std::cmp::Ordering::Greater); + } + + #[test] + fn test_legacy_compare_share_used() { + let step1 = create_test_step(5, 1, 1, 0.5, 0); + let step2 = create_test_step(5, 1, 2, 1.0, 0); + + assert_eq!(step1.legacy_compare(&step2), std::cmp::Ordering::Less); + assert_eq!(step2.legacy_compare(&step1), std::cmp::Ordering::Greater); + } + + #[test] + fn test_legacy_compare_local_priority() { + let step1 = create_test_step(5, 10, 1, 1.0, 0); + let step2 = create_test_step(5, 5, 2, 1.0, 0); + + assert_eq!(step1.legacy_compare(&step2), std::cmp::Ordering::Less); + assert_eq!(step2.legacy_compare(&step1), std::cmp::Ordering::Greater); + } + + #[test] + fn test_legacy_compare_build_id() { + let step1 = create_test_step(5, 1, 1, 1.0, 0); + let step2 = create_test_step(5, 1, 2, 1.0, 0); + + assert_eq!(step1.legacy_compare(&step2), std::cmp::Ordering::Less); + assert_eq!(step2.legacy_compare(&step1), std::cmp::Ordering::Greater); + } + + #[test] + fn test_legacy_compare_equal() { + let step1 = create_test_step(5, 1, 1, 1.0, 0); + let step2 = create_test_step(5, 1, 1, 1.0, 0); + + assert_eq!(step1.legacy_compare(&step2), std::cmp::Ordering::Equal); + } + + #[test] + fn test_compare_with_rdeps_global_priority() { + let step1 = create_test_step(10, 1, 1, 1.0, 0); + let step2 = create_test_step(5, 1, 2, 1.0, 0); + + assert_eq!(step1.compare_with_rdeps(&step2), std::cmp::Ordering::Less); + assert_eq!( + step2.compare_with_rdeps(&step1), + std::cmp::Ordering::Greater + ); + } + + #[test] + fn test_compare_with_rdeps_share_used() { + let step1 = create_test_step(5, 1, 1, 0.5, 0); + let step2 = create_test_step(5, 1, 2, 1.0, 0); + + assert_eq!(step1.compare_with_rdeps(&step2), std::cmp::Ordering::Less); + assert_eq!( + step2.compare_with_rdeps(&step1), + std::cmp::Ordering::Greater + ); + } + + #[test] + fn test_compare_with_rdeps_rdeps_len() { + let step1 = create_test_step(5, 1, 1, 1.0, 10); + let step2 = create_test_step(5, 1, 2, 1.0, 5); + + assert_eq!(step1.compare_with_rdeps(&step2), std::cmp::Ordering::Less); + assert_eq!( + step2.compare_with_rdeps(&step1), + std::cmp::Ordering::Greater + ); + } + + #[test] + fn test_compare_with_rdeps_local_priority() { + let step1 = create_test_step(5, 10, 1, 1.0, 0); + let step2 = create_test_step(5, 5, 2, 1.0, 0); + + assert_eq!(step1.compare_with_rdeps(&step2), std::cmp::Ordering::Less); + assert_eq!( + step2.compare_with_rdeps(&step1), + std::cmp::Ordering::Greater + ); + } + + #[test] + fn test_compare_with_rdeps_build_id() { + let step1 = create_test_step(5, 1, 1, 1.0, 0); + let step2 = create_test_step(5, 1, 2, 1.0, 0); + + assert_eq!(step1.compare_with_rdeps(&step2), std::cmp::Ordering::Less); + assert_eq!( + step2.compare_with_rdeps(&step1), + std::cmp::Ordering::Greater + ); + } + + #[test] + fn test_compare_with_rdeps_equal() { + let step1 = create_test_step(5, 1, 1, 1.0, 0); + let step2 = create_test_step(5, 1, 1, 1.0, 0); + + assert_eq!(step1.compare_with_rdeps(&step2), std::cmp::Ordering::Equal); + } + + #[test] + fn test_difference_between_compare_functions() { + // Same global priority, share used, local priority, and build ID + // But different rdeps_len - this should affect compare_with_rdeps but not legacy_compare + let step1 = create_test_step(5, 1, 1, 1.0, 10); + let step2 = create_test_step(5, 1, 1, 1.0, 5); + + assert_eq!(step1.legacy_compare(&step2), std::cmp::Ordering::Equal); + + assert_eq!(step1.compare_with_rdeps(&step2), std::cmp::Ordering::Less); + assert_eq!( + step2.compare_with_rdeps(&step1), + std::cmp::Ordering::Greater + ); + } +} diff --git a/src/queue-runner/src/state/uploader.rs b/src/queue-runner/src/state/uploader.rs new file mode 100644 index 000000000..c888b5e2c --- /dev/null +++ b/src/queue-runner/src/state/uploader.rs @@ -0,0 +1,165 @@ +use backon::ExponentialBuilder; +use backon::Retryable; +use nix_utils::BaseStore as _; + +// TODO: scheduling is shit, because if we crash/restart we need to start again as the builds are +// already done in the db. +// So we need to make this persistent! + +#[derive(Debug)] +struct Message { + store_paths: Vec, + log_remote_path: String, + log_local_path: String, +} + +pub struct Uploader { + upload_queue_sender: tokio::sync::mpsc::UnboundedSender, + upload_queue_receiver: tokio::sync::Mutex>, +} + +impl Default for Uploader { + fn default() -> Self { + Self::new() + } +} + +impl Uploader { + pub fn new() -> Self { + let (upload_queue_tx, upload_queue_rx) = tokio::sync::mpsc::unbounded_channel::(); + Self { + upload_queue_sender: upload_queue_tx, + upload_queue_receiver: tokio::sync::Mutex::new(upload_queue_rx), + } + } + + #[tracing::instrument(skip(self), err)] + pub fn schedule_upload( + &self, + store_paths: Vec, + log_remote_path: String, + log_local_path: String, + ) -> anyhow::Result<()> { + tracing::info!("Scheduling new path upload: {:?}", store_paths); + self.upload_queue_sender.send(Message { + store_paths, + log_remote_path, + log_local_path, + })?; + Ok(()) + } + + #[tracing::instrument(skip(self, local_store, remote_stores))] + async fn upload_msg( + &self, + local_store: nix_utils::LocalStore, + remote_stores: Vec, + msg: Message, + ) { + tracing::info!("Uploading {} paths", msg.store_paths.len()); + + let paths_to_copy = match local_store + .query_requisites(&msg.store_paths.iter().collect::>(), false) + .await + { + Ok(paths) => paths, + Err(e) => { + tracing::error!("Failed to query requisites: {e}"); + return; + } + }; + + for remote_store in remote_stores { + let bucket = &remote_store.cfg.client_config.bucket; + + // Upload log file with backon retry + let log_upload_result = (|| async { + let file = fs_err::tokio::File::open(&msg.log_local_path).await?; + let reader = Box::new(tokio::io::BufReader::new(file)); + + remote_store + .upsert_file_stream(&msg.log_remote_path, reader, "text/plain; charset=utf-8") + .await?; + + Ok::<(), anyhow::Error>(()) + }) + .retry( + ExponentialBuilder::default() + .with_max_delay(std::time::Duration::from_secs(30)) + .with_max_times(3), + ) + .await; + + if let Err(e) = log_upload_result { + tracing::error!("Failed to upload log file after retries: {e}"); + } + if msg.store_paths.is_empty() { + tracing::debug!("No NAR files to upload (presigned uploads enabled)"); + } else { + let paths_to_copy = remote_store + .query_missing_paths(paths_to_copy.clone()) + .await; + + let copy_result = (|| async { + remote_store + .copy_paths(&local_store, paths_to_copy.clone(), false) + .await?; + + Ok::<(), anyhow::Error>(()) + }) + .retry( + ExponentialBuilder::default() + .with_max_delay(std::time::Duration::from_secs(60)) + .with_max_times(5), + ) + .await; + + if let Err(e) = copy_result { + tracing::error!("Failed to copy paths after retries: {e}"); + } else { + tracing::debug!( + "Successfully uploaded {} paths to bucket {bucket}", + msg.store_paths.len() + ); + } + } + } + + tracing::info!("Finished uploading {} paths", msg.store_paths.len()); + } + + pub async fn upload_once( + &self, + local_store: nix_utils::LocalStore, + remote_stores: Vec, + ) { + let Some(msg) = ({ + let mut rx = self.upload_queue_receiver.lock().await; + rx.recv().await + }) else { + return; + }; + + self.upload_msg(local_store, remote_stores, msg).await; + } + + pub async fn upload_many( + &self, + local_store: nix_utils::LocalStore, + remote_stores: Vec, + limit: usize, + ) { + let mut messages: Vec = Vec::with_capacity(limit); + self.upload_queue_receiver + .lock() + .await + .recv_many(&mut messages, limit) + .await; + + let mut jobs = vec![]; + for msg in messages { + jobs.push(self.upload_msg(local_store.clone(), remote_stores.clone(), msg)); + } + futures::future::join_all(jobs).await; + } +} diff --git a/src/queue-runner/src/utils.rs b/src/queue-runner/src/utils.rs new file mode 100644 index 000000000..4ff82197f --- /dev/null +++ b/src/queue-runner/src/utils.rs @@ -0,0 +1,118 @@ +use db::models::BuildID; +use nix_utils::BaseStore as _; + +use crate::state::RemoteBuild; + +#[tracing::instrument(skip(db, store, res), err)] +pub async fn finish_build_step( + db: &db::Database, + store: &nix_utils::LocalStore, + build_id: BuildID, + step_nr: i32, + res: &RemoteBuild, + machine: Option, +) -> anyhow::Result<()> { + let mut conn = db.get().await?; + let mut tx = conn.begin_transaction().await?; + + debug_assert!(res.has_start_time()); + debug_assert!(res.has_stop_time()); + tracing::info!( + "Writing buildstep result in db. step_status={:?} start_time={:?} stop_time={:?}", + res.step_status, + res.get_start_time_as_i32(), + res.get_stop_time_as_i32(), + ); + tx.update_build_step_in_finish(db::models::UpdateBuildStepInFinish { + build_id, + step_nr, + status: res.step_status, + error_msg: res.error_msg.as_deref(), + start_time: res.get_start_time_as_i32()?, + stop_time: res.get_stop_time_as_i32()?, + machine: machine.as_deref(), + overhead: res.get_overhead(), + times_built: res.get_times_built(), + is_non_deterministic: res.get_is_non_deterministic(), + }) + .await?; + debug_assert!(!res.log_file.is_empty()); + debug_assert!(!res.log_file.contains('\t')); + + tx.notify_step_finished(build_id, step_nr, &res.log_file) + .await?; + + if res.step_status == db::models::BuildStatus::Success { + // Update the corresponding `BuildStepOutputs` row to add the output path + let drv_path = tx.get_drv_path_from_build_step(build_id, step_nr).await?; + if let Some(drv_path) = drv_path { + // If we've finished building, all the paths should be known + if let Some(drv) = + nix_utils::query_drv(store, &nix_utils::StorePath::new(&drv_path)).await? + { + for o in drv.outputs { + if let Some(path) = o.path { + tx.update_build_step_output( + build_id, + step_nr, + &o.name, + &store.print_store_path(&path), + ) + .await?; + } + } + } + } + } + + tx.commit().await?; + Ok(()) +} + +#[tracing::instrument(skip(db, store, o, remote_store), fields(%drv_path), err(level=tracing::Level::WARN))] +pub async fn substitute_output( + db: db::Database, + store: nix_utils::LocalStore, + o: nix_utils::DerivationOutput, + build_id: BuildID, + drv_path: &nix_utils::StorePath, + remote_store: Option<&binary_cache::S3BinaryCacheClient>, +) -> anyhow::Result { + let Some(path) = &o.path else { + return Ok(false); + }; + + let starttime = i32::try_from(jiff::Timestamp::now().as_second())?; // TODO + if let Err(e) = store.ensure_path(path).await { + tracing::debug!("Path not found, can't import={e}"); + return Ok(false); + } + if let Some(remote_store) = remote_store { + let paths_to_copy = store + .query_requisites(&[path], false) + .await + .unwrap_or_default(); + let paths_to_copy = remote_store.query_missing_paths(paths_to_copy).await; + if let Err(e) = remote_store.copy_paths(&store, paths_to_copy, false).await { + tracing::error!( + "Failed to copy paths to remote store({}): {e}", + remote_store.cfg.client_config.bucket + ); + } + } + let stoptime = i32::try_from(jiff::Timestamp::now().as_second())?; // TODO + + let mut db = db.get().await?; + let mut tx = db.begin_transaction().await?; + tx.create_substitution_step( + starttime, + stoptime, + build_id, + &store.print_store_path(drv_path), + (o.name.clone(), o.path.map(|p| store.print_store_path(&p))), + ) + .await?; + tx.commit().await?; + + Ok(true) +} diff --git a/t/meson.build b/t/meson.build index dd930dbd4..0f55c70b7 100644 --- a/t/meson.build +++ b/t/meson.build @@ -29,7 +29,8 @@ testenv.prepend('PERL5LIB', testenv.prepend('PATH', fs.parent(find_program('nix').full_path()), fs.parent(hydra_evaluator.full_path()), - fs.parent(hydra_queue_runner.full_path()), + fs.parent(rust_outputs[0].full_path()), + fs.parent(rust_outputs[1].full_path()), meson.project_source_root() / 'src/script', separator: ':' ) diff --git a/t/perlcritic.pl b/t/perlcritic.pl index 09172ede5..537cb4a3c 100755 --- a/t/perlcritic.pl +++ b/t/perlcritic.pl @@ -13,4 +13,4 @@ # Add src/lib to PERL5LIB so perlcritic can find our custom policies $ENV{PERL5LIB} = "src/lib" . ($ENV{PERL5LIB} ? ":$ENV{PERL5LIB}" : ""); -exec("perlcritic", "--quiet", ".") or die "Failed to execute perlcritic."; +exec("perlcritic", "--quiet", "src/", "t/") or die "Failed to execute perlcritic.";