From a0c2f73826921f8593648235663067b9e3b736e7 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 13 Mar 2024 12:29:41 +0100 Subject: [PATCH] start adding build to database before building, surface all build errors to the user --- ...8e1276f7f814f22b830c46d6b77edb209a033.json | 4 +- ...de41c195510e87fa643a4cae57864552962c.json} | 12 +- ...3ba324fde11d9f5bd07ae264761dc38e1f2a.json} | 18 +- ...ff9d5169f8bf7c50315ef96d838b37c18cf1.json} | 6 +- ...b6b6222f7d0d45aa938a1d52791c11629e070.json | 35 +++ ...4910dde51096de4ee2ad868cc5025161ab466.json | 2 +- ...75bb47f6ebc389b0aa61f7732d1f354dc5259.json | 16 +- ...500c0aa7bd9c4cd87544efc4307381e937d0.json} | 6 +- ...2ad28a198463534efbefcf06f45c0f49872af.json | 38 ++++ ...2eafefd3b80b3322116448901aa55f2d89e7.json} | 12 +- ...4160056d6cb620c55be459ead0f95c3523035.json | 2 +- ...265f616eef21a09ffaaa243fc26f107ad6c87.json | 12 +- ...0bd7ce1a9eeae15b48c36a8a5abeaaa4314c.json} | 16 +- ...ff8b2ec47de33f2d74e0f1475fdf165e1ac2.json} | 4 +- ...c8689ac75e55acad3fc85c4cf998259abfc6f.json | 22 ++ ...7068198f91d2bbd7f176054ac58bcda3fcac6.json | 23 ++ ...6c234eb0206d84805cbc8a38cddb0cd220d92.json | 23 ++ Cargo.lock | 1 + Cargo.toml | 1 + docker-compose.yml | 3 + ...3708_make_release_fields_optional.down.sql | 30 +++ ...103708_make_release_fields_optional.up.sql | 17 ++ ...182623_make_build_fields_optional.down.sql | 15 ++ ...13182623_make_build_fields_optional.up.sql | 7 + .../20240313184911_build_errors.down.sql | 1 + migrations/20240313184911_build_errors.up.sql | 1 + src/db/add_package.rs | 186 ++++++++++++++-- src/db/mod.rs | 3 +- src/docbuilder/rustwide_builder.rs | 41 +++- src/test/fakes.rs | 6 +- src/web/build_details.rs | 55 +++-- src/web/builds.rs | 45 ++-- src/web/crate_details.rs | 199 +++++++++++------- src/web/mod.rs | 52 +++-- src/web/releases.rs | 30 ++- src/web/rustdoc.rs | 77 ++++--- src/web/sitemap.rs | 4 +- src/web/source.rs | 18 +- templates/crate/build_details.html | 49 +++-- templates/crate/builds.html | 49 +++-- templates/crate/details.html | 52 +++-- templates/crate/features.html | 16 +- templates/crate/source.html | 82 ++++---- templates/macros.html | 11 +- 44 files changed, 947 insertions(+), 355 deletions(-) rename .sqlx/{query-162c05df1f44bb48d087b6e6e4b3a8ab868b6d0cc20143b176522c0791a7023c.json => query-1dae7bb925d677bb997ec28130f7de41c195510e87fa643a4cae57864552962c.json} (88%) rename .sqlx/{query-b29681d013e952b90a2fe01db50da562d6ef7ab5ab19c6c992235db5bd931e8c.json => query-224f12feb10c65e8345eb4346c123ba324fde11d9f5bd07ae264761dc38e1f2a.json} (69%) rename .sqlx/{query-d9f0166f4e46d6a601f0668606e1487eab21d8900c096f7e7e4629c210b3a371.json => query-5a92975ad3d17197b01235b08ea9ff9d5169f8bf7c50315ef96d838b37c18cf1.json} (66%) create mode 100644 .sqlx/query-5ffff1acc8acd27b2f6b9ed3f5fb6b6222f7d0d45aa938a1d52791c11629e070.json rename .sqlx/{query-9a3bcf29ba086cd26102227e024218a0fd4b7bbe2b229350523a1aa69a4d731f.json => query-99b0f4e27eea259dd4ff504ce935500c0aa7bd9c4cd87544efc4307381e937d0.json} (56%) create mode 100644 .sqlx/query-b2fa2e823f0f8e8fbd288cd0c102ad28a198463534efbefcf06f45c0f49872af.json rename .sqlx/{query-7e833553cf419d8fbc3a5a0c504f0af2fa3d4aaca158c3b616f2e683a943bccb.json => query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json} (52%) rename .sqlx/{query-913903768e8fd453e3246370a8147fb9828c88b1e5b8195bc0a0bcc037219923.json => query-df2a6468276a174f7993ba7cb5740bd7ce1a9eeae15b48c36a8a5abeaaa4314c.json} (69%) rename .sqlx/{query-5130f2d40dfc60c64e830b9567077f41b7472d1bb9c8a8943a74bf7df4afe8eb.json => query-ed7323a8d06dbb475ec470892c1fff8b2ec47de33f2d74e0f1475fdf165e1ac2.json} (53%) create mode 100644 .sqlx/query-f42825d44f0c14d5ad0bcf76150c8689ac75e55acad3fc85c4cf998259abfc6f.json create mode 100644 .sqlx/query-f586539bcbb4956768c424065ae7068198f91d2bbd7f176054ac58bcda3fcac6.json create mode 100644 .sqlx/query-f76e52369a85097581f2b2d74506c234eb0206d84805cbc8a38cddb0cd220d92.json create mode 100644 migrations/20240313103708_make_release_fields_optional.down.sql create mode 100644 migrations/20240313103708_make_release_fields_optional.up.sql create mode 100644 migrations/20240313182623_make_build_fields_optional.down.sql create mode 100644 migrations/20240313182623_make_build_fields_optional.up.sql create mode 100644 migrations/20240313184911_build_errors.down.sql create mode 100644 migrations/20240313184911_build_errors.up.sql diff --git a/.sqlx/query-0011936b31678ee644dff3f5f8d8e1276f7f814f22b830c46d6b77edb209a033.json b/.sqlx/query-0011936b31678ee644dff3f5f8d8e1276f7f814f22b830c46d6b77edb209a033.json index a578ca3f8..cad5698c0 100644 --- a/.sqlx/query-0011936b31678ee644dff3f5f8d8e1276f7f814f22b830c46d6b77edb209a033.json +++ b/.sqlx/query-0011936b31678ee644dff3f5f8d8e1276f7f814f22b830c46d6b77edb209a033.json @@ -20,8 +20,8 @@ ] }, "nullable": [ - false, - false + true, + true ] }, "hash": "0011936b31678ee644dff3f5f8d8e1276f7f814f22b830c46d6b77edb209a033" diff --git a/.sqlx/query-162c05df1f44bb48d087b6e6e4b3a8ab868b6d0cc20143b176522c0791a7023c.json b/.sqlx/query-1dae7bb925d677bb997ec28130f7de41c195510e87fa643a4cae57864552962c.json similarity index 88% rename from .sqlx/query-162c05df1f44bb48d087b6e6e4b3a8ab868b6d0cc20143b176522c0791a7023c.json rename to .sqlx/query-1dae7bb925d677bb997ec28130f7de41c195510e87fa643a4cae57864552962c.json index 3edf2a4b9..c0316f0f9 100644 --- a/.sqlx/query-162c05df1f44bb48d087b6e6e4b3a8ab868b6d0cc20143b176522c0791a7023c.json +++ b/.sqlx/query-1dae7bb925d677bb997ec28130f7de41c195510e87fa643a4cae57864552962c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n releases.id,\n releases.version,\n release_build_status.build_status as \"build_status!: BuildStatus\",\n releases.yanked,\n releases.is_library,\n releases.rustdoc_status,\n releases.target_name\n FROM releases\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n WHERE\n releases.crate_id = $1", + "query": "SELECT\n releases.id,\n releases.version,\n release_build_status.build_status as \"build_status!: BuildStatus\",\n releases.yanked,\n releases.is_library,\n releases.rustdoc_status,\n releases.target_name\n FROM releases\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n WHERE\n releases.crate_id = $1 AND\n release_build_status.build_status != 'in_progress'", "describe": { "columns": [ { @@ -59,11 +59,11 @@ false, false, false, - false, - false, - false, - false + true, + true, + true, + true ] }, - "hash": "162c05df1f44bb48d087b6e6e4b3a8ab868b6d0cc20143b176522c0791a7023c" + "hash": "1dae7bb925d677bb997ec28130f7de41c195510e87fa643a4cae57864552962c" } diff --git a/.sqlx/query-b29681d013e952b90a2fe01db50da562d6ef7ab5ab19c6c992235db5bd931e8c.json b/.sqlx/query-224f12feb10c65e8345eb4346c123ba324fde11d9f5bd07ae264761dc38e1f2a.json similarity index 69% rename from .sqlx/query-b29681d013e952b90a2fe01db50da562d6ef7ab5ab19c6c992235db5bd931e8c.json rename to .sqlx/query-224f12feb10c65e8345eb4346c123ba324fde11d9f5bd07ae264761dc38e1f2a.json index 6d50f853f..aebb0a875 100644 --- a/.sqlx/query-b29681d013e952b90a2fe01db50da562d6ef7ab5ab19c6c992235db5bd931e8c.json +++ b/.sqlx/query-224f12feb10c65e8345eb4346c123ba324fde11d9f5bd07ae264761dc38e1f2a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n builds.rustc_version,\n builds.docsrs_version,\n builds.build_status as \"build_status: BuildStatus\",\n builds.build_time,\n builds.output,\n releases.default_target\n FROM builds\n INNER JOIN releases ON releases.id = builds.rid\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE builds.id = $1 AND crates.name = $2 AND releases.version = $3", + "query": "SELECT\n builds.rustc_version,\n builds.docsrs_version,\n builds.build_status as \"build_status: BuildStatus\",\n builds.build_time,\n builds.output,\n builds.errors,\n releases.default_target\n FROM builds\n INNER JOIN releases ON releases.id = builds.rid\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE builds.id = $1 AND crates.name = $2 AND releases.version = $3", "describe": { "columns": [ { @@ -41,6 +41,11 @@ }, { "ordinal": 5, + "name": "errors", + "type_info": "Text" + }, + { + "ordinal": 6, "name": "default_target", "type_info": "Varchar" } @@ -53,13 +58,14 @@ ] }, "nullable": [ + true, + true, false, - false, - false, - false, true, - false + true, + true, + true ] }, - "hash": "b29681d013e952b90a2fe01db50da562d6ef7ab5ab19c6c992235db5bd931e8c" + "hash": "224f12feb10c65e8345eb4346c123ba324fde11d9f5bd07ae264761dc38e1f2a" } diff --git a/.sqlx/query-d9f0166f4e46d6a601f0668606e1487eab21d8900c096f7e7e4629c210b3a371.json b/.sqlx/query-5a92975ad3d17197b01235b08ea9ff9d5169f8bf7c50315ef96d838b37c18cf1.json similarity index 66% rename from .sqlx/query-d9f0166f4e46d6a601f0668606e1487eab21d8900c096f7e7e4629c210b3a371.json rename to .sqlx/query-5a92975ad3d17197b01235b08ea9ff9d5169f8bf7c50315ef96d838b37c18cf1.json index 624b78abb..a7c43486e 100644 --- a/.sqlx/query-d9f0166f4e46d6a601f0668606e1487eab21d8900c096f7e7e4629c210b3a371.json +++ b/.sqlx/query-5a92975ad3d17197b01235b08ea9ff9d5169f8bf7c50315ef96d838b37c18cf1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO builds (rid, rustc_version, docsrs_version, build_status, build_server)\n VALUES ($1, $2, $3, $4, $5)\n RETURNING id", + "query": "INSERT INTO builds(rid, build_status, build_server)\n VALUES ($1, $2, $3)\n RETURNING id", "describe": { "columns": [ { @@ -12,8 +12,6 @@ "parameters": { "Left": [ "Int4", - "Varchar", - "Varchar", { "Custom": { "name": "build_status", @@ -33,5 +31,5 @@ false ] }, - "hash": "d9f0166f4e46d6a601f0668606e1487eab21d8900c096f7e7e4629c210b3a371" + "hash": "5a92975ad3d17197b01235b08ea9ff9d5169f8bf7c50315ef96d838b37c18cf1" } diff --git a/.sqlx/query-5ffff1acc8acd27b2f6b9ed3f5fb6b6222f7d0d45aa938a1d52791c11629e070.json b/.sqlx/query-5ffff1acc8acd27b2f6b9ed3f5fb6b6222f7d0d45aa938a1d52791c11629e070.json new file mode 100644 index 000000000..19ceb7546 --- /dev/null +++ b/.sqlx/query-5ffff1acc8acd27b2f6b9ed3f5fb6b6222f7d0d45aa938a1d52791c11629e070.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE builds\n SET\n build_status = $1,\n errors = $2\n WHERE id = $3\n RETURNING rid", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "rid", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + { + "Custom": { + "name": "build_status", + "kind": { + "Enum": [ + "in_progress", + "success", + "failure" + ] + } + } + }, + "Text", + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5ffff1acc8acd27b2f6b9ed3f5fb6b6222f7d0d45aa938a1d52791c11629e070" +} diff --git a/.sqlx/query-65b0ead56880b369931c3a5ec324910dde51096de4ee2ad868cc5025161ab466.json b/.sqlx/query-65b0ead56880b369931c3a5ec324910dde51096de4ee2ad868cc5025161ab466.json index 466e6f723..5205f1517 100644 --- a/.sqlx/query-65b0ead56880b369931c3a5ec324910dde51096de4ee2ad868cc5025161ab466.json +++ b/.sqlx/query-65b0ead56880b369931c3a5ec324910dde51096de4ee2ad868cc5025161ab466.json @@ -26,7 +26,7 @@ }, "nullable": [ false, - false, + true, null ] }, diff --git a/.sqlx/query-65f40f6603b1f3e736566c3d4e275bb47f6ebc389b0aa61f7732d1f354dc5259.json b/.sqlx/query-65f40f6603b1f3e736566c3d4e275bb47f6ebc389b0aa61f7732d1f354dc5259.json index ae07a11c7..088212a18 100644 --- a/.sqlx/query-65f40f6603b1f3e736566c3d4e275bb47f6ebc389b0aa61f7732d1f354dc5259.json +++ b/.sqlx/query-65f40f6603b1f3e736566c3d4e275bb47f6ebc389b0aa61f7732d1f354dc5259.json @@ -200,19 +200,16 @@ true, true, true, - false, + true, false, null, - false, + true, false, true, true, true, - false, - false, - false, - false, - false, + true, + true, false, false, false, @@ -220,7 +217,10 @@ false, true, true, - false, + true, + true, + true, + true, null, true, true, diff --git a/.sqlx/query-9a3bcf29ba086cd26102227e024218a0fd4b7bbe2b229350523a1aa69a4d731f.json b/.sqlx/query-99b0f4e27eea259dd4ff504ce935500c0aa7bd9c4cd87544efc4307381e937d0.json similarity index 56% rename from .sqlx/query-9a3bcf29ba086cd26102227e024218a0fd4b7bbe2b229350523a1aa69a4d731f.json rename to .sqlx/query-99b0f4e27eea259dd4ff504ce935500c0aa7bd9c4cd87544efc4307381e937d0.json index 584d4ce86..0b4d342a2 100644 --- a/.sqlx/query-9a3bcf29ba086cd26102227e024218a0fd4b7bbe2b229350523a1aa69a4d731f.json +++ b/.sqlx/query-99b0f4e27eea259dd4ff504ce935500c0aa7bd9c4cd87544efc4307381e937d0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n releases.doc_targets\n FROM releases\n WHERE releases.id = $1;", + "query": "SELECT\n releases.doc_targets\n FROM releases\n WHERE releases.id = $1;", "describe": { "columns": [ { @@ -15,8 +15,8 @@ ] }, "nullable": [ - false + true ] }, - "hash": "9a3bcf29ba086cd26102227e024218a0fd4b7bbe2b229350523a1aa69a4d731f" + "hash": "99b0f4e27eea259dd4ff504ce935500c0aa7bd9c4cd87544efc4307381e937d0" } diff --git a/.sqlx/query-b2fa2e823f0f8e8fbd288cd0c102ad28a198463534efbefcf06f45c0f49872af.json b/.sqlx/query-b2fa2e823f0f8e8fbd288cd0c102ad28a198463534efbefcf06f45c0f49872af.json new file mode 100644 index 000000000..ade9b4eac --- /dev/null +++ b/.sqlx/query-b2fa2e823f0f8e8fbd288cd0c102ad28a198463534efbefcf06f45c0f49872af.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE builds\n SET\n rustc_version = $1,\n docsrs_version = $2,\n build_status = $3,\n build_server = $4,\n errors = $5,\n build_time = NOW()\n WHERE\n id = $6\n RETURNING rid", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "rid", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + { + "Custom": { + "name": "build_status", + "kind": { + "Enum": [ + "in_progress", + "success", + "failure" + ] + } + } + }, + "Text", + "Text", + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "b2fa2e823f0f8e8fbd288cd0c102ad28a198463534efbefcf06f45c0f49872af" +} diff --git a/.sqlx/query-7e833553cf419d8fbc3a5a0c504f0af2fa3d4aaca158c3b616f2e683a943bccb.json b/.sqlx/query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json similarity index 52% rename from .sqlx/query-7e833553cf419d8fbc3a5a0c504f0af2fa3d4aaca158c3b616f2e683a943bccb.json rename to .sqlx/query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json index 26913656c..bf07bcf56 100644 --- a/.sqlx/query-7e833553cf419d8fbc3a5a0c504f0af2fa3d4aaca158c3b616f2e683a943bccb.json +++ b/.sqlx/query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n crates.name,\n releases.version,\n releases.description,\n builds.build_time,\n releases.target_name,\n releases.rustdoc_status,\n repositories.stars as \"stars?\",\n EXISTS (\n SELECT 1\n FROM releases AS all_releases\n WHERE\n all_releases.crate_id = crates.id AND\n all_releases.yanked = false\n ) AS has_unyanked_releases\n\n FROM crates\n INNER JOIN releases ON crates.latest_version_id = releases.id\n INNER JOIN builds ON releases.id = builds.rid\n LEFT JOIN repositories ON releases.repository_id = repositories.id\n\n WHERE crates.name = ANY($1)", + "query": "SELECT\n crates.name,\n releases.version,\n releases.description,\n release_build_status.last_build_time,\n releases.target_name,\n releases.rustdoc_status,\n repositories.stars as \"stars?\",\n EXISTS (\n SELECT 1\n FROM releases AS all_releases\n WHERE\n all_releases.crate_id = crates.id AND\n all_releases.yanked = false\n ) AS has_unyanked_releases\n\n FROM crates\n INNER JOIN releases ON crates.latest_version_id = releases.id\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n LEFT JOIN repositories ON releases.repository_id = repositories.id\n\n WHERE\n crates.name = ANY($1) AND\n release_build_status.build_status <> 'in_progress'", "describe": { "columns": [ { @@ -20,7 +20,7 @@ }, { "ordinal": 3, - "name": "build_time", + "name": "last_build_time", "type_info": "Timestamptz" }, { @@ -53,12 +53,12 @@ false, false, true, - false, - false, - false, + true, + true, + true, false, null ] }, - "hash": "7e833553cf419d8fbc3a5a0c504f0af2fa3d4aaca158c3b616f2e683a943bccb" + "hash": "bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7" } diff --git a/.sqlx/query-bdddad099e891bb45ba3703d7144160056d6cb620c55be459ead0f95c3523035.json b/.sqlx/query-bdddad099e891bb45ba3703d7144160056d6cb620c55be459ead0f95c3523035.json index a597f557a..2ccec63d1 100644 --- a/.sqlx/query-bdddad099e891bb45ba3703d7144160056d6cb620c55be459ead0f95c3523035.json +++ b/.sqlx/query-bdddad099e891bb45ba3703d7144160056d6cb620c55be459ead0f95c3523035.json @@ -27,7 +27,7 @@ "nullable": [ false, false, - false + true ] }, "hash": "bdddad099e891bb45ba3703d7144160056d6cb620c55be459ead0f95c3523035" diff --git a/.sqlx/query-d471c22a41a040f720ac678b085265f616eef21a09ffaaa243fc26f107ad6c87.json b/.sqlx/query-d471c22a41a040f720ac678b085265f616eef21a09ffaaa243fc26f107ad6c87.json index f67066cdb..e1b04471a 100644 --- a/.sqlx/query-d471c22a41a040f720ac678b085265f616eef21a09ffaaa243fc26f107ad6c87.json +++ b/.sqlx/query-d471c22a41a040f720ac678b085265f616eef21a09ffaaa243fc26f107ad6c87.json @@ -59,12 +59,12 @@ false, false, true, - false, - false, - false, - false, - false, - false + true, + true, + true, + true, + true, + true ] }, "hash": "d471c22a41a040f720ac678b085265f616eef21a09ffaaa243fc26f107ad6c87" diff --git a/.sqlx/query-913903768e8fd453e3246370a8147fb9828c88b1e5b8195bc0a0bcc037219923.json b/.sqlx/query-df2a6468276a174f7993ba7cb5740bd7ce1a9eeae15b48c36a8a5abeaaa4314c.json similarity index 69% rename from .sqlx/query-913903768e8fd453e3246370a8147fb9828c88b1e5b8195bc0a0bcc037219923.json rename to .sqlx/query-df2a6468276a174f7993ba7cb5740bd7ce1a9eeae15b48c36a8a5abeaaa4314c.json index 3911e806e..de1ffe1d0 100644 --- a/.sqlx/query-913903768e8fd453e3246370a8147fb9828c88b1e5b8195bc0a0bcc037219923.json +++ b/.sqlx/query-df2a6468276a174f7993ba7cb5740bd7ce1a9eeae15b48c36a8a5abeaaa4314c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n builds.id,\n builds.rustc_version,\n builds.docsrs_version,\n builds.build_status as \"build_status: BuildStatus\",\n builds.build_time\n FROM builds\n INNER JOIN releases ON releases.id = builds.rid\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE crates.name = $1 AND releases.version = $2\n ORDER BY id DESC", + "query": "SELECT\n builds.id,\n builds.rustc_version,\n builds.docsrs_version,\n builds.build_status as \"build_status: BuildStatus\",\n builds.build_time,\n builds.errors\n FROM builds\n INNER JOIN releases ON releases.id = builds.rid\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE crates.name = $1 AND releases.version = $2\n ORDER BY id DESC", "describe": { "columns": [ { @@ -38,6 +38,11 @@ "ordinal": 4, "name": "build_time", "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "errors", + "type_info": "Text" } ], "parameters": { @@ -48,11 +53,12 @@ }, "nullable": [ false, + true, + true, false, - false, - false, - false + true, + true ] }, - "hash": "913903768e8fd453e3246370a8147fb9828c88b1e5b8195bc0a0bcc037219923" + "hash": "df2a6468276a174f7993ba7cb5740bd7ce1a9eeae15b48c36a8a5abeaaa4314c" } diff --git a/.sqlx/query-5130f2d40dfc60c64e830b9567077f41b7472d1bb9c8a8943a74bf7df4afe8eb.json b/.sqlx/query-ed7323a8d06dbb475ec470892c1fff8b2ec47de33f2d74e0f1475fdf165e1ac2.json similarity index 53% rename from .sqlx/query-5130f2d40dfc60c64e830b9567077f41b7472d1bb9c8a8943a74bf7df4afe8eb.json rename to .sqlx/query-ed7323a8d06dbb475ec470892c1fff8b2ec47de33f2d74e0f1475fdf165e1ac2.json index 032600f9a..251a03189 100644 --- a/.sqlx/query-5130f2d40dfc60c64e830b9567077f41b7472d1bb9c8a8943a74bf7df4afe8eb.json +++ b/.sqlx/query-ed7323a8d06dbb475ec470892c1fff8b2ec47de33f2d74e0f1475fdf165e1ac2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n releases.archive_storage,\n (\n SELECT id\n FROM builds\n WHERE builds.rid = releases.id\n ORDER BY build_time DESC\n LIMIT 1\n ) AS latest_build_id\n FROM releases\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE\n name = $1 AND\n version = $2", + "query": "SELECT\n releases.archive_storage,\n (\n SELECT id\n FROM builds\n WHERE\n builds.rid = releases.id AND\n builds.build_status = 'success'\n ORDER BY build_time DESC\n LIMIT 1\n ) AS latest_build_id\n FROM releases\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE\n name = $1 AND\n version = $2", "describe": { "columns": [ { @@ -25,5 +25,5 @@ null ] }, - "hash": "5130f2d40dfc60c64e830b9567077f41b7472d1bb9c8a8943a74bf7df4afe8eb" + "hash": "ed7323a8d06dbb475ec470892c1fff8b2ec47de33f2d74e0f1475fdf165e1ac2" } diff --git a/.sqlx/query-f42825d44f0c14d5ad0bcf76150c8689ac75e55acad3fc85c4cf998259abfc6f.json b/.sqlx/query-f42825d44f0c14d5ad0bcf76150c8689ac75e55acad3fc85c4cf998259abfc6f.json new file mode 100644 index 000000000..845d73dae --- /dev/null +++ b/.sqlx/query-f42825d44f0c14d5ad0bcf76150c8689ac75e55acad3fc85c4cf998259abfc6f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM builds WHERE rid = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f42825d44f0c14d5ad0bcf76150c8689ac75e55acad3fc85c4cf998259abfc6f" +} diff --git a/.sqlx/query-f586539bcbb4956768c424065ae7068198f91d2bbd7f176054ac58bcda3fcac6.json b/.sqlx/query-f586539bcbb4956768c424065ae7068198f91d2bbd7f176054ac58bcda3fcac6.json new file mode 100644 index 000000000..b25293ad7 --- /dev/null +++ b/.sqlx/query-f586539bcbb4956768c424065ae7068198f91d2bbd7f176054ac58bcda3fcac6.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM releases WHERE crate_id = $1 and version = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f586539bcbb4956768c424065ae7068198f91d2bbd7f176054ac58bcda3fcac6" +} diff --git a/.sqlx/query-f76e52369a85097581f2b2d74506c234eb0206d84805cbc8a38cddb0cd220d92.json b/.sqlx/query-f76e52369a85097581f2b2d74506c234eb0206d84805cbc8a38cddb0cd220d92.json new file mode 100644 index 000000000..54af1e0b8 --- /dev/null +++ b/.sqlx/query-f76e52369a85097581f2b2d74506c234eb0206d84805cbc8a38cddb0cd220d92.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO releases (crate_id, version, archive_storage)\n VALUES ($1, $2, TRUE)\n ON CONFLICT (crate_id, version) DO UPDATE\n SET -- this `SET` is needed so the id is always returned.\n version = EXCLUDED.version\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f76e52369a85097581f2b2d74506c234eb0206d84805cbc8a38cddb0cd220d92" +} diff --git a/Cargo.lock b/Cargo.lock index 55dc53d45..6831a838f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1618,6 +1618,7 @@ dependencies = [ "percent-encoding", "postgres", "postgres-types", + "pretty_assertions", "procfs", "prometheus", "r2d2", diff --git a/Cargo.toml b/Cargo.toml index 11e3ccc79..09d09ad8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,6 +131,7 @@ aws-smithy-types = "1.0.1" aws-smithy-runtime = {version = "1.0.1", features = ["client", "test-util"]} aws-smithy-http = "0.60.0" indoc = "2.0.0" +pretty_assertions = "1.4.0" [profile.dev.package."*"] opt-level = 2 diff --git a/docker-compose.yml b/docker-compose.yml index 6bad85bf3..aabf44cc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,9 @@ services: S3_ENDPOINT: http://s3:9000 AWS_ACCESS_KEY_ID: cratesfyi AWS_SECRET_ACCESS_KEY: secret_key + DOCSRS_MAX_LEGACY_POOL_SIZE: 10 + DOCSRS_MAX_POOL_SIZE: 10 + DOCSRS_MIN_POOL_IDLE: 1 env_file: - .env healthcheck: diff --git a/migrations/20240313103708_make_release_fields_optional.down.sql b/migrations/20240313103708_make_release_fields_optional.down.sql new file mode 100644 index 000000000..90116a626 --- /dev/null +++ b/migrations/20240313103708_make_release_fields_optional.down.sql @@ -0,0 +1,30 @@ +DELETE FROM releases WHERE ( + release_time IS NULL OR + target_name IS NULL OR + yanked IS NULL OR + is_library IS NULL OR + rustdoc_status IS NULL OR + have_examples IS NULL OR + downloads IS NULL OR + doc_targets IS NULL OR + default_target IS NULL OR + archive_storage IS NULL +); + +ALTER TABLE releases + ALTER COLUMN release_time SET NOT NULL, + ALTER COLUMN target_name SET NOT NULL, + ALTER COLUMN yanked SET NOT NULL, + ALTER COLUMN yanked SET DEFAULT false, + ALTER COLUMN is_library SET NOT NULL, + ALTER COLUMN is_library SET DEFAULT true, + ALTER COLUMN rustdoc_status SET NOT NULL, + ALTER COLUMN rustdoc_status SET DEFAULT false, + ALTER COLUMN test_status SET DEFAULT false, + ALTER COLUMN have_examples SET NOT NULL, + ALTER COLUMN have_examples SET DEFAULT false, + ALTER COLUMN downloads SET NOT NULL, + ALTER COLUMN downloads SET DEFAULT 0, + ALTER COLUMN doc_targets SET NOT NULL, + ALTER COLUMN doc_targets SET DEFAULT '[]'::json, + ALTER COLUMN default_target SET NOT NULL; diff --git a/migrations/20240313103708_make_release_fields_optional.up.sql b/migrations/20240313103708_make_release_fields_optional.up.sql new file mode 100644 index 000000000..681c1b064 --- /dev/null +++ b/migrations/20240313103708_make_release_fields_optional.up.sql @@ -0,0 +1,17 @@ +ALTER TABLE releases + ALTER COLUMN release_time DROP NOT NULL, + ALTER COLUMN target_name DROP NOT NULL, + ALTER COLUMN yanked DROP NOT NULL, + ALTER COLUMN yanked DROP DEFAULT, + ALTER COLUMN is_library DROP NOT NULL, + ALTER COLUMN is_library DROP DEFAULT, + ALTER COLUMN rustdoc_status DROP NOT NULL, + ALTER COLUMN rustdoc_status DROP DEFAULT, + ALTER COLUMN test_status DROP DEFAULT, + ALTER COLUMN have_examples DROP NOT NULL, + ALTER COLUMN have_examples DROP DEFAULT, + ALTER COLUMN downloads DROP NOT NULL, + ALTER COLUMN downloads DROP DEFAULT, + ALTER COLUMN doc_targets DROP NOT NULL, + ALTER COLUMN doc_targets DROP DEFAULT, + ALTER COLUMN default_target DROP NOT NULL; diff --git a/migrations/20240313182623_make_build_fields_optional.down.sql b/migrations/20240313182623_make_build_fields_optional.down.sql new file mode 100644 index 000000000..8877b8862 --- /dev/null +++ b/migrations/20240313182623_make_build_fields_optional.down.sql @@ -0,0 +1,15 @@ +DELETE FROM builds WHERE ( + rustc_version IS NULL OR + docsrs_version IS NULL OR + build_time IS NULL OR + build_server IS NULL +); + + +ALTER TABLE builds + ALTER COLUMN rustc_version SET NOT NULL, + ALTER COLUMN docsrs_version SET NOT NULL, + ALTER COLUMN build_time SET NOT NULL, + ALTER COLUMN build_time SET DEFAULT now(), + ALTER COLUMN build_server SET NOT NULL, + ALTER COLUMN build_server SET DEFAULT ''; diff --git a/migrations/20240313182623_make_build_fields_optional.up.sql b/migrations/20240313182623_make_build_fields_optional.up.sql new file mode 100644 index 000000000..9e7ec0d07 --- /dev/null +++ b/migrations/20240313182623_make_build_fields_optional.up.sql @@ -0,0 +1,7 @@ +ALTER TABLE builds + ALTER COLUMN rustc_version DROP NOT NULL, + ALTER COLUMN docsrs_version DROP NOT NULL, + ALTER COLUMN build_time DROP NOT NULL, + ALTER COLUMN build_time DROP DEFAULT, + ALTER COLUMN build_server DROP NOT NULL, + ALTER COLUMN build_server DROP DEFAULT; diff --git a/migrations/20240313184911_build_errors.down.sql b/migrations/20240313184911_build_errors.down.sql new file mode 100644 index 000000000..2b4eda132 --- /dev/null +++ b/migrations/20240313184911_build_errors.down.sql @@ -0,0 +1 @@ +ALTER TABLE builds DROP COLUMN errors; diff --git a/migrations/20240313184911_build_errors.up.sql b/migrations/20240313184911_build_errors.up.sql new file mode 100644 index 000000000..952986658 --- /dev/null +++ b/migrations/20240313184911_build_errors.up.sql @@ -0,0 +1 @@ +ALTER TABLE builds ADD COLUMN errors TEXT; diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 3bbcff263..ddd115f6f 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -182,6 +182,11 @@ pub async fn update_build_status(conn: &mut sqlx::PgConnection, release_id: i32) .execute(&mut *conn) .await?; + let crate_id = crate_id_from_release_id(&mut *conn, release_id).await?; + update_latest_version_id(&mut *conn, crate_id) + .await + .context("couldn't update latest version id")?; + Ok(()) } @@ -228,35 +233,66 @@ pub(crate) async fn add_doc_coverage( /// Adds a build into database #[instrument(skip(conn))] -pub(crate) async fn add_build_into_database( +pub(crate) async fn finish_build( conn: &mut sqlx::PgConnection, - release_id: i32, + build_id: i32, rustc_version: &str, docsrs_version: &str, build_status: BuildStatus, -) -> Result { - debug!("Adding build into database"); + errors: Option<&str>, +) -> Result<()> { + debug!("updating build after finishing"); let hostname = hostname::get()?; - let build_id = sqlx::query_scalar!( - "INSERT INTO builds (rid, rustc_version, docsrs_version, build_status, build_server) - VALUES ($1, $2, $3, $4, $5) - RETURNING id", - release_id, + let release_id = sqlx::query_scalar!( + "UPDATE builds + SET + rustc_version = $1, + docsrs_version = $2, + build_status = $3, + build_server = $4, + errors = $5, + build_time = NOW() + WHERE + id = $6 + RETURNING rid", rustc_version, docsrs_version, build_status as BuildStatus, hostname.to_str().unwrap_or(""), + errors, + build_id, ) .fetch_one(&mut *conn) .await?; update_build_status(conn, release_id).await?; - let crate_id = crate_id_from_release_id(&mut *conn, release_id).await?; - update_latest_version_id(&mut *conn, crate_id) - .await - .context("couldn't update latest version id")?; + Ok(()) +} + +#[instrument(skip(conn))] +pub(crate) async fn update_build_with_error( + conn: &mut sqlx::PgConnection, + build_id: i32, + errors: Option<&str>, +) -> Result { + debug!("updating build with error"); + let release_id = sqlx::query_scalar!( + "UPDATE builds + SET + build_status = $1, + errors = $2 + WHERE id = $3 + RETURNING rid", + BuildStatus::Failure as BuildStatus, + errors, + build_id, + ) + .fetch_one(&mut *conn) + .await?; + + update_build_status(conn, release_id).await?; Ok(build_id) } @@ -276,6 +312,51 @@ pub(crate) async fn initialize_crate(conn: &mut sqlx::PgConnection, name: &str) .map_err(Into::into) } +pub(crate) async fn initialize_release( + conn: &mut sqlx::PgConnection, + crate_id: i32, + version: &str, +) -> Result { + let release_id = sqlx::query_scalar!( + "INSERT INTO releases (crate_id, version, archive_storage) + VALUES ($1, $2, TRUE) + ON CONFLICT (crate_id, version) DO UPDATE + SET -- this `SET` is needed so the id is always returned. + version = EXCLUDED.version + RETURNING id", + crate_id, + version + ) + .fetch_one(&mut *conn) + .await?; + + update_build_status(conn, release_id).await?; + + Ok(release_id) +} + +pub(crate) async fn initialize_build( + conn: &mut sqlx::PgConnection, + release_id: i32, +) -> Result { + let hostname = hostname::get()?; + + let build_id = sqlx::query_scalar!( + "INSERT INTO builds(rid, build_status, build_server) + VALUES ($1, $2, $3) + RETURNING id", + release_id, + BuildStatus::InProgress as BuildStatus, + hostname.to_str().unwrap_or(""), + ) + .fetch_one(&mut *conn) + .await?; + + update_build_status(conn, release_id).await?; + + Ok(build_id) +} + /// Convert dependencies into Vec<(String, String, String, bool)> fn convert_dependencies(pkg: &MetadataPackage) -> Vec<(String, String, String, bool)> { pkg.dependencies @@ -644,8 +725,7 @@ mod test { fn new_owners() { async_wrapper(|env| async move { let mut conn = env.async_db().await.async_conn().await; - - let crate_id = initialize_crate(&mut conn, "").await?; + let crate_id = initialize_crate(&mut conn, "krate").await?; let owner1 = CrateOwner { avatar: "avatar".into(), @@ -685,7 +765,7 @@ mod test { fn update_owner_detais() { async_wrapper(|env| async move { let mut conn = env.async_db().await.async_conn().await; - let crate_id = initialize_crate(&mut conn, "").await?; + let crate_id = initialize_crate(&mut conn, "krate").await?; // set initial owner details update_owners_in_database( @@ -734,7 +814,7 @@ mod test { fn add_new_owners_and_delete_old() { async_wrapper(|env| async move { let mut conn = env.async_db().await.async_conn().await; - let crate_id = initialize_crate(&mut conn, "").await?; + let crate_id = initialize_crate(&mut conn, "krate").await?; // set initial owner details update_owners_in_database( @@ -846,4 +926,76 @@ mod test { Ok(()) } + + #[test] + fn test_initialize_crate() { + async_wrapper(|env| async move { + let mut conn = env.async_db().await.async_conn().await; + + let name = "krate"; + let crate_id = initialize_crate(&mut conn, name).await?; + + let id: i32 = sqlx::query_scalar!("SELECT id FROM crates WHERE name = $1", name) + .fetch_one(&mut *conn) + .await?; + + assert_eq!(crate_id, id); + + let same_crate_id = initialize_crate(&mut conn, name).await?; + assert_eq!(crate_id, same_crate_id); + + Ok(()) + }) + } + + #[test] + fn test_initialize_release() { + async_wrapper(|env| async move { + let mut conn = env.async_db().await.async_conn().await; + let name = "krate"; + let version = "0.1.0"; + let crate_id = initialize_crate(&mut conn, name).await?; + + let release_id = initialize_release(&mut conn, crate_id, version).await?; + + let id: i32 = sqlx::query_scalar!( + "SELECT id FROM releases WHERE crate_id = $1 and version = $2", + crate_id, + version + ) + .fetch_one(&mut *conn) + .await?; + + assert_eq!(release_id, id); + + let same_release_id = initialize_release(&mut conn, crate_id, version).await?; + assert_eq!(release_id, same_release_id); + + Ok(()) + }) + } + + #[test] + fn test_initialize_build() { + async_wrapper(|env| async move { + let mut conn = env.async_db().await.async_conn().await; + let name = "krate"; + let version = "0.1.0"; + let crate_id = initialize_crate(&mut conn, name).await?; + let release_id = initialize_release(&mut conn, crate_id, version).await?; + + let build_id = initialize_build(&mut conn, release_id).await?; + + let id: i32 = sqlx::query_scalar!("SELECT id FROM builds WHERE rid = $1", release_id) + .fetch_one(&mut *conn) + .await?; + + assert_eq!(build_id, id); + + let another_build_id = initialize_build(&mut conn, release_id).await?; + assert_ne!(build_id, another_build_id); + + Ok(()) + }) + } } diff --git a/src/db/mod.rs b/src/db/mod.rs index 0085ad564..822f98fe0 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -4,7 +4,8 @@ use sqlx::migrate::{Migrate, Migrator}; pub use self::add_package::update_latest_version_id; pub(crate) use self::add_package::{ - add_build_into_database, add_doc_coverage, add_package_into_database, + add_doc_coverage, add_package_into_database, finish_build, initialize_build, initialize_crate, + initialize_release, update_build_with_error, }; pub use self::{ add_package::{update_build_status, update_crate_data_in_database}, diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 3e0f9d78e..9dc50354a 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -1,7 +1,8 @@ use crate::db::file::add_path_into_database; use crate::db::{ - add_build_into_database, add_doc_coverage, add_package_into_database, - add_path_into_remote_archive, types::BuildStatus, update_crate_data_in_database, Pool, + add_doc_coverage, add_package_into_database, add_path_into_remote_archive, finish_build, + initialize_build, initialize_crate, initialize_release, types::BuildStatus, + update_build_with_error, update_crate_data_in_database, Pool, }; use crate::docbuilder::Limits; use crate::error::Result; @@ -368,8 +369,37 @@ impl RustwideBuilder { version: &str, kind: PackageKind<'_>, ) -> Result { - let mut conn = self.db.get()?; + let build_id = self.runtime.block_on(async { + let mut conn = self.db.get_async().await?; + let crate_id = initialize_crate(&mut conn, name).await?; + let release_id = initialize_release(&mut conn, crate_id, version).await?; + let build_id = initialize_build(&mut conn, release_id).await?; + Ok::(build_id) + })?; + + match self.build_package_inner(name, version, kind, build_id) { + Ok(successful) => Ok(successful), + Err(err) => self.runtime.block_on(async { + // NOTE: this might hide some errors from us, while only surfacing them in the build + // result. + // At some point we might introduce a special error type which additionally reports + // to sentry. + let mut conn = self.db.get_async().await?; + update_build_with_error(&mut conn, build_id, Some(&format!("{:?}", err))).await?; + + Ok(false) + }), + } + } + fn build_package_inner( + &mut self, + name: &str, + version: &str, + kind: PackageKind<'_>, + build_id: i32, + ) -> Result { + let mut conn = self.db.get()?; info!("building package {} {}", name, version); if is_blacklisted(&mut conn, name)? { @@ -608,12 +638,13 @@ impl RustwideBuilder { } else { BuildStatus::Failure }; - let build_id = self.runtime.block_on(add_build_into_database( + self.runtime.block_on(finish_build( &mut async_conn, - release_id, + build_id, &res.result.rustc_version, &res.result.docsrs_version, build_status, + None, ))?; { diff --git a/src/test/fakes.rs b/src/test/fakes.rs index 3d33d2444..5084a2182 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -612,12 +612,14 @@ impl FakeBuild { release_id: i32, default_target: &str, ) -> Result<()> { - let build_id = crate::db::add_build_into_database( + let build_id = crate::db::initialize_build(&mut *conn, release_id).await?; + crate::db::finish_build( &mut *conn, - release_id, + build_id, &self.rustc_version, &self.docsrs_version, self.build_status, + None, ) .await?; diff --git a/src/web/build_details.rs b/src/web/build_details.rs index 119414f0d..faf714f19 100644 --- a/src/web/build_details.rs +++ b/src/web/build_details.rs @@ -20,11 +20,12 @@ use std::sync::Arc; #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub(crate) struct BuildDetails { id: i32, - rustc_version: String, - docsrs_version: String, + rustc_version: Option, + docsrs_version: Option, build_status: BuildStatus, - build_time: DateTime, + build_time: Option>, output: String, + errors: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] @@ -63,6 +64,7 @@ pub(crate) async fn build_details_handler( builds.build_status as "build_status: BuildStatus", builds.build_time, builds.output, + builds.errors, releases.default_target FROM builds INNER JOIN releases ON releases.id = builds.rid @@ -81,26 +83,33 @@ pub(crate) async fn build_details_handler( } else { let prefix = format!("build-logs/{}/", id); - let current_filename = params + if let Some(current_filename) = params .filename - .unwrap_or_else(|| format!("{}.txt", row.default_target)); - - let path = format!("{prefix}{current_filename}"); - let file = File::from_path(&storage, &path, &config).await?; - ( - String::from_utf8(file.0.content).context("non utf8")?, - storage - .list_prefix(&prefix) // the result from S3 is ordered by key - .await - .map_ok(|path| { - path.strip_prefix(&prefix) - .expect("since we query for the prefix, it has to be always there") - .to_owned() - }) - .try_collect() - .await?, - Some(current_filename), - ) + .or(row.default_target.map(|target| format!("{}.txt", target))) + { + let path = format!("{prefix}{current_filename}"); + let file = File::from_path(&storage, &path, &config).await?; + ( + String::from_utf8(file.0.content).context("non utf8")?, + storage + .list_prefix(&prefix) // the result from S3 is ordered by key + .await + .map_ok(|path| { + path.strip_prefix(&prefix) + .expect("since we query for the prefix, it has to be always there") + .to_owned() + }) + .try_collect() + .await?, + Some(current_filename), + ) + } else { + // this can only happen when `releases.default_target` is NULL, + // which is the case for in-progress builds or builds which errored + // before we could determine the target. + // For the "error" case we show `row.errors`, which should contain what we need to see. + ("".into(), Vec::new(), None) + } }; Ok(BuildDetailsPage { @@ -112,6 +121,7 @@ pub(crate) async fn build_details_handler( build_status: row.build_status, build_time: row.build_time, output, + errors: row.errors, }, use_direct_platform_links: true, all_log_filenames, @@ -154,6 +164,7 @@ mod tests { env.frontend() .get("/crate/foo/0.1.0/builds") .send()? + .error_for_status()? .text()?, ); diff --git a/src/web/builds.rs b/src/web/builds.rs index d79b7bbc4..40bae507c 100644 --- a/src/web/builds.rs +++ b/src/web/builds.rs @@ -22,10 +22,11 @@ use std::sync::Arc; #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub(crate) struct Build { id: i32, - rustc_version: String, - docsrs_version: String, + rustc_version: Option, + docsrs_version: Option, build_status: BuildStatus, - build_time: DateTime, + build_time: Option>, + errors: Option, } #[derive(Debug, Clone, Serialize)] @@ -123,7 +124,8 @@ async fn get_builds( builds.rustc_version, builds.docsrs_version, builds.build_status as "build_status: BuildStatus", - builds.build_time + builds.build_time, + builds.errors FROM builds INNER JOIN releases ON releases.id = builds.rid INNER JOIN crates ON releases.crate_id = crates.id @@ -286,25 +288,30 @@ mod tests { .create()?; let response = env.frontend().get("/crate/foo/0.1.0/builds").send()?; - assert_cache_control(&response, CachePolicy::NoCaching, &env.config()); - let page = kuchikiki::parse_html().one(response.text()?); - let rows: Vec<_> = page - .select("ul > li a.release") - .unwrap() - .map(|row| row.text_contents()) - .collect(); + // FIXME: temporarily we don't show in-progress releases anywhere, which means we don't + // show releases without builds anywhere. + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + // assert_cache_control(&response, CachePolicy::NoCaching, &env.config()); + // let page = kuchikiki::parse_html().one(response.text()?); + + // let rows: Vec<_> = page + // .select("ul > li a.release") + // .unwrap() + // .map(|row| row.text_contents()) + // .collect(); - assert!(rows.is_empty()); + // assert!(rows.is_empty()); - let warning = page - .select_first(".warning") - .expect("missing warning element") - .text_contents(); + // let warning = page + // .select_first(".warning") + // .expect("missing warning element") + // .text_contents(); - assert!(warning.contains("has not built")); - assert!(warning.contains("queued")); - assert!(warning.contains("open an issue")); + // assert!(warning.contains("has not built")); + // assert!(warning.contains("queued")); + // assert!(warning.contains("open an issue")); Ok(()) }); diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index d28969845..86fe7e671 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -42,21 +42,21 @@ pub(crate) struct CrateDetails { readme: Option, #[serde(serialize_with = "optional_markdown")] rustdoc: Option, // this is description_long in database - release_time: DateTime, + release_time: Option>, build_status: BuildStatus, pub latest_build_id: Option, last_successful_build: Option, - pub rustdoc_status: bool, + pub rustdoc_status: Option, pub archive_storage: bool, repository_url: Option, homepage_url: Option, keywords: Option, - have_examples: bool, // need to check this manually - pub target_name: String, + have_examples: Option, // need to check this manually + pub target_name: Option, releases: Vec, repository_metadata: Option, pub(crate) metadata: MetaData, - is_library: bool, + is_library: Option, license: Option, pub(crate) documentation_url: Option, total_items: Option, @@ -103,10 +103,10 @@ pub(crate) struct Release { /// up here, but we still check. /// calculated in a database view : `release_build_status` pub build_status: BuildStatus, - pub yanked: bool, - pub is_library: bool, - pub rustdoc_status: bool, - pub target_name: String, + pub yanked: Option, + pub is_library: Option, + pub rustdoc_status: Option, + pub target_name: Option, } impl CrateDetails { @@ -219,9 +219,9 @@ impl CrateDetails { req_version: req_version.unwrap_or_else(|| ReqVersion::Exact(version.clone())), description: krate.description.clone(), rustdoc_status: krate.rustdoc_status, - target_name: Some(krate.target_name.clone()), + target_name: krate.target_name.clone(), default_target: krate.default_target, - doc_targets: MetaData::parse_doc_targets(krate.doc_targets), + doc_targets: krate.doc_targets.map(MetaData::parse_doc_targets), yanked: krate.yanked, rustdoc_css_file: krate .rustc_version @@ -280,7 +280,9 @@ impl CrateDetails { crate_details.last_successful_build = crate_details .releases .iter() - .filter(|release| release.build_status == BuildStatus::Success && !release.yanked) + .filter(|release| { + release.build_status == BuildStatus::Success && release.yanked == Some(false) + }) .map(|release| release.version.to_string()) .next(); } @@ -355,7 +357,7 @@ impl CrateDetails { pub(crate) fn latest_release(releases: &[Release]) -> Option<&Release> { if let Some(release) = releases.iter().find(|release| { release.version.pre.is_empty() - && !release.yanked + && release.yanked == Some(false) && release.build_status != BuildStatus::InProgress }) { Some(release) @@ -383,7 +385,8 @@ pub(crate) async fn releases_for_crate( FROM releases INNER JOIN release_build_status ON releases.id = release_build_status.rid WHERE - releases.crate_id = $1"#, + releases.crate_id = $1 AND + release_build_status.build_status != 'in_progress'"#, crate_id, ) .fetch(&mut *conn) @@ -500,18 +503,26 @@ pub(crate) async fn get_all_releases( .await? .into_canonical_req_version_or_else(|_| AxumNope::VersionNotFound)?; - let row = sqlx::query!( + if matched_release.build_status() != BuildStatus::Success { + // this handler should only be used for successful builds, so then we have all rows in the + // `releases` table filled with data. + // If we at some point need this view for in-progress releases or failed releases, we need + // to handle empty doc targets. + return Err(AxumNope::CrateNotFound); + } + + let doc_targets = sqlx::query_scalar!( "SELECT releases.doc_targets - FROM releases - WHERE releases.id = $1;", + FROM releases + WHERE releases.id = $1;", matched_release.id(), ) .fetch_optional(&mut *conn) .await? - .ok_or(AxumNope::CrateNotFound)?; - - let doc_targets = MetaData::parse_doc_targets(row.doc_targets); + .ok_or(AxumNope::CrateNotFound)? + .map(MetaData::parse_doc_targets) + .ok_or_else(|| anyhow!("empty doc targets for successful release"))?; let inner; let (target, inner_path) = { @@ -531,10 +542,15 @@ pub(crate) async fn get_all_releases( inner = inner_path.join("/"); (target, inner.trim_end_matches('/')) }; + + let target_name = matched_release + .target_name() + .ok_or_else(|| anyhow!("empty target name for succesful release"))?; + let inner_path = if inner_path.is_empty() { - format!("{}/index.html", matched_release.target_name()) + format!("{}/index.html", target_name) } else { - format!("{}/{inner_path}", matched_release.target_name()) + format!("{}/{inner_path}", target_name) }; let target = if target.is_empty() { @@ -620,10 +636,27 @@ pub(crate) async fn get_all_platforms_inner( .await? .ok_or(AxumNope::CrateNotFound)?; - let doc_targets = MetaData::parse_doc_targets(krate.doc_targets); + if krate.doc_targets.is_none() + || krate.default_target.is_none() + || matched_release.target_name().is_none() + { + // FIXME: is this really OK? just return an empty platform list in case of + // an early failure? + return Ok(PlatformList { + metadata: ShortMetadata { + name: params.name, + version: matched_release.version().clone(), + req_version: params.version.clone(), + doc_targets: Vec::new(), + }, + inner_path: "".into(), + use_direct_platform_links: is_crate_root, + current_target: "".into(), + } + .into_response()); + } - let latest_release = latest_release(&matched_release.all_releases) - .expect("we couldn't end up here without releases"); + let doc_targets = MetaData::parse_doc_targets(krate.doc_targets.unwrap()); // The path within this crate version's rustdoc output let inner; @@ -645,14 +678,17 @@ pub(crate) async fn get_all_platforms_inner( (target, inner.trim_end_matches('/')) }; let inner_path = if inner_path.is_empty() { - format!("{}/index.html", matched_release.target_name()) + format!("{}/index.html", matched_release.target_name().unwrap()) } else { - format!("{}/{inner_path}", matched_release.target_name()) + format!("{}/{inner_path}", matched_release.target_name().unwrap()) }; + let latest_release = latest_release(&matched_release.all_releases) + .expect("we couldn't end up here without releases"); + let current_target = if latest_release.build_status.is_success() { if target.is_empty() { - krate.default_target + krate.default_target.unwrap() } else { target.to_owned() } @@ -660,7 +696,7 @@ pub(crate) async fn get_all_platforms_inner( String::new() }; - let res = PlatformList { + Ok(PlatformList { metadata: ShortMetadata { name: params.name, version: matched_release.version().clone(), @@ -670,8 +706,8 @@ pub(crate) async fn get_all_platforms_inner( inner_path, use_direct_platform_links: is_crate_root, current_target, - }; - Ok(res.into_response()) + } + .into_response()) } pub(crate) async fn get_all_platforms_root( @@ -699,6 +735,7 @@ mod tests { use crate::{db::update_build_status, registry_api::CrateOwner}; use anyhow::Error; use kuchikiki::traits::TendrilSink; + use pretty_assertions::assert_eq; use reqwest::StatusCode; use std::collections::HashMap; @@ -940,74 +977,74 @@ mod tests { Release { version: semver::Version::parse("1.0.0")?, build_status: BuildStatus::Success, - yanked: false, - is_library: true, - rustdoc_status: true, + yanked: Some(false), + is_library: Some(true), + rustdoc_status: Some(true), id: details.releases[0].id, - target_name: "foo".to_owned(), + target_name: Some("foo".to_owned()), }, Release { version: semver::Version::parse("0.12.0")?, build_status: BuildStatus::Success, - yanked: false, - is_library: true, - rustdoc_status: true, + yanked: Some(false), + is_library: Some(true), + rustdoc_status: Some(true), id: details.releases[1].id, - target_name: "foo".to_owned(), + target_name: Some("foo".to_owned()), }, Release { version: semver::Version::parse("0.3.0")?, build_status: BuildStatus::Failure, - yanked: false, - is_library: true, - rustdoc_status: false, + yanked: Some(false), + is_library: Some(true), + rustdoc_status: Some(false), id: details.releases[2].id, - target_name: "foo".to_owned(), + target_name: Some("foo".to_owned()), }, Release { version: semver::Version::parse("0.2.0")?, build_status: BuildStatus::Success, - yanked: true, - is_library: true, - rustdoc_status: true, + yanked: Some(true), + is_library: Some(true), + rustdoc_status: Some(true), id: details.releases[3].id, - target_name: "foo".to_owned(), + target_name: Some("foo".to_owned()), }, Release { version: semver::Version::parse("0.2.0-alpha")?, build_status: BuildStatus::Success, - yanked: false, - is_library: true, - rustdoc_status: true, + yanked: Some(false), + is_library: Some(true), + rustdoc_status: Some(true), id: details.releases[4].id, - target_name: "foo".to_owned(), + target_name: Some("foo".to_owned()), }, Release { version: semver::Version::parse("0.1.1")?, build_status: BuildStatus::Success, - yanked: false, - is_library: true, - rustdoc_status: true, + yanked: Some(false), + is_library: Some(true), + rustdoc_status: Some(true), id: details.releases[5].id, - target_name: "foo".to_owned(), + target_name: Some("foo".to_owned()), }, Release { version: semver::Version::parse("0.1.0")?, build_status: BuildStatus::Success, - yanked: false, - is_library: true, - rustdoc_status: true, + yanked: Some(false), + is_library: Some(true), + rustdoc_status: Some(true), id: details.releases[6].id, - target_name: "foo".to_owned(), + target_name: Some("foo".to_owned()), }, Release { version: semver::Version::parse("0.0.1")?, build_status: BuildStatus::Failure, - yanked: false, - is_library: false, - rustdoc_status: false, + yanked: Some(false), + is_library: Some(false), + rustdoc_status: Some(false), id: details.releases[7].id, - target_name: "foo".to_owned(), + target_name: Some("foo".to_owned()), }, ] ); @@ -1222,22 +1259,26 @@ mod tests { ]) .create()?; - let page = kuchikiki::parse_html() - .one(env.frontend().get("/crate/foo/latest").send()?.text()?); - let link = page - .select_first("a.pure-menu-link[href='/crate/foo/0.1.0']") - .unwrap(); - - assert_eq!( - link.as_node() - .as_element() - .unwrap() - .attributes - .borrow() - .get("title") - .unwrap(), - "foo-0.1.0 is currently being built" - ); + let response = env.frontend().get("/crate/foo/latest").send()?; + // FIXME: temporarily we don't show in-progress releases anywhere, which means we don't + // show releases without builds anywhere. + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + // let page = kuchikiki::parse_html().one(response.text()?); + // let link = page + // .select_first("a.pure-menu-link[href='/crate/foo/0.1.0']") + // .unwrap(); + + // assert_eq!( + // link.as_node() + // .as_element() + // .unwrap() + // .attributes + // .borrow() + // .get("title") + // .unwrap(), + // "foo-0.1.0 is currently being built" + // ); Ok(()) }); diff --git a/src/web/mod.rs b/src/web/mod.rs index 2db225a02..923f7f331 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -2,6 +2,7 @@ pub mod page; +use crate::db::types::BuildStatus; use crate::utils::get_correct_docsrs_style_file; use crate::utils::report_error; use anyhow::{anyhow, bail, Context as _, Result}; @@ -58,6 +59,8 @@ use tower::ServiceBuilder; use tower_http::{catch_panic::CatchPanicLayer, timeout::TimeoutLayer, trace::TraceLayer}; use url::form_urlencoded; +use self::crate_details::Release; + // from https://github.com/servo/rust-url/blob/master/url/src/parser.rs // and https://github.com/tokio-rs/axum/blob/main/axum-extra/src/lib.rs const FRAGMENT: &AsciiSet = &CONTROLS.add(b' ').add(b'"').add(b'<').add(b'>').add(b'`'); @@ -214,12 +217,16 @@ impl MatchedRelease { self.release.id } + fn build_status(&self) -> BuildStatus { + self.release.build_status + } + fn rustdoc_status(&self) -> bool { - self.release.rustdoc_status + self.release.rustdoc_status.unwrap_or(false) } - fn target_name(&self) -> &str { - &self.release.target_name + fn target_name(&self) -> Option<&str> { + self.release.target_name.as_deref() } fn is_latest_url(&self) -> bool { @@ -262,7 +269,7 @@ async fn match_version( // first load and parse all versions of this crate, // `releases_for_crate` is already sorted, newest version first. - let mut releases = crate_details::releases_for_crate(conn, crate_id) + let releases = crate_details::releases_for_crate(conn, crate_id) .await .context("error fetching releases for crate")?; @@ -299,10 +306,11 @@ async fn match_version( }; // when matching semver requirements, we only want to look at non-yanked releases. - releases.retain(|r| !r.yanked); + let flt = |r: &&Release| r.yanked == Some(false); if let Some(release) = releases .iter() + .filter(flt) .find(|release| req_semver.matches(&release.version)) { return Ok(MatchedRelease { @@ -319,7 +327,8 @@ async fn match_version( // just return the latest prerelease. if req_semver == VersionReq::STAR { return releases - .first() + .iter() + .find(flt) .cloned() .map(|release| MatchedRelease { name: name.to_owned(), @@ -618,10 +627,10 @@ pub(crate) struct MetaData { pub(crate) req_version: ReqVersion, pub(crate) description: Option, pub(crate) target_name: Option, - pub(crate) rustdoc_status: bool, - pub(crate) default_target: String, - pub(crate) doc_targets: Vec, - pub(crate) yanked: bool, + pub(crate) rustdoc_status: Option, + pub(crate) default_target: Option, + pub(crate) doc_targets: Option>, + pub(crate) yanked: Option, /// CSS file to use depending on the rustdoc version used to generate this version of this /// crate. pub(crate) rustdoc_css_file: Option, @@ -667,12 +676,11 @@ impl MetaData { version: version.clone(), req_version: req_version.unwrap_or_else(|| ReqVersion::Exact(version.clone())), description: row.description, - target_name: Some(row.target_name), + target_name: row.target_name, rustdoc_status: row.rustdoc_status, default_target: row.default_target, - doc_targets: MetaData::parse_doc_targets(row.doc_targets), + doc_targets: row.doc_targets.map(MetaData::parse_doc_targets), yanked: row.yanked, - // rustdoc_css_file: get_correct_docsrs_style_file(&row.doc_rustc_version).unwrap(), rustdoc_css_file: row .rustc_version .as_deref() @@ -1076,13 +1084,13 @@ mod test { req_version: ReqVersion::Latest, description: Some("serde does stuff".to_string()), target_name: None, - rustdoc_status: true, - default_target: "x86_64-unknown-linux-gnu".to_string(), - doc_targets: vec![ + rustdoc_status: Some(true), + default_target: Some("x86_64-unknown-linux-gnu".to_string()), + doc_targets: Some(vec![ "x86_64-unknown-linux-gnu".to_string(), "arm64-unknown-linux-gnu".to_string(), - ], - yanked: false, + ]), + yanked: Some(false), rustdoc_css_file: Some("rustdoc.css".to_string()), }; @@ -1163,10 +1171,10 @@ mod test { req_version: ReqVersion::Latest, description: Some("Fake package".to_string()), target_name: Some("foo".to_string()), - rustdoc_status: true, - default_target: "x86_64-unknown-linux-gnu".to_string(), - doc_targets: vec![], - yanked: false, + rustdoc_status: Some(true), + default_target: Some("x86_64-unknown-linux-gnu".to_string()), + doc_targets: Some(vec![]), + yanked: Some(false), rustdoc_css_file: Some("rustdoc.css".to_string()), }, ); diff --git a/src/web/releases.rs b/src/web/releases.rs index f6fe8ea00..5f9417141 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -47,7 +47,7 @@ pub struct Release { description: Option, target_name: Option, rustdoc_status: bool, - pub(crate) build_time: DateTime, + pub(crate) build_time: Option>, stars: i32, has_unyanked_releases: Option, } @@ -89,7 +89,7 @@ pub(crate) async fn get_releases( releases.description, releases.target_name, releases.rustdoc_status, - release_build_status.last_build_time AS build_time, + release_build_status.last_build_time, repositories.stars FROM crates {1} @@ -97,7 +97,8 @@ pub(crate) async fn get_releases( LEFT JOIN repositories ON releases.repository_id = repositories.id WHERE ((NOT $3) OR (release_build_status.build_status = 'failure' AND releases.is_library = TRUE)) - AND {0} IS NOT NULL + AND {0} IS NOT NULL AND + release_build_status.build_status != 'in_progress' ORDER BY {0} DESC LIMIT $1 OFFSET $2", @@ -239,7 +240,7 @@ async fn get_search_results( crates.name, releases.version, releases.description, - builds.build_time, + release_build_status.last_build_time, releases.target_name, releases.rustdoc_status, repositories.stars as "stars?", @@ -253,10 +254,12 @@ async fn get_search_results( FROM crates INNER JOIN releases ON crates.latest_version_id = releases.id - INNER JOIN builds ON releases.id = builds.rid + INNER JOIN release_build_status ON releases.id = release_build_status.rid LEFT JOIN repositories ON releases.repository_id = repositories.id - WHERE crates.name = ANY($1)"#, + WHERE + crates.name = ANY($1) AND + release_build_status.build_status <> 'in_progress'"#, &names[..], ) .fetch(&mut *conn) @@ -267,9 +270,9 @@ async fn get_search_results( name: row.name, version: row.version, description: row.description, - build_time: row.build_time, - target_name: Some(row.target_name), - rustdoc_status: row.rustdoc_status, + build_time: row.last_build_time, + target_name: row.target_name, + rustdoc_status: row.rustdoc_status.unwrap_or(false), stars: row.stars.unwrap_or(0), has_unyanked_releases: row.has_unyanked_releases, }, @@ -512,7 +515,10 @@ async fn redirect_to_random_crate( Ok(axum_redirect(format!( "/{}/{}/{}/", - row.name, row.version, row.target_name + row.name, + row.version, + row.target_name + .expect("we only look at releases with docs, so target_name will exist") ))?) } else { report_error(&anyhow!("found no result in random crate search")); @@ -575,7 +581,9 @@ pub(crate) async fn search_handler( "/{}/{}/{}/", matchver.name, matchver.version(), - matchver.target_name(), + matchver + .target_name() + .expect("target name will exist when rustdoc_status is true"), ), queries, )? diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 8b36a7a0a..177991e44 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -214,23 +214,24 @@ pub(crate) async fn rustdoc_redirector_handler( let matched_release = matched_release.into_canonical_req_version(); - let mut target = params.target.as_deref(); - if target == Some("index.html") || target == Some(matched_release.target_name()) { - target = None; - } - if matched_release.rustdoc_status() { + let target_name = matched_release + .target_name() + .expect("when rustdoc_status is true, target name exists"); + let mut target = params.target.as_deref(); + if target == Some("index.html") || target == Some(target_name) { + target = None; + } + let url_str = if let Some(target) = target { format!( "/{crate_name}/{}/{target}/{}/", - matched_release.req_version, - matched_release.target_name(), + matched_release.req_version, target_name ) } else { format!( "/{crate_name}/{}/{}/", - matched_release.req_version, - matched_release.target_name(), + matched_release.req_version, target_name ) }; @@ -411,11 +412,7 @@ pub(crate) async fn rustdoc_html_server_handler( ) })?; - trace!("crate details"); - // Get the crate's details from the database - let krate = CrateDetails::from_matched_release(&mut conn, matched_release).await?; - - if !krate.rustdoc_status { + if !matched_release.rustdoc_status() { return Ok(axum_cached_redirect( format!("/crate/{}/{}", params.name, params.version), CachePolicy::ForeverInCdn, @@ -423,9 +420,19 @@ pub(crate) async fn rustdoc_html_server_handler( .into_response()); } + let krate = CrateDetails::from_matched_release(&mut conn, matched_release).await?; + // if visiting the full path to the default target, remove the target from the path // expects a req_path that looks like `[/:target]/.*` - if req_path.first().copied() == Some(&krate.metadata.default_target) { + if req_path.first().copied() + == Some( + krate + .metadata + .default_target + .as_ref() + .expect("when we have docs, this is always filled"), + ) + { return redirect( ¶ms.name, &krate.version, @@ -496,7 +503,7 @@ pub(crate) async fn rustdoc_html_server_handler( )? .into_response()) } else { - if storage_path == format!("{}/index.html", krate.target_name) { + if storage_path == format!("{}/index.html", krate.target_name.expect("we check rustdoc_status = true above, and with docs we have target_name")) { error!( krate = params.name, version = krate.version.to_string(), @@ -537,6 +544,8 @@ pub(crate) async fn rustdoc_html_server_handler( && krate .metadata .doc_targets + .as_ref() + .expect("with rustdoc_status=true we always have doc_targets") .iter() .any(|s| s == inner_path[0]) { @@ -551,14 +560,17 @@ pub(crate) async fn rustdoc_html_server_handler( // Find the path of the latest version for the `Go to latest` and `Permalink` links let mut current_target = String::new(); let target_redirect = if latest_release.build_status.is_success() { - let target = if target.is_empty() { - current_target.clone_from(&krate.metadata.default_target); - &krate.metadata.default_target + current_target = if target.is_empty() { + krate + .metadata + .default_target + .as_ref() + .expect("with docs we always have a default_target") + .clone() } else { - target.clone_into(&mut current_target); - target + target.to_owned() }; - format!("/target-redirect/{target}/{inner_path}") + format!("/target-redirect/{current_target}/{inner_path}") } else { "".to_string() }; @@ -641,6 +653,8 @@ fn path_for_version( let platform = if crate_details .metadata .doc_targets + .as_ref() + .expect("this method is only used when we have docs, so this field contains data") .iter() .any(|s| s == file_path[0]) && !file_path.is_empty() @@ -673,7 +687,10 @@ fn path_for_version( // else, don't try searching at all, we don't know how to find it last_component.strip_suffix(".rs.html") }; - let target_name = &crate_details.target_name; + let target_name = &crate_details + .target_name + .as_ref() + .expect("this method is only used when we have docs, so this field contains data"); let path = if platform.is_empty() { format!("{target_name}/") } else { @@ -699,6 +716,18 @@ pub(crate) async fn target_redirect_handler( let crate_details = CrateDetails::from_matched_release(&mut conn, matched_release).await?; + // this handler should only be used when we have docs. + // So we can assume here that we always have a default_target. + // the only case where this would be empty is when the build failed before calling rustdoc. + let default_target = crate_details + .metadata + .default_target + .as_ref() + .ok_or_else(|| { + error!("target_redirect_handler was called with release with missing default_target"); + AxumNope::VersionNotFound + })?; + // We're trying to find the storage location // for the requested path in the target-redirect. // *path always contains the target, @@ -709,7 +738,7 @@ pub(crate) async fn target_redirect_handler( let storage_location_for_path = { let mut pieces: Vec<_> = req_path.split('/').map(str::to_owned).collect(); - if pieces.first() == Some(&crate_details.metadata.default_target) { + if pieces.first() == Some(default_target) { pieces.remove(0); } diff --git a/src/web/sitemap.rs b/src/web/sitemap.rs index bc9927efb..b388a7c64 100644 --- a/src/web/sitemap.rs +++ b/src/web/sitemap.rs @@ -79,7 +79,9 @@ pub(crate) async fn sitemap_handler( .fetch(&mut *conn) .map_ok(|row| SitemapRow { crate_name: row.name, - target_name: row.target_name, + target_name: row + .target_name + .expect("when we have rustdoc_status=true, this field is filled"), last_modified: row .release_time // On Aug 27 2022 we added `` to all pages, diff --git a/src/web/source.rs b/src/web/source.rs index e10373c5e..10a768eb0 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -44,7 +44,6 @@ impl File { /// A list of source files #[derive(Debug, Clone, PartialEq, Serialize)] struct FileList { - metadata: MetaData, files: Vec, } @@ -136,10 +135,7 @@ impl FileList { } }); - Ok(Some(FileList { - metadata: MetaData::from_crate(conn, name, version, req_version).await?, - files: file_list, - })) + Ok(Some(FileList { files: file_list })) } else { Ok(None) } @@ -149,6 +145,7 @@ impl FileList { #[derive(Debug, Clone, Serialize)] struct SourcePage { file_list: FileList, + metadata: MetaData, show_parent_link: bool, file: Option, file_content: Option, @@ -210,7 +207,9 @@ pub(crate) async fn source_browser_handler( ( SELECT id FROM builds - WHERE builds.rid = releases.id + WHERE + builds.rid = releases.id AND + builds.build_status = 'success' ORDER BY build_time DESC LIMIT 1 ) AS latest_build_id @@ -308,6 +307,13 @@ pub(crate) async fn source_browser_handler( Ok(SourcePage { file_list, + metadata: MetaData::from_crate( + &mut conn, + ¶ms.name, + &version, + Some(params.version.clone()), + ) + .await?, show_parent_link: !current_folder.is_empty(), file, file_content, diff --git a/templates/crate/build_details.html b/templates/crate/build_details.html index 80e23dfbd..0f8bb9145 100644 --- a/templates/crate/build_details.html +++ b/templates/crate/build_details.html @@ -6,17 +6,21 @@ {%- endblock title -%} {%- block body_classes -%} -centered + centered {%- endblock body_classes -%} {%- block topbar -%} - {%- set latest_version = "" -%} - {%- set latest_path = "" -%} - {%- set target = "" -%} - {%- set inner_path = metadata.target_name ~ "/index.html" -%} - {%- set is_latest_version = true -%} - {%- set is_prerelease = false -%} - {%- include "rustdoc/topbar.html" -%} + {%- set latest_version = "" -%} + {%- set latest_path = "" -%} + {%- set target = "" -%} + {%- if metadata.target_name -%} + {%- set inner_path = metadata.target_name ~ "/index.html" -%} + {%- else -%} + {%- set inner_path = "" -%} + {%- endif -%} + {%- set is_latest_version = true -%} + {%- set is_prerelease = false -%} + {%- include "rustdoc/topbar.html" -%} {%- endblock topbar -%} {%- block header -%} @@ -27,7 +31,9 @@
- Build #{{ build_details.id }} {{ build_details.build_time | date(format="%+") }} + + Build #{{ build_details.id }} {%- if build_details.build_time -%} {{ build_details.build_time | date(format="%+") }} {%- endif -%} +
    @@ -51,13 +57,26 @@ {%- filter dedent -%} 
    -                    # rustc version
-                    {{ build_details.rustc_version }}
-                    # docs.rs version
-                    {{ build_details.docsrs_version }}
 
-                    # build log
-                    {{ build_details.output }}
+                    {%- if build_details.errors -%}
+                        # pre-build errors
+                        {{ build_details.errors }}
+                    {%- endif -%}
+
+                    {%- if build_details.rustc_version -%}
+                        # rustc version
+                        {{ build_details.rustc_version }}
+                    {%- endif -%}
+
+                    {%- if build_details.docsrs_version -%}
+                        # docs.rs version
+                        {{ build_details.docsrs_version }}
+                    {%- endif -%}
+
+                    {%- if build_details.output -%}
+                        # build log
+                        {{ build_details.output }}
+                    {%- endif -%}
    {%- endfilter -%}
diff --git a/templates/crate/builds.html b/templates/crate/builds.html index 775b48626..696fc95da 100644 --- a/templates/crate/builds.html +++ b/templates/crate/builds.html @@ -6,21 +6,25 @@ {%- endblock title -%} {%- block meta -%} - + {%- endblock -%} {%- block body_classes -%} -centered + centered {%- endblock body_classes -%} {%- block topbar -%} - {%- set latest_version = "" -%} - {%- set latest_path = "" -%} - {%- set target = "" -%} - {%- set inner_path = metadata.target_name ~ "/index.html" -%} - {%- set is_latest_version = true -%} - {%- set is_prerelease = false -%} - {%- include "rustdoc/topbar.html" -%} + {%- set latest_version = "" -%} + {%- set latest_path = "" -%} + {%- set target = "" -%} + {%- if metadata.target_name -%} + {%- set inner_path = metadata.target_name ~ "/index.html" -%} + {%- else -%} + {%- set inner_path = "" -%} + {%- endif -%} + {%- set is_latest_version = true -%} + {%- set is_prerelease = false -%} + {%- include "rustdoc/topbar.html" -%} {%- endblock topbar -%} {%- block header -%} @@ -44,22 +48,41 @@ {%- if build.build_status == "success" -%} {{ "check" | fas }} {%- elif build.build_status == "failure" -%} - {{ "times" | fas }} + {{ "triangle-exclamation" | fas }} {%- elif build.build_status == "in_progress" -%} {{ "gear" | fas(spin=true) }} {%- else -%} {{ "x" | fas }} {%- endif -%}
-
{{ build.rustc_version }}
-
{{ build.docsrs_version }}
-
{{ build.build_time | timeformat(relative=true) }}
+
+ {%- if build.rustc_version -%} + {{ build.rustc_version }} + {%- else -%} + — + {%- endif -%} +
+
+ {%- if build.docsrs_version -%} + {{ build.docsrs_version }} + {%- else -%} + — + {%- endif -%} +
+
+ {%- if build.build_time -%} + {{ build.build_time | timeformat(relative=true) }} + {%- else -%} + — + {%- endif -%} +
{%- endfor -%} {%- else -%} + {# FIXME: temporarily this will never be shown since we hide in-progress releases for now #}
docs.rs has not built {{ metadata.name }}-{{ metadata.version }}
diff --git a/templates/crate/details.html b/templates/crate/details.html index 5409224fe..310d38f63 100644 --- a/templates/crate/details.html +++ b/templates/crate/details.html @@ -10,15 +10,19 @@ {%- endblock meta -%} {%- block topbar -%} - {%- set metadata = details.metadata -%} - {%- set latest_version = "" -%} - {%- set latest_path = "" -%} - {%- set target = "" -%} - {%- set inner_path = details.metadata.target_name ~ "/index.html" -%} - {%- set is_latest_version = true -%} - {%- set is_prerelease = false -%} - {%- set use_direct_platform_links = true -%} - {%- include "rustdoc/topbar.html" -%} + {%- set metadata = details.metadata -%} + {%- set latest_version = "" -%} + {%- set latest_path = "" -%} + {%- set target = "" -%} + {%- if metadata.target_name -%} + {%- set inner_path = metadata.target_name ~ "/index.html" -%} + {%- else -%} + {%- set inner_path = "" -%} + {%- endif -%} + {%- set is_latest_version = true -%} + {%- set is_prerelease = false -%} + {%- set use_direct_platform_links = true -%} + {%- include "rustdoc/topbar.html" -%} {%- endblock topbar -%} {%- block header -%} @@ -101,17 +105,21 @@
@@ -141,13 +149,13 @@
{# If the release is not a library #} - {%- if not details.is_library -%} + {%- if details.is_library == false -%}
{{ details.name }}-{{ details.version }} is not a library.
{# If the release has been yanked and is a library #} - {%- elif details.yanked -%} + {%- elif details.metadata.yanked == true -%}
{{ details.name }}-{{ details.version }} has been yanked.
diff --git a/templates/crate/features.html b/templates/crate/features.html index 51f0733f7..c0ccce855 100644 --- a/templates/crate/features.html +++ b/templates/crate/features.html @@ -13,7 +13,11 @@ {%- set latest_version = "" -%} {%- set latest_path = "" -%} {%- set target = "" -%} - {%- set inner_path = metadata.target_name ~ "/index.html" -%} + {%- if metadata.target_name -%} + {%- set inner_path = metadata.target_name ~ "/index.html" -%} + {%- else -%} + {%- set inner_path = "" -%} + {%- endif -%} {%- set is_latest_version = true -%} {%- set is_prerelease = false -%} {%- include "rustdoc/topbar.html" -%} @@ -44,7 +48,10 @@ {%- else -%}
  • - Feature flags are not available for this release because it was built before features were collected by docs.rs. + + Feature flags are not available for this release because the build failed before we could retrieve them, + or it was built before features were collected by docs.rs. +
    • {%- endif -%} @@ -80,7 +87,10 @@

    {{ feature.name }}

    {%- elif features is iterable -%}

    This release does not have any feature flags.

    {%- else -%} -

    Feature flags are not available for this release because it was built before features were collected by docs.rs.

    +

    + Feature flags are not available for this release because the build failed before we could retrieve them, + or it was built before features were collected by docs.rs. +

    {%- endif -%}
    diff --git a/templates/crate/source.html b/templates/crate/source.html index 88356d3f2..5375c9a4e 100644 --- a/templates/crate/source.html +++ b/templates/crate/source.html @@ -2,23 +2,27 @@ {%- import "header/package_navigation.html" as navigation -%} {%- block title -%} - {{ macros::doc_title(name=file_list.metadata.name, version=file_list.metadata.version) }} + {{ macros::doc_title(name=metadata.name, version=metadata.version) }} {%- endblock title -%} {%- block topbar -%} - {%- set metadata = file_list.metadata -%} - {%- set latest_version = "" -%} - {%- set latest_path = "" -%} - {%- set target = "" -%} - {%- set inner_path = metadata.target_name ~ "/index.html" -%} - {%- set is_latest_version = true -%} - {%- set is_prerelease = false -%} - {%- include "rustdoc/topbar.html" -%} + {%- set metadata = metadata -%} + {%- set latest_version = "" -%} + {%- set latest_path = "" -%} + {%- set target = "" -%} + {%- if metadata.target_name -%} + {%- set inner_path = metadata.target_name ~ "/index.html" -%} + {%- else -%} + {%- set inner_path = "" -%} + {%- endif -%} + {%- set is_latest_version = true -%} + {%- set is_prerelease = false -%} + {%- include "rustdoc/topbar.html" -%} {%- endblock topbar -%} {%- block header -%} {# Set the active tab to the `source` tab #} - {{ navigation::package_navigation(metadata=file_list.metadata, active_tab="source") }} + {{ navigation::package_navigation(metadata=metadata, active_tab="source") }} {%- endblock header -%} {%- block body_classes -%} @@ -46,54 +50,54 @@ {%- for file in file_list.files -%}
  • - {# - Show a link to the file with a fancy icon. If the file is a directory, - `/` is appended to show the contents of the folder - #} + {# + Show a link to the file with a fancy icon. If the file is a directory, + `/` is appended to show the contents of the folder + #} - {# Directories #} + {# Directories #} {%- if file.mime == "dir" -%} {{ "folder-open" | far }} - {# Rust files #} + {# Rust files #} {%- elif file.mime == "text/rust" -%} {{ "rust" | fab }} - {# Cargo.lock #} + {# Cargo.lock #} {%- elif file.mime == "text/plain" and file.name == "Cargo.lock" -%} {{ "lock" | fas }} - {# Markdown files #} + {# Markdown files #} {% elif file.mime == "text/markdown" %} {{ "markdown" | fab }} - {# .gitignore #} + {# .gitignore #} {% elif file.mime == "text/plain" and file.name == ".gitignore" %} {{ "git-alt" | fab }} - {# - More ideas - FontAwesome v5: - ".application/x-bzip" - | "application/gzip" - | "application/x-bzip2" - | "application/vnd.rar" - | "application/x-tar" - | "application/zip" - | "application/x-7z-compressed" => https://fontawesome.com/icons/file-archive - "text/javascript" => https://fontawesome.com/icons/js - "application/java-archive" => https://fontawesome.com/icons/java - DevOpticons (https://github.com/file-icons/DevOpicons): - "text/rust" => https://github.com/file-icons/DevOpicons/blob/master/charmap.md#Rust - "text/css" => https://github.com/file-icons/DevOpicons/blob/master/charmap.md#CSS3,%20Full - "text/html" => https://github.com/file-icons/DevOpicons/blob/master/charmap.md#HTML5 - #} - - {# Text files or files which mime starts with `text` #} + {# + More ideas + FontAwesome v5: + ".application/x-bzip" + | "application/gzip" + | "application/x-bzip2" + | "application/vnd.rar" + | "application/x-tar" + | "application/zip" + | "application/x-7z-compressed" => https://fontawesome.com/icons/file-archive + "text/javascript" => https://fontawesome.com/icons/js + "application/java-archive" => https://fontawesome.com/icons/java + DevOpticons (https://github.com/file-icons/DevOpicons): + "text/rust" => https://github.com/file-icons/DevOpicons/blob/master/charmap.md#Rust + "text/css" => https://github.com/file-icons/DevOpicons/blob/master/charmap.md#CSS3,%20Full + "text/html" => https://github.com/file-icons/DevOpicons/blob/master/charmap.md#HTML5 + #} + + {# Text files or files which mime starts with `text` #} {%- elif file.mime == "text/plain" or file.mime | split(pat="/") | first == "text" -%} {{ "file-alt" | far }} - {# Binary files and any unrecognized types #} + {# Binary files and any unrecognized types #} {% else -%} {{ "file-archive" | far }} {%- endif -%} diff --git a/templates/macros.html b/templates/macros.html index dab2e7d0c..737b7ab6d 100644 --- a/templates/macros.html +++ b/templates/macros.html @@ -96,7 +96,10 @@ {%- for release in releases -%} {# The url for the release, `/crate/:name/:version` #} {# NOTE: `/` is part of target if it exists (to avoid `target-direct//path`) #} - {% if inner_path == "" %} {# /crate #} + {% if release.build_status == "failure" %} + {%- set release_url = "/crate/" ~ name ~ "/" ~ release.version -%} + {%- set retain_fragment = false -%} + {% elif inner_path == "" %} {# /crate #} {%- set release_url = "/crate/" ~ name ~ "/" ~ release.version -%} {%- set retain_fragment = false -%} {% else %} @@ -106,13 +109,13 @@ {# The release's name and version, `:name-:version` #} {%- set release_name = name ~ "-" ~ release.version -%} - {%- if not release.is_library -%} + {%- if release.is_library == false -%} {# If the release isn't a library, then display that warning #} {%- set warning = release_name ~ " is not a library" -%} - {%- elif release.yanked and release.build_status == "success" -%} + {%- elif release.yanked == true and release.build_status == "success" -%} {# If the release has been yanked and successfully built, display a warning #} {%- set warning = release_name ~ " is yanked" -%} - {%- elif release.yanked and release.build_status == "failure" -%} + {%- elif release.yanked == true and release.build_status == "failure" -%} {# If the release has been yanked and failed to build, display a warning #} {%- set warning = release_name ~ " is yanked and docs.rs failed to build it" -%} {%- elif release.build_status == "failure" -%}