diff --git a/.sqlx/query-081663d386dd8dac6fbf3e47ef39fa7238f29192674ced1b3a46329db6cc3b64.json b/.sqlx/query-081663d386dd8dac6fbf3e47ef39fa7238f29192674ced1b3a46329db6cc3b64.json new file mode 100644 index 000000000..c41e1a2e1 --- /dev/null +++ b/.sqlx/query-081663d386dd8dac6fbf3e47ef39fa7238f29192674ced1b3a46329db6cc3b64.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n kw.name as \"name!\",\n kw.slug as \"slug!\"\n FROM keywords as kw\n INNER JOIN keyword_rels as kwr on kw.id = kwr.kid\n WHERE kwr.rid = $1\n ORDER BY kw.name,kw.slug", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slug!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "081663d386dd8dac6fbf3e47ef39fa7238f29192674ced1b3a46329db6cc3b64" +} diff --git a/.sqlx/query-3a46fdff86a84ce2140ad14a5bdc951c1d43e600ef5bc25d23c1fe42f6136869.json b/.sqlx/query-3a46fdff86a84ce2140ad14a5bdc951c1d43e600ef5bc25d23c1fe42f6136869.json new file mode 100644 index 000000000..f33c7c861 --- /dev/null +++ b/.sqlx/query-3a46fdff86a84ce2140ad14a5bdc951c1d43e600ef5bc25d23c1fe42f6136869.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "ALTER SEQUENCE crates_id_seq RESTART WITH 1", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3a46fdff86a84ce2140ad14a5bdc951c1d43e600ef5bc25d23c1fe42f6136869" +} diff --git a/.sqlx/query-5a927fd22f0ad007a48820a7af73747f6a3f03e02097065cbb8958d50366a2b3.json b/.sqlx/query-5a927fd22f0ad007a48820a7af73747f6a3f03e02097065cbb8958d50366a2b3.json new file mode 100644 index 000000000..2c6ed4d89 --- /dev/null +++ b/.sqlx/query-5a927fd22f0ad007a48820a7af73747f6a3f03e02097065cbb8958d50366a2b3.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n kw.name as \"name!\",\n kw.slug as \"slug!\"\n FROM keywords as kw\n INNER JOIN keyword_rels as kwr on kw.id = kwr.kid\n WHERE kwr.rid = $1\n ORDER BY kw.name,kw.slug", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slug!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "5a927fd22f0ad007a48820a7af73747f6a3f03e02097065cbb8958d50366a2b3" +} diff --git a/.sqlx/query-688c28ae99e5511238a6cbe1c53e3abc3924b4ecffc216e1556a53dda936ceae.json b/.sqlx/query-688c28ae99e5511238a6cbe1c53e3abc3924b4ecffc216e1556a53dda936ceae.json new file mode 100644 index 000000000..dcc82bf13 --- /dev/null +++ b/.sqlx/query-688c28ae99e5511238a6cbe1c53e3abc3924b4ecffc216e1556a53dda936ceae.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO blacklisted_crates (crate_name) VALUES ($1);", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "688c28ae99e5511238a6cbe1c53e3abc3924b4ecffc216e1556a53dda936ceae" +} diff --git a/.sqlx/query-85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json b/.sqlx/query-85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json new file mode 100644 index 000000000..24c575ac8 --- /dev/null +++ b/.sqlx/query-85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n name as \"name!\",\n version as \"version!\",\n yanked\n FROM (\n SELECT\n crates.name,\n releases.version,\n releases.yanked\n FROM crates\n INNER JOIN releases ON releases.crate_id = crates.id\n UNION ALL\n -- crates & releases that are already queued\n -- don't have to be requeued.\n SELECT\n queue.name,\n queue.version,\n NULL as yanked\n FROM queue\n LEFT OUTER JOIN crates ON crates.name = queue.name\n LEFT OUTER JOIN releases ON (\n releases.crate_id = crates.id AND\n releases.version = queue.version\n )\n WHERE queue.attempt < $1 AND (\n crates.id IS NULL OR\n releases.id IS NULL\n )\n ) AS inp\n ORDER BY name, version", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "version!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "yanked", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95" +} diff --git a/.sqlx/query-864a659b317f451949bf793899a7ea5b78ebf8a962fbb12b88eb8f1d807fbab6.json b/.sqlx/query-864a659b317f451949bf793899a7ea5b78ebf8a962fbb12b88eb8f1d807fbab6.json new file mode 100644 index 000000000..cc5d0f370 --- /dev/null +++ b/.sqlx/query-864a659b317f451949bf793899a7ea5b78ebf8a962fbb12b88eb8f1d807fbab6.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM blacklisted_crates WHERE crate_name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "864a659b317f451949bf793899a7ea5b78ebf8a962fbb12b88eb8f1d807fbab6" +} diff --git a/.sqlx/query-87cd28092cc0d8646d48687bedd2bbb76530bcc207d2899736b8ca6e564d2a51.json b/.sqlx/query-87cd28092cc0d8646d48687bedd2bbb76530bcc207d2899736b8ca6e564d2a51.json new file mode 100644 index 000000000..c7c7353b1 --- /dev/null +++ b/.sqlx/query-87cd28092cc0d8646d48687bedd2bbb76530bcc207d2899736b8ca6e564d2a51.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE releases SET features = NULL WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "87cd28092cc0d8646d48687bedd2bbb76530bcc207d2899736b8ca6e564d2a51" +} diff --git a/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json b/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json new file mode 100644 index 000000000..4f5d74702 --- /dev/null +++ b/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE releases\n SET files = NULL\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8" +} diff --git a/.sqlx/query-96954280f19a0735b3cc3b567de2201bb4a5f789cb440163b02a4ec411010eef.json b/.sqlx/query-96954280f19a0735b3cc3b567de2201bb4a5f789cb440163b02a4ec411010eef.json new file mode 100644 index 000000000..9fb7618d5 --- /dev/null +++ b/.sqlx/query-96954280f19a0735b3cc3b567de2201bb4a5f789cb440163b02a4ec411010eef.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as \"count!\" FROM blacklisted_crates WHERE crate_name = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "96954280f19a0735b3cc3b567de2201bb4a5f789cb440163b02a4ec411010eef" +} diff --git a/.sqlx/query-abf12aa0c7f3d2d626c212f5e3d4903b6a9745d315b2b71c298934ac42649362.json b/.sqlx/query-abf12aa0c7f3d2d626c212f5e3d4903b6a9745d315b2b71c298934ac42649362.json new file mode 100644 index 000000000..7f51ff4d8 --- /dev/null +++ b/.sqlx/query-abf12aa0c7f3d2d626c212f5e3d4903b6a9745d315b2b71c298934ac42649362.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n releases.features \"features?: Vec\"\n FROM releases\n INNER JOIN crates ON crates.id = releases.crate_id\n WHERE crates.name = $1 AND releases.version = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "features?: Vec", + "type_info": { + "Custom": { + "name": "feature[]", + "kind": { + "Array": { + "Custom": { + "name": "feature", + "kind": { + "Composite": [ + [ + "name", + "Text" + ], + [ + "subfeatures", + "TextArray" + ] + ] + } + } + } + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "abf12aa0c7f3d2d626c212f5e3d4903b6a9745d315b2b71c298934ac42649362" +} diff --git a/.sqlx/query-b4ebc9f3e0770457003c321070f8301fc7b92dfda3fea892f1957eda0c3ae018.json b/.sqlx/query-b4ebc9f3e0770457003c321070f8301fc7b92dfda3fea892f1957eda0c3ae018.json new file mode 100644 index 000000000..b574cffd1 --- /dev/null +++ b/.sqlx/query-b4ebc9f3e0770457003c321070f8301fc7b92dfda3fea892f1957eda0c3ae018.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n r.rustdoc_status,\n r.is_library\n FROM\n crates as c\n INNER JOIN releases AS r ON c.id = r.crate_id\n LEFT OUTER JOIN doc_coverage AS cov ON r.id = cov.release_id\n WHERE\n c.name = $1 AND\n r.version = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "rustdoc_status", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "is_library", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "b4ebc9f3e0770457003c321070f8301fc7b92dfda3fea892f1957eda0c3ae018" +} diff --git a/.sqlx/query-c87fb1f05c8d726ab1211cf3a5d4e43ce08ac13c468ef4d90c28ab5fa8ec6ac7.json b/.sqlx/query-c87fb1f05c8d726ab1211cf3a5d4e43ce08ac13c468ef4d90c28ab5fa8ec6ac7.json new file mode 100644 index 000000000..751dfa4f0 --- /dev/null +++ b/.sqlx/query-c87fb1f05c8d726ab1211cf3a5d4e43ce08ac13c468ef4d90c28ab5fa8ec6ac7.json @@ -0,0 +1,71 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n r.rustdoc_status,\n r.default_target,\n r.doc_targets,\n r.archive_storage,\n cov.total_items,\n b.id as build_id,\n b.build_status::TEXT as build_status,\n b.docsrs_version,\n b.rustc_version\n FROM\n crates as c\n INNER JOIN releases AS r ON c.id = r.crate_id\n INNER JOIN builds as b ON r.id = b.rid\n LEFT OUTER JOIN doc_coverage AS cov ON r.id = cov.release_id\n WHERE\n c.name = $1 AND\n r.version = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "rustdoc_status", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "default_target", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "doc_targets", + "type_info": "Json" + }, + { + "ordinal": 3, + "name": "archive_storage", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "total_items", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "build_id", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "build_status", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "docsrs_version", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "rustc_version", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + true, + true, + false, + true, + false, + null, + true, + true + ] + }, + "hash": "c87fb1f05c8d726ab1211cf3a5d4e43ce08ac13c468ef4d90c28ab5fa8ec6ac7" +} diff --git a/.sqlx/query-ec35a33dd4ea3758a2f82ac5a441e60ca47f46dde782fce4fdcb42789cf2fd16.json b/.sqlx/query-ec35a33dd4ea3758a2f82ac5a441e60ca47f46dde782fce4fdcb42789cf2fd16.json new file mode 100644 index 000000000..ded21e53a --- /dev/null +++ b/.sqlx/query-ec35a33dd4ea3758a2f82ac5a441e60ca47f46dde782fce4fdcb42789cf2fd16.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE releases SET dependencies = dependencies::jsonb #- '{0,2}' WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "ec35a33dd4ea3758a2f82ac5a441e60ca47f46dde782fce4fdcb42789cf2fd16" +} diff --git a/.sqlx/query-f9e5a70e03175f8bbf44552e4def1aae33c0f1aba515ac53d1ddaf922bb1f9d9.json b/.sqlx/query-f9e5a70e03175f8bbf44552e4def1aae33c0f1aba515ac53d1ddaf922bb1f9d9.json new file mode 100644 index 000000000..fc50ff412 --- /dev/null +++ b/.sqlx/query-f9e5a70e03175f8bbf44552e4def1aae33c0f1aba515ac53d1ddaf922bb1f9d9.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT slug FROM keywords ORDER BY slug", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "slug", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "f9e5a70e03175f8bbf44552e4def1aae33c0f1aba515ac53d1ddaf922bb1f9d9" +} diff --git a/.sqlx/query-fb9dd5bf6d16a814c0ea07dd640a7dd6c020d7e01878567acdfa014f4f103b74.json b/.sqlx/query-fb9dd5bf6d16a814c0ea07dd640a7dd6c020d7e01878567acdfa014f4f103b74.json new file mode 100644 index 000000000..2c9ba39fc --- /dev/null +++ b/.sqlx/query-fb9dd5bf6d16a814c0ea07dd640a7dd6c020d7e01878567acdfa014f4f103b74.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT crate_name FROM blacklisted_crates ORDER BY crate_name asc;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "crate_name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "fb9dd5bf6d16a814c0ea07dd640a7dd6c020d7e01878567acdfa014f4f103b74" +} diff --git a/Cargo.lock b/Cargo.lock index f76df32ea..76fa0da28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1694,13 +1694,9 @@ dependencies = [ "once_cell", "path-slash", "percent-encoding", - "postgres", - "postgres-types", "pretty_assertions", "procfs", "prometheus", - "r2d2", - "r2d2_postgres", "rand 0.8.5", "rayon", "regex", @@ -1885,12 +1881,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - [[package]] name = "fallible-iterator" version = "0.3.0" @@ -4875,65 +4865,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "postgres" -version = "0.19.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c9ec84ab55b0f9e418675de50052d494ba893fd28c65769a6e68fcdacbee2b8" -dependencies = [ - "bytes", - "fallible-iterator 0.2.0", - "futures-util", - "log", - "tokio", - "tokio-postgres", -] - -[[package]] -name = "postgres-derive" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83145eba741b050ef981a9a1838c843fa7665e154383325aa8b440ae703180a2" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 2.0.77", -] - -[[package]] -name = "postgres-protocol" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acda0ebdebc28befa84bee35e651e4c5f09073d668c7aed4cf7e23c3cda84b23" -dependencies = [ - "base64 0.22.1", - "byteorder", - "bytes", - "fallible-iterator 0.2.0", - "hmac", - "md-5", - "memchr", - "rand 0.8.5", - "sha2", - "stringprep", -] - -[[package]] -name = "postgres-types" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02048d9e032fb3cc3413bbf7b83a15d84a5d419778e2628751896d856498eee9" -dependencies = [ - "bytes", - "chrono", - "fallible-iterator 0.2.0", - "postgres-derive", - "postgres-protocol", - "serde", - "serde_json", -] - [[package]] name = "powerfmt" version = "0.2.0" @@ -5048,27 +4979,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r2d2" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" -dependencies = [ - "log", - "parking_lot", - "scheduled-thread-pool", -] - -[[package]] -name = "r2d2_postgres" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7029c56be658cb54f321e0bee597810ee16796b735fa2559d7056bf06b12230b" -dependencies = [ - "postgres", - "r2d2", -] - [[package]] name = "rand" version = "0.7.3" @@ -5394,7 +5304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ "bitflags 2.6.0", - "fallible-iterator 0.3.0", + "fallible-iterator", "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", @@ -5595,15 +5505,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "scheduled-thread-pool" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" -dependencies = [ - "parking_lot", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -6698,32 +6599,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-postgres" -version = "0.7.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03adcf0147e203b6032c0b2d30be1415ba03bc348901f3ff1cc0df6a733e60c3" -dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator 0.2.0", - "futures-channel", - "futures-util", - "log", - "parking_lot", - "percent-encoding", - "phf 0.11.2", - "pin-project-lite", - "postgres-protocol", - "postgres-types", - "rand 0.8.5", - "socket2", - "tokio", - "tokio-util", - "whoami", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -7230,7 +7105,6 @@ checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ "redox_syscall", "wasite", - "web-sys", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 04ed458cc..64177f9f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,8 +35,6 @@ crates-index-diff = { version = "25.0.0", features = [ "max-performance" ]} reqwest = { version = "0.12", features = ["json", "gzip"] } semver = { version = "1.0.4", features = ["serde"] } slug = "0.1.1" -r2d2 = "0.8" -r2d2_postgres = "0.18" sqlx = { version = "0.8", features = [ "runtime-tokio", "postgres", "chrono" ] } url = { version = "2.1.1", features = ["serde"] } docsrs-metadata = { path = "crates/metadata" } @@ -59,7 +57,6 @@ lol_html = "1.0.0" font-awesome-as-a-crate = { path = "crates/font-awesome-as-a-crate" } dashmap = "6.0.0" string_cache = "0.8.0" -postgres-types = { version = "0.2", features = ["derive"] } zip = {version = "2.2.0", default-features = false, features = ["bzip2"]} bzip2 = "0.4.4" getrandom = "0.2.1" @@ -108,10 +105,6 @@ thread_local = "1.1.3" humantime = "2.1.0" constant_time_eq = "0.3.0" -[dependencies.postgres] -version = "0.19" -features = ["with-chrono-0_4", "with-serde_json-1"] - [target.'cfg(target_os = "linux")'.dependencies] # Process information procfs = "0.15.1" diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index dde6360db..1e36e2dd1 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -9,7 +9,7 @@ use anyhow::{anyhow, Context as _, Error, Result}; use axum::async_trait; use clap::{Parser, Subcommand, ValueEnum}; use docs_rs::cdn::CdnBackend; -use docs_rs::db::{self, add_path_into_database, Overrides, Pool, PoolClient}; +use docs_rs::db::{self, add_path_into_database, Overrides, Pool}; use docs_rs::repositories::RepositoryStatsUpdater; use docs_rs::utils::{ get_config, get_crate_pattern_and_priority, list_crate_priorities, queue_builder, @@ -771,22 +771,27 @@ enum BlacklistSubcommand { impl BlacklistSubcommand { fn handle_args(self, ctx: BinContext) -> Result<()> { - let conn = &mut *ctx.conn()?; - match self { - Self::List => { - let crates = db::blacklist::list_crates(conn) - .context("failed to list crates on blacklist")?; + ctx.runtime()?.block_on(async { + let conn = &mut *ctx.pool()?.get_async().await?; + match self { + Self::List => { + let crates = db::blacklist::list_crates(conn) + .await + .context("failed to list crates on blacklist")?; - println!("{}", crates.join("\n")); - } + println!("{}", crates.join("\n")); + } - Self::Add { crate_name } => db::blacklist::add_crate(conn, &crate_name) - .context("failed to add crate to blacklist")?, + Self::Add { crate_name } => db::blacklist::add_crate(conn, &crate_name) + .await + .context("failed to add crate to blacklist")?, - Self::Remove { crate_name } => db::blacklist::remove_crate(conn, &crate_name) - .context("failed to remove crate from blacklist")?, - } - Ok(()) + Self::Remove { crate_name } => db::blacklist::remove_crate(conn, &crate_name) + .await + .context("failed to remove crate from blacklist")?, + } + Ok(()) + }) } } @@ -840,10 +845,6 @@ impl BinContext { runtime: OnceCell::new(), } } - - fn conn(&self) -> Result { - Ok(self.pool()?.get()?) - } } macro_rules! lazy { diff --git a/src/config.rs b/src/config.rs index 5a257534f..32bbdbef4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -13,7 +13,6 @@ pub struct Config { // Database connection params pub(crate) database_url: String, - pub(crate) max_legacy_pool_size: u32, pub(crate) max_pool_size: u32, pub(crate) min_pool_idle: u32, @@ -153,8 +152,7 @@ impl Config { prefix: prefix.clone(), database_url: require_env("DOCSRS_DATABASE_URL")?, - max_legacy_pool_size: env("DOCSRS_MAX_LEGACY_POOL_SIZE", 45)?, - max_pool_size: env("DOCSRS_MAX_POOL_SIZE", 45)?, + max_pool_size: env("DOCSRS_MAX_POOL_SIZE", 90)?, min_pool_idle: env("DOCSRS_MIN_POOL_IDLE", 10)?, storage_backend: env("DOCSRS_STORAGE_BACKEND", StorageKind::Database)?, diff --git a/src/db/add_package.rs b/src/db/add_package.rs index b4bc05db9..027691d8c 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -722,36 +722,42 @@ mod test { #[test] fn new_keywords() { - wrapper(|env| { - let mut conn = env.db().conn(); + async_wrapper(|env| async move { + let mut conn = env.async_db().await.async_conn().await; let release_id = env - .fake_release() + .async_fake_release() + .await .name("dummy") .version("0.13.0") .keywords(vec!["kw 1".into(), "kw 2".into()]) - .create()?; + .create_async() + .await?; - let kw_r = conn - .query( - "SELECT kw.name,kw.slug - FROM keywords as kw - INNER JOIN keyword_rels as kwr on kw.id = kwr.kid - WHERE kwr.rid = $1 - ORDER BY kw.name,kw.slug", - &[&release_id], - )? - .into_iter() - .map(|row| (row.get::<_, String>(0), row.get::<_, String>(1))) - .collect::>(); + let kw_r = sqlx::query!( + r#"SELECT + kw.name as "name!", + kw.slug as "slug!" + FROM keywords as kw + INNER JOIN keyword_rels as kwr on kw.id = kwr.kid + WHERE kwr.rid = $1 + ORDER BY kw.name,kw.slug"#, + release_id + ) + .fetch_all(&mut *conn) + .await? + .into_iter() + .map(|row| (row.name, row.slug)) + .collect::>(); assert_eq!(kw_r[0], ("kw 1".into(), "kw-1".into())); assert_eq!(kw_r[1], ("kw 2".into(), "kw-2".into())); - let all_kw = conn - .query("SELECT slug FROM keywords ORDER BY slug", &[])? + let all_kw = sqlx::query!("SELECT slug FROM keywords ORDER BY slug") + .fetch_all(&mut *conn) + .await? .into_iter() - .map(|row| row.get::<_, String>(0)) + .map(|row| row.slug) .collect::>(); assert_eq!(all_kw, vec![String::from("kw-1"), "kw-2".into()]); @@ -782,41 +788,49 @@ mod test { #[test] fn updated_keywords() { - wrapper(|env| { - env.fake_release() + async_wrapper(|env| async move { + env.async_fake_release() + .await .name("dummy") .version("0.13.0") .keywords(vec!["kw 3".into(), "kw 4".into()]) - .create()?; + .create_async() + .await?; let release_id = env - .fake_release() + .async_fake_release() + .await .name("dummy") .version("0.13.0") .keywords(vec!["kw 1".into(), "kw 2".into()]) - .create()?; + .create_async() + .await?; - let mut conn = env.db().conn(); - let kw_r = conn - .query( - "SELECT kw.name,kw.slug - FROM keywords as kw - INNER JOIN keyword_rels as kwr on kw.id = kwr.kid - WHERE kwr.rid = $1 - ORDER BY kw.name,kw.slug", - &[&release_id], - )? - .into_iter() - .map(|row| (row.get::<_, String>(0), row.get::<_, String>(1))) - .collect::>(); + let mut conn = env.async_db().await.async_conn().await; + let kw_r = sqlx::query!( + r#"SELECT + kw.name as "name!", + kw.slug as "slug!" + FROM keywords as kw + INNER JOIN keyword_rels as kwr on kw.id = kwr.kid + WHERE kwr.rid = $1 + ORDER BY kw.name,kw.slug"#, + release_id + ) + .fetch_all(&mut *conn) + .await? + .into_iter() + .map(|row| (row.name, row.slug)) + .collect::>(); assert_eq!(kw_r[0], ("kw 1".into(), "kw-1".into())); assert_eq!(kw_r[1], ("kw 2".into(), "kw-2".into())); - let all_kw = conn - .query("SELECT slug FROM keywords ORDER BY slug", &[])? + let all_kw = sqlx::query!("SELECT slug FROM keywords ORDER BY slug") + .fetch_all(&mut *conn) + .await? .into_iter() - .map(|row| row.get::<_, String>(0)) + .map(|row| row.slug) .collect::>(); assert_eq!( diff --git a/src/db/blacklist.rs b/src/db/blacklist.rs index 37c713da9..f4e104521 100644 --- a/src/db/blacklist.rs +++ b/src/db/blacklist.rs @@ -1,5 +1,5 @@ use crate::error::Result; -use postgres::Client; +use futures_util::stream::TryStreamExt; #[derive(Debug, thiserror::Error)] enum BlacklistError { @@ -11,50 +11,55 @@ enum BlacklistError { } /// Returns whether the given name is blacklisted. -pub fn is_blacklisted(conn: &mut Client, name: &str) -> Result { - let rows = conn.query( - "SELECT COUNT(*) FROM blacklisted_crates WHERE crate_name = $1;", - &[&name], - )?; - let count: i64 = rows[0].get(0); - - Ok(count != 0) +pub async fn is_blacklisted(conn: &mut sqlx::PgConnection, name: &str) -> Result { + Ok(sqlx::query_scalar!( + r#"SELECT COUNT(*) as "count!" FROM blacklisted_crates WHERE crate_name = $1;"#, + name + ) + .fetch_one(conn) + .await? + != 0) } /// Returns the crate names on the blacklist, sorted ascending. -pub fn list_crates(conn: &mut Client) -> Result> { - let rows = conn.query( - "SELECT crate_name FROM blacklisted_crates ORDER BY crate_name asc;", - &[], - )?; - - Ok(rows.into_iter().map(|row| row.get(0)).collect()) +pub async fn list_crates(conn: &mut sqlx::PgConnection) -> Result> { + Ok( + sqlx::query!("SELECT crate_name FROM blacklisted_crates ORDER BY crate_name asc;") + .fetch(conn) + .map_ok(|row| row.crate_name) + .try_collect() + .await?, + ) } /// Adds a crate to the blacklist. -pub fn add_crate(conn: &mut Client, name: &str) -> Result<()> { - if is_blacklisted(conn, name)? { +pub async fn add_crate(conn: &mut sqlx::PgConnection, name: &str) -> Result<()> { + if is_blacklisted(&mut *conn, name).await? { return Err(BlacklistError::CrateAlreadyOnBlacklist(name.into()).into()); } - conn.execute( + sqlx::query!( "INSERT INTO blacklisted_crates (crate_name) VALUES ($1);", - &[&name], - )?; + name + ) + .execute(conn) + .await?; Ok(()) } /// Removes a crate from the blacklist. -pub fn remove_crate(conn: &mut Client, name: &str) -> Result<()> { - if !is_blacklisted(conn, name)? { +pub async fn remove_crate(conn: &mut sqlx::PgConnection, name: &str) -> Result<()> { + if !is_blacklisted(conn, name).await? { return Err(BlacklistError::CrateNotOnBlacklist(name.into()).into()); } - conn.execute( + sqlx::query!( "DELETE FROM blacklisted_crates WHERE crate_name = $1;", - &[&name], - )?; + name + ) + .execute(conn) + .await?; Ok(()) } @@ -65,41 +70,41 @@ mod tests { #[test] fn test_list_blacklist() { - crate::test::wrapper(|env| { - let db = env.db(); + crate::test::async_wrapper(|env| async move { + let mut conn = env.async_db().await.async_conn().await; // crates are added out of order to verify sorting - add_crate(&mut db.conn(), "crate A")?; - add_crate(&mut db.conn(), "crate C")?; - add_crate(&mut db.conn(), "crate B")?; + add_crate(&mut conn, "crate A").await?; + add_crate(&mut conn, "crate C").await?; + add_crate(&mut conn, "crate B").await?; - assert!(list_crates(&mut db.conn())? == vec!["crate A", "crate B", "crate C"]); + assert!(list_crates(&mut conn).await? == vec!["crate A", "crate B", "crate C"]); Ok(()) }); } #[test] fn test_add_to_and_remove_from_blacklist() { - crate::test::wrapper(|env| { - let db = env.db(); - - assert!(!is_blacklisted(&mut db.conn(), "crate foo")?); - add_crate(&mut db.conn(), "crate foo")?; - assert!(is_blacklisted(&mut db.conn(), "crate foo")?); - remove_crate(&mut db.conn(), "crate foo")?; - assert!(!is_blacklisted(&mut db.conn(), "crate foo")?); + crate::test::async_wrapper(|env| async move { + let mut conn = env.async_db().await.async_conn().await; + + assert!(!is_blacklisted(&mut conn, "crate foo").await?); + add_crate(&mut conn, "crate foo").await?; + assert!(is_blacklisted(&mut conn, "crate foo").await?); + remove_crate(&mut conn, "crate foo").await?; + assert!(!is_blacklisted(&mut conn, "crate foo").await?); Ok(()) }); } #[test] fn test_add_twice_to_blacklist() { - crate::test::wrapper(|env| { - let db = env.db(); + crate::test::async_wrapper(|env| async move { + let mut conn = env.async_db().await.async_conn().await; - add_crate(&mut db.conn(), "crate foo")?; - assert!(add_crate(&mut db.conn(), "crate foo").is_err()); - add_crate(&mut db.conn(), "crate bar")?; + add_crate(&mut conn, "crate foo").await?; + assert!(add_crate(&mut conn, "crate foo").await.is_err()); + add_crate(&mut conn, "crate bar").await?; Ok(()) }); @@ -107,10 +112,10 @@ mod tests { #[test] fn test_remove_non_existing_crate() { - crate::test::wrapper(|env| { - let db = env.db(); + crate::test::async_wrapper(|env| async move { + let mut conn = env.async_db().await.async_conn().await; - assert!(remove_crate(&mut db.conn(), "crate foo").is_err()); + assert!(remove_crate(&mut conn, "crate foo").await.is_err()); Ok(()) }); diff --git a/src/db/mod.rs b/src/db/mod.rs index 822f98fe0..c4b92ef19 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -12,7 +12,7 @@ pub use self::{ delete::{delete_crate, delete_version}, file::{add_path_into_database, add_path_into_remote_archive}, overrides::Overrides, - pool::{AsyncPoolClient, Pool, PoolClient, PoolError}, + pool::{AsyncPoolClient, Pool, PoolError}, }; mod add_package; diff --git a/src/db/pool.rs b/src/db/pool.rs index dc3c8c01c..32c71a63e 100644 --- a/src/db/pool.rs +++ b/src/db/pool.rs @@ -1,8 +1,6 @@ use crate::metrics::InstanceMetrics; use crate::Config; use futures_util::{future::BoxFuture, stream::BoxStream}; -use postgres::{Client, NoTls}; -use r2d2_postgres::PostgresConnectionManager; use sqlx::{postgres::PgPoolOptions, Executor}; use std::{ ops::{Deref, DerefMut}, @@ -12,16 +10,10 @@ use std::{ use tokio::runtime::Runtime; use tracing::debug; -pub type PoolClient = r2d2::PooledConnection>; - const DEFAULT_SCHEMA: &str = "public"; #[derive(Debug, Clone)] pub struct Pool { - #[cfg(test)] - pool: Arc>>>>, - #[cfg(not(test))] - pool: r2d2::Pool>, async_pool: sqlx::PgPool, runtime: Arc, metrics: Arc, @@ -56,26 +48,10 @@ impl Pool { metrics: Arc, schema: &str, ) -> Result { - let url = config - .database_url - .parse() - .map_err(PoolError::InvalidDatabaseUrl)?; - let acquire_timeout = Duration::from_secs(30); let max_lifetime = Duration::from_secs(30 * 60); let idle_timeout = Duration::from_secs(10 * 60); - let manager = PostgresConnectionManager::new(url, NoTls); - let pool = r2d2::Pool::builder() - .max_size(config.max_legacy_pool_size) - .min_idle(Some(config.min_pool_idle)) - .max_lifetime(Some(max_lifetime)) - .connection_timeout(acquire_timeout) - .idle_timeout(Some(idle_timeout)) - .connection_customizer(Box::new(SetSchema::new(schema))) - .build(manager) - .map_err(PoolError::PoolCreationFailed)?; - let _guard = runtime.enter(); let async_pool = PgPoolOptions::new() .max_connections(config.max_pool_size) @@ -107,41 +83,13 @@ impl Pool { .map_err(PoolError::AsyncPoolCreationFailed)?; Ok(Pool { - #[cfg(test)] - pool: Arc::new(std::sync::Mutex::new(Some(pool))), - #[cfg(not(test))] - pool, async_pool, metrics, runtime, - max_size: config.max_legacy_pool_size + config.max_pool_size, + max_size: config.max_pool_size, }) } - fn with_pool( - &self, - f: impl FnOnce(&r2d2::Pool>) -> R, - ) -> R { - #[cfg(test)] - { - f(self.pool.lock().unwrap().as_ref().unwrap()) - } - #[cfg(not(test))] - { - f(&self.pool) - } - } - - pub fn get(&self) -> Result { - match self.with_pool(|p| p.get()) { - Ok(conn) => Ok(conn), - Err(err) => { - self.metrics.failed_db_connections.inc(); - Err(PoolError::ClientError(err)) - } - } - } - pub async fn get_async(&self) -> Result { match self.async_pool.acquire().await { Ok(conn) => Ok(AsyncPoolClient { @@ -156,22 +104,16 @@ impl Pool { } pub(crate) fn used_connections(&self) -> u32 { - self.with_pool(|p| p.state().connections - p.state().idle_connections) - + (self.async_pool.size() - self.async_pool.num_idle() as u32) + self.async_pool.size() - self.async_pool.num_idle() as u32 } pub(crate) fn idle_connections(&self) -> u32 { - self.with_pool(|p| p.state().idle_connections) + self.async_pool.num_idle() as u32 + self.async_pool.num_idle() as u32 } pub(crate) fn max_size(&self) -> u32 { self.max_size } - - #[cfg(test)] - pub(crate) fn shutdown(&self) { - self.pool.lock().unwrap().take(); - } } /// This impl allows us to use our own pool as an executor for SQLx queries. @@ -257,45 +199,11 @@ impl Drop for AsyncPoolClient { } } -#[derive(Debug)] -struct SetSchema { - schema: String, -} - -impl SetSchema { - fn new(schema: &str) -> Self { - Self { - schema: schema.into(), - } - } -} - -impl r2d2::CustomizeConnection for SetSchema { - fn on_acquire(&self, conn: &mut Client) -> Result<(), postgres::Error> { - if self.schema != DEFAULT_SCHEMA { - conn.execute( - format!("SET search_path TO {}, {};", self.schema, DEFAULT_SCHEMA).as_str(), - &[], - )?; - } - Ok(()) - } -} - #[derive(Debug, thiserror::Error)] pub enum PoolError { - #[error("the provided database URL was not valid")] - InvalidDatabaseUrl(#[from] postgres::Error), - - #[error("failed to create the database connection pool")] - PoolCreationFailed(#[source] r2d2::Error), - #[error("failed to create the database connection pool")] AsyncPoolCreationFailed(#[source] sqlx::Error), - #[error("failed to get a database connection")] - ClientError(#[source] r2d2::Error), - #[error("failed to get a database connection")] AsyncClientError(#[source] sqlx::Error), } diff --git a/src/db/types.rs b/src/db/types.rs index 608b2d3ba..59d59cfb0 100644 --- a/src/db/types.rs +++ b/src/db/types.rs @@ -1,8 +1,7 @@ -use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, PartialEq, Eq, Serialize, FromSql, ToSql, sqlx::Type)] -#[postgres(name = "feature")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, sqlx::Type)] +#[sqlx(type_name = "feature")] pub struct Feature { pub(crate) name: String, pub(crate) subfeatures: Vec, diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index cb4360605..0b3fdd79f 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -394,10 +394,15 @@ impl RustwideBuilder { kind: PackageKind<'_>, build_id: i32, ) -> Result { - let mut conn = self.db.get()?; info!("building package {} {}", name, version); - if is_blacklisted(&mut conn, name)? { + let is_blacklisted = self.runtime.block_on(async { + let mut conn = self.db.get_async().await?; + + is_blacklisted(&mut conn, name).await + })?; + + if is_blacklisted { info!("skipping build of {}, crate has been blacklisted", name); return Ok(false); } @@ -1000,8 +1005,31 @@ impl Default for BuildPackageSummary { #[cfg(test)] mod tests { use super::*; - use crate::test::{assert_redirect, assert_success, wrapper, TestEnvironment}; - use serde_json::Value; + use crate::{ + db::types::Feature, + test::{assert_redirect, assert_success, wrapper, TestEnvironment}, + }; + + fn get_features( + env: &TestEnvironment, + name: &str, + version: &str, + ) -> Result>, sqlx::Error> { + env.runtime().block_on(async { + let mut conn = env.async_db().await.async_conn().await; + sqlx::query_scalar!( + r#"SELECT + releases.features "features?: Vec" + FROM releases + INNER JOIN crates ON crates.id = releases.crate_id + WHERE crates.name = $1 AND releases.version = $2"#, + name, + version, + ) + .fetch_one(&mut *conn) + .await + }) + } fn remove_cache_files(env: &TestEnvironment, crate_: &str, version: &str) -> Result<()> { let paths = [ @@ -1057,9 +1085,9 @@ mod tests { ); // check release record in the db (default and other targets) - let mut conn = env.db().conn(); - let row = conn - .query_one( + let row = env.runtime().block_on(async { + let mut conn = env.async_db().await.async_conn().await; + sqlx::query!( r#"SELECT r.rustdoc_status, r.default_target, @@ -1078,20 +1106,24 @@ mod tests { WHERE c.name = $1 AND r.version = $2"#, - &[&crate_, &version], + crate_, + version, ) - .unwrap(); + .fetch_one(&mut *conn) + .await + })?; - assert!(row.get::<_, bool>("rustdoc_status")); - assert_eq!(row.get::<_, String>("default_target"), default_target); - assert!(row.get::<_, Option>("total_items").is_some()); - assert!(row.get::<_, bool>("archive_storage")); - assert!(!row.get::<_, String>("docsrs_version").is_empty()); - assert!(!row.get::<_, String>("rustc_version").is_empty()); - assert_eq!(row.get::<_, String>("build_status"), "success"); + assert_eq!(row.rustdoc_status, Some(true)); + assert_eq!(row.default_target, Some(default_target.into())); + assert!(row.total_items.is_some()); + assert!(row.archive_storage); + assert!(!row.docsrs_version.unwrap().is_empty()); + assert!(!row.rustc_version.unwrap().is_empty()); + assert_eq!(row.build_status.unwrap(), "success"); let mut targets: Vec = row - .get::<_, Value>("doc_targets") + .doc_targets + .unwrap() .as_array() .unwrap() .iter() @@ -1172,10 +1204,7 @@ mod tests { assert_success(&target_url, web)?; assert!(storage - .exists(&format!( - "build-logs/{}/{target}.txt", - row.get::<_, i32>("build_id") - )) + .exists(&format!("build-logs/{}/{target}.txt", row.build_id)) .unwrap()); } } @@ -1207,9 +1236,9 @@ mod tests { ); // check release record in the db (default and other targets) - let mut conn = env.db().conn(); - let rows = conn - .query( + let row = env.runtime().block_on(async { + let mut conn = env.async_db().await.async_conn().await; + sqlx::query!( "SELECT r.rustdoc_status, r.is_library @@ -1220,13 +1249,15 @@ mod tests { WHERE c.name = $1 AND r.version = $2", - &[&crate_, &version], + crate_, + version ) - .unwrap(); - let row = rows.first().unwrap(); + .fetch_one(&mut *conn) + .await + })?; - assert!(!row.get::<_, bool>("rustdoc_status")); - assert!(!row.get::<_, bool>("is_library")); + assert_eq!(row.rustdoc_status, Some(false)); + assert_eq!(row.is_library, Some(false)); // doc archive exists let doc_archive = rustdoc_archive_path(crate_, version); @@ -1483,18 +1514,10 @@ mod tests { .successful ); - let mut conn = env.db().conn(); - let features: Vec = conn - .query_opt( - "SELECT releases.features FROM releases - INNER JOIN crates ON crates.id = releases.crate_id - WHERE crates.name = $1 AND releases.version = $2", - &[&crate_, &version], - )? - .context("missing release")? - .get(0); - - assert!(features.iter().any(|f| f.name == "serde_derive")); + assert!(get_features(env, crate_, version)? + .unwrap() + .iter() + .any(|f| f.name == "serde_derive")); Ok(()) }); @@ -1514,18 +1537,10 @@ mod tests { .successful ); - let mut conn = env.db().conn(); - let features: Vec = conn - .query_opt( - "SELECT releases.features FROM releases - INNER JOIN crates ON crates.id = releases.crate_id - WHERE crates.name = $1 AND releases.version = $2", - &[&crate_, &version], - )? - .context("missing release")? - .get(0); - - assert!(!features.iter().any(|f| f.name == "with_builtin_macros")); + assert!(!get_features(env, crate_, version)? + .unwrap() + .iter() + .any(|f| f.name == "with_builtin_macros")); Ok(()) }); diff --git a/src/test/mod.rs b/src/test/mod.rs index 23b530a8b..4f06a8dd6 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -2,7 +2,7 @@ mod fakes; pub(crate) use self::fakes::{fake_release_that_failed_before_build, FakeBuild}; use crate::cdn::CdnBackend; -use crate::db::{self, AsyncPoolClient, Pool, PoolClient}; +use crate::db::{self, AsyncPoolClient, Pool}; use crate::error::Result; use crate::repositories::RepositoryStatsUpdater; use crate::storage::{AsyncStorage, Storage, StorageKind}; @@ -333,7 +333,6 @@ impl TestEnvironment { // Use less connections for each test compared to production. config.max_pool_size = 4; - config.max_legacy_pool_size = 4; config.min_pool_idle = 0; // Use the database for storage, as it's faster than S3. @@ -658,31 +657,23 @@ impl TestDatabase { .await .expect("failed to get a connection out of the pool") } - - pub(crate) fn conn(&self) -> PoolClient { - self.pool - .get() - .expect("failed to get a connection out of the pool") - } } impl Drop for TestDatabase { fn drop(&mut self) { - let migration_result = self.runtime.block_on(async { + self.runtime.block_on(async { let mut conn = self.async_conn().await; - db::migrate(&mut conn, Some(0)).await - }); + let migration_result = db::migrate(&mut conn, Some(0)).await; - if let Err(e) = self.conn().execute( - format!("DROP SCHEMA {} CASCADE;", self.schema).as_str(), - &[], - ) { - error!("failed to drop test schema {}: {}", self.schema, e); - } - // Drop the connection pool so we don't leak database connections - self.pool.shutdown(); + if let Err(e) = sqlx::query(format!("DROP SCHEMA {} CASCADE;", self.schema).as_str()) + .execute(&mut *conn) + .await + { + error!("failed to drop test schema {}: {}", self.schema, e); + } - migration_result.expect("downgrading database works"); + migration_result.expect("downgrading database works"); + }); } } diff --git a/src/utils/consistency/db.rs b/src/utils/consistency/db.rs index 746ab4c7c..dc60c81e4 100644 --- a/src/utils/consistency/db.rs +++ b/src/utils/consistency/db.rs @@ -2,12 +2,13 @@ use super::data::{Crate, Crates, Release, Releases}; use crate::Config; use anyhow::Result; use itertools::Itertools; -use postgres::fallible_iterator::FallibleIterator; -use std::iter; -pub(super) fn load(conn: &mut postgres::Client, config: &Config) -> Result { - let rows = conn.query_raw( - "SELECT name, version, yanked +pub(super) async fn load(conn: &mut sqlx::PgConnection, config: &Config) -> Result { + let rows = sqlx::query!( + r#"SELECT + name as "name!", + version as "version!", + yanked FROM ( SELECT crates.name, @@ -18,7 +19,10 @@ pub(super) fn load(conn: &mut postgres::Client, config: &Config) -> Result Result Result<()> { - let mut conn = ctx.pool()?.get()?; let index = ctx.index()?; info!("Loading data from database..."); - let db_data = db::load(&mut conn, &*ctx.config()?) + let db_data = ctx + .runtime()? + .block_on(async { + let mut conn = ctx.pool()?.get_async().await?; + db::load(&mut conn, &*ctx.config()?).await + }) .context("Loading crate data from database for consistency check")?; tracing::info!("Loading data from index..."); @@ -148,27 +152,33 @@ where #[cfg(test)] mod tests { - use postgres_types::FromSql; - use super::diff::Difference; use super::*; use crate::test::{wrapper, TestEnvironment}; + use sqlx::Row as _; fn count(env: &TestEnvironment, sql: &str) -> Result { - Ok(env.db().conn().query_one(sql, &[])?.get::<_, i64>(0)) + Ok(env.runtime().block_on(async { + let mut conn = env.async_db().await.async_conn().await; + sqlx::query_scalar(sql).fetch_one(&mut *conn).await + })?) } - fn single_row(env: &TestEnvironment, sql: &str) -> Result> + fn single_row(env: &TestEnvironment, sql: &str) -> Result> where - T: for<'a> FromSql<'a>, + O: Send + Unpin + for<'r> sqlx::Decode<'r, sqlx::Postgres> + sqlx::Type, { - Ok(env - .db() - .conn() - .query(sql, &[])? - .iter() - .map(|row| row.get::<_, T>(0)) - .collect()) + env.runtime().block_on(async { + let mut conn = env.async_db().await.async_conn().await; + Ok::<_, anyhow::Error>( + sqlx::query(sql) + .fetch_all(&mut *conn) + .await? + .into_iter() + .map(|row| row.get(0)) + .collect(), + ) + }) } #[test] diff --git a/src/web/builds.rs b/src/web/builds.rs index ac8075ed4..fbe22b966 100644 --- a/src/web/builds.rs +++ b/src/web/builds.rs @@ -258,13 +258,14 @@ async fn get_builds( mod tests { use super::BuildStatus; use crate::{ + db::Overrides, test::{ assert_cache_control, fake_release_that_failed_before_build, wrapper, FakeBuild, TestEnvironment, }, web::cache::CachePolicy, }; - use chrono::{DateTime, Duration, Utc}; + use chrono::{DateTime, Utc}; use kuchikiki::traits::TendrilSink; use reqwest::StatusCode; @@ -587,17 +588,15 @@ mod tests { wrapper(|env| { env.fake_release().name("foo").version("0.1.0").create()?; - env.db().conn().query( - "INSERT INTO sandbox_overrides - (crate_name, max_memory_bytes, timeout_seconds, max_targets) - VALUES ($1, $2, $3, $4)", - &[ - &"foo", - &(6 * 1024 * 1024 * 1024i64), - &(Duration::try_hours(2).unwrap().num_seconds() as i32), - &1, - ], - )?; + env.runtime().block_on(async { + let mut conn = env.async_db().await.async_conn().await; + let limits = Overrides { + memory: Some(6 * 1024 * 1024 * 1024), + targets: Some(1), + timeout: Some(std::time::Duration::from_secs(2 * 60 * 60)), + }; + Overrides::save(&mut conn, "foo", limits).await + })?; let page = kuchikiki::parse_html().one( env.frontend() diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index e5ce5c3ce..7ff72706e 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -1630,9 +1630,12 @@ mod tests { .version("0.1.0") .create()?; - env.db() - .conn() - .query("UPDATE releases SET features = NULL WHERE id = $1", &[&id])?; + env.runtime().block_on(async { + let mut conn = env.async_db().await.async_conn().await; + sqlx::query!("UPDATE releases SET features = NULL WHERE id = $1", id) + .execute(&mut *conn) + .await + })?; let page = kuchikiki::parse_html().one( env.frontend() diff --git a/src/web/releases.rs b/src/web/releases.rs index 4097575d1..d0bfc0f3b 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -984,7 +984,7 @@ mod tests { #[test] fn im_feeling_lucky_with_stars() { wrapper(|env| { - { + env.runtime().block_on(async { // The normal test-setup will offset all primary sequences by 10k // to prevent errors with foreign key relations. // Random-crate-search relies on the sequence for the crates-table @@ -992,9 +992,11 @@ mod tests { // crate in the db breaks this test. // That's why we reset the id-sequence to zero for this test. - let mut conn = env.db().conn(); - conn.execute(r#"ALTER SEQUENCE crates_id_seq RESTART WITH 1"#, &[])?; - } + let mut conn = env.async_db().await.async_conn().await; + sqlx::query!(r#"ALTER SEQUENCE crates_id_seq RESTART WITH 1"#) + .execute(&mut *conn) + .await + })?; let web = env.frontend(); env.fake_release() diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 308d93ef3..af0da38b7 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -2041,17 +2041,21 @@ mod test { // regression test for https://github.com/rust-lang/docs.rs/pull/885#issuecomment-655147643 fn test_no_panic_on_missing_kind() { wrapper(|env| { - let db = env.db(); let id = env .fake_release() .name("strum") .version("0.13.0") .create()?; - // https://stackoverflow.com/questions/18209625/how-do-i-modify-fields-inside-the-new-postgresql-json-datatype - db.conn().query( - r#"UPDATE releases SET dependencies = dependencies::jsonb #- '{0,2}' WHERE id = $1"#, - &[&id], - )?; + + env.runtime().block_on(async { + let mut conn = env.async_db().await.async_conn().await; + // https://stackoverflow.com/questions/18209625/how-do-i-modify-fields-inside-the-new-postgresql-json-datatype + sqlx::query!( + r#"UPDATE releases SET dependencies = dependencies::jsonb #- '{0,2}' WHERE id = $1"#, id) + .execute(&mut *conn) + .await + })?; + let web = env.frontend(); assert_success("/strum/0.13.0/strum/", web)?; assert_success("/crate/strum/0.13.0/", web)?; diff --git a/src/web/source.rs b/src/web/source.rs index 840532c18..139a96e2e 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -487,12 +487,17 @@ mod tests { let web = env.frontend(); assert_success(path, web)?; - env.db().conn().execute( - "UPDATE releases + env.runtime().block_on(async { + let mut conn = env.async_db().await.async_conn().await; + sqlx::query!( + "UPDATE releases SET files = NULL WHERE id = $1", - &[&release_id], - )?; + release_id, + ) + .execute(&mut *conn) + .await + })?; assert!(web.get(path).send()?.status().is_success());