diff --git a/.sqlx/query-75310f51de2b9064c400dfa4ea16bac989fb9ce51a2efe7661c000aa6ad932d4.json b/.sqlx/query-75310f51de2b9064c400dfa4ea16bac989fb9ce51a2efe7661c000aa6ad932d4.json new file mode 100644 index 000000000..bcf5144f4 --- /dev/null +++ b/.sqlx/query-75310f51de2b9064c400dfa4ea16bac989fb9ce51a2efe7661c000aa6ad932d4.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name FROM crates ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "75310f51de2b9064c400dfa4ea16bac989fb9ce51a2efe7661c000aa6ad932d4" +} diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 66cc540b6..b17ef408c 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -20,6 +20,7 @@ use docs_rs::{ Context, Index, InstanceMetrics, PackageKind, RegistryApi, RustwideBuilder, ServiceMetrics, Storage, }; +use futures_util::StreamExt; use humantime::Duration; use once_cell::sync::OnceCell; use tokio::runtime::{Builder, Runtime}; @@ -486,6 +487,9 @@ enum DatabaseSubcommand { version: Option, }, + /// temporary commant to update the `crates.latest_version_id` field + UpdateLatestVersionId, + /// Updates Github/Gitlab stats for crates. UpdateRepositoryFields, @@ -544,6 +548,30 @@ impl DatabaseSubcommand { .context("Failed to run database migrations")? } + Self::UpdateLatestVersionId => { + let pool = ctx.pool()?; + ctx.runtime()? + .block_on(async { + let mut list_conn = pool.get_async().await?; + let mut update_conn = pool.get_async().await?; + + let mut result_stream = + sqlx::query!("SELECT id, name FROM crates ORDER BY name") + .fetch(&mut *list_conn); + + while let Some(row) = result_stream.next().await { + let row = row?; + + println!("handling crate {}", row.name); + + db::update_latest_version_id(&mut update_conn, row.id).await?; + } + + Ok::<(), anyhow::Error>(()) + }) + .context("Failed to update latest version id")? + } + Self::UpdateRepositoryFields => { ctx.runtime()? .block_on(ctx.repository_stats_updater()?.update_all_crates())?; @@ -787,6 +815,7 @@ impl Context for BinContext { self.instance_metrics()?, self.config()?, self.storage()?, + self.runtime()?, ); fn storage(self) -> Storage = { let runtime = self.runtime()?; diff --git a/src/build_queue.rs b/src/build_queue.rs index ee82160da..612f232eb 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -1,5 +1,5 @@ use crate::cdn; -use crate::db::{delete_crate, delete_version, Pool}; +use crate::db::{delete_crate, delete_version, update_latest_version_id, Pool}; use crate::docbuilder::PackageKind; use crate::error::Result; use crate::storage::Storage; @@ -8,11 +8,10 @@ use crate::Context; use crate::{Config, Index, InstanceMetrics, RustwideBuilder}; use anyhow::Context as _; use fn_error_context::context; - -use tracing::{debug, error, info}; - use std::collections::HashMap; use std::sync::Arc; +use tokio::runtime::Runtime; +use tracing::{debug, error, info}; #[derive(Debug, Clone, Eq, PartialEq, serde::Serialize)] pub(crate) struct QueuedCrate { @@ -30,6 +29,7 @@ pub struct BuildQueue { storage: Arc, pub(crate) db: Pool, metrics: Arc, + runtime: Arc, max_attempts: i32, } @@ -39,6 +39,7 @@ impl BuildQueue { metrics: Arc, config: Arc, storage: Arc, + runtime: Arc, ) -> Self { BuildQueue { max_attempts: config.build_attempts.into(), @@ -46,6 +47,7 @@ impl BuildQueue { db, metrics, storage, + runtime, } } @@ -403,16 +405,18 @@ impl BuildQueue { ) -> Result<()> { let activity = if yanked { "yanked" } else { "unyanked" }; - let rows = conn.execute( + let result = conn.query( "UPDATE releases SET yanked = $3 FROM crates WHERE crates.id = releases.crate_id AND name = $1 - AND version = $2", + AND version = $2 + RETURNING crates.id + ", &[&name, &version, &yanked], )?; - if rows != 1 { + if result.len() != 1 { match self .has_build_queued(name, version) .context("error trying to fetch build queue") @@ -434,6 +438,17 @@ impl BuildQueue { } else { debug!("{}-{} {}", name, version, activity); } + + if let Some(row) = result.first() { + let crate_id: i32 = row.get(0); + + self.runtime.block_on(async { + let mut conn = self.db.get_async().await?; + + update_latest_version_id(&mut conn, crate_id).await + })?; + } + Ok(()) } diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 5da6cafca..24e2bac2e 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -5,9 +5,9 @@ use crate::{ registry_api::{CrateData, CrateOwner, ReleaseData}, storage::CompressionAlgorithm, utils::MetadataPackage, - web::crate_details::CrateDetails, + web::crate_details::{latest_release, releases_for_crate}, }; -use anyhow::{anyhow, Context}; +use anyhow::Context; use futures_util::stream::TryStreamExt; use serde_json::Value; use slug::slugify; @@ -126,27 +126,27 @@ pub(crate) async fn add_package_into_database( add_keywords_into_database(conn, metadata_pkg, release_id).await?; add_compression_into_database(conn, compression_algorithms.into_iter(), release_id).await?; - let crate_details = CrateDetails::new( - &mut *conn, - &metadata_pkg.name, - &metadata_pkg.version, - &metadata_pkg.version, - ) - .await - .context("error when fetching crate-details")? - .ok_or_else(|| anyhow!("crate details not found directly after creating them"))?; + update_latest_version_id(&mut *conn, crate_id) + .await + .context("couldn't update latest version id")?; + + Ok(release_id) +} + +pub async fn update_latest_version_id(conn: &mut sqlx::PgConnection, crate_id: i32) -> Result<()> { + let releases = releases_for_crate(conn, crate_id).await?; sqlx::query!( "UPDATE crates SET latest_version_id = $2 WHERE id = $1", crate_id, - crate_details.latest_release().id, + latest_release(&releases).map(|release| release.id), ) .execute(&mut *conn) .await?; - Ok(release_id) + Ok(()) } pub(crate) async fn add_doc_coverage( diff --git a/src/db/delete.rs b/src/db/delete.rs index 98a908f27..c2920fb1a 100644 --- a/src/db/delete.rs +++ b/src/db/delete.rs @@ -1,6 +1,8 @@ -use crate::error::Result; -use crate::storage::{rustdoc_archive_path, source_archive_path, Storage}; -use crate::Config; +use crate::{ + error::Result, + storage::{rustdoc_archive_path, source_archive_path, Storage}, + Config, +}; use anyhow::Context as _; use fn_error_context::context; use postgres::Client; diff --git a/src/db/mod.rs b/src/db/mod.rs index 080838a32..f1acc436b 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -2,6 +2,7 @@ use anyhow::Result; use sqlx::migrate::{Migrate, Migrator}; +pub use self::add_package::update_latest_version_id; pub(crate) use self::add_package::{ add_build_into_database, add_doc_coverage, add_package_into_database, }; diff --git a/src/test/mod.rs b/src/test/mod.rs index 4490643e0..ae1409d95 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -372,6 +372,7 @@ impl TestEnvironment { self.instance_metrics(), self.config(), self.storage(), + self.runtime(), )) }) .clone() diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 266d3544a..1fc0973c7 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -13,7 +13,7 @@ use crate::{ }, AsyncStorage, }; -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; use axum::{ extract::{Extension, Path}, response::{IntoResponse, Response as AxumResponse}, @@ -302,11 +302,19 @@ impl CrateDetails { /// Returns the latest non-yanked, non-prerelease release of this crate (or latest /// yanked/prereleased if that is all that exist). - pub fn latest_release(&self) -> &Release { - self.releases - .iter() - .find(|release| release.version.pre.is_empty() && !release.yanked) - .unwrap_or(&self.releases[0]) + pub fn latest_release(&self) -> Result<&Release> { + latest_release(&self.releases).ok_or_else(|| anyhow!("crate without releases")) + } +} + +pub(crate) fn latest_release(releases: &[Release]) -> Option<&Release> { + if let Some(release) = releases + .iter() + .find(|release| release.version.pre.is_empty() && !release.yanked) + { + Some(release) + } else { + releases.first() } } @@ -996,7 +1004,7 @@ mod tests { .unwrap() }); assert_eq!( - details.latest_release().version, + details.latest_release().unwrap().version, semver::Version::parse("0.0.3")? ); } @@ -1026,7 +1034,7 @@ mod tests { .unwrap() }); assert_eq!( - details.latest_release().version, + details.latest_release().unwrap().version, semver::Version::parse("0.0.2")? ); } @@ -1057,7 +1065,7 @@ mod tests { .unwrap() }); assert_eq!( - details.latest_release().version, + details.latest_release().unwrap().version, semver::Version::parse("0.0.2")? ); } @@ -1096,7 +1104,7 @@ mod tests { .unwrap() }); assert_eq!( - details.latest_release().version, + details.latest_release().unwrap().version, semver::Version::parse("0.0.3")? ); } diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 6d7e7d8da..694100c71 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -562,7 +562,7 @@ pub(crate) async fn rustdoc_html_server_handler( rendering_time.step("find latest path"); - let latest_release = krate.latest_release(); + let latest_release = krate.latest_release()?; // Get the latest version of the crate let latest_version = latest_release.version.to_string();