From b37140d3ec2c256ec52e7da449e26749ac8bbf1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 6 Jan 2026 10:05:31 +0100 Subject: [PATCH 1/5] Do not check the old queue in the new queue anymore --- site/src/job_queue/mod.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/site/src/job_queue/mod.rs b/site/src/job_queue/mod.rs index 9a03db0ff..c4ca9fd77 100644 --- a/site/src/job_queue/mod.rs +++ b/site/src/job_queue/mod.rs @@ -35,8 +35,7 @@ async fn create_benchmark_request_master_commits( .filter(|c| now.signed_duration_since(c.time) < chrono::Duration::days(29)); for master_commit in master_commits { - if !index.contains_tag(&master_commit.sha) && conn.pr_of(&master_commit.sha).await.is_none() - { + if !index.contains_tag(&master_commit.sha) { let pr = master_commit.pr.unwrap_or(0); let benchmark = BenchmarkRequest::create_master( &master_commit.sha, @@ -345,12 +344,7 @@ pub async fn enqueue_benchmark_request( // If the parent job has been deleted from the database // but was already benchmarked then the collector will ignore // it as it will see it already has results. - if let Some(parent_sha) = request.parent_sha() { - // If the parent is in the old system, do not backfill it - if tx.conn().parent_of(parent_sha).await.is_some() { - continue; - } enqueue_job( &mut tx, parent_sha, From 17e2e89398e9712c5c95627a13b0b958afc83ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 6 Jan 2026 10:09:46 +0100 Subject: [PATCH 2/5] Stop using `pull_request_build` to implement `pr_of` --- database/src/pool.rs | 4 +--- database/src/pool/postgres.rs | 5 +---- database/src/pool/sqlite.rs | 9 +-------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/database/src/pool.rs b/database/src/pool.rs index 106ea54a3..6a35ab945 100644 --- a/database/src/pool.rs +++ b/database/src/pool.rs @@ -156,9 +156,7 @@ pub trait Connection: Send + Sync { /// (Currently only works for try commits) async fn parent_of(&self, sha: &str) -> Option; - /// Returns the PR of the parent commit, if available. - /// - /// (Currently only works for try commits) + /// Returns the PR associated with an artifact with the given SHA, if available. async fn pr_of(&self, sha: &str) -> Option; /// Returns the collection ids corresponding to the query. Usually just one. diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index 86ef188a2..57b3c7fcd 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -1258,10 +1258,7 @@ where } async fn pr_of(&self, sha: &str) -> Option { self.conn() - .query_opt( - "select pr from pull_request_build where bors_sha = $1", - &[&sha], - ) + .query_opt("SELECT pr FROM benchmark_request WHERE tag = $1", &[&sha]) .await .unwrap() .map(|r| r.get::<_, i32>(0) as u32) diff --git a/database/src/pool/sqlite.rs b/database/src/pool/sqlite.rs index 8872f08eb..7c395f31a 100644 --- a/database/src/pool/sqlite.rs +++ b/database/src/pool/sqlite.rs @@ -1048,14 +1048,7 @@ impl Connection for SqliteConnection { } async fn pr_of(&self, sha: &str) -> Option { - self.raw_ref() - .query_row( - "select pr from pull_request_build where bors_sha = ?", - params![sha], - |row| Ok(row.get(0).unwrap()), - ) - .optional() - .unwrap() + None } async fn list_self_profile( From 162eb6a3e54e5d8be8210b4978ff85899da7d86e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 6 Jan 2026 10:12:32 +0100 Subject: [PATCH 3/5] Stop using `pull_request_build` to implement `parent_of` --- database/src/pool.rs | 5 +---- database/src/pool/postgres.rs | 2 +- database/src/pool/sqlite.rs | 15 +++------------ 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/database/src/pool.rs b/database/src/pool.rs index 6a35ab945..58e5fb207 100644 --- a/database/src/pool.rs +++ b/database/src/pool.rs @@ -150,10 +150,7 @@ pub trait Connection: Send + Sync { async fn collector_remove_step(&self, aid: ArtifactIdNumber, step: &str); - // TODO: the following two functions do not work anymore and should not be used! - /// Returns the sha of the parent commit, if available. - /// - /// (Currently only works for try commits) + /// Returns the SHA of the parent of the given SHA commit, if available. async fn parent_of(&self, sha: &str) -> Option; /// Returns the PR associated with an artifact with the given SHA, if available. diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index 57b3c7fcd..91d01b101 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -1249,7 +1249,7 @@ where async fn parent_of(&self, sha: &str) -> Option { self.conn() .query_opt( - "select parent_sha from pull_request_build where bors_sha = $1", + "SELECT parent_sha FROM benchmark_request WHERE tag = $1", &[&sha], ) .await diff --git a/database/src/pool/sqlite.rs b/database/src/pool/sqlite.rs index 7c395f31a..7ccff5b09 100644 --- a/database/src/pool/sqlite.rs +++ b/database/src/pool/sqlite.rs @@ -1034,20 +1034,11 @@ impl Connection for SqliteConnection { .unwrap(); } - async fn parent_of(&self, sha: &str) -> Option { - let mut shas = self - .raw_ref() - .prepare_cached("select parent_sha from pull_request_build where bors_sha = ?") - .unwrap() - .query(params![sha]) - .unwrap() - .mapped(|row| Ok(row.get(0).unwrap())) - .collect::, _>>() - .unwrap(); - shas.pop() + async fn parent_of(&self, _sha: &str) -> Option { + None } - async fn pr_of(&self, sha: &str) -> Option { + async fn pr_of(&self, _sha: &str) -> Option { None } From a82c299239410a6288cd825131f2ba50be02f4bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 6 Jan 2026 10:13:56 +0100 Subject: [PATCH 4/5] Remove legacy tables from DB export/import --- database/src/bin/postgres-to-sqlite.rs | 95 ---------------- database/src/bin/sqlite-to-postgres.rs | 144 ------------------------- 2 files changed, 239 deletions(-) diff --git a/database/src/bin/postgres-to-sqlite.rs b/database/src/bin/postgres-to-sqlite.rs index 5a57c3316..a779e1cc9 100644 --- a/database/src/bin/postgres-to-sqlite.rs +++ b/database/src/bin/postgres-to-sqlite.rs @@ -77,33 +77,6 @@ impl Table for Artifact { } } -struct ArtifactCollectionDuration; - -impl Table for ArtifactCollectionDuration { - fn name(&self) -> &'static str { - "artifact_collection_duration" - } - - fn postgres_select_statement(&self, since_weeks_ago: Option) -> String { - let s = "select aid, date_recorded, duration from ".to_string() + self.name(); - with_filter_clause_maybe(s, ARTIFACT_JOIN_AND_WHERE, since_weeks_ago) - } - - fn sqlite_insert_statement(&self) -> &'static str { - "insert into artifact_collection_duration (aid, date_recorded, duration) VALUES (?, ?, ?)" - } - - fn sqlite_execute_insert(&self, statement: &mut rusqlite::Statement, row: tokio_postgres::Row) { - statement - .execute(params![ - row.get::<_, i32>(0), - row.get::<_, DateTime>(1).timestamp(), - row.get::<_, i32>(2) - ]) - .unwrap(); - } -} - struct Benchmark; impl Table for Benchmark { @@ -153,36 +126,6 @@ impl Table for Collection { } } -struct CollectorProgress; - -impl Table for CollectorProgress { - fn name(&self) -> &'static str { - "collector_progress" - } - - fn postgres_select_statement(&self, since_weeks_ago: Option) -> String { - let s = "select aid, step, start_time, end_time from ".to_string() + self.name(); - with_filter_clause_maybe(s, ARTIFACT_JOIN_AND_WHERE, since_weeks_ago) - } - - fn sqlite_insert_statement(&self) -> &'static str { - "insert into collector_progress (aid, step, start, end) VALUES (?, ?, ?, ?)" - } - - fn sqlite_execute_insert(&self, statement: &mut rusqlite::Statement, row: tokio_postgres::Row) { - statement - .execute(params![ - row.get::<_, i32>(0), - row.get::<_, &str>(1), - row.get::<_, Option>>(2) - .map(|d| d.timestamp()), - row.get::<_, Option>>(3) - .map(|d| d.timestamp()), - ]) - .unwrap(); - } -} - struct Error; impl Table for Error { @@ -269,41 +212,6 @@ impl Table for PstatSeries { } } -struct PullRequestBuild; - -impl Table for PullRequestBuild { - fn name(&self) -> &'static str { - "pull_request_build" - } - - fn postgres_select_statement(&self, _since_weeks_ago: Option) -> String { - "select bors_sha, pr, parent_sha, complete, requested, include, exclude, runs, backends from " - .to_string() - + self.name() - } - - fn sqlite_insert_statement(&self) -> &'static str { - "insert into pull_request_build (bors_sha, pr, parent_sha, complete, requested, include, exclude, runs, backends) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" - } - - fn sqlite_execute_insert(&self, statement: &mut rusqlite::Statement, row: tokio_postgres::Row) { - statement - .execute(params![ - row.get::<_, Option<&str>>(0), - row.get::<_, i32>(1), - row.get::<_, Option<&str>>(2), - row.get::<_, Option>(3), - row.get::<_, Option>>(4) - .map(|d| d.timestamp()), - row.get::<_, Option<&str>>(5), - row.get::<_, Option<&str>>(6), - row.get::<_, Option>(7), - row.get::<_, Option<&str>>(8), - ]) - .unwrap(); - } -} - struct RawSelfProfile; impl Table for RawSelfProfile { @@ -450,14 +358,11 @@ async fn main() -> anyhow::Result<()> { // Order matters to the extent necessary to satisfy foreign key constraints. let tables: &[&dyn Table] = &[ &Artifact, - &ArtifactCollectionDuration, &Benchmark, &Collection, - &CollectorProgress, &Error, &PstatSeries, &Pstat, - &PullRequestBuild, &RawSelfProfile, &RustcCompilation, &RuntimePstatSeries, diff --git a/database/src/bin/sqlite-to-postgres.rs b/database/src/bin/sqlite-to-postgres.rs index 35eefd62b..1eb2c6888 100644 --- a/database/src/bin/sqlite-to-postgres.rs +++ b/database/src/bin/sqlite-to-postgres.rs @@ -81,45 +81,6 @@ impl Table for Artifact { } } -struct ArtifactCollectionDuration; - -#[derive(Serialize)] -struct ArtifactCollectionDurationRow { - aid: i32, - date_recorded: DateTime, - duration: i32, -} - -impl Table for ArtifactCollectionDuration { - fn name() -> &'static str { - "artifact_collection_duration" - } - - fn sqlite_attributes() -> &'static str { - "aid, date_recorded, duration" - } - - fn postgres_attributes() -> &'static str { - "aid, date_recorded, duration" - } - - fn postgres_generated_id_attribute() -> Option<&'static str> { - None - } - - fn write_postgres_csv_row(writer: &mut csv::Writer, row: &rusqlite::Row) { - let date_recorded: i64 = row.get(1).unwrap(); - - writer - .serialize(ArtifactCollectionDurationRow { - aid: row.get(0).unwrap(), - date_recorded: Utc.timestamp_opt(date_recorded, 0).unwrap(), - duration: row.get(2).unwrap(), - }) - .unwrap(); - } -} - struct Benchmark; #[derive(Serialize)] @@ -193,50 +154,6 @@ impl Table for Collection { } } -struct CollectorProgress; - -#[derive(Serialize)] -struct CollectorProgressRow<'a> { - aid: i32, - step: &'a str, - start_time: Nullable>, - end_time: Nullable>, -} - -impl Table for CollectorProgress { - fn name() -> &'static str { - "collector_progress" - } - - fn sqlite_attributes() -> &'static str { - "aid, step, start, end" - } - - fn postgres_attributes() -> &'static str { - "aid, step, start_time, end_time" - } - - fn postgres_generated_id_attribute() -> Option<&'static str> { - None - } - - fn write_postgres_csv_row(writer: &mut csv::Writer, row: &rusqlite::Row) { - let start: Option = row.get(2).unwrap(); - let end: Option = row.get(3).unwrap(); - let start_time = Nullable(start.map(|seconds| Utc.timestamp_opt(seconds, 0).unwrap())); - let end_time = Nullable(end.map(|seconds| Utc.timestamp_opt(seconds, 0).unwrap())); - - writer - .serialize(CollectorProgressRow { - aid: row.get(0).unwrap(), - step: row.get_ref(1).unwrap().as_str().unwrap(), - start_time, - end_time, - }) - .unwrap(); - } -} - struct Error; #[derive(Serialize)] @@ -360,64 +277,6 @@ impl Table for PstatSeries { } } -struct PullRequestBuild; - -#[derive(Serialize)] -struct PullRequestBuildRow<'a> { - bors_sha: Nullable<&'a str>, - pr: i32, - parent_sha: Nullable<&'a str>, - complete: Nullable, - requested: Nullable>, - include: Nullable<&'a str>, - exclude: Nullable<&'a str>, - runs: Nullable, - commit_date: Nullable>, - backends: Nullable<&'a str>, -} - -impl Table for PullRequestBuild { - fn name() -> &'static str { - "pull_request_build" - } - - fn sqlite_attributes() -> &'static str { - "bors_sha, pr, parent_sha, complete, requested, include, exclude, runs, commit_date, backends" - } - - fn postgres_attributes() -> &'static str { - "bors_sha, pr, parent_sha, complete, requested, include, exclude, runs, commit_date, backends" - } - - fn postgres_generated_id_attribute() -> Option<&'static str> { - None - } - - fn write_postgres_csv_row(writer: &mut csv::Writer, row: &rusqlite::Row) { - let requested: Option = row.get(4).unwrap(); - let commit_date: Option = row.get(8).unwrap(); - - writer - .serialize(PullRequestBuildRow { - bors_sha: row.get_ref(0).unwrap().try_into().unwrap(), - pr: row.get(1).unwrap(), - parent_sha: row.get_ref(2).unwrap().try_into().unwrap(), - complete: row.get(3).unwrap(), - requested: Nullable( - requested.map(|seconds| Utc.timestamp_opt(seconds, 0).unwrap()), - ), - include: row.get_ref(5).unwrap().try_into().unwrap(), - exclude: row.get_ref(6).unwrap().try_into().unwrap(), - runs: row.get(7).unwrap(), - commit_date: Nullable( - commit_date.map(|seconds| Utc.timestamp_opt(seconds, 0).unwrap()), - ), - backends: row.get_ref(9).unwrap().try_into().unwrap(), - }) - .unwrap(); - } -} - struct RawSelfProfile; #[derive(Serialize)] @@ -731,14 +590,11 @@ async fn main() -> anyhow::Result<()> { disable_table_triggers(&postgres_tx, &tables).await; // Order matters to the extent necessary to satisfy foreign key constraints. copy::(&sqlite_tx, &postgres_tx).await; - copy::(&sqlite_tx, &postgres_tx).await; copy::(&sqlite_tx, &postgres_tx).await; copy::(&sqlite_tx, &postgres_tx).await; - copy::(&sqlite_tx, &postgres_tx).await; copy::(&sqlite_tx, &postgres_tx).await; copy::(&sqlite_tx, &postgres_tx).await; copy::(&sqlite_tx, &postgres_tx).await; - copy::(&sqlite_tx, &postgres_tx).await; copy::(&sqlite_tx, &postgres_tx).await; copy::(&sqlite_tx, &postgres_tx).await; copy::(&sqlite_tx, &postgres_tx).await; From c3f151dcc5c340138ed3e83f75876535010ec131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 6 Jan 2026 10:15:36 +0100 Subject: [PATCH 5/5] Remove mention of manually deleting data from the DB Using the `purge_artifact` command is a better approach. --- database/manual-modifications.md | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/database/manual-modifications.md b/database/manual-modifications.md index 2b7480045..6b158db2a 100644 --- a/database/manual-modifications.md +++ b/database/manual-modifications.md @@ -2,7 +2,7 @@ This document contains useful queries and commands that should be performed manually in exceptional situations. -## Remove data for a stable artifact from the DB +## Remove data for an artifact from the DB This is important for situations where there is some compilation error for a stable benchmark, with a stable release of the compiler. While this should be rare, it happens sometimes e.g. because of future incompatibility lints turning into errors. @@ -16,28 +16,3 @@ You can do that either using the following command: $ cargo run --bin collector purge_artifact # $ cargo run --bin collector purge_artifact 1.70.0 # Remove stable artifact 1.70.0 ``` - -Or using SQL queries. There are `ON DELETE CASCADE` clauses for `aid` (artifact ID) on tables that -reference it, so it should be enough to just delete the artifact from the `artifact` table. -The set of queries below show an example of removing the measured data and errors for Rust `1.69` -and `1.70`: -```sql -DELETE FROM artifact -WHERE name IN ('1.69.0', '1.70.0') AND - type = 'release'; -``` -After executing this query, the server should automatically re-benchmark these artifacts again. - -## Remove data for a master/try artifact from the DB -This is similar to removing a stable artifact, however it also has to be removed from the -`pull_request_build` table. - -```sql -DELETE FROM artifact -WHERE name = ""; - -DELETE FROM pull_request_build -WHERE bors_sha = ""; -``` -After that, the server has to be restarted, or you have to send a GET request to its `/perf/onpush` -endpoint.