diff --git a/collector/collect.sh b/collector/collect.sh deleted file mode 100755 index 0d66e5142..000000000 --- a/collector/collect.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# This script expects SITE_URL and DATABASE to be defined in the environment - -set -u -o pipefail - -echo "Running collector targeting $SITE_URL" - -export RUST_LOG=collector=trace,collector::sysroot=debug -export PATH="/home/collector/.cargo/bin:$PATH" - -while : ; do - # Update and rebuild the collector. - git pull - git reset --hard @{upstream} - - # Make sure we have a recent build, so that we can successfully build - # the collector. - rustup update stable - cargo +stable build --release -p collector - - target/release/collector bench_next $SITE_URL --self-profile --bench-rustc --db $DATABASE - STATUS=$? - echo finished run at `date` with exit code $STATUS - - # Wait a bit if the command has failed. - if [ $STATUS -ne 0 ]; then - sleep 120 - fi -done diff --git a/collector/src/api.rs b/collector/src/api.rs deleted file mode 100644 index 845bf6a50..000000000 --- a/collector/src/api.rs +++ /dev/null @@ -1,20 +0,0 @@ -pub mod next_artifact { - use database::Commit; - - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] - pub enum NextArtifact { - Release(String), - Commit { - commit: Commit, - include: Option, - exclude: Option, - runs: Option, - backends: Option, - }, - } - - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] - pub struct Response { - pub artifact: Option, - } -} diff --git a/collector/src/bin/collector.rs b/collector/src/bin/collector.rs index 880105198..19f1e00ee 100644 --- a/collector/src/bin/collector.rs +++ b/collector/src/bin/collector.rs @@ -30,7 +30,6 @@ use tabled::settings::style::Border; use tabled::settings::{Alignment, Color, Modify, Width}; use tokio::runtime::Runtime; -use collector::api::next_artifact::NextArtifact; use collector::artifact_stats::{ compile_and_get_stats, ArtifactStats, ArtifactWithStats, CargoProfile, }; @@ -601,21 +600,6 @@ enum Commands { purge: PurgeOption, }, - /// Benchmarks the next commit or release for perf.rust-lang.org - BenchNext { - /// Site URL - site_url: String, - - #[command(flatten)] - db: DbOption, - - #[command(flatten)] - bench_rustc: BenchRustcOption, - - #[command(flatten)] - self_profile: SelfProfileOption, - }, - /// Benchmarks a published toolchain for perf.rust-lang.org's dashboard BenchPublished { /// Toolchain (e.g. stable, beta, 1.26.0) @@ -822,7 +806,6 @@ fn main_result() -> anyhow::Result { mode: BinaryStatsMode::Local(_), .. } - | Commands::BenchNext { .. } | Commands::BenchPublished { .. } | Commands::InstallNext { .. } | Commands::Download(_) @@ -1064,169 +1047,6 @@ fn main_result() -> anyhow::Result { Ok(0) } - Commands::BenchNext { - site_url, - db, - bench_rustc, - self_profile, - } => { - log_db(&db); - println!("processing artifacts"); - let client = reqwest::blocking::Client::new(); - let response: collector::api::next_artifact::Response = client - .get(format!("{site_url}/perf/next_artifact")) - .send()? - .json()?; - let next = if let Some(c) = response.artifact { - c - } else { - println!("no artifact to benchmark"); - - // Sleep for a bit to avoid spamming the perf server too much - // This sleep serves to remove a needless sleep in `collector/collect.sh` when - // a benchmark was actually executed. - std::thread::sleep(Duration::from_secs(60 * 2)); - - // no missing artifacts - return Ok(0); - }; - - let res = std::panic::catch_unwind(|| { - let pool = database::Pool::open(&db.db); - let rt = build_async_runtime(); - - match next { - NextArtifact::Release(tag) => { - let toolchain = create_toolchain_from_published_version( - &tag, - &require_host_target_tuple(), - )?; - let conn = rt.block_on(pool.connection()); - rt.block_on(bench_published_artifact( - conn, - toolchain, - &benchmark_dirs, - None, - )) - } - NextArtifact::Commit { - commit, - include, - exclude, - runs, - backends: requested_backends, - } => { - // Parse the requested backends, with LLVM as a fallback if none or no valid - // ones were explicitly specified, which will be the case for the vast - // majority of cases. - let mut backends = vec![]; - if let Some(requested_backends) = requested_backends { - let requested_backends = requested_backends.to_lowercase(); - if requested_backends.contains("llvm") { - backends.push(CodegenBackend::Llvm); - } - if requested_backends.contains("cranelift") { - backends.push(CodegenBackend::Cranelift); - } - } - - if backends.is_empty() { - backends.push(CodegenBackend::Llvm); - } - - // FIXME: remove this when/if NextArtifact::Commit's include/exclude - // changed from Option to Vec - // to not to manually parse args - let split_args = |l: Option| -> Vec { - if let Some(l) = l { - l.split(',').map(|arg| arg.trim().to_owned()).collect() - } else { - vec![] - } - }; - let sha = commit.sha.to_string(); - let sysroot = rt - .block_on(Sysroot::install( - Path::new(TOOLCHAIN_CACHE_DIRECTORY), - sha.clone(), - &require_host_target_tuple(), - &backends, - )) - .map_err(SysrootDownloadError::as_anyhow_error) - .with_context(|| format!("failed to install sysroot for {commit:?}"))?; - - let mut benchmarks = get_compile_benchmarks( - &compile_benchmark_dir, - CompileBenchmarkFilter::Fuzzy { - include: &split_args(include), - exclude: &split_args(exclude), - exclude_suffix: &[], - }, - )?; - benchmarks.retain(|b| b.category().is_primary_or_secondary()); - - let artifact_id = ArtifactId::Commit(commit); - let mut conn = rt.block_on(pool.connection()); - let toolchain = Toolchain::from_sysroot(&sysroot, sha); - - let compile_config = CompileBenchmarkConfig { - benchmarks, - profiles: vec![ - Profile::Check, - Profile::Debug, - Profile::Doc, - Profile::Opt, - ], - scenarios: Scenario::all(), - backends, - iterations: runs.map(|v| v as usize), - is_self_profile: self_profile.self_profile, - bench_rustc: bench_rustc.bench_rustc, - targets: vec![Target::default()], - }; - let runtime_suite = rt.block_on(load_runtime_benchmarks( - conn.as_mut(), - &runtime_benchmark_dir, - CargoIsolationMode::Isolated, - None, - &toolchain, - &artifact_id, - None, - ))?; - - let runtime_config = RuntimeBenchmarkConfig { - runtime_suite, - filter: RuntimeBenchmarkFilter::keep_all(), - iterations: DEFAULT_RUNTIME_ITERATIONS, - target: Target::default(), - }; - let shared = SharedBenchmarkConfig { - artifact_id, - toolchain, - job_id: None, - }; - - rt.block_on(run_benchmarks( - conn.as_mut(), - shared, - Some(compile_config), - Some(runtime_config), - )) - } - } - }); - // We need to send a message to this endpoint even if the collector panics - client.post(format!("{site_url}/perf/onpush")).send()?; - - match res { - Ok(res) => res?, - Err(error) => { - log::error!("The collector has crashed\n{error:?}"); - } - } - Ok(0) - } - Commands::BenchPublished { toolchain, db } => { log_db(&db); let pool = database::Pool::open(&db.db); @@ -2267,8 +2087,6 @@ async fn run_benchmarks( let collector = init_collection(connection, &shared, compile.as_ref(), runtime.as_ref()).await; - let start = Instant::now(); - // Compile benchmarks let compile_result = if let Some(compile) = compile { let errors = bench_compile(connection, &shared, compile, &collector).await; @@ -2295,13 +2113,6 @@ async fn run_benchmarks( Ok(()) }; - if shared.job_id.is_none() { - let end = start.elapsed(); - connection - .record_duration(collector.artifact_row_id, end) - .await; - } - compile_result.and(runtime_result) } diff --git a/collector/src/lib.rs b/collector/src/lib.rs index dbee618a5..2769e2573 100644 --- a/collector/src/lib.rs +++ b/collector/src/lib.rs @@ -1,11 +1,9 @@ use chrono::NaiveDate; pub use database::{Commit, PatchName, QueryLabel}; use serde::Deserialize; -use std::cmp::PartialOrd; use std::fmt; use std::process::{self, Command}; -pub mod api; pub mod artifact_stats; pub mod benchmark_set; pub mod cargo; @@ -24,9 +22,6 @@ use hashbrown::HashSet; use process::Stdio; use std::time::{Duration, Instant}; -#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Deserialize)] -pub struct DeltaTime(#[serde(with = "round_float")] pub f64); - /// The bound for finding an artifact /// /// This can either be the upper or lower bound. diff --git a/database/schema.md b/database/schema.md index 77420b134..19c334271 100644 --- a/database/schema.md +++ b/database/schema.md @@ -82,27 +82,6 @@ Columns: * **id** (`integer`): Unique identifier * **perf_commit** (`text`): Commit sha / tag -### collector_progress - -Keeps track of the collector's start and finish time as well as which step it's currently on. - -Columns: - -* **aid** (`integer`): Artifact id, references the id in the artifact table -* **step** (`text`): The step the collector is currently benchmarking -* **start** (`timestamptz`): When the collector started -* **end** (`timestamptz`): When the collector finished - -### artifact_collection_duration - -Records how long benchmarking takes in seconds. - -Columns: - -* **aid** (`integer`): Artifact id, references the id in the artifact table -* **date_recorded** (`timestamptz`): When this was recorded -* **duration** (`integer`): How long the benchmarking took in seconds - ### benchmark The different types of compile-time benchmarks that are run. @@ -202,24 +181,6 @@ Columns: * **profile** (`text`): What type of compilation is happening - check build, optimized build (a.k.a. release build), debug build, or doc build. * **cache** (`text`): The incremental scenario used for the benchmark -### pull_request_build - -Records a pull request commit that is waiting in a queue to be benchmarked. - -First a merge commit is queued, then its artifacts are built by bors, and once the commit -is attached to the entry in this table, it can be benchmarked. - -* **bors_sha** (`text`): SHA of the commit that should be benchmarked -* **pr** (`integer`): number of the PR -* **parent_sha** (`text`): SHA of the parent commit, to which will the PR be compared -* **complete** (`boolean`): Specifies whether this commit has been already benchmarked or not -* **requested** (`timestamptz`): When was the commit queued -* **include** (`text`): Which benchmarks should be included (corresponds to the `--include` benchmark parameter), comma separated strings -* **exclude** (`text`): Which benchmarks should be excluded (corresponds to the `--exclude` benchmark parameter), comma separated strings -* **runs** (`integer`): How many iterations should be used by default for the benchmark run -* **commit_date** (`timestamptz`): When was the commit created -* **backends** (`text`): The codegen backends to use for the benchmarks (corresponds to the `--backends` benchmark parameter) - ### error Records an error within the application namely a; diff --git a/database/src/pool.rs b/database/src/pool.rs index 815b10cd8..106ea54a3 100644 --- a/database/src/pool.rs +++ b/database/src/pool.rs @@ -1,11 +1,11 @@ use crate::selector::CompileTestCase; use crate::{ - ArtifactCollection, ArtifactId, ArtifactIdNumber, BenchmarkJob, BenchmarkJobConclusion, - BenchmarkJobKind, BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestInsertResult, - BenchmarkRequestStatus, BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, - CollectorConfig, CompileBenchmark, PendingBenchmarkRequests, Target, + ArtifactId, ArtifactIdNumber, BenchmarkJob, BenchmarkJobConclusion, BenchmarkJobKind, + BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestInsertResult, BenchmarkRequestStatus, + BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectorConfig, CompileBenchmark, + PendingBenchmarkRequests, Target, }; -use crate::{CollectionId, Index, Profile, QueuedCommit, Scenario, Step}; +use crate::{CollectionId, Index, Profile, Scenario}; use chrono::{DateTime, Utc}; use hashbrown::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; @@ -45,10 +45,6 @@ pub trait Connection: Send + Sync { async fn artifact_by_name(&self, artifact: &str) -> Option; - /// This records the duration of a collection run, i.e., collecting all of - /// the statistics for a particular artifact. - async fn record_duration(&self, artifact: ArtifactIdNumber, duration: Duration); - /// One collection corresponds to all gathered metrics for a single iteration of a test case. async fn collection_id(&self, version: &str) -> CollectionId; async fn artifact_id(&self, artifact: &ArtifactId) -> ArtifactIdNumber; @@ -140,27 +136,11 @@ pub trait Connection: Send + Sync { ) -> Vec>>; async fn get_error(&self, artifact_row_id: ArtifactIdNumber) -> HashMap; - async fn queue_pr( - &self, - pr: u32, - include: Option<&str>, - exclude: Option<&str>, - runs: Option, - backends: Option<&str>, - ); - /// Returns true if this PR was queued waiting for a commit - async fn pr_attach_commit( - &self, - pr: u32, - sha: &str, - parent_sha: &str, - commit_date: Option>, - ) -> bool; - async fn queued_commits(&self) -> Vec; - async fn mark_complete(&self, sha: &str) -> Option; - // Collector status API + // TODO: these functions should be removed. The only useful user of them currently are runtime + // benchmarks, which should switch to something similar to + // `get_compile_test_cases_with_measurements`. async fn collector_start(&self, aid: ArtifactIdNumber, steps: &[String]); // Returns `true` if the step was started, i.e., it did not previously have @@ -170,12 +150,7 @@ pub trait Connection: Send + Sync { async fn collector_remove_step(&self, aid: ArtifactIdNumber, step: &str); - async fn in_progress_artifacts(&self) -> Vec; - - async fn in_progress_steps(&self, aid: &ArtifactId) -> Vec; - - async fn last_n_artifact_collections(&self, n: u32) -> Vec; - + // TODO: the following two functions do not work anymore and should not be used! /// Returns the sha of the parent commit, if available. /// /// (Currently only works for try commits) diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index 09b32b459..86ef188a2 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -3,14 +3,14 @@ use crate::pool::{ }; use crate::selector::CompileTestCase; use crate::{ - ArtifactCollection, ArtifactId, ArtifactIdNumber, Benchmark, BenchmarkJob, - BenchmarkJobConclusion, BenchmarkJobKind, BenchmarkJobStatus, BenchmarkRequest, - BenchmarkRequestIndex, BenchmarkRequestInsertResult, BenchmarkRequestStatus, - BenchmarkRequestType, BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectionId, - CollectorConfig, Commit, CommitType, CompileBenchmark, Date, Index, PendingBenchmarkRequests, - Profile, QueuedCommit, Scenario, Target, BENCHMARK_JOB_STATUS_FAILURE_STR, - BENCHMARK_JOB_STATUS_IN_PROGRESS_STR, BENCHMARK_JOB_STATUS_QUEUED_STR, - BENCHMARK_JOB_STATUS_SUCCESS_STR, BENCHMARK_REQUEST_MASTER_STR, BENCHMARK_REQUEST_RELEASE_STR, + ArtifactId, ArtifactIdNumber, Benchmark, BenchmarkJob, BenchmarkJobConclusion, + BenchmarkJobKind, BenchmarkJobStatus, BenchmarkRequest, BenchmarkRequestIndex, + BenchmarkRequestInsertResult, BenchmarkRequestStatus, BenchmarkRequestType, + BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectionId, CollectorConfig, + Commit, CommitType, CompileBenchmark, Date, Index, PendingBenchmarkRequests, Profile, Scenario, + Target, BENCHMARK_JOB_STATUS_FAILURE_STR, BENCHMARK_JOB_STATUS_IN_PROGRESS_STR, + BENCHMARK_JOB_STATUS_QUEUED_STR, BENCHMARK_JOB_STATUS_SUCCESS_STR, + BENCHMARK_REQUEST_MASTER_STR, BENCHMARK_REQUEST_RELEASE_STR, BENCHMARK_REQUEST_STATUS_ARTIFACTS_READY_STR, BENCHMARK_REQUEST_STATUS_COMPLETED_STR, BENCHMARK_REQUEST_STATUS_IN_PROGRESS_STR, BENCHMARK_REQUEST_STATUS_WAITING_FOR_ARTIFACTS_STR, BENCHMARK_REQUEST_TRY_STR, @@ -514,8 +514,6 @@ pub struct CachedStatements { select_pstat_series: Statement, get_error: Statement, collection_id: Statement, - record_duration: Statement, - in_progress_steps: Statement, get_benchmarks: Statement, insert_runtime_pstat_series: Statement, select_runtime_pstat_series: Statement, @@ -637,35 +635,6 @@ impl PostgresConnection { insert_pstat_series: conn.prepare("insert into pstat_series (crate, profile, scenario, backend, target, metric) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(), select_pstat_series: conn.prepare("select id from pstat_series where crate = $1 and profile = $2 and scenario = $3 and backend = $4 and target = $5 and metric = $6").await.unwrap(), collection_id: conn.prepare("insert into collection (perf_commit) VALUES ($1) returning id").await.unwrap(), - record_duration: conn.prepare(" - insert into artifact_collection_duration ( - aid, - date_recorded, - duration - ) VALUES ($1, CURRENT_TIMESTAMP, $2) - ON CONFLICT DO NOTHING - ").await.unwrap(), - in_progress_steps: conn.prepare(" - select step, - end_time is not null, - extract(epoch from interval '0 seconds'::interval + - coalesce(end_time, statement_timestamp()) - start_time)::int4, - extract( - epoch from interval '0 seconds'::interval + - (select cp.end_time - cp.start_time - from collector_progress as cp - join artifact on artifact.id = cp.aid - where - cp.aid != $1 - and cp.step = collector_progress.step - and cp.start_time is not null - and cp.end_time is not null - and artifact.type = 'master' - order by start_time desc - limit 1 - ))::int4 - from collector_progress where aid = $1 order by step - ").await.unwrap(), get_benchmarks: conn.prepare(" select name, category from benchmark @@ -819,16 +788,6 @@ where }) } - async fn record_duration(&self, artifact: ArtifactIdNumber, duration: Duration) { - self.conn() - .execute( - &self.statements().record_duration, - &[&(artifact.0 as i32), &(duration.as_secs() as i32)], - ) - .await - .unwrap(); - } - async fn load_index(&mut self) -> Index { Index { commits: self @@ -989,86 +948,6 @@ where .map(|row| (row.get(0), row.get(1))) .collect() } - async fn queue_pr( - &self, - pr: u32, - include: Option<&str>, - exclude: Option<&str>, - runs: Option, - backends: Option<&str>, - ) { - if let Err(e) = self.conn() - .execute( - "insert into pull_request_build (pr, complete, requested, include, exclude, runs, backends) VALUES ($1, false, CURRENT_TIMESTAMP, $2, $3, $4, $5)", - &[&(pr as i32), &include, &exclude, &runs, &backends], - ) - .await { - log::error!("failed to queue_pr({}, {:?}, {:?}, {:?}, {:?}): {:?}", pr, include, exclude, runs, backends, e); - } - } - async fn pr_attach_commit( - &self, - pr: u32, - sha: &str, - parent_sha: &str, - commit_date: Option>, - ) -> bool { - self.conn() - .execute( - "update pull_request_build SET bors_sha = $1, parent_sha = $2, commit_date = $4 - where pr = $3 and bors_sha is null", - &[&sha, &parent_sha, &(pr as i32), &commit_date], - ) - .await - .unwrap() - > 0 - } - async fn queued_commits(&self) -> Vec { - let rows = self - .conn() - .query( - "select pr, bors_sha, parent_sha, include, exclude, runs, commit_date, backends from pull_request_build - where complete is false and bors_sha is not null - order by requested asc", - &[], - ) - .await - .unwrap(); - rows.into_iter() - .map(|row| QueuedCommit { - pr: row.get::<_, i32>(0) as u32, - sha: row.get(1), - parent_sha: row.get(2), - include: row.get(3), - exclude: row.get(4), - runs: row.get(5), - commit_date: row.get::<_, Option<_>>(6).map(Date), - backends: row.get(7), - }) - .collect() - } - async fn mark_complete(&self, sha: &str) -> Option { - let row = self - .conn() - .query_opt( - "update pull_request_build SET complete = true - where bors_sha = $1 - returning pr, bors_sha, parent_sha, include, exclude, runs, commit_date, backends", - &[&sha], - ) - .await - .unwrap()?; - Some(QueuedCommit { - pr: row.get::<_, i32>(0) as u32, - sha: row.get(1), - parent_sha: row.get(2), - include: row.get(3), - exclude: row.get(4), - runs: row.get(5), - commit_date: row.get::<_, Option<_>>(6).map(Date), - backends: row.get(7), - }) - } async fn collection_id(&self, version: &str) -> CollectionId { CollectionId( self.conn() @@ -1367,102 +1246,6 @@ where .await .unwrap(); } - async fn in_progress_artifacts(&self) -> Vec { - let rows = self - .conn() - .query( - "select distinct aid from collector_progress where end_time is null order by aid limit 1", - &[], - ) - .await - .unwrap(); - let aids = rows - .into_iter() - .map(|row| row.get::<_, i32>(0)) - .collect::>(); - - let mut artifacts = Vec::new(); - for aid in aids { - let row = self - .conn() - .query_one( - "select name, date, type from artifact where id = $1", - &[&aid], - ) - .await; - - let row = match row { - Ok(row) => row, - Err(err) => { - log::error!("skipping aid={} -- no such artifact: {:?}", aid, err); - continue; - } - }; - - let ty = row.get::<_, String>(2); - artifacts.push(match ty.as_str() { - "try" | "master" => ArtifactId::Commit(Commit { - sha: row.get(0), - date: row - .get::<_, Option<_>>(1) - .map(Date) - .unwrap_or_else(|| Date::ymd_hms(2001, 1, 1, 0, 0, 0)), - r#type: CommitType::from_str(&ty).unwrap(), - }), - "release" => ArtifactId::Tag(row.get(0)), - _ => { - log::error!("unknown ty {:?}", ty); - continue; - } - }); - } - artifacts - } - async fn in_progress_steps(&self, artifact: &ArtifactId) -> Vec { - let aid = self.artifact_id(artifact).await; - - let steps = self - .conn() - .query(&self.statements().in_progress_steps, &[&(aid.0 as i32)]) - .await - .unwrap(); - - steps - .into_iter() - .map(|row| crate::Step { - name: row.get(0), - is_done: row.get(1), - duration: Duration::from_secs(row.get::<_, Option>(2).unwrap_or(0) as u64), - expected: Duration::from_secs(row.get::<_, Option>(3).unwrap_or(0) as u64), - }) - .collect() - } - async fn last_n_artifact_collections(&self, n: u32) -> Vec { - self.conn() - .query( - "select art.name, art.date, art.type, acd.date_recorded, acd.duration \ - from artifact_collection_duration as acd \ - join artifact as art on art.id = acd.aid \ - order by date_recorded desc \ - limit $1;", - &[&(n as i64)], - ) - .await - .unwrap() - .into_iter() - .map(|r| { - let sha = r.get::<_, String>(0); - let date = r.get::<_, Option>>(1); - let ty = r.get::<_, String>(2); - - ArtifactCollection { - artifact: parse_artifact_id(&ty, &sha, date), - end_time: r.get(3), - duration: Duration::from_secs(r.get::<_, i32>(4) as u64), - } - }) - .collect() - } async fn parent_of(&self, sha: &str) -> Option { self.conn() .query_opt( @@ -1616,13 +1399,6 @@ where .execute("DELETE FROM artifact WHERE name = $1", &[&info.name]) .await .unwrap(); - self.conn() - .execute( - "DELETE FROM pull_request_build WHERE bors_sha = $1", - &[&info.name], - ) - .await - .unwrap(); self.conn() .execute( "DELETE FROM benchmark_request WHERE tag = $1", diff --git a/database/src/pool/sqlite.rs b/database/src/pool/sqlite.rs index 028507146..8872f08eb 100644 --- a/database/src/pool/sqlite.rs +++ b/database/src/pool/sqlite.rs @@ -3,13 +3,12 @@ use crate::pool::{ }; use crate::selector::CompileTestCase; use crate::{ - ArtifactCollection, ArtifactId, Benchmark, BenchmarkJob, BenchmarkJobConclusion, - BenchmarkJobKind, BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestInsertResult, - BenchmarkRequestStatus, BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectionId, - CollectorConfig, Commit, CommitType, CompileBenchmark, Date, PendingBenchmarkRequests, Profile, - Target, + ArtifactId, Benchmark, BenchmarkJob, BenchmarkJobConclusion, BenchmarkJobKind, + BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestInsertResult, BenchmarkRequestStatus, + BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectionId, CollectorConfig, + Commit, CommitType, CompileBenchmark, Date, PendingBenchmarkRequests, Profile, Target, }; -use crate::{ArtifactIdNumber, Index, QueuedCommit}; +use crate::{ArtifactIdNumber, Index}; use chrono::{DateTime, TimeZone, Utc}; use hashbrown::{HashMap, HashSet}; use rusqlite::params; @@ -680,15 +679,6 @@ impl Connection for SqliteConnection { Some(parse_artifact_id(ty.as_str(), artifact, date)) } - async fn record_duration(&self, artifact: ArtifactIdNumber, duration: Duration) { - self.raw_ref() - .prepare_cached( - "insert or ignore into artifact_collection_duration (aid, date_recorded, duration) VALUES (?, strftime('%s','now'), ?)", - ) - .unwrap() - .execute(params![artifact.0, duration.as_secs() as i64]) - .unwrap(); - } async fn collection_id(&self, version: &str) -> CollectionId { let raw = self.raw_ref(); raw.execute( @@ -997,98 +987,6 @@ impl Connection for SqliteConnection { .collect::>() .unwrap() } - async fn queue_pr( - &self, - pr: u32, - include: Option<&str>, - exclude: Option<&str>, - runs: Option, - backends: Option<&str>, - ) { - self.raw_ref() - .prepare_cached( - "insert into pull_request_build (pr, complete, requested, include, exclude, runs, backends) VALUES (?, 0, strftime('%s','now'), ?, ?, ?, ?)", - ) - .unwrap() - .execute(params![pr, include, exclude, &runs, backends]) - .unwrap(); - } - async fn pr_attach_commit( - &self, - pr: u32, - sha: &str, - parent_sha: &str, - commit_date: Option>, - ) -> bool { - let timestamp = commit_date.map(|d| d.timestamp()); - self.raw_ref() - .prepare_cached( - "update pull_request_build SET bors_sha = ?, parent_sha = ?, commit_date = ? - where pr = ? and bors_sha is null", - ) - .unwrap() - .execute(params![sha, parent_sha, timestamp, pr]) - .unwrap() - > 0 - } - async fn queued_commits(&self) -> Vec { - self.raw_ref() - .prepare_cached( - "select pr, bors_sha, parent_sha, include, exclude, runs, commit_date, backends from pull_request_build - where complete is false and bors_sha is not null - order by requested asc", - ) - .unwrap() - .query(params![]) - .unwrap() - .mapped(|row| { - Ok(QueuedCommit { - pr: row.get(0).unwrap(), - sha: row.get(1).unwrap(), - parent_sha: row.get(2).unwrap(), - include: row.get(3).unwrap(), - exclude: row.get(4).unwrap(), - runs: row.get(5).unwrap(), - commit_date: row.get::<_, Option>(6).unwrap().map(|timestamp| Date(DateTime::from_timestamp(timestamp, 0).unwrap())), - backends: row.get(7).unwrap(), - }) - }) - .collect::, _>>() - .unwrap() - } - async fn mark_complete(&self, sha: &str) -> Option { - let count = self - .raw_ref() - .execute( - "update pull_request_build SET complete = 1 where bors_sha = ? and complete = 0", - params![sha], - ) - .unwrap(); - if count == 0 { - return None; - } - assert_eq!(count, 1, "sha is unique column"); - self.raw_ref() - .query_row( - "select pr, bors_sha, parent_sha, include, exclude, runs, commit_date, backends from pull_request_build - where bors_sha = ?", - params![sha], - |row| { - Ok(QueuedCommit { - pr: row.get(0).unwrap(), - sha: row.get(1).unwrap(), - parent_sha: row.get(2).unwrap(), - include: row.get(3).unwrap(), - exclude: row.get(4).unwrap(), - runs: row.get(5).unwrap(), - commit_date: row.get::<_, Option>(6).unwrap().map(|timestamp| Date(DateTime::from_timestamp(timestamp, 0).unwrap())), - backends: row.get(7).unwrap(), - }) - }, - ) - .optional() - .unwrap() - } async fn collector_start(&self, aid: ArtifactIdNumber, steps: &[String]) { // Clean out any leftover unterminated steps. self.raw_ref() @@ -1136,114 +1034,6 @@ impl Connection for SqliteConnection { .unwrap(); } - async fn in_progress_artifacts(&self) -> Vec { - let conn = self.raw_ref(); - let mut aids = conn - .prepare( - "select distinct aid from collector_progress \ - where end is null order by aid limit 1", - ) - .unwrap(); - - let aids = aids.query_map(params![], |r| r.get::<_, i32>(0)).unwrap(); - - let mut artifacts = Vec::new(); - for aid in aids { - let aid = aid.unwrap(); - let (name, date, ty) = conn - .query_row( - "select name, date, type from artifact where id = ?", - params![&aid], - |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, Option>(1)?, - r.get::<_, String>(2)?, - )) - }, - ) - .unwrap(); - - artifacts.push(match ty.as_str() { - "try" | "master" => ArtifactId::Commit(Commit { - sha: name, - date: date - .map(|d| Utc.timestamp_opt(d, 0).unwrap()) - .map(Date) - .unwrap_or_else(|| Date::ymd_hms(2001, 1, 1, 0, 0, 0)), - r#type: CommitType::from_str(&ty).unwrap(), - }), - "release" => ArtifactId::Tag(name), - _ => { - log::error!("unknown ty {:?}", ty); - continue; - } - }); - } - artifacts - } - async fn in_progress_steps(&self, artifact: &ArtifactId) -> Vec { - let aid = self.artifact_id(artifact).await; - - self.raw_ref() - .prepare( - " - select - step, - end is not null, - coalesce(end, strftime('%s', 'now')) - start, - (select end - start - from collector_progress as cp - where - cp.step = collector_progress.step - and cp.start is not null - and cp.end is not null - limit 1 - ) - from collector_progress where aid = ? order by step - ", - ) - .unwrap() - .query_map(params![&aid.0], |row| { - Ok(crate::Step { - name: row.get(0)?, - is_done: row.get(1)?, - duration: Duration::from_secs(row.get::<_, i64>(2).unwrap_or_default() as u64), - expected: Duration::from_secs(row.get::<_, i64>(3).unwrap_or_default() as u64), - }) - }) - .unwrap() - .map(|r| r.unwrap()) - .collect() - } - - async fn last_n_artifact_collections(&self, n: u32) -> Vec { - self.raw_ref() - .prepare_cached( - "select art.name, art.date, art.type, acd.date_recorded, acd.duration \ - from artifact_collection_duration as acd \ - join artifact as art on art.id = acd.aid \ - order by date_recorded desc \ - limit ?;", - ) - .unwrap() - .query(params![&n]) - .unwrap() - .mapped(|r| { - let sha = r.get::<_, String>(0)?; - let date = r.get::<_, Option>(1)?; - let ty = r.get::<_, String>(2)?; - - Ok(ArtifactCollection { - artifact: parse_artifact_id(&ty, &sha, date), - end_time: Utc.timestamp_opt(r.get(3)?, 0).unwrap(), - duration: Duration::from_secs(r.get(4)?), - }) - }) - .collect::, _>>() - .unwrap() - } - async fn parent_of(&self, sha: &str) -> Option { let mut shas = self .raw_ref() @@ -1308,12 +1098,6 @@ impl Connection for SqliteConnection { self.raw_ref() .execute("delete from artifact where name = ?1", [info.name]) .unwrap(); - self.raw_ref() - .execute( - "delete from pull_request_build where bors_sha = ?1", - [info.name], - ) - .unwrap(); } async fn insert_benchmark_request( diff --git a/site/frontend/package.json b/site/frontend/package.json index 9e0b58b40..ba5aa900f 100644 --- a/site/frontend/package.json +++ b/site/frontend/package.json @@ -39,10 +39,6 @@ "source": "src/pages/status.ts", "distDir": "dist/scripts" }, - "status_old": { - "source": "src/pages/status_old.ts", - "distDir": "dist/scripts" - }, "bootstrap": { "source": "src/pages/bootstrap.ts", "distDir": "dist/scripts" diff --git a/site/frontend/src/pages/status_old.ts b/site/frontend/src/pages/status_old.ts deleted file mode 100644 index 8e8c5caa3..000000000 --- a/site/frontend/src/pages/status_old.ts +++ /dev/null @@ -1,8 +0,0 @@ -import Status from "./status_old/page.vue"; -import {createApp} from "vue"; -import WithSuspense from "../components/with-suspense.vue"; - -const app = createApp(WithSuspense, { - component: Status, -}); -app.mount("#app"); diff --git a/site/frontend/src/pages/status_old/data.ts b/site/frontend/src/pages/status_old/data.ts deleted file mode 100644 index 48371985e..000000000 --- a/site/frontend/src/pages/status_old/data.ts +++ /dev/null @@ -1,66 +0,0 @@ -export interface Commit { - sha: string; - date: string; - type: "Try" | "Master"; -} - -export interface BenchmarkError { - name: string; - error: string; -} - -interface Step { - step: string; - is_done: boolean; - expected_duration: number; - current_progress: number; -} - -export type Artifact = - | { - Commit: Commit; - } - | { - Tag: string; - }; - -export type MissingReason = - | { - Master: { - pr: number; - parent_sha: string; - is_try_parent: boolean; - }; - } - | { - Try: { - pr: number; - parent_sha: string; - include: string | null; - exclude: string | null; - runs: number | null; - backends: string | null; - }; - } - | { - InProgress: MissingReason; - }; - -interface CurrentState { - artifact: Artifact; - progress: Step[]; -} - -export interface FinishedRun { - artifact: Artifact; - pr: number | null; - errors: BenchmarkError[]; - duration: number; - finished_at: number; -} - -export interface StatusResponse { - finished_runs: FinishedRun[]; - current: CurrentState | null; - missing: Array<[Commit, MissingReason]>; -} diff --git a/site/frontend/src/pages/status_old/expansion.ts b/site/frontend/src/pages/status_old/expansion.ts deleted file mode 100644 index c5347b40b..000000000 --- a/site/frontend/src/pages/status_old/expansion.ts +++ /dev/null @@ -1,19 +0,0 @@ -import {ref} from "vue"; - -export function useExpandedStore() { - const expanded = ref(new Set()); - - function isExpanded(sha: string) { - return expanded.value.has(sha); - } - - function toggleExpanded(sha: string) { - if (isExpanded(sha)) { - expanded.value.delete(sha); - } else { - expanded.value.add(sha); - } - } - - return {toggleExpanded, isExpanded}; -} diff --git a/site/frontend/src/pages/status_old/page.vue b/site/frontend/src/pages/status_old/page.vue deleted file mode 100644 index 19c96e3c3..000000000 --- a/site/frontend/src/pages/status_old/page.vue +++ /dev/null @@ -1,500 +0,0 @@ - - - - - diff --git a/site/frontend/src/urls.ts b/site/frontend/src/urls.ts index c653f34e6..f2f629270 100644 --- a/site/frontend/src/urls.ts +++ b/site/frontend/src/urls.ts @@ -4,7 +4,6 @@ export const INFO_URL = `${BASE_URL}/info`; export const DASHBOARD_DATA_URL = `${BASE_URL}/dashboard`; export const STATUS_DATA_URL = `${BASE_URL}/status_page`; -export const STATUS_DATA_OLD_URL = `${BASE_URL}/status_page_old`; export const BOOTSTRAP_DATA_URL = `${BASE_URL}/bootstrap`; export const GRAPH_DATA_URL = `${BASE_URL}/graphs`; export const COMPARE_DATA_URL = `${BASE_URL}/get`; diff --git a/site/frontend/templates/layout.html b/site/frontend/templates/layout.html index 585fd6f11..31795bdd8 100644 --- a/site/frontend/templates/layout.html +++ b/site/frontend/templates/layout.html @@ -12,7 +12,7 @@ {% block content %}{% endblock %}
diff --git a/site/frontend/templates/pages/status_old.html b/site/frontend/templates/pages/status_old.html deleted file mode 100644 index 1715f6cd9..000000000 --- a/site/frontend/templates/pages/status_old.html +++ /dev/null @@ -1,10 +0,0 @@ -{% extends "layout.html" %} -{% block head %} - -{% endblock %} -{% block content %} -
-{% endblock %} -{% block script %} - -{% endblock %} diff --git a/site/src/api.rs b/site/src/api.rs index e964b6204..3b6e080d1 100644 --- a/site/src/api.rs +++ b/site/src/api.rs @@ -347,54 +347,6 @@ pub mod comparison { } pub mod status { - use crate::load::MissingReason; - use database::ArtifactId; - use database::Commit; - use serde::{Deserialize, Serialize}; - - #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] - pub struct BenchmarkError { - pub name: String, - pub error: String, - } - - #[derive(Serialize, Debug)] - pub struct Step { - pub step: String, - pub is_done: bool, - // Seconds - pub expected_duration: u64, - // Seconds since start - pub current_progress: u64, - } - - #[derive(Serialize, Debug)] - pub struct CurrentState { - pub artifact: ArtifactId, - pub progress: Vec, - } - - #[derive(Serialize, Debug)] - pub struct FinishedRun { - pub artifact: ArtifactId, - pub pr: Option, - pub errors: Vec, - // In seconds - pub duration: u64, - // Unix timestamp - pub finished_at: u64, - } - - #[derive(Serialize, Debug)] - pub struct Response { - // Ordered from newest to oldest - pub finished_runs: Vec, - pub current: Option, - pub missing: Vec<(Commit, MissingReason)>, - } -} - -pub mod status_new { use chrono::{DateTime, Utc}; use hashbrown::HashMap; use serde::Serialize; diff --git a/site/src/github.rs b/site/src/github.rs index 299f18e3c..f995ea3b3 100644 --- a/site/src/github.rs +++ b/site/src/github.rs @@ -2,7 +2,7 @@ pub mod client; pub mod comparison_summary; use crate::api::github::Commit; -use crate::job_queue::{build_queue, should_use_job_queue}; +use crate::job_queue::build_queue; use crate::load::{SiteCtxt, TryCommit}; use chrono::Utc; use serde::Deserialize; @@ -22,7 +22,6 @@ pub const COMMENT_MARK_TEMPORARY: &str = ""; /// Used for comment that contains unrolled commits for merged rolled-up PRs. pub const COMMENT_MARK_ROLLUP: &str = ""; -pub use comparison_summary::post_finished; use database::{BenchmarkJobStatus, BenchmarkRequestStatus, Connection}; /// Enqueues try build artifacts and posts a message about them on the original rollup PR @@ -260,24 +259,14 @@ pub async fn enqueue_shas( }; let conn = ctxt.conn().await; - let queued = if should_use_job_queue(pr_number) { - conn.attach_shas_to_try_benchmark_request( + let queued = conn.attach_shas_to_try_benchmark_request( pr_number, &try_commit.sha, &try_commit.parent_sha, commit_response.commit.committer.date, ) .await - .map_err(|error| format!("Cannot attach SHAs to try benchmark request on PR {pr_number} and SHA {}: {error:?}", try_commit.sha))? - } else { - conn.pr_attach_commit( - pr_number, - &try_commit.sha, - &try_commit.parent_sha, - Some(commit_response.commit.committer.date), - ) - .await - }; + .map_err(|error| format!("Cannot attach SHAs to try benchmark request on PR {pr_number} and SHA {}: {error:?}", try_commit.sha))?; if queued { if !msg.is_empty() { msg.push('\n'); diff --git a/site/src/github/comparison_summary.rs b/site/src/github/comparison_summary.rs index 00e132ff3..f62b2fb65 100644 --- a/site/src/github/comparison_summary.rs +++ b/site/src/github/comparison_summary.rs @@ -4,69 +4,12 @@ use crate::comparison::{ }; use crate::load::SiteCtxt; -use database::{metric::Metric, ArtifactId, QueuedCommit}; +use database::{metric::Metric, QueuedCommit}; use crate::github::{COMMENT_MARK_ROLLUP, COMMENT_MARK_TEMPORARY, RUST_REPO_GITHUB_API_URL}; use humansize::BINARY; -use std::collections::HashSet; use std::fmt::Write; -/// Post messages to GitHub for all queued commits that have -/// not yet been marked as completed. -pub async fn post_finished(ctxt: &SiteCtxt) { - // If the github token is not configured, do not run this -- we don't want - // to mark things as complete without posting the comment. - if ctxt.config.keys.github_api_token.is_none() { - return; - } - let conn = ctxt.conn().await; - let index = ctxt.index.load(); - let mut known_commits = index - .commits() - .into_iter() - .map(|c| c.sha) - .collect::>(); - let (master_commits, queued_pr_commits, in_progress_artifacts) = futures::join!( - collector::master_commits(), - conn.queued_commits(), - conn.in_progress_artifacts() - ); - let master_commits = match master_commits { - Ok(mcs) => mcs.into_iter().map(|c| c.sha).collect::>(), - Err(e) => { - log::error!("posting finished did not load master commits: {:?}", e); - // If we can't fetch master commits, return. - // We'll eventually try again later - return; - } - }; - - for aid in in_progress_artifacts { - match aid { - ArtifactId::Commit(c) => { - known_commits.remove(&c.sha); - } - ArtifactId::Tag(_) => { - // do nothing, for now, though eventually we'll want an artifact queue - } - } - } - for queued_commit in queued_pr_commits - .into_iter() - .filter(|c| known_commits.contains(&c.sha)) - { - if let Some(completed) = conn.mark_complete(&queued_commit.sha).await { - assert_eq!(completed, queued_commit); - - let is_master_commit = master_commits.contains(&queued_commit.sha); - if let Err(error) = post_comparison_comment(ctxt, queued_commit, is_master_commit).await - { - log::error!("Cannot post comparison comment: {error:?}"); - } - } - } -} - /// Posts a comment to GitHub summarizing the comparison of the queued commit with its parent /// /// `is_master_commit` is used to differentiate messages for try runs and post-merge runs. diff --git a/site/src/job_queue/mod.rs b/site/src/job_queue/mod.rs index bc323083d..9a03db0ff 100644 --- a/site/src/job_queue/mod.rs +++ b/site/src/job_queue/mod.rs @@ -1,8 +1,8 @@ mod utils; use crate::github::comparison_summary::post_comparison_comment; -use crate::job_queue::utils::{parse_release_string, ExtractIf}; -use crate::load::{partition_in_place, SiteCtxt}; +use crate::job_queue::utils::{parse_release_string, partition_in_place, ExtractIf}; +use crate::load::SiteCtxt; use anyhow::Context; use chrono::Utc; use collector::benchmark_set::{ @@ -18,17 +18,6 @@ use parking_lot::RwLock; use std::sync::Arc; use tokio::time::{self, Duration, MissedTickBehavior}; -pub fn is_job_queue_enabled() -> bool { - std::env::var("USE_JOB_QUEUE") - .ok() - .and_then(|x| x.parse().ok()) - .unwrap_or(true) -} - -pub fn should_use_job_queue(_pr: u32) -> bool { - is_job_queue_enabled() -} - /// Store the latest master commits or do nothing if all of them are /// already in the database. async fn create_benchmark_request_master_commits( diff --git a/site/src/job_queue/utils.rs b/site/src/job_queue/utils.rs index 3ade039c4..ca1fe00c7 100644 --- a/site/src/job_queue/utils.rs +++ b/site/src/job_queue/utils.rs @@ -84,6 +84,47 @@ pub fn parse_release_string(url: &str) -> Option<(String, DateTime)> { None } +// Copy of Iterator::partition_in_place, which is currently unstable. +pub fn partition_in_place<'a, I, T: 'a, P>(mut iter: I, mut predicate: P) -> usize +where + I: Sized + DoubleEndedIterator, + P: FnMut(&T) -> bool, +{ + // FIXME: should we worry about the count overflowing? The only way to have more than + // `usize::MAX` mutable references is with ZSTs, which aren't useful to partition... + + // These closure "factory" functions exist to avoid genericity in `Self`. + + #[inline] + fn is_false<'a, T>( + predicate: &'a mut impl FnMut(&T) -> bool, + true_count: &'a mut usize, + ) -> impl FnMut(&&mut T) -> bool + 'a { + move |x| { + let p = predicate(&**x); + *true_count += p as usize; + !p + } + } + + #[inline] + fn is_true(predicate: &mut impl FnMut(&T) -> bool) -> impl FnMut(&&mut T) -> bool + '_ { + move |x| predicate(&**x) + } + + // Repeatedly find the first `false` and swap it with the last `true`. + let mut true_count = 0; + while let Some(head) = iter.find(is_false(&mut predicate, &mut true_count)) { + if let Some(tail) = iter.rfind(is_true(&mut predicate)) { + std::mem::swap(head, tail); + true_count += 1; + } else { + break; + } + } + true_count +} + #[cfg(test)] mod tests { use super::*; diff --git a/site/src/load.rs b/site/src/load.rs index f48afb025..84f054c30 100644 --- a/site/src/load.rs +++ b/site/src/load.rs @@ -1,66 +1,19 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::fs; use std::ops::RangeInclusive; -use std::sync::{Arc, LazyLock}; +use std::sync::Arc; use std::time::Instant; use arc_swap::{ArcSwap, Guard}; -use chrono::{Duration, Utc}; use log::error; use parking_lot::Mutex; -use regex::Regex; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use crate::self_profile::SelfProfileCache; use collector::compile::benchmark::category::Category; use collector::{Bound, MasterCommit}; use database::Pool; pub use database::{ArtifactId, Benchmark, Commit}; -use database::{CommitType, Date}; - -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -pub enum MissingReason { - /// This commmit has not yet been benchmarked - Master { - pr: u32, - parent_sha: String, - is_try_parent: bool, - }, - Try { - pr: u32, - parent_sha: String, - include: Option, - exclude: Option, - runs: Option, - backends: Option, - }, - InProgress(Option>), -} - -impl MissingReason { - fn pr(&self) -> Option { - let mut this = self; - loop { - match this { - MissingReason::Master { pr, .. } => return Some(*pr), - MissingReason::Try { pr, .. } => return Some(*pr), - MissingReason::InProgress(Some(s)) => this = s, - MissingReason::InProgress(None) => return None, - } - } - } - fn parent_sha(&self) -> Option<&str> { - let mut this = self; - loop { - match this { - MissingReason::Master { parent_sha, .. } => return Some(parent_sha.as_str()), - MissingReason::Try { parent_sha, .. } => return Some(parent_sha.as_str()), - MissingReason::InProgress(Some(s)) => this = s, - MissingReason::InProgress(None) => return None, - } - } - } -} #[derive(Clone, Debug, PartialEq, Eq)] pub struct TryCommit { @@ -188,65 +141,6 @@ impl SiteCtxt { self.pool.connection().await } - /// Returns the not yet tested commits - pub async fn missing_commits(&self) -> Vec<(Commit, MissingReason)> { - let conn = self.conn().await; - let (queued_pr_commits, in_progress_artifacts) = - futures::join!(conn.queued_commits(), conn.in_progress_artifacts()); - let master_commits = &self.get_master_commits().commits; - - let index = self.index.load(); - let all_commits = index - .commits() - .iter() - .map(|commit| commit.sha.clone()) - .collect::>(); - - calculate_missing( - master_commits.clone(), - queued_pr_commits, - in_progress_artifacts, - all_commits, - ) - } - - /// Returns the not yet tested published artifacts, sorted from newest to oldest. - pub async fn missing_published_artifacts(&self) -> anyhow::Result> { - let artifact_list: String = reqwest::get("https://static.rust-lang.org/manifests.txt") - .await? - .text() - .await?; - - let conn = self.conn().await; - - let index = self.index.load(); - let tested_artifacts: HashSet<_> = index.artifacts().collect(); - let in_progress_tagged_artifacts: HashSet<_> = conn - .in_progress_artifacts() - .await - .into_iter() - .filter_map(|artifact| match artifact { - ArtifactId::Commit(_) => None, - ArtifactId::Tag(tag) => Some(tag), - }) - .collect(); - - // Gather at most last 20 published artifacts that are not yet tested and - // are not in progress. - let artifacts: Vec<_> = artifact_list - .lines() - .rev() - .filter_map(parse_published_artifact_tag) - .take(20) - .filter(|artifact| { - !tested_artifacts.contains(artifact.as_str()) - && !in_progress_tagged_artifacts.contains(artifact.as_str()) - }) - .collect(); - - Ok(artifacts) - } - pub async fn get_benchmark_category_map(&self) -> HashMap { let benchmarks = self.pool.connection().await.get_compile_benchmarks().await; benchmarks @@ -283,747 +177,3 @@ impl SiteCtxt { commits } } - -/// Parses an artifact tag like `1.63.0` or `beta-2022-08-19` from a line taken from -/// `https://static.rust-lang.org/manifests.txt`. -fn parse_published_artifact_tag(line: &str) -> Option { - static VERSION_REGEX: LazyLock = - LazyLock::new(|| Regex::new(r"(\d+\.\d+.\d+)").unwrap()); - - let mut parts = line.rsplit('/'); - let name = parts.next(); - let date = parts.next(); - - if let Some(date) = date { - if let Some(name) = name { - // Create beta artifact in the form of beta-YYYY-MM-DD - if name == "channel-rust-beta.toml" { - return Some(format!("beta-{date}")); - } else if name.contains("beta") { - // No other beta releases are recognized as toolchains. - // - // We also have names like this: - // - // * channel-rust-1.75-beta.toml - // * channel-rust-1.75.0-beta.toml - // * channel-rust-1.75.0-beta.1.toml - // - // Which should get ignored for now, they're not consumable via rustup yet. - return None; - } else if let Some(capture) = VERSION_REGEX.captures(name) { - if let Some(version) = capture.get(1).map(|c| c.as_str()) { - return Some(version.to_string()); - } - } - } - } - None -} - -/// Calculating the missing commits. -fn calculate_missing( - master_commits: Vec, - queued_pr_commits: Vec, - in_progress_artifacts: Vec, - all_commits: HashSet, -) -> Vec<(Commit, MissingReason)> { - calculate_missing_from( - master_commits, - queued_pr_commits, - in_progress_artifacts, - all_commits, - Utc::now(), - ) -} - -/// Calculate the missing commits filtering out any that are 29 days or older than the supplied time. -/// -/// This is used by `calculate_missing` is exists as a separate function for testing purposes. -fn calculate_missing_from( - master_commits: Vec, - queued_pr_commits: Vec, - in_progress_artifacts: Vec, - mut all_commits: HashSet, - time: chrono::DateTime, -) -> Vec<(Commit, MissingReason)> { - let mut queue = master_commits - .into_iter() - .filter(|c| time.signed_duration_since(c.time) < Duration::days(29)) - .map(|c| { - ( - Commit { - sha: c.sha, - date: Date(c.time), - r#type: CommitType::Master, - }, - // All recent master commits should have an associated PR - MissingReason::Master { - pr: c.pr.unwrap_or(0), - parent_sha: c.parent_sha, - is_try_parent: false, - }, - ) - }) - .collect::>(); - let master_commits = queue - .iter() - .map(|(commit, _)| commit.sha.clone()) - .collect::>(); - for database::QueuedCommit { - sha, - parent_sha, - pr, - include, - exclude, - runs, - commit_date, - backends, - } in queued_pr_commits - .into_iter() - // filter out any queued PR master commits (leaving only try commits) - .filter(|queued_commit| !master_commits.contains(&queued_commit.sha)) - { - // Mark the parent commit as a try_parent. - if let Some((_, missing_reason)) = queue - .iter_mut() - .find(|(commit, _)| commit.sha == parent_sha.as_str()) - { - /* Mutates the parent by scanning the list again... bad. */ - if let MissingReason::Master { is_try_parent, .. } = missing_reason { - *is_try_parent = true; - } else { - unreachable!("try commit has non-master parent {:?}", missing_reason); - }; - } - queue.push(( - Commit { - sha: sha.to_string(), - date: commit_date.unwrap_or(Date::empty()), - r#type: CommitType::Try, - }, - MissingReason::Try { - pr, - parent_sha, - include, - exclude, - runs, - backends, - }, - )); - } - for aid in in_progress_artifacts { - match aid { - ArtifactId::Commit(aid_commit) => { - let previous = queue - .iter() - .find(|(commit, _)| commit.sha == aid_commit.sha) - .map(|pair| Box::new(pair.1.clone())); - all_commits.remove(&aid_commit.sha); - queue.insert(0, (aid_commit, MissingReason::InProgress(previous))); - } - ArtifactId::Tag(_) => { - // do nothing, for now, though eventually we'll want an artifact queue - } - } - } - let mut already_tested = all_commits.clone(); - let mut i = 0; - while i != queue.len() { - if !already_tested.insert(queue[i].0.sha.clone()) { - queue.remove(i); - } else { - i += 1; - } - } - sort_queue(all_commits.clone(), queue) -} - -fn sort_queue( - mut done: HashSet, - mut unordered_queue: Vec<(Commit, MissingReason)>, -) -> Vec<(Commit, MissingReason)> { - // A topological sort, where each "level" is additionally altered such that - // try commits come first, and then sorted by PR # (as a rough heuristic for - // earlier requests). - - let mut finished = 0; - while finished < unordered_queue.len() { - // The next level is those elements in the unordered queue which - // are ready to be benchmarked (i.e., those with parent in done or no - // parent). - let level_len = partition_in_place(unordered_queue[finished..].iter_mut(), |(_, mr)| { - mr.parent_sha().is_none_or(|parent| done.contains(parent)) - }); - - // No commit is ready for benchmarking. This can happen e.g. when a try parent commit - // was forcefully removed from the master branch of rust-lang/rust. In this case, just - // let the commits be benchmarked in the current order that we have, these benchmark runs - // just won't have a parent result available. - if level_len == 0 { - return unordered_queue; - } - - let level = &mut unordered_queue[finished..][..level_len]; - level.sort_unstable_by_key(|(c, mr)| { - ( - // InProgress MR go first (false < true) - mr.parent_sha().is_some(), - mr.pr().unwrap_or(0), - c.sha.clone(), - ) - }); - for (c, _) in level { - done.insert(c.sha.clone()); - } - finished += level_len; - } - unordered_queue -} - -// Copy of Iterator::partition_in_place, which is currently unstable. -pub fn partition_in_place<'a, I, T: 'a, P>(mut iter: I, mut predicate: P) -> usize -where - I: Sized + DoubleEndedIterator, - P: FnMut(&T) -> bool, -{ - // FIXME: should we worry about the count overflowing? The only way to have more than - // `usize::MAX` mutable references is with ZSTs, which aren't useful to partition... - - // These closure "factory" functions exist to avoid genericity in `Self`. - - #[inline] - fn is_false<'a, T>( - predicate: &'a mut impl FnMut(&T) -> bool, - true_count: &'a mut usize, - ) -> impl FnMut(&&mut T) -> bool + 'a { - move |x| { - let p = predicate(&**x); - *true_count += p as usize; - !p - } - } - - #[inline] - fn is_true(predicate: &mut impl FnMut(&T) -> bool) -> impl FnMut(&&mut T) -> bool + '_ { - move |x| predicate(&**x) - } - - // Repeatedly find the first `false` and swap it with the last `true`. - let mut true_count = 0; - while let Some(head) = iter.find(is_false(&mut predicate, &mut true_count)) { - if let Some(tail) = iter.rfind(is_true(&mut predicate)) { - std::mem::swap(head, tail); - true_count += 1; - } else { - break; - } - } - true_count -} - -/// One decimal place rounded percent -#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] -pub struct Percent(#[serde(with = "collector::round_float")] pub f64); - -#[cfg(test)] -mod tests { - use std::str::FromStr; - - use collector::MasterCommit; - use database::QueuedCommit; - - use super::*; - - /// Create a master missing reason - fn create_missing_reason_master( - parent_sha: &str, - pr: u32, - is_try_parent: bool, - ) -> MissingReason { - MissingReason::Master { - pr, - parent_sha: parent_sha.into(), - is_try_parent, - } - } - - /// Create a try misssing reason - fn create_missing_reason_try(parent_sha: &str, pr: u32) -> MissingReason { - MissingReason::Try { - pr, - parent_sha: parent_sha.into(), - include: None, - exclude: None, - runs: None, - backends: None, - } - } - - /// Create a Commit - fn create_commit(commit_sha: &str, time: chrono::DateTime, r#type: CommitType) -> Commit { - Commit { - sha: commit_sha.into(), - date: database::Date(time), - r#type, - } - } - - /// Create a (Commit, MissingReason::Master) pair - fn create_commit_master_pair( - commit_sha: &str, - parent_sha: &str, - pr: u32, - time: chrono::DateTime, - is_try_parent: bool, - ) -> (Commit, MissingReason) { - ( - create_commit(commit_sha, time, CommitType::Master), - create_missing_reason_master(parent_sha, pr, is_try_parent), - ) - } - - /// Create a (Commit, MissingReason::Try) pair - fn create_commit_try_pair( - commit_sha: &str, - parent_sha: &str, - pr: u32, - time: chrono::DateTime, - ) -> (Commit, MissingReason) { - ( - create_commit(commit_sha, time, CommitType::Try), - create_missing_reason_try(parent_sha, pr), - ) - } - - /// Create a master commit - fn create_master_commit( - sha: &str, - parent_sha: &str, - pr: Option, - time: chrono::DateTime, - ) -> MasterCommit { - MasterCommit { - sha: sha.into(), - parent_sha: parent_sha.into(), - pr, - time, - } - } - - /// Create a queued commit - fn create_queued_commit(sha: &str, parent_sha: &str, pr: u32) -> QueuedCommit { - QueuedCommit { - sha: sha.into(), - parent_sha: parent_sha.into(), - pr, - include: None, - exclude: None, - runs: None, - commit_date: None, - backends: None, - } - } - - // Checks that when we have a setup like the following, where a -> b means b - // is the parent of a (i.e., must be tested before we can report comparison - // results for a): - // - // a -> b - // -> try-on-a - // - // the resulting ordering is: - // - // b - // a - // try-on-a - // - // which ensures that as each commit finishes, we have the results for it. - // - // Note that try-on-a does *not* have a direct dependency on b's results - // being available; we could order b after ([a, try-on-a, b]) but this means - // that we have to be more careful about posting comparison results, and to - // most observers they expect those posted as soon as the PR's build in the - // queue finishes: not doing so will look odd to onlookers. - #[test] - fn try_commit_ancestors() { - let time = chrono::DateTime::from_str("2021-09-01T00:00:00.000Z").unwrap(); - let master_commits = vec![ - MasterCommit { - sha: "a".into(), - parent_sha: "b".into(), - pr: Some(2), - time, - }, - MasterCommit { - sha: "b".into(), - parent_sha: "c".into(), - pr: Some(1), - time, - }, - ]; - let queued_pr_commits = vec![ - QueuedCommit { - sha: "try-on-a".into(), - parent_sha: "a".into(), - pr: 3, - include: None, - exclude: None, - runs: None, - commit_date: None, - backends: None, - }, - QueuedCommit { - sha: "b".into(), - parent_sha: "c".into(), - pr: 1, - include: None, - exclude: None, - runs: None, - commit_date: None, - backends: None, - }, - QueuedCommit { - sha: "a".into(), - parent_sha: "b".into(), - pr: 2, - include: None, - exclude: None, - runs: None, - commit_date: None, - backends: None, - }, - ]; - let in_progress_artifacts = vec![]; - let mut all_commits = HashSet::new(); - all_commits.insert("c".into()); - - let expected = vec![ - ( - Commit { - sha: "b".into(), - date: database::Date(time), - r#type: CommitType::Master, - }, - MissingReason::Master { - pr: 1, - parent_sha: "c".into(), - is_try_parent: false, - }, - ), - ( - Commit { - sha: "a".into(), - date: database::Date(time), - r#type: CommitType::Master, - }, - MissingReason::Master { - pr: 2, - parent_sha: "b".into(), - is_try_parent: true, - }, - ), - ( - Commit { - sha: "try-on-a".into(), - date: database::Date(time), - r#type: CommitType::Try, - }, - MissingReason::Try { - pr: 3, - parent_sha: "a".into(), - include: None, - exclude: None, - runs: None, - backends: None, - }, - ), - ]; - let found = calculate_missing_from( - master_commits, - queued_pr_commits, - in_progress_artifacts, - all_commits, - time, - ); - assert_eq!(expected, found, "{expected:#?} != {found:#?}"); - } - - #[test] - fn calculates_missing_correct() { - let time = chrono::DateTime::from_str("2021-09-01T00:00:00.000Z").unwrap(); - let master_commits = vec![ - // An already tested commit - create_master_commit("abc", "def", Some(90), time), - // A queued PR commit - create_master_commit("foo", "bar", Some(77), time), - // A not yet tested commit - create_master_commit("123", "345", Some(11), time), - ]; - let queued_pr_commits = vec![ - // A master commit - create_queued_commit("foo", "bar", 77), - // A try run - create_queued_commit("baz", "foo", 1), - // Another try run that will become the head of the queue - create_queued_commit("yee", "rrr", 4), - ]; - let in_progress_artifacts = vec![]; - let mut all_commits = HashSet::new(); - all_commits.insert("abc".to_string()); - // Parent trailers - all_commits.insert(master_commits[0].parent_sha.clone()); - all_commits.insert(master_commits[1].parent_sha.clone()); - all_commits.insert(master_commits[2].parent_sha.clone()); - all_commits.insert(queued_pr_commits[2].parent_sha.clone()); - - let expected = vec![ - create_commit_try_pair("yee", "rrr", 4, time), - create_commit_master_pair("123", "345", 11, time, false), - create_commit_master_pair("foo", "bar", 77, time, true), - create_commit_try_pair("baz", "foo", 1, time), - ]; - - let result = calculate_missing_from( - master_commits, - queued_pr_commits, - in_progress_artifacts, - all_commits, - time, - ); - assert_eq!(expected, result); - } - - #[test] - fn queue_empty() { - let time = chrono::DateTime::from_str("2021-09-01T00:00:00.000Z").unwrap(); - // Pathalogical case of an empty queue - let master_commits = vec![]; - let queued_pr_commits = vec![]; - let in_progress_artifacts = vec![]; - let all_commits = HashSet::new(); - - let expected: Vec<(Commit, MissingReason)> = vec![]; - - let result = calculate_missing_from( - master_commits, - queued_pr_commits, - in_progress_artifacts, - all_commits, - time, - ); - assert_eq!(expected, result); - } - - #[test] - fn only_master_commits_already_tested() { - // If all master commits are in the set then the queue is in effect empty - let time = chrono::DateTime::from_str("2021-09-01T00:00:00.000Z").unwrap(); - let master_commits = vec![ - create_master_commit("m1", "p1", Some(10), time), - create_master_commit("m2", "m1", Some(11), time), - create_master_commit("m3", "m2", Some(12), time), - ]; - let queued_pr_commits = vec![]; - let in_progress_artifacts = vec![]; - let mut all_commits = HashSet::new(); - - all_commits.insert("m1".to_string()); - all_commits.insert("m2".to_string()); - all_commits.insert("m3".to_string()); - all_commits.insert("p1".to_string()); - - let result = calculate_missing_from( - master_commits, - queued_pr_commits, - in_progress_artifacts, - all_commits, - time, - ); - - let expected: Vec<(Commit, MissingReason)> = vec![]; - assert_eq!(expected, result); - } - - #[test] - fn multiple_try_with_same_parent() { - let time = chrono::DateTime::from_str("2021-09-01T00:00:00.000Z").unwrap(); - let master_commits = vec![create_master_commit("abc", "def", Some(1), time)]; - // scramble the order - let queued_pr_commits = vec![ - create_queued_commit("try3", "abc", 203), - create_queued_commit("try1", "abc", 201), - create_queued_commit("try2", "abc", 202), - ]; - let in_progress_artifacts = vec![]; - let mut all_commits = HashSet::new(); - all_commits.insert("abc".to_string()); - all_commits.insert("def".to_string()); - - let result = calculate_missing_from( - master_commits, - queued_pr_commits, - in_progress_artifacts, - all_commits, - time, - ); - - let expected = vec![ - create_commit_try_pair("try1", "abc", 201, time), - create_commit_try_pair("try2", "abc", 202, time), - create_commit_try_pair("try3", "abc", 203, time), - ]; - assert_eq!(expected, result); - } - - #[test] - fn in_progress_commit() { - // InProgress commits should get inserted into the queue - let time = chrono::DateTime::from_str("2021-09-01T00:00:00.000Z").unwrap(); - let master_commits = vec![create_master_commit("abc", "def", Some(50), time)]; - let queued_pr_commits = vec![]; - let in_progress_artifacts = vec![ArtifactId::from(create_commit( - "abc", - time, - CommitType::Master, - ))]; - let mut all_commits = HashSet::new(); - all_commits.insert("def".to_string()); - - let result = calculate_missing_from( - master_commits, - queued_pr_commits, - in_progress_artifacts, - all_commits, - time, - ); - - let expected = vec![( - create_commit("abc", time, CommitType::Master), - MissingReason::InProgress(Some(Box::new(create_missing_reason_master( - "def", 50, false, - )))), - )]; - assert_eq!(expected, result); - } - - #[test] - fn orphan_commit() { - let time = chrono::DateTime::from_str("2021-09-01T00:00:00.000Z").unwrap(); - let master_commits = vec![create_master_commit("x123", "ORPHAN", Some(69), time)]; - let queued_pr_commits = vec![]; - let in_progress_artifacts = vec![]; - let all_commits = HashSet::new(); - - let result = calculate_missing_from( - master_commits, - queued_pr_commits, - in_progress_artifacts, - all_commits, - time, - ); - - let expected = vec![create_commit_master_pair("x123", "ORPHAN", 69, time, false)]; - assert_eq!(expected, result); - } - - #[test] - fn artifact_tags() { - // ArtifactId with a type of `Tag` should be ignored - let time = chrono::DateTime::from_str("2021-09-01T00:00:00.000Z").unwrap(); - let master_commits = vec![create_master_commit("abc", "def", Some(50), time)]; - let queued_pr_commits = vec![]; - let in_progress_artifacts = vec![ArtifactId::Tag("nightly-2025-04-10".to_string())]; - let mut all_commits = HashSet::new(); - all_commits.insert("def".to_string()); - - let result = calculate_missing_from( - master_commits, - queued_pr_commits, - in_progress_artifacts, - all_commits, - time, - ); - - let expected = vec![create_commit_master_pair("abc", "def", 50, time, false)]; - assert_eq!(expected, result); - } - - #[test] - fn duplicate_commits() { - let time = chrono::DateTime::from_str("2021-09-01T00:00:00.000Z").unwrap(); - let master_commits = vec![ - create_master_commit("dup1", "m1", Some(69), time), - create_master_commit("dup1", "m1", Some(69), time), - ]; - let queued_pr_commits = vec![ - create_queued_commit("trydup", "dup1", 100), - create_queued_commit("trydup", "dup1", 100), - ]; - let in_progress_artifacts = vec![]; - let all_commits = HashSet::new(); - - let result = calculate_missing_from( - master_commits, - queued_pr_commits, - in_progress_artifacts, - all_commits, - time, - ); - - let expected = vec![ - create_commit_master_pair("dup1", "m1", 69, time, true), - create_commit_try_pair("trydup", "dup1", 100, time), - ]; - assert_eq!(expected, result); - } - - #[test] - fn parse_published_beta_artifact() { - assert_eq!( - parse_published_artifact_tag( - "static.rust-lang.org/dist/2022-08-15/channel-rust-beta.toml" - ), - Some("beta-2022-08-15".to_string()) - ); - } - - #[test] - fn parse_published_stable_artifact() { - assert_eq!( - parse_published_artifact_tag( - "static.rust-lang.org/dist/2022-08-15/channel-rust-1.63.0.toml" - ), - Some("1.63.0".to_string()) - ); - } - - #[test] - fn parse_published_beta_non_rustup_1() { - assert_eq!( - parse_published_artifact_tag( - "static.rust-lang.org/dist/2023-11-13/channel-rust-1.75-beta.toml" - ), - None - ); - } - - #[test] - fn parse_published_beta_non_rustup_2() { - assert_eq!( - parse_published_artifact_tag( - "static.rust-lang.org/dist/2023-11-13/channel-rust-1.75.0-beta.toml" - ), - None - ); - } - - #[test] - fn parse_published_beta_non_rustup_3() { - assert_eq!( - parse_published_artifact_tag( - "static.rust-lang.org/dist/2023-11-13/channel-rust-1.75.0-beta.1.toml" - ), - None - ); - } -} diff --git a/site/src/main.rs b/site/src/main.rs index c2e89a3ea..c149c494c 100644 --- a/site/src/main.rs +++ b/site/src/main.rs @@ -1,6 +1,6 @@ use futures::future::FutureExt; use parking_lot::RwLock; -use site::job_queue::{create_job_queue_process, is_job_queue_enabled}; +use site::job_queue::create_job_queue_process; use site::load; use std::env; use std::sync::Arc; @@ -49,9 +49,6 @@ async fn main() { eprintln!( "View the results in a web browser at 'http://localhost:{port}/compare.html'" ); - // Spawn off a task to post the results of any commit results that we - // are now aware of. - site::github::post_finished(&res).await; }) }) .fuse(); @@ -61,12 +58,8 @@ async fn main() { let create_job_queue_handler = |ctxt: Arc>>>| { task::spawn(async move { - if is_job_queue_enabled() { - create_job_queue_process(ctxt, Duration::from_secs(queue_update_interval_seconds)) - .await; - } else { - futures::future::pending::<()>().await; - } + create_job_queue_process(ctxt, Duration::from_secs(queue_update_interval_seconds)) + .await; }) .fuse() }; diff --git a/site/src/request_handlers.rs b/site/src/request_handlers.rs index d96ca8213..740999333 100644 --- a/site/src/request_handlers.rs +++ b/site/src/request_handlers.rs @@ -2,10 +2,8 @@ mod bootstrap; mod dashboard; mod github; mod graph; -mod next_artifact; mod self_profile; mod status_page; -mod status_page_new; pub use bootstrap::handle_bootstrap; pub use dashboard::handle_dashboard; @@ -14,13 +12,11 @@ pub use graph::{ handle_compile_detail_graphs, handle_compile_detail_sections, handle_graphs, handle_runtime_detail_graphs, }; -pub use next_artifact::handle_next_artifact; pub use self_profile::{ handle_self_profile, handle_self_profile_processed_download, handle_self_profile_raw, handle_self_profile_raw_download, }; -pub use status_page::handle_status_page_old; -pub use status_page_new::handle_status_page; +pub use status_page::handle_status_page; use crate::api::{info, ServerResult}; use crate::load::SiteCtxt; diff --git a/site/src/request_handlers/github.rs b/site/src/request_handlers/github.rs index 5aa1feb7f..46815beda 100644 --- a/site/src/request_handlers/github.rs +++ b/site/src/request_handlers/github.rs @@ -3,7 +3,6 @@ use crate::github::{ client, enqueue_shas, parse_homu_comment, rollup_pr_number, unroll_rollup, COMMENT_MARK_TEMPORARY, RUST_REPO_GITHUB_API_URL, }; -use crate::job_queue::should_use_job_queue; use crate::load::SiteCtxt; use database::{ @@ -79,16 +78,6 @@ async fn handle_issue( Ok(github::Response) } -fn get_awaiting_on_bors_message() -> String { - format!( - "Awaiting bors try build completion. - -@rustbot label: +S-waiting-on-perf - -{COMMENT_MARK_TEMPORARY}" - ) -} - /// The try request does not have a `sha` or a `parent_sha` but we need to keep a record /// of this commit existing. The DB ensures that there is only one non-completed /// try benchmark request per `pr`. @@ -114,7 +103,15 @@ async fn record_try_benchmark_request_without_artifacts( {COMMENT_MARK_TEMPORARY}" ) } - Ok(BenchmarkRequestInsertResult::Inserted) => get_awaiting_on_bors_message(), + Ok(BenchmarkRequestInsertResult::Inserted) => { + format!( + "Awaiting bors try build completion. + +@rustbot label: +S-waiting-on-perf + +{COMMENT_MARK_TEMPORARY}" + ) + } Err(e) => { log::error!("Failed to insert release benchmark request: {e}"); "Something went wrong! This is most likely an internal failure, please let us know on [Zulip](https://rust-lang.zulipchat.com/#narrow/channel/242791-t-infra)".to_string() @@ -153,26 +150,14 @@ async fn handle_rust_timer( Ok(cmd) => { let conn = ctxt.conn().await; - if should_use_job_queue(issue.number) { - record_try_benchmark_request_without_artifacts( - &*conn, - issue.number, - cmd.params.backends.unwrap_or(""), - cmd.params.profiles.unwrap_or(""), - cmd.params.targets.unwrap_or(""), - ) - .await - } else { - conn.queue_pr( - issue.number, - cmd.params.include, - cmd.params.exclude, - cmd.params.runs, - cmd.params.backends, - ) - .await; - get_awaiting_on_bors_message() - } + record_try_benchmark_request_without_artifacts( + &*conn, + issue.number, + cmd.params.backends.unwrap_or(""), + cmd.params.profiles.unwrap_or(""), + cmd.params.targets.unwrap_or(""), + ) + .await } Err(error) => { format!("Error occurred while parsing comment: {error}") @@ -203,25 +188,14 @@ async fn handle_rust_timer( { let conn = ctxt.conn().await; for command in &valid_build_cmds { - if should_use_job_queue(issue.number) { - record_try_benchmark_request_without_artifacts( - &*conn, - issue.number, - command.params.backends.unwrap_or(""), - command.params.profiles.unwrap_or(""), - command.params.targets.unwrap_or(""), - ) - .await; - } else { - conn.queue_pr( - issue.number, - command.params.include, - command.params.exclude, - command.params.runs, - command.params.backends, - ) - .await; - } + record_try_benchmark_request_without_artifacts( + &*conn, + issue.number, + command.params.backends.unwrap_or(""), + command.params.profiles.unwrap_or(""), + command.params.targets.unwrap_or(""), + ) + .await; } } @@ -383,7 +357,9 @@ struct BuildCommand<'a> { #[derive(Debug)] struct BenchmarkParameters<'a> { + #[allow(unused)] include: Option<&'a str>, + #[allow(unused)] exclude: Option<&'a str>, runs: Option, backends: Option<&'a str>, diff --git a/site/src/request_handlers/next_artifact.rs b/site/src/request_handlers/next_artifact.rs deleted file mode 100644 index 2eea831a8..000000000 --- a/site/src/request_handlers/next_artifact.rs +++ /dev/null @@ -1,84 +0,0 @@ -use crate::load::{MissingReason, SiteCtxt}; -use collector::api::next_artifact; - -use std::sync::Arc; - -pub async fn handle_next_artifact(ctxt: Arc) -> next_artifact::Response { - // Prefer benchmarking released artifacts first - match ctxt.missing_published_artifacts().await { - Ok(next_artifact) => { - if let Some(next_artifact) = next_artifact.into_iter().next() { - log::debug!("next_artifact: {next_artifact}"); - return next_artifact::Response { - artifact: Some(next_artifact::NextArtifact::Release(next_artifact)), - }; - } - } - Err(error) => log::error!("Failed to fetch missing artifacts: {error:?}"), - } - - let next_commit = ctxt.missing_commits().await.into_iter().next(); - - let next_commit = if let Some((commit, missing_reason)) = next_commit { - let missing_reason_dbg = format!("{missing_reason:?}"); - // If we're going to run a master commit next, make sure - // it's been enqueued in the pull_request_build table - if let MissingReason::Master { - pr, ref parent_sha, .. - } = missing_reason - { - let conn = ctxt.conn().await; - // TODO: add capability of doing the following in one step - // to avoid possibile illegal inbetween states. - conn.queue_pr(pr, None, None, None, None).await; - if !conn - .pr_attach_commit(pr, &commit.sha, parent_sha, None) - .await - { - log::error!("failed to attach commit {} to PR queue", commit.sha); - } - } - let (include, exclude, runs, backends) = match missing_reason { - crate::load::MissingReason::Try { - include, - exclude, - runs, - backends, - .. - } => (include, exclude, runs, backends), - crate::load::MissingReason::InProgress(Some(previous)) => { - if let crate::load::MissingReason::Try { - include, - exclude, - runs, - backends, - .. - } = *previous - { - (include, exclude, runs, backends) - } else { - (None, None, None, None) - } - } - _ => (None, None, None, None), - }; - log::debug!( - "next_commit: {} (missing: {})", - commit.sha, - missing_reason_dbg - ); - Some(next_artifact::NextArtifact::Commit { - commit, - include, - exclude, - runs, - backends, - }) - } else { - None - }; - - next_artifact::Response { - artifact: next_commit, - } -} diff --git a/site/src/request_handlers/status_page.rs b/site/src/request_handlers/status_page.rs index e5f6035f0..83728a4cb 100644 --- a/site/src/request_handlers/status_page.rs +++ b/site/src/request_handlers/status_page.rs @@ -1,83 +1,252 @@ -use std::str; -use std::sync::Arc; - use crate::api::status; -use crate::api::status::FinishedRun; +use crate::job_queue::build_queue; use crate::load::SiteCtxt; +use chrono::{DateTime, Utc}; +use database::{ + BenchmarkJob, BenchmarkJobStatus, BenchmarkRequest, BenchmarkRequestStatus, + BenchmarkRequestType, Connection, +}; +use hashbrown::HashMap; +use std::sync::Arc; +use std::time::Duration; -use database::{ArtifactId, Lookup}; +pub async fn handle_status_page(ctxt: Arc) -> anyhow::Result { + let conn = ctxt.conn().await; -// How many historical (finished) runs should be returned from the status API. -const FINISHED_RUN_COUNT: u64 = 5; + // The queue contains any in-progress request(s) and then the following requests in queue order + let queue = build_queue(&*conn).await?; + let completed = conn.get_last_n_completed_benchmark_requests(10).await?; -pub async fn handle_status_page_old(ctxt: Arc) -> status::Response { - let missing = ctxt.missing_commits().await; + // Figure out approximately how long was the most recent master benchmark request + let expected_duration = completed + .iter() + .filter(|req| req.request.is_master() && req.errors.is_empty()) + .filter_map(|req| match req.request.status() { + BenchmarkRequestStatus::Completed { duration, .. } => Some(duration), + _ => None, + }) + .next() + .unwrap_or(Duration::from_secs(3600)); - // FIXME: no current builds - let conn = ctxt.conn().await; - let current = if let Some(artifact) = conn.in_progress_artifacts().await.pop() { - let steps = conn - .in_progress_steps(&artifact) - .await - .into_iter() - .map(|s| crate::api::status::Step { - step: s.name, - is_done: s.is_done, - expected_duration: s.expected.as_secs(), - current_progress: s.duration.as_secs(), - }) - .collect(); + let in_progress_jobs = conn.get_jobs_of_in_progress_benchmark_requests().await?; - Some(crate::api::status::CurrentState { - artifact, - progress: steps, - }) + // Here we compute the estimated end time for queued requests, and convert the requests to their + // frontend representation. + // We assume that at most a single request is in progress + + let now = Utc::now(); + + // The estimated start time of the current in-progress request + let current_request_start = if let Some(req) = queue.first().take_if(|req| req.is_in_progress()) + { + // Here we need to somehow guess when did the current in-progress request actually start, + // as we do not have that information readily available + let request_jobs = in_progress_jobs + .get(req.tag().expect("In progress request without a tag")) + .map(|jobs| jobs.as_slice()) + .unwrap_or(&[]); + + // Take the earliest start time, if some job has already started + // If there are no started jobs yet, just fall back to the current time (we guess that a + // job will start "any time now") + request_jobs + .iter() + .filter_map(|job| match job.status() { + BenchmarkJobStatus::Queued => None, + BenchmarkJobStatus::InProgress { started_at, .. } + | BenchmarkJobStatus::Completed { started_at, .. } => Some(*started_at), + }) + .min() + .unwrap_or(now) } else { - None + // Assume that the next request (if any) will start at any given moment + now }; - // FIXME: load at least one master commit with errors, if no master commit is in last N commits? - // FIXME: cache this whole thing, or write a specific SQL query for it - let mut finished_runs = Vec::new(); - let idx = ctxt.index.load(); - - let recent_collections = conn - .last_n_artifact_collections(FINISHED_RUN_COUNT as u32) - .await; - for collection in recent_collections { - let errors = conn - .get_error(collection.artifact.lookup(&idx).unwrap()) - .await; - let mut pr = None; - if let ArtifactId::Commit(ref commit) = collection.artifact { - pr = conn.pr_of(&commit.sha).await; + // Estimate when the current in-progress request should end + // This ignores the fact that different kinds of requests (e.g. release ones) can have different + // durations, but these are rare and it's not worth the complexity to have multiple estimates + // here. + let current_request_end = current_request_start + expected_duration; + + let mut requests: Vec = queue + .into_iter() + .enumerate() + .map(|(index, req)| { + let estimated_end = if req.is_in_progress() { + current_request_end + } else { + current_request_end + expected_duration * (index as u32) + }; + request_to_ui(&req, HashMap::default(), Some(estimated_end)) + }) + .collect(); + + // We reverse the queued requests so that they start with the request that will be benchmarked the latest + requests.reverse(); + // And then we add the completed requests + requests.extend( + completed + .into_iter() + .map(|req| request_to_ui(&req.request, req.errors, None)), + ); + + let collectors = build_collectors(conn.as_ref(), &in_progress_jobs).await?; + + Ok(status::Response { + requests, + collectors, + }) +} + +async fn build_collectors( + conn: &dyn Connection, + in_progress_jobs: &HashMap>, +) -> anyhow::Result> { + let collectors = conn.get_collector_configs().await?; + let mut collector_map: HashMap = collectors + .into_iter() + .map(|c| { + ( + c.name().to_string(), + status::Collector { + name: c.name().to_string(), + target: c.target().to_string(), + benchmark_set: c.benchmark_set().get_id(), + is_active: c.is_active(), + last_heartbeat_at: c.last_heartbeat_at(), + date_added: c.date_added(), + jobs: vec![], + }, + ) + }) + .collect(); + + // This map is used to guess a future collector for jobs that haven't been dequeued yet + // (target, benchmark_set) -> collector name + let collector_guess_map: HashMap<(String, u32), String> = collector_map + .iter() + .map(|(name, collector)| { + ( + (collector.target.to_string(), collector.benchmark_set), + name.clone(), + ) + }) + .collect(); + + // Map jobs to collectors. Even if a collector has not started a job yet, we can guess if it + // will execute it or not, based on its target and benchmark set + for job in in_progress_jobs.values().flatten() { + let Some(collector_name) = job.collector_name().or_else(|| { + collector_guess_map + .get(&(job.target().to_string(), job.benchmark_set().get_id())) + .map(|n| n.as_str()) + }) else { + continue; + }; + if let Some(collector) = collector_map.get_mut(collector_name) { + collector.jobs.push(job_to_ui(job)); } - finished_runs.push(FinishedRun { - artifact: collection.artifact, - pr, - errors: errors - .into_iter() - .map(|(name, error)| { - let error = prettify_log(&error).unwrap_or(error); - status::BenchmarkError { name, error } + } + let mut collectors: Vec = collector_map.into_values().collect(); + collectors.sort_by(|c1, c2| c1.name.cmp(&c2.name)); + for collector in &mut collectors { + collector.jobs.sort_by(|j1, j2| { + let prio1 = job_status_to_priority(j1.status); + let prio2 = job_status_to_priority(j2.status); + prio1 + .cmp(&prio2) + .then(j1.deque_counter.cmp(&j2.deque_counter).reverse()) + .then_with(|| { + (&j1.target, &j1.backend, &j1.profile, j1.benchmark_set).cmp(&( + &j2.target, + &j2.backend, + &j2.profile, + j2.benchmark_set, + )) }) - .collect::>(), - duration: collection.duration.as_secs(), - finished_at: collection.end_time.timestamp() as u64, }); } - status::Response { - finished_runs, - missing, - current, + Ok(collectors) +} + +fn job_status_to_priority(status: status::BenchmarkJobStatus) -> u32 { + match status { + status::BenchmarkJobStatus::InProgress => 0, + status::BenchmarkJobStatus::Queued => 1, + status::BenchmarkJobStatus::Failed => 2, + status::BenchmarkJobStatus::Success => 3, + } +} + +fn request_to_ui( + req: &BenchmarkRequest, + errors: HashMap, + estimated_end: Option>, +) -> status::BenchmarkRequest { + let (completed_at, duration_s) = match req.status() { + BenchmarkRequestStatus::WaitingForArtifacts => (estimated_end, None), + BenchmarkRequestStatus::ArtifactsReady => (estimated_end, None), + BenchmarkRequestStatus::InProgress => (estimated_end, None), + BenchmarkRequestStatus::Completed { + completed_at, + duration, + } => (Some(completed_at), Some(duration.as_secs())), + }; + status::BenchmarkRequest { + tag: req.tag().expect("Missing request tag").to_string(), + pr: req.pr(), + status: match req.status() { + BenchmarkRequestStatus::WaitingForArtifacts => unreachable!(), + BenchmarkRequestStatus::ArtifactsReady => status::BenchmarkRequestStatus::Queued, + BenchmarkRequestStatus::InProgress => status::BenchmarkRequestStatus::InProgress, + BenchmarkRequestStatus::Completed { .. } => status::BenchmarkRequestStatus::Completed, + }, + request_type: match req.commit_type() { + BenchmarkRequestType::Try { .. } => status::BenchmarkRequestType::Try, + BenchmarkRequestType::Master { .. } => status::BenchmarkRequestType::Master, + BenchmarkRequestType::Release { .. } => status::BenchmarkRequestType::Release, + }, + created_at: req.created_at(), + completed_at, + duration_s, + errors, + end_estimated: estimated_end.is_some(), } } -fn prettify_log(log: &str) -> Option { - let mut lines = log.lines(); - let first = lines.next()?; - let log = &first[first.find('"')? + 1..]; - let log = &log[..log.find("\" }")?]; - Some(log.replace("\\n", "\n")) +fn job_to_ui(job: &BenchmarkJob) -> status::BenchmarkJob { + let (started_at, completed_at) = match job.status() { + BenchmarkJobStatus::Queued => (None, None), + BenchmarkJobStatus::InProgress { started_at, .. } => (Some(*started_at), None), + BenchmarkJobStatus::Completed { + started_at, + completed_at, + .. + } => (Some(*started_at), Some(*completed_at)), + }; + + status::BenchmarkJob { + request_tag: job.request_tag().to_string(), + kind: job.kind().to_string(), + target: job.target().as_str().to_string(), + backend: job.backend().as_str().to_string(), + profile: job.profile().as_str().to_string(), + benchmark_set: job.benchmark_set().get_id(), + created_at: job.created_at(), + started_at, + completed_at, + status: match job.status() { + BenchmarkJobStatus::Queued => status::BenchmarkJobStatus::Queued, + BenchmarkJobStatus::InProgress { .. } => status::BenchmarkJobStatus::InProgress, + BenchmarkJobStatus::Completed { success: true, .. } => { + status::BenchmarkJobStatus::Success + } + BenchmarkJobStatus::Completed { success: false, .. } => { + status::BenchmarkJobStatus::Failed + } + }, + deque_counter: job.deque_count(), + } } diff --git a/site/src/request_handlers/status_page_new.rs b/site/src/request_handlers/status_page_new.rs deleted file mode 100644 index b141ba952..000000000 --- a/site/src/request_handlers/status_page_new.rs +++ /dev/null @@ -1,254 +0,0 @@ -use crate::api::status_new; -use crate::job_queue::build_queue; -use crate::load::SiteCtxt; -use chrono::{DateTime, Utc}; -use database::{ - BenchmarkJob, BenchmarkJobStatus, BenchmarkRequest, BenchmarkRequestStatus, - BenchmarkRequestType, Connection, -}; -use hashbrown::HashMap; -use std::sync::Arc; -use std::time::Duration; - -pub async fn handle_status_page(ctxt: Arc) -> anyhow::Result { - let conn = ctxt.conn().await; - - // The queue contains any in-progress request(s) and then the following requests in queue order - let queue = build_queue(&*conn).await?; - let completed = conn.get_last_n_completed_benchmark_requests(10).await?; - - // Figure out approximately how long was the most recent master benchmark request - let expected_duration = completed - .iter() - .filter(|req| req.request.is_master() && req.errors.is_empty()) - .filter_map(|req| match req.request.status() { - BenchmarkRequestStatus::Completed { duration, .. } => Some(duration), - _ => None, - }) - .next() - .unwrap_or(Duration::from_secs(3600)); - - let in_progress_jobs = conn.get_jobs_of_in_progress_benchmark_requests().await?; - - // Here we compute the estimated end time for queued requests, and convert the requests to their - // frontend representation. - // We assume that at most a single request is in progress - - let now = Utc::now(); - - // The estimated start time of the current in-progress request - let current_request_start = if let Some(req) = queue.first().take_if(|req| req.is_in_progress()) - { - // Here we need to somehow guess when did the current in-progress request actually start, - // as we do not have that information readily available - let request_jobs = in_progress_jobs - .get(req.tag().expect("In progress request without a tag")) - .map(|jobs| jobs.as_slice()) - .unwrap_or(&[]); - - // Take the earliest start time, if some job has already started - // If there are no started jobs yet, just fall back to the current time (we guess that a - // job will start "any time now") - request_jobs - .iter() - .filter_map(|job| match job.status() { - BenchmarkJobStatus::Queued => None, - BenchmarkJobStatus::InProgress { started_at, .. } - | BenchmarkJobStatus::Completed { started_at, .. } => Some(*started_at), - }) - .min() - .unwrap_or(now) - } else { - // Assume that the next request (if any) will start at any given moment - now - }; - - // Estimate when the current in-progress request should end - // This ignores the fact that different kinds of requests (e.g. release ones) can have different - // durations, but these are rare and it's not worth the complexity to have multiple estimates - // here. - let current_request_end = current_request_start + expected_duration; - - let mut requests: Vec = queue - .into_iter() - .enumerate() - .map(|(index, req)| { - let estimated_end = if req.is_in_progress() { - current_request_end - } else { - current_request_end + expected_duration * (index as u32) - }; - request_to_ui(&req, HashMap::default(), Some(estimated_end)) - }) - .collect(); - - // We reverse the queued requests so that they start with the request that will be benchmarked the latest - requests.reverse(); - // And then we add the completed requests - requests.extend( - completed - .into_iter() - .map(|req| request_to_ui(&req.request, req.errors, None)), - ); - - let collectors = build_collectors(conn.as_ref(), &in_progress_jobs).await?; - - Ok(status_new::Response { - requests, - collectors, - }) -} - -async fn build_collectors( - conn: &dyn Connection, - in_progress_jobs: &HashMap>, -) -> anyhow::Result> { - let collectors = conn.get_collector_configs().await?; - let mut collector_map: HashMap = collectors - .into_iter() - .map(|c| { - ( - c.name().to_string(), - status_new::Collector { - name: c.name().to_string(), - target: c.target().to_string(), - benchmark_set: c.benchmark_set().get_id(), - is_active: c.is_active(), - last_heartbeat_at: c.last_heartbeat_at(), - date_added: c.date_added(), - jobs: vec![], - }, - ) - }) - .collect(); - - // This map is used to guess a future collector for jobs that haven't been dequeued yet - // (target, benchmark_set) -> collector name - let collector_guess_map: HashMap<(String, u32), String> = collector_map - .iter() - .map(|(name, collector)| { - ( - (collector.target.to_string(), collector.benchmark_set), - name.clone(), - ) - }) - .collect(); - - // Map jobs to collectors. Even if a collector has not started a job yet, we can guess if it - // will execute it or not, based on its target and benchmark set - for job in in_progress_jobs.values().flatten() { - let Some(collector_name) = job.collector_name().or_else(|| { - collector_guess_map - .get(&(job.target().to_string(), job.benchmark_set().get_id())) - .map(|n| n.as_str()) - }) else { - continue; - }; - if let Some(collector) = collector_map.get_mut(collector_name) { - collector.jobs.push(job_to_ui(job)); - } - } - let mut collectors: Vec = collector_map.into_values().collect(); - collectors.sort_by(|c1, c2| c1.name.cmp(&c2.name)); - for collector in &mut collectors { - collector.jobs.sort_by(|j1, j2| { - let prio1 = job_status_to_priority(j1.status); - let prio2 = job_status_to_priority(j2.status); - prio1 - .cmp(&prio2) - .then(j1.deque_counter.cmp(&j2.deque_counter).reverse()) - .then_with(|| { - (&j1.target, &j1.backend, &j1.profile, j1.benchmark_set).cmp(&( - &j2.target, - &j2.backend, - &j2.profile, - j2.benchmark_set, - )) - }) - }); - } - - Ok(collectors) -} - -fn job_status_to_priority(status: status_new::BenchmarkJobStatus) -> u32 { - match status { - status_new::BenchmarkJobStatus::InProgress => 0, - status_new::BenchmarkJobStatus::Queued => 1, - status_new::BenchmarkJobStatus::Failed => 2, - status_new::BenchmarkJobStatus::Success => 3, - } -} - -fn request_to_ui( - req: &BenchmarkRequest, - errors: HashMap, - estimated_end: Option>, -) -> status_new::BenchmarkRequest { - let (completed_at, duration_s) = match req.status() { - BenchmarkRequestStatus::WaitingForArtifacts => (estimated_end, None), - BenchmarkRequestStatus::ArtifactsReady => (estimated_end, None), - BenchmarkRequestStatus::InProgress => (estimated_end, None), - BenchmarkRequestStatus::Completed { - completed_at, - duration, - } => (Some(completed_at), Some(duration.as_secs())), - }; - status_new::BenchmarkRequest { - tag: req.tag().expect("Missing request tag").to_string(), - pr: req.pr(), - status: match req.status() { - BenchmarkRequestStatus::WaitingForArtifacts => unreachable!(), - BenchmarkRequestStatus::ArtifactsReady => status_new::BenchmarkRequestStatus::Queued, - BenchmarkRequestStatus::InProgress => status_new::BenchmarkRequestStatus::InProgress, - BenchmarkRequestStatus::Completed { .. } => { - status_new::BenchmarkRequestStatus::Completed - } - }, - request_type: match req.commit_type() { - BenchmarkRequestType::Try { .. } => status_new::BenchmarkRequestType::Try, - BenchmarkRequestType::Master { .. } => status_new::BenchmarkRequestType::Master, - BenchmarkRequestType::Release { .. } => status_new::BenchmarkRequestType::Release, - }, - created_at: req.created_at(), - completed_at, - duration_s, - errors, - end_estimated: estimated_end.is_some(), - } -} - -fn job_to_ui(job: &BenchmarkJob) -> status_new::BenchmarkJob { - let (started_at, completed_at) = match job.status() { - BenchmarkJobStatus::Queued => (None, None), - BenchmarkJobStatus::InProgress { started_at, .. } => (Some(*started_at), None), - BenchmarkJobStatus::Completed { - started_at, - completed_at, - .. - } => (Some(*started_at), Some(*completed_at)), - }; - - status_new::BenchmarkJob { - request_tag: job.request_tag().to_string(), - kind: job.kind().to_string(), - target: job.target().as_str().to_string(), - backend: job.backend().as_str().to_string(), - profile: job.profile().as_str().to_string(), - benchmark_set: job.benchmark_set().get_id(), - created_at: job.created_at(), - started_at, - completed_at, - status: match job.status() { - BenchmarkJobStatus::Queued => status_new::BenchmarkJobStatus::Queued, - BenchmarkJobStatus::InProgress { .. } => status_new::BenchmarkJobStatus::InProgress, - BenchmarkJobStatus::Completed { success: true, .. } => { - status_new::BenchmarkJobStatus::Success - } - BenchmarkJobStatus::Completed { success: false, .. } => { - status_new::BenchmarkJobStatus::Failed - } - }, - deque_counter: job.deque_count(), - } -} diff --git a/site/src/server.rs b/site/src/server.rs index c1b52b886..442f88746 100644 --- a/site/src/server.rs +++ b/site/src/server.rs @@ -6,17 +6,15 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::path::Path; use std::str::FromStr; -use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; use std::sync::{Arc, LazyLock}; -use std::time::Instant; use std::{fmt, str}; use futures::{future::FutureExt, stream::StreamExt}; use headers::{Authorization, CacheControl, ContentType, ETag, Header, HeaderMapExt, IfNoneMatch}; use http::header::CACHE_CONTROL; use hyper::StatusCode; -use log::{debug, error, info}; -use parking_lot::{Mutex, RwLock}; +use log::{error, info}; +use parking_lot::RwLock; use serde::de::DeserializeOwned; use serde::Serialize; use uuid::Uuid; @@ -29,7 +27,7 @@ use crate::load::{Config, SiteCtxt}; use crate::request_handlers; use crate::resources::{Payload, ResourceResolver}; -use database::{self, ArtifactId}; +use crate::job_queue::build_queue; pub type Request = http::Request; pub type Response = http::Response; @@ -49,52 +47,11 @@ macro_rules! check_http_method { #[derive(Clone)] struct Server { ctxt: Arc>>>, - updating: UpdatingStatus, } impl Server { fn new(ctxt: Arc>>>) -> Self { - Self { - ctxt, - updating: UpdatingStatus::new(), - } - } -} - -#[derive(Clone)] -struct UpdatingStatus(Arc); - -struct IsUpdating(Arc, hyper::body::Sender); - -impl Drop for IsUpdating { - fn drop(&mut self) { - self.0.store(false, AtomicOrdering::SeqCst); - if std::thread::panicking() { - let _ = self.1.try_send_data("panicked, try again".into()); - } else { - let _ = self.1.try_send_data("done".into()); - } - } -} - -impl UpdatingStatus { - fn new() -> Self { - UpdatingStatus(Arc::new(AtomicBool::new(false))) - } - - // Returns previous state - fn set_updating(&self) -> bool { - match self - .0 - .compare_exchange(false, true, AtomicOrdering::SeqCst, AtomicOrdering::SeqCst) - { - Ok(b) => b, - Err(b) => b, - } - } - - fn release_on_drop(&self, channel: hyper::body::Sender) -> IsUpdating { - IsUpdating(self.0.clone(), channel) + Self { ctxt } } } @@ -191,21 +148,22 @@ impl Server { async fn handle_metrics(&self, _req: Request) -> Response { use prometheus::Encoder; let ctxt: Arc = self.ctxt.read().as_ref().unwrap().clone(); - let idx = ctxt.index.load(); let mut buffer = Vec::new(); let r = prometheus::Registry::new(); - let missing_commits = ctxt.missing_commits().await; + let queue = build_queue(ctxt.pool.connection().await.as_mut()) + .await + .unwrap_or_default(); let queue_length = prometheus::IntGauge::new("rustc_perf_queue_length", "queue length").unwrap(); - queue_length.set(missing_commits.len() as i64); + queue_length.set(queue.len() as i64); r.register(Box::new(queue_length)).unwrap(); let queue_try_commits = prometheus::IntGauge::new("rustc_perf_queue_try_commits", "queued try commits") .unwrap(); - queue_try_commits.set(missing_commits.iter().filter(|(c, _)| c.is_try()).count() as i64); + queue_try_commits.set(queue.iter().filter(|req| req.is_try()).count() as i64); r.register(Box::new(queue_try_commits)).unwrap(); // Stores cache hits and misses of the self profile cache @@ -228,27 +186,6 @@ impl Server { self_profile_cache_misses.set(self_profile_stats.get_misses() as i64); r.register(Box::new(self_profile_cache_misses)).unwrap(); } - if let Some(last_commit) = idx.commits().last().cloned() { - let conn = ctxt.conn().await; - let steps = conn.in_progress_steps(&ArtifactId::from(last_commit)).await; - let g = prometheus::IntGaugeVec::new( - prometheus::core::Opts { - namespace: "rustc_perf".to_string(), - subsystem: String::new(), - name: String::from("step_duration_seconds"), - help: String::from("step duration"), - const_labels: HashMap::new(), - variable_labels: vec![], - }, - &["step"], - ) - .unwrap(); - for step in steps { - g.with_label_values(&[&step.name]) - .set(step.expected.as_secs() as i64); - } - r.register(Box::new(g)).unwrap(); - } let encoder = prometheus::TextEncoder::new(); let metric_families = r.gather(); @@ -256,59 +193,6 @@ impl Server { Response::new(buffer.into()) } - - async fn handle_push(&self, _req: Request) -> Response { - static LAST_UPDATE: LazyLock>> = LazyLock::new(|| Mutex::new(None)); - - let last = *LAST_UPDATE.lock(); - if let Some(last) = last { - let min = 60; // 1 minutes - let elapsed = last.elapsed(); - if elapsed < std::time::Duration::from_secs(min) { - return http::Response::builder() - .status(StatusCode::OK) - .header_typed(ContentType::text_utf8()) - .body(hyper::Body::from(format!( - "Refreshed too recently ({elapsed:?} ago). Please wait." - ))) - .unwrap(); - } - } - *LAST_UPDATE.lock() = Some(Instant::now()); - - // set to updating - let was_updating = self.updating.set_updating(); - - if was_updating { - return http::Response::builder() - .status(StatusCode::OK) - .header_typed(ContentType::text_utf8()) - .body(hyper::Body::from("Already updating!")) - .unwrap(); - } - - debug!("received onpush hook"); - - let (channel, body) = hyper::Body::channel(); - - let ctxt: Arc = self.ctxt.read().as_ref().unwrap().clone(); - let _updating = self.updating.release_on_drop(channel); - let mut conn = ctxt.conn().await; - let index = database::Index::load(&mut *conn).await; - eprintln!("index has {} commits", index.commits().len()); - ctxt.index.store(Arc::new(index)); - - // Refresh the landing page - ctxt.landing_page.store(Arc::new(None)); - - // Spawn off a task to post the results of any commit results that we - // are now aware of. - tokio::spawn(async move { - crate::github::post_finished(&ctxt).await; - }); - - Response::new(body) - } } #[derive(Debug)] @@ -393,16 +277,6 @@ async fn serve_req(server: Server, req: Request) -> Result { - return server - .handle_get_async(&req, request_handlers::handle_status_page_old) - .await; - } - "/perf/next_artifact" => { - return server - .handle_get_async(&req, request_handlers::handle_next_artifact) - .await; - } "/perf/triage" if *req.method() == http::Method::GET => { let ctxt: Arc = server.ctxt.read().as_ref().unwrap().clone(); let input: triage::Request = check!(parse_query_string(req.uri())); @@ -445,9 +319,6 @@ async fn serve_req(server: Server, req: Request) -> Result { return Ok(server.handle_metrics(req).await); } - "/perf/onpush" => { - return Ok(server.handle_push(req).await); - } "/perf/download-raw-self-profile" => { let ctxt: Arc = server.ctxt.read().as_ref().unwrap().clone(); let req = check!(parse_query_string(req.uri())); @@ -658,7 +529,6 @@ async fn handle_fs_path( | "/dashboard.html" | "/detailed-query.html" | "/help.html" - | "/status_old.html" | "/status.html" => resolve_template(relative_path).await, _ => match TEMPLATES.get_static_asset(relative_path, use_compression)? { Payload::Compressed(data) => {