diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs index 0b575b536cb6e..52e6aec37c48a 100644 --- a/compiler/rustc_middle/src/query/inner.rs +++ b/compiler/rustc_middle/src/query/inner.rs @@ -32,15 +32,15 @@ where /// Shared implementation of `tcx.$query(..)` and `tcx.at(span).$query(..)` /// for all queries. #[inline(always)] -pub(crate) fn query_get_at<'tcx, Cache>( +pub(crate) fn query_get_at<'tcx, C>( tcx: TyCtxt<'tcx>, - execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option, - query_cache: &Cache, + execute_query: fn(TyCtxt<'tcx>, Span, C::Key, QueryMode) -> Option, + query_cache: &C, span: Span, - key: Cache::Key, -) -> Cache::Value + key: C::Key, +) -> C::Value where - Cache: QueryCache, + C: QueryCache, { match try_get_cached(tcx, query_cache, &key) { Some(value) => value, @@ -51,14 +51,14 @@ where /// Shared implementation of `tcx.ensure_ok().$query(..)` for most queries, /// and `tcx.ensure_done().$query(..)` for all queries. #[inline] -pub(crate) fn query_ensure<'tcx, Cache>( +pub(crate) fn query_ensure<'tcx, C>( tcx: TyCtxt<'tcx>, - execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option, - query_cache: &Cache, - key: Cache::Key, + execute_query: fn(TyCtxt<'tcx>, Span, C::Key, QueryMode) -> Option, + query_cache: &C, + key: C::Key, check_cache: bool, ) where - Cache: QueryCache, + C: QueryCache, { if try_get_cached(tcx, query_cache, &key).is_none() { execute_query(tcx, DUMMY_SP, key, QueryMode::Ensure { check_cache }); @@ -68,15 +68,15 @@ pub(crate) fn query_ensure<'tcx, Cache>( /// Shared implementation of `tcx.ensure_ok().$query(..)` for queries that /// have the `return_result_from_ensure_ok` modifier. #[inline] -pub(crate) fn query_ensure_error_guaranteed<'tcx, Cache, T>( +pub(crate) fn query_ensure_error_guaranteed<'tcx, C, T>( tcx: TyCtxt<'tcx>, - execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option, - query_cache: &Cache, - key: Cache::Key, + execute_query: fn(TyCtxt<'tcx>, Span, C::Key, QueryMode) -> Option, + query_cache: &C, + key: C::Key, check_cache: bool, ) -> Result<(), ErrorGuaranteed> where - Cache: QueryCache>>, + C: QueryCache>>, Result: Erasable, { if let Some(res) = try_get_cached(tcx, query_cache, &key) { @@ -96,21 +96,20 @@ where } /// Common implementation of query feeding, used by `define_feedable!`. -pub(crate) fn query_feed<'tcx, Cache>( +pub(crate) fn query_feed<'tcx, C>( tcx: TyCtxt<'tcx>, dep_kind: DepKind, - query_vtable: &QueryVTable<'tcx, Cache>, - cache: &Cache, - key: Cache::Key, - value: Cache::Value, + query_vtable: &QueryVTable<'tcx, C>, + key: C::Key, + value: C::Value, ) where - Cache: QueryCache, - Cache::Key: DepNodeKey<'tcx>, + C: QueryCache, + C::Key: DepNodeKey<'tcx>, { let format_value = query_vtable.format_value; // Check whether the in-memory cache already has a value for this key. - match try_get_cached(tcx, cache, &key) { + match try_get_cached(tcx, &query_vtable.cache, &key) { Some(old) => { // The query already has a cached value for this key. // That's OK if both values are the same, i.e. they have the same hash, @@ -153,7 +152,7 @@ pub(crate) fn query_feed<'tcx, Cache>( query_vtable.hash_result, query_vtable.format_value, ); - cache.complete(key, value, dep_node_index); + query_vtable.cache.complete(key, value, dep_node_index); } } } diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 506d7616cbfc9..be09550ee4da5 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -14,9 +14,7 @@ pub use sealed::IntoQueryParam; use crate::dep_graph; use crate::dep_graph::{DepKind, DepNodeIndex, SerializedDepNodeIndex}; use crate::ich::StableHashingContext; -use crate::queries::{ - ExternProviders, PerQueryVTables, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates, -}; +use crate::queries::{ExternProviders, Providers, QueryArenas, QueryVTables}; use crate::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache}; use crate::query::stack::{QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra}; use crate::query::{QueryCache, QueryInfo, QueryJob}; @@ -101,17 +99,15 @@ pub enum QueryMode { Ensure { check_cache: bool }, } -/// Stores function pointers and other metadata for a particular query. +/// Stores data and metadata (e.g. function pointers) for a particular query. pub struct QueryVTable<'tcx, C: QueryCache> { pub name: &'static str, pub eval_always: bool, pub dep_kind: DepKind, /// How this query deals with query cycle errors. pub cycle_error_handling: CycleErrorHandling, - // Offset of this query's state field in the QueryStates struct - pub query_state: usize, - // Offset of this query's cache field in the QueryCaches struct - pub query_cache: usize, + pub state: QueryState<'tcx, C::Key>, + pub cache: C, pub will_cache_on_disk_for_key_fn: Option>, /// Function pointer that calls `tcx.$query(key)` for this query and @@ -140,10 +136,11 @@ pub struct QueryVTable<'tcx, C: QueryCache> { /// /// Used when reporting query cycle errors and similar problems. pub description_fn: fn(TyCtxt<'tcx>, C::Key) -> String, + + pub execute_query_fn: fn(TyCtxt<'tcx>, Span, C::Key, QueryMode) -> Option, } pub struct QuerySystemFns { - pub engine: QueryEngine, pub local_providers: Providers, pub extern_providers: ExternProviders, pub encode_query_results: for<'tcx> fn( @@ -155,10 +152,8 @@ pub struct QuerySystemFns { } pub struct QuerySystem<'tcx> { - pub states: QueryStates<'tcx>, pub arenas: WorkerLocal>, - pub caches: QueryCaches<'tcx>, - pub query_vtables: PerQueryVTables<'tcx>, + pub query_vtables: QueryVTables<'tcx>, /// This provides access to the incremental compilation on-disk cache for query results. /// Do not access this directly. It is only meant to be used by @@ -410,13 +405,6 @@ macro_rules! define_callbacks { )* } - #[derive(Default)] - pub struct QueryCaches<'tcx> { - $( - pub $name: $name::Storage<'tcx>, - )* - } - impl<'tcx> $crate::query::TyCtxtEnsureOk<'tcx> { $( $(#[$attr])* @@ -435,8 +423,8 @@ macro_rules! define_callbacks { (crate::query::inner::query_ensure) )( self.tcx, - self.tcx.query_system.fns.engine.$name, - &self.tcx.query_system.caches.$name, + self.tcx.query_system.query_vtables.$name.execute_query_fn, + &self.tcx.query_system.query_vtables.$name.cache, $crate::query::IntoQueryParam::into_query_param(key), false, ) @@ -451,8 +439,8 @@ macro_rules! define_callbacks { pub fn $name(self, key: query_helper_param_ty!($($K)*)) { crate::query::inner::query_ensure( self.tcx, - self.tcx.query_system.fns.engine.$name, - &self.tcx.query_system.caches.$name, + self.tcx.query_system.query_vtables.$name.execute_query_fn, + &self.tcx.query_system.query_vtables.$name.cache, $crate::query::IntoQueryParam::into_query_param(key), true, ); @@ -480,8 +468,8 @@ macro_rules! define_callbacks { erase::restore_val::<$V>(inner::query_get_at( self.tcx, - self.tcx.query_system.fns.engine.$name, - &self.tcx.query_system.caches.$name, + self.tcx.query_system.query_vtables.$name.execute_query_fn, + &self.tcx.query_system.query_vtables.$name.cache, self.span, $crate::query::IntoQueryParam::into_query_param(key), )) @@ -490,21 +478,12 @@ macro_rules! define_callbacks { } /// Holds a `QueryVTable` for each query. - /// - /// ("Per" just makes this pluralized name more visually distinct.) - pub struct PerQueryVTables<'tcx> { + pub struct QueryVTables<'tcx> { $( pub $name: ::rustc_middle::query::plumbing::QueryVTable<'tcx, $name::Storage<'tcx>>, )* } - #[derive(Default)] - pub struct QueryStates<'tcx> { - $( - pub $name: $crate::query::QueryState<'tcx, $name::Key<'tcx>>, - )* - } - pub struct Providers { $( /// This is the provider for the query. Use `Find references` on this to @@ -564,17 +543,6 @@ macro_rules! define_callbacks { impl Clone for ExternProviders { fn clone(&self) -> Self { *self } } - - pub struct QueryEngine { - $( - pub $name: for<'tcx> fn( - TyCtxt<'tcx>, - Span, - $name::Key<'tcx>, - $crate::query::QueryMode, - ) -> Option<$crate::query::erase::Erased<$V>>, - )* - } }; } @@ -601,7 +569,6 @@ macro_rules! define_feedable { tcx, dep_graph::DepKind::$name, &tcx.query_system.query_vtables.$name, - &tcx.query_system.caches.$name, key, erased_value, ); diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs index a180ea8327fd1..5ecb408dfc1fa 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_query_impl/src/dep_kind_vtables.rs @@ -102,18 +102,18 @@ mod non_query { /// Shared implementation of the [`DepKindVTable`] constructor for queries. /// Called from macro-generated code for each query. -pub(crate) fn make_dep_kind_vtable_for_query<'tcx, Q, Cache, const FLAGS: QueryFlags>( +pub(crate) fn make_dep_kind_vtable_for_query<'tcx, Q, C, const FLAGS: QueryFlags>( is_eval_always: bool, ) -> DepKindVTable<'tcx> where - Q: QueryDispatcherUnerased<'tcx, Cache, FLAGS>, - Cache: QueryCache + 'tcx, + Q: QueryDispatcherUnerased<'tcx, C, FLAGS>, + C: QueryCache + 'tcx, { let is_anon = FLAGS.is_anon; let key_fingerprint_style = if is_anon { KeyFingerprintStyle::Opaque } else { - >::key_fingerprint_style() + >::key_fingerprint_style() }; if is_anon || !key_fingerprint_style.reconstructible() { diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 1f064599332c1..f87b09c96cd19 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -244,12 +244,12 @@ fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>( match result { Ok(()) => { - let Some((v, index)) = query.query_cache(tcx).lookup(&key) else { + let Some((v, index)) = query.cache().lookup(&key) else { outline(|| { // We didn't find the query result in the query cache. Check if it was // poisoned due to a panic instead. let key_hash = sharded::make_hash(&key); - let shard = query.query_state(tcx).active.lock_shard_by_hash(key_hash); + let shard = query.state().active.lock_shard_by_hash(key_hash); match shard.find(key_hash, equivalent_key(&key)) { // The query we waited on panicked. Continue unwinding here. Some((_, ActiveKeyStatus::Poisoned)) => FatalError.raise(), @@ -278,9 +278,8 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b key: C::Key, dep_node: Option, ) -> (C::Value, Option) { - let state = query.query_state(tcx); let key_hash = sharded::make_hash(&key); - let mut state_lock = state.active.lock_shard_by_hash(key_hash); + let mut state_lock = query.state().active.lock_shard_by_hash(key_hash); // For the parallel compiler we need to check both the query cache and query state structures // while holding the state lock to ensure that 1) the query has not yet completed and 2) the @@ -289,7 +288,7 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b // executing, but another thread may have already completed the query and stores it result // in the query cache. if tcx.sess.threads() > 1 { - if let Some((value, index)) = query.query_cache(tcx).lookup(&key) { + if let Some((value, index)) = query.cache().lookup(&key) { tcx.prof.query_cache_hit(index.into()); return (value, Some(index)); } @@ -308,7 +307,7 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b // Drop the lock before we start executing the query drop(state_lock); - execute_job::(query, tcx, state, key, key_hash, id, dep_node) + execute_job::(query, tcx, key, key_hash, id, dep_node) } Entry::Occupied(mut entry) => { match &mut entry.get_mut().1 { @@ -340,7 +339,6 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>( query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, tcx: TyCtxt<'tcx>, - state: &'tcx QueryState<'tcx, C::Key>, key: C::Key, key_hash: u64, id: QueryJobId, @@ -348,7 +346,7 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>( ) -> (C::Value, Option) { // Set up a guard object that will automatically poison the query if a // panic occurs while executing the query (or any intermediate plumbing). - let job_guard = ActiveJobGuard { state, key, key_hash }; + let job_guard = ActiveJobGuard { state: query.state(), key, key_hash }; debug_assert_eq!(tcx.dep_graph.is_fully_enabled(), INCR); @@ -359,7 +357,7 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>( execute_job_non_incr(query, tcx, key, id) }; - let cache = query.query_cache(tcx); + let cache = query.cache(); if query.feedable() { // We should not compute queries that also got a value via feeding. // This can't happen, as query feeding adds the very dependencies to the fed query @@ -680,7 +678,7 @@ pub(crate) fn force_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>( ) { // We may be concurrently trying both execute and force a query. // Ensure that only one of them runs the query. - if let Some((_, index)) = query.query_cache(tcx).lookup(&key) { + if let Some((_, index)) = query.cache().lookup(&key) { tcx.prof.query_cache_hit(index.into()); return; } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 2804f5cfa055d..73621116e4529 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -14,9 +14,7 @@ use std::marker::ConstParamTy; use rustc_data_structures::sync::AtomicU64; use rustc_middle::dep_graph::{self, DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex}; -use rustc_middle::queries::{ - self, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates, -}; +use rustc_middle::queries::{self, ExternProviders, Providers}; use rustc_middle::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache}; use rustc_middle::query::plumbing::{ HashResult, QueryState, QuerySystem, QuerySystemFns, QueryVTable, @@ -104,26 +102,14 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> SemiDynamicQueryDispatcher<'t // Don't use this method to access query results, instead use the methods on TyCtxt. #[inline(always)] - fn query_state(self, tcx: TyCtxt<'tcx>) -> &'tcx QueryState<'tcx, C::Key> { - // Safety: - // This is just manually doing the subfield referencing through pointer math. - unsafe { - &*(&tcx.query_system.states as *const QueryStates<'tcx>) - .byte_add(self.vtable.query_state) - .cast::>() - } + fn state(self) -> &'tcx QueryState<'tcx, C::Key> { + &self.vtable.state } // Don't use this method to access query results, instead use the methods on TyCtxt. #[inline(always)] - fn query_cache(self, tcx: TyCtxt<'tcx>) -> &'tcx C { - // Safety: - // This is just manually doing the subfield referencing through pointer math. - unsafe { - &*(&tcx.query_system.caches as *const QueryCaches<'tcx>) - .byte_add(self.vtable.query_cache) - .cast::() - } + fn cache(self) -> &'tcx C { + &self.vtable.cache } /// Calls `tcx.$query(key)` for this query, and discards the returned value. @@ -243,13 +229,10 @@ pub fn query_system<'tcx>( incremental: bool, ) -> QuerySystem<'tcx> { QuerySystem { - states: Default::default(), arenas: Default::default(), - caches: Default::default(), - query_vtables: make_query_vtables(), + query_vtables: make_query_vtables(incremental), on_disk_cache, fns: QuerySystemFns { - engine: engine(incremental), local_providers, extern_providers, encode_query_results: encode_all_query_results, diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 31e81a62d5270..77a3531f29b4b 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -308,14 +308,15 @@ where QueryStackFrameExtra::new(description, span, def_kind) } -pub(crate) fn create_deferred_query_stack_frame<'tcx, Cache>( +pub(crate) fn create_deferred_query_stack_frame<'tcx, C>( tcx: TyCtxt<'tcx>, - vtable: &'tcx QueryVTable<'tcx, Cache>, - key: Cache::Key, + vtable: &'tcx QueryVTable<'tcx, C>, + key: C::Key, ) -> QueryStackFrame> where - Cache: QueryCache, - Cache::Key: Key + DynSend + DynSync, + C: QueryCache, + C::Key: Key + DynSend + DynSync, + QueryVTable<'tcx, C>: DynSync, { let kind = vtable.dep_kind; @@ -344,9 +345,8 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache, const FLAGS: Quer { let _timer = tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name()); - assert!(all_inactive(query.query_state(tcx))); - let cache = query.query_cache(tcx); - cache.iter(&mut |key, value, dep_node| { + assert!(all_inactive(query.state())); + query.cache().iter(&mut |key, value, dep_node| { if query.will_cache_on_disk_for_key(tcx, key) { let dep_node = SerializedDepNodeIndex::new(dep_node.index()); @@ -366,7 +366,7 @@ pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache, const FLAGS: QueryFlags ) { let _timer = tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name()); - let cache = query.query_cache(tcx); + let cache = query.cache(); let mut map = UnordMap::with_capacity(cache.len()); cache.iter(&mut |key, _, _| { let node = DepNode::construct(tcx, query.dep_kind(), key); @@ -558,7 +558,7 @@ macro_rules! define_queries { } } - pub(crate) fn make_query_vtable<'tcx>() + pub(crate) fn make_query_vtable<'tcx>(incremental: bool) -> QueryVTable<'tcx, queries::$name::Storage<'tcx>> { QueryVTable { @@ -566,8 +566,8 @@ macro_rules! define_queries { eval_always: is_eval_always!([$($modifiers)*]), dep_kind: dep_graph::DepKind::$name, cycle_error_handling: cycle_error_handling!([$($modifiers)*]), - query_state: std::mem::offset_of!(QueryStates<'tcx>, $name), - query_cache: std::mem::offset_of!(QueryCaches<'tcx>, $name), + state: Default::default(), + cache: Default::default(), will_cache_on_disk_for_key_fn: if_cache_on_disk!([$($modifiers)*] { Some(::rustc_middle::queries::_cache_on_disk_if_fns::$name) } { @@ -610,6 +610,11 @@ macro_rules! define_queries { hash_result: hash_result!([$($modifiers)*][queries::$name::Value<'tcx>]), format_value: |value| format!("{:?}", erase::restore_val::>(*value)), description_fn: $crate::queries::_description_fns::$name, + execute_query_fn: if incremental { + query_impl::$name::get_query_incr::__rust_end_short_backtrace + } else { + query_impl::$name::get_query_non_incr::__rust_end_short_backtrace + }, } } @@ -660,7 +665,8 @@ macro_rules! define_queries { }; // Call `gather_active_jobs_inner` to do the actual work. - let res = crate::execution::gather_active_jobs_inner(&tcx.query_system.states.$name, + let res = crate::execution::gather_active_jobs_inner( + &tcx.query_system.query_vtables.$name.state, tcx, make_frame, require_complete, @@ -686,7 +692,7 @@ macro_rules! define_queries { $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache( tcx, stringify!($name), - &tcx.query_system.caches.$name, + &tcx.query_system.query_vtables.$name.cache, string_cache, ) } @@ -718,22 +724,10 @@ macro_rules! define_queries { } })*} - pub(crate) fn engine(incremental: bool) -> QueryEngine { - if incremental { - QueryEngine { - $($name: query_impl::$name::get_query_incr::__rust_end_short_backtrace,)* - } - } else { - QueryEngine { - $($name: query_impl::$name::get_query_non_incr::__rust_end_short_backtrace,)* - } - } - } - - pub fn make_query_vtables<'tcx>() -> queries::PerQueryVTables<'tcx> { - queries::PerQueryVTables { + pub fn make_query_vtables<'tcx>(incremental: bool) -> queries::QueryVTables<'tcx> { + queries::QueryVTables { $( - $name: query_impl::$name::make_query_vtable(), + $name: query_impl::$name::make_query_vtable(incremental), )* } }