diff --git a/Cargo.lock b/Cargo.lock index 4f98a3facf9bb..4e0978a8cf214 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4093,7 +4093,6 @@ dependencies = [ "rustc_passes", "rustc_privacy", "rustc_query_impl", - "rustc_query_system", "rustc_resolve", "rustc_session", "rustc_span", @@ -4226,6 +4225,7 @@ dependencies = [ "bitflags", "either", "gsgdt", + "parking_lot", "polonius-engine", "rustc_abi", "rustc_apfloat", @@ -4512,22 +4512,17 @@ dependencies = [ name = "rustc_query_system" version = "0.0.0" dependencies = [ - "parking_lot", "rustc_abi", "rustc_ast", "rustc_data_structures", "rustc_errors", "rustc_feature", - "rustc_hashes", "rustc_hir", - "rustc_index", "rustc_macros", "rustc_serialize", "rustc_session", "rustc_span", - "rustc_thread_pool", "smallvec", - "tracing", ] [[package]] diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml index 3a3b4d59db868..c234e21b92541 100644 --- a/compiler/rustc_interface/Cargo.toml +++ b/compiler/rustc_interface/Cargo.toml @@ -35,7 +35,6 @@ rustc_parse = { path = "../rustc_parse" } rustc_passes = { path = "../rustc_passes" } rustc_privacy = { path = "../rustc_privacy" } rustc_query_impl = { path = "../rustc_query_impl" } -rustc_query_system = { path = "../rustc_query_system" } rustc_resolve = { path = "../rustc_resolve" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } diff --git a/compiler/rustc_interface/src/callbacks.rs b/compiler/rustc_interface/src/callbacks.rs index 3d8d5d59b118b..5aa1616397f8a 100644 --- a/compiler/rustc_interface/src/callbacks.rs +++ b/compiler/rustc_interface/src/callbacks.rs @@ -12,11 +12,10 @@ use std::fmt; use rustc_errors::{DiagInner, TRACK_DIAGNOSTIC}; -use rustc_middle::dep_graph::{DepNodeExt, TaskDepsRef}; +use rustc_middle::dep_graph::dep_node::default_dep_kind_debug; +use rustc_middle::dep_graph::{DepContext, DepKind, DepNode, DepNodeExt, TaskDepsRef}; use rustc_middle::ty::tls; use rustc_query_impl::QueryCtxt; -use rustc_query_system::dep_graph::dep_node::default_dep_kind_debug; -use rustc_query_system::dep_graph::{DepContext, DepKind, DepNode}; fn track_span_parent(def_id: rustc_span::def_id::LocalDefId) { tls::with_context_opt(|icx| { @@ -107,9 +106,9 @@ pub fn dep_node_debug(node: DepNode, f: &mut std::fmt::Formatter<'_>) -> std::fm pub fn setup_callbacks() { rustc_span::SPAN_TRACK.swap(&(track_span_parent as fn(_))); rustc_hir::def_id::DEF_ID_DEBUG.swap(&(def_id_debug as fn(_, &mut fmt::Formatter<'_>) -> _)); - rustc_query_system::dep_graph::dep_node::DEP_KIND_DEBUG + rustc_middle::dep_graph::dep_node::DEP_KIND_DEBUG .swap(&(dep_kind_debug as fn(_, &mut fmt::Formatter<'_>) -> _)); - rustc_query_system::dep_graph::dep_node::DEP_NODE_DEBUG + rustc_middle::dep_graph::dep_node::DEP_NODE_DEBUG .swap(&(dep_node_debug as fn(_, &mut fmt::Formatter<'_>) -> _)); TRACK_DIAGNOSTIC.swap(&(track_diagnostic as _)); } diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml index f9d5e7c027894..121e77614725f 100644 --- a/compiler/rustc_middle/Cargo.toml +++ b/compiler/rustc_middle/Cargo.toml @@ -8,6 +8,7 @@ edition = "2024" bitflags = "2.4.1" either = "1.5.0" gsgdt = "0.1.2" +parking_lot = "0.12" polonius-engine = "0.13.0" rustc_abi = { path = "../rustc_abi" } rustc_apfloat = "0.2.0" diff --git a/compiler/rustc_query_system/src/dep_graph/README.md b/compiler/rustc_middle/src/dep_graph/README.md similarity index 100% rename from compiler/rustc_query_system/src/dep_graph/README.md rename to compiler/rustc_middle/src/dep_graph/README.md diff --git a/compiler/rustc_query_system/src/dep_graph/debug.rs b/compiler/rustc_middle/src/dep_graph/debug.rs similarity index 100% rename from compiler/rustc_query_system/src/dep_graph/debug.rs rename to compiler/rustc_middle/src/dep_graph/debug.rs diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs index 5f90d1b67d9de..0033a1cd2337b 100644 --- a/compiler/rustc_middle/src/dep_graph/dep_node.rs +++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs @@ -1,12 +1,333 @@ +//! This module defines the [`DepNode`] type which the compiler uses to represent +//! nodes in the [dependency graph]. A `DepNode` consists of a [`DepKind`] (which +//! specifies the kind of thing it represents, like a piece of HIR, MIR, etc.) +//! and a [`Fingerprint`], a 128-bit hash value, the exact meaning of which +//! depends on the node's `DepKind`. Together, the kind and the fingerprint +//! fully identify a dependency node, even across multiple compilation sessions. +//! In other words, the value of the fingerprint does not depend on anything +//! that is specific to a given compilation session, like an unpredictable +//! interning key (e.g., `NodeId`, `DefId`, `Symbol`) or the numeric value of a +//! pointer. The concept behind this could be compared to how git commit hashes +//! uniquely identify a given commit. The fingerprinting approach has +//! a few advantages: +//! +//! * A `DepNode` can simply be serialized to disk and loaded in another session +//! without the need to do any "rebasing" (like we have to do for Spans and +//! NodeIds) or "retracing" (like we had to do for `DefId` in earlier +//! implementations of the dependency graph). +//! * A `Fingerprint` is just a bunch of bits, which allows `DepNode` to +//! implement `Copy`, `Sync`, `Send`, `Freeze`, etc. +//! * Since we just have a bit pattern, `DepNode` can be mapped from disk into +//! memory without any post-processing (e.g., "abomination-style" pointer +//! reconstruction). +//! * Because a `DepNode` is self-contained, we can instantiate `DepNodes` that +//! refer to things that do not exist anymore. In previous implementations +//! `DepNode` contained a `DefId`. A `DepNode` referring to something that +//! had been removed between the previous and the current compilation session +//! could not be instantiated because the current compilation session +//! contained no `DefId` for thing that had been removed. +//! +//! `DepNode` definition happens in `rustc_middle` with the +//! `define_dep_nodes!()` macro. This macro defines the `DepKind` enum. Each +//! `DepKind` has its own parameters that are needed at runtime in order to +//! construct a valid `DepNode` fingerprint. However, only `CompileCodegenUnit` +//! and `CompileMonoItem` are constructed explicitly (with +//! `make_compile_codegen_unit` and `make_compile_mono_item`). +//! +//! Because the macro sees what parameters a given `DepKind` requires, it can +//! "infer" some properties for each kind of `DepNode`: +//! +//! * Whether a `DepNode` of a given kind has any parameters at all. Some +//! `DepNode`s could represent global concepts with only one value. +//! * Whether it is possible, in principle, to reconstruct a query key from a +//! given `DepNode`. Many `DepKind`s only require a single `DefId` parameter, +//! in which case it is possible to map the node's fingerprint back to the +//! `DefId` it was computed from. In other cases, too much information gets +//! lost during fingerprint computation. +//! +//! `make_compile_codegen_unit` and `make_compile_mono_items`, together with +//! `DepNode::new()`, ensure that only valid `DepNode` instances can be +//! constructed. For example, the API does not allow for constructing +//! parameterless `DepNode`s with anything other than a zeroed out fingerprint. +//! More generally speaking, it relieves the user of the `DepNode` API of +//! having to know how to compute the expected fingerprint for a given set of +//! node parameters. +//! +//! [dependency graph]: https://rustc-dev-guide.rust-lang.org/query.html + +use std::fmt; +use std::hash::Hash; + +use rustc_data_structures::AtomicRef; +use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableOrd, ToStableHashKey}; use rustc_hir::def_id::DefId; use rustc_hir::definitions::DefPathHash; -use rustc_query_system::dep_graph::dep_node::DepKind; -use rustc_query_system::dep_graph::{DepContext, DepNode, FingerprintStyle}; +use rustc_macros::{Decodable, Encodable}; +use rustc_query_system::ich::StableHashingContext; use rustc_span::Symbol; +use super::{DepContext, FingerprintStyle, SerializedDepNodeIndex}; use crate::mir::mono::MonoItem; use crate::ty::TyCtxt; +/// This serves as an index into arrays built by `make_dep_kind_array`. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct DepKind { + variant: u16, +} + +impl DepKind { + #[inline] + pub const fn new(variant: u16) -> Self { + Self { variant } + } + + #[inline] + pub const fn as_inner(&self) -> u16 { + self.variant + } + + #[inline] + pub const fn as_usize(&self) -> usize { + self.variant as usize + } +} + +pub fn default_dep_kind_debug(kind: DepKind, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DepKind").field("variant", &kind.variant).finish() +} + +pub static DEP_KIND_DEBUG: AtomicRef) -> fmt::Result> = + AtomicRef::new(&(default_dep_kind_debug as fn(_, &mut fmt::Formatter<'_>) -> _)); + +impl fmt::Debug for DepKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (*DEP_KIND_DEBUG)(*self, f) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct DepNode { + pub kind: DepKind, + pub hash: PackedFingerprint, +} + +impl DepNode { + /// Creates a new, parameterless DepNode. This method will assert + /// that the DepNode corresponding to the given DepKind actually + /// does not require any parameters. + pub fn new_no_params(tcx: Tcx, kind: DepKind) -> DepNode + where + Tcx: super::DepContext, + { + debug_assert_eq!(tcx.fingerprint_style(kind), FingerprintStyle::Unit); + DepNode { kind, hash: Fingerprint::ZERO.into() } + } + + pub fn construct(tcx: Tcx, kind: DepKind, arg: &Key) -> DepNode + where + Tcx: super::DepContext, + Key: DepNodeKey, + { + let hash = arg.to_fingerprint(tcx); + let dep_node = DepNode { kind, hash: hash.into() }; + + #[cfg(debug_assertions)] + { + if !tcx.fingerprint_style(kind).reconstructible() + && (tcx.sess().opts.unstable_opts.incremental_info + || tcx.sess().opts.unstable_opts.query_dep_graph) + { + tcx.dep_graph().register_dep_node_debug_str(dep_node, || arg.to_debug_str(tcx)); + } + } + + dep_node + } + + /// Construct a DepNode from the given DepKind and DefPathHash. This + /// method will assert that the given DepKind actually requires a + /// single DefId/DefPathHash parameter. + pub fn from_def_path_hash(tcx: Tcx, def_path_hash: DefPathHash, kind: DepKind) -> Self + where + Tcx: super::DepContext, + { + debug_assert!(tcx.fingerprint_style(kind) == FingerprintStyle::DefPathHash); + DepNode { kind, hash: def_path_hash.0.into() } + } +} + +pub fn default_dep_node_debug(node: DepNode, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DepNode").field("kind", &node.kind).field("hash", &node.hash).finish() +} + +pub static DEP_NODE_DEBUG: AtomicRef) -> fmt::Result> = + AtomicRef::new(&(default_dep_node_debug as fn(_, &mut fmt::Formatter<'_>) -> _)); + +impl fmt::Debug for DepNode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (*DEP_NODE_DEBUG)(*self, f) + } +} + +/// Trait for query keys as seen by dependency-node tracking. +pub trait DepNodeKey: fmt::Debug + Sized { + fn fingerprint_style() -> FingerprintStyle; + + /// This method turns a query key into an opaque `Fingerprint` to be used + /// in `DepNode`. + fn to_fingerprint(&self, _: Tcx) -> Fingerprint; + + fn to_debug_str(&self, tcx: Tcx) -> String; + + /// This method tries to recover the query key from the given `DepNode`, + /// something which is needed when forcing `DepNode`s during red-green + /// evaluation. The query system will only call this method if + /// `fingerprint_style()` is not `FingerprintStyle::Opaque`. + /// It is always valid to return `None` here, in which case incremental + /// compilation will treat the query as having changed instead of forcing it. + fn recover(tcx: Tcx, dep_node: &DepNode) -> Option; +} + +// Blanket impl of `DepNodeKey`, which is specialized by other impls elsewhere. +impl DepNodeKey for T +where + T: for<'a> HashStable> + fmt::Debug, +{ + #[inline(always)] + default fn fingerprint_style() -> FingerprintStyle { + FingerprintStyle::Opaque + } + + #[inline(always)] + default fn to_fingerprint(&self, tcx: Tcx) -> Fingerprint { + tcx.with_stable_hashing_context(|mut hcx| { + let mut hasher = StableHasher::new(); + self.hash_stable(&mut hcx, &mut hasher); + hasher.finish() + }) + } + + #[inline(always)] + default fn to_debug_str(&self, tcx: Tcx) -> String { + // Make sure to print dep node params with reduced queries since printing + // may themselves call queries, which may lead to (possibly untracked!) + // query cycles. + tcx.with_reduced_queries(|| format!("{self:?}")) + } + + #[inline(always)] + default fn recover(_: Tcx, _: &DepNode) -> Option { + None + } +} + +/// This struct stores function pointers and other metadata for a particular DepKind. +/// +/// Information is retrieved by indexing the `DEP_KINDS` array using the integer value +/// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual +/// jump table instead of large matches. +pub struct DepKindVTable { + /// Anonymous queries cannot be replayed from one compiler invocation to the next. + /// When their result is needed, it is recomputed. They are useful for fine-grained + /// dependency tracking, and caching within one compiler invocation. + pub is_anon: bool, + + /// Eval-always queries do not track their dependencies, and are always recomputed, even if + /// their inputs have not changed since the last compiler invocation. The result is still + /// cached within one compiler invocation. + pub is_eval_always: bool, + + /// Indicates whether and how the query key can be recovered from its hashed fingerprint. + /// + /// The [`DepNodeKey`] trait determines the fingerprint style for each key type. + pub fingerprint_style: FingerprintStyle, + + /// The red/green evaluation system will try to mark a specific DepNode in the + /// dependency graph as green by recursively trying to mark the dependencies of + /// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode` + /// where we don't know if it is red or green and we therefore actually have + /// to recompute its value in order to find out. Since the only piece of + /// information that we have at that point is the `DepNode` we are trying to + /// re-evaluate, we need some way to re-run a query from just that. This is what + /// `force_from_dep_node()` implements. + /// + /// In the general case, a `DepNode` consists of a `DepKind` and an opaque + /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint + /// is usually constructed by computing a stable hash of the query-key that the + /// `DepNode` corresponds to. Consequently, it is not in general possible to go + /// back from hash to query-key (since hash functions are not reversible). For + /// this reason `force_from_dep_node()` is expected to fail from time to time + /// because we just cannot find out, from the `DepNode` alone, what the + /// corresponding query-key is and therefore cannot re-run the query. + /// + /// The system deals with this case letting `try_mark_green` fail which forces + /// the root query to be re-evaluated. + /// + /// Now, if `force_from_dep_node()` would always fail, it would be pretty useless. + /// Fortunately, we can use some contextual information that will allow us to + /// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we + /// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a + /// valid `DefPathHash`. Since we also always build a huge table that maps every + /// `DefPathHash` in the current codebase to the corresponding `DefId`, we have + /// everything we need to re-run the query. + /// + /// Take the `mir_promoted` query as an example. Like many other queries, it + /// just has a single parameter: the `DefId` of the item it will compute the + /// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode` + /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode` + /// is actually a `DefPathHash`, and can therefore just look up the corresponding + /// `DefId` in `tcx.def_path_hash_to_def_id`. + pub force_from_dep_node: + Option bool>, + + /// Invoke a query to put the on-disk cached value in memory. + pub try_load_from_on_disk_cache: Option, + + /// The name of this dep kind. + pub name: &'static &'static str, +} + +/// A "work product" corresponds to a `.o` (or other) file that we +/// save in between runs. These IDs do not have a `DefId` but rather +/// some independent path or string that persists between runs without +/// the need to be mapped or unmapped. (This ensures we can serialize +/// them even in the absence of a tcx.) +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)] +pub struct WorkProductId { + hash: Fingerprint, +} + +impl WorkProductId { + pub fn from_cgu_name(cgu_name: &str) -> WorkProductId { + let mut hasher = StableHasher::new(); + cgu_name.hash(&mut hasher); + WorkProductId { hash: hasher.finish() } + } +} + +impl HashStable for WorkProductId { + #[inline] + fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { + self.hash.hash_stable(hcx, hasher) + } +} +impl ToStableHashKey for WorkProductId { + type KeyType = Fingerprint; + #[inline] + fn to_stable_hash_key(&self, _: &HCX) -> Self::KeyType { + self.hash + } +} +impl StableOrd for WorkProductId { + // Fingerprint can use unstable (just a tuple of `u64`s), so WorkProductId can as well + const CAN_USE_UNSTABLE_SORT: bool = true; + + // `WorkProductId` sort order is not affected by (de)serialization. + const THIS_IMPLEMENTATION_HAS_BEEN_TRIPLE_CHECKED: () = (); +} + macro_rules! define_dep_nodes { ( $( @@ -172,3 +493,18 @@ pub fn dep_kind_from_label(label: &str) -> DepKind { dep_kind_from_label_string(label) .unwrap_or_else(|_| panic!("Query label {label} does not exist")) } + +// Some types are used a lot. Make sure they don't unintentionally get bigger. +#[cfg(target_pointer_width = "64")] +mod size_asserts { + use rustc_data_structures::static_assert_size; + + use super::*; + // tidy-alphabetical-start + static_assert_size!(DepKind, 2); + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + static_assert_size!(DepNode, 18); + #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] + static_assert_size!(DepNode, 24); + // tidy-alphabetical-end +} diff --git a/compiler/rustc_middle/src/dep_graph/dep_node_key.rs b/compiler/rustc_middle/src/dep_graph/dep_node_key.rs index e2fcd82c896ad..b1dfc15539bf9 100644 --- a/compiler/rustc_middle/src/dep_graph/dep_node_key.rs +++ b/compiler/rustc_middle/src/dep_graph/dep_node_key.rs @@ -2,9 +2,8 @@ use rustc_data_structures::fingerprint::Fingerprint; use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId, LocalModDefId, ModDefId}; use rustc_hir::definitions::DefPathHash; use rustc_hir::{HirId, ItemLocalId, OwnerId}; -use rustc_query_system::dep_graph::{DepContext, DepNode, DepNodeKey, FingerprintStyle}; -use crate::dep_graph::DepNodeExt; +use crate::dep_graph::{DepContext, DepNode, DepNodeExt, DepNodeKey, FingerprintStyle}; use crate::ty::TyCtxt; impl<'tcx> DepNodeKey> for () { diff --git a/compiler/rustc_query_system/src/dep_graph/edges.rs b/compiler/rustc_middle/src/dep_graph/edges.rs similarity index 100% rename from compiler/rustc_query_system/src/dep_graph/edges.rs rename to compiler/rustc_middle/src/dep_graph/edges.rs diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs similarity index 99% rename from compiler/rustc_query_system/src/dep_graph/graph.rs rename to compiler/rustc_middle/src/dep_graph/graph.rs index 94a04b148c4b9..3ef0511795b90 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_middle/src/dep_graph/graph.rs @@ -15,6 +15,8 @@ use rustc_data_structures::{assert_matches, outline}; use rustc_errors::DiagInner; use rustc_index::IndexVec; use rustc_macros::{Decodable, Encodable}; +use rustc_query_system::ich::StableHashingContext; +use rustc_query_system::query::QuerySideEffect; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder}; use rustc_session::Session; use tracing::{debug, instrument}; @@ -25,8 +27,8 @@ use super::query::DepGraphQuery; use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex}; use super::{DepContext, DepKind, DepNode, Deps, HasDepContext, WorkProductId}; use crate::dep_graph::edges::EdgesVec; -use crate::ich::StableHashingContext; -use crate::query::{QueryContext, QuerySideEffect}; +use crate::query::QueryContext; +use crate::verify_ich::incremental_verify_ich; pub struct DepGraph { data: Option>>, @@ -583,7 +585,7 @@ impl DepGraph { if let Some(prev_index) = data.previous.node_to_index_opt(&node) { let dep_node_index = data.colors.current(prev_index); if let Some(dep_node_index) = dep_node_index { - crate::query::incremental_verify_ich( + incremental_verify_ich( cx, data, result, diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs index 1056fa9b44627..723eed21f211d 100644 --- a/compiler/rustc_middle/src/dep_graph/mod.rs +++ b/compiler/rustc_middle/src/dep_graph/mod.rs @@ -1,26 +1,187 @@ +use std::panic; + use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_data_structures::sync::DynSync; use rustc_query_system::ich::StableHashingContext; use rustc_session::Session; +use tracing::instrument; +pub use self::dep_node::{ + DepKind, DepNode, DepNodeExt, DepNodeKey, WorkProductId, dep_kind_from_label, dep_kinds, + label_strs, +}; +pub use self::graph::{ + DepGraphData, DepNodeIndex, TaskDepsRef, WorkProduct, WorkProductMap, hash_result, +}; +use self::graph::{MarkFrame, print_markframe_trace}; +pub use self::query::DepGraphQuery; +pub use self::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; +pub use crate::dep_graph::debug::{DepNodeFilter, EdgeFilter}; use crate::ty::print::with_reduced_queries; use crate::ty::{self, TyCtxt}; -#[macro_use] -mod dep_node; +mod debug; +pub mod dep_node; mod dep_node_key; +mod edges; +mod graph; +mod query; +mod serialized; -pub use rustc_query_system::dep_graph::debug::{DepNodeFilter, EdgeFilter}; -pub use rustc_query_system::dep_graph::{ - DepContext, DepGraphQuery, DepKind, DepNode, DepNodeIndex, Deps, SerializedDepGraph, - SerializedDepNodeIndex, TaskDepsRef, WorkProduct, WorkProductId, WorkProductMap, hash_result, -}; +pub trait DepContext: Copy { + type Deps: Deps; + + /// Create a hashing context for hashing new results. + fn with_stable_hashing_context(self, f: impl FnOnce(StableHashingContext<'_>) -> R) -> R; + + /// Access the DepGraph. + fn dep_graph(&self) -> &graph::DepGraph; + + /// Access the profiler. + fn profiler(&self) -> &SelfProfilerRef; + + /// Access the compiler session. + fn sess(&self) -> &Session; + + fn dep_kind_vtable(&self, dep_node: DepKind) -> &dep_node::DepKindVTable; + + #[inline(always)] + fn fingerprint_style(self, kind: DepKind) -> FingerprintStyle { + self.dep_kind_vtable(kind).fingerprint_style + } + + #[inline(always)] + /// Return whether this kind always require evaluation. + fn is_eval_always(self, kind: DepKind) -> bool { + self.dep_kind_vtable(kind).is_eval_always + } + + /// Try to force a dep node to execute and see if it's green. + /// + /// Returns true if the query has actually been forced. It is valid that a query + /// fails to be forced, e.g. when the query key cannot be reconstructed from the + /// dep-node or when the query kind outright does not support it. + #[inline] + #[instrument(skip(self, frame), level = "debug")] + fn try_force_from_dep_node( + self, + dep_node: DepNode, + prev_index: SerializedDepNodeIndex, + frame: &MarkFrame<'_>, + ) -> bool { + if let Some(force_fn) = self.dep_kind_vtable(dep_node.kind).force_from_dep_node { + match panic::catch_unwind(panic::AssertUnwindSafe(|| { + force_fn(self, dep_node, prev_index) + })) { + Err(value) => { + if !value.is::() { + print_markframe_trace(self.dep_graph(), frame); + } + panic::resume_unwind(value) + } + Ok(query_has_been_forced) => query_has_been_forced, + } + } else { + false + } + } + + /// Load data from the on-disk cache. + fn try_load_from_on_disk_cache(self, dep_node: &DepNode) { + if let Some(try_load_fn) = self.dep_kind_vtable(dep_node.kind).try_load_from_on_disk_cache { + try_load_fn(self, *dep_node) + } + } + + fn with_reduced_queries(self, _: impl FnOnce() -> T) -> T; +} + +pub trait Deps: DynSync { + /// Execute the operation with provided dependencies. + fn with_deps(deps: TaskDepsRef<'_>, op: OP) -> R + where + OP: FnOnce() -> R; + + /// Access dependencies from current implicit context. + fn read_deps(op: OP) + where + OP: for<'a> FnOnce(TaskDepsRef<'a>); + + fn name(dep_kind: DepKind) -> &'static str; + + /// We use this for most things when incr. comp. is turned off. + const DEP_KIND_NULL: DepKind; + + /// We use this to create a forever-red node. + const DEP_KIND_RED: DepKind; + + /// We use this to create a side effect node. + const DEP_KIND_SIDE_EFFECT: DepKind; + + /// We use this to create the anon node with zero dependencies. + const DEP_KIND_ANON_ZERO_DEPS: DepKind; + + /// This is the highest value a `DepKind` can have. It's used during encoding to + /// pack information into the unused bits. + const DEP_KIND_MAX: u16; +} + +pub trait HasDepContext: Copy { + type Deps: self::Deps; + type DepContext: self::DepContext; + + fn dep_context(&self) -> &Self::DepContext; +} + +impl HasDepContext for T { + type Deps = T::Deps; + type DepContext = Self; -pub use self::dep_node::{DepNodeExt, dep_kind_from_label, dep_kinds, label_strs}; -pub(crate) use self::dep_node::{make_compile_codegen_unit, make_compile_mono_item, make_metadata}; + fn dep_context(&self) -> &Self::DepContext { + self + } +} + +impl HasDepContext for (T, Q) { + type Deps = T::Deps; + type DepContext = T::DepContext; + + fn dep_context(&self) -> &Self::DepContext { + self.0.dep_context() + } +} + +/// Describes the contents of the fingerprint generated by a given query. +/// +/// This is mainly for determining whether and how we can reconstruct a key +/// from the fingerprint. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub enum FingerprintStyle { + /// The fingerprint is actually a DefPathHash. + DefPathHash, + /// The fingerprint is actually a HirId. + HirId, + /// Query key was `()` or equivalent, so fingerprint is just zero. + Unit, + /// The fingerprint is an opaque hash, and a key cannot be reconstructed from it. + Opaque, +} + +impl FingerprintStyle { + #[inline] + pub const fn reconstructible(self) -> bool { + match self { + FingerprintStyle::DefPathHash | FingerprintStyle::Unit | FingerprintStyle::HirId => { + true + } + FingerprintStyle::Opaque => false, + } + } +} -pub type DepGraph = rustc_query_system::dep_graph::DepGraph; +pub type DepGraph = graph::DepGraph; -pub type DepKindVTable<'tcx> = rustc_query_system::dep_graph::DepKindVTable>; +pub type DepKindVTable<'tcx> = dep_node::DepKindVTable>; pub struct DepsType; diff --git a/compiler/rustc_query_system/src/dep_graph/query.rs b/compiler/rustc_middle/src/dep_graph/query.rs similarity index 100% rename from compiler/rustc_query_system/src/dep_graph/query.rs rename to compiler/rustc_middle/src/dep_graph/query.rs diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_middle/src/dep_graph/serialized.rs similarity index 100% rename from compiler/rustc_query_system/src/dep_graph/serialized.rs rename to compiler/rustc_middle/src/dep_graph/serialized.rs diff --git a/compiler/rustc_middle/src/error.rs b/compiler/rustc_middle/src/error.rs index d487caf4d3ac9..dfb99fb98513d 100644 --- a/compiler/rustc_middle/src/error.rs +++ b/compiler/rustc_middle/src/error.rs @@ -197,3 +197,20 @@ pub(crate) struct InvalidConstInValtree { pub span: Span, pub global_const_id: String, } + +#[derive(Diagnostic)] +#[diag("internal compiler error: reentrant incremental verify failure, suppressing message")] +pub(crate) struct Reentrant; + +#[derive(Diagnostic)] +#[diag("internal compiler error: encountered incremental compilation error with {$dep_node}")] +#[note("please follow the instructions below to create a bug report with the provided information")] +#[note("for incremental compilation bugs, having a reproduction is vital")] +#[note( + "an ideal reproduction consists of the code before and some patch that then triggers the bug when applied and compiled again" +)] +#[note("as a workaround, you can run {$run_cmd} to allow your project to compile")] +pub(crate) struct IncrementCompilation { + pub run_cmd: String, + pub dep_node: String, +} diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs index f0c85fec4ea6e..1c4c987aee920 100644 --- a/compiler/rustc_middle/src/lib.rs +++ b/compiler/rustc_middle/src/lib.rs @@ -51,6 +51,7 @@ #![feature(range_bounds_is_empty)] #![feature(rustc_attrs)] #![feature(sized_hierarchy)] +#![feature(trait_alias)] #![feature(try_blocks)] #![feature(try_trait_v2)] #![feature(try_trait_v2_residual)] @@ -80,6 +81,7 @@ pub mod thir; pub mod traits; pub mod ty; pub mod util; +pub mod verify_ich; #[macro_use] pub mod query; diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index c7ced84a527f7..577d226fc9d7a 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -18,6 +18,7 @@ use rustc_span::{Span, Symbol}; use rustc_target::spec::SymbolVisibility; use tracing::debug; +use crate::dep_graph::dep_node::{make_compile_codegen_unit, make_compile_mono_item}; use crate::dep_graph::{DepNode, WorkProduct, WorkProductId}; use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags; use crate::ty::{self, GenericArgs, Instance, InstanceKind, SymbolName, Ty, TyCtxt}; @@ -290,7 +291,7 @@ impl<'tcx> MonoItem<'tcx> { // Only used by rustc_codegen_cranelift pub fn codegen_dep_node(&self, tcx: TyCtxt<'tcx>) -> DepNode { - crate::dep_graph::make_compile_mono_item(tcx, self) + make_compile_mono_item(tcx, self) } /// Returns the item's `CrateNum` @@ -576,7 +577,7 @@ impl<'tcx> CodegenUnit<'tcx> { } pub fn codegen_dep_node(&self, tcx: TyCtxt<'tcx>) -> DepNode { - crate::dep_graph::make_compile_codegen_unit(tcx, self.name()) + make_compile_codegen_unit(tcx, self.name()) } } diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_middle/src/query/caches.rs similarity index 99% rename from compiler/rustc_query_system/src/query/caches.rs rename to compiler/rustc_middle/src/query/caches.rs index c1f5e5b670856..7424492ddc1f3 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_middle/src/query/caches.rs @@ -7,10 +7,10 @@ use rustc_data_structures::stable_hasher::HashStable; pub use rustc_data_structures::vec_cache::VecCache; use rustc_hir::def_id::LOCAL_CRATE; use rustc_index::Idx; +use rustc_query_system::ich::StableHashingContext; use rustc_span::def_id::{DefId, DefIndex}; use crate::dep_graph::DepNodeIndex; -use crate::ich::StableHashingContext; /// Traits that all query keys must satisfy. pub trait QueryCacheKey = Hash + Eq + Copy + Debug + for<'a> HashStable>; diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs index 594286bfa574e..b977172fcf9d2 100644 --- a/compiler/rustc_middle/src/query/inner.rs +++ b/compiler/rustc_middle/src/query/inner.rs @@ -1,13 +1,13 @@ //! Helper functions that serve as the immediate implementation of //! `tcx.$query(..)` and its variations. -use rustc_query_system::dep_graph::{DepKind, DepNodeKey}; -use rustc_query_system::query::{QueryCache, QueryMode}; use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span}; use crate::dep_graph; +use crate::dep_graph::{DepKind, DepNodeKey}; use crate::query::erase::{self, Erasable, Erased}; use crate::query::plumbing::QueryVTable; +use crate::query::{QueryCache, QueryMode}; use crate::ty::TyCtxt; /// Checks whether there is already a value for this key in the in-memory diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_middle/src/query/job.rs similarity index 92% rename from compiler/rustc_query_system/src/query/job.rs rename to compiler/rustc_middle/src/query/job.rs index 7349fe223ef94..f1a2b3a34d0e8 100644 --- a/compiler/rustc_query_system/src/query/job.rs +++ b/compiler/rustc_middle/src/query/job.rs @@ -6,9 +6,9 @@ use std::sync::Arc; use parking_lot::{Condvar, Mutex}; use rustc_span::Span; -use super::{QueryStackDeferred, QueryStackFrameExtra}; use crate::query::plumbing::CycleError; -use crate::query::{QueryContext, QueryStackFrame}; +use crate::query::stack::{QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra}; +use crate::ty::TyCtxt; /// Represents a span and a query key. #[derive(Clone, Debug)] @@ -98,13 +98,13 @@ impl<'tcx> QueryLatch<'tcx> { /// Awaits for the query job to complete. pub fn wait_on( &self, - qcx: impl QueryContext<'tcx>, + tcx: TyCtxt<'tcx>, query: Option, span: Span, ) -> Result<(), CycleError>> { let waiter = Arc::new(QueryWaiter { query, span, cycle: Mutex::new(None), condvar: Condvar::new() }); - self.wait_on_inner(qcx, &waiter); + self.wait_on_inner(tcx, &waiter); // FIXME: Get rid of this lock. We have ownership of the QueryWaiter // although another thread may still have a Arc reference so we cannot // use Arc::get_mut @@ -116,7 +116,7 @@ impl<'tcx> QueryLatch<'tcx> { } /// Awaits the caller on this latch by blocking the current thread. - fn wait_on_inner(&self, qcx: impl QueryContext<'tcx>, waiter: &Arc>) { + fn wait_on_inner(&self, tcx: TyCtxt<'tcx>, waiter: &Arc>) { let mut info = self.info.lock(); if !info.complete { // We push the waiter on to the `waiters` list. It can be accessed inside @@ -129,12 +129,11 @@ impl<'tcx> QueryLatch<'tcx> { // we have to be in the `wait` call. This is ensured by the deadlock handler // getting the self.info lock. rustc_thread_pool::mark_blocked(); - let proxy = qcx.jobserver_proxy(); - proxy.release_thread(); + tcx.jobserver_proxy.release_thread(); waiter.condvar.wait(&mut info); // Release the lock before we potentially block in `acquire_thread` drop(info); - proxy.acquire_thread(); + tcx.jobserver_proxy.acquire_thread(); } } diff --git a/compiler/rustc_middle/src/query/keys.rs b/compiler/rustc_middle/src/query/keys.rs index cccb7d51bd3ed..83eb4b9ad475b 100644 --- a/compiler/rustc_middle/src/query/keys.rs +++ b/compiler/rustc_middle/src/query/keys.rs @@ -5,12 +5,12 @@ use std::ffi::OsStr; use rustc_ast::tokenstream::TokenStream; use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId, LocalModDefId}; use rustc_hir::hir_id::OwnerId; -use rustc_query_system::dep_graph::DepNodeIndex; -use rustc_query_system::query::{DefIdCache, DefaultCache, SingleCache, VecCache}; use rustc_span::{DUMMY_SP, Ident, LocalExpnId, Span, Symbol}; +use crate::dep_graph::DepNodeIndex; use crate::infer::canonical::CanonicalQueryInput; use crate::mir::mono::CollectionMode; +use crate::query::{DefIdCache, DefaultCache, SingleCache, VecCache}; use crate::ty::fast_reject::SimplifiedType; use crate::ty::layout::ValidityRequirement; use crate::ty::{self, GenericArg, GenericArgsRef, Ty, TyCtxt}; @@ -28,7 +28,7 @@ pub trait Key: Sized { /// In practice the cache type must implement [`QueryCache`], though that /// constraint is not enforced here. /// - /// [`QueryCache`]: rustc_query_system::query::QueryCache + /// [`QueryCache`]: rustc_middle::query::QueryCache type Cache = DefaultCache; /// In the event that a cycle occurs, if no explicit span has been diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 62d5c1f9dd208..d16ca3c401d24 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -1,19 +1,32 @@ +use rustc_data_structures::jobserver::Proxy; use rustc_hir::def_id::LocalDefId; -pub use rustc_query_system::query::{QueryMode, QueryState}; +use rustc_query_system::query::QuerySideEffect; +pub use self::caches::{ + DefIdCache, DefaultCache, QueryCache, QueryCacheKey, SingleCache, VecCache, +}; +pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryLatch, QueryWaiter}; pub use self::keys::{AsLocalKey, Key, LocalCrate}; -pub use self::plumbing::{IntoQueryParam, TyCtxtAt, TyCtxtEnsureDone, TyCtxtEnsureOk}; +pub use self::plumbing::{ + ActiveKeyStatus, CycleError, CycleErrorHandling, IntoQueryParam, QueryMode, QueryState, + TyCtxtAt, TyCtxtEnsureDone, TyCtxtEnsureOk, +}; +pub use self::stack::{QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra}; +use crate::dep_graph::{DepNodeIndex, HasDepContext, SerializedDepNodeIndex}; pub use crate::queries::Providers; use crate::ty::TyCtxt; pub(crate) mod arena_cached; +mod caches; pub mod erase; pub(crate) mod inner; +mod job; mod keys; pub mod on_disk_cache; #[macro_use] pub mod plumbing; pub(crate) mod modifiers; +mod stack; pub fn describe_as_module(def_id: impl Into, tcx: TyCtxt<'_>) -> String { let def_id = def_id.into(); @@ -23,3 +36,18 @@ pub fn describe_as_module(def_id: impl Into, tcx: TyCtxt<'_>) -> Str format!("module `{}`", tcx.def_path_str(def_id)) } } + +pub trait QueryContext<'tcx>: HasDepContext { + /// Gets a jobserver reference which is used to release then acquire + /// a token while waiting on a query. + fn jobserver_proxy(&self) -> &Proxy; + + /// Load a side effect associated to the node in the previous session. + fn load_side_effect( + self, + prev_dep_node_index: SerializedDepNodeIndex, + ) -> Option; + + /// Register a side effect for the given node, for use in next session. + fn store_side_effect(self, dep_node_index: DepNodeIndex, side_effect: QuerySideEffect); +} diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 9a709d2c43c80..727e931482510 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -1,25 +1,70 @@ +use std::fmt::Debug; use std::ops::Deref; use rustc_data_structures::fingerprint::Fingerprint; +use rustc_data_structures::hash_table::HashTable; +use rustc_data_structures::sharded::Sharded; use rustc_data_structures::sync::{AtomicU64, WorkerLocal}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::hir_id::OwnerId; use rustc_macros::HashStable; -use rustc_query_system::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; use rustc_query_system::ich::StableHashingContext; -pub(crate) use rustc_query_system::query::QueryJobId; -use rustc_query_system::query::{CycleError, CycleErrorHandling, QueryCache}; use rustc_span::{ErrorGuaranteed, Span}; pub use sealed::IntoQueryParam; use crate::dep_graph; -use crate::dep_graph::DepKind; +use crate::dep_graph::{DepKind, DepNodeIndex, SerializedDepNodeIndex}; use crate::queries::{ ExternProviders, PerQueryVTables, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates, }; use crate::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache}; +use crate::query::stack::{QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra}; +use crate::query::{QueryCache, QueryInfo, QueryJob}; use crate::ty::TyCtxt; +/// For a particular query, keeps track of "active" keys, i.e. keys whose +/// evaluation has started but has not yet finished successfully. +/// +/// (Successful query evaluation for a key is represented by an entry in the +/// query's in-memory cache.) +pub struct QueryState<'tcx, K> { + pub active: Sharded)>>, +} + +impl<'tcx, K> Default for QueryState<'tcx, K> { + fn default() -> QueryState<'tcx, K> { + QueryState { active: Default::default() } + } +} + +/// For a particular query and key, tracks the status of a query evaluation +/// that has started, but has not yet finished successfully. +/// +/// (Successful query evaluation for a key is represented by an entry in the +/// query's in-memory cache.) +pub enum ActiveKeyStatus<'tcx> { + /// Some thread is already evaluating the query for this key. + /// + /// The enclosed [`QueryJob`] can be used to wait for it to finish. + Started(QueryJob<'tcx>), + + /// The query panicked. Queries trying to wait on this will raise a fatal error which will + /// silently panic. + Poisoned, +} + +/// How a particular query deals with query cycle errors. +/// +/// Inspected by the code that actually handles cycle errors, to decide what +/// approach to use. +#[derive(Copy, Clone)] +pub enum CycleErrorHandling { + Error, + Fatal, + DelayBug, + Stash, +} + pub type WillCacheOnDiskForKeyFn<'tcx, Key> = fn(tcx: TyCtxt<'tcx>, key: &Key) -> bool; pub type TryLoadFromDiskFn<'tcx, Key, Value> = fn( @@ -34,6 +79,28 @@ pub type IsLoadableFromDiskFn<'tcx, Key> = pub type HashResult = Option, &V) -> Fingerprint>; +#[derive(Clone, Debug)] +pub struct CycleError { + /// The query and related span that uses the cycle. + pub usage: Option<(Span, QueryStackFrame)>, + pub cycle: Vec>, +} + +impl<'tcx> CycleError> { + pub fn lift(&self) -> CycleError { + CycleError { + usage: self.usage.as_ref().map(|(span, frame)| (*span, frame.lift())), + cycle: self.cycle.iter().map(|info| info.lift()).collect(), + } + } +} + +#[derive(Debug)] +pub enum QueryMode { + Get, + Ensure { check_cache: bool }, +} + /// Stores function pointers and other metadata for a particular query. /// /// Used indirectly by query plumbing in `rustc_query_system` via a trait, diff --git a/compiler/rustc_middle/src/query/stack.rs b/compiler/rustc_middle/src/query/stack.rs new file mode 100644 index 0000000000000..b2b01517b7ee6 --- /dev/null +++ b/compiler/rustc_middle/src/query/stack.rs @@ -0,0 +1,112 @@ +use std::fmt::Debug; +use std::marker::PhantomData; +use std::mem::transmute; +use std::sync::Arc; + +use rustc_data_structures::sync::{DynSend, DynSync}; +use rustc_hashes::Hash64; +use rustc_hir::def::DefKind; +use rustc_span::Span; +use rustc_span::def_id::DefId; + +use crate::dep_graph::DepKind; + +/// Description of a frame in the query stack. +/// +/// This is mostly used in case of cycles for error reporting. +#[derive(Clone, Debug)] +pub struct QueryStackFrame { + /// This field initially stores a `QueryStackDeferred` during collection, + /// but can later be changed to `QueryStackFrameExtra` containing concrete information + /// by calling `lift`. This is done so that collecting query does not need to invoke + /// queries, instead `lift` will call queries in a more appropriate location. + pub info: I, + + pub dep_kind: DepKind, + /// This hash is used to deterministically pick + /// a query to remove cycles in the parallel compiler. + pub hash: Hash64, + pub def_id: Option, + /// A def-id that is extracted from a `Ty` in a query key + pub def_id_for_ty_in_cycle: Option, +} + +impl<'tcx> QueryStackFrame> { + #[inline] + pub fn new( + info: QueryStackDeferred<'tcx>, + dep_kind: DepKind, + hash: Hash64, + def_id: Option, + def_id_for_ty_in_cycle: Option, + ) -> Self { + Self { info, def_id, dep_kind, hash, def_id_for_ty_in_cycle } + } + + pub fn lift(&self) -> QueryStackFrame { + QueryStackFrame { + info: self.info.extract(), + dep_kind: self.dep_kind, + hash: self.hash, + def_id: self.def_id, + def_id_for_ty_in_cycle: self.def_id_for_ty_in_cycle, + } + } +} + +#[derive(Clone, Debug)] +pub struct QueryStackFrameExtra { + pub description: String, + pub span: Option, + pub def_kind: Option, +} + +impl QueryStackFrameExtra { + #[inline] + pub fn new(description: String, span: Option, def_kind: Option) -> Self { + Self { description, span, def_kind } + } + + // FIXME(eddyb) Get more valid `Span`s on queries. + #[inline] + pub fn default_span(&self, span: Span) -> Span { + if !span.is_dummy() { + return span; + } + self.span.unwrap_or(span) + } +} + +/// Track a 'side effect' for a particular query. +/// This is used to hold a closure which can create `QueryStackFrameExtra`. +#[derive(Clone)] +pub struct QueryStackDeferred<'tcx> { + _dummy: PhantomData<&'tcx ()>, + + // `extract` may contain references to 'tcx, but we can't tell drop checking that it won't + // access it in the destructor. + extract: Arc QueryStackFrameExtra + DynSync + DynSend>, +} + +impl<'tcx> QueryStackDeferred<'tcx> { + pub fn new( + context: C, + extract: fn(C) -> QueryStackFrameExtra, + ) -> Self { + let extract: Arc QueryStackFrameExtra + DynSync + DynSend + 'tcx> = + Arc::new(move || extract(context)); + // SAFETY: The `extract` closure does not access 'tcx in its destructor as the only + // captured variable is `context` which is Copy and cannot have a destructor. + Self { _dummy: PhantomData, extract: unsafe { transmute(extract) } } + } + + pub fn extract(&self) -> QueryStackFrameExtra { + (self.extract)() + } +} + +impl<'tcx> Debug for QueryStackDeferred<'tcx> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("QueryStackDeferred") + } +} diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index f638dd80864cb..430890d5a42d8 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -39,7 +39,6 @@ use rustc_hir::lang_items::LangItem; use rustc_hir::limit::Limit; use rustc_hir::{self as hir, HirId, Node, TraitCandidate, find_attr}; use rustc_index::IndexVec; -use rustc_query_system::dep_graph::DepNodeIndex; use rustc_query_system::ich::StableHashingContext; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder}; use rustc_session::Session; @@ -54,7 +53,8 @@ use rustc_type_ir::{CollectAndApply, TypeFlags, WithCachedTypeInfo, elaborate, s use tracing::{debug, instrument}; use crate::arena::Arena; -use crate::dep_graph::{DepGraph, DepKindVTable}; +use crate::dep_graph::dep_node::make_metadata; +use crate::dep_graph::{DepGraph, DepKindVTable, DepNodeIndex}; use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarKind}; use crate::lint::lint_level; use crate::metadata::ModChild; @@ -2751,7 +2751,7 @@ impl<'tcx> TyCtxt<'tcx> { } pub fn metadata_dep_node(self) -> crate::dep_graph::DepNode { - crate::dep_graph::make_metadata(self) + make_metadata(self) } pub fn needs_coroutine_by_move_body_def_id(self, def_id: DefId) -> bool { diff --git a/compiler/rustc_middle/src/ty/context/impl_interner.rs b/compiler/rustc_middle/src/ty/context/impl_interner.rs index 5a15d132048dd..7580cc65d530a 100644 --- a/compiler/rustc_middle/src/ty/context/impl_interner.rs +++ b/compiler/rustc_middle/src/ty/context/impl_interner.rs @@ -9,11 +9,11 @@ use rustc_hir::def::{CtorKind, CtorOf, DefKind}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::lang_items::LangItem; use rustc_hir::{self as hir}; -use rustc_query_system::dep_graph::DepNodeIndex; use rustc_span::{DUMMY_SP, Span, Symbol}; use rustc_type_ir::lang_items::{SolverAdtLangItem, SolverLangItem, SolverTraitLangItem}; use rustc_type_ir::{CollectAndApply, Interner, TypeFoldable, search_graph}; +use crate::dep_graph::DepNodeIndex; use crate::infer::canonical::CanonicalVarKinds; use crate::query::IntoQueryParam; use crate::traits::cache::WithDepNode; diff --git a/compiler/rustc_middle/src/ty/context/tls.rs b/compiler/rustc_middle/src/ty/context/tls.rs index a06f927928205..d37ad56c2e83d 100644 --- a/compiler/rustc_middle/src/ty/context/tls.rs +++ b/compiler/rustc_middle/src/ty/context/tls.rs @@ -4,7 +4,7 @@ use rustc_data_structures::sync; use super::{GlobalCtxt, TyCtxt}; use crate::dep_graph::TaskDepsRef; -use crate::query::plumbing::QueryJobId; +use crate::query::QueryJobId; /// This is the implicit state of rustc. It contains the current /// `TyCtxt` and query. It is updated when creating a local interner or diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_middle/src/verify_ich.rs similarity index 58% rename from compiler/rustc_query_system/src/query/plumbing.rs rename to compiler/rustc_middle/src/verify_ich.rs index d7a24d613583a..c5aec73679ec2 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/verify_ich.rs @@ -1,64 +1,10 @@ use std::cell::Cell; -use std::fmt::Debug; use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::hash_table::HashTable; -use rustc_data_structures::sharded::Sharded; -use rustc_span::Span; +use rustc_query_system::ich::StableHashingContext; use tracing::instrument; -use super::{QueryStackDeferred, QueryStackFrameExtra}; -use crate::dep_graph::{DepContext, DepGraphData}; -use crate::ich::StableHashingContext; -use crate::query::job::{QueryInfo, QueryJob}; -use crate::query::{QueryStackFrame, SerializedDepNodeIndex}; - -/// For a particular query, keeps track of "active" keys, i.e. keys whose -/// evaluation has started but has not yet finished successfully. -/// -/// (Successful query evaluation for a key is represented by an entry in the -/// query's in-memory cache.) -pub struct QueryState<'tcx, K> { - pub active: Sharded)>>, -} - -/// For a particular query and key, tracks the status of a query evaluation -/// that has started, but has not yet finished successfully. -/// -/// (Successful query evaluation for a key is represented by an entry in the -/// query's in-memory cache.) -pub enum ActiveKeyStatus<'tcx> { - /// Some thread is already evaluating the query for this key. - /// - /// The enclosed [`QueryJob`] can be used to wait for it to finish. - Started(QueryJob<'tcx>), - - /// The query panicked. Queries trying to wait on this will raise a fatal error which will - /// silently panic. - Poisoned, -} - -impl<'tcx, K> Default for QueryState<'tcx, K> { - fn default() -> QueryState<'tcx, K> { - QueryState { active: Default::default() } - } -} - -#[derive(Clone, Debug)] -pub struct CycleError { - /// The query and related span that uses the cycle. - pub usage: Option<(Span, QueryStackFrame)>, - pub cycle: Vec>, -} - -impl<'tcx> CycleError> { - pub fn lift(&self) -> CycleError { - CycleError { - usage: self.usage.as_ref().map(|(span, frame)| (*span, frame.lift())), - cycle: self.cycle.iter().map(|info| info.lift()).collect(), - } - } -} +use crate::dep_graph::{DepContext, DepGraphData, SerializedDepNodeIndex}; #[inline] #[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")] @@ -142,9 +88,3 @@ fn incremental_verify_ich_failed( INSIDE_VERIFY_PANIC.set(old_in_panic); } - -#[derive(Debug)] -pub enum QueryMode { - Get, - Ensure { check_cache: bool }, -} diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs index 92fd9c6734d04..99feeb86bac93 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_query_impl/src/dep_kind_vtables.rs @@ -1,8 +1,7 @@ use rustc_middle::bug; -use rustc_middle::dep_graph::DepKindVTable; +use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, FingerprintStyle}; +use rustc_middle::query::QueryCache; use rustc_middle::ty::TyCtxt; -use rustc_query_system::dep_graph::{DepNodeKey, FingerprintStyle}; -use rustc_query_system::query::QueryCache; use crate::plumbing::{force_from_dep_node_inner, try_load_from_on_disk_cache_inner}; use crate::{QueryCtxt, QueryDispatcherUnerased, QueryFlags}; diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 50fb4f29ed717..4df55e46e13ec 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -5,13 +5,13 @@ use rustc_data_structures::hash_table::{Entry, HashTable}; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::{outline, sharded, sync}; use rustc_errors::{Diag, FatalError, StashKey}; -use rustc_middle::dep_graph::DepsType; -use rustc_middle::ty::TyCtxt; -use rustc_query_system::dep_graph::{DepGraphData, DepNodeKey, HasDepContext}; -use rustc_query_system::query::{ +use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, DepsType, HasDepContext}; +use rustc_middle::query::{ ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryJob, QueryJobId, QueryLatch, - QueryMode, QueryStackDeferred, QueryStackFrame, QueryState, incremental_verify_ich, + QueryMode, QueryStackDeferred, QueryStackFrame, QueryState, }; +use rustc_middle::ty::TyCtxt; +use rustc_middle::verify_ich::incremental_verify_ich; use rustc_span::{DUMMY_SP, Span}; use crate::dep_graph::{DepContext, DepNode, DepNodeIndex}; @@ -238,7 +238,7 @@ fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>( // With parallel queries we might just have to wait on some other // thread. - let result = latch.wait_on(qcx, current, span); + let result = latch.wait_on(qcx.tcx, current, span); match result { Ok(()) => { diff --git a/compiler/rustc_query_impl/src/job.rs b/compiler/rustc_query_impl/src/job.rs index 19b8245b97e7a..8e554d4ed4e65 100644 --- a/compiler/rustc_query_impl/src/job.rs +++ b/compiler/rustc_query_impl/src/job.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::{Diag, DiagCtxtHandle}; use rustc_hir::def::DefKind; -use rustc_query_system::query::{ +use rustc_middle::query::{ CycleError, QueryInfo, QueryJob, QueryJobId, QueryLatch, QueryStackDeferred, QueryStackFrame, QueryWaiter, }; diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index feeb072f027af..6a4171afe4bc1 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -12,18 +12,16 @@ use std::marker::ConstParamTy; use rustc_data_structures::sync::AtomicU64; -use rustc_middle::dep_graph::{self, DepKind, DepNode, DepNodeIndex}; +use rustc_middle::dep_graph::{self, DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex}; use rustc_middle::queries::{ self, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates, }; -use rustc_middle::query::AsLocalKey; use rustc_middle::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache}; -use rustc_middle::query::plumbing::{HashResult, QuerySystem, QuerySystemFns, QueryVTable}; -use rustc_middle::ty::TyCtxt; -use rustc_query_system::dep_graph::SerializedDepNodeIndex; -use rustc_query_system::query::{ - CycleError, CycleErrorHandling, QueryCache, QueryMode, QueryState, +use rustc_middle::query::plumbing::{ + HashResult, QueryState, QuerySystem, QuerySystemFns, QueryVTable, }; +use rustc_middle::query::{AsLocalKey, CycleError, CycleErrorHandling, QueryCache, QueryMode}; +use rustc_middle::ty::TyCtxt; use rustc_span::{ErrorGuaranteed, Span}; pub use crate::dep_kind_vtables::make_dep_kind_vtables; diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 12cef31f58bb2..ad782b3150b08 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -16,22 +16,22 @@ use rustc_middle::bug; #[expect(unused_imports, reason = "used by doc comments")] use rustc_middle::dep_graph::DepKindVTable; use rustc_middle::dep_graph::{ - self, DepContext, DepNode, DepNodeIndex, DepsType, SerializedDepNodeIndex, dep_kinds, + self, DepContext, DepNode, DepNodeIndex, DepNodeKey, DepsType, HasDepContext, + SerializedDepNodeIndex, dep_kinds, }; -use rustc_middle::query::Key; use rustc_middle::query::on_disk_cache::{ AbsoluteBytePos, CacheDecoder, CacheEncoder, EncodedDepNodeIndex, }; use rustc_middle::query::plumbing::QueryVTable; +use rustc_middle::query::{ + Key, QueryCache, QueryContext, QueryJobId, QueryStackDeferred, QueryStackFrame, + QueryStackFrameExtra, +}; use rustc_middle::ty::codec::TyEncoder; use rustc_middle::ty::print::with_reduced_queries; use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::ty::{self, TyCtxt}; -use rustc_query_system::dep_graph::{DepNodeKey, HasDepContext}; -use rustc_query_system::query::{ - QueryCache, QueryContext, QueryJobId, QuerySideEffect, QueryStackDeferred, QueryStackFrame, - QueryStackFrameExtra, -}; +use rustc_query_system::query::QuerySideEffect; use rustc_serialize::{Decodable, Encodable}; use rustc_span::def_id::LOCAL_CRATE; @@ -209,16 +209,16 @@ pub fn query_key_hash_verify_all<'tcx>(tcx: TyCtxt<'tcx>) { macro_rules! cycle_error_handling { ([]) => {{ - rustc_query_system::query::CycleErrorHandling::Error + rustc_middle::query::CycleErrorHandling::Error }}; ([(cycle_fatal) $($rest:tt)*]) => {{ - rustc_query_system::query::CycleErrorHandling::Fatal + rustc_middle::query::CycleErrorHandling::Fatal }}; ([(cycle_stash) $($rest:tt)*]) => {{ - rustc_query_system::query::CycleErrorHandling::Stash + rustc_middle::query::CycleErrorHandling::Stash }}; ([(cycle_delay_bug) $($rest:tt)*]) => {{ - rustc_query_system::query::CycleErrorHandling::DelayBug + rustc_middle::query::CycleErrorHandling::DelayBug }}; ([$other:tt $($modifiers:tt)*]) => { cycle_error_handling!([$($modifiers)*]) @@ -276,8 +276,8 @@ macro_rules! feedable { macro_rules! hash_result { ([][$V:ty]) => {{ Some(|hcx, result| { - let result = ::rustc_middle::query::erase::restore_val::<$V>(*result); - ::rustc_query_system::dep_graph::hash_result(hcx, &result) + let result = rustc_middle::query::erase::restore_val::<$V>(*result); + rustc_middle::dep_graph::hash_result(hcx, &result) }) }}; ([(no_hash) $($rest:tt)*][$V:ty]) => {{ diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/profiling_support.rs index 7be75ea88acad..679fee49b6c6d 100644 --- a/compiler/rustc_query_impl/src/profiling_support.rs +++ b/compiler/rustc_query_impl/src/profiling_support.rs @@ -6,8 +6,8 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::profiling::SelfProfiler; use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LOCAL_CRATE, LocalDefId}; use rustc_hir::definitions::DefPathData; +use rustc_middle::query::QueryCache; use rustc_middle::ty::TyCtxt; -use rustc_query_system::query::QueryCache; pub(crate) struct QueryKeyStringCache { def_id_cache: FxHashMap, diff --git a/compiler/rustc_query_impl/src/values.rs b/compiler/rustc_query_impl/src/values.rs index 783e7a10d12ac..67bc6893a320d 100644 --- a/compiler/rustc_query_impl/src/values.rs +++ b/compiler/rustc_query_impl/src/values.rs @@ -8,10 +8,10 @@ use rustc_errors::{Applicability, MultiSpan, pluralize, struct_span_code_err}; use rustc_hir as hir; use rustc_hir::def::{DefKind, Res}; use rustc_middle::dep_graph::dep_kinds; +use rustc_middle::query::CycleError; use rustc_middle::query::plumbing::CyclePlaceholder; use rustc_middle::ty::{self, Representability, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; -use rustc_query_system::query::CycleError; use rustc_span::def_id::LocalDefId; use rustc_span::{ErrorGuaranteed, Span}; diff --git a/compiler/rustc_query_system/Cargo.toml b/compiler/rustc_query_system/Cargo.toml index 761a299eab775..bd12dcbfe0d18 100644 --- a/compiler/rustc_query_system/Cargo.toml +++ b/compiler/rustc_query_system/Cargo.toml @@ -5,20 +5,15 @@ edition = "2024" [dependencies] # tidy-alphabetical-start -parking_lot = "0.12" rustc_abi = { path = "../rustc_abi" } rustc_ast = { path = "../rustc_ast" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } rustc_feature = { path = "../rustc_feature" } -rustc_hashes = { path = "../rustc_hashes" } rustc_hir = { path = "../rustc_hir" } -rustc_index = { path = "../rustc_index" } rustc_macros = { path = "../rustc_macros" } rustc_serialize = { path = "../rustc_serialize" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } -rustc_thread_pool = { path = "../rustc_thread_pool" } smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } -tracing = "0.1" # tidy-alphabetical-end diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs deleted file mode 100644 index 5df50b2864b24..0000000000000 --- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs +++ /dev/null @@ -1,340 +0,0 @@ -//! This module defines the [`DepNode`] type which the compiler uses to represent -//! nodes in the [dependency graph]. A `DepNode` consists of a [`DepKind`] (which -//! specifies the kind of thing it represents, like a piece of HIR, MIR, etc.) -//! and a [`Fingerprint`], a 128-bit hash value, the exact meaning of which -//! depends on the node's `DepKind`. Together, the kind and the fingerprint -//! fully identify a dependency node, even across multiple compilation sessions. -//! In other words, the value of the fingerprint does not depend on anything -//! that is specific to a given compilation session, like an unpredictable -//! interning key (e.g., `NodeId`, `DefId`, `Symbol`) or the numeric value of a -//! pointer. The concept behind this could be compared to how git commit hashes -//! uniquely identify a given commit. The fingerprinting approach has -//! a few advantages: -//! -//! * A `DepNode` can simply be serialized to disk and loaded in another session -//! without the need to do any "rebasing" (like we have to do for Spans and -//! NodeIds) or "retracing" (like we had to do for `DefId` in earlier -//! implementations of the dependency graph). -//! * A `Fingerprint` is just a bunch of bits, which allows `DepNode` to -//! implement `Copy`, `Sync`, `Send`, `Freeze`, etc. -//! * Since we just have a bit pattern, `DepNode` can be mapped from disk into -//! memory without any post-processing (e.g., "abomination-style" pointer -//! reconstruction). -//! * Because a `DepNode` is self-contained, we can instantiate `DepNodes` that -//! refer to things that do not exist anymore. In previous implementations -//! `DepNode` contained a `DefId`. A `DepNode` referring to something that -//! had been removed between the previous and the current compilation session -//! could not be instantiated because the current compilation session -//! contained no `DefId` for thing that had been removed. -//! -//! `DepNode` definition happens in `rustc_middle` with the -//! `define_dep_nodes!()` macro. This macro defines the `DepKind` enum. Each -//! `DepKind` has its own parameters that are needed at runtime in order to -//! construct a valid `DepNode` fingerprint. However, only `CompileCodegenUnit` -//! and `CompileMonoItem` are constructed explicitly (with -//! `make_compile_codegen_unit` and `make_compile_mono_item`). -//! -//! Because the macro sees what parameters a given `DepKind` requires, it can -//! "infer" some properties for each kind of `DepNode`: -//! -//! * Whether a `DepNode` of a given kind has any parameters at all. Some -//! `DepNode`s could represent global concepts with only one value. -//! * Whether it is possible, in principle, to reconstruct a query key from a -//! given `DepNode`. Many `DepKind`s only require a single `DefId` parameter, -//! in which case it is possible to map the node's fingerprint back to the -//! `DefId` it was computed from. In other cases, too much information gets -//! lost during fingerprint computation. -//! -//! `make_compile_codegen_unit` and `make_compile_mono_items`, together with -//! `DepNode::new()`, ensure that only valid `DepNode` instances can be -//! constructed. For example, the API does not allow for constructing -//! parameterless `DepNode`s with anything other than a zeroed out fingerprint. -//! More generally speaking, it relieves the user of the `DepNode` API of -//! having to know how to compute the expected fingerprint for a given set of -//! node parameters. -//! -//! [dependency graph]: https://rustc-dev-guide.rust-lang.org/query.html - -use std::fmt; -use std::hash::Hash; - -use rustc_data_structures::AtomicRef; -use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableOrd, ToStableHashKey}; -use rustc_hir::definitions::DefPathHash; -use rustc_macros::{Decodable, Encodable}; - -use super::{DepContext, FingerprintStyle, SerializedDepNodeIndex}; -use crate::ich::StableHashingContext; - -/// This serves as an index into arrays built by `make_dep_kind_array`. -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -pub struct DepKind { - variant: u16, -} - -impl DepKind { - #[inline] - pub const fn new(variant: u16) -> Self { - Self { variant } - } - - #[inline] - pub const fn as_inner(&self) -> u16 { - self.variant - } - - #[inline] - pub const fn as_usize(&self) -> usize { - self.variant as usize - } -} - -pub fn default_dep_kind_debug(kind: DepKind, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("DepKind").field("variant", &kind.variant).finish() -} - -pub static DEP_KIND_DEBUG: AtomicRef) -> fmt::Result> = - AtomicRef::new(&(default_dep_kind_debug as fn(_, &mut fmt::Formatter<'_>) -> _)); - -impl fmt::Debug for DepKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - (*DEP_KIND_DEBUG)(*self, f) - } -} - -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -pub struct DepNode { - pub kind: DepKind, - pub hash: PackedFingerprint, -} - -impl DepNode { - /// Creates a new, parameterless DepNode. This method will assert - /// that the DepNode corresponding to the given DepKind actually - /// does not require any parameters. - pub fn new_no_params(tcx: Tcx, kind: DepKind) -> DepNode - where - Tcx: super::DepContext, - { - debug_assert_eq!(tcx.fingerprint_style(kind), FingerprintStyle::Unit); - DepNode { kind, hash: Fingerprint::ZERO.into() } - } - - pub fn construct(tcx: Tcx, kind: DepKind, arg: &Key) -> DepNode - where - Tcx: super::DepContext, - Key: DepNodeKey, - { - let hash = arg.to_fingerprint(tcx); - let dep_node = DepNode { kind, hash: hash.into() }; - - #[cfg(debug_assertions)] - { - if !tcx.fingerprint_style(kind).reconstructible() - && (tcx.sess().opts.unstable_opts.incremental_info - || tcx.sess().opts.unstable_opts.query_dep_graph) - { - tcx.dep_graph().register_dep_node_debug_str(dep_node, || arg.to_debug_str(tcx)); - } - } - - dep_node - } - - /// Construct a DepNode from the given DepKind and DefPathHash. This - /// method will assert that the given DepKind actually requires a - /// single DefId/DefPathHash parameter. - pub fn from_def_path_hash(tcx: Tcx, def_path_hash: DefPathHash, kind: DepKind) -> Self - where - Tcx: super::DepContext, - { - debug_assert!(tcx.fingerprint_style(kind) == FingerprintStyle::DefPathHash); - DepNode { kind, hash: def_path_hash.0.into() } - } -} - -pub fn default_dep_node_debug(node: DepNode, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("DepNode").field("kind", &node.kind).field("hash", &node.hash).finish() -} - -pub static DEP_NODE_DEBUG: AtomicRef) -> fmt::Result> = - AtomicRef::new(&(default_dep_node_debug as fn(_, &mut fmt::Formatter<'_>) -> _)); - -impl fmt::Debug for DepNode { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - (*DEP_NODE_DEBUG)(*self, f) - } -} - -/// Trait for query keys as seen by dependency-node tracking. -pub trait DepNodeKey: fmt::Debug + Sized { - fn fingerprint_style() -> FingerprintStyle; - - /// This method turns a query key into an opaque `Fingerprint` to be used - /// in `DepNode`. - fn to_fingerprint(&self, _: Tcx) -> Fingerprint; - - fn to_debug_str(&self, tcx: Tcx) -> String; - - /// This method tries to recover the query key from the given `DepNode`, - /// something which is needed when forcing `DepNode`s during red-green - /// evaluation. The query system will only call this method if - /// `fingerprint_style()` is not `FingerprintStyle::Opaque`. - /// It is always valid to return `None` here, in which case incremental - /// compilation will treat the query as having changed instead of forcing it. - fn recover(tcx: Tcx, dep_node: &DepNode) -> Option; -} - -// Blanket impl of `DepNodeKey`, which is specialized by other impls elsewhere. -impl DepNodeKey for T -where - T: for<'a> HashStable> + fmt::Debug, -{ - #[inline(always)] - default fn fingerprint_style() -> FingerprintStyle { - FingerprintStyle::Opaque - } - - #[inline(always)] - default fn to_fingerprint(&self, tcx: Tcx) -> Fingerprint { - tcx.with_stable_hashing_context(|mut hcx| { - let mut hasher = StableHasher::new(); - self.hash_stable(&mut hcx, &mut hasher); - hasher.finish() - }) - } - - #[inline(always)] - default fn to_debug_str(&self, tcx: Tcx) -> String { - // Make sure to print dep node params with reduced queries since printing - // may themselves call queries, which may lead to (possibly untracked!) - // query cycles. - tcx.with_reduced_queries(|| format!("{self:?}")) - } - - #[inline(always)] - default fn recover(_: Tcx, _: &DepNode) -> Option { - None - } -} - -/// This struct stores function pointers and other metadata for a particular DepKind. -/// -/// Information is retrieved by indexing the `DEP_KINDS` array using the integer value -/// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual -/// jump table instead of large matches. -pub struct DepKindVTable { - /// Anonymous queries cannot be replayed from one compiler invocation to the next. - /// When their result is needed, it is recomputed. They are useful for fine-grained - /// dependency tracking, and caching within one compiler invocation. - pub is_anon: bool, - - /// Eval-always queries do not track their dependencies, and are always recomputed, even if - /// their inputs have not changed since the last compiler invocation. The result is still - /// cached within one compiler invocation. - pub is_eval_always: bool, - - /// Indicates whether and how the query key can be recovered from its hashed fingerprint. - /// - /// The [`DepNodeKey`] trait determines the fingerprint style for each key type. - pub fingerprint_style: FingerprintStyle, - - /// The red/green evaluation system will try to mark a specific DepNode in the - /// dependency graph as green by recursively trying to mark the dependencies of - /// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode` - /// where we don't know if it is red or green and we therefore actually have - /// to recompute its value in order to find out. Since the only piece of - /// information that we have at that point is the `DepNode` we are trying to - /// re-evaluate, we need some way to re-run a query from just that. This is what - /// `force_from_dep_node()` implements. - /// - /// In the general case, a `DepNode` consists of a `DepKind` and an opaque - /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint - /// is usually constructed by computing a stable hash of the query-key that the - /// `DepNode` corresponds to. Consequently, it is not in general possible to go - /// back from hash to query-key (since hash functions are not reversible). For - /// this reason `force_from_dep_node()` is expected to fail from time to time - /// because we just cannot find out, from the `DepNode` alone, what the - /// corresponding query-key is and therefore cannot re-run the query. - /// - /// The system deals with this case letting `try_mark_green` fail which forces - /// the root query to be re-evaluated. - /// - /// Now, if `force_from_dep_node()` would always fail, it would be pretty useless. - /// Fortunately, we can use some contextual information that will allow us to - /// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we - /// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a - /// valid `DefPathHash`. Since we also always build a huge table that maps every - /// `DefPathHash` in the current codebase to the corresponding `DefId`, we have - /// everything we need to re-run the query. - /// - /// Take the `mir_promoted` query as an example. Like many other queries, it - /// just has a single parameter: the `DefId` of the item it will compute the - /// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode` - /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode` - /// is actually a `DefPathHash`, and can therefore just look up the corresponding - /// `DefId` in `tcx.def_path_hash_to_def_id`. - pub force_from_dep_node: - Option bool>, - - /// Invoke a query to put the on-disk cached value in memory. - pub try_load_from_on_disk_cache: Option, - - /// The name of this dep kind. - pub name: &'static &'static str, -} - -/// A "work product" corresponds to a `.o` (or other) file that we -/// save in between runs. These IDs do not have a `DefId` but rather -/// some independent path or string that persists between runs without -/// the need to be mapped or unmapped. (This ensures we can serialize -/// them even in the absence of a tcx.) -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)] -pub struct WorkProductId { - hash: Fingerprint, -} - -impl WorkProductId { - pub fn from_cgu_name(cgu_name: &str) -> WorkProductId { - let mut hasher = StableHasher::new(); - cgu_name.hash(&mut hasher); - WorkProductId { hash: hasher.finish() } - } -} - -impl HashStable for WorkProductId { - #[inline] - fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { - self.hash.hash_stable(hcx, hasher) - } -} -impl ToStableHashKey for WorkProductId { - type KeyType = Fingerprint; - #[inline] - fn to_stable_hash_key(&self, _: &HCX) -> Self::KeyType { - self.hash - } -} -impl StableOrd for WorkProductId { - // Fingerprint can use unstable (just a tuple of `u64`s), so WorkProductId can as well - const CAN_USE_UNSTABLE_SORT: bool = true; - - // `WorkProductId` sort order is not affected by (de)serialization. - const THIS_IMPLEMENTATION_HAS_BEEN_TRIPLE_CHECKED: () = (); -} - -// Some types are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(target_pointer_width = "64")] -mod size_asserts { - use rustc_data_structures::static_assert_size; - - use super::*; - // tidy-alphabetical-start - static_assert_size!(DepKind, 2); - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - static_assert_size!(DepNode, 18); - #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] - static_assert_size!(DepNode, 24); - // tidy-alphabetical-end -} diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs deleted file mode 100644 index da110ad2c77c3..0000000000000 --- a/compiler/rustc_query_system/src/dep_graph/mod.rs +++ /dev/null @@ -1,173 +0,0 @@ -pub mod debug; -pub mod dep_node; -mod edges; -mod graph; -mod query; -mod serialized; - -use std::panic; - -pub use dep_node::{DepKind, DepKindVTable, DepNode, DepNodeKey, WorkProductId}; -pub use graph::{ - DepGraph, DepGraphData, DepNodeIndex, TaskDepsRef, WorkProduct, WorkProductMap, hash_result, -}; -pub use query::DepGraphQuery; -use rustc_data_structures::profiling::SelfProfilerRef; -use rustc_data_structures::sync::DynSync; -use rustc_session::Session; -pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex}; -use tracing::instrument; - -use self::graph::{MarkFrame, print_markframe_trace}; -use crate::ich::StableHashingContext; - -pub trait DepContext: Copy { - type Deps: Deps; - - /// Create a hashing context for hashing new results. - fn with_stable_hashing_context(self, f: impl FnOnce(StableHashingContext<'_>) -> R) -> R; - - /// Access the DepGraph. - fn dep_graph(&self) -> &DepGraph; - - /// Access the profiler. - fn profiler(&self) -> &SelfProfilerRef; - - /// Access the compiler session. - fn sess(&self) -> &Session; - - fn dep_kind_vtable(&self, dep_node: DepKind) -> &DepKindVTable; - - #[inline(always)] - fn fingerprint_style(self, kind: DepKind) -> FingerprintStyle { - self.dep_kind_vtable(kind).fingerprint_style - } - - #[inline(always)] - /// Return whether this kind always require evaluation. - fn is_eval_always(self, kind: DepKind) -> bool { - self.dep_kind_vtable(kind).is_eval_always - } - - /// Try to force a dep node to execute and see if it's green. - /// - /// Returns true if the query has actually been forced. It is valid that a query - /// fails to be forced, e.g. when the query key cannot be reconstructed from the - /// dep-node or when the query kind outright does not support it. - #[inline] - #[instrument(skip(self, frame), level = "debug")] - fn try_force_from_dep_node( - self, - dep_node: DepNode, - prev_index: SerializedDepNodeIndex, - frame: &MarkFrame<'_>, - ) -> bool { - if let Some(force_fn) = self.dep_kind_vtable(dep_node.kind).force_from_dep_node { - match panic::catch_unwind(panic::AssertUnwindSafe(|| { - force_fn(self, dep_node, prev_index) - })) { - Err(value) => { - if !value.is::() { - print_markframe_trace(self.dep_graph(), frame); - } - panic::resume_unwind(value) - } - Ok(query_has_been_forced) => query_has_been_forced, - } - } else { - false - } - } - - /// Load data from the on-disk cache. - fn try_load_from_on_disk_cache(self, dep_node: &DepNode) { - if let Some(try_load_fn) = self.dep_kind_vtable(dep_node.kind).try_load_from_on_disk_cache { - try_load_fn(self, *dep_node) - } - } - - fn with_reduced_queries(self, _: impl FnOnce() -> T) -> T; -} - -pub trait Deps: DynSync { - /// Execute the operation with provided dependencies. - fn with_deps(deps: TaskDepsRef<'_>, op: OP) -> R - where - OP: FnOnce() -> R; - - /// Access dependencies from current implicit context. - fn read_deps(op: OP) - where - OP: for<'a> FnOnce(TaskDepsRef<'a>); - - fn name(dep_kind: DepKind) -> &'static str; - - /// We use this for most things when incr. comp. is turned off. - const DEP_KIND_NULL: DepKind; - - /// We use this to create a forever-red node. - const DEP_KIND_RED: DepKind; - - /// We use this to create a side effect node. - const DEP_KIND_SIDE_EFFECT: DepKind; - - /// We use this to create the anon node with zero dependencies. - const DEP_KIND_ANON_ZERO_DEPS: DepKind; - - /// This is the highest value a `DepKind` can have. It's used during encoding to - /// pack information into the unused bits. - const DEP_KIND_MAX: u16; -} - -pub trait HasDepContext: Copy { - type Deps: self::Deps; - type DepContext: self::DepContext; - - fn dep_context(&self) -> &Self::DepContext; -} - -impl HasDepContext for T { - type Deps = T::Deps; - type DepContext = Self; - - fn dep_context(&self) -> &Self::DepContext { - self - } -} - -impl HasDepContext for (T, Q) { - type Deps = T::Deps; - type DepContext = T::DepContext; - - fn dep_context(&self) -> &Self::DepContext { - self.0.dep_context() - } -} - -/// Describes the contents of the fingerprint generated by a given query. -/// -/// This is mainly for determining whether and how we can reconstruct a key -/// from the fingerprint. -#[derive(Debug, PartialEq, Eq, Copy, Clone)] -pub enum FingerprintStyle { - /// The fingerprint is actually a DefPathHash. - DefPathHash, - /// The fingerprint is actually a HirId. - HirId, - /// Query key was `()` or equivalent, so fingerprint is just zero. - Unit, - /// The fingerprint is an opaque hash, and a key cannot be reconstructed from it. - Opaque, -} - -impl FingerprintStyle { - #[inline] - pub const fn reconstructible(self) -> bool { - match self { - FingerprintStyle::DefPathHash | FingerprintStyle::Unit | FingerprintStyle::HirId => { - true - } - FingerprintStyle::Opaque => false, - } - } -} diff --git a/compiler/rustc_query_system/src/error.rs b/compiler/rustc_query_system/src/error.rs deleted file mode 100644 index f48afe6f75fa7..0000000000000 --- a/compiler/rustc_query_system/src/error.rs +++ /dev/null @@ -1,18 +0,0 @@ -use rustc_macros::Diagnostic; - -#[derive(Diagnostic)] -#[diag("internal compiler error: reentrant incremental verify failure, suppressing message")] -pub(crate) struct Reentrant; - -#[derive(Diagnostic)] -#[diag("internal compiler error: encountered incremental compilation error with {$dep_node}")] -#[note("please follow the instructions below to create a bug report with the provided information")] -#[note("for incremental compilation bugs, having a reproduction is vital")] -#[note( - "an ideal reproduction consists of the code before and some patch that then triggers the bug when applied and compiled again" -)] -#[note("as a workaround, you can run {$run_cmd} to allow your project to compile")] -pub(crate) struct IncrementCompilation { - pub run_cmd: String, - pub dep_node: String, -} diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs index e8d857cf31786..bb077d02422b9 100644 --- a/compiler/rustc_query_system/src/lib.rs +++ b/compiler/rustc_query_system/src/lib.rs @@ -2,10 +2,7 @@ #![allow(internal_features)] #![cfg_attr(bootstrap, feature(assert_matches))] #![feature(min_specialization)] -#![feature(trait_alias)] // tidy-alphabetical-end -pub mod dep_graph; -mod error; pub mod ich; pub mod query; diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs index c33af941f8026..87be4358fb8ba 100644 --- a/compiler/rustc_query_system/src/query/mod.rs +++ b/compiler/rustc_query_system/src/query/mod.rs @@ -1,139 +1,7 @@ use std::fmt::Debug; -use std::marker::PhantomData; -use std::mem::transmute; -use std::sync::Arc; -use rustc_data_structures::jobserver::Proxy; -use rustc_data_structures::sync::{DynSend, DynSync}; use rustc_errors::DiagInner; -use rustc_hashes::Hash64; -use rustc_hir::def::DefKind; use rustc_macros::{Decodable, Encodable}; -use rustc_span::Span; -use rustc_span::def_id::DefId; - -pub use self::caches::{ - DefIdCache, DefaultCache, QueryCache, QueryCacheKey, SingleCache, VecCache, -}; -pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryLatch, QueryWaiter}; -pub use self::plumbing::*; -use crate::dep_graph::{DepKind, DepNodeIndex, HasDepContext, SerializedDepNodeIndex}; - -mod caches; -mod job; -mod plumbing; - -/// How a particular query deals with query cycle errors. -/// -/// Inspected by the code that actually handles cycle errors, to decide what -/// approach to use. -#[derive(Copy, Clone)] -pub enum CycleErrorHandling { - Error, - Fatal, - DelayBug, - Stash, -} - -/// Description of a frame in the query stack. -/// -/// This is mostly used in case of cycles for error reporting. -#[derive(Clone, Debug)] -pub struct QueryStackFrame { - /// This field initially stores a `QueryStackDeferred` during collection, - /// but can later be changed to `QueryStackFrameExtra` containing concrete information - /// by calling `lift`. This is done so that collecting query does not need to invoke - /// queries, instead `lift` will call queries in a more appropriate location. - pub info: I, - - pub dep_kind: DepKind, - /// This hash is used to deterministically pick - /// a query to remove cycles in the parallel compiler. - pub hash: Hash64, - pub def_id: Option, - /// A def-id that is extracted from a `Ty` in a query key - pub def_id_for_ty_in_cycle: Option, -} - -impl<'tcx> QueryStackFrame> { - #[inline] - pub fn new( - info: QueryStackDeferred<'tcx>, - dep_kind: DepKind, - hash: Hash64, - def_id: Option, - def_id_for_ty_in_cycle: Option, - ) -> Self { - Self { info, def_id, dep_kind, hash, def_id_for_ty_in_cycle } - } - - fn lift(&self) -> QueryStackFrame { - QueryStackFrame { - info: self.info.extract(), - dep_kind: self.dep_kind, - hash: self.hash, - def_id: self.def_id, - def_id_for_ty_in_cycle: self.def_id_for_ty_in_cycle, - } - } -} - -#[derive(Clone, Debug)] -pub struct QueryStackFrameExtra { - pub description: String, - pub span: Option, - pub def_kind: Option, -} - -impl QueryStackFrameExtra { - #[inline] - pub fn new(description: String, span: Option, def_kind: Option) -> Self { - Self { description, span, def_kind } - } - - // FIXME(eddyb) Get more valid `Span`s on queries. - #[inline] - pub fn default_span(&self, span: Span) -> Span { - if !span.is_dummy() { - return span; - } - self.span.unwrap_or(span) - } -} - -/// Track a 'side effect' for a particular query. -/// This is used to hold a closure which can create `QueryStackFrameExtra`. -#[derive(Clone)] -pub struct QueryStackDeferred<'tcx> { - _dummy: PhantomData<&'tcx ()>, - - // `extract` may contain references to 'tcx, but we can't tell drop checking that it won't - // access it in the destructor. - extract: Arc QueryStackFrameExtra + DynSync + DynSend>, -} - -impl<'tcx> QueryStackDeferred<'tcx> { - pub fn new( - context: C, - extract: fn(C) -> QueryStackFrameExtra, - ) -> Self { - let extract: Arc QueryStackFrameExtra + DynSync + DynSend + 'tcx> = - Arc::new(move || extract(context)); - // SAFETY: The `extract` closure does not access 'tcx in its destructor as the only - // captured variable is `context` which is Copy and cannot have a destructor. - Self { _dummy: PhantomData, extract: unsafe { transmute(extract) } } - } - - pub fn extract(&self) -> QueryStackFrameExtra { - (self.extract)() - } -} - -impl<'tcx> Debug for QueryStackDeferred<'tcx> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("QueryStackDeferred") - } -} /// Tracks 'side effects' for a particular query. /// This struct is saved to disk along with the query result, @@ -152,18 +20,3 @@ pub enum QuerySideEffect { /// effect dep node as a dependency. Diagnostic(DiagInner), } - -pub trait QueryContext<'tcx>: HasDepContext { - /// Gets a jobserver reference which is used to release then acquire - /// a token while waiting on a query. - fn jobserver_proxy(&self) -> &Proxy; - - /// Load a side effect associated to the node in the previous session. - fn load_side_effect( - self, - prev_dep_node_index: SerializedDepNodeIndex, - ) -> Option; - - /// Register a side effect for the given node, for use in next session. - fn store_side_effect(self, dep_node_index: DepNodeIndex, side_effect: QuerySideEffect); -} diff --git a/triagebot.toml b/triagebot.toml index bc90371393b60..c9d6c1a730b7f 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -1556,6 +1556,7 @@ dep-bumps = [ "/compiler/rustc_llvm" = ["@cuviper"] "/compiler/rustc_codegen_llvm/src/debuginfo" = ["compiler", "debuginfo"] "/compiler/rustc_codegen_ssa" = ["compiler", "codegen"] +"/compiler/rustc_middle/src/dep_graph" = ["compiler", "incremental", "query-system"] "/compiler/rustc_middle/src/mir" = ["compiler", "mir"] "/compiler/rustc_middle/src/traits" = ["compiler", "types"] "/compiler/rustc_middle/src/ty" = ["compiler", "types"] @@ -1567,7 +1568,6 @@ dep-bumps = [ "/compiler/rustc_parse/src/lexer" = ["compiler", "lexer"] "/compiler/rustc_query_impl" = ["compiler", "query-system"] "/compiler/rustc_query_system" = ["compiler", "query-system"] -"/compiler/rustc_query_system/src/dep_graph" = ["compiler", "incremental", "query-system"] "/compiler/rustc_query_system/src/ich" = ["compiler", "incremental", "query-system"] "/compiler/rustc_trait_selection" = ["compiler", "types"] "/compiler/rustc_traits" = ["compiler", "types"]