From 3ebaa8b1f77bac217d5f878e11e414e9b3d7a526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sun, 1 Feb 2026 21:01:50 +0100 Subject: [PATCH] Remove generics from `DepGraph` --- .../src/assert_module_sources.rs | 3 +- compiler/rustc_driver_impl/src/pretty.rs | 3 +- .../rustc_incremental/src/assert_dep_graph.rs | 4 +- .../src/persist/dirty_clean.rs | 4 +- .../rustc_incremental/src/persist/load.rs | 4 +- .../rustc_incremental/src/persist/save.rs | 9 +- compiler/rustc_interface/src/passes.rs | 3 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 4 +- .../rustc_middle/src/dep_graph/dep_node.rs | 20 ++- compiler/rustc_middle/src/dep_graph/mod.rs | 21 ++- .../rustc_middle/src/query/on_disk_cache.rs | 3 +- compiler/rustc_middle/src/ty/context.rs | 6 +- compiler/rustc_query_impl/src/plumbing.rs | 10 +- compiler/rustc_query_system/src/cache.rs | 2 +- .../src/dep_graph/dep_node.rs | 24 ++- .../rustc_query_system/src/dep_graph/graph.rs | 137 +++++++++--------- .../rustc_query_system/src/dep_graph/mod.rs | 40 ++--- .../src/dep_graph/serialized.rs | 100 +++++++------ .../rustc_query_system/src/query/plumbing.rs | 46 +++--- compiler/rustc_symbol_mangling/src/test.rs | 3 +- 20 files changed, 248 insertions(+), 198 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs index 43e1e135a666b..dea1aeb75be7a 100644 --- a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs +++ b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs @@ -30,6 +30,7 @@ use rustc_data_structures::unord::{UnordMap, UnordSet}; use rustc_errors::{DiagArgValue, IntoDiagArg}; use rustc_hir as hir; use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::dep_graph::DepContext; use rustc_middle::mir::mono::CodegenUnitNameBuilder; use rustc_middle::ty::TyCtxt; use rustc_session::Session; @@ -41,7 +42,7 @@ use crate::errors; #[allow(missing_docs)] pub fn assert_module_sources(tcx: TyCtxt<'_>, set_reuse: &dyn Fn(&mut CguReuseTracker)) { - tcx.dep_graph.with_ignore(|| { + tcx.with_ignore(|| { if tcx.sess.opts.incremental.is_none() { return; } diff --git a/compiler/rustc_driver_impl/src/pretty.rs b/compiler/rustc_driver_impl/src/pretty.rs index 962d035db862f..f7d238a345d6c 100644 --- a/compiler/rustc_driver_impl/src/pretty.rs +++ b/compiler/rustc_driver_impl/src/pretty.rs @@ -5,6 +5,7 @@ use std::fmt::Write; use rustc_ast_pretty::pprust as pprust_ast; use rustc_middle::bug; +use rustc_middle::dep_graph::DepContext; use rustc_middle::mir::{write_mir_graphviz, write_mir_pretty}; use rustc_middle::ty::{self, TyCtxt}; use rustc_mir_build::thir::print::{thir_flat, thir_tree}; @@ -286,7 +287,7 @@ pub fn print<'tcx>(sess: &Session, ppm: PpMode, ex: PrintExtra<'tcx>) { } PpHirMode::Typed => { let annotation = HirTypedAnn { tcx, maybe_typeck_results: Cell::new(None) }; - tcx.dep_graph.with_ignore(|| f(&annotation)) + tcx.with_ignore(|| f(&annotation)) } } } diff --git a/compiler/rustc_incremental/src/assert_dep_graph.rs b/compiler/rustc_incremental/src/assert_dep_graph.rs index 0e04a2a784ec8..9e402d2823cb9 100644 --- a/compiler/rustc_incremental/src/assert_dep_graph.rs +++ b/compiler/rustc_incremental/src/assert_dep_graph.rs @@ -42,7 +42,7 @@ use rustc_data_structures::graph::linked_graph::{Direction, INCOMING, NodeIndex, use rustc_hir::def_id::{CRATE_DEF_ID, DefId, LocalDefId}; use rustc_hir::intravisit::{self, Visitor}; use rustc_middle::dep_graph::{ - DepGraphQuery, DepKind, DepNode, DepNodeExt, DepNodeFilter, EdgeFilter, dep_kinds, + DepContext, DepGraphQuery, DepKind, DepNode, DepNodeExt, DepNodeFilter, EdgeFilter, dep_kinds, }; use rustc_middle::hir::nested_filter; use rustc_middle::ty::TyCtxt; @@ -55,7 +55,7 @@ use crate::errors; #[allow(missing_docs)] pub(crate) fn assert_dep_graph(tcx: TyCtxt<'_>) { - tcx.dep_graph.with_ignore(|| { + tcx.with_ignore(|| { if tcx.sess.opts.unstable_opts.dump_dep_graph { tcx.dep_graph.with_query(dump_graph); } diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs index 71fb188952466..a0ade6939dcf1 100644 --- a/compiler/rustc_incremental/src/persist/dirty_clean.rs +++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs @@ -26,7 +26,7 @@ use rustc_hir::def_id::LocalDefId; use rustc_hir::{ Attribute, ImplItemKind, ItemKind as HirItem, Node as HirNode, TraitItemKind, intravisit, }; -use rustc_middle::dep_graph::{DepNode, DepNodeExt, dep_kind_from_label, label_strs}; +use rustc_middle::dep_graph::{DepContext, DepNode, DepNodeExt, dep_kind_from_label, label_strs}; use rustc_middle::hir::nested_filter; use rustc_middle::ty::TyCtxt; use rustc_span::{Span, Symbol, sym}; @@ -144,7 +144,7 @@ pub(crate) fn check_dirty_clean_annotations(tcx: TyCtxt<'_>) { return; } - tcx.dep_graph.with_ignore(|| { + tcx.with_ignore(|| { let mut dirty_clean_visitor = DirtyCleanVisitor { tcx, checked_attrs: Default::default() }; let crate_items = tcx.hir_crate_items(()); diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index e93cba2c8d0fc..ae647e35e1a44 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use rustc_data_structures::memmap::Mmap; use rustc_data_structures::unord::UnordMap; use rustc_hashes::Hash64; -use rustc_middle::dep_graph::{DepGraph, DepsType, SerializedDepGraph, WorkProductMap}; +use rustc_middle::dep_graph::{DEP_KIND_NAMES, DepGraph, SerializedDepGraph, WorkProductMap}; use rustc_middle::query::on_disk_cache::OnDiskCache; use rustc_serialize::Decodable; use rustc_serialize::opaque::MemDecoder; @@ -171,7 +171,7 @@ fn load_dep_graph(sess: &Session) -> LoadResult<(Arc, WorkPr return LoadResult::DataOutOfDate; } - let dep_graph = SerializedDepGraph::decode::(&mut decoder); + let dep_graph = SerializedDepGraph::decode(&mut decoder, DEP_KIND_NAMES); LoadResult::Ok { data: (dep_graph, prev_work_products) } } diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs index 58fea3278a839..9d14ab5d8a7fe 100644 --- a/compiler/rustc_incremental/src/persist/save.rs +++ b/compiler/rustc_incremental/src/persist/save.rs @@ -4,7 +4,8 @@ use std::sync::Arc; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::sync::join; use rustc_middle::dep_graph::{ - DepGraph, SerializedDepGraph, WorkProduct, WorkProductId, WorkProductMap, + DEP_KIND_VARIANTS, DepContext, DepGraph, DepsType, SerializedDepGraph, WorkProduct, + WorkProductId, WorkProductMap, }; use rustc_middle::ty::TyCtxt; use rustc_serialize::Encodable as RustcEncodable; @@ -27,7 +28,7 @@ use crate::errors; /// Trying to execute a query afterwards would attempt to read the result cache we just dropped. pub(crate) fn save_dep_graph(tcx: TyCtxt<'_>) { debug!("save_dep_graph()"); - tcx.dep_graph.with_ignore(|| { + tcx.with_ignore(|| { let sess = tcx.sess; if sess.opts.incremental.is_none() { return; @@ -89,7 +90,7 @@ pub fn save_work_product_index( } debug!("save_work_product_index()"); - dep_graph.assert_ignored(); + dep_graph.assert_ignored::(); let path = work_products_path(sess); file_format::save_in(sess, path, "work product index", |mut e| { encode_work_product_index(&new_work_products, &mut e); @@ -168,5 +169,5 @@ pub(crate) fn build_dep_graph( // First encode the commandline arguments hash sess.opts.dep_tracking_hash(false).encode(&mut encoder); - Some(DepGraph::new(sess, prev_graph, prev_work_products, encoder)) + Some(DepGraph::new(sess, prev_graph, prev_work_products, encoder, DEP_KIND_VARIANTS as usize)) } diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index f5fb8031ab0f0..b4a42d8fb9787 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -28,6 +28,7 @@ use rustc_lint::{BufferedEarlyLint, EarlyCheckNode, LintStore, unerased_lint_sto use rustc_metadata::EncodedMetadata; use rustc_metadata::creader::CStore; use rustc_middle::arena::Arena; +use rustc_middle::dep_graph::DepsType; use rustc_middle::ty::{self, CurrentGcx, GlobalCtxt, RegisteredTools, TyCtxt}; use rustc_middle::util::Providers; use rustc_parse::lexer::StripTokens; @@ -951,7 +952,7 @@ pub fn create_and_enter_global_ctxt FnOnce(TyCtxt<'tcx>) -> T>( // We're constructing the HIR here; we don't care what we will // read, since we haven't even constructed the *input* to // incr. comp. yet. - dep_graph.assert_ignored(); + dep_graph.assert_ignored::(); let query_result_on_disk_cache = rustc_incremental::load_query_result_cache(sess); diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 471c19ed4d4e7..f4594c1eb8735 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -17,7 +17,7 @@ use rustc_hir::def_id::{CRATE_DEF_ID, CRATE_DEF_INDEX, LOCAL_CRATE, LocalDefId, use rustc_hir::definitions::DefPathData; use rustc_hir::find_attr; use rustc_hir_pretty::id_to_string; -use rustc_middle::dep_graph::WorkProductId; +use rustc_middle::dep_graph::{DepContext, WorkProductId}; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::mir::interpret; use rustc_middle::query::Providers; @@ -2420,7 +2420,7 @@ impl Decodable for EncodedMetadata { pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { // Since encoding metadata is not in a query, and nothing is cached, // there's no need to do dep-graph tracking for any of it. - tcx.dep_graph.assert_ignored(); + tcx.assert_ignored(); // Generate the metadata stub manually, as that is a small file compared to full metadata. if let Some(ref_path) = ref_path { diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs index f5ed0570667cc..ec46effabb765 100644 --- a/compiler/rustc_middle/src/dep_graph/dep_node.rs +++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs @@ -5,6 +5,9 @@ use rustc_hir::{HirId, ItemLocalId, OwnerId}; pub use rustc_query_system::dep_graph::DepNode; use rustc_query_system::dep_graph::FingerprintStyle; pub use rustc_query_system::dep_graph::dep_node::DepKind; +use rustc_query_system::dep_graph::dep_node::{ + DEP_KIND_ANON_ZERO_DEPS, DEP_KIND_NULL, DEP_KIND_RED, DEP_KIND_SIDE_EFFECT, +}; pub(crate) use rustc_query_system::dep_graph::{DepContext, DepNodeParams}; use rustc_span::Symbol; @@ -47,7 +50,7 @@ macro_rules! define_dep_nodes { // This checks that the discriminants of the variants have been assigned consecutively // from 0 so that they can be used as a dense index. - pub(crate) const DEP_KIND_VARIANTS: u16 = { + pub const DEP_KIND_VARIANTS: u16 = { let deps = &[$(dep_kinds::$variant,)*]; let mut i = 0; while i < deps.len() { @@ -61,7 +64,7 @@ macro_rules! define_dep_nodes { /// List containing the name of each dep kind as a static string, /// indexable by `DepKind`. - pub(crate) const DEP_KIND_NAMES: &[&str] = &[ + pub const DEP_KIND_NAMES: &[&str] = &[ $( self::label_strs::$variant, )* ]; @@ -84,18 +87,27 @@ macro_rules! define_dep_nodes { // Create various data structures for each query, and also for a few things // that aren't queries. rustc_with_all_queries!(define_dep_nodes![ - /// We use this for most things when incr. comp. is turned off. + // Make sure this list matches the BuiltinDepKinds enum. [] fn Null() -> (), - /// We use this to create a forever-red node. [] fn Red() -> (), [] fn SideEffect() -> (), [] fn AnonZeroDeps() -> (), + // End of list. + [] fn TraitSelect() -> (), [] fn CompileCodegenUnit() -> (), [] fn CompileMonoItem() -> (), [] fn Metadata() -> (), ]); +const _: () = { + // Test that the list matches. + assert!(dep_kinds::Null.as_inner() == DEP_KIND_NULL.as_inner()); + assert!(dep_kinds::Red.as_inner() == DEP_KIND_RED.as_inner()); + assert!(dep_kinds::SideEffect.as_inner() == DEP_KIND_SIDE_EFFECT.as_inner()); + assert!(dep_kinds::AnonZeroDeps.as_inner() == DEP_KIND_ANON_ZERO_DEPS.as_inner()); +}; + // WARNING: `construct` is generic and does not know that `CompileCodegenUnit` takes `Symbol`s as keys. // Be very careful changing this type signature! pub(crate) fn make_compile_codegen_unit(tcx: TyCtxt<'_>, name: Symbol) -> DepNode { diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs index f28ba10d52e27..455f30808fb32 100644 --- a/compiler/rustc_middle/src/dep_graph/mod.rs +++ b/compiler/rustc_middle/src/dep_graph/mod.rs @@ -1,4 +1,5 @@ use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_query_system::dep_graph::{DEP_KIND_UNUSED_BITS, unused_dep_kind_bits}; use rustc_query_system::ich::StableHashingContext; use rustc_session::Session; @@ -8,7 +9,10 @@ use crate::ty::{self, TyCtxt}; #[macro_use] mod dep_node; -pub use dep_node::{DepKind, DepNode, DepNodeExt, dep_kind_from_label, dep_kinds, label_strs}; +pub use dep_node::{ + DEP_KIND_NAMES, DEP_KIND_VARIANTS, DepKind, DepNode, DepNodeExt, dep_kind_from_label, + dep_kinds, label_strs, +}; pub(crate) use dep_node::{make_compile_codegen_unit, make_compile_mono_item, make_metadata}; pub use rustc_query_system::dep_graph::debug::{DepNodeFilter, EdgeFilter}; pub use rustc_query_system::dep_graph::{ @@ -16,7 +20,7 @@ pub use rustc_query_system::dep_graph::{ TaskDepsRef, WorkProduct, WorkProductId, WorkProductMap, hash_result, }; -pub type DepGraph = rustc_query_system::dep_graph::DepGraph; +pub type DepGraph = rustc_query_system::dep_graph::DepGraph; pub type DepKindVTable<'tcx> = rustc_query_system::dep_graph::DepKindVTable>; @@ -43,18 +47,11 @@ impl Deps for DepsType { op(icx.task_deps) }) } - - fn name(dep_kind: DepKind) -> &'static str { - dep_node::DEP_KIND_NAMES[dep_kind.as_usize()] - } - - const DEP_KIND_NULL: DepKind = dep_kinds::Null; - const DEP_KIND_RED: DepKind = dep_kinds::Red; - const DEP_KIND_SIDE_EFFECT: DepKind = dep_kinds::SideEffect; - const DEP_KIND_ANON_ZERO_DEPS: DepKind = dep_kinds::AnonZeroDeps; - const DEP_KIND_MAX: u16 = dep_node::DEP_KIND_VARIANTS - 1; } +/// Verify that the unused bits for the dep kind matches the hardcoded value in `rustc_query_system`. +const _: [(); unused_dep_kind_bits(dep_node::DEP_KIND_VARIANTS)] = [(); DEP_KIND_UNUSED_BITS]; + impl<'tcx> DepContext for TyCtxt<'tcx> { type Deps = DepsType; diff --git a/compiler/rustc_middle/src/query/on_disk_cache.rs b/compiler/rustc_middle/src/query/on_disk_cache.rs index 5ef0c2500e7a9..6e589194b0d79 100644 --- a/compiler/rustc_middle/src/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/query/on_disk_cache.rs @@ -11,6 +11,7 @@ use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LOCAL_CRATE, LocalDefId, Stab use rustc_hir::definitions::DefPathHash; use rustc_index::{Idx, IndexVec}; use rustc_macros::{Decodable, Encodable}; +use rustc_query_system::dep_graph::DepContext; use rustc_query_system::query::QuerySideEffect; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder}; use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; @@ -222,7 +223,7 @@ impl OnDiskCache { pub fn serialize(&self, tcx: TyCtxt<'_>, encoder: FileEncoder) -> FileEncodeResult { // Serializing the `DepGraph` should not modify it. - tcx.dep_graph.with_ignore(|| { + tcx.with_ignore(|| { // Allocate `SourceFileIndex`es. let (file_to_file_index, file_index_to_stable_id) = { let files = tcx.sess.source_map().files(); diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 41e1388e31464..204fcc63df605 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -41,7 +41,7 @@ use rustc_hir::limit::Limit; use rustc_hir::{self as hir, HirId, Node, TraitCandidate, find_attr}; use rustc_index::IndexVec; use rustc_query_system::cache::WithDepNode; -use rustc_query_system::dep_graph::DepNodeIndex; +use rustc_query_system::dep_graph::{DepContext, DepNodeIndex}; use rustc_query_system::ich::StableHashingContext; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder}; use rustc_session::Session; @@ -1423,7 +1423,7 @@ impl<'tcx> TyCtxt<'tcx> { /// Can only be fed before queries are run, and is thus exempt from any /// incremental issues. Do not use except for the initial query feeding. pub fn feed_unit_query(self) -> TyCtxtFeed<'tcx, ()> { - self.dep_graph.assert_ignored(); + self.assert_ignored(); TyCtxtFeed { tcx: self, key: () } } @@ -2161,7 +2161,7 @@ impl<'tcx> TyCtxt<'tcx> { // We need to ensure that these side effects are re-run by the incr. comp. engine. // Depending on the forever-red node will tell the graph that the calling query // needs to be re-evaluated. - self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE); + self.read_index(DepNodeIndex::FOREVER_RED_NODE); let feed = TyCtxtFeed { tcx: self, key: def_id }; feed.def_kind(def_kind); diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index d1721f1f455f7..116524230413f 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -13,8 +13,8 @@ use rustc_hir::limit::Limit; use rustc_index::Idx; use rustc_middle::bug; use rustc_middle::dep_graph::{ - self, DepContext, DepKind, DepKindVTable, DepNode, DepNodeIndex, SerializedDepNodeIndex, - dep_kinds, + self, DepContext, DepKind, DepKindVTable, DepNode, DepNodeIndex, DepsType, + SerializedDepNodeIndex, dep_kinds, }; use rustc_middle::query::Key; use rustc_middle::query::on_disk_cache::{ @@ -468,9 +468,9 @@ where // The call to `with_query_deserialization` enforces that no new `DepNodes` // are created during deserialization. See the docs of that method for more // details. - let value = tcx - .dep_graph - .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index)); + let value = tcx.dep_graph.with_query_deserialization::(|| { + on_disk_cache.try_load_query_result(tcx, prev_index) + }); prof_timer.finish_with_query_invocation_id(index.into()); diff --git a/compiler/rustc_query_system/src/cache.rs b/compiler/rustc_query_system/src/cache.rs index 1b8332ad9e015..3848deb4929dd 100644 --- a/compiler/rustc_query_system/src/cache.rs +++ b/compiler/rustc_query_system/src/cache.rs @@ -52,7 +52,7 @@ impl WithDepNode { } pub fn get(&self, tcx: Tcx) -> T { - tcx.dep_graph().read_index(self.dep_node); + tcx.read_index(self.dep_node); self.cached_value.clone() } } diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs index b9d35bc5a9379..bc270824b9e1a 100644 --- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs +++ b/compiler/rustc_query_system/src/dep_graph/dep_node.rs @@ -90,6 +90,27 @@ impl DepKind { } } +/// This must match the use of the define_dep_nodes! macro. +#[repr(u16)] +enum BuiltinDepKinds { + /// We use this for most things when incr. comp. is turned off. + Null, + + /// We use this to create a forever-red node. + Red, + + /// We use this to create a side effect node. + SideEffect, + + /// We use this to create the anon node with zero dependencies. + AnonZeroDeps, +} + +pub const DEP_KIND_NULL: DepKind = DepKind::new(BuiltinDepKinds::Null as u16); +pub const DEP_KIND_RED: DepKind = DepKind::new(BuiltinDepKinds::Red as u16); +pub const DEP_KIND_SIDE_EFFECT: DepKind = DepKind::new(BuiltinDepKinds::SideEffect as u16); +pub const DEP_KIND_ANON_ZERO_DEPS: DepKind = DepKind::new(BuiltinDepKinds::AnonZeroDeps as u16); + pub fn default_dep_kind_debug(kind: DepKind, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("DepKind").field("variant", &kind.variant).finish() } @@ -135,7 +156,8 @@ impl DepNode { && (tcx.sess().opts.unstable_opts.incremental_info || tcx.sess().opts.unstable_opts.query_dep_graph) { - tcx.dep_graph().register_dep_node_debug_str(dep_node, || arg.to_debug_str(tcx)); + tcx.dep_graph() + .register_dep_node_debug_str(tcx, dep_node, || arg.to_debug_str(tcx)); } } diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 06e576baf22ad..10ab92c5c5213 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -24,12 +24,14 @@ use {super::debug::EdgeFilter, std::env}; use super::query::DepGraphQuery; use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex}; use super::{DepContext, DepKind, DepNode, Deps, HasDepContext, WorkProductId}; +use crate::dep_graph::dep_node::{DEP_KIND_ANON_ZERO_DEPS, DEP_KIND_RED, DEP_KIND_SIDE_EFFECT}; use crate::dep_graph::edges::EdgesVec; use crate::ich::StableHashingContext; use crate::query::{QueryContext, QuerySideEffect}; -pub struct DepGraph { - data: Option>>, +#[derive(Clone)] +pub struct DepGraph { + data: Option>, /// This field is used for assigning DepNodeIndices when running in /// non-incremental mode. Even in non-incremental mode we make sure that @@ -38,17 +40,6 @@ pub struct DepGraph { virtual_dep_node_index: Arc, } -/// Manual clone impl that does not require `D: Clone`. -impl Clone for DepGraph { - fn clone(&self) -> Self { - let Self { data, virtual_dep_node_index } = self; - Self { - data: Option::>::clone(data), - virtual_dep_node_index: Arc::clone(virtual_dep_node_index), - } - } -} - rustc_index::newtype_index! { pub struct DepNodeIndex {} } @@ -82,12 +73,12 @@ pub(super) enum DepNodeColor { Unknown, } -pub(crate) struct DepGraphData { +pub(crate) struct DepGraphData { /// The new encoding of the dependency graph, optimized for red/green /// tracking. The `current` field is the dependency graph of only the /// current compilation session: We don't merge the previous dep-graph into /// current one anymore, but we do reference shared data to save space. - current: CurrentDepGraph, + current: CurrentDepGraph, /// The dep-graph from the previous compilation session. It contains all /// nodes and edges as well as all fingerprints of nodes that have them. @@ -118,23 +109,29 @@ where stable_hasher.finish() } -impl DepGraph { +impl DepGraph { pub fn new( session: &Session, prev_graph: Arc, prev_work_products: WorkProductMap, encoder: FileEncoder, - ) -> DepGraph { + dep_kind_count: usize, + ) -> DepGraph { let prev_graph_node_count = prev_graph.node_count(); - let current = - CurrentDepGraph::new(session, prev_graph_node_count, encoder, Arc::clone(&prev_graph)); + let current = CurrentDepGraph::new( + session, + prev_graph_node_count, + encoder, + Arc::clone(&prev_graph), + dep_kind_count, + ); let colors = DepNodeColorMap::new(prev_graph_node_count); // Instantiate a node with zero dependencies only once for anonymous queries. let _green_node_index = current.alloc_new_node( - DepNode { kind: D::DEP_KIND_ANON_ZERO_DEPS, hash: current.anon_id_seed.into() }, + DepNode { kind: DEP_KIND_ANON_ZERO_DEPS, hash: current.anon_id_seed.into() }, EdgesVec::new(), Fingerprint::ZERO, ); @@ -142,7 +139,7 @@ impl DepGraph { // Instantiate a dependy-less red node only once for anonymous queries. let red_node_index = current.alloc_new_node( - DepNode { kind: D::DEP_KIND_RED, hash: Fingerprint::ZERO.into() }, + DepNode { kind: DEP_KIND_RED, hash: Fingerprint::ZERO.into() }, EdgesVec::new(), Fingerprint::ZERO, ); @@ -166,12 +163,12 @@ impl DepGraph { } } - pub fn new_disabled() -> DepGraph { + pub fn new_disabled() -> DepGraph { DepGraph { data: None, virtual_dep_node_index: Arc::new(AtomicU32::new(0)) } } #[inline] - pub(crate) fn data(&self) -> Option<&DepGraphData> { + pub(crate) fn data(&self) -> Option<&DepGraphData> { self.data.as_deref() } @@ -187,7 +184,7 @@ impl DepGraph { } } - pub fn assert_ignored(&self) { + pub fn assert_ignored(&self) { if let Some(..) = self.data { D::read_deps(|task_deps| { assert_matches!( @@ -199,13 +196,6 @@ impl DepGraph { } } - pub fn with_ignore(&self, op: OP) -> R - where - OP: FnOnce() -> R, - { - D::with_deps(TaskDepsRef::Ignore, op) - } - /// Used to wrap the deserialization of a query result from disk, /// This method enforces that no new `DepNodes` are created during /// query result deserialization. @@ -252,7 +242,7 @@ impl DepGraph { /// in the query infrastructure, and is not currently needed by the /// decoding of any query results. Should the need arise in the future, /// we should consider extending the query system with this functionality. - pub fn with_query_deserialization(&self, op: OP) -> R + pub fn with_query_deserialization(&self, op: OP) -> R where OP: FnOnce() -> R, { @@ -260,7 +250,7 @@ impl DepGraph { } #[inline(always)] - pub fn with_task, A: Debug, R>( + pub fn with_task( &self, key: DepNode, cx: Ctxt, @@ -274,7 +264,7 @@ impl DepGraph { } } - pub fn with_anon_task, OP, R>( + pub fn with_anon_task( &self, cx: Tcx, dep_kind: DepKind, @@ -286,7 +276,7 @@ impl DepGraph { match self.data() { Some(data) => { let (result, index) = data.with_anon_task_inner(cx, dep_kind, op); - self.read_index(index); + cx.read_index(index); (result, index) } None => (op(), self.next_virtual_depnode_index()), @@ -294,7 +284,7 @@ impl DepGraph { } } -impl DepGraphData { +impl DepGraphData { /// Starts a new dep-graph task. Dep-graph tasks are specified /// using a free function (`task`) and **not** a closure -- this /// is intentional because we want to exercise tight control over @@ -323,7 +313,7 @@ impl DepGraphData { /// /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/queries/incremental-compilation.html #[inline(always)] - pub(crate) fn with_task, A: Debug, R>( + pub(crate) fn with_task( &self, key: DepNode, cx: Ctxt, @@ -344,7 +334,7 @@ impl DepGraphData { ) }); - let with_deps = |task_deps| D::with_deps(task_deps, || task(cx, arg)); + let with_deps = |task_deps| Ctxt::Deps::with_deps(task_deps, || task(cx, arg)); let (result, edges) = if cx.dep_context().is_eval_always(key.kind) { (with_deps(TaskDepsRef::EvalAlways), EdgesVec::new()) } else { @@ -373,7 +363,7 @@ impl DepGraphData { /// FIXME: This could perhaps return a `WithDepNode` to ensure that the /// user of this function actually performs the read; we'll have to see /// how to make that work with `anon` in `execute_job_incr`, though. - pub(crate) fn with_anon_task_inner, OP, R>( + pub(crate) fn with_anon_task_inner( &self, cx: Tcx, dep_kind: DepKind, @@ -391,7 +381,7 @@ impl DepGraphData { None, 128, )); - let result = D::with_deps(TaskDepsRef::Allow(&task_deps), op); + let result = Tcx::Deps::with_deps(TaskDepsRef::Allow(&task_deps), op); let task_deps = task_deps.into_inner(); let reads = task_deps.reads; @@ -442,7 +432,7 @@ impl DepGraphData { } /// Intern the new `DepNode` with the dependencies up-to-now. - fn hash_result_and_alloc_node, R>( + fn hash_result_and_alloc_node( &self, cx: &Ctxt, node: DepNode, @@ -460,9 +450,9 @@ impl DepGraphData { } } -impl DepGraph { +impl DepGraph { #[inline] - pub fn read_index(&self, dep_node_index: DepNodeIndex) { + pub fn read_index(&self, dep_node_index: DepNodeIndex) { if let Some(ref data) = self.data { D::read_deps(|task_deps| { let mut task_deps = match task_deps { @@ -525,10 +515,10 @@ impl DepGraph { diagnostic: &DiagInner, ) { if let Some(ref data) = self.data { - D::read_deps(|task_deps| match task_deps { + Qcx::Deps::read_deps(|task_deps| match task_deps { TaskDepsRef::EvalAlways | TaskDepsRef::Ignore => return, TaskDepsRef::Forbid | TaskDepsRef::Allow(..) => { - self.read_index(data.encode_diagnostic(qcx, diagnostic)); + qcx.dep_context().read_index(data.encode_diagnostic(qcx, diagnostic)); } }) } @@ -561,7 +551,7 @@ impl DepGraph { /// FIXME: If the code is changed enough for this node to be marked before requiring the /// caller's node, we suppose that those changes will be enough to mark this node red and /// force a recomputation using the "normal" way. - pub fn with_feed_task, R>( + pub fn with_feed_task( &self, node: DepNode, cx: Ctxt, @@ -602,7 +592,7 @@ impl DepGraph { } let mut edges = EdgesVec::new(); - D::read_deps(|task_deps| match task_deps { + Ctxt::Deps::read_deps(|task_deps| match task_deps { TaskDepsRef::Allow(deps) => edges.extend(deps.lock().reads.iter().copied()), TaskDepsRef::EvalAlways => { edges.push(DepNodeIndex::FOREVER_RED_NODE); @@ -624,7 +614,7 @@ impl DepGraph { } } -impl DepGraphData { +impl DepGraphData { fn assert_dep_node_not_yet_allocated_in_current_session( &self, dep_node: &DepNode, @@ -682,7 +672,7 @@ impl DepGraphData { // Use `send_new` so we get an unique index, even though the dep node is not. let dep_node_index = self.current.encoder.send_new( DepNode { - kind: D::DEP_KIND_SIDE_EFFECT, + kind: DEP_KIND_SIDE_EFFECT, hash: PackedFingerprint::from(Fingerprint::ZERO), }, Fingerprint::ZERO, @@ -703,7 +693,7 @@ impl DepGraphData { qcx: Qcx, prev_index: SerializedDepNodeIndex, ) { - D::with_deps(TaskDepsRef::Ignore, || { + Qcx::Deps::with_deps(TaskDepsRef::Ignore, || { let side_effect = qcx.load_side_effect(prev_index).unwrap(); match &side_effect { @@ -719,7 +709,7 @@ impl DepGraphData { prev_index, &self.colors, DepNode { - kind: D::DEP_KIND_SIDE_EFFECT, + kind: DEP_KIND_SIDE_EFFECT, hash: PackedFingerprint::from(Fingerprint::ZERO), }, Fingerprint::ZERO, @@ -792,7 +782,7 @@ impl DepGraphData { } } -impl DepGraph { +impl DepGraph { /// Checks whether a previous work product exists for `v` and, if /// so, return the path that leads to it. Used to skip doing work. pub fn previous_work_product(&self, v: &WorkProductId) -> Option { @@ -823,16 +813,18 @@ impl DepGraph { #[cfg(debug_assertions)] #[inline(always)] - pub(crate) fn register_dep_node_debug_str(&self, dep_node: DepNode, debug_str_gen: F) - where - F: FnOnce() -> String, - { + pub(crate) fn register_dep_node_debug_str( + &self, + ctxt: impl DepContext, + dep_node: DepNode, + debug_str_gen: impl FnOnce() -> String, + ) { let dep_node_debug = &self.data.as_ref().unwrap().dep_node_debug; if dep_node_debug.borrow().contains_key(&dep_node) { return; } - let debug_str = self.with_ignore(debug_str_gen); + let debug_str = ctxt.with_ignore(debug_str_gen); dep_node_debug.borrow_mut().insert(dep_node, debug_str); } @@ -848,7 +840,7 @@ impl DepGraph { DepNodeColor::Unknown } - pub fn try_mark_green<'tcx, Qcx: QueryContext<'tcx, Deps = D>>( + pub fn try_mark_green<'tcx, Qcx: QueryContext<'tcx>>( &self, qcx: Qcx, dep_node: &DepNode, @@ -857,13 +849,13 @@ impl DepGraph { } } -impl DepGraphData { +impl DepGraphData { /// Try to mark a node index for the node dep_node. /// /// A node will have an index, when it's already been marked green, or when we can mark it /// green. This function will mark the current task as a reader of the specified node, when /// a node index can be found for that node. - pub(crate) fn try_mark_green<'tcx, Qcx: QueryContext<'tcx, Deps = D>>( + pub(crate) fn try_mark_green<'tcx, Qcx: QueryContext<'tcx>>( &self, qcx: Qcx, dep_node: &DepNode, @@ -890,7 +882,7 @@ impl DepGraphData { } #[instrument(skip(self, qcx, parent_dep_node_index, frame), level = "debug")] - fn try_mark_parent_green<'tcx, Qcx: QueryContext<'tcx, Deps = D>>( + fn try_mark_parent_green<'tcx, Qcx: QueryContext<'tcx>>( &self, qcx: Qcx, parent_dep_node_index: SerializedDepNodeIndex, @@ -979,7 +971,7 @@ impl DepGraphData { /// Try to mark a dep-node which existed in the previous compilation session as green. #[instrument(skip(self, qcx, prev_dep_node_index, frame), level = "debug")] - fn try_mark_previous_green<'tcx, Qcx: QueryContext<'tcx, Deps = D>>( + fn try_mark_previous_green<'tcx, Qcx: QueryContext<'tcx>>( &self, qcx: Qcx, prev_dep_node_index: SerializedDepNodeIndex, @@ -1020,7 +1012,7 @@ impl DepGraphData { } } -impl DepGraph { +impl DepGraph { /// Returns true if the given node has been marked as red during the /// current compilation session. Used in various assertions pub fn is_red(&self, dep_node: &DepNode) -> bool { @@ -1153,8 +1145,8 @@ rustc_index::newtype_index! { /// `anon_node_to_index` and `data`, or `prev_index_to_index` and `data`. When /// manipulating both, we acquire `anon_node_to_index` or `prev_index_to_index` /// first, and `data` second. -pub(super) struct CurrentDepGraph { - encoder: GraphEncoder, +pub(super) struct CurrentDepGraph { + encoder: GraphEncoder, anon_node_to_index: ShardedHashMap, /// This is used to verify that fingerprints do not change between the creation of a node @@ -1192,12 +1184,13 @@ pub(super) struct CurrentDepGraph { pub(super) total_duplicate_read_count: AtomicU64, } -impl CurrentDepGraph { +impl CurrentDepGraph { fn new( session: &Session, prev_graph_node_count: usize, encoder: FileEncoder, previous: Arc, + dep_kind_count: usize, ) -> Self { let mut stable_hasher = StableHasher::new(); previous.session_count().hash(&mut stable_hasher); @@ -1218,7 +1211,13 @@ impl CurrentDepGraph { session.opts.unstable_opts.incremental_verify_ich || cfg!(debug_assertions); CurrentDepGraph { - encoder: GraphEncoder::new(session, encoder, prev_graph_node_count, previous), + encoder: GraphEncoder::new( + session, + encoder, + prev_graph_node_count, + previous, + dep_kind_count, + ), anon_node_to_index: ShardedHashMap::with_capacity( // FIXME: The count estimate is off as anon nodes are only a portion of the nodes. new_node_count_estimate / sharded::shards(), @@ -1425,7 +1424,7 @@ impl DepNodeColorMap { #[inline(never)] #[cold] -pub(crate) fn print_markframe_trace(graph: &DepGraph, frame: &MarkFrame<'_>) { +pub(crate) fn print_markframe_trace(graph: &DepGraph, frame: &MarkFrame<'_>) { let data = graph.data.as_ref().unwrap(); eprintln!("there was a panic while trying to force a dep node"); @@ -1445,7 +1444,7 @@ pub(crate) fn print_markframe_trace(graph: &DepGraph, frame: &MarkFr #[cold] #[inline(never)] -fn panic_on_forbidden_read(data: &DepGraphData, dep_node_index: DepNodeIndex) -> ! { +fn panic_on_forbidden_read(data: &DepGraphData, dep_node_index: DepNodeIndex) -> ! { // We have to do an expensive reverse-lookup of the DepNode that // corresponds to `dep_node_index`, but that's OK since we are about // to ICE anyway. diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs index c3ac1c7bc2d5a..b93018cc0e82e 100644 --- a/compiler/rustc_query_system/src/dep_graph/mod.rs +++ b/compiler/rustc_query_system/src/dep_graph/mod.rs @@ -14,7 +14,9 @@ pub use query::DepGraphQuery; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sync::DynSync; use rustc_session::Session; -pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex}; +pub use serialized::{ + DEP_KIND_UNUSED_BITS, SerializedDepGraph, SerializedDepNodeIndex, unused_dep_kind_bits, +}; use tracing::instrument; use self::graph::{MarkFrame, print_markframe_trace}; @@ -27,7 +29,7 @@ pub trait DepContext: Copy { fn with_stable_hashing_context(self, f: impl FnOnce(StableHashingContext<'_>) -> R) -> R; /// Access the DepGraph. - fn dep_graph(&self) -> &DepGraph; + fn dep_graph(&self) -> &DepGraph; /// Access the profiler. fn profiler(&self) -> &SelfProfilerRef; @@ -37,6 +39,22 @@ pub trait DepContext: Copy { fn dep_kind_vtable(&self, dep_node: DepKind) -> &DepKindVTable; + #[inline] + fn read_index(self, dep_node_index: DepNodeIndex) { + self.dep_graph().read_index::(dep_node_index); + } + + fn assert_ignored(self) { + self.dep_graph().assert_ignored::(); + } + + fn with_ignore(&self, op: OP) -> R + where + OP: FnOnce() -> R, + { + Self::Deps::with_deps(TaskDepsRef::Ignore, op) + } + #[inline(always)] fn fingerprint_style(self, kind: DepKind) -> FingerprintStyle { self.dep_kind_vtable(kind).fingerprint_style @@ -98,24 +116,6 @@ pub trait Deps: DynSync { fn read_deps(op: OP) where OP: for<'a> FnOnce(TaskDepsRef<'a>); - - fn name(dep_kind: DepKind) -> &'static str; - - /// We use this for most things when incr. comp. is turned off. - const DEP_KIND_NULL: DepKind; - - /// We use this to create a forever-red node. - const DEP_KIND_RED: DepKind; - - /// We use this to create a side effect node. - const DEP_KIND_SIDE_EFFECT: DepKind; - - /// We use this to create the anon node with zero dependencies. - const DEP_KIND_ANON_ZERO_DEPS: DepKind; - - /// This is the highest value a `DepKind` can have. It's used during encoding to - /// pack information into the unused bits. - const DEP_KIND_MAX: u16; } pub trait HasDepContext: Copy { diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index aa4c928d3cdc8..ee2ef375fef33 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -41,7 +41,6 @@ use std::cell::RefCell; use std::cmp::max; -use std::marker::PhantomData; use std::sync::Arc; use std::sync::atomic::Ordering; use std::{iter, mem, u64}; @@ -61,7 +60,8 @@ use tracing::{debug, instrument}; use super::graph::{CurrentDepGraph, DepNodeColorMap}; use super::query::DepGraphQuery; -use super::{DepKind, DepNode, DepNodeIndex, Deps}; +use super::{DepKind, DepNode, DepNodeIndex}; +use crate::dep_graph::dep_node::{DEP_KIND_NULL, DEP_KIND_SIDE_EFFECT}; use crate::dep_graph::edges::EdgesVec; // The maximum value of `SerializedDepNodeIndex` leaves the upper two bits @@ -192,7 +192,7 @@ fn mask(bits: usize) -> usize { impl SerializedDepGraph { #[instrument(level = "debug", skip(d))] - pub fn decode(d: &mut MemDecoder<'_>) -> Arc { + pub fn decode(d: &mut MemDecoder<'_>, dep_kind_names: &[&str]) -> Arc { // The last 16 bytes are the node count and edge count. debug!("position: {:?}", d.position()); @@ -213,7 +213,7 @@ impl SerializedDepGraph { let graph_bytes = d.len() - (3 * IntEncodedWithFixedSize::ENCODED_SIZE) - d.position(); let mut nodes = IndexVec::from_elem_n( - DepNode { kind: D::DEP_KIND_NULL, hash: PackedFingerprint::from(Fingerprint::ZERO) }, + DepNode { kind: DEP_KIND_NULL, hash: PackedFingerprint::from(Fingerprint::ZERO) }, node_max, ); let mut fingerprints = IndexVec::from_elem_n(Fingerprint::ZERO, node_max); @@ -230,19 +230,18 @@ impl SerializedDepGraph { // least (34 byte header + 1 byte len + 64 bytes edge data), which is ~1%. A 2-byte leb128 // length is about the same fractional overhead and it amortizes for yet greater lengths. let mut edge_list_data = - Vec::with_capacity(graph_bytes - node_count * size_of::>()); + Vec::with_capacity(graph_bytes - node_count * size_of::()); for _ in 0..node_count { // Decode the header for this edge; the header packs together as many of the fixed-size // fields as possible to limit the number of times we update decoder state. - let node_header = - SerializedNodeHeader:: { bytes: d.read_array(), _marker: PhantomData }; + let node_header = SerializedNodeHeader { bytes: d.read_array() }; let index = node_header.index(); let node = &mut nodes[index]; // Make sure there's no duplicate indices in the dep graph. - assert!(node_header.node().kind != D::DEP_KIND_NULL && node.kind == D::DEP_KIND_NULL); + assert!(node_header.node().kind != DEP_KIND_NULL && node.kind == DEP_KIND_NULL); *node = node_header.node(); fingerprints[index] = node_header.fingerprint(); @@ -270,7 +269,7 @@ impl SerializedDepGraph { edge_list_data.extend(&[0u8; DEP_NODE_PAD]); // Read the number of each dep kind and use it to create an hash map with a suitable size. - let mut index: Vec<_> = (0..(D::DEP_KIND_MAX + 1)) + let mut index: Vec<_> = (0..dep_kind_names.len()) .map(|_| UnhashMap::with_capacity_and_hasher(d.read_u32() as usize, Default::default())) .collect(); @@ -279,8 +278,8 @@ impl SerializedDepGraph { for (idx, node) in nodes.iter_enumerated() { if index[node.kind.as_usize()].insert(node.hash, idx).is_some() { // Empty nodes and side effect nodes can have duplicates - if node.kind != D::DEP_KIND_NULL && node.kind != D::DEP_KIND_SIDE_EFFECT { - let name = D::name(node.kind); + if node.kind != DEP_KIND_NULL && node.kind != DEP_KIND_SIDE_EFFECT { + let name = dep_kind_names[node.kind.as_usize()]; panic!( "Error: A dep graph node ({name}) does not have an unique index. \ Running a clean build on a nightly compiler with `-Z incremental-verify-ich` \ @@ -310,13 +309,12 @@ impl SerializedDepGraph { /// * The `DepKind`'s discriminant (a u16, but not all bits are used...) /// * The byte width of the encoded edges for this node /// * In whatever bits remain, the length of the edge list for this node, if it fits -struct SerializedNodeHeader { +struct SerializedNodeHeader { // 2 bytes for the DepNode // 4 bytes for the index // 16 for Fingerprint in DepNode // 16 for Fingerprint in NodeInfo bytes: [u8; 38], - _marker: PhantomData, } // The fields of a `SerializedNodeHeader`, this struct is an implementation detail and exists only @@ -330,6 +328,14 @@ struct Unpacked { fingerprint: Fingerprint, } +/// This the hardcoded value of `unused_dep_kind_bits(dep_kind_count)`. +/// It is checked for correctness in `rustc_middle::dep_graph`. +pub const DEP_KIND_UNUSED_BITS: usize = 7; + +pub const fn unused_dep_kind_bits(dep_kind_count: u16) -> usize { + (dep_kind_count - 1).leading_zeros() as usize +} + // Bit fields, where // M: bits used to store the length of a node's edge list // N: bits used to store the byte width of elements of the edge list @@ -337,11 +343,11 @@ struct Unpacked { // 0..M length of the edge // M..M+N bytes per index // M+N..16 kind -impl SerializedNodeHeader { +impl SerializedNodeHeader { const TOTAL_BITS: usize = size_of::() * 8; const LEN_BITS: usize = Self::TOTAL_BITS - Self::KIND_BITS - Self::WIDTH_BITS; const WIDTH_BITS: usize = DEP_NODE_WIDTH_BITS; - const KIND_BITS: usize = Self::TOTAL_BITS - D::DEP_KIND_MAX.leading_zeros() as usize; + const KIND_BITS: usize = Self::TOTAL_BITS - DEP_KIND_UNUSED_BITS; const MAX_INLINE_LEN: usize = (u16::MAX as usize >> (Self::TOTAL_BITS - Self::LEN_BITS)) - 1; #[inline] @@ -377,14 +383,14 @@ impl SerializedNodeHeader { #[cfg(debug_assertions)] { - let res = Self { bytes, _marker: PhantomData }; + let res = Self { bytes }; assert_eq!(fingerprint, res.fingerprint()); assert_eq!(*node, res.node()); if let Some(len) = res.len() { assert_eq!(edge_count, len as usize); } } - Self { bytes, _marker: PhantomData } + Self { bytes } } #[inline] @@ -451,15 +457,10 @@ struct NodeInfo { } impl NodeInfo { - fn encode(&self, e: &mut MemEncoder, index: DepNodeIndex) { + fn encode(&self, e: &mut MemEncoder, index: DepNodeIndex) { let NodeInfo { ref node, fingerprint, ref edges } = *self; - let header = SerializedNodeHeader::::new( - node, - index, - fingerprint, - edges.max_index(), - edges.len(), - ); + let header = + SerializedNodeHeader::new(node, index, fingerprint, edges.max_index(), edges.len()); e.write_array(header.bytes); if header.len().is_none() { @@ -480,7 +481,7 @@ impl NodeInfo { /// the previous dep graph and expects all edges to already have a new dep node index assigned. /// This avoids the overhead of constructing `EdgesVec`, which would be needed to call `encode`. #[inline] - fn encode_promoted( + fn encode_promoted( e: &mut MemEncoder, node: &DepNode, index: DepNodeIndex, @@ -496,7 +497,7 @@ impl NodeInfo { let edge_max = edges.clone().map(|i| colors.current(i).unwrap().as_u32()).max().unwrap_or(0); - let header = SerializedNodeHeader::::new(node, index, fingerprint, edge_max, edge_count); + let header = SerializedNodeHeader::new(node, index, fingerprint, edge_max, edge_count); e.write_array(header.bytes); if header.len().is_none() { @@ -543,17 +544,24 @@ struct LocalEncoderResult { kind_stats: Vec, } -struct EncoderState { +struct EncoderState { next_node_index: AtomicU64, previous: Arc, file: Lock>, local: WorkerLocal>, stats: Option>>, - marker: PhantomData, } -impl EncoderState { - fn new(encoder: FileEncoder, record_stats: bool, previous: Arc) -> Self { +impl EncoderState { + fn new( + encoder: FileEncoder, + record_stats: bool, + previous: Arc, + dep_kind_count: usize, + ) -> Self { + // Verify that the unused bit count is correct. + assert_eq!(unused_dep_kind_bits(dep_kind_count.try_into().unwrap()), DEP_KIND_UNUSED_BITS); + Self { previous, next_node_index: AtomicU64::new(0), @@ -566,10 +574,9 @@ impl EncoderState { edge_count: 0, node_count: 0, encoder: MemEncoder::new(), - kind_stats: iter::repeat_n(0, D::DEP_KIND_MAX as usize + 1).collect(), + kind_stats: iter::repeat_n(0, dep_kind_count).collect(), }) }), - marker: PhantomData, } } @@ -658,7 +665,7 @@ impl EncoderState { record_graph: &Option>, local: &mut LocalEncoderState, ) { - node.encode::(&mut local.encoder, index); + node.encode(&mut local.encoder, index); self.flush_mem_encoder(&mut *local); self.record( &node.node, @@ -687,7 +694,7 @@ impl EncoderState { ) { let node = self.previous.index_to_node(prev_index); let fingerprint = self.previous.fingerprint_by_index(prev_index); - let edge_count = NodeInfo::encode_promoted::( + let edge_count = NodeInfo::encode_promoted( &mut local.encoder, node, index, @@ -712,7 +719,7 @@ impl EncoderState { ); } - fn finish(&self, profiler: &SelfProfilerRef, current: &CurrentDepGraph) -> FileEncodeResult { + fn finish(&self, profiler: &SelfProfilerRef, current: &CurrentDepGraph) -> FileEncodeResult { // Prevent more indices from being allocated. self.next_node_index.store(u32::MAX as u64 + 1, Ordering::SeqCst); @@ -735,7 +742,8 @@ impl EncoderState { let mut encoder = self.file.lock().take().unwrap(); - let mut kind_stats: Vec = iter::repeat_n(0, D::DEP_KIND_MAX as usize + 1).collect(); + let dep_kind_count = results.first().unwrap().kind_stats.len(); + let mut kind_stats: Vec = iter::repeat_n(0, dep_kind_count).collect(); let mut node_max = 0; let mut node_count = 0; @@ -778,7 +786,7 @@ impl EncoderState { fn print_incremental_info( &self, - current: &CurrentDepGraph, + current: &CurrentDepGraph, total_node_count: usize, total_edge_count: usize, ) { @@ -835,25 +843,31 @@ impl EncoderState { } } -pub(crate) struct GraphEncoder { +pub(crate) struct GraphEncoder { profiler: SelfProfilerRef, - status: EncoderState, + status: EncoderState, record_graph: Option>, } -impl GraphEncoder { +impl GraphEncoder { pub(crate) fn new( sess: &Session, encoder: FileEncoder, prev_node_count: usize, previous: Arc, + dep_kind_count: usize, ) -> Self { let record_graph = sess .opts .unstable_opts .query_dep_graph .then(|| Lock::new(DepGraphQuery::new(prev_node_count))); - let status = EncoderState::new(encoder, sess.opts.unstable_opts.incremental_info, previous); + let status = EncoderState::new( + encoder, + sess.opts.unstable_opts.incremental_info, + previous, + dep_kind_count, + ); GraphEncoder { status, record_graph, profiler: sess.prof.clone() } } @@ -948,7 +962,7 @@ impl GraphEncoder { } } - pub(crate) fn finish(&self, current: &CurrentDepGraph) -> FileEncodeResult { + pub(crate) fn finish(&self, current: &CurrentDepGraph) -> FileEncodeResult { let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph_finish"); self.status.finish(&self.profiler, current) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index fcd2e80a4fdc2..51adb9f13a791 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -279,7 +279,7 @@ where match cache.lookup(key) { Some((value, index)) => { tcx.profiler().query_cache_hit(index.into()); - tcx.dep_graph().read_index(index); + tcx.read_index(index); Some(value) } None => None, @@ -538,7 +538,7 @@ where fn execute_job_incr<'tcx, Q>( query: Q, qcx: Q::Qcx, - dep_graph_data: &DepGraphData<::Deps>, + dep_graph_data: &DepGraphData, key: Q::Key, mut dep_node_opt: Option, job_id: QueryJobId, @@ -592,7 +592,7 @@ where #[inline(always)] fn try_load_from_disk_and_cache_in_memory<'tcx, Q>( query: Q, - dep_graph_data: &DepGraphData<::Deps>, + dep_graph_data: &DepGraphData, qcx: Q::Qcx, key: &Q::Key, dep_node: &DepNode, @@ -659,7 +659,7 @@ where let prof_timer = qcx.dep_context().profiler().query_provider(); // The dep-graph for this computation is already in-place. - let result = qcx.dep_context().dep_graph().with_ignore(|| query.compute(qcx, *key)); + let result = qcx.dep_context().with_ignore(|| query.compute(qcx, *key)); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -688,7 +688,7 @@ where #[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")] pub(crate) fn incremental_verify_ich( tcx: Tcx, - dep_graph_data: &DepGraphData, + dep_graph_data: &DepGraphData, result: &V, prev_index: SerializedDepNodeIndex, hash_result: Option, &V) -> Fingerprint>, @@ -794,23 +794,23 @@ where let dep_node = query.construct_dep_node(*qcx.dep_context(), key); - let dep_graph = qcx.dep_context().dep_graph(); - let serialized_dep_node_index = match dep_graph.try_mark_green(qcx, &dep_node) { - None => { - // A None return from `try_mark_green` means that this is either - // a new dep node or that the dep node has already been marked red. - // Either way, we can't call `dep_graph.read()` as we don't have the - // DepNodeIndex. We must invoke the query itself. The performance cost - // this introduces should be negligible as we'll immediately hit the - // in-memory cache, or another query down the line will. - return (true, Some(dep_node)); - } - Some((serialized_dep_node_index, dep_node_index)) => { - dep_graph.read_index(dep_node_index); - qcx.dep_context().profiler().query_cache_hit(dep_node_index.into()); - serialized_dep_node_index - } - }; + let serialized_dep_node_index = + match qcx.dep_context().dep_graph().try_mark_green(qcx, &dep_node) { + None => { + // A None return from `try_mark_green` means that this is either + // a new dep node or that the dep node has already been marked red. + // Either way, we can't call `dep_graph.read()` as we don't have the + // DepNodeIndex. We must invoke the query itself. The performance cost + // this introduces should be negligible as we'll immediately hit the + // in-memory cache, or another query down the line will. + return (true, Some(dep_node)); + } + Some((serialized_dep_node_index, dep_node_index)) => { + qcx.dep_context().read_index(dep_node_index); + qcx.dep_context().profiler().query_cache_hit(dep_node_index.into()); + serialized_dep_node_index + } + }; // We do not need the value at all, so do not check the cache. if !check_cache { @@ -863,7 +863,7 @@ where let (result, dep_node_index) = ensure_sufficient_stack(|| try_execute_query::(query, qcx, span, key, dep_node)); if let Some(dep_node_index) = dep_node_index { - qcx.dep_context().dep_graph().read_index(dep_node_index) + qcx.dep_context().read_index(dep_node_index) } Some(result) } diff --git a/compiler/rustc_symbol_mangling/src/test.rs b/compiler/rustc_symbol_mangling/src/test.rs index 50935e7caf33c..5a7d917118284 100644 --- a/compiler/rustc_symbol_mangling/src/test.rs +++ b/compiler/rustc_symbol_mangling/src/test.rs @@ -5,6 +5,7 @@ //! paths etc in all kinds of annoying scenarios. use rustc_hir::def_id::LocalDefId; +use rustc_middle::dep_graph::DepContext; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{GenericArgs, Instance, TyCtxt}; use rustc_span::{Symbol, sym}; @@ -22,7 +23,7 @@ pub fn report_symbol_names(tcx: TyCtxt<'_>) { return; } - tcx.dep_graph.with_ignore(|| { + tcx.with_ignore(|| { let mut symbol_names = SymbolNamesTest { tcx }; let crate_items = tcx.hir_crate_items(());