Rollup of 3 pull requests

compiler/rustc_data_structures/src/graph/linked_graph/mod.rs

@@ -23,6 +23,7 @@
 use std::fmt::Debug;
 
 use rustc_index::bit_set::DenseBitSet;
+use rustc_index::{Idx, IndexSlice, IndexVec};
 use tracing::debug;
 
 #[cfg(test)]
@@ -45,13 +46,13 @@ mod tests;
 /// and does not implement those traits, so it has its own implementations of a
 /// few basic graph algorithms.
 pub struct LinkedGraph<N, E> {
-    nodes: Vec<Node<N>>,
+    nodes: IndexVec<NodeIndex, Node<N>>,
     edges: Vec<Edge<E>>,
 }
 
 pub struct Node<N> {
     first_edge: [EdgeIndex; 2], // see module comment
-    pub data: N,
+    pub data: Option<N>,
 }
 
 #[derive(Debug)]
@@ -62,7 +63,7 @@ pub struct Edge<E> {
     pub data: E,
 }
 
-#[derive(Copy, Clone, PartialEq, Debug)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
 pub struct NodeIndex(pub usize);
 
 #[derive(Copy, Clone, PartialEq, Debug)]
@@ -87,19 +88,29 @@ impl NodeIndex {
     }
 }
 
+impl Idx for NodeIndex {
+    fn new(idx: usize) -> NodeIndex {
+        NodeIndex(idx)
+    }
+
+    fn index(self) -> usize {
+        self.0
+    }
+}
+
 impl<N: Debug, E: Debug> LinkedGraph<N, E> {
     pub fn new() -> Self {
-        Self { nodes: Vec::new(), edges: Vec::new() }
+        Self { nodes: IndexVec::new(), edges: Vec::new() }
     }
 
     pub fn with_capacity(nodes: usize, edges: usize) -> Self {
-        Self { nodes: Vec::with_capacity(nodes), edges: Vec::with_capacity(edges) }
+        Self { nodes: IndexVec::with_capacity(nodes), edges: Vec::with_capacity(edges) }
     }
 
     // # Simple accessors
 
     #[inline]
-    pub fn all_nodes(&self) -> &[Node<N>] {
+    pub fn all_nodes(&self) -> &IndexSlice<NodeIndex, Node<N>> {
         &self.nodes
     }
 
@@ -124,22 +135,34 @@ impl<N: Debug, E: Debug> LinkedGraph<N, E> {
         NodeIndex(self.nodes.len())
     }
 
+    fn ensure_node(&mut self, idx: NodeIndex) -> &mut Node<N> {
+        self.nodes.ensure_contains_elem(idx, || Node {
+            first_edge: [INVALID_EDGE_INDEX, INVALID_EDGE_INDEX],
+            data: None,
+        })
+    }
+
+    pub fn add_node_with_idx(&mut self, idx: NodeIndex, data: N) {
+        let old_data = self.ensure_node(idx).data.replace(data);
+        debug_assert!(old_data.is_none());
+    }
+
     pub fn add_node(&mut self, data: N) -> NodeIndex {
         let idx = self.next_node_index();
-        self.nodes.push(Node { first_edge: [INVALID_EDGE_INDEX, INVALID_EDGE_INDEX], data });
+        self.add_node_with_idx(idx, data);
         idx
     }
 
     pub fn mut_node_data(&mut self, idx: NodeIndex) -> &mut N {
-        &mut self.nodes[idx.0].data
+        self.nodes[idx].data.as_mut().unwrap()
     }
 
     pub fn node_data(&self, idx: NodeIndex) -> &N {
-        &self.nodes[idx.0].data
+        self.nodes[idx].data.as_ref().unwrap()
     }
 
     pub fn node(&self, idx: NodeIndex) -> &Node<N> {
-        &self.nodes[idx.0]
+        &self.nodes[idx]
     }
 
     // # Edge construction and queries
@@ -154,16 +177,16 @@ impl<N: Debug, E: Debug> LinkedGraph<N, E> {
         let idx = self.next_edge_index();
 
         // read current first of the list of edges from each node
-        let source_first = self.nodes[source.0].first_edge[OUTGOING.repr];
-        let target_first = self.nodes[target.0].first_edge[INCOMING.repr];
+        let source_first = self.ensure_node(source).first_edge[OUTGOING.repr];
+        let target_first = self.ensure_node(target).first_edge[INCOMING.repr];
 
         // create the new edge, with the previous firsts from each node
         // as the next pointers
         self.edges.push(Edge { next_edge: [source_first, target_first], source, target, data });
 
         // adjust the firsts for each node target be the next object.
-        self.nodes[source.0].first_edge[OUTGOING.repr] = idx;
-        self.nodes[target.0].first_edge[INCOMING.repr] = idx;
+        self.nodes[source].first_edge[OUTGOING.repr] = idx;
+        self.nodes[target].first_edge[INCOMING.repr] = idx;
 
         idx
     }

compiler/rustc_data_structures/src/graph/linked_graph/tests.rs

@@ -40,7 +40,7 @@ fn each_node() {
     let expected = ["A", "B", "C", "D", "E", "F"];
     graph.each_node(|idx, node| {
         assert_eq!(&expected[idx.0], graph.node_data(idx));
-        assert_eq!(expected[idx.0], node.data);
+        assert_eq!(expected[idx.0], node.data.unwrap());
         true
     });
 }

compiler/rustc_middle/src/dep_graph/retained.rs

@@ -1,6 +1,5 @@
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::graph::linked_graph::{Direction, INCOMING, LinkedGraph, NodeIndex};
-use rustc_index::IndexVec;
 
 use super::{DepNode, DepNodeIndex};
 
@@ -13,7 +12,6 @@ use super::{DepNode, DepNodeIndex};
 pub struct RetainedDepGraph {
     pub inner: LinkedGraph<DepNode, ()>,
     pub indices: FxHashMap<DepNode, NodeIndex>,
-    pub dep_index_to_index: IndexVec<DepNodeIndex, Option<NodeIndex>>,
 }
 
 impl RetainedDepGraph {
@@ -23,27 +21,22 @@ impl RetainedDepGraph {
 
         let inner = LinkedGraph::with_capacity(node_count, edge_count);
         let indices = FxHashMap::default();
-        let dep_index_to_index = IndexVec::new();
 
-        Self { inner, indices, dep_index_to_index }
+        Self { inner, indices }
     }
 
     pub fn push(&mut self, index: DepNodeIndex, node: DepNode, edges: &[DepNodeIndex]) {
-        let source = self.inner.add_node(node);
-        self.dep_index_to_index.insert(index, source);
+        let source = NodeIndex(index.as_usize());
+        self.inner.add_node_with_idx(source, node);
         self.indices.insert(node, source);
 
         for &target in edges.iter() {
-            // We may miss the edges that are pushed while the `DepGraphQuery` is being accessed.
-            // Skip them to issues.
-            if let Some(&Some(target)) = self.dep_index_to_index.get(target) {
-                self.inner.add_edge(source, target, ());
-            }
+            self.inner.add_edge(source, NodeIndex(target.as_usize()), ());
         }
     }
 
     pub fn nodes(&self) -> Vec<&DepNode> {
-        self.inner.all_nodes().iter().map(|n| &n.data).collect()
+        self.inner.all_nodes().iter().map(|n| n.data.as_ref().unwrap()).collect()
     }
 
     pub fn edges(&self) -> Vec<(&DepNode, &DepNode)> {

compiler/rustc_query_impl/src/execution.rs

@@ -18,9 +18,9 @@ use rustc_span::{DUMMY_SP, Span};
 use tracing::warn;
 
 use crate::dep_graph::{DepNode, DepNodeIndex};
-use crate::for_each_query_vtable;
 use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack, report_cycle};
 use crate::plumbing::{current_query_job, next_job_id, start_query};
+use crate::query_impl::for_each_query_vtable;
 
 #[inline]
 fn equivalent_key<K: Eq, V>(k: K) -> impl Fn(&(K, V)) -> bool {

compiler/rustc_query_impl/src/lib.rs

@@ -10,26 +10,24 @@
 
 use rustc_data_structures::sync::AtomicU64;
 use rustc_middle::dep_graph;
-use rustc_middle::queries::{self, ExternProviders, Providers, TaggedQueryKey};
+use rustc_middle::queries::{ExternProviders, Providers};
+use rustc_middle::query::QueryCache;
 use rustc_middle::query::on_disk_cache::OnDiskCache;
 use rustc_middle::query::plumbing::{QuerySystem, QueryVTable};
-use rustc_middle::query::{AsLocalQueryKey, QueryCache, QueryMode};
 use rustc_middle::ty::TyCtxt;
-use rustc_span::Span;
 
 pub use crate::dep_kind_vtables::make_dep_kind_vtables;
 pub use crate::execution::{CollectActiveJobsKind, collect_active_jobs_from_all_queries};
 pub use crate::job::{QueryJobMap, break_query_cycles, print_query_stack};
 
-#[macro_use]
-mod plumbing;
-
 mod dep_kind_vtables;
 mod error;
 mod execution;
 mod from_cycle_error;
 mod job;
+mod plumbing;
 mod profiling_support;
+mod query_impl;
 
 /// Trait that knows how to look up the [`QueryVTable`] for a particular query.
 ///
@@ -51,7 +49,7 @@ pub fn query_system<'tcx>(
     on_disk_cache: Option<OnDiskCache>,
     incremental: bool,
 ) -> QuerySystem<'tcx> {
-    let mut query_vtables = make_query_vtables(incremental);
+    let mut query_vtables = query_impl::make_query_vtables(incremental);
     from_cycle_error::specialize_query_vtables(&mut query_vtables);
     QuerySystem {
         arenas: Default::default(),
@@ -63,8 +61,6 @@ pub fn query_system<'tcx>(
     }
 }
 
-rustc_middle::rustc_with_all_queries! { define_queries! }
-
 pub fn provide(providers: &mut rustc_middle::util::Providers) {
     providers.hooks.alloc_self_profile_query_strings =
         profiling_support::alloc_self_profile_query_strings;