From c54a2a53f83d9eb4fec161ecc304164d0303fbfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 27 Jun 2024 19:03:47 +0200 Subject: [PATCH 01/26] Make mtime of reproducible tarball dependent on git commit --- src/bootstrap/src/utils/tarball.rs | 26 ++++++++++++++++++++++- src/tools/rust-installer/src/tarballer.rs | 18 ++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/utils/tarball.rs b/src/bootstrap/src/utils/tarball.rs index 5cc319826dbf3..3c15bb296f39a 100644 --- a/src/bootstrap/src/utils/tarball.rs +++ b/src/bootstrap/src/utils/tarball.rs @@ -9,9 +9,9 @@ use std::path::{Path, PathBuf}; use crate::core::builder::Builder; use crate::core::{build_steps::dist::distdir, builder::Kind}; -use crate::utils::channel; use crate::utils::exec::BootstrapCommand; use crate::utils::helpers::{move_file, t}; +use crate::utils::{channel, helpers}; #[derive(Copy, Clone)] pub(crate) enum OverlayKind { @@ -351,6 +351,30 @@ impl<'a> Tarball<'a> { }; cmd.args(["--compression-profile", compression_profile]); + + // We want to use a pinned modification time for files in the archive + // to achieve better reproducibility. However, using the same mtime for all + // releases is not ideal, because it can break e.g. Cargo mtime checking + // (https://github.com/rust-lang/rust/issues/125578). + // Therefore, we set mtime to the date of the latest commit (if we're managed + // by git). In this way, the archive will still be always the same for a given commit + // (achieving reproducibility), but it will also change between different commits and + // Rust versions, so that it won't break mtime-based caches. + // + // Note that this only overrides the mtime of files, not directories, due to the + // limitations of the tarballer tool. Directories will have their mtime set to 2006. + + // Get the UTC timestamp of the last git commit, if we're under git. + // We need to use UTC, so that anyone who tries to rebuild from the same commit + // gets the same timestamp. + if self.builder.rust_info().is_managed_git_subrepository() { + // %ct means committer date + let timestamp = helpers::output( + helpers::git(Some(&self.builder.src)).arg("log").arg("-1").arg("--format=%ct"), + ); + cmd.args(["--override-file-mtime", timestamp.trim()]); + } + self.builder.run(cmd); // Ensure there are no symbolic links in the tarball. In particular, diff --git a/src/tools/rust-installer/src/tarballer.rs b/src/tools/rust-installer/src/tarballer.rs index 2f093e7ad17fd..b5e87a66ffc39 100644 --- a/src/tools/rust-installer/src/tarballer.rs +++ b/src/tools/rust-installer/src/tarballer.rs @@ -32,6 +32,12 @@ actor! { /// The formats used to compress the tarball. #[arg(value_name = "FORMAT", default_value_t)] compression_formats: CompressionFormats, + + /// Modification time that will be set for all files added to the archive. + /// The default is the date of the first Rust commit from 2006. + /// This serves for better reproducibility of the archives. + #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)] + override_file_mtime: u64, } } @@ -65,6 +71,8 @@ impl Tarballer { let buf = BufWriter::with_capacity(1024 * 1024, encoder); let mut builder = Builder::new(buf); // Make uid, gid and mtime deterministic to improve reproducibility + // The modification time of directories will be set to the date of the first Rust commit. + // The modification time of files will be set to `override_file_mtime` (see `append_path`). builder.mode(HeaderMode::Deterministic); let pool = rayon::ThreadPoolBuilder::new().num_threads(2).build().unwrap(); @@ -77,7 +85,7 @@ impl Tarballer { } for path in files { let src = Path::new(&self.work_dir).join(&path); - append_path(&mut builder, &src, &path) + append_path(&mut builder, &src, &path, self.override_file_mtime) .with_context(|| format!("failed to tar file '{}'", src.display()))?; } builder @@ -93,10 +101,16 @@ impl Tarballer { } } -fn append_path(builder: &mut Builder, src: &Path, path: &String) -> Result<()> { +fn append_path( + builder: &mut Builder, + src: &Path, + path: &String, + override_file_mtime: u64, +) -> Result<()> { let stat = symlink_metadata(src)?; let mut header = Header::new_gnu(); header.set_metadata_in_mode(&stat, HeaderMode::Deterministic); + header.set_mtime(override_file_mtime); if stat.file_type().is_symlink() { let link = read_link(src)?; From e1999f9a0815c891d71588045b96a76e8e2c45a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sat, 29 Jun 2024 23:30:57 +0200 Subject: [PATCH 02/26] Add support for mtime override to `generate` and `combine` rust-installer commands --- src/tools/rust-installer/src/combiner.rs | 9 ++++++++- src/tools/rust-installer/src/generator.rs | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/tools/rust-installer/src/combiner.rs b/src/tools/rust-installer/src/combiner.rs index 565d19d863f64..c211b34850a07 100644 --- a/src/tools/rust-installer/src/combiner.rs +++ b/src/tools/rust-installer/src/combiner.rs @@ -55,6 +55,12 @@ actor! { /// The formats used to compress the tarball #[arg(value_name = "FORMAT", default_value_t)] compression_formats: CompressionFormats, + + /// Modification time that will be set for all files added to the archive. + /// The default is the date of the first Rust commit from 2006. + /// This serves for better reproducibility of the archives. + #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)] + override_file_mtime: u64, } } @@ -145,7 +151,8 @@ impl Combiner { .input(self.package_name) .output(path_to_str(&output)?.into()) .compression_profile(self.compression_profile) - .compression_formats(self.compression_formats); + .compression_formats(self.compression_formats) + .override_file_mtime(self.override_file_mtime); tarballer.run()?; Ok(()) diff --git a/src/tools/rust-installer/src/generator.rs b/src/tools/rust-installer/src/generator.rs index b101e67d8df7f..035a394deebe6 100644 --- a/src/tools/rust-installer/src/generator.rs +++ b/src/tools/rust-installer/src/generator.rs @@ -61,6 +61,12 @@ actor! { /// The formats used to compress the tarball #[arg(value_name = "FORMAT", default_value_t)] compression_formats: CompressionFormats, + + /// Modification time that will be set for all files added to the archive. + /// The default is the date of the first Rust commit from 2006. + /// This serves for better reproducibility of the archives. + #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)] + override_file_mtime: u64, } } @@ -114,7 +120,8 @@ impl Generator { .input(self.package_name) .output(path_to_str(&output)?.into()) .compression_profile(self.compression_profile) - .compression_formats(self.compression_formats); + .compression_formats(self.compression_formats) + .override_file_mtime(self.override_file_mtime); tarballer.run()?; Ok(()) From 61963fabdf2eaf9a68be970dfad7eb470a0fc0d5 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Mon, 1 Jul 2024 10:14:42 +0000 Subject: [PATCH 03/26] Avoid an ICE reachable through const eval shenanigans --- .../rustc_hir_typeck/src/method/confirm.rs | 9 ++-- .../arbitrary-self-from-method-substs-ice.rs | 29 ++++++++++++ ...bitrary-self-from-method-substs-ice.stderr | 46 +++++++++++++++++++ ...ary-self-from-method-substs.default.stderr | 1 - 4 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 tests/ui/self/arbitrary-self-from-method-substs-ice.rs create mode 100644 tests/ui/self/arbitrary-self-from-method-substs-ice.stderr diff --git a/compiler/rustc_hir_typeck/src/method/confirm.rs b/compiler/rustc_hir_typeck/src/method/confirm.rs index 120e3239d1f1b..02844bcadac6e 100644 --- a/compiler/rustc_hir_typeck/src/method/confirm.rs +++ b/compiler/rustc_hir_typeck/src/method/confirm.rs @@ -510,9 +510,12 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> { .report_mismatched_types(&cause, method_self_ty, self_ty, terr) .emit(); } else { - error!("{self_ty} was a subtype of {method_self_ty} but now is not?"); - // This must already have errored elsewhere. - self.dcx().has_errors().unwrap(); + // This has/will have errored in wfcheck, which we cannot depend on from here, as typeck on functions + // may run before wfcheck if the function is used in const eval. + self.dcx().span_delayed_bug( + cause.span(), + format!("{self_ty} was a subtype of {method_self_ty} but now is not?"), + ); } } } diff --git a/tests/ui/self/arbitrary-self-from-method-substs-ice.rs b/tests/ui/self/arbitrary-self-from-method-substs-ice.rs new file mode 100644 index 0000000000000..8bf9f97e0b91d --- /dev/null +++ b/tests/ui/self/arbitrary-self-from-method-substs-ice.rs @@ -0,0 +1,29 @@ +//! The same as the non-ICE test, but const eval will run typeck of +//! `get` before running wfcheck (as that may in itself trigger const +//! eval again, and thus cause bogus cycles). This used to ICE because +//! we asserted that an error had already been emitted. + +use std::ops::Deref; + +struct Foo(u32); +impl Foo { + const fn get>(self: R) -> u32 { + //~^ ERROR: `R` cannot be used as the type of `self` + //~| ERROR destructor of `R` cannot be evaluated at compile-time + self.0 + //~^ ERROR cannot borrow here, since the borrowed element may contain interior mutability + //~| ERROR cannot call non-const fn `::deref` in constant function + } +} + +const FOO: () = { + let foo = Foo(1); + foo.get::<&Foo>(); +}; + +const BAR: [(); { + FOO; + 0 +}] = []; + +fn main() {} diff --git a/tests/ui/self/arbitrary-self-from-method-substs-ice.stderr b/tests/ui/self/arbitrary-self-from-method-substs-ice.stderr new file mode 100644 index 0000000000000..9e3851f9a6e72 --- /dev/null +++ b/tests/ui/self/arbitrary-self-from-method-substs-ice.stderr @@ -0,0 +1,46 @@ +error[E0658]: cannot borrow here, since the borrowed element may contain interior mutability + --> $DIR/arbitrary-self-from-method-substs-ice.rs:13:9 + | +LL | self.0 + | ^^^^ + | + = note: see issue #80384 for more information + = help: add `#![feature(const_refs_to_cell)]` to the crate attributes to enable + = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date + +error[E0015]: cannot call non-const fn `::deref` in constant functions + --> $DIR/arbitrary-self-from-method-substs-ice.rs:13:9 + | +LL | self.0 + | ^^^^^^ + | + = note: calls in constant functions are limited to constant functions, tuple structs and tuple variants +help: add `#![feature(const_trait_impl)]` to the crate attributes to enable + | +LL + #![feature(const_trait_impl)] + | + +error[E0493]: destructor of `R` cannot be evaluated at compile-time + --> $DIR/arbitrary-self-from-method-substs-ice.rs:10:43 + | +LL | const fn get>(self: R) -> u32 { + | ^^^^ the destructor for this type cannot be evaluated in constant functions +... +LL | } + | - value is dropped here + +error[E0658]: `R` cannot be used as the type of `self` without the `arbitrary_self_types` feature + --> $DIR/arbitrary-self-from-method-substs-ice.rs:10:49 + | +LL | const fn get>(self: R) -> u32 { + | ^ + | + = note: see issue #44874 for more information + = help: add `#![feature(arbitrary_self_types)]` to the crate attributes to enable + = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date + = help: consider changing to `self`, `&self`, `&mut self`, `self: Box`, `self: Rc`, `self: Arc`, or `self: Pin

` (where P is one of the previous types except `Self`) + +error: aborting due to 4 previous errors + +Some errors have detailed explanations: E0015, E0493, E0658. +For more information about an error, try `rustc --explain E0015`. diff --git a/tests/ui/self/arbitrary-self-from-method-substs.default.stderr b/tests/ui/self/arbitrary-self-from-method-substs.default.stderr index 6fff086a89c2e..4cc69666b8876 100644 --- a/tests/ui/self/arbitrary-self-from-method-substs.default.stderr +++ b/tests/ui/self/arbitrary-self-from-method-substs.default.stderr @@ -9,7 +9,6 @@ LL | fn get>(self: R) -> u32 { = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date = help: consider changing to `self`, `&self`, `&mut self`, `self: Box`, `self: Rc`, `self: Arc`, or `self: Pin

` (where P is one of the previous types except `Self`) - ERROR rustc_hir_typeck::method::confirm Foo was a subtype of &Foo but now is not? error: aborting due to 1 previous error For more information about this error, try `rustc --explain E0658`. From 9d5da39b3a4d3155dca12db9eb7ea6be6234e8a6 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Mon, 1 Jul 2024 10:26:27 +0000 Subject: [PATCH 04/26] Revert some ICE avoiding logic that is not necessary anymore --- compiler/rustc_hir_typeck/src/writeback.rs | 25 +++------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/compiler/rustc_hir_typeck/src/writeback.rs b/compiler/rustc_hir_typeck/src/writeback.rs index aea0114167eb9..ceb7480686d90 100644 --- a/compiler/rustc_hir_typeck/src/writeback.rs +++ b/compiler/rustc_hir_typeck/src/writeback.rs @@ -219,28 +219,9 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> { fn fix_index_builtin_expr(&mut self, e: &hir::Expr<'_>) { if let hir::ExprKind::Index(ref base, ref index, _) = e.kind { // All valid indexing looks like this; might encounter non-valid indexes at this point. - let base_ty = self.typeck_results.expr_ty_adjusted_opt(base); - if base_ty.is_none() { - // When encountering `return [0][0]` outside of a `fn` body we can encounter a base - // that isn't in the type table. We assume more relevant errors have already been - // emitted. (#64638) - assert!(self.tcx().dcx().has_errors().is_some(), "bad base: `{base:?}`"); - } - if let Some(base_ty) = base_ty - && let ty::Ref(_, base_ty_inner, _) = *base_ty.kind() - { - let index_ty = - self.typeck_results.expr_ty_adjusted_opt(index).unwrap_or_else(|| { - // When encountering `return [0][0]` outside of a `fn` body we would attempt - // to access an nonexistent index. We assume that more relevant errors will - // already have been emitted, so we only gate on this with an ICE if no - // error has been emitted. (#64638) - Ty::new_error_with_message( - self.fcx.tcx, - e.span, - format!("bad index {index:?} for base: `{base:?}`"), - ) - }); + let base_ty = self.typeck_results.expr_ty_adjusted(base); + if let ty::Ref(_, base_ty_inner, _) = *base_ty.kind() { + let index_ty = self.typeck_results.expr_ty_adjusted(index); if self.is_builtin_index(e, base_ty_inner, index_ty) { // Remove the method call record self.typeck_results.type_dependent_defs_mut().remove(e.hir_id); From a5d37238a1171ddcf0bbf37f5900bcb442678ffd Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Mon, 1 Jul 2024 10:31:41 +0000 Subject: [PATCH 05/26] Prefer item-local tainting checks over global error count checks --- compiler/rustc_hir_typeck/src/expr.rs | 2 +- compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs index 2fbc8a3f146a1..f3aece4e1d8aa 100644 --- a/compiler/rustc_hir_typeck/src/expr.rs +++ b/compiler/rustc_hir_typeck/src/expr.rs @@ -708,7 +708,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // else an error would have been flagged by the // `loops` pass for using break with an expression // where you are not supposed to. - assert!(expr_opt.is_none() || self.dcx().has_errors().is_some()); + assert!(expr_opt.is_none() || self.tainted_by_errors().is_some()); } // If we encountered a `break`, then (no surprise) it may be possible to break from the diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs index 430e12ff7b87b..ba7719bbb4e6d 100644 --- a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs +++ b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs @@ -1652,7 +1652,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { self.warn_if_unreachable(stmt.hir_id, stmt.span, "statement"); - // Hide the outer diverging and `has_errors` flags. + // Hide the outer diverging flags. let old_diverges = self.diverges.replace(Diverges::Maybe); match stmt.kind { From 814bfe9335fd7c941169e0ef15ae2cffeacc78eb Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Mon, 1 Jul 2024 10:35:16 +0000 Subject: [PATCH 06/26] This check should now be unreachable --- compiler/rustc_hir_typeck/src/expr_use_visitor.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs index 9a353bfe49d0e..c8ab0429ffc70 100644 --- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs +++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs @@ -734,9 +734,7 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx // struct; however, when EUV is run during typeck, it // may not. This will generate an error earlier in typeck, // so we can just ignore it. - if self.cx.tcx().dcx().has_errors().is_none() { - span_bug!(with_expr.span, "with expression doesn't evaluate to a struct"); - } + span_bug!(with_expr.span, "with expression doesn't evaluate to a struct"); } } From 1680b791d57013ca2cf5839a279e3d4d826c336c Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 27 Jun 2024 10:33:46 +1000 Subject: [PATCH 07/26] Simplify `CfgEval`. It can contain an owned value instead of a reference. --- compiler/rustc_builtin_macros/src/cfg_eval.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_builtin_macros/src/cfg_eval.rs b/compiler/rustc_builtin_macros/src/cfg_eval.rs index 03aff6f96330e..884cebf1939e4 100644 --- a/compiler/rustc_builtin_macros/src/cfg_eval.rs +++ b/compiler/rustc_builtin_macros/src/cfg_eval.rs @@ -38,16 +38,14 @@ pub(crate) fn cfg_eval( lint_node_id: NodeId, ) -> Annotatable { let features = Some(features); - CfgEval { cfg: &mut StripUnconfigured { sess, features, config_tokens: true, lint_node_id } } + CfgEval(StripUnconfigured { sess, features, config_tokens: true, lint_node_id }) .configure_annotatable(annotatable) // Since the item itself has already been configured by the `InvocationCollector`, // we know that fold result vector will contain exactly one element. .unwrap() } -struct CfgEval<'a, 'b> { - cfg: &'a mut StripUnconfigured<'b>, -} +struct CfgEval<'a>(StripUnconfigured<'a>); fn flat_map_annotatable( vis: &mut impl MutVisitor, @@ -125,9 +123,9 @@ fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool { res.is_break() } -impl CfgEval<'_, '_> { +impl CfgEval<'_> { fn configure(&mut self, node: T) -> Option { - self.cfg.configure(node) + self.0.configure(node) } fn configure_annotatable(&mut self, mut annotatable: Annotatable) -> Option { @@ -196,7 +194,7 @@ impl CfgEval<'_, '_> { // Re-parse the tokens, setting the `capture_cfg` flag to save extra information // to the captured `AttrTokenStream` (specifically, we capture // `AttrTokenTree::AttributesData` for all occurrences of `#[cfg]` and `#[cfg_attr]`) - let mut parser = Parser::new(&self.cfg.sess.psess, orig_tokens, None); + let mut parser = Parser::new(&self.0.sess.psess, orig_tokens, None); parser.capture_cfg = true; match parse_annotatable_with(&mut parser) { Ok(a) => annotatable = a, @@ -212,16 +210,16 @@ impl CfgEval<'_, '_> { } } -impl MutVisitor for CfgEval<'_, '_> { +impl MutVisitor for CfgEval<'_> { #[instrument(level = "trace", skip(self))] fn visit_expr(&mut self, expr: &mut P) { - self.cfg.configure_expr(expr, false); + self.0.configure_expr(expr, false); mut_visit::noop_visit_expr(expr, self); } #[instrument(level = "trace", skip(self))] fn visit_method_receiver_expr(&mut self, expr: &mut P) { - self.cfg.configure_expr(expr, true); + self.0.configure_expr(expr, true); mut_visit::noop_visit_expr(expr, self); } From d6c0b8117e4ccccd83c4a6e70eee8b12c51d1a18 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 27 Jun 2024 10:34:05 +1000 Subject: [PATCH 08/26] Fix a typo in a comment. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index b5480b6b7d210..4b1531c5ff3e4 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -340,7 +340,7 @@ impl<'a> Parser<'a> { // If we support tokens at all if let Some(target_tokens) = ret.tokens_mut() { if target_tokens.is_none() { - // Store se our newly captured tokens into the AST node + // Store our newly captured tokens into the AST node. *target_tokens = Some(tokens.clone()); } } From f852568fa601171f20f924a50478c33fd2661fba Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 27 Jun 2024 10:42:46 +1000 Subject: [PATCH 09/26] Change `AttrTokenStream::to_tokenstream` to `to_token_trees`. I.e. change the return type from `TokenStream` to `Vec`. Most of the callsites require a `TokenStream`, but the recursive call used to create `target_tokens` requires a `Vec`. It's easy to convert a `Vec` to a `TokenStream` (just call `TokenStream::new`) but it's harder to convert a `TokenStream` to a `Vec` (either iterate/clone/collect, or use `Lrc::into_inner` if appropriate). So this commit changes the return value to simplify that `target_tokens` call site. --- compiler/rustc_ast/src/attr/mod.rs | 14 ++++++++------ compiler/rustc_ast/src/tokenstream.rs | 23 +++++++---------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 593c78df3cdb8..65f1b5dbaf5b7 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -204,12 +204,14 @@ impl Attribute { pub fn tokens(&self) -> TokenStream { match &self.kind { - AttrKind::Normal(normal) => normal - .tokens - .as_ref() - .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}")) - .to_attr_token_stream() - .to_tokenstream(), + AttrKind::Normal(normal) => TokenStream::new( + normal + .tokens + .as_ref() + .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}")) + .to_attr_token_stream() + .to_token_trees(), + ), &AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone( token::DocComment(comment_kind, self.style, data), self.span, diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index b4ddbe20689e2..5b2d673316a52 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -180,14 +180,13 @@ impl AttrTokenStream { AttrTokenStream(Lrc::new(tokens)) } - /// Converts this `AttrTokenStream` to a plain `TokenStream`. + /// Converts this `AttrTokenStream` to a plain `Vec`. /// During conversion, `AttrTokenTree::Attributes` get 'flattened' /// back to a `TokenStream` of the form `outer_attr attr_target`. /// If there are inner attributes, they are inserted into the proper /// place in the attribute target tokens. - pub fn to_tokenstream(&self) -> TokenStream { - let trees: Vec<_> = self - .0 + pub fn to_token_trees(&self) -> Vec { + self.0 .iter() .flat_map(|tree| match &tree { AttrTokenTree::Token(inner, spacing) => { @@ -198,7 +197,7 @@ impl AttrTokenStream { *span, *spacing, *delim, - stream.to_tokenstream() + TokenStream::new(stream.to_token_trees()) ),] .into_iter() } @@ -208,14 +207,7 @@ impl AttrTokenStream { .partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer)); let (outer_attrs, inner_attrs) = data.attrs.split_at(idx); - let mut target_tokens: Vec<_> = data - .tokens - .to_attr_token_stream() - .to_tokenstream() - .0 - .iter() - .cloned() - .collect(); + let mut target_tokens = data.tokens.to_attr_token_stream().to_token_trees(); if !inner_attrs.is_empty() { let mut found = false; // Check the last two trees (to account for a trailing semi) @@ -260,8 +252,7 @@ impl AttrTokenStream { flat.into_iter() } }) - .collect(); - TokenStream::new(trees) + .collect() } } @@ -461,7 +452,7 @@ impl TokenStream { AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() }; AttrTokenStream::new(vec![AttrTokenTree::Attributes(attr_data)]) }; - attr_stream.to_tokenstream() + TokenStream::new(attr_stream.to_token_trees()) } pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { From 0cfd2473be4a5555018cfbca089a7ed1bfd16c53 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 27 Jun 2024 10:49:12 +1000 Subject: [PATCH 10/26] Rename `TokenStream::new` argument. `tts` is a better name than `streams` for a `Vec`. --- compiler/rustc_ast/src/tokenstream.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 5b2d673316a52..cabf18fd1530a 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -400,8 +400,8 @@ impl PartialEq for TokenStream { } impl TokenStream { - pub fn new(streams: Vec) -> TokenStream { - TokenStream(Lrc::new(streams)) + pub fn new(tts: Vec) -> TokenStream { + TokenStream(Lrc::new(tts)) } pub fn is_empty(&self) -> bool { From 7416c20cfd781de5608494d67975be94e8689c24 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 27 Jun 2024 11:15:38 +1000 Subject: [PATCH 11/26] Just `push` in `AttrTokenStream::to_token_trees`. Currently it uses a mixture of functional style (`flat_map`) and imperative style (`push`), which is a bit hard to read. This commit converts it to fully imperative, which is more concise and avoids the need for `smallvec`. --- compiler/rustc_ast/src/tokenstream.rs | 28 ++++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index cabf18fd1530a..655c18e45597a 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -23,7 +23,6 @@ use rustc_data_structures::sync::{self, Lrc}; use rustc_macros::{Decodable, Encodable, HashStable_Generic}; use rustc_serialize::{Decodable, Encodable}; use rustc_span::{sym, Span, SpanDecoder, SpanEncoder, Symbol, DUMMY_SP}; -use smallvec::{smallvec, SmallVec}; use std::borrow::Cow; use std::{cmp, fmt, iter}; @@ -186,20 +185,19 @@ impl AttrTokenStream { /// If there are inner attributes, they are inserted into the proper /// place in the attribute target tokens. pub fn to_token_trees(&self) -> Vec { - self.0 - .iter() - .flat_map(|tree| match &tree { + let mut res = Vec::with_capacity(self.0.len()); + for tree in self.0.iter() { + match tree { AttrTokenTree::Token(inner, spacing) => { - smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter() + res.push(TokenTree::Token(inner.clone(), *spacing)); } AttrTokenTree::Delimited(span, spacing, delim, stream) => { - smallvec![TokenTree::Delimited( + res.push(TokenTree::Delimited( *span, *spacing, *delim, - TokenStream::new(stream.to_token_trees()) - ),] - .into_iter() + TokenStream::new(stream.to_token_trees()), + )) } AttrTokenTree::Attributes(data) => { let idx = data @@ -243,16 +241,14 @@ impl AttrTokenStream { "Failed to find trailing delimited group in: {target_tokens:?}" ); } - let mut flat: SmallVec<[_; 1]> = - SmallVec::with_capacity(target_tokens.len() + outer_attrs.len()); for attr in outer_attrs { - flat.extend(attr.tokens().0.iter().cloned()); + res.extend(attr.tokens().0.iter().cloned()); } - flat.extend(target_tokens); - flat.into_iter() + res.extend(target_tokens); } - }) - .collect() + } + } + res } } From 36c30a968b60a2926a0f600feb78eaba618d8d9a Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 1 Jul 2024 14:32:47 +1000 Subject: [PATCH 12/26] Fix comment. Both the indenting, and the missing `)`. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 4b1531c5ff3e4..a5aaa9f052208 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -233,9 +233,9 @@ impl<'a> Parser<'a> { // 1. Our target doesn't support tokens at all (e.g we're parsing an `NtIdent`) // so there's nothing for us to do. // 2. Our target already has tokens set (e.g. we've parsed something - // like `#[my_attr] $item`. The actual parsing code takes care of prepending - // any attributes to the nonterminal, so we don't need to modify the - // already captured tokens. + // like `#[my_attr] $item`). The actual parsing code takes care of + // prepending any attributes to the nonterminal, so we don't need to + // modify the already captured tokens. // Note that this check is independent of `force_collect`- if we already // have tokens, or can't even store them, then there's never a need to // force collection of new tokens. From 2342770f49dcc7e98069efb2db3620d4803c7698 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 1 Jul 2024 16:50:06 +1000 Subject: [PATCH 13/26] Flip an if/else in `AttrTokenStream::to_attr_token_stream`. To put the simple case first. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index a5aaa9f052208..244a4f26fba85 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -112,7 +112,9 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { })) .take(self.num_calls); - if !self.replace_ranges.is_empty() { + if self.replace_ranges.is_empty() { + make_token_stream(tokens, self.break_last_token) + } else { let mut tokens: Vec<_> = tokens.collect(); let mut replace_ranges = self.replace_ranges.to_vec(); replace_ranges.sort_by_key(|(range, _)| range.start); @@ -165,8 +167,6 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { ); } make_token_stream(tokens.into_iter(), self.break_last_token) - } else { - make_token_stream(tokens, self.break_last_token) } } } From 8b5a7eb7f4840dcc27e1dcf4a0e102a417ad988f Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 1 Jul 2024 16:12:22 +1000 Subject: [PATCH 14/26] Move things around in `collect_tokens_trailing_token`. So that the `capturing` state is adjusted immediately before and after the call to `f`. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 244a4f26fba85..7c22cac350b02 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -218,16 +218,16 @@ impl<'a> Parser<'a> { let start_token = (self.token.clone(), self.token_spacing); let cursor_snapshot = self.token_cursor.clone(); let start_pos = self.num_bump_calls; - let has_outer_attrs = !attrs.attrs.is_empty(); - let prev_capturing = std::mem::replace(&mut self.capture_state.capturing, Capturing::Yes); let replace_ranges_start = self.capture_state.replace_ranges.len(); - let ret = f(self, attrs.attrs); - - self.capture_state.capturing = prev_capturing; - - let (mut ret, trailing) = ret?; + let (mut ret, trailing) = { + let prev_capturing = + std::mem::replace(&mut self.capture_state.capturing, Capturing::Yes); + let ret_and_trailing = f(self, attrs.attrs); + self.capture_state.capturing = prev_capturing; + ret_and_trailing? + }; // When we're not in `capture-cfg` mode, then bail out early if: // 1. Our target doesn't support tokens at all (e.g we're parsing an `NtIdent`) From f5b28968db07c96d1e2fe239d380fe5a418e85c5 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 1 Jul 2024 16:21:47 +1000 Subject: [PATCH 15/26] Move more things around in `collect_tokens_trailing_token`. To make things a little clearer, and to avoid some `mut` variables. --- .../rustc_parse/src/parser/attr_wrapper.rs | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 7c22cac350b02..10feccd21e286 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -276,37 +276,32 @@ impl<'a> Parser<'a> { let replace_ranges_end = self.capture_state.replace_ranges.len(); - let mut end_pos = self.num_bump_calls; - - let mut captured_trailing = false; - // Capture a trailing token if requested by the callback 'f' - match trailing { - TrailingToken::None => {} + let captured_trailing = match trailing { + TrailingToken::None => false, TrailingToken::Gt => { assert_eq!(self.token.kind, token::Gt); + false } TrailingToken::Semi => { assert_eq!(self.token.kind, token::Semi); - end_pos += 1; - captured_trailing = true; + true } - TrailingToken::MaybeComma => { - if self.token.kind == token::Comma { - end_pos += 1; - captured_trailing = true; - } - } - } + TrailingToken::MaybeComma => self.token.kind == token::Comma, + }; - // If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens), - // then extend the range of captured tokens to include it, since the parser - // was not actually bumped past it. When the `LazyAttrTokenStream` gets converted - // into an `AttrTokenStream`, we will create the proper token. - if self.break_last_token { - assert!(!captured_trailing, "Cannot set break_last_token and have trailing token"); - end_pos += 1; - } + assert!( + !(self.break_last_token && captured_trailing), + "Cannot set break_last_token and have trailing token" + ); + + let end_pos = self.num_bump_calls + + captured_trailing as usize + // If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens), then + // extend the range of captured tokens to include it, since the parser was not actually + // bumped past it. When the `LazyAttrTokenStream` gets converted into an + // `AttrTokenStream`, we will create the proper token. + + self.break_last_token as usize; let num_calls = end_pos - start_pos; From 3d750e270279d2d8689aee3ee5cdbf60faec42bd Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 2 Jul 2024 16:31:24 +1000 Subject: [PATCH 16/26] Shrink parser positions from `usize` to `u32`. The number of source code bytes can't exceed a `u32`'s range, so a token position also can't. This reduces the size of `Parser` and `LazyAttrTokenStreamImpl` by eight bytes each. --- compiler/rustc_expand/src/mbe/diagnostics.rs | 12 +++++------ compiler/rustc_expand/src/mbe/macro_parser.rs | 2 +- compiler/rustc_expand/src/mbe/macro_rules.rs | 4 ++-- .../rustc_parse/src/parser/attr_wrapper.rs | 21 ++++++++----------- compiler/rustc_parse/src/parser/mod.rs | 6 +++--- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/compiler/rustc_expand/src/mbe/diagnostics.rs b/compiler/rustc_expand/src/mbe/diagnostics.rs index bf475c1dc96ff..2df8b8f00f862 100644 --- a/compiler/rustc_expand/src/mbe/diagnostics.rs +++ b/compiler/rustc_expand/src/mbe/diagnostics.rs @@ -120,21 +120,21 @@ struct CollectTrackerAndEmitter<'a, 'cx, 'matcher> { struct BestFailure { token: Token, - position_in_tokenstream: usize, + position_in_tokenstream: u32, msg: &'static str, remaining_matcher: MatcherLoc, } impl BestFailure { - fn is_better_position(&self, position: usize) -> bool { + fn is_better_position(&self, position: u32) -> bool { position > self.position_in_tokenstream } } impl<'a, 'cx, 'matcher> Tracker<'matcher> for CollectTrackerAndEmitter<'a, 'cx, 'matcher> { - type Failure = (Token, usize, &'static str); + type Failure = (Token, u32, &'static str); - fn build_failure(tok: Token, position: usize, msg: &'static str) -> Self::Failure { + fn build_failure(tok: Token, position: u32, msg: &'static str) -> Self::Failure { (tok, position, msg) } @@ -211,9 +211,9 @@ impl<'matcher> FailureForwarder<'matcher> { } impl<'matcher> Tracker<'matcher> for FailureForwarder<'matcher> { - type Failure = (Token, usize, &'static str); + type Failure = (Token, u32, &'static str); - fn build_failure(tok: Token, position: usize, msg: &'static str) -> Self::Failure { + fn build_failure(tok: Token, position: u32, msg: &'static str) -> Self::Failure { (tok, position, msg) } diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 2fbd09fd9ae20..99a9d4f8912cd 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -452,7 +452,7 @@ impl TtParser { &mut self, matcher: &'matcher [MatcherLoc], token: &Token, - approx_position: usize, + approx_position: u32, track: &mut T, ) -> Option> { // Matcher positions that would be valid if the macro invocation was over now. Only diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index e43ba7c3a5a8f..88ec3d8366429 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -153,7 +153,7 @@ pub(super) trait Tracker<'matcher> { /// Arm failed to match. If the token is `token::Eof`, it indicates an unexpected /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. /// The usize is the approximate position of the token in the input token stream. - fn build_failure(tok: Token, position: usize, msg: &'static str) -> Self::Failure; + fn build_failure(tok: Token, position: u32, msg: &'static str) -> Self::Failure; /// This is called before trying to match next MatcherLoc on the current token. fn before_match_loc(&mut self, _parser: &TtParser, _matcher: &'matcher MatcherLoc) {} @@ -182,7 +182,7 @@ pub(super) struct NoopTracker; impl<'matcher> Tracker<'matcher> for NoopTracker { type Failure = (); - fn build_failure(_tok: Token, _position: usize, _msg: &'static str) -> Self::Failure {} + fn build_failure(_tok: Token, _position: u32, _msg: &'static str) -> Self::Failure {} fn description() -> &'static str { "none" diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 10feccd21e286..d0d01cec816fb 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -29,15 +29,15 @@ pub struct AttrWrapper { // The start of the outer attributes in the token cursor. // This allows us to create a `ReplaceRange` for the entire attribute // target, including outer attributes. - start_pos: usize, + start_pos: u32, } impl AttrWrapper { - pub(super) fn new(attrs: AttrVec, start_pos: usize) -> AttrWrapper { + pub(super) fn new(attrs: AttrVec, start_pos: u32) -> AttrWrapper { AttrWrapper { attrs, start_pos } } pub fn empty() -> AttrWrapper { - AttrWrapper { attrs: AttrVec::new(), start_pos: usize::MAX } + AttrWrapper { attrs: AttrVec::new(), start_pos: u32::MAX } } pub(crate) fn take_for_recovery(self, psess: &ParseSess) -> AttrVec { @@ -91,7 +91,7 @@ fn has_cfg_or_cfg_attr(attrs: &[Attribute]) -> bool { struct LazyAttrTokenStreamImpl { start_token: (Token, Spacing), cursor_snapshot: TokenCursor, - num_calls: usize, + num_calls: u32, break_last_token: bool, replace_ranges: Box<[ReplaceRange]>, } @@ -110,7 +110,7 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { let token = cursor_snapshot.next(); (FlatToken::Token(token.0), token.1) })) - .take(self.num_calls); + .take(self.num_calls as usize); if self.replace_ranges.is_empty() { make_token_stream(tokens, self.break_last_token) @@ -296,12 +296,12 @@ impl<'a> Parser<'a> { ); let end_pos = self.num_bump_calls - + captured_trailing as usize + + captured_trailing as u32 // If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens), then // extend the range of captured tokens to include it, since the parser was not actually // bumped past it. When the `LazyAttrTokenStream` gets converted into an // `AttrTokenStream`, we will create the proper token. - + self.break_last_token as usize; + + self.break_last_token as u32; let num_calls = end_pos - start_pos; @@ -313,14 +313,11 @@ impl<'a> Parser<'a> { // Grab any replace ranges that occur *inside* the current AST node. // We will perform the actual replacement when we convert the `LazyAttrTokenStream` // to an `AttrTokenStream`. - let start_calls: u32 = start_pos.try_into().unwrap(); self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end] .iter() .cloned() .chain(inner_attr_replace_ranges.iter().cloned()) - .map(|(range, tokens)| { - ((range.start - start_calls)..(range.end - start_calls), tokens) - }) + .map(|(range, tokens)| ((range.start - start_pos)..(range.end - start_pos), tokens)) .collect() }; @@ -459,6 +456,6 @@ mod size_asserts { use rustc_data_structures::static_assert_size; // tidy-alphabetical-start static_assert_size!(AttrWrapper, 16); - static_assert_size!(LazyAttrTokenStreamImpl, 104); + static_assert_size!(LazyAttrTokenStreamImpl, 96); // tidy-alphabetical-end } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index cfd0a72c056d5..5f16a3e1f3784 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -153,7 +153,7 @@ pub struct Parser<'a> { expected_tokens: Vec, token_cursor: TokenCursor, // The number of calls to `bump`, i.e. the position in the token stream. - num_bump_calls: usize, + num_bump_calls: u32, // During parsing we may sometimes need to 'unglue' a glued token into two // component tokens (e.g. '>>' into '>' and '>), so the parser can consume // them one at a time. This process bypasses the normal capturing mechanism @@ -192,7 +192,7 @@ pub struct Parser<'a> { // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure // it doesn't unintentionally get bigger. #[cfg(target_pointer_width = "64")] -rustc_data_structures::static_assert_size!(Parser<'_>, 264); +rustc_data_structures::static_assert_size!(Parser<'_>, 256); /// Stores span information about a closure. #[derive(Clone, Debug)] @@ -1572,7 +1572,7 @@ impl<'a> Parser<'a> { self.expected_tokens.clear(); } - pub fn approx_token_stream_pos(&self) -> usize { + pub fn approx_token_stream_pos(&self) -> u32 { self.num_bump_calls } } From 6f6015679fb0153bd35d1b56df2456bb6146a9af Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 2 Jul 2024 17:38:43 +1000 Subject: [PATCH 17/26] Rename `make_token_stream`. And update the comment. Clearly the return type of this function was changed at some point in the past, but its name and comment weren't updated to match. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index d0d01cec816fb..3afd178cc2b4e 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -113,7 +113,7 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { .take(self.num_calls as usize); if self.replace_ranges.is_empty() { - make_token_stream(tokens, self.break_last_token) + make_attr_token_stream(tokens, self.break_last_token) } else { let mut tokens: Vec<_> = tokens.collect(); let mut replace_ranges = self.replace_ranges.to_vec(); @@ -166,7 +166,7 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { new_tokens.into_iter().chain(filler), ); } - make_token_stream(tokens.into_iter(), self.break_last_token) + make_attr_token_stream(tokens.into_iter(), self.break_last_token) } } } @@ -374,10 +374,10 @@ impl<'a> Parser<'a> { } } -/// Converts a flattened iterator of tokens (including open and close delimiter tokens) -/// into a `TokenStream`, creating a `TokenTree::Delimited` for each matching pair -/// of open and close delims. -fn make_token_stream( +/// Converts a flattened iterator of tokens (including open and close delimiter tokens) into an +/// `AttrTokenStream`, creating an `AttrTokenTree::Delimited` for each matching pair of open and +/// close delims. +fn make_attr_token_stream( mut iter: impl Iterator, break_last_token: bool, ) -> AttrTokenStream { From edeebe675b39cbdca93ec5b4c79f9a0765678112 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 2 Jul 2024 20:29:01 +1000 Subject: [PATCH 18/26] Import `std::{iter,mem}`. --- .../rustc_parse/src/parser/attr_wrapper.rs | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 3afd178cc2b4e..13a647adfe3b5 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -9,6 +9,7 @@ use rustc_session::parse::ParseSess; use rustc_span::{sym, Span, DUMMY_SP}; use std::ops::Range; +use std::{iter, mem}; /// A wrapper type to ensure that the parser handles outer attributes correctly. /// When we parse outer attributes, we need to ensure that we capture tokens @@ -53,7 +54,7 @@ impl AttrWrapper { // FIXME: require passing an NT to prevent misuse of this method pub(crate) fn prepend_to_nt_inner(self, attrs: &mut AttrVec) { let mut self_attrs = self.attrs; - std::mem::swap(attrs, &mut self_attrs); + mem::swap(attrs, &mut self_attrs); attrs.extend(self_attrs); } @@ -104,13 +105,12 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { // produce an empty `TokenStream` if no calls were made, and omit the // final token otherwise. let mut cursor_snapshot = self.cursor_snapshot.clone(); - let tokens = - std::iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) - .chain(std::iter::repeat_with(|| { - let token = cursor_snapshot.next(); - (FlatToken::Token(token.0), token.1) - })) - .take(self.num_calls as usize); + let tokens = iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) + .chain(iter::repeat_with(|| { + let token = cursor_snapshot.next(); + (FlatToken::Token(token.0), token.1) + })) + .take(self.num_calls as usize); if self.replace_ranges.is_empty() { make_attr_token_stream(tokens, self.break_last_token) @@ -158,7 +158,7 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { // This keeps the total length of `tokens` constant throughout the // replacement process, allowing us to use all of the `ReplaceRanges` entries // without adjusting indices. - let filler = std::iter::repeat((FlatToken::Empty, Spacing::Alone)) + let filler = iter::repeat((FlatToken::Empty, Spacing::Alone)) .take(range.len() - new_tokens.len()); tokens.splice( @@ -222,8 +222,7 @@ impl<'a> Parser<'a> { let replace_ranges_start = self.capture_state.replace_ranges.len(); let (mut ret, trailing) = { - let prev_capturing = - std::mem::replace(&mut self.capture_state.capturing, Capturing::Yes); + let prev_capturing = mem::replace(&mut self.capture_state.capturing, Capturing::Yes); let ret_and_trailing = f(self, attrs.attrs); self.capture_state.capturing = prev_capturing; ret_and_trailing? From a21ba348964e6d72a3369daa4bc22af507eb7778 Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Sat, 29 Jun 2024 23:01:24 -0400 Subject: [PATCH 19/26] add TyCtxt::as_lang_item, use in new solver --- compiler/rustc_hir/src/lang_items.rs | 13 ++- .../rustc_middle/src/middle/lang_items.rs | 4 + compiler/rustc_middle/src/ty/context.rs | 95 +++++++++++-------- .../src/solve/assembly/mod.rs | 87 +++++++++-------- compiler/rustc_type_ir/src/interner.rs | 2 + 5 files changed, 126 insertions(+), 75 deletions(-) diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 3c44acb16577a..30c0e40206aaf 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -11,6 +11,7 @@ use crate::def_id::DefId; use crate::{MethodKind, Target}; use rustc_ast as ast; +use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_macros::{Decodable, Encodable, HashStable_Generic}; use rustc_span::symbol::{kw, sym, Symbol}; @@ -23,6 +24,7 @@ pub struct LanguageItems { /// Mappings from lang items to their possibly found [`DefId`]s. /// The index corresponds to the order in [`LangItem`]. items: [Option; std::mem::variant_count::()], + reverse_items: FxIndexMap, /// Lang items that were not found during collection. pub missing: Vec, } @@ -30,7 +32,11 @@ pub struct LanguageItems { impl LanguageItems { /// Construct an empty collection of lang items and no missing ones. pub fn new() -> Self { - Self { items: [None; std::mem::variant_count::()], missing: Vec::new() } + Self { + items: [None; std::mem::variant_count::()], + reverse_items: FxIndexMap::default(), + missing: Vec::new(), + } } pub fn get(&self, item: LangItem) -> Option { @@ -39,6 +45,11 @@ impl LanguageItems { pub fn set(&mut self, item: LangItem, def_id: DefId) { self.items[item as usize] = Some(def_id); + self.reverse_items.insert(def_id, item); + } + + pub fn from_def_id(&self, def_id: DefId) -> Option { + self.reverse_items.get(&def_id).copied() } pub fn iter(&self) -> impl Iterator + '_ { diff --git a/compiler/rustc_middle/src/middle/lang_items.rs b/compiler/rustc_middle/src/middle/lang_items.rs index e76d7af6e4aba..a0c9af436e207 100644 --- a/compiler/rustc_middle/src/middle/lang_items.rs +++ b/compiler/rustc_middle/src/middle/lang_items.rs @@ -27,6 +27,10 @@ impl<'tcx> TyCtxt<'tcx> { self.lang_items().get(lang_item) == Some(def_id) } + pub fn as_lang_item(self, def_id: DefId) -> Option { + self.lang_items().from_def_id(def_id) + } + /// Given a [`DefId`] of one of the [`Fn`], [`FnMut`] or [`FnOnce`] traits, /// returns a corresponding [`ty::ClosureKind`]. /// For any other [`DefId`] return `None`. diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 9225ae6300fc6..155a76713cab6 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -366,6 +366,10 @@ impl<'tcx> Interner for TyCtxt<'tcx> { self.is_lang_item(def_id, trait_lang_item_to_lang_item(lang_item)) } + fn as_lang_item(self, def_id: DefId) -> Option { + lang_item_to_trait_lang_item(self.lang_items().from_def_id(def_id)?) + } + fn associated_type_def_ids(self, def_id: DefId) -> impl IntoIterator { self.associated_items(def_id) .in_definition_order() @@ -584,46 +588,63 @@ impl<'tcx> Interner for TyCtxt<'tcx> { } } -fn trait_lang_item_to_lang_item(lang_item: TraitSolverLangItem) -> LangItem { - match lang_item { - TraitSolverLangItem::AsyncDestruct => LangItem::AsyncDestruct, - TraitSolverLangItem::AsyncFnKindHelper => LangItem::AsyncFnKindHelper, - TraitSolverLangItem::AsyncFnKindUpvars => LangItem::AsyncFnKindUpvars, - TraitSolverLangItem::AsyncFnOnceOutput => LangItem::AsyncFnOnceOutput, - TraitSolverLangItem::AsyncIterator => LangItem::AsyncIterator, - TraitSolverLangItem::CallOnceFuture => LangItem::CallOnceFuture, - TraitSolverLangItem::CallRefFuture => LangItem::CallRefFuture, - TraitSolverLangItem::Clone => LangItem::Clone, - TraitSolverLangItem::Copy => LangItem::Copy, - TraitSolverLangItem::Coroutine => LangItem::Coroutine, - TraitSolverLangItem::CoroutineReturn => LangItem::CoroutineReturn, - TraitSolverLangItem::CoroutineYield => LangItem::CoroutineYield, - TraitSolverLangItem::Destruct => LangItem::Destruct, - TraitSolverLangItem::DiscriminantKind => LangItem::DiscriminantKind, - TraitSolverLangItem::DynMetadata => LangItem::DynMetadata, - TraitSolverLangItem::EffectsMaybe => LangItem::EffectsMaybe, - TraitSolverLangItem::EffectsIntersection => LangItem::EffectsIntersection, - TraitSolverLangItem::EffectsIntersectionOutput => LangItem::EffectsIntersectionOutput, - TraitSolverLangItem::EffectsNoRuntime => LangItem::EffectsNoRuntime, - TraitSolverLangItem::EffectsRuntime => LangItem::EffectsRuntime, - TraitSolverLangItem::FnPtrTrait => LangItem::FnPtrTrait, - TraitSolverLangItem::FusedIterator => LangItem::FusedIterator, - TraitSolverLangItem::Future => LangItem::Future, - TraitSolverLangItem::FutureOutput => LangItem::FutureOutput, - TraitSolverLangItem::Iterator => LangItem::Iterator, - TraitSolverLangItem::Metadata => LangItem::Metadata, - TraitSolverLangItem::Option => LangItem::Option, - TraitSolverLangItem::PointeeTrait => LangItem::PointeeTrait, - TraitSolverLangItem::PointerLike => LangItem::PointerLike, - TraitSolverLangItem::Poll => LangItem::Poll, - TraitSolverLangItem::Sized => LangItem::Sized, - TraitSolverLangItem::TransmuteTrait => LangItem::TransmuteTrait, - TraitSolverLangItem::Tuple => LangItem::Tuple, - TraitSolverLangItem::Unpin => LangItem::Unpin, - TraitSolverLangItem::Unsize => LangItem::Unsize, +macro_rules! bidirectional_lang_item_map { + ($($name:ident),+ $(,)?) => { + fn trait_lang_item_to_lang_item(lang_item: TraitSolverLangItem) -> LangItem { + match lang_item { + $(TraitSolverLangItem::$name => LangItem::$name,)+ + } + } + + fn lang_item_to_trait_lang_item(lang_item: LangItem) -> Option { + Some(match lang_item { + $(LangItem::$name => TraitSolverLangItem::$name,)+ + _ => return None, + }) + } } } +bidirectional_lang_item_map! { +// tidy-alphabetical-start + AsyncDestruct, + AsyncFnKindHelper, + AsyncFnKindUpvars, + AsyncFnOnceOutput, + AsyncIterator, + CallOnceFuture, + CallRefFuture, + Clone, + Copy, + Coroutine, + CoroutineReturn, + CoroutineYield, + Destruct, + DiscriminantKind, + DynMetadata, + EffectsMaybe, + EffectsIntersection, + EffectsIntersectionOutput, + EffectsNoRuntime, + EffectsRuntime, + FnPtrTrait, + FusedIterator, + Future, + FutureOutput, + Iterator, + Metadata, + Option, + PointeeTrait, + PointerLike, + Poll, + Sized, + TransmuteTrait, + Tuple, + Unpin, + Unsize, +// tidy-alphabetical-end +} + impl<'tcx> rustc_type_ir::inherent::DefId> for DefId { fn as_local(self) -> Option { self.as_local() diff --git a/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs b/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs index 6ee684605ac60..20a0da16c7521 100644 --- a/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs @@ -387,48 +387,61 @@ where G::consider_auto_trait_candidate(self, goal) } else if cx.trait_is_alias(trait_def_id) { G::consider_trait_alias_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::Sized) { - G::consider_builtin_sized_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::Copy) - || cx.is_lang_item(trait_def_id, TraitSolverLangItem::Clone) - { - G::consider_builtin_copy_clone_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::PointerLike) { - G::consider_builtin_pointer_like_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::FnPtrTrait) { - G::consider_builtin_fn_ptr_trait_candidate(self, goal) } else if let Some(kind) = self.cx().fn_trait_kind_from_def_id(trait_def_id) { G::consider_builtin_fn_trait_candidates(self, goal, kind) } else if let Some(kind) = self.cx().async_fn_trait_kind_from_def_id(trait_def_id) { G::consider_builtin_async_fn_trait_candidates(self, goal, kind) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::AsyncFnKindHelper) { - G::consider_builtin_async_fn_kind_helper_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::Tuple) { - G::consider_builtin_tuple_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::PointeeTrait) { - G::consider_builtin_pointee_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::Future) { - G::consider_builtin_future_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::Iterator) { - G::consider_builtin_iterator_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::FusedIterator) { - G::consider_builtin_fused_iterator_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::AsyncIterator) { - G::consider_builtin_async_iterator_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::Coroutine) { - G::consider_builtin_coroutine_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::DiscriminantKind) { - G::consider_builtin_discriminant_kind_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::AsyncDestruct) { - G::consider_builtin_async_destruct_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::Destruct) { - G::consider_builtin_destruct_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::TransmuteTrait) { - G::consider_builtin_transmute_candidate(self, goal) - } else if cx.is_lang_item(trait_def_id, TraitSolverLangItem::EffectsIntersection) { - G::consider_builtin_effects_intersection_candidate(self, goal) } else { - Err(NoSolution) + match cx.as_lang_item(trait_def_id) { + Some(TraitSolverLangItem::Sized) => G::consider_builtin_sized_candidate(self, goal), + Some(TraitSolverLangItem::Copy | TraitSolverLangItem::Clone) => { + G::consider_builtin_copy_clone_candidate(self, goal) + } + Some(TraitSolverLangItem::PointerLike) => { + G::consider_builtin_pointer_like_candidate(self, goal) + } + Some(TraitSolverLangItem::FnPtrTrait) => { + G::consider_builtin_fn_ptr_trait_candidate(self, goal) + } + Some(TraitSolverLangItem::AsyncFnKindHelper) => { + G::consider_builtin_async_fn_kind_helper_candidate(self, goal) + } + Some(TraitSolverLangItem::Tuple) => G::consider_builtin_tuple_candidate(self, goal), + Some(TraitSolverLangItem::PointeeTrait) => { + G::consider_builtin_pointee_candidate(self, goal) + } + Some(TraitSolverLangItem::Future) => { + G::consider_builtin_future_candidate(self, goal) + } + Some(TraitSolverLangItem::Iterator) => { + G::consider_builtin_iterator_candidate(self, goal) + } + Some(TraitSolverLangItem::FusedIterator) => { + G::consider_builtin_fused_iterator_candidate(self, goal) + } + Some(TraitSolverLangItem::AsyncIterator) => { + G::consider_builtin_async_iterator_candidate(self, goal) + } + Some(TraitSolverLangItem::Coroutine) => { + G::consider_builtin_coroutine_candidate(self, goal) + } + Some(TraitSolverLangItem::DiscriminantKind) => { + G::consider_builtin_discriminant_kind_candidate(self, goal) + } + Some(TraitSolverLangItem::AsyncDestruct) => { + G::consider_builtin_async_destruct_candidate(self, goal) + } + Some(TraitSolverLangItem::Destruct) => { + G::consider_builtin_destruct_candidate(self, goal) + } + Some(TraitSolverLangItem::TransmuteTrait) => { + G::consider_builtin_transmute_candidate(self, goal) + } + Some(TraitSolverLangItem::EffectsIntersection) => { + G::consider_builtin_effects_intersection_candidate(self, goal) + } + _ => Err(NoSolution), + } }; candidates.extend(result); diff --git a/compiler/rustc_type_ir/src/interner.rs b/compiler/rustc_type_ir/src/interner.rs index 6665158c7cd34..84c7e6e8f2003 100644 --- a/compiler/rustc_type_ir/src/interner.rs +++ b/compiler/rustc_type_ir/src/interner.rs @@ -220,6 +220,8 @@ pub trait Interner: fn is_lang_item(self, def_id: Self::DefId, lang_item: TraitSolverLangItem) -> bool; + fn as_lang_item(self, def_id: Self::DefId) -> Option; + fn associated_type_def_ids(self, def_id: Self::DefId) -> impl IntoIterator; // FIXME: move `fast_reject` into `rustc_type_ir`. From 5a837515f28eed106dd031b0243d6476eba58dd9 Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Sat, 29 Jun 2024 23:05:35 -0400 Subject: [PATCH 20/26] Make fn traits into first-class TraitSolverLangItems to avoid needing fn_trait_kind_from_def_id --- compiler/rustc_middle/src/ty/context.rs | 16 +++++----- .../src/solve/assembly/mod.rs | 30 ++++++++++++++++--- compiler/rustc_type_ir/src/interner.rs | 4 --- compiler/rustc_type_ir/src/lang_items.rs | 6 ++++ 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 155a76713cab6..7c3d2df72faa2 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -537,14 +537,6 @@ impl<'tcx> Interner for TyCtxt<'tcx> { self.trait_def(trait_def_id).implement_via_object } - fn fn_trait_kind_from_def_id(self, trait_def_id: DefId) -> Option { - self.fn_trait_kind_from_def_id(trait_def_id) - } - - fn async_fn_trait_kind_from_def_id(self, trait_def_id: DefId) -> Option { - self.async_fn_trait_kind_from_def_id(trait_def_id) - } - fn supertrait_def_ids(self, trait_def_id: DefId) -> impl IntoIterator { self.supertrait_def_ids(trait_def_id) } @@ -608,8 +600,11 @@ macro_rules! bidirectional_lang_item_map { bidirectional_lang_item_map! { // tidy-alphabetical-start AsyncDestruct, + AsyncFn, AsyncFnKindHelper, AsyncFnKindUpvars, + AsyncFnMut, + AsyncFnOnce, AsyncFnOnceOutput, AsyncIterator, CallOnceFuture, @@ -622,11 +617,14 @@ bidirectional_lang_item_map! { Destruct, DiscriminantKind, DynMetadata, - EffectsMaybe, EffectsIntersection, EffectsIntersectionOutput, + EffectsMaybe, EffectsNoRuntime, EffectsRuntime, + Fn, + FnMut, + FnOnce, FnPtrTrait, FusedIterator, Future, diff --git a/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs b/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs index 20a0da16c7521..38a4f7dfe25cb 100644 --- a/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs @@ -387,16 +387,38 @@ where G::consider_auto_trait_candidate(self, goal) } else if cx.trait_is_alias(trait_def_id) { G::consider_trait_alias_candidate(self, goal) - } else if let Some(kind) = self.cx().fn_trait_kind_from_def_id(trait_def_id) { - G::consider_builtin_fn_trait_candidates(self, goal, kind) - } else if let Some(kind) = self.cx().async_fn_trait_kind_from_def_id(trait_def_id) { - G::consider_builtin_async_fn_trait_candidates(self, goal, kind) } else { match cx.as_lang_item(trait_def_id) { Some(TraitSolverLangItem::Sized) => G::consider_builtin_sized_candidate(self, goal), Some(TraitSolverLangItem::Copy | TraitSolverLangItem::Clone) => { G::consider_builtin_copy_clone_candidate(self, goal) } + Some(TraitSolverLangItem::Fn) => { + G::consider_builtin_fn_trait_candidates(self, goal, ty::ClosureKind::Fn) + } + Some(TraitSolverLangItem::FnMut) => { + G::consider_builtin_fn_trait_candidates(self, goal, ty::ClosureKind::FnMut) + } + Some(TraitSolverLangItem::FnOnce) => { + G::consider_builtin_fn_trait_candidates(self, goal, ty::ClosureKind::FnOnce) + } + Some(TraitSolverLangItem::AsyncFn) => { + G::consider_builtin_async_fn_trait_candidates(self, goal, ty::ClosureKind::Fn) + } + Some(TraitSolverLangItem::AsyncFnMut) => { + G::consider_builtin_async_fn_trait_candidates( + self, + goal, + ty::ClosureKind::FnMut, + ) + } + Some(TraitSolverLangItem::AsyncFnOnce) => { + G::consider_builtin_async_fn_trait_candidates( + self, + goal, + ty::ClosureKind::FnOnce, + ) + } Some(TraitSolverLangItem::PointerLike) => { G::consider_builtin_pointer_like_candidate(self, goal) } diff --git a/compiler/rustc_type_ir/src/interner.rs b/compiler/rustc_type_ir/src/interner.rs index 84c7e6e8f2003..f76a2f55c482e 100644 --- a/compiler/rustc_type_ir/src/interner.rs +++ b/compiler/rustc_type_ir/src/interner.rs @@ -254,10 +254,6 @@ pub trait Interner: fn trait_may_be_implemented_via_object(self, trait_def_id: Self::DefId) -> bool; - fn fn_trait_kind_from_def_id(self, trait_def_id: Self::DefId) -> Option; - - fn async_fn_trait_kind_from_def_id(self, trait_def_id: Self::DefId) -> Option; - fn supertrait_def_ids(self, trait_def_id: Self::DefId) -> impl IntoIterator; diff --git a/compiler/rustc_type_ir/src/lang_items.rs b/compiler/rustc_type_ir/src/lang_items.rs index cf00c37caa287..265a411882735 100644 --- a/compiler/rustc_type_ir/src/lang_items.rs +++ b/compiler/rustc_type_ir/src/lang_items.rs @@ -3,8 +3,11 @@ pub enum TraitSolverLangItem { // tidy-alphabetical-start AsyncDestruct, + AsyncFn, AsyncFnKindHelper, AsyncFnKindUpvars, + AsyncFnMut, + AsyncFnOnce, AsyncFnOnceOutput, AsyncIterator, CallOnceFuture, @@ -22,6 +25,9 @@ pub enum TraitSolverLangItem { EffectsMaybe, EffectsNoRuntime, EffectsRuntime, + Fn, + FnMut, + FnOnce, FnPtrTrait, FusedIterator, Future, From 64a3bd84d83a6138b6e7db82a53199246f292e53 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 21 Jun 2024 14:01:15 -0500 Subject: [PATCH 21/26] Always preserve user-written comments in assembly --- compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp | 3 ++- tests/assembly/asm-comments.rs | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/assembly/asm-comments.rs diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index c4cfc0b6dc6a1..a868c56c4e625 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -436,7 +436,8 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( Options.FunctionSections = FunctionSections; Options.UniqueSectionNames = UniqueSectionNames; Options.MCOptions.AsmVerbose = AsmComments; - Options.MCOptions.PreserveAsmComments = AsmComments; + // Always preserve comments that were written by the user + Options.MCOptions.PreserveAsmComments = true; Options.MCOptions.ABIName = ABIStr; if (SplitDwarfFile) { Options.MCOptions.SplitDwarfFile = SplitDwarfFile; diff --git a/tests/assembly/asm-comments.rs b/tests/assembly/asm-comments.rs new file mode 100644 index 0000000000000..557009975ddc9 --- /dev/null +++ b/tests/assembly/asm-comments.rs @@ -0,0 +1,12 @@ +//@ assembly-output: emit-asm +//@ only-x86_64 +// Check that comments in assembly get passed + +#![crate_type = "lib"] + +// CHECK-LABEL: test_comments: +#[no_mangle] +pub fn test_comments() { + // CHECK: example comment + unsafe { core::arch::asm!("nop // example comment") }; +} From c15a698f567356cea22e79686c2883f3557bae0b Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 21 Jun 2024 14:08:49 -0500 Subject: [PATCH 22/26] Rename the `asm-comments` compiler flag to `verbose-asm` Since this codegen flag now only controls LLVM-generated comments rather than all assembly comments, make the name more accurate (and also match Clang). --- compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs | 4 ++-- compiler/rustc_codegen_llvm/src/back/write.rs | 4 ++-- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 +- compiler/rustc_interface/src/tests.rs | 2 +- compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp | 4 ++-- compiler/rustc_session/src/options.rs | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 28a88dd2efea0..b72636a62248e 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -32,7 +32,7 @@ impl OwnedTargetMachine { unique_section_names: bool, trap_unreachable: bool, singletree: bool, - asm_comments: bool, + verbose_asm: bool, emit_stack_size_section: bool, relax_elf_relocations: bool, use_init_array: bool, @@ -64,7 +64,7 @@ impl OwnedTargetMachine { unique_section_names, trap_unreachable, singletree, - asm_comments, + verbose_asm, emit_stack_size_section, relax_elf_relocations, use_init_array, diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 5e481eb98f55f..2fda19bf0c914 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -214,7 +214,7 @@ pub fn target_machine_factory( sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; - let asm_comments = sess.opts.unstable_opts.asm_comments; + let verbose_asm = sess.opts.unstable_opts.verbose_asm; let relax_elf_relocations = sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); @@ -289,7 +289,7 @@ pub fn target_machine_factory( funique_section_names, trap_unreachable, singlethread, - asm_comments, + verbose_asm, emit_stack_size_section, relax_elf_relocations, use_init_array, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 132e1f9e8fd93..08e9e312827ce 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2185,7 +2185,7 @@ extern "C" { UniqueSectionNames: bool, TrapUnreachable: bool, Singlethread: bool, - AsmComments: bool, + VerboseAsm: bool, EmitStackSizeSection: bool, RelaxELFRelocations: bool, UseInitArray: bool, diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 9bd67a1154b7d..e2ba75dfd1902 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -757,7 +757,6 @@ fn test_unstable_options_tracking_hash() { // tidy-alphabetical-start tracked!(allow_features, Some(vec![String::from("lang_items")])); tracked!(always_encode_mir, true); - tracked!(asm_comments, true); tracked!(assume_incomplete_release, true); tracked!(binary_dep_depinfo, true); tracked!(box_noalias, false); @@ -862,6 +861,7 @@ fn test_unstable_options_tracking_hash() { tracked!(uninit_const_chunk_threshold, 123); tracked!(unleash_the_miri_inside_of_you, true); tracked!(use_ctors_section, Some(true)); + tracked!(verbose_asm, true); tracked!(verify_llvm_ir, true); tracked!(virtual_function_elimination, true); tracked!(wasi_exec_model, Some(WasiExecModel::Reactor)); diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index a868c56c4e625..283c4fbbb7cf4 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -407,7 +407,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, bool FunctionSections, bool DataSections, bool UniqueSectionNames, - bool TrapUnreachable, bool Singlethread, bool AsmComments, + bool TrapUnreachable, bool Singlethread, bool VerboseAsm, bool EmitStackSizeSection, bool RelaxELFRelocations, bool UseInitArray, const char *SplitDwarfFile, const char *OutputObjFile, const char *DebugInfoCompression, bool UseEmulatedTls, @@ -435,7 +435,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( Options.DataSections = DataSections; Options.FunctionSections = FunctionSections; Options.UniqueSectionNames = UniqueSectionNames; - Options.MCOptions.AsmVerbose = AsmComments; + Options.MCOptions.AsmVerbose = VerboseAsm; // Always preserve comments that were written by the user Options.MCOptions.PreserveAsmComments = true; Options.MCOptions.ABIName = ABIStr; diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 2e4421d50e313..7421cae65e4f8 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1630,8 +1630,6 @@ options! { "only allow the listed language features to be enabled in code (comma separated)"), always_encode_mir: bool = (false, parse_bool, [TRACKED], "encode MIR of all functions into the crate metadata (default: no)"), - asm_comments: bool = (false, parse_bool, [TRACKED], - "generate comments into the assembly (may change behavior) (default: no)"), assert_incr_state: Option = (None, parse_opt_string, [UNTRACKED], "assert that the incremental cache is in given state: \ either `loaded` or `not-loaded`."), @@ -2107,6 +2105,8 @@ written to standard error output)"), "Generate sync unwind tables instead of async unwind tables (default: no)"), validate_mir: bool = (false, parse_bool, [UNTRACKED], "validate MIR after each transformation"), + verbose_asm: bool = (false, parse_bool, [TRACKED], + "add descriptive comments from LLVM to the assembly (may change behavior) (default: no)"), #[rustc_lint_opt_deny_field_access("use `Session::verbose_internals` instead of this field")] verbose_internals: bool = (false, parse_bool, [TRACKED_NO_CRATE_HASH], "in general, enable more debug printouts (default: no)"), From 1a6893e14b7d08aee18acf82b9a08e1ba8534d7b Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 21 Jun 2024 14:17:47 -0500 Subject: [PATCH 23/26] Add documentation for -Zverbose-asm --- .../src/compiler-flags/verbose-asm.md | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/doc/unstable-book/src/compiler-flags/verbose-asm.md diff --git a/src/doc/unstable-book/src/compiler-flags/verbose-asm.md b/src/doc/unstable-book/src/compiler-flags/verbose-asm.md new file mode 100644 index 0000000000000..84eb90a14cf53 --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/verbose-asm.md @@ -0,0 +1,70 @@ +# `verbose-asm` + +The tracking issue for this feature is: [#126802](https://github.com/rust-lang/rust/issues/126802). + +------------------------ + +This enables passing `-Zverbose-asm` to get contextual comments added by LLVM. + +Sample code: + +```rust +#[no_mangle] +pub fn foo(a: i32, b: i32) -> i32 { + a + b +} +``` + +Default output: + +```asm +foo: + push rax + add edi, esi + mov dword ptr [rsp + 4], edi + seto al + jo .LBB0_2 + mov eax, dword ptr [rsp + 4] + pop rcx + ret +.LBB0_2: + lea rdi, [rip + .L__unnamed_1] + mov rax, qword ptr [rip + core::panicking::panic_const::panic_const_add_overflow::h9c85248fe0d735b2@GOTPCREL] + call rax + +.L__unnamed_2: + .ascii "/app/example.rs" + +.L__unnamed_1: + .quad .L__unnamed_2 + .asciz "\017\000\000\000\000\000\000\000\004\000\000\000\005\000\000" +``` + +With `-Zverbose-asm`: + +```asm +foo: # @foo +# %bb.0: + push rax + add edi, esi + mov dword ptr [rsp + 4], edi # 4-byte Spill + seto al + jo .LBB0_2 +# %bb.1: + mov eax, dword ptr [rsp + 4] # 4-byte Reload + pop rcx + ret +.LBB0_2: + lea rdi, [rip + .L__unnamed_1] + mov rax, qword ptr [rip + core::panicking::panic_const::panic_const_add_overflow::h9c85248fe0d735b2@GOTPCREL] + call rax + # -- End function +.L__unnamed_2: + .ascii "/app/example.rs" + +.L__unnamed_1: + .quad .L__unnamed_2 + .asciz "\017\000\000\000\000\000\000\000\004\000\000\000\005\000\000" + + # DW_AT_external +``` From 9e71c7b5a98fb1f18eade36c9188467616550ffc Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 3 Jul 2024 11:34:39 +0200 Subject: [PATCH 24/26] Small `run-make-support` API improvements --- src/tools/run-make-support/src/lib.rs | 6 ++++-- src/tools/run-make-support/src/rustc.rs | 3 ++- tests/run-make/emit-named-files/rmake.rs | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/tools/run-make-support/src/lib.rs b/src/tools/run-make-support/src/lib.rs index 294dae109425f..9a180fe4ad19a 100644 --- a/src/tools/run-make-support/src/lib.rs +++ b/src/tools/run-make-support/src/lib.rs @@ -321,8 +321,9 @@ pub fn set_host_rpath(cmd: &mut Command) { /// Read the contents of a file that cannot simply be read by /// read_to_string, due to invalid utf8 data, then assert that it contains `expected`. #[track_caller] -pub fn invalid_utf8_contains>(path: P, expected: &str) { +pub fn invalid_utf8_contains, S: AsRef>(path: P, expected: S) { let buffer = fs_wrapper::read(path.as_ref()); + let expected = expected.as_ref(); if !String::from_utf8_lossy(&buffer).contains(expected) { eprintln!("=== FILE CONTENTS (LOSSY) ==="); eprintln!("{}", String::from_utf8_lossy(&buffer)); @@ -335,8 +336,9 @@ pub fn invalid_utf8_contains>(path: P, expected: &str) { /// Read the contents of a file that cannot simply be read by /// read_to_string, due to invalid utf8 data, then assert that it does not contain `expected`. #[track_caller] -pub fn invalid_utf8_not_contains>(path: P, expected: &str) { +pub fn invalid_utf8_not_contains, S: AsRef>(path: P, expected: S) { let buffer = fs_wrapper::read(path.as_ref()); + let expected = expected.as_ref(); if String::from_utf8_lossy(&buffer).contains(expected) { eprintln!("=== FILE CONTENTS (LOSSY) ==="); eprintln!("{}", String::from_utf8_lossy(&buffer)); diff --git a/src/tools/run-make-support/src/rustc.rs b/src/tools/run-make-support/src/rustc.rs index 3f23c1b8f9e76..054836e87cfbc 100644 --- a/src/tools/run-make-support/src/rustc.rs +++ b/src/tools/run-make-support/src/rustc.rs @@ -86,7 +86,8 @@ impl Rustc { } /// Specify type(s) of output files to generate. - pub fn emit(&mut self, kinds: &str) -> &mut Self { + pub fn emit>(&mut self, kinds: S) -> &mut Self { + let kinds = kinds.as_ref(); self.cmd.arg(format!("--emit={kinds}")); self } diff --git a/tests/run-make/emit-named-files/rmake.rs b/tests/run-make/emit-named-files/rmake.rs index 79c3ee90c9875..a02c97fec4cba 100644 --- a/tests/run-make/emit-named-files/rmake.rs +++ b/tests/run-make/emit-named-files/rmake.rs @@ -4,7 +4,7 @@ use run_make_support::{fs_wrapper, rustc}; fn emit_and_check(out_dir: &Path, out_file: &str, format: &str) { let out_file = out_dir.join(out_file); - rustc().input("foo.rs").emit(&format!("{format}={}", out_file.display())).run(); + rustc().input("foo.rs").emit(format!("{format}={}", out_file.display())).run(); assert!(out_file.is_file()); } From bcbcbfff4452ca34c4cee6a7737141df39591668 Mon Sep 17 00:00:00 2001 From: klensy Date: Wed, 3 Jul 2024 14:23:09 +0300 Subject: [PATCH 25/26] bootstrap: pass correct struct size to winapi --- src/bootstrap/src/bin/rustc.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/bootstrap/src/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs index 009e62469b4d1..46e845f77ae1b 100644 --- a/src/bootstrap/src/bin/rustc.rs +++ b/src/bootstrap/src/bin/rustc.rs @@ -317,9 +317,7 @@ fn format_rusage_data(child: Child) -> Option { use windows::{ Win32::Foundation::HANDLE, - Win32::System::ProcessStatus::{ - K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS, PROCESS_MEMORY_COUNTERS_EX, - }, + Win32::System::ProcessStatus::{K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS}, Win32::System::Threading::GetProcessTimes, Win32::System::Time::FileTimeToSystemTime, }; @@ -331,6 +329,7 @@ fn format_rusage_data(child: Child) -> Option { let mut kernel_filetime = Default::default(); let mut kernel_time = Default::default(); let mut memory_counters = PROCESS_MEMORY_COUNTERS::default(); + let memory_counters_size = std::mem::size_of_val(&memory_counters); unsafe { GetProcessTimes( @@ -347,15 +346,9 @@ fn format_rusage_data(child: Child) -> Option { // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process // with the given handle and none of that process's children. - unsafe { - K32GetProcessMemoryInfo( - handle, - &mut memory_counters, - std::mem::size_of::() as u32, - ) - } - .ok() - .ok()?; + unsafe { K32GetProcessMemoryInfo(handle, &mut memory_counters, memory_counters_size as u32) } + .ok() + .ok()?; // Guide on interpreting these numbers: // https://docs.microsoft.com/en-us/windows/win32/psapi/process-memory-usage-information From 8b6435d5ce4efaecca51e530f0cb3c44d527f3c4 Mon Sep 17 00:00:00 2001 From: Santiago Pastorino Date: Tue, 2 Jul 2024 17:35:49 -0300 Subject: [PATCH 26/26] Add parse fail test using safe trait/impl trait --- .../unsafe-extern-blocks/safe-impl-trait.gated.stderr | 8 ++++++++ .../ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.rs | 8 ++++++++ .../unsafe-extern-blocks/safe-impl-trait.ungated.stderr | 8 ++++++++ .../unsafe-extern-blocks/safe-trait.gated.stderr | 8 ++++++++ tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.rs | 7 +++++++ .../unsafe-extern-blocks/safe-trait.ungated.stderr | 8 ++++++++ 6 files changed, 47 insertions(+) create mode 100644 tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.gated.stderr create mode 100644 tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.rs create mode 100644 tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.ungated.stderr create mode 100644 tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.gated.stderr create mode 100644 tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.rs create mode 100644 tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.ungated.stderr diff --git a/tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.gated.stderr b/tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.gated.stderr new file mode 100644 index 0000000000000..80e7a45f57e79 --- /dev/null +++ b/tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.gated.stderr @@ -0,0 +1,8 @@ +error: expected one of `!` or `::`, found keyword `impl` + --> $DIR/safe-impl-trait.rs:5:6 + | +LL | safe impl Bar for () { } + | ^^^^ expected one of `!` or `::` + +error: aborting due to 1 previous error + diff --git a/tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.rs b/tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.rs new file mode 100644 index 0000000000000..57c03e4d896be --- /dev/null +++ b/tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.rs @@ -0,0 +1,8 @@ +//@ revisions: gated ungated +#![cfg_attr(gated, feature(unsafe_extern_blocks))] + +trait Bar {} +safe impl Bar for () { } +//~^ ERROR expected one of `!` or `::`, found keyword `impl` + +fn main() {} diff --git a/tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.ungated.stderr b/tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.ungated.stderr new file mode 100644 index 0000000000000..80e7a45f57e79 --- /dev/null +++ b/tests/ui/rust-2024/unsafe-extern-blocks/safe-impl-trait.ungated.stderr @@ -0,0 +1,8 @@ +error: expected one of `!` or `::`, found keyword `impl` + --> $DIR/safe-impl-trait.rs:5:6 + | +LL | safe impl Bar for () { } + | ^^^^ expected one of `!` or `::` + +error: aborting due to 1 previous error + diff --git a/tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.gated.stderr b/tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.gated.stderr new file mode 100644 index 0000000000000..de84037f28c58 --- /dev/null +++ b/tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.gated.stderr @@ -0,0 +1,8 @@ +error: expected one of `!` or `::`, found keyword `trait` + --> $DIR/safe-trait.rs:4:6 + | +LL | safe trait Foo {} + | ^^^^^ expected one of `!` or `::` + +error: aborting due to 1 previous error + diff --git a/tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.rs b/tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.rs new file mode 100644 index 0000000000000..e73cb45b18877 --- /dev/null +++ b/tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.rs @@ -0,0 +1,7 @@ +//@ revisions: gated ungated +#![cfg_attr(gated, feature(unsafe_extern_blocks))] + +safe trait Foo {} +//~^ ERROR expected one of `!` or `::`, found keyword `trait` + +fn main() {} diff --git a/tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.ungated.stderr b/tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.ungated.stderr new file mode 100644 index 0000000000000..de84037f28c58 --- /dev/null +++ b/tests/ui/rust-2024/unsafe-extern-blocks/safe-trait.ungated.stderr @@ -0,0 +1,8 @@ +error: expected one of `!` or `::`, found keyword `trait` + --> $DIR/safe-trait.rs:4:6 + | +LL | safe trait Foo {} + | ^^^^^ expected one of `!` or `::` + +error: aborting due to 1 previous error +