From 4701025f56664bf6876390ca862a8ef726f23ef4 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Sun, 7 Apr 2019 17:40:56 +0200 Subject: [PATCH] -Z call-metadata: add call metadata to LLVM IR this implements eRFC rust-lang/rust#59412 --- Cargo.lock | 1 + src/librustc/ich/impls_ty.rs | 24 ++ src/librustc/query/mod.rs | 27 +- src/librustc/session/config.rs | 2 + src/librustc/ty/instance.rs | 37 +- src/librustc_codegen_llvm/Cargo.toml | 1 + src/librustc_codegen_llvm/abi.rs | 25 +- src/librustc_codegen_llvm/callee.rs | 9 +- src/librustc_codegen_llvm/common.rs | 164 ++++++++- src/librustc_codegen_llvm/llvm/ffi.rs | 6 + src/librustc_codegen_llvm/mono_item.rs | 4 + .../back/symbol_export.rs | 2 +- src/librustc_codegen_ssa/base.rs | 4 +- src/librustc_codegen_ssa/lib.rs | 9 + src/librustc_codegen_ssa/mir/block.rs | 57 ++- src/librustc_codegen_ssa/traits/abi.rs | 6 +- src/librustc_data_structures/stable_hasher.rs | 19 + src/librustc_mir/monomorphize/collector.rs | 341 ++++++++++++++---- src/librustc_mir/monomorphize/partitioning.rs | 43 ++- src/rustllvm/RustWrapper.cpp | 26 ++ 20 files changed, 705 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0256132be7db7..26b9dbeefc35a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2597,6 +2597,7 @@ dependencies = [ "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_llvm 0.0.0", + "smallvec 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/src/librustc/ich/impls_ty.rs b/src/librustc/ich/impls_ty.rs index ec1b0da681074..87d544d352110 100644 --- a/src/librustc/ich/impls_ty.rs +++ b/src/librustc/ich/impls_ty.rs @@ -54,6 +54,30 @@ impl<'a, 'gcx, T> ToStableHashKey> for &'gcx ty::List ToStableHashKey> for ty::Instance<'tcx> { + type KeyType = Fingerprint; + + #[inline] + fn to_stable_hash_key(&self, hcx: &StableHashingContext<'a>) -> Fingerprint { + let mut hasher = StableHasher::new(); + let mut hcx: StableHashingContext<'a> = hcx.clone(); + self.hash_stable(&mut hcx, &mut hasher); + hasher.finish() + } +} + +impl<'a, 'tcx> ToStableHashKey> for ty::ExistentialTraitRef<'tcx> { + type KeyType = Fingerprint; + + #[inline] + fn to_stable_hash_key(&self, hcx: &StableHashingContext<'a>) -> Fingerprint { + let mut hasher = StableHasher::new(); + let mut hcx: StableHashingContext<'a> = hcx.clone(); + self.hash_stable(&mut hcx, &mut hasher); + hasher.finish() + } +} + impl<'a, 'gcx> HashStable> for ty::subst::Kind<'gcx> { fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index cdf553d34e926..e830bcf4cdaaf 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -861,8 +861,13 @@ rustc_queries! { } Codegen { - query collect_and_partition_mono_items(_: CrateNum) - -> (Arc, Arc>>>) { + query collect_and_partition_mono_items(_: CrateNum) -> ( + Arc, + Arc>>>, + Arc>>, + Arc>>, + Arc, Arc>>>> + ) { eval_always desc { "collect_and_partition_mono_items" } } @@ -874,6 +879,24 @@ rustc_queries! { query backend_optimization_level(_: CrateNum) -> OptLevel { desc { "optimization level used by backend" } } + + // -Z call-metadata + // functions that may be called via a function pointer + query function_pointer(instance: ty::Instance<'tcx>) -> bool { + no_force + desc { "checking if `{}` may be called via a function pointer", instance } + } + + // trait methods that may be called via dynamic dispatch + query dynamic_dispatch(instance: ty::Instance<'tcx>) -> bool { + no_force + desc { "checking if `{}` may be called via dynamic dispatch", instance } + } + + query drop_glue(instance: ty::Instance<'tcx>) -> Option>>> { + no_force + desc { "checking if `{}` may be called by a trait object destructor", instance } + } } Other { diff --git a/src/librustc/session/config.rs b/src/librustc/session/config.rs index 988e4a9ff23a2..2766eaa0dbbcd 100644 --- a/src/librustc/session/config.rs +++ b/src/librustc/session/config.rs @@ -1444,6 +1444,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, the same values as the target option of the same name"), allow_features: Option> = (None, parse_opt_comma_list, [TRACKED], "only allow the listed language features to be enabled in code (space separated)"), + call_metadata: bool = (false, parse_bool, [UNTRACKED], + "adds call metadata to LLVM IR"), } pub fn default_lib_output() -> CrateType { diff --git a/src/librustc/ty/instance.rs b/src/librustc/ty/instance.rs index f54e69f352a4e..d58115840eff5 100644 --- a/src/librustc/ty/instance.rs +++ b/src/librustc/ty/instance.rs @@ -1,11 +1,12 @@ use crate::hir::Unsafety; -use crate::hir::def::Namespace; +use crate::hir::def::{Def, Namespace}; use crate::hir::def_id::DefId; -use crate::ty::{self, Ty, PolyFnSig, TypeFoldable, SubstsRef, TyCtxt}; +use crate::ty::{self, AssociatedItemContainer, Ty, PolyFnSig, TypeFoldable, SubstsRef, TyCtxt}; use crate::ty::print::{FmtPrinter, Printer}; use crate::traits; use rustc_target::spec::abi::Abi; use rustc_macros::HashStable; +use syntax::symbol::Symbol; use std::fmt; use std::iter; @@ -54,6 +55,38 @@ impl<'a, 'tcx> Instance<'tcx> { ) } + pub fn trait_ref_and_method( + &self, + tcx: TyCtxt<'a, 'tcx, 'tcx>, + ) -> Option<(ty::ExistentialTraitRef<'tcx>, Symbol)> { + let def_id = self.def_id(); + if let Some(Def::Method(..)) = tcx.describe_def(def_id) { + let item = tcx.associated_item(def_id); + + let trait_ref = match item.container { + AssociatedItemContainer::TraitContainer(trait_def_id) => { + ty::TraitRef::from_method(tcx, trait_def_id, self.substs) + } + + AssociatedItemContainer::ImplContainer(impl_def_id) => { + if let Some(trait_ref) = tcx.impl_trait_ref(impl_def_id) { + trait_ref + } else { + // inherent method + return None; + } + } + }; + + Some(( + ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref), + item.ident.name, + )) + } else { + None + } + } + fn fn_sig_noadjust(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>) -> PolyFnSig<'tcx> { let ty = self.ty(tcx); match ty.sty { diff --git a/src/librustc_codegen_llvm/Cargo.toml b/src/librustc_codegen_llvm/Cargo.toml index 841cf98164eb4..4e6738a35e8c6 100644 --- a/src/librustc_codegen_llvm/Cargo.toml +++ b/src/librustc_codegen_llvm/Cargo.toml @@ -16,6 +16,7 @@ num_cpus = "1.0" rustc-demangle = "0.1.4" rustc_llvm = { path = "../librustc_llvm" } memmap = "0.6" +smallvec = { version = "0.6.7", features = ["union", "may_dangle"] } [features] # This is used to convince Cargo to separately cache builds of `rustc_codegen_llvm` diff --git a/src/librustc_codegen_llvm/abi.rs b/src/librustc_codegen_llvm/abi.rs index 348616790b0bc..164ebeac86f7f 100644 --- a/src/librustc_codegen_llvm/abi.rs +++ b/src/librustc_codegen_llvm/abi.rs @@ -1,10 +1,11 @@ use crate::llvm::{self, AttributePlace}; use crate::builder::Builder; +use crate::common; use crate::context::CodegenCx; use crate::type_::Type; use crate::type_of::{LayoutLlvmExt, PointerKind}; use crate::value::Value; -use rustc_codegen_ssa::MemFlags; +use rustc_codegen_ssa::{CallKind, MemFlags}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::mir::operand::OperandValue; use rustc_target::abi::call::ArgType; @@ -856,9 +857,27 @@ impl AbiBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { fn apply_attrs_callsite( &mut self, ty: &FnType<'tcx, Ty<'tcx>>, - callsite: Self::Value + callsite: Self::Value, + call_kind: CallKind<'tcx>, ) { - ty.apply_attrs_callsite(self, callsite) + ty.apply_attrs_callsite(self, callsite); + + match call_kind { + // no extra metadata + CallKind::Direct => {} + + CallKind::FnPtr => { + common::add_callsite_fn_metadata(self.cx, ty, callsite) + } + + CallKind::DynamicDispatch(trait_ref, method) => { + common::add_callsite_dyn_metadata(self.cx, trait_ref, method, callsite) + } + + CallKind::DropGlue(trait_ref) => { + common::add_callsite_drop_metadata(self.cx, trait_ref, callsite) + } + } } fn get_param(&self, index: usize) -> Self::Value { diff --git a/src/librustc_codegen_llvm/callee.rs b/src/librustc_codegen_llvm/callee.rs index 2d732adcb9138..68f51b344b9d6 100644 --- a/src/librustc_codegen_llvm/callee.rs +++ b/src/librustc_codegen_llvm/callee.rs @@ -5,6 +5,7 @@ //! closure. use crate::attributes; +use crate::common; use crate::llvm; use crate::monomorphize::Instance; use crate::context::CodegenCx; @@ -16,7 +17,7 @@ use rustc::ty::layout::{LayoutOf, HasTyCtxt}; /// Codegens a reference to a fn/method item, monomorphizing and /// inlining as it goes. -/// +// /// # Parameters /// /// - `cx`: the crate context @@ -188,5 +189,11 @@ pub fn get_fn( cx.instances.borrow_mut().insert(instance, llfn); + common::add_define_metadata( + cx, + instance, + llfn, + ); + llfn } diff --git a/src/librustc_codegen_llvm/common.rs b/src/librustc_codegen_llvm/common.rs index 9554e54e4142a..89b01bfb726ef 100644 --- a/src/librustc_codegen_llvm/common.rs +++ b/src/librustc_codegen_llvm/common.rs @@ -2,7 +2,7 @@ //! Code that is useful in various codegen modules. -use crate::llvm::{self, True, False, Bool, BasicBlock, OperandBundleDef}; +use crate::llvm::{self, True, False, Bool, BasicBlock, Metadata, OperandBundleDef}; use crate::abi; use crate::consts; use crate::type_::Type; @@ -11,13 +11,20 @@ use crate::value::Value; use rustc_codegen_ssa::traits::*; use crate::consts::const_alloc_to_llvm; +use rustc_data_structures::fx::FxHashSet; +use rustc::ty; +use rustc::ty::{ExistentialTraitRef, FnSig, Ty, Instance}; use rustc::ty::layout::{HasDataLayout, LayoutOf, self, TyLayout, Size}; +use rustc::ty::layout::call::FnType; use rustc::mir::interpret::{Scalar, AllocKind, Allocation}; use rustc_codegen_ssa::mir::place::PlaceRef; +use std::ffi::CString; + use libc::{c_uint, c_char}; +use smallvec::SmallVec; -use syntax::symbol::LocalInternedString; +use syntax::symbol::{LocalInternedString, Symbol}; use syntax::ast::Mutability; pub use crate::context::CodegenCx; @@ -393,3 +400,156 @@ pub fn struct_in_context( fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 { ((hi as u128) << 64) | (lo as u128) } + +pub fn add_define_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + llfn: &'ll Value) { + if cx.sess().opts.debugging_opts.call_metadata { + if let Some(trait_refs) = cx.tcx.drop_glue(instance) { + add_define_drop_metadata(cx, &trait_refs, llfn); + } else { + let dyn_ = if cx.tcx.dynamic_dispatch(instance) { + Some(instance.trait_ref_and_method(cx.tcx).unwrap_or_else(|| { + bug!("`{}` marked for dynamic dispatch but it's not a trait method", instance) + })) + } else { + None + }; + + let fn_ = if cx.tcx.function_pointer(instance) { + let sig = cx.tcx.normalize_erasing_late_bound_regions( + ty::ParamEnv::reveal_all(), + &instance.fn_sig(cx.tcx), + ); + Some(sig) + } else { + None + }; + + if dyn_.is_some() || fn_.is_some() { + add_define_dyn_fn_metadata(cx, dyn_, fn_, llfn); + } + } + } +} + +fn add_define_dyn_fn_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, + dyn_: Option<(ExistentialTraitRef<'tcx>, Symbol)>, + fn_: Option>, + llfn: &'ll Value) { + let mut meta: SmallVec<[&Metadata; 8]> = SmallVec::new(); + + unsafe { + if let Some((trait_ref, method)) = dyn_ { + let trait_ref = CString::new(trait_ref.to_string()).unwrap(); + let method = CString::new(method.to_string()).unwrap(); + + meta.push(llvm::LLVMRustCreateMDString(cx.llcx, const_cstr!("dyn").as_ptr())); + meta.push(llvm::LLVMRustCreateMDString(cx.llcx, trait_ref.as_ptr())); + meta.push(llvm::LLVMRustCreateMDString(cx.llcx, method.as_ptr())); + } + + if let Some(sig) = fn_ { + let fn_ = CString::new(sig_to_string(sig.inputs(), sig.output())).unwrap(); + meta.push(llvm::LLVMRustCreateMDString(cx.llcx, const_cstr!("fn").as_ptr())); + meta.push(llvm::LLVMRustCreateMDString(cx.llcx, fn_.as_ptr())); + } + + let tuple = llvm::LLVMRustCreateMDTuple(cx.llcx, meta.as_ptr(), meta.len() as _); + + llvm::LLVMRustAddFunctionMetadata(llfn, tuple); + } +} + +fn add_define_drop_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, + trait_refs: &FxHashSet>, + llfn: &'ll Value) { + unsafe { + let scope = llvm::LLVMRustCreateMDString(cx.llcx, const_cstr!("drop").as_ptr()); + + let mut tuple = vec![scope]; + + for trait_ref in trait_refs { + let trait_ref = CString::new(trait_ref.to_string()).unwrap(); + let trait_ref = llvm::LLVMRustCreateMDString(cx.llcx, trait_ref.as_ptr()); + tuple.push(trait_ref); + } + + let tuple = llvm::LLVMRustCreateMDTuple(cx.llcx, tuple.as_ptr(), tuple.len() as _); + + llvm::LLVMRustAddFunctionMetadata(llfn, tuple); + } +} + +pub fn add_callsite_dyn_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, + trait_ref: ExistentialTraitRef<'tcx>, + method: Symbol, + llfn: &'ll Value) { + let trait_ref = CString::new(trait_ref.to_string()).unwrap(); + let method = CString::new(method.to_string()).unwrap(); + + unsafe { + let trait_ref = llvm::LLVMRustCreateMDString(cx.llcx, trait_ref.as_ptr()); + let method = llvm::LLVMRustCreateMDString(cx.llcx, method.as_ptr()); + let scope = llvm::LLVMRustCreateMDString(cx.llcx, const_cstr!("dyn").as_ptr()); + + let tuple = [scope, trait_ref, method]; + let tuple = llvm::LLVMRustCreateMDTuple(cx.llcx, tuple.as_ptr(), tuple.len() as _); + + llvm::LLVMRustAddInstructionMetadata(llfn, tuple); + } +} + +pub fn add_callsite_drop_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, + trait_ref: ExistentialTraitRef<'tcx>, + llfn: &'ll Value) { + let trait_ref = CString::new(trait_ref.to_string()).unwrap(); + + unsafe { + let scope = llvm::LLVMRustCreateMDString(cx.llcx, const_cstr!("drop").as_ptr()); + let trait_ref = llvm::LLVMRustCreateMDString(cx.llcx, trait_ref.as_ptr()); + + let tuple = [scope, trait_ref]; + + let tuple = llvm::LLVMRustCreateMDTuple(cx.llcx, tuple.as_ptr(), tuple.len() as _); + + llvm::LLVMRustAddInstructionMetadata(llfn, tuple); + } +} + +pub fn add_callsite_fn_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, + sig: &FnType<'tcx, Ty<'tcx>>, + llfn: &'ll Value) { + let args = sig.args.iter().map(|arg| arg.layout.ty).collect::>(); + let ret = sig.ret.layout.ty; + let fnty = CString::new(sig_to_string(&args, ret)).unwrap(); + + unsafe { + let scope = llvm::LLVMRustCreateMDString(cx.llcx, const_cstr!("fn").as_ptr()); + let id = llvm::LLVMRustCreateMDString(cx.llcx, fnty.as_ptr()); + let array = [scope, id]; + let tuple = llvm::LLVMRustCreateMDTuple(cx.llcx, array.as_ptr(), array.len() as _); + + llvm::LLVMRustAddInstructionMetadata(llfn, tuple); + } +} + +pub fn sig_to_string<'tcx>(args: &[Ty<'tcx>], ret: Ty<'tcx>) -> String { + use std::fmt::Write; + + let mut sig = "fn(".to_owned(); + let mut is_first = true; + for arg in args { + if is_first { + is_first = false; + } else { + sig.push_str(", "); + } + write!(sig, "{}", arg).ok(); + } + sig.push(')'); + + write!(sig, " -> {}", ret).ok(); + + sig +} diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs index f6ee8bec16a5a..941738cfa7a83 100644 --- a/src/librustc_codegen_llvm/llvm/ffi.rs +++ b/src/librustc_codegen_llvm/llvm/ffi.rs @@ -1820,4 +1820,10 @@ extern "C" { bytecode: *const c_char, bytecode_len: usize) -> bool; pub fn LLVMRustLinkerFree(linker: &'a mut Linker<'a>); + pub fn LLVMRustCreateMDString(C: &'a Context, S: *const c_char) -> &'a Metadata; + pub fn LLVMRustCreateMDTuple(C: &'a Context, + Ptr: *const &'a Metadata, + Count: c_uint) -> &'a Metadata; + pub fn LLVMRustAddFunctionMetadata(Fn: &Value, Metadata: &Metadata); + pub fn LLVMRustAddInstructionMetadata(Instr: &Value, Metadata: &Metadata); } diff --git a/src/librustc_codegen_llvm/mono_item.rs b/src/librustc_codegen_llvm/mono_item.rs index 7f0cdb9f58008..b295bffe4d056 100644 --- a/src/librustc_codegen_llvm/mono_item.rs +++ b/src/librustc_codegen_llvm/mono_item.rs @@ -1,5 +1,6 @@ use crate::attributes; use crate::base; +use crate::common; use crate::context::CodegenCx; use crate::llvm; use crate::monomorphize::Instance; @@ -44,8 +45,11 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> { !instance.substs.has_param_types()); let mono_sig = instance.fn_sig(self.tcx()); + let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); let lldecl = self.declare_fn(symbol_name, mono_sig); + common::add_define_metadata(self, instance, lldecl); + unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; base::set_link_section(lldecl, &attrs); if linkage == Linkage::LinkOnceODR || diff --git a/src/librustc_codegen_ssa/back/symbol_export.rs b/src/librustc_codegen_ssa/back/symbol_export.rs index 336f41b784a81..46e25909525fa 100644 --- a/src/librustc_codegen_ssa/back/symbol_export.rs +++ b/src/librustc_codegen_ssa/back/symbol_export.rs @@ -243,7 +243,7 @@ fn exported_symbols_provider_local<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, let need_visibility = tcx.sess.target.target.options.dynamic_linking && !tcx.sess.target.target.options.only_cdylib; - let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); + let (_, cgus, _, _, _) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); for (mono_item, &(linkage, visibility)) in cgus.iter() .flat_map(|cgu| cgu.items().iter()) { diff --git a/src/librustc_codegen_ssa/base.rs b/src/librustc_codegen_ssa/base.rs index 27e3e30669905..b5b40f13258ca 100644 --- a/src/librustc_codegen_ssa/base.rs +++ b/src/librustc_codegen_ssa/base.rs @@ -854,7 +854,7 @@ impl CrateInfo { } fn is_codegened_item(tcx: TyCtxt<'_, '_, '_>, id: DefId) -> bool { - let (all_mono_items, _) = + let (all_mono_items, _, _, _, _) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); all_mono_items.contains(&id) } @@ -879,7 +879,7 @@ pub fn provide_both(providers: &mut Providers<'_>) { config::OptLevel::SizeMin => config::OptLevel::Default, }; - let (defids, _) = tcx.collect_and_partition_mono_items(cratenum); + let (defids, _, _, _, _) = tcx.collect_and_partition_mono_items(cratenum); for id in &*defids { let hir::CodegenFnAttrs { optimize, .. } = tcx.codegen_fn_attrs(*id); match optimize { diff --git a/src/librustc_codegen_ssa/lib.rs b/src/librustc_codegen_ssa/lib.rs index e2917578c0ece..6e08edd605527 100644 --- a/src/librustc_codegen_ssa/lib.rs +++ b/src/librustc_codegen_ssa/lib.rs @@ -31,6 +31,7 @@ use std::path::PathBuf; use rustc::dep_graph::WorkProduct; use rustc::session::config::{OutputFilenames, OutputType}; +use rustc::ty::ExistentialTraitRef; use rustc::middle::lang_items::LangItem; use rustc::hir::def_id::CrateNum; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; @@ -122,6 +123,14 @@ pub enum ModuleKind { Allocator, } +#[derive(Copy, Clone, Debug, PartialEq)] +pub enum CallKind<'tcx> { + Direct, + DropGlue(ExistentialTraitRef<'tcx>), + DynamicDispatch(ExistentialTraitRef<'tcx>, Symbol), + FnPtr, +} + bitflags::bitflags! { pub struct MemFlags: u8 { const VOLATILE = 1 << 0; diff --git a/src/librustc_codegen_ssa/mir/block.rs b/src/librustc_codegen_ssa/mir/block.rs index e64c847db651b..5b7b41d295d10 100644 --- a/src/librustc_codegen_ssa/mir/block.rs +++ b/src/librustc_codegen_ssa/mir/block.rs @@ -1,5 +1,5 @@ use rustc::middle::lang_items; -use rustc::ty::{self, Ty, TypeFoldable}; +use rustc::ty::{self, Ty, TypeFoldable, InstanceDef}; use rustc::ty::layout::{self, LayoutOf, HasTyCtxt}; use rustc::mir::{self, Place, PlaceBase, Static, StaticKind}; use rustc::mir::interpret::InterpError; @@ -7,7 +7,7 @@ use rustc_target::abi::call::{ArgType, FnType, PassMode, IgnoreMode}; use rustc_target::spec::abi::Abi; use rustc_mir::monomorphize; use crate::base; -use crate::MemFlags; +use crate::{CallKind, MemFlags}; use crate::common::{self, IntPredicate}; use crate::meth; @@ -111,6 +111,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'a, 'tcx> { llargs: &[Bx::Value], destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>, cleanup: Option, + kind: CallKind<'tcx>, ) { if let Some(cleanup) = cleanup { let ret_bx = if let Some((_, target)) = destination { @@ -123,7 +124,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'a, 'tcx> { ret_bx, self.llblock(fx, cleanup), self.funclet(fx)); - bx.apply_attrs_callsite(&fn_ty, invokeret); + bx.apply_attrs_callsite(&fn_ty, invokeret, kind); if let Some((ret_dest, target)) = destination { let mut ret_bx = fx.build_block(target); @@ -132,7 +133,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'a, 'tcx> { } } else { let llret = bx.call(fn_ptr, &llargs, self.funclet(fx)); - bx.apply_attrs_callsite(&fn_ty, llret); + bx.apply_attrs_callsite(&fn_ty, llret, kind); if fx.mir[*self.bb].is_cleanup { // Cleanup is always the cold path. Don't inline // drop glue. Also, when there is a deeply-nested @@ -327,8 +328,12 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { args1 = [place.llval]; &args1[..] }; - let (drop_fn, fn_ty) = match ty.sty { - ty::Dynamic(..) => { + let (drop_fn, fn_ty, call_kind) = match ty.sty { + ty::Dynamic(ref trait_data, _) => { + let principal = self.cx.tcx().normalize_erasing_late_bound_regions( + ty::ParamEnv::reveal_all(), + &trait_data.principal().unwrap_or_else(|| bug!("trait object has no principal")), + ); let sig = drop_fn.fn_sig(self.cx.tcx()); let sig = self.cx.tcx().normalize_erasing_late_bound_regions( ty::ParamEnv::reveal_all(), @@ -337,16 +342,19 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let fn_ty = bx.new_vtable(sig, &[]); let vtable = args[1]; args = &args[..1]; - (meth::DESTRUCTOR.get_fn(&mut bx, vtable, &fn_ty), fn_ty) + (meth::DESTRUCTOR.get_fn(&mut bx, vtable, &fn_ty), + fn_ty, + CallKind::DropGlue(principal)) } _ => { (bx.get_fn(drop_fn), - bx.fn_type_of_instance(&drop_fn)) + bx.fn_type_of_instance(&drop_fn), + CallKind::Direct) } }; helper.do_call(self, &mut bx, fn_ty, drop_fn, args, Some((ReturnDest::Nothing, target)), - unwind); + unwind, call_kind); } fn codegen_assert_terminator<'b>( @@ -443,7 +451,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let llfn = bx.get_fn(instance); // Codegen the actual panic invoke/call. - helper.do_call(self, &mut bx, fn_ty, llfn, &args, None, cleanup); + helper.do_call(self, &mut bx, fn_ty, llfn, &args, None, cleanup, CallKind::Direct); } fn codegen_call_terminator<'b>( @@ -460,16 +468,26 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar. let callee = self.codegen_operand(&mut bx, func); - let (instance, mut llfn) = match callee.layout.ty.sty { + let (instance, mut llfn, call_kind) = match callee.layout.ty.sty { ty::FnDef(def_id, substs) => { - (Some(ty::Instance::resolve(bx.tcx(), - ty::ParamEnv::reveal_all(), - def_id, - substs).unwrap()), - None) + let instance = ty::Instance::resolve(bx.tcx(), + ty::ParamEnv::reveal_all(), + def_id, + substs).unwrap(); + let call_kind = if let InstanceDef::Virtual(..) = instance.def { + if let Some((trait_ref, method)) = instance.trait_ref_and_method(bx.tcx()) { + CallKind::DynamicDispatch(trait_ref, method) + } else { + bug!("virtual instance is not a trait method {:?}", instance); + } + } else { + CallKind::Direct + }; + + (Some(instance), None, call_kind) } ty::FnPtr(_) => { - (None, Some(callee.immediate())) + (None, Some(callee.immediate()), CallKind::FnPtr) } _ => bug!("{} is not callable", callee.layout.ty), }; @@ -568,6 +586,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &[msg_file_line_col], destination.as_ref().map(|(_, bb)| (ReturnDest::Nothing, *bb)), cleanup, + CallKind::Direct, ); } else { // a NOP @@ -713,7 +732,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let mut op = self.codegen_operand(&mut bx, arg); - if let (0, Some(ty::InstanceDef::Virtual(_, idx))) = (i, def) { + if let (0, Some(ty::InstanceDef::Virtual(_method_def_id, idx))) = (i, def) { if let Pair(..) = op.val { // In the case of Rc, we need to explicitly pass a // *mut RcBox with a Scalar (not ScalarPair) ABI. This is a hack @@ -788,7 +807,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { helper.do_call(self, &mut bx, fn_ty, fn_ptr, &llargs, destination.as_ref().map(|&(_, target)| (ret_dest, target)), - cleanup); + cleanup, call_kind); } } diff --git a/src/librustc_codegen_ssa/traits/abi.rs b/src/librustc_codegen_ssa/traits/abi.rs index a8fd4e1d2c7c7..66ee073890a67 100644 --- a/src/librustc_codegen_ssa/traits/abi.rs +++ b/src/librustc_codegen_ssa/traits/abi.rs @@ -1,3 +1,4 @@ +use crate::CallKind; use super::BackendTypes; use rustc::ty::{FnSig, Instance, Ty}; use rustc_target::abi::call::FnType; @@ -9,6 +10,9 @@ pub trait AbiMethods<'tcx> { } pub trait AbiBuilderMethods<'tcx>: BackendTypes { - fn apply_attrs_callsite(&mut self, ty: &FnType<'tcx, Ty<'tcx>>, callsite: Self::Value); + fn apply_attrs_callsite(&mut self, + ty: &FnType<'tcx, Ty<'tcx>>, + callsite: Self::Value, + kind: CallKind<'tcx>); fn get_param(&self, index: usize) -> Self::Value; } diff --git a/src/librustc_data_structures/stable_hasher.rs b/src/librustc_data_structures/stable_hasher.rs index 19343a9250df3..c8f8e07726551 100644 --- a/src/librustc_data_structures/stable_hasher.rs +++ b/src/librustc_data_structures/stable_hasher.rs @@ -298,6 +298,25 @@ impl HashStable for (T1, T2, T3, T4) } } +impl HashStable for (T1, T2, T3, T4, T5) + where T1: HashStable, + T2: HashStable, + T3: HashStable, + T4: HashStable, + T5: HashStable, +{ + fn hash_stable(&self, + ctx: &mut CTX, + hasher: &mut StableHasher) { + let (ref _0, ref _1, ref _2, ref _3, ref _4) = *self; + _0.hash_stable(ctx, hasher); + _1.hash_stable(ctx, hasher); + _2.hash_stable(ctx, hasher); + _3.hash_stable(ctx, hasher); + _4.hash_stable(ctx, hasher); + } +} + impl, CTX> HashStable for [T] { default fn hash_stable(&self, ctx: &mut CTX, diff --git a/src/librustc_mir/monomorphize/collector.rs b/src/librustc_mir/monomorphize/collector.rs index af875c2f9e8a1..be9a462201dfb 100644 --- a/src/librustc_mir/monomorphize/collector.rs +++ b/src/librustc_mir/monomorphize/collector.rs @@ -181,7 +181,9 @@ use rustc::hir::def_id::{DefId, LOCAL_CRATE}; use rustc::mir::interpret::{AllocId, ConstValue}; use rustc::middle::lang_items::{ExchangeMallocFnLangItem, StartFnLangItem}; use rustc::ty::subst::{InternalSubsts, SubstsRef}; -use rustc::ty::{self, TypeFoldable, Ty, TyCtxt, GenericParamDefKind}; +use rustc::ty::{ + self, GenericParamDefKind, InstanceDef, Ty, TyCtxt, TypeFoldable, ExistentialTraitRef, +}; use rustc::ty::adjustment::CustomCoerceUnsized; use rustc::session::config::EntryFnType; use rustc::mir::{self, Location, Place, PlaceBase, Promoted, Static, StaticKind}; @@ -199,6 +201,7 @@ use rustc_data_structures::bit_set::GrowableBitSet; use rustc_data_structures::sync::{MTRef, MTLock, ParallelIterator, par_iter}; use std::iter; +use std::sync::Arc; #[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)] pub enum MonoItemCollectionMode { @@ -282,21 +285,36 @@ impl<'tcx> InliningMap<'tcx> { } } -pub fn collect_crate_mono_items<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - mode: MonoItemCollectionMode) - -> (FxHashSet>, - InliningMap<'tcx>) { - let roots = time(tcx.sess, "collecting roots", || { - collect_roots(tcx, mode) - }); +pub fn collect_crate_mono_items<'a, 'tcx>( + tcx: TyCtxt<'a, 'tcx, 'tcx>, + mode: MonoItemCollectionMode, +) -> ( + FxHashSet>, + InliningMap<'tcx>, + FxHashSet>, + FxHashSet>, + FxHashMap, Arc>>>, +) { + let (roots, function_pointer, dynamic_dispatch, drop_glue) = time( + tcx.sess, + "collecting roots", + || { + collect_roots(tcx, mode) + }); debug!("Building mono item graph, beginning at roots"); let mut visited = MTLock::new(FxHashSet::default()); + let mut function_pointer = MTLock::new(function_pointer); + let mut dynamic_dispatch = MTLock::new(dynamic_dispatch); + let mut drop_glue = MTLock::new(drop_glue); let mut inlining_map = MTLock::new(InliningMap::new()); { let visited: MTRef<'_, _> = &mut visited; + let function_pointer: MTRef<'_, _> = &mut function_pointer; + let dynamic_dispatch: MTRef<'_, _> = &mut dynamic_dispatch; + let drop_glue: MTRef<'_, _> = &mut drop_glue; let inlining_map: MTRef<'_, _> = &mut inlining_map; time(tcx.sess, "collecting mono items", || { @@ -305,22 +323,39 @@ pub fn collect_crate_mono_items<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, collect_items_rec(tcx, root, visited, + function_pointer, + dynamic_dispatch, + drop_glue, &mut recursion_depths, inlining_map); }); }); } - (visited.into_inner(), inlining_map.into_inner()) + ( + visited.into_inner(), + inlining_map.into_inner(), + function_pointer.into_inner(), + dynamic_dispatch.into_inner(), + drop_glue.into_inner().into_iter().map(|(k, v)| (k, Arc::new(v))).collect(), + ) } // Find all non-generic items by walking the HIR. These items serve as roots to // start monomorphizing from. -fn collect_roots<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - mode: MonoItemCollectionMode) - -> Vec> { - debug!("Collecting roots"); +fn collect_roots<'a, 'tcx>( + tcx: TyCtxt<'a, 'tcx, 'tcx>, + mode: MonoItemCollectionMode, +) -> ( + Vec>, + FxHashSet>, + FxHashSet>, + FxHashMap, FxHashSet>>, +) { let mut roots = Vec::new(); + let mut function_pointer = FxHashSet::default(); + let mut dynamic_dispatch = FxHashSet::default(); + let mut drop_glue = FxHashMap::default(); { let entry_fn = tcx.entry_fn(LOCAL_CRATE); @@ -332,6 +367,9 @@ fn collect_roots<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, mode, entry_fn, output: &mut roots, + function_pointer: &mut function_pointer, + dynamic_dispatch: &mut dynamic_dispatch, + drop_glue: &mut drop_glue, }; tcx.hir().krate().visit_all_item_likes(&mut visitor); @@ -344,15 +382,20 @@ fn collect_roots<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, // can't actually be used, so we can just skip codegenning them. roots.retain(|root| root.is_instantiable(tcx)); - roots + (roots, function_pointer, dynamic_dispatch, drop_glue) } // Collect all monomorphized items reachable from `starting_point` -fn collect_items_rec<'a, 'tcx: 'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - starting_point: MonoItem<'tcx>, - visited: MTRef<'_, MTLock>>>, - recursion_depths: &mut DefIdMap, - inlining_map: MTRef<'_, MTLock>>) { +fn collect_items_rec<'a, 'tcx: 'a>( + tcx: TyCtxt<'a, 'tcx, 'tcx>, + starting_point: MonoItem<'tcx>, + visited: MTRef<'_, MTLock>>>, + function_pointer: MTRef<'_, MTLock>>>, + dynamic_dispatch: MTRef<'_, MTLock>>>, + drop_glue: MTRef<'_, MTLock, FxHashSet>>>>, + recursion_depths: &mut DefIdMap, + inlining_map: MTRef<'_, MTLock>>, +) { if !visited.lock_mut().insert(starting_point.clone()) { // We've been here already, no need to search again. return; @@ -360,6 +403,9 @@ fn collect_items_rec<'a, 'tcx: 'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>, debug!("BEGIN collect_items_rec({})", starting_point.to_string(tcx, true)); let mut neighbors = Vec::new(); + let mut function_pointer_ = FxHashSet::default(); + let mut dynamic_dispatch_ = FxHashSet::default(); + let mut drop_glue_ = FxHashMap::default(); let recursion_depth_reset; match starting_point { @@ -381,7 +427,15 @@ fn collect_items_rec<'a, 'tcx: 'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>, let param_env = ty::ParamEnv::reveal_all(); if let Ok(val) = tcx.const_eval(param_env.and(cid)) { - collect_const(tcx, val, InternalSubsts::empty(), &mut neighbors); + collect_const( + tcx, + val, + InternalSubsts::empty(), + &mut neighbors, + &mut function_pointer_, + &mut dynamic_dispatch_, + &mut drop_glue_, + ); } } MonoItem::Fn(instance) => { @@ -394,7 +448,14 @@ fn collect_items_rec<'a, 'tcx: 'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>, recursion_depths)); check_type_length_limit(tcx, instance); - collect_neighbours(tcx, instance, &mut neighbors); + collect_neighbours( + tcx, + instance, + &mut neighbors, + &mut function_pointer_, + &mut dynamic_dispatch_, + &mut drop_glue_, + ); } MonoItem::GlobalAsm(..) => { recursion_depth_reset = None; @@ -403,8 +464,26 @@ fn collect_items_rec<'a, 'tcx: 'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>, record_accesses(tcx, starting_point, &neighbors[..], inlining_map); + function_pointer.lock_mut().extend(function_pointer_); + dynamic_dispatch.lock_mut().extend(dynamic_dispatch_); + { + let drop_glue = drop_glue.lock_mut(); + for (k, v) in drop_glue_ { + drop_glue.entry(k).or_default().extend(v); + } + } + for neighbour in neighbors { - collect_items_rec(tcx, neighbour, visited, recursion_depths, inlining_map); + collect_items_rec( + tcx, + neighbour, + visited, + function_pointer, + dynamic_dispatch, + drop_glue, + recursion_depths, + inlining_map, + ); } if let Some((def_id, depth)) = recursion_depth_reset { @@ -517,6 +596,9 @@ struct MirNeighborCollector<'a, 'tcx: 'a> { tcx: TyCtxt<'a, 'tcx, 'tcx>, mir: &'a mir::Mir<'tcx>, output: &'a mut Vec>, + function_pointer: &'a mut FxHashSet>, + dynamic_dispatch: &'a mut FxHashSet>, + drop_glue: &'a mut FxHashMap, FxHashSet>>, param_substs: SubstsRef<'tcx>, } @@ -551,7 +633,9 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> { create_mono_items_for_vtable_methods(self.tcx, target_ty, source_ty, - self.output); + self.output, + self.dynamic_dispatch, + self.drop_glue); } } mir::Rvalue::Cast(mir::CastKind::ReifyFnPointer, ref operand, _) => { @@ -561,7 +645,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> { ty::ParamEnv::reveal_all(), &fn_ty, ); - visit_fn_use(self.tcx, fn_ty, false, &mut self.output); + visit_fn_use(self.tcx, fn_ty, false, &mut self.output, &mut self.function_pointer); } mir::Rvalue::Cast(mir::CastKind::ClosureFnPointer(_), ref operand, _) => { let source_ty = operand.ty(self.mir, self.tcx); @@ -601,7 +685,15 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> { fn visit_const(&mut self, constant: &&'tcx ty::Const<'tcx>, location: Location) { debug!("visiting const {:?} @ {:?}", *constant, location); - collect_const(self.tcx, **constant, self.param_substs, self.output); + collect_const( + self.tcx, + **constant, + self.param_substs, + self.output, + self.function_pointer, + self.dynamic_dispatch, + self.drop_glue, + ); self.super_const(constant); } @@ -621,7 +713,13 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> { ty::ParamEnv::reveal_all(), &callee_ty, ); - visit_fn_use(self.tcx, callee_ty, true, &mut self.output); + visit_fn_use( + self.tcx, + callee_ty, + true, + &mut self.output, + &mut self.function_pointer, + ); } mir::TerminatorKind::Drop { ref location, .. } | mir::TerminatorKind::DropAndReplace { ref location, .. } => { @@ -631,7 +729,12 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> { ty::ParamEnv::reveal_all(), &ty, ); - visit_drop_use(self.tcx, ty, true, self.output); + visit_drop_use( + self.tcx, + ty, + true, + self.output, + ); } mir::TerminatorKind::Goto { .. } | mir::TerminatorKind::SwitchInt { .. } | @@ -672,25 +775,33 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> { } } -fn visit_drop_use<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - ty: Ty<'tcx>, - is_direct_call: bool, - output: &mut Vec>) -{ +fn visit_drop_use<'a, 'tcx>( + tcx: TyCtxt<'a, 'tcx, 'tcx>, + ty: Ty<'tcx>, + is_direct_call: bool, + output: &mut Vec>, +) -> Instance<'tcx> { let instance = monomorphize::resolve_drop_in_place(tcx, ty); visit_instance_use(tcx, instance, is_direct_call, output); + instance } fn visit_fn_use<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, ty: Ty<'tcx>, is_direct_call: bool, - output: &mut Vec>) + output: &mut Vec>, + function_pointer: &mut FxHashSet>) { if let ty::FnDef(def_id, substs) = ty.sty { let instance = ty::Instance::resolve(tcx, ty::ParamEnv::reveal_all(), def_id, substs).unwrap(); + + if !is_direct_call { + function_pointer.insert(instance); + } + visit_instance_use(tcx, instance, is_direct_call, output); } } @@ -907,33 +1018,53 @@ fn create_fn_mono_item<'a, 'tcx>(instance: Instance<'tcx>) -> MonoItem<'tcx> { /// Creates a `MonoItem` for each method that is referenced by the vtable for /// the given trait/impl pair. -fn create_mono_items_for_vtable_methods<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - trait_ty: Ty<'tcx>, - impl_ty: Ty<'tcx>, - output: &mut Vec>) { +fn create_mono_items_for_vtable_methods<'a, 'tcx>( + tcx: TyCtxt<'a, 'tcx, 'tcx>, + trait_ty: Ty<'tcx>, + impl_ty: Ty<'tcx>, + output: &mut Vec>, + dynamic_dispatch: &mut FxHashSet>, + drop_glue: &mut FxHashMap, FxHashSet>>, +) { assert!(!trait_ty.needs_subst() && !trait_ty.has_escaping_bound_vars() && !impl_ty.needs_subst() && !impl_ty.has_escaping_bound_vars()); - if let ty::Dynamic(ref trait_ty, ..) = trait_ty.sty { - if let Some(principal) = trait_ty.principal() { + if let ty::Dynamic(ref trait_sty, ..) = trait_ty.sty { + // add the destructor + let drop_glue_instance = visit_drop_use(tcx, impl_ty, false, output); + + if let Some(principal) = trait_sty.principal() { + let drop_glue = drop_glue.entry(drop_glue_instance).or_default(); + let poly_trait_ref = principal.with_self_ty(tcx, impl_ty); assert!(!poly_trait_ref.has_escaping_bound_vars()); // Walk all methods of the trait, including those of its supertraits let methods = tcx.vtable_methods(poly_trait_ref); - let methods = methods.iter().cloned().filter_map(|method| method) - .map(|(def_id, substs)| ty::Instance::resolve_for_vtable( - tcx, - ty::ParamEnv::reveal_all(), - def_id, - substs).unwrap()) - .filter(|&instance| should_monomorphize_locally(tcx, &instance)) - .map(|instance| create_fn_mono_item(instance)); - output.extend(methods); - } + let methods = methods + .iter() + .cloned() + .filter_map(|method| method) + .map(|(def_id, substs)| { + ty::Instance::resolve_for_vtable( + tcx, + ty::ParamEnv::reveal_all(), + def_id, + substs, + ) + .unwrap() + }) + .filter(|instance| should_monomorphize_locally(tcx, &instance)); + + for instance in methods { + if let Some((trait_ref, _)) = instance.trait_ref_and_method(tcx) { + drop_glue.insert(trait_ref); + dynamic_dispatch.insert(instance); + } - // Also add the destructor - visit_drop_use(tcx, impl_ty, false, output); + output.push(create_fn_mono_item(instance)); + } + } } } @@ -945,6 +1076,9 @@ struct RootCollector<'b, 'a: 'b, 'tcx: 'a + 'b> { tcx: TyCtxt<'a, 'tcx, 'tcx>, mode: MonoItemCollectionMode, output: &'b mut Vec>, + function_pointer: &'b mut FxHashSet>, + dynamic_dispatch: &'b mut FxHashSet>, + drop_glue: &'b mut FxHashMap, FxHashSet>>, entry_fn: Option<(DefId, EntryFnType)>, } @@ -1011,7 +1145,15 @@ impl<'b, 'a, 'v> ItemLikeVisitor<'v> for RootCollector<'b, 'a, 'v> { let param_env = ty::ParamEnv::reveal_all(); if let Ok(val) = self.tcx.const_eval(param_env.and(cid)) { - collect_const(self.tcx, val, InternalSubsts::empty(), &mut self.output); + collect_const( + self.tcx, + val, + InternalSubsts::empty(), + &mut self.output, + &mut self.function_pointer, + &mut self.dynamic_dispatch, + &mut self.drop_glue, + ); } } hir::ItemKind::Fn(..) => { @@ -1174,6 +1316,9 @@ fn collect_miri<'a, 'tcx>( tcx: TyCtxt<'a, 'tcx, 'tcx>, alloc_id: AllocId, output: &mut Vec>, + function_pointer: &mut FxHashSet>, + dynamic_dispatch: &mut FxHashSet>, + drop_glue: &mut FxHashMap, FxHashSet>>, ) { let alloc_kind = tcx.alloc_map.lock().get(alloc_id); match alloc_kind { @@ -1186,13 +1331,57 @@ fn collect_miri<'a, 'tcx>( } Some(AllocKind::Memory(alloc)) => { trace!("collecting {:?} with {:#?}", alloc_id, alloc); - for &((), inner) in alloc.relocations.values() { - collect_miri(tcx, inner, output); + + // if this contains a drop glue instance then this is a const-evaluated trait object + let drop_glue_instance = alloc.relocations.values().filter_map(|(_, inner)| { + let kind = tcx.alloc_map.lock().get(*inner); + + if let Some(AllocKind::Function(instance)) = kind { + if let InstanceDef::DropGlue(..) = instance.def { + Some(instance) + } else { + None + } + } else { + None + } + }).next(); + + if let Some(drop_glue_instance) = drop_glue_instance { + let drop_glue = drop_glue.entry(drop_glue_instance).or_default(); + + // trait object in const / static context + for ((), inner) in alloc.relocations.values() { + let kind = tcx.alloc_map.lock().get(*inner); + + if let Some(AllocKind::Function(instance)) = kind { + if should_monomorphize_locally(tcx, &instance) { + trace!("collecting {:?} with {:#?}", alloc_id, instance); + + if let Some((trait_ref, _)) = instance.trait_ref_and_method(tcx) { + dynamic_dispatch.insert(instance); + drop_glue.insert(trait_ref); + } + + output.push(create_fn_mono_item(instance)); + } + } else { + bug!("unexpected kind: {:?}", kind); + } + } + } else { + for &((), inner) in alloc.relocations.values() { + collect_miri(tcx, inner, output, function_pointer, dynamic_dispatch, drop_glue); + } } }, Some(AllocKind::Function(fn_instance)) => { if should_monomorphize_locally(tcx, &fn_instance) { trace!("collecting {:?} with {:#?}", alloc_id, fn_instance); + + // function pointer in const / static context + function_pointer.insert(fn_instance); + output.push(create_fn_mono_item(fn_instance)); } } @@ -1201,10 +1390,14 @@ fn collect_miri<'a, 'tcx>( } /// Scan the MIR in order to find function calls, closures, and drop-glue -fn collect_neighbours<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - instance: Instance<'tcx>, - output: &mut Vec>) -{ +fn collect_neighbours<'a, 'tcx>( + tcx: TyCtxt<'a, 'tcx, 'tcx>, + instance: Instance<'tcx>, + output: &mut Vec>, + function_pointer: &mut FxHashSet>, + dynamic_dispatch: &mut FxHashSet>, + drop_glue: &mut FxHashMap, FxHashSet>>, +) { let mir = tcx.instance_mir(instance.def); MirNeighborCollector { @@ -1212,6 +1405,9 @@ fn collect_neighbours<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, mir: &mir, output, param_substs: instance.substs, + function_pointer, + dynamic_dispatch, + drop_glue, }.visit_mir(&mir); let param_env = ty::ParamEnv::reveal_all(); for i in 0..mir.promoted.len() { @@ -1222,7 +1418,17 @@ fn collect_neighbours<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, promoted: Some(i), }; match tcx.const_eval(param_env.and(cid)) { - Ok(val) => collect_const(tcx, val, instance.substs, output), + Ok(val) => { + collect_const( + tcx, + val, + instance.substs, + output, + function_pointer, + dynamic_dispatch, + drop_glue, + ) + }, Err(ErrorHandled::Reported) => {}, Err(ErrorHandled::TooGeneric) => span_bug!( mir.promoted[i].span, "collection encountered polymorphic constant", @@ -1245,16 +1451,19 @@ fn collect_const<'a, 'tcx>( constant: ty::Const<'tcx>, param_substs: SubstsRef<'tcx>, output: &mut Vec>, + function_pointer: &mut FxHashSet>, + dynamic_dispatch: &mut FxHashSet>, + drop_glue: &mut FxHashMap, FxHashSet>>, ) { debug!("visiting const {:?}", constant); match constant.val { ConstValue::Slice(Scalar::Ptr(ptr), _) | ConstValue::Scalar(Scalar::Ptr(ptr)) => - collect_miri(tcx, ptr.alloc_id, output), + collect_miri(tcx, ptr.alloc_id, output, function_pointer, dynamic_dispatch, drop_glue), ConstValue::ByRef(_ptr, alloc) => { for &((), id) in alloc.relocations.values() { - collect_miri(tcx, id, output); + collect_miri(tcx, id, output, function_pointer, dynamic_dispatch, drop_glue); } } ConstValue::Unevaluated(did, substs) => { @@ -1274,7 +1483,15 @@ fn collect_const<'a, 'tcx>( promoted: None, }; match tcx.const_eval(param_env.and(cid)) { - Ok(val) => collect_const(tcx, val, param_substs, output), + Ok(val) => collect_const( + tcx, + val, + param_substs, + output, + function_pointer, + dynamic_dispatch, + drop_glue, + ), Err(ErrorHandled::Reported) => {}, Err(ErrorHandled::TooGeneric) => span_bug!( tcx.def_span(did), "collection encountered polymorphic constant", diff --git a/src/librustc_mir/monomorphize/partitioning.rs b/src/librustc_mir/monomorphize/partitioning.rs index 4a2c05b201328..ac6739bd4afd5 100644 --- a/src/librustc_mir/monomorphize/partitioning.rs +++ b/src/librustc_mir/monomorphize/partitioning.rs @@ -103,7 +103,7 @@ use rustc::hir::def_id::{CrateNum, DefId, LOCAL_CRATE, CRATE_DEF_INDEX}; use rustc::hir::map::DefPathData; use rustc::mir::mono::{Linkage, Visibility, CodegenUnitNameBuilder}; use rustc::middle::exported_symbols::SymbolExportLevel; -use rustc::ty::{self, TyCtxt, InstanceDef}; +use rustc::ty::{self, TyCtxt, InstanceDef, Instance, ExistentialTraitRef}; use rustc::ty::print::characteristic_def_id_of_type; use rustc::ty::query::Providers; use rustc::util::common::time; @@ -891,8 +891,13 @@ fn debug_dump<'a, 'b, 'tcx, I>(tcx: TyCtxt<'a, 'tcx, 'tcx>, fn collect_and_partition_mono_items<'a, 'tcx>( tcx: TyCtxt<'a, 'tcx, 'tcx>, cnum: CrateNum, -) -> (Arc, Arc>>>) -{ +) -> ( + Arc, + Arc>>>, + Arc>>, + Arc>>, + Arc, Arc>>>>, +) { assert_eq!(cnum, LOCAL_CRATE); let collection_mode = match tcx.sess.opts.debugging_opts.print_mono_items { @@ -921,7 +926,7 @@ fn collect_and_partition_mono_items<'a, 'tcx>( } }; - let (items, inlining_map) = + let (items, inlining_map, function_pointer, dynamic_dispatch, drop_glue) = time(tcx.sess, "monomorphization collection", || { collector::collect_crate_mono_items(tcx, collection_mode) }); @@ -1009,21 +1014,45 @@ fn collect_and_partition_mono_items<'a, 'tcx>( } } - (Arc::new(mono_items), Arc::new(codegen_units)) + ( + Arc::new(mono_items), + Arc::new(codegen_units), + Arc::new(function_pointer), + Arc::new(dynamic_dispatch), + Arc::new(drop_glue), + ) } pub fn provide(providers: &mut Providers<'_>) { providers.collect_and_partition_mono_items = collect_and_partition_mono_items; + providers.function_pointer = |tcx, instance| { + let (_, _, function_pointer, _, _) = + tcx.collect_and_partition_mono_items(LOCAL_CRATE); + function_pointer.contains(&instance) + }; + + providers.dynamic_dispatch = |tcx, instance| { + let (_, _, _, dynamic_dispatch, _) = + tcx.collect_and_partition_mono_items(LOCAL_CRATE); + dynamic_dispatch.contains(&instance) + }; + + providers.drop_glue = |tcx, instance| { + let (_, _, _, _, drop_glue) = + tcx.collect_and_partition_mono_items(LOCAL_CRATE); + drop_glue.get(&instance).cloned() + }; + providers.is_codegened_item = |tcx, def_id| { - let (all_mono_items, _) = + let (all_mono_items, _, _, _, _) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); all_mono_items.contains(&def_id) }; providers.codegen_unit = |tcx, name| { - let (_, all) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); + let (_, all, _, _, _) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); all.iter() .find(|cgu| *cgu.name() == name) .cloned() diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp index 437e2d482efd6..1870579d91edd 100644 --- a/src/rustllvm/RustWrapper.cpp +++ b/src/rustllvm/RustWrapper.cpp @@ -1550,3 +1550,29 @@ extern "C" LLVMValueRef LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS),unwrap(RHS))); } + +extern "C" LLVMMetadataRef +LLVMRustCreateMDString(LLVMContextRef ContextRef, const char* S) { + LLVMContext &Context = *unwrap(ContextRef); + return wrap(MDString::get(Context, S)); +} + +extern "C" LLVMMetadataRef +LLVMRustCreateMDTuple(LLVMContextRef ContextRef, + LLVMMetadataRef *Ptr, + unsigned Count) { + LLVMContext &Context = *unwrap(ContextRef); + return wrap(MDTuple::get(Context, makeArrayRef(unwrap(Ptr), Count))); +} + +extern "C" void +LLVMRustAddFunctionMetadata(LLVMValueRef Fn, LLVMMetadataRef Metadata) { + Function* F = unwrap(Fn); + F->setMetadata("rust", unwrap(Metadata)); +} + +extern "C" void +LLVMRustAddInstructionMetadata(LLVMValueRef Instr, LLVMMetadataRef Metadata) { + Instruction* I = unwrap(Instr); + I->setMetadata("rust", unwrap(Metadata)); +}