Single commit with all changes

Cargo.lock

@@ -4121,6 +4121,7 @@ dependencies = [
 name = "rustc_monomorphize"
 version = "0.0.0"
 dependencies = [
+ "rustc_ast",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_fluent_macro",
@@ -4129,6 +4130,7 @@ dependencies = [
  "rustc_middle",
  "rustc_session",
  "rustc_span",
+ "rustc_symbol_mangling",
  "rustc_target",
  "serde",
  "serde_json",

compiler/rustc_builtin_macros/src/errors.rs

@@ -217,6 +217,60 @@ mod ad_fallback {
     }
 }
 
+#[derive(Diagnostic)]
+#[diag(builtin_macros_autodiff_unknown_activity)]
+pub(crate) struct AutoDiffUnknownActivity {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) act: String,
+}
+#[derive(Diagnostic)]
+#[diag(builtin_macros_autodiff_ty_activity)]
+pub(crate) struct AutoDiffInvalidTypeForActivity {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) act: String,
+}
+#[derive(Diagnostic)]
+#[diag(builtin_macros_autodiff_number_activities)]
+pub(crate) struct AutoDiffInvalidNumberActivities {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) expected: usize,
+    pub(crate) found: usize,
+}
+#[derive(Diagnostic)]
+#[diag(builtin_macros_autodiff_mode_activity)]
+pub(crate) struct AutoDiffInvalidApplicationModeAct {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) mode: String,
+    pub(crate) act: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_autodiff_mode)]
+pub(crate) struct AutoDiffInvalidMode {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) mode: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_autodiff)]
+pub(crate) struct AutoDiffInvalidApplication {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[cfg(not(llvm_enzyme))]
+#[derive(Diagnostic)]
+#[diag(builtin_macros_autodiff_not_build)]
+pub(crate) struct AutoDiffSupportNotBuild {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
 #[derive(Diagnostic)]
 #[diag(builtin_macros_concat_bytes_invalid)]
 pub(crate) struct ConcatBytesInvalid {

compiler/rustc_builtin_macros/src/lib.rs

@@ -16,6 +16,7 @@
 #![feature(proc_macro_internals)]
 #![feature(proc_macro_quote)]
 #![feature(rustdoc_internals)]
+#![cfg_attr(not(bootstrap), feature(autodiff))]
 #![feature(try_blocks)]
 #![warn(unreachable_pub)]
 // tidy-alphabetical-end

compiler/rustc_codegen_llvm/messages.ftl

@@ -51,6 +51,10 @@ codegen_llvm_prepare_thin_lto_module_with_llvm_err = failed to prepare thin LTO
 codegen_llvm_run_passes = failed to run LLVM passes
 codegen_llvm_run_passes_with_llvm_err = failed to run LLVM passes: {$llvm_err}
 
+codegen_llvm_prepare_autodiff = failed to prepare AutoDiff: src: {$src}, target: {$target}, {$error}
+codegen_llvm_prepare_autodiff_with_llvm_err = failed to prepare AutoDiff: {$llvm_err}, src: {$src}, target: {$target}, {$error}
+codegen_llvm_autodiff_without_lto = using the autodiff feature requires using fat-lto
+
 codegen_llvm_sanitizer_memtag_requires_mte =
     `-Zsanitizer=memtag` requires `-Ctarget-feature=+mte`
 

compiler/rustc_codegen_llvm/src/abi.rs

@@ -244,6 +244,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                     scratch_align,
                     bx.const_usize(copy_bytes),
                     MemFlags::empty(),
+                    None,
                 );
                 bx.lifetime_end(llscratch, scratch_size);
             }

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -1,5 +1,6 @@
 //! Set and unset common attributes on LLVM values.
 
+use rustc_ast::expand::autodiff_attrs::AutoDiffAttrs;
 use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr};
 use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::DefId;
@@ -332,6 +333,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
     instance: ty::Instance<'tcx>,
 ) {
     let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id());
+    let autodiff_attrs: &AutoDiffAttrs = cx.tcx.autodiff_attrs(instance.def_id());
 
     let mut to_add = SmallVec::<[_; 16]>::new();
 
@@ -349,6 +351,8 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
     let inline =
         if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) {
             InlineAttr::Hint
+        } else if autodiff_attrs.is_active() {
+            InlineAttr::Never
         } else {
             codegen_fn_attrs.inline
         };

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -616,7 +616,12 @@ pub(crate) fn run_pass_manager(
         }
         let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO };
         let opt_level = config.opt_level.unwrap_or(config::OptLevel::No);
-        write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage)?;
+
+        // We will run this again with different values in the context of automatic differentiation.
+        let first_run = true;
+        let noop = false;
+        debug!("running llvm pm opt pipeline");
+        write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, first_run, noop)?;
     }
     debug!("lto done");
     Ok(())
@@ -723,7 +728,12 @@ pub(crate) unsafe fn optimize_thin_module(
     let llcx = unsafe { llvm::LLVMRustContextCreate(cgcx.fewer_names) };
     let llmod_raw = parse_module(llcx, module_name, thin_module.data(), dcx)? as *const _;
     let mut module = ModuleCodegen {
-        module_llvm: ModuleLlvm { llmod_raw, llcx, tm: ManuallyDrop::new(tm) },
+        module_llvm: ModuleLlvm {
+            llmod_raw,
+            llcx,
+            tm: ManuallyDrop::new(tm),
+            typetrees: Default::default(),
+        },
         name: thin_module.name().to_string(),
         kind: ModuleKind::Regular,
     };