Rollup of 4 pull requests

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -28,6 +28,22 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[
     }
 }
 
+pub(crate) fn has_attr(llfn: &Value, idx: AttributePlace, attr: AttributeKind) -> bool {
+    llvm::HasAttributeAtIndex(llfn, idx, attr)
+}
+
+pub(crate) fn has_string_attr(llfn: &Value, name: *const i8) -> bool {
+    llvm::HasStringAttribute(llfn, name)
+}
+
+pub(crate) fn remove_from_llfn(llfn: &Value, place: AttributePlace, kind: AttributeKind) {
+    llvm::RemoveRustEnumAttributeAtIndex(llfn, place, kind);
+}
+
+pub(crate) fn remove_string_attr_from_llfn(llfn: &Value, name: *const i8) {
+    llvm::RemoveStringAttrFromFn(llfn, name);
+}
+
 /// Get LLVM attribute for the provided inline heuristic.
 #[inline]
 fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> {

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -28,8 +28,9 @@ use crate::back::write::{
 use crate::errors::{
     DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro,
 };
+use crate::llvm::AttributePlace::Function;
 use crate::llvm::{self, build_string};
-use crate::{LlvmCodegenBackend, ModuleLlvm};
+use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes};
 
 /// We keep track of the computed LTO cache keys from the previous
 /// session to determine which CGUs we can reuse.
@@ -666,6 +667,33 @@ pub(crate) fn run_pass_manager(
     }
 
     if cfg!(llvm_enzyme) && enable_ad && !thin {
+        let cx =
+            SimpleCx::new(module.module_llvm.llmod(), &module.module_llvm.llcx, cgcx.pointer_size);
+
+        for function in cx.get_functions() {
+            let enzyme_marker = CString::new("enzyme_marker").unwrap();
+            let marker_ptr = enzyme_marker.as_ptr();
+
+            if attributes::has_string_attr(function, marker_ptr) {
+                // Sanity check: Ensure 'noinline' is present before replacing it.
+                assert!(
+                    !attributes::has_attr(function, Function, llvm::AttributeKind::NoInline),
+                    "Expected __enzyme function to have 'noinline' before adding 'alwaysinline'"
+                );
+
+                attributes::remove_from_llfn(function, Function, llvm::AttributeKind::NoInline);
+                attributes::remove_string_attr_from_llfn(function, marker_ptr);
+
+                assert!(
+                    !attributes::has_string_attr(function, marker_ptr),
+                    "Expected function to not have 'enzyme_marker'"
+                );
+
+                let always_inline = llvm::AttributeKind::AlwaysInline.create_attr(cx.llcx);
+                attributes::apply_to_llfn(function, Function, &[always_inline]);
+            }
+        }
+
         let opt_stage = llvm::OptStage::FatLTO;
         let stage = write::AutodiffStage::PostAD;
         if !config.autodiff.contains(&config::AutoDiff::NoPostopt) {

compiler/rustc_codegen_llvm/src/builder/autodiff.rs

@@ -361,6 +361,11 @@ fn generate_enzyme_call<'ll>(
         let attr = llvm::AttributeKind::NoInline.create_attr(cx.llcx);
         attributes::apply_to_llfn(ad_fn, Function, &[attr]);
 
+        // We add a made-up attribute just such that we can recognize it after AD to update
+        // (no)-inline attributes. We'll then also remove this attribute.
+        let enzyme_marker_attr = llvm::CreateAttrString(cx.llcx, "enzyme_marker");
+        attributes::apply_to_llfn(outer_fn, Function, &[enzyme_marker_attr]);
+
         // first, remove all calls from fnc
         let entry = llvm::LLVMGetFirstBasicBlock(outer_fn);
         let br = llvm::LLVMRustGetTerminator(entry);

compiler/rustc_codegen_llvm/src/context.rs

@@ -698,6 +698,16 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
             llvm::LLVMMDStringInContext2(self.llcx(), name.as_ptr() as *const c_char, name.len())
         })
     }
+
+    pub(crate) fn get_functions(&self) -> Vec<&'ll Value> {
+        let mut functions = vec![];
+        let mut func = unsafe { llvm::LLVMGetFirstFunction(self.llmod()) };
+        while let Some(f) = func {
+            functions.push(f);
+            func = unsafe { llvm::LLVMGetNextFunction(f) }
+        }
+        functions
+    }
 }
 
 impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {

compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs

@@ -19,6 +19,15 @@ unsafe extern "C" {
     pub(crate) fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool;
     pub(crate) fn LLVMRustHasAttributeAtIndex(V: &Value, i: c_uint, Kind: AttributeKind) -> bool;
     pub(crate) fn LLVMRustGetArrayNumElements(Ty: &Type) -> u64;
+    pub(crate) fn LLVMRustHasFnAttribute(F: &Value, Name: *const c_char) -> bool;
+    pub(crate) fn LLVMRustRemoveFnAttribute(F: &Value, Name: *const c_char);
+    pub(crate) fn LLVMGetFirstFunction(M: &Module) -> Option<&Value>;
+    pub(crate) fn LLVMGetNextFunction(Fn: &Value) -> Option<&Value>;
+    pub(crate) fn LLVMRustRemoveEnumAttributeAtIndex(
+        Fn: &Value,
+        index: c_uint,
+        kind: AttributeKind,
+    );
 }
 
 unsafe extern "C" {

compiler/rustc_codegen_llvm/src/llvm/mod.rs

@@ -41,6 +41,32 @@ pub(crate) fn AddFunctionAttributes<'ll>(
     }
 }
 
+pub(crate) fn HasAttributeAtIndex<'ll>(
+    llfn: &'ll Value,
+    idx: AttributePlace,
+    kind: AttributeKind,
+) -> bool {
+    unsafe { LLVMRustHasAttributeAtIndex(llfn, idx.as_uint(), kind) }
+}
+
+pub(crate) fn HasStringAttribute<'ll>(llfn: &'ll Value, name: *const i8) -> bool {
+    unsafe { LLVMRustHasFnAttribute(llfn, name) }
+}
+
+pub(crate) fn RemoveStringAttrFromFn<'ll>(llfn: &'ll Value, name: *const i8) {
+    unsafe { LLVMRustRemoveFnAttribute(llfn, name) }
+}
+
+pub(crate) fn RemoveRustEnumAttributeAtIndex(
+    llfn: &Value,
+    place: AttributePlace,
+    kind: AttributeKind,
+) {
+    unsafe {
+        LLVMRustRemoveEnumAttributeAtIndex(llfn, place.as_uint(), kind);
+    }
+}
+
 pub(crate) fn AddCallSiteAttributes<'ll>(
     callsite: &'ll Value,
     idx: AttributePlace,

compiler/rustc_codegen_llvm/src/type_.rs

@@ -128,6 +128,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
         (**self).borrow().llcx
     }
 
+    pub(crate) fn llmod(&self) -> &'ll llvm::Module {
+        (**self).borrow().llmod
+    }
+
     pub(crate) fn isize_ty(&self) -> &'ll Type {
         (**self).borrow().isize_ty
     }

compiler/rustc_interface/src/tests.rs

@@ -1,7 +1,7 @@
 #![allow(rustc::bad_opt_access)]
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::BTreeMap;
 use std::num::NonZero;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::sync::atomic::AtomicBool;
 
 use rustc_abi::Align;
@@ -89,8 +89,8 @@ where
     S: Into<String>,
     I: IntoIterator<Item = S>,
 {
-    let locations: BTreeSet<CanonicalizedPath> =
-        locations.into_iter().map(|s| CanonicalizedPath::new(Path::new(&s.into()))).collect();
+    let locations =
+        locations.into_iter().map(|s| CanonicalizedPath::new(PathBuf::from(s.into()))).collect();
 
     ExternEntry {
         location: ExternLocation::ExactPaths(locations),

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -973,6 +973,25 @@ extern "C" LLVMMetadataRef LLVMRustDIGetInstMetadata(LLVMValueRef x) {
   return nullptr;
 }
 
+extern "C" void
+LLVMRustRemoveEnumAttributeAtIndex(LLVMValueRef F, size_t index,
+                                   LLVMRustAttributeKind RustAttr) {
+  LLVMRemoveEnumAttributeAtIndex(F, index, fromRust(RustAttr));
+}
+
+extern "C" bool LLVMRustHasFnAttribute(LLVMValueRef F, const char *Name) {
+  if (auto *Fn = dyn_cast<Function>(unwrap<Value>(F))) {
+    return Fn->hasFnAttribute(Name);
+  }
+  return false;
+}
+
+extern "C" void LLVMRustRemoveFnAttribute(LLVMValueRef Fn, const char *Name) {
+  if (auto *F = dyn_cast<Function>(unwrap<Value>(Fn))) {
+    F->removeFnAttr(Name);
+  }
+}
+
 extern "C" void LLVMRustGlobalAddMetadata(LLVMValueRef Global, unsigned Kind,
                                           LLVMMetadataRef MD) {
   unwrap<GlobalObject>(Global)->addMetadata(Kind, *unwrap<MDNode>(MD));

compiler/rustc_session/src/config.rs

@@ -2323,14 +2323,13 @@ pub fn parse_externs(
         let ExternOpt { crate_name: name, path, options } =
             split_extern_opt(early_dcx, unstable_opts, &arg).unwrap_or_else(|e| e.emit());
 
-        let path = path.map(|p| CanonicalizedPath::new(p.as_path()));
-
         let entry = externs.entry(name.to_owned());
 
         use std::collections::btree_map::Entry;
 
         let entry = if let Some(path) = path {
             // --extern prelude_name=some_file.rlib
+            let path = CanonicalizedPath::new(path);
             match entry {
                 Entry::Vacant(vacant) => {
                     let files = BTreeSet::from_iter(iter::once(path));

compiler/rustc_session/src/utils.rs

@@ -1,4 +1,4 @@
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::sync::OnceLock;
 
 use rustc_data_structures::profiling::VerboseTimingGuard;
@@ -104,8 +104,8 @@ pub struct CanonicalizedPath {
 }
 
 impl CanonicalizedPath {
-    pub fn new(path: &Path) -> Self {
-        Self { original: path.to_owned(), canonicalized: try_canonicalize(path).ok() }
+    pub fn new(path: PathBuf) -> Self {
+        Self { canonicalized: try_canonicalize(&path).ok(), original: path }
     }
 
     pub fn canonicalized(&self) -> &PathBuf {

library/core/src/ffi/c_str.rs

@@ -79,8 +79,9 @@ use crate::{fmt, ops, slice, str};
 ///
 /// fn my_string_safe() -> String {
 ///     let cstr = unsafe { CStr::from_ptr(my_string()) };
-///     // Get copy-on-write Cow<'_, str>, then guarantee a freshly-owned String allocation
-///     String::from_utf8_lossy(cstr.to_bytes()).to_string()
+///     // Get a copy-on-write Cow<'_, str>, then extract the
+///     // allocated String (or allocate a fresh one if needed).
+///     cstr.to_string_lossy().into_owned()
 /// }
 ///
 /// println!("string: {}", my_string_safe());

src/librustdoc/doctest/runner.rs

@@ -131,7 +131,22 @@ mod __doctest_mod {{
             .output()
             .expect(\"failed to run command\");
         if !out.status.success() {{
-            eprint!(\"{{}}\", String::from_utf8_lossy(&out.stderr));
+            if let Some(code) = out.status.code() {{
+                eprintln!(\"Test executable failed (exit status: {{code}}).\");
+            }} else {{
+                eprintln!(\"Test executable failed (terminated by signal).\");
+            }}
+            if !out.stdout.is_empty() || !out.stderr.is_empty() {{
+                eprintln!();
+            }}
+            if !out.stdout.is_empty() {{
+                eprintln!(\"stdout:\");
+                eprintln!(\"{{}}\", String::from_utf8_lossy(&out.stdout));
+            }}
+            if !out.stderr.is_empty() {{
+                eprintln!(\"stderr:\");
+                eprintln!(\"{{}}\", String::from_utf8_lossy(&out.stderr));
+            }}
             ExitCode::FAILURE
         }} else {{
             ExitCode::SUCCESS

tests/codegen/autodiff/inline.rs

@@ -0,0 +1,23 @@
+//@ compile-flags: -Zautodiff=Enable -C opt-level=3  -Clto=fat -Zautodiff=NoPostopt
+//@ no-prefer-dynamic
+//@ needs-enzyme
+
+#![feature(autodiff)]
+
+use std::autodiff::autodiff;
+
+#[autodiff(d_square, Reverse, Duplicated, Active)]
+fn square(x: &f64) -> f64 {
+    x * x
+}
+
+// CHECK: ; inline::d_square
+// CHECK-NEXT: ; Function Attrs: alwaysinline
+// CHECK-NOT: noinline
+// CHECK-NEXT: define internal fastcc void @_ZN6inline8d_square17h021c74e92c259cdeE
+fn main() {
+    let x = std::hint::black_box(3.0);
+    let mut dx1 = std::hint::black_box(1.0);
+    let _ = d_square(&x, &mut dx1, 1.0);
+    assert_eq!(dx1, 6.0);
+}

tests/rustdoc-ui/doctest/edition-2024-error-output.stdout

@@ -5,6 +5,9 @@ test $DIR/edition-2024-error-output.rs - (line 12) ... FAILED
 failures:
 
 ---- $DIR/edition-2024-error-output.rs - (line 12) stdout ----
+Test executable failed (exit status: 101).
+
+stderr:
 
 thread 'main' panicked at $TMP:6:1:
 assertion `left == right` failed
@@ -13,6 +16,7 @@ assertion `left == right` failed
 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
 
 
+
 failures:
     $DIR/edition-2024-error-output.rs - (line 12)
 

tests/rustdoc-ui/doctest/stdout-and-stderr.rs

@@ -0,0 +1,26 @@
+// This test ensures that the output is correctly generated when the
+// doctest fails. It checks when there is stderr and stdout, no stdout
+// and no stderr/stdout.
+//
+// This is a regression test for <https://github.com/rust-lang/rust/issues/140289>.
+
+//@ edition: 2024
+//@ compile-flags:--test --test-args=--test-threads=1
+//@ normalize-stdout: "tests/rustdoc-ui/doctest" -> "$$DIR"
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ normalize-stdout: "panicked at .+rs:" -> "panicked at $$TMP:"
+//@ failure-status: 101
+//@ rustc-env:RUST_BACKTRACE=0
+
+//! ```
+//! println!("######## from a DOC TEST ########");
+//! assert_eq!("doc", "test");
+//! ```
+//!
+//! ```
+//! assert_eq!("doc", "test");
+//! ```
+//!
+//! ```
+//! std::process::exit(1);
+//! ```

tests/rustdoc-ui/doctest/stdout-and-stderr.stdout

@@ -0,0 +1,46 @@
+
+running 3 tests
+test $DIR/stdout-and-stderr.rs - (line 15) ... FAILED
+test $DIR/stdout-and-stderr.rs - (line 20) ... FAILED
+test $DIR/stdout-and-stderr.rs - (line 24) ... FAILED
+
+failures:
+
+---- $DIR/stdout-and-stderr.rs - (line 15) stdout ----
+Test executable failed (exit status: 101).
+
+stdout:
+######## from a DOC TEST ########
+
+stderr:
+
+thread 'main' panicked at $TMP:7:1:
+assertion `left == right` failed
+  left: "doc"
+ right: "test"
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+
+
+---- $DIR/stdout-and-stderr.rs - (line 20) stdout ----
+Test executable failed (exit status: 101).
+
+stderr:
+
+thread 'main' panicked at $TMP:15:1:
+assertion `left == right` failed
+  left: "doc"
+ right: "test"
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+
+
+---- $DIR/stdout-and-stderr.rs - (line 24) stdout ----
+Test executable failed (exit status: 1).
+
+
+failures:
+    $DIR/stdout-and-stderr.rs - (line 15)
+    $DIR/stdout-and-stderr.rs - (line 20)
+    $DIR/stdout-and-stderr.rs - (line 24)
+
+test result: FAILED. 0 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+