diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index e8c42d16733ec..545708384d9df 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -28,6 +28,22 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ } } +pub(crate) fn has_attr(llfn: &Value, idx: AttributePlace, attr: AttributeKind) -> bool { + llvm::HasAttributeAtIndex(llfn, idx, attr) +} + +pub(crate) fn has_string_attr(llfn: &Value, name: *const i8) -> bool { + llvm::HasStringAttribute(llfn, name) +} + +pub(crate) fn remove_from_llfn(llfn: &Value, place: AttributePlace, kind: AttributeKind) { + llvm::RemoveRustEnumAttributeAtIndex(llfn, place, kind); +} + +pub(crate) fn remove_string_attr_from_llfn(llfn: &Value, name: *const i8) { + llvm::RemoveStringAttrFromFn(llfn, name); +} + /// Get LLVM attribute for the provided inline heuristic. #[inline] fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 925898d817371..9f00a4c27c341 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -28,8 +28,9 @@ use crate::back::write::{ use crate::errors::{ DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro, }; +use crate::llvm::AttributePlace::Function; use crate::llvm::{self, build_string}; -use crate::{LlvmCodegenBackend, ModuleLlvm}; +use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes}; /// We keep track of the computed LTO cache keys from the previous /// session to determine which CGUs we can reuse. @@ -666,6 +667,33 @@ pub(crate) fn run_pass_manager( } if cfg!(llvm_enzyme) && enable_ad && !thin { + let cx = + SimpleCx::new(module.module_llvm.llmod(), &module.module_llvm.llcx, cgcx.pointer_size); + + for function in cx.get_functions() { + let enzyme_marker = CString::new("enzyme_marker").unwrap(); + let marker_ptr = enzyme_marker.as_ptr(); + + if attributes::has_string_attr(function, marker_ptr) { + // Sanity check: Ensure 'noinline' is present before replacing it. + assert!( + !attributes::has_attr(function, Function, llvm::AttributeKind::NoInline), + "Expected __enzyme function to have 'noinline' before adding 'alwaysinline'" + ); + + attributes::remove_from_llfn(function, Function, llvm::AttributeKind::NoInline); + attributes::remove_string_attr_from_llfn(function, marker_ptr); + + assert!( + !attributes::has_string_attr(function, marker_ptr), + "Expected function to not have 'enzyme_marker'" + ); + + let always_inline = llvm::AttributeKind::AlwaysInline.create_attr(cx.llcx); + attributes::apply_to_llfn(function, Function, &[always_inline]); + } + } + let opt_stage = llvm::OptStage::FatLTO; let stage = write::AutodiffStage::PostAD; if !config.autodiff.contains(&config::AutoDiff::NoPostopt) { diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 0147bd5a66581..c5c13ac097a27 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -361,6 +361,11 @@ fn generate_enzyme_call<'ll>( let attr = llvm::AttributeKind::NoInline.create_attr(cx.llcx); attributes::apply_to_llfn(ad_fn, Function, &[attr]); + // We add a made-up attribute just such that we can recognize it after AD to update + // (no)-inline attributes. We'll then also remove this attribute. + let enzyme_marker_attr = llvm::CreateAttrString(cx.llcx, "enzyme_marker"); + attributes::apply_to_llfn(outer_fn, Function, &[enzyme_marker_attr]); + // first, remove all calls from fnc let entry = llvm::LLVMGetFirstBasicBlock(outer_fn); let br = llvm::LLVMRustGetTerminator(entry); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 4ec6999551898..ed50515b70716 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -698,6 +698,16 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { llvm::LLVMMDStringInContext2(self.llcx(), name.as_ptr() as *const c_char, name.len()) }) } + + pub(crate) fn get_functions(&self) -> Vec<&'ll Value> { + let mut functions = vec![]; + let mut func = unsafe { llvm::LLVMGetFirstFunction(self.llmod()) }; + while let Some(f) = func { + functions.push(f); + func = unsafe { llvm::LLVMGetNextFunction(f) } + } + functions + } } impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index a9b3bdf7344be..08987a3ad14dd 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -19,6 +19,15 @@ unsafe extern "C" { pub(crate) fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool; pub(crate) fn LLVMRustHasAttributeAtIndex(V: &Value, i: c_uint, Kind: AttributeKind) -> bool; pub(crate) fn LLVMRustGetArrayNumElements(Ty: &Type) -> u64; + pub(crate) fn LLVMRustHasFnAttribute(F: &Value, Name: *const c_char) -> bool; + pub(crate) fn LLVMRustRemoveFnAttribute(F: &Value, Name: *const c_char); + pub(crate) fn LLVMGetFirstFunction(M: &Module) -> Option<&Value>; + pub(crate) fn LLVMGetNextFunction(Fn: &Value) -> Option<&Value>; + pub(crate) fn LLVMRustRemoveEnumAttributeAtIndex( + Fn: &Value, + index: c_uint, + kind: AttributeKind, + ); } unsafe extern "C" { diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 6ca81c651ed42..d401440f3aef1 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -41,6 +41,32 @@ pub(crate) fn AddFunctionAttributes<'ll>( } } +pub(crate) fn HasAttributeAtIndex<'ll>( + llfn: &'ll Value, + idx: AttributePlace, + kind: AttributeKind, +) -> bool { + unsafe { LLVMRustHasAttributeAtIndex(llfn, idx.as_uint(), kind) } +} + +pub(crate) fn HasStringAttribute<'ll>(llfn: &'ll Value, name: *const i8) -> bool { + unsafe { LLVMRustHasFnAttribute(llfn, name) } +} + +pub(crate) fn RemoveStringAttrFromFn<'ll>(llfn: &'ll Value, name: *const i8) { + unsafe { LLVMRustRemoveFnAttribute(llfn, name) } +} + +pub(crate) fn RemoveRustEnumAttributeAtIndex( + llfn: &Value, + place: AttributePlace, + kind: AttributeKind, +) { + unsafe { + LLVMRustRemoveEnumAttributeAtIndex(llfn, place.as_uint(), kind); + } +} + pub(crate) fn AddCallSiteAttributes<'ll>( callsite: &'ll Value, idx: AttributePlace, diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index b89ce90d1a1dc..169036f515298 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -128,6 +128,10 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { (**self).borrow().llcx } + pub(crate) fn llmod(&self) -> &'ll llvm::Module { + (**self).borrow().llmod + } + pub(crate) fn isize_ty(&self) -> &'ll Type { (**self).borrow().isize_ty } diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 5c8c51c8bbcc2..0ceda2201344e 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -1,7 +1,7 @@ #![allow(rustc::bad_opt_access)] -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::num::NonZero; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::atomic::AtomicBool; use rustc_abi::Align; @@ -89,8 +89,8 @@ where S: Into, I: IntoIterator, { - let locations: BTreeSet = - locations.into_iter().map(|s| CanonicalizedPath::new(Path::new(&s.into()))).collect(); + let locations = + locations.into_iter().map(|s| CanonicalizedPath::new(PathBuf::from(s.into()))).collect(); ExternEntry { location: ExternLocation::ExactPaths(locations), diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 5f0e4d745e833..2871b3c02936e 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -973,6 +973,25 @@ extern "C" LLVMMetadataRef LLVMRustDIGetInstMetadata(LLVMValueRef x) { return nullptr; } +extern "C" void +LLVMRustRemoveEnumAttributeAtIndex(LLVMValueRef F, size_t index, + LLVMRustAttributeKind RustAttr) { + LLVMRemoveEnumAttributeAtIndex(F, index, fromRust(RustAttr)); +} + +extern "C" bool LLVMRustHasFnAttribute(LLVMValueRef F, const char *Name) { + if (auto *Fn = dyn_cast(unwrap(F))) { + return Fn->hasFnAttribute(Name); + } + return false; +} + +extern "C" void LLVMRustRemoveFnAttribute(LLVMValueRef Fn, const char *Name) { + if (auto *F = dyn_cast(unwrap(Fn))) { + F->removeFnAttr(Name); + } +} + extern "C" void LLVMRustGlobalAddMetadata(LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) { unwrap(Global)->addMetadata(Kind, *unwrap(MD)); diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 43955cc23a9e9..e2d36f6a4e2ff 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -2323,14 +2323,13 @@ pub fn parse_externs( let ExternOpt { crate_name: name, path, options } = split_extern_opt(early_dcx, unstable_opts, &arg).unwrap_or_else(|e| e.emit()); - let path = path.map(|p| CanonicalizedPath::new(p.as_path())); - let entry = externs.entry(name.to_owned()); use std::collections::btree_map::Entry; let entry = if let Some(path) = path { // --extern prelude_name=some_file.rlib + let path = CanonicalizedPath::new(path); match entry { Entry::Vacant(vacant) => { let files = BTreeSet::from_iter(iter::once(path)); diff --git a/compiler/rustc_session/src/utils.rs b/compiler/rustc_session/src/utils.rs index 2243e831b66ec..e9ddd66b5e8b3 100644 --- a/compiler/rustc_session/src/utils.rs +++ b/compiler/rustc_session/src/utils.rs @@ -1,4 +1,4 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::OnceLock; use rustc_data_structures::profiling::VerboseTimingGuard; @@ -104,8 +104,8 @@ pub struct CanonicalizedPath { } impl CanonicalizedPath { - pub fn new(path: &Path) -> Self { - Self { original: path.to_owned(), canonicalized: try_canonicalize(path).ok() } + pub fn new(path: PathBuf) -> Self { + Self { canonicalized: try_canonicalize(&path).ok(), original: path } } pub fn canonicalized(&self) -> &PathBuf { diff --git a/library/core/src/ffi/c_str.rs b/library/core/src/ffi/c_str.rs index 85e87445a1b43..ac07c645c0195 100644 --- a/library/core/src/ffi/c_str.rs +++ b/library/core/src/ffi/c_str.rs @@ -79,8 +79,9 @@ use crate::{fmt, ops, slice, str}; /// /// fn my_string_safe() -> String { /// let cstr = unsafe { CStr::from_ptr(my_string()) }; -/// // Get copy-on-write Cow<'_, str>, then guarantee a freshly-owned String allocation -/// String::from_utf8_lossy(cstr.to_bytes()).to_string() +/// // Get a copy-on-write Cow<'_, str>, then extract the +/// // allocated String (or allocate a fresh one if needed). +/// cstr.to_string_lossy().into_owned() /// } /// /// println!("string: {}", my_string_safe()); diff --git a/src/librustdoc/doctest/runner.rs b/src/librustdoc/doctest/runner.rs index 784d628805d97..39a4f23560a7e 100644 --- a/src/librustdoc/doctest/runner.rs +++ b/src/librustdoc/doctest/runner.rs @@ -131,7 +131,22 @@ mod __doctest_mod {{ .output() .expect(\"failed to run command\"); if !out.status.success() {{ - eprint!(\"{{}}\", String::from_utf8_lossy(&out.stderr)); + if let Some(code) = out.status.code() {{ + eprintln!(\"Test executable failed (exit status: {{code}}).\"); + }} else {{ + eprintln!(\"Test executable failed (terminated by signal).\"); + }} + if !out.stdout.is_empty() || !out.stderr.is_empty() {{ + eprintln!(); + }} + if !out.stdout.is_empty() {{ + eprintln!(\"stdout:\"); + eprintln!(\"{{}}\", String::from_utf8_lossy(&out.stdout)); + }} + if !out.stderr.is_empty() {{ + eprintln!(\"stderr:\"); + eprintln!(\"{{}}\", String::from_utf8_lossy(&out.stderr)); + }} ExitCode::FAILURE }} else {{ ExitCode::SUCCESS diff --git a/tests/codegen/autodiff/inline.rs b/tests/codegen/autodiff/inline.rs new file mode 100644 index 0000000000000..e90faa4aa3802 --- /dev/null +++ b/tests/codegen/autodiff/inline.rs @@ -0,0 +1,23 @@ +//@ compile-flags: -Zautodiff=Enable -C opt-level=3 -Clto=fat -Zautodiff=NoPostopt +//@ no-prefer-dynamic +//@ needs-enzyme + +#![feature(autodiff)] + +use std::autodiff::autodiff; + +#[autodiff(d_square, Reverse, Duplicated, Active)] +fn square(x: &f64) -> f64 { + x * x +} + +// CHECK: ; inline::d_square +// CHECK-NEXT: ; Function Attrs: alwaysinline +// CHECK-NOT: noinline +// CHECK-NEXT: define internal fastcc void @_ZN6inline8d_square17h021c74e92c259cdeE +fn main() { + let x = std::hint::black_box(3.0); + let mut dx1 = std::hint::black_box(1.0); + let _ = d_square(&x, &mut dx1, 1.0); + assert_eq!(dx1, 6.0); +} diff --git a/tests/rustdoc-ui/doctest/edition-2024-error-output.stdout b/tests/rustdoc-ui/doctest/edition-2024-error-output.stdout index 8f056a5f703ed..273d707123734 100644 --- a/tests/rustdoc-ui/doctest/edition-2024-error-output.stdout +++ b/tests/rustdoc-ui/doctest/edition-2024-error-output.stdout @@ -5,6 +5,9 @@ test $DIR/edition-2024-error-output.rs - (line 12) ... FAILED failures: ---- $DIR/edition-2024-error-output.rs - (line 12) stdout ---- +Test executable failed (exit status: 101). + +stderr: thread 'main' panicked at $TMP:6:1: assertion `left == right` failed @@ -13,6 +16,7 @@ assertion `left == right` failed note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + failures: $DIR/edition-2024-error-output.rs - (line 12) diff --git a/tests/rustdoc-ui/doctest/stdout-and-stderr.rs b/tests/rustdoc-ui/doctest/stdout-and-stderr.rs new file mode 100644 index 0000000000000..9b0c69d883910 --- /dev/null +++ b/tests/rustdoc-ui/doctest/stdout-and-stderr.rs @@ -0,0 +1,26 @@ +// This test ensures that the output is correctly generated when the +// doctest fails. It checks when there is stderr and stdout, no stdout +// and no stderr/stdout. +// +// This is a regression test for . + +//@ edition: 2024 +//@ compile-flags:--test --test-args=--test-threads=1 +//@ normalize-stdout: "tests/rustdoc-ui/doctest" -> "$$DIR" +//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME" +//@ normalize-stdout: "panicked at .+rs:" -> "panicked at $$TMP:" +//@ failure-status: 101 +//@ rustc-env:RUST_BACKTRACE=0 + +//! ``` +//! println!("######## from a DOC TEST ########"); +//! assert_eq!("doc", "test"); +//! ``` +//! +//! ``` +//! assert_eq!("doc", "test"); +//! ``` +//! +//! ``` +//! std::process::exit(1); +//! ``` diff --git a/tests/rustdoc-ui/doctest/stdout-and-stderr.stdout b/tests/rustdoc-ui/doctest/stdout-and-stderr.stdout new file mode 100644 index 0000000000000..b2febe1344f6d --- /dev/null +++ b/tests/rustdoc-ui/doctest/stdout-and-stderr.stdout @@ -0,0 +1,46 @@ + +running 3 tests +test $DIR/stdout-and-stderr.rs - (line 15) ... FAILED +test $DIR/stdout-and-stderr.rs - (line 20) ... FAILED +test $DIR/stdout-and-stderr.rs - (line 24) ... FAILED + +failures: + +---- $DIR/stdout-and-stderr.rs - (line 15) stdout ---- +Test executable failed (exit status: 101). + +stdout: +######## from a DOC TEST ######## + +stderr: + +thread 'main' panicked at $TMP:7:1: +assertion `left == right` failed + left: "doc" + right: "test" +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + +---- $DIR/stdout-and-stderr.rs - (line 20) stdout ---- +Test executable failed (exit status: 101). + +stderr: + +thread 'main' panicked at $TMP:15:1: +assertion `left == right` failed + left: "doc" + right: "test" +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + +---- $DIR/stdout-and-stderr.rs - (line 24) stdout ---- +Test executable failed (exit status: 1). + + +failures: + $DIR/stdout-and-stderr.rs - (line 15) + $DIR/stdout-and-stderr.rs - (line 20) + $DIR/stdout-and-stderr.rs - (line 24) + +test result: FAILED. 0 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME +