Skip to content

Commit 4d6d601

Browse files
borsbors
committed
Auto merge of #99574 - durin42:allocator-patch-redux, r=nikic
codegen: use new {re,de,}allocator annotations in llvm This obviates the patch that teaches LLVM internals about _rust_{re,de}alloc functions by putting annotations directly in the IR for the optimizer. The sole test change is required to anchor FileCheck to the body of the `box_uninitialized` method, so it doesn't see the `allocalign` on `__rust_alloc` and get mad about the string `alloca` showing up. Since I was there anyway, I added some checks on the attributes to prove the right attributes got set. r? `@nikic`
2 parents c11207e + 130a1df commit 4d6d601

File tree

14 files changed

+356
-4
lines changed

14 files changed

+356
-4
lines changed

‎compiler/rustc_codegen_llvm/src/attributes.rs

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ use smallvec::SmallVec;
1313

1414
use crate::attributes;
1515
use crate::llvm::AttributePlace::Function;
16-
use crate::llvm::{self, Attribute, AttributeKind, AttributePlace};
16+
use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace};
1717
use crate::llvm_util;
1818
pub use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr};
1919

@@ -227,6 +227,10 @@ pub(crate) fn default_optimisation_attrs<'ll>(
227227
attrs
228228
}
229229

230+
fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute {
231+
llvm::CreateAttrStringValue(llcx, "alloc-family", "__rust_alloc")
232+
}
233+
230234
/// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`)
231235
/// attributes.
232236
pub fn from_fn_attrs<'ll, 'tcx>(
@@ -309,11 +313,54 @@ pub fn from_fn_attrs<'ll, 'tcx>(
309313
// Need this for AArch64.
310314
to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false"));
311315
}
312-
if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) {
316+
if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR)
317+
|| codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED)
318+
{
319+
if llvm_util::get_version() >= (15, 0, 0) {
320+
to_add.push(create_alloc_family_attr(cx.llcx));
321+
// apply to argument place instead of function
322+
let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx);
323+
attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]);
324+
to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0));
325+
let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned;
326+
if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) {
327+
flags |= AllocKindFlags::Uninitialized;
328+
} else {
329+
flags |= AllocKindFlags::Zeroed;
330+
}
331+
to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags));
332+
}
313333
// apply to return place instead of function (unlike all other attributes applied in this function)
314334
let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx);
315335
attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]);
316336
}
337+
if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::REALLOCATOR) {
338+
if llvm_util::get_version() >= (15, 0, 0) {
339+
to_add.push(create_alloc_family_attr(cx.llcx));
340+
to_add.push(llvm::CreateAllocKindAttr(
341+
cx.llcx,
342+
AllocKindFlags::Realloc | AllocKindFlags::Aligned,
343+
));
344+
// applies to argument place instead of function place
345+
let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx);
346+
attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]);
347+
// apply to argument place instead of function
348+
let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx);
349+
attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]);
350+
to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3));
351+
}
352+
let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx);
353+
attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]);
354+
}
355+
if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::DEALLOCATOR) {
356+
if llvm_util::get_version() >= (15, 0, 0) {
357+
to_add.push(create_alloc_family_attr(cx.llcx));
358+
to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free));
359+
// applies to argument place instead of function place
360+
let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx);
361+
attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]);
362+
}
363+
}
317364
if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) {
318365
to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry"));
319366
}

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,9 @@ pub enum AttributeKind {
193193
SanitizeMemTag = 34,
194194
NoCfCheck = 35,
195195
ShadowCallStack = 36,
196+
AllocSize = 37,
197+
AllocatedPointer = 38,
198+
AllocAlign = 39,
196199
}
197200

198201
/// LLVMIntPredicate
@@ -986,6 +989,22 @@ pub mod debuginfo {
986989
}
987990
}
988991

992+
use bitflags::bitflags;
993+
// These values **must** match with LLVMRustAllocKindFlags
994+
bitflags! {
995+
#[repr(transparent)]
996+
#[derive(Default)]
997+
pub struct AllocKindFlags : u64 {
998+
const Unknown = 0;
999+
const Alloc = 1;
1000+
const Realloc = 1 << 1;
1001+
const Free = 1 << 2;
1002+
const Uninitialized = 1 << 3;
1003+
const Zeroed = 1 << 4;
1004+
const Aligned = 1 << 5;
1005+
}
1006+
}
1007+
9891008
extern "C" {
9901009
pub type ModuleBuffer;
9911010
}
@@ -1193,6 +1212,8 @@ extern "C" {
11931212
pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
11941213
pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
11951214
pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute;
1215+
pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute;
1216+
pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute;
11961217

11971218
// Operations on functions
11981219
pub fn LLVMRustGetOrInsertFunction<'a>(

‎compiler/rustc_codegen_llvm/src/llvm/mod.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,14 @@ pub fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute {
9595
unsafe { LLVMRustCreateUWTableAttr(llcx, async_) }
9696
}
9797

98+
pub fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute {
99+
unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) }
100+
}
101+
102+
pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute {
103+
unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) }
104+
}
105+
98106
#[derive(Copy, Clone)]
99107
pub enum AttributePlace {
100108
ReturnValue,

‎compiler/rustc_feature/src/builtin_attrs.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -532,6 +532,9 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
532532

533533
rustc_attr!(rustc_allocator, Normal, template!(Word), WarnFollowing, IMPL_DETAIL),
534534
rustc_attr!(rustc_allocator_nounwind, Normal, template!(Word), WarnFollowing, IMPL_DETAIL),
535+
rustc_attr!(rustc_reallocator, Normal, template!(Word), WarnFollowing, IMPL_DETAIL),
536+
rustc_attr!(rustc_deallocator, Normal, template!(Word), WarnFollowing, IMPL_DETAIL),
537+
rustc_attr!(rustc_allocator_zeroed, Normal, template!(Word), WarnFollowing, IMPL_DETAIL),
535538
gated!(
536539
alloc_error_handler, Normal, template!(Word), WarnFollowing,
537540
experimental!(alloc_error_handler)

‎compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ enum LLVMRustAttribute {
8686
SanitizeMemTag = 34,
8787
NoCfCheck = 35,
8888
ShadowCallStack = 36,
89+
AllocSize = 37,
90+
AllocatedPointer = 38,
91+
AllocAlign = 39,
8992
};
9093

9194
typedef struct OpaqueRustString *RustStringRef;

‎compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,14 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) {
234234
return Attribute::SanitizeMemTag;
235235
case ShadowCallStack:
236236
return Attribute::ShadowCallStack;
237+
case AllocSize:
238+
return Attribute::AllocSize;
239+
#if LLVM_VERSION_GE(15, 0)
240+
case AllocatedPointer:
241+
return Attribute::AllocatedPointer;
242+
case AllocAlign:
243+
return Attribute::AllocAlign;
244+
#endif
237245
}
238246
report_fatal_error("bad AttributeKind");
239247
}
@@ -305,6 +313,67 @@ extern "C" LLVMAttributeRef LLVMRustCreateUWTableAttr(LLVMContextRef C, bool Asy
305313
#endif
306314
}
307315

316+
extern "C" LLVMAttributeRef LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) {
317+
return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, None));
318+
}
319+
320+
#if LLVM_VERSION_GE(15, 0)
321+
322+
// These values **must** match ffi::AllocKindFlags.
323+
// It _happens_ to match the LLVM values of llvm::AllocFnKind,
324+
// but that's happenstance and we do explicit conversions before
325+
// passing them to LLVM.
326+
enum class LLVMRustAllocKindFlags : uint64_t {
327+
Unknown = 0,
328+
Alloc = 1,
329+
Realloc = 1 << 1,
330+
Free = 1 << 2,
331+
Uninitialized = 1 << 3,
332+
Zeroed = 1 << 4,
333+
Aligned = 1 << 5,
334+
};
335+
336+
static LLVMRustAllocKindFlags operator&(LLVMRustAllocKindFlags A, LLVMRustAllocKindFlags B) {
337+
return static_cast<LLVMRustAllocKindFlags>(static_cast<uint64_t>(A) &
338+
static_cast<uint64_t>(B));
339+
}
340+
341+
static bool isSet(LLVMRustAllocKindFlags F) { return F != LLVMRustAllocKindFlags::Unknown; }
342+
343+
static llvm::AllocFnKind allocKindFromRust(LLVMRustAllocKindFlags F) {
344+
llvm::AllocFnKind AFK = llvm::AllocFnKind::Unknown;
345+
if (isSet(F & LLVMRustAllocKindFlags::Alloc)) {
346+
AFK |= llvm::AllocFnKind::Alloc;
347+
}
348+
if (isSet(F & LLVMRustAllocKindFlags::Realloc)) {
349+
AFK |= llvm::AllocFnKind::Realloc;
350+
}
351+
if (isSet(F & LLVMRustAllocKindFlags::Free)) {
352+
AFK |= llvm::AllocFnKind::Free;
353+
}
354+
if (isSet(F & LLVMRustAllocKindFlags::Uninitialized)) {
355+
AFK |= llvm::AllocFnKind::Uninitialized;
356+
}
357+
if (isSet(F & LLVMRustAllocKindFlags::Zeroed)) {
358+
AFK |= llvm::AllocFnKind::Zeroed;
359+
}
360+
if (isSet(F & LLVMRustAllocKindFlags::Aligned)) {
361+
AFK |= llvm::AllocFnKind::Aligned;
362+
}
363+
return AFK;
364+
}
365+
#endif
366+
367+
extern "C" LLVMAttributeRef LLVMRustCreateAllocKindAttr(LLVMContextRef C, uint64_t AllocKindArg) {
368+
#if LLVM_VERSION_GE(15, 0)
369+
return wrap(Attribute::get(*unwrap(C), Attribute::AllocKind,
370+
static_cast<uint64_t>(allocKindFromRust(static_cast<LLVMRustAllocKindFlags>(AllocKindArg)))));
371+
#else
372+
report_fatal_error(
373+
"allockind attributes are new in LLVM 15 and should not be used on older LLVMs");
374+
#endif
375+
}
376+
308377
// Enable a fast-math flag
309378
//
310379
// https://llvm.org/docs/LangRef.html#fast-math-flags

‎compiler/rustc_middle/src/middle/codegen_fn_attrs.rs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ bitflags! {
5050
/// the hot path.
5151
const COLD = 1 << 0;
5252
/// `#[rustc_allocator]`: a hint to LLVM that the pointer returned from this
53-
/// function is never null.
53+
/// function is never null and the function has no side effects other than allocating.
5454
const ALLOCATOR = 1 << 1;
5555
/// An indicator that function will never unwind. Will become obsolete
5656
/// once C-unwind is fully stabilized.
@@ -91,6 +91,12 @@ bitflags! {
9191
const NO_COVERAGE = 1 << 15;
9292
/// `#[used(linker)]`: indicates that LLVM nor the linker can eliminate this function.
9393
const USED_LINKER = 1 << 16;
94+
/// `#[rustc_deallocator]`: a hint to LLVM that the function only deallocates memory.
95+
const DEALLOCATOR = 1 << 17;
96+
/// `#[rustc_reallocator]`: a hint to LLVM that the function only reallocates memory.
97+
const REALLOCATOR = 1 << 18;
98+
/// `#[rustc_allocator_zeroed]`: a hint to LLVM that the function only allocates zeroed memory.
99+
const ALLOCATOR_ZEROED = 1 << 19;
94100
}
95101
}
96102

‎compiler/rustc_span/src/symbol.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1202,6 +1202,7 @@ symbols! {
12021202
rustc,
12031203
rustc_allocator,
12041204
rustc_allocator_nounwind,
1205+
rustc_allocator_zeroed,
12051206
rustc_allow_const_fn_unstable,
12061207
rustc_allow_incoherent_impl,
12071208
rustc_allowed_through_unstable_modules,
@@ -1214,6 +1215,7 @@ symbols! {
12141215
rustc_const_stable,
12151216
rustc_const_unstable,
12161217
rustc_conversion_suggestion,
1218+
rustc_deallocator,
12171219
rustc_def_path,
12181220
rustc_diagnostic_item,
12191221
rustc_diagnostic_macros,
@@ -1258,6 +1260,7 @@ symbols! {
12581260
rustc_private,
12591261
rustc_proc_macro_decls,
12601262
rustc_promotable,
1263+
rustc_reallocator,
12611264
rustc_regions,
12621265
rustc_reservation_impl,
12631266
rustc_serialize,

‎compiler/rustc_typeck/src/collect.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2775,6 +2775,12 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
27752775
}
27762776
} else if attr.has_name(sym::rustc_allocator_nounwind) {
27772777
codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND;
2778+
} else if attr.has_name(sym::rustc_reallocator) {
2779+
codegen_fn_attrs.flags |= CodegenFnAttrFlags::REALLOCATOR;
2780+
} else if attr.has_name(sym::rustc_deallocator) {
2781+
codegen_fn_attrs.flags |= CodegenFnAttrFlags::DEALLOCATOR;
2782+
} else if attr.has_name(sym::rustc_allocator_zeroed) {
2783+
codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED;
27782784
} else if attr.has_name(sym::naked) {
27792785
codegen_fn_attrs.flags |= CodegenFnAttrFlags::NAKED;
27802786
} else if attr.has_name(sym::no_mangle) {

‎library/alloc/src/alloc.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,18 @@ extern "Rust" {
2525
// (the code expanding that attribute macro generates those functions), or to call
2626
// the default implementations in libstd (`__rdl_alloc` etc. in `library/std/src/alloc.rs`)
2727
// otherwise.
28-
// The rustc fork of LLVM also special-cases these function names to be able to optimize them
28+
// The rustc fork of LLVM 14 and earlier also special-cases these function names to be able to optimize them
2929
// like `malloc`, `realloc`, and `free`, respectively.
3030
#[rustc_allocator]
3131
#[rustc_allocator_nounwind]
3232
fn __rust_alloc(size: usize, align: usize) -> *mut u8;
33+
#[cfg_attr(not(bootstrap), rustc_deallocator)]
3334
#[rustc_allocator_nounwind]
3435
fn __rust_dealloc(ptr: *mut u8, size: usize, align: usize);
36+
#[cfg_attr(not(bootstrap), rustc_reallocator)]
3537
#[rustc_allocator_nounwind]
3638
fn __rust_realloc(ptr: *mut u8, old_size: usize, align: usize, new_size: usize) -> *mut u8;
39+
#[cfg_attr(not(bootstrap), rustc_allocator_zeroed)]
3740
#[rustc_allocator_nounwind]
3841
fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8;
3942
}

0 commit comments

Comments
 (0)