diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index c850786f834d9..96dcb21304787 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -31161,3 +31161,57 @@ This intrinsic is assumed to execute in the default :ref:`floating-point environment ` *except* for the rounding mode. This intrinsic is not supported on all targets. Some targets may not support all rounding modes. + +'``llvm.protected.field.ptr``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare ptr @llvm.protected.field.ptr(ptr ptr, i64 disc, i1 use_hw_encoding) + +Overview: +""""""""" + +The '``llvm.protected.field.ptr``' intrinsic returns a pointer to the +storage location of a pointer that has special properties as described +below. + +Arguments: +"""""""""" + +The first argument is the pointer specifying the location to store the +pointer. The second argument is the discriminator, which is used as an +input for the pointer encoding. The third argument specifies whether to +use a target-specific mechanism to encode the pointer. + +Semantics: +"""""""""" + +This intrinsic returns a pointer which may be used to store a +pointer at the specified address that is encoded using the specified +discriminator. Stores via the pointer will cause the stored pointer to be +blended with the second argument before being stored. The blend operation +shall be either a weak but cheap and target-independent operation (if +the third argument is 0) or a stronger target-specific operation (if the +third argument is 1). When loading from the pointer, the inverse operation +is done on the loaded pointer after it is loaded. Specifically, when the +third argument is 1, the pointer is signed (using pointer authentication +instructions or emulated PAC if not supported by the hardware) using +the discriminator before being stored, and authenticated after being +loaded. Note that it is currently unsupported to have the third argument +be 1 on targets other than AArch64. When the third argument is 0, it is +rotated left by 16 bits and the discriminator is subtracted before being +stored, and the discriminator is added and the pointer is rotated right +by 16 bits after being loaded. + +If the pointer is used other than for loading or storing (e.g. its +address escapes), that will disable all blending operations using +the deactivation symbol specified in the intrinsic's operand bundle. +The deactivation symbol operand bundle is copied onto any sign and auth +intrinsics that this intrinsic is lowered into. The intent is that the +deactivation symbol represents a field identifier. + +This intrinsic is used to implement structure protection. diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index bd6f94ac1286c..914424831b8a8 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -2850,6 +2850,12 @@ def int_experimental_convergence_anchor def int_experimental_convergence_loop : DefaultAttrsIntrinsic<[llvm_token_ty], [], [IntrNoMem, IntrConvergent]>; +//===----------------- Structure Protection Intrinsics --------------------===// + +def int_protected_field_ptr : + DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty, llvm_i1_ty], + [IntrNoMem, ImmArg>]>; + //===----------------------------------------------------------------------===// // Target-specific intrinsics //===----------------------------------------------------------------------===// diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp index 8de2c48581a1e..5d7129ae0cb87 100644 --- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp @@ -21,9 +21,11 @@ #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/Function.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/RuntimeLibcalls.h" #include "llvm/IR/Type.h" @@ -461,6 +463,162 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses( return Changed; } +namespace { + +enum class PointerEncoding { + Rotate, + PACCopyable, + PACNonCopyable, +}; + +bool expandProtectedFieldPtr(Function &Intr) { + Module &M = *Intr.getParent(); + + SmallPtrSet DSsToDeactivate; + SmallPtrSet LoadsStores; + + Type *Int8Ty = Type::getInt8Ty(M.getContext()); + Type *Int64Ty = Type::getInt64Ty(M.getContext()); + PointerType *PtrTy = PointerType::get(M.getContext(), 0); + + Function *SignIntr = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::ptrauth_sign, {}); + Function *AuthIntr = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::ptrauth_auth, {}); + + auto *EmuFnTy = FunctionType::get(Int64Ty, {Int64Ty, Int64Ty}, false); + FunctionCallee EmuSignIntr = M.getOrInsertFunction("__emupac_pacda", EmuFnTy); + FunctionCallee EmuAuthIntr = M.getOrInsertFunction("__emupac_autda", EmuFnTy); + + auto CreateSign = [&](IRBuilder<> &B, Value *Val, Value *Disc, + OperandBundleDef DSBundle) { + Function *F = B.GetInsertBlock()->getParent(); + Attribute FSAttr = F->getFnAttribute("target-features"); + if (FSAttr.isValid() && FSAttr.getValueAsString().contains("+pauth")) + return B.CreateCall(SignIntr, {Val, B.getInt32(2), Disc}, DSBundle); + return B.CreateCall(EmuSignIntr, {Val, Disc}, DSBundle); + }; + + auto CreateAuth = [&](IRBuilder<> &B, Value *Val, Value *Disc, + OperandBundleDef DSBundle) { + Function *F = B.GetInsertBlock()->getParent(); + Attribute FSAttr = F->getFnAttribute("target-features"); + if (FSAttr.isValid() && FSAttr.getValueAsString().contains("+pauth")) + return B.CreateCall(AuthIntr, {Val, B.getInt32(2), Disc}, DSBundle); + return B.CreateCall(EmuAuthIntr, {Val, Disc}, DSBundle); + }; + + auto GetDeactivationSymbol = [&](CallInst *Call) -> GlobalValue * { + if (auto Bundle = + Call->getOperandBundle(LLVMContext::OB_deactivation_symbol)) + return cast(Bundle->Inputs[0]); + return nullptr; + }; + + for (User *U : Intr.users()) { + auto *Call = cast(U); + auto *DS = GetDeactivationSymbol(Call); + + for (Use &U : Call->uses()) { + if (auto *LI = dyn_cast(U.getUser())) { + if (isa(LI->getType())) { + LoadsStores.insert(LI); + continue; + } + } + if (auto *SI = dyn_cast(U.getUser())) { + if (U.getOperandNo() == 1 && + isa(SI->getValueOperand()->getType())) { + LoadsStores.insert(SI); + continue; + } + } + // Comparisons against null cannot be used to recover the original + // pointer so we allow them. + if (auto *CI = dyn_cast(U.getUser())) { + if (auto *Op = dyn_cast(CI->getOperand(0))) + if (Op->isNullValue()) + continue; + if (auto *Op = dyn_cast(CI->getOperand(1))) + if (Op->isNullValue()) + continue; + } + if (DS) + DSsToDeactivate.insert(DS); + } + } + + for (Instruction *I : LoadsStores) { + auto *PointerOperand = isa(I) + ? cast(I)->getPointerOperand() + : cast(I)->getPointerOperand(); + auto *Call = cast(PointerOperand); + + auto *Disc = Call->getArgOperand(1); + bool UseHWEncoding = cast(Call->getArgOperand(2))->getZExtValue(); + + GlobalValue *DS = GetDeactivationSymbol(Call); + OperandBundleDef DSBundle("deactivation-symbol", DS); + + if (auto *LI = dyn_cast(I)) { + IRBuilder<> B(LI->getNextNode()); + auto *LIInt = cast(B.CreatePtrToInt(LI, B.getInt64Ty())); + Value *Auth; + if (UseHWEncoding) { + Auth = CreateAuth(B, LIInt, Disc, DSBundle); + } else { + Auth = B.CreateAdd(LIInt, Disc); + Auth = B.CreateIntrinsic( + Auth->getType(), Intrinsic::fshr, + {Auth, Auth, ConstantInt::get(Auth->getType(), 16)}); + } + LI->replaceAllUsesWith(B.CreateIntToPtr(Auth, B.getPtrTy())); + LIInt->setOperand(0, LI); + } else { + auto *SI = cast(I); + IRBuilder<> B(SI); + auto *SIValInt = + B.CreatePtrToInt(SI->getValueOperand(), B.getInt64Ty()); + Value *Sign; + if (UseHWEncoding) { + Sign = CreateSign(B, SIValInt, Disc, DSBundle); + } else { + Sign = B.CreateIntrinsic( + SIValInt->getType(), Intrinsic::fshl, + {SIValInt, SIValInt, ConstantInt::get(SIValInt->getType(), 16)}); + Sign = B.CreateSub(Sign, Disc); + } + SI->setOperand(0, B.CreateIntToPtr(Sign, B.getPtrTy())); + } + } + + for (User *U : llvm::make_early_inc_range(Intr.users())) { + auto *Call = cast(U); + auto *Pointer = Call->getArgOperand(0); + + Call->replaceAllUsesWith(Pointer); + Call->eraseFromParent(); + } + + if (!DSsToDeactivate.empty()) { + Constant *Nop = + ConstantExpr::getIntToPtr(ConstantInt::get(Int64Ty, 0xd503201f), PtrTy); + for (GlobalValue *OldDS : DSsToDeactivate) { + GlobalValue *DS = GlobalAlias::create( + Int8Ty, 0, GlobalValue::ExternalLinkage, OldDS->getName(), Nop, &M); + DS->setVisibility(GlobalValue::HiddenVisibility); + if (OldDS) { + DS->takeName(OldDS); + OldDS->replaceAllUsesWith(DS); + OldDS->eraseFromParent(); + } + } + } + return true; +} + +} + bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { // Map unique constants to globals. DenseMap CMap; @@ -598,6 +756,9 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { return lowerUnaryVectorIntrinsicAsLoop(M, CI); }); break; + case Intrinsic::protected_field_ptr: + Changed |= expandProtectedFieldPtr(F); + break; } } return Changed; diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/protected-field-pointer.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/protected-field-pointer.ll new file mode 100644 index 0000000000000..173240ce98717 --- /dev/null +++ b/llvm/test/Transforms/PreISelIntrinsicLowering/protected-field-pointer.ll @@ -0,0 +1,167 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 5 +; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck --check-prefix=NOPAUTH %s +; RUN: opt -passes=pre-isel-intrinsic-lowering -mattr=+pauth -S < %s | FileCheck --check-prefix=PAUTH %s + +target triple = "aarch64-unknown-linux-gnu" + +@ds1 = external global i8 +@ds2 = external global i8 +@ds3 = external global i8 +@ds4 = external global i8 +@ds5 = external global i8 +@ds6 = external global i8 + +;. +; NOPAUTH: @ds1 = external global i8 +; NOPAUTH: @ds2 = external global i8 +; NOPAUTH: @ds3 = external global i8 +; NOPAUTH: @ds4 = external global i8 +; NOPAUTH: @ds5 = external global i8 +; NOPAUTH: @ds6 = hidden alias i8, inttoptr (i64 3573751839 to ptr) +;. +; PAUTH: @ds1 = external global i8 +; PAUTH: @ds2 = external global i8 +; PAUTH: @ds3 = external global i8 +; PAUTH: @ds4 = external global i8 +; PAUTH: @ds5 = external global i8 +; PAUTH: @ds6 = hidden alias i8, inttoptr (i64 3573751839 to ptr) +;. +define ptr @load_hw(ptr %ptrptr) { +; NOPAUTH-LABEL: define ptr @load_hw( +; NOPAUTH-SAME: ptr [[PTRPTR:%.*]]) { +; NOPAUTH-NEXT: [[PTR:%.*]] = load ptr, ptr [[PTRPTR]], align 8 +; NOPAUTH-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64 +; NOPAUTH-NEXT: [[TMP2:%.*]] = call i64 @__emupac_autda(i64 [[TMP1]], i64 1) [ "deactivation-symbol"(ptr @ds1) ] +; NOPAUTH-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr +; NOPAUTH-NEXT: ret ptr [[TMP3]] +; +; PAUTH-LABEL: define ptr @load_hw( +; PAUTH-SAME: ptr [[PTRPTR:%.*]]) #[[ATTR0:[0-9]+]] { +; PAUTH-NEXT: [[PTR:%.*]] = load ptr, ptr [[PTRPTR]], align 8 +; PAUTH-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64 +; PAUTH-NEXT: [[TMP2:%.*]] = call i64 @llvm.ptrauth.auth(i64 [[TMP1]], i32 2, i64 1) [ "deactivation-symbol"(ptr @ds1) ] +; PAUTH-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr +; PAUTH-NEXT: ret ptr [[TMP3]] +; + %protptrptr = call ptr @llvm.protected.field.ptr(ptr %ptrptr, i64 1, i1 true) [ "deactivation-symbol"(ptr @ds1) ] + %ptr = load ptr, ptr %protptrptr + ret ptr %ptr +} + +define void @store_hw(ptr %ptrptr, ptr %ptr) { +; NOPAUTH-LABEL: define void @store_hw( +; NOPAUTH-SAME: ptr [[PTRPTR:%.*]], ptr [[PTR:%.*]]) { +; NOPAUTH-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64 +; NOPAUTH-NEXT: [[TMP2:%.*]] = call i64 @__emupac_pacda(i64 [[TMP1]], i64 2) [ "deactivation-symbol"(ptr @ds2) ] +; NOPAUTH-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr +; NOPAUTH-NEXT: store ptr [[TMP3]], ptr [[PTRPTR]], align 8 +; NOPAUTH-NEXT: ret void +; +; PAUTH-LABEL: define void @store_hw( +; PAUTH-SAME: ptr [[PTRPTR:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] { +; PAUTH-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64 +; PAUTH-NEXT: [[TMP2:%.*]] = call i64 @llvm.ptrauth.sign(i64 [[TMP1]], i32 2, i64 2) [ "deactivation-symbol"(ptr @ds2) ] +; PAUTH-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr +; PAUTH-NEXT: store ptr [[TMP3]], ptr [[PTRPTR]], align 8 +; PAUTH-NEXT: ret void +; + %protptrptr = call ptr @llvm.protected.field.ptr(ptr %ptrptr, i64 2, i1 true) [ "deactivation-symbol"(ptr @ds2) ] + store ptr %ptr, ptr %protptrptr + ret void +} + +define ptr @load_sw(ptr %ptrptr) { +; NOPAUTH-LABEL: define ptr @load_sw( +; NOPAUTH-SAME: ptr [[PTRPTR:%.*]]) { +; NOPAUTH-NEXT: [[PTR:%.*]] = load ptr, ptr [[PTRPTR]], align 8 +; NOPAUTH-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64 +; NOPAUTH-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 1 +; NOPAUTH-NEXT: [[TMP3:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 16) +; NOPAUTH-NEXT: [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr +; NOPAUTH-NEXT: ret ptr [[TMP4]] +; +; PAUTH-LABEL: define ptr @load_sw( +; PAUTH-SAME: ptr [[PTRPTR:%.*]]) #[[ATTR0]] { +; PAUTH-NEXT: [[PTR:%.*]] = load ptr, ptr [[PTRPTR]], align 8 +; PAUTH-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64 +; PAUTH-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 1 +; PAUTH-NEXT: [[TMP3:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 16) +; PAUTH-NEXT: [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr +; PAUTH-NEXT: ret ptr [[TMP4]] +; + %protptrptr = call ptr @llvm.protected.field.ptr(ptr %ptrptr, i64 1, i1 false) [ "deactivation-symbol"(ptr @ds3) ] + %ptr = load ptr, ptr %protptrptr + ret ptr %ptr +} + +define void @store_sw(ptr %ptrptr, ptr %ptr) { +; NOPAUTH-LABEL: define void @store_sw( +; NOPAUTH-SAME: ptr [[PTRPTR:%.*]], ptr [[PTR:%.*]]) { +; NOPAUTH-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64 +; NOPAUTH-NEXT: [[TMP2:%.*]] = call i64 @llvm.fshl.i64(i64 [[TMP1]], i64 [[TMP1]], i64 16) +; NOPAUTH-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 2 +; NOPAUTH-NEXT: [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr +; NOPAUTH-NEXT: store ptr [[TMP4]], ptr [[PTRPTR]], align 8 +; NOPAUTH-NEXT: ret void +; +; PAUTH-LABEL: define void @store_sw( +; PAUTH-SAME: ptr [[PTRPTR:%.*]], ptr [[PTR:%.*]]) #[[ATTR0]] { +; PAUTH-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64 +; PAUTH-NEXT: [[TMP2:%.*]] = call i64 @llvm.fshl.i64(i64 [[TMP1]], i64 [[TMP1]], i64 16) +; PAUTH-NEXT: [[TMP3:%.*]] = sub i64 [[TMP2]], 2 +; PAUTH-NEXT: [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr +; PAUTH-NEXT: store ptr [[TMP4]], ptr [[PTRPTR]], align 8 +; PAUTH-NEXT: ret void +; + %protptrptr = call ptr @llvm.protected.field.ptr(ptr %ptrptr, i64 2, i1 false) [ "deactivation-symbol"(ptr @ds4) ] + store ptr %ptr, ptr %protptrptr + ret void +} + +define i1 @compare(ptr %ptrptr) { +; NOPAUTH-LABEL: define i1 @compare( +; NOPAUTH-SAME: ptr [[PTRPTR:%.*]]) { +; NOPAUTH-NEXT: [[CMP1:%.*]] = icmp eq ptr [[PTRPTR]], null +; NOPAUTH-NEXT: [[CMP2:%.*]] = icmp eq ptr null, [[PTRPTR]] +; NOPAUTH-NEXT: [[CMP:%.*]] = or i1 [[CMP1]], [[CMP2]] +; NOPAUTH-NEXT: ret i1 [[CMP]] +; +; PAUTH-LABEL: define i1 @compare( +; PAUTH-SAME: ptr [[PTRPTR:%.*]]) #[[ATTR0]] { +; PAUTH-NEXT: [[CMP1:%.*]] = icmp eq ptr [[PTRPTR]], null +; PAUTH-NEXT: [[CMP2:%.*]] = icmp eq ptr null, [[PTRPTR]] +; PAUTH-NEXT: [[CMP:%.*]] = or i1 [[CMP1]], [[CMP2]] +; PAUTH-NEXT: ret i1 [[CMP]] +; + %protptrptr = call ptr @llvm.protected.field.ptr(ptr %ptrptr, i64 3, i1 true) [ "deactivation-symbol"(ptr @ds5) ] + %cmp1 = icmp eq ptr %protptrptr, null + %cmp2 = icmp eq ptr null, %protptrptr + %cmp = or i1 %cmp1, %cmp2 + ret i1 %cmp +} + +define ptr @escape(ptr %ptrptr) { +; NOPAUTH-LABEL: define ptr @escape( +; NOPAUTH-SAME: ptr [[PTRPTR:%.*]]) { +; NOPAUTH-NEXT: ret ptr [[PTRPTR]] +; +; PAUTH-LABEL: define ptr @escape( +; PAUTH-SAME: ptr [[PTRPTR:%.*]]) #[[ATTR0]] { +; PAUTH-NEXT: ret ptr [[PTRPTR]] +; + %protptrptr = call ptr @llvm.protected.field.ptr(ptr %ptrptr, i64 3, i1 true) [ "deactivation-symbol"(ptr @ds6) ] + ret ptr %protptrptr +} + +declare ptr @llvm.protected.field.ptr(ptr, i64, i1 immarg) +;. +; NOPAUTH: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(none) } +; NOPAUTH: attributes #[[ATTR1:[0-9]+]] = { nounwind memory(none) } +; NOPAUTH: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +;. +; PAUTH: attributes #[[ATTR0]] = { "target-features"="+pauth" } +; PAUTH: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(none) "target-features"="+pauth" } +; PAUTH: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(none) } +; PAUTH: attributes #[[ATTR3:[0-9]+]] = { nounwind memory(none) } +; PAUTH: attributes #[[ATTR4:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +;.