From 56c9223b06ee9129f48bc680427db922d557b540 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Wed, 30 Jul 2025 14:18:46 +0200 Subject: [PATCH 1/4] [AMDGPU] Check noalias.addrspace in mayAccessScratchThroughFlat PR #149247 made the MD accessible by the backend so we can now leverage it in the memory model. The first use case here is detecting if a flat op can access scratch memory. Benefits both the MemoryLegalizer and InsertWaitCnt. --- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 22 ++++++++++++++++++- .../AMDGPU/gfx1250-scratch-scope-se.ll | 5 ++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index f20b22d14c984..5f98b850cf9ec 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4249,6 +4249,24 @@ bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode); } +static bool hasNoAliasAddrSpaceScratch(const MachineMemOperand *MemOp) { + const MDNode *MD = MemOp->getAAInfo().NoAliasAddrSpace; + if (!MD) + return false; + + // This MD is structured in ranges [A, B) + // Check if PRIVATE is included in any of them. + for (unsigned I = 0, E = MD->getNumOperands() / 2; I != E; ++I) { + auto *Low = mdconst::extract(MD->getOperand(2 * I + 0)); + auto *High = mdconst::extract(MD->getOperand(2 * I + 1)); + if (Low->getValue().ule(AMDGPUAS::PRIVATE_ADDRESS) && + High->getValue().ugt(AMDGPUAS::PRIVATE_ADDRESS)) + return true; + } + + return false; +} + bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { if (!isFLAT(MI) || isFLATGlobal(MI)) return false; @@ -4271,7 +4289,9 @@ bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { // See if any memory operand specifies an address space that involves scratch. return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) { unsigned AS = Memop->getAddrSpace(); - return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS; + if (AS == AMDGPUAS::FLAT_ADDRESS) + return !hasNoAliasAddrSpaceScratch(Memop); + return AS == AMDGPUAS::PRIVATE_ADDRESS; }); } diff --git a/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll b/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll index d1e82a06077f5..99025f0a983c0 100644 --- a/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll +++ b/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll @@ -39,20 +39,19 @@ define void @test_flat_store_no_scratch_alloc(ptr %ptr, i32 %val) #0 { ret void } -; TODO: handle define void @test_flat_store_noalias_addrspace(ptr %ptr, i32 %val) { ; GCN-LABEL: test_flat_store_noalias_addrspace: ; GCN: ; %bb.0: ; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 ; GCN-NEXT: s_wait_kmcnt 0x0 -; GCN-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE +; GCN-NEXT: flat_store_b32 v[0:1], v2 ; GCN-NEXT: s_wait_dscnt 0x0 ; GCN-NEXT: s_set_pc_i64 s[30:31] store i32 %val, ptr %ptr, !noalias.addrspace !{i32 5, i32 6} ret void } -; TODO: would be nice to handle too +; TODO: would be nice to handle define void @test_flat_store_select(ptr addrspace(1) %a, ptr addrspace(3) %b, i1 %cond, i32 %val) { ; GCN-SDAG-LABEL: test_flat_store_select: ; GCN-SDAG: ; %bb.0: From fd716f321ad03c936c11edddf5f7f289f57685f3 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Fri, 1 Aug 2025 12:03:07 +0200 Subject: [PATCH 2/4] Use shared helper --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 17 ++------------ llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 23 ++----------------- .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 17 ++++++++++++++ llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 5 ++++ 4 files changed, 26 insertions(+), 36 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 8f44c03d95b43..f8383b7456901 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -17715,21 +17715,8 @@ atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) { static bool flatInstrMayAccessPrivate(const Instruction *I) { const MDNode *NoaliasAddrSpaceMD = I->getMetadata(LLVMContext::MD_noalias_addrspace); - if (!NoaliasAddrSpaceMD) - return true; - - for (unsigned I = 0, E = NoaliasAddrSpaceMD->getNumOperands() / 2; I != E; - ++I) { - auto *Low = mdconst::extract( - NoaliasAddrSpaceMD->getOperand(2 * I + 0)); - if (Low->getValue().uge(AMDGPUAS::PRIVATE_ADDRESS)) { - auto *High = mdconst::extract( - NoaliasAddrSpaceMD->getOperand(2 * I + 1)); - return High->getValue().ule(AMDGPUAS::PRIVATE_ADDRESS); - } - } - - return true; + return !AMDGPU::hasValueInRange(NoaliasAddrSpaceMD, + AMDGPUAS::PRIVATE_ADDRESS); } TargetLowering::AtomicExpansionKind diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 5f98b850cf9ec..6c80749cca243 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4249,24 +4249,6 @@ bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode); } -static bool hasNoAliasAddrSpaceScratch(const MachineMemOperand *MemOp) { - const MDNode *MD = MemOp->getAAInfo().NoAliasAddrSpace; - if (!MD) - return false; - - // This MD is structured in ranges [A, B) - // Check if PRIVATE is included in any of them. - for (unsigned I = 0, E = MD->getNumOperands() / 2; I != E; ++I) { - auto *Low = mdconst::extract(MD->getOperand(2 * I + 0)); - auto *High = mdconst::extract(MD->getOperand(2 * I + 1)); - if (Low->getValue().ule(AMDGPUAS::PRIVATE_ADDRESS) && - High->getValue().ugt(AMDGPUAS::PRIVATE_ADDRESS)) - return true; - } - - return false; -} - bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { if (!isFLAT(MI) || isFLATGlobal(MI)) return false; @@ -4284,13 +4266,12 @@ bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { if (MI.memoperands_empty()) return true; - // TODO (?): Does this need to be taught how to read noalias.addrspace ? - // See if any memory operand specifies an address space that involves scratch. return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) { unsigned AS = Memop->getAddrSpace(); if (AS == AMDGPUAS::FLAT_ADDRESS) - return !hasNoAliasAddrSpaceScratch(Memop); + return !AMDGPU::hasValueInRange(Memop->getAAInfo().NoAliasAddrSpace, + AMDGPUAS::PRIVATE_ADDRESS); return AS == AMDGPUAS::PRIVATE_ADDRESS; }); } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 00dcb9b52d4bd..621ae947238af 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -21,6 +21,7 @@ #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/IntrinsicsR600.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Metadata.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" @@ -1666,6 +1667,22 @@ getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) { return Vals; } +bool hasValueInRange(const MDNode *MD, unsigned Val) { + if (!MD) + return false; + + assert((MD->getNumOperands() % 2 == 0) && "invalid number of operands!"); + for (unsigned I = 0, E = MD->getNumOperands() / 2; I != E; ++I) { + auto *Low = mdconst::extract(MD->getOperand(2 * I + 0)); + auto *High = mdconst::extract(MD->getOperand(2 * I + 1)); + assert(Low->getValue().ult(High->getValue()) && "invalid range metadata!"); + if (Low->getValue().ule(Val) && High->getValue().ugt(Val)) + return true; + } + + return false; +} + unsigned getVmcntBitMask(const IsaVersion &Version) { return (1 << (getVmcntBitWidthLo(Version.Major) + getVmcntBitWidthHi(Version.Major))) - diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 1252e35d81e82..f060fa050d9a8 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -35,6 +35,7 @@ class MCInstrInfo; class MCRegisterClass; class MCRegisterInfo; class MCSubtargetInfo; +class MDNode; class StringRef; class Triple; class raw_ostream; @@ -1064,6 +1065,10 @@ SmallVector getIntegerVecAttribute(const Function &F, StringRef Name, std::optional> getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size); +/// Checks if \p Val is inside \p MD, a !range-like metadata. +/// Returns false if \p MD is null. +bool hasValueInRange(const MDNode *MD, unsigned Val); + /// Represents the counter values to wait for in an s_waitcnt instruction. /// /// Large values (including the maximum possible integer) can be used to From 757c736c95a3e7dc593e12ea03174b39aa009937 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Mon, 4 Aug 2025 11:07:36 +0200 Subject: [PATCH 3/4] comments --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 7 +++---- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 8 +++++--- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 13 +++++-------- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 3 +-- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index f8383b7456901..2f925415692a2 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -17713,10 +17713,9 @@ atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) { /// Return if a flat address space atomicrmw can access private memory. static bool flatInstrMayAccessPrivate(const Instruction *I) { - const MDNode *NoaliasAddrSpaceMD = - I->getMetadata(LLVMContext::MD_noalias_addrspace); - return !AMDGPU::hasValueInRange(NoaliasAddrSpaceMD, - AMDGPUAS::PRIVATE_ADDRESS); + const MDNode *MD = I->getMetadata(LLVMContext::MD_noalias_addrspace); + return !(MD && + AMDGPU::hasValueInRangeLikeMetadata(*MD, AMDGPUAS::PRIVATE_ADDRESS)); } TargetLowering::AtomicExpansionKind diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 6c80749cca243..0fe83fe664f5d 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4269,9 +4269,11 @@ bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { // See if any memory operand specifies an address space that involves scratch. return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) { unsigned AS = Memop->getAddrSpace(); - if (AS == AMDGPUAS::FLAT_ADDRESS) - return !AMDGPU::hasValueInRange(Memop->getAAInfo().NoAliasAddrSpace, - AMDGPUAS::PRIVATE_ADDRESS); + if (AS == AMDGPUAS::FLAT_ADDRESS) { + const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace; + return !(MD && AMDGPU::hasValueInRangeLikeMetadata( + *MD, AMDGPUAS::PRIVATE_ADDRESS)); + } return AS == AMDGPUAS::PRIVATE_ADDRESS; }); } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 621ae947238af..304f290a4db3a 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -1667,14 +1667,11 @@ getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) { return Vals; } -bool hasValueInRange(const MDNode *MD, unsigned Val) { - if (!MD) - return false; - - assert((MD->getNumOperands() % 2 == 0) && "invalid number of operands!"); - for (unsigned I = 0, E = MD->getNumOperands() / 2; I != E; ++I) { - auto *Low = mdconst::extract(MD->getOperand(2 * I + 0)); - auto *High = mdconst::extract(MD->getOperand(2 * I + 1)); +bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) { + assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!"); + for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) { + auto *Low = mdconst::extract(MD.getOperand(2 * I + 0)); + auto *High = mdconst::extract(MD.getOperand(2 * I + 1)); assert(Low->getValue().ult(High->getValue()) && "invalid range metadata!"); if (Low->getValue().ule(Val) && High->getValue().ugt(Val)) return true; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index f060fa050d9a8..0574c104f6a97 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -1066,8 +1066,7 @@ std::optional> getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size); /// Checks if \p Val is inside \p MD, a !range-like metadata. -/// Returns false if \p MD is null. -bool hasValueInRange(const MDNode *MD, unsigned Val); +bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val); /// Represents the counter values to wait for in an s_waitcnt instruction. /// From 269abdd021634f6d41a8655090ccc4f461413a69 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Thu, 7 Aug 2025 11:10:53 +0200 Subject: [PATCH 4/4] Comments --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 ++-- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 2f925415692a2..a74835258fb15 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -17714,8 +17714,8 @@ atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) { /// Return if a flat address space atomicrmw can access private memory. static bool flatInstrMayAccessPrivate(const Instruction *I) { const MDNode *MD = I->getMetadata(LLVMContext::MD_noalias_addrspace); - return !(MD && - AMDGPU::hasValueInRangeLikeMetadata(*MD, AMDGPUAS::PRIVATE_ADDRESS)); + return !MD || + !AMDGPU::hasValueInRangeLikeMetadata(*MD, AMDGPUAS::PRIVATE_ADDRESS); } TargetLowering::AtomicExpansionKind diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 0fe83fe664f5d..11cc999283504 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4271,8 +4271,8 @@ bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { unsigned AS = Memop->getAddrSpace(); if (AS == AMDGPUAS::FLAT_ADDRESS) { const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace; - return !(MD && AMDGPU::hasValueInRangeLikeMetadata( - *MD, AMDGPUAS::PRIVATE_ADDRESS)); + return !MD || !AMDGPU::hasValueInRangeLikeMetadata( + *MD, AMDGPUAS::PRIVATE_ADDRESS); } return AS == AMDGPUAS::PRIVATE_ADDRESS; });