[PredicateInfo] Use bitcast instead of ssa.copy (#151174)

PredicateInfo needs some no-op to which the predicate can be attached.
Currently this is an ssa.copy intrinsic. This PR replaces it with a
no-op bitcast.
    
Using a bitcast is more efficient because we don't have the overhead of
an overloaded intrinsic. It also makes things slightly simpler overall.

Author: nikic

llvm/include/llvm/Transforms/Utils/PredicateInfo.h

@@ -30,7 +30,7 @@
 /// %cmp = icmp eq i32, %x, 50
 /// br i1 %cmp, label %true, label %false
 /// true:
-/// %x.0 = call \@llvm.ssa_copy.i32(i32 %x)
+/// %x.0 = bitcast i32 %x to %x
 /// ret i32 %x.0
 /// false:
 /// ret i32 1
@@ -70,7 +70,7 @@ class raw_ostream;
 enum PredicateType { PT_Branch, PT_Assume, PT_Switch };
 
 /// Constraint for a predicate of the form "cmp Pred Op, OtherOp", where Op
-/// is the value the constraint applies to (the ssa.copy result).
+/// is the value the constraint applies to (the bitcast result).
 struct PredicateConstraint {
   CmpInst::Predicate Predicate;
   Value *OtherOp;
@@ -177,7 +177,6 @@ class PredicateInfo {
 public:
   LLVM_ABI PredicateInfo(Function &, DominatorTree &, AssumptionCache &,
                          BumpPtrAllocator &);
-  LLVM_ABI ~PredicateInfo();
 
   LLVM_ABI void verifyPredicateInfo() const;
 
@@ -200,10 +199,6 @@ class PredicateInfo {
   // the Predicate Info, they belong to the ValueInfo structs in the ValueInfos
   // vector.
   DenseMap<const Value *, const PredicateBase *> PredicateMap;
-  // The set of ssa_copy declarations we created with our custom mangling.
-  SmallSet<AssertingVH<Function>, 20> CreatedDeclarations;
-  // Cache of ssa.copy declaration for a given type.
-  SmallDenseMap<Type *, Function *> DeclarationCache;
 };
 
 /// Printer pass for \c PredicateInfo.

llvm/lib/Transforms/IPO/FunctionSpecialization.cpp

@@ -400,12 +400,6 @@ Constant *InstCostVisitor::visitFreezeInst(FreezeInst &I) {
 Constant *InstCostVisitor::visitCallBase(CallBase &I) {
   assert(LastVisited != KnownConstants.end() && "Invalid iterator!");
 
-  // Look through calls to ssa_copy intrinsics.
-  if (auto *II = dyn_cast<IntrinsicInst>(&I);
-      II && II->getIntrinsicID() == Intrinsic::ssa_copy) {
-    return LastVisited->second;
-  }
-
   Function *F = I.getCalledFunction();
   if (!F || !canConstantFoldCallTo(&I, F))
     return nullptr;
@@ -611,17 +605,15 @@ void FunctionSpecializer::promoteConstantStackValues(Function *F) {
   }
 }
 
-// ssa_copy intrinsics are introduced by the SCCP solver. These intrinsics
-// interfere with the promoteConstantStackValues() optimization.
+// The SCCP solver inserts bitcasts for PredicateInfo. These interfere with the
+// promoteConstantStackValues() optimization.
 static void removeSSACopy(Function &F) {
   for (BasicBlock &BB : F) {
     for (Instruction &Inst : llvm::make_early_inc_range(BB)) {
-      auto *II = dyn_cast<IntrinsicInst>(&Inst);
-      if (!II)
-        continue;
-      if (II->getIntrinsicID() != Intrinsic::ssa_copy)
+      auto *BC = dyn_cast<BitCastInst>(&Inst);
+      if (!BC || BC->getType() != BC->getOperand(0)->getType())
         continue;
-      Inst.replaceAllUsesWith(II->getOperand(0));
+      Inst.replaceAllUsesWith(BC->getOperand(0));
       Inst.eraseFromParent();
     }
   }

llvm/lib/Transforms/Scalar/NewGVN.cpp

@@ -651,7 +651,7 @@ class NewGVN {
   BitVector TouchedInstructions;
 
   DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange;
-  mutable DenseMap<const IntrinsicInst *, const Value *> PredicateSwapChoice;
+  mutable DenseMap<const BitCastInst *, const Value *> PredicateSwapChoice;
 
 #ifndef NDEBUG
   // Debugging for how many times each block and instruction got processed.
@@ -819,7 +819,7 @@ class NewGVN {
                                                  BasicBlock *PHIBlock) const;
   const Expression *performSymbolicAggrValueEvaluation(Instruction *) const;
   ExprResult performSymbolicCmpEvaluation(Instruction *) const;
-  ExprResult performSymbolicPredicateInfoEvaluation(IntrinsicInst *) const;
+  ExprResult performSymbolicPredicateInfoEvaluation(BitCastInst *) const;
 
   // Congruence finding.
   bool someEquivalentDominates(const Instruction *, const Instruction *) const;
@@ -841,7 +841,7 @@ class NewGVN {
   unsigned int getRank(const Value *) const;
   bool shouldSwapOperands(const Value *, const Value *) const;
   bool shouldSwapOperandsForPredicate(const Value *, const Value *,
-                                      const IntrinsicInst *I) const;
+                                      const BitCastInst *I) const;
 
   // Reachability handling.
   void updateReachableEdge(BasicBlock *, BasicBlock *);
@@ -1013,9 +1013,9 @@ void NewGVN::deleteExpression(const Expression *E) const {
 
 // If V is a predicateinfo copy, get the thing it is a copy of.
 static Value *getCopyOf(const Value *V) {
-  if (auto *II = dyn_cast<IntrinsicInst>(V))
-    if (II->getIntrinsicID() == Intrinsic::ssa_copy)
-      return II->getOperand(0);
+  if (auto *BC = dyn_cast<BitCastInst>(V))
+    if (BC->getType() == BC->getOperand(0)->getType())
+      return BC->getOperand(0);
   return nullptr;
 }
 
@@ -1604,7 +1604,7 @@ const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
 }
 
 NewGVN::ExprResult
-NewGVN::performSymbolicPredicateInfoEvaluation(IntrinsicInst *I) const {
+NewGVN::performSymbolicPredicateInfoEvaluation(BitCastInst *I) const {
   auto *PI = PredInfo->getPredicateInfoFor(I);
   if (!PI)
     return ExprResult::none();
@@ -1647,13 +1647,8 @@ NewGVN::performSymbolicPredicateInfoEvaluation(IntrinsicInst *I) const {
 NewGVN::ExprResult NewGVN::performSymbolicCallEvaluation(Instruction *I) const {
   auto *CI = cast<CallInst>(I);
   if (auto *II = dyn_cast<IntrinsicInst>(I)) {
-    // Intrinsics with the returned attribute are copies of arguments.
-    if (auto *ReturnedValue = II->getReturnedArgOperand()) {
-      if (II->getIntrinsicID() == Intrinsic::ssa_copy)
-        if (auto Res = performSymbolicPredicateInfoEvaluation(II))
-          return Res;
+    if (auto *ReturnedValue = II->getReturnedArgOperand())
       return ExprResult::some(createVariableOrConstant(ReturnedValue));
-    }
   }
 
   // FIXME: Currently the calls which may access the thread id may
@@ -2032,6 +2027,12 @@ NewGVN::performSymbolicEvaluation(Instruction *I,
     E = performSymbolicLoadEvaluation(I);
     break;
   case Instruction::BitCast:
+    // Intrinsics with the returned attribute are copies of arguments.
+    if (I->getType() == I->getOperand(0)->getType())
+      if (auto Res =
+              performSymbolicPredicateInfoEvaluation(cast<BitCastInst>(I)))
+        return Res;
+    [[fallthrough]];
   case Instruction::AddrSpaceCast:
   case Instruction::Freeze:
     return createExpression(I);
@@ -4075,8 +4076,7 @@ bool NewGVN::eliminateInstructions(Function &F) {
               if (DominatingLeader != Def) {
                 // Even if the instruction is removed, we still need to update
                 // flags/metadata due to downstreams users of the leader.
-                if (!match(DefI, m_Intrinsic<Intrinsic::ssa_copy>()))
-                  patchReplacementInstruction(DefI, DominatingLeader);
+                patchReplacementInstruction(DefI, DominatingLeader);
 
                 SmallVector<DbgVariableRecord *> DVRUsers;
                 findDbgUsers(DefI, DVRUsers);
@@ -4116,10 +4116,14 @@ bool NewGVN::eliminateInstructions(Function &F) {
 
           Value *DominatingLeader = EliminationStack.back();
 
-          auto *II = dyn_cast<IntrinsicInst>(DominatingLeader);
-          bool isSSACopy = II && II->getIntrinsicID() == Intrinsic::ssa_copy;
-          if (isSSACopy)
-            DominatingLeader = II->getOperand(0);
+          Instruction *SSACopy = nullptr;
+          if (auto *BC = dyn_cast<BitCastInst>(DominatingLeader)) {
+            if (BC->getType() == BC->getOperand(0)->getType() &&
+                PredInfo->getPredicateInfoFor(DominatingLeader)) {
+              SSACopy = BC;
+              DominatingLeader = BC->getOperand(0);
+            }
+          }
 
           // Don't replace our existing users with ourselves.
           if (U->get() == DominatingLeader)
@@ -4145,12 +4149,12 @@ bool NewGVN::eliminateInstructions(Function &F) {
             ProbablyDead.erase(cast<Instruction>(DominatingLeader));
           // For copy instructions, we use their operand as a leader,
           // which means we remove a user of the copy and it may become dead.
-          if (isSSACopy) {
-            auto It = UseCounts.find(II);
+          if (SSACopy) {
+            auto It = UseCounts.find(SSACopy);
             if (It != UseCounts.end()) {
               unsigned &IIUseCount = It->second;
               if (--IIUseCount == 0)
-                ProbablyDead.insert(II);
+                ProbablyDead.insert(SSACopy);
             }
           }
           ++LeaderUseCount;
@@ -4251,7 +4255,7 @@ bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const {
 }
 
 bool NewGVN::shouldSwapOperandsForPredicate(const Value *A, const Value *B,
-                                            const IntrinsicInst *I) const {
+                                            const BitCastInst *I) const {
   if (shouldSwapOperands(A, B)) {
     PredicateSwapChoice[I] = B;
     return true;

llvm/lib/Transforms/Utils/PredicateInfo.cpp

@@ -506,33 +506,19 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
     ValInfo->RenamedOp = (RenameStack.end() - Start) == RenameStack.begin()
                              ? OrigOp
                              : (RenameStack.end() - Start - 1)->Def;
-    auto CreateSSACopy = [this](IRBuilderBase &B, Value *Op,
-                                const Twine &Name = "") {
-      auto It = PI.DeclarationCache.try_emplace(Op->getType());
-      if (It.second) {
-        // The number of named values is used to detect if a new declaration
-        // was added. If so, that declaration is tracked so that it can be
-        // removed when the analysis is done. The corner case were a new
-        // declaration results in a name clash and the old name being renamed
-        // is not considered as that represents an invalid module.
-        auto NumDecls = F.getParent()->getNumNamedValues();
-        Function *IF = Intrinsic::getOrInsertDeclaration(
-            F.getParent(), Intrinsic::ssa_copy, Op->getType());
-        if (NumDecls != F.getParent()->getNumNamedValues())
-          PI.CreatedDeclarations.insert(IF);
-        It.first->second = IF;
-      }
-      return B.CreateCall(It.first->second, Op, Name);
+    auto CreateSSACopy = [](Instruction *InsertPt, Value *Op,
+                            const Twine &Name = "") {
+      // Use a no-op bitcast to represent ssa copy.
+      return new BitCastInst(Op, Op->getType(), Name, InsertPt->getIterator());
     };
     // For edge predicates, we can just place the operand in the block before
     // the terminator. For assume, we have to place it right after the assume
     // to ensure we dominate all uses except assume itself. Always insert
     // right before the terminator or after the assume, so that we insert in
     // proper order in the case of multiple predicateinfo in the same block.
     if (isa<PredicateWithEdge>(ValInfo)) {
-      IRBuilder<> B(getBranchTerminator(ValInfo));
-      CallInst *PIC =
-          CreateSSACopy(B, Op, Op->getName() + "." + Twine(Counter++));
+      BitCastInst *PIC = CreateSSACopy(getBranchTerminator(ValInfo), Op,
+                                       Op->getName() + "." + Twine(Counter++));
       PI.PredicateMap.insert({PIC, ValInfo});
       Result.Def = PIC;
     } else {
@@ -541,8 +527,7 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
              "Should not have gotten here without it being an assume");
       // Insert the predicate directly after the assume. While it also holds
       // directly before it, assume(i1 true) is not a useful fact.
-      IRBuilder<> B(PAssume->AssumeInst->getNextNode());
-      CallInst *PIC = CreateSSACopy(B, Op);
+      BitCastInst *PIC = CreateSSACopy(PAssume->AssumeInst->getNextNode(), Op);
       PI.PredicateMap.insert({PIC, ValInfo});
       Result.Def = PIC;
     }
@@ -710,23 +695,6 @@ PredicateInfo::PredicateInfo(Function &F, DominatorTree &DT,
   Builder.buildPredicateInfo();
 }
 
-// Remove all declarations we created . The PredicateInfo consumers are
-// responsible for remove the ssa_copy calls created.
-PredicateInfo::~PredicateInfo() {
-  // Collect function pointers in set first, as SmallSet uses a SmallVector
-  // internally and we have to remove the asserting value handles first.
-  SmallPtrSet<Function *, 20> FunctionPtrs;
-  for (const auto &F : CreatedDeclarations)
-    FunctionPtrs.insert(&*F);
-  CreatedDeclarations.clear();
-
-  for (Function *F : FunctionPtrs) {
-    assert(F->users().empty() &&
-           "PredicateInfo consumer did not remove all SSA copies.");
-    F->eraseFromParent();
-  }
-}
-
 std::optional<PredicateConstraint> PredicateBase::getConstraint() const {
   switch (Type) {
   case PT_Assume:
@@ -779,15 +747,16 @@ std::optional<PredicateConstraint> PredicateBase::getConstraint() const {
 
 void PredicateInfo::verifyPredicateInfo() const {}
 
-// Replace ssa_copy calls created by PredicateInfo with their operand.
+// Replace bitcasts created by PredicateInfo with their operand.
 static void replaceCreatedSSACopys(PredicateInfo &PredInfo, Function &F) {
   for (Instruction &Inst : llvm::make_early_inc_range(instructions(F))) {
     const auto *PI = PredInfo.getPredicateInfoFor(&Inst);
-    auto *II = dyn_cast<IntrinsicInst>(&Inst);
-    if (!PI || !II || II->getIntrinsicID() != Intrinsic::ssa_copy)
+    if (!PI)
       continue;
 
-    Inst.replaceAllUsesWith(II->getOperand(0));
+    assert(isa<BitCastInst>(Inst) &&
+           Inst.getType() == Inst.getOperand(0)->getType());
+    Inst.replaceAllUsesWith(Inst.getOperand(0));
     Inst.eraseFromParent();
   }
 }

llvm/lib/Transforms/Utils/SCCPSolver.cpp

@@ -777,10 +777,10 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
 
     for (BasicBlock &BB : F) {
       for (Instruction &Inst : llvm::make_early_inc_range(BB)) {
-        if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
-          if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
+        if (auto *BC = dyn_cast<BitCastInst>(&Inst)) {
+          if (BC->getType() == BC->getOperand(0)->getType()) {
             if (It->second->getPredicateInfoFor(&Inst)) {
-              Value *Op = II->getOperand(0);
+              Value *Op = BC->getOperand(0);
               Inst.replaceAllUsesWith(Op);
               Inst.eraseFromParent();
             }
@@ -1413,6 +1413,15 @@ void SCCPInstVisitor::visitCastInst(CastInst &I) {
   if (ValueState[&I].isOverdefined())
     return;
 
+  if (auto *BC = dyn_cast<BitCastInst>(&I)) {
+    if (BC->getType() == BC->getOperand(0)->getType()) {
+      if (const PredicateBase *PI = getPredicateInfoFor(&I)) {
+        handlePredicate(&I, I.getOperand(0), PI);
+        return;
+      }
+    }
+  }
+
   ValueLatticeElement OpSt = getValueState(I.getOperand(0));
   if (OpSt.isUnknownOrUndef())
     return;
@@ -2001,17 +2010,6 @@ void SCCPInstVisitor::handleCallResult(CallBase &CB) {
   Function *F = CB.getCalledFunction();
 
   if (auto *II = dyn_cast<IntrinsicInst>(&CB)) {
-    if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
-      if (ValueState[&CB].isOverdefined())
-        return;
-
-      Value *CopyOf = CB.getOperand(0);
-      const PredicateBase *PI = getPredicateInfoFor(&CB);
-      assert(PI && "Missing predicate info for ssa.copy");
-      handlePredicate(&CB, CopyOf, PI);
-      return;
-    }
-
     if (II->getIntrinsicID() == Intrinsic::vscale) {
       unsigned BitWidth = CB.getType()->getScalarSizeInBits();
       const ConstantRange Result = getVScaleRange(II->getFunction(), BitWidth);

llvm/test/Other/debugcounter-predicateinfo.ll

@@ -8,10 +8,10 @@ define fastcc void @barney() {
 ; CHECK-NEXT:    br label [[BB22:%.*]]
 ; CHECK:       bb22:
 ; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i32 undef, 2
-; CHECK:         [[TMP23_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP23]])
+; CHECK:         [[TMP23_0:%.*]] = bitcast i1 [[TMP23]] to i1
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[BB29:%.*]], label [[BB35:%.*]]
 ; CHECK:       bb29:
-; CHECK:         [[TMP23_0_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP23_0]])
+; CHECK:         [[TMP23_0_1:%.*]] = bitcast i1 [[TMP23_0]] to i1
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[BB33:%.*]], label [[BB35]]
 ; CHECK:       bb33:
 ; CHECK-NEXT:    br i1 [[TMP23_0_1]], label [[BB35]], label [[BB35]]

llvm/test/Transforms/FunctionSpecialization/ssa-copy.ll

@@ -53,17 +53,17 @@ exit4:
 ; PREDINF-NEXT:    br label %[[BLOCK1:.*]]
 ; PREDINF:       [[BLOCK1]]:
 ; PREDINF-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
-; PREDINF:         [[CMP_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[CMP]])
-; PREDINF:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; PREDINF:         [[X_4:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; PREDINF:         [[CMP_0:%.*]] = bitcast i1 [[CMP]] to i1
+; PREDINF:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; PREDINF:         [[X_4:%.*]] = bitcast i32 [[X]] to i32
 ; PREDINF-NEXT:    br i1 [[CMP]], label %[[BLOCK2:.*]], label %[[EXIT1:.*]]
 ; PREDINF:       [[BLOCK2]]:
-; PREDINF:         [[CMP_0_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[CMP_0]])
-; PREDINF:         [[X_0_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X_0]])
-; PREDINF:         [[X_0_3:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X_0]])
+; PREDINF:         [[CMP_0_1:%.*]] = bitcast i1 [[CMP_0]] to i1
+; PREDINF:         [[X_0_1:%.*]] = bitcast i32 [[X_0]] to i32
+; PREDINF:         [[X_0_3:%.*]] = bitcast i32 [[X_0]] to i32
 ; PREDINF-NEXT:    br i1 [[CMP_0]], label %[[BLOCK3:.*]], label %[[EXIT2:.*]]
 ; PREDINF:       [[BLOCK3]]:
-; PREDINF:         [[X_0_1_2:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X_0_1]])
+; PREDINF:         [[X_0_1_2:%.*]] = bitcast i32 [[X_0_1]] to i32
 ; PREDINF-NEXT:    br i1 [[CMP_0_1]], label %[[EXIT4:.*]], label %[[EXIT3:.*]]
 ; PREDINF:       [[EXIT1]]:
 ; PREDINF-NEXT:    ret i32 [[X_4]]