[InstCombine] Support well-defined recurrences in isGuaranteedNotToBeUndefOrPoison

llvm/include/llvm/Analysis/ValueTracking.h

@@ -50,6 +50,9 @@ constexpr unsigned MaxAnalysisRecursionDepth = 6;
 /// getUnderlyingObject().
 constexpr unsigned MaxLookupSearchDepth = 10;
 
+/// The max limit of the search depth in canFoldFreezeIntoRecurrence().
+constexpr unsigned MaxRecurrenceSearchDepth = 32;
+
 /// Determine which bits of V are known to be either zero or one and return
 /// them in the KnownZero/KnownOne bit sets.
 ///
@@ -773,6 +776,14 @@ LLVM_ABI bool canCreatePoison(const Operator *Op,
 /// impliesPoison returns true.
 LLVM_ABI bool impliesPoison(const Value *ValAssumedPoison, const Value *V);
 
+/// Detect if PN is a recurrence with a start value and some number of backedge
+/// values. We'll check whether we can push the freeze through the backedge
+/// values (possibly dropping poison flags along the way) until we reach the
+/// phi again. In that case, we can move the freeze to the start value.
+LLVM_ABI Use *canFoldFreezeIntoRecurrence(
+    PHINode *PN, const DominatorTree *DT, bool &StartNeedsFreeze,
+    SmallVectorImpl<Instruction *> *DropFlags = nullptr, unsigned Depth = 0);
+
 /// Return true if this function can prove that V does not have undef bits
 /// and is never poison. If V is an aggregate value or vector, check whether
 /// all elements (except padding) are not undef or poison.

llvm/lib/Analysis/ValueTracking.cpp

@@ -7569,6 +7569,69 @@ bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
 
 static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
 
+Use *llvm::canFoldFreezeIntoRecurrence(
+    PHINode *PN, const DominatorTree *DT, bool &StartNeedsFreeze,
+    SmallVectorImpl<Instruction *> *DropFlags, unsigned Depth) {
+  // Detect whether this is a recurrence with a start value and some number of
+  // backedge values. We'll check whether we can push the freeze through the
+  // backedge values (possibly dropping poison flags along the way) until we
+  // reach the phi again. In that case, we can move the freeze to the start
+  // value.
+  Use *StartU = nullptr;
+  SmallVector<Value *> Worklist;
+  for (Use &U : PN->incoming_values()) {
+    if (DT && DT->dominates(PN->getParent(), PN->getIncomingBlock(U))) {
+      // Add backedge value to worklist.
+      Worklist.push_back(U.get());
+      continue;
+    }
+
+    // Don't bother handling multiple start values.
+    if (StartU)
+      return nullptr;
+    StartU = &U;
+  }
+
+  if (!StartU || Worklist.empty())
+    return nullptr; // Not a recurrence.
+
+  Value *StartV = StartU->get();
+  BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
+  StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(
+      StartV, /*AC=*/nullptr, /*CtxI=*/nullptr, /*DT=*/nullptr, Depth);
+  // We can't insert freeze if the start value is the result of the
+  // terminator (e.g. an invoke).
+  if (StartNeedsFreeze && StartBB->getTerminator() == StartV)
+    return nullptr;
+
+  SmallPtrSet<Value *, 32> Visited;
+  while (!Worklist.empty()) {
+    Value *V = Worklist.pop_back_val();
+    if (!Visited.insert(V).second)
+      continue;
+
+    if (Visited.size() > MaxRecurrenceSearchDepth)
+      return nullptr; // Limit the total number of values we inspect.
+
+    // Assume that PN is non-poison, because it will be after the transform.
+    if (V == PN ||
+        isGuaranteedNotToBeUndefOrPoison(V, /*AC=*/nullptr, /*CtxI=*/nullptr,
+                                         /*DT=*/nullptr, Depth))
+      continue;
+
+    Instruction *I = dyn_cast<Instruction>(V);
+    if (!I || canCreateUndefOrPoison(cast<Operator>(I),
+                                     /*ConsiderFlagsAndMetadata*/ false))
+      return nullptr;
+
+    if (DropFlags)
+      DropFlags->push_back(I);
+    append_range(Worklist, I->operands());
+  }
+
+  return StartU;
+}
+
 static bool isGuaranteedNotToBeUndefOrPoison(
     const Value *V, AssumptionCache *AC, const Instruction *CtxI,
     const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
@@ -7657,6 +7720,13 @@ static bool isGuaranteedNotToBeUndefOrPoison(
       }
       if (IsWellDefined)
         return true;
+
+      bool StartNeedsFreeze;
+      if (canFoldFreezeIntoRecurrence(const_cast<PHINode *>(PN), DT,
+                                      StartNeedsFreeze, /*DropFlags=*/nullptr,
+                                      Depth) &&
+          !StartNeedsFreeze)
+        return true;
     } else if (!::canCreateUndefOrPoison(Opr, Kind,
                                          /*ConsiderFlagsAndMetadata*/ true) &&
                all_of(Opr->operands(), OpCheck))

llvm/lib/Transforms/InstCombine/InstructionCombining.cpp

@@ -4933,7 +4933,8 @@ InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating(FreezeInst &OrigFI) {
   // poison.
   Value *MaybePoisonOperand = nullptr;
   for (Value *V : OrigOpInst->operands()) {
-    if (isa<MetadataAsValue>(V) || isGuaranteedNotToBeUndefOrPoison(V) ||
+    if (isa<MetadataAsValue>(V) ||
+        isGuaranteedNotToBeUndefOrPoison(V, &AC, &OrigFI, &DT) ||
         // Treat identical operands as a single operand.
         (MaybePoisonOperand && MaybePoisonOperand == V))
       continue;
@@ -4959,64 +4960,19 @@ InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating(FreezeInst &OrigFI) {
 
 Instruction *InstCombinerImpl::foldFreezeIntoRecurrence(FreezeInst &FI,
                                                         PHINode *PN) {
-  // Detect whether this is a recurrence with a start value and some number of
-  // backedge values. We'll check whether we can push the freeze through the
-  // backedge values (possibly dropping poison flags along the way) until we
-  // reach the phi again. In that case, we can move the freeze to the start
-  // value.
-  Use *StartU = nullptr;
-  SmallVector<Value *> Worklist;
-  for (Use &U : PN->incoming_values()) {
-    if (DT.dominates(PN->getParent(), PN->getIncomingBlock(U))) {
-      // Add backedge value to worklist.
-      Worklist.push_back(U.get());
-      continue;
-    }
-
-    // Don't bother handling multiple start values.
-    if (StartU)
-      return nullptr;
-    StartU = &U;
-  }
-
-  if (!StartU || Worklist.empty())
-    return nullptr; // Not a recurrence.
-
-  Value *StartV = StartU->get();
-  BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
-  bool StartNeedsFreeze = !isGuaranteedNotToBeUndefOrPoison(StartV);
-  // We can't insert freeze if the start value is the result of the
-  // terminator (e.g. an invoke).
-  if (StartNeedsFreeze && StartBB->getTerminator() == StartV)
-    return nullptr;
-
-  SmallPtrSet<Value *, 32> Visited;
+  bool StartNeedsFreeze;
   SmallVector<Instruction *> DropFlags;
-  while (!Worklist.empty()) {
-    Value *V = Worklist.pop_back_val();
-    if (!Visited.insert(V).second)
-      continue;
-
-    if (Visited.size() > 32)
-      return nullptr; // Limit the total number of values we inspect.
-
-    // Assume that PN is non-poison, because it will be after the transform.
-    if (V == PN || isGuaranteedNotToBeUndefOrPoison(V))
-      continue;
-
-    Instruction *I = dyn_cast<Instruction>(V);
-    if (!I || canCreateUndefOrPoison(cast<Operator>(I),
-                                     /*ConsiderFlagsAndMetadata*/ false))
-      return nullptr;
-
-    DropFlags.push_back(I);
-    append_range(Worklist, I->operands());
-  }
+  Use *StartU =
+      canFoldFreezeIntoRecurrence(PN, &DT, StartNeedsFreeze, &DropFlags);
+  if (!StartU)
+    return nullptr;
 
   for (Instruction *I : DropFlags)
     I->dropPoisonGeneratingAnnotations();
 
   if (StartNeedsFreeze) {
+    Value *StartV = StartU->get();
+    BasicBlock *StartBB = PN->getIncomingBlock(*StartU);
     Builder.SetInsertPoint(StartBB->getTerminator());
     Value *FrozenStartV = Builder.CreateFreeze(StartV,
                                                StartV->getName() + ".fr");

llvm/test/Transforms/Attributor/dereferenceable-1.ll

@@ -95,7 +95,7 @@ define void @deref_phi_growing(ptr dereferenceable(4000) %a) {
 ; CHECK:       for.cond:
 ; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    [[A_ADDR_0:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[INCDEC_PTR:%.*]], [[FOR_INC]] ]
-; CHECK-NEXT:    call void @deref_phi_user(ptr nonnull [[A_ADDR_0]])
+; CHECK-NEXT:    call void @deref_phi_user(ptr noundef nonnull [[A_ADDR_0]])
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[A_ADDR_0]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[VAL]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
@@ -146,7 +146,7 @@ define void @deref_phi_shrinking(ptr dereferenceable(4000) %a) {
 ; CHECK:       for.cond:
 ; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    [[A_ADDR_0:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[INCDEC_PTR:%.*]], [[FOR_INC]] ]
-; CHECK-NEXT:    call void @deref_phi_user(ptr nonnull [[A_ADDR_0]])
+; CHECK-NEXT:    call void @deref_phi_user(ptr noundef nonnull [[A_ADDR_0]])
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[A_ADDR_0]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[VAL]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]

llvm/test/Transforms/Attributor/value-simplify.ll

@@ -626,7 +626,7 @@ define internal ptr @test_byval2(ptr byval(%struct.X) %a) {
 ; CHECK-NEXT:    [[A_PRIV:%.*]] = alloca [[STRUCT_X:%.*]], align 8
 ; CHECK-NEXT:    store ptr [[TMP0]], ptr [[A_PRIV]], align 8
 ; CHECK-NEXT:    call void @sync()
-; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[A_PRIV]], align 8
+; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[A_PRIV]], align 8, !invariant.load [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret ptr [[L]]
 ;
   call void @sync()
@@ -1359,7 +1359,7 @@ define internal i32 @ret_speculatable_expr(ptr %mem, i32 %a2) {
 ; CGSCC-SAME: (i32 [[TMP0:%.*]]) #[[ATTR1]] {
 ; CGSCC-NEXT:    [[MEM_PRIV:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    store i32 [[TMP0]], ptr [[MEM_PRIV]], align 4
-; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr [[MEM_PRIV]], align 4
+; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr [[MEM_PRIV]], align 4, !invariant.load [[META0]]
 ; CGSCC-NEXT:    [[MUL:%.*]] = mul i32 [[L]], 13
 ; CGSCC-NEXT:    [[ADD:%.*]] = add i32 [[MUL]], 7
 ; CGSCC-NEXT:    ret i32 [[ADD]]
@@ -1709,3 +1709,7 @@ define i32 @readExtInitZeroInit() {
 ; CGSCC: attributes #[[ATTR17]] = { nosync }
 ; CGSCC: attributes #[[ATTR18]] = { nounwind }
 ;.
+; TUNIT: [[META0]] = !{}
+;.
+; CGSCC: [[META0]] = !{}
+;.

llvm/test/Transforms/InstCombine/freeze-landingpad.ll

@@ -9,19 +9,18 @@ define i32 @propagate_freeze_in_landingpad() personality ptr null {
 ; CHECK:       invoke.bb1:
 ; CHECK-NEXT:    [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[NORMAL_RETURN:%.*]] ]
 ; CHECK-NEXT:    [[RES0:%.*]] = invoke i32 @foo()
-; CHECK-NEXT:    to label [[INVOKE_BB2:%.*]] unwind label [[EXCEPTIONAL_RETURN:%.*]]
+; CHECK-NEXT:            to label [[INVOKE_BB2:%.*]] unwind label [[EXCEPTIONAL_RETURN:%.*]]
 ; CHECK:       invoke.bb2:
 ; CHECK-NEXT:    [[RES1:%.*]] = invoke i32 @foo()
-; CHECK-NEXT:    to label [[NORMAL_RETURN]] unwind label [[EXCEPTIONAL_RETURN]]
+; CHECK-NEXT:            to label [[NORMAL_RETURN]] unwind label [[EXCEPTIONAL_RETURN]]
 ; CHECK:       normal_return:
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[X]], 1
 ; CHECK-NEXT:    br label [[INVOKE_BB1]]
 ; CHECK:       exceptional_return:
 ; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X]], [[INVOKE_BB1]] ], [ 0, [[INVOKE_BB2]] ]
 ; CHECK-NEXT:    [[LANDING_PAD:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT:    cleanup
-; CHECK-NEXT:    [[FR:%.*]] = freeze i32 [[PHI]]
-; CHECK-NEXT:    [[RES:%.*]] = shl i32 [[FR]], 1
+; CHECK-NEXT:            cleanup
+; CHECK-NEXT:    [[RES:%.*]] = shl nuw i32 [[PHI]], 1
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
 entry: