@@ -2358,61 +2358,57 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
2358
2358
}
2359
2359
2360
2360
void VPlanTransforms::simplifyEVLIVs (VPlan &Plan) {
2361
- auto ConvertEVLPhi = [](VPlan &Plan, VPBasicBlock *Entry,
2362
- VPEVLBasedIVPHIRecipe *EVLPhi) {
2363
- using namespace llvm ::VPlanPatternMatch;
2364
- VPValue *EVLIncrement = EVLPhi->getBackedgeValue ();
2365
-
2366
- // Convert EVLPhi to concrete recipe.
2367
- auto *ScalarR = VPBuilder (EVLPhi).createScalarPhi (
2368
- {EVLPhi->getStartValue (), EVLIncrement}, EVLPhi->getDebugLoc (),
2369
- " evl.based.iv" );
2370
- EVLPhi->replaceAllUsesWith (ScalarR);
2371
- EVLPhi->eraseFromParent ();
2372
-
2373
- // Find the latch-exiting block and convert to variable-length stepping.
2374
- // Before: (branch-on-count CanonicalIVInc, VectorTripCount)
2375
- // After: (branch-on-count EVLIVInc, TripCount)
2376
- auto Range =
2377
- VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow (Entry));
2378
- auto It = find_if (Range, [&](VPBasicBlock *VPBB) {
2379
- return any_of (VPBB->successors (),
2380
- [&](VPBlockBase *Succ) { return Succ == Entry; });
2381
- });
2382
- assert ((It != Range.end ()) && " LatchExiting is not found" );
2383
- VPBasicBlock *LatchExiting = *It;
2384
- auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator ());
2385
- VPValue *ScalarIVInc;
2386
- assert (LatchExitingBr &&
2387
- match (LatchExitingBr,
2388
- m_BranchOnCount (m_VPValue (ScalarIVInc),
2389
- m_Specific (&Plan.getVectorTripCount ()))) &&
2390
- " Unexpected terminator in EVL loop" );
2391
- LatchExitingBr->setOperand (1 , Plan.getTripCount ());
2392
- ScalarIVInc->replaceAllUsesWith (EVLIncrement);
2393
- VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe ();
2394
- VPRecipeBase *ScalarIV = IVIncR->getOperand (0 )->getDefiningRecipe ();
2395
- IVIncR->eraseFromParent ();
2396
- ScalarIV->eraseFromParent ();
2397
- };
2398
-
2361
+ using namespace llvm ::VPlanPatternMatch;
2399
2362
// Find EVL loop entries by locating VPEVLBasedIVPHIRecipe
2400
2363
// There should be only one EVL PHI in the entire plan
2401
2364
VPEVLBasedIVPHIRecipe *EVLPhi = nullptr ;
2402
- VPBasicBlock *EVLPhiBlock = nullptr ;
2403
2365
2404
2366
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2405
2367
vp_depth_first_shallow (Plan.getEntry ())))
2406
2368
for (VPRecipeBase &R : VPBB->phis ())
2407
2369
if (auto *PhiR = dyn_cast<VPEVLBasedIVPHIRecipe>(&R)) {
2408
- assert (!EVLPhi && " Found multiple EVL PHIs - only one expected" );
2370
+ assert (!EVLPhi && " Found multiple EVL PHIs. Only one expected" );
2409
2371
EVLPhi = PhiR;
2410
- EVLPhiBlock = VPBB;
2411
2372
}
2412
2373
2413
- // Process the single EVL PHI if found
2414
- if (EVLPhi)
2415
- ConvertEVLPhi (Plan, EVLPhiBlock, EVLPhi);
2374
+ // Early return if no EVL PHI is found
2375
+ if (!EVLPhi)
2376
+ return ;
2377
+
2378
+ VPBasicBlock *Entry = EVLPhi->getParent ();
2379
+ VPValue *EVLIncrement = EVLPhi->getBackedgeValue ();
2380
+
2381
+ // Convert EVLPhi to concrete recipe.
2382
+ auto *ScalarR =
2383
+ VPBuilder (EVLPhi).createScalarPhi ({EVLPhi->getStartValue (), EVLIncrement},
2384
+ EVLPhi->getDebugLoc (), " evl.based.iv" );
2385
+ EVLPhi->replaceAllUsesWith (ScalarR);
2386
+ EVLPhi->eraseFromParent ();
2387
+
2388
+ // Find the latch-exiting block and convert to variable-length stepping.
2389
+ // Before: (branch-on-count CanonicalIVInc, VectorTripCount)
2390
+ // After: (branch-on-count EVLIVInc, TripCount)
2391
+ auto Range =
2392
+ VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow (Entry));
2393
+ auto It = find_if (Range, [&Entry](VPBasicBlock *VPBB) {
2394
+ return any_of (VPBB->successors (),
2395
+ [&Entry](VPBlockBase *Succ) { return Succ == Entry; });
2396
+ });
2397
+ assert ((It != Range.end ()) && " LatchExiting is not found" );
2398
+ VPBasicBlock *LatchExiting = *It;
2399
+ auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator ());
2400
+ VPValue *ScalarIVInc;
2401
+ assert (LatchExitingBr &&
2402
+ match (LatchExitingBr,
2403
+ m_BranchOnCount (m_VPValue (ScalarIVInc),
2404
+ m_Specific (&Plan.getVectorTripCount ()))) &&
2405
+ " Unexpected terminator in EVL loop" );
2406
+ LatchExitingBr->setOperand (1 , Plan.getTripCount ());
2407
+ ScalarIVInc->replaceAllUsesWith (EVLIncrement);
2408
+ VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe ();
2409
+ VPRecipeBase *ScalarIV = IVIncR->getOperand (0 )->getDefiningRecipe ();
2410
+ IVIncR->eraseFromParent ();
2411
+ ScalarIV->eraseFromParent ();
2416
2412
}
2417
2413
2418
2414
void VPlanTransforms::dropPoisonGeneratingRecipes (
0 commit comments