@@ -652,3 +652,163 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
652
652
Term->addMetadata (LLVMContext::MD_prof, BranchWeights);
653
653
}
654
654
}
655
+
656
+ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath (VPlan &Plan) {
657
+ auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
658
+ auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
659
+ RedPhiR->getBackedgeValue ()->getDefiningRecipe ());
660
+ if (!MinMaxR)
661
+ return nullptr ;
662
+
663
+ auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
664
+ if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
665
+ !(RepR && isa<IntrinsicInst>(RepR->getUnderlyingInstr ())))
666
+ return nullptr ;
667
+
668
+ #ifndef NDEBUG
669
+ Intrinsic::ID RdxIntrinsicId =
670
+ RedPhiR->getRecurrenceKind () == RecurKind::FMaxNum ? Intrinsic::maxnum
671
+ : Intrinsic::minnum;
672
+ assert ((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
673
+ cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID () ==
674
+ RdxIntrinsicId) ||
675
+ (RepR &&
676
+ cast<IntrinsicInst>(RepR->getUnderlyingInstr ())->getIntrinsicID () ==
677
+ RdxIntrinsicId) &&
678
+ " Intrinsic did not match recurrence kind" );
679
+ #endif
680
+
681
+ if (MinMaxR->getOperand (0 ) == RedPhiR)
682
+ return MinMaxR->getOperand (1 );
683
+
684
+ assert (MinMaxR->getOperand (1 ) == RedPhiR &&
685
+ " Reduction phi operand expected" );
686
+ return MinMaxR->getOperand (0 );
687
+ };
688
+
689
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion ();
690
+ VPReductionPHIRecipe *RedPhiR = nullptr ;
691
+ bool HasUnsupportedPhi = false ;
692
+ for (auto &R : LoopRegion->getEntryBasicBlock ()->phis ()) {
693
+ if (isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe>(&R))
694
+ continue ;
695
+ auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
696
+ if (!Cur) {
697
+ // TODO: Also support fixed-order recurrence phis.
698
+ HasUnsupportedPhi = true ;
699
+ continue ;
700
+ }
701
+ // For now, only a single reduction is supported.
702
+ // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
703
+ if (RedPhiR)
704
+ return false ;
705
+ if (Cur->getRecurrenceKind () != RecurKind::FMaxNum &&
706
+ Cur->getRecurrenceKind () != RecurKind::FMinNum) {
707
+ HasUnsupportedPhi = true ;
708
+ continue ;
709
+ }
710
+ RedPhiR = Cur;
711
+ }
712
+
713
+ if (!RedPhiR)
714
+ return true ;
715
+
716
+ // We won't be able to resume execution in the scalar tail, if there are
717
+ // unsupported header phis or there is no scalar tail at all, due to
718
+ // tail-folding.
719
+ if (HasUnsupportedPhi || !Plan.hasScalarTail ())
720
+ return false ;
721
+
722
+ VPValue *MinMaxOp = GetMinMaxCompareValue (RedPhiR);
723
+ if (!MinMaxOp)
724
+ return false ;
725
+
726
+ RecurKind RedPhiRK = RedPhiR->getRecurrenceKind ();
727
+ assert ((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
728
+ " unsupported reduction" );
729
+
730
+ // / Check if the vector loop of \p Plan can early exit and restart
731
+ // / execution of last vector iteration in the scalar loop. This requires all
732
+ // / recipes up to early exit point be side-effect free as they are
733
+ // / re-executed. Currently we check that the loop is free of any recipe that
734
+ // / may write to memory. Expected to operate on an early VPlan w/o nested
735
+ // / regions.
736
+ for (VPBlockBase *VPB : vp_depth_first_shallow (
737
+ Plan.getVectorLoopRegion ()->getEntryBasicBlock ())) {
738
+ auto *VPBB = cast<VPBasicBlock>(VPB);
739
+ for (auto &R : *VPBB) {
740
+ if (R.mayWriteToMemory () &&
741
+ !match (&R, m_BranchOnCount (m_VPValue (), m_VPValue ())))
742
+ return false ;
743
+ }
744
+ }
745
+
746
+ VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock ();
747
+ VPBuilder Builder (LatchVPBB->getTerminator ());
748
+ auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator ());
749
+ assert (LatchExitingBranch->getOpcode () == VPInstruction::BranchOnCount &&
750
+ " Unexpected terminator" );
751
+ auto *IsLatchExitTaken =
752
+ Builder.createICmp (CmpInst::ICMP_EQ, LatchExitingBranch->getOperand (0 ),
753
+ LatchExitingBranch->getOperand (1 ));
754
+
755
+ VPValue *IsNaN = Builder.createFCmp (CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp);
756
+ VPValue *AnyNaN = Builder.createNaryOp (VPInstruction::AnyOf, {IsNaN});
757
+ auto *AnyExitTaken =
758
+ Builder.createNaryOp (Instruction::Or, {AnyNaN, IsLatchExitTaken});
759
+ Builder.createNaryOp (VPInstruction::BranchOnCond, AnyExitTaken);
760
+ LatchExitingBranch->eraseFromParent ();
761
+
762
+ // If we exit early due to NaNs, compute the final reduction result based on
763
+ // the reduction phi at the beginning of the last vector iteration.
764
+ auto *RdxResult = find_singleton<VPSingleDefRecipe>(
765
+ RedPhiR->users (), [](VPUser *U, bool ) -> VPSingleDefRecipe * {
766
+ auto *VPI = dyn_cast<VPInstruction>(U);
767
+ if (VPI && VPI->getOpcode () == VPInstruction::ComputeReductionResult)
768
+ return VPI;
769
+ return nullptr ;
770
+ });
771
+
772
+ auto *MiddleVPBB = Plan.getMiddleBlock ();
773
+ Builder.setInsertPoint (MiddleVPBB, MiddleVPBB->begin ());
774
+ auto *NewSel =
775
+ Builder.createSelect (AnyNaN, RedPhiR, RdxResult->getOperand (1 ));
776
+ RdxResult->setOperand (1 , NewSel);
777
+
778
+ auto *ScalarPH = Plan.getScalarPreheader ();
779
+ // Update resume phis for inductions in the scalar preheader. If AnyNaN is
780
+ // true, the resume from the start of the last vector iteration via the
781
+ // canonical IV, otherwise from the original value.
782
+ for (auto &R : ScalarPH->phis ()) {
783
+ auto *ResumeR = cast<VPPhi>(&R);
784
+ VPValue *VecV = ResumeR->getOperand (0 );
785
+ if (VecV == RdxResult)
786
+ continue ;
787
+ if (auto *DerivedIV = dyn_cast<VPDerivedIVRecipe>(VecV)) {
788
+ if (DerivedIV->getNumUsers () == 1 &&
789
+ DerivedIV->getOperand (1 ) == &Plan.getVectorTripCount ()) {
790
+ auto *NewSel = Builder.createSelect (AnyNaN, Plan.getCanonicalIV (),
791
+ &Plan.getVectorTripCount ());
792
+ DerivedIV->moveAfter (&*Builder.getInsertPoint ());
793
+ DerivedIV->setOperand (1 , NewSel);
794
+ continue ;
795
+ }
796
+ }
797
+ // Bail out and abandon the current, partially modified, VPlan if we
798
+ // encounter resume phi that cannot be updated yet.
799
+ if (VecV != &Plan.getVectorTripCount ()) {
800
+ LLVM_DEBUG (dbgs () << " Found resume phi we cannot update for VPlan with "
801
+ " FMaxNum/FMinNum reduction.\n " );
802
+ return false ;
803
+ }
804
+ auto *NewSel = Builder.createSelect (AnyNaN, Plan.getCanonicalIV (), VecV);
805
+ ResumeR->setOperand (0 , NewSel);
806
+ }
807
+
808
+ auto *MiddleTerm = MiddleVPBB->getTerminator ();
809
+ Builder.setInsertPoint (MiddleTerm);
810
+ VPValue *MiddleCond = MiddleTerm->getOperand (0 );
811
+ VPValue *NewCond = Builder.createAnd (MiddleCond, Builder.createNot (AnyNaN));
812
+ MiddleTerm->setOperand (0 , NewCond);
813
+ return true ;
814
+ }
0 commit comments