mirror of
https://github.com/llvm/llvm-project.git
synced 2025-04-18 19:16:43 +00:00
[VPlan] Track IsOrdered in VPReductionRecipe, remove use of ILV (NFCI).
Instead of using ILV.useOrderedReductions during ::execute, instead store the information at recipe construction. Another step towards making recipe'::execute independent of legacy ILV.
This commit is contained in:
Florian Hahn
parent
3f16ff4e68
commit
15d11a4de9
@ -573,10 +573,6 @@ public:
|
||||
/// Fix the non-induction PHIs in \p Plan.
|
||||
void fixNonInductionPHIs(VPlan &Plan, VPTransformState &State);
|
||||
|
||||
/// Returns true if the reordering of FP operations is not allowed, but we are
|
||||
/// able to vectorize with strict in-order reductions for the given RdxDesc.
|
||||
bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc);
|
||||
|
||||
/// Create a new phi node for the induction variable \p OrigPhi to resume
|
||||
/// iteration count in the scalar epilogue, from where the vectorized loop
|
||||
/// left off. \p Step is the SCEV-expanded induction step to use. In cases
|
||||
@ -3714,11 +3710,6 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPlan &Plan,
|
||||
}
|
||||
}
|
||||
|
||||
bool InnerLoopVectorizer::useOrderedReductions(
|
||||
const RecurrenceDescriptor &RdxDesc) {
|
||||
return Cost->useOrderedReductions(RdxDesc);
|
||||
}
|
||||
|
||||
void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
|
||||
// We should not collect Scalars more than once per VF. Right now, this
|
||||
// function is called from collectUniformsAndScalars(), which already does
|
||||
@ -9056,8 +9047,9 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
|
||||
if (CM.blockNeedsPredicationForAnyReason(BB))
|
||||
CondOp = RecipeBuilder.getBlockInMask(BB);
|
||||
|
||||
VPReductionRecipe *RedRecipe = new VPReductionRecipe(
|
||||
RdxDesc, CurrentLinkI, PreviousLink, VecOp, CondOp);
|
||||
VPReductionRecipe *RedRecipe =
|
||||
new VPReductionRecipe(RdxDesc, CurrentLinkI, PreviousLink, VecOp,
|
||||
CondOp, CM.useOrderedReductions(RdxDesc));
|
||||
// Append the recipe to the end of the VPBasicBlock because we need to
|
||||
// ensure that it comes after all of it's inputs, including CondOp.
|
||||
// Note that this transformation may leave over dead recipes (including
|
||||
@ -9307,57 +9299,6 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
|
||||
NeedsMaskForGaps);
|
||||
}
|
||||
|
||||
void VPReductionRecipe::execute(VPTransformState &State) {
|
||||
assert(!State.Instance && "Reduction being replicated.");
|
||||
Value *PrevInChain = State.get(getChainOp(), 0, /*IsScalar*/ true);
|
||||
RecurKind Kind = RdxDesc.getRecurrenceKind();
|
||||
bool IsOrdered = State.ILV->useOrderedReductions(RdxDesc);
|
||||
// Propagate the fast-math flags carried by the underlying instruction.
|
||||
IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
|
||||
State.Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
|
||||
for (unsigned Part = 0; Part < State.UF; ++Part) {
|
||||
Value *NewVecOp = State.get(getVecOp(), Part);
|
||||
if (VPValue *Cond = getCondOp()) {
|
||||
Value *NewCond = State.get(Cond, Part, State.VF.isScalar());
|
||||
VectorType *VecTy = dyn_cast<VectorType>(NewVecOp->getType());
|
||||
Type *ElementTy = VecTy ? VecTy->getElementType() : NewVecOp->getType();
|
||||
Value *Iden = RdxDesc.getRecurrenceIdentity(Kind, ElementTy,
|
||||
RdxDesc.getFastMathFlags());
|
||||
if (State.VF.isVector()) {
|
||||
Iden =
|
||||
State.Builder.CreateVectorSplat(VecTy->getElementCount(), Iden);
|
||||
}
|
||||
|
||||
Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, Iden);
|
||||
NewVecOp = Select;
|
||||
}
|
||||
Value *NewRed;
|
||||
Value *NextInChain;
|
||||
if (IsOrdered) {
|
||||
if (State.VF.isVector())
|
||||
NewRed = createOrderedReduction(State.Builder, RdxDesc, NewVecOp,
|
||||
PrevInChain);
|
||||
else
|
||||
NewRed = State.Builder.CreateBinOp(
|
||||
(Instruction::BinaryOps)RdxDesc.getOpcode(Kind), PrevInChain,
|
||||
NewVecOp);
|
||||
PrevInChain = NewRed;
|
||||
} else {
|
||||
PrevInChain = State.get(getChainOp(), Part, /*IsScalar*/ true);
|
||||
NewRed = createTargetReduction(State.Builder, RdxDesc, NewVecOp);
|
||||
}
|
||||
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
|
||||
NextInChain = createMinMaxOp(State.Builder, RdxDesc.getRecurrenceKind(),
|
||||
NewRed, PrevInChain);
|
||||
} else if (IsOrdered)
|
||||
NextInChain = NewRed;
|
||||
else
|
||||
NextInChain = State.Builder.CreateBinOp(
|
||||
(Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, PrevInChain);
|
||||
State.set(this, NextInChain, Part, /*IsScalar*/ true);
|
||||
}
|
||||
}
|
||||
|
||||
void VPReplicateRecipe::execute(VPTransformState &State) {
|
||||
Instruction *UI = getUnderlyingInstr();
|
||||
if (State.Instance) { // Generate a single instance.
|
||||
|
||||
@ -2075,13 +2075,15 @@ public:
|
||||
class VPReductionRecipe : public VPSingleDefRecipe {
|
||||
/// The recurrence decriptor for the reduction in question.
|
||||
const RecurrenceDescriptor &RdxDesc;
|
||||
bool IsOrdered;
|
||||
|
||||
public:
|
||||
VPReductionRecipe(const RecurrenceDescriptor &R, Instruction *I,
|
||||
VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp)
|
||||
VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
|
||||
bool IsOrdered)
|
||||
: VPSingleDefRecipe(VPDef::VPReductionSC,
|
||||
ArrayRef<VPValue *>({ChainOp, VecOp}), I),
|
||||
RdxDesc(R) {
|
||||
RdxDesc(R), IsOrdered(IsOrdered) {
|
||||
if (CondOp)
|
||||
addOperand(CondOp);
|
||||
}
|
||||
@ -2090,7 +2092,7 @@ public:
|
||||
|
||||
VPRecipeBase *clone() override {
|
||||
return new VPReductionRecipe(RdxDesc, getUnderlyingInstr(), getChainOp(),
|
||||
getVecOp(), getCondOp());
|
||||
getVecOp(), getCondOp(), IsOrdered);
|
||||
}
|
||||
|
||||
VP_CLASSOF_IMPL(VPDef::VPReductionSC)
|
||||
|
||||
@ -1518,7 +1518,58 @@ void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent,
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void VPReductionRecipe::execute(VPTransformState &State) {
|
||||
assert(!State.Instance && "Reduction being replicated.");
|
||||
Value *PrevInChain = State.get(getChainOp(), 0, /*IsScalar*/ true);
|
||||
RecurKind Kind = RdxDesc.getRecurrenceKind();
|
||||
// Propagate the fast-math flags carried by the underlying instruction.
|
||||
IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
|
||||
State.Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
|
||||
for (unsigned Part = 0; Part < State.UF; ++Part) {
|
||||
Value *NewVecOp = State.get(getVecOp(), Part);
|
||||
if (VPValue *Cond = getCondOp()) {
|
||||
Value *NewCond = State.get(Cond, Part, State.VF.isScalar());
|
||||
VectorType *VecTy = dyn_cast<VectorType>(NewVecOp->getType());
|
||||
Type *ElementTy = VecTy ? VecTy->getElementType() : NewVecOp->getType();
|
||||
Value *Iden = RdxDesc.getRecurrenceIdentity(Kind, ElementTy,
|
||||
RdxDesc.getFastMathFlags());
|
||||
if (State.VF.isVector()) {
|
||||
Iden = State.Builder.CreateVectorSplat(VecTy->getElementCount(), Iden);
|
||||
}
|
||||
|
||||
Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, Iden);
|
||||
NewVecOp = Select;
|
||||
}
|
||||
Value *NewRed;
|
||||
Value *NextInChain;
|
||||
if (IsOrdered) {
|
||||
if (State.VF.isVector())
|
||||
NewRed = createOrderedReduction(State.Builder, RdxDesc, NewVecOp,
|
||||
PrevInChain);
|
||||
else
|
||||
NewRed = State.Builder.CreateBinOp(
|
||||
(Instruction::BinaryOps)RdxDesc.getOpcode(Kind), PrevInChain,
|
||||
NewVecOp);
|
||||
PrevInChain = NewRed;
|
||||
} else {
|
||||
PrevInChain = State.get(getChainOp(), Part, /*IsScalar*/ true);
|
||||
NewRed = createTargetReduction(State.Builder, RdxDesc, NewVecOp);
|
||||
}
|
||||
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
|
||||
NextInChain = createMinMaxOp(State.Builder, RdxDesc.getRecurrenceKind(),
|
||||
NewRed, PrevInChain);
|
||||
} else if (IsOrdered)
|
||||
NextInChain = NewRed;
|
||||
else
|
||||
NextInChain = State.Builder.CreateBinOp(
|
||||
(Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, PrevInChain);
|
||||
State.set(this, NextInChain, Part, /*IsScalar*/ true);
|
||||
}
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
void VPReductionRecipe::print(raw_ostream &O, const Twine &Indent,
|
||||
VPSlotTracker &SlotTracker) const {
|
||||
O << Indent << "REDUCE ";
|
||||
|
||||
@ -1119,7 +1119,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
|
||||
VPValue VecOp;
|
||||
VPValue CondOp;
|
||||
VPReductionRecipe Recipe(RecurrenceDescriptor(), nullptr, &ChainOp, &CondOp,
|
||||
&VecOp);
|
||||
&VecOp, false);
|
||||
EXPECT_FALSE(Recipe.mayHaveSideEffects());
|
||||
EXPECT_FALSE(Recipe.mayReadFromMemory());
|
||||
EXPECT_FALSE(Recipe.mayWriteToMemory());
|
||||
@ -1287,7 +1287,7 @@ TEST(VPRecipeTest, CastVPReductionRecipeToVPUser) {
|
||||
VPValue VecOp;
|
||||
VPValue CondOp;
|
||||
VPReductionRecipe Recipe(RecurrenceDescriptor(), nullptr, &ChainOp, &CondOp,
|
||||
&VecOp);
|
||||
&VecOp, false);
|
||||
EXPECT_TRUE(isa<VPUser>(&Recipe));
|
||||
VPRecipeBase *BaseR = &Recipe;
|
||||
EXPECT_TRUE(isa<VPUser>(BaseR));
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user