mirror of
https://github.com/llvm/llvm-project.git
synced 2025-04-18 19:06:44 +00:00
[LV] Reduce register usage for scaled reductions (#133090)
This PR accounts for scaled reductions in `calculateRegisterUsage` to reflect the fact that the number of lanes in their output is smaller than the VF. Depends on https://github.com/llvm/llvm-project/pull/126437
This commit is contained in:
Sam Tebbs
GitHub
parent
5b384c3015
commit
b658a2e74a
@ -4878,6 +4878,16 @@ void LoopVectorizationCostModel::collectElementTypesForWidening() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the VF scaling factor applied to the recipe's output, if the recipe has
|
||||
/// one.
|
||||
static unsigned getVFScaleFactor(VPRecipeBase *R) {
|
||||
if (auto *RR = dyn_cast<VPReductionPHIRecipe>(R))
|
||||
return RR->getVFScaleFactor();
|
||||
if (auto *RR = dyn_cast<VPPartialReductionRecipe>(R))
|
||||
return RR->getVFScaleFactor();
|
||||
return 1;
|
||||
}
|
||||
|
||||
/// Estimate the register usage for \p Plan and vectorization factors in \p VFs
|
||||
/// by calculating the highest number of values that are live at a single
|
||||
/// location as a rough estimate. Returns the register usage for each VF in \p
|
||||
@ -5032,10 +5042,19 @@ calculateRegisterUsage(VPlan &Plan, ArrayRef<ElementCount> VFs,
|
||||
// even in the scalar case.
|
||||
RegUsage[ClassID] += 1;
|
||||
} else {
|
||||
// The output from scaled phis and scaled reductions actually has
|
||||
// fewer lanes than the VF.
|
||||
unsigned ScaleFactor = getVFScaleFactor(R);
|
||||
ElementCount VF = VFs[J].divideCoefficientBy(ScaleFactor);
|
||||
LLVM_DEBUG(if (VF != VFs[J]) {
|
||||
dbgs() << "LV(REG): Scaled down VF from " << VFs[J] << " to " << VF
|
||||
<< " for " << *R << "\n";
|
||||
});
|
||||
|
||||
for (VPValue *DefV : R->definedValues()) {
|
||||
Type *ScalarTy = TypeInfo.inferScalarType(DefV);
|
||||
unsigned ClassID = TTI.getRegisterClassForType(true, ScalarTy);
|
||||
RegUsage[ClassID] += GetRegUsage(ScalarTy, VFs[J]);
|
||||
RegUsage[ClassID] += GetRegUsage(ScalarTy, VF);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -9141,8 +9160,8 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
|
||||
if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
|
||||
return tryToWidenMemory(Instr, Operands, Range);
|
||||
|
||||
if (getScalingForReduction(Instr))
|
||||
return tryToCreatePartialReduction(Instr, Operands);
|
||||
if (std::optional<unsigned> ScaleFactor = getScalingForReduction(Instr))
|
||||
return tryToCreatePartialReduction(Instr, Operands, ScaleFactor.value());
|
||||
|
||||
if (!shouldWiden(Instr, Range))
|
||||
return nullptr;
|
||||
@ -9166,7 +9185,8 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
|
||||
|
||||
VPRecipeBase *
|
||||
VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
|
||||
ArrayRef<VPValue *> Operands) {
|
||||
ArrayRef<VPValue *> Operands,
|
||||
unsigned ScaleFactor) {
|
||||
assert(Operands.size() == 2 &&
|
||||
"Unexpected number of operands for partial reduction");
|
||||
|
||||
@ -9199,7 +9219,7 @@ VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
|
||||
BinOp = Builder.createSelect(Mask, BinOp, Zero, Reduction->getDebugLoc());
|
||||
}
|
||||
return new VPPartialReductionRecipe(ReductionOpcode, BinOp, Accumulator,
|
||||
Reduction);
|
||||
ScaleFactor, Reduction);
|
||||
}
|
||||
|
||||
void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
|
||||
|
||||
@ -178,7 +178,8 @@ public:
|
||||
/// Create and return a partial reduction recipe for a reduction instruction
|
||||
/// along with binary operation and reduction phi operands.
|
||||
VPRecipeBase *tryToCreatePartialReduction(Instruction *Reduction,
|
||||
ArrayRef<VPValue *> Operands);
|
||||
ArrayRef<VPValue *> Operands,
|
||||
unsigned ScaleFactor);
|
||||
|
||||
/// Set the recipe created for given ingredient.
|
||||
void setRecipe(Instruction *I, VPRecipeBase *R) {
|
||||
|
||||
@ -64,11 +64,10 @@ static cl::opt<bool> PrintVPlansInDotFormat(
|
||||
#define DEBUG_TYPE "loop-vectorize"
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
raw_ostream &llvm::operator<<(raw_ostream &OS, const VPValue &V) {
|
||||
const VPInstruction *Instr = dyn_cast<VPInstruction>(&V);
|
||||
VPSlotTracker SlotTracker(
|
||||
(Instr && Instr->getParent()) ? Instr->getParent()->getPlan() : nullptr);
|
||||
V.print(OS, SlotTracker);
|
||||
raw_ostream &llvm::operator<<(raw_ostream &OS, const VPRecipeBase &R) {
|
||||
const VPBasicBlock *Parent = R.getParent();
|
||||
VPSlotTracker SlotTracker(Parent ? Parent->getPlan() : nullptr);
|
||||
R.print(OS, "", SlotTracker);
|
||||
return OS;
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -2047,6 +2047,9 @@ public:
|
||||
/// Generate the phi/select nodes.
|
||||
void execute(VPTransformState &State) override;
|
||||
|
||||
/// Get the factor that the VF of this recipe's output should be scaled by.
|
||||
unsigned getVFScaleFactor() const { return VFScaleFactor; }
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
/// Print the recipe.
|
||||
void print(raw_ostream &O, const Twine &Indent,
|
||||
@ -2077,17 +2080,21 @@ public:
|
||||
/// scalar value.
|
||||
class VPPartialReductionRecipe : public VPSingleDefRecipe {
|
||||
unsigned Opcode;
|
||||
/// The divisor by which the VF of this recipe's output should be divided
|
||||
/// during execution.
|
||||
unsigned VFScaleFactor;
|
||||
|
||||
public:
|
||||
VPPartialReductionRecipe(Instruction *ReductionInst, VPValue *Op0,
|
||||
VPValue *Op1)
|
||||
VPValue *Op1, unsigned VFScaleFactor)
|
||||
: VPPartialReductionRecipe(ReductionInst->getOpcode(), Op0, Op1,
|
||||
ReductionInst) {}
|
||||
VFScaleFactor, ReductionInst) {}
|
||||
VPPartialReductionRecipe(unsigned Opcode, VPValue *Op0, VPValue *Op1,
|
||||
unsigned VFScaleFactor,
|
||||
Instruction *ReductionInst = nullptr)
|
||||
: VPSingleDefRecipe(VPDef::VPPartialReductionSC,
|
||||
ArrayRef<VPValue *>({Op0, Op1}), ReductionInst),
|
||||
Opcode(Opcode) {
|
||||
Opcode(Opcode), VFScaleFactor(VFScaleFactor) {
|
||||
[[maybe_unused]] auto *AccumulatorRecipe =
|
||||
getOperand(1)->getDefiningRecipe();
|
||||
assert((isa<VPReductionPHIRecipe>(AccumulatorRecipe) ||
|
||||
@ -2098,7 +2105,7 @@ public:
|
||||
|
||||
VPPartialReductionRecipe *clone() override {
|
||||
return new VPPartialReductionRecipe(Opcode, getOperand(0), getOperand(1),
|
||||
getUnderlyingInstr());
|
||||
VFScaleFactor, getUnderlyingInstr());
|
||||
}
|
||||
|
||||
VP_CLASSOF_IMPL(VPDef::VPPartialReductionSC)
|
||||
@ -2113,6 +2120,9 @@ public:
|
||||
/// Get the binary op's opcode.
|
||||
unsigned getOpcode() const { return Opcode; }
|
||||
|
||||
/// Get the factor that the VF of this recipe's output should be scaled by.
|
||||
unsigned getVFScaleFactor() const { return VFScaleFactor; }
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
/// Print the recipe.
|
||||
void print(raw_ostream &O, const Twine &Indent,
|
||||
|
||||
@ -516,8 +516,8 @@ VPInstruction *VPlanSlp::buildGraph(ArrayRef<VPValue *> Values) {
|
||||
auto *Inst = cast<VPInstruction>(Values[0])->getUnderlyingInstr();
|
||||
auto *VPI = new VPInstruction(Opcode, CombinedOperands, Inst->getDebugLoc());
|
||||
|
||||
LLVM_DEBUG(dbgs() << "Create VPInstruction " << *VPI << " "
|
||||
<< *cast<VPInstruction>(Values[0]) << "\n");
|
||||
LLVM_DEBUG(dbgs() << "Create VPInstruction " << *VPI << " " << Values[0]
|
||||
<< "\n");
|
||||
addCombined(Values, VPI);
|
||||
return VPI;
|
||||
}
|
||||
|
||||
@ -194,7 +194,7 @@ public:
|
||||
typedef DenseMap<Value *, VPValue *> Value2VPValueTy;
|
||||
typedef DenseMap<VPValue *, Value *> VPValue2ValueTy;
|
||||
|
||||
raw_ostream &operator<<(raw_ostream &OS, const VPValue &V);
|
||||
raw_ostream &operator<<(raw_ostream &OS, const VPRecipeBase &R);
|
||||
|
||||
/// This class augments VPValue with operands which provide the inverse def-use
|
||||
/// edges from VPValue's users to their defs.
|
||||
|
||||
@ -770,10 +770,10 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) {
|
||||
; CHECK-INTERLEAVED-LABEL: define i32 @dotp_unrolled(
|
||||
; CHECK-INTERLEAVED-SAME: i32 [[NUM_OUT:%.*]], i64 [[NUM_IN:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
|
||||
; CHECK-INTERLEAVED-NEXT: entry:
|
||||
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[NUM_IN]], 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[NUM_IN]], 32
|
||||
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
|
||||
; CHECK-INTERLEAVED: vector.ph:
|
||||
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[NUM_IN]], 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[NUM_IN]], 32
|
||||
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[NUM_IN]], [[N_MOD_VF]]
|
||||
; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]]
|
||||
; CHECK-INTERLEAVED: vector.body:
|
||||
@ -782,6 +782,10 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) {
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE10:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE16:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE17:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE1:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE11:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = or disjoint i64 [[INDEX]], 1
|
||||
@ -794,45 +798,81 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) {
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP9]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x i8>, ptr [[TMP38]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <16 x i8>, ptr [[TMP14]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = sext <16 x i8> [[WIDE_LOAD4]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = mul nsw <16 x i32> [[TMP15]], [[TMP13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP16]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP17]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = sext <16 x i8> [[WIDE_LOAD5]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP19]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = sext <16 x i8> [[WIDE_LOAD6]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = mul nsw <16 x i32> [[TMP18]], [[TMP20]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP21]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x i8>, ptr [[TMP22]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = sext <16 x i8> [[WIDE_LOAD8]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP24]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = sext <16 x i8> [[WIDE_LOAD9]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP26:%.*]] = mul nsw <16 x i32> [[TMP23]], [[TMP25]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE10]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP26]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP27]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP28:%.*]] = sext <16 x i8> [[WIDE_LOAD11]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP29]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP30:%.*]] = sext <16 x i8> [[WIDE_LOAD12]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP31:%.*]] = mul nsw <16 x i32> [[TMP28]], [[TMP30]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE13]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP31]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
||||
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = sext <16 x i8> [[WIDE_LOAD8]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP16]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = sext <16 x i8> [[WIDE_LOAD9]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = sext <16 x i8> [[WIDE_LOAD10]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = mul nsw <16 x i32> [[TMP17]], [[TMP13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = mul nsw <16 x i32> [[TMP18]], [[TMP14]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE1]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP19]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE11]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP20]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP21]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD13:%.*]] = load <16 x i8>, ptr [[TMP22]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = sext <16 x i8> [[WIDE_LOAD12]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP24:%.*]] = sext <16 x i8> [[WIDE_LOAD13]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP25]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD15:%.*]] = load <16 x i8>, ptr [[TMP26]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP27:%.*]] = sext <16 x i8> [[WIDE_LOAD14]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP28:%.*]] = sext <16 x i8> [[WIDE_LOAD15]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP29:%.*]] = mul nsw <16 x i32> [[TMP23]], [[TMP27]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP30:%.*]] = mul nsw <16 x i32> [[TMP24]], [[TMP28]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE16]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP29]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE17]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP30]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP31:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[TMP31]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD19:%.*]] = load <16 x i8>, ptr [[TMP32]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP33:%.*]] = sext <16 x i8> [[WIDE_LOAD18]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP34:%.*]] = sext <16 x i8> [[WIDE_LOAD19]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP35:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD20:%.*]] = load <16 x i8>, ptr [[TMP35]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD21:%.*]] = load <16 x i8>, ptr [[TMP36]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP37:%.*]] = sext <16 x i8> [[WIDE_LOAD20]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP56:%.*]] = sext <16 x i8> [[WIDE_LOAD21]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP39:%.*]] = mul nsw <16 x i32> [[TMP33]], [[TMP37]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP40:%.*]] = mul nsw <16 x i32> [[TMP34]], [[TMP56]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP39]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP40]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD24:%.*]] = load <16 x i8>, ptr [[TMP41]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD25:%.*]] = load <16 x i8>, ptr [[TMP42]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP43:%.*]] = sext <16 x i8> [[WIDE_LOAD24]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP44:%.*]] = sext <16 x i8> [[WIDE_LOAD25]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 16
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD26:%.*]] = load <16 x i8>, ptr [[TMP45]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD27:%.*]] = load <16 x i8>, ptr [[TMP46]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP47:%.*]] = sext <16 x i8> [[WIDE_LOAD26]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP48:%.*]] = sext <16 x i8> [[WIDE_LOAD27]] to <16 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP49:%.*]] = mul nsw <16 x i32> [[TMP43]], [[TMP47]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP50:%.*]] = mul nsw <16 x i32> [[TMP44]], [[TMP48]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE13]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP49]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE10]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP50]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
||||
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP51]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
|
||||
; CHECK-INTERLEAVED: middle.block:
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP33:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE13]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP34:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE10]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP35:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE7]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP36:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE10]], [[PARTIAL_REDUCE13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP52:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX30:%.*]] = add <4 x i32> [[PARTIAL_REDUCE]], [[PARTIAL_REDUCE7]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP53:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX30]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE17]], [[PARTIAL_REDUCE16]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP54:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX31]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE11]], [[PARTIAL_REDUCE1]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP55:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX32]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[NUM_IN]], [[N_VEC]]
|
||||
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
|
||||
; CHECK-INTERLEAVED: scalar.ph:
|
||||
@ -906,7 +946,6 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) {
|
||||
; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[NUM_IN]], [[N_VEC]]
|
||||
; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
|
||||
; CHECK-MAXBW: scalar.ph:
|
||||
;
|
||||
entry:
|
||||
br label %for.body
|
||||
|
||||
|
||||
@ -3256,65 +3256,65 @@ define dso_local void @not_dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum,
|
||||
; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]]
|
||||
; CHECK-INTERLEAVED: vector.body:
|
||||
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP64:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP58:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP52:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP40:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI10:%.*]] = phi <4 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP34:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI12:%.*]] = phi <4 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI14:%.*]] = phi <4 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD16:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <4 x i8> [[WIDE_LOAD16]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = shl nsw i64 [[INDEX]], 3
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP15]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC24:%.*]] = load <32 x i8>, ptr [[TMP17]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC25:%.*]] = shufflevector <32 x i8> [[WIDE_VEC24]], <32 x i8> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC26:%.*]] = shufflevector <32 x i8> [[WIDE_VEC24]], <32 x i8> poison, <4 x i32> <i32 1, i32 9, i32 17, i32 25>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC27:%.*]] = shufflevector <32 x i8> [[WIDE_VEC24]], <32 x i8> poison, <4 x i32> <i32 2, i32 10, i32 18, i32 26>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC28:%.*]] = shufflevector <32 x i8> [[WIDE_VEC24]], <32 x i8> poison, <4 x i32> <i32 3, i32 11, i32 19, i32 27>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC29:%.*]] = shufflevector <32 x i8> [[WIDE_VEC24]], <32 x i8> poison, <4 x i32> <i32 4, i32 12, i32 20, i32 28>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC30:%.*]] = shufflevector <32 x i8> [[WIDE_VEC24]], <32 x i8> poison, <4 x i32> <i32 5, i32 13, i32 21, i32 29>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC31:%.*]] = shufflevector <32 x i8> [[WIDE_VEC24]], <32 x i8> poison, <4 x i32> <i32 6, i32 14, i32 22, i32 30>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC32:%.*]] = shufflevector <32 x i8> [[WIDE_VEC24]], <32 x i8> poison, <4 x i32> <i32 7, i32 15, i32 23, i32 31>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = sext <4 x i8> [[STRIDED_VEC25]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = mul nsw <4 x i32> [[TMP19]], [[TMP13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP22]] = add <4 x i32> [[TMP21]], [[VEC_PHI14]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = sext <4 x i8> [[STRIDED_VEC26]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP27:%.*]] = mul nsw <4 x i32> [[TMP25]], [[TMP13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP28]] = add <4 x i32> [[TMP27]], [[VEC_PHI12]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP31:%.*]] = sext <4 x i8> [[STRIDED_VEC27]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP33:%.*]] = mul nsw <4 x i32> [[TMP31]], [[TMP13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP34]] = add <4 x i32> [[TMP33]], [[VEC_PHI10]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP37:%.*]] = sext <4 x i8> [[STRIDED_VEC28]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP39:%.*]] = mul nsw <4 x i32> [[TMP37]], [[TMP13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP40]] = add <4 x i32> [[TMP39]], [[VEC_PHI8]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP43:%.*]] = sext <4 x i8> [[STRIDED_VEC29]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP45:%.*]] = mul nsw <4 x i32> [[TMP43]], [[TMP13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP46]] = add <4 x i32> [[TMP45]], [[VEC_PHI6]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP49:%.*]] = sext <4 x i8> [[STRIDED_VEC30]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP51:%.*]] = mul nsw <4 x i32> [[TMP49]], [[TMP13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP52]] = add <4 x i32> [[TMP51]], [[VEC_PHI4]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP55:%.*]] = sext <4 x i8> [[STRIDED_VEC31]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP57:%.*]] = mul nsw <4 x i32> [[TMP55]], [[TMP13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP58]] = add <4 x i32> [[TMP57]], [[VEC_PHI2]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP61:%.*]] = sext <4 x i8> [[STRIDED_VEC32]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP63:%.*]] = mul nsw <4 x i32> [[TMP61]], [[TMP13]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP64]] = add <4 x i32> [[TMP63]], [[VEC_PHI]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP36:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP33:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = shl nsw i64 [[INDEX]], 3
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP11]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC:%.*]] = load <32 x i8>, ptr [[TMP12]], align 1
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC8:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 1, i32 9, i32 17, i32 25>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 2, i32 10, i32 18, i32 26>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 3, i32 11, i32 19, i32 27>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC11:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 4, i32 12, i32 20, i32 28>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC12:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 5, i32 13, i32 21, i32 29>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC13:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 6, i32 14, i32 22, i32 30>
|
||||
; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC14:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 7, i32 15, i32 23, i32 31>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[STRIDED_VEC]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = mul nsw <4 x i32> [[TMP13]], [[TMP10]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP15]] = add <4 x i32> [[TMP14]], [[VEC_PHI7]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = sext <4 x i8> [[STRIDED_VEC8]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = mul nsw <4 x i32> [[TMP16]], [[TMP10]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP18]] = add <4 x i32> [[TMP17]], [[VEC_PHI6]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = sext <4 x i8> [[STRIDED_VEC9]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = mul nsw <4 x i32> [[TMP19]], [[TMP10]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP21]] = add <4 x i32> [[TMP20]], [[VEC_PHI5]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = sext <4 x i8> [[STRIDED_VEC10]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = mul nsw <4 x i32> [[TMP22]], [[TMP10]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP24]] = add <4 x i32> [[TMP23]], [[VEC_PHI4]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = sext <4 x i8> [[STRIDED_VEC11]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP26:%.*]] = mul nsw <4 x i32> [[TMP25]], [[TMP10]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP27]] = add <4 x i32> [[TMP26]], [[VEC_PHI3]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP28:%.*]] = sext <4 x i8> [[STRIDED_VEC12]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP29:%.*]] = mul nsw <4 x i32> [[TMP28]], [[TMP10]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP30]] = add <4 x i32> [[TMP29]], [[VEC_PHI2]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP31:%.*]] = sext <4 x i8> [[STRIDED_VEC13]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP32:%.*]] = mul nsw <4 x i32> [[TMP31]], [[TMP10]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP33]] = add <4 x i32> [[TMP32]], [[VEC_PHI1]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP34:%.*]] = sext <4 x i8> [[STRIDED_VEC14]] to <4 x i32>
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP35:%.*]] = mul nsw <4 x i32> [[TMP34]], [[TMP10]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP36]] = add <4 x i32> [[TMP35]], [[VEC_PHI]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP66:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
||||
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP66]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
||||
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
|
||||
; CHECK-INTERLEAVED: middle.block:
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP38:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP64]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP47:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP58]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP48:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP52]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP46]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP42:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP40]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP50:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP34]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP44:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP28]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP53:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP22]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP38:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP36]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP39:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP33]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP40:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP30]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP27]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP42:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP24]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP43:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP21]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP44:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP18]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[TMP45:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
|
||||
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
|
||||
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
|
||||
; CHECK-INTERLEAVED: scalar.ph:
|
||||
@ -3419,7 +3419,6 @@ define dso_local void @not_dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum,
|
||||
; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
|
||||
; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
|
||||
; CHECK-MAXBW: scalar.ph:
|
||||
;
|
||||
entry:
|
||||
%cmp100 = icmp sgt i32 %n, 0
|
||||
br i1 %cmp100, label %for.body.lr.ph, label %for.cond.cleanup
|
||||
|
||||
@ -68,3 +68,183 @@ loop:
|
||||
exit:
|
||||
ret void
|
||||
}
|
||||
|
||||
define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32 %n) #1 {
|
||||
; CHECK-LABEL: LV: Checking a loop in 'dotp_high_register_pressure' from <stdin>
|
||||
; CHECK: LV(REG): VF = 16
|
||||
; CHECK-NEXT: LV(REG): Found max usage: 2 item
|
||||
; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 3 registers
|
||||
; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 40 registers
|
||||
; CHECK-NEXT: LV(REG): Found invariant usage: 2 item
|
||||
entry:
|
||||
%cmp100 = icmp sgt i32 %n, 0
|
||||
br i1 %cmp100, label %for.body.lr.ph, label %for.cond.cleanup
|
||||
|
||||
for.body.lr.ph: ; preds = %entry
|
||||
%arrayidx13 = getelementptr inbounds nuw i8, ptr %sum, i64 4
|
||||
%gep.b.12 = getelementptr inbounds nuw i8, ptr %sum, i64 8
|
||||
%arrayidx31 = getelementptr inbounds nuw i8, ptr %sum, i64 12
|
||||
%arrayidx40 = getelementptr inbounds nuw i8, ptr %sum, i64 16
|
||||
%arrayidx49 = getelementptr inbounds nuw i8, ptr %sum, i64 20
|
||||
%arrayidx58 = getelementptr inbounds nuw i8, ptr %sum, i64 24
|
||||
%arrayidx67 = getelementptr inbounds nuw i8, ptr %sum, i64 28
|
||||
%sum.promoted = load i32, ptr %sum, align 4
|
||||
%arrayidx13.promoted = load i32, ptr %arrayidx13, align 4
|
||||
%gep.b.12.promoted = load i32, ptr %gep.b.12, align 4
|
||||
%arrayidx31.promoted = load i32, ptr %arrayidx31, align 4
|
||||
%arrayidx40.promoted = load i32, ptr %arrayidx40, align 4
|
||||
%arrayidx49.promoted = load i32, ptr %arrayidx49, align 4
|
||||
%arrayidx58.promoted = load i32, ptr %arrayidx58, align 4
|
||||
%arrayidx67.promoted = load i32, ptr %arrayidx67, align 4
|
||||
%wide.trip.count = zext nneg i32 %n to i64
|
||||
br label %for.body
|
||||
|
||||
for.cond.for.cond.cleanup_crit_edge: ; preds = %for.body
|
||||
%add.lcssa = phi i32 [ %add.1, %for.body ]
|
||||
%add.2.lcssa = phi i32 [ %add.2, %for.body ]
|
||||
%add.3.lcssa = phi i32 [ %add.3, %for.body ]
|
||||
%add.4.lcssa = phi i32 [ %add.4, %for.body ]
|
||||
%add.5.lcssa = phi i32 [ %add.5, %for.body ]
|
||||
%add.6.lcssa = phi i32 [ %add.6, %for.body ]
|
||||
%add.7.lcssa = phi i32 [ %add.7, %for.body ]
|
||||
%add.8.lcssa = phi i32 [ %add.8, %for.body ]
|
||||
store i32 %add.lcssa, ptr %sum, align 4
|
||||
store i32 %add.2.lcssa, ptr %arrayidx13, align 4
|
||||
store i32 %add.3.lcssa, ptr %gep.b.12, align 4
|
||||
store i32 %add.4.lcssa, ptr %arrayidx31, align 4
|
||||
store i32 %add.5.lcssa, ptr %arrayidx40, align 4
|
||||
store i32 %add.6.lcssa, ptr %arrayidx49, align 4
|
||||
store i32 %add.7.lcssa, ptr %arrayidx58, align 4
|
||||
store i32 %add.8.lcssa, ptr %arrayidx67, align 4
|
||||
br label %for.cond.cleanup
|
||||
|
||||
for.cond.cleanup: ; preds = %for.cond.for.cond.cleanup_crit_edge, %entry
|
||||
ret void
|
||||
|
||||
for.body: ; preds = %for.body.lr.ph, %for.body
|
||||
%indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
|
||||
%0 = phi i32 [ %arrayidx67.promoted, %for.body.lr.ph ], [ %add.8, %for.body ]
|
||||
%1 = phi i32 [ %arrayidx58.promoted, %for.body.lr.ph ], [ %add.7, %for.body ]
|
||||
%2 = phi i32 [ %arrayidx49.promoted, %for.body.lr.ph ], [ %add.6, %for.body ]
|
||||
%3 = phi i32 [ %arrayidx40.promoted, %for.body.lr.ph ], [ %add.5, %for.body ]
|
||||
%4 = phi i32 [ %arrayidx31.promoted, %for.body.lr.ph ], [ %add.4, %for.body ]
|
||||
%5 = phi i32 [ %gep.b.12.promoted, %for.body.lr.ph ], [ %add.3, %for.body ]
|
||||
%6 = phi i32 [ %arrayidx13.promoted, %for.body.lr.ph ], [ %add.2, %for.body ]
|
||||
%7 = phi i32 [ %sum.promoted, %for.body.lr.ph ], [ %add.1, %for.body ]
|
||||
%arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
|
||||
%load.a = load i8, ptr %arrayidx, align 1
|
||||
%ext.a = zext i8 %load.a to i32
|
||||
%9 = shl nsw i64 %indvars.iv, 3
|
||||
%gep.b.1 = getelementptr inbounds nuw i8, ptr %b, i64 %9
|
||||
%load.b.1 = load i8, ptr %gep.b.1, align 1
|
||||
%ext.b.1 = sext i8 %load.b.1 to i32
|
||||
%mul.1 = mul nsw i32 %ext.b.1, %ext.a
|
||||
%add.1 = add nsw i32 %mul.1, %7
|
||||
%11 = or disjoint i64 %9, 1
|
||||
%gep.b.2 = getelementptr inbounds nuw i8, ptr %b, i64 %11
|
||||
%load.b.2 = load i8, ptr %gep.b.2, align 1
|
||||
%ext.b.2 = sext i8 %load.b.2 to i32
|
||||
%mul.2 = mul nsw i32 %ext.b.2, %ext.a
|
||||
%add.2 = add nsw i32 %mul.2, %6
|
||||
%13 = or disjoint i64 %9, 2
|
||||
%gep.b.3 = getelementptr inbounds nuw i8, ptr %b, i64 %13
|
||||
%load.b.3 = load i8, ptr %gep.b.3, align 1
|
||||
%ext.b.3 = sext i8 %load.b.3 to i32
|
||||
%mul.3 = mul nsw i32 %ext.b.3, %ext.a
|
||||
%add.3 = add nsw i32 %mul.3, %5
|
||||
%15 = or disjoint i64 %9, 3
|
||||
%gep.b.4 = getelementptr inbounds nuw i8, ptr %b, i64 %15
|
||||
%load.b.4 = load i8, ptr %gep.b.4, align 1
|
||||
%ext.b.4 = sext i8 %load.b.4 to i32
|
||||
%mul.4 = mul nsw i32 %ext.b.4, %ext.a
|
||||
%add.4 = add nsw i32 %mul.4, %4
|
||||
%17 = or disjoint i64 %9, 4
|
||||
%gep.b.5 = getelementptr inbounds nuw i8, ptr %b, i64 %17
|
||||
%load.b.5 = load i8, ptr %gep.b.5, align 1
|
||||
%ext.b.5 = sext i8 %load.b.5 to i32
|
||||
%mul.5 = mul nsw i32 %ext.b.5, %ext.a
|
||||
%add.5 = add nsw i32 %mul.5, %3
|
||||
%19 = or disjoint i64 %9, 5
|
||||
%gep.b.6 = getelementptr inbounds nuw i8, ptr %b, i64 %19
|
||||
%load.b.6 = load i8, ptr %gep.b.6, align 1
|
||||
%ext.b.6 = sext i8 %load.b.6 to i32
|
||||
%mul.6 = mul nsw i32 %ext.b.6, %ext.a
|
||||
%add.6 = add nsw i32 %mul.6, %2
|
||||
%21 = or disjoint i64 %9, 6
|
||||
%gep.b.7 = getelementptr inbounds nuw i8, ptr %b, i64 %21
|
||||
%load.b.7 = load i8, ptr %gep.b.7, align 1
|
||||
%ext.b.7 = sext i8 %load.b.7 to i32
|
||||
%mul.7 = mul nsw i32 %ext.b.7, %ext.a
|
||||
%add.7 = add nsw i32 %mul.7, %1
|
||||
%23 = or disjoint i64 %9, 7
|
||||
%gep.b.8 = getelementptr inbounds nuw i8, ptr %b, i64 %23
|
||||
%load.b.8 = load i8, ptr %gep.b.8, align 1
|
||||
%ext.b.8 = sext i8 %load.b.8 to i32
|
||||
%mul.8 = mul nsw i32 %ext.b.8, %ext.a
|
||||
%add.8 = add nsw i32 %mul.8, %0
|
||||
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
|
||||
%exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
|
||||
br i1 %exitcond.not, label %for.cond.for.cond.cleanup_crit_edge, label %for.body
|
||||
}
|
||||
|
||||
define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) {
|
||||
; CHECK-LABEL: LV: Checking a loop in 'dotp_unrolled' from <stdin>
|
||||
; CHECK: LV(REG): VF = 16
|
||||
; CHECK-NEXT: LV(REG): Found max usage: 2 item
|
||||
; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 9 registers
|
||||
; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 24 registers
|
||||
; CHECK-NEXT: LV(REG): Found invariant usage: 0 item
|
||||
entry:
|
||||
br label %for.body
|
||||
|
||||
for.body: ; preds = %entry, %for.body
|
||||
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
|
||||
%accum3 = phi i32 [ 0, %entry ], [ %add.a3, %for.body ]
|
||||
%accum2 = phi i32 [ 0, %entry ], [ %add.a2, %for.body ]
|
||||
%accum1 = phi i32 [ 0, %entry ], [ %add.a1, %for.body ]
|
||||
%accum0 = phi i32 [ 0, %entry ], [ %add.a0, %for.body ]
|
||||
%gep.a0 = getelementptr inbounds i8, ptr %a, i64 %iv
|
||||
%gep.b0 = getelementptr inbounds i8, ptr %b, i64 %iv
|
||||
%offset.1 = or disjoint i64 %iv, 1
|
||||
%gep.a1 = getelementptr inbounds i8, ptr %a, i64 %offset.1
|
||||
%gep.b1 = getelementptr inbounds i8, ptr %b, i64 %offset.1
|
||||
%offset.2 = or disjoint i64 %iv, 2
|
||||
%gep.a2 = getelementptr inbounds i8, ptr %a, i64 %offset.2
|
||||
%gep.b2 = getelementptr inbounds i8, ptr %b, i64 %offset.2
|
||||
%offset.3 = or disjoint i64 %iv, 3
|
||||
%gep.a3 = getelementptr inbounds i8, ptr %a, i64 %offset.3
|
||||
%gep.b3 = getelementptr inbounds i8, ptr %b, i64 %offset.3
|
||||
%load.a0 = load i8, ptr %gep.a0, align 1
|
||||
%ext.a0 = sext i8 %load.a0 to i32
|
||||
%load.b0 = load i8, ptr %gep.b0, align 1
|
||||
%ext.b0 = sext i8 %load.b0 to i32
|
||||
%mul.a0 = mul nsw i32 %ext.b0, %ext.a0
|
||||
%add.a0 = add nsw i32 %mul.a0, %accum0
|
||||
%load.a1 = load i8, ptr %gep.a1, align 1
|
||||
%ext.a1 = sext i8 %load.a1 to i32
|
||||
%load.b1 = load i8, ptr %gep.b1, align 1
|
||||
%ext.b1 = sext i8 %load.b1 to i32
|
||||
%mul.a1 = mul nsw i32 %ext.a1, %ext.b1
|
||||
%add.a1 = add nsw i32 %mul.a1, %accum1
|
||||
%load.a2 = load i8, ptr %gep.a2, align 1
|
||||
%ext.a2 = sext i8 %load.a2 to i32
|
||||
%load.b2 = load i8, ptr %gep.b2, align 1
|
||||
%ext.b2 = sext i8 %load.b2 to i32
|
||||
%mul.a2 = mul nsw i32 %ext.a2, %ext.b2
|
||||
%add.a2 = add nsw i32 %mul.a2, %accum2
|
||||
%load.a3 = load i8, ptr %gep.a3, align 1
|
||||
%ext.a3 = sext i8 %load.a3 to i32
|
||||
%load.b3 = load i8, ptr %gep.b3, align 1
|
||||
%ext.b3 = sext i8 %load.b3 to i32
|
||||
%mul.a3 = mul nsw i32 %ext.a3, %ext.b3
|
||||
%add.a3 = add nsw i32 %mul.a3, %accum3
|
||||
%iv.next = add nuw nsw i64 %iv, 1
|
||||
%exitcond.not = icmp eq i64 %iv.next, %num_in
|
||||
br i1 %exitcond.not, label %exit, label %for.body
|
||||
|
||||
exit: ; preds = %for.body
|
||||
%result0 = add nsw i32 %add.a0, %add.a1
|
||||
%result1 = add nsw i32 %add.a2, %add.a3
|
||||
%result = add nsw i32 %result0, %result1
|
||||
ret i32 %result
|
||||
}
|
||||
|
||||
@ -212,16 +212,16 @@ define void @test_muladd(ptr noalias nocapture %d1, ptr noalias nocapture readon
|
||||
; AVX1-NEXT: [[TMP43:%.*]] = sext <4 x i16> [[STRIDED_VEC22]] to <4 x i32>
|
||||
; AVX1-NEXT: [[TMP46:%.*]] = mul nsw <4 x i32> [[TMP42]], [[TMP38]]
|
||||
; AVX1-NEXT: [[TMP47:%.*]] = mul nsw <4 x i32> [[TMP43]], [[TMP39]]
|
||||
; AVX1-NEXT: [[TMP48:%.*]] = add nsw <4 x i32> [[TMP46]], [[TMP44]]
|
||||
; AVX1-NEXT: [[TMP49:%.*]] = add nsw <4 x i32> [[TMP47]], [[TMP45]]
|
||||
; AVX1-NEXT: [[TMP52:%.*]] = getelementptr inbounds i32, ptr [[D1:%.*]], i64 [[INDEX]]
|
||||
; AVX1-NEXT: [[TMP56:%.*]] = getelementptr inbounds i32, ptr [[TMP52]], i32 0
|
||||
; AVX1-NEXT: [[TMP57:%.*]] = getelementptr inbounds i32, ptr [[TMP52]], i32 4
|
||||
; AVX1-NEXT: store <4 x i32> [[TMP48]], ptr [[TMP56]], align 4
|
||||
; AVX1-NEXT: store <4 x i32> [[TMP49]], ptr [[TMP57]], align 4
|
||||
; AVX1-NEXT: [[TMP19:%.*]] = add nsw <4 x i32> [[TMP46]], [[TMP44]]
|
||||
; AVX1-NEXT: [[TMP20:%.*]] = add nsw <4 x i32> [[TMP47]], [[TMP45]]
|
||||
; AVX1-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[D1:%.*]], i64 [[INDEX]]
|
||||
; AVX1-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i32 0
|
||||
; AVX1-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i32 4
|
||||
; AVX1-NEXT: store <4 x i32> [[TMP19]], ptr [[TMP25]], align 4
|
||||
; AVX1-NEXT: store <4 x i32> [[TMP20]], ptr [[TMP26]], align 4
|
||||
; AVX1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
||||
; AVX1-NEXT: [[TMP60:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
||||
; AVX1-NEXT: br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
|
||||
; AVX1-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
||||
; AVX1-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
|
||||
; AVX1: middle.block:
|
||||
; AVX1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
|
||||
; AVX1-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user