diff --git a/llvm/include/llvm/CodeGen/ExpandVectorPredication.h b/llvm/include/llvm/CodeGen/ExpandVectorPredication.h index b69adb9db260..c42c644c99e9 100644 --- a/llvm/include/llvm/CodeGen/ExpandVectorPredication.h +++ b/llvm/include/llvm/CodeGen/ExpandVectorPredication.h @@ -13,11 +13,14 @@ namespace llvm { -class ExpandVectorPredicationPass - : public PassInfoMixin { -public: - PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); -}; +class TargetTransformInfo; +class VPIntrinsic; + +/// Expand a vector predication intrinsic. Returns true if the intrinsic was +/// removed/replaced. +bool expandVectorPredicationIntrinsic(VPIntrinsic &VPI, + const TargetTransformInfo &TTI); + } // end namespace llvm #endif // LLVM_CODEGEN_EXPANDVECTORPREDICATION_H diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index cafb9781698a..20273d069bf0 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -513,11 +513,6 @@ namespace llvm { // the corresponding function in a vector library (e.g., SVML, libmvec). FunctionPass *createReplaceWithVeclibLegacyPass(); - /// This pass expands the vector predication intrinsics into unpredicated - /// instructions with selects or just the explicit vector length into the - /// predicate mask. - FunctionPass *createExpandVectorPredicationPass(); - // Expands large div/rem instructions. FunctionPass *createExpandLargeDivRemPass(); diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h index d23d59862abb..c00e425b1319 100644 --- a/llvm/include/llvm/LinkAllPasses.h +++ b/llvm/include/llvm/LinkAllPasses.h @@ -119,7 +119,6 @@ namespace { (void) llvm::createMergeICmpsLegacyPass(); (void) llvm::createExpandLargeDivRemPass(); (void)llvm::createExpandMemCmpLegacyPass(); - (void) llvm::createExpandVectorPredicationPass(); std::string buf; llvm::raw_string_ostream os(buf); (void) llvm::createPrintModulePass(os); diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index e702721c299a..8e669ee57912 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -53,7 +53,6 @@ FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass(TM)) FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass(TM)) FUNCTION_PASS("expand-memcmp", ExpandMemCmpPass(TM)) FUNCTION_PASS("expand-reductions", ExpandReductionsPass()) -FUNCTION_PASS("expandvp", ExpandVectorPredicationPass()) FUNCTION_PASS("gc-lowering", GCLoweringPass()) FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass(TM)) FUNCTION_PASS("interleaved-access", InterleavedAccessPass(TM)) diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp index 97c6ee4773f2..5ffdbcda93e7 100644 --- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp +++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// This pass implements IR expansion for vector predication intrinsics, allowing +// This file implements IR expansion for vector predication intrinsics, allowing // targets to enable vector predication until just before codegen. // //===----------------------------------------------------------------------===// @@ -16,7 +16,6 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/VectorUtils.h" -#include "llvm/CodeGen/Passes.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" @@ -24,8 +23,6 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" -#include "llvm/InitializePasses.h" -#include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" @@ -137,7 +134,6 @@ namespace { // Expansion pass state at function scope. struct CachingVPExpander { - Function &F; const TargetTransformInfo &TTI; /// \returns A (fixed length) vector with ascending integer indices @@ -207,10 +203,10 @@ struct CachingVPExpander { bool UsingTTIOverrides; public: - CachingVPExpander(Function &F, const TargetTransformInfo &TTI) - : F(F), TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {} + CachingVPExpander(const TargetTransformInfo &TTI) + : TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {} - bool expandVectorPredication(); + bool expandVectorPredication(VPIntrinsic &VPI); }; //// CachingVPExpander { @@ -571,7 +567,7 @@ CachingVPExpander::expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder, VPIntrinsic &VPI) { assert(VPI.canIgnoreVectorLengthParam()); - const auto &DL = F.getDataLayout(); + const auto &DL = VPI.getDataLayout(); Value *MaskParam = VPI.getMaskParam(); Value *PtrParam = VPI.getMemoryPointerParam(); @@ -775,15 +771,6 @@ Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) { //// } CachingVPExpander -struct TransformJob { - VPIntrinsic *PI; - TargetTransformInfo::VPLegalization Strategy; - TransformJob(VPIntrinsic *PI, TargetTransformInfo::VPLegalization InitStrat) - : PI(PI), Strategy(InitStrat) {} - - bool isDone() const { return Strategy.shouldDoNothing(); } -}; - void sanitizeStrategy(VPIntrinsic &VPI, VPLegalization &LegalizeStrat) { // Operations with speculatable lanes do not strictly need predication. if (maySpeculateLanes(VPI)) { @@ -821,98 +808,43 @@ CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const { } /// Expand llvm.vp.* intrinsics as requested by \p TTI. -bool CachingVPExpander::expandVectorPredication() { - SmallVector Worklist; +bool CachingVPExpander::expandVectorPredication(VPIntrinsic &VPI) { + auto Strategy = getVPLegalizationStrategy(VPI); + sanitizeStrategy(VPI, Strategy); - // Collect all VPIntrinsics that need expansion and determine their expansion - // strategy. - for (auto &I : instructions(F)) { - auto *VPI = dyn_cast(&I); - if (!VPI) - continue; - auto VPStrat = getVPLegalizationStrategy(*VPI); - sanitizeStrategy(*VPI, VPStrat); - if (!VPStrat.shouldDoNothing()) - Worklist.emplace_back(VPI, VPStrat); + // Transform the EVL parameter. + switch (Strategy.EVLParamStrategy) { + case VPLegalization::Legal: + break; + case VPLegalization::Discard: + discardEVLParameter(VPI); + break; + case VPLegalization::Convert: + if (foldEVLIntoMask(VPI)) + ++NumFoldedVL; + break; } - if (Worklist.empty()) - return false; - // Transform all VPIntrinsics on the worklist. - LLVM_DEBUG(dbgs() << "\n:::: Transforming " << Worklist.size() - << " instructions ::::\n"); - for (TransformJob Job : Worklist) { - // Transform the EVL parameter. - switch (Job.Strategy.EVLParamStrategy) { - case VPLegalization::Legal: - break; - case VPLegalization::Discard: - discardEVLParameter(*Job.PI); - break; - case VPLegalization::Convert: - if (foldEVLIntoMask(*Job.PI)) - ++NumFoldedVL; - break; - } - Job.Strategy.EVLParamStrategy = VPLegalization::Legal; - - // Replace with a non-predicated operation. - switch (Job.Strategy.OpStrategy) { - case VPLegalization::Legal: - break; - case VPLegalization::Discard: - llvm_unreachable("Invalid strategy for operators."); - case VPLegalization::Convert: - expandPredication(*Job.PI); + // Replace with a non-predicated operation. + switch (Strategy.OpStrategy) { + case VPLegalization::Legal: + break; + case VPLegalization::Discard: + llvm_unreachable("Invalid strategy for operators."); + case VPLegalization::Convert: + if (Value *V = expandPredication(VPI); V != &VPI) { ++NumLoweredVPOps; - break; + // Return true if and only if the intrinsic was actually removed. + return true; } - Job.Strategy.OpStrategy = VPLegalization::Legal; - - assert(Job.isDone() && "incomplete transformation"); + break; } - return true; + return false; } -class ExpandVectorPredication : public FunctionPass { -public: - static char ID; - ExpandVectorPredication() : FunctionPass(ID) { - initializeExpandVectorPredicationPass(*PassRegistry::getPassRegistry()); - } - - bool runOnFunction(Function &F) override { - const auto *TTI = &getAnalysis().getTTI(F); - CachingVPExpander VPExpander(F, *TTI); - return VPExpander.expandVectorPredication(); - } - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired(); - AU.setPreservesCFG(); - } -}; } // namespace -char ExpandVectorPredication::ID; -INITIALIZE_PASS_BEGIN(ExpandVectorPredication, "expandvp", - "Expand vector predication intrinsics", false, false) -INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) -INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) -INITIALIZE_PASS_END(ExpandVectorPredication, "expandvp", - "Expand vector predication intrinsics", false, false) - -FunctionPass *llvm::createExpandVectorPredicationPass() { - return new ExpandVectorPredication(); -} - -PreservedAnalyses -ExpandVectorPredicationPass::run(Function &F, FunctionAnalysisManager &AM) { - const auto &TTI = AM.getResult(F); - CachingVPExpander VPExpander(F, TTI); - if (!VPExpander.expandVectorPredication()) - return PreservedAnalyses::all(); - PreservedAnalyses PA; - PA.preserveSet(); - return PA; +bool llvm::expandVectorPredicationIntrinsic(VPIntrinsic &VPI, + const TargetTransformInfo &TTI) { + return CachingVPExpander(TTI).expandVectorPredication(VPI); } diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp index 51dad439f57d..c0e08063be41 100644 --- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp @@ -16,6 +16,7 @@ #include "llvm/Analysis/ObjCARCUtil.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/ExpandVectorPredication.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" @@ -351,6 +352,16 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { return Changed; }); break; +#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \ + case Intrinsic::VPID: +#include "llvm/IR/VPIntrinsics.def" + Changed |= forEachCall(F, [&](CallInst *CI) { + Function *Parent = CI->getParent()->getParent(); + const TargetTransformInfo &TTI = LookupTTI(*Parent); + auto *VPI = cast(CI); + return expandVectorPredicationIntrinsic(*VPI, TTI); + }); + break; case Intrinsic::objc_autorelease: Changed |= lowerObjCCall(F, "objc_autorelease"); break; diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index eb74c6f3e5d4..3a20b340d45f 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -865,11 +865,6 @@ void TargetPassConfig::addIRPasses() { if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining) addPass(createPartiallyInlineLibCallsPass()); - // Expand vector predication intrinsics into standard IR instructions. - // This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction - // passes since it emits those kinds of intrinsics. - addPass(createExpandVectorPredicationPass()); - // Instrument function entry after all inlining. addPass(createPostInlineEntryExitInstrumenterPass()); diff --git a/llvm/test/CodeGen/AArch64/O0-pipeline.ll b/llvm/test/CodeGen/AArch64/O0-pipeline.ll index bfcb9a710f70..ba611493e1a7 100644 --- a/llvm/test/CodeGen/AArch64/O0-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O0-pipeline.ll @@ -22,7 +22,6 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Expand vector predication intrinsics ; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll index 017349aa32af..845634e8e983 100644 --- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll @@ -60,7 +60,6 @@ ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library ; CHECK-NEXT: Partially inline calls to library functions -; CHECK-NEXT: Expand vector predication intrinsics ; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll index 15f23eda241b..ee7f9375bf5d 100644 --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll @@ -43,7 +43,6 @@ ; GCN-O0-NEXT: FunctionPass Manager ; GCN-O0-NEXT: Expand Atomic instructions ; GCN-O0-NEXT: Remove unreachable blocks from the CFG -; GCN-O0-NEXT: Expand vector predication intrinsics ; GCN-O0-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; GCN-O0-NEXT: Scalarize Masked Memory Intrinsics ; GCN-O0-NEXT: Expand reduction intrinsics @@ -222,7 +221,6 @@ ; GCN-O1-NEXT: Constant Hoisting ; GCN-O1-NEXT: Replace intrinsics with calls to vector library ; GCN-O1-NEXT: Partially inline calls to library functions -; GCN-O1-NEXT: Expand vector predication intrinsics ; GCN-O1-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; GCN-O1-NEXT: Scalarize Masked Memory Intrinsics ; GCN-O1-NEXT: Expand reduction intrinsics @@ -508,7 +506,6 @@ ; GCN-O1-OPTS-NEXT: Constant Hoisting ; GCN-O1-OPTS-NEXT: Replace intrinsics with calls to vector library ; GCN-O1-OPTS-NEXT: Partially inline calls to library functions -; GCN-O1-OPTS-NEXT: Expand vector predication intrinsics ; GCN-O1-OPTS-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; GCN-O1-OPTS-NEXT: Scalarize Masked Memory Intrinsics ; GCN-O1-OPTS-NEXT: Expand reduction intrinsics @@ -813,7 +810,6 @@ ; GCN-O2-NEXT: Constant Hoisting ; GCN-O2-NEXT: Replace intrinsics with calls to vector library ; GCN-O2-NEXT: Partially inline calls to library functions -; GCN-O2-NEXT: Expand vector predication intrinsics ; GCN-O2-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; GCN-O2-NEXT: Scalarize Masked Memory Intrinsics ; GCN-O2-NEXT: Expand reduction intrinsics @@ -1126,7 +1122,6 @@ ; GCN-O3-NEXT: Constant Hoisting ; GCN-O3-NEXT: Replace intrinsics with calls to vector library ; GCN-O3-NEXT: Partially inline calls to library functions -; GCN-O3-NEXT: Expand vector predication intrinsics ; GCN-O3-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; GCN-O3-NEXT: Scalarize Masked Memory Intrinsics ; GCN-O3-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/ARM/O3-pipeline.ll b/llvm/test/CodeGen/ARM/O3-pipeline.ll index aa92c7ab4fb9..e74e4f2ad389 100644 --- a/llvm/test/CodeGen/ARM/O3-pipeline.ll +++ b/llvm/test/CodeGen/ARM/O3-pipeline.ll @@ -38,7 +38,6 @@ ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library ; CHECK-NEXT: Partially inline calls to library functions -; CHECK-NEXT: Expand vector predication intrinsics ; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/LoongArch/O0-pipeline.ll b/llvm/test/CodeGen/LoongArch/O0-pipeline.ll index 138f0c81238b..38c1dbcb1075 100644 --- a/llvm/test/CodeGen/LoongArch/O0-pipeline.ll +++ b/llvm/test/CodeGen/LoongArch/O0-pipeline.ll @@ -26,7 +26,6 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Expand vector predication intrinsics ; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll index c5c5342e303c..391888a38daf 100644 --- a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll +++ b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll @@ -61,7 +61,6 @@ ; LAXX-NEXT: Constant Hoisting ; LAXX-NEXT: Replace intrinsics with calls to vector library ; LAXX-NEXT: Partially inline calls to library functions -; LAXX-NEXT: Expand vector predication intrinsics ; LAXX-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; LAXX-NEXT: Scalarize Masked Memory Intrinsics ; LAXX-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/PowerPC/O0-pipeline.ll b/llvm/test/CodeGen/PowerPC/O0-pipeline.ll index f1c9da078a16..70b421f8c0c5 100644 --- a/llvm/test/CodeGen/PowerPC/O0-pipeline.ll +++ b/llvm/test/CodeGen/PowerPC/O0-pipeline.ll @@ -25,7 +25,6 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Expand vector predication intrinsics ; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/PowerPC/O3-pipeline.ll b/llvm/test/CodeGen/PowerPC/O3-pipeline.ll index be0fbf3abcda..f4f492782eb6 100644 --- a/llvm/test/CodeGen/PowerPC/O3-pipeline.ll +++ b/llvm/test/CodeGen/PowerPC/O3-pipeline.ll @@ -62,7 +62,6 @@ ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library ; CHECK-NEXT: Partially inline calls to library functions -; CHECK-NEXT: Expand vector predication intrinsics ; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/RISCV/O0-pipeline.ll b/llvm/test/CodeGen/RISCV/O0-pipeline.ll index fd2ba49ea861..7b2eec894a44 100644 --- a/llvm/test/CodeGen/RISCV/O0-pipeline.ll +++ b/llvm/test/CodeGen/RISCV/O0-pipeline.ll @@ -26,7 +26,6 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Expand vector predication intrinsics ; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll index d6d0cca6ddae..df3c690af155 100644 --- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll +++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll @@ -62,7 +62,6 @@ ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library ; CHECK-NEXT: Partially inline calls to library functions -; CHECK-NEXT: Expand vector predication intrinsics ; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll index 29d3c2795ffb..98b86384b844 100644 --- a/llvm/test/CodeGen/X86/O0-pipeline.ll +++ b/llvm/test/CodeGen/X86/O0-pipeline.ll @@ -26,7 +26,6 @@ ; CHECK-NEXT: Lower Garbage Collection Instructions ; CHECK-NEXT: Shadow Stack GC Lowering ; CHECK-NEXT: Remove unreachable blocks from the CFG -; CHECK-NEXT: Expand vector predication intrinsics ; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll index 275a42e02d74..6fcc1ed068f4 100644 --- a/llvm/test/CodeGen/X86/opt-pipeline.ll +++ b/llvm/test/CodeGen/X86/opt-pipeline.ll @@ -59,7 +59,6 @@ ; CHECK-NEXT: Constant Hoisting ; CHECK-NEXT: Replace intrinsics with calls to vector library ; CHECK-NEXT: Partially inline calls to library functions -; CHECK-NEXT: Expand vector predication intrinsics ; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics diff --git a/llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-fp-intrinsics.ll similarity index 99% rename from llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll rename to llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-fp-intrinsics.ll index bc89ddea6b85..d0b6fe27d823 100644 --- a/llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll +++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-fp-intrinsics.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -expandvp -S < %s | FileCheck %s +; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s define void @vp_fadd_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { ; CHECK-LABEL: define void @vp_fadd_v4f32( diff --git a/llvm/test/CodeGen/Generic/expand-vp-gather-scatter.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-gather-scatter.ll similarity index 99% rename from llvm/test/CodeGen/Generic/expand-vp-gather-scatter.ll rename to llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-gather-scatter.ll index 2e2dba50c845..23096eccc613 100644 --- a/llvm/test/CodeGen/Generic/expand-vp-gather-scatter.ll +++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-gather-scatter.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt --expandvp -S < %s | FileCheck %s +; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s ; Fixed vectors define <4 x i32> @vpgather_v4i32(<4 x ptr> %ptrs, <4 x i1> %m, i32 zeroext %evl) { diff --git a/llvm/test/CodeGen/Generic/expand-vp-load-store.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-load-store.ll similarity index 97% rename from llvm/test/CodeGen/Generic/expand-vp-load-store.ll rename to llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-load-store.ll index 5c6f1e858ce7..c966c3e7de0a 100644 --- a/llvm/test/CodeGen/Generic/expand-vp-load-store.ll +++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-load-store.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt --expandvp -S < %s | FileCheck %s -; RUN: opt --expandvp --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s +; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s +; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s ; Fixed vectors define <2 x i64> @vpload_v2i64(ptr %ptr, <2 x i1> %m, i32 zeroext %evl) { diff --git a/llvm/test/CodeGen/Generic/expand-vp.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp.ll similarity index 97% rename from llvm/test/CodeGen/Generic/expand-vp.ll rename to llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp.ll index 4fee9a533b94..92b36054356b 100644 --- a/llvm/test/CodeGen/Generic/expand-vp.ll +++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp.ll @@ -1,12 +1,12 @@ ; Partial expansion cases (still VP with parameter expansions). -; RUN: opt --expandvp --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=LEGAL_LEGAL -; RUN: opt --expandvp --expandvp-override-evl-transform=Discard --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=DISCARD_LEGAL -; RUN: opt --expandvp --expandvp-override-evl-transform=Convert --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=CONVERT_LEGAL +; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=LEGAL_LEGAL +; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Discard --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=DISCARD_LEGAL +; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Convert --expandvp-override-mask-transform=Legal -S < %s | FileCheck %s --check-prefix=CONVERT_LEGAL ; Full expansion cases (all expanded to non-VP). -; RUN: opt --expandvp --expandvp-override-evl-transform=Discard --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT -; RUN: opt --expandvp -S < %s | FileCheck %s --check-prefix=ALL-CONVERT -; RUN: opt --expandvp --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT -; RUN: opt --expandvp --expandvp-override-evl-transform=Convert --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT +; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Discard --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT +; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s --check-prefix=ALL-CONVERT +; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Legal --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT +; RUN: opt -passes=pre-isel-intrinsic-lowering --expandvp-override-evl-transform=Convert --expandvp-override-mask-transform=Convert -S < %s | FileCheck %s --check-prefix=ALL-CONVERT ; Fixed-width vectors diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp index e7bf192192b6..d3f7c2b7f819 100644 --- a/llvm/tools/llc/llc.cpp +++ b/llvm/tools/llc/llc.cpp @@ -342,7 +342,6 @@ int main(int argc, char **argv) { initializeVectorization(*Registry); initializeScalarizeMaskedMemIntrinLegacyPassPass(*Registry); initializeExpandReductionsPass(*Registry); - initializeExpandVectorPredicationPass(*Registry); initializeHardwareLoopsLegacyPass(*Registry); initializeTransformUtils(*Registry); initializeReplaceWithVeclibLegacyPass(*Registry); diff --git a/llvm/tools/opt/optdriver.cpp b/llvm/tools/opt/optdriver.cpp index b26aeb061851..eb4821d52140 100644 --- a/llvm/tools/opt/optdriver.cpp +++ b/llvm/tools/opt/optdriver.cpp @@ -443,7 +443,6 @@ extern "C" int optMain( initializePostInlineEntryExitInstrumenterPass(Registry); initializeUnreachableBlockElimLegacyPassPass(Registry); initializeExpandReductionsPass(Registry); - initializeExpandVectorPredicationPass(Registry); initializeWasmEHPreparePass(Registry); initializeWriteBitcodePassPass(Registry); initializeReplaceWithVeclibLegacyPass(Registry);