From 312409e464cd0e959e8ef8061144b2e8a4f02ab2 Mon Sep 17 00:00:00 2001 From: Sam Parker Date: Fri, 6 Sep 2019 08:24:41 +0000 Subject: [PATCH] [ARM] MVE Tail Predication The MVE and LOB extensions of Armv8.1m can be combined to enable 'tail predication' which removes the need for a scalar remainder loop after vectorization. Lane predication is performed implicitly via a system register. The effects of predication is described in Section B5.6.3 of the Armv8.1-m Arch Reference Manual, the key points being: - For vector operations that perform reduction across the vector and produce a scalar result, whether the value is accumulated or not. - For non-load instructions, the predicate flags determine if the destination register byte is updated with the new value or if the previous value is preserved. - For vector store instructions, whether the store occurs or not. - For vector load instructions, whether the value that is loaded or whether zeros are written to that element of the destination register. This patch implements a pass that takes a hardware loop, containing masked vector instructions, and converts it something that resembles an MVE tail predicated loop. Currently, if we had code generation, we'd generate a loop in which the VCTP would generate the predicate and VPST would then setup the value of VPR.PO. The loads and stores would be placed in VPT blocks so this is not tail predication, but normal VPT predication with the predicate based upon a element counting induction variable. Further work needs to be done to finally produce a true tail predicated loop. Because only the loads and stores are predicated, in both the LLVM IR and MIR level, we will restrict support to only lane-wise operations (no horizontal reductions). We will perform a final check on MIR during loop finalisation too. Another restriction, specific to MVE, is that all the vector instructions need operate on the same number of elements. This is because predication is performed at the byte level and this is set on entry to the loop, or by the VCTP instead. Differential Revision: https://reviews.llvm.org/D65884 llvm-svn: 371179 --- llvm/include/llvm/IR/IntrinsicsARM.td | 4 + llvm/lib/Target/ARM/ARM.h | 2 + llvm/lib/Target/ARM/ARMTargetMachine.cpp | 5 +- llvm/lib/Target/ARM/CMakeLists.txt | 1 + llvm/lib/Target/ARM/MVETailPredication.cpp | 469 +++++++++++++++++++ .../Thumb2/LowOverheadLoops/basic-tail-pred.ll | 385 ++++++++++++++++ .../test/CodeGen/Thumb2/LowOverheadLoops/nested.ll | 152 +++++++ .../Thumb2/LowOverheadLoops/tail-pred-narrow.ll | 54 +++ .../LowOverheadLoops/tail-pred-pattern-fail.ll | 505 +++++++++++++++++++++ .../Thumb2/LowOverheadLoops/tail-pred-widen.ll | 173 +++++++ .../CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll | 118 +++++ .../Thumb2/LowOverheadLoops/vector-unroll.ll | 118 +++++ 12 files changed, 1985 insertions(+), 1 deletion(-) create mode 100644 llvm/lib/Target/ARM/MVETailPredication.cpp create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-narrow.ll create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-pattern-fail.ll create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll create mode 100644 llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-unroll.ll diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td index 4da29ee..e13da61 100644 --- a/llvm/include/llvm/IR/IntrinsicsARM.td +++ b/llvm/include/llvm/IR/IntrinsicsARM.td @@ -777,6 +777,10 @@ class Neon_Dot_Intrinsic def int_arm_neon_udot : Neon_Dot_Intrinsic; def int_arm_neon_sdot : Neon_Dot_Intrinsic; +def int_arm_vctp8 : Intrinsic<[llvm_v16i1_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_vctp16 : Intrinsic<[llvm_v8i1_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_vctp32 : Intrinsic<[llvm_v4i1_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_vctp64 : Intrinsic<[llvm_v2i1_ty], [llvm_i32_ty], [IntrNoMem]>; // GNU eabi mcount def int_arm_gnu_eabi_mcount : Intrinsic<[], diff --git a/llvm/lib/Target/ARM/ARM.h b/llvm/lib/Target/ARM/ARM.h index bf8ed656..2e6f756 100644 --- a/llvm/lib/Target/ARM/ARM.h +++ b/llvm/lib/Target/ARM/ARM.h @@ -35,6 +35,7 @@ class MachineInstr; class MCInst; class PassRegistry; +Pass *createMVETailPredicationPass(); FunctionPass *createARMLowOverheadLoopsPass(); Pass *createARMParallelDSPPass(); FunctionPass *createARMISelDag(ARMBaseTargetMachine &TM, @@ -67,6 +68,7 @@ void initializeThumb2SizeReducePass(PassRegistry &); void initializeThumb2ITBlockPass(PassRegistry &); void initializeMVEVPTBlockPass(PassRegistry &); void initializeARMLowOverheadLoopsPass(PassRegistry &); +void initializeMVETailPredicationPass(PassRegistry &); } // end namespace llvm diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp index a8b324c..5c8007f 100644 --- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp +++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp @@ -96,6 +96,7 @@ extern "C" void LLVMInitializeARMTarget() { initializeARMExpandPseudoPass(Registry); initializeThumb2SizeReducePass(Registry); initializeMVEVPTBlockPass(Registry); + initializeMVETailPredicationPass(Registry); initializeARMLowOverheadLoopsPass(Registry); } @@ -447,8 +448,10 @@ bool ARMPassConfig::addPreISel() { MergeExternalByDefault)); } - if (TM->getOptLevel() != CodeGenOpt::None) + if (TM->getOptLevel() != CodeGenOpt::None) { addPass(createHardwareLoopsPass()); + addPass(createMVETailPredicationPass()); + } return false; } diff --git a/llvm/lib/Target/ARM/CMakeLists.txt b/llvm/lib/Target/ARM/CMakeLists.txt index 74d4312..5ce28f2 100644 --- a/llvm/lib/Target/ARM/CMakeLists.txt +++ b/llvm/lib/Target/ARM/CMakeLists.txt @@ -52,6 +52,7 @@ add_llvm_target(ARMCodeGen ARMTargetObjectFile.cpp ARMTargetTransformInfo.cpp MLxExpansionPass.cpp + MVETailPredication.cpp MVEVPTBlockPass.cpp Thumb1FrameLowering.cpp Thumb1InstrInfo.cpp diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp new file mode 100644 index 0000000..0079184 --- /dev/null +++ b/llvm/lib/Target/ARM/MVETailPredication.cpp @@ -0,0 +1,469 @@ +//===- MVETailPredication.cpp - MVE Tail Predication ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// Armv8.1m introduced MVE, M-Profile Vector Extension, and low-overhead +/// branches to help accelerate DSP applications. These two extensions can be +/// combined to provide implicit vector predication within a low-overhead loop. +/// The HardwareLoops pass inserts intrinsics identifying loops that the +/// backend will attempt to convert into a low-overhead loop. The vectorizer is +/// responsible for generating a vectorized loop in which the lanes are +/// predicated upon the iteration counter. This pass looks at these predicated +/// vector loops, that are targets for low-overhead loops, and prepares it for +/// code generation. Once the vectorizer has produced a masked loop, there's a +/// couple of final forms: +/// - A tail-predicated loop, with implicit predication. +/// - A loop containing multiple VCPT instructions, predicating multiple VPT +/// blocks of instructions operating on different vector types. + +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopPass.h" +#include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/ScalarEvolutionExpander.h" +#include "llvm/Analysis/ScalarEvolutionExpressions.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/PatternMatch.h" +#include "llvm/Support/Debug.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "ARM.h" +#include "ARMSubtarget.h" + +using namespace llvm; + +#define DEBUG_TYPE "mve-tail-predication" +#define DESC "Transform predicated vector loops to use MVE tail predication" + +static cl::opt +DisableTailPredication("disable-mve-tail-predication", cl::Hidden, + cl::init(true), + cl::desc("Disable MVE Tail Predication")); +namespace { + +class MVETailPredication : public LoopPass { + SmallVector MaskedInsts; + Loop *L = nullptr; + ScalarEvolution *SE = nullptr; + TargetTransformInfo *TTI = nullptr; + +public: + static char ID; + + MVETailPredication() : LoopPass(ID) { } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + AU.addPreserved(); + AU.setPreservesCFG(); + } + + bool runOnLoop(Loop *L, LPPassManager&) override; + +private: + + /// Perform the relevant checks on the loop and convert if possible. + bool TryConvert(Value *TripCount); + + /// Return whether this is a vectorized loop, that contains masked + /// load/stores. + bool IsPredicatedVectorLoop(); + + /// Compute a value for the total number of elements that the predicated + /// loop will process. + Value *ComputeElements(Value *TripCount, VectorType *VecTy); + + /// Is the icmp that generates an i1 vector, based upon a loop counter + /// and a limit that is defined outside the loop. + bool isTailPredicate(Value *Predicate, Value *NumElements); +}; + +} // end namespace + +static bool IsDecrement(Instruction &I) { + auto *Call = dyn_cast(&I); + if (!Call) + return false; + + Intrinsic::ID ID = Call->getIntrinsicID(); + return ID == Intrinsic::loop_decrement_reg; +} + +static bool IsMasked(Instruction *I) { + auto *Call = dyn_cast(I); + if (!Call) + return false; + + Intrinsic::ID ID = Call->getIntrinsicID(); + // TODO: Support gather/scatter expand/compress operations. + return ID == Intrinsic::masked_store || ID == Intrinsic::masked_load; +} + +bool MVETailPredication::runOnLoop(Loop *L, LPPassManager&) { + if (skipLoop(L) || DisableTailPredication) + return false; + + Function &F = *L->getHeader()->getParent(); + auto &TPC = getAnalysis(); + auto &TM = TPC.getTM(); + auto *ST = &TM.getSubtarget(F); + TTI = &getAnalysis().getTTI(F); + SE = &getAnalysis().getSE(); + this->L = L; + + // The MVE and LOB extensions are combined to enable tail-predication, but + // there's nothing preventing us from generating VCTP instructions for v8.1m. + if (!ST->hasMVEIntegerOps() || !ST->hasV8_1MMainlineOps()) { + LLVM_DEBUG(dbgs() << "TP: Not a v8.1m.main+mve target.\n"); + return false; + } + + BasicBlock *Preheader = L->getLoopPreheader(); + if (!Preheader) + return false; + + auto FindLoopIterations = [](BasicBlock *BB) -> IntrinsicInst* { + for (auto &I : *BB) { + auto *Call = dyn_cast(&I); + if (!Call) + continue; + + Intrinsic::ID ID = Call->getIntrinsicID(); + if (ID == Intrinsic::set_loop_iterations || + ID == Intrinsic::test_set_loop_iterations) + return cast(&I); + } + return nullptr; + }; + + // Look for the hardware loop intrinsic that sets the iteration count. + IntrinsicInst *Setup = FindLoopIterations(Preheader); + + // The test.set iteration could live in the pre- preheader. + if (!Setup) { + if (!Preheader->getSinglePredecessor()) + return false; + Setup = FindLoopIterations(Preheader->getSinglePredecessor()); + if (!Setup) + return false; + } + + // Search for the hardware loop intrinic that decrements the loop counter. + IntrinsicInst *Decrement = nullptr; + for (auto *BB : L->getBlocks()) { + for (auto &I : *BB) { + if (IsDecrement(I)) { + Decrement = cast(&I); + break; + } + } + } + + if (!Decrement) + return false; + + LLVM_DEBUG(dbgs() << "TP: Running on Loop: " << *L + << *Setup << "\n" + << *Decrement << "\n"); + bool Changed = TryConvert(Setup->getArgOperand(0)); + return Changed; +} + +bool MVETailPredication::isTailPredicate(Value *V, Value *NumElements) { + // Look for the following: + + // %trip.count.minus.1 = add i32 %N, -1 + // %broadcast.splatinsert10 = insertelement <4 x i32> undef, + // i32 %trip.count.minus.1, i32 0 + // %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, + // <4 x i32> undef, + // <4 x i32> zeroinitializer + // ... + // ... + // %index = phi i32 + // %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + // %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, + // <4 x i32> undef, + // <4 x i32> zeroinitializer + // %induction = add <4 x i32> %broadcast.splat, + // %pred = icmp ule <4 x i32> %induction, %broadcast.splat11 + + // And return whether V == %pred. + + using namespace PatternMatch; + + CmpInst::Predicate Pred; + Instruction *Shuffle = nullptr; + Instruction *Induction = nullptr; + + // The vector icmp + if (!match(V, m_ICmp(Pred, m_Instruction(Induction), + m_Instruction(Shuffle))) || + Pred != ICmpInst::ICMP_ULE || !L->isLoopInvariant(Shuffle)) + return false; + + // First find the stuff outside the loop which is setting up the limit + // vector.... + // The invariant shuffle that broadcast the limit into a vector. + Instruction *Insert = nullptr; + if (!match(Shuffle, m_ShuffleVector(m_Instruction(Insert), m_Undef(), + m_Zero()))) + return false; + + // Insert the limit into a vector. + Instruction *BECount = nullptr; + if (!match(Insert, m_InsertElement(m_Undef(), m_Instruction(BECount), + m_Zero()))) + return false; + + // The limit calculation, backedge count. + Value *TripCount = nullptr; + if (!match(BECount, m_Add(m_Value(TripCount), m_AllOnes()))) + return false; + + if (TripCount != NumElements) + return false; + + // Now back to searching inside the loop body... + // Find the add with takes the index iv and adds a constant vector to it. + Instruction *BroadcastSplat = nullptr; + Constant *Const = nullptr; + if (!match(Induction, m_Add(m_Instruction(BroadcastSplat), + m_Constant(Const)))) + return false; + + // Check that we're adding <0, 1, 2, 3... + if (auto *CDS = dyn_cast(Const)) { + for (unsigned i = 0; i < CDS->getNumElements(); ++i) { + if (CDS->getElementAsInteger(i) != i) + return false; + } + } else + return false; + + // The shuffle which broadcasts the index iv into a vector. + if (!match(BroadcastSplat, m_ShuffleVector(m_Instruction(Insert), m_Undef(), + m_Zero()))) + return false; + + // The insert element which initialises a vector with the index iv. + Instruction *IV = nullptr; + if (!match(Insert, m_InsertElement(m_Undef(), m_Instruction(IV), m_Zero()))) + return false; + + // The index iv. + auto *Phi = dyn_cast(IV); + if (!Phi) + return false; + + // TODO: Don't think we need to check the entry value. + Value *OnEntry = Phi->getIncomingValueForBlock(L->getLoopPreheader()); + if (!match(OnEntry, m_Zero())) + return false; + + Value *InLoop = Phi->getIncomingValueForBlock(L->getLoopLatch()); + unsigned Lanes = cast(Insert->getType())->getNumElements(); + + Instruction *LHS = nullptr; + if (!match(InLoop, m_Add(m_Instruction(LHS), m_SpecificInt(Lanes)))) + return false; + + return LHS == Phi; +} + +static VectorType* getVectorType(IntrinsicInst *I) { + unsigned TypeOp = I->getIntrinsicID() == Intrinsic::masked_load ? 0 : 1; + auto *PtrTy = cast(I->getOperand(TypeOp)->getType()); + return cast(PtrTy->getElementType()); +} + +bool MVETailPredication::IsPredicatedVectorLoop() { + // Check that the loop contains at least one masked load/store intrinsic. + // We only support 'normal' vector instructions - other than masked + // load/stores. + for (auto *BB : L->getBlocks()) { + for (auto &I : *BB) { + if (IsMasked(&I)) { + VectorType *VecTy = getVectorType(cast(&I)); + unsigned Lanes = VecTy->getNumElements(); + unsigned ElementWidth = VecTy->getScalarSizeInBits(); + // MVE vectors are 128-bit, but don't support 128 x i1. + // TODO: Can we support vectors larger than 128-bits? + unsigned MaxWidth = TTI->getRegisterBitWidth(true); + if (Lanes * ElementWidth != MaxWidth || Lanes == MaxWidth) + return false; + MaskedInsts.push_back(cast(&I)); + } else if (auto *Int = dyn_cast(&I)) { + for (auto &U : Int->args()) { + if (isa(U->getType())) + return false; + } + } + } + } + + return !MaskedInsts.empty(); +} + +Value* MVETailPredication::ComputeElements(Value *TripCount, + VectorType *VecTy) { + const SCEV *TripCountSE = SE->getSCEV(TripCount); + ConstantInt *VF = ConstantInt::get(cast(TripCount->getType()), + VecTy->getNumElements()); + + if (VF->equalsInt(1)) + return nullptr; + + // TODO: Support constant trip counts. + auto VisitAdd = [&](const SCEVAddExpr *S) -> const SCEVMulExpr* { + if (auto *Const = dyn_cast(S->getOperand(0))) { + if (Const->getAPInt() != -VF->getValue()) + return nullptr; + } else + return nullptr; + return dyn_cast(S->getOperand(1)); + }; + + auto VisitMul = [&](const SCEVMulExpr *S) -> const SCEVUDivExpr* { + if (auto *Const = dyn_cast(S->getOperand(0))) { + if (Const->getValue() != VF) + return nullptr; + } else + return nullptr; + return dyn_cast(S->getOperand(1)); + }; + + auto VisitDiv = [&](const SCEVUDivExpr *S) -> const SCEV* { + if (auto *Const = dyn_cast(S->getRHS())) { + if (Const->getValue() != VF) + return nullptr; + } else + return nullptr; + + if (auto *RoundUp = dyn_cast(S->getLHS())) { + if (auto *Const = dyn_cast(RoundUp->getOperand(0))) { + if (Const->getAPInt() != (VF->getValue() - 1)) + return nullptr; + } else + return nullptr; + + return RoundUp->getOperand(1); + } + return nullptr; + }; + + // TODO: Can we use SCEV helpers, such as findArrayDimensions, and friends to + // determine the numbers of elements instead? Looks like this is what is used + // for delinearization, but I'm not sure if it can be applied to the + // vectorized form - at least not without a bit more work than I feel + // comfortable with. + + // Search for Elems in the following SCEV: + // (1 + ((-VF + (VF * (((VF - 1) + %Elems) /u VF))) /u VF)) + const SCEV *Elems = nullptr; + if (auto *TC = dyn_cast(TripCountSE)) + if (auto *Div = dyn_cast(TC->getOperand(1))) + if (auto *Add = dyn_cast(Div->getLHS())) + if (auto *Mul = VisitAdd(Add)) + if (auto *Div = VisitMul(Mul)) + if (auto *Res = VisitDiv(Div)) + Elems = Res; + + if (!Elems) + return nullptr; + + Instruction *InsertPt = L->getLoopPreheader()->getTerminator(); + if (!isSafeToExpandAt(Elems, InsertPt, *SE)) + return nullptr; + + auto DL = L->getHeader()->getModule()->getDataLayout(); + SCEVExpander Expander(*SE, DL, "elements"); + return Expander.expandCodeFor(Elems, Elems->getType(), InsertPt); +} + +bool MVETailPredication::TryConvert(Value *TripCount) { + if (!IsPredicatedVectorLoop()) + return false; + + LLVM_DEBUG(dbgs() << "TP: Found predicated vector loop.\n"); + + // Walk through the masked intrinsics and try to find whether the predicate + // operand is generated from an induction variable. + Module *M = L->getHeader()->getModule(); + Type *Ty = IntegerType::get(M->getContext(), 32); + SmallPtrSet Predicates; + + for (auto *I : MaskedInsts) { + Intrinsic::ID ID = I->getIntrinsicID(); + unsigned PredOp = ID == Intrinsic::masked_load ? 2 : 3; + Value *Predicate = I->getArgOperand(PredOp); + if (Predicates.count(Predicate)) + continue; + + VectorType *VecTy = getVectorType(I); + Value *NumElements = ComputeElements(TripCount, VecTy); + if (!NumElements) + continue; + + if (!isTailPredicate(Predicate, NumElements)) { + LLVM_DEBUG(dbgs() << "TP: Not tail predicate: " << *Predicate << "\n"); + continue; + } + + LLVM_DEBUG(dbgs() << "TP: Found tail predicate: " << *Predicate << "\n"); + Predicates.insert(Predicate); + + // Insert a phi to count the number of elements processed by the loop. + IRBuilder<> Builder(L->getHeader()->getFirstNonPHI()); + PHINode *Processed = Builder.CreatePHI(Ty, 2); + Processed->addIncoming(NumElements, L->getLoopPreheader()); + + // Insert the intrinsic to represent the effect of tail predication. + Builder.SetInsertPoint(cast(Predicate)); + ConstantInt *Factor = + ConstantInt::get(cast(Ty), VecTy->getNumElements()); + Intrinsic::ID VCTPID; + switch (VecTy->getNumElements()) { + default: + llvm_unreachable("unexpected number of lanes"); + case 2: VCTPID = Intrinsic::arm_vctp64; break; + case 4: VCTPID = Intrinsic::arm_vctp32; break; + case 8: VCTPID = Intrinsic::arm_vctp16; break; + case 16: VCTPID = Intrinsic::arm_vctp8; break; + } + Function *VCTP = Intrinsic::getDeclaration(M, VCTPID); + // TODO: This add likely already exists in the loop. + Value *Remaining = Builder.CreateSub(Processed, Factor); + Value *TailPredicate = Builder.CreateCall(VCTP, Remaining); + Predicate->replaceAllUsesWith(TailPredicate); + + // Add the incoming value to the new phi. + Processed->addIncoming(Remaining, L->getLoopLatch()); + LLVM_DEBUG(dbgs() << "TP: Insert processed elements phi: " + << *Processed << "\n" + << "TP: Inserted VCTP: " << *TailPredicate << "\n"); + } + + for (auto I : L->blocks()) + DeleteDeadPHIs(I); + + return true; +} + +Pass *llvm::createMVETailPredicationPass() { + return new MVETailPredication(); +} + +char MVETailPredication::ID = 0; + +INITIALIZE_PASS_BEGIN(MVETailPredication, DEBUG_TYPE, DESC, false, false) +INITIALIZE_PASS_END(MVETailPredication, DEBUG_TYPE, DESC, false, false) diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll new file mode 100644 index 0000000..f01e6ab --- /dev/null +++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll @@ -0,0 +1,385 @@ +; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s + +; CHECK-LABEL: mul_v16i8 +; CHECK: vector.body: +; CHECK: %index = phi i32 +; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ] +; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 16 +; CHECK: [[VCTP:%[^ ]+]] = call <16 x i1> @llvm.arm.vctp8(i32 [[REMAINING]]) +; CHECK: [[LD0:%[^ ]+]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* {{.*}}, i32 4, <16 x i1> [[VCTP]], <16 x i8> undef) +; CHECK: [[LD1:%[^ ]+]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* {{.*}}, i32 4, <16 x i1> [[VCTP]], <16 x i8> undef) +; CHECK: tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> {{.*}}, <16 x i8>* {{.*}}, i32 4, <16 x i1> [[VCTP]]) +define dso_local arm_aapcs_vfpcc void @mul_v16i8(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 15 + %tmp9 = lshr i32 %tmp8, 4 + %tmp10 = shl nuw i32 %tmp9, 4 + %tmp11 = add i32 %tmp10, -16 + %tmp12 = lshr i32 %tmp11, 4 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <16 x i32> %broadcast.splatinsert10, <16 x i32> undef, <16 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer + %induction = add <16 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i8, i8* %a, i32 %index + %tmp1 = icmp ule <16 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i8* %tmp to <16 x i8>* + %wide.masked.load = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp2, i32 4, <16 x i1> %tmp1, <16 x i8> undef) + %tmp3 = getelementptr inbounds i8, i8* %b, i32 %index + %tmp4 = bitcast i8* %tmp3 to <16 x i8>* + %wide.masked.load2 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp4, i32 4, <16 x i1> %tmp1, <16 x i8> undef) + %mul = mul nsw <16 x i8> %wide.masked.load2, %wide.masked.load + %tmp6 = getelementptr inbounds i8, i8* %c, i32 %index + %tmp7 = bitcast i8* %tmp6 to <16 x i8>* + tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %mul, <16 x i8>* %tmp7, i32 4, <16 x i1> %tmp1) + %index.next = add i32 %index, 16 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; CHECK-LABEL: mul_v8i16 +; CHECK: vector.body: +; CHECK: %index = phi i32 +; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ] +; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 8 +; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[REMAINING]]) +; CHECK: [[LD0:%[^ ]+]] = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef) +; CHECK: [[LD1:%[^ ]+]] = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef) +; CHECK: tail call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> {{.*}}, <8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]]) +define dso_local arm_aapcs_vfpcc void @mul_v8i16(i16* noalias nocapture readonly %a, i16* noalias nocapture readonly %b, i16* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 7 + %tmp9 = lshr i32 %tmp8, 3 + %tmp10 = shl nuw i32 %tmp9, 3 + %tmp11 = add i32 %tmp10, -8 + %tmp12 = lshr i32 %tmp11, 3 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <8 x i32> %broadcast.splatinsert10, <8 x i32> undef, <8 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer + %induction = add <8 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i16, i16* %a, i32 %index + %tmp1 = icmp ule <8 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i16* %tmp to <8 x i16>* + %wide.masked.load = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp2, i32 4, <8 x i1> %tmp1, <8 x i16> undef) + %tmp3 = getelementptr inbounds i16, i16* %b, i32 %index + %tmp4 = bitcast i16* %tmp3 to <8 x i16>* + %wide.masked.load2 = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> %tmp1, <8 x i16> undef) + %mul = mul nsw <8 x i16> %wide.masked.load2, %wide.masked.load + %tmp6 = getelementptr inbounds i16, i16* %c, i32 %index + %tmp7 = bitcast i16* %tmp6 to <8 x i16>* + tail call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %mul, <8 x i16>* %tmp7, i32 4, <8 x i1> %tmp1) + %index.next = add i32 %index, 8 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; CHECK-LABEL: mul_v4i32 +; CHECK: vector.body: +; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ] +; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4 +; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[REMAINING]]) +; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef) +; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef) +; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]]) +define dso_local arm_aapcs_vfpcc void @mul_v4i32(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load2 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %mul = mul nsw <4 x i32> %wide.masked.load2, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %mul, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; CHECK-LABEL: copy_v2i64 +; CHECK: vector.body: +; CHECK: %index = phi i32 +; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ] +; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 2 +; CHECK: [[VCTP:%[^ ]+]] = call <2 x i1> @llvm.arm.vctp64(i32 [[REMAINING]]) +; CHECK: [[LD0:%[^ ]+]] = tail call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* {{.*}}, i32 4, <2 x i1> [[VCTP]], <2 x i64> undef) +; CHECK: tail call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> [[LD0]], <2 x i64>* {{.*}}, i32 4, <2 x i1> [[VCTP]]) +define void @copy_v2i64(i64* %a, i64* %b, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 1 + %tmp9 = lshr i32 %tmp8, 1 + %tmp10 = shl nuw i32 %tmp9, 1 + %tmp11 = add i32 %tmp10, -2 + %tmp12 = lshr i32 %tmp11, 1 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <2 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <2 x i32> %broadcast.splatinsert10, <2 x i32> undef, <2 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <2 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <2 x i32> %broadcast.splatinsert, <2 x i32> undef, <2 x i32> zeroinitializer + %induction = add <2 x i32> %broadcast.splat, + %tmp1 = icmp ule <2 x i32> %induction, %broadcast.splat11 + %tmp = getelementptr inbounds i64, i64* %a, i32 %index + %tmp2 = bitcast i64* %tmp to <2 x i64>* + %wide.masked.load = tail call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %tmp2, i32 4, <2 x i1> %tmp1, <2 x i64> undef) + %tmp3 = getelementptr inbounds i64, i64* %b, i32 %index + %tmp7 = bitcast i64* %tmp3 to <2 x i64>* + tail call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %wide.masked.load, <2 x i64>* %tmp7, i32 4, <2 x i1> %tmp1) + %index.next = add i32 %index, 2 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; CHECK-LABEL: split_vector +; CHECK: vector.body: +; CHECK: %index = phi i32 +; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ] +; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4 +; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[REMAINING]]) +; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef) +; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef) +; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]]) +define dso_local arm_aapcs_vfpcc void @split_vector(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %extract.1.low = shufflevector <4 x i32> %wide.masked.load, <4 x i32> undef, < 2 x i32> < i32 0, i32 2> + %extract.1.high = shufflevector <4 x i32> %wide.masked.load, <4 x i32> undef, < 2 x i32> < i32 1, i32 3> + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load2 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %extract.2.low = shufflevector <4 x i32> %wide.masked.load2, <4 x i32> undef, < 2 x i32> < i32 0, i32 2> + %extract.2.high = shufflevector <4 x i32> %wide.masked.load2, <4 x i32> undef, < 2 x i32> < i32 1, i32 3> + %mul = mul nsw <2 x i32> %extract.1.low, %extract.2.low + %sub = sub nsw <2 x i32> %extract.1.high, %extract.2.high + %combine = shufflevector <2 x i32> %mul, <2 x i32> %sub, <4 x i32> + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %combine, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; One of the loads now uses ult predicate. +; CHECK-LABEL: mismatch_load_pred +; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ] +; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4 +; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[REMAINING]]) +; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef) +; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> %wrong, <4 x i32> undef) +; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]]) +define dso_local arm_aapcs_vfpcc void @mismatch_load_pred(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %wrong = icmp ult <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %wrong, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; The store now uses ult predicate. +; CHECK-LABEL: mismatch_store_pred +; CHECK: %index = phi i32 +; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ] +; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4 +; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[REMAINING]]) +; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef) +; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef) +; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> %wrong) +define dso_local arm_aapcs_vfpcc void @mismatch_store_pred(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %wrong = icmp ult <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %wrong) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>) +declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>) +declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>) +declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32 immarg, <8 x i1>) +declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) +declare void @llvm.masked.store.v2i64.p0v2i64(<2 x i64>, <2 x i64>*, i32 immarg, <2 x i1>) +declare <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>*, i32 immarg, <2 x i1>, <2 x i64>) +declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>) +declare void @llvm.set.loop.iterations.i32(i32) +declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) + diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll new file mode 100644 index 0000000..a5e9228 --- /dev/null +++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll @@ -0,0 +1,152 @@ +; RUN: opt -mtriple=armv8.1m.main -mattr=+mve -S -mve-tail-predication -disable-mve-tail-predication=false %s -o - | FileCheck %s + +; TODO: Support extending loads +; CHECK-LABEL: mat_vec_sext_i16 +; CHECK-NOT: call {{.*}} @llvm.arm.vctp +define void @mat_vec_sext_i16(i16** nocapture readonly %A, i16* nocapture readonly %B, i32* noalias nocapture %C, i32 %N) { +entry: + %cmp24 = icmp eq i32 %N, 0 + br i1 %cmp24, label %for.cond.cleanup, label %for.cond1.preheader.us.preheader + +for.cond1.preheader.us.preheader: ; preds = %entry + %n.rnd.up = add i32 %N, 3 + %n.vec = and i32 %n.rnd.up, -4 + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert28 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat29 = shufflevector <4 x i32> %broadcast.splatinsert28, <4 x i32> undef, <4 x i32> zeroinitializer + %tmp = add i32 %n.vec, -4 + %tmp1 = lshr i32 %tmp, 2 + %tmp2 = add nuw nsw i32 %tmp1, 1 + br label %for.cond1.preheader.us + +for.cond1.preheader.us: ; preds = %middle.block, %for.cond1.preheader.us.preheader + %i.025.us = phi i32 [ %inc10.us, %middle.block ], [ 0, %for.cond1.preheader.us.preheader ] + %arrayidx.us = getelementptr inbounds i16*, i16** %A, i32 %i.025.us + %tmp3 = load i16*, i16** %arrayidx.us, align 4 + %arrayidx8.us = getelementptr inbounds i32, i32* %C, i32 %i.025.us + %arrayidx8.promoted.us = load i32, i32* %arrayidx8.us, align 4 + %tmp4 = insertelement <4 x i32> , i32 %arrayidx8.promoted.us, i32 0 + call void @llvm.set.loop.iterations.i32(i32 %tmp2) + br label %vector.body + +vector.body: ; preds = %vector.body, %for.cond1.preheader.us + %index = phi i32 [ 0, %for.cond1.preheader.us ], [ %index.next, %vector.body ] + %vec.phi = phi <4 x i32> [ %tmp4, %for.cond1.preheader.us ], [ %tmp14, %vector.body ] + %tmp5 = phi i32 [ %tmp2, %for.cond1.preheader.us ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp6 = getelementptr inbounds i16, i16* %tmp3, i32 %index + %tmp7 = icmp ule <4 x i32> %induction, %broadcast.splat29 + %tmp8 = bitcast i16* %tmp6 to <4 x i16>* + %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %tmp8, i32 2, <4 x i1> %tmp7, <4 x i16> undef) + %tmp9 = sext <4 x i16> %wide.masked.load to <4 x i32> + %tmp10 = getelementptr inbounds i16, i16* %B, i32 %index + %tmp11 = bitcast i16* %tmp10 to <4 x i16>* + %wide.masked.load30 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %tmp11, i32 2, <4 x i1> %tmp7, <4 x i16> undef) + %tmp12 = sext <4 x i16> %wide.masked.load30 to <4 x i32> + %tmp13 = mul nsw <4 x i32> %tmp12, %tmp9 + %tmp14 = add nsw <4 x i32> %tmp13, %vec.phi + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp5, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %middle.block + +middle.block: ; preds = %vector.body + %tmp17 = select <4 x i1> %tmp7, <4 x i32> %tmp14, <4 x i32> %vec.phi + %tmp18 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %tmp17) + store i32 %tmp18, i32* %arrayidx8.us, align 4 + %inc10.us = add nuw i32 %i.025.us, 1 + %exitcond27 = icmp eq i32 %inc10.us, %N + br i1 %exitcond27, label %for.cond.cleanup, label %for.cond1.preheader.us + +for.cond.cleanup: ; preds = %middle.block, %entry + ret void +} + +; CHECK-LABEL: mat_vec_i32 +; CHECK: phi +; CHECK: phi +; CHECK: phi +; CHECK: [[IV:%[^ ]+]] = phi i32 [ %N, %for.cond1.preheader.us ], [ [[REM:%[^ ]+]], %vector.body ] +; CHECK: [[REM]] = sub i32 [[IV]], 4 +; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[REM]]) +; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef) +; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef) +define void @mat_vec_i32(i32** nocapture readonly %A, i32* nocapture readonly %B, i32* noalias nocapture %C, i32 %N) { +entry: + %cmp23 = icmp eq i32 %N, 0 + br i1 %cmp23, label %for.cond.cleanup, label %for.cond1.preheader.us.preheader + +for.cond1.preheader.us.preheader: ; preds = %entry + %n.rnd.up = add i32 %N, 3 + %n.vec = and i32 %n.rnd.up, -4 + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert27 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat28 = shufflevector <4 x i32> %broadcast.splatinsert27, <4 x i32> undef, <4 x i32> zeroinitializer + %tmp = add i32 %n.vec, -4 + %tmp1 = lshr i32 %tmp, 2 + %tmp2 = add nuw nsw i32 %tmp1, 1 + br label %for.cond1.preheader.us + +for.cond1.preheader.us: ; preds = %middle.block, %for.cond1.preheader.us.preheader + %i.024.us = phi i32 [ %inc9.us, %middle.block ], [ 0, %for.cond1.preheader.us.preheader ] + %arrayidx.us = getelementptr inbounds i32*, i32** %A, i32 %i.024.us + %tmp3 = load i32*, i32** %arrayidx.us, align 4 + %arrayidx7.us = getelementptr inbounds i32, i32* %C, i32 %i.024.us + %arrayidx7.promoted.us = load i32, i32* %arrayidx7.us, align 4 + %tmp4 = insertelement <4 x i32> , i32 %arrayidx7.promoted.us, i32 0 + call void @llvm.set.loop.iterations.i32(i32 %tmp2) + br label %vector.body + +vector.body: ; preds = %vector.body, %for.cond1.preheader.us + %index = phi i32 [ 0, %for.cond1.preheader.us ], [ %index.next, %vector.body ] + %vec.phi = phi <4 x i32> [ %tmp4, %for.cond1.preheader.us ], [ %tmp12, %vector.body ] + %tmp5 = phi i32 [ %tmp2, %for.cond1.preheader.us ], [ %tmp13, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp6 = getelementptr inbounds i32, i32* %tmp3, i32 %index + %tmp7 = icmp ule <4 x i32> %induction, %broadcast.splat28 + %tmp8 = bitcast i32* %tmp6 to <4 x i32>* + %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %tmp7, <4 x i32> undef) + %tmp9 = getelementptr inbounds i32, i32* %B, i32 %index + %tmp10 = bitcast i32* %tmp9 to <4 x i32>* + %wide.masked.load29 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp10, i32 4, <4 x i1> %tmp7, <4 x i32> undef) + %tmp11 = mul nsw <4 x i32> %wide.masked.load29, %wide.masked.load + %tmp12 = add nsw <4 x i32> %vec.phi, %tmp11 + %index.next = add i32 %index, 4 + %tmp13 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp5, i32 1) + %tmp14 = icmp ne i32 %tmp13, 0 + br i1 %tmp14, label %vector.body, label %middle.block + +middle.block: ; preds = %vector.body + %tmp15 = select <4 x i1> %tmp7, <4 x i32> %tmp12, <4 x i32> %vec.phi + %tmp16 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %tmp15) + store i32 %tmp16, i32* %arrayidx7.us, align 4 + %inc9.us = add nuw i32 %i.024.us, 1 + %exitcond26 = icmp eq i32 %inc9.us, %N + br i1 %exitcond26, label %for.cond.cleanup, label %for.cond1.preheader.us + +for.cond.cleanup: ; preds = %middle.block, %entry + ret void +} + +; Function Attrs: argmemonly nounwind readonly willreturn +declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) #0 + +; Function Attrs: argmemonly nounwind readonly willreturn +declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32 immarg, <4 x i1>, <4 x i16>) #0 + +; Function Attrs: nounwind readnone willreturn +declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>) #1 + +; Function Attrs: noduplicate nounwind +declare void @llvm.set.loop.iterations.i32(i32) #2 + +; Function Attrs: noduplicate nounwind +declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #2 + +attributes #0 = { argmemonly nounwind readonly willreturn } +attributes #1 = { nounwind readnone willreturn } +attributes #2 = { noduplicate nounwind } diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-narrow.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-narrow.ll new file mode 100644 index 0000000..8c1534b --- /dev/null +++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-narrow.ll @@ -0,0 +1,54 @@ +; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s + +; TODO: We should be able to generate a vctp for the loads. +; CHECK-LABEL: trunc_v4i32_v4i16 +; CHECK-NOT: vcpt +define void @trunc_v4i32_v4i16(i32* readonly %a, i32* readonly %b, i16* %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load2 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %mul = mul nsw <4 x i32> %wide.masked.load2, %wide.masked.load + %trunc = trunc <4 x i32> %mul to <4 x i16> + %tmp6 = getelementptr inbounds i16, i16* %c, i32 %index + %tmp7 = bitcast i16* %tmp6 to <4 x i16>* + tail call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %trunc, <4 x i16>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) +declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32 immarg, <4 x i1>) +declare void @llvm.set.loop.iterations.i32(i32) +declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-pattern-fail.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-pattern-fail.ll new file mode 100644 index 0000000..1926bbe --- /dev/null +++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-pattern-fail.ll @@ -0,0 +1,505 @@ +; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s + +; The following functions should all fail to become tail-predicated. +; CHECK-NOT: call i32 @llvm.arm.vctp + +; trip.count.minus.1 has been inserted into element 1, not 0. +define dso_local arm_aapcs_vfpcc void @wrong_ph_insert_0(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 1 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; The insert isn't using an undef for operand 0. +define dso_local arm_aapcs_vfpcc void @wrong_ph_insert_def(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> , i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; The shuffle uses a defined value for operand 1. +define dso_local arm_aapcs_vfpcc void @wrong_ph_shuffle_1(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> , <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; The shuffle uses a non zero value for operand 2. +define dso_local arm_aapcs_vfpcc void @wrong_ph_shuffle_2(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; %N - 2 +define dso_local arm_aapcs_vfpcc void @trip_count_minus_2(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.2 = add i32 %N, -2 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.2, i32 1 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; index has been inserted at element 1, not 0. +define dso_local arm_aapcs_vfpcc void @wrong_loop_insert(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 1 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +define dso_local arm_aapcs_vfpcc void @wrong_loop_invalid_index_splat(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %incorrect = add i32 %index, 1 + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %incorrect, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; Now using ult, not ule for the vector icmp +define dso_local arm_aapcs_vfpcc void @wrong_pred_opcode(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ult <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; The add in the body uses 1, 2, 3, 4 +define void @wrong_body_broadcast_splat(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; Using a variable for the loop body broadcast. +define void @wrong_body_broadcast_splat_2(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N, <4 x i32> %offsets) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, %offsets + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; adding 5, instead of 4, to index. +define void @wrong_index_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %tmp5, <4 x i32>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 5 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) #1 +declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>) #2 +declare void @llvm.set.loop.iterations.i32(i32) #3 +declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #3 + diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll new file mode 100644 index 0000000..c1b957b --- /dev/null +++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll @@ -0,0 +1,173 @@ +; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s + +; CHECK-LABEL: expand_v8i16_v8i32 +; CHECK-NOT: call i32 @llvm.arm.vctp +define void @expand_v8i16_v8i32(i16* noalias nocapture readonly %a, i16* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 7 + %tmp9 = lshr i32 %tmp8, 3 + %tmp10 = shl nuw i32 %tmp9, 3 + %tmp11 = add i32 %tmp10, -8 + %tmp12 = lshr i32 %tmp11, 3 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <8 x i32> %broadcast.splatinsert10, <8 x i32> undef, <8 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer + %induction = add <8 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i16, i16* %a, i32 %index + %tmp1 = icmp ule <8 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i16* %tmp to <8 x i16>* + %wide.masked.load = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp2, i32 4, <8 x i1> %tmp1, <8 x i16> undef) + %tmp3 = getelementptr inbounds i16, i16* %b, i32 %index + %tmp4 = bitcast i16* %tmp3 to <8 x i16>* + %wide.masked.load2 = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> %tmp1, <8 x i16> undef) + %expand.1 = zext <8 x i16> %wide.masked.load to <8 x i32> + %expand.2 = zext <8 x i16> %wide.masked.load2 to <8 x i32> + %mul = mul nsw <8 x i32> %expand.2, %expand.1 + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %index + %tmp7 = bitcast i32* %tmp6 to <8 x i32>* + tail call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> %mul, <8 x i32>* %tmp7, i32 4, <8 x i1> %tmp1) + %index.next = add i32 %index, 8 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; CHECK-LABEL: expand_v8i16_v4i32 +; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[ELEMS_REM:%[^ ]+]], %vector.body ] +; CHECK: [[ELEMS_REM]] = sub i32 [[ELEMS]], 8 +; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[ELEMS_REM]]) +; CHECK: tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef) +; CHECK: %store.pred = icmp ule <4 x i32> %induction.store +; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> %store.pred) +; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> %store.pred) +define void @expand_v8i16_v4i32(i16* readonly %a, i16* readonly %b, i32* %c, i32* %d, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 7 + %tmp9 = lshr i32 %tmp8, 3 + %tmp10 = shl nuw i32 %tmp9, 3 + %tmp11 = add i32 %tmp10, -8 + %tmp12 = lshr i32 %tmp11, 3 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <8 x i32> %broadcast.splatinsert10, <8 x i32> undef, <8 x i32> zeroinitializer + %broadcast.splatinsert10.store = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11.store = shufflevector <4 x i32> %broadcast.splatinsert10.store, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %store.idx = phi i32 [ 0, %vector.ph ], [ %store.idx.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer + %induction = add <8 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i16, i16* %a, i32 %index + %tmp1 = icmp ule <8 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i16* %tmp to <8 x i16>* + %wide.masked.load = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp2, i32 4, <8 x i1> %tmp1, <8 x i16> undef) + %tmp3 = getelementptr inbounds i16, i16* %b, i32 %index + %tmp4 = bitcast i16* %tmp3 to <8 x i16>* + %wide.masked.load2 = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> %tmp1, <8 x i16> undef) + %extract.2.low = shufflevector <8 x i16> %wide.masked.load2, <8 x i16> undef, < 4 x i32> + %extract.2.high = shufflevector <8 x i16> %wide.masked.load2, <8 x i16> undef, < 4 x i32> + %expand.1 = zext <4 x i16> %extract.2.low to <4 x i32> + %expand.2 = zext <4 x i16> %extract.2.high to <4 x i32> + %mul = mul nsw <4 x i32> %expand.2, %expand.1 + %sub = mul nsw <4 x i32> %expand.1, %expand.2 + %broadcast.splatinsert.store = insertelement <4 x i32> undef, i32 %store.idx, i32 0 + %broadcast.splat.store = shufflevector <4 x i32> %broadcast.splatinsert.store, <4 x i32> undef, <4 x i32> zeroinitializer + %induction.store = add <4 x i32> %broadcast.splat.store, + %store.pred = icmp ule <4 x i32> %induction.store, %broadcast.splat11.store + %tmp6 = getelementptr inbounds i32, i32* %c, i32 %store.idx + %tmp7 = bitcast i32* %tmp6 to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %mul, <4 x i32>* %tmp7, i32 4, <4 x i1> %store.pred) + %gep = getelementptr inbounds i32, i32* %d, i32 %store.idx + %cast.gep = bitcast i32* %gep to <4 x i32>* + tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %sub, <4 x i32>* %cast.gep, i32 4, <4 x i1> %store.pred) + %store.idx.next = add i32 %store.idx, 4 + %index.next = add i32 %index, 8 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +; CHECK-LABEL: expand_v4i32_v4i64 +; CHECK-NOT: call i32 @llvm.arm.vctp +define void @expand_v4i32_v4i64(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i64* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 3 + %tmp9 = lshr i32 %tmp8, 2 + %tmp10 = shl nuw i32 %tmp9, 2 + %tmp11 = add i32 %tmp10, -4 + %tmp12 = lshr i32 %tmp11, 2 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph + %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] + %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ] + %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %induction = add <4 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i32, i32* %a, i32 %index + %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i32* %tmp to <4 x i32>* + %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index + %tmp4 = bitcast i32* %tmp3 to <4 x i32>* + %wide.masked.load2 = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef) + %expand.1 = zext <4 x i32> %wide.masked.load to <4 x i64> + %expand.2 = zext <4 x i32> %wide.masked.load2 to <4 x i64> + %mul = mul nsw <4 x i64> %expand.2, %expand.1 + %tmp6 = getelementptr inbounds i64, i64* %c, i32 %index + %tmp7 = bitcast i64* %tmp6 to <4 x i64>* + tail call void @llvm.masked.store.v4i64.p0v4i64(<4 x i64> %mul, <4 x i64>* %tmp7, i32 4, <4 x i1> %tmp1) + %index.next = add i32 %index, 4 + %tmp15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %tmp14, i32 1) + %tmp16 = icmp ne i32 %tmp15, 0 + br i1 %tmp16, label %vector.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %vector.body, %entry + ret void +} + +declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>) +declare void @llvm.masked.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, i32 immarg, <8 x i1>) +declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) +declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>) +declare void @llvm.masked.store.v4i64.p0v4i64(<4 x i64>, <4 x i64>*, i32 immarg, <4 x i1>) +declare void @llvm.set.loop.iterations.i32(i32) +declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll new file mode 100644 index 0000000..b308060 --- /dev/null +++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll @@ -0,0 +1,118 @@ +; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve %s -S -o - | FileCheck %s + +; CHECK-LABEL: reduction_i32 +; CHECK: phi i32 [ 0, %entry ] +; CHECK: phi <8 x i16> [ zeroinitializer, %entry ] +; CHECK: phi i32 +; CHECK: [[PHI:%[^ ]+]] = phi i32 [ %N, %entry ], [ [[ELEMS:%[^ ]+]], %vector.body ] +; CHECK: [[ELEMS]] = sub i32 [[PHI]], 8 +; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[ELEMS]]) +; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef) +; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp6, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef) +define i16 @reduction_i32(i16* nocapture readonly %A, i16* nocapture readonly %B, i32 %N) { +entry: + %tmp = add i32 %N, -1 + %n.rnd.up = add nuw nsw i32 %tmp, 8 + %n.vec = and i32 %n.rnd.up, -8 + %broadcast.splatinsert1 = insertelement <8 x i32> undef, i32 %tmp, i32 0 + %broadcast.splat2 = shufflevector <8 x i32> %broadcast.splatinsert1, <8 x i32> undef, <8 x i32> zeroinitializer + %0 = add i32 %n.vec, -8 + %1 = lshr i32 %0, 3 + %2 = add nuw nsw i32 %1, 1 + call void @llvm.set.loop.iterations.i32(i32 %2) + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ] + %vec.phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %tmp8, %vector.body ] + %3 = phi i32 [ %2, %entry ], [ %4, %vector.body ] + %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer + %induction = add <8 x i32> %broadcast.splat, + %tmp2 = getelementptr inbounds i16, i16* %A, i32 %index + %tmp3 = icmp ule <8 x i32> %induction, %broadcast.splat2 + %tmp4 = bitcast i16* %tmp2 to <8 x i16>* + %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> %tmp3, <8 x i16> undef) + %tmp5 = getelementptr inbounds i16, i16* %B, i32 %index + %tmp6 = bitcast i16* %tmp5 to <8 x i16>* + %wide.masked.load3 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp6, i32 4, <8 x i1> %tmp3, <8 x i16> undef) + %tmp7 = add <8 x i16> %wide.masked.load, %vec.phi + %tmp8 = add <8 x i16> %tmp7, %wide.masked.load3 + %index.next = add nuw nsw i32 %index, 8 + %4 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %3, i32 1) + %5 = icmp ne i32 %4, 0 + br i1 %5, label %vector.body, label %middle.block + +middle.block: ; preds = %vector.body + %vec.phi.lcssa = phi <8 x i16> [ %vec.phi, %vector.body ] + %.lcssa3 = phi <8 x i1> [ %tmp3, %vector.body ] + %.lcssa = phi <8 x i16> [ %tmp8, %vector.body ] + %tmp10 = select <8 x i1> %.lcssa3, <8 x i16> %.lcssa, <8 x i16> %vec.phi.lcssa + %rdx.shuf = shufflevector <8 x i16> %tmp10, <8 x i16> undef, <8 x i32> + %bin.rdx = add <8 x i16> %rdx.shuf, %tmp10 + %rdx.shuf4 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> + %bin.rdx5 = add <8 x i16> %rdx.shuf4, %bin.rdx + %rdx.shuf6 = shufflevector <8 x i16> %bin.rdx5, <8 x i16> undef, <8 x i32> + %bin.rdx7 = add <8 x i16> %rdx.shuf6, %bin.rdx5 + %tmp11 = extractelement <8 x i16> %bin.rdx7, i32 0 + ret i16 %tmp11 +} + +; CHECK-LABEL: reduction_i32_with_scalar +; CHECK: phi i32 [ 0, %entry ] +; CHECK: phi <8 x i16> [ zeroinitializer, %entry ] +; CHECK: phi i32 +; CHECK: [[PHI:%[^ ]+]] = phi i32 [ %N, %entry ], [ [[ELEMS:%[^ ]+]], %vector.body ] +; CHECK: [[ELEMS]] = sub i32 [[PHI]], 8 +; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[ELEMS]]) +; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef) +define i16 @reduction_i32_with_scalar(i16* nocapture readonly %A, i16 %B, i32 %N) local_unnamed_addr { +entry: + %tmp = add i32 %N, -1 + %n.rnd.up = add nuw nsw i32 %tmp, 8 + %n.vec = and i32 %n.rnd.up, -8 + %broadcast.splatinsert1 = insertelement <8 x i32> undef, i32 %tmp, i32 0 + %broadcast.splat2 = shufflevector <8 x i32> %broadcast.splatinsert1, <8 x i32> undef, <8 x i32> zeroinitializer + %broadcast.splatinsert3 = insertelement <8 x i16> undef, i16 %B, i32 0 + %broadcast.splat4 = shufflevector <8 x i16> %broadcast.splatinsert3, <8 x i16> undef, <8 x i32> zeroinitializer + %0 = add i32 %n.vec, -8 + %1 = lshr i32 %0, 3 + %2 = add nuw nsw i32 %1, 1 + call void @llvm.set.loop.iterations.i32(i32 %2) + br label %vector.body + +vector.body: ; preds = %vector.body, %entry + %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ] + %vec.phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %tmp6, %vector.body ] + %3 = phi i32 [ %2, %entry ], [ %4, %vector.body ] + %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer + %induction = add <8 x i32> %broadcast.splat, + %tmp2 = getelementptr inbounds i16, i16* %A, i32 %index + %tmp3 = icmp ule <8 x i32> %induction, %broadcast.splat2 + %tmp4 = bitcast i16* %tmp2 to <8 x i16>* + %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> %tmp3, <8 x i16> undef) + %tmp5 = add <8 x i16> %vec.phi, %broadcast.splat4 + %tmp6 = add <8 x i16> %tmp5, %wide.masked.load + %index.next = add nuw nsw i32 %index, 8 + %4 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %3, i32 1) + %5 = icmp ne i32 %4, 0 + br i1 %5, label %vector.body, label %middle.block + +middle.block: ; preds = %vector.body + %tmp8 = select <8 x i1> %tmp3, <8 x i16> %tmp6, <8 x i16> %vec.phi + %rdx.shuf = shufflevector <8 x i16> %tmp8, <8 x i16> undef, <8 x i32> + %bin.rdx = add <8 x i16> %rdx.shuf, %tmp8 + %rdx.shuf5 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> + %bin.rdx6 = add <8 x i16> %rdx.shuf5, %bin.rdx + %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx6, <8 x i16> undef, <8 x i32> + %bin.rdx8 = add <8 x i16> %rdx.shuf7, %bin.rdx6 + %tmp9 = extractelement <8 x i16> %bin.rdx8, i32 0 + ret i16 %tmp9 +} + +declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>) +declare void @llvm.set.loop.iterations.i32(i32) +declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) + + diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-unroll.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-unroll.ll new file mode 100644 index 0000000..19d9c89 --- /dev/null +++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-unroll.ll @@ -0,0 +1,118 @@ +; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s + +; TODO: The unrolled pattern is preventing the transform +; CHECK-LABEL: mul_v16i8_unroll +; CHECK-NOT: call i32 @llvm.arm.vcpt +define void @mul_v16i8_unroll(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) { +entry: + %cmp8 = icmp eq i32 %N, 0 + %tmp8 = add i32 %N, 15 + %tmp9 = lshr i32 %tmp8, 4 + %tmp10 = shl nuw i32 %tmp9, 4 + %tmp11 = add i32 %tmp10, -16 + %tmp12 = lshr i32 %tmp11, 4 + %tmp13 = add nuw nsw i32 %tmp12, 1 + br i1 %cmp8, label %for.cond.cleanup, label %vector.ph + +vector.ph: ; preds = %entry + %trip.count.minus.1 = add i32 %N, -1 + %broadcast.splatinsert10 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0 + %broadcast.splat11 = shufflevector <16 x i32> %broadcast.splatinsert10, <16 x i32> undef, <16 x i32> zeroinitializer + %xtraiter = and i32 %tmp13, 1 + %0 = icmp ult i32 %tmp12, 1 + br i1 %0, label %for.cond.cleanup.loopexit.unr-lcssa, label %vector.ph.new + +vector.ph.new: ; preds = %vector.ph + call void @llvm.set.loop.iterations.i32(i32 %tmp13) + %unroll_iter = sub i32 %tmp13, %xtraiter + br label %vector.body + +vector.body: ; preds = %vector.body, %vector.ph.new + %index = phi i32 [ 0, %vector.ph.new ], [ %index.next.1, %vector.body ] + %niter = phi i32 [ %unroll_iter, %vector.ph.new ], [ %niter.nsub.1, %vector.body ] + %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0 + %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer + %induction = add <16 x i32> %broadcast.splat, + %tmp = getelementptr inbounds i8, i8* %a, i32 %index + %tmp1 = icmp ule <16 x i32> %induction, %broadcast.splat11 + %tmp2 = bitcast i8* %tmp to <16 x i8>* + %wide.masked.load = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp2, i32 4, <16 x i1> %tmp1, <16 x i8> undef) + %tmp3 = getelementptr inbounds i8, i8* %b, i32 %index + %tmp4 = bitcast i8* %tmp3 to <16 x i8>* + %wide.masked.load2 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp4, i32 4, <16 x i1> %tmp1, <16 x i8> undef) + %mul = mul nsw <16 x i8> %wide.masked.load2, %wide.masked.load + %tmp6 = getelementptr inbounds i8, i8* %c, i32 %index + %tmp7 = bitcast i8* %tmp6 to <16 x i8>* + tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %mul, <16 x i8>* %tmp7, i32 4, <16 x i1> %tmp1) + %index.next = add nuw nsw i32 %index, 16 + %niter.nsub = sub i32 %niter, 1 + %broadcast.splatinsert.1 = insertelement <16 x i32> undef, i32 %index.next, i32 0 + %broadcast.splat.1 = shufflevector <16 x i32> %broadcast.splatinsert.1, <16 x i32> undef, <16 x i32> zeroinitializer + %induction.1 = add <16 x i32> %broadcast.splat.1, + %tmp.1 = getelementptr inbounds i8, i8* %a, i32 %index.next + %tmp1.1 = icmp ule <16 x i32> %induction.1, %broadcast.splat11 + %tmp2.1 = bitcast i8* %tmp.1 to <16 x i8>* + %wide.masked.load.1 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp2.1, i32 4, <16 x i1> %tmp1.1, <16 x i8> undef) + %tmp3.1 = getelementptr inbounds i8, i8* %b, i32 %index.next + %tmp4.1 = bitcast i8* %tmp3.1 to <16 x i8>* + %wide.masked.load2.1 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp4.1, i32 4, <16 x i1> %tmp1.1, <16 x i8> undef) + %mul.1 = mul nsw <16 x i8> %wide.masked.load2.1, %wide.masked.load.1 + %tmp6.1 = getelementptr inbounds i8, i8* %c, i32 %index.next + %tmp7.1 = bitcast i8* %tmp6.1 to <16 x i8>* + tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %mul.1, <16 x i8>* %tmp7.1, i32 4, <16 x i1> %tmp1.1) + %index.next.1 = add i32 %index.next, 16 + %niter.nsub.1 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %niter.nsub, i32 1) + %niter.ncmp.1 = icmp ne i32 %niter.nsub.1, 0 + br i1 %niter.ncmp.1, label %vector.body, label %for.cond.cleanup.loopexit.unr-lcssa.loopexit + +for.cond.cleanup.loopexit.unr-lcssa.loopexit: ; preds = %vector.body + %index.unr.ph = phi i32 [ %index.next.1, %vector.body ] + %tmp14.unr.ph = phi i32 [ -2, %vector.body ] + br label %for.cond.cleanup.loopexit.unr-lcssa + +for.cond.cleanup.loopexit.unr-lcssa: ; preds = %for.cond.cleanup.loopexit.unr-lcssa.loopexit, %vector.ph + %index.unr = phi i32 [ 0, %vector.ph ], [ %index.unr.ph, %for.cond.cleanup.loopexit.unr-lcssa.loopexit ] + %tmp14.unr = phi i32 [ %tmp13, %vector.ph ], [ %tmp14.unr.ph, %for.cond.cleanup.loopexit.unr-lcssa.loopexit ] + %lcmp.mod = icmp ne i32 %xtraiter, 0 + br i1 %lcmp.mod, label %vector.body.epil.preheader, label %for.cond.cleanup.loopexit + +vector.body.epil.preheader: ; preds = %for.cond.cleanup.loopexit.unr-lcssa + br label %vector.body.epil + +vector.body.epil: ; preds = %vector.body.epil.preheader + %index.epil = phi i32 [ %index.unr, %vector.body.epil.preheader ] + %tmp14.epil = phi i32 [ %tmp14.unr, %vector.body.epil.preheader ] + %broadcast.splatinsert.epil = insertelement <16 x i32> undef, i32 %index.epil, i32 0 + %broadcast.splat.epil = shufflevector <16 x i32> %broadcast.splatinsert.epil, <16 x i32> undef, <16 x i32> zeroinitializer + %induction.epil = add <16 x i32> %broadcast.splat.epil, + %tmp.epil = getelementptr inbounds i8, i8* %a, i32 %index.epil + %tmp1.epil = icmp ule <16 x i32> %induction.epil, %broadcast.splat11 + %tmp2.epil = bitcast i8* %tmp.epil to <16 x i8>* + %wide.masked.load.epil = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp2.epil, i32 4, <16 x i1> %tmp1.epil, <16 x i8> undef) + %tmp3.epil = getelementptr inbounds i8, i8* %b, i32 %index.epil + %tmp4.epil = bitcast i8* %tmp3.epil to <16 x i8>* + %wide.masked.load2.epil = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %tmp4.epil, i32 4, <16 x i1> %tmp1.epil, <16 x i8> undef) + %mul.epil = mul nsw <16 x i8> %wide.masked.load2.epil, %wide.masked.load.epil + %tmp6.epil = getelementptr inbounds i8, i8* %c, i32 %index.epil + %tmp7.epil = bitcast i8* %tmp6.epil to <16 x i8>* + tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %mul.epil, <16 x i8>* %tmp7.epil, i32 4, <16 x i1> %tmp1.epil) + %index.next.epil = add i32 %index.epil, 16 + %tmp15.epil = add nuw nsw i32 %tmp14.epil, -1 + %tmp16.epil = icmp ne i32 %tmp15.epil, 0 + br label %for.cond.cleanup.loopexit.epilog-lcssa + +for.cond.cleanup.loopexit.epilog-lcssa: ; preds = %vector.body.epil + br label %for.cond.cleanup.loopexit + +for.cond.cleanup.loopexit: ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.cond.cleanup.loopexit.epilog-lcssa + br label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry + ret void +} + +declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>) #1 +declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32 immarg, <16 x i1>) #2 +declare void @llvm.set.loop.iterations.i32(i32) #3 +declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #3 + -- 2.7.4