From 408ebe5e3aae3cfa1e960f74998b02e277591663 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 5 Sep 2022 10:48:29 +0100 Subject: [PATCH] [VPlan] Move VPWidenCallRecipe to VPlanRecipes.cpp (NFC). Depends on D132585. Reviewed By: Ayal Differential Revision: https://reviews.llvm.org/D132586 --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 62 ------------------------- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 58 +++++++++++++++++++++++ 2 files changed, 58 insertions(+), 62 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 5b50925..5448c31 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4153,63 +4153,6 @@ bool InnerLoopVectorizer::useOrderedReductions( return Cost->useOrderedReductions(RdxDesc); } -void InnerLoopVectorizer::widenCallInstruction( - CallInst &CI, VPValue *Def, VPUser &ArgOperands, VPTransformState &State, - Intrinsic::ID VectorIntrinsicID) { - assert(!isa(CI) && - "DbgInfoIntrinsic should have been dropped during VPlan construction"); - State.setDebugLocFromInst(&CI); - - SmallVector Tys; - for (Value *ArgOperand : CI.args()) - Tys.push_back(ToVectorTy(ArgOperand->getType(), VF.getKnownMinValue())); - - for (unsigned Part = 0; Part < UF; ++Part) { - SmallVector TysForDecl = {CI.getType()}; - SmallVector Args; - for (const auto &I : enumerate(ArgOperands.operands())) { - // Some intrinsics have a scalar argument - don't replace it with a - // vector. - Value *Arg; - if (!VectorIntrinsicID || - !isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index())) - Arg = State.get(I.value(), Part); - else - Arg = State.get(I.value(), VPIteration(0, 0)); - if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index())) - TysForDecl.push_back(Arg->getType()); - Args.push_back(Arg); - } - - Function *VectorF; - if (VectorIntrinsicID) { - // Use vector version of the intrinsic. - if (VF.isVector()) - TysForDecl[0] = VectorType::get(CI.getType()->getScalarType(), VF); - Module *M = State.Builder.GetInsertBlock()->getModule(); - VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl); - assert(VectorF && "Can't retrieve vector intrinsic."); - } else { - // Use vector version of the function call. - const VFShape Shape = VFShape::get(CI, VF, false /*HasGlobalPred*/); -#ifndef NDEBUG - assert(VFDatabase(CI).getVectorizedFunction(Shape) != nullptr && - "Can't create vector function."); -#endif - VectorF = VFDatabase(CI).getVectorizedFunction(Shape); - } - SmallVector OpBundles; - CI.getOperandBundlesAsDefs(OpBundles); - CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles); - - if (isa(V)) - V->copyFastMathFlags(&CI); - - State.set(Def, V, Part); - State.addMetadata(V, &CI); - } -} - void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) { // We should not collect Scalars more than once per VF. Right now, this // function is called from collectUniformsAndScalars(), which already does @@ -9316,11 +9259,6 @@ void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent, } #endif -void VPWidenCallRecipe::execute(VPTransformState &State) { - State.ILV->widenCallInstruction(*cast(getUnderlyingInstr()), this, - *this, State, VectorIntrinsicID); -} - void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { assert(!State.Instance && "Int or FP induction being replicated."); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 68ee682..2b4938c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -434,6 +434,64 @@ void VPInstruction::setFastMathFlags(FastMathFlags FMFNew) { FMF = FMFNew; } +void VPWidenCallRecipe::execute(VPTransformState &State) { + auto &CI = *cast(getUnderlyingInstr()); + assert(!isa(CI) && + "DbgInfoIntrinsic should have been dropped during VPlan construction"); + State.setDebugLocFromInst(&CI); + + SmallVector Tys; + for (Value *ArgOperand : CI.args()) + Tys.push_back( + ToVectorTy(ArgOperand->getType(), State.VF.getKnownMinValue())); + + for (unsigned Part = 0; Part < State.UF; ++Part) { + SmallVector TysForDecl = {CI.getType()}; + SmallVector Args; + for (const auto &I : enumerate(operands())) { + // Some intrinsics have a scalar argument - don't replace it with a + // vector. + Value *Arg; + if (VectorIntrinsicID == Intrinsic::not_intrinsic || + !isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index())) + Arg = State.get(I.value(), Part); + else + Arg = State.get(I.value(), VPIteration(0, 0)); + if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index())) + TysForDecl.push_back(Arg->getType()); + Args.push_back(Arg); + } + + Function *VectorF; + if (VectorIntrinsicID != Intrinsic::not_intrinsic) { + // Use vector version of the intrinsic. + if (State.VF.isVector()) + TysForDecl[0] = + VectorType::get(CI.getType()->getScalarType(), State.VF); + Module *M = State.Builder.GetInsertBlock()->getModule(); + VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl); + assert(VectorF && "Can't retrieve vector intrinsic."); + } else { + // Use vector version of the function call. + const VFShape Shape = VFShape::get(CI, State.VF, false /*HasGlobalPred*/); +#ifndef NDEBUG + assert(VFDatabase(CI).getVectorizedFunction(Shape) != nullptr && + "Can't create vector function."); +#endif + VectorF = VFDatabase(CI).getVectorizedFunction(Shape); + } + SmallVector OpBundles; + CI.getOperandBundlesAsDefs(OpBundles); + CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles); + + if (isa(V)) + V->copyFastMathFlags(&CI); + + State.set(this, V, Part); + State.addMetadata(V, &CI); + } +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { -- 2.7.4