return Cost->useOrderedReductions(RdxDesc);
}
-void InnerLoopVectorizer::widenCallInstruction(
- CallInst &CI, VPValue *Def, VPUser &ArgOperands, VPTransformState &State,
- Intrinsic::ID VectorIntrinsicID) {
- assert(!isa<DbgInfoIntrinsic>(CI) &&
- "DbgInfoIntrinsic should have been dropped during VPlan construction");
- State.setDebugLocFromInst(&CI);
-
- SmallVector<Type *, 4> Tys;
- for (Value *ArgOperand : CI.args())
- Tys.push_back(ToVectorTy(ArgOperand->getType(), VF.getKnownMinValue()));
-
- for (unsigned Part = 0; Part < UF; ++Part) {
- SmallVector<Type *, 2> TysForDecl = {CI.getType()};
- SmallVector<Value *, 4> Args;
- for (const auto &I : enumerate(ArgOperands.operands())) {
- // Some intrinsics have a scalar argument - don't replace it with a
- // vector.
- Value *Arg;
- if (!VectorIntrinsicID ||
- !isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index()))
- Arg = State.get(I.value(), Part);
- else
- Arg = State.get(I.value(), VPIteration(0, 0));
- if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
- TysForDecl.push_back(Arg->getType());
- Args.push_back(Arg);
- }
-
- Function *VectorF;
- if (VectorIntrinsicID) {
- // Use vector version of the intrinsic.
- if (VF.isVector())
- TysForDecl[0] = VectorType::get(CI.getType()->getScalarType(), VF);
- Module *M = State.Builder.GetInsertBlock()->getModule();
- VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
- assert(VectorF && "Can't retrieve vector intrinsic.");
- } else {
- // Use vector version of the function call.
- const VFShape Shape = VFShape::get(CI, VF, false /*HasGlobalPred*/);
-#ifndef NDEBUG
- assert(VFDatabase(CI).getVectorizedFunction(Shape) != nullptr &&
- "Can't create vector function.");
-#endif
- VectorF = VFDatabase(CI).getVectorizedFunction(Shape);
- }
- SmallVector<OperandBundleDef, 1> OpBundles;
- CI.getOperandBundlesAsDefs(OpBundles);
- CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);
-
- if (isa<FPMathOperator>(V))
- V->copyFastMathFlags(&CI);
-
- State.set(Def, V, Part);
- State.addMetadata(V, &CI);
- }
-}
-
void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
// We should not collect Scalars more than once per VF. Right now, this
// function is called from collectUniformsAndScalars(), which already does
}
#endif
-void VPWidenCallRecipe::execute(VPTransformState &State) {
- State.ILV->widenCallInstruction(*cast<CallInst>(getUnderlyingInstr()), this,
- *this, State, VectorIntrinsicID);
-}
-
void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Int or FP induction being replicated.");
FMF = FMFNew;
}
+void VPWidenCallRecipe::execute(VPTransformState &State) {
+ auto &CI = *cast<CallInst>(getUnderlyingInstr());
+ assert(!isa<DbgInfoIntrinsic>(CI) &&
+ "DbgInfoIntrinsic should have been dropped during VPlan construction");
+ State.setDebugLocFromInst(&CI);
+
+ SmallVector<Type *, 4> Tys;
+ for (Value *ArgOperand : CI.args())
+ Tys.push_back(
+ ToVectorTy(ArgOperand->getType(), State.VF.getKnownMinValue()));
+
+ for (unsigned Part = 0; Part < State.UF; ++Part) {
+ SmallVector<Type *, 2> TysForDecl = {CI.getType()};
+ SmallVector<Value *, 4> Args;
+ for (const auto &I : enumerate(operands())) {
+ // Some intrinsics have a scalar argument - don't replace it with a
+ // vector.
+ Value *Arg;
+ if (VectorIntrinsicID == Intrinsic::not_intrinsic ||
+ !isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index()))
+ Arg = State.get(I.value(), Part);
+ else
+ Arg = State.get(I.value(), VPIteration(0, 0));
+ if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
+ TysForDecl.push_back(Arg->getType());
+ Args.push_back(Arg);
+ }
+
+ Function *VectorF;
+ if (VectorIntrinsicID != Intrinsic::not_intrinsic) {
+ // Use vector version of the intrinsic.
+ if (State.VF.isVector())
+ TysForDecl[0] =
+ VectorType::get(CI.getType()->getScalarType(), State.VF);
+ Module *M = State.Builder.GetInsertBlock()->getModule();
+ VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
+ assert(VectorF && "Can't retrieve vector intrinsic.");
+ } else {
+ // Use vector version of the function call.
+ const VFShape Shape = VFShape::get(CI, State.VF, false /*HasGlobalPred*/);
+#ifndef NDEBUG
+ assert(VFDatabase(CI).getVectorizedFunction(Shape) != nullptr &&
+ "Can't create vector function.");
+#endif
+ VectorF = VFDatabase(CI).getVectorizedFunction(Shape);
+ }
+ SmallVector<OperandBundleDef, 1> OpBundles;
+ CI.getOperandBundlesAsDefs(OpBundles);
+ CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+
+ if (isa<FPMathOperator>(V))
+ V->copyFastMathFlags(&CI);
+
+ State.set(this, V, Part);
+ State.addMetadata(V, &CI);
+ }
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {