void widenInstruction(Instruction &I);
/// Widen a single call instruction within the innermost loop.
- void widenCallInstruction(CallInst &I);
+ void widenCallInstruction(CallInst &I, VPUser &ArgOperands,
+ VPTransformState &State);
/// Fix the vectorized code, taking care of header phi's, live-outs, and more.
void fixVectorizedLoop();
} // end of switch.
}
-void InnerLoopVectorizer::widenCallInstruction(CallInst &I) {
+void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPUser &ArgOperands,
+ VPTransformState &State) {
// Ignore dbg intrinsics.
// TODO: Debug intrinsics should be skipped/handled during VPlan construction
// rather than dropping them here.
for (unsigned Part = 0; Part < UF; ++Part) {
SmallVector<Value *, 4> Args;
- for (unsigned i = 0, ie = CI->getNumArgOperands(); i != ie; ++i) {
- Value *Arg = CI->getArgOperand(i);
+ for (auto &I : enumerate(ArgOperands.operands())) {
// Some intrinsics have a scalar argument - don't replace it with a
// vector.
- if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, i))
- Arg = getOrCreateVectorValue(CI->getArgOperand(i), Part);
+ Value *Arg;
+ if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index()))
+ Arg = State.get(I.value(), Part);
+ else
+ Arg = State.get(I.value(), {0, 0});
Args.push_back(Arg);
}
return new VPBlendRecipe(Phi, Masks);
}
-VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(Instruction *I,
- VFRange &Range) {
+VPWidenCallRecipe *
+VPRecipeBuilder::tryToWidenCall(Instruction *I, VFRange &Range, VPlan &Plan) {
bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
[&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range);
return nullptr;
// Success: widen this call.
- return new VPWidenCallRecipe(*CI);
+ auto VPValues = map_range(CI->arg_operands(), [&Plan](Value *Op) {
+ return Plan.getOrAddVPValue(Op);
+ });
+
+ return new VPWidenCallRecipe(*CI, VPValues);
}
VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range) {
// First, check for specific widening recipes that deal with calls, memory
// operations, inductions and Phi nodes.
- if ((Recipe = tryToWidenCall(Instr, Range)) ||
+ if ((Recipe = tryToWidenCall(Instr, Range, *Plan)) ||
(Recipe = tryToWidenMemory(Instr, Range, Plan)) ||
(Recipe = tryToOptimizeInduction(Instr, Range)) ||
(Recipe = tryToBlend(Instr, Plan)) ||
}
void VPWidenCallRecipe::execute(VPTransformState &State) {
- State.ILV->widenCallInstruction(Ingredient);
+ State.ILV->widenCallInstruction(Ingredient, User, State);
}
void VPWidenRecipe::execute(VPTransformState &State) {
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
VPUser(ArrayRef<VPValue *> Operands) : VPUser(VPValue::VPUserSC, Operands) {}
VPUser(std::initializer_list<VPValue *> Operands)
: VPUser(ArrayRef<VPValue *>(Operands)) {}
+ template <typename IterT>
+ VPUser(iterator_range<IterT> Operands) : VPValue(VPValue::VPUserSC) {
+ for (VPValue *Operand : Operands)
+ addOperand(Operand);
+ }
+
VPUser(const VPUser &) = delete;
VPUser &operator=(const VPUser &) = delete;