HorizontalReduction() = default;
/// Try to find a reduction tree.
- bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
- assert((!Phi || is_contained(Phi->operands(), B)) &&
+ bool matchAssociativeReduction(PHINode *Phi, Instruction *Inst) {
+ assert((!Phi || is_contained(Phi->operands(), Inst)) &&
"Phi needs to use the binary operator");
-
- RdxKind = getRdxKind(B);
+ assert((isa<BinaryOperator>(Inst) || isa<SelectInst>(Inst) ||
+ isa<IntrinsicInst>(Inst)) &&
+ "Expected binop, select, or intrinsic for reduction matching");
+ RdxKind = getRdxKind(Inst);
// We could have a initial reductions that is not an add.
// r *= v1 + v2 + v3 + v4
// In such a case start looking for a tree rooted in the first '+'.
if (Phi) {
- if (getLHS(RdxKind, B) == Phi) {
+ if (getLHS(RdxKind, Inst) == Phi) {
Phi = nullptr;
- B = dyn_cast<Instruction>(getRHS(RdxKind, B));
- if (!B)
+ Inst = dyn_cast<Instruction>(getRHS(RdxKind, Inst));
+ if (!Inst)
return false;
- RdxKind = getRdxKind(B);
- } else if (getRHS(RdxKind, B) == Phi) {
+ RdxKind = getRdxKind(Inst);
+ } else if (getRHS(RdxKind, Inst) == Phi) {
Phi = nullptr;
- B = dyn_cast<Instruction>(getLHS(RdxKind, B));
- if (!B)
+ Inst = dyn_cast<Instruction>(getLHS(RdxKind, Inst));
+ if (!Inst)
return false;
- RdxKind = getRdxKind(B);
+ RdxKind = getRdxKind(Inst);
}
}
- if (!isVectorizable(RdxKind, B))
+ if (!isVectorizable(RdxKind, Inst))
return false;
// Analyze "regular" integer/FP types for reductions - no target-specific
// types or pointers.
- Type *Ty = B->getType();
+ Type *Ty = Inst->getType();
if (!isValidElementType(Ty) || Ty->isPointerTy())
return false;
// Though the ultimate reduction may have multiple uses, its condition must
// have only single use.
- if (auto *SI = dyn_cast<SelectInst>(B))
- if (!SI->getCondition()->hasOneUse())
+ if (auto *Sel = dyn_cast<SelectInst>(Inst))
+ if (!Sel->getCondition()->hasOneUse())
return false;
- ReductionRoot = B;
+ ReductionRoot = Inst;
// The opcode for leaf values that we perform a reduction on.
// For example: load(x) + load(y) + load(z) + fptoui(w)
// Post order traverse the reduction tree starting at B. We only handle true
// trees containing only binary operators.
SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
- Stack.push_back(std::make_pair(B, getFirstOperandIndex(B)));
- initReductionOps(B);
+ Stack.push_back(std::make_pair(Inst, getFirstOperandIndex(Inst)));
+ initReductionOps(Inst);
while (!Stack.empty()) {
Instruction *TreeN = Stack.back().first;
unsigned EdgeToVisit = Stack.back().second++;
// Each tree node needs to have minimal number of users except for the
// ultimate reduction.
const bool IsRdxInst = EdgeRdxKind == RdxKind;
- if (EdgeInst != Phi && EdgeInst != B &&
- hasSameParent(EdgeInst, B->getParent(), IsRdxInst) &&
- hasRequiredNumberOfUses(isa<SelectInst>(B), EdgeInst) &&
+ if (EdgeInst != Phi && EdgeInst != Inst &&
+ hasSameParent(EdgeInst, Inst->getParent(), IsRdxInst) &&
+ hasRequiredNumberOfUses(isa<SelectInst>(Inst), EdgeInst) &&
(!LeafOpcode || LeafOpcode == EdgeInst->getOpcode() || IsRdxInst)) {
if (IsRdxInst) {
// We need to be able to reassociate the reduction operations.