From: Tyler Nowicki Date: Tue, 16 Jun 2015 18:07:34 +0000 (+0000) Subject: Rename Reduction variables/structures to Recurrence. X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=0a91310c7ffad8b597c1996fde68e5f0b6d27ad3;p=platform%2Fupstream%2Fllvm.git Rename Reduction variables/structures to Recurrence. A reduction is a special kind of recurrence. In the loop vectorizer we currently identify basic reductions. Future patches will extend this to identifying basic recurrences. llvm-svn: 239835 --- diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h index 28791f5..3936f1d 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -43,12 +43,23 @@ struct LICMSafetyInfo { {} }; -/// This POD struct holds information about a potential reduction operation. -class ReductionInstDesc { +/// The RecurrenceDescriptor is used to identify recurrences variables in a +/// loop. Reduction is a special case of recurrence that has uses of the +/// recurrence variable outside the loop. The method isReductionPHI identifies +/// reductions that are basic recurrences. +/// +/// Basic recurrences are defined as the summation, product, OR, AND, XOR, min, +/// or max of a set of terms. For example: for(i=0; i &Builder, - ReductionInstDesc::MinMaxReductionKind RK, + RecurrenceInstDesc::MinMaxRecurrenceKind RK, Value *Left, Value *Right); /// Returns true if Phi is a reduction of type Kind and adds it to the - /// ReductionDescriptor. - static bool AddReductionVar(PHINode *Phi, ReductionKind Kind, Loop *TheLoop, + /// RecurrenceDescriptor. + static bool AddReductionVar(PHINode *Phi, RecurrenceKind Kind, Loop *TheLoop, bool HasFunNoNaNAttr, - ReductionDescriptor &RedDes); + RecurrenceDescriptor &RedDes); - /// Returns true if Phi is a reduction in TheLoop. The ReductionDescriptor is + /// Returns true if Phi is a reduction in TheLoop. The RecurrenceDescriptor is /// returned in RedDes. static bool isReductionPHI(PHINode *Phi, Loop *TheLoop, - ReductionDescriptor &RedDes); + RecurrenceDescriptor &RedDes); - ReductionKind getReductionKind() { return Kind; } + RecurrenceKind getRecurrenceKind() { return Kind; } - ReductionInstDesc::MinMaxReductionKind getMinMaxReductionKind() { + RecurrenceInstDesc::MinMaxRecurrenceKind getMinMaxRecurrenceKind() { return MinMaxKind; } - TrackingVH getReductionStartValue() { return StartValue; } + TrackingVH getRecurrenceStartValue() { return StartValue; } Instruction *getLoopExitInstr() { return LoopExitInstr; } private: - // The starting value of the reduction. + // The starting value of the recurrence. // It does not have to be zero! TrackingVH StartValue; // The instruction who's value is used outside the loop. Instruction *LoopExitInstr; - // The kind of the reduction. - ReductionKind Kind; - // If this a min/max reduction the kind of reduction. - ReductionInstDesc::MinMaxReductionKind MinMaxKind; + // The kind of the recurrence. + RecurrenceKind Kind; + // If this a min/max recurrence the kind of recurrence. + RecurrenceInstDesc::MinMaxRecurrenceKind MinMaxKind; }; BasicBlock *InsertPreheaderForLoop(Loop *L, Pass *P); diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp index f584018..2554655 100644 --- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp +++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp @@ -598,8 +598,8 @@ struct LoopInterchange : public FunctionPass { bool LoopInterchangeLegality::areAllUsesReductions(Instruction *Ins, Loop *L) { return !std::any_of(Ins->user_begin(), Ins->user_end(), [=](User *U) -> bool { PHINode *UserIns = dyn_cast(U); - ReductionDescriptor RD; - return !UserIns || !ReductionDescriptor::isReductionPHI(UserIns, L, RD); + RecurrenceDescriptor RD; + return !UserIns || !RecurrenceDescriptor::isReductionPHI(UserIns, L, RD); }); } @@ -697,12 +697,12 @@ bool LoopInterchangeLegality::findInductionAndReductions( if (!L->getLoopLatch() || !L->getLoopPredecessor()) return false; for (BasicBlock::iterator I = L->getHeader()->begin(); isa(I); ++I) { - ReductionDescriptor RD; + RecurrenceDescriptor RD; PHINode *PHI = cast(I); ConstantInt *StepValue = nullptr; if (isInductionPHI(PHI, SE, StepValue)) Inductions.push_back(PHI); - else if (ReductionDescriptor::isReductionPHI(PHI, L, RD)) + else if (RecurrenceDescriptor::isReductionPHI(PHI, L, RD)) Reductions.push_back(PHI); else { DEBUG( diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index 5f25e6b..c37ecfc 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -26,17 +26,17 @@ using namespace llvm::PatternMatch; #define DEBUG_TYPE "loop-utils" -bool ReductionDescriptor::areAllUsesIn(Instruction *I, - SmallPtrSetImpl &Set) { +bool RecurrenceDescriptor::areAllUsesIn(Instruction *I, + SmallPtrSetImpl &Set) { for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; ++Use) if (!Set.count(dyn_cast(*Use))) return false; return true; } -bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, - Loop *TheLoop, bool HasFunNoNaNAttr, - ReductionDescriptor &RedDes) { +bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind, + Loop *TheLoop, bool HasFunNoNaNAttr, + RecurrenceDescriptor &RedDes) { if (Phi->getNumIncomingValues() != 2) return false; @@ -66,7 +66,7 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, // the number of instruction we saw from the recognized min/max pattern, // to make sure we only see exactly the two instructions. unsigned NumCmpSelectPatternInst = 0; - ReductionInstDesc ReduxDesc(false, nullptr); + RecurrenceInstDesc ReduxDesc(false, nullptr); SmallPtrSet VisitedInsts; SmallVector Worklist; @@ -111,8 +111,8 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, return false; // Any reduction instruction must be of one of the allowed kinds. - ReduxDesc = isReductionInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr); - if (!ReduxDesc.isReduction()) + ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr); + if (!ReduxDesc.isRecurrence()) return false; // A reduction operation must only have one use of the reduction value. @@ -164,7 +164,7 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, // Process instructions only once (termination). Each reduction cycle // value must only be used once, except by phi nodes and min/max // reductions which are represented as a cmp followed by a select. - ReductionInstDesc IgnoredVal(false, nullptr); + RecurrenceInstDesc IgnoredVal(false, nullptr); if (VisitedInsts.insert(UI).second) { if (isa(UI)) PHIs.push_back(UI); @@ -173,7 +173,7 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, } else if (!isa(UI) && ((!isa(UI) && !isa(UI) && !isa(UI)) || - !isMinMaxSelectCmpPattern(UI, IgnoredVal).isReduction())) + !isMinMaxSelectCmpPattern(UI, IgnoredVal).isRecurrence())) return false; // Remember that we completed the cycle. @@ -197,11 +197,11 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, // only have a single instruction with out-of-loop users. // The ExitInstruction(Instruction which is allowed to have out-of-loop users) - // is saved as part of the ReductionDescriptor. + // is saved as part of the RecurrenceDescriptor. // Save the description of this reduction variable. - ReductionDescriptor RD(RdxStart, ExitInstruction, Kind, - ReduxDesc.getMinMaxKind()); + RecurrenceDescriptor RD(RdxStart, ExitInstruction, Kind, + ReduxDesc.getMinMaxKind()); RedDes = RD; @@ -210,9 +210,9 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, /// Returns true if the instruction is a Select(ICmp(X, Y), X, Y) instruction /// pattern corresponding to a min(X, Y) or max(X, Y). -ReductionInstDesc -ReductionDescriptor::isMinMaxSelectCmpPattern(Instruction *I, - ReductionInstDesc &Prev) { +RecurrenceInstDesc +RecurrenceDescriptor::isMinMaxSelectCmpPattern(Instruction *I, + RecurrenceInstDesc &Prev) { assert((isa(I) || isa(I) || isa(I)) && "Expect a select instruction"); @@ -223,84 +223,84 @@ ReductionDescriptor::isMinMaxSelectCmpPattern(Instruction *I, // select. if ((Cmp = dyn_cast(I)) || (Cmp = dyn_cast(I))) { if (!Cmp->hasOneUse() || !(Select = dyn_cast(*I->user_begin()))) - return ReductionInstDesc(false, I); - return ReductionInstDesc(Select, Prev.getMinMaxKind()); + return RecurrenceInstDesc(false, I); + return RecurrenceInstDesc(Select, Prev.getMinMaxKind()); } // Only handle single use cases for now. if (!(Select = dyn_cast(I))) - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); if (!(Cmp = dyn_cast(I->getOperand(0))) && !(Cmp = dyn_cast(I->getOperand(0)))) - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); if (!Cmp->hasOneUse()) - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); Value *CmpLeft; Value *CmpRight; // Look for a min/max pattern. if (m_UMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_UIntMin); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_UIntMin); else if (m_UMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_UIntMax); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_UIntMax); else if (m_SMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_SIntMax); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_SIntMax); else if (m_SMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_SIntMin); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_SIntMin); else if (m_OrdFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMin); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_FloatMin); else if (m_OrdFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMax); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_FloatMax); else if (m_UnordFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMin); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_FloatMin); else if (m_UnordFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMax); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_FloatMax); - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); } -ReductionInstDesc ReductionDescriptor::isReductionInstr(Instruction *I, - ReductionKind Kind, - ReductionInstDesc &Prev, - bool HasFunNoNaNAttr) { +RecurrenceInstDesc +RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurrenceKind Kind, + RecurrenceInstDesc &Prev, + bool HasFunNoNaNAttr) { bool FP = I->getType()->isFloatingPointTy(); bool FastMath = FP && I->hasUnsafeAlgebra(); switch (I->getOpcode()) { default: - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); case Instruction::PHI: if (FP && (Kind != RK_FloatMult && Kind != RK_FloatAdd && Kind != RK_FloatMinMax)) - return ReductionInstDesc(false, I); - return ReductionInstDesc(I, Prev.getMinMaxKind()); + return RecurrenceInstDesc(false, I); + return RecurrenceInstDesc(I, Prev.getMinMaxKind()); case Instruction::Sub: case Instruction::Add: - return ReductionInstDesc(Kind == RK_IntegerAdd, I); + return RecurrenceInstDesc(Kind == RK_IntegerAdd, I); case Instruction::Mul: - return ReductionInstDesc(Kind == RK_IntegerMult, I); + return RecurrenceInstDesc(Kind == RK_IntegerMult, I); case Instruction::And: - return ReductionInstDesc(Kind == RK_IntegerAnd, I); + return RecurrenceInstDesc(Kind == RK_IntegerAnd, I); case Instruction::Or: - return ReductionInstDesc(Kind == RK_IntegerOr, I); + return RecurrenceInstDesc(Kind == RK_IntegerOr, I); case Instruction::Xor: - return ReductionInstDesc(Kind == RK_IntegerXor, I); + return RecurrenceInstDesc(Kind == RK_IntegerXor, I); case Instruction::FMul: - return ReductionInstDesc(Kind == RK_FloatMult && FastMath, I); + return RecurrenceInstDesc(Kind == RK_FloatMult && FastMath, I); case Instruction::FSub: case Instruction::FAdd: - return ReductionInstDesc(Kind == RK_FloatAdd && FastMath, I); + return RecurrenceInstDesc(Kind == RK_FloatAdd && FastMath, I); case Instruction::FCmp: case Instruction::ICmp: case Instruction::Select: if (Kind != RK_IntegerMinMax && (!HasFunNoNaNAttr || Kind != RK_FloatMinMax)) - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); return isMinMaxSelectCmpPattern(I, Prev); } } -bool ReductionDescriptor::hasMultipleUsesOf( +bool RecurrenceDescriptor::hasMultipleUsesOf( Instruction *I, SmallPtrSetImpl &Insts) { unsigned NumUses = 0; for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; @@ -313,8 +313,8 @@ bool ReductionDescriptor::hasMultipleUsesOf( return false; } -bool ReductionDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, - ReductionDescriptor &RedDes) { +bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, + RecurrenceDescriptor &RedDes) { bool HasFunNoNaNAttr = false; BasicBlock *Header = TheLoop->getHeader(); @@ -366,7 +366,8 @@ bool ReductionDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, /// This function returns the identity element (or neutral element) for /// the operation K. -Constant *ReductionDescriptor::getReductionIdentity(ReductionKind K, Type *Tp) { +Constant *RecurrenceDescriptor::getRecurrenceIdentity(RecurrenceKind K, + Type *Tp) { switch (K) { case RK_IntegerXor: case RK_IntegerAdd: @@ -386,12 +387,12 @@ Constant *ReductionDescriptor::getReductionIdentity(ReductionKind K, Type *Tp) { // Adding zero to a number does not change it. return ConstantFP::get(Tp, 0.0L); default: - llvm_unreachable("Unknown reduction kind"); + llvm_unreachable("Unknown recurrence kind"); } } -/// This function translates the reduction kind to an LLVM binary operator. -unsigned ReductionDescriptor::getReductionBinOp(ReductionKind Kind) { +/// This function translates the recurrence kind to an LLVM binary operator. +unsigned RecurrenceDescriptor::getRecurrenceBinOp(RecurrenceKind Kind) { switch (Kind) { case RK_IntegerAdd: return Instruction::Add; @@ -412,41 +413,40 @@ unsigned ReductionDescriptor::getReductionBinOp(ReductionKind Kind) { case RK_FloatMinMax: return Instruction::FCmp; default: - llvm_unreachable("Unknown reduction operation"); + llvm_unreachable("Unknown recurrence operation"); } } -Value * -ReductionDescriptor::createMinMaxOp(IRBuilder<> &Builder, - ReductionInstDesc::MinMaxReductionKind RK, - Value *Left, Value *Right) { +Value *RecurrenceDescriptor::createMinMaxOp( + IRBuilder<> &Builder, RecurrenceInstDesc::MinMaxRecurrenceKind RK, + Value *Left, Value *Right) { CmpInst::Predicate P = CmpInst::ICMP_NE; switch (RK) { default: - llvm_unreachable("Unknown min/max reduction kind"); - case ReductionInstDesc::MRK_UIntMin: + llvm_unreachable("Unknown min/max recurrence kind"); + case RecurrenceInstDesc::MRK_UIntMin: P = CmpInst::ICMP_ULT; break; - case ReductionInstDesc::MRK_UIntMax: + case RecurrenceInstDesc::MRK_UIntMax: P = CmpInst::ICMP_UGT; break; - case ReductionInstDesc::MRK_SIntMin: + case RecurrenceInstDesc::MRK_SIntMin: P = CmpInst::ICMP_SLT; break; - case ReductionInstDesc::MRK_SIntMax: + case RecurrenceInstDesc::MRK_SIntMax: P = CmpInst::ICMP_SGT; break; - case ReductionInstDesc::MRK_FloatMin: + case RecurrenceInstDesc::MRK_FloatMin: P = CmpInst::FCMP_OLT; break; - case ReductionInstDesc::MRK_FloatMax: + case RecurrenceInstDesc::MRK_FloatMax: P = CmpInst::FCMP_OGT; break; } Value *Cmp; - if (RK == ReductionInstDesc::MRK_FloatMin || - RK == ReductionInstDesc::MRK_FloatMax) + if (RK == RecurrenceInstDesc::MRK_FloatMin || + RK == RecurrenceInstDesc::MRK_FloatMax) Cmp = Builder.CreateFCmp(P, Left, Right, "rdx.minmax.cmp"); else Cmp = Builder.CreateICmp(P, Left, Right, "rdx.minmax.cmp"); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 95c9381..62de200 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -872,7 +872,7 @@ public: /// ReductionList contains the reduction descriptors for all /// of the reductions that were found in the loop. - typedef DenseMap ReductionList; + typedef DenseMap ReductionList; /// InductionList saves induction variables and maps them to the /// induction descriptor. @@ -3093,13 +3093,13 @@ void InnerLoopVectorizer::vectorizeLoop() { // Find the reduction variable descriptor. assert(Legal->getReductionVars()->count(RdxPhi) && "Unable to find the reduction variable"); - ReductionDescriptor RdxDesc = (*Legal->getReductionVars())[RdxPhi]; + RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[RdxPhi]; - ReductionDescriptor::ReductionKind RK = RdxDesc.getReductionKind(); - TrackingVH ReductionStartValue = RdxDesc.getReductionStartValue(); + RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind(); + TrackingVH ReductionStartValue = RdxDesc.getRecurrenceStartValue(); Instruction *LoopExitInst = RdxDesc.getLoopExitInstr(); - ReductionInstDesc::MinMaxReductionKind MinMaxKind = - RdxDesc.getMinMaxReductionKind(); + RecurrenceInstDesc::MinMaxRecurrenceKind MinMaxKind = + RdxDesc.getMinMaxRecurrenceKind(); setDebugLocFromInst(Builder, ReductionStartValue); // We need to generate a reduction vector from the incoming scalar. @@ -3116,8 +3116,8 @@ void InnerLoopVectorizer::vectorizeLoop() { // one for multiplication, -1 for And. Value *Identity; Value *VectorStart; - if (RK == ReductionDescriptor::RK_IntegerMinMax || - RK == ReductionDescriptor::RK_FloatMinMax) { + if (RK == RecurrenceDescriptor::RK_IntegerMinMax || + RK == RecurrenceDescriptor::RK_FloatMinMax) { // MinMax reduction have the start value as their identify. if (VF == 1) { VectorStart = Identity = ReductionStartValue; @@ -3127,8 +3127,8 @@ void InnerLoopVectorizer::vectorizeLoop() { } } else { // Handle other reduction kinds: - Constant *Iden = - ReductionDescriptor::getReductionIdentity(RK, VecTy->getScalarType()); + Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity( + RK, VecTy->getScalarType()); if (VF == 1) { Identity = Iden; // This vector is the Identity vector where the first element is the @@ -3185,7 +3185,7 @@ void InnerLoopVectorizer::vectorizeLoop() { // Reduce all of the unrolled parts into a single vector. Value *ReducedPartRdx = RdxParts[0]; - unsigned Op = ReductionDescriptor::getReductionBinOp(RK); + unsigned Op = RecurrenceDescriptor::getRecurrenceBinOp(RK); setDebugLocFromInst(Builder, ReducedPartRdx); for (unsigned part = 1; part < UF; ++part) { if (Op != Instruction::ICmp && Op != Instruction::FCmp) @@ -3194,7 +3194,7 @@ void InnerLoopVectorizer::vectorizeLoop() { Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxParts[part], ReducedPartRdx, "bin.rdx")); else - ReducedPartRdx = ReductionDescriptor::createMinMaxOp( + ReducedPartRdx = RecurrenceDescriptor::createMinMaxOp( Builder, MinMaxKind, ReducedPartRdx, RdxParts[part]); } @@ -3226,8 +3226,8 @@ void InnerLoopVectorizer::vectorizeLoop() { TmpVec = addFastMathFlag(Builder.CreateBinOp( (Instruction::BinaryOps)Op, TmpVec, Shuf, "bin.rdx")); else - TmpVec = ReductionDescriptor::createMinMaxOp(Builder, MinMaxKind, - TmpVec, Shuf); + TmpVec = RecurrenceDescriptor::createMinMaxOp(Builder, MinMaxKind, + TmpVec, Shuf); } // The result is in the first element of the vector. @@ -4040,8 +4040,8 @@ bool LoopVectorizationLegality::canVectorizeInstrs() { continue; } - if (ReductionDescriptor::isReductionPHI(Phi, TheLoop, - Reductions[Phi])) { + if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, + Reductions[Phi])) { AllowedExit.insert(Reductions[Phi].getLoopExitInstr()); continue; }