From 2cad359c91f79322ca5f304abd9c99d4f3a94a75 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Mon, 19 Nov 2018 16:51:57 +0000 Subject: [PATCH] Revert "[LICM] Make LICM able to hoist phis" This reverts commit r347190. llvm-svn: 347225 --- llvm/lib/Transforms/Scalar/LICM.cpp | 317 +----- llvm/test/Transforms/LICM/hoist-phi.ll | 1164 -------------------- .../LoopVectorize/invariant-store-vectorization.ll | 20 +- 3 files changed, 24 insertions(+), 1477 deletions(-) delete mode 100644 llvm/test/Transforms/LICM/hoist-phi.ll diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp index 9f6a34c..e30c25d 100644 --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -31,7 +31,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Scalar/LICM.h" -#include "llvm/ADT/SetOperations.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AliasSetTracker.h" @@ -42,7 +41,6 @@ #include "llvm/Analysis/GuardUtils.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/LoopIterator.h" #include "llvm/Analysis/LoopPass.h" #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/MemorySSA.h" @@ -77,8 +75,6 @@ using namespace llvm; #define DEBUG_TYPE "licm" -STATISTIC(NumCreatedBlocks, "Number of blocks created"); -STATISTIC(NumClonedBranches, "Number of branches cloned"); STATISTIC(NumSunk, "Number of instructions sunk out of loop"); STATISTIC(NumHoisted, "Number of instructions hoisted out of loop"); STATISTIC(NumMovedLoads, "Number of load insts hoisted or sunk"); @@ -107,7 +103,7 @@ static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop, const LoopSafetyInfo *SafetyInfo, TargetTransformInfo *TTI, bool &FreeInLoop); static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop, - BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo, + ICFLoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE); static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT, const Loop *CurLoop, ICFLoopSafetyInfo *SafetyInfo, @@ -441,225 +437,6 @@ bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, return Changed; } -// This is a helper class for hoistRegion to make it able to hoist control flow -// in order to be able to hoist phis. The way this works is that we initially -// start hoisting to the loop preheader, and when we see a loop invariant branch -// we make note of this. When we then come to hoist an instruction that's -// conditional on such a branch we duplicate the branch and the relevant control -// flow, then hoist the instruction into the block corresponding to its original -// block in the duplicated control flow. -class ControlFlowHoister { -private: - // Information about the loop we are hoisting from - LoopInfo *LI; - DominatorTree *DT; - Loop *CurLoop; - - // A map of blocks in the loop to the block their instructions will be hoisted - // to. - DenseMap HoistDestinationMap; - - // The branches that we can hoist, mapped to the block that marks a - // convergence point of their control flow. - DenseMap HoistableBranches; - -public: - ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop) - : LI(LI), DT(DT), CurLoop(CurLoop) {} - - void registerPossiblyHoistableBranch(BranchInst *BI) { - // We can only hoist conditional branches with loop invariant operands. - if (!BI->isConditional() || !CurLoop->hasLoopInvariantOperands(BI)) - return; - - // The branch destinations need to be in the loop, and we don't gain - // anything by duplicating conditional branches with duplicate successors, - // as it's essentially the same as an unconditional branch. - BasicBlock *TrueDest = BI->getSuccessor(0); - BasicBlock *FalseDest = BI->getSuccessor(1); - if (!CurLoop->contains(TrueDest) || !CurLoop->contains(FalseDest) || - TrueDest == FalseDest) - return; - - // We can hoist BI if one branch destination is the successor of the other, - // or both have common successor which we check by seeing if the - // intersection of their successors is non-empty. - // TODO: This could be expanded to allowing branches where both ends - // eventually converge to a single block. - SmallPtrSet TrueDestSucc, FalseDestSucc; - TrueDestSucc.insert(succ_begin(TrueDest), succ_end(TrueDest)); - FalseDestSucc.insert(succ_begin(FalseDest), succ_end(FalseDest)); - BasicBlock *CommonSucc = nullptr; - if (TrueDestSucc.count(FalseDest)) { - CommonSucc = FalseDest; - } else if (FalseDestSucc.count(TrueDest)) { - CommonSucc = TrueDest; - } else { - set_intersect(TrueDestSucc, FalseDestSucc); - // If there's one common successor use that. - if (TrueDestSucc.size() == 1) - CommonSucc = *TrueDestSucc.begin(); - // If there's more than one pick whichever appears first in the block list - // (we can't use the value returned by TrueDestSucc.begin() as it's - // unpredicatable which element gets returned). - else if (!TrueDestSucc.empty()) { - Function *F = TrueDest->getParent(); - auto IsSucc = [&](BasicBlock &BB) { return TrueDestSucc.count(&BB); }; - auto It = std::find_if(F->begin(), F->end(), IsSucc); - assert(It != F->end() && "Could not find successor in function"); - CommonSucc = &*It; - } - } - // The common successor has to be dominated by the branch, as otherwise - // there will be some other path to the successor that will not be - // controlled by this branch so any phi we hoist would be controlled by the - // wrong condition. This also takes care of avoiding hoisting of loop back - // edges. - // TODO: In some cases this could be relaxed if the successor is dominated - // by another block that's been hoisted and we can guarantee that the - // control flow has been replicated exactly. - if (CommonSucc && DT->dominates(BI, CommonSucc)) - HoistableBranches[BI] = CommonSucc; - } - - bool canHoistPHI(PHINode *PN) { - // The phi must have loop invariant operands. - if (!CurLoop->hasLoopInvariantOperands(PN)) - return false; - // We can hoist phis if the block they are in is the target of hoistable - // branches which cover all of the predecessors of the block. - SmallPtrSet PredecessorBlocks; - BasicBlock *BB = PN->getParent(); - for (BasicBlock *PredBB : predecessors(BB)) - PredecessorBlocks.insert(PredBB); - // If we have less predecessor blocks than predecessors then the phi will - // have more than one incoming value for the same block which we can't - // handle. - // TODO: This could be handled be erasing some of the duplicate incoming - // values. - if (PredecessorBlocks.size() != pred_size(BB)) - return false; - for (auto &Pair : HoistableBranches) { - if (Pair.second == BB) { - // Which blocks are predecessors via this branch depends on if the - // branch is triangle-like or diamond-like. - if (Pair.first->getSuccessor(0) == BB) { - PredecessorBlocks.erase(Pair.first->getParent()); - PredecessorBlocks.erase(Pair.first->getSuccessor(1)); - } else if (Pair.first->getSuccessor(1) == BB) { - PredecessorBlocks.erase(Pair.first->getParent()); - PredecessorBlocks.erase(Pair.first->getSuccessor(0)); - } else { - PredecessorBlocks.erase(Pair.first->getSuccessor(0)); - PredecessorBlocks.erase(Pair.first->getSuccessor(1)); - } - } - } - // PredecessorBlocks will now be empty if for every predecessor of BB we - // found a hoistable branch source. - return PredecessorBlocks.empty(); - } - - BasicBlock *getOrCreateHoistedBlock(BasicBlock *BB) { - // If BB has already been hoisted, return that - if (HoistDestinationMap.count(BB)) - return HoistDestinationMap[BB]; - - // Check if this block is conditional based on a pending branch - auto HasBBAsSuccessor = - [&](DenseMap::value_type &Pair) { - return BB != Pair.second && (Pair.first->getSuccessor(0) == BB || - Pair.first->getSuccessor(1) == BB); - }; - auto It = std::find_if(HoistableBranches.begin(), HoistableBranches.end(), - HasBBAsSuccessor); - - // If not involved in a pending branch, hoist to preheader - BasicBlock *InitialPreheader = CurLoop->getLoopPreheader(); - if (It == HoistableBranches.end()) { - LLVM_DEBUG(dbgs() << "LICM using " << InitialPreheader->getName() - << " as hoist destination for " << BB->getName() - << "\n"); - HoistDestinationMap[BB] = InitialPreheader; - return InitialPreheader; - } - BranchInst *BI = It->first; - assert(std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == - HoistableBranches.end() && - "BB is expected to be the target of at most one branch"); - - LLVMContext &C = BB->getContext(); - BasicBlock *TrueDest = BI->getSuccessor(0); - BasicBlock *FalseDest = BI->getSuccessor(1); - BasicBlock *CommonSucc = HoistableBranches[BI]; - BasicBlock *HoistTarget = getOrCreateHoistedBlock(BI->getParent()); - - // Create hoisted versions of blocks that currently don't have them - auto CreateHoistedBlock = [&](BasicBlock *Orig) { - if (HoistDestinationMap.count(Orig)) - return HoistDestinationMap[Orig]; - BasicBlock *New = - BasicBlock::Create(C, Orig->getName() + ".licm", Orig->getParent()); - HoistDestinationMap[Orig] = New; - DT->addNewBlock(New, HoistTarget); - if (CurLoop->getParentLoop()) - CurLoop->getParentLoop()->addBasicBlockToLoop(New, *LI); - ++NumCreatedBlocks; - LLVM_DEBUG(dbgs() << "LICM created " << New->getName() - << " as hoist destination for " << Orig->getName() - << "\n"); - return New; - }; - BasicBlock *HoistTrueDest = CreateHoistedBlock(TrueDest); - BasicBlock *HoistFalseDest = CreateHoistedBlock(FalseDest); - BasicBlock *HoistCommonSucc = CreateHoistedBlock(CommonSucc); - - // Link up these blocks with branches. - if (!HoistCommonSucc->getTerminator()) { - // The new common successor we've generated will branch to whatever that - // hoist target branched to. - BasicBlock *TargetSucc = HoistTarget->getSingleSuccessor(); - assert(TargetSucc && "Expected hoist target to have a single successor"); - HoistCommonSucc->moveBefore(TargetSucc); - BranchInst::Create(TargetSucc, HoistCommonSucc); - } - if (!HoistTrueDest->getTerminator()) { - HoistTrueDest->moveBefore(HoistCommonSucc); - BranchInst::Create(HoistCommonSucc, HoistTrueDest); - } - if (!HoistFalseDest->getTerminator()) { - HoistFalseDest->moveBefore(HoistCommonSucc); - BranchInst::Create(HoistCommonSucc, HoistFalseDest); - } - - // If BI is being cloned to what was originally the preheader then - // HoistCommonSucc will now be the new preheader. - if (HoistTarget == InitialPreheader) { - // Phis in the loop header now need to use the new preheader. - InitialPreheader->replaceSuccessorsPhiUsesWith(HoistCommonSucc); - // The new preheader dominates the loop header. - DomTreeNode *PreheaderNode = DT->getNode(HoistCommonSucc); - DomTreeNode *HeaderNode = DT->getNode(CurLoop->getHeader()); - DT->changeImmediateDominator(HeaderNode, PreheaderNode); - // The preheader hoist destination is now the new preheader, with the - // exception of the hoist destination of this branch. - for (auto &Pair : HoistDestinationMap) - if (Pair.second == InitialPreheader && Pair.first != BI->getParent()) - Pair.second = HoistCommonSucc; - } - - // Now finally clone BI. - ReplaceInstWithInst( - HoistTarget->getTerminator(), - BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition())); - ++NumClonedBranches; - - assert(CurLoop->getLoopPreheader() && - "Hoisting blocks should not have destroyed preheader"); - return HoistDestinationMap[BB]; - } -}; - /// Walk the specified region of the CFG (defined by all blocks dominated by /// the specified block, and that are in the current loop) in depth first /// order w.r.t the DominatorTree. This allows us to visit definitions before @@ -674,23 +451,13 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, CurLoop != nullptr && CurAST != nullptr && SafetyInfo != nullptr && "Unexpected input to hoistRegion"); - ControlFlowHoister CFH(LI, DT, CurLoop); - - // Keep track of instructions that have been hoisted, as they may need to be - // re-hoisted if they end up not dominating all of their uses. - SmallVector HoistedInstructions; - - // Record what the original preheader is, as we'll need it later if we need to - // re-hoist instructions. - BasicBlock *OriginalPreheader = CurLoop->getLoopPreheader(); + // We want to visit parents before children. We will enque all the parents + // before their children in the worklist and process the worklist in order. + SmallVector Worklist = collectChildrenInLoop(N, CurLoop); - // For PHI hoisting to work we need to hoist blocks before their successors. - // We can do this by iterating through the blocks in the loop in reverse - // post-order. - LoopBlocksRPO Worklist(CurLoop); - Worklist.perform(LI); bool Changed = false; - for (BasicBlock *BB : Worklist) { + for (DomTreeNode *DTN : Worklist) { + BasicBlock *BB = DTN->getBlock(); // Only need to process the contents of this block if it is not part of a // subloop (which would already have been processed). if (inSubLoop(BB, CurLoop, LI)) @@ -716,16 +483,13 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, // Try hoisting the instruction out to the preheader. We can only do // this if all of the operands of the instruction are loop invariant and // if it is safe to hoist the instruction. - // TODO: It may be safe to hoist if we are hoisting to a conditional block - // and we have accurately duplicated the control flow from the loop header - // to that block. + // if (CurLoop->hasLoopInvariantOperands(&I) && canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, true, ORE) && isSafeToExecuteUnconditionally( I, DT, CurLoop, SafetyInfo, ORE, CurLoop->getLoopPreheader()->getTerminator())) { - hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, ORE); - HoistedInstructions.push_back(&I); + hoist(I, DT, CurLoop, SafetyInfo, ORE); Changed = true; continue; } @@ -750,9 +514,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, I.replaceAllUsesWith(Product); eraseInstruction(I, *SafetyInfo, CurAST); - hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), - SafetyInfo, ORE); - HoistedInstructions.push_back(ReciprocalDivisor); + hoist(*ReciprocalDivisor, DT, CurLoop, SafetyInfo, ORE); Changed = true; continue; } @@ -764,58 +526,13 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, CurLoop->hasLoopInvariantOperands(&I) && SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) && SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop)) { - hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, ORE); - HoistedInstructions.push_back(&I); + hoist(I, DT, CurLoop, SafetyInfo, ORE); Changed = true; continue; } - - if (PHINode *PN = dyn_cast(&I)) { - if (CFH.canHoistPHI(PN)) { - // Redirect incoming blocks first to ensure that we create hoisted - // versions of those blocks before we hoist the phi. - for (unsigned int i = 0; i < PN->getNumIncomingValues(); ++i) - PN->setIncomingBlock( - i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i))); - hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, - ORE); - assert(DT->dominates(PN, BB) && "Conditional PHIs not expected"); - Changed = true; - continue; - } - } - - // Remember possibly hoistable branches so we can actually hoist them - // later if needed. - if (BranchInst *BI = dyn_cast(&I)) - CFH.registerPossiblyHoistableBranch(BI); } } - // If we hoisted instructions to a conditional block they may not dominate - // their uses that weren't hoisted (such as phis where some operands are not - // loop invariant). If so make them unconditional by moving them to the end of - // the original preheader, which is guaranteed to dominate everything in the - // loop. We iterate through the instructions in reverse order which ensures - // that when we rehoist an instruction we rehoist its operands. - Instruction *HoistPoint = OriginalPreheader->getTerminator(); - for (Instruction *I : reverse(HoistedInstructions)) { - if (!llvm::all_of(I->uses(), [&](Use &U) { return DT->dominates(I, U); })) { - LLVM_DEBUG(dbgs() << "LICM rehoisting to " << OriginalPreheader->getName() - << ": " << *I << "\n"); - moveInstructionBefore(*I, *HoistPoint, *SafetyInfo); - HoistPoint = I; - Changed = true; - } - } - - // Now that we've finished hoisting make sure that LI and DT are still valid. -#ifndef NDEBUG - assert(DT->verify(DominatorTree::VerificationLevel::Fast) && - "Dominator tree verification failed"); - LI->verify(*DT); -#endif - return Changed; } @@ -1383,9 +1100,9 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT, /// is safe to hoist, this instruction is called to do the dirty work. /// static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop, - BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo, - OptimizationRemarkEmitter *ORE) { - LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getName() << ": " << I + ICFLoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE) { + auto *Preheader = CurLoop->getLoopPreheader(); + LLVM_DEBUG(dbgs() << "LICM hoisting to " << Preheader->getName() << ": " << I << "\n"); ORE->emit([&]() { return OptimizationRemark(DEBUG_TYPE, "Hoisted", &I) << "hoisting " @@ -1403,12 +1120,8 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop, !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop)) I.dropUnknownNonDebugMetadata(); - if (isa(I)) - // Move the new node to the end of the phi list in the destination block. - moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo); - else - // Move the new node to the destination block, before its terminator. - moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo); + // Move the new node to the Preheader, before its terminator. + moveInstructionBefore(I, *Preheader->getTerminator(), *SafetyInfo); // Do not retain debug locations when we are moving instructions to different // basic blocks, because we want to avoid jumpy line tables. Calls, however, diff --git a/llvm/test/Transforms/LICM/hoist-phi.ll b/llvm/test/Transforms/LICM/hoist-phi.ll deleted file mode 100644 index e7a191b..0000000 --- a/llvm/test/Transforms/LICM/hoist-phi.ll +++ /dev/null @@ -1,1164 +0,0 @@ -; RUN: opt -S -licm < %s | FileCheck %s -; RUN: opt -passes='require,loop(licm)' -S < %s | FileCheck %s - -; CHECK-LABEL: @triangle_phi -define void @triangle_phi(i32 %x, i32* %p) { -; CHECK-LABEL: entry: -; CHECK: %cmp1 = icmp sgt i32 %x, 0 -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: %add = add i32 %x, 1 -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: phi i32 [ %add, %[[IF_LICM]] ], [ %x, %entry ] -; CHECK: store i32 %phi, i32* %p -; CHECK: %cmp2 = icmp ne i32 %phi, 0 -; CHECK: br label %loop - -loop: - %cmp1 = icmp sgt i32 %x, 0 - br i1 %cmp1, label %if, label %then - -if: - %add = add i32 %x, 1 - br label %then - -then: - %phi = phi i32 [ %add, %if ], [ %x, %loop ] - store i32 %phi, i32* %p - %cmp2 = icmp ne i32 %phi, 0 - br i1 %cmp2, label %loop, label %end - -end: - ret void -} - -; CHECK-LABEL: @diamond_phi -define void @diamond_phi(i32 %x, i32* %p) { -; CHECK-LABEL: entry: -; CHECK: %cmp1 = icmp sgt i32 %x, 0 -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: %add = add i32 %x, 1 -; CHECK: br label %[[THEN_LICM:.*]] - -; CHECK: [[ELSE_LICM]]: -; CHECK: %sub = sub i32 %x, 1 -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]] -; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ] -; CHECK: store i32 %phi, i32* %p -; CHECK: %cmp2 = icmp ne i32 %phi, 0 -; CHECK: br label %loop - -loop: - %cmp1 = icmp sgt i32 %x, 0 - br i1 %cmp1, label %if, label %else - -if: - %add = add i32 %x, 1 - br label %then - -else: - %sub = sub i32 %x, 1 - br label %then - -then: - %phi = phi i32 [ %add, %if ], [ %sub, %else ] - store i32 %phi, i32* %p - %cmp2 = icmp ne i32 %phi, 0 - br i1 %cmp2, label %loop, label %end - -end: - ret void -} - -; TODO: This is currently too complicated for us to be able to hoist the phi. -; CHECK-LABEL: @three_way_phi -define void @three_way_phi(i32 %x, i32* %p) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0 -; CHECK-DAG: %add = add i32 %x, 1 -; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0 -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]] - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM:.*]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %sub = sub i32 %x, 1 -; CHECK: br label %loop - -entry: - br label %loop - -loop: - %cmp1 = icmp sgt i32 %x, 0 - br i1 %cmp1, label %if, label %then - -if: - %add = add i32 %x, 1 - %cmp2 = icmp sgt i32 %add, 0 - br i1 %cmp2, label %if.if, label %then - -if.if: - %sub = sub i32 %x, 1 - br label %then - -then: - %phi = phi i32 [ 0, %loop ], [ %add, %if ], [ %sub, %if.if ] - store i32 %phi, i32* %p - %cmp3 = icmp ne i32 %phi, 0 - br i1 %cmp3, label %loop, label %end - -end: - ret void -} - -; TODO: This is currently too complicated for us to be able to hoist the phi. -; CHECK-LABEL: @tree_phi -define void @tree_phi(i32 %x, i32* %p) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0 -; CHECK-DAG: %add = add i32 %x, 1 -; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0 -; CHECK-DAG: %sub = sub i32 %x, 1 -; CHECK: br label %loop - -entry: - br label %loop - -loop: - %cmp1 = icmp sgt i32 %x, 0 - br i1 %cmp1, label %if, label %else - -if: - %add = add i32 %x, 1 - %cmp2 = icmp sgt i32 %add, 0 - br i1 %cmp2, label %if.if, label %if.else - -if.if: - br label %then - -if.else: - br label %then - -else: - %sub = sub i32 %x, 1 - br label %then - -then: - %phi = phi i32 [ %add, %if.if ], [ 0, %if.else ], [ %sub, %else ] - store i32 %phi, i32* %p - %cmp3 = icmp ne i32 %phi, 0 - br i1 %cmp3, label %loop, label %end - -end: - ret void -} - -; TODO: We can hoist the first phi, but not the second. -; CHECK-LABEL: @phi_phi -define void @phi_phi(i32 %x, i32* %p) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0 -; CHECK-DAG: %add = add i32 %x, 1 -; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0 -; CHECK-DAG: %sub = sub i32 %x, 1 -; CHECK: br i1 %cmp2, label %[[IF_IF_LICM:.*]], label %[[IF_ELSE_LICM:.*]] - -; CHECK: [[IF_IF_LICM]]: -; CHECK: br label %[[IF_THEN_LICM:.*]] - -; CHECK: [[IF_ELSE_LICM]]: -; CHECK: br label %[[IF_THEN_LICM]] - -; CHECK: [[IF_THEN_LICM]]: -; CHECK: %phi1 = phi i32 [ %add, %[[IF_IF_LICM]] ], [ 0, %[[IF_ELSE_LICM]] ] -; CHECK: br label %loop - -entry: - br label %loop - -loop: - %cmp1 = icmp sgt i32 %x, 0 - br i1 %cmp1, label %if, label %else - -if: - %add = add i32 %x, 1 - %cmp2 = icmp sgt i32 %add, 0 - br i1 %cmp2, label %if.if, label %if.else - -if.if: - br label %if.then - -if.else: - br label %if.then - -if.then: - %phi1 = phi i32 [ %add, %if.if ], [ 0, %if.else ] - br label %then - -else: - %sub = sub i32 %x, 1 - br label %then - -then: - %phi2 = phi i32 [ %phi1, %if.then ], [ %sub, %else ] - store i32 %phi2, i32* %p - %cmp3 = icmp ne i32 %phi2, 0 - br i1 %cmp3, label %loop, label %end - -end: - ret void -} - -; Check that we correctly duplicate empty control flow. -; CHECK-LABEL: @empty_triangle_phi -define i8 @empty_triangle_phi(i32 %x, i32 %y) { -; CHECK-LABEL: entry: -; CHECK: %cmp1 = icmp eq i32 %x, 0 -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ] -; CHECK: %cmp2 = icmp eq i32 %y, 0 -; CHECK: br label %loop - -loop: - %cmp1 = icmp eq i32 %x, 0 - br i1 %cmp1, label %if, label %then - -if: - br label %then - -then: - %phi = phi i8 [ 0, %if ], [ 1, %loop ] - %cmp2 = icmp eq i32 %y, 0 - br i1 %cmp2, label %end, label %loop - -end: - ret i8 %phi -} - -; CHECK-LABEL: @empty_diamond_phi -define i8 @empty_diamond_phi(i32 %x, i32 %y) { -; CHECK-LABEL: entry: -; CHECK: %cmp1 = icmp eq i32 %x, 0 -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM:.*]] - -; CHECK: [[ELSE_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ] -; CHECK: %cmp2 = icmp eq i32 %y, 0 -; CHECK: br label %loop - -loop: - %cmp1 = icmp eq i32 %x, 0 - br i1 %cmp1, label %if, label %else - -if: - br label %then - -else: - br label %then - -then: - %phi = phi i8 [ 0, %if ], [ 1, %else ] - %cmp2 = icmp eq i32 %y, 0 - br i1 %cmp2, label %end, label %loop - -end: - ret i8 %phi -} - -; Check that we correctly handle the case that the first thing we try to hoist is a phi. -; CHECK-LABEL: @empty_triangle_phi_first -define i8 @empty_triangle_phi_first(i32 %x, i1 %cond) { -; CHECK-LABEL: entry: -; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ] -; CHECK: %cmp = icmp eq i32 %x, 0 -; CHECK: br label %loop - -loop: - br i1 %cond, label %if, label %then - -if: - br label %then - -then: - %phi = phi i8 [ 0, %if ], [ 1, %loop ] - %cmp = icmp eq i32 %x, 0 - br i1 %cmp, label %end, label %loop - -end: - ret i8 %phi -} - -; CHECK-LABEL: @empty_diamond_phi -define i8 @empty_diamond_phi_first(i32 %x, i1 %cond) { -; CHECK-LABEL: entry: -; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM:.*]] - -; CHECK: [[ELSE_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ] -; CHECK: %cmp = icmp eq i32 %x, 0 -; CHECK: br label %loop - -loop: - br i1 %cond, label %if, label %else - -if: - br label %then - -else: - br label %then - -then: - %phi = phi i8 [ 0, %if ], [ 1, %else ] - %cmp = icmp eq i32 %x, 0 - br i1 %cmp, label %end, label %loop - -end: - ret i8 %phi -} - -; CHECK-LABEL: @empty_triangle_phi_first -define i8 @empty_triangle_phi_first_empty_loop_head(i32 %x, i1 %cond) { -; CHECK-LABEL: entry: -; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ] -; CHECK: %cmp = icmp eq i32 %x, 0 -; CHECK: br label %loop - -loop: - br label %test - -test: - br i1 %cond, label %if, label %then - -if: - br label %then - -then: - %phi = phi i8 [ 0, %if ], [ 1, %test ] - %cmp = icmp eq i32 %x, 0 - br i1 %cmp, label %end, label %loop - -end: - ret i8 %phi -} - -; CHECK-LABEL: @empty_diamond_phi_first_empty_loop_head -define i8 @empty_diamond_phi_first_empty_loop_head(i32 %x, i1 %cond) { -; CHECK-LABEL: entry: -; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM:.*]] - -; CHECK: [[ELSE_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ] -; CHECK: %cmp = icmp eq i32 %x, 0 -; CHECK: br label %loop - -loop: - br label %test - -test: - br i1 %cond, label %if, label %else - -if: - br label %then - -else: - br label %then - -then: - %phi = phi i8 [ 0, %if ], [ 1, %else ] - %cmp = icmp eq i32 %x, 0 - br i1 %cmp, label %end, label %loop - -end: - ret i8 %phi -} - -; The phi is on one branch of a diamond while simultaneously at the end of a -; triangle. Check that we duplicate the triangle and not the diamond. -; CHECK-LABEL: @triangle_diamond -define void @triangle_diamond(i32* %ptr, i32 %x, i32 %y) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cmp1 = icmp ne i32 %x, 0 -; CHECK-DAG: %cmp2 = icmp ne i32 %y, 0 -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %phi = phi i32 [ 0, %[[IF_LICM]] ], [ 127, %entry ] - -loop: - %cmp1 = icmp ne i32 %x, 0 - br i1 %cmp1, label %if, label %then - -if: - %cmp2 = icmp ne i32 %y, 0 - br i1 %cmp2, label %if.then, label %then - -then: - %phi = phi i32 [ 0, %if ], [ 127, %loop ] - store i32 %phi, i32* %ptr - br label %end - -if.then: - br label %end - -end: - br label %loop -} - -; As the previous, but the end of the diamond is the head of the loop. -; CHECK-LABEL: @triangle_diamond_backedge -define void @triangle_diamond_backedge(i32* %ptr, i32 %x, i32 %y) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cmp1 = icmp ne i32 %x, 0 -; CHECK-DAG: %cmp2 = icmp ne i32 %y, 0 -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %phi = phi i32 [ 0, %[[IF_LICM]] ], [ 127, %entry ] - -loop: - %cmp1 = icmp ne i32 %x, 0 - br i1 %cmp1, label %if, label %then - -if: - %cmp2 = icmp ne i32 %y, 0 - br i1 %cmp2, label %backedge, label %then - -then: - %phi = phi i32 [ 0, %if ], [ 127, %loop ] - store i32 %phi, i32* %ptr - br label %loop - -backedge: - br label %loop -} - -; TODO: The inner diamonds can be hoisted, but not currently the outer diamond -; CHECK-LABEL: @diamonds_inside_diamond -define void @diamonds_inside_diamond(i32 %x, i32* %p) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0 -; CHECK-DAG: %cmp3 = icmp slt i32 %x, -10 -; CHECK: br i1 %cmp3, label %[[ELSE_IF_LICM:.*]], label %[[ELSE_ELSE_LICM:.*]] -entry: - br label %loop - -; CHECK: [[ELSE_IF_LICM]]: -; CHECK: br label %[[ELSE_THEN_LICM:.*]] - -; CHECK: [[ELSE_ELSE_LICM]]: -; CHECK: br label %[[ELSE_THEN_LICM]] - -; CHECK: [[ELSE_THEN_LICM]]: -; CHECK: %phi2 = phi i32 [ 2, %[[ELSE_IF_LICM]] ], [ 3, %[[ELSE_ELSE_LICM]] ] -; CHECK: %cmp2 = icmp sgt i32 %x, 10 -; CHECK: br i1 %cmp2, label %[[IF_IF_LICM:.*]], label %[[IF_ELSE_LICM:.*]] - -; CHECK: [[IF_IF_LICM]]: -; CHECK: br label %[[IF_THEN_LICM:.*]] - -; CHECK: [[IF_ELSE_LICM]]: -; CHECK: br label %[[IF_THEN_LICM]] - -; CHECK: [[IF_THEN_LICM]]: -; CHECK: %phi1 = phi i32 [ 0, %[[IF_IF_LICM]] ], [ 1, %[[IF_ELSE_LICM]] ] -; CHECK: br label %loop - -loop: - %cmp1 = icmp sgt i32 %x, 0 - br i1 %cmp1, label %if, label %else - -if: - %cmp2 = icmp sgt i32 %x, 10 - br i1 %cmp2, label %if.if, label %if.else - -if.if: - br label %if.then - -if.else: - br label %if.then - -if.then: - %phi1 = phi i32 [ 0, %if.if ], [ 1, %if.else ] - br label %then - -else: - %cmp3 = icmp slt i32 %x, -10 - br i1 %cmp3, label %else.if, label %else.else - -else.if: - br label %else.then - -else.else: - br label %else.then - -else.then: - %phi2 = phi i32 [ 2, %else.if ], [ 3, %else.else ] - br label %then - -then: - %phi3 = phi i32 [ %phi1, %if.then ], [ %phi2, %else.then ] - store i32 %phi3, i32* %p - %cmp4 = icmp ne i32 %phi3, 0 - br i1 %cmp4, label %loop, label %end - -end: - ret void -} - -; We can hoist blocks that contain an edge that exits the loop by ignoring that -; edge in the hoisted block. -; CHECK-LABEL: @triangle_phi_loopexit -define void @triangle_phi_loopexit(i32 %x, i32* %p) { -; CHECK-LABEL: entry: -; CHECK-DAG: %add = add i32 %x, 1 -; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0 -; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %x, %entry ] - -loop: - %cmp1 = icmp sgt i32 %x, 0 - br i1 %cmp1, label %if, label %then - -if: - %add = add i32 %x, 1 - %cmp2 = icmp sgt i32 10, %add - br i1 %cmp2, label %then, label %end - -then: - %phi = phi i32 [ %add, %if ], [ %x, %loop ] - store i32 %phi, i32* %p - %cmp3 = icmp ne i32 %phi, 0 - br i1 %cmp3, label %loop, label %end - -end: - ret void -} - -; CHECK-LABEL: @diamond_phi_oneloopexit -define void @diamond_phi_oneloopexit(i32 %x, i32* %p) { -; CHECK-LABEL: entry: -; CHECK-DAG: %add = add i32 %x, 1 -; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0 -; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM:.*]] - -; CHECK: [[ELSE_LICM]]: -; CHECK: %sub = sub i32 %x, 1 -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]] -; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ] -; CHECK: %cmp3 = icmp ne i32 %phi, 0 -; CHECK: br label %loop - -loop: - %cmp1 = icmp sgt i32 %x, 0 - br i1 %cmp1, label %if, label %else - -if: - %add = add i32 %x, 1 - %cmp2 = icmp sgt i32 10, %add - br i1 %cmp2, label %then, label %end - -else: - %sub = sub i32 %x, 1 - br label %then - -then: - %phi = phi i32 [ %add, %if ], [ %sub, %else ] - store i32 %phi, i32* %p - %cmp3 = icmp ne i32 %phi, 0 - br i1 %cmp3, label %loop, label %end - -end: - ret void -} - -; CHECK-LABEL: @diamond_phi_twoloopexit -define void @diamond_phi_twoloopexit(i32 %x, i32* %p) { -; CHECK-LABEL: entry: -; CHECK-DAG: %sub = sub i32 %x, 1 -; CHECK-DAG: %add = add i32 %x, 1 -; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0 -; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add -; CHECK-DAG: %cmp3 = icmp sgt i32 10, %sub -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM:.*]] - -; CHECK: [[ELSE_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]] -; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ] -; CHECK: %cmp4 = icmp ne i32 %phi, 0 -; CHECK: br label %loop - -loop: - %cmp1 = icmp sgt i32 %x, 0 - br i1 %cmp1, label %if, label %else - -if: - %add = add i32 %x, 1 - %cmp2 = icmp sgt i32 10, %add - br i1 %cmp2, label %then, label %end - -else: - %sub = sub i32 %x, 1 - %cmp3 = icmp sgt i32 10, %sub - br i1 %cmp3, label %then, label %end - -then: - %phi = phi i32 [ %add, %if ], [ %sub, %else ] - store i32 %phi, i32* %p - %cmp4 = icmp ne i32 %phi, 0 - br i1 %cmp4, label %loop, label %end - -end: - ret void -} - -; The store cannot be hoisted, so add and shr cannot be hoisted into a -; conditional block. -; CHECK-LABEL: @conditional_use -define void @conditional_use(i32 %x, i32* %p) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cond = icmp ugt i32 %x, 0 -; CHECK-DAG: %add = add i32 %x, 5 -; CHECK-DAG: %shr = ashr i32 %add, 1 -; CHECK: br label %loop -entry: - br label %loop - -loop: - %cond = icmp ugt i32 %x, 0 - br i1 %cond, label %if, label %else - -; CHECK-LABEL: if: -; CHECK: store i32 %shr, i32* %p, align 4 -if: - %add = add i32 %x, 5 - %shr = ashr i32 %add, 1 - store i32 %shr, i32* %p, align 4 - br label %then - -else: - br label %then - -then: - br label %loop -} - -; A diamond with two triangles on the left and one on the right. This test is -; to check that we have a unique loop preheader when we hoist the store (and so -; don't fail an assertion). -; CHECK-LABEL: @triangles_in_diamond -define void @triangles_in_diamond(i32* %ptr) { -; CHECK-LABEL: entry: -; CHECK: store i32 0, i32* %ptr, align 4 -; CHECK: br label %loop -entry: - br label %loop - -loop: - br i1 undef, label %left_triangle_1, label %right_triangle - -left_triangle_1: - br i1 undef, label %left_triangle_1_if, label %left_triangle_2 - -left_triangle_1_if: - br label %left_triangle_2 - -left_triangle_2: - br i1 undef, label %left_triangle_2_if, label %left_triangle_2_then - -left_triangle_2_if: - br label %left_triangle_2_then - -left_triangle_2_then: - br label %loop.end - -right_triangle: - br i1 undef, label %right_triangle.if, label %right_triangle.then - -right_triangle.if: - br label %right_triangle.then - -right_triangle.then: - br label %loop.end - -loop.end: - store i32 0, i32* %ptr, align 4 - br label %loop -} - -; %cmp dominates its used after being hoisted, but not after %brmerge is rehoisted -; CHECK-LABEL: @rehoist -define void @rehoist(i8* %this, i32 %x) { -; CHECK-LABEL: entry: -; CHECK-DAG: %sub = add nsw i32 %x, -1 -; CHECK-DAG: %fptr = bitcast i8* %this to void (i8*)* -; CHECK-DAG: %cmp = icmp eq i32 0, %sub -; CHECK-DAG: %brmerge = or i1 %cmp, true -entry: - %sub = add nsw i32 %x, -1 - br label %loop - -loop: - br i1 undef, label %if1, label %else1 - -if1: - %fptr = bitcast i8* %this to void (i8*)* - call void %fptr(i8* %this) - br label %then1 - -else1: - br label %then1 - -then1: - %cmp = icmp eq i32 0, %sub - br i1 %cmp, label %end, label %else2 - -else2: - %brmerge = or i1 %cmp, true - br i1 %brmerge, label %if3, label %end - -if3: - br label %end - -end: - br label %loop -} - -; A test case that uses empty blocks in a way that can cause control flow -; hoisting to get confused. -; CHECK-LABEL: @empty_blocks_multiple_conditional_branches -define void @empty_blocks_multiple_conditional_branches(float %arg, float* %ptr) { -; CHECK-LABEL: entry -; CHECK-DAG: %div1 = fmul float %arg, 4.000000e+00 -; CHECK-DAG: %div2 = fmul float %arg, 2.000000e+00 -entry: - br label %loop - -; The exact path to the phi isn't checked here, because it depends on whether -; cond2 or cond3 is hoisted first -; CHECK: %phi = phi float [ 0.000000e+00, %{{.*}} ], [ %div1, %{{.*}} ] -; CHECK: br label %loop - -loop: - br i1 undef, label %backedge2, label %cond1 - -cond1: - br i1 undef, label %cond1.if, label %cond1.else - -cond1.else: - br label %cond3 - -cond1.if: - br label %cond1.if.next - -cond1.if.next: - br label %cond2 - -cond2: - %div1 = fmul float %arg, 4.000000e+00 - br i1 undef, label %cond2.if, label %cond2.then - -cond2.if: - br label %cond2.then - -cond2.then: - %phi = phi float [ 0.000000e+00, %cond2 ], [ %div1, %cond2.if ] - store float %phi, float* %ptr - br label %backedge2 - -cond3: - br i1 undef, label %cond3.then, label %cond3.if - -cond3.if: - %div2 = fmul float %arg, 2.000000e+00 - store float %div2, float* %ptr - br label %cond3.then - -cond3.then: - br label %loop - -backedge2: - br label %loop -} - -; We can't do much here, so mainly just check that we don't crash. -; CHECK-LABEL: @many_path_phi -define void @many_path_phi(i32* %ptr1, i32* %ptr2) { -; CHECK-LABEL: entry: -; CHECK-DAG: %gep3 = getelementptr inbounds i32, i32* %ptr2, i32 2 -; CHECK-DAG: %gep2 = getelementptr inbounds i32, i32* %ptr2, i32 2 -; CHECK: br label %loop -entry: - br label %loop - -loop: - %phi1 = phi i32 [ 0, %entry ], [ %phi2, %end ] - %cmp1 = icmp ugt i32 %phi1, 3 - br i1 %cmp1, label %cond2, label %cond1 - -cond1: - br i1 undef, label %end, label %cond1.else - -cond1.else: - %gep2 = getelementptr inbounds i32, i32* %ptr2, i32 2 - %val2 = load i32, i32* %gep2, align 4 - %cmp2 = icmp eq i32 %val2, 13 - br i1 %cmp2, label %cond1.end, label %end - -cond1.end: - br label %end - -cond2: - br i1 undef, label %end, label %cond2.else - -cond2.else: - %gep3 = getelementptr inbounds i32, i32* %ptr2, i32 2 - %val3 = load i32, i32* %gep3, align 4 - %cmp3 = icmp eq i32 %val3, 13 - br i1 %cmp3, label %cond2.end, label %end - -cond2.end: - br label %end - -end: - %phi2 = phi i32 [ 1, %cond1 ], [ 2, %cond1.else ], [ 3, %cond1.end ], [ 4, %cond2 ], [ 5, %cond2.else ], [ 6, %cond2.end ] - br label %loop -} - -; Check that we correctly handle the hoisting of %gep when theres a critical -; edge that branches to the preheader. -; CHECK-LABEL: @crit_edge -define void @crit_edge(i32* %ptr, i32 %idx, i1 %cond1, i1 %cond2) { -; CHECK-LABEL: entry: -; CHECK: %gep = getelementptr inbounds i32, i32* %ptr, i32 %idx -; CHECK: br label %preheader -entry: - br label %preheader - -preheader: - br label %loop - -loop: - br i1 %cond1, label %then, label %if - -if: - %gep = getelementptr inbounds i32, i32* %ptr, i32 %idx - %val = load i32, i32* %gep - br label %then - -then: - %phi = phi i32 [ %val, %if ], [ 0, %loop ] - store i32 %phi, i32* %ptr - br i1 %cond2, label %loop, label %crit_edge - -crit_edge: - br label %preheader -} - -; Check that the conditional sub is correctly hoisted from the inner loop to the -; preheader of the outer loop. -; CHECK-LABEL: @hoist_from_innermost_loop -define void @hoist_from_innermost_loop(i32 %nx, i32* %ptr) { -; CHECK-LABEL: entry: -; CHECK-DAG: %sub = sub nsw i32 0, %nx -; CHECK: br label %outer_loop -entry: - br label %outer_loop - -outer_loop: - br label %middle_loop - -middle_loop: - br label %inner_loop - -inner_loop: - br i1 undef, label %inner_loop_end, label %if - -if: - %sub = sub nsw i32 0, %nx - store i32 %sub, i32* %ptr, align 4 - br label %inner_loop_end - -inner_loop_end: - br i1 undef, label %inner_loop, label %middle_loop_end - -middle_loop_end: - br i1 undef, label %middle_loop, label %outer_loop_end - -outer_loop_end: - br label %outer_loop -} - -; We have a diamond starting from %if, but %if.if is also reachable from %loop, -; so %gep should not be conditionally hoisted. -; CHECK-LABEL: @diamond_with_extra_in_edge -define void @diamond_with_extra_in_edge(i32* %ptr1, i32* %ptr2, i32 %arg) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cmp2 = icmp ne i32 0, %arg -; CHECK-DAG: %gep = getelementptr i32, i32* %ptr1, i32 4 -; CHECK: br label %loop -entry: - br label %loop - -loop: - %phi1 = phi i32 [ 0, %entry ], [ %phi2, %then ] - %cmp1 = icmp ugt i32 16, %phi1 - br i1 %cmp1, label %if, label %if.if - -if: - %cmp2 = icmp ne i32 0, %arg - br i1 %cmp2, label %if.if, label %if.else - -if.if: - %gep = getelementptr i32, i32* %ptr1, i32 4 - %val = load i32, i32* %gep, align 4 - br label %then - -if.else: - br label %then - -then: - %phi2 = phi i32 [ %val, %if.if ], [ %phi1, %if.else ] - store i32 %phi2, i32* %ptr2, align 4 - br label %loop -} - -; %loop/%if/%then form a triangle, but %loop/%if/%then/%end also form a diamond. -; The triangle should be picked for conditional hoisting. -; CHECK-LABEL: @both_triangle_and_diamond -define void @both_triangle_and_diamond(i32* %ptr1, i32* %ptr2, i32 %arg) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cmp1 = icmp ne i32 0, %arg -; CHECK-DAG: %gep = getelementptr i32, i32* %ptr1, i32 4 -; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]] -entry: - br label %loop - -; CHECK: [[IF_LICM]]: -; CHECK: br label %[[THEN_LICM]] - -; CHECK: [[THEN_LICM]]: -; CHECK: %phi2 = phi i32 [ 0, %[[IF_LICM]] ], [ 1, %entry ] -; CHECK: br label %loop - -loop: - %phi1 = phi i32 [ 0, %entry ], [ %phi3, %end ] - %cmp1 = icmp ne i32 0, %arg - br i1 %cmp1, label %if, label %then - -if: - %gep = getelementptr i32, i32* %ptr1, i32 4 - %val = load i32, i32* %gep, align 4 - %cmp2 = icmp ugt i32 16, %phi1 - br i1 %cmp2, label %end, label %then - -then: - %phi2 = phi i32 [ 0, %if ], [ 1, %loop ] - br label %end - -end: - %phi3 = phi i32 [ %phi2, %then ], [ %val, %if ] - store i32 %phi3, i32* %ptr2, align 4 - br label %loop -} - -; We shouldn't duplicate the branch at the end of %loop and should instead hoist -; %val to %entry. -; CHECK-LABEL: @same_destination_branch -define i32 @same_destination_branch(i32 %arg1, i32 %arg2) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cmp1 = icmp ne i32 %arg2, 0 -; CHECK-DAG: %val = add i32 %arg1, 1 -; CHECK: br label %loop -entry: - br label %loop - -loop: - %phi = phi i32 [ 0, %entry ], [ %add, %then ] - %add = add i32 %phi, 1 - %cmp1 = icmp ne i32 %arg2, 0 - br i1 %cmp1, label %if, label %if - -if: - %val = add i32 %arg1, 1 - br label %then - -then: - %cmp2 = icmp ne i32 %val, %phi - br i1 %cmp2, label %loop, label %end - -end: - ret i32 %val -} - -; Diamond-like control flow but the left/right blocks actually have the same -; destinations. -; TODO: We could potentially hoist all of phi2-4, but currently only hoist phi2. -; CHECK-LABEL: @diamond_like_same_destinations -define i32 @diamond_like_same_destinations(i32 %arg1, i32 %arg2) { -; CHECK-LABEL: entry: -; CHECK-DAG: %cmp1 = icmp ne i32 %arg1, 0 -; CHECK-DAG: %cmp2 = icmp ugt i32 %arg2, 1 -; CHECK-DAG: %cmp3 = icmp ugt i32 %arg2, 2 -; CHECK: br i1 %cmp1, label %[[LEFT1_LICM:.*]], label %[[RIGHT1_LICM:.*]] -entry: - br label %loop - -; CHECK: [[LEFT1_LICM]]: -; CHECK: br label %[[LEFT2_LICM:.*]] - -; CHECK: [[RIGHT1_LICM]]: -; CHECK: br label %[[LEFT2_LICM]] - -; CHECK: [[LEFT2_LICM]]: -; CHECK: %phi2 = phi i32 [ 0, %[[LEFT1_LICM]] ], [ 1, %[[RIGHT1_LICM]] ] -; CHECK: br label %loop - -loop: - %phi1 = phi i32 [ 0, %entry ], [ %add, %loopend ] - %add = add i32 %phi1, 1 - %cmp1 = icmp ne i32 %arg1, 0 - br i1 %cmp1, label %left1, label %right1 - -left1: - %cmp2 = icmp ugt i32 %arg2, 1 - br i1 %cmp2, label %left2, label %right2 - -right1: - %cmp3 = icmp ugt i32 %arg2, 2 - br i1 %cmp3, label %left2, label %right2 - -left2: - %phi2 = phi i32 [ 0, %left1 ], [ 1, %right1 ] - br label %loopend - -right2: - %phi3 = phi i32 [ 2, %left1 ], [ 3, %right1 ] - br label %loopend - -loopend: - %phi4 = phi i32 [ %phi2, %left2 ], [ %phi3, %right2 ] - %cmp4 = icmp ne i32 %phi1, 32 - br i1 %cmp4, label %loop, label %end - -end: - ret i32 %phi4 -} - -; A phi with multiple incoming values for the same block due to a branch with -; two destinations that are actually the same. We can't hoist this. -; TODO: This could be hoisted by erasing one of the incoming values. -; CHECK-LABEL: @phi_multiple_values_same_block -define i32 @phi_multiple_values_same_block(i32 %arg) { -; CHECK-LABEL: entry: -; CHECK: %cmp = icmp sgt i32 %arg, 4 -; CHECK-NOT: phi -; CHECK: br label %loop -entry: - br label %loop - -loop: - %cmp = icmp sgt i32 %arg, 4 - br i1 %cmp, label %if, label %then - -if: - br i1 undef, label %then, label %then - -then: - %phi = phi i32 [ %arg, %loop ], [ 1, %if ], [ 1, %if ] - br i1 undef, label %exit, label %loop - -exit: - ret i32 %phi -} diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll index cf1257b..cc4c55c 100644 --- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll +++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll @@ -266,26 +266,19 @@ for.end: ; preds = %for.body ; variant/invariant values being stored to invariant address. ; test checks that the last element of the phi is extracted and scalar stored ; into the uniform address within the loop. -; Since the condition and the phi is loop invariant, they are LICM'ed before +; Since the condition and the phi is loop invariant, they are LICM'ed after ; vectorization. ; CHECK-LABEL: inv_val_store_to_inv_address_conditional_inv ; CHECK-NEXT: entry: -; CHECK-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* -; CHECK-NEXT: [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8* ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[NTRUNC]], [[K:%.*]] -; CHECK-NEXT: br i1 [[CMP]], label %[[COND_STORE_LICM:.*]], label %[[COND_STORE_K_LICM:.*]] -; CHECK: [[COND_STORE_LICM]]: -; CHECK-NEXT: br label %[[LATCH_LICM:.*]] -; CHECK: [[COND_STORE_K_LICM]]: -; CHECK-NEXT: br label %[[LATCH_LICM]] -; CHECK: [[LATCH_LICM]]: -; CHECK-NEXT: [[STOREVAL:%.*]] = phi i32 [ [[NTRUNC]], %[[COND_STORE_LICM]] ], [ [[K]], %[[COND_STORE_K_LICM]] ] ; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[N]], 1 ; CHECK-NEXT: [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: +; CHECK-NEXT: [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8* +; CHECK-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8* ; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i64 [[N]], 1 ; CHECK-NEXT: [[SMAX2:%.*]] = select i1 [[TMP1]], i64 [[N]], i64 1 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX2]] @@ -298,13 +291,17 @@ for.end: ; preds = %for.body ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> undef, i32 [[NTRUNC]], i32 0 ; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i1> undef, i1 [[CMP]], i32 3 +; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> undef, i32 [[K]], i32 3 +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32> [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[PREDPHI]], i32 3 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>* ; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32>* [[TMP7]], align 4 -; CHECK-NEXT: store i32 [[STOREVAL]], i32* [[A]], align 4 +; CHECK-NEXT: store i32 [[TMP5]], i32* [[A]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]] @@ -324,6 +321,7 @@ for.end: ; preds = %for.body ; CHECK: cond_store_k: ; CHECK-NEXT: br label [[LATCH]] ; CHECK: latch: +; CHECK-NEXT: [[STOREVAL:%.*]] = phi i32 [ [[NTRUNC]], [[COND_STORE]] ], [ [[K]], [[COND_STORE_K]] ] ; CHECK-NEXT: store i32 [[STOREVAL]], i32* [[A]], align 4 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] -- 2.7.4