#endif // DEBUG
-/*****************************************************************************
- * Find the natural loops, using dominators. Note that the test for
- * a loop is slightly different from the standard one, because we have
- * not done a depth first reordering of the basic blocks.
- */
-
-void Compiler::optFindNaturalLoops()
+namespace
{
-#ifdef DEBUG
- if (verbose)
+//------------------------------------------------------------------------
+// LoopSearch: Class that handles scanning a range of blocks to detect a loop,
+// moving blocks to make the loop body contiguous, and recording
+// the loop.
+//
+// We will use the following terminology:
+// HEAD - the basic block that flows into the loop ENTRY block (Currently MUST be lexically before entry).
+// Not part of the looping of the loop.
+// FIRST - the lexically first basic block (in bbNext order) within this loop.
+// TOP - the target of the backward edge from BOTTOM. In most cases FIRST and TOP are the same.
+// BOTTOM - the lexically last block in the loop (i.e. the block from which we jump to the top)
+// EXIT - the predecessor of loop's unique exit edge, if it has a unique exit edge; else nullptr
+// ENTRY - the entry in the loop (not necessarly the TOP), but there must be only one entry
+//
+// We (currently) require the body of a loop to be a contiguous (in bbNext order) sequence of basic blocks.
+// When the loop is identified, blocks will be moved out to make it a compact contiguous region if possible,
+// and in cases where compaction is not possible, we'll subsequently treat all blocks in the lexical range
+// between TOP and BOTTOM as part of the loop even if they aren't part of the SCC.
+// Regarding nesting: Since a given block can only have one back-edge (we only detect loops with back-edges
+// from BBJ_COND or BBJ_ALWAYS blocks), no two loops will share the same BOTTOM. Two loops may share the
+// same FIRST/TOP/ENTRY as reported by LoopSearch, and optCanonicalizeLoopNest will subsequently re-write
+// the CFG so that no two loops share the same FIRST/TOP/ENTRY anymore.
+//
+// |
+// v
+// head
+// |
+// | top/first <--+
+// | | |
+// | ... |
+// | | |
+// | v |
+// +---> entry |
+// | |
+// ... |
+// | |
+// v |
+// +-- exit/tail |
+// | | |
+// | ... |
+// | | |
+// | v |
+// | bottom ---+
+// |
+// +------+
+// |
+// v
+//
+class LoopSearch
+{
+
+ // Keeping track of which blocks are in the loop requires two block sets since we may add blocks
+ // as we go but the BlockSet type's max ID doesn't increase to accomodate them. Define a helper
+ // struct to make the ensuing code more readable.
+ struct LoopBlockSet
{
- printf("*************** In optFindNaturalLoops()\n");
- }
-#endif // DEBUG
+ private:
+ // Keep track of blocks with bbNum <= oldBlockMaxNum in a regular BlockSet, since
+ // it can hold all of them.
+ BlockSet oldBlocksInLoop; // Blocks with bbNum <= oldBlockMaxNum
- flowList* pred;
- flowList* predTop;
- flowList* predEntry;
+ // Keep track of blocks with bbNum > oldBlockMaxNum in a separate BlockSet, but
+ // indexing them by (blockNum - oldBlockMaxNum); since we won't generate more than
+ // one new block per old block, this must be sufficient to track any new blocks.
+ BlockSet newBlocksInLoop; // Blocks with bbNum > oldBlockMaxNum
- noway_assert(fgDomsComputed);
- assert(fgHasLoops);
+ Compiler* comp;
+ unsigned int oldBlockMaxNum;
-#if COUNT_LOOPS
- hasMethodLoops = false;
- loopsThisMethod = 0;
- loopOverflowThisMethod = false;
-#endif
+ public:
+ LoopBlockSet(Compiler* comp)
+ : oldBlocksInLoop(BlockSetOps::UninitVal())
+ , newBlocksInLoop(BlockSetOps::UninitVal())
+ , comp(comp)
+ , oldBlockMaxNum(comp->fgBBNumMax)
+ {
+ }
+
+ void Reset(unsigned int seedBlockNum)
+ {
+ if (BlockSetOps::MayBeUninit(oldBlocksInLoop))
+ {
+ // Either the block sets are uninitialized (and long), so we need to initialize
+ // them (and allocate their backing storage), or they are short and empty, so
+ // assigning MakeEmpty to them is as cheap as ClearD.
+ oldBlocksInLoop = BlockSetOps::MakeEmpty(comp);
+ newBlocksInLoop = BlockSetOps::MakeEmpty(comp);
+ }
+ else
+ {
+ // We know the backing storage is already allocated, so just clear it.
+ BlockSetOps::ClearD(comp, oldBlocksInLoop);
+ BlockSetOps::ClearD(comp, newBlocksInLoop);
+ }
+ assert(seedBlockNum <= oldBlockMaxNum);
+ BlockSetOps::AddElemD(comp, oldBlocksInLoop, seedBlockNum);
+ }
- /* We will use the following terminology:
- * HEAD - the basic block that flows into the loop ENTRY block (Currently MUST be lexically before entry).
- Not part of the looping of the loop.
- * FIRST - the lexically first basic block (in bbNext order) within this loop. (May be part of a nested loop,
- * but not the outer loop. ???)
- * TOP - the target of the backward edge from BOTTOM. In most cases FIRST and TOP are the same.
- * BOTTOM - the lexically last block in the loop (i.e. the block from which we jump to the top)
- * EXIT - the loop exit or the block right after the bottom
- * ENTRY - the entry in the loop (not necessarly the TOP), but there must be only one entry
- *
- * We (currently) require the body of a loop to be a contiguous (in bbNext order) sequence of basic blocks.
-
- |
- v
- head
- |
- | top/beg <--+
- | | |
- | ... |
- | | |
- | v |
- +---> entry |
- | |
- ... |
- | |
- v |
- +-- exit/tail |
- | | |
- | ... |
- | | |
- | v |
- | bottom ---+
- |
- +------+
- |
- v
+ bool CanRepresent(unsigned int blockNum)
+ {
+ // We can represent old blocks up to oldBlockMaxNum, and
+ // new blocks up to 2 * oldBlockMaxNum.
+ return (blockNum <= 2 * oldBlockMaxNum);
+ }
- */
+ bool IsMember(unsigned int blockNum)
+ {
+ if (blockNum > oldBlockMaxNum)
+ {
+ return BlockSetOps::IsMember(comp, newBlocksInLoop, blockNum - oldBlockMaxNum);
+ }
+ return BlockSetOps::IsMember(comp, oldBlocksInLoop, blockNum);
+ }
- bool mod = false;
- BasicBlock* head;
- BasicBlock* top;
- BasicBlock* bottom;
- BasicBlock* entry;
- BasicBlock* lastExit;
- unsigned char exitCount;
- unsigned int oldBlockMaxNum = fgBBNumMax;
- BlockSet bottomBlocks = BlockSetOps::MakeEmpty(this);
+ void Insert(unsigned int blockNum)
+ {
+ if (blockNum > oldBlockMaxNum)
+ {
+ BlockSetOps::AddElemD(comp, newBlocksInLoop, blockNum - oldBlockMaxNum);
+ }
+ else
+ {
+ BlockSetOps::AddElemD(comp, oldBlocksInLoop, blockNum);
+ }
+ }
+
+ bool TestAndInsert(unsigned int blockNum)
+ {
+ if (blockNum > oldBlockMaxNum)
+ {
+ unsigned int shiftedNum = blockNum - oldBlockMaxNum;
+ if (!BlockSetOps::IsMember(comp, newBlocksInLoop, shiftedNum))
+ {
+ BlockSetOps::AddElemD(comp, newBlocksInLoop, shiftedNum);
+ return false;
+ }
+ }
+ else
+ {
+ if (!BlockSetOps::IsMember(comp, oldBlocksInLoop, blockNum))
+ {
+ BlockSetOps::AddElemD(comp, oldBlocksInLoop, blockNum);
+ return false;
+ }
+ }
+ return true;
+ }
+ };
+
+ LoopBlockSet loopBlocks; // Set of blocks identified as part of the loop
+ Compiler* comp;
+
+ // See LoopSearch class comment header for a diagram relating these fields:
+ BasicBlock* head; // Predecessor of unique entry edge
+ BasicBlock* first; // Lexically first in-loop block
+ BasicBlock* top; // Successor of back-edge from BOTTOM
+ BasicBlock* bottom; // Predecessor of back-edge to TOP, also lexically last in-loop block
+ BasicBlock* entry; // Successor of unique entry edge
+
+ BasicBlock* lastExit; // Most recently discovered exit block
+ unsigned char exitCount; // Number of discovered exit edges
+ unsigned int oldBlockMaxNum; // Used to identify new blocks created during compaction
+ BlockSet bottomBlocks; // BOTTOM blocks of already-recorded loops
#ifdef DEBUG
- bool forgotExit = false;
+ bool forgotExit = false; // Flags a rare case where lastExit gets nulled out, for assertions
#endif
+ bool changedFlowGraph = false; // Signals that loop compaction has modified the flow graph
- // Make sure we've renumbered such that the bitsets can hold all the bits
- assert(oldBlockMaxNum <= fgCurBBEpochSize);
+public:
+ LoopSearch(Compiler* comp)
+ : loopBlocks(comp), comp(comp), oldBlockMaxNum(comp->fgBBNumMax), bottomBlocks(BlockSetOps::MakeEmpty(comp))
+ {
+ // Make sure we've renumbered such that the bitsets can hold all the bits
+ assert(comp->fgBBNumMax <= comp->fgCurBBEpochSize);
+ }
- for (head = fgFirstBB; head->bbNext; head = head->bbNext)
+ //------------------------------------------------------------------------
+ // RecordLoop: Notify the Compiler that a loop has been found.
+ //
+ // Return Value:
+ // true - Loop successfully recorded.
+ // false - Compiler has run out of loop descriptors; loop not recorded.
+ //
+ bool RecordLoop()
{
- top = head->bbNext;
- lastExit = nullptr;
- exitCount = 0;
+ /* At this point we have a compact loop - record it in the loop table
+ * If we found only one exit, record it in the table too
+ * (otherwise an exit = nullptr in the loop table means multiple exits) */
- // Blocks that are rarely run have a zero bbWeight and should
- // never be optimized here
+ BasicBlock* onlyExit = (exitCount == 1 ? lastExit : nullptr);
+ if (comp->optRecordLoop(head, first, top, entry, bottom, onlyExit, exitCount))
+ {
+ // Record the BOTTOM block for future reference before returning.
+ assert(bottom->bbNum <= oldBlockMaxNum);
+ BlockSetOps::AddElemD(comp, bottomBlocks, bottom->bbNum);
+ return true;
+ }
- if (top->bbWeight == BB_ZERO_WEIGHT)
+ // Unable to record this loop because the loop descriptor table overflowed.
+ return false;
+ }
+
+ //------------------------------------------------------------------------
+ // ChangedFlowGraph: Determine whether loop compaction has modified the flow graph.
+ //
+ // Return Value:
+ // true - The flow graph has been modified; fgUpdateChangedFlowGraph should
+ // be called (which is the caller's responsibility).
+ // false - The flow graph has not been modified by this LoopSearch.
+ //
+ bool ChangedFlowGraph()
+ {
+ return changedFlowGraph;
+ }
+
+ //------------------------------------------------------------------------
+ // FindLoop: Search for a loop with the given HEAD block and back-edge.
+ //
+ // Arguments:
+ // head - Block to be the HEAD of any loop identified
+ // top - Block to be the TOP of any loop identified
+ // bottom - Block to be the BOTTOM of any loop identified
+ //
+ // Return Value:
+ // true - Found a valid loop.
+ // false - Did not find a valid loop.
+ //
+ // Notes:
+ // May modify flow graph to make loop compact before returning.
+ // Will set instance fields to track loop's extent and exits if a valid
+ // loop is found, and potentially trash them otherwise.
+ //
+ bool FindLoop(BasicBlock* head, BasicBlock* top, BasicBlock* bottom)
+ {
+ /* Is this a loop candidate? - We look for "back edges", i.e. an edge from BOTTOM
+ * to TOP (note that this is an abuse of notation since this is not necessarily a back edge
+ * as the definition says, but merely an indication that we have a loop there).
+ * Thus, we have to be very careful and after entry discovery check that it is indeed
+ * the only place we enter the loop (especially for non-reducible flow graphs).
+ */
+
+ if (top->bbNum > bottom->bbNum) // is this a backward edge? (from BOTTOM to TOP)
{
- continue;
+ // Edge from BOTTOM to TOP is not a backward edge
+ return false;
}
- for (pred = top->bbPreds; pred; pred = pred->flNext)
+ if (bottom->bbNum > oldBlockMaxNum)
{
- /* Is this a loop candidate? - We look for "back edges", i.e. an edge from BOTTOM
- * to TOP (note that this is an abuse of notation since this is not necessarily a back edge
- * as the definition says, but merely an indication that we have a loop there).
- * Thus, we have to be very careful and after entry discovery check that it is indeed
- * the only place we enter the loop (especially for non-reducible flow graphs).
- */
+ // Not a true back-edge; bottom is a block added to reconnect fall-through during
+ // loop processing, so its block number does not reflect its position.
+ return false;
+ }
- bottom = pred->flBlock;
- exitCount = 0;
+ if ((bottom->bbJumpKind == BBJ_EHFINALLYRET) || (bottom->bbJumpKind == BBJ_EHFILTERRET) ||
+ (bottom->bbJumpKind == BBJ_EHCATCHRET) || (bottom->bbJumpKind == BBJ_CALLFINALLY) ||
+ (bottom->bbJumpKind == BBJ_SWITCH))
+ {
+ /* BBJ_EHFINALLYRET, BBJ_EHFILTERRET, BBJ_EHCATCHRET, and BBJ_CALLFINALLY can never form a loop.
+ * BBJ_SWITCH that has a backward jump appears only for labeled break. */
+ return false;
+ }
- if (top->bbNum <= bottom->bbNum) // is this a backward edge? (from BOTTOM to TOP)
- {
- if (bottom->bbNum > oldBlockMaxNum)
- {
- // Not a true back-edge; bottom is a block added to reconnect fall-through during
- // loop processing, so its block number does not reflect its position.
- goto NO_LOOP;
- }
+ /* The presence of a "back edge" is an indication that a loop might be present here
+ *
+ * LOOP:
+ * 1. A collection of STRONGLY CONNECTED nodes i.e. there is a path from any
+ * node in the loop to any other node in the loop (wholly within the loop)
+ * 2. The loop has a unique ENTRY, i.e. there is only one way to reach a node
+ * in the loop from outside the loop, and that is through the ENTRY
+ */
- if ((bottom->bbJumpKind == BBJ_EHFINALLYRET) || (bottom->bbJumpKind == BBJ_EHFILTERRET) ||
- (bottom->bbJumpKind == BBJ_EHCATCHRET) || (bottom->bbJumpKind == BBJ_CALLFINALLY) ||
- (bottom->bbJumpKind == BBJ_SWITCH))
- {
- /* BBJ_EHFINALLYRET, BBJ_EHFILTERRET, BBJ_EHCATCHRET, and BBJ_CALLFINALLY can never form a loop.
- * BBJ_SWITCH that has a backward jump appears only for labeled break. */
- goto NO_LOOP;
- }
+ /* Let's find the loop ENTRY */
+ BasicBlock* entry = FindEntry(head, top, bottom);
- BasicBlock* loopBlock;
+ if (entry == nullptr)
+ {
+ // For now, we only recognize loops where HEAD has some successor ENTRY in the loop.
+ return false;
+ }
- /* The presence of a "back edge" is an indication that a loop might be present here
- *
- * LOOP:
- * 1. A collection of STRONGLY CONNECTED nodes i.e. there is a path from any
- * node in the loop to any other node in the loop (wholly within the loop)
- * 2. The loop has a unique ENTRY, i.e. there is only one way to reach a node
- * in the loop from outside the loop, and that is through the ENTRY
- */
+ // Passed the basic checks; initialize instance state for this back-edge.
+ this->head = head;
+ this->top = top;
+ this->entry = entry;
+ this->bottom = bottom;
+ this->lastExit = nullptr;
+ this->exitCount = 0;
- /* Let's find the loop ENTRY */
+ // Now we find the "first" block -- the earliest block reachable within the loop.
+ // With our current algorithm, this is always the same as "top".
+ this->first = top;
- if (head->bbJumpKind == BBJ_ALWAYS)
- {
- if (head->bbJumpDest->bbNum <= bottom->bbNum && head->bbJumpDest->bbNum >= top->bbNum)
- {
- /* OK - we enter somewhere within the loop */
- entry = head->bbJumpDest;
+ if (!HasSingleEntryCycle())
+ {
+ // There isn't actually a loop between TOP and BOTTOM
+ return false;
+ }
- /* some useful asserts
- * Cannot enter at the top - should have being caught by redundant jumps */
+ if (!loopBlocks.IsMember(top->bbNum))
+ {
+ // The "back-edge" we identified isn't actually part of the flow cycle containing ENTRY
+ return false;
+ }
- assert((entry != top) || (head->bbFlags & BBF_KEEP_BBJ_ALWAYS));
- }
- else
- {
- /* special case - don't consider now */
- // assert (!"Loop entered in weird way!");
- goto NO_LOOP;
- }
- }
- // Can we fall through into the loop?
- else if (head->bbJumpKind == BBJ_NONE || head->bbJumpKind == BBJ_COND)
- {
- /* The ENTRY is at the TOP (a do-while loop) */
- entry = top;
- }
- else
- {
- goto NO_LOOP; // head does not flow into the loop bail for now
- }
+ // Disqualify loops where the first block of the loop is less nested in EH than
+ // the bottom block. That is, we don't want to handle loops where the back edge
+ // goes from within an EH region to a first block that is outside that same EH
+ // region. Note that we *do* handle loops where the first block is the *first*
+ // block of a more nested EH region (since it is legal to branch to the first
+ // block of an immediately more nested EH region). So, for example, disqualify
+ // this:
+ //
+ // BB02
+ // ...
+ // try {
+ // ...
+ // BB10 BBJ_COND => BB02
+ // ...
+ // }
+ //
+ // Here, BB10 is more nested than BB02.
- // Now we find the "first" block -- the earliest block reachable within the loop.
- // With our current algorithm, this is always the same as "top".
- BasicBlock* first = top;
+ if (bottom->hasTryIndex() && !comp->bbInTryRegions(bottom->getTryIndex(), first))
+ {
+ JITDUMP("Loop 'first' BB%02u is in an outer EH region compared to loop 'bottom' BB%02u. Rejecting "
+ "loop.\n",
+ first->bbNum, bottom->bbNum);
+ return false;
+ }
- // Now do a backwards flow walk from entry to see if we have a single-entry loop
- bool foundCycle = false;
+#if FEATURE_EH_FUNCLETS && defined(_TARGET_ARM_)
+ // Disqualify loops where the first block of the loop is a finally target.
+ // The main problem is when multiple loops share a 'first' block that is a finally
+ // target and we canonicalize the loops by adding a new loop head. In that case, we
+ // need to update the blocks so the finally target bit is moved to the newly created
+ // block, and removed from the old 'first' block. This is 'hard', so at this point
+ // in the RyuJIT codebase (when we don't expect to keep the "old" ARM32 code generator
+ // long-term), it's easier to disallow the loop than to update the flow graph to
+ // support this case.
+
+ if ((first->bbFlags & BBF_FINALLY_TARGET) != 0)
+ {
+ JITDUMP("Loop 'first' BB%02u is a finally target. Rejecting loop.\n", first->bbNum);
+ return false;
+ }
+#endif // FEATURE_EH_FUNCLETS && defined(_TARGET_ARM_)
- // Keeping track of which blocks are in the loop requires two block sets since we may add blocks
- // as we go but the BlockSet type's max ID doesn't increase to accomodate them. Define a helper
- // struct to make the ensuing code more readable.
- struct LoopBlockSet
- {
- private:
- // Keep track of blocks with bbNum <= oldBlockMaxNum in a regular BlockSet, since
- // it can hold all of them.
- BlockSet oldBlocksInLoop; // Blocks with bbNum <= oldBlockMaxNum
-
- // Keep track of blocks with bbNum > oldBlockMaxNum in a separate BlockSet, but
- // indexing them by (blockNum - oldBlockMaxNum); since we won't generate more than
- // one new block per old block, this must be sufficient to track any new blocks.
- BlockSet newBlocksInLoop; // Blocks with bbNum > oldBlockMaxNum
-
- Compiler* comp;
- unsigned int oldBlockMaxNum;
-
- public:
- LoopBlockSet(Compiler* comp, unsigned int oldBlockMaxNum, unsigned int seedBlockNum)
- : comp(comp), oldBlockMaxNum(oldBlockMaxNum)
- {
- assert(seedBlockNum <= oldBlockMaxNum);
- oldBlocksInLoop = BlockSetOps::MakeSingleton(comp, seedBlockNum);
- newBlocksInLoop = BlockSetOps::MakeEmpty(comp);
- }
+ // Compact the loop (sweep through it and move out any blocks that aren't part of the
+ // flow cycle), and find the exits.
+ if (!MakeCompactAndFindExits())
+ {
+ // Unable to preserve well-formed loop during compaction.
+ return false;
+ }
- bool isMember(unsigned int blockNum)
- {
- if (blockNum > oldBlockMaxNum)
- {
- return BlockSetOps::IsMember(comp, newBlocksInLoop, blockNum - oldBlockMaxNum);
- }
- return BlockSetOps::IsMember(comp, oldBlocksInLoop, blockNum);
- }
+ // We have a valid loop.
+ return true;
+ }
- void insert(unsigned int blockNum)
- {
- if (blockNum > oldBlockMaxNum)
- {
- BlockSetOps::AddElemD(comp, newBlocksInLoop, blockNum - oldBlockMaxNum);
- }
- else
- {
- BlockSetOps::AddElemD(comp, oldBlocksInLoop, blockNum);
- }
- }
+private:
+ //------------------------------------------------------------------------
+ // FindEntry: See if given HEAD flows to valid ENTRY between given TOP and BOTTOM
+ //
+ // Arguments:
+ // head - Block to be the HEAD of any loop identified
+ // top - Block to be the TOP of any loop identified
+ // bottom - Block to be the BOTTOM of any loop identified
+ //
+ // Return Value:
+ // Block to be the ENTRY of any loop identified, or nullptr if no
+ // such entry meeting our criteria can be found.
+ //
+ // Notes:
+ // Returns main entry if one is found, does not check for side-entries.
+ //
+ BasicBlock* FindEntry(BasicBlock* head, BasicBlock* top, BasicBlock* bottom)
+ {
+ if (head->bbJumpKind == BBJ_ALWAYS)
+ {
+ if (head->bbJumpDest->bbNum <= bottom->bbNum && head->bbJumpDest->bbNum >= top->bbNum)
+ {
+ /* OK - we enter somewhere within the loop */
+
+ /* some useful asserts
+ * Cannot enter at the top - should have being caught by redundant jumps */
- bool testAndInsert(unsigned int blockNum)
+ assert((head->bbJumpDest != top) || (head->bbFlags & BBF_KEEP_BBJ_ALWAYS));
+
+ return head->bbJumpDest;
+ }
+ else
+ {
+ /* special case - don't consider now */
+ // assert (!"Loop entered in weird way!");
+ return nullptr;
+ }
+ }
+ // Can we fall through into the loop?
+ else if (head->bbJumpKind == BBJ_NONE || head->bbJumpKind == BBJ_COND)
+ {
+ /* The ENTRY is at the TOP (a do-while loop) */
+ return top;
+ }
+ else
+ {
+ return nullptr; // head does not flow into the loop bail for now
+ }
+ }
+
+ //------------------------------------------------------------------------
+ // HasSingleEntryCycle: Perform a reverse flow walk from ENTRY, visiting
+ // only blocks between TOP and BOTTOM, to determine if such a cycle
+ // exists and if it has a single entry.
+ //
+ // Return Value:
+ // true - Found a single-entry cycle.
+ // false - Did not find a single-entry cycle.
+ //
+ // Notes:
+ // Will mark (in `loopBlocks`) all blocks found to participate in the
+ // cycle.
+ //
+ bool HasSingleEntryCycle()
+ {
+ // Now do a backwards flow walk from entry to see if we have a single-entry loop
+ bool foundCycle = false;
+
+ // Seed the loop block set and worklist with the entry block.
+ loopBlocks.Reset(entry->bbNum);
+ jitstd::list<BasicBlock*> worklist(comp->getAllocator());
+ worklist.push_back(entry);
+
+ while (!worklist.empty())
+ {
+ BasicBlock* block = worklist.back();
+ worklist.pop_back();
+
+ /* Make sure ENTRY dominates all blocks in the loop
+ * This is necessary to ensure condition 2. above
+ */
+ if (block->bbNum > oldBlockMaxNum)
+ {
+ // This is a new block we added to connect fall-through, so the
+ // recorded dominator information doesn't cover it. Just continue,
+ // and when we process its unique predecessor we'll abort if ENTRY
+ // doesn't dominate that.
+ }
+ else if (!comp->fgDominate(entry, block))
+ {
+ return false;
+ }
+
+ // Add preds to the worklist, checking for side-entries.
+ for (flowList* predIter = block->bbPreds; predIter != nullptr; predIter = predIter->flNext)
+ {
+ BasicBlock* pred = predIter->flBlock;
+
+ unsigned int testNum = PositionNum(pred);
+
+ if ((testNum < top->bbNum) || (testNum > bottom->bbNum))
+ {
+ // Pred is out of loop range
+ if (block == entry)
{
- if (blockNum > oldBlockMaxNum)
+ if (pred == head)
{
- unsigned int shiftedNum = blockNum - oldBlockMaxNum;
- if (!BlockSetOps::IsMember(comp, newBlocksInLoop, shiftedNum))
- {
- BlockSetOps::AddElemD(comp, newBlocksInLoop, shiftedNum);
- return false;
- }
+ // This is the single entry we expect.
+ continue;
}
- else
+ // ENTRY has some pred other than head outside the loop. If ENTRY does not
+ // dominate this pred, we'll consider this a side-entry and skip this loop;
+ // otherwise the loop is still valid and this may be a (flow-wise) back-edge
+ // of an outer loop. For the dominance test, if `pred` is a new block, use
+ // its unique predecessor since the dominator tree has info for that.
+ BasicBlock* effectivePred = (pred->bbNum > oldBlockMaxNum ? pred->bbPrev : pred);
+ if (comp->fgDominate(entry, effectivePred))
{
- if (!BlockSetOps::IsMember(comp, oldBlocksInLoop, blockNum))
- {
- BlockSetOps::AddElemD(comp, oldBlocksInLoop, blockNum);
- return false;
- }
+ // Outer loop back-edge
+ continue;
}
- return true;
}
- };
- // Also we'll sometimes want to get a block's position using its predecessor's
- // number if it is a new block, so define a helper for that.
- auto positionNum = [oldBlockMaxNum](BasicBlock* block) -> unsigned int {
- if (block->bbNum > oldBlockMaxNum)
- {
- // This must be a block we inserted to connect fall-through after moving blocks.
- // To determine if it's in the loop or not, use the number of its unique predecessor
- // block.
- assert(block->bbPreds->flBlock == block->bbPrev);
- assert(block->bbPreds->flNext == nullptr);
- return block->bbPrev->bbNum;
- }
- return block->bbNum;
- };
- // Seed the loop block set and worklist with the entry block.
- LoopBlockSet loopBlocks(this, oldBlockMaxNum, entry->bbNum);
- jitstd::list<BasicBlock*> worklist(getAllocator());
- worklist.push_back(entry);
+ // There are multiple entries to this loop, don't consider it.
+ return false;
+ }
- while (!worklist.empty())
+ if (pred == entry)
{
- BasicBlock* block = worklist.back();
- worklist.pop_back();
+ // We have indeed found a cycle in the flow graph.
+ foundCycle = true;
+ assert(loopBlocks.IsMember(pred->bbNum));
+ }
+ else if (!loopBlocks.TestAndInsert(pred->bbNum))
+ {
+ // Add this pred to the worklist
+ worklist.push_back(pred);
- /* Make sure ENTRY dominates all blocks in the loop
- * This is necessary to ensure condition 2. above
- */
- if (block->bbNum > oldBlockMaxNum)
- {
- // This is a new block we added to connect fall-through, so the
- // recorded dominator information doesn't cover it. Just continue,
- // and when we process its unique predecessor we'll abort if ENTRY
- // doesn't dominate that.
- }
- else if (!fgDominate(entry, block))
+ if ((pred->bbNext != nullptr) && (PositionNum(pred->bbNext) == pred->bbNum))
{
- goto NO_LOOP;
+ // We've created a new block immediately after `pred` to
+ // reconnect what was fall-through. Mark it as in-loop also;
+ // it needs to stay with `prev` and if it exits the loop we'd
+ // just need to re-create it if we tried to move it out.
+ loopBlocks.Insert(pred->bbNext->bbNum);
}
+ }
+ }
+ }
- // Add preds to the worklist, checking for side-entries.
- for (flowList* predIter = block->bbPreds; predIter != nullptr; predIter = predIter->flNext)
- {
- BasicBlock* pred = predIter->flBlock;
+ return foundCycle;
+ }
- unsigned int testNum = positionNum(pred);
+ //------------------------------------------------------------------------
+ // PositionNum: Get the number identifying a block's position per the
+ // lexical ordering that existed before searching for (and compacting)
+ // loops.
+ //
+ // Arguments:
+ // block - Block whose position is desired.
+ //
+ // Return Value:
+ // A number indicating that block's position relative to others.
+ //
+ // Notes:
+ // When the given block is a new one created during loop compaction,
+ // the number of its unique predecessor is returned.
+ //
+ unsigned int PositionNum(BasicBlock* block)
+ {
+ if (block->bbNum > oldBlockMaxNum)
+ {
+ // This must be a block we inserted to connect fall-through after moving blocks.
+ // To determine if it's in the loop or not, use the number of its unique predecessor
+ // block.
+ assert(block->bbPreds->flBlock == block->bbPrev);
+ assert(block->bbPreds->flNext == nullptr);
+ return block->bbPrev->bbNum;
+ }
+ return block->bbNum;
+ }
- if ((testNum < top->bbNum) || (testNum > bottom->bbNum))
- {
- // Pred is out of loop range
- if (block == entry)
- {
- if (pred == head)
- {
- // This is the single entry we expect.
- continue;
- }
- // ENTRY has some pred other than head outside the loop. If ENTRY does not
- // dominate this pred, we'll consider this a side-entry and skip this loop;
- // otherwise the loop is still valid and this may be a (flow-wise) back-edge
- // of an outer loop. For the dominance test, if `pred` is a new block, use
- // its unique predecessor since the dominator tree has info for that.
- BasicBlock* effectivePred = (pred->bbNum > oldBlockMaxNum ? pred->bbPrev : pred);
- if (fgDominate(entry, effectivePred))
- {
- // Outer loop back-edge
- continue;
- }
- }
+ //------------------------------------------------------------------------
+ // MakeCompactAndFindExits: Compact the loop (sweep through it and move out
+ // any blocks that aren't part of the flow cycle), and find the exits (set
+ // lastExit and exitCount).
+ //
+ // Return Value:
+ // true - Loop successfully compacted (or `loopBlocks` expanded to
+ // include all blocks in the lexical range), exits enumerated.
+ // false - Loop cannot be made compact and remain well-formed.
+ //
+ bool MakeCompactAndFindExits()
+ {
+ // Compaction (if it needs to happen) will require an insertion point.
+ BasicBlock* moveAfter = nullptr;
- // There are multiple entries to this loop, don't consider it.
- goto NO_LOOP;
- }
+ for (BasicBlock* previous = top->bbPrev; previous != bottom;)
+ {
+ BasicBlock* block = previous->bbNext;
- if (pred == entry)
- {
- // We have indeed found a cycle in the flow graph.
- foundCycle = true;
- assert(loopBlocks.isMember(pred->bbNum));
- }
- else if (!loopBlocks.testAndInsert(pred->bbNum))
- {
- // Add this pred to the worklist
- worklist.push_back(pred);
+ if (loopBlocks.IsMember(block->bbNum))
+ {
+ // This block is a member of the loop. Check to see if it may exit the loop.
+ CheckForExit(block);
- if ((pred->bbNext != nullptr) && (positionNum(pred->bbNext) == pred->bbNum))
- {
- // We've created a new block immediately after `pred` to
- // reconnect what was fall-through. Mark it as in-loop also;
- // it needs to stay with `prev` and if it exits the loop we'd
- // just need to re-create it if we tried to move it out.
- loopBlocks.insert(pred->bbNext->bbNum);
- }
- }
- }
- }
+ // Done processing this block; move on to the next.
+ previous = block;
+ continue;
+ }
+
+ // This blocks is lexically between TOP and BOTTOM, but it does not
+ // participate in the flow cycle. Check for a run of consecutive
+ // such blocks.
+ BasicBlock* lastNonLoopBlock = block;
+ BasicBlock* nextLoopBlock = block->bbNext;
+ while (!loopBlocks.IsMember(nextLoopBlock->bbNum))
+ {
+ lastNonLoopBlock = nextLoopBlock;
+ nextLoopBlock = nextLoopBlock->bbNext;
+ // This loop must terminate because we know BOTTOM is in loopBlocks.
+ }
+
+ // Choose an insertion point for non-loop blocks if we haven't yet done so.
+ if (moveAfter == nullptr)
+ {
+ moveAfter = FindInsertionPoint();
+ }
- if (!foundCycle)
+ if (!BasicBlock::sameEHRegion(previous, nextLoopBlock) || !BasicBlock::sameEHRegion(previous, moveAfter))
+ {
+ // EH regions would be ill-formed if we moved these blocks out.
+ // See if we can consider them loop blocks without introducing
+ // a side-entry.
+ if (CanTreatAsLoopBlocks(block, lastNonLoopBlock))
{
- // There isn't actually a loop between TOP and BOTTOM
- goto NO_LOOP;
+ // The call to `canTreatAsLoop` marked these blocks as part of the loop;
+ // iterate without updating `previous` so that we'll analyze them as part
+ // of the loop.
+ continue;
}
-
- if (!loopBlocks.isMember(top->bbNum))
+ else
{
- // The "back-edge" we identified isn't actually part of the flow cycle containing ENTRY
- goto NO_LOOP;
+ // We can't move these out of the loop or leave them in, so just give
+ // up on this loop.
+ return false;
}
+ }
- // Disqualify loops where the first block of the loop is less nested in EH than
- // the bottom block. That is, we don't want to handle loops where the back edge
- // goes from within an EH region to a first block that is outside that same EH
- // region. Note that we *do* handle loops where the first block is the *first*
- // block of a more nested EH region (since it is legal to branch to the first
- // block of an immediately more nested EH region). So, for example, disqualify
- // this:
- //
- // BB02
- // ...
- // try {
- // ...
- // BB10 BBJ_COND => BB02
- // ...
- // }
- //
- // Here, BB10 is more nested than BB02.
+ // Now physically move the blocks.
+ BasicBlock* moveBefore = moveAfter->bbNext;
- if (bottom->hasTryIndex() && !bbInTryRegions(bottom->getTryIndex(), first))
- {
- JITDUMP("Loop 'first' BB%02u is in an outer EH region compared to loop 'bottom' BB%02u. Rejecting "
- "loop.\n",
- first->bbNum, bottom->bbNum);
- goto NO_LOOP;
- }
+ comp->fgUnlinkRange(block, lastNonLoopBlock);
+ comp->fgMoveBlocksAfter(block, lastNonLoopBlock, moveAfter);
+ comp->ehUpdateLastBlocks(moveAfter, lastNonLoopBlock);
-#if FEATURE_EH_FUNCLETS && defined(_TARGET_ARM_)
- // Disqualify loops where the first block of the loop is a finally target.
- // The main problem is when multiple loops share a 'first' block that is a finally
- // target and we canonicalize the loops by adding a new loop head. In that case, we
- // need to update the blocks so the finally target bit is moved to the newly created
- // block, and removed from the old 'first' block. This is 'hard', so at this point
- // in the RyuJIT codebase (when we don't expect to keep the "old" ARM32 code generator
- // long-term), it's easier to disallow the loop than to update the flow graph to
- // support this case.
-
- if ((first->bbFlags & BBF_FINALLY_TARGET) != 0)
- {
- JITDUMP("Loop 'first' BB%02u is a finally target. Rejecting loop.\n", first->bbNum);
- goto NO_LOOP;
- }
-#endif // FEATURE_EH_FUNCLETS && defined(_TARGET_ARM_)
+ // Apply any adjustments needed for fallthrough at the boundaries of the moved region.
+ FixupFallThrough(moveAfter, moveBefore, block);
+ FixupFallThrough(lastNonLoopBlock, nextLoopBlock, moveBefore);
+ // Also apply any adjustments needed where the blocks were snipped out of the loop.
+ BasicBlock* newBlock = FixupFallThrough(previous, block, nextLoopBlock);
+ if (newBlock != nullptr)
+ {
+ // This new block is in the loop and is a loop exit.
+ loopBlocks.Insert(newBlock->bbNum);
+ lastExit = newBlock;
+ ++exitCount;
+ }
- // Compact the loop (sweep through it and move out any blocks that aren't part of the
- // flow cycle), and count the exits.
- BasicBlock* moveAfter = nullptr;
- for (BasicBlock* previous = top->bbPrev; previous != bottom;)
- {
- BasicBlock* block = previous->bbNext;
+ // Update moveAfter for the next insertion.
+ moveAfter = lastNonLoopBlock;
- if (!loopBlocks.isMember(block->bbNum))
- {
- // This code is lexically between TOP and BOTTOM, but it does not
- // participate in the flow cycle. Check for a run of consecutive
- // such blocks.
- BasicBlock* lastNonLoopBlock = block;
- BasicBlock* nextLoopBlock = block->bbNext;
- while (!loopBlocks.isMember(nextLoopBlock->bbNum))
- {
- lastNonLoopBlock = nextLoopBlock;
- nextLoopBlock = nextLoopBlock->bbNext;
- // This loop must terminate because we know BOTTOM is in loopBlocks.
- }
+ // Note that we've changed the flow graph, and continue without updating
+ // `previous` so that we'll process nextLoopBlock.
+ changedFlowGraph = true;
+ }
- if (moveAfter == nullptr)
- {
- // Find an insertion point for blocks we're going to move. Move them down
- // out of the loop, and if possible find a spot that won't break up fall-through.
- moveAfter = bottom;
- while (moveAfter->bbFallsThrough())
- {
- // Avoid splitting up this fall-through if we can; see if we can legally
- // change `moveAfter` to the next block after it.
- BasicBlock* next = moveAfter->bbNext;
+ if ((exitCount == 1) && (lastExit == nullptr))
+ {
+ // If we happen to have a loop with two exits, one of which goes to an
+ // infinite loop that's lexically nested inside it, where the inner loop
+ // can't be moved out, we can end up in this situation (because
+ // CanTreatAsLoopBlocks will have decremented the count expecting to find
+ // another exit later). Bump the exit count to 2, since downstream code
+ // will not be prepared for null lastExit with exitCount of 1.
+ assert(forgotExit);
+ exitCount = 2;
+ }
- if (!BasicBlock::sameEHRegion(moveAfter, next))
- {
- // Don't cross an EH region boundary
- break;
- }
+ // Loop compaction was successful
+ return true;
+ }
- if (next->isBBCallAlwaysPair())
- {
- // Don't split up a CallFinally/BBJ_ALWAYS pair
- break;
- }
+ //------------------------------------------------------------------------
+ // FindInsertionPoint: Find an appropriate spot to which blocks that are
+ // lexically between TOP and BOTTOM but not part of the flow cycle
+ // can be moved.
+ //
+ // Return Value:
+ // Block after which to insert moved blocks.
+ //
+ BasicBlock* FindInsertionPoint()
+ {
+ // Find an insertion point for blocks we're going to move. Move them down
+ // out of the loop, and if possible find a spot that won't break up fall-through.
+ BasicBlock* moveAfter = bottom;
+ while (moveAfter->bbFallsThrough())
+ {
+ // Keep looking for a better insertion point if we can.
+ BasicBlock* newMoveAfter = TryAdvanceInsertionPoint(moveAfter);
- if ((next->bbJumpKind == BBJ_ALWAYS) || (next->bbJumpKind == BBJ_COND))
- {
- unsigned int destNum = next->bbJumpDest->bbNum;
- if ((destNum >= top->bbNum) && (destNum <= bottom->bbNum) &&
- !loopBlocks.isMember(destNum))
- {
- // Reversing this branch out of block `next` could confuse this algorithm, so
- // don't.
- // We're checking for BBJ_ALWAYS and BBJ_COND only here -- we don't need to
- // check for BBJ_SWITCH because we'd never consider it a loop back-edge.
- break;
- }
- }
+ if (newMoveAfter == nullptr)
+ {
+ // Ran out of candidate insertion points, so just split up the fall-through.
+ return moveAfter;
+ }
- bool reversesBranch = false;
+ moveAfter = newMoveAfter;
+ }
- for (flowList* predIter = next->bbPreds; predIter != nullptr;
- predIter = predIter->flNext)
- {
- unsigned int predNum = predIter->flBlock->bbNum;
-
- if ((predNum >= top->bbNum) && (predNum <= bottom->bbNum) &&
- !loopBlocks.isMember(predNum))
- {
- reversesBranch = true;
- break;
- }
- }
+ return moveAfter;
+ }
- if (reversesBranch)
- {
- // Reversing this branch to block `next` could confuse this algorithm, so don't.
- break;
- }
+ //------------------------------------------------------------------------
+ // TryAdvanceInsertionPoint: Find the next legal insertion point after
+ // the given one, if one exists.
+ //
+ // Arguments:
+ // oldMoveAfter - Prior insertion point; find the next after this.
+ //
+ // Return Value:
+ // The next block after `oldMoveAfter` that is a legal insertion point
+ // (i.e. blocks being swept out of the loop can be moved immediately
+ // after it), if one exists, else nullptr.
+ //
+ BasicBlock* TryAdvanceInsertionPoint(BasicBlock* oldMoveAfter)
+ {
+ BasicBlock* newMoveAfter = oldMoveAfter->bbNext;
- if ((next->bbNum <= oldBlockMaxNum) &&
- BlockSetOps::IsMember(this, bottomBlocks, next->bbNum))
- {
- // This is the BOTTOM of another loop; don't move any blocks past it, to avoid
- // moving them out of that loop (we should have already done so when processing
- // that loop if it were legal).
- break;
- }
+ if (!BasicBlock::sameEHRegion(oldMoveAfter, newMoveAfter))
+ {
+ // Don't cross an EH region boundary.
+ return nullptr;
+ }
- // Advance moveAfter to avoid splitting up the fallthrough.
- moveAfter = next;
- }
- }
+ if ((newMoveAfter->bbJumpKind == BBJ_ALWAYS) || (newMoveAfter->bbJumpKind == BBJ_COND))
+ {
+ unsigned int destNum = newMoveAfter->bbJumpDest->bbNum;
+ if ((destNum >= top->bbNum) && (destNum <= bottom->bbNum) && !loopBlocks.IsMember(destNum))
+ {
+ // Reversing this branch out of block `newMoveAfter` could confuse this algorithm
+ // (in particular, the edge would still be numerically backwards but no longer be
+ // lexically backwards, so a lexical forward walk from TOP would not find BOTTOM),
+ // so don't do that.
+ // We're checking for BBJ_ALWAYS and BBJ_COND only here -- we don't need to
+ // check for BBJ_SWITCH because we'd never consider it a loop back-edge.
+ return nullptr;
+ }
+ }
- if (!BasicBlock::sameEHRegion(previous, nextLoopBlock) ||
- !BasicBlock::sameEHRegion(previous, moveAfter))
- {
- // EH regions would be ill-formed if we moved these blocks out.
- // See if we can consider them loop blocks without introducing
- // a side-entry.
- for (BasicBlock* testBlock = block; testBlock != nextLoopBlock;
- testBlock = testBlock->bbNext)
- {
- for (flowList* predIter = testBlock->bbPreds; predIter != nullptr;
- predIter = predIter->flNext)
- {
- BasicBlock* testPred = predIter->flBlock;
- unsigned int predPosNum = positionNum(testPred);
- unsigned int blockPosNum = positionNum(block);
- unsigned int lastNonLoopPosNum = positionNum(lastNonLoopBlock);
-
- if (loopBlocks.isMember(predPosNum) ||
- ((predPosNum >= blockPosNum) && (predPosNum <= lastNonLoopPosNum)))
- {
- // This pred is in the loop (or what will be the loop if we determine this
- // run of exit blocks doesn't include a side-entry).
-
- if (predPosNum < blockPosNum)
- {
- // We've already counted this block as an exit, so decrement the count.
- --exitCount;
- if (lastExit == testPred)
- {
- // Erase this now-bogus `lastExit` entry.
- lastExit = nullptr;
- INDEBUG(forgotExit = true);
- }
- }
- }
- else
- {
- // This pred is not in the loop, so this constitutes a side-entry.
- goto NO_LOOP;
- }
- }
+ // Similarly check to see if advancing to `newMoveAfter` would reverse the lexical order
+ // of an edge from the run of blocks being moved to `newMoveAfter` -- doing so would
+ // introduce a new lexical back-edge, which could (maybe?) confuse the loop search
+ // algorithm, and isn't desirable layout anyway.
+ for (flowList* predIter = newMoveAfter->bbPreds; predIter != nullptr; predIter = predIter->flNext)
+ {
+ unsigned int predNum = predIter->flBlock->bbNum;
- // Either we're going to abort the loop on a subsequent testBlock, or this
- // testBlock is part of the loop.
- loopBlocks.insert(testBlock->bbNum);
- }
+ if ((predNum >= top->bbNum) && (predNum <= bottom->bbNum) && !loopBlocks.IsMember(predNum))
+ {
+ // Don't make this forward edge a backwards edge.
+ return nullptr;
+ }
+ }
- // Now that we've marked these blocks as part of the loop, iterate without updating
- // `previous` so that we'll analyze them as part of the loop.
- continue;
- }
+ if (IsRecordedBottom(newMoveAfter))
+ {
+ // This is the BOTTOM of another loop; don't move any blocks past it, to avoid moving them
+ // out of that loop (we should have already done so when processing that loop if it were legal).
+ return nullptr;
+ }
- BasicBlock* moveBefore = moveAfter->bbNext;
+ // Advancing the insertion point is ok, except that we can't split up any CallFinally/BBJ_ALWAYS
+ // pair, so if we've got such a pair recurse to see if we can move past the whole thing.
+ return (newMoveAfter->isBBCallAlwaysPair() ? TryAdvanceInsertionPoint(newMoveAfter) : newMoveAfter);
+ }
- fgUnlinkRange(block, lastNonLoopBlock);
- fgMoveBlocksAfter(block, lastNonLoopBlock, moveAfter);
- ehUpdateLastBlocks(moveAfter, lastNonLoopBlock);
+ //------------------------------------------------------------------------
+ // isOuterBottom: Determine if the given block is the BOTTOM of a previously
+ // recorded loop.
+ //
+ // Arguments:
+ // block - Block to check for BOTTOM-ness.
+ //
+ // Return Value:
+ // true - The blocks was recorded as `bottom` of some earlier-processed loop.
+ // false - No loops yet recorded have this block as their `bottom`.
+ //
+ bool IsRecordedBottom(BasicBlock* block)
+ {
+ if (block->bbNum > oldBlockMaxNum)
+ {
+ // This is a new block, which can't be an outer bottom block because we only allow old blocks
+ // as BOTTOM.
+ return false;
+ }
+ return BlockSetOps::IsMember(comp, bottomBlocks, block->bbNum);
+ }
- // If lastNonLoopBlock fell through to nextLoopBlock, then it also would have been in the loop.
- assert(!lastNonLoopBlock->bbFallsThrough());
- if ((lastNonLoopBlock->bbJumpKind == BBJ_ALWAYS) &&
- (lastNonLoopBlock->bbJumpDest == moveBefore))
- {
- // We've created a goto-next, so change it to fall-through
- if (!fgOptimizeBranchToNext(lastNonLoopBlock, moveBefore, lastNonLoopBlock->bbPrev))
- {
- // If optimizing away the goto-next failed for some reason, mark it KEEP_BBJ_ALWAYS to
- // prevent assertions from complaining about it.
- lastNonLoopBlock->bbFlags |= BBF_KEEP_BBJ_ALWAYS;
- }
- }
+ //------------------------------------------------------------------------
+ // CanTreatAsLoopBlocks: If the given range of blocks can be treated as
+ // loop blocks, add them to loopBlockSet and return true. Otherwise,
+ // return false.
+ //
+ // Arguments:
+ // firstNonLoopBlock - First block in the run to be subsumed.
+ // lastNonLoopBlock - Last block in the run to be subsumed.
+ //
+ // Return Value:
+ // true - The blocks from `fistNonLoopBlock` to `lastNonLoopBlock` were
+ // successfully added to `loopBlocks`.
+ // false - Treating the blocks from `fistNonLoopBlock` to `lastNonLoopBlock`
+ // would not be legal (it would induce a side-entry).
+ //
+ // Notes:
+ // `loopBlocks` may be modified even if `false` is returned.
+ // `exitCount` and `lastExit` may be modified if this process identifies
+ // in-loop edges that were previously counted as exits.
+ //
+ bool CanTreatAsLoopBlocks(BasicBlock* firstNonLoopBlock, BasicBlock* lastNonLoopBlock)
+ {
+ BasicBlock* nextLoopBlock = lastNonLoopBlock->bbNext;
+ for (BasicBlock* testBlock = firstNonLoopBlock; testBlock != nextLoopBlock; testBlock = testBlock->bbNext)
+ {
+ for (flowList* predIter = testBlock->bbPreds; predIter != nullptr; predIter = predIter->flNext)
+ {
+ BasicBlock* testPred = predIter->flBlock;
+ unsigned int predPosNum = PositionNum(testPred);
+ unsigned int firstNonLoopPosNum = PositionNum(firstNonLoopBlock);
+ unsigned int lastNonLoopPosNum = PositionNum(lastNonLoopBlock);
- if (moveAfter->bbFallsThrough())
- {
- // We've just inserted blocks between moveAfter and moveBefore which it was supposed
- // to fall through to, so insert a jump to reconnect it to moveBefore.
- fgConnectFallThrough(moveAfter, moveBefore);
- }
- else if ((moveAfter->bbJumpKind == BBJ_ALWAYS) && (moveAfter->bbJumpDest == block))
+ if (loopBlocks.IsMember(predPosNum) ||
+ ((predPosNum >= firstNonLoopPosNum) && (predPosNum <= lastNonLoopPosNum)))
+ {
+ // This pred is in the loop (or what will be the loop if we determine this
+ // run of exit blocks doesn't include a side-entry).
+
+ if (predPosNum < firstNonLoopPosNum)
+ {
+ // We've already counted this block as an exit, so decrement the count.
+ --exitCount;
+ if (lastExit == testPred)
{
- // We've created a goto-next, so change it to fall-through
- if (!fgOptimizeBranchToNext(moveAfter, block, moveAfter->bbPrev))
- {
- // If optimizing away the goto-next failed for some reason, mark it KEEP_BBJ_ALWAYS to
- // prevent assertions from complaining about it.
- moveAfter->bbFlags |= BBF_KEEP_BBJ_ALWAYS;
- }
+ // Erase this now-bogus `lastExit` entry.
+ lastExit = nullptr;
+ INDEBUG(forgotExit = true);
}
+ }
+ }
+ else
+ {
+ // This pred is not in the loop, so this constitutes a side-entry.
+ return false;
+ }
+ }
- if (previous->bbFallsThrough())
- {
- // Need to reconnect the flow from `previous` to `block`.
+ // Either we're going to abort the loop on a subsequent testBlock, or this
+ // testBlock is part of the loop.
+ loopBlocks.Insert(testBlock->bbNum);
+ }
- if ((previous->bbJumpKind == BBJ_COND) && (previous->bbJumpDest == nextLoopBlock))
- {
- /* Reverse the jump condition */
- GenTree* test = previous->lastNode();
- noway_assert(test->OperIsConditionalJump());
+ // All blocks were ok to leave in the loop.
+ return true;
+ }
- if (test->OperGet() == GT_JTRUE)
- {
- GenTree* cond = gtReverseCond(test->gtOp.gtOp1);
- assert(cond ==
- test->gtOp.gtOp1); // Ensure `gtReverseCond` did not create a new node.
- test->gtOp.gtOp1 = cond;
- }
- else
- {
- gtReverseCond(test);
- }
+ //------------------------------------------------------------------------
+ // FixupFallThrough: Re-establish any broken control flow connectivity
+ // and eliminate any "goto-next"s that were created by changing the
+ // given block's lexical follower.
+ //
+ // Arguments:
+ // block - Block whose `bbNext` has changed.
+ // oldNext - Previous value of `block->bbNext`.
+ // newNext - New value of `block->bbNext`.
+ //
+ // Return Value:
+ // If a new block is created to reconnect flow, the new block is
+ // returned; otherwise, nullptr.
+ //
+ BasicBlock* FixupFallThrough(BasicBlock* block, BasicBlock* oldNext, BasicBlock* newNext)
+ {
+ // If we create a new block, that will be our return value.
+ BasicBlock* newBlock = nullptr;
- // Optimize the Conditional JUMP to go to the new target
- previous->bbJumpDest = block;
- }
- else
- {
- // Insert an unconditional jump to 'block' in place of the moved blocks.
- BasicBlock* newBlock = fgConnectFallThrough(previous, block);
+ if (block->bbFallsThrough())
+ {
+ // Need to reconnect the flow from `block` to `oldNext`.
- // The new block is in the loop and is a loop exit
- loopBlocks.insert(newBlock->bbNum);
- lastExit = newBlock;
- ++exitCount;
- }
- }
- else if ((previous->bbJumpKind == BBJ_ALWAYS) && (previous->bbJumpDest == nextLoopBlock))
- {
- // `previous` jumped exactly around the blocks being moved, so change it to fallthrough.
- if (!fgOptimizeBranchToNext(previous, nextLoopBlock, previous->bbPrev))
- {
- // If optimizing away the goto-next failed for some reason, mark it KEEP_BBJ_ALWAYS to
- // prevent assertions from complaining about it.
- previous->bbFlags |= BBF_KEEP_BBJ_ALWAYS;
- }
- }
+ if ((block->bbJumpKind == BBJ_COND) && (block->bbJumpDest == newNext))
+ {
+ /* Reverse the jump condition */
+ GenTree* test = block->lastNode();
+ noway_assert(test->OperIsConditionalJump());
- // Update moveAfter for the next insertion.
- moveAfter = lastNonLoopBlock;
+ if (test->OperGet() == GT_JTRUE)
+ {
+ GenTree* cond = comp->gtReverseCond(test->gtOp.gtOp1);
+ assert(cond == test->gtOp.gtOp1); // Ensure `gtReverseCond` did not create a new node.
+ test->gtOp.gtOp1 = cond;
+ }
+ else
+ {
+ comp->gtReverseCond(test);
+ }
- // Note that we've changed the flow graph, and continue without updating
- // `previous` so that we'll process nextLoopBlock.
- mod = true;
- continue;
- }
+ // Redirect the Conditional JUMP to go to `oldNext`
+ block->bbJumpDest = oldNext;
+ }
+ else
+ {
+ // Insert an unconditional jump to `oldNext` just after `block`.
+ newBlock = comp->fgConnectFallThrough(block, oldNext);
+ noway_assert((newBlock == nullptr) || loopBlocks.CanRepresent(newBlock->bbNum));
+ }
+ }
+ else if ((block->bbJumpKind == BBJ_ALWAYS) && (block->bbJumpDest == newNext))
+ {
+ // We've made `block`'s jump target its bbNext, so remove the jump.
+ if (!comp->fgOptimizeBranchToNext(block, newNext, block->bbPrev))
+ {
+ // If optimizing away the goto-next failed for some reason, mark it KEEP_BBJ_ALWAYS to
+ // prevent assertions from complaining about it.
+ block->bbFlags |= BBF_KEEP_BBJ_ALWAYS;
+ }
+ }
- // This block is a member of the loop. Check to see if it may exit the loop.
- BasicBlock* exitPoint;
+ // Make sure we don't leave around a goto-next unless it's marked KEEP_BBJ_ALWAYS.
+ assert((block->bbJumpKind != BBJ_COND) || (block->bbJumpKind != BBJ_ALWAYS) || (block->bbJumpDest != newNext) ||
+ ((block->bbFlags & BBF_KEEP_BBJ_ALWAYS) != 0));
+ return newBlock;
+ }
- switch (block->bbJumpKind)
- {
- case BBJ_COND:
- case BBJ_CALLFINALLY:
- case BBJ_ALWAYS:
- case BBJ_EHCATCHRET:
- assert(block->bbJumpDest);
- exitPoint = block->bbJumpDest;
-
- if (!loopBlocks.isMember(exitPoint->bbNum))
- {
- /* exit from a block other than BOTTOM */
- lastExit = block;
- exitCount++;
- }
- break;
+ //------------------------------------------------------------------------
+ // CheckForExit: Check if the given block has any successor edges that are
+ // loop exits, and update `lastExit` and `exitCount` if so.
+ //
+ // Arguments:
+ // block - Block whose successor edges are to be checked.
+ //
+ // Notes:
+ // If one block has multiple exiting successor edges, those are counted
+ // as multiple exits in `exitCount`.
+ //
+ void CheckForExit(BasicBlock* block)
+ {
+ BasicBlock* exitPoint;
- case BBJ_NONE:
- break;
+ switch (block->bbJumpKind)
+ {
+ case BBJ_COND:
+ case BBJ_CALLFINALLY:
+ case BBJ_ALWAYS:
+ case BBJ_EHCATCHRET:
+ assert(block->bbJumpDest);
+ exitPoint = block->bbJumpDest;
- case BBJ_EHFINALLYRET:
- case BBJ_EHFILTERRET:
- /* The "try" associated with this "finally" must be in the
- * same loop, so the finally block will return control inside the loop */
- break;
+ if (!loopBlocks.IsMember(exitPoint->bbNum))
+ {
+ /* exit from a block other than BOTTOM */
+ lastExit = block;
+ exitCount++;
+ }
+ break;
- case BBJ_THROW:
- case BBJ_RETURN:
- /* those are exits from the loop */
- lastExit = block;
- exitCount++;
- break;
+ case BBJ_NONE:
+ break;
- case BBJ_SWITCH:
+ case BBJ_EHFINALLYRET:
+ case BBJ_EHFILTERRET:
+ /* The "try" associated with this "finally" must be in the
+ * same loop, so the finally block will return control inside the loop */
+ break;
- unsigned jumpCnt;
- jumpCnt = block->bbJumpSwt->bbsCount;
- BasicBlock** jumpTab;
- jumpTab = block->bbJumpSwt->bbsDstTab;
+ case BBJ_THROW:
+ case BBJ_RETURN:
+ /* those are exits from the loop */
+ lastExit = block;
+ exitCount++;
+ break;
- do
- {
- noway_assert(*jumpTab);
- exitPoint = *jumpTab;
+ case BBJ_SWITCH:
- if (!loopBlocks.isMember(exitPoint->bbNum))
- {
- lastExit = block;
- exitCount++;
- }
- } while (++jumpTab, --jumpCnt);
- break;
+ unsigned jumpCnt;
+ jumpCnt = block->bbJumpSwt->bbsCount;
+ BasicBlock** jumpTab;
+ jumpTab = block->bbJumpSwt->bbsDstTab;
- default:
- noway_assert(!"Unexpected bbJumpKind");
- break;
- }
+ do
+ {
+ noway_assert(*jumpTab);
+ exitPoint = *jumpTab;
- if (block->bbFallsThrough() && !loopBlocks.isMember(block->bbNext->bbNum))
+ if (!loopBlocks.IsMember(exitPoint->bbNum))
{
- // Found a fall-through exit.
lastExit = block;
exitCount++;
}
+ } while (++jumpTab, --jumpCnt);
+ break;
- // Done processing this block; move on to the next.
- previous = block;
- }
+ default:
+ noway_assert(!"Unexpected bbJumpKind");
+ break;
+ }
- if ((exitCount == 1) && (lastExit == nullptr))
- {
- // If we happen to have a loop with two exits, one of which goes to an
- // infinite loop that's lexically nested inside it, where the inner loop
- // can't be moved out, we can end up in this situation (because
- // canTreatAsLoopBlocks will have decremented the count expecting to find
- // another exit later). Bump the exit count to 2, since downstream code
- // will not be prepared for null lastExit with exitCount of 1.
- assert(forgotExit);
- exitCount = 2;
- }
+ if (block->bbFallsThrough() && !loopBlocks.IsMember(block->bbNext->bbNum))
+ {
+ // Found a fall-through exit.
+ lastExit = block;
+ exitCount++;
+ }
+ }
+};
+}
- /* At this point we have a compact loop - record it in the loop table
- * If we found only one exit, record it in the table too
- * (otherwise an exit = nullptr in the loop table means multiple exits) */
+/*****************************************************************************
+ * Find the natural loops, using dominators. Note that the test for
+ * a loop is slightly different from the standard one, because we have
+ * not done a depth first reordering of the basic blocks.
+ */
- assert(pred);
- BasicBlock* onlyExit = (exitCount == 1 ? lastExit : nullptr);
- if (optRecordLoop(head, first, top, entry, bottom, onlyExit, exitCount))
- {
- // Record the BOTTOM block for future reference.
- assert(bottom->bbNum <= oldBlockMaxNum);
- BlockSetOps::AddElemD(this, bottomBlocks, bottom->bbNum);
- }
+void Compiler::optFindNaturalLoops()
+{
+#ifdef DEBUG
+ if (verbose)
+ {
+ printf("*************** In optFindNaturalLoops()\n");
+ }
+#endif // DEBUG
+
+ noway_assert(fgDomsComputed);
+ assert(fgHasLoops);
+
+#if COUNT_LOOPS
+ hasMethodLoops = false;
+ loopsThisMethod = 0;
+ loopOverflowThisMethod = false;
+#endif
+
+ LoopSearch search(this);
+
+ for (BasicBlock* head = fgFirstBB; head->bbNext; head = head->bbNext)
+ {
+ BasicBlock* top = head->bbNext;
+
+ // Blocks that are rarely run have a zero bbWeight and should
+ // never be optimized here
+
+ if (top->bbWeight == BB_ZERO_WEIGHT)
+ {
+ continue;
+ }
+
+ for (flowList* pred = top->bbPreds; pred; pred = pred->flNext)
+ {
+ if (search.FindLoop(head, top, pred->flBlock))
+ {
+ // Found a loop; record it and see if we've hit the limit.
+ bool recordedLoop = search.RecordLoop();
+
+ (void)recordedLoop; // avoid unusued variable warnings in COUNT_LOOPS and !DEBUG
#if COUNT_LOOPS
if (!hasMethodLoops)
/* keep track of the number of exits */
loopExitCountTable.record(static_cast<unsigned>(exitCount));
#else // COUNT_LOOPS
+ assert(recordedLoop);
if (optLoopCount == MAX_LOOP_NUM)
{
// We won't be able to record any more loops, so stop looking.
goto NO_MORE_LOOPS;
}
#endif // COUNT_LOOPS
- }
- /* current predecessor not good for a loop - continue with another one, if any */
- NO_LOOP:;
+ // Continue searching preds of `top` to see if any other are
+ // back-edges (this can happen for nested loops). The iteration
+ // is safe because the compaction we do only modifies predecessor
+ // lists of blocks that gain or lose fall-through from their
+ // `bbPrev`, but since the motion is from within the loop to below
+ // it, we know we're not altering the relationship between `top`
+ // and its `bbPrev`.
+ }
}
}
NO_MORE_LOOPS:
}
}
+ bool mod = search.ChangedFlowGraph();
+
// Make sure that loops are canonical: that every loop has a unique "top", by creating an empty "nop"
// one, if necessary, for loops containing others that share a "top."
for (unsigned char loopInd = 0; loopInd < optLoopCount; loopInd++)
projects / platform / upstream / dotnet / runtime.git / commitdiff