1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
5 /*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
6 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
11 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
13 Loop cloning optimizations comprise of the following steps:
14 - Loop detection logic which is existing logic in the JIT that records
15 loop information with loop flags.
16 - The next step is to identify loop optimization candidates. This is done
17 by optObtainLoopCloningOpts. The loop context variable is updated with
18 all the necessary information (for ex: block, stmt, tree information)
19 to do the optimization later.
20 a) This involves checking if the loop is well-formed with respect to
21 the optimization being performed.
22 b) In array bounds check case, reconstructing the morphed GT_INDEX
23 nodes back to their array representation.
24 i) The array index is stored in the "context" variable with
25 additional block, tree, stmt info.
26 - Once the optimization candidates are identified, we derive cloning conditions
27 For ex: to clone a simple "for (i=0; i<n; ++i) { a[i] }" loop, we need the
29 (a != null) && ((n >= 0) & (n <= a.length) & (stride > 0))
30 a) Note the short circuit AND for (a != null). These are called block
31 conditions or deref-conditions since these conditions need to be in their
32 own blocks to be able to short-circuit.
33 i) For a doubly nested loop on i, j, we would then have
35 (a != null) && (i < a.len) && (a[i] != null) && (j < a[i].len)
36 all short-circuiting creating blocks.
39 All conditions are checked before we enter the fast path. So fast
40 path gets as fast as it can be.
46 Therefore we will not clone if we exceed creating 4 blocks.
48 b) The other conditions called cloning conditions are transformed into LC_Condition
49 structs which are then optimized.
50 i) Optimization of conditions involves removing redundant condition checks.
51 ii) If some conditions evaluate to true statically, then they are removed.
52 iii) If any condition evaluates to false statically, then loop cloning is
53 aborted for that loop.
54 - Then the block splitting occurs and loop cloning conditions is transformed into
55 GenTree and added to the loop cloning choice block.
58 - Loop detection should have completed and the loop table should be
59 populated with the loop dscs.
60 - The loops that will be considered are the ones with the LPFLG_ITER
64 - For array based optimizations the loop choice condition is checked
65 before the loop body. This implies that the loop initializer statement
66 has not executed at the time of the check. So any loop cloning condition
67 involving the initial value of the loop counter cannot be condition checked
68 as it hasn't been assigned yet at the time of condition checking. Therefore
69 the initial value has to be statically known. This can be fixed with further
73 - The assumption is that the optimization candidates collected during the
74 identification phase will be the ones that will be optimized. In other words,
75 the loop that is present originally will be the fast path. Explicitly, the cloned
76 path will be the slow path and will be unoptimized. This allows us to
77 collect additional information at the same time of identifying the optimization
78 candidates. This later helps us to perform the optimizations during actual cloning.
79 - All loop cloning choice conditions will automatically be "AND"-ed. These are
80 bitwise AND operations.
81 - Perform short circuit AND for (array != null) side effect check
82 before hoisting (limit <= a.length) check.
83 For ex: to clone a simple "for (i=0; i<n; ++i) { a[i] }" loop, we need the
85 (a != null) && ((n >= 0) & (n <= a.length) & (stride > 0))
94 * Represents an array access and associated bounds checks.
95 * Array access is required have the array and indices in local variables.
96 * This struct is constructed using a GT_INDEX node that is broken into
102 unsigned arrLcl; // The array base local num
103 ExpandArrayStack<unsigned> indLcls; // The indices local nums
104 ExpandArrayStack<GenTree*> bndsChks; // The bounds checks nodes along each dimension.
105 unsigned rank; // Rank of the array
106 BasicBlock* useBlock; // Block where the [] occurs
108 ArrIndex(CompAllocator* alloc) : arrLcl(BAD_VAR_NUM), indLcls(alloc), bndsChks(alloc), rank(0), useBlock(nullptr)
113 void Print(unsigned dim = -1)
115 printf("V%02d", arrLcl);
116 for (unsigned i = 0; i < ((dim == -1) ? rank : dim); ++i)
118 printf("[V%02d]", indLcls.GetRef(i));
124 // Forward declarations
125 #define LC_OPT(en) struct en##OptInfo;
126 #include "loopcloningopts.h"
130 * LcOptInfo represents the optimization information for loop cloning,
131 * other classes are supposed to derive from this base class.
134 * LcMdArrayOptInfo is multi-dimensional array optimization for which the
135 * loop can be cloned.
136 * LcArrIndexOptInfo is a jagged array optimization for which the loop
139 * So LcOptInfo represents any type of optimization opportunity that
140 * occurs in a loop and the metadata for the optimization is stored in
148 #define LC_OPT(en) en,
149 #include "loopcloningopts.h"
154 LcOptInfo(void* optInfo, OptType optType) : optInfo(optInfo), optType(optType)
164 en##OptInfo* As##en##OptInfo() \
166 assert(optType == en); \
167 return reinterpret_cast<en##OptInfo*>(this); \
169 #include "loopcloningopts.h"
174 * Optimization info for a multi-dimensional array.
176 struct LcMdArrayOptInfo : public LcOptInfo
178 GenTreeArrElem* arrElem; // "arrElem" node of an MD array.
179 unsigned dim; // "dim" represents upto what level of the rank this optimization applies to.
180 // For example, a[i,j,k] could be the MD array "arrElem" but if "dim" is 2,
181 // then this node is treated as though it were a[i,j]
182 ArrIndex* index; // "index" cached computation in the form of an ArrIndex representation.
184 LcMdArrayOptInfo(GenTreeArrElem* arrElem, unsigned dim)
185 : LcOptInfo(this, LcMdArray), arrElem(arrElem), dim(dim), index(nullptr)
189 ArrIndex* GetArrIndexForDim(CompAllocator* alloc)
191 if (index == nullptr)
193 index = new (alloc) ArrIndex(alloc);
194 index->rank = arrElem->gtArrRank;
195 for (unsigned i = 0; i < dim; ++i)
197 index->indLcls.Push(arrElem->gtArrInds[i]->gtLclVarCommon.gtLclNum);
199 index->arrLcl = arrElem->gtArrObj->gtLclVarCommon.gtLclNum;
207 * Optimization info for a jagged array.
209 struct LcJaggedArrayOptInfo : public LcOptInfo
211 unsigned dim; // "dim" represents upto what level of the rank this optimization applies to.
212 // For example, a[i][j][k] could be the jagged array but if "dim" is 2,
213 // then this node is treated as though it were a[i][j]
214 ArrIndex arrIndex; // ArrIndex representation of the array.
215 GenTreePtr stmt; // "stmt" where the optimization opportunity occurs.
217 LcJaggedArrayOptInfo(ArrIndex& arrIndex, unsigned dim, GenTreePtr stmt)
218 : LcOptInfo(this, LcJaggedArray), dim(dim), arrIndex(arrIndex), stmt(stmt)
225 * Symbolic representation of a.length, or a[i][j].length or a[i,j].length and so on.
226 * OperType decides whether "arrLength" is invoked on the array or if it is just an array.
243 ArrType type; // The type of the array on which to invoke length operator.
244 ArrIndex* arrIndex; // ArrIndex representation of this array.
251 arrIndex->Print(dim);
259 int dim; // "dim" = which index to invoke arrLen on, if -1 invoke on the whole array
260 // Example 1: a[0][1][2] and dim = 2 implies a[0][1].length
261 // Example 2: a[0][1][2] and dim = -1 implies a[0][1][2].length
262 LC_Array() : type(Invalid), dim(-1)
265 LC_Array(ArrType type, ArrIndex* arrIndex, int dim, OperType oper)
266 : type(type), arrIndex(arrIndex), oper(oper), dim(dim)
270 LC_Array(ArrType type, ArrIndex* arrIndex, OperType oper) : type(type), arrIndex(arrIndex), oper(oper), dim(-1)
275 bool operator==(const LC_Array& that) const
277 assert(type != Invalid && that.type != Invalid);
279 // Types match and the array base matches.
280 if (type != that.type || arrIndex->arrLcl != that.arrIndex->arrLcl || oper != that.oper)
285 // If the dim ranks are not matching, quit.
286 int rank1 = GetDimRank();
287 int rank2 = that.GetDimRank();
293 // Check for the indices.
294 for (int i = 0; i < rank1; ++i)
296 if (arrIndex->indLcls[i] != that.arrIndex->indLcls[i])
304 // The max dim on which length is invoked.
305 int GetDimRank() const
307 return (dim < 0) ? (int)arrIndex->rank : dim;
310 // Get a tree representation for this symbolic a.length
311 GenTreePtr ToGenTree(Compiler* comp);
316 * Symbolic representation of either a constant like 1, 2 or a variable V02, V03 etc. or an "LC_Array" or the null
330 INT64 constant; // The constant value if this node is of type "Const", or the lcl num if "Var"
331 LC_Array arrLen; // The LC_Array if the type is "ArrLen"
332 IdentType type; // The type of this object
335 bool operator==(const LC_Ident& that) const
341 return (type == that.type) && constant == that.constant;
343 return (type == that.type) && (arrLen == that.arrLen);
345 return (type == that.type);
347 assert(!"Unknown LC_Ident type");
358 printf("%I64d", constant);
361 printf("V%02d", constant);
376 LC_Ident() : type(Invalid)
379 LC_Ident(INT64 constant, IdentType type) : constant(constant), type(type)
382 explicit LC_Ident(IdentType type) : type(type)
385 explicit LC_Ident(const LC_Array& arrLen) : arrLen(arrLen), type(ArrLen)
389 // Convert this symbolic representation into a tree node.
390 GenTreePtr ToGenTree(Compiler* comp);
395 * Symbolic representation of an expr that involves an "LC_Ident" or an "LC_Ident - constant"
411 bool operator==(const LC_Expr& that) const
413 assert(type != Invalid && that.type != Invalid);
415 // If the types don't match quit.
416 if (type != that.type)
421 // If the type involves arithmetic, the constant should match.
422 if (type == IdentPlusConst && constant != that.constant)
427 // Check if the ident match.
428 return (ident == that.ident);
434 if (type == IdentPlusConst)
436 printf("(%I64d - ", constant);
447 LC_Expr() : type(Invalid)
450 explicit LC_Expr(const LC_Ident& ident) : ident(ident), type(Ident)
453 LC_Expr(const LC_Ident& ident, INT64 constant) : ident(ident), constant(constant), type(IdentPlusConst)
457 // Convert LC_Expr into a tree node.
458 GenTreePtr ToGenTree(Compiler* comp);
463 * Symbolic representation of a conditional operation involving two "LC_Expr":
464 * LC_Expr < LC_Expr, for example: i > 0, i < a.length
476 printf(" %s ", GenTree::OpName(oper));
481 // Check if the condition evaluates statically to true or false, i < i => false, a.length > 0 => true
482 // The result is put in "pResult" parameter and is valid if the method returns "true". Otherwise, the
483 // condition could not be evaluated.
484 bool Evaluates(bool* pResult);
486 // Check if two conditions can be combined to yield one condition.
487 bool Combines(const LC_Condition& cond, LC_Condition* newCond);
492 LC_Condition(genTreeOps oper, const LC_Expr& op1, const LC_Expr& op2) : op1(op1), op2(op2), oper(oper)
496 // Convert this conditional operation into a GenTree.
497 GenTreePtr ToGenTree(Compiler* comp);
501 * A deref tree of an array expression.
502 * a[i][j][k], b[i] and a[i][y][k] are the occurrences in the loop, then, the tree would be:
519 const LC_Array array;
520 ExpandArrayStack<LC_Deref*>* children;
524 LC_Deref(const LC_Array& array, unsigned level) : array(array), children(nullptr), level(level)
528 LC_Deref* Find(unsigned lcl);
533 void EnsureChildren(CompAllocator* alloc);
534 static LC_Deref* Find(ExpandArrayStack<LC_Deref*>* children, unsigned lcl);
536 void DeriveLevelConditions(ExpandArrayStack<ExpandArrayStack<LC_Condition>*>* len);
538 void Print(unsigned indent = 0)
540 unsigned tab = 4 * indent;
541 printf("%*s%d,%d => {", tab, "", Lcl(), level);
542 if (children != nullptr)
544 for (unsigned i = 0; i < children->Size(); ++i)
552 (*children)[i]->Print(indent + 1);
554 (*((ExpandArray<LC_Deref*>*)children))[i]->Print(indent + 1);
558 printf("\n%*s}", tab, "");
565 * The "context" represents data that is used for making loop-cloning decisions.
566 * - The data is the collection of optimization opportunities
567 * - and the conditions (LC_Condition) that decide between the fast
568 * path or the slow path.
570 * BNF for LC_Condition:
571 * LC_Condition : LC_Expr genTreeOps LC_Expr
572 * LC_Expr : LC_Ident | LC_Ident + Constant
573 * LC_Ident : Constant | Var | LC_Array
575 * genTreeOps : GT_GE | GT_LE | GT_GT | GT_LT
578 struct LoopCloneContext
580 CompAllocator* alloc; // The allocator
581 ExpandArrayStack<LcOptInfo*>** optInfo; // The array of optimization opportunities found in each loop. (loop x
582 // optimization-opportunities)
583 ExpandArrayStack<LC_Condition>** conditions; // The array of conditions that influence which path to take for each
584 // loop. (loop x cloning-conditions)
585 ExpandArrayStack<LC_Array>** derefs; // The array of dereference conditions found in each loop. (loop x
587 ExpandArrayStack<ExpandArrayStack<LC_Condition>*>** blockConditions; // The array of block levels of conditions for
588 // each loop. (loop x level x conditions)
590 LoopCloneContext(unsigned loopCount, CompAllocator* alloc) : alloc(alloc)
592 optInfo = new (alloc) ExpandArrayStack<LcOptInfo*>*[loopCount];
593 conditions = new (alloc) ExpandArrayStack<LC_Condition>*[loopCount];
594 derefs = new (alloc) ExpandArrayStack<LC_Array>*[loopCount];
595 blockConditions = new (alloc) ExpandArrayStack<ExpandArrayStack<LC_Condition>*>*[loopCount];
596 for (unsigned i = 0; i < loopCount; ++i)
598 optInfo[i] = nullptr;
599 conditions[i] = nullptr;
601 blockConditions[i] = nullptr;
605 // Evaluate conditions into a JTRUE stmt and put it in the block. Reverse condition if 'reverse' is true.
606 void CondToStmtInBlock(Compiler* comp, ExpandArrayStack<LC_Condition>& conds, BasicBlock* block, bool reverse);
608 // Get all the optimization information for loop "loopNum"; This information is held in "optInfo" array.
609 // If NULL this allocates the optInfo[loopNum] array for "loopNum"
610 ExpandArrayStack<LcOptInfo*>* EnsureLoopOptInfo(unsigned loopNum);
612 // Get all the optimization information for loop "loopNum"; This information is held in "optInfo" array.
613 // If NULL this does not allocate the optInfo[loopNum] array for "loopNum"
614 ExpandArrayStack<LcOptInfo*>* GetLoopOptInfo(unsigned loopNum);
616 // Cancel all optimizations for loop "loopNum" by clearing out the "conditions" member if non-null
617 // and setting the optInfo to "null.", If "null", then the user of this class is not supposed to
619 void CancelLoopOptInfo(unsigned loopNum);
621 // Get the conditions that decide which loop to take for "loopNum." If NULL allocate an empty array.
622 ExpandArrayStack<LC_Condition>* EnsureConditions(unsigned loopNum);
624 // Get the conditions for loop. No allocation is performed.
625 ExpandArrayStack<LC_Condition>* GetConditions(unsigned loopNum);
627 // Ensure that the "deref" conditions array is allocated.
628 ExpandArrayStack<LC_Array>* EnsureDerefs(unsigned loopNum);
630 // Get block conditions for each loop, no allocation is performed.
631 ExpandArrayStack<ExpandArrayStack<LC_Condition>*>* GetBlockConditions(unsigned loopNum);
633 // Ensure that the block condition is present, if not allocate space.
634 ExpandArrayStack<ExpandArrayStack<LC_Condition>*>* EnsureBlockConditions(unsigned loopNum, unsigned totalBlocks);
636 // Print the block conditions for the loop.
637 void PrintBlockConditions(unsigned loopNum);
639 // Does the loop have block conditions?
640 bool HasBlockConditions(unsigned loopNum);
642 // Evaluate the conditions for "loopNum" and indicate if they are either all true or any of them are false.
643 // "pAllTrue" implies all the conditions are statically known to be true.
644 // "pAnyFalse" implies at least one condition is statically known to be false.
645 // If neither of them are true, then some conditions' evaluations are statically unknown.
647 // If all conditions yield true, then the caller doesn't need to clone the loop, but it can perform
648 // fast path optimizations.
649 // If any condition yields false, then the caller needs to abort cloning the loop (neither clone nor
650 // fast path optimizations.)
652 // Assumes the conditions involve an AND join operator.
653 void EvaluateConditions(unsigned loopNum, bool* pAllTrue, bool* pAnyFalse DEBUGARG(bool verbose));
656 void OptimizeConditions(ExpandArrayStack<LC_Condition>& conds);
659 // Optimize conditions to remove redundant conditions.
660 void OptimizeConditions(unsigned loopNum DEBUGARG(bool verbose));
662 void OptimizeBlockConditions(unsigned loopNum DEBUGARG(bool verbose));
665 void PrintConditions(unsigned loopNum);