1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
5 /*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
6 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
11 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
19 #include "allocacheck.h" // for alloca
21 /*****************************************************************************/
23 // Split a tree at the given point
24 // -- Introduces a new temporary variable
25 // -- evaluates *splitPoint into the new temp, in a new statement inserted before 'stmt'
26 // -- substitutes the temporary for '*splitPoint' in 'stmt'
27 // '*splitpoint' must be a node in 'stmt', which is within 'blk', and 'splitpoint' is a pointer
28 // to the link to that node, contained in its parent node.
29 GenTree* Compiler::fgMorphSplitTree(GenTree** splitPoint, // where to split
30 GenTree* stmt, // top level statement housing this tree
31 BasicBlock* blk) // block we are in
36 if ((*splitPoint)->OperIsAssignment())
38 // it's already being assigned so don't introduce a new one
39 newTree = *splitPoint;
40 temp = (*splitPoint)->gtGetOp1();
44 unsigned lclNum = lvaGrabTemp(true DEBUGARG("split tree"));
45 newTree = gtNewTempAssign(lclNum, *splitPoint);
46 temp = gtNewLclvNode(lclNum, (*splitPoint)->TypeGet());
49 GenTreePtr asg = gtNewStmt(newTree);
53 fgInsertStmtBefore(blk, stmt, asg);
58 // state carried over the tree walk, to be used in making
59 // a splitting decision.
62 // number of calls seen
65 // callback to determine if we should split here
66 Compiler::fgSplitPredicate* pred;
68 // root stmt of tree being processed
74 // Returns true if we should split the tree above this node.
75 // For ARM FP, handling multiple calls per tree via a local and
76 // greedy register allocator could result in a lot of shuffling.
77 // So let the global register allocator handle these cases.
78 bool shouldSplitARM(GenTree* tree, GenTree* parent, Compiler::fgWalkData* data)
81 && varTypeIsFloating(tree)
83 && !parent->OperIsAssignment())
85 // increment call count
86 SplitData* tmpState = (SplitData*) data->pCallbackData;
89 return tmpState->count > 1;
96 #endif // _TARGET_ARM_
98 // Callback for the tree walker, called once per node.
99 // Determines if we want to split, performs the split, and then processes the rest of the tree
100 Compiler::fgWalkResult Compiler::fgSplitHelper(GenTree** ppTree, fgWalkData* data)
102 GenTree* tree = *ppTree;
103 Compiler* comp = data->compiler;
105 SplitData* tmpState = (SplitData*) data->pCallbackData;
107 fgSplitPredicate* pred = tmpState->pred;
109 if (pred(tree, data->parent, data)) // does this look like somewhere we want to split?
111 //printf("tmpstate2 = %d %p r:%p tmp:%p tree:%p\n", tmpState->count, tmpState->pred, tmpState->root, tmpState, tree);
112 GenTree* result = comp->fgMorphSplitTree(ppTree, tmpState->root, comp->compCurBB);
114 GenTree* oldStatement = comp->compCurStmt;
115 comp->compCurStmt = result;
117 // because we are doing this in pre-order we also have to process
118 // the subtree that we have just split off
119 comp->fgSplitProcessOneTree(result, pred);
122 comp->compCurStmt = oldStatement;
124 return Compiler::WALK_SKIP_SUBTREES;
126 //else printf("tmpstate3 = %d %p r:%p tmp:%p tree:%p\n", tmpState->count, tmpState->pred, tmpState->root, tmpState, tree);
128 return Compiler::WALK_CONTINUE;
131 void Compiler::fgSplitProcessOneTree(GenTree* tree, fgSplitPredicate pred)
133 SplitData tmpState = {0};
134 tmpState.pred = pred;
135 tmpState.root = tree;
137 fgWalkTreePre(&(tree->gtStmt.gtStmtExpr),
142 // Split expression trees at points which in the case of ARM this is done.
143 void Compiler::fgSplitMethodTrees(void)
147 #else // _TARGET_ARM_
148 for (BasicBlock* block = fgFirstBB; block; block = block->bbNext)
151 for (GenTree* tree = block->bbTreeList; tree; tree = tree->gtNext)
153 assert(tree != tree->gtNext);
154 fgSplitProcessOneTree(tree, shouldSplitARM);
157 #endif // _TARGET_ARM_
161 // Convert the given node into a call to the specified helper passing
162 // the given argument list.
163 // Tries to fold constants and also adds an edge for overflow exception
164 // returns the morphed tree
165 GenTreePtr Compiler::fgMorphCastIntoHelper(GenTreePtr tree,
171 /* If the operand is a constant, we'll try to fold it */
172 if (oper->OperIsConst())
174 GenTreePtr oldTree = tree;
176 tree = gtFoldExprConst(tree); // This may not fold the constant (NaN ...)
179 return fgMorphTree(tree);
180 else if (tree->OperKind() & GTK_CONST)
181 return fgMorphConst(tree);
183 // assert that oper is unchanged and that it is still a GT_CAST node
184 noway_assert(tree->gtCast.CastOp() == oper);
185 noway_assert(tree->gtOper == GT_CAST);
187 result = fgMorphIntoHelperCall(tree, helper, gtNewArgList(oper));
188 assert(result == tree);
193 /*****************************************************************************
195 * Convert the given node into a call to the specified helper passing
196 * the given argument list.
199 GenTreePtr Compiler::fgMorphIntoHelperCall(GenTreePtr tree,
201 GenTreeArgList* args)
203 tree->ChangeOper(GT_CALL);
205 tree->gtFlags |= GTF_CALL;
206 tree->gtCall.gtCallType = CT_HELPER;
207 tree->gtCall.gtCallMethHnd = eeFindHelper(helper);
208 tree->gtCall.gtCallArgs = args;
209 tree->gtCall.gtCallObjp = NULL;
210 tree->gtCall.gtCallLateArgs = NULL;
211 tree->gtCall.fgArgInfo = NULL;
212 tree->gtCall.gtRetClsHnd = NULL;
213 tree->gtCall.gtCallRegUsedMask = RBM_NONE;
214 tree->gtCall.gtCallMoreFlags = 0;
215 tree->gtCall.gtInlineCandidateInfo = NULL;
216 tree->gtCall.gtControlExpr = NULL;
218 #ifdef FEATURE_READYTORUN_COMPILER
219 tree->gtCall.gtEntryPoint.addr = nullptr;
222 /* Perform the morphing */
224 tree = fgMorphArgs(tree->AsCall());
229 /*****************************************************************************
230 * This node should not be referenced by anyone now. Set its values to garbage
231 * to catch extra references
235 void DEBUG_DESTROY_NODE(GenTreePtr tree)
238 // printf("DEBUG_DESTROY_NODE for [0x%08x]\n", tree);
240 // Save gtOper in case we want to find out what this node was
241 tree->gtOperSave = tree->gtOper;
243 tree->gtType = TYP_UNDEF;
244 tree->gtFlags |= 0xFFFFFFFF & ~GTF_NODE_MASK;
245 if (tree->OperIsSimple())
248 tree->gtOp.gtOp2 = NULL;
250 // Must do this last, because the "gtOp" check above will fail otherwise.
251 // Don't call SetOper, because GT_COUNT is not a valid value
252 tree->gtOper = GT_COUNT;
257 /*****************************************************************************
259 * Determine if a relop must be morphed to a qmark to manifest a boolean value.
260 * This is done when code generation can't create straight-line code to do it.
262 bool Compiler::fgMorphRelopToQmark(GenTreePtr tree)
264 #ifndef LEGACY_BACKEND
266 #else // LEGACY_BACKEND
267 return (genActualType(tree->TypeGet()) == TYP_LONG) ||
268 varTypeIsFloating(tree->TypeGet());
269 #endif // LEGACY_BACKEND
273 /*****************************************************************************
275 * Morph a cast node (we perform some very simple transformations here).
279 #pragma warning(push)
280 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
282 GenTreePtr Compiler::fgMorphCast(GenTreePtr tree)
284 noway_assert(tree->gtOper == GT_CAST);
285 noway_assert(genTypeSize(TYP_I_IMPL) == sizeof(void*));
287 /* The first sub-operand is the thing being cast */
289 GenTreePtr oper = tree->gtCast.CastOp();
290 var_types srcType = genActualType(oper->TypeGet());
293 var_types dstType = tree->CastToType();
294 unsigned dstSize = genTypeSize(dstType);
296 // See if the cast has to be done in two steps. R -> I
297 if (varTypeIsFloating(srcType) && varTypeIsIntegral(dstType))
299 // Only x86 must go through TYP_DOUBLE to get to all
300 // integral types everybody else can get straight there
301 // except for when using helpers
302 if (srcType == TYP_FLOAT
303 #if !FEATURE_STACK_FP_X87
305 #if defined(_TARGET_ARM64_)
306 // Amd64: src = float, dst is overflow conversion.
307 // This goes through helper and hence src needs to be converted to double.
308 && tree->gtOverflow()
309 #elif defined(_TARGET_AMD64_)
310 // Amd64: src = float, dst = uint64 or overflow conversion.
311 // This goes through helper and hence src needs to be converted to double.
312 && (tree->gtOverflow() || (dstType == TYP_ULONG))
313 #elif defined(_TARGET_ARM_)
314 // Arm: src = float, dst = int64/uint64 or overflow conversion.
315 && (tree->gtOverflow() || varTypeIsLong(dstType))
318 #endif // FEATURE_STACK_FP_X87
321 oper = gtNewCastNode(TYP_DOUBLE, oper, TYP_DOUBLE);
324 // do we need to do it in two steps R -> I, '-> smallType
325 #if defined(_TARGET_ARM64_) || defined(_TARGET_AMD64_)
326 if (dstSize < genTypeSize(TYP_INT))
328 oper = gtNewCastNodeL(TYP_INT, oper, TYP_INT);
329 oper->gtFlags |= (tree->gtFlags & (GTF_UNSIGNED|GTF_OVERFLOW|GTF_EXCEPT));
330 tree->gtFlags &= ~GTF_UNSIGNED;
333 if (dstSize < sizeof(void*))
335 oper = gtNewCastNodeL(TYP_I_IMPL, oper, TYP_I_IMPL);
336 oper->gtFlags |= (tree->gtFlags & (GTF_OVERFLOW|GTF_EXCEPT));
341 /* Note that if we need to use a helper call then we can not morph oper */
342 if (!tree->gtOverflow())
344 #ifdef _TARGET_ARM64_ // On ARM64 All non-overflow checking conversions can be optimized
350 #ifdef _TARGET_X86_ // there is no rounding convert to integer instruction on ARM or x64 so skip this
351 if ((oper->gtOper == GT_INTRINSIC) &&
352 (oper->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Round))
354 /* optimization: conv.i4(round.d(d)) -> round.i(d) */
355 oper->gtType = dstType;
356 return fgMorphTree(oper);
358 // if SSE2 is not enabled, we need the helper
359 else if (!opts.compCanUseSSE2)
361 return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2INT, oper);
364 #endif // _TARGET_X86_
368 #if defined(_TARGET_ARM_) || defined(_TARGET_AMD64_)
369 case TYP_UINT: goto OPTIMIZECAST;
370 #else // _TARGET_ARM_
371 case TYP_UINT: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2UINT, oper);
372 #endif // _TARGET_ARM_
374 #ifdef _TARGET_AMD64_
375 // SSE2 has instructions to convert a float/double directly to a long
376 case TYP_LONG: goto OPTIMIZECAST;
378 case TYP_LONG: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2LNG, oper);
379 #endif //_TARGET_AMD64_
380 case TYP_ULONG: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2ULNG, oper);
383 #endif // _TARGET_ARM64_
389 case TYP_INT: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2INT_OVF, oper);
390 case TYP_UINT: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2UINT_OVF, oper);
391 case TYP_LONG: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2LNG_OVF, oper);
392 case TYP_ULONG: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2ULNG_OVF, oper);
396 noway_assert(!"Unexpected dstType");
399 #ifndef _TARGET_64BIT_
400 // The code generation phase (for x86 & ARM32) does not handle casts
401 // directly from [u]long to anything other than [u]int. Insert an
402 // intermediate cast to native int.
403 else if (varTypeIsLong(srcType) && varTypeIsSmall(dstType))
405 oper = gtNewCastNode(TYP_I_IMPL, oper, TYP_I_IMPL);
406 oper->gtFlags |= (tree->gtFlags & (GTF_OVERFLOW|GTF_EXCEPT|GTF_UNSIGNED));
407 tree->gtFlags &= ~GTF_UNSIGNED;
409 #endif //!_TARGET_64BIT_
412 else if ((dstType == TYP_FLOAT) && (srcType == TYP_DOUBLE) && (oper->gtOper == GT_CAST) && !varTypeIsLong(oper->gtCast.CastOp()))
414 // optimization: conv.r4(conv.r8(?)) -> conv.r4(d)
415 // except when the ultimate source is a long because there is no long-to-float helper, so it must be 2 step.
416 // This happens semi-frequently because there is no IL 'conv.r4.un'
417 oper->gtType = TYP_FLOAT;
418 oper->CastToType() = TYP_FLOAT;
419 return fgMorphTree(oper);
421 // converts long/ulong --> float/double casts into helper calls.
422 else if (varTypeIsFloating(dstType) && varTypeIsLong(srcType))
424 if (dstType == TYP_FLOAT)
426 // there is only a double helper, so we
427 // - change the dsttype to double
428 // - insert a cast from double to float
429 // - recurse into the resulting tree
430 tree->CastToType() = TYP_DOUBLE;
431 tree->gtType = TYP_DOUBLE;
433 tree = gtNewCastNode(TYP_FLOAT, tree, TYP_FLOAT);
435 return fgMorphTree(tree);
437 if (tree->gtFlags & GTF_UNSIGNED)
438 return fgMorphCastIntoHelper(tree, CORINFO_HELP_ULNG2DBL, oper);
439 return fgMorphCastIntoHelper(tree, CORINFO_HELP_LNG2DBL, oper);
441 #endif //_TARGET_ARM_
443 #ifdef _TARGET_AMD64_
444 // Do we have to do two step U4/8 -> R4/8 ?
445 // Codegen supports the following conversion as one-step operation
449 // The following conversions are performed as two-step operations using above.
450 // U4 -> R4/8 = U4-> Long -> R4/8
451 // U8 -> R4 = U8 -> R8 -> R4
452 else if ((tree->gtFlags & GTF_UNSIGNED) && varTypeIsFloating(dstType))
454 srcType = genUnsignedType(srcType);
456 if (srcType == TYP_ULONG)
458 if (dstType == TYP_FLOAT)
460 // Codegen can handle U8 -> R8 conversion.
461 // U8 -> R4 = U8 -> R8 -> R4
462 // - change the dsttype to double
463 // - insert a cast from double to float
464 // - recurse into the resulting tree
465 tree->CastToType() = TYP_DOUBLE;
466 tree->gtType = TYP_DOUBLE;
467 tree = gtNewCastNode(TYP_FLOAT, tree, TYP_FLOAT);
468 return fgMorphTree(tree);
471 else if (srcType == TYP_UINT)
473 oper = gtNewCastNode(TYP_LONG, oper, TYP_LONG);
474 oper->gtFlags |= (tree->gtFlags & (GTF_OVERFLOW|GTF_EXCEPT|GTF_UNSIGNED));
475 tree->gtFlags &= ~GTF_UNSIGNED;
478 #endif // _TARGET_AMD64_
481 // Do we have to do two step U4/8 -> R4/8 ?
482 else if ((tree->gtFlags & GTF_UNSIGNED) && varTypeIsFloating(dstType))
484 srcType = genUnsignedType(srcType);
486 if (srcType == TYP_ULONG)
488 return fgMorphCastIntoHelper(tree, CORINFO_HELP_ULNG2DBL, oper);
490 else if (srcType == TYP_UINT)
492 oper = gtNewCastNode(TYP_LONG, oper, TYP_LONG);
493 oper->gtFlags |= (tree->gtFlags & (GTF_OVERFLOW|GTF_EXCEPT|GTF_UNSIGNED));
494 tree->gtFlags &= ~GTF_UNSIGNED;
497 #endif //_TARGET_XARCH_
498 else if (varTypeIsGC(srcType) != varTypeIsGC(dstType))
500 // We are casting away GC information. we would like to just
501 // change the type to int, however this gives the emitter fits because
502 // it believes the variable is a GC variable at the begining of the
503 // instruction group, but is not turned non-gc by the code generator
504 // we fix this by copying the GC pointer to a non-gc pointer temp.
505 noway_assert(!varTypeIsGC(dstType) && "How can we have a cast to a GCRef here?");
507 // We generate an assignment to an int and then do the cast from an int. With this we avoid
508 // the gc problem and we allow casts to bytes, longs, etc...
509 unsigned lclNum = lvaGrabTemp(true DEBUGARG("Cast away GC"));
510 oper->gtType = TYP_I_IMPL;
511 GenTreePtr asg = gtNewTempAssign(lclNum, oper);
512 oper->gtType = srcType;
515 GenTreePtr cast = gtNewCastNode(tree->TypeGet(), gtNewLclvNode(lclNum, TYP_I_IMPL), dstType);
517 // Generate the comma tree
518 oper = gtNewOperNode(GT_COMMA, tree->TypeGet(), asg, cast);
520 return fgMorphTree(oper);
523 // Look for narrowing casts ([u]long -> [u]int) and try to push them
524 // down into the operand before morphing it.
526 // It doesn't matter if this is cast is from ulong or long (i.e. if
527 // GTF_UNSIGNED is set) because the transformation is only applied to
528 // overflow-insensitive narrowing casts, which always silently truncate.
530 // Note that casts from [u]long to small integer types are handled above.
531 if ((srcType == TYP_LONG) &&
532 ((dstType == TYP_INT) || (dstType == TYP_UINT)))
534 // As a special case, look for overflow-sensitive casts of an AND
535 // expression, and see if the second operand is a small constant. Since
536 // the result of an AND is bound by its smaller operand, it may be
537 // possible to prove that the cast won't overflow, which will in turn
538 // allow the cast's operand to be transformed.
539 if (tree->gtOverflow() && (oper->OperGet() == GT_AND))
541 GenTreePtr andOp2 = oper->gtOp.gtOp2;
543 // Special case to the special case: AND with a casted int.
544 if ((andOp2->OperGet() == GT_CAST) &&
545 (andOp2->gtCast.CastOp()->OperGet() == GT_CNS_INT))
547 // gtFoldExprConst will deal with whether the cast is signed or
548 // unsigned, or overflow-sensitive.
549 andOp2 = oper->gtOp.gtOp2 = gtFoldExprConst(andOp2);
552 // Look for a constant less than 2^{32} for a cast to uint, or less
553 // than 2^{31} for a cast to int.
554 int maxWidth = (dstType == TYP_UINT) ? 32 : 31;
556 if ((andOp2->OperGet() == GT_CNS_NATIVELONG) &&
557 ((andOp2->gtIntConCommon.LngValue() >> maxWidth) == 0))
559 // This cast can't overflow.
560 tree->gtFlags &= ~(GTF_OVERFLOW | GTF_EXCEPT);
564 // Only apply this transformation during global morph,
565 // when neither the cast node nor the oper node may throw an exception
566 // based on the upper 32 bits.
569 !tree->gtOverflow() &&
570 !oper->gtOverflowEx())
572 // For these operations the lower 32 bits of the result only depends
573 // upon the lower 32 bits of the operands
575 if ( (oper->OperGet() == GT_ADD) ||
576 (oper->OperGet() == GT_MUL) ||
577 (oper->OperGet() == GT_AND) ||
578 (oper->OperGet() == GT_OR) ||
579 (oper->OperGet() == GT_XOR) )
581 DEBUG_DESTROY_NODE(tree);
583 // Insert narrowing casts for op1 and op2
584 oper->gtOp.gtOp1 = gtNewCastNode(TYP_INT, oper->gtOp.gtOp1, dstType);
585 oper->gtOp.gtOp2 = gtNewCastNode(TYP_INT, oper->gtOp.gtOp2, dstType);
587 // Clear the GT_MUL_64RSLT if it is set
588 if (oper->gtOper == GT_MUL && (oper->gtFlags & GTF_MUL_64RSLT))
589 oper->gtFlags &= ~GTF_MUL_64RSLT;
591 // The operation now produces a 32-bit result.
592 oper->gtType = TYP_INT;
594 // Remorph the new tree as the casts that we added may be folded away.
595 return fgMorphTree(oper);
601 noway_assert(tree->gtOper == GT_CAST);
603 /* Morph the operand */
604 tree->gtCast.CastOp() = oper = fgMorphTree(oper);
606 /* Reset the call flag */
607 tree->gtFlags &= ~GTF_CALL;
609 /* unless we have an overflow cast, reset the except flag */
610 if (!tree->gtOverflow())
611 tree->gtFlags &= ~GTF_EXCEPT;
613 /* Just in case new side effects were introduced */
614 tree->gtFlags |= (oper->gtFlags & GTF_ALL_EFFECT);
616 srcType = oper->TypeGet();
618 /* if GTF_UNSIGNED is set then force srcType to an unsigned type */
619 if (tree->gtFlags & GTF_UNSIGNED)
620 srcType = genUnsignedType(srcType);
622 srcSize = genTypeSize(srcType);
624 if (!gtIsActiveCSE_Candidate(tree)) // tree cannot be a CSE candidate
626 /* See if we can discard the cast */
627 if (varTypeIsIntegral(srcType) && varTypeIsIntegral(dstType))
629 if (srcType == dstType) // Certainly if they are identical it is pointless
632 if (oper->OperGet() == GT_LCL_VAR && varTypeIsSmall(dstType))
634 unsigned varNum = oper->gtLclVarCommon.gtLclNum;
635 LclVarDsc * varDsc = &lvaTable[varNum];
636 if (varDsc->TypeGet() == dstType && varDsc->lvNormalizeOnStore())
640 bool unsignedSrc = varTypeIsUnsigned(srcType);
641 bool unsignedDst = varTypeIsUnsigned(dstType);
642 bool signsDiffer = (unsignedSrc != unsignedDst);
644 // For same sized casts with
645 // the same signs or non-overflow cast we discard them as well
646 if (srcSize == dstSize)
648 /* This should have been handled above */
649 noway_assert(varTypeIsGC(srcType) == varTypeIsGC(dstType));
654 if (!tree->gtOverflow())
656 /* For small type casts, when necessary we force
657 the src operand to the dstType and allow the
658 implied load from memory to perform the casting */
659 if (varTypeIsSmall(srcType))
661 switch (oper->gtOper)
667 oper->gtType = dstType;
677 if (srcSize < dstSize) // widening cast
679 // Keep any long casts
680 if (dstSize == sizeof(int))
682 // Only keep signed to unsigned widening cast with overflow check
683 if (!tree->gtOverflow() || !unsignedDst || unsignedSrc)
687 // Casts from signed->unsigned can never overflow while widening
689 if (unsignedSrc || !unsignedDst)
690 tree->gtFlags &= ~GTF_OVERFLOW;
694 // Try to narrow the operand of the cast and discard the cast
695 // Note: Do not narrow a cast that is marked as a CSE
696 // And do not narrow if the oper is marked as a CSE either
698 if (!tree->gtOverflow() &&
699 !gtIsActiveCSE_Candidate(oper) &&
700 (opts.compFlags & CLFLG_TREETRANS) &&
701 optNarrowTree(oper, srcType, dstType, tree->gtVNPair, false))
703 optNarrowTree(oper, srcType, dstType, tree->gtVNPair, true);
705 /* If oper is changed into a cast to TYP_INT, or to a GT_NOP, we may need to discard it */
706 if (oper->gtOper == GT_CAST && oper->CastToType() == genActualType(oper->CastFromType()))
708 oper = oper->gtCast.CastOp();
715 switch (oper->gtOper)
717 /* If the operand is a constant, we'll fold it */
723 GenTreePtr oldTree = tree;
725 tree = gtFoldExprConst(tree); // This may not fold the constant (NaN ...)
727 // Did we get a comma throw as a result of gtFoldExprConst?
728 if ((oldTree != tree) && (oldTree->gtOper != GT_COMMA))
730 noway_assert(fgIsCommaThrow(tree));
731 tree->gtOp.gtOp1 = fgMorphTree(tree->gtOp.gtOp1);
732 fgMorphTreeDone(tree);
735 else if (tree->gtOper != GT_CAST)
738 noway_assert(tree->gtCast.CastOp() == oper); // unchanged
743 /* Check for two consecutive casts into the same dstType */
744 if (!tree->gtOverflow())
746 var_types dstType2 = oper->CastToType();
747 if (dstType == dstType2)
752 /* If op1 is a mod node, mark it with the GTF_MOD_INT_RESULT flag
753 so that the code generator will know not to convert the result
754 of the idiv to a regpair */
756 if (dstType == TYP_INT)
757 tree->gtOp.gtOp1->gtFlags |= GTF_MOD_INT_RESULT;
761 if (dstType == TYP_UINT)
762 tree->gtOp.gtOp1->gtFlags |= GTF_MOD_INT_RESULT;
766 // Check for cast of a GT_COMMA with a throw overflow
767 // Bug 110829: Since this optimization will bash the types
768 // neither oper or commaOp2 can be CSE candidates
769 if (fgIsCommaThrow(oper) &&
770 !gtIsActiveCSE_Candidate(oper)) // oper can not be a CSE candidate
772 GenTreePtr commaOp2 = oper->gtOp.gtOp2;
774 if (!gtIsActiveCSE_Candidate(commaOp2)) // commaOp2 can not be a CSE candidate
776 // need type of oper to be same as tree
777 if (tree->gtType == TYP_LONG)
779 commaOp2->ChangeOperConst(GT_CNS_NATIVELONG);
780 commaOp2->gtIntConCommon.SetLngValue(0);
781 /* Change the types of oper and commaOp2 to TYP_LONG */
782 oper->gtType = commaOp2->gtType = TYP_LONG;
784 else if (varTypeIsFloating(tree->gtType))
786 commaOp2->ChangeOperConst(GT_CNS_DBL);
787 commaOp2->gtDblCon.gtDconVal = 0.0;
788 // Change the types of oper and commaOp2
789 // X87 promotes everything to TYP_DOUBLE
790 // But other's are a little more precise
791 const var_types newTyp
792 #if FEATURE_X87_DOUBLES
794 #else // FEATURE_X87_DOUBLES
796 #endif // FEATURE_X87_DOUBLES
797 oper->gtType = commaOp2->gtType = newTyp;
801 commaOp2->ChangeOperConst(GT_CNS_INT);
802 commaOp2->gtIntCon.gtIconVal = 0;
803 /* Change the types of oper and commaOp2 to TYP_INT */
804 oper->gtType = commaOp2->gtType = TYP_INT;
808 if (vnStore != nullptr)
810 fgValueNumberTreeConst(commaOp2);
813 /* Return the GT_COMMA node as the new tree */
820 } /* end switch (oper->gtOper) */
823 if (tree->gtOverflow())
824 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_OVERFLOW, fgPtrArgCntCur);
830 /* Here we've eliminated the cast, so just return it's operand */
831 assert(!gtIsActiveCSE_Candidate(tree)); // tree cannot be a CSE candidate
833 DEBUG_DESTROY_NODE(tree);
840 /*****************************************************************************
842 * Perform an unwrap operation on a Proxy object
845 GenTreePtr Compiler::fgUnwrapProxy(GenTreePtr objRef)
847 assert(info.compIsContextful &&
848 info.compUnwrapContextful &&
851 CORINFO_EE_INFO * pInfo = eeGetEEInfo();
854 // Perform the unwrap:
856 // This requires two extra indirections.
857 // We mark these indirections as 'invariant' and
858 // the CSE logic will hoist them when appropriate.
860 // Note that each dereference is a GC pointer
862 addTree = gtNewOperNode(GT_ADD, TYP_I_IMPL,
864 gtNewIconNode(pInfo->offsetOfTransparentProxyRP, TYP_I_IMPL));
866 objRef = gtNewOperNode(GT_IND, TYP_REF, addTree);
867 objRef->gtFlags |= GTF_IND_INVARIANT;
869 addTree = gtNewOperNode(GT_ADD, TYP_I_IMPL,
871 gtNewIconNode(pInfo->offsetOfRealProxyServer, TYP_I_IMPL));
873 objRef = gtNewOperNode(GT_IND, TYP_REF, addTree);
874 objRef->gtFlags |= GTF_IND_INVARIANT;
876 // objRef now hold the 'real this' reference (i.e. the unwrapped proxy)
880 /*****************************************************************************
882 * Morph an argument list; compute the pointer argument count in the process.
884 * NOTE: This function can be called from any place in the JIT to perform re-morphing
885 * due to graph altering modifications such as copy / constant propagation
888 unsigned UpdateGT_LISTFlags(GenTreePtr tree)
890 assert(tree->gtOper == GT_LIST);
893 if (tree->gtOp.gtOp2)
895 flags |= UpdateGT_LISTFlags(tree->gtOp.gtOp2);
898 flags |= (tree->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT);
900 tree->gtFlags &= ~GTF_ALL_EFFECT;
901 tree->gtFlags |= flags;
903 return tree->gtFlags;
906 fgArgInfo::fgArgInfo(Compiler * comp, GenTreePtr call, unsigned numArgs)
909 callTree = call; assert(call->IsCall());
910 argCount = 0; // filled in arg count, starts at zero
911 nextSlotNum = INIT_ARG_STACK_SLOT;
913 argTableSize = numArgs; // the allocated table size
914 argsComplete = false;
917 if (argTableSize == 0)
920 argTable = new(compiler, CMK_fgArgInfoPtrArr) fgArgTabEntryPtr[argTableSize];
923 /*****************************************************************************
925 * fgArgInfo Copy Constructor
927 * This method needs to act like a copy constructor for fgArgInfo.
928 * The newCall needs to have its fgArgInfo initialized such that
929 * we have newCall that is an exact copy of the oldCall.
930 * We have to take care since the argument information
931 * in the argTable contains pointers that must point to the
932 * new arguments and not the old arguments.
934 fgArgInfo::fgArgInfo(GenTreePtr newCall, GenTreePtr oldCall)
936 assert(oldCall->IsCall());
937 assert(newCall->IsCall());
939 fgArgInfoPtr oldArgInfo = oldCall->gtCall.fgArgInfo;
941 compiler = oldArgInfo->compiler;;
942 callTree = newCall; assert(newCall->IsCall());
943 argCount = 0; // filled in arg count, starts at zero
944 nextSlotNum = INIT_ARG_STACK_SLOT;
945 stkLevel = oldArgInfo->stkLevel;
946 argTableSize = oldArgInfo->argTableSize;
947 argsComplete = false;
949 if (argTableSize > 0)
951 argTable = new(compiler, CMK_fgArgInfoPtrArr) fgArgTabEntryPtr[argTableSize];
952 for (unsigned inx=0; inx<argTableSize; inx++)
954 argTable[inx] = NULL;
958 assert(oldArgInfo->argsComplete);
960 // We create local, artificial GenTreeArgLists that includes the gtCallObjp, if that exists, as first argument,
961 // so we can iterate over these argument lists more uniformly.
962 // Need to provide a temporary non-null first arguments to these constructors: if we use them, we'll replace them
963 GenTreeArgList* newArgs;
964 GenTreeArgList newArgObjp(newCall, newCall->gtCall.gtCallArgs);
965 GenTreeArgList* oldArgs;
966 GenTreeArgList oldArgObjp(oldCall, oldCall->gtCall.gtCallArgs);
968 if (newCall->gtCall.gtCallObjp == NULL)
970 assert(oldCall->gtCall.gtCallObjp == NULL);
971 newArgs = newCall->gtCall.gtCallArgs;
972 oldArgs = oldCall->gtCall.gtCallArgs;
976 assert(oldCall->gtCall.gtCallObjp != NULL);
977 newArgObjp.Current() = newCall->gtCall.gtCallArgs;
978 newArgs = &newArgObjp;
979 oldArgObjp.Current() = oldCall->gtCall.gtCallObjp;
980 oldArgs = &oldArgObjp;
985 GenTreeArgList* newParent = NULL;
986 GenTreeArgList* oldParent = NULL;
987 fgArgTabEntryPtr * oldArgTable = oldArgInfo->argTable;
988 bool scanRegArgs = false;
992 /* Get hold of the next argument values for the oldCall and newCall */
994 newCurr = newArgs->Current();
995 oldCurr = oldArgs->Current();
996 if (newArgs != &newArgObjp)
1003 assert(newParent == NULL && oldParent == NULL);
1005 newArgs = newArgs->Rest();
1006 oldArgs = oldArgs->Rest();
1008 fgArgTabEntryPtr oldArgTabEntry = NULL;
1009 fgArgTabEntryPtr newArgTabEntry = NULL;
1011 for (unsigned inx=0; inx<argTableSize; inx++)
1013 oldArgTabEntry = oldArgTable[inx];
1015 if (oldArgTabEntry->parent == oldParent)
1017 assert((oldParent == NULL) == (newParent == NULL));
1019 // We have found the matching "parent" field in oldArgTabEntry
1021 newArgTabEntry = new (compiler, CMK_fgArgInfo) fgArgTabEntry;
1023 // First block copy all fields
1025 *newArgTabEntry = *oldArgTabEntry;
1027 // Then update all GenTreePtr fields in the newArgTabEntry
1029 newArgTabEntry->parent = newParent;
1031 // The node field is likely to have been updated
1032 // to point at a node in the gtCallLateArgs list
1034 if (oldArgTabEntry->node == oldCurr)
1036 // node is not pointing into the gtCallLateArgs list
1037 newArgTabEntry->node = newCurr;
1041 // node must be pointing into the gtCallLateArgs list
1043 // We will fix this pointer up in the next loop
1045 newArgTabEntry->node = NULL; // For now we assign a NULL to this field
1050 // Now initialize the proper element in the argTable array
1052 argTable[inx] = newArgTabEntry;
1056 // We should have found the matching oldArgTabEntry and created the newArgTabEntry
1058 assert(newArgTabEntry != NULL);
1063 newArgs = newCall->gtCall.gtCallLateArgs;
1064 oldArgs = oldCall->gtCall.gtCallLateArgs;
1068 /* Get hold of the next argument values for the oldCall and newCall */
1070 assert(newArgs->IsList());
1072 newCurr = newArgs->Current();
1073 newArgs = newArgs->Rest();
1075 assert(oldArgs->IsList());
1077 oldCurr = oldArgs->Current();
1078 oldArgs = oldArgs->Rest();
1080 fgArgTabEntryPtr oldArgTabEntry = NULL;
1081 fgArgTabEntryPtr newArgTabEntry = NULL;
1083 for (unsigned inx=0; inx<argTableSize; inx++)
1085 oldArgTabEntry = oldArgTable[inx];
1087 if (oldArgTabEntry->node == oldCurr)
1089 // We have found the matching "node" field in oldArgTabEntry
1091 newArgTabEntry = argTable[inx];
1092 assert(newArgTabEntry != NULL);
1094 // update the "node" GenTreePtr fields in the newArgTabEntry
1096 assert(newArgTabEntry->node == NULL); // We previously assigned NULL to this field
1098 newArgTabEntry->node = newCurr;
1105 argCount = oldArgInfo->argCount;
1106 nextSlotNum = oldArgInfo->nextSlotNum;
1107 argsComplete = true;
1111 void fgArgInfo::AddArg(fgArgTabEntryPtr curArgTabEntry)
1113 assert(argCount < argTableSize);
1114 argTable[argCount] = curArgTabEntry;
1118 fgArgTabEntryPtr fgArgInfo::AddRegArg(unsigned argNum,
1125 fgArgTabEntryPtr curArgTabEntry = new(compiler, CMK_fgArgInfo) fgArgTabEntry;
1127 curArgTabEntry->argNum = argNum;
1128 curArgTabEntry->node = node;
1129 curArgTabEntry->parent = parent;
1130 curArgTabEntry->regNum = regNum;
1131 curArgTabEntry->slotNum = 0;
1132 curArgTabEntry->numRegs = numRegs;
1133 curArgTabEntry->numSlots = 0;
1134 curArgTabEntry->alignment = alignment;
1135 curArgTabEntry->lateArgInx = (unsigned)-1;
1136 curArgTabEntry->tmpNum = (unsigned)-1;
1137 curArgTabEntry->isSplit = false;
1138 curArgTabEntry->isTmp = false;
1139 curArgTabEntry->needTmp = false;
1140 curArgTabEntry->needPlace = false;
1141 curArgTabEntry->processed = false;
1142 curArgTabEntry->isHfaRegArg = false;
1143 curArgTabEntry->isBackFilled = false;
1144 curArgTabEntry->isNonStandard = false;
1146 AddArg(curArgTabEntry);
1147 return curArgTabEntry;
1150 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1151 fgArgTabEntryPtr fgArgInfo::AddRegArg(unsigned argNum,
1157 const bool isStruct,
1158 const regNumber otherRegNum,
1159 const SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR* const structDescPtr)
1161 fgArgTabEntryPtr curArgTabEntry = AddRegArg(argNum, node, parent, regNum, numRegs, alignment);
1162 assert(curArgTabEntry != nullptr);
1164 // The node of the ArgTabEntry could change after remorphing - it could be rewritten to a cpyblk or a
1165 // PlaceHolder node (in case of needed late argument, for example.)
1166 // This requires using of an extra flag. At creation time the state is right, so
1167 // and this assert enforces that.
1168 assert((varTypeIsStruct(node) && isStruct) || (!varTypeIsStruct(node) && !isStruct));
1169 curArgTabEntry->otherRegNum = otherRegNum; // Second reg for the struct
1170 curArgTabEntry->isStruct = isStruct; // is this a struct arg
1172 if (isStruct && structDescPtr != nullptr)
1174 curArgTabEntry->structDesc.CopyFrom(*structDescPtr);
1177 return curArgTabEntry;
1179 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1181 fgArgTabEntryPtr fgArgInfo::AddStkArg(unsigned argNum,
1186 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(const bool isStruct))
1188 fgArgTabEntryPtr curArgTabEntry = new(compiler, CMK_fgArgInfo) fgArgTabEntry;
1190 nextSlotNum = (unsigned)roundUp(nextSlotNum, alignment);
1192 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1193 // The node of the ArgTabEntry could change after remorphing - it could be rewritten to a cpyblk or a
1194 // PlaceHolder node (in case of needed late argument, for example.)
1195 // This reqires using of an extra flag. At creation time the state is right, so
1196 // and this assert enforces that.
1197 assert((varTypeIsStruct(node) && isStruct) || (!varTypeIsStruct(node) && !isStruct));
1198 curArgTabEntry->isStruct = isStruct; // is this a struct arg
1199 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1201 curArgTabEntry->argNum = argNum;
1202 curArgTabEntry->node = node;
1203 curArgTabEntry->parent = parent;
1204 curArgTabEntry->regNum = REG_STK;
1205 curArgTabEntry->slotNum = nextSlotNum;
1206 curArgTabEntry->numRegs = 0;
1207 curArgTabEntry->numSlots = numSlots;
1208 curArgTabEntry->alignment = alignment;
1209 curArgTabEntry->lateArgInx = (unsigned) -1;
1210 curArgTabEntry->tmpNum = (unsigned) -1;
1211 curArgTabEntry->isSplit = false;
1212 curArgTabEntry->isTmp = false;
1213 curArgTabEntry->needTmp = false;
1214 curArgTabEntry->needPlace = false;
1215 curArgTabEntry->processed = false;
1216 curArgTabEntry->isHfaRegArg = false;
1217 curArgTabEntry->isBackFilled = false;
1218 curArgTabEntry->isNonStandard = false;
1220 AddArg(curArgTabEntry);
1222 nextSlotNum += numSlots;
1223 return curArgTabEntry;
1226 void fgArgInfo::RemorphReset()
1228 nextSlotNum = INIT_ARG_STACK_SLOT;
1231 fgArgTabEntry* fgArgInfo::RemorphRegArg(unsigned argNum,
1238 fgArgTabEntryPtr curArgTabEntry = NULL;
1239 unsigned regArgInx = 0;
1242 for (inx=0; inx < argCount; inx++)
1244 curArgTabEntry = argTable[inx];
1245 if (curArgTabEntry->argNum == argNum)
1250 if (curArgTabEntry->parent != NULL)
1252 assert(curArgTabEntry->parent->IsList());
1253 argx = curArgTabEntry->parent->Current();
1254 isRegArg = (argx->gtFlags & GTF_LATE_ARG) != 0;
1258 argx = curArgTabEntry->node;
1267 // if this was a nonstandard arg the table is definitive
1268 if (curArgTabEntry->isNonStandard)
1269 regNum = curArgTabEntry->regNum;
1271 assert(curArgTabEntry->argNum == argNum);
1272 assert(curArgTabEntry->regNum == regNum);
1273 assert(curArgTabEntry->alignment == alignment);
1274 assert(curArgTabEntry->parent == parent);
1276 if (curArgTabEntry->node != node)
1278 GenTreePtr argx = NULL;
1279 unsigned regIndex = 0;
1281 /* process the register argument list */
1282 for (GenTreeArgList* list = callTree->gtCall.gtCallLateArgs; list; (regIndex++, list = list->Rest()))
1284 argx = list->Current();
1285 assert(!argx->IsArgPlaceHolderNode()); // No place holders nodes are in gtCallLateArgs;
1286 if (regIndex == regArgInx)
1289 assert(regIndex == regArgInx);
1290 assert(regArgInx == curArgTabEntry->lateArgInx);
1292 if (curArgTabEntry->node != argx)
1294 curArgTabEntry->node = argx;
1297 return curArgTabEntry;
1300 void fgArgInfo::RemorphStkArg(unsigned argNum,
1306 fgArgTabEntryPtr curArgTabEntry = NULL;
1307 bool isRegArg = false;
1308 unsigned regArgInx = 0;
1312 for (inx=0; inx < argCount; inx++)
1314 curArgTabEntry = argTable[inx];
1316 if (curArgTabEntry->parent != NULL)
1318 assert(curArgTabEntry->parent->IsList());
1319 argx = curArgTabEntry->parent->Current();
1320 isRegArg = (argx->gtFlags & GTF_LATE_ARG) != 0;
1324 argx = curArgTabEntry->node;
1328 if (curArgTabEntry->argNum == argNum)
1335 nextSlotNum = (unsigned) roundUp(nextSlotNum, alignment);
1337 assert(curArgTabEntry->argNum == argNum);
1338 assert(curArgTabEntry->slotNum == nextSlotNum);
1339 assert(curArgTabEntry->numSlots == numSlots);
1340 assert(curArgTabEntry->alignment == alignment);
1341 assert(curArgTabEntry->parent == parent);
1342 assert(parent->IsList());
1344 #if FEATURE_FIXED_OUT_ARGS
1345 if (curArgTabEntry->node != node)
1349 GenTreePtr argx = NULL;
1350 unsigned regIndex = 0;
1352 /* process the register argument list */
1353 for (GenTreeArgList * list = callTree->gtCall.gtCallLateArgs; list; list = list->Rest(), regIndex++)
1355 argx = list->Current();
1356 assert(!argx->IsArgPlaceHolderNode()); // No place holders nodes are in gtCallLateArgs;
1357 if (regIndex == regArgInx)
1360 assert(regIndex == regArgInx);
1361 assert(regArgInx == curArgTabEntry->lateArgInx);
1363 if (curArgTabEntry->node != argx)
1365 curArgTabEntry->node = argx;
1370 assert(parent->Current() == node);
1371 curArgTabEntry->node = node;
1375 curArgTabEntry->node = node;
1378 nextSlotNum += numSlots;
1381 void fgArgInfo::SplitArg(unsigned argNum,
1385 fgArgTabEntryPtr curArgTabEntry = NULL;
1386 assert(argNum < argCount);
1387 for (unsigned inx=0; inx < argCount; inx++)
1389 curArgTabEntry = argTable[inx];
1390 if (curArgTabEntry->argNum == argNum)
1394 assert(numRegs > 0);
1395 assert(numSlots > 0);
1397 curArgTabEntry->isSplit = true;
1398 curArgTabEntry->numRegs = numRegs;
1399 curArgTabEntry->numSlots = numSlots;
1401 nextSlotNum += numSlots;
1404 void fgArgInfo::EvalToTmp(unsigned argNum,
1408 fgArgTabEntryPtr curArgTabEntry = NULL;
1409 assert(argNum < argCount);
1410 for (unsigned inx=0; inx < argCount; inx++)
1412 curArgTabEntry = argTable[inx];
1413 if (curArgTabEntry->argNum == argNum)
1416 assert(curArgTabEntry->parent->Current() == newNode);
1418 curArgTabEntry->node = newNode;
1419 curArgTabEntry->tmpNum = tmpNum;
1420 curArgTabEntry->isTmp = true;
1423 void fgArgInfo::ArgsComplete()
1425 bool hasStackArgs = false;
1426 bool hasStructRegArg = false;
1428 for (unsigned curInx = 0; curInx < argCount; curInx++)
1430 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1431 assert(curArgTabEntry != NULL);
1432 GenTreePtr argx = curArgTabEntry->node;
1434 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1435 // If this is a struct, mark it for needing a tempVar.
1436 // In the copyblk and store this should have minimal perf impact since
1437 // the local vars where we copy/store to already exist and the logic for temp
1438 // var will not create a new one if it creates a tempVar from another tempVar.
1439 // (Debugging through the code, there was no new copy of data created, neither a new tempVar.)
1440 // The need for this arise from Lower::LowerArg.
1441 // In case of copyblk and store operation, the NewPutArg method will
1442 // not be invoked and the struct will not be loaded to be passed in
1443 // registers or by value on the stack.
1444 if (varTypeIsStruct(argx) FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY( || curArgTabEntry->isStruct))
1446 curArgTabEntry->needTmp = true;
1448 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1450 if (curArgTabEntry->regNum == REG_STK)
1452 hasStackArgs = true;
1453 #if !FEATURE_FIXED_OUT_ARGS
1454 // On x86 we use push instructions to pass arguments:
1455 // The non-register arguments are evaluated and pushed in order
1456 // and they are never evaluated into temps
1461 else // we have a register argument, next we look for a struct type.
1463 if (varTypeIsStruct(argx)
1464 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY( || curArgTabEntry->isStruct))
1466 hasStructRegArg = true;
1470 /* If the argument tree contains an assignment (GTF_ASG) then the argument and
1471 and every earlier argument (except constants) must be evaluated into temps
1472 since there may be other arguments that follow and they may use the value being assigned.
1474 EXAMPLE: ArgTab is "a, a=5, a"
1475 -> when we see the second arg "a=5"
1476 we know the first two arguments "a, a=5" have to be evaluated into temps
1478 For the case of an assignment, we only know that there exist some assignment someplace
1479 in the tree. We don't know what is being assigned so we are very conservative here
1480 and assume that any local variable could have been assigned.
1483 if (argx->gtFlags & GTF_ASG)
1485 // If this is not the only argument, or it's a copyblk, or it already evaluates the expression to
1486 // a tmp, then we need a temp in the late arg list.
1487 if ((argCount > 1) || argx->OperIsCopyBlkOp()
1488 #ifdef FEATURE_FIXED_OUT_ARGS
1489 || curArgTabEntry->isTmp // I protect this by "FEATURE_FIXED_OUT_ARGS" to preserve the property
1490 // that we only have late non-register args when that feature is on.
1491 #endif // FEATURE_FIXED_OUT_ARGS
1494 curArgTabEntry->needTmp = true;
1497 // For all previous arguments, unless they are a simple constant
1498 // we require that they be evaluated into temps
1499 for (unsigned prevInx = 0; prevInx < curInx; prevInx++)
1501 fgArgTabEntryPtr prevArgTabEntry = argTable[prevInx];
1502 assert(prevArgTabEntry->argNum < curArgTabEntry->argNum);
1504 assert(prevArgTabEntry->node);
1505 if (prevArgTabEntry->node->gtOper != GT_CNS_INT)
1507 prevArgTabEntry->needTmp = true;
1512 #if FEATURE_FIXED_OUT_ARGS
1513 // Like calls, if this argument has a tree that will do an inline throw,
1514 // a call to a jit helper, then we need to treat it like a call (but only
1515 // if there are/were any stack args).
1516 // This means unnesting, sorting, etc. Technically this is overly
1517 // conservative, but I want to avoid as much special-case debug-only code
1518 // as possible, so leveraging the GTF_CALL flag is the easiest.
1519 if (!(argx->gtFlags & GTF_CALL) &&
1520 (argx->gtFlags & GTF_EXCEPT) &&
1522 compiler->opts.compDbgCode &&
1523 (compiler->fgWalkTreePre(&argx, Compiler::fgChkThrowCB) == Compiler::WALK_ABORT))
1525 for (unsigned otherInx = 0; otherInx < argCount; otherInx++)
1527 if (otherInx == curInx)
1530 if (argTable[otherInx]->regNum == REG_STK)
1532 argx->gtFlags |= GTF_CALL;
1537 #endif // FEATURE_FIXED_OUT_ARGS
1539 /* If it contains a call (GTF_CALL) then itself and everything before the call
1540 with a GLOB_EFFECT must eval to temp (this is because everything with SIDE_EFFECT
1541 has to be kept in the right order since we will move the call to the first position)
1543 For calls we don't have to be quite as conservative as we are with an assignment
1544 since the call won't be modifying any non-address taken LclVars.
1547 if (argx->gtFlags & GTF_CALL)
1549 if (argCount > 1) // If this is not the only argument
1551 curArgTabEntry->needTmp = true;
1553 else if (varTypeIsFloating(argx->TypeGet()) && (argx->OperGet() == GT_CALL))
1555 // Spill all arguments that are floating point calls
1556 curArgTabEntry->needTmp = true;
1559 // All previous arguments may need to be evaluated into temps
1560 for (unsigned prevInx = 0; prevInx < curInx; prevInx++)
1562 fgArgTabEntryPtr prevArgTabEntry = argTable[prevInx];
1563 assert(prevArgTabEntry->argNum < curArgTabEntry->argNum);
1564 assert(prevArgTabEntry->node);
1566 // For all previous arguments, if they have any GTF_ALL_EFFECT
1567 // we require that they be evaluated into a temp
1568 if ((prevArgTabEntry->node->gtFlags & GTF_ALL_EFFECT) != 0)
1570 prevArgTabEntry->needTmp = true;
1572 #if FEATURE_FIXED_OUT_ARGS
1573 // Or, if they are stored into the FIXED_OUT_ARG area
1574 // we require that they be moved to the gtCallLateArgs
1575 // and replaced with a placeholder node
1576 else if (prevArgTabEntry->regNum == REG_STK)
1578 prevArgTabEntry->needPlace = true;
1584 #ifndef LEGACY_BACKEND
1585 // For RyuJIT backend we will expand a Multireg arg into a GT_LIST
1586 // with multiple indirections, so here we consider spilling it into a tmp LclVar.
1588 // Note that Arm32 is a LEGACY_BACKEND and it defines FEATURE_MULTIREG_ARGS
1589 // so we skip this for ARM32 until it is ported to use RyuJIT backend
1591 #if FEATURE_MULTIREG_ARGS
1592 if ((argx->TypeGet() == TYP_STRUCT) &&
1593 (curArgTabEntry->numRegs > 1) &&
1594 (curArgTabEntry->needTmp == false))
1596 if ((argx->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) != 0)
1598 // Spill multireg struct arguments that have Assignments or Calls embedded in them
1599 curArgTabEntry->needTmp = true;
1603 // We call gtPrepareCost to measure the cost of evaluating this tree
1604 compiler->gtPrepareCost(argx);
1606 if (argx->gtCostEx > (6 * IND_COST_EX))
1608 // Spill multireg struct arguments that are expensive to evaluate twice
1609 curArgTabEntry->needTmp = true;
1613 #endif // FEATURE_MULTIREG_ARGS
1614 #endif // LEGACY_BACKEND
1618 // We only care because we can't spill structs and qmarks involve a lot of spilling, but
1619 // if we don't have qmarks, then it doesn't matter.
1620 // So check for Qmark's globally once here, instead of inside the loop.
1622 const bool hasStructRegArgWeCareAbout = (hasStructRegArg && compiler->compQmarkUsed);
1624 #if FEATURE_FIXED_OUT_ARGS
1626 // For Arm/x64 we only care because we can't reorder a register
1627 // argument that uses GT_LCLHEAP. This is an optimization to
1628 // save a check inside the below loop.
1630 const bool hasStackArgsWeCareAbout = (hasStackArgs && compiler->compLocallocUsed);
1634 const bool hasStackArgsWeCareAbout = hasStackArgs;
1636 #endif // FEATURE_FIXED_OUT_ARGS
1638 // If we have any stack args we have to force the evaluation
1639 // of any arguments passed in registers that might throw an exception
1641 // Technically we only a required to handle the following two cases:
1642 // a GT_IND with GTF_IND_RNGCHK (only on x86) or
1643 // a GT_LCLHEAP node that allocates stuff on the stack
1645 if (hasStackArgsWeCareAbout || hasStructRegArgWeCareAbout)
1647 for (unsigned curInx = 0; curInx < argCount; curInx++)
1649 fgArgTabEntryPtr curArgTabEntry = argTable[curInx]; assert(curArgTabEntry != NULL);
1650 GenTreePtr argx = curArgTabEntry->node;
1652 // Examine the register args that are currently not marked needTmp
1654 if (!curArgTabEntry->needTmp && (curArgTabEntry->regNum != REG_STK))
1656 if (hasStackArgsWeCareAbout)
1658 #if !FEATURE_FIXED_OUT_ARGS
1659 // On x86 we previously recorded a stack depth of zero when
1660 // morphing the register arguments of any GT_IND with a GTF_IND_RNGCHK flag
1661 // Thus we can not reorder the argument after any stack based argument
1662 // (Note that GT_LCLHEAP sets the GTF_EXCEPT flag so we don't need to
1663 // check for it explicitly
1665 if (argx->gtFlags & GTF_EXCEPT)
1667 curArgTabEntry->needTmp = true;
1671 // For Arm/X64 we can't reorder a register argument that uses a GT_LCLHEAP
1673 if (argx->gtFlags & GTF_EXCEPT)
1675 assert(compiler->compLocallocUsed);
1677 // Returns WALK_ABORT if a GT_LCLHEAP node is encountered in the argx tree
1679 if (compiler->fgWalkTreePre(&argx, Compiler::fgChkLocAllocCB) == Compiler::WALK_ABORT)
1681 curArgTabEntry->needTmp = true;
1687 if (hasStructRegArgWeCareAbout)
1689 // Returns true if a GT_QMARK node is encountered in the argx tree
1691 if (compiler->fgWalkTreePre(&argx, Compiler::fgChkQmarkCB) == Compiler::WALK_ABORT)
1693 curArgTabEntry->needTmp = true;
1701 argsComplete = true;
1704 void fgArgInfo::SortArgs()
1706 assert(argsComplete == true);
1708 /* Shuffle the arguments around before we build the gtCallLateArgs list.
1709 The idea is to move all "simple" arguments like constants and local vars
1710 to the end of the table, and move the complex arguments towards the beginning
1711 of the table. This will help prevent registers from being spilled by
1712 allowing us to evaluate the more complex arguments before the simpler arguments.
1713 The argTable ends up looking like:
1714 +------------------------------------+ <--- argTable[argCount - 1]
1716 +------------------------------------+
1717 | local var / local field |
1718 +------------------------------------+
1719 | remaining arguments sorted by cost |
1720 +------------------------------------+
1721 | temps (argTable[].needTmp = true) |
1722 +------------------------------------+
1723 | args with calls (GTF_CALL) |
1724 +------------------------------------+ <--- argTable[0]
1728 if (compiler->verbose)
1730 printf("\nSorting the arguments:\n");
1734 /* Set the beginning and end for the new argument table */
1737 unsigned begTab = 0;
1738 unsigned endTab = argCount - 1;
1739 unsigned argsRemaining = argCount;
1741 // First take care of arguments that are constants.
1742 // [We use a backward iterator pattern]
1748 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1750 if (curArgTabEntry->regNum != REG_STK)
1753 // Skip any already processed args
1755 if (!curArgTabEntry->processed)
1757 GenTreePtr argx = curArgTabEntry->node;
1759 // put constants at the end of the table
1761 if (argx->gtOper == GT_CNS_INT)
1763 noway_assert(curInx <= endTab);
1765 curArgTabEntry->processed = true;
1767 // place curArgTabEntry at the endTab position by performing a swap
1769 if (curInx != endTab)
1771 argTable[curInx] = argTable[endTab];
1772 argTable[endTab] = curArgTabEntry;
1779 } while (curInx > 0);
1781 if (argsRemaining > 0)
1783 // Next take care of arguments that are calls.
1784 // [We use a forward iterator pattern]
1786 for (curInx = begTab; curInx <= endTab; curInx++)
1788 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1790 // Skip any already processed args
1792 if (!curArgTabEntry->processed)
1794 GenTreePtr argx = curArgTabEntry->node;
1796 // put calls at the beginning of the table
1798 if (argx->gtFlags & GTF_CALL)
1800 curArgTabEntry->processed = true;
1802 // place curArgTabEntry at the begTab position by performing a swap
1804 if (curInx != begTab)
1806 argTable[curInx] = argTable[begTab];
1807 argTable[begTab] = curArgTabEntry;
1817 if (argsRemaining > 0)
1819 // Next take care arguments that are temps.
1820 // These temps come before the arguments that are
1821 // ordinary local vars or local fields
1822 // since this will give them a better chance to become
1823 // enregistered into their actual argument register.
1824 // [We use a forward iterator pattern]
1826 for (curInx = begTab; curInx <= endTab; curInx++)
1828 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1830 // Skip any already processed args
1832 if (!curArgTabEntry->processed)
1834 if (curArgTabEntry->needTmp)
1836 curArgTabEntry->processed = true;
1838 // place curArgTabEntry at the begTab position by performing a swap
1840 if (curInx != begTab)
1842 argTable[curInx] = argTable[begTab];
1843 argTable[begTab] = curArgTabEntry;
1853 if (argsRemaining > 0)
1855 // Next take care of local var and local field arguments.
1856 // These are moved towards the end of the argument evaluation.
1857 // [We use a backward iterator pattern]
1859 curInx = endTab + 1;
1863 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1865 // Skip any already processed args
1867 if (!curArgTabEntry->processed)
1869 GenTreePtr argx = curArgTabEntry->node;
1871 if ((argx->gtOper == GT_LCL_VAR) || (argx->gtOper == GT_LCL_FLD))
1873 noway_assert(curInx <= endTab);
1875 curArgTabEntry->processed = true;
1877 // place curArgTabEntry at the endTab position by performing a swap
1879 if (curInx != endTab)
1881 argTable[curInx] = argTable[endTab];
1882 argTable[endTab] = curArgTabEntry;
1889 } while (curInx > begTab);
1892 // Finally, take care of all the remaining arguments.
1893 // Note that we fill in one arg at a time using a while loop.
1894 bool costsPrepared = false; // Only prepare tree costs once, the first time through this loop
1895 while (argsRemaining > 0)
1897 /* Find the most expensive arg remaining and evaluate it next */
1899 fgArgTabEntryPtr expensiveArgTabEntry = NULL;
1900 unsigned expensiveArg = UINT_MAX;
1901 unsigned expensiveArgCost = 0;
1903 // [We use a forward iterator pattern]
1905 for (curInx = begTab; curInx <= endTab; curInx++)
1907 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1909 // Skip any already processed args
1911 if (!curArgTabEntry->processed)
1913 GenTreePtr argx = curArgTabEntry->node;
1915 // We should have already handled these kinds of args
1916 assert (argx->gtOper != GT_LCL_VAR);
1917 assert (argx->gtOper != GT_LCL_FLD);
1918 assert (argx->gtOper != GT_CNS_INT);
1920 // This arg should either have no persistent side effects or be the last one in our table
1921 // assert(((argx->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) == 0) || (curInx == (argCount-1)));
1923 if (argsRemaining == 1)
1925 // This is the last arg to place
1926 expensiveArg = curInx;
1927 expensiveArgTabEntry = curArgTabEntry;
1928 assert(begTab == endTab);
1935 /* We call gtPrepareCost to measure the cost of evaluating this tree */
1936 compiler->gtPrepareCost(argx);
1939 if (argx->gtCostEx > expensiveArgCost)
1941 // Remember this arg as the most expensive one that we have yet seen
1942 expensiveArgCost = argx->gtCostEx;
1943 expensiveArg = curInx;
1944 expensiveArgTabEntry = curArgTabEntry;
1950 noway_assert(expensiveArg != UINT_MAX);
1952 // put the most expensive arg towards the beginning of the table
1954 expensiveArgTabEntry->processed = true;
1956 // place expensiveArgTabEntry at the begTab position by performing a swap
1958 if (expensiveArg != begTab)
1960 argTable[expensiveArg] = argTable[begTab];
1961 argTable[begTab] = expensiveArgTabEntry;
1967 costsPrepared = true; // If we have more expensive arguments, don't re-evaluate the tree cost on the next loop
1970 // The table should now be completely filled and thus begTab should now be adjacent to endTab
1971 // and regArgsRemaining should be zero
1972 assert(begTab == (endTab + 1));
1973 assert(argsRemaining == 0);
1975 #if !FEATURE_FIXED_OUT_ARGS
1976 // Finally build the regArgList
1978 callTree->gtCall.regArgList = NULL;
1979 callTree->gtCall.regArgListCount = regCount;
1981 unsigned regInx = 0;
1982 for (curInx = 0; curInx < argCount; curInx++)
1984 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1986 if (curArgTabEntry->regNum != REG_STK)
1988 // Encode the argument register in the register mask
1990 callTree->gtCall.regArgList[regInx] = curArgTabEntry->regNum;
1994 #endif // !FEATURE_FIXED_OUT_ARGS
1999 //------------------------------------------------------------------------------
2000 // fgMakeTmpArgNode : This function creates a tmp var only if needed.
2001 // We need this to be done in order to enforce ordering
2002 // of the evaluation of arguments.
2005 // tmpVarNum - the var num which we clone into the newly created temp var.
2008 // the newly created temp var tree.
2010 GenTreePtr Compiler::fgMakeTmpArgNode(unsigned tmpVarNum
2011 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(const bool passedInRegisters))
2013 LclVarDsc * varDsc = &lvaTable[tmpVarNum];
2014 assert(varDsc->lvIsTemp);
2015 var_types type = varDsc->TypeGet();
2017 // Create a copy of the temp to go into the late argument list
2018 GenTreePtr arg = gtNewLclvNode(tmpVarNum, type);
2019 GenTreePtr addrNode = nullptr;
2021 if (varTypeIsStruct(type))
2024 #if defined(_TARGET_AMD64_) || defined(_TARGET_ARM64_)
2026 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
2028 arg->gtFlags |= GTF_DONT_CSE;
2030 // If it is passed in registers, don't get the address of the var. Make it a
2031 // field instead. It will be loaded in registers with putarg_reg tree in lower.
2032 if (passedInRegisters)
2034 arg->ChangeOper(GT_LCL_FLD);
2039 arg = gtNewOperNode(GT_ADDR, type, arg);
2043 #else // !FEATURE_UNIX_AMD64_STRUCT_PASSING
2045 unsigned structSize = lvaLclExactSize(tmpVarNum);
2049 case 1: type = TYP_BYTE; break;
2050 case 2: type = TYP_SHORT; break;
2051 #if defined (_TARGET_AMD64_)
2052 case 4: type = TYP_INT; break;
2053 #elif defined(_TARGET_ARM64_)
2055 case 4: type = TYP_INT; break;
2058 case 7: type = TYP_I_IMPL; break;
2059 #endif // defined (_TARGET_ARM64_)
2061 switch (*lvaGetGcLayout(tmpVarNum))
2080 // If we didn't change the type of the struct, it means
2081 // its structure doesn't support to be passed directly through a
2082 // register, so we need to pass a pointer to the destination where
2083 // where we copied the struct to.
2084 if (type == varDsc->TypeGet())
2086 #if FEATURE_MULTIREG_ARGS
2087 #ifdef _TARGET_ARM64_
2088 assert(varTypeIsStruct(type));
2089 if (varDsc->lvIsMultiregStruct())
2091 // ToDo-ARM64: Consider using: arg->ChangeOper(GT_LCL_FLD);
2092 // as that is how FEATURE_UNIX_AMD64_STRUCT_PASSING works.
2093 // Create a GT_OBJ for the argument
2094 // This will be passed by value in two registers
2095 arg = gtNewOperNode(GT_ADDR, TYP_BYREF, arg);
2098 // Create an Obj of the temp to use it as a call argument.
2099 arg = gtNewObjNode(lvaGetStruct(tmpVarNum), arg);
2102 #endif // _TARGET_ARM64_
2103 #endif // FEATURE_MULTIREG_ARGS
2105 arg = gtNewOperNode(GT_ADDR, TYP_I_IMPL, arg);
2109 else // type was changed from a struct to a scalar type
2111 arg->ChangeOper(GT_LCL_FLD);
2114 #endif // !FEATURE_UNIX_AMD64_STRUCT_PASSING
2116 #else // not (_TARGET_AMD64_ or _TARGET_ARM64_)
2118 // other targets, we pass the struct by value
2119 assert(varTypeIsStruct(type));
2121 arg = gtNewOperNode(GT_ADDR, TYP_BYREF, arg);
2124 // Get a new Obj node temp to use it as a call argument
2125 arg = gtNewObjNode(lvaGetStruct(tmpVarNum), arg);
2126 arg->gtFlags |= GTF_EXCEPT;
2128 #endif // not (_TARGET_AMD64_ or _TARGET_ARM64_)
2130 } // (varTypeIsStruct(type))
2132 if (addrNode != nullptr)
2134 assert(addrNode->gtOper == GT_ADDR);
2136 // This will prevent this LclVar from being optimized away
2137 lvaSetVarAddrExposed(tmpVarNum);
2139 // the child of a GT_ADDR is required to have this flag set
2140 addrNode->gtOp.gtOp1->gtFlags |= GTF_DONT_CSE;
2146 void fgArgInfo::EvalArgsToTemps()
2148 assert(argsSorted == true);
2150 unsigned regArgInx = 0;
2151 // Now go through the argument table and perform the necessary evaluation into temps
2152 GenTreeArgList* tmpRegArgNext = NULL;
2153 for (unsigned curInx = 0; curInx < argCount; curInx++)
2155 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
2157 GenTreePtr argx = curArgTabEntry->node;
2158 GenTreePtr setupArg = NULL;
2161 #if !FEATURE_FIXED_OUT_ARGS
2162 // Only ever set for FEATURE_FIXED_OUT_ARGS
2163 assert(curArgTabEntry->needPlace == false);
2165 // On x86 and other archs that use push instructions to pass arguments:
2166 // Only the register arguments need to be replaced with placeholder nodes.
2167 // Stacked arguments are evaluated and pushed (or stored into the stack) in order.
2169 if (curArgTabEntry->regNum == REG_STK)
2173 if (curArgTabEntry->needTmp)
2177 if (curArgTabEntry->isTmp == true)
2179 // Create a copy of the temp to go into the late argument list
2180 tmpVarNum = curArgTabEntry->tmpNum;
2181 defArg = compiler->fgMakeTmpArgNode(
2183 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(argTable[curInx]->structDesc.passedInRegisters));
2185 // mark the original node as a late argument
2186 argx->gtFlags |= GTF_LATE_ARG;
2190 // Create a temp assignment for the argument
2191 // Put the temp in the gtCallLateArgs list
2193 if (compiler->verbose)
2195 printf("Argument with 'side effect'...\n");
2196 compiler->gtDispTree(argx);
2200 #if defined(_TARGET_AMD64_) && !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
2201 noway_assert(argx->gtType != TYP_STRUCT);
2204 tmpVarNum = compiler->lvaGrabTemp(true DEBUGARG("argument with side effect"));
2205 if (argx->gtOper == GT_MKREFANY)
2207 // For GT_MKREFANY, typically the actual struct copying does
2208 // not have any side-effects and can be delayed. So instead
2209 // of using a temp for the whole struct, we can just use a temp
2210 // for operand that that has a side-effect
2212 if ((argx->gtOp.gtOp2->gtFlags & GTF_ALL_EFFECT) == 0)
2214 operand = argx->gtOp.gtOp1;
2216 // In the early argument evaluation, place an assignment to the temp
2217 // from the source operand of the mkrefany
2218 setupArg = compiler->gtNewTempAssign(tmpVarNum, operand);
2220 // Replace the operand for the mkrefany with the new temp.
2221 argx->gtOp.gtOp1 = compiler->gtNewLclvNode(tmpVarNum, operand->TypeGet());
2223 else if ((argx->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT) == 0)
2225 operand = argx->gtOp.gtOp2;
2227 // In the early argument evaluation, place an assignment to the temp
2228 // from the source operand of the mkrefany
2229 setupArg = compiler->gtNewTempAssign(tmpVarNum, operand);
2231 // Replace the operand for the mkrefany with the new temp.
2232 argx->gtOp.gtOp2 = compiler->gtNewLclvNode(tmpVarNum, operand->TypeGet());
2236 if (setupArg != NULL)
2238 // Now keep the mkrefany for the late argument list
2241 // Clear the side-effect flags because now both op1 and op2 have no side-effects
2242 defArg->gtFlags &= ~GTF_ALL_EFFECT;
2246 setupArg = compiler->gtNewTempAssign(tmpVarNum, argx);
2248 #ifndef LEGACY_BACKEND
2249 if (compiler->fgOrder == Compiler::FGOrderLinear)
2251 // We'll reference this temporary variable just once
2252 // when we perform the function call after
2253 // setting up this argument.
2254 LclVarDsc* varDsc = compiler->lvaTable + tmpVarNum;
2255 varDsc->lvRefCnt = 1;
2257 #endif // !LEGACY_BACKEND
2259 if (setupArg->OperIsCopyBlkOp())
2260 setupArg = compiler->fgMorphCopyBlock(setupArg);
2262 /* Create a copy of the temp to go to the late argument list */
2263 defArg = compiler->gtNewLclvNode(tmpVarNum, genActualType(argx->gtType));
2265 curArgTabEntry->isTmp = true;
2266 curArgTabEntry->tmpNum = tmpVarNum;
2269 // Previously we might have thought the local was promoted, and thus the 'COPYBLK'
2270 // might have left holes in the used registers (see
2271 // fgAddSkippedRegsInPromotedStructArg).
2272 // Too bad we're not that smart for these intermediate temps...
2273 if (isValidIntArgReg(curArgTabEntry->regNum) && (curArgTabEntry->numRegs > 1))
2275 regNumber argReg = curArgTabEntry->regNum;
2276 regMaskTP allUsedRegs = genRegMask(curArgTabEntry->regNum);
2277 for (unsigned i = 1; i < curArgTabEntry->numRegs; i++)
2279 argReg = genRegArgNext(argReg);
2280 allUsedRegs |= genRegMask(argReg);
2282 callTree->gtCall.gtCallRegUsedMask |= allUsedRegs;
2284 #endif // _TARGET_ARM_
2287 /* mark the assignment as a late argument */
2288 setupArg->gtFlags |= GTF_LATE_ARG;
2291 if (compiler->verbose)
2293 printf("\n Evaluate to a temp:\n");
2294 compiler->gtDispTree(setupArg);
2299 else // curArgTabEntry->needTmp == false
2302 // Only register args are replaced with placeholder nodes
2303 // and the stack based arguments are evaluated and pushed in order.
2305 // On Arm/x64 - When needTmp is false and needPlace is false,
2306 // the non-register arguments are evaluated and stored in order.
2307 // When needPlace is true we have a nested call that comes after
2308 // this argument so we have to replace it in the gtCallArgs list
2309 // (the initial argument evaluation list) with a placeholder.
2311 if ((curArgTabEntry->regNum == REG_STK) && (curArgTabEntry->needPlace == false))
2314 /* No temp needed - move the whole node to the gtCallLateArgs list */
2316 /* The argument is deferred and put in the late argument list */
2320 // Create a placeholder node to put in its place in gtCallLateArgs.
2322 // For a struct type we also need to record the class handle of the arg.
2323 CORINFO_CLASS_HANDLE clsHnd = NO_CLASS_HANDLE;
2325 #if defined(_TARGET_AMD64_) && !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
2327 // All structs are either passed (and retyped) as integral types, OR they
2328 // are passed by reference.
2329 noway_assert(argx->gtType != TYP_STRUCT);
2331 #else // !defined(_TARGET_AMD64_) || defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
2333 if (varTypeIsStruct(defArg))
2335 // Need a temp to walk any GT_COMMA nodes when searching for the clsHnd
2336 GenTreePtr defArgTmp = defArg;
2338 // The GT_OBJ may be be a child of a GT_COMMA.
2339 while (defArgTmp->gtOper == GT_COMMA)
2341 defArgTmp = defArgTmp->gtOp.gtOp2;
2343 assert(varTypeIsStruct(defArgTmp));
2345 // We handle two opcodes: GT_MKREFANY and GT_OBJ.
2346 if (defArgTmp->gtOper == GT_MKREFANY)
2348 clsHnd = compiler->impGetRefAnyClass();
2350 else if (defArgTmp->gtOper == GT_OBJ)
2352 clsHnd = defArgTmp->AsObj()->gtClass;
2356 BADCODE("Unhandled struct argument tree in fgMorphArgs");
2360 #endif // !(defined(_TARGET_AMD64_) && !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING))
2362 setupArg = compiler->gtNewArgPlaceHolderNode(defArg->gtType, clsHnd);
2364 /* mark the placeholder node as a late argument */
2365 setupArg->gtFlags |= GTF_LATE_ARG;
2368 if (compiler->verbose)
2370 if (curArgTabEntry->regNum == REG_STK)
2372 printf("Deferred stack argument :\n");
2376 printf("Deferred argument ('%s'):\n", getRegName(curArgTabEntry->regNum));
2379 compiler->gtDispTree(argx);
2380 printf("Replaced with placeholder node:\n");
2381 compiler->gtDispTree(setupArg);
2386 if (setupArg != NULL)
2388 if (curArgTabEntry->parent)
2390 GenTreePtr parent = curArgTabEntry->parent;
2391 /* a normal argument from the list */
2392 noway_assert(parent->IsList());
2393 noway_assert(parent->gtOp.gtOp1 == argx);
2395 parent->gtOp.gtOp1 = setupArg;
2399 /* must be the gtCallObjp */
2400 noway_assert(callTree->gtCall.gtCallObjp == argx);
2402 callTree->gtCall.gtCallObjp = setupArg;
2406 /* deferred arg goes into the late argument list */
2408 if (tmpRegArgNext == NULL)
2410 tmpRegArgNext = compiler->gtNewArgList(defArg);
2411 callTree->gtCall.gtCallLateArgs = tmpRegArgNext;
2415 noway_assert(tmpRegArgNext->IsList());
2416 noway_assert(tmpRegArgNext->Current());
2417 tmpRegArgNext->gtOp.gtOp2 = compiler->gtNewArgList(defArg);
2418 tmpRegArgNext = tmpRegArgNext->Rest();
2421 curArgTabEntry->node = defArg;
2422 curArgTabEntry->lateArgInx = regArgInx++;
2426 if (compiler->verbose)
2428 printf("\nShuffled argument table: ");
2429 for (unsigned curInx = 0; curInx < argCount; curInx++)
2431 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
2433 if (curArgTabEntry->regNum != REG_STK)
2435 printf("%s ", getRegName( curArgTabEntry->regNum ));
2443 void fgArgInfo::RecordStkLevel(unsigned stkLvl)
2445 assert(!IsUninitialized(stkLvl));
2446 this->stkLevel = stkLvl;
2449 unsigned fgArgInfo::RetrieveStkLevel()
2451 assert(!IsUninitialized(stkLevel));
2455 // Return a conservative estimate of the stack size in bytes.
2456 // It will be used only on the intercepted-for-host code path to copy the arguments.
2457 int Compiler::fgEstimateCallStackSize(GenTreeCall* call)
2461 for (GenTreeArgList* args = call->gtCallArgs; args; args = args->Rest())
2467 if (numArgs > MAX_REG_ARG)
2468 numStkArgs = numArgs - MAX_REG_ARG;
2472 return numStkArgs * REGSIZE_BYTES;
2475 //------------------------------------------------------------------------------
2476 // fgMakeMultiUse : If the node is a local, clone it and increase the ref count
2477 // otherwise insert a comma form temp
2480 // ppTree - a pointer to the child node we will be replacing with the comma expression that
2481 // evaluates ppTree to a temp and returns the result
2484 // A fresh GT_LCL_VAR node referencing the temp which has not been used
2487 // The result tree MUST be added to the tree structure since the ref counts are
2488 // already incremented.
2490 GenTree* Compiler::fgMakeMultiUse(GenTree** pOp)
2492 GenTree* tree = *pOp;
2493 if (tree->IsLocal())
2495 auto result = gtClone(tree);
2496 if (lvaLocalVarRefCounted)
2498 lvaTable[tree->gtLclVarCommon.gtLclNum].incRefCnts(compCurBB->getBBWeight(this), this);
2504 GenTree* result = fgInsertCommaFormTemp(pOp);
2506 // At this point, *pOp is GT_COMMA(GT_ASG(V01, *pOp), V01) and result = V01
2507 // Therefore, the ref count has to be incremented 3 times for *pOp and result, if result will
2508 // be added by the caller.
2509 if (lvaLocalVarRefCounted)
2511 lvaTable[result->gtLclVarCommon.gtLclNum].incRefCnts(compCurBB->getBBWeight(this), this);
2512 lvaTable[result->gtLclVarCommon.gtLclNum].incRefCnts(compCurBB->getBBWeight(this), this);
2513 lvaTable[result->gtLclVarCommon.gtLclNum].incRefCnts(compCurBB->getBBWeight(this), this);
2521 //------------------------------------------------------------------------------
2522 // fgInsertCommaFormTemp: Create a new temporary variable to hold the result of *ppTree,
2523 // and replace *ppTree with comma(asg(newLcl, *ppTree), newLcl)
2526 // ppTree - a pointer to the child node we will be replacing with the comma expression that
2527 // evaluates ppTree to a temp and returns the result
2529 // structType - value type handle if the temp created is of TYP_STRUCT.
2532 // A fresh GT_LCL_VAR node referencing the temp which has not been used
2535 GenTree* Compiler::fgInsertCommaFormTemp(GenTree** ppTree, CORINFO_CLASS_HANDLE structType /*= nullptr*/)
2537 GenTree* subTree = *ppTree;
2539 unsigned lclNum = lvaGrabTemp(true DEBUGARG("fgInsertCommaFormTemp is creating a new local variable"));
2541 if (varTypeIsStruct(subTree))
2543 assert(structType != nullptr);
2544 lvaSetStruct(lclNum, structType, false);
2547 // If subTree->TypeGet() == TYP_STRUCT, gtNewTempAssign() will create a GT_COPYBLK tree.
2548 // The type of GT_COPYBLK is TYP_VOID. Therefore, we should use subTree->TypeGet() for
2549 // setting type of lcl vars created.
2550 GenTree* asg = gtNewTempAssign(lclNum, subTree);
2552 GenTree* load = new (this, GT_LCL_VAR) GenTreeLclVar(subTree->TypeGet(), lclNum, BAD_IL_OFFSET);
2554 GenTree* comma = gtNewOperNode(GT_COMMA, subTree->TypeGet(), asg, load);
2558 return new (this, GT_LCL_VAR) GenTreeLclVar(subTree->TypeGet(), lclNum, BAD_IL_OFFSET);
2562 //------------------------------------------------------------------------
2563 // fgMorphArgs: Walk and transform (morph) the arguments of a call
2566 // callNode - the call for which we are doing the argument morphing
2569 // Like most morph methods, this method returns the morphed node,
2570 // though in this case there are currently no scenarios where the
2571 // node itself is re-created.
2574 // This method is even less idempotent than most morph methods.
2575 // That is, it makes changes that should not be redone. It uses the existence
2576 // of gtCallLateArgs (the late arguments list) to determine if it has
2577 // already done that work.
2579 // The first time it is called (i.e. during global morphing), this method
2580 // computes the "late arguments". This is when it determines which arguments
2581 // need to be evaluated to temps prior to the main argument setup, and which
2582 // can be directly evaluated into the argument location. It also creates a
2583 // second argument list (gtCallLateArgs) that does the final placement of the
2584 // arguments, e.g. into registers or onto the stack.
2586 // The "non-late arguments", aka the gtCallArgs, are doing the in-order
2587 // evaluation of the arguments that might have side-effects, such as embedded
2588 // assignments, calls or possible throws. In these cases, it and earlier
2589 // arguments must be evaluated to temps.
2591 // On targets with a fixed outgoing argument area (FEATURE_FIXED_OUT_ARGS),
2592 // if we have any nested calls, we need to defer the copying of the argument
2593 // into the fixed argument area until after the call. If the argument did not
2594 // otherwise need to be computed into a temp, it is moved to gtCallLateArgs and
2595 // replaced in the "early" arg list (gtCallArgs) with a placeholder node.
2598 #pragma warning(push)
2599 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
2601 GenTreeCall* Compiler::fgMorphArgs(GenTreeCall* callNode)
2603 GenTreeCall* call = callNode->AsCall();
2608 unsigned flagsSummary = 0;
2609 unsigned genPtrArgCntSav = fgPtrArgCntCur;
2611 unsigned argIndex = 0;
2613 unsigned intArgRegNum = 0;
2614 unsigned fltArgRegNum = 0;
2616 regMaskTP argSkippedRegMask = RBM_NONE;
2618 #if defined(_TARGET_ARM_) || defined(_TARGET_AMD64_)
2619 regMaskTP fltArgSkippedRegMask = RBM_NONE;
2622 #if defined(_TARGET_X86_)
2623 unsigned maxRegArgs = MAX_REG_ARG; // X86: non-const, must be calculated
2625 const unsigned maxRegArgs = MAX_REG_ARG; // other arch: fixed constant number
2628 unsigned argSlots = 0;
2629 unsigned nonRegPassedStructSlots = 0;
2630 bool lateArgsComputed = (call->gtCallLateArgs != nullptr);
2631 bool callHasRetBuffArg = call->HasRetBufArg();
2633 #ifndef _TARGET_X86_ // i.e. _TARGET_AMD64_ or _TARGET_ARM_
2634 bool callIsVararg = call->IsVarargs();
2637 bool hasNonStandardArg = false;
2638 #ifndef LEGACY_BACKEND
2639 // data structure for keeping track of non-standard args we insert
2640 // (args that have a special meaning and are not passed following the normal
2641 // calling convention or even in the normal arg regs.
2642 struct NonStandardArg
2648 ArrayStack<NonStandardArg> nonStandardArgs(this, 2);
2649 #endif // !LEGACY_BACKEND
2651 // Process the late arguments (which were determined by a previous caller).
2652 // Do this before resetting fgPtrArgCntCur as fgMorphTree(call->gtCallLateArgs)
2653 // may need to refer to it.
2654 if (lateArgsComputed)
2656 // We need to reMorph the gtCallLateArgs early since that is what triggers
2657 // the expression folding and we need to have the final folded gtCallLateArgs
2658 // available when we call RemorphRegArg so that we correctly update the fgArgInfo
2659 // with the folded tree that represents the final optimized argument nodes.
2661 // However if a range-check needs to be generated for any of these late
2662 // arguments we also need to "know" what the stack depth will be when we generate
2663 // code to branch to the throw range check failure block as that is part of the
2664 // GC information contract for that block.
2666 // Since the late arguments are evaluated last we have pushed all of the
2667 // other arguments on the stack before we evaluate these late arguments,
2668 // so we record the stack depth on the first morph call when lateArgsComputed
2669 // was false (via RecordStkLevel) and then retrieve that value here (via RetrieveStkLevel)
2671 unsigned callStkLevel = call->fgArgInfo->RetrieveStkLevel();
2672 fgPtrArgCntCur += callStkLevel;
2673 call->gtCallLateArgs = fgMorphTree(call->gtCallLateArgs)->AsArgList();
2674 flagsSummary |= call->gtCallLateArgs->gtFlags;
2675 fgPtrArgCntCur -= callStkLevel;
2676 assert(call->fgArgInfo != nullptr);
2677 call->fgArgInfo->RemorphReset();
2681 // First we need to count the args
2682 unsigned numArgs = 0;
2683 if (call->gtCallObjp)
2685 for (args = call->gtCallArgs; (args != nullptr); args = args->gtOp.gtOp2)
2691 // insert nonstandard args (outside the calling convention)
2693 #if !defined(LEGACY_BACKEND) && !defined(_TARGET_X86_)
2694 // TODO-X86-CQ: Currently RyuJIT/x86 passes args on the stack, so this is not needed.
2695 // If/when we change that, the following code needs to be changed to correctly support the (TBD) managed calling
2696 // convention for x86/SSE.
2697 if (!lateArgsComputed)
2699 if (call->IsUnmanaged() && !opts.ShouldUsePInvokeHelpers())
2701 assert(!call->gtCallCookie);
2702 // Add a conservative estimate of the stack size in a special parameter (r11) at the call site.
2703 // It will be used only on the intercepted-for-host code path to copy the arguments.
2705 GenTree* cns = new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, fgEstimateCallStackSize(call));
2706 call->gtCallArgs = gtNewListNode(cns, call->gtCallArgs);
2707 NonStandardArg nsa = {REG_PINVOKE_COOKIE_PARAM, cns};
2710 nonStandardArgs.Push(nsa);
2712 else if (call->IsVirtualStub() &&
2713 (call->gtCallType == CT_INDIRECT) &&
2714 !call->IsTailCallViaHelper())
2716 // indirect VSD stubs need the base of the indirection cell to be
2717 // passed in addition. At this point that is the value in gtCallAddr.
2718 // The actual call target will be derived from gtCallAddr in call
2721 // If it is a VSD call getting dispatched via tail call helper,
2722 // fgMorphTailCall() would materialize stub addr as an additional
2723 // parameter added to the original arg list and hence no need to
2724 // add as a non-standard arg.
2726 GenTree* arg = call->gtCallAddr;
2727 if (arg->OperIsLocal())
2729 arg = gtClone(arg, true);
2733 call->gtCallAddr = fgInsertCommaFormTemp(&arg);
2734 call->gtFlags |= GTF_ASG;
2736 noway_assert(arg != nullptr);
2738 // And push the stub address onto the list of arguments
2739 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
2742 NonStandardArg nsa = {REG_VIRTUAL_STUB_PARAM, arg};
2744 nonStandardArgs.Push(nsa);
2746 else if (call->gtCallType == CT_INDIRECT && call->gtCallCookie)
2748 assert(!call->IsUnmanaged());
2750 // put cookie into R11
2751 GenTree* arg = call->gtCallCookie;
2752 noway_assert(arg != nullptr);
2753 call->gtCallCookie = nullptr;
2755 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
2758 NonStandardArg nsa = {REG_PINVOKE_COOKIE_PARAM, arg};
2760 nonStandardArgs.Push(nsa);
2762 // put destination into R10
2763 arg = gtClone(call->gtCallAddr, true);
2764 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
2767 NonStandardArg nsa2 = {REG_PINVOKE_TARGET_PARAM, arg};
2768 nonStandardArgs.Push(nsa2);
2770 // finally change this call to a helper call
2771 call->gtCallType = CT_HELPER;
2772 call->gtCallMethHnd = eeFindHelper(CORINFO_HELP_PINVOKE_CALLI);
2775 #endif // !defined(LEGACY_BACKEND) && !defined(_TARGET_X86_)
2777 // Allocate the fgArgInfo for the call node;
2779 call->fgArgInfo = new (this, CMK_Unknown) fgArgInfo(this, call, numArgs);
2783 fgFixupStructReturn(call);
2785 /* First we morph the argument subtrees ('this' pointer, arguments, etc.).
2786 * During the first call to fgMorphArgs we also record the
2787 * information about late arguments we have in 'fgArgInfo'.
2788 * This information is used later to contruct the gtCallLateArgs */
2790 /* Process the 'this' argument value, if present */
2792 argx = call->gtCallObjp;
2796 argx = fgMorphTree(argx);
2797 call->gtCallObjp = argx;
2798 flagsSummary |= argx->gtFlags;
2800 assert(call->gtCallType == CT_USER_FUNC ||
2801 call->gtCallType == CT_INDIRECT);
2803 assert(argIndex == 0);
2805 /* We must fill in or update the argInfo table */
2807 if (!lateArgsComputed)
2809 assert(varTypeIsGC(call->gtCallObjp->gtType) ||
2810 (call->gtCallObjp->gtType == TYP_I_IMPL));
2812 /* this is a register argument - put it in the table */
2813 call->fgArgInfo->AddRegArg(argIndex,
2816 genMapIntRegArgNumToRegNum(intArgRegNum),
2819 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
2823 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
2828 /* this is a register argument - possibly update it in the table */
2829 call->fgArgInfo->RemorphRegArg(argIndex, argx, NULL, genMapIntRegArgNumToRegNum(intArgRegNum), 1, 1);
2831 // this can't be a struct.
2832 assert(argx->gtType != TYP_STRUCT);
2834 // FIXME: Issue #4025 Why do we need floating type for 'this' argument
2835 /* Increment the argument register count and argument index */
2836 if (!varTypeIsFloating(argx->gtType) || opts.compUseSoftFP)
2839 #if defined(_TARGET_AMD64_) && !defined(UNIX_AMD64_ABI)
2840 fltArgSkippedRegMask |= genMapArgNumToRegMask(fltArgRegNum, TYP_FLOAT);
2847 #if defined(_TARGET_AMD64_) && !defined(UNIX_AMD64_ABI)
2848 argSkippedRegMask |= genMapArgNumToRegMask(intArgRegNum, TYP_I_IMPL);
2857 // Compute the maximum number of arguments that can be passed in registers.
2858 // For X86 we handle the varargs and unmanaged calling conventions
2860 if (call->gtFlags & GTF_CALL_POP_ARGS)
2862 noway_assert(intArgRegNum < MAX_REG_ARG);
2863 // No more register arguments for varargs (CALL_POP_ARGS)
2864 maxRegArgs = intArgRegNum;
2866 // Add in the ret buff arg
2867 if (callHasRetBuffArg)
2872 if (call->IsUnmanaged())
2874 noway_assert(intArgRegNum == 0);
2876 if (call->gtCallMoreFlags & GTF_CALL_M_UNMGD_THISCALL)
2878 noway_assert(call->gtCallArgs->gtOp.gtOp1->TypeGet() == TYP_I_IMPL ||
2879 call->gtCallArgs->gtOp.gtOp1->TypeGet() == TYP_BYREF ||
2880 call->gtCallArgs->gtOp.gtOp1->gtOper == GT_NOP); // the arg was already morphed to a register (fgMorph called twice)
2888 // Add in the ret buff arg
2889 if (callHasRetBuffArg)
2892 #endif // INLINE_NDIRECT
2893 #endif // _TARGET_X86_
2895 /* Morph the user arguments */
2897 #if defined(_TARGET_ARM_)
2899 // The ARM ABI has a concept of back-filling of floating-point argument registers, according
2900 // to the "Procedure Call Standard for the ARM Architecture" document, especially
2901 // section 6.1.2.3 "Parameter passing". Back-filling is where floating-point argument N+1 can
2902 // appear in a lower-numbered register than floating point argument N. That is, argument
2903 // register allocation is not strictly increasing. To support this, we need to keep track of unused
2904 // floating-point argument registers that we can back-fill. We only support 4-byte float and
2905 // 8-byte double types, and one to four element HFAs composed of these types. With this, we will
2906 // only back-fill single registers, since there is no way with these types to create
2907 // an alignment hole greater than one register. However, there can be up to 3 back-fill slots
2908 // available (with 16 FP argument registers). Consider this code:
2910 // struct HFA { float x, y, z; }; // a three element HFA
2911 // void bar(float a1, // passed in f0
2912 // double a2, // passed in f2/f3; skip f1 for alignment
2913 // HFA a3, // passed in f4/f5/f6
2914 // double a4, // passed in f8/f9; skip f7 for alignment. NOTE: it doesn't fit in the f1 back-fill slot
2915 // HFA a5, // passed in f10/f11/f12
2916 // double a6, // passed in f14/f15; skip f13 for alignment. NOTE: it doesn't fit in the f1 or f7 back-fill slots
2917 // float a7, // passed in f1 (back-filled)
2918 // float a8, // passed in f7 (back-filled)
2919 // float a9, // passed in f13 (back-filled)
2920 // float a10) // passed on the stack in [OutArg+0]
2922 // Note that if we ever support FP types with larger alignment requirements, then there could
2923 // be more than single register back-fills.
2925 // Once we assign a floating-pointer register to the stack, they all must be on the stack.
2926 // See "Procedure Call Standard for the ARM Architecture", section 6.1.2.3, "The back-filling
2927 // continues only so long as no VFP CPRC has been allocated to a slot on the stack."
2928 // We set anyFloatStackArgs to true when a floating-point argument has been assigned to the stack
2929 // and prevent any additional floating-point arguments from going in registers.
2931 bool anyFloatStackArgs = false;
2933 #endif // _TARGET_ARM_
2935 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
2936 bool nonRegPassableStruct = false;
2937 SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR structDesc;
2938 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
2940 bool hasStructArgument = false; // @TODO-ARM64-UNIX: Eemove this bool during a future refactoring
2941 bool hasMultiregStructArgs = false;
2942 for (args = call->gtCallArgs; args; args = args->gtOp.gtOp2)
2944 GenTreePtr * parentArgx = &args->gtOp.gtOp1;
2946 #if FEATURE_MULTIREG_ARGS
2947 if (!hasStructArgument)
2949 hasStructArgument = varTypeIsStruct(args->gtOp.gtOp1);
2951 #endif // FEATURE_MULTIREG_ARGS
2953 argx = fgMorphTree(*parentArgx);
2955 flagsSummary |= argx->gtFlags;
2957 assert(args->IsList());
2958 assert(argx == args->Current());
2960 /* Change the node to TYP_I_IMPL so we don't report GC info
2961 * NOTE: We deferred this from the importer because of the inliner */
2963 if (argx->IsVarAddr())
2964 argx->gtType = TYP_I_IMPL;
2966 bool passUsingFloatRegs;
2967 unsigned argAlign = 1;
2970 var_types hfaType = GetHfaType(argx);
2971 bool isHfaArg = varTypeIsFloating(hfaType);
2972 #endif // _TARGET_ARM_
2975 CORINFO_CLASS_HANDLE copyBlkClass = NULL;
2976 bool isRegArg = false;
2978 fgArgTabEntryPtr argEntry = NULL;
2980 if (lateArgsComputed)
2982 argEntry = gtArgEntryByArgNum(call, argIndex);
2987 bool passUsingIntRegs;
2988 if (lateArgsComputed)
2990 passUsingFloatRegs = isValidFloatArgReg(argEntry->regNum);
2991 passUsingIntRegs = isValidIntArgReg(argEntry->regNum);
2995 passUsingFloatRegs = !callIsVararg && (isHfaArg || varTypeIsFloating(argx)) && !opts.compUseSoftFP;
2996 passUsingIntRegs = passUsingFloatRegs ? false : (intArgRegNum < MAX_REG_ARG);
2999 GenTreePtr curArg = argx;
3000 // If late args have already been computed, use the node in the argument table.
3001 if (argEntry != NULL && argEntry->isTmp)
3003 curArg = argEntry->node;
3006 // We don't use the "size" return value from InferOpSizeAlign().
3007 codeGen->InferOpSizeAlign(curArg, &argAlign);
3009 argAlign = roundUp(argAlign, TARGET_POINTER_SIZE);
3010 argAlign /= TARGET_POINTER_SIZE;
3014 if (passUsingFloatRegs)
3016 if (fltArgRegNum % 2 == 1)
3018 fltArgSkippedRegMask |= genMapArgNumToRegMask(fltArgRegNum, TYP_FLOAT);
3022 else if (passUsingIntRegs)
3024 if (intArgRegNum % 2 == 1)
3026 argSkippedRegMask |= genMapArgNumToRegMask(intArgRegNum, TYP_I_IMPL);
3031 if (argSlots % 2 == 1)
3037 #elif defined(_TARGET_ARM64_)
3039 // TODO-ARM64-NYI: HFA/HVA
3040 if (lateArgsComputed)
3042 passUsingFloatRegs = isValidFloatArgReg(argEntry->regNum);
3046 passUsingFloatRegs = !callIsVararg && varTypeIsFloating(argx);
3049 #elif defined(_TARGET_AMD64_)
3050 #if defined(UNIX_AMD64_ABI)
3051 if (lateArgsComputed)
3053 passUsingFloatRegs = isValidFloatArgReg(argEntry->regNum);
3057 passUsingFloatRegs = varTypeIsFloating(argx);
3059 bool passUsingIntRegs;
3060 passUsingIntRegs = passUsingFloatRegs ? false : (intArgRegNum < MAX_REG_ARG);
3061 #else // !UNIX_AMD64_ABI
3062 passUsingFloatRegs = varTypeIsFloating(argx);
3063 #endif // !UNIX_AMD64_ABI
3064 #elif defined(_TARGET_X86_)
3066 passUsingFloatRegs = false;
3069 #error Unsupported or unset target architecture
3072 bool isBackFilled = false;
3073 unsigned nextFltArgRegNum = fltArgRegNum; // This is the next floating-point argument register number to use
3074 var_types structBaseType = TYP_STRUCT;
3076 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3077 unsigned int structFloatRegs = 0;
3078 unsigned int structIntRegs = 0;
3079 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3080 bool isStructArg = varTypeIsStruct(argx);
3082 if (lateArgsComputed)
3084 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3085 // Get the struct description for the already completed struct argument.
3086 fgArgTabEntryPtr fgEntryPtr = gtArgEntryByNode(call, argx);
3087 assert(fgEntryPtr != nullptr);
3089 // As described in few other places, this can happen when the argx was morphed
3090 // into an arg setup node - COPYBLK. The COPYBLK has always a type of void.
3091 // In such case the fgArgTabEntry keeps track of whether the original node (before morphing)
3092 // was a struct and the struct classification.
3093 isStructArg = fgEntryPtr->isStruct;
3097 structDesc.CopyFrom(fgEntryPtr->structDesc);
3099 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3101 assert(argEntry != NULL);
3102 if (argEntry->IsBackFilled())
3105 size = argEntry->numRegs;
3106 nextFltArgRegNum = genMapFloatRegNumToRegArgNum(argEntry->regNum);
3108 isBackFilled = true;
3110 else if (argEntry->regNum == REG_STK)
3113 assert(argEntry->numRegs == 0);
3114 size = argEntry->numSlots;
3119 assert(argEntry->numRegs > 0);
3120 size = argEntry->numRegs + argEntry->numSlots;
3123 // This size has now been computed
3129 // Figure out the size of the argument. This is either in number of registers, or number of register-sized
3130 // stack slots, or both if the argument is split between the registers and the stack.
3133 if (argx->IsArgPlaceHolderNode() || (!isStructArg))
3135 #if defined(_TARGET_AMD64_)
3136 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3139 size = 1; // On AMD64, all primitives fit in a single (64-bit) 'slot'
3143 size = (unsigned)(roundUp(info.compCompHnd->getClassSize(argx->gtArgPlace.gtArgPlaceClsHnd), TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE;
3144 eeGetSystemVAmd64PassStructInRegisterDescriptor(argx->gtArgPlace.gtArgPlaceClsHnd, &structDesc);
3147 hasMultiregStructArgs = true;
3150 #else // !FEATURE_UNIX_AMD64_STRUCT_PASSING
3151 size = 1; // On AMD64, all primitives fit in a single (64-bit) 'slot'
3152 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3153 #elif defined(_TARGET_ARM64_)
3156 // Structs are either passed in 1 or 2 (64-bit) slots
3157 size = (unsigned)(roundUp(info.compCompHnd->getClassSize(argx->gtArgPlace.gtArgPlaceClsHnd), TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE;
3160 size = 1; // Large structs are passed by reference (to a copy)
3164 hasMultiregStructArgs = true;
3166 // Note that there are some additional rules for size=2 structs,
3167 // (i.e they cannot be split betwen registers and the stack)
3171 size = 1; // On ARM64, all primitive types fit in a single (64-bit) 'slot'
3173 #elif defined(_TARGET_ARM_)
3176 size = (unsigned)(roundUp(info.compCompHnd->getClassSize(argx->gtArgPlace.gtArgPlaceClsHnd), TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE;
3181 size = genTypeStSz(argx->gtType);
3183 #elif defined(_TARGET_X86_)
3184 size = genTypeStSz(argx->gtType);
3186 #error Unsupported or unset target architecture
3187 #endif // _TARGET_XXX_
3192 size = GetHfaSlots(argx);
3197 // We handle two opcodes: GT_MKREFANY and GT_OBJ
3198 if (argx->gtOper == GT_MKREFANY)
3200 if (varTypeIsStruct(argx))
3204 #ifdef _TARGET_AMD64_
3205 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3206 if (varTypeIsStruct(argx))
3208 size = info.compCompHnd->getClassSize(impGetRefAnyClass());
3209 unsigned roundupSize = (unsigned)roundUp(size, TARGET_POINTER_SIZE);
3210 size = roundupSize / TARGET_POINTER_SIZE;
3211 eeGetSystemVAmd64PassStructInRegisterDescriptor(impGetRefAnyClass(), &structDesc);
3214 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3225 GenTreePtr argObj = argx;
3226 GenTreePtr* parentOfArgObj = parentArgx;
3228 assert(args->IsList());
3229 assert(argx == args->Current());
3231 /* The GT_OBJ may be be a child of a GT_COMMA */
3232 while (argObj->gtOper == GT_COMMA)
3234 parentOfArgObj = &argObj->gtOp.gtOp2;
3235 argObj = argObj->gtOp.gtOp2;
3238 if (argObj->gtOper != GT_OBJ)
3239 BADCODE("illegal argument tree in fgMorphArgs");
3241 CORINFO_CLASS_HANDLE objClass = argObj->gtObj.gtClass;
3242 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3243 eeGetSystemVAmd64PassStructInRegisterDescriptor(objClass, &structDesc);
3244 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3246 unsigned originalSize = info.compCompHnd->getClassSize(objClass);
3247 originalSize = (originalSize == 0 ? TARGET_POINTER_SIZE : originalSize);
3248 unsigned roundupSize = (unsigned)roundUp(originalSize, TARGET_POINTER_SIZE);
3249 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3250 // On System V OS-es a struct is never passed by reference.
3251 // It is either passed by value on the stack or in registers.
3252 bool passStructInRegisters = false;
3253 #else // !FEATURE_UNIX_AMD64_STRUCT_PASSING
3254 bool passStructByRef = false;
3255 #endif // !FEATURE_UNIX_AMD64_STRUCT_PASSING
3257 #ifndef _TARGET_X86_
3258 #ifndef FEATURE_UNIX_AMD64_STRUCT_PASSING
3259 // Check for struct argument with size 1, 2, 4 or 8 bytes
3260 // As we can optimize these by turning them into a GT_IND of the correct type
3261 if ((originalSize > TARGET_POINTER_SIZE) || ((originalSize & (originalSize - 1)) != 0))
3262 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3264 // Normalize 'size' to the number of pointer sized items
3265 // 'size' is the number of register slots that we will use to pass the argument
3266 size = roundupSize / TARGET_POINTER_SIZE;
3267 #if defined(_TARGET_AMD64_)
3268 #ifndef FEATURE_UNIX_AMD64_STRUCT_PASSING
3269 size = 1; // This must be copied to a temp and passed by address
3270 passStructByRef = true;
3271 copyBlkClass = objClass;
3272 #else // FEATURE_UNIX_AMD64_STRUCT_PASSING
3273 if (!structDesc.passedInRegisters)
3275 passStructInRegisters = false;
3276 copyBlkClass = NO_CLASS_HANDLE;
3280 // The objClass is used to materialize the struct on stack.
3281 passStructInRegisters = true;
3282 copyBlkClass = objClass;
3284 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3285 #elif defined(_TARGET_ARM64_)
3288 size = 1; // This must be copied to a temp and passed by address
3289 passStructByRef = true;
3290 copyBlkClass = objClass;
3295 // If we're passing a promoted struct local var,
3296 // we may need to skip some registers due to alignment; record those.
3297 GenTreePtr lclVar = fgIsIndirOfAddrOfLocal(argObj);
3300 LclVarDsc* varDsc = &lvaTable[lclVar->gtLclVarCommon.gtLclNum];
3301 if (varDsc->lvPromoted)
3303 assert(argObj->OperGet() == GT_OBJ);
3304 if (lvaGetPromotionType(varDsc) == PROMOTION_TYPE_INDEPENDENT)
3306 fgAddSkippedRegsInPromotedStructArg(varDsc, intArgRegNum, &argSkippedRegMask);
3310 #endif // _TARGET_ARM_
3312 #ifndef FEATURE_UNIX_AMD64_STRUCT_PASSING
3315 // change our GT_OBJ into a GT_IND of the correct type.
3316 structBaseType = argOrReturnTypeForStruct(originalSize, objClass, false /* forReturn */);
3317 // We've already ensured above that size is a power of 2, and less than pointer size.
3318 noway_assert(structBaseType != TYP_UNKNOWN);
3320 argObj->ChangeOper(GT_IND);
3322 // Now see if we can fold *(&X) into X
3323 if (argObj->gtOp.gtOp1->gtOper == GT_ADDR)
3325 GenTreePtr temp = argObj->gtOp.gtOp1->gtOp.gtOp1;
3327 // Keep the DONT_CSE flag in sync
3328 // (as the addr always marks it for its op1)
3329 temp->gtFlags &= ~GTF_DONT_CSE;
3330 temp->gtFlags |= (argObj->gtFlags & GTF_DONT_CSE);
3331 DEBUG_DESTROY_NODE(argObj->gtOp.gtOp1); // GT_ADDR
3332 DEBUG_DESTROY_NODE(argObj); // GT_IND
3335 *parentOfArgObj = temp;
3337 // If the OBJ had been the top level node, we've now changed argx.
3338 if (parentOfArgObj == parentArgx)
3341 if (argObj->gtOper == GT_LCL_VAR)
3343 unsigned lclNum = argObj->gtLclVarCommon.gtLclNum;
3344 LclVarDsc * varDsc = &lvaTable[lclNum];
3346 if (varDsc->lvPromoted)
3348 if (varDsc->lvFieldCnt == 1)
3350 // get the first and only promoted field
3351 LclVarDsc * fieldVarDsc = &lvaTable[varDsc->lvFieldLclStart];
3352 if (genTypeSize(fieldVarDsc->TypeGet()) >= originalSize)
3354 // we will use the first and only promoted field
3355 argObj->gtLclVarCommon.SetLclNum(varDsc->lvFieldLclStart);
3357 if (varTypeCanReg(fieldVarDsc->TypeGet()) && (genTypeSize(fieldVarDsc->TypeGet()) == originalSize))
3359 // Just use the existing field's type
3360 argObj->gtType = fieldVarDsc->TypeGet();
3364 // Can't use the existing field's type, so use GT_LCL_FLD to swizzle
3366 argObj->ChangeOper(GT_LCL_FLD);
3367 argObj->gtType = structBaseType;
3369 assert(varTypeCanReg(argObj->TypeGet()));
3370 assert(copyBlkClass == NO_CLASS_HANDLE);
3374 // use GT_LCL_FLD to swizzle the single field struct to a new type
3375 lvaSetVarDoNotEnregister(lclNum DEBUGARG(DNER_LocalField));
3376 argObj->ChangeOper(GT_LCL_FLD);
3377 argObj->gtType = structBaseType;
3382 // The struct fits into a single register, but it has been promoted into its
3383 // constituent fields, and so we have to re-assemble it
3384 copyBlkClass = objClass;
3386 // Alignment constraints may cause us not to use (to "skip") some argument registers.
3387 // Add those, if any, to the skipped (int) arg reg mask.
3388 fgAddSkippedRegsInPromotedStructArg(varDsc, intArgRegNum, &argSkippedRegMask);
3389 #endif // _TARGET_ARM_
3392 else if (!varTypeIsIntegralOrI(varDsc->TypeGet()))
3394 // Not a promoted struct, so just swizzle the type by using GT_LCL_FLD
3395 argObj->ChangeOper(GT_LCL_FLD);
3396 argObj->gtType = structBaseType;
3401 // Not a GT_LCL_VAR, so we can just change the type on the node
3402 argObj->gtType = structBaseType;
3404 assert(varTypeCanReg(argObj->TypeGet()) ||
3405 ((copyBlkClass != NO_CLASS_HANDLE) && varTypeIsIntegral(structBaseType)));
3409 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3411 #endif // not _TARGET_X86_
3412 // We still have a struct unless we converted the GT_OBJ into a GT_IND above...
3413 if ((structBaseType == TYP_STRUCT) &&
3414 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3415 !passStructInRegisters
3416 #else // !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3418 #endif // !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3421 // if the valuetype size is not a multiple of sizeof(void*),
3422 // we must copyblk to a temp before doing the obj to avoid
3423 // the obj reading memory past the end of the valuetype
3424 #if defined(_TARGET_X86_) && !defined(LEGACY_BACKEND)
3425 // TODO-X86-CQ: [1091733] Revisit for small structs, we should use push instruction
3426 copyBlkClass = objClass;
3427 size = roundupSize / TARGET_POINTER_SIZE; // Normalize size to number of pointer sized items
3428 #else // !defined(_TARGET_X86_) || defined(LEGACY_BACKEND)
3429 if (roundupSize > originalSize)
3431 copyBlkClass = objClass;
3433 // There are a few special cases where we can omit using a CopyBlk
3434 // where we normally would need to use one.
3436 GenTreePtr objAddr = argObj->gtObj.gtOp1;
3437 if (objAddr->gtOper == GT_ADDR)
3439 // exception : no need to use CopyBlk if the valuetype is on the stack
3440 if (objAddr->gtFlags & GTF_ADDR_ONSTACK)
3442 copyBlkClass = NO_CLASS_HANDLE;
3444 // exception : no need to use CopyBlk if the valuetype is already a struct local
3445 else if (objAddr->gtOp.gtOp1->gtOper == GT_LCL_VAR)
3447 copyBlkClass = NO_CLASS_HANDLE;
3452 size = roundupSize / TARGET_POINTER_SIZE; // Normalize size to number of pointer sized items
3453 #endif // !defined(_TARGET_X86_) || defined(LEGACY_BACKEND)
3458 hasMultiregStructArgs = true;
3462 // The 'size' value has now must have been set. (the original value of zero is an invalid value)
3466 // Figure out if the argument will be passed in a register.
3468 bool passedInRegisters = true;
3469 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3470 passedInRegisters = !isStructArg;
3471 if (!passedInRegisters)
3473 if (structDesc.passedInRegisters)
3475 passedInRegisters = true;
3479 passedInRegisters = false;
3484 if (passedInRegisters && isRegParamType(genActualType(argx->TypeGet())))
3487 if (passUsingFloatRegs)
3489 // First, see if it can be back-filled
3490 if (!anyFloatStackArgs && // Is it legal to back-fill? (We haven't put any FP args on the stack yet)
3491 (fltArgSkippedRegMask != RBM_NONE) && // Is there an available back-fill slot?
3492 (size == 1)) // The size to back-fill is one float register
3494 // Back-fill the register.
3495 isBackFilled = true;
3496 regMaskTP backFillBitMask = genFindLowestBit(fltArgSkippedRegMask);
3497 fltArgSkippedRegMask &= ~backFillBitMask; // Remove the back-filled register(s) from the skipped mask
3498 nextFltArgRegNum = genMapFloatRegNumToRegArgNum(genRegNumFromMask(backFillBitMask));
3499 assert(nextFltArgRegNum < MAX_FLOAT_REG_ARG);
3502 // Does the entire float, double, or HFA fit in the FP arg registers?
3503 // Check if the last register needed is still in the argument register range.
3504 isRegArg = (nextFltArgRegNum + size - 1) < MAX_FLOAT_REG_ARG;
3508 anyFloatStackArgs = true;
3513 isRegArg = intArgRegNum < MAX_REG_ARG;
3515 #elif _TARGET_ARM64_
3516 if (passUsingFloatRegs)
3518 // Check if the last register needed is still in the fp argument register range.
3519 isRegArg = (nextFltArgRegNum + (size - 1)) < MAX_FLOAT_REG_ARG;
3523 // Check if the last register needed is still in the int argument register range.
3524 isRegArg = (intArgRegNum + (size - 1)) < maxRegArgs;
3526 #else // not _TARGET_ARM_ or _TARGET_ARM64_
3528 #if defined(UNIX_AMD64_ABI)
3530 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3531 // Here a struct can be passed in register following the classifications of its members and size.
3532 // Now make sure there are actually enough registers to do so.
3535 for (unsigned int i = 0; i < structDesc.eightByteCount; i++)
3537 if (structDesc.IsIntegralSlot(i))
3541 else if (structDesc.IsSseSlot(i))
3547 if (((nextFltArgRegNum + structFloatRegs) > MAX_FLOAT_REG_ARG) ||
3548 ((intArgRegNum + structIntRegs) > MAX_REG_ARG))
3551 nonRegPassableStruct = true;
3556 nonRegPassableStruct = false;
3560 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3562 if (passUsingFloatRegs)
3564 isRegArg = nextFltArgRegNum < MAX_FLOAT_REG_ARG;
3568 isRegArg = intArgRegNum < MAX_REG_ARG;
3571 #else // !defined(UNIX_AMD64_ABI)
3572 isRegArg = (intArgRegNum+(size-1)) < maxRegArgs;
3573 #endif // !defined(UNIX_AMD64_ABI)
3574 #endif // _TARGET_ARM_
3580 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3581 nonRegPassableStruct = true;
3582 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3587 // Now we know if the argument goes in registers or not and how big it is,
3588 // whether we had to just compute it or this is a re-morph call and we looked it up.
3593 // If we ever allocate a floating point argument to the stack, then all
3594 // subsequent HFA/float/double arguments go on the stack.
3595 if (!isRegArg && passUsingFloatRegs)
3597 for (; fltArgRegNum < MAX_FLOAT_REG_ARG; ++fltArgRegNum)
3599 fltArgSkippedRegMask |= genMapArgNumToRegMask(fltArgRegNum, TYP_FLOAT);
3603 // If we think we're going to split a struct between integer registers and the stack, check to
3604 // see if we've already assigned a floating-point arg to the stack.
3605 if (isRegArg && // We decided above to use a register for the argument
3606 !passUsingFloatRegs && // We're using integer registers
3607 (intArgRegNum + size > MAX_REG_ARG) && // We're going to split a struct type onto registers and stack
3608 anyFloatStackArgs) // We've already used the stack for a floating-point argument
3610 isRegArg = false; // Change our mind; don't pass this struct partially in registers
3612 // Skip the rest of the integer argument registers
3613 for (; intArgRegNum < MAX_REG_ARG; ++intArgRegNum)
3615 argSkippedRegMask |= genMapArgNumToRegMask(intArgRegNum, TYP_I_IMPL);
3619 #endif // _TARGET_ARM_
3622 regNumber nextRegNum = REG_STK;
3623 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3624 regNumber nextOtherRegNum = REG_STK;
3626 if (isStructArg && structDesc.passedInRegisters)
3628 // It is a struct passed in registers. Assign the next available register.
3629 unsigned int curIntReg = intArgRegNum;
3630 unsigned int curFloatReg = nextFltArgRegNum;
3631 for (unsigned int i = 0; i < structDesc.eightByteCount; i++)
3633 if (structDesc.IsIntegralSlot(i))
3637 nextRegNum = genMapIntRegArgNumToRegNum(curIntReg);
3639 // For non-completed args the counters are incremented already
3640 // in the !lateArgsComputed above.
3641 if (lateArgsComputed)
3648 nextOtherRegNum = genMapIntRegArgNumToRegNum(curIntReg);
3650 if (lateArgsComputed)
3657 assert(false && "fgMorphArgs Invalid index for int classification.");
3662 else if (structDesc.IsSseSlot(i))
3666 nextRegNum = genMapFloatRegArgNumToRegNum(curFloatReg);
3668 if (lateArgsComputed)
3675 nextOtherRegNum = genMapFloatRegArgNumToRegNum(curFloatReg);
3677 if (lateArgsComputed)
3684 assert(false && "fgMorphArgs Invalid index for SSE classification.");
3692 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3694 // fill in or update the argInfo table
3695 nextRegNum = passUsingFloatRegs ? genMapFloatRegArgNumToRegNum(nextFltArgRegNum) : genMapIntRegArgNumToRegNum(intArgRegNum);
3698 #ifdef _TARGET_AMD64_
3699 #ifndef FEATURE_UNIX_AMD64_STRUCT_PASSING
3704 #ifndef LEGACY_BACKEND
3705 // If there are nonstandard args (outside the calling convention) they were inserted above
3706 // and noted them in a table so we can recognize them here and build their argInfo.
3708 // They should not affect the placement of any other args or stack space required.
3709 // Example: on AMD64 R10 and R11 are used for indirect VSD (generic interface) and cookie calls.
3710 bool nonStandardFound = false;
3711 for (int i=0; i<nonStandardArgs.Height(); i++)
3713 hasNonStandardArg = true;
3714 if (argx == nonStandardArgs.Index(i).node)
3716 fgArgTabEntry* argEntry = call->fgArgInfo->AddRegArg(argIndex,
3719 nonStandardArgs.Index(i).reg,
3722 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3726 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3728 argEntry->isNonStandard = true;
3730 nonStandardFound = true;
3734 if (nonStandardFound)
3736 #endif // !LEGACY_BACKEND
3738 if (!lateArgsComputed)
3740 // This is a register argument - put it in the table
3741 fgArgTabEntryPtr newArg = call->fgArgInfo->AddRegArg(argIndex,
3747 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3751 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3753 (void)newArg; //prevent "unused variable" error from GCC
3755 newArg->SetIsHfaRegArg(passUsingFloatRegs && isHfaArg); // Note that an HFA is passed in int regs for varargs
3756 newArg->SetIsBackFilled(isBackFilled);
3757 #endif // _TARGET_ARM_
3761 // This is a register argument - possibly update it in the table
3762 fgArgTabEntryPtr entry = call->fgArgInfo->RemorphRegArg(argIndex, argx, args, nextRegNum, size, argAlign);
3763 if (entry->isNonStandard)
3770 // Setup the next argRegNum value
3773 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3776 intArgRegNum += structIntRegs;
3777 fltArgRegNum += structFloatRegs;
3780 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3782 if (passUsingFloatRegs)
3784 fltArgRegNum += size;
3786 #if defined(_TARGET_AMD64_) && !defined(UNIX_AMD64_ABI)
3787 argSkippedRegMask |= genMapArgNumToRegMask(intArgRegNum, TYP_I_IMPL);
3788 intArgRegNum = min(intArgRegNum + size, MAX_REG_ARG);
3789 #endif // _TARGET_AMD64_
3791 if (fltArgRegNum > MAX_FLOAT_REG_ARG)
3793 // This indicates a partial enregistration of a struct type
3794 assert(varTypeIsStruct(argx));
3795 unsigned numRegsPartial = size - (fltArgRegNum - MAX_FLOAT_REG_ARG);
3796 assert((unsigned char)numRegsPartial == numRegsPartial);
3797 call->fgArgInfo->SplitArg(argIndex, numRegsPartial, size - numRegsPartial);
3798 fltArgRegNum = MAX_FLOAT_REG_ARG;
3800 #endif // _TARGET_ARM_
3804 intArgRegNum += size;
3806 #if defined(_TARGET_AMD64_) && !defined(UNIX_AMD64_ABI)
3807 fltArgSkippedRegMask |= genMapArgNumToRegMask(fltArgRegNum, TYP_DOUBLE);
3808 fltArgRegNum = min(fltArgRegNum + size, MAX_FLOAT_REG_ARG);
3809 #endif // _TARGET_AMD64_
3811 if (intArgRegNum > MAX_REG_ARG)
3813 // This indicates a partial enregistration of a struct type
3814 assert((isStructArg) || argx->OperIsCopyBlkOp() ||
3815 (argx->gtOper == GT_COMMA && (args->gtFlags & GTF_ASG)));
3816 unsigned numRegsPartial = size - (intArgRegNum - MAX_REG_ARG);
3817 assert((unsigned char)numRegsPartial == numRegsPartial);
3818 call->fgArgInfo->SplitArg(argIndex, numRegsPartial, size - numRegsPartial);
3819 intArgRegNum = MAX_REG_ARG;
3820 fgPtrArgCntCur += size - numRegsPartial;
3822 #endif // _TARGET_ARM_
3827 else // We have an argument that is not passed in a register
3829 fgPtrArgCntCur += size;
3831 // If the register arguments have not been determined then we must fill in the argInfo
3833 if (!lateArgsComputed)
3835 // This is a stack argument - put it in the table
3836 call->fgArgInfo->AddStkArg(argIndex, argx, args, size, argAlign FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(isStructArg));
3841 // This is a stack argument - possibly update it in the table
3842 call->fgArgInfo->RemorphStkArg(argIndex, argx, args, size, argAlign);
3845 if (copyBlkClass != NO_CLASS_HANDLE)
3847 noway_assert(!lateArgsComputed);
3848 fgMakeOutgoingStructArgCopy(call, args, argIndex, copyBlkClass FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(&structDesc));
3851 #ifndef LEGACY_BACKEND
3852 if (argx->gtOper == GT_MKREFANY)
3854 NYI_X86("MKREFANY");
3856 // 'Lower' the MKREFANY tree and insert it.
3857 noway_assert(!lateArgsComputed);
3860 // Here we don't need unsafe value cls check since the addr of temp is used only in mkrefany
3861 unsigned tmp = lvaGrabTemp(true DEBUGARG("by-value mkrefany struct argument"));
3862 lvaSetStruct(tmp, impGetRefAnyClass(), false);
3865 // Build the mkrefany as a comma node:
3866 // (tmp.ptr=argx),(tmp.type=handle)
3867 GenTreeLclFld* destPtrSlot = gtNewLclFldNode(tmp, TYP_I_IMPL, offsetof(CORINFO_RefAny, dataPtr));
3868 GenTreeLclFld* destTypeSlot = gtNewLclFldNode(tmp, TYP_I_IMPL, offsetof(CORINFO_RefAny, type));
3869 destPtrSlot->gtFieldSeq = GetFieldSeqStore()->CreateSingleton(GetRefanyDataField());
3870 destPtrSlot->gtFlags |= GTF_VAR_DEF;
3871 destTypeSlot->gtFieldSeq = GetFieldSeqStore()->CreateSingleton(GetRefanyTypeField());
3872 destTypeSlot->gtFlags |= GTF_VAR_DEF;
3874 GenTreePtr asgPtrSlot = gtNewAssignNode(destPtrSlot, argx->gtOp.gtOp1);
3875 GenTreePtr asgTypeSlot = gtNewAssignNode(destTypeSlot, argx->gtOp.gtOp2);
3876 GenTreePtr asg = gtNewOperNode(GT_COMMA, TYP_VOID, asgPtrSlot, asgTypeSlot);
3878 // Change the expression to "(tmp=val)"
3879 args->gtOp.gtOp1 = asg;
3881 // EvalArgsToTemps will cause tmp to actually get loaded as the argument
3882 call->fgArgInfo->EvalToTmp(argIndex, tmp, asg);
3883 lvaSetVarAddrExposed(tmp);
3885 #endif // !LEGACY_BACKEND
3888 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3889 if (nonRegPassableStruct)
3891 nonRegPassedStructSlots += size;
3894 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3898 } // end foreach argument loop
3900 if (!lateArgsComputed)
3902 call->fgArgInfo->ArgsComplete();
3904 call->gtCallRegUsedMask = genIntAllRegArgMask(intArgRegNum) & ~argSkippedRegMask;
3905 if (fltArgRegNum > 0)
3907 #if defined(_TARGET_ARM_) || defined(_TARGET_AMD64_)
3908 call->gtCallRegUsedMask |= genFltAllRegArgMask(fltArgRegNum) & ~fltArgSkippedRegMask;
3913 if (call->gtCallArgs)
3915 UpdateGT_LISTFlags(call->gtCallArgs);
3918 /* Process the function address, if indirect call */
3920 if (call->gtCallType == CT_INDIRECT)
3921 call->gtCallAddr = fgMorphTree(call->gtCallAddr);
3923 call->fgArgInfo->RecordStkLevel(fgPtrArgCntCur);
3925 if ((call->gtCallType == CT_INDIRECT) && (call->gtCallCookie != NULL))
3928 /* Remember the maximum value we ever see */
3930 if (fgPtrArgCntMax < fgPtrArgCntCur)
3931 fgPtrArgCntMax = fgPtrArgCntCur;
3933 /* The call will pop all the arguments we pushed */
3935 fgPtrArgCntCur = genPtrArgCntSav;
3937 #if FEATURE_FIXED_OUT_ARGS
3939 // Update the outgoing argument size.
3940 // If the call is a fast tail call, it will setup its arguments in incoming arg
3941 // area instead of the out-going arg area. Therefore, don't consider fast tail
3942 // calls to update lvaOutgoingArgSpaceSize.
3943 if (!call->IsFastTailCall())
3945 unsigned preallocatedArgCount;
3947 #if defined(_TARGET_ARMARCH_)
3948 // First slots go in registers only, no stack needed.
3949 // TODO-ARMArch-CQ: This calculation is only accurate for integer arguments,
3950 // and ignores floating point args (it is overly conservative in that case).
3951 if (argSlots <= MAX_REG_ARG)
3953 preallocatedArgCount = 0;
3957 preallocatedArgCount = argSlots - MAX_REG_ARG;
3959 #elif defined(UNIX_AMD64_ABI)
3960 opts.compNeedToAlignFrame = true;
3961 // First slots go in registers only, no stack needed.
3962 // TODO-Amd64-Unix-CQ This calculation is only accurate for integer arguments,
3963 // and ignores floating point args (it is overly conservative in that case).
3964 if (argSlots <= MAX_REG_ARG)
3966 preallocatedArgCount = nonRegPassedStructSlots;
3970 preallocatedArgCount = argSlots + nonRegPassedStructSlots - MAX_REG_ARG;
3972 #elif defined(_TARGET_AMD64_)
3973 preallocatedArgCount = max(4, argSlots);
3975 #error Unsupported or unset target architecture
3977 if (preallocatedArgCount * REGSIZE_BYTES > lvaOutgoingArgSpaceSize)
3979 lvaOutgoingArgSpaceSize = preallocatedArgCount * REGSIZE_BYTES;
3981 // If a function has localloc, we will need to move the outgoing arg space when the
3982 // localloc happens. When we do this, we need to maintain stack alignment. To avoid
3983 // leaving alignment-related holes when doing this move, make sure the outgoing
3984 // argument space size is a multiple of the stack alignment by aligning up to the next
3985 // stack alignment boundary.
3986 if (compLocallocUsed)
3988 lvaOutgoingArgSpaceSize = (lvaOutgoingArgSpaceSize + (STACK_ALIGN - 1)) & ~(STACK_ALIGN - 1);
3992 #endif // FEATURE_FIXED_OUT_ARGS
3994 /* Update the 'side effect' flags value for the call */
3996 call->gtFlags |= (flagsSummary & GTF_ALL_EFFECT);
3998 // If the register arguments have already been determined
3999 // or we have no register arguments then we are done.
4001 bool needEvalArgsToTemps = true;
4003 if (lateArgsComputed || (intArgRegNum == 0 && fltArgRegNum == 0 && !hasNonStandardArg && !hasStructArgument))
4005 needEvalArgsToTemps = false;
4008 if (needEvalArgsToTemps)
4010 // This is the first time that we morph this call AND it has register arguments.
4011 // Follow into the code below and do the 'defer or eval to temp' analysis.
4013 call->fgArgInfo->SortArgs();
4015 call->fgArgInfo->EvalArgsToTemps();
4017 // We may have updated the arguments
4018 if (call->gtCallArgs)
4020 UpdateGT_LISTFlags(call->gtCallArgs);
4024 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
4026 // Rewrite the struct args to be passed by value on stack or in registers.
4027 fgMorphSystemVStructArgs(call, hasStructArgument);
4029 #else // !FEATURE_UNIX_AMD64_STRUCT_PASSING
4031 // In the future we can migrate UNIX_AMD64 to use this
4032 // method instead of fgMorphSystemVStructArgs
4033 #ifndef LEGACY_BACKEND
4034 // We only build GT_LISTs for MultiReg structs for the RyuJIT backend
4035 if (hasMultiregStructArgs)
4037 fgMorphMultiregStructArgs(call);
4039 #endif // LEGACY_BACKEND
4041 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
4046 #pragma warning(pop)
4049 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
4050 // fgMorphSystemVStructArgs:
4051 // Rewrite the struct args to be passed by value on stack or in registers.
4054 // call: The call whose arguments need to be morphed.
4055 // hasStructArgument: Whether this call has struct arguments.
4057 void Compiler::fgMorphSystemVStructArgs(GenTreeCall* call, bool hasStructArgument)
4059 unsigned flagsSummary = 0;
4063 if (hasStructArgument)
4065 fgArgInfoPtr allArgInfo = call->fgArgInfo;
4067 for (args = call->gtCallArgs; args != nullptr; args = args->gtOp.gtOp2)
4069 // For late arguments the arg tree that is overridden is in the gtCallLateArgs list.
4070 // For such late args the gtCallArgList contains the setup arg node (evaluating the arg.)
4071 // The tree from the gtCallLateArgs list is passed to the callee. The fgArgEntry node contains the mapping
4072 // between the nodes in both lists. If the arg is not a late arg, the fgArgEntry->node points to itself,
4073 // otherwise points to the list in the late args list.
4074 bool isLateArg = (args->gtOp.gtOp1->gtFlags & GTF_LATE_ARG) != 0;
4075 fgArgTabEntryPtr fgEntryPtr = gtArgEntryByNode(call, args->gtOp.gtOp1);
4076 assert(fgEntryPtr != nullptr);
4077 GenTreePtr argx = fgEntryPtr->node;
4078 GenTreePtr lateList = nullptr;
4079 GenTreePtr lateNode = nullptr;
4083 for (GenTreePtr list = call->gtCallLateArgs; list; list = list->MoveNext())
4085 assert(list->IsList());
4087 GenTreePtr argNode = list->Current();
4088 if (argx == argNode)
4095 assert(lateList != nullptr && lateNode != nullptr);
4097 GenTreePtr arg = argx;
4098 bool argListCreated = false;
4100 var_types type = arg->TypeGet();
4102 if (varTypeIsStruct(type))
4104 var_types originalType = type;
4105 // If we have already processed the arg...
4106 if (arg->OperGet() == GT_LIST && varTypeIsStruct(arg))
4111 // If already OBJ it is set properly already.
4112 if (arg->OperGet() == GT_OBJ)
4114 assert(!fgEntryPtr->structDesc.passedInRegisters);
4119 arg->OperGet() == GT_LCL_VAR ||
4120 arg->OperGet() == GT_LCL_FLD ||
4121 (arg->OperGet() == GT_ADDR &&
4122 (arg->gtOp.gtOp1->OperGet() == GT_LCL_FLD ||
4123 arg->gtOp.gtOp1->OperGet() == GT_LCL_VAR)));
4125 GenTreeLclVarCommon* lclCommon = arg->OperGet() == GT_ADDR ?
4126 arg->gtOp.gtOp1->AsLclVarCommon() : arg->AsLclVarCommon();
4127 if (fgEntryPtr->structDesc.passedInRegisters)
4129 if (fgEntryPtr->structDesc.eightByteCount == 1)
4131 // Change the type and below the code will change the LclVar to a LCL_FLD
4132 type = GetTypeFromClassificationAndSizes(fgEntryPtr->structDesc.eightByteClassifications[0], fgEntryPtr->structDesc.eightByteSizes[0]);
4134 else if (fgEntryPtr->structDesc.eightByteCount == 2)
4136 // Create LCL_FLD for each eightbyte.
4137 argListCreated = true;
4139 // Second eightbyte.
4140 GenTreeLclFld* newLclField = new(this, GT_LCL_FLD) GenTreeLclFld(
4141 GetTypeFromClassificationAndSizes(
4142 fgEntryPtr->structDesc.eightByteClassifications[1],
4143 fgEntryPtr->structDesc.eightByteSizes[1]),
4144 lclCommon->gtLclNum,
4145 fgEntryPtr->structDesc.eightByteOffsets[1]);
4146 // Note this should actually be: secondNode = gtNewArgList(newLclField)
4147 GenTreeArgList* secondNode = gtNewListNode(newLclField, nullptr);
4148 secondNode->gtType = originalType; // Preserve the type. It is a special case.
4149 newLclField->gtFieldSeq = FieldSeqStore::NotAField();
4152 arg->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
4153 arg->gtType = GetTypeFromClassificationAndSizes(
4154 fgEntryPtr->structDesc.eightByteClassifications[0],
4155 fgEntryPtr->structDesc.eightByteSizes[0]);
4156 arg = gtNewListNode(arg, secondNode);
4157 arg->gtType = type; // Preserve the type. It is a special case.
4161 assert(false && "More than two eightbytes detected for CLR."); // No more than two eightbytes for the CLR.
4165 // If we didn't change the type of the struct, it means
4166 // its classification doesn't support to be passed directly through a
4167 // register, so we need to pass a pointer to the destination where
4168 // where we copied the struct to.
4169 if (!argListCreated)
4171 if (fgEntryPtr->structDesc.passedInRegisters)
4177 // Make sure this is an addr node.
4178 if (arg->OperGet() != GT_ADDR && arg->OperGet() != GT_LCL_VAR_ADDR)
4180 arg = gtNewOperNode(GT_ADDR, TYP_I_IMPL, arg);
4183 assert(arg->OperGet() == GT_ADDR || arg->OperGet() == GT_LCL_VAR_ADDR);
4185 // Create an Obj of the temp to use it as a call argument.
4186 arg = new (this, GT_OBJ) GenTreeObj(originalType, arg, lvaGetStruct(lclCommon->gtLclNum));
4187 arg->gtFlags |= GTF_EXCEPT;
4188 flagsSummary |= GTF_EXCEPT;
4195 bool isLateArg = (args->gtOp.gtOp1->gtFlags & GTF_LATE_ARG) != 0;
4196 fgArgTabEntryPtr fgEntryPtr = gtArgEntryByNode(call, args->gtOp.gtOp1);
4197 assert(fgEntryPtr != nullptr);
4198 GenTreePtr argx = fgEntryPtr->node;
4199 GenTreePtr lateList = nullptr;
4200 GenTreePtr lateNode = nullptr;
4203 for (GenTreePtr list = call->gtCallLateArgs; list; list = list->MoveNext())
4205 assert(list->IsList());
4207 GenTreePtr argNode = list->Current();
4208 if (argx == argNode)
4215 assert(lateList != nullptr && lateNode != nullptr);
4218 fgEntryPtr->node = arg;
4221 lateList->gtOp.gtOp1 = arg;
4225 args->gtOp.gtOp1 = arg;
4232 call->gtFlags |= (flagsSummary & GTF_ALL_EFFECT);
4234 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
4236 //-----------------------------------------------------------------------------
4237 // fgMorphMultiregStructArgs: Locate the TYP_STRUCT arguments and
4238 // call fgMorphMultiregStructArg on each of them.
4241 // call: a GenTreeCall node that has one or more TYP_STRUCT arguments
4244 // We only call fgMorphMultiregStructArg for the register passed TYP_STRUCT arguments.
4245 // The call to fgMorphMultiregStructArg will mutate the argument into the GT_LIST form
4246 // whicj is only used for register arguments.
4247 // If this method fails to find any TYP_STRUCT arguments it will assert.
4249 void Compiler::fgMorphMultiregStructArgs(GenTreeCall* call)
4253 bool foundStructArg = false;
4254 unsigned initialFlags = call->gtFlags;
4255 unsigned flagsSummary = 0;
4256 fgArgInfoPtr allArgInfo = call->fgArgInfo;
4258 // Currently only ARM64 is using this method to morph the MultiReg struct args
4259 // in the future AMD64_UNIX and for HFAs ARM32, will also use this method
4262 NYI_ARM("fgMorphMultiregStructArgs");
4265 assert("Logic error: no MultiregStructArgs for X86");
4267 #ifdef _TARGET_AMD64_
4268 #if defined(UNIX_AMD64_ABI)
4269 NYI_AMD64("fgMorphMultiregStructArgs (UNIX ABI)");
4272 assert("Logic error: no MultiregStructArgs for Windows X64 ABI");
4275 for (args = call->gtCallArgs; args != nullptr; args = args->gtOp.gtOp2)
4277 // For late arguments the arg tree that is overridden is in the gtCallLateArgs list.
4278 // For such late args the gtCallArgList contains the setup arg node (evaluating the arg.)
4279 // The tree from the gtCallLateArgs list is passed to the callee. The fgArgEntry node contains the mapping
4280 // between the nodes in both lists. If the arg is not a late arg, the fgArgEntry->node points to itself,
4281 // otherwise points to the list in the late args list.
4282 bool isLateArg = (args->gtOp.gtOp1->gtFlags & GTF_LATE_ARG) != 0;
4283 fgArgTabEntryPtr fgEntryPtr = gtArgEntryByNode(call, args->gtOp.gtOp1);
4284 assert(fgEntryPtr != nullptr);
4285 GenTreePtr argx = fgEntryPtr->node;
4286 GenTreePtr lateList = nullptr;
4287 GenTreePtr lateNode = nullptr;
4291 for (GenTreePtr list = call->gtCallLateArgs; list; list = list->MoveNext())
4293 assert(list->IsList());
4295 GenTreePtr argNode = list->Current();
4296 if (argx == argNode)
4303 assert(lateList != nullptr && lateNode != nullptr);
4306 GenTreePtr arg = argx;
4308 if (arg->TypeGet() == TYP_STRUCT)
4310 foundStructArg = true;
4312 arg = fgMorphMultiregStructArg(arg, fgEntryPtr);
4314 // Did we replace 'argx' with a new tree?
4317 fgEntryPtr->node = arg; // Record the new value for the arg in the fgEntryPtr->node
4319 // link the new arg node into either the late arg list or the gtCallArgs list
4322 lateList->gtOp.gtOp1 = arg;
4326 args->gtOp.gtOp1 = arg;
4332 // We should only call this method when we actually have one or more multireg struct args
4333 assert(foundStructArg);
4336 call->gtFlags |= (flagsSummary & GTF_ALL_EFFECT);
4340 //-----------------------------------------------------------------------------
4341 // fgMorphMultiregStructArg: Given a multireg TYP_STRUCT arg from a call argument list
4342 // Morph the argument into a set of GT_LIST nodes.
4345 // arg - A GenTree node containing a TYP_STRUCT arg that
4346 // is to be passed in multiple registers
4348 // arg must be a GT_OBJ or GT_LCL_VAR or GT_LCL_FLD of TYP_STRUCT that is suitable
4349 // for passing in multiple registers.
4350 // If arg is a LclVar we check if it is struct promoted and has the right number of fields
4351 // and if they are at the appropriate offsets we will use the struct promted fields
4352 // in the GT_LIST nodes that we create.
4353 // If we have a GT_LCL_VAR that isn't struct promoted or doesn't meet the requirements
4354 // we will use a set of GT_LCL_FLDs nodes to access the various portions of the struct
4355 // this also forces the struct to be stack allocated into the local frame.
4356 // For the GT_OBJ case will clone the address expression and generate two (or more)
4358 // Currently the implementation only handles ARM64 and will NYI for other architectures.
4359 // And for ARM64 we do not ye handle HFA arguments, so only 16-byte struct sizes are supported.
4361 GenTreePtr Compiler::fgMorphMultiregStructArg(GenTreePtr arg, fgArgTabEntryPtr fgEntryPtr)
4363 GenTreeArgList* newArg = nullptr;
4364 assert(arg->TypeGet() == TYP_STRUCT);
4365 GenTreePtr argValue = arg;
4367 #ifndef _TARGET_ARM64_
4368 NYI("fgMorphMultiregStructArg non-ARM64 implementation");
4371 // If we have a GT_OBJ of a GT_ADDR then
4372 // we set argValue to the child node ofthe GT_ADDR
4373 if (arg->OperGet() == GT_OBJ)
4375 GenTreePtr argAddr = arg->gtOp.gtOp1;
4377 if (argAddr->OperGet() == GT_ADDR)
4379 argValue = argAddr->gtOp.gtOp1;
4382 // We should still have a TYP_STRUCT
4383 assert(argValue->TypeGet() == TYP_STRUCT);
4385 // Are we passing a struct LclVar?
4387 if (argValue->OperGet() == GT_LCL_VAR)
4389 GenTreeLclVarCommon* varNode = argValue->AsLclVarCommon();
4390 unsigned varNum = varNode->gtLclNum;
4391 assert(varNum < lvaCount);
4392 LclVarDsc* varDsc = &lvaTable[varNum];
4394 // At this point any TYP_STRUCT LclVar must be a 16-byte pass by value argument
4395 assert(varDsc->lvSize() == 2 * TARGET_POINTER_SIZE);
4397 const BYTE * gcPtrs = varDsc->lvGcLayout;
4399 var_types type0 = getJitGCType(gcPtrs[0]);
4400 var_types type1 = getJitGCType(gcPtrs[1]);
4402 varDsc->lvIsMultiRegArgOrRet = true;
4404 // Is this LclVar a promoted struct with exactly two fields?
4405 if ((varDsc->lvPromoted) && (varDsc->lvFieldCnt == 2))
4407 // See if we have two promoted fields that start at offset 0 and 8?
4408 unsigned loVarNum = lvaGetFieldLocal(varDsc, 0);
4409 unsigned hiVarNum = lvaGetFieldLocal(varDsc, TARGET_POINTER_SIZE);
4411 // Did we find the promoted fields at the necessary offsets?
4412 if ((loVarNum != BAD_VAR_NUM) && (hiVarNum != BAD_VAR_NUM))
4414 LclVarDsc* loVarDsc = &lvaTable[loVarNum];
4415 LclVarDsc* hiVarDsc = &lvaTable[hiVarNum];
4417 var_types loType = loVarDsc->lvType;
4418 var_types hiType = hiVarDsc->lvType;
4420 if (varTypeIsFloating(loType) || varTypeIsFloating(hiType))
4422 // TODO-LSRA - It currently doesn't support the passing of floating point LCL_VARS in the integer registers
4423 // So for now we will use GT_LCLFLD's to pass this struct (it won't be enregistered)
4425 JITDUMP("Multireg struct V%02u will be passed using GT_LCLFLD because it has float fields.\n", varNum);
4427 // we call lvaSetVarDoNotEnregister and do the proper transformation below.
4432 // We can use the struct promoted field as the two arguments
4434 GenTreePtr loLclVar = gtNewLclvNode(loVarNum, loType, loVarNum);
4435 GenTreePtr hiLclVar = gtNewLclvNode(hiVarNum, hiType, hiVarNum);
4437 // Create a new tree for 'arg'
4438 // replace the existing LDOBJ(ADDR(LCLVAR))
4439 // with a LIST(LCLVAR-LO, LIST(LCLVAR-HI, nullptr))
4441 newArg = gtNewListNode(loLclVar, gtNewArgList(hiLclVar));
4446 // Check if we couldn't transform the LDOBJ(ADDR(LCLVAR)) into a struct promoted GT_LIST above
4447 if (newArg == nullptr)
4450 // We weren't able to pass this LclVar using it's struct promted fields
4452 // So instead we will create a list of GT_LCL_FLDs nodes to pass this struct
4454 lvaSetVarDoNotEnregister(varNum DEBUG_ARG(DNER_LocalField));
4456 // If this is going in the register area, we transform it here into a GT_LIST of LCLFLD's
4457 // If this is going in the outgoing arg area, it will be transformed later
4459 if (fgEntryPtr->regNum != REG_STK)
4461 GenTreeLclVarCommon* varNode = argValue->AsLclVarCommon();
4462 unsigned varNum = varNode->gtLclNum;
4463 assert(varNum < lvaCount);
4464 LclVarDsc* varDsc = &lvaTable[varNum];
4466 GenTreePtr loLclFld = gtNewLclFldNode(varNum, type0, 0);
4467 GenTreePtr hiLclFld = gtNewLclFldNode(varNum, type1, TARGET_POINTER_SIZE);
4469 // Create a new tree for 'arg'
4470 // replace the existing LDOBJ(ADDR(LCLVAR))
4471 // with a LIST(LCLFLD-LO, LIST(LCLFLD-HI, nullptr))
4473 newArg = gtNewListNode(loLclFld, gtNewArgList(hiLclFld));
4477 // Check if we already created a replacement newArg above
4478 if (newArg == nullptr)
4480 if (fgEntryPtr->regNum == REG_STK)
4482 // We leave this stack passed argument alone
4487 // Are we passing a GT_LCL_FLD which contain a 16-byte struct inside it?
4489 if (argValue->OperGet() == GT_LCL_FLD)
4491 GenTreeLclVarCommon* varNode = argValue->AsLclVarCommon();
4492 unsigned varNum = varNode->gtLclNum;
4493 assert(varNum < lvaCount);
4494 LclVarDsc* varDsc = &lvaTable[varNum];
4496 unsigned baseOffset = argValue->gtLclFld.gtLclOffs;
4497 unsigned baseIndex = baseOffset / TARGET_POINTER_SIZE;
4498 unsigned requiredSize = baseOffset + (2 * TARGET_POINTER_SIZE);
4500 // The allocated size of our LocalVar must be at least as big as requiredSize
4501 assert(varDsc->lvSize() >= requiredSize);
4503 const BYTE * gcPtrs = varDsc->lvGcLayout;
4505 var_types type0 = getJitGCType(gcPtrs[baseIndex+0]);
4506 var_types type1 = getJitGCType(gcPtrs[baseIndex+1]);
4509 // We create a list of two GT_LCL_FLDs nodes to pass this struct
4511 lvaSetVarDoNotEnregister(varNum DEBUG_ARG(DNER_LocalField));
4513 GenTreePtr loLclFld = gtNewLclFldNode(varNum, type0, baseOffset);
4514 GenTreePtr hiLclFld = gtNewLclFldNode(varNum, type1, baseOffset + TARGET_POINTER_SIZE);
4516 // Create a new tree for 'arg'
4517 // replace the existing LDOBJ(ADDR(LCLVAR))
4518 // with a LIST(LCLFLD-LO, LIST(LCLFLD-HI, nullptr))
4520 newArg = gtNewListNode(loLclFld, gtNewArgList(hiLclFld));
4522 // Are we passing a GT_OBJ struct?
4524 else if (argValue->OperGet() == GT_OBJ)
4526 GenTreeObj* argObj = argValue->AsObj();
4527 CORINFO_CLASS_HANDLE objClass = argObj->gtClass;
4529 int structSize = info.compCompHnd->getClassSize(objClass);
4530 assert(structSize <= 2 * TARGET_POINTER_SIZE);
4531 BYTE gcPtrs[2] = { TYPE_GC_NONE, TYPE_GC_NONE };
4532 info.compCompHnd->getClassGClayout(objClass, &gcPtrs[0]);
4534 var_types type0 = getJitGCType(gcPtrs[0]);
4535 var_types type1 = getJitGCType(gcPtrs[1]);
4537 GenTreePtr baseAddr = argObj->gtOp1;
4538 GenTreePtr baseAddrDup = gtCloneExpr(baseAddr);
4539 noway_assert(baseAddrDup != nullptr);
4541 var_types addrType = baseAddr->TypeGet();
4542 GenTreePtr loAddr = baseAddr;
4543 GenTreePtr hiAddr = gtNewOperNode(GT_ADD, addrType, baseAddrDup, gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL));
4544 GenTreePtr loValue = gtNewOperNode(GT_IND, type0, loAddr);
4545 GenTreePtr hiValue = gtNewOperNode(GT_IND, type1, hiAddr);
4547 // Create a new tree for 'arg'
4548 // replace the existing LDOBJ(EXPR)
4549 // with a LIST(IND(EXPR), LIST(IND(EXPR+8), nullptr))
4551 newArg = gtNewListNode(loValue, gtNewArgList(hiValue));
4554 // If we reach here we should have set newArg to something
4555 if (newArg == nullptr)
4558 gtDispTree(argValue);
4560 assert(!"Missing case in fgMorphMultiregStructArg");
4566 printf("fgMorphMultiregStructArg created tree:\n");
4571 arg = newArg; // consider calling fgMorphTree(newArg);
4575 // Make a copy of a struct variable if necessary, to pass to a callee.
4576 // returns: tree that computes address of the outgoing arg
4578 Compiler::fgMakeOutgoingStructArgCopy(GenTreeCall* call,
4581 CORINFO_CLASS_HANDLE copyBlkClass
4582 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(const SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR* const structDescPtr))
4584 GenTree* argx = args->Current();
4585 noway_assert(argx->gtOper != GT_MKREFANY);
4586 // See if we need to insert a copy at all
4587 // Case 1: don't need a copy if it is the last use of a local. We can't determine that all of the time
4588 // but if there is only one use and no loops, the use must be last.
4589 if (argx->gtOper == GT_OBJ)
4591 GenTree* lcl = argx->gtOp.gtOp1;
4592 if (lcl->OperIsLocal())
4594 unsigned varNum = lcl->AsLclVarCommon()->GetLclNum();
4595 if (lvaIsImplicitByRefLocal(varNum))
4597 LclVarDsc* varDsc = &lvaTable[varNum];
4598 if (varDsc->lvRefCnt == 1 && !fgMightHaveLoop())
4600 varDsc->lvRefCnt = 0;
4601 args->gtOp.gtOp1 = lcl;
4602 fgArgTabEntryPtr fp = Compiler::gtArgEntryByNode(call, argx);
4605 JITDUMP("did not have to make outgoing copy for V%2d", varNum);
4606 varDsc->lvRefCnt = 0;
4611 varDsc->lvRefCnt = 0;
4617 if (fgOutgoingArgTemps == nullptr)
4618 fgOutgoingArgTemps = hashBv::Create(this);
4623 // Attempt to find a local we have already used for an outgoing struct and reuse it.
4624 // We do not reuse within a statement.
4625 if (!opts.MinOpts())
4628 FOREACH_HBV_BIT_SET(lclNum, fgOutgoingArgTemps)
4630 LclVarDsc* varDsc = &lvaTable[lclNum];
4631 if (typeInfo::AreEquivalent(varDsc->lvVerTypeInfo, typeInfo(TI_STRUCT, copyBlkClass))
4632 && !fgCurrentlyInUseArgTemps->testBit(lclNum))
4634 tmp = (unsigned) lclNum;
4636 JITDUMP("reusing outgoing struct arg");
4643 // Create the CopyBlk tree and insert it.
4647 // Here We don't need unsafe value cls check, since the addr of this temp is used only in copyblk.
4648 tmp = lvaGrabTemp(true DEBUGARG("by-value struct argument"));
4649 lvaSetStruct(tmp, copyBlkClass, false);
4650 fgOutgoingArgTemps->setBit(tmp);
4653 fgCurrentlyInUseArgTemps->setBit(tmp);
4655 // TYP_SIMD structs should not be enregistered, since ABI requires it to be
4656 // allocated on stack and address of it needs to be passed.
4657 if (lclVarIsSIMDType(tmp))
4659 lvaSetVarDoNotEnregister(tmp DEBUGARG(DNER_IsStruct));
4662 // Create a reference to the temp
4663 GenTreePtr dest = gtNewLclvNode(tmp, lvaTable[tmp].lvType);
4664 dest->gtFlags |= (GTF_DONT_CSE | GTF_VAR_DEF); // This is a def of the local, "entire" by construction.
4665 dest = gtNewOperNode(GT_ADDR, TYP_BYREF, dest);
4666 lvaTable[tmp].incRefCnts(compCurBB->getBBWeight(this), this);
4669 if (argx->gtOper == GT_OBJ)
4671 src = argx->gtOp.gtOp1;
4675 argx->gtFlags |= GTF_DONT_CSE;
4676 src = gtNewOperNode(GT_ADDR, TYP_BYREF, argx);
4679 // Copy the valuetype to the temp
4680 GenTreePtr copyBlk = gtNewCpObjNode(dest, src, copyBlkClass, false);
4681 copyBlk = fgMorphCopyBlock(copyBlk);
4683 #if FEATURE_FIXED_OUT_ARGS
4685 // Do the copy early, and evalute the temp later (see EvalArgsToTemps)
4686 // When on Unix create LCL_FLD for structs passed in more than one registers. See fgMakeTmpArgNode
4687 GenTreePtr arg = copyBlk;
4689 #else // FEATURE_FIXED_OUT_ARGS
4691 // Structs are always on the stack, and thus never need temps
4692 // so we have to put the copy and temp all into one expression
4693 GenTreePtr arg = fgMakeTmpArgNode(
4695 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(structDescPtr->passedInRegisters));
4697 // Change the expression to "(tmp=val),tmp"
4698 arg = gtNewOperNode(GT_COMMA, arg->TypeGet(), copyBlk, arg);
4700 #endif // FEATURE_FIXED_OUT_ARGS
4702 args->gtOp.gtOp1 = arg;
4703 call->fgArgInfo->EvalToTmp(argIndex, tmp, arg);
4709 // See declaration for specification comment.
4710 void Compiler::fgAddSkippedRegsInPromotedStructArg(LclVarDsc* varDsc,
4711 unsigned firstArgRegNum,
4712 regMaskTP* pArgSkippedRegMask)
4714 assert(varDsc->lvPromoted);
4715 // There's no way to do these calculations without breaking abstraction and assuming that
4716 // integer register arguments are consecutive ints. They are on ARM.
4718 // To start, figure out what register contains the last byte of the first argument.
4719 LclVarDsc* firstFldVarDsc = &lvaTable[varDsc->lvFieldLclStart];
4720 unsigned lastFldRegOfLastByte = (firstFldVarDsc->lvFldOffset + firstFldVarDsc->lvExactSize - 1) / TARGET_POINTER_SIZE;;
4722 // Now we're keeping track of the register that the last field ended in; see what registers
4723 // subsequent fields start in, and whether any are skipped.
4724 // (We assume here the invariant that the fields are sorted in offset order.)
4725 for (unsigned fldVarOffset = 1; fldVarOffset < varDsc->lvFieldCnt; fldVarOffset++)
4727 unsigned fldVarNum = varDsc->lvFieldLclStart + fldVarOffset;
4728 LclVarDsc* fldVarDsc = &lvaTable[fldVarNum];
4729 unsigned fldRegOffset = fldVarDsc->lvFldOffset / TARGET_POINTER_SIZE;
4730 assert(fldRegOffset >= lastFldRegOfLastByte); // Assuming sorted fields.
4731 // This loop should enumerate the offsets of any registers skipped.
4732 // Find what reg contains the last byte:
4733 // And start at the first register after that. If that isn't the first reg of the current
4734 for (unsigned skippedRegOffsets = lastFldRegOfLastByte + 1; skippedRegOffsets < fldRegOffset; skippedRegOffsets++)
4736 // If the register number would not be an arg reg, we're done.
4737 if (firstArgRegNum + skippedRegOffsets >= MAX_REG_ARG) return;
4738 *pArgSkippedRegMask |= genRegMask(regNumber(firstArgRegNum + skippedRegOffsets));
4740 lastFldRegOfLastByte = (fldVarDsc->lvFldOffset + fldVarDsc->lvExactSize - 1) / TARGET_POINTER_SIZE;
4744 #endif // _TARGET_ARM_
4747 /*****************************************************************************
4749 * The companion to impFixupCallStructReturn. Now that the importer is done
4750 * and we no longer care as much about the declared return type, change to
4751 * precomputed native return type (at least for architectures that don't
4752 * always use return buffers for structs).
4755 void Compiler::fgFixupStructReturn(GenTreePtr callNode)
4757 GenTreeCall* call = callNode->AsCall();
4758 bool callHasRetBuffArg = call->HasRetBufArg();
4760 if (!callHasRetBuffArg && varTypeIsStruct(call))
4763 if (call->gtCall.IsVarargs() || !IsHfa(call))
4766 // Now that we are past the importer, re-type this node so the register predictor does
4768 call->gtType = genActualType((var_types)call->gtCall.gtReturnType);
4773 // Either we don't have a struct now or if struct, then it is HFA returned in regs.
4774 assert(!varTypeIsStruct(call) || (IsHfa(call) && !callHasRetBuffArg));
4775 #elif defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
4776 // Either we don't have a struct now or if struct, then it is a struct returned in regs or in return buffer.
4777 assert(!varTypeIsStruct(call) || call->HasMultiRegRetVal() || callHasRetBuffArg);
4779 // No more struct returns
4780 assert(call->TypeGet() != TYP_STRUCT);
4783 #if !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
4784 // If it was a struct return, it has been transformed into a call
4785 // with a return buffer (that returns TYP_VOID) or into a return
4786 // of a primitive/enregisterable type
4787 assert(!callHasRetBuffArg || (call->TypeGet() == TYP_VOID));
4792 /*****************************************************************************
4794 * A little helper used to rearrange nested commutative operations. The
4795 * effect is that nested associative, commutative operations are transformed
4796 * into a 'left-deep' tree, i.e. into something like this:
4798 * (((a op b) op c) op d) op...
4803 void Compiler::fgMoveOpsLeft(GenTreePtr tree)
4811 op1 = tree->gtOp.gtOp1;
4812 op2 = tree->gtOp.gtOp2;
4813 oper = tree->OperGet();
4815 noway_assert(GenTree::OperIsCommutative(oper));
4816 noway_assert(oper == GT_ADD || oper == GT_XOR || oper == GT_OR ||
4817 oper == GT_AND || oper == GT_MUL);
4818 noway_assert(!varTypeIsFloating(tree->TypeGet()) || !opts.genFPorder);
4819 noway_assert(oper == op2->gtOper);
4821 // Commutativity doesn't hold if overflow checks are needed
4823 if (tree->gtOverflowEx() || op2->gtOverflowEx())
4826 if (gtIsActiveCSE_Candidate(op2))
4828 // If we have marked op2 as a CSE candidate,
4829 // we can't perform a commutative reordering
4830 // because any value numbers that we computed for op2
4831 // will be incorrect after performing a commutative reordering
4836 if (oper == GT_MUL && (op2->gtFlags & GTF_MUL_64RSLT))
4839 // Check for GTF_ADDRMODE_NO_CSE flag on add/mul Binary Operators
4840 if ( ((oper == GT_ADD) || (oper == GT_MUL))
4841 && ((tree->gtFlags & GTF_ADDRMODE_NO_CSE) != 0) )
4846 if ( (tree->gtFlags | op2->gtFlags) & GTF_BOOLEAN )
4848 // We could deal with this, but we were always broken and just hit the assert
4849 // below regarding flags, which means it's not frequent, so will just bail out.
4854 noway_assert(!tree->gtOverflowEx() && !op2->gtOverflowEx());
4856 GenTreePtr ad1 = op2->gtOp.gtOp1;
4857 GenTreePtr ad2 = op2->gtOp.gtOp2;
4859 // Compiler::optOptimizeBools() can create GT_OR of two GC pointers yeilding a GT_INT
4860 // We can not reorder such GT_OR trees
4862 if (varTypeIsGC(ad1->TypeGet()) != varTypeIsGC(op2->TypeGet()))
4865 /* Change "(x op (y op z))" to "(x op y) op z" */
4866 /* ie. "(op1 op (ad1 op ad2))" to "(op1 op ad1) op ad2" */
4868 GenTreePtr new_op1 = op2;
4870 new_op1->gtOp.gtOp1 = op1;
4871 new_op1->gtOp.gtOp2 = ad1;
4873 /* Change the flags. */
4875 // Make sure we arent throwing away any flags
4876 noway_assert((new_op1->gtFlags & ~(
4878 GTF_DONT_CSE | // It is ok that new_op1->gtFlags contains GTF_DONT_CSE flag.
4879 GTF_REVERSE_OPS | // The reverse ops flag also can be set, it will be re-calculated
4880 GTF_NODE_MASK|GTF_ALL_EFFECT|GTF_UNSIGNED)) == 0);
4882 new_op1->gtFlags = (new_op1->gtFlags & (GTF_NODE_MASK | GTF_DONT_CSE)) | // Make sure we propagate GTF_DONT_CSE flag.
4883 (op1->gtFlags & GTF_ALL_EFFECT) |
4884 (ad1->gtFlags & GTF_ALL_EFFECT);
4886 /* Retype new_op1 if it has not/become a GC ptr. */
4888 if (varTypeIsGC(op1->TypeGet()))
4890 noway_assert((varTypeIsGC(tree->TypeGet()) && op2->TypeGet() == TYP_I_IMPL && oper == GT_ADD) || // byref(ref + (int+int))
4891 (varTypeIsI (tree->TypeGet()) && op2->TypeGet() == TYP_I_IMPL && oper == GT_OR)); // int(gcref | int(gcref|intval))
4893 new_op1->gtType = tree->gtType;
4895 else if (varTypeIsGC(ad2->TypeGet()))
4897 // Neither ad1 nor op1 are GC. So new_op1 isnt either
4898 noway_assert(op1->gtType == TYP_I_IMPL && ad1->gtType == TYP_I_IMPL);
4899 new_op1->gtType = TYP_I_IMPL;
4902 // If new_op1 is a new expression. Assign it a new unique value number.
4903 // vnStore is null before the ValueNumber phase has run
4904 if (vnStore != nullptr)
4906 // We can only keep the old value number on new_op1 if both op1 and ad2
4907 // have the same non-NoVN value numbers. Since op is commutative, comparing
4908 // only ad2 and op1 is enough.
4909 if ((op1->gtVNPair.GetLiberal() == ValueNumStore::NoVN) ||
4910 (ad2->gtVNPair.GetLiberal() == ValueNumStore::NoVN) ||
4911 (ad2->gtVNPair.GetLiberal() != op1->gtVNPair.GetLiberal()))
4913 new_op1->gtVNPair.SetBoth(vnStore->VNForExpr(new_op1->TypeGet()));
4917 tree->gtOp.gtOp1 = new_op1;
4918 tree->gtOp.gtOp2 = ad2;
4920 /* If 'new_op1' is now the same nested op, process it recursively */
4922 if ((ad1->gtOper == oper) && !ad1->gtOverflowEx())
4923 fgMoveOpsLeft(new_op1);
4925 /* If 'ad2' is now the same nested op, process it
4926 * Instead of recursion, we set up op1 and op2 for the next loop.
4932 while ((op2->gtOper == oper) && !op2->gtOverflowEx());
4939 /*****************************************************************************/
4941 void Compiler::fgSetRngChkTarget(GenTreePtr tree,
4944 GenTreeBoundsChk* bndsChk = nullptr;
4945 SpecialCodeKind kind = SCK_RNGCHK_FAIL;
4948 if ((tree->gtOper == GT_ARR_BOUNDS_CHECK) || (tree->gtOper == GT_SIMD_CHK))
4949 #else // FEATURE_SIMD
4950 if (tree->gtOper == GT_ARR_BOUNDS_CHECK)
4951 #endif // FEATURE_SIMD
4953 bndsChk = tree->AsBoundsChk();
4954 kind = tree->gtBoundsChk.gtThrowKind;
4958 noway_assert((tree->gtOper == GT_ARR_ELEM) || (tree->gtOper == GT_ARR_INDEX));
4962 unsigned callStkDepth = fgPtrArgCntCur;
4964 // only x86 pushes args
4965 const unsigned callStkDepth = 0;
4972 // we need to initialize this field
4973 if (fgGlobalMorph && bndsChk != nullptr)
4975 bndsChk->gtStkDepth = callStkDepth;
4979 if (!opts.compDbgCode)
4981 if (delay || compIsForInlining())
4983 /* We delay this until after loop-oriented range check
4984 analysis. For now we merely store the current stack
4985 level in the tree node.
4987 if (bndsChk != nullptr)
4989 noway_assert(!bndsChk->gtIndRngFailBB || previousCompletedPhase >= PHASE_OPTIMIZE_LOOPS);
4990 bndsChk->gtStkDepth = callStkDepth;
4995 /* Create/find the appropriate "range-fail" label */
4997 // fgPtrArgCntCur is only valid for global morph or if we walk full stmt.
4998 noway_assert((bndsChk != nullptr) || fgGlobalMorph);
5000 unsigned stkDepth = (bndsChk != nullptr) ? bndsChk->gtStkDepth
5003 BasicBlock * rngErrBlk = fgRngChkTarget(compCurBB, stkDepth, kind);
5005 /* Add the label to the indirection node */
5007 if (bndsChk != nullptr)
5009 bndsChk->gtIndRngFailBB = gtNewCodeRef(rngErrBlk);
5015 /*****************************************************************************
5017 * Expand a GT_INDEX node and fully morph the child operands
5019 * The orginal GT_INDEX node is bashed into the GT_IND node that accesses
5020 * the array element. We expand the GT_INDEX node into a larger tree that
5021 * evaluates the array base and index. The simplest expansion is a GT_COMMA
5022 * with a GT_ARR_BOUND_CHK and a GT_IND with a GTF_INX_RNGCHK flag.
5023 * For complex array or index expressions one or more GT_COMMA assignments
5024 * are inserted so that we only evaluate the array or index expressions once.
5026 * The fully expanded tree is then morphed. This causes gtFoldExpr to
5027 * perform local constant prop and reorder the constants in the tree and
5030 * We then parse the resulting array element expression in order to locate
5031 * and label the constants and variables that occur in the tree.
5034 const int MAX_ARR_COMPLEXITY = 4;
5035 const int MAX_INDEX_COMPLEXITY = 4;
5037 GenTreePtr Compiler::fgMorphArrayIndex(GenTreePtr tree)
5039 noway_assert(tree->gtOper == GT_INDEX);
5040 GenTreeIndex* asIndex = tree->AsIndex();
5042 var_types elemTyp = tree->TypeGet();
5043 unsigned elemSize = tree->gtIndex.gtIndElemSize;
5044 CORINFO_CLASS_HANDLE elemStructType = tree->gtIndex.gtStructElemClass;
5046 noway_assert(elemTyp != TYP_STRUCT || elemStructType != nullptr);
5049 if (featureSIMD && varTypeIsStruct(elemTyp) && elemSize <= getSIMDVectorRegisterByteLength())
5051 // If this is a SIMD type, this is the point at which we lose the type information,
5052 // so we need to set the correct type on the GT_IND.
5053 // (We don't care about the base type here, so we only check, but don't retain, the return value).
5054 unsigned simdElemSize = 0;
5055 if (getBaseTypeAndSizeOfSIMDType(elemStructType, &simdElemSize) != TYP_UNKNOWN)
5057 assert(simdElemSize == elemSize);
5058 elemTyp = getSIMDTypeForSize(elemSize);
5059 // This is the new type of the node.
5060 tree->gtType = elemTyp;
5061 // Now set elemStructType to null so that we don't confuse value numbering.
5062 elemStructType = nullptr;
5065 #endif // FEATURE_SIMD
5067 GenTreePtr arrRef = asIndex->Arr();
5068 GenTreePtr index = asIndex->Index();
5070 // Set up the the array length's offset into lenOffs
5071 // And the the first element's offset into elemOffs
5074 if (tree->gtFlags & GTF_INX_STRING_LAYOUT)
5076 lenOffs = offsetof(CORINFO_String, stringLen);
5077 elemOffs = offsetof(CORINFO_String, chars);
5078 tree->gtFlags &= ~GTF_INX_STRING_LAYOUT; // Clear this flag as it is used for GTF_IND_VOLATILE
5080 else if (tree->gtFlags & GTF_INX_REFARR_LAYOUT)
5082 lenOffs = offsetof(CORINFO_RefArray, length);
5083 elemOffs = eeGetEEInfo()->offsetOfObjArrayData;
5085 else // We have a standard array
5087 lenOffs = offsetof(CORINFO_Array, length);
5088 elemOffs = offsetof(CORINFO_Array, u1Elems);
5091 bool chkd = ((tree->gtFlags & GTF_INX_RNGCHK) != 0); // if false, range checking will be disabled
5092 bool nCSE = ((tree->gtFlags & GTF_DONT_CSE ) != 0);
5094 GenTreePtr arrRefDefn = nullptr; // non-NULL if we need to allocate a temp for the arrRef expression
5095 GenTreePtr indexDefn = nullptr; // non-NULL if we need to allocate a temp for the index expression
5096 GenTreePtr bndsChk = nullptr;
5098 // If we're doing range checking, introduce a GT_ARR_BOUNDS_CHECK node for the address.
5101 GenTreePtr arrRef2 = nullptr; // The second copy will be used in array address expression
5102 GenTreePtr index2 = nullptr;
5104 // If the arrRef expression involves an assignment, a call or reads from global memory,
5105 // then we *must* allocate a temporary in which to "localize" those values,
5106 // to ensure that the same values are used in the bounds check and the actual
5108 // Also we allocate the temporary when the arrRef is sufficiently complex/expensive.
5110 if ((arrRef->gtFlags & (GTF_ASG|GTF_CALL|GTF_GLOB_REF)) || gtComplexityExceeds(&arrRef, MAX_ARR_COMPLEXITY))
5112 unsigned arrRefTmpNum = lvaGrabTemp(true DEBUGARG("arr expr"));
5113 arrRefDefn = gtNewTempAssign(arrRefTmpNum, arrRef);
5114 arrRef = gtNewLclvNode(arrRefTmpNum, arrRef->TypeGet());
5115 arrRef2 = gtNewLclvNode(arrRefTmpNum, arrRef->TypeGet());
5119 arrRef2 = gtCloneExpr(arrRef);
5120 noway_assert(arrRef2 != nullptr);
5123 // If the index expression involves an assignment, a call or reads from global memory,
5124 // we *must* allocate a temporary in which to "localize" those values,
5125 // to ensure that the same values are used in the bounds check and the actual
5127 // Also we allocate the temporary when the index is sufficiently complex/expensive.
5129 if ((index->gtFlags & (GTF_ASG|GTF_CALL|GTF_GLOB_REF)) || gtComplexityExceeds(&index, MAX_ARR_COMPLEXITY))
5131 unsigned indexTmpNum = lvaGrabTemp(true DEBUGARG("arr expr"));
5132 indexDefn = gtNewTempAssign(indexTmpNum, index);
5133 index = gtNewLclvNode(indexTmpNum, index->TypeGet());
5134 index2 = gtNewLclvNode(indexTmpNum, index->TypeGet());
5138 index2 = gtCloneExpr(index);
5139 noway_assert(index2 != nullptr);
5142 // Next introduce a GT_ARR_BOUNDS_CHECK node
5143 var_types bndsChkType = TYP_INT; // By default, try to use 32-bit comparison for array bounds check.
5145 #ifdef _TARGET_64BIT_
5146 // The CLI Spec allows an array to be indexed by either an int32 or a native int. In the case
5147 // of a 64 bit architecture this means the array index can potentially be a TYP_LONG, so for this case,
5148 // the comparison will have to be widen to 64 bits.
5149 if (index->TypeGet() == TYP_I_IMPL)
5151 bndsChkType = TYP_I_IMPL;
5153 #endif // _TARGET_64BIT_
5155 GenTree* arrLen = new (this, GT_ARR_LENGTH) GenTreeArrLen(TYP_INT, arrRef, (int)lenOffs);
5157 if (bndsChkType != TYP_INT)
5159 arrLen = gtNewCastNode(bndsChkType, arrLen, bndsChkType);
5162 GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK) GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, arrLen, index, SCK_RNGCHK_FAIL);
5164 bndsChk = arrBndsChk;
5166 // Make sure to increment ref-counts if already ref-counted.
5167 if (lvaLocalVarRefCounted)
5169 lvaRecursiveIncRefCounts(index);
5170 lvaRecursiveIncRefCounts(arrRef);
5173 // Now we'll switch to using the second copies for arrRef and index
5174 // to compute the address expression
5180 // Create the "addr" which is "*(arrRef + ((index * elemSize) + elemOffs))"
5184 // Widen 'index' on 64-bit targets
5185 #ifdef _TARGET_64BIT_
5186 if (index->TypeGet() != TYP_I_IMPL)
5188 if (index->OperGet() == GT_CNS_INT)
5190 index->gtType = TYP_I_IMPL;
5194 index = gtNewCastNode(TYP_I_IMPL, index, TYP_I_IMPL);
5197 #endif // _TARGET_64BIT_
5199 /* Scale the index value if necessary */
5202 GenTreePtr size = gtNewIconNode(elemSize, TYP_I_IMPL);
5204 // Fix 392756 WP7 Crossgen
5206 // During codegen optGetArrayRefScaleAndIndex() makes the assumption that op2 of a GT_MUL node
5207 // is a constant and is not capable of handling CSE'ing the elemSize constant into a lclvar.
5208 // Hence to prevent the constant from becoming a CSE we mark it as NO_CSE.
5210 size->gtFlags |= GTF_DONT_CSE;
5212 /* Multiply by the array element size */
5213 addr = gtNewOperNode(GT_MUL, TYP_I_IMPL, index, size);
5220 /* Add the object ref to the element's offset */
5222 addr = gtNewOperNode(GT_ADD, TYP_BYREF, arrRef, addr);
5224 /* Add the first element's offset */
5226 GenTreePtr cns = gtNewIconNode(elemOffs, TYP_I_IMPL);
5228 addr = gtNewOperNode(GT_ADD, TYP_BYREF, addr, cns);
5230 #if SMALL_TREE_NODES
5231 assert(tree->gtFlags & GTF_NODE_LARGE);
5234 // Change the orginal GT_INDEX node into a GT_IND node
5235 tree->SetOper(GT_IND);
5237 // If the index node is a floating-point type, notify the compiler
5238 // we'll potentially use floating point registers at the time of codegen.
5239 if (varTypeIsFloating(tree->gtType))
5241 this->compFloatingPointUsed = true;
5244 // We've now consumed the GTF_INX_RNGCHK, and the node
5245 // is no longer a GT_INDEX node.
5246 tree->gtFlags &= ~GTF_INX_RNGCHK;
5248 tree->gtOp.gtOp1 = addr;
5250 // This is an array index expression.
5251 tree->gtFlags |= GTF_IND_ARR_INDEX;
5253 /* An indirection will cause a GPF if the address is null */
5254 tree->gtFlags |= GTF_EXCEPT;
5257 tree->gtFlags |= GTF_DONT_CSE;
5259 // Store information about it.
5260 GetArrayInfoMap()->Set(tree, ArrayInfo(elemTyp, elemSize, (int) elemOffs, elemStructType));
5262 // Remember this 'indTree' that we just created, as we still need to attach the fieldSeq information to it.
5264 GenTreePtr indTree = tree;
5266 // Did we create a bndsChk tree?
5269 // Use a GT_COMMA node to prepend the array bound check
5271 tree = gtNewOperNode(GT_COMMA, elemTyp, bndsChk, tree);
5273 /* Mark the indirection node as needing a range check */
5274 fgSetRngChkTarget(bndsChk);
5277 if (indexDefn != nullptr)
5279 // Use a GT_COMMA node to prepend the index assignment
5281 tree = gtNewOperNode(GT_COMMA, tree->TypeGet(), indexDefn, tree);
5283 if (arrRefDefn != nullptr)
5285 // Use a GT_COMMA node to prepend the arRef assignment
5287 tree = gtNewOperNode(GT_COMMA, tree->TypeGet(), arrRefDefn, tree);
5290 // Currently we morph the tree to perform some folding operations prior
5291 // to attaching fieldSeq info and labeling constant array index contributions
5295 // Ideally we just want to proceed to attaching fieldSeq info and labeling the
5296 // constant array index contributions, but the morphing operation may have changed
5297 // the 'tree' into something that now unconditionally throws an exception.
5299 // In such case the gtEffectiveVal could be a new tree or it's gtOper could be modified
5300 // or it could be left unchanged. If it is unchanged then we should not return,
5301 // instead we should proceed to attaching fieldSeq info, etc...
5303 GenTreePtr arrElem = tree->gtEffectiveVal();
5305 if (fgIsCommaThrow(tree))
5307 if ((arrElem != indTree) || // A new tree node may have been created
5308 (indTree->OperGet() != GT_IND)) // The GT_IND may have been changed to a GT_CNS_INT
5310 return tree; // Just return the Comma-Throw, don't try to attach the fieldSeq info, etc..
5314 assert(!fgGlobalMorph || (arrElem->gtFlags & GTF_MORPHED));
5316 addr = arrElem->gtOp.gtOp1;
5318 assert(addr->TypeGet() == TYP_BYREF);
5320 GenTreePtr cnsOff = nullptr;
5321 if (addr->OperGet() == GT_ADD)
5323 if (addr->gtOp.gtOp2->gtOper == GT_CNS_INT)
5325 cnsOff = addr->gtOp.gtOp2;
5326 addr = addr->gtOp.gtOp1;
5329 while ((addr->OperGet() == GT_ADD) || (addr->OperGet() == GT_SUB))
5331 assert(addr->TypeGet() == TYP_BYREF);
5332 GenTreePtr index = addr->gtOp.gtOp2;
5334 // Label any constant array index contributions with #ConstantIndex and any LclVars with GTF_VAR_ARR_INDEX
5335 index->LabelIndex(this);
5337 addr = addr->gtOp.gtOp1;
5339 assert(addr->TypeGet() == TYP_REF);
5341 else if (addr->OperGet() == GT_CNS_INT)
5346 FieldSeqNode* firstElemFseq = GetFieldSeqStore()->CreateSingleton(FieldSeqStore::FirstElemPseudoField);
5348 if ((cnsOff != nullptr) && (cnsOff->gtIntCon.gtIconVal == elemOffs))
5350 // Assign it the [#FirstElem] field sequence
5352 cnsOff->gtIntCon.gtFieldSeq = firstElemFseq;
5354 else // We have folded the first element's offset with the index expression
5356 // Build the [#ConstantIndex, #FirstElem] field sequence
5358 FieldSeqNode* constantIndexFseq = GetFieldSeqStore()->CreateSingleton(FieldSeqStore::ConstantIndexPseudoField);
5359 FieldSeqNode* fieldSeq = GetFieldSeqStore()->Append(constantIndexFseq, firstElemFseq);
5361 if (cnsOff == nullptr) // It must have folded into a zero offset
5363 // Record in the general zero-offset map.
5364 GetZeroOffsetFieldMap()->Set(addr, fieldSeq);
5368 cnsOff->gtIntCon.gtFieldSeq = fieldSeq;
5376 /*****************************************************************************
5378 * Wrap fixed stack arguments for varargs functions to go through varargs
5379 * cookie to access them, except for the cookie itself.
5381 * Non-x86 platforms are allowed to access all arguments directly
5382 * so we don't need this code.
5385 GenTreePtr Compiler::fgMorphStackArgForVarArgs(unsigned lclNum, var_types varType, unsigned lclOffs)
5387 /* For the fixed stack arguments of a varargs function, we need to go
5388 through the varargs cookies to access them, except for the
5391 LclVarDsc * varDsc = &lvaTable[lclNum];
5393 if (varDsc->lvIsParam && !varDsc->lvIsRegArg &&
5394 lclNum != lvaVarargsHandleArg)
5396 // Create a node representing the local pointing to the base of the args
5397 GenTreePtr ptrArg = gtNewOperNode(GT_SUB, TYP_I_IMPL,
5398 gtNewLclvNode(lvaVarargsBaseOfStkArgs, TYP_I_IMPL),
5399 gtNewIconNode(varDsc->lvStkOffs
5400 - codeGen->intRegState.rsCalleeRegArgNum*sizeof(void*)
5403 // Access the argument through the local
5404 GenTreePtr tree = gtNewOperNode(GT_IND, varType, ptrArg);
5405 tree->gtFlags |= GTF_IND_TGTANYWHERE;
5407 if (varDsc->lvAddrExposed)
5409 tree->gtFlags |= GTF_GLOB_REF;
5412 return fgMorphTree(tree);
5419 /*****************************************************************************
5421 * Transform the given GT_LCL_VAR tree for code generation.
5424 GenTreePtr Compiler::fgMorphLocalVar(GenTreePtr tree)
5426 noway_assert(tree->gtOper == GT_LCL_VAR);
5428 unsigned lclNum = tree->gtLclVarCommon.gtLclNum;
5429 var_types varType = lvaGetRealType(lclNum);
5430 LclVarDsc * varDsc = &lvaTable[lclNum];
5432 if (varDsc->lvAddrExposed)
5434 tree->gtFlags |= GTF_GLOB_REF;
5438 if (info.compIsVarArgs)
5440 GenTreePtr newTree = fgMorphStackArgForVarArgs(lclNum, varType, 0);
5441 if (newTree != NULL)
5444 #endif // _TARGET_X86_
5446 /* If not during the global morphing phase bail */
5451 bool varAddr = (tree->gtFlags & GTF_DONT_CSE) != 0;
5453 noway_assert(!(tree->gtFlags & GTF_VAR_DEF) || varAddr); // GTF_VAR_DEF should always imply varAddr
5456 varTypeIsSmall(varDsc->TypeGet()) &&
5457 varDsc->lvNormalizeOnLoad())
5459 #if LOCAL_ASSERTION_PROP
5460 /* Assertion prop can tell us to omit adding a cast here */
5461 if (optLocalAssertionProp &&
5462 optAssertionIsSubrange(tree, varType, apFull) != NO_ASSERTION_INDEX)
5467 /* Small-typed arguments and aliased locals are normalized on load.
5468 Other small-typed locals are normalized on store.
5469 Also, under the debugger as the debugger could write to the variable.
5470 If this is one of the former, insert a narrowing cast on the load.
5471 ie. Convert: var-short --> cast-short(var-int) */
5473 tree->gtType = TYP_INT;
5474 fgMorphTreeDone(tree);
5475 tree = gtNewCastNode(TYP_INT, tree, varType);
5476 fgMorphTreeDone(tree);
5484 /*****************************************************************************
5485 Grab a temp for big offset morphing.
5486 This method will grab a new temp if no temp of this "type" has been created.
5487 Or it will return the same cached one if it has been created.
5489 unsigned Compiler::fgGetBigOffsetMorphingTemp(var_types type)
5491 unsigned lclNum = fgBigOffsetMorphingTemps[type];
5493 if (lclNum == BAD_VAR_NUM) {
5494 // We haven't created a temp for this kind of type. Create one now.
5495 lclNum = lvaGrabTemp(false DEBUGARG("Big Offset Morphing"));
5496 fgBigOffsetMorphingTemps[type] = lclNum;
5499 // We better get the right type.
5500 noway_assert(lvaTable[lclNum].TypeGet() == type);
5503 noway_assert(lclNum != BAD_VAR_NUM);
5508 /*****************************************************************************
5510 * Transform the given GT_FIELD tree for code generation.
5513 GenTreePtr Compiler::fgMorphField(GenTreePtr tree, MorphAddrContext* mac)
5515 assert(tree->gtOper == GT_FIELD);
5517 noway_assert(tree->gtFlags & GTF_GLOB_REF);
5519 CORINFO_FIELD_HANDLE symHnd = tree->gtField.gtFldHnd;
5520 unsigned fldOffset = tree->gtField.gtFldOffset;
5521 GenTreePtr objRef = tree->gtField.gtFldObj;
5522 bool fieldMayOverlap = false;
5523 if (tree->gtField.gtFldMayOverlap)
5525 fieldMayOverlap = true;
5526 // Reset the flag because we may reuse the node.
5527 tree->gtField.gtFldMayOverlap = false;
5531 // if this field belongs to simd struct, tranlate it to simd instrinsic.
5532 if (mac == nullptr || mac->m_kind != MACK_Addr)
5534 GenTreePtr newTree = fgMorphFieldToSIMDIntrinsicGet(tree);
5535 if (newTree != tree)
5537 newTree = fgMorphSmpOp(newTree);
5541 else if (objRef != nullptr && objRef->OperGet() == GT_ADDR && objRef->OperIsSIMD())
5543 // We have a field of an SIMD intrinsic in an address-taken context.
5544 // We need to copy the SIMD result to a temp, and take the field of that.
5545 GenTree* copy = fgCopySIMDNode(objRef->gtOp.gtOp1->AsSIMD());
5546 objRef->gtOp.gtOp1 = copy;
5550 /* Is this an instance data member? */
5556 if (tree->gtFlags & GTF_IND_TLS_REF)
5557 NO_WAY("instance field can not be a TLS ref.");
5559 /* We'll create the expression "*(objRef + mem_offs)" */
5561 noway_assert(varTypeIsGC(objRef->TypeGet()) ||
5562 objRef->TypeGet() == TYP_I_IMPL);
5564 // An optimization for Contextful classes:
5565 // we unwrap the proxy when we have a 'this reference'
5566 if (info.compIsContextful &&
5567 info.compUnwrapContextful &&
5570 objRef = fgUnwrapProxy(objRef);
5574 Now we have a tree like this:
5576 +--------------------+
5578 +----------+---------+
5580 +--------------+-------------+
5581 | tree->gtField.gtFldObj |
5582 +--------------+-------------+
5585 We want to make it like this (when fldOffset is <= MAX_UNCHECKED_OFFSET_FOR_NULL_OBJECT):
5587 +--------------------+
5588 | GT_IND/GT_OBJ | tree
5589 +---------+----------+
5592 +---------+----------+
5594 +---------+----------+
5599 +-------------------+ +----------------------+
5600 | objRef | | fldOffset |
5601 | | | (when fldOffset !=0) |
5602 +-------------------+ +----------------------+
5605 or this (when fldOffset is > MAX_UNCHECKED_OFFSET_FOR_NULL_OBJECT):
5608 +--------------------+
5609 | GT_IND/GT_OBJ | tree
5610 +----------+---------+
5612 +----------+---------+
5614 +----------+---------+
5620 +---------+----------+ +---------+----------+
5621 comma | GT_COMMA | | "+" (i.e. GT_ADD) | addr
5622 +---------+----------+ +---------+----------+
5627 +-----+-----+ +-----+-----+ +---------+ +-----------+
5628 asg | GT_ASG | ind | GT_IND | | tmpLcl | | fldOffset |
5629 +-----+-----+ +-----+-----+ +---------+ +-----------+
5634 +-----+-----+ +-----+-----+ +-----------+
5635 | tmpLcl | | objRef | | tmpLcl |
5636 +-----------+ +-----------+ +-----------+
5641 var_types objRefType = objRef->TypeGet();
5643 GenTreePtr comma = NULL;
5645 bool addedExplicitNullCheck = false;
5647 // NULL mac means we encounter the GT_FIELD first. This denotes a dereference of the field,
5648 // and thus is equivalent to a MACK_Ind with zero offset.
5649 MorphAddrContext defMAC(MACK_Ind);
5650 if (mac == NULL) mac = &defMAC;
5652 // This flag is set to enable the "conservative" style of explicit null-check insertion.
5653 // This means that we insert an explicit null check whenever we create byref by adding a
5654 // constant offset to a ref, in a MACK_Addr context (meaning that the byref is not immediately
5655 // dereferenced). The alternative is "aggressive", which would not insert such checks (for
5656 // small offsets); in this plan, we would transfer some null-checking responsibility to
5657 // callee's of methods taking byref parameters. They would have to add explicit null checks
5658 // when creating derived byrefs from argument byrefs by adding constants to argument byrefs, in
5659 // contexts where the resulting derived byref is not immediately dereferenced (or if the offset is too
5660 // large). To make the "aggressive" scheme work, however, we'd also have to add explicit derived-from-null
5661 // checks for byref parameters to "external" methods implemented in C++, and in P/Invoke stubs.
5662 /// This is left here to point out how to implement it.
5663 #define CONSERVATIVE_NULL_CHECK_BYREF_CREATION 1
5665 // If the objRef is a GT_ADDR node, it, itself, never requires null checking. The expression
5666 // whose address is being taken is either a local or static variable, whose address is necessarily
5667 // non-null, or else it is a field dereference, which will do its own bounds checking if necessary.
5668 if (objRef->gtOper != GT_ADDR
5669 && ((mac->m_kind == MACK_Addr || mac->m_kind == MACK_Ind)
5670 && (!mac->m_allConstantOffsets
5671 || fgIsBigOffset(mac->m_totalOffset + fldOffset)
5672 #if CONSERVATIVE_NULL_CHECK_BYREF_CREATION
5673 || (mac->m_kind == MACK_Addr && (mac->m_totalOffset + fldOffset > 0))
5675 || (objRef->gtType == TYP_BYREF && mac->m_kind == MACK_Addr && (mac->m_totalOffset + fldOffset > 0))
5682 printf("Before explicit null check morphing:\n");
5688 // Create the "comma" subtree
5690 GenTreePtr asg = NULL;
5695 if (objRef->gtOper != GT_LCL_VAR)
5697 lclNum = fgGetBigOffsetMorphingTemp(genActualType(objRef->TypeGet()));
5699 // Create the "asg" node
5700 asg = gtNewTempAssign(lclNum, objRef);
5704 lclNum = objRef->gtLclVarCommon.gtLclNum;
5707 // Create the "nullchk" node
5708 nullchk = gtNewOperNode(GT_NULLCHECK,
5709 TYP_BYTE, // Make it TYP_BYTE so we only deference it for 1 byte.
5710 gtNewLclvNode(lclNum, objRefType));
5711 nullchk->gtFlags |= GTF_DONT_CSE; // Don't try to create a CSE for these TYP_BYTE indirections
5713 /* An indirection will cause a GPF if the address is null */
5714 nullchk->gtFlags |= GTF_EXCEPT;
5718 // Create the "comma" node.
5719 comma = gtNewOperNode(GT_COMMA,
5720 TYP_VOID, // We don't want to return anything from this "comma" node.
5721 // Set the type to TYP_VOID, so we can select "cmp" instruction
5722 // instead of "mov" instruction later on.
5731 addr = gtNewLclvNode(lclNum, objRefType); // Use "tmpLcl" to create "addr" node.
5733 addedExplicitNullCheck = true;
5735 else if (fldOffset == 0)
5737 // Generate the "addr" node.
5739 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5740 GetZeroOffsetFieldMap()->Set(addr, fieldSeq);
5747 #ifdef FEATURE_READYTORUN_COMPILER
5748 if (tree->gtField.gtFieldLookup.addr != nullptr)
5750 GenTreePtr baseOffset = gtNewIconEmbHndNode(tree->gtField.gtFieldLookup.addr, nullptr, GTF_ICON_FIELD_HDL);
5752 if (tree->gtField.gtFieldLookup.accessType == IAT_PVALUE)
5753 baseOffset = gtNewOperNode(GT_IND, TYP_I_IMPL, baseOffset);
5755 addr = gtNewOperNode(GT_ADD,
5756 (var_types)(objRefType == TYP_I_IMPL ? TYP_I_IMPL
5765 // Generate the "addr" node.
5766 /* Add the member offset to the object's address */
5767 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5768 addr = gtNewOperNode(GT_ADD,
5769 (var_types)(objRefType == TYP_I_IMPL ? TYP_I_IMPL
5772 gtNewIconHandleNode(fldOffset,
5777 // Now let's set the "tree" as a GT_IND tree.
5779 tree->SetOper(GT_IND);
5780 tree->gtOp.gtOp1 = addr;
5782 if (fgAddrCouldBeNull(addr))
5784 // This indirection can cause a GPF if the address could be null.
5785 tree->gtFlags |= GTF_EXCEPT;
5788 if (addedExplicitNullCheck)
5791 // Create "comma2" node and link it to "tree".
5794 comma2 = gtNewOperNode(GT_COMMA,
5795 addr->TypeGet(), // The type of "comma2" node is the same as the type of "addr" node.
5798 tree->gtOp.gtOp1 = comma2;
5804 if (addedExplicitNullCheck) {
5805 printf("After adding explicit null check:\n");
5812 else /* This is a static data member */
5814 if (tree->gtFlags & GTF_IND_TLS_REF)
5816 // Thread Local Storage static field reference
5818 // Field ref is a TLS 'Thread-Local-Storage' reference
5820 // Build this tree: IND(*) #
5828 // IND(I_IMPL) == [Base of this DLL's TLS]
5832 // / CNS(IdValue*4) or MUL
5834 // IND(I_IMPL) / CNS(4)
5836 // CNS(TLS_HDL,0x2C) IND
5840 // # Denotes the orginal node
5842 void ** pIdAddr = NULL;
5843 unsigned IdValue = info.compCompHnd->getFieldThreadLocalStoreID(symHnd, (void**) &pIdAddr);
5846 // If we can we access the TLS DLL index ID value directly
5847 // then pIdAddr will be NULL and
5848 // IdValue will be the actual TLS DLL index ID
5850 GenTreePtr dllRef = NULL;
5851 if (pIdAddr == NULL)
5854 dllRef = gtNewIconNode(IdValue*4, TYP_I_IMPL);
5858 dllRef = gtNewIconHandleNode((size_t)pIdAddr, GTF_ICON_STATIC_HDL);
5859 dllRef = gtNewOperNode(GT_IND, TYP_I_IMPL, dllRef);
5860 dllRef->gtFlags |= GTF_IND_INVARIANT;
5864 dllRef = gtNewOperNode(GT_MUL, TYP_I_IMPL, dllRef, gtNewIconNode(4, TYP_I_IMPL));
5867 #define WIN32_TLS_SLOTS (0x2C) // Offset from fs:[0] where the pointer to the slots resides
5869 // Mark this ICON as a TLS_HDL, codegen will use FS:[cns]
5871 GenTreePtr tlsRef = gtNewIconHandleNode(WIN32_TLS_SLOTS, GTF_ICON_TLS_HDL);
5873 tlsRef = gtNewOperNode(GT_IND, TYP_I_IMPL, tlsRef);
5877 /* Add the dllRef */
5878 tlsRef = gtNewOperNode(GT_ADD, TYP_I_IMPL, tlsRef, dllRef);
5881 /* indirect to have tlsRef point at the base of the DLLs Thread Local Storage */
5882 tlsRef = gtNewOperNode(GT_IND, TYP_I_IMPL, tlsRef);
5886 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5887 GenTreePtr fldOffsetNode = new(this, GT_CNS_INT) GenTreeIntCon(TYP_INT, fldOffset, fieldSeq);
5889 /* Add the TLS static field offset to the address */
5891 tlsRef = gtNewOperNode(GT_ADD, TYP_I_IMPL, tlsRef, fldOffsetNode);
5894 // Final indirect to get to actual value of TLS static field
5896 tree->SetOper(GT_IND);
5897 tree->gtOp.gtOp1 = tlsRef;
5899 noway_assert(tree->gtFlags & GTF_IND_TLS_REF);
5903 // Normal static field reference
5906 // If we can we access the static's address directly
5907 // then pFldAddr will be NULL and
5908 // fldAddr will be the actual address of the static field
5910 void ** pFldAddr = NULL;
5911 void * fldAddr = info.compCompHnd->getFieldAddress(symHnd, (void**) &pFldAddr);
5913 if (pFldAddr == NULL)
5915 #ifdef _TARGET_64BIT_
5916 if (IMAGE_REL_BASED_REL32 != eeGetRelocTypeHint(fldAddr))
5918 // The address is not directly addressible, so force it into a
5919 // constant, so we handle it properly
5921 GenTreePtr addr = gtNewIconHandleNode((size_t)fldAddr, GTF_ICON_STATIC_HDL);
5922 addr->gtType = TYP_I_IMPL;
5923 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5924 addr->gtIntCon.gtFieldSeq = fieldSeq;
5926 tree->SetOper(GT_IND);
5927 tree->gtOp.gtOp1 = addr;
5929 return fgMorphSmpOp(tree);
5932 #endif // _TARGET_64BIT_
5934 // Only volatile could be set, and it maps over
5935 noway_assert((tree->gtFlags & ~(GTF_FLD_VOLATILE | GTF_COMMON_MASK)) == 0);
5936 noway_assert(GTF_FLD_VOLATILE == GTF_IND_VOLATILE);
5937 tree->SetOper(GT_CLS_VAR);
5938 tree->gtClsVar.gtClsVarHnd = symHnd;
5939 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5940 tree->gtClsVar.gtFieldSeq = fieldSeq;
5947 GenTreePtr addr = gtNewIconHandleNode((size_t)pFldAddr, GTF_ICON_STATIC_HDL);
5949 // There are two cases here, either the static is RVA based,
5950 // in which case the type of the FIELD node is not a GC type
5951 // and the handle to the RVA is a TYP_I_IMPL. Or the FIELD node is
5952 // a GC type and the handle to it is a TYP_BYREF in the GC heap
5953 // because handles to statics now go into the large object heap
5955 var_types handleTyp = (var_types) (varTypeIsGC(tree->TypeGet()) ? TYP_BYREF
5957 GenTreePtr op1 = gtNewOperNode(GT_IND, handleTyp, addr);
5958 op1->gtFlags |= GTF_IND_INVARIANT;
5960 tree->SetOper(GT_IND);
5961 tree->gtOp.gtOp1 = op1;
5965 noway_assert(tree->gtOper == GT_IND);
5967 GenTreePtr res = fgMorphSmpOp(tree);
5969 if (fldOffset == 0 && res->OperGet() == GT_IND)
5971 GenTreePtr addr = res->gtOp.gtOp1;
5972 // Since we don't make a constant zero to attach the field sequence to, associate it with the "addr" node.
5973 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5974 fgAddFieldSeqForZeroOffset(addr, fieldSeq);
5982 //------------------------------------------------------------------------------
5983 // fgMorphCallInline: attempt to inline a call
5986 // call - call expression to inline, inline candidate
5987 // inlineResult - result tracking and reporting
5990 // Attempts to inline the call.
5992 // If successful, callee's IR is inserted in place of the call, and
5993 // is marked with an InlineContext.
5995 // If unsuccessful, the transformations done in anticpation of a
5996 // possible inline are undone, and the candidate flag on the call
5999 void Compiler::fgMorphCallInline(GenTreeCall* call, InlineResult* inlineResult)
6001 // The call must be a candiate for inlining.
6002 assert((call->gtFlags & GTF_CALL_INLINE_CANDIDATE) != 0);
6004 // Attempt the inline
6005 fgMorphCallInlineHelper(call, inlineResult);
6007 // We should have made up our minds one way or another....
6008 assert(inlineResult->IsDecided());
6010 // If we failed to inline, we have a bit of work to do to cleanup
6011 if (inlineResult->IsFailure())
6016 // Before we do any cleanup, create a failing InlineContext to
6017 // capture details of the inlining attempt.
6018 m_inlineStrategy->NewFailure(fgMorphStmt, inlineResult);
6022 // It was an inline candidate, but we haven't expanded it.
6023 if (call->gtCall.gtReturnType != TYP_VOID)
6025 // Detach the GT_CALL tree from the original statement by
6026 // hanging a "nothing" node to it. Later the "nothing" node will be removed
6027 // and the original GT_CALL tree will be picked up by the GT_RET_EXPR node.
6029 noway_assert(fgMorphStmt->gtStmt.gtStmtExpr == call);
6030 fgMorphStmt->gtStmt.gtStmtExpr = gtNewNothingNode();
6033 // Clear the Inline Candidate flag so we can ensure later we tried
6034 // inlining all candidates.
6036 call->gtFlags &= ~GTF_CALL_INLINE_CANDIDATE;
6040 /*****************************************************************************
6041 * Helper to attempt to inline a call
6042 * Sets success/failure in inline result
6043 * If success, modifies current method's IR with inlinee's IR
6044 * If failed, undoes any speculative modifications to current method
6047 void Compiler::fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result)
6049 // Don't expect any surprises here.
6050 assert(result->IsCandidate());
6052 if (lvaCount >= MAX_LV_NUM_COUNT_FOR_INLINING)
6054 // For now, attributing this to call site, though it's really
6055 // more of a budget issue (lvaCount currently includes all
6056 // caller and prospective callee locals). We still might be
6057 // able to inline other callees into this caller, or inline
6058 // this callee in other callers.
6059 result->NoteFatal(InlineObservation::CALLSITE_TOO_MANY_LOCALS);
6063 if (call->IsVirtual())
6065 result->NoteFatal(InlineObservation::CALLSITE_IS_VIRTUAL);
6069 // impMarkInlineCandidate() is expected not to mark tail prefixed calls
6070 // and recursive tail calls as inline candidates.
6071 noway_assert(!call->IsTailPrefixedCall());
6072 noway_assert(!call->IsImplicitTailCall() || !gtIsRecursiveCall(call));
6074 /* If the caller's stack frame is marked, then we can't do any inlining. Period.
6075 Although we have checked this in impCanInline, it is possible that later IL instructions
6076 might cause compNeedSecurityCheck to be set. Therefore we need to check it here again.
6079 if (opts.compNeedSecurityCheck)
6081 result->NoteFatal(InlineObservation::CALLER_NEEDS_SECURITY_CHECK);
6086 // Calling inlinee's compiler to inline the method.
6089 unsigned startVars = lvaCount;
6094 printf("Expanding INLINE_CANDIDATE in statement ");
6095 printTreeID(fgMorphStmt);
6096 printf(" in BB%02u:\n", compCurBB->bbNum);
6097 gtDispTree(fgMorphStmt);
6099 // printf("startVars=%d.\n", startVars);
6104 // Invoke the compiler to inline the call.
6107 fgInvokeInlineeCompiler(call, result);
6109 if (result->IsFailure())
6111 // Undo some changes made in anticipation of inlining...
6113 // Zero out the used locals
6114 memset(lvaTable + startVars, 0, (lvaCount - startVars) * sizeof(*lvaTable));
6115 for (unsigned i = startVars; i < lvaCount; i++)
6117 new (&lvaTable[i], jitstd::placement_t()) LclVarDsc(this); // call the constructor.
6120 lvaCount = startVars;
6125 // printf("Inlining failed. Restore lvaCount to %d.\n", lvaCount);
6137 // printf("After inlining lvaCount=%d.\n", lvaCount);
6142 /*****************************************************************************
6144 * Performs checks to see if this tail call can be optimized as epilog+jmp.
6146 bool Compiler::fgCanFastTailCall(GenTreeCall* callee)
6148 #if FEATURE_FASTTAILCALL
6149 // Reached here means that return types of caller and callee are tail call compatible.
6150 // In case of structs that can be returned in a register, compRetNativeType is set to the actual return type.
6152 // In an implicit tail call case callSig may not be available but it is guaranteed to be available
6153 // for explicit tail call cases. The reason implicit tail case callSig may not be available is that
6154 // a call node might be marked as an in-line candidate and could fail to be in-lined. In which case
6155 // fgInline() will replace return value place holder with call node using gtCloneExpr() which is
6156 // currently not copying/setting callSig.
6158 if (callee->IsTailPrefixedCall())
6160 assert(impTailCallRetTypeCompatible(info.compRetNativeType, info.compMethodInfo->args.retTypeClass,
6161 (var_types)callee->gtReturnType, callee->callSig->retTypeClass));
6165 // Note on vararg methods:
6166 // If the caller is vararg method, we don't know the number of arguments passed by caller's caller.
6167 // But we can be sure that in-coming arg area of vararg caller would be sufficient to hold its
6168 // fixed args. Therefore, we can allow a vararg method to fast tail call other methods as long as
6169 // out-going area required for callee is bounded by caller's fixed argument space.
6171 // Note that callee being a vararg method is not a problem since we can account the params being passed.
6173 // Count of caller args including implicit and hidden (i.e. thisPtr, RetBuf, GenericContext, VarargCookie)
6174 unsigned nCallerArgs = info.compArgsCount;
6176 // Count the callee args including implicit and hidden.
6177 // Note that GenericContext and VarargCookie are added by importer while
6178 // importing the call to gtCallArgs list along with explicit user args.
6179 unsigned nCalleeArgs = 0;
6180 if (callee->gtCallObjp) // thisPtr
6185 if (callee->HasRetBufArg()) // RetBuf
6189 // If callee has RetBuf param, caller too must have it.
6190 // Otherwise go the slow route.
6191 if (info.compRetBuffArg == BAD_VAR_NUM)
6197 // Count user args while tracking whether any of them is a multi-byte params
6198 // that cannot be passed in a register. Note that we don't need to count
6199 // non-standard and secret params passed in registers (e.g. R10, R11) since
6200 // these won't contribute to out-going arg size.
6201 bool hasMultiByteArgs = false;
6202 for (GenTreePtr args = callee->gtCallArgs; (args != nullptr) && !hasMultiByteArgs; args = args->gtOp.gtOp2)
6206 assert(args->IsList());
6207 GenTreePtr argx = args->gtOp.gtOp1;
6209 if (varTypeIsStruct(argx))
6211 // Actual arg may be a child of a GT_COMMA. Skip over comma opers.
6212 while (argx->gtOper == GT_COMMA)
6214 argx = argx->gtOp.gtOp2;
6217 // Get the size of the struct and see if it is register passable.
6218 if (argx->OperGet() == GT_OBJ)
6220 #if defined(_TARGET_AMD64_) || defined(_TARGET_ARM64_)
6222 unsigned typeSize = 0;
6223 hasMultiByteArgs = !VarTypeIsMultiByteAndCanEnreg(argx->TypeGet(), argx->gtObj.gtClass, &typeSize, false);
6225 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING) || defined(_TARGET_ARM64_)
6226 // On System V/arm64 the args could be a 2 eightbyte struct that is passed in two registers.
6227 // Account for the second eightbyte in the nCalleeArgs.
6228 // https://github.com/dotnet/coreclr/issues/2666
6229 // TODO-CQ-Amd64-Unix/arm64: Structs of size between 9 to 16 bytes are conservatively estimated
6230 // as two args, since they need two registers whereas nCallerArgs is
6231 // counting such an arg as one. This would mean we will not be optimizing
6232 // certain calls though technically possible.
6234 if (typeSize > TARGET_POINTER_SIZE)
6236 unsigned extraArgRegsToAdd = (typeSize / TARGET_POINTER_SIZE);
6237 nCalleeArgs += extraArgRegsToAdd;
6239 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING || _TARGET_ARM64_
6242 assert(!"Target platform ABI rules regarding passing struct type args in registers");
6244 #endif //_TARGET_AMD64_ || _TARGET_ARM64_
6249 hasMultiByteArgs = true;
6254 // Go the slow route, if it has multi-byte params
6255 if (hasMultiByteArgs)
6260 // If we reached here means that callee has only those argument types which can be passed in
6261 // a register and if passed on stack will occupy exactly one stack slot in out-going arg area.
6262 // If we are passing args on stack for callee and it has more args passed on stack than
6263 // caller, then fast tail call cannot be performed.
6265 // Note that the GC'ness of on stack args need not match since the arg setup area is marked
6266 // as non-interruptible for fast tail calls.
6267 if ((nCalleeArgs > MAX_REG_ARG) && (nCallerArgs < nCalleeArgs))
6279 /*****************************************************************************
6281 * Transform the given GT_CALL tree for tail call code generation.
6283 void Compiler::fgMorphTailCall(GenTreeCall* call)
6285 // x86 classic codegen doesn't require any morphing
6286 #if defined(_TARGET_X86_) && !defined(LEGACY_BACKEND)
6287 NYI_X86("Tail call morphing");
6288 #elif defined(_TARGET_ARM_)
6289 // For the helper-assisted tail calls, we need to push all the arguments
6290 // into a single list, and then add a few extra at the beginning
6292 // Check for PInvoke call types that we don't handle in codegen yet.
6293 assert(!call->IsUnmanaged());
6294 assert(call->IsVirtual() ||
6295 (call->gtCallType != CT_INDIRECT) ||
6296 (call->gtCallCookie == NULL));
6298 // First move the this pointer (if any) onto the regular arg list
6299 GenTreePtr thisPtr = NULL;
6300 if (call->gtCallObjp)
6302 GenTreePtr objp = call->gtCallObjp;
6303 call->gtCallObjp = NULL;
6305 if ((call->gtFlags & GTF_CALL_NULLCHECK) ||
6306 call->IsVirtualVtable())
6308 thisPtr = gtClone(objp, true);
6309 var_types vt = objp->TypeGet();
6310 if (thisPtr == NULL)
6312 // Too complex, so use a temp
6313 unsigned lclNum = lvaGrabTemp(true DEBUGARG("tail call thisptr"));
6314 GenTreePtr asg = gtNewTempAssign(lclNum, objp);
6315 if (!call->IsVirtualVtable())
6317 // Add an indirection to get the nullcheck
6318 GenTreePtr tmp = gtNewLclvNode(lclNum, vt);
6319 GenTreePtr ind = gtNewOperNode(GT_IND, TYP_INT, tmp);
6320 asg = gtNewOperNode(GT_COMMA, TYP_VOID, asg, ind);
6322 objp = gtNewOperNode(GT_COMMA, vt, asg, gtNewLclvNode(lclNum, vt));
6323 thisPtr = gtNewLclvNode(lclNum, vt);
6325 else if (!call->IsVirtualVtable())
6327 GenTreePtr ind = gtNewOperNode(GT_IND, TYP_INT, thisPtr);
6328 objp = gtNewOperNode(GT_COMMA, vt, ind, objp);
6329 thisPtr = gtClone(thisPtr, true);
6332 call->gtFlags &= ~GTF_CALL_NULLCHECK;
6335 GenTreeArgList** pList = &call->gtCallArgs;
6336 #if RETBUFARG_PRECEDES_THIS
6337 if (call->HasRetBufArg()) {
6338 pList = &(*pList)->Rest();
6340 #endif // RETBUFARG_PRECEDES_THIS
6341 *pList = gtNewListNode(objp, *pList);
6344 // Add the extra VSD parameter if needed
6345 CorInfoHelperTailCallSpecialHandling flags = CorInfoHelperTailCallSpecialHandling(0);
6346 if (call->IsVirtualStub())
6348 flags = CORINFO_TAILCALL_STUB_DISPATCH_ARG;
6351 if (call->gtCallType == CT_INDIRECT) {
6352 arg = gtClone(call->gtCallAddr, true);
6353 noway_assert(arg != NULL);
6356 noway_assert(call->gtCallMoreFlags & GTF_CALL_M_VIRTSTUB_REL_INDIRECT);
6357 ssize_t addr = ssize_t(call->gtStubCallStubAddr);
6358 arg = gtNewIconHandleNode(addr, GTF_ICON_FTN_ADDR);
6360 // Change the call type, so we can add the extra indirection here, rather than in codegen
6361 call->gtCallAddr = gtNewIconHandleNode(addr, GTF_ICON_FTN_ADDR);
6362 call->gtStubCallStubAddr = NULL;
6363 call->gtCallType = CT_INDIRECT;
6365 // Add the extra indirection to generate the real target
6366 call->gtCallAddr = gtNewOperNode(GT_IND, TYP_I_IMPL, call->gtCallAddr);
6367 call->gtFlags |= GTF_EXCEPT;
6369 // And push the stub address onto the list of arguments
6370 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
6372 else if (call->IsVirtualVtable())
6374 // TODO-ARM-NYI: for x64 handle CORINFO_TAILCALL_THIS_IN_SECRET_REGISTER
6376 noway_assert(thisPtr != NULL);
6378 GenTreePtr add = gtNewOperNode(GT_ADD, TYP_I_IMPL, thisPtr, gtNewIconNode(VPTR_OFFS, TYP_I_IMPL));
6379 GenTreePtr vtbl = gtNewOperNode(GT_IND, TYP_I_IMPL, add);
6380 vtbl->gtFlags |= GTF_EXCEPT;
6382 unsigned vtabOffsOfIndirection;
6383 unsigned vtabOffsAfterIndirection;
6384 info.compCompHnd->getMethodVTableOffset(call->gtCallMethHnd, &vtabOffsOfIndirection, &vtabOffsAfterIndirection);
6386 /* Get the appropriate vtable chunk */
6388 add = gtNewOperNode(GT_ADD, TYP_I_IMPL, vtbl, gtNewIconNode(vtabOffsOfIndirection, TYP_I_IMPL));
6389 vtbl = gtNewOperNode(GT_IND, TYP_I_IMPL, add);
6391 /* Now the appropriate vtable slot */
6393 add = gtNewOperNode(GT_ADD, TYP_I_IMPL, vtbl, gtNewIconNode(vtabOffsAfterIndirection, TYP_I_IMPL));
6394 vtbl = gtNewOperNode(GT_IND, TYP_I_IMPL, add);
6396 // Switch this to a plain indirect call
6397 call->gtFlags &= ~GTF_CALL_VIRT_KIND_MASK;
6398 assert(!call->IsVirtual());
6399 call->gtCallType = CT_INDIRECT;
6401 call->gtCallAddr = vtbl;
6402 call->gtCallCookie = NULL;
6403 call->gtFlags |= GTF_EXCEPT;
6406 // Now inject a placeholder for the real call target that codegen
6408 GenTreePtr arg = new (this, GT_NOP) GenTreeOp(GT_NOP, TYP_I_IMPL);
6409 codeGen->genMarkTreeInReg(arg, REG_TAILCALL_ADDR);
6410 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
6412 // Lastly inject the pointer for the copy routine
6413 noway_assert(call->callSig != NULL);
6414 void * pfnCopyArgs = info.compCompHnd->getTailCallCopyArgsThunk(call->callSig, flags);
6415 arg = gtNewIconHandleNode(ssize_t(pfnCopyArgs), GTF_ICON_FTN_ADDR);
6416 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
6418 // It is now a varargs tail call
6419 call->gtCallMoreFlags = GTF_CALL_M_VARARGS | GTF_CALL_M_TAILCALL;
6420 call->gtFlags &= ~GTF_CALL_POP_ARGS;
6422 #elif defined(_TARGET_AMD64_)
6423 // For the helper-assisted tail calls, we need to push all the arguments
6424 // into a single list, and then add a few extra at the beginning.
6426 // TailCallHelper(void *copyRoutine, void *callTarget, ....) - i.e We need to add
6427 // copyRoutine and callTarget extra params at the beginning. But callTarget is
6428 // determined by Lower phase. Therefore, we add a place holder arg for callTarget
6429 // here which will be later replaced with callTarget in tail call lowering.
6431 // Check for PInvoke call types that we don't handle in codegen yet.
6432 assert(!call->IsUnmanaged());
6433 assert(call->IsVirtual() ||
6434 (call->gtCallType != CT_INDIRECT) ||
6435 (call->gtCallCookie == NULL));
6437 // Don't support tail calling helper methods
6438 assert(call->gtCallType != CT_HELPER);
6440 // We come this route only for tail prefixed calls that cannot be dispatched as
6442 assert(!call->IsImplicitTailCall());
6443 assert(!fgCanFastTailCall(call));
6445 // First move the this pointer (if any) onto the regular arg list
6446 if (call->gtCallObjp)
6448 GenTreePtr thisPtr = nullptr;
6449 GenTreePtr objp = call->gtCallObjp;
6450 call->gtCallObjp = nullptr;
6452 if (call->NeedsNullCheck())
6454 // clone "this" if "this" has no side effects.
6455 if (!(objp->gtFlags & GTF_SIDE_EFFECT))
6457 thisPtr = gtClone(objp, true);
6460 var_types vt = objp->TypeGet();
6461 if (thisPtr == nullptr)
6463 // create a temp if either "this" has side effects or "this" is too complex to clone.
6466 unsigned lclNum = lvaGrabTemp(true DEBUGARG("tail call thisptr"));
6467 GenTreePtr asg = gtNewTempAssign(lclNum, objp);
6469 // COMMA(tmp = "this", deref(tmp))
6470 GenTreePtr tmp = gtNewLclvNode(lclNum, vt);
6471 GenTreePtr ind = gtNewOperNode(GT_IND, TYP_INT, tmp);
6472 asg = gtNewOperNode(GT_COMMA, TYP_VOID, asg, ind);
6474 // COMMA(COMMA(tmp = "this", deref(tmp)), tmp)
6475 thisPtr = gtNewOperNode(GT_COMMA, vt, asg, gtNewLclvNode(lclNum, vt));
6479 // thisPtr = COMMA(deref("this"), "this")
6480 GenTreePtr ind = gtNewOperNode(GT_IND, TYP_INT, thisPtr);
6481 thisPtr = gtNewOperNode(GT_COMMA, vt, ind, gtClone(objp, true));
6484 call->gtFlags &= ~GTF_CALL_NULLCHECK;
6491 GenTreeArgList** pList = &call->gtCallArgs;
6492 #if RETBUFARG_PRECEDES_THIS
6493 if (call->HasRetBufArg()) {
6494 pList = &(*pList)->Rest();
6496 #endif // RETBUFARG_PRECEDES_THIS
6498 // During rationalization tmp="this" and null check will
6499 // materialize as embedded stmts in right execution order.
6500 assert(thisPtr != nullptr);
6501 *pList = gtNewListNode(thisPtr, *pList);
6504 // Add the extra VSD parameter to arg list in case of VSD calls.
6505 // Tail call arg copying thunk will move this extra VSD parameter
6506 // to R11 before tail calling VSD stub. See CreateTailCallCopyArgsThunk()
6507 // in Stublinkerx86.cpp for more details.
6508 CorInfoHelperTailCallSpecialHandling flags = CorInfoHelperTailCallSpecialHandling(0);
6509 if (call->IsVirtualStub())
6511 GenTreePtr stubAddrArg;
6513 flags = CORINFO_TAILCALL_STUB_DISPATCH_ARG;
6515 if (call->gtCallType == CT_INDIRECT)
6517 stubAddrArg = gtClone(call->gtCallAddr, true);
6518 noway_assert(stubAddrArg != nullptr);
6522 noway_assert((call->gtCallMoreFlags & GTF_CALL_M_VIRTSTUB_REL_INDIRECT) != 0);
6524 ssize_t addr = ssize_t(call->gtStubCallStubAddr);
6525 stubAddrArg = gtNewIconHandleNode(addr, GTF_ICON_FTN_ADDR);
6528 // Push the stub address onto the list of arguments
6529 call->gtCallArgs = gtNewListNode(stubAddrArg, call->gtCallArgs);
6532 // Now inject a placeholder for the real call target that Lower phase will generate.
6533 GenTreePtr arg = gtNewIconNode(0, TYP_I_IMPL);
6534 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
6536 // Inject the pointer for the copy routine to be used for struct copying
6537 noway_assert(call->callSig != nullptr);
6538 void * pfnCopyArgs = info.compCompHnd->getTailCallCopyArgsThunk(call->callSig, flags);
6539 arg = gtNewIconHandleNode(ssize_t(pfnCopyArgs), GTF_ICON_FTN_ADDR);
6540 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
6542 // It is now a varargs tail call dispatched via helper.
6543 call->gtCallMoreFlags |= GTF_CALL_M_VARARGS | GTF_CALL_M_TAILCALL | GTF_CALL_M_TAILCALL_VIA_HELPER;
6544 call->gtFlags &= ~GTF_CALL_POP_ARGS;
6546 #endif //_TARGET_AMD64_
6550 //------------------------------------------------------------------------------
6551 // fgMorphRecursiveFastTailCallIntoLoop : Transform a recursive fast tail call into a loop.
6555 // block - basic block ending with a recursive fast tail call
6556 // recursiveTailCall - recursive tail call to transform
6559 // The legality of the transformation is ensured by the checks in endsWithTailCallConvertibleToLoop.
6561 void Compiler::fgMorphRecursiveFastTailCallIntoLoop(BasicBlock* block, GenTreeCall* recursiveTailCall)
6563 assert(recursiveTailCall->IsTailCallConvertibleToLoop());
6564 GenTreePtr last = fgGetLastTopLevelStmt(block);
6565 assert(recursiveTailCall == last->gtStmt.gtStmtExpr);
6567 // Transform recursive tail call into a loop.
6569 GenTreePtr earlyArgInsertionPoint = last;
6570 IL_OFFSETX callILOffset = last->gtStmt.gtStmtILoffsx;
6572 // Hoist arg setup statement for the 'this' argument.
6573 GenTreePtr thisArg = recursiveTailCall->gtCallObjp;
6574 if (thisArg && !thisArg->IsNothingNode() && !thisArg->IsArgPlaceHolderNode())
6576 GenTreePtr thisArgStmt = gtNewStmt(thisArg, callILOffset);
6577 fgInsertStmtBefore(block, earlyArgInsertionPoint, thisArgStmt);
6581 // All arguments whose trees may involve caller parameter local variables need to be assigned to temps first;
6582 // then the temps need to be assigned to the method parameters. This is done so that the caller
6583 // parameters are not re-assigned before call arguments depending on them are evaluated.
6584 // tmpAssignmentInsertionPoint and paramAssignmentInsertionPoint keep track of
6585 // where the next temp or parameter assignment should be inserted.
6587 // In the example below the first call argument (arg1 - 1) needs to be assigned to a temp first
6588 // while the second call argument (const 1) doesn't.
6589 // Basic block before tail recursion elimination:
6590 // ***** BB04, stmt 1 (top level)
6591 // [000037] ------------ * stmtExpr void (top level) (IL 0x00A...0x013)
6592 // [000033] --C - G------ - \--* call void RecursiveMethod
6593 // [000030] ------------ | / --* const int - 1
6594 // [000031] ------------arg0 in rcx + --* +int
6595 // [000029] ------------ | \--* lclVar int V00 arg1
6596 // [000032] ------------arg1 in rdx \--* const int 1
6599 // Basic block after tail recursion elimination :
6600 // ***** BB04, stmt 1 (top level)
6601 // [000051] ------------ * stmtExpr void (top level) (IL 0x00A... ? ? ? )
6602 // [000030] ------------ | / --* const int - 1
6603 // [000031] ------------ | / --* +int
6604 // [000029] ------------ | | \--* lclVar int V00 arg1
6605 // [000050] - A---------- \--* = int
6606 // [000049] D------N---- \--* lclVar int V02 tmp0
6608 // ***** BB04, stmt 2 (top level)
6609 // [000055] ------------ * stmtExpr void (top level) (IL 0x00A... ? ? ? )
6610 // [000052] ------------ | / --* lclVar int V02 tmp0
6611 // [000054] - A---------- \--* = int
6612 // [000053] D------N---- \--* lclVar int V00 arg0
6614 // ***** BB04, stmt 3 (top level)
6615 // [000058] ------------ * stmtExpr void (top level) (IL 0x00A... ? ? ? )
6616 // [000032] ------------ | / --* const int 1
6617 // [000057] - A---------- \--* = int
6618 // [000056] D------N---- \--* lclVar int V01 arg1
6620 GenTreePtr tmpAssignmentInsertionPoint = last;
6621 GenTreePtr paramAssignmentInsertionPoint = last;
6623 // Process early args. They may contain both setup statements for late args and actual args.
6624 // Early args don't include 'this' arg. We need to account for that so that the call to gtArgEntryByArgNum
6625 // below has the correct second argument.
6626 int earlyArgIndex = (thisArg == nullptr) ? 0 : 1;
6627 for (GenTreeArgList* earlyArgs = recursiveTailCall->gtCallArgs;
6628 earlyArgs != nullptr;
6629 (earlyArgIndex++, earlyArgs = earlyArgs->Rest()))
6631 GenTreePtr earlyArg = earlyArgs->Current();
6632 if (!earlyArg->IsNothingNode() && !earlyArg->IsArgPlaceHolderNode())
6634 if ((earlyArg->gtFlags & GTF_LATE_ARG) != 0)
6636 // This is a setup node so we need to hoist it.
6637 GenTreePtr earlyArgStmt = gtNewStmt(earlyArg, callILOffset);
6638 fgInsertStmtBefore(block, earlyArgInsertionPoint, earlyArgStmt);
6642 // This is an actual argument that needs to be assigned to the corresponding caller parameter.
6643 fgArgTabEntryPtr curArgTabEntry = gtArgEntryByArgNum(recursiveTailCall, earlyArgIndex);
6644 GenTreePtr paramAssignStmt = fgAssignRecursiveCallArgToCallerParam(earlyArg, curArgTabEntry, block, callILOffset,
6645 tmpAssignmentInsertionPoint, paramAssignmentInsertionPoint);
6646 if ((tmpAssignmentInsertionPoint == last) && (paramAssignStmt != nullptr))
6648 // All temp assignments will happen before the first param assignment.
6649 tmpAssignmentInsertionPoint = paramAssignStmt;
6655 // Process late args.
6656 int lateArgIndex = 0;
6657 for (GenTreeArgList* lateArgs = recursiveTailCall->gtCallLateArgs;
6658 lateArgs != nullptr;
6659 (lateArgIndex++, lateArgs = lateArgs->Rest()))
6661 // A late argument is an actual argument that needs to be assigned to the corresponding caller's parameter.
6662 GenTreePtr lateArg = lateArgs->Current();
6663 fgArgTabEntryPtr curArgTabEntry = gtArgEntryByLateArgIndex(recursiveTailCall, lateArgIndex);
6664 GenTreePtr paramAssignStmt = fgAssignRecursiveCallArgToCallerParam(lateArg, curArgTabEntry, block, callILOffset,
6665 tmpAssignmentInsertionPoint, paramAssignmentInsertionPoint);
6667 if ((tmpAssignmentInsertionPoint == last) && (paramAssignStmt != nullptr))
6669 // All temp assignments will happen before the first param assignment.
6670 tmpAssignmentInsertionPoint = paramAssignStmt;
6674 // If the method has starg.s 0 or ldarga.s 0 a special local (lvaArg0Var) is created so that
6675 // compThisArg stays immutable. Normally it's assigned in fgFirstBBScratch block. Since that
6676 // block won't be in the loop (it's assumed to have no predecessors), we need to update the special local here.
6677 if (!info.compIsStatic && (lvaArg0Var != info.compThisArg))
6679 var_types thisType = lvaTable[info.compThisArg].TypeGet();
6680 GenTreePtr arg0 = gtNewLclvNode(lvaArg0Var, thisType);
6681 GenTreePtr arg0Assignment = gtNewAssignNode(arg0, gtNewLclvNode(info.compThisArg, thisType));
6682 GenTreePtr arg0AssignmentStmt = gtNewStmt(arg0Assignment, callILOffset);
6683 fgInsertStmtBefore(block, paramAssignmentInsertionPoint, arg0AssignmentStmt);
6687 fgRemoveStmt(block, last);
6689 // Set the loop edge.
6690 block->bbJumpKind = BBJ_ALWAYS;
6691 block->bbJumpDest = fgFirstBBisScratch() ? fgFirstBB->bbNext : fgFirstBB;
6692 fgAddRefPred(block->bbJumpDest, block);
6693 block->bbFlags &= ~BBF_HAS_JMP;
6696 //------------------------------------------------------------------------------
6697 // fgAssignRecursiveCallArgToCallerParam : Assign argument to a recursive call to the corresponding caller parameter.
6701 // arg - argument to assign
6702 // argTabEntry - argument table entry corresponding to arg
6703 // block --- basic block the call is in
6704 // callILOffset - IL offset of the call
6705 // tmpAssignmentInsertionPoint - tree before which temp assignment should be inserted (if necessary)
6706 // paramAssignmentInsertionPoint - tree before which parameter assignment should be inserted
6709 // parameter assignment statement if one was inserted; nullptr otherwise.
6711 GenTreePtr Compiler::fgAssignRecursiveCallArgToCallerParam(GenTreePtr arg,
6712 fgArgTabEntryPtr argTabEntry,
6714 IL_OFFSETX callILOffset,
6715 GenTreePtr tmpAssignmentInsertionPoint,
6716 GenTreePtr paramAssignmentInsertionPoint)
6718 // Call arguments should be assigned to temps first and then the temps should be assigned to parameters because
6719 // some argument trees may reference parameters directly.
6721 GenTreePtr argInTemp = nullptr;
6722 unsigned originalArgNum = argTabEntry->argNum;
6723 bool needToAssignParameter = true;
6725 // TODO-CQ: enable calls with struct arguments passed in registers.
6726 noway_assert(!varTypeIsStruct(arg->TypeGet()));
6728 if ((argTabEntry->isTmp) || arg->IsCnsIntOrI() || arg->IsCnsFltOrDbl())
6730 // The argument is already assigned to a temp or is a const.
6733 else if (arg->OperGet() == GT_LCL_VAR)
6735 unsigned lclNum = arg->AsLclVar()->gtLclNum;
6736 LclVarDsc * varDsc = &lvaTable[lclNum];
6737 if (!varDsc->lvIsParam)
6739 // The argument is a non-parameter local so it doesn't need to be assigned to a temp.
6742 else if (lclNum == originalArgNum)
6744 // The argument is the same parameter local that we were about to assign so
6745 // we can skip the assignment.
6746 needToAssignParameter = false;
6750 // TODO: We don't need temp assignments if we can prove that the argument tree doesn't involve
6751 // any caller parameters. Some common cases are handled above but we may be able to eliminate
6752 // more temp assignments.
6754 GenTreePtr paramAssignStmt = nullptr;
6755 if (needToAssignParameter)
6757 if (argInTemp == nullptr)
6759 // The argument is not assigned to a temp. We need to create a new temp and insert an assignment.
6760 // TODO: we can avoid a temp assignment if we can prove that the argument tree
6761 // doesn't involve any caller parameters.
6762 unsigned tmpNum = lvaGrabTemp(true DEBUGARG("arg temp"));
6763 GenTreePtr tempSrc = arg;
6764 GenTreePtr tempDest = gtNewLclvNode(tmpNum, tempSrc->gtType);
6765 GenTreePtr tmpAssignNode = gtNewAssignNode(tempDest, tempSrc);
6766 GenTreePtr tmpAssignStmt = gtNewStmt(tmpAssignNode, callILOffset);
6767 fgInsertStmtBefore(block, tmpAssignmentInsertionPoint, tmpAssignStmt);
6768 argInTemp = gtNewLclvNode(tmpNum, tempSrc->gtType);
6771 // Now assign the temp to the parameter.
6772 LclVarDsc *paramDsc = lvaTable + originalArgNum;
6773 assert(paramDsc->lvIsParam);
6774 GenTreePtr paramDest = gtNewLclvNode(originalArgNum, paramDsc->lvType);
6775 GenTreePtr paramAssignNode = gtNewAssignNode(paramDest, argInTemp);
6776 paramAssignStmt = gtNewStmt(paramAssignNode, callILOffset);
6778 fgInsertStmtBefore(block, paramAssignmentInsertionPoint, paramAssignStmt);
6780 return paramAssignStmt;
6783 /*****************************************************************************
6785 * Transform the given GT_CALL tree for code generation.
6788 GenTreePtr Compiler::fgMorphCall(GenTreeCall* call)
6790 if (call->CanTailCall())
6792 // It should either be an explicit (i.e. tail prefixed) or an implicit tail call
6793 assert(call->IsTailPrefixedCall() ^ call->IsImplicitTailCall());
6795 // It cannot be an inline candidate
6796 assert(!call->IsInlineCandidate());
6798 const char * szFailReason = nullptr;
6799 bool hasStructParam = false;
6800 if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC)
6802 szFailReason = "Might turn into an intrinsic";
6805 if (opts.compNeedSecurityCheck)
6807 szFailReason = "Needs security check";
6809 else if (compLocallocUsed)
6811 szFailReason = "Localloc used";
6813 #ifdef _TARGET_AMD64_
6814 // Needed for Jit64 compat.
6815 // In future, enabling tail calls from methods that need GS cookie check
6816 // would require codegen side work to emit GS cookie check before a tail
6818 else if (getNeedsGSSecurityCookie())
6820 szFailReason = "GS Security cookie check";
6824 // DDB 99324: Just disable tailcall under compGcChecks stress mode.
6825 else if (opts.compGcChecks)
6827 szFailReason = "GcChecks";
6830 #if FEATURE_TAILCALL_OPT
6833 // We are still not sure whether it can be a tail call. Because, when converting
6834 // a call to an implicit tail call, we must check that there are no locals with
6835 // their address taken. If this is the case, we have to assume that the address
6836 // has been leaked and the current stack frame must live until after the final
6839 // Verify that none of vars has lvHasLdAddrOp or lvAddrExposed bit set. Note
6840 // that lvHasLdAddrOp is much more conservative. We cannot just base it on
6841 // lvAddrExposed alone since it is not guaranteed to be set on all VarDscs
6842 // during morph stage. The reason for also checking lvAddrExposed is that in case
6843 // of vararg methods user args are marked as addr exposed but not lvHasLdAddrOp.
6844 // The combination of lvHasLdAddrOp and lvAddrExposed though conservative allows us
6845 // never to be incorrect.
6847 // TODO-Throughput: have a compiler level flag to indicate whether method has vars whose
6848 // address is taken. Such a flag could be set whenever lvHasLdAddrOp or LvAddrExposed
6849 // is set. This avoids the need for iterating through all lcl vars of the current
6850 // method. Right now throughout the code base we are not consistently using 'set'
6851 // method to set lvHasLdAddrOp and lvAddrExposed flags.
6854 bool hasAddrExposedVars = false;
6855 bool hasStructPromotedParam = false;
6856 bool hasPinnedVars = false;
6858 for (varNum = 0, varDsc = lvaTable; varNum < lvaCount; varNum++, varDsc++)
6860 // If the method is marked as an explicit tail call we will skip the
6861 // following three hazard checks.
6862 // We still must check for any struct parameters and set 'hasStructParam'
6863 // so that we won't transform the recursive tail call into a loop.
6865 if (call->IsImplicitTailCall())
6867 if (varDsc->lvHasLdAddrOp || varDsc->lvAddrExposed)
6869 hasAddrExposedVars = true;
6872 if (varDsc->lvPromoted && varDsc->lvIsParam)
6874 hasStructPromotedParam = true;
6877 if (varDsc->lvPinned)
6879 // A tail call removes the method from the stack, which means the pinning
6880 // goes away for the callee. We can't allow that.
6881 hasPinnedVars = true;
6885 if (varTypeIsStruct(varDsc->TypeGet()) && varDsc->lvIsParam)
6887 hasStructParam = true;
6888 // This prevents transforming a recursive tail call into a loop
6889 // but doesn't prevent tail call optimization so we need to
6890 // look at the rest of parameters.
6895 if (hasAddrExposedVars)
6897 szFailReason = "Local address taken";
6899 if (hasStructPromotedParam)
6901 szFailReason = "Has Struct Promoted Param";
6905 szFailReason = "Has Pinned Vars";
6908 #endif // FEATURE_TAILCALL_OPT
6910 fgFixupStructReturn(call);
6912 var_types callType = call->TypeGet();
6914 // We have to ensure to pass the incoming retValBuf as the
6915 // outgoing one. Using a temp will not do as this function will
6916 // not regain control to do the copy.
6918 if (info.compRetBuffArg != BAD_VAR_NUM)
6920 noway_assert(callType == TYP_VOID);
6921 GenTreePtr retValBuf = call->gtCallArgs->gtOp.gtOp1;
6922 if (retValBuf->gtOper != GT_LCL_VAR ||
6923 retValBuf->gtLclVarCommon.gtLclNum != info.compRetBuffArg)
6925 szFailReason = "Need to copy return buffer";
6929 // If this is an opportunistic tail call and cannot be dispatched as
6930 // fast tail call, go the non-tail call route. This is done for perf
6933 // Avoid the cost of determining whether can be dispatched as fast tail
6934 // call if we already know that tail call cannot be honored for other
6936 bool canFastTailCall = false;
6937 if (szFailReason == nullptr)
6939 canFastTailCall = fgCanFastTailCall(call);
6940 if (!canFastTailCall)
6942 // Implicit or opportunistic tail calls are always dispatched via fast tail call
6943 // mechanism and never via tail call helper for perf.
6944 if (call->IsImplicitTailCall())
6946 szFailReason = "Opportunistic tail call cannot be dispatched as epilog+jmp";
6948 #ifndef LEGACY_BACKEND
6949 else if (!call->IsVirtualStub() && call->HasNonStandardArgs())
6951 // If we are here, it means that the call is an explicitly ".tail" prefixed and cannot be
6952 // dispatched as a fast tail call.
6954 // Methods with non-standard args will have indirection cell or cookie param passed
6955 // in callee trash register (e.g. R11). Tail call helper doesn't preserve it before
6956 // tail calling the target method and hence ".tail" prefix on such calls needs to be
6959 // Exception to the above rule: although Virtual Stub Dispatch (VSD) calls though require
6960 // extra stub param (e.g. in R11 on Amd64), they can still be called via tail call helper.
6961 // This is done by by adding stubAddr as an additional arg before the original list of
6962 // args. For more details see fgMorphTailCall() and CreateTailCallCopyArgsThunk()
6963 // in Stublinkerx86.cpp.
6964 szFailReason = "Method with non-standard args passed in callee trash register cannot be tail called via helper";
6966 #ifdef _TARGET_ARM64_
6969 // NYI - TAILCALL_RECURSIVE/TAILCALL_HELPER.
6970 // So, bail out if we can't make fast tail call.
6971 szFailReason = "Non-qualified fast tail call";
6974 #endif //LEGACY_BACKEND
6978 // Clear these flags before calling fgMorphCall() to avoid recursion.
6979 bool isTailPrefixed = call->IsTailPrefixedCall();
6980 call->gtCallMoreFlags &= ~GTF_CALL_M_EXPLICIT_TAILCALL;
6982 #if FEATURE_TAILCALL_OPT
6983 call->gtCallMoreFlags &= ~GTF_CALL_M_IMPLICIT_TAILCALL;
6987 if (!canFastTailCall && szFailReason == nullptr)
6989 szFailReason = "Non fast tail calls disabled for PAL based systems.";
6991 #endif // FEATURE_PAL
6993 if (szFailReason != nullptr)
6997 printf("\nRejecting tail call late for call ");
6999 printf(": %s\n", szFailReason);
7003 // for non user funcs, we have no handles to report
7004 info.compCompHnd->reportTailCallDecision(nullptr,
7005 (call->gtCallType == CT_USER_FUNC) ? call->gtCallMethHnd : nullptr,
7006 isTailPrefixed, TAILCALL_FAIL, szFailReason);
7011 #if !FEATURE_TAILCALL_OPT_SHARED_RETURN
7012 // We enable shared-ret tail call optimization for recursive calls even if
7013 // FEATURE_TAILCALL_OPT_SHARED_RETURN is not defined.
7014 if (gtIsRecursiveCall(call))
7017 // Many tailcalls will have call and ret in the same block, and thus be BBJ_RETURN,
7018 // but if the call falls through to a ret, and we are doing a tailcall, change it here.
7019 if (compCurBB->bbJumpKind != BBJ_RETURN)
7020 compCurBB->bbJumpKind = BBJ_RETURN;
7024 // Set this flag before calling fgMorphCall() to prevent inlining this call.
7025 call->gtCallMoreFlags |= GTF_CALL_M_TAILCALL;
7027 bool fastTailCallToLoop = false;
7028 #if FEATURE_TAILCALL_OPT
7029 // TODO-CQ: enable the transformation when the method has a struct parameter that can be passed in a register
7030 // or return type is a struct that can be passed in a register.
7032 // TODO-CQ: if the method being compiled requires generic context reported in gc-info (either through
7033 // hidden generic context param or through keep alive thisptr), then while transforming a recursive
7034 // call to such a method requires that the generic context stored on stack slot be updated. Right now,
7035 // fgMorphRecursiveFastTailCallIntoLoop() is not handling update of generic context while transforming
7036 // a recursive call into a loop. Another option is to modify gtIsRecursiveCall() to check that the
7037 // generic type parameters of both caller and callee generic method are the same.
7038 if (opts.compTailCallLoopOpt &&
7040 gtIsRecursiveCall(call) &&
7041 !lvaReportParamTypeArg() &&
7042 !lvaKeepAliveAndReportThis() &&
7043 !call->IsVirtual() &&
7045 !varTypeIsStruct(call->TypeGet()))
7047 call->gtCallMoreFlags |= GTF_CALL_M_TAILCALL_TO_LOOP;
7048 fastTailCallToLoop = true;
7052 // Do some target-specific transformations (before we process the args, etc.)
7053 // This is needed only for tail prefixed calls that cannot be dispatched as
7055 if (!canFastTailCall)
7057 fgMorphTailCall(call);
7060 // Implementation note : If we optimize tailcall to do a direct jump
7061 // to the target function (after stomping on the return address, etc),
7062 // without using CORINFO_HELP_TAILCALL, we have to make certain that
7063 // we don't starve the hijacking logic (by stomping on the hijacked
7064 // return address etc).
7066 // At this point, we are committed to do the tailcall.
7067 compTailCallUsed = true;
7069 CorInfoTailCall tailCallResult;
7071 if (fastTailCallToLoop)
7073 tailCallResult = TAILCALL_RECURSIVE;
7075 else if (canFastTailCall)
7077 tailCallResult = TAILCALL_OPTIMIZED;
7081 tailCallResult = TAILCALL_HELPER;
7084 // for non user funcs, we have no handles to report
7085 info.compCompHnd->reportTailCallDecision(nullptr,
7086 (call->gtCallType == CT_USER_FUNC) ? call->gtCallMethHnd : nullptr,
7091 // As we will actually call CORINFO_HELP_TAILCALL, set the callTyp to TYP_VOID.
7092 // to avoid doing any extra work for the return value.
7093 call->gtType = TYP_VOID;
7098 printf("\nGTF_CALL_M_TAILCALL bit set for call ");
7101 if (fastTailCallToLoop)
7103 printf("\nGTF_CALL_M_TAILCALL_TO_LOOP bit set for call ");
7110 GenTreePtr stmtExpr = fgMorphStmt->gtStmt.gtStmtExpr;
7111 bool deleteReturn = false;
7112 if (info.compRetBuffArg != BAD_VAR_NUM)
7114 // In this case we simply have a call followed by a return.
7115 noway_assert(fgMorphStmt->gtNext->gtStmt.gtStmtExpr->gtOper == GT_RETURN);
7116 deleteReturn = true;
7118 else if ((stmtExpr->gtOper == GT_ASG) && (fgMorphStmt->gtNext != nullptr))
7120 GenTreePtr nextStmtExpr = fgMorphStmt->gtNext->gtStmt.gtStmtExpr;
7121 noway_assert(nextStmtExpr->gtOper == GT_RETURN);
7122 // In this case we have an assignment of the result of the call, and then a return of the result of the assignment.
7123 // This can occur if impSpillStackEnsure() has introduced an assignment to a temp.
7124 noway_assert(stmtExpr->gtGetOp1()->OperIsLocal() &&
7125 nextStmtExpr->OperGet() == GT_RETURN &&
7126 nextStmtExpr->gtGetOp1() != nullptr &&
7127 nextStmtExpr->gtGetOp1()->OperIsLocal() &&
7128 stmtExpr->gtGetOp1()->AsLclVarCommon()->gtLclNum == nextStmtExpr->gtGetOp1()->AsLclVarCommon()->gtLclNum);
7129 deleteReturn = true;
7133 fgRemoveStmt(compCurBB, fgMorphStmt->gtNext);
7136 // For void calls, we would have created a GT_CALL in the stmt list.
7137 // For non-void calls, we would have created a GT_RETURN(GT_CAST(GT_CALL)).
7138 // For calls returning structs, we would have a void call, followed by a void return.
7139 // For debuggable code, it would be an assignment of the call to a temp
7140 // We want to get rid of any of this extra trees, and just leave
7143 bool tailCallFollowedByPopAndRet = false;
7147 noway_assert((stmtExpr->gtOper == GT_CALL && stmtExpr == call) || // Either a call stmt
7148 (stmtExpr->gtOper == GT_RETURN && (stmtExpr->gtOp.gtOp1 == call || stmtExpr->gtOp.gtOp1->gtOp.gtOp1 == call)) || // GT_RETURN(GT_CALL(..))
7149 (stmtExpr->gtOper == GT_ASG && stmtExpr->gtOp.gtOp2 == call)); // or var = call
7152 #ifdef _TARGET_AMD64_
7153 if ((stmtExpr->gtOper == GT_CALL) && (fgMorphStmt->gtNext != nullptr))
7155 // We have a stmt node after a tail call node. This must be a tail call occuring
7156 // in the following IL pattern
7160 // Since tail prefix is honored, we can get rid of the remaining two stmts
7161 // corresponding to pop and ret. Note that 'pop' may or may not result in
7162 // a new statement (see impImportBlockCode() for details).
7163 stmt = fgMorphStmt->gtNext;
7164 if (stmt->gtNext != nullptr)
7166 // We have a pop tree.
7167 // It must be side effect free.
7168 GenTreePtr ret = stmt->gtNext;
7169 noway_assert((stmt->gtStmt.gtStmtExpr->gtFlags & GTF_ALL_EFFECT) == 0);
7170 fgRemoveStmt(compCurBB, stmt);
7173 noway_assert(stmt->gtStmt.gtStmtExpr->gtOper == GT_RETURN);
7174 fgRemoveStmt(compCurBB, stmt);
7176 tailCallFollowedByPopAndRet = true;
7178 #else //!TARGET_AMD64_
7181 noway_assert(fgMorphStmt->gtNext == nullptr);
7184 #endif //!_TARGET_AMD64_
7186 fgMorphStmt->gtStmt.gtStmtExpr = call;
7188 // Tail call via helper: The VM can't use return address hijacking if we're
7189 // not going to return and the helper doesn't have enough info to safely poll,
7190 // so we poll before the tail call, if the block isn't already safe. Since
7191 // tail call via helper is a slow mechanism it doen't matter whether we emit
7192 // GC poll. This is done to be in parity with Jit64. Also this avoids GC info
7193 // size increase if all most all methods are expected to be tail calls (e.g. F#).
7195 // Note that we can avoid emitting GC-poll if we know that the current BB is
7196 // dominated by a Gc-SafePoint block. But we don't have dominator info at this
7197 // point. One option is to just add a place holder node for GC-poll (e.g. GT_GCPOLL)
7198 // here and remove it in lowering if the block is dominated by a GC-SafePoint. For
7199 // now it not clear whether optimizing slow tail calls is worth the effort. As a
7200 // low cost check, we check whether the first and current basic blocks are
7203 // Fast Tail call as epilog+jmp - No need to insert GC-poll. Instead, fgSetBlockOrder()
7204 // is going to mark the method as fully interruptible if the block containing this tail
7205 // call is reachable without executing any call.
7206 if (canFastTailCall ||
7207 (fgFirstBB->bbFlags & BBF_GC_SAFE_POINT) ||
7208 (compCurBB->bbFlags & BBF_GC_SAFE_POINT) ||
7209 !fgCreateGCPoll(GCPOLL_INLINE, compCurBB))
7211 // We didn't insert a poll block, so we need to morph the call now
7212 // (Normally it will get morphed when we get to the split poll block)
7213 GenTreePtr temp = fgMorphCall(call);
7214 noway_assert(temp == call);
7217 // Tail call via helper: we just call CORINFO_HELP_TAILCALL, and it jumps to
7218 // the target. So we don't need an epilog - just like CORINFO_HELP_THROW.
7220 // Fast tail call: in case of fast tail calls, we need a jmp epilog and
7221 // hence mark it as BBJ_RETURN with BBF_JMP flag set.
7222 noway_assert(compCurBB->bbJumpKind == BBJ_RETURN);
7224 if (canFastTailCall)
7226 compCurBB->bbFlags |= BBF_HAS_JMP;
7230 compCurBB->bbJumpKind = BBJ_THROW;
7233 // For non-void calls, we return a place holder which will be
7234 // used by the parent GT_RETURN node of this call. This should
7235 // not be done for tail calls occuring in the following IL pattern,
7236 // since this pattern is supported only in void returning methods.
7241 GenTree* result = call;
7243 if (!tailCallFollowedByPopAndRet && (callType != TYP_VOID) && info.compRetType != TYP_VOID)
7246 // Return a dummy node, as the return is already removed.
7247 if (callType == TYP_STRUCT)
7249 // This is a HFA, use float 0.
7250 callType = TYP_FLOAT;
7252 #elif defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
7253 // Return a dummy node, as the return is already removed.
7254 if (varTypeIsStruct(callType))
7256 // This is a register-returned struct. Return a 0.
7257 // The actual return registers are hacked in lower and the register allocator.
7262 // Return a dummy node, as the return is already removed.
7263 if (varTypeIsSIMD(callType))
7265 callType = TYP_DOUBLE;
7268 result = gtNewZeroConNode(genActualType(callType));
7269 result = fgMorphTree(result);
7277 if ((call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) == 0 &&
7278 (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_VIRTUAL_FUNC_PTR)
7279 #ifdef FEATURE_READYTORUN_COMPILER
7280 || call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_READYTORUN_VIRTUAL_FUNC_PTR)
7283 (call == fgMorphStmt->gtStmt.gtStmtExpr))
7285 // This is call to CORINFO_HELP_VIRTUAL_FUNC_PTR with ignored result.
7286 // Transform it into a null check.
7288 GenTreePtr thisPtr = call->gtCallArgs->gtOp.gtOp1;
7290 GenTreePtr nullCheck = gtNewOperNode(GT_IND, TYP_I_IMPL, thisPtr);
7291 nullCheck->gtFlags |= GTF_EXCEPT;
7293 return fgMorphTree(nullCheck);
7296 noway_assert(call->gtOper == GT_CALL);
7299 // Only count calls once (only in the global morph phase)
7303 if (call->gtCallType == CT_INDIRECT)
7306 optIndirectCallCount++;
7308 else if (call->gtCallType == CT_USER_FUNC)
7311 if (call->IsVirtual())
7312 optIndirectCallCount++;
7316 // Couldn't inline - remember that this BB contains method calls
7318 // If this is a 'regular' call, mark the basic block as
7319 // having a call (for computing full interruptibility).
7321 // Amd64 note: If this is a fast tail call then don't count it as a call
7322 // since we don't insert GC-polls but instead make the method fully GC
7324 #ifdef _TARGET_AMD64_
7325 if (!call->IsFastTailCall())
7328 if (call->gtCallType == CT_INDIRECT)
7330 compCurBB->bbFlags |= BBF_GC_SAFE_POINT;
7332 else if (call->gtCallType == CT_USER_FUNC)
7334 if ((call->gtCallMoreFlags & GTF_CALL_M_NOGCCHECK) == 0)
7335 compCurBB->bbFlags |= BBF_GC_SAFE_POINT;
7337 // otherwise we have a CT_HELPER
7340 // Morph Type.op_Equality and Type.op_Inequality
7341 // We need to do this before the arguments are morphed
7342 if ((call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC))
7344 CorInfoIntrinsics methodID = info.compCompHnd->getIntrinsicID(call->gtCallMethHnd);
7346 genTreeOps simpleOp = GT_CALL;
7347 if (methodID == CORINFO_INTRINSIC_TypeEQ)
7349 else if (methodID == CORINFO_INTRINSIC_TypeNEQ)
7352 if (simpleOp == GT_EQ || simpleOp == GT_NE)
7354 noway_assert(call->TypeGet() == TYP_INT);
7356 // Check for GetClassFromHandle(handle) and obj.GetType() both of which will only return RuntimeType objects.
7357 // Then if either operand is one of these two calls we can simplify op_Equality/op_Inequality to GT_NE/GT_NE:
7358 // One important invariance that should never change is that type equivalency is always equivalent to object
7359 // identity equality for runtime type objects in reflection. This is also reflected in RuntimeTypeHandle::TypeEquals.
7360 // If this invariance would ever be broken, we need to remove the optimization below.
7362 GenTreePtr op1 = call->gtCallArgs->gtOp.gtOp1;
7363 GenTreePtr op2 = call->gtCallArgs->gtOp.gtOp2->gtOp.gtOp1;
7365 if (gtCanOptimizeTypeEquality(op1) || gtCanOptimizeTypeEquality(op2))
7367 GenTreePtr compare = gtNewOperNode(simpleOp, TYP_INT, op1, op2);
7369 // fgMorphSmpOp will further optimize the following patterns:
7370 // 1. typeof(...) == typeof(...)
7371 // 2. typeof(...) == obj.GetType()
7372 return fgMorphTree(compare);
7377 // Make sure that return buffers containing GC pointers that aren't too large are pointers into the stack.
7378 GenTreePtr origDest = nullptr; // Will only become non-null if we do the transformation (and thus require copy-back).
7379 unsigned retValTmpNum = BAD_VAR_NUM;
7380 CORINFO_CLASS_HANDLE structHnd = nullptr;
7381 if (call->HasRetBufArg() &&
7382 call->gtCallLateArgs == nullptr) // Don't do this if we're re-morphing (which will make late args non-null).
7384 // We're enforcing the invariant that return buffers pointers (at least for
7385 // struct return types containing GC pointers) are never pointers into the heap.
7386 // The large majority of cases are address of local variables, which are OK.
7387 // Otherwise, allocate a local of the given struct type, pass its address,
7388 // then assign from that into the proper destination. (We don't need to do this
7389 // if we're passing the caller's ret buff arg to the callee, since the caller's caller
7390 // will maintain the same invariant.)
7392 GenTreePtr dest = call->gtCallArgs->gtOp.gtOp1;
7393 assert(dest->OperGet() != GT_ARGPLACE); // If it was, we'd be in a remorph, which we've already excluded above.
7394 if (dest->gtType == TYP_BYREF
7395 && !(dest->OperGet() == GT_ADDR && dest->gtOp.gtOp1->OperGet() == GT_LCL_VAR))
7397 // We'll exempt helper calls from this, assuming that the helper implementation
7398 // follows the old convention, and does whatever barrier is required.
7399 if (call->gtCallType != CT_HELPER)
7401 structHnd = call->gtRetClsHnd;
7402 if (info.compCompHnd->isStructRequiringStackAllocRetBuf(structHnd)
7403 && !((dest->OperGet() == GT_LCL_VAR || dest->OperGet() == GT_REG_VAR)
7404 && dest->gtLclVar.gtLclNum == info.compRetBuffArg))
7408 retValTmpNum = lvaGrabTemp(true DEBUGARG("substitute local for ret buff arg"));
7409 lvaSetStruct(retValTmpNum, structHnd, true);
7410 dest = gtNewOperNode(GT_ADDR, TYP_BYREF, gtNewLclvNode(retValTmpNum, TYP_STRUCT));
7415 call->gtCallArgs->gtOp.gtOp1 = dest;
7418 /* Process the "normal" argument list */
7419 call = fgMorphArgs(call);
7421 // Optimize get_ManagedThreadId(get_CurrentThread)
7422 noway_assert(call->gtOper == GT_CALL);
7424 // Morph stelem.ref helper call to store a null value, into a store into an array without the helper.
7425 // This needs to be done after the arguments are morphed to ensure constant propagation has already taken place.
7426 if ((call->gtCallType == CT_HELPER) && (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_ARRADDR_ST)))
7428 GenTreePtr value = gtArgEntryByArgNum(call, 2)->node;
7430 if (value->OperGet() == GT_CNS_INT && value->AsIntConCommon()->IconValue() == 0)
7432 GenTreePtr arr = gtArgEntryByArgNum(call, 0)->node;
7433 GenTreePtr index = gtArgEntryByArgNum(call, 1)->node;
7435 arr = gtClone(arr, true);
7438 index = gtClone(index, true);
7439 if (index != nullptr)
7441 value = gtClone(value);
7442 noway_assert(value != nullptr);
7444 GenTreePtr nullCheckedArr = impCheckForNullPointer(arr);
7445 GenTreePtr arrIndexNode = gtNewIndexRef(TYP_REF, nullCheckedArr, index);
7446 GenTreePtr arrStore = gtNewAssignNode(arrIndexNode, value);
7447 arrStore->gtFlags |= GTF_ASG;
7449 return fgMorphTree(arrStore);
7455 if ((call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) &&
7456 info.compCompHnd->getIntrinsicID(call->gtCallMethHnd) == CORINFO_INTRINSIC_GetManagedThreadId)
7458 noway_assert(origDest == NULL);
7459 noway_assert(call->gtCallLateArgs->gtOp.gtOp1 != NULL);
7461 GenTreePtr innerCall = call->gtCallLateArgs->gtOp.gtOp1;
7463 if (innerCall->gtOper == GT_CALL &&
7464 (innerCall->gtCall.gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) &&
7465 info.compCompHnd->getIntrinsicID(innerCall->gtCall.gtCallMethHnd) == CORINFO_INTRINSIC_GetCurrentManagedThread)
7467 // substitute expression with call to helper
7468 GenTreePtr newCall = gtNewHelperCallNode(CORINFO_HELP_GETCURRENTMANAGEDTHREADID, TYP_INT, 0);
7469 JITDUMP("get_ManagedThreadId(get_CurrentThread) folding performed\n");
7470 return fgMorphTree(newCall);
7474 if (origDest != NULL)
7476 GenTreePtr retValVarAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, gtNewLclvNode(retValTmpNum, TYP_STRUCT));
7477 // If the origDest expression was an assignment to a variable, it might be to an otherwise-unused
7478 // var, which would allow the whole assignment to be optimized away to a NOP. So in that case, make the
7479 // origDest into a comma that uses the var. Note that the var doesn't have to be a temp for this to
7481 if (origDest->OperGet() == GT_ASG)
7483 if (origDest->gtOp.gtOp1->OperGet() == GT_LCL_VAR)
7485 GenTreePtr var = origDest->gtOp.gtOp1;
7486 origDest = gtNewOperNode(GT_COMMA, var->TypeGet(), origDest,
7487 gtNewLclvNode(var->gtLclVar.gtLclNum, var->TypeGet()));
7490 GenTreePtr copyBlk = gtNewCpObjNode(origDest, retValVarAddr, structHnd, false);
7491 copyBlk = fgMorphTree(copyBlk);
7492 GenTree* result = gtNewOperNode(GT_COMMA, TYP_VOID, call, copyBlk);
7494 result->gtFlags |= GTF_MORPHED;
7503 /*****************************************************************************
7505 * Transform the given GTK_CONST tree for code generation.
7508 GenTreePtr Compiler::fgMorphConst(GenTreePtr tree)
7510 noway_assert(tree->OperKind() & GTK_CONST);
7512 /* Clear any exception flags or other unnecessary flags
7513 * that may have been set before folding this node to a constant */
7515 tree->gtFlags &= ~(GTF_ALL_EFFECT | GTF_REVERSE_OPS);
7517 if (tree->OperGet() != GT_CNS_STR)
7520 // TODO-CQ: Do this for compCurBB->isRunRarely(). Doing that currently will
7521 // guarantee slow performance for that block. Instead cache the return value
7522 // of CORINFO_HELP_STRCNS and go to cache first giving reasonable perf.
7524 if (compCurBB->bbJumpKind == BBJ_THROW)
7526 CorInfoHelpFunc helper = info.compCompHnd->getLazyStringLiteralHelper(tree->gtStrCon.gtScpHnd);
7527 if (helper != CORINFO_HELP_UNDEF)
7529 // For un-important blocks, we want to construct the string lazily
7531 GenTreeArgList *args;
7532 if (helper == CORINFO_HELP_STRCNS_CURRENT_MODULE)
7534 args = gtNewArgList(gtNewIconNode(RidFromToken(tree->gtStrCon.gtSconCPX), TYP_INT));
7538 args = gtNewArgList(gtNewIconNode(RidFromToken(tree->gtStrCon.gtSconCPX), TYP_INT),
7539 gtNewIconEmbScpHndNode(tree->gtStrCon.gtScpHnd));
7543 tree = gtNewHelperCallNode(helper, TYP_REF, 0, args);
7544 return fgMorphTree(tree);
7548 assert(tree->gtStrCon.gtScpHnd == info.compScopeHnd || !IsUninitialized(tree->gtStrCon.gtScpHnd));
7551 InfoAccessType iat = info.compCompHnd->constructStringLiteral(tree->gtStrCon.gtScpHnd,
7552 tree->gtStrCon.gtSconCPX,
7555 tree = gtNewStringLiteralNode(iat, pValue);
7557 return fgMorphTree(tree);
7560 /*****************************************************************************
7562 * Transform the given GTK_LEAF tree for code generation.
7565 GenTreePtr Compiler::fgMorphLeaf(GenTreePtr tree)
7567 noway_assert(tree->OperKind() & GTK_LEAF);
7569 if (tree->gtOper == GT_LCL_VAR)
7571 return fgMorphLocalVar(tree);
7574 else if (tree->gtOper == GT_LCL_FLD)
7576 if (info.compIsVarArgs)
7578 GenTreePtr newTree = fgMorphStackArgForVarArgs(tree->gtLclFld.gtLclNum, tree->gtType, tree->gtLclFld.gtLclOffs);
7579 if (newTree != NULL)
7583 #endif // _TARGET_X86_
7584 else if (tree->gtOper == GT_FTN_ADDR)
7586 CORINFO_CONST_LOOKUP addrInfo;
7588 #ifdef FEATURE_READYTORUN_COMPILER
7589 if (tree->gtFptrVal.gtEntryPoint.addr != nullptr)
7591 addrInfo = tree->gtFptrVal.gtEntryPoint;
7596 info.compCompHnd->getFunctionFixedEntryPoint(tree->gtFptrVal.gtFptrMethod, &addrInfo);
7599 // Refer to gtNewIconHandleNode() as the template for constructing a constant handle
7601 tree->SetOper(GT_CNS_INT);
7602 tree->gtIntConCommon.SetIconValue(ssize_t(addrInfo.handle));
7603 tree->gtFlags |= GTF_ICON_FTN_ADDR;
7605 switch (addrInfo.accessType)
7608 tree = gtNewOperNode(GT_IND, TYP_I_IMPL, tree);
7609 tree->gtFlags |= GTF_IND_INVARIANT;
7614 tree = gtNewOperNode(GT_IND, TYP_I_IMPL, tree);
7618 tree = gtNewOperNode(GT_NOP, tree->TypeGet(), tree); // prevents constant folding
7622 noway_assert(!"Unknown addrInfo.accessType");
7625 return fgMorphTree(tree);
7632 void Compiler::fgAssignSetVarDef(GenTreePtr tree)
7634 GenTreeLclVarCommon* lclVarCmnTree;
7635 bool isEntire = false;
7636 if (tree->DefinesLocal(this, &lclVarCmnTree, &isEntire))
7640 lclVarCmnTree->gtFlags |= GTF_VAR_DEF;
7644 // We consider partial definitions to be modeled as uses followed by definitions.
7645 // This captures the idea that precedings defs are not necessarily made redundant
7646 // by this definition.
7647 lclVarCmnTree->gtFlags |= (GTF_VAR_DEF | GTF_VAR_USEASG);
7652 GenTreePtr Compiler::fgMorphOneAsgBlockOp(GenTreePtr tree)
7654 genTreeOps oper = tree->gtOper;
7656 // Only xxBlk opcodes are possible
7657 noway_assert(tree->OperIsBlkOp());
7659 GenTreePtr dest = tree->gtOp.gtOp1->gtOp.gtOp1; // Dest address
7660 GenTreePtr src = tree->gtOp.gtOp1->gtOp.gtOp2; // Src
7661 GenTreePtr blkShape = tree->gtOp.gtOp2; // [size/clsHnd]
7662 bool volatil = tree->AsBlkOp()->IsVolatile();
7664 GenTreePtr lclVarTree;
7666 // The dest must be an address
7667 noway_assert(genActualType(dest->gtType) == TYP_I_IMPL ||
7668 dest->gtType == TYP_BYREF);
7670 // For COPYBLK the src must be an address
7671 noway_assert(!tree->OperIsCopyBlkOp() ||
7672 (genActualType( src->gtType) == TYP_I_IMPL ||
7673 src->gtType == TYP_BYREF));
7675 // For INITBLK the src must be a TYP_INT
7676 noway_assert(oper != GT_INITBLK ||
7677 (genActualType( src->gtType) == TYP_INT));
7679 // The size must be an integer type
7680 noway_assert(varTypeIsIntegral(blkShape->gtType));
7682 CORINFO_CLASS_HANDLE clsHnd;
7684 var_types type = TYP_UNDEF;
7686 if (blkShape->gtOper != GT_CNS_INT)
7690 // importer introduces cpblk nodes with src = GT_ADDR(GT_SIMD)
7691 // The SIMD type in question could be Vector2f which is 8-bytes in size.
7692 // The below check is to make sure that we don't turn that copyblk
7693 // into a assignment, since rationalizer logic will transform the
7694 // copyblk apropriately. Otherwise, the transormation made in this
7695 // routine will prevent rationalizer logic and we might end up with
7696 // GT_ADDR(GT_SIMD) node post rationalization, leading to a noway assert
7698 if (src->OperGet() == GT_ADDR && src->gtGetOp1()->OperGet() == GT_SIMD)
7702 if (!blkShape->IsIconHandle())
7705 size = blkShape->gtIntCon.gtIconVal;
7707 /* A four byte BLK_COPY can be treated as an integer asignment */
7710 #ifdef _TARGET_64BIT_
7717 clsHnd = (CORINFO_CLASS_HANDLE) blkShape->gtIntCon.gtIconVal;
7718 size = roundUp(info.compCompHnd->getClassSize(clsHnd), sizeof(void*));
7720 // Since we round up, we are not handling the case where we have a
7721 // non-dword sized struct with GC pointers.
7722 // The EE currently does not allow this, but we may change. Lets assert it
7724 noway_assert(info.compCompHnd->getClassSize(clsHnd) == size);
7726 if (size == REGSIZE_BYTES)
7729 info.compCompHnd->getClassGClayout(clsHnd, &gcPtr);
7730 type = getJitGCType(gcPtr);
7735 // See if we can do a simple transformation:
7737 // GT_ASG <TYP_size>
7739 // GT_IND GT_IND or CNS_INT
7748 goto ONE_SIMPLE_ASG;
7751 goto ONE_SIMPLE_ASG;
7753 #ifdef _TARGET_64BIT_
7756 goto ONE_SIMPLE_ASG;
7757 #endif // _TARGET_64BIT_
7760 noway_assert(type != TYP_UNDEF);
7764 noway_assert(size <= REGSIZE_BYTES);
7766 // For INITBLK, a non constant source is not going to allow us to fiddle
7767 // with the bits to create a single assigment.
7769 if ((oper == GT_INITBLK) && (src->gtOper != GT_CNS_INT))
7774 if (impIsAddressInLocal(dest, &lclVarTree))
7776 #if LOCAL_ASSERTION_PROP
7777 // Kill everything about dest
7778 if (optLocalAssertionProp)
7780 if (optAssertionCount > 0)
7782 fgKillDependentAssertions(lclVarTree->gtLclVarCommon.gtLclNum DEBUGARG(tree));
7785 #endif // LOCAL_ASSERTION_PROP
7787 unsigned lclNum = lclVarTree->gtLclVarCommon.gtLclNum;
7788 // A previous incarnation of this code also required the local not to be
7789 // address-exposed(=taken). That seems orthogonal to the decision of whether
7790 // to do field-wise assignments: being address-exposed will cause it to be
7791 // "dependently" promoted, so it will be in the right memory location. One possible
7792 // further reason for avoiding field-wise stores is that the struct might have alignment-induced
7793 // holes, whose contents could be meaningful in unsafe code. If we decide that's a valid
7794 // concern, then we could compromise, and say that address-exposed + fields do not completely cover the memory
7795 // of the struct prevent field-wise assignments. Same situation exists for the "src" decision.
7796 if (varTypeIsStruct(lclVarTree) &&
7797 (lvaTable[lclNum].lvPromoted || lclVarIsSIMDType(lclNum)))
7800 // Let fgMorphInitBlock handle it. (Since we'll need to do field-var-wise assignments.)
7804 if (!varTypeIsFloating(lclVarTree->TypeGet()) &&
7805 size == genTypeSize(var_types(lvaTable[lclNum].lvType)))
7807 // Use the dest local var directly.
7809 type = lvaTable[lclNum].lvType; // Make the type used in the GT_IND node match
7811 // If the block operation had been a write to a local var of a small int type,
7812 // of the exact size of the small int type, and the var is NormalizeOnStore,
7813 // we would have labeled it GTF_VAR_USEASG, because the block operation wouldn't
7814 // have done that normalization. If we're now making it into an assignment,
7815 // the NormalizeOnStore will work, and it can be a full def.
7816 if (lvaTable[lclNum].lvNormalizeOnStore())
7818 dest->gtFlags &= (~GTF_VAR_USEASG);
7825 // Could be a non-promoted struct, or a floating point type local, or
7826 // an int subject to a partial write. Don't enregister.
7827 lvaSetVarDoNotEnregister(lclNum DEBUGARG(DNER_LocalField));
7828 // Fall through to indirect the dest node.
7830 // Mark the local var tree as a definition point of the local.
7831 lclVarTree->gtFlags |= GTF_VAR_DEF;
7832 if (size < lvaTable[lclNum].lvExactSize) // If it's not a full-width assignment....
7833 lclVarTree->gtFlags |= GTF_VAR_USEASG;
7836 // Check to ensure we are not creating a reducible *(& ... )
7837 if (dest->gtOper == GT_ADDR)
7839 GenTreePtr addrOp = dest->gtOp.gtOp1;
7840 // Ignore reinterpret casts between int/gc
7841 if ((addrOp->TypeGet() == type) ||
7842 (varTypeIsIntegralOrI(addrOp) && (genTypeSize(addrOp->TypeGet()) == size)))
7845 type = addrOp->TypeGet();
7850 /* Indirect the dest node */
7852 dest = gtNewOperNode(GT_IND, type, dest);
7854 /* As long as we don't have more information about the destination we
7855 have to assume it could live anywhere (not just in the GC heap). Mark
7856 the GT_IND node so that we use the correct write barrier helper in case
7857 the field is a GC ref.
7860 dest->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF | GTF_IND_TGTANYWHERE);
7865 dest->gtFlags |= GTF_DONT_CSE;
7867 if (tree->OperIsCopyBlkOp())
7869 if (impIsAddressInLocal(src, &lclVarTree))
7871 unsigned lclNum = lclVarTree->gtLclVarCommon.gtLclNum;
7872 if (varTypeIsStruct(lclVarTree) &&
7873 (lvaTable[lclNum].lvPromoted || lclVarIsSIMDType(lclNum)))
7875 // Let fgMorphCopyBlock handle it.
7879 if (!varTypeIsFloating(lclVarTree->TypeGet()) &&
7880 size == genTypeSize(genActualType(lclVarTree->TypeGet())))
7882 /* Use the src local var directly */
7888 #ifndef LEGACY_BACKEND
7890 // The source argument of the copyblk can potentially
7891 // be accessed only through indir(addr(lclVar))
7892 // or indir(lclVarAddr) in rational form and liveness
7893 // won't account for these uses. That said,
7894 // we have to mark this local as address exposed so
7895 // we don't delete it as a dead store later on.
7896 unsigned lclVarNum = lclVarTree->gtLclVarCommon.gtLclNum;
7897 lvaTable[lclVarNum].lvAddrExposed = true;
7898 lvaSetVarDoNotEnregister(lclVarNum DEBUGARG(DNER_AddrExposed));
7900 #else // LEGACY_BACKEND
7901 lvaSetVarDoNotEnregister(lclVarTree->gtLclVarCommon.gtLclNum DEBUGARG(DNER_LocalField));
7902 #endif // LEGACY_BACKEND
7904 // Fall through to indirect the src node.
7908 /* Indirect the src node */
7910 src = gtNewOperNode(GT_IND, type, src);
7911 src->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF | GTF_IND_TGTANYWHERE);
7916 src->gtFlags |= GTF_DONT_CSE;
7918 else // (oper == GT_INITBLK)
7922 size_t cns = src->gtIntCon.gtIconVal;
7928 #ifdef _TARGET_64BIT_
7933 #endif // _TARGET_64BIT_
7935 src->gtType = type; // Make the type used in the GT_IND node match for TYP_REF
7937 // if we are using an GT_INITBLK on a GC type the value being assigned has to be zero (null)
7938 assert(!varTypeIsGC(type) || (cns == 0));
7941 src->gtIntCon.gtIconVal = cns;
7945 /* Create the assignment node */
7947 result = gtNewAssignNode(dest, src);
7948 result->gtType = type;
7958 //------------------------------------------------------------------------
7959 // fgMorphInitBlock: Perform the Morphing of a GT_INITBLK node
7962 // tree - a tree node with a gtOper of GT_INITBLK
7963 // the child nodes for tree have already been Morphed
7966 // We can return the orginal GT_INITBLK unmodified (least desirable, but always correct)
7967 // We can return a single assignment, when fgMorphOneAsgBlockOp transforms it (most desirable)
7968 // If we have performed struct promotion of the Dest() then we will try to
7969 // perform a field by field assignment for each of the promoted struct fields
7972 // If we leave it as a GT_INITBLK we will call lvaSetVarDoNotEnregister() with a reason of DNER_BlockOp
7973 // if the Dest() is a a struct that has a "CustomLayout" and "ConstainsHoles" then we
7974 // can not use a field by field assignment and must the orginal GT_INITBLK unmodified.
7976 GenTreePtr Compiler::fgMorphInitBlock(GenTreePtr tree)
7978 noway_assert(tree->gtOper == GT_INITBLK);
7980 JITDUMP("\nfgMorphInitBlock:");
7982 GenTreePtr oneAsgTree = fgMorphOneAsgBlockOp(tree);
7985 JITDUMP(" using oneAsgTree.\n");
7990 GenTreeInitBlk* initBlkOp = tree->AsInitBlk();
7992 GenTreePtr destAddr = initBlkOp->Dest();
7993 GenTreePtr initVal = initBlkOp->InitVal();
7994 GenTreePtr blockSize = initBlkOp->Size();
7996 // The dest must be an address
7997 noway_assert(genActualType(destAddr->gtType) == TYP_I_IMPL ||
7998 destAddr->gtType == TYP_BYREF);
8000 // The size must be an integer type
8001 assert(varTypeIsIntegral(blockSize->gtType));
8003 unsigned blockWidth = 0;
8004 bool blockWidthIsConst = false;
8006 if (blockSize->IsCnsIntOrI())
8008 blockWidthIsConst = true;
8009 blockWidth = unsigned(blockSize->gtIntConCommon.IconValue());
8012 GenTreeLclVarCommon* lclVarTree = nullptr;
8014 FieldSeqNode* destFldSeq = nullptr;
8015 unsigned destLclNum = BAD_VAR_NUM;
8016 LclVarDsc * destLclVar = nullptr;
8017 bool destDoFldAsg = false;
8019 if (destAddr->IsLocalAddrExpr(this, &lclVarTree, &destFldSeq))
8021 destLclNum = lclVarTree->gtLclNum;
8022 destLclVar = &lvaTable[destLclNum];
8024 #if LOCAL_ASSERTION_PROP
8025 // Kill everything about destLclNum (and its field locals)
8026 if (optLocalAssertionProp)
8028 if (optAssertionCount > 0)
8030 fgKillDependentAssertions(destLclNum DEBUGARG(tree));
8033 #endif // LOCAL_ASSERTION_PROP
8035 if (destLclVar->lvPromoted && blockWidthIsConst)
8037 noway_assert(varTypeIsStruct(destLclVar));
8038 noway_assert(!opts.MinOpts());
8039 if (destLclVar->lvAddrExposed & destLclVar->lvContainsHoles)
8041 JITDUMP(" dest is address exposed");
8045 if (blockWidth == destLclVar->lvExactSize)
8047 JITDUMP(" (destDoFldAsg=true)");
8048 // We may decide later that a copyblk is required when this struct has holes
8049 destDoFldAsg = true;
8053 JITDUMP(" with mismatched size");
8059 // Can we use field by field assignment for the dest?
8060 if (destDoFldAsg && destLclVar->lvCustomLayout && destLclVar->lvContainsHoles)
8062 JITDUMP(" dest contains holes");
8063 destDoFldAsg = false;
8066 JITDUMP(destDoFldAsg ? " using field by field initialization.\n"
8067 : " this requires an InitBlock.\n");
8069 if (!destDoFldAsg && (destLclVar != nullptr))
8071 // If destLclVar is not a reg-sized non-field-addressed struct, set it as DoNotEnregister.
8072 if (!destLclVar->lvRegStruct)
8074 // Mark it as DoNotEnregister.
8075 lvaSetVarDoNotEnregister(destLclNum DEBUGARG(DNER_BlockOp));
8079 // Mark the dest struct as DoNotEnreg
8080 // when they are LclVar structs and we are using a CopyBlock
8081 // or the struct is not promoted
8085 #if CPU_USES_BLOCK_MOVE
8086 compBlkOpUsed = true;
8091 // The initVal must be a constant of TYP_INT
8092 noway_assert(initVal->OperGet() == GT_CNS_INT);
8093 noway_assert(genActualType(initVal->gtType) == TYP_INT);
8095 // The dest must be of a struct type.
8096 noway_assert(varTypeIsStruct(destLclVar));
8099 // Now, convert InitBlock to individual assignments
8107 unsigned fieldLclNum;
8108 unsigned fieldCnt = destLclVar->lvFieldCnt;
8110 for (unsigned i=0; i<fieldCnt; ++i)
8112 fieldLclNum = destLclVar->lvFieldLclStart + i;
8113 dest = gtNewLclvNode(fieldLclNum, lvaTable[fieldLclNum].TypeGet());
8115 noway_assert(destAddr->gtOp.gtOp1->gtOper == GT_LCL_VAR);
8116 // If it had been labeled a "USEASG", assignments to the the individual promoted fields are not.
8117 dest->gtFlags |= destAddr->gtOp.gtOp1->gtFlags & ~(GTF_NODE_MASK | GTF_VAR_USEASG);
8119 srcCopy = gtCloneExpr(initVal);
8120 noway_assert(srcCopy != nullptr);
8122 // need type of oper to be same as tree
8123 if (dest->gtType == TYP_LONG)
8125 srcCopy->ChangeOperConst(GT_CNS_NATIVELONG);
8126 // copy and extend the value
8127 srcCopy->gtIntConCommon.SetLngValue(initVal->gtIntConCommon.IconValue());
8128 /* Change the types of srcCopy to TYP_LONG */
8129 srcCopy->gtType = TYP_LONG;
8131 else if (varTypeIsFloating(dest->gtType))
8133 srcCopy->ChangeOperConst(GT_CNS_DBL);
8134 // setup the bit pattern
8135 memset(&srcCopy->gtDblCon.gtDconVal, (int)initVal->gtIntCon.gtIconVal, sizeof(srcCopy->gtDblCon.gtDconVal));
8136 /* Change the types of srcCopy to TYP_DOUBLE */
8137 srcCopy->gtType = TYP_DOUBLE;
8141 noway_assert(srcCopy->gtOper == GT_CNS_INT);
8142 noway_assert(srcCopy->TypeGet() == TYP_INT);
8143 // setup the bit pattern
8144 memset(&srcCopy->gtIntCon.gtIconVal, (int)initVal->gtIntCon.gtIconVal, sizeof(srcCopy->gtIntCon.gtIconVal));
8147 srcCopy->gtType = dest->TypeGet();
8149 asg = gtNewAssignNode(dest, srcCopy);
8151 #if LOCAL_ASSERTION_PROP
8152 if (optLocalAssertionProp)
8154 optAssertionGen(asg);
8156 #endif // LOCAL_ASSERTION_PROP
8160 tree = gtNewOperNode(GT_COMMA,
8173 tree->gtFlags |= GTF_MORPHED;
8177 printf("fgMorphInitBlock (after):\n");
8186 //------------------------------------------------------------------------
8187 // fgMorphCopyBlock: Perform the Morphing of a GT_COPYBLK and GT_COPYOBJ nodes
8190 // tree - a tree node with a gtOper of GT_COPYBLK or GT_COPYOBJ
8191 // the child nodes for tree have already been Morphed
8194 // We can return the orginal GT_COPYBLK or GT_COPYOBJ unmodified (least desirable, but always correct)
8195 // We can return a single assignment, when fgMorphOneAsgBlockOp transforms it (most desirable)
8196 // If we have performed struct promotion of the Source() or the Dest() then we will try to
8197 // perform a field by field assignment for each of the promoted struct fields
8200 // If we leave it as a GT_COPYBLK or GT_COPYOBJ we will call lvaSetVarDoNotEnregister() on both Source() and Dest()
8201 // When performing a field by field assignment we can have one of Source() or Dest treated as a blob of bytes
8202 // and in such cases we will call lvaSetVarDoNotEnregister() on the one treated as a blob of bytes.
8203 // if the Source() or Dest() is a a struct that has a "CustomLayout" and "ConstainsHoles" then we
8204 // can not use a field by field assignment and must the orginal GT_COPYBLK unmodified.
8206 GenTreePtr Compiler::fgMorphCopyBlock(GenTreePtr tree)
8208 noway_assert(tree->OperIsCopyBlkOp());
8210 JITDUMP("\nfgMorphCopyBlock:");
8212 bool isLateArg = (tree->gtFlags & GTF_LATE_ARG) != 0;
8214 GenTreePtr oneAsgTree = fgMorphOneAsgBlockOp(tree);
8218 JITDUMP(" using oneAsgTree.\n");
8223 GenTreePtr destAddr;
8225 GenTreePtr blockSize;
8228 if (tree->OperGet() == GT_COPYBLK)
8230 GenTreeCpBlk* copyBlkOp = tree->AsCpBlk();
8233 destAddr = copyBlkOp->Dest();
8234 srcAddr = copyBlkOp->Source();
8235 blockSize = copyBlkOp->Size();
8239 GenTreeCpObj* copyObjOp = tree->AsCpObj();
8242 destAddr = copyObjOp->Dest();
8243 srcAddr = copyObjOp->Source();
8244 blockSize = copyObjOp->ClsTok();
8247 noway_assert(destAddr->TypeGet() == TYP_BYREF || destAddr->TypeGet() == TYP_I_IMPL);
8248 noway_assert(srcAddr->TypeGet() == TYP_BYREF || srcAddr->TypeGet() == TYP_I_IMPL);
8250 unsigned blockWidth = 0;
8251 bool blockWidthIsConst = false;
8253 if (blockSize->IsCnsIntOrI())
8255 blockWidthIsConst = true;
8256 if (blockSize->IsIconHandle(GTF_ICON_CLASS_HDL))
8258 CORINFO_CLASS_HANDLE clsHnd = (CORINFO_CLASS_HANDLE) blockSize->gtIntConCommon.IconValue();
8259 blockWidth = info.compCompHnd->getClassSize(clsHnd);
8263 blockWidth = unsigned(blockSize->gtIntConCommon.IconValue());
8267 GenTreeLclVarCommon* lclVarTree = nullptr;
8269 FieldSeqNode* destFldSeq = nullptr;
8270 unsigned destLclNum = BAD_VAR_NUM;
8271 LclVarDsc* destLclVar = nullptr;
8272 bool destDoFldAsg = false;
8273 bool destOnStack = false;
8275 if (destAddr->IsLocalAddrExpr(this, &lclVarTree, &destFldSeq))
8278 destLclNum = lclVarTree->gtLclNum;
8279 destLclVar = &lvaTable[destLclNum];
8281 #if LOCAL_ASSERTION_PROP
8282 // Kill everything about destLclNum (and its field locals)
8283 if (optLocalAssertionProp)
8285 if (optAssertionCount > 0)
8287 fgKillDependentAssertions(destLclNum DEBUGARG(tree));
8290 #endif // LOCAL_ASSERTION_PROP
8292 if (destLclVar->lvPromoted && blockWidthIsConst)
8294 noway_assert(varTypeIsStruct(destLclVar));
8295 noway_assert(!opts.MinOpts());
8297 if (blockWidth == destLclVar->lvExactSize)
8299 JITDUMP(" (destDoFldAsg=true)");
8300 // We may decide later that a copyblk is required when this struct has holes
8301 destDoFldAsg = true;
8305 JITDUMP(" with mismatched dest size");
8310 FieldSeqNode* srcFldSeq = nullptr;
8311 unsigned srcLclNum = BAD_VAR_NUM;
8312 LclVarDsc* srcLclVar = nullptr;
8313 bool srcDoFldAsg = false;
8315 if (srcAddr->IsLocalAddrExpr(this, &lclVarTree, &srcFldSeq))
8317 srcLclNum = lclVarTree->gtLclNum;
8318 srcLclVar = &lvaTable[srcLclNum];
8320 if (srcLclVar->lvPromoted && blockWidthIsConst)
8322 noway_assert(varTypeIsStruct(srcLclVar));
8323 noway_assert(!opts.MinOpts());
8325 if (blockWidth == srcLclVar->lvExactSize)
8327 JITDUMP(" (srcDoFldAsg=true)");
8328 // We may decide later that a copyblk is required when this struct has holes
8333 JITDUMP(" with mismatched src size");
8338 // Check to see if we are required to do a copy block because the struct contains holes
8339 // and either the src or dest is externally visible
8341 bool requiresCopyBlock = false;
8342 bool srcSingleLclVarAsg = false;
8344 // If either src or dest is a reg-sized non-field-addressed struct, keep the copyBlock.
8345 if ((destLclVar != nullptr && destLclVar->lvRegStruct) ||
8346 (srcLclVar != nullptr && srcLclVar->lvRegStruct))
8348 requiresCopyBlock = true;
8351 // Can we use field by field assignment for the dest?
8352 if (destDoFldAsg && destLclVar->lvCustomLayout && destLclVar->lvContainsHoles)
8354 JITDUMP(" dest contains custom layout and contains holes");
8355 // C++ style CopyBlock with holes
8356 requiresCopyBlock = true;
8359 // Can we use field by field assignment for the src?
8360 if (srcDoFldAsg && srcLclVar->lvCustomLayout && srcLclVar->lvContainsHoles)
8362 JITDUMP(" src contains custom layout and contains holes");
8363 // C++ style CopyBlock with holes
8364 requiresCopyBlock = true;
8367 if (tree->OperGet() == GT_COPYBLK && tree->AsCpBlk()->gtBlkOpGcUnsafe)
8369 requiresCopyBlock = true;
8372 // If we passed the above checks, then we will check these two
8373 if (!requiresCopyBlock)
8375 // Are both dest and src promoted structs?
8376 if (destDoFldAsg && srcDoFldAsg)
8378 // Both structs should be of the same type, if not we will use a copy block
8379 if (lvaTable[destLclNum].lvVerTypeInfo.GetClassHandle() != lvaTable[srcLclNum].lvVerTypeInfo.GetClassHandle())
8381 requiresCopyBlock = true; // Mismatched types, leave as a CopyBlock
8382 JITDUMP(" with mismatched types");
8385 // Are neither dest or src promoted structs?
8386 else if (!destDoFldAsg && !srcDoFldAsg)
8388 requiresCopyBlock = true; // Leave as a CopyBlock
8389 JITDUMP(" with no promoted structs");
8391 else if (destDoFldAsg)
8393 // Match the following kinds of trees:
8394 // fgMorphTree BB01, stmt 9 (before)
8395 // [000052] ------------ const int 8
8396 // [000053] -A--G------- copyBlk void
8397 // [000051] ------------ addr byref
8398 // [000050] ------------ lclVar long V07 loc5
8399 // [000054] --------R--- <list> void
8400 // [000049] ------------ addr byref
8401 // [000048] ------------ lclVar struct(P) V06 loc4
8402 // long V06.h (offs=0x00) -> V17 tmp9
8403 // Yields this transformation
8404 // fgMorphCopyBlock (after):
8405 // [000050] ------------ lclVar long V07 loc5
8406 // [000085] -A---------- = long
8407 // [000083] D------N---- lclVar long V17 tmp9
8409 if (blockWidthIsConst &&
8410 (destLclVar->lvFieldCnt == 1) &&
8411 (srcLclVar != nullptr) &&
8412 (blockWidth == genTypeSize(srcLclVar->TypeGet())))
8414 // Reject the following tree:
8415 // - seen on x86chk jit\jit64\hfa\main\hfa_sf3E_r.exe
8417 // fgMorphTree BB01, stmt 6 (before)
8418 // [000038] ------------- const int 4
8419 // [000039] -A--G-------- copyBlk void
8420 // [000037] ------------- addr byref
8421 // [000036] ------------- lclVar int V05 loc3
8422 // [000040] --------R---- <list> void
8423 // [000035] ------------- addr byref
8424 // [000034] ------------- lclVar struct(P) V04 loc2
8425 // float V04.f1 (offs=0x00) -> V13 tmp6
8426 // As this would framsform into
8427 // float V13 = int V05
8429 unsigned fieldLclNum = lvaTable[destLclNum].lvFieldLclStart;
8430 var_types destType = lvaTable[fieldLclNum].TypeGet();
8431 if (srcLclVar->TypeGet() == destType)
8433 srcSingleLclVarAsg = true;
8439 // If we require a copy block the set both of the field assign bools to false
8440 if (requiresCopyBlock)
8442 // If a copy block is required then we won't do field by field assignments
8443 destDoFldAsg = false;
8444 srcDoFldAsg = false;
8447 JITDUMP(requiresCopyBlock ? " this requires a CopyBlock.\n"
8448 : " using field by field assignments.\n");
8450 // Mark the dest/src structs as DoNotEnreg
8451 // when they are not reg-sized non-field-addressed structs and we are using a CopyBlock
8452 // or the struct is not promoted
8454 if (!destDoFldAsg && (destLclVar != nullptr))
8456 if (!destLclVar->lvRegStruct)
8458 // Mark it as DoNotEnregister.
8459 lvaSetVarDoNotEnregister(destLclNum DEBUGARG(DNER_BlockOp));
8463 if (!srcDoFldAsg && (srcLclVar != nullptr) && !srcSingleLclVarAsg)
8465 if (!srcLclVar->lvRegStruct)
8467 lvaSetVarDoNotEnregister(srcLclNum DEBUGARG(DNER_BlockOp));
8471 if (requiresCopyBlock)
8473 #if CPU_USES_BLOCK_MOVE
8474 compBlkOpUsed = true;
8476 // Note that the unrolling of CopyBlk is only implemented on some platforms
8477 // Currently that includes x64 and Arm64 but not x64 or Arm32
8478 #ifdef CPBLK_UNROLL_LIMIT
8479 // If we have a CopyObj with a dest on the stack
8480 // we will convert it into an GC Unsafe CopyBlk that is non-interruptible
8481 // when its size is small enouch to be completely unrolled (i.e. between [16..64] bytes)
8483 if (isCopyObj && destOnStack && blockWidthIsConst &&
8484 (blockWidth >= (2*TARGET_POINTER_SIZE)) && (blockWidth <= CPBLK_UNROLL_LIMIT))
8486 tree->SetOper(GT_COPYBLK);
8487 tree->AsCpBlk()->gtBlkOpGcUnsafe = true; // Mark as a GC unsage copy block
8488 blockSize->gtIntConCommon.SetIconValue(ssize_t(blockWidth));
8489 blockSize->gtFlags &= ~GTF_ICON_HDL_MASK; // Clear the GTF_ICON_CLASS_HDL flags
8492 // Liveness doesn't consider copyblk arguments of simple types as being
8493 // a use or def, so explicitly mark these variables as address-exposed.
8494 if (srcLclNum != BAD_VAR_NUM && !varTypeIsStruct(srcLclVar))
8496 JITDUMP("Non-struct copyBlk src V%02d is addr exposed\n", srcLclNum);
8497 lvaTable[srcLclNum].lvAddrExposed = true;
8500 if (destLclNum != BAD_VAR_NUM && !varTypeIsStruct(destLclVar))
8502 JITDUMP("Non-struct copyBlk dest V%02d is addr exposed\n", destLclNum);
8503 lvaTable[destLclNum].lvAddrExposed = true;
8510 // Otherwise we convert this CopyBlock into individual field by field assignments
8517 GenTreePtr addrSpill = nullptr;
8518 unsigned addrSpillTemp = BAD_VAR_NUM;
8519 bool addrSpillIsStackDest = false; // true if 'addrSpill' represents the address in our local stack frame
8521 unsigned fieldCnt = DUMMY_INIT(0);
8523 if (destDoFldAsg && srcDoFldAsg)
8525 // To do fieldwise assignments for both sides, they'd better be the same struct type!
8526 // All of these conditions were checked above...
8527 assert(destLclNum != BAD_VAR_NUM && srcLclNum != BAD_VAR_NUM);
8528 assert(lvaTable[destLclNum].lvVerTypeInfo.GetClassHandle() == lvaTable[srcLclNum].lvVerTypeInfo.GetClassHandle());
8529 assert(destLclVar != nullptr && srcLclVar != nullptr && destLclVar->lvFieldCnt == srcLclVar->lvFieldCnt);
8531 fieldCnt = destLclVar->lvFieldCnt;
8532 goto _AssignFields; // No need to spill the address to the temp. Go ahead to morph it into field assignments.
8534 else if (destDoFldAsg)
8536 fieldCnt = destLclVar->lvFieldCnt;
8540 assert(srcDoFldAsg);
8541 fieldCnt = srcLclVar->lvFieldCnt;
8546 noway_assert(!srcDoFldAsg);
8547 if (gtClone(srcAddr))
8549 // srcAddr is simple expression. No need to spill.
8550 noway_assert((srcAddr->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) == 0);
8554 // srcAddr is complex expression. Clone and spill it (unless the destination is
8555 // a struct local that only has one field, in which case we'd only use the
8556 // address value once...)
8557 if (destLclVar->lvFieldCnt > 1)
8559 addrSpill = gtCloneExpr(srcAddr); // addrSpill represents the 'srcAddr'
8560 noway_assert(addrSpill != nullptr);
8567 noway_assert(!destDoFldAsg);
8569 // If we're doing field-wise stores, to an address within a local, and we copy
8570 // the address into "addrSpill", do *not* declare the original local var node in the
8571 // field address as GTF_VAR_DEF and GTF_VAR_USEASG; we will declare each of the
8572 // field-wise assignments as an "indirect" assignment to the local.
8573 // ("lclVarTree" is a subtree of "destAddr"; make sure we remove the flags before
8575 if (lclVarTree != nullptr)
8577 lclVarTree->gtFlags &= ~(GTF_VAR_DEF | GTF_VAR_USEASG);
8580 if (gtClone(destAddr))
8582 // destAddr is simple expression. No need to spill
8583 noway_assert((destAddr->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) == 0);
8587 // destAddr is complex expression. Clone and spill it (unless
8588 // the source is a struct local that only has one field, in which case we'd only
8589 // use the address value once...)
8590 if (srcLclVar->lvFieldCnt > 1)
8592 addrSpill = gtCloneExpr(destAddr); // addrSpill represents the 'destAddr'
8593 noway_assert(addrSpill != nullptr);
8596 // TODO-CQ: this should be based on a more general
8597 // "BaseAddress" method, that handles fields of structs, before or after
8599 if (addrSpill != nullptr && addrSpill->OperGet() == GT_ADDR)
8601 if (addrSpill->gtOp.gtOp1->IsLocal())
8603 // We will *not* consider this to define the local, but rather have each individual field assign
8605 addrSpill->gtOp.gtOp1->gtFlags &= ~(GTF_LIVENESS_MASK);
8606 assert(lvaTable[addrSpill->gtOp.gtOp1->gtLclVarCommon.gtLclNum].lvLclBlockOpAddr == 1);
8607 addrSpillIsStackDest = true; // addrSpill represents the address of LclVar[varNum] in our local stack frame
8613 if (addrSpill != nullptr)
8615 // Spill the (complex) address to a BYREF temp.
8616 // Note, at most one address may need to be spilled.
8617 addrSpillTemp = lvaGrabTemp(true DEBUGARG("BlockOp address local"));
8619 lvaTable[addrSpillTemp].lvType = TYP_BYREF;
8621 if (addrSpillIsStackDest)
8623 lvaTable[addrSpillTemp].lvStackByref = true;
8626 tree = gtNewAssignNode(gtNewLclvNode(addrSpillTemp, TYP_BYREF),
8629 #ifndef LEGACY_BACKEND
8630 // If we are assigning the address of a LclVar here
8631 // liveness does not account for this kind of address taken use.
8633 // We have to mark this local as address exposed so
8634 // that we don't delete the definition for this LclVar
8635 // as a dead store later on.
8637 if (addrSpill->OperGet() == GT_ADDR)
8639 GenTreePtr addrOp = addrSpill->gtOp.gtOp1;
8640 if (addrOp->IsLocal())
8642 unsigned lclVarNum = addrOp->gtLclVarCommon.gtLclNum;
8643 lvaTable[lclVarNum].lvAddrExposed = true;
8644 lvaSetVarDoNotEnregister(lclVarNum DEBUGARG(DNER_AddrExposed));
8647 #endif // !LEGACY_BACKEND
8652 for (unsigned i=0; i<fieldCnt; ++i)
8654 FieldSeqNode* curFieldSeq = nullptr;
8657 noway_assert(destLclNum != BAD_VAR_NUM);
8658 unsigned fieldLclNum = lvaTable[destLclNum].lvFieldLclStart + i;
8659 dest = gtNewLclvNode(fieldLclNum, lvaTable[fieldLclNum].TypeGet());
8661 noway_assert(destAddr->gtOp.gtOp1->gtOper == GT_LCL_VAR);
8662 // If it had been labeled a "USEASG", assignments to the the individual promoted fields are not.
8663 dest->gtFlags |= destAddr->gtOp.gtOp1->gtFlags & ~(GTF_NODE_MASK | GTF_VAR_USEASG);
8667 noway_assert(srcDoFldAsg);
8668 noway_assert(srcLclNum != BAD_VAR_NUM);
8669 unsigned fieldLclNum = lvaTable[srcLclNum].lvFieldLclStart + i;
8673 assert(addrSpillTemp != BAD_VAR_NUM);
8674 dest = gtNewLclvNode(addrSpillTemp, TYP_BYREF);
8678 dest = gtCloneExpr(destAddr);
8679 noway_assert(dest != nullptr);
8681 // Is the address of a local?
8682 GenTreeLclVarCommon* lclVarTree = nullptr;
8683 bool isEntire = false;
8684 bool* pIsEntire = (blockWidthIsConst ? &isEntire : nullptr);
8685 if (dest->DefinesLocalAddr(this, blockWidth, &lclVarTree, pIsEntire))
8687 lclVarTree->gtFlags |= GTF_VAR_DEF;
8689 lclVarTree->gtFlags |= GTF_VAR_USEASG;
8693 GenTreePtr fieldOffsetNode = gtNewIconNode(lvaTable[fieldLclNum].lvFldOffset, TYP_I_IMPL);
8694 // Have to set the field sequence -- which means we need the field handle.
8695 CORINFO_CLASS_HANDLE classHnd = lvaTable[srcLclNum].lvVerTypeInfo.GetClassHandle();
8696 CORINFO_FIELD_HANDLE fieldHnd = info.compCompHnd->getFieldInClass(classHnd, lvaTable[fieldLclNum].lvFldOrdinal);
8697 curFieldSeq = GetFieldSeqStore()->CreateSingleton(fieldHnd);
8698 fieldOffsetNode->gtIntCon.gtFieldSeq = curFieldSeq;
8700 dest = gtNewOperNode(GT_ADD, TYP_BYREF,
8704 dest = gtNewOperNode(GT_IND, lvaTable[fieldLclNum].TypeGet(), dest);
8706 // !!! The destination could be on stack. !!!
8707 // This flag will let us choose the correct write barrier.
8708 dest->gtFlags |= GTF_IND_TGTANYWHERE;
8714 noway_assert(srcLclNum != BAD_VAR_NUM);
8715 unsigned fieldLclNum = lvaTable[srcLclNum].lvFieldLclStart + i;
8716 src = gtNewLclvNode(fieldLclNum, lvaTable[fieldLclNum].TypeGet());
8718 noway_assert(srcAddr->gtOp.gtOp1->gtOper == GT_LCL_VAR);
8719 src->gtFlags |= srcAddr->gtOp.gtOp1->gtFlags & ~GTF_NODE_MASK;
8723 noway_assert(destDoFldAsg);
8724 noway_assert(destLclNum != BAD_VAR_NUM);
8725 unsigned fieldLclNum = lvaTable[destLclNum].lvFieldLclStart + i;
8727 if (srcSingleLclVarAsg)
8729 noway_assert(fieldCnt == 1);
8730 noway_assert(srcLclVar != nullptr);
8731 noway_assert(addrSpill == nullptr);
8733 src = gtNewLclvNode(srcLclNum, srcLclVar->TypeGet());
8739 assert(addrSpillTemp != BAD_VAR_NUM);
8740 src = gtNewLclvNode(addrSpillTemp, TYP_BYREF);
8744 src = gtCloneExpr(srcAddr);
8745 noway_assert(src != nullptr);
8748 CORINFO_CLASS_HANDLE classHnd = lvaTable[destLclNum].lvVerTypeInfo.GetClassHandle();
8749 CORINFO_FIELD_HANDLE fieldHnd = info.compCompHnd->getFieldInClass(classHnd, lvaTable[fieldLclNum].lvFldOrdinal);
8750 curFieldSeq = GetFieldSeqStore()->CreateSingleton(fieldHnd);
8752 src = gtNewOperNode(GT_ADD, TYP_BYREF,
8754 new(this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL,
8755 lvaTable[fieldLclNum].lvFldOffset,
8758 src = gtNewOperNode(GT_IND, lvaTable[fieldLclNum].TypeGet(), src);
8762 noway_assert(dest->TypeGet() == src->TypeGet());
8764 asg = gtNewAssignNode(dest, src);
8766 // If we spilled the address, and we didn't do individual field assignments to promoted fields,
8767 // and it was of a local, record the assignment as an indirect update of a local.
8768 if (addrSpill && !destDoFldAsg && destLclNum != BAD_VAR_NUM)
8770 curFieldSeq = GetFieldSeqStore()->Append(destFldSeq, curFieldSeq);
8771 bool isEntire = (genTypeSize(var_types(lvaTable[destLclNum].lvType))
8772 == genTypeSize(dest->TypeGet()));
8773 IndirectAssignmentAnnotation* pIndirAnnot =
8774 new (this, CMK_Unknown) IndirectAssignmentAnnotation(destLclNum, curFieldSeq, isEntire);
8775 GetIndirAssignMap()->Set(asg, pIndirAnnot);
8778 #if LOCAL_ASSERTION_PROP
8779 if (optLocalAssertionProp)
8781 optAssertionGen(asg);
8783 #endif // LOCAL_ASSERTION_PROP
8787 tree = gtNewOperNode(GT_COMMA,
8801 tree->gtFlags |= GTF_LATE_ARG;
8805 tree->gtFlags |= GTF_MORPHED;
8809 printf("\nfgMorphCopyBlock (after):\n");
8818 // insert conversions and normalize to make tree amenable to register
8820 GenTree* Compiler::fgMorphForRegisterFP(GenTree *tree)
8822 GenTreePtr op1 = tree->gtOp.gtOp1;
8823 GenTreePtr op2 = tree->gtGetOp2();
8825 if (tree->OperIsArithmetic()
8826 && varTypeIsFloating(tree))
8828 if (op1->TypeGet() != tree->TypeGet())
8830 tree->gtOp.gtOp1 = gtNewCastNode(tree->TypeGet(), tree->gtOp.gtOp1, tree->TypeGet());
8832 if (op2->TypeGet() != tree->TypeGet())
8834 tree->gtOp.gtOp2 = gtNewCastNode(tree->TypeGet(), tree->gtOp.gtOp2, tree->TypeGet());
8837 else if (tree->OperIsCompare()
8838 && varTypeIsFloating(op1)
8839 && op1->TypeGet() != op2->TypeGet())
8841 // both had better be floating, just one bigger than other
8842 assert (varTypeIsFloating(op2));
8843 if (op1->TypeGet() == TYP_FLOAT)
8845 tree->gtOp.gtOp1 = gtNewCastNode(TYP_DOUBLE, tree->gtOp.gtOp1, TYP_DOUBLE);
8847 else if (op2->TypeGet() == TYP_FLOAT)
8849 tree->gtOp.gtOp2 = gtNewCastNode(TYP_DOUBLE, tree->gtOp.gtOp2, TYP_DOUBLE);
8856 GenTree* Compiler::fgMorphRecognizeBoxNullable(GenTree* compare)
8858 GenTree* op1 = compare->gtOp.gtOp1;
8859 GenTree* op2 = compare->gtOp.gtOp2;
8861 GenTreeCall* opCall;
8863 // recognize this pattern:
8865 // stmtExpr void (IL 0x000... ???)
8869 // call help ref HELPER.CORINFO_HELP_BOX_NULLABLE
8870 // const(h) long 0x7fed96836c8 class
8872 // ld.lclVar struct V00 arg0
8875 // which comes from this code (reported by customer as being slow) :
8877 // private static bool IsNull<T>(T arg)
8879 // return arg==null;
8883 if (op1->IsCnsIntOrI() && op2->IsHelperCall())
8886 opCall = op2->AsCall();
8888 else if (op1->IsHelperCall() && op2->IsCnsIntOrI())
8891 opCall = op1->AsCall();
8898 if (opCns->gtIntConCommon.IconValue() != 0)
8901 if (eeGetHelperNum(opCall->gtCallMethHnd) != CORINFO_HELP_BOX_NULLABLE)
8904 // replace the box with an access of the nullable 'hasValue' field which is at the zero offset
8905 GenTree* newOp = gtNewOperNode(GT_IND, TYP_BOOL, opCall->gtCall.gtCallArgs->gtOp.gtOp2->gtOp.gtOp1);
8908 compare->gtOp.gtOp1 = newOp;
8910 compare->gtOp.gtOp2 = newOp;
8917 //--------------------------------------------------------------------------------------
8918 // fgCopySIMDNode: make a copy of a SIMD intrinsic node, e.g. so that a field can be accessed.
8921 // simdNode - The GenTreeSIMD node to be copied
8924 // A comma node where op1 is the assignment of the simd node to a temp, and op2 is the temp lclVar.
8927 Compiler::fgCopySIMDNode(GenTreeSIMD* simdNode)
8929 // Copy the result of the SIMD intrinsic into a temp.
8930 unsigned lclNum = lvaGrabTemp(true DEBUGARG("Copy of SIMD intrinsic with field access"));
8932 CORINFO_CLASS_HANDLE simdHandle = NO_CLASS_HANDLE;
8933 // We only have fields of the fixed float vectors.
8934 noway_assert(simdNode->gtSIMDBaseType == TYP_FLOAT);
8935 switch(simdNode->gtSIMDSize)
8937 case 8: simdHandle = SIMDVector2Handle; break;
8938 case 12: simdHandle = SIMDVector3Handle; break;
8939 case 16: simdHandle = SIMDVector4Handle; break;
8940 default: noway_assert(!"field of unexpected SIMD type"); break;
8942 assert(simdHandle != NO_CLASS_HANDLE);
8944 lvaSetStruct(lclNum, simdHandle, false, true);
8945 lvaTable[lclNum].lvFieldAccessed = true;
8947 GenTree* asg = gtNewTempAssign(lclNum, simdNode);
8948 GenTree* newLclVarNode = new (this, GT_LCL_VAR) GenTreeLclVar(simdNode->TypeGet(), lclNum, BAD_IL_OFFSET);
8950 GenTree* comma = gtNewOperNode(GT_COMMA, simdNode->TypeGet(), asg, newLclVarNode);
8954 //--------------------------------------------------------------------------------------------------------------
8955 // getSIMDStructFromField:
8956 // Checking whether the field belongs to a simd struct or not. If it is, return the GenTreePtr for
8957 // the struct node, also base type, field index and simd size. If it is not, just return nullptr.
8958 // Usually if the tree node is from a simd lclvar which is not used in any SIMD intrinsic, then we
8959 // should return nullptr, since in this case we should treat SIMD struct as a regular struct.
8960 // However if no matter what, you just want get simd struct node, you can set the ignoreUsedInSIMDIntrinsic
8961 // as true. Then there will be no IsUsedInSIMDIntrinsic checking, and it will return SIMD struct node
8962 // if the struct is a SIMD struct.
8965 // tree - GentreePtr. This node will be checked to see this is a field which belongs to a simd
8966 // struct used for simd intrinsic or not.
8967 // pBaseTypeOut - var_types pointer, if the tree node is the tree we want, we set *pBaseTypeOut
8968 // to simd lclvar's base type.
8969 // indexOut - unsigned pointer, if the tree is used for simd intrinsic, we will set *indexOut
8970 // equals to the index number of this field.
8971 // simdSizeOut - unsigned pointer, if the tree is used for simd intrinsic, set the *simdSizeOut
8972 // equals to the simd struct size which this tree belongs to.
8973 // ignoreUsedInSIMDIntrinsic - bool. If this is set to true, then this function will ignore
8974 // the UsedInSIMDIntrinsic check.
8977 // A GenTreePtr which points the simd lclvar tree belongs to. If the tree is not the simd
8978 // instrinic related field, return nullptr.
8981 GenTreePtr Compiler::getSIMDStructFromField(GenTreePtr tree, var_types* pBaseTypeOut, unsigned* indexOut, unsigned* simdSizeOut, bool ignoreUsedInSIMDIntrinsic/*false*/)
8983 GenTreePtr ret = nullptr;
8984 if(tree->OperGet() == GT_FIELD)
8986 GenTreePtr objRef = tree->gtField.gtFldObj;
8987 if (objRef != nullptr)
8989 GenTreePtr obj = nullptr;
8990 if (objRef->gtOper == GT_ADDR)
8992 obj = objRef->gtOp.gtOp1;
8994 else if(ignoreUsedInSIMDIntrinsic)
9003 if (isSIMDTypeLocal(obj))
9005 unsigned lclNum = obj->gtLclVarCommon.gtLclNum;
9006 LclVarDsc* varDsc = &lvaTable[lclNum];
9007 if(varDsc->lvIsUsedInSIMDIntrinsic() || ignoreUsedInSIMDIntrinsic)
9009 *simdSizeOut = varDsc->lvExactSize;
9010 *pBaseTypeOut = getBaseTypeOfSIMDLocal(obj);
9014 else if (obj->OperGet() == GT_SIMD)
9017 GenTreeSIMD* simdNode = obj->AsSIMD();
9018 *simdSizeOut = simdNode->gtSIMDSize;
9019 *pBaseTypeOut = simdNode->gtSIMDBaseType;
9025 unsigned BaseTypeSize = genTypeSize(*pBaseTypeOut);
9026 *indexOut = tree->gtField.gtFldOffset / BaseTypeSize;
9031 /*****************************************************************************
9032 * If a read operation tries to access simd struct field, then transform the this
9033 * operation to to the SIMD intrinsic SIMDIntrinsicGetItem, and return the new tree.
9034 * Otherwise, return the old tree.
9036 * tree - GenTreePtr. If this pointer points to simd struct which is used for simd
9037 * intrinsic. We will morph it as simd intrinsic SIMDIntrinsicGetItem.
9039 * A GenTreePtr which points to the new tree. If the tree is not for simd intrinsic,
9043 GenTreePtr Compiler::fgMorphFieldToSIMDIntrinsicGet(GenTreePtr tree)
9046 var_types baseType = TYP_UNKNOWN;
9047 unsigned simdSize = 0;
9048 GenTreePtr simdStructNode = getSIMDStructFromField(tree, &baseType, &index, &simdSize);
9049 if(simdStructNode != nullptr)
9052 assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
9053 GenTree* op2 = gtNewIconNode(index);
9054 tree = gtNewSIMDNode(baseType, simdStructNode, op2, SIMDIntrinsicGetItem, baseType, simdSize);
9056 tree->gtFlags |= GTF_MORPHED;
9062 /*****************************************************************************
9063 * Transform an assignment of a SIMD struct field to SIMD intrinsic
9064 * SIMDIntrinsicGetItem, and return a new tree. If If it is not such an assignment,
9065 * then return the old tree.
9067 * tree - GenTreePtr. If this pointer points to simd struct which is used for simd
9068 * intrinsic. We will morph it as simd intrinsic set.
9070 * A GenTreePtr which points to the new tree. If the tree is not for simd intrinsic,
9074 GenTreePtr Compiler::fgMorphFieldAssignToSIMDIntrinsicSet(GenTreePtr tree)
9076 assert(tree->OperGet() == GT_ASG);
9077 GenTreePtr op1 = tree->gtGetOp1();
9078 GenTreePtr op2 = tree->gtGetOp2();
9081 var_types baseType = TYP_UNKNOWN;
9082 unsigned simdSize = 0;
9083 GenTreePtr simdOp1Struct = getSIMDStructFromField(op1, &baseType, &index, &simdSize);
9084 if (simdOp1Struct != nullptr)
9086 //Generate the simd set intrinsic
9087 assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
9089 SIMDIntrinsicID simdIntrinsicID = SIMDIntrinsicInvalid;
9093 simdIntrinsicID = SIMDIntrinsicSetX;
9096 simdIntrinsicID = SIMDIntrinsicSetY;
9099 simdIntrinsicID = SIMDIntrinsicSetZ;
9102 simdIntrinsicID = SIMDIntrinsicSetW;
9105 noway_assert("There is no set intrinsic for index bigger than 3");
9109 GenTreePtr newStruct = gtClone(simdOp1Struct);
9110 assert((newStruct != nullptr) && (varTypeIsSIMD(newStruct)));
9111 GenTreePtr simdTree = gtNewSIMDNode(newStruct->gtType, simdOp1Struct, op2, simdIntrinsicID, baseType, simdSize);
9112 GenTreePtr copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, newStruct);
9113 tree = gtNewBlkOpNode(GT_COPYBLK,
9115 gtNewOperNode(GT_ADDR, TYP_BYREF, simdTree),
9116 gtNewIconNode(simdSize),
9119 tree->gtFlags |= GTF_MORPHED;
9127 /*****************************************************************************
9129 * Transform the given GTK_SMPOP tree for code generation.
9133 #pragma warning(push)
9134 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
9136 GenTreePtr Compiler::fgMorphSmpOp(GenTreePtr tree, MorphAddrContext* mac)
9138 // this extra scope is a workaround for a gcc bug
9139 // the inline destructor for ALLOCA_CHECK confuses the control
9140 // flow and gcc thinks that the function never returns
9143 noway_assert(tree->OperKind() & GTK_SMPOP);
9145 /* The steps in this function are :
9146 o Perform required preorder processing
9147 o Process the first, then second operand, if any
9148 o Perform required postorder morphing
9149 o Perform optional postorder morphing if optimizing
9152 bool isQmarkColon = false;
9154 #if LOCAL_ASSERTION_PROP
9155 AssertionIndex origAssertionCount = DUMMY_INIT(0);
9156 AssertionDsc * origAssertionTab = DUMMY_INIT(NULL);
9158 AssertionIndex thenAssertionCount = DUMMY_INIT(0);
9159 AssertionDsc * thenAssertionTab = DUMMY_INIT(NULL);
9164 #if !FEATURE_STACK_FP_X87
9165 tree = fgMorphForRegisterFP(tree);
9169 genTreeOps oper = tree->OperGet();
9170 var_types typ = tree->TypeGet();
9171 GenTreePtr op1 = tree->gtOp.gtOp1;
9172 GenTreePtr op2 = tree->gtGetOp2();
9174 /*-------------------------------------------------------------------------
9175 * First do any PRE-ORDER processing
9180 // Some arithmetic operators need to use a helper call to the EE
9184 tree = fgDoNormalizeOnStore(tree);
9185 /* fgDoNormalizeOnStore can change op2 */
9186 noway_assert(op1 == tree->gtOp.gtOp1);
9187 op2 = tree->gtOp.gtOp2;
9191 // We should check whether op2 should be assigned to a SIMD field or not.
9192 // if it is, we should tranlate the tree to simd intrinsic
9193 GenTreePtr newTree = fgMorphFieldAssignToSIMDIntrinsicSet(tree);
9194 if (newTree != tree)
9197 oper = tree->OperGet();
9198 typ = tree->TypeGet();
9199 op1 = tree->gtOp.gtOp1;
9200 op2 = tree->gtGetOp2();
9222 /* We can't CSE the LHS of an assignment. Only r-values can be CSEed */
9223 op1->gtFlags |= GTF_DONT_CSE;
9228 /* op1 of a GT_ADDR is an l-value. Only r-values can be CSEed */
9229 op1->gtFlags |= GTF_DONT_CSE;
9237 if (op1->OperKind() & GTK_RELOP)
9239 noway_assert((oper == GT_JTRUE) || (op1->gtFlags & GTF_RELOP_QMARK));
9240 /* Mark the comparison node with GTF_RELOP_JMP_USED so it knows that it does
9241 not need to materialize the result as a 0 or 1. */
9243 /* We also mark it as DONT_CSE, as we don't handle QMARKs with nonRELOP op1s */
9244 op1->gtFlags |= (GTF_RELOP_JMP_USED | GTF_DONT_CSE);
9246 // Request that the codegen for op1 sets the condition flags
9247 // when it generates the code for op1.
9249 // Codegen for op1 must set the condition flags if
9250 // this method returns true.
9252 op1->gtRequestSetFlags();
9256 GenTreePtr effOp1 = op1->gtEffectiveVal();
9258 noway_assert( (effOp1->gtOper == GT_CNS_INT) &&
9259 ((effOp1->gtIntCon.gtIconVal == 0) || (effOp1->gtIntCon.gtIconVal == 1)) );
9264 #if LOCAL_ASSERTION_PROP
9265 if (optLocalAssertionProp)
9267 isQmarkColon = true;
9271 return fgMorphArrayIndex(tree);
9274 return fgMorphCast(tree);
9278 #ifndef _TARGET_64BIT_
9279 #if !LONG_MATH_REGPARAM
9280 if (typ == TYP_LONG)
9282 /* For (long)int1 * (long)int2, we dont actually do the
9283 casts, and just multiply the 32 bit values, which will
9284 give us the 64 bit result in edx:eax */
9287 if ((op1->gtOper == GT_CAST &&
9288 op2->gtOper == GT_CAST &&
9289 genActualType(op1->CastFromType()) == TYP_INT &&
9290 genActualType(op2->CastFromType()) == TYP_INT)&&
9291 !op1->gtOverflow() && !op2->gtOverflow())
9293 // The casts have to be of the same signedness.
9294 if ((op1->gtFlags & GTF_UNSIGNED) != (op2->gtFlags & GTF_UNSIGNED))
9296 //We see if we can force an int constant to change its signedness
9298 if (op1->gtCast.CastOp()->gtOper == GT_CNS_INT)
9300 else if (op2->gtCast.CastOp()->gtOper == GT_CNS_INT)
9305 if ( ((unsigned)(constOp->gtCast.CastOp()->gtIntCon.gtIconVal) < (unsigned)(0x80000000)) )
9306 constOp->gtFlags ^= GTF_UNSIGNED;
9311 // The only combination that can overflow
9312 if (tree->gtOverflow() && (tree->gtFlags & GTF_UNSIGNED) &&
9313 !( op1->gtFlags & GTF_UNSIGNED))
9316 /* Remaining combinations can never overflow during long mul. */
9318 tree->gtFlags &= ~GTF_OVERFLOW;
9320 /* Do unsigned mul only if the casts were unsigned */
9322 tree->gtFlags &= ~GTF_UNSIGNED;
9323 tree->gtFlags |= op1->gtFlags & GTF_UNSIGNED;
9325 /* Since we are committing to GTF_MUL_64RSLT, we don't want
9326 the casts to be folded away. So morph the castees directly */
9328 op1->gtOp.gtOp1 = fgMorphTree(op1->gtOp.gtOp1);
9329 op2->gtOp.gtOp1 = fgMorphTree(op2->gtOp.gtOp1);
9331 // Propagate side effect flags up the tree
9332 op1->gtFlags &= ~GTF_ALL_EFFECT;
9333 op1->gtFlags |= (op1->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT);
9334 op2->gtFlags &= ~GTF_ALL_EFFECT;
9335 op2->gtFlags |= (op2->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT);
9337 // If the GT_MUL can be altogether folded away, we should do that.
9339 if ((op1->gtCast.CastOp()->OperKind() &
9340 op2->gtCast.CastOp()->OperKind() & GTK_CONST) && opts.OptEnabled(CLFLG_CONSTANTFOLD))
9342 tree->gtOp.gtOp1 = op1 = gtFoldExprConst(op1);
9343 tree->gtOp.gtOp2 = op2 = gtFoldExprConst(op2);
9344 noway_assert(op1->OperKind() & op2->OperKind() & GTK_CONST);
9345 tree = gtFoldExprConst(tree);
9346 noway_assert(tree->OperIsConst());
9350 tree->gtFlags |= GTF_MUL_64RSLT;
9352 // If op1 and op2 are unsigned casts, we need to do an unsigned mult
9353 tree->gtFlags |= (op1->gtFlags & GTF_UNSIGNED);
9355 // Insert GT_NOP nodes for the cast operands so that they do not get folded
9356 // And propagate the new flags. We don't want to CSE the casts because
9357 // codegen expects GTF_MUL_64RSLT muls to have a certain layout.
9359 if (op1->gtCast.CastOp()->OperGet() != GT_NOP)
9361 op1->gtOp.gtOp1 = gtNewOperNode(GT_NOP, TYP_INT, op1->gtCast.CastOp());
9362 op1->gtFlags &= ~GTF_ALL_EFFECT;
9363 op1->gtFlags |= (op1->gtCast.CastOp()->gtFlags & GTF_ALL_EFFECT);
9364 op1->gtFlags |= GTF_DONT_CSE;
9367 if (op2->gtCast.CastOp()->OperGet() != GT_NOP)
9369 op2->gtOp.gtOp1 = gtNewOperNode(GT_NOP, TYP_INT, op2->gtCast.CastOp());
9370 op2->gtFlags &= ~GTF_ALL_EFFECT;
9371 op2->gtFlags |= (op2->gtCast.CastOp()->gtFlags & GTF_ALL_EFFECT);
9372 op2->gtFlags |= GTF_DONT_CSE;
9375 tree->gtFlags &= ~GTF_ALL_EFFECT;
9376 tree->gtFlags |= ((op1->gtFlags | op2->gtFlags) & GTF_ALL_EFFECT);
9378 goto DONE_MORPHING_CHILDREN;
9380 else if ((tree->gtFlags & GTF_MUL_64RSLT) == 0)
9383 if (tree->gtOverflow())
9384 helper = (tree->gtFlags & GTF_UNSIGNED) ? CORINFO_HELP_ULMUL_OVF
9385 : CORINFO_HELP_LMUL_OVF;
9387 helper = CORINFO_HELP_LMUL;
9389 goto USE_HELPER_FOR_ARITH;
9393 /* We are seeing this node again. We have decided to use
9394 GTF_MUL_64RSLT, so leave it alone. */
9396 assert(tree->gtIsValid64RsltMul());
9399 #endif // !LONG_MATH_REGPARAM
9400 #endif // !_TARGET_64BIT_
9406 #ifndef _TARGET_64BIT_
9407 #if !LONG_MATH_REGPARAM
9408 if (typ == TYP_LONG)
9410 helper = CORINFO_HELP_LDIV;
9411 goto USE_HELPER_FOR_ARITH;
9415 #if USE_HELPERS_FOR_INT_DIV
9416 if (typ == TYP_INT && !fgIsSignedDivOptimizable(op2))
9418 helper = CORINFO_HELP_DIV;
9419 goto USE_HELPER_FOR_ARITH;
9422 #endif // !_TARGET_64BIT_
9424 #ifndef LEGACY_BACKEND
9425 if (op2->gtOper == GT_CAST && op2->gtOp.gtOp1->IsCnsIntOrI())
9427 op2 = gtFoldExprConst(op2);
9430 if (fgShouldUseMagicNumberDivide(tree->AsOp()))
9432 tree = fgMorphDivByConst(tree->AsOp());
9433 op1 = tree->gtOp.gtOp1;
9434 op2 = tree->gtOp.gtOp2;
9436 #endif // !LEGACY_BACKEND
9442 #ifndef _TARGET_64BIT_
9443 #if !LONG_MATH_REGPARAM
9444 if (typ == TYP_LONG)
9446 helper = CORINFO_HELP_ULDIV;
9447 goto USE_HELPER_FOR_ARITH;
9450 #if USE_HELPERS_FOR_INT_DIV
9451 if (typ == TYP_INT && !fgIsUnsignedDivOptimizable(op2))
9453 helper = CORINFO_HELP_UDIV;
9454 goto USE_HELPER_FOR_ARITH;
9457 #endif // _TARGET_64BIT_
9463 if (varTypeIsFloating(typ))
9465 helper = CORINFO_HELP_DBLREM;
9467 if (op1->TypeGet() == TYP_FLOAT)
9468 if (op2->TypeGet() == TYP_FLOAT)
9469 helper = CORINFO_HELP_FLTREM;
9471 tree->gtOp.gtOp1 = op1 = gtNewCastNode(TYP_DOUBLE, op1, TYP_DOUBLE);
9473 if (op2->TypeGet() == TYP_FLOAT)
9474 tree->gtOp.gtOp2 = op2 = gtNewCastNode(TYP_DOUBLE, op2, TYP_DOUBLE);
9475 goto USE_HELPER_FOR_ARITH;
9478 // Do not use optimizations (unlike UMOD's idiv optimizing during codegen) for signed mod.
9479 // A similar optimization for signed mod will not work for a negative perfectly divisible
9480 // HI-word. To make it correct, we would need to divide without the sign and then flip the
9481 // result sign after mod. This requires 18 opcodes + flow making it not worthy to inline.
9482 goto ASSIGN_HELPER_FOR_MOD;
9486 #ifdef _TARGET_ARMARCH_
9488 // Note for _TARGET_ARMARCH_ we don't have a remainder instruction, so we don't do this optimization
9490 #else // _TARGET_XARCH
9491 /* If this is an unsigned long mod with op2 which is a cast to long from a
9492 constant int, then don't morph to a call to the helper. This can be done
9493 faster inline using idiv.
9497 if ((typ == TYP_LONG) && opts.OptEnabled(CLFLG_CONSTANTFOLD) &&
9498 ((tree->gtFlags & GTF_UNSIGNED) == (op1->gtFlags & GTF_UNSIGNED)) &&
9499 ((tree->gtFlags & GTF_UNSIGNED) == (op2->gtFlags & GTF_UNSIGNED)))
9501 if (op2->gtOper == GT_CAST &&
9502 op2->gtCast.CastOp()->gtOper == GT_CNS_INT &&
9503 op2->gtCast.CastOp()->gtIntCon.gtIconVal >= 2 &&
9504 op2->gtCast.CastOp()->gtIntCon.gtIconVal <= 0x3fffffff &&
9505 (tree->gtFlags & GTF_UNSIGNED) == (op2->gtCast.CastOp()->gtFlags & GTF_UNSIGNED))
9507 tree->gtOp.gtOp2 = op2 = fgMorphCast(op2);
9508 noway_assert(op2->gtOper == GT_CNS_NATIVELONG);
9511 if (op2->gtOper == GT_CNS_NATIVELONG &&
9512 op2->gtIntConCommon.LngValue() >= 2 &&
9513 op2->gtIntConCommon.LngValue() <= 0x3fffffff)
9515 tree->gtOp.gtOp1 = op1 = fgMorphTree(op1);
9516 noway_assert(op1->TypeGet() == TYP_LONG);
9518 // Update flags for op1 morph
9519 tree->gtFlags &= ~GTF_ALL_EFFECT;
9521 tree->gtFlags |= (op1->gtFlags & GTF_ALL_EFFECT); // Only update with op1 as op2 is a constant
9523 // If op1 is a constant, then do constant folding of the division operator
9524 if (op1->gtOper == GT_CNS_NATIVELONG)
9526 tree = gtFoldExpr(tree);
9531 #endif // _TARGET_XARCH
9533 ASSIGN_HELPER_FOR_MOD:
9535 // For "val % 1", return 0 if op1 doesn't have any side effects
9536 // and we are not in the CSE phase, we cannot discard 'tree'
9537 // because it may contain CSE expressions that we haven't yet examined.
9539 if (((op1->gtFlags & GTF_SIDE_EFFECT) == 0) && !optValnumCSE_phase)
9541 if (((op2->gtOper == GT_CNS_INT) && (op2->gtIntConCommon.IconValue() == 1))
9542 || ((op2->gtOper == GT_CNS_LNG) && (op2->gtIntConCommon.LngValue() == 1)))
9544 GenTreePtr zeroNode = gtNewZeroConNode(typ);
9546 zeroNode->gtFlags |= GTF_MORPHED;
9548 DEBUG_DESTROY_NODE(tree);
9553 #ifndef _TARGET_64BIT_
9554 #if !LONG_MATH_REGPARAM
9555 if (typ == TYP_LONG)
9557 helper = (oper == GT_UMOD) ? CORINFO_HELP_ULMOD : CORINFO_HELP_LMOD;
9558 goto USE_HELPER_FOR_ARITH;
9562 #if USE_HELPERS_FOR_INT_DIV
9565 if (oper == GT_UMOD && !fgIsUnsignedModOptimizable(op2))
9567 helper = CORINFO_HELP_UMOD;
9568 goto USE_HELPER_FOR_ARITH;
9570 else if (oper == GT_MOD && !fgIsSignedModOptimizable(op2))
9572 helper = CORINFO_HELP_MOD;
9573 goto USE_HELPER_FOR_ARITH;
9577 #endif // !_TARGET_64BIT_
9579 #ifndef LEGACY_BACKEND
9580 if (op2->gtOper == GT_CAST && op2->gtOp.gtOp1->IsCnsIntOrI())
9582 op2 = gtFoldExprConst(op2);
9585 #ifdef _TARGET_ARM64_
9587 // For ARM64 we don't have a remainder instruction,
9588 // The architecture manual suggests the following transformation to
9589 // generate code for such operator:
9591 // a % b = a - (a / b) * b;
9593 tree = fgMorphModToSubMulDiv(tree->AsOp());
9594 op1 = tree->gtOp.gtOp1;
9595 op2 = tree->gtOp.gtOp2;
9597 #else // !_TARGET_ARM64_
9599 if (oper != GT_UMOD && fgShouldUseMagicNumberDivide(tree->AsOp()))
9601 tree = fgMorphModByConst(tree->AsOp());
9602 op1 = tree->gtOp.gtOp1;
9603 op2 = tree->gtOp.gtOp2;
9606 #endif //_TARGET_ARM64_
9607 #endif // !LEGACY_BACKEND
9610 USE_HELPER_FOR_ARITH:
9612 /* We have to morph these arithmetic operations into helper calls
9613 before morphing the arguments (preorder), else the arguments
9614 won't get correct values of fgPtrArgCntCur.
9615 However, try to fold the tree first in case we end up with a
9616 simple node which won't need a helper call at all */
9618 noway_assert(tree->OperIsBinary());
9620 GenTreePtr oldTree = tree;
9622 tree = gtFoldExpr(tree);
9624 // Were we able to fold it ?
9625 // Note that gtFoldExpr may return a non-leaf even if successful
9626 // e.g. for something like "expr / 1" - see also bug #290853
9627 if (tree->OperIsLeaf() || (oldTree != tree))
9630 return (oldTree != tree) ? fgMorphTree(tree) : fgMorphLeaf(tree);
9633 // Did we fold it into a comma node with throw?
9634 if (tree->gtOper == GT_COMMA)
9636 noway_assert(fgIsCommaThrow(tree));
9637 return fgMorphTree(tree);
9640 return fgMorphIntoHelperCall(tree, helper, gtNewArgList(op1, op2));
9643 // normalize small integer return values
9644 if (fgGlobalMorph && varTypeIsSmall(info.compRetType) &&
9645 (op1 != NULL) && (op1->TypeGet() != TYP_VOID) &&
9646 fgCastNeeded(op1, info.compRetType))
9648 // Small-typed return values are normalized by the callee
9649 op1 = gtNewCastNode(TYP_INT, op1, info.compRetType);
9651 // Propagate GTF_COLON_COND
9652 op1->gtFlags|=(tree->gtFlags & GTF_COLON_COND);
9654 tree->gtOp.gtOp1 = fgMorphCast(op1);
9656 // Propagate side effect flags
9657 tree->gtFlags &= ~GTF_ALL_EFFECT;
9658 tree->gtFlags |= (tree->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT);
9667 // Check for typeof(...) == obj.GetType()
9668 // Also check for typeof(...) == typeof(...)
9669 // IMPORTANT NOTE: this optimization relies on a one-to-one mapping between
9670 // type handles and instances of System.Type
9671 // If this invariant is ever broken, the optimization will need updating
9673 #ifdef LEGACY_BACKEND
9674 if ( op1->gtOper == GT_CALL &&
9675 op2->gtOper == GT_CALL &&
9676 ((op1->gtCall.gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) || (op1->gtCall.gtCallType == CT_HELPER)) &&
9677 ((op2->gtCall.gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) || (op2->gtCall.gtCallType == CT_HELPER)))
9679 if ((((op1->gtOper == GT_INTRINSIC) && (op1->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Object_GetType)) ||
9680 ((op1->gtOper == GT_CALL) && (op1->gtCall.gtCallType == CT_HELPER))) &&
9681 (((op2->gtOper == GT_INTRINSIC) && (op2->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Object_GetType)) ||
9682 ((op2->gtOper == GT_CALL) && (op2->gtCall.gtCallType == CT_HELPER))))
9685 GenTreePtr pGetClassFromHandle;
9686 GenTreePtr pGetType;
9688 #ifdef LEGACY_BACKEND
9689 bool bOp1ClassFromHandle = gtIsTypeHandleToRuntimeTypeHelper(op1);
9690 bool bOp2ClassFromHandle = gtIsTypeHandleToRuntimeTypeHelper(op2);
9692 bool bOp1ClassFromHandle = op1->gtOper == GT_CALL ? gtIsTypeHandleToRuntimeTypeHelper(op1) : false;
9693 bool bOp2ClassFromHandle = op2->gtOper == GT_CALL ? gtIsTypeHandleToRuntimeTypeHelper(op2) : false;
9696 // Optimize typeof(...) == typeof(...)
9697 // Typically this occurs in generic code that attempts a type switch
9698 // e.g. typeof(T) == typeof(int)
9700 if (bOp1ClassFromHandle && bOp2ClassFromHandle)
9702 GenTreePtr classFromHandleArg1 = tree->gtOp.gtOp1->gtCall.gtCallArgs->gtOp.gtOp1;
9703 GenTreePtr classFromHandleArg2 = tree->gtOp.gtOp2->gtCall.gtCallArgs->gtOp.gtOp1;
9705 GenTreePtr compare = gtNewOperNode(oper, TYP_INT,
9706 classFromHandleArg1,
9707 classFromHandleArg2);
9709 compare->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED | GTF_RELOP_QMARK | GTF_DONT_CSE);
9712 return fgMorphTree(compare);
9714 else if (bOp1ClassFromHandle || bOp2ClassFromHandle)
9717 // Now check for GetClassFromHandle(handle) == obj.GetType()
9720 if (bOp1ClassFromHandle)
9722 pGetClassFromHandle = tree->gtOp.gtOp1;
9727 pGetClassFromHandle = tree->gtOp.gtOp2;
9731 GenTreePtr pGetClassFromHandleArgument = pGetClassFromHandle->gtCall.gtCallArgs->gtOp.gtOp1;
9732 GenTreePtr pConstLiteral = pGetClassFromHandleArgument;
9734 // Unwrap GT_NOP node used to prevent constant folding
9735 if (pConstLiteral->gtOper == GT_NOP && pConstLiteral->gtType == TYP_I_IMPL)
9737 pConstLiteral = pConstLiteral->gtOp.gtOp1;
9740 // In the ngen case, we have to go thru an indirection to get the right handle.
9741 if (pConstLiteral->gtOper == GT_IND)
9743 pConstLiteral = pConstLiteral->gtOp.gtOp1;
9745 #ifdef LEGACY_BACKEND
9747 if (pGetType->gtCall.gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC &&
9748 info.compCompHnd->getIntrinsicID(pGetType->gtCall.gtCallMethHnd) == CORINFO_INTRINSIC_Object_GetType &&
9750 if ((pGetType->gtOper == GT_INTRINSIC) && (pGetType->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Object_GetType) &&
9752 pConstLiteral->gtOper == GT_CNS_INT &&
9753 pConstLiteral->gtType == TYP_I_IMPL)
9755 CORINFO_CLASS_HANDLE clsHnd = CORINFO_CLASS_HANDLE(pConstLiteral->gtIntCon.gtCompileTimeHandle);
9757 if (info.compCompHnd->canInlineTypeCheckWithObjectVTable(clsHnd))
9759 // Method Table tree
9760 #ifdef LEGACY_BACKEND
9761 GenTreePtr objMT = gtNewOperNode(GT_IND, TYP_I_IMPL, pGetType->gtCall.gtCallObjp);
9763 GenTreePtr objMT = gtNewOperNode(GT_IND, TYP_I_IMPL, pGetType->gtUnOp.gtOp1);
9765 objMT->gtFlags |= GTF_EXCEPT; // Null ref exception if object is null
9766 compCurBB->bbFlags |= BBF_HAS_VTABREF;
9767 optMethodFlags |= OMF_HAS_VTABLEREF;
9769 // Method table constant
9770 GenTreePtr cnsMT = pGetClassFromHandleArgument;
9772 GenTreePtr compare = gtNewOperNode(oper, TYP_INT,
9776 compare->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED | GTF_RELOP_QMARK | GTF_DONT_CSE);
9779 return fgMorphTree(compare);
9784 fgMorphRecognizeBoxNullable(tree);
9785 op1 = tree->gtOp.gtOp1;
9786 op2 = tree->gtGetOp2();
9792 if (tree->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Round)
9794 switch (tree->TypeGet())
9797 return fgMorphIntoHelperCall(tree, CORINFO_HELP_DBLROUND, gtNewArgList(op1));
9799 return fgMorphIntoHelperCall(tree, CORINFO_HELP_FLTROUND, gtNewArgList(op1));
9811 #if !CPU_HAS_FP_SUPPORT
9812 tree = fgMorphToEmulatedFP(tree);
9815 /* Could this operator throw an exception? */
9816 if (fgGlobalMorph && tree->OperMayThrow())
9818 if ((tree->OperGet() != GT_IND) || fgAddrCouldBeNull(tree->gtOp.gtOp1))
9820 /* Mark the tree node as potentially throwing an exception */
9821 tree->gtFlags |= GTF_EXCEPT;
9825 /*-------------------------------------------------------------------------
9826 * Process the first operand, if any
9832 #if LOCAL_ASSERTION_PROP
9833 // If we are entering the "then" part of a Qmark-Colon we must
9834 // save the state of the current copy assignment table
9835 // so that we can restore this state when entering the "else" part
9838 noway_assert(optLocalAssertionProp);
9839 if (optAssertionCount)
9841 noway_assert(optAssertionCount <= optMaxAssertionCount); // else ALLOCA() is a bad idea
9842 unsigned tabSize = optAssertionCount * sizeof(AssertionDsc);
9843 origAssertionTab = (AssertionDsc*) ALLOCA(tabSize);
9844 origAssertionCount = optAssertionCount;
9845 memcpy(origAssertionTab, optAssertionTabPrivate, tabSize);
9849 origAssertionCount = 0;
9850 origAssertionTab = NULL;
9853 #endif // LOCAL_ASSERTION_PROP
9855 // We might need a new MorphAddressContext context. (These are used to convey
9856 // parent context about how addresses being calculated will be used; see the
9857 // specification comment for MorphAddrContext for full details.)
9858 // Assume it's an Ind context to start.
9859 MorphAddrContext subIndMac1(MACK_Ind);
9860 MorphAddrContext* subMac1 = mac;
9861 if (subMac1 == NULL || subMac1->m_kind == MACK_Ind || subMac1->m_kind == MACK_CopyBlock)
9863 switch (tree->gtOper)
9866 if (subMac1 == NULL)
9868 subMac1 = &subIndMac1;
9869 subMac1->m_kind = MACK_Addr;
9873 // In a comma, the incoming context only applies to the rightmost arg of the
9874 // comma list. The left arg (op1) gets a fresh context.
9879 assert(subMac1 == NULL); // Should only occur at top level, since value is void.
9880 subMac1 = &s_CopyBlockMAC;
9883 // If the list is the first arg of a copy block, its two args should be evaluated as
9884 // IND-context addresses, separately.
9885 if (subMac1 != NULL && subMac1->m_kind == MACK_CopyBlock)
9887 subMac1 = &subIndMac1;
9893 subMac1 = &subIndMac1;
9900 // For additions, if we're in an IND context keep track of whether
9901 // all offsets added to the address are constant, and their sum.
9902 if (tree->gtOper == GT_ADD && subMac1 != NULL)
9904 assert(subMac1->m_kind == MACK_Ind || subMac1->m_kind == MACK_Addr); // Can't be a CopyBlock.
9905 GenTreePtr otherOp = tree->gtOp.gtOp2;
9906 // Is the other operator a constant?
9907 if (otherOp->IsCnsIntOrI())
9909 ClrSafeInt<size_t> totalOffset(subMac1->m_totalOffset);
9910 totalOffset += otherOp->gtIntConCommon.IconValue();
9911 if (totalOffset.IsOverflow())
9913 // We will consider an offset so large as to overflow as "not a constant" --
9914 // we will do a null check.
9915 subMac1->m_allConstantOffsets = false;
9919 subMac1->m_totalOffset += otherOp->gtIntConCommon.IconValue();
9924 subMac1->m_allConstantOffsets = false;
9928 tree->gtOp.gtOp1 = op1 = fgMorphTree(op1, subMac1);
9930 #if LOCAL_ASSERTION_PROP
9931 // If we are exiting the "then" part of a Qmark-Colon we must
9932 // save the state of the current copy assignment table
9933 // so that we can merge this state with the "else" part exit
9936 noway_assert(optLocalAssertionProp);
9937 if (optAssertionCount)
9939 noway_assert(optAssertionCount <= optMaxAssertionCount); // else ALLOCA() is a bad idea
9940 unsigned tabSize = optAssertionCount * sizeof(AssertionDsc);
9941 thenAssertionTab = (AssertionDsc*) ALLOCA(tabSize);
9942 thenAssertionCount = optAssertionCount;
9943 memcpy(thenAssertionTab, optAssertionTabPrivate, tabSize);
9947 thenAssertionCount = 0;
9948 thenAssertionTab = NULL;
9951 #endif // LOCAL_ASSERTION_PROP
9953 /* Morphing along with folding and inlining may have changed the
9954 * side effect flags, so we have to reset them
9956 * NOTE: Don't reset the exception flags on nodes that may throw */
9958 noway_assert(tree->gtOper != GT_CALL);
9960 if ((tree->gtOper != GT_INTRINSIC) || !IsIntrinsicImplementedByUserCall(tree->gtIntrinsic.gtIntrinsicId))
9962 tree->gtFlags &= ~GTF_CALL;
9965 if (!tree->OperMayThrow())
9966 tree->gtFlags &= ~GTF_EXCEPT;
9968 /* Propagate the new flags */
9969 tree->gtFlags |= (op1->gtFlags & GTF_ALL_EFFECT);
9971 // &aliasedVar doesn't need GTF_GLOB_REF, though alisasedVar does
9972 // Similarly for clsVar
9973 if (oper == GT_ADDR && (op1->gtOper == GT_LCL_VAR || op1->gtOper == GT_CLS_VAR))
9974 tree->gtFlags &= ~GTF_GLOB_REF;
9977 /*-------------------------------------------------------------------------
9978 * Process the second operand, if any
9984 #if LOCAL_ASSERTION_PROP
9985 // If we are entering the "else" part of a Qmark-Colon we must
9986 // reset the state of the current copy assignment table
9989 noway_assert(optLocalAssertionProp);
9990 optAssertionReset(0);
9991 if (origAssertionCount)
9993 size_t tabSize = origAssertionCount * sizeof(AssertionDsc);
9994 memcpy(optAssertionTabPrivate, origAssertionTab, tabSize);
9995 optAssertionReset(origAssertionCount);
9998 #endif // LOCAL_ASSERTION_PROP
10000 // We might need a new MorphAddressContext context to use in evaluating op2.
10001 // (These are used to convey parent context about how addresses being calculated
10002 // will be used; see the specification comment for MorphAddrContext for full details.)
10003 // Assume it's an Ind context to start.
10004 MorphAddrContext subIndMac2(MACK_Ind);
10005 switch (tree->gtOper)
10008 if (mac != NULL && mac->m_kind == MACK_Ind)
10010 GenTreePtr otherOp = tree->gtOp.gtOp1;
10011 // Is the other operator a constant?
10012 if (otherOp->IsCnsIntOrI())
10014 mac->m_totalOffset += otherOp->gtIntConCommon.IconValue();
10018 mac->m_allConstantOffsets = false;
10023 if (mac != NULL && mac->m_kind == MACK_CopyBlock)
10031 tree->gtOp.gtOp2 = op2 = fgMorphTree(op2, mac);
10033 /* Propagate the side effect flags from op2 */
10035 tree->gtFlags |= (op2->gtFlags & GTF_ALL_EFFECT);
10037 #if LOCAL_ASSERTION_PROP
10038 // If we are exiting the "else" part of a Qmark-Colon we must
10039 // merge the state of the current copy assignment table with
10040 // that of the exit of the "then" part.
10043 noway_assert(optLocalAssertionProp);
10044 // If either exit table has zero entries then
10045 // the merged table also has zero entries
10046 if (optAssertionCount == 0 || thenAssertionCount == 0)
10048 optAssertionReset(0);
10052 size_t tabSize = optAssertionCount * sizeof(AssertionDsc);
10053 if ( (optAssertionCount != thenAssertionCount) ||
10054 (memcmp(thenAssertionTab, optAssertionTabPrivate, tabSize) != 0) )
10056 // Yes they are different so we have to find the merged set
10057 // Iterate over the copy asgn table removing any entries
10058 // that do not have an exact match in the thenAssertionTab
10059 AssertionIndex index = 1;
10060 while (index <= optAssertionCount)
10062 AssertionDsc* curAssertion = optGetAssertion(index);
10064 for (unsigned j=0; j < thenAssertionCount; j++)
10066 AssertionDsc* thenAssertion = &thenAssertionTab[j];
10068 // Do the left sides match?
10069 if ((curAssertion->op1.lcl.lclNum == thenAssertion->op1.lcl.lclNum) &&
10070 (curAssertion->assertionKind == thenAssertion->assertionKind))
10072 // Do the right sides match?
10073 if ((curAssertion->op2.kind == thenAssertion->op2.kind) &&
10074 (curAssertion->op2.lconVal == thenAssertion->op2.lconVal))
10085 // If we fall out of the loop above then we didn't find
10086 // any matching entry in the thenAssertionTab so it must
10087 // have been killed on that path so we remove it here
10090 // The data at optAssertionTabPrivate[i] is to be removed
10094 printf("The QMARK-COLON ");
10096 printf(" removes assertion candidate #%d\n", index);
10099 optAssertionRemove(index);
10102 // The data at optAssertionTabPrivate[i] is to be kept
10108 #endif // LOCAL_ASSERTION_PROP
10111 DONE_MORPHING_CHILDREN:
10113 /*-------------------------------------------------------------------------
10114 * Now do POST-ORDER processing
10117 #if FEATURE_FIXED_OUT_ARGS && !defined(_TARGET_64BIT_)
10118 // Variable shifts of a long end up being helper calls, so mark the tree as such. This
10119 // is potentially too conservative, since they'll get treated as having side effects.
10120 // It is important to mark them as calls so if they are part of an argument list,
10121 // they will get sorted and processed properly (for example, it is important to handle
10122 // all nested calls before putting struct arguments in the argument registers). We
10123 // could mark the trees just before argument processing, but it would require a full
10124 // tree walk of the argument tree, so we just do it here, instead, even though we'll
10125 // mark non-argument trees (that will still get converted to calls, anyway).
10126 if (GenTree::OperIsShift(oper) &&
10127 (tree->TypeGet() == TYP_LONG) &&
10128 (op2->OperGet() != GT_CNS_INT))
10130 tree->gtFlags |= GTF_CALL;
10132 #endif // FEATURE_FIXED_OUT_ARGS && !_TARGET_64BIT_
10134 if (varTypeIsGC(tree->TypeGet()) && (op1 && !varTypeIsGC(op1->TypeGet()))
10135 && (op2 && !varTypeIsGC(op2->TypeGet())))
10137 // The tree is really not GC but was marked as such. Now that the
10138 // children have been unmarked, unmark the tree too.
10140 // Remember that GT_COMMA inherits it's type only from op2
10141 if (tree->gtOper == GT_COMMA)
10142 tree->gtType = genActualType(op2->TypeGet());
10144 tree->gtType = genActualType(op1->TypeGet());
10147 GenTreePtr oldTree = tree;
10149 GenTreePtr qmarkOp1 = NULL;
10150 GenTreePtr qmarkOp2 = NULL;
10152 if ((tree->OperGet() == GT_QMARK) &&
10153 (tree->gtOp.gtOp2->OperGet() == GT_COLON))
10155 qmarkOp1 = oldTree->gtOp.gtOp2->gtOp.gtOp1;
10156 qmarkOp2 = oldTree->gtOp.gtOp2->gtOp.gtOp2;
10159 // Try to fold it, maybe we get lucky,
10160 tree = gtFoldExpr(tree);
10162 if (oldTree != tree)
10164 /* if gtFoldExpr returned op1 or op2 then we are done */
10165 if ((tree == op1) || (tree == op2) || (tree == qmarkOp1) || (tree == qmarkOp2))
10168 /* If we created a comma-throw tree then we need to morph op1 */
10169 if (fgIsCommaThrow(tree))
10171 tree->gtOp.gtOp1 = fgMorphTree(tree->gtOp.gtOp1);
10172 fgMorphTreeDone(tree);
10178 else if (tree->OperKind() & GTK_CONST)
10183 /* gtFoldExpr could have used setOper to change the oper */
10184 oper = tree->OperGet();
10185 typ = tree->TypeGet();
10187 /* gtFoldExpr could have changed op1 and op2 */
10188 op1 = tree->gtOp.gtOp1;
10189 op2 = tree->gtGetOp2();
10191 // Do we have an integer compare operation?
10193 if (tree->OperIsCompare() && varTypeIsIntegralOrI(tree->TypeGet()))
10195 // Are we comparing against zero?
10199 // Request that the codegen for op1 sets the condition flags
10200 // when it generates the code for op1.
10202 // Codegen for op1 must set the condition flags if
10203 // this method returns true.
10205 op1->gtRequestSetFlags();
10208 /*-------------------------------------------------------------------------
10209 * Perform the required oper-specific postorder morphing
10213 GenTreePtr cns1, cns2;
10214 GenTreePtr thenNode;
10215 GenTreePtr elseNode;
10216 size_t ival1, ival2;
10217 GenTreePtr lclVarTree;
10218 GenTreeLclVarCommon* lclVarCmnTree;
10219 FieldSeqNode* fieldSeq = NULL;
10225 lclVarTree = fgIsIndirOfAddrOfLocal(op1);
10226 if (lclVarTree != NULL)
10228 lclVarTree->gtFlags |= GTF_VAR_DEF;
10231 /* If we are storing a small type, we might be able to omit a cast */
10232 if ((op1->gtOper == GT_IND) && varTypeIsSmall(op1->TypeGet()))
10234 if (!gtIsActiveCSE_Candidate(op2) && (op2->gtOper == GT_CAST) && !op2->gtOverflow())
10236 var_types castType = op2->CastToType();
10238 // If we are performing a narrowing cast and
10239 // castType is larger or the same as op1's type
10240 // then we can discard the cast.
10242 if (varTypeIsSmall(castType) && (castType >= op1->TypeGet()))
10244 tree->gtOp.gtOp2 = op2 = op2->gtCast.CastOp();
10247 else if (op2->OperIsCompare() && varTypeIsByte(op1->TypeGet()))
10249 /* We don't need to zero extend the setcc instruction */
10250 op2->gtType = TYP_BYTE;
10253 // If we introduced a CSE we may need to undo the optimization above
10254 // (i.e. " op2->gtType = TYP_BYTE;" which depends upon op1 being a GT_IND of a byte type)
10255 // When we introduce the CSE we remove the GT_IND and subsitute a GT_LCL_VAR in it place.
10256 else if (op2->OperIsCompare() && (op2->gtType == TYP_BYTE) && (op1->gtOper == GT_LCL_VAR))
10258 unsigned varNum = op1->gtLclVarCommon.gtLclNum;
10259 LclVarDsc * varDsc = &lvaTable[varNum];
10261 /* We again need to zero extend the setcc instruction */
10262 op2->gtType = varDsc->TypeGet();
10270 fgAssignSetVarDef(tree);
10288 /* We can't CSE the LHS of an assignment */
10289 /* We also must set in the pre-morphing phase, otherwise assertionProp doesn't see it */
10290 op1->gtFlags |= GTF_DONT_CSE;
10296 /* Make sure we're allowed to do this */
10298 if (optValnumCSE_phase)
10300 // It is not safe to reorder/delete CSE's
10306 /* Check for "(expr +/- icon1) ==/!= (non-zero-icon2)" */
10308 if (cns2->gtOper == GT_CNS_INT && cns2->gtIntCon.gtIconVal != 0)
10310 op1 = tree->gtOp.gtOp1;
10312 /* Since this can occur repeatedly we use a while loop */
10314 while ((op1->gtOper == GT_ADD || op1->gtOper == GT_SUB) &&
10315 (op1->gtOp.gtOp2->gtOper == GT_CNS_INT) &&
10316 (op1->gtType == TYP_INT) &&
10317 (op1->gtOverflow() == false))
10319 /* Got it; change "x+icon1==icon2" to "x==icon2-icon1" */
10321 ival1 = op1->gtOp.gtOp2->gtIntCon.gtIconVal;
10322 ival2 = cns2->gtIntCon.gtIconVal;
10324 if (op1->gtOper == GT_ADD)
10332 cns2->gtIntCon.gtIconVal = ival2;
10334 #ifdef _TARGET_64BIT_
10335 // we need to properly re-sign-extend or truncate as needed.
10336 cns2->AsIntCon()->TruncateOrSignExtend32();
10337 #endif // _TARGET_64BIT_
10339 op1 = tree->gtOp.gtOp1 = op1->gtOp.gtOp1;
10344 // Here we look for the following tree
10350 ival2 = INT_MAX; // The value of INT_MAX for ival2 just means that the constant value is not 0 or 1
10352 // cast to unsigned allows test for both 0 and 1
10353 if ((cns2->gtOper == GT_CNS_INT) && (((size_t) cns2->gtIntConCommon.IconValue()) <= 1U))
10355 ival2 = (size_t) cns2->gtIntConCommon.IconValue();
10357 else // cast to UINT64 allows test for both 0 and 1
10358 if ((cns2->gtOper == GT_CNS_LNG) && (((UINT64) cns2->gtIntConCommon.LngValue()) <= 1ULL))
10360 ival2 = (size_t) cns2->gtIntConCommon.LngValue();
10363 if (ival2 != INT_MAX)
10365 // If we don't have a comma and relop, we can't do this optimization
10367 if ((op1->gtOper == GT_COMMA) && (op1->gtOp.gtOp2->OperIsCompare()))
10369 // Here we look for the following transformation
10371 // EQ/NE Possible REVERSE(RELOP)
10373 // COMMA CNS 0/1 -> COMMA relop_op2
10375 // x RELOP x relop_op1
10377 // relop_op1 relop_op2
10381 GenTreePtr comma = op1;
10382 GenTreePtr relop = comma->gtOp.gtOp2;
10384 GenTreePtr relop_op1 = relop->gtOp.gtOp1;
10386 bool reverse = ((ival2 == 0) == (oper == GT_EQ));
10390 gtReverseCond(relop);
10393 relop->gtOp.gtOp1 = comma;
10394 comma->gtOp.gtOp2 = relop_op1;
10396 // Comma now has fewer nodes underneath it, so we need to regenerate its flags
10397 comma->gtFlags &= ~GTF_ALL_EFFECT;
10398 comma->gtFlags |= (comma->gtOp.gtOp1->gtFlags) & GTF_ALL_EFFECT;
10399 comma->gtFlags |= (comma->gtOp.gtOp2->gtFlags) & GTF_ALL_EFFECT;
10401 noway_assert((relop->gtFlags & GTF_RELOP_JMP_USED) == 0);
10402 noway_assert((relop->gtFlags & GTF_REVERSE_OPS) == 0);
10403 relop->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED|GTF_RELOP_QMARK|GTF_DONT_CSE|GTF_ALL_EFFECT);
10408 if (op1->gtOper == GT_COMMA)
10410 // Here we look for the following tree
10411 // and when the LCL_VAR is a temp we can fold the tree:
10415 // COMMA CNS 0/1 -> RELOP CNS 0/1
10423 GenTreePtr asg = op1->gtOp.gtOp1;
10424 GenTreePtr lcl = op1->gtOp.gtOp2;
10426 /* Make sure that the left side of the comma is the assignment of the LCL_VAR */
10427 if (asg->gtOper != GT_ASG)
10430 /* The right side of the comma must be a LCL_VAR temp */
10431 if (lcl->gtOper != GT_LCL_VAR)
10434 unsigned lclNum = lcl->gtLclVarCommon.gtLclNum; noway_assert(lclNum < lvaCount);
10436 /* If the LCL_VAR is not a temp then bail, a temp has a single def */
10437 if (!lvaTable[lclNum].lvIsTemp)
10441 /* If the LCL_VAR is a CSE temp then bail, it could have multiple defs/uses */
10442 // Fix 383856 X86/ARM ILGEN
10443 if (lclNumIsCSE(lclNum))
10447 /* We also must be assigning the result of a RELOP */
10448 if (asg->gtOp.gtOp1->gtOper != GT_LCL_VAR)
10451 /* Both of the LCL_VAR must match */
10452 if (asg->gtOp.gtOp1->gtLclVarCommon.gtLclNum != lclNum)
10455 /* If right side of asg is not a RELOP then skip */
10456 if (!asg->gtOp.gtOp2->OperIsCompare())
10459 LclVarDsc * varDsc = lvaTable + lclNum;
10461 /* Set op1 to the right side of asg, (i.e. the RELOP) */
10462 op1 = asg->gtOp.gtOp2;
10464 DEBUG_DESTROY_NODE(asg->gtOp.gtOp1);
10465 DEBUG_DESTROY_NODE(lcl);
10467 /* This local variable should never be used again */
10469 // VSW 184221: Make RefCnt to zero to indicate that this local var
10470 // is not used any more. (Keey the lvType as is.)
10471 // Otherwise lvOnFrame will be set to true in Compiler::raMarkStkVars
10472 // And then emitter::emitEndCodeGen will assert in the following line:
10473 // noway_assert( dsc->lvTracked);
10475 noway_assert(varDsc->lvRefCnt == 0 || // lvRefCnt may not have been set yet.
10476 varDsc->lvRefCnt == 2 // Or, we assume this tmp should only be used here,
10477 // and it only shows up twice.
10479 lvaTable[lclNum].lvRefCnt = 0;
10480 lvaTable[lclNum].lvaResetSortAgainFlag(this);
10484 if (op1->OperIsCompare())
10486 // Here we look for the following tree
10488 // EQ/NE -> RELOP/!RELOP
10493 // Note that we will remove/destroy the EQ/NE node and move
10494 // the RELOP up into it's location.
10496 /* Here we reverse the RELOP if necessary */
10498 bool reverse = ((ival2 == 0) == (oper == GT_EQ));
10502 gtReverseCond(op1);
10505 /* Propagate gtType of tree into op1 in case it is TYP_BYTE for setcc optimization */
10506 op1->gtType = tree->gtType;
10508 noway_assert((op1->gtFlags & GTF_RELOP_JMP_USED) == 0);
10509 op1->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED|GTF_RELOP_QMARK|GTF_DONT_CSE);
10511 DEBUG_DESTROY_NODE(tree);
10517 // Now we check for a compare with the result of an '&' operator
10519 // Here we look for the following transformation:
10523 // AND CNS 0/1 -> AND CNS 0
10525 // RSZ/RSH CNS 1 x CNS (1 << y)
10529 if (op1->gtOper == GT_AND)
10531 GenTreePtr andOp = op1;
10532 GenTreePtr rshiftOp = andOp->gtOp.gtOp1;
10534 if ((rshiftOp->gtOper != GT_RSZ) && (rshiftOp->gtOper != GT_RSH))
10537 if (!rshiftOp->gtOp.gtOp2->IsCnsIntOrI())
10540 ssize_t shiftAmount = rshiftOp->gtOp.gtOp2->gtIntCon.gtIconVal;
10542 if (shiftAmount < 0)
10545 if (andOp->gtType == TYP_INT)
10547 if (!andOp->gtOp.gtOp2->IsCnsIntOrI())
10550 if (andOp->gtOp.gtOp2->gtIntCon.gtIconVal != 1)
10553 if (shiftAmount > 31)
10556 UINT32 newAndOperand = ((UINT32) 1) << shiftAmount;
10558 andOp->gtOp.gtOp2->gtIntCon.gtIconVal = newAndOperand;
10560 // Reverse the cond if necessary
10563 gtReverseCond(tree);
10564 cns2->gtIntCon.gtIconVal = 0;
10565 oper = tree->gtOper;
10569 else if (andOp->gtType == TYP_LONG)
10571 if (andOp->gtOp.gtOp2->gtOper != GT_CNS_NATIVELONG)
10574 if (andOp->gtOp.gtOp2->gtIntConCommon.LngValue() != 1)
10577 if (shiftAmount > 63)
10580 UINT64 newAndOperand = ((UINT64) 1) << shiftAmount;
10582 andOp->gtOp.gtOp2->gtIntConCommon.SetLngValue(newAndOperand);
10584 // Reverse the cond if necessary
10587 gtReverseCond(tree);
10588 cns2->gtIntConCommon.SetLngValue(0);
10589 oper = tree->gtOper;
10593 andOp->gtOp.gtOp1 = rshiftOp->gtOp.gtOp1;
10595 DEBUG_DESTROY_NODE(rshiftOp->gtOp.gtOp2);
10596 DEBUG_DESTROY_NODE(rshiftOp);
10598 } // END if (ival2 != INT_MAX)
10601 /* Now check for compares with small constant longs that can be cast to int */
10603 if (!cns2->OperIsConst())
10606 if (cns2->TypeGet() != TYP_LONG)
10609 /* Is the constant 31 bits or smaller? */
10611 if ((cns2->gtIntConCommon.LngValue() >> 31) != 0)
10614 /* Is the first comparand mask operation of type long ? */
10616 if (op1->gtOper != GT_AND)
10618 /* Another interesting case: cast from int */
10620 if (op1->gtOper == GT_CAST &&
10621 op1->CastFromType() == TYP_INT &&
10622 !gtIsActiveCSE_Candidate(op1) && // op1 cannot be a CSE candidate
10623 !op1->gtOverflow()) // cannot be an overflow checking cast
10625 /* Simply make this into an integer comparison */
10627 tree->gtOp.gtOp1 = op1->gtCast.CastOp();
10628 tree->gtOp.gtOp2 = gtNewIconNode((int)cns2->gtIntConCommon.LngValue(), TYP_INT);
10634 noway_assert(op1->TypeGet() == TYP_LONG && op1->OperGet() == GT_AND);
10636 /* Is the result of the mask effectively an INT ? */
10638 GenTreePtr andMask; andMask = op1->gtOp.gtOp2;
10639 if (andMask->gtOper != GT_CNS_NATIVELONG)
10641 if ((andMask->gtIntConCommon.LngValue() >> 32) != 0)
10644 /* Now we know that we can cast gtOp.gtOp1 of AND to int */
10646 op1->gtOp.gtOp1 = gtNewCastNode(TYP_INT,
10650 /* now replace the mask node (gtOp.gtOp2 of AND node) */
10652 noway_assert(andMask == op1->gtOp.gtOp2);
10654 ival1 = (int) andMask->gtIntConCommon.LngValue();
10655 andMask->SetOper(GT_CNS_INT);
10656 andMask->gtType = TYP_INT;
10657 andMask->gtIntCon.gtIconVal = ival1;
10659 /* now change the type of the AND node */
10661 op1->gtType = TYP_INT;
10663 /* finally we replace the comparand */
10665 ival2 = (int) cns2->gtIntConCommon.LngValue();
10666 cns2->SetOper(GT_CNS_INT);
10667 cns2->gtType = TYP_INT;
10669 noway_assert(cns2 == op2);
10670 cns2->gtIntCon.gtIconVal = ival2;
10679 if ((tree->gtFlags & GTF_UNSIGNED) == 0)
10681 if (op2->gtOper == GT_CNS_INT)
10684 /* Check for "expr relop 1" */
10685 if (cns2->gtIntCon.gtIconVal == +1)
10687 /* Check for "expr >= 1" */
10690 /* Change to "expr > 0" */
10694 /* Check for "expr < 1" */
10695 else if (oper == GT_LT)
10697 /* Change to "expr <= 0" */
10702 /* Check for "expr relop -1" */
10703 else if ((cns2->gtIntCon.gtIconVal == -1) && ((oper == GT_LE) || (oper == GT_GT)))
10705 /* Check for "expr <= -1" */
10708 /* Change to "expr < 0" */
10712 /* Check for "expr > -1" */
10713 else if (oper == GT_GT)
10715 /* Change to "expr >= 0" */
10719 // IF we get here we should be changing 'oper'
10720 assert(tree->OperGet() != oper);
10722 // Keep the old ValueNumber for 'tree' as the new expr
10723 // will still compute the same value as before
10724 tree->SetOper(oper, GenTree::PRESERVE_VN);
10725 cns2->gtIntCon.gtIconVal = 0;
10727 // vnStore is null before the ValueNumber phase has run
10728 if (vnStore != nullptr)
10730 // Update the ValueNumber for 'cns2', as we just changed it to 0
10731 fgValueNumberTreeConst(cns2);
10734 op2 = tree->gtOp.gtOp2 = gtFoldExpr(op2);
10742 noway_assert(tree->OperKind() & GTK_RELOP);
10744 /* Check if the result of the comparison is used for a jump.
10745 * If not then only the int (i.e. 32 bit) case is handled in
10746 * the code generator through the (x86) "set" instructions.
10747 * For the rest of the cases, the simplest way is to
10748 * "simulate" the comparison with ?:
10750 * On ARM, we previously used the IT instruction, but the IT instructions
10751 * have mostly been declared obsolete and off-limits, so all cases on ARM
10752 * get converted to ?: */
10754 if (!(tree->gtFlags & GTF_RELOP_JMP_USED) &&
10755 fgMorphRelopToQmark(op1))
10757 /* We convert it to "(CMP_TRUE) ? (1):(0)" */
10760 op1->gtFlags |= (GTF_RELOP_JMP_USED | GTF_RELOP_QMARK | GTF_DONT_CSE);
10761 op1->gtRequestSetFlags();
10763 op2 = new (this, GT_COLON) GenTreeColon(TYP_INT, gtNewIconNode(1), gtNewIconNode(0)
10765 op2 = fgMorphTree(op2);
10767 tree = gtNewQmarkNode(TYP_INT, op1, op2);
10769 fgMorphTreeDone(tree);
10777 /* If op1 is a comma throw node then we won't be keeping op2 */
10778 if (fgIsCommaThrow(op1))
10781 /* Get hold of the two branches */
10783 noway_assert(op2->OperGet() == GT_COLON);
10784 elseNode = op2->AsColon()->ElseNode();
10785 thenNode = op2->AsColon()->ThenNode();
10787 /* Try to hoist assignments out of qmark colon constructs.
10788 ie. replace (cond?(x=a):(x=b)) with (x=(cond?a:b)). */
10790 if (tree->TypeGet() == TYP_VOID &&
10791 thenNode->OperGet() == GT_ASG &&
10792 elseNode->OperGet() == GT_ASG &&
10793 thenNode->TypeGet() != TYP_LONG &&
10794 GenTree::Compare(thenNode->gtOp.gtOp1, elseNode->gtOp.gtOp1) &&
10795 thenNode->gtOp.gtOp2->TypeGet() == elseNode->gtOp.gtOp2->TypeGet())
10797 noway_assert(thenNode->TypeGet() == elseNode->TypeGet());
10799 GenTreePtr asg = thenNode;
10800 GenTreePtr colon = op2;
10801 colon->gtOp.gtOp1 = thenNode->gtOp.gtOp2;
10802 colon->gtOp.gtOp2 = elseNode->gtOp.gtOp2;
10803 tree->gtType = colon->gtType = asg->gtOp.gtOp2->gtType;
10804 asg->gtOp.gtOp2 = tree;
10806 // Asg will have all the flags that the QMARK had
10807 asg->gtFlags |= (tree->gtFlags & GTF_ALL_EFFECT);
10809 // Colon flag won't have the flags that x had.
10810 colon->gtFlags &= ~GTF_ALL_EFFECT;
10811 colon->gtFlags |= (colon->gtOp.gtOp1->gtFlags |
10812 colon->gtOp.gtOp2->gtFlags) & GTF_ALL_EFFECT;
10814 DEBUG_DESTROY_NODE(elseNode->gtOp.gtOp1);
10815 DEBUG_DESTROY_NODE(elseNode);
10821 /* If the 'else' branch is empty swap the two branches and reverse the condition */
10823 if (elseNode->IsNothingNode())
10825 /* This can only happen for VOID ?: */
10826 noway_assert(op2->gtType == TYP_VOID);
10828 /* If the thenNode and elseNode are both nop nodes then optimize away the QMARK */
10829 if (thenNode->IsNothingNode())
10831 // We may be able to throw away op1 (unless it has side-effects)
10833 if ((op1->gtFlags & GTF_SIDE_EFFECT) == 0)
10835 /* Just return a a Nop Node */
10840 /* Just return the relop, but clear the special flags. Note
10841 that we can't do that for longs and floats (see code under
10842 COMPARE label above) */
10844 if (!fgMorphRelopToQmark(op1->gtOp.gtOp1))
10846 op1->gtFlags &= ~(GTF_RELOP_QMARK | GTF_RELOP_JMP_USED);
10853 GenTreePtr tmp = elseNode;
10855 op2->AsColon()->ElseNode() = elseNode = thenNode;
10856 op2->AsColon()->ThenNode() = thenNode = tmp;
10857 gtReverseCond(op1);
10861 #if !defined(_TARGET_ARM_)
10862 // If we have (cond)?0:1, then we just return "cond" for TYP_INTs
10864 // Don't do this optimization for ARM: we always require assignment
10865 // to boolean to remain ?:, since we don't have any way to generate
10866 // this with straight-line code, like x86 does using setcc (at least
10867 // after the IT instruction is deprecated).
10869 if (genActualType(op1->gtOp.gtOp1->gtType) == TYP_INT &&
10870 genActualType(typ) == TYP_INT &&
10871 thenNode->gtOper == GT_CNS_INT &&
10872 elseNode->gtOper == GT_CNS_INT)
10874 ival1 = thenNode->gtIntCon.gtIconVal;
10875 ival2 = elseNode->gtIntCon.gtIconVal;
10877 // Is one constant 0 and the other 1?
10878 if ((ival1 | ival2) == 1 && (ival1 & ival2) == 0)
10880 // If the constants are {1, 0}, reverse the condition
10882 gtReverseCond(op1);
10884 // Unmark GTF_RELOP_JMP_USED on the condition node so it knows that it
10885 // needs to materialize the result as a 0 or 1.
10886 noway_assert(op1->gtFlags & (GTF_RELOP_QMARK | GTF_RELOP_JMP_USED));
10887 op1->gtFlags &= ~(GTF_RELOP_QMARK | GTF_RELOP_JMP_USED);
10889 DEBUG_DESTROY_NODE(tree);
10890 DEBUG_DESTROY_NODE(op2);
10895 #endif // !_TARGET_ARM_
10897 break; // end case GT_QMARK
10902 #ifndef _TARGET_64BIT_
10903 #if!LONG_MATH_REGPARAM
10904 if (typ == TYP_LONG)
10906 // This must be GTF_MUL_64RSLT
10907 assert(tree->gtIsValid64RsltMul());
10911 #endif // _TARGET_64BIT_
10916 if (tree->gtOverflow())
10919 /* Check for "op1 - cns2" , we change it to "op1 + (-cns2)" */
10922 if (op2->IsCnsIntOrI())
10924 /* Negate the constant and change the node to be "+" */
10926 op2->gtIntConCommon.SetIconValue(-op2->gtIntConCommon.IconValue());
10928 tree->ChangeOper(oper);
10932 /* Check for "cns1 - op2" , we change it to "(cns1 + (-op2))" */
10935 if (op1->IsCnsIntOrI())
10937 noway_assert(varTypeIsIntOrI(tree));
10939 tree->gtOp.gtOp2 = op2 = gtNewOperNode(GT_NEG, tree->gtType, op2); // The type of the new GT_NEG node should be the same
10940 // as the type of the tree, i.e. tree->gtType.
10941 fgMorphTreeDone(op2);
10944 tree->ChangeOper(oper);
10948 /* No match - exit */
10952 #ifdef _TARGET_ARM64_
10954 if (!varTypeIsFloating(tree->gtType))
10956 // Codegen for this instruction needs to be able to throw two exceptions:
10957 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_OVERFLOW, fgPtrArgCntCur);
10958 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_DIV_BY_ZERO, fgPtrArgCntCur);
10962 // Codegen for this instruction needs to be able to throw one exception:
10963 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_DIV_BY_ZERO, fgPtrArgCntCur);
10970 if (tree->gtOverflow())
10972 tree->gtRequestSetFlags();
10974 // Add the excptn-throwing basic block to jump to on overflow
10976 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_OVERFLOW, fgPtrArgCntCur);
10978 // We can't do any commutative morphing for overflow instructions
10989 /* Commute any non-REF constants to the right */
10992 if (op1->OperIsConst() && (op1->gtType != TYP_REF))
10994 // TODO-Review: We used to assert here that
10995 // noway_assert(!op2->OperIsConst() || !opts.OptEnabled(CLFLG_CONSTANTFOLD));
10996 // With modifications to AddrTaken==>AddrExposed, we did more assertion propagation,
10997 // and would sometimes hit this assertion. This may indicate a missed "remorph".
10998 // Task is to re-enable this assertion and investigate.
11000 /* Swap the operands */
11001 tree->gtOp.gtOp1 = op2;
11002 tree->gtOp.gtOp2 = op1;
11005 op2 = tree->gtOp.gtOp2;
11008 /* See if we can fold GT_ADD nodes. */
11010 if (oper == GT_ADD)
11012 /* Fold "((x+icon1)+(y+icon2)) to ((x+y)+(icon1+icon2))" */
11014 if (op1->gtOper == GT_ADD &&
11015 op2->gtOper == GT_ADD &&
11016 !gtIsActiveCSE_Candidate(op2) &&
11017 op1->gtOp.gtOp2->gtOper == GT_CNS_INT &&
11018 op2->gtOp.gtOp2->gtOper == GT_CNS_INT &&
11019 !op1->gtOverflow() &&
11020 !op2->gtOverflow() )
11022 cns1 = op1->gtOp.gtOp2;
11023 cns2 = op2->gtOp.gtOp2;
11024 cns1->gtIntCon.gtIconVal += cns2->gtIntCon.gtIconVal;
11025 #ifdef _TARGET_64BIT_
11026 if (cns1->TypeGet() == TYP_INT)
11028 // we need to properly re-sign-extend or truncate after adding two int constants above
11029 cns1->AsIntCon()->TruncateOrSignExtend32();
11031 #endif //_TARGET_64BIT_
11033 tree->gtOp.gtOp2 = cns1;
11034 DEBUG_DESTROY_NODE(cns2);
11036 op1->gtOp.gtOp2 = op2->gtOp.gtOp1;
11037 op1->gtFlags |= (op1->gtOp.gtOp2->gtFlags & GTF_ALL_EFFECT);
11038 DEBUG_DESTROY_NODE(op2);
11039 op2 = tree->gtOp.gtOp2;
11042 if (op2->IsCnsIntOrI() && varTypeIsIntegralOrI(typ))
11044 /* Fold "((x+icon1)+icon2) to (x+(icon1+icon2))" */
11046 if (op1->gtOper == GT_ADD &&
11047 !gtIsActiveCSE_Candidate(op1) &&
11048 op1->gtOp.gtOp2->IsCnsIntOrI() &&
11049 !op1->gtOverflow() &&
11050 op1->gtOp.gtOp2->OperGet() == op2->OperGet())
11052 cns1 = op1->gtOp.gtOp2;
11053 op2->gtIntConCommon.SetIconValue(cns1->gtIntConCommon.IconValue() + op2->gtIntConCommon.IconValue());
11054 #ifdef _TARGET_64BIT_
11055 if (op2->TypeGet() == TYP_INT)
11057 // we need to properly re-sign-extend or truncate after adding two int constants above
11058 op2->AsIntCon()->TruncateOrSignExtend32();
11060 #endif //_TARGET_64BIT_
11062 if (cns1->OperGet() == GT_CNS_INT)
11064 op2->gtIntCon.gtFieldSeq =
11065 GetFieldSeqStore()->Append(cns1->gtIntCon.gtFieldSeq,
11066 op2->gtIntCon.gtFieldSeq);
11068 DEBUG_DESTROY_NODE(cns1);
11070 tree->gtOp.gtOp1 = op1->gtOp.gtOp1;
11071 DEBUG_DESTROY_NODE(op1);
11072 op1 = tree->gtOp.gtOp1;
11077 if ((op2->gtIntConCommon.IconValue() == 0) && !gtIsActiveCSE_Candidate(tree))
11080 // If this addition is adding an offset to a null pointer,
11081 // avoid the work and yield the null pointer immediately.
11082 // Dereferencing the pointer in either case will have the
11085 if (!gtIsActiveCSE_Candidate(op1) && varTypeIsGC(op2->TypeGet()))
11087 op2->gtType = tree->gtType;
11088 DEBUG_DESTROY_NODE(op1);
11089 DEBUG_DESTROY_NODE(tree);
11093 // Remove the addition iff it won't change the tree type
11096 if (!gtIsActiveCSE_Candidate(op2) &&
11097 ((op1->TypeGet() == tree->TypeGet()) ||
11098 (op1->TypeGet() != TYP_REF)))
11100 if (fgGlobalMorph &&
11101 (op2->OperGet() == GT_CNS_INT) &&
11102 (op2->gtIntCon.gtFieldSeq != NULL) &&
11103 (op2->gtIntCon.gtFieldSeq != FieldSeqStore::NotAField()))
11105 fgAddFieldSeqForZeroOffset(op1, op2->gtIntCon.gtFieldSeq);
11108 DEBUG_DESTROY_NODE(op2);
11109 DEBUG_DESTROY_NODE(tree);
11116 /* See if we can fold GT_MUL by const nodes */
11117 else if (oper == GT_MUL && op2->IsCnsIntOrI() && !optValnumCSE_phase)
11119 #ifndef _TARGET_64BIT_
11120 noway_assert(typ <= TYP_UINT);
11121 #endif // _TARGET_64BIT_
11122 noway_assert(!tree->gtOverflow());
11124 ssize_t mult = op2->gtIntConCommon.IconValue();
11125 bool op2IsConstIndex = op2->OperGet() == GT_CNS_INT &&
11126 op2->gtIntCon.gtFieldSeq != nullptr &&
11127 op2->gtIntCon.gtFieldSeq->IsConstantIndexFieldSeq();
11129 assert(!op2IsConstIndex || op2->AsIntCon()->gtFieldSeq->m_next == nullptr);
11133 // We may be able to throw away op1 (unless it has side-effects)
11135 if ((op1->gtFlags & GTF_SIDE_EFFECT) == 0)
11137 DEBUG_DESTROY_NODE(op1);
11138 DEBUG_DESTROY_NODE(tree);
11139 return op2; // Just return the "0" node
11142 // We need to keep op1 for the side-effects. Hang it off
11145 tree->ChangeOper(GT_COMMA);
11149 size_t abs_mult = (mult >= 0) ? mult : -mult;
11150 size_t lowestBit = genFindLowestBit(abs_mult);
11151 bool changeToShift = false;
11153 // is it a power of two? (positive or negative)
11154 if (abs_mult == lowestBit)
11156 // if negative negate (min-int does not need negation)
11157 if (mult < 0 && mult != SSIZE_T_MIN)
11159 tree->gtOp.gtOp1 = op1 = gtNewOperNode(GT_NEG, op1->gtType, op1);
11160 fgMorphTreeDone(op1);
11163 // If "op2" is a constant array index, the other multiplicand must be a constant.
11164 // Transfer the annotation to the other one.
11165 if (op2->OperGet() == GT_CNS_INT &&
11166 op2->gtIntCon.gtFieldSeq != nullptr &&
11167 op2->gtIntCon.gtFieldSeq->IsConstantIndexFieldSeq())
11169 assert(op2->gtIntCon.gtFieldSeq->m_next == nullptr);
11170 GenTreePtr otherOp = op1;
11171 if (otherOp->OperGet() == GT_NEG)
11172 otherOp = otherOp->gtOp.gtOp1;
11173 assert(otherOp->OperGet() == GT_CNS_INT);
11174 assert(otherOp->gtIntCon.gtFieldSeq == FieldSeqStore::NotAField());
11175 otherOp->gtIntCon.gtFieldSeq = op2->gtIntCon.gtFieldSeq;
11180 DEBUG_DESTROY_NODE(op2);
11181 DEBUG_DESTROY_NODE(tree);
11185 /* Change the multiplication into a shift by log2(val) bits */
11186 op2->gtIntConCommon.SetIconValue(genLog2(abs_mult));
11187 changeToShift = true;
11190 else if ((lowestBit > 1) && jitIsScaleIndexMul(lowestBit) && optAvoidIntMult())
11192 int shift = genLog2(lowestBit);
11193 ssize_t factor = abs_mult >> shift;
11195 if (factor == 3 || factor == 5 || factor == 9)
11197 // if negative negate (min-int does not need negation)
11198 if (mult < 0 && mult != SSIZE_T_MIN)
11200 tree->gtOp.gtOp1 = op1 = gtNewOperNode(GT_NEG, op1->gtType, op1);
11201 fgMorphTreeDone(op1);
11204 GenTreePtr factorIcon = gtNewIconNode(factor, TYP_I_IMPL);
11205 if (op2IsConstIndex)
11207 factorIcon->AsIntCon()->gtFieldSeq = GetFieldSeqStore()->CreateSingleton(FieldSeqStore::ConstantIndexPseudoField);
11210 // change the multiplication into a smaller multiplication (by 3, 5 or 9) and a shift
11211 tree->gtOp.gtOp1 = op1 = gtNewOperNode(GT_MUL, tree->gtType, op1, factorIcon);
11212 fgMorphTreeDone(op1);
11214 op2->gtIntConCommon.SetIconValue(shift);
11215 changeToShift = true;
11218 #endif // LEA_AVAILABLE
11221 // vnStore is null before the ValueNumber phase has run
11222 if (vnStore != nullptr)
11224 // Update the ValueNumber for 'op2', as we just changed the constant
11225 fgValueNumberTreeConst(op2);
11228 // Keep the old ValueNumber for 'tree' as the new expr
11229 // will still compute the same value as before
11230 tree->ChangeOper(oper, GenTree::PRESERVE_VN);
11232 goto DONE_MORPHING_CHILDREN;
11235 else if (fgOperIsBitwiseRotationRoot(oper))
11237 tree = fgRecognizeAndMorphBitwiseRotation(tree);
11239 // fgRecognizeAndMorphBitwiseRotation may return a new tree
11240 oper = tree->OperGet();
11241 typ = tree->TypeGet();
11242 op1 = tree->gtOp.gtOp1;
11243 op2 = tree->gtOp.gtOp2;
11252 /* Any constant cases should have been folded earlier */
11253 noway_assert(!op1->OperIsConst() || !opts.OptEnabled(CLFLG_CONSTANTFOLD) || optValnumCSE_phase);
11258 noway_assert(varTypeIsFloating(op1->TypeGet()));
11260 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_ARITH_EXCPN, fgPtrArgCntCur);
11265 // Can not remove a GT_IND if it is currently a CSE candidate.
11266 if (gtIsActiveCSE_Candidate(tree))
11269 bool foldAndReturnTemp; foldAndReturnTemp = false;
11273 /* Try to Fold *(&X) into X */
11274 if (op1->gtOper == GT_ADDR)
11276 // Can not remove a GT_ADDR if it is currently a CSE candidate.
11277 if (gtIsActiveCSE_Candidate(op1))
11280 temp = op1->gtOp.gtOp1; // X
11282 // In the test below, if they're both TYP_STRUCT, this of course does *not* mean that
11283 // they are the *same* struct type. In fact, they almost certainly aren't. If the
11284 // address has an associated field sequence, that identifies this case; go through
11285 // the "lcl_fld" path rather than this one.
11286 FieldSeqNode* addrFieldSeq = NULL; // This is an unused out parameter below.
11287 if ( typ == temp->TypeGet()
11288 && !GetZeroOffsetFieldMap()->Lookup(op1, &addrFieldSeq))
11290 foldAndReturnTemp = true;
11292 else if (temp->OperIsLocal())
11294 unsigned lclNum = temp->gtLclVarCommon.gtLclNum;
11295 LclVarDsc * varDsc = &lvaTable[lclNum];
11297 // We will try to optimize when we have a promoted struct promoted with a zero lvFldOffset
11298 if (varDsc->lvPromoted && (varDsc->lvFldOffset == 0))
11300 noway_assert(varTypeIsStruct(varDsc));
11302 // We will try to optimize when we have a single field struct that is being struct promoted
11303 if (varDsc->lvFieldCnt == 1)
11305 unsigned lclNumFld = varDsc->lvFieldLclStart;
11306 // just grab the promoted field
11307 LclVarDsc * fieldVarDsc = &lvaTable[lclNumFld];
11309 // Also make sure that the tree type matches the fieldVarType and that it's lvFldOffset is zero
11310 if (fieldVarDsc->TypeGet() == tree->TypeGet() && (fieldVarDsc->lvFldOffset == 0))
11312 // We can just use the existing promoted field LclNum
11313 temp->gtLclVarCommon.SetLclNum(lclNumFld);
11314 temp->gtType = fieldVarDsc->TypeGet();
11316 foldAndReturnTemp = true;
11320 // If the type of the IND (typ) is a "small int", and the type of the local has the
11321 // same width, then we can reduce to just the local variable -- it will be
11322 // correctly normalized, and signed/unsigned differences won't matter.
11324 // The below transformation cannot be applied if the local var needs to be normalized on load.
11325 else if ( varTypeIsSmall(typ) &&
11326 (genTypeSize(lvaTable[lclNum].lvType) == genTypeSize(typ)) &&
11327 !lvaTable[lclNum].lvNormalizeOnLoad() )
11329 tree->gtType = temp->gtType;
11330 foldAndReturnTemp = true;
11334 // Assumes that when Lookup returns "false" it will leave "fieldSeq" unmodified (i.e. nullptr)
11335 assert(fieldSeq == nullptr);
11336 bool b = GetZeroOffsetFieldMap()->Lookup(op1, &fieldSeq);
11337 assert(b || fieldSeq == nullptr);
11339 if ((fieldSeq != nullptr) && (temp->OperGet() == GT_LCL_FLD))
11341 // Append the field sequence, change the type.
11342 temp->AsLclFld()->gtFieldSeq = GetFieldSeqStore()->Append(temp->AsLclFld()->gtFieldSeq, fieldSeq);
11343 temp->gtType = tree->TypeGet();
11345 foldAndReturnTemp = true;
11348 // Otherwise will will fold this into a GT_LCL_FLD below
11349 // where we check (temp != nullptr)
11351 else // !temp->OperIsLocal()
11353 // We don't try to fold away the GT_IND/GT_ADDR for this case
11357 else if (op1->OperGet() == GT_ADD)
11359 /* Try to change *(&lcl + cns) into lcl[cns] to prevent materialization of &lcl */
11361 if (op1->gtOp.gtOp1->OperGet() == GT_ADDR &&
11362 op1->gtOp.gtOp2->OperGet() == GT_CNS_INT
11363 && (!(opts.MinOpts() || opts.compDbgCode)))
11365 // No overflow arithmetic with pointers
11366 noway_assert(!op1->gtOverflow());
11368 temp = op1->gtOp.gtOp1->gtOp.gtOp1;
11369 if (!temp->OperIsLocal())
11375 // Can not remove the GT_ADDR if it is currently a CSE candidate.
11376 if (gtIsActiveCSE_Candidate(op1->gtOp.gtOp1))
11379 ival1 = op1->gtOp.gtOp2->gtIntCon.gtIconVal;
11380 fieldSeq = op1->gtOp.gtOp2->gtIntCon.gtFieldSeq;
11382 // Does the address have an associated zero-offset field sequence?
11383 FieldSeqNode* addrFieldSeq = NULL;
11384 if (GetZeroOffsetFieldMap()->Lookup(op1->gtOp.gtOp1, &addrFieldSeq))
11386 fieldSeq = GetFieldSeqStore()->Append(addrFieldSeq, fieldSeq);
11390 typ == temp->TypeGet() &&
11391 temp->TypeGet() != TYP_STRUCT)
11393 noway_assert(!varTypeIsGC(temp->TypeGet()));
11394 foldAndReturnTemp = true;
11398 // The emitter can't handle large offsets
11399 if (ival1 != (unsigned short)ival1)
11402 // The emitter can get confused by invalid offsets
11403 if (ival1 >= Compiler::lvaLclSize(temp->gtLclVarCommon.gtLclNum))
11406 #ifdef _TARGET_ARM_
11407 // Check for a LclVar TYP_STRUCT with misalignment on a Floating Point field
11409 if (varTypeIsFloating(tree->TypeGet()))
11411 if ((ival1 % emitTypeSize(tree->TypeGet())) != 0)
11413 tree->gtFlags |= GTF_IND_UNALIGNED;
11419 // Now we can fold this into a GT_LCL_FLD below
11420 // where we check (temp != nullptr)
11425 // If we have decided to fold, then temp cannot be nullptr
11426 if (foldAndReturnTemp)
11428 assert(temp != nullptr);
11432 if (temp != nullptr)
11434 noway_assert(op1->gtOper == GT_ADD || op1->gtOper == GT_ADDR);
11436 // If we haven't already decided to fold this expression
11438 if (!foldAndReturnTemp)
11440 noway_assert(temp->OperIsLocal());
11441 LclVarDsc* varDsc = &(lvaTable[temp->AsLclVarCommon()->gtLclNum]);
11442 // Make sure we don't separately promote the fields of this struct.
11443 if (varDsc->lvRegStruct)
11445 // We can enregister, but can't promote.
11446 varDsc->lvPromoted = false;
11450 lvaSetVarDoNotEnregister(temp->gtLclVarCommon.gtLclNum DEBUGARG(DNER_LocalField));
11453 // We will turn a GT_LCL_VAR into a GT_LCL_FLD with an gtLclOffs of 'ival'
11454 // or if we already have a GT_LCL_FLD we will adjust the gtLclOffs by adding 'ival'
11455 // Then we change the type of the GT_LCL_FLD to match the orginal GT_IND type.
11457 if (temp->OperGet() == GT_LCL_FLD)
11459 temp->AsLclFld()->gtLclOffs += (unsigned short)ival1;
11460 temp->AsLclFld()->gtFieldSeq =
11461 GetFieldSeqStore()->Append(temp->AsLclFld()->gtFieldSeq, fieldSeq);
11465 temp->ChangeOper(GT_LCL_FLD); // Note that this makes the gtFieldSeq "NotAField"...
11466 temp->AsLclFld()->gtLclOffs = (unsigned short)ival1;
11467 if (fieldSeq != NULL) // If it does represent a field, note that.
11468 temp->AsLclFld()->gtFieldSeq = fieldSeq;
11470 temp->gtType = tree->gtType;
11471 foldAndReturnTemp = true;
11474 assert(foldAndReturnTemp == true);
11476 // Keep the DONT_CSE flag in sync
11477 // (i.e keep the original value of this flag from tree)
11478 // as it can be set for 'temp' because a GT_ADDR always marks it for it's op1
11480 temp->gtFlags &= ~GTF_DONT_CSE;
11481 temp->gtFlags |= (tree->gtFlags & GTF_DONT_CSE);
11483 noway_assert(op1->gtOper == GT_ADD || op1->gtOper == GT_ADDR);
11484 noway_assert(temp->gtType == tree->gtType);
11486 if (op1->OperGet() == GT_ADD)
11488 DEBUG_DESTROY_NODE(op1->gtOp.gtOp1); // GT_ADDR
11489 DEBUG_DESTROY_NODE(op1->gtOp.gtOp2); // GT_CNS_INT
11491 DEBUG_DESTROY_NODE(op1); // GT_ADD or GT_ADDR
11492 DEBUG_DESTROY_NODE(tree); // GT_IND
11497 // Only do this optimization when we are in the global optimizer. Doing this after value numbering
11498 // could result in an invalid value number for the newly generated GT_IND node.
11499 if ((op1->OperGet() == GT_COMMA) && fgGlobalMorph)
11501 // Perform the transform IND(COMMA(x, ..., z)) == COMMA(x, ..., IND(z)).
11502 // TBD: this transformation is currently necessary for correctness -- it might
11503 // be good to analyze the failures that result if we don't do this, and fix them
11504 // in other ways. Ideally, this should be optional.
11505 GenTreePtr commaNode = op1;
11506 unsigned treeFlags = tree->gtFlags;
11507 commaNode->gtType = typ;
11508 commaNode->gtFlags = (treeFlags & ~GTF_REVERSE_OPS); // Bashing the GT_COMMA flags here is dangerous, clear the GTF_REVERSE_OPS at least.
11510 commaNode->gtFlags |= GTF_MORPHED;
11512 while (commaNode->gtOp.gtOp2->gtOper == GT_COMMA)
11514 commaNode = commaNode->gtOp.gtOp2;
11515 commaNode->gtType = typ;
11516 commaNode->gtFlags = (treeFlags & ~GTF_REVERSE_OPS); // Bashing the GT_COMMA flags here is dangerous, clear the GTF_REVERSE_OPS at least.
11518 commaNode->gtFlags |= GTF_MORPHED;
11521 bool wasArrIndex = (tree->gtFlags & GTF_IND_ARR_INDEX) != 0;
11525 bool b = GetArrayInfoMap()->Lookup(tree, &arrInfo);
11527 GetArrayInfoMap()->Remove(tree);
11530 op1 = gtNewOperNode(GT_IND, typ, commaNode->gtOp.gtOp2);
11531 op1->gtFlags = treeFlags;
11534 GetArrayInfoMap()->Set(op1, arrInfo);
11537 op1->gtFlags |= GTF_MORPHED;
11539 commaNode->gtOp.gtOp2 = op1;
11547 // Can not remove op1 if it is currently a CSE candidate.
11548 if (gtIsActiveCSE_Candidate(op1))
11551 if (op1->OperGet() == GT_IND)
11553 if ((op1->gtFlags & GTF_IND_ARR_INDEX) == 0)
11555 // Can not remove a GT_ADDR if it is currently a CSE candidate.
11556 if (gtIsActiveCSE_Candidate(tree))
11559 // Perform the transform ADDR(IND(...)) == (...).
11560 GenTreePtr addr = op1->gtOp.gtOp1;
11562 noway_assert(varTypeIsGC(addr->gtType) || addr->gtType == TYP_I_IMPL);
11564 DEBUG_DESTROY_NODE(op1);
11565 DEBUG_DESTROY_NODE(tree);
11570 else if (op1->gtOper == GT_CAST)
11572 GenTreePtr casting = op1->gtCast.CastOp();
11573 if (casting->gtOper == GT_LCL_VAR || casting->gtOper == GT_CLS_VAR)
11575 DEBUG_DESTROY_NODE(op1);
11576 tree->gtOp.gtOp1 = op1 = casting;
11579 else if ((op1->gtOper == GT_COMMA) && !optValnumCSE_phase)
11581 // Perform the transform ADDR(COMMA(x, ..., z)) == COMMA(x, ..., ADDR(z)).
11582 // (Be sure to mark "z" as an l-value...)
11583 GenTreePtr commaNode = op1;
11584 while (commaNode->gtOp.gtOp2->gtOper == GT_COMMA)
11586 commaNode = commaNode->gtOp.gtOp2;
11588 // The top-level addr might be annotated with a zeroOffset field.
11589 FieldSeqNode* zeroFieldSeq = nullptr;
11590 bool isZeroOffset = GetZeroOffsetFieldMap()->Lookup(tree, &zeroFieldSeq);
11592 commaNode->gtOp.gtOp2->gtFlags |= GTF_DONT_CSE;
11594 // If the node we're about to put under a GT_ADDR is a GT_IND, the indirection
11595 // doesn't need to be materialized, since we only want the addressing mode. Because
11596 // of this, this GT_IND is not a faulting indirection and we don't have to extract it
11597 // as a side effect.
11598 GenTree* commaOp2 = commaNode->gtOp.gtOp2;
11599 if (commaOp2->gtOper == GT_IND)
11601 commaOp2->gtFlags |= GTF_IND_NONFAULTING;
11604 op1 = gtNewOperNode(GT_ADDR, TYP_BYREF, commaOp2);
11608 // Transfer the annotation to the new GT_ADDR node.
11609 GetZeroOffsetFieldMap()->Set(op1, zeroFieldSeq);
11611 commaNode->gtOp.gtOp2 = op1;
11612 // Originally, I gave all the comma nodes type "byref". But the ADDR(IND(x)) == x transform
11613 // might give op1 a type different from byref (like, say, native int). So now go back and give
11614 // all the comma nodes the type of op1.
11616 while (commaNode->gtOper == GT_COMMA)
11618 commaNode->gtType = op1->gtType; commaNode->gtFlags |= op1->gtFlags;
11620 commaNode->gtFlags |= GTF_MORPHED;
11622 commaNode = commaNode->gtOp.gtOp2;
11628 /* op1 of a GT_ADDR is an l-value. Only r-values can be CSEed */
11629 op1->gtFlags |= GTF_DONT_CSE;
11635 /* Mark the nodes that are conditionally executed */
11636 fgWalkTreePre(&tree, gtMarkColonCond);
11638 /* Since we're doing this postorder we clear this if it got set by a child */
11639 fgRemoveRestOfBlock = false;
11644 /* Special case: trees that don't produce a value */
11645 if ((op2->OperKind() & GTK_ASGOP) ||
11646 (op2->OperGet() == GT_COMMA && op2->TypeGet() == TYP_VOID) ||
11649 typ = tree->gtType = TYP_VOID;
11652 // If we are in the Valuenum CSE phase then don't morph away anything as these
11653 // nodes may have CSE defs/uses in them.
11655 if (!optValnumCSE_phase)
11657 //Extract the side effects from the left side of the comma. Since they don't "go" anywhere, this is
11660 GenTreePtr op1SideEffects = NULL;
11661 // The addition of "GTF_MAKE_CSE" below prevents us from throwing away (for example)
11662 // hoisted expressions in loops.
11663 gtExtractSideEffList(op1, &op1SideEffects, (GTF_SIDE_EFFECT | GTF_MAKE_CSE));
11664 if (op1SideEffects)
11666 //Replace the left hand side with the side effect list.
11667 tree->gtOp.gtOp1 = op1SideEffects;
11668 tree->gtFlags |= (op1SideEffects->gtFlags & GTF_ALL_EFFECT);
11672 /* The left operand is worthless, throw it away */
11673 if (lvaLocalVarRefCounted)
11675 lvaRecursiveDecRefCounts(op1);
11677 op2->gtFlags |= (tree->gtFlags & (GTF_DONT_CSE | GTF_LATE_ARG));
11678 DEBUG_DESTROY_NODE(tree);
11679 DEBUG_DESTROY_NODE(op1);
11683 /* If the right operand is just a void nop node, throw it away */
11684 if (op2->IsNothingNode() && op1->gtType == TYP_VOID)
11686 op1->gtFlags |= (tree->gtFlags & (GTF_DONT_CSE | GTF_LATE_ARG));
11687 DEBUG_DESTROY_NODE(tree);
11688 DEBUG_DESTROY_NODE(op2);
11697 /* Special case if fgRemoveRestOfBlock is set to true */
11698 if (fgRemoveRestOfBlock)
11700 if (fgIsCommaThrow(op1, true))
11702 GenTreePtr throwNode = op1->gtOp.gtOp1;
11703 noway_assert(throwNode->gtType == TYP_VOID);
11708 noway_assert(op1->OperKind() & GTK_RELOP);
11709 noway_assert(op1->gtFlags & GTF_EXCEPT);
11711 // We need to keep op1 for the side-effects. Hang it off
11714 tree->ChangeOper(GT_COMMA);
11715 tree->gtOp.gtOp2 = op2 = gtNewNothingNode();
11717 // Additionally since we're eliminating the JTRUE
11718 // codegen won't like it if op1 is a RELOP of longs, floats or doubles.
11719 // So we change it into a GT_COMMA as well.
11720 op1->ChangeOper(GT_COMMA);
11721 op1->gtType = op1->gtOp.gtOp1->gtType;
11730 noway_assert(oper == tree->gtOper);
11732 // If we are in the Valuenum CSE phase then don't morph away anything as these
11733 // nodes may have CSE defs/uses in them.
11735 if (!optValnumCSE_phase && (oper != GT_ASG) && (oper != GT_COLON) && !tree->IsList())
11737 /* Check for op1 as a GT_COMMA with a unconditional throw node */
11738 if (op1 && fgIsCommaThrow(op1, true))
11740 if ((op1->gtFlags & GTF_COLON_COND) == 0)
11742 /* We can safely throw out the rest of the statements */
11743 fgRemoveRestOfBlock = true;
11746 GenTreePtr throwNode = op1->gtOp.gtOp1;
11747 noway_assert(throwNode->gtType == TYP_VOID);
11749 if (oper == GT_COMMA)
11751 /* Both tree and op1 are GT_COMMA nodes */
11752 /* Change the tree's op1 to the throw node: op1->gtOp.gtOp1 */
11753 tree->gtOp.gtOp1 = throwNode;
11756 else if (oper != GT_NOP)
11758 if (genActualType(typ) == genActualType(op1->gtType))
11760 /* The types match so, return the comma throw node as the new tree */
11765 if (typ == TYP_VOID)
11767 // Return the throw node
11772 GenTreePtr commaOp2 = op1->gtOp.gtOp2;
11774 // need type of oper to be same as tree
11775 if (typ == TYP_LONG)
11777 commaOp2->ChangeOperConst(GT_CNS_NATIVELONG);
11778 commaOp2->gtIntConCommon.SetLngValue(0);
11779 /* Change the types of oper and commaOp2 to TYP_LONG */
11780 op1->gtType = commaOp2->gtType = TYP_LONG;
11782 else if (varTypeIsFloating(typ))
11784 commaOp2->ChangeOperConst(GT_CNS_DBL);
11785 commaOp2->gtDblCon.gtDconVal = 0.0;
11786 /* Change the types of oper and commaOp2 to TYP_DOUBLE */
11787 op1->gtType = commaOp2->gtType = TYP_DOUBLE;
11791 commaOp2->ChangeOperConst(GT_CNS_INT);
11792 commaOp2->gtIntConCommon.SetIconValue(0);
11793 /* Change the types of oper and commaOp2 to TYP_INT */
11794 op1->gtType = commaOp2->gtType = TYP_INT;
11797 /* Return the GT_COMMA node as the new tree */
11804 /* Check for op2 as a GT_COMMA with a unconditional throw */
11806 if (op2 && fgIsCommaThrow(op2, true))
11808 if ((op2->gtFlags & GTF_COLON_COND) == 0)
11810 /* We can safely throw out the rest of the statements */
11811 fgRemoveRestOfBlock = true;
11814 // If op1 has no side-effects
11815 if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
11817 // If tree is an asg node
11818 if (tree->OperIsAssignment())
11820 /* Return the throw node as the new tree */
11821 return op2->gtOp.gtOp1;
11824 if (tree->OperGet() == GT_ARR_BOUNDS_CHECK)
11826 /* Return the throw node as the new tree */
11827 return op2->gtOp.gtOp1;
11830 // If tree is a comma node
11831 if (tree->OperGet() == GT_COMMA)
11833 /* Return the throw node as the new tree */
11834 return op2->gtOp.gtOp1;
11837 /* for the shift nodes the type of op2 can differ from the tree type */
11838 if ((typ == TYP_LONG) && (genActualType(op2->gtType) == TYP_INT))
11840 noway_assert(GenTree::OperIsShiftOrRotate(oper));
11842 GenTreePtr commaOp2 = op2->gtOp.gtOp2;
11844 commaOp2->ChangeOperConst(GT_CNS_NATIVELONG);
11845 commaOp2->gtIntConCommon.SetLngValue(0);
11847 /* Change the types of oper and commaOp2 to TYP_LONG */
11848 op2->gtType = commaOp2->gtType = TYP_LONG;
11851 if ((genActualType(typ) == TYP_INT) && (genActualType(op2->gtType) == TYP_LONG ||
11852 varTypeIsFloating(op2->TypeGet())))
11854 // An example case is comparison (say GT_GT) of two longs or floating point values.
11856 GenTreePtr commaOp2 = op2->gtOp.gtOp2;
11858 commaOp2->ChangeOperConst(GT_CNS_INT);
11859 commaOp2->gtIntCon.gtIconVal = 0;
11860 /* Change the types of oper and commaOp2 to TYP_INT */
11861 op2->gtType = commaOp2->gtType = TYP_INT;
11864 if ((typ == TYP_BYREF) && (genActualType(op2->gtType) == TYP_I_IMPL))
11866 noway_assert(tree->OperGet() == GT_ADD);
11868 GenTreePtr commaOp2 = op2->gtOp.gtOp2;
11870 commaOp2->ChangeOperConst(GT_CNS_INT);
11871 commaOp2->gtIntCon.gtIconVal = 0;
11872 /* Change the types of oper and commaOp2 to TYP_BYREF */
11873 op2->gtType = commaOp2->gtType = TYP_BYREF;
11876 /* types should now match */
11877 noway_assert( (genActualType(typ) == genActualType(op2->gtType)));
11879 /* Return the GT_COMMA node as the new tree */
11885 /*-------------------------------------------------------------------------
11886 * Optional morphing is done if tree transformations is permitted
11889 if ((opts.compFlags & CLFLG_TREETRANS) == 0)
11892 tree = fgMorphSmpOpOptional(tree->AsOp());
11894 } // extra scope for gcc workaround
11898 #pragma warning(pop)
11902 GenTree* Compiler::fgMorphSmpOpOptional(GenTreeOp* tree)
11904 genTreeOps oper = tree->gtOper;
11905 GenTree* op1 = tree->gtOp1;
11906 GenTree* op2 = tree->gtOp2;
11907 var_types typ = tree->TypeGet();
11909 if (GenTree::OperIsCommutative(oper))
11911 /* Swap the operands so that the more expensive one is 'op1' */
11913 if (tree->gtFlags & GTF_REVERSE_OPS)
11921 tree->gtFlags &= ~GTF_REVERSE_OPS;
11924 if (oper == op2->gtOper)
11926 /* Reorder nested operators at the same precedence level to be
11927 left-recursive. For example, change "(a+(b+c))" to the
11928 equivalent expression "((a+b)+c)".
11931 /* Things are handled differently for floating-point operators */
11933 if (!varTypeIsFloating(tree->TypeGet()))
11935 fgMoveOpsLeft(tree);
11945 /* Change "((x+icon)+y)" to "((x+y)+icon)"
11946 Don't reorder floating-point operations */
11948 if ((oper == GT_ADD) && !tree->gtOverflow() &&
11949 (op1->gtOper == GT_ADD) && ! op1->gtOverflow() && varTypeIsIntegralOrI(typ))
11951 GenTreePtr ad2 = op1->gtOp.gtOp2;
11953 if (op2->OperIsConst() == 0 &&
11954 ad2->OperIsConst() != 0)
11966 // And it swaps ad2 and op2. If (op2) is varTypeIsGC, then this implies that (tree) is
11967 // varTypeIsGC. If (op1) is not, then when we swap (ad2) and (op2), then we have a TYP_INT node
11968 // (op1) with a child that is varTypeIsGC. If we encounter that situation, make (op1) the same
11971 // Also, if (ad2) is varTypeIsGC then (tree) must also be (since op1 is), so no fixing is
11974 if (varTypeIsGC(op2->TypeGet()))
11976 noway_assert(varTypeIsGC(typ));
11981 op1 ->gtOp.gtOp2 = op2;
11982 op1->gtFlags |= op2->gtFlags & GTF_ALL_EFFECT;
11990 /*-------------------------------------------------------------------------
11991 * Perform optional oper-specific postorder morphing
11997 bool dstIsSafeLclVar;
12001 /* We'll convert "a = a <op> x" into "a <op>= x" */
12002 /* and also "a = x <op> a" into "a <op>= x" for communative ops */
12005 if (typ == TYP_LONG)
12009 /* Make sure we're allowed to do this */
12011 if (optValnumCSE_phase)
12013 // It is not safe to reorder/delete CSE's
12017 /* Are we assigning to a GT_LCL_VAR ? */
12019 dstIsSafeLclVar = (op1->gtOper == GT_LCL_VAR);
12021 /* If we have a GT_LCL_VAR, then is the address taken? */
12022 if (dstIsSafeLclVar)
12024 unsigned lclNum = op1->gtLclVarCommon.gtLclNum;
12025 LclVarDsc * varDsc = lvaTable + lclNum;
12027 noway_assert(lclNum < lvaCount);
12029 /* Is the address taken? */
12030 if (varDsc->lvAddrExposed)
12032 dstIsSafeLclVar = false;
12034 else if (op2->gtFlags & GTF_ASG)
12040 if (!dstIsSafeLclVar)
12042 if (op2->gtFlags & GTF_ASG)
12045 if ((op2->gtFlags & GTF_CALL) && (op1->gtFlags & GTF_ALL_EFFECT))
12049 /* Special case: a cast that can be thrown away */
12051 if (op1->gtOper == GT_IND &&
12052 op2->gtOper == GT_CAST &&
12053 !op2->gtOverflow() )
12059 srct = op2->gtCast.CastOp()->TypeGet();
12060 cast = (var_types) op2->CastToType();
12061 dstt = op1->TypeGet();
12063 /* Make sure these are all ints and precision is not lost */
12065 if (cast >= dstt && dstt <= TYP_INT && srct <= TYP_INT)
12066 op2 = tree->gtOp2 = op2->gtCast.CastOp();
12069 /* Make sure we have the operator range right */
12071 noway_assert(GT_SUB == GT_ADD + 1);
12072 noway_assert(GT_MUL == GT_ADD + 2);
12073 noway_assert(GT_DIV == GT_ADD + 3);
12074 noway_assert(GT_MOD == GT_ADD + 4);
12075 noway_assert(GT_UDIV== GT_ADD + 5);
12076 noway_assert(GT_UMOD== GT_ADD + 6);
12078 noway_assert(GT_OR == GT_ADD + 7);
12079 noway_assert(GT_XOR == GT_ADD + 8);
12080 noway_assert(GT_AND == GT_ADD + 9);
12082 noway_assert(GT_LSH == GT_ADD + 10);
12083 noway_assert(GT_RSH == GT_ADD + 11);
12084 noway_assert(GT_RSZ == GT_ADD + 12);
12086 /* Check for a suitable operator on the RHS */
12088 cmop = op2->OperGet();
12093 // GT_CHS only supported for integer types
12094 if ( varTypeIsFloating(tree->TypeGet()))
12100 // GT_ASG_MUL only supported for floating point types
12101 if (!varTypeIsFloating(tree->TypeGet()))
12108 if (op2->gtOverflow())
12110 /* Disable folding into "<op>=" if the result can be
12111 visible to anyone as <op> may throw an exception and
12112 the assignment should not proceed
12113 We are safe with an assignment to a local variables
12115 if (ehBlockHasExnFlowDsc(compCurBB))
12117 if (!dstIsSafeLclVar)
12120 #ifndef _TARGET_AMD64_
12121 // This is hard for byte-operations as we need to make
12122 // sure both operands are in RBM_BYTE_REGS.
12123 if (varTypeIsByte(op2->TypeGet()))
12125 #endif // _TARGET_AMD64_
12130 // GT_ASG_DIV only supported for floating point types
12131 if (!varTypeIsFloating(tree->TypeGet()))
12140 if (typ == TYP_LONG)
12150 /* TODO: allow non-const long assignment operators */
12152 if (typ == TYP_LONG && op2->gtOp.gtOp2->gtOper != GT_CNS_LNG)
12158 bool bReverse = false;
12159 bool bAsgOpFoldable = fgShouldCreateAssignOp(tree, &bReverse);
12160 if (bAsgOpFoldable)
12164 // We will transform this from "a = x <op> a" to "a <op>= x"
12165 // so we can now destroy the duplicate "a"
12166 DEBUG_DESTROY_NODE(op2->gtOp.gtOp2);
12167 op2->gtOp.gtOp2 = op2->gtOp.gtOp1;
12170 /* Special case: "x |= -1" and "x &= 0" */
12171 if (cmop == GT_AND || cmop == GT_OR)
12173 if (op2->gtOp.gtOp2->IsCnsIntOrI())
12175 ssize_t icon = op2->gtOp.gtOp2->gtIntCon.gtIconVal;
12177 noway_assert(typ <= TYP_UINT);
12179 if ((cmop == GT_AND && icon == 0) ||
12180 (cmop == GT_OR && icon == -1))
12182 /* Simply change to an assignment */
12183 tree->gtOp2 = op2->gtOp.gtOp2;
12189 if (cmop == GT_NEG)
12191 /* This is "x = -x;", use the flipsign operator */
12193 tree->ChangeOper (GT_CHS);
12195 if (op1->gtOper == GT_LCL_VAR)
12196 op1->gtFlags |= GTF_VAR_USEASG;
12198 tree->gtOp2 = gtNewIconNode(0, op1->TypeGet());
12203 if (cmop == GT_RSH && varTypeIsSmall(op1->TypeGet()) && varTypeIsUnsigned(op1->TypeGet()))
12205 // Changing from x = x op y to x op= y when x is a small integer type
12206 // makes the op size smaller (originally the op size was 32 bits, after
12207 // sign or zero extension of x, and there is an implicit truncation in the
12209 // This is ok in most cases because the upper bits were
12210 // lost when assigning the op result to a small type var,
12211 // but it may not be ok for the right shift operation where the higher bits
12212 // could be shifted into the lower bits and preserved.
12213 // Signed right shift of signed x still works (i.e. (sbyte)((int)(sbyte)x >>signed y) == (sbyte)x >>signed y))
12214 // as do unsigned right shift ((ubyte)((int)(ubyte)x >>unsigned y) == (ubyte)x >>unsigned y), but
12215 // signed right shift of an unigned small type may give the wrong result:
12216 // e.g. (ubyte)((int)(ubyte)0xf0 >>signed 4) == 0x0f,
12217 // but (ubyte)0xf0 >>signed 4 == 0xff which is incorrect.
12218 // The result becomes correct if we use >>unsigned instead of >>signed.
12219 noway_assert(op1->TypeGet() == op2->gtOp.gtOp1->TypeGet());
12223 /* Replace with an assignment operator */
12224 noway_assert(GT_ADD - GT_ADD == GT_ASG_ADD - GT_ASG_ADD);
12225 noway_assert(GT_SUB - GT_ADD == GT_ASG_SUB - GT_ASG_ADD);
12226 noway_assert(GT_OR - GT_ADD == GT_ASG_OR - GT_ASG_ADD);
12227 noway_assert(GT_XOR - GT_ADD == GT_ASG_XOR - GT_ASG_ADD);
12228 noway_assert(GT_AND - GT_ADD == GT_ASG_AND - GT_ASG_ADD);
12229 noway_assert(GT_LSH - GT_ADD == GT_ASG_LSH - GT_ASG_ADD);
12230 noway_assert(GT_RSH - GT_ADD == GT_ASG_RSH - GT_ASG_ADD);
12231 noway_assert(GT_RSZ - GT_ADD == GT_ASG_RSZ - GT_ASG_ADD);
12233 tree->SetOper((genTreeOps)(cmop - GT_ADD + GT_ASG_ADD));
12234 tree->gtOp2 = op2->gtOp.gtOp2;
12236 /* Propagate GTF_OVERFLOW */
12238 if (op2->gtOverflowEx())
12240 tree->gtType = op2->gtType;
12241 tree->gtFlags |= (op2->gtFlags &
12242 (GTF_OVERFLOW|GTF_EXCEPT|GTF_UNSIGNED));
12245 #if FEATURE_SET_FLAGS
12247 /* Propagate GTF_SET_FLAGS */
12248 if (op2->gtSetFlags())
12250 tree->gtRequestSetFlags();
12253 #endif // FEATURE_SET_FLAGS
12255 DEBUG_DESTROY_NODE(op2);
12258 /* The target is used as well as being defined */
12259 if (op1->OperIsLocal())
12260 op1->gtFlags |= GTF_VAR_USEASG;
12263 #if CPU_HAS_FP_SUPPORT
12264 /* Check for the special case "x += y * x;" */
12266 // GT_ASG_MUL only supported for floating point types
12267 if (cmop != GT_ADD && cmop != GT_SUB)
12270 if (op2->gtOper == GT_MUL && varTypeIsFloating(tree->TypeGet()))
12272 if (GenTree::Compare(op1, op2->gtOp.gtOp1))
12274 /* Change "x += x * y" into "x *= (y + 1)" */
12276 op2 = op2->gtOp.gtOp2;
12278 else if (GenTree::Compare(op1, op2->gtOp.gtOp2))
12280 /* Change "x += y * x" into "x *= (y + 1)" */
12282 op2 = op2->gtOp.gtOp1;
12287 op1 = gtNewDconNode(1.0);
12289 /* Now make the "*=" node */
12291 if (cmop == GT_ADD)
12293 /* Change "x += x * y" into "x *= (y + 1)" */
12295 tree->gtOp2 = op2 = gtNewOperNode(GT_ADD,
12302 /* Change "x -= x * y" into "x *= (1 - y)" */
12304 noway_assert(cmop == GT_SUB);
12305 tree->gtOp2 = op2 = gtNewOperNode(GT_SUB,
12310 tree->ChangeOper(GT_ASG_MUL);
12312 #endif // CPU_HAS_FP_SUPPORT
12320 /* Is the destination identical to the first RHS sub-operand? */
12322 if (GenTree::Compare(op1, op2->gtOp.gtOp1))
12324 /* This is "x = ~x" which is the same as "x ^= -1"
12325 * Transform the node into a GT_ASG_XOR */
12327 noway_assert(genActualType(typ) == TYP_INT ||
12328 genActualType(typ) == TYP_LONG);
12330 op2->gtOp.gtOp2 = (genActualType(typ) == TYP_INT)
12331 ? gtNewIconNode(-1)
12332 : gtNewLconNode(-1);
12347 /* Check for the case "(val + icon) * icon" */
12349 if (op2->gtOper == GT_CNS_INT &&
12350 op1->gtOper == GT_ADD)
12352 GenTreePtr add = op1->gtOp.gtOp2;
12354 if (add->IsCnsIntOrI() && (op2->GetScaleIndexMul() != 0))
12356 if (tree->gtOverflow() || op1->gtOverflow())
12361 ssize_t imul = op2->gtIntCon.gtIconVal;
12362 ssize_t iadd = add->gtIntCon.gtIconVal;
12364 /* Change '(val + iadd) * imul' -> '(val * imul) + (iadd * imul)' */
12367 tree->ChangeOper(oper);
12369 op2->gtIntCon.gtIconVal = iadd * imul;
12371 op1->ChangeOper(GT_MUL);
12373 add->gtIntCon.gtIconVal = imul;
12374 #ifdef _TARGET_64BIT_
12375 if (add->gtType == TYP_INT)
12377 // we need to properly re-sign-extend or truncate after multiplying two int constants above
12378 add->AsIntCon()->TruncateOrSignExtend32();
12380 #endif //_TARGET_64BIT_
12388 /* For "val / 1", just return "val" */
12390 if ((op2->gtOper == GT_CNS_INT) && (op2->gtIntConCommon.IconValue() == 1))
12392 DEBUG_DESTROY_NODE(tree);
12395 // Do this for "long" constants as well as ints.
12396 else if ((op2->gtOper == GT_CNS_LNG) && (op2->gtIntConCommon.LngValue() == 1))
12398 DEBUG_DESTROY_NODE(tree);
12406 /* Check for the case "(val + icon) << icon" */
12408 if (op2->IsCnsIntOrI() &&
12409 op1->gtOper == GT_ADD && !op1->gtOverflow())
12411 GenTreePtr cns = op1->gtOp.gtOp2;
12413 if (cns->IsCnsIntOrI() && (op2->GetScaleIndexShf() != 0))
12415 ssize_t ishf = op2->gtIntConCommon.IconValue();
12416 ssize_t iadd = cns->gtIntConCommon.IconValue();
12418 // printf("Changing '(val+icon1)<<icon2' into '(val<<icon2+icon1<<icon2)'\n");
12420 /* Change "(val + iadd) << ishf" into "(val<<ishf + iadd<<ishf)" */
12422 tree->ChangeOper(GT_ADD);
12423 ssize_t result = iadd << ishf;
12424 op2->gtIntConCommon.SetIconValue(result);
12425 #ifdef _TARGET_64BIT_
12426 if (op1->gtType == TYP_INT)
12428 op2->AsIntCon()->TruncateOrSignExtend32();
12430 #endif // _TARGET_64BIT_
12432 // we are reusing the shift amount node here, but the type we want is that of the shift result
12433 op2->gtType = op1->gtType;
12435 if (cns->gtOper == GT_CNS_INT &&
12436 cns->gtIntCon.gtFieldSeq != nullptr &&
12437 cns->gtIntCon.gtFieldSeq->IsConstantIndexFieldSeq())
12439 assert(cns->gtIntCon.gtFieldSeq->m_next == nullptr);
12440 op2->gtIntCon.gtFieldSeq = cns->gtIntCon.gtFieldSeq;
12443 op1->ChangeOper(GT_LSH);
12445 cns->gtIntConCommon.SetIconValue(ishf);
12453 if (!optValnumCSE_phase)
12455 /* "x ^ -1" is "~x" */
12457 if ((op2->gtOper == GT_CNS_INT) && (op2->gtIntConCommon.IconValue() == -1))
12459 tree->ChangeOper(GT_NOT);
12460 tree->gtOp2 = NULL;
12461 DEBUG_DESTROY_NODE(op2);
12463 else if ((op2->gtOper == GT_CNS_LNG) && (op2->gtIntConCommon.LngValue() == -1))
12465 tree->ChangeOper(GT_NOT);
12466 tree->gtOp2 = NULL;
12467 DEBUG_DESTROY_NODE(op2);
12469 else if ((op2->gtOper == GT_CNS_INT) && (op2->gtIntConCommon.IconValue() == 1) &&
12470 op1->OperIsCompare())
12472 /* "binaryVal ^ 1" is "!binaryVal" */
12473 gtReverseCond(op1);
12474 DEBUG_DESTROY_NODE(op2);
12475 DEBUG_DESTROY_NODE(tree);
12483 return fgMorphInitBlock(tree);
12488 return fgMorphCopyBlock(tree);
12498 // code to generate a magic number and shift amount for the magic number division
12499 // optimization. This code is previously from UTC where it notes it was taken from
12500 // _The_PowerPC_Compiler_Writer's_Guide_, pages 57-58.
12501 // The paper it is based on is "Division by invariant integers using multiplication"
12502 // by Torbjorn Granlund and Peter L. Montgomery in PLDI 94
12504 template <typename T>
12505 T GetSignedMagicNumberForDivide(T denom, int *shift /*out*/)
12507 // static SMAG smag;
12508 const int bits = sizeof(T) * 8;
12509 const int bits_minus_1 = bits - 1;
12511 typedef typename jitstd::make_unsigned<T>::type UT;
12513 const UT two_nminus1 = UT(1) << bits_minus_1;
12528 absDenom = abs(denom);
12529 t = two_nminus1 + ((unsigned int)denom >> 31);
12530 absNc = t - 1 - (t % absDenom); // absolute value of nc
12531 p = bits_minus_1; // initialize p
12532 q1 = two_nminus1 / absNc; // initialize q1 = 2^p / abs(nc)
12533 r1 = two_nminus1 - (q1 * absNc); // initialize r1 = rem(2^p, abs(nc))
12534 q2 = two_nminus1 / absDenom; // initialize q1 = 2^p / abs(denom)
12535 r2 = two_nminus1 - (q2 * absDenom); // initialize r1 = rem(2^p, abs(denom))
12540 q1 *= 2; // update q1 = 2^p / abs(nc)
12541 r1 *= 2; // update r1 = rem(2^p / abs(nc))
12543 if (r1 >= absNc) { // must be unsigned comparison
12548 q2 *= 2; // update q2 = 2^p / abs(denom)
12549 r2 *= 2; // update r2 = rem(2^p / abs(denom))
12551 if (r2 >= absDenom) { // must be unsigned comparison
12556 delta = absDenom - r2;
12557 } while (q1 < delta || (q1 == delta && r1 == 0));
12559 result_magic = q2 + 1; // resulting magic number
12561 result_magic = -result_magic;
12563 *shift = p - bits; // resulting shift
12565 return result_magic;
12569 bool Compiler::fgShouldUseMagicNumberDivide(GenTreeOp* tree)
12571 #ifdef _TARGET_ARM64_
12572 // TODO-ARM64-NYI: We don't have a 'mulHi' implementation yet for ARM64
12576 // During the optOptimizeValnumCSEs phase we can call fgMorph and when we do,
12577 // if this method returns true we will introduce a new LclVar and
12578 // a couple of new GenTree nodes, including an assignment to the new LclVar.
12579 // None of these new GenTree nodes will have valid ValueNumbers.
12580 // That is an invalid state for a GenTree node during the optOptimizeValnumCSEs phase.
12582 // Also during optAssertionProp when extracting side effects we can assert
12583 // during gtBuildCommaList if we have one tree that has Value Numbers
12584 // and another one that does not.
12586 if (!fgGlobalMorph)
12588 // We only perform the Magic Number Divide optimization during
12589 // the initial global morph phase
12593 if (tree->gtFlags & GTF_OVERFLOW)
12596 if (tree->gtOp2->gtOper != GT_CNS_INT && tree->gtOp2->gtOper != GT_CNS_LNG)
12599 ssize_t cons = tree->gtOp2->gtIntConCommon.IconValue();
12601 if (cons == 0 || cons == -1 || cons == 1)
12604 // codegen will expand these
12608 // someone else will fold this away, so don't make it complicated for them
12609 if (tree->gtOp1->IsCnsIntOrI())
12612 // There is no technical barrier to handling unsigned, however it is quite rare
12613 // and more work to support and test
12614 if (tree->gtFlags & GTF_UNSIGNED)
12622 // transform x%c -> x-((x/c)*c)
12624 GenTree* Compiler::fgMorphModByConst(GenTreeOp* tree)
12626 assert(fgShouldUseMagicNumberDivide(tree));
12628 var_types type = tree->gtType;
12630 GenTree* cns = tree->gtOp2;
12632 GenTree* numerator = fgMakeMultiUse(&tree->gtOp1);
12634 tree->SetOper(GT_DIV);
12636 GenTree* mul = gtNewOperNode(GT_MUL, type, tree, gtCloneExpr(cns));
12638 GenTree* sub = gtNewOperNode(GT_SUB, type, numerator, mul);
12641 sub->gtFlags |= GTF_MORPHED;
12647 // For ARM64 we don't have a remainder instruction,
12648 // The architecture manual suggests the following transformation to
12649 // generate code for such operator:
12651 // a % b = a - (a / b) * b;
12653 // This method will produce the above expression in 'a' and 'b' are
12654 // leaf nodes, otherwise, if any of them is not a leaf it will spill
12655 // its value into a temporary variable, an example:
12656 // (x * 2 - 1) % (y + 1) -> t1 - (t2 * ( comma(t1 = x * 2 - 1, t1) / comma(t2 = y + 1, t2) ) )
12658 GenTree* Compiler::fgMorphModToSubMulDiv(GenTreeOp* tree)
12660 #ifndef _TARGET_ARM64_
12661 assert(!"This should only be called for ARM64");
12664 if (tree->OperGet() == GT_MOD)
12666 tree->SetOper(GT_DIV);
12668 else if (tree->OperGet() == GT_UMOD)
12670 tree->SetOper(GT_UDIV);
12674 noway_assert(!"Illegal gtOper in fgMorphModToSubMulDiv");
12677 var_types type = tree->gtType;
12678 GenTree* denominator = tree->gtOp2;
12679 GenTree* numerator = tree->gtOp1;
12681 if (!numerator->OperIsLeaf())
12683 numerator = fgMakeMultiUse(&tree->gtOp1);
12686 if (!denominator->OperIsLeaf())
12688 denominator = fgMakeMultiUse(&tree->gtOp2);
12691 GenTree* mul = gtNewOperNode(GT_MUL, type, tree, gtCloneExpr(denominator));
12692 GenTree* sub = gtNewOperNode(GT_SUB, type, gtCloneExpr(numerator), mul);
12695 sub->gtFlags |= GTF_MORPHED;
12701 // Turn a division by a constant into a multiplication by constant + some adjustments
12702 // see comments on GetSignedMagicNumberForDivide for source of this algorithm.
12703 // returns: the transformed tree
12705 GenTree* Compiler::fgMorphDivByConst(GenTreeOp* tree)
12707 assert(fgShouldUseMagicNumberDivide(tree));
12709 JITDUMP("doing magic number divide optimization\n");
12711 int64_t denominator = tree->gtOp2->gtIntConCommon.IconValue();
12714 var_types type = tree->gtType;
12716 if (tree->gtType == TYP_INT)
12718 magic = GetSignedMagicNumberForDivide<int32_t>((int32_t) denominator, &shift);
12722 magic = GetSignedMagicNumberForDivide<int64_t>((int64_t) denominator, &shift);
12725 GenTree* numerator = nullptr;
12727 // If signs of the denominator and magic number don't match,
12728 // we will need to use the numerator again.
12729 if (signum(denominator) != signum(magic))
12731 numerator = fgMakeMultiUse(&tree->gtOp1);
12732 tree->gtFlags |= GTF_ASG;
12735 if (type == TYP_LONG)
12736 tree->gtOp2->gtIntConCommon.SetLngValue(magic);
12738 tree->gtOp2->gtIntConCommon.SetIconValue((ssize_t)magic);
12740 tree->SetOper(GT_MULHI);
12743 GenTree* mulresult = tree;
12745 JITDUMP("Multiply Result:\n");
12746 DISPTREE(mulresult);
12748 GenTree *adjusted = mulresult;
12750 if (denominator > 0 && magic < 0)
12752 // add the numerator back in
12753 adjusted = gtNewOperNode(GT_ADD, type, mulresult, numerator);
12755 else if (denominator < 0 && magic > 0)
12757 // subtract the numerator off
12758 adjusted = gtNewOperNode(GT_SUB, type, mulresult, numerator);
12762 adjusted = mulresult;
12765 GenTree* result1 = adjusted;
12768 result1 = gtNewOperNode(GT_RSH, type, adjusted, gtNewIconNode(shift, TYP_INT));
12771 GenTree* secondClone = fgMakeMultiUse(&result1);
12773 GenTree* result2 = gtNewOperNode(GT_RSZ, type, secondClone, gtNewIconNode(genTypeSize(type) * 8 - 1, type));
12776 GenTree* result = gtNewOperNode(GT_ADD, type, result1, result2);
12777 JITDUMP("Final Magic Number divide:\n");
12781 result->gtFlags |= GTF_MORPHED;
12787 //------------------------------------------------------------------------------
12788 // fgOperIsBitwiseRotationRoot : Check if the operation can be a root of a bitwise rotation tree.
12792 // oper - Operation to check
12795 // True if the operation can be a root of a bitwise rotation tree; false otherwise.
12797 bool Compiler::fgOperIsBitwiseRotationRoot(genTreeOps oper)
12799 return (oper == GT_OR) || (oper == GT_XOR);
12802 //------------------------------------------------------------------------------
12803 // fgRecognizeAndMorphBitwiseRotation : Check if the tree represents a left or right rotation. If so, return
12804 // an equivalent GT_ROL or GT_ROR tree; otherwise, return the original tree.
12807 // tree - tree to check for a rotation pattern
12810 // An equivalent GT_ROL or GT_ROR tree if a pattern is found; original tree otherwise.
12813 // The input is a GT_OR or a GT_XOR tree.
12815 GenTreePtr Compiler::fgRecognizeAndMorphBitwiseRotation(GenTreePtr tree)
12817 #ifndef LEGACY_BACKEND
12819 // Check for a rotation pattern, e.g.,
12832 // The patterns recognized:
12833 // (x << (y & M)) op (x >>> ((-y + N) & M))
12834 // (x >>> ((-y + N) & M)) op (x << (y & M))
12836 // (x << y) op (x >>> (-y + N))
12837 // (x >> > (-y + N)) op (x << y)
12839 // (x >>> (y & M)) op (x << ((-y + N) & M))
12840 // (x << ((-y + N) & M)) op (x >>> (y & M))
12842 // (x >>> y) op (x << (-y + N))
12843 // (x << (-y + N)) op (x >>> y)
12845 // (x << c1) op (x >>> c2)
12846 // (x >>> c1) op (x << c2)
12849 // c1 and c2 are const
12850 // c1 + c2 == bitsize(x)
12853 // M & (N - 1) == N - 1
12854 // op is either | or ^
12856 if (((tree->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) != 0) ||
12857 ((tree->gtFlags & GTF_ORDER_SIDEEFF) != 0))
12859 // We can't do anything if the tree has assignments, calls, or volatile
12860 // reads. Note that we allow GTF_EXCEPT side effect since any exceptions
12861 // thrown by the original tree will be thrown by the transformed tree as well.
12865 genTreeOps oper = tree->OperGet();
12866 assert(fgOperIsBitwiseRotationRoot(oper));
12868 // Check if we have an LSH on one side of the OR and an RSZ on the other side.
12869 GenTreePtr op1 = tree->gtGetOp1();
12870 GenTreePtr op2 = tree->gtGetOp2();
12871 GenTreePtr leftShiftTree = nullptr;
12872 GenTreePtr rightShiftTree = nullptr;
12873 if ((op1->OperGet() == GT_LSH) && (op2->OperGet() == GT_RSZ))
12875 leftShiftTree = op1;
12876 rightShiftTree = op2;
12878 else if ((op1->OperGet() == GT_RSZ) && (op2->OperGet() == GT_LSH))
12880 leftShiftTree = op2;
12881 rightShiftTree = op1;
12888 // Check if the trees representing the value to shift are identical.
12889 // We already checked that there are no side effects above.
12890 if (GenTree::Compare(leftShiftTree->gtGetOp1(), rightShiftTree->gtGetOp1()))
12892 GenTreePtr rotatedValue = leftShiftTree->gtGetOp1();
12893 var_types rotatedValueActualType = genActualType(rotatedValue->gtType);
12894 ssize_t rotatedValueBitSize = genTypeSize(rotatedValueActualType) * 8;
12895 noway_assert((rotatedValueBitSize == 32) || (rotatedValueBitSize == 64));
12896 GenTreePtr leftShiftIndex = leftShiftTree->gtGetOp2();
12897 GenTreePtr rightShiftIndex = rightShiftTree->gtGetOp2();
12899 // The shift index may be masked. At least (rotatedValueBitSize - 1) lower bits
12900 // shouldn't be masked for the transformation to be valid. If additional
12901 // higher bits are not masked, the transformation is still valid since the result
12902 // of MSIL shift instructions is unspecified if the shift amount is greater or equal
12903 // than the width of the value being shifted.
12904 ssize_t minimalMask = rotatedValueBitSize - 1;
12905 ssize_t leftShiftMask = -1;
12906 ssize_t rightShiftMask = -1;
12908 if ((leftShiftIndex->OperGet() == GT_AND))
12910 if (leftShiftIndex->gtGetOp2()->IsCnsIntOrI())
12912 leftShiftMask = leftShiftIndex->gtGetOp2()->gtIntCon.gtIconVal;
12913 leftShiftIndex = leftShiftIndex->gtGetOp1();
12921 if ((rightShiftIndex->OperGet() == GT_AND))
12923 if (rightShiftIndex->gtGetOp2()->IsCnsIntOrI())
12925 rightShiftMask = rightShiftIndex->gtGetOp2()->gtIntCon.gtIconVal;
12926 rightShiftIndex = rightShiftIndex->gtGetOp1();
12934 if (((minimalMask & leftShiftMask) != minimalMask) ||
12935 ((minimalMask & rightShiftMask) != minimalMask))
12937 // The shift index is overmasked, e.g., we have
12938 // something like (x << y & 15) or
12939 // (x >> (32 - y) & 15 with 32 bit x.
12940 // The transformation is not valid.
12944 GenTreePtr shiftIndexWithAdd = nullptr;
12945 GenTreePtr shiftIndexWithoutAdd = nullptr;
12946 genTreeOps rotateOp = GT_NONE;
12947 GenTreePtr rotateIndex = nullptr;
12949 if (leftShiftIndex->OperGet() == GT_ADD)
12951 shiftIndexWithAdd = leftShiftIndex;
12952 shiftIndexWithoutAdd = rightShiftIndex;
12955 else if (rightShiftIndex->OperGet() == GT_ADD)
12957 shiftIndexWithAdd = rightShiftIndex;
12958 shiftIndexWithoutAdd = leftShiftIndex;
12962 if (shiftIndexWithAdd != nullptr)
12964 if (shiftIndexWithAdd->gtGetOp2()->IsCnsIntOrI())
12966 if (shiftIndexWithAdd->gtGetOp2()->gtIntCon.gtIconVal == rotatedValueBitSize)
12968 if (shiftIndexWithAdd->gtGetOp1()->OperGet() == GT_NEG)
12970 if (GenTree::Compare(shiftIndexWithAdd->gtGetOp1()->gtGetOp1(), shiftIndexWithoutAdd))
12972 // We found one of these patterns:
12973 // (x << (y & M)) | (x >>> ((-y + N) & M))
12974 // (x << y) | (x >>> (-y + N))
12975 // (x >>> (y & M)) | (x << ((-y + N) & M))
12976 // (x >>> y) | (x << (-y + N))
12977 // where N == bitsize(x), M is const, and
12978 // M & (N - 1) == N - 1
12980 #ifndef _TARGET_64BIT_
12981 if (!shiftIndexWithoutAdd->IsCnsIntOrI() && (rotatedValueBitSize == 64))
12983 // TODO: we need to handle variable-sized long shifts specially on x86.
12984 // GT_LSH, GT_RSH, and GT_RSZ have helpers for this case. We may need
12985 // to add helpers for GT_ROL and GT_ROR.
12986 NYI("Rotation of a long value by variable amount");
12990 rotateIndex = shiftIndexWithoutAdd;
12996 else if ((leftShiftIndex->IsCnsIntOrI() &&
12997 rightShiftIndex->IsCnsIntOrI()))
12999 if (leftShiftIndex->gtIntCon.gtIconVal +
13000 rightShiftIndex->gtIntCon.gtIconVal == rotatedValueBitSize)
13002 // We found this pattern:
13003 // (x << c1) | (x >>> c2)
13004 // where c1 and c2 are const and c1 + c2 == bitsize(x)
13006 rotateIndex = leftShiftIndex;
13010 if (rotateIndex != nullptr)
13012 noway_assert(GenTree::OperIsRotate(rotateOp));
13014 unsigned inputTreeEffects = tree->gtFlags & GTF_ALL_EFFECT;
13016 // We can use the same tree only during global morph; reusing the tree in a later morph
13017 // may invalidate value numbers.
13020 tree->gtOp.gtOp1 = rotatedValue;
13021 tree->gtOp.gtOp2 = rotateIndex;
13022 tree->ChangeOper(rotateOp);
13023 noway_assert(inputTreeEffects == ((rotatedValue->gtFlags | rotateIndex->gtFlags) & GTF_ALL_EFFECT));
13027 tree = gtNewOperNode(rotateOp, rotatedValueActualType, rotatedValue, rotateIndex);
13028 noway_assert(inputTreeEffects == (tree->gtFlags & GTF_ALL_EFFECT));
13034 #endif //LEGACY_BACKEND
13038 #if !CPU_HAS_FP_SUPPORT
13039 GenTreePtr Compiler::fgMorphToEmulatedFP(GenTreePtr tree)
13042 genTreeOps oper = tree->OperGet();
13043 var_types typ = tree->TypeGet();
13044 GenTreePtr op1 = tree->gtOp.gtOp1;
13045 GenTreePtr op2 = tree->gtGetOp2();
13048 We have to use helper calls for all FP operations:
13050 FP operators that operate on FP values
13051 casts to and from FP
13052 comparisons of FP values
13055 if (varTypeIsFloating(typ) || (op1 && varTypeIsFloating(op1->TypeGet())))
13059 size_t argc = genTypeStSz(typ);
13061 /* Not all FP operations need helper calls */
13075 /* If the result isn't FP, it better be a compare or cast */
13077 if (!(varTypeIsFloating(typ) ||
13078 tree->OperIsCompare() || oper == GT_CAST))
13081 noway_assert(varTypeIsFloating(typ) ||
13082 tree->OperIsCompare() || oper == GT_CAST);
13085 /* Keep track of how many arguments we're passing */
13087 fgPtrArgCntCur += argc;
13089 /* Is this a binary operator? */
13093 /* Add the second operand to the argument count */
13095 fgPtrArgCntCur += argc; argc *= 2;
13097 /* What kind of an operator do we have? */
13101 case GT_ADD: helper = CPX_R4_ADD; break;
13102 case GT_SUB: helper = CPX_R4_SUB; break;
13103 case GT_MUL: helper = CPX_R4_MUL; break;
13104 case GT_DIV: helper = CPX_R4_DIV; break;
13105 // case GT_MOD: helper = CPX_R4_REM; break;
13107 case GT_EQ : helper = CPX_R4_EQ ; break;
13108 case GT_NE : helper = CPX_R4_NE ; break;
13109 case GT_LT : helper = CPX_R4_LT ; break;
13110 case GT_LE : helper = CPX_R4_LE ; break;
13111 case GT_GE : helper = CPX_R4_GE ; break;
13112 case GT_GT : helper = CPX_R4_GT ; break;
13118 noway_assert(!"unexpected FP binary op");
13122 args = gtNewArgList(tree->gtOp.gtOp2, tree->gtOp.gtOp1);
13132 noway_assert(!"FP cast");
13134 case GT_NEG: helper = CPX_R4_NEG; break;
13140 noway_assert(!"unexpected FP unary op");
13144 args = gtNewArgList(tree->gtOp.gtOp1);
13147 /* If we have double result/operands, modify the helper */
13149 if (typ == TYP_DOUBLE)
13151 noway_assert(CPX_R4_NEG+1 == CPX_R8_NEG);
13152 noway_assert(CPX_R4_ADD+1 == CPX_R8_ADD);
13153 noway_assert(CPX_R4_SUB+1 == CPX_R8_SUB);
13154 noway_assert(CPX_R4_MUL+1 == CPX_R8_MUL);
13155 noway_assert(CPX_R4_DIV+1 == CPX_R8_DIV);
13161 noway_assert(tree->OperIsCompare());
13163 noway_assert(CPX_R4_EQ+1 == CPX_R8_EQ);
13164 noway_assert(CPX_R4_NE+1 == CPX_R8_NE);
13165 noway_assert(CPX_R4_LT+1 == CPX_R8_LT);
13166 noway_assert(CPX_R4_LE+1 == CPX_R8_LE);
13167 noway_assert(CPX_R4_GE+1 == CPX_R8_GE);
13168 noway_assert(CPX_R4_GT+1 == CPX_R8_GT);
13171 tree = fgMorphIntoHelperCall(tree, helper, args);
13173 if (fgPtrArgCntMax < fgPtrArgCntCur)
13174 fgPtrArgCntMax = fgPtrArgCntCur;
13176 fgPtrArgCntCur -= argc;
13184 if (compCurBB == genReturnBB)
13186 /* This is the 'exitCrit' call at the exit label */
13188 noway_assert(op1->gtType == TYP_VOID);
13189 noway_assert(op2 == 0);
13191 tree->gtOp.gtOp1 = op1 = fgMorphTree(op1);
13196 /* This is a (real) return value -- check its type */
13199 if (genActualType(op1->TypeGet()) != genActualType(info.compRetType))
13201 bool allowMismatch = false;
13203 // Allow TYP_BYREF to be returned as TYP_I_IMPL and vice versa
13204 if ((info.compRetType == TYP_BYREF &&
13205 genActualType(op1->TypeGet()) == TYP_I_IMPL) ||
13206 (op1->TypeGet() == TYP_BYREF &&
13207 genActualType(info.compRetType) == TYP_I_IMPL))
13208 allowMismatch = true;
13210 if (varTypeIsFloating(info.compRetType) && varTypeIsFloating(op1->TypeGet()))
13211 allowMismatch = true;
13213 if (!allowMismatch)
13214 NO_WAY("Return type mismatch");
13225 /*****************************************************************************
13227 * Transform the given tree for code generation and return an equivalent tree.
13231 GenTreePtr Compiler::fgMorphTree(GenTreePtr tree, MorphAddrContext* mac)
13233 noway_assert(tree);
13234 noway_assert(tree->gtOper != GT_STMT);
13239 if ((unsigned)JitConfig.JitBreakMorphTree() == tree->gtTreeID)
13241 noway_assert(!"JitBreakMorphTree hit");
13247 int thisMorphNum = 0;
13248 if (verbose && treesBeforeAfterMorph)
13250 thisMorphNum = morphNum++;
13251 printf("\nfgMorphTree (before %d):\n", thisMorphNum);
13256 /*-------------------------------------------------------------------------
13257 * fgMorphTree() can potentially replace a tree with another, and the
13258 * caller has to store the return value correctly.
13259 * Turn this on to always make copy of "tree" here to shake out
13260 * hidden/unupdated references.
13265 if (compStressCompile(STRESS_GENERIC_CHECK, 0))
13269 #ifdef SMALL_TREE_NODES
13270 if (GenTree::s_gtNodeSizes[tree->gtOper] == TREE_NODE_SZ_SMALL)
13272 copy = gtNewLargeOperNode(GT_ADD, TYP_INT);
13277 copy = new (this, GT_CALL) GenTreeCall(TYP_INT);
13280 copy->CopyFrom(tree, this);
13282 #if defined (LATE_DISASM)
13283 // GT_CNS_INT is considered small, so CopyFrom() won't copy all fields
13284 if ((tree->gtOper == GT_CNS_INT) && tree->IsIconHandle())
13286 copy->gtIntCon.gtIconHdl.gtIconHdl1 = tree->gtIntCon.gtIconHdl.gtIconHdl1;
13287 copy->gtIntCon.gtIconHdl.gtIconHdl2 = tree->gtIntCon.gtIconHdl.gtIconHdl2;
13291 DEBUG_DESTROY_NODE(tree);
13298 /* Ensure that we haven't morphed this node already */
13299 assert(((tree->gtFlags & GTF_MORPHED) == 0) && "ERROR: Already morphed this node!");
13301 #if LOCAL_ASSERTION_PROP
13302 /* Before morphing the tree, we try to propagate any active assertions */
13303 if (optLocalAssertionProp)
13305 /* Do we have any active assertions? */
13307 if (optAssertionCount > 0)
13309 GenTreePtr newTree = tree;
13310 while (newTree != NULL)
13313 /* newTree is non-Null if we propagated an assertion */
13314 newTree = optAssertionProp(apFull, tree, NULL);
13316 noway_assert(tree != NULL);
13319 PREFAST_ASSUME(tree != NULL);
13323 /* Save the original un-morphed tree for fgMorphTreeDone */
13325 GenTreePtr oldTree = tree;
13327 /* Figure out what kind of a node we have */
13329 unsigned kind = tree->OperKind();
13331 /* Is this a constant node? */
13333 if (kind & GTK_CONST)
13335 tree = fgMorphConst(tree);
13339 /* Is this a leaf node? */
13341 if (kind & GTK_LEAF)
13343 tree = fgMorphLeaf(tree);
13347 /* Is it a 'simple' unary/binary operator? */
13349 if (kind & GTK_SMPOP)
13351 tree = fgMorphSmpOp(tree, mac);
13355 /* See what kind of a special operator we have here */
13357 switch (tree->OperGet())
13360 tree = fgMorphField(tree, mac);
13364 tree = fgMorphCall(tree->AsCall());
13367 case GT_ARR_BOUNDS_CHECK:
13368 #ifdef FEATURE_SIMD
13370 #endif // FEATURE_SIMD
13372 fgSetRngChkTarget(tree);
13374 GenTreeBoundsChk* bndsChk = tree->AsBoundsChk();
13375 bndsChk->gtArrLen = fgMorphTree(bndsChk->gtArrLen);
13376 bndsChk->gtIndex = fgMorphTree(bndsChk->gtIndex);
13377 // If the index is a comma(throw, x), just return that.
13378 if (!optValnumCSE_phase && fgIsCommaThrow(bndsChk->gtIndex))
13380 tree = bndsChk->gtIndex;
13383 // Propagate effects flags upwards
13384 bndsChk->gtFlags |= (bndsChk->gtArrLen->gtFlags & GTF_ALL_EFFECT);
13385 bndsChk->gtFlags |= (bndsChk->gtIndex->gtFlags & GTF_ALL_EFFECT);
13387 // Otherwise, we don't change the tree.
13392 tree->gtArrElem.gtArrObj = fgMorphTree(tree->gtArrElem.gtArrObj);
13393 tree->gtFlags |= tree->gtArrElem.gtArrObj->gtFlags & GTF_ALL_EFFECT;
13396 for (dim = 0; dim < tree->gtArrElem.gtArrRank; dim++)
13398 tree->gtArrElem.gtArrInds[dim] = fgMorphTree(tree->gtArrElem.gtArrInds[dim]);
13399 tree->gtFlags |= tree->gtArrElem.gtArrInds[dim]->gtFlags & GTF_ALL_EFFECT;
13402 fgSetRngChkTarget(tree, false);
13405 case GT_ARR_OFFSET:
13406 tree->gtArrOffs.gtOffset = fgMorphTree(tree->gtArrOffs.gtOffset);
13407 tree->gtFlags |= tree->gtArrOffs.gtOffset->gtFlags & GTF_ALL_EFFECT;
13408 tree->gtArrOffs.gtIndex = fgMorphTree(tree->gtArrOffs.gtIndex);
13409 tree->gtFlags |= tree->gtArrOffs.gtIndex->gtFlags & GTF_ALL_EFFECT;
13410 tree->gtArrOffs.gtArrObj = fgMorphTree(tree->gtArrOffs.gtArrObj);
13411 tree->gtFlags |= tree->gtArrOffs.gtArrObj->gtFlags & GTF_ALL_EFFECT;
13413 fgSetRngChkTarget(tree, false);
13417 tree->gtCmpXchg.gtOpLocation = fgMorphTree(tree->gtCmpXchg.gtOpLocation);
13418 tree->gtCmpXchg.gtOpValue = fgMorphTree(tree->gtCmpXchg.gtOpValue);
13419 tree->gtCmpXchg.gtOpComparand = fgMorphTree(tree->gtCmpXchg.gtOpComparand);
13426 noway_assert(!"unexpected operator");
13430 fgMorphTreeDone(tree, oldTree DEBUGARG(thisMorphNum));
13436 #if LOCAL_ASSERTION_PROP
13437 /*****************************************************************************
13439 * Kill all dependent assertions with regard to lclNum.
13443 void Compiler::fgKillDependentAssertions(unsigned lclNum
13444 DEBUGARG(GenTreePtr tree))
13446 LclVarDsc * varDsc = &lvaTable[lclNum];
13448 if (varDsc->lvPromoted)
13450 noway_assert(varTypeIsStruct(varDsc));
13452 // Kill the field locals.
13453 for (unsigned i = varDsc->lvFieldLclStart; i < varDsc->lvFieldLclStart + varDsc->lvFieldCnt; ++i)
13455 fgKillDependentAssertions(i DEBUGARG(tree));
13458 // Fall through to kill the struct local itself.
13461 /* All dependent assertions are killed here */
13463 ASSERT_TP killed = BitVecOps::MakeCopy(apTraits, GetAssertionDep(lclNum));
13467 AssertionIndex index = optAssertionCount;
13468 while (killed && (index > 0))
13470 if (BitVecOps::IsMember(apTraits, killed, index - 1))
13473 AssertionDsc* curAssertion = optGetAssertion(index);
13474 noway_assert((curAssertion->op1.lcl.lclNum == lclNum) ||
13475 ((curAssertion->op2.kind == O2K_LCLVAR_COPY) &&
13476 (curAssertion->op2.lcl.lclNum == lclNum)));
13479 printf("\nThe assignment ");
13481 printf(" using V%02u removes: ", curAssertion->op1.lcl.lclNum);
13482 optPrintAssertion(curAssertion);
13485 // Remove this bit from the killed mask
13486 BitVecOps::RemoveElemD(apTraits, killed, index - 1);
13488 optAssertionRemove(index);
13494 // killed mask should now be zero
13495 noway_assert(BitVecOps::IsEmpty(apTraits, killed));
13498 #endif // LOCAL_ASSERTION_PROP
13501 /*****************************************************************************
13503 * This function is called to complete the morphing of a tree node
13504 * It should only be called once for each node.
13505 * If DEBUG is defined the flag GTF_MORPHED is checked and updated,
13506 * to enforce the invariant that each node is only morphed once.
13507 * If LOCAL_ASSERTION_PROP is enabled the result tree may be replaced
13508 * by an equivalent tree.
13512 void Compiler::fgMorphTreeDone(GenTreePtr tree,
13513 GenTreePtr oldTree /* == NULL */
13514 DEBUGARG(int morphNum))
13517 if (verbose && treesBeforeAfterMorph)
13519 printf("\nfgMorphTree (after %d):\n", morphNum);
13521 printf(""); // in our logic this causes a flush
13525 if (!fgGlobalMorph)
13528 if ((oldTree != NULL) && (oldTree != tree))
13530 /* Ensure that we have morphed this node */
13531 assert((tree->gtFlags & GTF_MORPHED) && "ERROR: Did not morph this node!");
13534 TransferTestDataToNode(oldTree, tree);
13539 // Ensure that we haven't morphed this node already
13540 assert(((tree->gtFlags & GTF_MORPHED) == 0) && "ERROR: Already morphed this node!");
13543 if (tree->OperKind() & GTK_CONST)
13546 #if LOCAL_ASSERTION_PROP
13548 if (!optLocalAssertionProp)
13551 /* Do we have any active assertions? */
13553 if (optAssertionCount > 0)
13555 /* Is this an assignment to a local variable */
13557 if ((tree->OperKind() & GTK_ASGOP) &&
13558 (tree->gtOp.gtOp1->gtOper == GT_LCL_VAR || tree->gtOp.gtOp1->gtOper == GT_LCL_FLD))
13560 unsigned op1LclNum = tree->gtOp.gtOp1->gtLclVarCommon.gtLclNum; noway_assert(op1LclNum < lvaCount);
13561 fgKillDependentAssertions(op1LclNum DEBUGARG(tree));
13565 /* If this tree makes a new assertion - make it available */
13566 optAssertionGen(tree);
13568 #endif // LOCAL_ASSERTION_PROP
13573 /* Mark this node as being morphed */
13574 tree->gtFlags |= GTF_MORPHED;
13579 /*****************************************************************************
13581 * Check and fold blocks of type BBJ_COND and BBJ_SWITCH on constants
13582 * Returns true if we modified the flow graph
13585 bool Compiler::fgFoldConditional(BasicBlock * block)
13587 bool result = false;
13589 // We don't want to make any code unreachable
13590 if (opts.compDbgCode || opts.MinOpts())
13593 if (block->bbJumpKind == BBJ_COND)
13595 noway_assert(block->bbTreeList && block->bbTreeList->gtPrev);
13597 GenTreePtr stmt = block->bbTreeList->gtPrev;
13599 noway_assert(stmt->gtNext == NULL);
13601 if (stmt->gtStmt.gtStmtExpr->gtOper == GT_CALL)
13603 noway_assert(fgRemoveRestOfBlock);
13605 /* Unconditional throw - transform the basic block into a BBJ_THROW */
13606 fgConvertBBToThrowBB(block);
13608 /* Remove 'block' from the predecessor list of 'block->bbNext' */
13609 fgRemoveRefPred(block->bbNext, block);
13611 /* Remove 'block' from the predecessor list of 'block->bbJumpDest' */
13612 fgRemoveRefPred(block->bbJumpDest, block);
13617 printf("\nConditional folded at BB%02u\n", block->bbNum);
13618 printf("BB%02u becomes a BBJ_THROW\n", block->bbNum);
13624 noway_assert(stmt->gtStmt.gtStmtExpr->gtOper == GT_JTRUE);
13626 /* Did we fold the conditional */
13628 noway_assert(stmt->gtStmt.gtStmtExpr->gtOp.gtOp1);
13629 GenTreePtr cond; cond = stmt->gtStmt.gtStmtExpr->gtOp.gtOp1;
13631 if (cond->OperKind() & GTK_CONST)
13633 /* Yupee - we folded the conditional!
13634 * Remove the conditional statement */
13636 noway_assert(cond->gtOper == GT_CNS_INT);
13637 noway_assert((block->bbNext->countOfInEdges() > 0) &&
13638 (block->bbJumpDest->countOfInEdges() > 0));
13640 /* remove the statement from bbTreelist - No need to update
13641 * the reference counts since there are no lcl vars */
13642 fgRemoveStmt(block, stmt);
13644 // block is a BBJ_COND that we are folding the conditional for
13645 // bTaken is the path that will always be taken from block
13646 // bNotTaken is the path that will never be taken from block
13648 BasicBlock * bTaken;
13649 BasicBlock * bNotTaken;
13651 if (cond->gtIntCon.gtIconVal != 0)
13653 /* JTRUE 1 - transform the basic block into a BBJ_ALWAYS */
13654 block->bbJumpKind = BBJ_ALWAYS;
13655 bTaken = block->bbJumpDest;
13656 bNotTaken = block->bbNext;
13660 /* Unmark the loop if we are removing a backwards branch */
13661 /* dest block must also be marked as a loop head and */
13662 /* We must be able to reach the backedge block */
13663 if ((block->bbJumpDest->isLoopHead()) &&
13664 (block->bbJumpDest->bbNum <= block->bbNum) &&
13665 fgReachable(block->bbJumpDest, block))
13667 optUnmarkLoopBlocks(block->bbJumpDest, block);
13670 /* JTRUE 0 - transform the basic block into a BBJ_NONE */
13671 block->bbJumpKind = BBJ_NONE;
13672 noway_assert(!(block->bbFlags & BBF_NEEDS_GCPOLL));
13673 bTaken = block->bbNext;
13674 bNotTaken = block->bbJumpDest;
13677 if (fgHaveValidEdgeWeights)
13679 // We are removing an edge from block to bNotTaken
13680 // and we have already computed the edge weights, so
13681 // we will try to adjust some of the weights
13683 flowList * edgeTaken = fgGetPredForBlock(bTaken, block);
13684 BasicBlock * bUpdated = NULL; // non-NULL if we updated the weight of an internal block
13686 // We examine the taken edge (block -> bTaken)
13687 // if block has valid profile weight and bTaken does not we try to adjust bTaken's weight
13688 // else if bTaken has valid profile weight and block does not we try to adjust block's weight
13689 // We can only adjust the block weights when (the edge block -> bTaken) is the only edge into bTaken
13691 if (block->bbFlags & BBF_PROF_WEIGHT)
13693 // The edge weights for (block -> bTaken) are 100% of block's weight
13694 edgeTaken->flEdgeWeightMin = block->bbWeight;
13695 edgeTaken->flEdgeWeightMax = block->bbWeight;
13697 if ((bTaken->bbFlags & BBF_PROF_WEIGHT) == 0)
13699 if ((bTaken->countOfInEdges() == 1) || (bTaken->bbWeight < block->bbWeight))
13701 // Update the weight of bTaken
13702 bTaken->inheritWeight(block);
13707 else if (bTaken->bbFlags & BBF_PROF_WEIGHT)
13709 if (bTaken->countOfInEdges() == 1)
13711 // There is only one in edge to bTaken
13712 edgeTaken->flEdgeWeightMin = bTaken->bbWeight;
13713 edgeTaken->flEdgeWeightMax = bTaken->bbWeight;
13715 // Update the weight of block
13716 block->inheritWeight(bTaken);
13721 if (bUpdated != NULL)
13724 // Now fix the weights of the edges out of 'bUpdated'
13725 switch (bUpdated->bbJumpKind) {
13727 edge = fgGetPredForBlock(bUpdated->bbNext, bUpdated);
13728 edge->flEdgeWeightMax = bUpdated->bbWeight;
13731 edge = fgGetPredForBlock(bUpdated->bbNext, bUpdated);
13732 edge->flEdgeWeightMax = bUpdated->bbWeight;
13735 edge = fgGetPredForBlock(bUpdated->bbJumpDest, bUpdated);
13736 edge->flEdgeWeightMax = bUpdated->bbWeight;
13739 // We don't handle BBJ_SWITCH
13746 /* modify the flow graph */
13748 /* Remove 'block' from the predecessor list of 'bNotTaken' */
13749 fgRemoveRefPred(bNotTaken, block);
13754 printf("\nConditional folded at BB%02u\n", block->bbNum);
13755 printf("BB%02u becomes a %s", block->bbNum,
13756 block->bbJumpKind == BBJ_ALWAYS ? "BBJ_ALWAYS" : "BBJ_NONE");
13757 if (block->bbJumpKind == BBJ_ALWAYS)
13758 printf(" to BB%02u", block->bbJumpDest->bbNum);
13763 /* if the block was a loop condition we may have to modify
13764 * the loop table */
13766 for (unsigned loopNum = 0; loopNum < optLoopCount; loopNum++)
13768 /* Some loops may have been already removed by
13769 * loop unrolling or conditional folding */
13771 if (optLoopTable[loopNum].lpFlags & LPFLG_REMOVED)
13774 /* We are only interested in the loop bottom */
13776 if (optLoopTable[loopNum].lpBottom == block)
13778 if (cond->gtIntCon.gtIconVal == 0)
13780 /* This was a bogus loop (condition always false)
13781 * Remove the loop from the table */
13783 optLoopTable[loopNum].lpFlags |= LPFLG_REMOVED;
13787 printf("Removing loop L%02u (from BB%02u to BB%02u)\n\n",
13789 optLoopTable[loopNum].lpFirst ->bbNum,
13790 optLoopTable[loopNum].lpBottom->bbNum);
13800 else if (block->bbJumpKind == BBJ_SWITCH)
13802 noway_assert(block->bbTreeList && block->bbTreeList->gtPrev);
13804 GenTreePtr stmt = block->bbTreeList->gtPrev;
13806 noway_assert(stmt->gtNext == NULL);
13808 if (stmt->gtStmt.gtStmtExpr->gtOper == GT_CALL)
13810 noway_assert(fgRemoveRestOfBlock);
13812 /* Unconditional throw - transform the basic block into a BBJ_THROW */
13813 fgConvertBBToThrowBB(block);
13815 /* update the flow graph */
13817 unsigned jumpCnt = block->bbJumpSwt->bbsCount;
13818 BasicBlock * * jumpTab = block->bbJumpSwt->bbsDstTab;
13820 for (unsigned val = 0; val < jumpCnt; val++, jumpTab++)
13822 BasicBlock * curJump = *jumpTab;
13824 /* Remove 'block' from the predecessor list of 'curJump' */
13825 fgRemoveRefPred(curJump, block);
13831 printf("\nConditional folded at BB%02u\n", block->bbNum);
13832 printf("BB%02u becomes a BBJ_THROW\n", block->bbNum);
13839 noway_assert(stmt->gtStmt.gtStmtExpr->gtOper == GT_SWITCH);
13841 /* Did we fold the conditional */
13843 noway_assert(stmt->gtStmt.gtStmtExpr->gtOp.gtOp1);
13844 GenTreePtr cond; cond = stmt->gtStmt.gtStmtExpr->gtOp.gtOp1;
13846 if (cond->OperKind() & GTK_CONST)
13848 /* Yupee - we folded the conditional!
13849 * Remove the conditional statement */
13851 noway_assert(cond->gtOper == GT_CNS_INT);
13853 /* remove the statement from bbTreelist - No need to update
13854 * the reference counts since there are no lcl vars */
13855 fgRemoveStmt(block, stmt);
13857 /* modify the flow graph */
13859 /* Find the actual jump target */
13860 unsigned switchVal; switchVal = (unsigned)cond->gtIntCon.gtIconVal;
13861 unsigned jumpCnt; jumpCnt = block->bbJumpSwt->bbsCount;
13862 BasicBlock * * jumpTab; jumpTab = block->bbJumpSwt->bbsDstTab;
13863 bool foundVal; foundVal = false;
13865 for (unsigned val = 0; val < jumpCnt; val++, jumpTab++)
13867 BasicBlock * curJump = *jumpTab;
13869 assert (curJump->countOfInEdges() > 0);
13871 // If val matches switchVal or we are at the last entry and
13872 // we never found the switch value then set the new jump dest
13874 if ( (val == switchVal) || (!foundVal && (val == jumpCnt-1)))
13876 if (curJump != block->bbNext)
13878 /* transform the basic block into a BBJ_ALWAYS */
13879 block->bbJumpKind = BBJ_ALWAYS;
13880 block->bbJumpDest = curJump;
13882 //if we are jumping backwards, make sure we have a GC Poll.
13883 if (curJump->bbNum > block->bbNum)
13884 block->bbFlags &= ~BBF_NEEDS_GCPOLL;
13888 /* transform the basic block into a BBJ_NONE */
13889 block->bbJumpKind = BBJ_NONE;
13890 block->bbFlags &= ~BBF_NEEDS_GCPOLL;
13896 /* Remove 'block' from the predecessor list of 'curJump' */
13897 fgRemoveRefPred(curJump, block);
13903 printf("\nConditional folded at BB%02u\n", block->bbNum);
13904 printf("BB%02u becomes a %s", block->bbNum,
13905 block->bbJumpKind == BBJ_ALWAYS ? "BBJ_ALWAYS" : "BBJ_NONE");
13906 if (block->bbJumpKind == BBJ_ALWAYS)
13907 printf(" to BB%02u", block->bbJumpDest->bbNum);
13919 //*****************************************************************************
13921 // Morphs a single statement in a block.
13922 // Can be called anytime, unlike fgMorphStmts() which should only be called once.
13924 // Returns true if 'stmt' was removed from the block.
13925 // Returns false if 'stmt' is still in the block (even if other statements were removed).
13928 bool Compiler::fgMorphBlockStmt(BasicBlock * block,
13930 DEBUGARG(const char * msg) )
13932 noway_assert(stmt->gtOper == GT_STMT);
13935 compCurStmt = stmt;
13937 GenTreePtr morph = fgMorphTree(stmt->gtStmt.gtStmtExpr);
13939 // Bug 1106830 - During the CSE phase we can't just remove
13940 // morph->gtOp.gtOp2 as it could contain CSE expressions.
13941 // This leads to a noway_assert in OptCSE.cpp when
13942 // searching for the removed CSE ref. (using gtFindLink)
13944 if (!optValnumCSE_phase)
13946 /* Check for morph as a GT_COMMA with an unconditional throw */
13947 if (fgIsCommaThrow(morph, true))
13952 printf("Folding a top-level fgIsCommaThrow stmt\n");
13953 printf("Removing op2 as unreachable:\n");
13954 gtDispTree(morph->gtOp.gtOp2);
13958 /* Use the call as the new stmt */
13959 morph = morph->gtOp.gtOp1;
13960 noway_assert(morph->gtOper == GT_CALL);
13963 /* we can get a throw as a statement root*/
13964 if (fgIsThrow(morph))
13969 printf("We have a top-level fgIsThrow stmt\n");
13970 printf("Removing the rest of block as unreachable:\n");
13973 noway_assert((morph->gtFlags & GTF_COLON_COND) == 0);
13974 fgRemoveRestOfBlock = true;
13978 stmt->gtStmt.gtStmtExpr = morph;
13980 /* Can the entire tree be removed ? */
13982 bool removedStmt = fgCheckRemoveStmt(block, stmt);
13984 /* Or this is the last statement of a conditional branch that was just folded */
13986 if ((!removedStmt) && (stmt->gtNext == NULL) && !fgRemoveRestOfBlock)
13988 if (fgFoldConditional(block))
13990 if (block->bbJumpKind != BBJ_THROW)
13991 removedStmt = true;
13997 /* Have to re-do the evaluation order since for example
13998 * some later code does not expect constants as op1 */
13999 gtSetStmtInfo(stmt);
14001 /* Have to re-link the nodes for this statement */
14002 fgSetStmtSeq(stmt);
14008 printf("%s %s tree:\n", msg, (removedStmt ? "removed" : "morphed"));
14014 if (fgRemoveRestOfBlock)
14016 /* Remove the rest of the stmts in the block */
14018 while (stmt->gtNext)
14020 stmt = stmt->gtNext;
14021 noway_assert(stmt->gtOper == GT_STMT);
14023 fgRemoveStmt(block, stmt);
14026 // The rest of block has been removed
14027 // and we will always throw an exception
14029 // Update succesors of block
14030 fgRemoveBlockAsPred(block);
14032 // For compDbgCode, we prepend an empty BB as the firstBB, it is BBJ_NONE.
14033 // We should not convert it to a ThrowBB.
14034 if ((block != fgFirstBB) || ((fgFirstBB->bbFlags & BBF_INTERNAL) == 0) ) {
14035 // Convert block to a throw bb
14036 fgConvertBBToThrowBB(block);
14042 printf("\n%s Block BB%02u becomes a throw block.\n", msg, block->bbNum);
14045 fgRemoveRestOfBlock = false;
14048 return removedStmt;
14051 /*****************************************************************************
14053 * Morph the statements of the given block.
14054 * This function should be called just once for a block. Use fgMorphBlockStmt()
14055 * for reentrant calls.
14058 void Compiler::fgMorphStmts(BasicBlock * block,
14059 bool * mult, bool * lnot, bool * loadw)
14061 fgRemoveRestOfBlock = false;
14063 noway_assert(fgExpandInline == false);
14065 /* Make the current basic block address available globally */
14069 *mult = *lnot = *loadw = false;
14071 fgCurrentlyInUseArgTemps = hashBv::Create(this);
14073 GenTreePtr stmt, prev;
14074 for (stmt = block->bbTreeList, prev = NULL;
14076 prev = stmt->gtStmt.gtStmtExpr,
14077 stmt = stmt->gtNext)
14079 noway_assert(stmt->gtOper == GT_STMT);
14081 if (fgRemoveRestOfBlock)
14083 fgRemoveStmt(block, stmt);
14086 #ifdef FEATURE_SIMD
14087 if (!opts.MinOpts() &&
14088 stmt->gtStmt.gtStmtExpr->TypeGet() == TYP_FLOAT &&
14089 stmt->gtStmt.gtStmtExpr->OperGet() == GT_ASG)
14091 fgMorphCombineSIMDFieldAssignments(block, stmt);
14095 fgMorphStmt = stmt;
14096 compCurStmt = stmt;
14097 GenTreePtr tree = stmt->gtStmt.gtStmtExpr;
14101 if (stmt == block->bbTreeList)
14102 block->bbStmtNum = compCurStmtNum; // Set the block->bbStmtNum
14104 unsigned oldHash = verbose ? gtHashValue(tree) : DUMMY_INIT(~0);
14108 printf("\nfgMorphTree BB%02u, stmt %d (before)\n", block->bbNum, compCurStmtNum);
14113 /* Morph this statement tree */
14115 GenTreePtr morph = fgMorphTree(tree);
14117 // mark any outgoing arg temps as free so we can reuse them in the next statement.
14119 fgCurrentlyInUseArgTemps->ZeroAll();
14121 // Has fgMorphStmt been sneakily changed ?
14123 if (stmt->gtStmt.gtStmtExpr != tree)
14125 /* This must be tailcall. Ignore 'morph' and carry on with
14126 the tail-call node */
14128 morph = stmt->gtStmt.gtStmtExpr;
14129 noway_assert(compTailCallUsed);
14130 noway_assert((morph->gtOper == GT_CALL) && morph->AsCall()->IsTailCall());
14131 noway_assert(stmt->gtNext == NULL);
14133 GenTreeCall* call = morph->AsCall();
14135 // - a tail call dispatched via helper in which case block will be ending with BBJ_THROW or
14136 // - a fast call made as jmp in which case block will be ending with BBJ_RETURN and marked as containing a jmp.
14137 noway_assert((call->IsTailCallViaHelper() && (compCurBB->bbJumpKind == BBJ_THROW)) ||
14138 (call->IsFastTailCall() && (compCurBB->bbJumpKind == BBJ_RETURN) && (compCurBB->bbFlags & BBF_HAS_JMP)));
14140 else if (block != compCurBB)
14142 /* This must be a tail call that caused a GCPoll to get
14143 injected. We haven't actually morphed the call yet
14144 but the flag still got set, clear it here... */
14147 tree->gtFlags &= ~GTF_MORPHED;
14149 noway_assert(compTailCallUsed);
14150 noway_assert((tree->gtOper == GT_CALL) && tree->AsCall()->IsTailCall());
14151 noway_assert(stmt->gtNext == NULL);
14153 GenTreeCall* call = morph->AsCall();
14156 // - a tail call dispatched via helper in which case block will be ending with BBJ_THROW or
14157 // - a fast call made as jmp in which case block will be ending with BBJ_RETURN and marked as containing a jmp.
14158 noway_assert((call->IsTailCallViaHelper() && (compCurBB->bbJumpKind == BBJ_THROW)) ||
14159 (call->IsFastTailCall() && (compCurBB->bbJumpKind == BBJ_RETURN) && (compCurBB->bbFlags & BBF_HAS_JMP)));
14163 if (compStressCompile(STRESS_CLONE_EXPR, 30))
14165 // Clone all the trees to stress gtCloneExpr()
14169 printf("\nfgMorphTree (stressClone from):\n");
14173 morph = gtCloneExpr(morph);
14174 noway_assert(morph);
14178 printf("\nfgMorphTree (stressClone to):\n");
14183 /* If the hash value changes. we modified the tree during morphing */
14186 unsigned newHash = gtHashValue(morph);
14187 if (newHash != oldHash)
14189 printf("\nfgMorphTree BB%02u, stmt %d (after)\n", block->bbNum, compCurStmtNum);
14195 /* Check for morph as a GT_COMMA with an unconditional throw */
14196 if (!gtIsActiveCSE_Candidate(morph) && fgIsCommaThrow(morph, true))
14198 /* Use the call as the new stmt */
14199 morph = morph->gtOp.gtOp1;
14200 noway_assert(morph->gtOper == GT_CALL);
14201 noway_assert((morph->gtFlags & GTF_COLON_COND) == 0);
14203 fgRemoveRestOfBlock = true;
14206 stmt->gtStmt.gtStmtExpr = tree = morph;
14208 noway_assert(fgPtrArgCntCur == 0);
14210 if (fgRemoveRestOfBlock)
14213 /* Has the statement been optimized away */
14215 if (fgCheckRemoveStmt(block, stmt))
14218 /* Check if this block ends with a conditional branch that can be folded */
14220 if (fgFoldConditional(block))
14223 if (ehBlockHasExnFlowDsc(block))
14226 #if OPT_MULT_ADDSUB
14228 /* Note whether we have two or more +=/-= operators in a row */
14230 if (tree->gtOper == GT_ASG_ADD ||
14231 tree->gtOper == GT_ASG_SUB)
14233 if (prev && prev->gtOper == tree->gtOper)
14239 /* Note "x = a[i] & icon" followed by "x |= a[i] << 8" */
14241 if (tree->gtOper == GT_ASG_OR &&
14243 prev->gtOper == GT_ASG)
14249 if (fgRemoveRestOfBlock)
14251 if ((block->bbJumpKind == BBJ_COND) || (block->bbJumpKind == BBJ_SWITCH))
14253 GenTreePtr first = block->bbTreeList; noway_assert(first);
14254 GenTreePtr last = first->gtPrev; noway_assert(last && last->gtNext == NULL);
14255 GenTreePtr lastStmt = last->gtStmt.gtStmtExpr;
14257 if (((block->bbJumpKind == BBJ_COND ) && (lastStmt->gtOper == GT_JTRUE )) ||
14258 ((block->bbJumpKind == BBJ_SWITCH) && (lastStmt->gtOper == GT_SWITCH)) )
14260 GenTreePtr op1 = lastStmt->gtOp.gtOp1;
14262 if (op1->OperKind() & GTK_RELOP)
14264 /* Unmark the comparison node with GTF_RELOP_JMP_USED */
14265 op1->gtFlags &= ~GTF_RELOP_JMP_USED;
14268 last->gtStmt.gtStmtExpr = fgMorphTree(op1);
14272 /* Mark block as a BBJ_THROW block */
14273 fgConvertBBToThrowBB(block);
14276 noway_assert(fgExpandInline == false);
14278 #if FEATURE_FASTTAILCALL
14279 GenTreePtr recursiveTailCall = nullptr;
14280 if (block->endsWithTailCallConvertibleToLoop(this, &recursiveTailCall))
14282 fgMorphRecursiveFastTailCallIntoLoop(block, recursiveTailCall->AsCall());
14287 compCurBB = (BasicBlock*)INVALID_POINTER_VALUE;
14290 // Reset this back so that it doesn't leak out impacting other blocks
14291 fgRemoveRestOfBlock = false;
14294 /*****************************************************************************
14296 * Morph the blocks of the method.
14297 * Returns true if the basic block list is modified.
14298 * This function should be called just once.
14301 void Compiler::fgMorphBlocks()
14305 printf("\n*************** In fgMorphBlocks()\n");
14308 /* Since fgMorphTree can be called after various optimizations to re-arrange
14309 * the nodes we need a global flag to signal if we are during the one-pass
14310 * global morphing */
14312 fgGlobalMorph = true;
14314 #if LOCAL_ASSERTION_PROP
14316 // Local assertion prop is enabled if we are optimized
14318 optLocalAssertionProp = (!opts.compDbgCode && !opts.MinOpts());
14320 if (optLocalAssertionProp)
14323 // Initialize for local assertion prop
14325 optAssertionInit(true);
14327 #elif ASSERTION_PROP
14329 // If LOCAL_ASSERTION_PROP is not set
14330 // and we have global assertion prop
14331 // then local assertion prop is always off
14333 optLocalAssertionProp = false;
14337 /*-------------------------------------------------------------------------
14338 * Process all basic blocks in the function
14341 BasicBlock * block = fgFirstBB; noway_assert(block);
14344 compCurStmtNum = 0;
14349 #if OPT_MULT_ADDSUB
14357 bool loadw = false;
14361 printf("\nMorphing BB%02u of '%s'\n", block->bbNum, info.compFullName);
14364 #if LOCAL_ASSERTION_PROP
14365 if (optLocalAssertionProp)
14368 // Clear out any currently recorded assertion candidates
14369 // before processing each basic block,
14370 // also we must handle QMARK-COLON specially
14372 optAssertionReset(0);
14376 /* Process all statement trees in the basic block */
14380 fgMorphStmts(block, &mult, &lnot, &loadw);
14382 #if OPT_MULT_ADDSUB
14384 if (mult && (opts.compFlags & CLFLG_TREETRANS) &&
14385 !opts.compDbgCode && !opts.MinOpts())
14387 for (tree = block->bbTreeList; tree; tree = tree->gtNext)
14389 noway_assert(tree->gtOper == GT_STMT);
14390 GenTreePtr last = tree->gtStmt.gtStmtExpr;
14392 if (last->gtOper == GT_ASG_ADD ||
14393 last->gtOper == GT_ASG_SUB)
14398 GenTreePtr dst1 = last->gtOp.gtOp1;
14399 GenTreePtr src1 = last->gtOp.gtOp2;
14401 if (!last->IsCnsIntOrI())
14404 if (dst1->gtOper != GT_LCL_VAR)
14406 if (!src1->IsCnsIntOrI())
14414 /* Look at the next statement */
14416 temp = tree->gtNext;
14420 noway_assert(temp->gtOper == GT_STMT);
14421 next = temp->gtStmt.gtStmtExpr;
14423 if (next->gtOper != last->gtOper)
14425 if (next->gtType != last->gtType)
14428 dst2 = next->gtOp.gtOp1;
14429 src2 = next->gtOp.gtOp2;
14431 if (dst2->gtOper != GT_LCL_VAR)
14433 if (dst2->gtLclVarCommon.gtLclNum != dst1->gtLclVarCommon.gtLclNum)
14436 if (!src2->IsCnsIntOrI())
14439 if (last->gtOverflow() != next->gtOverflow())
14442 const ssize_t i1 = src1->gtIntCon.gtIconVal;
14443 const ssize_t i2 = src2->gtIntCon.gtIconVal;
14444 const ssize_t itemp = i1 + i2;
14446 /* if the operators are checking for overflow, check for overflow of the operands */
14448 if (next->gtOverflow())
14450 if (next->TypeGet() == TYP_LONG)
14452 if (next->gtFlags & GTF_UNSIGNED)
14454 ClrSafeInt<UINT64> si1(i1);
14455 if ((si1 + ClrSafeInt<UINT64>(i2)).IsOverflow())
14460 ClrSafeInt<INT64> si1(i1);
14461 if ((si1 + ClrSafeInt<INT64>(i2)).IsOverflow())
14465 else if (next->gtFlags & GTF_UNSIGNED)
14467 ClrSafeInt<UINT32> si1(i1);
14468 if ((si1 + ClrSafeInt<UINT32>(i2)).IsOverflow())
14473 ClrSafeInt<INT32> si1(i1);
14474 if ((si1 + ClrSafeInt<INT32>(i2)).IsOverflow())
14479 /* Fold the two increments/decrements into one */
14481 src1->gtIntCon.gtIconVal = itemp;
14482 #ifdef _TARGET_64BIT_
14483 if (src1->gtType == TYP_INT)
14485 src1->AsIntCon()->TruncateOrSignExtend32();
14487 #endif //_TARGET_64BIT_
14489 /* Remove the second statement completely */
14491 noway_assert(tree->gtNext == temp);
14492 noway_assert(temp->gtPrev == tree);
14496 noway_assert(temp->gtNext->gtPrev == temp);
14498 temp->gtNext->gtPrev = tree;
14499 tree->gtNext = temp->gtNext;
14505 noway_assert(block->bbTreeList->gtPrev == temp);
14507 block->bbTreeList->gtPrev = tree;
14520 /* Are we using a single return block? */
14522 if (block->bbJumpKind == BBJ_RETURN)
14524 if ((genReturnBB != nullptr) &&
14525 (genReturnBB != block) &&
14526 ((block->bbFlags & BBF_HAS_JMP) == 0))
14528 /* We'll jump to the genReturnBB */
14530 #if !defined(_TARGET_X86_)
14531 if (info.compFlags & CORINFO_FLG_SYNCH)
14533 fgConvertSyncReturnToLeave(block);
14536 #endif // !_TARGET_X86_
14538 block->bbJumpKind = BBJ_ALWAYS;
14539 block->bbJumpDest = genReturnBB;
14543 // Note 1: A block is not guaranteed to have a last stmt if its jump kind is BBJ_RETURN.
14544 // For example a method returning void could have an empty block with jump kind BBJ_RETURN.
14545 // Such blocks do materialize as part of in-lining.
14547 // Note 2: A block with jump kind BBJ_RETURN does not necessarily need to end with GT_RETURN.
14548 // It could end with a tail call or rejected tail call or monitor.exit or a GT_INTRINSIC.
14549 // For now it is safe to explicitly check whether last stmt is GT_RETURN if genReturnLocal
14552 // TODO: Need to characterize the last top level stmt of a block ending with BBJ_RETURN.
14554 GenTreePtr last = (block->bbTreeList != nullptr) ? block->bbTreeList->gtPrev : nullptr;
14555 GenTreePtr ret = (last != nullptr) ? last->gtStmt.gtStmtExpr : nullptr;
14557 //replace the GT_RETURN node to be a GT_ASG that stores the return value into genReturnLocal.
14558 if (genReturnLocal != BAD_VAR_NUM)
14560 // Method must be returning a value other than TYP_VOID.
14561 noway_assert(compMethodHasRetVal());
14563 // This block must be ending with a GT_RETURN
14564 noway_assert(last != nullptr);
14565 noway_assert(last->gtOper == GT_STMT);
14566 noway_assert(last->gtNext == nullptr);
14567 noway_assert(ret != nullptr);
14569 // GT_RETURN must have non-null operand as the method is returning the value assigned to genReturnLocal
14570 noway_assert(ret->OperGet() == GT_RETURN);
14571 noway_assert(ret->gtGetOp1() != nullptr);
14572 noway_assert(ret->gtGetOp2() == nullptr);
14574 last->gtStmt.gtStmtExpr = gtNewTempAssign(genReturnLocal, ret->gtGetOp1());
14576 //make sure that copy-prop ignores this assignment.
14577 last->gtStmt.gtStmtExpr->gtFlags |= GTF_DONT_CSE;
14579 else if (ret != nullptr && ret->OperGet() == GT_RETURN)
14581 // This block ends with a GT_RETURN
14582 noway_assert(last != nullptr);
14583 noway_assert(last->gtOper == GT_STMT);
14584 noway_assert(last->gtNext == nullptr);
14586 // Must be a void GT_RETURN with null operand; delete it as this block branches to oneReturn block
14587 noway_assert(ret->TypeGet() == TYP_VOID);
14588 noway_assert(ret->gtGetOp1() == nullptr);
14589 noway_assert(ret->gtGetOp2() == nullptr);
14591 fgRemoveStmt(block, last);
14597 printf("morph BB%02u to point at onereturn. New block is\n",
14599 fgTableDispBasicBlock(block);
14605 block = block->bbNext;
14609 /* We are done with the global morphing phase */
14611 fgGlobalMorph = false;
14616 fgDispBasicBlocks(true);
14622 /*****************************************************************************
14624 * Make some decisions about the kind of code to generate.
14627 void Compiler::fgSetOptions()
14630 /* Should we force fully interruptible code ? */
14633 if (JitConfig.JitFullyInt() ||
14634 compStressCompile(STRESS_GENERIC_VARN, 30))
14636 noway_assert(!codeGen->isGCTypeFixed());
14637 genInterruptible = true;
14641 #ifdef DEBUGGING_SUPPORT
14642 if (opts.compDbgCode)
14644 assert(!codeGen->isGCTypeFixed());
14645 genInterruptible = true; // debugging is easier this way ...
14649 /* Assume we won't need an explicit stack frame if this is allowed */
14652 // CORINFO_HELP_TAILCALL won't work with localloc because of the restoring of
14653 // the callee-saved registers.
14654 noway_assert(!compTailCallUsed || !compLocallocUsed);
14656 if (compLocallocUsed)
14657 codeGen->setFramePointerRequired(true);
14659 #ifdef _TARGET_X86_
14661 if (compTailCallUsed)
14662 codeGen->setFramePointerRequired(true);
14664 #endif // _TARGET_X86_
14666 if (!opts.genFPopt)
14667 codeGen->setFramePointerRequired(true);
14669 // Assert that the EH table has been initialized by now. Note that
14670 // compHndBBtabAllocCount never decreases; it is a high-water mark
14671 // of table allocation. In contrast, compHndBBtabCount does shrink
14672 // if we delete a dead EH region, and if it shrinks to zero, the
14673 // table pointer compHndBBtab is unreliable.
14674 assert(compHndBBtabAllocCount >= info.compXcptnsCount);
14676 #ifdef _TARGET_X86_
14678 // Note: this case, and the !X86 case below, should both use the
14679 // !X86 path. This would require a few more changes for X86 to use
14680 // compHndBBtabCount (the current number of EH clauses) instead of
14681 // info.compXcptnsCount (the number of EH clauses in IL), such as
14682 // in ehNeedsShadowSPslots(). This is because sometimes the IL has
14683 // an EH clause that we delete as statically dead code before we
14684 // get here, leaving no EH clauses left, and thus no requirement
14685 // to use a frame pointer because of EH. But until all the code uses
14686 // the same test, leave info.compXcptnsCount here.
14687 if (info.compXcptnsCount > 0)
14688 codeGen->setFramePointerRequiredEH(true);
14690 #else // !_TARGET_X86_
14692 if (compHndBBtabCount > 0)
14693 codeGen->setFramePointerRequiredEH(true);
14695 #endif // _TARGET_X86_
14697 // fpPtrArgCntMax records the maximum number of pushed arguments
14698 // Depending upon this value of the maximum number of pushed arguments
14699 // we may need to use an EBP frame or be partially interuptible
14702 if (!compCanEncodePtrArgCntMax())
14706 printf("Too many pushed arguments for fully interruptible encoding, marking method as partially interruptible\n");
14708 genInterruptible = false;
14710 if (fgPtrArgCntMax >= sizeof(unsigned))
14714 printf("Too many pushed arguments for an ESP based encoding, forcing an EBP frame\n");
14716 codeGen->setFramePointerRequiredGCInfo(true);
14720 if (info.compCallUnmanaged)
14722 codeGen->setFramePointerRequired(true); // Setup of Pinvoke frame currently requires an EBP style frame
14726 if (info.compPublishStubParam)
14728 codeGen->setFramePointerRequiredGCInfo(true);
14731 if (opts.compNeedSecurityCheck)
14733 codeGen->setFramePointerRequiredGCInfo(true);
14735 #ifndef JIT32_GCENCODER
14737 // The decoder only reports objects in frames with exceptions if the frame
14738 // is fully interruptible.
14739 // Even if there is no catch or other way to resume execution in this frame
14740 // the VM requires the security object to remain alive until later, so
14741 // Frames with security objects must be fully interruptible.
14742 genInterruptible = true;
14744 #endif // JIT32_GCENCODER
14747 if (compIsProfilerHookNeeded())
14749 codeGen->setFramePointerRequired(true);
14752 if (info.compIsVarArgs)
14754 // Code that initializes lvaVarargsBaseOfStkArgs requires this to be EBP relative.
14755 codeGen->setFramePointerRequiredGCInfo(true);
14758 if (lvaReportParamTypeArg())
14760 codeGen->setFramePointerRequiredGCInfo(true);
14763 // printf("method will %s be fully interruptible\n", genInterruptible ? " " : "not");
14767 /*****************************************************************************/
14769 GenTreePtr Compiler::fgInitThisClass()
14771 noway_assert(!compIsForInlining());
14773 CORINFO_LOOKUP_KIND kind = info.compCompHnd->getLocationOfThisType(info.compMethodHnd);
14775 if (!kind.needsRuntimeLookup)
14777 return fgGetSharedCCtor(info.compClassHnd);
14781 // Collectible types requires that for shared generic code, if we use the generic context paramter
14782 // that we report it. (This is a conservative approach, we could detect some cases particularly when the
14783 // context parameter is this that we don't need the eager reporting logic.)
14784 lvaGenericsContextUsed = true;
14786 switch (kind.runtimeLookupKind)
14788 case CORINFO_LOOKUP_THISOBJ :
14789 // This code takes a this pointer; but we need to pass the static method desc to get the right point in the hierarchy
14791 GenTreePtr vtTree = gtNewLclvNode(info.compThisArg, TYP_REF);
14792 // Vtable pointer of this object
14793 vtTree = gtNewOperNode(GT_IND, TYP_I_IMPL, vtTree);
14794 vtTree->gtFlags |= GTF_EXCEPT; // Null-pointer exception
14795 GenTreePtr methodHnd = gtNewIconEmbMethHndNode(info.compMethodHnd);
14797 return gtNewHelperCallNode(CORINFO_HELP_INITINSTCLASS,
14799 gtNewArgList(vtTree, methodHnd));
14803 case CORINFO_LOOKUP_CLASSPARAM :
14805 GenTreePtr vtTree = gtNewLclvNode(info.compTypeCtxtArg, TYP_I_IMPL);
14806 return gtNewHelperCallNode(CORINFO_HELP_INITCLASS,
14808 gtNewArgList(vtTree));
14811 case CORINFO_LOOKUP_METHODPARAM :
14813 GenTreePtr methHndTree = gtNewLclvNode(info.compTypeCtxtArg, TYP_I_IMPL);
14814 return gtNewHelperCallNode(CORINFO_HELP_INITINSTCLASS,
14816 gtNewArgList(gtNewIconNode(0),methHndTree));
14822 noway_assert(!"Unknown LOOKUP_KIND");
14828 /*****************************************************************************
14830 * Tree walk callback to make sure no GT_QMARK nodes are present in the tree,
14831 * except for the allowed ? 1 : 0; pattern.
14833 Compiler::fgWalkResult Compiler::fgAssertNoQmark(GenTreePtr* tree, fgWalkData* data)
14835 if ((*tree)->OperGet() == GT_QMARK)
14837 fgCheckQmarkAllowedForm(*tree);
14839 return WALK_CONTINUE;
14842 void Compiler::fgCheckQmarkAllowedForm(GenTree* tree)
14844 assert(tree->OperGet() == GT_QMARK);
14845 #ifndef LEGACY_BACKEND
14846 assert(!"Qmarks beyond morph disallowed.");
14847 #else // LEGACY_BACKEND
14848 GenTreePtr colon = tree->gtOp.gtOp2;
14850 assert(colon->gtOp.gtOp1->gtOper == GT_CNS_INT);
14851 assert(colon->gtOp.gtOp1->AsIntCon()->IconValue() == 0);
14853 assert(colon->gtOp.gtOp2->gtOper == GT_CNS_INT);
14854 assert(colon->gtOp.gtOp2->AsIntCon()->IconValue() == 1);
14855 #endif // LEGACY_BACKEND
14858 /*****************************************************************************
14860 * Verify that the importer has created GT_QMARK nodes in a way we can
14861 * process them. The following is allowed:
14863 * 1. A top level qmark. Top level qmark is of the form:
14864 * a) (bool) ? (void) : (void) OR
14865 * b) V0N = (bool) ? (type) : (type)
14867 * 2. Recursion is allowed at the top level, i.e., a GT_QMARK can be a child
14868 * of either op1 of colon or op2 of colon but not a child of any other
14871 void Compiler::fgPreExpandQmarkChecks(GenTreePtr expr)
14873 GenTreePtr topQmark = fgGetTopLevelQmark(expr);
14875 // If the top level Qmark is null, then scan the tree to make sure
14876 // there are no qmarks within it.
14877 if (topQmark == NULL)
14879 fgWalkTreePre(&expr, Compiler::fgAssertNoQmark, NULL);
14883 // We could probably expand the cond node also, but don't think the extra effort is necessary,
14884 // so let's just assert the cond node of a top level qmark doesn't have further top level qmarks.
14885 fgWalkTreePre(&topQmark->gtOp.gtOp1, Compiler::fgAssertNoQmark, NULL);
14887 fgPreExpandQmarkChecks(topQmark->gtOp.gtOp2->gtOp.gtOp1);
14888 fgPreExpandQmarkChecks(topQmark->gtOp.gtOp2->gtOp.gtOp2);
14893 /*****************************************************************************
14895 * Get the top level GT_QMARK node in a given "expr", return NULL if such a
14896 * node is not present. If the top level GT_QMARK node is assigned to a
14897 * GT_LCL_VAR, then return the lcl node in ppDst.
14900 GenTreePtr Compiler::fgGetTopLevelQmark(GenTreePtr expr, GenTreePtr* ppDst /* = NULL */)
14907 GenTreePtr topQmark = NULL;
14908 if (expr->gtOper == GT_QMARK)
14912 else if (expr->gtOper == GT_ASG &&
14913 expr->gtOp.gtOp2->gtOper == GT_QMARK &&
14914 expr->gtOp.gtOp1->gtOper == GT_LCL_VAR)
14916 topQmark = expr->gtOp.gtOp2;
14919 *ppDst = expr->gtOp.gtOp1;
14926 /*********************************************************************************
14928 * For a castclass helper call,
14929 * Importer creates the following tree:
14930 * tmp = (op1 == null) ? op1 : ((*op1 == (cse = op2, cse)) ? op1 : helper());
14932 * This method splits the qmark expression created by the importer into the
14933 * following blocks: (block, asg, cond1, cond2, helper, remainder)
14934 * Notice that op1 is the result for both the conditions. So we coalesce these
14935 * assignments into a single block instead of two blocks resulting a nested diamond.
14937 * +---------->-----------+
14941 * block-->asg-->cond1--+-->cond2--+-->helper--+-->remainder
14943 * We expect to achieve the following codegen:
14944 * mov rsi, rdx tmp = op1 // asgBlock
14945 * test rsi, rsi goto skip if tmp == null ? // cond1Block
14947 * mov rcx, 0x76543210 cns = op2 // cond2Block
14948 * cmp qword ptr [rsi], rcx goto skip if *tmp == op2
14950 * call CORINFO_HELP_CHKCASTCLASS_SPECIAL tmp = helper(cns, tmp) // helperBlock
14952 * SKIP: // remainderBlock
14953 * tmp has the result.
14956 void Compiler::fgExpandQmarkForCastInstOf(BasicBlock* block, GenTreePtr stmt)
14961 printf("\nExpanding CastInstOf qmark in BB%02u (before)\n", block->bbNum);
14962 fgDispBasicBlocks(block, block, true);
14966 GenTreePtr expr = stmt->gtStmt.gtStmtExpr;
14968 GenTreePtr dst = nullptr;
14969 GenTreePtr qmark = fgGetTopLevelQmark(expr, &dst);
14970 noway_assert(dst != nullptr);
14972 assert(qmark->gtFlags & GTF_QMARK_CAST_INSTOF);
14974 // Get cond, true, false exprs for the qmark.
14975 GenTreePtr condExpr = qmark->gtGetOp1();
14976 GenTreePtr trueExpr = qmark->gtGetOp2()->AsColon()->ThenNode();
14977 GenTreePtr falseExpr = qmark->gtGetOp2()->AsColon()->ElseNode();
14979 // Get cond, true, false exprs for the nested qmark.
14980 GenTreePtr nestedQmark = falseExpr;
14981 GenTreePtr cond2Expr;
14982 GenTreePtr true2Expr;
14983 GenTreePtr false2Expr;
14985 if (nestedQmark->gtOper == GT_QMARK)
14987 cond2Expr = nestedQmark->gtGetOp1();
14988 true2Expr = nestedQmark->gtGetOp2()->AsColon()->ThenNode();
14989 false2Expr = nestedQmark->gtGetOp2()->AsColon()->ElseNode();
14991 assert(cond2Expr->gtFlags & GTF_RELOP_QMARK);
14992 cond2Expr->gtFlags &= ~GTF_RELOP_QMARK;
14996 // This is a rare case that arises when we are doing minopts and encounter isinst of null
14997 // gtFoldExpr was still is able to optimize away part of the tree (but not all).
14998 // That means it does not match our pattern.
15000 // Rather than write code to handle this case, just fake up some nodes to make it match the common
15001 // case. Synthesize a comparison that is always true, and for the result-on-true, use the
15002 // entire subtree we expected to be the nested question op.
15004 cond2Expr = gtNewOperNode(GT_EQ, TYP_INT, gtNewIconNode(0, TYP_I_IMPL), gtNewIconNode(0, TYP_I_IMPL));
15005 true2Expr = nestedQmark;
15006 false2Expr = gtNewIconNode(0, TYP_I_IMPL);
15008 assert(false2Expr->OperGet() == trueExpr->OperGet());
15010 // Clear flags as they are now going to be part of JTRUE.
15011 assert(condExpr->gtFlags & GTF_RELOP_QMARK);
15012 condExpr->gtFlags &= ~GTF_RELOP_QMARK;
15014 // Create the chain of blocks. See method header comment.
15015 // The order of blocks after this is the following:
15016 // block ... asgBlock ... cond1Block ... cond2Block ... helperBlock ... remainderBlock
15018 // We need to remember flags that exist on 'block' that we want to propagate to 'remainderBlock',
15019 // if they are going to be cleared by fgSplitBlockAfterStatement(). We currently only do this only
15020 // for the GC safe point bit, the logic being that if 'block' was marked gcsafe, then surely
15021 // remainderBlock will still be GC safe.
15022 unsigned propagateFlags = block->bbFlags & BBF_GC_SAFE_POINT;
15023 BasicBlock* remainderBlock = fgSplitBlockAfterStatement(block, stmt);
15024 fgRemoveRefPred(remainderBlock, block); // We're going to put more blocks between block and remainderBlock.
15026 BasicBlock* helperBlock = fgNewBBafter(BBJ_NONE, block, true);
15027 BasicBlock* cond2Block = fgNewBBafter(BBJ_COND, block, true);
15028 BasicBlock* cond1Block = fgNewBBafter(BBJ_COND, block, true);
15029 BasicBlock* asgBlock = fgNewBBafter(BBJ_NONE, block, true);
15031 remainderBlock->bbFlags |= BBF_JMP_TARGET | BBF_HAS_LABEL | propagateFlags;
15033 // These blocks are only internal if 'block' is (but they've been set as internal by fgNewBBafter).
15034 // If they're not internal, mark them as imported to avoid asserts about un-imported blocks.
15035 if ((block->bbFlags & BBF_INTERNAL) == 0)
15037 helperBlock->bbFlags &= ~BBF_INTERNAL;
15038 cond2Block->bbFlags &= ~BBF_INTERNAL;
15039 cond1Block->bbFlags &= ~BBF_INTERNAL;
15040 asgBlock->bbFlags &= ~BBF_INTERNAL;
15041 helperBlock->bbFlags |= BBF_IMPORTED;
15042 cond2Block->bbFlags |= BBF_IMPORTED;
15043 cond1Block->bbFlags |= BBF_IMPORTED;
15044 asgBlock->bbFlags |= BBF_IMPORTED;
15047 // Chain the flow correctly.
15048 fgAddRefPred(asgBlock, block);
15049 fgAddRefPred(cond1Block, asgBlock);
15050 fgAddRefPred(cond2Block, cond1Block);
15051 fgAddRefPred(helperBlock, cond2Block);
15052 fgAddRefPred(remainderBlock, helperBlock);
15053 fgAddRefPred(remainderBlock, cond1Block);
15054 fgAddRefPred(remainderBlock, cond2Block);
15056 cond1Block->bbJumpDest = remainderBlock;
15057 cond2Block->bbJumpDest = remainderBlock;
15059 // Set the weights; some are guesses.
15060 asgBlock->inheritWeight(block);
15061 cond1Block->inheritWeight(block);
15062 cond2Block->inheritWeightPercentage(cond1Block, 50);
15063 helperBlock->inheritWeightPercentage(cond2Block, 50);
15065 // Append cond1 as JTRUE to cond1Block
15066 GenTreePtr jmpTree = gtNewOperNode(GT_JTRUE, TYP_VOID, condExpr);
15067 GenTreePtr jmpStmt = fgNewStmtFromTree(jmpTree, stmt->gtStmt.gtStmtILoffsx);
15068 fgInsertStmtAtEnd(cond1Block, jmpStmt);
15070 // Append cond2 as JTRUE to cond2Block
15071 jmpTree = gtNewOperNode(GT_JTRUE, TYP_VOID, cond2Expr);
15072 jmpStmt = fgNewStmtFromTree(jmpTree, stmt->gtStmt.gtStmtILoffsx);
15073 fgInsertStmtAtEnd(cond2Block, jmpStmt);
15075 // AsgBlock should get tmp = op1 assignment.
15076 trueExpr = gtNewTempAssign(dst->AsLclVarCommon()->GetLclNum(), trueExpr);
15077 GenTreePtr trueStmt = fgNewStmtFromTree(trueExpr, stmt->gtStmt.gtStmtILoffsx);
15078 fgInsertStmtAtEnd(asgBlock, trueStmt);
15080 // Since we are adding helper in the JTRUE false path, reverse the cond2 and add the helper.
15081 gtReverseCond(cond2Expr);
15082 GenTreePtr helperExpr = gtNewTempAssign(dst->AsLclVarCommon()->GetLclNum(), true2Expr);
15083 GenTreePtr helperStmt = fgNewStmtFromTree(helperExpr, stmt->gtStmt.gtStmtILoffsx);
15084 fgInsertStmtAtEnd(helperBlock, helperStmt);
15086 // Finally remove the nested qmark stmt.
15087 fgRemoveStmt(block, stmt);
15092 printf("\nExpanding CastInstOf qmark in BB%02u (after)\n", block->bbNum);
15093 fgDispBasicBlocks(block, remainderBlock, true);
15098 /*****************************************************************************
15100 * Expand a statement with a top level qmark node. There are three cases, based
15101 * on whether the qmark has both "true" and "false" arms, or just one of them.
15112 * S0 -->-- ~C -->-- T F -->-- S1
15117 * -----------------------------------------
15126 * S0 -->-- ~C -->-- T -->-- S1
15128 * +-->-------------+
15131 * -----------------------------------------
15140 * S0 -->-- C -->-- F -->-- S1
15142 * +-->------------+
15145 * If the qmark assigns to a variable, then create tmps for "then"
15146 * and "else" results and assign the temp to the variable as a writeback step.
15148 void Compiler::fgExpandQmarkStmt(BasicBlock* block, GenTreePtr stmt)
15150 GenTreePtr expr = stmt->gtStmt.gtStmtExpr;
15152 // Retrieve the Qmark node to be expanded.
15153 GenTreePtr dst = nullptr;
15154 GenTreePtr qmark = fgGetTopLevelQmark(expr, &dst);
15155 if (qmark == nullptr)
15160 if (qmark->gtFlags & GTF_QMARK_CAST_INSTOF)
15162 fgExpandQmarkForCastInstOf(block, stmt);
15169 printf("\nExpanding top-level qmark in BB%02u (before)\n", block->bbNum);
15170 fgDispBasicBlocks(block, block, true);
15174 // Retrieve the operands.
15175 GenTreePtr condExpr = qmark->gtGetOp1();
15176 GenTreePtr trueExpr = qmark->gtGetOp2()->AsColon()->ThenNode();
15177 GenTreePtr falseExpr = qmark->gtGetOp2()->AsColon()->ElseNode();
15179 assert(condExpr->gtFlags & GTF_RELOP_QMARK);
15180 condExpr->gtFlags &= ~GTF_RELOP_QMARK;
15182 assert(!varTypeIsFloating(condExpr->TypeGet()));
15184 bool hasTrueExpr = (trueExpr->OperGet() != GT_NOP);
15185 bool hasFalseExpr = (falseExpr->OperGet() != GT_NOP);
15186 assert(hasTrueExpr || hasFalseExpr); // We expect to have at least one arm of the qmark!
15188 // Create remainder, cond and "else" blocks. After this, the blocks are in this order:
15189 // block ... condBlock ... elseBlock ... remainderBlock
15191 // We need to remember flags that exist on 'block' that we want to propagate to 'remainderBlock',
15192 // if they are going to be cleared by fgSplitBlockAfterStatement(). We currently only do this only
15193 // for the GC safe point bit, the logic being that if 'block' was marked gcsafe, then surely
15194 // remainderBlock will still be GC safe.
15195 unsigned propagateFlags = block->bbFlags & BBF_GC_SAFE_POINT;
15196 BasicBlock* remainderBlock = fgSplitBlockAfterStatement(block, stmt);
15197 fgRemoveRefPred(remainderBlock, block); // We're going to put more blocks between block and remainderBlock.
15199 BasicBlock* condBlock = fgNewBBafter(BBJ_COND, block, true);
15200 BasicBlock* elseBlock = fgNewBBafter(BBJ_NONE, condBlock, true);
15202 // These blocks are only internal if 'block' is (but they've been set as internal by fgNewBBafter).
15203 // If they're not internal, mark them as imported to avoid asserts about un-imported blocks.
15204 if ((block->bbFlags & BBF_INTERNAL) == 0)
15206 condBlock->bbFlags &= ~BBF_INTERNAL;
15207 elseBlock->bbFlags &= ~BBF_INTERNAL;
15208 condBlock->bbFlags |= BBF_IMPORTED;
15209 elseBlock->bbFlags |= BBF_IMPORTED;
15212 remainderBlock->bbFlags |= BBF_JMP_TARGET | BBF_HAS_LABEL | propagateFlags;
15214 condBlock->inheritWeight(block);
15216 fgAddRefPred(condBlock, block);
15217 fgAddRefPred(elseBlock, condBlock);
15218 fgAddRefPred(remainderBlock, elseBlock);
15220 BasicBlock* thenBlock = nullptr;
15221 if (hasTrueExpr && hasFalseExpr)
15226 // S0 -->-- ~C -->-- T F -->-- S1
15231 gtReverseCond(condExpr);
15232 condBlock->bbJumpDest = elseBlock;
15234 thenBlock = fgNewBBafter(BBJ_ALWAYS, condBlock, true);
15235 thenBlock->bbJumpDest = remainderBlock;
15236 if ((block->bbFlags & BBF_INTERNAL) == 0)
15238 thenBlock->bbFlags &= ~BBF_INTERNAL;
15239 thenBlock->bbFlags |= BBF_IMPORTED;
15242 elseBlock->bbFlags |= (BBF_JMP_TARGET | BBF_HAS_LABEL);
15244 fgAddRefPred(thenBlock, condBlock);
15245 fgAddRefPred(remainderBlock, thenBlock);
15247 thenBlock->inheritWeightPercentage(condBlock, 50);
15248 elseBlock->inheritWeightPercentage(condBlock, 50);
15250 else if (hasTrueExpr)
15253 // S0 -->-- ~C -->-- T -->-- S1
15255 // +-->-------------+
15258 gtReverseCond(condExpr);
15259 condBlock->bbJumpDest = remainderBlock;
15260 fgAddRefPred(remainderBlock, condBlock);
15261 // Since we have no false expr, use the one we'd already created.
15262 thenBlock = elseBlock;
15263 elseBlock = nullptr;
15265 thenBlock->inheritWeightPercentage(condBlock, 50);
15267 else if (hasFalseExpr)
15270 // S0 -->-- C -->-- F -->-- S1
15272 // +-->------------+
15275 condBlock->bbJumpDest = remainderBlock;
15276 fgAddRefPred(remainderBlock, condBlock);
15278 elseBlock->inheritWeightPercentage(condBlock, 50);
15281 GenTreePtr jmpTree = gtNewOperNode(GT_JTRUE, TYP_VOID, qmark->gtGetOp1());
15282 GenTreePtr jmpStmt = fgNewStmtFromTree(jmpTree, stmt->gtStmt.gtStmtILoffsx);
15283 fgInsertStmtAtEnd(condBlock, jmpStmt);
15285 // Remove the original qmark statement.
15286 fgRemoveStmt(block, stmt);
15288 // Since we have top level qmarks, we either have a dst for it in which case
15289 // we need to create tmps for true and falseExprs, else just don't bother
15291 unsigned lclNum = BAD_VAR_NUM;
15292 if (dst != nullptr)
15294 assert(dst->gtOper == GT_LCL_VAR);
15295 lclNum = dst->gtLclVar.gtLclNum;
15299 assert(qmark->TypeGet() == TYP_VOID);
15304 if (dst != nullptr)
15306 trueExpr = gtNewTempAssign(lclNum, trueExpr);
15308 GenTreePtr trueStmt = fgNewStmtFromTree(trueExpr, stmt->gtStmt.gtStmtILoffsx);
15309 fgInsertStmtAtEnd(thenBlock, trueStmt);
15312 // Assign the falseExpr into the dst or tmp, insert in elseBlock
15315 if (dst != nullptr)
15317 falseExpr = gtNewTempAssign(lclNum, falseExpr);
15319 GenTreePtr falseStmt = fgNewStmtFromTree(falseExpr, stmt->gtStmt.gtStmtILoffsx);
15320 fgInsertStmtAtEnd(elseBlock, falseStmt);
15326 printf("\nExpanding top-level qmark in BB%02u (after)\n", block->bbNum);
15327 fgDispBasicBlocks(block, remainderBlock, true);
15332 /*****************************************************************************
15334 * Expand GT_QMARK nodes from the flow graph into basic blocks.
15338 void Compiler::fgExpandQmarkNodes()
15342 for (BasicBlock* block = fgFirstBB; block; block = block->bbNext)
15344 for (GenTreePtr stmt = block->bbTreeList; stmt; stmt = stmt->gtNext)
15346 GenTreePtr expr = stmt->gtStmt.gtStmtExpr;
15348 fgPreExpandQmarkChecks(expr);
15350 fgExpandQmarkStmt(block, stmt);
15354 fgPostExpandQmarkChecks();
15357 compQmarkRationalized = true;
15361 /*****************************************************************************
15363 * Make sure we don't have any more GT_QMARK nodes.
15366 void Compiler::fgPostExpandQmarkChecks()
15368 for (BasicBlock* block = fgFirstBB; block; block = block->bbNext)
15370 for (GenTreePtr stmt = block->bbTreeList; stmt; stmt = stmt->gtNext)
15372 GenTreePtr expr = stmt->gtStmt.gtStmtExpr;
15373 fgWalkTreePre(&expr, Compiler::fgAssertNoQmark, NULL);
15379 /*****************************************************************************
15381 * Transform all basic blocks for codegen.
15384 void Compiler::fgMorph()
15386 noway_assert(!compIsForInlining()); // Inlinee's compiler should never reach here.
15388 fgOutgoingArgTemps = nullptr;
15392 printf("*************** In fgMorph()\n");
15394 fgDispBasicBlocks(true);
15397 // Insert call to class constructor as the first basic block if
15398 // we were asked to do so.
15399 if (info.compCompHnd->initClass(NULL /* field */, info.compMethodHnd /* method */,
15400 impTokenLookupContextHandle /* context */) & CORINFO_INITCLASS_USE_HELPER)
15402 fgEnsureFirstBBisScratch();
15403 fgInsertStmtAtBeg(fgFirstBB, fgInitThisClass());
15407 if (opts.compGcChecks)
15409 for (unsigned i = 0; i < info.compArgsCount; i++)
15411 if (lvaTable[i].TypeGet() == TYP_REF)
15413 // confirm that the argument is a GC pointer (for debugging (GC stress))
15414 GenTreePtr op = gtNewLclvNode(i, TYP_REF);
15415 GenTreeArgList* args = gtNewArgList(op);
15416 op = gtNewHelperCallNode(CORINFO_HELP_CHECK_OBJ, TYP_VOID, 0, args);
15418 fgEnsureFirstBBisScratch();
15419 fgInsertStmtAtEnd(fgFirstBB, op);
15424 if (opts.compStackCheckOnRet)
15426 lvaReturnEspCheck = lvaGrabTempWithImplicitUse(false DEBUGARG("ReturnEspCheck"));
15427 lvaTable[lvaReturnEspCheck].lvType = TYP_INT;
15430 if (opts.compStackCheckOnCall)
15432 lvaCallEspCheck = lvaGrabTempWithImplicitUse(false DEBUGARG("CallEspCheck"));
15433 lvaTable[lvaCallEspCheck].lvType = TYP_INT;
15437 /* Filter out unimported BBs */
15439 fgRemoveEmptyBlocks();
15441 /* Add any internal blocks/trees we may need */
15446 fgMultipleNots = false;
15450 /* Inliner could add basic blocks. Check that the flowgraph data is up-to-date */
15451 fgDebugCheckBBlist(false, false);
15457 JITDUMP("trees after inlining\n");
15458 DBEXEC(VERBOSE, fgDispBasicBlocks(true));
15461 RecordStateAtEndOfInlining(); // Record "start" values for post-inlining cycles and elapsed time.
15464 /* Inliner could add basic blocks. Check that the flowgraph data is up-to-date */
15465 fgDebugCheckBBlist(false, false);
15468 /* For x64 and ARM64 we need to mark irregular parameters early so that they don't get promoted */
15469 fgMarkImplicitByRefArgs();
15471 /* Promote struct locals if necessary */
15472 fgPromoteStructs();
15474 /* Now it is the time to figure out what locals have address-taken. */
15475 fgMarkAddressExposedLocals();
15478 /* Now that locals have address-taken marked, we can safely apply stress. */
15480 fgStress64RsltMul();
15483 /* Morph the trees in all the blocks of the method */
15488 JITDUMP("trees after fgMorphBlocks\n");
15489 DBEXEC(VERBOSE, fgDispBasicBlocks(true));
15492 /* Decide the kind of code we want to generate */
15496 fgExpandQmarkNodes();
15504 /*****************************************************************************
15506 * Promoting struct locals
15508 void Compiler::fgPromoteStructs()
15512 printf("*************** In fgPromoteStructs()\n");
15515 if (!opts.OptEnabled(CLFLG_STRUCTPROMOTE))
15518 if (fgNoStructPromotion)
15522 // The code in this #if has been useful in debugging struct promotion issues, by
15523 // enabling selective enablement of the struct promotion optimization according to
15526 unsigned methHash = info.compMethodHash();
15527 char* lostr = getenv("structpromohashlo");
15528 unsigned methHashLo = 0;
15531 sscanf_s(lostr, "%x", &methHashLo);
15533 char* histr = getenv("structpromohashhi");
15534 unsigned methHashHi = UINT32_MAX;
15537 sscanf_s(histr, "%x", &methHashHi);
15539 if (methHash < methHashLo || methHash > methHashHi)
15545 printf("Promoting structs for method %s, hash = 0x%x.\n",
15546 info.compFullName, info.compMethodHash());
15547 printf(""); // in our logic this causes a flush
15552 if (info.compIsVarArgs)
15555 if (getNeedsGSSecurityCookie())
15558 // The lvaTable might grow as we grab temps. Make a local copy here.
15560 unsigned startLvaCount = lvaCount;
15563 // Loop through the original lvaTable. Looking for struct locals to be promoted.
15566 lvaStructPromotionInfo structPromotionInfo;
15567 bool tooManyLocals = false;
15569 for (unsigned lclNum = 0;
15570 lclNum < startLvaCount;
15573 // Whether this var got promoted
15574 bool promotedVar = false;
15575 LclVarDsc* varDsc = &lvaTable[lclNum];
15577 #ifdef FEATURE_SIMD
15578 if (varDsc->lvSIMDType && varDsc->lvUsedInSIMDIntrinsic)
15580 // If we have marked this as lvUsedInSIMDIntrinsic, then we do not want to promote
15581 // its fields. Instead, we will attempt to enregister the entire struct.
15582 varDsc->lvRegStruct = true;
15585 #endif //FEATURE_SIMD
15586 // Don't promote if we have reached the tracking limit.
15587 if (lvaHaveManyLocals())
15589 // Print the message first time when we detected this condition
15590 if (!tooManyLocals)
15592 JITDUMP("Stopped promoting struct fields, due to too many locals.\n");
15594 tooManyLocals = true;
15596 #if !FEATURE_MULTIREG_STRUCT_PROMOTE
15597 else if (varDsc->lvIsMultiRegArgOrRet)
15599 JITDUMP("Skipping V%02u: marked lvIsMultiRegArgOrRet.\n", lclNum);
15601 #endif // !FEATURE_MULTIREG_STRUCT_PROMOTE
15602 else if (varTypeIsStruct(varDsc))
15604 lvaCanPromoteStructVar(lclNum, &structPromotionInfo);
15605 bool canPromote = structPromotionInfo.canPromote;
15607 // We start off with shouldPromote same as canPromote.
15608 // Based on further profitablity checks done below, shouldPromote
15609 // could be set to false.
15610 bool shouldPromote = canPromote;
15615 // We *can* promote; *should* we promote?
15616 // We should only do so if promotion has potential savings. One source of savings
15617 // is if a field of the struct is accessed, since this access will be turned into
15618 // an access of the corresponding promoted field variable. Even if there are no
15619 // field accesses, but only block-level operations on the whole struct, if the struct
15620 // has only one or two fields, then doing those block operations field-wise is probably faster
15621 // than doing a whole-variable block operation (e.g., a hardware "copy loop" on x86).
15622 // So if no fields are accessed independently, and there are three or more fields,
15623 // then do not promote.
15624 if (structPromotionInfo.fieldCnt > 2 && !varDsc->lvFieldAccessed)
15626 JITDUMP("Not promoting promotable struct local V%02u: #fields = %d, fieldAccessed = %d.\n",
15627 lclNum, structPromotionInfo.fieldCnt, varDsc->lvFieldAccessed);
15628 shouldPromote = false;
15630 #if defined(_TARGET_AMD64_) || defined(_TARGET_ARM64_)
15631 // TODO-PERF - Only do this when the LclVar is used in an argument context
15632 // TODO-ARM64 - HFA support should also eliminate the need for this.
15633 // TODO-LSRA - Currently doesn't support the passing of floating point LCL_VARS in the integer registers
15635 // For now we currently don't promote structs with a single float field
15636 // Promoting it can cause us to shuffle it back and forth between the int and
15637 // the float regs when it is used as a argument, which is very expensive for XARCH
15639 else if ((structPromotionInfo.fieldCnt == 1) &&
15640 varTypeIsFloating(structPromotionInfo.fields[0].fldType))
15642 JITDUMP("Not promoting promotable struct local V%02u: #fields = %d because it is a struct with single float field.\n",
15643 lclNum, structPromotionInfo.fieldCnt);
15644 shouldPromote = false;
15646 #endif // _TARGET_AMD64_ || _TARGET_ARM64_
15648 #if !FEATURE_MULTIREG_STRUCT_PROMOTE
15649 #if defined(_TARGET_ARM64_)
15651 // For now we currently don't promote structs that could be passed in registers
15653 else if (varDsc->lvIsMultiregStruct())
15655 JITDUMP("Not promoting promotable struct local V%02u (size==%d): ",
15656 lclNum, lvaLclExactSize(lclNum));
15657 shouldPromote = false;
15659 #endif // _TARGET_ARM64_
15660 #endif // !FEATURE_MULTIREG_STRUCT_PROMOTE
15661 else if (varDsc->lvIsParam)
15663 #if FEATURE_MULTIREG_STRUCT_PROMOTE
15664 if (varDsc->lvIsMultiregStruct() && // Is this a variable holding a value that is passed in multiple registers?
15665 (structPromotionInfo.fieldCnt != 2)) // Does it have exactly two fields
15667 JITDUMP("Not promoting multireg struct local V%02u, because lvIsParam is true and #fields != 2\n",
15669 shouldPromote = false;
15672 #endif // !FEATURE_MULTIREG_STRUCT_PROMOTE
15674 // TODO-PERF - Implement struct promotion for incoming multireg structs
15675 // Currently it hits assert(lvFieldCnt==1) in lclvar.cpp line 4417
15677 if (structPromotionInfo.fieldCnt != 1)
15679 JITDUMP("Not promoting promotable struct local V%02u, because lvIsParam is true and #fields = %d.\n",
15680 lclNum, structPromotionInfo.fieldCnt);
15681 shouldPromote = false;
15686 // If the lvRefCnt is zero and we have a struct promoted parameter we can end up with an extra store of the the
15687 // incoming register into the stack frame slot.
15688 // In that case, we would like to avoid promortion.
15689 // However we haven't yet computed the lvRefCnt values so we can't do that.
15693 // Often-useful debugging code: if you've narrowed down a struct-promotion problem to a single
15694 // method, this allows you to select a subset of the vars to promote (by 1-based ordinal number).
15695 static int structPromoVarNum = 0;
15696 structPromoVarNum++;
15697 if (atoi(getenv("structpromovarnumlo")) <= structPromoVarNum && structPromoVarNum <= atoi(getenv("structpromovarnumhi")))
15702 assert(canPromote);
15704 // Promote the this struct local var.
15705 lvaPromoteStructVar(lclNum, &structPromotionInfo);
15706 promotedVar = true;
15708 #ifdef _TARGET_ARM_
15709 if (structPromotionInfo.requiresScratchVar)
15711 // Ensure that the scratch variable is allocated, in case we
15712 // pass a promoted struct as an argument.
15713 if (lvaPromotedStructAssemblyScratchVar == BAD_VAR_NUM)
15715 lvaPromotedStructAssemblyScratchVar =
15716 lvaGrabTempWithImplicitUse(false DEBUGARG("promoted struct assembly scratch var."));
15717 lvaTable[lvaPromotedStructAssemblyScratchVar].lvType = TYP_I_IMPL;
15720 #endif // _TARGET_ARM_
15725 #ifdef FEATURE_SIMD
15726 if (!promotedVar && varDsc->lvSIMDType && !varDsc->lvFieldAccessed)
15728 // Even if we have not used this in a SIMD intrinsic, if it is not being promoted,
15729 // we will treat it as a reg struct.
15730 varDsc->lvRegStruct = true;
15732 #endif // FEATURE_SIMD
15738 Compiler::fgWalkResult Compiler::fgMorphStructField(GenTreePtr tree, fgWalkData* fgWalkPre)
15740 noway_assert(tree->OperGet() == GT_FIELD);
15741 noway_assert(tree->gtFlags & GTF_GLOB_REF);
15743 GenTreePtr objRef = tree->gtField.gtFldObj;
15745 /* Is this an instance data member? */
15749 if (objRef->gtOper == GT_ADDR)
15751 GenTreePtr obj = objRef->gtOp.gtOp1;
15753 if (obj->gtOper == GT_LCL_VAR)
15755 unsigned lclNum = obj->gtLclVarCommon.gtLclNum;
15756 LclVarDsc* varDsc = &lvaTable[lclNum];
15758 if (varTypeIsStruct(obj))
15760 if (varDsc->lvPromoted)
15763 unsigned fldOffset = tree->gtField.gtFldOffset;
15764 unsigned fieldLclIndex = lvaGetFieldLocal(varDsc, fldOffset);
15765 noway_assert(fieldLclIndex != BAD_VAR_NUM);
15767 tree->SetOper(GT_LCL_VAR);
15768 tree->gtLclVarCommon.SetLclNum(fieldLclIndex);
15769 tree->gtType = lvaTable[fieldLclIndex].TypeGet();
15770 tree->gtFlags &= GTF_NODE_MASK;
15771 tree->gtFlags &= ~GTF_GLOB_REF;
15773 GenTreePtr parent = fgWalkPre->parentStack->Index(1);
15774 if ((parent->gtOper == GT_ASG) && (parent->gtOp.gtOp1 == tree))
15776 tree->gtFlags |= GTF_VAR_DEF;
15777 tree->gtFlags |= GTF_DONT_CSE;
15782 printf("Replacing the field in promoted struct with a local var:\n");
15783 fgWalkPre->printModified = true;
15786 return WALK_SKIP_SUBTREES;
15792 // A "normed struct" is a struct that the VM tells us is a basic type. This can only happen if
15793 // the struct contains a single element, and that element is 4 bytes (on x64 it can also be 8 bytes).
15794 // Normally, the type of the local var and the type of GT_FIELD are equivalent. However, there
15795 // is one extremely rare case where that won't be true. An enum type is a special value type
15796 // that contains exactly one element of a primitive integer type (that, for CLS programs is named "value__").
15797 // The VM tells us that a local var of that enum type is the primitive type of the enum's single field.
15798 // It turns out that it is legal for IL to access this field using ldflda or ldfld. For example:
15800 // .class public auto ansi sealed mynamespace.e_t extends [mscorlib]System.Enum
15802 // .field public specialname rtspecialname int16 value__
15803 // .field public static literal valuetype mynamespace.e_t one = int16(0x0000)
15805 // .method public hidebysig static void Main() cil managed
15807 // .locals init (valuetype mynamespace.e_t V_0)
15810 // ldflda int16 mynamespace.e_t::value__
15814 // Normally, compilers will not generate the ldflda, since it is superfluous.
15816 // In the example, the lclVar is short, but the JIT promotes all trees using this local to the
15817 // "actual type", that is, INT. But the GT_FIELD is still SHORT. So, in the case of a type
15818 // mismatch like this, don't do this morphing. The local var may end up getting marked as
15819 // address taken, and the appropriate SHORT load will be done from memory in that case.
15821 if (tree->TypeGet() == obj->TypeGet())
15823 tree->ChangeOper(GT_LCL_VAR);
15824 tree->gtLclVarCommon.SetLclNum(lclNum);
15825 tree->gtFlags &= GTF_NODE_MASK;
15827 GenTreePtr parent = fgWalkPre->parentStack->Index(1);
15828 if ((parent->gtOper == GT_ASG) && (parent->gtOp.gtOp1 == tree))
15830 tree->gtFlags |= GTF_VAR_DEF;
15831 tree->gtFlags |= GTF_DONT_CSE;
15836 printf("Replacing the field in normed struct with the local var:\n");
15837 fgWalkPre->printModified = true;
15840 return WALK_SKIP_SUBTREES;
15847 return WALK_CONTINUE;
15850 Compiler::fgWalkResult Compiler::fgMorphLocalField(GenTreePtr tree, fgWalkData* fgWalkPre)
15852 noway_assert(tree->OperGet() == GT_LCL_FLD);
15854 unsigned lclNum = tree->gtLclFld.gtLclNum;
15855 LclVarDsc* varDsc = &lvaTable[lclNum];
15857 if (varTypeIsStruct(varDsc) && (varDsc->lvPromoted))
15860 unsigned fldOffset = tree->gtLclFld.gtLclOffs;
15861 unsigned fieldLclIndex = 0;
15862 LclVarDsc* fldVarDsc = NULL;
15864 if (fldOffset != BAD_VAR_NUM)
15866 fieldLclIndex = lvaGetFieldLocal(varDsc, fldOffset);
15867 noway_assert(fieldLclIndex != BAD_VAR_NUM);
15868 fldVarDsc = &lvaTable[fieldLclIndex];
15871 if (fldOffset != BAD_VAR_NUM && genTypeSize(fldVarDsc->TypeGet()) == genTypeSize(tree->gtType)
15872 #ifdef _TARGET_X86_
15873 && varTypeIsFloating(fldVarDsc->TypeGet()) == varTypeIsFloating(tree->gtType)
15877 // There is an existing sub-field we can use
15878 tree->gtLclFld.SetLclNum(fieldLclIndex);
15880 // We need to keep the types 'compatible'. If we can switch back to a GT_LCL_VAR
15881 assert(varTypeIsIntegralOrI(tree->TypeGet()));
15882 if (varTypeCanReg(fldVarDsc->TypeGet()))
15884 // If the type is integer-ish, then we can use it as-is
15885 tree->ChangeOper(GT_LCL_VAR);
15886 assert(tree->gtLclVarCommon.gtLclNum == fieldLclIndex);
15887 tree->gtType = fldVarDsc->TypeGet();
15891 printf("Replacing the GT_LCL_FLD in promoted struct with a local var:\n");
15892 fgWalkPre->printModified = true;
15897 GenTreePtr parent = fgWalkPre->parentStack->Index(1);
15898 if ((parent->gtOper == GT_ASG) && (parent->gtOp.gtOp1 == tree))
15900 tree->gtFlags |= GTF_VAR_DEF;
15901 tree->gtFlags |= GTF_DONT_CSE;
15906 // There is no existing field that has all the parts that we need
15907 // So we must ensure that the struct lives in memory.
15908 lvaSetVarDoNotEnregister(lclNum DEBUGARG(DNER_LocalField));
15911 // We can't convert this guy to a float because he really does have his
15913 varDsc->lvKeepType = 1;
15917 return WALK_SKIP_SUBTREES;
15920 return WALK_CONTINUE;
15923 /*****************************************************************************
15925 * Mark irregular parameters. For x64 this is 3, 5, 6, 7, >8 byte structs that are passed by reference.
15926 * For ARM64, this is structs larger than 16 bytes that are passed by reference.
15928 void Compiler::fgMarkImplicitByRefArgs()
15930 #if defined(_TARGET_AMD64_) || defined(_TARGET_ARM64_)
15933 printf("\n*************** In fgMarkImplicitByRefs()\n");
15936 for (unsigned lclNum = 0; lclNum < lvaCount; lclNum++)
15938 LclVarDsc* varDsc = &lvaTable[lclNum];
15940 assert(!varDsc->lvPromoted); // Called in the wrong order?
15942 if (varDsc->lvIsParam && varTypeIsStruct(varDsc))
15946 if (varDsc->lvSize() > REGSIZE_BYTES)
15948 size = varDsc->lvSize();
15952 CORINFO_CLASS_HANDLE typeHnd = varDsc->lvVerTypeInfo.GetClassHandle();
15953 size = info.compCompHnd->getClassSize(typeHnd);
15957 #if !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
15958 #if defined(_TARGET_AMD64_)
15959 if (size > REGSIZE_BYTES || (size & (size - 1)) != 0)
15960 #elif defined(_TARGET_ARM64_)
15964 // Previously nobody was ever setting lvIsParam and lvIsTemp on the same local
15965 // So I am now using it to indicate that this is one of the weird implicit
15967 // The address taken cleanup will look for references to locals marked like
15968 // this, and transform them appropriately.
15969 varDsc->lvIsTemp = 1;
15971 // Also marking them as BYREF will hide them from struct promotion.
15972 varDsc->lvType = TYP_BYREF;
15974 varDsc->lvRefCnt = 0;
15976 // Since this previously was a TYP_STRUCT and we have changed it to a TYP_BYREF
15977 // make sure that the following flag is not set as these will force SSA to
15978 // exclude tracking/enregistering these LclVars. (see fgExcludeFromSsa)
15980 varDsc->lvOverlappingFields = 0; // This flag could have been set, clear it.
15983 // This should not be converted to a double in stress mode,
15984 // because it is really a pointer
15985 varDsc->lvKeepType = 1;
15988 #endif // !FEATURE_UNIX_AMD64_STRUCT_PASSING
15992 #endif // _TARGET_AMD64_ || _TARGET_ARM64_
15995 /*****************************************************************************
15997 * Morph irregular parameters
15998 * for x64 and ARM64 this means turning them into byrefs, adding extra indirs.
16000 bool Compiler::fgMorphImplicitByRefArgs(GenTreePtr tree, fgWalkData* fgWalkPre)
16002 #if !defined(_TARGET_AMD64_) && !defined(_TARGET_ARM64_)
16006 #else // _TARGET_AMD64_ || _TARGET_ARM64_
16008 assert((tree->gtOper == GT_LCL_VAR) ||
16009 ((tree->gtOper == GT_ADDR) && (tree->gtOp.gtOp1->gtOper == GT_LCL_VAR)));
16011 bool isAddr = (tree->gtOper == GT_ADDR);
16012 GenTreePtr lclVarTree = isAddr ? tree->gtOp.gtOp1 : tree;
16013 unsigned lclNum = lclVarTree->gtLclVarCommon.gtLclNum;
16014 LclVarDsc* lclVarDsc = &lvaTable[lclNum];
16016 if (!lvaIsImplicitByRefLocal(lclNum))
16018 // We only need to tranform the 'marked' implicit by ref parameters
16022 // The SIMD transformation to coalesce contiguous references to SIMD vector fields will
16023 // re-invoke the traversal to mark address-taken locals.
16024 // So, we may encounter a tree that has already been transformed to TYP_BYREF.
16025 // If we do, leave it as-is.
16026 if (!varTypeIsStruct(lclVarTree))
16028 assert(lclVarTree->TypeGet() == TYP_BYREF);
16032 // We are overloading the lvRefCnt field here because real ref counts have not been set.
16033 lclVarDsc->lvRefCnt++;
16037 // change &X into just plain X
16038 tree->CopyFrom(lclVarTree, this);
16039 tree->gtType = TYP_BYREF;
16044 printf("Replacing address of implicit by ref struct parameter with byref:\n");
16045 fgWalkPre->printModified = true;
16051 // Change X into *X
16052 // First, save the original type, then change the tree type to TYP_BYREF (otherwise we
16053 // will get an assert when we try to clone the lclVar node because the lclVar is now TYP_BYREF
16054 // and the types have to match). The reason we clone the lclVar is that we don't pass a
16055 // possible-modified tree back to the caller, so we modify the original lclVar node in-place
16057 var_types structType = tree->gtType;
16058 lclVarTree = gtClone(tree);
16059 // Now, set the types appropriately.
16060 lclVarTree->gtType = TYP_BYREF;
16061 tree->gtType = structType;
16062 // Now, "insert" the GT_IND by changing the oper of the original node and setting its op1.
16063 tree->SetOper(GT_IND);
16064 tree->gtOp.gtOp1 = lclVarTree;
16065 // TODO-CQ: If the VM ever stops violating the ABI and passing heap references
16066 // we could remove TGTANYWHERE
16067 tree->gtFlags = ((tree->gtFlags & GTF_COMMON_MASK) | GTF_IND_TGTANYWHERE);
16072 printf("Replacing value of implicit by ref struct parameter with indir of parameter:\n");
16073 gtDispTree(tree, nullptr, nullptr, true);
16074 fgWalkPre->printModified = true;
16081 #endif // _TARGET_AMD64_ || _TARGET_ARM64_
16086 // An "AddrExposedContext" expresses the calling context in which an address expression occurs.
16087 enum AddrExposedContext
16089 AXC_None, // None of the below seen yet.
16090 AXC_Ind, // The address being computed is to be dereferenced.
16091 AXC_Addr, // We're computing a raw address (not dereferenced, at least not immediately).
16092 AXC_IndWide, // A block operation dereferenced an address referencing more bytes than the address
16093 // addresses -- if the address addresses a field of a struct local, we need to consider
16094 // the entire local address taken (not just the field).
16095 AXC_AddrWide, // The address being computed will be dereferenced by a block operation that operates
16096 // on more bytes than the width of the storage location addressed. If this is a
16097 // field of a promoted struct local, declare the entire struct local address-taken.
16098 AXC_InitBlk, // An GT_INITBLK is the immediate parent. The first argument is in an IND context.
16099 AXC_CopyBlk, // An GT_COPYBLK is the immediate parent. The first argument is in a GT_LIST, whose
16100 // args should be evaluated in an IND context.
16101 AXC_IndAdd, // A GT_ADD is the immediate parent, and it was evaluated in an IND contxt.
16102 // If one arg is a constant int, evaluate the other in an IND context. Otherwise, none.
16105 typedef ArrayStack<AddrExposedContext> AXCStack;
16107 // We use pre-post to simulate passing an argument in a recursion, via a stack.
16108 Compiler::fgWalkResult Compiler::fgMarkAddrTakenLocalsPostCB(GenTreePtr* pTree,
16109 fgWalkData* fgWalkPre)
16111 AXCStack* axcStack = reinterpret_cast<AXCStack*>(fgWalkPre->pCallbackData);
16112 (void)axcStack->Pop();
16113 return WALK_CONTINUE;
16116 Compiler::fgWalkResult Compiler::fgMarkAddrTakenLocalsPreCB(GenTreePtr* pTree,
16117 fgWalkData* fgWalkPre)
16119 GenTreePtr tree = *pTree;
16120 Compiler* comp = fgWalkPre->compiler;
16121 AXCStack* axcStack = reinterpret_cast<AXCStack*>(fgWalkPre->pCallbackData);
16122 AddrExposedContext axc = axcStack->Top();
16124 // In some situations, we have to figure out what the effective context is in which to
16125 // evaluate the current tree, depending on which argument position it is in its parent.
16127 // If the parent was an initblock, and this is its first argument, we're in
16128 // and "ind" context.
16134 // In both cases, the second argument is an integer struct size. That should have a "none" context.
16135 // The first argument is a GT_LIST. For GT_COPYBLK, both args of the list are addresses
16136 // that are dereferenced; for GT_INITBLK, the first is. We pass "axc" to the GT_LIST;
16137 // which will pass it to its arguments; these will decide whether they're in an Ind context
16138 // depending on "axc" and which argument they are.
16139 // A GT_INITBLK's first argument is a GT_LIST, whose first argument is an address
16140 // that should be considered to be dereferenced, and whose second argument the integer
16141 // (byte) value to fill the block with. The second argument of the GT_INITBLK is also
16142 // an integer, the block size.
16143 GenTreePtr parent = fgWalkPre->parentStack->Index(1);
16144 if (parent->gtOp.gtOp2 == tree &&
16145 parent->OperIsBlkOp())
16149 else if (parent->OperGet() == GT_LIST)
16151 genTreeOps axcOper = fgWalkPre->parentStack->Index(2)->OperGet();
16152 assert((axc == AXC_InitBlk && axcOper == GT_INITBLK) ||
16153 (axc == AXC_CopyBlk && GenTree::OperIsCopyBlkOp(axcOper)));
16155 // The block operation will derefence its argument(s) -- usually. If the size of the initblk
16156 // or copyblk exceeds the size of a storage location whose address is used as one of the
16157 // arguments, then we have to consider that storage location (indeed, it's underlying containing
16158 // location) to be address taken. So get the width of the initblk or copyblk.
16159 GenTreePtr widthNode = fgWalkPre->parentStack->Index(2)->gtOp.gtOp2;
16160 unsigned width = UINT_MAX; // If it's not a constant, assume it's maximally big.
16161 if (widthNode->IsCnsIntOrI())
16163 if (widthNode->IsIconHandle())
16165 // If it's a handle, it must be a class handle. We only create such block operations
16166 // for initialization of struct types, so the type of the argument(s) will match this
16167 // type, by construction. Set the width to zero to make sure nothing fits in it.
16168 assert(widthNode->IsIconHandle(GTF_ICON_CLASS_HDL));
16173 ssize_t swidth = widthNode->gtIntConCommon.IconValue();
16174 assert(swidth > 0); // Well-formedness of the block operation node...
16175 width = unsigned(swidth);
16179 if (parent->gtOp.gtOp1 == tree)
16181 // First argument is (potentially) dereferenced by both kinds of block operations.
16182 if (tree->OperGet() == GT_ADDR && !comp->fgFitsInOrNotLoc(tree->gtOp.gtOp1, width))
16191 else if (axc == AXC_CopyBlk)
16193 assert(parent->gtOp.gtOp2 == tree);
16194 if (tree->OperGet() == GT_ADDR && !comp->fgFitsInOrNotLoc(tree->gtOp.gtOp1, width))
16213 GenTreePtr parent = fgWalkPre->parentStack->Index(1);
16214 assert(parent->OperGet() == GT_ADD);
16215 // Is one of the args a constant representing a field offset,
16216 // and is this the other? If so, Ind context.
16217 if (parent->gtOp.gtOp1->IsCnsIntOrI() && parent->gtOp.gtOp2 == tree)
16221 else if (parent->gtOp.gtOp2->IsCnsIntOrI() && parent->gtOp.gtOp1 == tree)
16236 // Now recurse properly for the tree.
16237 switch (tree->gtOper)
16241 if (axc != AXC_Addr)
16243 axcStack->Push(AXC_Ind);
16247 axcStack->Push(AXC_None);
16249 return WALK_CONTINUE;
16252 axcStack->Push(AXC_InitBlk);
16253 return WALK_CONTINUE;
16257 axcStack->Push(AXC_CopyBlk);
16258 return WALK_CONTINUE;
16261 if (axc == AXC_InitBlk || axc == AXC_CopyBlk)
16263 axcStack->Push(axc);
16267 axcStack->Push(AXC_None);
16269 return WALK_CONTINUE;
16272 // Taking the address of an array element never takes the address of a local.
16273 axcStack->Push(AXC_None);
16274 return WALK_CONTINUE;
16277 // If we have ADDR(lcl), and "lcl" is an implicit byref parameter, fgMorphImplicitByRefArgs will
16278 // convert to just "lcl". This is never an address-context use, since the local is already a
16279 // byref after this transformation.
16280 if (tree->gtOp.gtOp1->OperGet() == GT_LCL_VAR && comp->fgMorphImplicitByRefArgs(tree, fgWalkPre))
16282 // Push something to keep the PostCB, which will pop it, happy.
16283 axcStack->Push(AXC_None);
16284 // In the first case, tree may no longer be a leaf, but we're done with it; is a leaf in the second case.
16285 return WALK_SKIP_SUBTREES;
16288 #ifdef FEATURE_SIMD
16289 if (tree->gtOp.gtOp1->OperGet() == GT_SIMD)
16291 axcStack->Push(AXC_None);
16294 #endif // FEATURE_SIMD
16295 if (axc == AXC_Ind)
16297 axcStack->Push(AXC_None);
16299 else if (axc == AXC_IndWide)
16301 axcStack->Push(AXC_AddrWide);
16305 assert(axc == AXC_None);
16306 axcStack->Push(AXC_Addr);
16308 return WALK_CONTINUE;
16311 // First, handle a couple of special cases: field of promoted struct local, field
16312 // of "normed" struct.
16313 if (comp->fgMorphStructField(tree, fgWalkPre) == WALK_SKIP_SUBTREES)
16315 // It (may have) replaced the field with a local var or local field. If we're in an addr context,
16316 // label it addr-taken.
16317 if (tree->OperIsLocal() && (axc == AXC_Addr || axc == AXC_AddrWide))
16319 unsigned lclNum = tree->gtLclVarCommon.gtLclNum;
16320 comp->lvaSetVarAddrExposed(lclNum);
16321 if (axc == AXC_AddrWide)
16323 LclVarDsc* varDsc = &comp->lvaTable[lclNum];
16324 if (varDsc->lvIsStructField)
16326 comp->lvaSetVarAddrExposed(varDsc->lvParentLcl);
16330 // Push something to keep the PostCB, which will pop it, happy.
16331 axcStack->Push(AXC_None);
16332 return WALK_SKIP_SUBTREES;
16336 // GT_FIELD is an implicit deref.
16337 if (axc == AXC_Addr)
16339 axcStack->Push(AXC_None);
16341 else if (axc == AXC_AddrWide)
16343 axcStack->Push(AXC_IndWide);
16347 axcStack->Push(AXC_Ind);
16349 return WALK_CONTINUE;
16354 assert(axc != AXC_Addr);
16355 // This recognizes certain forms, and does all the work. In that case, returns WALK_SKIP_SUBTREES,
16356 // else WALK_CONTINUE. We do the same here.
16357 fgWalkResult res = comp->fgMorphLocalField(tree, fgWalkPre);
16358 if (res == WALK_SKIP_SUBTREES && tree->OperGet() == GT_LCL_VAR && (axc == AXC_Addr || axc == AXC_AddrWide))
16360 unsigned lclNum = tree->gtLclVarCommon.gtLclNum;
16361 comp->lvaSetVarAddrExposed(lclNum);
16362 if (axc == AXC_AddrWide)
16364 LclVarDsc* varDsc = &comp->lvaTable[lclNum];
16365 if (varDsc->lvIsStructField)
16367 comp->lvaSetVarAddrExposed(varDsc->lvParentLcl);
16371 // Must push something; if res is WALK_SKIP_SUBTREES, doesn't matter
16372 // what, but something to be popped by the post callback. If we're going
16373 // to analyze children, the LCL_FLD creates an Ind context, so use that.
16374 axcStack->Push(AXC_Ind);
16379 // On some architectures, some arguments are passed implicitly by reference.
16380 // Modify the trees to reflect that, if this local is one of those.
16381 if (comp->fgMorphImplicitByRefArgs(tree, fgWalkPre))
16383 // We can't be in an address context; the ADDR(lcl), where lcl is an implicit byref param, was
16384 // handled earlier. (And we can't have added anything to this address, since it was implicit.)
16385 assert(axc != AXC_Addr);
16389 if (axc == AXC_Addr || axc == AXC_AddrWide)
16391 unsigned lclNum = tree->gtLclVarCommon.gtLclNum;
16392 comp->lvaSetVarAddrExposed(lclNum);
16393 if (axc == AXC_AddrWide)
16395 LclVarDsc* varDsc = &comp->lvaTable[lclNum];
16396 if (varDsc->lvIsStructField)
16398 comp->lvaSetVarAddrExposed(varDsc->lvParentLcl);
16402 // We may need to Quirk the storage size for this LCL_VAR
16403 // some PInvoke signatures incorrectly specify a ByRef to an INT32
16404 // when they actually write a SIZE_T or INT64
16405 if (axc == AXC_Addr)
16407 comp->gtCheckQuirkAddrExposedLclVar(tree, fgWalkPre->parentStack);
16411 // Push something to keep the PostCB, which will pop it, happy.
16412 axcStack->Push(AXC_None);
16413 // In the first case, tree may no longer be a leaf, but we're done with it; is a leaf in the second case.
16414 return WALK_SKIP_SUBTREES;
16417 assert(axc != AXC_Addr);
16418 // See below about treating pointer operations as wider indirection.
16419 if (tree->gtOp.gtOp1->gtType == TYP_BYREF || tree->gtOp.gtOp2->gtType == TYP_BYREF)
16421 axcStack->Push(AXC_IndWide);
16423 else if (axc == AXC_Ind)
16425 // Let the children know that the parent was a GT_ADD, to be evaluated in an IND context.
16426 // If it's an add of a constant and an address, and the constant represents a field,
16427 // then we'll evaluate the address argument in an Ind context; otherwise, the None context.
16428 axcStack->Push(AXC_IndAdd);
16432 axcStack->Push(axc);
16434 return WALK_CONTINUE;
16436 // !!! Treat Pointer Operations as Wider Indirection
16438 // If we are performing pointer operations, make sure we treat that as equivalent to a wider
16439 // indirection. This is because the pointers could be pointing to the address of struct fields
16440 // and could be used to perform operations on the whole struct or passed to another method.
16442 // When visiting a node in this pre-order walk, we do not know if we would in the future
16443 // encounter a GT_ADDR of a GT_FIELD below.
16445 // Note: GT_ADDR of a GT_FIELD is always a TYP_BYREF.
16446 // So let us be conservative and treat TYP_BYREF operations as AXC_IndWide and propagate a
16447 // wider indirection context down the expr tree.
16449 // Example, in unsafe code,
16451 // IL_000e 12 00 ldloca.s 0x0
16452 // IL_0010 7c 02 00 00 04 ldflda 0x4000002
16453 // IL_0015 12 00 ldloca.s 0x0
16454 // IL_0017 7c 01 00 00 04 ldflda 0x4000001
16457 // When visiting the GT_SUB node, if the types of either of the GT_SUB's operand are BYREF, then
16458 // consider GT_SUB to be equivalent of an AXC_IndWide.
16460 // Similarly for pointer comparisons and pointer escaping as integers through conversions, treat
16461 // them as AXC_IndWide.
16485 if ((tree->gtOp.gtOp1->gtType == TYP_BYREF) || (tree->OperIsBinary() && (tree->gtOp.gtOp2->gtType == TYP_BYREF)))
16487 axcStack->Push(AXC_IndWide);
16488 return WALK_CONTINUE;
16493 // To be safe/conservative: pass Addr through, but not Ind -- otherwise, revert to "None". We must
16494 // handle the "Ind" propogation explicitly above.
16495 if (axc == AXC_Addr || axc == AXC_AddrWide)
16497 axcStack->Push(axc);
16501 axcStack->Push(AXC_None);
16503 return WALK_CONTINUE;
16507 bool Compiler::fgFitsInOrNotLoc(GenTreePtr tree, unsigned width)
16509 if (tree->TypeGet() != TYP_STRUCT)
16511 return width <= genTypeSize(tree->TypeGet());
16513 else if (tree->OperGet() == GT_LCL_VAR)
16515 assert(tree->TypeGet() == TYP_STRUCT);
16516 unsigned lclNum = tree->gtLclVarCommon.gtLclNum;
16517 return width <= lvaTable[lclNum].lvExactSize;
16519 else if (tree->OperGet() == GT_FIELD)
16521 CORINFO_CLASS_HANDLE fldClass = info.compCompHnd->getFieldClass (tree->gtField.gtFldHnd);
16522 return width <= info.compCompHnd->getClassSize(fldClass);
16531 void Compiler::fgAddFieldSeqForZeroOffset(GenTreePtr op1, FieldSeqNode* fieldSeq)
16533 assert(op1->TypeGet() == TYP_BYREF || op1->TypeGet() == TYP_I_IMPL || op1->TypeGet() == TYP_REF);
16535 switch (op1->OperGet())
16538 if (op1->gtOp.gtOp1->OperGet() == GT_LCL_FLD)
16540 GenTreeLclFld* lclFld = op1->gtOp.gtOp1->AsLclFld();
16541 lclFld->gtFieldSeq = GetFieldSeqStore()->Append(lclFld->gtFieldSeq, fieldSeq);
16546 if (op1->gtOp.gtOp1->OperGet() == GT_CNS_INT)
16548 FieldSeqNode* op1Fs = op1->gtOp.gtOp1->gtIntCon.gtFieldSeq;
16551 op1Fs = GetFieldSeqStore()->Append(op1Fs, fieldSeq);
16552 op1->gtOp.gtOp1->gtIntCon.gtFieldSeq = op1Fs;
16555 else if (op1->gtOp.gtOp2->OperGet() == GT_CNS_INT)
16557 FieldSeqNode* op2Fs = op1->gtOp.gtOp2->gtIntCon.gtFieldSeq;
16560 op2Fs = GetFieldSeqStore()->Append(op2Fs, fieldSeq);
16561 op1->gtOp.gtOp2->gtIntCon.gtFieldSeq = op2Fs;
16568 FieldSeqNode* op1Fs = op1->gtIntCon.gtFieldSeq;
16571 op1Fs = GetFieldSeqStore()->Append(op1Fs, fieldSeq);
16572 op1->gtIntCon.gtFieldSeq = op1Fs;
16578 // Record in the general zero-offset map.
16579 GetZeroOffsetFieldMap()->Set(op1, fieldSeq);
16584 /*****************************************************************************
16586 * Mark address-taken locals.
16589 void Compiler::fgMarkAddressExposedLocals()
16593 printf("\n*************** In fgMarkAddressExposedLocals()\n");
16596 BasicBlock* block = fgFirstBB;
16597 noway_assert(block);
16601 /* Make the current basic block address available globally */
16607 for (stmt = block->bbTreeList;
16609 stmt = stmt->gtNext)
16611 // Call Compiler::fgMarkAddrTakenLocalsCB on each node
16612 AXCStack stk(this);
16613 stk.Push(AXC_None); // We start in neither an addr or ind context.
16614 fgWalkTree(&stmt->gtStmt.gtStmtExpr,
16615 fgMarkAddrTakenLocalsPreCB,
16616 fgMarkAddrTakenLocalsPostCB,
16620 block = block->bbNext;
16626 // fgNodesMayInterfere:
16627 // return true if moving nodes relative to each other can change the result of a computation
16630 // read: a node which reads
16633 bool Compiler::fgNodesMayInterfere(GenTree* write, GenTree* read)
16635 LclVarDsc* srcVar = nullptr;
16636 bool srcAliased = false;
16637 bool dstAliased = false;
16639 bool readIsIndir = read->OperIsIndir() || read->OperIsImplicitIndir();
16640 bool writeIsIndir = write->OperIsIndir() || write->OperIsImplicitIndir();
16642 if (read->OperIsLocal())
16643 srcVar = &lvaTable[read->gtLclVarCommon.gtLclNum];
16647 if (srcVar && srcVar->lvAddrExposed)
16649 else if (readIsIndir)
16653 else if (write->OperIsLocal())
16655 LclVarDsc* dstVar = &lvaTable[write->gtLclVarCommon.gtLclNum];
16658 return dstVar->lvAddrExposed;
16660 else if (read->OperIsLocal())
16662 if (read->gtLclVarCommon.gtLclNum == write->gtLclVarCommon.gtLclNum)
16677 /** This predicate decides whether we will fold a tree with the structure:
16678 * x = x <op> y where x could be any arbitrary expression into
16681 * This modification is only performed when the target architecture supports
16682 * complex addressing modes. In the case of ARM for example, this transformation
16683 * yields no benefit.
16685 * In case this functions decides we can proceed to fold into an assignment operator
16686 * we need to inspect whether the operator is commutative to tell fgMorph whether we need to
16687 * reverse the tree due to the fact we saw x = y <op> x and we want to fold that into
16688 * x <op>= y because the operator property.
16690 bool Compiler::fgShouldCreateAssignOp(GenTreePtr tree, bool* bReverse)
16692 #if CPU_LOAD_STORE_ARCH
16693 /* In the case of a load/store architecture, there's no gain by doing any of this, we bail. */
16695 #elif !defined(LEGACY_BACKEND)
16697 #else // defined(LEGACY_BACKEND)
16700 GenTreePtr op1 = tree->gtOp.gtOp1;
16701 GenTreePtr op2 = tree->gtGetOp2();
16702 genTreeOps cmop = op2->OperGet();
16704 /* Is the destination identical to the first RHS sub-operand? */
16705 if (GenTree::Compare(op1, op2->gtOp.gtOp1))
16708 Do not transform the following tree
16710 [0024CFA4] ----------- const int 1
16711 [0024CFDC] ----G------ | int
16712 [0024CF5C] ----------- lclVar ubyte V01 tmp0
16713 [0024D05C] -A--G------ = ubyte
16714 [0024D014] D------N--- lclVar ubyte V01 tmp0
16718 [0024CFA4] ----------- const int 1
16719 [0024D05C] -A--G------ |= ubyte
16720 [0024D014] U------N--- lclVar ubyte V01 tmp0
16722 , when V01 is a struct field local.
16725 if (op1->gtOper == GT_LCL_VAR &&
16726 varTypeIsSmall(op1->TypeGet()) &&
16727 op1->TypeGet() != op2->gtOp.gtOp2->TypeGet())
16729 unsigned lclNum = op1->gtLclVarCommon.gtLclNum;
16730 LclVarDsc* varDsc = lvaTable + lclNum;
16732 if (varDsc->lvIsStructField)
16741 else if (GenTree::OperIsCommutative(cmop))
16743 /* For commutative ops only, check for "a = x <op> a" */
16745 /* Should we be doing this at all? */
16746 if ((opts.compFlags & CLFLG_TREETRANS) == 0)
16751 /* Can we swap the operands to cmop ... */
16752 if ((op2->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT) &&
16753 (op2->gtOp.gtOp2->gtFlags & GTF_ALL_EFFECT) )
16755 // Both sides must have side effects to prevent swap */
16759 /* Is the destination identical to the second RHS sub-operand? */
16760 if (GenTree::Compare(op1, op2->gtOp.gtOp2))
16767 #endif // defined(LEGACY_BACKEND)
16770 // Static variables.
16771 Compiler::MorphAddrContext Compiler::s_CopyBlockMAC(Compiler::MACK_CopyBlock);
16773 #ifdef FEATURE_SIMD
16775 //-----------------------------------------------------------------------------------
16776 // fgMorphCombineSIMDFieldAssignments:
16777 // If the RHS of the input stmt is a read for simd vector X Field, then this function
16778 // will keep reading next few stmts based on the vector size(2, 3, 4).
16779 // If the next stmts LHS are located contiguous and RHS are also located
16780 // contiguous, then we replace those statements with a copyblk.
16783 // block - BasicBlock*. block which stmt belongs to
16784 // stmt - GenTreeStmt*. the stmt node we want to check
16787 // if this funciton successfully optimized the stmts, then return true. Otherwise
16790 bool Compiler::fgMorphCombineSIMDFieldAssignments(BasicBlock* block, GenTreePtr stmt)
16793 noway_assert(stmt->gtOper == GT_STMT);
16794 GenTreePtr tree = stmt->gtStmt.gtStmtExpr;
16795 assert(tree->OperGet() == GT_ASG);
16797 GenTreePtr originalLHS = tree->gtOp.gtOp1;
16798 GenTreePtr prevLHS = tree->gtOp.gtOp1;
16799 GenTreePtr prevRHS = tree->gtOp.gtOp2;
16800 unsigned index = 0;
16801 var_types baseType = TYP_UNKNOWN;
16802 unsigned simdSize = 0;
16803 GenTreePtr simdStructNode = getSIMDStructFromField(prevRHS, &baseType, &index, &simdSize, true);
16805 if (simdStructNode == nullptr ||
16807 baseType != TYP_FLOAT)
16809 // if the RHS is not from a SIMD vector field X, then there is no need to check further.
16813 int assignmentsCount = simdSize / genTypeSize(baseType) - 1;
16814 int remainingAssignments = assignmentsCount;
16815 GenTreePtr curStmt = stmt->gtNext;
16816 GenTreePtr lastStmt = stmt;
16818 while (curStmt != nullptr && remainingAssignments > 0)
16820 GenTreePtr exp = curStmt->gtStmt.gtStmtExpr;
16821 if (exp->OperGet() != GT_ASG)
16825 GenTreePtr curLHS = exp->gtGetOp1();
16826 GenTreePtr curRHS = exp->gtGetOp2();
16828 if (!areArgumentsContiguous(prevLHS, curLHS) ||
16829 !areArgumentsContiguous(prevRHS, curRHS))
16834 remainingAssignments--;
16838 lastStmt = curStmt;
16839 curStmt = curStmt->gtNext;
16842 if (remainingAssignments > 0)
16844 // if the left assignments number is bigger than zero, then this means
16845 // that the assignments are not assgining to the contiguously memory
16846 // locations from same vector.
16852 printf("\nFound contiguous assignments from a SIMD vector to memory.\n");
16853 printf("From BB%02u, stmt", block->bbNum);
16855 printf(" to stmt");
16856 printTreeID(lastStmt);
16862 for (int i = 0; i < assignmentsCount; i++)
16864 fgRemoveStmt(block, stmt->gtNext);
16867 GenTree* copyBlkDst = createAddressNodeForSIMDInit(originalLHS, simdSize);
16868 if (simdStructNode->OperIsLocal())
16870 setLclRelatedToSIMDIntrinsic(simdStructNode);
16872 GenTree* copyBlkAddr = copyBlkDst;
16873 if (copyBlkAddr->gtOper == GT_LEA)
16875 copyBlkAddr = copyBlkAddr->AsAddrMode()->Base();
16877 GenTreeLclVarCommon* localDst = nullptr;
16878 if (copyBlkAddr->IsLocalAddrExpr(this, &localDst, nullptr))
16880 setLclRelatedToSIMDIntrinsic(localDst);
16883 GenTree* simdStructAddr;
16884 if (simdStructNode->TypeGet() == TYP_BYREF)
16886 assert(simdStructNode->OperIsLocal());
16887 assert(lvaIsImplicitByRefLocal(simdStructNode->AsLclVarCommon()->gtLclNum));
16888 simdStructAddr = simdStructNode;
16892 assert(varTypeIsSIMD(simdStructNode));
16893 simdStructAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, simdStructNode);
16899 printf("\nBB%02u stmt", block->bbNum);
16901 printf("(before)\n");
16906 tree = gtNewBlkOpNode(GT_COPYBLK,
16909 gtNewIconNode(simdSize),
16912 stmt->gtStmt.gtStmtExpr = tree;
16914 // Since we generated a new address node which didn't exist before,
16915 // we should expose this address manually here.
16916 AXCStack stk(this);
16917 stk.Push(AXC_None);
16918 fgWalkTree(&stmt->gtStmt.gtStmtExpr,
16919 fgMarkAddrTakenLocalsPreCB,
16920 fgMarkAddrTakenLocalsPostCB,
16926 printf("\nReplaced BB%02u stmt", block->bbNum);
16928 printf("(after)\n");
16935 #endif //FEATURE_SIMD