1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
5 /*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
6 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
11 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
19 #include "allocacheck.h" // for alloca
21 /*****************************************************************************/
23 // Split a tree at the given point
24 // -- Introduces a new temporary variable
25 // -- evaluates *splitPoint into the new temp, in a new statement inserted before 'stmt'
26 // -- substitutes the temporary for '*splitPoint' in 'stmt'
27 // '*splitpoint' must be a node in 'stmt', which is within 'blk', and 'splitpoint' is a pointer
28 // to the link to that node, contained in its parent node.
29 GenTree* Compiler::fgMorphSplitTree(GenTree** splitPoint, // where to split
30 GenTree* stmt, // top level statement housing this tree
31 BasicBlock* blk) // block we are in
36 if ((*splitPoint)->OperIsAssignment())
38 // it's already being assigned so don't introduce a new one
39 newTree = *splitPoint;
40 temp = (*splitPoint)->gtGetOp1();
44 unsigned lclNum = lvaGrabTemp(true DEBUGARG("split tree"));
45 newTree = gtNewTempAssign(lclNum, *splitPoint);
46 temp = gtNewLclvNode(lclNum, (*splitPoint)->TypeGet());
49 GenTreePtr asg = gtNewStmt(newTree);
53 fgInsertStmtBefore(blk, stmt, asg);
58 // state carried over the tree walk, to be used in making
59 // a splitting decision.
62 // number of calls seen
65 // callback to determine if we should split here
66 Compiler::fgSplitPredicate* pred;
68 // root stmt of tree being processed
74 // Returns true if we should split the tree above this node.
75 // For ARM FP, handling multiple calls per tree via a local and
76 // greedy register allocator could result in a lot of shuffling.
77 // So let the global register allocator handle these cases.
78 bool shouldSplitARM(GenTree* tree, GenTree* parent, Compiler::fgWalkData* data)
81 && varTypeIsFloating(tree)
83 && !parent->OperIsAssignment())
85 // increment call count
86 SplitData* tmpState = (SplitData*) data->pCallbackData;
89 return tmpState->count > 1;
96 #endif // _TARGET_ARM_
98 // Callback for the tree walker, called once per node.
99 // Determines if we want to split, performs the split, and then processes the rest of the tree
100 Compiler::fgWalkResult Compiler::fgSplitHelper(GenTree** ppTree, fgWalkData* data)
102 GenTree* tree = *ppTree;
103 Compiler* comp = data->compiler;
105 SplitData* tmpState = (SplitData*) data->pCallbackData;
107 fgSplitPredicate* pred = tmpState->pred;
109 if (pred(tree, data->parent, data)) // does this look like somewhere we want to split?
111 //printf("tmpstate2 = %d %p r:%p tmp:%p tree:%p\n", tmpState->count, tmpState->pred, tmpState->root, tmpState, tree);
112 GenTree* result = comp->fgMorphSplitTree(ppTree, tmpState->root, comp->compCurBB);
114 GenTree* oldStatement = comp->compCurStmt;
115 comp->compCurStmt = result;
117 // because we are doing this in pre-order we also have to process
118 // the subtree that we have just split off
119 comp->fgSplitProcessOneTree(result, pred);
122 comp->compCurStmt = oldStatement;
124 return Compiler::WALK_SKIP_SUBTREES;
126 //else printf("tmpstate3 = %d %p r:%p tmp:%p tree:%p\n", tmpState->count, tmpState->pred, tmpState->root, tmpState, tree);
128 return Compiler::WALK_CONTINUE;
131 void Compiler::fgSplitProcessOneTree(GenTree* tree, fgSplitPredicate pred)
133 SplitData tmpState = {0};
134 tmpState.pred = pred;
135 tmpState.root = tree;
137 fgWalkTreePre(&(tree->gtStmt.gtStmtExpr),
142 // Split expression trees at points which in the case of ARM this is done.
143 void Compiler::fgSplitMethodTrees(void)
147 #else // _TARGET_ARM_
148 for (BasicBlock* block = fgFirstBB; block; block = block->bbNext)
151 for (GenTree* tree = block->bbTreeList; tree; tree = tree->gtNext)
153 assert(tree != tree->gtNext);
154 fgSplitProcessOneTree(tree, shouldSplitARM);
157 #endif // _TARGET_ARM_
161 // Convert the given node into a call to the specified helper passing
162 // the given argument list.
163 // Tries to fold constants and also adds an edge for overflow exception
164 // returns the morphed tree
165 GenTreePtr Compiler::fgMorphCastIntoHelper(GenTreePtr tree,
171 /* If the operand is a constant, we'll try to fold it */
172 if (oper->OperIsConst())
174 GenTreePtr oldTree = tree;
176 tree = gtFoldExprConst(tree); // This may not fold the constant (NaN ...)
179 return fgMorphTree(tree);
180 else if (tree->OperKind() & GTK_CONST)
181 return fgMorphConst(tree);
183 // assert that oper is unchanged and that it is still a GT_CAST node
184 noway_assert(tree->gtCast.CastOp() == oper);
185 noway_assert(tree->gtOper == GT_CAST);
187 result = fgMorphIntoHelperCall(tree, helper, gtNewArgList(oper));
188 assert(result == tree);
193 /*****************************************************************************
195 * Convert the given node into a call to the specified helper passing
196 * the given argument list.
199 GenTreePtr Compiler::fgMorphIntoHelperCall(GenTreePtr tree,
201 GenTreeArgList* args)
203 tree->ChangeOper(GT_CALL);
205 tree->gtFlags |= GTF_CALL;
206 tree->gtCall.gtCallType = CT_HELPER;
207 tree->gtCall.gtCallMethHnd = eeFindHelper(helper);
208 tree->gtCall.gtCallArgs = args;
209 tree->gtCall.gtCallObjp = NULL;
210 tree->gtCall.gtCallLateArgs = NULL;
211 tree->gtCall.fgArgInfo = NULL;
212 tree->gtCall.gtRetClsHnd = NULL;
213 tree->gtCall.gtCallRegUsedMask = RBM_NONE;
214 tree->gtCall.gtCallMoreFlags = 0;
215 tree->gtCall.gtInlineCandidateInfo = NULL;
216 tree->gtCall.gtControlExpr = NULL;
218 #ifdef FEATURE_READYTORUN_COMPILER
219 tree->gtCall.gtEntryPoint.addr = nullptr;
222 /* Perform the morphing */
224 tree = fgMorphArgs(tree->AsCall());
229 /*****************************************************************************
230 * This node should not be referenced by anyone now. Set its values to garbage
231 * to catch extra references
235 void DEBUG_DESTROY_NODE(GenTreePtr tree)
238 // printf("DEBUG_DESTROY_NODE for [0x%08x]\n", tree);
240 // Save gtOper in case we want to find out what this node was
241 tree->gtOperSave = tree->gtOper;
243 tree->gtType = TYP_UNDEF;
244 tree->gtFlags |= 0xFFFFFFFF & ~GTF_NODE_MASK;
245 if (tree->OperIsSimple())
248 tree->gtOp.gtOp2 = NULL;
250 // Must do this last, because the "gtOp" check above will fail otherwise.
251 // Don't call SetOper, because GT_COUNT is not a valid value
252 tree->gtOper = GT_COUNT;
257 /*****************************************************************************
259 * Determine if a relop must be morphed to a qmark to manifest a boolean value.
260 * This is done when code generation can't create straight-line code to do it.
262 bool Compiler::fgMorphRelopToQmark(GenTreePtr tree)
264 #ifndef LEGACY_BACKEND
266 #else // LEGACY_BACKEND
267 return (genActualType(tree->TypeGet()) == TYP_LONG) ||
268 varTypeIsFloating(tree->TypeGet());
269 #endif // LEGACY_BACKEND
273 /*****************************************************************************
275 * Morph a cast node (we perform some very simple transformations here).
279 #pragma warning(push)
280 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
282 GenTreePtr Compiler::fgMorphCast(GenTreePtr tree)
284 noway_assert(tree->gtOper == GT_CAST);
285 noway_assert(genTypeSize(TYP_I_IMPL) == sizeof(void*));
287 /* The first sub-operand is the thing being cast */
289 GenTreePtr oper = tree->gtCast.CastOp();
290 var_types srcType = genActualType(oper->TypeGet());
293 var_types dstType = tree->CastToType();
294 unsigned dstSize = genTypeSize(dstType);
296 // See if the cast has to be done in two steps. R -> I
297 if (varTypeIsFloating(srcType) && varTypeIsIntegral(dstType))
299 // Only x86 must go through TYP_DOUBLE to get to all
300 // integral types everybody else can get straight there
301 // except for when using helpers
302 if (srcType == TYP_FLOAT
303 #if !FEATURE_STACK_FP_X87
305 #if defined(_TARGET_ARM64_)
306 // Amd64: src = float, dst is overflow conversion.
307 // This goes through helper and hence src needs to be converted to double.
308 && tree->gtOverflow()
309 #elif defined(_TARGET_AMD64_)
310 // Amd64: src = float, dst = uint64 or overflow conversion.
311 // This goes through helper and hence src needs to be converted to double.
312 && (tree->gtOverflow() || (dstType == TYP_ULONG))
313 #elif defined(_TARGET_ARM_)
314 // Arm: src = float, dst = int64/uint64 or overflow conversion.
315 && (tree->gtOverflow() || varTypeIsLong(dstType))
318 #endif // FEATURE_STACK_FP_X87
321 oper = gtNewCastNode(TYP_DOUBLE, oper, TYP_DOUBLE);
324 // do we need to do it in two steps R -> I, '-> smallType
325 #if defined(_TARGET_ARM64_) || defined(_TARGET_AMD64_)
326 if (dstSize < genTypeSize(TYP_INT))
328 oper = gtNewCastNodeL(TYP_INT, oper, TYP_INT);
329 oper->gtFlags |= (tree->gtFlags & (GTF_UNSIGNED|GTF_OVERFLOW|GTF_EXCEPT));
330 tree->gtFlags &= ~GTF_UNSIGNED;
333 if (dstSize < sizeof(void*))
335 oper = gtNewCastNodeL(TYP_I_IMPL, oper, TYP_I_IMPL);
336 oper->gtFlags |= (tree->gtFlags & (GTF_OVERFLOW|GTF_EXCEPT));
341 /* Note that if we need to use a helper call then we can not morph oper */
342 if (!tree->gtOverflow())
344 #ifdef _TARGET_ARM64_ // On ARM64 All non-overflow checking conversions can be optimized
350 #ifdef _TARGET_X86_ // there is no rounding convert to integer instruction on ARM or x64 so skip this
351 if ((oper->gtOper == GT_INTRINSIC) &&
352 (oper->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Round))
354 /* optimization: conv.i4(round.d(d)) -> round.i(d) */
355 oper->gtType = dstType;
356 return fgMorphTree(oper);
358 // if SSE2 is not enabled, we need the helper
359 else if (!opts.compCanUseSSE2)
361 return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2INT, oper);
364 #endif // _TARGET_X86_
368 #if defined(_TARGET_ARM_) || defined(_TARGET_AMD64_)
369 case TYP_UINT: goto OPTIMIZECAST;
370 #else // _TARGET_ARM_
371 case TYP_UINT: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2UINT, oper);
372 #endif // _TARGET_ARM_
374 #ifdef _TARGET_AMD64_
375 // SSE2 has instructions to convert a float/double directly to a long
376 case TYP_LONG: goto OPTIMIZECAST;
378 case TYP_LONG: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2LNG, oper);
379 #endif //_TARGET_AMD64_
380 case TYP_ULONG: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2ULNG, oper);
383 #endif // _TARGET_ARM64_
389 case TYP_INT: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2INT_OVF, oper);
390 case TYP_UINT: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2UINT_OVF, oper);
391 case TYP_LONG: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2LNG_OVF, oper);
392 case TYP_ULONG: return fgMorphCastIntoHelper(tree, CORINFO_HELP_DBL2ULNG_OVF, oper);
396 noway_assert(!"Unexpected dstType");
399 #ifndef _TARGET_64BIT_
400 // The code generation phase (for x86 & ARM32) does not handle casts
401 // directly from [u]long to anything other than [u]int. Insert an
402 // intermediate cast to native int.
403 else if (varTypeIsLong(srcType) && varTypeIsSmall(dstType))
405 oper = gtNewCastNode(TYP_I_IMPL, oper, TYP_I_IMPL);
406 oper->gtFlags |= (tree->gtFlags & (GTF_OVERFLOW|GTF_EXCEPT|GTF_UNSIGNED));
407 tree->gtFlags &= ~GTF_UNSIGNED;
409 #endif //!_TARGET_64BIT_
412 else if ((dstType == TYP_FLOAT) && (srcType == TYP_DOUBLE) && (oper->gtOper == GT_CAST) && !varTypeIsLong(oper->gtCast.CastOp()))
414 // optimization: conv.r4(conv.r8(?)) -> conv.r4(d)
415 // except when the ultimate source is a long because there is no long-to-float helper, so it must be 2 step.
416 // This happens semi-frequently because there is no IL 'conv.r4.un'
417 oper->gtType = TYP_FLOAT;
418 oper->CastToType() = TYP_FLOAT;
419 return fgMorphTree(oper);
421 // converts long/ulong --> float/double casts into helper calls.
422 else if (varTypeIsFloating(dstType) && varTypeIsLong(srcType))
424 if (dstType == TYP_FLOAT)
426 // there is only a double helper, so we
427 // - change the dsttype to double
428 // - insert a cast from double to float
429 // - recurse into the resulting tree
430 tree->CastToType() = TYP_DOUBLE;
431 tree->gtType = TYP_DOUBLE;
433 tree = gtNewCastNode(TYP_FLOAT, tree, TYP_FLOAT);
435 return fgMorphTree(tree);
437 if (tree->gtFlags & GTF_UNSIGNED)
438 return fgMorphCastIntoHelper(tree, CORINFO_HELP_ULNG2DBL, oper);
439 return fgMorphCastIntoHelper(tree, CORINFO_HELP_LNG2DBL, oper);
441 #endif //_TARGET_ARM_
443 #ifdef _TARGET_AMD64_
444 // Do we have to do two step U4/8 -> R4/8 ?
445 // Codegen supports the following conversion as one-step operation
449 // The following conversions are performed as two-step operations using above.
450 // U4 -> R4/8 = U4-> Long -> R4/8
451 // U8 -> R4 = U8 -> R8 -> R4
452 else if ((tree->gtFlags & GTF_UNSIGNED) && varTypeIsFloating(dstType))
454 srcType = genUnsignedType(srcType);
456 if (srcType == TYP_ULONG)
458 if (dstType == TYP_FLOAT)
460 // Codegen can handle U8 -> R8 conversion.
461 // U8 -> R4 = U8 -> R8 -> R4
462 // - change the dsttype to double
463 // - insert a cast from double to float
464 // - recurse into the resulting tree
465 tree->CastToType() = TYP_DOUBLE;
466 tree->gtType = TYP_DOUBLE;
467 tree = gtNewCastNode(TYP_FLOAT, tree, TYP_FLOAT);
468 return fgMorphTree(tree);
471 else if (srcType == TYP_UINT)
473 oper = gtNewCastNode(TYP_LONG, oper, TYP_LONG);
474 oper->gtFlags |= (tree->gtFlags & (GTF_OVERFLOW|GTF_EXCEPT|GTF_UNSIGNED));
475 tree->gtFlags &= ~GTF_UNSIGNED;
478 #endif // _TARGET_AMD64_
481 // Do we have to do two step U4/8 -> R4/8 ?
482 else if ((tree->gtFlags & GTF_UNSIGNED) && varTypeIsFloating(dstType))
484 srcType = genUnsignedType(srcType);
486 if (srcType == TYP_ULONG)
488 return fgMorphCastIntoHelper(tree, CORINFO_HELP_ULNG2DBL, oper);
490 else if (srcType == TYP_UINT)
492 oper = gtNewCastNode(TYP_LONG, oper, TYP_LONG);
493 oper->gtFlags |= (tree->gtFlags & (GTF_OVERFLOW|GTF_EXCEPT|GTF_UNSIGNED));
494 tree->gtFlags &= ~GTF_UNSIGNED;
497 #endif //_TARGET_XARCH_
498 else if (varTypeIsGC(srcType) != varTypeIsGC(dstType))
500 // We are casting away GC information. we would like to just
501 // change the type to int, however this gives the emitter fits because
502 // it believes the variable is a GC variable at the begining of the
503 // instruction group, but is not turned non-gc by the code generator
504 // we fix this by copying the GC pointer to a non-gc pointer temp.
505 noway_assert(!varTypeIsGC(dstType) && "How can we have a cast to a GCRef here?");
507 // We generate an assignment to an int and then do the cast from an int. With this we avoid
508 // the gc problem and we allow casts to bytes, longs, etc...
509 unsigned lclNum = lvaGrabTemp(true DEBUGARG("Cast away GC"));
510 oper->gtType = TYP_I_IMPL;
511 GenTreePtr asg = gtNewTempAssign(lclNum, oper);
512 oper->gtType = srcType;
515 GenTreePtr cast = gtNewCastNode(tree->TypeGet(), gtNewLclvNode(lclNum, TYP_I_IMPL), dstType);
517 // Generate the comma tree
518 oper = gtNewOperNode(GT_COMMA, tree->TypeGet(), asg, cast);
520 return fgMorphTree(oper);
523 // Look for narrowing casts ([u]long -> [u]int) and try to push them
524 // down into the operand before morphing it.
526 // It doesn't matter if this is cast is from ulong or long (i.e. if
527 // GTF_UNSIGNED is set) because the transformation is only applied to
528 // overflow-insensitive narrowing casts, which always silently truncate.
530 // Note that casts from [u]long to small integer types are handled above.
531 if ((srcType == TYP_LONG) &&
532 ((dstType == TYP_INT) || (dstType == TYP_UINT)))
534 // As a special case, look for overflow-sensitive casts of an AND
535 // expression, and see if the second operand is a small constant. Since
536 // the result of an AND is bound by its smaller operand, it may be
537 // possible to prove that the cast won't overflow, which will in turn
538 // allow the cast's operand to be transformed.
539 if (tree->gtOverflow() && (oper->OperGet() == GT_AND))
541 GenTreePtr andOp2 = oper->gtOp.gtOp2;
543 // Special case to the special case: AND with a casted int.
544 if ((andOp2->OperGet() == GT_CAST) &&
545 (andOp2->gtCast.CastOp()->OperGet() == GT_CNS_INT))
547 // gtFoldExprConst will deal with whether the cast is signed or
548 // unsigned, or overflow-sensitive.
549 andOp2 = oper->gtOp.gtOp2 = gtFoldExprConst(andOp2);
552 // Look for a constant less than 2^{32} for a cast to uint, or less
553 // than 2^{31} for a cast to int.
554 int maxWidth = (dstType == TYP_UINT) ? 32 : 31;
556 if ((andOp2->OperGet() == GT_CNS_NATIVELONG) &&
557 ((andOp2->gtIntConCommon.LngValue() >> maxWidth) == 0))
559 // This cast can't overflow.
560 tree->gtFlags &= ~(GTF_OVERFLOW | GTF_EXCEPT);
564 // Only apply this transformation during global morph,
565 // when neither the cast node nor the oper node may throw an exception
566 // based on the upper 32 bits.
569 !tree->gtOverflow() &&
570 !oper->gtOverflowEx())
572 // For these operations the lower 32 bits of the result only depends
573 // upon the lower 32 bits of the operands
575 if ( (oper->OperGet() == GT_ADD) ||
576 (oper->OperGet() == GT_MUL) ||
577 (oper->OperGet() == GT_AND) ||
578 (oper->OperGet() == GT_OR) ||
579 (oper->OperGet() == GT_XOR) )
581 DEBUG_DESTROY_NODE(tree);
583 // Insert narrowing casts for op1 and op2
584 oper->gtOp.gtOp1 = gtNewCastNode(TYP_INT, oper->gtOp.gtOp1, dstType);
585 oper->gtOp.gtOp2 = gtNewCastNode(TYP_INT, oper->gtOp.gtOp2, dstType);
587 // Clear the GT_MUL_64RSLT if it is set
588 if (oper->gtOper == GT_MUL && (oper->gtFlags & GTF_MUL_64RSLT))
589 oper->gtFlags &= ~GTF_MUL_64RSLT;
591 // The operation now produces a 32-bit result.
592 oper->gtType = TYP_INT;
594 // Remorph the new tree as the casts that we added may be folded away.
595 return fgMorphTree(oper);
601 noway_assert(tree->gtOper == GT_CAST);
603 /* Morph the operand */
604 tree->gtCast.CastOp() = oper = fgMorphTree(oper);
606 /* Reset the call flag */
607 tree->gtFlags &= ~GTF_CALL;
609 /* unless we have an overflow cast, reset the except flag */
610 if (!tree->gtOverflow())
611 tree->gtFlags &= ~GTF_EXCEPT;
613 /* Just in case new side effects were introduced */
614 tree->gtFlags |= (oper->gtFlags & GTF_ALL_EFFECT);
616 srcType = oper->TypeGet();
618 /* if GTF_UNSIGNED is set then force srcType to an unsigned type */
619 if (tree->gtFlags & GTF_UNSIGNED)
620 srcType = genUnsignedType(srcType);
622 srcSize = genTypeSize(srcType);
624 if (!gtIsActiveCSE_Candidate(tree)) // tree cannot be a CSE candidate
626 /* See if we can discard the cast */
627 if (varTypeIsIntegral(srcType) && varTypeIsIntegral(dstType))
629 if (srcType == dstType) // Certainly if they are identical it is pointless
632 if (oper->OperGet() == GT_LCL_VAR && varTypeIsSmall(dstType))
634 unsigned varNum = oper->gtLclVarCommon.gtLclNum;
635 LclVarDsc * varDsc = &lvaTable[varNum];
636 if (varDsc->TypeGet() == dstType && varDsc->lvNormalizeOnStore())
640 bool unsignedSrc = varTypeIsUnsigned(srcType);
641 bool unsignedDst = varTypeIsUnsigned(dstType);
642 bool signsDiffer = (unsignedSrc != unsignedDst);
644 // For same sized casts with
645 // the same signs or non-overflow cast we discard them as well
646 if (srcSize == dstSize)
648 /* This should have been handled above */
649 noway_assert(varTypeIsGC(srcType) == varTypeIsGC(dstType));
654 if (!tree->gtOverflow())
656 /* For small type casts, when necessary we force
657 the src operand to the dstType and allow the
658 implied load from memory to perform the casting */
659 if (varTypeIsSmall(srcType))
661 switch (oper->gtOper)
667 oper->gtType = dstType;
677 if (srcSize < dstSize) // widening cast
679 // Keep any long casts
680 if (dstSize == sizeof(int))
682 // Only keep signed to unsigned widening cast with overflow check
683 if (!tree->gtOverflow() || !unsignedDst || unsignedSrc)
687 // Casts from signed->unsigned can never overflow while widening
689 if (unsignedSrc || !unsignedDst)
690 tree->gtFlags &= ~GTF_OVERFLOW;
694 // Try to narrow the operand of the cast and discard the cast
695 // Note: Do not narrow a cast that is marked as a CSE
696 // And do not narrow if the oper is marked as a CSE either
698 if (!tree->gtOverflow() &&
699 !gtIsActiveCSE_Candidate(oper) &&
700 (opts.compFlags & CLFLG_TREETRANS) &&
701 optNarrowTree(oper, srcType, dstType, tree->gtVNPair, false))
703 optNarrowTree(oper, srcType, dstType, tree->gtVNPair, true);
705 /* If oper is changed into a cast to TYP_INT, or to a GT_NOP, we may need to discard it */
706 if (oper->gtOper == GT_CAST && oper->CastToType() == genActualType(oper->CastFromType()))
708 oper = oper->gtCast.CastOp();
715 switch (oper->gtOper)
717 /* If the operand is a constant, we'll fold it */
723 GenTreePtr oldTree = tree;
725 tree = gtFoldExprConst(tree); // This may not fold the constant (NaN ...)
727 // Did we get a comma throw as a result of gtFoldExprConst?
728 if ((oldTree != tree) && (oldTree->gtOper != GT_COMMA))
730 noway_assert(fgIsCommaThrow(tree));
731 tree->gtOp.gtOp1 = fgMorphTree(tree->gtOp.gtOp1);
732 fgMorphTreeDone(tree);
735 else if (tree->gtOper != GT_CAST)
738 noway_assert(tree->gtCast.CastOp() == oper); // unchanged
743 /* Check for two consecutive casts into the same dstType */
744 if (!tree->gtOverflow())
746 var_types dstType2 = oper->CastToType();
747 if (dstType == dstType2)
752 /* If op1 is a mod node, mark it with the GTF_MOD_INT_RESULT flag
753 so that the code generator will know not to convert the result
754 of the idiv to a regpair */
756 if (dstType == TYP_INT)
757 tree->gtOp.gtOp1->gtFlags |= GTF_MOD_INT_RESULT;
761 if (dstType == TYP_UINT)
762 tree->gtOp.gtOp1->gtFlags |= GTF_MOD_INT_RESULT;
766 // Check for cast of a GT_COMMA with a throw overflow
767 // Bug 110829: Since this optimization will bash the types
768 // neither oper or commaOp2 can be CSE candidates
769 if (fgIsCommaThrow(oper) &&
770 !gtIsActiveCSE_Candidate(oper)) // oper can not be a CSE candidate
772 GenTreePtr commaOp2 = oper->gtOp.gtOp2;
774 if (!gtIsActiveCSE_Candidate(commaOp2)) // commaOp2 can not be a CSE candidate
776 // need type of oper to be same as tree
777 if (tree->gtType == TYP_LONG)
779 commaOp2->ChangeOperConst(GT_CNS_NATIVELONG);
780 commaOp2->gtIntConCommon.SetLngValue(0);
781 /* Change the types of oper and commaOp2 to TYP_LONG */
782 oper->gtType = commaOp2->gtType = TYP_LONG;
784 else if (varTypeIsFloating(tree->gtType))
786 commaOp2->ChangeOperConst(GT_CNS_DBL);
787 commaOp2->gtDblCon.gtDconVal = 0.0;
788 // Change the types of oper and commaOp2
789 // X87 promotes everything to TYP_DOUBLE
790 // But other's are a little more precise
791 const var_types newTyp
792 #if FEATURE_X87_DOUBLES
794 #else // FEATURE_X87_DOUBLES
796 #endif // FEATURE_X87_DOUBLES
797 oper->gtType = commaOp2->gtType = newTyp;
801 commaOp2->ChangeOperConst(GT_CNS_INT);
802 commaOp2->gtIntCon.gtIconVal = 0;
803 /* Change the types of oper and commaOp2 to TYP_INT */
804 oper->gtType = commaOp2->gtType = TYP_INT;
808 if (vnStore != nullptr)
810 fgValueNumberTreeConst(commaOp2);
813 /* Return the GT_COMMA node as the new tree */
820 } /* end switch (oper->gtOper) */
823 if (tree->gtOverflow())
824 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_OVERFLOW, fgPtrArgCntCur);
830 /* Here we've eliminated the cast, so just return it's operand */
831 assert(!gtIsActiveCSE_Candidate(tree)); // tree cannot be a CSE candidate
833 DEBUG_DESTROY_NODE(tree);
840 /*****************************************************************************
842 * Perform an unwrap operation on a Proxy object
845 GenTreePtr Compiler::fgUnwrapProxy(GenTreePtr objRef)
847 assert(info.compIsContextful &&
848 info.compUnwrapContextful &&
851 CORINFO_EE_INFO * pInfo = eeGetEEInfo();
854 // Perform the unwrap:
856 // This requires two extra indirections.
857 // We mark these indirections as 'invariant' and
858 // the CSE logic will hoist them when appropriate.
860 // Note that each dereference is a GC pointer
862 addTree = gtNewOperNode(GT_ADD, TYP_I_IMPL,
864 gtNewIconNode(pInfo->offsetOfTransparentProxyRP, TYP_I_IMPL));
866 objRef = gtNewOperNode(GT_IND, TYP_REF, addTree);
867 objRef->gtFlags |= GTF_IND_INVARIANT;
869 addTree = gtNewOperNode(GT_ADD, TYP_I_IMPL,
871 gtNewIconNode(pInfo->offsetOfRealProxyServer, TYP_I_IMPL));
873 objRef = gtNewOperNode(GT_IND, TYP_REF, addTree);
874 objRef->gtFlags |= GTF_IND_INVARIANT;
876 // objRef now hold the 'real this' reference (i.e. the unwrapped proxy)
880 /*****************************************************************************
882 * Morph an argument list; compute the pointer argument count in the process.
884 * NOTE: This function can be called from any place in the JIT to perform re-morphing
885 * due to graph altering modifications such as copy / constant propagation
888 unsigned UpdateGT_LISTFlags(GenTreePtr tree)
890 assert(tree->gtOper == GT_LIST);
893 if (tree->gtOp.gtOp2)
895 flags |= UpdateGT_LISTFlags(tree->gtOp.gtOp2);
898 flags |= (tree->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT);
900 tree->gtFlags &= ~GTF_ALL_EFFECT;
901 tree->gtFlags |= flags;
903 return tree->gtFlags;
906 fgArgInfo::fgArgInfo(Compiler * comp, GenTreePtr call, unsigned numArgs)
909 callTree = call; assert(call->IsCall());
910 argCount = 0; // filled in arg count, starts at zero
911 nextSlotNum = INIT_ARG_STACK_SLOT;
913 argTableSize = numArgs; // the allocated table size
914 argsComplete = false;
917 if (argTableSize == 0)
920 argTable = new(compiler, CMK_fgArgInfoPtrArr) fgArgTabEntryPtr[argTableSize];
923 /*****************************************************************************
925 * fgArgInfo Copy Constructor
927 * This method needs to act like a copy constructor for fgArgInfo.
928 * The newCall needs to have its fgArgInfo initialized such that
929 * we have newCall that is an exact copy of the oldCall.
930 * We have to take care since the argument information
931 * in the argTable contains pointers that must point to the
932 * new arguments and not the old arguments.
934 fgArgInfo::fgArgInfo(GenTreePtr newCall, GenTreePtr oldCall)
936 assert(oldCall->IsCall());
937 assert(newCall->IsCall());
939 fgArgInfoPtr oldArgInfo = oldCall->gtCall.fgArgInfo;
941 compiler = oldArgInfo->compiler;;
942 callTree = newCall; assert(newCall->IsCall());
943 argCount = 0; // filled in arg count, starts at zero
944 nextSlotNum = INIT_ARG_STACK_SLOT;
945 stkLevel = oldArgInfo->stkLevel;
946 argTableSize = oldArgInfo->argTableSize;
947 argsComplete = false;
949 if (argTableSize > 0)
951 argTable = new(compiler, CMK_fgArgInfoPtrArr) fgArgTabEntryPtr[argTableSize];
952 for (unsigned inx=0; inx<argTableSize; inx++)
954 argTable[inx] = NULL;
958 assert(oldArgInfo->argsComplete);
960 // We create local, artificial GenTreeArgLists that includes the gtCallObjp, if that exists, as first argument,
961 // so we can iterate over these argument lists more uniformly.
962 // Need to provide a temporary non-null first arguments to these constructors: if we use them, we'll replace them
963 GenTreeArgList* newArgs;
964 GenTreeArgList newArgObjp(newCall, newCall->gtCall.gtCallArgs);
965 GenTreeArgList* oldArgs;
966 GenTreeArgList oldArgObjp(oldCall, oldCall->gtCall.gtCallArgs);
968 if (newCall->gtCall.gtCallObjp == NULL)
970 assert(oldCall->gtCall.gtCallObjp == NULL);
971 newArgs = newCall->gtCall.gtCallArgs;
972 oldArgs = oldCall->gtCall.gtCallArgs;
976 assert(oldCall->gtCall.gtCallObjp != NULL);
977 newArgObjp.Current() = newCall->gtCall.gtCallArgs;
978 newArgs = &newArgObjp;
979 oldArgObjp.Current() = oldCall->gtCall.gtCallObjp;
980 oldArgs = &oldArgObjp;
985 GenTreeArgList* newParent = NULL;
986 GenTreeArgList* oldParent = NULL;
987 fgArgTabEntryPtr * oldArgTable = oldArgInfo->argTable;
988 bool scanRegArgs = false;
992 /* Get hold of the next argument values for the oldCall and newCall */
994 newCurr = newArgs->Current();
995 oldCurr = oldArgs->Current();
996 if (newArgs != &newArgObjp)
1003 assert(newParent == NULL && oldParent == NULL);
1005 newArgs = newArgs->Rest();
1006 oldArgs = oldArgs->Rest();
1008 fgArgTabEntryPtr oldArgTabEntry = NULL;
1009 fgArgTabEntryPtr newArgTabEntry = NULL;
1011 for (unsigned inx=0; inx<argTableSize; inx++)
1013 oldArgTabEntry = oldArgTable[inx];
1015 if (oldArgTabEntry->parent == oldParent)
1017 assert((oldParent == NULL) == (newParent == NULL));
1019 // We have found the matching "parent" field in oldArgTabEntry
1021 newArgTabEntry = new (compiler, CMK_fgArgInfo) fgArgTabEntry;
1023 // First block copy all fields
1025 *newArgTabEntry = *oldArgTabEntry;
1027 // Then update all GenTreePtr fields in the newArgTabEntry
1029 newArgTabEntry->parent = newParent;
1031 // The node field is likely to have been updated
1032 // to point at a node in the gtCallLateArgs list
1034 if (oldArgTabEntry->node == oldCurr)
1036 // node is not pointing into the gtCallLateArgs list
1037 newArgTabEntry->node = newCurr;
1041 // node must be pointing into the gtCallLateArgs list
1043 // We will fix this pointer up in the next loop
1045 newArgTabEntry->node = NULL; // For now we assign a NULL to this field
1050 // Now initialize the proper element in the argTable array
1052 argTable[inx] = newArgTabEntry;
1056 // We should have found the matching oldArgTabEntry and created the newArgTabEntry
1058 assert(newArgTabEntry != NULL);
1063 newArgs = newCall->gtCall.gtCallLateArgs;
1064 oldArgs = oldCall->gtCall.gtCallLateArgs;
1068 /* Get hold of the next argument values for the oldCall and newCall */
1070 assert(newArgs->IsList());
1072 newCurr = newArgs->Current();
1073 newArgs = newArgs->Rest();
1075 assert(oldArgs->IsList());
1077 oldCurr = oldArgs->Current();
1078 oldArgs = oldArgs->Rest();
1080 fgArgTabEntryPtr oldArgTabEntry = NULL;
1081 fgArgTabEntryPtr newArgTabEntry = NULL;
1083 for (unsigned inx=0; inx<argTableSize; inx++)
1085 oldArgTabEntry = oldArgTable[inx];
1087 if (oldArgTabEntry->node == oldCurr)
1089 // We have found the matching "node" field in oldArgTabEntry
1091 newArgTabEntry = argTable[inx];
1092 assert(newArgTabEntry != NULL);
1094 // update the "node" GenTreePtr fields in the newArgTabEntry
1096 assert(newArgTabEntry->node == NULL); // We previously assigned NULL to this field
1098 newArgTabEntry->node = newCurr;
1105 argCount = oldArgInfo->argCount;
1106 nextSlotNum = oldArgInfo->nextSlotNum;
1107 argsComplete = true;
1111 void fgArgInfo::AddArg(fgArgTabEntryPtr curArgTabEntry)
1113 assert(argCount < argTableSize);
1114 argTable[argCount] = curArgTabEntry;
1118 fgArgTabEntryPtr fgArgInfo::AddRegArg(unsigned argNum,
1125 fgArgTabEntryPtr curArgTabEntry = new(compiler, CMK_fgArgInfo) fgArgTabEntry;
1127 curArgTabEntry->argNum = argNum;
1128 curArgTabEntry->node = node;
1129 curArgTabEntry->parent = parent;
1130 curArgTabEntry->regNum = regNum;
1131 curArgTabEntry->slotNum = 0;
1132 curArgTabEntry->numRegs = numRegs;
1133 curArgTabEntry->numSlots = 0;
1134 curArgTabEntry->alignment = alignment;
1135 curArgTabEntry->lateArgInx = (unsigned)-1;
1136 curArgTabEntry->tmpNum = (unsigned)-1;
1137 curArgTabEntry->isSplit = false;
1138 curArgTabEntry->isTmp = false;
1139 curArgTabEntry->needTmp = false;
1140 curArgTabEntry->needPlace = false;
1141 curArgTabEntry->processed = false;
1142 curArgTabEntry->isHfaRegArg = false;
1143 curArgTabEntry->isBackFilled = false;
1144 curArgTabEntry->isNonStandard = false;
1146 AddArg(curArgTabEntry);
1147 return curArgTabEntry;
1150 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1151 fgArgTabEntryPtr fgArgInfo::AddRegArg(unsigned argNum,
1157 const bool isStruct,
1158 const regNumber otherRegNum,
1159 const SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR* const structDescPtr)
1161 fgArgTabEntryPtr curArgTabEntry = AddRegArg(argNum, node, parent, regNum, numRegs, alignment);
1162 assert(curArgTabEntry != nullptr);
1164 // The node of the ArgTabEntry could change after remorphing - it could be rewritten to a cpyblk or a
1165 // PlaceHolder node (in case of needed late argument, for example.)
1166 // This requires using of an extra flag. At creation time the state is right, so
1167 // and this assert enforces that.
1168 assert((varTypeIsStruct(node) && isStruct) || (!varTypeIsStruct(node) && !isStruct));
1169 curArgTabEntry->otherRegNum = otherRegNum; // Second reg for the struct
1170 curArgTabEntry->isStruct = isStruct; // is this a struct arg
1172 if (isStruct && structDescPtr != nullptr)
1174 curArgTabEntry->structDesc.CopyFrom(*structDescPtr);
1177 return curArgTabEntry;
1179 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1181 fgArgTabEntryPtr fgArgInfo::AddStkArg(unsigned argNum,
1186 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(const bool isStruct))
1188 fgArgTabEntryPtr curArgTabEntry = new(compiler, CMK_fgArgInfo) fgArgTabEntry;
1190 nextSlotNum = (unsigned)roundUp(nextSlotNum, alignment);
1192 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1193 // The node of the ArgTabEntry could change after remorphing - it could be rewritten to a cpyblk or a
1194 // PlaceHolder node (in case of needed late argument, for example.)
1195 // This reqires using of an extra flag. At creation time the state is right, so
1196 // and this assert enforces that.
1197 assert((varTypeIsStruct(node) && isStruct) || (!varTypeIsStruct(node) && !isStruct));
1198 curArgTabEntry->isStruct = isStruct; // is this a struct arg
1199 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1201 curArgTabEntry->argNum = argNum;
1202 curArgTabEntry->node = node;
1203 curArgTabEntry->parent = parent;
1204 curArgTabEntry->regNum = REG_STK;
1205 curArgTabEntry->slotNum = nextSlotNum;
1206 curArgTabEntry->numRegs = 0;
1207 curArgTabEntry->numSlots = numSlots;
1208 curArgTabEntry->alignment = alignment;
1209 curArgTabEntry->lateArgInx = (unsigned) -1;
1210 curArgTabEntry->tmpNum = (unsigned) -1;
1211 curArgTabEntry->isSplit = false;
1212 curArgTabEntry->isTmp = false;
1213 curArgTabEntry->needTmp = false;
1214 curArgTabEntry->needPlace = false;
1215 curArgTabEntry->processed = false;
1216 curArgTabEntry->isHfaRegArg = false;
1217 curArgTabEntry->isBackFilled = false;
1218 curArgTabEntry->isNonStandard = false;
1220 AddArg(curArgTabEntry);
1222 nextSlotNum += numSlots;
1223 return curArgTabEntry;
1226 void fgArgInfo::RemorphReset()
1228 nextSlotNum = INIT_ARG_STACK_SLOT;
1231 fgArgTabEntry* fgArgInfo::RemorphRegArg(unsigned argNum,
1238 fgArgTabEntryPtr curArgTabEntry = NULL;
1239 unsigned regArgInx = 0;
1242 for (inx=0; inx < argCount; inx++)
1244 curArgTabEntry = argTable[inx];
1245 if (curArgTabEntry->argNum == argNum)
1250 if (curArgTabEntry->parent != NULL)
1252 assert(curArgTabEntry->parent->IsList());
1253 argx = curArgTabEntry->parent->Current();
1254 isRegArg = (argx->gtFlags & GTF_LATE_ARG) != 0;
1258 argx = curArgTabEntry->node;
1267 // if this was a nonstandard arg the table is definitive
1268 if (curArgTabEntry->isNonStandard)
1269 regNum = curArgTabEntry->regNum;
1271 assert(curArgTabEntry->argNum == argNum);
1272 assert(curArgTabEntry->regNum == regNum);
1273 assert(curArgTabEntry->alignment == alignment);
1274 assert(curArgTabEntry->parent == parent);
1276 if (curArgTabEntry->node != node)
1278 GenTreePtr argx = NULL;
1279 unsigned regIndex = 0;
1281 /* process the register argument list */
1282 for (GenTreeArgList* list = callTree->gtCall.gtCallLateArgs; list; (regIndex++, list = list->Rest()))
1284 argx = list->Current();
1285 assert(!argx->IsArgPlaceHolderNode()); // No place holders nodes are in gtCallLateArgs;
1286 if (regIndex == regArgInx)
1289 assert(regIndex == regArgInx);
1290 assert(regArgInx == curArgTabEntry->lateArgInx);
1292 if (curArgTabEntry->node != argx)
1294 curArgTabEntry->node = argx;
1297 return curArgTabEntry;
1300 void fgArgInfo::RemorphStkArg(unsigned argNum,
1306 fgArgTabEntryPtr curArgTabEntry = NULL;
1307 bool isRegArg = false;
1308 unsigned regArgInx = 0;
1312 for (inx=0; inx < argCount; inx++)
1314 curArgTabEntry = argTable[inx];
1316 if (curArgTabEntry->parent != NULL)
1318 assert(curArgTabEntry->parent->IsList());
1319 argx = curArgTabEntry->parent->Current();
1320 isRegArg = (argx->gtFlags & GTF_LATE_ARG) != 0;
1324 argx = curArgTabEntry->node;
1328 if (curArgTabEntry->argNum == argNum)
1335 nextSlotNum = (unsigned) roundUp(nextSlotNum, alignment);
1337 assert(curArgTabEntry->argNum == argNum);
1338 assert(curArgTabEntry->slotNum == nextSlotNum);
1339 assert(curArgTabEntry->numSlots == numSlots);
1340 assert(curArgTabEntry->alignment == alignment);
1341 assert(curArgTabEntry->parent == parent);
1342 assert(parent->IsList());
1344 #if FEATURE_FIXED_OUT_ARGS
1345 if (curArgTabEntry->node != node)
1349 GenTreePtr argx = NULL;
1350 unsigned regIndex = 0;
1352 /* process the register argument list */
1353 for (GenTreeArgList * list = callTree->gtCall.gtCallLateArgs; list; list = list->Rest(), regIndex++)
1355 argx = list->Current();
1356 assert(!argx->IsArgPlaceHolderNode()); // No place holders nodes are in gtCallLateArgs;
1357 if (regIndex == regArgInx)
1360 assert(regIndex == regArgInx);
1361 assert(regArgInx == curArgTabEntry->lateArgInx);
1363 if (curArgTabEntry->node != argx)
1365 curArgTabEntry->node = argx;
1370 assert(parent->Current() == node);
1371 curArgTabEntry->node = node;
1375 curArgTabEntry->node = node;
1378 nextSlotNum += numSlots;
1381 void fgArgInfo::SplitArg(unsigned argNum,
1385 fgArgTabEntryPtr curArgTabEntry = NULL;
1386 assert(argNum < argCount);
1387 for (unsigned inx=0; inx < argCount; inx++)
1389 curArgTabEntry = argTable[inx];
1390 if (curArgTabEntry->argNum == argNum)
1394 assert(numRegs > 0);
1395 assert(numSlots > 0);
1397 curArgTabEntry->isSplit = true;
1398 curArgTabEntry->numRegs = numRegs;
1399 curArgTabEntry->numSlots = numSlots;
1401 nextSlotNum += numSlots;
1404 void fgArgInfo::EvalToTmp(unsigned argNum,
1408 fgArgTabEntryPtr curArgTabEntry = NULL;
1409 assert(argNum < argCount);
1410 for (unsigned inx=0; inx < argCount; inx++)
1412 curArgTabEntry = argTable[inx];
1413 if (curArgTabEntry->argNum == argNum)
1416 assert(curArgTabEntry->parent->Current() == newNode);
1418 curArgTabEntry->node = newNode;
1419 curArgTabEntry->tmpNum = tmpNum;
1420 curArgTabEntry->isTmp = true;
1423 void fgArgInfo::ArgsComplete()
1425 bool hasStackArgs = false;
1426 bool hasStructRegArg = false;
1428 for (unsigned curInx = 0; curInx < argCount; curInx++)
1430 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1431 assert(curArgTabEntry != NULL);
1432 GenTreePtr argx = curArgTabEntry->node;
1434 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1435 // If this is a struct, mark it for needing a tempVar.
1436 // In the copyblk and store this should have minimal perf impact since
1437 // the local vars where we copy/store to already exist and the logic for temp
1438 // var will not create a new one if it creates a tempVar from another tempVar.
1439 // (Debugging through the code, there was no new copy of data created, neither a new tempVar.)
1440 // The need for this arise from Lower::LowerArg.
1441 // In case of copyblk and store operation, the NewPutArg method will
1442 // not be invoked and the struct will not be loaded to be passed in
1443 // registers or by value on the stack.
1444 if (varTypeIsStruct(argx) FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY( || curArgTabEntry->isStruct))
1446 curArgTabEntry->needTmp = true;
1448 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
1450 if (curArgTabEntry->regNum == REG_STK)
1452 hasStackArgs = true;
1453 #if !FEATURE_FIXED_OUT_ARGS
1454 // On x86 we use push instructions to pass arguments:
1455 // The non-register arguments are evaluated and pushed in order
1456 // and they are never evaluated into temps
1461 else // we have a register argument, next we look for a struct type.
1463 if (varTypeIsStruct(argx)
1464 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY( || curArgTabEntry->isStruct))
1466 hasStructRegArg = true;
1470 /* If the argument tree contains an assignment (GTF_ASG) then the argument and
1471 and every earlier argument (except constants) must be evaluated into temps
1472 since there may be other arguments that follow and they may use the value being assigned.
1474 EXAMPLE: ArgTab is "a, a=5, a"
1475 -> when we see the second arg "a=5"
1476 we know the first two arguments "a, a=5" have to be evaluated into temps
1478 For the case of an assignment, we only know that there exist some assignment someplace
1479 in the tree. We don't know what is being assigned so we are very conservative here
1480 and assume that any local variable could have been assigned.
1483 if (argx->gtFlags & GTF_ASG)
1485 // If this is not the only argument, or it's a copyblk, or it already evaluates the expression to
1486 // a tmp, then we need a temp in the late arg list.
1487 if ((argCount > 1) || argx->OperIsCopyBlkOp()
1488 #ifdef FEATURE_FIXED_OUT_ARGS
1489 || curArgTabEntry->isTmp // I protect this by "FEATURE_FIXED_OUT_ARGS" to preserve the property
1490 // that we only have late non-register args when that feature is on.
1491 #endif // FEATURE_FIXED_OUT_ARGS
1494 curArgTabEntry->needTmp = true;
1497 // For all previous arguments, unless they are a simple constant
1498 // we require that they be evaluated into temps
1499 for (unsigned prevInx = 0; prevInx < curInx; prevInx++)
1501 fgArgTabEntryPtr prevArgTabEntry = argTable[prevInx];
1502 assert(prevArgTabEntry->argNum < curArgTabEntry->argNum);
1504 assert(prevArgTabEntry->node);
1505 if (prevArgTabEntry->node->gtOper != GT_CNS_INT)
1507 prevArgTabEntry->needTmp = true;
1512 #if FEATURE_FIXED_OUT_ARGS
1513 // Like calls, if this argument has a tree that will do an inline throw,
1514 // a call to a jit helper, then we need to treat it like a call (but only
1515 // if there are/were any stack args).
1516 // This means unnesting, sorting, etc. Technically this is overly
1517 // conservative, but I want to avoid as much special-case debug-only code
1518 // as possible, so leveraging the GTF_CALL flag is the easiest.
1519 if (!(argx->gtFlags & GTF_CALL) &&
1520 (argx->gtFlags & GTF_EXCEPT) &&
1522 compiler->opts.compDbgCode &&
1523 (compiler->fgWalkTreePre(&argx, Compiler::fgChkThrowCB) == Compiler::WALK_ABORT))
1525 for (unsigned otherInx = 0; otherInx < argCount; otherInx++)
1527 if (otherInx == curInx)
1530 if (argTable[otherInx]->regNum == REG_STK)
1532 argx->gtFlags |= GTF_CALL;
1537 #endif // FEATURE_FIXED_OUT_ARGS
1539 /* If it contains a call (GTF_CALL) then itself and everything before the call
1540 with a GLOB_EFFECT must eval to temp (this is because everything with SIDE_EFFECT
1541 has to be kept in the right order since we will move the call to the first position)
1543 For calls we don't have to be quite as conservative as we are with an assignment
1544 since the call won't be modifying any non-address taken LclVars.
1547 if (argx->gtFlags & GTF_CALL)
1549 if (argCount > 1) // If this is not the only argument
1551 curArgTabEntry->needTmp = true;
1553 else if (varTypeIsFloating(argx->TypeGet()) && (argx->OperGet() == GT_CALL))
1555 // Spill all arguments that are floating point calls
1556 curArgTabEntry->needTmp = true;
1559 // All previous arguments may need to be evaluated into temps
1560 for (unsigned prevInx = 0; prevInx < curInx; prevInx++)
1562 fgArgTabEntryPtr prevArgTabEntry = argTable[prevInx];
1563 assert(prevArgTabEntry->argNum < curArgTabEntry->argNum);
1564 assert(prevArgTabEntry->node);
1566 // For all previous arguments, if they have any GTF_ALL_EFFECT
1567 // we require that they be evaluated into a temp
1568 if ((prevArgTabEntry->node->gtFlags & GTF_ALL_EFFECT) != 0)
1570 prevArgTabEntry->needTmp = true;
1572 #if FEATURE_FIXED_OUT_ARGS
1573 // Or, if they are stored into the FIXED_OUT_ARG area
1574 // we require that they be moved to the gtCallLateArgs
1575 // and replaced with a placeholder node
1576 else if (prevArgTabEntry->regNum == REG_STK)
1578 prevArgTabEntry->needPlace = true;
1584 #ifndef LEGACY_BACKEND
1585 // For RyuJIT backend we will expand a Multireg arg into a GT_LIST
1586 // with multiple indirections, so here we consider spilling it into a tmp LclVar.
1588 // Note that Arm32 is a LEGACY_BACKEND and it defines FEATURE_MULTIREG_ARGS
1589 // so we skip this for ARM32 until it is ported to use RyuJIT backend
1591 #if FEATURE_MULTIREG_ARGS
1592 if ((argx->TypeGet() == TYP_STRUCT) &&
1593 (curArgTabEntry->numRegs > 1) &&
1594 (curArgTabEntry->needTmp == false))
1596 if ((argx->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) != 0)
1598 // Spill multireg struct arguments that have Assignments or Calls embedded in them
1599 curArgTabEntry->needTmp = true;
1603 // We call gtPrepareCost to measure the cost of evaluating this tree
1604 compiler->gtPrepareCost(argx);
1606 if (argx->gtCostEx > (6 * IND_COST_EX))
1608 // Spill multireg struct arguments that are expensive to evaluate twice
1609 curArgTabEntry->needTmp = true;
1613 #endif // FEATURE_MULTIREG_ARGS
1614 #endif // LEGACY_BACKEND
1618 // We only care because we can't spill structs and qmarks involve a lot of spilling, but
1619 // if we don't have qmarks, then it doesn't matter.
1620 // So check for Qmark's globally once here, instead of inside the loop.
1622 const bool hasStructRegArgWeCareAbout = (hasStructRegArg && compiler->compQmarkUsed);
1624 #if FEATURE_FIXED_OUT_ARGS
1626 // For Arm/x64 we only care because we can't reorder a register
1627 // argument that uses GT_LCLHEAP. This is an optimization to
1628 // save a check inside the below loop.
1630 const bool hasStackArgsWeCareAbout = (hasStackArgs && compiler->compLocallocUsed);
1634 const bool hasStackArgsWeCareAbout = hasStackArgs;
1636 #endif // FEATURE_FIXED_OUT_ARGS
1638 // If we have any stack args we have to force the evaluation
1639 // of any arguments passed in registers that might throw an exception
1641 // Technically we only a required to handle the following two cases:
1642 // a GT_IND with GTF_IND_RNGCHK (only on x86) or
1643 // a GT_LCLHEAP node that allocates stuff on the stack
1645 if (hasStackArgsWeCareAbout || hasStructRegArgWeCareAbout)
1647 for (unsigned curInx = 0; curInx < argCount; curInx++)
1649 fgArgTabEntryPtr curArgTabEntry = argTable[curInx]; assert(curArgTabEntry != NULL);
1650 GenTreePtr argx = curArgTabEntry->node;
1652 // Examine the register args that are currently not marked needTmp
1654 if (!curArgTabEntry->needTmp && (curArgTabEntry->regNum != REG_STK))
1656 if (hasStackArgsWeCareAbout)
1658 #if !FEATURE_FIXED_OUT_ARGS
1659 // On x86 we previously recorded a stack depth of zero when
1660 // morphing the register arguments of any GT_IND with a GTF_IND_RNGCHK flag
1661 // Thus we can not reorder the argument after any stack based argument
1662 // (Note that GT_LCLHEAP sets the GTF_EXCEPT flag so we don't need to
1663 // check for it explicitly
1665 if (argx->gtFlags & GTF_EXCEPT)
1667 curArgTabEntry->needTmp = true;
1671 // For Arm/X64 we can't reorder a register argument that uses a GT_LCLHEAP
1673 if (argx->gtFlags & GTF_EXCEPT)
1675 assert(compiler->compLocallocUsed);
1677 // Returns WALK_ABORT if a GT_LCLHEAP node is encountered in the argx tree
1679 if (compiler->fgWalkTreePre(&argx, Compiler::fgChkLocAllocCB) == Compiler::WALK_ABORT)
1681 curArgTabEntry->needTmp = true;
1687 if (hasStructRegArgWeCareAbout)
1689 // Returns true if a GT_QMARK node is encountered in the argx tree
1691 if (compiler->fgWalkTreePre(&argx, Compiler::fgChkQmarkCB) == Compiler::WALK_ABORT)
1693 curArgTabEntry->needTmp = true;
1701 argsComplete = true;
1704 void fgArgInfo::SortArgs()
1706 assert(argsComplete == true);
1708 /* Shuffle the arguments around before we build the gtCallLateArgs list.
1709 The idea is to move all "simple" arguments like constants and local vars
1710 to the end of the table, and move the complex arguments towards the beginning
1711 of the table. This will help prevent registers from being spilled by
1712 allowing us to evaluate the more complex arguments before the simpler arguments.
1713 The argTable ends up looking like:
1714 +------------------------------------+ <--- argTable[argCount - 1]
1716 +------------------------------------+
1717 | local var / local field |
1718 +------------------------------------+
1719 | remaining arguments sorted by cost |
1720 +------------------------------------+
1721 | temps (argTable[].needTmp = true) |
1722 +------------------------------------+
1723 | args with calls (GTF_CALL) |
1724 +------------------------------------+ <--- argTable[0]
1728 if (compiler->verbose)
1730 printf("\nSorting the arguments:\n");
1734 /* Set the beginning and end for the new argument table */
1737 unsigned begTab = 0;
1738 unsigned endTab = argCount - 1;
1739 unsigned argsRemaining = argCount;
1741 // First take care of arguments that are constants.
1742 // [We use a backward iterator pattern]
1748 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1750 if (curArgTabEntry->regNum != REG_STK)
1753 // Skip any already processed args
1755 if (!curArgTabEntry->processed)
1757 GenTreePtr argx = curArgTabEntry->node;
1759 // put constants at the end of the table
1761 if (argx->gtOper == GT_CNS_INT)
1763 noway_assert(curInx <= endTab);
1765 curArgTabEntry->processed = true;
1767 // place curArgTabEntry at the endTab position by performing a swap
1769 if (curInx != endTab)
1771 argTable[curInx] = argTable[endTab];
1772 argTable[endTab] = curArgTabEntry;
1779 } while (curInx > 0);
1781 if (argsRemaining > 0)
1783 // Next take care of arguments that are calls.
1784 // [We use a forward iterator pattern]
1786 for (curInx = begTab; curInx <= endTab; curInx++)
1788 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1790 // Skip any already processed args
1792 if (!curArgTabEntry->processed)
1794 GenTreePtr argx = curArgTabEntry->node;
1796 // put calls at the beginning of the table
1798 if (argx->gtFlags & GTF_CALL)
1800 curArgTabEntry->processed = true;
1802 // place curArgTabEntry at the begTab position by performing a swap
1804 if (curInx != begTab)
1806 argTable[curInx] = argTable[begTab];
1807 argTable[begTab] = curArgTabEntry;
1817 if (argsRemaining > 0)
1819 // Next take care arguments that are temps.
1820 // These temps come before the arguments that are
1821 // ordinary local vars or local fields
1822 // since this will give them a better chance to become
1823 // enregistered into their actual argument register.
1824 // [We use a forward iterator pattern]
1826 for (curInx = begTab; curInx <= endTab; curInx++)
1828 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1830 // Skip any already processed args
1832 if (!curArgTabEntry->processed)
1834 if (curArgTabEntry->needTmp)
1836 curArgTabEntry->processed = true;
1838 // place curArgTabEntry at the begTab position by performing a swap
1840 if (curInx != begTab)
1842 argTable[curInx] = argTable[begTab];
1843 argTable[begTab] = curArgTabEntry;
1853 if (argsRemaining > 0)
1855 // Next take care of local var and local field arguments.
1856 // These are moved towards the end of the argument evaluation.
1857 // [We use a backward iterator pattern]
1859 curInx = endTab + 1;
1863 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1865 // Skip any already processed args
1867 if (!curArgTabEntry->processed)
1869 GenTreePtr argx = curArgTabEntry->node;
1871 if ((argx->gtOper == GT_LCL_VAR) || (argx->gtOper == GT_LCL_FLD))
1873 noway_assert(curInx <= endTab);
1875 curArgTabEntry->processed = true;
1877 // place curArgTabEntry at the endTab position by performing a swap
1879 if (curInx != endTab)
1881 argTable[curInx] = argTable[endTab];
1882 argTable[endTab] = curArgTabEntry;
1889 } while (curInx > begTab);
1892 // Finally, take care of all the remaining arguments.
1893 // Note that we fill in one arg at a time using a while loop.
1894 bool costsPrepared = false; // Only prepare tree costs once, the first time through this loop
1895 while (argsRemaining > 0)
1897 /* Find the most expensive arg remaining and evaluate it next */
1899 fgArgTabEntryPtr expensiveArgTabEntry = NULL;
1900 unsigned expensiveArg = UINT_MAX;
1901 unsigned expensiveArgCost = 0;
1903 // [We use a forward iterator pattern]
1905 for (curInx = begTab; curInx <= endTab; curInx++)
1907 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1909 // Skip any already processed args
1911 if (!curArgTabEntry->processed)
1913 GenTreePtr argx = curArgTabEntry->node;
1915 // We should have already handled these kinds of args
1916 assert (argx->gtOper != GT_LCL_VAR);
1917 assert (argx->gtOper != GT_LCL_FLD);
1918 assert (argx->gtOper != GT_CNS_INT);
1920 // This arg should either have no persistent side effects or be the last one in our table
1921 // assert(((argx->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) == 0) || (curInx == (argCount-1)));
1923 if (argsRemaining == 1)
1925 // This is the last arg to place
1926 expensiveArg = curInx;
1927 expensiveArgTabEntry = curArgTabEntry;
1928 assert(begTab == endTab);
1935 /* We call gtPrepareCost to measure the cost of evaluating this tree */
1936 compiler->gtPrepareCost(argx);
1939 if (argx->gtCostEx > expensiveArgCost)
1941 // Remember this arg as the most expensive one that we have yet seen
1942 expensiveArgCost = argx->gtCostEx;
1943 expensiveArg = curInx;
1944 expensiveArgTabEntry = curArgTabEntry;
1950 noway_assert(expensiveArg != UINT_MAX);
1952 // put the most expensive arg towards the beginning of the table
1954 expensiveArgTabEntry->processed = true;
1956 // place expensiveArgTabEntry at the begTab position by performing a swap
1958 if (expensiveArg != begTab)
1960 argTable[expensiveArg] = argTable[begTab];
1961 argTable[begTab] = expensiveArgTabEntry;
1967 costsPrepared = true; // If we have more expensive arguments, don't re-evaluate the tree cost on the next loop
1970 // The table should now be completely filled and thus begTab should now be adjacent to endTab
1971 // and regArgsRemaining should be zero
1972 assert(begTab == (endTab + 1));
1973 assert(argsRemaining == 0);
1975 #if !FEATURE_FIXED_OUT_ARGS
1976 // Finally build the regArgList
1978 callTree->gtCall.regArgList = NULL;
1979 callTree->gtCall.regArgListCount = regCount;
1981 unsigned regInx = 0;
1982 for (curInx = 0; curInx < argCount; curInx++)
1984 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
1986 if (curArgTabEntry->regNum != REG_STK)
1988 // Encode the argument register in the register mask
1990 callTree->gtCall.regArgList[regInx] = curArgTabEntry->regNum;
1994 #endif // !FEATURE_FIXED_OUT_ARGS
1999 //------------------------------------------------------------------------------
2000 // fgMakeTmpArgNode : This function creates a tmp var only if needed.
2001 // We need this to be done in order to enforce ordering
2002 // of the evaluation of arguments.
2005 // tmpVarNum - the var num which we clone into the newly created temp var.
2008 // the newly created temp var tree.
2010 GenTreePtr Compiler::fgMakeTmpArgNode(unsigned tmpVarNum
2011 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(const bool passedInRegisters))
2013 LclVarDsc * varDsc = &lvaTable[tmpVarNum];
2014 assert(varDsc->lvIsTemp);
2015 var_types type = varDsc->TypeGet();
2017 // Create a copy of the temp to go into the late argument list
2018 GenTreePtr arg = gtNewLclvNode(tmpVarNum, type);
2019 GenTreePtr addrNode = nullptr;
2021 if (varTypeIsStruct(type))
2024 #if defined(_TARGET_AMD64_) || defined(_TARGET_ARM64_)
2026 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
2028 arg->gtFlags |= GTF_DONT_CSE;
2030 // If it is passed in registers, don't get the address of the var. Make it a
2031 // field instead. It will be loaded in registers with putarg_reg tree in lower.
2032 if (passedInRegisters)
2034 arg->ChangeOper(GT_LCL_FLD);
2039 arg = gtNewOperNode(GT_ADDR, type, arg);
2043 #else // !FEATURE_UNIX_AMD64_STRUCT_PASSING
2045 unsigned structSize = lvaLclExactSize(tmpVarNum);
2049 case 1: type = TYP_BYTE; break;
2050 case 2: type = TYP_SHORT; break;
2051 #if defined (_TARGET_AMD64_)
2052 case 4: type = TYP_INT; break;
2053 #elif defined(_TARGET_ARM64_)
2055 case 4: type = TYP_INT; break;
2058 case 7: type = TYP_I_IMPL; break;
2059 #endif // defined (_TARGET_ARM64_)
2061 switch (*lvaGetGcLayout(tmpVarNum))
2080 // If we didn't change the type of the struct, it means
2081 // its structure doesn't support to be passed directly through a
2082 // register, so we need to pass a pointer to the destination where
2083 // where we copied the struct to.
2084 if (type == varDsc->TypeGet())
2086 #if FEATURE_MULTIREG_ARGS
2087 #ifdef _TARGET_ARM64_
2088 assert(varTypeIsStruct(type));
2089 if (varDsc->lvIsMultiregStruct())
2091 // ToDo-ARM64: Consider using: arg->ChangeOper(GT_LCL_FLD);
2092 // as that is how FEATURE_UNIX_AMD64_STRUCT_PASSING works.
2093 // Create a GT_OBJ for the argument
2094 // This will be passed by value in two registers
2095 arg = gtNewOperNode(GT_ADDR, TYP_BYREF, arg);
2098 // Create an Obj of the temp to use it as a call argument.
2099 arg = gtNewObjNode(lvaGetStruct(tmpVarNum), arg);
2102 #endif // _TARGET_ARM64_
2103 #endif // FEATURE_MULTIREG_ARGS
2105 arg = gtNewOperNode(GT_ADDR, TYP_I_IMPL, arg);
2109 else // type was changed from a struct to a scalar type
2111 arg->ChangeOper(GT_LCL_FLD);
2114 #endif // !FEATURE_UNIX_AMD64_STRUCT_PASSING
2116 #else // not (_TARGET_AMD64_ or _TARGET_ARM64_)
2118 // other targets, we pass the struct by value
2119 assert(varTypeIsStruct(type));
2121 arg = gtNewOperNode(GT_ADDR, TYP_BYREF, arg);
2124 // Get a new Obj node temp to use it as a call argument
2125 arg = gtNewObjNode(lvaGetStruct(tmpVarNum), arg);
2126 arg->gtFlags |= GTF_EXCEPT;
2128 #endif // not (_TARGET_AMD64_ or _TARGET_ARM64_)
2130 } // (varTypeIsStruct(type))
2132 if (addrNode != nullptr)
2134 assert(addrNode->gtOper == GT_ADDR);
2136 // This will prevent this LclVar from being optimized away
2137 lvaSetVarAddrExposed(tmpVarNum);
2139 // the child of a GT_ADDR is required to have this flag set
2140 addrNode->gtOp.gtOp1->gtFlags |= GTF_DONT_CSE;
2146 void fgArgInfo::EvalArgsToTemps()
2148 assert(argsSorted == true);
2150 unsigned regArgInx = 0;
2151 // Now go through the argument table and perform the necessary evaluation into temps
2152 GenTreeArgList* tmpRegArgNext = NULL;
2153 for (unsigned curInx = 0; curInx < argCount; curInx++)
2155 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
2157 GenTreePtr argx = curArgTabEntry->node;
2158 GenTreePtr setupArg = NULL;
2161 #if !FEATURE_FIXED_OUT_ARGS
2162 // Only ever set for FEATURE_FIXED_OUT_ARGS
2163 assert(curArgTabEntry->needPlace == false);
2165 // On x86 and other archs that use push instructions to pass arguments:
2166 // Only the register arguments need to be replaced with placeholder nodes.
2167 // Stacked arguments are evaluated and pushed (or stored into the stack) in order.
2169 if (curArgTabEntry->regNum == REG_STK)
2173 if (curArgTabEntry->needTmp)
2177 if (curArgTabEntry->isTmp == true)
2179 // Create a copy of the temp to go into the late argument list
2180 tmpVarNum = curArgTabEntry->tmpNum;
2181 defArg = compiler->fgMakeTmpArgNode(
2183 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(argTable[curInx]->structDesc.passedInRegisters));
2185 // mark the original node as a late argument
2186 argx->gtFlags |= GTF_LATE_ARG;
2190 // Create a temp assignment for the argument
2191 // Put the temp in the gtCallLateArgs list
2193 if (compiler->verbose)
2195 printf("Argument with 'side effect'...\n");
2196 compiler->gtDispTree(argx);
2200 #if defined(_TARGET_AMD64_) && !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
2201 noway_assert(argx->gtType != TYP_STRUCT);
2204 tmpVarNum = compiler->lvaGrabTemp(true DEBUGARG("argument with side effect"));
2205 if (argx->gtOper == GT_MKREFANY)
2207 // For GT_MKREFANY, typically the actual struct copying does
2208 // not have any side-effects and can be delayed. So instead
2209 // of using a temp for the whole struct, we can just use a temp
2210 // for operand that that has a side-effect
2212 if ((argx->gtOp.gtOp2->gtFlags & GTF_ALL_EFFECT) == 0)
2214 operand = argx->gtOp.gtOp1;
2216 // In the early argument evaluation, place an assignment to the temp
2217 // from the source operand of the mkrefany
2218 setupArg = compiler->gtNewTempAssign(tmpVarNum, operand);
2220 // Replace the operand for the mkrefany with the new temp.
2221 argx->gtOp.gtOp1 = compiler->gtNewLclvNode(tmpVarNum, operand->TypeGet());
2223 else if ((argx->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT) == 0)
2225 operand = argx->gtOp.gtOp2;
2227 // In the early argument evaluation, place an assignment to the temp
2228 // from the source operand of the mkrefany
2229 setupArg = compiler->gtNewTempAssign(tmpVarNum, operand);
2231 // Replace the operand for the mkrefany with the new temp.
2232 argx->gtOp.gtOp2 = compiler->gtNewLclvNode(tmpVarNum, operand->TypeGet());
2236 if (setupArg != NULL)
2238 // Now keep the mkrefany for the late argument list
2241 // Clear the side-effect flags because now both op1 and op2 have no side-effects
2242 defArg->gtFlags &= ~GTF_ALL_EFFECT;
2246 setupArg = compiler->gtNewTempAssign(tmpVarNum, argx);
2248 #ifndef LEGACY_BACKEND
2249 if (compiler->fgOrder == Compiler::FGOrderLinear)
2251 // We'll reference this temporary variable just once
2252 // when we perform the function call after
2253 // setting up this argument.
2254 LclVarDsc* varDsc = compiler->lvaTable + tmpVarNum;
2255 varDsc->lvRefCnt = 1;
2257 #endif // !LEGACY_BACKEND
2259 if (setupArg->OperIsCopyBlkOp())
2260 setupArg = compiler->fgMorphCopyBlock(setupArg);
2262 /* Create a copy of the temp to go to the late argument list */
2263 defArg = compiler->gtNewLclvNode(tmpVarNum, genActualType(argx->gtType));
2265 curArgTabEntry->isTmp = true;
2266 curArgTabEntry->tmpNum = tmpVarNum;
2269 // Previously we might have thought the local was promoted, and thus the 'COPYBLK'
2270 // might have left holes in the used registers (see
2271 // fgAddSkippedRegsInPromotedStructArg).
2272 // Too bad we're not that smart for these intermediate temps...
2273 if (isValidIntArgReg(curArgTabEntry->regNum) && (curArgTabEntry->numRegs > 1))
2275 regNumber argReg = curArgTabEntry->regNum;
2276 regMaskTP allUsedRegs = genRegMask(curArgTabEntry->regNum);
2277 for (unsigned i = 1; i < curArgTabEntry->numRegs; i++)
2279 argReg = genRegArgNext(argReg);
2280 allUsedRegs |= genRegMask(argReg);
2282 callTree->gtCall.gtCallRegUsedMask |= allUsedRegs;
2284 #endif // _TARGET_ARM_
2287 /* mark the assignment as a late argument */
2288 setupArg->gtFlags |= GTF_LATE_ARG;
2291 if (compiler->verbose)
2293 printf("\n Evaluate to a temp:\n");
2294 compiler->gtDispTree(setupArg);
2299 else // curArgTabEntry->needTmp == false
2302 // Only register args are replaced with placeholder nodes
2303 // and the stack based arguments are evaluated and pushed in order.
2305 // On Arm/x64 - When needTmp is false and needPlace is false,
2306 // the non-register arguments are evaluated and stored in order.
2307 // When needPlace is true we have a nested call that comes after
2308 // this argument so we have to replace it in the gtCallArgs list
2309 // (the initial argument evaluation list) with a placeholder.
2311 if ((curArgTabEntry->regNum == REG_STK) && (curArgTabEntry->needPlace == false))
2314 /* No temp needed - move the whole node to the gtCallLateArgs list */
2316 /* The argument is deferred and put in the late argument list */
2320 // Create a placeholder node to put in its place in gtCallLateArgs.
2322 // For a struct type we also need to record the class handle of the arg.
2323 CORINFO_CLASS_HANDLE clsHnd = NO_CLASS_HANDLE;
2325 #if defined(_TARGET_AMD64_) && !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
2327 // All structs are either passed (and retyped) as integral types, OR they
2328 // are passed by reference.
2329 noway_assert(argx->gtType != TYP_STRUCT);
2331 #else // !defined(_TARGET_AMD64_) || defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
2333 if (varTypeIsStruct(defArg))
2335 // Need a temp to walk any GT_COMMA nodes when searching for the clsHnd
2336 GenTreePtr defArgTmp = defArg;
2338 // The GT_OBJ may be be a child of a GT_COMMA.
2339 while (defArgTmp->gtOper == GT_COMMA)
2341 defArgTmp = defArgTmp->gtOp.gtOp2;
2343 assert(varTypeIsStruct(defArgTmp));
2345 // We handle two opcodes: GT_MKREFANY and GT_OBJ.
2346 if (defArgTmp->gtOper == GT_MKREFANY)
2348 clsHnd = compiler->impGetRefAnyClass();
2350 else if (defArgTmp->gtOper == GT_OBJ)
2352 clsHnd = defArgTmp->AsObj()->gtClass;
2356 BADCODE("Unhandled struct argument tree in fgMorphArgs");
2360 #endif // !(defined(_TARGET_AMD64_) && !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING))
2362 setupArg = compiler->gtNewArgPlaceHolderNode(defArg->gtType, clsHnd);
2364 /* mark the placeholder node as a late argument */
2365 setupArg->gtFlags |= GTF_LATE_ARG;
2368 if (compiler->verbose)
2370 if (curArgTabEntry->regNum == REG_STK)
2372 printf("Deferred stack argument :\n");
2376 printf("Deferred argument ('%s'):\n", getRegName(curArgTabEntry->regNum));
2379 compiler->gtDispTree(argx);
2380 printf("Replaced with placeholder node:\n");
2381 compiler->gtDispTree(setupArg);
2386 if (setupArg != NULL)
2388 if (curArgTabEntry->parent)
2390 GenTreePtr parent = curArgTabEntry->parent;
2391 /* a normal argument from the list */
2392 noway_assert(parent->IsList());
2393 noway_assert(parent->gtOp.gtOp1 == argx);
2395 parent->gtOp.gtOp1 = setupArg;
2399 /* must be the gtCallObjp */
2400 noway_assert(callTree->gtCall.gtCallObjp == argx);
2402 callTree->gtCall.gtCallObjp = setupArg;
2406 /* deferred arg goes into the late argument list */
2408 if (tmpRegArgNext == NULL)
2410 tmpRegArgNext = compiler->gtNewArgList(defArg);
2411 callTree->gtCall.gtCallLateArgs = tmpRegArgNext;
2415 noway_assert(tmpRegArgNext->IsList());
2416 noway_assert(tmpRegArgNext->Current());
2417 tmpRegArgNext->gtOp.gtOp2 = compiler->gtNewArgList(defArg);
2418 tmpRegArgNext = tmpRegArgNext->Rest();
2421 curArgTabEntry->node = defArg;
2422 curArgTabEntry->lateArgInx = regArgInx++;
2426 if (compiler->verbose)
2428 printf("\nShuffled argument table: ");
2429 for (unsigned curInx = 0; curInx < argCount; curInx++)
2431 fgArgTabEntryPtr curArgTabEntry = argTable[curInx];
2433 if (curArgTabEntry->regNum != REG_STK)
2435 printf("%s ", getRegName( curArgTabEntry->regNum ));
2443 void fgArgInfo::RecordStkLevel(unsigned stkLvl)
2445 assert(!IsUninitialized(stkLvl));
2446 this->stkLevel = stkLvl;
2449 unsigned fgArgInfo::RetrieveStkLevel()
2451 assert(!IsUninitialized(stkLevel));
2455 // Return a conservative estimate of the stack size in bytes.
2456 // It will be used only on the intercepted-for-host code path to copy the arguments.
2457 int Compiler::fgEstimateCallStackSize(GenTreeCall* call)
2461 for (GenTreeArgList* args = call->gtCallArgs; args; args = args->Rest())
2467 if (numArgs > MAX_REG_ARG)
2468 numStkArgs = numArgs - MAX_REG_ARG;
2472 return numStkArgs * REGSIZE_BYTES;
2475 //------------------------------------------------------------------------------
2476 // fgMakeMultiUse : If the node is a local, clone it and increase the ref count
2477 // otherwise insert a comma form temp
2480 // ppTree - a pointer to the child node we will be replacing with the comma expression that
2481 // evaluates ppTree to a temp and returns the result
2484 // A fresh GT_LCL_VAR node referencing the temp which has not been used
2487 // The result tree MUST be added to the tree structure since the ref counts are
2488 // already incremented.
2490 GenTree* Compiler::fgMakeMultiUse(GenTree** pOp)
2492 GenTree* tree = *pOp;
2493 if (tree->IsLocal())
2495 auto result = gtClone(tree);
2496 if (lvaLocalVarRefCounted)
2498 lvaTable[tree->gtLclVarCommon.gtLclNum].incRefCnts(compCurBB->getBBWeight(this), this);
2504 GenTree* result = fgInsertCommaFormTemp(pOp);
2506 // At this point, *pOp is GT_COMMA(GT_ASG(V01, *pOp), V01) and result = V01
2507 // Therefore, the ref count has to be incremented 3 times for *pOp and result, if result will
2508 // be added by the caller.
2509 if (lvaLocalVarRefCounted)
2511 lvaTable[result->gtLclVarCommon.gtLclNum].incRefCnts(compCurBB->getBBWeight(this), this);
2512 lvaTable[result->gtLclVarCommon.gtLclNum].incRefCnts(compCurBB->getBBWeight(this), this);
2513 lvaTable[result->gtLclVarCommon.gtLclNum].incRefCnts(compCurBB->getBBWeight(this), this);
2521 //------------------------------------------------------------------------------
2522 // fgInsertCommaFormTemp: Create a new temporary variable to hold the result of *ppTree,
2523 // and replace *ppTree with comma(asg(newLcl, *ppTree), newLcl)
2526 // ppTree - a pointer to the child node we will be replacing with the comma expression that
2527 // evaluates ppTree to a temp and returns the result
2529 // structType - value type handle if the temp created is of TYP_STRUCT.
2532 // A fresh GT_LCL_VAR node referencing the temp which has not been used
2535 GenTree* Compiler::fgInsertCommaFormTemp(GenTree** ppTree, CORINFO_CLASS_HANDLE structType /*= nullptr*/)
2537 GenTree* subTree = *ppTree;
2539 unsigned lclNum = lvaGrabTemp(true DEBUGARG("fgInsertCommaFormTemp is creating a new local variable"));
2541 if (varTypeIsStruct(subTree))
2543 assert(structType != nullptr);
2544 lvaSetStruct(lclNum, structType, false);
2547 // If subTree->TypeGet() == TYP_STRUCT, gtNewTempAssign() will create a GT_COPYBLK tree.
2548 // The type of GT_COPYBLK is TYP_VOID. Therefore, we should use subTree->TypeGet() for
2549 // setting type of lcl vars created.
2550 GenTree* asg = gtNewTempAssign(lclNum, subTree);
2552 GenTree* load = new (this, GT_LCL_VAR) GenTreeLclVar(subTree->TypeGet(), lclNum, BAD_IL_OFFSET);
2554 GenTree* comma = gtNewOperNode(GT_COMMA, subTree->TypeGet(), asg, load);
2558 return new (this, GT_LCL_VAR) GenTreeLclVar(subTree->TypeGet(), lclNum, BAD_IL_OFFSET);
2562 //------------------------------------------------------------------------
2563 // fgMorphArgs: Walk and transform (morph) the arguments of a call
2566 // callNode - the call for which we are doing the argument morphing
2569 // Like most morph methods, this method returns the morphed node,
2570 // though in this case there are currently no scenarios where the
2571 // node itself is re-created.
2574 // This method is even less idempotent than most morph methods.
2575 // That is, it makes changes that should not be redone. It uses the existence
2576 // of gtCallLateArgs (the late arguments list) to determine if it has
2577 // already done that work.
2579 // The first time it is called (i.e. during global morphing), this method
2580 // computes the "late arguments". This is when it determines which arguments
2581 // need to be evaluated to temps prior to the main argument setup, and which
2582 // can be directly evaluated into the argument location. It also creates a
2583 // second argument list (gtCallLateArgs) that does the final placement of the
2584 // arguments, e.g. into registers or onto the stack.
2586 // The "non-late arguments", aka the gtCallArgs, are doing the in-order
2587 // evaluation of the arguments that might have side-effects, such as embedded
2588 // assignments, calls or possible throws. In these cases, it and earlier
2589 // arguments must be evaluated to temps.
2591 // On targets with a fixed outgoing argument area (FEATURE_FIXED_OUT_ARGS),
2592 // if we have any nested calls, we need to defer the copying of the argument
2593 // into the fixed argument area until after the call. If the argument did not
2594 // otherwise need to be computed into a temp, it is moved to gtCallLateArgs and
2595 // replaced in the "early" arg list (gtCallArgs) with a placeholder node.
2598 #pragma warning(push)
2599 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
2601 GenTreeCall* Compiler::fgMorphArgs(GenTreeCall* callNode)
2603 GenTreeCall* call = callNode->AsCall();
2608 unsigned flagsSummary = 0;
2609 unsigned genPtrArgCntSav = fgPtrArgCntCur;
2611 unsigned argIndex = 0;
2613 unsigned intArgRegNum = 0;
2614 unsigned fltArgRegNum = 0;
2616 regMaskTP argSkippedRegMask = RBM_NONE;
2618 #if defined(_TARGET_ARM_) || defined(_TARGET_AMD64_)
2619 regMaskTP fltArgSkippedRegMask = RBM_NONE;
2622 #if defined(_TARGET_X86_)
2623 unsigned maxRegArgs = MAX_REG_ARG; // X86: non-const, must be calculated
2625 const unsigned maxRegArgs = MAX_REG_ARG; // other arch: fixed constant number
2628 unsigned argSlots = 0;
2629 unsigned nonRegPassedStructSlots = 0;
2630 bool lateArgsComputed = (call->gtCallLateArgs != nullptr);
2631 bool callHasRetBuffArg = call->HasRetBufArg();
2633 #ifndef _TARGET_X86_ // i.e. _TARGET_AMD64_ or _TARGET_ARM_
2634 bool callIsVararg = call->IsVarargs();
2637 bool hasNonStandardArg = false;
2638 #ifndef LEGACY_BACKEND
2639 // data structure for keeping track of non-standard args we insert
2640 // (args that have a special meaning and are not passed following the normal
2641 // calling convention or even in the normal arg regs.
2642 struct NonStandardArg
2648 ArrayStack<NonStandardArg> nonStandardArgs(this, 2);
2649 #endif // !LEGACY_BACKEND
2651 // Process the late arguments (which were determined by a previous caller).
2652 // Do this before resetting fgPtrArgCntCur as fgMorphTree(call->gtCallLateArgs)
2653 // may need to refer to it.
2654 if (lateArgsComputed)
2656 // We need to reMorph the gtCallLateArgs early since that is what triggers
2657 // the expression folding and we need to have the final folded gtCallLateArgs
2658 // available when we call RemorphRegArg so that we correctly update the fgArgInfo
2659 // with the folded tree that represents the final optimized argument nodes.
2661 // However if a range-check needs to be generated for any of these late
2662 // arguments we also need to "know" what the stack depth will be when we generate
2663 // code to branch to the throw range check failure block as that is part of the
2664 // GC information contract for that block.
2666 // Since the late arguments are evaluated last we have pushed all of the
2667 // other arguments on the stack before we evaluate these late arguments,
2668 // so we record the stack depth on the first morph call when lateArgsComputed
2669 // was false (via RecordStkLevel) and then retrieve that value here (via RetrieveStkLevel)
2671 unsigned callStkLevel = call->fgArgInfo->RetrieveStkLevel();
2672 fgPtrArgCntCur += callStkLevel;
2673 call->gtCallLateArgs = fgMorphTree(call->gtCallLateArgs)->AsArgList();
2674 flagsSummary |= call->gtCallLateArgs->gtFlags;
2675 fgPtrArgCntCur -= callStkLevel;
2676 assert(call->fgArgInfo != nullptr);
2677 call->fgArgInfo->RemorphReset();
2681 // First we need to count the args
2682 unsigned numArgs = 0;
2683 if (call->gtCallObjp)
2685 for (args = call->gtCallArgs; (args != nullptr); args = args->gtOp.gtOp2)
2691 // insert nonstandard args (outside the calling convention)
2693 #if !defined(LEGACY_BACKEND) && !defined(_TARGET_X86_)
2694 // TODO-X86-CQ: Currently RyuJIT/x86 passes args on the stack, so this is not needed.
2695 // If/when we change that, the following code needs to be changed to correctly support the (TBD) managed calling
2696 // convention for x86/SSE.
2697 if (!lateArgsComputed)
2699 if (call->IsUnmanaged() && !opts.ShouldUsePInvokeHelpers())
2701 assert(!call->gtCallCookie);
2702 // Add a conservative estimate of the stack size in a special parameter (r11) at the call site.
2703 // It will be used only on the intercepted-for-host code path to copy the arguments.
2705 GenTree* cns = new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, fgEstimateCallStackSize(call));
2706 call->gtCallArgs = gtNewListNode(cns, call->gtCallArgs);
2707 NonStandardArg nsa = {REG_PINVOKE_COOKIE_PARAM, cns};
2710 nonStandardArgs.Push(nsa);
2712 else if (call->IsVirtualStub() &&
2713 (call->gtCallType == CT_INDIRECT) &&
2714 !call->IsTailCallViaHelper())
2716 // indirect VSD stubs need the base of the indirection cell to be
2717 // passed in addition. At this point that is the value in gtCallAddr.
2718 // The actual call target will be derived from gtCallAddr in call
2721 // If it is a VSD call getting dispatched via tail call helper,
2722 // fgMorphTailCall() would materialize stub addr as an additional
2723 // parameter added to the original arg list and hence no need to
2724 // add as a non-standard arg.
2726 GenTree* arg = call->gtCallAddr;
2727 if (arg->OperIsLocal())
2729 arg = gtClone(arg, true);
2733 call->gtCallAddr = fgInsertCommaFormTemp(&arg);
2734 call->gtFlags |= GTF_ASG;
2736 noway_assert(arg != nullptr);
2738 // And push the stub address onto the list of arguments
2739 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
2742 NonStandardArg nsa = {REG_VIRTUAL_STUB_PARAM, arg};
2744 nonStandardArgs.Push(nsa);
2746 else if (call->gtCallType == CT_INDIRECT && call->gtCallCookie)
2748 assert(!call->IsUnmanaged());
2750 // put cookie into R11
2751 GenTree* arg = call->gtCallCookie;
2752 noway_assert(arg != nullptr);
2753 call->gtCallCookie = nullptr;
2755 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
2758 NonStandardArg nsa = {REG_PINVOKE_COOKIE_PARAM, arg};
2760 nonStandardArgs.Push(nsa);
2762 // put destination into R10
2763 arg = gtClone(call->gtCallAddr, true);
2764 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
2767 NonStandardArg nsa2 = {REG_PINVOKE_TARGET_PARAM, arg};
2768 nonStandardArgs.Push(nsa2);
2770 // finally change this call to a helper call
2771 call->gtCallType = CT_HELPER;
2772 call->gtCallMethHnd = eeFindHelper(CORINFO_HELP_PINVOKE_CALLI);
2775 #endif // !defined(LEGACY_BACKEND) && !defined(_TARGET_X86_)
2777 // Allocate the fgArgInfo for the call node;
2779 call->fgArgInfo = new (this, CMK_Unknown) fgArgInfo(this, call, numArgs);
2783 fgFixupStructReturn(call);
2785 /* First we morph the argument subtrees ('this' pointer, arguments, etc.).
2786 * During the first call to fgMorphArgs we also record the
2787 * information about late arguments we have in 'fgArgInfo'.
2788 * This information is used later to contruct the gtCallLateArgs */
2790 /* Process the 'this' argument value, if present */
2792 argx = call->gtCallObjp;
2796 argx = fgMorphTree(argx);
2797 call->gtCallObjp = argx;
2798 flagsSummary |= argx->gtFlags;
2800 assert(call->gtCallType == CT_USER_FUNC ||
2801 call->gtCallType == CT_INDIRECT);
2803 assert(argIndex == 0);
2805 /* We must fill in or update the argInfo table */
2807 if (!lateArgsComputed)
2809 assert(varTypeIsGC(call->gtCallObjp->gtType) ||
2810 (call->gtCallObjp->gtType == TYP_I_IMPL));
2812 /* this is a register argument - put it in the table */
2813 call->fgArgInfo->AddRegArg(argIndex,
2816 genMapIntRegArgNumToRegNum(intArgRegNum),
2819 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
2823 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
2828 /* this is a register argument - possibly update it in the table */
2829 call->fgArgInfo->RemorphRegArg(argIndex, argx, NULL, genMapIntRegArgNumToRegNum(intArgRegNum), 1, 1);
2831 // this can't be a struct.
2832 assert(argx->gtType != TYP_STRUCT);
2834 // FIXME: Issue #4025 Why do we need floating type for 'this' argument
2835 /* Increment the argument register count and argument index */
2836 if (!varTypeIsFloating(argx->gtType) || opts.compUseSoftFP)
2839 #if defined(_TARGET_AMD64_) && !defined(UNIX_AMD64_ABI)
2840 fltArgSkippedRegMask |= genMapArgNumToRegMask(fltArgRegNum, TYP_FLOAT);
2847 #if defined(_TARGET_AMD64_) && !defined(UNIX_AMD64_ABI)
2848 argSkippedRegMask |= genMapArgNumToRegMask(intArgRegNum, TYP_I_IMPL);
2857 // Compute the maximum number of arguments that can be passed in registers.
2858 // For X86 we handle the varargs and unmanaged calling conventions
2860 if (call->gtFlags & GTF_CALL_POP_ARGS)
2862 noway_assert(intArgRegNum < MAX_REG_ARG);
2863 // No more register arguments for varargs (CALL_POP_ARGS)
2864 maxRegArgs = intArgRegNum;
2866 // Add in the ret buff arg
2867 if (callHasRetBuffArg)
2871 if (call->IsUnmanaged())
2873 noway_assert(intArgRegNum == 0);
2875 if (call->gtCallMoreFlags & GTF_CALL_M_UNMGD_THISCALL)
2877 noway_assert(call->gtCallArgs->gtOp.gtOp1->TypeGet() == TYP_I_IMPL ||
2878 call->gtCallArgs->gtOp.gtOp1->TypeGet() == TYP_BYREF ||
2879 call->gtCallArgs->gtOp.gtOp1->gtOper == GT_NOP); // the arg was already morphed to a register (fgMorph called twice)
2887 // Add in the ret buff arg
2888 if (callHasRetBuffArg)
2891 #endif // _TARGET_X86_
2893 /* Morph the user arguments */
2895 #if defined(_TARGET_ARM_)
2897 // The ARM ABI has a concept of back-filling of floating-point argument registers, according
2898 // to the "Procedure Call Standard for the ARM Architecture" document, especially
2899 // section 6.1.2.3 "Parameter passing". Back-filling is where floating-point argument N+1 can
2900 // appear in a lower-numbered register than floating point argument N. That is, argument
2901 // register allocation is not strictly increasing. To support this, we need to keep track of unused
2902 // floating-point argument registers that we can back-fill. We only support 4-byte float and
2903 // 8-byte double types, and one to four element HFAs composed of these types. With this, we will
2904 // only back-fill single registers, since there is no way with these types to create
2905 // an alignment hole greater than one register. However, there can be up to 3 back-fill slots
2906 // available (with 16 FP argument registers). Consider this code:
2908 // struct HFA { float x, y, z; }; // a three element HFA
2909 // void bar(float a1, // passed in f0
2910 // double a2, // passed in f2/f3; skip f1 for alignment
2911 // HFA a3, // passed in f4/f5/f6
2912 // double a4, // passed in f8/f9; skip f7 for alignment. NOTE: it doesn't fit in the f1 back-fill slot
2913 // HFA a5, // passed in f10/f11/f12
2914 // double a6, // passed in f14/f15; skip f13 for alignment. NOTE: it doesn't fit in the f1 or f7 back-fill slots
2915 // float a7, // passed in f1 (back-filled)
2916 // float a8, // passed in f7 (back-filled)
2917 // float a9, // passed in f13 (back-filled)
2918 // float a10) // passed on the stack in [OutArg+0]
2920 // Note that if we ever support FP types with larger alignment requirements, then there could
2921 // be more than single register back-fills.
2923 // Once we assign a floating-pointer register to the stack, they all must be on the stack.
2924 // See "Procedure Call Standard for the ARM Architecture", section 6.1.2.3, "The back-filling
2925 // continues only so long as no VFP CPRC has been allocated to a slot on the stack."
2926 // We set anyFloatStackArgs to true when a floating-point argument has been assigned to the stack
2927 // and prevent any additional floating-point arguments from going in registers.
2929 bool anyFloatStackArgs = false;
2931 #endif // _TARGET_ARM_
2933 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
2934 bool nonRegPassableStruct = false;
2935 SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR structDesc;
2936 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
2938 bool hasStructArgument = false; // @TODO-ARM64-UNIX: Eemove this bool during a future refactoring
2939 bool hasMultiregStructArgs = false;
2940 for (args = call->gtCallArgs; args; args = args->gtOp.gtOp2)
2942 GenTreePtr * parentArgx = &args->gtOp.gtOp1;
2944 #if FEATURE_MULTIREG_ARGS
2945 if (!hasStructArgument)
2947 hasStructArgument = varTypeIsStruct(args->gtOp.gtOp1);
2949 #endif // FEATURE_MULTIREG_ARGS
2951 argx = fgMorphTree(*parentArgx);
2953 flagsSummary |= argx->gtFlags;
2955 assert(args->IsList());
2956 assert(argx == args->Current());
2958 /* Change the node to TYP_I_IMPL so we don't report GC info
2959 * NOTE: We deferred this from the importer because of the inliner */
2961 if (argx->IsVarAddr())
2962 argx->gtType = TYP_I_IMPL;
2964 bool passUsingFloatRegs;
2965 unsigned argAlign = 1;
2968 var_types hfaType = GetHfaType(argx);
2969 bool isHfaArg = varTypeIsFloating(hfaType);
2970 #endif // _TARGET_ARM_
2973 CORINFO_CLASS_HANDLE copyBlkClass = NULL;
2974 bool isRegArg = false;
2976 fgArgTabEntryPtr argEntry = NULL;
2978 if (lateArgsComputed)
2980 argEntry = gtArgEntryByArgNum(call, argIndex);
2985 bool passUsingIntRegs;
2986 if (lateArgsComputed)
2988 passUsingFloatRegs = isValidFloatArgReg(argEntry->regNum);
2989 passUsingIntRegs = isValidIntArgReg(argEntry->regNum);
2993 passUsingFloatRegs = !callIsVararg && (isHfaArg || varTypeIsFloating(argx)) && !opts.compUseSoftFP;
2994 passUsingIntRegs = passUsingFloatRegs ? false : (intArgRegNum < MAX_REG_ARG);
2997 GenTreePtr curArg = argx;
2998 // If late args have already been computed, use the node in the argument table.
2999 if (argEntry != NULL && argEntry->isTmp)
3001 curArg = argEntry->node;
3004 // We don't use the "size" return value from InferOpSizeAlign().
3005 codeGen->InferOpSizeAlign(curArg, &argAlign);
3007 argAlign = roundUp(argAlign, TARGET_POINTER_SIZE);
3008 argAlign /= TARGET_POINTER_SIZE;
3012 if (passUsingFloatRegs)
3014 if (fltArgRegNum % 2 == 1)
3016 fltArgSkippedRegMask |= genMapArgNumToRegMask(fltArgRegNum, TYP_FLOAT);
3020 else if (passUsingIntRegs)
3022 if (intArgRegNum % 2 == 1)
3024 argSkippedRegMask |= genMapArgNumToRegMask(intArgRegNum, TYP_I_IMPL);
3029 if (argSlots % 2 == 1)
3035 #elif defined(_TARGET_ARM64_)
3037 // TODO-ARM64-NYI: HFA/HVA
3038 if (lateArgsComputed)
3040 passUsingFloatRegs = isValidFloatArgReg(argEntry->regNum);
3044 passUsingFloatRegs = !callIsVararg && varTypeIsFloating(argx);
3047 #elif defined(_TARGET_AMD64_)
3048 #if defined(UNIX_AMD64_ABI)
3049 if (lateArgsComputed)
3051 passUsingFloatRegs = isValidFloatArgReg(argEntry->regNum);
3055 passUsingFloatRegs = varTypeIsFloating(argx);
3057 bool passUsingIntRegs;
3058 passUsingIntRegs = passUsingFloatRegs ? false : (intArgRegNum < MAX_REG_ARG);
3059 #else // !UNIX_AMD64_ABI
3060 passUsingFloatRegs = varTypeIsFloating(argx);
3061 #endif // !UNIX_AMD64_ABI
3062 #elif defined(_TARGET_X86_)
3064 passUsingFloatRegs = false;
3067 #error Unsupported or unset target architecture
3070 bool isBackFilled = false;
3071 unsigned nextFltArgRegNum = fltArgRegNum; // This is the next floating-point argument register number to use
3072 var_types structBaseType = TYP_STRUCT;
3074 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3075 unsigned int structFloatRegs = 0;
3076 unsigned int structIntRegs = 0;
3077 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3078 bool isStructArg = varTypeIsStruct(argx);
3080 if (lateArgsComputed)
3082 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3083 // Get the struct description for the already completed struct argument.
3084 fgArgTabEntryPtr fgEntryPtr = gtArgEntryByNode(call, argx);
3085 assert(fgEntryPtr != nullptr);
3087 // As described in few other places, this can happen when the argx was morphed
3088 // into an arg setup node - COPYBLK. The COPYBLK has always a type of void.
3089 // In such case the fgArgTabEntry keeps track of whether the original node (before morphing)
3090 // was a struct and the struct classification.
3091 isStructArg = fgEntryPtr->isStruct;
3095 structDesc.CopyFrom(fgEntryPtr->structDesc);
3097 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3099 assert(argEntry != NULL);
3100 if (argEntry->IsBackFilled())
3103 size = argEntry->numRegs;
3104 nextFltArgRegNum = genMapFloatRegNumToRegArgNum(argEntry->regNum);
3106 isBackFilled = true;
3108 else if (argEntry->regNum == REG_STK)
3111 assert(argEntry->numRegs == 0);
3112 size = argEntry->numSlots;
3117 assert(argEntry->numRegs > 0);
3118 size = argEntry->numRegs + argEntry->numSlots;
3121 // This size has now been computed
3127 // Figure out the size of the argument. This is either in number of registers, or number of register-sized
3128 // stack slots, or both if the argument is split between the registers and the stack.
3131 if (argx->IsArgPlaceHolderNode() || (!isStructArg))
3133 #if defined(_TARGET_AMD64_)
3134 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3137 size = 1; // On AMD64, all primitives fit in a single (64-bit) 'slot'
3141 size = (unsigned)(roundUp(info.compCompHnd->getClassSize(argx->gtArgPlace.gtArgPlaceClsHnd), TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE;
3142 eeGetSystemVAmd64PassStructInRegisterDescriptor(argx->gtArgPlace.gtArgPlaceClsHnd, &structDesc);
3145 hasMultiregStructArgs = true;
3148 #else // !FEATURE_UNIX_AMD64_STRUCT_PASSING
3149 size = 1; // On AMD64, all primitives fit in a single (64-bit) 'slot'
3150 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3151 #elif defined(_TARGET_ARM64_)
3154 // Structs are either passed in 1 or 2 (64-bit) slots
3155 size = (unsigned)(roundUp(info.compCompHnd->getClassSize(argx->gtArgPlace.gtArgPlaceClsHnd), TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE;
3158 size = 1; // Large structs are passed by reference (to a copy)
3162 hasMultiregStructArgs = true;
3164 // Note that there are some additional rules for size=2 structs,
3165 // (i.e they cannot be split betwen registers and the stack)
3169 size = 1; // On ARM64, all primitive types fit in a single (64-bit) 'slot'
3171 #elif defined(_TARGET_ARM_)
3174 size = (unsigned)(roundUp(info.compCompHnd->getClassSize(argx->gtArgPlace.gtArgPlaceClsHnd), TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE;
3179 size = genTypeStSz(argx->gtType);
3181 #elif defined(_TARGET_X86_)
3182 size = genTypeStSz(argx->gtType);
3184 #error Unsupported or unset target architecture
3185 #endif // _TARGET_XXX_
3190 size = GetHfaSlots(argx);
3195 // We handle two opcodes: GT_MKREFANY and GT_OBJ
3196 if (argx->gtOper == GT_MKREFANY)
3198 if (varTypeIsStruct(argx))
3202 #ifdef _TARGET_AMD64_
3203 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3204 if (varTypeIsStruct(argx))
3206 size = info.compCompHnd->getClassSize(impGetRefAnyClass());
3207 unsigned roundupSize = (unsigned)roundUp(size, TARGET_POINTER_SIZE);
3208 size = roundupSize / TARGET_POINTER_SIZE;
3209 eeGetSystemVAmd64PassStructInRegisterDescriptor(impGetRefAnyClass(), &structDesc);
3212 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3223 GenTreePtr argObj = argx;
3224 GenTreePtr* parentOfArgObj = parentArgx;
3226 assert(args->IsList());
3227 assert(argx == args->Current());
3229 /* The GT_OBJ may be be a child of a GT_COMMA */
3230 while (argObj->gtOper == GT_COMMA)
3232 parentOfArgObj = &argObj->gtOp.gtOp2;
3233 argObj = argObj->gtOp.gtOp2;
3236 if (argObj->gtOper != GT_OBJ)
3237 BADCODE("illegal argument tree in fgMorphArgs");
3239 CORINFO_CLASS_HANDLE objClass = argObj->gtObj.gtClass;
3240 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3241 eeGetSystemVAmd64PassStructInRegisterDescriptor(objClass, &structDesc);
3242 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3244 unsigned originalSize = info.compCompHnd->getClassSize(objClass);
3245 originalSize = (originalSize == 0 ? TARGET_POINTER_SIZE : originalSize);
3246 unsigned roundupSize = (unsigned)roundUp(originalSize, TARGET_POINTER_SIZE);
3247 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3248 // On System V OS-es a struct is never passed by reference.
3249 // It is either passed by value on the stack or in registers.
3250 bool passStructInRegisters = false;
3251 #else // !FEATURE_UNIX_AMD64_STRUCT_PASSING
3252 bool passStructByRef = false;
3253 #endif // !FEATURE_UNIX_AMD64_STRUCT_PASSING
3255 #ifndef _TARGET_X86_
3256 #ifndef FEATURE_UNIX_AMD64_STRUCT_PASSING
3257 // Check for struct argument with size 1, 2, 4 or 8 bytes
3258 // As we can optimize these by turning them into a GT_IND of the correct type
3259 if ((originalSize > TARGET_POINTER_SIZE) || ((originalSize & (originalSize - 1)) != 0))
3260 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3262 // Normalize 'size' to the number of pointer sized items
3263 // 'size' is the number of register slots that we will use to pass the argument
3264 size = roundupSize / TARGET_POINTER_SIZE;
3265 #if defined(_TARGET_AMD64_)
3266 #ifndef FEATURE_UNIX_AMD64_STRUCT_PASSING
3267 size = 1; // This must be copied to a temp and passed by address
3268 passStructByRef = true;
3269 copyBlkClass = objClass;
3270 #else // FEATURE_UNIX_AMD64_STRUCT_PASSING
3271 if (!structDesc.passedInRegisters)
3273 passStructInRegisters = false;
3274 copyBlkClass = NO_CLASS_HANDLE;
3278 // The objClass is used to materialize the struct on stack.
3279 passStructInRegisters = true;
3280 copyBlkClass = objClass;
3282 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3283 #elif defined(_TARGET_ARM64_)
3286 size = 1; // This must be copied to a temp and passed by address
3287 passStructByRef = true;
3288 copyBlkClass = objClass;
3293 // If we're passing a promoted struct local var,
3294 // we may need to skip some registers due to alignment; record those.
3295 GenTreePtr lclVar = fgIsIndirOfAddrOfLocal(argObj);
3298 LclVarDsc* varDsc = &lvaTable[lclVar->gtLclVarCommon.gtLclNum];
3299 if (varDsc->lvPromoted)
3301 assert(argObj->OperGet() == GT_OBJ);
3302 if (lvaGetPromotionType(varDsc) == PROMOTION_TYPE_INDEPENDENT)
3304 fgAddSkippedRegsInPromotedStructArg(varDsc, intArgRegNum, &argSkippedRegMask);
3308 #endif // _TARGET_ARM_
3310 #ifndef FEATURE_UNIX_AMD64_STRUCT_PASSING
3313 // change our GT_OBJ into a GT_IND of the correct type.
3314 structBaseType = argOrReturnTypeForStruct(originalSize, objClass, false /* forReturn */);
3315 // We've already ensured above that size is a power of 2, and less than pointer size.
3316 noway_assert(structBaseType != TYP_UNKNOWN);
3318 argObj->ChangeOper(GT_IND);
3320 // Now see if we can fold *(&X) into X
3321 if (argObj->gtOp.gtOp1->gtOper == GT_ADDR)
3323 GenTreePtr temp = argObj->gtOp.gtOp1->gtOp.gtOp1;
3325 // Keep the DONT_CSE flag in sync
3326 // (as the addr always marks it for its op1)
3327 temp->gtFlags &= ~GTF_DONT_CSE;
3328 temp->gtFlags |= (argObj->gtFlags & GTF_DONT_CSE);
3329 DEBUG_DESTROY_NODE(argObj->gtOp.gtOp1); // GT_ADDR
3330 DEBUG_DESTROY_NODE(argObj); // GT_IND
3333 *parentOfArgObj = temp;
3335 // If the OBJ had been the top level node, we've now changed argx.
3336 if (parentOfArgObj == parentArgx)
3339 if (argObj->gtOper == GT_LCL_VAR)
3341 unsigned lclNum = argObj->gtLclVarCommon.gtLclNum;
3342 LclVarDsc * varDsc = &lvaTable[lclNum];
3344 if (varDsc->lvPromoted)
3346 if (varDsc->lvFieldCnt == 1)
3348 // get the first and only promoted field
3349 LclVarDsc * fieldVarDsc = &lvaTable[varDsc->lvFieldLclStart];
3350 if (genTypeSize(fieldVarDsc->TypeGet()) >= originalSize)
3352 // we will use the first and only promoted field
3353 argObj->gtLclVarCommon.SetLclNum(varDsc->lvFieldLclStart);
3355 if (varTypeCanReg(fieldVarDsc->TypeGet()) && (genTypeSize(fieldVarDsc->TypeGet()) == originalSize))
3357 // Just use the existing field's type
3358 argObj->gtType = fieldVarDsc->TypeGet();
3362 // Can't use the existing field's type, so use GT_LCL_FLD to swizzle
3364 argObj->ChangeOper(GT_LCL_FLD);
3365 argObj->gtType = structBaseType;
3367 assert(varTypeCanReg(argObj->TypeGet()));
3368 assert(copyBlkClass == NO_CLASS_HANDLE);
3372 // use GT_LCL_FLD to swizzle the single field struct to a new type
3373 lvaSetVarDoNotEnregister(lclNum DEBUGARG(DNER_LocalField));
3374 argObj->ChangeOper(GT_LCL_FLD);
3375 argObj->gtType = structBaseType;
3380 // The struct fits into a single register, but it has been promoted into its
3381 // constituent fields, and so we have to re-assemble it
3382 copyBlkClass = objClass;
3384 // Alignment constraints may cause us not to use (to "skip") some argument registers.
3385 // Add those, if any, to the skipped (int) arg reg mask.
3386 fgAddSkippedRegsInPromotedStructArg(varDsc, intArgRegNum, &argSkippedRegMask);
3387 #endif // _TARGET_ARM_
3390 else if (!varTypeIsIntegralOrI(varDsc->TypeGet()))
3392 // Not a promoted struct, so just swizzle the type by using GT_LCL_FLD
3393 argObj->ChangeOper(GT_LCL_FLD);
3394 argObj->gtType = structBaseType;
3399 // Not a GT_LCL_VAR, so we can just change the type on the node
3400 argObj->gtType = structBaseType;
3402 assert(varTypeCanReg(argObj->TypeGet()) ||
3403 ((copyBlkClass != NO_CLASS_HANDLE) && varTypeIsIntegral(structBaseType)));
3407 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3409 #endif // not _TARGET_X86_
3410 // We still have a struct unless we converted the GT_OBJ into a GT_IND above...
3411 if ((structBaseType == TYP_STRUCT) &&
3412 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3413 !passStructInRegisters
3414 #else // !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3416 #endif // !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3419 // if the valuetype size is not a multiple of sizeof(void*),
3420 // we must copyblk to a temp before doing the obj to avoid
3421 // the obj reading memory past the end of the valuetype
3422 #if defined(_TARGET_X86_) && !defined(LEGACY_BACKEND)
3423 // TODO-X86-CQ: [1091733] Revisit for small structs, we should use push instruction
3424 copyBlkClass = objClass;
3425 size = roundupSize / TARGET_POINTER_SIZE; // Normalize size to number of pointer sized items
3426 #else // !defined(_TARGET_X86_) || defined(LEGACY_BACKEND)
3427 if (roundupSize > originalSize)
3429 copyBlkClass = objClass;
3431 // There are a few special cases where we can omit using a CopyBlk
3432 // where we normally would need to use one.
3434 GenTreePtr objAddr = argObj->gtObj.gtOp1;
3435 if (objAddr->gtOper == GT_ADDR)
3437 // exception : no need to use CopyBlk if the valuetype is on the stack
3438 if (objAddr->gtFlags & GTF_ADDR_ONSTACK)
3440 copyBlkClass = NO_CLASS_HANDLE;
3442 // exception : no need to use CopyBlk if the valuetype is already a struct local
3443 else if (objAddr->gtOp.gtOp1->gtOper == GT_LCL_VAR)
3445 copyBlkClass = NO_CLASS_HANDLE;
3450 size = roundupSize / TARGET_POINTER_SIZE; // Normalize size to number of pointer sized items
3451 #endif // !defined(_TARGET_X86_) || defined(LEGACY_BACKEND)
3456 hasMultiregStructArgs = true;
3460 // The 'size' value has now must have been set. (the original value of zero is an invalid value)
3464 // Figure out if the argument will be passed in a register.
3466 bool passedInRegisters = true;
3467 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3468 passedInRegisters = !isStructArg;
3469 if (!passedInRegisters)
3471 if (structDesc.passedInRegisters)
3473 passedInRegisters = true;
3477 passedInRegisters = false;
3482 if (passedInRegisters && isRegParamType(genActualType(argx->TypeGet())))
3485 if (passUsingFloatRegs)
3487 // First, see if it can be back-filled
3488 if (!anyFloatStackArgs && // Is it legal to back-fill? (We haven't put any FP args on the stack yet)
3489 (fltArgSkippedRegMask != RBM_NONE) && // Is there an available back-fill slot?
3490 (size == 1)) // The size to back-fill is one float register
3492 // Back-fill the register.
3493 isBackFilled = true;
3494 regMaskTP backFillBitMask = genFindLowestBit(fltArgSkippedRegMask);
3495 fltArgSkippedRegMask &= ~backFillBitMask; // Remove the back-filled register(s) from the skipped mask
3496 nextFltArgRegNum = genMapFloatRegNumToRegArgNum(genRegNumFromMask(backFillBitMask));
3497 assert(nextFltArgRegNum < MAX_FLOAT_REG_ARG);
3500 // Does the entire float, double, or HFA fit in the FP arg registers?
3501 // Check if the last register needed is still in the argument register range.
3502 isRegArg = (nextFltArgRegNum + size - 1) < MAX_FLOAT_REG_ARG;
3506 anyFloatStackArgs = true;
3511 isRegArg = intArgRegNum < MAX_REG_ARG;
3513 #elif _TARGET_ARM64_
3514 if (passUsingFloatRegs)
3516 // Check if the last register needed is still in the fp argument register range.
3517 isRegArg = (nextFltArgRegNum + (size - 1)) < MAX_FLOAT_REG_ARG;
3521 // Check if the last register needed is still in the int argument register range.
3522 isRegArg = (intArgRegNum + (size - 1)) < maxRegArgs;
3524 #else // not _TARGET_ARM_ or _TARGET_ARM64_
3526 #if defined(UNIX_AMD64_ABI)
3528 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3529 // Here a struct can be passed in register following the classifications of its members and size.
3530 // Now make sure there are actually enough registers to do so.
3533 for (unsigned int i = 0; i < structDesc.eightByteCount; i++)
3535 if (structDesc.IsIntegralSlot(i))
3539 else if (structDesc.IsSseSlot(i))
3545 if (((nextFltArgRegNum + structFloatRegs) > MAX_FLOAT_REG_ARG) ||
3546 ((intArgRegNum + structIntRegs) > MAX_REG_ARG))
3549 nonRegPassableStruct = true;
3554 nonRegPassableStruct = false;
3558 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3560 if (passUsingFloatRegs)
3562 isRegArg = nextFltArgRegNum < MAX_FLOAT_REG_ARG;
3566 isRegArg = intArgRegNum < MAX_REG_ARG;
3569 #else // !defined(UNIX_AMD64_ABI)
3570 isRegArg = (intArgRegNum+(size-1)) < maxRegArgs;
3571 #endif // !defined(UNIX_AMD64_ABI)
3572 #endif // _TARGET_ARM_
3578 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3579 nonRegPassableStruct = true;
3580 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3585 // Now we know if the argument goes in registers or not and how big it is,
3586 // whether we had to just compute it or this is a re-morph call and we looked it up.
3591 // If we ever allocate a floating point argument to the stack, then all
3592 // subsequent HFA/float/double arguments go on the stack.
3593 if (!isRegArg && passUsingFloatRegs)
3595 for (; fltArgRegNum < MAX_FLOAT_REG_ARG; ++fltArgRegNum)
3597 fltArgSkippedRegMask |= genMapArgNumToRegMask(fltArgRegNum, TYP_FLOAT);
3601 // If we think we're going to split a struct between integer registers and the stack, check to
3602 // see if we've already assigned a floating-point arg to the stack.
3603 if (isRegArg && // We decided above to use a register for the argument
3604 !passUsingFloatRegs && // We're using integer registers
3605 (intArgRegNum + size > MAX_REG_ARG) && // We're going to split a struct type onto registers and stack
3606 anyFloatStackArgs) // We've already used the stack for a floating-point argument
3608 isRegArg = false; // Change our mind; don't pass this struct partially in registers
3610 // Skip the rest of the integer argument registers
3611 for (; intArgRegNum < MAX_REG_ARG; ++intArgRegNum)
3613 argSkippedRegMask |= genMapArgNumToRegMask(intArgRegNum, TYP_I_IMPL);
3617 #endif // _TARGET_ARM_
3620 regNumber nextRegNum = REG_STK;
3621 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3622 regNumber nextOtherRegNum = REG_STK;
3624 if (isStructArg && structDesc.passedInRegisters)
3626 // It is a struct passed in registers. Assign the next available register.
3627 unsigned int curIntReg = intArgRegNum;
3628 unsigned int curFloatReg = nextFltArgRegNum;
3629 for (unsigned int i = 0; i < structDesc.eightByteCount; i++)
3631 if (structDesc.IsIntegralSlot(i))
3635 nextRegNum = genMapIntRegArgNumToRegNum(curIntReg);
3637 // For non-completed args the counters are incremented already
3638 // in the !lateArgsComputed above.
3639 if (lateArgsComputed)
3646 nextOtherRegNum = genMapIntRegArgNumToRegNum(curIntReg);
3648 if (lateArgsComputed)
3655 assert(false && "fgMorphArgs Invalid index for int classification.");
3660 else if (structDesc.IsSseSlot(i))
3664 nextRegNum = genMapFloatRegArgNumToRegNum(curFloatReg);
3666 if (lateArgsComputed)
3673 nextOtherRegNum = genMapFloatRegArgNumToRegNum(curFloatReg);
3675 if (lateArgsComputed)
3682 assert(false && "fgMorphArgs Invalid index for SSE classification.");
3690 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3692 // fill in or update the argInfo table
3693 nextRegNum = passUsingFloatRegs ? genMapFloatRegArgNumToRegNum(nextFltArgRegNum) : genMapIntRegArgNumToRegNum(intArgRegNum);
3696 #ifdef _TARGET_AMD64_
3697 #ifndef FEATURE_UNIX_AMD64_STRUCT_PASSING
3702 #ifndef LEGACY_BACKEND
3703 // If there are nonstandard args (outside the calling convention) they were inserted above
3704 // and noted them in a table so we can recognize them here and build their argInfo.
3706 // They should not affect the placement of any other args or stack space required.
3707 // Example: on AMD64 R10 and R11 are used for indirect VSD (generic interface) and cookie calls.
3708 bool nonStandardFound = false;
3709 for (int i=0; i<nonStandardArgs.Height(); i++)
3711 hasNonStandardArg = true;
3712 if (argx == nonStandardArgs.Index(i).node)
3714 fgArgTabEntry* argEntry = call->fgArgInfo->AddRegArg(argIndex,
3717 nonStandardArgs.Index(i).reg,
3720 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3724 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3726 argEntry->isNonStandard = true;
3728 nonStandardFound = true;
3732 if (nonStandardFound)
3734 #endif // !LEGACY_BACKEND
3736 if (!lateArgsComputed)
3738 // This is a register argument - put it in the table
3739 fgArgTabEntryPtr newArg = call->fgArgInfo->AddRegArg(argIndex,
3745 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3749 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3751 (void)newArg; //prevent "unused variable" error from GCC
3753 newArg->SetIsHfaRegArg(passUsingFloatRegs && isHfaArg); // Note that an HFA is passed in int regs for varargs
3754 newArg->SetIsBackFilled(isBackFilled);
3755 #endif // _TARGET_ARM_
3759 // This is a register argument - possibly update it in the table
3760 fgArgTabEntryPtr entry = call->fgArgInfo->RemorphRegArg(argIndex, argx, args, nextRegNum, size, argAlign);
3761 if (entry->isNonStandard)
3768 // Setup the next argRegNum value
3771 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3774 intArgRegNum += structIntRegs;
3775 fltArgRegNum += structFloatRegs;
3778 #endif // defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
3780 if (passUsingFloatRegs)
3782 fltArgRegNum += size;
3784 #if defined(_TARGET_AMD64_) && !defined(UNIX_AMD64_ABI)
3785 argSkippedRegMask |= genMapArgNumToRegMask(intArgRegNum, TYP_I_IMPL);
3786 intArgRegNum = min(intArgRegNum + size, MAX_REG_ARG);
3787 #endif // _TARGET_AMD64_
3789 if (fltArgRegNum > MAX_FLOAT_REG_ARG)
3791 // This indicates a partial enregistration of a struct type
3792 assert(varTypeIsStruct(argx));
3793 unsigned numRegsPartial = size - (fltArgRegNum - MAX_FLOAT_REG_ARG);
3794 assert((unsigned char)numRegsPartial == numRegsPartial);
3795 call->fgArgInfo->SplitArg(argIndex, numRegsPartial, size - numRegsPartial);
3796 fltArgRegNum = MAX_FLOAT_REG_ARG;
3798 #endif // _TARGET_ARM_
3802 intArgRegNum += size;
3804 #if defined(_TARGET_AMD64_) && !defined(UNIX_AMD64_ABI)
3805 fltArgSkippedRegMask |= genMapArgNumToRegMask(fltArgRegNum, TYP_DOUBLE);
3806 fltArgRegNum = min(fltArgRegNum + size, MAX_FLOAT_REG_ARG);
3807 #endif // _TARGET_AMD64_
3809 if (intArgRegNum > MAX_REG_ARG)
3811 // This indicates a partial enregistration of a struct type
3812 assert((isStructArg) || argx->OperIsCopyBlkOp() ||
3813 (argx->gtOper == GT_COMMA && (args->gtFlags & GTF_ASG)));
3814 unsigned numRegsPartial = size - (intArgRegNum - MAX_REG_ARG);
3815 assert((unsigned char)numRegsPartial == numRegsPartial);
3816 call->fgArgInfo->SplitArg(argIndex, numRegsPartial, size - numRegsPartial);
3817 intArgRegNum = MAX_REG_ARG;
3818 fgPtrArgCntCur += size - numRegsPartial;
3820 #endif // _TARGET_ARM_
3825 else // We have an argument that is not passed in a register
3827 fgPtrArgCntCur += size;
3829 // If the register arguments have not been determined then we must fill in the argInfo
3831 if (!lateArgsComputed)
3833 // This is a stack argument - put it in the table
3834 call->fgArgInfo->AddStkArg(argIndex, argx, args, size, argAlign FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(isStructArg));
3839 // This is a stack argument - possibly update it in the table
3840 call->fgArgInfo->RemorphStkArg(argIndex, argx, args, size, argAlign);
3843 if (copyBlkClass != NO_CLASS_HANDLE)
3845 noway_assert(!lateArgsComputed);
3846 fgMakeOutgoingStructArgCopy(call, args, argIndex, copyBlkClass FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(&structDesc));
3849 #ifndef LEGACY_BACKEND
3850 if (argx->gtOper == GT_MKREFANY)
3852 NYI_X86("MKREFANY");
3854 // 'Lower' the MKREFANY tree and insert it.
3855 noway_assert(!lateArgsComputed);
3858 // Here we don't need unsafe value cls check since the addr of temp is used only in mkrefany
3859 unsigned tmp = lvaGrabTemp(true DEBUGARG("by-value mkrefany struct argument"));
3860 lvaSetStruct(tmp, impGetRefAnyClass(), false);
3863 // Build the mkrefany as a comma node:
3864 // (tmp.ptr=argx),(tmp.type=handle)
3865 GenTreeLclFld* destPtrSlot = gtNewLclFldNode(tmp, TYP_I_IMPL, offsetof(CORINFO_RefAny, dataPtr));
3866 GenTreeLclFld* destTypeSlot = gtNewLclFldNode(tmp, TYP_I_IMPL, offsetof(CORINFO_RefAny, type));
3867 destPtrSlot->gtFieldSeq = GetFieldSeqStore()->CreateSingleton(GetRefanyDataField());
3868 destPtrSlot->gtFlags |= GTF_VAR_DEF;
3869 destTypeSlot->gtFieldSeq = GetFieldSeqStore()->CreateSingleton(GetRefanyTypeField());
3870 destTypeSlot->gtFlags |= GTF_VAR_DEF;
3872 GenTreePtr asgPtrSlot = gtNewAssignNode(destPtrSlot, argx->gtOp.gtOp1);
3873 GenTreePtr asgTypeSlot = gtNewAssignNode(destTypeSlot, argx->gtOp.gtOp2);
3874 GenTreePtr asg = gtNewOperNode(GT_COMMA, TYP_VOID, asgPtrSlot, asgTypeSlot);
3876 // Change the expression to "(tmp=val)"
3877 args->gtOp.gtOp1 = asg;
3879 // EvalArgsToTemps will cause tmp to actually get loaded as the argument
3880 call->fgArgInfo->EvalToTmp(argIndex, tmp, asg);
3881 lvaSetVarAddrExposed(tmp);
3883 #endif // !LEGACY_BACKEND
3886 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
3887 if (nonRegPassableStruct)
3889 nonRegPassedStructSlots += size;
3892 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
3896 } // end foreach argument loop
3898 if (!lateArgsComputed)
3900 call->fgArgInfo->ArgsComplete();
3902 call->gtCallRegUsedMask = genIntAllRegArgMask(intArgRegNum) & ~argSkippedRegMask;
3903 if (fltArgRegNum > 0)
3905 #if defined(_TARGET_ARM_) || defined(_TARGET_AMD64_)
3906 call->gtCallRegUsedMask |= genFltAllRegArgMask(fltArgRegNum) & ~fltArgSkippedRegMask;
3911 if (call->gtCallArgs)
3913 UpdateGT_LISTFlags(call->gtCallArgs);
3916 /* Process the function address, if indirect call */
3918 if (call->gtCallType == CT_INDIRECT)
3919 call->gtCallAddr = fgMorphTree(call->gtCallAddr);
3921 call->fgArgInfo->RecordStkLevel(fgPtrArgCntCur);
3923 if ((call->gtCallType == CT_INDIRECT) && (call->gtCallCookie != NULL))
3926 /* Remember the maximum value we ever see */
3928 if (fgPtrArgCntMax < fgPtrArgCntCur)
3929 fgPtrArgCntMax = fgPtrArgCntCur;
3931 /* The call will pop all the arguments we pushed */
3933 fgPtrArgCntCur = genPtrArgCntSav;
3935 #if FEATURE_FIXED_OUT_ARGS
3937 // Update the outgoing argument size.
3938 // If the call is a fast tail call, it will setup its arguments in incoming arg
3939 // area instead of the out-going arg area. Therefore, don't consider fast tail
3940 // calls to update lvaOutgoingArgSpaceSize.
3941 if (!call->IsFastTailCall())
3943 unsigned preallocatedArgCount;
3945 #if defined(_TARGET_ARMARCH_)
3946 // First slots go in registers only, no stack needed.
3947 // TODO-ARMArch-CQ: This calculation is only accurate for integer arguments,
3948 // and ignores floating point args (it is overly conservative in that case).
3949 if (argSlots <= MAX_REG_ARG)
3951 preallocatedArgCount = 0;
3955 preallocatedArgCount = argSlots - MAX_REG_ARG;
3957 #elif defined(UNIX_AMD64_ABI)
3958 opts.compNeedToAlignFrame = true;
3959 // First slots go in registers only, no stack needed.
3960 // TODO-Amd64-Unix-CQ This calculation is only accurate for integer arguments,
3961 // and ignores floating point args (it is overly conservative in that case).
3962 if (argSlots <= MAX_REG_ARG)
3964 preallocatedArgCount = nonRegPassedStructSlots;
3968 preallocatedArgCount = argSlots + nonRegPassedStructSlots - MAX_REG_ARG;
3970 #elif defined(_TARGET_AMD64_)
3971 preallocatedArgCount = max(4, argSlots);
3973 #error Unsupported or unset target architecture
3975 if (preallocatedArgCount * REGSIZE_BYTES > lvaOutgoingArgSpaceSize)
3977 lvaOutgoingArgSpaceSize = preallocatedArgCount * REGSIZE_BYTES;
3979 // If a function has localloc, we will need to move the outgoing arg space when the
3980 // localloc happens. When we do this, we need to maintain stack alignment. To avoid
3981 // leaving alignment-related holes when doing this move, make sure the outgoing
3982 // argument space size is a multiple of the stack alignment by aligning up to the next
3983 // stack alignment boundary.
3984 if (compLocallocUsed)
3986 lvaOutgoingArgSpaceSize = (lvaOutgoingArgSpaceSize + (STACK_ALIGN - 1)) & ~(STACK_ALIGN - 1);
3990 #endif // FEATURE_FIXED_OUT_ARGS
3992 /* Update the 'side effect' flags value for the call */
3994 call->gtFlags |= (flagsSummary & GTF_ALL_EFFECT);
3996 // If the register arguments have already been determined
3997 // or we have no register arguments then we are done.
3999 bool needEvalArgsToTemps = true;
4001 if (lateArgsComputed || (intArgRegNum == 0 && fltArgRegNum == 0 && !hasNonStandardArg && !hasStructArgument))
4003 needEvalArgsToTemps = false;
4006 if (needEvalArgsToTemps)
4008 // This is the first time that we morph this call AND it has register arguments.
4009 // Follow into the code below and do the 'defer or eval to temp' analysis.
4011 call->fgArgInfo->SortArgs();
4013 call->fgArgInfo->EvalArgsToTemps();
4015 // We may have updated the arguments
4016 if (call->gtCallArgs)
4018 UpdateGT_LISTFlags(call->gtCallArgs);
4022 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
4024 // Rewrite the struct args to be passed by value on stack or in registers.
4025 fgMorphSystemVStructArgs(call, hasStructArgument);
4027 #else // !FEATURE_UNIX_AMD64_STRUCT_PASSING
4029 // In the future we can migrate UNIX_AMD64 to use this
4030 // method instead of fgMorphSystemVStructArgs
4031 #ifndef LEGACY_BACKEND
4032 // We only build GT_LISTs for MultiReg structs for the RyuJIT backend
4033 if (hasMultiregStructArgs)
4035 fgMorphMultiregStructArgs(call);
4037 #endif // LEGACY_BACKEND
4039 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
4044 #pragma warning(pop)
4047 #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING
4048 // fgMorphSystemVStructArgs:
4049 // Rewrite the struct args to be passed by value on stack or in registers.
4052 // call: The call whose arguments need to be morphed.
4053 // hasStructArgument: Whether this call has struct arguments.
4055 void Compiler::fgMorphSystemVStructArgs(GenTreeCall* call, bool hasStructArgument)
4057 unsigned flagsSummary = 0;
4061 if (hasStructArgument)
4063 fgArgInfoPtr allArgInfo = call->fgArgInfo;
4065 for (args = call->gtCallArgs; args != nullptr; args = args->gtOp.gtOp2)
4067 // For late arguments the arg tree that is overridden is in the gtCallLateArgs list.
4068 // For such late args the gtCallArgList contains the setup arg node (evaluating the arg.)
4069 // The tree from the gtCallLateArgs list is passed to the callee. The fgArgEntry node contains the mapping
4070 // between the nodes in both lists. If the arg is not a late arg, the fgArgEntry->node points to itself,
4071 // otherwise points to the list in the late args list.
4072 bool isLateArg = (args->gtOp.gtOp1->gtFlags & GTF_LATE_ARG) != 0;
4073 fgArgTabEntryPtr fgEntryPtr = gtArgEntryByNode(call, args->gtOp.gtOp1);
4074 assert(fgEntryPtr != nullptr);
4075 GenTreePtr argx = fgEntryPtr->node;
4076 GenTreePtr lateList = nullptr;
4077 GenTreePtr lateNode = nullptr;
4081 for (GenTreePtr list = call->gtCallLateArgs; list; list = list->MoveNext())
4083 assert(list->IsList());
4085 GenTreePtr argNode = list->Current();
4086 if (argx == argNode)
4093 assert(lateList != nullptr && lateNode != nullptr);
4095 GenTreePtr arg = argx;
4096 bool argListCreated = false;
4098 var_types type = arg->TypeGet();
4100 if (varTypeIsStruct(type))
4102 var_types originalType = type;
4103 // If we have already processed the arg...
4104 if (arg->OperGet() == GT_LIST && varTypeIsStruct(arg))
4109 // If already OBJ it is set properly already.
4110 if (arg->OperGet() == GT_OBJ)
4112 assert(!fgEntryPtr->structDesc.passedInRegisters);
4117 arg->OperGet() == GT_LCL_VAR ||
4118 arg->OperGet() == GT_LCL_FLD ||
4119 (arg->OperGet() == GT_ADDR &&
4120 (arg->gtOp.gtOp1->OperGet() == GT_LCL_FLD ||
4121 arg->gtOp.gtOp1->OperGet() == GT_LCL_VAR)));
4123 GenTreeLclVarCommon* lclCommon = arg->OperGet() == GT_ADDR ?
4124 arg->gtOp.gtOp1->AsLclVarCommon() : arg->AsLclVarCommon();
4125 if (fgEntryPtr->structDesc.passedInRegisters)
4127 if (fgEntryPtr->structDesc.eightByteCount == 1)
4129 // Change the type and below the code will change the LclVar to a LCL_FLD
4130 type = GetTypeFromClassificationAndSizes(fgEntryPtr->structDesc.eightByteClassifications[0], fgEntryPtr->structDesc.eightByteSizes[0]);
4132 else if (fgEntryPtr->structDesc.eightByteCount == 2)
4134 // Create LCL_FLD for each eightbyte.
4135 argListCreated = true;
4137 // Second eightbyte.
4138 GenTreeLclFld* newLclField = new(this, GT_LCL_FLD) GenTreeLclFld(
4139 GetTypeFromClassificationAndSizes(
4140 fgEntryPtr->structDesc.eightByteClassifications[1],
4141 fgEntryPtr->structDesc.eightByteSizes[1]),
4142 lclCommon->gtLclNum,
4143 fgEntryPtr->structDesc.eightByteOffsets[1]);
4144 // Note this should actually be: secondNode = gtNewArgList(newLclField)
4145 GenTreeArgList* secondNode = gtNewListNode(newLclField, nullptr);
4146 secondNode->gtType = originalType; // Preserve the type. It is a special case.
4147 newLclField->gtFieldSeq = FieldSeqStore::NotAField();
4150 arg->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField();
4151 arg->gtType = GetTypeFromClassificationAndSizes(
4152 fgEntryPtr->structDesc.eightByteClassifications[0],
4153 fgEntryPtr->structDesc.eightByteSizes[0]);
4154 arg = gtNewListNode(arg, secondNode);
4155 arg->gtType = type; // Preserve the type. It is a special case.
4159 assert(false && "More than two eightbytes detected for CLR."); // No more than two eightbytes for the CLR.
4163 // If we didn't change the type of the struct, it means
4164 // its classification doesn't support to be passed directly through a
4165 // register, so we need to pass a pointer to the destination where
4166 // where we copied the struct to.
4167 if (!argListCreated)
4169 if (fgEntryPtr->structDesc.passedInRegisters)
4175 // Make sure this is an addr node.
4176 if (arg->OperGet() != GT_ADDR && arg->OperGet() != GT_LCL_VAR_ADDR)
4178 arg = gtNewOperNode(GT_ADDR, TYP_I_IMPL, arg);
4181 assert(arg->OperGet() == GT_ADDR || arg->OperGet() == GT_LCL_VAR_ADDR);
4183 // Create an Obj of the temp to use it as a call argument.
4184 arg = new (this, GT_OBJ) GenTreeObj(originalType, arg, lvaGetStruct(lclCommon->gtLclNum));
4185 arg->gtFlags |= GTF_EXCEPT;
4186 flagsSummary |= GTF_EXCEPT;
4193 bool isLateArg = (args->gtOp.gtOp1->gtFlags & GTF_LATE_ARG) != 0;
4194 fgArgTabEntryPtr fgEntryPtr = gtArgEntryByNode(call, args->gtOp.gtOp1);
4195 assert(fgEntryPtr != nullptr);
4196 GenTreePtr argx = fgEntryPtr->node;
4197 GenTreePtr lateList = nullptr;
4198 GenTreePtr lateNode = nullptr;
4201 for (GenTreePtr list = call->gtCallLateArgs; list; list = list->MoveNext())
4203 assert(list->IsList());
4205 GenTreePtr argNode = list->Current();
4206 if (argx == argNode)
4213 assert(lateList != nullptr && lateNode != nullptr);
4216 fgEntryPtr->node = arg;
4219 lateList->gtOp.gtOp1 = arg;
4223 args->gtOp.gtOp1 = arg;
4230 call->gtFlags |= (flagsSummary & GTF_ALL_EFFECT);
4232 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING
4234 //-----------------------------------------------------------------------------
4235 // fgMorphMultiregStructArgs: Locate the TYP_STRUCT arguments and
4236 // call fgMorphMultiregStructArg on each of them.
4239 // call: a GenTreeCall node that has one or more TYP_STRUCT arguments
4242 // We only call fgMorphMultiregStructArg for the register passed TYP_STRUCT arguments.
4243 // The call to fgMorphMultiregStructArg will mutate the argument into the GT_LIST form
4244 // whicj is only used for register arguments.
4245 // If this method fails to find any TYP_STRUCT arguments it will assert.
4247 void Compiler::fgMorphMultiregStructArgs(GenTreeCall* call)
4251 bool foundStructArg = false;
4252 unsigned initialFlags = call->gtFlags;
4253 unsigned flagsSummary = 0;
4254 fgArgInfoPtr allArgInfo = call->fgArgInfo;
4256 // Currently only ARM64 is using this method to morph the MultiReg struct args
4257 // in the future AMD64_UNIX and for HFAs ARM32, will also use this method
4260 NYI_ARM("fgMorphMultiregStructArgs");
4263 assert("Logic error: no MultiregStructArgs for X86");
4265 #ifdef _TARGET_AMD64_
4266 #if defined(UNIX_AMD64_ABI)
4267 NYI_AMD64("fgMorphMultiregStructArgs (UNIX ABI)");
4270 assert("Logic error: no MultiregStructArgs for Windows X64 ABI");
4273 for (args = call->gtCallArgs; args != nullptr; args = args->gtOp.gtOp2)
4275 // For late arguments the arg tree that is overridden is in the gtCallLateArgs list.
4276 // For such late args the gtCallArgList contains the setup arg node (evaluating the arg.)
4277 // The tree from the gtCallLateArgs list is passed to the callee. The fgArgEntry node contains the mapping
4278 // between the nodes in both lists. If the arg is not a late arg, the fgArgEntry->node points to itself,
4279 // otherwise points to the list in the late args list.
4280 bool isLateArg = (args->gtOp.gtOp1->gtFlags & GTF_LATE_ARG) != 0;
4281 fgArgTabEntryPtr fgEntryPtr = gtArgEntryByNode(call, args->gtOp.gtOp1);
4282 assert(fgEntryPtr != nullptr);
4283 GenTreePtr argx = fgEntryPtr->node;
4284 GenTreePtr lateList = nullptr;
4285 GenTreePtr lateNode = nullptr;
4289 for (GenTreePtr list = call->gtCallLateArgs; list; list = list->MoveNext())
4291 assert(list->IsList());
4293 GenTreePtr argNode = list->Current();
4294 if (argx == argNode)
4301 assert(lateList != nullptr && lateNode != nullptr);
4304 GenTreePtr arg = argx;
4306 if (arg->TypeGet() == TYP_STRUCT)
4308 foundStructArg = true;
4310 arg = fgMorphMultiregStructArg(arg, fgEntryPtr);
4312 // Did we replace 'argx' with a new tree?
4315 fgEntryPtr->node = arg; // Record the new value for the arg in the fgEntryPtr->node
4317 // link the new arg node into either the late arg list or the gtCallArgs list
4320 lateList->gtOp.gtOp1 = arg;
4324 args->gtOp.gtOp1 = arg;
4330 // We should only call this method when we actually have one or more multireg struct args
4331 assert(foundStructArg);
4334 call->gtFlags |= (flagsSummary & GTF_ALL_EFFECT);
4338 //-----------------------------------------------------------------------------
4339 // fgMorphMultiregStructArg: Given a multireg TYP_STRUCT arg from a call argument list
4340 // Morph the argument into a set of GT_LIST nodes.
4343 // arg - A GenTree node containing a TYP_STRUCT arg that
4344 // is to be passed in multiple registers
4346 // arg must be a GT_OBJ or GT_LCL_VAR or GT_LCL_FLD of TYP_STRUCT that is suitable
4347 // for passing in multiple registers.
4348 // If arg is a LclVar we check if it is struct promoted and has the right number of fields
4349 // and if they are at the appropriate offsets we will use the struct promted fields
4350 // in the GT_LIST nodes that we create.
4351 // If we have a GT_LCL_VAR that isn't struct promoted or doesn't meet the requirements
4352 // we will use a set of GT_LCL_FLDs nodes to access the various portions of the struct
4353 // this also forces the struct to be stack allocated into the local frame.
4354 // For the GT_OBJ case will clone the address expression and generate two (or more)
4356 // Currently the implementation only handles ARM64 and will NYI for other architectures.
4357 // And for ARM64 we do not ye handle HFA arguments, so only 16-byte struct sizes are supported.
4359 GenTreePtr Compiler::fgMorphMultiregStructArg(GenTreePtr arg, fgArgTabEntryPtr fgEntryPtr)
4361 GenTreeArgList* newArg = nullptr;
4362 assert(arg->TypeGet() == TYP_STRUCT);
4363 GenTreePtr argValue = arg;
4365 #ifndef _TARGET_ARM64_
4366 NYI("fgMorphMultiregStructArg non-ARM64 implementation");
4369 // If we have a GT_OBJ of a GT_ADDR then
4370 // we set argValue to the child node ofthe GT_ADDR
4371 if (arg->OperGet() == GT_OBJ)
4373 GenTreePtr argAddr = arg->gtOp.gtOp1;
4375 if (argAddr->OperGet() == GT_ADDR)
4377 argValue = argAddr->gtOp.gtOp1;
4380 // We should still have a TYP_STRUCT
4381 assert(argValue->TypeGet() == TYP_STRUCT);
4383 // Are we passing a struct LclVar?
4385 if (argValue->OperGet() == GT_LCL_VAR)
4387 GenTreeLclVarCommon* varNode = argValue->AsLclVarCommon();
4388 unsigned varNum = varNode->gtLclNum;
4389 assert(varNum < lvaCount);
4390 LclVarDsc* varDsc = &lvaTable[varNum];
4392 // At this point any TYP_STRUCT LclVar must be a 16-byte pass by value argument
4393 assert(varDsc->lvSize() == 2 * TARGET_POINTER_SIZE);
4395 const BYTE * gcPtrs = varDsc->lvGcLayout;
4397 var_types type0 = getJitGCType(gcPtrs[0]);
4398 var_types type1 = getJitGCType(gcPtrs[1]);
4400 varDsc->lvIsMultiRegArgOrRet = true;
4402 // Is this LclVar a promoted struct with exactly two fields?
4403 if ((varDsc->lvPromoted) && (varDsc->lvFieldCnt == 2))
4405 // See if we have two promoted fields that start at offset 0 and 8?
4406 unsigned loVarNum = lvaGetFieldLocal(varDsc, 0);
4407 unsigned hiVarNum = lvaGetFieldLocal(varDsc, TARGET_POINTER_SIZE);
4409 // Did we find the promoted fields at the necessary offsets?
4410 if ((loVarNum != BAD_VAR_NUM) && (hiVarNum != BAD_VAR_NUM))
4412 LclVarDsc* loVarDsc = &lvaTable[loVarNum];
4413 LclVarDsc* hiVarDsc = &lvaTable[hiVarNum];
4415 var_types loType = loVarDsc->lvType;
4416 var_types hiType = hiVarDsc->lvType;
4418 if (varTypeIsFloating(loType) || varTypeIsFloating(hiType))
4420 // TODO-LSRA - It currently doesn't support the passing of floating point LCL_VARS in the integer registers
4421 // So for now we will use GT_LCLFLD's to pass this struct (it won't be enregistered)
4423 JITDUMP("Multireg struct V%02u will be passed using GT_LCLFLD because it has float fields.\n", varNum);
4425 // we call lvaSetVarDoNotEnregister and do the proper transformation below.
4430 // We can use the struct promoted field as the two arguments
4432 GenTreePtr loLclVar = gtNewLclvNode(loVarNum, loType, loVarNum);
4433 GenTreePtr hiLclVar = gtNewLclvNode(hiVarNum, hiType, hiVarNum);
4435 // Create a new tree for 'arg'
4436 // replace the existing LDOBJ(ADDR(LCLVAR))
4437 // with a LIST(LCLVAR-LO, LIST(LCLVAR-HI, nullptr))
4439 newArg = gtNewListNode(loLclVar, gtNewArgList(hiLclVar));
4444 // Check if we couldn't transform the LDOBJ(ADDR(LCLVAR)) into a struct promoted GT_LIST above
4445 if (newArg == nullptr)
4448 // We weren't able to pass this LclVar using it's struct promted fields
4450 // So instead we will create a list of GT_LCL_FLDs nodes to pass this struct
4452 lvaSetVarDoNotEnregister(varNum DEBUG_ARG(DNER_LocalField));
4454 // If this is going in the register area, we transform it here into a GT_LIST of LCLFLD's
4455 // If this is going in the outgoing arg area, it will be transformed later
4457 if (fgEntryPtr->regNum != REG_STK)
4459 GenTreeLclVarCommon* varNode = argValue->AsLclVarCommon();
4460 unsigned varNum = varNode->gtLclNum;
4461 assert(varNum < lvaCount);
4462 LclVarDsc* varDsc = &lvaTable[varNum];
4464 GenTreePtr loLclFld = gtNewLclFldNode(varNum, type0, 0);
4465 GenTreePtr hiLclFld = gtNewLclFldNode(varNum, type1, TARGET_POINTER_SIZE);
4467 // Create a new tree for 'arg'
4468 // replace the existing LDOBJ(ADDR(LCLVAR))
4469 // with a LIST(LCLFLD-LO, LIST(LCLFLD-HI, nullptr))
4471 newArg = gtNewListNode(loLclFld, gtNewArgList(hiLclFld));
4475 // Check if we already created a replacement newArg above
4476 if (newArg == nullptr)
4478 if (fgEntryPtr->regNum == REG_STK)
4480 // We leave this stack passed argument alone
4485 // Are we passing a GT_LCL_FLD which contain a 16-byte struct inside it?
4487 if (argValue->OperGet() == GT_LCL_FLD)
4489 GenTreeLclVarCommon* varNode = argValue->AsLclVarCommon();
4490 unsigned varNum = varNode->gtLclNum;
4491 assert(varNum < lvaCount);
4492 LclVarDsc* varDsc = &lvaTable[varNum];
4494 unsigned baseOffset = argValue->gtLclFld.gtLclOffs;
4495 unsigned baseIndex = baseOffset / TARGET_POINTER_SIZE;
4496 unsigned requiredSize = baseOffset + (2 * TARGET_POINTER_SIZE);
4498 // The allocated size of our LocalVar must be at least as big as requiredSize
4499 assert(varDsc->lvSize() >= requiredSize);
4501 const BYTE * gcPtrs = varDsc->lvGcLayout;
4503 var_types type0 = getJitGCType(gcPtrs[baseIndex+0]);
4504 var_types type1 = getJitGCType(gcPtrs[baseIndex+1]);
4507 // We create a list of two GT_LCL_FLDs nodes to pass this struct
4509 lvaSetVarDoNotEnregister(varNum DEBUG_ARG(DNER_LocalField));
4511 GenTreePtr loLclFld = gtNewLclFldNode(varNum, type0, baseOffset);
4512 GenTreePtr hiLclFld = gtNewLclFldNode(varNum, type1, baseOffset + TARGET_POINTER_SIZE);
4514 // Create a new tree for 'arg'
4515 // replace the existing LDOBJ(ADDR(LCLVAR))
4516 // with a LIST(LCLFLD-LO, LIST(LCLFLD-HI, nullptr))
4518 newArg = gtNewListNode(loLclFld, gtNewArgList(hiLclFld));
4520 // Are we passing a GT_OBJ struct?
4522 else if (argValue->OperGet() == GT_OBJ)
4524 GenTreeObj* argObj = argValue->AsObj();
4525 CORINFO_CLASS_HANDLE objClass = argObj->gtClass;
4527 int structSize = info.compCompHnd->getClassSize(objClass);
4528 assert(structSize <= 2 * TARGET_POINTER_SIZE);
4529 BYTE gcPtrs[2] = { TYPE_GC_NONE, TYPE_GC_NONE };
4530 info.compCompHnd->getClassGClayout(objClass, &gcPtrs[0]);
4532 var_types type0 = getJitGCType(gcPtrs[0]);
4533 var_types type1 = getJitGCType(gcPtrs[1]);
4535 GenTreePtr baseAddr = argObj->gtOp1;
4536 GenTreePtr baseAddrDup = gtCloneExpr(baseAddr);
4537 noway_assert(baseAddrDup != nullptr);
4539 var_types addrType = baseAddr->TypeGet();
4540 GenTreePtr loAddr = baseAddr;
4541 GenTreePtr hiAddr = gtNewOperNode(GT_ADD, addrType, baseAddrDup, gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL));
4542 GenTreePtr loValue = gtNewOperNode(GT_IND, type0, loAddr);
4543 GenTreePtr hiValue = gtNewOperNode(GT_IND, type1, hiAddr);
4545 // Create a new tree for 'arg'
4546 // replace the existing LDOBJ(EXPR)
4547 // with a LIST(IND(EXPR), LIST(IND(EXPR+8), nullptr))
4549 newArg = gtNewListNode(loValue, gtNewArgList(hiValue));
4552 // If we reach here we should have set newArg to something
4553 if (newArg == nullptr)
4556 gtDispTree(argValue);
4558 assert(!"Missing case in fgMorphMultiregStructArg");
4564 printf("fgMorphMultiregStructArg created tree:\n");
4569 arg = newArg; // consider calling fgMorphTree(newArg);
4573 // Make a copy of a struct variable if necessary, to pass to a callee.
4574 // returns: tree that computes address of the outgoing arg
4576 Compiler::fgMakeOutgoingStructArgCopy(GenTreeCall* call,
4579 CORINFO_CLASS_HANDLE copyBlkClass
4580 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(const SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR* const structDescPtr))
4582 GenTree* argx = args->Current();
4583 noway_assert(argx->gtOper != GT_MKREFANY);
4584 // See if we need to insert a copy at all
4585 // Case 1: don't need a copy if it is the last use of a local. We can't determine that all of the time
4586 // but if there is only one use and no loops, the use must be last.
4587 if (argx->gtOper == GT_OBJ)
4589 GenTree* lcl = argx->gtOp.gtOp1;
4590 if (lcl->OperIsLocal())
4592 unsigned varNum = lcl->AsLclVarCommon()->GetLclNum();
4593 if (lvaIsImplicitByRefLocal(varNum))
4595 LclVarDsc* varDsc = &lvaTable[varNum];
4596 if (varDsc->lvRefCnt == 1 && !fgMightHaveLoop())
4598 varDsc->lvRefCnt = 0;
4599 args->gtOp.gtOp1 = lcl;
4600 fgArgTabEntryPtr fp = Compiler::gtArgEntryByNode(call, argx);
4603 JITDUMP("did not have to make outgoing copy for V%2d", varNum);
4604 varDsc->lvRefCnt = 0;
4609 varDsc->lvRefCnt = 0;
4615 if (fgOutgoingArgTemps == nullptr)
4616 fgOutgoingArgTemps = hashBv::Create(this);
4621 // Attempt to find a local we have already used for an outgoing struct and reuse it.
4622 // We do not reuse within a statement.
4623 if (!opts.MinOpts())
4626 FOREACH_HBV_BIT_SET(lclNum, fgOutgoingArgTemps)
4628 LclVarDsc* varDsc = &lvaTable[lclNum];
4629 if (typeInfo::AreEquivalent(varDsc->lvVerTypeInfo, typeInfo(TI_STRUCT, copyBlkClass))
4630 && !fgCurrentlyInUseArgTemps->testBit(lclNum))
4632 tmp = (unsigned) lclNum;
4634 JITDUMP("reusing outgoing struct arg");
4641 // Create the CopyBlk tree and insert it.
4645 // Here We don't need unsafe value cls check, since the addr of this temp is used only in copyblk.
4646 tmp = lvaGrabTemp(true DEBUGARG("by-value struct argument"));
4647 lvaSetStruct(tmp, copyBlkClass, false);
4648 fgOutgoingArgTemps->setBit(tmp);
4651 fgCurrentlyInUseArgTemps->setBit(tmp);
4653 // TYP_SIMD structs should not be enregistered, since ABI requires it to be
4654 // allocated on stack and address of it needs to be passed.
4655 if (lclVarIsSIMDType(tmp))
4657 lvaSetVarDoNotEnregister(tmp DEBUGARG(DNER_IsStruct));
4660 // Create a reference to the temp
4661 GenTreePtr dest = gtNewLclvNode(tmp, lvaTable[tmp].lvType);
4662 dest->gtFlags |= (GTF_DONT_CSE | GTF_VAR_DEF); // This is a def of the local, "entire" by construction.
4663 dest = gtNewOperNode(GT_ADDR, TYP_BYREF, dest);
4664 lvaTable[tmp].incRefCnts(compCurBB->getBBWeight(this), this);
4667 if (argx->gtOper == GT_OBJ)
4669 src = argx->gtOp.gtOp1;
4673 argx->gtFlags |= GTF_DONT_CSE;
4674 src = gtNewOperNode(GT_ADDR, TYP_BYREF, argx);
4677 // Copy the valuetype to the temp
4678 GenTreePtr copyBlk = gtNewCpObjNode(dest, src, copyBlkClass, false);
4679 copyBlk = fgMorphCopyBlock(copyBlk);
4681 #if FEATURE_FIXED_OUT_ARGS
4683 // Do the copy early, and evalute the temp later (see EvalArgsToTemps)
4684 // When on Unix create LCL_FLD for structs passed in more than one registers. See fgMakeTmpArgNode
4685 GenTreePtr arg = copyBlk;
4687 #else // FEATURE_FIXED_OUT_ARGS
4689 // Structs are always on the stack, and thus never need temps
4690 // so we have to put the copy and temp all into one expression
4691 GenTreePtr arg = fgMakeTmpArgNode(
4693 FEATURE_UNIX_AMD64_STRUCT_PASSING_ONLY_ARG(structDescPtr->passedInRegisters));
4695 // Change the expression to "(tmp=val),tmp"
4696 arg = gtNewOperNode(GT_COMMA, arg->TypeGet(), copyBlk, arg);
4698 #endif // FEATURE_FIXED_OUT_ARGS
4700 args->gtOp.gtOp1 = arg;
4701 call->fgArgInfo->EvalToTmp(argIndex, tmp, arg);
4707 // See declaration for specification comment.
4708 void Compiler::fgAddSkippedRegsInPromotedStructArg(LclVarDsc* varDsc,
4709 unsigned firstArgRegNum,
4710 regMaskTP* pArgSkippedRegMask)
4712 assert(varDsc->lvPromoted);
4713 // There's no way to do these calculations without breaking abstraction and assuming that
4714 // integer register arguments are consecutive ints. They are on ARM.
4716 // To start, figure out what register contains the last byte of the first argument.
4717 LclVarDsc* firstFldVarDsc = &lvaTable[varDsc->lvFieldLclStart];
4718 unsigned lastFldRegOfLastByte = (firstFldVarDsc->lvFldOffset + firstFldVarDsc->lvExactSize - 1) / TARGET_POINTER_SIZE;;
4720 // Now we're keeping track of the register that the last field ended in; see what registers
4721 // subsequent fields start in, and whether any are skipped.
4722 // (We assume here the invariant that the fields are sorted in offset order.)
4723 for (unsigned fldVarOffset = 1; fldVarOffset < varDsc->lvFieldCnt; fldVarOffset++)
4725 unsigned fldVarNum = varDsc->lvFieldLclStart + fldVarOffset;
4726 LclVarDsc* fldVarDsc = &lvaTable[fldVarNum];
4727 unsigned fldRegOffset = fldVarDsc->lvFldOffset / TARGET_POINTER_SIZE;
4728 assert(fldRegOffset >= lastFldRegOfLastByte); // Assuming sorted fields.
4729 // This loop should enumerate the offsets of any registers skipped.
4730 // Find what reg contains the last byte:
4731 // And start at the first register after that. If that isn't the first reg of the current
4732 for (unsigned skippedRegOffsets = lastFldRegOfLastByte + 1; skippedRegOffsets < fldRegOffset; skippedRegOffsets++)
4734 // If the register number would not be an arg reg, we're done.
4735 if (firstArgRegNum + skippedRegOffsets >= MAX_REG_ARG) return;
4736 *pArgSkippedRegMask |= genRegMask(regNumber(firstArgRegNum + skippedRegOffsets));
4738 lastFldRegOfLastByte = (fldVarDsc->lvFldOffset + fldVarDsc->lvExactSize - 1) / TARGET_POINTER_SIZE;
4742 #endif // _TARGET_ARM_
4745 /*****************************************************************************
4747 * The companion to impFixupCallStructReturn. Now that the importer is done
4748 * and we no longer care as much about the declared return type, change to
4749 * precomputed native return type (at least for architectures that don't
4750 * always use return buffers for structs).
4753 void Compiler::fgFixupStructReturn(GenTreePtr callNode)
4755 GenTreeCall* call = callNode->AsCall();
4756 bool callHasRetBuffArg = call->HasRetBufArg();
4758 if (!callHasRetBuffArg && varTypeIsStruct(call))
4761 if (call->gtCall.IsVarargs() || !IsHfa(call))
4764 // Now that we are past the importer, re-type this node so the register predictor does
4766 call->gtType = genActualType((var_types)call->gtCall.gtReturnType);
4771 // Either we don't have a struct now or if struct, then it is HFA returned in regs.
4772 assert(!varTypeIsStruct(call) || (IsHfa(call) && !callHasRetBuffArg));
4773 #elif defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
4774 // Either we don't have a struct now or if struct, then it is a struct returned in regs or in return buffer.
4775 assert(!varTypeIsStruct(call) || call->HasMultiRegRetVal() || callHasRetBuffArg);
4777 // No more struct returns
4778 assert(call->TypeGet() != TYP_STRUCT);
4781 #if !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
4782 // If it was a struct return, it has been transformed into a call
4783 // with a return buffer (that returns TYP_VOID) or into a return
4784 // of a primitive/enregisterable type
4785 assert(!callHasRetBuffArg || (call->TypeGet() == TYP_VOID));
4790 /*****************************************************************************
4792 * A little helper used to rearrange nested commutative operations. The
4793 * effect is that nested associative, commutative operations are transformed
4794 * into a 'left-deep' tree, i.e. into something like this:
4796 * (((a op b) op c) op d) op...
4801 void Compiler::fgMoveOpsLeft(GenTreePtr tree)
4809 op1 = tree->gtOp.gtOp1;
4810 op2 = tree->gtOp.gtOp2;
4811 oper = tree->OperGet();
4813 noway_assert(GenTree::OperIsCommutative(oper));
4814 noway_assert(oper == GT_ADD || oper == GT_XOR || oper == GT_OR ||
4815 oper == GT_AND || oper == GT_MUL);
4816 noway_assert(!varTypeIsFloating(tree->TypeGet()) || !opts.genFPorder);
4817 noway_assert(oper == op2->gtOper);
4819 // Commutativity doesn't hold if overflow checks are needed
4821 if (tree->gtOverflowEx() || op2->gtOverflowEx())
4824 if (gtIsActiveCSE_Candidate(op2))
4826 // If we have marked op2 as a CSE candidate,
4827 // we can't perform a commutative reordering
4828 // because any value numbers that we computed for op2
4829 // will be incorrect after performing a commutative reordering
4834 if (oper == GT_MUL && (op2->gtFlags & GTF_MUL_64RSLT))
4837 // Check for GTF_ADDRMODE_NO_CSE flag on add/mul Binary Operators
4838 if ( ((oper == GT_ADD) || (oper == GT_MUL))
4839 && ((tree->gtFlags & GTF_ADDRMODE_NO_CSE) != 0) )
4844 if ( (tree->gtFlags | op2->gtFlags) & GTF_BOOLEAN )
4846 // We could deal with this, but we were always broken and just hit the assert
4847 // below regarding flags, which means it's not frequent, so will just bail out.
4852 noway_assert(!tree->gtOverflowEx() && !op2->gtOverflowEx());
4854 GenTreePtr ad1 = op2->gtOp.gtOp1;
4855 GenTreePtr ad2 = op2->gtOp.gtOp2;
4857 // Compiler::optOptimizeBools() can create GT_OR of two GC pointers yeilding a GT_INT
4858 // We can not reorder such GT_OR trees
4860 if (varTypeIsGC(ad1->TypeGet()) != varTypeIsGC(op2->TypeGet()))
4863 /* Change "(x op (y op z))" to "(x op y) op z" */
4864 /* ie. "(op1 op (ad1 op ad2))" to "(op1 op ad1) op ad2" */
4866 GenTreePtr new_op1 = op2;
4868 new_op1->gtOp.gtOp1 = op1;
4869 new_op1->gtOp.gtOp2 = ad1;
4871 /* Change the flags. */
4873 // Make sure we arent throwing away any flags
4874 noway_assert((new_op1->gtFlags & ~(
4876 GTF_DONT_CSE | // It is ok that new_op1->gtFlags contains GTF_DONT_CSE flag.
4877 GTF_REVERSE_OPS | // The reverse ops flag also can be set, it will be re-calculated
4878 GTF_NODE_MASK|GTF_ALL_EFFECT|GTF_UNSIGNED)) == 0);
4880 new_op1->gtFlags = (new_op1->gtFlags & (GTF_NODE_MASK | GTF_DONT_CSE)) | // Make sure we propagate GTF_DONT_CSE flag.
4881 (op1->gtFlags & GTF_ALL_EFFECT) |
4882 (ad1->gtFlags & GTF_ALL_EFFECT);
4884 /* Retype new_op1 if it has not/become a GC ptr. */
4886 if (varTypeIsGC(op1->TypeGet()))
4888 noway_assert((varTypeIsGC(tree->TypeGet()) && op2->TypeGet() == TYP_I_IMPL && oper == GT_ADD) || // byref(ref + (int+int))
4889 (varTypeIsI (tree->TypeGet()) && op2->TypeGet() == TYP_I_IMPL && oper == GT_OR)); // int(gcref | int(gcref|intval))
4891 new_op1->gtType = tree->gtType;
4893 else if (varTypeIsGC(ad2->TypeGet()))
4895 // Neither ad1 nor op1 are GC. So new_op1 isnt either
4896 noway_assert(op1->gtType == TYP_I_IMPL && ad1->gtType == TYP_I_IMPL);
4897 new_op1->gtType = TYP_I_IMPL;
4900 // If new_op1 is a new expression. Assign it a new unique value number.
4901 // vnStore is null before the ValueNumber phase has run
4902 if (vnStore != nullptr)
4904 // We can only keep the old value number on new_op1 if both op1 and ad2
4905 // have the same non-NoVN value numbers. Since op is commutative, comparing
4906 // only ad2 and op1 is enough.
4907 if ((op1->gtVNPair.GetLiberal() == ValueNumStore::NoVN) ||
4908 (ad2->gtVNPair.GetLiberal() == ValueNumStore::NoVN) ||
4909 (ad2->gtVNPair.GetLiberal() != op1->gtVNPair.GetLiberal()))
4911 new_op1->gtVNPair.SetBoth(vnStore->VNForExpr(new_op1->TypeGet()));
4915 tree->gtOp.gtOp1 = new_op1;
4916 tree->gtOp.gtOp2 = ad2;
4918 /* If 'new_op1' is now the same nested op, process it recursively */
4920 if ((ad1->gtOper == oper) && !ad1->gtOverflowEx())
4921 fgMoveOpsLeft(new_op1);
4923 /* If 'ad2' is now the same nested op, process it
4924 * Instead of recursion, we set up op1 and op2 for the next loop.
4930 while ((op2->gtOper == oper) && !op2->gtOverflowEx());
4937 /*****************************************************************************/
4939 void Compiler::fgSetRngChkTarget(GenTreePtr tree,
4942 GenTreeBoundsChk* bndsChk = nullptr;
4943 SpecialCodeKind kind = SCK_RNGCHK_FAIL;
4946 if ((tree->gtOper == GT_ARR_BOUNDS_CHECK) || (tree->gtOper == GT_SIMD_CHK))
4947 #else // FEATURE_SIMD
4948 if (tree->gtOper == GT_ARR_BOUNDS_CHECK)
4949 #endif // FEATURE_SIMD
4951 bndsChk = tree->AsBoundsChk();
4952 kind = tree->gtBoundsChk.gtThrowKind;
4956 noway_assert((tree->gtOper == GT_ARR_ELEM) || (tree->gtOper == GT_ARR_INDEX));
4960 unsigned callStkDepth = fgPtrArgCntCur;
4962 // only x86 pushes args
4963 const unsigned callStkDepth = 0;
4970 // we need to initialize this field
4971 if (fgGlobalMorph && bndsChk != nullptr)
4973 bndsChk->gtStkDepth = callStkDepth;
4977 if (!opts.compDbgCode)
4979 if (delay || compIsForInlining())
4981 /* We delay this until after loop-oriented range check
4982 analysis. For now we merely store the current stack
4983 level in the tree node.
4985 if (bndsChk != nullptr)
4987 noway_assert(!bndsChk->gtIndRngFailBB || previousCompletedPhase >= PHASE_OPTIMIZE_LOOPS);
4988 bndsChk->gtStkDepth = callStkDepth;
4993 /* Create/find the appropriate "range-fail" label */
4995 // fgPtrArgCntCur is only valid for global morph or if we walk full stmt.
4996 noway_assert((bndsChk != nullptr) || fgGlobalMorph);
4998 unsigned stkDepth = (bndsChk != nullptr) ? bndsChk->gtStkDepth
5001 BasicBlock * rngErrBlk = fgRngChkTarget(compCurBB, stkDepth, kind);
5003 /* Add the label to the indirection node */
5005 if (bndsChk != nullptr)
5007 bndsChk->gtIndRngFailBB = gtNewCodeRef(rngErrBlk);
5013 /*****************************************************************************
5015 * Expand a GT_INDEX node and fully morph the child operands
5017 * The orginal GT_INDEX node is bashed into the GT_IND node that accesses
5018 * the array element. We expand the GT_INDEX node into a larger tree that
5019 * evaluates the array base and index. The simplest expansion is a GT_COMMA
5020 * with a GT_ARR_BOUND_CHK and a GT_IND with a GTF_INX_RNGCHK flag.
5021 * For complex array or index expressions one or more GT_COMMA assignments
5022 * are inserted so that we only evaluate the array or index expressions once.
5024 * The fully expanded tree is then morphed. This causes gtFoldExpr to
5025 * perform local constant prop and reorder the constants in the tree and
5028 * We then parse the resulting array element expression in order to locate
5029 * and label the constants and variables that occur in the tree.
5032 const int MAX_ARR_COMPLEXITY = 4;
5033 const int MAX_INDEX_COMPLEXITY = 4;
5035 GenTreePtr Compiler::fgMorphArrayIndex(GenTreePtr tree)
5037 noway_assert(tree->gtOper == GT_INDEX);
5038 GenTreeIndex* asIndex = tree->AsIndex();
5040 var_types elemTyp = tree->TypeGet();
5041 unsigned elemSize = tree->gtIndex.gtIndElemSize;
5042 CORINFO_CLASS_HANDLE elemStructType = tree->gtIndex.gtStructElemClass;
5044 noway_assert(elemTyp != TYP_STRUCT || elemStructType != nullptr);
5047 if (featureSIMD && varTypeIsStruct(elemTyp) && elemSize <= getSIMDVectorRegisterByteLength())
5049 // If this is a SIMD type, this is the point at which we lose the type information,
5050 // so we need to set the correct type on the GT_IND.
5051 // (We don't care about the base type here, so we only check, but don't retain, the return value).
5052 unsigned simdElemSize = 0;
5053 if (getBaseTypeAndSizeOfSIMDType(elemStructType, &simdElemSize) != TYP_UNKNOWN)
5055 assert(simdElemSize == elemSize);
5056 elemTyp = getSIMDTypeForSize(elemSize);
5057 // This is the new type of the node.
5058 tree->gtType = elemTyp;
5059 // Now set elemStructType to null so that we don't confuse value numbering.
5060 elemStructType = nullptr;
5063 #endif // FEATURE_SIMD
5065 GenTreePtr arrRef = asIndex->Arr();
5066 GenTreePtr index = asIndex->Index();
5068 // Set up the the array length's offset into lenOffs
5069 // And the the first element's offset into elemOffs
5072 if (tree->gtFlags & GTF_INX_STRING_LAYOUT)
5074 lenOffs = offsetof(CORINFO_String, stringLen);
5075 elemOffs = offsetof(CORINFO_String, chars);
5076 tree->gtFlags &= ~GTF_INX_STRING_LAYOUT; // Clear this flag as it is used for GTF_IND_VOLATILE
5078 else if (tree->gtFlags & GTF_INX_REFARR_LAYOUT)
5080 lenOffs = offsetof(CORINFO_RefArray, length);
5081 elemOffs = eeGetEEInfo()->offsetOfObjArrayData;
5083 else // We have a standard array
5085 lenOffs = offsetof(CORINFO_Array, length);
5086 elemOffs = offsetof(CORINFO_Array, u1Elems);
5089 bool chkd = ((tree->gtFlags & GTF_INX_RNGCHK) != 0); // if false, range checking will be disabled
5090 bool nCSE = ((tree->gtFlags & GTF_DONT_CSE ) != 0);
5092 GenTreePtr arrRefDefn = nullptr; // non-NULL if we need to allocate a temp for the arrRef expression
5093 GenTreePtr indexDefn = nullptr; // non-NULL if we need to allocate a temp for the index expression
5094 GenTreePtr bndsChk = nullptr;
5096 // If we're doing range checking, introduce a GT_ARR_BOUNDS_CHECK node for the address.
5099 GenTreePtr arrRef2 = nullptr; // The second copy will be used in array address expression
5100 GenTreePtr index2 = nullptr;
5102 // If the arrRef expression involves an assignment, a call or reads from global memory,
5103 // then we *must* allocate a temporary in which to "localize" those values,
5104 // to ensure that the same values are used in the bounds check and the actual
5106 // Also we allocate the temporary when the arrRef is sufficiently complex/expensive.
5108 if ((arrRef->gtFlags & (GTF_ASG|GTF_CALL|GTF_GLOB_REF)) || gtComplexityExceeds(&arrRef, MAX_ARR_COMPLEXITY))
5110 unsigned arrRefTmpNum = lvaGrabTemp(true DEBUGARG("arr expr"));
5111 arrRefDefn = gtNewTempAssign(arrRefTmpNum, arrRef);
5112 arrRef = gtNewLclvNode(arrRefTmpNum, arrRef->TypeGet());
5113 arrRef2 = gtNewLclvNode(arrRefTmpNum, arrRef->TypeGet());
5117 arrRef2 = gtCloneExpr(arrRef);
5118 noway_assert(arrRef2 != nullptr);
5121 // If the index expression involves an assignment, a call or reads from global memory,
5122 // we *must* allocate a temporary in which to "localize" those values,
5123 // to ensure that the same values are used in the bounds check and the actual
5125 // Also we allocate the temporary when the index is sufficiently complex/expensive.
5127 if ((index->gtFlags & (GTF_ASG|GTF_CALL|GTF_GLOB_REF)) || gtComplexityExceeds(&index, MAX_ARR_COMPLEXITY))
5129 unsigned indexTmpNum = lvaGrabTemp(true DEBUGARG("arr expr"));
5130 indexDefn = gtNewTempAssign(indexTmpNum, index);
5131 index = gtNewLclvNode(indexTmpNum, index->TypeGet());
5132 index2 = gtNewLclvNode(indexTmpNum, index->TypeGet());
5136 index2 = gtCloneExpr(index);
5137 noway_assert(index2 != nullptr);
5140 // Next introduce a GT_ARR_BOUNDS_CHECK node
5141 var_types bndsChkType = TYP_INT; // By default, try to use 32-bit comparison for array bounds check.
5143 #ifdef _TARGET_64BIT_
5144 // The CLI Spec allows an array to be indexed by either an int32 or a native int. In the case
5145 // of a 64 bit architecture this means the array index can potentially be a TYP_LONG, so for this case,
5146 // the comparison will have to be widen to 64 bits.
5147 if (index->TypeGet() == TYP_I_IMPL)
5149 bndsChkType = TYP_I_IMPL;
5151 #endif // _TARGET_64BIT_
5153 GenTree* arrLen = new (this, GT_ARR_LENGTH) GenTreeArrLen(TYP_INT, arrRef, (int)lenOffs);
5155 if (bndsChkType != TYP_INT)
5157 arrLen = gtNewCastNode(bndsChkType, arrLen, bndsChkType);
5160 GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK) GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, arrLen, index, SCK_RNGCHK_FAIL);
5162 bndsChk = arrBndsChk;
5164 // Make sure to increment ref-counts if already ref-counted.
5165 if (lvaLocalVarRefCounted)
5167 lvaRecursiveIncRefCounts(index);
5168 lvaRecursiveIncRefCounts(arrRef);
5171 // Now we'll switch to using the second copies for arrRef and index
5172 // to compute the address expression
5178 // Create the "addr" which is "*(arrRef + ((index * elemSize) + elemOffs))"
5182 // Widen 'index' on 64-bit targets
5183 #ifdef _TARGET_64BIT_
5184 if (index->TypeGet() != TYP_I_IMPL)
5186 if (index->OperGet() == GT_CNS_INT)
5188 index->gtType = TYP_I_IMPL;
5192 index = gtNewCastNode(TYP_I_IMPL, index, TYP_I_IMPL);
5195 #endif // _TARGET_64BIT_
5197 /* Scale the index value if necessary */
5200 GenTreePtr size = gtNewIconNode(elemSize, TYP_I_IMPL);
5202 // Fix 392756 WP7 Crossgen
5204 // During codegen optGetArrayRefScaleAndIndex() makes the assumption that op2 of a GT_MUL node
5205 // is a constant and is not capable of handling CSE'ing the elemSize constant into a lclvar.
5206 // Hence to prevent the constant from becoming a CSE we mark it as NO_CSE.
5208 size->gtFlags |= GTF_DONT_CSE;
5210 /* Multiply by the array element size */
5211 addr = gtNewOperNode(GT_MUL, TYP_I_IMPL, index, size);
5218 /* Add the object ref to the element's offset */
5220 addr = gtNewOperNode(GT_ADD, TYP_BYREF, arrRef, addr);
5222 /* Add the first element's offset */
5224 GenTreePtr cns = gtNewIconNode(elemOffs, TYP_I_IMPL);
5226 addr = gtNewOperNode(GT_ADD, TYP_BYREF, addr, cns);
5228 #if SMALL_TREE_NODES
5229 assert(tree->gtFlags & GTF_NODE_LARGE);
5232 // Change the orginal GT_INDEX node into a GT_IND node
5233 tree->SetOper(GT_IND);
5235 // If the index node is a floating-point type, notify the compiler
5236 // we'll potentially use floating point registers at the time of codegen.
5237 if (varTypeIsFloating(tree->gtType))
5239 this->compFloatingPointUsed = true;
5242 // We've now consumed the GTF_INX_RNGCHK, and the node
5243 // is no longer a GT_INDEX node.
5244 tree->gtFlags &= ~GTF_INX_RNGCHK;
5246 tree->gtOp.gtOp1 = addr;
5248 // This is an array index expression.
5249 tree->gtFlags |= GTF_IND_ARR_INDEX;
5251 /* An indirection will cause a GPF if the address is null */
5252 tree->gtFlags |= GTF_EXCEPT;
5255 tree->gtFlags |= GTF_DONT_CSE;
5257 // Store information about it.
5258 GetArrayInfoMap()->Set(tree, ArrayInfo(elemTyp, elemSize, (int) elemOffs, elemStructType));
5260 // Remember this 'indTree' that we just created, as we still need to attach the fieldSeq information to it.
5262 GenTreePtr indTree = tree;
5264 // Did we create a bndsChk tree?
5267 // Use a GT_COMMA node to prepend the array bound check
5269 tree = gtNewOperNode(GT_COMMA, elemTyp, bndsChk, tree);
5271 /* Mark the indirection node as needing a range check */
5272 fgSetRngChkTarget(bndsChk);
5275 if (indexDefn != nullptr)
5277 // Use a GT_COMMA node to prepend the index assignment
5279 tree = gtNewOperNode(GT_COMMA, tree->TypeGet(), indexDefn, tree);
5281 if (arrRefDefn != nullptr)
5283 // Use a GT_COMMA node to prepend the arRef assignment
5285 tree = gtNewOperNode(GT_COMMA, tree->TypeGet(), arrRefDefn, tree);
5288 // Currently we morph the tree to perform some folding operations prior
5289 // to attaching fieldSeq info and labeling constant array index contributions
5293 // Ideally we just want to proceed to attaching fieldSeq info and labeling the
5294 // constant array index contributions, but the morphing operation may have changed
5295 // the 'tree' into something that now unconditionally throws an exception.
5297 // In such case the gtEffectiveVal could be a new tree or it's gtOper could be modified
5298 // or it could be left unchanged. If it is unchanged then we should not return,
5299 // instead we should proceed to attaching fieldSeq info, etc...
5301 GenTreePtr arrElem = tree->gtEffectiveVal();
5303 if (fgIsCommaThrow(tree))
5305 if ((arrElem != indTree) || // A new tree node may have been created
5306 (indTree->OperGet() != GT_IND)) // The GT_IND may have been changed to a GT_CNS_INT
5308 return tree; // Just return the Comma-Throw, don't try to attach the fieldSeq info, etc..
5312 assert(!fgGlobalMorph || (arrElem->gtFlags & GTF_MORPHED));
5314 addr = arrElem->gtOp.gtOp1;
5316 assert(addr->TypeGet() == TYP_BYREF);
5318 GenTreePtr cnsOff = nullptr;
5319 if (addr->OperGet() == GT_ADD)
5321 if (addr->gtOp.gtOp2->gtOper == GT_CNS_INT)
5323 cnsOff = addr->gtOp.gtOp2;
5324 addr = addr->gtOp.gtOp1;
5327 while ((addr->OperGet() == GT_ADD) || (addr->OperGet() == GT_SUB))
5329 assert(addr->TypeGet() == TYP_BYREF);
5330 GenTreePtr index = addr->gtOp.gtOp2;
5332 // Label any constant array index contributions with #ConstantIndex and any LclVars with GTF_VAR_ARR_INDEX
5333 index->LabelIndex(this);
5335 addr = addr->gtOp.gtOp1;
5337 assert(addr->TypeGet() == TYP_REF);
5339 else if (addr->OperGet() == GT_CNS_INT)
5344 FieldSeqNode* firstElemFseq = GetFieldSeqStore()->CreateSingleton(FieldSeqStore::FirstElemPseudoField);
5346 if ((cnsOff != nullptr) && (cnsOff->gtIntCon.gtIconVal == elemOffs))
5348 // Assign it the [#FirstElem] field sequence
5350 cnsOff->gtIntCon.gtFieldSeq = firstElemFseq;
5352 else // We have folded the first element's offset with the index expression
5354 // Build the [#ConstantIndex, #FirstElem] field sequence
5356 FieldSeqNode* constantIndexFseq = GetFieldSeqStore()->CreateSingleton(FieldSeqStore::ConstantIndexPseudoField);
5357 FieldSeqNode* fieldSeq = GetFieldSeqStore()->Append(constantIndexFseq, firstElemFseq);
5359 if (cnsOff == nullptr) // It must have folded into a zero offset
5361 // Record in the general zero-offset map.
5362 GetZeroOffsetFieldMap()->Set(addr, fieldSeq);
5366 cnsOff->gtIntCon.gtFieldSeq = fieldSeq;
5374 /*****************************************************************************
5376 * Wrap fixed stack arguments for varargs functions to go through varargs
5377 * cookie to access them, except for the cookie itself.
5379 * Non-x86 platforms are allowed to access all arguments directly
5380 * so we don't need this code.
5383 GenTreePtr Compiler::fgMorphStackArgForVarArgs(unsigned lclNum, var_types varType, unsigned lclOffs)
5385 /* For the fixed stack arguments of a varargs function, we need to go
5386 through the varargs cookies to access them, except for the
5389 LclVarDsc * varDsc = &lvaTable[lclNum];
5391 if (varDsc->lvIsParam && !varDsc->lvIsRegArg &&
5392 lclNum != lvaVarargsHandleArg)
5394 // Create a node representing the local pointing to the base of the args
5395 GenTreePtr ptrArg = gtNewOperNode(GT_SUB, TYP_I_IMPL,
5396 gtNewLclvNode(lvaVarargsBaseOfStkArgs, TYP_I_IMPL),
5397 gtNewIconNode(varDsc->lvStkOffs
5398 - codeGen->intRegState.rsCalleeRegArgNum*sizeof(void*)
5401 // Access the argument through the local
5402 GenTreePtr tree = gtNewOperNode(GT_IND, varType, ptrArg);
5403 tree->gtFlags |= GTF_IND_TGTANYWHERE;
5405 if (varDsc->lvAddrExposed)
5407 tree->gtFlags |= GTF_GLOB_REF;
5410 return fgMorphTree(tree);
5417 /*****************************************************************************
5419 * Transform the given GT_LCL_VAR tree for code generation.
5422 GenTreePtr Compiler::fgMorphLocalVar(GenTreePtr tree)
5424 noway_assert(tree->gtOper == GT_LCL_VAR);
5426 unsigned lclNum = tree->gtLclVarCommon.gtLclNum;
5427 var_types varType = lvaGetRealType(lclNum);
5428 LclVarDsc * varDsc = &lvaTable[lclNum];
5430 if (varDsc->lvAddrExposed)
5432 tree->gtFlags |= GTF_GLOB_REF;
5436 if (info.compIsVarArgs)
5438 GenTreePtr newTree = fgMorphStackArgForVarArgs(lclNum, varType, 0);
5439 if (newTree != NULL)
5442 #endif // _TARGET_X86_
5444 /* If not during the global morphing phase bail */
5449 bool varAddr = (tree->gtFlags & GTF_DONT_CSE) != 0;
5451 noway_assert(!(tree->gtFlags & GTF_VAR_DEF) || varAddr); // GTF_VAR_DEF should always imply varAddr
5454 varTypeIsSmall(varDsc->TypeGet()) &&
5455 varDsc->lvNormalizeOnLoad())
5457 #if LOCAL_ASSERTION_PROP
5458 /* Assertion prop can tell us to omit adding a cast here */
5459 if (optLocalAssertionProp &&
5460 optAssertionIsSubrange(tree, varType, apFull) != NO_ASSERTION_INDEX)
5465 /* Small-typed arguments and aliased locals are normalized on load.
5466 Other small-typed locals are normalized on store.
5467 Also, under the debugger as the debugger could write to the variable.
5468 If this is one of the former, insert a narrowing cast on the load.
5469 ie. Convert: var-short --> cast-short(var-int) */
5471 tree->gtType = TYP_INT;
5472 fgMorphTreeDone(tree);
5473 tree = gtNewCastNode(TYP_INT, tree, varType);
5474 fgMorphTreeDone(tree);
5482 /*****************************************************************************
5483 Grab a temp for big offset morphing.
5484 This method will grab a new temp if no temp of this "type" has been created.
5485 Or it will return the same cached one if it has been created.
5487 unsigned Compiler::fgGetBigOffsetMorphingTemp(var_types type)
5489 unsigned lclNum = fgBigOffsetMorphingTemps[type];
5491 if (lclNum == BAD_VAR_NUM) {
5492 // We haven't created a temp for this kind of type. Create one now.
5493 lclNum = lvaGrabTemp(false DEBUGARG("Big Offset Morphing"));
5494 fgBigOffsetMorphingTemps[type] = lclNum;
5497 // We better get the right type.
5498 noway_assert(lvaTable[lclNum].TypeGet() == type);
5501 noway_assert(lclNum != BAD_VAR_NUM);
5506 /*****************************************************************************
5508 * Transform the given GT_FIELD tree for code generation.
5511 GenTreePtr Compiler::fgMorphField(GenTreePtr tree, MorphAddrContext* mac)
5513 assert(tree->gtOper == GT_FIELD);
5515 noway_assert(tree->gtFlags & GTF_GLOB_REF);
5517 CORINFO_FIELD_HANDLE symHnd = tree->gtField.gtFldHnd;
5518 unsigned fldOffset = tree->gtField.gtFldOffset;
5519 GenTreePtr objRef = tree->gtField.gtFldObj;
5520 bool fieldMayOverlap = false;
5521 if (tree->gtField.gtFldMayOverlap)
5523 fieldMayOverlap = true;
5524 // Reset the flag because we may reuse the node.
5525 tree->gtField.gtFldMayOverlap = false;
5529 // if this field belongs to simd struct, tranlate it to simd instrinsic.
5530 if (mac == nullptr || mac->m_kind != MACK_Addr)
5532 GenTreePtr newTree = fgMorphFieldToSIMDIntrinsicGet(tree);
5533 if (newTree != tree)
5535 newTree = fgMorphSmpOp(newTree);
5539 else if (objRef != nullptr && objRef->OperGet() == GT_ADDR && objRef->OperIsSIMD())
5541 // We have a field of an SIMD intrinsic in an address-taken context.
5542 // We need to copy the SIMD result to a temp, and take the field of that.
5543 GenTree* copy = fgCopySIMDNode(objRef->gtOp.gtOp1->AsSIMD());
5544 objRef->gtOp.gtOp1 = copy;
5548 /* Is this an instance data member? */
5554 if (tree->gtFlags & GTF_IND_TLS_REF)
5555 NO_WAY("instance field can not be a TLS ref.");
5557 /* We'll create the expression "*(objRef + mem_offs)" */
5559 noway_assert(varTypeIsGC(objRef->TypeGet()) ||
5560 objRef->TypeGet() == TYP_I_IMPL);
5562 // An optimization for Contextful classes:
5563 // we unwrap the proxy when we have a 'this reference'
5564 if (info.compIsContextful &&
5565 info.compUnwrapContextful &&
5568 objRef = fgUnwrapProxy(objRef);
5572 Now we have a tree like this:
5574 +--------------------+
5576 +----------+---------+
5578 +--------------+-------------+
5579 | tree->gtField.gtFldObj |
5580 +--------------+-------------+
5583 We want to make it like this (when fldOffset is <= MAX_UNCHECKED_OFFSET_FOR_NULL_OBJECT):
5585 +--------------------+
5586 | GT_IND/GT_OBJ | tree
5587 +---------+----------+
5590 +---------+----------+
5592 +---------+----------+
5597 +-------------------+ +----------------------+
5598 | objRef | | fldOffset |
5599 | | | (when fldOffset !=0) |
5600 +-------------------+ +----------------------+
5603 or this (when fldOffset is > MAX_UNCHECKED_OFFSET_FOR_NULL_OBJECT):
5606 +--------------------+
5607 | GT_IND/GT_OBJ | tree
5608 +----------+---------+
5610 +----------+---------+
5612 +----------+---------+
5618 +---------+----------+ +---------+----------+
5619 comma | GT_COMMA | | "+" (i.e. GT_ADD) | addr
5620 +---------+----------+ +---------+----------+
5625 +-----+-----+ +-----+-----+ +---------+ +-----------+
5626 asg | GT_ASG | ind | GT_IND | | tmpLcl | | fldOffset |
5627 +-----+-----+ +-----+-----+ +---------+ +-----------+
5632 +-----+-----+ +-----+-----+ +-----------+
5633 | tmpLcl | | objRef | | tmpLcl |
5634 +-----------+ +-----------+ +-----------+
5639 var_types objRefType = objRef->TypeGet();
5641 GenTreePtr comma = NULL;
5643 bool addedExplicitNullCheck = false;
5645 // NULL mac means we encounter the GT_FIELD first. This denotes a dereference of the field,
5646 // and thus is equivalent to a MACK_Ind with zero offset.
5647 MorphAddrContext defMAC(MACK_Ind);
5648 if (mac == NULL) mac = &defMAC;
5650 // This flag is set to enable the "conservative" style of explicit null-check insertion.
5651 // This means that we insert an explicit null check whenever we create byref by adding a
5652 // constant offset to a ref, in a MACK_Addr context (meaning that the byref is not immediately
5653 // dereferenced). The alternative is "aggressive", which would not insert such checks (for
5654 // small offsets); in this plan, we would transfer some null-checking responsibility to
5655 // callee's of methods taking byref parameters. They would have to add explicit null checks
5656 // when creating derived byrefs from argument byrefs by adding constants to argument byrefs, in
5657 // contexts where the resulting derived byref is not immediately dereferenced (or if the offset is too
5658 // large). To make the "aggressive" scheme work, however, we'd also have to add explicit derived-from-null
5659 // checks for byref parameters to "external" methods implemented in C++, and in P/Invoke stubs.
5660 /// This is left here to point out how to implement it.
5661 #define CONSERVATIVE_NULL_CHECK_BYREF_CREATION 1
5663 // If the objRef is a GT_ADDR node, it, itself, never requires null checking. The expression
5664 // whose address is being taken is either a local or static variable, whose address is necessarily
5665 // non-null, or else it is a field dereference, which will do its own bounds checking if necessary.
5666 if (objRef->gtOper != GT_ADDR
5667 && ((mac->m_kind == MACK_Addr || mac->m_kind == MACK_Ind)
5668 && (!mac->m_allConstantOffsets
5669 || fgIsBigOffset(mac->m_totalOffset + fldOffset)
5670 #if CONSERVATIVE_NULL_CHECK_BYREF_CREATION
5671 || (mac->m_kind == MACK_Addr && (mac->m_totalOffset + fldOffset > 0))
5673 || (objRef->gtType == TYP_BYREF && mac->m_kind == MACK_Addr && (mac->m_totalOffset + fldOffset > 0))
5680 printf("Before explicit null check morphing:\n");
5686 // Create the "comma" subtree
5688 GenTreePtr asg = NULL;
5693 if (objRef->gtOper != GT_LCL_VAR)
5695 lclNum = fgGetBigOffsetMorphingTemp(genActualType(objRef->TypeGet()));
5697 // Create the "asg" node
5698 asg = gtNewTempAssign(lclNum, objRef);
5702 lclNum = objRef->gtLclVarCommon.gtLclNum;
5705 // Create the "nullchk" node
5706 nullchk = gtNewOperNode(GT_NULLCHECK,
5707 TYP_BYTE, // Make it TYP_BYTE so we only deference it for 1 byte.
5708 gtNewLclvNode(lclNum, objRefType));
5709 nullchk->gtFlags |= GTF_DONT_CSE; // Don't try to create a CSE for these TYP_BYTE indirections
5711 /* An indirection will cause a GPF if the address is null */
5712 nullchk->gtFlags |= GTF_EXCEPT;
5716 // Create the "comma" node.
5717 comma = gtNewOperNode(GT_COMMA,
5718 TYP_VOID, // We don't want to return anything from this "comma" node.
5719 // Set the type to TYP_VOID, so we can select "cmp" instruction
5720 // instead of "mov" instruction later on.
5729 addr = gtNewLclvNode(lclNum, objRefType); // Use "tmpLcl" to create "addr" node.
5731 addedExplicitNullCheck = true;
5733 else if (fldOffset == 0)
5735 // Generate the "addr" node.
5737 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5738 GetZeroOffsetFieldMap()->Set(addr, fieldSeq);
5745 #ifdef FEATURE_READYTORUN_COMPILER
5746 if (tree->gtField.gtFieldLookup.addr != nullptr)
5748 GenTreePtr baseOffset = gtNewIconEmbHndNode(tree->gtField.gtFieldLookup.addr, nullptr, GTF_ICON_FIELD_HDL);
5750 if (tree->gtField.gtFieldLookup.accessType == IAT_PVALUE)
5751 baseOffset = gtNewOperNode(GT_IND, TYP_I_IMPL, baseOffset);
5753 addr = gtNewOperNode(GT_ADD,
5754 (var_types)(objRefType == TYP_I_IMPL ? TYP_I_IMPL
5763 // Generate the "addr" node.
5764 /* Add the member offset to the object's address */
5765 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5766 addr = gtNewOperNode(GT_ADD,
5767 (var_types)(objRefType == TYP_I_IMPL ? TYP_I_IMPL
5770 gtNewIconHandleNode(fldOffset,
5775 // Now let's set the "tree" as a GT_IND tree.
5777 tree->SetOper(GT_IND);
5778 tree->gtOp.gtOp1 = addr;
5780 if (fgAddrCouldBeNull(addr))
5782 // This indirection can cause a GPF if the address could be null.
5783 tree->gtFlags |= GTF_EXCEPT;
5786 if (addedExplicitNullCheck)
5789 // Create "comma2" node and link it to "tree".
5792 comma2 = gtNewOperNode(GT_COMMA,
5793 addr->TypeGet(), // The type of "comma2" node is the same as the type of "addr" node.
5796 tree->gtOp.gtOp1 = comma2;
5802 if (addedExplicitNullCheck) {
5803 printf("After adding explicit null check:\n");
5810 else /* This is a static data member */
5812 if (tree->gtFlags & GTF_IND_TLS_REF)
5814 // Thread Local Storage static field reference
5816 // Field ref is a TLS 'Thread-Local-Storage' reference
5818 // Build this tree: IND(*) #
5826 // IND(I_IMPL) == [Base of this DLL's TLS]
5830 // / CNS(IdValue*4) or MUL
5832 // IND(I_IMPL) / CNS(4)
5834 // CNS(TLS_HDL,0x2C) IND
5838 // # Denotes the orginal node
5840 void ** pIdAddr = NULL;
5841 unsigned IdValue = info.compCompHnd->getFieldThreadLocalStoreID(symHnd, (void**) &pIdAddr);
5844 // If we can we access the TLS DLL index ID value directly
5845 // then pIdAddr will be NULL and
5846 // IdValue will be the actual TLS DLL index ID
5848 GenTreePtr dllRef = NULL;
5849 if (pIdAddr == NULL)
5852 dllRef = gtNewIconNode(IdValue*4, TYP_I_IMPL);
5856 dllRef = gtNewIconHandleNode((size_t)pIdAddr, GTF_ICON_STATIC_HDL);
5857 dllRef = gtNewOperNode(GT_IND, TYP_I_IMPL, dllRef);
5858 dllRef->gtFlags |= GTF_IND_INVARIANT;
5862 dllRef = gtNewOperNode(GT_MUL, TYP_I_IMPL, dllRef, gtNewIconNode(4, TYP_I_IMPL));
5865 #define WIN32_TLS_SLOTS (0x2C) // Offset from fs:[0] where the pointer to the slots resides
5867 // Mark this ICON as a TLS_HDL, codegen will use FS:[cns]
5869 GenTreePtr tlsRef = gtNewIconHandleNode(WIN32_TLS_SLOTS, GTF_ICON_TLS_HDL);
5871 tlsRef = gtNewOperNode(GT_IND, TYP_I_IMPL, tlsRef);
5875 /* Add the dllRef */
5876 tlsRef = gtNewOperNode(GT_ADD, TYP_I_IMPL, tlsRef, dllRef);
5879 /* indirect to have tlsRef point at the base of the DLLs Thread Local Storage */
5880 tlsRef = gtNewOperNode(GT_IND, TYP_I_IMPL, tlsRef);
5884 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5885 GenTreePtr fldOffsetNode = new(this, GT_CNS_INT) GenTreeIntCon(TYP_INT, fldOffset, fieldSeq);
5887 /* Add the TLS static field offset to the address */
5889 tlsRef = gtNewOperNode(GT_ADD, TYP_I_IMPL, tlsRef, fldOffsetNode);
5892 // Final indirect to get to actual value of TLS static field
5894 tree->SetOper(GT_IND);
5895 tree->gtOp.gtOp1 = tlsRef;
5897 noway_assert(tree->gtFlags & GTF_IND_TLS_REF);
5901 // Normal static field reference
5904 // If we can we access the static's address directly
5905 // then pFldAddr will be NULL and
5906 // fldAddr will be the actual address of the static field
5908 void ** pFldAddr = NULL;
5909 void * fldAddr = info.compCompHnd->getFieldAddress(symHnd, (void**) &pFldAddr);
5911 if (pFldAddr == NULL)
5913 #ifdef _TARGET_64BIT_
5914 if (IMAGE_REL_BASED_REL32 != eeGetRelocTypeHint(fldAddr))
5916 // The address is not directly addressible, so force it into a
5917 // constant, so we handle it properly
5919 GenTreePtr addr = gtNewIconHandleNode((size_t)fldAddr, GTF_ICON_STATIC_HDL);
5920 addr->gtType = TYP_I_IMPL;
5921 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5922 addr->gtIntCon.gtFieldSeq = fieldSeq;
5924 tree->SetOper(GT_IND);
5925 tree->gtOp.gtOp1 = addr;
5927 return fgMorphSmpOp(tree);
5930 #endif // _TARGET_64BIT_
5932 // Only volatile could be set, and it maps over
5933 noway_assert((tree->gtFlags & ~(GTF_FLD_VOLATILE | GTF_COMMON_MASK)) == 0);
5934 noway_assert(GTF_FLD_VOLATILE == GTF_IND_VOLATILE);
5935 tree->SetOper(GT_CLS_VAR);
5936 tree->gtClsVar.gtClsVarHnd = symHnd;
5937 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5938 tree->gtClsVar.gtFieldSeq = fieldSeq;
5945 GenTreePtr addr = gtNewIconHandleNode((size_t)pFldAddr, GTF_ICON_STATIC_HDL);
5947 // There are two cases here, either the static is RVA based,
5948 // in which case the type of the FIELD node is not a GC type
5949 // and the handle to the RVA is a TYP_I_IMPL. Or the FIELD node is
5950 // a GC type and the handle to it is a TYP_BYREF in the GC heap
5951 // because handles to statics now go into the large object heap
5953 var_types handleTyp = (var_types) (varTypeIsGC(tree->TypeGet()) ? TYP_BYREF
5955 GenTreePtr op1 = gtNewOperNode(GT_IND, handleTyp, addr);
5956 op1->gtFlags |= GTF_IND_INVARIANT;
5958 tree->SetOper(GT_IND);
5959 tree->gtOp.gtOp1 = op1;
5963 noway_assert(tree->gtOper == GT_IND);
5965 GenTreePtr res = fgMorphSmpOp(tree);
5967 if (fldOffset == 0 && res->OperGet() == GT_IND)
5969 GenTreePtr addr = res->gtOp.gtOp1;
5970 // Since we don't make a constant zero to attach the field sequence to, associate it with the "addr" node.
5971 FieldSeqNode* fieldSeq = fieldMayOverlap ? FieldSeqStore::NotAField() : GetFieldSeqStore()->CreateSingleton(symHnd);
5972 fgAddFieldSeqForZeroOffset(addr, fieldSeq);
5980 //------------------------------------------------------------------------------
5981 // fgMorphCallInline: attempt to inline a call
5984 // call - call expression to inline, inline candidate
5985 // inlineResult - result tracking and reporting
5988 // Attempts to inline the call.
5990 // If successful, callee's IR is inserted in place of the call, and
5991 // is marked with an InlineContext.
5993 // If unsuccessful, the transformations done in anticpation of a
5994 // possible inline are undone, and the candidate flag on the call
5997 void Compiler::fgMorphCallInline(GenTreeCall* call, InlineResult* inlineResult)
5999 // The call must be a candiate for inlining.
6000 assert((call->gtFlags & GTF_CALL_INLINE_CANDIDATE) != 0);
6002 // Attempt the inline
6003 fgMorphCallInlineHelper(call, inlineResult);
6005 // We should have made up our minds one way or another....
6006 assert(inlineResult->IsDecided());
6008 // If we failed to inline, we have a bit of work to do to cleanup
6009 if (inlineResult->IsFailure())
6014 // Before we do any cleanup, create a failing InlineContext to
6015 // capture details of the inlining attempt.
6016 m_inlineStrategy->NewFailure(fgMorphStmt, inlineResult);
6020 // It was an inline candidate, but we haven't expanded it.
6021 if (call->gtCall.gtReturnType != TYP_VOID)
6023 // Detach the GT_CALL tree from the original statement by
6024 // hanging a "nothing" node to it. Later the "nothing" node will be removed
6025 // and the original GT_CALL tree will be picked up by the GT_RET_EXPR node.
6027 noway_assert(fgMorphStmt->gtStmt.gtStmtExpr == call);
6028 fgMorphStmt->gtStmt.gtStmtExpr = gtNewNothingNode();
6031 // Clear the Inline Candidate flag so we can ensure later we tried
6032 // inlining all candidates.
6034 call->gtFlags &= ~GTF_CALL_INLINE_CANDIDATE;
6038 /*****************************************************************************
6039 * Helper to attempt to inline a call
6040 * Sets success/failure in inline result
6041 * If success, modifies current method's IR with inlinee's IR
6042 * If failed, undoes any speculative modifications to current method
6045 void Compiler::fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result)
6047 // Don't expect any surprises here.
6048 assert(result->IsCandidate());
6050 if (lvaCount >= MAX_LV_NUM_COUNT_FOR_INLINING)
6052 // For now, attributing this to call site, though it's really
6053 // more of a budget issue (lvaCount currently includes all
6054 // caller and prospective callee locals). We still might be
6055 // able to inline other callees into this caller, or inline
6056 // this callee in other callers.
6057 result->NoteFatal(InlineObservation::CALLSITE_TOO_MANY_LOCALS);
6061 if (call->IsVirtual())
6063 result->NoteFatal(InlineObservation::CALLSITE_IS_VIRTUAL);
6067 // impMarkInlineCandidate() is expected not to mark tail prefixed calls
6068 // and recursive tail calls as inline candidates.
6069 noway_assert(!call->IsTailPrefixedCall());
6070 noway_assert(!call->IsImplicitTailCall() || !gtIsRecursiveCall(call));
6072 /* If the caller's stack frame is marked, then we can't do any inlining. Period.
6073 Although we have checked this in impCanInline, it is possible that later IL instructions
6074 might cause compNeedSecurityCheck to be set. Therefore we need to check it here again.
6077 if (opts.compNeedSecurityCheck)
6079 result->NoteFatal(InlineObservation::CALLER_NEEDS_SECURITY_CHECK);
6084 // Calling inlinee's compiler to inline the method.
6087 unsigned startVars = lvaCount;
6092 printf("Expanding INLINE_CANDIDATE in statement ");
6093 printTreeID(fgMorphStmt);
6094 printf(" in BB%02u:\n", compCurBB->bbNum);
6095 gtDispTree(fgMorphStmt);
6097 // printf("startVars=%d.\n", startVars);
6102 // Invoke the compiler to inline the call.
6105 fgInvokeInlineeCompiler(call, result);
6107 if (result->IsFailure())
6109 // Undo some changes made in anticipation of inlining...
6111 // Zero out the used locals
6112 memset(lvaTable + startVars, 0, (lvaCount - startVars) * sizeof(*lvaTable));
6113 for (unsigned i = startVars; i < lvaCount; i++)
6115 new (&lvaTable[i], jitstd::placement_t()) LclVarDsc(this); // call the constructor.
6118 lvaCount = startVars;
6123 // printf("Inlining failed. Restore lvaCount to %d.\n", lvaCount);
6135 // printf("After inlining lvaCount=%d.\n", lvaCount);
6140 /*****************************************************************************
6142 * Performs checks to see if this tail call can be optimized as epilog+jmp.
6144 bool Compiler::fgCanFastTailCall(GenTreeCall* callee)
6146 #if FEATURE_FASTTAILCALL
6147 // Reached here means that return types of caller and callee are tail call compatible.
6148 // In case of structs that can be returned in a register, compRetNativeType is set to the actual return type.
6150 // In an implicit tail call case callSig may not be available but it is guaranteed to be available
6151 // for explicit tail call cases. The reason implicit tail case callSig may not be available is that
6152 // a call node might be marked as an in-line candidate and could fail to be in-lined. In which case
6153 // fgInline() will replace return value place holder with call node using gtCloneExpr() which is
6154 // currently not copying/setting callSig.
6156 if (callee->IsTailPrefixedCall())
6158 assert(impTailCallRetTypeCompatible(info.compRetNativeType, info.compMethodInfo->args.retTypeClass,
6159 (var_types)callee->gtReturnType, callee->callSig->retTypeClass));
6163 // Note on vararg methods:
6164 // If the caller is vararg method, we don't know the number of arguments passed by caller's caller.
6165 // But we can be sure that in-coming arg area of vararg caller would be sufficient to hold its
6166 // fixed args. Therefore, we can allow a vararg method to fast tail call other methods as long as
6167 // out-going area required for callee is bounded by caller's fixed argument space.
6169 // Note that callee being a vararg method is not a problem since we can account the params being passed.
6171 // Count of caller args including implicit and hidden (i.e. thisPtr, RetBuf, GenericContext, VarargCookie)
6172 unsigned nCallerArgs = info.compArgsCount;
6174 // Count the callee args including implicit and hidden.
6175 // Note that GenericContext and VarargCookie are added by importer while
6176 // importing the call to gtCallArgs list along with explicit user args.
6177 unsigned nCalleeArgs = 0;
6178 if (callee->gtCallObjp) // thisPtr
6183 if (callee->HasRetBufArg()) // RetBuf
6187 // If callee has RetBuf param, caller too must have it.
6188 // Otherwise go the slow route.
6189 if (info.compRetBuffArg == BAD_VAR_NUM)
6195 // Count user args while tracking whether any of them is a multi-byte params
6196 // that cannot be passed in a register. Note that we don't need to count
6197 // non-standard and secret params passed in registers (e.g. R10, R11) since
6198 // these won't contribute to out-going arg size.
6199 bool hasMultiByteArgs = false;
6200 for (GenTreePtr args = callee->gtCallArgs; (args != nullptr) && !hasMultiByteArgs; args = args->gtOp.gtOp2)
6204 assert(args->IsList());
6205 GenTreePtr argx = args->gtOp.gtOp1;
6207 if (varTypeIsStruct(argx))
6209 // Actual arg may be a child of a GT_COMMA. Skip over comma opers.
6210 while (argx->gtOper == GT_COMMA)
6212 argx = argx->gtOp.gtOp2;
6215 // Get the size of the struct and see if it is register passable.
6216 if (argx->OperGet() == GT_OBJ)
6218 #if defined(_TARGET_AMD64_) || defined(_TARGET_ARM64_)
6220 unsigned typeSize = 0;
6221 hasMultiByteArgs = !VarTypeIsMultiByteAndCanEnreg(argx->TypeGet(), argx->gtObj.gtClass, &typeSize, false);
6223 #if defined(FEATURE_UNIX_AMD64_STRUCT_PASSING) || defined(_TARGET_ARM64_)
6224 // On System V/arm64 the args could be a 2 eightbyte struct that is passed in two registers.
6225 // Account for the second eightbyte in the nCalleeArgs.
6226 // https://github.com/dotnet/coreclr/issues/2666
6227 // TODO-CQ-Amd64-Unix/arm64: Structs of size between 9 to 16 bytes are conservatively estimated
6228 // as two args, since they need two registers whereas nCallerArgs is
6229 // counting such an arg as one. This would mean we will not be optimizing
6230 // certain calls though technically possible.
6232 if (typeSize > TARGET_POINTER_SIZE)
6234 unsigned extraArgRegsToAdd = (typeSize / TARGET_POINTER_SIZE);
6235 nCalleeArgs += extraArgRegsToAdd;
6237 #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING || _TARGET_ARM64_
6240 assert(!"Target platform ABI rules regarding passing struct type args in registers");
6242 #endif //_TARGET_AMD64_ || _TARGET_ARM64_
6247 hasMultiByteArgs = true;
6252 // Go the slow route, if it has multi-byte params
6253 if (hasMultiByteArgs)
6258 // If we reached here means that callee has only those argument types which can be passed in
6259 // a register and if passed on stack will occupy exactly one stack slot in out-going arg area.
6260 // If we are passing args on stack for callee and it has more args passed on stack than
6261 // caller, then fast tail call cannot be performed.
6263 // Note that the GC'ness of on stack args need not match since the arg setup area is marked
6264 // as non-interruptible for fast tail calls.
6265 if ((nCalleeArgs > MAX_REG_ARG) && (nCallerArgs < nCalleeArgs))
6277 /*****************************************************************************
6279 * Transform the given GT_CALL tree for tail call code generation.
6281 void Compiler::fgMorphTailCall(GenTreeCall* call)
6283 // x86 classic codegen doesn't require any morphing
6284 #if defined(_TARGET_X86_) && !defined(LEGACY_BACKEND)
6285 NYI_X86("Tail call morphing");
6286 #elif defined(_TARGET_ARM_)
6287 // For the helper-assisted tail calls, we need to push all the arguments
6288 // into a single list, and then add a few extra at the beginning
6290 // Check for PInvoke call types that we don't handle in codegen yet.
6291 assert(!call->IsUnmanaged());
6292 assert(call->IsVirtual() ||
6293 (call->gtCallType != CT_INDIRECT) ||
6294 (call->gtCallCookie == NULL));
6296 // First move the this pointer (if any) onto the regular arg list
6297 GenTreePtr thisPtr = NULL;
6298 if (call->gtCallObjp)
6300 GenTreePtr objp = call->gtCallObjp;
6301 call->gtCallObjp = NULL;
6303 if ((call->gtFlags & GTF_CALL_NULLCHECK) ||
6304 call->IsVirtualVtable())
6306 thisPtr = gtClone(objp, true);
6307 var_types vt = objp->TypeGet();
6308 if (thisPtr == NULL)
6310 // Too complex, so use a temp
6311 unsigned lclNum = lvaGrabTemp(true DEBUGARG("tail call thisptr"));
6312 GenTreePtr asg = gtNewTempAssign(lclNum, objp);
6313 if (!call->IsVirtualVtable())
6315 // Add an indirection to get the nullcheck
6316 GenTreePtr tmp = gtNewLclvNode(lclNum, vt);
6317 GenTreePtr ind = gtNewOperNode(GT_IND, TYP_INT, tmp);
6318 asg = gtNewOperNode(GT_COMMA, TYP_VOID, asg, ind);
6320 objp = gtNewOperNode(GT_COMMA, vt, asg, gtNewLclvNode(lclNum, vt));
6321 thisPtr = gtNewLclvNode(lclNum, vt);
6323 else if (!call->IsVirtualVtable())
6325 GenTreePtr ind = gtNewOperNode(GT_IND, TYP_INT, thisPtr);
6326 objp = gtNewOperNode(GT_COMMA, vt, ind, objp);
6327 thisPtr = gtClone(thisPtr, true);
6330 call->gtFlags &= ~GTF_CALL_NULLCHECK;
6333 GenTreeArgList** pList = &call->gtCallArgs;
6334 #if RETBUFARG_PRECEDES_THIS
6335 if (call->HasRetBufArg()) {
6336 pList = &(*pList)->Rest();
6338 #endif // RETBUFARG_PRECEDES_THIS
6339 *pList = gtNewListNode(objp, *pList);
6342 // Add the extra VSD parameter if needed
6343 CorInfoHelperTailCallSpecialHandling flags = CorInfoHelperTailCallSpecialHandling(0);
6344 if (call->IsVirtualStub())
6346 flags = CORINFO_TAILCALL_STUB_DISPATCH_ARG;
6349 if (call->gtCallType == CT_INDIRECT) {
6350 arg = gtClone(call->gtCallAddr, true);
6351 noway_assert(arg != NULL);
6354 noway_assert(call->gtCallMoreFlags & GTF_CALL_M_VIRTSTUB_REL_INDIRECT);
6355 ssize_t addr = ssize_t(call->gtStubCallStubAddr);
6356 arg = gtNewIconHandleNode(addr, GTF_ICON_FTN_ADDR);
6358 // Change the call type, so we can add the extra indirection here, rather than in codegen
6359 call->gtCallAddr = gtNewIconHandleNode(addr, GTF_ICON_FTN_ADDR);
6360 call->gtStubCallStubAddr = NULL;
6361 call->gtCallType = CT_INDIRECT;
6363 // Add the extra indirection to generate the real target
6364 call->gtCallAddr = gtNewOperNode(GT_IND, TYP_I_IMPL, call->gtCallAddr);
6365 call->gtFlags |= GTF_EXCEPT;
6367 // And push the stub address onto the list of arguments
6368 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
6370 else if (call->IsVirtualVtable())
6372 // TODO-ARM-NYI: for x64 handle CORINFO_TAILCALL_THIS_IN_SECRET_REGISTER
6374 noway_assert(thisPtr != NULL);
6376 GenTreePtr add = gtNewOperNode(GT_ADD, TYP_I_IMPL, thisPtr, gtNewIconNode(VPTR_OFFS, TYP_I_IMPL));
6377 GenTreePtr vtbl = gtNewOperNode(GT_IND, TYP_I_IMPL, add);
6378 vtbl->gtFlags |= GTF_EXCEPT;
6380 unsigned vtabOffsOfIndirection;
6381 unsigned vtabOffsAfterIndirection;
6382 info.compCompHnd->getMethodVTableOffset(call->gtCallMethHnd, &vtabOffsOfIndirection, &vtabOffsAfterIndirection);
6384 /* Get the appropriate vtable chunk */
6386 add = gtNewOperNode(GT_ADD, TYP_I_IMPL, vtbl, gtNewIconNode(vtabOffsOfIndirection, TYP_I_IMPL));
6387 vtbl = gtNewOperNode(GT_IND, TYP_I_IMPL, add);
6389 /* Now the appropriate vtable slot */
6391 add = gtNewOperNode(GT_ADD, TYP_I_IMPL, vtbl, gtNewIconNode(vtabOffsAfterIndirection, TYP_I_IMPL));
6392 vtbl = gtNewOperNode(GT_IND, TYP_I_IMPL, add);
6394 // Switch this to a plain indirect call
6395 call->gtFlags &= ~GTF_CALL_VIRT_KIND_MASK;
6396 assert(!call->IsVirtual());
6397 call->gtCallType = CT_INDIRECT;
6399 call->gtCallAddr = vtbl;
6400 call->gtCallCookie = NULL;
6401 call->gtFlags |= GTF_EXCEPT;
6404 // Now inject a placeholder for the real call target that codegen
6406 GenTreePtr arg = new (this, GT_NOP) GenTreeOp(GT_NOP, TYP_I_IMPL);
6407 codeGen->genMarkTreeInReg(arg, REG_TAILCALL_ADDR);
6408 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
6410 // Lastly inject the pointer for the copy routine
6411 noway_assert(call->callSig != NULL);
6412 void * pfnCopyArgs = info.compCompHnd->getTailCallCopyArgsThunk(call->callSig, flags);
6413 arg = gtNewIconHandleNode(ssize_t(pfnCopyArgs), GTF_ICON_FTN_ADDR);
6414 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
6416 // It is now a varargs tail call
6417 call->gtCallMoreFlags = GTF_CALL_M_VARARGS | GTF_CALL_M_TAILCALL;
6418 call->gtFlags &= ~GTF_CALL_POP_ARGS;
6420 #elif defined(_TARGET_AMD64_)
6421 // For the helper-assisted tail calls, we need to push all the arguments
6422 // into a single list, and then add a few extra at the beginning.
6424 // TailCallHelper(void *copyRoutine, void *callTarget, ....) - i.e We need to add
6425 // copyRoutine and callTarget extra params at the beginning. But callTarget is
6426 // determined by Lower phase. Therefore, we add a place holder arg for callTarget
6427 // here which will be later replaced with callTarget in tail call lowering.
6429 // Check for PInvoke call types that we don't handle in codegen yet.
6430 assert(!call->IsUnmanaged());
6431 assert(call->IsVirtual() ||
6432 (call->gtCallType != CT_INDIRECT) ||
6433 (call->gtCallCookie == NULL));
6435 // Don't support tail calling helper methods
6436 assert(call->gtCallType != CT_HELPER);
6438 // We come this route only for tail prefixed calls that cannot be dispatched as
6440 assert(!call->IsImplicitTailCall());
6441 assert(!fgCanFastTailCall(call));
6443 // First move the this pointer (if any) onto the regular arg list
6444 if (call->gtCallObjp)
6446 GenTreePtr thisPtr = nullptr;
6447 GenTreePtr objp = call->gtCallObjp;
6448 call->gtCallObjp = nullptr;
6450 if (call->NeedsNullCheck())
6452 // clone "this" if "this" has no side effects.
6453 if (!(objp->gtFlags & GTF_SIDE_EFFECT))
6455 thisPtr = gtClone(objp, true);
6458 var_types vt = objp->TypeGet();
6459 if (thisPtr == nullptr)
6461 // create a temp if either "this" has side effects or "this" is too complex to clone.
6464 unsigned lclNum = lvaGrabTemp(true DEBUGARG("tail call thisptr"));
6465 GenTreePtr asg = gtNewTempAssign(lclNum, objp);
6467 // COMMA(tmp = "this", deref(tmp))
6468 GenTreePtr tmp = gtNewLclvNode(lclNum, vt);
6469 GenTreePtr ind = gtNewOperNode(GT_IND, TYP_INT, tmp);
6470 asg = gtNewOperNode(GT_COMMA, TYP_VOID, asg, ind);
6472 // COMMA(COMMA(tmp = "this", deref(tmp)), tmp)
6473 thisPtr = gtNewOperNode(GT_COMMA, vt, asg, gtNewLclvNode(lclNum, vt));
6477 // thisPtr = COMMA(deref("this"), "this")
6478 GenTreePtr ind = gtNewOperNode(GT_IND, TYP_INT, thisPtr);
6479 thisPtr = gtNewOperNode(GT_COMMA, vt, ind, gtClone(objp, true));
6482 call->gtFlags &= ~GTF_CALL_NULLCHECK;
6489 GenTreeArgList** pList = &call->gtCallArgs;
6490 #if RETBUFARG_PRECEDES_THIS
6491 if (call->HasRetBufArg()) {
6492 pList = &(*pList)->Rest();
6494 #endif // RETBUFARG_PRECEDES_THIS
6496 // During rationalization tmp="this" and null check will
6497 // materialize as embedded stmts in right execution order.
6498 assert(thisPtr != nullptr);
6499 *pList = gtNewListNode(thisPtr, *pList);
6502 // Add the extra VSD parameter to arg list in case of VSD calls.
6503 // Tail call arg copying thunk will move this extra VSD parameter
6504 // to R11 before tail calling VSD stub. See CreateTailCallCopyArgsThunk()
6505 // in Stublinkerx86.cpp for more details.
6506 CorInfoHelperTailCallSpecialHandling flags = CorInfoHelperTailCallSpecialHandling(0);
6507 if (call->IsVirtualStub())
6509 GenTreePtr stubAddrArg;
6511 flags = CORINFO_TAILCALL_STUB_DISPATCH_ARG;
6513 if (call->gtCallType == CT_INDIRECT)
6515 stubAddrArg = gtClone(call->gtCallAddr, true);
6516 noway_assert(stubAddrArg != nullptr);
6520 noway_assert((call->gtCallMoreFlags & GTF_CALL_M_VIRTSTUB_REL_INDIRECT) != 0);
6522 ssize_t addr = ssize_t(call->gtStubCallStubAddr);
6523 stubAddrArg = gtNewIconHandleNode(addr, GTF_ICON_FTN_ADDR);
6526 // Push the stub address onto the list of arguments
6527 call->gtCallArgs = gtNewListNode(stubAddrArg, call->gtCallArgs);
6530 // Now inject a placeholder for the real call target that Lower phase will generate.
6531 GenTreePtr arg = gtNewIconNode(0, TYP_I_IMPL);
6532 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
6534 // Inject the pointer for the copy routine to be used for struct copying
6535 noway_assert(call->callSig != nullptr);
6536 void * pfnCopyArgs = info.compCompHnd->getTailCallCopyArgsThunk(call->callSig, flags);
6537 arg = gtNewIconHandleNode(ssize_t(pfnCopyArgs), GTF_ICON_FTN_ADDR);
6538 call->gtCallArgs = gtNewListNode(arg, call->gtCallArgs);
6540 // It is now a varargs tail call dispatched via helper.
6541 call->gtCallMoreFlags |= GTF_CALL_M_VARARGS | GTF_CALL_M_TAILCALL | GTF_CALL_M_TAILCALL_VIA_HELPER;
6542 call->gtFlags &= ~GTF_CALL_POP_ARGS;
6544 #endif //_TARGET_AMD64_
6548 //------------------------------------------------------------------------------
6549 // fgMorphRecursiveFastTailCallIntoLoop : Transform a recursive fast tail call into a loop.
6553 // block - basic block ending with a recursive fast tail call
6554 // recursiveTailCall - recursive tail call to transform
6557 // The legality of the transformation is ensured by the checks in endsWithTailCallConvertibleToLoop.
6559 void Compiler::fgMorphRecursiveFastTailCallIntoLoop(BasicBlock* block, GenTreeCall* recursiveTailCall)
6561 assert(recursiveTailCall->IsTailCallConvertibleToLoop());
6562 GenTreePtr last = fgGetLastTopLevelStmt(block);
6563 assert(recursiveTailCall == last->gtStmt.gtStmtExpr);
6565 // Transform recursive tail call into a loop.
6567 GenTreePtr earlyArgInsertionPoint = last;
6568 IL_OFFSETX callILOffset = last->gtStmt.gtStmtILoffsx;
6570 // Hoist arg setup statement for the 'this' argument.
6571 GenTreePtr thisArg = recursiveTailCall->gtCallObjp;
6572 if (thisArg && !thisArg->IsNothingNode() && !thisArg->IsArgPlaceHolderNode())
6574 GenTreePtr thisArgStmt = gtNewStmt(thisArg, callILOffset);
6575 fgInsertStmtBefore(block, earlyArgInsertionPoint, thisArgStmt);
6579 // All arguments whose trees may involve caller parameter local variables need to be assigned to temps first;
6580 // then the temps need to be assigned to the method parameters. This is done so that the caller
6581 // parameters are not re-assigned before call arguments depending on them are evaluated.
6582 // tmpAssignmentInsertionPoint and paramAssignmentInsertionPoint keep track of
6583 // where the next temp or parameter assignment should be inserted.
6585 // In the example below the first call argument (arg1 - 1) needs to be assigned to a temp first
6586 // while the second call argument (const 1) doesn't.
6587 // Basic block before tail recursion elimination:
6588 // ***** BB04, stmt 1 (top level)
6589 // [000037] ------------ * stmtExpr void (top level) (IL 0x00A...0x013)
6590 // [000033] --C - G------ - \--* call void RecursiveMethod
6591 // [000030] ------------ | / --* const int - 1
6592 // [000031] ------------arg0 in rcx + --* +int
6593 // [000029] ------------ | \--* lclVar int V00 arg1
6594 // [000032] ------------arg1 in rdx \--* const int 1
6597 // Basic block after tail recursion elimination :
6598 // ***** BB04, stmt 1 (top level)
6599 // [000051] ------------ * stmtExpr void (top level) (IL 0x00A... ? ? ? )
6600 // [000030] ------------ | / --* const int - 1
6601 // [000031] ------------ | / --* +int
6602 // [000029] ------------ | | \--* lclVar int V00 arg1
6603 // [000050] - A---------- \--* = int
6604 // [000049] D------N---- \--* lclVar int V02 tmp0
6606 // ***** BB04, stmt 2 (top level)
6607 // [000055] ------------ * stmtExpr void (top level) (IL 0x00A... ? ? ? )
6608 // [000052] ------------ | / --* lclVar int V02 tmp0
6609 // [000054] - A---------- \--* = int
6610 // [000053] D------N---- \--* lclVar int V00 arg0
6612 // ***** BB04, stmt 3 (top level)
6613 // [000058] ------------ * stmtExpr void (top level) (IL 0x00A... ? ? ? )
6614 // [000032] ------------ | / --* const int 1
6615 // [000057] - A---------- \--* = int
6616 // [000056] D------N---- \--* lclVar int V01 arg1
6618 GenTreePtr tmpAssignmentInsertionPoint = last;
6619 GenTreePtr paramAssignmentInsertionPoint = last;
6621 // Process early args. They may contain both setup statements for late args and actual args.
6622 // Early args don't include 'this' arg. We need to account for that so that the call to gtArgEntryByArgNum
6623 // below has the correct second argument.
6624 int earlyArgIndex = (thisArg == nullptr) ? 0 : 1;
6625 for (GenTreeArgList* earlyArgs = recursiveTailCall->gtCallArgs;
6626 earlyArgs != nullptr;
6627 (earlyArgIndex++, earlyArgs = earlyArgs->Rest()))
6629 GenTreePtr earlyArg = earlyArgs->Current();
6630 if (!earlyArg->IsNothingNode() && !earlyArg->IsArgPlaceHolderNode())
6632 if ((earlyArg->gtFlags & GTF_LATE_ARG) != 0)
6634 // This is a setup node so we need to hoist it.
6635 GenTreePtr earlyArgStmt = gtNewStmt(earlyArg, callILOffset);
6636 fgInsertStmtBefore(block, earlyArgInsertionPoint, earlyArgStmt);
6640 // This is an actual argument that needs to be assigned to the corresponding caller parameter.
6641 fgArgTabEntryPtr curArgTabEntry = gtArgEntryByArgNum(recursiveTailCall, earlyArgIndex);
6642 GenTreePtr paramAssignStmt = fgAssignRecursiveCallArgToCallerParam(earlyArg, curArgTabEntry, block, callILOffset,
6643 tmpAssignmentInsertionPoint, paramAssignmentInsertionPoint);
6644 if ((tmpAssignmentInsertionPoint == last) && (paramAssignStmt != nullptr))
6646 // All temp assignments will happen before the first param assignment.
6647 tmpAssignmentInsertionPoint = paramAssignStmt;
6653 // Process late args.
6654 int lateArgIndex = 0;
6655 for (GenTreeArgList* lateArgs = recursiveTailCall->gtCallLateArgs;
6656 lateArgs != nullptr;
6657 (lateArgIndex++, lateArgs = lateArgs->Rest()))
6659 // A late argument is an actual argument that needs to be assigned to the corresponding caller's parameter.
6660 GenTreePtr lateArg = lateArgs->Current();
6661 fgArgTabEntryPtr curArgTabEntry = gtArgEntryByLateArgIndex(recursiveTailCall, lateArgIndex);
6662 GenTreePtr paramAssignStmt = fgAssignRecursiveCallArgToCallerParam(lateArg, curArgTabEntry, block, callILOffset,
6663 tmpAssignmentInsertionPoint, paramAssignmentInsertionPoint);
6665 if ((tmpAssignmentInsertionPoint == last) && (paramAssignStmt != nullptr))
6667 // All temp assignments will happen before the first param assignment.
6668 tmpAssignmentInsertionPoint = paramAssignStmt;
6672 // If the method has starg.s 0 or ldarga.s 0 a special local (lvaArg0Var) is created so that
6673 // compThisArg stays immutable. Normally it's assigned in fgFirstBBScratch block. Since that
6674 // block won't be in the loop (it's assumed to have no predecessors), we need to update the special local here.
6675 if (!info.compIsStatic && (lvaArg0Var != info.compThisArg))
6677 var_types thisType = lvaTable[info.compThisArg].TypeGet();
6678 GenTreePtr arg0 = gtNewLclvNode(lvaArg0Var, thisType);
6679 GenTreePtr arg0Assignment = gtNewAssignNode(arg0, gtNewLclvNode(info.compThisArg, thisType));
6680 GenTreePtr arg0AssignmentStmt = gtNewStmt(arg0Assignment, callILOffset);
6681 fgInsertStmtBefore(block, paramAssignmentInsertionPoint, arg0AssignmentStmt);
6685 fgRemoveStmt(block, last);
6687 // Set the loop edge.
6688 block->bbJumpKind = BBJ_ALWAYS;
6689 block->bbJumpDest = fgFirstBBisScratch() ? fgFirstBB->bbNext : fgFirstBB;
6690 fgAddRefPred(block->bbJumpDest, block);
6691 block->bbFlags &= ~BBF_HAS_JMP;
6694 //------------------------------------------------------------------------------
6695 // fgAssignRecursiveCallArgToCallerParam : Assign argument to a recursive call to the corresponding caller parameter.
6699 // arg - argument to assign
6700 // argTabEntry - argument table entry corresponding to arg
6701 // block --- basic block the call is in
6702 // callILOffset - IL offset of the call
6703 // tmpAssignmentInsertionPoint - tree before which temp assignment should be inserted (if necessary)
6704 // paramAssignmentInsertionPoint - tree before which parameter assignment should be inserted
6707 // parameter assignment statement if one was inserted; nullptr otherwise.
6709 GenTreePtr Compiler::fgAssignRecursiveCallArgToCallerParam(GenTreePtr arg,
6710 fgArgTabEntryPtr argTabEntry,
6712 IL_OFFSETX callILOffset,
6713 GenTreePtr tmpAssignmentInsertionPoint,
6714 GenTreePtr paramAssignmentInsertionPoint)
6716 // Call arguments should be assigned to temps first and then the temps should be assigned to parameters because
6717 // some argument trees may reference parameters directly.
6719 GenTreePtr argInTemp = nullptr;
6720 unsigned originalArgNum = argTabEntry->argNum;
6721 bool needToAssignParameter = true;
6723 // TODO-CQ: enable calls with struct arguments passed in registers.
6724 noway_assert(!varTypeIsStruct(arg->TypeGet()));
6726 if ((argTabEntry->isTmp) || arg->IsCnsIntOrI() || arg->IsCnsFltOrDbl())
6728 // The argument is already assigned to a temp or is a const.
6731 else if (arg->OperGet() == GT_LCL_VAR)
6733 unsigned lclNum = arg->AsLclVar()->gtLclNum;
6734 LclVarDsc * varDsc = &lvaTable[lclNum];
6735 if (!varDsc->lvIsParam)
6737 // The argument is a non-parameter local so it doesn't need to be assigned to a temp.
6740 else if (lclNum == originalArgNum)
6742 // The argument is the same parameter local that we were about to assign so
6743 // we can skip the assignment.
6744 needToAssignParameter = false;
6748 // TODO: We don't need temp assignments if we can prove that the argument tree doesn't involve
6749 // any caller parameters. Some common cases are handled above but we may be able to eliminate
6750 // more temp assignments.
6752 GenTreePtr paramAssignStmt = nullptr;
6753 if (needToAssignParameter)
6755 if (argInTemp == nullptr)
6757 // The argument is not assigned to a temp. We need to create a new temp and insert an assignment.
6758 // TODO: we can avoid a temp assignment if we can prove that the argument tree
6759 // doesn't involve any caller parameters.
6760 unsigned tmpNum = lvaGrabTemp(true DEBUGARG("arg temp"));
6761 GenTreePtr tempSrc = arg;
6762 GenTreePtr tempDest = gtNewLclvNode(tmpNum, tempSrc->gtType);
6763 GenTreePtr tmpAssignNode = gtNewAssignNode(tempDest, tempSrc);
6764 GenTreePtr tmpAssignStmt = gtNewStmt(tmpAssignNode, callILOffset);
6765 fgInsertStmtBefore(block, tmpAssignmentInsertionPoint, tmpAssignStmt);
6766 argInTemp = gtNewLclvNode(tmpNum, tempSrc->gtType);
6769 // Now assign the temp to the parameter.
6770 LclVarDsc *paramDsc = lvaTable + originalArgNum;
6771 assert(paramDsc->lvIsParam);
6772 GenTreePtr paramDest = gtNewLclvNode(originalArgNum, paramDsc->lvType);
6773 GenTreePtr paramAssignNode = gtNewAssignNode(paramDest, argInTemp);
6774 paramAssignStmt = gtNewStmt(paramAssignNode, callILOffset);
6776 fgInsertStmtBefore(block, paramAssignmentInsertionPoint, paramAssignStmt);
6778 return paramAssignStmt;
6781 /*****************************************************************************
6783 * Transform the given GT_CALL tree for code generation.
6786 GenTreePtr Compiler::fgMorphCall(GenTreeCall* call)
6788 if (call->CanTailCall())
6790 // It should either be an explicit (i.e. tail prefixed) or an implicit tail call
6791 assert(call->IsTailPrefixedCall() ^ call->IsImplicitTailCall());
6793 // It cannot be an inline candidate
6794 assert(!call->IsInlineCandidate());
6796 const char * szFailReason = nullptr;
6797 bool hasStructParam = false;
6798 if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC)
6800 szFailReason = "Might turn into an intrinsic";
6803 if (opts.compNeedSecurityCheck)
6805 szFailReason = "Needs security check";
6807 else if (compLocallocUsed)
6809 szFailReason = "Localloc used";
6811 #ifdef _TARGET_AMD64_
6812 // Needed for Jit64 compat.
6813 // In future, enabling tail calls from methods that need GS cookie check
6814 // would require codegen side work to emit GS cookie check before a tail
6816 else if (getNeedsGSSecurityCookie())
6818 szFailReason = "GS Security cookie check";
6822 // DDB 99324: Just disable tailcall under compGcChecks stress mode.
6823 else if (opts.compGcChecks)
6825 szFailReason = "GcChecks";
6828 #if FEATURE_TAILCALL_OPT
6831 // We are still not sure whether it can be a tail call. Because, when converting
6832 // a call to an implicit tail call, we must check that there are no locals with
6833 // their address taken. If this is the case, we have to assume that the address
6834 // has been leaked and the current stack frame must live until after the final
6837 // Verify that none of vars has lvHasLdAddrOp or lvAddrExposed bit set. Note
6838 // that lvHasLdAddrOp is much more conservative. We cannot just base it on
6839 // lvAddrExposed alone since it is not guaranteed to be set on all VarDscs
6840 // during morph stage. The reason for also checking lvAddrExposed is that in case
6841 // of vararg methods user args are marked as addr exposed but not lvHasLdAddrOp.
6842 // The combination of lvHasLdAddrOp and lvAddrExposed though conservative allows us
6843 // never to be incorrect.
6845 // TODO-Throughput: have a compiler level flag to indicate whether method has vars whose
6846 // address is taken. Such a flag could be set whenever lvHasLdAddrOp or LvAddrExposed
6847 // is set. This avoids the need for iterating through all lcl vars of the current
6848 // method. Right now throughout the code base we are not consistently using 'set'
6849 // method to set lvHasLdAddrOp and lvAddrExposed flags.
6852 bool hasAddrExposedVars = false;
6853 bool hasStructPromotedParam = false;
6854 bool hasPinnedVars = false;
6856 for (varNum = 0, varDsc = lvaTable; varNum < lvaCount; varNum++, varDsc++)
6858 // If the method is marked as an explicit tail call we will skip the
6859 // following three hazard checks.
6860 // We still must check for any struct parameters and set 'hasStructParam'
6861 // so that we won't transform the recursive tail call into a loop.
6863 if (call->IsImplicitTailCall())
6865 if (varDsc->lvHasLdAddrOp || varDsc->lvAddrExposed)
6867 hasAddrExposedVars = true;
6870 if (varDsc->lvPromoted && varDsc->lvIsParam)
6872 hasStructPromotedParam = true;
6875 if (varDsc->lvPinned)
6877 // A tail call removes the method from the stack, which means the pinning
6878 // goes away for the callee. We can't allow that.
6879 hasPinnedVars = true;
6883 if (varTypeIsStruct(varDsc->TypeGet()) && varDsc->lvIsParam)
6885 hasStructParam = true;
6886 // This prevents transforming a recursive tail call into a loop
6887 // but doesn't prevent tail call optimization so we need to
6888 // look at the rest of parameters.
6893 if (hasAddrExposedVars)
6895 szFailReason = "Local address taken";
6897 if (hasStructPromotedParam)
6899 szFailReason = "Has Struct Promoted Param";
6903 szFailReason = "Has Pinned Vars";
6906 #endif // FEATURE_TAILCALL_OPT
6908 fgFixupStructReturn(call);
6910 var_types callType = call->TypeGet();
6912 // We have to ensure to pass the incoming retValBuf as the
6913 // outgoing one. Using a temp will not do as this function will
6914 // not regain control to do the copy.
6916 if (info.compRetBuffArg != BAD_VAR_NUM)
6918 noway_assert(callType == TYP_VOID);
6919 GenTreePtr retValBuf = call->gtCallArgs->gtOp.gtOp1;
6920 if (retValBuf->gtOper != GT_LCL_VAR ||
6921 retValBuf->gtLclVarCommon.gtLclNum != info.compRetBuffArg)
6923 szFailReason = "Need to copy return buffer";
6927 // If this is an opportunistic tail call and cannot be dispatched as
6928 // fast tail call, go the non-tail call route. This is done for perf
6931 // Avoid the cost of determining whether can be dispatched as fast tail
6932 // call if we already know that tail call cannot be honored for other
6934 bool canFastTailCall = false;
6935 if (szFailReason == nullptr)
6937 canFastTailCall = fgCanFastTailCall(call);
6938 if (!canFastTailCall)
6940 // Implicit or opportunistic tail calls are always dispatched via fast tail call
6941 // mechanism and never via tail call helper for perf.
6942 if (call->IsImplicitTailCall())
6944 szFailReason = "Opportunistic tail call cannot be dispatched as epilog+jmp";
6946 #ifndef LEGACY_BACKEND
6947 else if (!call->IsVirtualStub() && call->HasNonStandardArgs())
6949 // If we are here, it means that the call is an explicitly ".tail" prefixed and cannot be
6950 // dispatched as a fast tail call.
6952 // Methods with non-standard args will have indirection cell or cookie param passed
6953 // in callee trash register (e.g. R11). Tail call helper doesn't preserve it before
6954 // tail calling the target method and hence ".tail" prefix on such calls needs to be
6957 // Exception to the above rule: although Virtual Stub Dispatch (VSD) calls though require
6958 // extra stub param (e.g. in R11 on Amd64), they can still be called via tail call helper.
6959 // This is done by by adding stubAddr as an additional arg before the original list of
6960 // args. For more details see fgMorphTailCall() and CreateTailCallCopyArgsThunk()
6961 // in Stublinkerx86.cpp.
6962 szFailReason = "Method with non-standard args passed in callee trash register cannot be tail called via helper";
6964 #ifdef _TARGET_ARM64_
6967 // NYI - TAILCALL_RECURSIVE/TAILCALL_HELPER.
6968 // So, bail out if we can't make fast tail call.
6969 szFailReason = "Non-qualified fast tail call";
6972 #endif //LEGACY_BACKEND
6976 // Clear these flags before calling fgMorphCall() to avoid recursion.
6977 bool isTailPrefixed = call->IsTailPrefixedCall();
6978 call->gtCallMoreFlags &= ~GTF_CALL_M_EXPLICIT_TAILCALL;
6980 #if FEATURE_TAILCALL_OPT
6981 call->gtCallMoreFlags &= ~GTF_CALL_M_IMPLICIT_TAILCALL;
6985 if (!canFastTailCall && szFailReason == nullptr)
6987 szFailReason = "Non fast tail calls disabled for PAL based systems.";
6989 #endif // FEATURE_PAL
6991 if (szFailReason != nullptr)
6995 printf("\nRejecting tail call late for call ");
6997 printf(": %s\n", szFailReason);
7001 // for non user funcs, we have no handles to report
7002 info.compCompHnd->reportTailCallDecision(nullptr,
7003 (call->gtCallType == CT_USER_FUNC) ? call->gtCallMethHnd : nullptr,
7004 isTailPrefixed, TAILCALL_FAIL, szFailReason);
7009 #if !FEATURE_TAILCALL_OPT_SHARED_RETURN
7010 // We enable shared-ret tail call optimization for recursive calls even if
7011 // FEATURE_TAILCALL_OPT_SHARED_RETURN is not defined.
7012 if (gtIsRecursiveCall(call))
7015 // Many tailcalls will have call and ret in the same block, and thus be BBJ_RETURN,
7016 // but if the call falls through to a ret, and we are doing a tailcall, change it here.
7017 if (compCurBB->bbJumpKind != BBJ_RETURN)
7018 compCurBB->bbJumpKind = BBJ_RETURN;
7022 // Set this flag before calling fgMorphCall() to prevent inlining this call.
7023 call->gtCallMoreFlags |= GTF_CALL_M_TAILCALL;
7025 bool fastTailCallToLoop = false;
7026 #if FEATURE_TAILCALL_OPT
7027 // TODO-CQ: enable the transformation when the method has a struct parameter that can be passed in a register
7028 // or return type is a struct that can be passed in a register.
7030 // TODO-CQ: if the method being compiled requires generic context reported in gc-info (either through
7031 // hidden generic context param or through keep alive thisptr), then while transforming a recursive
7032 // call to such a method requires that the generic context stored on stack slot be updated. Right now,
7033 // fgMorphRecursiveFastTailCallIntoLoop() is not handling update of generic context while transforming
7034 // a recursive call into a loop. Another option is to modify gtIsRecursiveCall() to check that the
7035 // generic type parameters of both caller and callee generic method are the same.
7036 if (opts.compTailCallLoopOpt &&
7038 gtIsRecursiveCall(call) &&
7039 !lvaReportParamTypeArg() &&
7040 !lvaKeepAliveAndReportThis() &&
7041 !call->IsVirtual() &&
7043 !varTypeIsStruct(call->TypeGet()))
7045 call->gtCallMoreFlags |= GTF_CALL_M_TAILCALL_TO_LOOP;
7046 fastTailCallToLoop = true;
7050 // Do some target-specific transformations (before we process the args, etc.)
7051 // This is needed only for tail prefixed calls that cannot be dispatched as
7053 if (!canFastTailCall)
7055 fgMorphTailCall(call);
7058 // Implementation note : If we optimize tailcall to do a direct jump
7059 // to the target function (after stomping on the return address, etc),
7060 // without using CORINFO_HELP_TAILCALL, we have to make certain that
7061 // we don't starve the hijacking logic (by stomping on the hijacked
7062 // return address etc).
7064 // At this point, we are committed to do the tailcall.
7065 compTailCallUsed = true;
7067 CorInfoTailCall tailCallResult;
7069 if (fastTailCallToLoop)
7071 tailCallResult = TAILCALL_RECURSIVE;
7073 else if (canFastTailCall)
7075 tailCallResult = TAILCALL_OPTIMIZED;
7079 tailCallResult = TAILCALL_HELPER;
7082 // for non user funcs, we have no handles to report
7083 info.compCompHnd->reportTailCallDecision(nullptr,
7084 (call->gtCallType == CT_USER_FUNC) ? call->gtCallMethHnd : nullptr,
7089 // As we will actually call CORINFO_HELP_TAILCALL, set the callTyp to TYP_VOID.
7090 // to avoid doing any extra work for the return value.
7091 call->gtType = TYP_VOID;
7096 printf("\nGTF_CALL_M_TAILCALL bit set for call ");
7099 if (fastTailCallToLoop)
7101 printf("\nGTF_CALL_M_TAILCALL_TO_LOOP bit set for call ");
7108 GenTreePtr stmtExpr = fgMorphStmt->gtStmt.gtStmtExpr;
7109 bool deleteReturn = false;
7110 if (info.compRetBuffArg != BAD_VAR_NUM)
7112 // In this case we simply have a call followed by a return.
7113 noway_assert(fgMorphStmt->gtNext->gtStmt.gtStmtExpr->gtOper == GT_RETURN);
7114 deleteReturn = true;
7116 else if ((stmtExpr->gtOper == GT_ASG) && (fgMorphStmt->gtNext != nullptr))
7118 GenTreePtr nextStmtExpr = fgMorphStmt->gtNext->gtStmt.gtStmtExpr;
7119 noway_assert(nextStmtExpr->gtOper == GT_RETURN);
7120 // In this case we have an assignment of the result of the call, and then a return of the result of the assignment.
7121 // This can occur if impSpillStackEnsure() has introduced an assignment to a temp.
7122 noway_assert(stmtExpr->gtGetOp1()->OperIsLocal() &&
7123 nextStmtExpr->OperGet() == GT_RETURN &&
7124 nextStmtExpr->gtGetOp1() != nullptr &&
7125 nextStmtExpr->gtGetOp1()->OperIsLocal() &&
7126 stmtExpr->gtGetOp1()->AsLclVarCommon()->gtLclNum == nextStmtExpr->gtGetOp1()->AsLclVarCommon()->gtLclNum);
7127 deleteReturn = true;
7131 fgRemoveStmt(compCurBB, fgMorphStmt->gtNext);
7134 // For void calls, we would have created a GT_CALL in the stmt list.
7135 // For non-void calls, we would have created a GT_RETURN(GT_CAST(GT_CALL)).
7136 // For calls returning structs, we would have a void call, followed by a void return.
7137 // For debuggable code, it would be an assignment of the call to a temp
7138 // We want to get rid of any of this extra trees, and just leave
7141 bool tailCallFollowedByPopAndRet = false;
7145 noway_assert((stmtExpr->gtOper == GT_CALL && stmtExpr == call) || // Either a call stmt
7146 (stmtExpr->gtOper == GT_RETURN && (stmtExpr->gtOp.gtOp1 == call || stmtExpr->gtOp.gtOp1->gtOp.gtOp1 == call)) || // GT_RETURN(GT_CALL(..))
7147 (stmtExpr->gtOper == GT_ASG && stmtExpr->gtOp.gtOp2 == call)); // or var = call
7150 #ifdef _TARGET_AMD64_
7151 if ((stmtExpr->gtOper == GT_CALL) && (fgMorphStmt->gtNext != nullptr))
7153 // We have a stmt node after a tail call node. This must be a tail call occuring
7154 // in the following IL pattern
7158 // Since tail prefix is honored, we can get rid of the remaining two stmts
7159 // corresponding to pop and ret. Note that 'pop' may or may not result in
7160 // a new statement (see impImportBlockCode() for details).
7161 stmt = fgMorphStmt->gtNext;
7162 if (stmt->gtNext != nullptr)
7164 // We have a pop tree.
7165 // It must be side effect free.
7166 GenTreePtr ret = stmt->gtNext;
7167 noway_assert((stmt->gtStmt.gtStmtExpr->gtFlags & GTF_ALL_EFFECT) == 0);
7168 fgRemoveStmt(compCurBB, stmt);
7171 noway_assert(stmt->gtStmt.gtStmtExpr->gtOper == GT_RETURN);
7172 fgRemoveStmt(compCurBB, stmt);
7174 tailCallFollowedByPopAndRet = true;
7176 #else //!TARGET_AMD64_
7179 noway_assert(fgMorphStmt->gtNext == nullptr);
7182 #endif //!_TARGET_AMD64_
7184 fgMorphStmt->gtStmt.gtStmtExpr = call;
7186 // Tail call via helper: The VM can't use return address hijacking if we're
7187 // not going to return and the helper doesn't have enough info to safely poll,
7188 // so we poll before the tail call, if the block isn't already safe. Since
7189 // tail call via helper is a slow mechanism it doen't matter whether we emit
7190 // GC poll. This is done to be in parity with Jit64. Also this avoids GC info
7191 // size increase if all most all methods are expected to be tail calls (e.g. F#).
7193 // Note that we can avoid emitting GC-poll if we know that the current BB is
7194 // dominated by a Gc-SafePoint block. But we don't have dominator info at this
7195 // point. One option is to just add a place holder node for GC-poll (e.g. GT_GCPOLL)
7196 // here and remove it in lowering if the block is dominated by a GC-SafePoint. For
7197 // now it not clear whether optimizing slow tail calls is worth the effort. As a
7198 // low cost check, we check whether the first and current basic blocks are
7201 // Fast Tail call as epilog+jmp - No need to insert GC-poll. Instead, fgSetBlockOrder()
7202 // is going to mark the method as fully interruptible if the block containing this tail
7203 // call is reachable without executing any call.
7204 if (canFastTailCall ||
7205 (fgFirstBB->bbFlags & BBF_GC_SAFE_POINT) ||
7206 (compCurBB->bbFlags & BBF_GC_SAFE_POINT) ||
7207 !fgCreateGCPoll(GCPOLL_INLINE, compCurBB))
7209 // We didn't insert a poll block, so we need to morph the call now
7210 // (Normally it will get morphed when we get to the split poll block)
7211 GenTreePtr temp = fgMorphCall(call);
7212 noway_assert(temp == call);
7215 // Tail call via helper: we just call CORINFO_HELP_TAILCALL, and it jumps to
7216 // the target. So we don't need an epilog - just like CORINFO_HELP_THROW.
7218 // Fast tail call: in case of fast tail calls, we need a jmp epilog and
7219 // hence mark it as BBJ_RETURN with BBF_JMP flag set.
7220 noway_assert(compCurBB->bbJumpKind == BBJ_RETURN);
7222 if (canFastTailCall)
7224 compCurBB->bbFlags |= BBF_HAS_JMP;
7228 compCurBB->bbJumpKind = BBJ_THROW;
7231 // For non-void calls, we return a place holder which will be
7232 // used by the parent GT_RETURN node of this call. This should
7233 // not be done for tail calls occuring in the following IL pattern,
7234 // since this pattern is supported only in void returning methods.
7239 GenTree* result = call;
7241 if (!tailCallFollowedByPopAndRet && (callType != TYP_VOID) && info.compRetType != TYP_VOID)
7244 // Return a dummy node, as the return is already removed.
7245 if (callType == TYP_STRUCT)
7247 // This is a HFA, use float 0.
7248 callType = TYP_FLOAT;
7250 #elif defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
7251 // Return a dummy node, as the return is already removed.
7252 if (varTypeIsStruct(callType))
7254 // This is a register-returned struct. Return a 0.
7255 // The actual return registers are hacked in lower and the register allocator.
7260 // Return a dummy node, as the return is already removed.
7261 if (varTypeIsSIMD(callType))
7263 callType = TYP_DOUBLE;
7266 result = gtNewZeroConNode(genActualType(callType));
7267 result = fgMorphTree(result);
7275 if ((call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) == 0 &&
7276 (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_VIRTUAL_FUNC_PTR)
7277 #ifdef FEATURE_READYTORUN_COMPILER
7278 || call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_READYTORUN_VIRTUAL_FUNC_PTR)
7281 (call == fgMorphStmt->gtStmt.gtStmtExpr))
7283 // This is call to CORINFO_HELP_VIRTUAL_FUNC_PTR with ignored result.
7284 // Transform it into a null check.
7286 GenTreePtr thisPtr = call->gtCallArgs->gtOp.gtOp1;
7288 GenTreePtr nullCheck = gtNewOperNode(GT_IND, TYP_I_IMPL, thisPtr);
7289 nullCheck->gtFlags |= GTF_EXCEPT;
7291 return fgMorphTree(nullCheck);
7294 noway_assert(call->gtOper == GT_CALL);
7297 // Only count calls once (only in the global morph phase)
7301 if (call->gtCallType == CT_INDIRECT)
7304 optIndirectCallCount++;
7306 else if (call->gtCallType == CT_USER_FUNC)
7309 if (call->IsVirtual())
7310 optIndirectCallCount++;
7314 // Couldn't inline - remember that this BB contains method calls
7316 // If this is a 'regular' call, mark the basic block as
7317 // having a call (for computing full interruptibility).
7319 // Amd64 note: If this is a fast tail call then don't count it as a call
7320 // since we don't insert GC-polls but instead make the method fully GC
7322 #ifdef _TARGET_AMD64_
7323 if (!call->IsFastTailCall())
7326 if (call->gtCallType == CT_INDIRECT)
7328 compCurBB->bbFlags |= BBF_GC_SAFE_POINT;
7330 else if (call->gtCallType == CT_USER_FUNC)
7332 if ((call->gtCallMoreFlags & GTF_CALL_M_NOGCCHECK) == 0)
7333 compCurBB->bbFlags |= BBF_GC_SAFE_POINT;
7335 // otherwise we have a CT_HELPER
7338 // Morph Type.op_Equality and Type.op_Inequality
7339 // We need to do this before the arguments are morphed
7340 if ((call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC))
7342 CorInfoIntrinsics methodID = info.compCompHnd->getIntrinsicID(call->gtCallMethHnd);
7344 genTreeOps simpleOp = GT_CALL;
7345 if (methodID == CORINFO_INTRINSIC_TypeEQ)
7347 else if (methodID == CORINFO_INTRINSIC_TypeNEQ)
7350 if (simpleOp == GT_EQ || simpleOp == GT_NE)
7352 noway_assert(call->TypeGet() == TYP_INT);
7354 // Check for GetClassFromHandle(handle) and obj.GetType() both of which will only return RuntimeType objects.
7355 // Then if either operand is one of these two calls we can simplify op_Equality/op_Inequality to GT_NE/GT_NE:
7356 // One important invariance that should never change is that type equivalency is always equivalent to object
7357 // identity equality for runtime type objects in reflection. This is also reflected in RuntimeTypeHandle::TypeEquals.
7358 // If this invariance would ever be broken, we need to remove the optimization below.
7360 GenTreePtr op1 = call->gtCallArgs->gtOp.gtOp1;
7361 GenTreePtr op2 = call->gtCallArgs->gtOp.gtOp2->gtOp.gtOp1;
7363 if (gtCanOptimizeTypeEquality(op1) || gtCanOptimizeTypeEquality(op2))
7365 GenTreePtr compare = gtNewOperNode(simpleOp, TYP_INT, op1, op2);
7367 // fgMorphSmpOp will further optimize the following patterns:
7368 // 1. typeof(...) == typeof(...)
7369 // 2. typeof(...) == obj.GetType()
7370 return fgMorphTree(compare);
7375 // Make sure that return buffers containing GC pointers that aren't too large are pointers into the stack.
7376 GenTreePtr origDest = nullptr; // Will only become non-null if we do the transformation (and thus require copy-back).
7377 unsigned retValTmpNum = BAD_VAR_NUM;
7378 CORINFO_CLASS_HANDLE structHnd = nullptr;
7379 if (call->HasRetBufArg() &&
7380 call->gtCallLateArgs == nullptr) // Don't do this if we're re-morphing (which will make late args non-null).
7382 // We're enforcing the invariant that return buffers pointers (at least for
7383 // struct return types containing GC pointers) are never pointers into the heap.
7384 // The large majority of cases are address of local variables, which are OK.
7385 // Otherwise, allocate a local of the given struct type, pass its address,
7386 // then assign from that into the proper destination. (We don't need to do this
7387 // if we're passing the caller's ret buff arg to the callee, since the caller's caller
7388 // will maintain the same invariant.)
7390 GenTreePtr dest = call->gtCallArgs->gtOp.gtOp1;
7391 assert(dest->OperGet() != GT_ARGPLACE); // If it was, we'd be in a remorph, which we've already excluded above.
7392 if (dest->gtType == TYP_BYREF
7393 && !(dest->OperGet() == GT_ADDR && dest->gtOp.gtOp1->OperGet() == GT_LCL_VAR))
7395 // We'll exempt helper calls from this, assuming that the helper implementation
7396 // follows the old convention, and does whatever barrier is required.
7397 if (call->gtCallType != CT_HELPER)
7399 structHnd = call->gtRetClsHnd;
7400 if (info.compCompHnd->isStructRequiringStackAllocRetBuf(structHnd)
7401 && !((dest->OperGet() == GT_LCL_VAR || dest->OperGet() == GT_REG_VAR)
7402 && dest->gtLclVar.gtLclNum == info.compRetBuffArg))
7406 retValTmpNum = lvaGrabTemp(true DEBUGARG("substitute local for ret buff arg"));
7407 lvaSetStruct(retValTmpNum, structHnd, true);
7408 dest = gtNewOperNode(GT_ADDR, TYP_BYREF, gtNewLclvNode(retValTmpNum, TYP_STRUCT));
7413 call->gtCallArgs->gtOp.gtOp1 = dest;
7416 /* Process the "normal" argument list */
7417 call = fgMorphArgs(call);
7419 // Optimize get_ManagedThreadId(get_CurrentThread)
7420 noway_assert(call->gtOper == GT_CALL);
7422 // Morph stelem.ref helper call to store a null value, into a store into an array without the helper.
7423 // This needs to be done after the arguments are morphed to ensure constant propagation has already taken place.
7424 if ((call->gtCallType == CT_HELPER) && (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_ARRADDR_ST)))
7426 GenTreePtr value = gtArgEntryByArgNum(call, 2)->node;
7428 if (value->OperGet() == GT_CNS_INT && value->AsIntConCommon()->IconValue() == 0)
7430 GenTreePtr arr = gtArgEntryByArgNum(call, 0)->node;
7431 GenTreePtr index = gtArgEntryByArgNum(call, 1)->node;
7433 arr = gtClone(arr, true);
7436 index = gtClone(index, true);
7437 if (index != nullptr)
7439 value = gtClone(value);
7440 noway_assert(value != nullptr);
7442 GenTreePtr nullCheckedArr = impCheckForNullPointer(arr);
7443 GenTreePtr arrIndexNode = gtNewIndexRef(TYP_REF, nullCheckedArr, index);
7444 GenTreePtr arrStore = gtNewAssignNode(arrIndexNode, value);
7445 arrStore->gtFlags |= GTF_ASG;
7447 return fgMorphTree(arrStore);
7453 if ((call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) &&
7454 info.compCompHnd->getIntrinsicID(call->gtCallMethHnd) == CORINFO_INTRINSIC_GetManagedThreadId)
7456 noway_assert(origDest == NULL);
7457 noway_assert(call->gtCallLateArgs->gtOp.gtOp1 != NULL);
7459 GenTreePtr innerCall = call->gtCallLateArgs->gtOp.gtOp1;
7461 if (innerCall->gtOper == GT_CALL &&
7462 (innerCall->gtCall.gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) &&
7463 info.compCompHnd->getIntrinsicID(innerCall->gtCall.gtCallMethHnd) == CORINFO_INTRINSIC_GetCurrentManagedThread)
7465 // substitute expression with call to helper
7466 GenTreePtr newCall = gtNewHelperCallNode(CORINFO_HELP_GETCURRENTMANAGEDTHREADID, TYP_INT, 0);
7467 JITDUMP("get_ManagedThreadId(get_CurrentThread) folding performed\n");
7468 return fgMorphTree(newCall);
7472 if (origDest != NULL)
7474 GenTreePtr retValVarAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, gtNewLclvNode(retValTmpNum, TYP_STRUCT));
7475 // If the origDest expression was an assignment to a variable, it might be to an otherwise-unused
7476 // var, which would allow the whole assignment to be optimized away to a NOP. So in that case, make the
7477 // origDest into a comma that uses the var. Note that the var doesn't have to be a temp for this to
7479 if (origDest->OperGet() == GT_ASG)
7481 if (origDest->gtOp.gtOp1->OperGet() == GT_LCL_VAR)
7483 GenTreePtr var = origDest->gtOp.gtOp1;
7484 origDest = gtNewOperNode(GT_COMMA, var->TypeGet(), origDest,
7485 gtNewLclvNode(var->gtLclVar.gtLclNum, var->TypeGet()));
7488 GenTreePtr copyBlk = gtNewCpObjNode(origDest, retValVarAddr, structHnd, false);
7489 copyBlk = fgMorphTree(copyBlk);
7490 GenTree* result = gtNewOperNode(GT_COMMA, TYP_VOID, call, copyBlk);
7492 result->gtFlags |= GTF_MORPHED;
7501 /*****************************************************************************
7503 * Transform the given GTK_CONST tree for code generation.
7506 GenTreePtr Compiler::fgMorphConst(GenTreePtr tree)
7508 noway_assert(tree->OperKind() & GTK_CONST);
7510 /* Clear any exception flags or other unnecessary flags
7511 * that may have been set before folding this node to a constant */
7513 tree->gtFlags &= ~(GTF_ALL_EFFECT | GTF_REVERSE_OPS);
7515 if (tree->OperGet() != GT_CNS_STR)
7518 // TODO-CQ: Do this for compCurBB->isRunRarely(). Doing that currently will
7519 // guarantee slow performance for that block. Instead cache the return value
7520 // of CORINFO_HELP_STRCNS and go to cache first giving reasonable perf.
7522 if (compCurBB->bbJumpKind == BBJ_THROW)
7524 CorInfoHelpFunc helper = info.compCompHnd->getLazyStringLiteralHelper(tree->gtStrCon.gtScpHnd);
7525 if (helper != CORINFO_HELP_UNDEF)
7527 // For un-important blocks, we want to construct the string lazily
7529 GenTreeArgList *args;
7530 if (helper == CORINFO_HELP_STRCNS_CURRENT_MODULE)
7532 args = gtNewArgList(gtNewIconNode(RidFromToken(tree->gtStrCon.gtSconCPX), TYP_INT));
7536 args = gtNewArgList(gtNewIconNode(RidFromToken(tree->gtStrCon.gtSconCPX), TYP_INT),
7537 gtNewIconEmbScpHndNode(tree->gtStrCon.gtScpHnd));
7541 tree = gtNewHelperCallNode(helper, TYP_REF, 0, args);
7542 return fgMorphTree(tree);
7546 assert(tree->gtStrCon.gtScpHnd == info.compScopeHnd || !IsUninitialized(tree->gtStrCon.gtScpHnd));
7549 InfoAccessType iat = info.compCompHnd->constructStringLiteral(tree->gtStrCon.gtScpHnd,
7550 tree->gtStrCon.gtSconCPX,
7553 tree = gtNewStringLiteralNode(iat, pValue);
7555 return fgMorphTree(tree);
7558 /*****************************************************************************
7560 * Transform the given GTK_LEAF tree for code generation.
7563 GenTreePtr Compiler::fgMorphLeaf(GenTreePtr tree)
7565 noway_assert(tree->OperKind() & GTK_LEAF);
7567 if (tree->gtOper == GT_LCL_VAR)
7569 return fgMorphLocalVar(tree);
7572 else if (tree->gtOper == GT_LCL_FLD)
7574 if (info.compIsVarArgs)
7576 GenTreePtr newTree = fgMorphStackArgForVarArgs(tree->gtLclFld.gtLclNum, tree->gtType, tree->gtLclFld.gtLclOffs);
7577 if (newTree != NULL)
7581 #endif // _TARGET_X86_
7582 else if (tree->gtOper == GT_FTN_ADDR)
7584 CORINFO_CONST_LOOKUP addrInfo;
7586 #ifdef FEATURE_READYTORUN_COMPILER
7587 if (tree->gtFptrVal.gtEntryPoint.addr != nullptr)
7589 addrInfo = tree->gtFptrVal.gtEntryPoint;
7594 info.compCompHnd->getFunctionFixedEntryPoint(tree->gtFptrVal.gtFptrMethod, &addrInfo);
7597 // Refer to gtNewIconHandleNode() as the template for constructing a constant handle
7599 tree->SetOper(GT_CNS_INT);
7600 tree->gtIntConCommon.SetIconValue(ssize_t(addrInfo.handle));
7601 tree->gtFlags |= GTF_ICON_FTN_ADDR;
7603 switch (addrInfo.accessType)
7606 tree = gtNewOperNode(GT_IND, TYP_I_IMPL, tree);
7607 tree->gtFlags |= GTF_IND_INVARIANT;
7612 tree = gtNewOperNode(GT_IND, TYP_I_IMPL, tree);
7616 tree = gtNewOperNode(GT_NOP, tree->TypeGet(), tree); // prevents constant folding
7620 noway_assert(!"Unknown addrInfo.accessType");
7623 return fgMorphTree(tree);
7630 void Compiler::fgAssignSetVarDef(GenTreePtr tree)
7632 GenTreeLclVarCommon* lclVarCmnTree;
7633 bool isEntire = false;
7634 if (tree->DefinesLocal(this, &lclVarCmnTree, &isEntire))
7638 lclVarCmnTree->gtFlags |= GTF_VAR_DEF;
7642 // We consider partial definitions to be modeled as uses followed by definitions.
7643 // This captures the idea that precedings defs are not necessarily made redundant
7644 // by this definition.
7645 lclVarCmnTree->gtFlags |= (GTF_VAR_DEF | GTF_VAR_USEASG);
7650 GenTreePtr Compiler::fgMorphOneAsgBlockOp(GenTreePtr tree)
7652 genTreeOps oper = tree->gtOper;
7654 // Only xxBlk opcodes are possible
7655 noway_assert(tree->OperIsBlkOp());
7657 GenTreePtr dest = tree->gtOp.gtOp1->gtOp.gtOp1; // Dest address
7658 GenTreePtr src = tree->gtOp.gtOp1->gtOp.gtOp2; // Src
7659 GenTreePtr blkShape = tree->gtOp.gtOp2; // [size/clsHnd]
7660 bool volatil = tree->AsBlkOp()->IsVolatile();
7662 GenTreePtr lclVarTree;
7664 // The dest must be an address
7665 noway_assert(genActualType(dest->gtType) == TYP_I_IMPL ||
7666 dest->gtType == TYP_BYREF);
7668 // For COPYBLK the src must be an address
7669 noway_assert(!tree->OperIsCopyBlkOp() ||
7670 (genActualType( src->gtType) == TYP_I_IMPL ||
7671 src->gtType == TYP_BYREF));
7673 // For INITBLK the src must be a TYP_INT
7674 noway_assert(oper != GT_INITBLK ||
7675 (genActualType( src->gtType) == TYP_INT));
7677 // The size must be an integer type
7678 noway_assert(varTypeIsIntegral(blkShape->gtType));
7680 CORINFO_CLASS_HANDLE clsHnd;
7682 var_types type = TYP_UNDEF;
7684 if (blkShape->gtOper != GT_CNS_INT)
7688 // importer introduces cpblk nodes with src = GT_ADDR(GT_SIMD)
7689 // The SIMD type in question could be Vector2f which is 8-bytes in size.
7690 // The below check is to make sure that we don't turn that copyblk
7691 // into a assignment, since rationalizer logic will transform the
7692 // copyblk apropriately. Otherwise, the transormation made in this
7693 // routine will prevent rationalizer logic and we might end up with
7694 // GT_ADDR(GT_SIMD) node post rationalization, leading to a noway assert
7696 if (src->OperGet() == GT_ADDR && src->gtGetOp1()->OperGet() == GT_SIMD)
7700 if (!blkShape->IsIconHandle())
7703 size = blkShape->gtIntCon.gtIconVal;
7705 /* A four byte BLK_COPY can be treated as an integer asignment */
7708 #ifdef _TARGET_64BIT_
7715 clsHnd = (CORINFO_CLASS_HANDLE) blkShape->gtIntCon.gtIconVal;
7716 size = roundUp(info.compCompHnd->getClassSize(clsHnd), sizeof(void*));
7718 // Since we round up, we are not handling the case where we have a
7719 // non-dword sized struct with GC pointers.
7720 // The EE currently does not allow this, but we may change. Lets assert it
7722 noway_assert(info.compCompHnd->getClassSize(clsHnd) == size);
7724 if (size == REGSIZE_BYTES)
7727 info.compCompHnd->getClassGClayout(clsHnd, &gcPtr);
7728 type = getJitGCType(gcPtr);
7733 // See if we can do a simple transformation:
7735 // GT_ASG <TYP_size>
7737 // GT_IND GT_IND or CNS_INT
7746 goto ONE_SIMPLE_ASG;
7749 goto ONE_SIMPLE_ASG;
7751 #ifdef _TARGET_64BIT_
7754 goto ONE_SIMPLE_ASG;
7755 #endif // _TARGET_64BIT_
7758 noway_assert(type != TYP_UNDEF);
7762 noway_assert(size <= REGSIZE_BYTES);
7764 // For INITBLK, a non constant source is not going to allow us to fiddle
7765 // with the bits to create a single assigment.
7767 if ((oper == GT_INITBLK) && (src->gtOper != GT_CNS_INT))
7772 if (impIsAddressInLocal(dest, &lclVarTree))
7774 #if LOCAL_ASSERTION_PROP
7775 // Kill everything about dest
7776 if (optLocalAssertionProp)
7778 if (optAssertionCount > 0)
7780 fgKillDependentAssertions(lclVarTree->gtLclVarCommon.gtLclNum DEBUGARG(tree));
7783 #endif // LOCAL_ASSERTION_PROP
7785 unsigned lclNum = lclVarTree->gtLclVarCommon.gtLclNum;
7786 // A previous incarnation of this code also required the local not to be
7787 // address-exposed(=taken). That seems orthogonal to the decision of whether
7788 // to do field-wise assignments: being address-exposed will cause it to be
7789 // "dependently" promoted, so it will be in the right memory location. One possible
7790 // further reason for avoiding field-wise stores is that the struct might have alignment-induced
7791 // holes, whose contents could be meaningful in unsafe code. If we decide that's a valid
7792 // concern, then we could compromise, and say that address-exposed + fields do not completely cover the memory
7793 // of the struct prevent field-wise assignments. Same situation exists for the "src" decision.
7794 if (varTypeIsStruct(lclVarTree) &&
7795 (lvaTable[lclNum].lvPromoted || lclVarIsSIMDType(lclNum)))
7798 // Let fgMorphInitBlock handle it. (Since we'll need to do field-var-wise assignments.)
7802 if (!varTypeIsFloating(lclVarTree->TypeGet()) &&
7803 size == genTypeSize(var_types(lvaTable[lclNum].lvType)))
7805 // Use the dest local var directly.
7807 type = lvaTable[lclNum].lvType; // Make the type used in the GT_IND node match
7809 // If the block operation had been a write to a local var of a small int type,
7810 // of the exact size of the small int type, and the var is NormalizeOnStore,
7811 // we would have labeled it GTF_VAR_USEASG, because the block operation wouldn't
7812 // have done that normalization. If we're now making it into an assignment,
7813 // the NormalizeOnStore will work, and it can be a full def.
7814 if (lvaTable[lclNum].lvNormalizeOnStore())
7816 dest->gtFlags &= (~GTF_VAR_USEASG);
7823 // Could be a non-promoted struct, or a floating point type local, or
7824 // an int subject to a partial write. Don't enregister.
7825 lvaSetVarDoNotEnregister(lclNum DEBUGARG(DNER_LocalField));
7826 // Fall through to indirect the dest node.
7828 // Mark the local var tree as a definition point of the local.
7829 lclVarTree->gtFlags |= GTF_VAR_DEF;
7830 if (size < lvaTable[lclNum].lvExactSize) // If it's not a full-width assignment....
7831 lclVarTree->gtFlags |= GTF_VAR_USEASG;
7834 // Check to ensure we are not creating a reducible *(& ... )
7835 if (dest->gtOper == GT_ADDR)
7837 GenTreePtr addrOp = dest->gtOp.gtOp1;
7838 // Ignore reinterpret casts between int/gc
7839 if ((addrOp->TypeGet() == type) ||
7840 (varTypeIsIntegralOrI(addrOp) && (genTypeSize(addrOp->TypeGet()) == size)))
7843 type = addrOp->TypeGet();
7848 /* Indirect the dest node */
7850 dest = gtNewOperNode(GT_IND, type, dest);
7852 /* As long as we don't have more information about the destination we
7853 have to assume it could live anywhere (not just in the GC heap). Mark
7854 the GT_IND node so that we use the correct write barrier helper in case
7855 the field is a GC ref.
7858 dest->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF | GTF_IND_TGTANYWHERE);
7863 dest->gtFlags |= GTF_DONT_CSE;
7865 if (tree->OperIsCopyBlkOp())
7867 if (impIsAddressInLocal(src, &lclVarTree))
7869 unsigned lclNum = lclVarTree->gtLclVarCommon.gtLclNum;
7870 if (varTypeIsStruct(lclVarTree) &&
7871 (lvaTable[lclNum].lvPromoted || lclVarIsSIMDType(lclNum)))
7873 // Let fgMorphCopyBlock handle it.
7877 if (!varTypeIsFloating(lclVarTree->TypeGet()) &&
7878 size == genTypeSize(genActualType(lclVarTree->TypeGet())))
7880 /* Use the src local var directly */
7886 #ifndef LEGACY_BACKEND
7888 // The source argument of the copyblk can potentially
7889 // be accessed only through indir(addr(lclVar))
7890 // or indir(lclVarAddr) in rational form and liveness
7891 // won't account for these uses. That said,
7892 // we have to mark this local as address exposed so
7893 // we don't delete it as a dead store later on.
7894 unsigned lclVarNum = lclVarTree->gtLclVarCommon.gtLclNum;
7895 lvaTable[lclVarNum].lvAddrExposed = true;
7896 lvaSetVarDoNotEnregister(lclVarNum DEBUGARG(DNER_AddrExposed));
7898 #else // LEGACY_BACKEND
7899 lvaSetVarDoNotEnregister(lclVarTree->gtLclVarCommon.gtLclNum DEBUGARG(DNER_LocalField));
7900 #endif // LEGACY_BACKEND
7902 // Fall through to indirect the src node.
7906 /* Indirect the src node */
7908 src = gtNewOperNode(GT_IND, type, src);
7909 src->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF | GTF_IND_TGTANYWHERE);
7914 src->gtFlags |= GTF_DONT_CSE;
7916 else // (oper == GT_INITBLK)
7920 size_t cns = src->gtIntCon.gtIconVal;
7926 #ifdef _TARGET_64BIT_
7931 #endif // _TARGET_64BIT_
7933 src->gtType = type; // Make the type used in the GT_IND node match for TYP_REF
7935 // if we are using an GT_INITBLK on a GC type the value being assigned has to be zero (null)
7936 assert(!varTypeIsGC(type) || (cns == 0));
7939 src->gtIntCon.gtIconVal = cns;
7943 /* Create the assignment node */
7945 result = gtNewAssignNode(dest, src);
7946 result->gtType = type;
7956 //------------------------------------------------------------------------
7957 // fgMorphInitBlock: Perform the Morphing of a GT_INITBLK node
7960 // tree - a tree node with a gtOper of GT_INITBLK
7961 // the child nodes for tree have already been Morphed
7964 // We can return the orginal GT_INITBLK unmodified (least desirable, but always correct)
7965 // We can return a single assignment, when fgMorphOneAsgBlockOp transforms it (most desirable)
7966 // If we have performed struct promotion of the Dest() then we will try to
7967 // perform a field by field assignment for each of the promoted struct fields
7970 // If we leave it as a GT_INITBLK we will call lvaSetVarDoNotEnregister() with a reason of DNER_BlockOp
7971 // if the Dest() is a a struct that has a "CustomLayout" and "ConstainsHoles" then we
7972 // can not use a field by field assignment and must the orginal GT_INITBLK unmodified.
7974 GenTreePtr Compiler::fgMorphInitBlock(GenTreePtr tree)
7976 noway_assert(tree->gtOper == GT_INITBLK);
7978 JITDUMP("\nfgMorphInitBlock:");
7980 GenTreePtr oneAsgTree = fgMorphOneAsgBlockOp(tree);
7983 JITDUMP(" using oneAsgTree.\n");
7988 GenTreeInitBlk* initBlkOp = tree->AsInitBlk();
7990 GenTreePtr destAddr = initBlkOp->Dest();
7991 GenTreePtr initVal = initBlkOp->InitVal();
7992 GenTreePtr blockSize = initBlkOp->Size();
7994 // The dest must be an address
7995 noway_assert(genActualType(destAddr->gtType) == TYP_I_IMPL ||
7996 destAddr->gtType == TYP_BYREF);
7998 // The size must be an integer type
7999 assert(varTypeIsIntegral(blockSize->gtType));
8001 unsigned blockWidth = 0;
8002 bool blockWidthIsConst = false;
8004 if (blockSize->IsCnsIntOrI())
8006 blockWidthIsConst = true;
8007 blockWidth = unsigned(blockSize->gtIntConCommon.IconValue());
8010 GenTreeLclVarCommon* lclVarTree = nullptr;
8012 FieldSeqNode* destFldSeq = nullptr;
8013 unsigned destLclNum = BAD_VAR_NUM;
8014 LclVarDsc * destLclVar = nullptr;
8015 bool destDoFldAsg = false;
8017 if (destAddr->IsLocalAddrExpr(this, &lclVarTree, &destFldSeq))
8019 destLclNum = lclVarTree->gtLclNum;
8020 destLclVar = &lvaTable[destLclNum];
8022 #if LOCAL_ASSERTION_PROP
8023 // Kill everything about destLclNum (and its field locals)
8024 if (optLocalAssertionProp)
8026 if (optAssertionCount > 0)
8028 fgKillDependentAssertions(destLclNum DEBUGARG(tree));
8031 #endif // LOCAL_ASSERTION_PROP
8033 if (destLclVar->lvPromoted && blockWidthIsConst)
8035 noway_assert(varTypeIsStruct(destLclVar));
8036 noway_assert(!opts.MinOpts());
8037 if (destLclVar->lvAddrExposed & destLclVar->lvContainsHoles)
8039 JITDUMP(" dest is address exposed");
8043 if (blockWidth == destLclVar->lvExactSize)
8045 JITDUMP(" (destDoFldAsg=true)");
8046 // We may decide later that a copyblk is required when this struct has holes
8047 destDoFldAsg = true;
8051 JITDUMP(" with mismatched size");
8057 // Can we use field by field assignment for the dest?
8058 if (destDoFldAsg && destLclVar->lvCustomLayout && destLclVar->lvContainsHoles)
8060 JITDUMP(" dest contains holes");
8061 destDoFldAsg = false;
8064 JITDUMP(destDoFldAsg ? " using field by field initialization.\n"
8065 : " this requires an InitBlock.\n");
8067 if (!destDoFldAsg && (destLclVar != nullptr))
8069 // If destLclVar is not a reg-sized non-field-addressed struct, set it as DoNotEnregister.
8070 if (!destLclVar->lvRegStruct)
8072 // Mark it as DoNotEnregister.
8073 lvaSetVarDoNotEnregister(destLclNum DEBUGARG(DNER_BlockOp));
8077 // Mark the dest struct as DoNotEnreg
8078 // when they are LclVar structs and we are using a CopyBlock
8079 // or the struct is not promoted
8083 #if CPU_USES_BLOCK_MOVE
8084 compBlkOpUsed = true;
8089 // The initVal must be a constant of TYP_INT
8090 noway_assert(initVal->OperGet() == GT_CNS_INT);
8091 noway_assert(genActualType(initVal->gtType) == TYP_INT);
8093 // The dest must be of a struct type.
8094 noway_assert(varTypeIsStruct(destLclVar));
8097 // Now, convert InitBlock to individual assignments
8105 unsigned fieldLclNum;
8106 unsigned fieldCnt = destLclVar->lvFieldCnt;
8108 for (unsigned i=0; i<fieldCnt; ++i)
8110 fieldLclNum = destLclVar->lvFieldLclStart + i;
8111 dest = gtNewLclvNode(fieldLclNum, lvaTable[fieldLclNum].TypeGet());
8113 noway_assert(destAddr->gtOp.gtOp1->gtOper == GT_LCL_VAR);
8114 // If it had been labeled a "USEASG", assignments to the the individual promoted fields are not.
8115 dest->gtFlags |= destAddr->gtOp.gtOp1->gtFlags & ~(GTF_NODE_MASK | GTF_VAR_USEASG);
8117 srcCopy = gtCloneExpr(initVal);
8118 noway_assert(srcCopy != nullptr);
8120 // need type of oper to be same as tree
8121 if (dest->gtType == TYP_LONG)
8123 srcCopy->ChangeOperConst(GT_CNS_NATIVELONG);
8124 // copy and extend the value
8125 srcCopy->gtIntConCommon.SetLngValue(initVal->gtIntConCommon.IconValue());
8126 /* Change the types of srcCopy to TYP_LONG */
8127 srcCopy->gtType = TYP_LONG;
8129 else if (varTypeIsFloating(dest->gtType))
8131 srcCopy->ChangeOperConst(GT_CNS_DBL);
8132 // setup the bit pattern
8133 memset(&srcCopy->gtDblCon.gtDconVal, (int)initVal->gtIntCon.gtIconVal, sizeof(srcCopy->gtDblCon.gtDconVal));
8134 /* Change the types of srcCopy to TYP_DOUBLE */
8135 srcCopy->gtType = TYP_DOUBLE;
8139 noway_assert(srcCopy->gtOper == GT_CNS_INT);
8140 noway_assert(srcCopy->TypeGet() == TYP_INT);
8141 // setup the bit pattern
8142 memset(&srcCopy->gtIntCon.gtIconVal, (int)initVal->gtIntCon.gtIconVal, sizeof(srcCopy->gtIntCon.gtIconVal));
8145 srcCopy->gtType = dest->TypeGet();
8147 asg = gtNewAssignNode(dest, srcCopy);
8149 #if LOCAL_ASSERTION_PROP
8150 if (optLocalAssertionProp)
8152 optAssertionGen(asg);
8154 #endif // LOCAL_ASSERTION_PROP
8158 tree = gtNewOperNode(GT_COMMA,
8171 tree->gtFlags |= GTF_MORPHED;
8175 printf("fgMorphInitBlock (after):\n");
8184 //------------------------------------------------------------------------
8185 // fgMorphCopyBlock: Perform the Morphing of a GT_COPYBLK and GT_COPYOBJ nodes
8188 // tree - a tree node with a gtOper of GT_COPYBLK or GT_COPYOBJ
8189 // the child nodes for tree have already been Morphed
8192 // We can return the orginal GT_COPYBLK or GT_COPYOBJ unmodified (least desirable, but always correct)
8193 // We can return a single assignment, when fgMorphOneAsgBlockOp transforms it (most desirable)
8194 // If we have performed struct promotion of the Source() or the Dest() then we will try to
8195 // perform a field by field assignment for each of the promoted struct fields
8198 // If we leave it as a GT_COPYBLK or GT_COPYOBJ we will call lvaSetVarDoNotEnregister() on both Source() and Dest()
8199 // When performing a field by field assignment we can have one of Source() or Dest treated as a blob of bytes
8200 // and in such cases we will call lvaSetVarDoNotEnregister() on the one treated as a blob of bytes.
8201 // if the Source() or Dest() is a a struct that has a "CustomLayout" and "ConstainsHoles" then we
8202 // can not use a field by field assignment and must the orginal GT_COPYBLK unmodified.
8204 GenTreePtr Compiler::fgMorphCopyBlock(GenTreePtr tree)
8206 noway_assert(tree->OperIsCopyBlkOp());
8208 JITDUMP("\nfgMorphCopyBlock:");
8210 bool isLateArg = (tree->gtFlags & GTF_LATE_ARG) != 0;
8212 GenTreePtr oneAsgTree = fgMorphOneAsgBlockOp(tree);
8216 JITDUMP(" using oneAsgTree.\n");
8221 GenTreePtr destAddr;
8223 GenTreePtr blockSize;
8226 if (tree->OperGet() == GT_COPYBLK)
8228 GenTreeCpBlk* copyBlkOp = tree->AsCpBlk();
8231 destAddr = copyBlkOp->Dest();
8232 srcAddr = copyBlkOp->Source();
8233 blockSize = copyBlkOp->Size();
8237 GenTreeCpObj* copyObjOp = tree->AsCpObj();
8240 destAddr = copyObjOp->Dest();
8241 srcAddr = copyObjOp->Source();
8242 blockSize = copyObjOp->ClsTok();
8245 noway_assert(destAddr->TypeGet() == TYP_BYREF || destAddr->TypeGet() == TYP_I_IMPL);
8246 noway_assert(srcAddr->TypeGet() == TYP_BYREF || srcAddr->TypeGet() == TYP_I_IMPL);
8248 unsigned blockWidth = 0;
8249 bool blockWidthIsConst = false;
8251 if (blockSize->IsCnsIntOrI())
8253 blockWidthIsConst = true;
8254 if (blockSize->IsIconHandle(GTF_ICON_CLASS_HDL))
8256 CORINFO_CLASS_HANDLE clsHnd = (CORINFO_CLASS_HANDLE) blockSize->gtIntConCommon.IconValue();
8257 blockWidth = info.compCompHnd->getClassSize(clsHnd);
8261 blockWidth = unsigned(blockSize->gtIntConCommon.IconValue());
8265 GenTreeLclVarCommon* lclVarTree = nullptr;
8267 FieldSeqNode* destFldSeq = nullptr;
8268 unsigned destLclNum = BAD_VAR_NUM;
8269 LclVarDsc* destLclVar = nullptr;
8270 bool destDoFldAsg = false;
8271 bool destOnStack = false;
8273 if (destAddr->IsLocalAddrExpr(this, &lclVarTree, &destFldSeq))
8276 destLclNum = lclVarTree->gtLclNum;
8277 destLclVar = &lvaTable[destLclNum];
8279 #if LOCAL_ASSERTION_PROP
8280 // Kill everything about destLclNum (and its field locals)
8281 if (optLocalAssertionProp)
8283 if (optAssertionCount > 0)
8285 fgKillDependentAssertions(destLclNum DEBUGARG(tree));
8288 #endif // LOCAL_ASSERTION_PROP
8290 if (destLclVar->lvPromoted && blockWidthIsConst)
8292 noway_assert(varTypeIsStruct(destLclVar));
8293 noway_assert(!opts.MinOpts());
8295 if (blockWidth == destLclVar->lvExactSize)
8297 JITDUMP(" (destDoFldAsg=true)");
8298 // We may decide later that a copyblk is required when this struct has holes
8299 destDoFldAsg = true;
8303 JITDUMP(" with mismatched dest size");
8308 FieldSeqNode* srcFldSeq = nullptr;
8309 unsigned srcLclNum = BAD_VAR_NUM;
8310 LclVarDsc* srcLclVar = nullptr;
8311 bool srcDoFldAsg = false;
8313 if (srcAddr->IsLocalAddrExpr(this, &lclVarTree, &srcFldSeq))
8315 srcLclNum = lclVarTree->gtLclNum;
8316 srcLclVar = &lvaTable[srcLclNum];
8318 if (srcLclVar->lvPromoted && blockWidthIsConst)
8320 noway_assert(varTypeIsStruct(srcLclVar));
8321 noway_assert(!opts.MinOpts());
8323 if (blockWidth == srcLclVar->lvExactSize)
8325 JITDUMP(" (srcDoFldAsg=true)");
8326 // We may decide later that a copyblk is required when this struct has holes
8331 JITDUMP(" with mismatched src size");
8336 // Check to see if we are required to do a copy block because the struct contains holes
8337 // and either the src or dest is externally visible
8339 bool requiresCopyBlock = false;
8340 bool srcSingleLclVarAsg = false;
8342 // If either src or dest is a reg-sized non-field-addressed struct, keep the copyBlock.
8343 if ((destLclVar != nullptr && destLclVar->lvRegStruct) ||
8344 (srcLclVar != nullptr && srcLclVar->lvRegStruct))
8346 requiresCopyBlock = true;
8349 // Can we use field by field assignment for the dest?
8350 if (destDoFldAsg && destLclVar->lvCustomLayout && destLclVar->lvContainsHoles)
8352 JITDUMP(" dest contains custom layout and contains holes");
8353 // C++ style CopyBlock with holes
8354 requiresCopyBlock = true;
8357 // Can we use field by field assignment for the src?
8358 if (srcDoFldAsg && srcLclVar->lvCustomLayout && srcLclVar->lvContainsHoles)
8360 JITDUMP(" src contains custom layout and contains holes");
8361 // C++ style CopyBlock with holes
8362 requiresCopyBlock = true;
8365 if (tree->OperGet() == GT_COPYBLK && tree->AsCpBlk()->gtBlkOpGcUnsafe)
8367 requiresCopyBlock = true;
8370 // If we passed the above checks, then we will check these two
8371 if (!requiresCopyBlock)
8373 // Are both dest and src promoted structs?
8374 if (destDoFldAsg && srcDoFldAsg)
8376 // Both structs should be of the same type, if not we will use a copy block
8377 if (lvaTable[destLclNum].lvVerTypeInfo.GetClassHandle() != lvaTable[srcLclNum].lvVerTypeInfo.GetClassHandle())
8379 requiresCopyBlock = true; // Mismatched types, leave as a CopyBlock
8380 JITDUMP(" with mismatched types");
8383 // Are neither dest or src promoted structs?
8384 else if (!destDoFldAsg && !srcDoFldAsg)
8386 requiresCopyBlock = true; // Leave as a CopyBlock
8387 JITDUMP(" with no promoted structs");
8389 else if (destDoFldAsg)
8391 // Match the following kinds of trees:
8392 // fgMorphTree BB01, stmt 9 (before)
8393 // [000052] ------------ const int 8
8394 // [000053] -A--G------- copyBlk void
8395 // [000051] ------------ addr byref
8396 // [000050] ------------ lclVar long V07 loc5
8397 // [000054] --------R--- <list> void
8398 // [000049] ------------ addr byref
8399 // [000048] ------------ lclVar struct(P) V06 loc4
8400 // long V06.h (offs=0x00) -> V17 tmp9
8401 // Yields this transformation
8402 // fgMorphCopyBlock (after):
8403 // [000050] ------------ lclVar long V07 loc5
8404 // [000085] -A---------- = long
8405 // [000083] D------N---- lclVar long V17 tmp9
8407 if (blockWidthIsConst &&
8408 (destLclVar->lvFieldCnt == 1) &&
8409 (srcLclVar != nullptr) &&
8410 (blockWidth == genTypeSize(srcLclVar->TypeGet())))
8412 // Reject the following tree:
8413 // - seen on x86chk jit\jit64\hfa\main\hfa_sf3E_r.exe
8415 // fgMorphTree BB01, stmt 6 (before)
8416 // [000038] ------------- const int 4
8417 // [000039] -A--G-------- copyBlk void
8418 // [000037] ------------- addr byref
8419 // [000036] ------------- lclVar int V05 loc3
8420 // [000040] --------R---- <list> void
8421 // [000035] ------------- addr byref
8422 // [000034] ------------- lclVar struct(P) V04 loc2
8423 // float V04.f1 (offs=0x00) -> V13 tmp6
8424 // As this would framsform into
8425 // float V13 = int V05
8427 unsigned fieldLclNum = lvaTable[destLclNum].lvFieldLclStart;
8428 var_types destType = lvaTable[fieldLclNum].TypeGet();
8429 if (srcLclVar->TypeGet() == destType)
8431 srcSingleLclVarAsg = true;
8437 // If we require a copy block the set both of the field assign bools to false
8438 if (requiresCopyBlock)
8440 // If a copy block is required then we won't do field by field assignments
8441 destDoFldAsg = false;
8442 srcDoFldAsg = false;
8445 JITDUMP(requiresCopyBlock ? " this requires a CopyBlock.\n"
8446 : " using field by field assignments.\n");
8448 // Mark the dest/src structs as DoNotEnreg
8449 // when they are not reg-sized non-field-addressed structs and we are using a CopyBlock
8450 // or the struct is not promoted
8452 if (!destDoFldAsg && (destLclVar != nullptr))
8454 if (!destLclVar->lvRegStruct)
8456 // Mark it as DoNotEnregister.
8457 lvaSetVarDoNotEnregister(destLclNum DEBUGARG(DNER_BlockOp));
8461 if (!srcDoFldAsg && (srcLclVar != nullptr) && !srcSingleLclVarAsg)
8463 if (!srcLclVar->lvRegStruct)
8465 lvaSetVarDoNotEnregister(srcLclNum DEBUGARG(DNER_BlockOp));
8469 if (requiresCopyBlock)
8471 #if CPU_USES_BLOCK_MOVE
8472 compBlkOpUsed = true;
8474 // Note that the unrolling of CopyBlk is only implemented on some platforms
8475 // Currently that includes x64 and Arm64 but not x64 or Arm32
8476 #ifdef CPBLK_UNROLL_LIMIT
8477 // If we have a CopyObj with a dest on the stack
8478 // we will convert it into an GC Unsafe CopyBlk that is non-interruptible
8479 // when its size is small enouch to be completely unrolled (i.e. between [16..64] bytes)
8481 if (isCopyObj && destOnStack && blockWidthIsConst &&
8482 (blockWidth >= (2*TARGET_POINTER_SIZE)) && (blockWidth <= CPBLK_UNROLL_LIMIT))
8484 tree->SetOper(GT_COPYBLK);
8485 tree->AsCpBlk()->gtBlkOpGcUnsafe = true; // Mark as a GC unsage copy block
8486 blockSize->gtIntConCommon.SetIconValue(ssize_t(blockWidth));
8487 blockSize->gtFlags &= ~GTF_ICON_HDL_MASK; // Clear the GTF_ICON_CLASS_HDL flags
8490 // Liveness doesn't consider copyblk arguments of simple types as being
8491 // a use or def, so explicitly mark these variables as address-exposed.
8492 if (srcLclNum != BAD_VAR_NUM && !varTypeIsStruct(srcLclVar))
8494 JITDUMP("Non-struct copyBlk src V%02d is addr exposed\n", srcLclNum);
8495 lvaTable[srcLclNum].lvAddrExposed = true;
8498 if (destLclNum != BAD_VAR_NUM && !varTypeIsStruct(destLclVar))
8500 JITDUMP("Non-struct copyBlk dest V%02d is addr exposed\n", destLclNum);
8501 lvaTable[destLclNum].lvAddrExposed = true;
8508 // Otherwise we convert this CopyBlock into individual field by field assignments
8515 GenTreePtr addrSpill = nullptr;
8516 unsigned addrSpillTemp = BAD_VAR_NUM;
8517 bool addrSpillIsStackDest = false; // true if 'addrSpill' represents the address in our local stack frame
8519 unsigned fieldCnt = DUMMY_INIT(0);
8521 if (destDoFldAsg && srcDoFldAsg)
8523 // To do fieldwise assignments for both sides, they'd better be the same struct type!
8524 // All of these conditions were checked above...
8525 assert(destLclNum != BAD_VAR_NUM && srcLclNum != BAD_VAR_NUM);
8526 assert(lvaTable[destLclNum].lvVerTypeInfo.GetClassHandle() == lvaTable[srcLclNum].lvVerTypeInfo.GetClassHandle());
8527 assert(destLclVar != nullptr && srcLclVar != nullptr && destLclVar->lvFieldCnt == srcLclVar->lvFieldCnt);
8529 fieldCnt = destLclVar->lvFieldCnt;
8530 goto _AssignFields; // No need to spill the address to the temp. Go ahead to morph it into field assignments.
8532 else if (destDoFldAsg)
8534 fieldCnt = destLclVar->lvFieldCnt;
8538 assert(srcDoFldAsg);
8539 fieldCnt = srcLclVar->lvFieldCnt;
8544 noway_assert(!srcDoFldAsg);
8545 if (gtClone(srcAddr))
8547 // srcAddr is simple expression. No need to spill.
8548 noway_assert((srcAddr->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) == 0);
8552 // srcAddr is complex expression. Clone and spill it (unless the destination is
8553 // a struct local that only has one field, in which case we'd only use the
8554 // address value once...)
8555 if (destLclVar->lvFieldCnt > 1)
8557 addrSpill = gtCloneExpr(srcAddr); // addrSpill represents the 'srcAddr'
8558 noway_assert(addrSpill != nullptr);
8565 noway_assert(!destDoFldAsg);
8567 // If we're doing field-wise stores, to an address within a local, and we copy
8568 // the address into "addrSpill", do *not* declare the original local var node in the
8569 // field address as GTF_VAR_DEF and GTF_VAR_USEASG; we will declare each of the
8570 // field-wise assignments as an "indirect" assignment to the local.
8571 // ("lclVarTree" is a subtree of "destAddr"; make sure we remove the flags before
8573 if (lclVarTree != nullptr)
8575 lclVarTree->gtFlags &= ~(GTF_VAR_DEF | GTF_VAR_USEASG);
8578 if (gtClone(destAddr))
8580 // destAddr is simple expression. No need to spill
8581 noway_assert((destAddr->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) == 0);
8585 // destAddr is complex expression. Clone and spill it (unless
8586 // the source is a struct local that only has one field, in which case we'd only
8587 // use the address value once...)
8588 if (srcLclVar->lvFieldCnt > 1)
8590 addrSpill = gtCloneExpr(destAddr); // addrSpill represents the 'destAddr'
8591 noway_assert(addrSpill != nullptr);
8594 // TODO-CQ: this should be based on a more general
8595 // "BaseAddress" method, that handles fields of structs, before or after
8597 if (addrSpill != nullptr && addrSpill->OperGet() == GT_ADDR)
8599 if (addrSpill->gtOp.gtOp1->IsLocal())
8601 // We will *not* consider this to define the local, but rather have each individual field assign
8603 addrSpill->gtOp.gtOp1->gtFlags &= ~(GTF_LIVENESS_MASK);
8604 assert(lvaTable[addrSpill->gtOp.gtOp1->gtLclVarCommon.gtLclNum].lvLclBlockOpAddr == 1);
8605 addrSpillIsStackDest = true; // addrSpill represents the address of LclVar[varNum] in our local stack frame
8611 if (addrSpill != nullptr)
8613 // Spill the (complex) address to a BYREF temp.
8614 // Note, at most one address may need to be spilled.
8615 addrSpillTemp = lvaGrabTemp(true DEBUGARG("BlockOp address local"));
8617 lvaTable[addrSpillTemp].lvType = TYP_BYREF;
8619 if (addrSpillIsStackDest)
8621 lvaTable[addrSpillTemp].lvStackByref = true;
8624 tree = gtNewAssignNode(gtNewLclvNode(addrSpillTemp, TYP_BYREF),
8627 #ifndef LEGACY_BACKEND
8628 // If we are assigning the address of a LclVar here
8629 // liveness does not account for this kind of address taken use.
8631 // We have to mark this local as address exposed so
8632 // that we don't delete the definition for this LclVar
8633 // as a dead store later on.
8635 if (addrSpill->OperGet() == GT_ADDR)
8637 GenTreePtr addrOp = addrSpill->gtOp.gtOp1;
8638 if (addrOp->IsLocal())
8640 unsigned lclVarNum = addrOp->gtLclVarCommon.gtLclNum;
8641 lvaTable[lclVarNum].lvAddrExposed = true;
8642 lvaSetVarDoNotEnregister(lclVarNum DEBUGARG(DNER_AddrExposed));
8645 #endif // !LEGACY_BACKEND
8650 for (unsigned i=0; i<fieldCnt; ++i)
8652 FieldSeqNode* curFieldSeq = nullptr;
8655 noway_assert(destLclNum != BAD_VAR_NUM);
8656 unsigned fieldLclNum = lvaTable[destLclNum].lvFieldLclStart + i;
8657 dest = gtNewLclvNode(fieldLclNum, lvaTable[fieldLclNum].TypeGet());
8659 noway_assert(destAddr->gtOp.gtOp1->gtOper == GT_LCL_VAR);
8660 // If it had been labeled a "USEASG", assignments to the the individual promoted fields are not.
8661 dest->gtFlags |= destAddr->gtOp.gtOp1->gtFlags & ~(GTF_NODE_MASK | GTF_VAR_USEASG);
8665 noway_assert(srcDoFldAsg);
8666 noway_assert(srcLclNum != BAD_VAR_NUM);
8667 unsigned fieldLclNum = lvaTable[srcLclNum].lvFieldLclStart + i;
8671 assert(addrSpillTemp != BAD_VAR_NUM);
8672 dest = gtNewLclvNode(addrSpillTemp, TYP_BYREF);
8676 dest = gtCloneExpr(destAddr);
8677 noway_assert(dest != nullptr);
8679 // Is the address of a local?
8680 GenTreeLclVarCommon* lclVarTree = nullptr;
8681 bool isEntire = false;
8682 bool* pIsEntire = (blockWidthIsConst ? &isEntire : nullptr);
8683 if (dest->DefinesLocalAddr(this, blockWidth, &lclVarTree, pIsEntire))
8685 lclVarTree->gtFlags |= GTF_VAR_DEF;
8687 lclVarTree->gtFlags |= GTF_VAR_USEASG;
8691 GenTreePtr fieldOffsetNode = gtNewIconNode(lvaTable[fieldLclNum].lvFldOffset, TYP_I_IMPL);
8692 // Have to set the field sequence -- which means we need the field handle.
8693 CORINFO_CLASS_HANDLE classHnd = lvaTable[srcLclNum].lvVerTypeInfo.GetClassHandle();
8694 CORINFO_FIELD_HANDLE fieldHnd = info.compCompHnd->getFieldInClass(classHnd, lvaTable[fieldLclNum].lvFldOrdinal);
8695 curFieldSeq = GetFieldSeqStore()->CreateSingleton(fieldHnd);
8696 fieldOffsetNode->gtIntCon.gtFieldSeq = curFieldSeq;
8698 dest = gtNewOperNode(GT_ADD, TYP_BYREF,
8702 dest = gtNewOperNode(GT_IND, lvaTable[fieldLclNum].TypeGet(), dest);
8704 // !!! The destination could be on stack. !!!
8705 // This flag will let us choose the correct write barrier.
8706 dest->gtFlags |= GTF_IND_TGTANYWHERE;
8712 noway_assert(srcLclNum != BAD_VAR_NUM);
8713 unsigned fieldLclNum = lvaTable[srcLclNum].lvFieldLclStart + i;
8714 src = gtNewLclvNode(fieldLclNum, lvaTable[fieldLclNum].TypeGet());
8716 noway_assert(srcAddr->gtOp.gtOp1->gtOper == GT_LCL_VAR);
8717 src->gtFlags |= srcAddr->gtOp.gtOp1->gtFlags & ~GTF_NODE_MASK;
8721 noway_assert(destDoFldAsg);
8722 noway_assert(destLclNum != BAD_VAR_NUM);
8723 unsigned fieldLclNum = lvaTable[destLclNum].lvFieldLclStart + i;
8725 if (srcSingleLclVarAsg)
8727 noway_assert(fieldCnt == 1);
8728 noway_assert(srcLclVar != nullptr);
8729 noway_assert(addrSpill == nullptr);
8731 src = gtNewLclvNode(srcLclNum, srcLclVar->TypeGet());
8737 assert(addrSpillTemp != BAD_VAR_NUM);
8738 src = gtNewLclvNode(addrSpillTemp, TYP_BYREF);
8742 src = gtCloneExpr(srcAddr);
8743 noway_assert(src != nullptr);
8746 CORINFO_CLASS_HANDLE classHnd = lvaTable[destLclNum].lvVerTypeInfo.GetClassHandle();
8747 CORINFO_FIELD_HANDLE fieldHnd = info.compCompHnd->getFieldInClass(classHnd, lvaTable[fieldLclNum].lvFldOrdinal);
8748 curFieldSeq = GetFieldSeqStore()->CreateSingleton(fieldHnd);
8750 src = gtNewOperNode(GT_ADD, TYP_BYREF,
8752 new(this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL,
8753 lvaTable[fieldLclNum].lvFldOffset,
8756 src = gtNewOperNode(GT_IND, lvaTable[fieldLclNum].TypeGet(), src);
8760 noway_assert(dest->TypeGet() == src->TypeGet());
8762 asg = gtNewAssignNode(dest, src);
8764 // If we spilled the address, and we didn't do individual field assignments to promoted fields,
8765 // and it was of a local, record the assignment as an indirect update of a local.
8766 if (addrSpill && !destDoFldAsg && destLclNum != BAD_VAR_NUM)
8768 curFieldSeq = GetFieldSeqStore()->Append(destFldSeq, curFieldSeq);
8769 bool isEntire = (genTypeSize(var_types(lvaTable[destLclNum].lvType))
8770 == genTypeSize(dest->TypeGet()));
8771 IndirectAssignmentAnnotation* pIndirAnnot =
8772 new (this, CMK_Unknown) IndirectAssignmentAnnotation(destLclNum, curFieldSeq, isEntire);
8773 GetIndirAssignMap()->Set(asg, pIndirAnnot);
8776 #if LOCAL_ASSERTION_PROP
8777 if (optLocalAssertionProp)
8779 optAssertionGen(asg);
8781 #endif // LOCAL_ASSERTION_PROP
8785 tree = gtNewOperNode(GT_COMMA,
8799 tree->gtFlags |= GTF_LATE_ARG;
8803 tree->gtFlags |= GTF_MORPHED;
8807 printf("\nfgMorphCopyBlock (after):\n");
8816 // insert conversions and normalize to make tree amenable to register
8818 GenTree* Compiler::fgMorphForRegisterFP(GenTree *tree)
8820 GenTreePtr op1 = tree->gtOp.gtOp1;
8821 GenTreePtr op2 = tree->gtGetOp2();
8823 if (tree->OperIsArithmetic()
8824 && varTypeIsFloating(tree))
8826 if (op1->TypeGet() != tree->TypeGet())
8828 tree->gtOp.gtOp1 = gtNewCastNode(tree->TypeGet(), tree->gtOp.gtOp1, tree->TypeGet());
8830 if (op2->TypeGet() != tree->TypeGet())
8832 tree->gtOp.gtOp2 = gtNewCastNode(tree->TypeGet(), tree->gtOp.gtOp2, tree->TypeGet());
8835 else if (tree->OperIsCompare()
8836 && varTypeIsFloating(op1)
8837 && op1->TypeGet() != op2->TypeGet())
8839 // both had better be floating, just one bigger than other
8840 assert (varTypeIsFloating(op2));
8841 if (op1->TypeGet() == TYP_FLOAT)
8843 tree->gtOp.gtOp1 = gtNewCastNode(TYP_DOUBLE, tree->gtOp.gtOp1, TYP_DOUBLE);
8845 else if (op2->TypeGet() == TYP_FLOAT)
8847 tree->gtOp.gtOp2 = gtNewCastNode(TYP_DOUBLE, tree->gtOp.gtOp2, TYP_DOUBLE);
8854 GenTree* Compiler::fgMorphRecognizeBoxNullable(GenTree* compare)
8856 GenTree* op1 = compare->gtOp.gtOp1;
8857 GenTree* op2 = compare->gtOp.gtOp2;
8859 GenTreeCall* opCall;
8861 // recognize this pattern:
8863 // stmtExpr void (IL 0x000... ???)
8867 // call help ref HELPER.CORINFO_HELP_BOX_NULLABLE
8868 // const(h) long 0x7fed96836c8 class
8870 // ld.lclVar struct V00 arg0
8873 // which comes from this code (reported by customer as being slow) :
8875 // private static bool IsNull<T>(T arg)
8877 // return arg==null;
8881 if (op1->IsCnsIntOrI() && op2->IsHelperCall())
8884 opCall = op2->AsCall();
8886 else if (op1->IsHelperCall() && op2->IsCnsIntOrI())
8889 opCall = op1->AsCall();
8896 if (opCns->gtIntConCommon.IconValue() != 0)
8899 if (eeGetHelperNum(opCall->gtCallMethHnd) != CORINFO_HELP_BOX_NULLABLE)
8902 // replace the box with an access of the nullable 'hasValue' field which is at the zero offset
8903 GenTree* newOp = gtNewOperNode(GT_IND, TYP_BOOL, opCall->gtCall.gtCallArgs->gtOp.gtOp2->gtOp.gtOp1);
8906 compare->gtOp.gtOp1 = newOp;
8908 compare->gtOp.gtOp2 = newOp;
8915 //--------------------------------------------------------------------------------------
8916 // fgCopySIMDNode: make a copy of a SIMD intrinsic node, e.g. so that a field can be accessed.
8919 // simdNode - The GenTreeSIMD node to be copied
8922 // A comma node where op1 is the assignment of the simd node to a temp, and op2 is the temp lclVar.
8925 Compiler::fgCopySIMDNode(GenTreeSIMD* simdNode)
8927 // Copy the result of the SIMD intrinsic into a temp.
8928 unsigned lclNum = lvaGrabTemp(true DEBUGARG("Copy of SIMD intrinsic with field access"));
8930 CORINFO_CLASS_HANDLE simdHandle = NO_CLASS_HANDLE;
8931 // We only have fields of the fixed float vectors.
8932 noway_assert(simdNode->gtSIMDBaseType == TYP_FLOAT);
8933 switch(simdNode->gtSIMDSize)
8935 case 8: simdHandle = SIMDVector2Handle; break;
8936 case 12: simdHandle = SIMDVector3Handle; break;
8937 case 16: simdHandle = SIMDVector4Handle; break;
8938 default: noway_assert(!"field of unexpected SIMD type"); break;
8940 assert(simdHandle != NO_CLASS_HANDLE);
8942 lvaSetStruct(lclNum, simdHandle, false, true);
8943 lvaTable[lclNum].lvFieldAccessed = true;
8945 GenTree* asg = gtNewTempAssign(lclNum, simdNode);
8946 GenTree* newLclVarNode = new (this, GT_LCL_VAR) GenTreeLclVar(simdNode->TypeGet(), lclNum, BAD_IL_OFFSET);
8948 GenTree* comma = gtNewOperNode(GT_COMMA, simdNode->TypeGet(), asg, newLclVarNode);
8952 //--------------------------------------------------------------------------------------------------------------
8953 // getSIMDStructFromField:
8954 // Checking whether the field belongs to a simd struct or not. If it is, return the GenTreePtr for
8955 // the struct node, also base type, field index and simd size. If it is not, just return nullptr.
8956 // Usually if the tree node is from a simd lclvar which is not used in any SIMD intrinsic, then we
8957 // should return nullptr, since in this case we should treat SIMD struct as a regular struct.
8958 // However if no matter what, you just want get simd struct node, you can set the ignoreUsedInSIMDIntrinsic
8959 // as true. Then there will be no IsUsedInSIMDIntrinsic checking, and it will return SIMD struct node
8960 // if the struct is a SIMD struct.
8963 // tree - GentreePtr. This node will be checked to see this is a field which belongs to a simd
8964 // struct used for simd intrinsic or not.
8965 // pBaseTypeOut - var_types pointer, if the tree node is the tree we want, we set *pBaseTypeOut
8966 // to simd lclvar's base type.
8967 // indexOut - unsigned pointer, if the tree is used for simd intrinsic, we will set *indexOut
8968 // equals to the index number of this field.
8969 // simdSizeOut - unsigned pointer, if the tree is used for simd intrinsic, set the *simdSizeOut
8970 // equals to the simd struct size which this tree belongs to.
8971 // ignoreUsedInSIMDIntrinsic - bool. If this is set to true, then this function will ignore
8972 // the UsedInSIMDIntrinsic check.
8975 // A GenTreePtr which points the simd lclvar tree belongs to. If the tree is not the simd
8976 // instrinic related field, return nullptr.
8979 GenTreePtr Compiler::getSIMDStructFromField(GenTreePtr tree, var_types* pBaseTypeOut, unsigned* indexOut, unsigned* simdSizeOut, bool ignoreUsedInSIMDIntrinsic/*false*/)
8981 GenTreePtr ret = nullptr;
8982 if(tree->OperGet() == GT_FIELD)
8984 GenTreePtr objRef = tree->gtField.gtFldObj;
8985 if (objRef != nullptr)
8987 GenTreePtr obj = nullptr;
8988 if (objRef->gtOper == GT_ADDR)
8990 obj = objRef->gtOp.gtOp1;
8992 else if(ignoreUsedInSIMDIntrinsic)
9001 if (isSIMDTypeLocal(obj))
9003 unsigned lclNum = obj->gtLclVarCommon.gtLclNum;
9004 LclVarDsc* varDsc = &lvaTable[lclNum];
9005 if(varDsc->lvIsUsedInSIMDIntrinsic() || ignoreUsedInSIMDIntrinsic)
9007 *simdSizeOut = varDsc->lvExactSize;
9008 *pBaseTypeOut = getBaseTypeOfSIMDLocal(obj);
9012 else if (obj->OperGet() == GT_SIMD)
9015 GenTreeSIMD* simdNode = obj->AsSIMD();
9016 *simdSizeOut = simdNode->gtSIMDSize;
9017 *pBaseTypeOut = simdNode->gtSIMDBaseType;
9023 unsigned BaseTypeSize = genTypeSize(*pBaseTypeOut);
9024 *indexOut = tree->gtField.gtFldOffset / BaseTypeSize;
9029 /*****************************************************************************
9030 * If a read operation tries to access simd struct field, then transform the this
9031 * operation to to the SIMD intrinsic SIMDIntrinsicGetItem, and return the new tree.
9032 * Otherwise, return the old tree.
9034 * tree - GenTreePtr. If this pointer points to simd struct which is used for simd
9035 * intrinsic. We will morph it as simd intrinsic SIMDIntrinsicGetItem.
9037 * A GenTreePtr which points to the new tree. If the tree is not for simd intrinsic,
9041 GenTreePtr Compiler::fgMorphFieldToSIMDIntrinsicGet(GenTreePtr tree)
9044 var_types baseType = TYP_UNKNOWN;
9045 unsigned simdSize = 0;
9046 GenTreePtr simdStructNode = getSIMDStructFromField(tree, &baseType, &index, &simdSize);
9047 if(simdStructNode != nullptr)
9050 assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
9051 GenTree* op2 = gtNewIconNode(index);
9052 tree = gtNewSIMDNode(baseType, simdStructNode, op2, SIMDIntrinsicGetItem, baseType, simdSize);
9054 tree->gtFlags |= GTF_MORPHED;
9060 /*****************************************************************************
9061 * Transform an assignment of a SIMD struct field to SIMD intrinsic
9062 * SIMDIntrinsicGetItem, and return a new tree. If If it is not such an assignment,
9063 * then return the old tree.
9065 * tree - GenTreePtr. If this pointer points to simd struct which is used for simd
9066 * intrinsic. We will morph it as simd intrinsic set.
9068 * A GenTreePtr which points to the new tree. If the tree is not for simd intrinsic,
9072 GenTreePtr Compiler::fgMorphFieldAssignToSIMDIntrinsicSet(GenTreePtr tree)
9074 assert(tree->OperGet() == GT_ASG);
9075 GenTreePtr op1 = tree->gtGetOp1();
9076 GenTreePtr op2 = tree->gtGetOp2();
9079 var_types baseType = TYP_UNKNOWN;
9080 unsigned simdSize = 0;
9081 GenTreePtr simdOp1Struct = getSIMDStructFromField(op1, &baseType, &index, &simdSize);
9082 if (simdOp1Struct != nullptr)
9084 //Generate the simd set intrinsic
9085 assert(simdSize >= ((index + 1) * genTypeSize(baseType)));
9087 SIMDIntrinsicID simdIntrinsicID = SIMDIntrinsicInvalid;
9091 simdIntrinsicID = SIMDIntrinsicSetX;
9094 simdIntrinsicID = SIMDIntrinsicSetY;
9097 simdIntrinsicID = SIMDIntrinsicSetZ;
9100 simdIntrinsicID = SIMDIntrinsicSetW;
9103 noway_assert("There is no set intrinsic for index bigger than 3");
9107 GenTreePtr newStruct = gtClone(simdOp1Struct);
9108 assert((newStruct != nullptr) && (varTypeIsSIMD(newStruct)));
9109 GenTreePtr simdTree = gtNewSIMDNode(newStruct->gtType, simdOp1Struct, op2, simdIntrinsicID, baseType, simdSize);
9110 GenTreePtr copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, newStruct);
9111 tree = gtNewBlkOpNode(GT_COPYBLK,
9113 gtNewOperNode(GT_ADDR, TYP_BYREF, simdTree),
9114 gtNewIconNode(simdSize),
9117 tree->gtFlags |= GTF_MORPHED;
9125 /*****************************************************************************
9127 * Transform the given GTK_SMPOP tree for code generation.
9131 #pragma warning(push)
9132 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
9134 GenTreePtr Compiler::fgMorphSmpOp(GenTreePtr tree, MorphAddrContext* mac)
9136 // this extra scope is a workaround for a gcc bug
9137 // the inline destructor for ALLOCA_CHECK confuses the control
9138 // flow and gcc thinks that the function never returns
9141 noway_assert(tree->OperKind() & GTK_SMPOP);
9143 /* The steps in this function are :
9144 o Perform required preorder processing
9145 o Process the first, then second operand, if any
9146 o Perform required postorder morphing
9147 o Perform optional postorder morphing if optimizing
9150 bool isQmarkColon = false;
9152 #if LOCAL_ASSERTION_PROP
9153 AssertionIndex origAssertionCount = DUMMY_INIT(0);
9154 AssertionDsc * origAssertionTab = DUMMY_INIT(NULL);
9156 AssertionIndex thenAssertionCount = DUMMY_INIT(0);
9157 AssertionDsc * thenAssertionTab = DUMMY_INIT(NULL);
9162 #if !FEATURE_STACK_FP_X87
9163 tree = fgMorphForRegisterFP(tree);
9167 genTreeOps oper = tree->OperGet();
9168 var_types typ = tree->TypeGet();
9169 GenTreePtr op1 = tree->gtOp.gtOp1;
9170 GenTreePtr op2 = tree->gtGetOp2();
9172 /*-------------------------------------------------------------------------
9173 * First do any PRE-ORDER processing
9178 // Some arithmetic operators need to use a helper call to the EE
9182 tree = fgDoNormalizeOnStore(tree);
9183 /* fgDoNormalizeOnStore can change op2 */
9184 noway_assert(op1 == tree->gtOp.gtOp1);
9185 op2 = tree->gtOp.gtOp2;
9189 // We should check whether op2 should be assigned to a SIMD field or not.
9190 // if it is, we should tranlate the tree to simd intrinsic
9191 GenTreePtr newTree = fgMorphFieldAssignToSIMDIntrinsicSet(tree);
9192 if (newTree != tree)
9195 oper = tree->OperGet();
9196 typ = tree->TypeGet();
9197 op1 = tree->gtOp.gtOp1;
9198 op2 = tree->gtGetOp2();
9220 /* We can't CSE the LHS of an assignment. Only r-values can be CSEed */
9221 op1->gtFlags |= GTF_DONT_CSE;
9226 /* op1 of a GT_ADDR is an l-value. Only r-values can be CSEed */
9227 op1->gtFlags |= GTF_DONT_CSE;
9235 if (op1->OperKind() & GTK_RELOP)
9237 noway_assert((oper == GT_JTRUE) || (op1->gtFlags & GTF_RELOP_QMARK));
9238 /* Mark the comparison node with GTF_RELOP_JMP_USED so it knows that it does
9239 not need to materialize the result as a 0 or 1. */
9241 /* We also mark it as DONT_CSE, as we don't handle QMARKs with nonRELOP op1s */
9242 op1->gtFlags |= (GTF_RELOP_JMP_USED | GTF_DONT_CSE);
9244 // Request that the codegen for op1 sets the condition flags
9245 // when it generates the code for op1.
9247 // Codegen for op1 must set the condition flags if
9248 // this method returns true.
9250 op1->gtRequestSetFlags();
9254 GenTreePtr effOp1 = op1->gtEffectiveVal();
9256 noway_assert( (effOp1->gtOper == GT_CNS_INT) &&
9257 ((effOp1->gtIntCon.gtIconVal == 0) || (effOp1->gtIntCon.gtIconVal == 1)) );
9262 #if LOCAL_ASSERTION_PROP
9263 if (optLocalAssertionProp)
9265 isQmarkColon = true;
9269 return fgMorphArrayIndex(tree);
9272 return fgMorphCast(tree);
9276 #ifndef _TARGET_64BIT_
9277 #if !LONG_MATH_REGPARAM
9278 if (typ == TYP_LONG)
9280 /* For (long)int1 * (long)int2, we dont actually do the
9281 casts, and just multiply the 32 bit values, which will
9282 give us the 64 bit result in edx:eax */
9285 if ((op1->gtOper == GT_CAST &&
9286 op2->gtOper == GT_CAST &&
9287 genActualType(op1->CastFromType()) == TYP_INT &&
9288 genActualType(op2->CastFromType()) == TYP_INT)&&
9289 !op1->gtOverflow() && !op2->gtOverflow())
9291 // The casts have to be of the same signedness.
9292 if ((op1->gtFlags & GTF_UNSIGNED) != (op2->gtFlags & GTF_UNSIGNED))
9294 //We see if we can force an int constant to change its signedness
9296 if (op1->gtCast.CastOp()->gtOper == GT_CNS_INT)
9298 else if (op2->gtCast.CastOp()->gtOper == GT_CNS_INT)
9303 if ( ((unsigned)(constOp->gtCast.CastOp()->gtIntCon.gtIconVal) < (unsigned)(0x80000000)) )
9304 constOp->gtFlags ^= GTF_UNSIGNED;
9309 // The only combination that can overflow
9310 if (tree->gtOverflow() && (tree->gtFlags & GTF_UNSIGNED) &&
9311 !( op1->gtFlags & GTF_UNSIGNED))
9314 /* Remaining combinations can never overflow during long mul. */
9316 tree->gtFlags &= ~GTF_OVERFLOW;
9318 /* Do unsigned mul only if the casts were unsigned */
9320 tree->gtFlags &= ~GTF_UNSIGNED;
9321 tree->gtFlags |= op1->gtFlags & GTF_UNSIGNED;
9323 /* Since we are committing to GTF_MUL_64RSLT, we don't want
9324 the casts to be folded away. So morph the castees directly */
9326 op1->gtOp.gtOp1 = fgMorphTree(op1->gtOp.gtOp1);
9327 op2->gtOp.gtOp1 = fgMorphTree(op2->gtOp.gtOp1);
9329 // Propagate side effect flags up the tree
9330 op1->gtFlags &= ~GTF_ALL_EFFECT;
9331 op1->gtFlags |= (op1->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT);
9332 op2->gtFlags &= ~GTF_ALL_EFFECT;
9333 op2->gtFlags |= (op2->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT);
9335 // If the GT_MUL can be altogether folded away, we should do that.
9337 if ((op1->gtCast.CastOp()->OperKind() &
9338 op2->gtCast.CastOp()->OperKind() & GTK_CONST) && opts.OptEnabled(CLFLG_CONSTANTFOLD))
9340 tree->gtOp.gtOp1 = op1 = gtFoldExprConst(op1);
9341 tree->gtOp.gtOp2 = op2 = gtFoldExprConst(op2);
9342 noway_assert(op1->OperKind() & op2->OperKind() & GTK_CONST);
9343 tree = gtFoldExprConst(tree);
9344 noway_assert(tree->OperIsConst());
9348 tree->gtFlags |= GTF_MUL_64RSLT;
9350 // If op1 and op2 are unsigned casts, we need to do an unsigned mult
9351 tree->gtFlags |= (op1->gtFlags & GTF_UNSIGNED);
9353 // Insert GT_NOP nodes for the cast operands so that they do not get folded
9354 // And propagate the new flags. We don't want to CSE the casts because
9355 // codegen expects GTF_MUL_64RSLT muls to have a certain layout.
9357 if (op1->gtCast.CastOp()->OperGet() != GT_NOP)
9359 op1->gtOp.gtOp1 = gtNewOperNode(GT_NOP, TYP_INT, op1->gtCast.CastOp());
9360 op1->gtFlags &= ~GTF_ALL_EFFECT;
9361 op1->gtFlags |= (op1->gtCast.CastOp()->gtFlags & GTF_ALL_EFFECT);
9362 op1->gtFlags |= GTF_DONT_CSE;
9365 if (op2->gtCast.CastOp()->OperGet() != GT_NOP)
9367 op2->gtOp.gtOp1 = gtNewOperNode(GT_NOP, TYP_INT, op2->gtCast.CastOp());
9368 op2->gtFlags &= ~GTF_ALL_EFFECT;
9369 op2->gtFlags |= (op2->gtCast.CastOp()->gtFlags & GTF_ALL_EFFECT);
9370 op2->gtFlags |= GTF_DONT_CSE;
9373 tree->gtFlags &= ~GTF_ALL_EFFECT;
9374 tree->gtFlags |= ((op1->gtFlags | op2->gtFlags) & GTF_ALL_EFFECT);
9376 goto DONE_MORPHING_CHILDREN;
9378 else if ((tree->gtFlags & GTF_MUL_64RSLT) == 0)
9381 if (tree->gtOverflow())
9382 helper = (tree->gtFlags & GTF_UNSIGNED) ? CORINFO_HELP_ULMUL_OVF
9383 : CORINFO_HELP_LMUL_OVF;
9385 helper = CORINFO_HELP_LMUL;
9387 goto USE_HELPER_FOR_ARITH;
9391 /* We are seeing this node again. We have decided to use
9392 GTF_MUL_64RSLT, so leave it alone. */
9394 assert(tree->gtIsValid64RsltMul());
9397 #endif // !LONG_MATH_REGPARAM
9398 #endif // !_TARGET_64BIT_
9404 #ifndef _TARGET_64BIT_
9405 #if !LONG_MATH_REGPARAM
9406 if (typ == TYP_LONG)
9408 helper = CORINFO_HELP_LDIV;
9409 goto USE_HELPER_FOR_ARITH;
9413 #if USE_HELPERS_FOR_INT_DIV
9414 if (typ == TYP_INT && !fgIsSignedDivOptimizable(op2))
9416 helper = CORINFO_HELP_DIV;
9417 goto USE_HELPER_FOR_ARITH;
9420 #endif // !_TARGET_64BIT_
9422 #ifndef LEGACY_BACKEND
9423 if (op2->gtOper == GT_CAST && op2->gtOp.gtOp1->IsCnsIntOrI())
9425 op2 = gtFoldExprConst(op2);
9428 if (fgShouldUseMagicNumberDivide(tree->AsOp()))
9430 tree = fgMorphDivByConst(tree->AsOp());
9431 op1 = tree->gtOp.gtOp1;
9432 op2 = tree->gtOp.gtOp2;
9434 #endif // !LEGACY_BACKEND
9440 #ifndef _TARGET_64BIT_
9441 #if !LONG_MATH_REGPARAM
9442 if (typ == TYP_LONG)
9444 helper = CORINFO_HELP_ULDIV;
9445 goto USE_HELPER_FOR_ARITH;
9448 #if USE_HELPERS_FOR_INT_DIV
9449 if (typ == TYP_INT && !fgIsUnsignedDivOptimizable(op2))
9451 helper = CORINFO_HELP_UDIV;
9452 goto USE_HELPER_FOR_ARITH;
9455 #endif // _TARGET_64BIT_
9461 if (varTypeIsFloating(typ))
9463 helper = CORINFO_HELP_DBLREM;
9465 if (op1->TypeGet() == TYP_FLOAT)
9466 if (op2->TypeGet() == TYP_FLOAT)
9467 helper = CORINFO_HELP_FLTREM;
9469 tree->gtOp.gtOp1 = op1 = gtNewCastNode(TYP_DOUBLE, op1, TYP_DOUBLE);
9471 if (op2->TypeGet() == TYP_FLOAT)
9472 tree->gtOp.gtOp2 = op2 = gtNewCastNode(TYP_DOUBLE, op2, TYP_DOUBLE);
9473 goto USE_HELPER_FOR_ARITH;
9476 // Do not use optimizations (unlike UMOD's idiv optimizing during codegen) for signed mod.
9477 // A similar optimization for signed mod will not work for a negative perfectly divisible
9478 // HI-word. To make it correct, we would need to divide without the sign and then flip the
9479 // result sign after mod. This requires 18 opcodes + flow making it not worthy to inline.
9480 goto ASSIGN_HELPER_FOR_MOD;
9484 #ifdef _TARGET_ARMARCH_
9486 // Note for _TARGET_ARMARCH_ we don't have a remainder instruction, so we don't do this optimization
9488 #else // _TARGET_XARCH
9489 /* If this is an unsigned long mod with op2 which is a cast to long from a
9490 constant int, then don't morph to a call to the helper. This can be done
9491 faster inline using idiv.
9495 if ((typ == TYP_LONG) && opts.OptEnabled(CLFLG_CONSTANTFOLD) &&
9496 ((tree->gtFlags & GTF_UNSIGNED) == (op1->gtFlags & GTF_UNSIGNED)) &&
9497 ((tree->gtFlags & GTF_UNSIGNED) == (op2->gtFlags & GTF_UNSIGNED)))
9499 if (op2->gtOper == GT_CAST &&
9500 op2->gtCast.CastOp()->gtOper == GT_CNS_INT &&
9501 op2->gtCast.CastOp()->gtIntCon.gtIconVal >= 2 &&
9502 op2->gtCast.CastOp()->gtIntCon.gtIconVal <= 0x3fffffff &&
9503 (tree->gtFlags & GTF_UNSIGNED) == (op2->gtCast.CastOp()->gtFlags & GTF_UNSIGNED))
9505 tree->gtOp.gtOp2 = op2 = fgMorphCast(op2);
9506 noway_assert(op2->gtOper == GT_CNS_NATIVELONG);
9509 if (op2->gtOper == GT_CNS_NATIVELONG &&
9510 op2->gtIntConCommon.LngValue() >= 2 &&
9511 op2->gtIntConCommon.LngValue() <= 0x3fffffff)
9513 tree->gtOp.gtOp1 = op1 = fgMorphTree(op1);
9514 noway_assert(op1->TypeGet() == TYP_LONG);
9516 // Update flags for op1 morph
9517 tree->gtFlags &= ~GTF_ALL_EFFECT;
9519 tree->gtFlags |= (op1->gtFlags & GTF_ALL_EFFECT); // Only update with op1 as op2 is a constant
9521 // If op1 is a constant, then do constant folding of the division operator
9522 if (op1->gtOper == GT_CNS_NATIVELONG)
9524 tree = gtFoldExpr(tree);
9529 #endif // _TARGET_XARCH
9531 ASSIGN_HELPER_FOR_MOD:
9533 // For "val % 1", return 0 if op1 doesn't have any side effects
9534 // and we are not in the CSE phase, we cannot discard 'tree'
9535 // because it may contain CSE expressions that we haven't yet examined.
9537 if (((op1->gtFlags & GTF_SIDE_EFFECT) == 0) && !optValnumCSE_phase)
9539 if (((op2->gtOper == GT_CNS_INT) && (op2->gtIntConCommon.IconValue() == 1))
9540 || ((op2->gtOper == GT_CNS_LNG) && (op2->gtIntConCommon.LngValue() == 1)))
9542 GenTreePtr zeroNode = gtNewZeroConNode(typ);
9544 zeroNode->gtFlags |= GTF_MORPHED;
9546 DEBUG_DESTROY_NODE(tree);
9551 #ifndef _TARGET_64BIT_
9552 #if !LONG_MATH_REGPARAM
9553 if (typ == TYP_LONG)
9555 helper = (oper == GT_UMOD) ? CORINFO_HELP_ULMOD : CORINFO_HELP_LMOD;
9556 goto USE_HELPER_FOR_ARITH;
9560 #if USE_HELPERS_FOR_INT_DIV
9563 if (oper == GT_UMOD && !fgIsUnsignedModOptimizable(op2))
9565 helper = CORINFO_HELP_UMOD;
9566 goto USE_HELPER_FOR_ARITH;
9568 else if (oper == GT_MOD && !fgIsSignedModOptimizable(op2))
9570 helper = CORINFO_HELP_MOD;
9571 goto USE_HELPER_FOR_ARITH;
9575 #endif // !_TARGET_64BIT_
9577 #ifndef LEGACY_BACKEND
9578 if (op2->gtOper == GT_CAST && op2->gtOp.gtOp1->IsCnsIntOrI())
9580 op2 = gtFoldExprConst(op2);
9583 #ifdef _TARGET_ARM64_
9585 // For ARM64 we don't have a remainder instruction,
9586 // The architecture manual suggests the following transformation to
9587 // generate code for such operator:
9589 // a % b = a - (a / b) * b;
9591 tree = fgMorphModToSubMulDiv(tree->AsOp());
9592 op1 = tree->gtOp.gtOp1;
9593 op2 = tree->gtOp.gtOp2;
9595 #else // !_TARGET_ARM64_
9597 if (oper != GT_UMOD && fgShouldUseMagicNumberDivide(tree->AsOp()))
9599 tree = fgMorphModByConst(tree->AsOp());
9600 op1 = tree->gtOp.gtOp1;
9601 op2 = tree->gtOp.gtOp2;
9604 #endif //_TARGET_ARM64_
9605 #endif // !LEGACY_BACKEND
9608 USE_HELPER_FOR_ARITH:
9610 /* We have to morph these arithmetic operations into helper calls
9611 before morphing the arguments (preorder), else the arguments
9612 won't get correct values of fgPtrArgCntCur.
9613 However, try to fold the tree first in case we end up with a
9614 simple node which won't need a helper call at all */
9616 noway_assert(tree->OperIsBinary());
9618 GenTreePtr oldTree = tree;
9620 tree = gtFoldExpr(tree);
9622 // Were we able to fold it ?
9623 // Note that gtFoldExpr may return a non-leaf even if successful
9624 // e.g. for something like "expr / 1" - see also bug #290853
9625 if (tree->OperIsLeaf() || (oldTree != tree))
9628 return (oldTree != tree) ? fgMorphTree(tree) : fgMorphLeaf(tree);
9631 // Did we fold it into a comma node with throw?
9632 if (tree->gtOper == GT_COMMA)
9634 noway_assert(fgIsCommaThrow(tree));
9635 return fgMorphTree(tree);
9638 return fgMorphIntoHelperCall(tree, helper, gtNewArgList(op1, op2));
9641 // normalize small integer return values
9642 if (fgGlobalMorph && varTypeIsSmall(info.compRetType) &&
9643 (op1 != NULL) && (op1->TypeGet() != TYP_VOID) &&
9644 fgCastNeeded(op1, info.compRetType))
9646 // Small-typed return values are normalized by the callee
9647 op1 = gtNewCastNode(TYP_INT, op1, info.compRetType);
9649 // Propagate GTF_COLON_COND
9650 op1->gtFlags|=(tree->gtFlags & GTF_COLON_COND);
9652 tree->gtOp.gtOp1 = fgMorphCast(op1);
9654 // Propagate side effect flags
9655 tree->gtFlags &= ~GTF_ALL_EFFECT;
9656 tree->gtFlags |= (tree->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT);
9665 // Check for typeof(...) == obj.GetType()
9666 // Also check for typeof(...) == typeof(...)
9667 // IMPORTANT NOTE: this optimization relies on a one-to-one mapping between
9668 // type handles and instances of System.Type
9669 // If this invariant is ever broken, the optimization will need updating
9671 #ifdef LEGACY_BACKEND
9672 if ( op1->gtOper == GT_CALL &&
9673 op2->gtOper == GT_CALL &&
9674 ((op1->gtCall.gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) || (op1->gtCall.gtCallType == CT_HELPER)) &&
9675 ((op2->gtCall.gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) || (op2->gtCall.gtCallType == CT_HELPER)))
9677 if ((((op1->gtOper == GT_INTRINSIC) && (op1->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Object_GetType)) ||
9678 ((op1->gtOper == GT_CALL) && (op1->gtCall.gtCallType == CT_HELPER))) &&
9679 (((op2->gtOper == GT_INTRINSIC) && (op2->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Object_GetType)) ||
9680 ((op2->gtOper == GT_CALL) && (op2->gtCall.gtCallType == CT_HELPER))))
9683 GenTreePtr pGetClassFromHandle;
9684 GenTreePtr pGetType;
9686 #ifdef LEGACY_BACKEND
9687 bool bOp1ClassFromHandle = gtIsTypeHandleToRuntimeTypeHelper(op1);
9688 bool bOp2ClassFromHandle = gtIsTypeHandleToRuntimeTypeHelper(op2);
9690 bool bOp1ClassFromHandle = op1->gtOper == GT_CALL ? gtIsTypeHandleToRuntimeTypeHelper(op1) : false;
9691 bool bOp2ClassFromHandle = op2->gtOper == GT_CALL ? gtIsTypeHandleToRuntimeTypeHelper(op2) : false;
9694 // Optimize typeof(...) == typeof(...)
9695 // Typically this occurs in generic code that attempts a type switch
9696 // e.g. typeof(T) == typeof(int)
9698 if (bOp1ClassFromHandle && bOp2ClassFromHandle)
9700 GenTreePtr classFromHandleArg1 = tree->gtOp.gtOp1->gtCall.gtCallArgs->gtOp.gtOp1;
9701 GenTreePtr classFromHandleArg2 = tree->gtOp.gtOp2->gtCall.gtCallArgs->gtOp.gtOp1;
9703 GenTreePtr compare = gtNewOperNode(oper, TYP_INT,
9704 classFromHandleArg1,
9705 classFromHandleArg2);
9707 compare->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED | GTF_RELOP_QMARK | GTF_DONT_CSE);
9710 return fgMorphTree(compare);
9712 else if (bOp1ClassFromHandle || bOp2ClassFromHandle)
9715 // Now check for GetClassFromHandle(handle) == obj.GetType()
9718 if (bOp1ClassFromHandle)
9720 pGetClassFromHandle = tree->gtOp.gtOp1;
9725 pGetClassFromHandle = tree->gtOp.gtOp2;
9729 GenTreePtr pGetClassFromHandleArgument = pGetClassFromHandle->gtCall.gtCallArgs->gtOp.gtOp1;
9730 GenTreePtr pConstLiteral = pGetClassFromHandleArgument;
9732 // Unwrap GT_NOP node used to prevent constant folding
9733 if (pConstLiteral->gtOper == GT_NOP && pConstLiteral->gtType == TYP_I_IMPL)
9735 pConstLiteral = pConstLiteral->gtOp.gtOp1;
9738 // In the ngen case, we have to go thru an indirection to get the right handle.
9739 if (pConstLiteral->gtOper == GT_IND)
9741 pConstLiteral = pConstLiteral->gtOp.gtOp1;
9743 #ifdef LEGACY_BACKEND
9745 if (pGetType->gtCall.gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC &&
9746 info.compCompHnd->getIntrinsicID(pGetType->gtCall.gtCallMethHnd) == CORINFO_INTRINSIC_Object_GetType &&
9748 if ((pGetType->gtOper == GT_INTRINSIC) && (pGetType->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Object_GetType) &&
9750 pConstLiteral->gtOper == GT_CNS_INT &&
9751 pConstLiteral->gtType == TYP_I_IMPL)
9753 CORINFO_CLASS_HANDLE clsHnd = CORINFO_CLASS_HANDLE(pConstLiteral->gtIntCon.gtCompileTimeHandle);
9755 if (info.compCompHnd->canInlineTypeCheckWithObjectVTable(clsHnd))
9757 // Method Table tree
9758 #ifdef LEGACY_BACKEND
9759 GenTreePtr objMT = gtNewOperNode(GT_IND, TYP_I_IMPL, pGetType->gtCall.gtCallObjp);
9761 GenTreePtr objMT = gtNewOperNode(GT_IND, TYP_I_IMPL, pGetType->gtUnOp.gtOp1);
9763 objMT->gtFlags |= GTF_EXCEPT; // Null ref exception if object is null
9764 compCurBB->bbFlags |= BBF_HAS_VTABREF;
9765 optMethodFlags |= OMF_HAS_VTABLEREF;
9767 // Method table constant
9768 GenTreePtr cnsMT = pGetClassFromHandleArgument;
9770 GenTreePtr compare = gtNewOperNode(oper, TYP_INT,
9774 compare->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED | GTF_RELOP_QMARK | GTF_DONT_CSE);
9777 return fgMorphTree(compare);
9782 fgMorphRecognizeBoxNullable(tree);
9783 op1 = tree->gtOp.gtOp1;
9784 op2 = tree->gtGetOp2();
9790 if (tree->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Round)
9792 switch (tree->TypeGet())
9795 return fgMorphIntoHelperCall(tree, CORINFO_HELP_DBLROUND, gtNewArgList(op1));
9797 return fgMorphIntoHelperCall(tree, CORINFO_HELP_FLTROUND, gtNewArgList(op1));
9809 #if !CPU_HAS_FP_SUPPORT
9810 tree = fgMorphToEmulatedFP(tree);
9813 /* Could this operator throw an exception? */
9814 if (fgGlobalMorph && tree->OperMayThrow())
9816 if ((tree->OperGet() != GT_IND) || fgAddrCouldBeNull(tree->gtOp.gtOp1))
9818 /* Mark the tree node as potentially throwing an exception */
9819 tree->gtFlags |= GTF_EXCEPT;
9823 /*-------------------------------------------------------------------------
9824 * Process the first operand, if any
9830 #if LOCAL_ASSERTION_PROP
9831 // If we are entering the "then" part of a Qmark-Colon we must
9832 // save the state of the current copy assignment table
9833 // so that we can restore this state when entering the "else" part
9836 noway_assert(optLocalAssertionProp);
9837 if (optAssertionCount)
9839 noway_assert(optAssertionCount <= optMaxAssertionCount); // else ALLOCA() is a bad idea
9840 unsigned tabSize = optAssertionCount * sizeof(AssertionDsc);
9841 origAssertionTab = (AssertionDsc*) ALLOCA(tabSize);
9842 origAssertionCount = optAssertionCount;
9843 memcpy(origAssertionTab, optAssertionTabPrivate, tabSize);
9847 origAssertionCount = 0;
9848 origAssertionTab = NULL;
9851 #endif // LOCAL_ASSERTION_PROP
9853 // We might need a new MorphAddressContext context. (These are used to convey
9854 // parent context about how addresses being calculated will be used; see the
9855 // specification comment for MorphAddrContext for full details.)
9856 // Assume it's an Ind context to start.
9857 MorphAddrContext subIndMac1(MACK_Ind);
9858 MorphAddrContext* subMac1 = mac;
9859 if (subMac1 == NULL || subMac1->m_kind == MACK_Ind || subMac1->m_kind == MACK_CopyBlock)
9861 switch (tree->gtOper)
9864 if (subMac1 == NULL)
9866 subMac1 = &subIndMac1;
9867 subMac1->m_kind = MACK_Addr;
9871 // In a comma, the incoming context only applies to the rightmost arg of the
9872 // comma list. The left arg (op1) gets a fresh context.
9877 assert(subMac1 == NULL); // Should only occur at top level, since value is void.
9878 subMac1 = &s_CopyBlockMAC;
9881 // If the list is the first arg of a copy block, its two args should be evaluated as
9882 // IND-context addresses, separately.
9883 if (subMac1 != NULL && subMac1->m_kind == MACK_CopyBlock)
9885 subMac1 = &subIndMac1;
9891 subMac1 = &subIndMac1;
9898 // For additions, if we're in an IND context keep track of whether
9899 // all offsets added to the address are constant, and their sum.
9900 if (tree->gtOper == GT_ADD && subMac1 != NULL)
9902 assert(subMac1->m_kind == MACK_Ind || subMac1->m_kind == MACK_Addr); // Can't be a CopyBlock.
9903 GenTreePtr otherOp = tree->gtOp.gtOp2;
9904 // Is the other operator a constant?
9905 if (otherOp->IsCnsIntOrI())
9907 ClrSafeInt<size_t> totalOffset(subMac1->m_totalOffset);
9908 totalOffset += otherOp->gtIntConCommon.IconValue();
9909 if (totalOffset.IsOverflow())
9911 // We will consider an offset so large as to overflow as "not a constant" --
9912 // we will do a null check.
9913 subMac1->m_allConstantOffsets = false;
9917 subMac1->m_totalOffset += otherOp->gtIntConCommon.IconValue();
9922 subMac1->m_allConstantOffsets = false;
9926 tree->gtOp.gtOp1 = op1 = fgMorphTree(op1, subMac1);
9928 #if LOCAL_ASSERTION_PROP
9929 // If we are exiting the "then" part of a Qmark-Colon we must
9930 // save the state of the current copy assignment table
9931 // so that we can merge this state with the "else" part exit
9934 noway_assert(optLocalAssertionProp);
9935 if (optAssertionCount)
9937 noway_assert(optAssertionCount <= optMaxAssertionCount); // else ALLOCA() is a bad idea
9938 unsigned tabSize = optAssertionCount * sizeof(AssertionDsc);
9939 thenAssertionTab = (AssertionDsc*) ALLOCA(tabSize);
9940 thenAssertionCount = optAssertionCount;
9941 memcpy(thenAssertionTab, optAssertionTabPrivate, tabSize);
9945 thenAssertionCount = 0;
9946 thenAssertionTab = NULL;
9949 #endif // LOCAL_ASSERTION_PROP
9951 /* Morphing along with folding and inlining may have changed the
9952 * side effect flags, so we have to reset them
9954 * NOTE: Don't reset the exception flags on nodes that may throw */
9956 noway_assert(tree->gtOper != GT_CALL);
9958 if ((tree->gtOper != GT_INTRINSIC) || !IsIntrinsicImplementedByUserCall(tree->gtIntrinsic.gtIntrinsicId))
9960 tree->gtFlags &= ~GTF_CALL;
9963 if (!tree->OperMayThrow())
9964 tree->gtFlags &= ~GTF_EXCEPT;
9966 /* Propagate the new flags */
9967 tree->gtFlags |= (op1->gtFlags & GTF_ALL_EFFECT);
9969 // &aliasedVar doesn't need GTF_GLOB_REF, though alisasedVar does
9970 // Similarly for clsVar
9971 if (oper == GT_ADDR && (op1->gtOper == GT_LCL_VAR || op1->gtOper == GT_CLS_VAR))
9972 tree->gtFlags &= ~GTF_GLOB_REF;
9975 /*-------------------------------------------------------------------------
9976 * Process the second operand, if any
9982 #if LOCAL_ASSERTION_PROP
9983 // If we are entering the "else" part of a Qmark-Colon we must
9984 // reset the state of the current copy assignment table
9987 noway_assert(optLocalAssertionProp);
9988 optAssertionReset(0);
9989 if (origAssertionCount)
9991 size_t tabSize = origAssertionCount * sizeof(AssertionDsc);
9992 memcpy(optAssertionTabPrivate, origAssertionTab, tabSize);
9993 optAssertionReset(origAssertionCount);
9996 #endif // LOCAL_ASSERTION_PROP
9998 // We might need a new MorphAddressContext context to use in evaluating op2.
9999 // (These are used to convey parent context about how addresses being calculated
10000 // will be used; see the specification comment for MorphAddrContext for full details.)
10001 // Assume it's an Ind context to start.
10002 MorphAddrContext subIndMac2(MACK_Ind);
10003 switch (tree->gtOper)
10006 if (mac != NULL && mac->m_kind == MACK_Ind)
10008 GenTreePtr otherOp = tree->gtOp.gtOp1;
10009 // Is the other operator a constant?
10010 if (otherOp->IsCnsIntOrI())
10012 mac->m_totalOffset += otherOp->gtIntConCommon.IconValue();
10016 mac->m_allConstantOffsets = false;
10021 if (mac != NULL && mac->m_kind == MACK_CopyBlock)
10029 tree->gtOp.gtOp2 = op2 = fgMorphTree(op2, mac);
10031 /* Propagate the side effect flags from op2 */
10033 tree->gtFlags |= (op2->gtFlags & GTF_ALL_EFFECT);
10035 #if LOCAL_ASSERTION_PROP
10036 // If we are exiting the "else" part of a Qmark-Colon we must
10037 // merge the state of the current copy assignment table with
10038 // that of the exit of the "then" part.
10041 noway_assert(optLocalAssertionProp);
10042 // If either exit table has zero entries then
10043 // the merged table also has zero entries
10044 if (optAssertionCount == 0 || thenAssertionCount == 0)
10046 optAssertionReset(0);
10050 size_t tabSize = optAssertionCount * sizeof(AssertionDsc);
10051 if ( (optAssertionCount != thenAssertionCount) ||
10052 (memcmp(thenAssertionTab, optAssertionTabPrivate, tabSize) != 0) )
10054 // Yes they are different so we have to find the merged set
10055 // Iterate over the copy asgn table removing any entries
10056 // that do not have an exact match in the thenAssertionTab
10057 AssertionIndex index = 1;
10058 while (index <= optAssertionCount)
10060 AssertionDsc* curAssertion = optGetAssertion(index);
10062 for (unsigned j=0; j < thenAssertionCount; j++)
10064 AssertionDsc* thenAssertion = &thenAssertionTab[j];
10066 // Do the left sides match?
10067 if ((curAssertion->op1.lcl.lclNum == thenAssertion->op1.lcl.lclNum) &&
10068 (curAssertion->assertionKind == thenAssertion->assertionKind))
10070 // Do the right sides match?
10071 if ((curAssertion->op2.kind == thenAssertion->op2.kind) &&
10072 (curAssertion->op2.lconVal == thenAssertion->op2.lconVal))
10083 // If we fall out of the loop above then we didn't find
10084 // any matching entry in the thenAssertionTab so it must
10085 // have been killed on that path so we remove it here
10088 // The data at optAssertionTabPrivate[i] is to be removed
10092 printf("The QMARK-COLON ");
10094 printf(" removes assertion candidate #%d\n", index);
10097 optAssertionRemove(index);
10100 // The data at optAssertionTabPrivate[i] is to be kept
10106 #endif // LOCAL_ASSERTION_PROP
10109 DONE_MORPHING_CHILDREN:
10111 /*-------------------------------------------------------------------------
10112 * Now do POST-ORDER processing
10115 #if FEATURE_FIXED_OUT_ARGS && !defined(_TARGET_64BIT_)
10116 // Variable shifts of a long end up being helper calls, so mark the tree as such. This
10117 // is potentially too conservative, since they'll get treated as having side effects.
10118 // It is important to mark them as calls so if they are part of an argument list,
10119 // they will get sorted and processed properly (for example, it is important to handle
10120 // all nested calls before putting struct arguments in the argument registers). We
10121 // could mark the trees just before argument processing, but it would require a full
10122 // tree walk of the argument tree, so we just do it here, instead, even though we'll
10123 // mark non-argument trees (that will still get converted to calls, anyway).
10124 if (GenTree::OperIsShift(oper) &&
10125 (tree->TypeGet() == TYP_LONG) &&
10126 (op2->OperGet() != GT_CNS_INT))
10128 tree->gtFlags |= GTF_CALL;
10130 #endif // FEATURE_FIXED_OUT_ARGS && !_TARGET_64BIT_
10132 if (varTypeIsGC(tree->TypeGet()) && (op1 && !varTypeIsGC(op1->TypeGet()))
10133 && (op2 && !varTypeIsGC(op2->TypeGet())))
10135 // The tree is really not GC but was marked as such. Now that the
10136 // children have been unmarked, unmark the tree too.
10138 // Remember that GT_COMMA inherits it's type only from op2
10139 if (tree->gtOper == GT_COMMA)
10140 tree->gtType = genActualType(op2->TypeGet());
10142 tree->gtType = genActualType(op1->TypeGet());
10145 GenTreePtr oldTree = tree;
10147 GenTreePtr qmarkOp1 = NULL;
10148 GenTreePtr qmarkOp2 = NULL;
10150 if ((tree->OperGet() == GT_QMARK) &&
10151 (tree->gtOp.gtOp2->OperGet() == GT_COLON))
10153 qmarkOp1 = oldTree->gtOp.gtOp2->gtOp.gtOp1;
10154 qmarkOp2 = oldTree->gtOp.gtOp2->gtOp.gtOp2;
10157 // Try to fold it, maybe we get lucky,
10158 tree = gtFoldExpr(tree);
10160 if (oldTree != tree)
10162 /* if gtFoldExpr returned op1 or op2 then we are done */
10163 if ((tree == op1) || (tree == op2) || (tree == qmarkOp1) || (tree == qmarkOp2))
10166 /* If we created a comma-throw tree then we need to morph op1 */
10167 if (fgIsCommaThrow(tree))
10169 tree->gtOp.gtOp1 = fgMorphTree(tree->gtOp.gtOp1);
10170 fgMorphTreeDone(tree);
10176 else if (tree->OperKind() & GTK_CONST)
10181 /* gtFoldExpr could have used setOper to change the oper */
10182 oper = tree->OperGet();
10183 typ = tree->TypeGet();
10185 /* gtFoldExpr could have changed op1 and op2 */
10186 op1 = tree->gtOp.gtOp1;
10187 op2 = tree->gtGetOp2();
10189 // Do we have an integer compare operation?
10191 if (tree->OperIsCompare() && varTypeIsIntegralOrI(tree->TypeGet()))
10193 // Are we comparing against zero?
10197 // Request that the codegen for op1 sets the condition flags
10198 // when it generates the code for op1.
10200 // Codegen for op1 must set the condition flags if
10201 // this method returns true.
10203 op1->gtRequestSetFlags();
10206 /*-------------------------------------------------------------------------
10207 * Perform the required oper-specific postorder morphing
10211 GenTreePtr cns1, cns2;
10212 GenTreePtr thenNode;
10213 GenTreePtr elseNode;
10214 size_t ival1, ival2;
10215 GenTreePtr lclVarTree;
10216 GenTreeLclVarCommon* lclVarCmnTree;
10217 FieldSeqNode* fieldSeq = NULL;
10223 lclVarTree = fgIsIndirOfAddrOfLocal(op1);
10224 if (lclVarTree != NULL)
10226 lclVarTree->gtFlags |= GTF_VAR_DEF;
10229 /* If we are storing a small type, we might be able to omit a cast */
10230 if ((op1->gtOper == GT_IND) && varTypeIsSmall(op1->TypeGet()))
10232 if (!gtIsActiveCSE_Candidate(op2) && (op2->gtOper == GT_CAST) && !op2->gtOverflow())
10234 var_types castType = op2->CastToType();
10236 // If we are performing a narrowing cast and
10237 // castType is larger or the same as op1's type
10238 // then we can discard the cast.
10240 if (varTypeIsSmall(castType) && (castType >= op1->TypeGet()))
10242 tree->gtOp.gtOp2 = op2 = op2->gtCast.CastOp();
10245 else if (op2->OperIsCompare() && varTypeIsByte(op1->TypeGet()))
10247 /* We don't need to zero extend the setcc instruction */
10248 op2->gtType = TYP_BYTE;
10251 // If we introduced a CSE we may need to undo the optimization above
10252 // (i.e. " op2->gtType = TYP_BYTE;" which depends upon op1 being a GT_IND of a byte type)
10253 // When we introduce the CSE we remove the GT_IND and subsitute a GT_LCL_VAR in it place.
10254 else if (op2->OperIsCompare() && (op2->gtType == TYP_BYTE) && (op1->gtOper == GT_LCL_VAR))
10256 unsigned varNum = op1->gtLclVarCommon.gtLclNum;
10257 LclVarDsc * varDsc = &lvaTable[varNum];
10259 /* We again need to zero extend the setcc instruction */
10260 op2->gtType = varDsc->TypeGet();
10268 fgAssignSetVarDef(tree);
10286 /* We can't CSE the LHS of an assignment */
10287 /* We also must set in the pre-morphing phase, otherwise assertionProp doesn't see it */
10288 op1->gtFlags |= GTF_DONT_CSE;
10294 /* Make sure we're allowed to do this */
10296 if (optValnumCSE_phase)
10298 // It is not safe to reorder/delete CSE's
10304 /* Check for "(expr +/- icon1) ==/!= (non-zero-icon2)" */
10306 if (cns2->gtOper == GT_CNS_INT && cns2->gtIntCon.gtIconVal != 0)
10308 op1 = tree->gtOp.gtOp1;
10310 /* Since this can occur repeatedly we use a while loop */
10312 while ((op1->gtOper == GT_ADD || op1->gtOper == GT_SUB) &&
10313 (op1->gtOp.gtOp2->gtOper == GT_CNS_INT) &&
10314 (op1->gtType == TYP_INT) &&
10315 (op1->gtOverflow() == false))
10317 /* Got it; change "x+icon1==icon2" to "x==icon2-icon1" */
10319 ival1 = op1->gtOp.gtOp2->gtIntCon.gtIconVal;
10320 ival2 = cns2->gtIntCon.gtIconVal;
10322 if (op1->gtOper == GT_ADD)
10330 cns2->gtIntCon.gtIconVal = ival2;
10332 #ifdef _TARGET_64BIT_
10333 // we need to properly re-sign-extend or truncate as needed.
10334 cns2->AsIntCon()->TruncateOrSignExtend32();
10335 #endif // _TARGET_64BIT_
10337 op1 = tree->gtOp.gtOp1 = op1->gtOp.gtOp1;
10342 // Here we look for the following tree
10348 ival2 = INT_MAX; // The value of INT_MAX for ival2 just means that the constant value is not 0 or 1
10350 // cast to unsigned allows test for both 0 and 1
10351 if ((cns2->gtOper == GT_CNS_INT) && (((size_t) cns2->gtIntConCommon.IconValue()) <= 1U))
10353 ival2 = (size_t) cns2->gtIntConCommon.IconValue();
10355 else // cast to UINT64 allows test for both 0 and 1
10356 if ((cns2->gtOper == GT_CNS_LNG) && (((UINT64) cns2->gtIntConCommon.LngValue()) <= 1ULL))
10358 ival2 = (size_t) cns2->gtIntConCommon.LngValue();
10361 if (ival2 != INT_MAX)
10363 // If we don't have a comma and relop, we can't do this optimization
10365 if ((op1->gtOper == GT_COMMA) && (op1->gtOp.gtOp2->OperIsCompare()))
10367 // Here we look for the following transformation
10369 // EQ/NE Possible REVERSE(RELOP)
10371 // COMMA CNS 0/1 -> COMMA relop_op2
10373 // x RELOP x relop_op1
10375 // relop_op1 relop_op2
10379 GenTreePtr comma = op1;
10380 GenTreePtr relop = comma->gtOp.gtOp2;
10382 GenTreePtr relop_op1 = relop->gtOp.gtOp1;
10384 bool reverse = ((ival2 == 0) == (oper == GT_EQ));
10388 gtReverseCond(relop);
10391 relop->gtOp.gtOp1 = comma;
10392 comma->gtOp.gtOp2 = relop_op1;
10394 // Comma now has fewer nodes underneath it, so we need to regenerate its flags
10395 comma->gtFlags &= ~GTF_ALL_EFFECT;
10396 comma->gtFlags |= (comma->gtOp.gtOp1->gtFlags) & GTF_ALL_EFFECT;
10397 comma->gtFlags |= (comma->gtOp.gtOp2->gtFlags) & GTF_ALL_EFFECT;
10399 noway_assert((relop->gtFlags & GTF_RELOP_JMP_USED) == 0);
10400 noway_assert((relop->gtFlags & GTF_REVERSE_OPS) == 0);
10401 relop->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED|GTF_RELOP_QMARK|GTF_DONT_CSE|GTF_ALL_EFFECT);
10406 if (op1->gtOper == GT_COMMA)
10408 // Here we look for the following tree
10409 // and when the LCL_VAR is a temp we can fold the tree:
10413 // COMMA CNS 0/1 -> RELOP CNS 0/1
10421 GenTreePtr asg = op1->gtOp.gtOp1;
10422 GenTreePtr lcl = op1->gtOp.gtOp2;
10424 /* Make sure that the left side of the comma is the assignment of the LCL_VAR */
10425 if (asg->gtOper != GT_ASG)
10428 /* The right side of the comma must be a LCL_VAR temp */
10429 if (lcl->gtOper != GT_LCL_VAR)
10432 unsigned lclNum = lcl->gtLclVarCommon.gtLclNum; noway_assert(lclNum < lvaCount);
10434 /* If the LCL_VAR is not a temp then bail, a temp has a single def */
10435 if (!lvaTable[lclNum].lvIsTemp)
10439 /* If the LCL_VAR is a CSE temp then bail, it could have multiple defs/uses */
10440 // Fix 383856 X86/ARM ILGEN
10441 if (lclNumIsCSE(lclNum))
10445 /* We also must be assigning the result of a RELOP */
10446 if (asg->gtOp.gtOp1->gtOper != GT_LCL_VAR)
10449 /* Both of the LCL_VAR must match */
10450 if (asg->gtOp.gtOp1->gtLclVarCommon.gtLclNum != lclNum)
10453 /* If right side of asg is not a RELOP then skip */
10454 if (!asg->gtOp.gtOp2->OperIsCompare())
10457 LclVarDsc * varDsc = lvaTable + lclNum;
10459 /* Set op1 to the right side of asg, (i.e. the RELOP) */
10460 op1 = asg->gtOp.gtOp2;
10462 DEBUG_DESTROY_NODE(asg->gtOp.gtOp1);
10463 DEBUG_DESTROY_NODE(lcl);
10465 /* This local variable should never be used again */
10467 // VSW 184221: Make RefCnt to zero to indicate that this local var
10468 // is not used any more. (Keey the lvType as is.)
10469 // Otherwise lvOnFrame will be set to true in Compiler::raMarkStkVars
10470 // And then emitter::emitEndCodeGen will assert in the following line:
10471 // noway_assert( dsc->lvTracked);
10473 noway_assert(varDsc->lvRefCnt == 0 || // lvRefCnt may not have been set yet.
10474 varDsc->lvRefCnt == 2 // Or, we assume this tmp should only be used here,
10475 // and it only shows up twice.
10477 lvaTable[lclNum].lvRefCnt = 0;
10478 lvaTable[lclNum].lvaResetSortAgainFlag(this);
10482 if (op1->OperIsCompare())
10484 // Here we look for the following tree
10486 // EQ/NE -> RELOP/!RELOP
10491 // Note that we will remove/destroy the EQ/NE node and move
10492 // the RELOP up into it's location.
10494 /* Here we reverse the RELOP if necessary */
10496 bool reverse = ((ival2 == 0) == (oper == GT_EQ));
10500 gtReverseCond(op1);
10503 /* Propagate gtType of tree into op1 in case it is TYP_BYTE for setcc optimization */
10504 op1->gtType = tree->gtType;
10506 noway_assert((op1->gtFlags & GTF_RELOP_JMP_USED) == 0);
10507 op1->gtFlags |= tree->gtFlags & (GTF_RELOP_JMP_USED|GTF_RELOP_QMARK|GTF_DONT_CSE);
10509 DEBUG_DESTROY_NODE(tree);
10515 // Now we check for a compare with the result of an '&' operator
10517 // Here we look for the following transformation:
10521 // AND CNS 0/1 -> AND CNS 0
10523 // RSZ/RSH CNS 1 x CNS (1 << y)
10527 if (op1->gtOper == GT_AND)
10529 GenTreePtr andOp = op1;
10530 GenTreePtr rshiftOp = andOp->gtOp.gtOp1;
10532 if ((rshiftOp->gtOper != GT_RSZ) && (rshiftOp->gtOper != GT_RSH))
10535 if (!rshiftOp->gtOp.gtOp2->IsCnsIntOrI())
10538 ssize_t shiftAmount = rshiftOp->gtOp.gtOp2->gtIntCon.gtIconVal;
10540 if (shiftAmount < 0)
10543 if (andOp->gtType == TYP_INT)
10545 if (!andOp->gtOp.gtOp2->IsCnsIntOrI())
10548 if (andOp->gtOp.gtOp2->gtIntCon.gtIconVal != 1)
10551 if (shiftAmount > 31)
10554 UINT32 newAndOperand = ((UINT32) 1) << shiftAmount;
10556 andOp->gtOp.gtOp2->gtIntCon.gtIconVal = newAndOperand;
10558 // Reverse the cond if necessary
10561 gtReverseCond(tree);
10562 cns2->gtIntCon.gtIconVal = 0;
10563 oper = tree->gtOper;
10567 else if (andOp->gtType == TYP_LONG)
10569 if (andOp->gtOp.gtOp2->gtOper != GT_CNS_NATIVELONG)
10572 if (andOp->gtOp.gtOp2->gtIntConCommon.LngValue() != 1)
10575 if (shiftAmount > 63)
10578 UINT64 newAndOperand = ((UINT64) 1) << shiftAmount;
10580 andOp->gtOp.gtOp2->gtIntConCommon.SetLngValue(newAndOperand);
10582 // Reverse the cond if necessary
10585 gtReverseCond(tree);
10586 cns2->gtIntConCommon.SetLngValue(0);
10587 oper = tree->gtOper;
10591 andOp->gtOp.gtOp1 = rshiftOp->gtOp.gtOp1;
10593 DEBUG_DESTROY_NODE(rshiftOp->gtOp.gtOp2);
10594 DEBUG_DESTROY_NODE(rshiftOp);
10596 } // END if (ival2 != INT_MAX)
10599 /* Now check for compares with small constant longs that can be cast to int */
10601 if (!cns2->OperIsConst())
10604 if (cns2->TypeGet() != TYP_LONG)
10607 /* Is the constant 31 bits or smaller? */
10609 if ((cns2->gtIntConCommon.LngValue() >> 31) != 0)
10612 /* Is the first comparand mask operation of type long ? */
10614 if (op1->gtOper != GT_AND)
10616 /* Another interesting case: cast from int */
10618 if (op1->gtOper == GT_CAST &&
10619 op1->CastFromType() == TYP_INT &&
10620 !gtIsActiveCSE_Candidate(op1) && // op1 cannot be a CSE candidate
10621 !op1->gtOverflow()) // cannot be an overflow checking cast
10623 /* Simply make this into an integer comparison */
10625 tree->gtOp.gtOp1 = op1->gtCast.CastOp();
10626 tree->gtOp.gtOp2 = gtNewIconNode((int)cns2->gtIntConCommon.LngValue(), TYP_INT);
10632 noway_assert(op1->TypeGet() == TYP_LONG && op1->OperGet() == GT_AND);
10634 /* Is the result of the mask effectively an INT ? */
10636 GenTreePtr andMask; andMask = op1->gtOp.gtOp2;
10637 if (andMask->gtOper != GT_CNS_NATIVELONG)
10639 if ((andMask->gtIntConCommon.LngValue() >> 32) != 0)
10642 /* Now we know that we can cast gtOp.gtOp1 of AND to int */
10644 op1->gtOp.gtOp1 = gtNewCastNode(TYP_INT,
10648 /* now replace the mask node (gtOp.gtOp2 of AND node) */
10650 noway_assert(andMask == op1->gtOp.gtOp2);
10652 ival1 = (int) andMask->gtIntConCommon.LngValue();
10653 andMask->SetOper(GT_CNS_INT);
10654 andMask->gtType = TYP_INT;
10655 andMask->gtIntCon.gtIconVal = ival1;
10657 /* now change the type of the AND node */
10659 op1->gtType = TYP_INT;
10661 /* finally we replace the comparand */
10663 ival2 = (int) cns2->gtIntConCommon.LngValue();
10664 cns2->SetOper(GT_CNS_INT);
10665 cns2->gtType = TYP_INT;
10667 noway_assert(cns2 == op2);
10668 cns2->gtIntCon.gtIconVal = ival2;
10677 if ((tree->gtFlags & GTF_UNSIGNED) == 0)
10679 if (op2->gtOper == GT_CNS_INT)
10682 /* Check for "expr relop 1" */
10683 if (cns2->gtIntCon.gtIconVal == +1)
10685 /* Check for "expr >= 1" */
10688 /* Change to "expr > 0" */
10692 /* Check for "expr < 1" */
10693 else if (oper == GT_LT)
10695 /* Change to "expr <= 0" */
10700 /* Check for "expr relop -1" */
10701 else if ((cns2->gtIntCon.gtIconVal == -1) && ((oper == GT_LE) || (oper == GT_GT)))
10703 /* Check for "expr <= -1" */
10706 /* Change to "expr < 0" */
10710 /* Check for "expr > -1" */
10711 else if (oper == GT_GT)
10713 /* Change to "expr >= 0" */
10717 // IF we get here we should be changing 'oper'
10718 assert(tree->OperGet() != oper);
10720 // Keep the old ValueNumber for 'tree' as the new expr
10721 // will still compute the same value as before
10722 tree->SetOper(oper, GenTree::PRESERVE_VN);
10723 cns2->gtIntCon.gtIconVal = 0;
10725 // vnStore is null before the ValueNumber phase has run
10726 if (vnStore != nullptr)
10728 // Update the ValueNumber for 'cns2', as we just changed it to 0
10729 fgValueNumberTreeConst(cns2);
10732 op2 = tree->gtOp.gtOp2 = gtFoldExpr(op2);
10740 noway_assert(tree->OperKind() & GTK_RELOP);
10742 /* Check if the result of the comparison is used for a jump.
10743 * If not then only the int (i.e. 32 bit) case is handled in
10744 * the code generator through the (x86) "set" instructions.
10745 * For the rest of the cases, the simplest way is to
10746 * "simulate" the comparison with ?:
10748 * On ARM, we previously used the IT instruction, but the IT instructions
10749 * have mostly been declared obsolete and off-limits, so all cases on ARM
10750 * get converted to ?: */
10752 if (!(tree->gtFlags & GTF_RELOP_JMP_USED) &&
10753 fgMorphRelopToQmark(op1))
10755 /* We convert it to "(CMP_TRUE) ? (1):(0)" */
10758 op1->gtFlags |= (GTF_RELOP_JMP_USED | GTF_RELOP_QMARK | GTF_DONT_CSE);
10759 op1->gtRequestSetFlags();
10761 op2 = new (this, GT_COLON) GenTreeColon(TYP_INT, gtNewIconNode(1), gtNewIconNode(0)
10763 op2 = fgMorphTree(op2);
10765 tree = gtNewQmarkNode(TYP_INT, op1, op2);
10767 fgMorphTreeDone(tree);
10775 /* If op1 is a comma throw node then we won't be keeping op2 */
10776 if (fgIsCommaThrow(op1))
10779 /* Get hold of the two branches */
10781 noway_assert(op2->OperGet() == GT_COLON);
10782 elseNode = op2->AsColon()->ElseNode();
10783 thenNode = op2->AsColon()->ThenNode();
10785 /* Try to hoist assignments out of qmark colon constructs.
10786 ie. replace (cond?(x=a):(x=b)) with (x=(cond?a:b)). */
10788 if (tree->TypeGet() == TYP_VOID &&
10789 thenNode->OperGet() == GT_ASG &&
10790 elseNode->OperGet() == GT_ASG &&
10791 thenNode->TypeGet() != TYP_LONG &&
10792 GenTree::Compare(thenNode->gtOp.gtOp1, elseNode->gtOp.gtOp1) &&
10793 thenNode->gtOp.gtOp2->TypeGet() == elseNode->gtOp.gtOp2->TypeGet())
10795 noway_assert(thenNode->TypeGet() == elseNode->TypeGet());
10797 GenTreePtr asg = thenNode;
10798 GenTreePtr colon = op2;
10799 colon->gtOp.gtOp1 = thenNode->gtOp.gtOp2;
10800 colon->gtOp.gtOp2 = elseNode->gtOp.gtOp2;
10801 tree->gtType = colon->gtType = asg->gtOp.gtOp2->gtType;
10802 asg->gtOp.gtOp2 = tree;
10804 // Asg will have all the flags that the QMARK had
10805 asg->gtFlags |= (tree->gtFlags & GTF_ALL_EFFECT);
10807 // Colon flag won't have the flags that x had.
10808 colon->gtFlags &= ~GTF_ALL_EFFECT;
10809 colon->gtFlags |= (colon->gtOp.gtOp1->gtFlags |
10810 colon->gtOp.gtOp2->gtFlags) & GTF_ALL_EFFECT;
10812 DEBUG_DESTROY_NODE(elseNode->gtOp.gtOp1);
10813 DEBUG_DESTROY_NODE(elseNode);
10819 /* If the 'else' branch is empty swap the two branches and reverse the condition */
10821 if (elseNode->IsNothingNode())
10823 /* This can only happen for VOID ?: */
10824 noway_assert(op2->gtType == TYP_VOID);
10826 /* If the thenNode and elseNode are both nop nodes then optimize away the QMARK */
10827 if (thenNode->IsNothingNode())
10829 // We may be able to throw away op1 (unless it has side-effects)
10831 if ((op1->gtFlags & GTF_SIDE_EFFECT) == 0)
10833 /* Just return a a Nop Node */
10838 /* Just return the relop, but clear the special flags. Note
10839 that we can't do that for longs and floats (see code under
10840 COMPARE label above) */
10842 if (!fgMorphRelopToQmark(op1->gtOp.gtOp1))
10844 op1->gtFlags &= ~(GTF_RELOP_QMARK | GTF_RELOP_JMP_USED);
10851 GenTreePtr tmp = elseNode;
10853 op2->AsColon()->ElseNode() = elseNode = thenNode;
10854 op2->AsColon()->ThenNode() = thenNode = tmp;
10855 gtReverseCond(op1);
10859 #if !defined(_TARGET_ARM_)
10860 // If we have (cond)?0:1, then we just return "cond" for TYP_INTs
10862 // Don't do this optimization for ARM: we always require assignment
10863 // to boolean to remain ?:, since we don't have any way to generate
10864 // this with straight-line code, like x86 does using setcc (at least
10865 // after the IT instruction is deprecated).
10867 if (genActualType(op1->gtOp.gtOp1->gtType) == TYP_INT &&
10868 genActualType(typ) == TYP_INT &&
10869 thenNode->gtOper == GT_CNS_INT &&
10870 elseNode->gtOper == GT_CNS_INT)
10872 ival1 = thenNode->gtIntCon.gtIconVal;
10873 ival2 = elseNode->gtIntCon.gtIconVal;
10875 // Is one constant 0 and the other 1?
10876 if ((ival1 | ival2) == 1 && (ival1 & ival2) == 0)
10878 // If the constants are {1, 0}, reverse the condition
10880 gtReverseCond(op1);
10882 // Unmark GTF_RELOP_JMP_USED on the condition node so it knows that it
10883 // needs to materialize the result as a 0 or 1.
10884 noway_assert(op1->gtFlags & (GTF_RELOP_QMARK | GTF_RELOP_JMP_USED));
10885 op1->gtFlags &= ~(GTF_RELOP_QMARK | GTF_RELOP_JMP_USED);
10887 DEBUG_DESTROY_NODE(tree);
10888 DEBUG_DESTROY_NODE(op2);
10893 #endif // !_TARGET_ARM_
10895 break; // end case GT_QMARK
10900 #ifndef _TARGET_64BIT_
10901 #if!LONG_MATH_REGPARAM
10902 if (typ == TYP_LONG)
10904 // This must be GTF_MUL_64RSLT
10905 assert(tree->gtIsValid64RsltMul());
10909 #endif // _TARGET_64BIT_
10914 if (tree->gtOverflow())
10917 /* Check for "op1 - cns2" , we change it to "op1 + (-cns2)" */
10920 if (op2->IsCnsIntOrI())
10922 /* Negate the constant and change the node to be "+" */
10924 op2->gtIntConCommon.SetIconValue(-op2->gtIntConCommon.IconValue());
10926 tree->ChangeOper(oper);
10930 /* Check for "cns1 - op2" , we change it to "(cns1 + (-op2))" */
10933 if (op1->IsCnsIntOrI())
10935 noway_assert(varTypeIsIntOrI(tree));
10937 tree->gtOp.gtOp2 = op2 = gtNewOperNode(GT_NEG, tree->gtType, op2); // The type of the new GT_NEG node should be the same
10938 // as the type of the tree, i.e. tree->gtType.
10939 fgMorphTreeDone(op2);
10942 tree->ChangeOper(oper);
10946 /* No match - exit */
10950 #ifdef _TARGET_ARM64_
10952 if (!varTypeIsFloating(tree->gtType))
10954 // Codegen for this instruction needs to be able to throw two exceptions:
10955 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_OVERFLOW, fgPtrArgCntCur);
10956 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_DIV_BY_ZERO, fgPtrArgCntCur);
10960 // Codegen for this instruction needs to be able to throw one exception:
10961 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_DIV_BY_ZERO, fgPtrArgCntCur);
10968 if (tree->gtOverflow())
10970 tree->gtRequestSetFlags();
10972 // Add the excptn-throwing basic block to jump to on overflow
10974 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_OVERFLOW, fgPtrArgCntCur);
10976 // We can't do any commutative morphing for overflow instructions
10987 /* Commute any non-REF constants to the right */
10990 if (op1->OperIsConst() && (op1->gtType != TYP_REF))
10992 // TODO-Review: We used to assert here that
10993 // noway_assert(!op2->OperIsConst() || !opts.OptEnabled(CLFLG_CONSTANTFOLD));
10994 // With modifications to AddrTaken==>AddrExposed, we did more assertion propagation,
10995 // and would sometimes hit this assertion. This may indicate a missed "remorph".
10996 // Task is to re-enable this assertion and investigate.
10998 /* Swap the operands */
10999 tree->gtOp.gtOp1 = op2;
11000 tree->gtOp.gtOp2 = op1;
11003 op2 = tree->gtOp.gtOp2;
11006 /* See if we can fold GT_ADD nodes. */
11008 if (oper == GT_ADD)
11010 /* Fold "((x+icon1)+(y+icon2)) to ((x+y)+(icon1+icon2))" */
11012 if (op1->gtOper == GT_ADD &&
11013 op2->gtOper == GT_ADD &&
11014 !gtIsActiveCSE_Candidate(op2) &&
11015 op1->gtOp.gtOp2->gtOper == GT_CNS_INT &&
11016 op2->gtOp.gtOp2->gtOper == GT_CNS_INT &&
11017 !op1->gtOverflow() &&
11018 !op2->gtOverflow() )
11020 cns1 = op1->gtOp.gtOp2;
11021 cns2 = op2->gtOp.gtOp2;
11022 cns1->gtIntCon.gtIconVal += cns2->gtIntCon.gtIconVal;
11023 #ifdef _TARGET_64BIT_
11024 if (cns1->TypeGet() == TYP_INT)
11026 // we need to properly re-sign-extend or truncate after adding two int constants above
11027 cns1->AsIntCon()->TruncateOrSignExtend32();
11029 #endif //_TARGET_64BIT_
11031 tree->gtOp.gtOp2 = cns1;
11032 DEBUG_DESTROY_NODE(cns2);
11034 op1->gtOp.gtOp2 = op2->gtOp.gtOp1;
11035 op1->gtFlags |= (op1->gtOp.gtOp2->gtFlags & GTF_ALL_EFFECT);
11036 DEBUG_DESTROY_NODE(op2);
11037 op2 = tree->gtOp.gtOp2;
11040 if (op2->IsCnsIntOrI() && varTypeIsIntegralOrI(typ))
11042 /* Fold "((x+icon1)+icon2) to (x+(icon1+icon2))" */
11044 if (op1->gtOper == GT_ADD &&
11045 !gtIsActiveCSE_Candidate(op1) &&
11046 op1->gtOp.gtOp2->IsCnsIntOrI() &&
11047 !op1->gtOverflow() &&
11048 op1->gtOp.gtOp2->OperGet() == op2->OperGet())
11050 cns1 = op1->gtOp.gtOp2;
11051 op2->gtIntConCommon.SetIconValue(cns1->gtIntConCommon.IconValue() + op2->gtIntConCommon.IconValue());
11052 #ifdef _TARGET_64BIT_
11053 if (op2->TypeGet() == TYP_INT)
11055 // we need to properly re-sign-extend or truncate after adding two int constants above
11056 op2->AsIntCon()->TruncateOrSignExtend32();
11058 #endif //_TARGET_64BIT_
11060 if (cns1->OperGet() == GT_CNS_INT)
11062 op2->gtIntCon.gtFieldSeq =
11063 GetFieldSeqStore()->Append(cns1->gtIntCon.gtFieldSeq,
11064 op2->gtIntCon.gtFieldSeq);
11066 DEBUG_DESTROY_NODE(cns1);
11068 tree->gtOp.gtOp1 = op1->gtOp.gtOp1;
11069 DEBUG_DESTROY_NODE(op1);
11070 op1 = tree->gtOp.gtOp1;
11075 if ((op2->gtIntConCommon.IconValue() == 0) && !gtIsActiveCSE_Candidate(tree))
11078 // If this addition is adding an offset to a null pointer,
11079 // avoid the work and yield the null pointer immediately.
11080 // Dereferencing the pointer in either case will have the
11083 if (!gtIsActiveCSE_Candidate(op1) && varTypeIsGC(op2->TypeGet()))
11085 op2->gtType = tree->gtType;
11086 DEBUG_DESTROY_NODE(op1);
11087 DEBUG_DESTROY_NODE(tree);
11091 // Remove the addition iff it won't change the tree type
11094 if (!gtIsActiveCSE_Candidate(op2) &&
11095 ((op1->TypeGet() == tree->TypeGet()) ||
11096 (op1->TypeGet() != TYP_REF)))
11098 if (fgGlobalMorph &&
11099 (op2->OperGet() == GT_CNS_INT) &&
11100 (op2->gtIntCon.gtFieldSeq != NULL) &&
11101 (op2->gtIntCon.gtFieldSeq != FieldSeqStore::NotAField()))
11103 fgAddFieldSeqForZeroOffset(op1, op2->gtIntCon.gtFieldSeq);
11106 DEBUG_DESTROY_NODE(op2);
11107 DEBUG_DESTROY_NODE(tree);
11114 /* See if we can fold GT_MUL by const nodes */
11115 else if (oper == GT_MUL && op2->IsCnsIntOrI() && !optValnumCSE_phase)
11117 #ifndef _TARGET_64BIT_
11118 noway_assert(typ <= TYP_UINT);
11119 #endif // _TARGET_64BIT_
11120 noway_assert(!tree->gtOverflow());
11122 ssize_t mult = op2->gtIntConCommon.IconValue();
11123 bool op2IsConstIndex = op2->OperGet() == GT_CNS_INT &&
11124 op2->gtIntCon.gtFieldSeq != nullptr &&
11125 op2->gtIntCon.gtFieldSeq->IsConstantIndexFieldSeq();
11127 assert(!op2IsConstIndex || op2->AsIntCon()->gtFieldSeq->m_next == nullptr);
11131 // We may be able to throw away op1 (unless it has side-effects)
11133 if ((op1->gtFlags & GTF_SIDE_EFFECT) == 0)
11135 DEBUG_DESTROY_NODE(op1);
11136 DEBUG_DESTROY_NODE(tree);
11137 return op2; // Just return the "0" node
11140 // We need to keep op1 for the side-effects. Hang it off
11143 tree->ChangeOper(GT_COMMA);
11147 size_t abs_mult = (mult >= 0) ? mult : -mult;
11148 size_t lowestBit = genFindLowestBit(abs_mult);
11149 bool changeToShift = false;
11151 // is it a power of two? (positive or negative)
11152 if (abs_mult == lowestBit)
11154 // if negative negate (min-int does not need negation)
11155 if (mult < 0 && mult != SSIZE_T_MIN)
11157 tree->gtOp.gtOp1 = op1 = gtNewOperNode(GT_NEG, op1->gtType, op1);
11158 fgMorphTreeDone(op1);
11161 // If "op2" is a constant array index, the other multiplicand must be a constant.
11162 // Transfer the annotation to the other one.
11163 if (op2->OperGet() == GT_CNS_INT &&
11164 op2->gtIntCon.gtFieldSeq != nullptr &&
11165 op2->gtIntCon.gtFieldSeq->IsConstantIndexFieldSeq())
11167 assert(op2->gtIntCon.gtFieldSeq->m_next == nullptr);
11168 GenTreePtr otherOp = op1;
11169 if (otherOp->OperGet() == GT_NEG)
11170 otherOp = otherOp->gtOp.gtOp1;
11171 assert(otherOp->OperGet() == GT_CNS_INT);
11172 assert(otherOp->gtIntCon.gtFieldSeq == FieldSeqStore::NotAField());
11173 otherOp->gtIntCon.gtFieldSeq = op2->gtIntCon.gtFieldSeq;
11178 DEBUG_DESTROY_NODE(op2);
11179 DEBUG_DESTROY_NODE(tree);
11183 /* Change the multiplication into a shift by log2(val) bits */
11184 op2->gtIntConCommon.SetIconValue(genLog2(abs_mult));
11185 changeToShift = true;
11188 else if ((lowestBit > 1) && jitIsScaleIndexMul(lowestBit) && optAvoidIntMult())
11190 int shift = genLog2(lowestBit);
11191 ssize_t factor = abs_mult >> shift;
11193 if (factor == 3 || factor == 5 || factor == 9)
11195 // if negative negate (min-int does not need negation)
11196 if (mult < 0 && mult != SSIZE_T_MIN)
11198 tree->gtOp.gtOp1 = op1 = gtNewOperNode(GT_NEG, op1->gtType, op1);
11199 fgMorphTreeDone(op1);
11202 GenTreePtr factorIcon = gtNewIconNode(factor, TYP_I_IMPL);
11203 if (op2IsConstIndex)
11205 factorIcon->AsIntCon()->gtFieldSeq = GetFieldSeqStore()->CreateSingleton(FieldSeqStore::ConstantIndexPseudoField);
11208 // change the multiplication into a smaller multiplication (by 3, 5 or 9) and a shift
11209 tree->gtOp.gtOp1 = op1 = gtNewOperNode(GT_MUL, tree->gtType, op1, factorIcon);
11210 fgMorphTreeDone(op1);
11212 op2->gtIntConCommon.SetIconValue(shift);
11213 changeToShift = true;
11216 #endif // LEA_AVAILABLE
11219 // vnStore is null before the ValueNumber phase has run
11220 if (vnStore != nullptr)
11222 // Update the ValueNumber for 'op2', as we just changed the constant
11223 fgValueNumberTreeConst(op2);
11226 // Keep the old ValueNumber for 'tree' as the new expr
11227 // will still compute the same value as before
11228 tree->ChangeOper(oper, GenTree::PRESERVE_VN);
11230 goto DONE_MORPHING_CHILDREN;
11233 else if (fgOperIsBitwiseRotationRoot(oper))
11235 tree = fgRecognizeAndMorphBitwiseRotation(tree);
11237 // fgRecognizeAndMorphBitwiseRotation may return a new tree
11238 oper = tree->OperGet();
11239 typ = tree->TypeGet();
11240 op1 = tree->gtOp.gtOp1;
11241 op2 = tree->gtOp.gtOp2;
11250 /* Any constant cases should have been folded earlier */
11251 noway_assert(!op1->OperIsConst() || !opts.OptEnabled(CLFLG_CONSTANTFOLD) || optValnumCSE_phase);
11256 noway_assert(varTypeIsFloating(op1->TypeGet()));
11258 fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_ARITH_EXCPN, fgPtrArgCntCur);
11263 // Can not remove a GT_IND if it is currently a CSE candidate.
11264 if (gtIsActiveCSE_Candidate(tree))
11267 bool foldAndReturnTemp; foldAndReturnTemp = false;
11271 /* Try to Fold *(&X) into X */
11272 if (op1->gtOper == GT_ADDR)
11274 // Can not remove a GT_ADDR if it is currently a CSE candidate.
11275 if (gtIsActiveCSE_Candidate(op1))
11278 temp = op1->gtOp.gtOp1; // X
11280 // In the test below, if they're both TYP_STRUCT, this of course does *not* mean that
11281 // they are the *same* struct type. In fact, they almost certainly aren't. If the
11282 // address has an associated field sequence, that identifies this case; go through
11283 // the "lcl_fld" path rather than this one.
11284 FieldSeqNode* addrFieldSeq = NULL; // This is an unused out parameter below.
11285 if ( typ == temp->TypeGet()
11286 && !GetZeroOffsetFieldMap()->Lookup(op1, &addrFieldSeq))
11288 foldAndReturnTemp = true;
11290 else if (temp->OperIsLocal())
11292 unsigned lclNum = temp->gtLclVarCommon.gtLclNum;
11293 LclVarDsc * varDsc = &lvaTable[lclNum];
11295 // We will try to optimize when we have a promoted struct promoted with a zero lvFldOffset
11296 if (varDsc->lvPromoted && (varDsc->lvFldOffset == 0))
11298 noway_assert(varTypeIsStruct(varDsc));
11300 // We will try to optimize when we have a single field struct that is being struct promoted
11301 if (varDsc->lvFieldCnt == 1)
11303 unsigned lclNumFld = varDsc->lvFieldLclStart;
11304 // just grab the promoted field
11305 LclVarDsc * fieldVarDsc = &lvaTable[lclNumFld];
11307 // Also make sure that the tree type matches the fieldVarType and that it's lvFldOffset is zero
11308 if (fieldVarDsc->TypeGet() == tree->TypeGet() && (fieldVarDsc->lvFldOffset == 0))
11310 // We can just use the existing promoted field LclNum
11311 temp->gtLclVarCommon.SetLclNum(lclNumFld);
11312 temp->gtType = fieldVarDsc->TypeGet();
11314 foldAndReturnTemp = true;
11318 // If the type of the IND (typ) is a "small int", and the type of the local has the
11319 // same width, then we can reduce to just the local variable -- it will be
11320 // correctly normalized, and signed/unsigned differences won't matter.
11322 // The below transformation cannot be applied if the local var needs to be normalized on load.
11323 else if ( varTypeIsSmall(typ) &&
11324 (genTypeSize(lvaTable[lclNum].lvType) == genTypeSize(typ)) &&
11325 !lvaTable[lclNum].lvNormalizeOnLoad() )
11327 tree->gtType = temp->gtType;
11328 foldAndReturnTemp = true;
11332 // Assumes that when Lookup returns "false" it will leave "fieldSeq" unmodified (i.e. nullptr)
11333 assert(fieldSeq == nullptr);
11334 bool b = GetZeroOffsetFieldMap()->Lookup(op1, &fieldSeq);
11335 assert(b || fieldSeq == nullptr);
11337 if ((fieldSeq != nullptr) && (temp->OperGet() == GT_LCL_FLD))
11339 // Append the field sequence, change the type.
11340 temp->AsLclFld()->gtFieldSeq = GetFieldSeqStore()->Append(temp->AsLclFld()->gtFieldSeq, fieldSeq);
11341 temp->gtType = tree->TypeGet();
11343 foldAndReturnTemp = true;
11346 // Otherwise will will fold this into a GT_LCL_FLD below
11347 // where we check (temp != nullptr)
11349 else // !temp->OperIsLocal()
11351 // We don't try to fold away the GT_IND/GT_ADDR for this case
11355 else if (op1->OperGet() == GT_ADD)
11357 /* Try to change *(&lcl + cns) into lcl[cns] to prevent materialization of &lcl */
11359 if (op1->gtOp.gtOp1->OperGet() == GT_ADDR &&
11360 op1->gtOp.gtOp2->OperGet() == GT_CNS_INT
11361 && (!(opts.MinOpts() || opts.compDbgCode)))
11363 // No overflow arithmetic with pointers
11364 noway_assert(!op1->gtOverflow());
11366 temp = op1->gtOp.gtOp1->gtOp.gtOp1;
11367 if (!temp->OperIsLocal())
11373 // Can not remove the GT_ADDR if it is currently a CSE candidate.
11374 if (gtIsActiveCSE_Candidate(op1->gtOp.gtOp1))
11377 ival1 = op1->gtOp.gtOp2->gtIntCon.gtIconVal;
11378 fieldSeq = op1->gtOp.gtOp2->gtIntCon.gtFieldSeq;
11380 // Does the address have an associated zero-offset field sequence?
11381 FieldSeqNode* addrFieldSeq = NULL;
11382 if (GetZeroOffsetFieldMap()->Lookup(op1->gtOp.gtOp1, &addrFieldSeq))
11384 fieldSeq = GetFieldSeqStore()->Append(addrFieldSeq, fieldSeq);
11388 typ == temp->TypeGet() &&
11389 temp->TypeGet() != TYP_STRUCT)
11391 noway_assert(!varTypeIsGC(temp->TypeGet()));
11392 foldAndReturnTemp = true;
11396 // The emitter can't handle large offsets
11397 if (ival1 != (unsigned short)ival1)
11400 // The emitter can get confused by invalid offsets
11401 if (ival1 >= Compiler::lvaLclSize(temp->gtLclVarCommon.gtLclNum))
11404 #ifdef _TARGET_ARM_
11405 // Check for a LclVar TYP_STRUCT with misalignment on a Floating Point field
11407 if (varTypeIsFloating(tree->TypeGet()))
11409 if ((ival1 % emitTypeSize(tree->TypeGet())) != 0)
11411 tree->gtFlags |= GTF_IND_UNALIGNED;
11417 // Now we can fold this into a GT_LCL_FLD below
11418 // where we check (temp != nullptr)
11423 // If we have decided to fold, then temp cannot be nullptr
11424 if (foldAndReturnTemp)
11426 assert(temp != nullptr);
11430 if (temp != nullptr)
11432 noway_assert(op1->gtOper == GT_ADD || op1->gtOper == GT_ADDR);
11434 // If we haven't already decided to fold this expression
11436 if (!foldAndReturnTemp)
11438 noway_assert(temp->OperIsLocal());
11439 LclVarDsc* varDsc = &(lvaTable[temp->AsLclVarCommon()->gtLclNum]);
11440 // Make sure we don't separately promote the fields of this struct.
11441 if (varDsc->lvRegStruct)
11443 // We can enregister, but can't promote.
11444 varDsc->lvPromoted = false;
11448 lvaSetVarDoNotEnregister(temp->gtLclVarCommon.gtLclNum DEBUGARG(DNER_LocalField));
11451 // We will turn a GT_LCL_VAR into a GT_LCL_FLD with an gtLclOffs of 'ival'
11452 // or if we already have a GT_LCL_FLD we will adjust the gtLclOffs by adding 'ival'
11453 // Then we change the type of the GT_LCL_FLD to match the orginal GT_IND type.
11455 if (temp->OperGet() == GT_LCL_FLD)
11457 temp->AsLclFld()->gtLclOffs += (unsigned short)ival1;
11458 temp->AsLclFld()->gtFieldSeq =
11459 GetFieldSeqStore()->Append(temp->AsLclFld()->gtFieldSeq, fieldSeq);
11463 temp->ChangeOper(GT_LCL_FLD); // Note that this makes the gtFieldSeq "NotAField"...
11464 temp->AsLclFld()->gtLclOffs = (unsigned short)ival1;
11465 if (fieldSeq != NULL) // If it does represent a field, note that.
11466 temp->AsLclFld()->gtFieldSeq = fieldSeq;
11468 temp->gtType = tree->gtType;
11469 foldAndReturnTemp = true;
11472 assert(foldAndReturnTemp == true);
11474 // Keep the DONT_CSE flag in sync
11475 // (i.e keep the original value of this flag from tree)
11476 // as it can be set for 'temp' because a GT_ADDR always marks it for it's op1
11478 temp->gtFlags &= ~GTF_DONT_CSE;
11479 temp->gtFlags |= (tree->gtFlags & GTF_DONT_CSE);
11481 noway_assert(op1->gtOper == GT_ADD || op1->gtOper == GT_ADDR);
11482 noway_assert(temp->gtType == tree->gtType);
11484 if (op1->OperGet() == GT_ADD)
11486 DEBUG_DESTROY_NODE(op1->gtOp.gtOp1); // GT_ADDR
11487 DEBUG_DESTROY_NODE(op1->gtOp.gtOp2); // GT_CNS_INT
11489 DEBUG_DESTROY_NODE(op1); // GT_ADD or GT_ADDR
11490 DEBUG_DESTROY_NODE(tree); // GT_IND
11495 // Only do this optimization when we are in the global optimizer. Doing this after value numbering
11496 // could result in an invalid value number for the newly generated GT_IND node.
11497 if ((op1->OperGet() == GT_COMMA) && fgGlobalMorph)
11499 // Perform the transform IND(COMMA(x, ..., z)) == COMMA(x, ..., IND(z)).
11500 // TBD: this transformation is currently necessary for correctness -- it might
11501 // be good to analyze the failures that result if we don't do this, and fix them
11502 // in other ways. Ideally, this should be optional.
11503 GenTreePtr commaNode = op1;
11504 unsigned treeFlags = tree->gtFlags;
11505 commaNode->gtType = typ;
11506 commaNode->gtFlags = (treeFlags & ~GTF_REVERSE_OPS); // Bashing the GT_COMMA flags here is dangerous, clear the GTF_REVERSE_OPS at least.
11508 commaNode->gtFlags |= GTF_MORPHED;
11510 while (commaNode->gtOp.gtOp2->gtOper == GT_COMMA)
11512 commaNode = commaNode->gtOp.gtOp2;
11513 commaNode->gtType = typ;
11514 commaNode->gtFlags = (treeFlags & ~GTF_REVERSE_OPS); // Bashing the GT_COMMA flags here is dangerous, clear the GTF_REVERSE_OPS at least.
11516 commaNode->gtFlags |= GTF_MORPHED;
11519 bool wasArrIndex = (tree->gtFlags & GTF_IND_ARR_INDEX) != 0;
11523 bool b = GetArrayInfoMap()->Lookup(tree, &arrInfo);
11525 GetArrayInfoMap()->Remove(tree);
11528 op1 = gtNewOperNode(GT_IND, typ, commaNode->gtOp.gtOp2);
11529 op1->gtFlags = treeFlags;
11532 GetArrayInfoMap()->Set(op1, arrInfo);
11535 op1->gtFlags |= GTF_MORPHED;
11537 commaNode->gtOp.gtOp2 = op1;
11545 // Can not remove op1 if it is currently a CSE candidate.
11546 if (gtIsActiveCSE_Candidate(op1))
11549 if (op1->OperGet() == GT_IND)
11551 if ((op1->gtFlags & GTF_IND_ARR_INDEX) == 0)
11553 // Can not remove a GT_ADDR if it is currently a CSE candidate.
11554 if (gtIsActiveCSE_Candidate(tree))
11557 // Perform the transform ADDR(IND(...)) == (...).
11558 GenTreePtr addr = op1->gtOp.gtOp1;
11560 noway_assert(varTypeIsGC(addr->gtType) || addr->gtType == TYP_I_IMPL);
11562 DEBUG_DESTROY_NODE(op1);
11563 DEBUG_DESTROY_NODE(tree);
11568 else if (op1->gtOper == GT_CAST)
11570 GenTreePtr casting = op1->gtCast.CastOp();
11571 if (casting->gtOper == GT_LCL_VAR || casting->gtOper == GT_CLS_VAR)
11573 DEBUG_DESTROY_NODE(op1);
11574 tree->gtOp.gtOp1 = op1 = casting;
11577 else if ((op1->gtOper == GT_COMMA) && !optValnumCSE_phase)
11579 // Perform the transform ADDR(COMMA(x, ..., z)) == COMMA(x, ..., ADDR(z)).
11580 // (Be sure to mark "z" as an l-value...)
11581 GenTreePtr commaNode = op1;
11582 while (commaNode->gtOp.gtOp2->gtOper == GT_COMMA)
11584 commaNode = commaNode->gtOp.gtOp2;
11586 // The top-level addr might be annotated with a zeroOffset field.
11587 FieldSeqNode* zeroFieldSeq = nullptr;
11588 bool isZeroOffset = GetZeroOffsetFieldMap()->Lookup(tree, &zeroFieldSeq);
11590 commaNode->gtOp.gtOp2->gtFlags |= GTF_DONT_CSE;
11592 // If the node we're about to put under a GT_ADDR is a GT_IND, the indirection
11593 // doesn't need to be materialized, since we only want the addressing mode. Because
11594 // of this, this GT_IND is not a faulting indirection and we don't have to extract it
11595 // as a side effect.
11596 GenTree* commaOp2 = commaNode->gtOp.gtOp2;
11597 if (commaOp2->gtOper == GT_IND)
11599 commaOp2->gtFlags |= GTF_IND_NONFAULTING;
11602 op1 = gtNewOperNode(GT_ADDR, TYP_BYREF, commaOp2);
11606 // Transfer the annotation to the new GT_ADDR node.
11607 GetZeroOffsetFieldMap()->Set(op1, zeroFieldSeq);
11609 commaNode->gtOp.gtOp2 = op1;
11610 // Originally, I gave all the comma nodes type "byref". But the ADDR(IND(x)) == x transform
11611 // might give op1 a type different from byref (like, say, native int). So now go back and give
11612 // all the comma nodes the type of op1.
11614 while (commaNode->gtOper == GT_COMMA)
11616 commaNode->gtType = op1->gtType; commaNode->gtFlags |= op1->gtFlags;
11618 commaNode->gtFlags |= GTF_MORPHED;
11620 commaNode = commaNode->gtOp.gtOp2;
11626 /* op1 of a GT_ADDR is an l-value. Only r-values can be CSEed */
11627 op1->gtFlags |= GTF_DONT_CSE;
11633 /* Mark the nodes that are conditionally executed */
11634 fgWalkTreePre(&tree, gtMarkColonCond);
11636 /* Since we're doing this postorder we clear this if it got set by a child */
11637 fgRemoveRestOfBlock = false;
11642 /* Special case: trees that don't produce a value */
11643 if ((op2->OperKind() & GTK_ASGOP) ||
11644 (op2->OperGet() == GT_COMMA && op2->TypeGet() == TYP_VOID) ||
11647 typ = tree->gtType = TYP_VOID;
11650 // If we are in the Valuenum CSE phase then don't morph away anything as these
11651 // nodes may have CSE defs/uses in them.
11653 if (!optValnumCSE_phase)
11655 //Extract the side effects from the left side of the comma. Since they don't "go" anywhere, this is
11658 GenTreePtr op1SideEffects = NULL;
11659 // The addition of "GTF_MAKE_CSE" below prevents us from throwing away (for example)
11660 // hoisted expressions in loops.
11661 gtExtractSideEffList(op1, &op1SideEffects, (GTF_SIDE_EFFECT | GTF_MAKE_CSE));
11662 if (op1SideEffects)
11664 //Replace the left hand side with the side effect list.
11665 tree->gtOp.gtOp1 = op1SideEffects;
11666 tree->gtFlags |= (op1SideEffects->gtFlags & GTF_ALL_EFFECT);
11670 /* The left operand is worthless, throw it away */
11671 if (lvaLocalVarRefCounted)
11673 lvaRecursiveDecRefCounts(op1);
11675 op2->gtFlags |= (tree->gtFlags & (GTF_DONT_CSE | GTF_LATE_ARG));
11676 DEBUG_DESTROY_NODE(tree);
11677 DEBUG_DESTROY_NODE(op1);
11681 /* If the right operand is just a void nop node, throw it away */
11682 if (op2->IsNothingNode() && op1->gtType == TYP_VOID)
11684 op1->gtFlags |= (tree->gtFlags & (GTF_DONT_CSE | GTF_LATE_ARG));
11685 DEBUG_DESTROY_NODE(tree);
11686 DEBUG_DESTROY_NODE(op2);
11695 /* Special case if fgRemoveRestOfBlock is set to true */
11696 if (fgRemoveRestOfBlock)
11698 if (fgIsCommaThrow(op1, true))
11700 GenTreePtr throwNode = op1->gtOp.gtOp1;
11701 noway_assert(throwNode->gtType == TYP_VOID);
11706 noway_assert(op1->OperKind() & GTK_RELOP);
11707 noway_assert(op1->gtFlags & GTF_EXCEPT);
11709 // We need to keep op1 for the side-effects. Hang it off
11712 tree->ChangeOper(GT_COMMA);
11713 tree->gtOp.gtOp2 = op2 = gtNewNothingNode();
11715 // Additionally since we're eliminating the JTRUE
11716 // codegen won't like it if op1 is a RELOP of longs, floats or doubles.
11717 // So we change it into a GT_COMMA as well.
11718 op1->ChangeOper(GT_COMMA);
11719 op1->gtType = op1->gtOp.gtOp1->gtType;
11728 noway_assert(oper == tree->gtOper);
11730 // If we are in the Valuenum CSE phase then don't morph away anything as these
11731 // nodes may have CSE defs/uses in them.
11733 if (!optValnumCSE_phase && (oper != GT_ASG) && (oper != GT_COLON) && !tree->IsList())
11735 /* Check for op1 as a GT_COMMA with a unconditional throw node */
11736 if (op1 && fgIsCommaThrow(op1, true))
11738 if ((op1->gtFlags & GTF_COLON_COND) == 0)
11740 /* We can safely throw out the rest of the statements */
11741 fgRemoveRestOfBlock = true;
11744 GenTreePtr throwNode = op1->gtOp.gtOp1;
11745 noway_assert(throwNode->gtType == TYP_VOID);
11747 if (oper == GT_COMMA)
11749 /* Both tree and op1 are GT_COMMA nodes */
11750 /* Change the tree's op1 to the throw node: op1->gtOp.gtOp1 */
11751 tree->gtOp.gtOp1 = throwNode;
11754 else if (oper != GT_NOP)
11756 if (genActualType(typ) == genActualType(op1->gtType))
11758 /* The types match so, return the comma throw node as the new tree */
11763 if (typ == TYP_VOID)
11765 // Return the throw node
11770 GenTreePtr commaOp2 = op1->gtOp.gtOp2;
11772 // need type of oper to be same as tree
11773 if (typ == TYP_LONG)
11775 commaOp2->ChangeOperConst(GT_CNS_NATIVELONG);
11776 commaOp2->gtIntConCommon.SetLngValue(0);
11777 /* Change the types of oper and commaOp2 to TYP_LONG */
11778 op1->gtType = commaOp2->gtType = TYP_LONG;
11780 else if (varTypeIsFloating(typ))
11782 commaOp2->ChangeOperConst(GT_CNS_DBL);
11783 commaOp2->gtDblCon.gtDconVal = 0.0;
11784 /* Change the types of oper and commaOp2 to TYP_DOUBLE */
11785 op1->gtType = commaOp2->gtType = TYP_DOUBLE;
11789 commaOp2->ChangeOperConst(GT_CNS_INT);
11790 commaOp2->gtIntConCommon.SetIconValue(0);
11791 /* Change the types of oper and commaOp2 to TYP_INT */
11792 op1->gtType = commaOp2->gtType = TYP_INT;
11795 /* Return the GT_COMMA node as the new tree */
11802 /* Check for op2 as a GT_COMMA with a unconditional throw */
11804 if (op2 && fgIsCommaThrow(op2, true))
11806 if ((op2->gtFlags & GTF_COLON_COND) == 0)
11808 /* We can safely throw out the rest of the statements */
11809 fgRemoveRestOfBlock = true;
11812 // If op1 has no side-effects
11813 if ((op1->gtFlags & GTF_ALL_EFFECT) == 0)
11815 // If tree is an asg node
11816 if (tree->OperIsAssignment())
11818 /* Return the throw node as the new tree */
11819 return op2->gtOp.gtOp1;
11822 if (tree->OperGet() == GT_ARR_BOUNDS_CHECK)
11824 /* Return the throw node as the new tree */
11825 return op2->gtOp.gtOp1;
11828 // If tree is a comma node
11829 if (tree->OperGet() == GT_COMMA)
11831 /* Return the throw node as the new tree */
11832 return op2->gtOp.gtOp1;
11835 /* for the shift nodes the type of op2 can differ from the tree type */
11836 if ((typ == TYP_LONG) && (genActualType(op2->gtType) == TYP_INT))
11838 noway_assert(GenTree::OperIsShiftOrRotate(oper));
11840 GenTreePtr commaOp2 = op2->gtOp.gtOp2;
11842 commaOp2->ChangeOperConst(GT_CNS_NATIVELONG);
11843 commaOp2->gtIntConCommon.SetLngValue(0);
11845 /* Change the types of oper and commaOp2 to TYP_LONG */
11846 op2->gtType = commaOp2->gtType = TYP_LONG;
11849 if ((genActualType(typ) == TYP_INT) && (genActualType(op2->gtType) == TYP_LONG ||
11850 varTypeIsFloating(op2->TypeGet())))
11852 // An example case is comparison (say GT_GT) of two longs or floating point values.
11854 GenTreePtr commaOp2 = op2->gtOp.gtOp2;
11856 commaOp2->ChangeOperConst(GT_CNS_INT);
11857 commaOp2->gtIntCon.gtIconVal = 0;
11858 /* Change the types of oper and commaOp2 to TYP_INT */
11859 op2->gtType = commaOp2->gtType = TYP_INT;
11862 if ((typ == TYP_BYREF) && (genActualType(op2->gtType) == TYP_I_IMPL))
11864 noway_assert(tree->OperGet() == GT_ADD);
11866 GenTreePtr commaOp2 = op2->gtOp.gtOp2;
11868 commaOp2->ChangeOperConst(GT_CNS_INT);
11869 commaOp2->gtIntCon.gtIconVal = 0;
11870 /* Change the types of oper and commaOp2 to TYP_BYREF */
11871 op2->gtType = commaOp2->gtType = TYP_BYREF;
11874 /* types should now match */
11875 noway_assert( (genActualType(typ) == genActualType(op2->gtType)));
11877 /* Return the GT_COMMA node as the new tree */
11883 /*-------------------------------------------------------------------------
11884 * Optional morphing is done if tree transformations is permitted
11887 if ((opts.compFlags & CLFLG_TREETRANS) == 0)
11890 tree = fgMorphSmpOpOptional(tree->AsOp());
11892 } // extra scope for gcc workaround
11896 #pragma warning(pop)
11900 GenTree* Compiler::fgMorphSmpOpOptional(GenTreeOp* tree)
11902 genTreeOps oper = tree->gtOper;
11903 GenTree* op1 = tree->gtOp1;
11904 GenTree* op2 = tree->gtOp2;
11905 var_types typ = tree->TypeGet();
11907 if (GenTree::OperIsCommutative(oper))
11909 /* Swap the operands so that the more expensive one is 'op1' */
11911 if (tree->gtFlags & GTF_REVERSE_OPS)
11919 tree->gtFlags &= ~GTF_REVERSE_OPS;
11922 if (oper == op2->gtOper)
11924 /* Reorder nested operators at the same precedence level to be
11925 left-recursive. For example, change "(a+(b+c))" to the
11926 equivalent expression "((a+b)+c)".
11929 /* Things are handled differently for floating-point operators */
11931 if (!varTypeIsFloating(tree->TypeGet()))
11933 fgMoveOpsLeft(tree);
11943 /* Change "((x+icon)+y)" to "((x+y)+icon)"
11944 Don't reorder floating-point operations */
11946 if ((oper == GT_ADD) && !tree->gtOverflow() &&
11947 (op1->gtOper == GT_ADD) && ! op1->gtOverflow() && varTypeIsIntegralOrI(typ))
11949 GenTreePtr ad2 = op1->gtOp.gtOp2;
11951 if (op2->OperIsConst() == 0 &&
11952 ad2->OperIsConst() != 0)
11964 // And it swaps ad2 and op2. If (op2) is varTypeIsGC, then this implies that (tree) is
11965 // varTypeIsGC. If (op1) is not, then when we swap (ad2) and (op2), then we have a TYP_INT node
11966 // (op1) with a child that is varTypeIsGC. If we encounter that situation, make (op1) the same
11969 // Also, if (ad2) is varTypeIsGC then (tree) must also be (since op1 is), so no fixing is
11972 if (varTypeIsGC(op2->TypeGet()))
11974 noway_assert(varTypeIsGC(typ));
11979 op1 ->gtOp.gtOp2 = op2;
11980 op1->gtFlags |= op2->gtFlags & GTF_ALL_EFFECT;
11988 /*-------------------------------------------------------------------------
11989 * Perform optional oper-specific postorder morphing
11995 bool dstIsSafeLclVar;
11999 /* We'll convert "a = a <op> x" into "a <op>= x" */
12000 /* and also "a = x <op> a" into "a <op>= x" for communative ops */
12003 if (typ == TYP_LONG)
12007 /* Make sure we're allowed to do this */
12009 if (optValnumCSE_phase)
12011 // It is not safe to reorder/delete CSE's
12015 /* Are we assigning to a GT_LCL_VAR ? */
12017 dstIsSafeLclVar = (op1->gtOper == GT_LCL_VAR);
12019 /* If we have a GT_LCL_VAR, then is the address taken? */
12020 if (dstIsSafeLclVar)
12022 unsigned lclNum = op1->gtLclVarCommon.gtLclNum;
12023 LclVarDsc * varDsc = lvaTable + lclNum;
12025 noway_assert(lclNum < lvaCount);
12027 /* Is the address taken? */
12028 if (varDsc->lvAddrExposed)
12030 dstIsSafeLclVar = false;
12032 else if (op2->gtFlags & GTF_ASG)
12038 if (!dstIsSafeLclVar)
12040 if (op2->gtFlags & GTF_ASG)
12043 if ((op2->gtFlags & GTF_CALL) && (op1->gtFlags & GTF_ALL_EFFECT))
12047 /* Special case: a cast that can be thrown away */
12049 if (op1->gtOper == GT_IND &&
12050 op2->gtOper == GT_CAST &&
12051 !op2->gtOverflow() )
12057 srct = op2->gtCast.CastOp()->TypeGet();
12058 cast = (var_types) op2->CastToType();
12059 dstt = op1->TypeGet();
12061 /* Make sure these are all ints and precision is not lost */
12063 if (cast >= dstt && dstt <= TYP_INT && srct <= TYP_INT)
12064 op2 = tree->gtOp2 = op2->gtCast.CastOp();
12067 /* Make sure we have the operator range right */
12069 noway_assert(GT_SUB == GT_ADD + 1);
12070 noway_assert(GT_MUL == GT_ADD + 2);
12071 noway_assert(GT_DIV == GT_ADD + 3);
12072 noway_assert(GT_MOD == GT_ADD + 4);
12073 noway_assert(GT_UDIV== GT_ADD + 5);
12074 noway_assert(GT_UMOD== GT_ADD + 6);
12076 noway_assert(GT_OR == GT_ADD + 7);
12077 noway_assert(GT_XOR == GT_ADD + 8);
12078 noway_assert(GT_AND == GT_ADD + 9);
12080 noway_assert(GT_LSH == GT_ADD + 10);
12081 noway_assert(GT_RSH == GT_ADD + 11);
12082 noway_assert(GT_RSZ == GT_ADD + 12);
12084 /* Check for a suitable operator on the RHS */
12086 cmop = op2->OperGet();
12091 // GT_CHS only supported for integer types
12092 if ( varTypeIsFloating(tree->TypeGet()))
12098 // GT_ASG_MUL only supported for floating point types
12099 if (!varTypeIsFloating(tree->TypeGet()))
12106 if (op2->gtOverflow())
12108 /* Disable folding into "<op>=" if the result can be
12109 visible to anyone as <op> may throw an exception and
12110 the assignment should not proceed
12111 We are safe with an assignment to a local variables
12113 if (ehBlockHasExnFlowDsc(compCurBB))
12115 if (!dstIsSafeLclVar)
12118 #ifndef _TARGET_AMD64_
12119 // This is hard for byte-operations as we need to make
12120 // sure both operands are in RBM_BYTE_REGS.
12121 if (varTypeIsByte(op2->TypeGet()))
12123 #endif // _TARGET_AMD64_
12128 // GT_ASG_DIV only supported for floating point types
12129 if (!varTypeIsFloating(tree->TypeGet()))
12138 if (typ == TYP_LONG)
12148 /* TODO: allow non-const long assignment operators */
12150 if (typ == TYP_LONG && op2->gtOp.gtOp2->gtOper != GT_CNS_LNG)
12156 bool bReverse = false;
12157 bool bAsgOpFoldable = fgShouldCreateAssignOp(tree, &bReverse);
12158 if (bAsgOpFoldable)
12162 // We will transform this from "a = x <op> a" to "a <op>= x"
12163 // so we can now destroy the duplicate "a"
12164 DEBUG_DESTROY_NODE(op2->gtOp.gtOp2);
12165 op2->gtOp.gtOp2 = op2->gtOp.gtOp1;
12168 /* Special case: "x |= -1" and "x &= 0" */
12169 if (cmop == GT_AND || cmop == GT_OR)
12171 if (op2->gtOp.gtOp2->IsCnsIntOrI())
12173 ssize_t icon = op2->gtOp.gtOp2->gtIntCon.gtIconVal;
12175 noway_assert(typ <= TYP_UINT);
12177 if ((cmop == GT_AND && icon == 0) ||
12178 (cmop == GT_OR && icon == -1))
12180 /* Simply change to an assignment */
12181 tree->gtOp2 = op2->gtOp.gtOp2;
12187 if (cmop == GT_NEG)
12189 /* This is "x = -x;", use the flipsign operator */
12191 tree->ChangeOper (GT_CHS);
12193 if (op1->gtOper == GT_LCL_VAR)
12194 op1->gtFlags |= GTF_VAR_USEASG;
12196 tree->gtOp2 = gtNewIconNode(0, op1->TypeGet());
12201 if (cmop == GT_RSH && varTypeIsSmall(op1->TypeGet()) && varTypeIsUnsigned(op1->TypeGet()))
12203 // Changing from x = x op y to x op= y when x is a small integer type
12204 // makes the op size smaller (originally the op size was 32 bits, after
12205 // sign or zero extension of x, and there is an implicit truncation in the
12207 // This is ok in most cases because the upper bits were
12208 // lost when assigning the op result to a small type var,
12209 // but it may not be ok for the right shift operation where the higher bits
12210 // could be shifted into the lower bits and preserved.
12211 // Signed right shift of signed x still works (i.e. (sbyte)((int)(sbyte)x >>signed y) == (sbyte)x >>signed y))
12212 // as do unsigned right shift ((ubyte)((int)(ubyte)x >>unsigned y) == (ubyte)x >>unsigned y), but
12213 // signed right shift of an unigned small type may give the wrong result:
12214 // e.g. (ubyte)((int)(ubyte)0xf0 >>signed 4) == 0x0f,
12215 // but (ubyte)0xf0 >>signed 4 == 0xff which is incorrect.
12216 // The result becomes correct if we use >>unsigned instead of >>signed.
12217 noway_assert(op1->TypeGet() == op2->gtOp.gtOp1->TypeGet());
12221 /* Replace with an assignment operator */
12222 noway_assert(GT_ADD - GT_ADD == GT_ASG_ADD - GT_ASG_ADD);
12223 noway_assert(GT_SUB - GT_ADD == GT_ASG_SUB - GT_ASG_ADD);
12224 noway_assert(GT_OR - GT_ADD == GT_ASG_OR - GT_ASG_ADD);
12225 noway_assert(GT_XOR - GT_ADD == GT_ASG_XOR - GT_ASG_ADD);
12226 noway_assert(GT_AND - GT_ADD == GT_ASG_AND - GT_ASG_ADD);
12227 noway_assert(GT_LSH - GT_ADD == GT_ASG_LSH - GT_ASG_ADD);
12228 noway_assert(GT_RSH - GT_ADD == GT_ASG_RSH - GT_ASG_ADD);
12229 noway_assert(GT_RSZ - GT_ADD == GT_ASG_RSZ - GT_ASG_ADD);
12231 tree->SetOper((genTreeOps)(cmop - GT_ADD + GT_ASG_ADD));
12232 tree->gtOp2 = op2->gtOp.gtOp2;
12234 /* Propagate GTF_OVERFLOW */
12236 if (op2->gtOverflowEx())
12238 tree->gtType = op2->gtType;
12239 tree->gtFlags |= (op2->gtFlags &
12240 (GTF_OVERFLOW|GTF_EXCEPT|GTF_UNSIGNED));
12243 #if FEATURE_SET_FLAGS
12245 /* Propagate GTF_SET_FLAGS */
12246 if (op2->gtSetFlags())
12248 tree->gtRequestSetFlags();
12251 #endif // FEATURE_SET_FLAGS
12253 DEBUG_DESTROY_NODE(op2);
12256 /* The target is used as well as being defined */
12257 if (op1->OperIsLocal())
12258 op1->gtFlags |= GTF_VAR_USEASG;
12261 #if CPU_HAS_FP_SUPPORT
12262 /* Check for the special case "x += y * x;" */
12264 // GT_ASG_MUL only supported for floating point types
12265 if (cmop != GT_ADD && cmop != GT_SUB)
12268 if (op2->gtOper == GT_MUL && varTypeIsFloating(tree->TypeGet()))
12270 if (GenTree::Compare(op1, op2->gtOp.gtOp1))
12272 /* Change "x += x * y" into "x *= (y + 1)" */
12274 op2 = op2->gtOp.gtOp2;
12276 else if (GenTree::Compare(op1, op2->gtOp.gtOp2))
12278 /* Change "x += y * x" into "x *= (y + 1)" */
12280 op2 = op2->gtOp.gtOp1;
12285 op1 = gtNewDconNode(1.0);
12287 /* Now make the "*=" node */
12289 if (cmop == GT_ADD)
12291 /* Change "x += x * y" into "x *= (y + 1)" */
12293 tree->gtOp2 = op2 = gtNewOperNode(GT_ADD,
12300 /* Change "x -= x * y" into "x *= (1 - y)" */
12302 noway_assert(cmop == GT_SUB);
12303 tree->gtOp2 = op2 = gtNewOperNode(GT_SUB,
12308 tree->ChangeOper(GT_ASG_MUL);
12310 #endif // CPU_HAS_FP_SUPPORT
12318 /* Is the destination identical to the first RHS sub-operand? */
12320 if (GenTree::Compare(op1, op2->gtOp.gtOp1))
12322 /* This is "x = ~x" which is the same as "x ^= -1"
12323 * Transform the node into a GT_ASG_XOR */
12325 noway_assert(genActualType(typ) == TYP_INT ||
12326 genActualType(typ) == TYP_LONG);
12328 op2->gtOp.gtOp2 = (genActualType(typ) == TYP_INT)
12329 ? gtNewIconNode(-1)
12330 : gtNewLconNode(-1);
12345 /* Check for the case "(val + icon) * icon" */
12347 if (op2->gtOper == GT_CNS_INT &&
12348 op1->gtOper == GT_ADD)
12350 GenTreePtr add = op1->gtOp.gtOp2;
12352 if (add->IsCnsIntOrI() && (op2->GetScaleIndexMul() != 0))
12354 if (tree->gtOverflow() || op1->gtOverflow())
12359 ssize_t imul = op2->gtIntCon.gtIconVal;
12360 ssize_t iadd = add->gtIntCon.gtIconVal;
12362 /* Change '(val + iadd) * imul' -> '(val * imul) + (iadd * imul)' */
12365 tree->ChangeOper(oper);
12367 op2->gtIntCon.gtIconVal = iadd * imul;
12369 op1->ChangeOper(GT_MUL);
12371 add->gtIntCon.gtIconVal = imul;
12372 #ifdef _TARGET_64BIT_
12373 if (add->gtType == TYP_INT)
12375 // we need to properly re-sign-extend or truncate after multiplying two int constants above
12376 add->AsIntCon()->TruncateOrSignExtend32();
12378 #endif //_TARGET_64BIT_
12386 /* For "val / 1", just return "val" */
12388 if ((op2->gtOper == GT_CNS_INT) && (op2->gtIntConCommon.IconValue() == 1))
12390 DEBUG_DESTROY_NODE(tree);
12393 // Do this for "long" constants as well as ints.
12394 else if ((op2->gtOper == GT_CNS_LNG) && (op2->gtIntConCommon.LngValue() == 1))
12396 DEBUG_DESTROY_NODE(tree);
12404 /* Check for the case "(val + icon) << icon" */
12406 if (op2->IsCnsIntOrI() &&
12407 op1->gtOper == GT_ADD && !op1->gtOverflow())
12409 GenTreePtr cns = op1->gtOp.gtOp2;
12411 if (cns->IsCnsIntOrI() && (op2->GetScaleIndexShf() != 0))
12413 ssize_t ishf = op2->gtIntConCommon.IconValue();
12414 ssize_t iadd = cns->gtIntConCommon.IconValue();
12416 // printf("Changing '(val+icon1)<<icon2' into '(val<<icon2+icon1<<icon2)'\n");
12418 /* Change "(val + iadd) << ishf" into "(val<<ishf + iadd<<ishf)" */
12420 tree->ChangeOper(GT_ADD);
12421 ssize_t result = iadd << ishf;
12422 op2->gtIntConCommon.SetIconValue(result);
12423 #ifdef _TARGET_64BIT_
12424 if (op1->gtType == TYP_INT)
12426 op2->AsIntCon()->TruncateOrSignExtend32();
12428 #endif // _TARGET_64BIT_
12430 // we are reusing the shift amount node here, but the type we want is that of the shift result
12431 op2->gtType = op1->gtType;
12433 if (cns->gtOper == GT_CNS_INT &&
12434 cns->gtIntCon.gtFieldSeq != nullptr &&
12435 cns->gtIntCon.gtFieldSeq->IsConstantIndexFieldSeq())
12437 assert(cns->gtIntCon.gtFieldSeq->m_next == nullptr);
12438 op2->gtIntCon.gtFieldSeq = cns->gtIntCon.gtFieldSeq;
12441 op1->ChangeOper(GT_LSH);
12443 cns->gtIntConCommon.SetIconValue(ishf);
12451 if (!optValnumCSE_phase)
12453 /* "x ^ -1" is "~x" */
12455 if ((op2->gtOper == GT_CNS_INT) && (op2->gtIntConCommon.IconValue() == -1))
12457 tree->ChangeOper(GT_NOT);
12458 tree->gtOp2 = NULL;
12459 DEBUG_DESTROY_NODE(op2);
12461 else if ((op2->gtOper == GT_CNS_LNG) && (op2->gtIntConCommon.LngValue() == -1))
12463 tree->ChangeOper(GT_NOT);
12464 tree->gtOp2 = NULL;
12465 DEBUG_DESTROY_NODE(op2);
12467 else if ((op2->gtOper == GT_CNS_INT) && (op2->gtIntConCommon.IconValue() == 1) &&
12468 op1->OperIsCompare())
12470 /* "binaryVal ^ 1" is "!binaryVal" */
12471 gtReverseCond(op1);
12472 DEBUG_DESTROY_NODE(op2);
12473 DEBUG_DESTROY_NODE(tree);
12481 return fgMorphInitBlock(tree);
12486 return fgMorphCopyBlock(tree);
12496 // code to generate a magic number and shift amount for the magic number division
12497 // optimization. This code is previously from UTC where it notes it was taken from
12498 // _The_PowerPC_Compiler_Writer's_Guide_, pages 57-58.
12499 // The paper it is based on is "Division by invariant integers using multiplication"
12500 // by Torbjorn Granlund and Peter L. Montgomery in PLDI 94
12502 template <typename T>
12503 T GetSignedMagicNumberForDivide(T denom, int *shift /*out*/)
12505 // static SMAG smag;
12506 const int bits = sizeof(T) * 8;
12507 const int bits_minus_1 = bits - 1;
12509 typedef typename jitstd::make_unsigned<T>::type UT;
12511 const UT two_nminus1 = UT(1) << bits_minus_1;
12526 absDenom = abs(denom);
12527 t = two_nminus1 + ((unsigned int)denom >> 31);
12528 absNc = t - 1 - (t % absDenom); // absolute value of nc
12529 p = bits_minus_1; // initialize p
12530 q1 = two_nminus1 / absNc; // initialize q1 = 2^p / abs(nc)
12531 r1 = two_nminus1 - (q1 * absNc); // initialize r1 = rem(2^p, abs(nc))
12532 q2 = two_nminus1 / absDenom; // initialize q1 = 2^p / abs(denom)
12533 r2 = two_nminus1 - (q2 * absDenom); // initialize r1 = rem(2^p, abs(denom))
12538 q1 *= 2; // update q1 = 2^p / abs(nc)
12539 r1 *= 2; // update r1 = rem(2^p / abs(nc))
12541 if (r1 >= absNc) { // must be unsigned comparison
12546 q2 *= 2; // update q2 = 2^p / abs(denom)
12547 r2 *= 2; // update r2 = rem(2^p / abs(denom))
12549 if (r2 >= absDenom) { // must be unsigned comparison
12554 delta = absDenom - r2;
12555 } while (q1 < delta || (q1 == delta && r1 == 0));
12557 result_magic = q2 + 1; // resulting magic number
12559 result_magic = -result_magic;
12561 *shift = p - bits; // resulting shift
12563 return result_magic;
12567 bool Compiler::fgShouldUseMagicNumberDivide(GenTreeOp* tree)
12569 #ifdef _TARGET_ARM64_
12570 // TODO-ARM64-NYI: We don't have a 'mulHi' implementation yet for ARM64
12574 // During the optOptimizeValnumCSEs phase we can call fgMorph and when we do,
12575 // if this method returns true we will introduce a new LclVar and
12576 // a couple of new GenTree nodes, including an assignment to the new LclVar.
12577 // None of these new GenTree nodes will have valid ValueNumbers.
12578 // That is an invalid state for a GenTree node during the optOptimizeValnumCSEs phase.
12580 // Also during optAssertionProp when extracting side effects we can assert
12581 // during gtBuildCommaList if we have one tree that has Value Numbers
12582 // and another one that does not.
12584 if (!fgGlobalMorph)
12586 // We only perform the Magic Number Divide optimization during
12587 // the initial global morph phase
12591 if (tree->gtFlags & GTF_OVERFLOW)
12594 if (tree->gtOp2->gtOper != GT_CNS_INT && tree->gtOp2->gtOper != GT_CNS_LNG)
12597 ssize_t cons = tree->gtOp2->gtIntConCommon.IconValue();
12599 if (cons == 0 || cons == -1 || cons == 1)
12602 // codegen will expand these
12603 if (cons == SSIZE_T_MIN || isPow2(abs(cons)))
12606 // someone else will fold this away, so don't make it complicated for them
12607 if (tree->gtOp1->IsCnsIntOrI())
12610 // There is no technical barrier to handling unsigned, however it is quite rare
12611 // and more work to support and test
12612 if (tree->gtFlags & GTF_UNSIGNED)
12620 // transform x%c -> x-((x/c)*c)
12622 GenTree* Compiler::fgMorphModByConst(GenTreeOp* tree)
12624 assert(fgShouldUseMagicNumberDivide(tree));
12626 var_types type = tree->gtType;
12628 GenTree* cns = tree->gtOp2;
12630 GenTree* numerator = fgMakeMultiUse(&tree->gtOp1);
12632 tree->SetOper(GT_DIV);
12634 GenTree* mul = gtNewOperNode(GT_MUL, type, tree, gtCloneExpr(cns));
12636 GenTree* sub = gtNewOperNode(GT_SUB, type, numerator, mul);
12639 sub->gtFlags |= GTF_MORPHED;
12645 // For ARM64 we don't have a remainder instruction,
12646 // The architecture manual suggests the following transformation to
12647 // generate code for such operator:
12649 // a % b = a - (a / b) * b;
12651 // This method will produce the above expression in 'a' and 'b' are
12652 // leaf nodes, otherwise, if any of them is not a leaf it will spill
12653 // its value into a temporary variable, an example:
12654 // (x * 2 - 1) % (y + 1) -> t1 - (t2 * ( comma(t1 = x * 2 - 1, t1) / comma(t2 = y + 1, t2) ) )
12656 GenTree* Compiler::fgMorphModToSubMulDiv(GenTreeOp* tree)
12658 #ifndef _TARGET_ARM64_
12659 assert(!"This should only be called for ARM64");
12662 if (tree->OperGet() == GT_MOD)
12664 tree->SetOper(GT_DIV);
12666 else if (tree->OperGet() == GT_UMOD)
12668 tree->SetOper(GT_UDIV);
12672 noway_assert(!"Illegal gtOper in fgMorphModToSubMulDiv");
12675 var_types type = tree->gtType;
12676 GenTree* denominator = tree->gtOp2;
12677 GenTree* numerator = tree->gtOp1;
12679 if (!numerator->OperIsLeaf())
12681 numerator = fgMakeMultiUse(&tree->gtOp1);
12684 if (!denominator->OperIsLeaf())
12686 denominator = fgMakeMultiUse(&tree->gtOp2);
12689 GenTree* mul = gtNewOperNode(GT_MUL, type, tree, gtCloneExpr(denominator));
12690 GenTree* sub = gtNewOperNode(GT_SUB, type, gtCloneExpr(numerator), mul);
12693 sub->gtFlags |= GTF_MORPHED;
12699 // Turn a division by a constant into a multiplication by constant + some adjustments
12700 // see comments on GetSignedMagicNumberForDivide for source of this algorithm.
12701 // returns: the transformed tree
12703 GenTree* Compiler::fgMorphDivByConst(GenTreeOp* tree)
12705 assert(fgShouldUseMagicNumberDivide(tree));
12707 JITDUMP("doing magic number divide optimization\n");
12709 int64_t denominator = tree->gtOp2->gtIntConCommon.IconValue();
12712 var_types type = tree->gtType;
12714 if (tree->gtType == TYP_INT)
12716 magic = GetSignedMagicNumberForDivide<int32_t>((int32_t) denominator, &shift);
12720 magic = GetSignedMagicNumberForDivide<int64_t>((int64_t) denominator, &shift);
12723 GenTree* numerator = nullptr;
12725 // If signs of the denominator and magic number don't match,
12726 // we will need to use the numerator again.
12727 if (signum(denominator) != signum(magic))
12729 numerator = fgMakeMultiUse(&tree->gtOp1);
12730 tree->gtFlags |= GTF_ASG;
12733 if (type == TYP_LONG)
12734 tree->gtOp2->gtIntConCommon.SetLngValue(magic);
12736 tree->gtOp2->gtIntConCommon.SetIconValue((ssize_t)magic);
12738 tree->SetOper(GT_MULHI);
12741 GenTree* mulresult = tree;
12743 JITDUMP("Multiply Result:\n");
12744 DISPTREE(mulresult);
12746 GenTree *adjusted = mulresult;
12748 if (denominator > 0 && magic < 0)
12750 // add the numerator back in
12751 adjusted = gtNewOperNode(GT_ADD, type, mulresult, numerator);
12753 else if (denominator < 0 && magic > 0)
12755 // subtract the numerator off
12756 adjusted = gtNewOperNode(GT_SUB, type, mulresult, numerator);
12760 adjusted = mulresult;
12763 GenTree* result1 = adjusted;
12766 result1 = gtNewOperNode(GT_RSH, type, adjusted, gtNewIconNode(shift, TYP_INT));
12769 GenTree* secondClone = fgMakeMultiUse(&result1);
12771 GenTree* result2 = gtNewOperNode(GT_RSZ, type, secondClone, gtNewIconNode(genTypeSize(type) * 8 - 1, type));
12774 GenTree* result = gtNewOperNode(GT_ADD, type, result1, result2);
12775 JITDUMP("Final Magic Number divide:\n");
12779 result->gtFlags |= GTF_MORPHED;
12785 //------------------------------------------------------------------------------
12786 // fgOperIsBitwiseRotationRoot : Check if the operation can be a root of a bitwise rotation tree.
12790 // oper - Operation to check
12793 // True if the operation can be a root of a bitwise rotation tree; false otherwise.
12795 bool Compiler::fgOperIsBitwiseRotationRoot(genTreeOps oper)
12797 return (oper == GT_OR) || (oper == GT_XOR);
12800 //------------------------------------------------------------------------------
12801 // fgRecognizeAndMorphBitwiseRotation : Check if the tree represents a left or right rotation. If so, return
12802 // an equivalent GT_ROL or GT_ROR tree; otherwise, return the original tree.
12805 // tree - tree to check for a rotation pattern
12808 // An equivalent GT_ROL or GT_ROR tree if a pattern is found; original tree otherwise.
12811 // The input is a GT_OR or a GT_XOR tree.
12813 GenTreePtr Compiler::fgRecognizeAndMorphBitwiseRotation(GenTreePtr tree)
12815 #ifndef LEGACY_BACKEND
12817 // Check for a rotation pattern, e.g.,
12830 // The patterns recognized:
12831 // (x << (y & M)) op (x >>> ((-y + N) & M))
12832 // (x >>> ((-y + N) & M)) op (x << (y & M))
12834 // (x << y) op (x >>> (-y + N))
12835 // (x >> > (-y + N)) op (x << y)
12837 // (x >>> (y & M)) op (x << ((-y + N) & M))
12838 // (x << ((-y + N) & M)) op (x >>> (y & M))
12840 // (x >>> y) op (x << (-y + N))
12841 // (x << (-y + N)) op (x >>> y)
12843 // (x << c1) op (x >>> c2)
12844 // (x >>> c1) op (x << c2)
12847 // c1 and c2 are const
12848 // c1 + c2 == bitsize(x)
12851 // M & (N - 1) == N - 1
12852 // op is either | or ^
12854 if (((tree->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) != 0) ||
12855 ((tree->gtFlags & GTF_ORDER_SIDEEFF) != 0))
12857 // We can't do anything if the tree has assignments, calls, or volatile
12858 // reads. Note that we allow GTF_EXCEPT side effect since any exceptions
12859 // thrown by the original tree will be thrown by the transformed tree as well.
12863 genTreeOps oper = tree->OperGet();
12864 assert(fgOperIsBitwiseRotationRoot(oper));
12866 // Check if we have an LSH on one side of the OR and an RSZ on the other side.
12867 GenTreePtr op1 = tree->gtGetOp1();
12868 GenTreePtr op2 = tree->gtGetOp2();
12869 GenTreePtr leftShiftTree = nullptr;
12870 GenTreePtr rightShiftTree = nullptr;
12871 if ((op1->OperGet() == GT_LSH) && (op2->OperGet() == GT_RSZ))
12873 leftShiftTree = op1;
12874 rightShiftTree = op2;
12876 else if ((op1->OperGet() == GT_RSZ) && (op2->OperGet() == GT_LSH))
12878 leftShiftTree = op2;
12879 rightShiftTree = op1;
12886 // Check if the trees representing the value to shift are identical.
12887 // We already checked that there are no side effects above.
12888 if (GenTree::Compare(leftShiftTree->gtGetOp1(), rightShiftTree->gtGetOp1()))
12890 GenTreePtr rotatedValue = leftShiftTree->gtGetOp1();
12891 var_types rotatedValueActualType = genActualType(rotatedValue->gtType);
12892 ssize_t rotatedValueBitSize = genTypeSize(rotatedValueActualType) * 8;
12893 noway_assert((rotatedValueBitSize == 32) || (rotatedValueBitSize == 64));
12894 GenTreePtr leftShiftIndex = leftShiftTree->gtGetOp2();
12895 GenTreePtr rightShiftIndex = rightShiftTree->gtGetOp2();
12897 // The shift index may be masked. At least (rotatedValueBitSize - 1) lower bits
12898 // shouldn't be masked for the transformation to be valid. If additional
12899 // higher bits are not masked, the transformation is still valid since the result
12900 // of MSIL shift instructions is unspecified if the shift amount is greater or equal
12901 // than the width of the value being shifted.
12902 ssize_t minimalMask = rotatedValueBitSize - 1;
12903 ssize_t leftShiftMask = -1;
12904 ssize_t rightShiftMask = -1;
12906 if ((leftShiftIndex->OperGet() == GT_AND))
12908 if (leftShiftIndex->gtGetOp2()->IsCnsIntOrI())
12910 leftShiftMask = leftShiftIndex->gtGetOp2()->gtIntCon.gtIconVal;
12911 leftShiftIndex = leftShiftIndex->gtGetOp1();
12919 if ((rightShiftIndex->OperGet() == GT_AND))
12921 if (rightShiftIndex->gtGetOp2()->IsCnsIntOrI())
12923 rightShiftMask = rightShiftIndex->gtGetOp2()->gtIntCon.gtIconVal;
12924 rightShiftIndex = rightShiftIndex->gtGetOp1();
12932 if (((minimalMask & leftShiftMask) != minimalMask) ||
12933 ((minimalMask & rightShiftMask) != minimalMask))
12935 // The shift index is overmasked, e.g., we have
12936 // something like (x << y & 15) or
12937 // (x >> (32 - y) & 15 with 32 bit x.
12938 // The transformation is not valid.
12942 GenTreePtr shiftIndexWithAdd = nullptr;
12943 GenTreePtr shiftIndexWithoutAdd = nullptr;
12944 genTreeOps rotateOp = GT_NONE;
12945 GenTreePtr rotateIndex = nullptr;
12947 if (leftShiftIndex->OperGet() == GT_ADD)
12949 shiftIndexWithAdd = leftShiftIndex;
12950 shiftIndexWithoutAdd = rightShiftIndex;
12953 else if (rightShiftIndex->OperGet() == GT_ADD)
12955 shiftIndexWithAdd = rightShiftIndex;
12956 shiftIndexWithoutAdd = leftShiftIndex;
12960 if (shiftIndexWithAdd != nullptr)
12962 if (shiftIndexWithAdd->gtGetOp2()->IsCnsIntOrI())
12964 if (shiftIndexWithAdd->gtGetOp2()->gtIntCon.gtIconVal == rotatedValueBitSize)
12966 if (shiftIndexWithAdd->gtGetOp1()->OperGet() == GT_NEG)
12968 if (GenTree::Compare(shiftIndexWithAdd->gtGetOp1()->gtGetOp1(), shiftIndexWithoutAdd))
12970 // We found one of these patterns:
12971 // (x << (y & M)) | (x >>> ((-y + N) & M))
12972 // (x << y) | (x >>> (-y + N))
12973 // (x >>> (y & M)) | (x << ((-y + N) & M))
12974 // (x >>> y) | (x << (-y + N))
12975 // where N == bitsize(x), M is const, and
12976 // M & (N - 1) == N - 1
12978 #ifndef _TARGET_64BIT_
12979 if (!shiftIndexWithoutAdd->IsCnsIntOrI() && (rotatedValueBitSize == 64))
12981 // TODO: we need to handle variable-sized long shifts specially on x86.
12982 // GT_LSH, GT_RSH, and GT_RSZ have helpers for this case. We may need
12983 // to add helpers for GT_ROL and GT_ROR.
12984 NYI("Rotation of a long value by variable amount");
12988 rotateIndex = shiftIndexWithoutAdd;
12994 else if ((leftShiftIndex->IsCnsIntOrI() &&
12995 rightShiftIndex->IsCnsIntOrI()))
12997 if (leftShiftIndex->gtIntCon.gtIconVal +
12998 rightShiftIndex->gtIntCon.gtIconVal == rotatedValueBitSize)
13000 // We found this pattern:
13001 // (x << c1) | (x >>> c2)
13002 // where c1 and c2 are const and c1 + c2 == bitsize(x)
13004 rotateIndex = leftShiftIndex;
13008 if (rotateIndex != nullptr)
13010 noway_assert(GenTree::OperIsRotate(rotateOp));
13012 unsigned inputTreeEffects = tree->gtFlags & GTF_ALL_EFFECT;
13014 // We can use the same tree only during global morph; reusing the tree in a later morph
13015 // may invalidate value numbers.
13018 tree->gtOp.gtOp1 = rotatedValue;
13019 tree->gtOp.gtOp2 = rotateIndex;
13020 tree->ChangeOper(rotateOp);
13021 noway_assert(inputTreeEffects == ((rotatedValue->gtFlags | rotateIndex->gtFlags) & GTF_ALL_EFFECT));
13025 tree = gtNewOperNode(rotateOp, rotatedValueActualType, rotatedValue, rotateIndex);
13026 noway_assert(inputTreeEffects == (tree->gtFlags & GTF_ALL_EFFECT));
13032 #endif //LEGACY_BACKEND
13036 #if !CPU_HAS_FP_SUPPORT
13037 GenTreePtr Compiler::fgMorphToEmulatedFP(GenTreePtr tree)
13040 genTreeOps oper = tree->OperGet();
13041 var_types typ = tree->TypeGet();
13042 GenTreePtr op1 = tree->gtOp.gtOp1;
13043 GenTreePtr op2 = tree->gtGetOp2();
13046 We have to use helper calls for all FP operations:
13048 FP operators that operate on FP values
13049 casts to and from FP
13050 comparisons of FP values
13053 if (varTypeIsFloating(typ) || (op1 && varTypeIsFloating(op1->TypeGet())))
13057 size_t argc = genTypeStSz(typ);
13059 /* Not all FP operations need helper calls */
13073 /* If the result isn't FP, it better be a compare or cast */
13075 if (!(varTypeIsFloating(typ) ||
13076 tree->OperIsCompare() || oper == GT_CAST))
13079 noway_assert(varTypeIsFloating(typ) ||
13080 tree->OperIsCompare() || oper == GT_CAST);
13083 /* Keep track of how many arguments we're passing */
13085 fgPtrArgCntCur += argc;
13087 /* Is this a binary operator? */
13091 /* Add the second operand to the argument count */
13093 fgPtrArgCntCur += argc; argc *= 2;
13095 /* What kind of an operator do we have? */
13099 case GT_ADD: helper = CPX_R4_ADD; break;
13100 case GT_SUB: helper = CPX_R4_SUB; break;
13101 case GT_MUL: helper = CPX_R4_MUL; break;
13102 case GT_DIV: helper = CPX_R4_DIV; break;
13103 // case GT_MOD: helper = CPX_R4_REM; break;
13105 case GT_EQ : helper = CPX_R4_EQ ; break;
13106 case GT_NE : helper = CPX_R4_NE ; break;
13107 case GT_LT : helper = CPX_R4_LT ; break;
13108 case GT_LE : helper = CPX_R4_LE ; break;
13109 case GT_GE : helper = CPX_R4_GE ; break;
13110 case GT_GT : helper = CPX_R4_GT ; break;
13116 noway_assert(!"unexpected FP binary op");
13120 args = gtNewArgList(tree->gtOp.gtOp2, tree->gtOp.gtOp1);
13130 noway_assert(!"FP cast");
13132 case GT_NEG: helper = CPX_R4_NEG; break;
13138 noway_assert(!"unexpected FP unary op");
13142 args = gtNewArgList(tree->gtOp.gtOp1);
13145 /* If we have double result/operands, modify the helper */
13147 if (typ == TYP_DOUBLE)
13149 noway_assert(CPX_R4_NEG+1 == CPX_R8_NEG);
13150 noway_assert(CPX_R4_ADD+1 == CPX_R8_ADD);
13151 noway_assert(CPX_R4_SUB+1 == CPX_R8_SUB);
13152 noway_assert(CPX_R4_MUL+1 == CPX_R8_MUL);
13153 noway_assert(CPX_R4_DIV+1 == CPX_R8_DIV);
13159 noway_assert(tree->OperIsCompare());
13161 noway_assert(CPX_R4_EQ+1 == CPX_R8_EQ);
13162 noway_assert(CPX_R4_NE+1 == CPX_R8_NE);
13163 noway_assert(CPX_R4_LT+1 == CPX_R8_LT);
13164 noway_assert(CPX_R4_LE+1 == CPX_R8_LE);
13165 noway_assert(CPX_R4_GE+1 == CPX_R8_GE);
13166 noway_assert(CPX_R4_GT+1 == CPX_R8_GT);
13169 tree = fgMorphIntoHelperCall(tree, helper, args);
13171 if (fgPtrArgCntMax < fgPtrArgCntCur)
13172 fgPtrArgCntMax = fgPtrArgCntCur;
13174 fgPtrArgCntCur -= argc;
13182 if (compCurBB == genReturnBB)
13184 /* This is the 'exitCrit' call at the exit label */
13186 noway_assert(op1->gtType == TYP_VOID);
13187 noway_assert(op2 == 0);
13189 tree->gtOp.gtOp1 = op1 = fgMorphTree(op1);
13194 /* This is a (real) return value -- check its type */
13197 if (genActualType(op1->TypeGet()) != genActualType(info.compRetType))
13199 bool allowMismatch = false;
13201 // Allow TYP_BYREF to be returned as TYP_I_IMPL and vice versa
13202 if ((info.compRetType == TYP_BYREF &&
13203 genActualType(op1->TypeGet()) == TYP_I_IMPL) ||
13204 (op1->TypeGet() == TYP_BYREF &&
13205 genActualType(info.compRetType) == TYP_I_IMPL))
13206 allowMismatch = true;
13208 if (varTypeIsFloating(info.compRetType) && varTypeIsFloating(op1->TypeGet()))
13209 allowMismatch = true;
13211 if (!allowMismatch)
13212 NO_WAY("Return type mismatch");
13223 /*****************************************************************************
13225 * Transform the given tree for code generation and return an equivalent tree.
13229 GenTreePtr Compiler::fgMorphTree(GenTreePtr tree, MorphAddrContext* mac)
13231 noway_assert(tree);
13232 noway_assert(tree->gtOper != GT_STMT);
13237 if ((unsigned)JitConfig.JitBreakMorphTree() == tree->gtTreeID)
13239 noway_assert(!"JitBreakMorphTree hit");
13245 int thisMorphNum = 0;
13246 if (verbose && treesBeforeAfterMorph)
13248 thisMorphNum = morphNum++;
13249 printf("\nfgMorphTree (before %d):\n", thisMorphNum);
13254 /*-------------------------------------------------------------------------
13255 * fgMorphTree() can potentially replace a tree with another, and the
13256 * caller has to store the return value correctly.
13257 * Turn this on to always make copy of "tree" here to shake out
13258 * hidden/unupdated references.
13263 if (compStressCompile(STRESS_GENERIC_CHECK, 0))
13267 #ifdef SMALL_TREE_NODES
13268 if (GenTree::s_gtNodeSizes[tree->gtOper] == TREE_NODE_SZ_SMALL)
13270 copy = gtNewLargeOperNode(GT_ADD, TYP_INT);
13275 copy = new (this, GT_CALL) GenTreeCall(TYP_INT);
13278 copy->CopyFrom(tree, this);
13280 #if defined (LATE_DISASM)
13281 // GT_CNS_INT is considered small, so CopyFrom() won't copy all fields
13282 if ((tree->gtOper == GT_CNS_INT) && tree->IsIconHandle())
13284 copy->gtIntCon.gtIconHdl.gtIconHdl1 = tree->gtIntCon.gtIconHdl.gtIconHdl1;
13285 copy->gtIntCon.gtIconHdl.gtIconHdl2 = tree->gtIntCon.gtIconHdl.gtIconHdl2;
13289 DEBUG_DESTROY_NODE(tree);
13296 /* Ensure that we haven't morphed this node already */
13297 assert(((tree->gtFlags & GTF_MORPHED) == 0) && "ERROR: Already morphed this node!");
13299 #if LOCAL_ASSERTION_PROP
13300 /* Before morphing the tree, we try to propagate any active assertions */
13301 if (optLocalAssertionProp)
13303 /* Do we have any active assertions? */
13305 if (optAssertionCount > 0)
13307 GenTreePtr newTree = tree;
13308 while (newTree != NULL)
13311 /* newTree is non-Null if we propagated an assertion */
13312 newTree = optAssertionProp(apFull, tree, NULL);
13314 noway_assert(tree != NULL);
13317 PREFAST_ASSUME(tree != NULL);
13321 /* Save the original un-morphed tree for fgMorphTreeDone */
13323 GenTreePtr oldTree = tree;
13325 /* Figure out what kind of a node we have */
13327 unsigned kind = tree->OperKind();
13329 /* Is this a constant node? */
13331 if (kind & GTK_CONST)
13333 tree = fgMorphConst(tree);
13337 /* Is this a leaf node? */
13339 if (kind & GTK_LEAF)
13341 tree = fgMorphLeaf(tree);
13345 /* Is it a 'simple' unary/binary operator? */
13347 if (kind & GTK_SMPOP)
13349 tree = fgMorphSmpOp(tree, mac);
13353 /* See what kind of a special operator we have here */
13355 switch (tree->OperGet())
13358 tree = fgMorphField(tree, mac);
13362 tree = fgMorphCall(tree->AsCall());
13365 case GT_ARR_BOUNDS_CHECK:
13366 #ifdef FEATURE_SIMD
13368 #endif // FEATURE_SIMD
13370 fgSetRngChkTarget(tree);
13372 GenTreeBoundsChk* bndsChk = tree->AsBoundsChk();
13373 bndsChk->gtArrLen = fgMorphTree(bndsChk->gtArrLen);
13374 bndsChk->gtIndex = fgMorphTree(bndsChk->gtIndex);
13375 // If the index is a comma(throw, x), just return that.
13376 if (!optValnumCSE_phase && fgIsCommaThrow(bndsChk->gtIndex))
13378 tree = bndsChk->gtIndex;
13381 // Propagate effects flags upwards
13382 bndsChk->gtFlags |= (bndsChk->gtArrLen->gtFlags & GTF_ALL_EFFECT);
13383 bndsChk->gtFlags |= (bndsChk->gtIndex->gtFlags & GTF_ALL_EFFECT);
13385 // Otherwise, we don't change the tree.
13390 tree->gtArrElem.gtArrObj = fgMorphTree(tree->gtArrElem.gtArrObj);
13391 tree->gtFlags |= tree->gtArrElem.gtArrObj->gtFlags & GTF_ALL_EFFECT;
13394 for (dim = 0; dim < tree->gtArrElem.gtArrRank; dim++)
13396 tree->gtArrElem.gtArrInds[dim] = fgMorphTree(tree->gtArrElem.gtArrInds[dim]);
13397 tree->gtFlags |= tree->gtArrElem.gtArrInds[dim]->gtFlags & GTF_ALL_EFFECT;
13400 fgSetRngChkTarget(tree, false);
13403 case GT_ARR_OFFSET:
13404 tree->gtArrOffs.gtOffset = fgMorphTree(tree->gtArrOffs.gtOffset);
13405 tree->gtFlags |= tree->gtArrOffs.gtOffset->gtFlags & GTF_ALL_EFFECT;
13406 tree->gtArrOffs.gtIndex = fgMorphTree(tree->gtArrOffs.gtIndex);
13407 tree->gtFlags |= tree->gtArrOffs.gtIndex->gtFlags & GTF_ALL_EFFECT;
13408 tree->gtArrOffs.gtArrObj = fgMorphTree(tree->gtArrOffs.gtArrObj);
13409 tree->gtFlags |= tree->gtArrOffs.gtArrObj->gtFlags & GTF_ALL_EFFECT;
13411 fgSetRngChkTarget(tree, false);
13415 tree->gtCmpXchg.gtOpLocation = fgMorphTree(tree->gtCmpXchg.gtOpLocation);
13416 tree->gtCmpXchg.gtOpValue = fgMorphTree(tree->gtCmpXchg.gtOpValue);
13417 tree->gtCmpXchg.gtOpComparand = fgMorphTree(tree->gtCmpXchg.gtOpComparand);
13424 noway_assert(!"unexpected operator");
13428 fgMorphTreeDone(tree, oldTree DEBUGARG(thisMorphNum));
13434 #if LOCAL_ASSERTION_PROP
13435 /*****************************************************************************
13437 * Kill all dependent assertions with regard to lclNum.
13441 void Compiler::fgKillDependentAssertions(unsigned lclNum
13442 DEBUGARG(GenTreePtr tree))
13444 LclVarDsc * varDsc = &lvaTable[lclNum];
13446 if (varDsc->lvPromoted)
13448 noway_assert(varTypeIsStruct(varDsc));
13450 // Kill the field locals.
13451 for (unsigned i = varDsc->lvFieldLclStart; i < varDsc->lvFieldLclStart + varDsc->lvFieldCnt; ++i)
13453 fgKillDependentAssertions(i DEBUGARG(tree));
13456 // Fall through to kill the struct local itself.
13459 /* All dependent assertions are killed here */
13461 ASSERT_TP killed = BitVecOps::MakeCopy(apTraits, GetAssertionDep(lclNum));
13465 AssertionIndex index = optAssertionCount;
13466 while (killed && (index > 0))
13468 if (BitVecOps::IsMember(apTraits, killed, index - 1))
13471 AssertionDsc* curAssertion = optGetAssertion(index);
13472 noway_assert((curAssertion->op1.lcl.lclNum == lclNum) ||
13473 ((curAssertion->op2.kind == O2K_LCLVAR_COPY) &&
13474 (curAssertion->op2.lcl.lclNum == lclNum)));
13477 printf("\nThe assignment ");
13479 printf(" using V%02u removes: ", curAssertion->op1.lcl.lclNum);
13480 optPrintAssertion(curAssertion);
13483 // Remove this bit from the killed mask
13484 BitVecOps::RemoveElemD(apTraits, killed, index - 1);
13486 optAssertionRemove(index);
13492 // killed mask should now be zero
13493 noway_assert(BitVecOps::IsEmpty(apTraits, killed));
13496 #endif // LOCAL_ASSERTION_PROP
13499 /*****************************************************************************
13501 * This function is called to complete the morphing of a tree node
13502 * It should only be called once for each node.
13503 * If DEBUG is defined the flag GTF_MORPHED is checked and updated,
13504 * to enforce the invariant that each node is only morphed once.
13505 * If LOCAL_ASSERTION_PROP is enabled the result tree may be replaced
13506 * by an equivalent tree.
13510 void Compiler::fgMorphTreeDone(GenTreePtr tree,
13511 GenTreePtr oldTree /* == NULL */
13512 DEBUGARG(int morphNum))
13515 if (verbose && treesBeforeAfterMorph)
13517 printf("\nfgMorphTree (after %d):\n", morphNum);
13519 printf(""); // in our logic this causes a flush
13523 if (!fgGlobalMorph)
13526 if ((oldTree != NULL) && (oldTree != tree))
13528 /* Ensure that we have morphed this node */
13529 assert((tree->gtFlags & GTF_MORPHED) && "ERROR: Did not morph this node!");
13532 TransferTestDataToNode(oldTree, tree);
13537 // Ensure that we haven't morphed this node already
13538 assert(((tree->gtFlags & GTF_MORPHED) == 0) && "ERROR: Already morphed this node!");
13541 if (tree->OperKind() & GTK_CONST)
13544 #if LOCAL_ASSERTION_PROP
13546 if (!optLocalAssertionProp)
13549 /* Do we have any active assertions? */
13551 if (optAssertionCount > 0)
13553 /* Is this an assignment to a local variable */
13555 if ((tree->OperKind() & GTK_ASGOP) &&
13556 (tree->gtOp.gtOp1->gtOper == GT_LCL_VAR || tree->gtOp.gtOp1->gtOper == GT_LCL_FLD))
13558 unsigned op1LclNum = tree->gtOp.gtOp1->gtLclVarCommon.gtLclNum; noway_assert(op1LclNum < lvaCount);
13559 fgKillDependentAssertions(op1LclNum DEBUGARG(tree));
13563 /* If this tree makes a new assertion - make it available */
13564 optAssertionGen(tree);
13566 #endif // LOCAL_ASSERTION_PROP
13571 /* Mark this node as being morphed */
13572 tree->gtFlags |= GTF_MORPHED;
13577 /*****************************************************************************
13579 * Check and fold blocks of type BBJ_COND and BBJ_SWITCH on constants
13580 * Returns true if we modified the flow graph
13583 bool Compiler::fgFoldConditional(BasicBlock * block)
13585 bool result = false;
13587 // We don't want to make any code unreachable
13588 if (opts.compDbgCode || opts.MinOpts())
13591 if (block->bbJumpKind == BBJ_COND)
13593 noway_assert(block->bbTreeList && block->bbTreeList->gtPrev);
13595 GenTreePtr stmt = block->bbTreeList->gtPrev;
13597 noway_assert(stmt->gtNext == NULL);
13599 if (stmt->gtStmt.gtStmtExpr->gtOper == GT_CALL)
13601 noway_assert(fgRemoveRestOfBlock);
13603 /* Unconditional throw - transform the basic block into a BBJ_THROW */
13604 fgConvertBBToThrowBB(block);
13606 /* Remove 'block' from the predecessor list of 'block->bbNext' */
13607 fgRemoveRefPred(block->bbNext, block);
13609 /* Remove 'block' from the predecessor list of 'block->bbJumpDest' */
13610 fgRemoveRefPred(block->bbJumpDest, block);
13615 printf("\nConditional folded at BB%02u\n", block->bbNum);
13616 printf("BB%02u becomes a BBJ_THROW\n", block->bbNum);
13622 noway_assert(stmt->gtStmt.gtStmtExpr->gtOper == GT_JTRUE);
13624 /* Did we fold the conditional */
13626 noway_assert(stmt->gtStmt.gtStmtExpr->gtOp.gtOp1);
13627 GenTreePtr cond; cond = stmt->gtStmt.gtStmtExpr->gtOp.gtOp1;
13629 if (cond->OperKind() & GTK_CONST)
13631 /* Yupee - we folded the conditional!
13632 * Remove the conditional statement */
13634 noway_assert(cond->gtOper == GT_CNS_INT);
13635 noway_assert((block->bbNext->countOfInEdges() > 0) &&
13636 (block->bbJumpDest->countOfInEdges() > 0));
13638 /* remove the statement from bbTreelist - No need to update
13639 * the reference counts since there are no lcl vars */
13640 fgRemoveStmt(block, stmt);
13642 // block is a BBJ_COND that we are folding the conditional for
13643 // bTaken is the path that will always be taken from block
13644 // bNotTaken is the path that will never be taken from block
13646 BasicBlock * bTaken;
13647 BasicBlock * bNotTaken;
13649 if (cond->gtIntCon.gtIconVal != 0)
13651 /* JTRUE 1 - transform the basic block into a BBJ_ALWAYS */
13652 block->bbJumpKind = BBJ_ALWAYS;
13653 bTaken = block->bbJumpDest;
13654 bNotTaken = block->bbNext;
13658 /* Unmark the loop if we are removing a backwards branch */
13659 /* dest block must also be marked as a loop head and */
13660 /* We must be able to reach the backedge block */
13661 if ((block->bbJumpDest->isLoopHead()) &&
13662 (block->bbJumpDest->bbNum <= block->bbNum) &&
13663 fgReachable(block->bbJumpDest, block))
13665 optUnmarkLoopBlocks(block->bbJumpDest, block);
13668 /* JTRUE 0 - transform the basic block into a BBJ_NONE */
13669 block->bbJumpKind = BBJ_NONE;
13670 noway_assert(!(block->bbFlags & BBF_NEEDS_GCPOLL));
13671 bTaken = block->bbNext;
13672 bNotTaken = block->bbJumpDest;
13675 if (fgHaveValidEdgeWeights)
13677 // We are removing an edge from block to bNotTaken
13678 // and we have already computed the edge weights, so
13679 // we will try to adjust some of the weights
13681 flowList * edgeTaken = fgGetPredForBlock(bTaken, block);
13682 BasicBlock * bUpdated = NULL; // non-NULL if we updated the weight of an internal block
13684 // We examine the taken edge (block -> bTaken)
13685 // if block has valid profile weight and bTaken does not we try to adjust bTaken's weight
13686 // else if bTaken has valid profile weight and block does not we try to adjust block's weight
13687 // We can only adjust the block weights when (the edge block -> bTaken) is the only edge into bTaken
13689 if (block->bbFlags & BBF_PROF_WEIGHT)
13691 // The edge weights for (block -> bTaken) are 100% of block's weight
13692 edgeTaken->flEdgeWeightMin = block->bbWeight;
13693 edgeTaken->flEdgeWeightMax = block->bbWeight;
13695 if ((bTaken->bbFlags & BBF_PROF_WEIGHT) == 0)
13697 if ((bTaken->countOfInEdges() == 1) || (bTaken->bbWeight < block->bbWeight))
13699 // Update the weight of bTaken
13700 bTaken->inheritWeight(block);
13705 else if (bTaken->bbFlags & BBF_PROF_WEIGHT)
13707 if (bTaken->countOfInEdges() == 1)
13709 // There is only one in edge to bTaken
13710 edgeTaken->flEdgeWeightMin = bTaken->bbWeight;
13711 edgeTaken->flEdgeWeightMax = bTaken->bbWeight;
13713 // Update the weight of block
13714 block->inheritWeight(bTaken);
13719 if (bUpdated != NULL)
13722 // Now fix the weights of the edges out of 'bUpdated'
13723 switch (bUpdated->bbJumpKind) {
13725 edge = fgGetPredForBlock(bUpdated->bbNext, bUpdated);
13726 edge->flEdgeWeightMax = bUpdated->bbWeight;
13729 edge = fgGetPredForBlock(bUpdated->bbNext, bUpdated);
13730 edge->flEdgeWeightMax = bUpdated->bbWeight;
13733 edge = fgGetPredForBlock(bUpdated->bbJumpDest, bUpdated);
13734 edge->flEdgeWeightMax = bUpdated->bbWeight;
13737 // We don't handle BBJ_SWITCH
13744 /* modify the flow graph */
13746 /* Remove 'block' from the predecessor list of 'bNotTaken' */
13747 fgRemoveRefPred(bNotTaken, block);
13752 printf("\nConditional folded at BB%02u\n", block->bbNum);
13753 printf("BB%02u becomes a %s", block->bbNum,
13754 block->bbJumpKind == BBJ_ALWAYS ? "BBJ_ALWAYS" : "BBJ_NONE");
13755 if (block->bbJumpKind == BBJ_ALWAYS)
13756 printf(" to BB%02u", block->bbJumpDest->bbNum);
13761 /* if the block was a loop condition we may have to modify
13762 * the loop table */
13764 for (unsigned loopNum = 0; loopNum < optLoopCount; loopNum++)
13766 /* Some loops may have been already removed by
13767 * loop unrolling or conditional folding */
13769 if (optLoopTable[loopNum].lpFlags & LPFLG_REMOVED)
13772 /* We are only interested in the loop bottom */
13774 if (optLoopTable[loopNum].lpBottom == block)
13776 if (cond->gtIntCon.gtIconVal == 0)
13778 /* This was a bogus loop (condition always false)
13779 * Remove the loop from the table */
13781 optLoopTable[loopNum].lpFlags |= LPFLG_REMOVED;
13785 printf("Removing loop L%02u (from BB%02u to BB%02u)\n\n",
13787 optLoopTable[loopNum].lpFirst ->bbNum,
13788 optLoopTable[loopNum].lpBottom->bbNum);
13798 else if (block->bbJumpKind == BBJ_SWITCH)
13800 noway_assert(block->bbTreeList && block->bbTreeList->gtPrev);
13802 GenTreePtr stmt = block->bbTreeList->gtPrev;
13804 noway_assert(stmt->gtNext == NULL);
13806 if (stmt->gtStmt.gtStmtExpr->gtOper == GT_CALL)
13808 noway_assert(fgRemoveRestOfBlock);
13810 /* Unconditional throw - transform the basic block into a BBJ_THROW */
13811 fgConvertBBToThrowBB(block);
13813 /* update the flow graph */
13815 unsigned jumpCnt = block->bbJumpSwt->bbsCount;
13816 BasicBlock * * jumpTab = block->bbJumpSwt->bbsDstTab;
13818 for (unsigned val = 0; val < jumpCnt; val++, jumpTab++)
13820 BasicBlock * curJump = *jumpTab;
13822 /* Remove 'block' from the predecessor list of 'curJump' */
13823 fgRemoveRefPred(curJump, block);
13829 printf("\nConditional folded at BB%02u\n", block->bbNum);
13830 printf("BB%02u becomes a BBJ_THROW\n", block->bbNum);
13837 noway_assert(stmt->gtStmt.gtStmtExpr->gtOper == GT_SWITCH);
13839 /* Did we fold the conditional */
13841 noway_assert(stmt->gtStmt.gtStmtExpr->gtOp.gtOp1);
13842 GenTreePtr cond; cond = stmt->gtStmt.gtStmtExpr->gtOp.gtOp1;
13844 if (cond->OperKind() & GTK_CONST)
13846 /* Yupee - we folded the conditional!
13847 * Remove the conditional statement */
13849 noway_assert(cond->gtOper == GT_CNS_INT);
13851 /* remove the statement from bbTreelist - No need to update
13852 * the reference counts since there are no lcl vars */
13853 fgRemoveStmt(block, stmt);
13855 /* modify the flow graph */
13857 /* Find the actual jump target */
13858 unsigned switchVal; switchVal = (unsigned)cond->gtIntCon.gtIconVal;
13859 unsigned jumpCnt; jumpCnt = block->bbJumpSwt->bbsCount;
13860 BasicBlock * * jumpTab; jumpTab = block->bbJumpSwt->bbsDstTab;
13861 bool foundVal; foundVal = false;
13863 for (unsigned val = 0; val < jumpCnt; val++, jumpTab++)
13865 BasicBlock * curJump = *jumpTab;
13867 assert (curJump->countOfInEdges() > 0);
13869 // If val matches switchVal or we are at the last entry and
13870 // we never found the switch value then set the new jump dest
13872 if ( (val == switchVal) || (!foundVal && (val == jumpCnt-1)))
13874 if (curJump != block->bbNext)
13876 /* transform the basic block into a BBJ_ALWAYS */
13877 block->bbJumpKind = BBJ_ALWAYS;
13878 block->bbJumpDest = curJump;
13880 //if we are jumping backwards, make sure we have a GC Poll.
13881 if (curJump->bbNum > block->bbNum)
13882 block->bbFlags &= ~BBF_NEEDS_GCPOLL;
13886 /* transform the basic block into a BBJ_NONE */
13887 block->bbJumpKind = BBJ_NONE;
13888 block->bbFlags &= ~BBF_NEEDS_GCPOLL;
13894 /* Remove 'block' from the predecessor list of 'curJump' */
13895 fgRemoveRefPred(curJump, block);
13901 printf("\nConditional folded at BB%02u\n", block->bbNum);
13902 printf("BB%02u becomes a %s", block->bbNum,
13903 block->bbJumpKind == BBJ_ALWAYS ? "BBJ_ALWAYS" : "BBJ_NONE");
13904 if (block->bbJumpKind == BBJ_ALWAYS)
13905 printf(" to BB%02u", block->bbJumpDest->bbNum);
13917 //*****************************************************************************
13919 // Morphs a single statement in a block.
13920 // Can be called anytime, unlike fgMorphStmts() which should only be called once.
13922 // Returns true if 'stmt' was removed from the block.
13923 // Returns false if 'stmt' is still in the block (even if other statements were removed).
13926 bool Compiler::fgMorphBlockStmt(BasicBlock * block,
13928 DEBUGARG(const char * msg) )
13930 noway_assert(stmt->gtOper == GT_STMT);
13933 compCurStmt = stmt;
13935 GenTreePtr morph = fgMorphTree(stmt->gtStmt.gtStmtExpr);
13937 // Bug 1106830 - During the CSE phase we can't just remove
13938 // morph->gtOp.gtOp2 as it could contain CSE expressions.
13939 // This leads to a noway_assert in OptCSE.cpp when
13940 // searching for the removed CSE ref. (using gtFindLink)
13942 if (!optValnumCSE_phase)
13944 /* Check for morph as a GT_COMMA with an unconditional throw */
13945 if (fgIsCommaThrow(morph, true))
13950 printf("Folding a top-level fgIsCommaThrow stmt\n");
13951 printf("Removing op2 as unreachable:\n");
13952 gtDispTree(morph->gtOp.gtOp2);
13956 /* Use the call as the new stmt */
13957 morph = morph->gtOp.gtOp1;
13958 noway_assert(morph->gtOper == GT_CALL);
13961 /* we can get a throw as a statement root*/
13962 if (fgIsThrow(morph))
13967 printf("We have a top-level fgIsThrow stmt\n");
13968 printf("Removing the rest of block as unreachable:\n");
13971 noway_assert((morph->gtFlags & GTF_COLON_COND) == 0);
13972 fgRemoveRestOfBlock = true;
13976 stmt->gtStmt.gtStmtExpr = morph;
13978 /* Can the entire tree be removed ? */
13980 bool removedStmt = fgCheckRemoveStmt(block, stmt);
13982 /* Or this is the last statement of a conditional branch that was just folded */
13984 if ((!removedStmt) && (stmt->gtNext == NULL) && !fgRemoveRestOfBlock)
13986 if (fgFoldConditional(block))
13988 if (block->bbJumpKind != BBJ_THROW)
13989 removedStmt = true;
13995 /* Have to re-do the evaluation order since for example
13996 * some later code does not expect constants as op1 */
13997 gtSetStmtInfo(stmt);
13999 /* Have to re-link the nodes for this statement */
14000 fgSetStmtSeq(stmt);
14006 printf("%s %s tree:\n", msg, (removedStmt ? "removed" : "morphed"));
14012 if (fgRemoveRestOfBlock)
14014 /* Remove the rest of the stmts in the block */
14016 while (stmt->gtNext)
14018 stmt = stmt->gtNext;
14019 noway_assert(stmt->gtOper == GT_STMT);
14021 fgRemoveStmt(block, stmt);
14024 // The rest of block has been removed
14025 // and we will always throw an exception
14027 // Update succesors of block
14028 fgRemoveBlockAsPred(block);
14030 // For compDbgCode, we prepend an empty BB as the firstBB, it is BBJ_NONE.
14031 // We should not convert it to a ThrowBB.
14032 if ((block != fgFirstBB) || ((fgFirstBB->bbFlags & BBF_INTERNAL) == 0) ) {
14033 // Convert block to a throw bb
14034 fgConvertBBToThrowBB(block);
14040 printf("\n%s Block BB%02u becomes a throw block.\n", msg, block->bbNum);
14043 fgRemoveRestOfBlock = false;
14046 return removedStmt;
14049 /*****************************************************************************
14051 * Morph the statements of the given block.
14052 * This function should be called just once for a block. Use fgMorphBlockStmt()
14053 * for reentrant calls.
14056 void Compiler::fgMorphStmts(BasicBlock * block,
14057 bool * mult, bool * lnot, bool * loadw)
14059 fgRemoveRestOfBlock = false;
14061 noway_assert(fgExpandInline == false);
14063 /* Make the current basic block address available globally */
14067 *mult = *lnot = *loadw = false;
14069 fgCurrentlyInUseArgTemps = hashBv::Create(this);
14071 GenTreePtr stmt, prev;
14072 for (stmt = block->bbTreeList, prev = NULL;
14074 prev = stmt->gtStmt.gtStmtExpr,
14075 stmt = stmt->gtNext)
14077 noway_assert(stmt->gtOper == GT_STMT);
14079 if (fgRemoveRestOfBlock)
14081 fgRemoveStmt(block, stmt);
14084 #ifdef FEATURE_SIMD
14085 if (!opts.MinOpts() &&
14086 stmt->gtStmt.gtStmtExpr->TypeGet() == TYP_FLOAT &&
14087 stmt->gtStmt.gtStmtExpr->OperGet() == GT_ASG)
14089 fgMorphCombineSIMDFieldAssignments(block, stmt);
14093 fgMorphStmt = stmt;
14094 compCurStmt = stmt;
14095 GenTreePtr tree = stmt->gtStmt.gtStmtExpr;
14099 if (stmt == block->bbTreeList)
14100 block->bbStmtNum = compCurStmtNum; // Set the block->bbStmtNum
14102 unsigned oldHash = verbose ? gtHashValue(tree) : DUMMY_INIT(~0);
14106 printf("\nfgMorphTree BB%02u, stmt %d (before)\n", block->bbNum, compCurStmtNum);
14111 /* Morph this statement tree */
14113 GenTreePtr morph = fgMorphTree(tree);
14115 // mark any outgoing arg temps as free so we can reuse them in the next statement.
14117 fgCurrentlyInUseArgTemps->ZeroAll();
14119 // Has fgMorphStmt been sneakily changed ?
14121 if (stmt->gtStmt.gtStmtExpr != tree)
14123 /* This must be tailcall. Ignore 'morph' and carry on with
14124 the tail-call node */
14126 morph = stmt->gtStmt.gtStmtExpr;
14127 noway_assert(compTailCallUsed);
14128 noway_assert((morph->gtOper == GT_CALL) && morph->AsCall()->IsTailCall());
14129 noway_assert(stmt->gtNext == NULL);
14131 GenTreeCall* call = morph->AsCall();
14133 // - a tail call dispatched via helper in which case block will be ending with BBJ_THROW or
14134 // - a fast call made as jmp in which case block will be ending with BBJ_RETURN and marked as containing a jmp.
14135 noway_assert((call->IsTailCallViaHelper() && (compCurBB->bbJumpKind == BBJ_THROW)) ||
14136 (call->IsFastTailCall() && (compCurBB->bbJumpKind == BBJ_RETURN) && (compCurBB->bbFlags & BBF_HAS_JMP)));
14138 else if (block != compCurBB)
14140 /* This must be a tail call that caused a GCPoll to get
14141 injected. We haven't actually morphed the call yet
14142 but the flag still got set, clear it here... */
14145 tree->gtFlags &= ~GTF_MORPHED;
14147 noway_assert(compTailCallUsed);
14148 noway_assert((tree->gtOper == GT_CALL) && tree->AsCall()->IsTailCall());
14149 noway_assert(stmt->gtNext == NULL);
14151 GenTreeCall* call = morph->AsCall();
14154 // - a tail call dispatched via helper in which case block will be ending with BBJ_THROW or
14155 // - a fast call made as jmp in which case block will be ending with BBJ_RETURN and marked as containing a jmp.
14156 noway_assert((call->IsTailCallViaHelper() && (compCurBB->bbJumpKind == BBJ_THROW)) ||
14157 (call->IsFastTailCall() && (compCurBB->bbJumpKind == BBJ_RETURN) && (compCurBB->bbFlags & BBF_HAS_JMP)));
14161 if (compStressCompile(STRESS_CLONE_EXPR, 30))
14163 // Clone all the trees to stress gtCloneExpr()
14167 printf("\nfgMorphTree (stressClone from):\n");
14171 morph = gtCloneExpr(morph);
14172 noway_assert(morph);
14176 printf("\nfgMorphTree (stressClone to):\n");
14181 /* If the hash value changes. we modified the tree during morphing */
14184 unsigned newHash = gtHashValue(morph);
14185 if (newHash != oldHash)
14187 printf("\nfgMorphTree BB%02u, stmt %d (after)\n", block->bbNum, compCurStmtNum);
14193 /* Check for morph as a GT_COMMA with an unconditional throw */
14194 if (!gtIsActiveCSE_Candidate(morph) && fgIsCommaThrow(morph, true))
14196 /* Use the call as the new stmt */
14197 morph = morph->gtOp.gtOp1;
14198 noway_assert(morph->gtOper == GT_CALL);
14199 noway_assert((morph->gtFlags & GTF_COLON_COND) == 0);
14201 fgRemoveRestOfBlock = true;
14204 stmt->gtStmt.gtStmtExpr = tree = morph;
14206 noway_assert(fgPtrArgCntCur == 0);
14208 if (fgRemoveRestOfBlock)
14211 /* Has the statement been optimized away */
14213 if (fgCheckRemoveStmt(block, stmt))
14216 /* Check if this block ends with a conditional branch that can be folded */
14218 if (fgFoldConditional(block))
14221 if (ehBlockHasExnFlowDsc(block))
14224 #if OPT_MULT_ADDSUB
14226 /* Note whether we have two or more +=/-= operators in a row */
14228 if (tree->gtOper == GT_ASG_ADD ||
14229 tree->gtOper == GT_ASG_SUB)
14231 if (prev && prev->gtOper == tree->gtOper)
14237 /* Note "x = a[i] & icon" followed by "x |= a[i] << 8" */
14239 if (tree->gtOper == GT_ASG_OR &&
14241 prev->gtOper == GT_ASG)
14247 if (fgRemoveRestOfBlock)
14249 if ((block->bbJumpKind == BBJ_COND) || (block->bbJumpKind == BBJ_SWITCH))
14251 GenTreePtr first = block->bbTreeList; noway_assert(first);
14252 GenTreePtr last = first->gtPrev; noway_assert(last && last->gtNext == NULL);
14253 GenTreePtr lastStmt = last->gtStmt.gtStmtExpr;
14255 if (((block->bbJumpKind == BBJ_COND ) && (lastStmt->gtOper == GT_JTRUE )) ||
14256 ((block->bbJumpKind == BBJ_SWITCH) && (lastStmt->gtOper == GT_SWITCH)) )
14258 GenTreePtr op1 = lastStmt->gtOp.gtOp1;
14260 if (op1->OperKind() & GTK_RELOP)
14262 /* Unmark the comparison node with GTF_RELOP_JMP_USED */
14263 op1->gtFlags &= ~GTF_RELOP_JMP_USED;
14266 last->gtStmt.gtStmtExpr = fgMorphTree(op1);
14270 /* Mark block as a BBJ_THROW block */
14271 fgConvertBBToThrowBB(block);
14274 noway_assert(fgExpandInline == false);
14276 #if FEATURE_FASTTAILCALL
14277 GenTreePtr recursiveTailCall = nullptr;
14278 if (block->endsWithTailCallConvertibleToLoop(this, &recursiveTailCall))
14280 fgMorphRecursiveFastTailCallIntoLoop(block, recursiveTailCall->AsCall());
14285 compCurBB = (BasicBlock*)INVALID_POINTER_VALUE;
14288 // Reset this back so that it doesn't leak out impacting other blocks
14289 fgRemoveRestOfBlock = false;
14292 /*****************************************************************************
14294 * Morph the blocks of the method.
14295 * Returns true if the basic block list is modified.
14296 * This function should be called just once.
14299 void Compiler::fgMorphBlocks()
14303 printf("\n*************** In fgMorphBlocks()\n");
14306 /* Since fgMorphTree can be called after various optimizations to re-arrange
14307 * the nodes we need a global flag to signal if we are during the one-pass
14308 * global morphing */
14310 fgGlobalMorph = true;
14312 #if LOCAL_ASSERTION_PROP
14314 // Local assertion prop is enabled if we are optimized
14316 optLocalAssertionProp = (!opts.compDbgCode && !opts.MinOpts());
14318 if (optLocalAssertionProp)
14321 // Initialize for local assertion prop
14323 optAssertionInit(true);
14325 #elif ASSERTION_PROP
14327 // If LOCAL_ASSERTION_PROP is not set
14328 // and we have global assertion prop
14329 // then local assertion prop is always off
14331 optLocalAssertionProp = false;
14335 /*-------------------------------------------------------------------------
14336 * Process all basic blocks in the function
14339 BasicBlock * block = fgFirstBB; noway_assert(block);
14342 compCurStmtNum = 0;
14347 #if OPT_MULT_ADDSUB
14355 bool loadw = false;
14359 printf("\nMorphing BB%02u of '%s'\n", block->bbNum, info.compFullName);
14362 #if LOCAL_ASSERTION_PROP
14363 if (optLocalAssertionProp)
14366 // Clear out any currently recorded assertion candidates
14367 // before processing each basic block,
14368 // also we must handle QMARK-COLON specially
14370 optAssertionReset(0);
14374 /* Process all statement trees in the basic block */
14378 fgMorphStmts(block, &mult, &lnot, &loadw);
14380 #if OPT_MULT_ADDSUB
14382 if (mult && (opts.compFlags & CLFLG_TREETRANS) &&
14383 !opts.compDbgCode && !opts.MinOpts())
14385 for (tree = block->bbTreeList; tree; tree = tree->gtNext)
14387 noway_assert(tree->gtOper == GT_STMT);
14388 GenTreePtr last = tree->gtStmt.gtStmtExpr;
14390 if (last->gtOper == GT_ASG_ADD ||
14391 last->gtOper == GT_ASG_SUB)
14396 GenTreePtr dst1 = last->gtOp.gtOp1;
14397 GenTreePtr src1 = last->gtOp.gtOp2;
14399 if (!last->IsCnsIntOrI())
14402 if (dst1->gtOper != GT_LCL_VAR)
14404 if (!src1->IsCnsIntOrI())
14412 /* Look at the next statement */
14414 temp = tree->gtNext;
14418 noway_assert(temp->gtOper == GT_STMT);
14419 next = temp->gtStmt.gtStmtExpr;
14421 if (next->gtOper != last->gtOper)
14423 if (next->gtType != last->gtType)
14426 dst2 = next->gtOp.gtOp1;
14427 src2 = next->gtOp.gtOp2;
14429 if (dst2->gtOper != GT_LCL_VAR)
14431 if (dst2->gtLclVarCommon.gtLclNum != dst1->gtLclVarCommon.gtLclNum)
14434 if (!src2->IsCnsIntOrI())
14437 if (last->gtOverflow() != next->gtOverflow())
14440 const ssize_t i1 = src1->gtIntCon.gtIconVal;
14441 const ssize_t i2 = src2->gtIntCon.gtIconVal;
14442 const ssize_t itemp = i1 + i2;
14444 /* if the operators are checking for overflow, check for overflow of the operands */
14446 if (next->gtOverflow())
14448 if (next->TypeGet() == TYP_LONG)
14450 if (next->gtFlags & GTF_UNSIGNED)
14452 ClrSafeInt<UINT64> si1(i1);
14453 if ((si1 + ClrSafeInt<UINT64>(i2)).IsOverflow())
14458 ClrSafeInt<INT64> si1(i1);
14459 if ((si1 + ClrSafeInt<INT64>(i2)).IsOverflow())
14463 else if (next->gtFlags & GTF_UNSIGNED)
14465 ClrSafeInt<UINT32> si1(i1);
14466 if ((si1 + ClrSafeInt<UINT32>(i2)).IsOverflow())
14471 ClrSafeInt<INT32> si1(i1);
14472 if ((si1 + ClrSafeInt<INT32>(i2)).IsOverflow())
14477 /* Fold the two increments/decrements into one */
14479 src1->gtIntCon.gtIconVal = itemp;
14480 #ifdef _TARGET_64BIT_
14481 if (src1->gtType == TYP_INT)
14483 src1->AsIntCon()->TruncateOrSignExtend32();
14485 #endif //_TARGET_64BIT_
14487 /* Remove the second statement completely */
14489 noway_assert(tree->gtNext == temp);
14490 noway_assert(temp->gtPrev == tree);
14494 noway_assert(temp->gtNext->gtPrev == temp);
14496 temp->gtNext->gtPrev = tree;
14497 tree->gtNext = temp->gtNext;
14503 noway_assert(block->bbTreeList->gtPrev == temp);
14505 block->bbTreeList->gtPrev = tree;
14518 /* Are we using a single return block? */
14520 if (block->bbJumpKind == BBJ_RETURN)
14522 if ((genReturnBB != nullptr) &&
14523 (genReturnBB != block) &&
14524 ((block->bbFlags & BBF_HAS_JMP) == 0))
14526 /* We'll jump to the genReturnBB */
14528 #if !defined(_TARGET_X86_)
14529 if (info.compFlags & CORINFO_FLG_SYNCH)
14531 fgConvertSyncReturnToLeave(block);
14534 #endif // !_TARGET_X86_
14536 block->bbJumpKind = BBJ_ALWAYS;
14537 block->bbJumpDest = genReturnBB;
14541 // Note 1: A block is not guaranteed to have a last stmt if its jump kind is BBJ_RETURN.
14542 // For example a method returning void could have an empty block with jump kind BBJ_RETURN.
14543 // Such blocks do materialize as part of in-lining.
14545 // Note 2: A block with jump kind BBJ_RETURN does not necessarily need to end with GT_RETURN.
14546 // It could end with a tail call or rejected tail call or monitor.exit or a GT_INTRINSIC.
14547 // For now it is safe to explicitly check whether last stmt is GT_RETURN if genReturnLocal
14550 // TODO: Need to characterize the last top level stmt of a block ending with BBJ_RETURN.
14552 GenTreePtr last = (block->bbTreeList != nullptr) ? block->bbTreeList->gtPrev : nullptr;
14553 GenTreePtr ret = (last != nullptr) ? last->gtStmt.gtStmtExpr : nullptr;
14555 //replace the GT_RETURN node to be a GT_ASG that stores the return value into genReturnLocal.
14556 if (genReturnLocal != BAD_VAR_NUM)
14558 // Method must be returning a value other than TYP_VOID.
14559 noway_assert(compMethodHasRetVal());
14561 // This block must be ending with a GT_RETURN
14562 noway_assert(last != nullptr);
14563 noway_assert(last->gtOper == GT_STMT);
14564 noway_assert(last->gtNext == nullptr);
14565 noway_assert(ret != nullptr);
14567 // GT_RETURN must have non-null operand as the method is returning the value assigned to genReturnLocal
14568 noway_assert(ret->OperGet() == GT_RETURN);
14569 noway_assert(ret->gtGetOp1() != nullptr);
14570 noway_assert(ret->gtGetOp2() == nullptr);
14572 last->gtStmt.gtStmtExpr = gtNewTempAssign(genReturnLocal, ret->gtGetOp1());
14574 //make sure that copy-prop ignores this assignment.
14575 last->gtStmt.gtStmtExpr->gtFlags |= GTF_DONT_CSE;
14577 else if (ret != nullptr && ret->OperGet() == GT_RETURN)
14579 // This block ends with a GT_RETURN
14580 noway_assert(last != nullptr);
14581 noway_assert(last->gtOper == GT_STMT);
14582 noway_assert(last->gtNext == nullptr);
14584 // Must be a void GT_RETURN with null operand; delete it as this block branches to oneReturn block
14585 noway_assert(ret->TypeGet() == TYP_VOID);
14586 noway_assert(ret->gtGetOp1() == nullptr);
14587 noway_assert(ret->gtGetOp2() == nullptr);
14589 fgRemoveStmt(block, last);
14595 printf("morph BB%02u to point at onereturn. New block is\n",
14597 fgTableDispBasicBlock(block);
14603 block = block->bbNext;
14607 /* We are done with the global morphing phase */
14609 fgGlobalMorph = false;
14614 fgDispBasicBlocks(true);
14620 /*****************************************************************************
14622 * Make some decisions about the kind of code to generate.
14625 void Compiler::fgSetOptions()
14628 /* Should we force fully interruptible code ? */
14631 if (JitConfig.JitFullyInt() ||
14632 compStressCompile(STRESS_GENERIC_VARN, 30))
14634 noway_assert(!codeGen->isGCTypeFixed());
14635 genInterruptible = true;
14639 #ifdef DEBUGGING_SUPPORT
14640 if (opts.compDbgCode)
14642 assert(!codeGen->isGCTypeFixed());
14643 genInterruptible = true; // debugging is easier this way ...
14647 /* Assume we won't need an explicit stack frame if this is allowed */
14650 // CORINFO_HELP_TAILCALL won't work with localloc because of the restoring of
14651 // the callee-saved registers.
14652 noway_assert(!compTailCallUsed || !compLocallocUsed);
14654 if (compLocallocUsed)
14655 codeGen->setFramePointerRequired(true);
14657 #ifdef _TARGET_X86_
14659 if (compTailCallUsed)
14660 codeGen->setFramePointerRequired(true);
14662 #endif // _TARGET_X86_
14664 if (!opts.genFPopt)
14665 codeGen->setFramePointerRequired(true);
14667 // Assert that the EH table has been initialized by now. Note that
14668 // compHndBBtabAllocCount never decreases; it is a high-water mark
14669 // of table allocation. In contrast, compHndBBtabCount does shrink
14670 // if we delete a dead EH region, and if it shrinks to zero, the
14671 // table pointer compHndBBtab is unreliable.
14672 assert(compHndBBtabAllocCount >= info.compXcptnsCount);
14674 #ifdef _TARGET_X86_
14676 // Note: this case, and the !X86 case below, should both use the
14677 // !X86 path. This would require a few more changes for X86 to use
14678 // compHndBBtabCount (the current number of EH clauses) instead of
14679 // info.compXcptnsCount (the number of EH clauses in IL), such as
14680 // in ehNeedsShadowSPslots(). This is because sometimes the IL has
14681 // an EH clause that we delete as statically dead code before we
14682 // get here, leaving no EH clauses left, and thus no requirement
14683 // to use a frame pointer because of EH. But until all the code uses
14684 // the same test, leave info.compXcptnsCount here.
14685 if (info.compXcptnsCount > 0)
14686 codeGen->setFramePointerRequiredEH(true);
14688 #else // !_TARGET_X86_
14690 if (compHndBBtabCount > 0)
14691 codeGen->setFramePointerRequiredEH(true);
14693 #endif // _TARGET_X86_
14695 // fpPtrArgCntMax records the maximum number of pushed arguments
14696 // Depending upon this value of the maximum number of pushed arguments
14697 // we may need to use an EBP frame or be partially interuptible
14700 if (!compCanEncodePtrArgCntMax())
14704 printf("Too many pushed arguments for fully interruptible encoding, marking method as partially interruptible\n");
14706 genInterruptible = false;
14708 if (fgPtrArgCntMax >= sizeof(unsigned))
14712 printf("Too many pushed arguments for an ESP based encoding, forcing an EBP frame\n");
14714 codeGen->setFramePointerRequiredGCInfo(true);
14717 if (info.compCallUnmanaged)
14719 codeGen->setFramePointerRequired(true); // Setup of Pinvoke frame currently requires an EBP style frame
14722 if (info.compPublishStubParam)
14724 codeGen->setFramePointerRequiredGCInfo(true);
14727 if (opts.compNeedSecurityCheck)
14729 codeGen->setFramePointerRequiredGCInfo(true);
14731 #ifndef JIT32_GCENCODER
14733 // The decoder only reports objects in frames with exceptions if the frame
14734 // is fully interruptible.
14735 // Even if there is no catch or other way to resume execution in this frame
14736 // the VM requires the security object to remain alive until later, so
14737 // Frames with security objects must be fully interruptible.
14738 genInterruptible = true;
14740 #endif // JIT32_GCENCODER
14743 if (compIsProfilerHookNeeded())
14745 codeGen->setFramePointerRequired(true);
14748 if (info.compIsVarArgs)
14750 // Code that initializes lvaVarargsBaseOfStkArgs requires this to be EBP relative.
14751 codeGen->setFramePointerRequiredGCInfo(true);
14754 if (lvaReportParamTypeArg())
14756 codeGen->setFramePointerRequiredGCInfo(true);
14759 // printf("method will %s be fully interruptible\n", genInterruptible ? " " : "not");
14763 /*****************************************************************************/
14765 GenTreePtr Compiler::fgInitThisClass()
14767 noway_assert(!compIsForInlining());
14769 CORINFO_LOOKUP_KIND kind = info.compCompHnd->getLocationOfThisType(info.compMethodHnd);
14771 if (!kind.needsRuntimeLookup)
14773 return fgGetSharedCCtor(info.compClassHnd);
14777 // Collectible types requires that for shared generic code, if we use the generic context paramter
14778 // that we report it. (This is a conservative approach, we could detect some cases particularly when the
14779 // context parameter is this that we don't need the eager reporting logic.)
14780 lvaGenericsContextUsed = true;
14782 switch (kind.runtimeLookupKind)
14784 case CORINFO_LOOKUP_THISOBJ :
14785 // This code takes a this pointer; but we need to pass the static method desc to get the right point in the hierarchy
14787 GenTreePtr vtTree = gtNewLclvNode(info.compThisArg, TYP_REF);
14788 // Vtable pointer of this object
14789 vtTree = gtNewOperNode(GT_IND, TYP_I_IMPL, vtTree);
14790 vtTree->gtFlags |= GTF_EXCEPT; // Null-pointer exception
14791 GenTreePtr methodHnd = gtNewIconEmbMethHndNode(info.compMethodHnd);
14793 return gtNewHelperCallNode(CORINFO_HELP_INITINSTCLASS,
14795 gtNewArgList(vtTree, methodHnd));
14799 case CORINFO_LOOKUP_CLASSPARAM :
14801 GenTreePtr vtTree = gtNewLclvNode(info.compTypeCtxtArg, TYP_I_IMPL);
14802 return gtNewHelperCallNode(CORINFO_HELP_INITCLASS,
14804 gtNewArgList(vtTree));
14807 case CORINFO_LOOKUP_METHODPARAM :
14809 GenTreePtr methHndTree = gtNewLclvNode(info.compTypeCtxtArg, TYP_I_IMPL);
14810 return gtNewHelperCallNode(CORINFO_HELP_INITINSTCLASS,
14812 gtNewArgList(gtNewIconNode(0),methHndTree));
14818 noway_assert(!"Unknown LOOKUP_KIND");
14824 /*****************************************************************************
14826 * Tree walk callback to make sure no GT_QMARK nodes are present in the tree,
14827 * except for the allowed ? 1 : 0; pattern.
14829 Compiler::fgWalkResult Compiler::fgAssertNoQmark(GenTreePtr* tree, fgWalkData* data)
14831 if ((*tree)->OperGet() == GT_QMARK)
14833 fgCheckQmarkAllowedForm(*tree);
14835 return WALK_CONTINUE;
14838 void Compiler::fgCheckQmarkAllowedForm(GenTree* tree)
14840 assert(tree->OperGet() == GT_QMARK);
14841 #ifndef LEGACY_BACKEND
14842 assert(!"Qmarks beyond morph disallowed.");
14843 #else // LEGACY_BACKEND
14844 GenTreePtr colon = tree->gtOp.gtOp2;
14846 assert(colon->gtOp.gtOp1->gtOper == GT_CNS_INT);
14847 assert(colon->gtOp.gtOp1->AsIntCon()->IconValue() == 0);
14849 assert(colon->gtOp.gtOp2->gtOper == GT_CNS_INT);
14850 assert(colon->gtOp.gtOp2->AsIntCon()->IconValue() == 1);
14851 #endif // LEGACY_BACKEND
14854 /*****************************************************************************
14856 * Verify that the importer has created GT_QMARK nodes in a way we can
14857 * process them. The following is allowed:
14859 * 1. A top level qmark. Top level qmark is of the form:
14860 * a) (bool) ? (void) : (void) OR
14861 * b) V0N = (bool) ? (type) : (type)
14863 * 2. Recursion is allowed at the top level, i.e., a GT_QMARK can be a child
14864 * of either op1 of colon or op2 of colon but not a child of any other
14867 void Compiler::fgPreExpandQmarkChecks(GenTreePtr expr)
14869 GenTreePtr topQmark = fgGetTopLevelQmark(expr);
14871 // If the top level Qmark is null, then scan the tree to make sure
14872 // there are no qmarks within it.
14873 if (topQmark == NULL)
14875 fgWalkTreePre(&expr, Compiler::fgAssertNoQmark, NULL);
14879 // We could probably expand the cond node also, but don't think the extra effort is necessary,
14880 // so let's just assert the cond node of a top level qmark doesn't have further top level qmarks.
14881 fgWalkTreePre(&topQmark->gtOp.gtOp1, Compiler::fgAssertNoQmark, NULL);
14883 fgPreExpandQmarkChecks(topQmark->gtOp.gtOp2->gtOp.gtOp1);
14884 fgPreExpandQmarkChecks(topQmark->gtOp.gtOp2->gtOp.gtOp2);
14889 /*****************************************************************************
14891 * Get the top level GT_QMARK node in a given "expr", return NULL if such a
14892 * node is not present. If the top level GT_QMARK node is assigned to a
14893 * GT_LCL_VAR, then return the lcl node in ppDst.
14896 GenTreePtr Compiler::fgGetTopLevelQmark(GenTreePtr expr, GenTreePtr* ppDst /* = NULL */)
14903 GenTreePtr topQmark = NULL;
14904 if (expr->gtOper == GT_QMARK)
14908 else if (expr->gtOper == GT_ASG &&
14909 expr->gtOp.gtOp2->gtOper == GT_QMARK &&
14910 expr->gtOp.gtOp1->gtOper == GT_LCL_VAR)
14912 topQmark = expr->gtOp.gtOp2;
14915 *ppDst = expr->gtOp.gtOp1;
14922 /*********************************************************************************
14924 * For a castclass helper call,
14925 * Importer creates the following tree:
14926 * tmp = (op1 == null) ? op1 : ((*op1 == (cse = op2, cse)) ? op1 : helper());
14928 * This method splits the qmark expression created by the importer into the
14929 * following blocks: (block, asg, cond1, cond2, helper, remainder)
14930 * Notice that op1 is the result for both the conditions. So we coalesce these
14931 * assignments into a single block instead of two blocks resulting a nested diamond.
14933 * +---------->-----------+
14937 * block-->asg-->cond1--+-->cond2--+-->helper--+-->remainder
14939 * We expect to achieve the following codegen:
14940 * mov rsi, rdx tmp = op1 // asgBlock
14941 * test rsi, rsi goto skip if tmp == null ? // cond1Block
14943 * mov rcx, 0x76543210 cns = op2 // cond2Block
14944 * cmp qword ptr [rsi], rcx goto skip if *tmp == op2
14946 * call CORINFO_HELP_CHKCASTCLASS_SPECIAL tmp = helper(cns, tmp) // helperBlock
14948 * SKIP: // remainderBlock
14949 * tmp has the result.
14952 void Compiler::fgExpandQmarkForCastInstOf(BasicBlock* block, GenTreePtr stmt)
14957 printf("\nExpanding CastInstOf qmark in BB%02u (before)\n", block->bbNum);
14958 fgDispBasicBlocks(block, block, true);
14962 GenTreePtr expr = stmt->gtStmt.gtStmtExpr;
14964 GenTreePtr dst = nullptr;
14965 GenTreePtr qmark = fgGetTopLevelQmark(expr, &dst);
14966 noway_assert(dst != nullptr);
14968 assert(qmark->gtFlags & GTF_QMARK_CAST_INSTOF);
14970 // Get cond, true, false exprs for the qmark.
14971 GenTreePtr condExpr = qmark->gtGetOp1();
14972 GenTreePtr trueExpr = qmark->gtGetOp2()->AsColon()->ThenNode();
14973 GenTreePtr falseExpr = qmark->gtGetOp2()->AsColon()->ElseNode();
14975 // Get cond, true, false exprs for the nested qmark.
14976 GenTreePtr nestedQmark = falseExpr;
14977 GenTreePtr cond2Expr;
14978 GenTreePtr true2Expr;
14979 GenTreePtr false2Expr;
14981 if (nestedQmark->gtOper == GT_QMARK)
14983 cond2Expr = nestedQmark->gtGetOp1();
14984 true2Expr = nestedQmark->gtGetOp2()->AsColon()->ThenNode();
14985 false2Expr = nestedQmark->gtGetOp2()->AsColon()->ElseNode();
14987 assert(cond2Expr->gtFlags & GTF_RELOP_QMARK);
14988 cond2Expr->gtFlags &= ~GTF_RELOP_QMARK;
14992 // This is a rare case that arises when we are doing minopts and encounter isinst of null
14993 // gtFoldExpr was still is able to optimize away part of the tree (but not all).
14994 // That means it does not match our pattern.
14996 // Rather than write code to handle this case, just fake up some nodes to make it match the common
14997 // case. Synthesize a comparison that is always true, and for the result-on-true, use the
14998 // entire subtree we expected to be the nested question op.
15000 cond2Expr = gtNewOperNode(GT_EQ, TYP_INT, gtNewIconNode(0, TYP_I_IMPL), gtNewIconNode(0, TYP_I_IMPL));
15001 true2Expr = nestedQmark;
15002 false2Expr = gtNewIconNode(0, TYP_I_IMPL);
15004 assert(false2Expr->OperGet() == trueExpr->OperGet());
15006 // Clear flags as they are now going to be part of JTRUE.
15007 assert(condExpr->gtFlags & GTF_RELOP_QMARK);
15008 condExpr->gtFlags &= ~GTF_RELOP_QMARK;
15010 // Create the chain of blocks. See method header comment.
15011 // The order of blocks after this is the following:
15012 // block ... asgBlock ... cond1Block ... cond2Block ... helperBlock ... remainderBlock
15014 // We need to remember flags that exist on 'block' that we want to propagate to 'remainderBlock',
15015 // if they are going to be cleared by fgSplitBlockAfterStatement(). We currently only do this only
15016 // for the GC safe point bit, the logic being that if 'block' was marked gcsafe, then surely
15017 // remainderBlock will still be GC safe.
15018 unsigned propagateFlags = block->bbFlags & BBF_GC_SAFE_POINT;
15019 BasicBlock* remainderBlock = fgSplitBlockAfterStatement(block, stmt);
15020 fgRemoveRefPred(remainderBlock, block); // We're going to put more blocks between block and remainderBlock.
15022 BasicBlock* helperBlock = fgNewBBafter(BBJ_NONE, block, true);
15023 BasicBlock* cond2Block = fgNewBBafter(BBJ_COND, block, true);
15024 BasicBlock* cond1Block = fgNewBBafter(BBJ_COND, block, true);
15025 BasicBlock* asgBlock = fgNewBBafter(BBJ_NONE, block, true);
15027 remainderBlock->bbFlags |= BBF_JMP_TARGET | BBF_HAS_LABEL | propagateFlags;
15029 // These blocks are only internal if 'block' is (but they've been set as internal by fgNewBBafter).
15030 // If they're not internal, mark them as imported to avoid asserts about un-imported blocks.
15031 if ((block->bbFlags & BBF_INTERNAL) == 0)
15033 helperBlock->bbFlags &= ~BBF_INTERNAL;
15034 cond2Block->bbFlags &= ~BBF_INTERNAL;
15035 cond1Block->bbFlags &= ~BBF_INTERNAL;
15036 asgBlock->bbFlags &= ~BBF_INTERNAL;
15037 helperBlock->bbFlags |= BBF_IMPORTED;
15038 cond2Block->bbFlags |= BBF_IMPORTED;
15039 cond1Block->bbFlags |= BBF_IMPORTED;
15040 asgBlock->bbFlags |= BBF_IMPORTED;
15043 // Chain the flow correctly.
15044 fgAddRefPred(asgBlock, block);
15045 fgAddRefPred(cond1Block, asgBlock);
15046 fgAddRefPred(cond2Block, cond1Block);
15047 fgAddRefPred(helperBlock, cond2Block);
15048 fgAddRefPred(remainderBlock, helperBlock);
15049 fgAddRefPred(remainderBlock, cond1Block);
15050 fgAddRefPred(remainderBlock, cond2Block);
15052 cond1Block->bbJumpDest = remainderBlock;
15053 cond2Block->bbJumpDest = remainderBlock;
15055 // Set the weights; some are guesses.
15056 asgBlock->inheritWeight(block);
15057 cond1Block->inheritWeight(block);
15058 cond2Block->inheritWeightPercentage(cond1Block, 50);
15059 helperBlock->inheritWeightPercentage(cond2Block, 50);
15061 // Append cond1 as JTRUE to cond1Block
15062 GenTreePtr jmpTree = gtNewOperNode(GT_JTRUE, TYP_VOID, condExpr);
15063 GenTreePtr jmpStmt = fgNewStmtFromTree(jmpTree, stmt->gtStmt.gtStmtILoffsx);
15064 fgInsertStmtAtEnd(cond1Block, jmpStmt);
15066 // Append cond2 as JTRUE to cond2Block
15067 jmpTree = gtNewOperNode(GT_JTRUE, TYP_VOID, cond2Expr);
15068 jmpStmt = fgNewStmtFromTree(jmpTree, stmt->gtStmt.gtStmtILoffsx);
15069 fgInsertStmtAtEnd(cond2Block, jmpStmt);
15071 // AsgBlock should get tmp = op1 assignment.
15072 trueExpr = gtNewTempAssign(dst->AsLclVarCommon()->GetLclNum(), trueExpr);
15073 GenTreePtr trueStmt = fgNewStmtFromTree(trueExpr, stmt->gtStmt.gtStmtILoffsx);
15074 fgInsertStmtAtEnd(asgBlock, trueStmt);
15076 // Since we are adding helper in the JTRUE false path, reverse the cond2 and add the helper.
15077 gtReverseCond(cond2Expr);
15078 GenTreePtr helperExpr = gtNewTempAssign(dst->AsLclVarCommon()->GetLclNum(), true2Expr);
15079 GenTreePtr helperStmt = fgNewStmtFromTree(helperExpr, stmt->gtStmt.gtStmtILoffsx);
15080 fgInsertStmtAtEnd(helperBlock, helperStmt);
15082 // Finally remove the nested qmark stmt.
15083 fgRemoveStmt(block, stmt);
15088 printf("\nExpanding CastInstOf qmark in BB%02u (after)\n", block->bbNum);
15089 fgDispBasicBlocks(block, remainderBlock, true);
15094 /*****************************************************************************
15096 * Expand a statement with a top level qmark node. There are three cases, based
15097 * on whether the qmark has both "true" and "false" arms, or just one of them.
15108 * S0 -->-- ~C -->-- T F -->-- S1
15113 * -----------------------------------------
15122 * S0 -->-- ~C -->-- T -->-- S1
15124 * +-->-------------+
15127 * -----------------------------------------
15136 * S0 -->-- C -->-- F -->-- S1
15138 * +-->------------+
15141 * If the qmark assigns to a variable, then create tmps for "then"
15142 * and "else" results and assign the temp to the variable as a writeback step.
15144 void Compiler::fgExpandQmarkStmt(BasicBlock* block, GenTreePtr stmt)
15146 GenTreePtr expr = stmt->gtStmt.gtStmtExpr;
15148 // Retrieve the Qmark node to be expanded.
15149 GenTreePtr dst = nullptr;
15150 GenTreePtr qmark = fgGetTopLevelQmark(expr, &dst);
15151 if (qmark == nullptr)
15156 if (qmark->gtFlags & GTF_QMARK_CAST_INSTOF)
15158 fgExpandQmarkForCastInstOf(block, stmt);
15165 printf("\nExpanding top-level qmark in BB%02u (before)\n", block->bbNum);
15166 fgDispBasicBlocks(block, block, true);
15170 // Retrieve the operands.
15171 GenTreePtr condExpr = qmark->gtGetOp1();
15172 GenTreePtr trueExpr = qmark->gtGetOp2()->AsColon()->ThenNode();
15173 GenTreePtr falseExpr = qmark->gtGetOp2()->AsColon()->ElseNode();
15175 assert(condExpr->gtFlags & GTF_RELOP_QMARK);
15176 condExpr->gtFlags &= ~GTF_RELOP_QMARK;
15178 assert(!varTypeIsFloating(condExpr->TypeGet()));
15180 bool hasTrueExpr = (trueExpr->OperGet() != GT_NOP);
15181 bool hasFalseExpr = (falseExpr->OperGet() != GT_NOP);
15182 assert(hasTrueExpr || hasFalseExpr); // We expect to have at least one arm of the qmark!
15184 // Create remainder, cond and "else" blocks. After this, the blocks are in this order:
15185 // block ... condBlock ... elseBlock ... remainderBlock
15187 // We need to remember flags that exist on 'block' that we want to propagate to 'remainderBlock',
15188 // if they are going to be cleared by fgSplitBlockAfterStatement(). We currently only do this only
15189 // for the GC safe point bit, the logic being that if 'block' was marked gcsafe, then surely
15190 // remainderBlock will still be GC safe.
15191 unsigned propagateFlags = block->bbFlags & BBF_GC_SAFE_POINT;
15192 BasicBlock* remainderBlock = fgSplitBlockAfterStatement(block, stmt);
15193 fgRemoveRefPred(remainderBlock, block); // We're going to put more blocks between block and remainderBlock.
15195 BasicBlock* condBlock = fgNewBBafter(BBJ_COND, block, true);
15196 BasicBlock* elseBlock = fgNewBBafter(BBJ_NONE, condBlock, true);
15198 // These blocks are only internal if 'block' is (but they've been set as internal by fgNewBBafter).
15199 // If they're not internal, mark them as imported to avoid asserts about un-imported blocks.
15200 if ((block->bbFlags & BBF_INTERNAL) == 0)
15202 condBlock->bbFlags &= ~BBF_INTERNAL;
15203 elseBlock->bbFlags &= ~BBF_INTERNAL;
15204 condBlock->bbFlags |= BBF_IMPORTED;
15205 elseBlock->bbFlags |= BBF_IMPORTED;
15208 remainderBlock->bbFlags |= BBF_JMP_TARGET | BBF_HAS_LABEL | propagateFlags;
15210 condBlock->inheritWeight(block);
15212 fgAddRefPred(condBlock, block);
15213 fgAddRefPred(elseBlock, condBlock);
15214 fgAddRefPred(remainderBlock, elseBlock);
15216 BasicBlock* thenBlock = nullptr;
15217 if (hasTrueExpr && hasFalseExpr)
15222 // S0 -->-- ~C -->-- T F -->-- S1
15227 gtReverseCond(condExpr);
15228 condBlock->bbJumpDest = elseBlock;
15230 thenBlock = fgNewBBafter(BBJ_ALWAYS, condBlock, true);
15231 thenBlock->bbJumpDest = remainderBlock;
15232 if ((block->bbFlags & BBF_INTERNAL) == 0)
15234 thenBlock->bbFlags &= ~BBF_INTERNAL;
15235 thenBlock->bbFlags |= BBF_IMPORTED;
15238 elseBlock->bbFlags |= (BBF_JMP_TARGET | BBF_HAS_LABEL);
15240 fgAddRefPred(thenBlock, condBlock);
15241 fgAddRefPred(remainderBlock, thenBlock);
15243 thenBlock->inheritWeightPercentage(condBlock, 50);
15244 elseBlock->inheritWeightPercentage(condBlock, 50);
15246 else if (hasTrueExpr)
15249 // S0 -->-- ~C -->-- T -->-- S1
15251 // +-->-------------+
15254 gtReverseCond(condExpr);
15255 condBlock->bbJumpDest = remainderBlock;
15256 fgAddRefPred(remainderBlock, condBlock);
15257 // Since we have no false expr, use the one we'd already created.
15258 thenBlock = elseBlock;
15259 elseBlock = nullptr;
15261 thenBlock->inheritWeightPercentage(condBlock, 50);
15263 else if (hasFalseExpr)
15266 // S0 -->-- C -->-- F -->-- S1
15268 // +-->------------+
15271 condBlock->bbJumpDest = remainderBlock;
15272 fgAddRefPred(remainderBlock, condBlock);
15274 elseBlock->inheritWeightPercentage(condBlock, 50);
15277 GenTreePtr jmpTree = gtNewOperNode(GT_JTRUE, TYP_VOID, qmark->gtGetOp1());
15278 GenTreePtr jmpStmt = fgNewStmtFromTree(jmpTree, stmt->gtStmt.gtStmtILoffsx);
15279 fgInsertStmtAtEnd(condBlock, jmpStmt);
15281 // Remove the original qmark statement.
15282 fgRemoveStmt(block, stmt);
15284 // Since we have top level qmarks, we either have a dst for it in which case
15285 // we need to create tmps for true and falseExprs, else just don't bother
15287 unsigned lclNum = BAD_VAR_NUM;
15288 if (dst != nullptr)
15290 assert(dst->gtOper == GT_LCL_VAR);
15291 lclNum = dst->gtLclVar.gtLclNum;
15295 assert(qmark->TypeGet() == TYP_VOID);
15300 if (dst != nullptr)
15302 trueExpr = gtNewTempAssign(lclNum, trueExpr);
15304 GenTreePtr trueStmt = fgNewStmtFromTree(trueExpr, stmt->gtStmt.gtStmtILoffsx);
15305 fgInsertStmtAtEnd(thenBlock, trueStmt);
15308 // Assign the falseExpr into the dst or tmp, insert in elseBlock
15311 if (dst != nullptr)
15313 falseExpr = gtNewTempAssign(lclNum, falseExpr);
15315 GenTreePtr falseStmt = fgNewStmtFromTree(falseExpr, stmt->gtStmt.gtStmtILoffsx);
15316 fgInsertStmtAtEnd(elseBlock, falseStmt);
15322 printf("\nExpanding top-level qmark in BB%02u (after)\n", block->bbNum);
15323 fgDispBasicBlocks(block, remainderBlock, true);
15328 /*****************************************************************************
15330 * Expand GT_QMARK nodes from the flow graph into basic blocks.
15334 void Compiler::fgExpandQmarkNodes()
15338 for (BasicBlock* block = fgFirstBB; block; block = block->bbNext)
15340 for (GenTreePtr stmt = block->bbTreeList; stmt; stmt = stmt->gtNext)
15342 GenTreePtr expr = stmt->gtStmt.gtStmtExpr;
15344 fgPreExpandQmarkChecks(expr);
15346 fgExpandQmarkStmt(block, stmt);
15350 fgPostExpandQmarkChecks();
15353 compQmarkRationalized = true;
15357 /*****************************************************************************
15359 * Make sure we don't have any more GT_QMARK nodes.
15362 void Compiler::fgPostExpandQmarkChecks()
15364 for (BasicBlock* block = fgFirstBB; block; block = block->bbNext)
15366 for (GenTreePtr stmt = block->bbTreeList; stmt; stmt = stmt->gtNext)
15368 GenTreePtr expr = stmt->gtStmt.gtStmtExpr;
15369 fgWalkTreePre(&expr, Compiler::fgAssertNoQmark, NULL);
15375 /*****************************************************************************
15377 * Transform all basic blocks for codegen.
15380 void Compiler::fgMorph()
15382 noway_assert(!compIsForInlining()); // Inlinee's compiler should never reach here.
15384 fgOutgoingArgTemps = nullptr;
15388 printf("*************** In fgMorph()\n");
15390 fgDispBasicBlocks(true);
15393 // Insert call to class constructor as the first basic block if
15394 // we were asked to do so.
15395 if (info.compCompHnd->initClass(NULL /* field */, info.compMethodHnd /* method */,
15396 impTokenLookupContextHandle /* context */) & CORINFO_INITCLASS_USE_HELPER)
15398 fgEnsureFirstBBisScratch();
15399 fgInsertStmtAtBeg(fgFirstBB, fgInitThisClass());
15403 if (opts.compGcChecks)
15405 for (unsigned i = 0; i < info.compArgsCount; i++)
15407 if (lvaTable[i].TypeGet() == TYP_REF)
15409 // confirm that the argument is a GC pointer (for debugging (GC stress))
15410 GenTreePtr op = gtNewLclvNode(i, TYP_REF);
15411 GenTreeArgList* args = gtNewArgList(op);
15412 op = gtNewHelperCallNode(CORINFO_HELP_CHECK_OBJ, TYP_VOID, 0, args);
15414 fgEnsureFirstBBisScratch();
15415 fgInsertStmtAtEnd(fgFirstBB, op);
15420 if (opts.compStackCheckOnRet)
15422 lvaReturnEspCheck = lvaGrabTempWithImplicitUse(false DEBUGARG("ReturnEspCheck"));
15423 lvaTable[lvaReturnEspCheck].lvType = TYP_INT;
15426 if (opts.compStackCheckOnCall)
15428 lvaCallEspCheck = lvaGrabTempWithImplicitUse(false DEBUGARG("CallEspCheck"));
15429 lvaTable[lvaCallEspCheck].lvType = TYP_INT;
15433 /* Filter out unimported BBs */
15435 fgRemoveEmptyBlocks();
15437 /* Add any internal blocks/trees we may need */
15442 fgMultipleNots = false;
15446 /* Inliner could add basic blocks. Check that the flowgraph data is up-to-date */
15447 fgDebugCheckBBlist(false, false);
15453 JITDUMP("trees after inlining\n");
15454 DBEXEC(VERBOSE, fgDispBasicBlocks(true));
15457 RecordStateAtEndOfInlining(); // Record "start" values for post-inlining cycles and elapsed time.
15460 /* Inliner could add basic blocks. Check that the flowgraph data is up-to-date */
15461 fgDebugCheckBBlist(false, false);
15464 /* For x64 and ARM64 we need to mark irregular parameters early so that they don't get promoted */
15465 fgMarkImplicitByRefArgs();
15467 /* Promote struct locals if necessary */
15468 fgPromoteStructs();
15470 /* Now it is the time to figure out what locals have address-taken. */
15471 fgMarkAddressExposedLocals();
15474 /* Now that locals have address-taken marked, we can safely apply stress. */
15476 fgStress64RsltMul();
15479 /* Morph the trees in all the blocks of the method */
15484 JITDUMP("trees after fgMorphBlocks\n");
15485 DBEXEC(VERBOSE, fgDispBasicBlocks(true));
15488 /* Decide the kind of code we want to generate */
15492 fgExpandQmarkNodes();
15500 /*****************************************************************************
15502 * Promoting struct locals
15504 void Compiler::fgPromoteStructs()
15508 printf("*************** In fgPromoteStructs()\n");
15511 if (!opts.OptEnabled(CLFLG_STRUCTPROMOTE))
15514 if (fgNoStructPromotion)
15518 // The code in this #if has been useful in debugging struct promotion issues, by
15519 // enabling selective enablement of the struct promotion optimization according to
15522 unsigned methHash = info.compMethodHash();
15523 char* lostr = getenv("structpromohashlo");
15524 unsigned methHashLo = 0;
15527 sscanf_s(lostr, "%x", &methHashLo);
15529 char* histr = getenv("structpromohashhi");
15530 unsigned methHashHi = UINT32_MAX;
15533 sscanf_s(histr, "%x", &methHashHi);
15535 if (methHash < methHashLo || methHash > methHashHi)
15541 printf("Promoting structs for method %s, hash = 0x%x.\n",
15542 info.compFullName, info.compMethodHash());
15543 printf(""); // in our logic this causes a flush
15548 if (info.compIsVarArgs)
15551 if (getNeedsGSSecurityCookie())
15554 // The lvaTable might grow as we grab temps. Make a local copy here.
15556 unsigned startLvaCount = lvaCount;
15559 // Loop through the original lvaTable. Looking for struct locals to be promoted.
15562 lvaStructPromotionInfo structPromotionInfo;
15563 bool tooManyLocals = false;
15565 for (unsigned lclNum = 0;
15566 lclNum < startLvaCount;
15569 // Whether this var got promoted
15570 bool promotedVar = false;
15571 LclVarDsc* varDsc = &lvaTable[lclNum];
15573 #ifdef FEATURE_SIMD
15574 if (varDsc->lvSIMDType && varDsc->lvUsedInSIMDIntrinsic)
15576 // If we have marked this as lvUsedInSIMDIntrinsic, then we do not want to promote
15577 // its fields. Instead, we will attempt to enregister the entire struct.
15578 varDsc->lvRegStruct = true;
15581 #endif //FEATURE_SIMD
15582 // Don't promote if we have reached the tracking limit.
15583 if (lvaHaveManyLocals())
15585 // Print the message first time when we detected this condition
15586 if (!tooManyLocals)
15588 JITDUMP("Stopped promoting struct fields, due to too many locals.\n");
15590 tooManyLocals = true;
15592 #if !FEATURE_MULTIREG_STRUCT_PROMOTE
15593 else if (varDsc->lvIsMultiRegArgOrRet)
15595 JITDUMP("Skipping V%02u: marked lvIsMultiRegArgOrRet.\n", lclNum);
15597 #endif // !FEATURE_MULTIREG_STRUCT_PROMOTE
15598 else if (varTypeIsStruct(varDsc))
15600 lvaCanPromoteStructVar(lclNum, &structPromotionInfo);
15601 bool canPromote = structPromotionInfo.canPromote;
15603 // We start off with shouldPromote same as canPromote.
15604 // Based on further profitablity checks done below, shouldPromote
15605 // could be set to false.
15606 bool shouldPromote = canPromote;
15611 // We *can* promote; *should* we promote?
15612 // We should only do so if promotion has potential savings. One source of savings
15613 // is if a field of the struct is accessed, since this access will be turned into
15614 // an access of the corresponding promoted field variable. Even if there are no
15615 // field accesses, but only block-level operations on the whole struct, if the struct
15616 // has only one or two fields, then doing those block operations field-wise is probably faster
15617 // than doing a whole-variable block operation (e.g., a hardware "copy loop" on x86).
15618 // So if no fields are accessed independently, and there are three or more fields,
15619 // then do not promote.
15620 if (structPromotionInfo.fieldCnt > 2 && !varDsc->lvFieldAccessed)
15622 JITDUMP("Not promoting promotable struct local V%02u: #fields = %d, fieldAccessed = %d.\n",
15623 lclNum, structPromotionInfo.fieldCnt, varDsc->lvFieldAccessed);
15624 shouldPromote = false;
15626 #if defined(_TARGET_AMD64_) || defined(_TARGET_ARM64_)
15627 // TODO-PERF - Only do this when the LclVar is used in an argument context
15628 // TODO-ARM64 - HFA support should also eliminate the need for this.
15629 // TODO-LSRA - Currently doesn't support the passing of floating point LCL_VARS in the integer registers
15631 // For now we currently don't promote structs with a single float field
15632 // Promoting it can cause us to shuffle it back and forth between the int and
15633 // the float regs when it is used as a argument, which is very expensive for XARCH
15635 else if ((structPromotionInfo.fieldCnt == 1) &&
15636 varTypeIsFloating(structPromotionInfo.fields[0].fldType))
15638 JITDUMP("Not promoting promotable struct local V%02u: #fields = %d because it is a struct with single float field.\n",
15639 lclNum, structPromotionInfo.fieldCnt);
15640 shouldPromote = false;
15642 #endif // _TARGET_AMD64_ || _TARGET_ARM64_
15644 #if !FEATURE_MULTIREG_STRUCT_PROMOTE
15645 #if defined(_TARGET_ARM64_)
15647 // For now we currently don't promote structs that could be passed in registers
15649 else if (varDsc->lvIsMultiregStruct())
15651 JITDUMP("Not promoting promotable struct local V%02u (size==%d): ",
15652 lclNum, lvaLclExactSize(lclNum));
15653 shouldPromote = false;
15655 #endif // _TARGET_ARM64_
15656 #endif // !FEATURE_MULTIREG_STRUCT_PROMOTE
15657 else if (varDsc->lvIsParam)
15659 #if FEATURE_MULTIREG_STRUCT_PROMOTE
15660 if (varDsc->lvIsMultiregStruct() && // Is this a variable holding a value that is passed in multiple registers?
15661 (structPromotionInfo.fieldCnt != 2)) // Does it have exactly two fields
15663 JITDUMP("Not promoting multireg struct local V%02u, because lvIsParam is true and #fields != 2\n",
15665 shouldPromote = false;
15668 #endif // !FEATURE_MULTIREG_STRUCT_PROMOTE
15670 // TODO-PERF - Implement struct promotion for incoming multireg structs
15671 // Currently it hits assert(lvFieldCnt==1) in lclvar.cpp line 4417
15673 if (structPromotionInfo.fieldCnt != 1)
15675 JITDUMP("Not promoting promotable struct local V%02u, because lvIsParam is true and #fields = %d.\n",
15676 lclNum, structPromotionInfo.fieldCnt);
15677 shouldPromote = false;
15682 // If the lvRefCnt is zero and we have a struct promoted parameter we can end up with an extra store of the the
15683 // incoming register into the stack frame slot.
15684 // In that case, we would like to avoid promortion.
15685 // However we haven't yet computed the lvRefCnt values so we can't do that.
15689 // Often-useful debugging code: if you've narrowed down a struct-promotion problem to a single
15690 // method, this allows you to select a subset of the vars to promote (by 1-based ordinal number).
15691 static int structPromoVarNum = 0;
15692 structPromoVarNum++;
15693 if (atoi(getenv("structpromovarnumlo")) <= structPromoVarNum && structPromoVarNum <= atoi(getenv("structpromovarnumhi")))
15698 assert(canPromote);
15700 // Promote the this struct local var.
15701 lvaPromoteStructVar(lclNum, &structPromotionInfo);
15702 promotedVar = true;
15704 #ifdef _TARGET_ARM_
15705 if (structPromotionInfo.requiresScratchVar)
15707 // Ensure that the scratch variable is allocated, in case we
15708 // pass a promoted struct as an argument.
15709 if (lvaPromotedStructAssemblyScratchVar == BAD_VAR_NUM)
15711 lvaPromotedStructAssemblyScratchVar =
15712 lvaGrabTempWithImplicitUse(false DEBUGARG("promoted struct assembly scratch var."));
15713 lvaTable[lvaPromotedStructAssemblyScratchVar].lvType = TYP_I_IMPL;
15716 #endif // _TARGET_ARM_
15721 #ifdef FEATURE_SIMD
15722 if (!promotedVar && varDsc->lvSIMDType && !varDsc->lvFieldAccessed)
15724 // Even if we have not used this in a SIMD intrinsic, if it is not being promoted,
15725 // we will treat it as a reg struct.
15726 varDsc->lvRegStruct = true;
15728 #endif // FEATURE_SIMD
15734 Compiler::fgWalkResult Compiler::fgMorphStructField(GenTreePtr tree, fgWalkData* fgWalkPre)
15736 noway_assert(tree->OperGet() == GT_FIELD);
15737 noway_assert(tree->gtFlags & GTF_GLOB_REF);
15739 GenTreePtr objRef = tree->gtField.gtFldObj;
15741 /* Is this an instance data member? */
15745 if (objRef->gtOper == GT_ADDR)
15747 GenTreePtr obj = objRef->gtOp.gtOp1;
15749 if (obj->gtOper == GT_LCL_VAR)
15751 unsigned lclNum = obj->gtLclVarCommon.gtLclNum;
15752 LclVarDsc* varDsc = &lvaTable[lclNum];
15754 if (varTypeIsStruct(obj))
15756 if (varDsc->lvPromoted)
15759 unsigned fldOffset = tree->gtField.gtFldOffset;
15760 unsigned fieldLclIndex = lvaGetFieldLocal(varDsc, fldOffset);
15761 noway_assert(fieldLclIndex != BAD_VAR_NUM);
15763 tree->SetOper(GT_LCL_VAR);
15764 tree->gtLclVarCommon.SetLclNum(fieldLclIndex);
15765 tree->gtType = lvaTable[fieldLclIndex].TypeGet();
15766 tree->gtFlags &= GTF_NODE_MASK;
15767 tree->gtFlags &= ~GTF_GLOB_REF;
15769 GenTreePtr parent = fgWalkPre->parentStack->Index(1);
15770 if ((parent->gtOper == GT_ASG) && (parent->gtOp.gtOp1 == tree))
15772 tree->gtFlags |= GTF_VAR_DEF;
15773 tree->gtFlags |= GTF_DONT_CSE;
15778 printf("Replacing the field in promoted struct with a local var:\n");
15779 fgWalkPre->printModified = true;
15782 return WALK_SKIP_SUBTREES;
15788 // A "normed struct" is a struct that the VM tells us is a basic type. This can only happen if
15789 // the struct contains a single element, and that element is 4 bytes (on x64 it can also be 8 bytes).
15790 // Normally, the type of the local var and the type of GT_FIELD are equivalent. However, there
15791 // is one extremely rare case where that won't be true. An enum type is a special value type
15792 // that contains exactly one element of a primitive integer type (that, for CLS programs is named "value__").
15793 // The VM tells us that a local var of that enum type is the primitive type of the enum's single field.
15794 // It turns out that it is legal for IL to access this field using ldflda or ldfld. For example:
15796 // .class public auto ansi sealed mynamespace.e_t extends [mscorlib]System.Enum
15798 // .field public specialname rtspecialname int16 value__
15799 // .field public static literal valuetype mynamespace.e_t one = int16(0x0000)
15801 // .method public hidebysig static void Main() cil managed
15803 // .locals init (valuetype mynamespace.e_t V_0)
15806 // ldflda int16 mynamespace.e_t::value__
15810 // Normally, compilers will not generate the ldflda, since it is superfluous.
15812 // In the example, the lclVar is short, but the JIT promotes all trees using this local to the
15813 // "actual type", that is, INT. But the GT_FIELD is still SHORT. So, in the case of a type
15814 // mismatch like this, don't do this morphing. The local var may end up getting marked as
15815 // address taken, and the appropriate SHORT load will be done from memory in that case.
15817 if (tree->TypeGet() == obj->TypeGet())
15819 tree->ChangeOper(GT_LCL_VAR);
15820 tree->gtLclVarCommon.SetLclNum(lclNum);
15821 tree->gtFlags &= GTF_NODE_MASK;
15823 GenTreePtr parent = fgWalkPre->parentStack->Index(1);
15824 if ((parent->gtOper == GT_ASG) && (parent->gtOp.gtOp1 == tree))
15826 tree->gtFlags |= GTF_VAR_DEF;
15827 tree->gtFlags |= GTF_DONT_CSE;
15832 printf("Replacing the field in normed struct with the local var:\n");
15833 fgWalkPre->printModified = true;
15836 return WALK_SKIP_SUBTREES;
15843 return WALK_CONTINUE;
15846 Compiler::fgWalkResult Compiler::fgMorphLocalField(GenTreePtr tree, fgWalkData* fgWalkPre)
15848 noway_assert(tree->OperGet() == GT_LCL_FLD);
15850 unsigned lclNum = tree->gtLclFld.gtLclNum;
15851 LclVarDsc* varDsc = &lvaTable[lclNum];
15853 if (varTypeIsStruct(varDsc) && (varDsc->lvPromoted))
15856 unsigned fldOffset = tree->gtLclFld.gtLclOffs;
15857 unsigned fieldLclIndex = 0;
15858 LclVarDsc* fldVarDsc = NULL;
15860 if (fldOffset != BAD_VAR_NUM)
15862 fieldLclIndex = lvaGetFieldLocal(varDsc, fldOffset);
15863 noway_assert(fieldLclIndex != BAD_VAR_NUM);
15864 fldVarDsc = &lvaTable[fieldLclIndex];
15867 if (fldOffset != BAD_VAR_NUM && genTypeSize(fldVarDsc->TypeGet()) == genTypeSize(tree->gtType)
15868 #ifdef _TARGET_X86_
15869 && varTypeIsFloating(fldVarDsc->TypeGet()) == varTypeIsFloating(tree->gtType)
15873 // There is an existing sub-field we can use
15874 tree->gtLclFld.SetLclNum(fieldLclIndex);
15876 // We need to keep the types 'compatible'. If we can switch back to a GT_LCL_VAR
15877 assert(varTypeIsIntegralOrI(tree->TypeGet()));
15878 if (varTypeCanReg(fldVarDsc->TypeGet()))
15880 // If the type is integer-ish, then we can use it as-is
15881 tree->ChangeOper(GT_LCL_VAR);
15882 assert(tree->gtLclVarCommon.gtLclNum == fieldLclIndex);
15883 tree->gtType = fldVarDsc->TypeGet();
15887 printf("Replacing the GT_LCL_FLD in promoted struct with a local var:\n");
15888 fgWalkPre->printModified = true;
15893 GenTreePtr parent = fgWalkPre->parentStack->Index(1);
15894 if ((parent->gtOper == GT_ASG) && (parent->gtOp.gtOp1 == tree))
15896 tree->gtFlags |= GTF_VAR_DEF;
15897 tree->gtFlags |= GTF_DONT_CSE;
15902 // There is no existing field that has all the parts that we need
15903 // So we must ensure that the struct lives in memory.
15904 lvaSetVarDoNotEnregister(lclNum DEBUGARG(DNER_LocalField));
15907 // We can't convert this guy to a float because he really does have his
15909 varDsc->lvKeepType = 1;
15913 return WALK_SKIP_SUBTREES;
15916 return WALK_CONTINUE;
15919 /*****************************************************************************
15921 * Mark irregular parameters. For x64 this is 3, 5, 6, 7, >8 byte structs that are passed by reference.
15922 * For ARM64, this is structs larger than 16 bytes that are passed by reference.
15924 void Compiler::fgMarkImplicitByRefArgs()
15926 #if defined(_TARGET_AMD64_) || defined(_TARGET_ARM64_)
15929 printf("\n*************** In fgMarkImplicitByRefs()\n");
15932 for (unsigned lclNum = 0; lclNum < lvaCount; lclNum++)
15934 LclVarDsc* varDsc = &lvaTable[lclNum];
15936 assert(!varDsc->lvPromoted); // Called in the wrong order?
15938 if (varDsc->lvIsParam && varTypeIsStruct(varDsc))
15942 if (varDsc->lvSize() > REGSIZE_BYTES)
15944 size = varDsc->lvSize();
15948 CORINFO_CLASS_HANDLE typeHnd = varDsc->lvVerTypeInfo.GetClassHandle();
15949 size = info.compCompHnd->getClassSize(typeHnd);
15953 #if !defined(FEATURE_UNIX_AMD64_STRUCT_PASSING)
15954 #if defined(_TARGET_AMD64_)
15955 if (size > REGSIZE_BYTES || (size & (size - 1)) != 0)
15956 #elif defined(_TARGET_ARM64_)
15960 // Previously nobody was ever setting lvIsParam and lvIsTemp on the same local
15961 // So I am now using it to indicate that this is one of the weird implicit
15963 // The address taken cleanup will look for references to locals marked like
15964 // this, and transform them appropriately.
15965 varDsc->lvIsTemp = 1;
15967 // Also marking them as BYREF will hide them from struct promotion.
15968 varDsc->lvType = TYP_BYREF;
15970 varDsc->lvRefCnt = 0;
15972 // Since this previously was a TYP_STRUCT and we have changed it to a TYP_BYREF
15973 // make sure that the following flag is not set as these will force SSA to
15974 // exclude tracking/enregistering these LclVars. (see fgExcludeFromSsa)
15976 varDsc->lvOverlappingFields = 0; // This flag could have been set, clear it.
15979 // This should not be converted to a double in stress mode,
15980 // because it is really a pointer
15981 varDsc->lvKeepType = 1;
15984 #endif // !FEATURE_UNIX_AMD64_STRUCT_PASSING
15988 #endif // _TARGET_AMD64_ || _TARGET_ARM64_
15991 /*****************************************************************************
15993 * Morph irregular parameters
15994 * for x64 and ARM64 this means turning them into byrefs, adding extra indirs.
15996 bool Compiler::fgMorphImplicitByRefArgs(GenTreePtr tree, fgWalkData* fgWalkPre)
15998 #if !defined(_TARGET_AMD64_) && !defined(_TARGET_ARM64_)
16002 #else // _TARGET_AMD64_ || _TARGET_ARM64_
16004 assert((tree->gtOper == GT_LCL_VAR) ||
16005 ((tree->gtOper == GT_ADDR) && (tree->gtOp.gtOp1->gtOper == GT_LCL_VAR)));
16007 bool isAddr = (tree->gtOper == GT_ADDR);
16008 GenTreePtr lclVarTree = isAddr ? tree->gtOp.gtOp1 : tree;
16009 unsigned lclNum = lclVarTree->gtLclVarCommon.gtLclNum;
16010 LclVarDsc* lclVarDsc = &lvaTable[lclNum];
16012 if (!lvaIsImplicitByRefLocal(lclNum))
16014 // We only need to tranform the 'marked' implicit by ref parameters
16018 // The SIMD transformation to coalesce contiguous references to SIMD vector fields will
16019 // re-invoke the traversal to mark address-taken locals.
16020 // So, we may encounter a tree that has already been transformed to TYP_BYREF.
16021 // If we do, leave it as-is.
16022 if (!varTypeIsStruct(lclVarTree))
16024 assert(lclVarTree->TypeGet() == TYP_BYREF);
16028 // We are overloading the lvRefCnt field here because real ref counts have not been set.
16029 lclVarDsc->lvRefCnt++;
16033 // change &X into just plain X
16034 tree->CopyFrom(lclVarTree, this);
16035 tree->gtType = TYP_BYREF;
16040 printf("Replacing address of implicit by ref struct parameter with byref:\n");
16041 fgWalkPre->printModified = true;
16047 // Change X into *X
16048 // First, save the original type, then change the tree type to TYP_BYREF (otherwise we
16049 // will get an assert when we try to clone the lclVar node because the lclVar is now TYP_BYREF
16050 // and the types have to match). The reason we clone the lclVar is that we don't pass a
16051 // possible-modified tree back to the caller, so we modify the original lclVar node in-place
16053 var_types structType = tree->gtType;
16054 lclVarTree = gtClone(tree);
16055 // Now, set the types appropriately.
16056 lclVarTree->gtType = TYP_BYREF;
16057 tree->gtType = structType;
16058 // Now, "insert" the GT_IND by changing the oper of the original node and setting its op1.
16059 tree->SetOper(GT_IND);
16060 tree->gtOp.gtOp1 = lclVarTree;
16061 // TODO-CQ: If the VM ever stops violating the ABI and passing heap references
16062 // we could remove TGTANYWHERE
16063 tree->gtFlags = ((tree->gtFlags & GTF_COMMON_MASK) | GTF_IND_TGTANYWHERE);
16068 printf("Replacing value of implicit by ref struct parameter with indir of parameter:\n");
16069 gtDispTree(tree, nullptr, nullptr, true);
16070 fgWalkPre->printModified = true;
16077 #endif // _TARGET_AMD64_ || _TARGET_ARM64_
16082 // An "AddrExposedContext" expresses the calling context in which an address expression occurs.
16083 enum AddrExposedContext
16085 AXC_None, // None of the below seen yet.
16086 AXC_Ind, // The address being computed is to be dereferenced.
16087 AXC_Addr, // We're computing a raw address (not dereferenced, at least not immediately).
16088 AXC_IndWide, // A block operation dereferenced an address referencing more bytes than the address
16089 // addresses -- if the address addresses a field of a struct local, we need to consider
16090 // the entire local address taken (not just the field).
16091 AXC_AddrWide, // The address being computed will be dereferenced by a block operation that operates
16092 // on more bytes than the width of the storage location addressed. If this is a
16093 // field of a promoted struct local, declare the entire struct local address-taken.
16094 AXC_InitBlk, // An GT_INITBLK is the immediate parent. The first argument is in an IND context.
16095 AXC_CopyBlk, // An GT_COPYBLK is the immediate parent. The first argument is in a GT_LIST, whose
16096 // args should be evaluated in an IND context.
16097 AXC_IndAdd, // A GT_ADD is the immediate parent, and it was evaluated in an IND contxt.
16098 // If one arg is a constant int, evaluate the other in an IND context. Otherwise, none.
16101 typedef ArrayStack<AddrExposedContext> AXCStack;
16103 // We use pre-post to simulate passing an argument in a recursion, via a stack.
16104 Compiler::fgWalkResult Compiler::fgMarkAddrTakenLocalsPostCB(GenTreePtr* pTree,
16105 fgWalkData* fgWalkPre)
16107 AXCStack* axcStack = reinterpret_cast<AXCStack*>(fgWalkPre->pCallbackData);
16108 (void)axcStack->Pop();
16109 return WALK_CONTINUE;
16112 Compiler::fgWalkResult Compiler::fgMarkAddrTakenLocalsPreCB(GenTreePtr* pTree,
16113 fgWalkData* fgWalkPre)
16115 GenTreePtr tree = *pTree;
16116 Compiler* comp = fgWalkPre->compiler;
16117 AXCStack* axcStack = reinterpret_cast<AXCStack*>(fgWalkPre->pCallbackData);
16118 AddrExposedContext axc = axcStack->Top();
16120 // In some situations, we have to figure out what the effective context is in which to
16121 // evaluate the current tree, depending on which argument position it is in its parent.
16123 // If the parent was an initblock, and this is its first argument, we're in
16124 // and "ind" context.
16130 // In both cases, the second argument is an integer struct size. That should have a "none" context.
16131 // The first argument is a GT_LIST. For GT_COPYBLK, both args of the list are addresses
16132 // that are dereferenced; for GT_INITBLK, the first is. We pass "axc" to the GT_LIST;
16133 // which will pass it to its arguments; these will decide whether they're in an Ind context
16134 // depending on "axc" and which argument they are.
16135 // A GT_INITBLK's first argument is a GT_LIST, whose first argument is an address
16136 // that should be considered to be dereferenced, and whose second argument the integer
16137 // (byte) value to fill the block with. The second argument of the GT_INITBLK is also
16138 // an integer, the block size.
16139 GenTreePtr parent = fgWalkPre->parentStack->Index(1);
16140 if (parent->gtOp.gtOp2 == tree &&
16141 parent->OperIsBlkOp())
16145 else if (parent->OperGet() == GT_LIST)
16147 genTreeOps axcOper = fgWalkPre->parentStack->Index(2)->OperGet();
16148 assert((axc == AXC_InitBlk && axcOper == GT_INITBLK) ||
16149 (axc == AXC_CopyBlk && GenTree::OperIsCopyBlkOp(axcOper)));
16151 // The block operation will derefence its argument(s) -- usually. If the size of the initblk
16152 // or copyblk exceeds the size of a storage location whose address is used as one of the
16153 // arguments, then we have to consider that storage location (indeed, it's underlying containing
16154 // location) to be address taken. So get the width of the initblk or copyblk.
16155 GenTreePtr widthNode = fgWalkPre->parentStack->Index(2)->gtOp.gtOp2;
16156 unsigned width = UINT_MAX; // If it's not a constant, assume it's maximally big.
16157 if (widthNode->IsCnsIntOrI())
16159 if (widthNode->IsIconHandle())
16161 // If it's a handle, it must be a class handle. We only create such block operations
16162 // for initialization of struct types, so the type of the argument(s) will match this
16163 // type, by construction. Set the width to zero to make sure nothing fits in it.
16164 assert(widthNode->IsIconHandle(GTF_ICON_CLASS_HDL));
16169 ssize_t swidth = widthNode->gtIntConCommon.IconValue();
16170 assert(swidth > 0); // Well-formedness of the block operation node...
16171 width = unsigned(swidth);
16175 if (parent->gtOp.gtOp1 == tree)
16177 // First argument is (potentially) dereferenced by both kinds of block operations.
16178 if (tree->OperGet() == GT_ADDR && !comp->fgFitsInOrNotLoc(tree->gtOp.gtOp1, width))
16187 else if (axc == AXC_CopyBlk)
16189 assert(parent->gtOp.gtOp2 == tree);
16190 if (tree->OperGet() == GT_ADDR && !comp->fgFitsInOrNotLoc(tree->gtOp.gtOp1, width))
16209 GenTreePtr parent = fgWalkPre->parentStack->Index(1);
16210 assert(parent->OperGet() == GT_ADD);
16211 // Is one of the args a constant representing a field offset,
16212 // and is this the other? If so, Ind context.
16213 if (parent->gtOp.gtOp1->IsCnsIntOrI() && parent->gtOp.gtOp2 == tree)
16217 else if (parent->gtOp.gtOp2->IsCnsIntOrI() && parent->gtOp.gtOp1 == tree)
16232 // Now recurse properly for the tree.
16233 switch (tree->gtOper)
16237 if (axc != AXC_Addr)
16239 axcStack->Push(AXC_Ind);
16243 axcStack->Push(AXC_None);
16245 return WALK_CONTINUE;
16248 axcStack->Push(AXC_InitBlk);
16249 return WALK_CONTINUE;
16253 axcStack->Push(AXC_CopyBlk);
16254 return WALK_CONTINUE;
16257 if (axc == AXC_InitBlk || axc == AXC_CopyBlk)
16259 axcStack->Push(axc);
16263 axcStack->Push(AXC_None);
16265 return WALK_CONTINUE;
16268 // Taking the address of an array element never takes the address of a local.
16269 axcStack->Push(AXC_None);
16270 return WALK_CONTINUE;
16273 // If we have ADDR(lcl), and "lcl" is an implicit byref parameter, fgMorphImplicitByRefArgs will
16274 // convert to just "lcl". This is never an address-context use, since the local is already a
16275 // byref after this transformation.
16276 if (tree->gtOp.gtOp1->OperGet() == GT_LCL_VAR && comp->fgMorphImplicitByRefArgs(tree, fgWalkPre))
16278 // Push something to keep the PostCB, which will pop it, happy.
16279 axcStack->Push(AXC_None);
16280 // In the first case, tree may no longer be a leaf, but we're done with it; is a leaf in the second case.
16281 return WALK_SKIP_SUBTREES;
16284 #ifdef FEATURE_SIMD
16285 if (tree->gtOp.gtOp1->OperGet() == GT_SIMD)
16287 axcStack->Push(AXC_None);
16290 #endif // FEATURE_SIMD
16291 if (axc == AXC_Ind)
16293 axcStack->Push(AXC_None);
16295 else if (axc == AXC_IndWide)
16297 axcStack->Push(AXC_AddrWide);
16301 assert(axc == AXC_None);
16302 axcStack->Push(AXC_Addr);
16304 return WALK_CONTINUE;
16307 // First, handle a couple of special cases: field of promoted struct local, field
16308 // of "normed" struct.
16309 if (comp->fgMorphStructField(tree, fgWalkPre) == WALK_SKIP_SUBTREES)
16311 // It (may have) replaced the field with a local var or local field. If we're in an addr context,
16312 // label it addr-taken.
16313 if (tree->OperIsLocal() && (axc == AXC_Addr || axc == AXC_AddrWide))
16315 unsigned lclNum = tree->gtLclVarCommon.gtLclNum;
16316 comp->lvaSetVarAddrExposed(lclNum);
16317 if (axc == AXC_AddrWide)
16319 LclVarDsc* varDsc = &comp->lvaTable[lclNum];
16320 if (varDsc->lvIsStructField)
16322 comp->lvaSetVarAddrExposed(varDsc->lvParentLcl);
16326 // Push something to keep the PostCB, which will pop it, happy.
16327 axcStack->Push(AXC_None);
16328 return WALK_SKIP_SUBTREES;
16332 // GT_FIELD is an implicit deref.
16333 if (axc == AXC_Addr)
16335 axcStack->Push(AXC_None);
16337 else if (axc == AXC_AddrWide)
16339 axcStack->Push(AXC_IndWide);
16343 axcStack->Push(AXC_Ind);
16345 return WALK_CONTINUE;
16350 assert(axc != AXC_Addr);
16351 // This recognizes certain forms, and does all the work. In that case, returns WALK_SKIP_SUBTREES,
16352 // else WALK_CONTINUE. We do the same here.
16353 fgWalkResult res = comp->fgMorphLocalField(tree, fgWalkPre);
16354 if (res == WALK_SKIP_SUBTREES && tree->OperGet() == GT_LCL_VAR && (axc == AXC_Addr || axc == AXC_AddrWide))
16356 unsigned lclNum = tree->gtLclVarCommon.gtLclNum;
16357 comp->lvaSetVarAddrExposed(lclNum);
16358 if (axc == AXC_AddrWide)
16360 LclVarDsc* varDsc = &comp->lvaTable[lclNum];
16361 if (varDsc->lvIsStructField)
16363 comp->lvaSetVarAddrExposed(varDsc->lvParentLcl);
16367 // Must push something; if res is WALK_SKIP_SUBTREES, doesn't matter
16368 // what, but something to be popped by the post callback. If we're going
16369 // to analyze children, the LCL_FLD creates an Ind context, so use that.
16370 axcStack->Push(AXC_Ind);
16375 // On some architectures, some arguments are passed implicitly by reference.
16376 // Modify the trees to reflect that, if this local is one of those.
16377 if (comp->fgMorphImplicitByRefArgs(tree, fgWalkPre))
16379 // We can't be in an address context; the ADDR(lcl), where lcl is an implicit byref param, was
16380 // handled earlier. (And we can't have added anything to this address, since it was implicit.)
16381 assert(axc != AXC_Addr);
16385 if (axc == AXC_Addr || axc == AXC_AddrWide)
16387 unsigned lclNum = tree->gtLclVarCommon.gtLclNum;
16388 comp->lvaSetVarAddrExposed(lclNum);
16389 if (axc == AXC_AddrWide)
16391 LclVarDsc* varDsc = &comp->lvaTable[lclNum];
16392 if (varDsc->lvIsStructField)
16394 comp->lvaSetVarAddrExposed(varDsc->lvParentLcl);
16398 // We may need to Quirk the storage size for this LCL_VAR
16399 // some PInvoke signatures incorrectly specify a ByRef to an INT32
16400 // when they actually write a SIZE_T or INT64
16401 if (axc == AXC_Addr)
16403 comp->gtCheckQuirkAddrExposedLclVar(tree, fgWalkPre->parentStack);
16407 // Push something to keep the PostCB, which will pop it, happy.
16408 axcStack->Push(AXC_None);
16409 // In the first case, tree may no longer be a leaf, but we're done with it; is a leaf in the second case.
16410 return WALK_SKIP_SUBTREES;
16413 assert(axc != AXC_Addr);
16414 // See below about treating pointer operations as wider indirection.
16415 if (tree->gtOp.gtOp1->gtType == TYP_BYREF || tree->gtOp.gtOp2->gtType == TYP_BYREF)
16417 axcStack->Push(AXC_IndWide);
16419 else if (axc == AXC_Ind)
16421 // Let the children know that the parent was a GT_ADD, to be evaluated in an IND context.
16422 // If it's an add of a constant and an address, and the constant represents a field,
16423 // then we'll evaluate the address argument in an Ind context; otherwise, the None context.
16424 axcStack->Push(AXC_IndAdd);
16428 axcStack->Push(axc);
16430 return WALK_CONTINUE;
16432 // !!! Treat Pointer Operations as Wider Indirection
16434 // If we are performing pointer operations, make sure we treat that as equivalent to a wider
16435 // indirection. This is because the pointers could be pointing to the address of struct fields
16436 // and could be used to perform operations on the whole struct or passed to another method.
16438 // When visiting a node in this pre-order walk, we do not know if we would in the future
16439 // encounter a GT_ADDR of a GT_FIELD below.
16441 // Note: GT_ADDR of a GT_FIELD is always a TYP_BYREF.
16442 // So let us be conservative and treat TYP_BYREF operations as AXC_IndWide and propagate a
16443 // wider indirection context down the expr tree.
16445 // Example, in unsafe code,
16447 // IL_000e 12 00 ldloca.s 0x0
16448 // IL_0010 7c 02 00 00 04 ldflda 0x4000002
16449 // IL_0015 12 00 ldloca.s 0x0
16450 // IL_0017 7c 01 00 00 04 ldflda 0x4000001
16453 // When visiting the GT_SUB node, if the types of either of the GT_SUB's operand are BYREF, then
16454 // consider GT_SUB to be equivalent of an AXC_IndWide.
16456 // Similarly for pointer comparisons and pointer escaping as integers through conversions, treat
16457 // them as AXC_IndWide.
16481 if ((tree->gtOp.gtOp1->gtType == TYP_BYREF) || (tree->OperIsBinary() && (tree->gtOp.gtOp2->gtType == TYP_BYREF)))
16483 axcStack->Push(AXC_IndWide);
16484 return WALK_CONTINUE;
16489 // To be safe/conservative: pass Addr through, but not Ind -- otherwise, revert to "None". We must
16490 // handle the "Ind" propogation explicitly above.
16491 if (axc == AXC_Addr || axc == AXC_AddrWide)
16493 axcStack->Push(axc);
16497 axcStack->Push(AXC_None);
16499 return WALK_CONTINUE;
16503 bool Compiler::fgFitsInOrNotLoc(GenTreePtr tree, unsigned width)
16505 if (tree->TypeGet() != TYP_STRUCT)
16507 return width <= genTypeSize(tree->TypeGet());
16509 else if (tree->OperGet() == GT_LCL_VAR)
16511 assert(tree->TypeGet() == TYP_STRUCT);
16512 unsigned lclNum = tree->gtLclVarCommon.gtLclNum;
16513 return width <= lvaTable[lclNum].lvExactSize;
16515 else if (tree->OperGet() == GT_FIELD)
16517 CORINFO_CLASS_HANDLE fldClass = info.compCompHnd->getFieldClass (tree->gtField.gtFldHnd);
16518 return width <= info.compCompHnd->getClassSize(fldClass);
16527 void Compiler::fgAddFieldSeqForZeroOffset(GenTreePtr op1, FieldSeqNode* fieldSeq)
16529 assert(op1->TypeGet() == TYP_BYREF || op1->TypeGet() == TYP_I_IMPL || op1->TypeGet() == TYP_REF);
16531 switch (op1->OperGet())
16534 if (op1->gtOp.gtOp1->OperGet() == GT_LCL_FLD)
16536 GenTreeLclFld* lclFld = op1->gtOp.gtOp1->AsLclFld();
16537 lclFld->gtFieldSeq = GetFieldSeqStore()->Append(lclFld->gtFieldSeq, fieldSeq);
16542 if (op1->gtOp.gtOp1->OperGet() == GT_CNS_INT)
16544 FieldSeqNode* op1Fs = op1->gtOp.gtOp1->gtIntCon.gtFieldSeq;
16547 op1Fs = GetFieldSeqStore()->Append(op1Fs, fieldSeq);
16548 op1->gtOp.gtOp1->gtIntCon.gtFieldSeq = op1Fs;
16551 else if (op1->gtOp.gtOp2->OperGet() == GT_CNS_INT)
16553 FieldSeqNode* op2Fs = op1->gtOp.gtOp2->gtIntCon.gtFieldSeq;
16556 op2Fs = GetFieldSeqStore()->Append(op2Fs, fieldSeq);
16557 op1->gtOp.gtOp2->gtIntCon.gtFieldSeq = op2Fs;
16564 FieldSeqNode* op1Fs = op1->gtIntCon.gtFieldSeq;
16567 op1Fs = GetFieldSeqStore()->Append(op1Fs, fieldSeq);
16568 op1->gtIntCon.gtFieldSeq = op1Fs;
16574 // Record in the general zero-offset map.
16575 GetZeroOffsetFieldMap()->Set(op1, fieldSeq);
16580 /*****************************************************************************
16582 * Mark address-taken locals.
16585 void Compiler::fgMarkAddressExposedLocals()
16589 printf("\n*************** In fgMarkAddressExposedLocals()\n");
16592 BasicBlock* block = fgFirstBB;
16593 noway_assert(block);
16597 /* Make the current basic block address available globally */
16603 for (stmt = block->bbTreeList;
16605 stmt = stmt->gtNext)
16607 // Call Compiler::fgMarkAddrTakenLocalsCB on each node
16608 AXCStack stk(this);
16609 stk.Push(AXC_None); // We start in neither an addr or ind context.
16610 fgWalkTree(&stmt->gtStmt.gtStmtExpr,
16611 fgMarkAddrTakenLocalsPreCB,
16612 fgMarkAddrTakenLocalsPostCB,
16616 block = block->bbNext;
16622 // fgNodesMayInterfere:
16623 // return true if moving nodes relative to each other can change the result of a computation
16626 // read: a node which reads
16629 bool Compiler::fgNodesMayInterfere(GenTree* write, GenTree* read)
16631 LclVarDsc* srcVar = nullptr;
16632 bool srcAliased = false;
16633 bool dstAliased = false;
16635 bool readIsIndir = read->OperIsIndir() || read->OperIsImplicitIndir();
16636 bool writeIsIndir = write->OperIsIndir() || write->OperIsImplicitIndir();
16638 if (read->OperIsLocal())
16639 srcVar = &lvaTable[read->gtLclVarCommon.gtLclNum];
16643 if (srcVar && srcVar->lvAddrExposed)
16645 else if (readIsIndir)
16649 else if (write->OperIsLocal())
16651 LclVarDsc* dstVar = &lvaTable[write->gtLclVarCommon.gtLclNum];
16654 return dstVar->lvAddrExposed;
16656 else if (read->OperIsLocal())
16658 if (read->gtLclVarCommon.gtLclNum == write->gtLclVarCommon.gtLclNum)
16673 /** This predicate decides whether we will fold a tree with the structure:
16674 * x = x <op> y where x could be any arbitrary expression into
16677 * This modification is only performed when the target architecture supports
16678 * complex addressing modes. In the case of ARM for example, this transformation
16679 * yields no benefit.
16681 * In case this functions decides we can proceed to fold into an assignment operator
16682 * we need to inspect whether the operator is commutative to tell fgMorph whether we need to
16683 * reverse the tree due to the fact we saw x = y <op> x and we want to fold that into
16684 * x <op>= y because the operator property.
16686 bool Compiler::fgShouldCreateAssignOp(GenTreePtr tree, bool* bReverse)
16688 #if CPU_LOAD_STORE_ARCH
16689 /* In the case of a load/store architecture, there's no gain by doing any of this, we bail. */
16691 #elif !defined(LEGACY_BACKEND)
16693 #else // defined(LEGACY_BACKEND)
16696 GenTreePtr op1 = tree->gtOp.gtOp1;
16697 GenTreePtr op2 = tree->gtGetOp2();
16698 genTreeOps cmop = op2->OperGet();
16700 /* Is the destination identical to the first RHS sub-operand? */
16701 if (GenTree::Compare(op1, op2->gtOp.gtOp1))
16704 Do not transform the following tree
16706 [0024CFA4] ----------- const int 1
16707 [0024CFDC] ----G------ | int
16708 [0024CF5C] ----------- lclVar ubyte V01 tmp0
16709 [0024D05C] -A--G------ = ubyte
16710 [0024D014] D------N--- lclVar ubyte V01 tmp0
16714 [0024CFA4] ----------- const int 1
16715 [0024D05C] -A--G------ |= ubyte
16716 [0024D014] U------N--- lclVar ubyte V01 tmp0
16718 , when V01 is a struct field local.
16721 if (op1->gtOper == GT_LCL_VAR &&
16722 varTypeIsSmall(op1->TypeGet()) &&
16723 op1->TypeGet() != op2->gtOp.gtOp2->TypeGet())
16725 unsigned lclNum = op1->gtLclVarCommon.gtLclNum;
16726 LclVarDsc* varDsc = lvaTable + lclNum;
16728 if (varDsc->lvIsStructField)
16737 else if (GenTree::OperIsCommutative(cmop))
16739 /* For commutative ops only, check for "a = x <op> a" */
16741 /* Should we be doing this at all? */
16742 if ((opts.compFlags & CLFLG_TREETRANS) == 0)
16747 /* Can we swap the operands to cmop ... */
16748 if ((op2->gtOp.gtOp1->gtFlags & GTF_ALL_EFFECT) &&
16749 (op2->gtOp.gtOp2->gtFlags & GTF_ALL_EFFECT) )
16751 // Both sides must have side effects to prevent swap */
16755 /* Is the destination identical to the second RHS sub-operand? */
16756 if (GenTree::Compare(op1, op2->gtOp.gtOp2))
16763 #endif // defined(LEGACY_BACKEND)
16766 // Static variables.
16767 Compiler::MorphAddrContext Compiler::s_CopyBlockMAC(Compiler::MACK_CopyBlock);
16769 #ifdef FEATURE_SIMD
16771 //-----------------------------------------------------------------------------------
16772 // fgMorphCombineSIMDFieldAssignments:
16773 // If the RHS of the input stmt is a read for simd vector X Field, then this function
16774 // will keep reading next few stmts based on the vector size(2, 3, 4).
16775 // If the next stmts LHS are located contiguous and RHS are also located
16776 // contiguous, then we replace those statements with a copyblk.
16779 // block - BasicBlock*. block which stmt belongs to
16780 // stmt - GenTreeStmt*. the stmt node we want to check
16783 // if this funciton successfully optimized the stmts, then return true. Otherwise
16786 bool Compiler::fgMorphCombineSIMDFieldAssignments(BasicBlock* block, GenTreePtr stmt)
16789 noway_assert(stmt->gtOper == GT_STMT);
16790 GenTreePtr tree = stmt->gtStmt.gtStmtExpr;
16791 assert(tree->OperGet() == GT_ASG);
16793 GenTreePtr originalLHS = tree->gtOp.gtOp1;
16794 GenTreePtr prevLHS = tree->gtOp.gtOp1;
16795 GenTreePtr prevRHS = tree->gtOp.gtOp2;
16796 unsigned index = 0;
16797 var_types baseType = TYP_UNKNOWN;
16798 unsigned simdSize = 0;
16799 GenTreePtr simdStructNode = getSIMDStructFromField(prevRHS, &baseType, &index, &simdSize, true);
16801 if (simdStructNode == nullptr ||
16803 baseType != TYP_FLOAT)
16805 // if the RHS is not from a SIMD vector field X, then there is no need to check further.
16809 int assignmentsCount = simdSize / genTypeSize(baseType) - 1;
16810 int remainingAssignments = assignmentsCount;
16811 GenTreePtr curStmt = stmt->gtNext;
16812 GenTreePtr lastStmt = stmt;
16814 while (curStmt != nullptr && remainingAssignments > 0)
16816 GenTreePtr exp = curStmt->gtStmt.gtStmtExpr;
16817 if (exp->OperGet() != GT_ASG)
16821 GenTreePtr curLHS = exp->gtGetOp1();
16822 GenTreePtr curRHS = exp->gtGetOp2();
16824 if (!areArgumentsContiguous(prevLHS, curLHS) ||
16825 !areArgumentsContiguous(prevRHS, curRHS))
16830 remainingAssignments--;
16834 lastStmt = curStmt;
16835 curStmt = curStmt->gtNext;
16838 if (remainingAssignments > 0)
16840 // if the left assignments number is bigger than zero, then this means
16841 // that the assignments are not assgining to the contiguously memory
16842 // locations from same vector.
16848 printf("\nFound contiguous assignments from a SIMD vector to memory.\n");
16849 printf("From BB%02u, stmt", block->bbNum);
16851 printf(" to stmt");
16852 printTreeID(lastStmt);
16858 for (int i = 0; i < assignmentsCount; i++)
16860 fgRemoveStmt(block, stmt->gtNext);
16863 GenTree* copyBlkDst = createAddressNodeForSIMDInit(originalLHS, simdSize);
16864 if (simdStructNode->OperIsLocal())
16866 setLclRelatedToSIMDIntrinsic(simdStructNode);
16868 GenTree* copyBlkAddr = copyBlkDst;
16869 if (copyBlkAddr->gtOper == GT_LEA)
16871 copyBlkAddr = copyBlkAddr->AsAddrMode()->Base();
16873 GenTreeLclVarCommon* localDst = nullptr;
16874 if (copyBlkAddr->IsLocalAddrExpr(this, &localDst, nullptr))
16876 setLclRelatedToSIMDIntrinsic(localDst);
16879 GenTree* simdStructAddr;
16880 if (simdStructNode->TypeGet() == TYP_BYREF)
16882 assert(simdStructNode->OperIsLocal());
16883 assert(lvaIsImplicitByRefLocal(simdStructNode->AsLclVarCommon()->gtLclNum));
16884 simdStructAddr = simdStructNode;
16888 assert(varTypeIsSIMD(simdStructNode));
16889 simdStructAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, simdStructNode);
16895 printf("\nBB%02u stmt", block->bbNum);
16897 printf("(before)\n");
16902 tree = gtNewBlkOpNode(GT_COPYBLK,
16905 gtNewIconNode(simdSize),
16908 stmt->gtStmt.gtStmtExpr = tree;
16910 // Since we generated a new address node which didn't exist before,
16911 // we should expose this address manually here.
16912 AXCStack stk(this);
16913 stk.Push(AXC_None);
16914 fgWalkTree(&stmt->gtStmt.gtStmtExpr,
16915 fgMarkAddrTakenLocalsPreCB,
16916 fgMarkAddrTakenLocalsPostCB,
16922 printf("\nReplaced BB%02u stmt", block->bbNum);
16924 printf("(after)\n");
16931 #endif //FEATURE_SIMD