1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
5 /*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
6 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10 XX Does the register allocation and puts the remaining lclVars on the stack XX
12 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
13 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
22 #if FEATURE_FP_REGALLOC
23 Compiler::enumConfigRegisterFP Compiler::raConfigRegisterFP()
25 DWORD val = JitConfig.JitRegisterFP();
27 return (enumConfigRegisterFP)(val & 0x3);
29 #endif // FEATURE_FP_REGALLOC
31 regMaskTP Compiler::raConfigRestrictMaskFP()
33 regMaskTP result = RBM_NONE;
35 #if FEATURE_FP_REGALLOC
36 switch (raConfigRegisterFP())
38 case CONFIG_REGISTER_FP_NONE:
41 case CONFIG_REGISTER_FP_CALLEE_TRASH:
42 result = RBM_FLT_CALLEE_TRASH;
44 case CONFIG_REGISTER_FP_CALLEE_SAVED:
45 result = RBM_FLT_CALLEE_SAVED;
47 case CONFIG_REGISTER_FP_FULL:
48 result = RBM_ALLFLOAT;
57 DWORD Compiler::getCanDoubleAlign()
60 if (compStressCompile(STRESS_DBL_ALN, 20))
61 return MUST_DOUBLE_ALIGN;
63 return JitConfig.JitDoubleAlign();
65 return DEFAULT_DOUBLE_ALIGN;
69 //------------------------------------------------------------------------
70 // shouldDoubleAlign: Determine whether to double-align the frame
73 // refCntStk - sum of ref counts for all stack based variables
74 // refCntEBP - sum of ref counts for EBP enregistered variables
75 // refCntWtdEBP - sum of wtd ref counts for EBP enregistered variables
76 // refCntStkParam - sum of ref counts for all stack based parameters
77 // refCntWtdStkDbl - sum of wtd ref counts for stack based doubles (including structs
78 // with double fields).
81 // Returns true if this method estimates that a double-aligned frame would be beneficial
84 // The impact of a double-aligned frame is computed as follows:
85 // - We save a byte of code for each parameter reference (they are frame-pointer relative)
86 // - We pay a byte of code for each non-parameter stack reference.
87 // - We save the misalignment penalty and possible cache-line crossing penalty.
88 // This is estimated as 0 for SMALL_CODE, 16 for FAST_CODE and 4 otherwise.
89 // - We pay 7 extra bytes for:
91 // LEA ESP,[EBP-offset]
92 // AND ESP,-8 to double align ESP
93 // - We pay one extra memory reference for each variable that could have been enregistered in EBP (refCntWtdEBP).
95 // If the misalignment penalty is estimated to be less than the bytes used, we don't double align.
96 // Otherwise, we compare the weighted ref count of ebp-enregistered variables aginst double the
97 // ref count for double-aligned values.
99 bool Compiler::shouldDoubleAlign(
100 unsigned refCntStk, unsigned refCntEBP, unsigned refCntWtdEBP, unsigned refCntStkParam, unsigned refCntWtdStkDbl)
102 bool doDoubleAlign = false;
103 const unsigned DBL_ALIGN_SETUP_SIZE = 7;
105 unsigned bytesUsed = refCntStk + refCntEBP - refCntStkParam + DBL_ALIGN_SETUP_SIZE;
106 unsigned misaligned_weight = 4;
108 if (compCodeOpt() == Compiler::SMALL_CODE)
109 misaligned_weight = 0;
111 if (compCodeOpt() == Compiler::FAST_CODE)
112 misaligned_weight *= 4;
114 JITDUMP("\nDouble alignment:\n");
115 JITDUMP(" Bytes that could be saved by not using EBP frame: %i\n", bytesUsed);
116 JITDUMP(" Sum of weighted ref counts for EBP enregistered variables: %i\n", refCntWtdEBP);
117 JITDUMP(" Sum of weighted ref counts for weighted stack based doubles: %i\n", refCntWtdStkDbl);
119 if (bytesUsed > ((refCntWtdStkDbl * misaligned_weight) / BB_UNITY_WEIGHT))
121 JITDUMP(" Predicting not to double-align ESP to save %d bytes of code.\n", bytesUsed);
123 else if (refCntWtdEBP > refCntWtdStkDbl * 2)
125 // TODO-CQ: On P4 2 Proc XEON's, SciMark.FFT degrades if SciMark.FFT.transform_internal is
126 // not double aligned.
127 // Here are the numbers that make this not double-aligned.
128 // refCntWtdStkDbl = 0x164
129 // refCntWtdEBP = 0x1a4
130 // We think we do need to change the heuristic to be in favor of double-align.
132 JITDUMP(" Predicting not to double-align ESP to allow EBP to be used to enregister variables.\n");
136 // OK we passed all of the benefit tests, so we'll predict a double aligned frame.
137 JITDUMP(" Predicting to create a double-aligned frame\n");
138 doDoubleAlign = true;
140 return doDoubleAlign;
142 #endif // DOUBLE_ALIGN
144 #ifdef LEGACY_BACKEND // We don't use any of the old register allocator functions when LSRA is used instead.
146 void Compiler::raInit()
148 #if FEATURE_STACK_FP_X87
149 /* We have not assigned any FP variables to registers yet */
151 VarSetOps::AssignNoCopy(this, optAllFPregVars, VarSetOps::UninitVal());
153 codeGen->intRegState.rsIsFloat = false;
154 codeGen->floatRegState.rsIsFloat = true;
156 rpReverseEBPenreg = false;
159 rpPassesPessimize = rpPassesMax - 3;
160 if (opts.compDbgCode)
164 rpStkPredict = (unsigned)-1;
165 rpFrameType = FT_NOT_SET;
167 rpMustCreateEBPCalled = false;
168 rpRegAllocDone = false;
169 rpMaskPInvokeEpilogIntf = RBM_NONE;
171 rpPredictMap[PREDICT_NONE] = RBM_NONE;
172 rpPredictMap[PREDICT_ADDR] = RBM_NONE;
174 #if FEATURE_FP_REGALLOC
175 rpPredictMap[PREDICT_REG] = RBM_ALLINT | RBM_ALLFLOAT;
176 rpPredictMap[PREDICT_SCRATCH_REG] = RBM_ALLINT | RBM_ALLFLOAT;
178 rpPredictMap[PREDICT_REG] = RBM_ALLINT;
179 rpPredictMap[PREDICT_SCRATCH_REG] = RBM_ALLINT;
182 #define REGDEF(name, rnum, mask, sname) rpPredictMap[PREDICT_REG_##name] = RBM_##name;
183 #include "register.h"
185 #if defined(_TARGET_ARM_)
187 rpPredictMap[PREDICT_PAIR_R0R1] = RBM_R0 | RBM_R1;
188 rpPredictMap[PREDICT_PAIR_R2R3] = RBM_R2 | RBM_R3;
189 rpPredictMap[PREDICT_REG_SP] = RBM_ILLEGAL;
191 #elif defined(_TARGET_AMD64_)
193 rpPredictMap[PREDICT_NOT_REG_EAX] = RBM_ALLINT & ~RBM_EAX;
194 rpPredictMap[PREDICT_NOT_REG_ECX] = RBM_ALLINT & ~RBM_ECX;
195 rpPredictMap[PREDICT_REG_ESP] = RBM_ILLEGAL;
197 #elif defined(_TARGET_X86_)
199 rpPredictMap[PREDICT_NOT_REG_EAX] = RBM_ALLINT & ~RBM_EAX;
200 rpPredictMap[PREDICT_NOT_REG_ECX] = RBM_ALLINT & ~RBM_ECX;
201 rpPredictMap[PREDICT_REG_ESP] = RBM_ILLEGAL;
202 rpPredictMap[PREDICT_PAIR_EAXEDX] = RBM_EAX | RBM_EDX;
203 rpPredictMap[PREDICT_PAIR_ECXEBX] = RBM_ECX | RBM_EBX;
207 rpBestRecordedPrediction = NULL;
210 /*****************************************************************************
212 * The following table(s) determines the order in which registers are considered
213 * for variables to live in
216 const regNumber* Compiler::raGetRegVarOrder(var_types regType, unsigned* wbVarOrderSize)
218 #if FEATURE_FP_REGALLOC
219 if (varTypeIsFloating(regType))
221 static const regNumber raRegVarOrderFlt[] = {REG_VAR_ORDER_FLT};
222 const unsigned raRegVarOrderFltSize = _countof(raRegVarOrderFlt);
224 if (wbVarOrderSize != NULL)
225 *wbVarOrderSize = raRegVarOrderFltSize;
227 return &raRegVarOrderFlt[0];
232 static const regNumber raRegVarOrder[] = {REG_VAR_ORDER};
233 const unsigned raRegVarOrderSize = _countof(raRegVarOrder);
235 if (wbVarOrderSize != NULL)
236 *wbVarOrderSize = raRegVarOrderSize;
238 return &raRegVarOrder[0];
244 /*****************************************************************************
246 * Dump out the variable interference graph
250 void Compiler::raDumpVarIntf()
255 printf("Var. interference graph for %s\n", info.compFullName);
257 for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
259 /* Ignore the variable if it's not tracked */
261 if (!varDsc->lvTracked)
264 /* Get hold of the index and the interference mask for the variable */
265 unsigned varIndex = varDsc->lvVarIndex;
267 printf(" V%02u,T%02u and ", lclNum, varIndex);
271 for (refIndex = 0; refIndex < lvaTrackedCount; refIndex++)
273 if (VarSetOps::IsMember(this, lvaVarIntf[varIndex], refIndex))
274 printf("T%02u ", refIndex);
285 /*****************************************************************************
287 * Dump out the register interference graph
290 void Compiler::raDumpRegIntf()
292 printf("Reg. interference graph for %s\n", info.compFullName);
297 for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
301 /* Ignore the variable if it's not tracked */
303 if (!varDsc->lvTracked)
306 /* Get hold of the index and the interference mask for the variable */
308 varNum = varDsc->lvVarIndex;
310 printf(" V%02u,T%02u and ", lclNum, varNum);
312 if (varDsc->IsFloatRegType())
314 #if !FEATURE_STACK_FP_X87
315 for (regNumber regNum = REG_FP_FIRST; regNum <= REG_FP_LAST; regNum = REG_NEXT(regNum))
317 if (VarSetOps::IsMember(this, raLclRegIntf[regNum], varNum))
318 printf("%3s ", getRegName(regNum, true));
326 for (regNumber regNum = REG_INT_FIRST; regNum <= REG_INT_LAST; regNum = REG_NEXT(regNum))
328 if (VarSetOps::IsMember(this, raLclRegIntf[regNum], varNum))
329 printf("%3s ", getRegName(regNum));
342 /*****************************************************************************
344 * We'll adjust the ref counts based on interference
348 void Compiler::raAdjustVarIntf()
350 // This method was not correct and has been disabled.
354 /*****************************************************************************/
355 /*****************************************************************************/
356 /* Determine register mask for a call/return from type.
359 inline regMaskTP Compiler::genReturnRegForTree(GenTreePtr tree)
361 var_types type = tree->TypeGet();
363 if (varTypeIsStruct(type) && IsHfa(tree))
365 int retSlots = GetHfaCount(tree);
366 return ((1 << retSlots) - 1) << REG_FLOATRET;
369 const static regMaskTP returnMap[TYP_COUNT] = {
370 RBM_ILLEGAL, // TYP_UNDEF,
371 RBM_NONE, // TYP_VOID,
372 RBM_INTRET, // TYP_BOOL,
373 RBM_INTRET, // TYP_BYTE,
374 RBM_INTRET, // TYP_UBYTE,
375 RBM_INTRET, // TYP_SHORT,
376 RBM_INTRET, // TYP_USHORT,
377 RBM_INTRET, // TYP_INT,
378 RBM_INTRET, // TYP_UINT,
379 RBM_LNGRET, // TYP_LONG,
380 RBM_LNGRET, // TYP_ULONG,
381 RBM_FLOATRET, // TYP_FLOAT,
382 RBM_DOUBLERET, // TYP_DOUBLE,
383 RBM_INTRET, // TYP_REF,
384 RBM_INTRET, // TYP_BYREF,
385 RBM_ILLEGAL, // TYP_STRUCT,
386 RBM_ILLEGAL, // TYP_BLK,
387 RBM_ILLEGAL, // TYP_LCLBLK,
388 RBM_ILLEGAL, // TYP_UNKNOWN,
391 assert((unsigned)type < _countof(returnMap));
392 assert(returnMap[TYP_LONG] == RBM_LNGRET);
393 assert(returnMap[TYP_DOUBLE] == RBM_DOUBLERET);
394 assert(returnMap[TYP_REF] == RBM_INTRET);
395 assert(returnMap[TYP_STRUCT] == RBM_ILLEGAL);
397 regMaskTP result = returnMap[type];
398 assert(result != RBM_ILLEGAL);
402 /*****************************************************************************/
404 /****************************************************************************/
408 static void dispLifeSet(Compiler* comp, VARSET_VALARG_TP mask, VARSET_VALARG_TP life)
413 for (lclNum = 0, varDsc = comp->lvaTable; lclNum < comp->lvaCount; lclNum++, varDsc++)
415 if (!varDsc->lvTracked)
418 if (!VarSetOps::IsMember(comp, mask, varDsc->lvVarIndex))
421 if (VarSetOps::IsMember(comp, life, varDsc->lvVarIndex))
422 printf("V%02u ", lclNum);
428 /*****************************************************************************/
430 /*****************************************************************************
432 * Debugging helpers - display variables liveness info.
435 void dispFPvarsInBBlist(BasicBlock* beg, BasicBlock* end, VARSET_TP mask, Compiler* comp)
439 printf("BB%02u: ", beg->bbNum);
442 dispLifeSet(comp, mask, beg->bbLiveIn);
446 dispLifeSet(comp, mask, beg->bbLiveOut);
449 if (beg->bbFlags & BBF_VISITED)
450 printf(" inner=%u", beg->bbFPinVars);
457 } while (beg != end);
460 #if FEATURE_STACK_FP_X87
461 void Compiler::raDispFPlifeInfo()
465 for (block = fgFirstBB; block; block = block->bbNext)
469 printf("BB%02u: in = [ ", block->bbNum);
470 dispLifeSet(this, optAllFloatVars, block->bbLiveIn);
473 VARSET_TP life(VarSetOps::MakeCopy(this, block->bbLiveIn));
474 for (stmt = block->bbTreeList; stmt; stmt = stmt->gtNext)
478 noway_assert(stmt->gtOper == GT_STMT);
480 for (tree = stmt->gtStmt.gtStmtList; tree; tree = tree->gtNext)
482 VarSetOps::AssignNoCopy(this, life, fgUpdateLiveSet(life, tree));
484 dispLifeSet(this, optAllFloatVars, life);
486 gtDispTree(tree, 0, NULL, true);
492 printf("BB%02u: out = [ ", block->bbNum);
493 dispLifeSet(this, optAllFloatVars, block->bbLiveOut);
497 #endif // FEATURE_STACK_FP_X87
498 /*****************************************************************************/
500 /*****************************************************************************/
502 /*****************************************************************************/
504 void Compiler::raSetRegVarOrder(
505 var_types regType, regNumber* customVarOrder, unsigned* customVarOrderSize, regMaskTP prefReg, regMaskTP avoidReg)
507 unsigned normalVarOrderSize;
508 const regNumber* normalVarOrder = raGetRegVarOrder(regType, &normalVarOrderSize);
510 unsigned listIndex = 0;
511 regMaskTP usedReg = avoidReg;
513 noway_assert(*customVarOrderSize >= normalVarOrderSize);
517 /* First place the preferred registers at the start of customVarOrder */
522 for (index = 0; index < normalVarOrderSize; index++)
524 regNum = normalVarOrder[index];
525 regBit = genRegMask(regNum);
527 if (usedReg & regBit)
530 if (prefReg & regBit)
533 noway_assert(listIndex < normalVarOrderSize);
534 customVarOrder[listIndex++] = regNum;
541 #if CPU_HAS_BYTE_REGS
542 /* Then if byteable registers are preferred place them */
544 if (prefReg & RBM_BYTE_REG_FLAG)
546 for (index = 0; index < normalVarOrderSize; index++)
548 regNum = normalVarOrder[index];
549 regBit = genRegMask(regNum);
551 if (usedReg & regBit)
554 if (RBM_BYTE_REGS & regBit)
557 noway_assert(listIndex < normalVarOrderSize);
558 customVarOrder[listIndex++] = regNum;
563 #endif // CPU_HAS_BYTE_REGS
566 /* Now place all the non-preferred registers */
568 for (index = 0; index < normalVarOrderSize; index++)
570 regNumber regNum = normalVarOrder[index];
571 regMaskTP regBit = genRegMask(regNum);
573 if (usedReg & regBit)
577 noway_assert(listIndex < normalVarOrderSize);
578 customVarOrder[listIndex++] = regNum;
583 /* Now place the "avoid" registers */
585 for (index = 0; index < normalVarOrderSize; index++)
587 regNumber regNum = normalVarOrder[index];
588 regMaskTP regBit = genRegMask(regNum);
590 if (avoidReg & regBit)
592 noway_assert(listIndex < normalVarOrderSize);
593 customVarOrder[listIndex++] = regNum;
601 *customVarOrderSize = listIndex;
602 noway_assert(listIndex == normalVarOrderSize);
605 /*****************************************************************************
607 * Setup the raAvoidArgRegMask and rsCalleeRegArgMaskLiveIn
610 void Compiler::raSetupArgMasks(RegState* regState)
612 /* Determine the registers holding incoming register arguments */
613 /* and setup raAvoidArgRegMask to the set of registers that we */
614 /* may want to avoid when enregistering the locals. */
616 regState->rsCalleeRegArgMaskLiveIn = RBM_NONE;
617 raAvoidArgRegMask = RBM_NONE;
619 LclVarDsc* argsEnd = lvaTable + info.compArgsCount;
621 for (LclVarDsc* argDsc = lvaTable; argDsc < argsEnd; argDsc++)
623 noway_assert(argDsc->lvIsParam);
625 // Is it a register argument ?
626 if (!argDsc->lvIsRegArg)
629 // only process args that apply to the current register file
630 if ((argDsc->IsFloatRegType() && !info.compIsVarArgs && !opts.compUseSoftFP) != regState->rsIsFloat)
635 // Is it dead on entry ??
636 // In certain cases such as when compJmpOpUsed is true,
637 // or when we have a generic type context arg that we must report
638 // then the arguments have to be kept alive throughout the prolog.
639 // So we have to consider it as live on entry.
641 bool keepArgAlive = compJmpOpUsed;
642 if ((unsigned(info.compTypeCtxtArg) != BAD_VAR_NUM) && lvaReportParamTypeArg() &&
643 ((lvaTable + info.compTypeCtxtArg) == argDsc))
648 if (!keepArgAlive && argDsc->lvTracked && !VarSetOps::IsMember(this, fgFirstBB->bbLiveIn, argDsc->lvVarIndex))
653 // The code to set the regState for each arg is outlined for shared use
655 regNumber inArgReg = raUpdateRegStateForArg(regState, argDsc);
657 // Do we need to try to avoid this incoming arg registers?
659 // If it's not tracked, don't do the stuff below.
660 if (!argDsc->lvTracked)
663 // If the incoming arg is used after a call it is live accross
664 // a call and will have to be allocated to a caller saved
665 // register anyway (a very common case).
667 // In this case it is pointless to ask that the higher ref count
668 // locals to avoid using the incoming arg register
670 unsigned argVarIndex = argDsc->lvVarIndex;
672 /* Does the incoming register and the arg variable interfere? */
674 if (!VarSetOps::IsMember(this, raLclRegIntf[inArgReg], argVarIndex))
676 // No they do not interfere,
677 // so we add inArgReg to raAvoidArgRegMask
679 raAvoidArgRegMask |= genRegMask(inArgReg);
682 if (argDsc->lvType == TYP_DOUBLE)
684 // Avoid the double register argument pair for register allocation.
685 if (!VarSetOps::IsMember(this, raLclRegIntf[inArgReg + 1], argVarIndex))
687 raAvoidArgRegMask |= genRegMask(static_cast<regNumber>(inArgReg + 1));
694 #endif // LEGACY_BACKEND
696 // The code to set the regState for each arg is outlined for shared use
697 // by linear scan. (It is not shared for System V AMD64 platform.)
698 regNumber Compiler::raUpdateRegStateForArg(RegState* regState, LclVarDsc* argDsc)
700 regNumber inArgReg = argDsc->lvArgReg;
701 regMaskTP inArgMask = genRegMask(inArgReg);
703 if (regState->rsIsFloat)
705 noway_assert(inArgMask & RBM_FLTARG_REGS);
707 else // regState is for the integer registers
709 // This might be the fixed return buffer register argument (on ARM64)
710 // We check and allow inArgReg to be theFixedRetBuffReg
711 if (hasFixedRetBuffReg() && (inArgReg == theFixedRetBuffReg()))
713 // We should have a TYP_BYREF or TYP_I_IMPL arg and not a TYP_STRUCT arg
714 noway_assert(argDsc->lvType == TYP_BYREF || argDsc->lvType == TYP_I_IMPL);
715 // We should have recorded the variable number for the return buffer arg
716 noway_assert(info.compRetBuffArg != BAD_VAR_NUM);
718 else // we have a regular arg
720 noway_assert(inArgMask & RBM_ARG_REGS);
724 regState->rsCalleeRegArgMaskLiveIn |= inArgMask;
727 if (argDsc->lvType == TYP_DOUBLE)
729 if (info.compIsVarArgs || opts.compUseSoftFP)
731 assert((inArgReg == REG_R0) || (inArgReg == REG_R2));
732 assert(!regState->rsIsFloat);
736 assert(regState->rsIsFloat);
737 assert(emitter::isDoubleReg(inArgReg));
739 regState->rsCalleeRegArgMaskLiveIn |= genRegMask((regNumber)(inArgReg + 1));
741 else if (argDsc->lvType == TYP_LONG)
743 assert((inArgReg == REG_R0) || (inArgReg == REG_R2));
744 assert(!regState->rsIsFloat);
745 regState->rsCalleeRegArgMaskLiveIn |= genRegMask((regNumber)(inArgReg + 1));
747 #endif // _TARGET_ARM_
749 #if FEATURE_MULTIREG_ARGS
750 if (varTypeIsStruct(argDsc->lvType))
752 if (argDsc->lvIsHfaRegArg())
754 assert(regState->rsIsFloat);
755 unsigned cSlots = GetHfaCount(argDsc->lvVerTypeInfo.GetClassHandleForValueClass());
756 for (unsigned i = 1; i < cSlots; i++)
758 assert(inArgReg + i <= LAST_FP_ARGREG);
759 regState->rsCalleeRegArgMaskLiveIn |= genRegMask(static_cast<regNumber>(inArgReg + i));
764 unsigned cSlots = argDsc->lvSize() / TARGET_POINTER_SIZE;
765 for (unsigned i = 1; i < cSlots; i++)
767 regNumber nextArgReg = (regNumber)(inArgReg + i);
768 if (nextArgReg > REG_ARG_LAST)
772 assert(regState->rsIsFloat == false);
773 regState->rsCalleeRegArgMaskLiveIn |= genRegMask(nextArgReg);
777 #endif // FEATURE_MULTIREG_ARGS
782 #ifdef LEGACY_BACKEND // We don't use any of the old register allocator functions when LSRA is used instead.
784 /*****************************************************************************
786 * Assign variables to live in registers, etc.
789 void Compiler::raAssignVars()
793 printf("*************** In raAssignVars()\n");
795 /* We need to keep track of which registers we ever touch */
797 codeGen->regSet.rsClearRegsModified();
799 #if FEATURE_STACK_FP_X87
800 // FP register allocation
801 raEnregisterVarsStackFP();
802 raGenerateFPRefCounts();
805 /* Predict registers used by code generation */
806 rpPredictRegUse(); // New reg predictor/allocator
808 // Change all unused promoted non-argument struct locals to a non-GC type (in this case TYP_INT)
809 // so that the gc tracking logic and lvMustInit logic will ignore them.
814 for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
816 if (varDsc->lvType != TYP_STRUCT)
819 if (!varDsc->lvPromoted)
822 if (varDsc->lvIsParam)
825 if (varDsc->lvRefCnt > 0)
831 printf("Mark unused struct local V%02u\n", lclNum);
834 lvaPromotionType promotionType = lvaGetPromotionType(varDsc);
836 if (promotionType == PROMOTION_TYPE_DEPENDENT)
838 // This should only happen when all its field locals are unused as well.
840 for (unsigned varNum = varDsc->lvFieldLclStart; varNum < varDsc->lvFieldLclStart + varDsc->lvFieldCnt;
843 noway_assert(lvaTable[varNum].lvRefCnt == 0);
848 noway_assert(promotionType == PROMOTION_TYPE_INDEPENDENT);
851 varDsc->lvUnusedStruct = 1;
854 // Change such struct locals to ints
856 varDsc->lvType = TYP_INT; // Bash to a non-gc type.
857 noway_assert(!varDsc->lvTracked);
858 noway_assert(!varDsc->lvRegister);
859 varDsc->lvOnFrame = false; // Force it not to be onstack.
860 varDsc->lvMustInit = false; // Force not to init it.
861 varDsc->lvStkOffs = 0; // Set it to anything other than BAD_STK_OFFS to make genSetScopeInfo() happy
865 /*****************************************************************************/
866 /*****************************************************************************/
868 /*****************************************************************************
870 * Given a regNumber return the correct predictReg enum value
873 inline static rpPredictReg rpGetPredictForReg(regNumber reg)
875 return (rpPredictReg)(((int)reg) + ((int)PREDICT_REG_FIRST));
878 /*****************************************************************************
880 * Given a varIndex return the correct predictReg enum value
883 inline static rpPredictReg rpGetPredictForVarIndex(unsigned varIndex)
885 return (rpPredictReg)(varIndex + ((int)PREDICT_REG_VAR_T00));
888 /*****************************************************************************
890 * Given a rpPredictReg return the correct varNumber value
893 inline static unsigned rpGetVarIndexForPredict(rpPredictReg predict)
895 return (unsigned)predict - (unsigned)PREDICT_REG_VAR_T00;
898 /*****************************************************************************
900 * Given a rpPredictReg return true if it specifies a Txx register
903 inline static bool rpHasVarIndexForPredict(rpPredictReg predict)
905 if ((predict >= PREDICT_REG_VAR_T00) && (predict <= PREDICT_REG_VAR_MAX))
911 /*****************************************************************************
913 * Given a regmask return the correct predictReg enum value
916 static rpPredictReg rpGetPredictForMask(regMaskTP regmask)
918 rpPredictReg result = PREDICT_NONE;
919 if (regmask != 0) /* Check if regmask has zero bits set */
921 if (((regmask - 1) & regmask) == 0) /* Check if regmask has one bit set */
924 assert(FitsIn<DWORD>(regmask));
925 BitScanForward(®, (DWORD)regmask);
926 return rpGetPredictForReg((regNumber)reg);
929 #if defined(_TARGET_ARM_)
930 /* It has multiple bits set */
931 else if (regmask == (RBM_R0 | RBM_R1))
933 result = PREDICT_PAIR_R0R1;
935 else if (regmask == (RBM_R2 | RBM_R3))
937 result = PREDICT_PAIR_R2R3;
939 #elif defined(_TARGET_X86_)
940 /* It has multiple bits set */
941 else if (regmask == (RBM_EAX | RBM_EDX))
943 result = PREDICT_PAIR_EAXEDX;
945 else if (regmask == (RBM_ECX | RBM_EBX))
947 result = PREDICT_PAIR_ECXEBX;
950 else /* It doesn't match anything */
952 result = PREDICT_NONE;
953 assert(!"unreachable");
954 NO_WAY("bad regpair");
960 /*****************************************************************************
962 * Record a variable to register(s) interference
965 bool Compiler::rpRecordRegIntf(regMaskTP regMask, VARSET_VALARG_TP life DEBUGARG(const char* msg))
968 bool addedIntf = false;
972 for (regNumber regNum = REG_FIRST; regNum < REG_COUNT; regNum = REG_NEXT(regNum))
974 regMaskTP regBit = genRegMask(regNum);
976 if (regMask & regBit)
978 VARSET_TP newIntf(VarSetOps::Diff(this, life, raLclRegIntf[regNum]));
979 if (!VarSetOps::IsEmpty(this, newIntf))
984 VarSetOps::Iter newIntfIter(this, newIntf);
986 while (newIntfIter.NextElem(&varNum))
988 unsigned lclNum = lvaTrackedToVarNum[varNum];
989 LclVarDsc* varDsc = &lvaTable[varNum];
990 #if FEATURE_FP_REGALLOC
991 // Only print the useful interferences
992 // i.e. floating point LclVar interference with floating point registers
993 // or integer LclVar interference with general purpose registers
994 if (varTypeIsFloating(varDsc->TypeGet()) == genIsValidFloatReg(regNum))
997 printf("Record interference between V%02u,T%02u and %s -- %s\n", lclNum, varNum,
998 getRegName(regNum), msg);
1004 VarSetOps::UnionD(this, raLclRegIntf[regNum], newIntf);
1016 /*****************************************************************************
1018 * Record a new variable to variable(s) interference
1021 bool Compiler::rpRecordVarIntf(unsigned varNum, VARSET_VALARG_TP intfVar DEBUGARG(const char* msg))
1023 noway_assert((varNum >= 0) && (varNum < lvaTrackedCount));
1024 noway_assert(!VarSetOps::IsEmpty(this, intfVar));
1026 VARSET_TP oneVar(VarSetOps::MakeEmpty(this));
1027 VarSetOps::AddElemD(this, oneVar, varNum);
1029 bool newIntf = fgMarkIntf(intfVar, oneVar);
1032 rpAddedVarIntf = true;
1035 if (verbose && newIntf)
1037 for (unsigned oneNum = 0; oneNum < lvaTrackedCount; oneNum++)
1039 if (VarSetOps::IsMember(this, intfVar, oneNum))
1041 unsigned lclNum = lvaTrackedToVarNum[varNum];
1042 unsigned lclOne = lvaTrackedToVarNum[oneNum];
1043 printf("Record interference between V%02u,T%02u and V%02u,T%02u -- %s\n", lclNum, varNum, lclOne,
1053 /*****************************************************************************
1055 * Determine preferred register mask for a given predictReg value
1058 inline regMaskTP Compiler::rpPredictRegMask(rpPredictReg predictReg, var_types type)
1060 if (rpHasVarIndexForPredict(predictReg))
1061 predictReg = PREDICT_REG;
1063 noway_assert((unsigned)predictReg < _countof(rpPredictMap));
1064 noway_assert(rpPredictMap[predictReg] != RBM_ILLEGAL);
1066 regMaskTP regAvailForType = rpPredictMap[predictReg];
1067 if (varTypeIsFloating(type))
1069 regAvailForType &= RBM_ALLFLOAT;
1073 regAvailForType &= RBM_ALLINT;
1076 if (type == TYP_DOUBLE)
1078 if ((predictReg >= PREDICT_REG_F0) && (predictReg <= PREDICT_REG_F31))
1080 // Fix 388433 ARM JitStress WP7
1081 if ((regAvailForType & RBM_DBL_REGS) != 0)
1083 regAvailForType |= (regAvailForType << 1);
1087 regAvailForType = RBM_NONE;
1092 return regAvailForType;
1095 /*****************************************************************************
1097 * Predict register choice for a type.
1099 * Adds the predicted registers to rsModifiedRegsMask.
1101 regMaskTP Compiler::rpPredictRegPick(var_types type, rpPredictReg predictReg, regMaskTP lockedRegs)
1103 regMaskTP preferReg = rpPredictRegMask(predictReg, type);
1107 // Add any reserved register to the lockedRegs
1108 lockedRegs |= codeGen->regSet.rsMaskResvd;
1110 /* Clear out the lockedRegs from preferReg */
1111 preferReg &= ~lockedRegs;
1113 if (rpAsgVarNum != -1)
1115 noway_assert((rpAsgVarNum >= 0) && (rpAsgVarNum < (int)lclMAX_TRACKED));
1117 /* Don't pick the register used by rpAsgVarNum either */
1118 LclVarDsc* tgtVar = lvaTable + lvaTrackedToVarNum[rpAsgVarNum];
1119 noway_assert(tgtVar->lvRegNum != REG_STK);
1121 preferReg &= ~genRegMask(tgtVar->lvRegNum);
1135 #ifdef _TARGET_AMD64_
1137 #endif // _TARGET_AMD64_
1139 // expand preferReg to all non-locked registers if no bits set
1140 preferReg = codeGen->regSet.rsUseIfZero(preferReg & RBM_ALLINT, RBM_ALLINT & ~lockedRegs);
1142 if (preferReg == 0) // no bits set?
1144 // Add one predefined spill choice register if no bits set.
1145 // (The jit will introduce one spill temp)
1146 preferReg |= RBM_SPILL_CHOICE;
1147 rpPredictSpillCnt++;
1151 printf("Predict one spill temp\n");
1157 /* Iterate the registers in the order specified by rpRegTmpOrder */
1159 for (unsigned index = 0; index < REG_TMP_ORDER_COUNT; index++)
1161 regNum = rpRegTmpOrder[index];
1162 regBits = genRegMask(regNum);
1164 if ((preferReg & regBits) == regBits)
1170 /* Otherwise we have allocated all registers, so do nothing */
1173 #ifndef _TARGET_AMD64_
1176 if ((preferReg == 0) || // no bits set?
1177 ((preferReg & (preferReg - 1)) == 0)) // or only one bit set?
1179 // expand preferReg to all non-locked registers
1180 preferReg = RBM_ALLINT & ~lockedRegs;
1183 if (preferReg == 0) // no bits set?
1185 // Add EAX:EDX to the registers
1186 // (The jit will introduce two spill temps)
1187 preferReg = RBM_PAIR_TMP;
1188 rpPredictSpillCnt += 2;
1191 printf("Predict two spill temps\n");
1194 else if ((preferReg & (preferReg - 1)) == 0) // only one bit set?
1196 if ((preferReg & RBM_PAIR_TMP_LO) == 0)
1198 // Add EAX to the registers
1199 // (The jit will introduce one spill temp)
1200 preferReg |= RBM_PAIR_TMP_LO;
1204 // Add EDX to the registers
1205 // (The jit will introduce one spill temp)
1206 preferReg |= RBM_PAIR_TMP_HI;
1208 rpPredictSpillCnt++;
1211 printf("Predict one spill temp\n");
1216 regPair = codeGen->regSet.rsFindRegPairNo(preferReg);
1217 if (regPair != REG_PAIR_NONE)
1219 regBits = genRegPairMask(regPair);
1223 /* Otherwise we have allocated all registers, so do nothing */
1225 #endif // _TARGET_AMD64_
1234 #if FEATURE_FP_REGALLOC
1235 regMaskTP restrictMask;
1236 restrictMask = (raConfigRestrictMaskFP() | RBM_FLT_CALLEE_TRASH);
1237 assert((restrictMask & RBM_SPILL_CHOICE_FLT) == RBM_SPILL_CHOICE_FLT);
1239 // expand preferReg to all available non-locked registers if no bits set
1240 preferReg = codeGen->regSet.rsUseIfZero(preferReg & restrictMask, restrictMask & ~lockedRegs);
1241 regMaskTP preferDouble;
1242 preferDouble = preferReg & (preferReg >> 1);
1244 if ((preferReg == 0) // no bits set?
1246 || ((type == TYP_DOUBLE) &&
1247 ((preferReg & (preferReg >> 1)) == 0)) // or two consecutive bits set for TYP_DOUBLE
1251 // Add one predefined spill choice register if no bits set.
1252 // (The jit will introduce one spill temp)
1253 preferReg |= RBM_SPILL_CHOICE_FLT;
1254 rpPredictSpillCnt++;
1258 printf("Predict one spill temp (float)\n");
1262 assert(preferReg != 0);
1264 /* Iterate the registers in the order specified by raRegFltTmpOrder */
1266 for (unsigned index = 0; index < REG_FLT_TMP_ORDER_COUNT; index++)
1268 regNum = raRegFltTmpOrder[index];
1269 regBits = genRegMask(regNum);
1271 if (varTypeIsFloating(type))
1274 if (type == TYP_DOUBLE)
1276 if ((regBits & RBM_DBL_REGS) == 0)
1278 continue; // We must restrict the set to the double registers
1282 // TYP_DOUBLE use two consecutive registers
1283 regBits |= genRegMask(REG_NEXT(regNum));
1287 // See if COMPlus_JitRegisterFP is restricting this FP register
1289 if ((restrictMask & regBits) != regBits)
1293 if ((preferReg & regBits) == regBits)
1298 /* Otherwise we have allocated all registers, so do nothing */
1301 #else // !FEATURE_FP_REGALLOC
1308 noway_assert(!"unexpected type in reg use prediction");
1311 /* Abnormal return */
1312 noway_assert(!"Ran out of registers in rpPredictRegPick");
1317 * If during the first prediction we need to allocate
1318 * one of the registers that we used for coloring locals
1319 * then flag this by setting rpPredictAssignAgain.
1320 * We will have to go back and repredict the registers
1322 if ((rpPasses == 0) && ((rpPredictAssignMask & regBits) == regBits))
1323 rpPredictAssignAgain = true;
1325 // Add a register interference to each of the last use variables
1326 if (!VarSetOps::IsEmpty(this, rpLastUseVars) || !VarSetOps::IsEmpty(this, rpUseInPlace))
1328 VARSET_TP lastUse(VarSetOps::MakeEmpty(this));
1329 VarSetOps::Assign(this, lastUse, rpLastUseVars);
1330 VARSET_TP inPlaceUse(VarSetOps::MakeEmpty(this));
1331 VarSetOps::Assign(this, inPlaceUse, rpUseInPlace);
1332 // While we still have any lastUse or inPlaceUse bits
1333 VARSET_TP useUnion(VarSetOps::Union(this, lastUse, inPlaceUse));
1335 VARSET_TP varAsSet(VarSetOps::MakeEmpty(this));
1336 VarSetOps::Iter iter(this, useUnion);
1337 unsigned varNum = 0;
1338 while (iter.NextElem(&varNum))
1340 // We'll need this for one of the calls...
1341 VarSetOps::ClearD(this, varAsSet);
1342 VarSetOps::AddElemD(this, varAsSet, varNum);
1344 // If this varBit and lastUse?
1345 if (VarSetOps::IsMember(this, lastUse, varNum))
1347 // Record a register to variable interference
1348 rpRecordRegIntf(regBits, varAsSet DEBUGARG("last use RegPick"));
1351 // If this varBit and inPlaceUse?
1352 if (VarSetOps::IsMember(this, inPlaceUse, varNum))
1354 // Record a register to variable interference
1355 rpRecordRegIntf(regBits, varAsSet DEBUGARG("used in place RegPick"));
1359 codeGen->regSet.rsSetRegsModified(regBits);
1364 /*****************************************************************************
1366 * Predict integer register use for generating an address mode for a tree,
1367 * by setting tree->gtUsedRegs to all registers used by this tree and its
1369 * tree - is the child of a GT_IND node
1370 * type - the type of the GT_IND node (floating point/integer)
1371 * lockedRegs - are the registers which are currently held by
1372 * a previously evaluated node.
1373 * rsvdRegs - registers which should not be allocated because they will
1374 * be needed to evaluate a node in the future
1375 * - Also if rsvdRegs has the RBM_LASTUSE bit set then
1376 * the rpLastUseVars set should be saved and restored
1377 * so that we don't add any new variables to rpLastUseVars
1378 * lenCSE - is non-NULL only when we have a lenCSE expression
1380 * Return the scratch registers to be held by this tree. (one or two registers
1381 * to form an address expression)
1384 regMaskTP Compiler::rpPredictAddressMode(
1385 GenTreePtr tree, var_types type, regMaskTP lockedRegs, regMaskTP rsvdRegs, GenTreePtr lenCSE)
1390 genTreeOps oper = tree->OperGet();
1397 bool hasTwoAddConst = false;
1398 bool restoreLastUseVars = false;
1399 VARSET_TP oldLastUseVars(VarSetOps::MakeEmpty(this));
1401 /* do we need to save and restore the rpLastUseVars set ? */
1402 if ((rsvdRegs & RBM_LASTUSE) && (lenCSE == NULL))
1404 restoreLastUseVars = true;
1405 VarSetOps::Assign(this, oldLastUseVars, rpLastUseVars);
1407 rsvdRegs &= ~RBM_LASTUSE;
1409 /* if not an add, then just force it to a register */
1413 if (oper == GT_ARR_ELEM)
1415 regMask = rpPredictTreeRegUse(tree, PREDICT_NONE, lockedRegs, rsvdRegs);
1424 op1 = tree->gtOp.gtOp1;
1425 op2 = tree->gtOp.gtOp2;
1426 rev = ((tree->gtFlags & GTF_REVERSE_OPS) != 0);
1428 /* look for (x + y) + icon address mode */
1430 if (op2->OperGet() == GT_CNS_INT)
1432 cns = op2->gtIntCon.gtIconVal;
1434 /* if not an add, then just force op1 into a register */
1435 if (op1->OperGet() != GT_ADD)
1438 hasTwoAddConst = true;
1440 /* Record the 'rev' flag, reverse evaluation order */
1441 rev = ((op1->gtFlags & GTF_REVERSE_OPS) != 0);
1443 op2 = op1->gtOp.gtOp2;
1444 op1 = op1->gtOp.gtOp1; // Overwrite op1 last!!
1447 /* Check for CNS_INT or LSH of CNS_INT in op2 slot */
1450 if (op2->OperGet() == GT_LSH)
1452 if (op2->gtOp.gtOp2->OperGet() == GT_CNS_INT)
1454 sh = op2->gtOp.gtOp2->gtIntCon.gtIconVal;
1455 opTemp = op2->gtOp.gtOp1;
1469 if (opTemp->OperGet() == GT_NOP)
1471 opTemp = opTemp->gtOp.gtOp1;
1474 // Is this a const operand?
1475 if (opTemp->OperGet() == GT_CNS_INT)
1477 // Compute the new cns value that Codegen will end up using
1478 cns += (opTemp->gtIntCon.gtIconVal << sh);
1484 /* Check for LSH in op1 slot */
1486 if (op1->OperGet() != GT_LSH)
1489 opTemp = op1->gtOp.gtOp2;
1491 if (opTemp->OperGet() != GT_CNS_INT)
1494 sh = opTemp->gtIntCon.gtIconVal;
1496 /* Check for LSH of 0, special case */
1500 #if defined(_TARGET_XARCH_)
1502 /* Check for LSH of 1 2 or 3 */
1506 #elif defined(_TARGET_ARM_)
1508 /* Check for LSH of 1 to 30 */
1518 /* Matched a leftShift by 'sh' subtree, move op1 down */
1519 op1 = op1->gtOp.gtOp1;
1523 /* Now we have to evaluate op1 and op2 into registers */
1525 /* Evaluate op1 and op2 in the correct order */
1528 op2Mask = rpPredictTreeRegUse(op2, PREDICT_REG, lockedRegs, rsvdRegs | op1->gtRsvdRegs);
1529 op1Mask = rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs | op2Mask, rsvdRegs);
1533 op1Mask = rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs, rsvdRegs | op2->gtRsvdRegs);
1534 op2Mask = rpPredictTreeRegUse(op2, PREDICT_REG, lockedRegs | op1Mask, rsvdRegs);
1537 /* If op1 and op2 must be spilled and reloaded then
1538 * op1 and op2 might be reloaded into the same register
1539 * This can only happen when all the registers are lockedRegs
1541 if ((op1Mask == op2Mask) && (op1Mask != 0))
1543 /* We'll need to grab a different register for op2 */
1544 op2Mask = rpPredictRegPick(TYP_INT, PREDICT_REG, op1Mask);
1548 // On the ARM we need a scratch register to evaluate the shifted operand for trees that have this form
1549 // [op2 + op1<<sh + cns]
1550 // when op1 is an enregistered variable, thus the op1Mask is RBM_NONE
1552 if (hasTwoAddConst && (sh != 0) && (op1Mask == RBM_NONE))
1554 op1Mask |= rpPredictRegPick(TYP_INT, PREDICT_REG, (lockedRegs | op1Mask | op2Mask));
1558 // On the ARM we will need at least one scratch register for trees that have this form:
1559 // [op1 + op2 + cns] or [op1 + op2<<sh + cns]
1560 // or for a float/double or long when we have both op1 and op2
1561 // or when we have an 'cns' that is too large for the ld/st instruction
1563 if (hasTwoAddConst || varTypeIsFloating(type) || (type == TYP_LONG) || !codeGen->validDispForLdSt(cns, type))
1565 op2Mask |= rpPredictRegPick(TYP_INT, PREDICT_REG, (lockedRegs | op1Mask | op2Mask));
1569 // If we create a CSE that immediately dies then we may need to add an additional register interference
1570 // so we don't color the CSE into R3
1572 if (!rev && (op1Mask != RBM_NONE) && (op2->OperGet() == GT_COMMA))
1574 opTemp = op2->gtOp.gtOp2;
1575 if (opTemp->OperGet() == GT_LCL_VAR)
1577 unsigned varNum = opTemp->gtLclVar.gtLclNum;
1578 LclVarDsc* varDsc = &lvaTable[varNum];
1580 if (varDsc->lvTracked && !VarSetOps::IsMember(this, compCurLife, varDsc->lvVarIndex))
1582 rpRecordRegIntf(RBM_TMP_0,
1583 VarSetOps::MakeSingleton(this, varDsc->lvVarIndex) DEBUGARG("dead CSE (gt_ind)"));
1589 regMask = (op1Mask | op2Mask);
1590 tree->gtUsedRegs = (regMaskSmall)regMask;
1595 /* now we have to evaluate op1 into a register */
1597 op1Mask = rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs, rsvdRegs);
1602 // On the ARM we will need another scratch register when we have an 'cns' that is too large for the ld/st
1605 if (!codeGen->validDispForLdSt(cns, type))
1607 op2Mask |= rpPredictRegPick(TYP_INT, PREDICT_REG, (lockedRegs | op1Mask | op2Mask));
1611 regMask = (op1Mask | op2Mask);
1612 tree->gtUsedRegs = (regMaskSmall)regMask;
1617 #if !CPU_LOAD_STORE_ARCH
1618 if (oper == GT_CNS_INT)
1620 /* Indirect of a constant does not require a register */
1626 /* now we have to evaluate tree into a register */
1627 regMask = rpPredictTreeRegUse(tree, PREDICT_REG, lockedRegs, rsvdRegs);
1631 regMaskTP regUse = tree->gtUsedRegs;
1633 if (!VarSetOps::IsEmpty(this, compCurLife))
1635 // Add interference between the current set of life variables and
1636 // the set of temporary registers need to evaluate the sub tree
1639 rpRecordRegIntf(regUse, compCurLife DEBUGARG("tmp use (gt_ind)"));
1643 /* Do we need to resore the oldLastUseVars value */
1644 if (restoreLastUseVars)
1647 * If we used a GT_ASG targeted register then we need to add
1648 * a variable interference between any new last use variables
1649 * and the GT_ASG targeted register
1651 if (!VarSetOps::Equal(this, rpLastUseVars, oldLastUseVars) && rpAsgVarNum != -1)
1653 rpRecordVarIntf(rpAsgVarNum,
1654 VarSetOps::Diff(this, rpLastUseVars, oldLastUseVars) DEBUGARG("asgn conflict (gt_ind)"));
1656 VarSetOps::Assign(this, rpLastUseVars, oldLastUseVars);
1662 /*****************************************************************************
1667 void Compiler::rpPredictRefAssign(unsigned lclNum)
1669 LclVarDsc* varDsc = lvaTable + lclNum;
1671 varDsc->lvRefAssign = 1;
1673 #if NOGC_WRITE_BARRIERS
1677 if (!VarSetOps::IsMember(this, raLclRegIntf[REG_EDX], varDsc->lvVarIndex))
1678 printf("Record interference between V%02u,T%02u and REG WRITE BARRIER -- ref assign\n", lclNum,
1679 varDsc->lvVarIndex);
1683 /* Make sure that write barrier pointer variables never land in EDX */
1684 VarSetOps::AddElemD(this, raLclRegIntf[REG_EDX], varDsc->lvVarIndex);
1685 #endif // NOGC_WRITE_BARRIERS
1688 /*****************************************************************************
1690 * Predict the internal temp physical register usage for a block assignment tree,
1691 * by setting tree->gtUsedRegs.
1692 * Records the internal temp physical register usage for this tree.
1693 * Returns a mask of interfering registers for this tree.
1695 * Each of the switch labels in this function updates regMask and assigns tree->gtUsedRegs
1696 * to the set of scratch registers needed when evaluating the tree.
1697 * Generally tree->gtUsedRegs and the return value retMask are the same, except when the
1698 * parameter "lockedRegs" conflicts with the computed tree->gtUsedRegs, in which case we
1699 * predict additional internal temp physical registers to spill into.
1701 * tree - is the child of a GT_IND node
1702 * predictReg - what type of register does the tree need
1703 * lockedRegs - are the registers which are currently held by a previously evaluated node.
1704 * Don't modify lockedRegs as it is used at the end to compute a spill mask.
1705 * rsvdRegs - registers which should not be allocated because they will
1706 * be needed to evaluate a node in the future
1707 * - Also, if rsvdRegs has the RBM_LASTUSE bit set then
1708 * the rpLastUseVars set should be saved and restored
1709 * so that we don't add any new variables to rpLastUseVars.
1711 regMaskTP Compiler::rpPredictBlkAsgRegUse(GenTreePtr tree,
1712 rpPredictReg predictReg,
1713 regMaskTP lockedRegs,
1716 regMaskTP regMask = RBM_NONE;
1717 regMaskTP interferingRegs = RBM_NONE;
1719 bool hasGCpointer = false;
1720 bool dstIsOnStack = false;
1721 bool useMemHelper = false;
1722 bool useBarriers = false;
1723 GenTreeBlk* dst = tree->gtGetOp1()->AsBlk();
1724 GenTreePtr dstAddr = dst->Addr();
1725 GenTreePtr srcAddrOrFill = tree->gtGetOp2IfPresent();
1727 size_t blkSize = dst->gtBlkSize;
1729 hasGCpointer = (dst->HasGCPtr());
1731 bool isCopyBlk = tree->OperIsCopyBlkOp();
1732 bool isCopyObj = isCopyBlk && hasGCpointer;
1733 bool isInitBlk = tree->OperIsInitBlkOp();
1737 assert(srcAddrOrFill->OperIsIndir());
1738 srcAddrOrFill = srcAddrOrFill->AsIndir()->Addr();
1742 // For initBlk, we don't need to worry about the GC pointers.
1743 hasGCpointer = false;
1750 dstIsOnStack = (dstAddr->gtOper == GT_ADDR && (dstAddr->gtFlags & GTF_ADDR_ONSTACK));
1755 if (srcAddrOrFill->OperGet() != GT_CNS_INT)
1757 useMemHelper = true;
1763 useMemHelper = true;
1766 if (hasGCpointer && !dstIsOnStack)
1773 // On ARM For COPYBLK & INITBLK we have special treatment for constant lengths.
1775 if (!useMemHelper && !useBarriers)
1777 bool useLoop = false;
1778 unsigned fullStoreCount = blkSize / TARGET_POINTER_SIZE;
1780 // A mask to use to force the predictor to choose low registers (to reduce code size)
1781 regMaskTP avoidReg = (RBM_R12 | RBM_LR);
1783 // Allow the src and dst to be used in place, unless we use a loop, in which
1784 // case we will need scratch registers as we will be writing to them.
1785 rpPredictReg srcAndDstPredict = PREDICT_REG;
1787 // Will we be using a loop to implement this INITBLK/COPYBLK?
1788 if ((isCopyBlk && (fullStoreCount >= 8)) || (isInitBlk && (fullStoreCount >= 16)))
1791 avoidReg = RBM_NONE;
1792 srcAndDstPredict = PREDICT_SCRATCH_REG;
1795 if (tree->gtFlags & GTF_REVERSE_OPS)
1797 regMask |= rpPredictTreeRegUse(srcAddrOrFill, srcAndDstPredict, lockedRegs,
1798 dstAddr->gtRsvdRegs | avoidReg | RBM_LASTUSE);
1799 regMask |= rpPredictTreeRegUse(dstAddr, srcAndDstPredict, lockedRegs | regMask, avoidReg);
1803 regMask |= rpPredictTreeRegUse(dstAddr, srcAndDstPredict, lockedRegs,
1804 srcAddrOrFill->gtRsvdRegs | avoidReg | RBM_LASTUSE);
1805 regMask |= rpPredictTreeRegUse(srcAddrOrFill, srcAndDstPredict, lockedRegs | regMask, avoidReg);
1808 // We need at least one scratch register for a copyBlk
1811 // Pick a low register to reduce the code size
1812 regMask |= rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | regMask | avoidReg);
1819 // We need a second temp register for a copyBlk (our code gen is load two/store two)
1820 // Pick another low register to reduce the code size
1821 regMask |= rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | regMask | avoidReg);
1824 // We need a loop index register
1825 regMask |= rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | regMask);
1828 tree->gtUsedRegs = dstAddr->gtUsedRegs | srcAddrOrFill->gtUsedRegs | (regMaskSmall)regMask;
1830 return interferingRegs;
1833 // What order should the Dest, Val/Src, and Size be calculated
1834 GenTreePtr opsPtr[3];
1835 regMaskTP regsPtr[3];
1837 #if defined(_TARGET_XARCH_)
1838 fgOrderBlockOps(tree, RBM_EDI, (isInitBlk) ? RBM_EAX : RBM_ESI, RBM_ECX, opsPtr, regsPtr);
1840 // We're going to use these, might as well make them available now
1842 codeGen->regSet.rsSetRegsModified(RBM_EDI | RBM_ECX);
1844 codeGen->regSet.rsSetRegsModified(RBM_ESI);
1846 #elif defined(_TARGET_ARM_)
1850 // For all other cases that involve non-constants, we just call memcpy/memset
1852 fgOrderBlockOps(tree, RBM_ARG_0, RBM_ARG_1, RBM_ARG_2, opsPtr, regsPtr);
1853 interferingRegs |= RBM_CALLEE_TRASH;
1856 printf("Adding interference with RBM_CALLEE_TRASH for memcpy/memset\n");
1861 assert(useBarriers);
1864 fgOrderBlockOps(tree, RBM_ARG_0, RBM_ARG_1, REG_TMP_1, opsPtr, regsPtr);
1866 // For this case Codegen will call the CORINFO_HELP_ASSIGN_BYREF helper
1867 interferingRegs |= RBM_CALLEE_TRASH_NOGC;
1870 printf("Adding interference with RBM_CALLEE_TRASH_NOGC for Byref WriteBarrier\n");
1873 #else // !_TARGET_X86_ && !_TARGET_ARM_
1874 #error "Non-ARM or x86 _TARGET_ in RegPredict for INITBLK/COPYBLK"
1875 #endif // !_TARGET_X86_ && !_TARGET_ARM_
1876 regMaskTP opsPtr2RsvdRegs = opsPtr[2] == nullptr ? RBM_NONE : opsPtr[2]->gtRsvdRegs;
1877 regMask |= rpPredictTreeRegUse(opsPtr[0], rpGetPredictForMask(regsPtr[0]), lockedRegs,
1878 opsPtr[1]->gtRsvdRegs | opsPtr2RsvdRegs | RBM_LASTUSE);
1879 regMask |= regsPtr[0];
1880 opsPtr[0]->gtUsedRegs |= regsPtr[0];
1881 rpRecordRegIntf(regsPtr[0], compCurLife DEBUGARG("movsd dest"));
1883 regMask |= rpPredictTreeRegUse(opsPtr[1], rpGetPredictForMask(regsPtr[1]), lockedRegs | regMask,
1884 opsPtr2RsvdRegs | RBM_LASTUSE);
1885 regMask |= regsPtr[1];
1886 opsPtr[1]->gtUsedRegs |= regsPtr[1];
1887 rpRecordRegIntf(regsPtr[1], compCurLife DEBUGARG("movsd src"));
1889 regMaskSmall opsPtr2UsedRegs = (regMaskSmall)regsPtr[2];
1890 if (opsPtr[2] == nullptr)
1892 // If we have no "size" node, we will predict that regsPtr[2] will be used for the size.
1893 // Note that it is quite possible that no register is required, but this preserves
1895 regMask |= rpPredictRegPick(TYP_INT, rpGetPredictForMask(regsPtr[2]), lockedRegs | regMask);
1896 rpRecordRegIntf(regsPtr[2], compCurLife DEBUGARG("tmp use"));
1900 regMask |= rpPredictTreeRegUse(opsPtr[2], rpGetPredictForMask(regsPtr[2]), lockedRegs | regMask, RBM_NONE);
1901 opsPtr[2]->gtUsedRegs |= opsPtr2UsedRegs;
1903 regMask |= opsPtr2UsedRegs;
1905 tree->gtUsedRegs = opsPtr[0]->gtUsedRegs | opsPtr[1]->gtUsedRegs | opsPtr2UsedRegs | (regMaskSmall)regMask;
1906 return interferingRegs;
1909 /*****************************************************************************
1911 * Predict the internal temp physical register usage for a tree by setting tree->gtUsedRegs.
1912 * Returns a regMask with the internal temp physical register usage for this tree.
1914 * Each of the switch labels in this function updates regMask and assigns tree->gtUsedRegs
1915 * to the set of scratch registers needed when evaluating the tree.
1916 * Generally tree->gtUsedRegs and the return value retMask are the same, except when the
1917 * parameter "lockedRegs" conflicts with the computed tree->gtUsedRegs, in which case we
1918 * predict additional internal temp physical registers to spill into.
1920 * tree - is the child of a GT_IND node
1921 * predictReg - what type of register does the tree need
1922 * lockedRegs - are the registers which are currently held by a previously evaluated node.
1923 * Don't modify lockedRegs as it is used at the end to compute a spill mask.
1924 * rsvdRegs - registers which should not be allocated because they will
1925 * be needed to evaluate a node in the future
1926 * - Also, if rsvdRegs has the RBM_LASTUSE bit set then
1927 * the rpLastUseVars set should be saved and restored
1928 * so that we don't add any new variables to rpLastUseVars.
1931 #pragma warning(disable : 4701)
1934 #pragma warning(push)
1935 #pragma warning(disable : 21000) // Suppress PREFast warning about overly large function
1937 regMaskTP Compiler::rpPredictTreeRegUse(GenTreePtr tree,
1938 rpPredictReg predictReg,
1939 regMaskTP lockedRegs,
1942 regMaskTP regMask = DUMMY_INIT(RBM_ILLEGAL);
1945 rpPredictReg op1PredictReg;
1946 rpPredictReg op2PredictReg;
1947 LclVarDsc* varDsc = NULL;
1948 VARSET_TP oldLastUseVars(VarSetOps::UninitVal());
1950 VARSET_TP varBits(VarSetOps::UninitVal());
1951 VARSET_TP lastUseVarBits(VarSetOps::MakeEmpty(this));
1953 bool restoreLastUseVars = false;
1954 regMaskTP interferingRegs = RBM_NONE;
1957 // if (verbose) printf("rpPredictTreeRegUse() [%08x]\n", tree);
1959 noway_assert(((RBM_ILLEGAL & RBM_ALLINT) == 0));
1960 noway_assert(RBM_ILLEGAL);
1961 noway_assert((lockedRegs & RBM_ILLEGAL) == 0);
1962 /* impossible values, to make sure that we set them */
1963 tree->gtUsedRegs = RBM_ILLEGAL;
1966 /* Figure out what kind of a node we have */
1968 genTreeOps oper = tree->OperGet();
1969 var_types type = tree->TypeGet();
1970 unsigned kind = tree->OperKind();
1972 // In the comma case, we care about whether this is "effectively" ADDR(IND(...))
1973 genTreeOps effectiveOper = tree->gtEffectiveVal()->OperGet();
1974 if ((predictReg == PREDICT_ADDR) && (effectiveOper != GT_IND))
1975 predictReg = PREDICT_NONE;
1976 else if (rpHasVarIndexForPredict(predictReg))
1978 // The only place where predictReg is set to a var is in the PURE
1979 // assignment case where varIndex is the var being assigned to.
1980 // We need to check whether the variable is used between here and
1981 // its redefinition.
1982 unsigned varIndex = rpGetVarIndexForPredict(predictReg);
1983 unsigned lclNum = lvaTrackedToVarNum[varIndex];
1985 for (GenTreePtr nextTree = tree->gtNext; nextTree != NULL && !found; nextTree = nextTree->gtNext)
1987 if (nextTree->gtOper == GT_LCL_VAR && nextTree->gtLclVarCommon.gtLclNum == lclNum)
1989 // Is this the pure assignment?
1990 if ((nextTree->gtFlags & GTF_VAR_DEF) == 0)
1992 predictReg = PREDICT_SCRATCH_REG;
2001 if (rsvdRegs & RBM_LASTUSE)
2003 restoreLastUseVars = true;
2004 VarSetOps::Assign(this, oldLastUseVars, rpLastUseVars);
2005 rsvdRegs &= ~RBM_LASTUSE;
2008 /* Is this a constant or leaf node? */
2010 if (kind & (GTK_CONST | GTK_LEAF))
2012 bool lastUse = false;
2013 regMaskTP enregMask = RBM_NONE;
2019 // Codegen for floating point constants on the ARM is currently
2020 // movw/movt rT1, <lo32 bits>
2021 // movw/movt rT2, <hi32 bits>
2022 // vmov.i2d dT0, rT1,rT2
2024 // For TYP_FLOAT one integer register is required
2026 // These integer register(s) immediately die
2027 tmpMask = rpPredictRegPick(TYP_INT, PREDICT_REG, lockedRegs | rsvdRegs);
2028 if (type == TYP_DOUBLE)
2030 // For TYP_DOUBLE a second integer register is required
2032 tmpMask |= rpPredictRegPick(TYP_INT, PREDICT_REG, lockedRegs | rsvdRegs | tmpMask);
2035 // We also need a floating point register that we keep
2037 if (predictReg == PREDICT_NONE)
2038 predictReg = PREDICT_SCRATCH_REG;
2040 regMask = rpPredictRegPick(type, predictReg, lockedRegs | rsvdRegs);
2041 tree->gtUsedRegs = regMask | tmpMask;
2048 if (rpHasVarIndexForPredict(predictReg))
2050 unsigned tgtIndex = rpGetVarIndexForPredict(predictReg);
2051 rpAsgVarNum = tgtIndex;
2053 // We don't need any register as we plan on writing to the rpAsgVarNum register
2054 predictReg = PREDICT_NONE;
2056 LclVarDsc* tgtVar = lvaTable + lvaTrackedToVarNum[tgtIndex];
2057 tgtVar->lvDependReg = true;
2059 if (type == TYP_LONG)
2061 assert(oper == GT_CNS_LNG);
2063 if (tgtVar->lvOtherReg == REG_STK)
2065 // Well we do need one register for a partially enregistered
2067 predictReg = PREDICT_SCRATCH_REG;
2073 #if !CPU_LOAD_STORE_ARCH
2074 /* If the constant is a handle then it will need to have a relocation
2075 applied to it. It will need to be loaded into a register.
2076 But never throw away an existing hint.
2078 if (opts.compReloc && tree->IsCnsIntOrI() && tree->IsIconHandle())
2081 if (predictReg == PREDICT_NONE)
2082 predictReg = PREDICT_SCRATCH_REG;
2091 if ((predictReg == PREDICT_NONE) && (genActualType(type) == TYP_INT) &&
2092 (genTypeSize(type) < sizeof(int)))
2094 predictReg = PREDICT_SCRATCH_REG;
2097 // Unaligned loads/stores for floating point values must first be loaded into integer register(s)
2099 if ((tree->gtFlags & GTF_IND_UNALIGNED) && varTypeIsFloating(type))
2101 // These integer register(s) immediately die
2102 tmpMask = rpPredictRegPick(TYP_INT, PREDICT_REG, lockedRegs | rsvdRegs);
2103 // Two integer registers are required for a TYP_DOUBLE
2104 if (type == TYP_DOUBLE)
2105 tmpMask |= rpPredictRegPick(TYP_INT, PREDICT_REG, lockedRegs | rsvdRegs | tmpMask);
2107 // We need a temp register in some cases of loads/stores to a class var
2108 if (predictReg == PREDICT_NONE)
2110 predictReg = PREDICT_SCRATCH_REG;
2113 if (rpHasVarIndexForPredict(predictReg))
2115 unsigned tgtIndex = rpGetVarIndexForPredict(predictReg);
2116 rpAsgVarNum = tgtIndex;
2118 // We don't need any register as we plan on writing to the rpAsgVarNum register
2119 predictReg = PREDICT_NONE;
2121 LclVarDsc* tgtVar = lvaTable + lvaTrackedToVarNum[tgtIndex];
2122 tgtVar->lvDependReg = true;
2124 if (type == TYP_LONG)
2126 if (tgtVar->lvOtherReg == REG_STK)
2128 // Well we do need one register for a partially enregistered
2130 predictReg = PREDICT_SCRATCH_REG;
2138 // Check for a misalignment on a Floating Point field
2140 if (varTypeIsFloating(type))
2142 if ((tree->gtLclFld.gtLclOffs % emitTypeSize(tree->TypeGet())) != 0)
2144 // These integer register(s) immediately die
2145 tmpMask = rpPredictRegPick(TYP_INT, PREDICT_REG, lockedRegs | rsvdRegs);
2146 // Two integer registers are required for a TYP_DOUBLE
2147 if (type == TYP_DOUBLE)
2148 tmpMask |= rpPredictRegPick(TYP_INT, PREDICT_REG, lockedRegs | rsvdRegs | tmpMask);
2157 varDsc = lvaTable + tree->gtLclVarCommon.gtLclNum;
2159 VarSetOps::Assign(this, varBits, fgGetVarBits(tree));
2160 compUpdateLifeVar</*ForCodeGen*/ false>(tree, &lastUseVarBits);
2161 lastUse = !VarSetOps::IsEmpty(this, lastUseVarBits);
2163 #if FEATURE_STACK_FP_X87
2164 // If it's a floating point var, there's nothing to do
2165 if (varTypeIsFloating(type))
2167 tree->gtUsedRegs = RBM_NONE;
2173 // If the variable is already a register variable, no need to go further.
2174 if (oper == GT_REG_VAR)
2177 /* Apply the type of predictReg to the LCL_VAR */
2179 if (predictReg == PREDICT_REG)
2182 if (varDsc->lvRegNum == REG_STK)
2187 else if (predictReg == PREDICT_SCRATCH_REG)
2189 noway_assert(predictReg == PREDICT_SCRATCH_REG);
2191 /* Is this the last use of a local var? */
2194 if (VarSetOps::IsEmptyIntersection(this, rpUseInPlace, lastUseVarBits))
2195 goto PREDICT_REG_COMMON;
2198 else if (rpHasVarIndexForPredict(predictReg))
2200 /* Get the tracked local variable that has an lvVarIndex of tgtIndex1 */
2202 unsigned tgtIndex1 = rpGetVarIndexForPredict(predictReg);
2203 LclVarDsc* tgtVar = lvaTable + lvaTrackedToVarNum[tgtIndex1];
2204 VarSetOps::MakeSingleton(this, tgtIndex1);
2206 noway_assert(tgtVar->lvVarIndex == tgtIndex1);
2207 noway_assert(tgtVar->lvRegNum != REG_STK); /* Must have been enregistered */
2208 #ifndef _TARGET_AMD64_
2209 // On amd64 we have the occasional spec-allowed implicit conversion from TYP_I_IMPL to TYP_INT
2210 // so this assert is meaningless
2211 noway_assert((type != TYP_LONG) || (tgtVar->TypeGet() == TYP_LONG));
2212 #endif // !_TARGET_AMD64_
2214 if (varDsc->lvTracked)
2217 srcIndex = varDsc->lvVarIndex;
2219 // If this register has it's last use here then we will prefer
2220 // to color to the same register as tgtVar.
2224 * Add an entry in the lvaVarPref graph to indicate
2225 * that it would be worthwhile to color these two variables
2226 * into the same physical register.
2227 * This will help us avoid having an extra copy instruction
2229 VarSetOps::AddElemD(this, lvaVarPref[srcIndex], tgtIndex1);
2230 VarSetOps::AddElemD(this, lvaVarPref[tgtIndex1], srcIndex);
2233 // Add a variable interference from srcIndex to each of the last use variables
2234 if (!VarSetOps::IsEmpty(this, rpLastUseVars))
2236 rpRecordVarIntf(srcIndex, rpLastUseVars DEBUGARG("src reg conflict"));
2239 rpAsgVarNum = tgtIndex1;
2241 /* We will rely on the target enregistered variable from the GT_ASG */
2248 if (genIsValidFloatReg(varDsc->lvRegNum))
2250 enregMask = genRegMaskFloat(varDsc->lvRegNum, varDsc->TypeGet());
2254 enregMask = genRegMask(varDsc->lvRegNum);
2258 if ((type == TYP_DOUBLE) && (varDsc->TypeGet() == TYP_FLOAT))
2260 // We need to compute the intermediate value using a TYP_DOUBLE
2261 // but we storing the result in a TYP_SINGLE enregistered variable
2268 /* We can't trust a prediction of rsvdRegs or lockedRegs sets */
2269 if (enregMask & (rsvdRegs | lockedRegs))
2273 #ifndef _TARGET_64BIT_
2274 if (type == TYP_LONG)
2276 if (varDsc->lvOtherReg != REG_STK)
2278 tmpMask = genRegMask(varDsc->lvOtherReg);
2279 enregMask |= tmpMask;
2281 /* We can't trust a prediction of rsvdRegs or lockedRegs sets */
2282 if (tmpMask & (rsvdRegs | lockedRegs))
2285 else // lvOtherReg == REG_STK
2290 #endif // _TARGET_64BIT_
2293 varDsc->lvDependReg = true;
2297 /* Does not need a register */
2298 predictReg = PREDICT_NONE;
2299 // noway_assert(!VarSetOps::IsEmpty(this, varBits));
2300 VarSetOps::UnionD(this, rpUseInPlace, varBits);
2302 else // (grabCount > 0)
2304 #ifndef _TARGET_64BIT_
2305 /* For TYP_LONG and we only need one register then change the type to TYP_INT */
2306 if ((type == TYP_LONG) && (grabCount == 1))
2308 /* We will need to pick one register */
2310 // noway_assert(!VarSetOps::IsEmpty(this, varBits));
2311 VarSetOps::UnionD(this, rpUseInPlace, varBits);
2313 noway_assert((type == TYP_DOUBLE) ||
2314 (grabCount == (genTypeSize(genActualType(type)) / REGSIZE_BYTES)));
2315 #else // !_TARGET_64BIT_
2316 noway_assert(grabCount == 1);
2317 #endif // !_TARGET_64BIT_
2320 else if (type == TYP_STRUCT)
2323 // TODO-ARM-Bug?: Passing structs in registers on ARM hits an assert here when
2324 // predictReg is PREDICT_REG_R0 to PREDICT_REG_R3
2325 // As a workaround we just bash it to PREDICT_NONE here
2327 if (predictReg != PREDICT_NONE)
2328 predictReg = PREDICT_NONE;
2330 // Currently predictReg is saying that we will not need any scratch registers
2331 noway_assert(predictReg == PREDICT_NONE);
2333 /* We may need to sign or zero extend a small type when pushing a struct */
2334 if (varDsc->lvPromoted && !varDsc->lvAddrExposed)
2336 for (unsigned varNum = varDsc->lvFieldLclStart;
2337 varNum < varDsc->lvFieldLclStart + varDsc->lvFieldCnt; varNum++)
2339 LclVarDsc* fldVar = lvaTable + varNum;
2341 if (fldVar->lvStackAligned())
2343 // When we are stack aligned Codegen will just use
2344 // a push instruction and thus doesn't need any register
2345 // since we can push both a register or a stack frame location
2349 if (varTypeIsByte(fldVar->TypeGet()))
2351 // We will need to reserve one byteable register,
2354 predictReg = PREDICT_SCRATCH_REG;
2355 #if CPU_HAS_BYTE_REGS
2356 // It is best to enregister this fldVar in a byteable register
2358 fldVar->addPrefReg(RBM_BYTE_REG_FLAG, this);
2361 else if (varTypeIsShort(fldVar->TypeGet()))
2363 bool isEnregistered = fldVar->lvTracked && (fldVar->lvRegNum != REG_STK);
2364 // If fldVar is not enregistered then we will need a scratch register
2366 if (!isEnregistered)
2368 // We will need either an int register or a byte register
2369 // If we are not requesting a byte register we will request an int register
2371 if (type != TYP_BYTE)
2373 predictReg = PREDICT_SCRATCH_REG;
2381 regMaskTP preferReg = rpPredictRegMask(predictReg, type);
2384 if ((genTypeStSz(type) == 1) || (genCountBits(preferReg) <= genTypeStSz(type)))
2386 varDsc->addPrefReg(preferReg, this);
2390 break; /* end of case GT_LCL_VAR */
2393 tree->gtUsedRegs = RBM_NONE;
2396 #if defined(_TARGET_ARM_) && defined(PROFILING_SUPPORTED)
2397 // Mark the registers required to emit a tailcall profiler callback
2398 if (compIsProfilerHookNeeded())
2400 tree->gtUsedRegs |= RBM_PROFILER_JMP_USED;
2407 } /* end of switch (oper) */
2409 /* If we don't need to evaluate to register, regmask is the empty set */
2410 /* Otherwise we grab a temp for the local variable */
2412 if (predictReg == PREDICT_NONE)
2416 regMask = rpPredictRegPick(type, predictReg, lockedRegs | rsvdRegs | enregMask);
2418 if ((oper == GT_LCL_VAR) && (tree->TypeGet() == TYP_STRUCT))
2420 /* We need to sign or zero extend a small type when pushing a struct */
2421 noway_assert((type == TYP_INT) || (type == TYP_BYTE));
2423 varDsc = lvaTable + tree->gtLclVarCommon.gtLclNum;
2424 noway_assert(varDsc->lvPromoted && !varDsc->lvAddrExposed);
2426 for (unsigned varNum = varDsc->lvFieldLclStart; varNum < varDsc->lvFieldLclStart + varDsc->lvFieldCnt;
2429 LclVarDsc* fldVar = lvaTable + varNum;
2430 if (fldVar->lvTracked)
2432 VARSET_TP fldBit(VarSetOps::MakeSingleton(this, fldVar->lvVarIndex));
2433 rpRecordRegIntf(regMask, fldBit DEBUGARG(
2434 "need scratch register when pushing a small field of a struct"));
2440 /* Update the set of lastUse variables that we encountered so far */
2443 VarSetOps::UnionD(this, rpLastUseVars, lastUseVarBits);
2444 VARSET_TP varAsSet(VarSetOps::MakeCopy(this, lastUseVarBits));
2447 * Add interference from any previously locked temps into this last use variable.
2451 rpRecordRegIntf(lockedRegs, varAsSet DEBUGARG("last use Predict lockedRegs"));
2454 * Add interference from any reserved temps into this last use variable.
2458 rpRecordRegIntf(rsvdRegs, varAsSet DEBUGARG("last use Predict rsvdRegs"));
2461 * For partially enregistered longs add an interference with the
2462 * register return by rpPredictRegPick
2464 if ((type == TYP_INT) && (tree->TypeGet() == TYP_LONG))
2466 rpRecordRegIntf(regMask, varAsSet DEBUGARG("last use with partial enreg"));
2470 tree->gtUsedRegs = (regMaskSmall)regMask;
2474 /* Is it a 'simple' unary/binary operator? */
2476 if (kind & GTK_SMPOP)
2478 GenTreePtr op1 = tree->gtOp.gtOp1;
2479 GenTreePtr op2 = tree->gtGetOp2IfPresent();
2481 GenTreePtr opsPtr[3];
2482 regMaskTP regsPtr[3];
2484 VARSET_TP startAsgUseInPlaceVars(VarSetOps::UninitVal());
2490 /* Is the value being assigned into a LCL_VAR? */
2491 if (op1->gtOper == GT_LCL_VAR)
2493 varDsc = lvaTable + op1->gtLclVarCommon.gtLclNum;
2495 /* Are we assigning a LCL_VAR the result of a call? */
2496 if (op2->gtOper == GT_CALL)
2498 /* Set a preferred register for the LCL_VAR */
2499 if (isRegPairType(varDsc->TypeGet()))
2500 varDsc->addPrefReg(RBM_LNGRET, this);
2501 else if (!varTypeIsFloating(varDsc->TypeGet()))
2502 varDsc->addPrefReg(RBM_INTRET, this);
2503 #ifdef _TARGET_AMD64_
2505 varDsc->addPrefReg(RBM_FLOATRET, this);
2508 * When assigning the result of a call we don't
2509 * bother trying to target the right side of the
2510 * assignment, since we have a fixed calling convention.
2513 else if (varDsc->lvTracked)
2515 // We interfere with uses in place
2516 if (!VarSetOps::IsEmpty(this, rpUseInPlace))
2518 rpRecordVarIntf(varDsc->lvVarIndex, rpUseInPlace DEBUGARG("Assign UseInPlace conflict"));
2521 // Did we predict that this local will be fully enregistered?
2522 // and the assignment type is the same as the expression type?
2523 // and it is dead on the right side of the assignment?
2524 // and we current have no other rpAsgVarNum active?
2526 if ((varDsc->lvRegNum != REG_STK) && ((type != TYP_LONG) || (varDsc->lvOtherReg != REG_STK)) &&
2527 (type == op2->TypeGet()) && (op1->gtFlags & GTF_VAR_DEF) && (rpAsgVarNum == -1))
2530 // Yes, we should try to target the right side (op2) of this
2531 // assignment into the (enregistered) tracked variable.
2534 op1PredictReg = PREDICT_NONE; /* really PREDICT_REG, but we've already done the check */
2535 op2PredictReg = rpGetPredictForVarIndex(varDsc->lvVarIndex);
2537 // Remember that this is a new use in place
2539 // We've added "new UseInPlace"; remove from the global set.
2540 VarSetOps::RemoveElemD(this, rpUseInPlace, varDsc->lvVarIndex);
2542 // Note that later when we walk down to the leaf node for op2
2543 // if we decide to actually use the register for the 'varDsc'
2544 // to enregister the operand, the we will set rpAsgVarNum to
2545 // varDsc->lvVarIndex, by extracting this value using
2546 // rpGetVarIndexForPredict()
2548 // Also we reset rpAsgVarNum back to -1 after we have finished
2549 // predicting the current GT_ASG node
2555 else if (tree->OperIsBlkOp())
2557 interferingRegs |= rpPredictBlkAsgRegUse(tree, predictReg, lockedRegs, rsvdRegs);
2574 /* We can't use "reg <op>= addr" for TYP_LONG or if op2 is a short type */
2575 if ((type != TYP_LONG) && !varTypeIsSmall(op2->gtType))
2577 /* Is the value being assigned into an enregistered LCL_VAR? */
2578 /* For debug code we only allow a simple op2 to be assigned */
2579 if ((op1->gtOper == GT_LCL_VAR) && (!opts.compDbgCode || rpCanAsgOperWithoutReg(op2, false)))
2581 varDsc = lvaTable + op1->gtLclVarCommon.gtLclNum;
2582 /* Did we predict that this local will be enregistered? */
2583 if (varDsc->lvRegNum != REG_STK)
2585 /* Yes, we can use "reg <op>= addr" */
2587 op1PredictReg = PREDICT_NONE; /* really PREDICT_REG, but we've already done the check */
2588 op2PredictReg = PREDICT_NONE;
2595 #if CPU_LOAD_STORE_ARCH
2598 op1PredictReg = PREDICT_REG;
2599 op2PredictReg = PREDICT_REG;
2605 * Otherwise, initialize the normal forcing of operands:
2608 op1PredictReg = PREDICT_ADDR;
2609 op2PredictReg = PREDICT_REG;
2614 #if !CPU_LOAD_STORE_ARCH
2615 if (op2PredictReg != PREDICT_NONE)
2617 /* Is the value being assigned a simple one? */
2618 if (rpCanAsgOperWithoutReg(op2, false))
2619 op2PredictReg = PREDICT_NONE;
2623 bool simpleAssignment;
2624 simpleAssignment = false;
2626 if ((oper == GT_ASG) && (op1->gtOper == GT_LCL_VAR))
2628 // Add a variable interference from the assign target
2629 // to each of the last use variables
2630 if (!VarSetOps::IsEmpty(this, rpLastUseVars))
2632 varDsc = lvaTable + op1->gtLclVarCommon.gtLclNum;
2634 if (varDsc->lvTracked)
2636 unsigned varIndex = varDsc->lvVarIndex;
2638 rpRecordVarIntf(varIndex, rpLastUseVars DEBUGARG("Assign conflict"));
2642 /* Record whether this tree is a simple assignment to a local */
2644 simpleAssignment = ((type != TYP_LONG) || !opts.compDbgCode);
2647 bool requireByteReg;
2648 requireByteReg = false;
2650 #if CPU_HAS_BYTE_REGS
2651 /* Byte-assignments need the byte registers, unless op1 is an enregistered local */
2653 if (varTypeIsByte(type) &&
2654 ((op1->gtOper != GT_LCL_VAR) || (lvaTable[op1->gtLclVarCommon.gtLclNum].lvRegNum == REG_STK)))
2657 // Byte-assignments typically need a byte register
2658 requireByteReg = true;
2660 if (op1->gtOper == GT_LCL_VAR)
2662 varDsc = lvaTable + op1->gtLclVar.gtLclNum;
2664 // Did we predict that this local will be enregistered?
2665 if (varDsc->lvTracked && (varDsc->lvRegNum != REG_STK) && (oper != GT_CHS))
2667 // We don't require a byte register when op1 is an enregistered local */
2668 requireByteReg = false;
2671 // Is op1 part of an Assign-Op or is the RHS a simple memory indirection?
2672 if ((oper != GT_ASG) || (op2->gtOper == GT_IND) || (op2->gtOper == GT_CLS_VAR))
2674 // We should try to put op1 in an byte register
2675 varDsc->addPrefReg(RBM_BYTE_REG_FLAG, this);
2681 VarSetOps::Assign(this, startAsgUseInPlaceVars, rpUseInPlace);
2683 bool isWriteBarrierAsgNode;
2684 isWriteBarrierAsgNode = codeGen->gcInfo.gcIsWriteBarrierAsgNode(tree);
2686 GCInfo::WriteBarrierForm wbf;
2687 if (isWriteBarrierAsgNode)
2688 wbf = codeGen->gcInfo.gcIsWriteBarrierCandidate(tree->gtOp.gtOp1, tree->gtOp.gtOp2);
2690 wbf = GCInfo::WBF_NoBarrier;
2693 regMaskTP wbaLockedRegs;
2694 wbaLockedRegs = lockedRegs;
2695 if (isWriteBarrierAsgNode)
2697 #if defined(_TARGET_X86_) && NOGC_WRITE_BARRIERS
2699 if (wbf != GCInfo::WBF_NoBarrier_CheckNotHeapInDebug)
2702 wbaLockedRegs |= RBM_WRITE_BARRIER;
2703 op1->gtRsvdRegs |= RBM_WRITE_BARRIER; // This will steer op2 away from REG_WRITE_BARRIER
2704 assert(REG_WRITE_BARRIER == REG_EDX);
2705 op1PredictReg = PREDICT_REG_EDX;
2710 #endif // defined(_TARGET_X86_) && NOGC_WRITE_BARRIERS
2712 #if defined(DEBUG) || !(defined(_TARGET_X86_) && NOGC_WRITE_BARRIERS)
2715 op1PredictReg = PREDICT_REG_ECX;
2716 op2PredictReg = PREDICT_REG_EDX;
2717 #elif defined(_TARGET_ARM_)
2718 op1PredictReg = PREDICT_REG_R0;
2719 op2PredictReg = PREDICT_REG_R1;
2721 // This is my best guess as to what the previous code meant by checking "gtRngChkLen() == NULL".
2722 if ((op1->OperGet() == GT_IND) && (op1->gtOp.gtOp1->OperGet() != GT_ARR_BOUNDS_CHECK))
2724 op1 = op1->gtOp.gtOp1;
2726 #else // !_TARGET_X86_ && !_TARGET_ARM_
2727 #error "Non-ARM or x86 _TARGET_ in RegPredict for WriteBarrierAsg"
2733 /* Are we supposed to evaluate RHS first? */
2735 if (tree->gtFlags & GTF_REVERSE_OPS)
2737 op2Mask = rpPredictTreeRegUse(op2, op2PredictReg, lockedRegs, rsvdRegs | op1->gtRsvdRegs);
2739 #if CPU_HAS_BYTE_REGS
2740 // Should we insure that op2 gets evaluated into a byte register?
2741 if (requireByteReg && ((op2Mask & RBM_BYTE_REGS) == 0))
2743 // We need to grab a byte-able register, (i.e. EAX, EDX, ECX, EBX)
2744 // and we can't select one that is already reserved (i.e. lockedRegs)
2746 op2Mask |= rpPredictRegPick(type, PREDICT_SCRATCH_REG, (lockedRegs | RBM_NON_BYTE_REGS));
2747 op2->gtUsedRegs |= op2Mask;
2749 // No longer a simple assignment because we're using extra registers and might
2750 // have interference between op1 and op2. See DevDiv #136681
2751 simpleAssignment = false;
2755 * For a simple assignment we don't want the op2Mask to be
2756 * marked as interferring with the LCL_VAR, since it is likely
2757 * that we will want to enregister the LCL_VAR in exactly
2758 * the register that is used to compute op2
2760 tmpMask = lockedRegs;
2762 if (!simpleAssignment)
2765 regMask = rpPredictTreeRegUse(op1, op1PredictReg, tmpMask, RBM_NONE);
2767 // Did we relax the register prediction for op1 and op2 above ?
2768 // - because we are depending upon op1 being enregistered
2770 if ((op1PredictReg == PREDICT_NONE) &&
2771 ((op2PredictReg == PREDICT_NONE) || rpHasVarIndexForPredict(op2PredictReg)))
2773 /* We must be assigning into an enregistered LCL_VAR */
2774 noway_assert(op1->gtOper == GT_LCL_VAR);
2775 varDsc = lvaTable + op1->gtLclVar.gtLclNum;
2776 noway_assert(varDsc->lvRegNum != REG_STK);
2778 /* We need to set lvDependReg, in case we lose the enregistration of op1 */
2779 varDsc->lvDependReg = true;
2784 // For the case of simpleAssignments op2 should always be evaluated first
2785 noway_assert(!simpleAssignment);
2787 regMask = rpPredictTreeRegUse(op1, op1PredictReg, lockedRegs, rsvdRegs | op2->gtRsvdRegs);
2788 if (isWriteBarrierAsgNode)
2790 wbaLockedRegs |= op1->gtUsedRegs;
2792 op2Mask = rpPredictTreeRegUse(op2, op2PredictReg, wbaLockedRegs | regMask, RBM_NONE);
2794 #if CPU_HAS_BYTE_REGS
2795 // Should we insure that op2 gets evaluated into a byte register?
2796 if (requireByteReg && ((op2Mask & RBM_BYTE_REGS) == 0))
2798 // We need to grab a byte-able register, (i.e. EAX, EDX, ECX, EBX)
2799 // and we can't select one that is already reserved (i.e. lockedRegs or regMask)
2802 rpPredictRegPick(type, PREDICT_SCRATCH_REG, (lockedRegs | regMask | RBM_NON_BYTE_REGS));
2803 op2->gtUsedRegs |= op2Mask;
2808 if (rpHasVarIndexForPredict(op2PredictReg))
2813 if (isWriteBarrierAsgNode)
2815 #if NOGC_WRITE_BARRIERS
2817 if (wbf != GCInfo::WBF_NoBarrier_CheckNotHeapInDebug)
2821 /* Steer computation away from REG_WRITE_BARRIER as the pointer is
2822 passed to the write-barrier call in REG_WRITE_BARRIER */
2826 if (op1->gtOper == GT_IND)
2828 GenTreePtr rv1, rv2;
2832 /* Special handling of indirect assigns for write barrier */
2834 bool yes = codeGen->genCreateAddrMode(op1->gtOp.gtOp1, -1, true, RBM_NONE, &rev, &rv1, &rv2,
2837 /* Check address mode for enregisterable locals */
2841 if (rv1 != NULL && rv1->gtOper == GT_LCL_VAR)
2843 rpPredictRefAssign(rv1->gtLclVarCommon.gtLclNum);
2845 if (rv2 != NULL && rv2->gtOper == GT_LCL_VAR)
2847 rpPredictRefAssign(rv2->gtLclVarCommon.gtLclNum);
2852 if (op2->gtOper == GT_LCL_VAR)
2854 rpPredictRefAssign(op2->gtLclVarCommon.gtLclNum);
2857 // Add a register interference for REG_WRITE_BARRIER to each of the last use variables
2858 if (!VarSetOps::IsEmpty(this, rpLastUseVars))
2860 rpRecordRegIntf(RBM_WRITE_BARRIER,
2861 rpLastUseVars DEBUGARG("WriteBarrier and rpLastUseVars conflict"));
2863 tree->gtUsedRegs |= RBM_WRITE_BARRIER;
2868 #endif // NOGC_WRITE_BARRIERS
2870 #if defined(DEBUG) || !NOGC_WRITE_BARRIERS
2875 printf("Adding interference with RBM_CALLEE_TRASH_NOGC for NoGC WriteBarrierAsg\n");
2878 // For the ARM target we have an optimized JIT Helper
2879 // that only trashes a subset of the callee saved registers
2882 // NOTE: Adding it to the gtUsedRegs will cause the interference to
2883 // be added appropriately
2885 // the RBM_CALLEE_TRASH_NOGC set is killed. We will record this in interferingRegs
2886 // instead of gtUsedRegs, because the latter will be modified later, but we need
2887 // to remember to add the interference.
2889 interferingRegs |= RBM_CALLEE_TRASH_NOGC;
2891 op1->gtUsedRegs |= RBM_R0;
2892 op2->gtUsedRegs |= RBM_R1;
2893 #else // _TARGET_ARM_
2897 printf("Adding interference with RBM_CALLEE_TRASH for NoGC WriteBarrierAsg\n");
2899 // We have to call a normal JIT helper to perform the Write Barrier Assignment
2900 // It will trash the callee saved registers
2902 tree->gtUsedRegs |= RBM_CALLEE_TRASH;
2903 #endif // _TARGET_ARM_
2905 #endif // defined(DEBUG) || !NOGC_WRITE_BARRIERS
2908 if (simpleAssignment)
2911 * Consider a simple assignment to a local:
2915 * Since the "=" node is visited after the variable
2916 * is marked live (assuming it's live after the
2917 * assignment), we don't want to use the register
2918 * use mask of the "=" node but rather that of the
2921 tree->gtUsedRegs = op1->gtUsedRegs;
2925 tree->gtUsedRegs = op1->gtUsedRegs | op2->gtUsedRegs;
2927 VarSetOps::Assign(this, rpUseInPlace, startAsgUseInPlaceVars);
2933 /* assigning shift operators */
2935 noway_assert(type != TYP_LONG);
2937 #if CPU_LOAD_STORE_ARCH
2938 predictReg = PREDICT_ADDR;
2940 predictReg = PREDICT_NONE;
2943 /* shift count is handled same as ordinary shift */
2944 goto HANDLE_SHIFT_COUNT;
2947 regMask = rpPredictTreeRegUse(op1, PREDICT_ADDR, lockedRegs, RBM_LASTUSE);
2949 if ((regMask == RBM_NONE) && (predictReg >= PREDICT_REG))
2951 // We need a scratch register for the LEA instruction
2952 regMask = rpPredictRegPick(TYP_INT, predictReg, lockedRegs | rsvdRegs);
2955 tree->gtUsedRegs = op1->gtUsedRegs | (regMaskSmall)regMask;
2960 /* Cannot cast to VOID */
2961 noway_assert(type != TYP_VOID);
2963 /* cast to long is special */
2964 if (type == TYP_LONG && op1->gtType <= TYP_INT)
2966 noway_assert(tree->gtCast.gtCastType == TYP_LONG || tree->gtCast.gtCastType == TYP_ULONG);
2967 #if CPU_LONG_USES_REGPAIR
2968 rpPredictReg predictRegHi = PREDICT_SCRATCH_REG;
2970 if (rpHasVarIndexForPredict(predictReg))
2972 unsigned tgtIndex = rpGetVarIndexForPredict(predictReg);
2973 rpAsgVarNum = tgtIndex;
2975 // We don't need any register as we plan on writing to the rpAsgVarNum register
2976 predictReg = PREDICT_NONE;
2978 LclVarDsc* tgtVar = lvaTable + lvaTrackedToVarNum[tgtIndex];
2979 tgtVar->lvDependReg = true;
2981 if (tgtVar->lvOtherReg != REG_STK)
2983 predictRegHi = PREDICT_NONE;
2988 if (predictReg == PREDICT_NONE)
2990 predictReg = PREDICT_SCRATCH_REG;
2993 // If we are widening an int into a long using a targeted register pair we
2994 // should retarget so that the low part get loaded into the appropriate register
2995 else if (predictReg == PREDICT_PAIR_R0R1)
2997 predictReg = PREDICT_REG_R0;
2998 predictRegHi = PREDICT_REG_R1;
3000 else if (predictReg == PREDICT_PAIR_R2R3)
3002 predictReg = PREDICT_REG_R2;
3003 predictRegHi = PREDICT_REG_R3;
3007 // If we are widening an int into a long using a targeted register pair we
3008 // should retarget so that the low part get loaded into the appropriate register
3009 else if (predictReg == PREDICT_PAIR_EAXEDX)
3011 predictReg = PREDICT_REG_EAX;
3012 predictRegHi = PREDICT_REG_EDX;
3014 else if (predictReg == PREDICT_PAIR_ECXEBX)
3016 predictReg = PREDICT_REG_ECX;
3017 predictRegHi = PREDICT_REG_EBX;
3021 regMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs);
3023 #if CPU_LONG_USES_REGPAIR
3024 if (predictRegHi != PREDICT_NONE)
3026 // Now get one more reg for the upper part
3027 regMask |= rpPredictRegPick(TYP_INT, predictRegHi, lockedRegs | rsvdRegs | regMask);
3030 tree->gtUsedRegs = op1->gtUsedRegs | (regMaskSmall)regMask;
3034 /* cast from long is special - it frees a register */
3035 if (type <= TYP_INT // nice. this presumably is intended to mean "signed int and shorter types"
3036 && op1->gtType == TYP_LONG)
3038 if ((predictReg == PREDICT_NONE) || rpHasVarIndexForPredict(predictReg))
3039 predictReg = PREDICT_REG;
3041 regMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs);
3043 // If we have 2 or more regs, free one of them
3044 if (!genMaxOneBit(regMask))
3046 /* Clear the 2nd lowest bit in regMask */
3047 /* First set tmpMask to the lowest bit in regMask */
3048 tmpMask = genFindLowestBit(regMask);
3049 /* Next find the second lowest bit in regMask */
3050 tmpMask = genFindLowestBit(regMask & ~tmpMask);
3051 /* Clear this bit from regmask */
3052 regMask &= ~tmpMask;
3054 tree->gtUsedRegs = op1->gtUsedRegs;
3058 #if CPU_HAS_BYTE_REGS
3059 /* cast from signed-byte is special - it uses byteable registers */
3060 if (type == TYP_INT)
3062 var_types smallType;
3064 if (genTypeSize(tree->gtCast.CastOp()->TypeGet()) < genTypeSize(tree->gtCast.gtCastType))
3065 smallType = tree->gtCast.CastOp()->TypeGet();
3067 smallType = tree->gtCast.gtCastType;
3069 if (smallType == TYP_BYTE)
3071 regMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs);
3073 if ((regMask & RBM_BYTE_REGS) == 0)
3074 regMask = rpPredictRegPick(type, PREDICT_SCRATCH_REG, RBM_NON_BYTE_REGS);
3076 tree->gtUsedRegs = (regMaskSmall)regMask;
3082 #if FEATURE_STACK_FP_X87
3083 /* cast to float/double is special */
3084 if (varTypeIsFloating(type))
3086 switch (op1->TypeGet())
3088 /* uses fild, so don't need to be loaded to reg */
3091 rpPredictTreeRegUse(op1, PREDICT_NONE, lockedRegs, rsvdRegs);
3092 tree->gtUsedRegs = op1->gtUsedRegs;
3100 /* Casting from integral type to floating type is special */
3101 if (!varTypeIsFloating(type) && varTypeIsFloating(op1->TypeGet()))
3103 if (opts.compCanUseSSE2)
3105 // predict for SSE2 based casting
3106 if (predictReg <= PREDICT_REG)
3107 predictReg = PREDICT_SCRATCH_REG;
3108 regMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs);
3110 // Get one more int reg to hold cast result
3111 regMask |= rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | rsvdRegs | regMask);
3112 tree->gtUsedRegs = op1->gtUsedRegs | (regMaskSmall)regMask;
3118 #if FEATURE_FP_REGALLOC
3119 // Are we casting between int to float or float to int
3120 // Fix 388428 ARM JitStress WP7
3121 if (varTypeIsFloating(type) != varTypeIsFloating(op1->TypeGet()))
3123 // op1 needs to go into a register
3124 regMask = rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs, rsvdRegs);
3127 if (varTypeIsFloating(op1->TypeGet()))
3129 // We also need a fp scratch register for the convert operation
3130 regMask |= rpPredictRegPick((genTypeStSz(type) == 1) ? TYP_FLOAT : TYP_DOUBLE,
3131 PREDICT_SCRATCH_REG, regMask | lockedRegs | rsvdRegs);
3134 // We also need a register to hold the result
3135 regMask |= rpPredictRegPick(type, PREDICT_SCRATCH_REG, regMask | lockedRegs | rsvdRegs);
3136 tree->gtUsedRegs = op1->gtUsedRegs | (regMaskSmall)regMask;
3141 /* otherwise must load op1 into a register */
3146 #ifdef _TARGET_XARCH_
3147 if (tree->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Round && tree->TypeGet() == TYP_INT)
3149 // This is a special case to handle the following
3150 // optimization: conv.i4(round.d(d)) -> round.i(d)
3151 // if flowgraph 3186
3153 if (predictReg <= PREDICT_REG)
3154 predictReg = PREDICT_SCRATCH_REG;
3156 rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs);
3158 regMask = rpPredictRegPick(TYP_INT, predictReg, lockedRegs | rsvdRegs);
3160 tree->gtUsedRegs = op1->gtUsedRegs | (regMaskSmall)regMask;
3168 if (tree->TypeGet() == TYP_LONG)
3170 // On ARM this consumes an extra register for the '0' value
3171 if (predictReg <= PREDICT_REG)
3172 predictReg = PREDICT_SCRATCH_REG;
3174 regMaskTP op1Mask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs);
3176 regMask = rpPredictRegPick(TYP_INT, predictReg, lockedRegs | op1Mask | rsvdRegs);
3178 tree->gtUsedRegs = op1->gtUsedRegs | (regMaskSmall)regMask;
3181 #endif // _TARGET_ARM_
3186 // these unary operators will write new values
3187 // and thus will need a scratch register
3189 /* generic unary operators */
3191 if (predictReg <= PREDICT_REG)
3192 predictReg = PREDICT_SCRATCH_REG;
3197 // these unary operators do not write new values
3198 // and thus won't need a scratch register
3199 CLANG_FORMAT_COMMENT_ANCHOR;
3204 tree->gtUsedRegs = 0;
3209 regMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs);
3210 tree->gtUsedRegs = op1->gtUsedRegs;
3214 case GT_NULLCHECK: // At this point, nullcheck is just like an IND...
3216 bool intoReg = true;
3217 VARSET_TP startIndUseInPlaceVars(VarSetOps::MakeCopy(this, rpUseInPlace));
3219 if (fgIsIndirOfAddrOfLocal(tree) != NULL)
3221 compUpdateLifeVar</*ForCodeGen*/ false>(tree);
3224 if (predictReg == PREDICT_ADDR)
3228 else if (predictReg == PREDICT_NONE)
3230 if (type != TYP_LONG)
3236 predictReg = PREDICT_REG;
3240 /* forcing to register? */
3241 if (intoReg && (type != TYP_LONG))
3243 rsvdRegs |= RBM_LASTUSE;
3249 /* check for address mode */
3250 regMask = rpPredictAddressMode(op1, type, lockedRegs, rsvdRegs, lenCSE);
3253 #if CPU_LOAD_STORE_ARCH
3254 // We may need a scratch register for loading a long
3255 if (type == TYP_LONG)
3257 /* This scratch register immediately dies */
3258 tmpMask = rpPredictRegPick(TYP_BYREF, PREDICT_REG, op1->gtUsedRegs | lockedRegs | rsvdRegs);
3260 #endif // CPU_LOAD_STORE_ARCH
3263 // Unaligned loads/stores for floating point values must first be loaded into integer register(s)
3265 if ((tree->gtFlags & GTF_IND_UNALIGNED) && varTypeIsFloating(type))
3267 /* These integer register(s) immediately die */
3268 tmpMask = rpPredictRegPick(TYP_INT, PREDICT_REG, op1->gtUsedRegs | lockedRegs | rsvdRegs);
3269 // Two integer registers are required for a TYP_DOUBLE
3270 if (type == TYP_DOUBLE)
3272 rpPredictRegPick(TYP_INT, PREDICT_REG, op1->gtUsedRegs | lockedRegs | rsvdRegs | tmpMask);
3276 /* forcing to register? */
3279 regMaskTP lockedMask = lockedRegs | rsvdRegs;
3282 // We will compute a new regMask that holds the register(s)
3283 // that we will load the indirection into.
3285 CLANG_FORMAT_COMMENT_ANCHOR;
3287 #ifndef _TARGET_64BIT_
3288 if (type == TYP_LONG)
3290 // We need to use multiple load instructions here:
3291 // For the first register we can not choose
3292 // any registers that are being used in place or
3293 // any register in the current regMask
3295 regMask = rpPredictRegPick(TYP_INT, predictReg, regMask | lockedMask);
3297 // For the second register we can choose a register that was
3298 // used in place or any register in the old now overwritten regMask
3299 // but not the same register that we picked above in 'regMask'
3301 VarSetOps::Assign(this, rpUseInPlace, startIndUseInPlaceVars);
3302 regMask |= rpPredictRegPick(TYP_INT, predictReg, regMask | lockedMask);
3307 // We will use one load instruction here:
3308 // The load target register can be a register that was used in place
3309 // or one of the register from the orginal regMask.
3311 VarSetOps::Assign(this, rpUseInPlace, startIndUseInPlaceVars);
3312 regMask = rpPredictRegPick(type, predictReg, lockedMask);
3315 else if (predictReg != PREDICT_ADDR)
3317 /* Unless the caller specified PREDICT_ADDR */
3318 /* we don't return the temp registers used */
3319 /* to form the address */
3324 tree->gtUsedRegs = (regMaskSmall)(regMask | tmpMask);
3336 /* Floating point comparison uses EAX for flags */
3337 if (varTypeIsFloating(op1->TypeGet()))
3343 if (!(tree->gtFlags & GTF_RELOP_JMP_USED))
3345 // Some comparisons are converted to ?:
3346 noway_assert(!fgMorphRelopToQmark(op1));
3348 if (predictReg <= PREDICT_REG)
3349 predictReg = PREDICT_SCRATCH_REG;
3351 // The set instructions need a byte register
3352 regMask = rpPredictRegPick(TYP_BYTE, predictReg, lockedRegs | rsvdRegs);
3357 #ifdef _TARGET_XARCH_
3359 // Optimize the compare with a constant cases for xarch
3360 if (op1->gtOper == GT_CNS_INT)
3362 if (op2->gtOper == GT_CNS_INT)
3364 rpPredictTreeRegUse(op1, PREDICT_SCRATCH_REG, lockedRegs, rsvdRegs | op2->gtRsvdRegs);
3365 rpPredictTreeRegUse(op2, PREDICT_NONE, lockedRegs | tmpMask, RBM_LASTUSE);
3366 tree->gtUsedRegs = op2->gtUsedRegs;
3369 else if (op2->gtOper == GT_CNS_INT)
3371 rpPredictTreeRegUse(op1, PREDICT_NONE, lockedRegs, rsvdRegs);
3372 tree->gtUsedRegs = op1->gtUsedRegs;
3375 else if (op2->gtOper == GT_CNS_LNG)
3377 regMaskTP op1Mask = rpPredictTreeRegUse(op1, PREDICT_ADDR, lockedRegs, rsvdRegs);
3379 // We also need one extra register to read values from
3380 tmpMask = rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | op1Mask | rsvdRegs);
3381 #endif // _TARGET_X86_
3382 tree->gtUsedRegs = (regMaskSmall)tmpMask | op1->gtUsedRegs;
3385 #endif // _TARGET_XARCH_
3388 unsigned op1TypeSize;
3389 unsigned op2TypeSize;
3391 op1TypeSize = genTypeSize(op1->TypeGet());
3392 op2TypeSize = genTypeSize(op2->TypeGet());
3394 op1PredictReg = PREDICT_REG;
3395 op2PredictReg = PREDICT_REG;
3397 if (tree->gtFlags & GTF_REVERSE_OPS)
3399 #ifdef _TARGET_XARCH_
3400 if (op1TypeSize == sizeof(int))
3401 op1PredictReg = PREDICT_NONE;
3404 tmpMask = rpPredictTreeRegUse(op2, op2PredictReg, lockedRegs, rsvdRegs | op1->gtRsvdRegs);
3405 rpPredictTreeRegUse(op1, op1PredictReg, lockedRegs | tmpMask, RBM_LASTUSE);
3409 #ifdef _TARGET_XARCH_
3410 // For full DWORD compares we can have
3412 // op1 is an address mode and op2 is a register
3414 // op1 is a register and op2 is an address mode
3416 if ((op2TypeSize == sizeof(int)) && (op1TypeSize == op2TypeSize))
3418 if (op2->gtOper == GT_LCL_VAR)
3420 unsigned lclNum = op2->gtLclVar.gtLclNum;
3421 varDsc = lvaTable + lclNum;
3422 /* Did we predict that this local will be enregistered? */
3423 if (varDsc->lvTracked && (varDsc->lvRegNum != REG_STK))
3425 op1PredictReg = PREDICT_ADDR;
3429 // Codegen will generate cmp reg,[mem] for 4 or 8-byte types, but not for 1 or 2 byte types
3430 if ((op1PredictReg != PREDICT_ADDR) && (op2TypeSize >= sizeof(int)))
3431 op2PredictReg = PREDICT_ADDR;
3432 #endif // _TARGET_XARCH_
3434 tmpMask = rpPredictTreeRegUse(op1, op1PredictReg, lockedRegs, rsvdRegs | op2->gtRsvdRegs);
3436 if ((op2->gtOper != GT_CNS_INT) || !codeGen->validImmForAlu(op2->gtIntCon.gtIconVal))
3439 rpPredictTreeRegUse(op2, op2PredictReg, lockedRegs | tmpMask, RBM_LASTUSE);
3443 #ifdef _TARGET_XARCH_
3444 // In some cases in genCondSetFlags(), we need to use a temporary register (via rsPickReg())
3445 // to generate a sign/zero extension before doing a compare. Save a register for this purpose
3446 // if one of the registers is small and the types aren't equal.
3448 if (regMask == RBM_NONE)
3450 rpPredictReg op1xPredictReg, op2xPredictReg;
3451 GenTreePtr op1x, op2x;
3452 if (tree->gtFlags & GTF_REVERSE_OPS) // TODO: do we really need to handle this case?
3454 op1xPredictReg = op2PredictReg;
3455 op2xPredictReg = op1PredictReg;
3461 op1xPredictReg = op1PredictReg;
3462 op2xPredictReg = op2PredictReg;
3466 if ((op1xPredictReg < PREDICT_REG) && // op1 doesn't get a register (probably an indir)
3467 (op2xPredictReg >= PREDICT_REG) && // op2 gets a register
3468 varTypeIsSmall(op1x->TypeGet())) // op1 is smaller than an int
3470 bool needTmp = false;
3472 // If op1x is a byte, and op2x is not a byteable register, we'll need a temp.
3473 // We could predict a byteable register for op2x, but what if we don't get it?
3474 // So, be conservative and always ask for a temp. There are a couple small CQ losses as a
3476 if (varTypeIsByte(op1x->TypeGet()))
3482 if (op2x->gtOper == GT_LCL_VAR) // this will be a GT_REG_VAR during code generation
3484 if (genActualType(op1x->TypeGet()) != lvaGetActualType(op2x->gtLclVar.gtLclNum))
3489 if (op1x->TypeGet() != op2x->TypeGet())
3495 regMask = rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | rsvdRegs);
3499 #endif // _TARGET_XARCH_
3501 tree->gtUsedRegs = (regMaskSmall)regMask | op1->gtUsedRegs | op2->gtUsedRegs;
3506 #ifndef _TARGET_AMD64_
3507 if (type == TYP_LONG)
3509 assert(tree->gtIsValid64RsltMul());
3511 /* Strip out the cast nodes */
3513 noway_assert(op1->gtOper == GT_CAST && op2->gtOper == GT_CAST);
3514 op1 = op1->gtCast.CastOp();
3515 op2 = op2->gtCast.CastOp();
3519 #endif // !_TARGET_AMD64_
3522 #if defined(_TARGET_X86_)
3523 // This will done by a 64-bit imul "imul eax, reg"
3524 // (i.e. EDX:EAX = EAX * reg)
3526 /* Are we supposed to evaluate op2 first? */
3527 if (tree->gtFlags & GTF_REVERSE_OPS)
3529 rpPredictTreeRegUse(op2, PREDICT_PAIR_TMP_LO, lockedRegs, rsvdRegs | op1->gtRsvdRegs);
3530 rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs | RBM_PAIR_TMP_LO, RBM_LASTUSE);
3534 rpPredictTreeRegUse(op1, PREDICT_PAIR_TMP_LO, lockedRegs, rsvdRegs | op2->gtRsvdRegs);
3535 rpPredictTreeRegUse(op2, PREDICT_REG, lockedRegs | RBM_PAIR_TMP_LO, RBM_LASTUSE);
3538 /* set gtUsedRegs to EAX, EDX and the registers needed by op1 and op2 */
3540 tree->gtUsedRegs = RBM_PAIR_TMP | op1->gtUsedRegs | op2->gtUsedRegs;
3542 /* set regMask to the set of held registers */
3544 regMask = RBM_PAIR_TMP_LO;
3546 if (type == TYP_LONG)
3547 regMask |= RBM_PAIR_TMP_HI;
3549 #elif defined(_TARGET_ARM_)
3550 // This will done by a 4 operand multiply
3552 // Are we supposed to evaluate op2 first?
3553 if (tree->gtFlags & GTF_REVERSE_OPS)
3555 rpPredictTreeRegUse(op2, PREDICT_REG, lockedRegs, rsvdRegs | op1->gtRsvdRegs);
3556 rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs, RBM_LASTUSE);
3560 rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs, rsvdRegs | op2->gtRsvdRegs);
3561 rpPredictTreeRegUse(op2, PREDICT_REG, lockedRegs, RBM_LASTUSE);
3564 // set regMask to the set of held registers,
3565 // the two scratch register we need to compute the mul result
3567 regMask = rpPredictRegPick(TYP_LONG, PREDICT_SCRATCH_REG, lockedRegs | rsvdRegs);
3569 // set gtUsedRegs toregMask and the registers needed by op1 and op2
3571 tree->gtUsedRegs = regMask | op1->gtUsedRegs | op2->gtUsedRegs;
3573 #else // !_TARGET_X86_ && !_TARGET_ARM_
3574 #error "Non-ARM or x86 _TARGET_ in RegPredict for 64-bit imul"
3581 /* We use imulEAX for most unsigned multiply operations */
3582 if (tree->gtOverflow())
3584 if ((tree->gtFlags & GTF_UNSIGNED) || varTypeIsSmall(tree->TypeGet()))
3599 tree->gtUsedRegs = 0;
3601 if (predictReg <= PREDICT_REG)
3602 predictReg = PREDICT_SCRATCH_REG;
3607 if (tree->gtFlags & GTF_REVERSE_OPS)
3609 op1PredictReg = PREDICT_REG;
3610 #if !CPU_LOAD_STORE_ARCH
3611 if (genTypeSize(op1->gtType) >= sizeof(int))
3612 op1PredictReg = PREDICT_NONE;
3614 regMask = rpPredictTreeRegUse(op2, predictReg, lockedRegs, rsvdRegs | op1->gtRsvdRegs);
3615 rpPredictTreeRegUse(op1, op1PredictReg, lockedRegs | regMask, RBM_LASTUSE);
3619 op2PredictReg = PREDICT_REG;
3620 #if !CPU_LOAD_STORE_ARCH
3621 if (genTypeSize(op2->gtType) >= sizeof(int))
3622 op2PredictReg = PREDICT_NONE;
3624 regMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs | op2->gtRsvdRegs);
3626 // For most ALU operations we can generate a single instruction that encodes
3627 // a small immediate integer constant value. (except for multiply)
3629 if ((op2->gtOper == GT_CNS_INT) && (oper != GT_MUL))
3631 ssize_t ival = op2->gtIntCon.gtIconVal;
3632 if (codeGen->validImmForAlu(ival))
3634 op2PredictReg = PREDICT_NONE;
3636 else if (codeGen->validImmForAdd(ival, INS_FLAGS_DONT_CARE) &&
3637 ((oper == GT_ADD) || (oper == GT_SUB)))
3639 op2PredictReg = PREDICT_NONE;
3642 if (op2PredictReg == PREDICT_NONE)
3644 op2->gtUsedRegs = RBM_NONE;
3649 rpPredictTreeRegUse(op2, op2PredictReg, lockedRegs | regMask, RBM_LASTUSE);
3652 tree->gtUsedRegs = (regMaskSmall)regMask | op1->gtUsedRegs | op2->gtUsedRegs;
3654 #if CPU_HAS_BYTE_REGS
3655 /* We have special register requirements for byte operations */
3657 if (varTypeIsByte(tree->TypeGet()))
3659 /* For 8 bit arithmetic, one operands has to be in a
3660 byte-addressable register, and the other has to be
3661 in a byte-addrble reg or in memory. Assume its in a reg */
3663 regMaskTP regByteMask = 0;
3664 regMaskTP op1ByteMask = op1->gtUsedRegs;
3666 if (!(op1->gtUsedRegs & RBM_BYTE_REGS))
3668 // Pick a Byte register to use for op1
3669 regByteMask = rpPredictRegPick(TYP_BYTE, PREDICT_REG, lockedRegs | rsvdRegs);
3670 op1ByteMask = regByteMask;
3673 if (!(op2->gtUsedRegs & RBM_BYTE_REGS))
3675 // Pick a Byte register to use for op2, avoiding the one used by op1
3676 regByteMask |= rpPredictRegPick(TYP_BYTE, PREDICT_REG, lockedRegs | rsvdRegs | op1ByteMask);
3681 tree->gtUsedRegs |= regByteMask;
3682 regMask = regByteMask;
3694 /* non-integer division handled in generic way */
3695 if (!varTypeIsIntegral(type))
3697 tree->gtUsedRegs = 0;
3698 if (predictReg <= PREDICT_REG)
3699 predictReg = PREDICT_SCRATCH_REG;
3700 goto GENERIC_BINARY;
3703 #ifndef _TARGET_64BIT_
3705 if (type == TYP_LONG && (oper == GT_MOD || oper == GT_UMOD))
3707 /* Special case: a mod with an int op2 is done inline using idiv or div
3708 to avoid a costly call to the helper */
3710 noway_assert((op2->gtOper == GT_CNS_LNG) &&
3711 (op2->gtLngCon.gtLconVal == int(op2->gtLngCon.gtLconVal)));
3713 #if defined(_TARGET_X86_) || defined(_TARGET_ARM_)
3714 if (tree->gtFlags & GTF_REVERSE_OPS)
3716 tmpMask = rpPredictTreeRegUse(op2, PREDICT_REG, lockedRegs | RBM_PAIR_TMP,
3717 rsvdRegs | op1->gtRsvdRegs);
3718 tmpMask |= rpPredictTreeRegUse(op1, PREDICT_PAIR_TMP, lockedRegs | tmpMask, RBM_LASTUSE);
3722 tmpMask = rpPredictTreeRegUse(op1, PREDICT_PAIR_TMP, lockedRegs, rsvdRegs | op2->gtRsvdRegs);
3724 rpPredictTreeRegUse(op2, PREDICT_REG, lockedRegs | tmpMask | RBM_PAIR_TMP, RBM_LASTUSE);
3726 regMask = RBM_PAIR_TMP;
3727 #else // !_TARGET_X86_ && !_TARGET_ARM_
3728 #error "Non-ARM or x86 _TARGET_ in RegPredict for 64-bit MOD"
3729 #endif // !_TARGET_X86_ && !_TARGET_ARM_
3732 (regMaskSmall)(regMask | op1->gtUsedRegs | op2->gtUsedRegs |
3733 rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, regMask | tmpMask));
3737 #endif // _TARGET_64BIT_
3739 /* no divide immediate, so force integer constant which is not
3740 * a power of two to register
3743 if (op2->OperKind() & GTK_CONST)
3745 ssize_t ival = op2->gtIntConCommon.IconValue();
3747 /* Is the divisor a power of 2 ? */
3749 if (ival > 0 && genMaxOneBit(size_t(ival)))
3754 op2PredictReg = PREDICT_SCRATCH_REG;
3758 /* Non integer constant also must be enregistered */
3759 op2PredictReg = PREDICT_REG;
3762 regMaskTP trashedMask;
3763 trashedMask = DUMMY_INIT(RBM_ILLEGAL);
3764 regMaskTP op1ExcludeMask;
3765 op1ExcludeMask = DUMMY_INIT(RBM_ILLEGAL);
3766 regMaskTP op2ExcludeMask;
3767 op2ExcludeMask = DUMMY_INIT(RBM_ILLEGAL);
3769 #ifdef _TARGET_XARCH_
3770 /* Consider the case "a / b" - we'll need to trash EDX (via "CDQ") before
3771 * we can safely allow the "b" value to die. Unfortunately, if we simply
3772 * mark the node "b" as using EDX, this will not work if "b" is a register
3773 * variable that dies with this particular reference. Thus, if we want to
3774 * avoid this situation (where we would have to spill the variable from
3775 * EDX to someplace else), we need to explicitly mark the interference
3776 * of the variable at this point.
3779 if (op2->gtOper == GT_LCL_VAR)
3781 unsigned lclNum = op2->gtLclVarCommon.gtLclNum;
3782 varDsc = lvaTable + lclNum;
3783 if (varDsc->lvTracked)
3788 if (!VarSetOps::IsMember(this, raLclRegIntf[REG_EAX], varDsc->lvVarIndex))
3789 printf("Record interference between V%02u,T%02u and EAX -- int divide\n", lclNum,
3790 varDsc->lvVarIndex);
3791 if (!VarSetOps::IsMember(this, raLclRegIntf[REG_EDX], varDsc->lvVarIndex))
3792 printf("Record interference between V%02u,T%02u and EDX -- int divide\n", lclNum,
3793 varDsc->lvVarIndex);
3796 VarSetOps::AddElemD(this, raLclRegIntf[REG_EAX], varDsc->lvVarIndex);
3797 VarSetOps::AddElemD(this, raLclRegIntf[REG_EDX], varDsc->lvVarIndex);
3801 /* set the held register based on opcode */
3802 if (oper == GT_DIV || oper == GT_UDIV)
3806 trashedMask = (RBM_EAX | RBM_EDX);
3808 op2ExcludeMask = (RBM_EAX | RBM_EDX);
3810 #endif // _TARGET_XARCH_
3813 trashedMask = RBM_NONE;
3814 op1ExcludeMask = RBM_NONE;
3815 op2ExcludeMask = RBM_NONE;
3818 /* set the lvPref reg if possible */
3821 * Walking the gtNext link twice from here should get us back
3822 * to our parent node, if this is an simple assignment tree.
3824 dest = tree->gtNext;
3825 if (dest && (dest->gtOper == GT_LCL_VAR) && dest->gtNext && (dest->gtNext->OperKind() & GTK_ASGOP) &&
3826 dest->gtNext->gtOp.gtOp2 == tree)
3828 varDsc = lvaTable + dest->gtLclVarCommon.gtLclNum;
3829 varDsc->addPrefReg(regMask, this);
3831 #ifdef _TARGET_XARCH_
3832 op1PredictReg = PREDICT_REG_EDX; /* Normally target op1 into EDX */
3834 op1PredictReg = PREDICT_SCRATCH_REG;
3837 /* are we supposed to evaluate op2 first? */
3838 if (tree->gtFlags & GTF_REVERSE_OPS)
3840 tmpMask = rpPredictTreeRegUse(op2, op2PredictReg, lockedRegs | op2ExcludeMask,
3841 rsvdRegs | op1->gtRsvdRegs);
3842 rpPredictTreeRegUse(op1, op1PredictReg, lockedRegs | tmpMask | op1ExcludeMask, RBM_LASTUSE);
3846 tmpMask = rpPredictTreeRegUse(op1, op1PredictReg, lockedRegs | op1ExcludeMask,
3847 rsvdRegs | op2->gtRsvdRegs);
3848 rpPredictTreeRegUse(op2, op2PredictReg, tmpMask | lockedRegs | op2ExcludeMask, RBM_LASTUSE);
3853 /* grab EAX, EDX for this tree node */
3854 tree->gtUsedRegs = (regMaskSmall)trashedMask | op1->gtUsedRegs | op2->gtUsedRegs;
3862 if (predictReg <= PREDICT_REG)
3863 predictReg = PREDICT_SCRATCH_REG;
3865 #ifndef _TARGET_64BIT_
3866 if (type == TYP_LONG)
3868 if (op2->IsCnsIntOrI())
3870 regMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs);
3871 // no register used by op2
3872 op2->gtUsedRegs = 0;
3873 tree->gtUsedRegs = op1->gtUsedRegs;
3877 // since RBM_LNGARG_0 and RBM_SHIFT_LNG are hardwired we can't have them in the locked registers
3878 tmpMask = lockedRegs;
3879 tmpMask &= ~RBM_LNGARG_0;
3880 tmpMask &= ~RBM_SHIFT_LNG;
3882 // op2 goes to RBM_SHIFT, op1 to the RBM_LNGARG_0 pair
3883 if (tree->gtFlags & GTF_REVERSE_OPS)
3885 rpPredictTreeRegUse(op2, PREDICT_REG_SHIFT_LNG, tmpMask, RBM_NONE);
3886 tmpMask |= RBM_SHIFT_LNG;
3887 // Ensure that the RBM_SHIFT_LNG register interfere with op2's compCurLife
3888 // Fix 383843 X86/ARM ILGEN
3889 rpRecordRegIntf(RBM_SHIFT_LNG, compCurLife DEBUGARG("SHIFT_LNG arg setup"));
3890 rpPredictTreeRegUse(op1, PREDICT_PAIR_LNGARG_0, tmpMask, RBM_LASTUSE);
3894 rpPredictTreeRegUse(op1, PREDICT_PAIR_LNGARG_0, tmpMask, RBM_NONE);
3895 tmpMask |= RBM_LNGARG_0;
3896 // Ensure that the RBM_LNGARG_0 registers interfere with op1's compCurLife
3897 // Fix 383839 ARM ILGEN
3898 rpRecordRegIntf(RBM_LNGARG_0, compCurLife DEBUGARG("LNGARG_0 arg setup"));
3899 rpPredictTreeRegUse(op2, PREDICT_REG_SHIFT_LNG, tmpMask, RBM_LASTUSE);
3901 regMask = RBM_LNGRET; // function return registers
3902 op1->gtUsedRegs |= RBM_LNGARG_0;
3903 op2->gtUsedRegs |= RBM_SHIFT_LNG;
3905 tree->gtUsedRegs = op1->gtUsedRegs | op2->gtUsedRegs;
3907 // We are using a helper function to do shift:
3909 tree->gtUsedRegs |= RBM_CALLEE_TRASH;
3913 #endif // _TARGET_64BIT_
3915 #ifdef _TARGET_XARCH_
3916 if (!op2->IsCnsIntOrI())
3917 predictReg = PREDICT_NOT_REG_ECX;
3921 // Note that this code is also used by assigning shift operators (i.e. GT_ASG_LSH)
3923 regMaskTP tmpRsvdRegs;
3925 if ((tree->gtFlags & GTF_REVERSE_OPS) == 0)
3927 regMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs | op2->gtRsvdRegs);
3928 rsvdRegs = RBM_LASTUSE;
3929 tmpRsvdRegs = RBM_NONE;
3934 // Special case op1 of a constant
3935 if (op1->IsCnsIntOrI())
3936 tmpRsvdRegs = RBM_LASTUSE; // Allow a last use to occur in op2; See
3937 // System.Xml.Schema.BitSet:Get(int):bool
3939 tmpRsvdRegs = op1->gtRsvdRegs;
3943 if (!op2->IsCnsIntOrI())
3945 if ((REG_SHIFT != REG_NA) && ((RBM_SHIFT & tmpRsvdRegs) == 0))
3947 op2PredictReg = PREDICT_REG_SHIFT;
3951 op2PredictReg = PREDICT_REG;
3954 /* evaluate shift count into a register, likely the PREDICT_REG_SHIFT register */
3955 op2Mask = rpPredictTreeRegUse(op2, op2PredictReg, lockedRegs | regMask, tmpRsvdRegs);
3957 // If our target arch has a REG_SHIFT register then
3958 // we set the PrefReg when we have a LclVar for op2
3959 // we add an interference with REG_SHIFT for any other LclVars alive at op2
3960 if (REG_SHIFT != REG_NA)
3962 VARSET_TP liveSet(VarSetOps::MakeCopy(this, compCurLife));
3964 while (op2->gtOper == GT_COMMA)
3966 op2 = op2->gtOp.gtOp2;
3969 if (op2->gtOper == GT_LCL_VAR)
3971 varDsc = lvaTable + op2->gtLclVarCommon.gtLclNum;
3972 varDsc->setPrefReg(REG_SHIFT, this);
3973 if (varDsc->lvTracked)
3975 VarSetOps::RemoveElemD(this, liveSet, varDsc->lvVarIndex);
3979 // Ensure that we have a register interference with the LclVar in tree's LiveSet,
3980 // excluding the LclVar that was used for the shift amount as it is read-only
3981 // and can be kept alive through the shift operation
3983 rpRecordRegIntf(RBM_SHIFT, liveSet DEBUGARG("Variable Shift Register"));
3984 // In case op2Mask doesn't contain the required shift register,
3985 // we will or it in now.
3986 op2Mask |= RBM_SHIFT;
3990 if (tree->gtFlags & GTF_REVERSE_OPS)
3992 assert(regMask == RBM_NONE);
3993 regMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs | op2Mask, rsvdRegs | RBM_LASTUSE);
3996 #if CPU_HAS_BYTE_REGS
3997 if (varTypeIsByte(type))
3999 // Fix 383789 X86 ILGEN
4000 // Fix 383813 X86 ILGEN
4001 // Fix 383828 X86 ILGEN
4002 if (op1->gtOper == GT_LCL_VAR)
4004 varDsc = lvaTable + op1->gtLclVar.gtLclNum;
4005 if (varDsc->lvTracked)
4007 VARSET_TP op1VarBit(VarSetOps::MakeSingleton(this, varDsc->lvVarIndex));
4009 // Ensure that we don't assign a Non-Byteable register for op1's LCL_VAR
4010 rpRecordRegIntf(RBM_NON_BYTE_REGS, op1VarBit DEBUGARG("Non Byte Register"));
4013 if ((regMask & RBM_BYTE_REGS) == 0)
4015 // We need to grab a byte-able register, (i.e. EAX, EDX, ECX, EBX)
4016 // and we can't select one that is already reserved (i.e. lockedRegs or regMask)
4019 rpPredictRegPick(type, PREDICT_SCRATCH_REG, (lockedRegs | regMask | RBM_NON_BYTE_REGS));
4023 tree->gtUsedRegs = (regMaskSmall)(regMask | op2Mask);
4029 if (tree->gtFlags & GTF_REVERSE_OPS)
4031 if (predictReg == PREDICT_NONE)
4033 predictReg = PREDICT_REG;
4035 else if (rpHasVarIndexForPredict(predictReg))
4037 /* Don't propagate the use of tgt reg use in a GT_COMMA */
4038 predictReg = PREDICT_SCRATCH_REG;
4041 regMask = rpPredictTreeRegUse(op2, predictReg, lockedRegs, rsvdRegs);
4042 rpPredictTreeRegUse(op1, PREDICT_NONE, lockedRegs | regMask, RBM_LASTUSE);
4046 rpPredictTreeRegUse(op1, PREDICT_NONE, lockedRegs, RBM_LASTUSE);
4048 /* CodeGen will enregister the op2 side of a GT_COMMA */
4049 if (predictReg == PREDICT_NONE)
4051 predictReg = PREDICT_REG;
4053 else if (rpHasVarIndexForPredict(predictReg))
4055 /* Don't propagate the use of tgt reg use in a GT_COMMA */
4056 predictReg = PREDICT_SCRATCH_REG;
4059 regMask = rpPredictTreeRegUse(op2, predictReg, lockedRegs, rsvdRegs);
4061 // tree should only accumulate the used registers from the op2 side of the GT_COMMA
4063 tree->gtUsedRegs = op2->gtUsedRegs;
4064 if ((op2->gtOper == GT_LCL_VAR) && (rsvdRegs != 0))
4066 LclVarDsc* op2VarDsc = lvaTable + op2->gtLclVarCommon.gtLclNum;
4068 if (op2VarDsc->lvTracked)
4070 VARSET_TP op2VarBit(VarSetOps::MakeSingleton(this, op2VarDsc->lvVarIndex));
4071 rpRecordRegIntf(rsvdRegs, op2VarBit DEBUGARG("comma use"));
4078 noway_assert(op1 != NULL && op2 != NULL);
4081 * If the gtUsedRegs conflicts with lockedRegs
4082 * then we going to have to spill some registers
4083 * into the non-trashed register set to keep it alive
4087 regMaskTP spillRegs;
4088 spillRegs = lockedRegs & tree->gtUsedRegs;
4092 /* Find the next register that needs to be spilled */
4093 tmpMask = genFindLowestBit(spillRegs);
4098 printf("Predict spill of %s before: ", getRegName(genRegNumFromMask(tmpMask)));
4099 gtDispTree(tree, 0, NULL, true);
4102 /* In Codegen it will typically introduce a spill temp here */
4103 /* rather than relocating the register to a non trashed reg */
4104 rpPredictSpillCnt++;
4107 /* Remove it from the spillRegs and lockedRegs*/
4108 spillRegs &= ~tmpMask;
4109 lockedRegs &= ~tmpMask;
4112 VARSET_TP startQmarkCondUseInPlaceVars(VarSetOps::MakeCopy(this, rpUseInPlace));
4114 /* Evaluate the <cond> subtree */
4115 rpPredictTreeRegUse(op1, PREDICT_NONE, lockedRegs, RBM_LASTUSE);
4116 VarSetOps::Assign(this, rpUseInPlace, startQmarkCondUseInPlaceVars);
4117 tree->gtUsedRegs = op1->gtUsedRegs;
4119 noway_assert(op2->gtOper == GT_COLON);
4120 if (rpHasVarIndexForPredict(predictReg) && ((op2->gtFlags & (GTF_ASG | GTF_CALL)) != 0))
4122 // Don't try to target the register specified in predictReg when we have complex subtrees
4124 predictReg = PREDICT_SCRATCH_REG;
4126 GenTreePtr elseTree = op2->AsColon()->ElseNode();
4127 GenTreePtr thenTree = op2->AsColon()->ThenNode();
4129 noway_assert(thenTree != NULL && elseTree != NULL);
4131 // Update compCurLife to only those vars live on the <then> subtree
4133 VarSetOps::Assign(this, compCurLife, tree->gtQmark.gtThenLiveSet);
4135 if (type == TYP_VOID)
4137 /* Evaluate the <then> subtree */
4138 rpPredictTreeRegUse(thenTree, PREDICT_NONE, lockedRegs, RBM_LASTUSE);
4140 predictReg = PREDICT_NONE;
4144 // A mask to use to force the predictor to choose low registers (to reduce code size)
4145 regMaskTP avoidRegs = RBM_NONE;
4147 avoidRegs = (RBM_R12 | RBM_LR);
4149 if (predictReg <= PREDICT_REG)
4150 predictReg = PREDICT_SCRATCH_REG;
4152 /* Evaluate the <then> subtree */
4154 rpPredictTreeRegUse(thenTree, predictReg, lockedRegs, rsvdRegs | avoidRegs | RBM_LASTUSE);
4158 rpPredictReg op1PredictReg = rpGetPredictForMask(regMask);
4159 if (op1PredictReg != PREDICT_NONE)
4160 predictReg = op1PredictReg;
4164 VarSetOps::Assign(this, rpUseInPlace, startQmarkCondUseInPlaceVars);
4166 /* Evaluate the <else> subtree */
4167 // First record the post-then liveness, and reset the current liveness to the else
4169 CLANG_FORMAT_COMMENT_ANCHOR;
4172 VARSET_TP postThenLive(VarSetOps::MakeCopy(this, compCurLife));
4175 VarSetOps::Assign(this, compCurLife, tree->gtQmark.gtElseLiveSet);
4177 rpPredictTreeRegUse(elseTree, predictReg, lockedRegs, rsvdRegs | RBM_LASTUSE);
4178 tree->gtUsedRegs |= thenTree->gtUsedRegs | elseTree->gtUsedRegs;
4180 // The then and the else are "virtual basic blocks" that form a control-flow diamond.
4181 // They each have only one successor, which they share. Their live-out sets must equal the
4182 // live-in set of this virtual successor block, and thus must be the same. We can assert
4183 // that equality here.
4184 assert(VarSetOps::Equal(this, compCurLife, postThenLive));
4188 regMaskTP reloadMask = RBM_NONE;
4192 regMaskTP reloadReg;
4194 /* Get an extra register to hold it */
4195 reloadReg = rpPredictRegPick(TYP_INT, PREDICT_REG, lockedRegs | regMask | reloadMask);
4199 printf("Predict reload into %s after : ", getRegName(genRegNumFromMask(reloadReg)));
4200 gtDispTree(tree, 0, NULL, true);
4203 reloadMask |= reloadReg;
4208 /* update the gtUsedRegs mask */
4209 tree->gtUsedRegs |= reloadMask;
4216 tree->gtUsedRegs = RBM_NONE;
4219 /* Is there a return value? */
4222 #if FEATURE_FP_REGALLOC
4223 if (varTypeIsFloating(type))
4225 predictReg = PREDICT_FLTRET;
4226 if (type == TYP_FLOAT)
4227 regMask = RBM_FLOATRET;
4229 regMask = RBM_DOUBLERET;
4233 if (isRegPairType(type))
4235 predictReg = PREDICT_LNGRET;
4236 regMask = RBM_LNGRET;
4240 predictReg = PREDICT_INTRET;
4241 regMask = RBM_INTRET;
4243 if (info.compCallUnmanaged)
4245 lockedRegs |= (RBM_PINVOKE_TCB | RBM_PINVOKE_FRAME);
4247 rpPredictTreeRegUse(op1, predictReg, lockedRegs, RBM_LASTUSE);
4248 tree->gtUsedRegs = op1->gtUsedRegs | (regMaskSmall)regMask;
4251 #if defined(_TARGET_ARM_) && defined(PROFILING_SUPPORTED)
4252 // When on Arm under profiler, to emit Leave callback we would need RBM_PROFILER_RETURN_USED.
4253 // We could optimize on registers based on int/long or no return value. But to
4254 // keep it simple we will mark entire RBM_PROFILER_RETURN_USED as used regs here.
4255 if (compIsProfilerHookNeeded())
4257 tree->gtUsedRegs |= RBM_PROFILER_RET_USED;
4266 rpPredictTreeRegUse(op1, PREDICT_NONE, lockedRegs, RBM_LASTUSE);
4267 regMask = genReturnRegForTree(tree);
4268 tree->gtUsedRegs = op1->gtUsedRegs | (regMaskSmall)regMask;
4271 tree->gtUsedRegs = 0;
4277 /* This must be a test of a relational operator */
4279 noway_assert(op1->OperIsCompare());
4281 /* Only condition code set by this operation */
4283 rpPredictTreeRegUse(op1, PREDICT_NONE, lockedRegs, RBM_NONE);
4285 tree->gtUsedRegs = op1->gtUsedRegs;
4291 noway_assert(type <= TYP_INT);
4292 noway_assert(compCurBB->bbJumpKind == BBJ_SWITCH);
4295 regMask = rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs, RBM_NONE);
4296 unsigned jumpCnt = compCurBB->bbJumpSwt->bbsCount;
4299 // Table based switch requires an extra register for the table base
4300 regMask |= rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | regMask);
4302 tree->gtUsedRegs = op1->gtUsedRegs | regMask;
4304 #else // !_TARGET_ARM_
4305 rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs, RBM_NONE);
4306 tree->gtUsedRegs = op1->gtUsedRegs;
4307 #endif // _TARGET_ARM_
4312 if (predictReg <= PREDICT_REG)
4313 predictReg = PREDICT_SCRATCH_REG;
4315 rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs);
4316 // Need a reg to load exponent into
4317 regMask = rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | rsvdRegs);
4318 tree->gtUsedRegs = (regMaskSmall)regMask | op1->gtUsedRegs;
4322 regMask = rpPredictTreeRegUse(op1, PREDICT_SCRATCH_REG, lockedRegs, rsvdRegs);
4326 if (info.compInitMem)
4328 // We zero out two registers in the ARM codegen path
4330 rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | rsvdRegs | regMask | op2Mask);
4334 op1->gtUsedRegs |= (regMaskSmall)regMask;
4335 tree->gtUsedRegs = op1->gtUsedRegs | (regMaskSmall)op2Mask;
4337 // The result will be put in the reg we picked for the size
4338 // regMask = <already set as we want it to be>
4345 if (predictReg <= PREDICT_REG)
4346 predictReg = PREDICT_SCRATCH_REG;
4348 regMaskTP avoidRegs = (RBM_R12 | RBM_LR); // A mask to use to force the predictor to choose low
4349 // registers (to reduce code size)
4351 tmpMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs | avoidRegs);
4354 if (fgIsIndirOfAddrOfLocal(tree) != NULL)
4356 compUpdateLifeVar</*ForCodeGen*/ false>(tree);
4360 unsigned objSize = info.compCompHnd->getClassSize(tree->gtObj.gtClass);
4361 regMaskTP preferReg = rpPredictRegMask(predictReg, TYP_I_IMPL);
4362 // If it has one bit set, and that's an arg reg...
4363 if (preferReg != RBM_NONE && genMaxOneBit(preferReg) && ((preferReg & RBM_ARG_REGS) != 0))
4365 // We are passing the 'obj' in the argument registers
4367 regNumber rn = genRegNumFromMask(preferReg);
4369 // Add the registers used to pass the 'obj' to regMask.
4370 for (unsigned i = 0; i < objSize / 4; i++)
4372 if (rn == MAX_REG_ARG)
4375 regMask |= genRegMask(rn);
4376 rn = genRegArgNext(rn);
4381 // We are passing the 'obj' in the outgoing arg space
4382 // We will need one register to load into unless the 'obj' size is 4 or less.
4386 regMask = rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | tmpMask | avoidRegs);
4389 tree->gtUsedRegs = (regMaskSmall)(regMask | tmpMask);
4391 #else // !_TARGET_ARM_
4393 #endif // _TARGET_ARM_
4399 regMaskTP preferReg = rpPredictRegMask(predictReg, TYP_I_IMPL);
4401 if ((((preferReg - 1) & preferReg) == 0) && ((preferReg & RBM_ARG_REGS) != 0))
4403 // A MKREFANY takes up two registers.
4404 regNumber rn = genRegNumFromMask(preferReg);
4406 if (rn < MAX_REG_ARG)
4408 regMask |= genRegMask(rn);
4409 rn = genRegArgNext(rn);
4410 if (rn < MAX_REG_ARG)
4411 regMask |= genRegMask(rn);
4414 if (regMask != RBM_NONE)
4416 // Condensation of GENERIC_BINARY path.
4417 assert((tree->gtFlags & GTF_REVERSE_OPS) == 0);
4418 op2PredictReg = PREDICT_REG;
4419 regMaskTP regMaskOp1 = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs | op2->gtRsvdRegs);
4420 rpPredictTreeRegUse(op2, op2PredictReg, lockedRegs | regMaskOp1, RBM_LASTUSE);
4421 regMask |= op1->gtUsedRegs | op2->gtUsedRegs;
4422 tree->gtUsedRegs = (regMaskSmall)regMask;
4425 tree->gtUsedRegs = op1->gtUsedRegs;
4426 #endif // _TARGET_ARM_
4427 goto GENERIC_BINARY;
4434 goto GENERIC_BINARY;
4438 // Ensure we can write to op2. op2 will hold the output.
4439 if (predictReg < PREDICT_SCRATCH_REG)
4440 predictReg = PREDICT_SCRATCH_REG;
4442 if (tree->gtFlags & GTF_REVERSE_OPS)
4444 op2Mask = rpPredictTreeRegUse(op2, predictReg, lockedRegs, rsvdRegs);
4445 regMask = rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs, rsvdRegs | op2Mask);
4449 regMask = rpPredictTreeRegUse(op1, PREDICT_REG, lockedRegs, rsvdRegs);
4450 op2Mask = rpPredictTreeRegUse(op2, PREDICT_SCRATCH_REG, lockedRegs, rsvdRegs | regMask);
4452 tree->gtUsedRegs = (regMaskSmall)(regMask | op2Mask);
4459 // This unary operator simply passes through the value from its child (much like GT_NOP)
4460 // and thus won't need a scratch register.
4461 regMask = rpPredictTreeRegUse(op1, predictReg, lockedRegs, rsvdRegs);
4462 tree->gtUsedRegs = op1->gtUsedRegs;
4469 noway_assert(!"unexpected simple operator in reg use prediction");
4474 /* See what kind of a special operator we have here */
4479 GenTreeArgList* list;
4481 unsigned regArgsNum;
4483 regMaskTP regArgMask;
4484 regMaskTP curArgMask;
4490 /* initialize so we can just or in various bits */
4491 tree->gtUsedRegs = RBM_NONE;
4493 #if GTF_CALL_REG_SAVE
4495 * Unless the GTF_CALL_REG_SAVE flag is set,
4496 * we can't preserve the RBM_CALLEE_TRASH registers.
4497 * (likewise we can't preserve the return registers)
4498 * So we remove them from the lockedRegs set and
4499 * record any of them in the keepMask
4502 if (tree->gtFlags & GTF_CALL_REG_SAVE)
4504 regMaskTP trashMask = genReturnRegForTree(tree);
4506 keepMask = lockedRegs & trashMask;
4507 lockedRegs &= ~trashMask;
4512 keepMask = lockedRegs & RBM_CALLEE_TRASH;
4513 lockedRegs &= ~RBM_CALLEE_TRASH;
4519 /* Is there an object pointer? */
4520 if (tree->gtCall.gtCallObjp)
4522 /* Evaluate the instance pointer first */
4524 args = tree->gtCall.gtCallObjp;
4526 /* the objPtr always goes to an integer register (through temp or directly) */
4527 noway_assert(regArgsNum == 0);
4530 /* Must be passed in a register */
4532 noway_assert(args->gtFlags & GTF_LATE_ARG);
4534 /* Must be either a deferred reg arg node or a GT_ASG node */
4536 noway_assert(args->IsArgPlaceHolderNode() || args->IsNothingNode() || (args->gtOper == GT_ASG) ||
4537 args->OperIsCopyBlkOp() || (args->gtOper == GT_COMMA));
4539 if (!args->IsArgPlaceHolderNode())
4541 rpPredictTreeRegUse(args, PREDICT_NONE, lockedRegs, RBM_LASTUSE);
4544 VARSET_TP startArgUseInPlaceVars(VarSetOps::UninitVal());
4545 VarSetOps::Assign(this, startArgUseInPlaceVars, rpUseInPlace);
4547 /* process argument list */
4548 for (list = tree->gtCall.gtCallArgs; list; list = list->Rest())
4550 args = list->Current();
4552 if (args->gtFlags & GTF_LATE_ARG)
4554 /* Must be either a Placeholder/NOP node or a GT_ASG node */
4556 noway_assert(args->IsArgPlaceHolderNode() || args->IsNothingNode() || (args->gtOper == GT_ASG) ||
4557 args->OperIsCopyBlkOp() || (args->gtOper == GT_COMMA));
4559 if (!args->IsArgPlaceHolderNode())
4561 rpPredictTreeRegUse(args, PREDICT_NONE, lockedRegs, RBM_LASTUSE);
4568 #ifdef FEATURE_FIXED_OUT_ARGS
4569 // We'll store this argument into the outgoing argument area
4570 // It needs to be in a register to be stored.
4572 predictReg = PREDICT_REG;
4574 #else // !FEATURE_FIXED_OUT_ARGS
4575 // We'll generate a push for this argument
4577 predictReg = PREDICT_NONE;
4578 if (varTypeIsSmall(args->TypeGet()))
4580 /* We may need to sign or zero extend a small type using a register */
4581 predictReg = PREDICT_SCRATCH_REG;
4585 rpPredictTreeRegUse(args, predictReg, lockedRegs, RBM_LASTUSE);
4587 VarSetOps::Assign(this, rpUseInPlace, startArgUseInPlaceVars);
4588 tree->gtUsedRegs |= args->gtUsedRegs;
4591 /* Is there a late argument list */
4594 regArgMask = RBM_NONE; // Set of argument registers that have already been setup.
4597 /* process the late argument list */
4598 for (list = tree->gtCall.gtCallLateArgs; list; regIndex++)
4600 // If the current argument being copied is a promoted struct local, set this pointer to its description.
4601 LclVarDsc* promotedStructLocal = NULL;
4603 curArgMask = RBM_NONE; // Set of argument registers that are going to be setup by this arg
4604 tmpMask = RBM_NONE; // Set of additional temp registers that are need only to setup the current arg
4606 assert(list->OperIsList());
4608 args = list->Current();
4609 list = list->Rest();
4611 assert(!args->IsArgPlaceHolderNode()); // No place holders nodes are in gtCallLateArgs;
4613 fgArgTabEntryPtr curArgTabEntry = gtArgEntryByNode(tree->AsCall(), args);
4614 assert(curArgTabEntry);
4616 regNumber regNum = curArgTabEntry->regNum; // first register use to pass this argument
4618 curArgTabEntry->numSlots; // number of outgoing arg stack slots used by this argument
4620 rpPredictReg argPredictReg;
4621 regMaskTP avoidReg = RBM_NONE;
4623 if (regNum != REG_STK)
4625 argPredictReg = rpGetPredictForReg(regNum);
4626 curArgMask |= genRegMask(regNum);
4630 assert(numSlots > 0);
4631 argPredictReg = PREDICT_NONE;
4633 // Force the predictor to choose a low register when regNum is REG_STK to reduce code bloat
4634 avoidReg = (RBM_R12 | RBM_LR);
4639 // For TYP_LONG or TYP_DOUBLE register arguments we need to add the second argument register
4641 if ((regNum != REG_STK) && ((args->TypeGet() == TYP_LONG) || (args->TypeGet() == TYP_DOUBLE)))
4643 // 64-bit longs and doubles require 2 consecutive argument registers
4644 curArgMask |= genRegMask(REG_NEXT(regNum));
4646 else if (args->TypeGet() == TYP_STRUCT)
4648 GenTreePtr argx = args;
4649 GenTreePtr lclVarTree = NULL;
4651 /* The GT_OBJ may be be a child of a GT_COMMA */
4652 while (argx->gtOper == GT_COMMA)
4654 argx = argx->gtOp.gtOp2;
4656 unsigned originalSize = 0;
4658 if (argx->gtOper == GT_OBJ)
4660 originalSize = info.compCompHnd->getClassSize(argx->gtObj.gtClass);
4662 // Is it the address of a promoted struct local?
4663 if (argx->gtObj.gtOp1->gtOper == GT_ADDR && argx->gtObj.gtOp1->gtOp.gtOp1->gtOper == GT_LCL_VAR)
4665 lclVarTree = argx->gtObj.gtOp1->gtOp.gtOp1;
4666 LclVarDsc* varDsc = &lvaTable[lclVarTree->gtLclVarCommon.gtLclNum];
4667 if (varDsc->lvPromoted)
4668 promotedStructLocal = varDsc;
4671 else if (argx->gtOper == GT_LCL_VAR)
4673 varDsc = lvaTable + argx->gtLclVarCommon.gtLclNum;
4674 originalSize = varDsc->lvSize();
4676 // Is it a promoted struct local?
4677 if (varDsc->lvPromoted)
4678 promotedStructLocal = varDsc;
4680 else if (argx->gtOper == GT_MKREFANY)
4682 originalSize = 2 * TARGET_POINTER_SIZE;
4686 noway_assert(!"Can't predict unsupported TYP_STRUCT arg kind");
4689 // We only pass arguments differently if it a struct local "independently" promoted, which
4690 // allows the field locals can be independently enregistered.
4691 if (promotedStructLocal != NULL)
4693 if (lvaGetPromotionType(promotedStructLocal) != PROMOTION_TYPE_INDEPENDENT)
4694 promotedStructLocal = NULL;
4697 unsigned slots = ((unsigned)(roundUp(originalSize, TARGET_POINTER_SIZE))) / REGSIZE_BYTES;
4699 // Are we passing a TYP_STRUCT in multiple integer registers?
4700 // if so set up curArgMask to reflect this
4701 // Also slots is updated to reflect the number of outgoing arg slots that we will write
4702 if (regNum != REG_STK)
4704 regNumber regLast = (curArgTabEntry->isHfaRegArg) ? LAST_FP_ARGREG : REG_ARG_LAST;
4705 assert(genIsValidReg(regNum));
4706 regNumber nextReg = REG_NEXT(regNum);
4708 while (slots > 0 && nextReg <= regLast)
4710 curArgMask |= genRegMask(nextReg);
4711 nextReg = REG_NEXT(nextReg);
4716 if ((promotedStructLocal != NULL) && (curArgMask != RBM_NONE))
4718 // All or a portion of this struct will be placed in the argument registers indicated by
4719 // "curArgMask". We build in knowledge of the order in which the code is generated here, so
4720 // that the second arg to be evaluated interferes with the reg for the first, the third with
4721 // the regs for the first and second, etc. But since we always place the stack slots before
4722 // placing the register slots we do not add inteferences for any part of the struct that gets
4723 // passed on the stack.
4726 PREDICT_NONE; // We will target the indivual fields into registers but not the whole struct
4727 regMaskTP prevArgMask = RBM_NONE;
4728 for (unsigned i = 0; i < promotedStructLocal->lvFieldCnt; i++)
4730 LclVarDsc* fieldVarDsc = &lvaTable[promotedStructLocal->lvFieldLclStart + i];
4731 if (fieldVarDsc->lvTracked)
4733 assert(lclVarTree != NULL);
4734 if (prevArgMask != RBM_NONE)
4736 rpRecordRegIntf(prevArgMask, VarSetOps::MakeSingleton(this, fieldVarDsc->lvVarIndex)
4737 DEBUGARG("fieldVar/argReg"));
4740 // Now see many registers this uses up.
4741 unsigned firstRegOffset = fieldVarDsc->lvFldOffset / TARGET_POINTER_SIZE;
4742 unsigned nextAfterLastRegOffset =
4743 (fieldVarDsc->lvFldOffset + fieldVarDsc->lvExactSize + TARGET_POINTER_SIZE - 1) /
4744 TARGET_POINTER_SIZE;
4745 unsigned nextAfterLastArgRegOffset =
4746 min(nextAfterLastRegOffset,
4747 genIsValidIntReg(regNum) ? REG_NEXT(REG_ARG_LAST) : REG_NEXT(LAST_FP_ARGREG));
4749 for (unsigned regOffset = firstRegOffset; regOffset < nextAfterLastArgRegOffset;
4752 prevArgMask |= genRegMask(regNumber(regNum + regOffset));
4755 if (nextAfterLastRegOffset > nextAfterLastArgRegOffset)
4760 if ((fieldVarDsc->lvFldOffset % TARGET_POINTER_SIZE) == 0)
4762 // Add the argument register used here as a preferred register for this fieldVarDsc
4764 regNumber firstRegUsed = regNumber(regNum + firstRegOffset);
4765 fieldVarDsc->setPrefReg(firstRegUsed, this);
4768 compUpdateLifeVar</*ForCodeGen*/ false>(argx);
4771 // If slots is greater than zero then part or all of this TYP_STRUCT
4772 // argument is passed in the outgoing argument area. (except HFA arg)
4774 if ((slots > 0) && !curArgTabEntry->isHfaRegArg)
4776 // We will need a register to address the TYP_STRUCT
4777 // Note that we can use an argument register in curArgMask as in
4778 // codegen we pass the stack portion of the argument before we
4779 // setup the register part.
4782 // Force the predictor to choose a LOW_REG here to reduce code bloat
4783 avoidReg = (RBM_R12 | RBM_LR);
4785 assert(tmpMask == RBM_NONE);
4786 tmpMask = rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | regArgMask | avoidReg);
4788 // If slots > 1 then we will need a second register to perform the load/store into the outgoing
4792 tmpMask |= rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG,
4793 lockedRegs | regArgMask | tmpMask | avoidReg);
4796 } // (args->TypeGet() == TYP_STRUCT)
4797 #endif // _TARGET_ARM_
4799 // If we have a promotedStructLocal we don't need to call rpPredictTreeRegUse(args, ...
4800 // as we have already calculated the correct tmpMask and curArgMask values and
4801 // by calling rpPredictTreeRegUse we would just add unnecessary register inteferences.
4803 if (promotedStructLocal == NULL)
4805 /* Target the appropriate argument register */
4806 tmpMask |= rpPredictTreeRegUse(args, argPredictReg, lockedRegs | regArgMask, RBM_LASTUSE);
4809 // We mark OBJ(ADDR(LOCAL)) with GTF_VAR_DEATH since the local is required to live
4810 // for the duration of the OBJ.
4811 if (args->OperGet() == GT_OBJ && (args->gtFlags & GTF_VAR_DEATH))
4813 GenTreePtr lclVarTree = fgIsIndirOfAddrOfLocal(args);
4814 assert(lclVarTree != NULL); // Or else would not be marked with GTF_VAR_DEATH.
4815 compUpdateLifeVar</*ForCodeGen*/ false>(lclVarTree);
4818 regArgMask |= curArgMask;
4819 args->gtUsedRegs |= (tmpMask | regArgMask);
4820 tree->gtUsedRegs |= args->gtUsedRegs;
4821 tree->gtCall.gtCallLateArgs->gtUsedRegs |= args->gtUsedRegs;
4823 if (args->gtUsedRegs != RBM_NONE)
4825 // Add register interference with the set of registers used or in use when we evaluated
4826 // the current arg, with whatever is alive after the current arg
4828 rpRecordRegIntf(args->gtUsedRegs, compCurLife DEBUGARG("register arg setup"));
4830 VarSetOps::Assign(this, rpUseInPlace, startArgUseInPlaceVars);
4832 assert(list == NULL);
4834 #ifdef LEGACY_BACKEND
4835 #if CPU_LOAD_STORE_ARCH
4836 #ifdef FEATURE_READYTORUN_COMPILER
4837 if (tree->gtCall.IsR2RRelativeIndir())
4839 tree->gtUsedRegs |= RBM_R2R_INDIRECT_PARAM;
4841 #endif // FEATURE_READYTORUN_COMPILER
4842 #endif // CPU_LOAD_STORE_ARCH
4843 #endif // LEGACY_BACKEND
4845 regMaskTP callAddrMask;
4846 callAddrMask = RBM_NONE;
4847 #if CPU_LOAD_STORE_ARCH
4848 predictReg = PREDICT_SCRATCH_REG;
4850 predictReg = PREDICT_NONE;
4853 switch (tree->gtFlags & GTF_CALL_VIRT_KIND_MASK)
4855 case GTF_CALL_VIRT_STUB:
4857 // We only want to record an interference between the virtual stub
4858 // param reg and anything that's live AFTER the call, but we've not
4859 // yet processed the indirect target. So add virtualStubParamInfo.regMask
4860 // to interferingRegs.
4861 interferingRegs |= virtualStubParamInfo->GetRegMask();
4864 printf("Adding interference with Virtual Stub Param\n");
4866 codeGen->regSet.rsSetRegsModified(virtualStubParamInfo->GetRegMask());
4868 if (tree->gtCall.gtCallType == CT_INDIRECT)
4870 predictReg = virtualStubParamInfo->GetPredict();
4874 case GTF_CALL_VIRT_VTABLE:
4875 predictReg = PREDICT_SCRATCH_REG;
4878 case GTF_CALL_NONVIRT:
4879 predictReg = PREDICT_SCRATCH_REG;
4883 if (tree->gtCall.gtCallType == CT_INDIRECT)
4885 #if defined(_TARGET_ARM_) || defined(_TARGET_AMD64_)
4886 if (tree->gtCall.gtCallCookie)
4888 codeGen->regSet.rsSetRegsModified(RBM_PINVOKE_COOKIE_PARAM | RBM_PINVOKE_TARGET_PARAM);
4890 callAddrMask |= rpPredictTreeRegUse(tree->gtCall.gtCallCookie, PREDICT_REG_PINVOKE_COOKIE_PARAM,
4891 lockedRegs | regArgMask, RBM_LASTUSE);
4893 // Just in case we predict some other registers, force interference with our two special
4894 // parameters: PINVOKE_COOKIE_PARAM & PINVOKE_TARGET_PARAM
4895 callAddrMask |= (RBM_PINVOKE_COOKIE_PARAM | RBM_PINVOKE_TARGET_PARAM);
4897 predictReg = PREDICT_REG_PINVOKE_TARGET_PARAM;
4901 rpPredictTreeRegUse(tree->gtCall.gtCallAddr, predictReg, lockedRegs | regArgMask, RBM_LASTUSE);
4903 else if (predictReg != PREDICT_NONE)
4905 callAddrMask |= rpPredictRegPick(TYP_I_IMPL, predictReg, lockedRegs | regArgMask);
4908 if (tree->gtFlags & GTF_CALL_UNMANAGED)
4910 // Need a register for tcbReg
4912 rpPredictRegPick(TYP_I_IMPL, PREDICT_SCRATCH_REG, lockedRegs | regArgMask | callAddrMask);
4913 #if CPU_LOAD_STORE_ARCH
4914 // Need an extra register for tmpReg
4916 rpPredictRegPick(TYP_I_IMPL, PREDICT_SCRATCH_REG, lockedRegs | regArgMask | callAddrMask);
4920 tree->gtUsedRegs |= callAddrMask;
4922 /* After the call restore the orginal value of lockedRegs */
4923 lockedRegs |= keepMask;
4925 /* set the return register */
4926 regMask = genReturnRegForTree(tree);
4928 if (regMask & rsvdRegs)
4930 // We will need to relocate the return register value
4931 regMaskTP intRegMask = (regMask & RBM_ALLINT);
4932 #if FEATURE_FP_REGALLOC
4933 regMaskTP floatRegMask = (regMask & RBM_ALLFLOAT);
4939 if (intRegMask == RBM_INTRET)
4941 regMask |= rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, rsvdRegs | regMask);
4943 else if (intRegMask == RBM_LNGRET)
4945 regMask |= rpPredictRegPick(TYP_LONG, PREDICT_SCRATCH_REG, rsvdRegs | regMask);
4949 noway_assert(!"unexpected return regMask");
4953 #if FEATURE_FP_REGALLOC
4956 if (floatRegMask == RBM_FLOATRET)
4958 regMask |= rpPredictRegPick(TYP_FLOAT, PREDICT_SCRATCH_REG, rsvdRegs | regMask);
4960 else if (floatRegMask == RBM_DOUBLERET)
4962 regMask |= rpPredictRegPick(TYP_DOUBLE, PREDICT_SCRATCH_REG, rsvdRegs | regMask);
4964 else // HFA return case
4966 for (unsigned f = 0; f < genCountBits(floatRegMask); f++)
4968 regMask |= rpPredictRegPick(TYP_FLOAT, PREDICT_SCRATCH_REG, rsvdRegs | regMask);
4975 /* the return registers (if any) are killed */
4976 tree->gtUsedRegs |= regMask;
4978 #if GTF_CALL_REG_SAVE
4979 if (!(tree->gtFlags & GTF_CALL_REG_SAVE))
4982 /* the RBM_CALLEE_TRASH set are killed (i.e. EAX,ECX,EDX) */
4983 tree->gtUsedRegs |= RBM_CALLEE_TRASH;
4987 #if defined(_TARGET_ARM_) && defined(PROFILING_SUPPORTED)
4988 // Mark required registers for emitting tailcall profiler callback as used
4989 if (compIsProfilerHookNeeded() && tree->gtCall.IsTailCall() && (tree->gtCall.gtCallType == CT_USER_FUNC))
4991 tree->gtUsedRegs |= RBM_PROFILER_TAIL_USED;
4998 // Figure out which registers can't be touched
5000 for (dim = 0; dim < tree->gtArrElem.gtArrRank; dim++)
5001 rsvdRegs |= tree->gtArrElem.gtArrInds[dim]->gtRsvdRegs;
5003 regMask = rpPredictTreeRegUse(tree->gtArrElem.gtArrObj, PREDICT_REG, lockedRegs, rsvdRegs);
5008 #if CPU_LOAD_STORE_ARCH
5009 // We need a register to load the bounds of the MD array
5010 regMask |= rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | regMask);
5013 for (dim = 0; dim < tree->gtArrElem.gtArrRank; dim++)
5015 /* We need scratch registers to compute index-lower_bound.
5016 Also, gtArrInds[0]'s register will be used as the second
5017 addressability register (besides gtArrObj's) */
5019 regMaskTP dimMask = rpPredictTreeRegUse(tree->gtArrElem.gtArrInds[dim], PREDICT_SCRATCH_REG,
5020 lockedRegs | regMask | dimsMask, rsvdRegs);
5024 dimsMask |= dimMask;
5026 #ifdef _TARGET_XARCH_
5027 // INS_imul doesnt have an immediate constant.
5028 if (!jitIsScaleIndexMul(tree->gtArrElem.gtArrElemSize))
5029 regMask |= rpPredictRegPick(TYP_INT, PREDICT_SCRATCH_REG, lockedRegs | regMask | dimsMask);
5031 tree->gtUsedRegs = (regMaskSmall)(regMask | dimsMask);
5036 #ifdef _TARGET_XARCH_
5037 rsvdRegs |= RBM_EAX;
5039 if (tree->gtCmpXchg.gtOpLocation->OperGet() == GT_LCL_VAR)
5041 regMask = rpPredictTreeRegUse(tree->gtCmpXchg.gtOpLocation, PREDICT_REG, lockedRegs, rsvdRegs);
5045 regMask = rpPredictTreeRegUse(tree->gtCmpXchg.gtOpLocation, PREDICT_ADDR, lockedRegs, rsvdRegs);
5047 op2Mask = rpPredictTreeRegUse(tree->gtCmpXchg.gtOpValue, PREDICT_REG, lockedRegs, rsvdRegs | regMask);
5049 #ifdef _TARGET_XARCH_
5050 rsvdRegs &= ~RBM_EAX;
5051 tmpMask = rpPredictTreeRegUse(tree->gtCmpXchg.gtOpComparand, PREDICT_REG_EAX, lockedRegs,
5052 rsvdRegs | regMask | op2Mask);
5053 tree->gtUsedRegs = (regMaskSmall)(RBM_EAX | regMask | op2Mask | tmpMask);
5054 predictReg = PREDICT_REG_EAX; // When this is done the result is always in EAX.
5057 tree->gtUsedRegs = (regMaskSmall)(regMask | op2Mask | tmpMask);
5062 case GT_ARR_BOUNDS_CHECK:
5064 regMaskTP opArrLenRsvd = rsvdRegs | tree->gtBoundsChk.gtIndex->gtRsvdRegs;
5065 regMask = rpPredictTreeRegUse(tree->gtBoundsChk.gtArrLen, PREDICT_REG, lockedRegs, opArrLenRsvd);
5066 rpPredictTreeRegUse(tree->gtBoundsChk.gtIndex, PREDICT_REG, lockedRegs | regMask, RBM_LASTUSE);
5069 (regMaskSmall)regMask | tree->gtBoundsChk.gtArrLen->gtUsedRegs | tree->gtBoundsChk.gtIndex->gtUsedRegs;
5074 NO_WAY("unexpected special operator in reg use prediction");
5081 /* make sure we set them to something reasonable */
5082 if (tree->gtUsedRegs & RBM_ILLEGAL)
5083 noway_assert(!"used regs not set properly in reg use prediction");
5085 if (regMask & RBM_ILLEGAL)
5086 noway_assert(!"return value not set propery in reg use prediction");
5091 * If the gtUsedRegs conflicts with lockedRegs
5092 * then we going to have to spill some registers
5093 * into the non-trashed register set to keep it alive
5095 regMaskTP spillMask;
5096 spillMask = tree->gtUsedRegs & lockedRegs;
5102 /* Find the next register that needs to be spilled */
5103 tmpMask = genFindLowestBit(spillMask);
5108 printf("Predict spill of %s before: ", getRegName(genRegNumFromMask(tmpMask)));
5109 gtDispTree(tree, 0, NULL, true);
5110 if ((tmpMask & regMask) == 0)
5112 printf("Predict reload of %s after : ", getRegName(genRegNumFromMask(tmpMask)));
5113 gtDispTree(tree, 0, NULL, true);
5117 /* In Codegen it will typically introduce a spill temp here */
5118 /* rather than relocating the register to a non trashed reg */
5119 rpPredictSpillCnt++;
5121 /* Remove it from the spillMask */
5122 spillMask &= ~tmpMask;
5127 * If the return registers in regMask conflicts with the lockedRegs
5128 * then we allocate extra registers for the reload of the conflicting
5131 * Set spillMask to the set of locked registers that have to be reloaded here.
5132 * reloadMask is set to the extra registers that are used to reload
5133 * the spilled lockedRegs.
5136 noway_assert(regMask != DUMMY_INIT(RBM_ILLEGAL));
5137 spillMask = lockedRegs & regMask;
5141 /* Remove the spillMask from regMask */
5142 regMask &= ~spillMask;
5144 regMaskTP reloadMask = RBM_NONE;
5147 /* Get an extra register to hold it */
5148 regMaskTP reloadReg = rpPredictRegPick(TYP_INT, PREDICT_REG, lockedRegs | regMask | reloadMask);
5152 printf("Predict reload into %s after : ", getRegName(genRegNumFromMask(reloadReg)));
5153 gtDispTree(tree, 0, NULL, true);
5156 reloadMask |= reloadReg;
5158 /* Remove it from the spillMask */
5159 spillMask &= ~genFindLowestBit(spillMask);
5162 /* Update regMask to use the reloadMask */
5163 regMask |= reloadMask;
5165 /* update the gtUsedRegs mask */
5166 tree->gtUsedRegs |= (regMaskSmall)regMask;
5169 regMaskTP regUse = tree->gtUsedRegs;
5170 regUse |= interferingRegs;
5172 if (!VarSetOps::IsEmpty(this, compCurLife))
5174 // Add interference between the current set of live variables and
5175 // the set of temporary registers need to evaluate the sub tree
5178 rpRecordRegIntf(regUse, compCurLife DEBUGARG("tmp use"));
5182 if (rpAsgVarNum != -1)
5184 // Add interference between the registers used (if any)
5185 // and the assignment target variable
5188 rpRecordRegIntf(regUse, VarSetOps::MakeSingleton(this, rpAsgVarNum) DEBUGARG("tgt var tmp use"));
5191 // Add a variable interference from rpAsgVarNum (i.e. the enregistered left hand
5192 // side of the assignment passed here using PREDICT_REG_VAR_Txx)
5193 // to the set of currently live variables. This new interference will prevent us
5194 // from using the register value used here for enregistering different live variable
5196 if (!VarSetOps::IsEmpty(this, compCurLife))
5198 rpRecordVarIntf(rpAsgVarNum, compCurLife DEBUGARG("asg tgt live conflict"));
5202 /* Do we need to resore the oldLastUseVars value */
5203 if (restoreLastUseVars)
5205 /* If we used a GT_ASG targeted register then we need to add
5206 * a variable interference between any new last use variables
5207 * and the GT_ASG targeted register
5209 if (!VarSetOps::Equal(this, rpLastUseVars, oldLastUseVars) && rpAsgVarNum != -1)
5211 rpRecordVarIntf(rpAsgVarNum, VarSetOps::Diff(this, rpLastUseVars, oldLastUseVars)
5212 DEBUGARG("asgn tgt last use conflict"));
5214 VarSetOps::Assign(this, rpLastUseVars, oldLastUseVars);
5220 #pragma warning(pop)
5223 #endif // LEGACY_BACKEND
5225 /****************************************************************************/
5226 /* Returns true when we must create an EBP frame
5227 This is used to force most managed methods to have EBP based frames
5228 which allows the ETW kernel stackwalker to walk the stacks of managed code
5229 this allows the kernel to perform light weight profiling
5231 bool Compiler::rpMustCreateEBPFrame(INDEBUG(const char** wbReason))
5233 bool result = false;
5235 const char* reason = nullptr;
5239 if (!result && (opts.MinOpts() || opts.compDbgCode))
5241 INDEBUG(reason = "Debug Code");
5244 if (!result && (info.compMethodInfo->ILCodeSize > DEFAULT_MAX_INLINE_SIZE))
5246 INDEBUG(reason = "IL Code Size");
5249 if (!result && (fgBBcount > 3))
5251 INDEBUG(reason = "BasicBlock Count");
5254 if (!result && fgHasLoops)
5256 INDEBUG(reason = "Method has Loops");
5259 if (!result && (optCallCount >= 2))
5261 INDEBUG(reason = "Call Count");
5264 if (!result && (optIndirectCallCount >= 1))
5266 INDEBUG(reason = "Indirect Call");
5269 #endif // ETW_EBP_FRAMED
5271 // VM wants to identify the containing frame of an InlinedCallFrame always
5272 // via the frame register never the stack register so we need a frame.
5273 if (!result && (optNativeCallCount != 0))
5275 INDEBUG(reason = "Uses PInvoke");
5279 #ifdef _TARGET_ARM64_
5280 // TODO-ARM64-NYI: This is temporary: force a frame pointer-based frame until genFnProlog can handle non-frame
5284 INDEBUG(reason = "Temporary ARM64 force frame pointer");
5287 #endif // _TARGET_ARM64_
5290 if ((result == true) && (wbReason != nullptr))
5299 #ifdef LEGACY_BACKEND // We don't use any of the old register allocator functions when LSRA is used instead.
5301 /*****************************************************************************
5303 * Predict which variables will be assigned to registers
5304 * This is x86 specific and only predicts the integer registers and
5305 * must be conservative, any register that is predicted to be enregister
5306 * must end up being enregistered.
5308 * The rpPredictTreeRegUse takes advantage of the LCL_VARS that are
5309 * predicted to be enregistered to minimize calls to rpPredictRegPick.
5314 #pragma warning(push)
5315 #pragma warning(disable : 21000) // Suppress PREFast warning about overly large function
5317 regMaskTP Compiler::rpPredictAssignRegVars(regMaskTP regAvail)
5321 if (rpPasses <= rpPassesPessimize)
5323 // Assume that we won't have to reverse EBP enregistration
5324 rpReverseEBPenreg = false;
5326 // Set the default rpFrameType based upon codeGen->isFramePointerRequired()
5327 if (codeGen->isFramePointerRequired() || codeGen->isFrameRequired())
5328 rpFrameType = FT_EBP_FRAME;
5330 rpFrameType = FT_ESP_FRAME;
5334 // If we are using FPBASE as the frame register, we cannot also use it for
5336 if (rpFrameType == FT_EBP_FRAME)
5338 regAvail &= ~RBM_FPBASE;
5340 #endif // !ETW_EBP_FRAMED
5343 rpPredictAssignMask = regAvail;
5345 raSetupArgMasks(&codeGen->intRegState);
5346 #if !FEATURE_STACK_FP_X87
5347 raSetupArgMasks(&codeGen->floatRegState);
5350 // If there is a secret stub param, it is also live in
5351 if (info.compPublishStubParam)
5353 codeGen->intRegState.rsCalleeRegArgMaskLiveIn |= RBM_SECRET_STUB_PARAM;
5356 if (regAvail == RBM_NONE)
5361 for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
5363 #if FEATURE_STACK_FP_X87
5364 if (!varDsc->IsFloatRegType())
5367 varDsc->lvRegNum = REG_STK;
5368 if (isRegPairType(varDsc->lvType))
5369 varDsc->lvOtherReg = REG_STK;
5377 printf("\nCompiler::rpPredictAssignRegVars pass #%d", rpPasses);
5378 printf("\n Available registers = ");
5379 dspRegMask(regAvail);
5384 if (regAvail == RBM_NONE)
5389 /* We cannot change the lvVarIndexes at this point, so we */
5390 /* can only re-order the existing set of tracked variables */
5391 /* Which will change the order in which we select the */
5392 /* locals for enregistering. */
5394 assert(lvaTrackedFixed); // We should have already set this to prevent us from adding any new tracked variables.
5396 // Should not be set unless optimizing
5397 noway_assert((lvaSortAgain == false) || (opts.MinOpts() == false));
5403 fgDebugCheckBBlist();
5406 /* Initialize the weighted count of variables that could have */
5407 /* been enregistered but weren't */
5408 unsigned refCntStk = 0; // sum of ref counts for all stack based variables
5409 unsigned refCntEBP = 0; // sum of ref counts for EBP enregistered variables
5410 unsigned refCntWtdEBP = 0; // sum of wtd ref counts for EBP enregistered variables
5412 unsigned refCntStkParam; // sum of ref counts for all stack based parameters
5413 unsigned refCntWtdStkDbl; // sum of wtd ref counts for stack based doubles
5415 #if FEATURE_STACK_FP_X87
5416 refCntStkParam = raCntStkParamDblStackFP;
5417 refCntWtdStkDbl = raCntWtdStkDblStackFP;
5418 refCntStk = raCntStkStackFP;
5421 refCntWtdStkDbl = 0;
5423 #endif // FEATURE_STACK_FP_X87
5425 #endif // DOUBLE_ALIGN
5427 /* Set of registers used to enregister variables in the predition */
5428 regMaskTP regUsed = RBM_NONE;
5430 /*-------------------------------------------------------------------------
5432 * Predict/Assign the enregistered locals in ref-count order
5436 VARSET_TP unprocessedVars(VarSetOps::MakeFull(this));
5438 unsigned FPRegVarLiveInCnt;
5439 FPRegVarLiveInCnt = 0; // How many enregistered doubles are live on entry to the method
5442 for (unsigned sortNum = 0; sortNum < lvaCount; sortNum++)
5444 bool notWorthy = false;
5448 regMaskTP regAvailForType;
5451 unsigned customVarOrderSize;
5452 regNumber customVarOrder[MAX_VAR_ORDER_SIZE];
5454 regNumber saveOtherReg;
5456 varDsc = lvaRefSorted[sortNum];
5458 #if FEATURE_STACK_FP_X87
5459 if (varTypeIsFloating(varDsc->TypeGet()))
5462 if (lvaIsFieldOfDependentlyPromotedStruct(varDsc))
5464 // Field local of a PROMOTION_TYPE_DEPENDENT struct should not
5465 // be en-registered.
5466 noway_assert(!varDsc->lvRegister);
5473 /* Check the set of invariant things that would prevent enregistration */
5475 /* Ignore the variable if it's not tracked */
5476 if (!varDsc->lvTracked)
5479 /* Get hold of the index and the interference mask for the variable */
5480 varIndex = varDsc->lvVarIndex;
5482 // Remove 'varIndex' from unprocessedVars
5483 VarSetOps::RemoveElemD(this, unprocessedVars, varIndex);
5485 // Skip the variable if it's marked as DoNotEnregister.
5487 if (varDsc->lvDoNotEnregister)
5490 /* TODO: For now if we have JMP all register args go to stack
5491 * TODO: Later consider extending the life of the argument or make a copy of it */
5493 if (compJmpOpUsed && varDsc->lvIsRegArg)
5496 /* Skip the variable if the ref count is zero */
5498 if (varDsc->lvRefCnt == 0)
5501 /* Ignore field of PROMOTION_TYPE_DEPENDENT type of promoted struct */
5503 if (lvaIsFieldOfDependentlyPromotedStruct(varDsc))
5508 /* Is the unweighted ref count too low to be interesting? */
5510 if (!varDsc->lvIsStructField && // We do encourage enregistering field locals.
5511 (varDsc->lvRefCnt <= 1))
5513 /* Sometimes it's useful to enregister a variable with only one use */
5514 /* arguments referenced in loops are one example */
5516 if (varDsc->lvIsParam && varDsc->lvRefCntWtd > BB_UNITY_WEIGHT)
5517 goto OK_TO_ENREGISTER;
5519 /* If the variable has a preferred register set it may be useful to put it there */
5520 if (varDsc->lvPrefReg && varDsc->lvIsRegArg)
5521 goto OK_TO_ENREGISTER;
5523 /* Keep going; the table is sorted by "weighted" ref count */
5529 if (varTypeIsFloating(varDsc->TypeGet()))
5531 regType = varDsc->TypeGet();
5532 regAvailForType = regAvail & RBM_ALLFLOAT;
5537 regAvailForType = regAvail & RBM_ALLINT;
5541 isDouble = (varDsc->TypeGet() == TYP_DOUBLE);
5545 regAvailForType &= RBM_DBL_REGS; // We must restrict the set to the double registers
5549 /* If we don't have any registers available then skip the enregistration attempt */
5550 if (regAvailForType == RBM_NONE)
5553 // On the pessimize passes don't even try to enregister LONGS
5554 if (isRegPairType(varDsc->lvType))
5556 if (rpPasses > rpPassesPessimize)
5558 else if (rpLostEnreg && (rpPasses == rpPassesPessimize))
5562 // Set of registers to avoid when performing register allocation
5563 avoidReg = RBM_NONE;
5565 if (!varDsc->lvIsRegArg)
5567 /* For local variables,
5568 * avoid the incoming arguments,
5569 * but only if you conflict with them */
5571 if (raAvoidArgRegMask != 0)
5574 LclVarDsc* argsEnd = lvaTable + info.compArgsCount;
5576 for (argDsc = lvaTable; argDsc < argsEnd; argDsc++)
5578 if (!argDsc->lvIsRegArg)
5581 bool isFloat = argDsc->IsFloatRegType();
5582 regNumber inArgReg = argDsc->lvArgReg;
5583 regMaskTP inArgBit = genRegMask(inArgReg);
5585 // Is this inArgReg in the raAvoidArgRegMask set?
5587 if (!(raAvoidArgRegMask & inArgBit))
5590 noway_assert(argDsc->lvIsParam);
5591 noway_assert(inArgBit & (isFloat ? RBM_FLTARG_REGS : RBM_ARG_REGS));
5593 unsigned locVarIndex = varDsc->lvVarIndex;
5594 unsigned argVarIndex = argDsc->lvVarIndex;
5596 /* Does this variable interfere with the arg variable ? */
5597 if (VarSetOps::IsMember(this, lvaVarIntf[locVarIndex], argVarIndex))
5599 noway_assert(VarSetOps::IsMember(this, lvaVarIntf[argVarIndex], locVarIndex));
5600 /* Yes, so try to avoid the incoming arg reg */
5601 avoidReg |= inArgBit;
5605 noway_assert(!VarSetOps::IsMember(this, lvaVarIntf[argVarIndex], locVarIndex));
5611 // Now we will try to predict which register the variable
5612 // could be enregistered in
5614 customVarOrderSize = MAX_VAR_ORDER_SIZE;
5616 raSetRegVarOrder(regType, customVarOrder, &customVarOrderSize, varDsc->lvPrefReg, avoidReg);
5619 saveOtherReg = DUMMY_INIT(REG_NA);
5621 for (regInx = 0; regInx < customVarOrderSize; regInx++)
5623 regNumber regNum = customVarOrder[regInx];
5624 regMaskTP regBits = genRegMask(regNum);
5626 /* Skip this register if it isn't available */
5627 if ((regAvailForType & regBits) == 0)
5630 /* Skip this register if it interferes with the variable */
5632 if (VarSetOps::IsMember(this, raLclRegIntf[regNum], varIndex))
5635 if (varTypeIsFloating(regType))
5640 regNumber regNext = REG_NEXT(regNum);
5641 regBits |= genRegMask(regNext);
5643 /* Skip if regNext interferes with the variable */
5644 if (VarSetOps::IsMember(this, raLclRegIntf[regNext], varIndex))
5650 bool firstUseOfReg = ((regBits & (regUsed | codeGen->regSet.rsGetModifiedRegsMask())) == 0);
5651 bool lessThanTwoRefWtd = (varDsc->lvRefCntWtd < (2 * BB_UNITY_WEIGHT));
5652 bool calleeSavedReg = ((regBits & RBM_CALLEE_SAVED) != 0);
5654 /* Skip this register if the weighted ref count is less than two
5655 and we are considering a unused callee saved register */
5657 if (lessThanTwoRefWtd && // less than two references (weighted)
5658 firstUseOfReg && // first use of this register
5659 calleeSavedReg) // callee saved register
5661 unsigned int totalRefCntWtd = varDsc->lvRefCntWtd;
5663 // psc is abbeviation for possibleSameColor
5664 VARSET_TP pscVarSet(VarSetOps::Diff(this, unprocessedVars, lvaVarIntf[varIndex]));
5666 VarSetOps::Iter pscIndexIter(this, pscVarSet);
5667 unsigned pscIndex = 0;
5668 while (pscIndexIter.NextElem(&pscIndex))
5670 LclVarDsc* pscVar = lvaTable + lvaTrackedToVarNum[pscIndex];
5671 totalRefCntWtd += pscVar->lvRefCntWtd;
5672 if (totalRefCntWtd > (2 * BB_UNITY_WEIGHT))
5676 if (totalRefCntWtd <= (2 * BB_UNITY_WEIGHT))
5679 continue; // not worth spilling a callee saved register
5681 // otherwise we will spill this callee saved registers,
5682 // because its uses when combined with the uses of
5683 // other yet to be processed candidates exceed our threshold.
5684 // totalRefCntWtd = totalRefCntWtd;
5687 /* Looks good - mark the variable as living in the register */
5689 if (isRegPairType(varDsc->lvType))
5691 if (firstHalf == false)
5693 /* Enregister the first half of the long */
5694 varDsc->lvRegNum = regNum;
5695 saveOtherReg = varDsc->lvOtherReg;
5696 varDsc->lvOtherReg = REG_STK;
5701 /* Ensure 'well-formed' register pairs */
5702 /* (those returned by gen[Pick|Grab]RegPair) */
5704 if (regNum < varDsc->lvRegNum)
5706 varDsc->lvOtherReg = varDsc->lvRegNum;
5707 varDsc->lvRegNum = regNum;
5711 varDsc->lvOtherReg = regNum;
5718 varDsc->lvRegNum = regNum;
5722 varDsc->lvOtherReg = REG_NEXT(regNum);
5727 if (regNum == REG_FPBASE)
5729 refCntEBP += varDsc->lvRefCnt;
5730 refCntWtdEBP += varDsc->lvRefCntWtd;
5732 if (varDsc->lvIsParam)
5734 refCntStkParam += varDsc->lvRefCnt;
5739 /* Record this register in the regUsed set */
5742 /* The register is now ineligible for all interfering variables */
5744 VarSetOps::UnionD(this, raLclRegIntf[regNum], lvaVarIntf[varIndex]);
5749 regNumber secondHalf = REG_NEXT(regNum);
5750 VarSetOps::Iter iter(this, lvaVarIntf[varIndex]);
5751 unsigned intfIndex = 0;
5752 while (iter.NextElem(&intfIndex))
5754 VarSetOps::AddElemD(this, raLclRegIntf[secondHalf], intfIndex);
5759 /* If a register argument, remove its incoming register
5760 * from the "avoid" list */
5762 if (varDsc->lvIsRegArg)
5764 raAvoidArgRegMask &= ~genRegMask(varDsc->lvArgReg);
5768 raAvoidArgRegMask &= ~genRegMask(REG_NEXT(varDsc->lvArgReg));
5773 /* A variable of TYP_LONG can take two registers */
5777 // Since we have successfully enregistered this variable it is
5778 // now time to move on and consider the next variable
5784 noway_assert(isRegPairType(varDsc->lvType));
5786 /* This TYP_LONG is partially enregistered */
5788 noway_assert(saveOtherReg != DUMMY_INIT(REG_NA));
5790 if (varDsc->lvDependReg && (saveOtherReg != REG_STK))
5795 raAddToStkPredict(varDsc->lvRefCntWtd);
5800 if (varDsc->lvDependReg)
5807 /* Weighted count of variables that could have been enregistered but weren't */
5808 raAddToStkPredict(varDsc->lvRefCntWtd);
5810 if (isRegPairType(varDsc->lvType) && (varDsc->lvOtherReg == REG_STK))
5811 raAddToStkPredict(varDsc->lvRefCntWtd);
5815 varDsc->lvRegister = false;
5817 varDsc->lvRegNum = REG_STK;
5818 if (isRegPairType(varDsc->lvType))
5819 varDsc->lvOtherReg = REG_STK;
5821 /* unweighted count of variables that were not enregistered */
5823 refCntStk += varDsc->lvRefCnt;
5826 if (varDsc->lvIsParam)
5828 refCntStkParam += varDsc->lvRefCnt;
5832 /* Is it a stack based double? */
5833 /* Note that double params are excluded since they can not be double aligned */
5834 if (varDsc->lvType == TYP_DOUBLE)
5836 refCntWtdStkDbl += varDsc->lvRefCntWtd;
5844 gtDispLclVar((unsigned)(varDsc - lvaTable));
5845 if (varDsc->lvTracked)
5846 printf("T%02u", varDsc->lvVarIndex);
5849 printf(" (refcnt=%2u,refwtd=%s) not enregistered", varDsc->lvRefCnt, refCntWtd2str(varDsc->lvRefCntWtd));
5850 if (varDsc->lvDoNotEnregister)
5851 printf(", do-not-enregister");
5859 varDsc->lvRegister = true;
5861 // Record the fact that we enregistered a stack arg when tail call is used.
5862 if (compJmpOpUsed && !varDsc->lvIsRegArg)
5864 rpMaskPInvokeEpilogIntf |= genRegMask(varDsc->lvRegNum);
5865 if (isRegPairType(varDsc->lvType))
5867 rpMaskPInvokeEpilogIntf |= genRegMask(varDsc->lvOtherReg);
5875 gtDispLclVar((unsigned)(varDsc - lvaTable));
5876 printf("T%02u (refcnt=%2u,refwtd=%s) predicted to be assigned to ", varIndex, varDsc->lvRefCnt,
5877 refCntWtd2str(varDsc->lvRefCntWtd));
5878 varDsc->PrintVarReg();
5882 printf(":%s", getRegName(varDsc->lvOtherReg));
5891 noway_assert(refCntEBP == 0);
5898 printf("; refCntStk = %u\n", refCntStk);
5900 printf("; refCntEBP = %u\n", refCntEBP);
5901 if (refCntWtdEBP > 0)
5902 printf("; refCntWtdEBP = %u\n", refCntWtdEBP);
5904 if (refCntStkParam > 0)
5905 printf("; refCntStkParam = %u\n", refCntStkParam);
5906 if (refCntWtdStkDbl > 0)
5907 printf("; refCntWtdStkDbl = %u\n", refCntWtdStkDbl);
5912 /* Determine how the EBP register should be used */
5913 CLANG_FORMAT_COMMENT_ANCHOR;
5917 if (!codeGen->isFramePointerRequired())
5919 noway_assert(getCanDoubleAlign() < COUNT_DOUBLE_ALIGN);
5922 First let us decide if we should use EBP to create a
5923 double-aligned frame, instead of enregistering variables
5926 if (getCanDoubleAlign() == MUST_DOUBLE_ALIGN)
5928 rpFrameType = FT_DOUBLE_ALIGN_FRAME;
5929 goto REVERSE_EBP_ENREG;
5932 if (getCanDoubleAlign() == CAN_DOUBLE_ALIGN && (refCntWtdStkDbl > 0))
5934 if (shouldDoubleAlign(refCntStk, refCntEBP, refCntWtdEBP, refCntStkParam, refCntWtdStkDbl))
5936 rpFrameType = FT_DOUBLE_ALIGN_FRAME;
5937 goto REVERSE_EBP_ENREG;
5942 #endif // DOUBLE_ALIGN
5944 if (!codeGen->isFramePointerRequired() && !codeGen->isFrameRequired())
5946 #ifdef _TARGET_XARCH_
5948 /* If we are using EBP to enregister variables then
5949 will we actually save bytes by setting up an EBP frame?
5951 Each stack reference is an extra byte of code if we use
5954 Here we measure the savings that we get by using EBP to
5955 enregister variables vs. the cost in code size that we
5956 pay when using an ESP based frame.
5958 We pay one byte of code for each refCntStk
5959 but we save one byte (or more) for each refCntEBP.
5961 Our savings are the elimination of a stack memory read/write.
5962 We use the loop weighted value of
5963 refCntWtdEBP * mem_access_weight (0, 3, 6)
5964 to represent this savings.
5967 // We also pay 5 extra bytes for the MOV EBP,ESP and LEA ESP,[EBP-0x10]
5968 // to set up an EBP frame in the prolog and epilog
5969 #define EBP_FRAME_SETUP_SIZE 5
5972 if (refCntStk > (refCntEBP + EBP_FRAME_SETUP_SIZE))
5974 unsigned bytesSaved = refCntStk - (refCntEBP + EBP_FRAME_SETUP_SIZE);
5975 unsigned mem_access_weight = 3;
5977 if (compCodeOpt() == SMALL_CODE)
5978 mem_access_weight = 0;
5979 else if (compCodeOpt() == FAST_CODE)
5980 mem_access_weight *= 2;
5982 if (bytesSaved > ((refCntWtdEBP * mem_access_weight) / BB_UNITY_WEIGHT))
5984 /* It's not be a good idea to use EBP in our predictions */
5985 CLANG_FORMAT_COMMENT_ANCHOR;
5987 if (verbose && (refCntEBP > 0))
5988 printf("; Predicting that it's not worth using EBP to enregister variables\n");
5990 rpFrameType = FT_EBP_FRAME;
5991 goto REVERSE_EBP_ENREG;
5994 #endif // _TARGET_XARCH_
5996 if ((rpFrameType == FT_NOT_SET) || (rpFrameType == FT_ESP_FRAME))
6001 if (rpMustCreateEBPCalled == false)
6003 rpMustCreateEBPCalled = true;
6004 if (rpMustCreateEBPFrame(INDEBUG(&reason)))
6008 printf("; Decided to create an EBP based frame for ETW stackwalking (%s)\n", reason);
6010 codeGen->setFrameRequired(true);
6012 rpFrameType = FT_EBP_FRAME;
6013 goto REVERSE_EBP_ENREG;
6023 noway_assert(rpFrameType != FT_ESP_FRAME);
6025 rpReverseEBPenreg = true;
6030 noway_assert(regUsed & RBM_FPBASE);
6032 regUsed &= ~RBM_FPBASE;
6034 /* variables that were enregistered in EBP become stack based variables */
6035 raAddToStkPredict(refCntWtdEBP);
6039 /* We're going to have to undo some predicted enregistered variables */
6040 for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
6042 /* Is this a register variable? */
6043 if (varDsc->lvRegNum != REG_STK)
6045 if (isRegPairType(varDsc->lvType))
6047 /* Only one can be EBP */
6048 if (varDsc->lvRegNum == REG_FPBASE || varDsc->lvOtherReg == REG_FPBASE)
6050 if (varDsc->lvRegNum == REG_FPBASE)
6051 varDsc->lvRegNum = varDsc->lvOtherReg;
6053 varDsc->lvOtherReg = REG_STK;
6055 if (varDsc->lvRegNum == REG_STK)
6056 varDsc->lvRegister = false;
6058 if (varDsc->lvDependReg)
6068 if ((varDsc->lvRegNum == REG_FPBASE) && (!varDsc->IsFloatRegType()))
6070 varDsc->lvRegNum = REG_STK;
6072 varDsc->lvRegister = false;
6074 if (varDsc->lvDependReg)
6080 printf("; reversing enregisteration of V%02u,T%02u (refcnt=%2u,refwtd=%4u%s)\n", lclNum,
6081 varDsc->lvVarIndex, varDsc->lvRefCnt, varDsc->lvRefCntWtd / 2,
6082 (varDsc->lvRefCntWtd & 1) ? ".5" : "");
6090 #endif // ETW_EBP_FRAMED
6095 for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
6097 /* Clear the lvDependReg flag for next iteration of the predictor */
6098 varDsc->lvDependReg = false;
6100 // If we set rpLostEnreg and this is the first pessimize pass
6101 // then reverse the enreg of all TYP_LONG
6102 if (rpLostEnreg && isRegPairType(varDsc->lvType) && (rpPasses == rpPassesPessimize))
6104 varDsc->lvRegNum = REG_STK;
6105 varDsc->lvOtherReg = REG_STK;
6110 if (verbose && raNewBlocks)
6112 printf("\nAdded FP register killing blocks:\n");
6113 fgDispBasicBlocks();
6117 noway_assert(rpFrameType != FT_NOT_SET);
6119 /* return the set of registers used to enregister variables */
6123 #pragma warning(pop)
6126 /*****************************************************************************
6128 * Predict register use for every tree in the function. Note that we do this
6129 * at different times (not to mention in a totally different way) for x86 vs
6132 void Compiler::rpPredictRegUse()
6139 // We might want to adjust the ref counts based on interference
6142 regMaskTP allAcceptableRegs = RBM_ALLINT;
6144 #if FEATURE_FP_REGALLOC
6145 allAcceptableRegs |= raConfigRestrictMaskFP();
6148 allAcceptableRegs &= ~codeGen->regSet.rsMaskResvd; // Remove any register reserved for special purposes
6150 /* For debuggable code, genJumpToThrowHlpBlk() generates an inline call
6151 to acdHelper(). This is done implicitly, without creating a GT_CALL
6152 node. Hence, this interference is be handled implicitly by
6153 restricting the registers used for enregistering variables */
6155 if (opts.compDbgCode)
6157 allAcceptableRegs &= RBM_CALLEE_SAVED;
6160 /* Compute the initial regmask to use for the first pass */
6161 regMaskTP regAvail = RBM_CALLEE_SAVED & allAcceptableRegs;
6164 #if CPU_USES_BLOCK_MOVE
6165 /* If we might need to generate a rep mov instruction */
6166 /* remove ESI and EDI */
6168 regAvail &= ~(RBM_ESI | RBM_EDI);
6172 /* If we using longs then we remove ESI to allow */
6173 /* ESI:EBX to be saved accross a call */
6175 regAvail &= ~(RBM_ESI);
6179 // For the first register allocation pass we don't want to color using r4
6180 // as we want to allow it to be used to color the internal temps instead
6181 // when r0,r1,r2,r3 are all in use.
6183 regAvail &= ~(RBM_R4);
6187 // We never have EBP available when ETW_EBP_FRAME is defined
6188 regAvail &= ~RBM_FPBASE;
6190 /* If a frame pointer is required then we remove EBP */
6191 if (codeGen->isFramePointerRequired() || codeGen->isFrameRequired())
6192 regAvail &= ~RBM_FPBASE;
6196 BOOL fJitNoRegLoc = JitConfig.JitNoRegLoc();
6198 regAvail = RBM_NONE;
6201 if ((opts.compFlags & CLFLG_REGVAR) == 0)
6202 regAvail = RBM_NONE;
6204 #if FEATURE_STACK_FP_X87
6205 VarSetOps::AssignNoCopy(this, optAllNonFPvars, VarSetOps::MakeEmpty(this));
6206 VarSetOps::AssignNoCopy(this, optAllFloatVars, VarSetOps::MakeEmpty(this));
6208 // Calculate the set of all tracked FP/non-FP variables
6209 // into optAllFloatVars and optAllNonFPvars
6214 for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
6216 /* Ignore the variable if it's not tracked */
6218 if (!varDsc->lvTracked)
6221 /* Get hold of the index and the interference mask for the variable */
6223 unsigned varNum = varDsc->lvVarIndex;
6225 /* add to the set of all tracked FP/non-FP variables */
6227 if (varDsc->IsFloatRegType())
6228 VarSetOps::AddElemD(this, optAllFloatVars, varNum);
6230 VarSetOps::AddElemD(this, optAllNonFPvars, varNum);
6234 for (unsigned i = 0; i < REG_COUNT; i++)
6236 VarSetOps::AssignNoCopy(this, raLclRegIntf[i], VarSetOps::MakeEmpty(this));
6238 for (unsigned i = 0; i < lvaTrackedCount; i++)
6240 VarSetOps::AssignNoCopy(this, lvaVarPref[i], VarSetOps::MakeEmpty(this));
6243 raNewBlocks = false;
6244 rpPredictAssignAgain = false;
6247 bool mustPredict = true;
6248 unsigned stmtNum = 0;
6249 unsigned oldStkPredict = DUMMY_INIT(~0);
6250 VARSET_TP oldLclRegIntf[REG_COUNT];
6252 for (unsigned i = 0; i < REG_COUNT; i++)
6254 VarSetOps::AssignNoCopy(this, oldLclRegIntf[i], VarSetOps::MakeEmpty(this));
6259 /* Assign registers to variables using the variable/register interference
6260 graph (raLclRegIntf[]) calculated in the previous pass */
6261 regUsed = rpPredictAssignRegVars(regAvail);
6263 mustPredict |= rpLostEnreg;
6266 // See if we previously reserved REG_R10 and try to make it available if we have a small frame now
6267 if ((rpPasses == 0) && ((codeGen->regSet.rsMaskResvd & RBM_OPT_RSVD) != 0) &&
6268 !compRsvdRegCheck(REGALLOC_FRAME_LAYOUT))
6270 // We can release our reservation on R10 and use it to color registers
6271 codeGen->regSet.rsMaskResvd &= ~RBM_OPT_RSVD;
6272 allAcceptableRegs |= RBM_OPT_RSVD;
6276 /* Is our new prediction good enough?? */
6279 /* For small methods (less than 12 stmts), we add a */
6280 /* extra pass if we are predicting the use of some */
6281 /* of the caller saved registers. */
6282 /* This fixes RAID perf bug 43440 VB Ackerman function */
6284 if ((rpPasses == 1) && (stmtNum <= 12) && (regUsed & RBM_CALLEE_SAVED))
6289 /* If every variable was fully enregistered then we're done */
6290 if (rpStkPredict == 0)
6293 // This was a successful prediction. Record it, in case it turns out to be the best one.
6294 rpRecordPrediction();
6298 noway_assert(oldStkPredict != (unsigned)DUMMY_INIT(~0));
6300 // Be careful about overflow
6301 unsigned highStkPredict = (rpStkPredict * 2 < rpStkPredict) ? ULONG_MAX : rpStkPredict * 2;
6302 if (oldStkPredict < highStkPredict)
6305 if (rpStkPredict < rpPasses * 8)
6308 if (rpPasses >= (rpPassesMax - 1))
6313 /* We will do another pass */;
6317 if (JitConfig.JitAssertOnMaxRAPasses())
6319 noway_assert(rpPasses < rpPassesMax &&
6320 "This may not a bug, but dev team should look and see what is happening");
6324 // The "64" here had been "VARSET_SZ". It is unclear why this number is connected with
6325 // the (max) size of a VARSET. We've eliminated this constant, so I left this as a constant. We hope
6326 // that we're phasing out this code, anyway, and this leaves the behavior the way that it was.
6327 if (rpPasses > (rpPassesMax - rpPassesPessimize) + 64)
6329 NO_WAY("we seem to be stuck in an infinite loop. breaking out");
6338 printf("\n; Another pass due to rpLostEnreg");
6340 printf("\n; Another pass due to rpAddedVarIntf");
6341 if ((rpPasses == 1) && rpPredictAssignAgain)
6342 printf("\n; Another pass due to rpPredictAssignAgain");
6344 printf("\n; Register predicting pass# %d\n", rpPasses + 1);
6348 /* Zero the variable/register interference graph */
6349 for (unsigned i = 0; i < REG_COUNT; i++)
6351 VarSetOps::ClearD(this, raLclRegIntf[i]);
6354 // if there are PInvoke calls and compLvFrameListRoot is enregistered,
6355 // it must not be in a register trashed by the callee
6356 if (info.compLvFrameListRoot != BAD_VAR_NUM)
6358 assert(!opts.ShouldUsePInvokeHelpers());
6359 noway_assert(info.compLvFrameListRoot < lvaCount);
6361 LclVarDsc* pinvokeVarDsc = &lvaTable[info.compLvFrameListRoot];
6363 if (pinvokeVarDsc->lvTracked)
6365 rpRecordRegIntf(RBM_CALLEE_TRASH, VarSetOps::MakeSingleton(this, pinvokeVarDsc->lvVarIndex)
6366 DEBUGARG("compLvFrameListRoot"));
6368 // We would prefer to have this be enregister in the PINVOKE_TCB register
6369 pinvokeVarDsc->addPrefReg(RBM_PINVOKE_TCB, this);
6372 // If we're using a single return block, the p/invoke epilog code trashes ESI and EDI (in the
6373 // worst case). Make sure that the return value compiler temp that we create for the single
6374 // return block knows about this interference.
6375 if (genReturnLocal != BAD_VAR_NUM)
6377 noway_assert(genReturnBB);
6378 LclVarDsc* localTmp = &lvaTable[genReturnLocal];
6379 if (localTmp->lvTracked)
6381 rpRecordRegIntf(RBM_PINVOKE_TCB | RBM_PINVOKE_FRAME,
6382 VarSetOps::MakeSingleton(this, localTmp->lvVarIndex) DEBUGARG("genReturnLocal"));
6388 if (compFloatingPointUsed)
6390 bool hasMustInitFloat = false;
6392 // if we have any must-init floating point LclVars then we will add register interferences
6393 // for the arguments with RBM_SCRATCH
6394 // this is so that if we need to reset the initReg to REG_SCRATCH in Compiler::genFnProlog()
6395 // we won't home the arguments into REG_SCRATCH
6400 for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
6402 if (varDsc->lvMustInit && varTypeIsFloating(varDsc->TypeGet()))
6404 hasMustInitFloat = true;
6409 if (hasMustInitFloat)
6411 for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
6413 // If is an incoming argument, that is tracked and not floating-point
6414 if (varDsc->lvIsParam && varDsc->lvTracked && !varTypeIsFloating(varDsc->TypeGet()))
6416 rpRecordRegIntf(RBM_SCRATCH, VarSetOps::MakeSingleton(this, varDsc->lvVarIndex)
6417 DEBUGARG("arg home with must-init fp"));
6425 rpAddedVarIntf = false;
6426 rpLostEnreg = false;
6428 /* Walk the basic blocks and predict reg use for each tree */
6430 for (BasicBlock* block = fgFirstBB; block != NULL; block = block->bbNext)
6434 compCurLifeTree = NULL;
6435 VarSetOps::Assign(this, compCurLife, block->bbLiveIn);
6439 for (stmt = block->FirstNonPhiDef(); stmt != NULL; stmt = stmt->gtNext)
6441 noway_assert(stmt->gtOper == GT_STMT);
6443 rpPredictSpillCnt = 0;
6444 VarSetOps::AssignNoCopy(this, rpLastUseVars, VarSetOps::MakeEmpty(this));
6445 VarSetOps::AssignNoCopy(this, rpUseInPlace, VarSetOps::MakeEmpty(this));
6447 GenTreePtr tree = stmt->gtStmt.gtStmtExpr;
6452 printf("\nRegister predicting BB%02u, stmt %d\n", block->bbNum, stmtNum);
6457 rpPredictTreeRegUse(tree, PREDICT_NONE, RBM_NONE, RBM_NONE);
6459 noway_assert(rpAsgVarNum == -1);
6461 if (rpPredictSpillCnt > tmpIntSpillMax)
6462 tmpIntSpillMax = rpPredictSpillCnt;
6467 /* Decide whether we need to set mustPredict */
6468 mustPredict = false;
6471 // The spill count may be now high enough that we now need to reserve r10. If this is the case, we'll need to
6472 // reserve r10, and if it was used, throw out the last prediction and repredict.
6473 if (((codeGen->regSet.rsMaskResvd & RBM_OPT_RSVD) == 0) && compRsvdRegCheck(REGALLOC_FRAME_LAYOUT))
6475 codeGen->regSet.rsMaskResvd |= RBM_OPT_RSVD;
6476 allAcceptableRegs &= ~RBM_OPT_RSVD;
6477 if ((regUsed & RBM_OPT_RSVD) != 0)
6480 rpBestRecordedPrediction = nullptr;
6496 if ((opts.compFlags & CLFLG_REGVAR) == 0)
6499 if (rpPredictAssignAgain)
6507 /* Calculate the new value to use for regAvail */
6509 regAvail = allAcceptableRegs;
6511 /* If a frame pointer is required then we remove EBP */
6512 if (codeGen->isFramePointerRequired() || codeGen->isFrameRequired())
6513 regAvail &= ~RBM_FPBASE;
6516 // We never have EBP available when ETW_EBP_FRAME is defined
6517 regAvail &= ~RBM_FPBASE;
6520 // If we have done n-passes then we must continue to pessimize the
6521 // interference graph by or-ing the interferences from the previous pass
6523 if (rpPasses > rpPassesPessimize)
6525 for (unsigned regInx = 0; regInx < REG_COUNT; regInx++)
6526 VarSetOps::UnionD(this, raLclRegIntf[regInx], oldLclRegIntf[regInx]);
6528 /* If we reverse an EBP enregistration then keep it that way */
6529 if (rpReverseEBPenreg)
6530 regAvail &= ~RBM_FPBASE;
6538 /* Save the old variable/register interference graph */
6539 for (unsigned i = 0; i < REG_COUNT; i++)
6541 VarSetOps::Assign(this, oldLclRegIntf[i], raLclRegIntf[i]);
6543 oldStkPredict = rpStkPredict;
6544 } // end of while (true)
6548 // If we recorded a better feasible allocation than we ended up with, go back to using it.
6549 rpUseRecordedPredictionIfBetter();
6552 codeGen->setDoubleAlign(false);
6555 switch (rpFrameType)
6558 noway_assert(!"rpFrameType not set correctly!");
6561 noway_assert(!codeGen->isFramePointerRequired());
6562 noway_assert(!codeGen->isFrameRequired());
6563 codeGen->setFramePointerUsed(false);
6566 noway_assert((regUsed & RBM_FPBASE) == 0);
6567 codeGen->setFramePointerUsed(true);
6570 case FT_DOUBLE_ALIGN_FRAME:
6571 noway_assert((regUsed & RBM_FPBASE) == 0);
6572 noway_assert(!codeGen->isFramePointerRequired());
6573 codeGen->setFramePointerUsed(false);
6574 codeGen->setDoubleAlign(true);
6579 /* Record the set of registers that we need */
6580 codeGen->regSet.rsClearRegsModified();
6581 if (regUsed != RBM_NONE)
6583 codeGen->regSet.rsSetRegsModified(regUsed);
6586 /* We need genFullPtrRegMap if :
6587 * The method is fully interruptible, or
6588 * We are generating an EBP-less frame (for stack-pointer deltas)
6591 genFullPtrRegMap = (genInterruptible || !codeGen->isFramePointerUsed());
6597 printf("# rpPasses was %u for %s\n", rpPasses, info.compFullName);
6598 printf(" rpStkPredict was %u\n", rpStkPredict);
6601 rpRegAllocDone = true;
6604 #endif // LEGACY_BACKEND
6606 /*****************************************************************************
6608 * Mark all variables as to whether they live on the stack frame
6609 * (part or whole), and if so what the base is (FP or SP).
6612 void Compiler::raMarkStkVars()
6617 for (lclNum = 0, varDsc = lvaTable; lclNum < lvaCount; lclNum++, varDsc++)
6619 // For RyuJIT, lvOnFrame is set by LSRA, except in the case of zero-ref, which is set below.
6620 CLANG_FORMAT_COMMENT_ANCHOR;
6622 #ifdef LEGACY_BACKEND
6623 varDsc->lvOnFrame = false;
6624 #endif // LEGACY_BACKEND
6626 if (lvaIsFieldOfDependentlyPromotedStruct(varDsc))
6628 noway_assert(!varDsc->lvRegister);
6632 /* Fully enregistered variables don't need any frame space */
6634 if (varDsc->lvRegister)
6636 if (!isRegPairType(varDsc->TypeGet()))
6641 /* For "large" variables make sure both halves are enregistered */
6643 if (varDsc->lvRegNum != REG_STK && varDsc->lvOtherReg != REG_STK)
6648 /* Unused variables typically don't get any frame space */
6649 else if (varDsc->lvRefCnt == 0)
6651 bool needSlot = false;
6653 bool stkFixedArgInVarArgs =
6654 info.compIsVarArgs && varDsc->lvIsParam && !varDsc->lvIsRegArg && lclNum != lvaVarargsHandleArg;
6656 // If its address has been exposed, ignore lvRefCnt. However, exclude
6657 // fixed arguments in varargs method as lvOnFrame shouldn't be set
6658 // for them as we don't want to explicitly report them to GC.
6660 if (!stkFixedArgInVarArgs)
6662 needSlot |= varDsc->lvAddrExposed;
6665 #if FEATURE_FIXED_OUT_ARGS
6667 /* Is this the dummy variable representing GT_LCLBLK ? */
6668 needSlot |= (lclNum == lvaOutgoingArgSpaceVar);
6670 #endif // FEATURE_FIXED_OUT_ARGS
6673 /* For debugging, note that we have to reserve space even for
6674 unused variables if they are ever in scope. However, this is not
6675 an issue as fgExtendDbgLifetimes() adds an initialization and
6676 variables in scope will not have a zero ref-cnt.
6678 if (opts.compDbgCode && !varDsc->lvIsParam && varDsc->lvTracked)
6680 for (unsigned scopeNum = 0; scopeNum < info.compVarScopesCount; scopeNum++)
6682 noway_assert(info.compVarScopes[scopeNum].vsdVarNum != lclNum);
6687 For Debug Code, we have to reserve space even if the variable is never
6688 in scope. We will also need to initialize it if it is a GC var.
6689 So we set lvMustInit and artifically bump up the ref-cnt.
6692 if (opts.compDbgCode && !stkFixedArgInVarArgs && lclNum < info.compLocalsCount)
6696 if (lvaTypeIsGC(lclNum))
6698 varDsc->lvRefCnt = 1;
6701 if (!varDsc->lvIsParam)
6703 varDsc->lvMustInit = true;
6707 #ifndef LEGACY_BACKEND
6708 varDsc->lvOnFrame = needSlot;
6709 #endif // !LEGACY_BACKEND
6712 /* Clear the lvMustInit flag in case it is set */
6713 varDsc->lvMustInit = false;
6719 #ifndef LEGACY_BACKEND
6720 if (!varDsc->lvOnFrame)
6724 #endif // !LEGACY_BACKEND
6727 /* The variable (or part of it) lives on the stack frame */
6729 noway_assert((varDsc->lvType != TYP_UNDEF) && (varDsc->lvType != TYP_VOID) && (varDsc->lvType != TYP_UNKNOWN));
6730 #if FEATURE_FIXED_OUT_ARGS
6731 noway_assert((lclNum == lvaOutgoingArgSpaceVar) || lvaLclSize(lclNum) != 0);
6732 #else // FEATURE_FIXED_OUT_ARGS
6733 noway_assert(lvaLclSize(lclNum) != 0);
6734 #endif // FEATURE_FIXED_OUT_ARGS
6736 varDsc->lvOnFrame = true; // Our prediction is that the final home for this local variable will be in the
6740 varDsc->lvFramePointerBased = codeGen->isFramePointerUsed();
6744 if (codeGen->doDoubleAlign())
6746 noway_assert(codeGen->isFramePointerUsed() == false);
6748 /* All arguments are off of EBP with double-aligned frames */
6750 if (varDsc->lvIsParam && !varDsc->lvIsRegArg)
6752 varDsc->lvFramePointerBased = true;
6758 /* Some basic checks */
6760 // It must be in a register, on frame, or have zero references.
6762 noway_assert(varDsc->lvIsInReg() || varDsc->lvOnFrame || varDsc->lvRefCnt == 0);
6764 #ifndef LEGACY_BACKEND
6765 // We can't have both lvRegister and lvOnFrame for RyuJIT
6766 noway_assert(!varDsc->lvRegister || !varDsc->lvOnFrame);
6767 #else // LEGACY_BACKEND
6769 /* If both lvRegister and lvOnFrame are set, it must be partially enregistered */
6770 noway_assert(!varDsc->lvRegister || !varDsc->lvOnFrame ||
6771 (varDsc->lvType == TYP_LONG && varDsc->lvOtherReg == REG_STK));
6772 #endif // LEGACY_BACKEND
6776 // For varargs functions, there should be no direct references to
6777 // parameter variables except for 'this' (because these were morphed
6778 // in the importer) and the 'arglist' parameter (which is not a GC
6779 // pointer). and the return buffer argument (if we are returning a
6781 // This is important because we don't want to try to report them
6782 // to the GC, as the frame offsets in these local varables would
6785 if (varDsc->lvIsParam && raIsVarargsStackArg(lclNum))
6787 if (!varDsc->lvPromoted && !varDsc->lvIsStructField)
6789 noway_assert(varDsc->lvRefCnt == 0 && !varDsc->lvRegister && !varDsc->lvOnFrame);
6796 #ifdef LEGACY_BACKEND
6797 void Compiler::rpRecordPrediction()
6799 if (rpBestRecordedPrediction == NULL || rpStkPredict < rpBestRecordedStkPredict)
6801 if (rpBestRecordedPrediction == NULL)
6803 rpBestRecordedPrediction =
6804 reinterpret_cast<VarRegPrediction*>(compGetMemArray(lvaCount, sizeof(VarRegPrediction)));
6806 for (unsigned k = 0; k < lvaCount; k++)
6808 rpBestRecordedPrediction[k].m_isEnregistered = lvaTable[k].lvRegister;
6809 rpBestRecordedPrediction[k].m_regNum = (regNumberSmall)lvaTable[k].GetRegNum();
6810 rpBestRecordedPrediction[k].m_otherReg = (regNumberSmall)lvaTable[k].GetOtherReg();
6812 rpBestRecordedStkPredict = rpStkPredict;
6813 JITDUMP("Recorded a feasible reg prediction with weighted stack use count %d.\n", rpBestRecordedStkPredict);
6817 void Compiler::rpUseRecordedPredictionIfBetter()
6819 JITDUMP("rpStkPredict is %d; previous feasible reg prediction is %d.\n", rpStkPredict,
6820 rpBestRecordedPrediction != NULL ? rpBestRecordedStkPredict : 0);
6821 if (rpBestRecordedPrediction != NULL && rpStkPredict > rpBestRecordedStkPredict)
6823 JITDUMP("Reverting to a previously-recorded feasible reg prediction with weighted stack use count %d.\n",
6824 rpBestRecordedStkPredict);
6826 for (unsigned k = 0; k < lvaCount; k++)
6828 lvaTable[k].lvRegister = rpBestRecordedPrediction[k].m_isEnregistered;
6829 lvaTable[k].SetRegNum(static_cast<regNumber>(rpBestRecordedPrediction[k].m_regNum));
6830 lvaTable[k].SetOtherReg(static_cast<regNumber>(rpBestRecordedPrediction[k].m_otherReg));
6834 #endif // LEGACY_BACKEND