1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
4 /*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
5 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
9 XX Represents the method data we are currently JIT-compiling. XX
10 XX An instance of this class is created for every method we JIT. XX
11 XX This contains all the info needed for the method. So allocating a XX
12 XX a new instance per method makes it thread-safe. XX
13 XX It should be used to do all the memory management for the compiler run. XX
15 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
16 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
19 /*****************************************************************************/
22 /*****************************************************************************/
28 #include "jithashtable.h"
30 #include "debuginfo.h"
38 #include "cycletimer.h"
40 #include "arraystack.h"
42 #include "jitexpandarray.h"
43 #include "tinyarray.h"
45 #include "jittelemetry.h"
46 #include "namedintrinsiclist.h"
51 #include "codegeninterface.h"
53 #include "jitgcinfo.h"
55 #if DUMP_GC_TABLES && defined(JIT32_GCENCODER)
61 // Forward declaration
63 inline var_types genActualType(T value);
65 #include "hwintrinsic.h"
67 #include "simdashwintrinsic.h"
69 /*****************************************************************************
70 * Forward declarations
73 struct InfoHdr; // defined in GCInfo.h
74 struct escapeMapping_t; // defined in fgdiagnostic.cpp
75 class emitter; // defined in emit.h
76 struct ShadowParamVarInfo; // defined in GSChecks.cpp
77 struct InitVarDscInfo; // defined in registerargconvention.h
78 class FgStack; // defined in fgbasic.cpp
79 class Instrumentor; // defined in fgprofile.cpp
80 class SpanningTreeVisitor; // defined in fgprofile.cpp
81 class CSE_DataFlow; // defined in OptCSE.cpp
82 class OptBoolsDsc; // defined in optimizer.cpp
83 struct RelopImplicationInfo; // defined in redundantbranchopts.cpp
84 struct JumpThreadInfo; // defined in redundantbranchopts.cpp
85 class ProfileSynthesis; // defined in profilesynthesis.h
90 class Lowering; // defined in lower.h
92 // The following are defined in this file, Compiler.h
96 /*****************************************************************************
102 /*****************************************************************************/
105 // Declare global operator new overloads that use the compiler's arena allocator
108 void* operator new(size_t n, Compiler* context, CompMemKind cmk);
109 void* operator new[](size_t n, Compiler* context, CompMemKind cmk);
111 // Requires the definitions of "operator new" so including "LoopCloning.h" after the definitions.
112 #include "loopcloning.h"
114 /*****************************************************************************/
116 /* This is included here and not earlier as it needs the definition of "CSE"
117 * which is defined in the section above */
119 /*****************************************************************************/
121 unsigned genLog2(unsigned value);
122 unsigned genLog2(unsigned __int64 value);
124 unsigned ReinterpretHexAsDecimal(unsigned in);
126 /*****************************************************************************/
128 const unsigned FLG_CCTOR = (CORINFO_FLG_CONSTRUCTOR | CORINFO_FLG_STATIC);
131 const int BAD_STK_OFFS = 0xBAADF00D; // for LclVarDsc::lvStkOffs
134 //------------------------------------------------------------------------
135 // HFA info shared by LclVarDsc and CallArgABIInformation
136 //------------------------------------------------------------------------
137 inline bool IsHfa(CorInfoHFAElemType kind)
139 return kind != CORINFO_HFA_ELEM_NONE;
141 inline var_types HfaTypeFromElemKind(CorInfoHFAElemType kind)
145 case CORINFO_HFA_ELEM_FLOAT:
147 case CORINFO_HFA_ELEM_DOUBLE:
150 case CORINFO_HFA_ELEM_VECTOR64:
152 case CORINFO_HFA_ELEM_VECTOR128:
155 case CORINFO_HFA_ELEM_NONE:
158 assert(!"Invalid HfaElemKind");
162 inline CorInfoHFAElemType HfaElemKindFromType(var_types type)
167 return CORINFO_HFA_ELEM_FLOAT;
169 return CORINFO_HFA_ELEM_DOUBLE;
172 return CORINFO_HFA_ELEM_VECTOR64;
174 return CORINFO_HFA_ELEM_VECTOR128;
177 return CORINFO_HFA_ELEM_NONE;
179 assert(!"Invalid HFA Type");
180 return CORINFO_HFA_ELEM_NONE;
184 // The following holds the Local var info (scope information)
185 typedef const char* VarName; // Actual ASCII string
188 unsigned vsdVarNum; // (remapped) LclVarDsc number
189 unsigned vsdLVnum; // 'which' in eeGetLVinfo().
190 // Also, it is the index of this entry in the info.compVarScopes array,
191 // which is useful since the array is also accessed via the
192 // compEnterScopeList and compExitScopeList sorted arrays.
194 IL_OFFSET vsdLifeBeg; // instr offset of beg of life
195 IL_OFFSET vsdLifeEnd; // instr offset of end of life
198 VarName vsdName; // name of the var
202 // This class stores information associated with a LclVar SSA definition.
205 // The basic block where the definition occurs. Definitions of uninitialized variables
206 // are considered to occur at the start of the first basic block (fgFirstBB).
208 // TODO-Cleanup: In the case of uninitialized variables the block is set to nullptr by
209 // SsaBuilder and changed to fgFirstBB during value numbering. It would be useful to
210 // investigate and perhaps eliminate this rather unexpected behavior.
211 BasicBlock* m_block = nullptr;
212 // The store node that generates the definition, or nullptr for definitions
213 // of uninitialized variables.
214 GenTreeLclVarCommon* m_defNode = nullptr;
215 // The SSA number associated with the previous definition for partial (GTF_USEASG) defs.
216 unsigned m_useDefSsaNum = SsaConfig::RESERVED_SSA_NUM;
217 // Number of uses of this SSA def (may be an over-estimate).
218 // May not be accurate for for promoted fields.
219 unsigned short m_numUses = 0;
220 // True if there may be phi args uses of this def
221 // May not be accurate for for promoted fields.
222 // (false implies all uses are non-phi).
223 bool m_hasPhiUse = false;
224 // True if there may be uses of the def in a different block.
225 // May not be accurate for for promoted fields.
226 bool m_hasGlobalUse = false;
233 LclSsaVarDsc(BasicBlock* block) : m_block(block)
237 LclSsaVarDsc(BasicBlock* block, GenTreeLclVarCommon* defNode) : m_block(block)
242 BasicBlock* GetBlock() const
247 void SetBlock(BasicBlock* block)
252 GenTreeLclVarCommon* GetDefNode() const
257 void SetDefNode(GenTreeLclVarCommon* defNode)
259 assert((defNode == nullptr) || defNode->OperIsLocalStore());
263 unsigned GetUseDefSsaNum() const
265 return m_useDefSsaNum;
268 void SetUseDefSsaNum(unsigned ssaNum)
270 m_useDefSsaNum = ssaNum;
273 unsigned GetNumUses() const
278 void AddUse(BasicBlock* block)
280 if (block != m_block)
282 m_hasGlobalUse = true;
285 if (m_numUses < USHRT_MAX)
291 void AddPhiUse(BasicBlock* block)
297 bool HasPhiUse() const
302 bool HasGlobalUse() const
304 return m_hasGlobalUse;
307 ValueNumPair m_vnPair;
310 // This class stores information associated with a memory SSA definition.
314 ValueNumPair m_vnPair;
317 //------------------------------------------------------------------------
318 // SsaDefArray: A resizable array of SSA definitions.
320 // Unlike an ordinary resizable array implementation, this allows only element
321 // addition (by calling AllocSsaNum) and has special handling for RESERVED_SSA_NUM
322 // (basically it's a 1-based array). The array doesn't impose any particular
323 // requirements on the elements it stores and AllocSsaNum forwards its arguments
324 // to the array element constructor, this way the array supports both LclSsaVarDsc
325 // and SsaMemDef elements.
327 template <typename T>
331 unsigned m_arraySize;
334 static_assert_no_msg(SsaConfig::RESERVED_SSA_NUM == 0);
335 static_assert_no_msg(SsaConfig::FIRST_SSA_NUM == 1);
337 // Get the minimum valid SSA number.
338 unsigned GetMinSsaNum() const
340 return SsaConfig::FIRST_SSA_NUM;
343 // Increase (double) the size of the array.
344 void GrowArray(CompAllocator alloc)
346 unsigned oldSize = m_arraySize;
347 unsigned newSize = max(2, oldSize * 2);
349 T* newArray = alloc.allocate<T>(newSize);
351 for (unsigned i = 0; i < oldSize; i++)
353 newArray[i] = m_array[i];
357 m_arraySize = newSize;
361 // Construct an empty SsaDefArray.
362 SsaDefArray() : m_array(nullptr), m_arraySize(0), m_count(0)
366 // Reset the array (used only if the SSA form is reconstructed).
372 // Allocate a new SSA number (starting with SsaConfig::FIRST_SSA_NUM).
373 template <class... Args>
374 unsigned AllocSsaNum(CompAllocator alloc, Args&&... args)
376 if (m_count == m_arraySize)
381 unsigned ssaNum = GetMinSsaNum() + m_count;
382 m_array[m_count++] = T(std::forward<Args>(args)...);
384 // Ensure that the first SSA number we allocate is SsaConfig::FIRST_SSA_NUM
385 assert((ssaNum == SsaConfig::FIRST_SSA_NUM) || (m_count > 1));
390 // Get the number of SSA definitions in the array.
391 unsigned GetCount() const
396 // Get a pointer to the SSA definition at the specified index.
397 T* GetSsaDefByIndex(unsigned index) const
399 assert(index < m_count);
400 return &m_array[index];
403 // Check if the specified SSA number is valid.
404 bool IsValidSsaNum(unsigned ssaNum) const
406 return (GetMinSsaNum() <= ssaNum) && (ssaNum < (GetMinSsaNum() + m_count));
409 // Get a pointer to the SSA definition associated with the specified SSA number.
410 T* GetSsaDef(unsigned ssaNum) const
412 assert(ssaNum != SsaConfig::RESERVED_SSA_NUM);
413 return GetSsaDefByIndex(ssaNum - GetMinSsaNum());
416 // Get an SSA number associated with the specified SSA def (that must be in this array).
417 unsigned GetSsaNum(T* ssaDef) const
419 assert((m_array <= ssaDef) && (ssaDef < &m_array[m_count]));
420 return GetMinSsaNum() + static_cast<unsigned>(ssaDef - &m_array[0]);
426 RCS_INVALID, // not valid to get/set ref counts
427 RCS_EARLY, // early counts for struct promotion and struct passing
428 RCS_NORMAL, // normal ref counts (from lvaMarkRefs onward)
432 // Reasons why we can't enregister a local.
433 enum class DoNotEnregisterReason
436 AddrExposed, // the address of this local is exposed.
437 DontEnregStructs, // struct enregistration is disabled.
438 NotRegSizeStruct, // the struct size does not much any register size, usually the struct size is too big.
439 LocalField, // the local is accessed with LCL_FLD, note we can do it not only for struct locals.
441 LiveInOutOfHandler, // the local is alive in and out of exception handler and not single def.
442 BlockOp, // Is read or written via a block operation.
443 IsStructArg, // Is a struct passed as an argument in a way that requires a stack location.
444 DepField, // It is a field of a dependently promoted struct
445 NoRegVars, // opts.compFlags & CLFLG_REGVAR is not set
446 MinOptsGC, // It is a GC Ref and we are compiling MinOpts
447 #if !defined(TARGET_64BIT)
448 LongParamField, // It is a decomposed field of a long parameter.
450 #ifdef JIT32_GCENCODER
453 LclAddrNode, // the local is accessed with LCL_ADDR_VAR/FLD.
455 StoreBlkSrc, // the local is used as STORE_BLK source.
456 SwizzleArg, // the local is passed using LCL_FLD as another type.
457 BlockOpRet, // the struct is returned and it promoted or there is a cast.
458 ReturnSpCheck, // the local is used to do SP check on return from function
459 CallSpCheck, // the local is used to do SP check on every call
460 SimdUserForcesDep, // a promoted struct was used by a SIMD/HWI node; it must be dependently promoted
461 HiddenBufferStructArg // the argument is a hidden return buffer passed to a method.
464 enum class AddressExposedReason
467 PARENT_EXPOSED, // This is a promoted field but the parent is exposed.
468 TOO_CONSERVATIVE, // Were marked as exposed to be conservative, fix these places.
469 ESCAPE_ADDRESS, // The address is escaping, for example, passed as call argument.
470 WIDE_INDIR, // We access via indirection with wider type.
471 OSR_EXPOSED, // It was exposed in the original method, osr has to repeat it.
472 STRESS_LCL_FLD, // Stress mode replaces localVar with localFld and makes them addrExposed.
473 DISPATCH_RET_BUF, // Caller return buffer dispatch.
474 STRESS_POISON_IMPLICIT_BYREFS, // This is an implicit byref we want to poison.
475 EXTERNALLY_VISIBLE_IMPLICITLY, // Local is visible externally without explicit escape in JIT IR.
476 // For example because it is used by GC or is the outgoing arg area
477 // that belongs to callees.
485 // The constructor. Most things can just be zero'ed.
487 // Initialize the ArgRegs to REG_STK.
488 // Morph will update if this local is passed in a register.
491 #if FEATURE_MULTIREG_ARGS
492 , _lvOtherArgReg(REG_STK)
493 #endif // FEATURE_MULTIREG_ARGS
494 , lvClassHnd(NO_CLASS_HANDLE)
499 // note this only packs because var_types is a typedef of unsigned char
500 var_types lvType : 5; // TYP_INT/LONG/FLOAT/DOUBLE/REF
502 unsigned char lvIsParam : 1; // is this a parameter?
503 unsigned char lvIsRegArg : 1; // is this an argument that was passed by register?
504 unsigned char lvFramePointerBased : 1; // 0 = off of REG_SPBASE (e.g., ESP), 1 = off of REG_FPBASE (e.g., EBP)
506 unsigned char lvOnFrame : 1; // (part of) the variable lives on the frame
507 unsigned char lvRegister : 1; // assigned to live in a register? For RyuJIT backend, this is only set if the
508 // variable is in the same register for the entire function.
509 unsigned char lvTracked : 1; // is this a tracked variable?
510 bool lvTrackedNonStruct()
512 return lvTracked && lvType != TYP_STRUCT;
515 unsigned char lvTrackedWithoutIndex : 1; // Tracked but has no lvVarIndex (i.e. only valid GTF_VAR_DEATH flags, used
516 // by physical promotion)
518 unsigned char lvPinned : 1; // is this a pinned variable?
520 unsigned char lvMustInit : 1; // must be initialized
523 bool m_addrExposed : 1; // The address of this variable is "exposed" -- passed as an argument, stored in a
524 // global location, etc.
525 // We cannot reason reliably about the value of the variable.
527 unsigned char lvDoNotEnregister : 1; // Do not enregister this variable.
528 unsigned char lvFieldAccessed : 1; // The var is a struct local, and a field of the variable is accessed. Affects
530 unsigned char lvLiveInOutOfHndlr : 1; // The variable is live in or out of an exception handler, and therefore must
531 // be on the stack (at least at those boundaries.)
533 unsigned char lvInSsa : 1; // The variable is in SSA form (set by SsaBuilder)
534 unsigned char lvIsCSE : 1; // Indicates if this LclVar is a CSE variable.
535 unsigned char lvHasLdAddrOp : 1; // has ldloca or ldarga opcode on this local.
537 unsigned char lvHasILStoreOp : 1; // there is at least one STLOC or STARG on this local
538 unsigned char lvHasMultipleILStoreOp : 1; // there is more than one STLOC on this local
540 unsigned char lvIsTemp : 1; // Short-lifetime compiler temp
542 #if FEATURE_IMPLICIT_BYREFS
543 // Set if the argument is an implicit byref.
544 unsigned char lvIsImplicitByRef : 1;
545 // Set if the local appears as a last use that will be passed as an implicit byref.
546 unsigned char lvIsLastUseCopyOmissionCandidate : 1;
547 #endif // FEATURE_IMPLICIT_BYREFS
549 #if defined(TARGET_LOONGARCH64)
550 unsigned char lvIs4Field1 : 1; // Set if the 1st field is int or float within struct for LA-ABI64.
551 unsigned char lvIs4Field2 : 1; // Set if the 2nd field is int or float within struct for LA-ABI64.
552 unsigned char lvIsSplit : 1; // Set if the argument is splited.
553 #endif // defined(TARGET_LOONGARCH64)
555 #if defined(TARGET_RISCV64)
556 unsigned char lvIs4Field1 : 1; // Set if the 1st field is int or float within struct for RISCV64.
557 unsigned char lvIs4Field2 : 1; // Set if the 2nd field is int or float within struct for RISCV64.
558 unsigned char lvIsSplit : 1; // Set if the argument is splited.
559 #endif // defined(TARGET_RISCV64)
561 unsigned char lvIsBoolean : 1; // set if variable is boolean
562 unsigned char lvSingleDef : 1; // variable has a single def. Used to identify ref type locals that can get type
565 unsigned char lvSingleDefRegCandidate : 1; // variable has a single def and hence is a register candidate
566 // Currently, this is only used to decide if an EH variable can be
567 // a register candidate or not.
569 unsigned char lvDisqualifySingleDefRegCandidate : 1; // tracks variable that are disqualified from register
572 unsigned char lvSpillAtSingleDef : 1; // variable has a single def (as determined by LSRA interval scan)
573 // and is spilled making it candidate to spill right after the
574 // first (and only) definition.
575 // Note: We cannot reuse lvSingleDefRegCandidate because it is set
576 // in earlier phase and the information might not be appropriate
579 unsigned char lvVolatileHint : 1; // hint for AssertionProp
582 unsigned char lvStructDoubleAlign : 1; // Must we double align this struct?
583 #endif // !TARGET_64BIT
585 unsigned char lvQuirkToLong : 1; // Quirk to allocate this LclVar as a 64-bit long
588 unsigned char lvKeepType : 1; // Don't change the type of this variable
589 unsigned char lvNoLclFldStress : 1; // Can't apply local field stress on this one
591 unsigned char lvIsPtr : 1; // Might this be used in an address computation? (used by buffer overflow security
593 unsigned char lvIsUnsafeBuffer : 1; // Does this contain an unsafe buffer requiring buffer overflow security checks?
594 unsigned char lvPromoted : 1; // True when this local is a promoted struct, a normed struct, or a "split" long on a
595 // 32-bit target. For implicit byref parameters, this gets hijacked between
596 // fgRetypeImplicitByRefArgs and fgMarkDemotedImplicitByRefArgs to indicate whether
597 // references to the arg are being rewritten as references to a promoted shadow local.
598 unsigned char lvIsStructField : 1; // Is this local var a field of a promoted struct local?
599 unsigned char lvContainsHoles : 1; // Is this a promoted struct whose fields do not cover the struct local?
601 // True for a promoted struct that has significant padding in it.
602 // Significant padding is any data in the struct that is not covered by a
603 // promoted field and that the EE told us we need to preserve on block
605 unsigned char lvAnySignificantPadding : 1;
607 unsigned char lvIsMultiRegArg : 1; // true if this is a multireg LclVar struct used in an argument context
608 unsigned char lvIsMultiRegRet : 1; // true if this is a multireg LclVar struct assigned from a multireg call
611 unsigned char lvHiddenBufferStructArg : 1; // True when this struct (or its field) are passed as hidden buffer
615 #ifdef FEATURE_HFA_FIELDS_PRESENT
616 CorInfoHFAElemType _lvHfaElemKind : 3; // What kind of an HFA this is (CORINFO_HFA_ELEM_NONE if it is not an HFA).
617 #endif // FEATURE_HFA_FIELDS_PRESENT
620 // TODO-Cleanup: See the note on lvSize() - this flag is only in use by asserts that are checking for struct
621 // types, and is needed because of cases where TYP_STRUCT is bashed to an integral type.
622 // Consider cleaning this up so this workaround is not required.
623 unsigned char lvUnusedStruct : 1; // All references to this promoted struct are through its field locals.
624 // I.e. there is no longer any reference to the struct directly.
625 // In this case we can simply remove this struct local.
627 unsigned char lvUndoneStructPromotion : 1; // The struct promotion was undone and hence there should be no
628 // reference to the fields of this struct.
631 unsigned char lvLRACandidate : 1; // Tracked for linear scan register allocation purposes
634 unsigned char lvUsedInSIMDIntrinsic : 1; // This tells lclvar is used for simd intrinsic
635 #endif // FEATURE_SIMD
637 unsigned char lvRegStruct : 1; // This is a reg-sized non-field-addressed struct.
639 unsigned char lvClassIsExact : 1; // lvClassHandle is the exact type
642 unsigned char lvClassInfoUpdated : 1; // true if this var has updated class handle or exactness
645 unsigned char lvImplicitlyReferenced : 1; // true if there are non-IR references to this local (prolog, epilog, gc,
648 unsigned char lvSuppressedZeroInit : 1; // local needs zero init if we transform tail call to loop
650 unsigned char lvHasExplicitInit : 1; // The local is explicitly initialized and doesn't need zero initialization in
651 // the prolog. If the local has gc pointers, there are no gc-safe points
652 // between the prolog and the explicit initialization.
654 unsigned char lvIsOSRLocal : 1; // Root method local in an OSR method. Any stack home will be on the Tier0 frame.
655 // Initial value will be defined by Tier0. Requires special handing in prolog.
657 unsigned char lvIsOSRExposedLocal : 1; // OSR local that was address exposed in Tier0
659 unsigned char lvRedefinedInEmbeddedStatement : 1; // Local has redefinitions inside embedded statements that
660 // disqualify it from local copy prop.
662 unsigned char lvIsNeverNegative : 1; // The local is known to be never negative
666 unsigned lvFieldLclStart; // The index of the local var representing the first field in the promoted struct
667 // local. For implicit byref parameters, this gets hijacked between
668 // fgRetypeImplicitByRefArgs and fgMarkDemotedImplicitByRefArgs to point to the
669 // struct local created to model the parameter's struct promotion, if any.
670 unsigned lvParentLcl; // The index of the local var representing the parent (i.e. the promoted struct local).
671 // Valid on promoted struct local fields.
674 unsigned char lvFieldCnt; // Number of fields in the promoted VarDsc.
675 unsigned char lvFldOffset;
676 unsigned char lvFldOrdinal;
679 unsigned char lvSingleDefDisqualifyReason = 'H';
682 unsigned char lvAllDefsAreNoGc : 1; // For pinned locals: true if all defs of this local are no-gc
684 #if FEATURE_MULTIREG_ARGS
685 regNumber lvRegNumForSlot(unsigned slotNum)
689 return (regNumber)_lvArgReg;
691 else if (slotNum == 1)
693 return GetOtherArgReg();
697 assert(false && "Invalid slotNum!");
702 #endif // FEATURE_MULTIREG_ARGS
704 CorInfoHFAElemType GetLvHfaElemKind() const
706 #ifdef FEATURE_HFA_FIELDS_PRESENT
707 return _lvHfaElemKind;
709 NOWAY_MSG("GetLvHfaElemKind");
710 return CORINFO_HFA_ELEM_NONE;
711 #endif // FEATURE_HFA_FIELDS_PRESENT
714 void SetLvHfaElemKind(CorInfoHFAElemType elemKind)
716 #ifdef FEATURE_HFA_FIELDS_PRESENT
717 _lvHfaElemKind = elemKind;
719 NOWAY_MSG("SetLvHfaElemKind");
720 #endif // FEATURE_HFA_FIELDS_PRESENT
725 if (GlobalJitOptions::compFeatureHfa)
727 return IsHfa(GetLvHfaElemKind());
735 bool lvIsHfaRegArg() const
737 if (GlobalJitOptions::compFeatureHfa)
739 return lvIsRegArg && lvIsHfa();
747 //------------------------------------------------------------------------------
748 // lvHfaSlots: Get the number of slots used by an HFA local
751 // On Arm64 - Returns 1-4 indicating the number of register slots used by the HFA
752 // On Arm32 - Returns the total number of single FP register slots used by the HFA, max is 8
754 unsigned lvHfaSlots() const
757 assert(varTypeIsStruct(lvType));
760 slots = lvExactSize() / sizeof(float);
762 #elif defined(TARGET_ARM64)
763 switch (GetLvHfaElemKind())
765 case CORINFO_HFA_ELEM_NONE:
766 assert(!"lvHfaSlots called for non-HFA");
768 case CORINFO_HFA_ELEM_FLOAT:
769 assert((lvExactSize() % 4) == 0);
770 slots = lvExactSize() >> 2;
772 case CORINFO_HFA_ELEM_DOUBLE:
773 case CORINFO_HFA_ELEM_VECTOR64:
774 assert((lvExactSize() % 8) == 0);
775 slots = lvExactSize() >> 3;
777 case CORINFO_HFA_ELEM_VECTOR128:
778 assert((lvExactSize() % 16) == 0);
779 slots = lvExactSize() >> 4;
785 #endif // TARGET_ARM64
789 // lvIsMultiRegArgOrRet()
790 // returns true if this is a multireg LclVar struct used in an argument context
791 // or if this is a multireg LclVar struct assigned from a multireg call
792 bool lvIsMultiRegArgOrRet()
794 return lvIsMultiRegArg || lvIsMultiRegRet;
799 DoNotEnregisterReason m_doNotEnregReason;
801 AddressExposedReason m_addrExposedReason;
804 void SetDoNotEnregReason(DoNotEnregisterReason reason)
806 m_doNotEnregReason = reason;
809 DoNotEnregisterReason GetDoNotEnregReason() const
811 return m_doNotEnregReason;
814 AddressExposedReason GetAddrExposedReason() const
816 return m_addrExposedReason;
821 void SetAddressExposed(bool value DEBUGARG(AddressExposedReason reason))
823 m_addrExposed = value;
824 INDEBUG(m_addrExposedReason = reason);
827 void CleanAddressExposed()
829 m_addrExposed = false;
832 bool IsAddressExposed() const
834 return m_addrExposed;
838 void SetHiddenBufferStructArg(char value)
840 lvHiddenBufferStructArg = value;
843 bool IsHiddenBufferStructArg() const
845 return lvHiddenBufferStructArg;
850 regNumberSmall _lvRegNum; // Used to store the register this variable is in (or, the low register of a
851 // register pair). It is set during codegen any time the
852 // variable is enregistered (lvRegister is only set
853 // to non-zero if the variable gets the same register assignment for its entire
855 #if !defined(TARGET_64BIT)
856 regNumberSmall _lvOtherReg; // Used for "upper half" of long var.
857 #endif // !defined(TARGET_64BIT)
859 regNumberSmall _lvArgReg; // The (first) register in which this argument is passed.
861 #if FEATURE_MULTIREG_ARGS
862 regNumberSmall _lvOtherArgReg; // Used for the second part of the struct passed in a register.
863 // Note this is defined but not used by ARM32
864 #endif // FEATURE_MULTIREG_ARGS
866 regNumberSmall _lvArgInitReg; // the register into which the argument is moved at entry
869 // The register number is stored in a small format (8 bits), but the getters return and the setters take
870 // a full-size (unsigned) format, to localize the casts here.
872 /////////////////////
874 regNumber GetRegNum() const
876 return (regNumber)_lvRegNum;
879 void SetRegNum(regNumber reg)
881 _lvRegNum = (regNumberSmall)reg;
882 assert(_lvRegNum == reg);
885 /////////////////////
887 #if defined(TARGET_64BIT)
889 regNumber GetOtherReg() const
891 assert(!"shouldn't get here"); // can't use "unreached();" because it's NORETURN, which causes C4072
892 // "unreachable code" warnings
896 void SetOtherReg(regNumber reg)
898 assert(!"shouldn't get here"); // can't use "unreached();" because it's NORETURN, which causes C4072
899 // "unreachable code" warnings
901 #else // !TARGET_64BIT
903 regNumber GetOtherReg() const
905 return (regNumber)_lvOtherReg;
908 void SetOtherReg(regNumber reg)
910 _lvOtherReg = (regNumberSmall)reg;
911 assert(_lvOtherReg == reg);
913 #endif // !TARGET_64BIT
915 /////////////////////
917 regNumber GetArgReg() const
919 return (regNumber)_lvArgReg;
922 void SetArgReg(regNumber reg)
924 _lvArgReg = (regNumberSmall)reg;
925 assert(_lvArgReg == reg);
928 #if FEATURE_MULTIREG_ARGS
930 regNumber GetOtherArgReg() const
932 return (regNumber)_lvOtherArgReg;
935 void SetOtherArgReg(regNumber reg)
937 _lvOtherArgReg = (regNumberSmall)reg;
938 assert(_lvOtherArgReg == reg);
940 #endif // FEATURE_MULTIREG_ARGS
943 // Is this is a SIMD struct which is used for SIMD intrinsic?
944 bool lvIsUsedInSIMDIntrinsic() const
946 return lvUsedInSIMDIntrinsic;
949 bool lvIsUsedInSIMDIntrinsic() const
955 // Is this is local never negative?
956 bool IsNeverNegative() const
958 return lvIsNeverNegative;
961 // Is this is local never negative?
962 void SetIsNeverNegative(bool value)
964 lvIsNeverNegative = value;
967 /////////////////////
969 regNumber GetArgInitReg() const
971 return (regNumber)_lvArgInitReg;
974 void SetArgInitReg(regNumber reg)
976 _lvArgInitReg = (regNumberSmall)reg;
977 assert(_lvArgInitReg == reg);
980 /////////////////////
982 bool lvIsRegCandidate() const
984 return lvLRACandidate != 0;
987 bool lvIsInReg() const
989 return lvIsRegCandidate() && (GetRegNum() != REG_STK);
992 regMaskTP lvRegMask() const
994 regMaskTP regMask = RBM_NONE;
995 if (GetRegNum() != REG_STK)
997 if (varTypeUsesIntReg(this))
999 regMask = genRegMask(GetRegNum());
1003 assert(varTypeUsesFloatReg(this));
1004 regMask = genRegMaskFloat(GetRegNum() ARM_ARG(TypeGet()));
1010 //-----------------------------------------------------------------------------
1011 // AllFieldDeathFlags: Get a bitset of flags that represents all fields dying.
1014 // A bit mask that has GTF_VAR_FIELD_DEATH0 to GTF_VAR_FIELD_DEATH3 set,
1015 // depending on how many fields this promoted local has.
1018 // Only usable for promoted locals.
1020 GenTreeFlags AllFieldDeathFlags() const
1022 assert(lvPromoted && (lvFieldCnt > 0) && (lvFieldCnt <= 4));
1023 GenTreeFlags flags = static_cast<GenTreeFlags>(((1 << lvFieldCnt) - 1) << FIELD_LAST_USE_SHIFT);
1024 assert((flags & ~GTF_VAR_DEATH_MASK) == 0);
1028 //-----------------------------------------------------------------------------
1029 // FullDeathFlags: Get a bitset of flags that represents this local fully dying.
1032 // For promoted locals, this returns AllFieldDeathFlags(). Otherwise
1033 // returns GTF_VAR_DEATH.
1035 GenTreeFlags FullDeathFlags() const
1037 return lvPromoted ? AllFieldDeathFlags() : GTF_VAR_DEATH;
1040 unsigned short lvVarIndex; // variable tracking index
1043 unsigned short m_lvRefCnt; // unweighted (real) reference count. For implicit by reference
1044 // parameters, this gets hijacked from fgResetImplicitByRefRefCount
1045 // through fgMarkDemotedImplicitByRefArgs, to provide a static
1046 // appearance count (computed during address-exposed analysis)
1047 // that fgMakeOutgoingStructArgCopy consults during global morph
1048 // to determine if eliding its copy is legal.
1050 weight_t m_lvRefCntWtd; // weighted reference count
1053 unsigned short lvRefCnt(RefCountState state = RCS_NORMAL) const;
1054 void incLvRefCnt(unsigned short delta, RefCountState state = RCS_NORMAL);
1055 void setLvRefCnt(unsigned short newValue, RefCountState state = RCS_NORMAL);
1056 void incLvRefCntSaturating(unsigned short delta, RefCountState state = RCS_NORMAL);
1058 weight_t lvRefCntWtd(RefCountState state = RCS_NORMAL) const;
1059 void incLvRefCntWtd(weight_t delta, RefCountState state = RCS_NORMAL);
1060 void setLvRefCntWtd(weight_t newValue, RefCountState state = RCS_NORMAL);
1063 int lvStkOffs; // stack offset of home in bytes.
1066 int GetStackOffset() const
1071 void SetStackOffset(int offset)
1076 unsigned lvExactSize() const;
1077 unsigned lvSize() const;
1079 size_t lvArgStackSize() const;
1081 unsigned lvSlotNum; // original slot # (if remapped)
1083 // class handle for the local or null if not known or not a class
1084 CORINFO_CLASS_HANDLE lvClassHnd;
1087 ClassLayout* m_layout; // layout info for structs
1090 var_types TypeGet() const
1092 return (var_types)lvType;
1094 bool lvStackAligned() const
1096 assert(lvIsStructField);
1097 return ((lvFldOffset % TARGET_POINTER_SIZE) == 0);
1100 // NormalizeOnLoad Rules:
1101 // 1. All small locals are actually TYP_INT locals.
1102 // 2. NOL locals are such that not all definitions can be controlled by the compiler and so the upper bits can
1103 // be undefined.For parameters this is the case because of ABI.For struct fields - because of padding.For
1104 // address - exposed locals - because not all stores are direct.
1105 // 3. Hence, all NOL uses(unless proven otherwise) are assumed in morph to have undefined upper bits and
1106 // explicit casts have be inserted to "normalize" them back to conform to IL semantics.
1107 bool lvNormalizeOnLoad() const
1109 return varTypeIsSmall(TypeGet()) &&
1110 // OSR exposed locals were normalize on load in the Tier0 frame so must be so for OSR too.
1111 (lvIsParam || m_addrExposed || lvIsStructField || lvIsOSRExposedLocal);
1114 bool lvNormalizeOnStore() const
1116 return varTypeIsSmall(TypeGet()) &&
1117 // OSR exposed locals were normalize on load in the Tier0 frame so must be so for OSR too.
1118 !(lvIsParam || m_addrExposed || lvIsStructField || lvIsOSRExposedLocal);
1121 void incRefCnts(weight_t weight, Compiler* pComp, RefCountState state = RCS_NORMAL, bool propagate = true);
1123 var_types GetHfaType() const
1125 if (GlobalJitOptions::compFeatureHfa)
1128 return HfaTypeFromElemKind(GetLvHfaElemKind());
1136 void SetHfaType(var_types type)
1138 if (GlobalJitOptions::compFeatureHfa)
1140 CorInfoHFAElemType elemKind = HfaElemKindFromType(type);
1141 SetLvHfaElemKind(elemKind);
1142 // Ensure we've allocated enough bits.
1143 assert(GetLvHfaElemKind() == elemKind);
1147 // Returns true if this variable contains GC pointers (including being a GC pointer itself).
1148 bool HasGCPtr() const
1150 return varTypeIsGC(lvType) || ((lvType == TYP_STRUCT) && m_layout->HasGCPtr());
1153 // Returns the layout of a struct variable or implicit byref.
1154 ClassLayout* GetLayout() const
1156 #if FEATURE_IMPLICIT_BYREFS
1157 assert(varTypeIsStruct(TypeGet()) || (lvIsImplicitByRef && (TypeGet() == TYP_BYREF)));
1159 assert(varTypeIsStruct(TypeGet()));
1164 // Sets the layout of a struct variable.
1165 void SetLayout(ClassLayout* layout)
1167 assert(varTypeIsStruct(lvType));
1168 assert((m_layout == nullptr) || ClassLayout::AreCompatible(m_layout, layout));
1172 // Grow the size of a block layout local.
1173 void GrowBlockLayout(ClassLayout* layout)
1175 assert(varTypeIsStruct(lvType));
1176 assert((m_layout == nullptr) || (m_layout->IsBlockLayout() && (m_layout->GetSize() <= layout->GetSize())));
1177 assert(layout->IsBlockLayout());
1181 SsaDefArray<LclSsaVarDsc> lvPerSsaData;
1183 // True if ssaNum is a viable ssaNum for this local.
1184 bool IsValidSsaNum(unsigned ssaNum) const
1186 return lvPerSsaData.IsValidSsaNum(ssaNum);
1189 // Returns the address of the per-Ssa data for the given ssaNum (which is required
1190 // not to be the SsaConfig::RESERVED_SSA_NUM, which indicates that the variable is
1191 // not an SSA variable).
1192 LclSsaVarDsc* GetPerSsaData(unsigned ssaNum) const
1194 return lvPerSsaData.GetSsaDef(ssaNum);
1197 // Returns the SSA number for "ssaDef". Requires "ssaDef" to be a valid definition
1198 // of this variable.
1199 unsigned GetSsaNumForSsaDef(LclSsaVarDsc* ssaDef)
1201 return lvPerSsaData.GetSsaNum(ssaDef);
1204 var_types GetRegisterType(const GenTreeLclVarCommon* tree) const;
1206 var_types GetRegisterType() const;
1208 var_types GetStackSlotHomeType() const;
1210 bool IsEnregisterableType() const
1212 return GetRegisterType() != TYP_UNDEF;
1215 bool IsEnregisterableLcl() const
1217 if (lvDoNotEnregister)
1221 return IsEnregisterableType();
1224 //-----------------------------------------------------------------------------
1225 // IsAlwaysAliveInMemory: Determines if this variable's value is always
1226 // up-to-date on stack. This is possible if this is an EH-var or
1227 // we decided to spill after single-def.
1229 bool IsAlwaysAliveInMemory() const
1231 return lvLiveInOutOfHndlr || lvSpillAtSingleDef;
1234 bool CanBeReplacedWithItsField(Compiler* comp) const;
1238 const char* lvReason;
1240 void PrintVarReg() const
1242 printf("%s", getRegName(GetRegNum()));
1246 }; // class LclVarDsc
1248 enum class SymbolicIntegerValue : int32_t
1266 inline constexpr bool operator>(SymbolicIntegerValue left, SymbolicIntegerValue right)
1268 return static_cast<int32_t>(left) > static_cast<int32_t>(right);
1271 inline constexpr bool operator>=(SymbolicIntegerValue left, SymbolicIntegerValue right)
1273 return static_cast<int32_t>(left) >= static_cast<int32_t>(right);
1276 inline constexpr bool operator<(SymbolicIntegerValue left, SymbolicIntegerValue right)
1278 return static_cast<int32_t>(left) < static_cast<int32_t>(right);
1281 inline constexpr bool operator<=(SymbolicIntegerValue left, SymbolicIntegerValue right)
1283 return static_cast<int32_t>(left) <= static_cast<int32_t>(right);
1286 // Represents an integral range useful for reasoning about integral casts.
1287 // It uses a symbolic representation for lower and upper bounds so
1288 // that it can efficiently handle integers of all sizes on all hosts.
1290 // Note that the ranges represented by this class are **always** in the
1291 // "signed" domain. This is so that if we know the range a node produces, it
1292 // can be trivially used to determine if a cast above the node does or does not
1293 // overflow, which requires that the interpretation of integers be the same both
1294 // for the "input" and "output". We choose signed interpretation here because it
1295 // produces nice continuous ranges and because IR uses sign-extension for constants.
1297 // Some examples of how ranges are computed for casts:
1298 // 1. CAST_OVF(ubyte <- uint): does not overflow for [0..UBYTE_MAX], produces the
1299 // same range - all casts that do not change the representation, i. e. have the same
1300 // "actual" input and output type, have the same "input" and "output" range.
1301 // 2. CAST_OVF(ulong <- uint): never overflows => the "input" range is [INT_MIN..INT_MAX]
1302 // (aka all possible 32 bit integers). Produces [0..UINT_MAX] (aka all possible 32
1303 // bit integers zero-extended to 64 bits).
1304 // 3. CAST_OVF(int <- uint): overflows for inputs larger than INT_MAX <=> less than 0
1305 // when interpreting as signed => the "input" range is [0..INT_MAX], the same range
1306 // being the produced one as the node does not change the width of the integer.
1311 SymbolicIntegerValue m_lowerBound;
1312 SymbolicIntegerValue m_upperBound;
1315 IntegralRange() = default;
1317 IntegralRange(SymbolicIntegerValue lowerBound, SymbolicIntegerValue upperBound)
1318 : m_lowerBound(lowerBound), m_upperBound(upperBound)
1320 assert(lowerBound <= upperBound);
1323 SymbolicIntegerValue GetLowerBound() const
1325 return m_lowerBound;
1328 SymbolicIntegerValue GetUpperBound() const
1330 return m_upperBound;
1333 bool Contains(int64_t value) const;
1335 bool Contains(IntegralRange other) const
1337 return (m_lowerBound <= other.m_lowerBound) && (other.m_upperBound <= m_upperBound);
1340 bool IsNonNegative() const
1342 return m_lowerBound >= SymbolicIntegerValue::Zero;
1345 bool Equals(IntegralRange other) const
1347 return (m_lowerBound == other.m_lowerBound) && (m_upperBound == other.m_upperBound);
1350 static int64_t SymbolicToRealValue(SymbolicIntegerValue value);
1351 static SymbolicIntegerValue LowerBoundForType(var_types type);
1352 static SymbolicIntegerValue UpperBoundForType(var_types type);
1354 static IntegralRange ForType(var_types type)
1356 return {LowerBoundForType(type), UpperBoundForType(type)};
1359 static IntegralRange ForNode(GenTree* node, Compiler* compiler);
1360 static IntegralRange ForCastInput(GenTreeCast* cast);
1361 static IntegralRange ForCastOutput(GenTreeCast* cast, Compiler* compiler);
1362 static IntegralRange Union(IntegralRange range1, IntegralRange range2);
1365 static void Print(IntegralRange range);
1370 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1371 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1375 XX The temporary lclVars allocated by the compiler for code generation XX
1377 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1378 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1381 /*****************************************************************************
1383 * The following keeps track of temporaries allocated in the stack frame
1384 * during code-generation (after register allocation). These spill-temps are
1385 * only used if we run out of registers while evaluating a tree.
1387 * These are different from the more common temps allocated by lvaGrabTemp().
1398 static const int BAD_TEMP_OFFSET = 0xDDDDDDDD; // used as a sentinel "bad value" for tdOffs in DEBUG
1406 TempDsc(int _tdNum, unsigned _tdSize, var_types _tdType) : tdNum(_tdNum), tdSize((BYTE)_tdSize), tdType(_tdType)
1409 // temps must have a negative number (so they have a different number from all local variables)
1411 tdOffs = BAD_TEMP_OFFSET;
1413 if (tdNum != _tdNum)
1415 IMPL_LIMITATION("too many spill temps");
1420 bool tdLegalOffset() const
1422 return tdOffs != BAD_TEMP_OFFSET;
1426 int tdTempOffs() const
1428 assert(tdLegalOffset());
1431 void tdSetTempOffs(int offs)
1434 assert(tdLegalOffset());
1436 void tdAdjustTempOffs(int offs)
1439 assert(tdLegalOffset());
1442 int tdTempNum() const
1447 unsigned tdTempSize() const
1451 var_types tdTempType() const
1457 // Specify compiler data that a phase might modify
1458 enum class PhaseStatus : unsigned
1460 MODIFIED_NOTHING, // Phase did not make any changes that warrant running post-phase checks or dumping
1461 // the main jit data strutures.
1462 MODIFIED_EVERYTHING, // Phase made changes that warrant running post-phase checks or dumping
1463 // the main jit data strutures.
1466 // interface to hide linearscan implementation from rest of compiler
1467 class LinearScanInterface
1470 virtual PhaseStatus doLinearScan() = 0;
1471 virtual void recordVarLocationsAtStartOfBB(BasicBlock* bb) = 0;
1472 virtual bool willEnregisterLocalVars() const = 0;
1473 #if TRACK_LSRA_STATS
1474 virtual void dumpLsraStatsCsv(FILE* file) = 0;
1475 virtual void dumpLsraStatsSummary(FILE* file) = 0;
1476 #endif // TRACK_LSRA_STATS
1479 LinearScanInterface* getLinearScanAllocator(Compiler* comp);
1481 // This enumeration names the phases into which we divide compilation. The phases should completely
1482 // partition a compilation.
1485 #define CompPhaseNameMacro(enum_nm, string_nm, hasChildren, parent, measureIR) enum_nm,
1486 #include "compphases.h"
1490 extern const char* PhaseNames[];
1491 extern const char* PhaseEnums[];
1493 // Specify which checks should be run after each phase
1496 enum class PhaseChecks : unsigned int
1499 CHECK_IR = 1 << 0, // ir flags, etc
1500 CHECK_UNIQUE = 1 << 1, // tree node uniqueness
1501 CHECK_FG = 1 << 2, // flow graph integrity
1502 CHECK_EH = 1 << 3, // eh table integrity
1503 CHECK_LOOPS = 1 << 4, // loop table integrity
1504 CHECK_PROFILE = 1 << 5, // profile data integrity
1505 CHECK_LINKED_LOCALS = 1 << 6, // check linked list of locals
1508 inline constexpr PhaseChecks operator ~(PhaseChecks a)
1510 return (PhaseChecks)(~(unsigned int)a);
1513 inline constexpr PhaseChecks operator |(PhaseChecks a, PhaseChecks b)
1515 return (PhaseChecks)((unsigned int)a | (unsigned int)b);
1518 inline constexpr PhaseChecks operator &(PhaseChecks a, PhaseChecks b)
1520 return (PhaseChecks)((unsigned int)a & (unsigned int)b);
1523 inline PhaseChecks& operator |=(PhaseChecks& a, PhaseChecks b)
1525 return a = (PhaseChecks)((unsigned int)a | (unsigned int)b);
1528 inline PhaseChecks& operator &=(PhaseChecks& a, PhaseChecks b)
1530 return a = (PhaseChecks)((unsigned int)a & (unsigned int)b);
1533 inline PhaseChecks& operator ^=(PhaseChecks& a, PhaseChecks b)
1535 return a = (PhaseChecks)((unsigned int)a ^ (unsigned int)b);
1539 // Specify which dumps should be run after each phase
1541 enum class PhaseDumps
1547 // The following enum provides a simple 1:1 mapping to CLR API's
1548 enum API_ICorJitInfo_Names
1550 #define DEF_CLR_API(name) API_##name,
1551 #include "ICorJitInfo_names_generated.h"
1555 enum class FlowGraphUpdates
1557 COMPUTE_BASICS = 0, // renumber blocks, reachability, etc
1558 COMPUTE_DOMS = 1 << 0, // recompute dominators
1559 COMPUTE_RETURNS = 1 << 1, // recompute return blocks
1560 COMPUTE_LOOPS = 1 << 2, // recompute loop table
1563 inline constexpr FlowGraphUpdates operator|(FlowGraphUpdates a, FlowGraphUpdates b)
1565 return (FlowGraphUpdates)((unsigned int)a | (unsigned int)b);
1568 inline constexpr FlowGraphUpdates operator&(FlowGraphUpdates a, FlowGraphUpdates b)
1570 return (FlowGraphUpdates)((unsigned int)a & (unsigned int)b);
1573 // Profile checking options
1576 enum class ProfileChecks : unsigned int
1579 CHECK_CLASSIC = 1 << 0, // check "classic" jit weights
1580 CHECK_LIKELY = 1 << 1, // check likelihood based weights
1581 RAISE_ASSERT = 1 << 2, // assert on check failure
1582 CHECK_ALL_BLOCKS = 1 << 3, // check blocks even if bbHasProfileWeight is false
1585 inline constexpr ProfileChecks operator ~(ProfileChecks a)
1587 return (ProfileChecks)(~(unsigned int)a);
1590 inline constexpr ProfileChecks operator |(ProfileChecks a, ProfileChecks b)
1592 return (ProfileChecks)((unsigned int)a | (unsigned int)b);
1595 inline constexpr ProfileChecks operator &(ProfileChecks a, ProfileChecks b)
1597 return (ProfileChecks)((unsigned int)a & (unsigned int)b);
1600 inline ProfileChecks& operator |=(ProfileChecks& a, ProfileChecks b)
1602 return a = (ProfileChecks)((unsigned int)a | (unsigned int)b);
1605 inline ProfileChecks& operator &=(ProfileChecks& a, ProfileChecks b)
1607 return a = (ProfileChecks)((unsigned int)a & (unsigned int)b);
1610 inline ProfileChecks& operator ^=(ProfileChecks& a, ProfileChecks b)
1612 return a = (ProfileChecks)((unsigned int)a ^ (unsigned int)b);
1615 inline bool hasFlag(const ProfileChecks& flagSet, const ProfileChecks& flag)
1617 return ((flagSet & flag) == flag);
1620 //---------------------------------------------------------------
1621 // Compilation time.
1624 // A "CompTimeInfo" is a structure for tracking the compilation time of one or more methods.
1625 // We divide a compilation into a sequence of contiguous phases, and track the total (per-thread) cycles
1626 // of the compilation, as well as the cycles for each phase. We also track the number of bytecodes.
1627 // If there is a failure in reading a timer at any point, the "CompTimeInfo" becomes invalid, as indicated
1628 // by "m_timerFailure" being true.
1629 // If FEATURE_JIT_METHOD_PERF is not set, we define a minimal form of this, enough to let other code compile.
1632 #ifdef FEATURE_JIT_METHOD_PERF
1633 // The string names of the phases.
1634 static const char* PhaseNames[];
1636 static bool PhaseHasChildren[];
1637 static int PhaseParent[];
1638 static bool PhaseReportsIRSize[];
1640 unsigned m_byteCodeBytes;
1641 unsigned __int64 m_totalCycles;
1642 unsigned __int64 m_invokesByPhase[PHASE_NUMBER_OF];
1643 unsigned __int64 m_cyclesByPhase[PHASE_NUMBER_OF];
1644 #if MEASURE_CLRAPI_CALLS
1645 unsigned __int64 m_CLRinvokesByPhase[PHASE_NUMBER_OF];
1646 unsigned __int64 m_CLRcyclesByPhase[PHASE_NUMBER_OF];
1649 unsigned m_nodeCountAfterPhase[PHASE_NUMBER_OF];
1651 // For better documentation, we call EndPhase on
1652 // non-leaf phases. We should also call EndPhase on the
1653 // last leaf subphase; obviously, the elapsed cycles between the EndPhase
1654 // for the last leaf subphase and the EndPhase for an ancestor should be very small.
1655 // We add all such "redundant end phase" intervals to this variable below; we print
1656 // it out in a report, so we can verify that it is, indeed, very small. If it ever
1657 // isn't, this means that we're doing something significant between the end of the last
1658 // declared subphase and the end of its parent.
1659 unsigned __int64 m_parentPhaseEndSlop;
1660 bool m_timerFailure;
1662 #if MEASURE_CLRAPI_CALLS
1663 // The following measures the time spent inside each individual CLR API call.
1664 unsigned m_allClrAPIcalls;
1665 unsigned m_perClrAPIcalls[API_ICorJitInfo_Names::API_COUNT];
1666 unsigned __int64 m_allClrAPIcycles;
1667 unsigned __int64 m_perClrAPIcycles[API_ICorJitInfo_Names::API_COUNT];
1668 unsigned __int32 m_maxClrAPIcycles[API_ICorJitInfo_Names::API_COUNT];
1669 #endif // MEASURE_CLRAPI_CALLS
1671 CompTimeInfo(unsigned byteCodeBytes);
1675 #ifdef FEATURE_JIT_METHOD_PERF
1677 #if MEASURE_CLRAPI_CALLS
1678 struct WrapICorJitInfo;
1681 // This class summarizes the JIT time information over the course of a run: the number of methods compiled,
1682 // and the total and maximum timings. (These are instances of the "CompTimeInfo" type described above).
1683 // The operation of adding a single method's timing to the summary may be performed concurrently by several
1684 // threads, so it is protected by a lock.
1685 // This class is intended to be used as a singleton type, with only a single instance.
1686 class CompTimeSummaryInfo
1688 // This lock protects the fields of all CompTimeSummaryInfo(s) (of which we expect there to be one).
1689 static CritSecObject s_compTimeSummaryLock;
1693 CompTimeInfo m_total;
1694 CompTimeInfo m_maximum;
1696 int m_numFilteredMethods;
1697 CompTimeInfo m_filtered;
1699 // This can use what ever data you want to determine if the value to be added
1700 // belongs in the filtered section (it's always included in the unfiltered section)
1701 bool IncludedInFilteredData(CompTimeInfo& info);
1704 // This is the unique CompTimeSummaryInfo object for this instance of the runtime.
1705 static CompTimeSummaryInfo s_compTimeSummary;
1707 CompTimeSummaryInfo()
1708 : m_numMethods(0), m_totMethods(0), m_total(0), m_maximum(0), m_numFilteredMethods(0), m_filtered(0)
1712 // Assumes that "info" is a completed CompTimeInfo for a compilation; adds it to the summary.
1713 // This is thread safe.
1714 void AddInfo(CompTimeInfo& info, bool includePhases);
1716 // Print the summary information to "f".
1717 // This is not thread-safe; assumed to be called by only one thread.
1718 void Print(FILE* f);
1721 // A JitTimer encapsulates a CompTimeInfo for a single compilation. It also tracks the start of compilation,
1722 // and when the current phase started. This is intended to be part of a Compilation object.
1726 unsigned __int64 m_start; // Start of the compilation.
1727 unsigned __int64 m_curPhaseStart; // Start of the current phase.
1728 #if MEASURE_CLRAPI_CALLS
1729 unsigned __int64 m_CLRcallStart; // Start of the current CLR API call (if any).
1730 unsigned __int64 m_CLRcallInvokes; // CLR API invokes under current outer so far
1731 unsigned __int64 m_CLRcallCycles; // CLR API cycles under current outer so far.
1732 int m_CLRcallAPInum; // The enum/index of the current CLR API call (or -1).
1733 static double s_cyclesPerSec; // Cached for speedier measurements
1736 Phases m_lastPhase; // The last phase that was completed (or (Phases)-1 to start).
1738 CompTimeInfo m_info; // The CompTimeInfo for this compilation.
1740 static CritSecObject s_csvLock; // Lock to protect the time log file.
1741 static FILE* s_csvFile; // The time log file handle.
1742 void PrintCsvMethodStats(Compiler* comp);
1745 void* operator new(size_t);
1746 void* operator new[](size_t);
1747 void operator delete(void*);
1748 void operator delete[](void*);
1751 // Initialized the timer instance
1752 JitTimer(unsigned byteCodeSize);
1754 static JitTimer* Create(Compiler* comp, unsigned byteCodeSize)
1756 return ::new (comp, CMK_Unknown) JitTimer(byteCodeSize);
1759 static void PrintCsvHeader();
1761 // Ends the current phase (argument is for a redundant check).
1762 void EndPhase(Compiler* compiler, Phases phase);
1764 #if MEASURE_CLRAPI_CALLS
1765 // Start and end a timed CLR API call.
1766 void CLRApiCallEnter(unsigned apix);
1767 void CLRApiCallLeave(unsigned apix);
1768 #endif // MEASURE_CLRAPI_CALLS
1770 // Completes the timing of the current method, which is assumed to have "byteCodeBytes" bytes of bytecode,
1771 // and adds it to "sum".
1772 void Terminate(Compiler* comp, CompTimeSummaryInfo& sum, bool includePhases);
1774 // Attempts to query the cycle counter of the current thread. If successful, returns "true" and sets
1775 // *cycles to the cycle counter value. Otherwise, returns false and sets the "m_timerFailure" flag of
1776 // "m_info" to true.
1777 bool GetThreadCycles(unsigned __int64* cycles)
1779 bool res = CycleTimer::GetThreadCyclesS(cycles);
1782 m_info.m_timerFailure = true;
1787 static void Shutdown();
1789 #endif // FEATURE_JIT_METHOD_PERF
1791 //------------------- Function/Funclet info -------------------------------
1792 enum FuncKind : BYTE
1794 FUNC_ROOT, // The main/root function (always id==0)
1795 FUNC_HANDLER, // a funclet associated with an EH handler (finally, fault, catch, filter handler)
1796 FUNC_FILTER, // a funclet associated with an EH filter
1805 BYTE funFlags; // Currently unused, just here for padding
1806 unsigned short funEHIndex; // index, into the ebd table, of innermost EH clause corresponding to this
1807 // funclet. It is only valid if funKind field indicates this is a
1808 // EH-related funclet: FUNC_HANDLER or FUNC_FILTER
1810 #if defined(TARGET_AMD64)
1812 // TODO-AMD64-Throughput: make the AMD64 info more like the ARM info to avoid having this large static array.
1813 emitLocation* startLoc;
1814 emitLocation* endLoc;
1815 emitLocation* coldStartLoc; // locations for the cold section, if there is one.
1816 emitLocation* coldEndLoc;
1817 UNWIND_INFO unwindHeader;
1818 // Maximum of 255 UNWIND_CODE 'nodes' and then the unwind header. If there are an odd
1819 // number of codes, the VM or Zapper will 4-byte align the whole thing.
1820 BYTE unwindCodes[offsetof(UNWIND_INFO, UnwindCode) + (0xFF * sizeof(UNWIND_CODE))];
1821 unsigned unwindCodeSlot;
1823 #elif defined(TARGET_X86)
1825 emitLocation* startLoc;
1826 emitLocation* endLoc;
1827 emitLocation* coldStartLoc; // locations for the cold section, if there is one.
1828 emitLocation* coldEndLoc;
1830 #elif defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
1832 UnwindInfo uwi; // Unwind information for this function/funclet's hot section
1833 UnwindInfo* uwiCold; // Unwind information for this function/funclet's cold section
1834 // Note: we only have a pointer here instead of the actual object,
1835 // to save memory in the JIT case (compared to the NGEN case),
1836 // where we don't have any cold section.
1837 // Note 2: we currently don't support hot/cold splitting in functions
1838 // with EH, so uwiCold will be NULL for all funclets.
1840 emitLocation* startLoc;
1841 emitLocation* endLoc;
1842 emitLocation* coldStartLoc; // locations for the cold section, if there is one.
1843 emitLocation* coldEndLoc;
1845 #endif // TARGET_ARMARCH || TARGET_LOONGARCH64 || TARGET_RISCV64
1847 #if defined(FEATURE_CFI_SUPPORT)
1848 jitstd::vector<CFI_CODE>* cfiCodes;
1849 #endif // FEATURE_CFI_SUPPORT
1851 // Eventually we may want to move rsModifiedRegsMask, lvaOutgoingArgSize, and anything else
1852 // that isn't shared between the main function body and funclets.
1862 // XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1863 // We have the ability to mark source expressions with "Test Labels."
1864 // These drive assertions within the JIT, or internal JIT testing. For example, we could label expressions
1865 // that should be CSE defs, and other expressions that should uses of those defs, with a shared label.
1867 enum TestLabel // This must be kept identical to System.Runtime.CompilerServices.JitTestLabel.TestLabel.
1870 TL_VN, // Defines a "VN equivalence class". (For full VN, including exceptions thrown).
1871 TL_VNNorm, // Like above, but uses the non-exceptional value of the expression.
1872 TL_CSE_Def, // This must be identified in the JIT as a CSE def
1873 TL_CSE_Use, // This must be identified in the JIT as a CSE use
1874 TL_LoopHoist, // Expression must (or must not) be hoisted out of the loop.
1877 struct TestLabelAndNum
1882 TestLabelAndNum() : m_tl(TestLabel(0)), m_num(0)
1887 typedef JitHashTable<GenTree*, JitPtrKeyFuncs<GenTree>, TestLabelAndNum> NodeToTestDataMap;
1889 // XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1892 //-------------------------------------------------------------------------
1893 // LoopFlags: flags for the loop table.
1895 enum LoopFlags : unsigned short
1899 // LPFLG_UNUSED = 0x0001,
1900 // LPFLG_UNUSED = 0x0002,
1901 LPFLG_ITER = 0x0004, // loop of form: for (i = icon or expression; test_condition(); i++)
1902 // LPFLG_UNUSED = 0x0008,
1904 LPFLG_CONTAINS_CALL = 0x0010, // If executing the loop body *may* execute a call
1905 // LPFLG_UNUSED = 0x0020,
1906 LPFLG_CONST_INIT = 0x0040, // iterator is initialized with a constant (found in lpConstInit)
1907 LPFLG_SIMD_LIMIT = 0x0080, // iterator is compared with vector element count (found in lpConstLimit)
1909 LPFLG_VAR_LIMIT = 0x0100, // iterator is compared with a local var (var # found in lpVarLimit)
1910 LPFLG_CONST_LIMIT = 0x0200, // iterator is compared with a constant (found in lpConstLimit)
1911 LPFLG_ARRLEN_LIMIT = 0x0400, // iterator is compared with a.len or a[i].len (found in lpArrLenLimit)
1912 LPFLG_HAS_PREHEAD = 0x0800, // lpHead is known to be a preHead for this loop
1914 LPFLG_REMOVED = 0x1000, // has been removed from the loop table (unrolled or optimized away)
1915 LPFLG_DONT_UNROLL = 0x2000, // do not unroll this loop
1916 LPFLG_ASGVARS_YES = 0x4000, // "lpAsgVars" has been computed
1917 LPFLG_ASGVARS_INC = 0x8000, // "lpAsgVars" is incomplete -- vars beyond those representable in an AllVarSet
1918 // type are assigned to.
1921 inline constexpr LoopFlags operator~(LoopFlags a)
1923 return (LoopFlags)(~(unsigned short)a);
1926 inline constexpr LoopFlags operator|(LoopFlags a, LoopFlags b)
1928 return (LoopFlags)((unsigned short)a | (unsigned short)b);
1931 inline constexpr LoopFlags operator&(LoopFlags a, LoopFlags b)
1933 return (LoopFlags)((unsigned short)a & (unsigned short)b);
1936 inline LoopFlags& operator|=(LoopFlags& a, LoopFlags b)
1938 return a = (LoopFlags)((unsigned short)a | (unsigned short)b);
1941 inline LoopFlags& operator&=(LoopFlags& a, LoopFlags b)
1943 return a = (LoopFlags)((unsigned short)a & (unsigned short)b);
1946 // The following holds information about instr offsets in terms of generated code.
1948 enum class IPmappingDscKind
1950 Prolog, // The mapping represents the start of a prolog.
1951 Epilog, // The mapping represents the start of an epilog.
1952 NoMapping, // This does not map to any IL offset.
1953 Normal, // The mapping maps to an IL offset.
1958 emitLocation ipmdNativeLoc; // the emitter location of the native code corresponding to the IL offset
1959 IPmappingDscKind ipmdKind; // The kind of mapping
1960 ILLocation ipmdLoc; // The location for normal mappings
1961 bool ipmdIsLabel; // Can this code be a branch label?
1964 struct RichIPMapping
1966 emitLocation nativeLoc;
1967 DebugInfo debugInfo;
1970 // Current kind of node threading stored in GenTree::gtPrev and GenTree::gtNext.
1971 // See fgNodeThreading for more information.
1972 enum class NodeThreading
1975 AllLocals, // Locals are threaded (after local morph when optimizing)
1976 AllTrees, // All nodes are threaded (after gtSetBlockOrder)
1977 LIR, // Nodes are in LIR form (after rationalization)
1981 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1982 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1984 XX The big guy. The sections are currently organized as : XX
1986 XX o GenTree and BasicBlock XX
1998 XX o PrologScopeInfo XX
1999 XX o CodeGenerator XX
2004 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
2005 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
2008 struct HWIntrinsicInfo;
2012 friend class emitter;
2013 friend class UnwindInfo;
2014 friend class UnwindFragmentInfo;
2015 friend class UnwindEpilogInfo;
2016 friend class JitTimer;
2017 friend class LinearScan;
2018 friend class Rationalizer;
2020 friend class Lowering;
2021 friend class CSE_DataFlow;
2022 friend class CSE_Heuristic;
2023 friend class CodeGenInterface;
2024 friend class CodeGen;
2025 friend class LclVarDsc;
2026 friend class TempDsc;
2028 friend class ObjectAllocator;
2029 friend class LocalAddressVisitor;
2030 friend struct Statement;
2031 friend struct GenTree;
2032 friend class MorphInitBlockHelper;
2033 friend class MorphCopyBlockHelper;
2034 friend class SharedTempsScope;
2035 friend class CallArgs;
2036 friend class IndirectCallTransformer;
2037 friend class ProfileSynthesis;
2038 friend class LocalsUseVisitor;
2039 friend class Promotion;
2040 friend class ReplaceVisitor;
2042 #ifdef FEATURE_HW_INTRINSICS
2043 friend struct HWIntrinsicInfo;
2044 friend struct SimdAsHWIntrinsicInfo;
2045 #endif // FEATURE_HW_INTRINSICS
2047 #ifndef TARGET_64BIT
2048 friend class DecomposeLongs;
2049 #endif // !TARGET_64BIT
2052 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
2053 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
2055 XX Misc structs definitions XX
2057 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
2058 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
2062 hashBvGlobalData hbvGlobalData; // Used by the hashBv bitvector package.
2067 bool shouldUseVerboseTrees();
2068 bool asciiTrees; // If true, dump trees using only ASCII characters
2069 bool shouldDumpASCIITrees();
2070 bool verboseSsa; // If true, produce especially verbose dump output in SSA construction.
2071 bool shouldUseVerboseSsa();
2072 bool treesBeforeAfterMorph; // If true, print trees before/after morphing (paired by an intra-compilation id:
2073 int morphNum; // This counts the trees that have been morphed, allowing us to label each uniquely.
2074 void makeExtraStructQueries(CORINFO_CLASS_HANDLE structHandle, int level); // Make queries recursively 'level' deep.
2076 const char* VarNameToStr(VarName name)
2081 DWORD expensiveDebugCheckLevel;
2084 GenTree* impStoreMultiRegValueToVar(GenTree* op,
2085 CORINFO_CLASS_HANDLE hClass DEBUGARG(CorInfoCallConvExtension callConv));
2088 bool isTrivialPointerSizedStruct(CORINFO_CLASS_HANDLE clsHnd) const;
2089 #endif // TARGET_X86
2091 //-------------------------------------------------------------------------
2092 // Functions to handle homogeneous floating-point aggregates (HFAs) in ARM/ARM64.
2093 // HFAs are one to four element structs where each element is the same
2094 // type, either all float or all double. We handle HVAs (one to four elements of
2095 // vector types) uniformly with HFAs. HFAs are treated specially
2096 // in the ARM/ARM64 Procedure Call Standards, specifically, they are passed in
2097 // floating-point registers instead of the general purpose registers.
2100 bool IsHfa(CORINFO_CLASS_HANDLE hClass);
2101 var_types GetHfaType(CORINFO_CLASS_HANDLE hClass);
2102 unsigned GetHfaCount(CORINFO_CLASS_HANDLE hClass);
2104 bool IsMultiRegReturnedType(CORINFO_CLASS_HANDLE hClass, CorInfoCallConvExtension callConv);
2106 //-------------------------------------------------------------------------
2107 // The following is used for validating format of EH table
2111 typedef struct EHNodeDsc* pEHNodeDsc;
2113 EHNodeDsc* ehnTree; // root of the tree comprising the EHnodes.
2114 EHNodeDsc* ehnNext; // root of the tree comprising the EHnodes.
2127 EHBlockType ehnBlockType; // kind of EH block
2128 IL_OFFSET ehnStartOffset; // IL offset of start of the EH block
2129 IL_OFFSET ehnEndOffset; // IL offset past end of the EH block. (TODO: looks like verInsertEhNode() sets this to
2130 // the last IL offset, not "one past the last one", i.e., the range Start to End is
2132 pEHNodeDsc ehnNext; // next (non-nested) block in sequential order
2133 pEHNodeDsc ehnChild; // leftmost nested block
2135 pEHNodeDsc ehnTryNode; // for filters and handlers, the corresponding try node
2136 pEHNodeDsc ehnHandlerNode; // for a try node, the corresponding handler node
2138 pEHNodeDsc ehnFilterNode; // if this is a try node and has a filter, otherwise 0
2139 pEHNodeDsc ehnEquivalent; // if blockType=tryNode, start offset and end offset is same,
2141 void ehnSetTryNodeType()
2143 ehnBlockType = TryNode;
2145 void ehnSetFilterNodeType()
2147 ehnBlockType = FilterNode;
2149 void ehnSetHandlerNodeType()
2151 ehnBlockType = HandlerNode;
2153 void ehnSetFinallyNodeType()
2155 ehnBlockType = FinallyNode;
2157 void ehnSetFaultNodeType()
2159 ehnBlockType = FaultNode;
2162 bool ehnIsTryBlock()
2164 return ehnBlockType == TryNode;
2166 bool ehnIsFilterBlock()
2168 return ehnBlockType == FilterNode;
2170 bool ehnIsHandlerBlock()
2172 return ehnBlockType == HandlerNode;
2174 bool ehnIsFinallyBlock()
2176 return ehnBlockType == FinallyNode;
2178 bool ehnIsFaultBlock()
2180 return ehnBlockType == FaultNode;
2183 // returns true if there is any overlap between the two nodes
2184 static bool ehnIsOverlap(pEHNodeDsc node1, pEHNodeDsc node2)
2186 if (node1->ehnStartOffset < node2->ehnStartOffset)
2188 return (node1->ehnEndOffset >= node2->ehnStartOffset);
2192 return (node1->ehnStartOffset <= node2->ehnEndOffset);
2196 // fails with BADCODE if inner is not completely nested inside outer
2197 static bool ehnIsNested(pEHNodeDsc inner, pEHNodeDsc outer)
2199 return ((inner->ehnStartOffset >= outer->ehnStartOffset) && (inner->ehnEndOffset <= outer->ehnEndOffset));
2203 //-------------------------------------------------------------------------
2204 // Exception handling functions
2207 #if !defined(FEATURE_EH_FUNCLETS)
2209 bool ehNeedsShadowSPslots()
2211 return (info.compXcptnsCount || opts.compDbgEnC);
2214 // 0 for methods with no EH
2215 // 1 for methods with non-nested EH, or where only the try blocks are nested
2216 // 2 for a method with a catch within a catch
2218 unsigned ehMaxHndNestingCount;
2220 #endif // !FEATURE_EH_FUNCLETS
2222 static bool jitIsBetween(unsigned value, unsigned start, unsigned end);
2223 static bool jitIsBetweenInclusive(unsigned value, unsigned start, unsigned end);
2225 bool bbInCatchHandlerILRange(BasicBlock* blk);
2226 bool bbInFilterILRange(BasicBlock* blk);
2227 bool bbInFilterBBRange(BasicBlock* blk);
2228 bool bbInTryRegions(unsigned regionIndex, BasicBlock* blk);
2229 bool bbInExnFlowRegions(unsigned regionIndex, BasicBlock* blk);
2230 bool bbInHandlerRegions(unsigned regionIndex, BasicBlock* blk);
2231 bool bbInCatchHandlerRegions(BasicBlock* tryBlk, BasicBlock* hndBlk);
2232 unsigned short bbFindInnermostCommonTryRegion(BasicBlock* bbOne, BasicBlock* bbTwo);
2234 unsigned short bbFindInnermostTryRegionContainingHandlerRegion(unsigned handlerIndex);
2235 unsigned short bbFindInnermostHandlerRegionContainingTryRegion(unsigned tryIndex);
2237 // Returns true if "block" is the start of a try region.
2238 bool bbIsTryBeg(BasicBlock* block);
2240 // Returns true if "block" is the start of a handler or filter region.
2241 bool bbIsHandlerBeg(BasicBlock* block);
2243 // Returns true iff "block" is where control flows if an exception is raised in the
2244 // try region, and sets "*regionIndex" to the index of the try for the handler.
2245 // Differs from "IsHandlerBeg" in the case of filters, where this is true for the first
2246 // block of the filter, but not for the filter's handler.
2247 bool bbIsExFlowBlock(BasicBlock* block, unsigned* regionIndex);
2249 bool ehHasCallableHandlers();
2251 // Return the EH descriptor for the given region index.
2252 EHblkDsc* ehGetDsc(unsigned regionIndex);
2254 // Return the EH index given a region descriptor.
2255 unsigned ehGetIndex(EHblkDsc* ehDsc);
2257 // Return the EH descriptor index of the enclosing try, for the given region index.
2258 unsigned ehGetEnclosingTryIndex(unsigned regionIndex);
2260 // Return the EH descriptor index of the enclosing handler, for the given region index.
2261 unsigned ehGetEnclosingHndIndex(unsigned regionIndex);
2263 // Return the EH descriptor for the most nested 'try' region this BasicBlock is a member of (or nullptr if this
2264 // block is not in a 'try' region).
2265 EHblkDsc* ehGetBlockTryDsc(BasicBlock* block);
2267 // Return the EH descriptor for the most nested filter or handler region this BasicBlock is a member of (or nullptr
2268 // if this block is not in a filter or handler region).
2269 EHblkDsc* ehGetBlockHndDsc(BasicBlock* block);
2271 // Return the EH descriptor for the most nested region that may handle exceptions raised in this BasicBlock (or
2272 // nullptr if this block's exceptions propagate to caller).
2273 EHblkDsc* ehGetBlockExnFlowDsc(BasicBlock* block);
2275 EHblkDsc* ehIsBlockTryLast(BasicBlock* block);
2276 EHblkDsc* ehIsBlockHndLast(BasicBlock* block);
2277 bool ehIsBlockEHLast(BasicBlock* block);
2279 bool ehBlockHasExnFlowDsc(BasicBlock* block);
2281 // Return the region index of the most nested EH region this block is in.
2282 unsigned ehGetMostNestedRegionIndex(BasicBlock* block, bool* inTryRegion);
2284 // Find the true enclosing try index, ignoring 'mutual protect' try. Uses IL ranges to check.
2285 unsigned ehTrueEnclosingTryIndexIL(unsigned regionIndex);
2287 // Return the index of the most nested enclosing region for a particular EH region. Returns NO_ENCLOSING_INDEX
2288 // if there is no enclosing region. If the returned index is not NO_ENCLOSING_INDEX, then '*inTryRegion'
2289 // is set to 'true' if the enclosing region is a 'try', or 'false' if the enclosing region is a handler.
2290 // (It can never be a filter.)
2291 unsigned ehGetEnclosingRegionIndex(unsigned regionIndex, bool* inTryRegion);
2293 // A block has been deleted. Update the EH table appropriately.
2294 void ehUpdateForDeletedBlock(BasicBlock* block);
2296 // Determine whether a block can be deleted while preserving the EH normalization rules.
2297 bool ehCanDeleteEmptyBlock(BasicBlock* block);
2299 // Update the 'last' pointers in the EH table to reflect new or deleted blocks in an EH region.
2300 void ehUpdateLastBlocks(BasicBlock* oldLast, BasicBlock* newLast);
2302 // For a finally handler, find the region index that the BBJ_CALLFINALLY lives in that calls the handler,
2303 // or NO_ENCLOSING_INDEX if the BBJ_CALLFINALLY lives in the main function body. Normally, the index
2304 // is the same index as the handler (and the BBJ_CALLFINALLY lives in the 'try' region), but for AMD64 the
2305 // BBJ_CALLFINALLY lives in the enclosing try or handler region, whichever is more nested, or the main function
2306 // body. If the returned index is not NO_ENCLOSING_INDEX, then '*inTryRegion' is set to 'true' if the
2307 // BBJ_CALLFINALLY lives in the returned index's 'try' region, or 'false' if lives in the handler region. (It never
2308 // lives in a filter.)
2309 unsigned ehGetCallFinallyRegionIndex(unsigned finallyIndex, bool* inTryRegion);
2311 // Find the range of basic blocks in which all BBJ_CALLFINALLY will be found that target the 'finallyIndex' region's
2312 // handler. Set begBlk to the first block, and endBlk to the block after the last block of the range
2313 // (nullptr if the last block is the last block in the program).
2314 // Precondition: 'finallyIndex' is the EH region of a try/finally clause.
2315 void ehGetCallFinallyBlockRange(unsigned finallyIndex, BasicBlock** begBlk, BasicBlock** endBlk);
2318 // Given a BBJ_CALLFINALLY block and the EH region index of the finally it is calling, return
2319 // 'true' if the BBJ_CALLFINALLY is in the correct EH region.
2320 bool ehCallFinallyInCorrectRegion(BasicBlock* blockCallFinally, unsigned finallyIndex);
2323 #if defined(FEATURE_EH_FUNCLETS)
2324 // Do we need a PSPSym in the main function? For codegen purposes, we only need one
2325 // if there is a filter that protects a region with a nested EH clause (such as a
2326 // try/catch nested in the 'try' body of a try/filter/filter-handler). See
2327 // genFuncletProlog() for more details. However, the VM seems to use it for more
2328 // purposes, maybe including debugging. Until we are sure otherwise, always create
2329 // a PSPSym for functions with any EH.
2330 bool ehNeedsPSPSym() const
2335 return compHndBBtabCount > 0;
2336 #endif // TARGET_X86
2339 bool ehAnyFunclets(); // Are there any funclets in this function?
2340 unsigned ehFuncletCount(); // Return the count of funclets in the function
2342 unsigned bbThrowIndex(BasicBlock* blk); // Get the index to use as the cache key for sharing throw blocks
2344 #else // !FEATURE_EH_FUNCLETS
2346 bool ehAnyFunclets()
2350 unsigned ehFuncletCount()
2355 unsigned bbThrowIndex(BasicBlock* blk)
2357 return blk->bbTryIndex;
2358 } // Get the index to use as the cache key for sharing throw blocks
2359 #endif // !FEATURE_EH_FUNCLETS
2361 // Returns a FlowEdge representing the "EH predecessors" of "blk". These are the normal predecessors of
2362 // "blk", plus one special case: if "blk" is the first block of a handler, considers the predecessor(s) of the
2363 // first block of the corresponding try region to be "EH predecessors". (If there is a single such predecessor,
2364 // for example, we want to consider that the immediate dominator of the catch clause start block, so it's
2365 // convenient to also consider it a predecessor.)
2366 FlowEdge* BlockPredsWithEH(BasicBlock* blk);
2368 // This table is useful for memoization of the method above.
2369 typedef JitHashTable<BasicBlock*, JitPtrKeyFuncs<BasicBlock>, FlowEdge*> BlockToFlowEdgeMap;
2370 BlockToFlowEdgeMap* m_blockToEHPreds;
2371 BlockToFlowEdgeMap* GetBlockToEHPreds()
2373 if (m_blockToEHPreds == nullptr)
2375 m_blockToEHPreds = new (getAllocator()) BlockToFlowEdgeMap(getAllocator());
2377 return m_blockToEHPreds;
2380 void* ehEmitCookie(BasicBlock* block);
2381 UNATIVE_OFFSET ehCodeOffset(BasicBlock* block);
2383 EHblkDsc* ehInitHndRange(BasicBlock* src, IL_OFFSET* hndBeg, IL_OFFSET* hndEnd, bool* inFilter);
2385 EHblkDsc* ehInitTryRange(BasicBlock* src, IL_OFFSET* tryBeg, IL_OFFSET* tryEnd);
2387 EHblkDsc* ehInitHndBlockRange(BasicBlock* blk, BasicBlock** hndBeg, BasicBlock** hndLast, bool* inFilter);
2389 EHblkDsc* ehInitTryBlockRange(BasicBlock* blk, BasicBlock** tryBeg, BasicBlock** tryLast);
2391 void fgSetTryBeg(EHblkDsc* handlerTab, BasicBlock* newTryBeg);
2393 void fgSetTryEnd(EHblkDsc* handlerTab, BasicBlock* newTryLast);
2395 void fgSetHndEnd(EHblkDsc* handlerTab, BasicBlock* newHndLast);
2397 void fgSkipRmvdBlocks(EHblkDsc* handlerTab);
2399 void fgAllocEHTable();
2401 void fgRemoveEHTableEntry(unsigned XTnum);
2403 #if defined(FEATURE_EH_FUNCLETS)
2405 EHblkDsc* fgAddEHTableEntry(unsigned XTnum);
2407 #endif // FEATURE_EH_FUNCLETS
2409 void fgSortEHTable();
2411 // Causes the EH table to obey some well-formedness conditions, by inserting
2412 // empty BB's when necessary:
2413 // * No block is both the first block of a handler and the first block of a try.
2414 // * No block is the first block of multiple 'try' regions.
2415 // * No block is the last block of multiple EH regions.
2416 void fgNormalizeEH();
2417 bool fgNormalizeEHCase1();
2418 bool fgNormalizeEHCase2();
2419 bool fgNormalizeEHCase3();
2421 bool fgCreateFiltersForGenericExceptions();
2423 void fgCheckForLoopsInHandlers();
2426 void dispIncomingEHClause(unsigned num, const CORINFO_EH_CLAUSE& clause);
2427 void dispOutgoingEHClause(unsigned num, const CORINFO_EH_CLAUSE& clause);
2428 void fgVerifyHandlerTab();
2429 void fgDispHandlerTab();
2432 bool fgNeedToSortEHTable;
2434 void verInitEHTree(unsigned numEHClauses);
2435 void verInsertEhNode(CORINFO_EH_CLAUSE* clause, EHblkDsc* handlerTab);
2436 void verInsertEhNodeInTree(EHNodeDsc** ppRoot, EHNodeDsc* node);
2437 void verInsertEhNodeParent(EHNodeDsc** ppRoot, EHNodeDsc* node);
2438 void verCheckNestingLevel(EHNodeDsc* initRoot);
2441 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
2442 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
2444 XX GenTree and BasicBlock XX
2446 XX Functions to allocate and display the GenTrees and BasicBlocks XX
2448 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
2449 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
2452 // Functions to create nodes
2453 Statement* gtNewStmt(GenTree* expr = nullptr);
2454 Statement* gtNewStmt(GenTree* expr, const DebugInfo& di);
2457 GenTree* gtNewOperNode(genTreeOps oper, var_types type, GenTree* op1);
2459 // For binary opers.
2460 GenTreeOp* gtNewOperNode(genTreeOps oper, var_types type, GenTree* op1, GenTree* op2);
2462 GenTreeCC* gtNewCC(genTreeOps oper, var_types type, GenCondition cond);
2463 GenTreeOpCC* gtNewOperCC(genTreeOps oper, var_types type, GenCondition cond, GenTree* op1, GenTree* op2);
2465 GenTreeColon* gtNewColonNode(var_types type, GenTree* thenNode, GenTree* elseNode);
2466 GenTreeQmark* gtNewQmarkNode(var_types type, GenTree* cond, GenTreeColon* colon);
2468 GenTree* gtNewLargeOperNode(genTreeOps oper,
2469 var_types type = TYP_I_IMPL,
2470 GenTree* op1 = nullptr,
2471 GenTree* op2 = nullptr);
2473 GenTreeIntCon* gtNewIconNode(ssize_t value, var_types type = TYP_INT);
2474 GenTreeIntCon* gtNewIconNode(unsigned fieldOffset, FieldSeq* fieldSeq);
2475 GenTreeIntCon* gtNewNull();
2476 GenTreeIntCon* gtNewTrue();
2477 GenTreeIntCon* gtNewFalse();
2479 GenTree* gtNewPhysRegNode(regNumber reg, var_types type);
2481 GenTree* gtNewJmpTableNode();
2483 GenTree* gtNewIndOfIconHandleNode(var_types indType, size_t value, GenTreeFlags iconFlags, bool isInvariant);
2485 GenTreeIntCon* gtNewIconHandleNode(size_t value, GenTreeFlags flags, FieldSeq* fields = nullptr);
2487 GenTreeFlags gtTokenToIconFlags(unsigned token);
2489 GenTree* gtNewIconEmbHndNode(void* value, void* pValue, GenTreeFlags flags, void* compileTimeHandle);
2491 GenTree* gtNewIconEmbScpHndNode(CORINFO_MODULE_HANDLE scpHnd);
2492 GenTree* gtNewIconEmbClsHndNode(CORINFO_CLASS_HANDLE clsHnd);
2493 GenTree* gtNewIconEmbMethHndNode(CORINFO_METHOD_HANDLE methHnd);
2494 GenTree* gtNewIconEmbFldHndNode(CORINFO_FIELD_HANDLE fldHnd);
2496 GenTree* gtNewStringLiteralNode(InfoAccessType iat, void* pValue);
2497 GenTreeIntCon* gtNewStringLiteralLength(GenTreeStrCon* node);
2499 GenTree* gtNewLconNode(__int64 value);
2501 GenTree* gtNewDconNodeF(float value);
2502 GenTree* gtNewDconNodeD(double value);
2503 GenTree* gtNewDconNode(float value, var_types type) = delete; // use gtNewDconNodeF instead
2504 GenTree* gtNewDconNode(double value, var_types type);
2506 GenTree* gtNewSconNode(int CPX, CORINFO_MODULE_HANDLE scpHandle);
2508 GenTreeVecCon* gtNewVconNode(var_types type);
2510 GenTreeVecCon* gtNewVconNode(var_types type, void* data);
2512 GenTree* gtNewAllBitsSetConNode(var_types type);
2514 GenTree* gtNewZeroConNode(var_types type);
2516 GenTree* gtNewOneConNode(var_types type, var_types simdBaseType = TYP_UNDEF);
2518 GenTree* gtNewGenericCon(var_types type, uint8_t* cnsVal);
2520 GenTree* gtNewConWithPattern(var_types type, uint8_t pattern);
2522 GenTreeLclVar* gtNewStoreLclVarNode(unsigned lclNum, GenTree* data);
2524 GenTreeLclFld* gtNewStoreLclFldNode(
2525 unsigned lclNum, var_types type, ClassLayout* layout, unsigned offset, GenTree* data);
2527 GenTreeLclFld* gtNewStoreLclFldNode(unsigned lclNum, var_types type, unsigned offset, GenTree* data)
2529 return gtNewStoreLclFldNode(lclNum, type, (type == TYP_STRUCT) ? data->GetLayout(this) : nullptr, offset, data);
2532 GenTree* gtNewPutArgReg(var_types type, GenTree* arg, regNumber argReg);
2534 GenTree* gtNewBitCastNode(var_types type, GenTree* arg);
2537 GenTreeCall* gtNewCallNode(gtCallTypes callType,
2538 CORINFO_METHOD_HANDLE handle,
2540 const DebugInfo& di = DebugInfo());
2542 GenTreeCall* gtNewIndCallNode(GenTree* addr, var_types type, const DebugInfo& di = DebugInfo());
2544 GenTreeCall* gtNewHelperCallNode(
2545 unsigned helper, var_types type, GenTree* arg1 = nullptr, GenTree* arg2 = nullptr, GenTree* arg3 = nullptr);
2547 GenTreeCall* gtNewRuntimeLookupHelperCallNode(CORINFO_RUNTIME_LOOKUP* pRuntimeLookup,
2549 void* compileTimeHandle);
2551 GenTreeLclVar* gtNewLclvNode(unsigned lnum, var_types type DEBUGARG(IL_OFFSET offs = BAD_IL_OFFSET));
2552 GenTreeLclVar* gtNewLclVarNode(unsigned lclNum, var_types type = TYP_UNDEF);
2553 GenTreeLclVar* gtNewLclLNode(unsigned lnum, var_types type DEBUGARG(IL_OFFSET offs = BAD_IL_OFFSET));
2555 GenTreeLclFld* gtNewLclVarAddrNode(unsigned lclNum, var_types type = TYP_I_IMPL);
2556 GenTreeLclFld* gtNewLclAddrNode(unsigned lclNum, unsigned lclOffs, var_types type = TYP_I_IMPL);
2558 GenTreeConditional* gtNewConditionalNode(
2559 genTreeOps oper, GenTree* cond, GenTree* op1, GenTree* op2, var_types type);
2562 void SetOpLclRelatedToSIMDIntrinsic(GenTree* op);
2565 #ifdef FEATURE_HW_INTRINSICS
2566 GenTreeHWIntrinsic* gtNewSimdHWIntrinsicNode(var_types type,
2567 NamedIntrinsic hwIntrinsicID,
2568 CorInfoType simdBaseJitType,
2570 GenTreeHWIntrinsic* gtNewSimdHWIntrinsicNode(var_types type,
2572 NamedIntrinsic hwIntrinsicID,
2573 CorInfoType simdBaseJitType,
2575 GenTreeHWIntrinsic* gtNewSimdHWIntrinsicNode(var_types type,
2578 NamedIntrinsic hwIntrinsicID,
2579 CorInfoType simdBaseJitType,
2581 GenTreeHWIntrinsic* gtNewSimdHWIntrinsicNode(var_types type,
2585 NamedIntrinsic hwIntrinsicID,
2586 CorInfoType simdBaseJitType,
2588 GenTreeHWIntrinsic* gtNewSimdHWIntrinsicNode(var_types type,
2593 NamedIntrinsic hwIntrinsicID,
2594 CorInfoType simdBaseJitType,
2596 GenTreeHWIntrinsic* gtNewSimdHWIntrinsicNode(var_types type,
2598 size_t operandCount,
2599 NamedIntrinsic hwIntrinsicID,
2600 CorInfoType simdBaseJitType,
2602 GenTreeHWIntrinsic* gtNewSimdHWIntrinsicNode(var_types type,
2603 IntrinsicNodeBuilder&& nodeBuilder,
2604 NamedIntrinsic hwIntrinsicID,
2605 CorInfoType simdBaseJitType,
2608 GenTreeHWIntrinsic* gtNewSimdAsHWIntrinsicNode(var_types type,
2609 NamedIntrinsic hwIntrinsicID,
2610 CorInfoType simdBaseJitType,
2613 return gtNewSimdHWIntrinsicNode(type, hwIntrinsicID, simdBaseJitType, simdSize);
2616 GenTreeHWIntrinsic* gtNewSimdAsHWIntrinsicNode(
2617 var_types type, GenTree* op1, NamedIntrinsic hwIntrinsicID, CorInfoType simdBaseJitType, unsigned simdSize)
2619 return gtNewSimdHWIntrinsicNode(type, op1, hwIntrinsicID, simdBaseJitType, simdSize);
2622 GenTreeHWIntrinsic* gtNewSimdAsHWIntrinsicNode(var_types type,
2625 NamedIntrinsic hwIntrinsicID,
2626 CorInfoType simdBaseJitType,
2629 return gtNewSimdHWIntrinsicNode(type, op1, op2, hwIntrinsicID, simdBaseJitType, simdSize);
2632 GenTreeHWIntrinsic* gtNewSimdAsHWIntrinsicNode(var_types type,
2636 NamedIntrinsic hwIntrinsicID,
2637 CorInfoType simdBaseJitType,
2640 return gtNewSimdHWIntrinsicNode(type, op1, op2, op3, hwIntrinsicID, simdBaseJitType, simdSize);
2643 GenTree* gtNewSimdAbsNode(
2644 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2646 GenTree* gtNewSimdBinOpNode(genTreeOps op,
2650 CorInfoType simdBaseJitType,
2653 GenTree* gtNewSimdCeilNode(
2654 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2656 GenTree* gtNewSimdCmpOpNode(genTreeOps op,
2660 CorInfoType simdBaseJitType,
2663 GenTree* gtNewSimdCmpOpAllNode(genTreeOps op,
2667 CorInfoType simdBaseJitType,
2670 GenTree* gtNewSimdCmpOpAnyNode(genTreeOps op,
2674 CorInfoType simdBaseJitType,
2677 GenTree* gtNewSimdCndSelNode(var_types type,
2681 CorInfoType simdBaseJitType,
2684 GenTree* gtNewSimdCreateBroadcastNode(
2685 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2687 GenTree* gtNewSimdCreateScalarNode(
2688 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2690 GenTree* gtNewSimdCreateScalarUnsafeNode(
2691 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2693 GenTree* gtNewSimdDotProdNode(var_types type,
2696 CorInfoType simdBaseJitType,
2699 GenTree* gtNewSimdFloorNode(
2700 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2702 GenTree* gtNewSimdGetElementNode(var_types type,
2705 CorInfoType simdBaseJitType,
2708 GenTree* gtNewSimdGetLowerNode(var_types type,
2710 CorInfoType simdBaseJitType,
2713 GenTree* gtNewSimdGetUpperNode(var_types type,
2715 CorInfoType simdBaseJitType,
2718 GenTree* gtNewSimdLoadNode(
2719 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2721 GenTree* gtNewSimdLoadAlignedNode(
2722 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2724 GenTree* gtNewSimdLoadNonTemporalNode(
2725 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2727 GenTree* gtNewSimdMaxNode(var_types type,
2730 CorInfoType simdBaseJitType,
2733 GenTree* gtNewSimdMinNode(var_types type,
2736 CorInfoType simdBaseJitType,
2739 GenTree* gtNewSimdNarrowNode(var_types type,
2742 CorInfoType simdBaseJitType,
2745 GenTree* gtNewSimdShuffleNode(var_types type,
2748 CorInfoType simdBaseJitType,
2751 GenTree* gtNewSimdSqrtNode(
2752 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2754 GenTree* gtNewSimdStoreNode(
2755 GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize);
2757 GenTree* gtNewSimdStoreAlignedNode(
2758 GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize);
2760 GenTree* gtNewSimdStoreNonTemporalNode(
2761 GenTree* op1, GenTree* op2, CorInfoType simdBaseJitType, unsigned simdSize);
2763 GenTree* gtNewSimdSumNode(
2764 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2766 #if defined(TARGET_XARCH)
2767 GenTree* gtNewSimdTernaryLogicNode(var_types type,
2772 CorInfoType simdBaseJitType,
2774 #endif // TARGET_XARCH
2776 GenTree* gtNewSimdUnOpNode(genTreeOps op,
2779 CorInfoType simdBaseJitType,
2782 GenTree* gtNewSimdWidenLowerNode(
2783 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2785 GenTree* gtNewSimdWidenUpperNode(
2786 var_types type, GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize);
2788 GenTree* gtNewSimdWithElementNode(var_types type,
2792 CorInfoType simdBaseJitType,
2795 GenTree* gtNewSimdWithLowerNode(var_types type,
2798 CorInfoType simdBaseJitType,
2801 GenTree* gtNewSimdWithUpperNode(var_types type,
2804 CorInfoType simdBaseJitType,
2807 GenTreeHWIntrinsic* gtNewScalarHWIntrinsicNode(var_types type, NamedIntrinsic hwIntrinsicID);
2808 GenTreeHWIntrinsic* gtNewScalarHWIntrinsicNode(var_types type, GenTree* op1, NamedIntrinsic hwIntrinsicID);
2809 GenTreeHWIntrinsic* gtNewScalarHWIntrinsicNode(var_types type,
2812 NamedIntrinsic hwIntrinsicID);
2813 GenTreeHWIntrinsic* gtNewScalarHWIntrinsicNode(
2814 var_types type, GenTree* op1, GenTree* op2, GenTree* op3, NamedIntrinsic hwIntrinsicID);
2815 CorInfoType getBaseJitTypeFromArgIfNeeded(NamedIntrinsic intrinsic,
2816 CORINFO_CLASS_HANDLE clsHnd,
2817 CORINFO_SIG_INFO* sig,
2818 CorInfoType simdBaseJitType);
2821 GenTreeFieldList* gtConvertTableOpToFieldList(GenTree* op, unsigned fieldCount);
2823 #endif // FEATURE_HW_INTRINSICS
2825 GenTree* gtNewMustThrowException(unsigned helper, var_types type, CORINFO_CLASS_HANDLE clsHnd);
2827 GenTreeLclFld* gtNewLclFldNode(unsigned lnum, var_types type, unsigned offset);
2828 GenTreeRetExpr* gtNewInlineCandidateReturnExpr(GenTreeCall* inlineCandidate, var_types type);
2830 GenTreeFieldAddr* gtNewFieldAddrNode(var_types type,
2831 CORINFO_FIELD_HANDLE fldHnd,
2832 GenTree* obj = nullptr,
2835 GenTreeFieldAddr* gtNewFieldAddrNode(CORINFO_FIELD_HANDLE fldHnd, GenTree* obj, unsigned offset)
2837 return gtNewFieldAddrNode(varTypeIsGC(obj) ? TYP_BYREF : TYP_I_IMPL, fldHnd, obj, offset);
2840 GenTreeIndexAddr* gtNewIndexAddr(GenTree* arrayOp,
2843 CORINFO_CLASS_HANDLE elemClassHandle,
2844 unsigned firstElemOffset,
2845 unsigned lengthOffset);
2847 GenTreeIndexAddr* gtNewArrayIndexAddr(GenTree* arrayOp,
2850 CORINFO_CLASS_HANDLE elemClassHandle);
2852 GenTreeIndir* gtNewIndexIndir(GenTreeIndexAddr* indexAddr);
2854 void gtAnnotateNewArrLen(GenTree* arrLen, BasicBlock* block);
2856 GenTreeArrLen* gtNewArrLen(var_types typ, GenTree* arrayOp, int lenOffset, BasicBlock* block);
2858 GenTreeMDArr* gtNewMDArrLen(GenTree* arrayOp, unsigned dim, unsigned rank, BasicBlock* block);
2860 GenTreeMDArr* gtNewMDArrLowerBound(GenTree* arrayOp, unsigned dim, unsigned rank, BasicBlock* block);
2862 void gtInitializeStoreNode(GenTree* store, GenTree* data);
2864 void gtInitializeIndirNode(GenTreeIndir* indir, GenTreeFlags indirFlags);
2866 GenTreeBlk* gtNewBlkIndir(ClassLayout* layout, GenTree* addr, GenTreeFlags indirFlags = GTF_EMPTY);
2868 GenTreeIndir* gtNewIndir(var_types typ, GenTree* addr, GenTreeFlags indirFlags = GTF_EMPTY);
2870 GenTreeBlk* gtNewStoreBlkNode(
2871 ClassLayout* layout, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY);
2873 GenTreeStoreInd* gtNewStoreIndNode(
2874 var_types type, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY);
2876 GenTree* gtNewLoadValueNode(
2877 var_types type, ClassLayout* layout, GenTree* addr, GenTreeFlags indirFlags = GTF_EMPTY);
2879 GenTree* gtNewLoadValueNode(ClassLayout* layout, GenTree* addr, GenTreeFlags indirFlags = GTF_EMPTY)
2881 return gtNewLoadValueNode(layout->GetType(), layout, addr, indirFlags);
2884 GenTree* gtNewLoadValueNode(var_types type, GenTree* addr, GenTreeFlags indirFlags = GTF_EMPTY)
2886 return gtNewLoadValueNode(type, nullptr, addr, indirFlags);
2889 GenTree* gtNewStoreValueNode(
2890 var_types type, ClassLayout* layout, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY);
2892 GenTree* gtNewStoreValueNode(ClassLayout* layout, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY)
2894 return gtNewStoreValueNode(layout->GetType(), layout, addr, data, indirFlags);
2897 GenTree* gtNewStoreValueNode(var_types type, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY)
2899 return gtNewStoreValueNode(type, nullptr, addr, data, indirFlags);
2902 GenTree* gtNewNullCheck(GenTree* addr, BasicBlock* basicBlock);
2904 var_types gtTypeForNullCheck(GenTree* tree);
2905 void gtChangeOperToNullCheck(GenTree* tree, BasicBlock* block);
2907 GenTree* gtNewTempStore(unsigned tmp,
2909 unsigned curLevel = CHECK_SPILL_NONE,
2910 Statement** pAfterStmt = nullptr,
2911 const DebugInfo& di = DebugInfo(),
2912 BasicBlock* block = nullptr);
2914 GenTree* gtNewRefCOMfield(GenTree* objPtr,
2915 CORINFO_RESOLVED_TOKEN* pResolvedToken,
2916 CORINFO_ACCESS_FLAGS access,
2917 CORINFO_FIELD_INFO* pFieldInfo,
2921 GenTree* gtNewNothingNode();
2923 GenTree* gtUnusedValNode(GenTree* expr);
2925 GenTree* gtNewKeepAliveNode(GenTree* op);
2927 GenTreeCast* gtNewCastNode(var_types typ, GenTree* op1, bool fromUnsigned, var_types castType);
2929 GenTreeCast* gtNewCastNodeL(var_types typ, GenTree* op1, bool fromUnsigned, var_types castType);
2931 GenTreeAllocObj* gtNewAllocObjNode(
2932 unsigned int helper, bool helperHasSideEffects, CORINFO_CLASS_HANDLE clsHnd, var_types type, GenTree* op1);
2934 GenTreeAllocObj* gtNewAllocObjNode(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool useParent);
2936 GenTree* gtNewRuntimeLookup(CORINFO_GENERIC_HANDLE hnd, CorInfoGenericHandleType hndTyp, GenTree* lookupTree);
2938 GenTreeIndir* gtNewMethodTableLookup(GenTree* obj);
2940 //------------------------------------------------------------------------
2941 // Other GenTree functions
2943 GenTree* gtClone(GenTree* tree, bool complexOK = false);
2945 // If `tree` is a lclVar with lclNum `varNum`, return an IntCns with value `varVal`; otherwise,
2946 // create a copy of `tree`, adding specified flags, replacing uses of lclVar `deepVarNum` with
2947 // IntCnses with value `deepVarVal`.
2948 GenTree* gtCloneExpr(
2949 GenTree* tree, GenTreeFlags addFlags, unsigned varNum, int varVal, unsigned deepVarNum, int deepVarVal);
2951 // Create a copy of `tree`, optionally adding specified flags, and optionally mapping uses of local
2952 // `varNum` to int constants with value `varVal`.
2953 GenTree* gtCloneExpr(GenTree* tree,
2954 GenTreeFlags addFlags = GTF_EMPTY,
2955 unsigned varNum = BAD_VAR_NUM,
2958 return gtCloneExpr(tree, addFlags, varNum, varVal, varNum, varVal);
2961 Statement* gtCloneStmt(Statement* stmt)
2963 GenTree* exprClone = gtCloneExpr(stmt->GetRootNode());
2964 return gtNewStmt(exprClone, stmt->GetDebugInfo());
2967 // Internal helper for cloning a call
2968 GenTreeCall* gtCloneExprCallHelper(GenTreeCall* call,
2969 GenTreeFlags addFlags = GTF_EMPTY,
2970 unsigned deepVarNum = BAD_VAR_NUM,
2971 int deepVarVal = 0);
2973 // Create copy of an inline or guarded devirtualization candidate tree.
2974 GenTreeCall* gtCloneCandidateCall(GenTreeCall* call);
2976 void gtUpdateSideEffects(Statement* stmt, GenTree* tree);
2978 void gtUpdateTreeAncestorsSideEffects(GenTree* tree);
2980 void gtUpdateStmtSideEffects(Statement* stmt);
2982 void gtUpdateNodeSideEffects(GenTree* tree);
2984 void gtUpdateNodeOperSideEffects(GenTree* tree);
2986 // Returns "true" iff the complexity (not formally defined, but first interpretation
2987 // is #of nodes in subtree) of "tree" is greater than "limit".
2988 // (This is somewhat redundant with the "GetCostEx()/GetCostSz()" fields, but can be used
2989 // before they have been set.)
2990 bool gtComplexityExceeds(GenTree* tree, unsigned limit);
2992 GenTree* gtReverseCond(GenTree* tree);
2994 static bool gtHasRef(GenTree* tree, unsigned lclNum);
2996 bool gtHasLocalsWithAddrOp(GenTree* tree);
2997 bool gtHasAddressExposedLocals(GenTree* tree);
2999 unsigned gtSetCallArgsOrder(CallArgs* args, bool lateArgs, int* callCostEx, int* callCostSz);
3000 unsigned gtSetMultiOpOrder(GenTreeMultiOp* multiOp);
3002 void gtWalkOp(GenTree** op1, GenTree** op2, GenTree* base, bool constOnly);
3005 unsigned gtHashValue(GenTree* tree);
3007 GenTree* gtWalkOpEffectiveVal(GenTree* op);
3010 void gtPrepareCost(GenTree* tree);
3011 bool gtIsLikelyRegVar(GenTree* tree);
3012 void gtGetLclVarNodeCost(GenTreeLclVar* node, int* pCostEx, int* pCostSz, bool isLikelyRegVar);
3013 void gtGetLclFldNodeCost(GenTreeLclFld* node, int* pCostEx, int* pCostSz);
3014 bool gtGetIndNodeCost(GenTreeIndir* node, int* pCostEx, int* pCostSz);
3016 // Returns true iff the secondNode can be swapped with firstNode.
3017 bool gtCanSwapOrder(GenTree* firstNode, GenTree* secondNode);
3019 // Given an address expression, compute its costs and addressing mode opportunities,
3020 // and mark addressing mode candidates as GTF_DONT_CSE.
3021 // TODO-Throughput - Consider actually instantiating these early, to avoid
3022 // having to re-run the algorithm that looks for them (might also improve CQ).
3023 bool gtMarkAddrMode(GenTree* addr, int* costEx, int* costSz, var_types type);
3025 unsigned gtSetEvalOrder(GenTree* tree);
3027 void gtSetStmtInfo(Statement* stmt);
3029 // Returns "true" iff "node" has any of the side effects in "flags".
3030 bool gtNodeHasSideEffects(GenTree* node, GenTreeFlags flags);
3032 // Returns "true" iff "tree" or its (transitive) children have any of the side effects in "flags".
3033 bool gtTreeHasSideEffects(GenTree* tree, GenTreeFlags flags);
3035 void gtExtractSideEffList(GenTree* expr,
3037 GenTreeFlags GenTreeFlags = GTF_SIDE_EFFECT,
3038 bool ignoreRoot = false);
3041 BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitPointUse);
3043 // Static fields of struct types (and sometimes the types that those are reduced to) are represented by having the
3044 // static field contain an object pointer to the boxed struct. This simplifies the GC implementation...but
3045 // complicates the JIT somewhat. This predicate returns "true" iff a node with type "fieldNodeType", representing
3046 // the given "fldHnd", is such an object pointer.
3047 bool gtIsStaticFieldPtrToBoxedStruct(var_types fieldNodeType, CORINFO_FIELD_HANDLE fldHnd);
3049 bool gtStoreDefinesField(
3050 LclVarDsc* fieldVarDsc, ssize_t offset, unsigned size, ssize_t* pFieldStoreOffset, unsigned* pFieldStoreSize);
3052 void gtPeelOffsets(GenTree** addr, target_ssize_t* offset, FieldSeq** fldSeq);
3054 // Return true if call is a recursive call; return false otherwise.
3055 // Note when inlining, this looks for calls back to the root method.
3056 bool gtIsRecursiveCall(GenTreeCall* call)
3058 return gtIsRecursiveCall(call->gtCallMethHnd);
3061 bool gtIsRecursiveCall(CORINFO_METHOD_HANDLE callMethodHandle)
3063 return (callMethodHandle == impInlineRoot()->info.compMethodHnd);
3066 //-------------------------------------------------------------------------
3068 GenTree* gtFoldExpr(GenTree* tree);
3069 GenTree* gtFoldExprConst(GenTree* tree);
3070 GenTree* gtFoldIndirConst(GenTreeIndir* indir);
3071 GenTree* gtFoldExprSpecial(GenTree* tree);
3072 GenTree* gtFoldBoxNullable(GenTree* tree);
3073 GenTree* gtFoldExprCompare(GenTree* tree);
3074 GenTree* gtFoldExprConditional(GenTree* tree);
3075 GenTree* gtFoldExprCall(GenTreeCall* call);
3076 GenTree* gtFoldTypeCompare(GenTree* tree);
3077 GenTree* gtFoldTypeEqualityCall(bool isEq, GenTree* op1, GenTree* op2);
3079 // Options to control behavior of gtTryRemoveBoxUpstreamEffects
3080 enum BoxRemovalOptions
3082 BR_REMOVE_AND_NARROW, // remove effects, minimize remaining work, return possibly narrowed source tree
3083 BR_REMOVE_AND_NARROW_WANT_TYPE_HANDLE, // remove effects and minimize remaining work, return type handle tree
3084 BR_REMOVE_BUT_NOT_NARROW, // remove effects, return original source tree
3085 BR_DONT_REMOVE, // check if removal is possible, return copy source tree
3086 BR_DONT_REMOVE_WANT_TYPE_HANDLE, // check if removal is possible, return type handle tree
3087 BR_MAKE_LOCAL_COPY // revise box to copy to temp local and return local's address
3090 GenTree* gtTryRemoveBoxUpstreamEffects(GenTree* tree, BoxRemovalOptions options = BR_REMOVE_AND_NARROW);
3091 GenTree* gtOptimizeEnumHasFlag(GenTree* thisOp, GenTree* flagOp);
3093 //-------------------------------------------------------------------------
3094 // Get the handle for a ref type.
3095 CORINFO_CLASS_HANDLE gtGetClassHandle(GenTree* tree, bool* pIsExact, bool* pIsNonNull);
3096 // Get the class handle for an helper call
3097 CORINFO_CLASS_HANDLE gtGetHelperCallClassHandle(GenTreeCall* call, bool* pIsExact, bool* pIsNonNull);
3098 // Get the element handle for an array of ref type.
3099 CORINFO_CLASS_HANDLE gtGetArrayElementClassHandle(GenTree* array);
3100 // Get a class handle from a helper call argument
3101 CORINFO_CLASS_HANDLE gtGetHelperArgClassHandle(GenTree* array);
3102 // Get the class handle for a field
3103 CORINFO_CLASS_HANDLE gtGetFieldClassHandle(CORINFO_FIELD_HANDLE fieldHnd, bool* pIsExact, bool* pIsNonNull);
3104 // Check if this tree is a typeof()
3105 bool gtIsTypeof(GenTree* tree, CORINFO_CLASS_HANDLE* handle = nullptr);
3107 GenTreeLclVarCommon* gtCallGetDefinedRetBufLclAddr(GenTreeCall* call);
3109 //-------------------------------------------------------------------------
3110 // Functions to display the trees
3113 void gtDispNode(GenTree* tree, IndentStack* indentStack, _In_z_ const char* msg, bool isLIR);
3115 void gtDispConst(GenTree* tree);
3116 void gtDispLeaf(GenTree* tree, IndentStack* indentStack);
3117 void gtDispLocal(GenTreeLclVarCommon* tree, IndentStack* indentStack);
3118 void gtDispNodeName(GenTree* tree);
3119 #if FEATURE_MULTIREG_RET
3120 unsigned gtDispMultiRegCount(GenTree* tree);
3122 void gtDispRegVal(GenTree* tree);
3123 void gtDispVN(GenTree* tree);
3124 void gtDispCommonEndLine(GenTree* tree);
3136 void gtDispChild(GenTree* child,
3137 IndentStack* indentStack,
3139 _In_opt_ const char* msg = nullptr,
3140 bool topOnly = false);
3141 void gtDispTree(GenTree* tree,
3142 IndentStack* indentStack = nullptr,
3143 _In_opt_ const char* msg = nullptr,
3144 bool topOnly = false,
3145 bool isLIR = false);
3146 void gtGetLclVarNameInfo(unsigned lclNum, const char** ilKindOut, const char** ilNameOut, unsigned* ilNumOut);
3147 int gtGetLclVarName(unsigned lclNum, char* buf, unsigned buf_remaining);
3148 char* gtGetLclVarName(unsigned lclNum);
3149 void gtDispLclVar(unsigned lclNum, bool padForBiggestDisp = true);
3150 void gtDispLclVarStructType(unsigned lclNum);
3151 void gtDispSsaName(unsigned lclNum, unsigned ssaNum, bool isDef);
3152 void gtDispClassLayout(ClassLayout* layout, var_types type);
3153 void gtDispILLocation(const ILLocation& loc);
3154 void gtDispStmt(Statement* stmt, const char* msg = nullptr);
3155 void gtDispBlockStmts(BasicBlock* block);
3156 void gtPrintArgPrefix(GenTreeCall* call, CallArg* arg, char** bufp, unsigned* bufLength);
3157 const char* gtGetWellKnownArgNameForArgMsg(WellKnownArg arg);
3158 void gtGetArgMsg(GenTreeCall* call, CallArg* arg, char* bufp, unsigned bufLength);
3159 void gtGetLateArgMsg(GenTreeCall* call, CallArg* arg, char* bufp, unsigned bufLength);
3160 void gtDispArgList(GenTreeCall* call, GenTree* lastCallOperand, IndentStack* indentStack);
3161 void gtDispFieldSeq(FieldSeq* fieldSeq, ssize_t offset);
3163 void gtDispRange(LIR::ReadOnlyRange const& range);
3165 void gtDispTreeRange(LIR::Range& containingRange, GenTree* tree);
3167 void gtDispLIRNode(GenTree* node, const char* prefixMsg = nullptr);
3179 typedef fgWalkResult(fgWalkPreFn)(GenTree** pTree, fgWalkData* data);
3180 typedef fgWalkResult(fgWalkPostFn)(GenTree** pTree, fgWalkData* data);
3182 static fgWalkPreFn gtMarkColonCond;
3183 static fgWalkPreFn gtClearColonCond;
3187 GenTree* nodeToFind;
3192 FindLinkData gtFindLink(Statement* stmt, GenTree* node);
3193 bool gtHasCatchArg(GenTree* tree);
3195 typedef ArrayStack<GenTree*> GenTreeStack;
3197 //=========================================================================
3198 // BasicBlock functions
3200 // This is a debug flag we will use to assert when creating block during codegen
3201 // as this interferes with procedure splitting. If you know what you're doing, set
3202 // it to true before creating the block. (DEBUG only)
3203 bool fgSafeBasicBlockCreation;
3206 BasicBlock* bbNewBasicBlock(BBjumpKinds jumpKind);
3209 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
3210 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
3214 XX The variables to be used by the code generator. XX
3216 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
3217 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
3221 // For both PROMOTION_TYPE_NONE and PROMOTION_TYPE_DEPENDENT the struct will
3222 // be placed in the stack frame and it's fields must be laid out sequentially.
3224 // For PROMOTION_TYPE_INDEPENDENT each of the struct's fields is replaced by
3225 // a local variable that can be enregistered or placed in the stack frame.
3226 // The fields do not need to be laid out sequentially
3228 enum lvaPromotionType
3230 PROMOTION_TYPE_NONE, // The struct local is not promoted
3231 PROMOTION_TYPE_INDEPENDENT, // The struct local is promoted,
3232 // and its field locals are independent of its parent struct local.
3233 PROMOTION_TYPE_DEPENDENT // The struct local is promoted,
3234 // but its field locals depend on its parent struct local.
3237 /*****************************************************************************/
3239 enum FrameLayoutState
3242 INITIAL_FRAME_LAYOUT,
3243 PRE_REGALLOC_FRAME_LAYOUT,
3244 REGALLOC_FRAME_LAYOUT,
3245 TENTATIVE_FRAME_LAYOUT,
3250 RefCountState lvaRefCountState; // Current local ref count state
3252 bool lvaLocalVarRefCounted() const
3254 return lvaRefCountState == RCS_NORMAL;
3257 bool lvaTrackedFixed; // true: We cannot add new 'tracked' variable
3258 unsigned lvaCount; // total number of locals, which includes function arguments,
3259 // special arguments, IL local variables, and JIT temporary variables
3261 LclVarDsc* lvaTable; // variable descriptor table
3262 unsigned lvaTableCnt; // lvaTable size (>= lvaCount)
3264 unsigned lvaTrackedCount; // actual # of locals being tracked
3265 unsigned lvaTrackedCountInSizeTUnits; // min # of size_t's sufficient to hold a bit for all the locals being tracked
3268 VARSET_TP lvaTrackedVars; // set of tracked variables
3270 #ifndef TARGET_64BIT
3271 VARSET_TP lvaLongVars; // set of long (64-bit) variables
3273 VARSET_TP lvaFloatVars; // set of floating-point (32-bit and 64-bit) variables
3275 unsigned lvaCurEpoch; // VarSets are relative to a specific set of tracked var indices.
3276 // It that changes, this changes. VarSets from different epochs
3277 // cannot be meaningfully combined.
3279 unsigned GetCurLVEpoch()
3284 // reverse map of tracked number to var number
3285 unsigned lvaTrackedToVarNumSize;
3286 unsigned* lvaTrackedToVarNum;
3290 // # of procs compiled a with double-aligned stack
3291 static unsigned s_lvaDoubleAlignedProcsCount;
3295 // Getters and setters for address-exposed and do-not-enregister local var properties.
3296 bool lvaVarAddrExposed(unsigned varNum) const;
3297 void lvaSetVarAddrExposed(unsigned varNum DEBUGARG(AddressExposedReason reason));
3298 void lvaSetHiddenBufferStructArg(unsigned varNum);
3299 void lvaSetVarLiveInOutOfHandler(unsigned varNum);
3300 bool lvaVarDoNotEnregister(unsigned varNum);
3302 void lvSetMinOptsDoNotEnreg();
3304 bool lvaEnregEHVars;
3305 bool lvaEnregMultiRegVars;
3307 void lvaSetVarDoNotEnregister(unsigned varNum DEBUGARG(DoNotEnregisterReason reason));
3309 unsigned lvaVarargsHandleArg;
3311 unsigned lvaVarargsBaseOfStkArgs; // Pointer (computed based on incoming varargs handle) to the start of the stack
3313 #endif // TARGET_X86
3315 unsigned lvaInlinedPInvokeFrameVar; // variable representing the InlinedCallFrame
3316 unsigned lvaReversePInvokeFrameVar; // variable representing the reverse PInvoke frame
3317 unsigned lvaMonAcquired; // boolean variable introduced into in synchronized methods
3318 // that tracks whether the lock has been taken
3320 unsigned lvaArg0Var; // The lclNum of arg0. Normally this will be info.compThisArg.
3321 // However, if there is a "ldarga 0" or "starg 0" in the IL,
3322 // we will redirect all "ldarg(a) 0" and "starg 0" to this temp.
3324 unsigned lvaInlineeReturnSpillTemp; // The temp to spill the non-VOID return expression
3325 // in case there are multiple BBJ_RETURN blocks in the inlinee
3326 // or if the inlinee has GC ref locals.
3328 #if FEATURE_FIXED_OUT_ARGS
3329 unsigned lvaOutgoingArgSpaceVar; // var that represents outgoing argument space
3330 PhasedVar<unsigned> lvaOutgoingArgSpaceSize; // size of fixed outgoing argument space
3331 #endif // FEATURE_FIXED_OUT_ARGS
3333 static unsigned GetOutgoingArgByteSize(unsigned sizeWithoutPadding)
3335 return roundUp(sizeWithoutPadding, TARGET_POINTER_SIZE);
3338 // Variable representing the return address. The helper-based tailcall
3339 // mechanism passes the address of the return address to a runtime helper
3340 // where it is used to detect tail-call chains.
3341 unsigned lvaRetAddrVar;
3343 #if defined(DEBUG) && defined(TARGET_XARCH)
3345 unsigned lvaReturnSpCheck; // Stores SP to confirm it is not corrupted on return.
3347 #endif // defined(DEBUG) && defined(TARGET_XARCH)
3349 #if defined(DEBUG) && defined(TARGET_X86)
3351 unsigned lvaCallSpCheck; // Stores SP to confirm it is not corrupted after every call.
3353 #endif // defined(DEBUG) && defined(TARGET_X86)
3355 bool lvaGenericsContextInUse;
3357 bool lvaKeepAliveAndReportThis(); // Synchronized instance method of a reference type, or
3358 // CORINFO_GENERICS_CTXT_FROM_THIS?
3359 bool lvaReportParamTypeArg(); // Exceptions and CORINFO_GENERICS_CTXT_FROM_PARAMTYPEARG?
3361 //-------------------------------------------------------------------------
3362 // All these frame offsets are inter-related and must be kept in sync
3364 #if !defined(FEATURE_EH_FUNCLETS)
3365 // This is used for the callable handlers
3366 unsigned lvaShadowSPslotsVar; // Block-layout TYP_STRUCT variable for all the shadow SP slots
3367 #endif // FEATURE_EH_FUNCLETS
3369 int lvaCachedGenericContextArgOffs;
3370 int lvaCachedGenericContextArgOffset(); // For CORINFO_CALLCONV_PARAMTYPE and if generic context is passed as
3373 #ifdef JIT32_GCENCODER
3375 unsigned lvaLocAllocSPvar; // variable which stores the value of ESP after the last alloca/localloc
3377 #endif // JIT32_GCENCODER
3379 unsigned lvaNewObjArrayArgs; // variable with arguments for new MD array helper
3381 // TODO-Review: Prior to reg predict we reserve 24 bytes for Spill temps.
3382 // after the reg predict we will use a computed maxTmpSize
3383 // which is based upon the number of spill temps predicted by reg predict
3384 // All this is necessary because if we under-estimate the size of the spill
3385 // temps we could fail when encoding instructions that reference stack offsets for ARM.
3387 // Pre codegen max spill temp size.
3388 static const unsigned MAX_SPILL_TEMP_SIZE = 24;
3390 //-------------------------------------------------------------------------
3392 unsigned lvaGetMaxSpillTempSize();
3394 bool lvaIsPreSpilled(unsigned lclNum, regMaskTP preSpillMask);
3395 #endif // TARGET_ARM
3396 void lvaAssignFrameOffsets(FrameLayoutState curState);
3397 void lvaFixVirtualFrameOffsets();
3398 void lvaUpdateArgWithInitialReg(LclVarDsc* varDsc);
3399 void lvaUpdateArgsWithInitialReg();
3400 void lvaAssignVirtualFrameOffsetsToArgs();
3401 #ifdef UNIX_AMD64_ABI
3402 int lvaAssignVirtualFrameOffsetToArg(unsigned lclNum, unsigned argSize, int argOffs, int* callerArgOffset);
3403 #else // !UNIX_AMD64_ABI
3404 int lvaAssignVirtualFrameOffsetToArg(unsigned lclNum, unsigned argSize, int argOffs);
3405 #endif // !UNIX_AMD64_ABI
3406 void lvaAssignVirtualFrameOffsetsToLocals();
3407 int lvaAllocLocalAndSetVirtualOffset(unsigned lclNum, unsigned size, int stkOffs);
3409 // Returns true if compCalleeRegsPushed (including RBP if used as frame pointer) is even.
3410 bool lvaIsCalleeSavedIntRegCountEven();
3412 void lvaAlignFrame();
3413 void lvaAssignFrameOffsetsToPromotedStructs();
3414 int lvaAllocateTemps(int stkOffs, bool mustDoubleAlign);
3417 void lvaDumpRegLocation(unsigned lclNum);
3418 void lvaDumpFrameLocation(unsigned lclNum);
3419 void lvaDumpEntry(unsigned lclNum, FrameLayoutState curState, size_t refCntWtdWidth = 6);
3420 void lvaTableDump(FrameLayoutState curState = NO_FRAME_LAYOUT); // NO_FRAME_LAYOUT means use the current frame
3421 // layout state defined by lvaDoneFrameLayout
3424 // Limit frames size to 1GB. The maximum is 2GB in theory - make it intentionally smaller
3425 // to avoid bugs from borderline cases.
3426 #define MAX_FrameSize 0x3FFFFFFF
3427 void lvaIncrementFrameSize(unsigned size);
3429 unsigned lvaFrameSize(FrameLayoutState curState);
3431 // Returns the caller-SP-relative offset for the SP/FP relative offset determined by FP based.
3432 int lvaToCallerSPRelativeOffset(int offs, bool isFpBased, bool forRootFrame = true) const;
3434 // Returns the caller-SP-relative offset for the local variable "varNum."
3435 int lvaGetCallerSPRelativeOffset(unsigned varNum);
3437 // Returns the SP-relative offset for the local variable "varNum". Illegal to ask this for functions with localloc.
3438 int lvaGetSPRelativeOffset(unsigned varNum);
3440 int lvaToInitialSPRelativeOffset(unsigned offset, bool isFpBased);
3441 int lvaGetInitialSPRelativeOffset(unsigned varNum);
3443 // True if this is an OSR compilation and this local is potentially
3444 // located on the original method stack frame.
3445 bool lvaIsOSRLocal(unsigned varNum);
3447 //------------------------ For splitting types ----------------------------
3449 void lvaInitTypeRef();
3451 void lvaInitArgs(InitVarDscInfo* varDscInfo);
3452 void lvaInitThisPtr(InitVarDscInfo* varDscInfo);
3453 void lvaInitRetBuffArg(InitVarDscInfo* varDscInfo, bool useFixedRetBufReg);
3454 void lvaInitUserArgs(InitVarDscInfo* varDscInfo, unsigned skipArgs, unsigned takeArgs);
3455 void lvaInitGenericsCtxt(InitVarDscInfo* varDscInfo);
3456 void lvaInitVarArgsHandle(InitVarDscInfo* varDscInfo);
3458 void lvaInitVarDsc(LclVarDsc* varDsc,
3460 CorInfoType corInfoType,
3461 CORINFO_CLASS_HANDLE typeHnd,
3462 CORINFO_ARG_LIST_HANDLE varList,
3463 CORINFO_SIG_INFO* varSig);
3465 var_types lvaGetActualType(unsigned lclNum);
3466 var_types lvaGetRealType(unsigned lclNum);
3468 //-------------------------------------------------------------------------
3472 LclVarDsc* lvaGetDesc(unsigned lclNum)
3474 assert(lclNum < lvaCount);
3475 return &lvaTable[lclNum];
3478 LclVarDsc* lvaGetDesc(unsigned lclNum) const
3480 assert(lclNum < lvaCount);
3481 return &lvaTable[lclNum];
3484 LclVarDsc* lvaGetDesc(const GenTreeLclVarCommon* lclVar)
3486 return lvaGetDesc(lclVar->GetLclNum());
3489 unsigned lvaTrackedIndexToLclNum(unsigned trackedIndex)
3491 assert(trackedIndex < lvaTrackedCount);
3492 unsigned lclNum = lvaTrackedToVarNum[trackedIndex];
3493 assert(lclNum < lvaCount);
3497 LclVarDsc* lvaGetDescByTrackedIndex(unsigned trackedIndex)
3499 return lvaGetDesc(lvaTrackedIndexToLclNum(trackedIndex));
3502 unsigned lvaGetLclNum(const LclVarDsc* varDsc)
3504 assert((lvaTable <= varDsc) && (varDsc < lvaTable + lvaCount)); // varDsc must point within the table
3505 assert(((char*)varDsc - (char*)lvaTable) % sizeof(LclVarDsc) ==
3506 0); // varDsc better not point in the middle of a variable
3507 unsigned varNum = (unsigned)(varDsc - lvaTable);
3508 assert(varDsc == &lvaTable[varNum]);
3512 unsigned lvaLclSize(unsigned varNum);
3513 unsigned lvaLclExactSize(unsigned varNum);
3515 bool lvaHaveManyLocals(float percent = 1.0f) const;
3517 unsigned lvaGrabTemp(bool shortLifetime DEBUGARG(const char* reason));
3518 unsigned lvaGrabTemps(unsigned cnt DEBUGARG(const char* reason));
3519 unsigned lvaGrabTempWithImplicitUse(bool shortLifetime DEBUGARG(const char* reason));
3521 void lvaSortByRefCount();
3523 PhaseStatus lvaMarkLocalVars(); // Local variable ref-counting
3524 void lvaComputeRefCounts(bool isRecompute, bool setSlotNumbers);
3525 void lvaMarkLocalVars(BasicBlock* block, bool isRecompute);
3527 void lvaAllocOutgoingArgSpaceVar(); // Set up lvaOutgoingArgSpaceVar
3530 struct lvaStressLclFldArgs
3532 Compiler* m_pCompiler;
3536 static fgWalkPreFn lvaStressLclFldCB;
3537 void lvaStressLclFld();
3538 unsigned lvaStressLclFldPadding(unsigned lclNum);
3540 void lvaDispVarSet(VARSET_VALARG_TP set, VARSET_VALARG_TP allVars);
3541 void lvaDispVarSet(VARSET_VALARG_TP set);
3546 int lvaFrameAddress(int varNum, bool mustBeFPBased, regNumber* pBaseReg, int addrModeOffset, bool isFloatUsage);
3548 int lvaFrameAddress(int varNum, bool* pFPbased);
3551 bool lvaIsParameter(unsigned varNum);
3552 bool lvaIsRegArgument(unsigned varNum);
3553 bool lvaIsOriginalThisArg(unsigned varNum); // Is this varNum the original this argument?
3554 bool lvaIsOriginalThisReadOnly(); // return true if there is no place in the code
3555 // that writes to arg0
3558 bool lvaIsArgAccessedViaVarArgsCookie(unsigned lclNum)
3560 if (!info.compIsVarArgs)
3565 LclVarDsc* varDsc = lvaGetDesc(lclNum);
3566 return varDsc->lvIsParam && !varDsc->lvIsRegArg && (lclNum != lvaVarargsHandleArg);
3568 #endif // TARGET_X86
3570 bool lvaIsImplicitByRefLocal(unsigned lclNum) const;
3571 bool lvaIsLocalImplicitlyAccessedByRef(unsigned lclNum) const;
3573 // Returns true if this local var is a multireg struct
3574 bool lvaIsMultiregStruct(LclVarDsc* varDsc, bool isVararg);
3576 // If the local is a TYP_STRUCT, get/set a class handle describing it
3577 void lvaSetStruct(unsigned varNum, ClassLayout* layout, bool unsafeValueClsCheck);
3578 void lvaSetStruct(unsigned varNum, CORINFO_CLASS_HANDLE typeHnd, bool unsafeValueClsCheck);
3579 void lvaSetStructUsedAsVarArg(unsigned varNum);
3581 // If the local is TYP_REF, set or update the associated class information.
3582 void lvaSetClass(unsigned varNum, CORINFO_CLASS_HANDLE clsHnd, bool isExact = false);
3583 void lvaSetClass(unsigned varNum, GenTree* tree, CORINFO_CLASS_HANDLE stackHandle = nullptr);
3584 void lvaUpdateClass(unsigned varNum, CORINFO_CLASS_HANDLE clsHnd, bool isExact = false);
3585 void lvaUpdateClass(unsigned varNum, GenTree* tree, CORINFO_CLASS_HANDLE stackHandle = nullptr);
3587 #define MAX_NumOfFieldsInPromotableStruct 4 // Maximum number of fields in promotable struct
3589 // Info about struct type fields.
3590 struct lvaStructFieldInfo
3592 // Class handle for SIMD type recognition, see CORINFO_TYPE_LAYOUT_NODE
3593 // for more details on the restrictions.
3594 CORINFO_CLASS_HANDLE fldSIMDTypeHnd = NO_CLASS_HANDLE;
3595 uint8_t fldOffset = 0;
3596 uint8_t fldOrdinal = 0;
3597 var_types fldType = TYP_UNDEF;
3598 unsigned fldSize = 0;
3601 // Field handle for diagnostic purposes only. See CORINFO_TYPE_LAYOUT_NODE.
3602 CORINFO_FIELD_HANDLE diagFldHnd = NO_FIELD_HANDLE;
3606 // Info about a struct type, instances of which may be candidates for promotion.
3607 struct lvaStructPromotionInfo
3609 CORINFO_CLASS_HANDLE typeHnd;
3612 bool anySignificantPadding;
3614 unsigned char fieldCnt;
3615 lvaStructFieldInfo fields[MAX_NumOfFieldsInPromotableStruct];
3617 lvaStructPromotionInfo(CORINFO_CLASS_HANDLE typeHnd = nullptr)
3620 , containsHoles(false)
3621 , anySignificantPadding(false)
3622 , fieldsSorted(false)
3628 // This class is responsible for checking validity and profitability of struct promotion.
3629 // If it is both legal and profitable, then TryPromoteStructVar promotes the struct and initializes
3630 // necessary information for fgMorphStructField to use.
3631 class StructPromotionHelper
3634 StructPromotionHelper(Compiler* compiler);
3636 bool CanPromoteStructType(CORINFO_CLASS_HANDLE typeHnd);
3637 bool TryPromoteStructVar(unsigned lclNum);
3640 structPromotionInfo.typeHnd = NO_CLASS_HANDLE;
3644 bool CanPromoteStructVar(unsigned lclNum);
3645 bool ShouldPromoteStructVar(unsigned lclNum);
3646 void PromoteStructVar(unsigned lclNum);
3647 void SortStructFields();
3649 var_types TryPromoteValueClassAsPrimitive(CORINFO_TYPE_LAYOUT_NODE* treeNodes, size_t maxTreeNodes, size_t index);
3650 void AdvanceSubTree(CORINFO_TYPE_LAYOUT_NODE* treeNodes, size_t maxTreeNodes, size_t* index);
3654 lvaStructPromotionInfo structPromotionInfo;
3657 StructPromotionHelper* structPromotionHelper;
3659 unsigned lvaGetFieldLocal(const LclVarDsc* varDsc, unsigned int fldOffset);
3660 lvaPromotionType lvaGetPromotionType(const LclVarDsc* varDsc);
3661 lvaPromotionType lvaGetPromotionType(unsigned varNum);
3662 lvaPromotionType lvaGetParentPromotionType(const LclVarDsc* varDsc);
3663 lvaPromotionType lvaGetParentPromotionType(unsigned varNum);
3664 bool lvaIsFieldOfDependentlyPromotedStruct(const LclVarDsc* varDsc);
3665 bool lvaIsGCTracked(const LclVarDsc* varDsc);
3667 #if defined(FEATURE_SIMD)
3668 bool lvaMapSimd12ToSimd16(const LclVarDsc* varDsc)
3670 assert(varDsc->lvType == TYP_SIMD12);
3672 #if defined(TARGET_64BIT)
3673 assert(compMacOsArm64Abi() || varDsc->lvSize() == 16);
3674 #endif // defined(TARGET_64BIT)
3676 // We make local variable SIMD12 types 16 bytes instead of just 12.
3677 // lvSize() will return 16 bytes for SIMD12, even for fields.
3678 // However, we can't do that mapping if the var is a dependently promoted struct field.
3679 // Such a field must remain its exact size within its parent struct unless it is a single
3680 // field *and* it is the only field in a struct of 16 bytes.
3681 if (varDsc->lvSize() != 16)
3685 if (lvaIsFieldOfDependentlyPromotedStruct(varDsc))
3687 LclVarDsc* parentVarDsc = lvaGetDesc(varDsc->lvParentLcl);
3688 return (parentVarDsc->lvFieldCnt == 1) && (parentVarDsc->lvSize() == 16);
3692 #endif // defined(FEATURE_SIMD)
3694 unsigned lvaGSSecurityCookie; // LclVar number
3695 bool lvaTempsHaveLargerOffsetThanVars();
3697 // Returns "true" iff local variable "lclNum" is in SSA form.
3698 bool lvaInSsa(unsigned lclNum) const
3700 return lvaGetDesc(lclNum)->lvInSsa;
3703 unsigned lvaStubArgumentVar; // variable representing the secret stub argument coming in EAX
3705 #if defined(FEATURE_EH_FUNCLETS)
3706 unsigned lvaPSPSym; // variable representing the PSPSym
3709 InlineInfo* impInlineInfo; // Only present for inlinees
3710 InlineStrategy* m_inlineStrategy;
3712 InlineContext* compInlineContext; // Always present
3714 // The Compiler* that is the root of the inlining tree of which "this" is a member.
3715 Compiler* impInlineRoot();
3717 #if defined(DEBUG) || defined(INLINE_DATA)
3718 unsigned __int64 getInlineCycleCount()
3720 return m_compCycles;
3722 #endif // defined(DEBUG) || defined(INLINE_DATA)
3724 bool fgNoStructPromotion; // Set to TRUE to turn off struct promotion for this method.
3725 bool fgNoStructParamPromotion; // Set to TRUE to turn off struct promotion for parameters this method.
3727 //=========================================================================
3729 //=========================================================================
3732 //---------------- Local variable ref-counting ----------------------------
3734 void lvaMarkLclRefs(GenTree* tree, BasicBlock* block, Statement* stmt, bool isRecompute);
3735 bool IsDominatedByExceptionalEntry(BasicBlock* block);
3736 void SetVolatileHint(LclVarDsc* varDsc);
3738 // Keeps the mapping from SSA #'s to VN's for the implicit memory variables.
3739 SsaDefArray<SsaMemDef> lvMemoryPerSsaData;
3742 // Returns the address of the per-Ssa data for memory at the given ssaNum (which is required
3743 // not to be the SsaConfig::RESERVED_SSA_NUM, which indicates that the variable is
3744 // not an SSA variable).
3745 SsaMemDef* GetMemoryPerSsaData(unsigned ssaNum)
3747 return lvMemoryPerSsaData.GetSsaDef(ssaNum);
3751 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
3752 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
3756 XX Imports the given method and converts it to semantic trees XX
3758 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
3759 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
3766 PREFIX_TAILCALL_EXPLICIT = 0x00000001, // call has "tail" IL prefix
3767 PREFIX_TAILCALL_IMPLICIT =
3768 0x00000010, // call is treated as having "tail" prefix even though there is no "tail" IL prefix
3769 PREFIX_TAILCALL_STRESS =
3770 0x00000100, // call doesn't "tail" IL prefix but is treated as explicit because of tail call stress
3771 PREFIX_TAILCALL = (PREFIX_TAILCALL_EXPLICIT | PREFIX_TAILCALL_IMPLICIT | PREFIX_TAILCALL_STRESS),
3772 PREFIX_VOLATILE = 0x00001000,
3773 PREFIX_UNALIGNED = 0x00010000,
3774 PREFIX_CONSTRAINED = 0x00100000,
3775 PREFIX_READONLY = 0x01000000
3778 static void impValidateMemoryAccessOpcode(const BYTE* codeAddr, const BYTE* codeEndp, bool volatilePrefix);
3779 static OPCODE impGetNonPrefixOpcode(const BYTE* codeAddr, const BYTE* codeEndp);
3780 static GenTreeFlags impPrefixFlagsToIndirFlags(unsigned prefixFlags);
3781 static bool impOpcodeIsCallOpcode(OPCODE opcode);
3786 void impFixPredLists();
3788 CORINFO_CLASS_HANDLE impGetRefAnyClass();
3789 CORINFO_CLASS_HANDLE impGetRuntimeArgumentHandle();
3790 CORINFO_CLASS_HANDLE impGetTypeHandleClass();
3791 CORINFO_CLASS_HANDLE impGetStringClass();
3792 CORINFO_CLASS_HANDLE impGetObjectClass();
3794 // Returns underlying type of handles returned by ldtoken instruction
3795 var_types GetRuntimeHandleUnderlyingType()
3797 // RuntimeTypeHandle is backed by raw pointer on NativeAOT and by object reference on other runtimes
3798 return IsTargetAbi(CORINFO_NATIVEAOT_ABI) ? TYP_I_IMPL : TYP_REF;
3801 void impDevirtualizeCall(GenTreeCall* call,
3802 CORINFO_RESOLVED_TOKEN* pResolvedToken,
3803 CORINFO_METHOD_HANDLE* method,
3804 unsigned* methodFlags,
3805 CORINFO_CONTEXT_HANDLE* contextHandle,
3806 CORINFO_CONTEXT_HANDLE* exactContextHandle,
3807 bool isLateDevirtualization,
3808 bool isExplicitTailCall,
3809 IL_OFFSET ilOffset = BAD_IL_OFFSET);
3811 bool impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset);
3813 enum class GDVProbeType
3818 MethodAndClassProfile,
3821 GDVProbeType compClassifyGDVProbeType(GenTreeCall* call);
3823 //=========================================================================
3825 //=========================================================================
3828 //-------------------- Stack manipulation ---------------------------------
3830 unsigned impStkSize; // Size of the full stack
3832 #define SMALL_STACK_SIZE 16 // number of elements in impSmallStack
3834 struct SavedStack // used to save/restore stack contents.
3836 unsigned ssDepth; // number of values on stack
3837 StackEntry* ssTrees; // saved tree values
3840 bool impIsPrimitive(CorInfoType type);
3841 bool impILConsumesAddr(const BYTE* codeAddr);
3843 void impResolveToken(const BYTE* addr, CORINFO_RESOLVED_TOKEN* pResolvedToken, CorInfoTokenKind kind);
3845 void impPushOnStack(GenTree* tree, typeInfo ti);
3846 StackEntry impPopStack();
3847 void impPopStack(unsigned n);
3848 StackEntry& impStackTop(unsigned n = 0);
3849 unsigned impStackHeight();
3851 void impSaveStackState(SavedStack* savePtr, bool copy);
3852 void impRestoreStackState(SavedStack* savePtr);
3854 GenTree* impImportLdvirtftn(GenTree* thisPtr, CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_CALL_INFO* pCallInfo);
3856 enum class BoxPatterns
3860 MakeInlineObservation = 2,
3863 int impBoxPatternMatch(CORINFO_RESOLVED_TOKEN* pResolvedToken,
3864 const BYTE* codeAddr,
3865 const BYTE* codeEndp,
3867 void impImportAndPushBox(CORINFO_RESOLVED_TOKEN* pResolvedToken);
3869 void impImportNewObjArray(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_CALL_INFO* pCallInfo);
3871 bool impCanPInvokeInline();
3872 bool impCanPInvokeInlineCallSite(BasicBlock* block);
3873 void impCheckForPInvokeCall(
3874 GenTreeCall* call, CORINFO_METHOD_HANDLE methHnd, CORINFO_SIG_INFO* sig, unsigned mflags, BasicBlock* block);
3875 GenTreeCall* impImportIndirectCall(CORINFO_SIG_INFO* sig, const DebugInfo& di = DebugInfo());
3876 void impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* sig);
3878 void impInsertHelperCall(CORINFO_HELPER_DESC* helperCall);
3879 void impHandleAccessAllowed(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall);
3880 void impHandleAccessAllowedInternal(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall);
3882 var_types impImportCall(OPCODE opcode,
3883 CORINFO_RESOLVED_TOKEN* pResolvedToken,
3884 CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, // Is this a "constrained." call on a
3886 GenTree* newobjThis,
3888 CORINFO_CALL_INFO* callInfo,
3889 IL_OFFSET rawILOffset);
3891 CORINFO_CLASS_HANDLE impGetSpecialIntrinsicExactReturnType(GenTreeCall* call);
3893 GenTree* impFixupCallStructReturn(GenTreeCall* call, CORINFO_CLASS_HANDLE retClsHnd);
3895 GenTree* impFixupStructReturnType(GenTree* op);
3898 var_types impImportJitTestLabelMark(int numArgs);
3901 GenTree* impInitClass(CORINFO_RESOLVED_TOKEN* pResolvedToken);
3903 GenTree* impImportStaticReadOnlyField(CORINFO_FIELD_HANDLE field, CORINFO_CLASS_HANDLE ownerCls);
3905 GenTree* impImportStaticFieldAddress(CORINFO_RESOLVED_TOKEN* pResolvedToken,
3906 CORINFO_ACCESS_FLAGS access,
3907 CORINFO_FIELD_INFO* pFieldInfo,
3909 GenTreeFlags* pIndirFlags,
3910 bool* pIsHoistable = nullptr);
3911 void impAnnotateFieldIndir(GenTreeIndir* indir);
3913 static void impBashVarAddrsToI(GenTree* tree1, GenTree* tree2 = nullptr);
3915 GenTree* impImplicitIorI4Cast(GenTree* tree, var_types dstTyp, bool zeroExtend = false);
3917 GenTree* impImplicitR4orR8Cast(GenTree* tree, var_types dstTyp);
3919 void impImportLeave(BasicBlock* block);
3920 void impResetLeaveBlock(BasicBlock* block, unsigned jmpAddr);
3921 GenTree* impTypeIsAssignable(GenTree* typeTo, GenTree* typeFrom);
3923 // Mirrors StringComparison.cs
3924 enum StringComparison
3927 OrdinalIgnoreCase = 5
3929 enum StringComparisonJoint
3931 Eq, // (d1 == cns1) && (s2 == cns2)
3932 Xor, // (d1 ^ cns1) | (s2 ^ cns2)
3934 GenTree* impStringEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* sig, unsigned methodFlags);
3935 GenTree* impSpanEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* sig, unsigned methodFlags);
3936 GenTree* impExpandHalfConstEquals(GenTreeLclVarCommon* data,
3943 StringComparison cmpMode);
3944 GenTree* impCreateCompareInd(GenTreeLclVarCommon* obj,
3948 StringComparison ignoreCase,
3949 StringComparisonJoint joint = Eq);
3950 GenTree* impExpandHalfConstEqualsSWAR(
3951 GenTreeLclVarCommon* data, WCHAR* cns, int len, int dataOffset, StringComparison cmpMode);
3952 GenTree* impExpandHalfConstEqualsSIMD(
3953 GenTreeLclVarCommon* data, WCHAR* cns, int len, int dataOffset, StringComparison cmpMode);
3954 GenTreeStrCon* impGetStrConFromSpan(GenTree* span);
3956 GenTree* impIntrinsic(GenTree* newobjThis,
3957 CORINFO_CLASS_HANDLE clsHnd,
3958 CORINFO_METHOD_HANDLE method,
3959 CORINFO_SIG_INFO* sig,
3960 unsigned methodFlags,
3961 CORINFO_RESOLVED_TOKEN* pResolvedToken,
3965 CORINFO_RESOLVED_TOKEN* pContstrainedResolvedToken,
3966 CORINFO_THIS_TRANSFORM constraintCallThisTransform,
3967 NamedIntrinsic* pIntrinsicName,
3968 bool* isSpecialIntrinsic = nullptr);
3969 GenTree* impMathIntrinsic(CORINFO_METHOD_HANDLE method,
3970 CORINFO_SIG_INFO* sig,
3972 NamedIntrinsic intrinsicName,
3974 GenTree* impMinMaxIntrinsic(CORINFO_METHOD_HANDLE method,
3975 CORINFO_SIG_INFO* sig,
3976 CorInfoType callJitType,
3977 NamedIntrinsic intrinsicName,
3982 NamedIntrinsic lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method);
3983 NamedIntrinsic lookupPrimitiveFloatNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName);
3984 NamedIntrinsic lookupPrimitiveIntNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName);
3985 GenTree* impUnsupportedNamedIntrinsic(unsigned helper,
3986 CORINFO_METHOD_HANDLE method,
3987 CORINFO_SIG_INFO* sig,
3990 GenTree* impSRCSUnsafeIntrinsic(NamedIntrinsic intrinsic,
3991 CORINFO_CLASS_HANDLE clsHnd,
3992 CORINFO_METHOD_HANDLE method,
3993 CORINFO_SIG_INFO* sig,
3994 CORINFO_RESOLVED_TOKEN* pResolvedToken);
3996 GenTree* impPrimitiveNamedIntrinsic(NamedIntrinsic intrinsic,
3997 CORINFO_CLASS_HANDLE clsHnd,
3998 CORINFO_METHOD_HANDLE method,
3999 CORINFO_SIG_INFO* sig);
4001 #ifdef FEATURE_HW_INTRINSICS
4002 GenTree* impHWIntrinsic(NamedIntrinsic intrinsic,
4003 CORINFO_CLASS_HANDLE clsHnd,
4004 CORINFO_METHOD_HANDLE method,
4005 CORINFO_SIG_INFO* sig,
4007 GenTree* impSimdAsHWIntrinsic(NamedIntrinsic intrinsic,
4008 CORINFO_CLASS_HANDLE clsHnd,
4009 CORINFO_METHOD_HANDLE method,
4010 CORINFO_SIG_INFO* sig,
4011 GenTree* newobjThis);
4014 bool compSupportsHWIntrinsic(CORINFO_InstructionSet isa);
4016 GenTree* impSimdAsHWIntrinsicSpecial(NamedIntrinsic intrinsic,
4017 CORINFO_CLASS_HANDLE clsHnd,
4018 CORINFO_SIG_INFO* sig,
4020 CorInfoType simdBaseJitType,
4022 GenTree* newobjThis);
4024 GenTree* impSpecialIntrinsic(NamedIntrinsic intrinsic,
4025 CORINFO_CLASS_HANDLE clsHnd,
4026 CORINFO_METHOD_HANDLE method,
4027 CORINFO_SIG_INFO* sig,
4028 CorInfoType simdBaseJitType,
4032 GenTree* getArgForHWIntrinsic(var_types argType,
4033 CORINFO_CLASS_HANDLE argClass,
4034 bool expectAddr = false,
4035 GenTree* newobjThis = nullptr);
4036 GenTree* impNonConstFallback(NamedIntrinsic intrinsic, var_types simdType, CorInfoType simdBaseJitType);
4037 GenTree* addRangeCheckIfNeeded(
4038 NamedIntrinsic intrinsic, GenTree* immOp, bool mustExpand, int immLowerBound, int immUpperBound);
4039 GenTree* addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound, int immUpperBound);
4041 #endif // FEATURE_HW_INTRINSICS
4042 GenTree* impArrayAccessIntrinsic(CORINFO_CLASS_HANDLE clsHnd,
4043 CORINFO_SIG_INFO* sig,
4046 NamedIntrinsic intrinsicName);
4047 GenTree* impInitializeArrayIntrinsic(CORINFO_SIG_INFO* sig);
4048 GenTree* impCreateSpanIntrinsic(CORINFO_SIG_INFO* sig);
4050 GenTree* impKeepAliveIntrinsic(GenTree* objToKeepAlive);
4052 GenTree* impMethodPointer(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_CALL_INFO* pCallInfo);
4054 GenTree* impTransformThis(GenTree* thisPtr,
4055 CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken,
4056 CORINFO_THIS_TRANSFORM transform);
4058 //----------------- Manipulating the trees and stmts ----------------------
4060 Statement* impStmtList; // Statements for the BB being imported.
4061 Statement* impLastStmt; // The last statement for the current BB.
4064 static const unsigned CHECK_SPILL_ALL = static_cast<unsigned>(-1);
4065 static const unsigned CHECK_SPILL_NONE = static_cast<unsigned>(-2);
4067 void impBeginTreeList();
4068 void impEndTreeList(BasicBlock* block, Statement* firstStmt, Statement* lastStmt);
4069 void impEndTreeList(BasicBlock* block);
4070 void impAppendStmtCheck(Statement* stmt, unsigned chkLevel);
4071 void impAppendStmt(Statement* stmt, unsigned chkLevel, bool checkConsumedDebugInfo = true);
4072 void impAppendStmt(Statement* stmt);
4073 void impInsertStmtBefore(Statement* stmt, Statement* stmtBefore);
4074 Statement* impAppendTree(GenTree* tree, unsigned chkLevel, const DebugInfo& di, bool checkConsumedDebugInfo = true);
4075 void impStoreTemp(unsigned lclNum,
4078 Statement** pAfterStmt = nullptr,
4079 const DebugInfo& di = DebugInfo(),
4080 BasicBlock* block = nullptr);
4081 Statement* impExtractLastStmt();
4082 GenTree* impCloneExpr(GenTree* tree,
4085 Statement** pAfterStmt DEBUGARG(const char* reason));
4086 GenTree* impStoreStruct(GenTree* store,
4088 Statement** pAfterStmt = nullptr,
4089 const DebugInfo& di = DebugInfo(),
4090 BasicBlock* block = nullptr);
4091 GenTree* impStoreStructPtr(GenTree* destAddr, GenTree* value, unsigned curLevel);
4093 GenTree* impGetNodeAddr(GenTree* val, unsigned curLevel, GenTreeFlags* pDerefFlags);
4095 var_types impNormStructType(CORINFO_CLASS_HANDLE structHnd, CorInfoType* simdBaseJitType = nullptr);
4097 GenTree* impNormStructVal(GenTree* structVal, unsigned curLevel);
4099 GenTree* impTokenToHandle(CORINFO_RESOLVED_TOKEN* pResolvedToken,
4100 bool* pRuntimeLookup = nullptr,
4101 bool mustRestoreHandle = false,
4102 bool importParent = false);
4104 GenTree* impParentClassTokenToHandle(CORINFO_RESOLVED_TOKEN* pResolvedToken,
4105 bool* pRuntimeLookup = nullptr,
4106 bool mustRestoreHandle = false)
4108 return impTokenToHandle(pResolvedToken, pRuntimeLookup, mustRestoreHandle, true);
4111 GenTree* impLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken,
4112 CORINFO_LOOKUP* pLookup,
4114 void* compileTimeHandle);
4116 GenTree* getRuntimeContextTree(CORINFO_RUNTIME_LOOKUP_KIND kind);
4118 GenTree* impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken,
4119 CORINFO_LOOKUP* pLookup,
4120 void* compileTimeHandle);
4122 GenTree* impReadyToRunLookupToTree(CORINFO_CONST_LOOKUP* pLookup, GenTreeFlags flags, void* compileTimeHandle);
4124 GenTreeCall* impReadyToRunHelperToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken,
4125 CorInfoHelpFunc helper,
4127 CORINFO_LOOKUP_KIND* pGenericLookupKind = nullptr,
4128 GenTree* arg1 = nullptr);
4130 bool impIsCastHelperEligibleForClassProbe(GenTree* tree);
4131 bool impIsCastHelperMayHaveProfileData(CorInfoHelpFunc helper);
4133 GenTree* impCastClassOrIsInstToTree(
4134 GenTree* op1, GenTree* op2, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass, IL_OFFSET ilOffset);
4136 GenTree* impOptimizeCastClassOrIsInst(GenTree* op1, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass);
4138 bool VarTypeIsMultiByteAndCanEnreg(var_types type,
4139 CORINFO_CLASS_HANDLE typeClass,
4143 CorInfoCallConvExtension callConv);
4145 bool IsIntrinsicImplementedByUserCall(NamedIntrinsic intrinsicName);
4146 bool IsTargetIntrinsic(NamedIntrinsic intrinsicName);
4147 bool IsMathIntrinsic(NamedIntrinsic intrinsicName);
4148 bool IsMathIntrinsic(GenTree* tree);
4151 //----------------- Importing the method ----------------------------------
4153 CORINFO_CONTEXT_HANDLE impTokenLookupContextHandle; // The context used for looking up tokens.
4156 unsigned impCurOpcOffs;
4157 const char* impCurOpcName;
4158 bool impNestedStackSpill;
4160 // For displaying instrs with generated native code (-n:B)
4161 Statement* impLastILoffsStmt; // oldest stmt added for which we did not call SetLastILOffset().
4162 void impNoteLastILoffs();
4165 // Debug info of current statement being imported. It gets set to contain
4166 // no IL location (!impCurStmtDI.GetLocation().IsValid) after it has been
4167 // set in the appended trees. Then it gets updated at IL instructions for
4168 // which we have to report mapping info.
4169 // It will always contain the current inline context.
4170 DebugInfo impCurStmtDI;
4172 DebugInfo impCreateDIWithCurrentStackInfo(IL_OFFSET offs, bool isCall);
4173 void impCurStmtOffsSet(IL_OFFSET offs);
4175 void impNoteBranchOffs();
4177 unsigned impInitBlockLineInfo();
4179 bool impIsThis(GenTree* obj);
4181 void impPopCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call);
4184 static bool impCheckImplicitArgumentCoercion(var_types sigType, var_types nodeType);
4187 void impPopReverseCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call, unsigned skipReverseCount);
4189 //---------------- Spilling the importer stack ----------------------------
4191 // The maximum number of bytes of IL processed without clean stack state.
4192 // It allows to limit the maximum tree size and depth.
4193 static const unsigned MAX_TREE_SIZE = 200;
4194 bool impCanSpillNow(OPCODE prevOpcode);
4200 SavedStack pdSavedStack;
4203 PendingDsc* impPendingList; // list of BBs currently waiting to be imported.
4204 PendingDsc* impPendingFree; // Freed up dscs that can be reused
4206 // We keep a byte-per-block map (dynamically extended) in the top-level Compiler object of a compilation.
4207 JitExpandArray<BYTE> impPendingBlockMembers;
4209 // Return the byte for "b" (allocating/extending impPendingBlockMembers if necessary.)
4210 // Operates on the map in the top-level ancestor.
4211 BYTE impGetPendingBlockMember(BasicBlock* blk)
4213 return impInlineRoot()->impPendingBlockMembers.Get(blk->bbInd());
4216 // Set the byte for "b" to "val" (allocating/extending impPendingBlockMembers if necessary.)
4217 // Operates on the map in the top-level ancestor.
4218 void impSetPendingBlockMember(BasicBlock* blk, BYTE val)
4220 impInlineRoot()->impPendingBlockMembers.Set(blk->bbInd(), val);
4223 bool impCanReimport;
4225 bool impSpillStackEntry(unsigned level,
4229 bool bAssertOnRecursion,
4234 void impSpillStackEnsure(bool spillLeaves = false);
4235 void impEvalSideEffects();
4236 void impSpillSpecialSideEff();
4237 void impSpillSideEffect(bool spillGlobEffects, unsigned chkLevel DEBUGARG(const char* reason));
4238 void impSpillSideEffects(bool spillGlobEffects, unsigned chkLevel DEBUGARG(const char* reason));
4239 void impSpillLclRefs(unsigned lclNum, unsigned chkLevel);
4241 BasicBlock* impPushCatchArgOnStack(BasicBlock* hndBlk, CORINFO_CLASS_HANDLE clsHnd, bool isSingleBlockFilter);
4243 bool impBlockIsInALoop(BasicBlock* block);
4244 void impImportBlockCode(BasicBlock* block);
4246 void impReimportMarkBlock(BasicBlock* block);
4247 void impReimportMarkSuccessors(BasicBlock* block);
4249 void impVerifyEHBlock(BasicBlock* block, bool isTryStart);
4251 void impImportBlockPending(BasicBlock* block);
4253 // Similar to impImportBlockPending, but assumes that block has already been imported once and is being
4254 // reimported for some reason. It specifically does *not* look at verCurrentState to set the EntryState
4255 // for the block, but instead, just re-uses the block's existing EntryState.
4256 void impReimportBlockPending(BasicBlock* block);
4258 var_types impGetByRefResultType(genTreeOps oper, bool fUnsigned, GenTree** pOp1, GenTree** pOp2);
4260 void impImportBlock(BasicBlock* block);
4262 // Assumes that "block" is a basic block that completes with a non-empty stack. We will assign the values
4263 // on the stack to local variables (the "spill temp" variables). The successor blocks will assume that
4264 // its incoming stack contents are in those locals. This requires "block" and its successors to agree on
4265 // the variables that will be used -- and for all the predecessors of those successors, and the
4266 // successors of those predecessors, etc. Call such a set of blocks closed under alternating
4267 // successor/predecessor edges a "spill clique." A block is a "predecessor" or "successor" member of the
4268 // clique (or, conceivably, both). Each block has a specified sequence of incoming and outgoing spill
4269 // temps. If "block" already has its outgoing spill temps assigned (they are always a contiguous series
4270 // of local variable numbers, so we represent them with the base local variable number), returns that.
4271 // Otherwise, picks a set of spill temps, and propagates this choice to all blocks in the spill clique of
4272 // which "block" is a member (asserting, in debug mode, that no block in this clique had its spill temps
4273 // chosen already. More precisely, that the incoming or outgoing spill temps are not chosen, depending
4274 // on which kind of member of the clique the block is).
4275 unsigned impGetSpillTmpBase(BasicBlock* block);
4277 // Assumes that "block" is a basic block that completes with a non-empty stack. We have previously
4278 // assigned the values on the stack to local variables (the "spill temp" variables). The successor blocks
4279 // will assume that its incoming stack contents are in those locals. This requires "block" and its
4280 // successors to agree on the variables and their types that will be used. The CLI spec allows implicit
4281 // conversions between 'int' and 'native int' or 'float' and 'double' stack types. So one predecessor can
4282 // push an int and another can push a native int. For 64-bit we have chosen to implement this by typing
4283 // the "spill temp" as native int, and then importing (or re-importing as needed) so that all the
4284 // predecessors in the "spill clique" push a native int (sign-extending if needed), and all the
4285 // successors receive a native int. Similarly float and double are unified to double.
4286 // This routine is called after a type-mismatch is detected, and it will walk the spill clique to mark
4287 // blocks for re-importation as appropriate (both successors, so they get the right incoming type, and
4288 // predecessors, so they insert an upcast if needed).
4289 void impReimportSpillClique(BasicBlock* block);
4291 // When we compute a "spill clique" (see above) these byte-maps are allocated to have a byte per basic
4292 // block, and represent the predecessor and successor members of the clique currently being computed.
4293 // *** Access to these will need to be locked in a parallel compiler.
4294 JitExpandArray<BYTE> impSpillCliquePredMembers;
4295 JitExpandArray<BYTE> impSpillCliqueSuccMembers;
4303 // Abstract class for receiving a callback while walking a spill clique
4304 class SpillCliqueWalker
4307 virtual void Visit(SpillCliqueDir predOrSucc, BasicBlock* blk) = 0;
4310 // This class is used for setting the bbStkTempsIn and bbStkTempsOut on the blocks within a spill clique
4311 class SetSpillTempsBase : public SpillCliqueWalker
4316 SetSpillTempsBase(unsigned baseTmp) : m_baseTmp(baseTmp)
4319 virtual void Visit(SpillCliqueDir predOrSucc, BasicBlock* blk);
4322 // This class is used for implementing impReimportSpillClique part on each block within the spill clique
4323 class ReimportSpillClique : public SpillCliqueWalker
4328 ReimportSpillClique(Compiler* pComp) : m_pComp(pComp)
4331 virtual void Visit(SpillCliqueDir predOrSucc, BasicBlock* blk);
4334 // This is the heart of the algorithm for walking spill cliques. It invokes callback->Visit for each
4335 // predecessor or successor within the spill clique
4336 void impWalkSpillCliqueFromPred(BasicBlock* pred, SpillCliqueWalker* callback);
4338 // For a BasicBlock that has already been imported, the EntryState has an array of GenTrees for the
4339 // incoming locals. This walks that list an resets the types of the GenTrees to match the types of
4340 // the VarDscs. They get out of sync when we have int/native int issues (see impReimportSpillClique).
4341 void impRetypeEntryStateTemps(BasicBlock* blk);
4343 BYTE impSpillCliqueGetMember(SpillCliqueDir predOrSucc, BasicBlock* blk);
4344 void impSpillCliqueSetMember(SpillCliqueDir predOrSucc, BasicBlock* blk, BYTE val);
4346 GenTreeLclVar* impCreateLocalNode(unsigned lclNum DEBUGARG(IL_OFFSET offset));
4347 void impLoadVar(unsigned lclNum, IL_OFFSET offset);
4348 void impLoadArg(unsigned ilArgNum, IL_OFFSET offset);
4349 void impLoadLoc(unsigned ilLclNum, IL_OFFSET offset);
4350 bool impReturnInstruction(int prefixFlags, OPCODE& opcode);
4351 void impPoisonImplicitByrefsBeforeReturn();
4353 // A free list of linked list nodes used to represent to-do stacks of basic blocks.
4354 struct BlockListNode
4357 BlockListNode* m_next;
4358 BlockListNode(BasicBlock* blk, BlockListNode* next = nullptr) : m_blk(blk), m_next(next)
4361 void* operator new(size_t sz, Compiler* comp);
4363 BlockListNode* impBlockListNodeFreeList;
4365 void FreeBlockListNode(BlockListNode* node);
4367 var_types mangleVarArgsType(var_types type);
4369 regNumber getCallArgIntRegister(regNumber floatReg);
4370 regNumber getCallArgFloatRegister(regNumber intReg);
4372 static unsigned jitTotalMethodCompiled;
4375 static LONG jitNestingLevel;
4378 static bool impIsInvariant(const GenTree* tree);
4379 static bool impIsAddressInLocal(const GenTree* tree, GenTree** lclVarTreeOut = nullptr);
4381 void impMakeDiscretionaryInlineObservations(InlineInfo* pInlineInfo, InlineResult* inlineResult);
4383 // STATIC inlining decision based on the IL code.
4384 void impCanInlineIL(CORINFO_METHOD_HANDLE fncHandle,
4385 CORINFO_METHOD_INFO* methInfo,
4387 InlineResult* inlineResult);
4389 void impCheckCanInline(GenTreeCall* call,
4390 uint8_t candidateIndex,
4391 CORINFO_METHOD_HANDLE fncHandle,
4393 CORINFO_CONTEXT_HANDLE exactContextHnd,
4394 InlineCandidateInfo** ppInlineCandidateInfo,
4395 InlineResult* inlineResult);
4397 void impInlineRecordArgInfo(InlineInfo* pInlineInfo, CallArg* arg, unsigned argNum, InlineResult* inlineResult);
4399 void impInlineInitVars(InlineInfo* pInlineInfo);
4401 unsigned impInlineFetchLocal(unsigned lclNum DEBUGARG(const char* reason));
4403 GenTree* impInlineFetchArg(unsigned lclNum, InlArgInfo* inlArgInfo, InlLclVarInfo* lclTypeInfo);
4405 bool impInlineIsThis(GenTree* tree, InlArgInfo* inlArgInfo);
4407 bool impInlineIsGuaranteedThisDerefBeforeAnySideEffects(GenTree* additionalTree,
4408 CallArgs* additionalCallArgs,
4409 GenTree* dereferencedAddress,
4410 InlArgInfo* inlArgInfo);
4412 void impMarkInlineCandidate(GenTree* call,
4413 CORINFO_CONTEXT_HANDLE exactContextHnd,
4414 bool exactContextNeedsRuntimeLookup,
4415 CORINFO_CALL_INFO* callInfo,
4416 IL_OFFSET ilOffset);
4418 void impMarkInlineCandidateHelper(GenTreeCall* call,
4419 uint8_t candidateIndex,
4420 CORINFO_CONTEXT_HANDLE exactContextHnd,
4421 bool exactContextNeedsRuntimeLookup,
4422 CORINFO_CALL_INFO* callInfo,
4424 InlineResult* inlineResult);
4426 bool impTailCallRetTypeCompatible(bool allowWidening,
4427 var_types callerRetType,
4428 CORINFO_CLASS_HANDLE callerRetTypeClass,
4429 CorInfoCallConvExtension callerCallConv,
4430 var_types calleeRetType,
4431 CORINFO_CLASS_HANDLE calleeRetTypeClass,
4432 CorInfoCallConvExtension calleeCallConv);
4434 bool impIsTailCallILPattern(
4435 bool tailPrefixed, OPCODE curOpcode, const BYTE* codeAddrOfNextOpcode, const BYTE* codeEnd, bool isRecursive);
4437 bool impIsImplicitTailCallCandidate(
4438 OPCODE curOpcode, const BYTE* codeAddrOfNextOpcode, const BYTE* codeEnd, int prefixFlags, bool isRecursive);
4440 bool impIsClassExact(CORINFO_CLASS_HANDLE classHnd);
4441 bool impCanSkipCovariantStoreCheck(GenTree* value, GenTree* array);
4443 methodPointerInfo* impAllocateMethodPointerInfo(const CORINFO_RESOLVED_TOKEN& token, mdToken tokenConstrained);
4446 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
4447 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
4451 XX Info about the basic-blocks, their contents and the flow analysis XX
4453 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
4454 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
4458 BasicBlock* fgFirstBB; // Beginning of the basic block list
4459 BasicBlock* fgLastBB; // End of the basic block list
4460 BasicBlock* fgFirstColdBlock; // First block to be placed in the cold section
4461 BasicBlock* fgEntryBB; // For OSR, the original method's entry point
4462 BasicBlock* fgOSREntryBB; // For OSR, the logical entry point (~ patchpoint)
4463 #if defined(FEATURE_EH_FUNCLETS)
4464 BasicBlock* fgFirstFuncletBB; // First block of outlined funclets (to allow block insertion before the funclets)
4466 BasicBlock* fgFirstBBScratch; // Block inserted for initialization stuff. Is nullptr if no such block has been
4468 BasicBlockList* fgReturnBlocks; // list of BBJ_RETURN blocks
4469 unsigned fgEdgeCount; // # of control flow edges between the BBs
4470 unsigned fgBBcount; // # of BBs in the method
4472 unsigned fgBBcountAtCodegen; // # of BBs in the method at the start of codegen
4473 jitstd::vector<BasicBlock*>* fgBBOrder; // ordered vector of BBs
4475 unsigned fgBBNumMax; // The max bbNum that has been assigned to basic blocks
4476 unsigned fgDomBBcount; // # of BBs for which we have dominator and reachability information
4477 BasicBlock** fgBBReversePostorder; // Blocks in reverse postorder
4479 // After the dominance tree is computed, we cache a DFS preorder number and DFS postorder number to compute
4480 // dominance queries in O(1). fgDomTreePreOrder and fgDomTreePostOrder are arrays giving the block's preorder and
4481 // postorder number, respectively. The arrays are indexed by basic block number. (Note that blocks are numbered
4482 // starting from one. Thus, we always waste element zero. This makes debugging easier and makes the code less likely
4483 // to suffer from bugs stemming from forgetting to add or subtract one from the block number to form an array
4484 // index). The arrays are of size fgBBNumMax + 1.
4485 unsigned* fgDomTreePreOrder;
4486 unsigned* fgDomTreePostOrder;
4488 // Dominator tree used by SSA construction and copy propagation (the two are expected to use the same tree
4489 // in order to avoid the need for SSA reconstruction and an "out of SSA" phase).
4490 DomTreeNode* fgSsaDomTree;
4492 bool fgBBVarSetsInited;
4494 // Track how many artificial ref counts we've added to fgEntryBB (for OSR)
4495 unsigned fgEntryBBExtraRefs;
4497 // Allocate array like T* a = new T[fgBBNumMax + 1];
4498 // Using helper so we don't keep forgetting +1.
4499 template <typename T>
4500 T* fgAllocateTypeForEachBlk(CompMemKind cmk = CMK_Unknown)
4502 return getAllocator(cmk).allocate<T>(fgBBNumMax + 1);
4505 // BlockSets are relative to a specific set of BasicBlock numbers. If that changes
4506 // (if the blocks are renumbered), this changes. BlockSets from different epochs
4507 // cannot be meaningfully combined. Note that new blocks can be created with higher
4508 // block numbers without changing the basic block epoch. These blocks *cannot*
4509 // participate in a block set until the blocks are all renumbered, causing the epoch
4510 // to change. This is useful if continuing to use previous block sets is valuable.
4511 // If the epoch is zero, then it is uninitialized, and block sets can't be used.
4512 unsigned fgCurBBEpoch;
4514 unsigned GetCurBasicBlockEpoch()
4516 return fgCurBBEpoch;
4519 // The number of basic blocks in the current epoch. When the blocks are renumbered,
4520 // this is fgBBcount. As blocks are added, fgBBcount increases, fgCurBBEpochSize remains
4521 // the same, until a new BasicBlock epoch is created, such as when the blocks are all renumbered.
4522 unsigned fgCurBBEpochSize;
4524 // The number of "size_t" elements required to hold a bitset large enough for fgCurBBEpochSize
4525 // bits. This is precomputed to avoid doing math every time BasicBlockBitSetTraits::GetArrSize() is called.
4526 unsigned fgBBSetCountInSizeTUnits;
4528 void NewBasicBlockEpoch()
4530 INDEBUG(unsigned oldEpochArrSize = fgBBSetCountInSizeTUnits);
4532 // We have a new epoch. Compute and cache the size needed for new BlockSets.
4534 fgCurBBEpochSize = fgBBNumMax + 1;
4535 fgBBSetCountInSizeTUnits =
4536 roundUp(fgCurBBEpochSize, (unsigned)(sizeof(size_t) * 8)) / unsigned(sizeof(size_t) * 8);
4539 // All BlockSet objects are now invalid!
4540 fgReachabilitySetsValid = false; // the bbReach sets are now invalid!
4541 fgEnterBlksSetValid = false; // the fgEnterBlks set is now invalid!
4545 unsigned epochArrSize = BasicBlockBitSetTraits::GetArrSize(this);
4546 printf("\nNew BlockSet epoch %d, # of blocks (including unused BB00): %u, bitset array size: %u (%s)",
4547 fgCurBBEpoch, fgCurBBEpochSize, epochArrSize, (epochArrSize <= 1) ? "short" : "long");
4548 if ((fgCurBBEpoch != 1) && ((oldEpochArrSize <= 1) != (epochArrSize <= 1)))
4550 // If we're not just establishing the first epoch, and the epoch array size has changed such that we're
4551 // going to change our bitset representation from short (just a size_t bitset) to long (a pointer to an
4552 // array of size_t bitsets), then print that out.
4553 printf("; NOTE: BlockSet size was previously %s!", (oldEpochArrSize <= 1) ? "short" : "long");
4560 void EnsureBasicBlockEpoch()
4562 if (fgCurBBEpochSize != fgBBNumMax + 1)
4564 NewBasicBlockEpoch();
4568 BasicBlock* fgNewBasicBlock(BBjumpKinds jumpKind);
4569 bool fgEnsureFirstBBisScratch();
4570 bool fgFirstBBisScratch();
4571 bool fgBBisScratch(BasicBlock* block);
4573 void fgExtendEHRegionBefore(BasicBlock* block);
4574 void fgExtendEHRegionAfter(BasicBlock* block);
4576 BasicBlock* fgNewBBbefore(BBjumpKinds jumpKind, BasicBlock* block, bool extendRegion);
4578 BasicBlock* fgNewBBafter(BBjumpKinds jumpKind, BasicBlock* block, bool extendRegion);
4580 BasicBlock* fgNewBBFromTreeAfter(BBjumpKinds jumpKind, BasicBlock* block, GenTree* tree, DebugInfo& debugInfo, bool updateSideEffects = false);
4582 BasicBlock* fgNewBBinRegion(BBjumpKinds jumpKind,
4585 BasicBlock* nearBlk,
4586 bool putInFilter = false,
4587 bool runRarely = false,
4588 bool insertAtEnd = false);
4590 BasicBlock* fgNewBBinRegion(BBjumpKinds jumpKind,
4592 bool runRarely = false,
4593 bool insertAtEnd = false);
4595 BasicBlock* fgNewBBinRegion(BBjumpKinds jumpKind);
4597 BasicBlock* fgNewBBinRegionWorker(BBjumpKinds jumpKind,
4598 BasicBlock* afterBlk,
4599 unsigned xcptnIndex,
4600 bool putInTryRegion);
4602 void fgInsertBBbefore(BasicBlock* insertBeforeBlk, BasicBlock* newBlk);
4603 void fgInsertBBafter(BasicBlock* insertAfterBlk, BasicBlock* newBlk);
4604 void fgUnlinkBlock(BasicBlock* block);
4606 #ifdef FEATURE_JIT_METHOD_PERF
4607 unsigned fgMeasureIR();
4608 #endif // FEATURE_JIT_METHOD_PERF
4610 bool fgModified; // True if the flow graph has been modified recently
4611 bool fgPredsComputed; // Have we computed the bbPreds list
4612 bool fgDomsComputed; // Have we computed the dominator sets?
4613 bool fgReturnBlocksComputed; // Have we computed the return blocks list?
4614 bool fgOptimizedFinally; // Did we optimize any try-finallys?
4616 bool fgHasSwitch; // any BBJ_SWITCH jumps?
4618 BlockSet fgEnterBlks; // Set of blocks which have a special transfer of control; the "entry" blocks plus EH handler
4620 #if defined(FEATURE_EH_FUNCLETS) && defined(TARGET_ARM)
4621 BlockSet fgAlwaysBlks; // Set of blocks which are BBJ_ALWAYS part of BBJ_CALLFINALLY/BBJ_ALWAYS pair that should
4622 // never be removed due to a requirement to use the BBJ_ALWAYS for generating code and
4623 // not have "retless" blocks.
4625 #endif // defined(FEATURE_EH_FUNCLETS) && defined(TARGET_ARM)
4628 bool fgReachabilitySetsValid; // Are the bbReach sets valid?
4629 bool fgEnterBlksSetValid; // Is the fgEnterBlks set valid?
4632 bool fgRemoveRestOfBlock; // true if we know that we will throw
4633 bool fgStmtRemoved; // true if we remove statements -> need new DFA
4640 // There are two modes for ordering of the trees.
4641 // - In FGOrderTree, the dominant ordering is the tree order, and the nodes contained in
4642 // each tree and sub-tree are contiguous, and can be traversed (in gtNext/gtPrev order)
4643 // by traversing the tree according to the order of the operands.
4644 // - In FGOrderLinear, the dominant ordering is the linear order.
4645 FlowGraphOrder fgOrder;
4647 // The following are flags that keep track of the state of internal data structures
4649 // Even in tree form (fgOrder == FGOrderTree) the trees are threaded in a
4650 // doubly linked lists during certain phases of the compilation.
4651 // - Local morph threads all locals to be used for early liveness and
4652 // forward sub when optimizing. This is kept valid until after forward sub.
4653 // The first local is kept in Statement::GetTreeList() and the last
4654 // local in Statement::GetTreeListEnd(). fgSequenceLocals can be used
4655 // to (re-)sequence a statement into this form, and
4656 // Statement::LocalsTreeList for range-based iteration. The order must
4657 // match tree order.
4659 // - fgSetBlockOrder threads all nodes. This is kept valid until LIR form.
4660 // In this form the first node is given by Statement::GetTreeList and the
4661 // last node is given by Statement::GetRootNode(). fgSetStmtSeq can be used
4662 // to (re-)sequence a statement into this form, and Statement::TreeList for
4663 // range-based iteration. The order must match tree order.
4665 // - Rationalization links all nodes into linear form which is kept until
4666 // the end of compilation. The first and last nodes are stored in the block.
4667 NodeThreading fgNodeThreading;
4668 bool fgCanRelocateEHRegions; // true if we are allowed to relocate the EH regions
4669 bool fgEdgeWeightsComputed; // true after we have called fgComputeEdgeWeights
4670 bool fgHaveValidEdgeWeights; // true if we were successful in computing all of the edge weights
4671 bool fgSlopUsedInEdgeWeights; // true if their was some slop used when computing the edge weights
4672 bool fgRangeUsedInEdgeWeights; // true if some of the edgeWeight are expressed in Min..Max form
4673 weight_t fgCalledCount; // count of the number of times this method was called
4674 // This is derived from the profile data
4675 // or is BB_UNITY_WEIGHT when we don't have profile data
4677 #if defined(FEATURE_EH_FUNCLETS)
4678 bool fgFuncletsCreated; // true if the funclet creation phase has been run
4679 #endif // FEATURE_EH_FUNCLETS
4681 bool fgGlobalMorph; // indicates if we are during the global morphing phase
4682 // since fgMorphTree can be called from several places
4684 bool impBoxTempInUse; // the temp below is valid and available
4685 unsigned impBoxTemp; // a temporary that is used for boxing
4688 bool jitFallbackCompile; // Are we doing a fallback compile? That is, have we executed a NO_WAY assert,
4689 // and we are trying to compile again in a "safer", minopts mode?
4693 unsigned impInlinedCodeSize;
4694 bool fgPrintInlinedMethods;
4697 jitstd::vector<FlowEdge*>* fgPredListSortVector;
4699 //-------------------------------------------------------------------------
4703 PhaseStatus fgImport();
4705 PhaseStatus fgTransformIndirectCalls();
4707 PhaseStatus fgTransformPatchpoints();
4709 PhaseStatus fgMorphInit();
4711 PhaseStatus fgInline();
4713 PhaseStatus fgRemoveEmptyTry();
4715 PhaseStatus fgRemoveEmptyFinally();
4717 PhaseStatus fgMergeFinallyChains();
4719 PhaseStatus fgCloneFinally();
4721 void fgCleanupContinuation(BasicBlock* continuation);
4723 #if defined(FEATURE_EH_FUNCLETS) && defined(TARGET_ARM)
4725 PhaseStatus fgUpdateFinallyTargetFlags();
4727 void fgClearAllFinallyTargetBits();
4729 void fgAddFinallyTargetFlags();
4731 void fgFixFinallyTargetFlags(BasicBlock* pred, BasicBlock* succ, BasicBlock* newBlock);
4733 #endif // defined(FEATURE_EH_FUNCLETS) && defined(TARGET_ARM)
4734 PhaseStatus fgTailMergeThrows();
4735 void fgTailMergeThrowsFallThroughHelper(BasicBlock* predBlock,
4736 BasicBlock* nonCanonicalBlock,
4737 BasicBlock* canonicalBlock,
4738 FlowEdge* predEdge);
4739 void fgTailMergeThrowsJumpToHelper(BasicBlock* predBlock,
4740 BasicBlock* nonCanonicalBlock,
4741 BasicBlock* canonicalBlock,
4742 FlowEdge* predEdge);
4744 bool fgRetargetBranchesToCanonicalCallFinally(BasicBlock* block,
4745 BasicBlock* handler,
4746 BlockToBlockMap& continuationMap);
4748 GenTree* fgGetCritSectOfStaticMethod();
4750 #if defined(FEATURE_EH_FUNCLETS)
4752 void fgAddSyncMethodEnterExit();
4754 GenTree* fgCreateMonitorTree(unsigned lvaMonitorBool, unsigned lvaThisVar, BasicBlock* block, bool enter);
4756 void fgConvertSyncReturnToLeave(BasicBlock* block);
4758 #endif // FEATURE_EH_FUNCLETS
4760 void fgAddReversePInvokeEnterExit();
4762 bool fgMoreThanOneReturnBlock();
4764 // The number of separate return points in the method.
4765 unsigned fgReturnCount;
4767 PhaseStatus fgAddInternal();
4769 enum class FoldResult
4772 FOLD_CHANGED_CONTROL_FLOW,
4773 FOLD_REMOVED_LAST_STMT,
4774 FOLD_ALTERED_LAST_STMT,
4777 FoldResult fgFoldConditional(BasicBlock* block);
4779 void fgMorphStmts(BasicBlock* block);
4780 void fgMorphBlocks();
4782 void fgMergeBlockReturn(BasicBlock* block);
4784 bool fgMorphBlockStmt(BasicBlock* block, Statement* stmt DEBUGARG(const char* msg));
4785 void fgMorphStmtBlockOps(BasicBlock* block, Statement* stmt);
4787 //------------------------------------------------------------------------------------------------------------
4788 // MorphMDArrayTempCache: a simple cache of compiler temporaries in the local variable table, used to minimize
4789 // the number of locals allocated when doing early multi-dimensional array operation expansion. Two types of
4790 // temps are created and cached (due to the two types of temps needed by the MD array expansion): TYP_INT and
4791 // TYP_REF. `GrabTemp` either returns an available temp from the cache or allocates a new temp and returns it
4792 // after adding it to the cache. `Reset` makes all the temps in the cache available for subsequent re-use.
4794 class MorphMDArrayTempCache
4800 TempList(Compiler* compiler)
4801 : m_compiler(compiler), m_first(nullptr), m_insertPtr(&m_first), m_nextAvail(nullptr)
4809 m_nextAvail = m_first;
4815 Node(unsigned tmp) : next(nullptr), tmp(tmp)
4823 Compiler* m_compiler;
4829 TempList intTemps; // Temps for genActualType() == TYP_INT
4830 TempList refTemps; // Temps for TYP_REF
4833 MorphMDArrayTempCache(Compiler* compiler) : intTemps(compiler), refTemps(compiler)
4837 unsigned GrabTemp(var_types type);
4846 bool fgMorphArrayOpsStmt(MorphMDArrayTempCache* pTempCache, BasicBlock* block, Statement* stmt);
4847 PhaseStatus fgMorphArrayOps();
4849 void fgSetOptions();
4852 void fgPreExpandQmarkChecks(GenTree* expr);
4853 void fgPostExpandQmarkChecks();
4856 IL_OFFSET fgFindBlockILOffset(BasicBlock* block);
4857 void fgFixEntryFlowForOSR();
4859 BasicBlock* fgSplitBlockAtBeginning(BasicBlock* curr);
4860 BasicBlock* fgSplitBlockAtEnd(BasicBlock* curr);
4861 BasicBlock* fgSplitBlockAfterStatement(BasicBlock* curr, Statement* stmt);
4862 BasicBlock* fgSplitBlockAfterNode(BasicBlock* curr, GenTree* node); // for LIR
4863 BasicBlock* fgSplitEdge(BasicBlock* curr, BasicBlock* succ);
4864 BasicBlock* fgSplitBlockBeforeTree(BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitNodeUse);
4866 Statement* fgNewStmtFromTree(GenTree* tree, BasicBlock* block, const DebugInfo& di);
4867 Statement* fgNewStmtFromTree(GenTree* tree);
4868 Statement* fgNewStmtFromTree(GenTree* tree, BasicBlock* block);
4869 Statement* fgNewStmtFromTree(GenTree* tree, const DebugInfo& di);
4871 GenTree* fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst = nullptr);
4872 void fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt);
4873 void fgExpandQmarkStmt(BasicBlock* block, Statement* stmt);
4874 void fgExpandQmarkNodes();
4876 // Do "simple lowering." This functionality is (conceptually) part of "general"
4877 // lowering that is distributed between fgMorph and the lowering phase of LSRA.
4878 PhaseStatus fgSimpleLowering();
4880 bool fgSimpleLowerCastOfSmpOp(LIR::Range& range, GenTreeCast* cast);
4882 #if FEATURE_LOOP_ALIGN
4883 PhaseStatus placeLoopAlignInstructions();
4886 // This field keep the R2R helper call that would be inserted to trigger the constructor
4887 // of the static class. It is set as nongc or gc static base if they are imported, so
4888 // CSE can eliminate the repeated call, or the chepeast helper function that triggers it.
4889 CorInfoHelpFunc m_preferredInitCctor;
4890 void fgSetPreferredInitCctor();
4892 GenTree* fgInitThisClass();
4894 GenTreeCall* fgGetStaticsCCtorHelper(CORINFO_CLASS_HANDLE cls, CorInfoHelpFunc helper, uint32_t typeIndex = 0);
4896 GenTreeCall* fgGetSharedCCtor(CORINFO_CLASS_HANDLE cls);
4898 bool backendRequiresLocalVarLifetimes()
4900 return !opts.MinOpts() || m_pLinearScan->willEnregisterLocalVars();
4903 void fgLocalVarLiveness();
4905 void fgLocalVarLivenessInit();
4907 void fgPerNodeLocalVarLiveness(GenTree* node);
4908 void fgPerBlockLocalVarLiveness();
4910 #if defined(FEATURE_HW_INTRINSICS)
4911 void fgPerNodeLocalVarLiveness(GenTreeHWIntrinsic* hwintrinsic);
4912 #endif // FEATURE_HW_INTRINSICS
4914 void fgAddHandlerLiveVars(BasicBlock* block, VARSET_TP& ehHandlerLiveVars);
4916 void fgLiveVarAnalysis(bool updateInternalOnly = false);
4918 void fgComputeLifeCall(VARSET_TP& life, GenTreeCall* call);
4920 void fgComputeLifeTrackedLocalUse(VARSET_TP& life, LclVarDsc& varDsc, GenTreeLclVarCommon* node);
4921 bool fgComputeLifeTrackedLocalDef(VARSET_TP& life,
4922 VARSET_VALARG_TP keepAliveVars,
4924 GenTreeLclVarCommon* node);
4925 bool fgComputeLifeUntrackedLocal(VARSET_TP& life,
4926 VARSET_VALARG_TP keepAliveVars,
4928 GenTreeLclVarCommon* lclVarNode);
4929 bool fgComputeLifeLocal(VARSET_TP& life, VARSET_VALARG_TP keepAliveVars, GenTree* lclVarNode);
4931 GenTree* fgTryRemoveDeadStoreEarly(Statement* stmt, GenTreeLclVarCommon* dst);
4933 void fgComputeLife(VARSET_TP& life,
4936 VARSET_VALARG_TP volatileVars,
4937 bool* pStmtInfoDirty DEBUGARG(bool* treeModf));
4939 void fgComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VARSET_VALARG_TP volatileVars);
4941 bool fgTryRemoveNonLocal(GenTree* node, LIR::Range* blockRange);
4943 bool fgTryRemoveDeadStoreLIR(GenTree* store, GenTreeLclVarCommon* lclNode, BasicBlock* block);
4945 bool fgRemoveDeadStore(GenTree** pTree,
4947 VARSET_VALARG_TP life,
4949 bool* pStmtInfoDirty,
4950 bool* pStoreRemoved DEBUGARG(bool* treeModf));
4952 void fgInterBlockLocalVarLiveness();
4954 // Blocks: convenience methods for enabling range-based `for` iteration over the function's blocks, e.g.:
4955 // 1. for (BasicBlock* const block : compiler->Blocks()) ...
4956 // 2. for (BasicBlock* const block : compiler->Blocks(startBlock)) ...
4957 // 3. for (BasicBlock* const block : compiler->Blocks(startBlock, endBlock)) ...
4958 // In case (1), the block list can be empty. In case (2), `startBlock` can be nullptr. In case (3),
4959 // both `startBlock` and `endBlock` must be non-null.
4961 BasicBlockSimpleList Blocks() const
4963 return BasicBlockSimpleList(fgFirstBB);
4966 BasicBlockSimpleList Blocks(BasicBlock* startBlock) const
4968 return BasicBlockSimpleList(startBlock);
4971 BasicBlockRangeList Blocks(BasicBlock* startBlock, BasicBlock* endBlock) const
4973 return BasicBlockRangeList(startBlock, endBlock);
4976 // This array, managed by the SSA numbering infrastructure, keeps "outlined composite SSA numbers".
4977 // See "SsaNumInfo::GetNum" for more details on when this is needed.
4978 JitExpandArrayStack<unsigned>* m_outlinedCompositeSsaNums;
4980 // This map tracks nodes whose value numbers explicitly or implicitly depend on memory states.
4981 // The map provides the entry block of the most closely enclosing loop that
4982 // defines the memory region accessed when defining the nodes's VN.
4984 // This information should be consulted when considering hoisting node out of a loop, as the VN
4985 // for the node will only be valid within the indicated loop.
4987 // It is not fine-grained enough to track memory dependence within loops, so cannot be used
4988 // for more general code motion.
4990 // If a node does not have an entry in the map we currently assume the VN is not memory dependent
4991 // and so memory does not constrain hoisting.
4993 typedef JitHashTable<GenTree*, JitPtrKeyFuncs<GenTree>, BasicBlock*> NodeToLoopMemoryBlockMap;
4994 NodeToLoopMemoryBlockMap* m_nodeToLoopMemoryBlockMap;
4995 NodeToLoopMemoryBlockMap* GetNodeToLoopMemoryBlockMap()
4997 if (m_nodeToLoopMemoryBlockMap == nullptr)
4999 m_nodeToLoopMemoryBlockMap = new (getAllocator()) NodeToLoopMemoryBlockMap(getAllocator());
5001 return m_nodeToLoopMemoryBlockMap;
5004 typedef JitHashTable<void*, JitPtrKeyFuncs<void>, CORINFO_RUNTIME_LOOKUP> SignatureToLookupInfoMap;
5005 SignatureToLookupInfoMap* m_signatureToLookupInfoMap;
5006 SignatureToLookupInfoMap* GetSignatureToLookupInfoMap()
5008 if (m_signatureToLookupInfoMap == nullptr)
5010 m_signatureToLookupInfoMap = new (getAllocator()) SignatureToLookupInfoMap(getAllocator());
5012 return m_signatureToLookupInfoMap;
5015 void optRecordLoopMemoryDependence(GenTree* tree, BasicBlock* block, ValueNum memoryVN);
5016 void optCopyLoopMemoryDependence(GenTree* fromTree, GenTree* toTree);
5018 inline bool PreciseRefCountsRequired();
5020 // Performs SSA conversion.
5021 PhaseStatus fgSsaBuild();
5023 // Reset any data structures to the state expected by "fgSsaBuild", so it can be run again.
5024 void fgResetForSsa();
5026 unsigned fgSsaPassesCompleted; // Number of times fgSsaBuild has been run.
5027 bool fgSsaChecksEnabled; // True if SSA info can be cross-checked versus IR
5030 void DumpSsaSummary();
5033 // Returns "true" if this is a special variable that is never zero initialized in the prolog.
5034 inline bool fgVarIsNeverZeroInitializedInProlog(unsigned varNum);
5036 // Returns "true" if the variable needs explicit zero initialization.
5037 inline bool fgVarNeedsExplicitZeroInit(unsigned varNum, bool bbInALoop, bool bbIsReturn);
5039 // The value numbers for this compilation.
5040 ValueNumStore* vnStore;
5043 ValueNumStore* GetValueNumStore()
5048 // Do value numbering (assign a value number to each
5050 PhaseStatus fgValueNumber();
5052 void fgValueNumberLocalStore(GenTree* storeNode,
5053 GenTreeLclVarCommon* lclDefNode,
5057 bool normalize = true);
5059 void fgValueNumberArrayElemLoad(GenTree* loadTree, VNFuncApp* addrFunc);
5061 void fgValueNumberArrayElemStore(GenTree* storeNode, VNFuncApp* addrFunc, unsigned storeSize, ValueNum value);
5063 void fgValueNumberFieldLoad(GenTree* loadTree, GenTree* baseAddr, FieldSeq* fieldSeq, ssize_t offset);
5065 void fgValueNumberFieldStore(
5066 GenTree* storeNode, GenTree* baseAddr, FieldSeq* fieldSeq, ssize_t offset, unsigned storeSize, ValueNum value);
5068 bool fgValueNumberConstLoad(GenTreeIndir* tree);
5070 // Compute the value number for a byref-exposed load of the given type via the given pointerVN.
5071 ValueNum fgValueNumberByrefExposedLoad(var_types type, ValueNum pointerVN);
5073 unsigned fgVNPassesCompleted; // Number of times fgValueNumber has been run.
5075 // Utility functions for fgValueNumber.
5077 // Perform value-numbering for the trees in "blk".
5078 void fgValueNumberBlock(BasicBlock* blk);
5080 // Requires that "entryBlock" is the entry block of loop "loopNum", and that "loopNum" is the
5081 // innermost loop of which "entryBlock" is the entry. Returns the value number that should be
5082 // assumed for the memoryKind at the start "entryBlk".
5083 ValueNum fgMemoryVNForLoopSideEffects(MemoryKind memoryKind, BasicBlock* entryBlock, unsigned loopNum);
5085 // Called when an operation (performed by "tree", described by "msg") may cause the GcHeap to be mutated.
5086 // As GcHeap is a subset of ByrefExposed, this will also annotate the ByrefExposed mutation.
5087 void fgMutateGcHeap(GenTree* tree DEBUGARG(const char* msg));
5089 // Called when an operation (performed by "tree", described by "msg") may cause an address-exposed local to be
5091 void fgMutateAddressExposedLocal(GenTree* tree DEBUGARG(const char* msg));
5093 // For a GC heap store at curTree, record the new curMemoryVN's and update curTree's MemorySsaMap.
5094 // As GcHeap is a subset of ByrefExposed, this will also record the ByrefExposed store.
5095 void recordGcHeapStore(GenTree* curTree, ValueNum gcHeapVN DEBUGARG(const char* msg));
5097 // For a store to an address-exposed local at curTree, record the new curMemoryVN and update curTree's MemorySsaMap.
5098 void recordAddressExposedLocalStore(GenTree* curTree, ValueNum memoryVN DEBUGARG(const char* msg));
5100 void fgSetCurrentMemoryVN(MemoryKind memoryKind, ValueNum newMemoryVN);
5102 // Tree caused an update in the current memory VN. If "tree" has an associated heap SSA #, record that
5103 // value in that SSA #.
5104 void fgValueNumberRecordMemorySsa(MemoryKind memoryKind, GenTree* tree);
5106 // The input 'tree' is a leaf node that is a constant
5107 // Assign the proper value number to the tree
5108 void fgValueNumberTreeConst(GenTree* tree);
5110 // If the VN store has been initialized, reassign the
5111 // proper value number to the constant tree.
5112 void fgUpdateConstTreeValueNumber(GenTree* tree);
5114 // Assumes that all inputs to "tree" have had value numbers assigned; assigns a VN to tree.
5115 // (With some exceptions: the VN of the lhs of an assignment is assigned as part of the
5117 void fgValueNumberTree(GenTree* tree);
5119 void fgValueNumberStore(GenTree* tree);
5121 void fgValueNumberSsaVarDef(GenTreeLclVarCommon* lcl);
5123 // Does value-numbering for a cast tree.
5124 void fgValueNumberCastTree(GenTree* tree);
5126 // Does value-numbering for a bitcast tree.
5127 void fgValueNumberBitCast(GenTree* tree);
5129 // Does value-numbering for an intrinsic tree.
5130 void fgValueNumberIntrinsic(GenTree* tree);
5132 void fgValueNumberArrIndexAddr(GenTreeArrAddr* arrAddr);
5134 #ifdef FEATURE_HW_INTRINSICS
5135 // Does value-numbering for a GT_HWINTRINSIC tree
5136 void fgValueNumberHWIntrinsic(GenTreeHWIntrinsic* tree);
5137 #endif // FEATURE_HW_INTRINSICS
5139 // Does value-numbering for a call. We interpret some helper calls.
5140 void fgValueNumberCall(GenTreeCall* call);
5142 // Does value-numbering for a helper representing a cast operation.
5143 void fgValueNumberCastHelper(GenTreeCall* call);
5145 // Does value-numbering for a helper "call" that has a VN function symbol "vnf".
5146 void fgValueNumberHelperCallFunc(GenTreeCall* call, VNFunc vnf, ValueNumPair vnpExc);
5148 // Requires "helpCall" to be a helper call. Assigns it a value number;
5149 // we understand the semantics of some of the calls. Returns "true" if
5150 // the call may modify the heap (we assume arbitrary memory side effects if so).
5151 bool fgValueNumberHelperCall(GenTreeCall* helpCall);
5153 // Requires that "helpFunc" is one of the pure Jit Helper methods.
5154 // Returns the corresponding VNFunc to use for value numbering
5155 VNFunc fgValueNumberJitHelperMethodVNFunc(CorInfoHelpFunc helpFunc);
5157 // Adds the exception set for the current tree node which has a memory indirection operation
5158 void fgValueNumberAddExceptionSetForIndirection(GenTree* tree, GenTree* baseAddr);
5160 // Adds the exception sets for the current tree node which is performing a division or modulus operation
5161 void fgValueNumberAddExceptionSetForDivision(GenTree* tree);
5163 // Adds the exception set for the current tree node which is performing a overflow checking operation
5164 void fgValueNumberAddExceptionSetForOverflow(GenTree* tree);
5166 // Adds the exception set for the current tree node which is performing a bounds check operation
5167 void fgValueNumberAddExceptionSetForBoundsCheck(GenTree* tree);
5169 // Adds the exception set for the current tree node which is performing a ckfinite operation
5170 void fgValueNumberAddExceptionSetForCkFinite(GenTree* tree);
5172 // Adds the exception sets for the current tree node
5173 void fgValueNumberAddExceptionSet(GenTree* tree);
5176 void fgDebugCheckExceptionSets();
5179 // These are the current value number for the memory implicit variables while
5180 // doing value numbering. These are the value numbers under the "liberal" interpretation
5181 // of memory values; the "conservative" interpretation needs no VN, since every access of
5182 // memory yields an unknown value.
5183 ValueNum fgCurMemoryVN[MemoryKindCount];
5185 // Return a "pseudo"-class handle for an array element type. If "elemType" is TYP_STRUCT,
5186 // requires "elemStructType" to be non-null (and to have a low-order zero). Otherwise, low order bit
5187 // is 1, and the rest is an encoding of "elemTyp".
5188 static CORINFO_CLASS_HANDLE EncodeElemType(var_types elemTyp, CORINFO_CLASS_HANDLE elemStructType)
5190 if (elemStructType != nullptr)
5192 assert(varTypeIsStruct(elemTyp) || elemTyp == TYP_REF || elemTyp == TYP_BYREF ||
5193 varTypeIsIntegral(elemTyp));
5194 assert((size_t(elemStructType) & 0x1) == 0x0); // Make sure the encoding below is valid.
5195 return elemStructType;
5199 assert(elemTyp != TYP_STRUCT);
5200 elemTyp = varTypeToSigned(elemTyp);
5201 return CORINFO_CLASS_HANDLE(size_t(elemTyp) << 1 | 0x1);
5204 // If "clsHnd" is the result of an "EncodePrim" call, returns true and sets "*pPrimType" to the
5205 // var_types it represents. Otherwise, returns TYP_STRUCT (on the assumption that "clsHnd" is
5206 // the struct type of the element).
5207 static var_types DecodeElemType(CORINFO_CLASS_HANDLE clsHnd)
5209 size_t clsHndVal = size_t(clsHnd);
5210 if (clsHndVal & 0x1)
5212 return var_types(clsHndVal >> 1);
5220 bool GetObjectHandleAndOffset(GenTree* tree, ssize_t* byteOffset, CORINFO_OBJECT_HANDLE* pObj);
5222 // Convert a BYTE which represents the VM's CorInfoGCtype to the JIT's var_types
5223 var_types getJitGCType(BYTE gcType);
5225 // Returns true if the provided type should be treated as a primitive type
5226 // for the unmanaged calling conventions.
5227 bool isNativePrimitiveStructType(CORINFO_CLASS_HANDLE clsHnd);
5229 enum structPassingKind
5231 SPK_Unknown, // Invalid value, never returned
5232 SPK_PrimitiveType, // The struct is passed/returned using a primitive type.
5233 SPK_EnclosingType, // Like SPK_Primitive type, but used for return types that
5234 // require a primitive type temp that is larger than the struct size.
5235 // Currently used for structs of size 3, 5, 6, or 7 bytes.
5236 SPK_ByValue, // The struct is passed/returned by value (using the ABI rules)
5237 // for ARM64 and UNIX_X64 in multiple registers. (when all of the
5238 // parameters registers are used, then the stack will be used)
5239 // for X86 passed on the stack, for ARM32 passed in registers
5240 // or the stack or split between registers and the stack.
5241 SPK_ByValueAsHfa, // The struct is passed/returned as an HFA in multiple registers.
5243 }; // The struct is passed/returned by reference to a copy/buffer.
5245 // Get the "primitive" type that is used when we are given a struct of size 'structSize'.
5246 // For pointer sized structs the 'clsHnd' is used to determine if the struct contains GC ref.
5247 // A "primitive" type is one of the scalar types: byte, short, int, long, ref, float, double
5248 // If we can't or shouldn't use a "primitive" type then TYP_UNKNOWN is returned.
5250 // isVarArg is passed for use on Windows Arm64 to change the decision returned regarding
5253 var_types getPrimitiveTypeForStruct(unsigned structSize, CORINFO_CLASS_HANDLE clsHnd, bool isVarArg);
5255 // Get the type that is used to pass values of the given struct type.
5256 // isVarArg is passed for use on Windows Arm64 to change the decision returned regarding
5259 var_types getArgTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
5260 structPassingKind* wbPassStruct,
5262 unsigned structSize);
5264 // Get the type that is used to return values of the given struct type.
5265 // If the size is unknown, pass 0 and it will be determined from 'clsHnd'.
5266 var_types getReturnTypeForStruct(CORINFO_CLASS_HANDLE clsHnd,
5267 CorInfoCallConvExtension callConv,
5268 structPassingKind* wbPassStruct = nullptr,
5269 unsigned structSize = 0);
5272 // Print a representation of "vnp" or "vn" on standard output.
5273 // If "level" is non-zero, we also print out a partial expansion of the value.
5274 void vnpPrint(ValueNumPair vnp, unsigned level);
5275 void vnPrint(ValueNum vn, unsigned level);
5278 bool fgDominate(const BasicBlock* b1, const BasicBlock* b2); // Return true if b1 dominates b2
5280 // Dominator computation member functions
5281 // Not exposed outside Compiler
5283 bool fgReachable(BasicBlock* b1, BasicBlock* b2); // Returns true if block b1 can reach block b2
5285 // Compute immediate dominators, the dominator tree and and its pre/post-order travsersal numbers.
5286 void fgComputeDoms();
5288 void fgCompDominatedByExceptionalEntryBlocks();
5290 BlockSet_ValRet_T fgGetDominatorSet(BasicBlock* block); // Returns a set of blocks that dominate the given block.
5291 // Note: this is relatively slow compared to calling fgDominate(),
5292 // especially if dealing with a single block versus block check.
5294 void fgComputeReachabilitySets(); // Compute bbReach sets. (Also sets BBF_GC_SAFE_POINT flag on blocks.)
5296 void fgComputeReturnBlocks(); // Initialize fgReturnBlocks to a list of BBJ_RETURN blocks.
5298 void fgComputeEnterBlocksSet(); // Compute the set of entry blocks, 'fgEnterBlks'.
5300 // Remove blocks determined to be unreachable by the 'canRemoveBlock'.
5301 template <typename CanRemoveBlockBody>
5302 bool fgRemoveUnreachableBlocks(CanRemoveBlockBody canRemoveBlock);
5304 PhaseStatus fgComputeReachability(); // Perform flow graph node reachability analysis.
5306 bool fgRemoveDeadBlocks(); // Identify and remove dead blocks.
5308 BasicBlock* fgIntersectDom(BasicBlock* a, BasicBlock* b); // Intersect two immediate dominator sets.
5310 void fgDfsReversePostorder();
5311 void fgDfsReversePostorderHelper(BasicBlock* block,
5313 unsigned& preorderIndex,
5314 unsigned& reversePostorderIndex);
5316 BlockSet_ValRet_T fgDomFindStartNodes(); // Computes which basic blocks don't have incoming edges in the flow graph.
5317 // Returns this as a set.
5319 INDEBUG(void fgDispDomTree(DomTreeNode* domTree);) // Helper that prints out the Dominator Tree in debug builds.
5321 DomTreeNode* fgBuildDomTree(); // Once we compute all the immediate dominator sets for each node in the flow graph
5322 // (performed by fgComputeDoms), this procedure builds the dominance tree represented
5325 // In order to speed up the queries of the form 'Does A dominates B', we can perform a DFS preorder and postorder
5326 // traversal of the dominance tree and the dominance query will become A dominates B iif preOrder(A) <= preOrder(B)
5327 // && postOrder(A) >= postOrder(B) making the computation O(1).
5328 void fgNumberDomTree(DomTreeNode* domTree);
5330 // When the flow graph changes, we need to update the block numbers, reachability sets,
5331 // dominators, and possibly loops.
5333 void fgUpdateChangedFlowGraph(FlowGraphUpdates updates);
5343 // Initialize the per-block variable sets (used for liveness analysis).
5344 void fgInitBlockVarSets();
5346 PhaseStatus StressSplitTree();
5347 void SplitTreesRandomly();
5348 void SplitTreesRemoveCommas();
5350 template <bool (Compiler::*ExpansionFunction)(BasicBlock**, Statement*, GenTreeCall*)>
5351 PhaseStatus fgExpandHelper(bool skipRarelyRunBlocks);
5353 template <bool (Compiler::*ExpansionFunction)(BasicBlock**, Statement*, GenTreeCall*)>
5354 bool fgExpandHelperForBlock(BasicBlock** pBlock);
5356 PhaseStatus fgExpandRuntimeLookups();
5357 bool fgExpandRuntimeLookupsForCall(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call);
5359 PhaseStatus fgExpandThreadLocalAccess();
5360 bool fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call);
5362 PhaseStatus fgExpandStaticInit();
5363 bool fgExpandStaticInitForCall(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call);
5365 PhaseStatus fgVNBasedIntrinsicExpansion();
5366 bool fgVNBasedIntrinsicExpansionForCall(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call);
5367 bool fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call);
5369 PhaseStatus fgInsertGCPolls();
5370 BasicBlock* fgCreateGCPoll(GCPollType pollType, BasicBlock* block);
5372 // Requires that "block" is a block that returns from
5373 // a finally. Returns the number of successors (jump targets of
5374 // of blocks in the covered "try" that did a "LEAVE".)
5375 unsigned fgNSuccsOfFinallyRet(BasicBlock* block);
5377 // Requires that "block" is a block that returns (in the sense of BBJ_EHFINALLYRET) from
5378 // a finally. Returns its "i"th successor (jump targets of
5379 // of blocks in the covered "try" that did a "LEAVE".)
5380 // Requires that "i" < fgNSuccsOfFinallyRet(block).
5381 BasicBlock* fgSuccOfFinallyRet(BasicBlock* block, unsigned i);
5384 // Factor out common portions of the impls of the methods above.
5385 void fgSuccOfFinallyRetWork(BasicBlock* block, unsigned i, BasicBlock** bres, unsigned* nres);
5388 // For many purposes, it is desirable to be able to enumerate the *distinct* targets of a switch statement,
5389 // skipping duplicate targets. (E.g., in flow analyses that are only interested in the set of possible targets.)
5390 // SwitchUniqueSuccSet contains the non-duplicated switch targets.
5391 // (Code that modifies the jump table of a switch has an obligation to call Compiler::UpdateSwitchTableTarget,
5392 // which in turn will call the "UpdateTarget" method of this type if a SwitchUniqueSuccSet has already
5393 // been computed for the switch block. If a switch block is deleted or is transformed into a non-switch,
5394 // we leave the entry associated with the block, but it will no longer be accessed.)
5395 struct SwitchUniqueSuccSet
5397 unsigned numDistinctSuccs; // Number of distinct targets of the switch.
5398 BasicBlock** nonDuplicates; // Array of "numDistinctSuccs", containing all the distinct switch target
5401 // The switch block "switchBlk" just had an entry with value "from" modified to the value "to".
5402 // Update "this" as necessary: if "from" is no longer an element of the jump table of "switchBlk",
5403 // remove it from "this", and ensure that "to" is a member. Use "alloc" to do any required allocation.
5404 void UpdateTarget(CompAllocator alloc, BasicBlock* switchBlk, BasicBlock* from, BasicBlock* to);
5407 typedef JitHashTable<BasicBlock*, JitPtrKeyFuncs<BasicBlock>, SwitchUniqueSuccSet> BlockToSwitchDescMap;
5410 // Maps BasicBlock*'s that end in switch statements to SwitchUniqueSuccSets that allow
5411 // iteration over only the distinct successors.
5412 BlockToSwitchDescMap* m_switchDescMap;
5415 BlockToSwitchDescMap* GetSwitchDescMap(bool createIfNull = true)
5417 if ((m_switchDescMap == nullptr) && createIfNull)
5419 m_switchDescMap = new (getAllocator()) BlockToSwitchDescMap(getAllocator());
5421 return m_switchDescMap;
5424 // Invalidate the map of unique switch block successors. For example, since the hash key of the map
5425 // depends on block numbers, we must invalidate the map when the blocks are renumbered, to ensure that
5426 // we don't accidentally look up and return the wrong switch data.
5427 void InvalidateUniqueSwitchSuccMap()
5429 m_switchDescMap = nullptr;
5432 // Requires "switchBlock" to be a block that ends in a switch. Returns
5433 // the corresponding SwitchUniqueSuccSet.
5434 SwitchUniqueSuccSet GetDescriptorForSwitch(BasicBlock* switchBlk);
5436 // The switch block "switchBlk" just had an entry with value "from" modified to the value "to".
5437 // Update "this" as necessary: if "from" is no longer an element of the jump table of "switchBlk",
5438 // remove it from "this", and ensure that "to" is a member.
5439 void UpdateSwitchTableTarget(BasicBlock* switchBlk, BasicBlock* from, BasicBlock* to);
5441 // Remove the "SwitchUniqueSuccSet" of "switchBlk" in the BlockToSwitchDescMap.
5442 void fgInvalidateSwitchDescMapEntry(BasicBlock* switchBlk);
5444 BasicBlock* fgFirstBlockOfHandler(BasicBlock* block);
5446 bool fgIsFirstBlockOfFilterOrHandler(BasicBlock* block);
5448 FlowEdge* fgGetPredForBlock(BasicBlock* block, BasicBlock* blockPred);
5450 FlowEdge* fgGetPredForBlock(BasicBlock* block, BasicBlock* blockPred, FlowEdge*** ptrToPred);
5452 FlowEdge* fgRemoveRefPred(BasicBlock* block, BasicBlock* blockPred);
5454 FlowEdge* fgRemoveAllRefPreds(BasicBlock* block, BasicBlock* blockPred);
5456 void fgRemoveBlockAsPred(BasicBlock* block);
5458 void fgChangeSwitchBlock(BasicBlock* oldSwitchBlock, BasicBlock* newSwitchBlock);
5460 void fgReplaceSwitchJumpTarget(BasicBlock* blockSwitch, BasicBlock* newTarget, BasicBlock* oldTarget);
5462 void fgReplaceJumpTarget(BasicBlock* block, BasicBlock* newTarget, BasicBlock* oldTarget);
5464 void fgReplacePred(BasicBlock* block, BasicBlock* oldPred, BasicBlock* newPred);
5466 // initializingPreds is only 'true' when we are computing preds in fgLinkBasicBlocks()
5467 template <bool initializingPreds = false>
5468 FlowEdge* fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowEdge* oldEdge = nullptr);
5470 void fgFindBasicBlocks();
5472 bool fgIsBetterFallThrough(BasicBlock* bCur, BasicBlock* bAlt);
5474 bool fgCheckEHCanInsertAfterBlock(BasicBlock* blk, unsigned regionIndex, bool putInTryRegion);
5476 BasicBlock* fgFindInsertPoint(unsigned regionIndex,
5477 bool putInTryRegion,
5478 BasicBlock* startBlk,
5480 BasicBlock* nearBlk,
5481 BasicBlock* jumpBlk,
5484 unsigned fgGetNestingLevel(BasicBlock* block, unsigned* pFinallyNesting = nullptr);
5486 PhaseStatus fgPostImportationCleanup();
5488 void fgRemoveStmt(BasicBlock* block, Statement* stmt DEBUGARG(bool isUnlink = false));
5489 void fgUnlinkStmt(BasicBlock* block, Statement* stmt);
5491 bool fgCheckRemoveStmt(BasicBlock* block, Statement* stmt);
5493 bool fgCreateLoopPreHeader(unsigned lnum);
5495 void fgUnreachableBlock(BasicBlock* block);
5497 void fgRemoveConditionalJump(BasicBlock* block);
5499 BasicBlock* fgLastBBInMainFunction();
5501 BasicBlock* fgEndBBAfterMainFunction();
5503 BasicBlock* fgGetDomSpeculatively(const BasicBlock* block);
5505 void fgUnlinkRange(BasicBlock* bBeg, BasicBlock* bEnd);
5507 void fgRemoveBlock(BasicBlock* block, bool unreachable);
5509 bool fgCanCompactBlocks(BasicBlock* block, BasicBlock* bNext);
5511 void fgCompactBlocks(BasicBlock* block, BasicBlock* bNext);
5513 void fgUpdateLoopsAfterCompacting(BasicBlock* block, BasicBlock* bNext);
5515 BasicBlock* fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst);
5517 bool fgRenumberBlocks();
5519 bool fgExpandRarelyRunBlocks();
5521 bool fgEhAllowsMoveBlock(BasicBlock* bBefore, BasicBlock* bAfter);
5523 void fgMoveBlocksAfter(BasicBlock* bStart, BasicBlock* bEnd, BasicBlock* insertAfterBlk);
5525 PhaseStatus fgTailMerge();
5527 enum FG_RELOCATE_TYPE
5529 FG_RELOCATE_TRY, // relocate the 'try' region
5530 FG_RELOCATE_HANDLER // relocate the handler region (including the filter if necessary)
5532 BasicBlock* fgRelocateEHRange(unsigned regionIndex, FG_RELOCATE_TYPE relocateType);
5534 #if defined(FEATURE_EH_FUNCLETS)
5535 #if defined(TARGET_ARM)
5536 void fgClearFinallyTargetBit(BasicBlock* block);
5537 #endif // defined(TARGET_ARM)
5538 bool fgIsIntraHandlerPred(BasicBlock* predBlock, BasicBlock* block);
5539 bool fgAnyIntraHandlerPreds(BasicBlock* block);
5540 void fgInsertFuncletPrologBlock(BasicBlock* block);
5541 void fgCreateFuncletPrologBlocks();
5542 PhaseStatus fgCreateFunclets();
5543 #else // !FEATURE_EH_FUNCLETS
5544 bool fgRelocateEHRegions();
5545 #endif // !FEATURE_EH_FUNCLETS
5547 bool fgOptimizeUncondBranchToSimpleCond(BasicBlock* block, BasicBlock* target);
5549 bool fgBlockEndFavorsTailDuplication(BasicBlock* block, unsigned lclNum);
5551 bool fgBlockIsGoodTailDuplicationCandidate(BasicBlock* block, unsigned* lclNum);
5553 bool fgOptimizeEmptyBlock(BasicBlock* block);
5555 bool fgOptimizeBranchToEmptyUnconditional(BasicBlock* block, BasicBlock* bDest);
5557 bool fgOptimizeBranch(BasicBlock* bJump);
5559 bool fgOptimizeSwitchBranches(BasicBlock* block);
5561 bool fgOptimizeBranchToNext(BasicBlock* block, BasicBlock* bNext, BasicBlock* bPrev);
5563 bool fgOptimizeSwitchJumps();
5565 void fgPrintEdgeWeights();
5567 PhaseStatus fgComputeBlockAndEdgeWeights();
5568 bool fgComputeMissingBlockWeights(weight_t* returnWeight);
5569 bool fgComputeCalledCount(weight_t returnWeight);
5570 PhaseStatus fgComputeEdgeWeights();
5572 bool fgReorderBlocks(bool useProfile);
5574 #ifdef FEATURE_EH_FUNCLETS
5575 bool fgFuncletsAreCold();
5576 #endif // FEATURE_EH_FUNCLETS
5578 PhaseStatus fgDetermineFirstColdBlock();
5580 bool fgIsForwardBranch(BasicBlock* bJump, BasicBlock* bSrc = nullptr);
5582 bool fgUpdateFlowGraph(bool doTailDup = false, bool isPhase = false);
5583 PhaseStatus fgUpdateFlowGraphPhase();
5585 PhaseStatus fgFindOperOrder();
5587 // method that returns if you should split here
5588 typedef bool(fgSplitPredicate)(GenTree* tree, GenTree* parent, fgWalkData* data);
5590 PhaseStatus fgSetBlockOrder();
5592 void fgRemoveReturnBlock(BasicBlock* block);
5594 /* Helper code that has been factored out */
5595 inline void fgConvertBBToThrowBB(BasicBlock* block);
5597 bool fgCastNeeded(GenTree* tree, var_types toType);
5599 // The following check for loops that don't execute calls
5600 bool fgLoopCallMarked;
5602 void fgLoopCallTest(BasicBlock* srcBB, BasicBlock* dstBB);
5603 void fgLoopCallMark();
5605 void fgMarkLoopHead(BasicBlock* block);
5607 unsigned fgGetCodeEstimate(BasicBlock* block);
5610 enum class PhasePosition
5615 const char* fgProcessEscapes(const char* nameIn, escapeMapping_t* map);
5616 static void fgDumpTree(FILE* fgxFile, GenTree* const tree);
5617 FILE* fgOpenFlowGraphFile(bool* wbDontClose, Phases phase, PhasePosition pos, const char* type);
5618 bool fgDumpFlowGraph(Phases phase, PhasePosition pos);
5619 #endif // DUMP_FLOWGRAPHS
5625 void fgDispBBLiveness(BasicBlock* block);
5626 void fgDispBBLiveness();
5627 void fgTableDispBasicBlock(BasicBlock* block, int ibcColWidth = 0);
5628 void fgDispBasicBlocks(BasicBlock* firstBlock, BasicBlock* lastBlock, bool dumpTrees);
5629 void fgDispBasicBlocks(bool dumpTrees = false);
5630 void fgDumpStmtTree(Statement* stmt, unsigned bbNum);
5631 void fgDumpBlock(BasicBlock* block);
5632 void fgDumpTrees(BasicBlock* firstBlock, BasicBlock* lastBlock);
5634 static fgWalkPreFn fgStress64RsltMulCB;
5635 void fgStress64RsltMul();
5636 void fgDebugCheckUpdate();
5638 void fgDebugCheckBBNumIncreasing();
5639 void fgDebugCheckBBlist(bool checkBBNum = false, bool checkBBRefs = true);
5640 void fgDebugCheckBlockLinks();
5641 void fgDebugCheckLinks(bool morphTrees = false);
5642 void fgDebugCheckStmtsList(BasicBlock* block, bool morphTrees);
5643 void fgDebugCheckNodeLinks(BasicBlock* block, Statement* stmt);
5644 void fgDebugCheckLinkedLocals();
5645 void fgDebugCheckNodesUniqueness();
5646 void fgDebugCheckLoopTable();
5647 void fgDebugCheckSsa();
5649 void fgDebugCheckFlags(GenTree* tree);
5650 void fgDebugCheckDispFlags(GenTree* tree, GenTreeFlags dispFlags, GenTreeDebugFlags debugFlags);
5651 void fgDebugCheckFlagsHelper(GenTree* tree, GenTreeFlags actualFlags, GenTreeFlags expectedFlags);
5652 void fgDebugCheckTryFinallyExits();
5653 void fgDebugCheckProfileWeights();
5654 void fgDebugCheckProfileWeights(ProfileChecks checks);
5655 bool fgDebugCheckIncomingProfileData(BasicBlock* block, ProfileChecks checks);
5656 bool fgDebugCheckOutgoingProfileData(BasicBlock* block, ProfileChecks checks);
5660 static bool fgProfileWeightsEqual(weight_t weight1, weight_t weight2, weight_t epsilon = 0.01);
5661 static bool fgProfileWeightsConsistent(weight_t weight1, weight_t weight2);
5663 static GenTree* fgGetFirstNode(GenTree* tree);
5665 //--------------------- Walking the trees in the IR -----------------------
5670 fgWalkPreFn* wtprVisitorFn;
5671 fgWalkPostFn* wtpoVisitorFn;
5672 void* pCallbackData; // user-provided data
5673 GenTree* parent; // parent of current node, provided to callback
5674 bool wtprLclsOnly; // whether to only visit lclvar nodes
5676 bool printModified; // callback can use this
5680 fgWalkResult fgWalkTreePre(GenTree** pTree,
5681 fgWalkPreFn* visitor,
5682 void* pCallBackData = nullptr,
5683 bool lclVarsOnly = false,
5684 bool computeStack = false);
5686 fgWalkResult fgWalkTree(GenTree** pTree,
5687 fgWalkPreFn* preVisitor,
5688 fgWalkPostFn* postVisitor,
5689 void* pCallBackData = nullptr);
5691 void fgWalkAllTreesPre(fgWalkPreFn* visitor, void* pCallBackData);
5695 fgWalkResult fgWalkTreePost(GenTree** pTree,
5696 fgWalkPostFn* visitor,
5697 void* pCallBackData = nullptr,
5698 bool computeStack = false);
5700 /**************************************************************************
5702 *************************************************************************/
5705 friend class SsaBuilder;
5706 friend struct ValueNumberState;
5708 //--------------------- Detect the basic blocks ---------------------------
5710 BasicBlock** fgBBs; // Table of pointers to the BBs
5712 void fgInitBBLookup();
5713 BasicBlock* fgLookupBB(unsigned addr);
5715 bool fgCanSwitchToOptimized();
5716 void fgSwitchToOptimized(const char* reason);
5718 bool fgMayExplicitTailCall();
5720 void fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, FixedBitVect* jumpTarget);
5722 void fgMarkBackwardJump(BasicBlock* startBlock, BasicBlock* endBlock);
5724 void fgLinkBasicBlocks();
5726 unsigned fgMakeBasicBlocks(const BYTE* codeAddr, IL_OFFSET codeSize, FixedBitVect* jumpTarget);
5728 void fgCheckBasicBlockControlFlow();
5730 void fgControlFlowPermitted(BasicBlock* blkSrc,
5731 BasicBlock* blkDest,
5732 bool IsLeave = false /* is the src a leave block */);
5734 bool fgFlowToFirstBlockOfInnerTry(BasicBlock* blkSrc, BasicBlock* blkDest, bool sibling);
5736 void fgObserveInlineConstants(OPCODE opcode, const FgStack& stack, bool isInlining);
5738 void fgAdjustForAddressExposedOrWrittenThis();
5740 unsigned fgStressBBProf()
5743 unsigned result = JitConfig.JitStressBBProf();
5746 if (compStressCompile(STRESS_BB_PROFILE, 15))
5757 bool fgHaveProfileData();
5758 bool fgHaveProfileWeights();
5759 bool fgGetProfileWeightForBasicBlock(IL_OFFSET offset, weight_t* weight);
5761 Instrumentor* fgCountInstrumentor;
5762 Instrumentor* fgHistogramInstrumentor;
5764 PhaseStatus fgPrepareToInstrumentMethod();
5765 PhaseStatus fgInstrumentMethod();
5766 PhaseStatus fgIncorporateProfileData();
5767 bool fgIncorporateBlockCounts();
5768 bool fgIncorporateEdgeCounts();
5771 const char* fgPgoFailReason;
5773 ICorJitInfo::PgoSource fgPgoSource;
5774 ICorJitInfo::PgoInstrumentationSchema* fgPgoSchema;
5776 UINT32 fgPgoSchemaCount;
5777 HRESULT fgPgoQueryResult;
5778 UINT32 fgNumProfileRuns;
5779 UINT32 fgPgoBlockCounts;
5780 UINT32 fgPgoEdgeCounts;
5781 UINT32 fgPgoClassProfiles;
5782 UINT32 fgPgoMethodProfiles;
5783 unsigned fgPgoInlineePgo;
5784 unsigned fgPgoInlineeNoPgo;
5785 unsigned fgPgoInlineeNoPgoSingleBlock;
5786 bool fgPgoHaveWeights;
5788 void WalkSpanningTree(SpanningTreeVisitor* visitor);
5789 void fgSetProfileWeight(BasicBlock* block, weight_t weight);
5790 void fgApplyProfileScale();
5791 bool fgHaveSufficientProfileWeights();
5792 bool fgHaveTrustedProfileWeights();
5794 // fgIsUsingProfileWeights - returns true if we have real profile data for this method
5795 // or if we have some fake profile data for the stress mode
5796 bool fgIsUsingProfileWeights()
5798 return (fgHaveProfileWeights() || fgStressBBProf());
5801 // fgProfileRunsCount - returns total number of scenario runs for the profile data
5802 // or BB_UNITY_WEIGHT_UNSIGNED when we aren't using profile data.
5803 unsigned fgProfileRunsCount()
5805 return fgIsUsingProfileWeights() ? fgNumProfileRuns : BB_UNITY_WEIGHT_UNSIGNED;
5808 //-------- Insert a statement at the start or end of a basic block --------
5812 static bool fgBlockContainsStatementBounded(BasicBlock* block, Statement* stmt, bool answerOnBoundExceeded = true);
5816 Statement* fgNewStmtAtBeg(BasicBlock* block, GenTree* tree, const DebugInfo& di = DebugInfo());
5817 void fgInsertStmtAtEnd(BasicBlock* block, Statement* stmt);
5818 Statement* fgNewStmtAtEnd(BasicBlock* block, GenTree* tree, const DebugInfo& di = DebugInfo());
5819 Statement* fgNewStmtNearEnd(BasicBlock* block, GenTree* tree, const DebugInfo& di = DebugInfo());
5822 void fgInsertStmtNearEnd(BasicBlock* block, Statement* stmt);
5823 void fgInsertStmtAtBeg(BasicBlock* block, Statement* stmt);
5826 void fgInsertStmtAfter(BasicBlock* block, Statement* insertionPoint, Statement* stmt);
5827 void fgInsertStmtBefore(BasicBlock* block, Statement* insertionPoint, Statement* stmt);
5830 Statement* fgInsertStmtListAfter(BasicBlock* block, Statement* stmtAfter, Statement* stmtList);
5832 // Create a new temporary variable to hold the result of *ppTree,
5833 // and transform the graph accordingly.
5834 GenTree* fgInsertCommaFormTemp(GenTree** ppTree);
5835 TempInfo fgMakeTemp(GenTree* rhs);
5836 GenTree* fgMakeMultiUse(GenTree** ppTree);
5838 // Recognize a bitwise rotation pattern and convert into a GT_ROL or a GT_ROR node.
5839 GenTree* fgRecognizeAndMorphBitwiseRotation(GenTree* tree);
5840 bool fgOperIsBitwiseRotationRoot(genTreeOps oper);
5842 #if !defined(TARGET_64BIT)
5843 // Recognize and morph a long multiplication with 32 bit operands.
5844 GenTreeOp* fgRecognizeAndMorphLongMul(GenTreeOp* mul);
5845 GenTreeOp* fgMorphLongMul(GenTreeOp* mul);
5848 //-------- Determine the order in which the trees will be evaluated -------
5850 void fgSetStmtSeq(Statement* stmt);
5853 GenTree* fgSetTreeSeq(GenTree* tree, bool isLIR = false);
5854 void fgSetBlockOrder(BasicBlock* block);
5856 //------------------------- Morphing --------------------------------------
5858 unsigned fgPtrArgCntMax;
5861 //------------------------------------------------------------------------
5862 // fgGetPtrArgCntMax: Return the maximum number of pointer-sized stack arguments that calls inside this method
5863 // can push on the stack. This value is calculated during morph.
5866 // Returns fgPtrArgCntMax, that is a private field.
5868 unsigned fgGetPtrArgCntMax() const
5870 return fgPtrArgCntMax;
5873 //------------------------------------------------------------------------
5874 // fgSetPtrArgCntMax: Set the maximum number of pointer-sized stack arguments that calls inside this method
5875 // can push on the stack. This function is used during StackLevelSetter to fix incorrect morph calculations.
5877 void fgSetPtrArgCntMax(unsigned argCntMax)
5879 fgPtrArgCntMax = argCntMax;
5882 bool compCanEncodePtrArgCntMax();
5885 hashBv* fgAvailableOutgoingArgTemps;
5886 ArrayStack<unsigned>* fgUsedSharedTemps;
5888 void fgSetRngChkTarget(GenTree* tree, bool delay = true);
5890 BasicBlock* fgSetRngChkTargetInner(SpecialCodeKind kind, bool delay);
5893 void fgMoveOpsLeft(GenTree* tree);
5896 bool fgIsCommaThrow(GenTree* tree, bool forFolding = false);
5898 bool fgIsThrow(GenTree* tree);
5900 bool fgInDifferentRegions(BasicBlock* blk1, BasicBlock* blk2);
5901 bool fgIsBlockCold(BasicBlock* block);
5903 GenTree* fgMorphCastIntoHelper(GenTree* tree, int helper, GenTree* oper);
5905 GenTree* fgMorphIntoHelperCall(
5906 GenTree* tree, int helper, bool morphArgs, GenTree* arg1 = nullptr, GenTree* arg2 = nullptr);
5908 // A "MorphAddrContext" carries information from the surrounding context. If we are evaluating a byref address,
5909 // it is useful to know whether the address will be immediately dereferenced, or whether the address value will
5910 // be used, perhaps by passing it as an argument to a called method. This affects how null checking is done:
5911 // for sufficiently small offsets, we can rely on OS page protection to implicitly null-check addresses that we
5912 // know will be dereferenced. To know that reliance on implicit null checking is sound, we must further know that
5913 // all offsets between the top-level indirection and the bottom are constant, and that their sum is sufficiently
5914 // small; hence the other fields of MorphAddrContext.
5915 struct MorphAddrContext
5917 size_t m_totalOffset = 0; // Sum of offsets between the top-level indirection and here (current context).
5918 bool m_used = false; // Whether this context was used to elide a null check.
5922 GenTree* getSIMDStructFromField(GenTree* tree,
5924 unsigned* simdSizeOut,
5925 bool ignoreUsedInSIMDIntrinsic = false);
5926 bool fgMorphCombineSIMDFieldStores(BasicBlock* block, Statement* stmt);
5927 void impMarkContiguousSIMDFieldStores(Statement* stmt);
5929 // fgPreviousCandidateSIMDFieldStoreStmt is only used for tracking previous simd field assignment
5930 // in function: Compiler::impMarkContiguousSIMDFieldStores.
5931 Statement* fgPreviousCandidateSIMDFieldStoreStmt;
5933 #endif // FEATURE_SIMD
5934 GenTree* fgMorphIndexAddr(GenTreeIndexAddr* tree);
5935 GenTree* fgMorphExpandCast(GenTreeCast* tree);
5936 GenTreeFieldList* fgMorphLclArgToFieldlist(GenTreeLclVarCommon* lcl);
5937 GenTreeCall* fgMorphArgs(GenTreeCall* call);
5939 void fgMakeOutgoingStructArgCopy(GenTreeCall* call, CallArg* arg);
5941 GenTree* fgMorphLeafLocal(GenTreeLclVarCommon* lclNode);
5943 GenTree* fgMorphExpandStackArgForVarArgs(GenTreeLclVarCommon* lclNode);
5944 #endif // TARGET_X86
5945 GenTree* fgMorphExpandImplicitByRefArg(GenTreeLclVarCommon* lclNode);
5946 GenTree* fgMorphExpandLocal(GenTreeLclVarCommon* lclNode);
5949 bool fgAddrCouldBeNull(GenTree* addr);
5950 void fgAssignSetVarDef(GenTree* tree);
5953 GenTree* fgMorphFieldAddr(GenTree* tree, MorphAddrContext* mac);
5954 GenTree* fgMorphExpandInstanceField(GenTree* tree, MorphAddrContext* mac);
5955 GenTree* fgMorphExpandTlsFieldAddr(GenTree* tree);
5956 bool fgCanFastTailCall(GenTreeCall* call, const char** failReason);
5957 #if FEATURE_FASTTAILCALL
5958 bool fgCallHasMustCopyByrefParameter(GenTreeCall* call);
5959 bool fgCallArgWillPointIntoLocalFrame(GenTreeCall* call, CallArg& arg);
5962 bool fgCheckStmtAfterTailCall();
5963 GenTree* fgMorphTailCallViaHelpers(GenTreeCall* call, CORINFO_TAILCALL_HELPERS& help);
5964 bool fgCanTailCallViaJitHelper(GenTreeCall* call);
5965 void fgMorphTailCallViaJitHelper(GenTreeCall* call);
5966 GenTree* fgCreateCallDispatcherAndGetResult(GenTreeCall* origCall,
5967 CORINFO_METHOD_HANDLE callTargetStubHnd,
5968 CORINFO_METHOD_HANDLE dispatcherHnd);
5969 GenTree* getLookupTree(CORINFO_RESOLVED_TOKEN* pResolvedToken,
5970 CORINFO_LOOKUP* pLookup,
5971 GenTreeFlags handleFlags,
5972 void* compileTimeHandle);
5973 GenTree* getRuntimeLookupTree(CORINFO_RESOLVED_TOKEN* pResolvedToken,
5974 CORINFO_LOOKUP* pLookup,
5975 void* compileTimeHandle);
5976 GenTree* getVirtMethodPointerTree(GenTree* thisPtr,
5977 CORINFO_RESOLVED_TOKEN* pResolvedToken,
5978 CORINFO_CALL_INFO* pCallInfo);
5979 GenTree* getTokenHandleTree(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool parent);
5981 GenTree* fgMorphPotentialTailCall(GenTreeCall* call);
5982 void fgValidateIRForTailCall(GenTreeCall* call);
5983 GenTree* fgGetStubAddrArg(GenTreeCall* call);
5984 unsigned fgGetArgParameterLclNum(GenTreeCall* call, CallArg* arg);
5985 void fgMorphRecursiveFastTailCallIntoLoop(BasicBlock* block, GenTreeCall* recursiveTailCall);
5986 Statement* fgAssignRecursiveCallArgToCallerParam(GenTree* arg,
5988 unsigned lclParamNum,
5990 const DebugInfo& callDI,
5991 Statement* tmpAssignmentInsertionPoint,
5992 Statement* paramAssignmentInsertionPoint);
5993 GenTree* fgMorphCall(GenTreeCall* call);
5994 GenTree* fgExpandVirtualVtableCallTarget(GenTreeCall* call);
5996 void fgMorphCallInline(GenTreeCall* call, InlineResult* result);
5997 void fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
5999 void fgNoteNonInlineCandidate(Statement* stmt, GenTreeCall* call);
6000 static fgWalkPreFn fgFindNonInlineCandidate;
6002 GenTree* fgOptimizeDelegateConstructor(GenTreeCall* call,
6003 CORINFO_CONTEXT_HANDLE* ExactContextHnd,
6004 methodPointerInfo* ldftnToken);
6005 GenTree* fgMorphLeaf(GenTree* tree);
6007 GenTree* fgMorphInitBlock(GenTree* tree);
6008 GenTree* fgMorphCopyBlock(GenTree* tree);
6009 GenTree* fgMorphStoreDynBlock(GenTreeStoreDynBlk* tree);
6011 GenTree* fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optAssertionPropDone = nullptr);
6012 void fgTryReplaceStructLocalWithField(GenTree* tree);
6013 GenTree* fgMorphFinalizeIndir(GenTreeIndir* indir);
6014 GenTree* fgOptimizeCast(GenTreeCast* cast);
6015 GenTree* fgOptimizeCastOnStore(GenTree* store);
6016 GenTree* fgOptimizeBitCast(GenTreeUnOp* bitCast);
6017 GenTree* fgOptimizeEqualityComparisonWithConst(GenTreeOp* cmp);
6018 GenTree* fgOptimizeRelationalComparisonWithConst(GenTreeOp* cmp);
6019 GenTree* fgOptimizeRelationalComparisonWithFullRangeConst(GenTreeOp* cmp);
6020 #ifdef FEATURE_HW_INTRINSICS
6021 GenTree* fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node);
6023 GenTree* fgOptimizeCommutativeArithmetic(GenTreeOp* tree);
6024 GenTree* fgOptimizeRelationalComparisonWithCasts(GenTreeOp* cmp);
6025 GenTree* fgOptimizeAddition(GenTreeOp* add);
6026 GenTree* fgOptimizeMultiply(GenTreeOp* mul);
6027 GenTree* fgOptimizeBitwiseAnd(GenTreeOp* andOp);
6028 GenTree* fgOptimizeBitwiseXor(GenTreeOp* xorOp);
6029 GenTree* fgPropagateCommaThrow(GenTree* parent, GenTreeOp* commaThrow, GenTreeFlags precedingSideEffects);
6030 GenTree* fgMorphRetInd(GenTreeUnOp* tree);
6031 GenTree* fgMorphModToZero(GenTreeOp* tree);
6032 GenTree* fgMorphModToSubMulDiv(GenTreeOp* tree);
6033 GenTree* fgMorphUModToAndSub(GenTreeOp* tree);
6034 GenTree* fgMorphSmpOpOptional(GenTreeOp* tree, bool* optAssertionPropDone);
6035 GenTree* fgMorphMultiOp(GenTreeMultiOp* multiOp);
6036 GenTree* fgMorphConst(GenTree* tree);
6038 GenTreeOp* fgMorphCommutative(GenTreeOp* tree);
6040 GenTree* fgMorphReduceAddOps(GenTree* tree);
6043 GenTree* fgMorphTree(GenTree* tree, MorphAddrContext* mac = nullptr);
6046 void fgKillDependentAssertionsSingle(unsigned lclNum DEBUGARG(GenTree* tree));
6047 void fgKillDependentAssertions(unsigned lclNum DEBUGARG(GenTree* tree));
6048 void fgMorphTreeDone(GenTree* tree);
6049 void fgMorphTreeDone(GenTree* tree, bool optAssertionPropDone, bool isMorphedTree DEBUGARG(int morphNum = 0));
6051 Statement* fgMorphStmt;
6052 unsigned fgBigOffsetMorphingTemps[TYP_COUNT];
6054 unsigned fgGetFieldMorphingTemp(GenTreeFieldAddr* fieldNode);
6056 //----------------------- Liveness analysis -------------------------------
6058 VARSET_TP fgCurUseSet; // vars used by block (before an assignment)
6059 VARSET_TP fgCurDefSet; // vars assigned by block (before a use)
6061 MemoryKindSet fgCurMemoryUse; // True iff the current basic block uses memory.
6062 MemoryKindSet fgCurMemoryDef; // True iff the current basic block modifies memory.
6063 MemoryKindSet fgCurMemoryHavoc; // True if the current basic block is known to set memory to a "havoc" value.
6065 bool byrefStatesMatchGcHeapStates; // True iff GcHeap and ByrefExposed memory have all the same def points.
6067 PhaseStatus fgEarlyLiveness();
6069 void fgMarkUseDef(GenTreeLclVarCommon* tree);
6071 void fgBeginScopeLife(VARSET_TP* inScope, VarScopeDsc* var);
6072 void fgEndScopeLife(VARSET_TP* inScope, VarScopeDsc* var);
6074 void fgMarkInScope(BasicBlock* block, VARSET_VALARG_TP inScope);
6075 void fgUnmarkInScope(BasicBlock* block, VARSET_VALARG_TP unmarkScope);
6077 void fgExtendDbgScopes();
6078 void fgExtendDbgLifetimes();
6081 void fgDispDebugScopes();
6084 //-------------------------------------------------------------------------
6086 // The following keeps track of any code we've added for things like array
6087 // range checking or explicit calls to enable GC, and so on.
6092 AddCodeDsc* acdNext;
6093 BasicBlock* acdDstBlk; // block to which we jump
6095 SpecialCodeKind acdKind; // what kind of a special block is this?
6096 #if !FEATURE_FIXED_OUT_ARGS
6097 bool acdStkLvlInit; // has acdStkLvl value been already set?
6098 unsigned acdStkLvl; // stack level in stack slots.
6099 #endif // !FEATURE_FIXED_OUT_ARGS
6103 static unsigned acdHelper(SpecialCodeKind codeKind);
6105 AddCodeDsc* fgAddCodeList;
6107 bool fgRngChkThrowAdded;
6108 AddCodeDsc* fgExcptnTargetCache[SCK_COUNT];
6110 BasicBlock* fgRngChkTarget(BasicBlock* block, SpecialCodeKind kind);
6112 BasicBlock* fgAddCodeRef(BasicBlock* srcBlk, unsigned refData, SpecialCodeKind kind);
6115 AddCodeDsc* fgFindExcptnTarget(SpecialCodeKind kind, unsigned refData);
6117 bool fgUseThrowHelperBlocks();
6119 AddCodeDsc* fgGetAdditionalCodeDescriptors()
6121 return fgAddCodeList;
6125 bool fgIsCodeAdded();
6127 bool fgIsThrowHlpBlk(BasicBlock* block);
6129 #if !FEATURE_FIXED_OUT_ARGS
6130 unsigned fgThrowHlpBlkStkLevel(BasicBlock* block);
6131 #endif // !FEATURE_FIXED_OUT_ARGS
6133 unsigned fgCheckInlineDepthAndRecursion(InlineInfo* inlineInfo);
6134 bool IsDisallowedRecursiveInline(InlineContext* ancestor, InlineInfo* inlineInfo);
6135 bool ContextComplexityExceeds(CORINFO_CONTEXT_HANDLE handle, int max);
6136 bool MethodInstantiationComplexityExceeds(CORINFO_METHOD_HANDLE handle, int& cur, int max);
6137 bool TypeInstantiationComplexityExceeds(CORINFO_CLASS_HANDLE handle, int& cur, int max);
6139 void fgInvokeInlineeCompiler(GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
6140 void fgInsertInlineeBlocks(InlineInfo* pInlineInfo);
6141 Statement* fgInlinePrependStatements(InlineInfo* inlineInfo);
6142 void fgInlineAppendStatements(InlineInfo* inlineInfo, BasicBlock* block, Statement* stmt);
6145 static fgWalkPreFn fgDebugCheckInlineCandidates;
6147 void CheckNoTransformableIndirectCallsRemain();
6148 static fgWalkPreFn fgDebugCheckForTransformableIndirectCalls;
6151 PhaseStatus fgPromoteStructs();
6152 void fgMorphLocalField(GenTree* tree, GenTree* parent);
6154 // Reset the refCount for implicit byrefs.
6155 void fgResetImplicitByRefRefCount();
6157 // Identify all candidates for last-use copy omission.
6158 PhaseStatus fgMarkImplicitByRefCopyOmissionCandidates();
6160 // Change implicit byrefs' types from struct to pointer, and for any that were
6161 // promoted, create new promoted struct temps.
6162 PhaseStatus fgRetypeImplicitByRefArgs();
6164 // Clear up annotations for any struct promotion temps created for implicit byrefs.
6165 void fgMarkDemotedImplicitByRefArgs();
6167 PhaseStatus fgMarkAddressExposedLocals();
6168 void fgSequenceLocals(Statement* stmt);
6170 PhaseStatus PhysicalPromotion();
6172 PhaseStatus fgForwardSub();
6173 bool fgForwardSubBlock(BasicBlock* block);
6174 bool fgForwardSubStatement(Statement* statement);
6175 bool fgForwardSubHasStoreInterference(Statement* defStmt, Statement* nextStmt, GenTree* nextStmtUse);
6176 void fgForwardSubUpdateLiveness(GenTree* newSubListFirst, GenTree* newSubListLast);
6178 // The given local variable, required to be a struct variable, is being assigned via
6179 // a "lclField", to make it masquerade as an integral type in the ABI. Make sure that
6180 // the variable is not enregistered, and is therefore not promoted independently.
6181 void fgLclFldAssign(unsigned lclNum);
6183 enum TypeProducerKind
6185 TPK_Unknown = 0, // May not be a RuntimeType
6186 TPK_Handle = 1, // RuntimeType via handle
6187 TPK_GetType = 2, // RuntimeType via Object.get_Type()
6188 TPK_Null = 3, // Tree value is null
6189 TPK_Other = 4 // RuntimeType via other means
6192 TypeProducerKind gtGetTypeProducerKind(GenTree* tree);
6193 bool gtIsTypeHandleToRuntimeTypeHelper(GenTreeCall* call);
6194 bool gtIsTypeHandleToRuntimeTypeHandleHelper(GenTreeCall* call, CorInfoHelpFunc* pHelper = nullptr);
6195 bool gtIsActiveCSE_Candidate(GenTree* tree);
6197 bool gtTreeContainsOper(GenTree* tree, genTreeOps op);
6198 ExceptionSetFlags gtCollectExceptions(GenTree* tree);
6201 bool fgIsBigOffset(size_t offset);
6204 bool fgNeedReturnSpillTemp();
6207 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
6208 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
6212 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
6213 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
6219 PhaseStatus rangeCheckPhase();
6220 GenTree* optRemoveRangeCheck(GenTreeBoundsChk* check, GenTree* comma, Statement* stmt);
6221 GenTree* optRemoveStandaloneRangeCheck(GenTreeBoundsChk* check, Statement* stmt);
6222 void optRemoveCommaBasedRangeCheck(GenTree* comma, Statement* stmt);
6225 // Do hoisting for all loops.
6226 PhaseStatus optHoistLoopCode();
6228 // To represent sets of VN's that have already been hoisted in outer loops.
6229 typedef JitHashTable<ValueNum, JitSmallPrimitiveKeyFuncs<ValueNum>, bool> VNSet;
6231 struct LoopHoistContext
6234 // The set of variables hoisted in the current loop (or nullptr if there are none).
6235 VNSet* m_pHoistedInCurLoop;
6238 // Value numbers of expressions that have been hoisted in the current (or most recent) loop in the nest.
6239 // Previous decisions on loop-invariance of value numbers in the current loop.
6240 VNSet m_curLoopVnInvariantCache;
6242 // Get the VN cache for current loop
6243 VNSet* GetHoistedInCurLoop(Compiler* comp)
6245 if (m_pHoistedInCurLoop == nullptr)
6247 m_pHoistedInCurLoop = new (comp->getAllocatorLoopHoist()) VNSet(comp->getAllocatorLoopHoist());
6249 return m_pHoistedInCurLoop;
6252 // Return the so far collected VNs in cache for current loop and reset it.
6253 void ResetHoistedInCurLoop()
6255 m_pHoistedInCurLoop = nullptr;
6256 JITDUMP("Resetting m_pHoistedInCurLoop\n");
6259 LoopHoistContext(Compiler* comp)
6260 : m_pHoistedInCurLoop(nullptr), m_curLoopVnInvariantCache(comp->getAllocatorLoopHoist())
6265 // Do hoisting of all loops nested within loop "lnum" (an index into the optLoopTable), followed
6266 // by the loop "lnum" itself.
6267 bool optHoistLoopNest(unsigned lnum, LoopHoistContext* hoistCtxt);
6269 // Do hoisting for a particular loop ("lnum" is an index into the optLoopTable.)
6270 bool optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt);
6272 // Hoist all expressions in "blocks" that are invariant in loop "loopNum" (an index into the optLoopTable)
6273 // outside of that loop.
6274 void optHoistLoopBlocks(unsigned loopNum, ArrayStack<BasicBlock*>* blocks, LoopHoistContext* hoistContext);
6276 // Return true if the tree looks profitable to hoist out of loop 'lnum'.
6277 bool optIsProfitableToHoistTree(GenTree* tree, unsigned lnum);
6279 // Performs the hoisting 'tree' into the PreHeader for loop 'lnum'
6280 void optHoistCandidate(GenTree* tree, BasicBlock* treeBb, unsigned lnum, LoopHoistContext* hoistCtxt);
6282 // Note the new SSA uses in tree
6283 void optRecordSsaUses(GenTree* tree, BasicBlock* block);
6285 // Returns true iff the ValueNum "vn" represents a value that is loop-invariant in "lnum".
6286 // Constants and init values are always loop invariant.
6287 // VNPhi's connect VN's to the SSA definition, so we can know if the SSA def occurs in the loop.
6288 bool optVNIsLoopInvariant(ValueNum vn, unsigned lnum, VNSet* recordedVNs);
6290 // If "blk" is the entry block of a natural loop, returns true and sets "*pLnum" to the index of the loop
6291 // in the loop table.
6292 bool optBlockIsLoopEntry(BasicBlock* blk, unsigned* pLnum);
6294 // Records the set of "side effects" of all loops: fields (object instance and static)
6295 // written to, and SZ-array element type equivalence classes updated.
6296 void optComputeLoopSideEffects();
6298 // Compute the sets of long and float vars (lvaLongVars, lvaFloatVars).
6299 void optComputeInterestingVarSets();
6302 bool optAnyChildNotRemoved(unsigned loopNum);
6305 // Mark a loop as removed.
6306 void optMarkLoopRemoved(unsigned loopNum);
6309 // Requires "lnum" to be the index of an outermost loop in the loop table. Traverses the body of that loop,
6310 // including all nested loops, and records the set of "side effects" of the loop: fields (object instance and
6311 // static) written to, and SZ-array element type equivalence classes updated.
6312 void optComputeLoopNestSideEffects(unsigned lnum);
6314 // Given a loop number 'lnum' mark it and any nested loops as having 'memoryHavoc'
6315 void optRecordLoopNestsMemoryHavoc(unsigned lnum, MemoryKindSet memoryHavoc);
6317 // Add the side effects of "blk" (which is required to be within a loop) to all loops of which it is a part.
6318 // Returns false if we encounter a block that is not marked as being inside a loop.
6320 bool optComputeLoopSideEffectsOfBlock(BasicBlock* blk);
6322 // Hoist the expression "expr" out of loop "lnum".
6323 void optPerformHoistExpr(GenTree* expr, BasicBlock* exprBb, unsigned lnum);
6326 PhaseStatus optOptimizeBools();
6329 PhaseStatus optInvertLoops(); // Invert loops so they're entered at top and tested at bottom.
6330 PhaseStatus optOptimizeFlow(); // Simplify flow graph and do tail duplication
6331 PhaseStatus optOptimizeLayout(); // Optimize the BasicBlock layout of the method
6332 PhaseStatus optSetBlockWeights();
6333 PhaseStatus optFindLoopsPhase(); // Finds loops and records them in the loop table
6335 void optFindLoops();
6337 PhaseStatus optCloneLoops();
6338 void optCloneLoop(unsigned loopInd, LoopCloneContext* context);
6339 PhaseStatus optUnrollLoops(); // Unrolls loops (needs to have cost info)
6340 void optRemoveRedundantZeroInits();
6341 PhaseStatus optIfConversion(); // If conversion
6344 // This enumeration describes what is killed by a call.
6348 CALLINT_NONE, // no interference (most helpers)
6349 CALLINT_REF_INDIRS, // kills GC ref indirections (SETFIELD OBJ)
6350 CALLINT_SCL_INDIRS, // kills non GC ref indirections (SETFIELD non-OBJ)
6351 CALLINT_ALL_INDIRS, // kills both GC ref and non GC ref indirections (SETFIELD STRUCT)
6352 CALLINT_ALL, // kills everything (normal method call)
6355 enum class FieldKindForVN
6362 // A "LoopDsc" describes a ("natural") loop. We (currently) require the body of a loop to be a contiguous (in
6363 // bbNext order) sequence of basic blocks. (At times, we may require the blocks in a loop to be "properly numbered"
6364 // in bbNext order; we use comparisons on the bbNum to decide order.)
6365 // The blocks that define the body are
6366 // top <= entry <= bottom
6367 // The "head" of the loop is a block outside the loop that has "entry" as a successor. We only support loops with a
6368 // single 'head' block. The meanings of these blocks are given in the definitions below. Also see the picture at
6369 // Compiler::optFindNaturalLoops().
6372 BasicBlock* lpHead; // HEAD of the loop (not part of the looping of the loop) -- has ENTRY as a successor.
6373 BasicBlock* lpTop; // loop TOP (the back edge from lpBottom reaches here). Lexically first block (in bbNext
6374 // order) reachable in this loop.
6375 BasicBlock* lpEntry; // the ENTRY in the loop (in most cases TOP or BOTTOM)
6376 BasicBlock* lpBottom; // loop BOTTOM (from here we have a back edge to the TOP)
6377 BasicBlock* lpExit; // if a single exit loop this is the EXIT (in most cases BOTTOM)
6379 callInterf lpAsgCall; // "callInterf" for calls in the loop
6380 ALLVARSET_TP lpAsgVars; // set of vars assigned within the loop (all vars, not just tracked)
6381 varRefKinds lpAsgInds : 8; // set of inds modified within the loop
6385 unsigned char lpExitCnt; // number of exits from the loop
6387 unsigned char lpParent; // The index of the most-nested loop that completely contains this one,
6388 // or else BasicBlock::NOT_IN_LOOP if no such loop exists.
6389 unsigned char lpChild; // The index of a nested loop, or else BasicBlock::NOT_IN_LOOP if no child exists.
6390 // (Actually, an "immediately" nested loop --
6391 // no other child of this loop is a parent of lpChild.)
6392 unsigned char lpSibling; // The index of another loop that is an immediate child of lpParent,
6393 // or else BasicBlock::NOT_IN_LOOP. One can enumerate all the children of a loop
6394 // by following "lpChild" then "lpSibling" links.
6396 bool lpLoopHasMemoryHavoc[MemoryKindCount]; // The loop contains an operation that we assume has arbitrary
6397 // memory side effects. If this is set, the fields below
6398 // may not be accurate (since they become irrelevant.)
6400 VARSET_TP lpVarInOut; // The set of variables that are IN or OUT during the execution of this loop
6401 VARSET_TP lpVarUseDef; // The set of variables that are USE or DEF during the execution of this loop
6403 // The following counts are used for hoisting profitability checks.
6405 int lpHoistedExprCount; // The register count for the non-FP expressions from inside this loop that have been
6407 int lpLoopVarCount; // The register count for the non-FP LclVars that are read/written inside this loop
6408 int lpVarInOutCount; // The register count for the non-FP LclVars that are alive inside or across this loop
6410 int lpHoistedFPExprCount; // The register count for the FP expressions from inside this loop that have been
6412 int lpLoopVarFPCount; // The register count for the FP LclVars that are read/written inside this loop
6413 int lpVarInOutFPCount; // The register count for the FP LclVars that are alive inside or across this loop
6415 typedef JitHashTable<CORINFO_FIELD_HANDLE, JitPtrKeyFuncs<struct CORINFO_FIELD_STRUCT_>, FieldKindForVN>
6417 FieldHandleSet* lpFieldsModified; // This has entries for all static field and object instance fields modified
6420 typedef JitHashTable<CORINFO_CLASS_HANDLE, JitPtrKeyFuncs<struct CORINFO_CLASS_STRUCT_>, bool> ClassHandleSet;
6421 ClassHandleSet* lpArrayElemTypesModified; // Bits set indicate the set of sz array element types such that
6422 // arrays of that type are modified
6425 // Adds the variable liveness information for 'blk' to 'this' LoopDsc
6426 void AddVariableLiveness(Compiler* comp, BasicBlock* blk);
6428 inline void AddModifiedField(Compiler* comp, CORINFO_FIELD_HANDLE fldHnd, FieldKindForVN fieldKind);
6429 // This doesn't *always* take a class handle -- it can also take primitive types, encoded as class handles
6430 // (shifted left, with a low-order bit set to distinguish.)
6431 // Use the {Encode/Decode}ElemType methods to construct/destruct these.
6432 inline void AddModifiedElemType(Compiler* comp, CORINFO_CLASS_HANDLE structHnd);
6434 /* The following values are set only for iterator loops, i.e. has the flag LPFLG_ITER set */
6436 GenTree* lpIterTree; // The "i = i <op> const" tree
6437 unsigned lpIterVar() const; // iterator variable #
6438 int lpIterConst() const; // the constant with which the iterator is incremented
6439 genTreeOps lpIterOper() const; // the type of the operation on the iterator (ADD, SUB, etc.)
6440 void VERIFY_lpIterTree() const;
6442 var_types lpIterOperType() const; // For overflow instructions
6444 // Set to the block where we found the initialization for LPFLG_CONST_INIT loops.
6445 // Initially, this will be 'head', but 'head' might change if we insert a loop pre-header block.
6446 BasicBlock* lpInitBlock;
6448 int lpConstInit; // initial constant value of iterator : Valid if LPFLG_CONST_INIT
6450 // The following is for LPFLG_ITER loops only (i.e. the loop condition is "i RELOP const or var")
6452 GenTree* lpTestTree; // pointer to the node containing the loop test
6453 genTreeOps lpTestOper() const; // the type of the comparison between the iterator and the limit (GT_LE, GT_GE,
6456 bool lpIsIncreasingLoop() const; // if the loop iterator increases from low to high value.
6457 bool lpIsDecreasingLoop() const; // if the loop iterator decreases from high to low value.
6459 void VERIFY_lpTestTree() const;
6461 bool lpIsReversed() const; // true if the iterator node is the second operand in the loop condition
6462 GenTree* lpIterator() const; // the iterator node in the loop test
6463 GenTree* lpLimit() const; // the limit node in the loop test
6465 // Limit constant value of iterator - loop condition is "i RELOP const"
6466 // : Valid if LPFLG_CONST_LIMIT
6467 int lpConstLimit() const;
6469 // The lclVar # in the loop condition ( "i RELOP lclVar" )
6470 // : Valid if LPFLG_VAR_LIMIT
6471 unsigned lpVarLimit() const;
6473 // The array length in the loop condition ( "i RELOP arr.len" or "i RELOP arr[i][j].len" )
6474 // : Valid if LPFLG_ARRLEN_LIMIT
6475 bool lpArrLenLimit(Compiler* comp, ArrIndex* index) const;
6477 // Returns "true" iff this is a "top entry" loop.
6478 bool lpIsTopEntry() const
6480 if (lpHead->bbNext == lpEntry)
6482 assert(lpHead->bbFallsThrough());
6483 assert(lpTop == lpEntry);
6492 // Returns "true" iff this is removed loop.
6493 bool lpIsRemoved() const
6495 return (lpFlags & LPFLG_REMOVED) != 0;
6498 // Returns "true" iff "*this" contains the blk.
6499 bool lpContains(BasicBlock* blk) const
6501 return lpTop->bbNum <= blk->bbNum && blk->bbNum <= lpBottom->bbNum;
6504 // Returns "true" iff "*this" (properly) contains the range [top, bottom] (allowing tops
6505 // to be equal, but requiring bottoms to be different.)
6506 bool lpContains(BasicBlock* top, BasicBlock* bottom) const
6508 return lpTop->bbNum <= top->bbNum && bottom->bbNum < lpBottom->bbNum;
6511 // Returns "true" iff "*this" (properly) contains "lp2" (allowing tops to be equal, but requiring
6512 // bottoms to be different.)
6513 bool lpContains(const LoopDsc& lp2) const
6515 return lpContains(lp2.lpTop, lp2.lpBottom);
6518 // Returns "true" iff "*this" is (properly) contained by the range [top, bottom]
6519 // (allowing tops to be equal, but requiring bottoms to be different.)
6520 bool lpContainedBy(BasicBlock* top, BasicBlock* bottom) const
6522 return top->bbNum <= lpTop->bbNum && lpBottom->bbNum < bottom->bbNum;
6525 // Returns "true" iff "*this" is (properly) contained by "lp2"
6526 // (allowing tops to be equal, but requiring bottoms to be different.)
6527 bool lpContainedBy(const LoopDsc& lp2) const
6529 return lpContainedBy(lp2.lpTop, lp2.lpBottom);
6532 // Returns "true" iff "*this" is disjoint from the range [top, bottom].
6533 bool lpDisjoint(BasicBlock* top, BasicBlock* bottom) const
6535 return bottom->bbNum < lpTop->bbNum || lpBottom->bbNum < top->bbNum;
6537 // Returns "true" iff "*this" is disjoint from "lp2".
6538 bool lpDisjoint(const LoopDsc& lp2) const
6540 return lpDisjoint(lp2.lpTop, lp2.lpBottom);
6543 // Returns "true" iff the loop is well-formed (see code for defn).
6544 bool lpWellFormed() const
6546 return lpTop->bbNum <= lpEntry->bbNum && lpEntry->bbNum <= lpBottom->bbNum &&
6547 (lpHead->bbNum < lpTop->bbNum || lpHead->bbNum > lpBottom->bbNum);
6551 void lpValidatePreHeader() const
6553 // If this is called, we expect there to be a pre-header.
6554 assert(lpFlags & LPFLG_HAS_PREHEAD);
6556 // The pre-header must unconditionally enter the loop.
6557 assert(lpHead->GetUniqueSucc() == lpEntry);
6559 // The loop block must be marked as a pre-header.
6560 assert(lpHead->bbFlags & BBF_LOOP_PREHEADER);
6562 // The loop entry must have a single non-loop predecessor, which is the pre-header.
6563 // We can't assume here that the bbNum are properly ordered, so we can't do a simple lpContained()
6564 // check. So, we defer this check, which will be done by `fgDebugCheckLoopTable()`.
6568 // LoopBlocks: convenience method for enabling range-based `for` iteration over all the
6569 // blocks in a loop, e.g.:
6570 // for (BasicBlock* const block : loop->LoopBlocks()) ...
6571 // Currently, the loop blocks are expected to be in linear, lexical, `bbNext` order
6572 // from `lpTop` through `lpBottom`, inclusive. All blocks in this range are considered
6573 // to be part of the loop.
6575 BasicBlockRangeList LoopBlocks() const
6577 return BasicBlockRangeList(lpTop, lpBottom);
6582 bool fgMightHaveLoop(); // returns true if there are any back edges
6583 bool fgHasLoops; // True if this method has any loops, set in fgComputeReachability
6586 LoopDsc* optLoopTable; // loop descriptor table
6587 bool optLoopTableValid; // info in loop table should be valid
6588 bool optLoopsRequirePreHeaders; // Do we require that all loops (in the loop table) have pre-headers?
6589 unsigned char optLoopCount; // number of tracked loops
6590 unsigned char loopAlignCandidates; // number of loops identified for alignment
6592 // Every time we rebuild the loop table, we increase the global "loop epoch". Any loop indices or
6593 // loop table pointers from the previous epoch are invalid.
6594 // TODO: validate this in some way?
6595 unsigned optCurLoopEpoch;
6600 JITDUMP("New loop epoch %d\n", optCurLoopEpoch);
6604 unsigned char loopsAligned; // number of loops actually aligned
6607 bool optRecordLoop(BasicBlock* head,
6612 unsigned char exitCnt);
6614 PhaseStatus optClearLoopIterInfo();
6617 void optPrintLoopInfo(unsigned lnum, bool printVerbose = false);
6618 void optPrintLoopInfo(const LoopDsc* loop, bool printVerbose = false);
6619 void optPrintLoopTable();
6623 unsigned optCallCount; // number of calls made in the method
6624 unsigned optIndirectCallCount; // number of virtual, interface and indirect calls made in the method
6625 unsigned optNativeCallCount; // number of Pinvoke/Native calls made in the method
6626 unsigned optLoopsCloned; // number of loops cloned in the current method.
6629 void optCheckPreds();
6632 void optResetLoopInfo();
6633 void optFindAndScaleGeneralLoopBlocks();
6635 // Determine if there are any potential loops, and set BBF_LOOP_HEAD on potential loop heads.
6636 void optMarkLoopHeads();
6638 void optScaleLoopBlocks(BasicBlock* begBlk, BasicBlock* endBlk);
6640 void optUnmarkLoopBlocks(BasicBlock* begBlk, BasicBlock* endBlk);
6642 void optUpdateLoopsBeforeRemoveBlock(BasicBlock* block, bool skipUnmarkLoop = false);
6644 bool optIsLoopTestEvalIntoTemp(Statement* testStmt, Statement** newTestStmt);
6645 unsigned optIsLoopIncrTree(GenTree* incr);
6646 bool optCheckIterInLoopTest(unsigned loopInd, GenTree* test, unsigned iterVar);
6647 bool optComputeIterInfo(GenTree* incr, BasicBlock* from, BasicBlock* to, unsigned* pIterVar);
6648 bool optPopulateInitInfo(unsigned loopInd, BasicBlock* initBlock, GenTree* init, unsigned iterVar);
6649 bool optExtractInitTestIncr(
6650 BasicBlock** pInitBlock, BasicBlock* bottom, BasicBlock* exit, GenTree** ppInit, GenTree** ppTest, GenTree** ppIncr);
6652 void optFindNaturalLoops();
6654 void optIdentifyLoopsForAlignment();
6656 // Ensures that all the loops in the loop nest rooted at "loopInd" (an index into the loop table) are 'canonical' --
6657 // each loop has a unique "top." Returns "true" iff the flowgraph has been modified.
6658 bool optCanonicalizeLoopNest(unsigned char loopInd);
6660 // Ensures that the loop "loopInd" (an index into the loop table) is 'canonical' -- it has a unique "top,"
6661 // unshared with any other loop. Returns "true" iff the flowgraph has been modified
6662 bool optCanonicalizeLoop(unsigned char loopInd);
6664 enum class LoopCanonicalizationOption
6670 bool optCanonicalizeLoopCore(unsigned char loopInd, LoopCanonicalizationOption option);
6672 // Requires "l1" to be a valid loop table index, and not "BasicBlock::NOT_IN_LOOP".
6673 // Requires "l2" to be a valid loop table index, or else "BasicBlock::NOT_IN_LOOP".
6674 // Returns true iff "l2" is not NOT_IN_LOOP, and "l1" contains "l2".
6675 // A loop contains itself.
6676 bool optLoopContains(unsigned l1, unsigned l2) const;
6678 // Returns the lpEntry for given preheader block of a loop
6679 BasicBlock* optLoopEntry(BasicBlock* preHeader);
6681 // Updates the loop table by changing loop "loopInd", whose head is required
6682 // to be "from", to be "to". Also performs this transformation for any
6683 // loop nested in "loopInd" that shares the same head as "loopInd".
6684 void optUpdateLoopHead(unsigned loopInd, BasicBlock* from, BasicBlock* to);
6686 enum class RedirectBlockOption
6688 DoNotChangePredLists, // do not modify pred lists
6689 UpdatePredLists, // add/remove to pred lists
6690 AddToPredLists, // only add to pred lists
6693 void optRedirectBlock(BasicBlock* blk,
6694 BlockToBlockMap* redirectMap,
6695 const RedirectBlockOption = RedirectBlockOption::DoNotChangePredLists);
6697 // Marks the containsCall information to "lnum" and any parent loops.
6698 void AddContainsCallAllContainingLoops(unsigned lnum);
6700 // Adds the variable liveness information from 'blk' to "lnum" and any parent loops.
6701 void AddVariableLivenessAllContainingLoops(unsigned lnum, BasicBlock* blk);
6703 // Adds "fldHnd" to the set of modified fields of "lnum" and any parent loops.
6704 void AddModifiedFieldAllContainingLoops(unsigned lnum, CORINFO_FIELD_HANDLE fldHnd, FieldKindForVN fieldKind);
6706 // Adds "elemType" to the set of modified array element types of "lnum" and any parent loops.
6707 void AddModifiedElemTypeAllContainingLoops(unsigned lnum, CORINFO_CLASS_HANDLE elemType);
6709 // Requires that "from" and "to" have the same "bbJumpKind" (perhaps because "to" is a clone
6710 // of "from".) Copies the jump destination from "from" to "to".
6711 void optCopyBlkDest(BasicBlock* from, BasicBlock* to);
6713 // Returns true if 'block' is an entry block for any loop in 'optLoopTable'
6714 bool optIsLoopEntry(BasicBlock* block) const;
6716 // The depth of the loop described by "lnum" (an index into the loop table.) (0 == top level)
6717 unsigned optLoopDepth(unsigned lnum) const
6719 assert(lnum < optLoopCount);
6721 while ((lnum = optLoopTable[lnum].lpParent) != BasicBlock::NOT_IN_LOOP)
6728 // Struct used in optInvertWhileLoop to count interesting constructs to boost the profitability score.
6729 struct OptInvertCountTreeInfoType
6731 int sharedStaticHelperCount;
6732 int arrayLengthCount;
6735 OptInvertCountTreeInfoType optInvertCountTreeInfo(GenTree* tree);
6737 bool optInvertWhileLoop(BasicBlock* block);
6738 bool optIfConvert(BasicBlock* block);
6741 static bool optIterSmallOverflow(int iterAtExit, var_types incrType);
6742 static bool optIterSmallUnderflow(int iterAtExit, var_types decrType);
6744 bool optComputeLoopRep(int constInit,
6747 genTreeOps iterOper,
6749 genTreeOps testOper,
6752 unsigned* iterCount);
6758 ALLVARSET_TP ivaMaskVal; // Set of variables assigned to. This is a set of all vars, not tracked vars.
6759 unsigned ivaVar; // Variable we are interested in, or -1
6760 varRefKinds ivaMaskInd; // What kind of indirect assignments are there?
6761 callInterf ivaMaskCall; // What kind of calls are there?
6762 bool ivaMaskIncomplete; // Variables not representable in ivaMaskVal were assigned to.
6765 bool optIsVarAssignedWithDesc(Statement* stmt, isVarAssgDsc* dsc);
6767 bool optIsVarAssigned(BasicBlock* beg, BasicBlock* end, GenTree* skip, unsigned var);
6769 bool optIsVarAssgLoop(unsigned lnum, unsigned var);
6771 bool optIsSetAssgLoop(unsigned lnum, ALLVARSET_VALARG_TP vars, varRefKinds inds = VR_NONE);
6773 bool optNarrowTree(GenTree* tree, var_types srct, var_types dstt, ValueNumPair vnpNarrow, bool doit);
6776 // The following is the upper limit on how many expressions we'll keep track
6777 // of for the CSE analysis.
6779 static const unsigned MAX_CSE_CNT = EXPSET_SZ;
6781 static const int MIN_CSE_COST = 2;
6783 // BitVec trait information only used by the optCSE_canSwap() method, for the CSE_defMask and CSE_useMask.
6784 // This BitVec uses one bit per CSE candidate
6785 BitVecTraits* cseMaskTraits; // one bit per CSE candidate
6787 // BitVec trait information for computing CSE availability using the CSE_DataFlow algorithm.
6788 // Two bits are allocated per CSE candidate to compute CSE availability
6789 // plus an extra bit to handle the initial unvisited case.
6790 // (See CSE_DataFlow::EndMerge for an explanation of why this is necessary.)
6792 // The two bits per CSE candidate have the following meanings:
6793 // 11 - The CSE is available, and is also available when considering calls as killing availability.
6794 // 10 - The CSE is available, but is not available when considering calls as killing availability.
6795 // 00 - The CSE is not available
6796 // 01 - An illegal combination
6798 BitVecTraits* cseLivenessTraits;
6800 //-----------------------------------------------------------------------------------------------------------------
6801 // getCSEnum2bit: Return the normalized index to use in the EXPSET_TP for the CSE with the given CSE index.
6802 // Each GenTree has a `gtCSEnum` field. Zero is reserved to mean this node is not a CSE, positive values indicate
6803 // CSE uses, and negative values indicate CSE defs. The caller must pass a non-zero positive value, as from
6806 static unsigned genCSEnum2bit(unsigned CSEnum)
6808 assert((CSEnum > 0) && (CSEnum <= MAX_CSE_CNT));
6812 //-----------------------------------------------------------------------------------------------------------------
6813 // getCSEAvailBit: Return the bit used by CSE dataflow sets (bbCseGen, etc.) for the availability bit for a CSE.
6815 static unsigned getCSEAvailBit(unsigned CSEnum)
6817 return genCSEnum2bit(CSEnum) * 2;
6820 //-----------------------------------------------------------------------------------------------------------------
6821 // getCSEAvailCrossCallBit: Return the bit used by CSE dataflow sets (bbCseGen, etc.) for the availability bit
6822 // for a CSE considering calls as killing availability bit (see description above).
6824 static unsigned getCSEAvailCrossCallBit(unsigned CSEnum)
6826 return getCSEAvailBit(CSEnum) + 1;
6829 void optPrintCSEDataFlowSet(EXPSET_VALARG_TP cseDataFlowSet, bool includeBits = true);
6831 EXPSET_TP cseCallKillsMask; // Computed once - A mask that is used to kill available CSEs at callsites
6833 /* Generic list of nodes - used by the CSE logic */
6843 treeStmtLst* tslNext;
6844 GenTree* tslTree; // tree node
6845 Statement* tslStmt; // statement containing the tree
6846 BasicBlock* tslBlock; // block containing the statement
6849 // The following logic keeps track of expressions via a simple hash table.
6853 CSEdsc* csdNextInBucket; // used by the hash table
6854 size_t csdHashKey; // the original hashkey
6855 ssize_t csdConstDefValue; // When we CSE similar constants, this is the value that we use as the def
6856 ValueNum csdConstDefVN; // When we CSE similar constants, this is the ValueNumber that we use for the LclVar
6858 unsigned csdIndex; // 1..optCSECandidateCount
6859 bool csdIsSharedConst; // true if this CSE is a shared const
6860 bool csdLiveAcrossCall;
6862 unsigned short csdDefCount; // definition count
6863 unsigned short csdUseCount; // use count (excluding the implicit uses at defs)
6865 weight_t csdDefWtCnt; // weighted def count
6866 weight_t csdUseWtCnt; // weighted use count (excluding the implicit uses at defs)
6868 GenTree* csdTree; // treenode containing the 1st occurrence
6869 Statement* csdStmt; // stmt containing the 1st occurrence
6870 BasicBlock* csdBlock; // block containing the 1st occurrence
6872 treeStmtLst* csdTreeList; // list of matching tree nodes: head
6873 treeStmtLst* csdTreeLast; // list of matching tree nodes: tail
6875 ValueNum defExcSetPromise; // The exception set that is now required for all defs of this CSE.
6876 // This will be set to NoVN if we decide to abandon this CSE
6878 ValueNum defExcSetCurrent; // The set of exceptions we currently can use for CSE uses.
6880 ValueNum defConservNormVN; // if all def occurrences share the same conservative normal value
6881 // number, this will reflect it; otherwise, NoVN.
6882 // not used for shared const CSE's
6885 static const size_t s_optCSEhashSizeInitial;
6886 static const size_t s_optCSEhashGrowthFactor;
6887 static const size_t s_optCSEhashBucketSize;
6888 size_t optCSEhashSize; // The current size of hashtable
6889 size_t optCSEhashCount; // Number of entries in hashtable
6890 size_t optCSEhashMaxCountBeforeResize; // Number of entries before resize
6891 CSEdsc** optCSEhash;
6894 typedef JitHashTable<GenTree*, JitPtrKeyFuncs<GenTree>, GenTree*> NodeToNodeMap;
6896 NodeToNodeMap* optCseCheckedBoundMap; // Maps bound nodes to ancestor compares that should be
6897 // re-numbered with the bound to improve range check elimination
6899 // Given a compare, look for a cse candidate checked bound feeding it and add a map entry if found.
6900 void optCseUpdateCheckedBoundMap(GenTree* compare);
6904 CSEdsc* optCSEfindDsc(unsigned index);
6905 bool optUnmarkCSE(GenTree* tree);
6907 // user defined callback data for the tree walk function optCSE_MaskHelper()
6908 struct optCSE_MaskData
6910 EXPSET_TP CSE_defMask;
6911 EXPSET_TP CSE_useMask;
6914 // Treewalk helper for optCSE_DefMask and optCSE_UseMask
6915 static fgWalkPreFn optCSE_MaskHelper;
6917 // This function walks all the node for an given tree
6918 // and return the mask of CSE definitions and uses for the tree
6920 void optCSE_GetMaskData(GenTree* tree, optCSE_MaskData* pMaskData);
6922 // Given a binary tree node return true if it is safe to swap the order of evaluation for op1 and op2.
6923 bool optCSE_canSwap(GenTree* firstNode, GenTree* secondNode);
6925 struct optCSEcostCmpEx
6927 bool operator()(const CSEdsc* op1, const CSEdsc* op2);
6929 struct optCSEcostCmpSz
6931 bool operator()(const CSEdsc* op1, const CSEdsc* op2);
6934 void optCleanupCSEs();
6937 void optEnsureClearCSEInfo();
6940 static bool Is_Shared_Const_CSE(size_t key)
6942 return ((key & TARGET_SIGN_BIT) != 0);
6945 // returns the encoded key
6946 static size_t Encode_Shared_Const_CSE_Value(size_t key)
6948 return TARGET_SIGN_BIT | (key >> CSE_CONST_SHARED_LOW_BITS);
6951 // returns the original key
6952 static size_t Decode_Shared_Const_CSE_Value(size_t enckey)
6954 assert(Is_Shared_Const_CSE(enckey));
6955 return (enckey & ~TARGET_SIGN_BIT) << CSE_CONST_SHARED_LOW_BITS;
6958 /**************************************************************************
6959 * Value Number based CSEs
6960 *************************************************************************/
6962 // String to use for formatting CSE numbers. Note that this is the positive number, e.g., from GET_CSE_INDEX().
6963 #define FMT_CSE "CSE #%02u"
6966 PhaseStatus optOptimizeValnumCSEs();
6969 void optValnumCSE_Init();
6970 unsigned optValnumCSE_Index(GenTree* tree, Statement* stmt);
6971 bool optValnumCSE_Locate();
6972 void optValnumCSE_InitDataFlow();
6973 void optValnumCSE_DataFlow();
6974 void optValnumCSE_Availability();
6975 bool optValnumCSE_Heuristic();
6977 bool optDoCSE; // True when we have found a duplicate CSE tree
6978 bool optValnumCSE_phase; // True when we are executing the optOptimizeValnumCSEs() phase
6979 unsigned optCSECandidateCount; // Count of CSE's candidates
6980 unsigned optCSEstart; // The first local variable number that is a CSE
6981 unsigned optCSEcount; // The total count of CSE's introduced.
6982 weight_t optCSEweight; // The weight of the current block when we are doing PerformCSE
6984 bool optIsCSEcandidate(GenTree* tree);
6986 // lclNumIsTrueCSE returns true if the LclVar was introduced by the CSE phase of the compiler
6988 bool lclNumIsTrueCSE(unsigned lclNum) const
6990 return ((optCSEcount > 0) && (lclNum >= optCSEstart) && (lclNum < optCSEstart + optCSEcount));
6993 // lclNumIsCSE returns true if the LclVar should be treated like a CSE with regards to constant prop.
6995 bool lclNumIsCSE(unsigned lclNum) const
6997 return lvaGetDesc(lclNum)->lvIsCSE;
7001 bool optConfigDisableCSE();
7002 bool optConfigDisableCSE2();
7005 void optOptimizeCSEs();
7007 static callInterf optCallInterf(GenTreeCall* call);
7010 // VN based copy propagation.
7012 // In DEBUG builds, we'd like to know the tree that the SSA definition was pushed for.
7013 // While for ordinary SSA defs it will be available (as a store) in the SSA descriptor,
7014 // for locals which will use "definitions from uses", it will not be, so we store it
7015 // in this class instead.
7016 class CopyPropSsaDef
7018 LclSsaVarDsc* m_ssaDef;
7023 CopyPropSsaDef(LclSsaVarDsc* ssaDef, GenTree* defNode)
7026 , m_defNode(defNode)
7031 LclSsaVarDsc* GetSsaDef() const
7037 GenTree* GetDefNode() const
7044 typedef ArrayStack<CopyPropSsaDef> CopyPropSsaDefStack;
7045 typedef JitHashTable<unsigned, JitSmallPrimitiveKeyFuncs<unsigned>, CopyPropSsaDefStack*> LclNumToLiveDefsMap;
7047 // Copy propagation functions.
7048 bool optCopyProp(BasicBlock* block,
7050 GenTreeLclVarCommon* tree,
7052 LclNumToLiveDefsMap* curSsaName);
7053 void optBlockCopyPropPopStacks(BasicBlock* block, LclNumToLiveDefsMap* curSsaName);
7054 bool optBlockCopyProp(BasicBlock* block, LclNumToLiveDefsMap* curSsaName);
7055 void optCopyPropPushDef(GenTree* defNode, GenTreeLclVarCommon* lclNode, LclNumToLiveDefsMap* curSsaName);
7056 int optCopyProp_LclVarScore(const LclVarDsc* lclVarDsc, const LclVarDsc* copyVarDsc, bool preferOp2);
7057 PhaseStatus optVnCopyProp();
7058 INDEBUG(void optDumpCopyPropStack(LclNumToLiveDefsMap* curSsaName));
7060 /**************************************************************************
7061 * Early value propagation
7062 *************************************************************************/
7068 SSAName(unsigned lvNum, unsigned ssaNum) : m_lvNum(lvNum), m_ssaNum(ssaNum)
7072 static unsigned GetHashCode(SSAName ssaNm)
7074 return (ssaNm.m_lvNum << 16) | (ssaNm.m_ssaNum);
7077 static bool Equals(SSAName ssaNm1, SSAName ssaNm2)
7079 return (ssaNm1.m_lvNum == ssaNm2.m_lvNum) && (ssaNm1.m_ssaNum == ssaNm2.m_ssaNum);
7083 PhaseStatus optVNBasedDeadStoreRemoval();
7087 #define OMF_HAS_NEWARRAY 0x00000001 // Method contains 'new' of an SD array
7088 #define OMF_HAS_NEWOBJ 0x00000002 // Method contains 'new' of an object type.
7089 #define OMF_HAS_ARRAYREF 0x00000004 // Method contains array element loads or stores.
7090 #define OMF_HAS_NULLCHECK 0x00000008 // Method contains null check.
7091 #define OMF_HAS_FATPOINTER 0x00000010 // Method contains call, that needs fat pointer transformation.
7092 #define OMF_HAS_OBJSTACKALLOC 0x00000020 // Method contains an object allocated on the stack.
7093 #define OMF_HAS_GUARDEDDEVIRT 0x00000040 // Method contains guarded devirtualization candidate
7094 #define OMF_HAS_EXPRUNTIMELOOKUP 0x00000080 // Method contains a runtime lookup to an expandable dictionary.
7095 #define OMF_HAS_PATCHPOINT 0x00000100 // Method contains patchpoints
7096 #define OMF_NEEDS_GCPOLLS 0x00000200 // Method needs GC polls
7097 #define OMF_HAS_FROZEN_OBJECTS 0x00000400 // Method has frozen objects (REF constant int)
7098 #define OMF_HAS_PARTIAL_COMPILATION_PATCHPOINT 0x00000800 // Method contains partial compilation patchpoints
7099 #define OMF_HAS_TAILCALL_SUCCESSOR 0x00001000 // Method has potential tail call in a non BBJ_RETURN block
7100 #define OMF_HAS_MDNEWARRAY 0x00002000 // Method contains 'new' of an MD array
7101 #define OMF_HAS_MDARRAYREF 0x00004000 // Method contains multi-dimensional intrinsic array element loads or stores.
7102 #define OMF_HAS_STATIC_INIT 0x00008000 // Method has static initializations we might want to partially inline
7103 #define OMF_HAS_TLS_FIELD 0x00010000 // Method contains TLS field access
7104 #define OMF_HAS_SPECIAL_INTRINSICS 0x00020000 // Method contains special intrinsics expanded in late phases
7108 bool doesMethodHaveFatPointer()
7110 return (optMethodFlags & OMF_HAS_FATPOINTER) != 0;
7113 void setMethodHasFatPointer()
7115 optMethodFlags |= OMF_HAS_FATPOINTER;
7118 void clearMethodHasFatPointer()
7120 optMethodFlags &= ~OMF_HAS_FATPOINTER;
7123 void addFatPointerCandidate(GenTreeCall* call);
7125 bool doesMethodHaveFrozenObjects() const
7127 return (optMethodFlags & OMF_HAS_FROZEN_OBJECTS) != 0;
7130 void setMethodHasFrozenObjects()
7132 optMethodFlags |= OMF_HAS_FROZEN_OBJECTS;
7135 bool doesMethodHaveStaticInit()
7137 return (optMethodFlags & OMF_HAS_STATIC_INIT) != 0;
7140 void setMethodHasStaticInit()
7142 optMethodFlags |= OMF_HAS_STATIC_INIT;
7145 bool doesMethodHaveGuardedDevirtualization() const
7147 return (optMethodFlags & OMF_HAS_GUARDEDDEVIRT) != 0;
7150 void setMethodHasGuardedDevirtualization()
7152 optMethodFlags |= OMF_HAS_GUARDEDDEVIRT;
7155 bool methodHasTlsFieldAccess()
7157 return (optMethodFlags & OMF_HAS_TLS_FIELD) != 0;
7160 void setMethodHasTlsFieldAccess()
7162 optMethodFlags |= OMF_HAS_TLS_FIELD;
7165 bool doesMethodHaveSpecialIntrinsics()
7167 return (optMethodFlags & OMF_HAS_SPECIAL_INTRINSICS) != 0;
7170 void setMethodHasSpecialIntrinsics()
7172 optMethodFlags |= OMF_HAS_SPECIAL_INTRINSICS;
7175 void pickGDV(GenTreeCall* call,
7178 CORINFO_CLASS_HANDLE* classGuesses,
7179 CORINFO_METHOD_HANDLE* methodGuesses,
7180 int* candidatesCount,
7181 unsigned* likelihoods);
7183 void considerGuardedDevirtualization(GenTreeCall* call,
7186 CORINFO_METHOD_HANDLE baseMethod,
7187 CORINFO_CLASS_HANDLE baseClass,
7188 CORINFO_CONTEXT_HANDLE* pContextHandle);
7190 bool isCompatibleMethodGDV(GenTreeCall* call, CORINFO_METHOD_HANDLE gdvTarget);
7192 void addGuardedDevirtualizationCandidate(GenTreeCall* call,
7193 CORINFO_METHOD_HANDLE methodHandle,
7194 CORINFO_CLASS_HANDLE classHandle,
7195 CORINFO_CONTEXT_HANDLE contextHandle,
7196 unsigned methodAttr,
7198 unsigned likelihood);
7200 int getGDVMaxTypeChecks()
7202 int typeChecks = JitConfig.JitGuardedDevirtualizationMaxTypeChecks();
7205 // Negative value means "it's up to JIT to decide"
7206 if (IsTargetAbi(CORINFO_NATIVEAOT_ABI) && !opts.jitFlags->IsSet(JitFlags::JIT_FLAG_SIZE_OPT))
7211 // We plan to use 3 for CoreCLR too, but we need to make sure it doesn't regress performance
7212 // as CoreCLR heavily relies on Dynamic PGO while for NativeAOT we *usually* don't have it and
7213 // can only perform the "exact" devirtualization.
7217 // MAX_GDV_TYPE_CHECKS is the upper limit. The constant can be changed, we just suspect that even
7218 // 4 type checks is already too much.
7219 return min(MAX_GDV_TYPE_CHECKS, typeChecks);
7222 bool doesMethodHaveExpRuntimeLookup()
7224 return (optMethodFlags & OMF_HAS_EXPRUNTIMELOOKUP) != 0;
7227 void setMethodHasExpRuntimeLookup()
7229 optMethodFlags |= OMF_HAS_EXPRUNTIMELOOKUP;
7232 bool doesMethodHavePatchpoints()
7234 return (optMethodFlags & OMF_HAS_PATCHPOINT) != 0;
7237 void setMethodHasPatchpoint()
7239 optMethodFlags |= OMF_HAS_PATCHPOINT;
7242 bool doesMethodHavePartialCompilationPatchpoints()
7244 return (optMethodFlags & OMF_HAS_PARTIAL_COMPILATION_PATCHPOINT) != 0;
7247 void setMethodHasPartialCompilationPatchpoint()
7249 optMethodFlags |= OMF_HAS_PARTIAL_COMPILATION_PATCHPOINT;
7252 unsigned optMethodFlags;
7254 bool doesMethodHaveNoReturnCalls()
7256 return optNoReturnCallCount > 0;
7259 void setMethodHasNoReturnCalls()
7261 optNoReturnCallCount++;
7264 unsigned optNoReturnCallCount;
7266 // Recursion bound controls how far we can go backwards tracking for a SSA value.
7267 // No throughput diff was found with backward walk bound between 3-8.
7268 static const int optEarlyPropRecurBound = 5;
7270 enum class optPropKind
7277 typedef JitHashTable<unsigned, JitSmallPrimitiveKeyFuncs<unsigned>, GenTree*> LocalNumberToNullCheckTreeMap;
7279 GenTree* getArrayLengthFromAllocation(GenTree* tree DEBUGARG(BasicBlock* block));
7280 GenTree* optPropGetValueRec(unsigned lclNum, unsigned ssaNum, optPropKind valueKind, int walkDepth);
7281 GenTree* optPropGetValue(unsigned lclNum, unsigned ssaNum, optPropKind valueKind);
7282 GenTree* optEarlyPropRewriteTree(GenTree* tree, LocalNumberToNullCheckTreeMap* nullCheckMap);
7283 bool optDoEarlyPropForBlock(BasicBlock* block);
7284 bool optDoEarlyPropForFunc();
7285 PhaseStatus optEarlyProp();
7286 bool optFoldNullCheck(GenTree* tree, LocalNumberToNullCheckTreeMap* nullCheckMap);
7287 GenTree* optFindNullCheckToFold(GenTree* tree, LocalNumberToNullCheckTreeMap* nullCheckMap);
7288 bool optIsNullCheckFoldingLegal(GenTree* tree,
7289 GenTree* nullCheckTree,
7290 GenTree** nullCheckParent,
7291 Statement** nullCheckStmt);
7292 bool optCanMoveNullCheckPastTree(GenTree* tree,
7293 unsigned nullCheckLclNum,
7295 bool checkSideEffectSummary);
7297 void optCheckFlagsAreSet(unsigned methodFlag,
7298 const char* methodFlagStr,
7300 const char* bbFlagStr,
7302 BasicBlock* basicBlock);
7305 // Redundant branch opts
7307 PhaseStatus optRedundantBranches();
7308 bool optRedundantRelop(BasicBlock* const block);
7309 bool optRedundantBranch(BasicBlock* const block);
7310 bool optJumpThreadDom(BasicBlock* const block, BasicBlock* const domBlock, bool domIsSameRelop);
7311 bool optJumpThreadPhi(BasicBlock* const block, GenTree* tree, ValueNum treeNormVN);
7312 bool optJumpThreadCheck(BasicBlock* const block, BasicBlock* const domBlock);
7313 bool optJumpThreadCore(JumpThreadInfo& jti);
7314 bool optReachable(BasicBlock* const fromBlock, BasicBlock* const toBlock, BasicBlock* const excludedBlock);
7315 BitVecTraits* optReachableBitVecTraits;
7316 BitVec optReachableBitVec;
7317 void optRelopImpliesRelop(RelopImplicationInfo* rii);
7319 /**************************************************************************
7320 * Value/Assertion propagation
7321 *************************************************************************/
7323 // Data structures for assertion prop
7324 BitVecTraits* apTraits;
7327 enum optAssertionKind
7344 O1K_CONSTANT_LOOP_BND,
7345 O1K_CONSTANT_LOOP_BND_UN,
7352 enum optOp2Kind : uint16_t
7367 optAssertionKind assertionKind;
7370 unsigned lclNum; // assigned to or property of this local var number
7378 struct AssertionDscOp1
7380 optOp1Kind kind; // a normal LclVar, or Exact-type or Subtype
7387 struct AssertionDscOp2
7389 optOp2Kind kind; // a const or copy assignment
7391 uint16_t m_encodedIconFlags; // encoded icon gtFlags, don't use directly
7396 ssize_t iconVal; // integer
7397 #if !defined(HOST_64BIT)
7398 unsigned padding; // unused; ensures iconFlags does not overlap lconVal
7412 assert(m_encodedIconFlags <= 0xFF);
7413 return m_encodedIconFlags != 0;
7415 GenTreeFlags GetIconFlag()
7417 // number of trailing zeros in GTF_ICON_HDL_MASK
7418 const uint16_t iconMaskTzc = 24;
7419 static_assert_no_msg((0xFF000000 == GTF_ICON_HDL_MASK) && (GTF_ICON_HDL_MASK >> iconMaskTzc) == 0xFF);
7421 GenTreeFlags flags = (GenTreeFlags)(m_encodedIconFlags << iconMaskTzc);
7422 assert((flags & ~GTF_ICON_HDL_MASK) == 0);
7425 void SetIconFlag(GenTreeFlags flags, FieldSeq* fieldSeq = nullptr)
7427 const uint16_t iconMaskTzc = 24;
7428 assert((flags & ~GTF_ICON_HDL_MASK) == 0);
7429 m_encodedIconFlags = flags >> iconMaskTzc;
7430 u1.fieldSeq = fieldSeq;
7434 bool IsCheckedBoundArithBound()
7436 return ((assertionKind == OAK_EQUAL || assertionKind == OAK_NOT_EQUAL) && op1.kind == O1K_BOUND_OPER_BND);
7438 bool IsCheckedBoundBound()
7440 return ((assertionKind == OAK_EQUAL || assertionKind == OAK_NOT_EQUAL) && op1.kind == O1K_BOUND_LOOP_BND);
7442 bool IsConstantBound()
7444 return ((assertionKind == OAK_EQUAL || assertionKind == OAK_NOT_EQUAL) &&
7445 (op1.kind == O1K_CONSTANT_LOOP_BND));
7447 bool IsConstantBoundUnsigned()
7449 return ((assertionKind == OAK_EQUAL || assertionKind == OAK_NOT_EQUAL) &&
7450 (op1.kind == O1K_CONSTANT_LOOP_BND_UN));
7452 bool IsBoundsCheckNoThrow()
7454 return ((assertionKind == OAK_NO_THROW) && (op1.kind == O1K_ARR_BND));
7457 bool IsCopyAssertion()
7459 return ((assertionKind == OAK_EQUAL) && (op1.kind == O1K_LCLVAR) && (op2.kind == O2K_LCLVAR_COPY));
7462 bool IsConstantInt32Assertion()
7464 return ((assertionKind == OAK_EQUAL) || (assertionKind == OAK_NOT_EQUAL)) && (op2.kind == O2K_CONST_INT);
7467 bool CanPropLclVar()
7469 return assertionKind == OAK_EQUAL && op1.kind == O1K_LCLVAR;
7472 bool CanPropEqualOrNotEqual()
7474 return assertionKind == OAK_EQUAL || assertionKind == OAK_NOT_EQUAL;
7477 bool CanPropNonNull()
7479 return assertionKind == OAK_NOT_EQUAL && op2.vn == ValueNumStore::VNForNull();
7482 bool CanPropBndsCheck()
7484 return op1.kind == O1K_ARR_BND;
7487 bool CanPropSubRange()
7489 return assertionKind == OAK_SUBRANGE && op1.kind == O1K_LCLVAR;
7492 static bool SameKind(AssertionDsc* a1, AssertionDsc* a2)
7494 return a1->assertionKind == a2->assertionKind && a1->op1.kind == a2->op1.kind &&
7495 a1->op2.kind == a2->op2.kind;
7498 static bool ComplementaryKind(optAssertionKind kind, optAssertionKind kind2)
7500 if (kind == OAK_EQUAL)
7502 return kind2 == OAK_NOT_EQUAL;
7504 else if (kind == OAK_NOT_EQUAL)
7506 return kind2 == OAK_EQUAL;
7511 bool HasSameOp1(AssertionDsc* that, bool vnBased)
7513 if (op1.kind != that->op1.kind)
7517 else if (op1.kind == O1K_ARR_BND)
7520 return (op1.bnd.vnIdx == that->op1.bnd.vnIdx) && (op1.bnd.vnLen == that->op1.bnd.vnLen);
7524 return ((vnBased && (op1.vn == that->op1.vn)) ||
7525 (!vnBased && (op1.lcl.lclNum == that->op1.lcl.lclNum)));
7529 bool HasSameOp2(AssertionDsc* that, bool vnBased)
7531 if (op2.kind != that->op2.kind)
7538 case O2K_IND_CNS_INT:
7540 return ((op2.u1.iconVal == that->op2.u1.iconVal) && (op2.GetIconFlag() == that->op2.GetIconFlag()));
7542 case O2K_CONST_LONG:
7543 return (op2.lconVal == that->op2.lconVal);
7545 case O2K_CONST_DOUBLE:
7546 // exact match because of positive and negative zero.
7547 return (memcmp(&op2.dconVal, &that->op2.dconVal, sizeof(double)) == 0);
7552 case O2K_LCLVAR_COPY:
7553 return (op2.lcl.lclNum == that->op2.lcl.lclNum) &&
7554 (!vnBased || (op2.lcl.ssaNum == that->op2.lcl.ssaNum));
7557 return op2.u2.Equals(that->op2.u2);
7560 // we will return false
7564 assert(!"Unexpected value for op2.kind in AssertionDsc.");
7571 bool Complementary(AssertionDsc* that, bool vnBased)
7573 return ComplementaryKind(assertionKind, that->assertionKind) && HasSameOp1(that, vnBased) &&
7574 HasSameOp2(that, vnBased);
7577 bool Equals(AssertionDsc* that, bool vnBased)
7579 if (assertionKind != that->assertionKind)
7583 else if (assertionKind == OAK_NO_THROW)
7585 assert(op2.kind == O2K_INVALID);
7586 return HasSameOp1(that, vnBased);
7590 return HasSameOp1(that, vnBased) && HasSameOp2(that, vnBased);
7596 static fgWalkPreFn optVNAssertionPropCurStmtVisitor;
7598 bool optLocalAssertionProp; // indicates that we are performing local assertion prop
7599 bool optAssertionPropagated; // set to true if we modified the trees
7600 bool optAssertionPropagatedCurrentStmt;
7602 GenTree* optAssertionPropCurrentTree;
7604 AssertionIndex* optComplementaryAssertionMap;
7605 JitExpandArray<ASSERT_TP>* optAssertionDep; // table that holds dependent assertions (assertions
7606 // using the value of a local var) for each local var
7607 AssertionDsc* optAssertionTabPrivate; // table that holds info about value assignments
7608 AssertionIndex optAssertionCount; // total number of assertions in the assertion table
7609 AssertionIndex optMaxAssertionCount;
7610 bool optCanPropLclVar;
7611 bool optCanPropEqual;
7612 bool optCanPropNonNull;
7613 bool optCanPropBndsChk;
7614 bool optCanPropSubRange;
7617 void optVnNonNullPropCurStmt(BasicBlock* block, Statement* stmt, GenTree* tree);
7618 fgWalkResult optVNConstantPropCurStmt(BasicBlock* block, Statement* stmt, GenTree* tree);
7619 GenTree* optVNConstantPropOnJTrue(BasicBlock* block, GenTree* test);
7620 GenTree* optVNConstantPropOnTree(BasicBlock* block, GenTree* tree);
7621 GenTree* optExtractSideEffListFromConst(GenTree* tree);
7623 AssertionIndex GetAssertionCount()
7625 return optAssertionCount;
7627 ASSERT_TP* bbJtrueAssertionOut;
7628 typedef JitHashTable<ValueNum, JitSmallPrimitiveKeyFuncs<ValueNum>, ASSERT_TP> ValueNumToAssertsMap;
7629 ValueNumToAssertsMap* optValueNumToAsserts;
7631 // Assertion prop helpers.
7632 ASSERT_TP& GetAssertionDep(unsigned lclNum);
7633 AssertionDsc* optGetAssertion(AssertionIndex assertIndex);
7634 void optAssertionInit(bool isLocalProp);
7635 void optAssertionTraitsInit(AssertionIndex assertionCount);
7636 void optAssertionReset(AssertionIndex limit);
7637 void optAssertionRemove(AssertionIndex index);
7639 // Assertion prop data flow functions.
7640 PhaseStatus optAssertionPropMain();
7641 Statement* optVNAssertionPropCurStmt(BasicBlock* block, Statement* stmt);
7642 bool optIsTreeKnownIntValue(bool vnBased, GenTree* tree, ssize_t* pConstant, GenTreeFlags* pIconFlags);
7643 ASSERT_TP* optInitAssertionDataflowFlags();
7644 ASSERT_TP* optComputeAssertionGen();
7646 // Assertion Gen functions.
7647 void optAssertionGen(GenTree* tree);
7648 AssertionIndex optAssertionGenCast(GenTreeCast* cast);
7649 AssertionIndex optAssertionGenPhiDefn(GenTree* tree);
7650 AssertionInfo optCreateJTrueBoundsAssertion(GenTree* tree);
7651 AssertionInfo optAssertionGenJtrue(GenTree* tree);
7652 AssertionIndex optCreateJtrueAssertions(GenTree* op1,
7654 Compiler::optAssertionKind assertionKind,
7655 bool helperCallArgs = false);
7656 AssertionIndex optFindComplementary(AssertionIndex assertionIndex);
7657 void optMapComplementary(AssertionIndex assertionIndex, AssertionIndex index);
7659 ValueNum optConservativeNormalVN(GenTree* tree);
7661 ssize_t optCastConstantSmall(ssize_t iconVal, var_types smallType);
7663 // Assertion creation functions.
7664 AssertionIndex optCreateAssertion(GenTree* op1,
7666 optAssertionKind assertionKind,
7667 bool helperCallArgs = false);
7669 AssertionIndex optFinalizeCreatingAssertion(AssertionDsc* assertion);
7671 bool optTryExtractSubrangeAssertion(GenTree* source, IntegralRange* pRange);
7673 void optCreateComplementaryAssertion(AssertionIndex assertionIndex,
7676 bool helperCallArgs = false);
7678 bool optAssertionVnInvolvesNan(AssertionDsc* assertion);
7679 AssertionIndex optAddAssertion(AssertionDsc* assertion);
7680 void optAddVnAssertionMapping(ValueNum vn, AssertionIndex index);
7682 void optPrintVnAssertionMapping();
7684 ASSERT_TP optGetVnMappedAssertions(ValueNum vn);
7686 // Used for respective assertion propagations.
7687 AssertionIndex optAssertionIsSubrange(GenTree* tree, IntegralRange range, ASSERT_VALARG_TP assertions);
7688 AssertionIndex optAssertionIsSubtype(GenTree* tree, GenTree* methodTableArg, ASSERT_VALARG_TP assertions);
7689 AssertionIndex optAssertionIsNonNullInternal(GenTree* op, ASSERT_VALARG_TP assertions DEBUGARG(bool* pVnBased));
7690 bool optAssertionIsNonNull(GenTree* op,
7691 ASSERT_VALARG_TP assertions DEBUGARG(bool* pVnBased) DEBUGARG(AssertionIndex* pIndex));
7693 AssertionIndex optGlobalAssertionIsEqualOrNotEqual(ASSERT_VALARG_TP assertions, GenTree* op1, GenTree* op2);
7694 AssertionIndex optGlobalAssertionIsEqualOrNotEqualZero(ASSERT_VALARG_TP assertions, GenTree* op1);
7695 AssertionIndex optLocalAssertionIsEqualOrNotEqual(
7696 optOp1Kind op1Kind, unsigned lclNum, optOp2Kind op2Kind, ssize_t cnsVal, ASSERT_VALARG_TP assertions);
7698 // Assertion prop for lcl var functions.
7699 bool optAssertionProp_LclVarTypeCheck(GenTree* tree, LclVarDsc* lclVarDsc, LclVarDsc* copyVarDsc);
7700 GenTree* optCopyAssertionProp(AssertionDsc* curAssertion,
7701 GenTreeLclVarCommon* tree,
7702 Statement* stmt DEBUGARG(AssertionIndex index));
7703 GenTree* optConstantAssertionProp(AssertionDsc* curAssertion,
7704 GenTreeLclVarCommon* tree,
7705 Statement* stmt DEBUGARG(AssertionIndex index));
7706 bool optIsProfitableToSubstitute(GenTree* dest, BasicBlock* destBlock, GenTree* value);
7707 bool optZeroObjAssertionProp(GenTree* tree, ASSERT_VALARG_TP assertions);
7709 // Assertion propagation functions.
7710 GenTree* optAssertionProp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt, BasicBlock* block);
7711 GenTree* optAssertionProp_LclVar(ASSERT_VALARG_TP assertions, GenTreeLclVarCommon* tree, Statement* stmt);
7712 GenTree* optAssertionProp_LclFld(ASSERT_VALARG_TP assertions, GenTreeLclVarCommon* tree, Statement* stmt);
7713 GenTree* optAssertionProp_LocalStore(ASSERT_VALARG_TP assertions, GenTreeLclVarCommon* store, Statement* stmt);
7714 GenTree* optAssertionProp_BlockStore(ASSERT_VALARG_TP assertions, GenTreeBlk* store, Statement* stmt);
7715 GenTree* optAssertionProp_Return(ASSERT_VALARG_TP assertions, GenTreeUnOp* ret, Statement* stmt);
7716 GenTree* optAssertionProp_Ind(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
7717 GenTree* optAssertionProp_Cast(ASSERT_VALARG_TP assertions, GenTreeCast* cast, Statement* stmt);
7718 GenTree* optAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCall* call, Statement* stmt);
7719 GenTree* optAssertionProp_RelOp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
7720 GenTree* optAssertionProp_Comma(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
7721 GenTree* optAssertionProp_BndsChk(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
7722 GenTree* optAssertionPropGlobal_RelOp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
7723 GenTree* optAssertionPropLocal_RelOp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt);
7724 GenTree* optAssertionProp_Update(GenTree* newTree, GenTree* tree, Statement* stmt);
7725 GenTree* optNonNullAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCall* call);
7726 bool optNonNullAssertionProp_Ind(ASSERT_VALARG_TP assertions, GenTree* indir);
7728 // Implied assertion functions.
7729 void optImpliedAssertions(AssertionIndex assertionIndex, ASSERT_TP& activeAssertions);
7730 void optImpliedByTypeOfAssertions(ASSERT_TP& activeAssertions);
7731 void optImpliedByCopyAssertion(AssertionDsc* copyAssertion, AssertionDsc* depAssertion, ASSERT_TP& result);
7732 void optImpliedByConstAssertion(AssertionDsc* curAssertion, ASSERT_TP& result);
7735 void optPrintAssertion(AssertionDsc* newAssertion, AssertionIndex assertionIndex = 0);
7736 void optPrintAssertionIndex(AssertionIndex index);
7737 void optPrintAssertionIndices(ASSERT_TP assertions);
7738 void optDebugCheckAssertion(AssertionDsc* assertion);
7739 void optDebugCheckAssertions(AssertionIndex AssertionIndex);
7742 static void optDumpAssertionIndices(const char* header, ASSERT_TP assertions, const char* footer = nullptr);
7743 static void optDumpAssertionIndices(ASSERT_TP assertions, const char* footer = nullptr);
7745 /**************************************************************************
7747 *************************************************************************/
7750 struct LoopCloneVisitorInfo
7752 LoopCloneContext* context;
7754 const unsigned loopNum;
7755 const bool cloneForArrayBounds;
7756 const bool cloneForGDVTests;
7757 LoopCloneVisitorInfo(LoopCloneContext* context,
7760 bool cloneForArrayBounds,
7761 bool cloneForGDVTests)
7765 , cloneForArrayBounds(cloneForArrayBounds)
7766 , cloneForGDVTests(cloneForGDVTests)
7771 bool optIsStackLocalInvariant(unsigned loopNum, unsigned lclNum);
7772 bool optExtractArrIndex(GenTree* tree, ArrIndex* result, unsigned lhsNum, bool* topLevelIsFinal);
7773 bool optReconstructArrIndexHelp(GenTree* tree, ArrIndex* result, unsigned lhsNum, bool* topLevelIsFinal);
7774 bool optReconstructArrIndex(GenTree* tree, ArrIndex* result);
7775 bool optIdentifyLoopOptInfo(unsigned loopNum, LoopCloneContext* context);
7776 static fgWalkPreFn optCanOptimizeByLoopCloningVisitor;
7777 fgWalkResult optCanOptimizeByLoopCloning(GenTree* tree, LoopCloneVisitorInfo* info);
7778 bool optObtainLoopCloningOpts(LoopCloneContext* context);
7779 bool optIsLoopClonable(unsigned loopInd);
7780 bool optCheckLoopCloningGDVTestProfitable(GenTreeOp* guard, LoopCloneVisitorInfo* info);
7781 bool optIsHandleOrIndirOfHandle(GenTree* tree, GenTreeFlags handleType);
7783 static bool optLoopCloningEnabled();
7786 void optDebugLogLoopCloning(BasicBlock* block, Statement* insertBefore);
7788 void optPerformStaticOptimizations(unsigned loopNum, LoopCloneContext* context DEBUGARG(bool fastPath));
7789 bool optComputeDerefConditions(unsigned loopNum, LoopCloneContext* context);
7790 bool optDeriveLoopCloningConditions(unsigned loopNum, LoopCloneContext* context);
7791 BasicBlock* optInsertLoopChoiceConditions(LoopCloneContext* context,
7793 BasicBlock* slowHead,
7794 BasicBlock* insertAfter);
7797 ssize_t optGetArrayRefScaleAndIndex(GenTree* mul, GenTree** pIndex DEBUGARG(bool bRngChk));
7799 bool optReachWithoutCall(BasicBlock* srcBB, BasicBlock* dstBB);
7802 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
7803 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
7807 XX Does the register allocation and puts the remaining lclVars on the stack XX
7809 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
7810 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
7814 regNumber raUpdateRegStateForArg(RegState* regState, LclVarDsc* argDsc);
7816 void raMarkStkVars();
7818 #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE
7819 #if defined(TARGET_AMD64)
7820 static bool varTypeNeedsPartialCalleeSave(var_types type)
7822 assert(type != TYP_STRUCT);
7823 assert((type < TYP_SIMD32) || (type == TYP_SIMD32) || (type == TYP_SIMD64));
7824 return type >= TYP_SIMD32;
7826 #elif defined(TARGET_ARM64)
7827 static bool varTypeNeedsPartialCalleeSave(var_types type)
7829 assert(type != TYP_STRUCT);
7830 // ARM64 ABI FP Callee save registers only require Callee to save lower 8 Bytes
7831 // For SIMD types longer than 8 bytes Caller is responsible for saving and restoring Upper bytes.
7832 return ((type == TYP_SIMD16) || (type == TYP_SIMD12));
7834 #else // !defined(TARGET_AMD64) && !defined(TARGET_ARM64)
7835 #error("Unknown target architecture for FEATURE_PARTIAL_SIMD_CALLEE_SAVE")
7836 #endif // !defined(TARGET_AMD64) && !defined(TARGET_ARM64)
7837 #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE
7840 // Some things are used by both LSRA and regpredict allocators.
7842 FrameType rpFrameType;
7843 bool rpMustCreateEBPCalled; // Set to true after we have called rpMustCreateEBPFrame once
7845 bool rpMustCreateEBPFrame(INDEBUG(const char** wbReason));
7848 Lowering* m_pLowering; // Lowering; needed to Lower IR that's added or modified after Lowering.
7849 LinearScanInterface* m_pLinearScan; // Linear Scan allocator
7851 /* raIsVarargsStackArg is called by raMaskStkVars and by
7852 lvaComputeRefCounts. It identifies the special case
7853 where a varargs function has a parameter passed on the
7854 stack, other than the special varargs handle. Such parameters
7855 require special treatment, because they cannot be tracked
7856 by the GC (their offsets in the stack are not known
7860 bool raIsVarargsStackArg(unsigned lclNum)
7864 LclVarDsc* varDsc = lvaGetDesc(lclNum);
7866 assert(varDsc->lvIsParam);
7868 return (info.compIsVarArgs && !varDsc->lvIsRegArg && (lclNum != lvaVarargsHandleArg));
7874 #endif // TARGET_X86
7878 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
7879 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
7883 XX Get to the class and method info from the Execution Engine given XX
7884 XX tokens for the class and method XX
7886 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
7887 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
7893 void eeGetCallInfo(CORINFO_RESOLVED_TOKEN* pResolvedToken,
7894 CORINFO_RESOLVED_TOKEN* pConstrainedToken,
7895 CORINFO_CALLINFO_FLAGS flags,
7896 CORINFO_CALL_INFO* pResult);
7898 void eeGetFieldInfo(CORINFO_RESOLVED_TOKEN* pResolvedToken,
7899 CORINFO_ACCESS_FLAGS flags,
7900 CORINFO_FIELD_INFO* pResult);
7904 bool eeIsValueClass(CORINFO_CLASS_HANDLE clsHnd);
7905 bool eeIsIntrinsic(CORINFO_METHOD_HANDLE ftn);
7906 bool eeIsFieldStatic(CORINFO_FIELD_HANDLE fldHnd);
7908 var_types eeGetFieldType(CORINFO_FIELD_HANDLE fldHnd, CORINFO_CLASS_HANDLE* pStructHnd = nullptr);
7910 template <typename TPrint>
7911 void eeAppendPrint(class StringPrinter* printer, TPrint print);
7912 // Conventions: the "base" primitive printing functions take StringPrinter*
7913 // and do not do any SPMI handling. There are then convenience printing
7914 // functions exposed on top that have SPMI handling and additional buffer
7915 // handling. Note that the strings returned are never truncated here.
7916 void eePrintJitType(class StringPrinter* printer, var_types jitType);
7917 void eePrintType(class StringPrinter* printer, CORINFO_CLASS_HANDLE clsHnd, bool includeInstantiation);
7918 void eePrintTypeOrJitAlias(class StringPrinter* printer, CORINFO_CLASS_HANDLE clsHnd, bool includeInstantiation);
7919 void eePrintMethod(class StringPrinter* printer,
7920 CORINFO_CLASS_HANDLE clsHnd,
7921 CORINFO_METHOD_HANDLE methodHnd,
7922 CORINFO_SIG_INFO* sig,
7923 bool includeClassInstantiation,
7924 bool includeMethodInstantiation,
7925 bool includeSignature,
7926 bool includeReturnType,
7927 bool includeThisSpecifier);
7929 void eePrintField(class StringPrinter* printer, CORINFO_FIELD_HANDLE fldHnd, bool includeType);
7931 const char* eeGetMethodFullName(CORINFO_METHOD_HANDLE hnd,
7932 bool includeReturnType = true,
7933 bool includeThisSpecifier = true,
7934 char* buffer = nullptr,
7935 size_t bufferSize = 0);
7937 const char* eeGetMethodName(CORINFO_METHOD_HANDLE methHnd, char* buffer = nullptr, size_t bufferSize = 0);
7939 const char* eeGetFieldName(CORINFO_FIELD_HANDLE fldHnd,
7941 char* buffer = nullptr,
7942 size_t bufferSize = 0);
7944 const char* eeGetClassName(CORINFO_CLASS_HANDLE clsHnd, char* buffer = nullptr, size_t bufferSize = 0);
7946 void eePrintObjectDescription(const char* prefix, CORINFO_OBJECT_HANDLE handle);
7947 const char* eeGetShortClassName(CORINFO_CLASS_HANDLE clsHnd);
7950 unsigned eeTryGetClassSize(CORINFO_CLASS_HANDLE clsHnd);
7953 unsigned compMethodHash(CORINFO_METHOD_HANDLE methodHandle);
7955 var_types eeGetArgType(CORINFO_ARG_LIST_HANDLE list, CORINFO_SIG_INFO* sig);
7956 var_types eeGetArgType(CORINFO_ARG_LIST_HANDLE list, CORINFO_SIG_INFO* sig, bool* isPinned);
7957 CORINFO_CLASS_HANDLE eeGetArgClass(CORINFO_SIG_INFO* sig, CORINFO_ARG_LIST_HANDLE list);
7958 CORINFO_CLASS_HANDLE eeGetClassFromContext(CORINFO_CONTEXT_HANDLE context);
7959 unsigned eeGetArgSize(CORINFO_ARG_LIST_HANDLE list, CORINFO_SIG_INFO* sig);
7960 static unsigned eeGetArgSizeAlignment(var_types type, bool isFloatHfa);
7962 // VOM info, method sigs
7964 void eeGetSig(unsigned sigTok,
7965 CORINFO_MODULE_HANDLE scope,
7966 CORINFO_CONTEXT_HANDLE context,
7967 CORINFO_SIG_INFO* retSig);
7969 void eeGetCallSiteSig(unsigned sigTok,
7970 CORINFO_MODULE_HANDLE scope,
7971 CORINFO_CONTEXT_HANDLE context,
7972 CORINFO_SIG_INFO* retSig);
7974 void eeGetMethodSig(CORINFO_METHOD_HANDLE methHnd, CORINFO_SIG_INFO* retSig, CORINFO_CLASS_HANDLE owner = nullptr);
7976 CORINFO_EE_INFO eeInfo;
7977 bool eeInfoInitialized;
7979 CORINFO_EE_INFO* eeGetEEInfo();
7981 // Gets the offset of a SDArray's first element
7982 static unsigned eeGetArrayDataOffset();
7984 // Get the offset of a MDArray's first element
7985 static unsigned eeGetMDArrayDataOffset(unsigned rank);
7987 // Get the offset of a MDArray's dimension length for a given dimension.
7988 static unsigned eeGetMDArrayLengthOffset(unsigned rank, unsigned dimension);
7990 // Get the offset of a MDArray's lower bound for a given dimension.
7991 static unsigned eeGetMDArrayLowerBoundOffset(unsigned rank, unsigned dimension);
7993 GenTree* eeGetPInvokeCookie(CORINFO_SIG_INFO* szMetaSig);
7995 // Returns the page size for the target machine as reported by the EE.
7996 target_size_t eeGetPageSize()
7998 return (target_size_t)eeGetEEInfo()->osPageSize;
8001 //------------------------------------------------------------------------
8002 // VirtualStubParam: virtual stub dispatch extra parameter (slot address).
8004 // It represents Abi and target specific registers for the parameter.
8006 class VirtualStubParamInfo
8009 VirtualStubParamInfo(bool isNativeAOT)
8011 #if defined(TARGET_X86)
8014 #elif defined(TARGET_AMD64)
8025 #elif defined(TARGET_ARM)
8036 #elif defined(TARGET_ARM64)
8039 #elif defined(TARGET_LOONGARCH64)
8042 #elif defined(TARGET_RISCV64)
8046 #error Unsupported or unset target architecture
8050 regNumber GetReg() const
8055 _regMask_enum GetRegMask() const
8062 _regMask_enum regMask;
8065 VirtualStubParamInfo* virtualStubParamInfo;
8067 bool IsTargetAbi(CORINFO_RUNTIME_ABI abi)
8069 return eeGetEEInfo()->targetAbi == abi;
8072 bool generateCFIUnwindCodes()
8074 #if defined(FEATURE_CFI_SUPPORT)
8075 return TargetOS::IsUnix && IsTargetAbi(CORINFO_NATIVEAOT_ABI);
8081 // Debugging support - Line number info
8083 void eeGetStmtOffsets();
8085 unsigned eeBoundariesCount;
8087 ICorDebugInfo::OffsetMapping* eeBoundaries; // Boundaries to report to the EE
8088 void eeSetLIcount(unsigned count);
8089 void eeSetLIinfo(unsigned which, UNATIVE_OFFSET offs, IPmappingDscKind kind, const ILLocation& loc);
8093 static void eeDispILOffs(IL_OFFSET offs);
8094 static void eeDispSourceMappingOffs(uint32_t offs);
8095 static void eeDispLineInfo(const ICorDebugInfo::OffsetMapping* line);
8096 void eeDispLineInfos();
8099 // Debugging support - Local var info
8103 unsigned eeVarsCount;
8105 struct VarResultInfo
8107 UNATIVE_OFFSET startOffset;
8108 UNATIVE_OFFSET endOffset;
8110 CodeGenInterface::siVarLoc loc;
8112 void eeSetLVcount(unsigned count);
8113 void eeSetLVinfo(unsigned which,
8114 UNATIVE_OFFSET startOffs,
8115 UNATIVE_OFFSET length,
8117 const CodeGenInterface::siVarLoc& loc);
8121 void eeDispVar(ICorDebugInfo::NativeVarInfo* var);
8122 void eeDispVars(CORINFO_METHOD_HANDLE ftn, ULONG32 cVars, ICorDebugInfo::NativeVarInfo* vars);
8125 // ICorJitInfo wrappers
8127 void eeAllocMem(AllocMemArgs* args, const UNATIVE_OFFSET roDataSectionAlignment);
8129 void eeReserveUnwindInfo(bool isFunclet, bool isColdCode, ULONG unwindSize);
8131 void eeAllocUnwindInfo(BYTE* pHotCode,
8137 CorJitFuncKind funcKind);
8139 void eeSetEHcount(unsigned cEH);
8141 void eeSetEHinfo(unsigned EHnumber, const CORINFO_EH_CLAUSE* clause);
8143 WORD eeGetRelocTypeHint(void* target);
8145 // ICorStaticInfo wrapper functions
8147 #if defined(UNIX_AMD64_ABI)
8149 static void dumpSystemVClassificationType(SystemVClassificationType ct);
8152 void eeGetSystemVAmd64PassStructInRegisterDescriptor(
8153 /*IN*/ CORINFO_CLASS_HANDLE structHnd,
8154 /*OUT*/ SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR* structPassInRegDescPtr);
8155 #endif // UNIX_AMD64_ABI
8157 template <typename ParamType>
8158 bool eeRunWithErrorTrap(void (*function)(ParamType*), ParamType* param)
8160 return eeRunWithErrorTrapImp(reinterpret_cast<void (*)(void*)>(function), reinterpret_cast<void*>(param));
8163 bool eeRunWithErrorTrapImp(void (*function)(void*), void* param);
8165 template <typename ParamType>
8166 bool eeRunWithSPMIErrorTrap(void (*function)(ParamType*), ParamType* param)
8168 return eeRunWithSPMIErrorTrapImp(reinterpret_cast<void (*)(void*)>(function), reinterpret_cast<void*>(param));
8171 template <typename Functor>
8172 bool eeRunFunctorWithSPMIErrorTrap(Functor f)
8174 return eeRunWithSPMIErrorTrap<Functor>([](Functor* pf) { (*pf)(); }, &f);
8177 bool eeRunWithSPMIErrorTrapImp(void (*function)(void*), void* param);
8179 // Utility functions
8181 static CORINFO_METHOD_HANDLE eeFindHelper(unsigned helper);
8182 static CorInfoHelpFunc eeGetHelperNum(CORINFO_METHOD_HANDLE method);
8184 enum StaticHelperReturnValue
8186 SHRV_STATIC_BASE_PTR,
8189 static bool IsStaticHelperEligibleForExpansion(GenTree* tree,
8190 bool* isGc = nullptr,
8191 StaticHelperReturnValue* retValKind = nullptr);
8192 static bool IsSharedStaticHelper(GenTree* tree);
8193 static bool IsGcSafePoint(GenTreeCall* call);
8195 static CORINFO_FIELD_HANDLE eeFindJitDataOffs(unsigned jitDataOffs);
8196 // returns true/false if 'field' is a Jit Data offset
8197 static bool eeIsJitDataOffs(CORINFO_FIELD_HANDLE field);
8198 // returns a number < 0 if 'field' is not a Jit Data offset, otherwise the data offset (limited to 2GB)
8199 static int eeGetJitDataOffs(CORINFO_FIELD_HANDLE field);
8201 /*****************************************************************************/
8204 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8205 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8209 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8210 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8214 CodeGenInterface* codeGen;
8216 // Record the instr offset mapping to the generated code
8218 jitstd::list<IPmappingDsc> genIPmappings;
8219 jitstd::list<RichIPMapping> genRichIPmappings;
8221 // Managed RetVal - A side hash table meant to record the mapping from a
8222 // GT_CALL node to its debug info. This info is used to emit sequence points
8223 // that can be used by debugger to determine the native offset at which the
8224 // managed RetVal will be available.
8226 // In fact we can store debug info in a GT_CALL node. This was ruled out in
8227 // favor of a side table for two reasons: 1) We need debug info for only those
8228 // GT_CALL nodes (created during importation) that correspond to an IL call and
8229 // whose return type is other than TYP_VOID. 2) GT_CALL node is a frequently used
8230 // structure and IL offset is needed only when generating debuggable code. Therefore
8231 // it is desirable to avoid memory size penalty in retail scenarios.
8232 typedef JitHashTable<GenTree*, JitPtrKeyFuncs<GenTree>, DebugInfo> CallSiteDebugInfoTable;
8233 CallSiteDebugInfoTable* genCallSite2DebugInfoMap;
8235 unsigned genReturnLocal; // Local number for the return value when applicable.
8236 BasicBlock* genReturnBB; // jumped to when not optimizing for speed.
8238 // The following properties are part of CodeGenContext. Getters are provided here for
8239 // convenience and backward compatibility, but the properties can only be set by invoking
8240 // the setter on CodeGenContext directly.
8242 emitter* GetEmitter() const
8244 return codeGen->GetEmitter();
8247 bool isFramePointerUsed() const
8249 return codeGen->isFramePointerUsed();
8252 bool GetInterruptible()
8254 return codeGen->GetInterruptible();
8256 void SetInterruptible(bool value)
8258 codeGen->SetInterruptible(value);
8262 const bool genDoubleAlign()
8264 return codeGen->doDoubleAlign();
8266 DWORD getCanDoubleAlign();
8267 bool shouldDoubleAlign(unsigned refCntStk,
8269 weight_t refCntWtdReg,
8270 unsigned refCntStkParam,
8271 weight_t refCntWtdStkDbl);
8272 #endif // DOUBLE_ALIGN
8274 bool IsFullPtrRegMapRequired()
8276 return codeGen->IsFullPtrRegMapRequired();
8278 void SetFullPtrRegMapRequired(bool value)
8280 codeGen->SetFullPtrRegMapRequired(value);
8283 // Things that MAY belong either in CodeGen or CodeGenContext
8285 #if defined(FEATURE_EH_FUNCLETS)
8286 FuncInfoDsc* compFuncInfos;
8287 unsigned short compCurrFuncIdx;
8288 unsigned short compFuncInfoCount;
8290 unsigned short compFuncCount()
8292 assert(fgFuncletsCreated);
8293 return compFuncInfoCount;
8296 #else // !FEATURE_EH_FUNCLETS
8298 // This is a no-op when there are no funclets!
8299 void genUpdateCurrentFunclet(BasicBlock* block)
8304 FuncInfoDsc compFuncInfoRoot;
8306 static const unsigned compCurrFuncIdx = 0;
8308 unsigned short compFuncCount()
8313 #endif // !FEATURE_EH_FUNCLETS
8315 FuncInfoDsc* funCurrentFunc();
8316 void funSetCurrentFunc(unsigned funcIdx);
8317 FuncInfoDsc* funGetFunc(unsigned funcIdx);
8318 unsigned int funGetFuncIdx(BasicBlock* block);
8322 VARSET_TP compCurLife; // current live variables
8323 GenTree* compCurLifeTree; // node after which compCurLife has been computed
8325 // Compare the given "newLife" with last set of live variables and update
8326 // codeGen "gcInfo", siScopes, "regSet" with the new variable's homes/liveness.
8327 template <bool ForCodeGen>
8328 void compChangeLife(VARSET_VALARG_TP newLife);
8330 // Update the GC's masks, register's masks and reports change on variable's homes given a set of
8331 // current live variables if changes have happened since "compCurLife".
8332 template <bool ForCodeGen>
8333 inline void compUpdateLife(VARSET_VALARG_TP newLife);
8335 // Gets a register mask that represent the kill set for a helper call since
8336 // not all JIT Helper calls follow the standard ABI on the target architecture.
8337 regMaskTP compHelperCallKillSet(CorInfoHelpFunc helper);
8340 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8341 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8345 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8346 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8349 #if !defined(__GNUC__)
8350 #pragma region Unwind information
8355 // Infrastructure functions: start/stop/reserve/emit.
8358 void unwindBegProlog();
8359 void unwindEndProlog();
8360 void unwindBegEpilog();
8361 void unwindEndEpilog();
8362 void unwindReserve();
8363 void unwindEmit(void* pHotCode, void* pColdCode);
8366 // Specific unwind information functions: called by code generation to indicate a particular
8367 // prolog or epilog unwindable instruction has been generated.
8370 void unwindPush(regNumber reg);
8371 void unwindAllocStack(unsigned size);
8372 void unwindSetFrameReg(regNumber reg, unsigned offset);
8373 void unwindSaveReg(regNumber reg, unsigned offset);
8375 #if defined(TARGET_ARM)
8376 void unwindPushMaskInt(regMaskTP mask);
8377 void unwindPushMaskFloat(regMaskTP mask);
8378 void unwindPopMaskInt(regMaskTP mask);
8379 void unwindPopMaskFloat(regMaskTP mask);
8380 void unwindBranch16(); // The epilog terminates with a 16-bit branch (e.g., "bx lr")
8381 void unwindNop(unsigned codeSizeInBytes); // Generate unwind NOP code. 'codeSizeInBytes' is 2 or 4 bytes. Only
8382 // called via unwindPadding().
8383 void unwindPadding(); // Generate a sequence of unwind NOP codes representing instructions between the last
8384 // instruction and the current location.
8385 #endif // TARGET_ARM
8387 #if defined(TARGET_ARM64)
8389 void unwindPadding(); // Generate a sequence of unwind NOP codes representing instructions between the last
8390 // instruction and the current location.
8391 void unwindSaveReg(regNumber reg, int offset); // str reg, [sp, #offset]
8392 void unwindSaveRegPreindexed(regNumber reg, int offset); // str reg, [sp, #offset]!
8393 void unwindSaveRegPair(regNumber reg1, regNumber reg2, int offset); // stp reg1, reg2, [sp, #offset]
8394 void unwindSaveRegPairPreindexed(regNumber reg1, regNumber reg2, int offset); // stp reg1, reg2, [sp, #offset]!
8395 void unwindSaveNext(); // unwind code: save_next
8396 void unwindReturn(regNumber reg); // ret lr
8397 #endif // defined(TARGET_ARM64)
8399 #if defined(TARGET_LOONGARCH64)
8401 void unwindPadding(); // Generate a sequence of unwind NOP codes representing instructions between the last
8402 // instruction and the current location.
8403 void unwindSaveReg(regNumber reg, int offset);
8404 void unwindSaveRegPair(regNumber reg1, regNumber reg2, int offset);
8405 void unwindReturn(regNumber reg);
8406 #endif // defined(TARGET_LOONGARCH64)
8408 #if defined(TARGET_RISCV64)
8410 void unwindPadding(); // Generate a sequence of unwind NOP codes representing instructions between the last
8411 // instruction and the current location.
8412 void unwindSaveReg(regNumber reg, int offset);
8413 void unwindSaveRegPair(regNumber reg1, regNumber reg2, int offset);
8414 void unwindReturn(regNumber reg);
8415 #endif // defined(TARGET_RISCV64)
8418 // Private "helper" functions for the unwind implementation.
8422 #if defined(FEATURE_EH_FUNCLETS)
8423 void unwindGetFuncLocations(FuncInfoDsc* func,
8424 bool getHotSectionData,
8425 /* OUT */ emitLocation** ppStartLoc,
8426 /* OUT */ emitLocation** ppEndLoc);
8427 #endif // FEATURE_EH_FUNCLETS
8429 void unwindReserveFunc(FuncInfoDsc* func);
8430 void unwindEmitFunc(FuncInfoDsc* func, void* pHotCode, void* pColdCode);
8432 #if defined(TARGET_AMD64) || (defined(TARGET_X86) && defined(FEATURE_EH_FUNCLETS))
8434 void unwindReserveFuncHelper(FuncInfoDsc* func, bool isHotCode);
8435 void unwindEmitFuncHelper(FuncInfoDsc* func, void* pHotCode, void* pColdCode, bool isHotCode);
8437 #endif // TARGET_AMD64 || (TARGET_X86 && FEATURE_EH_FUNCLETS)
8439 UNATIVE_OFFSET unwindGetCurrentOffset(FuncInfoDsc* func);
8441 #if defined(TARGET_AMD64)
8443 void unwindBegPrologWindows();
8444 void unwindPushWindows(regNumber reg);
8445 void unwindAllocStackWindows(unsigned size);
8446 void unwindSetFrameRegWindows(regNumber reg, unsigned offset);
8447 void unwindSaveRegWindows(regNumber reg, unsigned offset);
8449 #ifdef UNIX_AMD64_ABI
8450 void unwindSaveRegCFI(regNumber reg, unsigned offset);
8451 #endif // UNIX_AMD64_ABI
8452 #elif defined(TARGET_ARM)
8454 void unwindPushPopMaskInt(regMaskTP mask, bool useOpsize16);
8455 void unwindPushPopMaskFloat(regMaskTP mask);
8457 #endif // TARGET_ARM
8459 #if defined(FEATURE_CFI_SUPPORT)
8460 short mapRegNumToDwarfReg(regNumber reg);
8461 void createCfiCode(FuncInfoDsc* func, UNATIVE_OFFSET codeOffset, UCHAR opcode, short dwarfReg, INT offset = 0);
8462 void unwindPushPopCFI(regNumber reg);
8463 void unwindBegPrologCFI();
8464 void unwindPushPopMaskCFI(regMaskTP regMask, bool isFloat);
8465 void unwindAllocStackCFI(unsigned size);
8466 void unwindSetFrameRegCFI(regNumber reg, unsigned offset);
8467 void unwindEmitFuncCFI(FuncInfoDsc* func, void* pHotCode, void* pColdCode);
8469 void DumpCfiInfo(bool isHotCode,
8470 UNATIVE_OFFSET startOffset,
8471 UNATIVE_OFFSET endOffset,
8473 const CFI_CODE* const pCfiCode);
8476 #endif // FEATURE_CFI_SUPPORT
8478 #if !defined(__GNUC__)
8479 #pragma endregion // Note: region is NOT under !defined(__GNUC__)
8483 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8484 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8488 XX Info about SIMD types, methods and the SIMD assembly (i.e. the assembly XX
8489 XX that contains the distinguished, well-known SIMD type definitions). XX
8491 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8492 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
8495 bool IsBaselineSimdIsaSupported()
8498 #if defined(TARGET_XARCH)
8499 CORINFO_InstructionSet minimumIsa = InstructionSet_SSE2;
8500 #elif defined(TARGET_ARM64)
8501 CORINFO_InstructionSet minimumIsa = InstructionSet_AdvSimd;
8502 #elif defined(TARGET_LOONGARCH64)
8503 // TODO: supporting SIMD feature for LoongArch64.
8504 assert(!"unimplemented yet on LA");
8505 CORINFO_InstructionSet minimumIsa = 0;
8507 #error Unsupported platform
8508 #endif // !TARGET_XARCH && !TARGET_ARM64 && !TARGET_LOONGARCH64
8510 return compOpportunisticallyDependsOn(minimumIsa);
8517 bool IsBaselineSimdIsaSupportedDebugOnly()
8520 #if defined(TARGET_XARCH)
8521 CORINFO_InstructionSet minimumIsa = InstructionSet_SSE2;
8522 #elif defined(TARGET_ARM64)
8523 CORINFO_InstructionSet minimumIsa = InstructionSet_AdvSimd;
8525 #error Unsupported platform
8526 #endif // !TARGET_XARCH && !TARGET_ARM64
8528 return compIsaSupportedDebugOnly(minimumIsa);
8531 #endif // FEATURE_SIMD
8535 bool isIntrinsicType(CORINFO_CLASS_HANDLE clsHnd)
8537 return info.compCompHnd->isIntrinsicType(clsHnd);
8540 const char* getClassNameFromMetadata(CORINFO_CLASS_HANDLE cls, const char** namespaceName)
8542 return info.compCompHnd->getClassNameFromMetadata(cls, namespaceName);
8545 CORINFO_CLASS_HANDLE getTypeInstantiationArgument(CORINFO_CLASS_HANDLE cls, unsigned index)
8547 return info.compCompHnd->getTypeInstantiationArgument(cls, index);
8550 bool isNumericsNamespace(const char* ns)
8552 return strcmp(ns, "System.Numerics") == 0;
8555 bool isRuntimeIntrinsicsNamespace(const char* ns)
8557 return strcmp(ns, "System.Runtime.Intrinsics") == 0;
8561 // Have we identified any SIMD types?
8562 // This is currently used by struct promotion to avoid getting type information for a struct
8563 // field to see if it is a SIMD type, if we haven't seen any SIMD types or operations in
8565 bool _usesSIMDTypes;
8566 bool usesSIMDTypes()
8568 return _usesSIMDTypes;
8570 void setUsesSIMDTypes(bool value)
8572 _usesSIMDTypes = value;
8575 // This is a temp lclVar allocated on the stack as TYP_SIMD. It is used to implement intrinsics
8576 // that require indexed access to the individual fields of the vector, which is not well supported
8577 // by the hardware. It is allocated when/if such situations are encountered during Lowering.
8578 unsigned lvaSIMDInitTempVarNum;
8580 struct SIMDHandlesCache
8582 CORINFO_CLASS_HANDLE PlaneHandle;
8583 CORINFO_CLASS_HANDLE QuaternionHandle;
8584 CORINFO_CLASS_HANDLE Vector2Handle;
8585 CORINFO_CLASS_HANDLE Vector3Handle;
8586 CORINFO_CLASS_HANDLE Vector4Handle;
8587 CORINFO_CLASS_HANDLE VectorHandle;
8591 memset(this, 0, sizeof(*this));
8595 SIMDHandlesCache* m_simdHandleCache;
8597 // Returns true if this is a SIMD type that should be considered an opaque
8598 // vector type (i.e. do not analyze or promote its fields).
8599 // Note that all but the fixed vector types are opaque, even though they may
8600 // actually be declared as having fields.
8601 bool isOpaqueSIMDType(CORINFO_CLASS_HANDLE structHandle) const
8603 // We order the checks roughly by expected hit count so early exits are possible
8605 if (m_simdHandleCache == nullptr)
8610 if (structHandle == m_simdHandleCache->Vector4Handle)
8615 if (structHandle == m_simdHandleCache->Vector3Handle)
8620 if (structHandle == m_simdHandleCache->Vector2Handle)
8625 if (structHandle == m_simdHandleCache->QuaternionHandle)
8630 if (structHandle == m_simdHandleCache->PlaneHandle)
8638 bool isOpaqueSIMDType(ClassLayout* layout) const
8640 if (layout->IsBlockLayout())
8645 return isOpaqueSIMDType(layout->GetClassHandle());
8648 // Returns true if the lclVar is an opaque SIMD type.
8649 bool isOpaqueSIMDLclVar(const LclVarDsc* varDsc) const
8651 if (!varTypeIsSIMD(varDsc))
8656 if (varDsc->GetLayout() == nullptr)
8661 return isOpaqueSIMDType(varDsc->GetLayout());
8664 bool isSIMDClass(CORINFO_CLASS_HANDLE clsHnd)
8666 if (isIntrinsicType(clsHnd))
8668 const char* namespaceName = nullptr;
8669 (void)getClassNameFromMetadata(clsHnd, &namespaceName);
8670 return isNumericsNamespace(namespaceName);
8675 bool isHWSIMDClass(CORINFO_CLASS_HANDLE clsHnd)
8677 #ifdef FEATURE_HW_INTRINSICS
8678 if (isIntrinsicType(clsHnd))
8680 const char* namespaceName = nullptr;
8681 (void)getClassNameFromMetadata(clsHnd, &namespaceName);
8682 return isRuntimeIntrinsicsNamespace(namespaceName);
8684 #endif // FEATURE_HW_INTRINSICS
8688 bool isSIMDorHWSIMDClass(CORINFO_CLASS_HANDLE clsHnd)
8690 return isSIMDClass(clsHnd) || isHWSIMDClass(clsHnd);
8693 // Get the base (element) type and size in bytes for a SIMD type. Returns CORINFO_TYPE_UNDEF
8694 // if it is not a SIMD type or is an unsupported base JIT type.
8695 CorInfoType getBaseJitTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes = nullptr);
8697 CorInfoType getBaseJitTypeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd)
8699 return getBaseJitTypeAndSizeOfSIMDType(typeHnd, nullptr);
8702 GenTree* impSIMDPopStack();
8704 void setLclRelatedToSIMDIntrinsic(GenTree* tree);
8705 bool areFieldsContiguous(GenTreeIndir* op1, GenTreeIndir* op2);
8706 bool areLocalFieldsContiguous(GenTreeLclFld* first, GenTreeLclFld* second);
8707 bool areArrayElementsContiguous(GenTree* op1, GenTree* op2);
8708 bool areArgumentsContiguous(GenTree* op1, GenTree* op2);
8709 GenTree* CreateAddressNodeForSimdHWIntrinsicCreate(GenTree* tree, var_types simdBaseType, unsigned simdSize);
8711 // Get the size of the SIMD type in bytes
8712 int getSIMDTypeSizeInBytes(CORINFO_CLASS_HANDLE typeHnd)
8714 unsigned sizeBytes = 0;
8715 (void)getBaseJitTypeAndSizeOfSIMDType(typeHnd, &sizeBytes);
8719 // Get the number of elements of baseType of SIMD vector given by its size and baseType
8720 static int getSIMDVectorLength(unsigned simdSize, var_types baseType);
8722 // Get the number of elements of baseType of SIMD vector given by its type handle
8723 int getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd);
8725 // Get preferred alignment of SIMD type.
8726 int getSIMDTypeAlignment(var_types simdType);
8728 // Get the number of bytes in a System.Numeric.Vector<T> for the current compilation.
8729 // Note - cannot be used for System.Runtime.Intrinsic
8730 uint32_t getVectorTByteLength()
8732 // We need to report the ISA dependency to the VM so that scenarios
8733 // such as R2R work correctly for larger vector sizes, so we always
8734 // do `compExactlyDependsOn` for such cases.
8735 CLANG_FORMAT_COMMENT_ANCHOR;
8737 #if defined(TARGET_XARCH)
8738 // TODO-XArch: Add support for 512-bit Vector<T>
8739 assert(!compIsaSupportedDebugOnly(InstructionSet_VectorT512));
8741 if (compExactlyDependsOn(InstructionSet_VectorT256))
8743 assert(!compIsaSupportedDebugOnly(InstructionSet_VectorT128));
8744 return YMM_REGSIZE_BYTES;
8746 else if (compExactlyDependsOn(InstructionSet_VectorT128))
8748 return XMM_REGSIZE_BYTES;
8752 // TODO: We should be returning 0 here, but there are a number of
8753 // places that don't quite get handled correctly in that scenario
8755 return XMM_REGSIZE_BYTES;
8757 #elif defined(TARGET_ARM64)
8758 if (compExactlyDependsOn(InstructionSet_VectorT128))
8760 return FP_REGSIZE_BYTES;
8764 // TODO: We should be returning 0 here, but there are a number of
8765 // places that don't quite get handled correctly in that scenario
8767 return FP_REGSIZE_BYTES;
8770 assert(!"getVectorTByteLength() unimplemented on target arch");
8775 // The minimum and maximum possible number of bytes in a SIMD vector.
8777 // getMaxVectorByteLength
8778 // The minimum SIMD size supported by System.Numeric.Vectors or System.Runtime.Intrinsic
8779 // Arm.AdvSimd: 16-byte Vector<T> and Vector128<T>
8780 // X86.SSE: 16-byte Vector<T> and Vector128<T>
8781 // X86.AVX: 16-byte Vector<T> and Vector256<T>
8782 // X86.AVX2: 32-byte Vector<T> and Vector256<T>
8783 // X86.AVX512F: 32-byte Vector<T> and Vector512<T>
8784 uint32_t getMaxVectorByteLength() const
8786 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH)
8787 if (compOpportunisticallyDependsOn(InstructionSet_AVX512F))
8789 return ZMM_REGSIZE_BYTES;
8791 else if (compOpportunisticallyDependsOn(InstructionSet_AVX))
8793 return YMM_REGSIZE_BYTES;
8795 else if (compOpportunisticallyDependsOn(InstructionSet_SSE))
8797 return XMM_REGSIZE_BYTES;
8801 // TODO: We should be returning 0 here, but there are a number of
8802 // places that don't quite get handled correctly in that scenario
8804 return XMM_REGSIZE_BYTES;
8806 #elif defined(TARGET_ARM64)
8807 if (compOpportunisticallyDependsOn(InstructionSet_AdvSimd))
8809 return FP_REGSIZE_BYTES;
8813 // TODO: We should be returning 0 here, but there are a number of
8814 // places that don't quite get handled correctly in that scenario
8816 return FP_REGSIZE_BYTES;
8819 assert(!"getMaxVectorByteLength() unimplemented on target arch");
8824 //------------------------------------------------------------------------
8825 // getPreferredVectorByteLength: Gets the preferred length, in bytes, to use for vectorization
8827 uint32_t getPreferredVectorByteLength() const
8829 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH)
8830 uint32_t preferredVectorByteLength = opts.preferredVectorByteLength;
8832 if (preferredVectorByteLength != 0)
8834 return min(getMaxVectorByteLength(), preferredVectorByteLength);
8836 #endif // FEATURE_HW_INTRINSICS && TARGET_XARCH
8838 return getMaxVectorByteLength();
8841 //------------------------------------------------------------------------
8842 // roundUpSIMDSize: rounds the given size up to the nearest SIMD size
8843 // available on the target. Examples on XARCH:
8846 // size: 30 -> YMM (or XMM if target doesn't support AVX)
8847 // size: 70 -> ZMM (or YMM or XMM depending on target)
8850 // size - size of the data to process with SIMD
8853 // It's only supposed to be used for scenarios where we can
8854 // perform an overlapped load/store.
8856 uint32_t roundUpSIMDSize(unsigned size)
8858 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH)
8859 uint32_t maxSize = getPreferredVectorByteLength();
8860 assert(maxSize <= ZMM_REGSIZE_BYTES);
8862 if ((size <= XMM_REGSIZE_BYTES) && (maxSize > XMM_REGSIZE_BYTES))
8864 return XMM_REGSIZE_BYTES;
8867 if ((size <= YMM_REGSIZE_BYTES) && (maxSize > YMM_REGSIZE_BYTES))
8869 return YMM_REGSIZE_BYTES;
8873 #elif defined(TARGET_ARM64)
8874 assert(getMaxVectorByteLength() == FP_REGSIZE_BYTES);
8875 return FP_REGSIZE_BYTES;
8877 assert(!"roundUpSIMDSize() unimplemented on target arch");
8882 //------------------------------------------------------------------------
8883 // roundDownSIMDSize: rounds the given size down to the nearest SIMD size
8884 // available on the target. Examples on XARCH:
8887 // size: 30 -> XMM (not enough for AVX)
8888 // size: 60 -> YMM (or XMM if target doesn't support AVX)
8889 // size: 70 -> ZMM/YMM/XMM whatever the current system can offer
8892 // size - size of the data to process with SIMD
8894 uint32_t roundDownSIMDSize(unsigned size)
8896 #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH)
8897 uint32_t maxSize = getPreferredVectorByteLength();
8898 assert(maxSize <= ZMM_REGSIZE_BYTES);
8900 if (size >= maxSize)
8902 // Size is bigger than max SIMD size the current target supports
8906 if ((size >= YMM_REGSIZE_BYTES) && (maxSize >= YMM_REGSIZE_BYTES))
8908 // Size is >= YMM but not enough for ZMM -> YMM
8909 return YMM_REGSIZE_BYTES;
8912 // Return 0 if size is even less than XMM, otherwise - XMM
8913 return (size >= XMM_REGSIZE_BYTES) ? XMM_REGSIZE_BYTES : 0;
8914 #elif defined(TARGET_ARM64)
8915 assert(getMaxVectorByteLength() == FP_REGSIZE_BYTES);
8916 return (size >= FP_REGSIZE_BYTES) ? FP_REGSIZE_BYTES : 0;
8918 assert(!"roundDownSIMDSize() unimplemented on target arch");
8923 uint32_t getMinVectorByteLength()
8925 return emitTypeSize(TYP_SIMD8);
8929 // Returns the codegen type for a given SIMD size.
8930 static var_types getSIMDTypeForSize(unsigned size)
8932 var_types simdType = TYP_UNDEF;
8935 simdType = TYP_SIMD8;
8937 else if (size == 12)
8939 simdType = TYP_SIMD12;
8941 else if (size == 16)
8943 simdType = TYP_SIMD16;
8945 #if defined(TARGET_XARCH)
8946 else if (size == 32)
8948 simdType = TYP_SIMD32;
8950 else if (size == 64)
8952 simdType = TYP_SIMD64;
8954 #endif // TARGET_XARCH
8957 noway_assert(!"Unexpected size for SIMD type");
8963 unsigned getSIMDInitTempVarNum(var_types simdType);
8965 #else // !FEATURE_SIMD
8966 bool isOpaqueSIMDLclVar(LclVarDsc* varDsc)
8970 unsigned int roundUpSIMDSize(unsigned size)
8974 unsigned int roundDownSIMDSize(unsigned size)
8978 #endif // FEATURE_SIMD
8981 // Similar to roundUpSIMDSize, but for General Purpose Registers (GPR)
8982 unsigned roundUpGPRSize(unsigned size)
8984 if (size > 4 && (REGSIZE_BYTES == 8))
8992 return size; // 2, 1, 0
8995 var_types roundDownMaxType(unsigned size)
8998 var_types result = TYP_UNDEF;
9000 if (IsBaselineSimdIsaSupported() && (roundDownSIMDSize(size) > 0))
9002 return getSIMDTypeForSize(roundDownSIMDSize(size));
9005 int nearestPow2 = 1 << BitOperations::Log2((unsigned)size);
9006 switch (min(nearestPow2, REGSIZE_BYTES))
9015 assert(REGSIZE_BYTES == 8);
9029 //------------------------------------------------------------------------
9030 // getUnrollThreshold: Calculates the unrolling threshold for the given operation
9033 // type - kind of the operation (memset/memcpy)
9034 // canUseSimd - whether it is allowed to use SIMD or not
9037 // The unrolling threshold for the given operation in bytes
9039 unsigned int getUnrollThreshold(UnrollKind type, bool canUseSimd = true)
9041 unsigned maxRegSize = REGSIZE_BYTES;
9042 unsigned threshold = maxRegSize;
9044 #if defined(FEATURE_SIMD)
9047 maxRegSize = getPreferredVectorByteLength();
9049 #if defined(TARGET_XARCH)
9050 assert(maxRegSize <= ZMM_REGSIZE_BYTES);
9051 threshold = maxRegSize;
9052 #elif defined(TARGET_ARM64)
9053 // ldp/stp instructions can load/store two 16-byte vectors at once, e.g.:
9058 threshold = maxRegSize * 2;
9061 #if defined(TARGET_XARCH)
9064 // Compatibility with previous logic: we used to allow memset:128/memcpy:64
9065 // on AMD64 (and 64/32 on x86) for cases where we don't use SIMD
9066 // see https://github.com/dotnet/runtime/issues/83297
9072 if (type == UnrollKind::Memset)
9074 // Typically, memset-like operations require less instructions than memcpy
9078 // Use 4 as a multiplier by default, thus, the final threshold will be:
9080 // | arch | memset | memcpy |
9081 // |-------------|--------|--------|
9082 // | x86 avx512 | 512 | 256 |
9083 // | x86 avx | 256 | 128 |
9084 // | x86 sse | 128 | 64 |
9085 // | arm64 | 256 | 128 | ldp/stp (2x128bit)
9086 // | arm | 32 | 16 | no SIMD support
9087 // | loongarch64 | 64 | 32 | no SIMD support
9089 // We might want to use a different multiplier for truly hot/cold blocks based on PGO data
9093 if (type == UnrollKind::Memmove)
9095 // NOTE: Memmove's unrolling is currently limited with LSRA -
9096 // up to LinearScan::MaxInternalCount number of temp regs, e.g. 5*16=80 bytes on arm64
9097 threshold = maxRegSize * 4;
9103 // Use to determine if a struct *might* be a SIMD type. As this function only takes a size, many
9104 // structs will fit the criteria.
9105 bool structSizeMightRepresentSIMDType(size_t structSize)
9108 return (structSize >= getMinVectorByteLength()) && (structSize <= getMaxVectorByteLength());
9111 #endif // FEATURE_SIMD
9114 #ifdef FEATURE_HW_INTRINSICS
9115 static bool vnEncodesResultTypeForHWIntrinsic(NamedIntrinsic hwIntrinsicID);
9116 #endif // FEATURE_HW_INTRINSICS
9119 // Returns true if the TYP_SIMD locals on stack are aligned at their
9120 // preferred byte boundary specified by getSIMDTypeAlignment().
9122 // As per the Intel manual, the preferred alignment for AVX vectors is
9123 // 32-bytes. It is not clear whether additional stack space used in
9124 // aligning stack is worth the benefit and for now will use 16-byte
9125 // alignment for AVX 256-bit vectors with unaligned load/stores to/from
9126 // memory. On x86, the stack frame is aligned to 4 bytes. We need to extend
9127 // existing support for double (8-byte) alignment to 16 or 32 byte
9128 // alignment for frames with local SIMD vars, if that is determined to be
9131 // On Amd64 and SysV, RSP+8 is aligned on entry to the function (before
9132 // prolog has run). This means that in RBP-based frames RBP will be 16-byte
9133 // aligned. For RSP-based frames these are only sometimes aligned, depending
9134 // on the frame size.
9136 bool isSIMDTypeLocalAligned(unsigned varNum)
9138 #if defined(FEATURE_SIMD) && ALIGN_SIMD_TYPES
9139 LclVarDsc* lcl = lvaGetDesc(varNum);
9140 if (varTypeIsSIMD(lcl))
9142 // TODO-Cleanup: Can't this use the lvExactSize on the varDsc?
9143 int alignment = getSIMDTypeAlignment(lcl->TypeGet());
9144 if (alignment <= STACK_ALIGN)
9147 int off = lvaFrameAddress(varNum, &rbpBased);
9148 // On SysV and Winx64 ABIs RSP+8 will be 16-byte aligned at the
9149 // first instruction of a function. If our frame is RBP based
9150 // then RBP will always be 16 bytes aligned, so we can simply
9151 // check the offset.
9154 return (off % alignment) == 0;
9157 // For RSP-based frame the alignment of RSP depends on our
9158 // locals. rsp+8 is aligned on entry and we just subtract frame
9159 // size so it is not hard to compute. Note that the compiler
9160 // tries hard to make sure the frame size means RSP will be
9161 // 16-byte aligned, but for leaf functions without locals (i.e.
9162 // frameSize = 0) it will not be.
9163 int frameSize = codeGen->genTotalFrameSize();
9164 return ((8 - frameSize + off) % alignment) == 0;
9167 #endif // FEATURE_SIMD
9173 // Answer the question: Is a particular ISA supported?
9174 // Use this api when asking the question so that future
9175 // ISA questions can be asked correctly or when asserting
9176 // support/nonsupport for an instruction set
9177 bool compIsaSupportedDebugOnly(CORINFO_InstructionSet isa) const
9179 #if defined(TARGET_XARCH) || defined(TARGET_ARM64)
9180 return opts.compSupportsISA.HasInstructionSet(isa);
9187 bool notifyInstructionSetUsage(CORINFO_InstructionSet isa, bool supported) const;
9189 // Answer the question: Is a particular ISA allowed to be used implicitly by optimizations?
9190 // The result of this api call will exactly match the target machine
9191 // on which the function is executed (except for CoreLib, where there are special rules)
9192 bool compExactlyDependsOn(CORINFO_InstructionSet isa) const
9194 #if defined(TARGET_XARCH) || defined(TARGET_ARM64)
9195 if ((opts.compSupportsISAReported.HasInstructionSet(isa)) == false)
9197 if (notifyInstructionSetUsage(isa, (opts.compSupportsISA.HasInstructionSet(isa))))
9198 ((Compiler*)this)->opts.compSupportsISAExactly.AddInstructionSet(isa);
9199 ((Compiler*)this)->opts.compSupportsISAReported.AddInstructionSet(isa);
9201 return (opts.compSupportsISAExactly.HasInstructionSet(isa));
9207 // Answer the question: Is a particular ISA allowed to be used implicitly by optimizations?
9208 // The result of this api call will match the target machine if the result is true.
9209 // If the result is false, then the target machine may have support for the instruction.
9210 bool compOpportunisticallyDependsOn(CORINFO_InstructionSet isa) const
9212 if (opts.compSupportsISA.HasInstructionSet(isa))
9214 return compExactlyDependsOn(isa);
9222 // Answer the question: Is a particular ISA supported for explicit hardware intrinsics?
9223 bool compHWIntrinsicDependsOn(CORINFO_InstructionSet isa) const
9225 // Report intent to use the ISA to the EE
9226 compExactlyDependsOn(isa);
9227 return opts.compSupportsISA.HasInstructionSet(isa);
9231 //------------------------------------------------------------------------
9232 // IsBaselineVector512IsaSupportedDebugOnly - Does isa support exist for Vector512.
9235 // `true` if AVX512F, AVX512BW, AVX512CD, AVX512DQ, and AVX512VL are supported.
9237 bool IsBaselineVector512IsaSupportedDebugOnly() const
9240 return compIsaSupportedDebugOnly(InstructionSet_AVX512F);
9247 //------------------------------------------------------------------------
9248 // IsBaselineVector512IsaSupportedOpportunistically - Does opportunistic isa support exist for Vector512.
9251 // `true` if AVX512F, AVX512BW, AVX512CD, AVX512DQ, and AVX512VL are supported.
9253 bool IsBaselineVector512IsaSupportedOpportunistically() const
9256 return compOpportunisticallyDependsOn(InstructionSet_AVX512F);
9263 bool canUseVexEncoding() const
9265 return compOpportunisticallyDependsOn(InstructionSet_AVX);
9268 //------------------------------------------------------------------------
9269 // canUseEvexEncoding - Answer the question: Is Evex encoding supported on this target.
9272 // `true` if Evex encoding is supported, `false` if not.
9274 bool canUseEvexEncoding() const
9276 return compOpportunisticallyDependsOn(InstructionSet_AVX512F);
9279 //------------------------------------------------------------------------
9280 // DoJitStressEvexEncoding- Answer the question: Do we force EVEX encoding.
9283 // `true` if user requests EVEX encoding and it's safe, `false` if not.
9285 bool DoJitStressEvexEncoding() const
9288 // Using JitStressEVEXEncoding flag will force instructions which would
9289 // otherwise use VEX encoding but can be EVEX encoded to use EVEX encoding
9290 // This requires AVX512F, AVX512BW, AVX512CD, AVX512DQ, and AVX512VL support
9292 if (JitConfig.JitStressEvexEncoding() && IsBaselineVector512IsaSupportedOpportunistically())
9294 assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F));
9295 assert(compIsaSupportedDebugOnly(InstructionSet_AVX512F_VL));
9296 assert(compIsaSupportedDebugOnly(InstructionSet_AVX512BW));
9297 assert(compIsaSupportedDebugOnly(InstructionSet_AVX512BW_VL));
9298 assert(compIsaSupportedDebugOnly(InstructionSet_AVX512CD));
9299 assert(compIsaSupportedDebugOnly(InstructionSet_AVX512CD_VL));
9300 assert(compIsaSupportedDebugOnly(InstructionSet_AVX512DQ));
9301 assert(compIsaSupportedDebugOnly(InstructionSet_AVX512DQ_VL));
9309 #endif // TARGET_XARCH
9312 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
9313 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
9317 XX Generic info about the compilation and the method being compiled. XX
9318 XX It is responsible for driving the other phases. XX
9319 XX It is also responsible for all the memory management. XX
9321 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
9322 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
9326 Compiler* InlineeCompiler; // The Compiler instance for the inlinee
9328 InlineResult* compInlineResult; // The result of importing the inlinee method.
9330 bool compDoAggressiveInlining; // If true, mark every method as CORINFO_FLG_FORCEINLINE
9331 bool compJmpOpUsed; // Does the method do a JMP
9332 bool compLongUsed; // Does the method use TYP_LONG
9333 bool compFloatingPointUsed; // Does the method use TYP_FLOAT or TYP_DOUBLE
9334 bool compTailCallUsed; // Does the method do a tailcall
9335 bool compTailPrefixSeen; // Does the method IL have tail. prefix
9336 bool compMayConvertTailCallToLoop; // Does the method have a recursive tail call that we may convert to a loop?
9337 bool compLocallocSeen; // Does the method IL have localloc opcode
9338 bool compLocallocUsed; // Does the method use localloc.
9339 bool compLocallocOptimized; // Does the method have an optimized localloc
9340 bool compQmarkUsed; // Does the method use GT_QMARK/GT_COLON
9341 bool compQmarkRationalized; // Is it allowed to use a GT_QMARK/GT_COLON node.
9342 bool compHasBackwardJump; // Does the method (or some inlinee) have a lexically backwards jump?
9343 bool compHasBackwardJumpInHandler; // Does the method have a lexically backwards jump in a handler?
9344 bool compSwitchedToOptimized; // Codegen initially was Tier0 but jit switched to FullOpts
9345 bool compSwitchedToMinOpts; // Codegen initially was Tier1/FullOpts but jit switched to MinOpts
9346 bool compSuppressedZeroInit; // There are vars with lvSuppressedZeroInit set
9348 // NOTE: These values are only reliable after
9349 // the importing is completely finished.
9352 // State information - which phases have completed?
9353 // These are kept together for easy discoverability
9355 bool bRangeAllowStress;
9356 bool compCodeGenDone;
9357 int64_t compNumStatementLinksTraversed; // # of links traversed while doing debug checks
9358 bool fgNormalizeEHDone; // Has the flowgraph EH normalization phase been done?
9359 size_t compSizeEstimate; // The estimated size of the method as per `gtSetEvalOrder`.
9360 size_t compCycleEstimate; // The estimated cycle count of the method as per `gtSetEvalOrder`
9361 bool compPoisoningAnyImplicitByrefs; // Importer inserted IR before returns to poison implicit byrefs
9365 bool fgLocalVarLivenessDone; // Note that this one is used outside of debug.
9366 bool fgLocalVarLivenessChanged;
9367 bool fgIsDoingEarlyLiveness;
9368 bool fgDidEarlyLiveness;
9369 bool compPostImportationCleanupDone;
9371 bool compRationalIRForm;
9373 bool compUsesThrowHelper; // There is a call to a THROW_HELPER for the compiled method.
9375 bool compGeneratingProlog;
9376 bool compGeneratingEpilog;
9377 bool compGeneratingUnwindProlog;
9378 bool compGeneratingUnwindEpilog;
9379 bool compNeedsGSSecurityCookie; // There is an unsafe buffer (or localloc) on the stack.
9380 // Insert cookie on frame and code to check the cookie, like VC++ -GS.
9381 bool compGSReorderStackLayout; // There is an unsafe buffer on the stack, reorder locals and make local
9382 // copies of susceptible parameters to avoid buffer overrun attacks through locals/params
9383 bool getNeedsGSSecurityCookie() const
9385 return compNeedsGSSecurityCookie;
9387 void setNeedsGSSecurityCookie()
9389 compNeedsGSSecurityCookie = true;
9392 FrameLayoutState lvaDoneFrameLayout; // The highest frame layout state that we've completed. During
9393 // frame layout calculations, this is the level we are currently
9396 //---------------------------- JITing options -----------------------------
9409 JitFlags* jitFlags; // all flags passed from the EE
9411 // The instruction sets that the compiler is allowed to emit.
9412 CORINFO_InstructionSetFlags compSupportsISA;
9413 // The instruction sets that were reported to the VM as being used by the current method. Subset of
9415 CORINFO_InstructionSetFlags compSupportsISAReported;
9416 // The instruction sets that the compiler is allowed to take advantage of implicitly during optimizations.
9417 // Subset of compSupportsISA.
9418 // The instruction sets available in compSupportsISA and not available in compSupportsISAExactly can be only
9419 // used via explicit hardware intrinsics.
9420 CORINFO_InstructionSetFlags compSupportsISAExactly;
9422 void setSupportedISAs(CORINFO_InstructionSetFlags isas)
9424 compSupportsISA = isas;
9427 unsigned compFlags; // method attributes
9428 unsigned instrCount;
9429 unsigned lvRefCount;
9431 codeOptimize compCodeOpt; // what type of code optimizations
9433 #if defined(TARGET_XARCH)
9434 uint32_t preferredVectorByteLength;
9435 #endif // TARGET_XARCH
9437 // optimize maximally and/or favor speed over size?
9439 #define DEFAULT_MIN_OPTS_CODE_SIZE 60000
9440 #define DEFAULT_MIN_OPTS_INSTR_COUNT 20000
9441 #define DEFAULT_MIN_OPTS_BB_COUNT 2000
9442 #define DEFAULT_MIN_OPTS_LV_NUM_COUNT 2000
9443 #define DEFAULT_MIN_OPTS_LV_REF_COUNT 8000
9445 // Maximum number of locals before turning off the inlining
9446 #define MAX_LV_NUM_COUNT_FOR_INLINING 512
9449 bool compMinOptsIsSet;
9451 mutable bool compMinOptsIsUsed;
9453 bool MinOpts() const
9455 assert(compMinOptsIsSet);
9456 compMinOptsIsUsed = true;
9459 bool IsMinOptsSet() const
9461 return compMinOptsIsSet;
9464 bool MinOpts() const
9468 bool IsMinOptsSet() const
9470 return compMinOptsIsSet;
9474 bool OptimizationDisabled() const
9476 return MinOpts() || compDbgCode;
9478 bool OptimizationEnabled() const
9480 return !OptimizationDisabled();
9483 void SetMinOpts(bool val)
9485 assert(!compMinOptsIsUsed);
9486 assert(!compMinOptsIsSet || (compMinOpts == val));
9488 compMinOptsIsSet = true;
9491 // true if the CLFLG_* for an optimization is set.
9492 bool OptEnabled(unsigned optFlag) const
9494 return !!(compFlags & optFlag);
9497 #ifdef FEATURE_READYTORUN
9498 bool IsReadyToRun() const
9500 return jitFlags->IsSet(JitFlags::JIT_FLAG_READYTORUN);
9503 bool IsReadyToRun() const
9509 // Check if the compilation is control-flow guard enabled.
9510 bool IsCFGEnabled() const
9512 #if defined(TARGET_ARM64) || defined(TARGET_AMD64)
9513 // On these platforms we assume the register that the target is
9514 // passed in is preserved by the validator and take care to get the
9515 // target from the register for the call (even in debug mode).
9516 static_assert_no_msg((RBM_VALIDATE_INDIRECT_CALL_TRASH & (1 << REG_VALIDATE_INDIRECT_CALL_ADDR)) == 0);
9517 if (JitConfig.JitForceControlFlowGuard())
9520 return jitFlags->IsSet(JitFlags::JIT_FLAG_ENABLE_CFG);
9522 // The remaining platforms are not supported and would require some
9526 // The ARM32 validator does not preserve any volatile registers
9527 // which means we have to take special care to allocate and use a
9528 // callee-saved register (reloading the target from memory is a
9532 // On x86 some VSD calls disassemble the call site and expect an
9533 // indirect call which is fundamentally incompatible with CFG.
9534 // This would require a different way to pass this information
9541 #ifdef FEATURE_ON_STACK_REPLACEMENT
9544 return jitFlags->IsSet(JitFlags::JIT_FLAG_OSR);
9553 bool IsTier0() const
9555 return jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0);
9558 bool IsInstrumented() const
9560 return jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR);
9563 bool IsInstrumentedAndOptimized() const
9565 return IsInstrumented() && jitFlags->IsSet(JitFlags::JIT_FLAG_BBOPT);
9568 bool DoEarlyBlockMerging() const
9570 if (jitFlags->IsSet(JitFlags::JIT_FLAG_DEBUG_EnC) || jitFlags->IsSet(JitFlags::JIT_FLAG_DEBUG_CODE))
9575 if (jitFlags->IsSet(JitFlags::JIT_FLAG_MIN_OPT) && !jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0))
9583 // true if we should use the PINVOKE_{BEGIN,END} helpers instead of generating
9584 // PInvoke transitions inline. Normally used by R2R, but also used when generating a reverse pinvoke frame, as
9585 // the current logic for frame setup initializes and pushes
9586 // the InlinedCallFrame before performing the Reverse PInvoke transition, which is invalid (as frames cannot
9587 // safely be pushed/popped while the thread is in a preemptive state.).
9588 bool ShouldUsePInvokeHelpers()
9590 return jitFlags->IsSet(JitFlags::JIT_FLAG_USE_PINVOKE_HELPERS) ||
9591 jitFlags->IsSet(JitFlags::JIT_FLAG_REVERSE_PINVOKE);
9594 // true if we should use insert the REVERSE_PINVOKE_{ENTER,EXIT} helpers in the method
9596 bool IsReversePInvoke()
9598 return jitFlags->IsSet(JitFlags::JIT_FLAG_REVERSE_PINVOKE);
9601 bool compScopeInfo; // Generate the LocalVar info ?
9602 bool compDbgCode; // Generate debugger-friendly code?
9603 bool compDbgInfo; // Gather debugging info?
9606 #ifdef PROFILING_SUPPORTED
9607 bool compNoPInvokeInlineCB;
9609 static const bool compNoPInvokeInlineCB;
9613 bool compGcChecks; // Check arguments and return values to ensure they are sane
9616 #if defined(DEBUG) && defined(TARGET_XARCH)
9618 bool compStackCheckOnRet; // Check stack pointer on return to ensure it is correct.
9620 #endif // defined(DEBUG) && defined(TARGET_XARCH)
9622 #if defined(DEBUG) && defined(TARGET_X86)
9624 bool compStackCheckOnCall; // Check stack pointer after call to ensure it is correct. Only for x86.
9626 #endif // defined(DEBUG) && defined(TARGET_X86)
9628 bool compReloc; // Generate relocs for pointers in code, true for all ngen/prejit codegen
9631 #if defined(TARGET_XARCH)
9632 bool compEnablePCRelAddr; // Whether absolute addr be encoded as PC-rel offset by RyuJIT where possible
9636 #ifdef UNIX_AMD64_ABI
9637 // This flag is indicating if there is a need to align the frame.
9638 // On AMD64-Windows, if there are calls, 4 slots for the outgoing ars are allocated, except for
9639 // FastTailCall. This slots makes the frame size non-zero, so alignment logic will be called.
9640 // On AMD64-Unix, there are no such slots. There is a possibility to have calls in the method with frame size of
9641 // 0. The frame alignment logic won't kick in. This flags takes care of the AMD64-Unix case by remembering that
9642 // there are calls and making sure the frame alignment logic is executed.
9643 bool compNeedToAlignFrame;
9644 #endif // UNIX_AMD64_ABI
9646 bool compProcedureSplitting; // Separate cold code from hot code
9648 bool genFPorder; // Preserve FP order (operations are non-commutative)
9649 bool genFPopt; // Can we do frame-pointer-omission optimization?
9650 bool altJit; // True if we are an altjit and are compiling this method
9653 bool optRepeat; // Repeat optimizer phases k times
9656 bool disAsm; // Display native code as it is generated
9657 bool disTesting; // Display BEGIN METHOD/END METHOD anchors for disasm testing
9658 bool dspDiffable; // Makes the Jit Dump 'diff-able' (currently uses same DOTNET_* flag as disDiffable)
9659 bool disDiffable; // Makes the Disassembly code 'diff-able'
9660 bool disAlignment; // Display alignment boundaries in disassembly code
9661 bool disCodeBytes; // Display instruction code bytes in disassembly code
9663 bool compProcedureSplittingEH; // Separate cold code from hot code for functions with EH
9664 bool dspCode; // Display native code generated
9665 bool dspEHTable; // Display the EH table reported to the VM
9666 bool dspDebugInfo; // Display the Debug info reported to the VM
9667 bool dspInstrs; // Display the IL instructions intermixed with the native code output
9668 bool dspLines; // Display source-code lines intermixed with native code output
9669 bool dmpHex; // Display raw bytes in hex of native code output
9670 bool varNames; // Display variables names in native code output
9671 bool disAsmSpilled; // Display native code when any register spilling occurs
9672 bool disasmWithGC; // Display GC info interleaved with disassembly.
9673 bool disAddr; // Display process address next to each instruction in disassembly code
9674 bool disAsm2; // Display native code after it is generated using external disassembler
9675 bool dspOrder; // Display names of each of the methods that we ngen/jit
9676 bool dspUnwind; // Display the unwind info output
9677 bool compLongAddress; // Force using large pseudo instructions for long address
9678 // (IF_LARGEJMP/IF_LARGEADR/IF_LARGLDC)
9679 bool dspGCtbls; // Display the GC tables
9682 // Default numbers used to perform loop alignment. All the numbers are chosen
9683 // based on experimenting with various benchmarks.
9685 // Default minimum loop block weight required to enable loop alignment.
9686 #define DEFAULT_ALIGN_LOOP_MIN_BLOCK_WEIGHT 4
9688 // By default a loop will be aligned at 32B address boundary to get better
9689 // performance as per architecture manuals.
9690 #define DEFAULT_ALIGN_LOOP_BOUNDARY 0x20
9692 // For non-adaptive loop alignment, by default, only align a loop whose size is
9693 // at most 3 times the alignment block size. If the loop is bigger than that, it is most
9694 // likely complicated enough that loop alignment will not impact performance.
9695 #define DEFAULT_MAX_LOOPSIZE_FOR_ALIGN DEFAULT_ALIGN_LOOP_BOUNDARY * 3
9697 // By default only loops with a constant iteration count less than or equal to this will be unrolled
9698 #define DEFAULT_UNROLL_LOOP_MAX_ITERATION_COUNT 4
9701 // Loop alignment variables
9703 // If set, for non-adaptive alignment, ensure loop jmps are not on or cross alignment boundary.
9704 bool compJitAlignLoopForJcc;
9706 // For non-adaptive alignment, minimum loop size (in bytes) for which alignment will be done.
9707 unsigned short compJitAlignLoopMaxCodeSize;
9709 // Minimum weight needed for the first block of a loop to make it a candidate for alignment.
9710 unsigned short compJitAlignLoopMinBlockWeight;
9712 // For non-adaptive alignment, address boundary (power of 2) at which loop alignment should
9713 // be done. By default, 32B.
9714 unsigned short compJitAlignLoopBoundary;
9716 // Padding limit to align a loop.
9717 unsigned short compJitAlignPaddingLimit;
9719 // If set, perform adaptive loop alignment that limits number of padding based on loop size.
9720 bool compJitAlignLoopAdaptive;
9722 // If set, tries to hide alignment instructions behind unconditional jumps.
9723 bool compJitHideAlignBehindJmp;
9725 // If set, tracks the hidden return buffer for struct arg.
9726 bool compJitOptimizeStructHiddenBuffer;
9728 // Iteration limit to unroll a loop.
9729 unsigned short compJitUnrollLoopMaxIterationCount;
9732 bool doLateDisasm; // Run the late disassembler
9733 #endif // LATE_DISASM
9735 #if DUMP_GC_TABLES && !defined(DEBUG)
9736 #pragma message("NOTE: this non-debug build has GC ptr table dumping always enabled!")
9737 static const bool dspGCtbls = true;
9740 #ifdef PROFILING_SUPPORTED
9741 // Whether to emit Enter/Leave/TailCall hooks using a dummy stub (DummyProfilerELTStub()).
9742 // This option helps make the JIT behave as if it is running under a profiler.
9743 bool compJitELTHookEnabled;
9744 #endif // PROFILING_SUPPORTED
9746 #if FEATURE_TAILCALL_OPT
9747 // Whether opportunistic or implicit tail call optimization is enabled.
9748 bool compTailCallOpt;
9749 // Whether optimization of transforming a recursive tail call into a loop is enabled.
9750 bool compTailCallLoopOpt;
9753 #if FEATURE_FASTTAILCALL
9754 // Whether fast tail calls are allowed.
9755 bool compFastTailCalls;
9756 #endif // FEATURE_FASTTAILCALL
9758 #if defined(TARGET_ARM64)
9759 // Decision about whether to save FP/LR registers with callee-saved registers (see
9760 // DOTNET_JitSaveFpLrWithCalleSavedRegisters).
9761 int compJitSaveFpLrWithCalleeSavedRegisters;
9762 #endif // defined(TARGET_ARM64)
9764 #ifdef CONFIGURABLE_ARM_ABI
9765 bool compUseSoftFP = false;
9768 static const bool compUseSoftFP = true;
9769 #else // !ARM_SOFTFP
9770 static const bool compUseSoftFP = false;
9771 #endif // ARM_SOFTFP
9772 #endif // CONFIGURABLE_ARM_ABI
9774 // Collect 64 bit counts for PGO data.
9775 bool compCollect64BitCounts;
9779 static bool s_pAltJitExcludeAssembliesListInitialized;
9780 static AssemblyNamesList2* s_pAltJitExcludeAssembliesList;
9783 static bool s_pJitDisasmIncludeAssembliesListInitialized;
9784 static AssemblyNamesList2* s_pJitDisasmIncludeAssembliesList;
9786 static bool s_pJitFunctionFileInitialized;
9787 static MethodSet* s_pJitMethodSet;
9789 // silence warning of cast to greater size. It is easier to silence than construct code the compiler is happy with, and
9790 // it is safe in this case
9791 #pragma warning(push)
9792 #pragma warning(disable : 4312)
9794 template <typename T>
9797 return (p == ZERO) ? ZERO : (opts.dspDiffable ? T(0xD1FFAB1E) : p);
9800 template <typename T>
9803 return (o == ZERO) ? ZERO : (opts.dspDiffable ? T(0xD1FFAB1E) : o);
9805 #pragma warning(pop)
9807 #pragma warning(push)
9808 #pragma warning(disable : 4312)
9809 template <typename T>
9815 template <typename T>
9820 #pragma warning(pop)
9825 static int dspTreeID(GenTree* tree)
9827 return tree->gtTreeID;
9830 static void printStmtID(Statement* stmt)
9832 assert(stmt != nullptr);
9833 printf(FMT_STMT, stmt->GetID());
9836 static void printTreeID(GenTree* tree)
9838 if (tree == nullptr)
9844 printf("[%06d]", dspTreeID(tree));
9848 const char* devirtualizationDetailToString(CORINFO_DEVIRTUALIZATION_DETAIL detail);
9853 #define STRESS_MODES \
9857 /* "Variations" stress areas which we try to mix up with each other. */ \
9858 /* These should not be exhaustively used as they might */ \
9859 /* hide/trivialize other areas */ \
9862 STRESS_MODE(DBL_ALN) \
9863 STRESS_MODE(LCL_FLDS) \
9864 STRESS_MODE(UNROLL_LOOPS) \
9865 STRESS_MODE(MAKE_CSE) \
9866 STRESS_MODE(LEGACY_INLINE) \
9867 STRESS_MODE(CLONE_EXPR) \
9869 STRESS_MODE(MERGED_RETURNS) \
9870 STRESS_MODE(BB_PROFILE) \
9871 STRESS_MODE(OPT_BOOLS_GC) \
9872 STRESS_MODE(OPT_BOOLS_COMPARE_CHAIN_COST) \
9873 STRESS_MODE(REMORPH_TREES) \
9874 STRESS_MODE(64RSLT_MUL) \
9875 STRESS_MODE(DO_WHILE_LOOPS) \
9876 STRESS_MODE(MIN_OPTS) \
9877 STRESS_MODE(REVERSE_FLAG) /* Will set GTF_REVERSE_OPS whenever we can */ \
9878 STRESS_MODE(TAILCALL) /* Will make the call as a tailcall whenever legal */ \
9879 STRESS_MODE(CATCH_ARG) /* Will spill catch arg */ \
9880 STRESS_MODE(UNSAFE_BUFFER_CHECKS) \
9881 STRESS_MODE(NULL_OBJECT_CHECK) \
9882 STRESS_MODE(RANDOM_INLINE) \
9883 STRESS_MODE(SWITCH_CMP_BR_EXPANSION) \
9884 STRESS_MODE(GENERIC_VARN) \
9885 STRESS_MODE(PROFILER_CALLBACKS) /* Will generate profiler hooks for ELT callbacks */ \
9886 STRESS_MODE(BYREF_PROMOTION) /* Change undoPromotion decisions for byrefs */ \
9887 STRESS_MODE(PROMOTE_FEWER_STRUCTS)/* Don't promote some structs that can be promoted */ \
9888 STRESS_MODE(VN_BUDGET)/* Randomize the VN budget */ \
9889 STRESS_MODE(SSA_INFO) /* Select lower thresholds for "complex" SSA num encoding */ \
9890 STRESS_MODE(SPLIT_TREES_RANDOMLY) /* Split all statements at a random tree */ \
9891 STRESS_MODE(SPLIT_TREES_REMOVE_COMMAS) /* Remove all GT_COMMA nodes */ \
9892 STRESS_MODE(NO_OLD_PROMOTION) /* Do not use old promotion */ \
9893 STRESS_MODE(PHYSICAL_PROMOTION) /* Use physical promotion */ \
9894 STRESS_MODE(PHYSICAL_PROMOTION_COST) \
9896 /* After COUNT_VARN, stress level 2 does all of these all the time */ \
9898 STRESS_MODE(COUNT_VARN) \
9900 /* "Check" stress areas that can be exhaustively used if we */ \
9901 /* dont care about performance at all */ \
9903 STRESS_MODE(FORCE_INLINE) /* Treat every method as AggressiveInlining */ \
9904 STRESS_MODE(CHK_FLOW_UPDATE) \
9905 STRESS_MODE(EMITTER) \
9906 STRESS_MODE(CHK_REIMPORT) \
9907 STRESS_MODE(GENERIC_CHECK) \
9908 STRESS_MODE(IF_CONVERSION_COST) \
9909 STRESS_MODE(IF_CONVERSION_INNER_LOOPS) \
9910 STRESS_MODE(POISON_IMPLICIT_BYREFS) \
9915 #define STRESS_MODE(mode) STRESS_##mode,
9922 static const LPCWSTR s_compStressModeNamesW[STRESS_COUNT + 1];
9923 static const char* s_compStressModeNames[STRESS_COUNT + 1];
9924 BYTE compActiveStressModes[STRESS_COUNT];
9927 #define MAX_STRESS_WEIGHT 100
9929 bool compStressCompile(compStressArea stressArea, unsigned weightPercentage);
9930 bool compStressCompileHelper(compStressArea stressArea, unsigned weightPercentage);
9931 static unsigned compStressAreaHash(compStressArea area);
9935 bool compInlineStress()
9937 return compStressCompile(STRESS_LEGACY_INLINE, 50);
9940 bool compRandomInlineStress()
9942 return compStressCompile(STRESS_RANDOM_INLINE, 50);
9945 bool compPromoteFewerStructs(unsigned lclNum);
9949 bool compTailCallStress()
9952 // Do not stress tailcalls in IL stubs as the runtime creates several IL
9953 // stubs to implement the tailcall mechanism, which would then
9954 // recursively create more IL stubs.
9955 return !opts.jitFlags->IsSet(JitFlags::JIT_FLAG_IL_STUB) &&
9956 (JitConfig.TailcallStress() != 0 || compStressCompile(STRESS_TAILCALL, 5));
9962 const char* compGetTieringName(bool wantShortName = false) const;
9963 const char* compGetPgoSourceName() const;
9964 const char* compGetStressMessage() const;
9966 codeOptimize compCodeOpt() const
9969 // Switching between size & speed has measurable throughput impact
9970 // (3.5% on NGen CoreLib when measured). It used to be enabled for
9971 // DEBUG, but should generate identical code between CHK & RET builds,
9972 // so that's not acceptable.
9973 // TODO-Throughput: Figure out what to do about size vs. speed & throughput.
9974 // Investigate the cause of the throughput regression.
9976 return opts.compCodeOpt;
9978 return BLENDED_CODE;
9982 //--------------------- Info about the procedure --------------------------
9986 COMP_HANDLE compCompHnd;
9987 CORINFO_MODULE_HANDLE compScopeHnd;
9988 CORINFO_CLASS_HANDLE compClassHnd;
9989 CORINFO_METHOD_HANDLE compMethodHnd;
9990 CORINFO_METHOD_INFO* compMethodInfo;
9992 bool hasCircularClassConstraints;
9993 bool hasCircularMethodConstraints;
9995 #if defined(DEBUG) || defined(LATE_DISASM) || DUMP_FLOWGRAPHS || DUMP_GC_TABLES
9997 const char* compMethodName;
9998 const char* compClassName;
9999 const char* compFullName;
10000 double compPerfScore;
10001 int compMethodSuperPMIIndex; // useful when debugging under SuperPMI
10003 #endif // defined(DEBUG) || defined(LATE_DISASM) || DUMP_FLOWGRAPHS
10005 #if defined(DEBUG) || defined(INLINE_DATA)
10006 // Method hash is logically const, but computed
10007 // on first demand.
10008 mutable unsigned compMethodHashPrivate;
10009 unsigned compMethodHash() const;
10010 #endif // defined(DEBUG) || defined(INLINE_DATA)
10012 #ifdef PSEUDORANDOM_NOP_INSERTION
10013 // things for pseudorandom nop insertion
10014 unsigned compChecksum;
10018 // The following holds the FLG_xxxx flags for the method we're compiling.
10019 unsigned compFlags;
10021 // The following holds the class attributes for the method we're compiling.
10022 unsigned compClassAttr;
10024 const BYTE* compCode;
10025 IL_OFFSET compILCodeSize; // The IL code size
10026 IL_OFFSET compILImportSize; // Estimated amount of IL actually imported
10027 IL_OFFSET compILEntry; // The IL entry point (normally 0)
10028 PatchpointInfo* compPatchpointInfo; // Patchpoint data for OSR (normally nullptr)
10029 UNATIVE_OFFSET compNativeCodeSize; // The native code size, after instructions are issued. This
10030 // is less than (compTotalHotCodeSize + compTotalColdCodeSize) only if:
10031 // (1) the code is not hot/cold split, and we issued less code than we expected, or
10032 // (2) the code is hot/cold split, and we issued less code than we expected
10033 // in the cold section (the hot section will always be padded out to compTotalHotCodeSize).
10035 bool compIsStatic : 1; // Is the method static (no 'this' pointer)?
10036 bool compIsVarArgs : 1; // Does the method have varargs parameters?
10037 bool compInitMem : 1; // Is the CORINFO_OPT_INIT_LOCALS bit set in the method info options?
10038 bool compProfilerCallback : 1; // JIT inserted a profiler Enter callback
10039 bool compPublishStubParam : 1; // EAX captured in prolog will be available through an intrinsic
10040 bool compHasNextCallRetAddr : 1; // The NextCallReturnAddress intrinsic is used.
10042 var_types compRetType; // Return type of the method as declared in IL (including SIMD normalization)
10043 var_types compRetNativeType; // Normalized return type as per target arch ABI
10044 unsigned compILargsCount; // Number of arguments (incl. implicit but not hidden)
10045 unsigned compArgsCount; // Number of arguments (incl. implicit and hidden)
10047 #if FEATURE_FASTTAILCALL
10048 unsigned compArgStackSize; // Incoming argument stack size in bytes
10049 #endif // FEATURE_FASTTAILCALL
10051 unsigned compRetBuffArg; // position of hidden return param var (0, 1) (BAD_VAR_NUM means not present);
10052 int compTypeCtxtArg; // position of hidden param for type context for generic code (CORINFO_CALLCONV_PARAMTYPE)
10053 unsigned compThisArg; // position of implicit this pointer param (not to be confused with lvaArg0Var)
10054 unsigned compILlocalsCount; // Number of vars : args + locals (incl. implicit but not hidden)
10055 unsigned compLocalsCount; // Number of vars : args + locals (incl. implicit and hidden)
10056 unsigned compMaxStack;
10057 UNATIVE_OFFSET compTotalHotCodeSize; // Total number of bytes of Hot Code in the method
10058 UNATIVE_OFFSET compTotalColdCodeSize; // Total number of bytes of Cold Code in the method
10060 unsigned compUnmanagedCallCountWithGCTransition; // count of unmanaged calls with GC transition.
10062 CorInfoCallConvExtension compCallConv; // The entry-point calling convention for this method.
10064 unsigned compLvFrameListRoot; // lclNum for the Frame root
10065 unsigned compXcptnsCount; // Number of exception-handling clauses read in the method's IL.
10066 // You should generally use compHndBBtabCount instead: it is the
10067 // current number of EH clauses (after additions like synchronized
10068 // methods and funclets, and removals like unreachable code deletion).
10070 Target::ArgOrder compArgOrder;
10072 bool compMatchedVM; // true if the VM is "matched": either the JIT is a cross-compiler
10073 // and the VM expects that, or the JIT is a "self-host" compiler
10074 // (e.g., x86 hosted targeting x86) and the VM expects that.
10076 /* The following holds IL scope information about local variables.
10079 unsigned compVarScopesCount;
10080 VarScopeDsc* compVarScopes;
10082 /* The following holds information about instr offsets for
10083 * which we need to report IP-mappings
10086 IL_OFFSET* compStmtOffsets; // sorted
10087 unsigned compStmtOffsetsCount;
10088 ICorDebugInfo::BoundaryTypes compStmtOffsetsImplicit;
10090 // Number of class profile probes in this method
10091 unsigned compHandleHistogramProbeCount;
10093 #ifdef TARGET_ARM64
10094 bool compNeedsConsecutiveRegisters;
10100 // Are we running a replay under SuperPMI?
10101 bool RunningSuperPmiReplay() const
10103 return info.compMethodSuperPMIIndex != -1;
10107 ReturnTypeDesc compRetTypeDesc; // ABI return type descriptor for the method
10109 //------------------------------------------------------------------------
10110 // compMethodHasRetVal: Does this method return some kind of value?
10113 // If this method returns a struct via a return buffer, whether that
10114 // buffer's address needs to be returned, otherwise whether signature
10115 // return type is not "TYP_VOID".
10117 bool compMethodHasRetVal() const
10119 return (info.compRetBuffArg != BAD_VAR_NUM) ? compMethodReturnsRetBufAddr() : (info.compRetType != TYP_VOID);
10122 // Returns true if the method being compiled returns RetBuf addr as its return value
10123 bool compMethodReturnsRetBufAddr() const
10125 // There are cases where implicit RetBuf argument should be explicitly returned in a register.
10126 // In such cases the return type is changed to TYP_BYREF and appropriate IR is generated.
10127 // These cases are:
10128 CLANG_FORMAT_COMMENT_ANCHOR;
10129 #ifdef TARGET_AMD64
10130 // 1. on x64 Windows and Unix the address of RetBuf needs to be returned by
10131 // methods with hidden RetBufArg in RAX. In such case GT_RETURN is of TYP_BYREF,
10132 // returning the address of RetBuf.
10133 return (info.compRetBuffArg != BAD_VAR_NUM);
10134 #else // TARGET_AMD64
10135 #ifdef PROFILING_SUPPORTED
10136 // 2. Profiler Leave callback expects the address of retbuf as return value for
10137 // methods with hidden RetBuf argument. impReturnInstruction() when profiler
10138 // callbacks are needed creates GT_RETURN(TYP_BYREF, op1 = Addr of RetBuf) for
10139 // methods with hidden RetBufArg.
10140 if (compIsProfilerHookNeeded())
10142 return (info.compRetBuffArg != BAD_VAR_NUM);
10145 // 3. Windows ARM64 native instance calling convention requires the address of RetBuff
10146 // to be returned in x0.
10147 CLANG_FORMAT_COMMENT_ANCHOR;
10148 #if defined(TARGET_ARM64)
10149 if (TargetOS::IsWindows)
10151 auto callConv = info.compCallConv;
10152 if (callConvIsInstanceMethodCallConv(callConv))
10154 return (info.compRetBuffArg != BAD_VAR_NUM);
10157 #endif // TARGET_ARM64
10158 // 4. x86 unmanaged calling conventions require the address of RetBuff to be returned in eax.
10159 CLANG_FORMAT_COMMENT_ANCHOR;
10160 #if defined(TARGET_X86)
10161 if (info.compCallConv != CorInfoCallConvExtension::Managed)
10163 return (info.compRetBuffArg != BAD_VAR_NUM);
10168 #endif // TARGET_AMD64
10171 //------------------------------------------------------------------------
10172 // compMethodReturnsMultiRegRetType: Does this method return a multi-reg value?
10175 // If this method returns a value in multiple registers, "true", "false"
10178 bool compMethodReturnsMultiRegRetType() const
10180 return compRetTypeDesc.IsMultiRegRetType();
10183 bool compEnregLocals()
10185 return ((opts.compFlags & CLFLG_REGVAR) != 0);
10188 bool compEnregStructLocals()
10190 return (JitConfig.JitEnregStructLocals() != 0);
10193 bool compObjectStackAllocation()
10195 return (JitConfig.JitObjectStackAllocation() != 0);
10198 // Returns true if the method requires a PInvoke prolog and epilog
10199 bool compMethodRequiresPInvokeFrame()
10201 return (info.compUnmanagedCallCountWithGCTransition > 0);
10204 // Returns true if address-exposed user variables should be poisoned with a recognizable value
10205 bool compShouldPoisonFrame()
10207 #ifdef FEATURE_ON_STACK_REPLACEMENT
10211 return !info.compInitMem && opts.compDbgCode;
10214 // Returns true if the jit supports having patchpoints in this method.
10215 // Optionally, get the reason why not.
10216 bool compCanHavePatchpoints(const char** reason = nullptr);
10220 void compDispLocalVars();
10225 class ClassLayoutTable* m_classLayoutTable;
10227 class ClassLayoutTable* typCreateClassLayoutTable();
10228 class ClassLayoutTable* typGetClassLayoutTable();
10231 // Get the layout having the specified layout number.
10232 ClassLayout* typGetLayoutByNum(unsigned layoutNum);
10233 // Get the layout number of the specified layout.
10234 unsigned typGetLayoutNum(ClassLayout* layout);
10235 // Get the layout having the specified size but no class handle.
10236 ClassLayout* typGetBlkLayout(unsigned blockSize);
10237 // Get the number of a layout having the specified size but no class handle.
10238 unsigned typGetBlkLayoutNum(unsigned blockSize);
10239 // Get the layout for the specified class handle.
10240 ClassLayout* typGetObjLayout(CORINFO_CLASS_HANDLE classHandle);
10241 // Get the number of a layout for the specified class handle.
10242 unsigned typGetObjLayoutNum(CORINFO_CLASS_HANDLE classHandle);
10244 var_types TypeHandleToVarType(CORINFO_CLASS_HANDLE handle, ClassLayout** pLayout = nullptr);
10245 var_types TypeHandleToVarType(CorInfoType jitType, CORINFO_CLASS_HANDLE handle, ClassLayout** pLayout = nullptr);
10247 //-------------------------- Global Compiler Data ------------------------------------
10251 static LONG s_compMethodsCount; // to produce unique label names
10256 unsigned compGenTreeID;
10257 unsigned compStatementID;
10258 unsigned compBasicBlockID;
10262 BasicBlock* compCurBB; // the current basic block in process
10263 Statement* compCurStmt; // the current statement in process
10264 GenTree* compCurTree; // the current tree in process
10266 // The following is used to create the 'method JIT info' block.
10267 size_t compInfoBlkSize;
10268 BYTE* compInfoBlkAddr;
10270 EHblkDsc* compHndBBtab; // array of EH data
10271 unsigned compHndBBtabCount; // element count of used elements in EH data array
10272 unsigned compHndBBtabAllocCount; // element count of allocated elements in EH data array
10274 #if defined(TARGET_X86)
10276 //-------------------------------------------------------------------------
10277 // Tracking of region covered by the monitor in synchronized methods
10278 void* syncStartEmitCookie; // the emitter cookie for first instruction after the call to MON_ENTER
10279 void* syncEndEmitCookie; // the emitter cookie for first instruction after the call to MON_EXIT
10281 #endif // !TARGET_X86
10283 Phases mostRecentlyActivePhase; // the most recently active phase
10284 PhaseChecks activePhaseChecks; // the currently active phase checks
10285 PhaseDumps activePhaseDumps; // the currently active phase dumps
10287 //-------------------------------------------------------------------------
10288 // The following keeps track of how many bytes of local frame space we've
10289 // grabbed so far in the current function, and how many argument bytes we
10290 // need to pop when we return.
10293 unsigned compLclFrameSize; // secObject+lclBlk+locals+temps
10295 // Count of callee-saved regs we pushed in the prolog.
10296 // Does not include EBP for isFramePointerUsed() and double-aligned frames.
10297 // In case of Amd64 this doesn't include float regs saved on stack.
10298 unsigned compCalleeRegsPushed;
10300 #if defined(TARGET_XARCH)
10301 // Mask of callee saved float regs on stack.
10302 regMaskTP compCalleeFPRegsSavedMask;
10304 #ifdef TARGET_AMD64
10305 // Quirk for VS debug-launch scenario to work:
10306 // Bytes of padding between save-reg area and locals.
10307 #define VSQUIRK_STACK_PAD (2 * REGSIZE_BYTES)
10308 unsigned compVSQuirkStackPaddingNeeded;
10311 unsigned compArgSize; // total size of arguments in bytes (including register args (lvIsRegArg))
10313 #if defined(TARGET_ARM) || defined(TARGET_RISCV64)
10314 bool compHasSplitParam;
10317 unsigned compMapILargNum(unsigned ILargNum); // map accounting for hidden args
10318 unsigned compMapILvarNum(unsigned ILvarNum); // map accounting for hidden args
10319 unsigned compMap2ILvarNum(unsigned varNum) const; // map accounting for hidden args
10321 #if defined(TARGET_ARM64)
10324 // Frame type (1-5)
10327 // Distance from established (method body) SP to base of callee save area
10328 int calleeSaveSpOffset;
10330 // Amount to subtract from SP before saving (prolog) OR
10331 // to add to SP after restoring (epilog) callee saves
10332 int calleeSaveSpDelta;
10334 // Distance from established SP to where caller's FP was saved
10335 int offsetSpToSavedFp;
10339 //-------------------------------------------------------------------------
10341 static void compStartup(); // One-time initialization
10342 static void compShutdown(); // One-time finalization
10344 void compInit(ArenaAllocator* pAlloc,
10345 CORINFO_METHOD_HANDLE methodHnd,
10346 COMP_HANDLE compHnd,
10347 CORINFO_METHOD_INFO* methodInfo,
10348 InlineInfo* inlineInfo);
10351 static void compDisplayStaticSizes();
10353 //------------ Some utility functions --------------
10355 void* compGetHelperFtn(CorInfoHelpFunc ftnNum, /* IN */
10356 void** ppIndirection); /* OUT */
10358 // Several JIT/EE interface functions return a CorInfoType, and also return a
10359 // class handle as an out parameter if the type is a value class. Returns the
10360 // size of the type these describe.
10361 unsigned compGetTypeSize(CorInfoType cit, CORINFO_CLASS_HANDLE clsHnd);
10364 // Components used by the compiler may write unit test suites, and
10365 // have them run within this method. They will be run only once per process, and only
10366 // in debug. (Perhaps should be under the control of a DOTNET_ flag.)
10367 // These should fail by asserting.
10368 void compDoComponentUnitTestsOnce();
10371 int compCompile(CORINFO_MODULE_HANDLE classPtr,
10372 void** methodCodePtr,
10373 uint32_t* methodCodeSize,
10374 JitFlags* compileFlags);
10375 void compCompileFinish();
10376 int compCompileHelper(CORINFO_MODULE_HANDLE classPtr,
10377 COMP_HANDLE compHnd,
10378 CORINFO_METHOD_INFO* methodInfo,
10379 void** methodCodePtr,
10380 uint32_t* methodCodeSize,
10381 JitFlags* compileFlag);
10383 ArenaAllocator* compGetArenaAllocator();
10385 void generatePatchpointInfo();
10387 #if MEASURE_MEM_ALLOC
10388 static bool s_dspMemStats; // Display per-phase memory statistics for every function
10389 #endif // MEASURE_MEM_ALLOC
10391 #if LOOP_HOIST_STATS
10392 unsigned m_loopsConsidered;
10393 bool m_curLoopHasHoistedExpression;
10394 unsigned m_loopsWithHoistedExpressions;
10395 unsigned m_totalHoistedExpressions;
10397 void AddLoopHoistStats();
10398 void PrintPerMethodLoopHoistStats();
10400 static CritSecObject s_loopHoistStatsLock; // This lock protects the data structures below.
10401 static unsigned s_loopsConsidered;
10402 static unsigned s_loopsWithHoistedExpressions;
10403 static unsigned s_totalHoistedExpressions;
10405 static void PrintAggregateLoopHoistStats(FILE* f);
10406 #endif // LOOP_HOIST_STATS
10408 #if TRACK_ENREG_STATS
10409 class EnregisterStats
10412 unsigned m_totalNumberOfVars;
10413 unsigned m_totalNumberOfStructVars;
10414 unsigned m_totalNumberOfEnregVars;
10415 unsigned m_totalNumberOfStructEnregVars;
10417 unsigned m_addrExposed;
10418 unsigned m_hiddenStructArg;
10419 unsigned m_VMNeedsStackAddr;
10420 unsigned m_localField;
10421 unsigned m_blockOp;
10422 unsigned m_dontEnregStructs;
10423 unsigned m_notRegSizeStruct;
10424 unsigned m_structArg;
10425 unsigned m_lclAddrNode;
10426 unsigned m_castTakesAddr;
10427 unsigned m_storeBlkSrc;
10428 unsigned m_swizzleArg;
10429 unsigned m_blockOpRet;
10430 unsigned m_returnSpCheck;
10431 unsigned m_callSpCheck;
10432 unsigned m_simdUserForcesDep;
10433 unsigned m_liveInOutHndlr;
10434 unsigned m_depField;
10435 unsigned m_noRegVars;
10436 unsigned m_minOptsGC;
10437 #ifdef JIT32_GCENCODER
10438 unsigned m_PinningRef;
10439 #endif // JIT32_GCENCODER
10440 #if !defined(TARGET_64BIT)
10441 unsigned m_longParamField;
10442 #endif // !TARGET_64BIT
10443 unsigned m_parentExposed;
10444 unsigned m_tooConservative;
10445 unsigned m_escapeAddress;
10446 unsigned m_osrExposed;
10447 unsigned m_stressLclFld;
10448 unsigned m_dispatchRetBuf;
10449 unsigned m_wideIndir;
10450 unsigned m_stressPoisonImplicitByrefs;
10451 unsigned m_externallyVisibleImplicitly;
10454 void RecordLocal(const LclVarDsc* varDsc);
10455 void Dump(FILE* fout) const;
10458 static EnregisterStats s_enregisterStats;
10459 #endif // TRACK_ENREG_STATS
10461 bool compIsForInlining() const;
10462 bool compDonotInline();
10465 // Get the default fill char value we randomize this value when JitStress is enabled.
10466 static unsigned char compGetJitDefaultFill(Compiler* comp);
10468 const char* compLocalVarName(unsigned varNum, unsigned offs);
10469 VarName compVarName(regNumber reg, bool isFloatReg = false);
10470 const char* compFPregVarName(unsigned fpReg, bool displayVar = false);
10471 void compDspSrcLinesByNativeIP(UNATIVE_OFFSET curIP);
10472 void compDspSrcLinesByLineNum(unsigned line, bool seek = false);
10474 const char* compRegNameForSize(regNumber reg, size_t size);
10475 const char* compRegVarName(regNumber reg, bool displayVar = false, bool isFloatReg = false);
10477 //-------------------------------------------------------------------------
10479 struct VarScopeListNode
10482 VarScopeListNode* next;
10483 static VarScopeListNode* Create(VarScopeDsc* value, CompAllocator alloc)
10485 VarScopeListNode* node = new (alloc) VarScopeListNode;
10486 node->data = value;
10487 node->next = nullptr;
10492 struct VarScopeMapInfo
10494 VarScopeListNode* head;
10495 VarScopeListNode* tail;
10496 static VarScopeMapInfo* Create(VarScopeListNode* node, CompAllocator alloc)
10498 VarScopeMapInfo* info = new (alloc) VarScopeMapInfo;
10505 // Max value of scope count for which we would use linear search; for larger values we would use hashtable lookup.
10506 static const unsigned MAX_LINEAR_FIND_LCL_SCOPELIST = 32;
10508 typedef JitHashTable<unsigned, JitSmallPrimitiveKeyFuncs<unsigned>, VarScopeMapInfo*> VarNumToScopeDscMap;
10510 // Map to keep variables' scope indexed by varNum containing it's scope dscs at the index.
10511 VarNumToScopeDscMap* compVarScopeMap;
10513 VarScopeDsc* compFindLocalVar(unsigned varNum, unsigned lifeBeg, unsigned lifeEnd);
10515 VarScopeDsc* compFindLocalVar(unsigned varNum, unsigned offs);
10517 VarScopeDsc* compFindLocalVarLinear(unsigned varNum, unsigned offs);
10519 void compInitVarScopeMap();
10521 VarScopeDsc** compEnterScopeList; // List has the offsets where variables
10522 // enter scope, sorted by instr offset
10523 unsigned compNextEnterScope;
10525 VarScopeDsc** compExitScopeList; // List has the offsets where variables
10526 // go out of scope, sorted by instr offset
10527 unsigned compNextExitScope;
10529 void compInitScopeLists();
10531 void compResetScopeLists();
10533 VarScopeDsc* compGetNextEnterScope(unsigned offs, bool scan = false);
10535 VarScopeDsc* compGetNextExitScope(unsigned offs, bool scan = false);
10537 void compProcessScopesUntil(unsigned offset,
10538 VARSET_TP* inScope,
10539 void (Compiler::*enterScopeFn)(VARSET_TP* inScope, VarScopeDsc*),
10540 void (Compiler::*exitScopeFn)(VARSET_TP* inScope, VarScopeDsc*));
10543 void compDispScopeLists();
10546 bool compIsProfilerHookNeeded() const;
10548 //-------------------------------------------------------------------------
10549 /* Statistical Data Gathering */
10551 void compJitStats(); // call this function and enable
10552 // various ifdef's below for statistical data
10555 void compCallArgStats();
10556 static void compDispCallArgStats(FILE* fout);
10559 //-------------------------------------------------------------------------
10566 ArenaAllocator* compArenaAllocator;
10569 void compFunctionTraceStart();
10570 void compFunctionTraceEnd(void* methodCodePtr, ULONG methodCodeSize, bool isNYI);
10573 size_t compMaxUncheckedOffsetForNullObject;
10575 void compInitOptions(JitFlags* compileFlags);
10577 void compSetProcessor();
10578 void compInitDebuggingInfo();
10579 void compSetOptimizationLevel();
10580 #if defined(TARGET_ARMARCH) || defined(TARGET_RISCV64)
10581 bool compRsvdRegCheck(FrameLayoutState curState);
10583 void compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFlags* compileFlags);
10585 // Clear annotations produced during optimizations; to be used between iterations when repeating opts.
10586 void ResetOptAnnotations();
10588 // Regenerate loop descriptors; to be used between iterations when repeating opts.
10589 void RecomputeLoopInfo();
10591 #ifdef PROFILING_SUPPORTED
10592 // Data required for generating profiler Enter/Leave/TailCall hooks
10594 bool compProfilerHookNeeded; // Whether profiler Enter/Leave/TailCall hook needs to be generated for the method
10595 void* compProfilerMethHnd; // Profiler handle of the method being compiled. Passed as param to ELT callbacks
10596 bool compProfilerMethHndIndirected; // Whether compProfilerHandle is pointer to the handle or is an actual handle
10600 // Assumes called as part of process shutdown; does any compiler-specific work associated with that.
10601 static void ProcessShutdownWork(ICorStaticInfo* statInfo);
10603 CompAllocator getAllocator(CompMemKind cmk = CMK_Generic)
10605 return CompAllocator(compArenaAllocator, cmk);
10608 CompAllocator getAllocatorGC()
10610 return getAllocator(CMK_GC);
10613 CompAllocator getAllocatorLoopHoist()
10615 return getAllocator(CMK_LoopHoist);
10619 CompAllocator getAllocatorDebugOnly()
10621 return getAllocator(CMK_DebugOnly);
10626 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10627 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10629 XX IL verification stuff XX
10632 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10633 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10637 EntryState verCurrentState;
10639 void verInitBBEntryState(BasicBlock* block, EntryState* currentState);
10641 void verInitCurrentState();
10642 void verResetCurrentState(BasicBlock* block, EntryState* currentState);
10644 void verConvertBBToThrowVerificationException(BasicBlock* block DEBUGARG(bool logMsg));
10645 void verHandleVerificationFailure(BasicBlock* block DEBUGARG(bool logMsg));
10646 typeInfo verMakeTypeInfoForLocal(unsigned lclNum);
10647 typeInfo verMakeTypeInfo(CORINFO_CLASS_HANDLE clsHnd); // converts from jit type representation to typeInfo
10648 typeInfo verMakeTypeInfo(CorInfoType ciType,
10649 CORINFO_CLASS_HANDLE clsHnd); // converts from jit type representation to typeInfo
10651 typeInfo verParseArgSigToTypeInfo(CORINFO_SIG_INFO* sig, CORINFO_ARG_LIST_HANDLE args);
10653 bool verCheckTailCallConstraint(OPCODE opcode,
10654 CORINFO_RESOLVED_TOKEN* pResolvedToken,
10655 CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken);
10659 // One line log function. Default level is 0. Increasing it gives you
10660 // more log information
10662 // levels are currently unused: #define JITDUMP(level,...) ();
10663 void JitLogEE(unsigned level, const char* fmt, ...);
10665 bool compDebugBreak;
10667 bool compJitHaltMethod();
10669 void dumpRegMask(regMaskTP regs) const;
10674 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10675 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10677 XX GS Security checks for unsafe buffers XX
10679 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10680 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
10683 struct ShadowParamVarInfo
10685 FixedBitVect* assignGroup; // the closure set of variables whose values depend on each other
10686 unsigned shadowCopy; // Lcl var num, if not valid set to BAD_VAR_NUM
10688 static bool mayNeedShadowCopy(LclVarDsc* varDsc)
10690 #if defined(TARGET_AMD64)
10691 // GS cookie logic to create shadow slots, create trees to copy reg args to shadow
10692 // slots and update all trees to refer to shadow slots is done immediately after
10693 // fgMorph(). Lsra could potentially mark a param as DoNotEnregister after JIT determines
10694 // not to shadow a parameter. Also, LSRA could potentially spill a param which is passed
10695 // in register. Therefore, conservatively all params may need a shadow copy. Note that
10696 // GS cookie logic further checks whether the param is a ptr or an unsafe buffer before
10697 // creating a shadow slot even though this routine returns true.
10699 // TODO-AMD64-CQ: Revisit this conservative approach as it could create more shadow slots than
10700 // required. There are two cases under which a reg arg could potentially be used from its
10702 // a) LSRA marks it as DoNotEnregister (see LinearScan::identifyCandidates())
10703 // b) LSRA spills it
10705 // Possible solution to address case (a)
10706 // - The conditions under which LSRA marks a varDsc as DoNotEnregister could be checked
10707 // in this routine. Note that live out of exception handler is something we may not be
10708 // able to do it here since GS cookie logic is invoked ahead of liveness computation.
10709 // Therefore, for methods with exception handling and need GS cookie check we might have
10710 // to take conservative approach.
10712 // Possible solution to address case (b)
10713 // - Whenever a parameter passed in an argument register needs to be spilled by LSRA, we
10714 // create a new spill temp if the method needs GS cookie check.
10715 return varDsc->lvIsParam;
10716 #else // !defined(TARGET_AMD64)
10717 return varDsc->lvIsParam && !varDsc->lvIsRegArg;
10724 printf("assignGroup [%p]; shadowCopy: [%d];\n", assignGroup, shadowCopy);
10729 GSCookie* gsGlobalSecurityCookieAddr; // Address of global cookie for unsafe buffer checks
10730 GSCookie gsGlobalSecurityCookieVal; // Value of global cookie if addr is NULL
10731 ShadowParamVarInfo* gsShadowVarInfo; // Table used by shadow param analysis code
10733 PhaseStatus gsPhase();
10734 void gsGSChecksInitCookie(); // Grabs cookie variable
10735 void gsCopyShadowParams(); // Identify vulnerable params and create dhadow copies
10736 bool gsFindVulnerableParams(); // Shadow param analysis code
10737 void gsParamsToShadows(); // Insert copy code and replave param uses by shadow
10739 static fgWalkPreFn gsMarkPtrsAndAssignGroups; // Shadow param analysis tree-walk
10740 static fgWalkPreFn gsReplaceShadowParams; // Shadow param replacement tree-walk
10742 #define DEFAULT_MAX_INLINE_SIZE 100 // Methods with > DEFAULT_MAX_INLINE_SIZE IL bytes will never be inlined.
10743 // This can be overwritten by setting DOTNET_JITInlineSize env variable.
10745 #define DEFAULT_MAX_INLINE_DEPTH 20 // Methods at more than this level deep will not be inlined
10747 #define DEFAULT_MAX_FORCE_INLINE_DEPTH 1 // Methods at more than this level deep will not be force inlined
10749 #define DEFAULT_MAX_LOCALLOC_TO_LOCAL_SIZE 32 // fixed locallocs of this size or smaller will convert to local buffers
10752 #ifdef FEATURE_JIT_METHOD_PERF
10753 JitTimer* pCompJitTimer; // Timer data structure (by phases) for current compilation.
10754 static CompTimeSummaryInfo s_compJitTimerSummary; // Summary of the Timer information for the whole run.
10756 static LPCWSTR JitTimeLogCsv(); // Retrieve the file name for CSV from ConfigDWORD.
10757 static LPCWSTR compJitTimeLogFilename; // If a log file for JIT time is desired, filename to write it to.
10759 void BeginPhase(Phases phase); // Indicate the start of the given phase.
10760 void EndPhase(Phases phase); // Indicate the end of the given phase.
10762 #if MEASURE_CLRAPI_CALLS
10763 // Thin wrappers that call into JitTimer (if present).
10764 inline void CLRApiCallEnter(unsigned apix);
10765 inline void CLRApiCallLeave(unsigned apix);
10768 inline void CLR_API_Enter(API_ICorJitInfo_Names ename);
10769 inline void CLR_API_Leave(API_ICorJitInfo_Names ename);
10774 #if defined(DEBUG) || defined(INLINE_DATA)
10775 // These variables are associated with maintaining SQM data about compile time.
10776 unsigned __int64 m_compCyclesAtEndOfInlining; // The thread-virtualized cycle count at the end of the inlining phase
10777 // in the current compilation.
10778 unsigned __int64 m_compCycles; // Net cycle count for current compilation
10779 DWORD m_compTickCountAtEndOfInlining; // The result of GetTickCount() (# ms since some epoch marker) at the end of
10780 // the inlining phase in the current compilation.
10781 #endif // defined(DEBUG) || defined(INLINE_DATA)
10783 // Records the SQM-relevant (cycles and tick count). Should be called after inlining is complete.
10784 // (We do this after inlining because this marks the last point at which the JIT is likely to cause
10785 // type-loading and class initialization).
10786 void RecordStateAtEndOfInlining();
10787 // Assumes being called at the end of compilation. Update the SQM state.
10788 void RecordStateAtEndOfCompilation();
10791 #if FUNC_INFO_LOGGING
10792 static LPCWSTR compJitFuncInfoFilename; // If a log file for per-function information is required, this is the
10793 // filename to write it to.
10794 static FILE* compJitFuncInfoFile; // And this is the actual FILE* to write to.
10795 #endif // FUNC_INFO_LOGGING
10797 Compiler* prevCompiler; // Previous compiler on stack for TLS Compiler* linked list for reentrant compilers.
10800 void RecordNowayAssert(const char* filename, unsigned line, const char* condStr);
10801 #endif // MEASURE_NOWAY
10803 #ifndef FEATURE_TRACELOGGING
10804 // Should we actually fire the noway assert body and the exception handler?
10805 bool compShouldThrowOnNoway();
10806 #else // FEATURE_TRACELOGGING
10807 // Should we actually fire the noway assert body and the exception handler?
10808 bool compShouldThrowOnNoway(const char* filename, unsigned line);
10810 // Telemetry instance to use per method compilation.
10811 JitTelemetry compJitTelemetry;
10813 // Get common parameters that have to be logged with most telemetry data.
10814 void compGetTelemetryDefaults(const char** assemblyName,
10815 const char** scopeName,
10816 const char** methodName,
10817 unsigned* methodHash);
10818 #endif // !FEATURE_TRACELOGGING
10822 NodeToTestDataMap* m_nodeTestData;
10824 static const unsigned FIRST_LOOP_HOIST_CSE_CLASS = 1000;
10825 unsigned m_loopHoistCSEClass; // LoopHoist test annotations turn into CSE requirements; we
10826 // label them with CSE Class #'s starting at FIRST_LOOP_HOIST_CSE_CLASS.
10827 // Current kept in this.
10829 NodeToTestDataMap* GetNodeTestData()
10831 Compiler* compRoot = impInlineRoot();
10832 if (compRoot->m_nodeTestData == nullptr)
10834 compRoot->m_nodeTestData = new (getAllocatorDebugOnly()) NodeToTestDataMap(getAllocatorDebugOnly());
10836 return compRoot->m_nodeTestData;
10839 typedef JitHashTable<GenTree*, JitPtrKeyFuncs<GenTree>, int> NodeToIntMap;
10841 // Returns the set (i.e., the domain of the result map) of nodes that are keys in m_nodeTestData, and
10842 // currently occur in the AST graph.
10843 NodeToIntMap* FindReachableNodesInNodeTestData();
10845 // Node "from" is being eliminated, and being replaced by node "to". If "from" had any associated
10846 // test data, associate that data with "to".
10847 void TransferTestDataToNode(GenTree* from, GenTree* to);
10849 // These are the methods that test that the various conditions implied by the
10850 // test attributes are satisfied.
10851 void JitTestCheckSSA(); // SSA builder tests.
10852 void JitTestCheckVN(); // Value numbering tests.
10855 FieldSeqStore* m_fieldSeqStore;
10857 FieldSeqStore* GetFieldSeqStore()
10859 Compiler* compRoot = impInlineRoot();
10860 if (compRoot->m_fieldSeqStore == nullptr)
10862 CompAllocator alloc = getAllocator(CMK_FieldSeqStore);
10863 compRoot->m_fieldSeqStore = new (alloc) FieldSeqStore(alloc);
10865 return compRoot->m_fieldSeqStore;
10868 typedef JitHashTable<GenTree*, JitPtrKeyFuncs<GenTree>, unsigned> NodeToUnsignedMap;
10870 NodeToUnsignedMap* m_memorySsaMap[MemoryKindCount];
10872 // In some cases, we want to assign intermediate SSA #'s to memory states, and know what nodes create those memory
10873 // states. (We do this for try blocks, where, if the try block doesn't do a call that loses track of the memory
10874 // state, all the possible memory states are possible initial states of the corresponding catch block(s).)
10875 NodeToUnsignedMap* GetMemorySsaMap(MemoryKind memoryKind)
10877 if (memoryKind == GcHeap && byrefStatesMatchGcHeapStates)
10879 // Use the same map for GCHeap and ByrefExposed when their states match.
10880 memoryKind = ByrefExposed;
10883 assert(memoryKind < MemoryKindCount);
10884 Compiler* compRoot = impInlineRoot();
10885 if (compRoot->m_memorySsaMap[memoryKind] == nullptr)
10887 // Create a CompAllocator that labels sub-structure with CMK_MemorySsaMap, and use that for allocation.
10888 CompAllocator ialloc(getAllocator(CMK_MemorySsaMap));
10889 compRoot->m_memorySsaMap[memoryKind] = new (ialloc) NodeToUnsignedMap(ialloc);
10891 return compRoot->m_memorySsaMap[memoryKind];
10894 // The Refany type is the only struct type whose structure is implicitly assumed by IL. We need its fields.
10895 CORINFO_CLASS_HANDLE m_refAnyClass;
10896 CORINFO_FIELD_HANDLE GetRefanyDataField()
10898 if (m_refAnyClass == nullptr)
10900 m_refAnyClass = info.compCompHnd->getBuiltinClass(CLASSID_TYPED_BYREF);
10902 return info.compCompHnd->getFieldInClass(m_refAnyClass, 0);
10904 CORINFO_FIELD_HANDLE GetRefanyTypeField()
10906 if (m_refAnyClass == nullptr)
10908 m_refAnyClass = info.compCompHnd->getBuiltinClass(CLASSID_TYPED_BYREF);
10910 return info.compCompHnd->getFieldInClass(m_refAnyClass, 1);
10913 #if VARSET_COUNTOPS
10914 static BitSetSupport::BitSetOpCounter m_varsetOpCounter;
10916 #if ALLVARSET_COUNTOPS
10917 static BitSetSupport::BitSetOpCounter m_allvarsetOpCounter;
10920 static HelperCallProperties s_helperCallProperties;
10922 #ifdef UNIX_AMD64_ABI
10923 static var_types GetTypeFromClassificationAndSizes(SystemVClassificationType classType, int size);
10924 static var_types GetEightByteType(const SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR& structDesc,
10927 static void GetStructTypeOffset(const SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR& structDesc,
10930 unsigned __int8* offset0,
10931 unsigned __int8* offset1);
10933 void GetStructTypeOffset(CORINFO_CLASS_HANDLE typeHnd,
10936 unsigned __int8* offset0,
10937 unsigned __int8* offset1);
10939 #endif // defined(UNIX_AMD64_ABI)
10941 void fgMorphMultiregStructArgs(GenTreeCall* call);
10942 GenTree* fgMorphMultiregStructArg(CallArg* arg);
10944 bool killGCRefs(GenTree* tree);
10946 #if defined(TARGET_AMD64)
10948 // The following are for initializing register allocator "constants" defined in targetamd64.h
10949 // that now depend upon runtime ISA information, e.g., the presence of AVX512F/VL, which increases
10950 // the number of SIMD (xmm, ymm, and zmm) registers from 16 to 32.
10951 // As only 64-bit xarch has the capability to have the additional registers, we limit the changes
10952 // to TARGET_AMD64 only.
10954 // Users of these values need to define four accessor functions:
10956 // regMaskTP get_RBM_ALLFLOAT();
10957 // regMaskTP get_RBM_FLT_CALLEE_TRASH();
10958 // unsigned get_CNT_CALLEE_TRASH_FLOAT();
10959 // unsigned get_AVAILABLE_REG_COUNT();
10961 // which return the values of these variables.
10963 // This was done to avoid polluting all `targetXXX.h` macro definitions with a compiler parameter, where only
10964 // TARGET_AMD64 requires one.
10966 regMaskTP rbmAllFloat;
10967 regMaskTP rbmFltCalleeTrash;
10968 unsigned cntCalleeTrashFloat;
10971 FORCEINLINE regMaskTP get_RBM_ALLFLOAT() const
10973 return this->rbmAllFloat;
10975 FORCEINLINE regMaskTP get_RBM_FLT_CALLEE_TRASH() const
10977 return this->rbmFltCalleeTrash;
10979 FORCEINLINE unsigned get_CNT_CALLEE_TRASH_FLOAT() const
10981 return this->cntCalleeTrashFloat;
10984 #endif // TARGET_AMD64
10986 #if defined(TARGET_XARCH)
10988 // The following are for initializing register allocator "constants" defined in targetamd64.h
10989 // that now depend upon runtime ISA information, e.g., the presence of AVX512F/VL, which adds
10990 // 8 mask registers for use.
10992 // Users of these values need to define four accessor functions:
10994 // regMaskTP get_RBM_ALLMASK();
10995 // regMaskTP get_RBM_MSK_CALLEE_TRASH();
10996 // unsigned get_CNT_CALLEE_TRASH_MASK();
10997 // unsigned get_AVAILABLE_REG_COUNT();
10999 // which return the values of these variables.
11001 // This was done to avoid polluting all `targetXXX.h` macro definitions with a compiler parameter, where only
11002 // TARGET_XARCH requires one.
11004 regMaskTP rbmAllMask;
11005 regMaskTP rbmMskCalleeTrash;
11006 unsigned cntCalleeTrashMask;
11007 regMaskTP varTypeCalleeTrashRegs[TYP_COUNT];
11010 FORCEINLINE regMaskTP get_RBM_ALLMASK() const
11012 return this->rbmAllMask;
11014 FORCEINLINE regMaskTP get_RBM_MSK_CALLEE_TRASH() const
11016 return this->rbmMskCalleeTrash;
11018 FORCEINLINE unsigned get_CNT_CALLEE_TRASH_MASK() const
11020 return this->cntCalleeTrashMask;
11022 #endif // TARGET_XARCH
11024 }; // end of class Compiler
11026 //---------------------------------------------------------------------------------------------------------------------
11027 // GenTreeVisitor: a flexible tree walker implemented using the curiously-recurring-template pattern.
11029 // This class implements a configurable walker for IR trees. There are five configuration options (defaults values are
11030 // shown in parentheses):
11032 // - ComputeStack (false): when true, the walker will push each node onto the `m_ancestors` stack. "Ancestors" is a bit
11033 // of a misnomer, as the first entry will always be the current node.
11035 // - DoPreOrder (false): when true, the walker will invoke `TVisitor::PreOrderVisit` with the current node as an
11036 // argument before visiting the node's operands.
11038 // - DoPostOrder (false): when true, the walker will invoke `TVisitor::PostOrderVisit` with the current node as an
11039 // argument after visiting the node's operands.
11041 // - DoLclVarsOnly (false): when true, the walker will only invoke `TVisitor::PreOrderVisit` for lclVar nodes.
11042 // `DoPreOrder` must be true if this option is true.
11044 // - UseExecutionOrder (false): when true, then walker will visit a node's operands in execution order (e.g. if a
11045 // binary operator has the `GTF_REVERSE_OPS` flag set, the second operand will be
11046 // visited before the first).
11048 // At least one of `DoPreOrder` and `DoPostOrder` must be specified.
11050 // A simple pre-order visitor might look something like the following:
11052 // class CountingVisitor final : public GenTreeVisitor<CountingVisitor>
11057 // DoPreOrder = true
11060 // unsigned m_count;
11062 // CountingVisitor(Compiler* compiler)
11063 // : GenTreeVisitor<CountingVisitor>(compiler), m_count(0)
11067 // Compiler::fgWalkResult PreOrderVisit(GenTree* node)
11073 // This visitor would then be used like so:
11075 // CountingVisitor countingVisitor(compiler);
11076 // countingVisitor.WalkTree(root);
11078 template <typename TVisitor>
11079 class GenTreeVisitor
11082 typedef Compiler::fgWalkResult fgWalkResult;
11086 ComputeStack = false,
11087 DoPreOrder = false,
11088 DoPostOrder = false,
11089 DoLclVarsOnly = false,
11090 UseExecutionOrder = false,
11093 Compiler* m_compiler;
11094 ArrayStack<GenTree*> m_ancestors;
11096 GenTreeVisitor(Compiler* compiler) : m_compiler(compiler), m_ancestors(compiler->getAllocator(CMK_ArrayStack))
11098 assert(compiler != nullptr);
11100 static_assert_no_msg(TVisitor::DoPreOrder || TVisitor::DoPostOrder);
11101 static_assert_no_msg(!TVisitor::DoLclVarsOnly || TVisitor::DoPreOrder);
11104 fgWalkResult PreOrderVisit(GenTree** use, GenTree* user)
11106 return fgWalkResult::WALK_CONTINUE;
11109 fgWalkResult PostOrderVisit(GenTree** use, GenTree* user)
11111 return fgWalkResult::WALK_CONTINUE;
11115 fgWalkResult WalkTree(GenTree** use, GenTree* user)
11117 assert(use != nullptr);
11119 GenTree* node = *use;
11121 if (TVisitor::ComputeStack)
11123 m_ancestors.Push(node);
11126 fgWalkResult result = fgWalkResult::WALK_CONTINUE;
11127 if (TVisitor::DoPreOrder && !TVisitor::DoLclVarsOnly)
11129 result = reinterpret_cast<TVisitor*>(this)->PreOrderVisit(use, user);
11130 if (result == fgWalkResult::WALK_ABORT)
11136 if ((node == nullptr) || (result == fgWalkResult::WALK_SKIP_SUBTREES))
11142 switch (node->OperGet())
11148 if (TVisitor::DoLclVarsOnly)
11150 result = reinterpret_cast<TVisitor*>(this)->PreOrderVisit(use, user);
11151 if (result == fgWalkResult::WALK_ABORT)
11168 case GT_MEMORYBARRIER:
11173 case GT_START_NONGC:
11174 case GT_START_PREEMPTGC:
11176 #if !defined(FEATURE_EH_FUNCLETS)
11178 #endif // !FEATURE_EH_FUNCLETS
11181 case GT_CLS_VAR_ADDR:
11184 case GT_PINVOKE_PROLOG:
11185 case GT_PINVOKE_EPILOG:
11189 // Lclvar unary operators
11190 case GT_STORE_LCL_VAR:
11191 case GT_STORE_LCL_FLD:
11192 if (TVisitor::DoLclVarsOnly)
11194 result = reinterpret_cast<TVisitor*>(this)->PreOrderVisit(use, user);
11195 if (result == fgWalkResult::WALK_ABORT)
11202 // Standard unary operators
11209 case GT_ARR_LENGTH:
11210 case GT_MDARR_LENGTH:
11211 case GT_MDARR_LOWER_BOUND:
11224 case GT_PUTARG_REG:
11225 case GT_PUTARG_STK:
11226 case GT_RETURNTRAP:
11228 case GT_FIELD_ADDR:
11231 case GT_RUNTIMELOOKUP:
11234 case GT_INC_SATURATE:
11236 GenTreeUnOp* const unOp = node->AsUnOp();
11237 if (unOp->gtOp1 != nullptr)
11239 result = WalkTree(&unOp->gtOp1, unOp);
11240 if (result == fgWalkResult::WALK_ABORT)
11250 for (GenTreePhi::Use& use : node->AsPhi()->Uses())
11252 result = WalkTree(&use.NodeRef(), node);
11253 if (result == fgWalkResult::WALK_ABORT)
11260 case GT_FIELD_LIST:
11261 for (GenTreeFieldList::Use& use : node->AsFieldList()->Uses())
11263 result = WalkTree(&use.NodeRef(), node);
11264 if (result == fgWalkResult::WALK_ABORT)
11273 GenTreeCmpXchg* const cmpXchg = node->AsCmpXchg();
11275 result = WalkTree(&cmpXchg->gtOpLocation, cmpXchg);
11276 if (result == fgWalkResult::WALK_ABORT)
11280 result = WalkTree(&cmpXchg->gtOpValue, cmpXchg);
11281 if (result == fgWalkResult::WALK_ABORT)
11285 result = WalkTree(&cmpXchg->gtOpComparand, cmpXchg);
11286 if (result == fgWalkResult::WALK_ABORT)
11295 GenTreeArrElem* const arrElem = node->AsArrElem();
11297 result = WalkTree(&arrElem->gtArrObj, arrElem);
11298 if (result == fgWalkResult::WALK_ABORT)
11303 const unsigned rank = arrElem->gtArrRank;
11304 for (unsigned dim = 0; dim < rank; dim++)
11306 result = WalkTree(&arrElem->gtArrInds[dim], arrElem);
11307 if (result == fgWalkResult::WALK_ABORT)
11315 case GT_STORE_DYN_BLK:
11317 GenTreeStoreDynBlk* const dynBlock = node->AsStoreDynBlk();
11319 result = WalkTree(&dynBlock->gtOp1, dynBlock);
11320 if (result == fgWalkResult::WALK_ABORT)
11324 result = WalkTree(&dynBlock->gtOp2, dynBlock);
11325 if (result == fgWalkResult::WALK_ABORT)
11329 result = WalkTree(&dynBlock->gtDynamicSize, dynBlock);
11330 if (result == fgWalkResult::WALK_ABORT)
11339 GenTreeCall* const call = node->AsCall();
11341 for (CallArg& arg : call->gtArgs.EarlyArgs())
11343 result = WalkTree(&arg.EarlyNodeRef(), call);
11344 if (result == fgWalkResult::WALK_ABORT)
11350 for (CallArg& arg : call->gtArgs.LateArgs())
11352 result = WalkTree(&arg.LateNodeRef(), call);
11353 if (result == fgWalkResult::WALK_ABORT)
11359 if (call->gtCallType == CT_INDIRECT)
11361 if (call->gtCallCookie != nullptr)
11363 result = WalkTree(&call->gtCallCookie, call);
11364 if (result == fgWalkResult::WALK_ABORT)
11370 result = WalkTree(&call->gtCallAddr, call);
11371 if (result == fgWalkResult::WALK_ABORT)
11377 if (call->gtControlExpr != nullptr)
11379 result = WalkTree(&call->gtControlExpr, call);
11380 if (result == fgWalkResult::WALK_ABORT)
11389 #if defined(FEATURE_HW_INTRINSICS)
11390 case GT_HWINTRINSIC:
11391 if (TVisitor::UseExecutionOrder && node->IsReverseOp())
11393 assert(node->AsMultiOp()->GetOperandCount() == 2);
11394 result = WalkTree(&node->AsMultiOp()->Op(2), node);
11395 if (result == fgWalkResult::WALK_ABORT)
11399 result = WalkTree(&node->AsMultiOp()->Op(1), node);
11400 if (result == fgWalkResult::WALK_ABORT)
11407 for (GenTree** use : node->AsMultiOp()->UseEdges())
11409 result = WalkTree(use, node);
11410 if (result == fgWalkResult::WALK_ABORT)
11417 #endif // defined(FEATURE_HW_INTRINSICS)
11421 GenTreeConditional* const conditional = node->AsConditional();
11423 result = WalkTree(&conditional->gtCond, conditional);
11424 if (result == fgWalkResult::WALK_ABORT)
11428 result = WalkTree(&conditional->gtOp1, conditional);
11429 if (result == fgWalkResult::WALK_ABORT)
11433 result = WalkTree(&conditional->gtOp2, conditional);
11434 if (result == fgWalkResult::WALK_ABORT)
11444 assert(node->OperIsBinary());
11446 GenTreeOp* const op = node->AsOp();
11448 GenTree** op1Use = &op->gtOp1;
11449 GenTree** op2Use = &op->gtOp2;
11451 if (TVisitor::UseExecutionOrder && node->IsReverseOp())
11453 std::swap(op1Use, op2Use);
11456 if (*op1Use != nullptr)
11458 result = WalkTree(op1Use, op);
11459 if (result == fgWalkResult::WALK_ABORT)
11465 if (*op2Use != nullptr)
11467 result = WalkTree(op2Use, op);
11468 if (result == fgWalkResult::WALK_ABORT)
11478 // Finally, visit the current node
11479 if (TVisitor::DoPostOrder)
11481 result = reinterpret_cast<TVisitor*>(this)->PostOrderVisit(use, user);
11484 if (TVisitor::ComputeStack)
11493 template <bool doPreOrder, bool doPostOrder, bool doLclVarsOnly, bool useExecutionOrder>
11494 class GenericTreeWalker final
11495 : public GenTreeVisitor<GenericTreeWalker<doPreOrder, doPostOrder, doLclVarsOnly, useExecutionOrder>>
11500 ComputeStack = false,
11501 DoPreOrder = doPreOrder,
11502 DoPostOrder = doPostOrder,
11503 DoLclVarsOnly = doLclVarsOnly,
11504 UseExecutionOrder = useExecutionOrder,
11508 Compiler::fgWalkData* m_walkData;
11511 GenericTreeWalker(Compiler::fgWalkData* walkData)
11512 : GenTreeVisitor<GenericTreeWalker<doPreOrder, doPostOrder, doLclVarsOnly, useExecutionOrder>>(
11513 walkData->compiler)
11514 , m_walkData(walkData)
11516 assert(walkData != nullptr);
11519 Compiler::fgWalkResult PreOrderVisit(GenTree** use, GenTree* user)
11521 m_walkData->parent = user;
11522 return m_walkData->wtprVisitorFn(use, m_walkData);
11525 Compiler::fgWalkResult PostOrderVisit(GenTree** use, GenTree* user)
11527 m_walkData->parent = user;
11528 return m_walkData->wtpoVisitorFn(use, m_walkData);
11532 // A dominator tree visitor implemented using the curiously-recurring-template pattern, similar to GenTreeVisitor.
11533 template <typename TVisitor>
11534 class DomTreeVisitor
11537 Compiler* const m_compiler;
11538 DomTreeNode* const m_domTree;
11540 DomTreeVisitor(Compiler* compiler, DomTreeNode* domTree) : m_compiler(compiler), m_domTree(domTree)
11548 void PreOrderVisit(BasicBlock* block)
11552 void PostOrderVisit(BasicBlock* block)
11561 //------------------------------------------------------------------------
11562 // WalkTree: Walk the dominator tree, starting from fgFirstBB.
11565 // This performs a non-recursive, non-allocating walk of the tree by using
11566 // DomTreeNode's firstChild and nextSibling links to locate the children of
11567 // a node and BasicBlock's bbIDom parent link to go back up the tree when
11568 // no more children are left.
11570 // Forests are also supported, provided that all the roots are chained via
11571 // DomTreeNode::nextSibling to fgFirstBB.
11575 static_cast<TVisitor*>(this)->Begin();
11577 for (BasicBlock *next, *block = m_compiler->fgFirstBB; block != nullptr; block = next)
11579 static_cast<TVisitor*>(this)->PreOrderVisit(block);
11581 next = m_domTree[block->bbNum].firstChild;
11583 if (next != nullptr)
11585 assert(next->bbIDom == block);
11591 static_cast<TVisitor*>(this)->PostOrderVisit(block);
11593 next = m_domTree[block->bbNum].nextSibling;
11595 if (next != nullptr)
11597 assert(next->bbIDom == block->bbIDom);
11601 block = block->bbIDom;
11603 } while (block != nullptr);
11606 static_cast<TVisitor*>(this)->End();
11610 // EHClauses: adapter class for forward iteration of the exception handling table using range-based `for`, e.g.:
11611 // for (EHblkDsc* const ehDsc : EHClauses(compiler))
11618 // Forward iterator for the exception handling table entries. Iteration is in table order.
11625 iterator(EHblkDsc* ehDsc) : m_ehDsc(ehDsc)
11629 EHblkDsc* operator*() const
11634 iterator& operator++()
11640 bool operator!=(const iterator& i) const
11642 return m_ehDsc != i.m_ehDsc;
11647 EHClauses(Compiler* comp) : m_begin(comp->compHndBBtab), m_end(comp->compHndBBtab + comp->compHndBBtabCount)
11649 assert((m_begin != nullptr) || (m_begin == m_end));
11652 iterator begin() const
11654 return iterator(m_begin);
11657 iterator end() const
11659 return iterator(m_end);
11664 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
11665 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
11667 XX Miscellaneous Compiler stuff XX
11669 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
11670 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
11673 class StringPrinter
11675 CompAllocator m_alloc;
11677 size_t m_bufferMax;
11678 size_t m_bufferIndex = 0;
11680 void Grow(size_t newSize);
11683 StringPrinter(CompAllocator alloc, char* buffer = nullptr, size_t bufferMax = 0)
11684 : m_alloc(alloc), m_buffer(buffer), m_bufferMax(bufferMax)
11686 if ((m_buffer == nullptr) || (m_bufferMax == 0))
11689 m_buffer = alloc.allocate<char>(m_bufferMax);
11692 m_buffer[0] = '\0';
11697 return m_bufferIndex;
11702 assert(m_buffer[GetLength()] == '\0');
11705 void Truncate(size_t newLength)
11707 assert(newLength <= m_bufferIndex);
11708 m_bufferIndex = newLength;
11709 m_buffer[m_bufferIndex] = '\0';
11712 void Append(const char* str);
11713 void Append(char chr);
11716 /*****************************************************************************
11718 * Variables to keep track of total code amounts.
11723 extern size_t grossVMsize;
11724 extern size_t grossNCsize;
11725 extern size_t totalNCsize;
11727 extern unsigned genMethodICnt;
11728 extern unsigned genMethodNCnt;
11729 extern size_t gcHeaderISize;
11730 extern size_t gcPtrMapISize;
11731 extern size_t gcHeaderNSize;
11732 extern size_t gcPtrMapNSize;
11734 #endif // DISPLAY_SIZES
11736 /*****************************************************************************
11738 * Variables to keep track of basic block counts (more data on 1 BB methods)
11741 #if COUNT_BASIC_BLOCKS
11742 extern Histogram bbCntTable;
11743 extern Histogram bbOneBBSizeTable;
11744 extern Histogram domsChangedIterationTable;
11745 extern Histogram computeReachabilitySetsIterationTable;
11746 extern Histogram computeReachabilityIterationTable;
11749 /*****************************************************************************
11751 * Used by optFindNaturalLoops to gather statistical information such as
11752 * - total number of natural loops
11753 * - number of loops with 1, 2, ... exit conditions
11754 * - number of loops that have an iterator (for like)
11755 * - number of loops that have a constant iterator
11760 extern unsigned totalLoopMethods; // counts the total number of methods that have natural loops
11761 extern unsigned maxLoopsPerMethod; // counts the maximum number of loops a method has
11762 extern unsigned totalLoopOverflows; // # of methods that identified more loops than we can represent
11763 extern unsigned totalLoopCount; // counts the total number of natural loops
11764 extern unsigned totalUnnatLoopCount; // counts the total number of (not-necessarily natural) loops
11765 extern unsigned totalUnnatLoopOverflows; // # of methods that identified more unnatural loops than we can represent
11766 extern unsigned iterLoopCount; // counts the # of loops with an iterator (for like)
11767 extern unsigned constIterLoopCount; // counts the # of loops with a constant iterator (for like)
11768 extern bool hasMethodLoops; // flag to keep track if we already counted a method as having loops
11769 extern unsigned loopsThisMethod; // counts the number of loops in the current method
11770 extern bool loopOverflowThisMethod; // True if we exceeded the max # of loops in the method.
11771 extern Histogram loopCountTable; // Histogram of loop counts
11772 extern Histogram loopExitCountTable; // Histogram of loop exit counts
11774 #endif // COUNT_LOOPS
11776 #if MEASURE_BLOCK_SIZE
11777 extern size_t genFlowNodeSize;
11778 extern size_t genFlowNodeCnt;
11779 #endif // MEASURE_BLOCK_SIZE
11781 #if MEASURE_NODE_SIZE
11782 struct NodeSizeStats
11786 genTreeNodeCnt = 0;
11787 genTreeNodeSize = 0;
11788 genTreeNodeActualSize = 0;
11791 // Count of tree nodes allocated.
11792 unsigned __int64 genTreeNodeCnt;
11794 // The size we allocate.
11795 unsigned __int64 genTreeNodeSize;
11797 // The actual size of the node. Note that the actual size will likely be smaller
11798 // than the allocated size, but we sometimes use SetOper()/ChangeOper() to change
11799 // a smaller node to a larger one. TODO-Cleanup: add stats on
11800 // SetOper()/ChangeOper() usage to quantify this.
11801 unsigned __int64 genTreeNodeActualSize;
11803 extern NodeSizeStats genNodeSizeStats; // Total node size stats
11804 extern NodeSizeStats genNodeSizeStatsPerFunc; // Per-function node size stats
11805 extern Histogram genTreeNcntHist;
11806 extern Histogram genTreeNsizHist;
11807 #endif // MEASURE_NODE_SIZE
11809 /*****************************************************************************
11810 * Count fatal errors (including noway_asserts).
11814 extern unsigned fatal_badCode;
11815 extern unsigned fatal_noWay;
11816 extern unsigned fatal_implLimitation;
11817 extern unsigned fatal_NOMEM;
11818 extern unsigned fatal_noWayAssertBody;
11820 extern unsigned fatal_noWayAssertBodyArgs;
11822 extern unsigned fatal_NYI;
11823 #endif // MEASURE_FATAL
11825 /*****************************************************************************
11829 #ifdef TARGET_XARCH
11831 const instruction INS_SHIFT_LEFT_LOGICAL = INS_shl;
11832 const instruction INS_SHIFT_RIGHT_LOGICAL = INS_shr;
11833 const instruction INS_SHIFT_RIGHT_ARITHM = INS_sar;
11835 const instruction INS_AND = INS_and;
11836 const instruction INS_OR = INS_or;
11837 const instruction INS_XOR = INS_xor;
11838 const instruction INS_NEG = INS_neg;
11839 const instruction INS_TEST = INS_test;
11840 const instruction INS_MUL = INS_imul;
11841 const instruction INS_SIGNED_DIVIDE = INS_idiv;
11842 const instruction INS_UNSIGNED_DIVIDE = INS_div;
11843 const instruction INS_BREAKPOINT = INS_int3;
11844 const instruction INS_ADDC = INS_adc;
11845 const instruction INS_SUBC = INS_sbb;
11846 const instruction INS_NOT = INS_not;
11848 #endif // TARGET_XARCH
11852 const instruction INS_SHIFT_LEFT_LOGICAL = INS_lsl;
11853 const instruction INS_SHIFT_RIGHT_LOGICAL = INS_lsr;
11854 const instruction INS_SHIFT_RIGHT_ARITHM = INS_asr;
11856 const instruction INS_AND = INS_and;
11857 const instruction INS_OR = INS_orr;
11858 const instruction INS_XOR = INS_eor;
11859 const instruction INS_NEG = INS_rsb;
11860 const instruction INS_TEST = INS_tst;
11861 const instruction INS_MUL = INS_mul;
11862 const instruction INS_MULADD = INS_mla;
11863 const instruction INS_SIGNED_DIVIDE = INS_sdiv;
11864 const instruction INS_UNSIGNED_DIVIDE = INS_udiv;
11865 const instruction INS_BREAKPOINT = INS_bkpt;
11866 const instruction INS_ADDC = INS_adc;
11867 const instruction INS_SUBC = INS_sbc;
11868 const instruction INS_NOT = INS_mvn;
11870 const instruction INS_ABS = INS_vabs;
11871 const instruction INS_SQRT = INS_vsqrt;
11873 #endif // TARGET_ARM
11875 #ifdef TARGET_ARM64
11877 const instruction INS_MULADD = INS_madd;
11878 inline const instruction INS_BREAKPOINT_osHelper()
11880 // GDB needs the encoding of brk #0
11881 // Windbg needs the encoding of brk #F000
11882 return TargetOS::IsUnix ? INS_brk_unix : INS_brk_windows;
11884 #define INS_BREAKPOINT INS_BREAKPOINT_osHelper()
11886 const instruction INS_ABS = INS_fabs;
11887 const instruction INS_SQRT = INS_fsqrt;
11889 #endif // TARGET_ARM64
11891 #ifdef TARGET_LOONGARCH64
11892 const instruction INS_BREAKPOINT = INS_break;
11893 const instruction INS_MULADD = INS_fmadd_d; // NOTE: default is double.
11894 const instruction INS_ABS = INS_fabs_d; // NOTE: default is double.
11895 const instruction INS_SQRT = INS_fsqrt_d; // NOTE: default is double.
11896 #endif // TARGET_LOONGARCH64
11898 #ifdef TARGET_RISCV64
11899 const instruction INS_BREAKPOINT = INS_ebreak;
11900 #endif // TARGET_RISCV64
11902 /*****************************************************************************/
11904 extern const BYTE genTypeSizes[];
11905 extern const BYTE genTypeAlignments[];
11906 extern const BYTE genTypeStSzs[];
11907 extern const BYTE genActualTypes[];
11909 /*****************************************************************************/
11912 void dumpConvertedVarSet(Compiler* comp, VARSET_VALARG_TP vars);
11915 #include "compiler.hpp" // All the shared inline functions
11917 /*****************************************************************************/
11918 #endif //_COMPILER_H_
11919 /*****************************************************************************/