1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 /*****************************************************************************/
14 #include "jitgcinfo.h"
16 /*****************************************************************************/
18 #pragma warning(disable : 4200) // allow arrays of 0 size inside structs
21 /*****************************************************************************/
26 #define EMITVERBOSE (emitComp->verbose)
30 #define EMIT_GC_VERBOSE 0
32 #define EMIT_GC_VERBOSE (emitComp->verbose)
36 #define EMIT_INSTLIST_VERBOSE 0
38 #define EMIT_INSTLIST_VERBOSE (emitComp->verbose)
42 #define EMIT_BACKWARDS_NAVIGATION 1 // If 1, enable backwards navigation code for MIR (insGroup/instrDesc).
44 #define EMIT_BACKWARDS_NAVIGATION 0
47 /*****************************************************************************/
56 void emitterStats(FILE* fout);
57 void emitterStaticStats(FILE* fout); // Static stats about the emitter (data structure offsets, sizes, etc.)
60 void printRegMaskInt(regMaskTP mask);
62 /*****************************************************************************/
63 /* Forward declarations */
69 typedef void (*emitSplitCallbackType)(void* context, emitLocation* emitLoc);
71 /*****************************************************************************/
73 //-----------------------------------------------------------------------------
75 inline bool needsGC(GCtype gcType)
77 if (gcType == GCT_NONE)
83 assert(gcType == GCT_GCREF || gcType == GCT_BYREF);
88 //-----------------------------------------------------------------------------
92 inline bool IsValidGCtype(GCtype gcType)
94 return (gcType == GCT_NONE || gcType == GCT_GCREF || gcType == GCT_BYREF);
97 // Get a string name to represent the GC type
99 inline const char* GCtypeStr(GCtype gcType)
110 assert(!"Invalid GCtype");
117 /*****************************************************************************/
120 #define INTERESTING_JUMP_NUM -1 // set to 0 to see all jump info
121 //#define INTERESTING_JUMP_NUM 0
124 /*****************************************************************************
126 * Represent an emitter location.
132 emitLocation() : ig(nullptr), codePos(0)
136 emitLocation(insGroup* _ig) : ig(_ig), codePos(0)
140 emitLocation(insGroup* _ig, unsigned _codePos)
142 SetLocation(_ig, _codePos);
145 emitLocation(emitter* emit)
147 CaptureLocation(emit);
150 emitLocation(void* emitCookie) : ig((insGroup*)emitCookie), codePos(0)
154 // A constructor for code that needs to call it explicitly.
157 *this = emitLocation();
160 void CaptureLocation(emitter* emit);
161 void SetLocation(insGroup* _ig, unsigned _codePos);
162 void SetLocation(emitLocation newLocation);
164 bool IsCurrentLocation(emitter* emit) const;
166 // This function is highly suspect, since it presumes knowledge of the codePos "cookie",
167 // and doesn't look at the 'ig' pointer.
168 bool IsOffsetZero() const
170 return (codePos == 0);
173 UNATIVE_OFFSET CodeOffset(emitter* emit) const;
175 insGroup* GetIG() const
180 int GetInsNum() const;
181 int GetInsOffset() const;
183 bool operator!=(const emitLocation& other) const
185 return (ig != other.ig) || (codePos != other.codePos);
188 bool operator==(const emitLocation& other) const
190 return !(*this != other);
195 // Things we could validate:
196 // 1. the instruction group pointer is non-nullptr.
197 // 2. 'ig' is a legal pointer to an instruction group.
198 // 3. 'codePos' is a legal offset into 'ig'.
199 // Currently, we just do #1.
200 // #2 and #3 should only be done in DEBUG, if they are implemented.
210 UNATIVE_OFFSET GetFuncletPrologOffset(emitter* emit) const;
212 bool IsPreviousInsNum(emitter* emit) const;
215 void Print(LONG compMethodID) const;
219 insGroup* ig; // the instruction group
220 unsigned codePos; // the code position within the IG (see emitCurOffset())
223 /************************************************************************/
224 /* The following describes an instruction group */
225 /************************************************************************/
227 enum insGroupPlaceholderType : unsigned char
229 IGPT_PROLOG, // currently unused
231 #if defined(FEATURE_EH_FUNCLETS)
234 #endif // FEATURE_EH_FUNCLETS
237 #if defined(_MSC_VER) && defined(TARGET_ARM)
238 // ARM aligns structures that contain 64-bit ints or doubles on 64-bit boundaries. This causes unwanted
239 // padding to be added to the end, so sizeof() is unnecessarily big.
242 #endif // defined(_MSC_VER) && defined(TARGET_ARM)
244 struct insPlaceholderGroupData
248 VARSET_TP igPhInitGCrefVars;
249 regMaskTP igPhInitGCrefRegs;
250 regMaskTP igPhInitByrefRegs;
251 VARSET_TP igPhPrevGCrefVars;
252 regMaskTP igPhPrevGCrefRegs;
253 regMaskTP igPhPrevByrefRegs;
254 insGroupPlaceholderType igPhType;
255 }; // end of struct insPlaceholderGroupData
261 #if EMIT_BACKWARDS_NAVIGATION
266 insGroup* igSelf; // for consistency checking
268 #if defined(DEBUG) || defined(LATE_DISASM)
269 weight_t igWeight; // the block weight used for this insGroup
270 double igPerfScore; // The PerfScore for this insGroup
274 BasicBlock* lastGeneratedBlock; // The last block that generated code into this insGroup.
275 jitstd::list<BasicBlock*> igBlocks; // All the blocks that generated code into this insGroup.
276 size_t igDataSize; // size of instrDesc data pointed to by 'igData'
279 UNATIVE_OFFSET igNum; // for ordering (and display) purposes
280 UNATIVE_OFFSET igOffs; // offset of this group within method
281 unsigned int igFuncIdx; // Which function/funclet does this belong to? (Index into Compiler::compFuncInfos array.)
282 unsigned short igFlags; // see IGF_xxx below
283 unsigned short igSize; // # of bytes of code in this group
285 #if FEATURE_LOOP_ALIGN
286 insGroup* igLoopBackEdge; // "last" back-edge that branches back to an aligned loop head.
289 #define IGF_GC_VARS 0x0001 // new set of live GC ref variables
290 #define IGF_BYREF_REGS 0x0002 // new set of live by-ref registers
291 #if defined(FEATURE_EH_FUNCLETS) && defined(TARGET_ARM)
292 #define IGF_FINALLY_TARGET 0x0004 // this group is the start of a basic block that is returned to after a finally.
293 #endif // defined(FEATURE_EH_FUNCLETS) && defined(TARGET_ARM)
294 #define IGF_FUNCLET_PROLOG 0x0008 // this group belongs to a funclet prolog
295 #define IGF_FUNCLET_EPILOG 0x0010 // this group belongs to a funclet epilog.
296 #define IGF_EPILOG 0x0020 // this group belongs to a main function epilog
297 #define IGF_NOGCINTERRUPT 0x0040 // this IG is in a no-interrupt region (prolog, epilog, etc.)
298 #define IGF_UPD_ISZ 0x0080 // some instruction sizes updated
299 #define IGF_PLACEHOLDER 0x0100 // this is a placeholder group, to be filled in later
300 #define IGF_EXTEND 0x0200 // this block is conceptually an extension of the previous block
301 // and the emitter should continue to track GC info as if there was no new block.
302 #define IGF_HAS_ALIGN 0x0400 // this group contains an alignment instruction(s) at the end to align either the next
303 // IG, or, if this IG contains with an unconditional branch, some subsequent IG.
304 #define IGF_REMOVED_ALIGN 0x0800 // IG was marked as having an alignment instruction(s), but was later unmarked
305 // without updating the IG's size/offsets.
306 #define IGF_HAS_REMOVABLE_JMP 0x1000 // this group ends with an unconditional jump which is a candidate for removal
308 #define IGF_HAS_REMOVED_INSTR 0x2000 // this group has an instruction that was removed.
311 // Mask of IGF_* flags that should be propagated to new blocks when they are created.
312 // This allows prologs and epilogs to be any number of IGs, but still be
313 // automatically marked properly.
314 #if defined(FEATURE_EH_FUNCLETS)
316 #define IGF_PROPAGATE_MASK (IGF_EPILOG | IGF_FUNCLET_PROLOG | IGF_FUNCLET_EPILOG)
318 #define IGF_PROPAGATE_MASK (IGF_EPILOG | IGF_FUNCLET_PROLOG)
320 #else // !FEATURE_EH_FUNCLETS
321 #define IGF_PROPAGATE_MASK (IGF_EPILOG)
322 #endif // !FEATURE_EH_FUNCLETS
324 // Try to do better packing based on how large regMaskSmall is (8, 16, or 64 bits).
325 CLANG_FORMAT_COMMENT_ANCHOR;
327 #if !(REGMASK_BITS <= 32)
328 regMaskSmall igGCregs; // set of registers with live GC refs
329 #endif // !(REGMASK_BITS <= 32)
332 BYTE* igData; // addr of instruction descriptors
333 insPlaceholderGroupData* igPhData; // when igFlags & IGF_PLACEHOLDER
336 #if EMIT_BACKWARDS_NAVIGATION
337 // Last instruction in group, if any (nullptr if none); used for backwards navigation.
338 // (Should be type emitter::instrDesc*).
340 #endif // EMIT_BACKWARDS_NAVIGATION
342 #if EMIT_TRACK_STACK_DEPTH
343 unsigned igStkLvl; // stack level on entry
344 #endif // EMIT_TRACK_STACK_DEPTH
346 #if REGMASK_BITS <= 32
347 regMaskSmall igGCregs; // set of registers with live GC refs
348 #endif // REGMASK_BITS <= 32
350 unsigned char igInsCnt; // # of instructions in this group
352 VARSET_VALRET_TP igGCvars() const
354 assert(igFlags & IGF_GC_VARS);
356 BYTE* ptr = (BYTE*)igData;
357 ptr -= sizeof(VARSET_TP);
359 return *(VARSET_TP*)ptr;
362 unsigned igByrefRegs() const
364 assert(igFlags & IGF_BYREF_REGS);
366 BYTE* ptr = (BYTE*)igData;
368 if (igFlags & IGF_GC_VARS)
370 ptr -= sizeof(VARSET_TP);
373 ptr -= sizeof(unsigned);
375 return *(unsigned*)ptr;
378 bool endsWithAlignInstr() const
380 return (igFlags & IGF_HAS_ALIGN) != 0;
383 // hadAlignInstr: Checks if this IG was ever marked as aligned and later
384 // decided to not align. Sometimes, a loop is marked as not
385 // needing alignment, but the igSize was not adjusted immediately.
386 // This method is used during loopSize calculation, where we adjust
387 // the loop size by removed alignment bytes.
388 bool hadAlignInstr() const
390 return (igFlags & IGF_REMOVED_ALIGN) != 0;
393 }; // end of struct insGroup
395 // For AMD64 the maximum prolog/epilog size supported on the OS is 256 bytes
396 // Since it is incorrect for us to be jumping across funclet prolog/epilogs
397 // we will use the following estimate as the maximum placeholder size.
399 #define MAX_PLACEHOLDER_IG_SIZE 256
401 #if defined(_MSC_VER) && defined(TARGET_ARM)
403 #endif // defined(_MSC_VER) && defined(TARGET_ARM)
405 /*****************************************************************************/
407 #define DEFINE_ID_OPS
408 #include "emitfmts.h"
413 LVA_STANDARD_ENCODING = 0,
414 LVA_LARGE_OFFSET = 1,
415 LVA_COMPILER_TEMP = 2,
419 struct emitLclVarAddr
422 void initLclVarAddr(int varNum, unsigned offset);
424 int lvaVarNum() const; // Returns the variable to access. Note that it returns a negative number for compiler spill
426 unsigned lvaOffset() const; // returns the offset into the variable to access
428 // This struct should be 32 bits in size for the release build.
429 // We have this constraint because this type is used in a union
430 // with several other pointer sized types in the instrDesc struct.
433 unsigned _lvaVarNum : 15; // Usually the lvaVarNum
434 unsigned _lvaExtra : 15; // Usually the lvaOffset
435 unsigned _lvaTag : 2; // tag field to support larger varnums
440 iaut_ALIGNED_POINTER = 0x0,
441 iaut_DATA_OFFSET = 0x1,
442 iaut_INST_COUNT = 0x2,
443 iaut_UNUSED_TAG = 0x3,
451 friend class emitLocation;
452 friend class Compiler;
453 friend class CodeGen;
454 friend class CodeGenInterface;
457 /*************************************************************************
459 * Define the public entry points.
466 // There seem to be some cases where this is used without being initialized via CodeGen::inst_set_SV_var().
471 SetUseVEXEncoding(false);
472 SetUseEvexEncoding(false);
473 #endif // TARGET_XARCH
475 emitDataSecCur = nullptr;
481 /************************************************************************/
482 /* Miscellaneous stuff */
483 /************************************************************************/
489 size_t m_debugInfoSize;
491 typedef GCInfo::varPtrDsc varPtrDsc;
492 typedef GCInfo::regPtrDsc regPtrDsc;
493 typedef GCInfo::CallDsc callDsc;
495 void* emitGetMem(size_t sz);
497 enum opSize : unsigned
505 #if defined(TARGET_XARCH)
520 #define OPSIZE_INVALID ((opSize)0xffff)
522 static const emitAttr emitSizeDecode[];
524 static emitter::opSize emitEncodeSize(emitAttr size);
525 static emitAttr emitDecodeSize(emitter::opSize ensz);
527 // Currently, we only allow one IG for the prolog
528 bool emitIGisInProlog(const insGroup* ig)
530 return ig == emitPrologIG;
533 bool emitIGisInEpilog(const insGroup* ig)
535 return (ig != nullptr) && ((ig->igFlags & IGF_EPILOG) != 0);
538 #if defined(FEATURE_EH_FUNCLETS)
540 bool emitIGisInFuncletProlog(const insGroup* ig)
542 return (ig != nullptr) && ((ig->igFlags & IGF_FUNCLET_PROLOG) != 0);
545 bool emitIGisInFuncletEpilog(const insGroup* ig)
547 return (ig != nullptr) && ((ig->igFlags & IGF_FUNCLET_EPILOG) != 0);
550 #endif // FEATURE_EH_FUNCLETS
552 // If "ig" corresponds to the start of a basic block that is the
553 // target of a funclet return, generate GC information for it's start
554 // address "cp", as if it were the return address of a call.
555 void emitGenGCInfoIfFuncletRetTarget(insGroup* ig, BYTE* cp);
557 void emitRecomputeIGoffsets();
559 void emitDispCommentForHandle(size_t handle, size_t cookie, GenTreeFlags flags);
561 /************************************************************************/
562 /* The following describes a single instruction */
563 /************************************************************************/
565 enum insFormat : unsigned
567 #define IF_DEF(en, op1, op2) IF_##en,
568 #include "emitfmts.h"
575 #define AM_DISP_BITS ((sizeof(unsigned) * 8) - 2 * (REGNUM_BITS + 1) - 2)
576 #define AM_DISP_BIG_VAL (-(1 << (AM_DISP_BITS - 1)))
577 #define AM_DISP_MIN (-((1 << (AM_DISP_BITS - 1)) - 1))
578 #define AM_DISP_MAX (+((1 << (AM_DISP_BITS - 1)) - 1))
582 regNumber amBaseReg : REGNUM_BITS + 1;
583 regNumber amIndxReg : REGNUM_BITS + 1;
584 emitter::opSize amScale : 2;
585 int amDisp : AM_DISP_BITS;
588 #endif // TARGET_XARCH
590 struct instrDescDebugInfo
593 size_t idSize; // size of the instruction descriptor
594 unsigned idVarRefOffs; // IL offset for LclVar reference
595 unsigned idVarRefOffs2; // IL offset for 2nd LclVar reference (in case this is a pair)
596 size_t idMemCookie; // compile time handle (check idFlags)
597 GenTreeFlags idFlags; // for determining type of handle in idMemCookie
598 bool idFinallyCall; // Branch instruction is a call to finally
599 bool idCatchRet; // Instruction is for a catch 'return'
600 CORINFO_SIG_INFO* idCallSig; // Used to report native call site signatures to the EE
604 unsigned insEncodeSetFlags(insFlags sf);
606 enum insSize : unsigned
610 ISZ_48BIT // pseudo-instruction for conditional branch with imm24 range,
611 // encoded as IT of condition followed by an unconditional branch
614 unsigned insEncodeShiftOpts(insOpts opt);
615 unsigned insEncodePUW_G0(insOpts opt, int imm);
616 unsigned insEncodePUW_H0(insOpts opt, int imm);
625 // The assembly instruction
626 #if defined(TARGET_XARCH)
627 static_assert_no_msg(INS_count <= 1024);
628 instruction _idIns : 10;
629 #define MAX_ENCODED_SIZE 15
630 #elif defined(TARGET_ARM64)
631 #define INSTR_ENCODED_SIZE 4
632 static_assert_no_msg(INS_count <= 512);
633 instruction _idIns : 9;
634 #elif defined(TARGET_LOONGARCH64)
635 // TODO-LoongArch64: not include SIMD-vector.
636 static_assert_no_msg(INS_count <= 512);
637 instruction _idIns : 9;
639 static_assert_no_msg(INS_count <= 256);
640 instruction _idIns : 8;
641 #endif // !(defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64))
643 // The format for the instruction
644 #if defined(TARGET_XARCH)
645 static_assert_no_msg(IF_COUNT <= 128);
646 insFormat _idInsFmt : 7;
647 #elif defined(TARGET_LOONGARCH64)
648 unsigned _idCodeSize : 5; // the instruction(s) size of this instrDesc described.
649 #elif defined(TARGET_RISCV64)
650 unsigned _idCodeSize : 6; // the instruction(s) size of this instrDesc described.
652 static_assert_no_msg(IF_COUNT <= 256);
653 insFormat _idInsFmt : 8;
657 instrDesc() = delete; // Do not stack alloc this due to debug info that has to come before it.
659 instruction idIns() const
663 void idIns(instruction ins)
665 assert((ins != INS_invalid) && (ins < INS_count));
668 bool idInsIs(instruction ins) const
670 return idIns() == ins;
672 template <typename... T>
673 bool idInsIs(instruction ins, T... rest) const
675 return idInsIs(ins) || idInsIs(rest...);
678 #if defined(TARGET_LOONGARCH64)
679 insFormat idInsFmt() const
680 { // not used for LOONGARCH64.
683 void idInsFmt(insFormat insFmt)
686 #elif defined(TARGET_RISCV64)
687 insFormat idInsFmt() const
689 NYI_RISCV64("idInsFmt-----unimplemented on RISCV64 yet----");
692 void idInsFmt(insFormat insFmt)
694 NYI_RISCV64("idInsFmt-----unimplemented on RISCV64 yet----");
697 insFormat idInsFmt() const
701 void idInsFmt(insFormat insFmt)
703 #if defined(TARGET_ARM64)
704 noway_assert(insFmt != IF_NONE); // Only the x86 emitter uses IF_NONE, it is invalid for ARM64 (and ARM32)
706 assert(insFmt < IF_COUNT);
711 ////////////////////////////////////////////////////////////////////////
712 // Space taken up to here:
717 // loongarch64: 14 bits
721 #if defined(TARGET_XARCH)
722 unsigned _idCodeSize : 4; // size of instruction in bytes. Max size of an Intel instruction is 15 bytes.
723 opSize _idOpSize : 3; // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16, 5=32
724 // At this point we have fully consumed first DWORD so that next field
725 // doesn't cross a byte boundary.
726 #elif defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
727 /* _idOpSize defined below. */
729 opSize _idOpSize : 2; // operand size: 0=1 , 1=2 , 2=4 , 3=8
730 #endif // TARGET_ARM64 || TARGET_LOONGARCH64 || TARGET_RISCV64
732 // On Amd64, this is where the second DWORD begins
733 // On System V a call could return a struct in 2 registers. The instrDescCGCA struct below has member that
734 // stores the GC-ness of the second register.
735 // It is added to the instrDescCGCA and not here (the base struct) since it is not needed by all the
736 // instructions. This struct (instrDesc) is very carefully kept to be no more than 128 bytes. There is no more
737 // space to add members for keeping GC-ness of the second return registers. It will also bloat the base struct
738 // unnecessarily since the GC-ness of the second register is only needed for call instructions.
739 // The instrDescCGCA struct's member keeping the GC-ness of the first return register is _idcSecondRetRegGCType.
740 GCtype _idGCref : 2; // GCref operand? (value is a "GCtype")
742 // The idReg1 and idReg2 fields hold the first and second register
743 // operand(s), whenever these are present. Note that currently the
744 // size of these fields is 6 bits on all targets, and care needs to
745 // be taken to make sure all of these fields stay reasonably packed.
747 // Note that we use the _idReg1 and _idReg2 fields to hold
748 // the live gcrefReg mask for the call instructions on x86/x64
750 regNumber _idReg1 : REGNUM_BITS; // register num
751 regNumber _idReg2 : REGNUM_BITS;
753 ////////////////////////////////////////////////////////////////////////
754 // Space taken up to here:
759 // loongarch64: 28 bits
762 unsigned _idSmallDsc : 1; // is this a "small" descriptor?
763 unsigned _idLargeCns : 1; // does a large constant follow?
764 unsigned _idLargeDsp : 1; // does a large displacement follow?
765 unsigned _idLargeCall : 1; // large call descriptor used
767 unsigned _idBound : 1; // jump target / frame offset bound
768 unsigned _idCallRegPtr : 1; // IL indirect calls: addr in reg
769 unsigned _idCallAddr : 1; // IL indirect calls: can make a direct call to iiaAddr
770 unsigned _idNoGC : 1; // Some helpers don't get recorded in GC tables
771 #if defined(TARGET_XARCH)
772 unsigned _idEvexbContext : 1; // does EVEX.b need to be set.
773 #endif // TARGET_XARCH
776 opSize _idOpSize : 3; // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16
777 insOpts _idInsOpt : 6; // options for instructions
778 unsigned _idLclVar : 1; // access a local on stack
779 unsigned _idLclVarPair : 1 // carries information for 2 GC lcl vars.
782 #ifdef TARGET_LOONGARCH64
783 // TODO-LoongArch64: maybe delete on future.
784 opSize _idOpSize : 3; // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16
785 insOpts _idInsOpt : 6; // loongarch options for special: placeholders. e.g emitIns_R_C, also identifying the
786 // accessing a local on stack.
787 unsigned _idLclVar : 1; // access a local on stack.
790 #ifdef TARGET_RISCV64
791 // TODO-RISCV64: maybe delete on future
792 opSize _idOpSize : 3; // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16
793 insOpts _idInsOpt : 6; // options for instructions
794 unsigned _idLclVar : 1; // access a local on stack
798 insSize _idInsSize : 2; // size of instruction: 16, 32 or 48 bits
799 insFlags _idInsFlags : 1; // will this instruction set the flags
800 unsigned _idLclVar : 1; // access a local on stack
801 unsigned _idLclFPBase : 1; // access a local on stack - SP based offset
802 insOpts _idInsOpt : 3; // options for Load/Store instructions
805 ////////////////////////////////////////////////////////////////////////
806 // Space taken up to here:
811 // loongarch64: 46 bits
815 // How many bits have been used beyond the first 32?
816 // Define ID_EXTRA_BITFIELD_BITS to that number.
818 CLANG_FORMAT_COMMENT_ANCHOR;
820 #if defined(TARGET_ARM)
821 #define ID_EXTRA_BITFIELD_BITS (16)
822 #elif defined(TARGET_ARM64)
823 #define ID_EXTRA_BITFIELD_BITS (18)
824 #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
825 #define ID_EXTRA_BITFIELD_BITS (14)
826 #elif defined(TARGET_XARCH)
827 #define ID_EXTRA_BITFIELD_BITS (15)
829 #error Unsupported or unset target architecture
832 unsigned _idCnsReloc : 1; // LargeCns is an RVA and needs reloc tag
833 unsigned _idDspReloc : 1; // LargeDsp is an RVA and needs reloc tag
835 #define ID_EXTRA_RELOC_BITS (2)
837 #if EMIT_BACKWARDS_NAVIGATION
839 // "Pointer" to previous instrDesc in this group. If zero, there is
840 // no previous instrDesc. If non-zero, then _idScaledPrevOffset * 4
841 // is the size in bytes of the previous instrDesc; subtract that from
842 // the current instrDesc* to reach the previous one.
843 // All instrDesc types are <= 56 bytes, but we also need m_debugInfoSize,
844 // which is pointer sized, so 5 bits are required on 64-bit and 4 bits
846 CLANG_FORMAT_COMMENT_ANCHOR;
849 unsigned _idScaledPrevOffset : 5;
850 #define ID_EXTRA_PREV_OFFSET_BITS (5)
852 unsigned _idScaledPrevOffset : 4;
853 #define ID_EXTRA_PREV_OFFSET_BITS (4)
856 #else // !EMIT_BACKWARDS_NAVIGATION
857 #define ID_EXTRA_PREV_OFFSET_BITS (0)
858 #endif // !EMIT_BACKWARDS_NAVIGATION
860 ////////////////////////////////////////////////////////////////////////
861 // Space taken up to here (with/without prev offset, assuming host==target):
866 // loongarch64: 53/48 bits
867 // risc-v: 53/48 bits
868 CLANG_FORMAT_COMMENT_ANCHOR;
870 #define ID_EXTRA_BITS (ID_EXTRA_RELOC_BITS + ID_EXTRA_BITFIELD_BITS + ID_EXTRA_PREV_OFFSET_BITS)
872 /* Use whatever bits are left over for small constants */
874 #define ID_BIT_SMALL_CNS (32 - ID_EXTRA_BITS)
875 C_ASSERT(ID_BIT_SMALL_CNS > 0);
877 ////////////////////////////////////////////////////////////////////////
878 // Small constant size (with/without prev offset, assuming host==target):
883 // loongarch64: 11/16 bits
884 // risc-v: 11/16 bits
885 CLANG_FORMAT_COMMENT_ANCHOR;
887 #define ID_ADJ_SMALL_CNS (int)(1 << (ID_BIT_SMALL_CNS - 1))
888 #define ID_CNT_SMALL_CNS (int)(1 << ID_BIT_SMALL_CNS)
890 #define ID_MIN_SMALL_CNS (int)(0 - ID_ADJ_SMALL_CNS)
891 #define ID_MAX_SMALL_CNS (int)(ID_CNT_SMALL_CNS - ID_ADJ_SMALL_CNS - 1)
893 // We encounter many constants, but there is a disproportionate amount that are in the range [-1, +4]
894 // and otherwise powers of 2. We therefore allow the tracked range here to include negative values.
895 signed _idSmallCns : ID_BIT_SMALL_CNS;
897 ////////////////////////////////////////////////////////////////////////
898 // Space taken up to here: 64 bits, all architectures, by design.
899 ////////////////////////////////////////////////////////////////////////
902 // This is the end of the 'small' instrDesc which is the same on all platforms
904 // If you add lots more fields that need to be cleared (such
905 // as various flags), you might need to update the body of
906 // emitter::emitAllocInstr() to clear them.
908 // SMALL_IDSC_SIZE is this size, in bytes.
910 CLANG_FORMAT_COMMENT_ANCHOR;
912 #define SMALL_IDSC_SIZE 8
915 instrDescDebugInfo* idDebugOnlyInfo() const
917 const char* addr = reinterpret_cast<const char*>(this);
918 return *reinterpret_cast<instrDescDebugInfo* const*>(addr - sizeof(instrDescDebugInfo*));
920 void idDebugOnlyInfo(instrDescDebugInfo* info)
922 char* addr = reinterpret_cast<char*>(this);
923 *reinterpret_cast<instrDescDebugInfo**>(addr - sizeof(instrDescDebugInfo*)) = info;
927 CLANG_FORMAT_COMMENT_ANCHOR;
932 // TODO-Cleanup: We should really add a DEBUG-only tag to this union so we can add asserts
933 // about reading what we think is here, to avoid unexpected corruption issues.
935 #if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64)
936 emitLclVarAddr iiaLclVar;
938 BasicBlock* iiaBBlabel;
939 insGroup* iiaIGlabel;
942 emitAddrMode iiaAddrMode;
943 #endif // TARGET_XARCH
945 CORINFO_FIELD_HANDLE iiaFieldHnd; // iiaFieldHandle is also used to encode
946 // an offset into the JIT data constant area
947 bool iiaIsJitDataOffset() const;
948 int iiaGetJitDataOffset() const;
950 // iiaEncodedInstrCount and its accessor functions are used to specify an instruction
951 // count for jumps, instead of using a label and multiple blocks. This is used in the
952 // prolog as well as for IF_LARGEJMP pseudo-branch instructions.
953 int iiaEncodedInstrCount;
955 bool iiaHasInstrCount() const
957 return (iiaEncodedInstrCount & iaut_MASK) == iaut_INST_COUNT;
959 int iiaGetInstrCount() const
961 assert(iiaHasInstrCount());
962 return (iiaEncodedInstrCount >> iaut_SHIFT);
964 void iiaSetInstrCount(int count)
966 assert(abs(count) < 10);
967 iiaEncodedInstrCount = (count << iaut_SHIFT) | iaut_INST_COUNT;
970 #ifdef TARGET_ARMARCH
975 // For 64-bit architecture this 32-bit structure can pack with these unsigned bit fields
976 emitLclVarAddr iiaLclVar;
977 unsigned _idReg3Scaled : 1; // Reg3 is scaled by idOpSize bits
978 GCtype _idGCref2 : 2;
980 regNumber _idReg3 : REGNUM_BITS;
981 regNumber _idReg4 : REGNUM_BITS;
983 #elif defined(TARGET_XARCH)
986 regNumber _idReg3 : REGNUM_BITS;
987 regNumber _idReg4 : REGNUM_BITS;
989 #elif defined(TARGET_LOONGARCH64)
992 unsigned int iiaEncodedInstr; // instruction's binary encoding.
993 regNumber _idReg3 : REGNUM_BITS;
994 regNumber _idReg4 : REGNUM_BITS;
999 int iiaJmpOffset; // temporary saving the offset of jmp or data.
1000 emitLclVarAddr iiaLclVar;
1003 void iiaSetInstrEncode(unsigned int encode)
1005 iiaEncodedInstr = encode;
1007 unsigned int iiaGetInstrEncode() const
1009 return iiaEncodedInstr;
1012 void iiaSetJmpOffset(int offset)
1014 iiaJmpOffset = offset;
1016 int iiaGetJmpOffset() const
1018 return iiaJmpOffset;
1020 #elif defined(TARGET_RISCV64)
1023 regNumber _idReg3 : REGNUM_BITS;
1024 regNumber _idReg4 : REGNUM_BITS;
1025 unsigned int iiaEncodedInstr; // instruction's binary encoding.
1028 void iiaSetInstrEncode(unsigned int encode)
1030 iiaEncodedInstr = encode;
1032 unsigned int iiaGetInstrEncode() const
1034 return iiaEncodedInstr;
1036 #endif // defined(TARGET_RISCV64)
1040 /* Trivial wrappers to return properly typed enums */
1042 bool idIsSmallDsc() const
1044 return (_idSmallDsc != 0);
1046 void idSetIsSmallDsc()
1051 #if defined(TARGET_XARCH)
1053 unsigned idCodeSize() const
1057 void idCodeSize(unsigned sz)
1059 assert(sz <= 15); // Intel decoder limit.
1061 assert(sz == _idCodeSize);
1064 #elif defined(TARGET_ARM64)
1066 inline bool idIsEmptyAlign() const
1068 return (idIns() == INS_align) && (idInsOpt() == INS_OPTS_NONE);
1071 unsigned idCodeSize() const
1079 // b<cond> + b<uncond>
1083 if (isVectorRegister(idReg1()))
1085 // (adrp + ldr + fmov) or (adrp + add + ld1)
1095 if (idIsEmptyAlign())
1107 #elif defined(TARGET_ARM)
1109 bool idInstrIsT1() const
1111 return (_idInsSize == ISZ_16BIT);
1113 unsigned idCodeSize() const
1115 unsigned result = (_idInsSize == ISZ_16BIT) ? 2 : (_idInsSize == ISZ_32BIT) ? 4 : 6;
1118 insSize idInsSize() const
1122 void idInsSize(insSize isz)
1125 assert(isz == _idInsSize);
1127 insFlags idInsFlags() const
1131 void idInsFlags(insFlags sf)
1134 assert(sf == _idInsFlags);
1137 #elif defined(TARGET_LOONGARCH64)
1138 unsigned idCodeSize() const
1142 void idCodeSize(unsigned sz)
1144 // LoongArch64's instrDesc is not always meaning only one instruction.
1145 // e.g. the `emitter::emitIns_I_la` for emitting the immediates.
1149 #elif defined(TARGET_RISCV64)
1150 unsigned idCodeSize() const
1154 void idCodeSize(unsigned sz)
1156 // RISCV64's instrDesc is not always meaning only one instruction.
1157 // e.g. the `emitter::emitLoadImmediate` for emitting the immediates.
1163 emitAttr idOpSize() const
1165 return emitDecodeSize(_idOpSize);
1167 void idOpSize(emitAttr opsz)
1169 _idOpSize = emitEncodeSize(opsz);
1172 GCtype idGCref() const
1174 return (GCtype)_idGCref;
1176 void idGCref(GCtype gctype)
1181 regNumber idReg1() const
1185 void idReg1(regNumber reg)
1188 assert(reg == _idReg1);
1192 GCtype idGCrefReg2() const
1194 assert(!idIsSmallDsc());
1195 return (GCtype)idAddr()->_idGCref2;
1197 void idGCrefReg2(GCtype gctype)
1199 assert(!idIsSmallDsc());
1200 idAddr()->_idGCref2 = gctype;
1202 #endif // TARGET_ARM64
1204 regNumber idReg2() const
1208 void idReg2(regNumber reg)
1211 assert(reg == _idReg2);
1214 #if defined(TARGET_XARCH)
1215 regNumber idReg3() const
1217 assert(!idIsSmallDsc());
1218 return idAddr()->_idReg3;
1220 void idReg3(regNumber reg)
1222 assert(!idIsSmallDsc());
1223 idAddr()->_idReg3 = reg;
1224 assert(reg == idAddr()->_idReg3);
1227 regNumber idReg4() const
1229 assert(!idIsSmallDsc());
1230 return idAddr()->_idReg4;
1232 void idReg4(regNumber reg)
1234 assert(!idIsSmallDsc());
1235 idAddr()->_idReg4 = reg;
1236 assert(reg == idAddr()->_idReg4);
1239 bool idHasReg1() const
1241 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1242 return (isInfo & (IS_R1_RD | IS_R1_RW | IS_R1_WR)) != 0;
1244 bool idIsReg1Read() const
1246 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1247 return (isInfo & (IS_R1_RD | IS_R1_RW)) != 0;
1249 bool idIsReg1Write() const
1251 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1252 return (isInfo & (IS_R1_RW | IS_R1_WR)) != 0;
1255 bool idHasReg2() const
1257 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1258 return (isInfo & (IS_R2_RD | IS_R2_RW | IS_R2_WR)) != 0;
1260 bool idIsReg2Read() const
1262 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1263 return (isInfo & (IS_R2_RD | IS_R2_RW)) != 0;
1265 bool idIsReg2Write() const
1267 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1268 return (isInfo & (IS_R2_RW | IS_R2_WR)) != 0;
1271 bool idHasReg3() const
1273 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1274 return (isInfo & (IS_R3_RD | IS_R3_RW | IS_R3_WR)) != 0;
1276 bool idIsReg3Read() const
1278 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1279 return (isInfo & (IS_R3_RD | IS_R3_RW)) != 0;
1281 bool idIsReg3Write() const
1283 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1284 return (isInfo & (IS_R3_RW | IS_R3_WR)) != 0;
1287 bool idHasReg4() const
1289 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1290 return (isInfo & (IS_R4_RD | IS_R4_RW | IS_R4_WR)) != 0;
1292 bool idIsReg4Read() const
1294 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1295 return (isInfo & (IS_R4_RD | IS_R4_RW)) != 0;
1297 bool idIsReg4Write() const
1299 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1300 return (isInfo & (IS_R4_RW | IS_R4_WR)) != 0;
1303 bool idHasMemGen() const
1305 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1306 return (isInfo & (IS_GM_RD | IS_GM_RW | IS_GM_WR)) != 0;
1308 bool idHasMemGenRead() const
1310 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1311 return (isInfo & (IS_GM_RD | IS_GM_RW)) != 0;
1313 bool idHasMemGenWrite() const
1315 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1316 return (isInfo & (IS_GM_RW | IS_GM_WR)) != 0;
1319 bool idHasMemStk() const
1321 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1322 return (isInfo & (IS_SF_RD | IS_SF_RW | IS_SF_WR)) != 0;
1324 bool idHasMemStkRead() const
1326 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1327 return (isInfo & (IS_SF_RD | IS_SF_RW)) != 0;
1329 bool idHasMemStkWrite() const
1331 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1332 return (isInfo & (IS_SF_RW | IS_SF_WR)) != 0;
1335 bool idHasMemAdr() const
1337 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1338 return (isInfo & (IS_AM_RD | IS_AM_RW | IS_AM_WR)) != 0;
1340 bool idHasMemAdrRead() const
1342 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1343 return (isInfo & (IS_AM_RD | IS_AM_RW)) != 0;
1345 bool idHasMemAdrWrite() const
1347 IS_INFO isInfo = emitGetSchedInfo(idInsFmt());
1348 return (isInfo & (IS_AM_RW | IS_AM_WR)) != 0;
1351 bool idHasMem() const
1353 return idHasMemGen() || idHasMemStk() || idHasMemAdr();
1355 bool idHasMemRead() const
1357 return idHasMemGenRead() || idHasMemStkRead() || idHasMemAdrRead();
1359 bool idHasMemWrite() const
1361 return idHasMemGenWrite() || idHasMemStkWrite() || idHasMemAdrWrite();
1363 #endif // defined(TARGET_XARCH)
1364 #ifdef TARGET_ARMARCH
1365 insOpts idInsOpt() const
1367 return (insOpts)_idInsOpt;
1369 void idInsOpt(insOpts opt)
1372 assert(opt == _idInsOpt);
1375 regNumber idReg3() const
1377 assert(!idIsSmallDsc());
1378 return idAddr()->_idReg3;
1380 void idReg3(regNumber reg)
1382 assert(!idIsSmallDsc());
1383 idAddr()->_idReg3 = reg;
1384 assert(reg == idAddr()->_idReg3);
1386 regNumber idReg4() const
1388 assert(!idIsSmallDsc());
1389 return idAddr()->_idReg4;
1391 void idReg4(regNumber reg)
1393 assert(!idIsSmallDsc());
1394 idAddr()->_idReg4 = reg;
1395 assert(reg == idAddr()->_idReg4);
1398 bool idReg3Scaled() const
1400 assert(!idIsSmallDsc());
1401 return (idAddr()->_idReg3Scaled == 1);
1403 void idReg3Scaled(bool val)
1405 assert(!idIsSmallDsc());
1406 idAddr()->_idReg3Scaled = val ? 1 : 0;
1408 #endif // TARGET_ARM64
1410 #endif // TARGET_ARMARCH
1412 #ifdef TARGET_LOONGARCH64
1413 insOpts idInsOpt() const
1415 return (insOpts)_idInsOpt;
1417 void idInsOpt(insOpts opt)
1420 assert(opt == _idInsOpt);
1423 regNumber idReg3() const
1425 assert(!idIsSmallDsc());
1426 return idAddr()->_idReg3;
1428 void idReg3(regNumber reg)
1430 assert(!idIsSmallDsc());
1431 idAddr()->_idReg3 = reg;
1432 assert(reg == idAddr()->_idReg3);
1434 regNumber idReg4() const
1436 assert(!idIsSmallDsc());
1437 return idAddr()->_idReg4;
1439 void idReg4(regNumber reg)
1441 assert(!idIsSmallDsc());
1442 idAddr()->_idReg4 = reg;
1443 assert(reg == idAddr()->_idReg4);
1446 #endif // TARGET_LOONGARCH64
1448 #ifdef TARGET_RISCV64
1449 insOpts idInsOpt() const
1451 return (insOpts)_idInsOpt;
1453 void idInsOpt(insOpts opt)
1456 assert(opt == _idInsOpt);
1459 regNumber idReg3() const
1461 assert(!idIsSmallDsc());
1462 return idAddr()->_idReg3;
1464 void idReg3(regNumber reg)
1466 assert(!idIsSmallDsc());
1467 idAddr()->_idReg3 = reg;
1468 assert(reg == idAddr()->_idReg3);
1470 regNumber idReg4() const
1472 assert(!idIsSmallDsc());
1473 return idAddr()->_idReg4;
1475 void idReg4(regNumber reg)
1477 assert(!idIsSmallDsc());
1478 idAddr()->_idReg4 = reg;
1479 assert(reg == idAddr()->_idReg4);
1482 #endif // TARGET_RISCV64
1484 inline static bool fitsInSmallCns(cnsval_ssize_t val)
1486 return ((val >= ID_MIN_SMALL_CNS) && (val <= ID_MAX_SMALL_CNS));
1489 bool idIsLargeCns() const
1491 return _idLargeCns != 0;
1493 void idSetIsLargeCns()
1498 bool idIsLargeDsp() const
1500 return _idLargeDsp != 0;
1502 void idSetIsLargeDsp()
1506 void idSetIsSmallDsp()
1511 bool idIsLargeCall() const
1513 return _idLargeCall != 0;
1515 void idSetIsLargeCall()
1520 bool idIsBound() const
1522 return _idBound != 0;
1529 bool idIsCallRegPtr() const
1531 return _idCallRegPtr != 0;
1533 void idSetIsCallRegPtr()
1538 // Only call instructions that call helper functions may be marked as "IsNoGC", indicating
1539 // that a thread executing such a call cannot be stopped for GC. Thus, in partially-interruptible
1540 // code, it is not necessary to generate GC info for a call so labeled.
1541 bool idIsNoGC() const
1543 return _idNoGC != 0;
1545 void idSetIsNoGC(bool val)
1551 bool idIsEvexbContext() const
1553 return _idEvexbContext != 0;
1555 void idSetEvexbContext()
1557 assert(_idEvexbContext == 0);
1558 _idEvexbContext = 1;
1559 assert(_idEvexbContext == 1);
1563 #ifdef TARGET_ARMARCH
1564 bool idIsLclVar() const
1566 return _idLclVar != 0;
1568 void idSetIsLclVar()
1573 bool idIsLclVarPair() const
1575 return _idLclVarPair != 0;
1577 void idSetIsLclVarPair()
1581 #endif // TARGET_ARM64
1582 #endif // TARGET_ARMARCH
1584 #if defined(TARGET_ARM)
1585 bool idIsLclFPBase() const
1587 return _idLclFPBase != 0;
1589 void idSetIsLclFPBase()
1593 #endif // defined(TARGET_ARM)
1595 #ifdef TARGET_LOONGARCH64
1596 bool idIsLclVar() const
1598 return _idLclVar != 0;
1600 void idSetIsLclVar()
1604 #endif // TARGET_LOONGARCH64
1606 #ifdef TARGET_RISCV64
1607 bool idIsLclVar() const
1609 return _idLclVar != 0;
1611 void idSetIsLclVar()
1615 #endif // TARGET_RISCV64
1617 bool idIsCnsReloc() const
1619 return _idCnsReloc != 0;
1621 void idSetIsCnsReloc()
1626 bool idIsDspReloc() const
1628 return _idDspReloc != 0;
1630 void idSetIsDspReloc(bool val = true)
1634 bool idIsReloc() const
1636 return idIsDspReloc() || idIsCnsReloc();
1639 void idSetRelocFlags(emitAttr attr)
1641 _idCnsReloc = (EA_IS_CNS_RELOC(attr) ? 1 : 0);
1642 _idDspReloc = (EA_IS_DSP_RELOC(attr) ? 1 : 0);
1645 #if EMIT_BACKWARDS_NAVIGATION
1647 // Return the stored size of the previous instrDesc in bytes, or zero if there
1648 // is no previous instrDesc in this group.
1649 unsigned idPrevSize()
1651 return _idScaledPrevOffset * 4;
1653 void idSetPrevSize(unsigned prevInstrDescSizeInBytes)
1655 assert(prevInstrDescSizeInBytes % 4 == 0);
1656 _idScaledPrevOffset = prevInstrDescSizeInBytes / 4;
1657 assert(idPrevSize() == prevInstrDescSizeInBytes);
1660 #endif // EMIT_BACKWARDS_NAVIGATION
1662 signed idSmallCns() const
1666 void idSmallCns(cnsval_ssize_t value)
1668 assert(fitsInSmallCns(value));
1669 _idSmallCns = value;
1670 assert(value == idSmallCns());
1673 inline const idAddrUnion* idAddr() const
1675 assert(!idIsSmallDsc());
1676 return &this->_idAddrUnion;
1679 inline idAddrUnion* idAddr()
1681 assert(!idIsSmallDsc());
1682 return &this->_idAddrUnion;
1684 }; // End of struct instrDesc
1686 #if defined(TARGET_XARCH)
1687 insFormat getMemoryOperation(instrDesc* id) const;
1688 insFormat ExtractMemoryFormat(insFormat insFmt) const;
1689 #elif defined(TARGET_ARM64)
1690 void getMemoryOperation(instrDesc* id, unsigned* pMemAccessKind, bool* pIsLocalAccess);
1693 #if defined(DEBUG) || defined(LATE_DISASM)
1695 #define PERFSCORE_THROUGHPUT_ILLEGAL -1024.0f
1697 #define PERFSCORE_THROUGHPUT_ZERO 0.0f // Only used for pseudo-instructions that don't generate code
1699 #define PERFSCORE_THROUGHPUT_6X (1.0f / 6.0f) // Hextuple issue
1700 #define PERFSCORE_THROUGHPUT_5X 0.20f // Pentuple issue
1701 #define PERFSCORE_THROUGHPUT_4X 0.25f // Quad issue
1702 #define PERFSCORE_THROUGHPUT_3X (1.0f / 3.0f) // Three issue
1703 #define PERFSCORE_THROUGHPUT_2X 0.5f // Dual issue
1705 #define PERFSCORE_THROUGHPUT_1C 1.0f // Single Issue
1707 #define PERFSCORE_THROUGHPUT_2C 2.0f // slower - 2 cycles
1708 #define PERFSCORE_THROUGHPUT_3C 3.0f // slower - 3 cycles
1709 #define PERFSCORE_THROUGHPUT_4C 4.0f // slower - 4 cycles
1710 #define PERFSCORE_THROUGHPUT_5C 5.0f // slower - 5 cycles
1711 #define PERFSCORE_THROUGHPUT_6C 6.0f // slower - 6 cycles
1712 #define PERFSCORE_THROUGHPUT_7C 7.0f // slower - 7 cycles
1713 #define PERFSCORE_THROUGHPUT_8C 8.0f // slower - 8 cycles
1714 #define PERFSCORE_THROUGHPUT_9C 9.0f // slower - 9 cycles
1715 #define PERFSCORE_THROUGHPUT_10C 10.0f // slower - 10 cycles
1716 #define PERFSCORE_THROUGHPUT_13C 13.0f // slower - 13 cycles
1717 #define PERFSCORE_THROUGHPUT_19C 19.0f // slower - 19 cycles
1718 #define PERFSCORE_THROUGHPUT_25C 25.0f // slower - 25 cycles
1719 #define PERFSCORE_THROUGHPUT_33C 33.0f // slower - 33 cycles
1720 #define PERFSCORE_THROUGHPUT_50C 50.0f // slower - 50 cycles
1721 #define PERFSCORE_THROUGHPUT_52C 52.0f // slower - 52 cycles
1722 #define PERFSCORE_THROUGHPUT_57C 57.0f // slower - 57 cycles
1723 #define PERFSCORE_THROUGHPUT_140C 140.0f // slower - 140 cycles
1725 #define PERFSCORE_LATENCY_ILLEGAL -1024.0f
1727 #define PERFSCORE_LATENCY_ZERO 0.0f
1728 #define PERFSCORE_LATENCY_1C 1.0f
1729 #define PERFSCORE_LATENCY_2C 2.0f
1730 #define PERFSCORE_LATENCY_3C 3.0f
1731 #define PERFSCORE_LATENCY_4C 4.0f
1732 #define PERFSCORE_LATENCY_5C 5.0f
1733 #define PERFSCORE_LATENCY_6C 6.0f
1734 #define PERFSCORE_LATENCY_7C 7.0f
1735 #define PERFSCORE_LATENCY_8C 8.0f
1736 #define PERFSCORE_LATENCY_9C 9.0f
1737 #define PERFSCORE_LATENCY_10C 10.0f
1738 #define PERFSCORE_LATENCY_11C 11.0f
1739 #define PERFSCORE_LATENCY_12C 12.0f
1740 #define PERFSCORE_LATENCY_13C 13.0f
1741 #define PERFSCORE_LATENCY_15C 15.0f
1742 #define PERFSCORE_LATENCY_16C 16.0f
1743 #define PERFSCORE_LATENCY_18C 18.0f
1744 #define PERFSCORE_LATENCY_20C 20.0f
1745 #define PERFSCORE_LATENCY_22C 22.0f
1746 #define PERFSCORE_LATENCY_23C 23.0f
1747 #define PERFSCORE_LATENCY_26C 26.0f
1748 #define PERFSCORE_LATENCY_62C 62.0f
1749 #define PERFSCORE_LATENCY_69C 69.0f
1750 #define PERFSCORE_LATENCY_140C 140.0f
1751 #define PERFSCORE_LATENCY_400C 400.0f // Intel microcode issue with these instructions
1753 #define PERFSCORE_LATENCY_BRANCH_DIRECT 1.0f // cost of an unconditional branch
1754 #define PERFSCORE_LATENCY_BRANCH_COND 2.0f // includes cost of a possible misprediction
1755 #define PERFSCORE_LATENCY_BRANCH_INDIRECT 2.0f // includes cost of a possible misprediction
1757 #if defined(TARGET_XARCH)
1759 // a read,write or modify from stack location, possible def to use latency from L0 cache
1760 #define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_2C
1761 #define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_2C
1762 #define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_5C
1764 // a read, write or modify from constant location, possible def to use latency from L0 cache
1765 #define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_2C
1766 #define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_2C
1767 #define PERFSCORE_LATENCY_RD_WR_CONST_ADDR PERFSCORE_LATENCY_5C
1769 // a read, write or modify from memory location, possible def to use latency from L0 or L1 cache
1770 // plus an extra cost (of 1.0) for a increased chance of a cache miss
1771 #define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_3C
1772 #define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_3C
1773 #define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_6C
1775 #elif defined(TARGET_ARM64) || defined(TARGET_ARM)
1777 // a read,write or modify from stack location, possible def to use latency from L0 cache
1778 #define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_3C
1779 #define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_1C
1780 #define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_3C
1782 // a read, write or modify from constant location, possible def to use latency from L0 cache
1783 #define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_3C
1784 #define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_1C
1785 #define PERFSCORE_LATENCY_RD_WR_CONST_ADDR PERFSCORE_LATENCY_3C
1787 // a read, write or modify from memory location, possible def to use latency from L0 or L1 cache
1788 // plus an extra cost (of 1.0) for a increased chance of a cache miss
1789 #define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_4C
1790 #define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_1C
1791 #define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_4C
1793 #elif defined(TARGET_LOONGARCH64)
1794 // a read,write or modify from stack location, possible def to use latency from L0 cache
1795 #define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_3C
1796 #define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_1C
1797 #define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_3C
1799 // a read, write or modify from constant location, possible def to use latency from L0 cache
1800 #define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_3C
1801 #define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_1C
1802 #define PERFSCORE_LATENCY_RD_WR_CONST_ADDR PERFSCORE_LATENCY_3C
1804 // a read, write or modify from memory location, possible def to use latency from L0 or L1 cache
1805 // plus an extra cost (of 1.0) for a increased chance of a cache miss
1806 #define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_4C
1807 #define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_1C
1808 #define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_4C
1810 #elif defined(TARGET_RISCV64)
1811 // a read,write or modify from stack location, possible def to use latency from L0 cache
1812 #define PERFSCORE_LATENCY_RD_STACK PERFSCORE_LATENCY_3C
1813 #define PERFSCORE_LATENCY_WR_STACK PERFSCORE_LATENCY_1C
1814 #define PERFSCORE_LATENCY_RD_WR_STACK PERFSCORE_LATENCY_3C
1816 // a read, write or modify from constant location, possible def to use latency from L0 cache
1817 #define PERFSCORE_LATENCY_RD_CONST_ADDR PERFSCORE_LATENCY_3C
1818 #define PERFSCORE_LATENCY_WR_CONST_ADDR PERFSCORE_LATENCY_1C
1819 #define PERFSCORE_LATENCY_RD_WR_CONST_ADDR PERFSCORE_LATENCY_3C
1821 // a read, write or modify from memory location, possible def to use latency from L0 or L1 cache
1822 // plus an extra cost (of 1.0) for a increased chance of a cache miss
1823 #define PERFSCORE_LATENCY_RD_GENERAL PERFSCORE_LATENCY_4C
1824 #define PERFSCORE_LATENCY_WR_GENERAL PERFSCORE_LATENCY_1C
1825 #define PERFSCORE_LATENCY_RD_WR_GENERAL PERFSCORE_LATENCY_4C
1827 #endif // TARGET_XXX
1829 // Make this an enum:
1831 #define PERFSCORE_MEMORY_NONE 0
1832 #define PERFSCORE_MEMORY_READ 1
1833 #define PERFSCORE_MEMORY_WRITE 2
1834 #define PERFSCORE_MEMORY_READ_WRITE 3
1836 #define PERFSCORE_CODESIZE_COST_HOT 0.10f
1837 #define PERFSCORE_CODESIZE_COST_COLD 0.01f
1839 #define PERFSCORE_CALLEE_SPILL_COST 0.75f
1841 struct insExecutionCharacteristics
1843 float insThroughput;
1845 unsigned insMemoryAccessKind;
1848 float insEvaluateExecutionCost(instrDesc* id);
1850 insExecutionCharacteristics getInsExecutionCharacteristics(instrDesc* id);
1852 void perfScoreUnhandledInstruction(instrDesc* id, insExecutionCharacteristics* result);
1854 #endif // defined(DEBUG) || defined(LATE_DISASM)
1856 weight_t getCurrentBlockWeight();
1858 void dispIns(instrDesc* id);
1860 void appendToCurIG(instrDesc* id);
1862 /********************************************************************************************/
1864 struct instrDescJmp : instrDesc
1866 instrDescJmp() = delete;
1868 instrDescJmp* idjNext; // next jump in the group/method
1869 insGroup* idjIG; // containing group
1872 BYTE* idjAddr; // address of jump ins (for patching)
1875 // Before jump emission, this is the byte offset within IG of the jump instruction.
1876 // After emission, for forward jumps, this is the target offset -- in bytes from the
1877 // beginning of the function -- of the target instruction of the jump, used to
1878 // determine if this jump needs to be patched.
1880 #if defined(TARGET_XARCH)
1882 // indicates that the jump was added at the end of a BBJ_ALWAYS basic block and is
1883 // a candidate for being removed if it jumps to the next instruction
1884 unsigned idjIsRemovableJmpCandidate : 1;
1888 unsigned idjShort : 1; // is the jump known to be a short one?
1889 unsigned idjKeepLong : 1; // should the jump be kept long? (used for hot to cold and cold to hot jumps)
1892 #if FEATURE_LOOP_ALIGN
1893 struct instrDescAlign : instrDesc
1895 instrDescAlign() = delete;
1897 instrDescAlign* idaNext; // next align in the group/method
1898 insGroup* idaIG; // containing group
1899 insGroup* idaLoopHeadPredIG; // The IG before the loop IG.
1900 // If no 'jmp' instructions were found until idaLoopHeadPredIG,
1901 // then idaLoopHeadPredIG == idaIG.
1903 bool isPlacedAfterJmp; // Is the 'align' instruction placed after jmp. Used to decide
1904 // if the instruction cost should be included in PerfScore
1905 // calculation or not.
1908 inline insGroup* loopHeadIG()
1910 assert(idaLoopHeadPredIG);
1911 return idaLoopHeadPredIG->igNext;
1914 void removeAlignFlags()
1916 idaIG->igFlags &= ~IGF_HAS_ALIGN;
1917 idaIG->igFlags |= IGF_REMOVED_ALIGN;
1920 void emitCheckAlignFitInCurIG(unsigned nAlignInstr);
1921 #endif // FEATURE_LOOP_ALIGN
1923 #if !defined(TARGET_ARM64) // This shouldn't be needed for ARM32, either, but I don't want to touch the ARM32 JIT.
1924 struct instrDescLbl : instrDescJmp
1926 emitLclVarAddr dstLclVar;
1928 #endif // !TARGET_ARM64
1930 struct instrDescCns : instrDesc // large const
1932 instrDescCns() = delete;
1934 cnsval_ssize_t idcCnsVal;
1937 struct instrDescDsp : instrDesc // large displacement
1939 instrDescDsp() = delete;
1941 target_ssize_t iddDspVal;
1944 struct instrDescCnsDsp : instrDesc // large cons + disp
1946 instrDescCnsDsp() = delete;
1948 target_ssize_t iddcCnsVal;
1954 struct instrDescAmd : instrDesc // large addrmode disp
1956 instrDescAmd() = delete;
1961 struct instrDescCnsAmd : instrDesc // large cons + addrmode disp
1963 instrDescCnsAmd() = delete;
1969 #endif // TARGET_XARCH
1972 struct instrDescLclVarPair : instrDesc // contains 2 gc vars to be tracked
1974 instrDescLclVarPair() = delete;
1976 emitLclVarAddr iiaLclVar2;
1979 struct instrDescLclVarPairCns : instrDescCns // contains 2 gc vars to be tracked, with large cons
1981 instrDescLclVarPairCns() = delete;
1983 emitLclVarAddr iiaLclVar2;
1987 struct instrDescCGCA : instrDesc // call with ...
1989 instrDescCGCA() = delete;
1991 VARSET_TP idcGCvars; // ... updated GC vars or
1992 ssize_t idcDisp; // ... big addrmode disp
1993 regMaskTP idcGcrefRegs; // ... gcref registers
1994 regMaskTP idcByrefRegs; // ... byref registers
1995 unsigned idcArgCnt; // ... lots of args or (<0 ==> caller pops args)
1997 #if MULTIREG_HAS_SECOND_GC_RET
1998 // This method handle the GC-ness of the second register in a 2 register returned struct on System V.
1999 GCtype idSecondGCref() const
2001 return (GCtype)_idcSecondRetRegGCType;
2003 void idSecondGCref(GCtype gctype)
2005 _idcSecondRetRegGCType = gctype;
2009 // This member stores the GC-ness of the second register in a 2 register returned struct on System V.
2010 // It is added to the call struct since it is not needed by the base instrDesc struct, which keeps GC-ness
2011 // of the first register for the instCall nodes.
2012 // The base instrDesc is very carefully kept to be no more than 128 bytes. There is no more space to add members
2013 // for keeping GC-ness of the second return registers. It will also bloat the base struct unnecessarily
2014 // since the GC-ness of the second register is only needed for call instructions.
2015 // The base struct's member keeping the GC-ness of the first return register is _idGCref.
2016 GCtype _idcSecondRetRegGCType : 2; // ... GC type for the second return register.
2017 #endif // MULTIREG_HAS_SECOND_GC_RET
2020 // TODO-Cleanup: Uses of stack-allocated instrDescs should be refactored to be unnecessary.
2021 template <typename T>
2022 struct inlineInstrDesc
2025 instrDescDebugInfo* idDebugInfo;
2026 alignas(alignof(T)) char idStorage[sizeof(T)];
2029 inlineInstrDesc() : idDebugInfo(nullptr), idStorage()
2031 static_assert_no_msg((offsetof(inlineInstrDesc<T>, idStorage) - sizeof(instrDescDebugInfo*)) ==
2032 offsetof(inlineInstrDesc<T>, idDebugInfo));
2037 return reinterpret_cast<T*>(idStorage);
2043 struct instrDescReloc : instrDesc
2045 instrDescReloc() = delete;
2050 BYTE* emitGetInsRelocValue(instrDesc* id);
2052 #endif // TARGET_ARM
2054 insUpdateModes emitInsUpdateMode(instruction ins);
2055 insFormat emitInsModeFormat(instruction ins, insFormat base);
2057 static const BYTE emitInsModeFmtTab[];
2059 static const unsigned emitInsModeFmtCnt;
2062 size_t emitGetInstrDescSize(const instrDesc* id);
2066 ssize_t emitGetInsCns(instrDesc* id);
2067 ssize_t emitGetInsDsp(instrDesc* id);
2068 ssize_t emitGetInsAmd(instrDesc* id);
2070 ssize_t emitGetInsCIdisp(instrDesc* id);
2071 unsigned emitGetInsCIargs(instrDesc* id);
2073 inline emitAttr emitGetMemOpSize(instrDesc* id) const;
2074 inline emitAttr emitGetBaseMemOpSize(instrDesc*) const;
2076 // Return the argument count for a direct call "id".
2077 int emitGetInsCDinfo(instrDesc* id);
2079 static const IS_INFO emitGetSchedInfo(insFormat f);
2080 #endif // TARGET_XARCH
2082 cnsval_ssize_t emitGetInsSC(const instrDesc* id) const;
2083 unsigned emitInsCount;
2085 /************************************************************************/
2086 /* A few routines used for debug display purposes */
2087 /************************************************************************/
2089 static const char* emitIfName(unsigned f);
2092 unsigned emitVarRefOffs;
2094 #define emitVarRefOffs 0
2097 const char* emitRegName(regNumber reg, emitAttr size = EA_PTRSIZE, bool varName = true) const;
2098 const char* emitFloatRegName(regNumber reg, emitAttr size = EA_PTRSIZE, bool varName = true);
2100 // GC Info changes are not readily available at each instruction.
2101 // We use debug-only sets to track the per-instruction state, and to remember
2102 // what the state was at the last time it was output (instruction or label).
2103 VARSET_TP debugPrevGCrefVars;
2104 VARSET_TP debugThisGCrefVars;
2105 regPtrDsc* debugPrevRegPtrDsc;
2106 regMaskTP debugPrevGCrefRegs;
2107 regMaskTP debugPrevByrefRegs;
2108 void emitDispInsIndent();
2109 void emitDispGCDeltaTitle(const char* title);
2110 void emitDispGCRegDelta(const char* title, regMaskTP prevRegs, regMaskTP curRegs);
2111 void emitDispGCVarDelta();
2112 void emitDispRegPtrListDelta();
2113 void emitDispGCInfoDelta();
2115 void emitDispIGflags(unsigned flags);
2116 void emitDispIG(insGroup* ig,
2117 bool displayFunc = false,
2118 bool displayInstructions = false,
2119 bool displayLocation = true);
2120 void emitDispIGlist(bool displayInstructions = false);
2121 void emitDispGCinfo();
2122 void emitDispJumpList();
2123 void emitDispClsVar(CORINFO_FIELD_HANDLE fldHnd, ssize_t offs, bool reloc = false);
2124 void emitDispFrameRef(int varx, int disp, int offs, bool asmfm);
2125 void emitDispInsAddr(const BYTE* code);
2126 void emitDispInsOffs(unsigned offs, bool doffs);
2127 void emitDispInsHex(instrDesc* id, BYTE* code, size_t sz);
2128 void emitDispEmbBroadcastCount(instrDesc* id);
2129 void emitDispIns(instrDesc* id,
2134 BYTE* pCode = nullptr,
2136 insGroup* ig = nullptr);
2138 /************************************************************************/
2139 /* Method prolog and epilog */
2140 /************************************************************************/
2142 unsigned emitPrologEndPos;
2144 unsigned emitEpilogCnt;
2145 UNATIVE_OFFSET emitEpilogSize;
2149 void emitStartExitSeq(); // Mark the start of the "return" sequence
2150 emitLocation emitExitSeqBegLoc;
2151 UNATIVE_OFFSET emitExitSeqSize; // minimum size of any return sequence - the 'ret' after the epilog
2153 #endif // TARGET_XARCH
2155 insGroup* emitPlaceholderList; // per method placeholder list - head
2156 insGroup* emitPlaceholderLast; // per method placeholder list - tail
2158 #ifdef JIT32_GCENCODER
2160 // The x86 GC encoder needs to iterate over a list of epilogs to generate a table of
2161 // epilog offsets. Epilogs always start at the beginning of an IG, so save the first
2162 // IG of the epilog, and use it to find the epilog offset at the end of code generation.
2168 EpilogList() : elNext(nullptr), elLoc()
2173 EpilogList* emitEpilogList; // per method epilog list - head
2174 EpilogList* emitEpilogLast; // per method epilog list - tail
2177 void emitStartEpilog();
2179 bool emitHasEpilogEnd();
2181 size_t emitGenEpilogLst(size_t (*fp)(void*, unsigned), void* cp);
2183 #endif // JIT32_GCENCODER
2185 void emitBegPrologEpilog(insGroup* igPh);
2186 void emitEndPrologEpilog();
2188 void emitBegFnEpilog(insGroup* igPh);
2189 void emitEndFnEpilog();
2191 #if defined(FEATURE_EH_FUNCLETS)
2193 void emitBegFuncletProlog(insGroup* igPh);
2194 void emitEndFuncletProlog();
2196 void emitBegFuncletEpilog(insGroup* igPh);
2197 void emitEndFuncletEpilog();
2199 #endif // FEATURE_EH_FUNCLETS
2201 /************************************************************************/
2202 /* Methods to record a code position and later convert to offset */
2203 /************************************************************************/
2205 unsigned emitFindInsNum(const insGroup* ig, const instrDesc* id) const;
2206 UNATIVE_OFFSET emitFindOffset(const insGroup* ig, unsigned insNum) const;
2208 /************************************************************************/
2209 /* Members and methods used to issue (encode) instructions. */
2210 /************************************************************************/
2213 // If we have started issuing instructions from the list of instrDesc, this is set
2217 BYTE* emitCodeBlock; // Hot code block
2218 BYTE* emitColdCodeBlock; // Cold code block
2219 BYTE* emitConsBlock; // Read-only (constant) data block
2220 size_t writeableOffset; // Offset applied to a code address to get memory location that can be written
2222 UNATIVE_OFFSET emitTotalHotCodeSize;
2223 UNATIVE_OFFSET emitTotalColdCodeSize;
2225 UNATIVE_OFFSET emitCurCodeOffs(const BYTE* dst) const
2228 if ((dst >= emitCodeBlock) && (dst <= (emitCodeBlock + emitTotalHotCodeSize)))
2230 distance = (dst - emitCodeBlock);
2234 assert(emitFirstColdIG);
2235 assert(emitColdCodeBlock);
2236 assert((dst >= emitColdCodeBlock) && (dst <= (emitColdCodeBlock + emitTotalColdCodeSize)));
2238 distance = (dst - emitColdCodeBlock + emitTotalHotCodeSize);
2240 noway_assert((UNATIVE_OFFSET)distance == distance);
2241 return (UNATIVE_OFFSET)distance;
2244 BYTE* emitOffsetToPtr(UNATIVE_OFFSET offset) const
2246 if (offset < emitTotalHotCodeSize)
2248 return emitCodeBlock + offset;
2252 assert(offset < (emitTotalHotCodeSize + emitTotalColdCodeSize));
2254 return emitColdCodeBlock + (offset - emitTotalHotCodeSize);
2258 BYTE* emitDataOffsetToPtr(UNATIVE_OFFSET offset)
2260 assert(offset < emitDataSize());
2261 return emitConsBlock + offset;
2264 bool emitJumpCrossHotColdBoundary(size_t srcOffset, size_t dstOffset)
2266 if (emitTotalColdCodeSize == 0)
2271 assert(srcOffset < (emitTotalHotCodeSize + emitTotalColdCodeSize));
2272 assert(dstOffset < (emitTotalHotCodeSize + emitTotalColdCodeSize));
2274 return ((srcOffset < emitTotalHotCodeSize) != (dstOffset < emitTotalHotCodeSize));
2277 unsigned char emitOutputByte(BYTE* dst, ssize_t val);
2278 unsigned char emitOutputWord(BYTE* dst, ssize_t val);
2279 unsigned char emitOutputLong(BYTE* dst, ssize_t val);
2280 unsigned char emitOutputSizeT(BYTE* dst, ssize_t val);
2282 #if !defined(HOST_64BIT)
2283 #if defined(TARGET_X86)
2284 unsigned char emitOutputByte(BYTE* dst, size_t val);
2285 unsigned char emitOutputWord(BYTE* dst, size_t val);
2286 unsigned char emitOutputLong(BYTE* dst, size_t val);
2287 unsigned char emitOutputSizeT(BYTE* dst, size_t val);
2289 unsigned char emitOutputByte(BYTE* dst, unsigned __int64 val);
2290 unsigned char emitOutputWord(BYTE* dst, unsigned __int64 val);
2291 unsigned char emitOutputLong(BYTE* dst, unsigned __int64 val);
2292 unsigned char emitOutputSizeT(BYTE* dst, unsigned __int64 val);
2293 #endif // defined(TARGET_X86)
2294 #endif // !defined(HOST_64BIT)
2296 #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
2297 unsigned int emitCounts_INS_OPTS_J;
2298 #endif // TARGET_LOONGARCH64 || TARGET_RISCV64
2300 instrDesc* emitFirstInstrDesc(BYTE* idData) const;
2301 void emitAdvanceInstrDesc(instrDesc** id, size_t idSize) const;
2302 size_t emitIssue1Instr(insGroup* ig, instrDesc* id, BYTE** dp);
2303 size_t emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp);
2305 bool emitHasFramePtr;
2307 #ifdef PSEUDORANDOM_NOP_INSERTION
2308 bool emitInInstrumentation;
2309 #endif // PSEUDORANDOM_NOP_INSERTION
2312 bool emitChkAlign; // perform some alignment checks
2315 insGroup* emitCurIG;
2317 void emitSetShortJump(instrDescJmp* id);
2318 void emitSetMediumJump(instrDescJmp* id);
2321 CORINFO_FIELD_HANDLE emitBlkConst(const void* cnsAddr, unsigned cnsSize, unsigned cnsAlign, var_types elemType);
2324 #if defined(TARGET_AMD64)
2325 regMaskTP rbmFltCalleeTrash;
2327 FORCEINLINE regMaskTP get_RBM_FLT_CALLEE_TRASH() const
2329 return this->rbmFltCalleeTrash;
2331 #endif // TARGET_AMD64
2333 #if defined(TARGET_XARCH)
2334 regMaskTP rbmMskCalleeTrash;
2336 FORCEINLINE regMaskTP get_RBM_MSK_CALLEE_TRASH() const
2338 return this->rbmMskCalleeTrash;
2340 #endif // TARGET_AMD64
2342 CORINFO_FIELD_HANDLE emitFltOrDblConst(double constValue, emitAttr attr);
2343 #if defined(FEATURE_SIMD)
2344 CORINFO_FIELD_HANDLE emitSimd8Const(simd8_t constValue);
2345 CORINFO_FIELD_HANDLE emitSimd16Const(simd16_t constValue);
2346 #if defined(TARGET_XARCH)
2347 CORINFO_FIELD_HANDLE emitSimd32Const(simd32_t constValue);
2348 CORINFO_FIELD_HANDLE emitSimd64Const(simd64_t constValue);
2349 #endif // TARGET_XARCH
2350 #endif // FEATURE_SIMD
2351 regNumber emitInsBinary(instruction ins, emitAttr attr, GenTree* dst, GenTree* src);
2352 regNumber emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, GenTree* src1, GenTree* src2);
2353 void emitInsLoadInd(instruction ins, emitAttr attr, regNumber dstReg, GenTreeIndir* mem);
2354 void emitInsStoreInd(instruction ins, emitAttr attr, GenTreeStoreInd* mem);
2355 void emitInsStoreLcl(instruction ins, emitAttr attr, GenTreeLclVarCommon* varNode);
2356 insFormat emitMapFmtForIns(insFormat fmt, instruction ins);
2357 insFormat emitMapFmtAtoM(insFormat fmt);
2358 void emitHandleMemOp(GenTreeIndir* indir, instrDesc* id, insFormat fmt, instruction ins);
2359 void spillIntArgRegsToShadowSlots();
2362 bool emitIsInstrWritingToReg(instrDesc* id, regNumber reg);
2363 bool emitDoesInsModifyFlags(instruction ins);
2364 #endif // TARGET_XARCH
2366 /************************************************************************/
2367 /* The logic that creates and keeps track of instruction groups */
2368 /************************************************************************/
2370 // The IG buffer size is the size, in bytes, of the single, global instruction group buffer.
2371 // It is computed dynamically based on SC_IG_BUFFER_NUM_SMALL_DESCS, SC_IG_BUFFER_NUM_LARGE_DESCS,
2372 // and whether a debug info pointer is being saved.
2374 // When a label is reached, or the buffer is filled, the precise amount of the buffer that was
2375 // used is copied to a newly allocated, precisely sized buffer, and the global buffer is reset
2376 // for use with the next set of instructions (see emitSavIG). If the buffer was filled before
2377 // reaching a label, the next instruction group will be an "overflow", or "extension" group
2378 // (marked with IGF_EXTEND). Thus, the size of the global buffer shouldn't matter (as long as it
2379 // can hold at least one of the largest instruction descriptor forms), since we can always overflow
2380 // to subsequent instruction groups.
2382 // The only place where this fixed instruction group size is a problem is in the main function prolog,
2383 // where we only support a single instruction group, and no extension groups. We should really fix that.
2384 // Thus, the buffer size needs to be large enough to hold the maximum number of instructions that
2385 // can possibly be generated into the prolog instruction group. That is difficult to statically determine.
2387 // If we do generate an overflow prolog group, we will hit a NOWAY assert and fall back to MinOpts.
2388 // This should reduce the number of instructions generated into the prolog.
2390 // Note that OSR prologs require additional code not seen in normal prologs.
2392 // Also, note that DEBUG and non-DEBUG builds have different instrDesc sizes, and there are multiple
2393 // sizes of instruction descriptors, so the number of instructions that will fit in the largest
2394 // instruction group depends on the instruction mix as well as DEBUG/non-DEBUG build type. See the
2395 // EMITTER_STATS output for various statistics related to this.
2397 CLANG_FORMAT_COMMENT_ANCHOR;
2399 #if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
2400 // ARM32/64, LoongArch and RISC-V can require a bigger prolog instruction group. One scenario
2401 // is where a function uses all the incoming integer and single-precision floating-point arguments,
2402 // and must store them all to the frame on entry. If the frame is very large, we generate
2407 // for each store, or, to load arguments into registers:
2408 // movz xip1, #0x6cd0
2409 // movk xip1, #2 LSL #16
2410 // ldr w8, [fp, xip1] // [V10 arg10]
2411 // which eats up our insGroup buffer.
2412 #define SC_IG_BUFFER_NUM_SMALL_DESCS 0
2413 #define SC_IG_BUFFER_NUM_LARGE_DESCS 200
2416 #define SC_IG_BUFFER_NUM_SMALL_DESCS 14
2417 #define SC_IG_BUFFER_NUM_LARGE_DESCS 50
2418 #endif // !(TARGET_ARMARCH || TARGET_LOONGARCH64 || TARGET_RISCV64)
2420 size_t emitIGbuffSize;
2422 insGroup* emitIGlist; // first instruction group
2423 insGroup* emitIGlast; // last instruction group
2424 insGroup* emitIGthis; // issued instruction group
2426 insGroup* emitPrologIG; // prolog instruction group
2428 instrDescJmp* emitJumpList; // list of local jumps in method
2429 instrDescJmp* emitJumpLast; // last of local jumps in method
2430 void emitJumpDistBind(); // Bind all the local jumps in method
2431 bool emitContainsRemovableJmpCandidates;
2432 void emitRemoveJumpToNextInst(); // try to remove unconditional jumps to the next instruction
2434 #if FEATURE_LOOP_ALIGN
2435 instrDescAlign* emitCurIGAlignList; // list of align instructions in current IG
2436 unsigned emitLastLoopStart; // Start IG of last inner loop
2437 unsigned emitLastLoopEnd; // End IG of last inner loop
2438 unsigned emitLastAlignedIgNum; // last IG that has align instruction
2439 instrDescAlign* emitAlignList; // list of all align instructions in method
2440 instrDescAlign* emitAlignLast; // last align instruction in method
2442 // Points to the most recent added align instruction. If there are multiple align instructions like in arm64 or
2443 // non-adaptive alignment on xarch, this points to the first align instruction of the series of align instructions.
2444 instrDescAlign* emitAlignLastGroup;
2446 unsigned getLoopSize(insGroup* igLoopHeader,
2447 unsigned maxLoopSize DEBUG_ARG(bool isAlignAdjusted) DEBUG_ARG(UNATIVE_OFFSET containingIGNum)
2448 DEBUG_ARG(UNATIVE_OFFSET loopHeadPredIGNum)); // Get the smallest loop size
2449 void emitLoopAlignment(DEBUG_ARG1(bool isPlacedBehindJmp));
2450 bool emitEndsWithAlignInstr(); // Validate if newLabel is appropriate
2451 void emitSetLoopBackEdge(BasicBlock* loopTopBlock);
2452 void emitLoopAlignAdjustments(); // Predict if loop alignment is needed and make appropriate adjustments
2453 unsigned emitCalculatePaddingForLoopAlignment(insGroup* ig,
2454 size_t offset DEBUG_ARG(bool isAlignAdjusted)
2455 DEBUG_ARG(UNATIVE_OFFSET containingIGNum)
2456 DEBUG_ARG(UNATIVE_OFFSET loopHeadPredIGNum));
2458 void emitLoopAlign(unsigned paddingBytes, bool isFirstAlign DEBUG_ARG(bool isPlacedBehindJmp));
2459 void emitLongLoopAlign(unsigned alignmentBoundary DEBUG_ARG(bool isPlacedBehindJmp));
2460 instrDescAlign* emitAlignInNextIG(instrDescAlign* alignInstr);
2461 void emitConnectAlignInstrWithCurIG();
2465 void emitCheckFuncletBranch(instrDesc* jmp, insGroup* jmpIG); // Check for illegal branches between funclets
2467 bool emitFwdJumps; // forward jumps present?
2468 unsigned emitNoGCRequestCount; // Count of number of nested "NO GC" region requests we have.
2469 bool emitNoGCIG; // Are we generating IGF_NOGCINTERRUPT insGroups (for prologs, epilogs, etc.)
2470 bool emitForceNewIG; // If we generate an instruction, and not another instruction group, force create a new emitAdd
2471 // instruction group.
2473 BYTE* emitCurIGfreeNext; // next available byte in buffer
2474 BYTE* emitCurIGfreeEndp; // one byte past the last available byte in buffer
2475 BYTE* emitCurIGfreeBase; // first byte address
2477 unsigned emitCurIGinsCnt; // # of collected instr's in buffer
2478 unsigned emitCurIGsize; // estimated code size of current group in bytes
2479 UNATIVE_OFFSET emitCurCodeOffset; // current code offset within group
2480 UNATIVE_OFFSET emitTotalCodeSize; // bytes of code in entire method
2482 insGroup* emitFirstColdIG; // first cold instruction group
2484 void emitSetFirstColdIGCookie(void* bbEmitCookie)
2486 emitFirstColdIG = (insGroup*)bbEmitCookie;
2489 int emitOffsAdj; // current code offset adjustment
2491 instrDescJmp* emitCurIGjmpList; // list of jumps in current IG
2493 // emitPrev* and emitInit* are only used during code generation, not during
2494 // emission (issuing), to determine what GC values to store into an IG.
2495 // Note that only the Vars ones are actually used, apparently due to bugs
2496 // in that tracking. See emitSavIG(): the important use of ByrefRegs is commented
2497 // out, and GCrefRegs is always saved.
2499 VARSET_TP emitPrevGCrefVars;
2500 regMaskTP emitPrevGCrefRegs;
2501 regMaskTP emitPrevByrefRegs;
2503 VARSET_TP emitInitGCrefVars;
2504 regMaskTP emitInitGCrefRegs;
2505 regMaskTP emitInitByrefRegs;
2507 // If this is set, we ignore comparing emitPrev* and emitInit* to determine
2508 // whether to save GC state (to save space in the IG), and always save it.
2510 bool emitForceStoreGCState;
2512 // emitThis* variables are used during emission, to track GC updates
2513 // on a per-instruction basis. During code generation, per-instruction
2514 // tracking is done with variables gcVarPtrSetCur, gcRegGCrefSetCur,
2515 // and gcRegByrefSetCur. However, these are also used for a slightly
2516 // different purpose during code generation: to try to minimize the
2517 // amount of GC data stored to an IG, by only storing deltas from what
2518 // we expect to see at an IG boundary. Also, only emitThisGCrefVars is
2519 // really the only one used; the others seem to be calculated, but not
2520 // used due to bugs.
2522 VARSET_TP emitThisGCrefVars;
2523 regMaskTP emitThisGCrefRegs; // Current set of registers holding GC references
2524 regMaskTP emitThisByrefRegs; // Current set of registers holding BYREF references
2526 bool emitThisGCrefVset; // Is "emitThisGCrefVars" up to date?
2528 regNumber emitSyncThisObjReg; // where is "this" enregistered for synchronized methods?
2530 #if MULTIREG_HAS_SECOND_GC_RET
2531 void emitSetSecondRetRegGCType(instrDescCGCA* id, emitAttr secondRetSize);
2532 #endif // MULTIREG_HAS_SECOND_GC_RET
2534 static void emitEncodeCallGCregs(regMaskTP regs, instrDesc* id);
2535 static unsigned emitDecodeCallGCregs(instrDesc* id);
2537 unsigned emitNxtIGnum;
2539 #ifdef PSEUDORANDOM_NOP_INSERTION
2541 // random nop insertion to break up nop sleds
2542 unsigned emitNextNop;
2543 bool emitRandomNops;
2545 void emitEnableRandomNops()
2547 emitRandomNops = true;
2549 void emitDisableRandomNops()
2551 emitRandomNops = false;
2554 #endif // PSEUDORANDOM_NOP_INSERTION
2556 insGroup* emitAllocAndLinkIG();
2557 insGroup* emitAllocIG();
2558 void emitInitIG(insGroup* ig);
2559 void emitInsertIGAfter(insGroup* insertAfterIG, insGroup* ig);
2563 #if !defined(JIT32_GCENCODER)
2564 void emitDisableGC();
2565 void emitEnableGC();
2566 #endif // !defined(JIT32_GCENCODER)
2568 #if defined(TARGET_XARCH)
2569 static bool emitAlignInstHasNoCode(instrDesc* id);
2570 static bool emitInstHasNoCode(instrDesc* id);
2571 static bool emitJmpInstHasNoCode(instrDesc* id);
2574 void emitGenIG(insGroup* ig);
2575 insGroup* emitSavIG(bool emitAdd = false);
2576 void emitNxtIG(bool extend = false);
2579 void emitRemoveLastInstruction();
2582 bool emitCurIGnonEmpty()
2584 return (emitCurIG && emitCurIGfreeNext > emitCurIGfreeBase);
2587 #define EMIT_MAX_IG_INS_COUNT 256
2589 #if EMIT_BACKWARDS_NAVIGATION
2590 #define EMIT_MAX_PEEPHOLE_INS_COUNT 32 // The max number of previous instructions to navigate through for peepholes.
2591 #endif // EMIT_BACKWARDS_NAVIGATION
2593 instrDesc* emitLastIns;
2594 insGroup* emitLastInsIG;
2596 #if EMIT_BACKWARDS_NAVIGATION
2597 unsigned emitLastInsFullSize;
2598 #endif // EMIT_BACKWARDS_NAVIGATION
2600 // Check to see if the last instruction is available.
2601 inline bool emitHasLastIns() const
2603 return (emitLastIns != nullptr);
2606 // Checks to see if we can cross between the two given IG boundaries.
2608 // We have the following checks:
2609 // 1. Looking backwards across an IG boundary can only be done if we're in an extension IG.
2610 // 2. The IG of the previous instruction must have the same GC interrupt status as the current IG.
2611 inline bool isInsIGSafeForPeepholeOptimization(insGroup* prevInsIG, insGroup* curInsIG) const
2613 if (prevInsIG == curInsIG)
2619 return (curInsIG->igFlags & IGF_EXTEND) &&
2620 ((prevInsIG->igFlags & IGF_NOGCINTERRUPT) == (curInsIG->igFlags & IGF_NOGCINTERRUPT));
2624 // Check if a peephole optimization involving emitLastIns is safe.
2626 // We have the following checks:
2627 // 1. There must be a non-null emitLastIns to consult (thus, we have a known "last" instruction).
2628 // 2. `emitForceNewIG` is not set: this prevents peepholes from crossing nogc boundaries where
2629 // the next instruction is forced to create a new IG.
2630 bool emitCanPeepholeLastIns() const
2632 assert(emitHasLastIns() == (emitLastInsIG != nullptr));
2634 return emitHasLastIns() && // there is an emitLastInstr
2635 !emitForceNewIG && // and we're not about to start a new IG.
2636 isInsIGSafeForPeepholeOptimization(emitLastInsIG, emitCurIG);
2639 enum emitPeepholeResult
2645 // Visits the last emitted instructions.
2646 // Must be safe to do - use emitCanPeepholeLastIns for checking.
2647 // Action is a function type: instrDesc* -> emitPeepholeResult
2648 template <typename Action>
2649 void emitPeepholeIterateLastInstrs(Action action)
2651 assert(emitCanPeepholeLastIns());
2653 #if EMIT_BACKWARDS_NAVIGATION
2657 if (!emitGetLastIns(&curInsIG, &id))
2660 for (unsigned i = 0; i < EMIT_MAX_PEEPHOLE_INS_COUNT; i++)
2662 assert(id != nullptr);
2666 case PEEPHOLE_ABORT:
2668 case PEEPHOLE_CONTINUE:
2670 insGroup* savedInsIG = curInsIG;
2671 if (emitPrevID(curInsIG, id))
2673 if (isInsIGSafeForPeepholeOptimization(curInsIG, savedInsIG))
2689 #else // EMIT_BACKWARDS_NAVIGATION
2690 action(emitLastIns);
2691 #endif // !EMIT_BACKWARDS_NAVIGATION
2694 #ifdef TARGET_ARMARCH
2695 instrDesc* emitLastMemBarrier;
2699 void emitCheckIGList();
2702 // Terminates any in-progress instruction group, making the current IG a new empty one.
2703 // Mark this instruction group as having a label; return the new instruction group.
2704 // Sets the emitter's record of the currently live GC variables
2705 // and registers. The "isFinallyTarget" parameter indicates that the current location is
2706 // the start of a basic block that is returned to after a finally clause in non-exceptional execution.
2707 void* emitAddLabel(VARSET_VALARG_TP GCvars,
2708 regMaskTP gcrefRegs,
2709 regMaskTP byrefRegs,
2710 bool isFinallyTarget = false DEBUG_ARG(BasicBlock* block = nullptr));
2712 // Same as above, except the label is added and is conceptually "inline" in
2713 // the current block. Thus it extends the previous block and the emitter
2714 // continues to track GC info as if there was no label.
2715 void* emitAddInlineLabel();
2717 void emitPrintLabel(const insGroup* ig) const;
2718 const char* emitLabelString(const insGroup* ig) const;
2720 #if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
2722 void emitGetInstrDescs(insGroup* ig, instrDesc** id, int* insCnt);
2724 bool emitGetLocationInfo(emitLocation* emitLoc, insGroup** pig, instrDesc** pid, int* pinsRemaining = NULL);
2726 bool emitNextID(insGroup*& ig, instrDesc*& id, int& insRemaining);
2728 typedef void (*emitProcessInstrFunc_t)(instrDesc* id, void* context);
2730 void emitWalkIDs(emitLocation* locFrom, emitProcessInstrFunc_t processFunc, void* context);
2732 static void emitGenerateUnwindNop(instrDesc* id, void* context);
2734 #endif // TARGET_ARMARCH || TARGET_LOONGARCH64
2736 #if EMIT_BACKWARDS_NAVIGATION
2737 bool emitPrevID(insGroup*& ig, instrDesc*& id);
2738 bool emitGetLastIns(insGroup** pig, instrDesc** pid);
2739 #endif // EMIT_BACKWARDS_NAVIGATION
2742 void emitMarkStackLvl(unsigned stackLevel);
2745 #if defined(FEATURE_SIMD)
2746 void emitStoreSimd12ToLclOffset(unsigned varNum, unsigned offset, regNumber dataReg, GenTree* tmpRegProvider);
2747 #endif // FEATURE_SIMD
2749 int emitNextRandomNop();
2752 // Functions for allocating instrDescs.
2754 // The emitAllocXXX functions are the base level that allocate memory, and do little else.
2755 // The emitters themselves use emitNewXXX, which might be thin wrappers over the emitAllocXXX functions.
2758 void* emitAllocAnyInstr(size_t sz, emitAttr attr);
2760 instrDesc* emitAllocInstr(emitAttr attr)
2763 emitTotalIDescCnt++;
2764 #endif // EMITTER_STATS
2765 return (instrDesc*)emitAllocAnyInstr(sizeof(instrDesc), attr);
2768 instrDescJmp* emitAllocInstrJmp()
2771 emitTotalIDescJmpCnt++;
2772 #endif // EMITTER_STATS
2773 return (instrDescJmp*)emitAllocAnyInstr(sizeof(instrDescJmp), EA_1BYTE);
2776 #if !defined(TARGET_ARM64)
2777 instrDescLbl* emitAllocInstrLbl()
2780 emitTotalIDescLblCnt++;
2781 #endif // EMITTER_STATS
2782 return (instrDescLbl*)emitAllocAnyInstr(sizeof(instrDescLbl), EA_4BYTE);
2784 #endif // TARGET_ARM64
2786 #if defined(TARGET_ARM64)
2787 instrDescLclVarPair* emitAllocInstrLclVarPair(emitAttr attr)
2790 emitTotalIDescLclVarPairCnt++;
2791 #endif // EMITTER_STATS
2792 instrDescLclVarPair* result = (instrDescLclVarPair*)emitAllocAnyInstr(sizeof(instrDescLclVarPair), attr);
2793 result->idSetIsLclVarPair();
2797 instrDescLclVarPairCns* emitAllocInstrLclVarPairCns(emitAttr attr, cnsval_size_t cns)
2800 emitTotalIDescLclVarPairCnsCnt++;
2801 #endif // EMITTER_STATS
2802 instrDescLclVarPairCns* result =
2803 (instrDescLclVarPairCns*)emitAllocAnyInstr(sizeof(instrDescLclVarPairCns), attr);
2804 result->idSetIsLargeCns();
2805 result->idSetIsLclVarPair();
2806 result->idcCnsVal = cns;
2809 #endif // TARGET_ARM64
2811 instrDescCns* emitAllocInstrCns(emitAttr attr)
2814 emitTotalIDescCnsCnt++;
2815 #endif // EMITTER_STATS
2816 return (instrDescCns*)emitAllocAnyInstr(sizeof(instrDescCns), attr);
2819 instrDescCns* emitAllocInstrCns(emitAttr attr, cnsval_size_t cns)
2821 instrDescCns* result = emitAllocInstrCns(attr);
2822 result->idSetIsLargeCns();
2823 result->idcCnsVal = cns;
2827 instrDescDsp* emitAllocInstrDsp(emitAttr attr)
2830 emitTotalIDescDspCnt++;
2831 #endif // EMITTER_STATS
2832 return (instrDescDsp*)emitAllocAnyInstr(sizeof(instrDescDsp), attr);
2835 instrDescCnsDsp* emitAllocInstrCnsDsp(emitAttr attr)
2838 emitTotalIDescCnsDspCnt++;
2839 #endif // EMITTER_STATS
2840 return (instrDescCnsDsp*)emitAllocAnyInstr(sizeof(instrDescCnsDsp), attr);
2845 instrDescAmd* emitAllocInstrAmd(emitAttr attr)
2848 emitTotalIDescAmdCnt++;
2849 #endif // EMITTER_STATS
2850 return (instrDescAmd*)emitAllocAnyInstr(sizeof(instrDescAmd), attr);
2853 instrDescCnsAmd* emitAllocInstrCnsAmd(emitAttr attr)
2856 emitTotalIDescCnsAmdCnt++;
2857 #endif // EMITTER_STATS
2858 return (instrDescCnsAmd*)emitAllocAnyInstr(sizeof(instrDescCnsAmd), attr);
2861 #endif // TARGET_XARCH
2863 instrDescCGCA* emitAllocInstrCGCA(emitAttr attr)
2866 emitTotalIDescCGCACnt++;
2867 #endif // EMITTER_STATS
2868 return (instrDescCGCA*)emitAllocAnyInstr(sizeof(instrDescCGCA), attr);
2871 #if FEATURE_LOOP_ALIGN
2872 instrDescAlign* emitAllocInstrAlign()
2875 emitTotalIDescAlignCnt++;
2876 #endif // EMITTER_STATS
2877 return (instrDescAlign*)emitAllocAnyInstr(sizeof(instrDescAlign), EA_1BYTE);
2879 instrDescAlign* emitNewInstrAlign();
2882 instrDesc* emitNewInstrSmall(emitAttr attr);
2883 instrDesc* emitNewInstr(emitAttr attr = EA_4BYTE);
2884 instrDesc* emitNewInstrSC(emitAttr attr, cnsval_ssize_t cns);
2885 instrDesc* emitNewInstrCns(emitAttr attr, cnsval_ssize_t cns);
2886 instrDesc* emitNewInstrDsp(emitAttr attr, target_ssize_t dsp);
2887 instrDesc* emitNewInstrCnsDsp(emitAttr attr, target_ssize_t cns, int dsp);
2889 instrDesc* emitNewInstrReloc(emitAttr attr, BYTE* addr);
2890 #endif // TARGET_ARM
2891 instrDescJmp* emitNewInstrJmp();
2893 #if !defined(TARGET_ARM64)
2894 instrDescLbl* emitNewInstrLbl();
2896 instrDesc* emitNewInstrLclVarPair(emitAttr attr, cnsval_ssize_t cns);
2897 #endif // !TARGET_ARM64
2899 static const BYTE emitFmtToOps[];
2902 static const unsigned emitFmtCount;
2905 bool emitIsSmallInsDsc(instrDesc* id) const;
2907 size_t emitSizeOfInsDsc(instrDesc* id) const;
2909 /************************************************************************/
2910 /* The following keeps track of stack-based GC values */
2911 /************************************************************************/
2913 unsigned emitTrkVarCnt;
2914 int* emitGCrFrameOffsTab; // Offsets of tracked stack ptr vars (varTrkIndex -> stkOffs)
2916 unsigned emitGCrFrameOffsCnt; // Number of tracked stack ptr vars
2917 int emitGCrFrameOffsMin; // Min offset of a tracked stack ptr var
2918 int emitGCrFrameOffsMax; // Max offset of a tracked stack ptr var
2919 bool emitContTrkPtrLcls; // All lcl between emitGCrFrameOffsMin/Max are only tracked stack ptr vars
2920 varPtrDsc** emitGCrFrameLiveTab; // Cache of currently live varPtrs (stkOffs -> varPtrDsc)
2922 int emitArgFrameOffsMin;
2923 int emitArgFrameOffsMax;
2925 int emitLclFrameOffsMin;
2926 int emitLclFrameOffsMax;
2928 int emitSyncThisObjOffs; // what is the offset of "this" for synchronized methods?
2931 void emitSetFrameRangeGCRs(int offsLo, int offsHi);
2932 void emitSetFrameRangeLcls(int offsLo, int offsHi);
2933 void emitSetFrameRangeArgs(int offsLo, int offsHi);
2935 bool emitIsWithinFrameRangeGCRs(int offs)
2937 return (offs >= emitGCrFrameOffsMin) && (offs < emitGCrFrameOffsMax);
2940 static instruction emitJumpKindToIns(emitJumpKind jumpKind);
2941 static emitJumpKind emitInsToJumpKind(instruction ins);
2942 static emitJumpKind emitReverseJumpKind(emitJumpKind jumpKind);
2945 #ifndef TARGET_LOONGARCH64
2946 void emitInsSanityCheck(instrDesc* id);
2950 #ifdef TARGET_ARMARCH
2951 // Returns true if instruction "id->idIns()" writes to a register that might be used to contain a GC
2952 // pointer. This exempts the SP and PC registers, and floating point registers. Memory access
2953 // instructions that pre- or post-increment their memory address registers are *not* considered to write
2954 // to GC registers, even if that memory address is a by-ref: such an instruction cannot change the GC
2955 // status of that register, since it must be a byref before and remains one after.
2957 // This may return false positives.
2958 bool emitInsMayWriteToGCReg(instrDesc* id);
2960 // Returns "true" if instruction "id->idIns()" writes to a LclVar stack location.
2961 bool emitInsWritesToLclVarStackLoc(instrDesc* id);
2963 // Returns true if the instruction may write to more than one register.
2964 bool emitInsMayWriteMultipleRegs(instrDesc* id);
2966 // Returns "true" if instruction "id->idIns()" writes to a LclVar stack slot pair.
2967 bool emitInsWritesToLclVarStackLocPair(instrDesc* id);
2968 #elif defined(TARGET_LOONGARCH64)
2969 bool emitInsMayWriteToGCReg(instruction ins);
2970 bool emitInsWritesToLclVarStackLoc(instrDesc* id);
2971 #elif defined(TARGET_RISCV64)
2972 bool emitInsMayWriteToGCReg(instruction ins);
2973 bool emitInsWritesToLclVarStackLoc(instrDesc* id);
2974 #endif // TARGET_LOONGARCH64
2976 /************************************************************************/
2977 /* The following is used to distinguish helper vs non-helper calls */
2978 /************************************************************************/
2980 static bool emitNoGChelper(CorInfoHelpFunc helpFunc);
2981 static bool emitNoGChelper(CORINFO_METHOD_HANDLE methHnd);
2983 /************************************************************************/
2984 /* The following logic keeps track of live GC ref values */
2985 /************************************************************************/
2987 bool emitFullArgInfo; // full arg info (including non-ptr arg)?
2988 bool emitFullGCinfo; // full GC pointer maps?
2989 bool emitFullyInt; // fully interruptible code?
2991 regMaskTP emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd);
2993 // Gets a register mask that represent the kill set for a NoGC helper call.
2994 regMaskTP emitGetGCRegsKilledByNoGCCall(CorInfoHelpFunc helper);
2996 #if EMIT_TRACK_STACK_DEPTH
2997 unsigned emitCntStackDepth; // 0 in prolog/epilog, One DWORD elsewhere
2998 unsigned emitMaxStackDepth; // actual computed max. stack depth
3001 /* Stack modelling wrt GC */
3003 bool emitSimpleStkUsed; // using the "simple" stack table?
3006 struct // if emitSimpleStkUsed==true
3008 #define BITS_IN_BYTE (8)
3009 #define MAX_SIMPLE_STK_DEPTH (BITS_IN_BYTE * sizeof(unsigned))
3011 unsigned emitSimpleStkMask; // bit per pushed dword (if it fits. Lowest bit <==> last pushed arg)
3012 unsigned emitSimpleByrefStkMask; // byref qualifier for emitSimpleStkMask
3015 struct // if emitSimpleStkUsed==false
3017 BYTE emitArgTrackLcl[16]; // small local table to avoid malloc
3018 BYTE* emitArgTrackTab; // base of the argument tracking stack
3019 BYTE* emitArgTrackTop; // top of the argument tracking stack
3020 USHORT emitGcArgTrackCnt; // count of pending arg records (stk-depth for frameless methods, gc ptrs on stk
3021 // for framed methods)
3025 unsigned emitCurStackLvl; // amount of bytes pushed on stack
3027 #if EMIT_TRACK_STACK_DEPTH
3028 /* Functions for stack tracking */
3030 void emitStackPush(BYTE* addr, GCtype gcType);
3032 void emitStackPushN(BYTE* addr, unsigned count);
3034 void emitStackPop(BYTE* addr, bool isCall, unsigned char callInstrSize, unsigned count = 1);
3036 void emitStackKillArgs(BYTE* addr, unsigned count, unsigned char callInstrSize);
3038 void emitRecordGCcall(BYTE* codePos, unsigned char callInstrSize);
3040 // Helpers for the above
3042 void emitStackPushLargeStk(BYTE* addr, GCtype gcType, unsigned count = 1);
3043 void emitStackPopLargeStk(BYTE* addr, bool isCall, unsigned char callInstrSize, unsigned count = 1);
3044 #endif // EMIT_TRACK_STACK_DEPTH
3046 /* Liveness of stack variables, and registers */
3048 void emitUpdateLiveGCvars(VARSET_VALARG_TP vars, BYTE* addr);
3049 void emitUpdateLiveGCregs(GCtype gcType, regMaskTP regs, BYTE* addr);
3052 const char* emitGetFrameReg();
3053 void emitDispRegSet(regMaskTP regs);
3054 void emitDispVarSet();
3057 void emitGCregLiveUpd(GCtype gcType, regNumber reg, BYTE* addr);
3058 void emitGCregLiveSet(GCtype gcType, regMaskTP mask, BYTE* addr, bool isThis);
3059 void emitGCregDeadUpdMask(regMaskTP, BYTE* addr);
3060 void emitGCregDeadUpd(regNumber reg, BYTE* addr);
3061 void emitGCregDeadSet(GCtype gcType, regMaskTP mask, BYTE* addr);
3063 void emitGCvarLiveUpd(int offs, int varNum, GCtype gcType, BYTE* addr DEBUG_ARG(unsigned actualVarNum));
3064 void emitGCvarLiveSet(int offs, GCtype gcType, BYTE* addr, ssize_t disp = -1);
3065 void emitGCvarDeadUpd(int offs, BYTE* addr DEBUG_ARG(unsigned varNum));
3066 void emitGCvarDeadSet(int offs, BYTE* addr, ssize_t disp = -1);
3068 GCtype emitRegGCtype(regNumber reg);
3070 // We have a mixture of code emission methods, some of which return the size of the emitted instruction,
3071 // requiring the caller to add this to the current code pointer (dst += <call to emit code>), others of which
3072 // return the updated code pointer (dst = <call to emit code>). Sometimes we'd like to get the size of
3073 // the generated instruction for the latter style. This method accomplishes that --
3074 // "emitCodeWithInstructionSize(dst, <call to emitCode>, &instrSize)" will do the call, and set
3075 // "*instrSize" to the after-before code pointer difference. Returns the result of the call. (And
3076 // asserts that the instruction size fits in an unsigned char.)
3077 static BYTE* emitCodeWithInstructionSize(BYTE* codePtrBefore, BYTE* newCodePointer, unsigned char* instrSize);
3079 /************************************************************************/
3080 /* The following logic keeps track of initialized data sections */
3081 /************************************************************************/
3083 /* One of these is allocated for every blob of initialized data */
3087 // Note to use alignments greater than 64 requires modification in the VM
3088 // to support larger alignments (see ICorJitInfo::allocMem)
3090 const static unsigned MIN_DATA_ALIGN = 4;
3091 const static unsigned MAX_DATA_ALIGN = 64;
3100 dataSection* dsNext;
3101 UNATIVE_OFFSET dsSize;
3103 var_types dsDataType;
3105 // variable-sized array used to store the constant data
3106 // or BasicBlock* array in the block cases.
3110 /* These describe the entire initialized/uninitialized data sections */
3114 dataSection* dsdList;
3115 dataSection* dsdLast;
3116 UNATIVE_OFFSET dsdOffs;
3117 UNATIVE_OFFSET alignment; // in bytes, defaults to 4
3119 dataSecDsc() : dsdList(nullptr), dsdLast(nullptr), dsdOffs(0), alignment(4)
3124 dataSecDsc emitConsDsc;
3126 dataSection* emitDataSecCur;
3128 void emitOutputDataSec(dataSecDsc* sec, BYTE* dst);
3129 void emitDispDataSec(dataSecDsc* section, BYTE* dst);
3131 /************************************************************************/
3132 /* Handles to the current class and method. */
3133 /************************************************************************/
3135 COMP_HANDLE emitCmpHandle;
3137 /************************************************************************/
3138 /* Helpers for interface to EE */
3139 /************************************************************************/
3143 #define emitRecordRelocation(location, target, fRelocType) \
3144 emitRecordRelocationHelp(location, target, fRelocType, #fRelocType)
3146 #define emitRecordRelocationWithAddlDelta(location, target, fRelocType, addlDelta) \
3147 emitRecordRelocationHelp(location, target, fRelocType, #fRelocType, addlDelta)
3149 void emitRecordRelocationHelp(void* location, /* IN */
3150 void* target, /* IN */
3151 uint16_t fRelocType, /* IN */
3152 const char* relocTypeName, /* IN */
3153 int32_t addlDelta = 0); /* IN */
3157 void emitRecordRelocationWithAddlDelta(void* location, /* IN */
3158 void* target, /* IN */
3159 uint16_t fRelocType, /* IN */
3160 int32_t addlDelta) /* IN */
3162 emitRecordRelocation(location, target, fRelocType, addlDelta);
3165 void emitRecordRelocation(void* location, /* IN */
3166 void* target, /* IN */
3167 uint16_t fRelocType, /* IN */
3168 int32_t addlDelta = 0); /* IN */
3173 void emitHandlePCRelativeMov32(void* location, /* IN */
3174 void* target); /* IN */
3177 void emitRecordCallSite(ULONG instrOffset, /* IN */
3178 CORINFO_SIG_INFO* callSig, /* IN */
3179 CORINFO_METHOD_HANDLE methodHandle); /* IN */
3182 // This is a scratch buffer used to minimize the number of sig info structs
3183 // we have to allocate for recordCallSite.
3184 CORINFO_SIG_INFO* emitScratchSigInfo;
3187 /************************************************************************/
3188 /* Logic to collect and display statistics */
3189 /************************************************************************/
3193 friend void emitterStats(FILE* fout);
3194 friend void emitterStaticStats(FILE* fout);
3196 static size_t emitSizeMethod;
3198 static unsigned emitTotalInsCnt;
3200 static unsigned emitCurPrologInsCnt; // current number of prolog instrDescs
3201 static size_t emitCurPrologIGSize; // current size of prolog instrDescs
3202 static unsigned emitMaxPrologInsCnt; // maximum number of prolog instrDescs
3203 static size_t emitMaxPrologIGSize; // maximum size of prolog instrDescs
3205 static unsigned emitTotalIGcnt; // total number of insGroup allocated
3206 static unsigned emitTotalPhIGcnt; // total number of insPlaceholderGroupData allocated
3207 static unsigned emitTotalIGicnt;
3208 static size_t emitTotalIGsize;
3209 static unsigned emitTotalIGmcnt; // total method count
3210 static unsigned emitTotalIGExtend; // total number of 'emitExtend' (typically overflow) groups
3211 static unsigned emitTotalIGjmps;
3212 static unsigned emitTotalIGptrs;
3214 static unsigned emitTotalIDescSmallCnt;
3215 static unsigned emitTotalIDescCnt;
3216 static unsigned emitTotalIDescCnsCnt;
3217 static unsigned emitTotalIDescDspCnt;
3219 static unsigned emitTotalIDescLclVarPairCnt;
3220 static unsigned emitTotalIDescLclVarPairCnsCnt;
3221 #endif // TARGET_ARM64
3223 static unsigned emitTotalIDescRelocCnt;
3224 #endif // TARGET_ARM
3226 static unsigned emitTotalIDescAmdCnt;
3227 static unsigned emitTotalIDescCnsAmdCnt;
3228 #endif // TARGET_XARCH
3229 static unsigned emitTotalIDescCnsDspCnt;
3230 #if FEATURE_LOOP_ALIGN
3231 static unsigned emitTotalIDescAlignCnt;
3232 #endif // FEATURE_LOOP_ALIGN
3233 static unsigned emitTotalIDescJmpCnt;
3234 #if !defined(TARGET_ARM64)
3235 static unsigned emitTotalIDescLblCnt;
3236 #endif // !defined(TARGET_ARM64)
3237 static unsigned emitTotalIDescCGCACnt;
3239 static size_t emitTotMemAlloc;
3241 static unsigned emitSmallDspCnt;
3242 static unsigned emitLargeDspCnt;
3244 #define SMALL_CNS_TSZ 256
3245 static unsigned emitSmallCns[SMALL_CNS_TSZ];
3246 static unsigned emitSmallCnsCnt;
3247 static unsigned emitLargeCnsCnt;
3248 static unsigned emitInt8CnsCnt;
3249 static unsigned emitInt16CnsCnt;
3250 static unsigned emitInt32CnsCnt;
3251 static unsigned emitNegCnsCnt;
3252 static unsigned emitPow2CnsCnt;
3254 static unsigned emitIFcounts[IF_COUNT];
3256 void TrackCns(cnsval_ssize_t value)
3262 if (value >= INT8_MIN)
3266 else if (value >= INT16_MIN)
3270 else if (value >= INT32_MIN)
3275 else if (value <= INT8_MAX)
3279 else if (value <= INT16_MAX)
3283 else if (value <= INT32_MAX)
3294 void TrackSmallCns(cnsval_ssize_t value)
3296 // We only track a subset of the allowed small constants and
3297 // so we'll split the tracked range between positive/negative
3298 // aggregating those outside the tracked range into the min/max
3301 assert(fitsInSmallCns(value));
3304 if (value >= ((SMALL_CNS_TSZ / 2) - 1))
3306 index = static_cast<uint32_t>(SMALL_CNS_TSZ - 1);
3308 else if (value >= (0 - SMALL_CNS_TSZ / 2))
3310 index = static_cast<uint32_t>(value + (SMALL_CNS_TSZ / 2));
3314 emitSmallCns[index]++;
3319 void TrackLargeCns(cnsval_ssize_t value)
3324 #endif // EMITTER_STATS
3326 /*************************************************************************
3328 * Define any target-dependent emitter members.
3331 #include "emitdef.h"
3333 // It would be better if this were a constructor, but that would entail revamping the allocation
3334 // infrastructure of the entire JIT...
3337 VarSetOps::AssignNoCopy(emitComp, emitPrevGCrefVars, VarSetOps::MakeEmpty(emitComp));
3338 VarSetOps::AssignNoCopy(emitComp, emitInitGCrefVars, VarSetOps::MakeEmpty(emitComp));
3339 VarSetOps::AssignNoCopy(emitComp, emitThisGCrefVars, VarSetOps::MakeEmpty(emitComp));
3341 VarSetOps::AssignNoCopy(emitComp, debugPrevGCrefVars, VarSetOps::MakeEmpty(emitComp));
3342 VarSetOps::AssignNoCopy(emitComp, debugThisGCrefVars, VarSetOps::MakeEmpty(emitComp));
3343 debugPrevRegPtrDsc = nullptr;
3344 debugPrevGCrefRegs = RBM_NONE;
3345 debugPrevByrefRegs = RBM_NONE;
3350 /*****************************************************************************
3352 * Define any target-dependent inlines.
3355 #include "emitinl.h"
3357 inline void emitter::instrDesc::checkSizes()
3359 C_ASSERT(SMALL_IDSC_SIZE == offsetof(instrDesc, _idAddrUnion));
3362 /*****************************************************************************
3364 * Returns true if the given instruction descriptor is a "small
3365 * constant" one (i.e. one of the descriptors that don't have all instrDesc
3366 * fields allocated).
3369 inline bool emitter::emitIsSmallInsDsc(instrDesc* id) const
3371 return id->idIsSmallDsc();
3374 /*****************************************************************************
3376 * Given an instruction, return its "update mode" (RD/WR/RW).
3379 inline insUpdateModes emitter::emitInsUpdateMode(instruction ins)
3382 assert((unsigned)ins < emitInsModeFmtCnt);
3384 return (insUpdateModes)emitInsModeFmtTab[ins];
3387 /*****************************************************************************
3389 * Return the number of epilog blocks generated so far.
3392 inline unsigned emitter::emitGetEpilogCnt()
3394 return emitEpilogCnt;
3397 /*****************************************************************************
3399 * Return the current size of the specified data section.
3402 inline UNATIVE_OFFSET emitter::emitDataSize()
3404 return emitConsDsc.dsdOffs;
3407 /*****************************************************************************
3409 * Return a handle to the current position in the output stream. This can
3410 * be later converted to an actual code offset in bytes.
3413 inline void* emitter::emitCurBlock()
3418 /*****************************************************************************
3420 * The emitCurOffset() method returns a cookie that identifies the current
3421 * position in the instruction stream. Due to things like scheduling (and
3422 * the fact that the final size of some instructions cannot be known until
3423 * the end of code generation), we return a value with the instruction number
3424 * and its estimated offset to the caller.
3427 inline unsigned emitGetInsNumFromCodePos(unsigned codePos)
3429 return (codePos & 0xFFFF);
3432 inline unsigned emitGetInsOfsFromCodePos(unsigned codePos)
3434 return (codePos >> 16);
3437 inline unsigned emitter::emitCurOffset()
3439 return emitSpecifiedOffset(emitCurIGinsCnt, emitCurIGsize);
3442 inline unsigned emitter::emitSpecifiedOffset(unsigned insCount, unsigned igSize)
3444 unsigned codePos = insCount + (igSize << 16);
3446 assert(emitGetInsOfsFromCodePos(codePos) == igSize);
3447 assert(emitGetInsNumFromCodePos(codePos) == insCount);
3452 extern const unsigned short emitTypeSizes[TYP_COUNT];
3455 inline emitAttr emitTypeSize(T type)
3457 assert(TypeGet(type) < TYP_COUNT);
3458 assert(emitTypeSizes[TypeGet(type)] > 0);
3459 return (emitAttr)emitTypeSizes[TypeGet(type)];
3462 extern const unsigned short emitTypeActSz[TYP_COUNT];
3465 inline emitAttr emitActualTypeSize(T type)
3467 assert(TypeGet(type) < TYP_COUNT);
3468 assert(emitTypeActSz[TypeGet(type)] > 0);
3469 return (emitAttr)emitTypeActSz[TypeGet(type)];
3472 /*****************************************************************************
3474 * Convert between an operand size in bytes and a smaller encoding used for
3475 * storage in instruction descriptors.
3478 /* static */ inline emitter::opSize emitter::emitEncodeSize(emitAttr size)
3480 assert((size != EA_UNKNOWN) && ((size & EA_SIZE_MASK) == size));
3481 return static_cast<emitter::opSize>(genLog2(size));
3484 /* static */ inline emitAttr emitter::emitDecodeSize(emitter::opSize ensz)
3486 assert(static_cast<unsigned>(ensz) < OPSZ_COUNT);
3487 return emitSizeDecode[ensz];
3490 /*****************************************************************************
3492 * Little helpers to allocate various flavors of instructions.
3495 inline emitter::instrDesc* emitter::emitNewInstrSmall(emitAttr attr)
3499 id = (instrDesc*)emitAllocAnyInstr(SMALL_IDSC_SIZE, attr);
3500 id->idSetIsSmallDsc();
3503 emitTotalIDescSmallCnt++;
3504 #endif // EMITTER_STATS
3509 inline emitter::instrDesc* emitter::emitNewInstr(emitAttr attr)
3511 // This is larger than the Small Descr
3512 return emitAllocInstr(attr);
3515 inline emitter::instrDescJmp* emitter::emitNewInstrJmp()
3517 return emitAllocInstrJmp();
3520 #if FEATURE_LOOP_ALIGN
3521 inline emitter::instrDescAlign* emitter::emitNewInstrAlign()
3523 instrDescAlign* newInstr = emitAllocInstrAlign();
3524 newInstr->idIns(INS_align);
3527 newInstr->idInsFmt(IF_SN_0A);
3528 newInstr->idInsOpt(INS_OPTS_ALIGN);
3534 #if !defined(TARGET_ARM64)
3535 inline emitter::instrDescLbl* emitter::emitNewInstrLbl()
3537 return emitAllocInstrLbl();
3540 inline emitter::instrDesc* emitter::emitNewInstrLclVarPair(emitAttr attr, cnsval_ssize_t cns)
3543 emitTotalIDescCnt++;
3544 emitTotalIDescCnsCnt++;
3545 #endif // EMITTER_STATS
3547 if (instrDesc::fitsInSmallCns(cns))
3549 instrDescLclVarPair* id = emitAllocInstrLclVarPair(attr);
3550 id->idSmallCns(cns);
3560 instrDescLclVarPairCns* id = emitAllocInstrLclVarPairCns(attr, cns);
3569 #endif // !TARGET_ARM64
3571 inline emitter::instrDesc* emitter::emitNewInstrDsp(emitAttr attr, target_ssize_t dsp)
3575 instrDesc* id = emitAllocInstr(attr);
3585 instrDescDsp* id = emitAllocInstrDsp(attr);
3587 id->idSetIsLargeDsp();
3588 id->iddDspVal = dsp;
3598 /*****************************************************************************
3600 * Allocate an instruction descriptor for an instruction with a constant operand.
3601 * The instruction descriptor uses the idAddrUnion to save additional info
3602 * so the smallest size that this can be is sizeof(instrDesc).
3603 * Note that this very similar to emitter::emitNewInstrSC(), except it never
3604 * allocates a small descriptor.
3606 inline emitter::instrDesc* emitter::emitNewInstrCns(emitAttr attr, cnsval_ssize_t cns)
3608 if (instrDesc::fitsInSmallCns(cns))
3610 instrDesc* id = emitAllocInstr(attr);
3611 id->idSmallCns(cns);
3621 instrDescCns* id = emitAllocInstrCns(attr, cns);
3631 /*****************************************************************************
3633 * Get the instrDesc size, general purpose version
3637 inline size_t emitter::emitGetInstrDescSize(const instrDesc* id)
3639 if (id->idIsSmallDsc())
3641 return SMALL_IDSC_SIZE;
3643 else if (id->idIsLargeCns())
3646 if (id->idIsLclVarPair())
3648 return sizeof(instrDescLclVarPairCns);
3651 return sizeof(instrDescCns);
3656 if (id->idIsLclVarPair())
3658 return sizeof(instrDescLclVarPair);
3661 return sizeof(instrDesc);
3665 /*****************************************************************************
3667 * Allocate an instruction descriptor for an instruction with a small integer
3668 * constant operand. This is the same as emitNewInstrCns() except that here
3669 * any constant that is small enough for instrDesc::fitsInSmallCns() only gets
3670 * allocated SMALL_IDSC_SIZE bytes (and is thus a small descriptor, whereas
3671 * emitNewInstrCns() always allocates at least sizeof(instrDesc)).
3674 inline emitter::instrDesc* emitter::emitNewInstrSC(emitAttr attr, cnsval_ssize_t cns)
3676 if (instrDesc::fitsInSmallCns(cns))
3678 instrDesc* id = emitNewInstrSmall(attr);
3679 id->idSmallCns(cns);
3689 instrDescCns* id = emitAllocInstrCns(attr, cns);
3701 inline emitter::instrDesc* emitter::emitNewInstrReloc(emitAttr attr, BYTE* addr)
3703 assert(EA_IS_RELOC(attr));
3705 instrDescReloc* id = (instrDescReloc*)emitAllocAnyInstr(sizeof(instrDescReloc), attr);
3706 assert(id->idIsReloc());
3708 id->idrRelocVal = addr;
3711 emitTotalIDescRelocCnt++;
3712 #endif // EMITTER_STATS
3717 #endif // TARGET_ARM
3721 /*****************************************************************************
3723 * The following helpers should be used to access the various values that
3724 * get stored in different places within the instruction descriptor.
3727 inline ssize_t emitter::emitGetInsCns(instrDesc* id)
3729 return id->idIsLargeCns() ? ((instrDescCns*)id)->idcCnsVal : id->idSmallCns();
3732 inline ssize_t emitter::emitGetInsDsp(instrDesc* id)
3734 if (id->idIsLargeDsp())
3736 if (id->idIsLargeCns())
3738 return ((instrDescCnsDsp*)id)->iddcDspVal;
3740 return ((instrDescDsp*)id)->iddDspVal;
3745 /*****************************************************************************
3747 * Get hold of the argument count for an indirect call.
3750 inline unsigned emitter::emitGetInsCIargs(instrDesc* id)
3752 if (id->idIsLargeCall())
3754 return ((instrDescCGCA*)id)->idcArgCnt;
3758 assert(id->idIsLargeDsp() == false);
3759 assert(id->idIsLargeCns() == false);
3761 ssize_t cns = emitGetInsCns(id);
3762 assert((unsigned)cns == (size_t)cns);
3763 return (unsigned)cns;
3767 //-----------------------------------------------------------------------------
3768 // emitGetMemOpSize: Get the memory operand size of instrDesc.
3770 // Note: there are cases when embedded broadcast is enabled, so the memory operand
3771 // size is different from the intrinsic simd size, we will check here if emitter is
3772 // emiting a embedded broadcast enabled instruction.
3775 // id - Instruction descriptor
3777 emitAttr emitter::emitGetMemOpSize(instrDesc* id) const
3779 if (id->idIsEvexbContext())
3781 // should have the assumption that Evex.b now stands for the embedded broadcast context.
3782 // reference: Section 2.7.5 in Intel 64 and ia-32 architectures software developer's manual volume 2.
3783 ssize_t inputSize = GetInputSizeInBytes(id);
3797 return emitGetBaseMemOpSize(id);
3801 //-----------------------------------------------------------------------------
3802 // emitGetMemOpSize: Get the memory operand size of instrDesc.
3804 // Note: vextractf128 has a 128-bit output (register or memory) but a 256-bit input (register).
3805 // vinsertf128 is the inverse with a 256-bit output (register), a 256-bit input(register),
3806 // and a 128-bit input (register or memory).
3807 // Similarly, vextractf64x4 has a 256-bit output and 128-bit input and vinsertf64x4 the inverse
3808 // This method is mainly used for such instructions to return the appropriate memory operand
3809 // size, otherwise returns the regular operand size of the instruction.
3812 // id - Instruction descriptor
3814 emitAttr emitter::emitGetBaseMemOpSize(instrDesc* id) const
3817 emitAttr defaultSize = id->idOpSize();
3818 instruction ins = id->idIns();
3824 case INS_vpbroadcastb:
3830 case INS_pextrw_sse41:
3834 case INS_vpbroadcastw:
3864 case INS_vbroadcastss:
3865 case INS_vfmadd132ss:
3866 case INS_vfmadd213ss:
3867 case INS_vfmadd231ss:
3868 case INS_vfmsub132ss:
3869 case INS_vfmsub213ss:
3870 case INS_vfmsub231ss:
3871 case INS_vfnmadd132ss:
3872 case INS_vfnmadd213ss:
3873 case INS_vfnmadd231ss:
3874 case INS_vfnmsub132ss:
3875 case INS_vfnmsub213ss:
3876 case INS_vfnmsub231ss:
3877 case INS_vpbroadcastd:
3910 case INS_vbroadcastsd:
3911 case INS_vfmadd132sd:
3912 case INS_vfmadd213sd:
3913 case INS_vfmadd231sd:
3914 case INS_vfmsub132sd:
3915 case INS_vfmsub213sd:
3916 case INS_vfmsub231sd:
3917 case INS_vfnmadd132sd:
3918 case INS_vfnmadd213sd:
3919 case INS_vfnmadd231sd:
3920 case INS_vfnmsub132sd:
3921 case INS_vfnmsub213sd:
3922 case INS_vfnmsub231sd:
3923 case INS_vpbroadcastq:
3931 if (defaultSize == 64)
3935 else if (defaultSize == 32)
3941 assert(defaultSize == 16);
3965 insTupleType tupleType = insTupleTypeInfo(ins);
3966 unsigned memSize = 0;
3970 case INS_TT_HALF_MEM:
3972 memSize = defaultSize / 2;
3976 case INS_TT_QUARTER_MEM:
3978 memSize = defaultSize / 4;
3982 case INS_TT_EIGHTH_MEM:
3984 memSize = defaultSize / 8;
3994 return EA_ATTR(memSize);
3997 case INS_vbroadcastf128:
3998 case INS_vbroadcasti128:
3999 case INS_vextractf128:
4000 case INS_vextracti128:
4001 case INS_vinsertf128:
4002 case INS_vinserti128:
4007 case INS_vextractf32x8:
4008 case INS_vextracti32x8:
4009 case INS_vextractf64x4:
4010 case INS_vextracti64x4:
4011 case INS_vinsertf32x8:
4012 case INS_vinserti32x8:
4013 case INS_vinsertf64x4:
4014 case INS_vinserti64x4:
4021 if (defaultSize == 64)
4025 else if (defaultSize == 32)
4031 assert(defaultSize == 16);
4043 #endif // TARGET_XARCH
4045 /*****************************************************************************
4047 * Returns true if the given register contains a live GC ref.
4050 inline GCtype emitter::emitRegGCtype(regNumber reg)
4052 assert(emitIssuing);
4054 if ((emitThisGCrefRegs & genRegMask(reg)) != 0)
4058 else if ((emitThisByrefRegs & genRegMask(reg)) != 0)
4070 #if EMIT_TRACK_STACK_DEPTH
4071 #define CHECK_STACK_DEPTH() assert((int)emitCurStackLvl >= 0)
4073 #define CHECK_STACK_DEPTH()
4078 /*****************************************************************************
4080 * Return true when a given code offset is properly aligned for the target
4083 inline bool IsCodeAligned(UNATIVE_OFFSET offset)
4085 return ((offset & (CODE_ALIGN - 1)) == 0);
4089 inline BYTE* emitter::emitCodeWithInstructionSize(BYTE* codePtrBefore, BYTE* newCodePointer, unsigned char* instrSize)
4091 // DLD: Perhaps this method should return the instruction size, and we should do dst += <that size>
4092 // as is done in other cases?
4093 assert(newCodePointer >= codePtrBefore);
4094 ClrSafeInt<unsigned char> callInstrSizeSafe = ClrSafeInt<unsigned char>(newCodePointer - codePtrBefore);
4095 assert(!callInstrSizeSafe.IsOverflow());
4096 *instrSize = callInstrSizeSafe.Value();
4097 return newCodePointer;
4100 /*****************************************************************************/
4102 /*****************************************************************************/