1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
5 // This class contains all the data & functionality for code generation
6 // of a method, except for the target-specific elements, which are
7 // primarily in the Target class.
12 #include "codegeninterface.h"
13 #include "compiler.h" // temporary??
15 #include "jitgcinfo.h"
17 class CodeGen final : public CodeGenInterface
20 friend class DisAssembler;
23 // This could use further abstraction
24 CodeGen(Compiler* theCompiler);
26 virtual void genGenerateCode(void** codePtr, uint32_t* nativeSizeOfCode);
28 void genGenerateMachineCode();
29 void genEmitMachineCode();
30 void genEmitUnwindDebugGCandEH();
32 // TODO-Cleanup: Abstract out the part of this that finds the addressing mode, and
34 virtual bool genCreateAddrMode(
35 GenTree* addr, bool fold, bool* revPtr, GenTree** rv1Ptr, GenTree** rv2Ptr, unsigned* mulPtr, ssize_t* cnsPtr);
38 #if defined(TARGET_XARCH)
39 // Bit masks used in negating a float or double number.
40 // This is to avoid creating more than one data constant for these bitmasks when a
41 // method has more than one GT_NEG operation on floating point values.
42 CORINFO_FIELD_HANDLE negBitmaskFlt;
43 CORINFO_FIELD_HANDLE negBitmaskDbl;
45 // Bit masks used in computing Math.Abs() of a float or double number.
46 CORINFO_FIELD_HANDLE absBitmaskFlt;
47 CORINFO_FIELD_HANDLE absBitmaskDbl;
49 // Bit mask used in zeroing the 3rd element of a SIMD12
50 CORINFO_FIELD_HANDLE zroSimd12Elm3;
52 // Bit mask used in U8 -> double conversion to adjust the result.
53 CORINFO_FIELD_HANDLE u8ToDblBitmask;
55 // Generates SSE2 code for the given tree as "Operand BitWiseOp BitMask"
56 void genSSE2BitwiseOp(GenTree* treeNode);
58 // Generates SSE41 code for the given tree as a round operation
59 void genSSE41RoundOp(GenTreeOp* treeNode);
61 instruction simdAlignedMovIns()
63 // We use movaps when non-VEX because it is a smaller instruction;
64 // however the VEX version vmovaps would be used which is the same size as vmovdqa;
65 // also vmovdqa has more available CPU ports on older processors so we switch to that
66 return compiler->canUseVexEncoding() ? INS_movdqa : INS_movaps;
68 instruction simdUnalignedMovIns()
70 // We use movups when non-VEX because it is a smaller instruction;
71 // however the VEX version vmovups would be used which is the same size as vmovdqu;
72 // but vmovdqu has more available CPU ports on older processors so we switch to that
73 return compiler->canUseVexEncoding() ? INS_movdqu : INS_movups;
75 #endif // defined(TARGET_XARCH)
77 void genPrepForCompiler();
79 void genMarkLabelsForCodegen();
81 regNumber genFramePointerReg()
83 if (isFramePointerUsed())
93 static bool genShouldRoundFP();
95 static GenTreeIndir indirForm(var_types type, GenTree* base);
96 static GenTreeStoreInd storeIndirForm(var_types type, GenTree* base, GenTree* data);
98 GenTreeIntCon intForm(var_types type, ssize_t value);
100 void genRangeCheck(GenTree* node);
102 void genLockedInstructions(GenTreeOp* node);
104 void genCodeForLockAdd(GenTreeOp* node);
108 // On some targets such as the ARM we may need to have an extra reserved register
109 // that is used when addressing stack based locals and stack based temps.
110 // This method returns the regNumber that should be used when an extra register
111 // is needed to access the stack based locals and stack based temps.
113 regNumber rsGetRsvdReg()
115 // We should have already added this register to the mask
116 // of reserved registers in regSet.rdMaskResvd
117 noway_assert((regSet.rsMaskResvd & RBM_OPT_RSVD) != 0);
121 #endif // REG_OPT_RSVD
123 //-------------------------------------------------------------------------
125 bool genUseBlockInit; // true if we plan to block-initialize the local stack frame
126 unsigned genInitStkLclCnt; // The count of local variables that we need to zero init
128 void SubtractStackLevel(unsigned adjustment)
130 assert(genStackLevel >= adjustment);
131 unsigned newStackLevel = genStackLevel - adjustment;
132 if (genStackLevel != newStackLevel)
134 JITDUMP("Adjusting stack level from %d to %d\n", genStackLevel, newStackLevel);
136 genStackLevel = newStackLevel;
139 void AddStackLevel(unsigned adjustment)
141 unsigned newStackLevel = genStackLevel + adjustment;
142 if (genStackLevel != newStackLevel)
144 JITDUMP("Adjusting stack level from %d to %d\n", genStackLevel, newStackLevel);
146 genStackLevel = newStackLevel;
149 void SetStackLevel(unsigned newStackLevel)
151 if (genStackLevel != newStackLevel)
153 JITDUMP("Setting stack level from %d to %d\n", genStackLevel, newStackLevel);
155 genStackLevel = newStackLevel;
158 //-------------------------------------------------------------------------
162 // Allocates storage for the GC info, writes the GC info into that storage, records the address of the
163 // GC info of the method with the EE, and returns a pointer to the "info" portion (just post-header) of
164 // the GC info. Requires "codeSize" to be the size of the generated code, "prologSize" and "epilogSize"
165 // to be the sizes of the prolog and epilog, respectively. In DEBUG, makes a check involving the
166 // "codePtr", assumed to be a pointer to the start of the generated code.
167 CLANG_FORMAT_COMMENT_ANCHOR;
169 #ifdef JIT32_GCENCODER
170 void* genCreateAndStoreGCInfo(unsigned codeSize, unsigned prologSize, unsigned epilogSize DEBUGARG(void* codePtr));
171 void* genCreateAndStoreGCInfoJIT32(unsigned codeSize,
173 unsigned epilogSize DEBUGARG(void* codePtr));
174 #else // !JIT32_GCENCODER
175 void genCreateAndStoreGCInfo(unsigned codeSize, unsigned prologSize, unsigned epilogSize DEBUGARG(void* codePtr));
176 void genCreateAndStoreGCInfoX64(unsigned codeSize, unsigned prologSize DEBUGARG(void* codePtr));
177 #endif // !JIT32_GCENCODER
179 /**************************************************************************
181 *************************************************************************/
184 // the current (pending) label ref, a label which has been referenced but not yet seen
185 BasicBlock* genPendingCallLabel;
188 uint32_t* nativeSizeOfCode;
193 // Last instr we have displayed for dspInstrs
194 unsigned genCurDispOffset;
196 static const char* genInsName(instruction ins);
197 const char* genInsDisplayName(emitter::instrDesc* id);
199 static const char* genSizeStr(emitAttr size);
201 void genInitialize();
203 void genInitializeRegisterState();
205 void genCodeForBBlist();
208 void genSpillVar(GenTree* tree);
211 void genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize, regNumber callTarget = REG_NA);
213 void genGCWriteBarrier(GenTreeStoreInd* store, GCInfo::WriteBarrierForm wbf);
215 BasicBlock* genCreateTempLabel();
218 void genLogLabel(BasicBlock* bb);
221 void genDefineTempLabel(BasicBlock* label);
222 void genDefineInlineTempLabel(BasicBlock* label);
224 void genAdjustStackLevel(BasicBlock* block);
226 void genExitCode(BasicBlock* block);
228 void genJumpToThrowHlpBlk(emitJumpKind jumpKind, SpecialCodeKind codeKind, BasicBlock* failBlk = nullptr);
230 #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
231 void genJumpToThrowHlpBlk_la(SpecialCodeKind codeKind,
234 BasicBlock* failBlk = nullptr,
235 regNumber reg2 = REG_R0);
237 void genCheckOverflow(GenTree* tree);
240 //-------------------------------------------------------------------------
242 // Prolog/epilog generation
244 //-------------------------------------------------------------------------
250 // Prolog functions and data (there are a few exceptions for more generally used things)
253 void genEstablishFramePointer(int delta, bool reportUnwindData);
254 #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
255 void genFnPrologCalleeRegArgs();
257 void genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbered, RegState* regState);
259 void genEnregisterIncomingStackArgs();
260 #if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
261 void genEnregisterOSRArgsAndLocals(regNumber initReg, bool* pInitRegZeroed);
263 void genEnregisterOSRArgsAndLocals();
265 void genCheckUseBlockInit();
266 #if defined(UNIX_AMD64_ABI) && defined(FEATURE_SIMD)
267 void genClearStackVec3ArgUpperBits();
268 #endif // UNIX_AMD64_ABI && FEATURE_SIMD
270 #if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
271 bool genInstrWithConstant(instruction ins,
277 bool inUnwindRegion = false);
279 void genStackPointerAdjustment(ssize_t spAdjustment, regNumber tmpReg, bool* pTmpRegIsZero, bool reportUnwindData);
281 void genPrologSaveRegPair(regNumber reg1,
285 bool useSaveNextPair,
287 bool* pTmpRegIsZero);
289 void genPrologSaveReg(regNumber reg1, int spOffset, int spDelta, regNumber tmpReg, bool* pTmpRegIsZero);
291 void genEpilogRestoreRegPair(regNumber reg1,
295 bool useSaveNextPair,
297 bool* pTmpRegIsZero);
299 void genEpilogRestoreReg(regNumber reg1, int spOffset, int spDelta, regNumber tmpReg, bool* pTmpRegIsZero);
301 // A simple struct to keep register pairs for prolog and epilog.
306 bool useSaveNextPair;
308 RegPair(regNumber reg1) : reg1(reg1), reg2(REG_NA), useSaveNextPair(false)
312 RegPair(regNumber reg1, regNumber reg2) : reg1(reg1), reg2(reg2), useSaveNextPair(false)
314 assert(reg2 == REG_NEXT(reg1));
318 static void genBuildRegPairsStack(regMaskTP regsMask, ArrayStack<RegPair>* regStack);
319 static void genSetUseSaveNextPairs(ArrayStack<RegPair>* regStack);
321 static int genGetSlotSizeForRegsInMask(regMaskTP regsMask);
323 void genSaveCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset);
324 void genRestoreCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset);
326 void genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowestCalleeSavedOffset, int spDelta);
327 void genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta);
329 void genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed);
332 void genPushCalleeSavedRegisters();
335 #if defined(TARGET_AMD64)
336 void genOSRRecordTier0CalleeSavedRegistersAndFrame();
337 void genOSRSaveRemainingCalleeSavedRegisters();
338 #endif // TARGET_AMD64
340 #if defined(TARGET_RISCV64)
341 void genStackProbe(ssize_t frameSize, regNumber rOffset, regNumber rLimit, regNumber rPageSize);
344 void genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskTP maskArgRegsLiveIn);
346 void genPoisonFrame(regMaskTP bbRegLiveIn);
348 #if defined(TARGET_ARM)
350 bool genInstrWithConstant(
351 instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, ssize_t imm, insFlags flags, regNumber tmpReg);
353 bool genStackPointerAdjustment(ssize_t spAdjustment, regNumber tmpReg);
355 void genPushFltRegs(regMaskTP regMask);
356 void genPopFltRegs(regMaskTP regMask);
357 regMaskTP genStackAllocRegisterMask(unsigned frameSize, regMaskTP maskCalleeSavedFloat);
359 regMaskTP genJmpCallArgMask();
361 void genFreeLclFrame(unsigned frameSize,
362 /* IN OUT */ bool* pUnwindStarted);
364 void genMov32RelocatableDisplacement(BasicBlock* block, regNumber reg);
365 void genMov32RelocatableDataLabel(unsigned value, regNumber reg);
366 void genMov32RelocatableImmediate(emitAttr size, BYTE* addr, regNumber reg);
368 bool genUsedPopToReturn; // True if we use the pop into PC to return,
369 // False if we didn't and must branch to LR to return.
371 // A set of information that is used by funclet prolog and epilog generation. It is collected once, before
372 // funclet prologs and epilogs are generated, and used by all funclet prologs and epilogs, which must all be the
374 struct FuncletFrameInfoDsc
376 regMaskTP fiSaveRegs; // Set of registers saved in the funclet prolog (includes LR)
377 unsigned fiFunctionCallerSPtoFPdelta; // Delta between caller SP and the frame pointer
378 unsigned fiSpDelta; // Stack pointer delta
379 unsigned fiPSP_slot_SP_offset; // PSP slot offset from SP
380 int fiPSP_slot_CallerSP_offset; // PSP slot offset from Caller SP
383 FuncletFrameInfoDsc genFuncletInfo;
385 #elif defined(TARGET_ARM64)
387 // A set of information that is used by funclet prolog and epilog generation. It is collected once, before
388 // funclet prologs and epilogs are generated, and used by all funclet prologs and epilogs, which must all be the
390 struct FuncletFrameInfoDsc
392 regMaskTP fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes LR)
393 int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function
395 int fiSP_to_FPLR_save_delta; // FP/LR register save offset from SP (positive)
396 int fiSP_to_PSP_slot_delta; // PSP slot offset from SP (positive)
397 int fiSP_to_CalleeSave_delta; // First callee-saved register slot offset from SP (positive)
398 int fiCallerSP_to_PSP_slot_delta; // PSP slot offset from Caller SP (negative)
399 int fiFrameType; // Funclet frame types are numbered. See genFuncletProlog() for details.
400 int fiSpDelta1; // Stack pointer delta 1 (negative)
401 int fiSpDelta2; // Stack pointer delta 2 (negative)
404 FuncletFrameInfoDsc genFuncletInfo;
406 #elif defined(TARGET_AMD64)
408 // A set of information that is used by funclet prolog and epilog generation. It is collected once, before
409 // funclet prologs and epilogs are generated, and used by all funclet prologs and epilogs, which must all be the
411 struct FuncletFrameInfoDsc
413 unsigned fiFunction_InitialSP_to_FP_delta; // Delta between Initial-SP and the frame pointer
414 unsigned fiSpDelta; // Stack pointer delta
415 int fiPSP_slot_InitialSP_offset; // PSP slot offset from Initial-SP
418 FuncletFrameInfoDsc genFuncletInfo;
420 #elif defined(TARGET_LOONGARCH64)
422 // A set of information that is used by funclet prolog and epilog generation.
423 // It is collected once, before funclet prologs and epilogs are generated,
424 // and used by all funclet prologs and epilogs, which must all be the same.
425 struct FuncletFrameInfoDsc
427 regMaskTP fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA)
428 int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function
430 int fiSP_to_CalleeSaved_delta; // CalleeSaved register save offset from SP (positive)
431 int fiCalleeSavedPadding; // CalleeSaved offset padding (positive)
432 int fiSP_to_PSP_slot_delta; // PSP slot offset from SP (positive)
433 int fiCallerSP_to_PSP_slot_delta; // PSP slot offset from Caller SP (negative)
434 int fiSpDelta; // Stack pointer delta (negative)
437 FuncletFrameInfoDsc genFuncletInfo;
439 #elif defined(TARGET_RISCV64)
441 // A set of information that is used by funclet prolog and epilog generation.
442 // It is collected once, before funclet prologs and epilogs are generated,
443 // and used by all funclet prologs and epilogs, which must all be the same.
444 struct FuncletFrameInfoDsc
446 regMaskTP fiSaveRegs; // Set of callee-saved registers saved in the funclet prolog (includes RA)
447 int fiFunction_CallerSP_to_FP_delta; // Delta between caller SP and the frame pointer in the parent function
449 int fiSP_to_CalleeSaved_delta; // CalleeSaved register save offset from SP (positive)
450 int fiCalleeSavedPadding; // CalleeSaved offset padding (positive)
451 int fiSP_to_PSP_slot_delta; // PSP slot offset from SP (positive)
452 int fiCallerSP_to_PSP_slot_delta; // PSP slot offset from Caller SP (negative)
453 int fiSpDelta; // Stack pointer delta (negative)
456 FuncletFrameInfoDsc genFuncletInfo;
458 #endif // TARGET_ARM, TARGET_ARM64, TARGET_AMD64, TARGET_LOONGARCH64, TARGET_RISCV64
460 #if defined(TARGET_XARCH)
462 // Save/Restore callee saved float regs to stack
463 void genPreserveCalleeSavedFltRegs(unsigned lclFrameSize);
464 void genRestoreCalleeSavedFltRegs(unsigned lclFrameSize);
465 // Generate VZeroupper instruction to avoid AVX/SSE transition penalty
466 void genVzeroupperIfNeeded(bool check256bitOnly = true);
468 #endif // TARGET_XARCH
470 void genZeroInitFltRegs(const regMaskTP& initFltRegs, const regMaskTP& initDblRegs, const regNumber& initReg);
472 regNumber genGetZeroReg(regNumber initReg, bool* pInitRegZeroed);
474 void genZeroInitFrame(int untrLclHi, int untrLclLo, regNumber initReg, bool* pInitRegZeroed);
475 void genZeroInitFrameUsingBlockInit(int untrLclHi, int untrLclLo, regNumber initReg, bool* pInitRegZeroed);
477 void genReportGenericContextArg(regNumber initReg, bool* pInitRegZeroed);
479 void genSetGSSecurityCookie(regNumber initReg, bool* pInitRegZeroed);
481 void genFinalizeFrame();
483 #ifdef PROFILING_SUPPORTED
484 void genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed);
485 void genProfilingLeaveCallback(unsigned helper);
486 #endif // PROFILING_SUPPORTED
489 void genEmitCall(int callType,
490 CORINFO_METHOD_HANDLE methHnd,
491 INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo)
493 X86_ARG(int argSize),
495 MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
502 void genEmitCallIndir(int callType,
503 CORINFO_METHOD_HANDLE methHnd,
504 INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo)
506 X86_ARG(int argSize),
508 MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
516 CLANG_FORMAT_COMMENT_ANCHOR;
518 #if defined(TARGET_ARM)
519 bool genCanUsePopToReturn(regMaskTP maskPopRegsInt, bool jmpEpilog);
522 #if defined(TARGET_ARM64)
524 void genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog);
526 #else // !defined(TARGET_ARM64)
528 void genPopCalleeSavedRegisters(bool jmpEpilog = false);
530 #if defined(TARGET_XARCH)
531 unsigned genPopCalleeSavedRegistersFromMask(regMaskTP rsPopRegs);
532 #endif // !defined(TARGET_XARCH)
534 #endif // !defined(TARGET_ARM64)
537 // Common or driving functions
540 void genReserveProlog(BasicBlock* block); // currently unused
541 void genReserveEpilog(BasicBlock* block);
543 void genFnEpilog(BasicBlock* block);
545 #if defined(FEATURE_EH_FUNCLETS)
547 void genReserveFuncletProlog(BasicBlock* block);
548 void genReserveFuncletEpilog(BasicBlock* block);
549 void genFuncletProlog(BasicBlock* block);
550 void genFuncletEpilog();
551 void genCaptureFuncletPrologEpilogInfo();
553 /*-----------------------------------------------------------------------------
555 * Set the main function PSPSym value in the frame.
556 * Funclets use different code to load the PSP sym and save it in their frame.
557 * See the document "CLR ABI.md" for a full description of the PSPSym.
558 * The PSPSym section of that document is copied here.
560 ***********************************
561 * The name PSPSym stands for Previous Stack Pointer Symbol. It is how a funclet
562 * accesses locals from the main function body.
564 * First, two definitions.
566 * Caller-SP is the value of the stack pointer in a function's caller before the call
567 * instruction is executed. That is, when function A calls function B, Caller-SP for B
568 * is the value of the stack pointer immediately before the call instruction in A
569 * (calling B) was executed. Note that this definition holds for both AMD64, which
570 * pushes the return value when a call instruction is executed, and for ARM, which
571 * doesn't. For AMD64, Caller-SP is the address above the call return address.
573 * Initial-SP is the initial value of the stack pointer after the fixed-size portion of
574 * the frame has been allocated. That is, before any "alloca"-type allocations.
576 * The PSPSym is a pointer-sized local variable in the frame of the main function and
577 * of each funclet. The value stored in PSPSym is the value of Initial-SP/Caller-SP
578 * for the main function. The stack offset of the PSPSym is reported to the VM in the
579 * GC information header. The value reported in the GC information is the offset of the
580 * PSPSym from Initial-SP/Caller-SP. (Note that both the value stored, and the way the
581 * value is reported to the VM, differs between architectures. In particular, note that
582 * most things in the GC information header are reported as offsets relative to Caller-SP,
583 * but PSPSym on AMD64 is one (maybe the only) exception.)
585 * The VM uses the PSPSym to find other locals it cares about (such as the generics context
586 * in a funclet frame). The JIT uses it to re-establish the frame pointer register, so that
587 * the frame pointer is the same value in a funclet as it is in the main function body.
589 * When a funclet is called, it is passed the Establisher Frame Pointer. For AMD64 this is
590 * true for all funclets and it is passed as the first argument in RCX, but for ARM this is
591 * only true for first pass funclets (currently just filters) and it is passed as the second
592 * argument in R1. The Establisher Frame Pointer is a stack pointer of an interesting "parent"
593 * frame in the exception processing system. For the CLR, it points either to the main function
594 * frame or a dynamically enclosing funclet frame from the same function, for the funclet being
595 * invoked. The value of the Establisher Frame Pointer is Initial-SP on AMD64, Caller-SP on ARM.
597 * Using the establisher frame, the funclet wants to load the value of the PSPSym. Since we
598 * don't know if the Establisher Frame is from the main function or a funclet, we design the
599 * main function and funclet frame layouts to place the PSPSym at an identical, small, constant
600 * offset from the Establisher Frame in each case. (This is also required because we only report
601 * a single offset to the PSPSym in the GC information, and that offset must be valid for the main
602 * function and all of its funclets). Then, the funclet uses this known offset to compute the
603 * PSPSym address and read its value. From this, it can compute the value of the frame pointer
604 * (which is a constant offset from the PSPSym value) and set the frame register to be the same
605 * as the parent function. Also, the funclet writes the value of the PSPSym to its own frame's
606 * PSPSym. This "copying" of the PSPSym happens for every funclet invocation, in particular,
607 * for every nested funclet invocation.
609 * On ARM, for all second pass funclets (finally, fault, catch, and filter-handler) the VM
610 * restores all non-volatile registers to their values within the parent frame. This includes
611 * the frame register (R11). Thus, the PSPSym is not used to recompute the frame pointer register
612 * in this case, though the PSPSym is copied to the funclet's frame, as for all funclets.
614 * Catch, Filter, and Filter-handlers also get an Exception object (GC ref) as an argument
615 * (REG_EXCEPTION_OBJECT). On AMD64 it is the second argument and thus passed in RDX. On
616 * ARM this is the first argument and passed in R0.
618 * (Note that the JIT64 source code contains a comment that says, "The current CLR doesn't always
619 * pass the correct establisher frame to the funclet. Funclet may receive establisher frame of
620 * funclet when expecting that of original routine." It indicates this is the reason that a PSPSym
621 * is required in all funclets as well as the main function, whereas if the establisher frame was
622 * correctly reported, the PSPSym could be omitted in some cases.)
623 ***********************************
625 void genSetPSPSym(regNumber initReg, bool* pInitRegZeroed);
627 void genUpdateCurrentFunclet(BasicBlock* block);
628 #if defined(TARGET_ARM)
629 void genInsertNopForUnwinder(BasicBlock* block);
632 #else // !FEATURE_EH_FUNCLETS
634 // This is a no-op when there are no funclets!
635 void genUpdateCurrentFunclet(BasicBlock* block)
640 #endif // !FEATURE_EH_FUNCLETS
642 void genGeneratePrologsAndEpilogs();
644 #if defined(DEBUG) && defined(TARGET_ARM64)
645 void genArm64EmitterUnitTests();
648 #if defined(DEBUG) && defined(TARGET_LOONGARCH64)
649 void genLoongArch64EmitterUnitTests();
652 #if defined(DEBUG) && defined(LATE_DISASM) && defined(TARGET_AMD64)
653 void genAmd64EmitterUnitTests();
657 virtual void SetSaveFpLrWithAllCalleeSavedRegisters(bool value);
658 virtual bool IsSaveFpLrWithAllCalleeSavedRegisters() const;
659 bool genSaveFpLrWithAllCalleeSavedRegisters;
660 bool genForceFuncletFrameType5;
661 #endif // TARGET_ARM64
663 //-------------------------------------------------------------------------
665 // End prolog/epilog generation
667 //-------------------------------------------------------------------------
669 void genSinglePush();
671 regMaskTP genPushRegs(regMaskTP regs, regMaskTP* byrefRegs, regMaskTP* noRefRegs);
672 void genPopRegs(regMaskTP regs, regMaskTP byrefRegs, regMaskTP noRefRegs);
675 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
676 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
678 XX Debugging Support XX
680 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
681 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
685 void genIPmappingDisp(unsigned mappingNum, const IPmappingDsc* ipMapping);
686 void genIPmappingListDisp();
689 void genIPmappingAdd(IPmappingDscKind kind, const DebugInfo& di, bool isLabel);
690 void genIPmappingAddToFront(IPmappingDscKind kind, const DebugInfo& di, bool isLabel);
691 void genIPmappingGen();
692 void genAddRichIPMappingHere(const DebugInfo& di);
694 void genReportRichDebugInfo();
696 void genRecordRichDebugInfoInlineTree(InlineContext* context, ICorDebugInfo::InlineTreeNode* tree);
699 void genReportRichDebugInfoToFile();
700 void genReportRichDebugInfoInlineTreeToFile(FILE* file, InlineContext* context, bool* first);
703 void genEnsureCodeEmitted(const DebugInfo& di);
705 //-------------------------------------------------------------------------
706 // scope info for the variables
708 void genSetScopeInfo(unsigned which,
709 UNATIVE_OFFSET startOffs,
710 UNATIVE_OFFSET length,
716 void genSetScopeInfo();
717 // Send VariableLiveRanges as debug info to the debugger
718 void genSetScopeInfoUsingVariableRanges();
722 void checkICodeDebugInfo();
724 // The logic used to report debug info on debug code is the same for ScopeInfo and
726 void siBeginBlock(BasicBlock* block);
727 void siEndBlock(BasicBlock* block);
729 // VariableLiveRange and siScope needs this method to report variables on debug code
730 void siOpenScopesForNonTrackedVars(const BasicBlock* block, unsigned int lastBlockILEndOffset);
733 #if defined(FEATURE_EH_FUNCLETS)
734 bool siInFuncletRegion; // Have we seen the start of the funclet region?
735 #endif // FEATURE_EH_FUNCLETS
737 IL_OFFSET siLastEndOffs; // IL offset of the (exclusive) end of the last block processed
740 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
741 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
743 XX PrologScopeInfo XX
745 XX We need special handling in the prolog block, as the parameter variables XX
746 XX may not be in the same position described by genLclVarTable - they all XX
747 XX start out on the stack XX
749 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
750 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
757 NATIVE_OFFSET psiGetVarStackOffset(const LclVarDsc* lclVarDsc) const;
759 /*****************************************************************************
762 * This struct holds the LocalVarInfo in terms of the generated native code
763 * after a call to genSetScopeInfo()
769 struct TrnslLocalVarInfo
774 UNATIVE_OFFSET tlviStartPC;
780 // Array of scopes of LocalVars in terms of native code
782 TrnslLocalVarInfo* genTrnslLocalVarInfo;
783 unsigned genTrnslLocalVarCount;
786 void genSetRegToConst(regNumber targetReg, var_types targetType, GenTree* tree);
787 #if defined(FEATURE_SIMD)
788 void genSetRegToConst(regNumber targetReg, var_types targetType, simd_t* val);
790 void genCodeForTreeNode(GenTree* treeNode);
791 void genCodeForBinary(GenTreeOp* treeNode);
793 #if defined(TARGET_X86)
794 void genCodeForLongUMod(GenTreeOp* node);
797 void genCodeForDivMod(GenTreeOp* treeNode);
798 void genCodeForMul(GenTreeOp* treeNode);
799 void genCodeForIncSaturate(GenTree* treeNode);
800 void genCodeForMulHi(GenTreeOp* treeNode);
801 void genLeaInstruction(GenTreeAddrMode* lea);
802 void genSetRegToCond(regNumber dstReg, GenTree* tree);
804 #if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
805 void genScaledAdd(emitAttr attr,
809 int scale RISCV64_ARG(regNumber scaleTempReg));
810 #endif // TARGET_ARMARCH || TARGET_LOONGARCH64 || TARGET_RISCV64
812 #if defined(TARGET_ARMARCH)
813 void genCodeForMulLong(GenTreeOp* mul);
814 #endif // TARGET_ARMARCH
816 #if !defined(TARGET_64BIT)
817 void genLongToIntCast(GenTree* treeNode);
820 // Generate code for a GT_BITCAST that is not contained.
821 void genCodeForBitCast(GenTreeOp* treeNode);
823 // Generate the instruction to move a value between register files
824 void genBitCast(var_types targetType, regNumber targetReg, var_types srcType, regNumber srcReg);
827 struct GenIntCastDesc
832 CHECK_SMALL_INT_RANGE,
836 CHECK_POSITIVE_INT_RANGE,
844 ZERO_EXTEND_SMALL_INT,
845 SIGN_EXTEND_SMALL_INT,
850 LOAD_ZERO_EXTEND_SMALL_INT,
851 LOAD_SIGN_EXTEND_SMALL_INT,
853 LOAD_ZERO_EXTEND_INT,
854 LOAD_SIGN_EXTEND_INT,
860 CheckKind m_checkKind;
861 unsigned m_checkSrcSize;
862 int m_checkSmallIntMin;
863 int m_checkSmallIntMax;
864 ExtendKind m_extendKind;
865 unsigned m_extendSrcSize;
868 GenIntCastDesc(GenTreeCast* cast);
870 CheckKind CheckKind() const
875 unsigned CheckSrcSize() const
877 assert(m_checkKind != CHECK_NONE);
878 return m_checkSrcSize;
881 int CheckSmallIntMin() const
883 assert(m_checkKind == CHECK_SMALL_INT_RANGE);
884 return m_checkSmallIntMin;
887 int CheckSmallIntMax() const
889 assert(m_checkKind == CHECK_SMALL_INT_RANGE);
890 return m_checkSmallIntMax;
893 ExtendKind ExtendKind() const
898 unsigned ExtendSrcSize() const
900 return m_extendSrcSize;
905 void genIntCastOverflowCheck(GenTreeCast* cast, const GenIntCastDesc& desc, regNumber reg);
906 void genIntToIntCast(GenTreeCast* cast);
907 void genFloatToFloatCast(GenTree* treeNode);
908 void genFloatToIntCast(GenTree* treeNode);
909 void genIntToFloatCast(GenTree* treeNode);
910 void genCkfinite(GenTree* treeNode);
911 void genCodeForCompare(GenTreeOp* tree);
913 void genCodeForCCMP(GenTreeCCMP* ccmp);
915 void genCodeForSelect(GenTreeOp* select);
916 void genIntrinsic(GenTreeIntrinsic* treeNode);
917 void genPutArgStk(GenTreePutArgStk* treeNode);
918 void genPutArgReg(GenTreeOp* tree);
919 #if FEATURE_ARG_SPLIT
920 void genPutArgSplit(GenTreePutArgSplit* treeNode);
921 #endif // FEATURE_ARG_SPLIT
923 #if defined(TARGET_XARCH)
924 unsigned getBaseVarForPutArgStk(GenTree* treeNode);
925 #endif // TARGET_XARCH
927 unsigned getFirstArgWithStackSlot();
929 void genCompareFloat(GenTree* treeNode);
930 void genCompareInt(GenTree* treeNode);
932 bool genCanAvoidEmittingCompareAgainstZero(GenTree* tree, var_types opType);
933 GenTree* genTryFindFlagsConsumer(GenTree* flagsProducer, GenCondition** condition);
938 insOpts genGetSimdInsOpt(emitAttr size, var_types elementType);
940 void genSimdUpperSave(GenTreeIntrinsic* node);
941 void genSimdUpperRestore(GenTreeIntrinsic* node);
943 void genSimd12UpperClear(regNumber tgtReg);
945 // TYP_SIMD12 (i.e Vector3 of size 12 bytes) is not a hardware supported size and requires
946 // two reads/writes on 64-bit targets. These routines abstract reading/writing of Vector3
947 // values through an indirection. Note that Vector3 locals allocated on stack would have
948 // their size rounded to TARGET_POINTER_SIZE (which is 8 bytes on 64-bit targets) and hence
949 // Vector3 locals could be treated as TYP_SIMD16 while reading/writing.
950 void genStoreIndTypeSimd12(GenTreeStoreInd* treeNode);
951 void genLoadIndTypeSimd12(GenTreeIndir* treeNode);
952 void genStoreLclTypeSimd12(GenTreeLclVarCommon* treeNode);
953 void genLoadLclTypeSimd12(GenTreeLclVarCommon* treeNode);
955 void genEmitStoreLclTypeSimd12(GenTree* store, unsigned lclNum, unsigned offset);
956 void genEmitLoadLclTypeSimd12(regNumber tgtReg, unsigned lclNum, unsigned offset);
957 #endif // TARGET_XARCH
959 void genStoreSimd12ToStack(regNumber dataReg, regNumber tmpReg);
960 void genPutArgStkSimd12(GenTreePutArgStk* treeNode);
962 #endif // FEATURE_SIMD
964 #ifdef FEATURE_HW_INTRINSICS
965 void genHWIntrinsic(GenTreeHWIntrinsic* node);
966 #if defined(TARGET_XARCH)
967 void genHWIntrinsic_R_RM(GenTreeHWIntrinsic* node, instruction ins, emitAttr attr, regNumber reg, GenTree* rmOp);
968 void genHWIntrinsic_R_RM_I(GenTreeHWIntrinsic* node, instruction ins, emitAttr attr, int8_t ival);
969 void genHWIntrinsic_R_R_RM(GenTreeHWIntrinsic* node, instruction ins, emitAttr attr);
970 void genHWIntrinsic_R_R_RM(
971 GenTreeHWIntrinsic* node, instruction ins, emitAttr attr, regNumber targetReg, regNumber op1Reg, GenTree* op2);
972 void genHWIntrinsic_R_R_RM_I(GenTreeHWIntrinsic* node, instruction ins, emitAttr attr, int8_t ival);
973 void genHWIntrinsic_R_R_RM_R(GenTreeHWIntrinsic* node, instruction ins, emitAttr attr);
974 void genHWIntrinsic_R_R_R_RM(
975 instruction ins, emitAttr attr, regNumber targetReg, regNumber op1Reg, regNumber op2Reg, GenTree* op3);
976 void genHWIntrinsic_R_R_R_RM_I(GenTreeHWIntrinsic* node, instruction ins, emitAttr attr, int8_t ival);
977 void genBaseIntrinsic(GenTreeHWIntrinsic* node);
978 void genX86BaseIntrinsic(GenTreeHWIntrinsic* node);
979 void genSSEIntrinsic(GenTreeHWIntrinsic* node);
980 void genSSE2Intrinsic(GenTreeHWIntrinsic* node);
981 void genSSE41Intrinsic(GenTreeHWIntrinsic* node);
982 void genSSE42Intrinsic(GenTreeHWIntrinsic* node);
983 void genAvxFamilyIntrinsic(GenTreeHWIntrinsic* node);
984 void genAESIntrinsic(GenTreeHWIntrinsic* node);
985 void genBMI1OrBMI2Intrinsic(GenTreeHWIntrinsic* node);
986 void genFMAIntrinsic(GenTreeHWIntrinsic* node);
987 void genPermuteVar2x(GenTreeHWIntrinsic* node);
988 void genLZCNTIntrinsic(GenTreeHWIntrinsic* node);
989 void genPCLMULQDQIntrinsic(GenTreeHWIntrinsic* node);
990 void genPOPCNTIntrinsic(GenTreeHWIntrinsic* node);
991 void genXCNTIntrinsic(GenTreeHWIntrinsic* node, instruction ins);
992 void genX86SerializeIntrinsic(GenTreeHWIntrinsic* node);
993 template <typename HWIntrinsicSwitchCaseBody>
994 void genHWIntrinsicJumpTableFallback(NamedIntrinsic intrinsic,
995 regNumber nonConstImmReg,
998 HWIntrinsicSwitchCaseBody emitSwCase);
999 #endif // defined(TARGET_XARCH)
1002 class HWIntrinsicImmOpHelper final
1005 HWIntrinsicImmOpHelper(CodeGen* codeGen, GenTree* immOp, GenTreeHWIntrinsic* intrin);
1010 // Returns true after the last call to EmitCaseEnd() (i.e. this signals that code generation is done).
1013 return (immValue > immUpperBound);
1016 // Returns a value of the immediate operand that should be used for a case.
1017 int ImmValue() const
1023 // Returns true if immOp is non contained immediate (i.e. the value of the immediate operand is enregistered in
1025 bool NonConstImmOp() const
1027 return nonConstImmReg != REG_NA;
1030 // Returns true if a non constant immediate operand can be either 0 or 1.
1031 bool TestImmOpZeroOrOne() const
1033 assert(NonConstImmOp());
1034 return (immLowerBound == 0) && (immUpperBound == 1);
1037 emitter* GetEmitter() const
1039 return codeGen->GetEmitter();
1042 CodeGen* const codeGen;
1043 BasicBlock* endLabel;
1044 BasicBlock* nonZeroLabel;
1048 regNumber nonConstImmReg;
1049 regNumber branchTargetReg;
1051 #endif // TARGET_ARM64
1053 #endif // FEATURE_HW_INTRINSICS
1055 #if !defined(TARGET_64BIT)
1057 // CodeGen for Long Ints
1059 void genStoreLongLclVar(GenTree* treeNode);
1061 #endif // !defined(TARGET_64BIT)
1063 //-------------------------------------------------------------------------
1064 // genUpdateLifeStore: Do liveness udpate after tree store instructions
1065 // were emitted, update result var's home if it was stored on stack.
1068 // tree - Gentree node
1069 // targetReg - of the tree
1070 // varDsc - result value's variable
1074 __forceinline void genUpdateLifeStore(GenTree* tree, regNumber targetReg, LclVarDsc* varDsc)
1076 if (targetReg != REG_NA)
1078 genProduceReg(tree);
1082 genUpdateLife(tree);
1083 varDsc->SetRegNum(REG_STK);
1087 // Do liveness update for register produced by the current node in codegen after
1088 // code has been emitted for it.
1089 void genProduceReg(GenTree* tree);
1090 void genSpillLocal(unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum);
1091 void genUnspillLocal(
1092 unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum, bool reSpill, bool isLastUse);
1093 void genUnspillRegIfNeeded(GenTree* tree);
1094 void genUnspillRegIfNeeded(GenTree* tree, unsigned multiRegIndex);
1095 regNumber genConsumeReg(GenTree* tree);
1096 regNumber genConsumeReg(GenTree* tree, unsigned multiRegIndex);
1097 void genCopyRegIfNeeded(GenTree* tree, regNumber needReg);
1098 void genConsumeRegAndCopy(GenTree* tree, regNumber needReg);
1100 void genConsumeIfReg(GenTree* tree)
1102 if (!tree->isContained())
1104 (void)genConsumeReg(tree);
1108 void genRegCopy(GenTree* tree);
1109 regNumber genRegCopy(GenTree* tree, unsigned multiRegIndex);
1110 void genTransferRegGCState(regNumber dst, regNumber src);
1111 void genConsumeAddress(GenTree* addr);
1112 void genConsumeAddrMode(GenTreeAddrMode* mode);
1113 void genSetBlockSize(GenTreeBlk* blkNode, regNumber sizeReg);
1114 void genConsumeBlockSrc(GenTreeBlk* blkNode);
1115 void genSetBlockSrc(GenTreeBlk* blkNode, regNumber srcReg);
1116 void genConsumeBlockOp(GenTreeBlk* blkNode, regNumber dstReg, regNumber srcReg, regNumber sizeReg);
1118 #ifdef FEATURE_PUT_STRUCT_ARG_STK
1119 void genConsumePutStructArgStk(GenTreePutArgStk* putArgStkNode,
1123 #endif // FEATURE_PUT_STRUCT_ARG_STK
1124 #if FEATURE_ARG_SPLIT
1125 void genConsumeArgSplitStruct(GenTreePutArgSplit* putArgNode);
1126 #endif // FEATURE_ARG_SPLIT
1128 void genConsumeRegs(GenTree* tree);
1129 void genConsumeOperands(GenTreeOp* tree);
1130 #if defined(FEATURE_SIMD) || defined(FEATURE_HW_INTRINSICS)
1131 void genConsumeMultiOpOperands(GenTreeMultiOp* tree);
1133 void genEmitGSCookieCheck(bool pushReg);
1134 void genCodeForShift(GenTree* tree);
1136 #if defined(TARGET_X86) || defined(TARGET_ARM)
1137 void genCodeForShiftLong(GenTree* tree);
1141 void genCodeForShiftRMW(GenTreeStoreInd* storeInd);
1142 #endif // TARGET_XARCH
1144 void genCodeForCast(GenTreeOp* tree);
1145 void genCodeForLclAddr(GenTreeLclFld* lclAddrNode);
1146 void genCodeForIndexAddr(GenTreeIndexAddr* tree);
1147 void genCodeForIndir(GenTreeIndir* tree);
1148 void genCodeForNegNot(GenTree* tree);
1149 void genCodeForBswap(GenTree* tree);
1150 bool genCanOmitNormalizationForBswap16(GenTree* tree);
1151 void genCodeForLclVar(GenTreeLclVar* tree);
1152 void genCodeForLclFld(GenTreeLclFld* tree);
1153 void genCodeForStoreLclFld(GenTreeLclFld* tree);
1154 void genCodeForStoreLclVar(GenTreeLclVar* tree);
1155 void genCodeForReturnTrap(GenTreeOp* tree);
1156 void genCodeForStoreInd(GenTreeStoreInd* tree);
1157 void genCodeForSwap(GenTreeOp* tree);
1158 void genCodeForCpObj(GenTreeBlk* cpObjNode);
1159 void genCodeForCpBlkRepMovs(GenTreeBlk* cpBlkNode);
1160 void genCodeForCpBlkUnroll(GenTreeBlk* cpBlkNode);
1162 void genCodeForCpBlkHelper(GenTreeBlk* cpBlkNode);
1164 void genCodeForPhysReg(GenTreePhysReg* tree);
1165 void genCodeForNullCheck(GenTreeIndir* tree);
1166 void genCodeForCmpXchg(GenTreeCmpXchg* tree);
1167 void genCodeForReuseVal(GenTree* treeNode);
1169 void genAlignStackBeforeCall(GenTreePutArgStk* putArgStk);
1170 void genAlignStackBeforeCall(GenTreeCall* call);
1171 void genRemoveAlignmentAfterCall(GenTreeCall* call, unsigned bias = 0);
1173 #if defined(UNIX_X86_ABI)
1175 unsigned curNestedAlignment; // Keep track of alignment adjustment required during codegen.
1176 unsigned maxNestedAlignment; // The maximum amount of alignment adjustment required.
1178 void SubtractNestedAlignment(unsigned adjustment)
1180 assert(curNestedAlignment >= adjustment);
1181 unsigned newNestedAlignment = curNestedAlignment - adjustment;
1182 if (curNestedAlignment != newNestedAlignment)
1184 JITDUMP("Adjusting stack nested alignment from %d to %d\n", curNestedAlignment, newNestedAlignment);
1186 curNestedAlignment = newNestedAlignment;
1189 void AddNestedAlignment(unsigned adjustment)
1191 unsigned newNestedAlignment = curNestedAlignment + adjustment;
1192 if (curNestedAlignment != newNestedAlignment)
1194 JITDUMP("Adjusting stack nested alignment from %d to %d\n", curNestedAlignment, newNestedAlignment);
1196 curNestedAlignment = newNestedAlignment;
1198 if (curNestedAlignment > maxNestedAlignment)
1200 JITDUMP("Max stack nested alignment changed from %d to %d\n", maxNestedAlignment, curNestedAlignment);
1201 maxNestedAlignment = curNestedAlignment;
1208 void genPutArgStkFieldList(GenTreePutArgStk* putArgStk, unsigned outArgVarNum);
1209 #endif // !TARGET_X86
1211 #ifdef FEATURE_PUT_STRUCT_ARG_STK
1213 bool genAdjustStackForPutArgStk(GenTreePutArgStk* putArgStk);
1214 void genPushReg(var_types type, regNumber srcReg);
1215 void genPutArgStkFieldList(GenTreePutArgStk* putArgStk);
1216 #endif // TARGET_X86
1218 void genPutStructArgStk(GenTreePutArgStk* treeNode);
1220 unsigned genMove8IfNeeded(unsigned size, regNumber tmpReg, GenTree* src, unsigned offset);
1221 unsigned genMove4IfNeeded(unsigned size, regNumber tmpReg, GenTree* src, unsigned offset);
1222 unsigned genMove2IfNeeded(unsigned size, regNumber tmpReg, GenTree* src, unsigned offset);
1223 unsigned genMove1IfNeeded(unsigned size, regNumber tmpReg, GenTree* src, unsigned offset);
1224 void genCodeForLoadOffset(instruction ins, emitAttr size, regNumber dst, GenTree* base, unsigned offset);
1225 void genStoreRegToStackArg(var_types type, regNumber reg, int offset);
1226 void genStructPutArgRepMovs(GenTreePutArgStk* putArgStkNode);
1227 void genStructPutArgUnroll(GenTreePutArgStk* putArgStkNode);
1229 void genStructPutArgPush(GenTreePutArgStk* putArgStkNode);
1231 void genStructPutArgPartialRepMovs(GenTreePutArgStk* putArgStkNode);
1233 #endif // FEATURE_PUT_STRUCT_ARG_STK
1235 void genCodeForStoreBlk(GenTreeBlk* storeBlkNode);
1237 void genCodeForInitBlkHelper(GenTreeBlk* initBlkNode);
1239 void genCodeForInitBlkRepStos(GenTreeBlk* initBlkNode);
1240 void genCodeForInitBlkUnroll(GenTreeBlk* initBlkNode);
1241 void genJumpTable(GenTree* tree);
1242 void genTableBasedSwitch(GenTree* tree);
1243 #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
1244 instruction genGetInsForOper(GenTree* treeNode);
1246 instruction genGetInsForOper(genTreeOps oper, var_types type);
1248 bool genEmitOptimizedGCWriteBarrier(GCInfo::WriteBarrierForm writeBarrierForm, GenTree* addr, GenTree* data);
1249 GenTree* getCallTarget(const GenTreeCall* call, CORINFO_METHOD_HANDLE* methHnd);
1250 regNumber getCallIndirectionCellReg(GenTreeCall* call);
1251 void genCall(GenTreeCall* call);
1252 void genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackArgBytes));
1253 void genDefinePendingCallLabel(GenTreeCall* call);
1254 void genJmpMethod(GenTree* jmp);
1255 BasicBlock* genCallFinally(BasicBlock* block);
1256 #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
1257 // TODO: refactor for LA.
1258 void genCodeForJumpCompare(GenTreeOpCC* tree);
1260 #if defined(TARGET_ARM64)
1261 void genCodeForJumpCompare(GenTreeOpCC* tree);
1262 void genCodeForBfiz(GenTreeOp* tree);
1263 #endif // TARGET_ARM64
1265 #if defined(FEATURE_EH_FUNCLETS)
1266 void genEHCatchRet(BasicBlock* block);
1267 #else // !FEATURE_EH_FUNCLETS
1268 void genEHFinallyOrFilterRet(BasicBlock* block);
1269 #endif // !FEATURE_EH_FUNCLETS
1271 void genMultiRegStoreToSIMDLocal(GenTreeLclVar* lclNode);
1272 void genMultiRegStoreToLocal(GenTreeLclVar* lclNode);
1274 // Codegen for multi-register struct returns.
1275 bool isStructReturn(GenTree* treeNode);
1277 void genSIMDSplitReturn(GenTree* src, ReturnTypeDesc* retTypeDesc);
1279 void genStructReturn(GenTree* treeNode);
1281 #if defined(TARGET_X86) || defined(TARGET_ARM)
1282 void genLongReturn(GenTree* treeNode);
1283 #endif // TARGET_X86 || TARGET_ARM
1285 #if defined(TARGET_X86)
1286 void genFloatReturn(GenTree* treeNode);
1287 #endif // TARGET_X86
1289 #if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
1290 void genSimpleReturn(GenTree* treeNode);
1291 #endif // TARGET_ARM64 || TARGET_LOONGARCH64 || TARGET_RISCV64
1293 void genReturn(GenTree* treeNode);
1296 void genStackPointerConstantAdjustment(ssize_t spDelta, bool trackSpAdjustments);
1297 void genStackPointerConstantAdjustmentWithProbe(ssize_t spDelta, bool trackSpAdjustments);
1298 target_ssize_t genStackPointerConstantAdjustmentLoopWithProbe(ssize_t spDelta, bool trackSpAdjustments);
1299 void genStackPointerDynamicAdjustmentWithProbe(regNumber regSpDelta);
1300 #else // !TARGET_XARCH
1301 void genStackPointerConstantAdjustment(ssize_t spDelta, regNumber regTmp);
1302 void genStackPointerConstantAdjustmentWithProbe(ssize_t spDelta, regNumber regTmp);
1303 target_ssize_t genStackPointerConstantAdjustmentLoopWithProbe(ssize_t spDelta, regNumber regTmp);
1304 #endif // !TARGET_XARCH
1306 void genLclHeap(GenTree* tree);
1307 void genCodeForMemmove(GenTreeBlk* tree);
1309 bool genIsRegCandidateLocal(GenTree* tree)
1311 if (!tree->IsLocal())
1315 return compiler->lvaGetDesc(tree->AsLclVarCommon())->lvIsRegCandidate();
1318 #ifdef FEATURE_PUT_STRUCT_ARG_STK
1321 #else // !TARGET_X86
1322 unsigned m_stkArgVarNum;
1323 unsigned m_stkArgOffset;
1324 #endif // !TARGET_X86
1325 #endif // !FEATURE_PUT_STRUCT_ARG_STK
1327 #if defined(DEBUG) && defined(TARGET_XARCH)
1328 void genStackPointerCheck(bool doStackPointerCheck,
1329 unsigned lvaStackPointerVar,
1331 regNumber regTmp = REG_NA);
1332 #endif // defined(DEBUG) && defined(TARGET_XARCH)
1335 GenTree* lastConsumedNode;
1336 void genNumberOperandUse(GenTree* const operand, int& useNum) const;
1337 void genCheckConsumeNode(GenTree* const node);
1339 inline void genCheckConsumeNode(GenTree* treeNode)
1345 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1346 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1350 XX The interface to generate a machine-instruction. XX
1351 XX Currently specific to x86 XX
1352 XX TODO-Cleanup: Consider factoring this out of CodeGen XX
1354 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1355 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
1359 void instGen(instruction ins);
1360 #if defined(TARGET_XARCH)
1361 void inst_JMP(emitJumpKind jmp, BasicBlock* tgtBlock, bool isRemovableJmpCandidate = false);
1363 void inst_JMP(emitJumpKind jmp, BasicBlock* tgtBlock);
1366 void inst_SET(emitJumpKind condition, regNumber reg);
1368 void inst_RV(instruction ins, regNumber reg, var_types type, emitAttr size = EA_UNKNOWN);
1370 void inst_Mov(var_types dstType,
1374 emitAttr size = EA_UNKNOWN,
1375 insFlags flags = INS_FLAGS_DONT_CARE);
1377 void inst_Mov_Extend(var_types srcType,
1382 emitAttr size = EA_UNKNOWN,
1383 insFlags flags = INS_FLAGS_DONT_CARE);
1385 void inst_RV_RV(instruction ins,
1388 var_types type = TYP_I_IMPL,
1389 emitAttr size = EA_UNKNOWN,
1390 insFlags flags = INS_FLAGS_DONT_CARE);
1392 void inst_RV_RV_RV(instruction ins,
1397 insFlags flags = INS_FLAGS_DONT_CARE);
1399 void inst_IV(instruction ins, cnsval_ssize_t val);
1400 void inst_IV_handle(instruction ins, cnsval_ssize_t val);
1403 instruction ins, regNumber reg, target_ssize_t val, emitAttr size, insFlags flags = INS_FLAGS_DONT_CARE);
1405 void inst_ST_RV(instruction ins, TempDsc* tmp, unsigned ofs, regNumber reg, var_types type);
1407 void inst_FS_ST(instruction ins, emitAttr size, TempDsc* tmp, unsigned ofs);
1409 void inst_TT_RV(instruction ins, emitAttr size, GenTree* tree, regNumber reg);
1411 void inst_RV_SH(instruction ins, emitAttr size, regNumber reg, unsigned val, insFlags flags = INS_FLAGS_DONT_CARE);
1413 #if defined(TARGET_XARCH)
1415 enum class OperandKind{
1416 ClsVar, // [CLS_VAR_ADDR] - "C" in the emitter.
1417 Local, // [Local or spill temp + offset] - "S" in the emitter.
1418 Indir, // [base+index*scale+disp] - "A" in the emitter.
1419 Imm, // immediate - "I" in the emitter.
1420 Reg // reg - "R" in the emitter.
1429 CORINFO_FIELD_HANDLE m_fieldHnd;
1439 GenTreeIndir* m_indir;
1440 var_types m_indirType;
1444 ssize_t m_immediate;
1445 bool m_immediateNeedsReloc;
1454 OperandDesc(CORINFO_FIELD_HANDLE fieldHnd) : m_kind(OperandKind::ClsVar), m_fieldHnd(fieldHnd)
1458 OperandDesc(int varNum, uint16_t offset) : m_kind(OperandKind::Local), m_varNum(varNum), m_offset(offset)
1462 OperandDesc(GenTreeIndir* indir)
1463 : m_kind(OperandKind::Indir), m_addr(indir->Addr()), m_indir(indir), m_indirType(indir->TypeGet())
1467 OperandDesc(var_types indirType, GenTree* addr)
1468 : m_kind(OperandKind::Indir), m_addr(addr), m_indir(nullptr), m_indirType(indirType)
1472 OperandDesc(ssize_t immediate, bool immediateNeedsReloc)
1473 : m_kind(OperandKind::Imm), m_immediate(immediate), m_immediateNeedsReloc(immediateNeedsReloc)
1477 OperandDesc(regNumber reg) : m_kind(OperandKind::Reg), m_reg(reg)
1481 OperandKind GetKind() const
1486 CORINFO_FIELD_HANDLE GetFieldHnd() const
1488 assert(m_kind == OperandKind::ClsVar);
1492 int GetVarNum() const
1494 assert(m_kind == OperandKind::Local);
1498 int GetLclOffset() const
1500 assert(m_kind == OperandKind::Local);
1504 // TODO-Cleanup: instead of this rather unsightly workaround with
1505 // "indirForm", create a new abstraction for address modes to pass
1506 // to the emitter (or at least just use "addr"...).
1507 GenTreeIndir* GetIndirForm(GenTreeIndir* pIndirForm)
1509 if (m_indir == nullptr)
1511 GenTreeIndir indirForm = CodeGen::indirForm(m_indirType, m_addr);
1512 memcpy((void*)pIndirForm, (void*)&indirForm, sizeof(GenTreeIndir));
1516 pIndirForm = m_indir;
1522 ssize_t GetImmediate() const
1524 assert(m_kind == OperandKind::Imm);
1528 emitAttr GetEmitAttrForImmediate(emitAttr baseAttr) const
1530 assert(m_kind == OperandKind::Imm);
1531 return m_immediateNeedsReloc ? EA_SET_FLG(baseAttr, EA_CNS_RELOC_FLG) : baseAttr;
1534 regNumber GetReg() const
1539 bool IsContained() const
1541 return m_kind != OperandKind::Reg;
1545 OperandDesc genOperandDesc(GenTree* op);
1547 void inst_TT(instruction ins, emitAttr size, GenTree* op1);
1548 void inst_RV_TT(instruction ins, emitAttr size, regNumber op1Reg, GenTree* op2);
1549 void inst_RV_RV_IV(instruction ins, emitAttr size, regNumber reg1, regNumber reg2, unsigned ival);
1550 void inst_RV_TT_IV(instruction ins, emitAttr attr, regNumber reg1, GenTree* rmOp, int ival);
1551 void inst_RV_RV_TT(instruction ins, emitAttr size, regNumber targetReg, regNumber op1Reg, GenTree* op2, bool isRMW);
1552 void inst_RV_RV_TT_IV(
1553 instruction ins, emitAttr size, regNumber targetReg, regNumber op1Reg, GenTree* op2, int8_t ival, bool isRMW);
1556 void inst_set_SV_var(GenTree* tree);
1559 bool arm_Valid_Imm_For_Instr(instruction ins, target_ssize_t imm, insFlags flags);
1560 bool arm_Valid_Imm_For_Add(target_ssize_t imm, insFlags flag);
1561 bool arm_Valid_Imm_For_Add_SP(target_ssize_t imm);
1564 instruction ins_Move_Extend(var_types srcType, bool srcInReg);
1566 instruction ins_Copy(var_types dstType);
1567 instruction ins_Copy(regNumber srcReg, var_types dstType);
1568 #if defined(TARGET_XARCH)
1569 instruction ins_FloatConv(var_types to, var_types from, emitAttr attr);
1570 #elif defined(TARGET_ARM)
1571 instruction ins_FloatConv(var_types to, var_types from);
1573 instruction ins_MathOp(genTreeOps oper, var_types type);
1575 void instGen_Return(unsigned stkArgSize);
1579 BARRIER_FULL, // full barrier
1580 BARRIER_LOAD_ONLY, // load barier
1583 void instGen_MemoryBarrier(BarrierKind barrierKind = BARRIER_FULL);
1585 void instGen_Set_Reg_To_Zero(emitAttr size, regNumber reg, insFlags flags = INS_FLAGS_DONT_CARE);
1587 void instGen_Set_Reg_To_Imm(emitAttr size,
1590 insFlags flags = INS_FLAGS_DONT_CARE DEBUGARG(size_t targetHandle = 0)
1591 DEBUGARG(GenTreeFlags gtFlags = GTF_EMPTY));
1594 instruction genMapShiftInsToShiftByConstantIns(instruction ins, int shiftByValue);
1595 #endif // TARGET_XARCH
1597 #if defined(TARGET_ARM64)
1598 static insCond JumpKindToInsCond(emitJumpKind condition);
1599 static insOpts ShiftOpToInsOpts(genTreeOps op);
1600 #elif defined(TARGET_XARCH)
1601 static instruction JumpKindToCmov(emitJumpKind condition);
1604 #if !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64)
1605 // Maps a GenCondition code to a sequence of conditional jumps or other conditional instructions
1606 // such as X86's SETcc. A sequence of instructions rather than just a single one is required for
1607 // certain floating point conditions.
1608 // For example, X86's UCOMISS sets ZF to indicate equality but it also sets it, together with PF,
1609 // to indicate an unordered result. So for GenCondition::FEQ we first need to check if PF is 0
1610 // and then jump if ZF is 1:
1611 // JP fallThroughBlock
1613 // fallThroughBlock:
1617 // This is very similar to the way shortcircuit evaluation of bool AND and OR operators works so
1618 // in order to make the GenConditionDesc mapping tables easier to read, a bool expression-like
1619 // pattern is used to encode the above:
1620 // { EJ_jnp, GT_AND, EJ_je }
1621 // { EJ_jp, GT_OR, EJ_jne }
1623 // For more details check inst_JCC and inst_SETCC functions.
1625 struct GenConditionDesc
1627 emitJumpKind jumpKind1;
1629 emitJumpKind jumpKind2;
1632 static const GenConditionDesc& Get(GenCondition condition)
1634 assert(condition.GetCode() < ArrLen(map));
1635 const GenConditionDesc& desc = map[condition.GetCode()];
1636 assert(desc.jumpKind1 != EJ_NONE);
1637 assert((desc.oper == GT_NONE) || (desc.oper == GT_AND) || (desc.oper == GT_OR));
1638 assert((desc.oper == GT_NONE) == (desc.jumpKind2 == EJ_NONE));
1643 static const GenConditionDesc map[32];
1646 void inst_JCC(GenCondition condition, BasicBlock* target);
1647 void inst_SETCC(GenCondition condition, var_types type, regNumber dstReg);
1649 void genCodeForJcc(GenTreeCC* tree);
1650 void genCodeForSetcc(GenTreeCC* setcc);
1651 void genCodeForJTrue(GenTreeOp* jtrue);
1652 #endif // !TARGET_LOONGARCH64 && !TARGET_RISCV64
1655 // A simple phase that just invokes a method on the codegen instance
1657 class CodeGenPhase final : public Phase
1660 CodeGenPhase(CodeGen* _codeGen, Phases _phase, void (CodeGen::*_action)())
1661 : Phase(_codeGen->GetCompiler(), _phase), codeGen(_codeGen), action(_action)
1666 virtual PhaseStatus DoPhase() override
1668 (codeGen->*action)();
1669 return PhaseStatus::MODIFIED_EVERYTHING;
1674 void (CodeGen::*action)();
1677 // Wrapper for using CodeGenPhase
1679 inline void DoPhase(CodeGen* _codeGen, Phases _phase, void (CodeGen::*_action)())
1681 CodeGenPhase phase(_codeGen, _phase, _action);
1685 #endif // _CODEGEN_H_