From 9e202d4e48212e7e18cd51fbae38244660ad9733 Mon Sep 17 00:00:00 2001 From: Kyungwoo Lee Date: Fri, 6 May 2016 08:11:08 -0700 Subject: [PATCH] ARM64: Enable Long Address Fixes https://github.com/dotnet/coreclr/issues/3668 Currently ARM64 codegen can have reference within +/-1 MB due to encoding restriction in `b/adr/ldr` instructions. This is normally okay assuming each function is reasonably small, but certainly not working for large method which also can be formed with an aggressive inlining probably like crossgen/corert scenarios. In addition, for hot/cold code separation long address is a prerequisite since reference can be across different regions which are arbitrary. In fact, we need additional relocations which are not in this change yet. In details, this supports long address for conditional jump/address loading/constant loading operations by default while they can be shortened later by `emitJumpDistBind()` if they can fit into the smaller encoding. Logically those operations now can reach within +/-4GB address range. Note I haven't extended unconditional jump in this change for simplicity so it can reach within +/-128MB same as before. `emitOutputLJ` is extended to finally encode these operations. There are 3 pseudo instructions introduced. These can be expanded either short/long form. 1. Conditional jump. See `emitIns_J()` a. Short form(`IF_BI_0B`): `b rel_addr` b. Long form(`IF_LARGEJMP`): ``` b $LABEL b rel_addr (unconditional jump) $LABEL: ``` 2. Load label(address computation). See `emitIns_R_L()` a. Short form(`IF_DI_1E`): `adr x, [rel_addr]` b. Long form(`IF_LARGEADR`): ``` adrp x, [rel_page_addr] add x, x, page_offs ``` 3. Load constant (from JIT data). See `emitIns_R_C()` a. Short form(`IF_LS_1A`): `ldr x, [rel_addr]` b. Long form(`IF_LARGLDC`): ``` adrp x, [rel_page_addr] ldr x, [x, page_offs] (fmov v, x in case loading vector constant) ``` In addition, JIT data is aligned on 8 byte to be accessible from large load. Replaced JitLargeBranches by JitLongAddress to test stress on these operations. Commit migrated from https://github.com/dotnet/coreclr/commit/61fe4641665e84089dcceeabbea3e5faa0f693ce --- src/coreclr/src/jit/codegenarm64.cpp | 17 +- src/coreclr/src/jit/compiler.cpp | 6 +- src/coreclr/src/jit/compiler.h | 2 +- src/coreclr/src/jit/emit.cpp | 71 +++- src/coreclr/src/jit/emit.h | 29 +- src/coreclr/src/jit/emitarm.cpp | 3 +- src/coreclr/src/jit/emitarm64.cpp | 605 +++++++++++++++++++++++++--------- src/coreclr/src/jit/emitarm64.h | 21 +- src/coreclr/src/jit/emitfmtsarm64.h | 4 +- src/coreclr/src/jit/flowgraph.cpp | 13 + src/coreclr/src/jit/jitconfigvalues.h | 2 +- src/coreclr/src/jit/lowerarm64.cpp | 14 + src/coreclr/src/jit/target.h | 14 +- src/coreclr/tests/arm64/Tests.lst | 2 +- 14 files changed, 622 insertions(+), 181 deletions(-) diff --git a/src/coreclr/src/jit/codegenarm64.cpp b/src/coreclr/src/jit/codegenarm64.cpp index c8bef50..4885fb1 100644 --- a/src/coreclr/src/jit/codegenarm64.cpp +++ b/src/coreclr/src/jit/codegenarm64.cpp @@ -2065,7 +2065,9 @@ void CodeGen::genCodeForBBlist() break; case BBJ_EHCATCHRET: - getEmitter()->emitIns_R_L(INS_adr, EA_4BYTE_DSP_RELOC, block->bbJumpDest, REG_INTRET); + // For long address (default): `adrp + add` will be emitted. + // For short address (proven later): `adr` will be emitted. + getEmitter()->emitIns_R_L(INS_adr, EA_PTRSIZE, block->bbJumpDest, REG_INTRET); __fallthrough; @@ -2248,10 +2250,17 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types tar } else { + // Get a temp integer register to compute long address. + regMaskTP addrRegMask = tree->gtRsvdRegs; + regNumber addrReg = genRegNumFromMask(addrRegMask); + noway_assert(addrReg != REG_NA); + // We must load the FP constant from the constant pool // Emit a data section constant for the float or double constant. CORINFO_FIELD_HANDLE hnd = emit->emitFltOrDblConst(dblConst); - emit->emitIns_R_C(INS_ldr, size, targetReg, hnd, 0); + // For long address (default): `adrp + ldr + fmov` will be emitted. + // For short address (proven later), `ldr` will be emitted. + emit->emitIns_R_C(INS_ldr, size, targetReg, addrReg, hnd, 0); } } break; @@ -3271,6 +3280,9 @@ CodeGen::genCodeForTreeNode(GenTreePtr treeNode) case GT_LABEL: genPendingCallLabel = genCreateTempLabel(); treeNode->gtLabel.gtLabBB = genPendingCallLabel; + + // For long address (default): `adrp + add` will be emitted. + // For short address (proven later): `adr` will be emitted. emit->emitIns_R_L(INS_adr, EA_PTRSIZE, genPendingCallLabel, targetReg); break; @@ -4203,6 +4215,7 @@ CodeGen::genJumpTable(GenTree* treeNode) getEmitter()->emitIns_R_C(INS_lea, emitTypeSize(TYP_I_IMPL), treeNode->gtRegNum, + REG_NA, compiler->eeFindJitDataOffs(jmpTabBase), 0); genProduceReg(treeNode); diff --git a/src/coreclr/src/jit/compiler.cpp b/src/coreclr/src/jit/compiler.cpp index 3b2c3d6..15dba0d 100644 --- a/src/coreclr/src/jit/compiler.cpp +++ b/src/coreclr/src/jit/compiler.cpp @@ -2465,7 +2465,7 @@ void Compiler::compInitOptions(CORJIT_FLAGS* jitFlags) opts.disAsm2 = false; opts.dspUnwind = false; s_dspMemStats = false; - opts.compLargeBranches = false; + opts.compLongAddress = false; opts.compJitELTHookEnabled = false; #ifdef LATE_DISASM @@ -2534,8 +2534,8 @@ void Compiler::compInitOptions(CORJIT_FLAGS* jitFlags) if (JitConfig.DisplayMemStats() != 0) s_dspMemStats = true; - if (JitConfig.JitLargeBranches() != 0) - opts.compLargeBranches = true; + if (JitConfig.JitLongAddress() != 0) + opts.compLongAddress = true; } if (verboseDump) diff --git a/src/coreclr/src/jit/compiler.h b/src/coreclr/src/jit/compiler.h index 95995c3..d757b68 100644 --- a/src/coreclr/src/jit/compiler.h +++ b/src/coreclr/src/jit/compiler.h @@ -7554,7 +7554,7 @@ public : bool dspOrder; // Display names of each of the methods that we ngen/jit bool dspUnwind; // Display the unwind info output bool dspDiffable; // Makes the Jit Dump 'diff-able' (currently uses same COMPlus_* flag as disDiffable) - bool compLargeBranches; // Force using large conditional branches + bool compLongAddress;// Force using large pseudo instructions for long address (IF_LARGEJMP/IF_LARGEADR/IF_LARGLDC) bool dspGCtbls; // Display the GC tables #endif diff --git a/src/coreclr/src/jit/emit.cpp b/src/coreclr/src/jit/emit.cpp index 9b8aadd..cea1509 100644 --- a/src/coreclr/src/jit/emit.cpp +++ b/src/coreclr/src/jit/emit.cpp @@ -3721,6 +3721,8 @@ AGAIN: else if (emitIsUncondJump(jmp)) { // Nothing to do; we don't shrink these. + assert(jmp->idjShort); + ssz = JMP_SIZE_SMALL; } else if (emitIsCmpJump(jmp)) { @@ -3732,6 +3734,12 @@ AGAIN: nsd = LBL_DIST_SMALL_MAX_NEG; psd = LBL_DIST_SMALL_MAX_POS; } + else if (emitIsLoadConstant(jmp)) + { + ssz = LDC_SIZE_SMALL; + nsd = LDC_DIST_SMALL_MAX_NEG; + psd = LDC_DIST_SMALL_MAX_POS; + } else { assert(!"Unknown jump instruction"); @@ -3793,6 +3801,42 @@ AGAIN: // If this is a jump via register, the instruction size does not change, so we are done. +#if defined(_TARGET_ARM64_) + // JIT code and data will be allocated together for arm64 so the relative offset to JIT data is known. + // In case such offset can be encodeable for `ldr` (+-1MB), shorten it. + if (emitIsLoadConstant(jmp)) + { + // Reference to JIT data + assert(jmp->idAddr()->iiaIsJitDataOffset()); + assert(jmp->idIsBound()); + UNATIVE_OFFSET srcOffs = jmpIG->igOffs + jmp->idjOffs; + + int doff = jmp->idAddr()->iiaGetJitDataOffset(); + assert(doff >= 0); + ssize_t imm = emitGetInsSC(jmp); + assert((imm >= 0) && (imm < 0x1000)); // 0x1000 is arbitrary, currently 'imm' is always 0 + + unsigned dataOffs = (unsigned)(doff + imm); + assert(dataOffs < emitDataSize()); + + // Conservately assume JIT data starts after the entire code size. + // TODO-ARM64: we might consider only hot code size which will be computed later in emitComputeCodeSizes(). + assert(emitTotalCodeSize > 0); + UNATIVE_OFFSET maxDstOffs = emitTotalCodeSize + dataOffs; + + // Check if the distance is within the encoding length. + UNATIVE_OFFSET jmpDist = maxDstOffs - srcOffs; + extra = jmpDist - psd; + if (extra <= 0) + { + goto SHORT_JMP; + } + + // Keep the large form. + continue; + } +#endif + /* Have we bound this jump's target already? */ if (jmp->idIsBound()) @@ -3981,10 +4025,6 @@ AGAIN: } } - // TODO-ARM64-NYI: we couldn't shorten the branch/label load into a size that fits a single b.cond or adr instruction. - // We need to implement multiple-instruction sequences to handle large ranges, e.g. b.!cond / b, and adrp/adr. - NYI_ARM64("Implement pseudo-instructions for large conditional branch and large load label address"); - /* We arrive here if the jump couldn't be made short, at least for now */ /* We had better not have eagerly marked the jump as short @@ -4109,7 +4149,9 @@ AGAIN: // insSize isz = emitInsSize(jmp->idInsFmt()); // jmp->idInsSize(isz); #elif defined(_TARGET_ARM64_) - // TODO-ARM64-NYI: Support large conditional pseudo-op branches + // The size of IF_LARGEJMP/IF_LARGEADR/IF_LARGELDC are 8 or 12. + // All other code size is 4. + assert((sizeDif == 4) || (sizeDif == 8)); #else #error Unsupported or unset target architecture #endif @@ -4505,14 +4547,21 @@ unsigned emitter::emitEndCodeGen(Compiler *comp, NYI_ARM64("Need to handle fix-up to data from cold code."); } - emitCmpHandle->allocMem(emitTotalHotCodeSize + emitConsDsc.dsdOffs, emitTotalColdCodeSize, + UNATIVE_OFFSET roDataAlignmentDelta = 0; + if (emitConsDsc.dsdOffs) + { + UNATIVE_OFFSET roDataAlignment = sizeof(void*); // 8 Byte align by default. + roDataAlignmentDelta = (UNATIVE_OFFSET)ALIGN_UP(emitTotalHotCodeSize, roDataAlignment) - emitTotalHotCodeSize; + assert((roDataAlignmentDelta == 0) || (roDataAlignmentDelta == 4)); + } + emitCmpHandle->allocMem(emitTotalHotCodeSize + roDataAlignmentDelta + emitConsDsc.dsdOffs, emitTotalColdCodeSize, 0, xcptnsCount, allocMemFlag, (void**)&codeBlock, (void**)&coldCodeBlock, (void**)&consBlock); - consBlock = codeBlock + emitTotalHotCodeSize; + consBlock = codeBlock + emitTotalHotCodeSize + roDataAlignmentDelta; #else emitCmpHandle->allocMem( emitTotalHotCodeSize, emitTotalColdCodeSize, @@ -4973,7 +5022,8 @@ unsigned emitter::emitEndCodeGen(Compiler *comp, // Presumably we could also just call "emitOutputLJ(NULL, adr, jmp)", like for long jumps? *(short int *)adr -= (short)adj; #elif defined(_TARGET_ARM64_) - NYI_ARM64("Fill this in for Arm64"); + assert(!jmp->idAddr()->iiaHasInstrCount()); + emitOutputLJ(NULL, adr, jmp); #else #error Unsupported or unset target architecture #endif @@ -4983,12 +5033,9 @@ unsigned emitter::emitEndCodeGen(Compiler *comp, // Patch Forward non-Short Jump #if defined(_TARGET_XARCH_) *(int *)adr -= adj; -#elif defined(_TARGET_ARM_) +#elif defined(_TARGET_ARMARCH_) assert(!jmp->idAddr()->iiaHasInstrCount()); - // This handles both medium and long jumps emitOutputLJ(NULL, adr, jmp); -#elif defined(_TARGET_ARM64_) - NYI_ARM64("Fill this in for Arm64"); #else #error Unsupported or unset target architecture #endif diff --git a/src/coreclr/src/jit/emit.h b/src/coreclr/src/jit/emit.h index 67adcdf..41fc46c 100644 --- a/src/coreclr/src/jit/emit.h +++ b/src/coreclr/src/jit/emit.h @@ -983,8 +983,32 @@ protected: void idCodeSize(unsigned sz) { _idCodeSize = sz; assert(sz == _idCodeSize); } #elif defined(_TARGET_ARM64_) - - unsigned idCodeSize() const { return 4; } + unsigned idCodeSize() const { + int size = 4; + switch (idInsFmt()) + { + case IF_LARGEADR: + // adrp + add + case IF_LARGEJMP: + // b + b + size = 8; + break; + case IF_LARGELDC: + if (isVectorRegister(idReg1())) + { + // adrp + ldr + fmov + size = 12; + } + else + { + // adrp + ldr + size = 8; + } + break; + } + + return size; + } #elif defined(_TARGET_ARM_) @@ -1797,6 +1821,7 @@ public: void emitSetFrameRangeArgs(int offsLo, int offsHi); static instruction emitJumpKindToIns(emitJumpKind jumpKind); + static emitJumpKind emitInsToJumpKind(instruction ins); static emitJumpKind emitReverseJumpKind(emitJumpKind jumpKind); #ifdef _TARGET_ARM_ diff --git a/src/coreclr/src/jit/emitarm.cpp b/src/coreclr/src/jit/emitarm.cpp index 99049e4..e2daab4 100644 --- a/src/coreclr/src/jit/emitarm.cpp +++ b/src/coreclr/src/jit/emitarm.cpp @@ -10,7 +10,6 @@ XX XX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */ - #include "jitpch.h" #ifdef _MSC_VER #pragma hdrstop @@ -4280,7 +4279,7 @@ void emitter::emitIns_J(instruction ins, id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); #ifdef DEBUG - if (emitComp->opts.compLargeBranches) // Force long branches + if (emitComp->opts.compLongAddress) // Force long branches id->idjKeepLong = 1; #endif // DEBUG } diff --git a/src/coreclr/src/jit/emitarm64.cpp b/src/coreclr/src/jit/emitarm64.cpp index 33843d3..3eec0ea 100644 --- a/src/coreclr/src/jit/emitarm64.cpp +++ b/src/coreclr/src/jit/emitarm64.cpp @@ -56,6 +56,25 @@ const emitJumpKind emitReverseJumpKinds[] = } /***************************************************************************** +* Look up the jump kind for an instruction. It better be a conditional +* branch instruction with a jump kind! +*/ + +/*static*/ emitJumpKind emitter::emitInsToJumpKind(instruction ins) +{ + for (unsigned i = 0; i < ArrLen(emitJumpKindInstructions); i++) + { + if (ins == emitJumpKindInstructions[i]) + { + emitJumpKind ret = (emitJumpKind)i; + assert(EJ_NONE < ret && ret < EJ_COUNT); + return ret; + } + } + unreached(); +} + +/***************************************************************************** * Reverse the conditional jump */ @@ -157,6 +176,11 @@ void emitter::emitInsSanityCheck(instrDesc *id) case IF_BI_0B: // BI_0B ......iiiiiiiiii iiiiiiiiiiii.... simm19:00 break; + case IF_LARGEJMP: + case IF_LARGEADR: + case IF_LARGELDC: + break; + case IF_BI_0C: // BI_0C ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 break; @@ -6321,6 +6345,7 @@ void emitter::emitIns_S_I (instruction ins, void emitter::emitIns_R_C (instruction ins, emitAttr attr, regNumber reg, + regNumber addrReg, CORINFO_FIELD_HANDLE fldHnd, int offs) { @@ -6332,24 +6357,27 @@ void emitter::emitIns_R_C (instruction ins, int disp = 0; instrDescJmp* id = emitNewInstrJmp(); - // TODO-ARM64-CQ: use unscaled loads? - /* Figure out the encoding format of the instruction */ switch (ins) { case INS_ldr: + fmt = IF_LARGELDC; if (isVectorRegister(reg)) { assert(isValidScalarDatasize(size)); + // For vector (float/double) register, we should have an integer address reg to + // compute long address which consists of page address and page offset. + // For integer constant, this is not needed since the dest reg can be used to + // compute address as well as contain the final contents. + assert(isGeneralRegister(reg) || (addrReg != REG_NA)); } else { assert(isGeneralRegister(reg)); assert(isValidGeneralDatasize(size)); } - fmt = IF_LS_1A; break; default: - break; + unreached(); } assert(fmt != IF_NONE); @@ -6360,9 +6388,36 @@ void emitter::emitIns_R_C (instruction ins, id->idSmallCns(offs); id->idOpSize(size); id->idAddr()->iiaFieldHnd = fldHnd; - id->idSetIsBound(); + id->idSetIsBound(); // We won't patch address since we will know the exact distance once JIT code and data are allocated together. - id->idReg1(reg); + id->idReg1(reg); // destination register that will get the constant value. + id->idReg2(addrReg); // integer register to compute long address (used for vector dest when we end up with long address) + id->idjShort = false; // Assume loading constant from long address + + // Keep it long if it's in cold code. + id->idjKeepLong = emitComp->fgIsBlockCold(emitComp->compCurBB); + +#ifdef DEBUG + if (emitComp->opts.compLongAddress) + id->idjKeepLong = 1; +#endif // DEBUG + + // If it's possible to be shortened, then put it in jump list + // to be revisited by emitJumpDistBind. + if (!id->idjKeepLong) + { + /* Record the jump's IG and offset within it */ + id->idjIG = emitCurIG; + id->idjOffs = emitCurIGsize; + + /* Append this jump to this IG's jump list */ + id->idjNext = emitCurIGjmpList; + emitCurIGjmpList = id; + +#if EMITTER_STATS + emitTotalIGjmps++; +#endif + } dispIns(id); appendToCurIG(id); @@ -6515,8 +6570,27 @@ void emitter::emitIns_R_ARX (instruction ins, */ void emitter::emitSetShortJump(instrDescJmp * id) { - // All jumps are the same size. - // NYI: support large conditional branches via pseudo-op. + if (id->idjKeepLong) + return; + + insFormat fmt = IF_NONE; + if (emitIsCondJump(id)) + { + fmt = IF_BI_0B; + } + else if (emitIsLoadLabel(id)) + { + fmt = IF_DI_1E; + } + else if (emitIsLoadConstant(id)) + { + fmt = IF_LS_1A; + } + else { + unreached(); + } + + id->idInsFmt(fmt); id->idjShort = true; } @@ -6534,23 +6608,20 @@ void emitter::emitIns_R_L (instruction ins, insFormat fmt = IF_NONE; - instrDescJmp *id; switch (ins) { case INS_adr: - case INS_adrp: - fmt = IF_DI_1E; + fmt = IF_LARGEADR; break; default: - // TODO-Cleanup: add unreached() here - break; + unreached(); } - assert(fmt == IF_DI_1E); - id = emitNewInstrJmp(); + instrDescJmp* id = emitNewInstrJmp(); id->idIns(ins); id->idInsFmt(fmt); + id->idjShort = false; id->idAddr()->iiaBBlabel = dst; id->idReg1(reg); id->idOpSize(EA_PTRSIZE); @@ -6563,11 +6634,13 @@ void emitter::emitIns_R_L (instruction ins, } #endif // DEBUG - /* Assume the label reference will be long */ - - id->idjShort = 0; id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); - + +#ifdef DEBUG + if (emitComp->opts.compLongAddress) + id->idjKeepLong = 1; +#endif // DEBUG + /* Record the jump's IG and offset within it */ id->idjIG = emitCurIG; @@ -6582,13 +6655,6 @@ void emitter::emitIns_R_L (instruction ins, emitTotalIGjmps++; #endif - /* Figure out the max. size of the instruction */ - - if (!id->idjKeepLong) - { - // TODO-ARM64-NYI: handle large labels: larger than adr can do (+/-1MB) - } - dispIns(id); appendToCurIG(id); } @@ -6631,10 +6697,14 @@ void emitter::emitIns_J(instruction ins, } /* Figure out the encoding format of the instruction */ + + bool idjShort = false; switch (ins) { case INS_bl_local: case INS_b: + // Unconditional jump is a single form. + idjShort = true; fmt = IF_BI_0A; break; @@ -6652,20 +6722,20 @@ void emitter::emitIns_J(instruction ins, case INS_blt: case INS_bgt: case INS_ble: - // TODO-ARM64-CQ: fmt = IF_LARGEJMP; /* Assume the jump will be long */ - fmt = IF_BI_0B; + // Assume conditional jump is long. + fmt = IF_LARGEJMP; break; + default: - // TODO-Cleanup: add unreached() here + unreached(); break; } - assert((fmt == IF_BI_0A) || - (fmt == IF_BI_0B)); instrDescJmp* id = emitNewInstrJmp(); id->idIns(ins); id->idInsFmt(fmt); + id->idjShort = idjShort; #ifdef DEBUG // Mark the finally call @@ -6675,19 +6745,22 @@ void emitter::emitIns_J(instruction ins, } #endif // DEBUG - /* Assume the jump will be long */ - - // TODO-ARM64-Cleanup: there is only one size jump on ARM64. Clean this up. - id->idjShort = 1; if (dst != nullptr) { id->idAddr()->iiaBBlabel = dst; - id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); + + // Skip unconditional jump that has a single form. + // TODO-ARM64-NYI: enable hot/cold splittingNYI. + // The target needs to be relocated. + if (!idjShort) + { + id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); #ifdef DEBUG - if (emitComp->opts.compLargeBranches) // Force long branches - id->idjKeepLong = 1; + if (emitComp->opts.compLongAddress) // Force long branches + id->idjKeepLong = 1; #endif // DEBUG + } } else { @@ -6712,13 +6785,6 @@ void emitter::emitIns_J(instruction ins, emitTotalIGjmps++; #endif - /* Figure out the max. size of the jump/call instruction */ - - if (!id->idjKeepLong) - { - // TODO-ARM64-NYI: handle large conditional branches: larger than b.cond can do (+/-1MB) - } - dispIns(id); appendToCurIG(id); } @@ -7832,12 +7898,13 @@ BYTE* emitter::emitOutputLJ(insGroup *ig, BYTE *dst, instrDesc *i ssize_t distVal; ssize_t loBits; + // Set default ins/fmt from id. instruction ins = id->idIns(); insFormat fmt = id->idInsFmt(); - code_t code = emitInsCode(ins, fmt); // Basic instruction encoding bool loadLabel = false; bool isJump = false; + bool loadConstant = false; switch (ins) { @@ -7855,7 +7922,16 @@ BYTE* emitter::emitOutputLJ(insGroup *ig, BYTE *dst, instrDesc *i case INS_ldr: case INS_ldrsw: case INS_adr: - loadLabel = true; + case INS_adrp: + // Any reference to JIT data is assumed to load constant. + if (id->idAddr()->iiaIsJitDataOffset()) + { + loadConstant = true; + } + else + { + loadLabel = true; + } break; } @@ -7864,6 +7940,78 @@ BYTE* emitter::emitOutputLJ(insGroup *ig, BYTE *dst, instrDesc *i srcOffs = emitCurCodeOffs(dst); srcAddr = emitOffsetToPtr(srcOffs); + if (loadConstant) + { + /* This is actually a reference to the JIT data section */ + int doff = id->idAddr()->iiaGetJitDataOffset(); + assert(doff >= 0); + ssize_t imm = emitGetInsSC(id); + assert((imm >= 0) && (imm < 0x1000)); // 0x1000 is arbitrary, currently 'imm' is always 0 + + unsigned dataOffs = (unsigned)(doff + imm); + assert(dataOffs < emitDataSize()); + dstAddr = emitDataOffsetToPtr(dataOffs); + + regNumber dstReg = id->idReg1(); + regNumber addrReg = dstReg; // an integer register to compute long address. + emitAttr opSize = id->idOpSize(); + + if (id->idjShort) + { + // ldr x/v, [rel addr] -- load constant from current addr(ip) + rel addr. + assert(ins == INS_ldr); + assert(fmt == IF_LS_1A); + distVal = (ssize_t)(dstAddr - srcAddr); + dst = emitOutputShortConstant(dst, ins, fmt, distVal, dstReg, opSize); + } + else + { + // adrp x, [rel page addr] -- compute page address: current page addr + rel page addr + assert(fmt == IF_LARGELDC); + ssize_t relPageAddr = (((ssize_t)dstAddr & 0xFFFFFFFFFFFFF000LL) - ((ssize_t)srcAddr & 0xFFFFFFFFFFFFF000LL)) >> 12; + if (isVectorRegister(dstReg)) + { + // Update addrReg with the reserved integer register + // since we cannot use dstReg (vector) to load constant directly from memory. + addrReg = id->idReg2(); + assert(isGeneralRegister(addrReg)); + } + ins = INS_adrp; + fmt = IF_DI_1E; + dst = emitOutputShortAddress(dst, ins, fmt, relPageAddr, addrReg); + + // ldr x, [x, page offs] -- load constant from page address + page offset into integer register. + ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits + assert(isValidUimm12(imm12)); + ins = INS_ldr; + fmt = IF_LS_2B; + dst = emitOutputShortConstant(dst, ins, fmt, imm12, addrReg, opSize); + + // fmov v, d -- copy constant in integer register to vector register. + // This is needed only for vector constant. + if (addrReg != dstReg) + { + // fmov Vd,Rn DV_2I X00111100X100111 000000nnnnnddddd 1E27 0000 Vd,Rn (scalar, from general) + assert(isVectorRegister(dstReg) && isGeneralRegister(addrReg)); + ins = INS_fmov; + fmt = IF_DV_2I; + code_t code = emitInsCode(ins, fmt); + + code |= insEncodeReg_Vd(dstReg); // ddddd + code |= insEncodeReg_Rn(addrReg); // nnnnn + if (id->idOpSize() == EA_8BYTE) + { + code |= 0x80400000; // X ... X + } + dst += emitOutput_Instr(dst, code); + } + } + + return dst; + } + + assert(loadLabel || isJump); + if (id->idAddr()->iiaHasInstrCount()) { assert(ig != NULL); @@ -7877,30 +8025,6 @@ BYTE* emitter::emitOutputLJ(insGroup *ig, BYTE *dst, instrDesc *i dstOffs = ig->igOffs + emitFindOffset(ig, (insNum + 1 + instrCount)); dstAddr = emitOffsetToPtr(dstOffs); } - else if (id->idAddr()->iiaIsJitDataOffset()) - { - assert(loadLabel); - - /* This is actually a reference to the JIT data section */ - - int doff = id->idAddr()->iiaGetJitDataOffset(); - assert(doff >= 0); - ssize_t imm = emitGetInsSC(id); - assert((imm >= 0) && (imm < 0x1000)); // 0x1000 is arbitrary, currently 'imm' is always 0 - - unsigned dataOffs = (unsigned) (doff + imm); - assert(dataOffs < emitDataSize()); - dstAddr = emitDataOffsetToPtr(dataOffs); - dstOffs = (unsigned) ((ssize_t) (dstAddr - srcAddr) + srcOffs); - assert((dstOffs & 3) == 0); - - // Failing the following assertion means the corresponding JIT data is not within +/-1MB range - // from the current code reference. This could happen for a large method or extremely large - // amount of JIT data for the method, or access it from cold method. - // Ideally, we should detect such case earlier to expand the code sequence using a fix-up - // similar to emitIns_R_AI. - assert(isValidSimm19(dstOffs)); - } else { dstOffs = id->idAddr()->iiaIGlabel->igOffs; @@ -7977,120 +8101,277 @@ BYTE* emitter::emitOutputLJ(insGroup *ig, BYTE *dst, instrDesc *i } #endif - loBits = (distVal & 3); - distVal >>= 2; // branch offset encodings are scaled by 4. - /* For forward jumps, record the address of the distance value */ id->idjTemp.idjAddr = (distVal > 0) ? dst : NULL; - assert(emitJumpCrossHotColdBoundary(srcOffs, dstOffs) == false); + if (emitJumpCrossHotColdBoundary(srcOffs, dstOffs)) + { + assert(!id->idjShort); + NYI_ARM64("Relocation Support for long address"); + } + assert(insOptsNone(id->idInsOpt())); if (isJump) { - assert(!id->idjKeepLong); - - // branch offsets must be a multiple of 4 - noway_assert(loBits == 0); - - if (fmt == IF_BI_0A) - { - // INS_b or INS_bl_local - noway_assert(isValidSimm26(distVal)); - distVal &= 0x3FFFFFFLL; - code |= distVal; - } - else if (fmt == IF_BI_0B) + if (id->idjShort) { - // INS_beq, INS_bne, etc... - noway_assert(isValidSimm19(distVal)); - distVal &= 0x7FFFFLL; - code |= distVal << 5; + // Short conditional/unconditional jump + assert(!id->idjKeepLong); + assert(emitJumpCrossHotColdBoundary(srcOffs, dstOffs) == false); + assert((fmt == IF_BI_0A) || (fmt == IF_BI_0B)); } - else if (fmt == IF_BI_1A) // BI_1A X.......iiiiiiii iiiiiiiiiiittttt Rt simm19:00 + else { - // INS_cbz or INS_cbnz - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt + // Long conditional jump + assert(fmt == IF_LARGEJMP); + // This is a pseudo-instruction format representing a large conditional branch, to allow + // us to get a greater branch target range than we can get by using a straightforward conditional + // branch. It is encoded as a short conditional branch that branches around a long unconditional + // branch. + // + // Conceptually, we have: + // + // b L_target + // + // The code we emit is: + // + // b L_not // 4 bytes. Note that we reverse the condition. + // b L_target // 4 bytes + // L_not: + // + // Note that we don't actually insert any blocks: we simply encode "b L_not" as a branch with + // the correct offset. Note also that this works for both integer and floating-point conditions, because + // the condition inversion takes ordered/unordered into account, preserving NaN behavior. For example, + // "GT" (greater than) is inverted to "LE" (less than, equal, or unordered). + dst = emitOutputShortBranch(dst, + emitJumpKindToIns(emitReverseJumpKind(emitInsToJumpKind(ins))), // reverse the conditional instruction + IF_BI_0B, + 8, /* 8 bytes from start of this large conditional pseudo-instruction to L_not. */ + nullptr /* only used for tbz/tbnzcbz/cbnz */); - noway_assert(isValidSimm19(distVal)); - distVal &= 0x7FFFFLL; // 19 bits - code |= distVal << 5; - } - else if (fmt == IF_BI_1B) // BI_1B B.......bbbbbiii iiiiiiiiiiittttt Rt imm6, simm14:00 - { - // INS_tbz or INS_tbnz - ssize_t imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); + // Now, pretend we've got a normal unconditional branch, and fall through to the code to emit that. + ins = INS_b; + fmt = IF_BI_0A; - if (imm & 0x20) // test bit 32-63 ? - { - code |= 0x80000000; // B - } - code |= ((imm & 0x1F) << 19); // bbbbb - code |= insEncodeReg_Rt(id->idReg1()); // ttttt + // The distVal was computed based on the beginning of the pseudo-instruction. + // So subtract the size of the conditional branch so that it is relative to the + // unconditional branch. + distVal -= 4; + } - noway_assert(isValidSimm14(distVal)); - distVal &= 0x3FFFLL; // 14 bits - code |= distVal << 5; + dst = emitOutputShortBranch(dst, ins, fmt, distVal, id); + } + else if (loadLabel) + { + regNumber dstReg = id->idReg1(); + if (id->idjShort) + { + // adr x, [rel addr] -- compute address: current addr(ip) + rel addr. + assert(ins == INS_adr); + assert(fmt == IF_DI_1E); + dst = emitOutputShortAddress(dst, ins, fmt, distVal, dstReg); } else { - assert(!"Unknown fmt"); + // adrp x, [rel page addr] -- compute page address: current page addr + rel page addr + assert(fmt == IF_LARGEADR); + ssize_t relPageAddr = (((ssize_t)dstAddr & 0xFFFFFFFFFFFFF000LL) - ((ssize_t)srcAddr & 0xFFFFFFFFFFFFF000LL)) >> 12; + dst = emitOutputShortAddress(dst, INS_adrp, IF_DI_1E, relPageAddr, dstReg); + + // add x, x, page offs -- compute address = page addr + page offs + ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits + assert(isValidUimm12(imm12)); + code_t code = emitInsCode(INS_add, IF_DI_2A); // DI_2A X0010001shiiiiii iiiiiinnnnnddddd 1100 0000 imm(i12, sh) + code |= insEncodeDatasize(EA_8BYTE); // X + code |= ((code_t)imm12 << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rd(dstReg); // ddddd + code |= insEncodeReg_Rn(dstReg); // nnnnn + dst += emitOutput_Instr(dst, code); } } - else if (loadLabel) + + return dst; +} + +/***************************************************************************** +* +* Output a short branch instruction. +*/ +BYTE* emitter::emitOutputShortBranch(BYTE *dst, instruction ins, insFormat fmt, ssize_t distVal, instrDescJmp* id) +{ + code_t code = emitInsCode(ins, fmt); + + ssize_t loBits = (distVal & 3); + noway_assert(loBits == 0); + distVal >>= 2; // branch offset encodings are scaled by 4. + + if (fmt == IF_BI_0A) + { + // INS_b or INS_bl_local + noway_assert(isValidSimm26(distVal)); + distVal &= 0x3FFFFFFLL; + code |= distVal; + } + else if (fmt == IF_BI_0B) // BI_0B 01010100iiiiiiii iiiiiiiiiiiXXXXX simm19:00 + { + // INS_beq, INS_bne, etc... + noway_assert(isValidSimm19(distVal)); + distVal &= 0x7FFFFLL; + code |= distVal << 5; + } + else if (fmt == IF_BI_1A) // BI_1A X.......iiiiiiii iiiiiiiiiiittttt Rt simm19:00 + { + // INS_cbz or INS_cbnz + assert(id != nullptr); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + + noway_assert(isValidSimm19(distVal)); + distVal &= 0x7FFFFLL; // 19 bits + code |= distVal << 5; + } + else if (fmt == IF_BI_1B) // BI_1B B.......bbbbbiii iiiiiiiiiiittttt Rt imm6, simm14:00 { - if (fmt == IF_LS_1A) // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt simm21 + // INS_tbz or INS_tbnz + assert(id != nullptr); + ssize_t imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); + + if (imm & 0x20) // test bit 32-63 ? { - // INS_ldr or INS_ldrsw (PC-Relative) + code |= 0x80000000; // B + } + code |= ((imm & 0x1F) << 19); // bbbbb + code |= insEncodeReg_Rt(id->idReg1()); // ttttt - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX V - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - } - else - { - assert(isGeneralRegister(id->idReg1())); - // insEncodeDatasizeLS is not quite right for this case. - // So just specialize it. - if ((ins == INS_ldr) && (id->idOpSize() == EA_8BYTE)) - { - // set the operation size in bit 30 - code |= 0x40000000; - } + noway_assert(isValidSimm14(distVal)); + distVal &= 0x3FFFLL; // 14 bits + code |= distVal << 5; + } + else + { + assert(!"Unknown fmt for emitOutputShortBranch"); + } - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - } + dst += emitOutput_Instr(dst, code); + + return dst; +} + +/***************************************************************************** +* +* Output a short address instruction. +*/ +BYTE* emitter::emitOutputShortAddress(BYTE *dst, instruction ins, insFormat fmt, ssize_t distVal, regNumber reg) +{ + ssize_t loBits = (distVal & 3); + distVal >>= 2; + + code_t code = emitInsCode(ins, fmt); + if (fmt == IF_DI_1E) // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 + { + // INS_adr or INS_adrp + code |= insEncodeReg_Rd(reg); // ddddd + + noway_assert(isValidSimm19(distVal)); + distVal &= 0x7FFFFLL; // 19 bits + code |= distVal << 5; + code |= loBits << 29; // 2 bits + } + else + { + assert(!"Unknown fmt for emitOutputShortAddress"); + } - noway_assert(loBits == 0); - noway_assert(isValidSimm19(distVal)); - distVal &= 0x7FFFFLL; // 19 bits - code |= distVal << 5; + dst += emitOutput_Instr(dst, code); + + return dst; +} + +/***************************************************************************** +* +* Output a short constant instruction. +*/ +BYTE* emitter::emitOutputShortConstant(BYTE *dst, instruction ins, insFormat fmt, ssize_t imm, regNumber reg, emitAttr opSize) +{ + code_t code = emitInsCode(ins, fmt); + + if (fmt == IF_LS_1A) + { + // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt simm21 + // INS_ldr or INS_ldrsw (PC-Relative) + + ssize_t loBits = (imm & 3); + noway_assert(loBits == 0); + ssize_t distVal = imm >>= 2; // load offset encodings are scaled by 4. + + noway_assert(isValidSimm19(distVal)); + + // Is the target a vector register? + if (isVectorRegister(reg)) + { + code |= insEncodeDatasizeVLS(code, opSize); // XX V + code |= insEncodeReg_Vt(reg); // ttttt } - else if (fmt == IF_DI_1E) // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 + else { - // INS_adr or INS_adrp - code |= insEncodeReg_Rd(id->idReg1()); // ddddd + assert(isGeneralRegister(reg)); + // insEncodeDatasizeLS is not quite right for this case. + // So just specialize it. + if ((ins == INS_ldr) && (opSize == EA_8BYTE)) + { + // set the operation size in bit 30 + code |= 0x40000000; + } + + code |= insEncodeReg_Rt(reg); // ttttt + } - noway_assert(isValidSimm19(distVal)); - distVal &= 0x7FFFFLL; // 19 bits - code |= distVal << 5; - code |= loBits << 29; // 2 bits + distVal &= 0x7FFFFLL; // 19 bits + code |= distVal << 5; + } + else if (fmt == IF_LS_2B) + { + // ldr Rt,[Xn+pimm12] LS_2B 1X11100101iiiiii iiiiiinnnnnttttt B940 0000 imm(0-4095<<{2,3}) + // INS_ldr or INS_ldrsw (PC-Relative) + noway_assert(isValidUimm12(imm)); + assert(isGeneralRegister(reg)); + + if (opSize == EA_8BYTE) + { + // insEncodeDatasizeLS is not quite right for this case. + // So just specialize it. + if (ins == INS_ldr) + { + // set the operation size in bit 30 + code |= 0x40000000; + } + // Low 3 bits should be 0 -- 8 byte JIT data should be aligned on 8 byte. + assert((imm & 7) == 0); + imm >>= 3; } else { - assert(!"Unknown fmt"); + assert(opSize == EA_4BYTE); + // Low 2 bits should be 0 -- 4 byte aligned data. + assert((imm & 3) == 0); + imm >>= 2; } + + code |= insEncodeReg_Rt(reg); // ttttt + code |= insEncodeReg_Rn(reg); // nnnnn + code |= imm << 10; + } + else + { + assert(!"Unknown fmt for emitOutputShortConstant"); } - dst += emitOutput_Instr(dst, code); + dst += emitOutput_Instr(dst, code); - return dst; + return dst; } - /***************************************************************************** * * Output a call instruction. @@ -8243,6 +8524,7 @@ size_t emitter::emitOutputInstr(insGroup *ig, case IF_BI_0A: // BI_0A ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 case IF_BI_0B: // BI_0B ......iiiiiiiiii iiiiiiiiiii..... simm19:00 + case IF_LARGEJMP: assert(id->idGCref() == GCT_NONE); assert(id->idIsBound()); dst = emitOutputLJ(ig, dst, id); @@ -8293,6 +8575,7 @@ size_t emitter::emitOutputInstr(insGroup *ig, break; case IF_LS_1A: // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt PC imm(1MB) + case IF_LARGELDC: assert(insOptsNone(id->idInsOpt())); assert(id->idIsBound()); @@ -8484,6 +8767,7 @@ size_t emitter::emitOutputInstr(insGroup *ig, break; case IF_DI_1E: // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 + case IF_LARGEADR: assert(insOptsNone(id->idInsOpt())); if (id->idIsReloc()) { @@ -9821,6 +10105,10 @@ void emitter::emitDispInsHex(BYTE * code, size_t sz) { printf(" %08X ", (*((code_t *) code))); } + else + { + printf(" "); + } } } @@ -9904,7 +10192,12 @@ void emitter::emitDispIns(instrDesc * id, case IF_BI_0A: // BI_0A ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 case IF_BI_0B: // BI_0B ......iiiiiiiiii iiiiiiiiiii..... simm19:00 + case IF_LARGEJMP: { + if (fmt == IF_LARGEJMP) + { + printf("(LARGEJMP)"); + } if (id->idAddr()->iiaHasInstrCount()) { int instrCount = id->idAddr()->iiaGetInstrCount(); @@ -9995,6 +10288,7 @@ void emitter::emitDispIns(instrDesc * id, break; case IF_LS_1A: // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt PC imm(1MB) + case IF_LARGELDC: assert(insOptsNone(id->idInsOpt())); emitDispReg(id->idReg1(), size, true); imm = emitGetInsSC(id); @@ -10002,6 +10296,10 @@ void emitter::emitDispIns(instrDesc * id, /* Is this actually a reference to a data section? */ doffs = Compiler::eeGetJitDataOffs(id->idAddr()->iiaFieldHnd); + if (fmt == IF_LARGELDC) + { + printf("(LARGELDC)"); + } printf("["); if (doffs >= 0) { @@ -10120,8 +10418,13 @@ void emitter::emitDispIns(instrDesc * id, break; case IF_DI_1E: // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 + case IF_LARGEADR: assert(insOptsNone(id->idInsOpt())); emitDispReg(id->idReg1(), size, true); + if (fmt == IF_LARGEADR) + { + printf("(LARGEADR)"); + } if (id->idIsBound()) { printf("G_M%03u_IG%02u", Compiler::s_compMethodsCount, id->idAddr()->iiaIGlabel->igNum); diff --git a/src/coreclr/src/jit/emitarm64.h b/src/coreclr/src/jit/emitarm64.h index f7549fa..a3f1845 100644 --- a/src/coreclr/src/jit/emitarm64.h +++ b/src/coreclr/src/jit/emitarm64.h @@ -775,6 +775,7 @@ public: void emitIns_R_C (instruction ins, emitAttr attr, regNumber reg, + regNumber tmpReg, CORINFO_FIELD_HANDLE fldHnd, int offs); @@ -899,6 +900,9 @@ public: BYTE* emitOutputLJ (insGroup *ig, BYTE *dst, instrDesc *i); unsigned emitOutputCall(insGroup *ig, BYTE *dst, instrDesc *i, code_t code); + BYTE* emitOutputShortBranch(BYTE *dst, instruction ins, insFormat fmt, ssize_t distVal, instrDescJmp* id); + BYTE* emitOutputShortAddress(BYTE *dst, instruction ins, insFormat fmt, ssize_t distVal, regNumber reg); + BYTE* emitOutputShortConstant(BYTE *dst, instruction ins, insFormat fmt, ssize_t distVal, regNumber reg, emitAttr opSize); /***************************************************************************** * @@ -907,7 +911,8 @@ public: inline bool emitIsCondJump(instrDesc *jmp) { - return (jmp->idInsFmt() == IF_BI_0B); + return ((jmp->idInsFmt() == IF_BI_0B) || + (jmp->idInsFmt() == IF_LARGEJMP)); } @@ -949,7 +954,19 @@ inline bool emitIsDirectCall(instrDesc *call) inline bool emitIsLoadLabel(instrDesc *jmp) { - return (jmp->idInsFmt() == IF_DI_1E); // adr or adrp + return ((jmp->idInsFmt() == IF_DI_1E) || // adr or arp + (jmp->idInsFmt() == IF_LARGEADR)); +} + +/***************************************************************************** +* +* Given a instrDesc, return true if it's a load constant instruction. +*/ + +inline bool emitIsLoadConstant(instrDesc *jmp) +{ + return ((jmp->idInsFmt() == IF_LS_1A) || // ldr + (jmp->idInsFmt() == IF_LARGELDC)); } #endif // _TARGET_ARM64_ diff --git a/src/coreclr/src/jit/emitfmtsarm64.h b/src/coreclr/src/jit/emitfmtsarm64.h index 06cde03..294bb38 100644 --- a/src/coreclr/src/jit/emitfmtsarm64.h +++ b/src/coreclr/src/jit/emitfmtsarm64.h @@ -40,7 +40,9 @@ enum ID_OPS IF_DEF(NONE, IS_NONE, NONE) // IF_DEF(LABEL, IS_NONE, JMP ) // label -IF_DEF(LARGEJMP, IS_NONE, JMP) // large conditional branch pseudo-op +IF_DEF(LARGEJMP, IS_NONE, JMP) // large conditional branch pseudo-op (cond branch + uncond branch) +IF_DEF(LARGEADR, IS_NONE, JMP) // large address pseudo-op (adrp + add) +IF_DEF(LARGELDC, IS_NONE, JMP) // large constant pseudo-op (adrp + ldr) ///////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/coreclr/src/jit/flowgraph.cpp b/src/coreclr/src/jit/flowgraph.cpp index 2cfed2e..24312f2 100644 --- a/src/coreclr/src/jit/flowgraph.cpp +++ b/src/coreclr/src/jit/flowgraph.cpp @@ -6666,6 +6666,19 @@ bool Compiler::fgInDifferentRegions(BasicBlock *blk1, BasicBlock *blk2) return ((blk1->bbFlags & BBF_COLD)!= (blk2->bbFlags & BBF_COLD)); } +bool Compiler::fgIsBlockCold(BasicBlock *blk) +{ + noway_assert(blk != NULL); + + if (fgFirstColdBlock == NULL) + { + return false; + } + + return ((blk->bbFlags & BBF_COLD) != 0); +} + + /***************************************************************************** * This function returns true if tree is a GT_COMMA node with a call * that unconditionally throws an exception diff --git a/src/coreclr/src/jit/jitconfigvalues.h b/src/coreclr/src/jit/jitconfigvalues.h index cb4724d..b82bab9 100644 --- a/src/coreclr/src/jit/jitconfigvalues.h +++ b/src/coreclr/src/jit/jitconfigvalues.h @@ -58,7 +58,7 @@ CONFIG_INTEGER(JitInlineAdditionalMultiplier, W("JitInlineAdditionalMultiplier") CONFIG_INTEGER(JitInlinePrintStats, W("JitInlinePrintStats"), 0) CONFIG_INTEGER(JitInlineSize, W("JITInlineSize"), DEFAULT_MAX_INLINE_SIZE) CONFIG_INTEGER(JitInlineDepth, W("JITInlineDepth"), DEFAULT_MAX_INLINE_DEPTH) -CONFIG_INTEGER(JitLargeBranches, W("JitLargeBranches"), 0) // Force using the largest conditional branch format +CONFIG_INTEGER(JitLongAddress, W("JitLongAddress"), 0) // Force using the large pseudo instruction form for long address CONFIG_INTEGER(JitMaxTempAssert, W("JITMaxTempAssert"), 1) CONFIG_INTEGER(JitMaxUncheckedOffset, W("JitMaxUncheckedOffset"), 8) CONFIG_INTEGER(JitMinOpts, W("JITMinOpts"), 0) // Forces MinOpts diff --git a/src/coreclr/src/jit/lowerarm64.cpp b/src/coreclr/src/jit/lowerarm64.cpp index f4411e6..e6b8d23 100644 --- a/src/coreclr/src/jit/lowerarm64.cpp +++ b/src/coreclr/src/jit/lowerarm64.cpp @@ -201,6 +201,20 @@ void Lowering::TreeNodeInfoInit(GenTree* stmt) case GT_CNS_DBL: info->srcCount = 0; info->dstCount = 1; + { + GenTreeDblCon *dblConst = tree->AsDblCon(); + double constValue = dblConst->gtDblCon.gtDconVal; + + if (emitter::emitIns_valid_imm_for_fmov(constValue)) + { + // Directly encode constant to instructions. + } + else + { + // Reserve int to load constant from memory (IF_LARGELDC) + info->internalIntCount = 1; + } + } break; case GT_QMARK: diff --git a/src/coreclr/src/jit/target.h b/src/coreclr/src/jit/target.h index 2f43f2c..4810dfe 100644 --- a/src/coreclr/src/jit/target.h +++ b/src/coreclr/src/jit/target.h @@ -1713,13 +1713,21 @@ typedef unsigned short regPairNoSmall; // arm: need 12 bits #define LBL_DIST_SMALL_MAX_POS (+1048575) #define LBL_SIZE_SMALL (4) - #define LBL_SIZE_LARGE (8) // NYI + #define LBL_SIZE_LARGE (8) #define JCC_DIST_SMALL_MAX_NEG (-1048576) - #define JCC_DIST_SMALL_MAX_POS (+1048572) + #define JCC_DIST_SMALL_MAX_POS (+1048575) #define JCC_SIZE_SMALL (4) - #define JCC_SIZE_LARGE (8) // NYI + #define JCC_SIZE_LARGE (8) + + #define LDC_DIST_SMALL_MAX_NEG (-1048576) + #define LDC_DIST_SMALL_MAX_POS (+1048575) + + #define LDC_SIZE_SMALL (4) + #define LDC_SIZE_LARGE (8) + + #define JMP_SIZE_SMALL (4) #else #error Unsupported or unset target architecture diff --git a/src/coreclr/tests/arm64/Tests.lst b/src/coreclr/tests/arm64/Tests.lst index d189310..d6bdd4a 100644 --- a/src/coreclr/tests/arm64/Tests.lst +++ b/src/coreclr/tests/arm64/Tests.lst @@ -39127,7 +39127,7 @@ RelativePath=JIT\Regression\VS-ia64-JIT\V1.2-M02\b28158\b28158\b28158.cmd WorkingDir=JIT\Regression\VS-ia64-JIT\V1.2-M02\b28158\b28158 Expected=0 MaxAllowedDurationSeconds=600 -Categories=Pri0;JIT;EXPECTED_FAIL;ISSUE_3668;LONG_RUNNING +Categories=Pri0;JIT;EXPECTED_PASS HostStyle=0 [b28158_64.cmd_5696] RelativePath=JIT\Regression\VS-ia64-JIT\V1.2-M02\b28158\b28158_64\b28158_64.cmd -- 2.7.4