From: Jakob Botsch Nielsen Date: Sun, 9 Oct 2022 18:27:46 +0000 (+0200) Subject: JIT: Move impInitializeArrayIntrinsic, impCreateSpanIntrinsic, impIntrinsic, impSRCSU... X-Git-Tag: accepted/tizen/unified/riscv/20231226.055536~5986 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=a29bbc8066661ec60642b0e0607ed5a9b99552f6;p=platform%2Fupstream%2Fdotnet%2Fruntime.git JIT: Move impInitializeArrayIntrinsic, impCreateSpanIntrinsic, impIntrinsic, impSRCSUnsafeIntrinsic --- diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 39657dd..d826843 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3072,2286 +3072,6 @@ GenTree* Compiler::impImplicitR4orR8Cast(GenTree* tree, var_types dstTyp) return tree; } -//------------------------------------------------------------------------ -// impInitializeArrayIntrinsic: Attempts to replace a call to InitializeArray -// with a GT_COPYBLK node. -// -// Arguments: -// sig - The InitializeArray signature. -// -// Return Value: -// A pointer to the newly created GT_COPYBLK node if the replacement succeeds or -// nullptr otherwise. -// -// Notes: -// The function recognizes the following IL pattern: -// ldc or a list of ldc / -// newarr or newobj -// dup -// ldtoken -// call InitializeArray -// The lower bounds need not be constant except when the array rank is 1. -// The function recognizes all kinds of arrays thus enabling a small runtime -// such as NativeAOT to skip providing an implementation for InitializeArray. - -GenTree* Compiler::impInitializeArrayIntrinsic(CORINFO_SIG_INFO* sig) -{ - assert(sig->numArgs == 2); - - GenTree* fieldTokenNode = impStackTop(0).val; - GenTree* arrayLocalNode = impStackTop(1).val; - - // - // Verify that the field token is known and valid. Note that it's also - // possible for the token to come from reflection, in which case we cannot do - // the optimization and must therefore revert to calling the helper. You can - // see an example of this in bvt\DynIL\initarray2.exe (in Main). - // - - // Check to see if the ldtoken helper call is what we see here. - if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) || - (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD))) - { - return nullptr; - } - - // Strip helper call away - fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode(); - - if (fieldTokenNode->gtOper == GT_IND) - { - fieldTokenNode = fieldTokenNode->AsOp()->gtOp1; - } - - // Check for constant - if (fieldTokenNode->gtOper != GT_CNS_INT) - { - return nullptr; - } - - CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle; - if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr)) - { - return nullptr; - } - - // - // We need to get the number of elements in the array and the size of each element. - // We verify that the newarr statement is exactly what we expect it to be. - // If it's not then we just return NULL and we don't optimize this call - // - - // It is possible the we don't have any statements in the block yet. - if (impLastStmt == nullptr) - { - return nullptr; - } - - // - // We start by looking at the last statement, making sure it's an assignment, and - // that the target of the assignment is the array passed to InitializeArray. - // - GenTree* arrayAssignment = impLastStmt->GetRootNode(); - if ((arrayAssignment->gtOper != GT_ASG) || (arrayAssignment->AsOp()->gtOp1->gtOper != GT_LCL_VAR) || - (arrayLocalNode->gtOper != GT_LCL_VAR) || (arrayAssignment->AsOp()->gtOp1->AsLclVarCommon()->GetLclNum() != - arrayLocalNode->AsLclVarCommon()->GetLclNum())) - { - return nullptr; - } - - // - // Make sure that the object being assigned is a helper call. - // - - GenTree* newArrayCall = arrayAssignment->AsOp()->gtOp2; - if ((newArrayCall->gtOper != GT_CALL) || (newArrayCall->AsCall()->gtCallType != CT_HELPER)) - { - return nullptr; - } - - // - // Verify that it is one of the new array helpers. - // - - bool isMDArray = false; - - if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_DIRECT) && - newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_OBJ) && - newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_VC) && - newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_ALIGN8) -#ifdef FEATURE_READYTORUN - && newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1) -#endif - ) - { - if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEW_MDARR)) - { - return nullptr; - } - - isMDArray = true; - } - - CORINFO_CLASS_HANDLE arrayClsHnd = (CORINFO_CLASS_HANDLE)newArrayCall->AsCall()->compileTimeHelperArgumentHandle; - - // - // Make sure we found a compile time handle to the array - // - - if (!arrayClsHnd) - { - return nullptr; - } - - unsigned rank = 0; - S_UINT32 numElements; - - if (isMDArray) - { - rank = info.compCompHnd->getArrayRank(arrayClsHnd); - - if (rank == 0) - { - return nullptr; - } - - assert(newArrayCall->AsCall()->gtArgs.CountArgs() == 3); - GenTree* numArgsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode(); - GenTree* argsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(2)->GetNode(); - - // - // The number of arguments should be a constant between 1 and 64. The rank can't be 0 - // so at least one length must be present and the rank can't exceed 32 so there can - // be at most 64 arguments - 32 lengths and 32 lower bounds. - // - - if ((!numArgsArg->IsCnsIntOrI()) || (numArgsArg->AsIntCon()->IconValue() < 1) || - (numArgsArg->AsIntCon()->IconValue() > 64)) - { - return nullptr; - } - - unsigned numArgs = static_cast(numArgsArg->AsIntCon()->IconValue()); - bool lowerBoundsSpecified; - - if (numArgs == rank * 2) - { - lowerBoundsSpecified = true; - } - else if (numArgs == rank) - { - lowerBoundsSpecified = false; - - // - // If the rank is 1 and a lower bound isn't specified then the runtime creates - // a SDArray. Note that even if a lower bound is specified it can be 0 and then - // we get a SDArray as well, see the for loop below. - // - - if (rank == 1) - { - isMDArray = false; - } - } - else - { - return nullptr; - } - - // - // The rank is known to be at least 1 so we can start with numElements being 1 - // to avoid the need to special case the first dimension. - // - - numElements = S_UINT32(1); - - struct Match - { - static bool IsArgsFieldInit(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs) - { - return (tree->OperGet() == GT_ASG) && IsArgsFieldIndir(tree->gtGetOp1(), index, lvaNewObjArrayArgs) && - IsArgsAddr(tree->gtGetOp1()->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs); - } - - static bool IsArgsFieldIndir(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs) - { - return (tree->OperGet() == GT_IND) && (tree->gtGetOp1()->OperGet() == GT_ADD) && - (tree->gtGetOp1()->gtGetOp2()->IsIntegralConst(sizeof(INT32) * index)) && - IsArgsAddr(tree->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs); - } - - static bool IsArgsAddr(GenTree* tree, unsigned lvaNewObjArrayArgs) - { - return (tree->OperGet() == GT_ADDR) && (tree->gtGetOp1()->OperGet() == GT_LCL_VAR) && - (tree->gtGetOp1()->AsLclVar()->GetLclNum() == lvaNewObjArrayArgs); - } - - static bool IsComma(GenTree* tree) - { - return (tree != nullptr) && (tree->OperGet() == GT_COMMA); - } - }; - - unsigned argIndex = 0; - GenTree* comma; - - for (comma = argsArg; Match::IsComma(comma); comma = comma->gtGetOp2()) - { - if (lowerBoundsSpecified) - { - // - // In general lower bounds can be ignored because they're not needed to - // calculate the total number of elements. But for single dimensional arrays - // we need to know if the lower bound is 0 because in this case the runtime - // creates a SDArray and this affects the way the array data offset is calculated. - // - - if (rank == 1) - { - GenTree* lowerBoundAssign = comma->gtGetOp1(); - assert(Match::IsArgsFieldInit(lowerBoundAssign, argIndex, lvaNewObjArrayArgs)); - GenTree* lowerBoundNode = lowerBoundAssign->gtGetOp2(); - - if (lowerBoundNode->IsIntegralConst(0)) - { - isMDArray = false; - } - } - - comma = comma->gtGetOp2(); - argIndex++; - } - - GenTree* lengthNodeAssign = comma->gtGetOp1(); - assert(Match::IsArgsFieldInit(lengthNodeAssign, argIndex, lvaNewObjArrayArgs)); - GenTree* lengthNode = lengthNodeAssign->gtGetOp2(); - - if (!lengthNode->IsCnsIntOrI()) - { - return nullptr; - } - - numElements *= S_SIZE_T(lengthNode->AsIntCon()->IconValue()); - argIndex++; - } - - assert((comma != nullptr) && Match::IsArgsAddr(comma, lvaNewObjArrayArgs)); - - if (argIndex != numArgs) - { - return nullptr; - } - } - else - { - // - // Make sure there are exactly two arguments: the array class and - // the number of elements. - // - - GenTree* arrayLengthNode; - -#ifdef FEATURE_READYTORUN - if (newArrayCall->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1)) - { - // Array length is 1st argument for readytorun helper - arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); - } - else -#endif - { - // Array length is 2nd argument for regular helper - arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode(); - } - - // - // This optimization is only valid for a constant array size. - // - if (arrayLengthNode->gtOper != GT_CNS_INT) - { - return nullptr; - } - - numElements = S_SIZE_T(arrayLengthNode->AsIntCon()->gtIconVal); - - if (!info.compCompHnd->isSDArray(arrayClsHnd)) - { - return nullptr; - } - } - - CORINFO_CLASS_HANDLE elemClsHnd; - var_types elementType = JITtype2varType(info.compCompHnd->getChildType(arrayClsHnd, &elemClsHnd)); - - // - // Note that genTypeSize will return zero for non primitive types, which is exactly - // what we want (size will then be 0, and we will catch this in the conditional below). - // Note that we don't expect this to fail for valid binaries, so we assert in the - // non-verification case (the verification case should not assert but rather correctly - // handle bad binaries). This assert is not guarding any specific invariant, but rather - // saying that we don't expect this to happen, and if it is hit, we need to investigate - // why. - // - - S_UINT32 elemSize(genTypeSize(elementType)); - S_UINT32 size = elemSize * S_UINT32(numElements); - - if (size.IsOverflow()) - { - return nullptr; - } - - if ((size.Value() == 0) || (varTypeIsGC(elementType))) - { - return nullptr; - } - - void* initData = info.compCompHnd->getArrayInitializationData(fieldToken, size.Value()); - if (!initData) - { - return nullptr; - } - - // - // At this point we are ready to commit to implementing the InitializeArray - // intrinsic using a struct assignment. Pop the arguments from the stack and - // return the struct assignment node. - // - - impPopStack(); - impPopStack(); - - const unsigned blkSize = size.Value(); - unsigned dataOffset; - - if (isMDArray) - { - dataOffset = eeGetMDArrayDataOffset(rank); - } - else - { - dataOffset = eeGetArrayDataOffset(); - } - - GenTree* dstAddr = gtNewOperNode(GT_ADD, TYP_BYREF, arrayLocalNode, gtNewIconNode(dataOffset, TYP_I_IMPL)); - GenTree* dst = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, dstAddr, typGetBlkLayout(blkSize)); - GenTree* src = gtNewIndOfIconHandleNode(TYP_STRUCT, (size_t)initData, GTF_ICON_CONST_PTR, true); - -#ifdef DEBUG - src->gtGetOp1()->AsIntCon()->gtTargetHandle = THT_InitializeArrayIntrinsics; -#endif - - return gtNewBlkOpNode(dst, // dst - src, // src - false, // volatile - true); // copyBlock -} - -GenTree* Compiler::impCreateSpanIntrinsic(CORINFO_SIG_INFO* sig) -{ - assert(sig->numArgs == 1); - assert(sig->sigInst.methInstCount == 1); - - GenTree* fieldTokenNode = impStackTop(0).val; - - // - // Verify that the field token is known and valid. Note that it's also - // possible for the token to come from reflection, in which case we cannot do - // the optimization and must therefore revert to calling the helper. You can - // see an example of this in bvt\DynIL\initarray2.exe (in Main). - // - - // Check to see if the ldtoken helper call is what we see here. - if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) || - (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD))) - { - return nullptr; - } - - // Strip helper call away - fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); - if (fieldTokenNode->gtOper == GT_IND) - { - fieldTokenNode = fieldTokenNode->AsOp()->gtOp1; - } - - // Check for constant - if (fieldTokenNode->gtOper != GT_CNS_INT) - { - return nullptr; - } - - CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle; - if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr)) - { - return nullptr; - } - - CORINFO_CLASS_HANDLE fieldOwnerHnd = info.compCompHnd->getFieldClass(fieldToken); - - CORINFO_CLASS_HANDLE fieldClsHnd; - var_types fieldElementType = - JITtype2varType(info.compCompHnd->getFieldType(fieldToken, &fieldClsHnd, fieldOwnerHnd)); - unsigned totalFieldSize; - - // Most static initialization data fields are of some structure, but it is possible for them to be of various - // primitive types as well - if (fieldElementType == var_types::TYP_STRUCT) - { - totalFieldSize = info.compCompHnd->getClassSize(fieldClsHnd); - } - else - { - totalFieldSize = genTypeSize(fieldElementType); - } - - // Limit to primitive or enum type - see ArrayNative::GetSpanDataFrom() - CORINFO_CLASS_HANDLE targetElemHnd = sig->sigInst.methInst[0]; - if (info.compCompHnd->getTypeForPrimitiveValueClass(targetElemHnd) == CORINFO_TYPE_UNDEF) - { - return nullptr; - } - - const unsigned targetElemSize = info.compCompHnd->getClassSize(targetElemHnd); - assert(targetElemSize != 0); - - const unsigned count = totalFieldSize / targetElemSize; - if (count == 0) - { - return nullptr; - } - - void* data = info.compCompHnd->getArrayInitializationData(fieldToken, totalFieldSize); - if (!data) - { - return nullptr; - } - - // - // Ready to commit to the work - // - - impPopStack(); - - // Turn count and pointer value into constants. - GenTree* lengthValue = gtNewIconNode(count, TYP_INT); - GenTree* pointerValue = gtNewIconHandleNode((size_t)data, GTF_ICON_CONST_PTR); - - // Construct ReadOnlySpan to return. - CORINFO_CLASS_HANDLE spanHnd = sig->retTypeClass; - unsigned spanTempNum = lvaGrabTemp(true DEBUGARG("ReadOnlySpan for CreateSpan")); - lvaSetStruct(spanTempNum, spanHnd, false); - - GenTreeLclFld* pointerField = gtNewLclFldNode(spanTempNum, TYP_BYREF, 0); - GenTree* pointerFieldAsg = gtNewAssignNode(pointerField, pointerValue); - - GenTreeLclFld* lengthField = gtNewLclFldNode(spanTempNum, TYP_INT, TARGET_POINTER_SIZE); - GenTree* lengthFieldAsg = gtNewAssignNode(lengthField, lengthValue); - - // Now append a few statements the initialize the span - impAppendTree(lengthFieldAsg, CHECK_SPILL_NONE, impCurStmtDI); - impAppendTree(pointerFieldAsg, CHECK_SPILL_NONE, impCurStmtDI); - - // And finally create a tree that points at the span. - return impCreateLocalNode(spanTempNum DEBUGARG(0)); -} - -//------------------------------------------------------------------------ -// impIntrinsic: possibly expand intrinsic call into alternate IR sequence -// -// Arguments: -// newobjThis - for constructor calls, the tree for the newly allocated object -// clsHnd - handle for the intrinsic method's class -// method - handle for the intrinsic method -// sig - signature of the intrinsic method -// methodFlags - CORINFO_FLG_XXX flags of the intrinsic method -// memberRef - the token for the intrinsic method -// readonlyCall - true if call has a readonly prefix -// tailCall - true if call is in tail position -// pConstrainedResolvedToken -- resolved token for constrained call, or nullptr -// if call is not constrained -// constraintCallThisTransform -- this transform to apply for a constrained call -// pIntrinsicName [OUT] -- intrinsic name (see enumeration in namedintrinsiclist.h) -// for "traditional" jit intrinsics -// isSpecialIntrinsic [OUT] -- set true if intrinsic expansion is a call -// that is amenable to special downstream optimization opportunities -// -// Returns: -// IR tree to use in place of the call, or nullptr if the jit should treat -// the intrinsic call like a normal call. -// -// pIntrinsicName set to non-illegal value if the call is recognized as a -// traditional jit intrinsic, even if the intrinsic is not expaned. -// -// isSpecial set true if the expansion is subject to special -// optimizations later in the jit processing -// -// Notes: -// On success the IR tree may be a call to a different method or an inline -// sequence. If it is a call, then the intrinsic processing here is responsible -// for handling all the special cases, as upon return to impImportCall -// expanded intrinsics bypass most of the normal call processing. -// -// Intrinsics are generally not recognized in minopts and debug codegen. -// -// However, certain traditional intrinsics are identifed as "must expand" -// if there is no fallback implementation to invoke; these must be handled -// in all codegen modes. -// -// New style intrinsics (where the fallback implementation is in IL) are -// identified as "must expand" if they are invoked from within their -// own method bodies. -// -GenTree* Compiler::impIntrinsic(GenTree* newobjThis, - CORINFO_CLASS_HANDLE clsHnd, - CORINFO_METHOD_HANDLE method, - CORINFO_SIG_INFO* sig, - unsigned methodFlags, - int memberRef, - bool readonlyCall, - bool tailCall, - CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, - CORINFO_THIS_TRANSFORM constraintCallThisTransform, - NamedIntrinsic* pIntrinsicName, - bool* isSpecialIntrinsic) -{ - bool mustExpand = false; - bool isSpecial = false; - const bool isIntrinsic = (methodFlags & CORINFO_FLG_INTRINSIC) != 0; - - NamedIntrinsic ni = lookupNamedIntrinsic(method); - - if (isIntrinsic) - { - // The recursive non-virtual calls to Jit intrinsics are must-expand by convention. - mustExpand = gtIsRecursiveCall(method) && !(methodFlags & CORINFO_FLG_VIRTUAL); - } - else - { - // For mismatched VM (AltJit) we want to check all methods as intrinsic to ensure - // we get more accurate codegen. This particularly applies to HWIntrinsic usage - assert(!info.compMatchedVM); - } - - // We specially support the following on all platforms to allow for dead - // code optimization and to more generally support recursive intrinsics. - - if (isIntrinsic) - { - if (ni == NI_IsSupported_True) - { - assert(sig->numArgs == 0); - return gtNewIconNode(true); - } - - if (ni == NI_IsSupported_False) - { - assert(sig->numArgs == 0); - return gtNewIconNode(false); - } - - if (ni == NI_Throw_PlatformNotSupportedException) - { - return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); - } - - if ((ni > NI_SRCS_UNSAFE_START) && (ni < NI_SRCS_UNSAFE_END)) - { - assert(!mustExpand); - return impSRCSUnsafeIntrinsic(ni, clsHnd, method, sig); - } - } - -#ifdef FEATURE_HW_INTRINSICS - if ((ni > NI_HW_INTRINSIC_START) && (ni < NI_HW_INTRINSIC_END)) - { - if (!isIntrinsic) - { -#if defined(TARGET_XARCH) - // We can't guarantee that all overloads for the xplat intrinsics can be - // handled by the AltJit, so limit only the platform specific intrinsics - assert((NI_Vector256_Xor + 1) == NI_X86Base_BitScanForward); - - if (ni < NI_Vector256_Xor) -#elif defined(TARGET_ARM64) - // We can't guarantee that all overloads for the xplat intrinsics can be - // handled by the AltJit, so limit only the platform specific intrinsics - assert((NI_Vector128_Xor + 1) == NI_AdvSimd_Abs); - - if (ni < NI_Vector128_Xor) -#else -#error Unsupported platform -#endif - { - // Several of the NI_Vector64/128/256 APIs do not have - // all overloads as intrinsic today so they will assert - return nullptr; - } - } - - GenTree* hwintrinsic = impHWIntrinsic(ni, clsHnd, method, sig, mustExpand); - - if (mustExpand && (hwintrinsic == nullptr)) - { - return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_NOT_IMPLEMENTED, method, sig, mustExpand); - } - - return hwintrinsic; - } - - if (isIntrinsic && (ni > NI_SIMD_AS_HWINTRINSIC_START) && (ni < NI_SIMD_AS_HWINTRINSIC_END)) - { - // These intrinsics aren't defined recursively and so they will never be mustExpand - // Instead, they provide software fallbacks that will be executed instead. - - assert(!mustExpand); - return impSimdAsHWIntrinsic(ni, clsHnd, method, sig, newobjThis); - } -#endif // FEATURE_HW_INTRINSICS - - if (!isIntrinsic) - { - // Outside the cases above, there are many intrinsics which apply to only a - // subset of overload and where simply matching by name may cause downstream - // asserts or other failures. Math.Min is one example, where it only applies - // to the floating-point overloads. - return nullptr; - } - - *pIntrinsicName = ni; - - if (ni == NI_System_StubHelpers_GetStubContext) - { - // must be done regardless of DbgCode and MinOpts - return gtNewLclvNode(lvaStubArgumentVar, TYP_I_IMPL); - } - - if (ni == NI_System_StubHelpers_NextCallReturnAddress) - { - // For now we just avoid inlining anything into these methods since - // this intrinsic is only rarely used. We could do this better if we - // wanted to by trying to match which call is the one we need to get - // the return address of. - info.compHasNextCallRetAddr = true; - return new (this, GT_LABEL) GenTree(GT_LABEL, TYP_I_IMPL); - } - - switch (ni) - { - // CreateSpan must be expanded for NativeAOT - case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan: - case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray: - mustExpand |= IsTargetAbi(CORINFO_NATIVEAOT_ABI); - break; - - case NI_Internal_Runtime_MethodTable_Of: - case NI_System_Activator_AllocatorOf: - case NI_System_Activator_DefaultConstructorOf: - case NI_System_EETypePtr_EETypePtrOf: - mustExpand = true; - break; - - default: - break; - } - - GenTree* retNode = nullptr; - - // Under debug and minopts, only expand what is required. - // NextCallReturnAddress intrinsic returns the return address of the next call. - // If that call is an intrinsic and is expanded, codegen for NextCallReturnAddress will fail. - // To avoid that we conservatively expand only required intrinsics in methods that call - // the NextCallReturnAddress intrinsic. - if (!mustExpand && (opts.OptimizationDisabled() || info.compHasNextCallRetAddr)) - { - *pIntrinsicName = NI_Illegal; - return retNode; - } - - CorInfoType callJitType = sig->retType; - var_types callType = JITtype2varType(callJitType); - - /* First do the intrinsics which are always smaller than a call */ - - if (ni != NI_Illegal) - { - assert(retNode == nullptr); - switch (ni) - { - case NI_Array_Address: - case NI_Array_Get: - case NI_Array_Set: - retNode = impArrayAccessIntrinsic(clsHnd, sig, memberRef, readonlyCall, ni); - break; - - case NI_System_String_Equals: - { - retNode = impStringEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags); - break; - } - - case NI_System_MemoryExtensions_Equals: - case NI_System_MemoryExtensions_SequenceEqual: - { - retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags); - break; - } - - case NI_System_String_StartsWith: - { - retNode = impStringEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags); - break; - } - - case NI_System_MemoryExtensions_StartsWith: - { - retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags); - break; - } - - case NI_System_MemoryExtensions_AsSpan: - case NI_System_String_op_Implicit: - { - assert(sig->numArgs == 1); - isSpecial = impStackTop().val->OperIs(GT_CNS_STR); - break; - } - - case NI_System_String_get_Chars: - { - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - GenTree* addr = gtNewIndexAddr(op1, op2, TYP_USHORT, NO_CLASS_HANDLE, OFFSETOF__CORINFO_String__chars, - OFFSETOF__CORINFO_String__stringLen); - retNode = gtNewIndexIndir(addr->AsIndexAddr()); - break; - } - - case NI_System_String_get_Length: - { - GenTree* op1 = impPopStack().val; - if (op1->OperIs(GT_CNS_STR)) - { - // Optimize `ldstr + String::get_Length()` to CNS_INT - // e.g. "Hello".Length => 5 - GenTreeIntCon* iconNode = gtNewStringLiteralLength(op1->AsStrCon()); - if (iconNode != nullptr) - { - retNode = iconNode; - break; - } - } - GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, op1, OFFSETOF__CORINFO_String__stringLen, compCurBB); - op1 = arrLen; - - // Getting the length of a null string should throw - op1->gtFlags |= GTF_EXCEPT; - - retNode = op1; - break; - } - - case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan: - { - retNode = impCreateSpanIntrinsic(sig); - break; - } - - case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray: - { - retNode = impInitializeArrayIntrinsic(sig); - break; - } - - case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant: - { - GenTree* op1 = impPopStack().val; - if (op1->OperIsConst()) - { - // op1 is a known constant, replace with 'true'. - retNode = gtNewIconNode(1); - JITDUMP("\nExpanding RuntimeHelpers.IsKnownConstant to true early\n"); - // We can also consider FTN_ADDR and typeof(T) here - } - else - { - // op1 is not a known constant, we'll do the expansion in morph - retNode = new (this, GT_INTRINSIC) GenTreeIntrinsic(TYP_INT, op1, ni, method); - JITDUMP("\nConverting RuntimeHelpers.IsKnownConstant to:\n"); - DISPTREE(retNode); - } - break; - } - - case NI_Internal_Runtime_MethodTable_Of: - case NI_System_Activator_AllocatorOf: - case NI_System_Activator_DefaultConstructorOf: - case NI_System_EETypePtr_EETypePtrOf: - { - assert(IsTargetAbi(CORINFO_NATIVEAOT_ABI)); // Only NativeAOT supports it. - CORINFO_RESOLVED_TOKEN resolvedToken; - resolvedToken.tokenContext = impTokenLookupContextHandle; - resolvedToken.tokenScope = info.compScopeHnd; - resolvedToken.token = memberRef; - resolvedToken.tokenType = CORINFO_TOKENKIND_Method; - - CORINFO_GENERICHANDLE_RESULT embedInfo; - info.compCompHnd->expandRawHandleIntrinsic(&resolvedToken, &embedInfo); - - GenTree* rawHandle = impLookupToTree(&resolvedToken, &embedInfo.lookup, gtTokenToIconFlags(memberRef), - embedInfo.compileTimeHandle); - if (rawHandle == nullptr) - { - return nullptr; - } - - noway_assert(genTypeSize(rawHandle->TypeGet()) == genTypeSize(TYP_I_IMPL)); - - unsigned rawHandleSlot = lvaGrabTemp(true DEBUGARG("rawHandle")); - impAssignTempGen(rawHandleSlot, rawHandle, clsHnd, CHECK_SPILL_NONE); - - GenTree* lclVar = gtNewLclvNode(rawHandleSlot, TYP_I_IMPL); - GenTree* lclVarAddr = gtNewOperNode(GT_ADDR, TYP_I_IMPL, lclVar); - var_types resultType = JITtype2varType(sig->retType); - if (resultType == TYP_STRUCT) - { - retNode = gtNewObjNode(sig->retTypeClass, lclVarAddr); - } - else - { - retNode = gtNewIndir(resultType, lclVarAddr); - } - break; - } - - case NI_System_Span_get_Item: - case NI_System_ReadOnlySpan_get_Item: - { - // Have index, stack pointer-to Span s on the stack. Expand to: - // - // For Span - // Comma - // BoundsCheck(index, s->_length) - // s->_reference + index * sizeof(T) - // - // For ReadOnlySpan -- same expansion, as it now returns a readonly ref - // - // Signature should show one class type parameter, which - // we need to examine. - assert(sig->sigInst.classInstCount == 1); - assert(sig->numArgs == 1); - CORINFO_CLASS_HANDLE spanElemHnd = sig->sigInst.classInst[0]; - const unsigned elemSize = info.compCompHnd->getClassSize(spanElemHnd); - assert(elemSize > 0); - - const bool isReadOnly = (ni == NI_System_ReadOnlySpan_get_Item); - - JITDUMP("\nimpIntrinsic: Expanding %sSpan.get_Item, T=%s, sizeof(T)=%u\n", - isReadOnly ? "ReadOnly" : "", eeGetClassName(spanElemHnd), elemSize); - - GenTree* index = impPopStack().val; - GenTree* ptrToSpan = impPopStack().val; - GenTree* indexClone = nullptr; - GenTree* ptrToSpanClone = nullptr; - assert(genActualType(index) == TYP_INT); - assert(ptrToSpan->TypeGet() == TYP_BYREF); - -#if defined(DEBUG) - if (verbose) - { - printf("with ptr-to-span\n"); - gtDispTree(ptrToSpan); - printf("and index\n"); - gtDispTree(index); - } -#endif // defined(DEBUG) - - // We need to use both index and ptr-to-span twice, so clone or spill. - index = impCloneExpr(index, &indexClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL, - nullptr DEBUGARG("Span.get_Item index")); - ptrToSpan = impCloneExpr(ptrToSpan, &ptrToSpanClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL, - nullptr DEBUGARG("Span.get_Item ptrToSpan")); - - // Bounds check - CORINFO_FIELD_HANDLE lengthHnd = info.compCompHnd->getFieldInClass(clsHnd, 1); - const unsigned lengthOffset = info.compCompHnd->getFieldOffset(lengthHnd); - GenTree* length = gtNewFieldRef(TYP_INT, lengthHnd, ptrToSpan, lengthOffset); - GenTree* boundsCheck = new (this, GT_BOUNDS_CHECK) GenTreeBoundsChk(index, length, SCK_RNGCHK_FAIL); - - // Element access - index = indexClone; - -#ifdef TARGET_64BIT - if (index->OperGet() == GT_CNS_INT) - { - index->gtType = TYP_I_IMPL; - } - else - { - index = gtNewCastNode(TYP_I_IMPL, index, true, TYP_I_IMPL); - } -#endif - - if (elemSize != 1) - { - GenTree* sizeofNode = gtNewIconNode(static_cast(elemSize), TYP_I_IMPL); - index = gtNewOperNode(GT_MUL, TYP_I_IMPL, index, sizeofNode); - } - - CORINFO_FIELD_HANDLE ptrHnd = info.compCompHnd->getFieldInClass(clsHnd, 0); - const unsigned ptrOffset = info.compCompHnd->getFieldOffset(ptrHnd); - GenTree* data = gtNewFieldRef(TYP_BYREF, ptrHnd, ptrToSpanClone, ptrOffset); - GenTree* result = gtNewOperNode(GT_ADD, TYP_BYREF, data, index); - - // Prepare result - var_types resultType = JITtype2varType(sig->retType); - assert(resultType == result->TypeGet()); - retNode = gtNewOperNode(GT_COMMA, resultType, boundsCheck, result); - - break; - } - - case NI_System_RuntimeTypeHandle_GetValueInternal: - { - GenTree* op1 = impStackTop(0).val; - if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) && - gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall())) - { - // Old tree - // Helper-RuntimeTypeHandle -> TreeToGetNativeTypeHandle - // - // New tree - // TreeToGetNativeTypeHandle - - // Remove call to helper and return the native TypeHandle pointer that was the parameter - // to that helper. - - op1 = impPopStack().val; - - // Get native TypeHandle argument to old helper - assert(op1->AsCall()->gtArgs.CountArgs() == 1); - op1 = op1->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); - retNode = op1; - } - // Call the regular function. - break; - } - - case NI_System_Type_GetTypeFromHandle: - { - GenTree* op1 = impStackTop(0).val; - CorInfoHelpFunc typeHandleHelper; - if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) && - gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall(), &typeHandleHelper)) - { - op1 = impPopStack().val; - // Replace helper with a more specialized helper that returns RuntimeType - if (typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE) - { - typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE; - } - else - { - assert(typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL); - typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL; - } - assert(op1->AsCall()->gtArgs.CountArgs() == 1); - op1 = gtNewHelperCallNode(typeHandleHelper, TYP_REF, - op1->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()); - op1->gtType = TYP_REF; - retNode = op1; - } - break; - } - - case NI_System_Type_op_Equality: - case NI_System_Type_op_Inequality: - { - JITDUMP("Importing Type.op_*Equality intrinsic\n"); - GenTree* op1 = impStackTop(1).val; - GenTree* op2 = impStackTop(0).val; - GenTree* optTree = gtFoldTypeEqualityCall(ni == NI_System_Type_op_Equality, op1, op2); - if (optTree != nullptr) - { - // Success, clean up the evaluation stack. - impPopStack(); - impPopStack(); - - // See if we can optimize even further, to a handle compare. - optTree = gtFoldTypeCompare(optTree); - - // See if we can now fold a handle compare to a constant. - optTree = gtFoldExpr(optTree); - - retNode = optTree; - } - else - { - // Retry optimizing these later - isSpecial = true; - } - break; - } - - case NI_System_Enum_HasFlag: - { - GenTree* thisOp = impStackTop(1).val; - GenTree* flagOp = impStackTop(0).val; - GenTree* optTree = gtOptimizeEnumHasFlag(thisOp, flagOp); - - if (optTree != nullptr) - { - // Optimization successful. Pop the stack for real. - impPopStack(); - impPopStack(); - retNode = optTree; - } - else - { - // Retry optimizing this during morph. - isSpecial = true; - } - - break; - } - - case NI_System_Type_IsAssignableFrom: - { - GenTree* typeTo = impStackTop(1).val; - GenTree* typeFrom = impStackTop(0).val; - - retNode = impTypeIsAssignable(typeTo, typeFrom); - break; - } - - case NI_System_Type_IsAssignableTo: - { - GenTree* typeTo = impStackTop(0).val; - GenTree* typeFrom = impStackTop(1).val; - - retNode = impTypeIsAssignable(typeTo, typeFrom); - break; - } - - case NI_System_Type_get_IsValueType: - case NI_System_Type_get_IsByRefLike: - { - // Optimize - // - // call Type.GetTypeFromHandle (which is replaced with CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE) - // call Type.IsXXX - // - // to `true` or `false` - // e.g., `typeof(int).IsValueType` => `true` - // e.g., `typeof(Span).IsByRefLike` => `true` - if (impStackTop().val->IsCall()) - { - GenTreeCall* call = impStackTop().val->AsCall(); - if (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE)) - { - assert(call->gtArgs.CountArgs() == 1); - CORINFO_CLASS_HANDLE hClass = - gtGetHelperArgClassHandle(call->gtArgs.GetArgByIndex(0)->GetEarlyNode()); - if (hClass != NO_CLASS_HANDLE) - { - switch (ni) - { - case NI_System_Type_get_IsValueType: - retNode = gtNewIconNode(eeIsValueClass(hClass) ? 1 : 0); - break; - case NI_System_Type_get_IsByRefLike: - retNode = gtNewIconNode( - (info.compCompHnd->getClassAttribs(hClass) & CORINFO_FLG_BYREF_LIKE) ? 1 : 0); - break; - default: - NO_WAY("Intrinsic not supported in this path."); - } - impPopStack(); // drop CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE call - } - } - } - break; - } - - case NI_System_Threading_Thread_get_ManagedThreadId: - { - if (impStackTop().val->OperIs(GT_RET_EXPR)) - { - GenTreeCall* call = impStackTop().val->AsRetExpr()->gtInlineCandidate->AsCall(); - if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) - { - if (lookupNamedIntrinsic(call->gtCallMethHnd) == NI_System_Threading_Thread_get_CurrentThread) - { - // drop get_CurrentThread() call - impPopStack(); - call->ReplaceWith(gtNewNothingNode(), this); - retNode = gtNewHelperCallNode(CORINFO_HELP_GETCURRENTMANAGEDTHREADID, TYP_INT); - } - } - } - break; - } - -#ifdef TARGET_ARM64 - // Intrinsify Interlocked.Or and Interlocked.And only for arm64-v8.1 (and newer) - // TODO-CQ: Implement for XArch (https://github.com/dotnet/runtime/issues/32239). - case NI_System_Threading_Interlocked_Or: - case NI_System_Threading_Interlocked_And: - { - if (compOpportunisticallyDependsOn(InstructionSet_Atomics)) - { - assert(sig->numArgs == 2); - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - genTreeOps op = (ni == NI_System_Threading_Interlocked_Or) ? GT_XORR : GT_XAND; - retNode = gtNewOperNode(op, genActualType(callType), op1, op2); - retNode->gtFlags |= GTF_GLOB_REF | GTF_ASG; - } - break; - } -#endif // TARGET_ARM64 - -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) - // TODO-ARM-CQ: reenable treating InterlockedCmpXchg32 operation as intrinsic - case NI_System_Threading_Interlocked_CompareExchange: - { - var_types retType = JITtype2varType(sig->retType); - if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4)) - { - break; - } - if ((retType != TYP_INT) && (retType != TYP_LONG)) - { - break; - } - - assert(callType != TYP_STRUCT); - assert(sig->numArgs == 3); - - GenTree* op3 = impPopStack().val; // comparand - GenTree* op2 = impPopStack().val; // value - GenTree* op1 = impPopStack().val; // location - - GenTree* node = new (this, GT_CMPXCHG) GenTreeCmpXchg(genActualType(callType), op1, op2, op3); - - node->AsCmpXchg()->gtOpLocation->gtFlags |= GTF_DONT_CSE; - retNode = node; - break; - } - - case NI_System_Threading_Interlocked_Exchange: - case NI_System_Threading_Interlocked_ExchangeAdd: - { - assert(callType != TYP_STRUCT); - assert(sig->numArgs == 2); - - var_types retType = JITtype2varType(sig->retType); - if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4)) - { - break; - } - if ((retType != TYP_INT) && (retType != TYP_LONG)) - { - break; - } - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - - // This creates: - // val - // XAdd - // addr - // field (for example) - // - // In the case where the first argument is the address of a local, we might - // want to make this *not* make the var address-taken -- but atomic instructions - // on a local are probably pretty useless anyway, so we probably don't care. - - op1 = gtNewOperNode(ni == NI_System_Threading_Interlocked_ExchangeAdd ? GT_XADD : GT_XCHG, - genActualType(callType), op1, op2); - op1->gtFlags |= GTF_GLOB_REF | GTF_ASG; - retNode = op1; - break; - } -#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64) - - case NI_System_Threading_Interlocked_MemoryBarrier: - case NI_System_Threading_Interlocked_ReadMemoryBarrier: - { - assert(sig->numArgs == 0); - - GenTree* op1 = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID); - op1->gtFlags |= GTF_GLOB_REF | GTF_ASG; - - // On XARCH `NI_System_Threading_Interlocked_ReadMemoryBarrier` fences need not be emitted. - // However, we still need to capture the effect on reordering. - if (ni == NI_System_Threading_Interlocked_ReadMemoryBarrier) - { - op1->gtFlags |= GTF_MEMORYBARRIER_LOAD; - } - - retNode = op1; - break; - } - -#ifdef FEATURE_HW_INTRINSICS - case NI_System_Math_FusedMultiplyAdd: - { -#ifdef TARGET_XARCH - if (compExactlyDependsOn(InstructionSet_FMA)) - { - assert(varTypeIsFloating(callType)); - - // We are constructing a chain of intrinsics similar to: - // return FMA.MultiplyAddScalar( - // Vector128.CreateScalarUnsafe(x), - // Vector128.CreateScalarUnsafe(y), - // Vector128.CreateScalarUnsafe(z) - // ).ToScalar(); - - GenTree* op3 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, - NI_Vector128_CreateScalarUnsafe, callJitType, 16); - GenTree* op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, - NI_Vector128_CreateScalarUnsafe, callJitType, 16); - GenTree* op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, - NI_Vector128_CreateScalarUnsafe, callJitType, 16); - GenTree* res = - gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_FMA_MultiplyAddScalar, callJitType, 16); - - retNode = gtNewSimdHWIntrinsicNode(callType, res, NI_Vector128_ToScalar, callJitType, 16); - break; - } -#elif defined(TARGET_ARM64) - if (compExactlyDependsOn(InstructionSet_AdvSimd)) - { - assert(varTypeIsFloating(callType)); - - // We are constructing a chain of intrinsics similar to: - // return AdvSimd.FusedMultiplyAddScalar( - // Vector64.Create{ScalarUnsafe}(z), - // Vector64.Create{ScalarUnsafe}(y), - // Vector64.Create{ScalarUnsafe}(x) - // ).ToScalar(); - - NamedIntrinsic createVector64 = - (callType == TYP_DOUBLE) ? NI_Vector64_Create : NI_Vector64_CreateScalarUnsafe; - - constexpr unsigned int simdSize = 8; - - GenTree* op3 = - gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); - GenTree* op2 = - gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); - GenTree* op1 = - gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); - - // Note that AdvSimd.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 + op2 * op3 - // while Math{F}.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 * op2 + op3 - retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op3, op2, op1, NI_AdvSimd_FusedMultiplyAddScalar, - callJitType, simdSize); - - retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector64_ToScalar, callJitType, simdSize); - break; - } -#endif - - // TODO-CQ-XArch: Ideally we would create a GT_INTRINSIC node for fma, however, that currently - // requires more extensive changes to valuenum to support methods with 3 operands - - // We want to generate a GT_INTRINSIC node in the case the call can't be treated as - // a target intrinsic so that we can still benefit from CSE and constant folding. - - break; - } -#endif // FEATURE_HW_INTRINSICS - - case NI_System_Math_Abs: - case NI_System_Math_Acos: - case NI_System_Math_Acosh: - case NI_System_Math_Asin: - case NI_System_Math_Asinh: - case NI_System_Math_Atan: - case NI_System_Math_Atanh: - case NI_System_Math_Atan2: - case NI_System_Math_Cbrt: - case NI_System_Math_Ceiling: - case NI_System_Math_Cos: - case NI_System_Math_Cosh: - case NI_System_Math_Exp: - case NI_System_Math_Floor: - case NI_System_Math_FMod: - case NI_System_Math_ILogB: - case NI_System_Math_Log: - case NI_System_Math_Log2: - case NI_System_Math_Log10: - { - retNode = impMathIntrinsic(method, sig, callType, ni, tailCall); - break; - } -#if defined(TARGET_ARM64) - // ARM64 has fmax/fmin which are IEEE754:2019 minimum/maximum compatible - // TODO-XARCH-CQ: Enable this for XARCH when one of the arguments is a constant - // so we can then emit maxss/minss and avoid NaN/-0.0 handling - case NI_System_Math_Max: - case NI_System_Math_Min: -#endif - -#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) - case NI_System_Math_Max: - case NI_System_Math_Min: - { - assert(varTypeIsFloating(callType)); - assert(sig->numArgs == 2); - - GenTreeDblCon* cnsNode = nullptr; - GenTree* otherNode = nullptr; - - GenTree* op2 = impStackTop().val; - GenTree* op1 = impStackTop(1).val; - - if (op2->IsCnsFltOrDbl()) - { - cnsNode = op2->AsDblCon(); - otherNode = op1; - } - else if (op1->IsCnsFltOrDbl()) - { - cnsNode = op1->AsDblCon(); - otherNode = op2; - } - - if (cnsNode == nullptr) - { - // no constant node, nothing to do - break; - } - - if (otherNode->IsCnsFltOrDbl()) - { - // both are constant, we can fold this operation completely. Pop both peeked values - - if (ni == NI_System_Math_Max) - { - cnsNode->SetDconValue( - FloatingPointUtils::maximum(cnsNode->DconValue(), otherNode->AsDblCon()->DconValue())); - } - else - { - assert(ni == NI_System_Math_Min); - cnsNode->SetDconValue( - FloatingPointUtils::minimum(cnsNode->DconValue(), otherNode->AsDblCon()->DconValue())); - } - - retNode = cnsNode; - - impPopStack(); - impPopStack(); - DEBUG_DESTROY_NODE(otherNode); - - break; - } - - // only one is constant, we can fold in specialized scenarios - - if (cnsNode->IsFloatNaN()) - { - impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("spill side effects before propagating NaN")); - - // maxsd, maxss, minsd, and minss all return op2 if either is NaN - // we require NaN to be propagated so ensure the known NaN is op2 - - impPopStack(); - impPopStack(); - DEBUG_DESTROY_NODE(otherNode); - - retNode = cnsNode; - break; - } - - if (!compOpportunisticallyDependsOn(InstructionSet_SSE2)) - { - break; - } - - if (ni == NI_System_Math_Max) - { - // maxsd, maxss return op2 if both inputs are 0 of either sign - // we require +0 to be greater than -0, so we can't handle if - // the known constant is +0. This is because if the unknown value - // is -0, we'd need the cns to be op2. But if the unknown value - // is NaN, we'd need the cns to be op1 instead. - - if (cnsNode->IsFloatPositiveZero()) - { - break; - } - - // Given the checks, op1 can safely be the cns and op2 the other node - - ni = (callType == TYP_DOUBLE) ? NI_SSE2_Max : NI_SSE_Max; - - // one is constant and we know its something we can handle, so pop both peeked values - - op1 = cnsNode; - op2 = otherNode; - } - else - { - assert(ni == NI_System_Math_Min); - - // minsd, minss return op2 if both inputs are 0 of either sign - // we require -0 to be lesser than +0, so we can't handle if - // the known constant is -0. This is because if the unknown value - // is +0, we'd need the cns to be op2. But if the unknown value - // is NaN, we'd need the cns to be op1 instead. - - if (cnsNode->IsFloatNegativeZero()) - { - break; - } - - // Given the checks, op1 can safely be the cns and op2 the other node - - ni = (callType == TYP_DOUBLE) ? NI_SSE2_Min : NI_SSE_Min; - - // one is constant and we know its something we can handle, so pop both peeked values - - op1 = cnsNode; - op2 = otherNode; - } - - assert(op1->IsCnsFltOrDbl() && !op2->IsCnsFltOrDbl()); - - impPopStack(); - impPopStack(); - - GenTreeVecCon* vecCon = gtNewVconNode(TYP_SIMD16); - - if (callJitType == CORINFO_TYPE_FLOAT) - { - vecCon->gtSimd16Val.f32[0] = (float)op1->AsDblCon()->DconValue(); - } - else - { - vecCon->gtSimd16Val.f64[0] = op1->AsDblCon()->DconValue(); - } - - op1 = vecCon; - op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, NI_Vector128_CreateScalarUnsafe, callJitType, 16); - - retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, ni, callJitType, 16); - retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector128_ToScalar, callJitType, 16); - - break; - } -#endif - - case NI_System_Math_Pow: - case NI_System_Math_Round: - case NI_System_Math_Sin: - case NI_System_Math_Sinh: - case NI_System_Math_Sqrt: - case NI_System_Math_Tan: - case NI_System_Math_Tanh: - case NI_System_Math_Truncate: - { - retNode = impMathIntrinsic(method, sig, callType, ni, tailCall); - break; - } - - case NI_System_Array_Clone: - case NI_System_Collections_Generic_Comparer_get_Default: - case NI_System_Collections_Generic_EqualityComparer_get_Default: - case NI_System_Object_MemberwiseClone: - case NI_System_Threading_Thread_get_CurrentThread: - { - // Flag for later handling. - isSpecial = true; - break; - } - - case NI_System_Object_GetType: - { - JITDUMP("\n impIntrinsic: call to Object.GetType\n"); - GenTree* op1 = impStackTop(0).val; - - // If we're calling GetType on a boxed value, just get the type directly. - if (op1->IsBoxedValue()) - { - JITDUMP("Attempting to optimize box(...).getType() to direct type construction\n"); - - // Try and clean up the box. Obtain the handle we - // were going to pass to the newobj. - GenTree* boxTypeHandle = gtTryRemoveBoxUpstreamEffects(op1, BR_REMOVE_AND_NARROW_WANT_TYPE_HANDLE); - - if (boxTypeHandle != nullptr) - { - // Note we don't need to play the TYP_STRUCT games here like - // do for LDTOKEN since the return value of this operator is Type, - // not RuntimeTypeHandle. - impPopStack(); - GenTree* runtimeType = - gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, boxTypeHandle); - retNode = runtimeType; - } - } - - // If we have a constrained callvirt with a "box this" transform - // we know we have a value class and hence an exact type. - // - // If so, instead of boxing and then extracting the type, just - // construct the type directly. - if ((retNode == nullptr) && (pConstrainedResolvedToken != nullptr) && - (constraintCallThisTransform == CORINFO_BOX_THIS)) - { - // Ensure this is one of the simple box cases (in particular, rule out nullables). - const CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pConstrainedResolvedToken->hClass); - const bool isSafeToOptimize = (boxHelper == CORINFO_HELP_BOX); - - if (isSafeToOptimize) - { - JITDUMP("Optimizing constrained box-this obj.getType() to direct type construction\n"); - impPopStack(); - GenTree* typeHandleOp = - impTokenToHandle(pConstrainedResolvedToken, nullptr, true /* mustRestoreHandle */); - if (typeHandleOp == nullptr) - { - assert(compDonotInline()); - return nullptr; - } - GenTree* runtimeType = - gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, typeHandleOp); - retNode = runtimeType; - } - } - -#ifdef DEBUG - if (retNode != nullptr) - { - JITDUMP("Optimized result for call to GetType is\n"); - if (verbose) - { - gtDispTree(retNode); - } - } -#endif - - // Else expand as an intrinsic, unless the call is constrained, - // in which case we defer expansion to allow impImportCall do the - // special constraint processing. - if ((retNode == nullptr) && (pConstrainedResolvedToken == nullptr)) - { - JITDUMP("Expanding as special intrinsic\n"); - impPopStack(); - op1 = new (this, GT_INTRINSIC) GenTreeIntrinsic(genActualType(callType), op1, ni, method); - - // Set the CALL flag to indicate that the operator is implemented by a call. - // Set also the EXCEPTION flag because the native implementation of - // NI_System_Object_GetType intrinsic can throw NullReferenceException. - op1->gtFlags |= (GTF_CALL | GTF_EXCEPT); - retNode = op1; - // Might be further optimizable, so arrange to leave a mark behind - isSpecial = true; - } - - if (retNode == nullptr) - { - JITDUMP("Leaving as normal call\n"); - // Might be further optimizable, so arrange to leave a mark behind - isSpecial = true; - } - - break; - } - - case NI_System_Array_GetLength: - case NI_System_Array_GetLowerBound: - case NI_System_Array_GetUpperBound: - { - // System.Array.GetLength(Int32) method: - // public int GetLength(int dimension) - // System.Array.GetLowerBound(Int32) method: - // public int GetLowerBound(int dimension) - // System.Array.GetUpperBound(Int32) method: - // public int GetUpperBound(int dimension) - // - // Only implement these as intrinsics for multi-dimensional arrays. - // Only handle constant dimension arguments. - - GenTree* gtDim = impStackTop().val; - GenTree* gtArr = impStackTop(1).val; - - if (gtDim->IsIntegralConst()) - { - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE arrCls = gtGetClassHandle(gtArr, &isExact, &isNonNull); - if (arrCls != NO_CLASS_HANDLE) - { - unsigned rank = info.compCompHnd->getArrayRank(arrCls); - if ((rank > 1) && !info.compCompHnd->isSDArray(arrCls)) - { - // `rank` is guaranteed to be <=32 (see MAX_RANK in vm\array.h). Any constant argument - // is `int` sized. - INT64 dimValue = gtDim->AsIntConCommon()->IntegralValue(); - assert((unsigned int)dimValue == dimValue); - unsigned dim = (unsigned int)dimValue; - if (dim < rank) - { - // This is now known to be a multi-dimension array with a constant dimension - // that is in range; we can expand it as an intrinsic. - - impPopStack().val; // Pop the dim and array object; we already have a pointer to them. - impPopStack().val; - - // Make sure there are no global effects in the array (such as it being a function - // call), so we can mark the generated indirection with GTF_IND_INVARIANT. In the - // GetUpperBound case we need the cloned object, since we refer to the array - // object twice. In the other cases, we don't need to clone. - GenTree* gtArrClone = nullptr; - if (((gtArr->gtFlags & GTF_GLOB_EFFECT) != 0) || (ni == NI_System_Array_GetUpperBound)) - { - gtArr = impCloneExpr(gtArr, >ArrClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL, - nullptr DEBUGARG("MD intrinsics array")); - } - - switch (ni) - { - case NI_System_Array_GetLength: - { - // Generate *(array + offset-to-length-array + sizeof(int) * dim) - unsigned offs = eeGetMDArrayLengthOffset(rank, dim); - GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); - GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); - retNode = gtNewIndir(TYP_INT, gtAddr); - retNode->gtFlags |= GTF_IND_INVARIANT; - break; - } - case NI_System_Array_GetLowerBound: - { - // Generate *(array + offset-to-bounds-array + sizeof(int) * dim) - unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim); - GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); - GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); - retNode = gtNewIndir(TYP_INT, gtAddr); - retNode->gtFlags |= GTF_IND_INVARIANT; - break; - } - case NI_System_Array_GetUpperBound: - { - assert(gtArrClone != nullptr); - - // Generate: - // *(array + offset-to-length-array + sizeof(int) * dim) + - // *(array + offset-to-bounds-array + sizeof(int) * dim) - 1 - unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim); - GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); - GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); - GenTree* gtLowerBound = gtNewIndir(TYP_INT, gtAddr); - gtLowerBound->gtFlags |= GTF_IND_INVARIANT; - - offs = eeGetMDArrayLengthOffset(rank, dim); - gtOffs = gtNewIconNode(offs, TYP_I_IMPL); - gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArrClone, gtOffs); - GenTree* gtLength = gtNewIndir(TYP_INT, gtAddr); - gtLength->gtFlags |= GTF_IND_INVARIANT; - - GenTree* gtSum = gtNewOperNode(GT_ADD, TYP_INT, gtLowerBound, gtLength); - GenTree* gtOne = gtNewIconNode(1, TYP_INT); - retNode = gtNewOperNode(GT_SUB, TYP_INT, gtSum, gtOne); - break; - } - default: - unreached(); - } - } - } - } - } - break; - } - - case NI_System_Buffers_Binary_BinaryPrimitives_ReverseEndianness: - { - assert(sig->numArgs == 1); - - // We expect the return type of the ReverseEndianness routine to match the type of the - // one and only argument to the method. We use a special instruction for 16-bit - // BSWAPs since on x86 processors this is implemented as ROR <16-bit reg>, 8. Additionally, - // we only emit 64-bit BSWAP instructions on 64-bit archs; if we're asked to perform a - // 64-bit byte swap on a 32-bit arch, we'll fall to the default case in the switch block below. - - switch (sig->retType) - { - case CorInfoType::CORINFO_TYPE_SHORT: - case CorInfoType::CORINFO_TYPE_USHORT: - retNode = gtNewCastNode(TYP_INT, gtNewOperNode(GT_BSWAP16, TYP_INT, impPopStack().val), false, - callType); - break; - - case CorInfoType::CORINFO_TYPE_INT: - case CorInfoType::CORINFO_TYPE_UINT: -#ifdef TARGET_64BIT - case CorInfoType::CORINFO_TYPE_LONG: - case CorInfoType::CORINFO_TYPE_ULONG: -#endif // TARGET_64BIT - retNode = gtNewOperNode(GT_BSWAP, callType, impPopStack().val); - break; - - default: - // This default case gets hit on 32-bit archs when a call to a 64-bit overload - // of ReverseEndianness is encountered. In that case we'll let JIT treat this as a standard - // method call, where the implementation decomposes the operation into two 32-bit - // bswap routines. If the input to the 64-bit function is a constant, then we rely - // on inlining + constant folding of 32-bit bswaps to effectively constant fold - // the 64-bit call site. - break; - } - - break; - } - - // Fold PopCount for constant input - case NI_System_Numerics_BitOperations_PopCount: - { - assert(sig->numArgs == 1); - if (impStackTop().val->IsIntegralConst()) - { - typeInfo argType = verParseArgSigToTypeInfo(sig, sig->args).NormaliseForStack(); - INT64 cns = impPopStack().val->AsIntConCommon()->IntegralValue(); - if (argType.IsType(TI_LONG)) - { - retNode = gtNewIconNode(genCountBits(cns), callType); - } - else - { - assert(argType.IsType(TI_INT)); - retNode = gtNewIconNode(genCountBits(static_cast(cns)), callType); - } - } - break; - } - - case NI_System_GC_KeepAlive: - { - retNode = impKeepAliveIntrinsic(impPopStack().val); - break; - } - - case NI_System_BitConverter_DoubleToInt64Bits: - { - GenTree* op1 = impStackTop().val; - assert(varTypeIsFloating(op1)); - - if (op1->IsCnsFltOrDbl()) - { - impPopStack(); - - double f64Cns = op1->AsDblCon()->DconValue(); - retNode = gtNewLconNode(*reinterpret_cast(&f64Cns)); - } -#if TARGET_64BIT - else - { - // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work - impPopStack(); - - if (op1->TypeGet() != TYP_DOUBLE) - { - op1 = gtNewCastNode(TYP_DOUBLE, op1, false, TYP_DOUBLE); - } - retNode = gtNewBitCastNode(TYP_LONG, op1); - } -#endif - break; - } - - case NI_System_BitConverter_Int32BitsToSingle: - { - GenTree* op1 = impPopStack().val; - assert(varTypeIsInt(op1)); - - if (op1->IsIntegralConst()) - { - int32_t i32Cns = (int32_t)op1->AsIntConCommon()->IconValue(); - retNode = gtNewDconNode(*reinterpret_cast(&i32Cns), TYP_FLOAT); - } - else - { - retNode = gtNewBitCastNode(TYP_FLOAT, op1); - } - break; - } - - case NI_System_BitConverter_Int64BitsToDouble: - { - GenTree* op1 = impStackTop().val; - assert(varTypeIsLong(op1)); - - if (op1->IsIntegralConst()) - { - impPopStack(); - - int64_t i64Cns = op1->AsIntConCommon()->LngValue(); - retNode = gtNewDconNode(*reinterpret_cast(&i64Cns)); - } -#if TARGET_64BIT - else - { - // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work - impPopStack(); - - retNode = gtNewBitCastNode(TYP_DOUBLE, op1); - } -#endif - break; - } - - case NI_System_BitConverter_SingleToInt32Bits: - { - GenTree* op1 = impPopStack().val; - assert(varTypeIsFloating(op1)); - - if (op1->IsCnsFltOrDbl()) - { - float f32Cns = (float)op1->AsDblCon()->DconValue(); - retNode = gtNewIconNode(*reinterpret_cast(&f32Cns)); - } - else - { - if (op1->TypeGet() != TYP_FLOAT) - { - op1 = gtNewCastNode(TYP_FLOAT, op1, false, TYP_FLOAT); - } - retNode = gtNewBitCastNode(TYP_INT, op1); - } - break; - } - - default: - break; - } - } - - if (mustExpand && (retNode == nullptr)) - { - assert(!"Unhandled must expand intrinsic, throwing PlatformNotSupportedException"); - return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); - } - - // Optionally report if this intrinsic is special - // (that is, potentially re-optimizable during morph). - if (isSpecialIntrinsic != nullptr) - { - *isSpecialIntrinsic = isSpecial; - } - - return retNode; -} - -GenTree* Compiler::impSRCSUnsafeIntrinsic(NamedIntrinsic intrinsic, - CORINFO_CLASS_HANDLE clsHnd, - CORINFO_METHOD_HANDLE method, - CORINFO_SIG_INFO* sig) -{ - // NextCallRetAddr requires a CALL, so return nullptr. - if (info.compHasNextCallRetAddr) - { - return nullptr; - } - - assert(sig->sigInst.classInstCount == 0); - - switch (intrinsic) - { - case NI_SRCS_UNSAFE_Add: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // sizeof !!T - // conv.i - // mul - // add - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1, op2); - - op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL); - - unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); - - if (classSize != 1) - { - GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL); - op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size); - } - - var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2); - return gtNewOperNode(GT_ADD, type, op1, op2); - } - - case NI_SRCS_UNSAFE_AddByteOffset: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // add - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1, op2); - - var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2); - return gtNewOperNode(GT_ADD, type, op1, op2); - } - - case NI_SRCS_UNSAFE_AreSame: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // ceq - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - - GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, op2); - return gtFoldExpr(tmp); - } - - case NI_SRCS_UNSAFE_As: - { - assert((sig->sigInst.methInstCount == 1) || (sig->sigInst.methInstCount == 2)); - - // ldarg.0 - // ret - - return impPopStack().val; - } - - case NI_SRCS_UNSAFE_AsPointer: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // conv.u - // ret - - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1); - - return gtNewCastNode(TYP_I_IMPL, op1, /* uns */ false, TYP_I_IMPL); - } - - case NI_SRCS_UNSAFE_AsRef: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ret - - return impPopStack().val; - } - - case NI_SRCS_UNSAFE_ByteOffset: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.1 - // ldarg.0 - // sub - // ret - - impSpillSideEffect(true, verCurrentState.esStackDepth - - 2 DEBUGARG("Spilling op1 side effects for Unsafe.ByteOffset")); - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1, op2); - - var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op2, &op1); - return gtNewOperNode(GT_SUB, type, op2, op1); - } - - case NI_SRCS_UNSAFE_Copy: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // ldobj !!T - // stobj !!T - // ret - - return nullptr; - } - - case NI_SRCS_UNSAFE_CopyBlock: - { - assert(sig->sigInst.methInstCount == 0); - - // ldarg.0 - // ldarg.1 - // ldarg.2 - // cpblk - // ret - - return nullptr; - } - - case NI_SRCS_UNSAFE_CopyBlockUnaligned: - { - assert(sig->sigInst.methInstCount == 0); - - // ldarg.0 - // ldarg.1 - // ldarg.2 - // unaligned. 0x1 - // cpblk - // ret - - return nullptr; - } - - case NI_SRCS_UNSAFE_InitBlock: - { - assert(sig->sigInst.methInstCount == 0); - - // ldarg.0 - // ldarg.1 - // ldarg.2 - // initblk - // ret - - return nullptr; - } - - case NI_SRCS_UNSAFE_InitBlockUnaligned: - { - assert(sig->sigInst.methInstCount == 0); - - // ldarg.0 - // ldarg.1 - // ldarg.2 - // unaligned. 0x1 - // initblk - // ret - - return nullptr; - } - - case NI_SRCS_UNSAFE_IsAddressGreaterThan: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // cgt.un - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - - GenTree* tmp = gtNewOperNode(GT_GT, TYP_INT, op1, op2); - tmp->gtFlags |= GTF_UNSIGNED; - return gtFoldExpr(tmp); - } - - case NI_SRCS_UNSAFE_IsAddressLessThan: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // clt.un - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - - GenTree* tmp = gtNewOperNode(GT_LT, TYP_INT, op1, op2); - tmp->gtFlags |= GTF_UNSIGNED; - return gtFoldExpr(tmp); - } - - case NI_SRCS_UNSAFE_IsNullRef: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldc.i4.0 - // conv.u - // ceq - // ret - - GenTree* op1 = impPopStack().val; - GenTree* cns = gtNewIconNode(0, TYP_BYREF); - GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, cns); - return gtFoldExpr(tmp); - } - - case NI_SRCS_UNSAFE_NullRef: - { - assert(sig->sigInst.methInstCount == 1); - - // ldc.i4.0 - // conv.u - // ret - - return gtNewIconNode(0, TYP_BYREF); - } - - case NI_SRCS_UNSAFE_Read: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldobj !!T - // ret - - return nullptr; - } - - case NI_SRCS_UNSAFE_ReadUnaligned: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // unaligned. 0x1 - // ldobj !!T - // ret - - return nullptr; - } - - case NI_SRCS_UNSAFE_SizeOf: - { - assert(sig->sigInst.methInstCount == 1); - - // sizeof !!T - // ret - - unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); - return gtNewIconNode(classSize, TYP_INT); - } - - case NI_SRCS_UNSAFE_SkipInit: - { - assert(sig->sigInst.methInstCount == 1); - - // ret - - GenTree* op1 = impPopStack().val; - - if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) - { - return gtUnusedValNode(op1); - } - else - { - return gtNewNothingNode(); - } - } - - case NI_SRCS_UNSAFE_Subtract: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // sizeof !!T - // conv.i - // mul - // sub - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1, op2); - - op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL); - - unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); - - if (classSize != 1) - { - GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL); - op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size); - } - - var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2); - return gtNewOperNode(GT_SUB, type, op1, op2); - } - - case NI_SRCS_UNSAFE_SubtractByteOffset: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // sub - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1, op2); - - var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2); - return gtNewOperNode(GT_SUB, type, op1, op2); - } - - case NI_SRCS_UNSAFE_Unbox: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // unbox !!T - // ret - - return nullptr; - } - - case NI_SRCS_UNSAFE_Write: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // stobj !!T - // ret - - return nullptr; - } - - case NI_SRCS_UNSAFE_WriteUnaligned: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // unaligned. 0x01 - // stobj !!T - // ret - - return nullptr; - } - - default: - { - unreached(); - } - } -} - GenTree* Compiler::impTypeIsAssignable(GenTree* typeTo, GenTree* typeFrom) { // Optimize patterns like: diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index e5e251f..785c229 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1789,3 +1789,2283 @@ void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* s } } } + +//------------------------------------------------------------------------ +// impInitializeArrayIntrinsic: Attempts to replace a call to InitializeArray +// with a GT_COPYBLK node. +// +// Arguments: +// sig - The InitializeArray signature. +// +// Return Value: +// A pointer to the newly created GT_COPYBLK node if the replacement succeeds or +// nullptr otherwise. +// +// Notes: +// The function recognizes the following IL pattern: +// ldc or a list of ldc / +// newarr or newobj +// dup +// ldtoken +// call InitializeArray +// The lower bounds need not be constant except when the array rank is 1. +// The function recognizes all kinds of arrays thus enabling a small runtime +// such as NativeAOT to skip providing an implementation for InitializeArray. + +GenTree* Compiler::impInitializeArrayIntrinsic(CORINFO_SIG_INFO* sig) +{ + assert(sig->numArgs == 2); + + GenTree* fieldTokenNode = impStackTop(0).val; + GenTree* arrayLocalNode = impStackTop(1).val; + + // + // Verify that the field token is known and valid. Note that it's also + // possible for the token to come from reflection, in which case we cannot do + // the optimization and must therefore revert to calling the helper. You can + // see an example of this in bvt\DynIL\initarray2.exe (in Main). + // + + // Check to see if the ldtoken helper call is what we see here. + if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) || + (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD))) + { + return nullptr; + } + + // Strip helper call away + fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode(); + + if (fieldTokenNode->gtOper == GT_IND) + { + fieldTokenNode = fieldTokenNode->AsOp()->gtOp1; + } + + // Check for constant + if (fieldTokenNode->gtOper != GT_CNS_INT) + { + return nullptr; + } + + CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle; + if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr)) + { + return nullptr; + } + + // + // We need to get the number of elements in the array and the size of each element. + // We verify that the newarr statement is exactly what we expect it to be. + // If it's not then we just return NULL and we don't optimize this call + // + + // It is possible the we don't have any statements in the block yet. + if (impLastStmt == nullptr) + { + return nullptr; + } + + // + // We start by looking at the last statement, making sure it's an assignment, and + // that the target of the assignment is the array passed to InitializeArray. + // + GenTree* arrayAssignment = impLastStmt->GetRootNode(); + if ((arrayAssignment->gtOper != GT_ASG) || (arrayAssignment->AsOp()->gtOp1->gtOper != GT_LCL_VAR) || + (arrayLocalNode->gtOper != GT_LCL_VAR) || (arrayAssignment->AsOp()->gtOp1->AsLclVarCommon()->GetLclNum() != + arrayLocalNode->AsLclVarCommon()->GetLclNum())) + { + return nullptr; + } + + // + // Make sure that the object being assigned is a helper call. + // + + GenTree* newArrayCall = arrayAssignment->AsOp()->gtOp2; + if ((newArrayCall->gtOper != GT_CALL) || (newArrayCall->AsCall()->gtCallType != CT_HELPER)) + { + return nullptr; + } + + // + // Verify that it is one of the new array helpers. + // + + bool isMDArray = false; + + if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_DIRECT) && + newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_OBJ) && + newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_VC) && + newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_ALIGN8) +#ifdef FEATURE_READYTORUN + && newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1) +#endif + ) + { + if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEW_MDARR)) + { + return nullptr; + } + + isMDArray = true; + } + + CORINFO_CLASS_HANDLE arrayClsHnd = (CORINFO_CLASS_HANDLE)newArrayCall->AsCall()->compileTimeHelperArgumentHandle; + + // + // Make sure we found a compile time handle to the array + // + + if (!arrayClsHnd) + { + return nullptr; + } + + unsigned rank = 0; + S_UINT32 numElements; + + if (isMDArray) + { + rank = info.compCompHnd->getArrayRank(arrayClsHnd); + + if (rank == 0) + { + return nullptr; + } + + assert(newArrayCall->AsCall()->gtArgs.CountArgs() == 3); + GenTree* numArgsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode(); + GenTree* argsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(2)->GetNode(); + + // + // The number of arguments should be a constant between 1 and 64. The rank can't be 0 + // so at least one length must be present and the rank can't exceed 32 so there can + // be at most 64 arguments - 32 lengths and 32 lower bounds. + // + + if ((!numArgsArg->IsCnsIntOrI()) || (numArgsArg->AsIntCon()->IconValue() < 1) || + (numArgsArg->AsIntCon()->IconValue() > 64)) + { + return nullptr; + } + + unsigned numArgs = static_cast(numArgsArg->AsIntCon()->IconValue()); + bool lowerBoundsSpecified; + + if (numArgs == rank * 2) + { + lowerBoundsSpecified = true; + } + else if (numArgs == rank) + { + lowerBoundsSpecified = false; + + // + // If the rank is 1 and a lower bound isn't specified then the runtime creates + // a SDArray. Note that even if a lower bound is specified it can be 0 and then + // we get a SDArray as well, see the for loop below. + // + + if (rank == 1) + { + isMDArray = false; + } + } + else + { + return nullptr; + } + + // + // The rank is known to be at least 1 so we can start with numElements being 1 + // to avoid the need to special case the first dimension. + // + + numElements = S_UINT32(1); + + struct Match + { + static bool IsArgsFieldInit(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs) + { + return (tree->OperGet() == GT_ASG) && IsArgsFieldIndir(tree->gtGetOp1(), index, lvaNewObjArrayArgs) && + IsArgsAddr(tree->gtGetOp1()->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs); + } + + static bool IsArgsFieldIndir(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs) + { + return (tree->OperGet() == GT_IND) && (tree->gtGetOp1()->OperGet() == GT_ADD) && + (tree->gtGetOp1()->gtGetOp2()->IsIntegralConst(sizeof(INT32) * index)) && + IsArgsAddr(tree->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs); + } + + static bool IsArgsAddr(GenTree* tree, unsigned lvaNewObjArrayArgs) + { + return (tree->OperGet() == GT_ADDR) && (tree->gtGetOp1()->OperGet() == GT_LCL_VAR) && + (tree->gtGetOp1()->AsLclVar()->GetLclNum() == lvaNewObjArrayArgs); + } + + static bool IsComma(GenTree* tree) + { + return (tree != nullptr) && (tree->OperGet() == GT_COMMA); + } + }; + + unsigned argIndex = 0; + GenTree* comma; + + for (comma = argsArg; Match::IsComma(comma); comma = comma->gtGetOp2()) + { + if (lowerBoundsSpecified) + { + // + // In general lower bounds can be ignored because they're not needed to + // calculate the total number of elements. But for single dimensional arrays + // we need to know if the lower bound is 0 because in this case the runtime + // creates a SDArray and this affects the way the array data offset is calculated. + // + + if (rank == 1) + { + GenTree* lowerBoundAssign = comma->gtGetOp1(); + assert(Match::IsArgsFieldInit(lowerBoundAssign, argIndex, lvaNewObjArrayArgs)); + GenTree* lowerBoundNode = lowerBoundAssign->gtGetOp2(); + + if (lowerBoundNode->IsIntegralConst(0)) + { + isMDArray = false; + } + } + + comma = comma->gtGetOp2(); + argIndex++; + } + + GenTree* lengthNodeAssign = comma->gtGetOp1(); + assert(Match::IsArgsFieldInit(lengthNodeAssign, argIndex, lvaNewObjArrayArgs)); + GenTree* lengthNode = lengthNodeAssign->gtGetOp2(); + + if (!lengthNode->IsCnsIntOrI()) + { + return nullptr; + } + + numElements *= S_SIZE_T(lengthNode->AsIntCon()->IconValue()); + argIndex++; + } + + assert((comma != nullptr) && Match::IsArgsAddr(comma, lvaNewObjArrayArgs)); + + if (argIndex != numArgs) + { + return nullptr; + } + } + else + { + // + // Make sure there are exactly two arguments: the array class and + // the number of elements. + // + + GenTree* arrayLengthNode; + +#ifdef FEATURE_READYTORUN + if (newArrayCall->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1)) + { + // Array length is 1st argument for readytorun helper + arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); + } + else +#endif + { + // Array length is 2nd argument for regular helper + arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode(); + } + + // + // This optimization is only valid for a constant array size. + // + if (arrayLengthNode->gtOper != GT_CNS_INT) + { + return nullptr; + } + + numElements = S_SIZE_T(arrayLengthNode->AsIntCon()->gtIconVal); + + if (!info.compCompHnd->isSDArray(arrayClsHnd)) + { + return nullptr; + } + } + + CORINFO_CLASS_HANDLE elemClsHnd; + var_types elementType = JITtype2varType(info.compCompHnd->getChildType(arrayClsHnd, &elemClsHnd)); + + // + // Note that genTypeSize will return zero for non primitive types, which is exactly + // what we want (size will then be 0, and we will catch this in the conditional below). + // Note that we don't expect this to fail for valid binaries, so we assert in the + // non-verification case (the verification case should not assert but rather correctly + // handle bad binaries). This assert is not guarding any specific invariant, but rather + // saying that we don't expect this to happen, and if it is hit, we need to investigate + // why. + // + + S_UINT32 elemSize(genTypeSize(elementType)); + S_UINT32 size = elemSize * S_UINT32(numElements); + + if (size.IsOverflow()) + { + return nullptr; + } + + if ((size.Value() == 0) || (varTypeIsGC(elementType))) + { + return nullptr; + } + + void* initData = info.compCompHnd->getArrayInitializationData(fieldToken, size.Value()); + if (!initData) + { + return nullptr; + } + + // + // At this point we are ready to commit to implementing the InitializeArray + // intrinsic using a struct assignment. Pop the arguments from the stack and + // return the struct assignment node. + // + + impPopStack(); + impPopStack(); + + const unsigned blkSize = size.Value(); + unsigned dataOffset; + + if (isMDArray) + { + dataOffset = eeGetMDArrayDataOffset(rank); + } + else + { + dataOffset = eeGetArrayDataOffset(); + } + + GenTree* dstAddr = gtNewOperNode(GT_ADD, TYP_BYREF, arrayLocalNode, gtNewIconNode(dataOffset, TYP_I_IMPL)); + GenTree* dst = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, dstAddr, typGetBlkLayout(blkSize)); + GenTree* src = gtNewIndOfIconHandleNode(TYP_STRUCT, (size_t)initData, GTF_ICON_CONST_PTR, true); + +#ifdef DEBUG + src->gtGetOp1()->AsIntCon()->gtTargetHandle = THT_InitializeArrayIntrinsics; +#endif + + return gtNewBlkOpNode(dst, // dst + src, // src + false, // volatile + true); // copyBlock +} + +GenTree* Compiler::impCreateSpanIntrinsic(CORINFO_SIG_INFO* sig) +{ + assert(sig->numArgs == 1); + assert(sig->sigInst.methInstCount == 1); + + GenTree* fieldTokenNode = impStackTop(0).val; + + // + // Verify that the field token is known and valid. Note that it's also + // possible for the token to come from reflection, in which case we cannot do + // the optimization and must therefore revert to calling the helper. You can + // see an example of this in bvt\DynIL\initarray2.exe (in Main). + // + + // Check to see if the ldtoken helper call is what we see here. + if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) || + (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD))) + { + return nullptr; + } + + // Strip helper call away + fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); + if (fieldTokenNode->gtOper == GT_IND) + { + fieldTokenNode = fieldTokenNode->AsOp()->gtOp1; + } + + // Check for constant + if (fieldTokenNode->gtOper != GT_CNS_INT) + { + return nullptr; + } + + CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle; + if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr)) + { + return nullptr; + } + + CORINFO_CLASS_HANDLE fieldOwnerHnd = info.compCompHnd->getFieldClass(fieldToken); + + CORINFO_CLASS_HANDLE fieldClsHnd; + var_types fieldElementType = + JITtype2varType(info.compCompHnd->getFieldType(fieldToken, &fieldClsHnd, fieldOwnerHnd)); + unsigned totalFieldSize; + + // Most static initialization data fields are of some structure, but it is possible for them to be of various + // primitive types as well + if (fieldElementType == var_types::TYP_STRUCT) + { + totalFieldSize = info.compCompHnd->getClassSize(fieldClsHnd); + } + else + { + totalFieldSize = genTypeSize(fieldElementType); + } + + // Limit to primitive or enum type - see ArrayNative::GetSpanDataFrom() + CORINFO_CLASS_HANDLE targetElemHnd = sig->sigInst.methInst[0]; + if (info.compCompHnd->getTypeForPrimitiveValueClass(targetElemHnd) == CORINFO_TYPE_UNDEF) + { + return nullptr; + } + + const unsigned targetElemSize = info.compCompHnd->getClassSize(targetElemHnd); + assert(targetElemSize != 0); + + const unsigned count = totalFieldSize / targetElemSize; + if (count == 0) + { + return nullptr; + } + + void* data = info.compCompHnd->getArrayInitializationData(fieldToken, totalFieldSize); + if (!data) + { + return nullptr; + } + + // + // Ready to commit to the work + // + + impPopStack(); + + // Turn count and pointer value into constants. + GenTree* lengthValue = gtNewIconNode(count, TYP_INT); + GenTree* pointerValue = gtNewIconHandleNode((size_t)data, GTF_ICON_CONST_PTR); + + // Construct ReadOnlySpan to return. + CORINFO_CLASS_HANDLE spanHnd = sig->retTypeClass; + unsigned spanTempNum = lvaGrabTemp(true DEBUGARG("ReadOnlySpan for CreateSpan")); + lvaSetStruct(spanTempNum, spanHnd, false); + + GenTreeLclFld* pointerField = gtNewLclFldNode(spanTempNum, TYP_BYREF, 0); + GenTree* pointerFieldAsg = gtNewAssignNode(pointerField, pointerValue); + + GenTreeLclFld* lengthField = gtNewLclFldNode(spanTempNum, TYP_INT, TARGET_POINTER_SIZE); + GenTree* lengthFieldAsg = gtNewAssignNode(lengthField, lengthValue); + + // Now append a few statements the initialize the span + impAppendTree(lengthFieldAsg, CHECK_SPILL_NONE, impCurStmtDI); + impAppendTree(pointerFieldAsg, CHECK_SPILL_NONE, impCurStmtDI); + + // And finally create a tree that points at the span. + return impCreateLocalNode(spanTempNum DEBUGARG(0)); +} + +//------------------------------------------------------------------------ +// impIntrinsic: possibly expand intrinsic call into alternate IR sequence +// +// Arguments: +// newobjThis - for constructor calls, the tree for the newly allocated object +// clsHnd - handle for the intrinsic method's class +// method - handle for the intrinsic method +// sig - signature of the intrinsic method +// methodFlags - CORINFO_FLG_XXX flags of the intrinsic method +// memberRef - the token for the intrinsic method +// readonlyCall - true if call has a readonly prefix +// tailCall - true if call is in tail position +// pConstrainedResolvedToken -- resolved token for constrained call, or nullptr +// if call is not constrained +// constraintCallThisTransform -- this transform to apply for a constrained call +// pIntrinsicName [OUT] -- intrinsic name (see enumeration in namedintrinsiclist.h) +// for "traditional" jit intrinsics +// isSpecialIntrinsic [OUT] -- set true if intrinsic expansion is a call +// that is amenable to special downstream optimization opportunities +// +// Returns: +// IR tree to use in place of the call, or nullptr if the jit should treat +// the intrinsic call like a normal call. +// +// pIntrinsicName set to non-illegal value if the call is recognized as a +// traditional jit intrinsic, even if the intrinsic is not expaned. +// +// isSpecial set true if the expansion is subject to special +// optimizations later in the jit processing +// +// Notes: +// On success the IR tree may be a call to a different method or an inline +// sequence. If it is a call, then the intrinsic processing here is responsible +// for handling all the special cases, as upon return to impImportCall +// expanded intrinsics bypass most of the normal call processing. +// +// Intrinsics are generally not recognized in minopts and debug codegen. +// +// However, certain traditional intrinsics are identifed as "must expand" +// if there is no fallback implementation to invoke; these must be handled +// in all codegen modes. +// +// New style intrinsics (where the fallback implementation is in IL) are +// identified as "must expand" if they are invoked from within their +// own method bodies. +// +GenTree* Compiler::impIntrinsic(GenTree* newobjThis, + CORINFO_CLASS_HANDLE clsHnd, + CORINFO_METHOD_HANDLE method, + CORINFO_SIG_INFO* sig, + unsigned methodFlags, + int memberRef, + bool readonlyCall, + bool tailCall, + CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, + CORINFO_THIS_TRANSFORM constraintCallThisTransform, + NamedIntrinsic* pIntrinsicName, + bool* isSpecialIntrinsic) +{ + bool mustExpand = false; + bool isSpecial = false; + const bool isIntrinsic = (methodFlags & CORINFO_FLG_INTRINSIC) != 0; + + NamedIntrinsic ni = lookupNamedIntrinsic(method); + + if (isIntrinsic) + { + // The recursive non-virtual calls to Jit intrinsics are must-expand by convention. + mustExpand = gtIsRecursiveCall(method) && !(methodFlags & CORINFO_FLG_VIRTUAL); + } + else + { + // For mismatched VM (AltJit) we want to check all methods as intrinsic to ensure + // we get more accurate codegen. This particularly applies to HWIntrinsic usage + assert(!info.compMatchedVM); + } + + // We specially support the following on all platforms to allow for dead + // code optimization and to more generally support recursive intrinsics. + + if (isIntrinsic) + { + if (ni == NI_IsSupported_True) + { + assert(sig->numArgs == 0); + return gtNewIconNode(true); + } + + if (ni == NI_IsSupported_False) + { + assert(sig->numArgs == 0); + return gtNewIconNode(false); + } + + if (ni == NI_Throw_PlatformNotSupportedException) + { + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); + } + + if ((ni > NI_SRCS_UNSAFE_START) && (ni < NI_SRCS_UNSAFE_END)) + { + assert(!mustExpand); + return impSRCSUnsafeIntrinsic(ni, clsHnd, method, sig); + } + } + +#ifdef FEATURE_HW_INTRINSICS + if ((ni > NI_HW_INTRINSIC_START) && (ni < NI_HW_INTRINSIC_END)) + { + if (!isIntrinsic) + { +#if defined(TARGET_XARCH) + // We can't guarantee that all overloads for the xplat intrinsics can be + // handled by the AltJit, so limit only the platform specific intrinsics + assert((NI_Vector256_Xor + 1) == NI_X86Base_BitScanForward); + + if (ni < NI_Vector256_Xor) +#elif defined(TARGET_ARM64) + // We can't guarantee that all overloads for the xplat intrinsics can be + // handled by the AltJit, so limit only the platform specific intrinsics + assert((NI_Vector128_Xor + 1) == NI_AdvSimd_Abs); + + if (ni < NI_Vector128_Xor) +#else +#error Unsupported platform +#endif + { + // Several of the NI_Vector64/128/256 APIs do not have + // all overloads as intrinsic today so they will assert + return nullptr; + } + } + + GenTree* hwintrinsic = impHWIntrinsic(ni, clsHnd, method, sig, mustExpand); + + if (mustExpand && (hwintrinsic == nullptr)) + { + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_NOT_IMPLEMENTED, method, sig, mustExpand); + } + + return hwintrinsic; + } + + if (isIntrinsic && (ni > NI_SIMD_AS_HWINTRINSIC_START) && (ni < NI_SIMD_AS_HWINTRINSIC_END)) + { + // These intrinsics aren't defined recursively and so they will never be mustExpand + // Instead, they provide software fallbacks that will be executed instead. + + assert(!mustExpand); + return impSimdAsHWIntrinsic(ni, clsHnd, method, sig, newobjThis); + } +#endif // FEATURE_HW_INTRINSICS + + if (!isIntrinsic) + { + // Outside the cases above, there are many intrinsics which apply to only a + // subset of overload and where simply matching by name may cause downstream + // asserts or other failures. Math.Min is one example, where it only applies + // to the floating-point overloads. + return nullptr; + } + + *pIntrinsicName = ni; + + if (ni == NI_System_StubHelpers_GetStubContext) + { + // must be done regardless of DbgCode and MinOpts + return gtNewLclvNode(lvaStubArgumentVar, TYP_I_IMPL); + } + + if (ni == NI_System_StubHelpers_NextCallReturnAddress) + { + // For now we just avoid inlining anything into these methods since + // this intrinsic is only rarely used. We could do this better if we + // wanted to by trying to match which call is the one we need to get + // the return address of. + info.compHasNextCallRetAddr = true; + return new (this, GT_LABEL) GenTree(GT_LABEL, TYP_I_IMPL); + } + + switch (ni) + { + // CreateSpan must be expanded for NativeAOT + case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan: + case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray: + mustExpand |= IsTargetAbi(CORINFO_NATIVEAOT_ABI); + break; + + case NI_Internal_Runtime_MethodTable_Of: + case NI_System_Activator_AllocatorOf: + case NI_System_Activator_DefaultConstructorOf: + case NI_System_EETypePtr_EETypePtrOf: + mustExpand = true; + break; + + default: + break; + } + + GenTree* retNode = nullptr; + + // Under debug and minopts, only expand what is required. + // NextCallReturnAddress intrinsic returns the return address of the next call. + // If that call is an intrinsic and is expanded, codegen for NextCallReturnAddress will fail. + // To avoid that we conservatively expand only required intrinsics in methods that call + // the NextCallReturnAddress intrinsic. + if (!mustExpand && (opts.OptimizationDisabled() || info.compHasNextCallRetAddr)) + { + *pIntrinsicName = NI_Illegal; + return retNode; + } + + CorInfoType callJitType = sig->retType; + var_types callType = JITtype2varType(callJitType); + + /* First do the intrinsics which are always smaller than a call */ + + if (ni != NI_Illegal) + { + assert(retNode == nullptr); + switch (ni) + { + case NI_Array_Address: + case NI_Array_Get: + case NI_Array_Set: + retNode = impArrayAccessIntrinsic(clsHnd, sig, memberRef, readonlyCall, ni); + break; + + case NI_System_String_Equals: + { + retNode = impStringEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags); + break; + } + + case NI_System_MemoryExtensions_Equals: + case NI_System_MemoryExtensions_SequenceEqual: + { + retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags); + break; + } + + case NI_System_String_StartsWith: + { + retNode = impStringEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags); + break; + } + + case NI_System_MemoryExtensions_StartsWith: + { + retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags); + break; + } + + case NI_System_MemoryExtensions_AsSpan: + case NI_System_String_op_Implicit: + { + assert(sig->numArgs == 1); + isSpecial = impStackTop().val->OperIs(GT_CNS_STR); + break; + } + + case NI_System_String_get_Chars: + { + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + GenTree* addr = gtNewIndexAddr(op1, op2, TYP_USHORT, NO_CLASS_HANDLE, OFFSETOF__CORINFO_String__chars, + OFFSETOF__CORINFO_String__stringLen); + retNode = gtNewIndexIndir(addr->AsIndexAddr()); + break; + } + + case NI_System_String_get_Length: + { + GenTree* op1 = impPopStack().val; + if (op1->OperIs(GT_CNS_STR)) + { + // Optimize `ldstr + String::get_Length()` to CNS_INT + // e.g. "Hello".Length => 5 + GenTreeIntCon* iconNode = gtNewStringLiteralLength(op1->AsStrCon()); + if (iconNode != nullptr) + { + retNode = iconNode; + break; + } + } + GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, op1, OFFSETOF__CORINFO_String__stringLen, compCurBB); + op1 = arrLen; + + // Getting the length of a null string should throw + op1->gtFlags |= GTF_EXCEPT; + + retNode = op1; + break; + } + + case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan: + { + retNode = impCreateSpanIntrinsic(sig); + break; + } + + case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray: + { + retNode = impInitializeArrayIntrinsic(sig); + break; + } + + case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant: + { + GenTree* op1 = impPopStack().val; + if (op1->OperIsConst()) + { + // op1 is a known constant, replace with 'true'. + retNode = gtNewIconNode(1); + JITDUMP("\nExpanding RuntimeHelpers.IsKnownConstant to true early\n"); + // We can also consider FTN_ADDR and typeof(T) here + } + else + { + // op1 is not a known constant, we'll do the expansion in morph + retNode = new (this, GT_INTRINSIC) GenTreeIntrinsic(TYP_INT, op1, ni, method); + JITDUMP("\nConverting RuntimeHelpers.IsKnownConstant to:\n"); + DISPTREE(retNode); + } + break; + } + + case NI_Internal_Runtime_MethodTable_Of: + case NI_System_Activator_AllocatorOf: + case NI_System_Activator_DefaultConstructorOf: + case NI_System_EETypePtr_EETypePtrOf: + { + assert(IsTargetAbi(CORINFO_NATIVEAOT_ABI)); // Only NativeAOT supports it. + CORINFO_RESOLVED_TOKEN resolvedToken; + resolvedToken.tokenContext = impTokenLookupContextHandle; + resolvedToken.tokenScope = info.compScopeHnd; + resolvedToken.token = memberRef; + resolvedToken.tokenType = CORINFO_TOKENKIND_Method; + + CORINFO_GENERICHANDLE_RESULT embedInfo; + info.compCompHnd->expandRawHandleIntrinsic(&resolvedToken, &embedInfo); + + GenTree* rawHandle = impLookupToTree(&resolvedToken, &embedInfo.lookup, gtTokenToIconFlags(memberRef), + embedInfo.compileTimeHandle); + if (rawHandle == nullptr) + { + return nullptr; + } + + noway_assert(genTypeSize(rawHandle->TypeGet()) == genTypeSize(TYP_I_IMPL)); + + unsigned rawHandleSlot = lvaGrabTemp(true DEBUGARG("rawHandle")); + impAssignTempGen(rawHandleSlot, rawHandle, clsHnd, CHECK_SPILL_NONE); + + GenTree* lclVar = gtNewLclvNode(rawHandleSlot, TYP_I_IMPL); + GenTree* lclVarAddr = gtNewOperNode(GT_ADDR, TYP_I_IMPL, lclVar); + var_types resultType = JITtype2varType(sig->retType); + if (resultType == TYP_STRUCT) + { + retNode = gtNewObjNode(sig->retTypeClass, lclVarAddr); + } + else + { + retNode = gtNewIndir(resultType, lclVarAddr); + } + break; + } + + case NI_System_Span_get_Item: + case NI_System_ReadOnlySpan_get_Item: + { + // Have index, stack pointer-to Span s on the stack. Expand to: + // + // For Span + // Comma + // BoundsCheck(index, s->_length) + // s->_reference + index * sizeof(T) + // + // For ReadOnlySpan -- same expansion, as it now returns a readonly ref + // + // Signature should show one class type parameter, which + // we need to examine. + assert(sig->sigInst.classInstCount == 1); + assert(sig->numArgs == 1); + CORINFO_CLASS_HANDLE spanElemHnd = sig->sigInst.classInst[0]; + const unsigned elemSize = info.compCompHnd->getClassSize(spanElemHnd); + assert(elemSize > 0); + + const bool isReadOnly = (ni == NI_System_ReadOnlySpan_get_Item); + + JITDUMP("\nimpIntrinsic: Expanding %sSpan.get_Item, T=%s, sizeof(T)=%u\n", + isReadOnly ? "ReadOnly" : "", eeGetClassName(spanElemHnd), elemSize); + + GenTree* index = impPopStack().val; + GenTree* ptrToSpan = impPopStack().val; + GenTree* indexClone = nullptr; + GenTree* ptrToSpanClone = nullptr; + assert(genActualType(index) == TYP_INT); + assert(ptrToSpan->TypeGet() == TYP_BYREF); + +#if defined(DEBUG) + if (verbose) + { + printf("with ptr-to-span\n"); + gtDispTree(ptrToSpan); + printf("and index\n"); + gtDispTree(index); + } +#endif // defined(DEBUG) + + // We need to use both index and ptr-to-span twice, so clone or spill. + index = impCloneExpr(index, &indexClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL, + nullptr DEBUGARG("Span.get_Item index")); + ptrToSpan = impCloneExpr(ptrToSpan, &ptrToSpanClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL, + nullptr DEBUGARG("Span.get_Item ptrToSpan")); + + // Bounds check + CORINFO_FIELD_HANDLE lengthHnd = info.compCompHnd->getFieldInClass(clsHnd, 1); + const unsigned lengthOffset = info.compCompHnd->getFieldOffset(lengthHnd); + GenTree* length = gtNewFieldRef(TYP_INT, lengthHnd, ptrToSpan, lengthOffset); + GenTree* boundsCheck = new (this, GT_BOUNDS_CHECK) GenTreeBoundsChk(index, length, SCK_RNGCHK_FAIL); + + // Element access + index = indexClone; + +#ifdef TARGET_64BIT + if (index->OperGet() == GT_CNS_INT) + { + index->gtType = TYP_I_IMPL; + } + else + { + index = gtNewCastNode(TYP_I_IMPL, index, true, TYP_I_IMPL); + } +#endif + + if (elemSize != 1) + { + GenTree* sizeofNode = gtNewIconNode(static_cast(elemSize), TYP_I_IMPL); + index = gtNewOperNode(GT_MUL, TYP_I_IMPL, index, sizeofNode); + } + + CORINFO_FIELD_HANDLE ptrHnd = info.compCompHnd->getFieldInClass(clsHnd, 0); + const unsigned ptrOffset = info.compCompHnd->getFieldOffset(ptrHnd); + GenTree* data = gtNewFieldRef(TYP_BYREF, ptrHnd, ptrToSpanClone, ptrOffset); + GenTree* result = gtNewOperNode(GT_ADD, TYP_BYREF, data, index); + + // Prepare result + var_types resultType = JITtype2varType(sig->retType); + assert(resultType == result->TypeGet()); + retNode = gtNewOperNode(GT_COMMA, resultType, boundsCheck, result); + + break; + } + + case NI_System_RuntimeTypeHandle_GetValueInternal: + { + GenTree* op1 = impStackTop(0).val; + if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) && + gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall())) + { + // Old tree + // Helper-RuntimeTypeHandle -> TreeToGetNativeTypeHandle + // + // New tree + // TreeToGetNativeTypeHandle + + // Remove call to helper and return the native TypeHandle pointer that was the parameter + // to that helper. + + op1 = impPopStack().val; + + // Get native TypeHandle argument to old helper + assert(op1->AsCall()->gtArgs.CountArgs() == 1); + op1 = op1->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); + retNode = op1; + } + // Call the regular function. + break; + } + + case NI_System_Type_GetTypeFromHandle: + { + GenTree* op1 = impStackTop(0).val; + CorInfoHelpFunc typeHandleHelper; + if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) && + gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall(), &typeHandleHelper)) + { + op1 = impPopStack().val; + // Replace helper with a more specialized helper that returns RuntimeType + if (typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE) + { + typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE; + } + else + { + assert(typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL); + typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL; + } + assert(op1->AsCall()->gtArgs.CountArgs() == 1); + op1 = gtNewHelperCallNode(typeHandleHelper, TYP_REF, + op1->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()); + op1->gtType = TYP_REF; + retNode = op1; + } + break; + } + + case NI_System_Type_op_Equality: + case NI_System_Type_op_Inequality: + { + JITDUMP("Importing Type.op_*Equality intrinsic\n"); + GenTree* op1 = impStackTop(1).val; + GenTree* op2 = impStackTop(0).val; + GenTree* optTree = gtFoldTypeEqualityCall(ni == NI_System_Type_op_Equality, op1, op2); + if (optTree != nullptr) + { + // Success, clean up the evaluation stack. + impPopStack(); + impPopStack(); + + // See if we can optimize even further, to a handle compare. + optTree = gtFoldTypeCompare(optTree); + + // See if we can now fold a handle compare to a constant. + optTree = gtFoldExpr(optTree); + + retNode = optTree; + } + else + { + // Retry optimizing these later + isSpecial = true; + } + break; + } + + case NI_System_Enum_HasFlag: + { + GenTree* thisOp = impStackTop(1).val; + GenTree* flagOp = impStackTop(0).val; + GenTree* optTree = gtOptimizeEnumHasFlag(thisOp, flagOp); + + if (optTree != nullptr) + { + // Optimization successful. Pop the stack for real. + impPopStack(); + impPopStack(); + retNode = optTree; + } + else + { + // Retry optimizing this during morph. + isSpecial = true; + } + + break; + } + + case NI_System_Type_IsAssignableFrom: + { + GenTree* typeTo = impStackTop(1).val; + GenTree* typeFrom = impStackTop(0).val; + + retNode = impTypeIsAssignable(typeTo, typeFrom); + break; + } + + case NI_System_Type_IsAssignableTo: + { + GenTree* typeTo = impStackTop(0).val; + GenTree* typeFrom = impStackTop(1).val; + + retNode = impTypeIsAssignable(typeTo, typeFrom); + break; + } + + case NI_System_Type_get_IsValueType: + case NI_System_Type_get_IsByRefLike: + { + // Optimize + // + // call Type.GetTypeFromHandle (which is replaced with CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE) + // call Type.IsXXX + // + // to `true` or `false` + // e.g., `typeof(int).IsValueType` => `true` + // e.g., `typeof(Span).IsByRefLike` => `true` + if (impStackTop().val->IsCall()) + { + GenTreeCall* call = impStackTop().val->AsCall(); + if (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE)) + { + assert(call->gtArgs.CountArgs() == 1); + CORINFO_CLASS_HANDLE hClass = + gtGetHelperArgClassHandle(call->gtArgs.GetArgByIndex(0)->GetEarlyNode()); + if (hClass != NO_CLASS_HANDLE) + { + switch (ni) + { + case NI_System_Type_get_IsValueType: + retNode = gtNewIconNode(eeIsValueClass(hClass) ? 1 : 0); + break; + case NI_System_Type_get_IsByRefLike: + retNode = gtNewIconNode( + (info.compCompHnd->getClassAttribs(hClass) & CORINFO_FLG_BYREF_LIKE) ? 1 : 0); + break; + default: + NO_WAY("Intrinsic not supported in this path."); + } + impPopStack(); // drop CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE call + } + } + } + break; + } + + case NI_System_Threading_Thread_get_ManagedThreadId: + { + if (impStackTop().val->OperIs(GT_RET_EXPR)) + { + GenTreeCall* call = impStackTop().val->AsRetExpr()->gtInlineCandidate->AsCall(); + if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) + { + if (lookupNamedIntrinsic(call->gtCallMethHnd) == NI_System_Threading_Thread_get_CurrentThread) + { + // drop get_CurrentThread() call + impPopStack(); + call->ReplaceWith(gtNewNothingNode(), this); + retNode = gtNewHelperCallNode(CORINFO_HELP_GETCURRENTMANAGEDTHREADID, TYP_INT); + } + } + } + break; + } + +#ifdef TARGET_ARM64 + // Intrinsify Interlocked.Or and Interlocked.And only for arm64-v8.1 (and newer) + // TODO-CQ: Implement for XArch (https://github.com/dotnet/runtime/issues/32239). + case NI_System_Threading_Interlocked_Or: + case NI_System_Threading_Interlocked_And: + { + if (compOpportunisticallyDependsOn(InstructionSet_Atomics)) + { + assert(sig->numArgs == 2); + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + genTreeOps op = (ni == NI_System_Threading_Interlocked_Or) ? GT_XORR : GT_XAND; + retNode = gtNewOperNode(op, genActualType(callType), op1, op2); + retNode->gtFlags |= GTF_GLOB_REF | GTF_ASG; + } + break; + } +#endif // TARGET_ARM64 + +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) + // TODO-ARM-CQ: reenable treating InterlockedCmpXchg32 operation as intrinsic + case NI_System_Threading_Interlocked_CompareExchange: + { + var_types retType = JITtype2varType(sig->retType); + if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4)) + { + break; + } + if ((retType != TYP_INT) && (retType != TYP_LONG)) + { + break; + } + + assert(callType != TYP_STRUCT); + assert(sig->numArgs == 3); + + GenTree* op3 = impPopStack().val; // comparand + GenTree* op2 = impPopStack().val; // value + GenTree* op1 = impPopStack().val; // location + + GenTree* node = new (this, GT_CMPXCHG) GenTreeCmpXchg(genActualType(callType), op1, op2, op3); + + node->AsCmpXchg()->gtOpLocation->gtFlags |= GTF_DONT_CSE; + retNode = node; + break; + } + + case NI_System_Threading_Interlocked_Exchange: + case NI_System_Threading_Interlocked_ExchangeAdd: + { + assert(callType != TYP_STRUCT); + assert(sig->numArgs == 2); + + var_types retType = JITtype2varType(sig->retType); + if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4)) + { + break; + } + if ((retType != TYP_INT) && (retType != TYP_LONG)) + { + break; + } + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + // This creates: + // val + // XAdd + // addr + // field (for example) + // + // In the case where the first argument is the address of a local, we might + // want to make this *not* make the var address-taken -- but atomic instructions + // on a local are probably pretty useless anyway, so we probably don't care. + + op1 = gtNewOperNode(ni == NI_System_Threading_Interlocked_ExchangeAdd ? GT_XADD : GT_XCHG, + genActualType(callType), op1, op2); + op1->gtFlags |= GTF_GLOB_REF | GTF_ASG; + retNode = op1; + break; + } +#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64) + + case NI_System_Threading_Interlocked_MemoryBarrier: + case NI_System_Threading_Interlocked_ReadMemoryBarrier: + { + assert(sig->numArgs == 0); + + GenTree* op1 = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID); + op1->gtFlags |= GTF_GLOB_REF | GTF_ASG; + + // On XARCH `NI_System_Threading_Interlocked_ReadMemoryBarrier` fences need not be emitted. + // However, we still need to capture the effect on reordering. + if (ni == NI_System_Threading_Interlocked_ReadMemoryBarrier) + { + op1->gtFlags |= GTF_MEMORYBARRIER_LOAD; + } + + retNode = op1; + break; + } + +#ifdef FEATURE_HW_INTRINSICS + case NI_System_Math_FusedMultiplyAdd: + { +#ifdef TARGET_XARCH + if (compExactlyDependsOn(InstructionSet_FMA)) + { + assert(varTypeIsFloating(callType)); + + // We are constructing a chain of intrinsics similar to: + // return FMA.MultiplyAddScalar( + // Vector128.CreateScalarUnsafe(x), + // Vector128.CreateScalarUnsafe(y), + // Vector128.CreateScalarUnsafe(z) + // ).ToScalar(); + + GenTree* op3 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, + NI_Vector128_CreateScalarUnsafe, callJitType, 16); + GenTree* op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, + NI_Vector128_CreateScalarUnsafe, callJitType, 16); + GenTree* op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, + NI_Vector128_CreateScalarUnsafe, callJitType, 16); + GenTree* res = + gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_FMA_MultiplyAddScalar, callJitType, 16); + + retNode = gtNewSimdHWIntrinsicNode(callType, res, NI_Vector128_ToScalar, callJitType, 16); + break; + } +#elif defined(TARGET_ARM64) + if (compExactlyDependsOn(InstructionSet_AdvSimd)) + { + assert(varTypeIsFloating(callType)); + + // We are constructing a chain of intrinsics similar to: + // return AdvSimd.FusedMultiplyAddScalar( + // Vector64.Create{ScalarUnsafe}(z), + // Vector64.Create{ScalarUnsafe}(y), + // Vector64.Create{ScalarUnsafe}(x) + // ).ToScalar(); + + NamedIntrinsic createVector64 = + (callType == TYP_DOUBLE) ? NI_Vector64_Create : NI_Vector64_CreateScalarUnsafe; + + constexpr unsigned int simdSize = 8; + + GenTree* op3 = + gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); + GenTree* op2 = + gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); + GenTree* op1 = + gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); + + // Note that AdvSimd.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 + op2 * op3 + // while Math{F}.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 * op2 + op3 + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op3, op2, op1, NI_AdvSimd_FusedMultiplyAddScalar, + callJitType, simdSize); + + retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector64_ToScalar, callJitType, simdSize); + break; + } +#endif + + // TODO-CQ-XArch: Ideally we would create a GT_INTRINSIC node for fma, however, that currently + // requires more extensive changes to valuenum to support methods with 3 operands + + // We want to generate a GT_INTRINSIC node in the case the call can't be treated as + // a target intrinsic so that we can still benefit from CSE and constant folding. + + break; + } +#endif // FEATURE_HW_INTRINSICS + + case NI_System_Math_Abs: + case NI_System_Math_Acos: + case NI_System_Math_Acosh: + case NI_System_Math_Asin: + case NI_System_Math_Asinh: + case NI_System_Math_Atan: + case NI_System_Math_Atanh: + case NI_System_Math_Atan2: + case NI_System_Math_Cbrt: + case NI_System_Math_Ceiling: + case NI_System_Math_Cos: + case NI_System_Math_Cosh: + case NI_System_Math_Exp: + case NI_System_Math_Floor: + case NI_System_Math_FMod: + case NI_System_Math_ILogB: + case NI_System_Math_Log: + case NI_System_Math_Log2: + case NI_System_Math_Log10: + { + retNode = impMathIntrinsic(method, sig, callType, ni, tailCall); + break; + } +#if defined(TARGET_ARM64) + // ARM64 has fmax/fmin which are IEEE754:2019 minimum/maximum compatible + // TODO-XARCH-CQ: Enable this for XARCH when one of the arguments is a constant + // so we can then emit maxss/minss and avoid NaN/-0.0 handling + case NI_System_Math_Max: + case NI_System_Math_Min: +#endif + +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) + case NI_System_Math_Max: + case NI_System_Math_Min: + { + assert(varTypeIsFloating(callType)); + assert(sig->numArgs == 2); + + GenTreeDblCon* cnsNode = nullptr; + GenTree* otherNode = nullptr; + + GenTree* op2 = impStackTop().val; + GenTree* op1 = impStackTop(1).val; + + if (op2->IsCnsFltOrDbl()) + { + cnsNode = op2->AsDblCon(); + otherNode = op1; + } + else if (op1->IsCnsFltOrDbl()) + { + cnsNode = op1->AsDblCon(); + otherNode = op2; + } + + if (cnsNode == nullptr) + { + // no constant node, nothing to do + break; + } + + if (otherNode->IsCnsFltOrDbl()) + { + // both are constant, we can fold this operation completely. Pop both peeked values + + if (ni == NI_System_Math_Max) + { + cnsNode->SetDconValue( + FloatingPointUtils::maximum(cnsNode->DconValue(), otherNode->AsDblCon()->DconValue())); + } + else + { + assert(ni == NI_System_Math_Min); + cnsNode->SetDconValue( + FloatingPointUtils::minimum(cnsNode->DconValue(), otherNode->AsDblCon()->DconValue())); + } + + retNode = cnsNode; + + impPopStack(); + impPopStack(); + DEBUG_DESTROY_NODE(otherNode); + + break; + } + + // only one is constant, we can fold in specialized scenarios + + if (cnsNode->IsFloatNaN()) + { + impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("spill side effects before propagating NaN")); + + // maxsd, maxss, minsd, and minss all return op2 if either is NaN + // we require NaN to be propagated so ensure the known NaN is op2 + + impPopStack(); + impPopStack(); + DEBUG_DESTROY_NODE(otherNode); + + retNode = cnsNode; + break; + } + + if (!compOpportunisticallyDependsOn(InstructionSet_SSE2)) + { + break; + } + + if (ni == NI_System_Math_Max) + { + // maxsd, maxss return op2 if both inputs are 0 of either sign + // we require +0 to be greater than -0, so we can't handle if + // the known constant is +0. This is because if the unknown value + // is -0, we'd need the cns to be op2. But if the unknown value + // is NaN, we'd need the cns to be op1 instead. + + if (cnsNode->IsFloatPositiveZero()) + { + break; + } + + // Given the checks, op1 can safely be the cns and op2 the other node + + ni = (callType == TYP_DOUBLE) ? NI_SSE2_Max : NI_SSE_Max; + + // one is constant and we know its something we can handle, so pop both peeked values + + op1 = cnsNode; + op2 = otherNode; + } + else + { + assert(ni == NI_System_Math_Min); + + // minsd, minss return op2 if both inputs are 0 of either sign + // we require -0 to be lesser than +0, so we can't handle if + // the known constant is -0. This is because if the unknown value + // is +0, we'd need the cns to be op2. But if the unknown value + // is NaN, we'd need the cns to be op1 instead. + + if (cnsNode->IsFloatNegativeZero()) + { + break; + } + + // Given the checks, op1 can safely be the cns and op2 the other node + + ni = (callType == TYP_DOUBLE) ? NI_SSE2_Min : NI_SSE_Min; + + // one is constant and we know its something we can handle, so pop both peeked values + + op1 = cnsNode; + op2 = otherNode; + } + + assert(op1->IsCnsFltOrDbl() && !op2->IsCnsFltOrDbl()); + + impPopStack(); + impPopStack(); + + GenTreeVecCon* vecCon = gtNewVconNode(TYP_SIMD16); + + if (callJitType == CORINFO_TYPE_FLOAT) + { + vecCon->gtSimd16Val.f32[0] = (float)op1->AsDblCon()->DconValue(); + } + else + { + vecCon->gtSimd16Val.f64[0] = op1->AsDblCon()->DconValue(); + } + + op1 = vecCon; + op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, NI_Vector128_CreateScalarUnsafe, callJitType, 16); + + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, ni, callJitType, 16); + retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector128_ToScalar, callJitType, 16); + + break; + } +#endif + + case NI_System_Math_Pow: + case NI_System_Math_Round: + case NI_System_Math_Sin: + case NI_System_Math_Sinh: + case NI_System_Math_Sqrt: + case NI_System_Math_Tan: + case NI_System_Math_Tanh: + case NI_System_Math_Truncate: + { + retNode = impMathIntrinsic(method, sig, callType, ni, tailCall); + break; + } + + case NI_System_Array_Clone: + case NI_System_Collections_Generic_Comparer_get_Default: + case NI_System_Collections_Generic_EqualityComparer_get_Default: + case NI_System_Object_MemberwiseClone: + case NI_System_Threading_Thread_get_CurrentThread: + { + // Flag for later handling. + isSpecial = true; + break; + } + + case NI_System_Object_GetType: + { + JITDUMP("\n impIntrinsic: call to Object.GetType\n"); + GenTree* op1 = impStackTop(0).val; + + // If we're calling GetType on a boxed value, just get the type directly. + if (op1->IsBoxedValue()) + { + JITDUMP("Attempting to optimize box(...).getType() to direct type construction\n"); + + // Try and clean up the box. Obtain the handle we + // were going to pass to the newobj. + GenTree* boxTypeHandle = gtTryRemoveBoxUpstreamEffects(op1, BR_REMOVE_AND_NARROW_WANT_TYPE_HANDLE); + + if (boxTypeHandle != nullptr) + { + // Note we don't need to play the TYP_STRUCT games here like + // do for LDTOKEN since the return value of this operator is Type, + // not RuntimeTypeHandle. + impPopStack(); + GenTree* runtimeType = + gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, boxTypeHandle); + retNode = runtimeType; + } + } + + // If we have a constrained callvirt with a "box this" transform + // we know we have a value class and hence an exact type. + // + // If so, instead of boxing and then extracting the type, just + // construct the type directly. + if ((retNode == nullptr) && (pConstrainedResolvedToken != nullptr) && + (constraintCallThisTransform == CORINFO_BOX_THIS)) + { + // Ensure this is one of the simple box cases (in particular, rule out nullables). + const CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pConstrainedResolvedToken->hClass); + const bool isSafeToOptimize = (boxHelper == CORINFO_HELP_BOX); + + if (isSafeToOptimize) + { + JITDUMP("Optimizing constrained box-this obj.getType() to direct type construction\n"); + impPopStack(); + GenTree* typeHandleOp = + impTokenToHandle(pConstrainedResolvedToken, nullptr, true /* mustRestoreHandle */); + if (typeHandleOp == nullptr) + { + assert(compDonotInline()); + return nullptr; + } + GenTree* runtimeType = + gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, typeHandleOp); + retNode = runtimeType; + } + } + +#ifdef DEBUG + if (retNode != nullptr) + { + JITDUMP("Optimized result for call to GetType is\n"); + if (verbose) + { + gtDispTree(retNode); + } + } +#endif + + // Else expand as an intrinsic, unless the call is constrained, + // in which case we defer expansion to allow impImportCall do the + // special constraint processing. + if ((retNode == nullptr) && (pConstrainedResolvedToken == nullptr)) + { + JITDUMP("Expanding as special intrinsic\n"); + impPopStack(); + op1 = new (this, GT_INTRINSIC) GenTreeIntrinsic(genActualType(callType), op1, ni, method); + + // Set the CALL flag to indicate that the operator is implemented by a call. + // Set also the EXCEPTION flag because the native implementation of + // NI_System_Object_GetType intrinsic can throw NullReferenceException. + op1->gtFlags |= (GTF_CALL | GTF_EXCEPT); + retNode = op1; + // Might be further optimizable, so arrange to leave a mark behind + isSpecial = true; + } + + if (retNode == nullptr) + { + JITDUMP("Leaving as normal call\n"); + // Might be further optimizable, so arrange to leave a mark behind + isSpecial = true; + } + + break; + } + + case NI_System_Array_GetLength: + case NI_System_Array_GetLowerBound: + case NI_System_Array_GetUpperBound: + { + // System.Array.GetLength(Int32) method: + // public int GetLength(int dimension) + // System.Array.GetLowerBound(Int32) method: + // public int GetLowerBound(int dimension) + // System.Array.GetUpperBound(Int32) method: + // public int GetUpperBound(int dimension) + // + // Only implement these as intrinsics for multi-dimensional arrays. + // Only handle constant dimension arguments. + + GenTree* gtDim = impStackTop().val; + GenTree* gtArr = impStackTop(1).val; + + if (gtDim->IsIntegralConst()) + { + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE arrCls = gtGetClassHandle(gtArr, &isExact, &isNonNull); + if (arrCls != NO_CLASS_HANDLE) + { + unsigned rank = info.compCompHnd->getArrayRank(arrCls); + if ((rank > 1) && !info.compCompHnd->isSDArray(arrCls)) + { + // `rank` is guaranteed to be <=32 (see MAX_RANK in vm\array.h). Any constant argument + // is `int` sized. + INT64 dimValue = gtDim->AsIntConCommon()->IntegralValue(); + assert((unsigned int)dimValue == dimValue); + unsigned dim = (unsigned int)dimValue; + if (dim < rank) + { + // This is now known to be a multi-dimension array with a constant dimension + // that is in range; we can expand it as an intrinsic. + + impPopStack().val; // Pop the dim and array object; we already have a pointer to them. + impPopStack().val; + + // Make sure there are no global effects in the array (such as it being a function + // call), so we can mark the generated indirection with GTF_IND_INVARIANT. In the + // GetUpperBound case we need the cloned object, since we refer to the array + // object twice. In the other cases, we don't need to clone. + GenTree* gtArrClone = nullptr; + if (((gtArr->gtFlags & GTF_GLOB_EFFECT) != 0) || (ni == NI_System_Array_GetUpperBound)) + { + gtArr = impCloneExpr(gtArr, >ArrClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL, + nullptr DEBUGARG("MD intrinsics array")); + } + + switch (ni) + { + case NI_System_Array_GetLength: + { + // Generate *(array + offset-to-length-array + sizeof(int) * dim) + unsigned offs = eeGetMDArrayLengthOffset(rank, dim); + GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); + GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); + retNode = gtNewIndir(TYP_INT, gtAddr); + retNode->gtFlags |= GTF_IND_INVARIANT; + break; + } + case NI_System_Array_GetLowerBound: + { + // Generate *(array + offset-to-bounds-array + sizeof(int) * dim) + unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim); + GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); + GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); + retNode = gtNewIndir(TYP_INT, gtAddr); + retNode->gtFlags |= GTF_IND_INVARIANT; + break; + } + case NI_System_Array_GetUpperBound: + { + assert(gtArrClone != nullptr); + + // Generate: + // *(array + offset-to-length-array + sizeof(int) * dim) + + // *(array + offset-to-bounds-array + sizeof(int) * dim) - 1 + unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim); + GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); + GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); + GenTree* gtLowerBound = gtNewIndir(TYP_INT, gtAddr); + gtLowerBound->gtFlags |= GTF_IND_INVARIANT; + + offs = eeGetMDArrayLengthOffset(rank, dim); + gtOffs = gtNewIconNode(offs, TYP_I_IMPL); + gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArrClone, gtOffs); + GenTree* gtLength = gtNewIndir(TYP_INT, gtAddr); + gtLength->gtFlags |= GTF_IND_INVARIANT; + + GenTree* gtSum = gtNewOperNode(GT_ADD, TYP_INT, gtLowerBound, gtLength); + GenTree* gtOne = gtNewIconNode(1, TYP_INT); + retNode = gtNewOperNode(GT_SUB, TYP_INT, gtSum, gtOne); + break; + } + default: + unreached(); + } + } + } + } + } + break; + } + + case NI_System_Buffers_Binary_BinaryPrimitives_ReverseEndianness: + { + assert(sig->numArgs == 1); + + // We expect the return type of the ReverseEndianness routine to match the type of the + // one and only argument to the method. We use a special instruction for 16-bit + // BSWAPs since on x86 processors this is implemented as ROR <16-bit reg>, 8. Additionally, + // we only emit 64-bit BSWAP instructions on 64-bit archs; if we're asked to perform a + // 64-bit byte swap on a 32-bit arch, we'll fall to the default case in the switch block below. + + switch (sig->retType) + { + case CorInfoType::CORINFO_TYPE_SHORT: + case CorInfoType::CORINFO_TYPE_USHORT: + retNode = gtNewCastNode(TYP_INT, gtNewOperNode(GT_BSWAP16, TYP_INT, impPopStack().val), false, + callType); + break; + + case CorInfoType::CORINFO_TYPE_INT: + case CorInfoType::CORINFO_TYPE_UINT: +#ifdef TARGET_64BIT + case CorInfoType::CORINFO_TYPE_LONG: + case CorInfoType::CORINFO_TYPE_ULONG: +#endif // TARGET_64BIT + retNode = gtNewOperNode(GT_BSWAP, callType, impPopStack().val); + break; + + default: + // This default case gets hit on 32-bit archs when a call to a 64-bit overload + // of ReverseEndianness is encountered. In that case we'll let JIT treat this as a standard + // method call, where the implementation decomposes the operation into two 32-bit + // bswap routines. If the input to the 64-bit function is a constant, then we rely + // on inlining + constant folding of 32-bit bswaps to effectively constant fold + // the 64-bit call site. + break; + } + + break; + } + + // Fold PopCount for constant input + case NI_System_Numerics_BitOperations_PopCount: + { + assert(sig->numArgs == 1); + if (impStackTop().val->IsIntegralConst()) + { + typeInfo argType = verParseArgSigToTypeInfo(sig, sig->args).NormaliseForStack(); + INT64 cns = impPopStack().val->AsIntConCommon()->IntegralValue(); + if (argType.IsType(TI_LONG)) + { + retNode = gtNewIconNode(genCountBits(cns), callType); + } + else + { + assert(argType.IsType(TI_INT)); + retNode = gtNewIconNode(genCountBits(static_cast(cns)), callType); + } + } + break; + } + + case NI_System_GC_KeepAlive: + { + retNode = impKeepAliveIntrinsic(impPopStack().val); + break; + } + + case NI_System_BitConverter_DoubleToInt64Bits: + { + GenTree* op1 = impStackTop().val; + assert(varTypeIsFloating(op1)); + + if (op1->IsCnsFltOrDbl()) + { + impPopStack(); + + double f64Cns = op1->AsDblCon()->DconValue(); + retNode = gtNewLconNode(*reinterpret_cast(&f64Cns)); + } +#if TARGET_64BIT + else + { + // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work + impPopStack(); + + if (op1->TypeGet() != TYP_DOUBLE) + { + op1 = gtNewCastNode(TYP_DOUBLE, op1, false, TYP_DOUBLE); + } + retNode = gtNewBitCastNode(TYP_LONG, op1); + } +#endif + break; + } + + case NI_System_BitConverter_Int32BitsToSingle: + { + GenTree* op1 = impPopStack().val; + assert(varTypeIsInt(op1)); + + if (op1->IsIntegralConst()) + { + int32_t i32Cns = (int32_t)op1->AsIntConCommon()->IconValue(); + retNode = gtNewDconNode(*reinterpret_cast(&i32Cns), TYP_FLOAT); + } + else + { + retNode = gtNewBitCastNode(TYP_FLOAT, op1); + } + break; + } + + case NI_System_BitConverter_Int64BitsToDouble: + { + GenTree* op1 = impStackTop().val; + assert(varTypeIsLong(op1)); + + if (op1->IsIntegralConst()) + { + impPopStack(); + + int64_t i64Cns = op1->AsIntConCommon()->LngValue(); + retNode = gtNewDconNode(*reinterpret_cast(&i64Cns)); + } +#if TARGET_64BIT + else + { + // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work + impPopStack(); + + retNode = gtNewBitCastNode(TYP_DOUBLE, op1); + } +#endif + break; + } + + case NI_System_BitConverter_SingleToInt32Bits: + { + GenTree* op1 = impPopStack().val; + assert(varTypeIsFloating(op1)); + + if (op1->IsCnsFltOrDbl()) + { + float f32Cns = (float)op1->AsDblCon()->DconValue(); + retNode = gtNewIconNode(*reinterpret_cast(&f32Cns)); + } + else + { + if (op1->TypeGet() != TYP_FLOAT) + { + op1 = gtNewCastNode(TYP_FLOAT, op1, false, TYP_FLOAT); + } + retNode = gtNewBitCastNode(TYP_INT, op1); + } + break; + } + + default: + break; + } + } + + if (mustExpand && (retNode == nullptr)) + { + assert(!"Unhandled must expand intrinsic, throwing PlatformNotSupportedException"); + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); + } + + // Optionally report if this intrinsic is special + // (that is, potentially re-optimizable during morph). + if (isSpecialIntrinsic != nullptr) + { + *isSpecialIntrinsic = isSpecial; + } + + return retNode; +} + +GenTree* Compiler::impSRCSUnsafeIntrinsic(NamedIntrinsic intrinsic, + CORINFO_CLASS_HANDLE clsHnd, + CORINFO_METHOD_HANDLE method, + CORINFO_SIG_INFO* sig) +{ + // NextCallRetAddr requires a CALL, so return nullptr. + if (info.compHasNextCallRetAddr) + { + return nullptr; + } + + assert(sig->sigInst.classInstCount == 0); + + switch (intrinsic) + { + case NI_SRCS_UNSAFE_Add: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // sizeof !!T + // conv.i + // mul + // add + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); + + op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL); + + unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); + + if (classSize != 1) + { + GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL); + op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size); + } + + var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2); + return gtNewOperNode(GT_ADD, type, op1, op2); + } + + case NI_SRCS_UNSAFE_AddByteOffset: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // add + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); + + var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2); + return gtNewOperNode(GT_ADD, type, op1, op2); + } + + case NI_SRCS_UNSAFE_AreSame: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // ceq + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, op2); + return gtFoldExpr(tmp); + } + + case NI_SRCS_UNSAFE_As: + { + assert((sig->sigInst.methInstCount == 1) || (sig->sigInst.methInstCount == 2)); + + // ldarg.0 + // ret + + return impPopStack().val; + } + + case NI_SRCS_UNSAFE_AsPointer: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // conv.u + // ret + + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1); + + return gtNewCastNode(TYP_I_IMPL, op1, /* uns */ false, TYP_I_IMPL); + } + + case NI_SRCS_UNSAFE_AsRef: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ret + + return impPopStack().val; + } + + case NI_SRCS_UNSAFE_ByteOffset: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.1 + // ldarg.0 + // sub + // ret + + impSpillSideEffect(true, verCurrentState.esStackDepth - + 2 DEBUGARG("Spilling op1 side effects for Unsafe.ByteOffset")); + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); + + var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op2, &op1); + return gtNewOperNode(GT_SUB, type, op2, op1); + } + + case NI_SRCS_UNSAFE_Copy: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // ldobj !!T + // stobj !!T + // ret + + return nullptr; + } + + case NI_SRCS_UNSAFE_CopyBlock: + { + assert(sig->sigInst.methInstCount == 0); + + // ldarg.0 + // ldarg.1 + // ldarg.2 + // cpblk + // ret + + return nullptr; + } + + case NI_SRCS_UNSAFE_CopyBlockUnaligned: + { + assert(sig->sigInst.methInstCount == 0); + + // ldarg.0 + // ldarg.1 + // ldarg.2 + // unaligned. 0x1 + // cpblk + // ret + + return nullptr; + } + + case NI_SRCS_UNSAFE_InitBlock: + { + assert(sig->sigInst.methInstCount == 0); + + // ldarg.0 + // ldarg.1 + // ldarg.2 + // initblk + // ret + + return nullptr; + } + + case NI_SRCS_UNSAFE_InitBlockUnaligned: + { + assert(sig->sigInst.methInstCount == 0); + + // ldarg.0 + // ldarg.1 + // ldarg.2 + // unaligned. 0x1 + // initblk + // ret + + return nullptr; + } + + case NI_SRCS_UNSAFE_IsAddressGreaterThan: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // cgt.un + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + GenTree* tmp = gtNewOperNode(GT_GT, TYP_INT, op1, op2); + tmp->gtFlags |= GTF_UNSIGNED; + return gtFoldExpr(tmp); + } + + case NI_SRCS_UNSAFE_IsAddressLessThan: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // clt.un + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + GenTree* tmp = gtNewOperNode(GT_LT, TYP_INT, op1, op2); + tmp->gtFlags |= GTF_UNSIGNED; + return gtFoldExpr(tmp); + } + + case NI_SRCS_UNSAFE_IsNullRef: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldc.i4.0 + // conv.u + // ceq + // ret + + GenTree* op1 = impPopStack().val; + GenTree* cns = gtNewIconNode(0, TYP_BYREF); + GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, cns); + return gtFoldExpr(tmp); + } + + case NI_SRCS_UNSAFE_NullRef: + { + assert(sig->sigInst.methInstCount == 1); + + // ldc.i4.0 + // conv.u + // ret + + return gtNewIconNode(0, TYP_BYREF); + } + + case NI_SRCS_UNSAFE_Read: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldobj !!T + // ret + + return nullptr; + } + + case NI_SRCS_UNSAFE_ReadUnaligned: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // unaligned. 0x1 + // ldobj !!T + // ret + + return nullptr; + } + + case NI_SRCS_UNSAFE_SizeOf: + { + assert(sig->sigInst.methInstCount == 1); + + // sizeof !!T + // ret + + unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); + return gtNewIconNode(classSize, TYP_INT); + } + + case NI_SRCS_UNSAFE_SkipInit: + { + assert(sig->sigInst.methInstCount == 1); + + // ret + + GenTree* op1 = impPopStack().val; + + if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) + { + return gtUnusedValNode(op1); + } + else + { + return gtNewNothingNode(); + } + } + + case NI_SRCS_UNSAFE_Subtract: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // sizeof !!T + // conv.i + // mul + // sub + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); + + op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL); + + unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); + + if (classSize != 1) + { + GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL); + op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size); + } + + var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2); + return gtNewOperNode(GT_SUB, type, op1, op2); + } + + case NI_SRCS_UNSAFE_SubtractByteOffset: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // sub + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); + + var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2); + return gtNewOperNode(GT_SUB, type, op1, op2); + } + + case NI_SRCS_UNSAFE_Unbox: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // unbox !!T + // ret + + return nullptr; + } + + case NI_SRCS_UNSAFE_Write: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // stobj !!T + // ret + + return nullptr; + } + + case NI_SRCS_UNSAFE_WriteUnaligned: + { + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // unaligned. 0x01 + // stobj !!T + // ret + + return nullptr; + } + + default: + { + unreached(); + } + } +}