return tree;
}
-//------------------------------------------------------------------------
-// impInitializeArrayIntrinsic: Attempts to replace a call to InitializeArray
-// with a GT_COPYBLK node.
-//
-// Arguments:
-// sig - The InitializeArray signature.
-//
-// Return Value:
-// A pointer to the newly created GT_COPYBLK node if the replacement succeeds or
-// nullptr otherwise.
-//
-// Notes:
-// The function recognizes the following IL pattern:
-// ldc <length> or a list of ldc <lower bound>/<length>
-// newarr or newobj
-// dup
-// ldtoken <field handle>
-// call InitializeArray
-// The lower bounds need not be constant except when the array rank is 1.
-// The function recognizes all kinds of arrays thus enabling a small runtime
-// such as NativeAOT to skip providing an implementation for InitializeArray.
-
-GenTree* Compiler::impInitializeArrayIntrinsic(CORINFO_SIG_INFO* sig)
-{
- assert(sig->numArgs == 2);
-
- GenTree* fieldTokenNode = impStackTop(0).val;
- GenTree* arrayLocalNode = impStackTop(1).val;
-
- //
- // Verify that the field token is known and valid. Note that it's also
- // possible for the token to come from reflection, in which case we cannot do
- // the optimization and must therefore revert to calling the helper. You can
- // see an example of this in bvt\DynIL\initarray2.exe (in Main).
- //
-
- // Check to see if the ldtoken helper call is what we see here.
- if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) ||
- (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD)))
- {
- return nullptr;
- }
-
- // Strip helper call away
- fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode();
-
- if (fieldTokenNode->gtOper == GT_IND)
- {
- fieldTokenNode = fieldTokenNode->AsOp()->gtOp1;
- }
-
- // Check for constant
- if (fieldTokenNode->gtOper != GT_CNS_INT)
- {
- return nullptr;
- }
-
- CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle;
- if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr))
- {
- return nullptr;
- }
-
- //
- // We need to get the number of elements in the array and the size of each element.
- // We verify that the newarr statement is exactly what we expect it to be.
- // If it's not then we just return NULL and we don't optimize this call
- //
-
- // It is possible the we don't have any statements in the block yet.
- if (impLastStmt == nullptr)
- {
- return nullptr;
- }
-
- //
- // We start by looking at the last statement, making sure it's an assignment, and
- // that the target of the assignment is the array passed to InitializeArray.
- //
- GenTree* arrayAssignment = impLastStmt->GetRootNode();
- if ((arrayAssignment->gtOper != GT_ASG) || (arrayAssignment->AsOp()->gtOp1->gtOper != GT_LCL_VAR) ||
- (arrayLocalNode->gtOper != GT_LCL_VAR) || (arrayAssignment->AsOp()->gtOp1->AsLclVarCommon()->GetLclNum() !=
- arrayLocalNode->AsLclVarCommon()->GetLclNum()))
- {
- return nullptr;
- }
-
- //
- // Make sure that the object being assigned is a helper call.
- //
-
- GenTree* newArrayCall = arrayAssignment->AsOp()->gtOp2;
- if ((newArrayCall->gtOper != GT_CALL) || (newArrayCall->AsCall()->gtCallType != CT_HELPER))
- {
- return nullptr;
- }
-
- //
- // Verify that it is one of the new array helpers.
- //
-
- bool isMDArray = false;
-
- if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_DIRECT) &&
- newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_OBJ) &&
- newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_VC) &&
- newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_ALIGN8)
-#ifdef FEATURE_READYTORUN
- && newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1)
-#endif
- )
- {
- if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEW_MDARR))
- {
- return nullptr;
- }
-
- isMDArray = true;
- }
-
- CORINFO_CLASS_HANDLE arrayClsHnd = (CORINFO_CLASS_HANDLE)newArrayCall->AsCall()->compileTimeHelperArgumentHandle;
-
- //
- // Make sure we found a compile time handle to the array
- //
-
- if (!arrayClsHnd)
- {
- return nullptr;
- }
-
- unsigned rank = 0;
- S_UINT32 numElements;
-
- if (isMDArray)
- {
- rank = info.compCompHnd->getArrayRank(arrayClsHnd);
-
- if (rank == 0)
- {
- return nullptr;
- }
-
- assert(newArrayCall->AsCall()->gtArgs.CountArgs() == 3);
- GenTree* numArgsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode();
- GenTree* argsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(2)->GetNode();
-
- //
- // The number of arguments should be a constant between 1 and 64. The rank can't be 0
- // so at least one length must be present and the rank can't exceed 32 so there can
- // be at most 64 arguments - 32 lengths and 32 lower bounds.
- //
-
- if ((!numArgsArg->IsCnsIntOrI()) || (numArgsArg->AsIntCon()->IconValue() < 1) ||
- (numArgsArg->AsIntCon()->IconValue() > 64))
- {
- return nullptr;
- }
-
- unsigned numArgs = static_cast<unsigned>(numArgsArg->AsIntCon()->IconValue());
- bool lowerBoundsSpecified;
-
- if (numArgs == rank * 2)
- {
- lowerBoundsSpecified = true;
- }
- else if (numArgs == rank)
- {
- lowerBoundsSpecified = false;
-
- //
- // If the rank is 1 and a lower bound isn't specified then the runtime creates
- // a SDArray. Note that even if a lower bound is specified it can be 0 and then
- // we get a SDArray as well, see the for loop below.
- //
-
- if (rank == 1)
- {
- isMDArray = false;
- }
- }
- else
- {
- return nullptr;
- }
-
- //
- // The rank is known to be at least 1 so we can start with numElements being 1
- // to avoid the need to special case the first dimension.
- //
-
- numElements = S_UINT32(1);
-
- struct Match
- {
- static bool IsArgsFieldInit(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs)
- {
- return (tree->OperGet() == GT_ASG) && IsArgsFieldIndir(tree->gtGetOp1(), index, lvaNewObjArrayArgs) &&
- IsArgsAddr(tree->gtGetOp1()->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs);
- }
-
- static bool IsArgsFieldIndir(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs)
- {
- return (tree->OperGet() == GT_IND) && (tree->gtGetOp1()->OperGet() == GT_ADD) &&
- (tree->gtGetOp1()->gtGetOp2()->IsIntegralConst(sizeof(INT32) * index)) &&
- IsArgsAddr(tree->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs);
- }
-
- static bool IsArgsAddr(GenTree* tree, unsigned lvaNewObjArrayArgs)
- {
- return (tree->OperGet() == GT_ADDR) && (tree->gtGetOp1()->OperGet() == GT_LCL_VAR) &&
- (tree->gtGetOp1()->AsLclVar()->GetLclNum() == lvaNewObjArrayArgs);
- }
-
- static bool IsComma(GenTree* tree)
- {
- return (tree != nullptr) && (tree->OperGet() == GT_COMMA);
- }
- };
-
- unsigned argIndex = 0;
- GenTree* comma;
-
- for (comma = argsArg; Match::IsComma(comma); comma = comma->gtGetOp2())
- {
- if (lowerBoundsSpecified)
- {
- //
- // In general lower bounds can be ignored because they're not needed to
- // calculate the total number of elements. But for single dimensional arrays
- // we need to know if the lower bound is 0 because in this case the runtime
- // creates a SDArray and this affects the way the array data offset is calculated.
- //
-
- if (rank == 1)
- {
- GenTree* lowerBoundAssign = comma->gtGetOp1();
- assert(Match::IsArgsFieldInit(lowerBoundAssign, argIndex, lvaNewObjArrayArgs));
- GenTree* lowerBoundNode = lowerBoundAssign->gtGetOp2();
-
- if (lowerBoundNode->IsIntegralConst(0))
- {
- isMDArray = false;
- }
- }
-
- comma = comma->gtGetOp2();
- argIndex++;
- }
-
- GenTree* lengthNodeAssign = comma->gtGetOp1();
- assert(Match::IsArgsFieldInit(lengthNodeAssign, argIndex, lvaNewObjArrayArgs));
- GenTree* lengthNode = lengthNodeAssign->gtGetOp2();
-
- if (!lengthNode->IsCnsIntOrI())
- {
- return nullptr;
- }
-
- numElements *= S_SIZE_T(lengthNode->AsIntCon()->IconValue());
- argIndex++;
- }
-
- assert((comma != nullptr) && Match::IsArgsAddr(comma, lvaNewObjArrayArgs));
-
- if (argIndex != numArgs)
- {
- return nullptr;
- }
- }
- else
- {
- //
- // Make sure there are exactly two arguments: the array class and
- // the number of elements.
- //
-
- GenTree* arrayLengthNode;
-
-#ifdef FEATURE_READYTORUN
- if (newArrayCall->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1))
- {
- // Array length is 1st argument for readytorun helper
- arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(0)->GetNode();
- }
- else
-#endif
- {
- // Array length is 2nd argument for regular helper
- arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode();
- }
-
- //
- // This optimization is only valid for a constant array size.
- //
- if (arrayLengthNode->gtOper != GT_CNS_INT)
- {
- return nullptr;
- }
-
- numElements = S_SIZE_T(arrayLengthNode->AsIntCon()->gtIconVal);
-
- if (!info.compCompHnd->isSDArray(arrayClsHnd))
- {
- return nullptr;
- }
- }
-
- CORINFO_CLASS_HANDLE elemClsHnd;
- var_types elementType = JITtype2varType(info.compCompHnd->getChildType(arrayClsHnd, &elemClsHnd));
-
- //
- // Note that genTypeSize will return zero for non primitive types, which is exactly
- // what we want (size will then be 0, and we will catch this in the conditional below).
- // Note that we don't expect this to fail for valid binaries, so we assert in the
- // non-verification case (the verification case should not assert but rather correctly
- // handle bad binaries). This assert is not guarding any specific invariant, but rather
- // saying that we don't expect this to happen, and if it is hit, we need to investigate
- // why.
- //
-
- S_UINT32 elemSize(genTypeSize(elementType));
- S_UINT32 size = elemSize * S_UINT32(numElements);
-
- if (size.IsOverflow())
- {
- return nullptr;
- }
-
- if ((size.Value() == 0) || (varTypeIsGC(elementType)))
- {
- return nullptr;
- }
-
- void* initData = info.compCompHnd->getArrayInitializationData(fieldToken, size.Value());
- if (!initData)
- {
- return nullptr;
- }
-
- //
- // At this point we are ready to commit to implementing the InitializeArray
- // intrinsic using a struct assignment. Pop the arguments from the stack and
- // return the struct assignment node.
- //
-
- impPopStack();
- impPopStack();
-
- const unsigned blkSize = size.Value();
- unsigned dataOffset;
-
- if (isMDArray)
- {
- dataOffset = eeGetMDArrayDataOffset(rank);
- }
- else
- {
- dataOffset = eeGetArrayDataOffset();
- }
-
- GenTree* dstAddr = gtNewOperNode(GT_ADD, TYP_BYREF, arrayLocalNode, gtNewIconNode(dataOffset, TYP_I_IMPL));
- GenTree* dst = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, dstAddr, typGetBlkLayout(blkSize));
- GenTree* src = gtNewIndOfIconHandleNode(TYP_STRUCT, (size_t)initData, GTF_ICON_CONST_PTR, true);
-
-#ifdef DEBUG
- src->gtGetOp1()->AsIntCon()->gtTargetHandle = THT_InitializeArrayIntrinsics;
-#endif
-
- return gtNewBlkOpNode(dst, // dst
- src, // src
- false, // volatile
- true); // copyBlock
-}
-
-GenTree* Compiler::impCreateSpanIntrinsic(CORINFO_SIG_INFO* sig)
-{
- assert(sig->numArgs == 1);
- assert(sig->sigInst.methInstCount == 1);
-
- GenTree* fieldTokenNode = impStackTop(0).val;
-
- //
- // Verify that the field token is known and valid. Note that it's also
- // possible for the token to come from reflection, in which case we cannot do
- // the optimization and must therefore revert to calling the helper. You can
- // see an example of this in bvt\DynIL\initarray2.exe (in Main).
- //
-
- // Check to see if the ldtoken helper call is what we see here.
- if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) ||
- (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD)))
- {
- return nullptr;
- }
-
- // Strip helper call away
- fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetNode();
- if (fieldTokenNode->gtOper == GT_IND)
- {
- fieldTokenNode = fieldTokenNode->AsOp()->gtOp1;
- }
-
- // Check for constant
- if (fieldTokenNode->gtOper != GT_CNS_INT)
- {
- return nullptr;
- }
-
- CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle;
- if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr))
- {
- return nullptr;
- }
-
- CORINFO_CLASS_HANDLE fieldOwnerHnd = info.compCompHnd->getFieldClass(fieldToken);
-
- CORINFO_CLASS_HANDLE fieldClsHnd;
- var_types fieldElementType =
- JITtype2varType(info.compCompHnd->getFieldType(fieldToken, &fieldClsHnd, fieldOwnerHnd));
- unsigned totalFieldSize;
-
- // Most static initialization data fields are of some structure, but it is possible for them to be of various
- // primitive types as well
- if (fieldElementType == var_types::TYP_STRUCT)
- {
- totalFieldSize = info.compCompHnd->getClassSize(fieldClsHnd);
- }
- else
- {
- totalFieldSize = genTypeSize(fieldElementType);
- }
-
- // Limit to primitive or enum type - see ArrayNative::GetSpanDataFrom()
- CORINFO_CLASS_HANDLE targetElemHnd = sig->sigInst.methInst[0];
- if (info.compCompHnd->getTypeForPrimitiveValueClass(targetElemHnd) == CORINFO_TYPE_UNDEF)
- {
- return nullptr;
- }
-
- const unsigned targetElemSize = info.compCompHnd->getClassSize(targetElemHnd);
- assert(targetElemSize != 0);
-
- const unsigned count = totalFieldSize / targetElemSize;
- if (count == 0)
- {
- return nullptr;
- }
-
- void* data = info.compCompHnd->getArrayInitializationData(fieldToken, totalFieldSize);
- if (!data)
- {
- return nullptr;
- }
-
- //
- // Ready to commit to the work
- //
-
- impPopStack();
-
- // Turn count and pointer value into constants.
- GenTree* lengthValue = gtNewIconNode(count, TYP_INT);
- GenTree* pointerValue = gtNewIconHandleNode((size_t)data, GTF_ICON_CONST_PTR);
-
- // Construct ReadOnlySpan<T> to return.
- CORINFO_CLASS_HANDLE spanHnd = sig->retTypeClass;
- unsigned spanTempNum = lvaGrabTemp(true DEBUGARG("ReadOnlySpan<T> for CreateSpan<T>"));
- lvaSetStruct(spanTempNum, spanHnd, false);
-
- GenTreeLclFld* pointerField = gtNewLclFldNode(spanTempNum, TYP_BYREF, 0);
- GenTree* pointerFieldAsg = gtNewAssignNode(pointerField, pointerValue);
-
- GenTreeLclFld* lengthField = gtNewLclFldNode(spanTempNum, TYP_INT, TARGET_POINTER_SIZE);
- GenTree* lengthFieldAsg = gtNewAssignNode(lengthField, lengthValue);
-
- // Now append a few statements the initialize the span
- impAppendTree(lengthFieldAsg, CHECK_SPILL_NONE, impCurStmtDI);
- impAppendTree(pointerFieldAsg, CHECK_SPILL_NONE, impCurStmtDI);
-
- // And finally create a tree that points at the span.
- return impCreateLocalNode(spanTempNum DEBUGARG(0));
-}
-
-//------------------------------------------------------------------------
-// impIntrinsic: possibly expand intrinsic call into alternate IR sequence
-//
-// Arguments:
-// newobjThis - for constructor calls, the tree for the newly allocated object
-// clsHnd - handle for the intrinsic method's class
-// method - handle for the intrinsic method
-// sig - signature of the intrinsic method
-// methodFlags - CORINFO_FLG_XXX flags of the intrinsic method
-// memberRef - the token for the intrinsic method
-// readonlyCall - true if call has a readonly prefix
-// tailCall - true if call is in tail position
-// pConstrainedResolvedToken -- resolved token for constrained call, or nullptr
-// if call is not constrained
-// constraintCallThisTransform -- this transform to apply for a constrained call
-// pIntrinsicName [OUT] -- intrinsic name (see enumeration in namedintrinsiclist.h)
-// for "traditional" jit intrinsics
-// isSpecialIntrinsic [OUT] -- set true if intrinsic expansion is a call
-// that is amenable to special downstream optimization opportunities
-//
-// Returns:
-// IR tree to use in place of the call, or nullptr if the jit should treat
-// the intrinsic call like a normal call.
-//
-// pIntrinsicName set to non-illegal value if the call is recognized as a
-// traditional jit intrinsic, even if the intrinsic is not expaned.
-//
-// isSpecial set true if the expansion is subject to special
-// optimizations later in the jit processing
-//
-// Notes:
-// On success the IR tree may be a call to a different method or an inline
-// sequence. If it is a call, then the intrinsic processing here is responsible
-// for handling all the special cases, as upon return to impImportCall
-// expanded intrinsics bypass most of the normal call processing.
-//
-// Intrinsics are generally not recognized in minopts and debug codegen.
-//
-// However, certain traditional intrinsics are identifed as "must expand"
-// if there is no fallback implementation to invoke; these must be handled
-// in all codegen modes.
-//
-// New style intrinsics (where the fallback implementation is in IL) are
-// identified as "must expand" if they are invoked from within their
-// own method bodies.
-//
-GenTree* Compiler::impIntrinsic(GenTree* newobjThis,
- CORINFO_CLASS_HANDLE clsHnd,
- CORINFO_METHOD_HANDLE method,
- CORINFO_SIG_INFO* sig,
- unsigned methodFlags,
- int memberRef,
- bool readonlyCall,
- bool tailCall,
- CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken,
- CORINFO_THIS_TRANSFORM constraintCallThisTransform,
- NamedIntrinsic* pIntrinsicName,
- bool* isSpecialIntrinsic)
-{
- bool mustExpand = false;
- bool isSpecial = false;
- const bool isIntrinsic = (methodFlags & CORINFO_FLG_INTRINSIC) != 0;
-
- NamedIntrinsic ni = lookupNamedIntrinsic(method);
-
- if (isIntrinsic)
- {
- // The recursive non-virtual calls to Jit intrinsics are must-expand by convention.
- mustExpand = gtIsRecursiveCall(method) && !(methodFlags & CORINFO_FLG_VIRTUAL);
- }
- else
- {
- // For mismatched VM (AltJit) we want to check all methods as intrinsic to ensure
- // we get more accurate codegen. This particularly applies to HWIntrinsic usage
- assert(!info.compMatchedVM);
- }
-
- // We specially support the following on all platforms to allow for dead
- // code optimization and to more generally support recursive intrinsics.
-
- if (isIntrinsic)
- {
- if (ni == NI_IsSupported_True)
- {
- assert(sig->numArgs == 0);
- return gtNewIconNode(true);
- }
-
- if (ni == NI_IsSupported_False)
- {
- assert(sig->numArgs == 0);
- return gtNewIconNode(false);
- }
-
- if (ni == NI_Throw_PlatformNotSupportedException)
- {
- return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
- }
-
- if ((ni > NI_SRCS_UNSAFE_START) && (ni < NI_SRCS_UNSAFE_END))
- {
- assert(!mustExpand);
- return impSRCSUnsafeIntrinsic(ni, clsHnd, method, sig);
- }
- }
-
-#ifdef FEATURE_HW_INTRINSICS
- if ((ni > NI_HW_INTRINSIC_START) && (ni < NI_HW_INTRINSIC_END))
- {
- if (!isIntrinsic)
- {
-#if defined(TARGET_XARCH)
- // We can't guarantee that all overloads for the xplat intrinsics can be
- // handled by the AltJit, so limit only the platform specific intrinsics
- assert((NI_Vector256_Xor + 1) == NI_X86Base_BitScanForward);
-
- if (ni < NI_Vector256_Xor)
-#elif defined(TARGET_ARM64)
- // We can't guarantee that all overloads for the xplat intrinsics can be
- // handled by the AltJit, so limit only the platform specific intrinsics
- assert((NI_Vector128_Xor + 1) == NI_AdvSimd_Abs);
-
- if (ni < NI_Vector128_Xor)
-#else
-#error Unsupported platform
-#endif
- {
- // Several of the NI_Vector64/128/256 APIs do not have
- // all overloads as intrinsic today so they will assert
- return nullptr;
- }
- }
-
- GenTree* hwintrinsic = impHWIntrinsic(ni, clsHnd, method, sig, mustExpand);
-
- if (mustExpand && (hwintrinsic == nullptr))
- {
- return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_NOT_IMPLEMENTED, method, sig, mustExpand);
- }
-
- return hwintrinsic;
- }
-
- if (isIntrinsic && (ni > NI_SIMD_AS_HWINTRINSIC_START) && (ni < NI_SIMD_AS_HWINTRINSIC_END))
- {
- // These intrinsics aren't defined recursively and so they will never be mustExpand
- // Instead, they provide software fallbacks that will be executed instead.
-
- assert(!mustExpand);
- return impSimdAsHWIntrinsic(ni, clsHnd, method, sig, newobjThis);
- }
-#endif // FEATURE_HW_INTRINSICS
-
- if (!isIntrinsic)
- {
- // Outside the cases above, there are many intrinsics which apply to only a
- // subset of overload and where simply matching by name may cause downstream
- // asserts or other failures. Math.Min is one example, where it only applies
- // to the floating-point overloads.
- return nullptr;
- }
-
- *pIntrinsicName = ni;
-
- if (ni == NI_System_StubHelpers_GetStubContext)
- {
- // must be done regardless of DbgCode and MinOpts
- return gtNewLclvNode(lvaStubArgumentVar, TYP_I_IMPL);
- }
-
- if (ni == NI_System_StubHelpers_NextCallReturnAddress)
- {
- // For now we just avoid inlining anything into these methods since
- // this intrinsic is only rarely used. We could do this better if we
- // wanted to by trying to match which call is the one we need to get
- // the return address of.
- info.compHasNextCallRetAddr = true;
- return new (this, GT_LABEL) GenTree(GT_LABEL, TYP_I_IMPL);
- }
-
- switch (ni)
- {
- // CreateSpan must be expanded for NativeAOT
- case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan:
- case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray:
- mustExpand |= IsTargetAbi(CORINFO_NATIVEAOT_ABI);
- break;
-
- case NI_Internal_Runtime_MethodTable_Of:
- case NI_System_Activator_AllocatorOf:
- case NI_System_Activator_DefaultConstructorOf:
- case NI_System_EETypePtr_EETypePtrOf:
- mustExpand = true;
- break;
-
- default:
- break;
- }
-
- GenTree* retNode = nullptr;
-
- // Under debug and minopts, only expand what is required.
- // NextCallReturnAddress intrinsic returns the return address of the next call.
- // If that call is an intrinsic and is expanded, codegen for NextCallReturnAddress will fail.
- // To avoid that we conservatively expand only required intrinsics in methods that call
- // the NextCallReturnAddress intrinsic.
- if (!mustExpand && (opts.OptimizationDisabled() || info.compHasNextCallRetAddr))
- {
- *pIntrinsicName = NI_Illegal;
- return retNode;
- }
-
- CorInfoType callJitType = sig->retType;
- var_types callType = JITtype2varType(callJitType);
-
- /* First do the intrinsics which are always smaller than a call */
-
- if (ni != NI_Illegal)
- {
- assert(retNode == nullptr);
- switch (ni)
- {
- case NI_Array_Address:
- case NI_Array_Get:
- case NI_Array_Set:
- retNode = impArrayAccessIntrinsic(clsHnd, sig, memberRef, readonlyCall, ni);
- break;
-
- case NI_System_String_Equals:
- {
- retNode = impStringEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags);
- break;
- }
-
- case NI_System_MemoryExtensions_Equals:
- case NI_System_MemoryExtensions_SequenceEqual:
- {
- retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags);
- break;
- }
-
- case NI_System_String_StartsWith:
- {
- retNode = impStringEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags);
- break;
- }
-
- case NI_System_MemoryExtensions_StartsWith:
- {
- retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags);
- break;
- }
-
- case NI_System_MemoryExtensions_AsSpan:
- case NI_System_String_op_Implicit:
- {
- assert(sig->numArgs == 1);
- isSpecial = impStackTop().val->OperIs(GT_CNS_STR);
- break;
- }
-
- case NI_System_String_get_Chars:
- {
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
- GenTree* addr = gtNewIndexAddr(op1, op2, TYP_USHORT, NO_CLASS_HANDLE, OFFSETOF__CORINFO_String__chars,
- OFFSETOF__CORINFO_String__stringLen);
- retNode = gtNewIndexIndir(addr->AsIndexAddr());
- break;
- }
-
- case NI_System_String_get_Length:
- {
- GenTree* op1 = impPopStack().val;
- if (op1->OperIs(GT_CNS_STR))
- {
- // Optimize `ldstr + String::get_Length()` to CNS_INT
- // e.g. "Hello".Length => 5
- GenTreeIntCon* iconNode = gtNewStringLiteralLength(op1->AsStrCon());
- if (iconNode != nullptr)
- {
- retNode = iconNode;
- break;
- }
- }
- GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, op1, OFFSETOF__CORINFO_String__stringLen, compCurBB);
- op1 = arrLen;
-
- // Getting the length of a null string should throw
- op1->gtFlags |= GTF_EXCEPT;
-
- retNode = op1;
- break;
- }
-
- case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan:
- {
- retNode = impCreateSpanIntrinsic(sig);
- break;
- }
-
- case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray:
- {
- retNode = impInitializeArrayIntrinsic(sig);
- break;
- }
-
- case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant:
- {
- GenTree* op1 = impPopStack().val;
- if (op1->OperIsConst())
- {
- // op1 is a known constant, replace with 'true'.
- retNode = gtNewIconNode(1);
- JITDUMP("\nExpanding RuntimeHelpers.IsKnownConstant to true early\n");
- // We can also consider FTN_ADDR and typeof(T) here
- }
- else
- {
- // op1 is not a known constant, we'll do the expansion in morph
- retNode = new (this, GT_INTRINSIC) GenTreeIntrinsic(TYP_INT, op1, ni, method);
- JITDUMP("\nConverting RuntimeHelpers.IsKnownConstant to:\n");
- DISPTREE(retNode);
- }
- break;
- }
-
- case NI_Internal_Runtime_MethodTable_Of:
- case NI_System_Activator_AllocatorOf:
- case NI_System_Activator_DefaultConstructorOf:
- case NI_System_EETypePtr_EETypePtrOf:
- {
- assert(IsTargetAbi(CORINFO_NATIVEAOT_ABI)); // Only NativeAOT supports it.
- CORINFO_RESOLVED_TOKEN resolvedToken;
- resolvedToken.tokenContext = impTokenLookupContextHandle;
- resolvedToken.tokenScope = info.compScopeHnd;
- resolvedToken.token = memberRef;
- resolvedToken.tokenType = CORINFO_TOKENKIND_Method;
-
- CORINFO_GENERICHANDLE_RESULT embedInfo;
- info.compCompHnd->expandRawHandleIntrinsic(&resolvedToken, &embedInfo);
-
- GenTree* rawHandle = impLookupToTree(&resolvedToken, &embedInfo.lookup, gtTokenToIconFlags(memberRef),
- embedInfo.compileTimeHandle);
- if (rawHandle == nullptr)
- {
- return nullptr;
- }
-
- noway_assert(genTypeSize(rawHandle->TypeGet()) == genTypeSize(TYP_I_IMPL));
-
- unsigned rawHandleSlot = lvaGrabTemp(true DEBUGARG("rawHandle"));
- impAssignTempGen(rawHandleSlot, rawHandle, clsHnd, CHECK_SPILL_NONE);
-
- GenTree* lclVar = gtNewLclvNode(rawHandleSlot, TYP_I_IMPL);
- GenTree* lclVarAddr = gtNewOperNode(GT_ADDR, TYP_I_IMPL, lclVar);
- var_types resultType = JITtype2varType(sig->retType);
- if (resultType == TYP_STRUCT)
- {
- retNode = gtNewObjNode(sig->retTypeClass, lclVarAddr);
- }
- else
- {
- retNode = gtNewIndir(resultType, lclVarAddr);
- }
- break;
- }
-
- case NI_System_Span_get_Item:
- case NI_System_ReadOnlySpan_get_Item:
- {
- // Have index, stack pointer-to Span<T> s on the stack. Expand to:
- //
- // For Span<T>
- // Comma
- // BoundsCheck(index, s->_length)
- // s->_reference + index * sizeof(T)
- //
- // For ReadOnlySpan<T> -- same expansion, as it now returns a readonly ref
- //
- // Signature should show one class type parameter, which
- // we need to examine.
- assert(sig->sigInst.classInstCount == 1);
- assert(sig->numArgs == 1);
- CORINFO_CLASS_HANDLE spanElemHnd = sig->sigInst.classInst[0];
- const unsigned elemSize = info.compCompHnd->getClassSize(spanElemHnd);
- assert(elemSize > 0);
-
- const bool isReadOnly = (ni == NI_System_ReadOnlySpan_get_Item);
-
- JITDUMP("\nimpIntrinsic: Expanding %sSpan<T>.get_Item, T=%s, sizeof(T)=%u\n",
- isReadOnly ? "ReadOnly" : "", eeGetClassName(spanElemHnd), elemSize);
-
- GenTree* index = impPopStack().val;
- GenTree* ptrToSpan = impPopStack().val;
- GenTree* indexClone = nullptr;
- GenTree* ptrToSpanClone = nullptr;
- assert(genActualType(index) == TYP_INT);
- assert(ptrToSpan->TypeGet() == TYP_BYREF);
-
-#if defined(DEBUG)
- if (verbose)
- {
- printf("with ptr-to-span\n");
- gtDispTree(ptrToSpan);
- printf("and index\n");
- gtDispTree(index);
- }
-#endif // defined(DEBUG)
-
- // We need to use both index and ptr-to-span twice, so clone or spill.
- index = impCloneExpr(index, &indexClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL,
- nullptr DEBUGARG("Span.get_Item index"));
- ptrToSpan = impCloneExpr(ptrToSpan, &ptrToSpanClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL,
- nullptr DEBUGARG("Span.get_Item ptrToSpan"));
-
- // Bounds check
- CORINFO_FIELD_HANDLE lengthHnd = info.compCompHnd->getFieldInClass(clsHnd, 1);
- const unsigned lengthOffset = info.compCompHnd->getFieldOffset(lengthHnd);
- GenTree* length = gtNewFieldRef(TYP_INT, lengthHnd, ptrToSpan, lengthOffset);
- GenTree* boundsCheck = new (this, GT_BOUNDS_CHECK) GenTreeBoundsChk(index, length, SCK_RNGCHK_FAIL);
-
- // Element access
- index = indexClone;
-
-#ifdef TARGET_64BIT
- if (index->OperGet() == GT_CNS_INT)
- {
- index->gtType = TYP_I_IMPL;
- }
- else
- {
- index = gtNewCastNode(TYP_I_IMPL, index, true, TYP_I_IMPL);
- }
-#endif
-
- if (elemSize != 1)
- {
- GenTree* sizeofNode = gtNewIconNode(static_cast<ssize_t>(elemSize), TYP_I_IMPL);
- index = gtNewOperNode(GT_MUL, TYP_I_IMPL, index, sizeofNode);
- }
-
- CORINFO_FIELD_HANDLE ptrHnd = info.compCompHnd->getFieldInClass(clsHnd, 0);
- const unsigned ptrOffset = info.compCompHnd->getFieldOffset(ptrHnd);
- GenTree* data = gtNewFieldRef(TYP_BYREF, ptrHnd, ptrToSpanClone, ptrOffset);
- GenTree* result = gtNewOperNode(GT_ADD, TYP_BYREF, data, index);
-
- // Prepare result
- var_types resultType = JITtype2varType(sig->retType);
- assert(resultType == result->TypeGet());
- retNode = gtNewOperNode(GT_COMMA, resultType, boundsCheck, result);
-
- break;
- }
-
- case NI_System_RuntimeTypeHandle_GetValueInternal:
- {
- GenTree* op1 = impStackTop(0).val;
- if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) &&
- gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall()))
- {
- // Old tree
- // Helper-RuntimeTypeHandle -> TreeToGetNativeTypeHandle
- //
- // New tree
- // TreeToGetNativeTypeHandle
-
- // Remove call to helper and return the native TypeHandle pointer that was the parameter
- // to that helper.
-
- op1 = impPopStack().val;
-
- // Get native TypeHandle argument to old helper
- assert(op1->AsCall()->gtArgs.CountArgs() == 1);
- op1 = op1->AsCall()->gtArgs.GetArgByIndex(0)->GetNode();
- retNode = op1;
- }
- // Call the regular function.
- break;
- }
-
- case NI_System_Type_GetTypeFromHandle:
- {
- GenTree* op1 = impStackTop(0).val;
- CorInfoHelpFunc typeHandleHelper;
- if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) &&
- gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall(), &typeHandleHelper))
- {
- op1 = impPopStack().val;
- // Replace helper with a more specialized helper that returns RuntimeType
- if (typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE)
- {
- typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE;
- }
- else
- {
- assert(typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL);
- typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL;
- }
- assert(op1->AsCall()->gtArgs.CountArgs() == 1);
- op1 = gtNewHelperCallNode(typeHandleHelper, TYP_REF,
- op1->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode());
- op1->gtType = TYP_REF;
- retNode = op1;
- }
- break;
- }
-
- case NI_System_Type_op_Equality:
- case NI_System_Type_op_Inequality:
- {
- JITDUMP("Importing Type.op_*Equality intrinsic\n");
- GenTree* op1 = impStackTop(1).val;
- GenTree* op2 = impStackTop(0).val;
- GenTree* optTree = gtFoldTypeEqualityCall(ni == NI_System_Type_op_Equality, op1, op2);
- if (optTree != nullptr)
- {
- // Success, clean up the evaluation stack.
- impPopStack();
- impPopStack();
-
- // See if we can optimize even further, to a handle compare.
- optTree = gtFoldTypeCompare(optTree);
-
- // See if we can now fold a handle compare to a constant.
- optTree = gtFoldExpr(optTree);
-
- retNode = optTree;
- }
- else
- {
- // Retry optimizing these later
- isSpecial = true;
- }
- break;
- }
-
- case NI_System_Enum_HasFlag:
- {
- GenTree* thisOp = impStackTop(1).val;
- GenTree* flagOp = impStackTop(0).val;
- GenTree* optTree = gtOptimizeEnumHasFlag(thisOp, flagOp);
-
- if (optTree != nullptr)
- {
- // Optimization successful. Pop the stack for real.
- impPopStack();
- impPopStack();
- retNode = optTree;
- }
- else
- {
- // Retry optimizing this during morph.
- isSpecial = true;
- }
-
- break;
- }
-
- case NI_System_Type_IsAssignableFrom:
- {
- GenTree* typeTo = impStackTop(1).val;
- GenTree* typeFrom = impStackTop(0).val;
-
- retNode = impTypeIsAssignable(typeTo, typeFrom);
- break;
- }
-
- case NI_System_Type_IsAssignableTo:
- {
- GenTree* typeTo = impStackTop(0).val;
- GenTree* typeFrom = impStackTop(1).val;
-
- retNode = impTypeIsAssignable(typeTo, typeFrom);
- break;
- }
-
- case NI_System_Type_get_IsValueType:
- case NI_System_Type_get_IsByRefLike:
- {
- // Optimize
- //
- // call Type.GetTypeFromHandle (which is replaced with CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE)
- // call Type.IsXXX
- //
- // to `true` or `false`
- // e.g., `typeof(int).IsValueType` => `true`
- // e.g., `typeof(Span<int>).IsByRefLike` => `true`
- if (impStackTop().val->IsCall())
- {
- GenTreeCall* call = impStackTop().val->AsCall();
- if (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE))
- {
- assert(call->gtArgs.CountArgs() == 1);
- CORINFO_CLASS_HANDLE hClass =
- gtGetHelperArgClassHandle(call->gtArgs.GetArgByIndex(0)->GetEarlyNode());
- if (hClass != NO_CLASS_HANDLE)
- {
- switch (ni)
- {
- case NI_System_Type_get_IsValueType:
- retNode = gtNewIconNode(eeIsValueClass(hClass) ? 1 : 0);
- break;
- case NI_System_Type_get_IsByRefLike:
- retNode = gtNewIconNode(
- (info.compCompHnd->getClassAttribs(hClass) & CORINFO_FLG_BYREF_LIKE) ? 1 : 0);
- break;
- default:
- NO_WAY("Intrinsic not supported in this path.");
- }
- impPopStack(); // drop CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE call
- }
- }
- }
- break;
- }
-
- case NI_System_Threading_Thread_get_ManagedThreadId:
- {
- if (impStackTop().val->OperIs(GT_RET_EXPR))
- {
- GenTreeCall* call = impStackTop().val->AsRetExpr()->gtInlineCandidate->AsCall();
- if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC)
- {
- if (lookupNamedIntrinsic(call->gtCallMethHnd) == NI_System_Threading_Thread_get_CurrentThread)
- {
- // drop get_CurrentThread() call
- impPopStack();
- call->ReplaceWith(gtNewNothingNode(), this);
- retNode = gtNewHelperCallNode(CORINFO_HELP_GETCURRENTMANAGEDTHREADID, TYP_INT);
- }
- }
- }
- break;
- }
-
-#ifdef TARGET_ARM64
- // Intrinsify Interlocked.Or and Interlocked.And only for arm64-v8.1 (and newer)
- // TODO-CQ: Implement for XArch (https://github.com/dotnet/runtime/issues/32239).
- case NI_System_Threading_Interlocked_Or:
- case NI_System_Threading_Interlocked_And:
- {
- if (compOpportunisticallyDependsOn(InstructionSet_Atomics))
- {
- assert(sig->numArgs == 2);
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
- genTreeOps op = (ni == NI_System_Threading_Interlocked_Or) ? GT_XORR : GT_XAND;
- retNode = gtNewOperNode(op, genActualType(callType), op1, op2);
- retNode->gtFlags |= GTF_GLOB_REF | GTF_ASG;
- }
- break;
- }
-#endif // TARGET_ARM64
-
-#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
- // TODO-ARM-CQ: reenable treating InterlockedCmpXchg32 operation as intrinsic
- case NI_System_Threading_Interlocked_CompareExchange:
- {
- var_types retType = JITtype2varType(sig->retType);
- if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4))
- {
- break;
- }
- if ((retType != TYP_INT) && (retType != TYP_LONG))
- {
- break;
- }
-
- assert(callType != TYP_STRUCT);
- assert(sig->numArgs == 3);
-
- GenTree* op3 = impPopStack().val; // comparand
- GenTree* op2 = impPopStack().val; // value
- GenTree* op1 = impPopStack().val; // location
-
- GenTree* node = new (this, GT_CMPXCHG) GenTreeCmpXchg(genActualType(callType), op1, op2, op3);
-
- node->AsCmpXchg()->gtOpLocation->gtFlags |= GTF_DONT_CSE;
- retNode = node;
- break;
- }
-
- case NI_System_Threading_Interlocked_Exchange:
- case NI_System_Threading_Interlocked_ExchangeAdd:
- {
- assert(callType != TYP_STRUCT);
- assert(sig->numArgs == 2);
-
- var_types retType = JITtype2varType(sig->retType);
- if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4))
- {
- break;
- }
- if ((retType != TYP_INT) && (retType != TYP_LONG))
- {
- break;
- }
-
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
-
- // This creates:
- // val
- // XAdd
- // addr
- // field (for example)
- //
- // In the case where the first argument is the address of a local, we might
- // want to make this *not* make the var address-taken -- but atomic instructions
- // on a local are probably pretty useless anyway, so we probably don't care.
-
- op1 = gtNewOperNode(ni == NI_System_Threading_Interlocked_ExchangeAdd ? GT_XADD : GT_XCHG,
- genActualType(callType), op1, op2);
- op1->gtFlags |= GTF_GLOB_REF | GTF_ASG;
- retNode = op1;
- break;
- }
-#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64)
-
- case NI_System_Threading_Interlocked_MemoryBarrier:
- case NI_System_Threading_Interlocked_ReadMemoryBarrier:
- {
- assert(sig->numArgs == 0);
-
- GenTree* op1 = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID);
- op1->gtFlags |= GTF_GLOB_REF | GTF_ASG;
-
- // On XARCH `NI_System_Threading_Interlocked_ReadMemoryBarrier` fences need not be emitted.
- // However, we still need to capture the effect on reordering.
- if (ni == NI_System_Threading_Interlocked_ReadMemoryBarrier)
- {
- op1->gtFlags |= GTF_MEMORYBARRIER_LOAD;
- }
-
- retNode = op1;
- break;
- }
-
-#ifdef FEATURE_HW_INTRINSICS
- case NI_System_Math_FusedMultiplyAdd:
- {
-#ifdef TARGET_XARCH
- if (compExactlyDependsOn(InstructionSet_FMA))
- {
- assert(varTypeIsFloating(callType));
-
- // We are constructing a chain of intrinsics similar to:
- // return FMA.MultiplyAddScalar(
- // Vector128.CreateScalarUnsafe(x),
- // Vector128.CreateScalarUnsafe(y),
- // Vector128.CreateScalarUnsafe(z)
- // ).ToScalar();
-
- GenTree* op3 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val,
- NI_Vector128_CreateScalarUnsafe, callJitType, 16);
- GenTree* op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val,
- NI_Vector128_CreateScalarUnsafe, callJitType, 16);
- GenTree* op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val,
- NI_Vector128_CreateScalarUnsafe, callJitType, 16);
- GenTree* res =
- gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_FMA_MultiplyAddScalar, callJitType, 16);
-
- retNode = gtNewSimdHWIntrinsicNode(callType, res, NI_Vector128_ToScalar, callJitType, 16);
- break;
- }
-#elif defined(TARGET_ARM64)
- if (compExactlyDependsOn(InstructionSet_AdvSimd))
- {
- assert(varTypeIsFloating(callType));
-
- // We are constructing a chain of intrinsics similar to:
- // return AdvSimd.FusedMultiplyAddScalar(
- // Vector64.Create{ScalarUnsafe}(z),
- // Vector64.Create{ScalarUnsafe}(y),
- // Vector64.Create{ScalarUnsafe}(x)
- // ).ToScalar();
-
- NamedIntrinsic createVector64 =
- (callType == TYP_DOUBLE) ? NI_Vector64_Create : NI_Vector64_CreateScalarUnsafe;
-
- constexpr unsigned int simdSize = 8;
-
- GenTree* op3 =
- gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize);
- GenTree* op2 =
- gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize);
- GenTree* op1 =
- gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize);
-
- // Note that AdvSimd.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 + op2 * op3
- // while Math{F}.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 * op2 + op3
- retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op3, op2, op1, NI_AdvSimd_FusedMultiplyAddScalar,
- callJitType, simdSize);
-
- retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector64_ToScalar, callJitType, simdSize);
- break;
- }
-#endif
-
- // TODO-CQ-XArch: Ideally we would create a GT_INTRINSIC node for fma, however, that currently
- // requires more extensive changes to valuenum to support methods with 3 operands
-
- // We want to generate a GT_INTRINSIC node in the case the call can't be treated as
- // a target intrinsic so that we can still benefit from CSE and constant folding.
-
- break;
- }
-#endif // FEATURE_HW_INTRINSICS
-
- case NI_System_Math_Abs:
- case NI_System_Math_Acos:
- case NI_System_Math_Acosh:
- case NI_System_Math_Asin:
- case NI_System_Math_Asinh:
- case NI_System_Math_Atan:
- case NI_System_Math_Atanh:
- case NI_System_Math_Atan2:
- case NI_System_Math_Cbrt:
- case NI_System_Math_Ceiling:
- case NI_System_Math_Cos:
- case NI_System_Math_Cosh:
- case NI_System_Math_Exp:
- case NI_System_Math_Floor:
- case NI_System_Math_FMod:
- case NI_System_Math_ILogB:
- case NI_System_Math_Log:
- case NI_System_Math_Log2:
- case NI_System_Math_Log10:
- {
- retNode = impMathIntrinsic(method, sig, callType, ni, tailCall);
- break;
- }
-#if defined(TARGET_ARM64)
- // ARM64 has fmax/fmin which are IEEE754:2019 minimum/maximum compatible
- // TODO-XARCH-CQ: Enable this for XARCH when one of the arguments is a constant
- // so we can then emit maxss/minss and avoid NaN/-0.0 handling
- case NI_System_Math_Max:
- case NI_System_Math_Min:
-#endif
-
-#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH)
- case NI_System_Math_Max:
- case NI_System_Math_Min:
- {
- assert(varTypeIsFloating(callType));
- assert(sig->numArgs == 2);
-
- GenTreeDblCon* cnsNode = nullptr;
- GenTree* otherNode = nullptr;
-
- GenTree* op2 = impStackTop().val;
- GenTree* op1 = impStackTop(1).val;
-
- if (op2->IsCnsFltOrDbl())
- {
- cnsNode = op2->AsDblCon();
- otherNode = op1;
- }
- else if (op1->IsCnsFltOrDbl())
- {
- cnsNode = op1->AsDblCon();
- otherNode = op2;
- }
-
- if (cnsNode == nullptr)
- {
- // no constant node, nothing to do
- break;
- }
-
- if (otherNode->IsCnsFltOrDbl())
- {
- // both are constant, we can fold this operation completely. Pop both peeked values
-
- if (ni == NI_System_Math_Max)
- {
- cnsNode->SetDconValue(
- FloatingPointUtils::maximum(cnsNode->DconValue(), otherNode->AsDblCon()->DconValue()));
- }
- else
- {
- assert(ni == NI_System_Math_Min);
- cnsNode->SetDconValue(
- FloatingPointUtils::minimum(cnsNode->DconValue(), otherNode->AsDblCon()->DconValue()));
- }
-
- retNode = cnsNode;
-
- impPopStack();
- impPopStack();
- DEBUG_DESTROY_NODE(otherNode);
-
- break;
- }
-
- // only one is constant, we can fold in specialized scenarios
-
- if (cnsNode->IsFloatNaN())
- {
- impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("spill side effects before propagating NaN"));
-
- // maxsd, maxss, minsd, and minss all return op2 if either is NaN
- // we require NaN to be propagated so ensure the known NaN is op2
-
- impPopStack();
- impPopStack();
- DEBUG_DESTROY_NODE(otherNode);
-
- retNode = cnsNode;
- break;
- }
-
- if (!compOpportunisticallyDependsOn(InstructionSet_SSE2))
- {
- break;
- }
-
- if (ni == NI_System_Math_Max)
- {
- // maxsd, maxss return op2 if both inputs are 0 of either sign
- // we require +0 to be greater than -0, so we can't handle if
- // the known constant is +0. This is because if the unknown value
- // is -0, we'd need the cns to be op2. But if the unknown value
- // is NaN, we'd need the cns to be op1 instead.
-
- if (cnsNode->IsFloatPositiveZero())
- {
- break;
- }
-
- // Given the checks, op1 can safely be the cns and op2 the other node
-
- ni = (callType == TYP_DOUBLE) ? NI_SSE2_Max : NI_SSE_Max;
-
- // one is constant and we know its something we can handle, so pop both peeked values
-
- op1 = cnsNode;
- op2 = otherNode;
- }
- else
- {
- assert(ni == NI_System_Math_Min);
-
- // minsd, minss return op2 if both inputs are 0 of either sign
- // we require -0 to be lesser than +0, so we can't handle if
- // the known constant is -0. This is because if the unknown value
- // is +0, we'd need the cns to be op2. But if the unknown value
- // is NaN, we'd need the cns to be op1 instead.
-
- if (cnsNode->IsFloatNegativeZero())
- {
- break;
- }
-
- // Given the checks, op1 can safely be the cns and op2 the other node
-
- ni = (callType == TYP_DOUBLE) ? NI_SSE2_Min : NI_SSE_Min;
-
- // one is constant and we know its something we can handle, so pop both peeked values
-
- op1 = cnsNode;
- op2 = otherNode;
- }
-
- assert(op1->IsCnsFltOrDbl() && !op2->IsCnsFltOrDbl());
-
- impPopStack();
- impPopStack();
-
- GenTreeVecCon* vecCon = gtNewVconNode(TYP_SIMD16);
-
- if (callJitType == CORINFO_TYPE_FLOAT)
- {
- vecCon->gtSimd16Val.f32[0] = (float)op1->AsDblCon()->DconValue();
- }
- else
- {
- vecCon->gtSimd16Val.f64[0] = op1->AsDblCon()->DconValue();
- }
-
- op1 = vecCon;
- op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, NI_Vector128_CreateScalarUnsafe, callJitType, 16);
-
- retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, ni, callJitType, 16);
- retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector128_ToScalar, callJitType, 16);
-
- break;
- }
-#endif
-
- case NI_System_Math_Pow:
- case NI_System_Math_Round:
- case NI_System_Math_Sin:
- case NI_System_Math_Sinh:
- case NI_System_Math_Sqrt:
- case NI_System_Math_Tan:
- case NI_System_Math_Tanh:
- case NI_System_Math_Truncate:
- {
- retNode = impMathIntrinsic(method, sig, callType, ni, tailCall);
- break;
- }
-
- case NI_System_Array_Clone:
- case NI_System_Collections_Generic_Comparer_get_Default:
- case NI_System_Collections_Generic_EqualityComparer_get_Default:
- case NI_System_Object_MemberwiseClone:
- case NI_System_Threading_Thread_get_CurrentThread:
- {
- // Flag for later handling.
- isSpecial = true;
- break;
- }
-
- case NI_System_Object_GetType:
- {
- JITDUMP("\n impIntrinsic: call to Object.GetType\n");
- GenTree* op1 = impStackTop(0).val;
-
- // If we're calling GetType on a boxed value, just get the type directly.
- if (op1->IsBoxedValue())
- {
- JITDUMP("Attempting to optimize box(...).getType() to direct type construction\n");
-
- // Try and clean up the box. Obtain the handle we
- // were going to pass to the newobj.
- GenTree* boxTypeHandle = gtTryRemoveBoxUpstreamEffects(op1, BR_REMOVE_AND_NARROW_WANT_TYPE_HANDLE);
-
- if (boxTypeHandle != nullptr)
- {
- // Note we don't need to play the TYP_STRUCT games here like
- // do for LDTOKEN since the return value of this operator is Type,
- // not RuntimeTypeHandle.
- impPopStack();
- GenTree* runtimeType =
- gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, boxTypeHandle);
- retNode = runtimeType;
- }
- }
-
- // If we have a constrained callvirt with a "box this" transform
- // we know we have a value class and hence an exact type.
- //
- // If so, instead of boxing and then extracting the type, just
- // construct the type directly.
- if ((retNode == nullptr) && (pConstrainedResolvedToken != nullptr) &&
- (constraintCallThisTransform == CORINFO_BOX_THIS))
- {
- // Ensure this is one of the simple box cases (in particular, rule out nullables).
- const CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pConstrainedResolvedToken->hClass);
- const bool isSafeToOptimize = (boxHelper == CORINFO_HELP_BOX);
-
- if (isSafeToOptimize)
- {
- JITDUMP("Optimizing constrained box-this obj.getType() to direct type construction\n");
- impPopStack();
- GenTree* typeHandleOp =
- impTokenToHandle(pConstrainedResolvedToken, nullptr, true /* mustRestoreHandle */);
- if (typeHandleOp == nullptr)
- {
- assert(compDonotInline());
- return nullptr;
- }
- GenTree* runtimeType =
- gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, typeHandleOp);
- retNode = runtimeType;
- }
- }
-
-#ifdef DEBUG
- if (retNode != nullptr)
- {
- JITDUMP("Optimized result for call to GetType is\n");
- if (verbose)
- {
- gtDispTree(retNode);
- }
- }
-#endif
-
- // Else expand as an intrinsic, unless the call is constrained,
- // in which case we defer expansion to allow impImportCall do the
- // special constraint processing.
- if ((retNode == nullptr) && (pConstrainedResolvedToken == nullptr))
- {
- JITDUMP("Expanding as special intrinsic\n");
- impPopStack();
- op1 = new (this, GT_INTRINSIC) GenTreeIntrinsic(genActualType(callType), op1, ni, method);
-
- // Set the CALL flag to indicate that the operator is implemented by a call.
- // Set also the EXCEPTION flag because the native implementation of
- // NI_System_Object_GetType intrinsic can throw NullReferenceException.
- op1->gtFlags |= (GTF_CALL | GTF_EXCEPT);
- retNode = op1;
- // Might be further optimizable, so arrange to leave a mark behind
- isSpecial = true;
- }
-
- if (retNode == nullptr)
- {
- JITDUMP("Leaving as normal call\n");
- // Might be further optimizable, so arrange to leave a mark behind
- isSpecial = true;
- }
-
- break;
- }
-
- case NI_System_Array_GetLength:
- case NI_System_Array_GetLowerBound:
- case NI_System_Array_GetUpperBound:
- {
- // System.Array.GetLength(Int32) method:
- // public int GetLength(int dimension)
- // System.Array.GetLowerBound(Int32) method:
- // public int GetLowerBound(int dimension)
- // System.Array.GetUpperBound(Int32) method:
- // public int GetUpperBound(int dimension)
- //
- // Only implement these as intrinsics for multi-dimensional arrays.
- // Only handle constant dimension arguments.
-
- GenTree* gtDim = impStackTop().val;
- GenTree* gtArr = impStackTop(1).val;
-
- if (gtDim->IsIntegralConst())
- {
- bool isExact = false;
- bool isNonNull = false;
- CORINFO_CLASS_HANDLE arrCls = gtGetClassHandle(gtArr, &isExact, &isNonNull);
- if (arrCls != NO_CLASS_HANDLE)
- {
- unsigned rank = info.compCompHnd->getArrayRank(arrCls);
- if ((rank > 1) && !info.compCompHnd->isSDArray(arrCls))
- {
- // `rank` is guaranteed to be <=32 (see MAX_RANK in vm\array.h). Any constant argument
- // is `int` sized.
- INT64 dimValue = gtDim->AsIntConCommon()->IntegralValue();
- assert((unsigned int)dimValue == dimValue);
- unsigned dim = (unsigned int)dimValue;
- if (dim < rank)
- {
- // This is now known to be a multi-dimension array with a constant dimension
- // that is in range; we can expand it as an intrinsic.
-
- impPopStack().val; // Pop the dim and array object; we already have a pointer to them.
- impPopStack().val;
-
- // Make sure there are no global effects in the array (such as it being a function
- // call), so we can mark the generated indirection with GTF_IND_INVARIANT. In the
- // GetUpperBound case we need the cloned object, since we refer to the array
- // object twice. In the other cases, we don't need to clone.
- GenTree* gtArrClone = nullptr;
- if (((gtArr->gtFlags & GTF_GLOB_EFFECT) != 0) || (ni == NI_System_Array_GetUpperBound))
- {
- gtArr = impCloneExpr(gtArr, >ArrClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL,
- nullptr DEBUGARG("MD intrinsics array"));
- }
-
- switch (ni)
- {
- case NI_System_Array_GetLength:
- {
- // Generate *(array + offset-to-length-array + sizeof(int) * dim)
- unsigned offs = eeGetMDArrayLengthOffset(rank, dim);
- GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL);
- GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs);
- retNode = gtNewIndir(TYP_INT, gtAddr);
- retNode->gtFlags |= GTF_IND_INVARIANT;
- break;
- }
- case NI_System_Array_GetLowerBound:
- {
- // Generate *(array + offset-to-bounds-array + sizeof(int) * dim)
- unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim);
- GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL);
- GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs);
- retNode = gtNewIndir(TYP_INT, gtAddr);
- retNode->gtFlags |= GTF_IND_INVARIANT;
- break;
- }
- case NI_System_Array_GetUpperBound:
- {
- assert(gtArrClone != nullptr);
-
- // Generate:
- // *(array + offset-to-length-array + sizeof(int) * dim) +
- // *(array + offset-to-bounds-array + sizeof(int) * dim) - 1
- unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim);
- GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL);
- GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs);
- GenTree* gtLowerBound = gtNewIndir(TYP_INT, gtAddr);
- gtLowerBound->gtFlags |= GTF_IND_INVARIANT;
-
- offs = eeGetMDArrayLengthOffset(rank, dim);
- gtOffs = gtNewIconNode(offs, TYP_I_IMPL);
- gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArrClone, gtOffs);
- GenTree* gtLength = gtNewIndir(TYP_INT, gtAddr);
- gtLength->gtFlags |= GTF_IND_INVARIANT;
-
- GenTree* gtSum = gtNewOperNode(GT_ADD, TYP_INT, gtLowerBound, gtLength);
- GenTree* gtOne = gtNewIconNode(1, TYP_INT);
- retNode = gtNewOperNode(GT_SUB, TYP_INT, gtSum, gtOne);
- break;
- }
- default:
- unreached();
- }
- }
- }
- }
- }
- break;
- }
-
- case NI_System_Buffers_Binary_BinaryPrimitives_ReverseEndianness:
- {
- assert(sig->numArgs == 1);
-
- // We expect the return type of the ReverseEndianness routine to match the type of the
- // one and only argument to the method. We use a special instruction for 16-bit
- // BSWAPs since on x86 processors this is implemented as ROR <16-bit reg>, 8. Additionally,
- // we only emit 64-bit BSWAP instructions on 64-bit archs; if we're asked to perform a
- // 64-bit byte swap on a 32-bit arch, we'll fall to the default case in the switch block below.
-
- switch (sig->retType)
- {
- case CorInfoType::CORINFO_TYPE_SHORT:
- case CorInfoType::CORINFO_TYPE_USHORT:
- retNode = gtNewCastNode(TYP_INT, gtNewOperNode(GT_BSWAP16, TYP_INT, impPopStack().val), false,
- callType);
- break;
-
- case CorInfoType::CORINFO_TYPE_INT:
- case CorInfoType::CORINFO_TYPE_UINT:
-#ifdef TARGET_64BIT
- case CorInfoType::CORINFO_TYPE_LONG:
- case CorInfoType::CORINFO_TYPE_ULONG:
-#endif // TARGET_64BIT
- retNode = gtNewOperNode(GT_BSWAP, callType, impPopStack().val);
- break;
-
- default:
- // This default case gets hit on 32-bit archs when a call to a 64-bit overload
- // of ReverseEndianness is encountered. In that case we'll let JIT treat this as a standard
- // method call, where the implementation decomposes the operation into two 32-bit
- // bswap routines. If the input to the 64-bit function is a constant, then we rely
- // on inlining + constant folding of 32-bit bswaps to effectively constant fold
- // the 64-bit call site.
- break;
- }
-
- break;
- }
-
- // Fold PopCount for constant input
- case NI_System_Numerics_BitOperations_PopCount:
- {
- assert(sig->numArgs == 1);
- if (impStackTop().val->IsIntegralConst())
- {
- typeInfo argType = verParseArgSigToTypeInfo(sig, sig->args).NormaliseForStack();
- INT64 cns = impPopStack().val->AsIntConCommon()->IntegralValue();
- if (argType.IsType(TI_LONG))
- {
- retNode = gtNewIconNode(genCountBits(cns), callType);
- }
- else
- {
- assert(argType.IsType(TI_INT));
- retNode = gtNewIconNode(genCountBits(static_cast<unsigned>(cns)), callType);
- }
- }
- break;
- }
-
- case NI_System_GC_KeepAlive:
- {
- retNode = impKeepAliveIntrinsic(impPopStack().val);
- break;
- }
-
- case NI_System_BitConverter_DoubleToInt64Bits:
- {
- GenTree* op1 = impStackTop().val;
- assert(varTypeIsFloating(op1));
-
- if (op1->IsCnsFltOrDbl())
- {
- impPopStack();
-
- double f64Cns = op1->AsDblCon()->DconValue();
- retNode = gtNewLconNode(*reinterpret_cast<int64_t*>(&f64Cns));
- }
-#if TARGET_64BIT
- else
- {
- // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work
- impPopStack();
-
- if (op1->TypeGet() != TYP_DOUBLE)
- {
- op1 = gtNewCastNode(TYP_DOUBLE, op1, false, TYP_DOUBLE);
- }
- retNode = gtNewBitCastNode(TYP_LONG, op1);
- }
-#endif
- break;
- }
-
- case NI_System_BitConverter_Int32BitsToSingle:
- {
- GenTree* op1 = impPopStack().val;
- assert(varTypeIsInt(op1));
-
- if (op1->IsIntegralConst())
- {
- int32_t i32Cns = (int32_t)op1->AsIntConCommon()->IconValue();
- retNode = gtNewDconNode(*reinterpret_cast<float*>(&i32Cns), TYP_FLOAT);
- }
- else
- {
- retNode = gtNewBitCastNode(TYP_FLOAT, op1);
- }
- break;
- }
-
- case NI_System_BitConverter_Int64BitsToDouble:
- {
- GenTree* op1 = impStackTop().val;
- assert(varTypeIsLong(op1));
-
- if (op1->IsIntegralConst())
- {
- impPopStack();
-
- int64_t i64Cns = op1->AsIntConCommon()->LngValue();
- retNode = gtNewDconNode(*reinterpret_cast<double*>(&i64Cns));
- }
-#if TARGET_64BIT
- else
- {
- // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work
- impPopStack();
-
- retNode = gtNewBitCastNode(TYP_DOUBLE, op1);
- }
-#endif
- break;
- }
-
- case NI_System_BitConverter_SingleToInt32Bits:
- {
- GenTree* op1 = impPopStack().val;
- assert(varTypeIsFloating(op1));
-
- if (op1->IsCnsFltOrDbl())
- {
- float f32Cns = (float)op1->AsDblCon()->DconValue();
- retNode = gtNewIconNode(*reinterpret_cast<int32_t*>(&f32Cns));
- }
- else
- {
- if (op1->TypeGet() != TYP_FLOAT)
- {
- op1 = gtNewCastNode(TYP_FLOAT, op1, false, TYP_FLOAT);
- }
- retNode = gtNewBitCastNode(TYP_INT, op1);
- }
- break;
- }
-
- default:
- break;
- }
- }
-
- if (mustExpand && (retNode == nullptr))
- {
- assert(!"Unhandled must expand intrinsic, throwing PlatformNotSupportedException");
- return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
- }
-
- // Optionally report if this intrinsic is special
- // (that is, potentially re-optimizable during morph).
- if (isSpecialIntrinsic != nullptr)
- {
- *isSpecialIntrinsic = isSpecial;
- }
-
- return retNode;
-}
-
-GenTree* Compiler::impSRCSUnsafeIntrinsic(NamedIntrinsic intrinsic,
- CORINFO_CLASS_HANDLE clsHnd,
- CORINFO_METHOD_HANDLE method,
- CORINFO_SIG_INFO* sig)
-{
- // NextCallRetAddr requires a CALL, so return nullptr.
- if (info.compHasNextCallRetAddr)
- {
- return nullptr;
- }
-
- assert(sig->sigInst.classInstCount == 0);
-
- switch (intrinsic)
- {
- case NI_SRCS_UNSAFE_Add:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldarg.1
- // sizeof !!T
- // conv.i
- // mul
- // add
- // ret
-
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
- impBashVarAddrsToI(op1, op2);
-
- op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL);
-
- unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]);
-
- if (classSize != 1)
- {
- GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL);
- op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size);
- }
-
- var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2);
- return gtNewOperNode(GT_ADD, type, op1, op2);
- }
-
- case NI_SRCS_UNSAFE_AddByteOffset:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldarg.1
- // add
- // ret
-
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
- impBashVarAddrsToI(op1, op2);
-
- var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2);
- return gtNewOperNode(GT_ADD, type, op1, op2);
- }
-
- case NI_SRCS_UNSAFE_AreSame:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldarg.1
- // ceq
- // ret
-
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
-
- GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, op2);
- return gtFoldExpr(tmp);
- }
-
- case NI_SRCS_UNSAFE_As:
- {
- assert((sig->sigInst.methInstCount == 1) || (sig->sigInst.methInstCount == 2));
-
- // ldarg.0
- // ret
-
- return impPopStack().val;
- }
-
- case NI_SRCS_UNSAFE_AsPointer:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // conv.u
- // ret
-
- GenTree* op1 = impPopStack().val;
- impBashVarAddrsToI(op1);
-
- return gtNewCastNode(TYP_I_IMPL, op1, /* uns */ false, TYP_I_IMPL);
- }
-
- case NI_SRCS_UNSAFE_AsRef:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ret
-
- return impPopStack().val;
- }
-
- case NI_SRCS_UNSAFE_ByteOffset:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.1
- // ldarg.0
- // sub
- // ret
-
- impSpillSideEffect(true, verCurrentState.esStackDepth -
- 2 DEBUGARG("Spilling op1 side effects for Unsafe.ByteOffset"));
-
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
- impBashVarAddrsToI(op1, op2);
-
- var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op2, &op1);
- return gtNewOperNode(GT_SUB, type, op2, op1);
- }
-
- case NI_SRCS_UNSAFE_Copy:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldarg.1
- // ldobj !!T
- // stobj !!T
- // ret
-
- return nullptr;
- }
-
- case NI_SRCS_UNSAFE_CopyBlock:
- {
- assert(sig->sigInst.methInstCount == 0);
-
- // ldarg.0
- // ldarg.1
- // ldarg.2
- // cpblk
- // ret
-
- return nullptr;
- }
-
- case NI_SRCS_UNSAFE_CopyBlockUnaligned:
- {
- assert(sig->sigInst.methInstCount == 0);
-
- // ldarg.0
- // ldarg.1
- // ldarg.2
- // unaligned. 0x1
- // cpblk
- // ret
-
- return nullptr;
- }
-
- case NI_SRCS_UNSAFE_InitBlock:
- {
- assert(sig->sigInst.methInstCount == 0);
-
- // ldarg.0
- // ldarg.1
- // ldarg.2
- // initblk
- // ret
-
- return nullptr;
- }
-
- case NI_SRCS_UNSAFE_InitBlockUnaligned:
- {
- assert(sig->sigInst.methInstCount == 0);
-
- // ldarg.0
- // ldarg.1
- // ldarg.2
- // unaligned. 0x1
- // initblk
- // ret
-
- return nullptr;
- }
-
- case NI_SRCS_UNSAFE_IsAddressGreaterThan:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldarg.1
- // cgt.un
- // ret
-
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
-
- GenTree* tmp = gtNewOperNode(GT_GT, TYP_INT, op1, op2);
- tmp->gtFlags |= GTF_UNSIGNED;
- return gtFoldExpr(tmp);
- }
-
- case NI_SRCS_UNSAFE_IsAddressLessThan:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldarg.1
- // clt.un
- // ret
-
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
-
- GenTree* tmp = gtNewOperNode(GT_LT, TYP_INT, op1, op2);
- tmp->gtFlags |= GTF_UNSIGNED;
- return gtFoldExpr(tmp);
- }
-
- case NI_SRCS_UNSAFE_IsNullRef:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldc.i4.0
- // conv.u
- // ceq
- // ret
-
- GenTree* op1 = impPopStack().val;
- GenTree* cns = gtNewIconNode(0, TYP_BYREF);
- GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, cns);
- return gtFoldExpr(tmp);
- }
-
- case NI_SRCS_UNSAFE_NullRef:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldc.i4.0
- // conv.u
- // ret
-
- return gtNewIconNode(0, TYP_BYREF);
- }
-
- case NI_SRCS_UNSAFE_Read:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldobj !!T
- // ret
-
- return nullptr;
- }
-
- case NI_SRCS_UNSAFE_ReadUnaligned:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // unaligned. 0x1
- // ldobj !!T
- // ret
-
- return nullptr;
- }
-
- case NI_SRCS_UNSAFE_SizeOf:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // sizeof !!T
- // ret
-
- unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]);
- return gtNewIconNode(classSize, TYP_INT);
- }
-
- case NI_SRCS_UNSAFE_SkipInit:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ret
-
- GenTree* op1 = impPopStack().val;
-
- if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
- {
- return gtUnusedValNode(op1);
- }
- else
- {
- return gtNewNothingNode();
- }
- }
-
- case NI_SRCS_UNSAFE_Subtract:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldarg.1
- // sizeof !!T
- // conv.i
- // mul
- // sub
- // ret
-
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
- impBashVarAddrsToI(op1, op2);
-
- op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL);
-
- unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]);
-
- if (classSize != 1)
- {
- GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL);
- op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size);
- }
-
- var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2);
- return gtNewOperNode(GT_SUB, type, op1, op2);
- }
-
- case NI_SRCS_UNSAFE_SubtractByteOffset:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldarg.1
- // sub
- // ret
-
- GenTree* op2 = impPopStack().val;
- GenTree* op1 = impPopStack().val;
- impBashVarAddrsToI(op1, op2);
-
- var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2);
- return gtNewOperNode(GT_SUB, type, op1, op2);
- }
-
- case NI_SRCS_UNSAFE_Unbox:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // unbox !!T
- // ret
-
- return nullptr;
- }
-
- case NI_SRCS_UNSAFE_Write:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldarg.1
- // stobj !!T
- // ret
-
- return nullptr;
- }
-
- case NI_SRCS_UNSAFE_WriteUnaligned:
- {
- assert(sig->sigInst.methInstCount == 1);
-
- // ldarg.0
- // ldarg.1
- // unaligned. 0x01
- // stobj !!T
- // ret
-
- return nullptr;
- }
-
- default:
- {
- unreached();
- }
- }
-}
-
GenTree* Compiler::impTypeIsAssignable(GenTree* typeTo, GenTree* typeFrom)
{
// Optimize patterns like:
}
}
}
+
+//------------------------------------------------------------------------
+// impInitializeArrayIntrinsic: Attempts to replace a call to InitializeArray
+// with a GT_COPYBLK node.
+//
+// Arguments:
+// sig - The InitializeArray signature.
+//
+// Return Value:
+// A pointer to the newly created GT_COPYBLK node if the replacement succeeds or
+// nullptr otherwise.
+//
+// Notes:
+// The function recognizes the following IL pattern:
+// ldc <length> or a list of ldc <lower bound>/<length>
+// newarr or newobj
+// dup
+// ldtoken <field handle>
+// call InitializeArray
+// The lower bounds need not be constant except when the array rank is 1.
+// The function recognizes all kinds of arrays thus enabling a small runtime
+// such as NativeAOT to skip providing an implementation for InitializeArray.
+
+GenTree* Compiler::impInitializeArrayIntrinsic(CORINFO_SIG_INFO* sig)
+{
+ assert(sig->numArgs == 2);
+
+ GenTree* fieldTokenNode = impStackTop(0).val;
+ GenTree* arrayLocalNode = impStackTop(1).val;
+
+ //
+ // Verify that the field token is known and valid. Note that it's also
+ // possible for the token to come from reflection, in which case we cannot do
+ // the optimization and must therefore revert to calling the helper. You can
+ // see an example of this in bvt\DynIL\initarray2.exe (in Main).
+ //
+
+ // Check to see if the ldtoken helper call is what we see here.
+ if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) ||
+ (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD)))
+ {
+ return nullptr;
+ }
+
+ // Strip helper call away
+ fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode();
+
+ if (fieldTokenNode->gtOper == GT_IND)
+ {
+ fieldTokenNode = fieldTokenNode->AsOp()->gtOp1;
+ }
+
+ // Check for constant
+ if (fieldTokenNode->gtOper != GT_CNS_INT)
+ {
+ return nullptr;
+ }
+
+ CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle;
+ if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr))
+ {
+ return nullptr;
+ }
+
+ //
+ // We need to get the number of elements in the array and the size of each element.
+ // We verify that the newarr statement is exactly what we expect it to be.
+ // If it's not then we just return NULL and we don't optimize this call
+ //
+
+ // It is possible the we don't have any statements in the block yet.
+ if (impLastStmt == nullptr)
+ {
+ return nullptr;
+ }
+
+ //
+ // We start by looking at the last statement, making sure it's an assignment, and
+ // that the target of the assignment is the array passed to InitializeArray.
+ //
+ GenTree* arrayAssignment = impLastStmt->GetRootNode();
+ if ((arrayAssignment->gtOper != GT_ASG) || (arrayAssignment->AsOp()->gtOp1->gtOper != GT_LCL_VAR) ||
+ (arrayLocalNode->gtOper != GT_LCL_VAR) || (arrayAssignment->AsOp()->gtOp1->AsLclVarCommon()->GetLclNum() !=
+ arrayLocalNode->AsLclVarCommon()->GetLclNum()))
+ {
+ return nullptr;
+ }
+
+ //
+ // Make sure that the object being assigned is a helper call.
+ //
+
+ GenTree* newArrayCall = arrayAssignment->AsOp()->gtOp2;
+ if ((newArrayCall->gtOper != GT_CALL) || (newArrayCall->AsCall()->gtCallType != CT_HELPER))
+ {
+ return nullptr;
+ }
+
+ //
+ // Verify that it is one of the new array helpers.
+ //
+
+ bool isMDArray = false;
+
+ if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_DIRECT) &&
+ newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_OBJ) &&
+ newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_VC) &&
+ newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_ALIGN8)
+#ifdef FEATURE_READYTORUN
+ && newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1)
+#endif
+ )
+ {
+ if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEW_MDARR))
+ {
+ return nullptr;
+ }
+
+ isMDArray = true;
+ }
+
+ CORINFO_CLASS_HANDLE arrayClsHnd = (CORINFO_CLASS_HANDLE)newArrayCall->AsCall()->compileTimeHelperArgumentHandle;
+
+ //
+ // Make sure we found a compile time handle to the array
+ //
+
+ if (!arrayClsHnd)
+ {
+ return nullptr;
+ }
+
+ unsigned rank = 0;
+ S_UINT32 numElements;
+
+ if (isMDArray)
+ {
+ rank = info.compCompHnd->getArrayRank(arrayClsHnd);
+
+ if (rank == 0)
+ {
+ return nullptr;
+ }
+
+ assert(newArrayCall->AsCall()->gtArgs.CountArgs() == 3);
+ GenTree* numArgsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode();
+ GenTree* argsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(2)->GetNode();
+
+ //
+ // The number of arguments should be a constant between 1 and 64. The rank can't be 0
+ // so at least one length must be present and the rank can't exceed 32 so there can
+ // be at most 64 arguments - 32 lengths and 32 lower bounds.
+ //
+
+ if ((!numArgsArg->IsCnsIntOrI()) || (numArgsArg->AsIntCon()->IconValue() < 1) ||
+ (numArgsArg->AsIntCon()->IconValue() > 64))
+ {
+ return nullptr;
+ }
+
+ unsigned numArgs = static_cast<unsigned>(numArgsArg->AsIntCon()->IconValue());
+ bool lowerBoundsSpecified;
+
+ if (numArgs == rank * 2)
+ {
+ lowerBoundsSpecified = true;
+ }
+ else if (numArgs == rank)
+ {
+ lowerBoundsSpecified = false;
+
+ //
+ // If the rank is 1 and a lower bound isn't specified then the runtime creates
+ // a SDArray. Note that even if a lower bound is specified it can be 0 and then
+ // we get a SDArray as well, see the for loop below.
+ //
+
+ if (rank == 1)
+ {
+ isMDArray = false;
+ }
+ }
+ else
+ {
+ return nullptr;
+ }
+
+ //
+ // The rank is known to be at least 1 so we can start with numElements being 1
+ // to avoid the need to special case the first dimension.
+ //
+
+ numElements = S_UINT32(1);
+
+ struct Match
+ {
+ static bool IsArgsFieldInit(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs)
+ {
+ return (tree->OperGet() == GT_ASG) && IsArgsFieldIndir(tree->gtGetOp1(), index, lvaNewObjArrayArgs) &&
+ IsArgsAddr(tree->gtGetOp1()->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs);
+ }
+
+ static bool IsArgsFieldIndir(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs)
+ {
+ return (tree->OperGet() == GT_IND) && (tree->gtGetOp1()->OperGet() == GT_ADD) &&
+ (tree->gtGetOp1()->gtGetOp2()->IsIntegralConst(sizeof(INT32) * index)) &&
+ IsArgsAddr(tree->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs);
+ }
+
+ static bool IsArgsAddr(GenTree* tree, unsigned lvaNewObjArrayArgs)
+ {
+ return (tree->OperGet() == GT_ADDR) && (tree->gtGetOp1()->OperGet() == GT_LCL_VAR) &&
+ (tree->gtGetOp1()->AsLclVar()->GetLclNum() == lvaNewObjArrayArgs);
+ }
+
+ static bool IsComma(GenTree* tree)
+ {
+ return (tree != nullptr) && (tree->OperGet() == GT_COMMA);
+ }
+ };
+
+ unsigned argIndex = 0;
+ GenTree* comma;
+
+ for (comma = argsArg; Match::IsComma(comma); comma = comma->gtGetOp2())
+ {
+ if (lowerBoundsSpecified)
+ {
+ //
+ // In general lower bounds can be ignored because they're not needed to
+ // calculate the total number of elements. But for single dimensional arrays
+ // we need to know if the lower bound is 0 because in this case the runtime
+ // creates a SDArray and this affects the way the array data offset is calculated.
+ //
+
+ if (rank == 1)
+ {
+ GenTree* lowerBoundAssign = comma->gtGetOp1();
+ assert(Match::IsArgsFieldInit(lowerBoundAssign, argIndex, lvaNewObjArrayArgs));
+ GenTree* lowerBoundNode = lowerBoundAssign->gtGetOp2();
+
+ if (lowerBoundNode->IsIntegralConst(0))
+ {
+ isMDArray = false;
+ }
+ }
+
+ comma = comma->gtGetOp2();
+ argIndex++;
+ }
+
+ GenTree* lengthNodeAssign = comma->gtGetOp1();
+ assert(Match::IsArgsFieldInit(lengthNodeAssign, argIndex, lvaNewObjArrayArgs));
+ GenTree* lengthNode = lengthNodeAssign->gtGetOp2();
+
+ if (!lengthNode->IsCnsIntOrI())
+ {
+ return nullptr;
+ }
+
+ numElements *= S_SIZE_T(lengthNode->AsIntCon()->IconValue());
+ argIndex++;
+ }
+
+ assert((comma != nullptr) && Match::IsArgsAddr(comma, lvaNewObjArrayArgs));
+
+ if (argIndex != numArgs)
+ {
+ return nullptr;
+ }
+ }
+ else
+ {
+ //
+ // Make sure there are exactly two arguments: the array class and
+ // the number of elements.
+ //
+
+ GenTree* arrayLengthNode;
+
+#ifdef FEATURE_READYTORUN
+ if (newArrayCall->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1))
+ {
+ // Array length is 1st argument for readytorun helper
+ arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(0)->GetNode();
+ }
+ else
+#endif
+ {
+ // Array length is 2nd argument for regular helper
+ arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode();
+ }
+
+ //
+ // This optimization is only valid for a constant array size.
+ //
+ if (arrayLengthNode->gtOper != GT_CNS_INT)
+ {
+ return nullptr;
+ }
+
+ numElements = S_SIZE_T(arrayLengthNode->AsIntCon()->gtIconVal);
+
+ if (!info.compCompHnd->isSDArray(arrayClsHnd))
+ {
+ return nullptr;
+ }
+ }
+
+ CORINFO_CLASS_HANDLE elemClsHnd;
+ var_types elementType = JITtype2varType(info.compCompHnd->getChildType(arrayClsHnd, &elemClsHnd));
+
+ //
+ // Note that genTypeSize will return zero for non primitive types, which is exactly
+ // what we want (size will then be 0, and we will catch this in the conditional below).
+ // Note that we don't expect this to fail for valid binaries, so we assert in the
+ // non-verification case (the verification case should not assert but rather correctly
+ // handle bad binaries). This assert is not guarding any specific invariant, but rather
+ // saying that we don't expect this to happen, and if it is hit, we need to investigate
+ // why.
+ //
+
+ S_UINT32 elemSize(genTypeSize(elementType));
+ S_UINT32 size = elemSize * S_UINT32(numElements);
+
+ if (size.IsOverflow())
+ {
+ return nullptr;
+ }
+
+ if ((size.Value() == 0) || (varTypeIsGC(elementType)))
+ {
+ return nullptr;
+ }
+
+ void* initData = info.compCompHnd->getArrayInitializationData(fieldToken, size.Value());
+ if (!initData)
+ {
+ return nullptr;
+ }
+
+ //
+ // At this point we are ready to commit to implementing the InitializeArray
+ // intrinsic using a struct assignment. Pop the arguments from the stack and
+ // return the struct assignment node.
+ //
+
+ impPopStack();
+ impPopStack();
+
+ const unsigned blkSize = size.Value();
+ unsigned dataOffset;
+
+ if (isMDArray)
+ {
+ dataOffset = eeGetMDArrayDataOffset(rank);
+ }
+ else
+ {
+ dataOffset = eeGetArrayDataOffset();
+ }
+
+ GenTree* dstAddr = gtNewOperNode(GT_ADD, TYP_BYREF, arrayLocalNode, gtNewIconNode(dataOffset, TYP_I_IMPL));
+ GenTree* dst = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, dstAddr, typGetBlkLayout(blkSize));
+ GenTree* src = gtNewIndOfIconHandleNode(TYP_STRUCT, (size_t)initData, GTF_ICON_CONST_PTR, true);
+
+#ifdef DEBUG
+ src->gtGetOp1()->AsIntCon()->gtTargetHandle = THT_InitializeArrayIntrinsics;
+#endif
+
+ return gtNewBlkOpNode(dst, // dst
+ src, // src
+ false, // volatile
+ true); // copyBlock
+}
+
+GenTree* Compiler::impCreateSpanIntrinsic(CORINFO_SIG_INFO* sig)
+{
+ assert(sig->numArgs == 1);
+ assert(sig->sigInst.methInstCount == 1);
+
+ GenTree* fieldTokenNode = impStackTop(0).val;
+
+ //
+ // Verify that the field token is known and valid. Note that it's also
+ // possible for the token to come from reflection, in which case we cannot do
+ // the optimization and must therefore revert to calling the helper. You can
+ // see an example of this in bvt\DynIL\initarray2.exe (in Main).
+ //
+
+ // Check to see if the ldtoken helper call is what we see here.
+ if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) ||
+ (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD)))
+ {
+ return nullptr;
+ }
+
+ // Strip helper call away
+ fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetNode();
+ if (fieldTokenNode->gtOper == GT_IND)
+ {
+ fieldTokenNode = fieldTokenNode->AsOp()->gtOp1;
+ }
+
+ // Check for constant
+ if (fieldTokenNode->gtOper != GT_CNS_INT)
+ {
+ return nullptr;
+ }
+
+ CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle;
+ if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr))
+ {
+ return nullptr;
+ }
+
+ CORINFO_CLASS_HANDLE fieldOwnerHnd = info.compCompHnd->getFieldClass(fieldToken);
+
+ CORINFO_CLASS_HANDLE fieldClsHnd;
+ var_types fieldElementType =
+ JITtype2varType(info.compCompHnd->getFieldType(fieldToken, &fieldClsHnd, fieldOwnerHnd));
+ unsigned totalFieldSize;
+
+ // Most static initialization data fields are of some structure, but it is possible for them to be of various
+ // primitive types as well
+ if (fieldElementType == var_types::TYP_STRUCT)
+ {
+ totalFieldSize = info.compCompHnd->getClassSize(fieldClsHnd);
+ }
+ else
+ {
+ totalFieldSize = genTypeSize(fieldElementType);
+ }
+
+ // Limit to primitive or enum type - see ArrayNative::GetSpanDataFrom()
+ CORINFO_CLASS_HANDLE targetElemHnd = sig->sigInst.methInst[0];
+ if (info.compCompHnd->getTypeForPrimitiveValueClass(targetElemHnd) == CORINFO_TYPE_UNDEF)
+ {
+ return nullptr;
+ }
+
+ const unsigned targetElemSize = info.compCompHnd->getClassSize(targetElemHnd);
+ assert(targetElemSize != 0);
+
+ const unsigned count = totalFieldSize / targetElemSize;
+ if (count == 0)
+ {
+ return nullptr;
+ }
+
+ void* data = info.compCompHnd->getArrayInitializationData(fieldToken, totalFieldSize);
+ if (!data)
+ {
+ return nullptr;
+ }
+
+ //
+ // Ready to commit to the work
+ //
+
+ impPopStack();
+
+ // Turn count and pointer value into constants.
+ GenTree* lengthValue = gtNewIconNode(count, TYP_INT);
+ GenTree* pointerValue = gtNewIconHandleNode((size_t)data, GTF_ICON_CONST_PTR);
+
+ // Construct ReadOnlySpan<T> to return.
+ CORINFO_CLASS_HANDLE spanHnd = sig->retTypeClass;
+ unsigned spanTempNum = lvaGrabTemp(true DEBUGARG("ReadOnlySpan<T> for CreateSpan<T>"));
+ lvaSetStruct(spanTempNum, spanHnd, false);
+
+ GenTreeLclFld* pointerField = gtNewLclFldNode(spanTempNum, TYP_BYREF, 0);
+ GenTree* pointerFieldAsg = gtNewAssignNode(pointerField, pointerValue);
+
+ GenTreeLclFld* lengthField = gtNewLclFldNode(spanTempNum, TYP_INT, TARGET_POINTER_SIZE);
+ GenTree* lengthFieldAsg = gtNewAssignNode(lengthField, lengthValue);
+
+ // Now append a few statements the initialize the span
+ impAppendTree(lengthFieldAsg, CHECK_SPILL_NONE, impCurStmtDI);
+ impAppendTree(pointerFieldAsg, CHECK_SPILL_NONE, impCurStmtDI);
+
+ // And finally create a tree that points at the span.
+ return impCreateLocalNode(spanTempNum DEBUGARG(0));
+}
+
+//------------------------------------------------------------------------
+// impIntrinsic: possibly expand intrinsic call into alternate IR sequence
+//
+// Arguments:
+// newobjThis - for constructor calls, the tree for the newly allocated object
+// clsHnd - handle for the intrinsic method's class
+// method - handle for the intrinsic method
+// sig - signature of the intrinsic method
+// methodFlags - CORINFO_FLG_XXX flags of the intrinsic method
+// memberRef - the token for the intrinsic method
+// readonlyCall - true if call has a readonly prefix
+// tailCall - true if call is in tail position
+// pConstrainedResolvedToken -- resolved token for constrained call, or nullptr
+// if call is not constrained
+// constraintCallThisTransform -- this transform to apply for a constrained call
+// pIntrinsicName [OUT] -- intrinsic name (see enumeration in namedintrinsiclist.h)
+// for "traditional" jit intrinsics
+// isSpecialIntrinsic [OUT] -- set true if intrinsic expansion is a call
+// that is amenable to special downstream optimization opportunities
+//
+// Returns:
+// IR tree to use in place of the call, or nullptr if the jit should treat
+// the intrinsic call like a normal call.
+//
+// pIntrinsicName set to non-illegal value if the call is recognized as a
+// traditional jit intrinsic, even if the intrinsic is not expaned.
+//
+// isSpecial set true if the expansion is subject to special
+// optimizations later in the jit processing
+//
+// Notes:
+// On success the IR tree may be a call to a different method or an inline
+// sequence. If it is a call, then the intrinsic processing here is responsible
+// for handling all the special cases, as upon return to impImportCall
+// expanded intrinsics bypass most of the normal call processing.
+//
+// Intrinsics are generally not recognized in minopts and debug codegen.
+//
+// However, certain traditional intrinsics are identifed as "must expand"
+// if there is no fallback implementation to invoke; these must be handled
+// in all codegen modes.
+//
+// New style intrinsics (where the fallback implementation is in IL) are
+// identified as "must expand" if they are invoked from within their
+// own method bodies.
+//
+GenTree* Compiler::impIntrinsic(GenTree* newobjThis,
+ CORINFO_CLASS_HANDLE clsHnd,
+ CORINFO_METHOD_HANDLE method,
+ CORINFO_SIG_INFO* sig,
+ unsigned methodFlags,
+ int memberRef,
+ bool readonlyCall,
+ bool tailCall,
+ CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken,
+ CORINFO_THIS_TRANSFORM constraintCallThisTransform,
+ NamedIntrinsic* pIntrinsicName,
+ bool* isSpecialIntrinsic)
+{
+ bool mustExpand = false;
+ bool isSpecial = false;
+ const bool isIntrinsic = (methodFlags & CORINFO_FLG_INTRINSIC) != 0;
+
+ NamedIntrinsic ni = lookupNamedIntrinsic(method);
+
+ if (isIntrinsic)
+ {
+ // The recursive non-virtual calls to Jit intrinsics are must-expand by convention.
+ mustExpand = gtIsRecursiveCall(method) && !(methodFlags & CORINFO_FLG_VIRTUAL);
+ }
+ else
+ {
+ // For mismatched VM (AltJit) we want to check all methods as intrinsic to ensure
+ // we get more accurate codegen. This particularly applies to HWIntrinsic usage
+ assert(!info.compMatchedVM);
+ }
+
+ // We specially support the following on all platforms to allow for dead
+ // code optimization and to more generally support recursive intrinsics.
+
+ if (isIntrinsic)
+ {
+ if (ni == NI_IsSupported_True)
+ {
+ assert(sig->numArgs == 0);
+ return gtNewIconNode(true);
+ }
+
+ if (ni == NI_IsSupported_False)
+ {
+ assert(sig->numArgs == 0);
+ return gtNewIconNode(false);
+ }
+
+ if (ni == NI_Throw_PlatformNotSupportedException)
+ {
+ return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
+ }
+
+ if ((ni > NI_SRCS_UNSAFE_START) && (ni < NI_SRCS_UNSAFE_END))
+ {
+ assert(!mustExpand);
+ return impSRCSUnsafeIntrinsic(ni, clsHnd, method, sig);
+ }
+ }
+
+#ifdef FEATURE_HW_INTRINSICS
+ if ((ni > NI_HW_INTRINSIC_START) && (ni < NI_HW_INTRINSIC_END))
+ {
+ if (!isIntrinsic)
+ {
+#if defined(TARGET_XARCH)
+ // We can't guarantee that all overloads for the xplat intrinsics can be
+ // handled by the AltJit, so limit only the platform specific intrinsics
+ assert((NI_Vector256_Xor + 1) == NI_X86Base_BitScanForward);
+
+ if (ni < NI_Vector256_Xor)
+#elif defined(TARGET_ARM64)
+ // We can't guarantee that all overloads for the xplat intrinsics can be
+ // handled by the AltJit, so limit only the platform specific intrinsics
+ assert((NI_Vector128_Xor + 1) == NI_AdvSimd_Abs);
+
+ if (ni < NI_Vector128_Xor)
+#else
+#error Unsupported platform
+#endif
+ {
+ // Several of the NI_Vector64/128/256 APIs do not have
+ // all overloads as intrinsic today so they will assert
+ return nullptr;
+ }
+ }
+
+ GenTree* hwintrinsic = impHWIntrinsic(ni, clsHnd, method, sig, mustExpand);
+
+ if (mustExpand && (hwintrinsic == nullptr))
+ {
+ return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_NOT_IMPLEMENTED, method, sig, mustExpand);
+ }
+
+ return hwintrinsic;
+ }
+
+ if (isIntrinsic && (ni > NI_SIMD_AS_HWINTRINSIC_START) && (ni < NI_SIMD_AS_HWINTRINSIC_END))
+ {
+ // These intrinsics aren't defined recursively and so they will never be mustExpand
+ // Instead, they provide software fallbacks that will be executed instead.
+
+ assert(!mustExpand);
+ return impSimdAsHWIntrinsic(ni, clsHnd, method, sig, newobjThis);
+ }
+#endif // FEATURE_HW_INTRINSICS
+
+ if (!isIntrinsic)
+ {
+ // Outside the cases above, there are many intrinsics which apply to only a
+ // subset of overload and where simply matching by name may cause downstream
+ // asserts or other failures. Math.Min is one example, where it only applies
+ // to the floating-point overloads.
+ return nullptr;
+ }
+
+ *pIntrinsicName = ni;
+
+ if (ni == NI_System_StubHelpers_GetStubContext)
+ {
+ // must be done regardless of DbgCode and MinOpts
+ return gtNewLclvNode(lvaStubArgumentVar, TYP_I_IMPL);
+ }
+
+ if (ni == NI_System_StubHelpers_NextCallReturnAddress)
+ {
+ // For now we just avoid inlining anything into these methods since
+ // this intrinsic is only rarely used. We could do this better if we
+ // wanted to by trying to match which call is the one we need to get
+ // the return address of.
+ info.compHasNextCallRetAddr = true;
+ return new (this, GT_LABEL) GenTree(GT_LABEL, TYP_I_IMPL);
+ }
+
+ switch (ni)
+ {
+ // CreateSpan must be expanded for NativeAOT
+ case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan:
+ case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray:
+ mustExpand |= IsTargetAbi(CORINFO_NATIVEAOT_ABI);
+ break;
+
+ case NI_Internal_Runtime_MethodTable_Of:
+ case NI_System_Activator_AllocatorOf:
+ case NI_System_Activator_DefaultConstructorOf:
+ case NI_System_EETypePtr_EETypePtrOf:
+ mustExpand = true;
+ break;
+
+ default:
+ break;
+ }
+
+ GenTree* retNode = nullptr;
+
+ // Under debug and minopts, only expand what is required.
+ // NextCallReturnAddress intrinsic returns the return address of the next call.
+ // If that call is an intrinsic and is expanded, codegen for NextCallReturnAddress will fail.
+ // To avoid that we conservatively expand only required intrinsics in methods that call
+ // the NextCallReturnAddress intrinsic.
+ if (!mustExpand && (opts.OptimizationDisabled() || info.compHasNextCallRetAddr))
+ {
+ *pIntrinsicName = NI_Illegal;
+ return retNode;
+ }
+
+ CorInfoType callJitType = sig->retType;
+ var_types callType = JITtype2varType(callJitType);
+
+ /* First do the intrinsics which are always smaller than a call */
+
+ if (ni != NI_Illegal)
+ {
+ assert(retNode == nullptr);
+ switch (ni)
+ {
+ case NI_Array_Address:
+ case NI_Array_Get:
+ case NI_Array_Set:
+ retNode = impArrayAccessIntrinsic(clsHnd, sig, memberRef, readonlyCall, ni);
+ break;
+
+ case NI_System_String_Equals:
+ {
+ retNode = impStringEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags);
+ break;
+ }
+
+ case NI_System_MemoryExtensions_Equals:
+ case NI_System_MemoryExtensions_SequenceEqual:
+ {
+ retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags);
+ break;
+ }
+
+ case NI_System_String_StartsWith:
+ {
+ retNode = impStringEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags);
+ break;
+ }
+
+ case NI_System_MemoryExtensions_StartsWith:
+ {
+ retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags);
+ break;
+ }
+
+ case NI_System_MemoryExtensions_AsSpan:
+ case NI_System_String_op_Implicit:
+ {
+ assert(sig->numArgs == 1);
+ isSpecial = impStackTop().val->OperIs(GT_CNS_STR);
+ break;
+ }
+
+ case NI_System_String_get_Chars:
+ {
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+ GenTree* addr = gtNewIndexAddr(op1, op2, TYP_USHORT, NO_CLASS_HANDLE, OFFSETOF__CORINFO_String__chars,
+ OFFSETOF__CORINFO_String__stringLen);
+ retNode = gtNewIndexIndir(addr->AsIndexAddr());
+ break;
+ }
+
+ case NI_System_String_get_Length:
+ {
+ GenTree* op1 = impPopStack().val;
+ if (op1->OperIs(GT_CNS_STR))
+ {
+ // Optimize `ldstr + String::get_Length()` to CNS_INT
+ // e.g. "Hello".Length => 5
+ GenTreeIntCon* iconNode = gtNewStringLiteralLength(op1->AsStrCon());
+ if (iconNode != nullptr)
+ {
+ retNode = iconNode;
+ break;
+ }
+ }
+ GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, op1, OFFSETOF__CORINFO_String__stringLen, compCurBB);
+ op1 = arrLen;
+
+ // Getting the length of a null string should throw
+ op1->gtFlags |= GTF_EXCEPT;
+
+ retNode = op1;
+ break;
+ }
+
+ case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan:
+ {
+ retNode = impCreateSpanIntrinsic(sig);
+ break;
+ }
+
+ case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray:
+ {
+ retNode = impInitializeArrayIntrinsic(sig);
+ break;
+ }
+
+ case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant:
+ {
+ GenTree* op1 = impPopStack().val;
+ if (op1->OperIsConst())
+ {
+ // op1 is a known constant, replace with 'true'.
+ retNode = gtNewIconNode(1);
+ JITDUMP("\nExpanding RuntimeHelpers.IsKnownConstant to true early\n");
+ // We can also consider FTN_ADDR and typeof(T) here
+ }
+ else
+ {
+ // op1 is not a known constant, we'll do the expansion in morph
+ retNode = new (this, GT_INTRINSIC) GenTreeIntrinsic(TYP_INT, op1, ni, method);
+ JITDUMP("\nConverting RuntimeHelpers.IsKnownConstant to:\n");
+ DISPTREE(retNode);
+ }
+ break;
+ }
+
+ case NI_Internal_Runtime_MethodTable_Of:
+ case NI_System_Activator_AllocatorOf:
+ case NI_System_Activator_DefaultConstructorOf:
+ case NI_System_EETypePtr_EETypePtrOf:
+ {
+ assert(IsTargetAbi(CORINFO_NATIVEAOT_ABI)); // Only NativeAOT supports it.
+ CORINFO_RESOLVED_TOKEN resolvedToken;
+ resolvedToken.tokenContext = impTokenLookupContextHandle;
+ resolvedToken.tokenScope = info.compScopeHnd;
+ resolvedToken.token = memberRef;
+ resolvedToken.tokenType = CORINFO_TOKENKIND_Method;
+
+ CORINFO_GENERICHANDLE_RESULT embedInfo;
+ info.compCompHnd->expandRawHandleIntrinsic(&resolvedToken, &embedInfo);
+
+ GenTree* rawHandle = impLookupToTree(&resolvedToken, &embedInfo.lookup, gtTokenToIconFlags(memberRef),
+ embedInfo.compileTimeHandle);
+ if (rawHandle == nullptr)
+ {
+ return nullptr;
+ }
+
+ noway_assert(genTypeSize(rawHandle->TypeGet()) == genTypeSize(TYP_I_IMPL));
+
+ unsigned rawHandleSlot = lvaGrabTemp(true DEBUGARG("rawHandle"));
+ impAssignTempGen(rawHandleSlot, rawHandle, clsHnd, CHECK_SPILL_NONE);
+
+ GenTree* lclVar = gtNewLclvNode(rawHandleSlot, TYP_I_IMPL);
+ GenTree* lclVarAddr = gtNewOperNode(GT_ADDR, TYP_I_IMPL, lclVar);
+ var_types resultType = JITtype2varType(sig->retType);
+ if (resultType == TYP_STRUCT)
+ {
+ retNode = gtNewObjNode(sig->retTypeClass, lclVarAddr);
+ }
+ else
+ {
+ retNode = gtNewIndir(resultType, lclVarAddr);
+ }
+ break;
+ }
+
+ case NI_System_Span_get_Item:
+ case NI_System_ReadOnlySpan_get_Item:
+ {
+ // Have index, stack pointer-to Span<T> s on the stack. Expand to:
+ //
+ // For Span<T>
+ // Comma
+ // BoundsCheck(index, s->_length)
+ // s->_reference + index * sizeof(T)
+ //
+ // For ReadOnlySpan<T> -- same expansion, as it now returns a readonly ref
+ //
+ // Signature should show one class type parameter, which
+ // we need to examine.
+ assert(sig->sigInst.classInstCount == 1);
+ assert(sig->numArgs == 1);
+ CORINFO_CLASS_HANDLE spanElemHnd = sig->sigInst.classInst[0];
+ const unsigned elemSize = info.compCompHnd->getClassSize(spanElemHnd);
+ assert(elemSize > 0);
+
+ const bool isReadOnly = (ni == NI_System_ReadOnlySpan_get_Item);
+
+ JITDUMP("\nimpIntrinsic: Expanding %sSpan<T>.get_Item, T=%s, sizeof(T)=%u\n",
+ isReadOnly ? "ReadOnly" : "", eeGetClassName(spanElemHnd), elemSize);
+
+ GenTree* index = impPopStack().val;
+ GenTree* ptrToSpan = impPopStack().val;
+ GenTree* indexClone = nullptr;
+ GenTree* ptrToSpanClone = nullptr;
+ assert(genActualType(index) == TYP_INT);
+ assert(ptrToSpan->TypeGet() == TYP_BYREF);
+
+#if defined(DEBUG)
+ if (verbose)
+ {
+ printf("with ptr-to-span\n");
+ gtDispTree(ptrToSpan);
+ printf("and index\n");
+ gtDispTree(index);
+ }
+#endif // defined(DEBUG)
+
+ // We need to use both index and ptr-to-span twice, so clone or spill.
+ index = impCloneExpr(index, &indexClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL,
+ nullptr DEBUGARG("Span.get_Item index"));
+ ptrToSpan = impCloneExpr(ptrToSpan, &ptrToSpanClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL,
+ nullptr DEBUGARG("Span.get_Item ptrToSpan"));
+
+ // Bounds check
+ CORINFO_FIELD_HANDLE lengthHnd = info.compCompHnd->getFieldInClass(clsHnd, 1);
+ const unsigned lengthOffset = info.compCompHnd->getFieldOffset(lengthHnd);
+ GenTree* length = gtNewFieldRef(TYP_INT, lengthHnd, ptrToSpan, lengthOffset);
+ GenTree* boundsCheck = new (this, GT_BOUNDS_CHECK) GenTreeBoundsChk(index, length, SCK_RNGCHK_FAIL);
+
+ // Element access
+ index = indexClone;
+
+#ifdef TARGET_64BIT
+ if (index->OperGet() == GT_CNS_INT)
+ {
+ index->gtType = TYP_I_IMPL;
+ }
+ else
+ {
+ index = gtNewCastNode(TYP_I_IMPL, index, true, TYP_I_IMPL);
+ }
+#endif
+
+ if (elemSize != 1)
+ {
+ GenTree* sizeofNode = gtNewIconNode(static_cast<ssize_t>(elemSize), TYP_I_IMPL);
+ index = gtNewOperNode(GT_MUL, TYP_I_IMPL, index, sizeofNode);
+ }
+
+ CORINFO_FIELD_HANDLE ptrHnd = info.compCompHnd->getFieldInClass(clsHnd, 0);
+ const unsigned ptrOffset = info.compCompHnd->getFieldOffset(ptrHnd);
+ GenTree* data = gtNewFieldRef(TYP_BYREF, ptrHnd, ptrToSpanClone, ptrOffset);
+ GenTree* result = gtNewOperNode(GT_ADD, TYP_BYREF, data, index);
+
+ // Prepare result
+ var_types resultType = JITtype2varType(sig->retType);
+ assert(resultType == result->TypeGet());
+ retNode = gtNewOperNode(GT_COMMA, resultType, boundsCheck, result);
+
+ break;
+ }
+
+ case NI_System_RuntimeTypeHandle_GetValueInternal:
+ {
+ GenTree* op1 = impStackTop(0).val;
+ if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) &&
+ gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall()))
+ {
+ // Old tree
+ // Helper-RuntimeTypeHandle -> TreeToGetNativeTypeHandle
+ //
+ // New tree
+ // TreeToGetNativeTypeHandle
+
+ // Remove call to helper and return the native TypeHandle pointer that was the parameter
+ // to that helper.
+
+ op1 = impPopStack().val;
+
+ // Get native TypeHandle argument to old helper
+ assert(op1->AsCall()->gtArgs.CountArgs() == 1);
+ op1 = op1->AsCall()->gtArgs.GetArgByIndex(0)->GetNode();
+ retNode = op1;
+ }
+ // Call the regular function.
+ break;
+ }
+
+ case NI_System_Type_GetTypeFromHandle:
+ {
+ GenTree* op1 = impStackTop(0).val;
+ CorInfoHelpFunc typeHandleHelper;
+ if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) &&
+ gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall(), &typeHandleHelper))
+ {
+ op1 = impPopStack().val;
+ // Replace helper with a more specialized helper that returns RuntimeType
+ if (typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE)
+ {
+ typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE;
+ }
+ else
+ {
+ assert(typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL);
+ typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL;
+ }
+ assert(op1->AsCall()->gtArgs.CountArgs() == 1);
+ op1 = gtNewHelperCallNode(typeHandleHelper, TYP_REF,
+ op1->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode());
+ op1->gtType = TYP_REF;
+ retNode = op1;
+ }
+ break;
+ }
+
+ case NI_System_Type_op_Equality:
+ case NI_System_Type_op_Inequality:
+ {
+ JITDUMP("Importing Type.op_*Equality intrinsic\n");
+ GenTree* op1 = impStackTop(1).val;
+ GenTree* op2 = impStackTop(0).val;
+ GenTree* optTree = gtFoldTypeEqualityCall(ni == NI_System_Type_op_Equality, op1, op2);
+ if (optTree != nullptr)
+ {
+ // Success, clean up the evaluation stack.
+ impPopStack();
+ impPopStack();
+
+ // See if we can optimize even further, to a handle compare.
+ optTree = gtFoldTypeCompare(optTree);
+
+ // See if we can now fold a handle compare to a constant.
+ optTree = gtFoldExpr(optTree);
+
+ retNode = optTree;
+ }
+ else
+ {
+ // Retry optimizing these later
+ isSpecial = true;
+ }
+ break;
+ }
+
+ case NI_System_Enum_HasFlag:
+ {
+ GenTree* thisOp = impStackTop(1).val;
+ GenTree* flagOp = impStackTop(0).val;
+ GenTree* optTree = gtOptimizeEnumHasFlag(thisOp, flagOp);
+
+ if (optTree != nullptr)
+ {
+ // Optimization successful. Pop the stack for real.
+ impPopStack();
+ impPopStack();
+ retNode = optTree;
+ }
+ else
+ {
+ // Retry optimizing this during morph.
+ isSpecial = true;
+ }
+
+ break;
+ }
+
+ case NI_System_Type_IsAssignableFrom:
+ {
+ GenTree* typeTo = impStackTop(1).val;
+ GenTree* typeFrom = impStackTop(0).val;
+
+ retNode = impTypeIsAssignable(typeTo, typeFrom);
+ break;
+ }
+
+ case NI_System_Type_IsAssignableTo:
+ {
+ GenTree* typeTo = impStackTop(0).val;
+ GenTree* typeFrom = impStackTop(1).val;
+
+ retNode = impTypeIsAssignable(typeTo, typeFrom);
+ break;
+ }
+
+ case NI_System_Type_get_IsValueType:
+ case NI_System_Type_get_IsByRefLike:
+ {
+ // Optimize
+ //
+ // call Type.GetTypeFromHandle (which is replaced with CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE)
+ // call Type.IsXXX
+ //
+ // to `true` or `false`
+ // e.g., `typeof(int).IsValueType` => `true`
+ // e.g., `typeof(Span<int>).IsByRefLike` => `true`
+ if (impStackTop().val->IsCall())
+ {
+ GenTreeCall* call = impStackTop().val->AsCall();
+ if (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE))
+ {
+ assert(call->gtArgs.CountArgs() == 1);
+ CORINFO_CLASS_HANDLE hClass =
+ gtGetHelperArgClassHandle(call->gtArgs.GetArgByIndex(0)->GetEarlyNode());
+ if (hClass != NO_CLASS_HANDLE)
+ {
+ switch (ni)
+ {
+ case NI_System_Type_get_IsValueType:
+ retNode = gtNewIconNode(eeIsValueClass(hClass) ? 1 : 0);
+ break;
+ case NI_System_Type_get_IsByRefLike:
+ retNode = gtNewIconNode(
+ (info.compCompHnd->getClassAttribs(hClass) & CORINFO_FLG_BYREF_LIKE) ? 1 : 0);
+ break;
+ default:
+ NO_WAY("Intrinsic not supported in this path.");
+ }
+ impPopStack(); // drop CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE call
+ }
+ }
+ }
+ break;
+ }
+
+ case NI_System_Threading_Thread_get_ManagedThreadId:
+ {
+ if (impStackTop().val->OperIs(GT_RET_EXPR))
+ {
+ GenTreeCall* call = impStackTop().val->AsRetExpr()->gtInlineCandidate->AsCall();
+ if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC)
+ {
+ if (lookupNamedIntrinsic(call->gtCallMethHnd) == NI_System_Threading_Thread_get_CurrentThread)
+ {
+ // drop get_CurrentThread() call
+ impPopStack();
+ call->ReplaceWith(gtNewNothingNode(), this);
+ retNode = gtNewHelperCallNode(CORINFO_HELP_GETCURRENTMANAGEDTHREADID, TYP_INT);
+ }
+ }
+ }
+ break;
+ }
+
+#ifdef TARGET_ARM64
+ // Intrinsify Interlocked.Or and Interlocked.And only for arm64-v8.1 (and newer)
+ // TODO-CQ: Implement for XArch (https://github.com/dotnet/runtime/issues/32239).
+ case NI_System_Threading_Interlocked_Or:
+ case NI_System_Threading_Interlocked_And:
+ {
+ if (compOpportunisticallyDependsOn(InstructionSet_Atomics))
+ {
+ assert(sig->numArgs == 2);
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+ genTreeOps op = (ni == NI_System_Threading_Interlocked_Or) ? GT_XORR : GT_XAND;
+ retNode = gtNewOperNode(op, genActualType(callType), op1, op2);
+ retNode->gtFlags |= GTF_GLOB_REF | GTF_ASG;
+ }
+ break;
+ }
+#endif // TARGET_ARM64
+
+#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
+ // TODO-ARM-CQ: reenable treating InterlockedCmpXchg32 operation as intrinsic
+ case NI_System_Threading_Interlocked_CompareExchange:
+ {
+ var_types retType = JITtype2varType(sig->retType);
+ if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4))
+ {
+ break;
+ }
+ if ((retType != TYP_INT) && (retType != TYP_LONG))
+ {
+ break;
+ }
+
+ assert(callType != TYP_STRUCT);
+ assert(sig->numArgs == 3);
+
+ GenTree* op3 = impPopStack().val; // comparand
+ GenTree* op2 = impPopStack().val; // value
+ GenTree* op1 = impPopStack().val; // location
+
+ GenTree* node = new (this, GT_CMPXCHG) GenTreeCmpXchg(genActualType(callType), op1, op2, op3);
+
+ node->AsCmpXchg()->gtOpLocation->gtFlags |= GTF_DONT_CSE;
+ retNode = node;
+ break;
+ }
+
+ case NI_System_Threading_Interlocked_Exchange:
+ case NI_System_Threading_Interlocked_ExchangeAdd:
+ {
+ assert(callType != TYP_STRUCT);
+ assert(sig->numArgs == 2);
+
+ var_types retType = JITtype2varType(sig->retType);
+ if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4))
+ {
+ break;
+ }
+ if ((retType != TYP_INT) && (retType != TYP_LONG))
+ {
+ break;
+ }
+
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+
+ // This creates:
+ // val
+ // XAdd
+ // addr
+ // field (for example)
+ //
+ // In the case where the first argument is the address of a local, we might
+ // want to make this *not* make the var address-taken -- but atomic instructions
+ // on a local are probably pretty useless anyway, so we probably don't care.
+
+ op1 = gtNewOperNode(ni == NI_System_Threading_Interlocked_ExchangeAdd ? GT_XADD : GT_XCHG,
+ genActualType(callType), op1, op2);
+ op1->gtFlags |= GTF_GLOB_REF | GTF_ASG;
+ retNode = op1;
+ break;
+ }
+#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64)
+
+ case NI_System_Threading_Interlocked_MemoryBarrier:
+ case NI_System_Threading_Interlocked_ReadMemoryBarrier:
+ {
+ assert(sig->numArgs == 0);
+
+ GenTree* op1 = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID);
+ op1->gtFlags |= GTF_GLOB_REF | GTF_ASG;
+
+ // On XARCH `NI_System_Threading_Interlocked_ReadMemoryBarrier` fences need not be emitted.
+ // However, we still need to capture the effect on reordering.
+ if (ni == NI_System_Threading_Interlocked_ReadMemoryBarrier)
+ {
+ op1->gtFlags |= GTF_MEMORYBARRIER_LOAD;
+ }
+
+ retNode = op1;
+ break;
+ }
+
+#ifdef FEATURE_HW_INTRINSICS
+ case NI_System_Math_FusedMultiplyAdd:
+ {
+#ifdef TARGET_XARCH
+ if (compExactlyDependsOn(InstructionSet_FMA))
+ {
+ assert(varTypeIsFloating(callType));
+
+ // We are constructing a chain of intrinsics similar to:
+ // return FMA.MultiplyAddScalar(
+ // Vector128.CreateScalarUnsafe(x),
+ // Vector128.CreateScalarUnsafe(y),
+ // Vector128.CreateScalarUnsafe(z)
+ // ).ToScalar();
+
+ GenTree* op3 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val,
+ NI_Vector128_CreateScalarUnsafe, callJitType, 16);
+ GenTree* op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val,
+ NI_Vector128_CreateScalarUnsafe, callJitType, 16);
+ GenTree* op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val,
+ NI_Vector128_CreateScalarUnsafe, callJitType, 16);
+ GenTree* res =
+ gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_FMA_MultiplyAddScalar, callJitType, 16);
+
+ retNode = gtNewSimdHWIntrinsicNode(callType, res, NI_Vector128_ToScalar, callJitType, 16);
+ break;
+ }
+#elif defined(TARGET_ARM64)
+ if (compExactlyDependsOn(InstructionSet_AdvSimd))
+ {
+ assert(varTypeIsFloating(callType));
+
+ // We are constructing a chain of intrinsics similar to:
+ // return AdvSimd.FusedMultiplyAddScalar(
+ // Vector64.Create{ScalarUnsafe}(z),
+ // Vector64.Create{ScalarUnsafe}(y),
+ // Vector64.Create{ScalarUnsafe}(x)
+ // ).ToScalar();
+
+ NamedIntrinsic createVector64 =
+ (callType == TYP_DOUBLE) ? NI_Vector64_Create : NI_Vector64_CreateScalarUnsafe;
+
+ constexpr unsigned int simdSize = 8;
+
+ GenTree* op3 =
+ gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize);
+ GenTree* op2 =
+ gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize);
+ GenTree* op1 =
+ gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize);
+
+ // Note that AdvSimd.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 + op2 * op3
+ // while Math{F}.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 * op2 + op3
+ retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op3, op2, op1, NI_AdvSimd_FusedMultiplyAddScalar,
+ callJitType, simdSize);
+
+ retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector64_ToScalar, callJitType, simdSize);
+ break;
+ }
+#endif
+
+ // TODO-CQ-XArch: Ideally we would create a GT_INTRINSIC node for fma, however, that currently
+ // requires more extensive changes to valuenum to support methods with 3 operands
+
+ // We want to generate a GT_INTRINSIC node in the case the call can't be treated as
+ // a target intrinsic so that we can still benefit from CSE and constant folding.
+
+ break;
+ }
+#endif // FEATURE_HW_INTRINSICS
+
+ case NI_System_Math_Abs:
+ case NI_System_Math_Acos:
+ case NI_System_Math_Acosh:
+ case NI_System_Math_Asin:
+ case NI_System_Math_Asinh:
+ case NI_System_Math_Atan:
+ case NI_System_Math_Atanh:
+ case NI_System_Math_Atan2:
+ case NI_System_Math_Cbrt:
+ case NI_System_Math_Ceiling:
+ case NI_System_Math_Cos:
+ case NI_System_Math_Cosh:
+ case NI_System_Math_Exp:
+ case NI_System_Math_Floor:
+ case NI_System_Math_FMod:
+ case NI_System_Math_ILogB:
+ case NI_System_Math_Log:
+ case NI_System_Math_Log2:
+ case NI_System_Math_Log10:
+ {
+ retNode = impMathIntrinsic(method, sig, callType, ni, tailCall);
+ break;
+ }
+#if defined(TARGET_ARM64)
+ // ARM64 has fmax/fmin which are IEEE754:2019 minimum/maximum compatible
+ // TODO-XARCH-CQ: Enable this for XARCH when one of the arguments is a constant
+ // so we can then emit maxss/minss and avoid NaN/-0.0 handling
+ case NI_System_Math_Max:
+ case NI_System_Math_Min:
+#endif
+
+#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH)
+ case NI_System_Math_Max:
+ case NI_System_Math_Min:
+ {
+ assert(varTypeIsFloating(callType));
+ assert(sig->numArgs == 2);
+
+ GenTreeDblCon* cnsNode = nullptr;
+ GenTree* otherNode = nullptr;
+
+ GenTree* op2 = impStackTop().val;
+ GenTree* op1 = impStackTop(1).val;
+
+ if (op2->IsCnsFltOrDbl())
+ {
+ cnsNode = op2->AsDblCon();
+ otherNode = op1;
+ }
+ else if (op1->IsCnsFltOrDbl())
+ {
+ cnsNode = op1->AsDblCon();
+ otherNode = op2;
+ }
+
+ if (cnsNode == nullptr)
+ {
+ // no constant node, nothing to do
+ break;
+ }
+
+ if (otherNode->IsCnsFltOrDbl())
+ {
+ // both are constant, we can fold this operation completely. Pop both peeked values
+
+ if (ni == NI_System_Math_Max)
+ {
+ cnsNode->SetDconValue(
+ FloatingPointUtils::maximum(cnsNode->DconValue(), otherNode->AsDblCon()->DconValue()));
+ }
+ else
+ {
+ assert(ni == NI_System_Math_Min);
+ cnsNode->SetDconValue(
+ FloatingPointUtils::minimum(cnsNode->DconValue(), otherNode->AsDblCon()->DconValue()));
+ }
+
+ retNode = cnsNode;
+
+ impPopStack();
+ impPopStack();
+ DEBUG_DESTROY_NODE(otherNode);
+
+ break;
+ }
+
+ // only one is constant, we can fold in specialized scenarios
+
+ if (cnsNode->IsFloatNaN())
+ {
+ impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("spill side effects before propagating NaN"));
+
+ // maxsd, maxss, minsd, and minss all return op2 if either is NaN
+ // we require NaN to be propagated so ensure the known NaN is op2
+
+ impPopStack();
+ impPopStack();
+ DEBUG_DESTROY_NODE(otherNode);
+
+ retNode = cnsNode;
+ break;
+ }
+
+ if (!compOpportunisticallyDependsOn(InstructionSet_SSE2))
+ {
+ break;
+ }
+
+ if (ni == NI_System_Math_Max)
+ {
+ // maxsd, maxss return op2 if both inputs are 0 of either sign
+ // we require +0 to be greater than -0, so we can't handle if
+ // the known constant is +0. This is because if the unknown value
+ // is -0, we'd need the cns to be op2. But if the unknown value
+ // is NaN, we'd need the cns to be op1 instead.
+
+ if (cnsNode->IsFloatPositiveZero())
+ {
+ break;
+ }
+
+ // Given the checks, op1 can safely be the cns and op2 the other node
+
+ ni = (callType == TYP_DOUBLE) ? NI_SSE2_Max : NI_SSE_Max;
+
+ // one is constant and we know its something we can handle, so pop both peeked values
+
+ op1 = cnsNode;
+ op2 = otherNode;
+ }
+ else
+ {
+ assert(ni == NI_System_Math_Min);
+
+ // minsd, minss return op2 if both inputs are 0 of either sign
+ // we require -0 to be lesser than +0, so we can't handle if
+ // the known constant is -0. This is because if the unknown value
+ // is +0, we'd need the cns to be op2. But if the unknown value
+ // is NaN, we'd need the cns to be op1 instead.
+
+ if (cnsNode->IsFloatNegativeZero())
+ {
+ break;
+ }
+
+ // Given the checks, op1 can safely be the cns and op2 the other node
+
+ ni = (callType == TYP_DOUBLE) ? NI_SSE2_Min : NI_SSE_Min;
+
+ // one is constant and we know its something we can handle, so pop both peeked values
+
+ op1 = cnsNode;
+ op2 = otherNode;
+ }
+
+ assert(op1->IsCnsFltOrDbl() && !op2->IsCnsFltOrDbl());
+
+ impPopStack();
+ impPopStack();
+
+ GenTreeVecCon* vecCon = gtNewVconNode(TYP_SIMD16);
+
+ if (callJitType == CORINFO_TYPE_FLOAT)
+ {
+ vecCon->gtSimd16Val.f32[0] = (float)op1->AsDblCon()->DconValue();
+ }
+ else
+ {
+ vecCon->gtSimd16Val.f64[0] = op1->AsDblCon()->DconValue();
+ }
+
+ op1 = vecCon;
+ op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, NI_Vector128_CreateScalarUnsafe, callJitType, 16);
+
+ retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, ni, callJitType, 16);
+ retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector128_ToScalar, callJitType, 16);
+
+ break;
+ }
+#endif
+
+ case NI_System_Math_Pow:
+ case NI_System_Math_Round:
+ case NI_System_Math_Sin:
+ case NI_System_Math_Sinh:
+ case NI_System_Math_Sqrt:
+ case NI_System_Math_Tan:
+ case NI_System_Math_Tanh:
+ case NI_System_Math_Truncate:
+ {
+ retNode = impMathIntrinsic(method, sig, callType, ni, tailCall);
+ break;
+ }
+
+ case NI_System_Array_Clone:
+ case NI_System_Collections_Generic_Comparer_get_Default:
+ case NI_System_Collections_Generic_EqualityComparer_get_Default:
+ case NI_System_Object_MemberwiseClone:
+ case NI_System_Threading_Thread_get_CurrentThread:
+ {
+ // Flag for later handling.
+ isSpecial = true;
+ break;
+ }
+
+ case NI_System_Object_GetType:
+ {
+ JITDUMP("\n impIntrinsic: call to Object.GetType\n");
+ GenTree* op1 = impStackTop(0).val;
+
+ // If we're calling GetType on a boxed value, just get the type directly.
+ if (op1->IsBoxedValue())
+ {
+ JITDUMP("Attempting to optimize box(...).getType() to direct type construction\n");
+
+ // Try and clean up the box. Obtain the handle we
+ // were going to pass to the newobj.
+ GenTree* boxTypeHandle = gtTryRemoveBoxUpstreamEffects(op1, BR_REMOVE_AND_NARROW_WANT_TYPE_HANDLE);
+
+ if (boxTypeHandle != nullptr)
+ {
+ // Note we don't need to play the TYP_STRUCT games here like
+ // do for LDTOKEN since the return value of this operator is Type,
+ // not RuntimeTypeHandle.
+ impPopStack();
+ GenTree* runtimeType =
+ gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, boxTypeHandle);
+ retNode = runtimeType;
+ }
+ }
+
+ // If we have a constrained callvirt with a "box this" transform
+ // we know we have a value class and hence an exact type.
+ //
+ // If so, instead of boxing and then extracting the type, just
+ // construct the type directly.
+ if ((retNode == nullptr) && (pConstrainedResolvedToken != nullptr) &&
+ (constraintCallThisTransform == CORINFO_BOX_THIS))
+ {
+ // Ensure this is one of the simple box cases (in particular, rule out nullables).
+ const CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pConstrainedResolvedToken->hClass);
+ const bool isSafeToOptimize = (boxHelper == CORINFO_HELP_BOX);
+
+ if (isSafeToOptimize)
+ {
+ JITDUMP("Optimizing constrained box-this obj.getType() to direct type construction\n");
+ impPopStack();
+ GenTree* typeHandleOp =
+ impTokenToHandle(pConstrainedResolvedToken, nullptr, true /* mustRestoreHandle */);
+ if (typeHandleOp == nullptr)
+ {
+ assert(compDonotInline());
+ return nullptr;
+ }
+ GenTree* runtimeType =
+ gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, typeHandleOp);
+ retNode = runtimeType;
+ }
+ }
+
+#ifdef DEBUG
+ if (retNode != nullptr)
+ {
+ JITDUMP("Optimized result for call to GetType is\n");
+ if (verbose)
+ {
+ gtDispTree(retNode);
+ }
+ }
+#endif
+
+ // Else expand as an intrinsic, unless the call is constrained,
+ // in which case we defer expansion to allow impImportCall do the
+ // special constraint processing.
+ if ((retNode == nullptr) && (pConstrainedResolvedToken == nullptr))
+ {
+ JITDUMP("Expanding as special intrinsic\n");
+ impPopStack();
+ op1 = new (this, GT_INTRINSIC) GenTreeIntrinsic(genActualType(callType), op1, ni, method);
+
+ // Set the CALL flag to indicate that the operator is implemented by a call.
+ // Set also the EXCEPTION flag because the native implementation of
+ // NI_System_Object_GetType intrinsic can throw NullReferenceException.
+ op1->gtFlags |= (GTF_CALL | GTF_EXCEPT);
+ retNode = op1;
+ // Might be further optimizable, so arrange to leave a mark behind
+ isSpecial = true;
+ }
+
+ if (retNode == nullptr)
+ {
+ JITDUMP("Leaving as normal call\n");
+ // Might be further optimizable, so arrange to leave a mark behind
+ isSpecial = true;
+ }
+
+ break;
+ }
+
+ case NI_System_Array_GetLength:
+ case NI_System_Array_GetLowerBound:
+ case NI_System_Array_GetUpperBound:
+ {
+ // System.Array.GetLength(Int32) method:
+ // public int GetLength(int dimension)
+ // System.Array.GetLowerBound(Int32) method:
+ // public int GetLowerBound(int dimension)
+ // System.Array.GetUpperBound(Int32) method:
+ // public int GetUpperBound(int dimension)
+ //
+ // Only implement these as intrinsics for multi-dimensional arrays.
+ // Only handle constant dimension arguments.
+
+ GenTree* gtDim = impStackTop().val;
+ GenTree* gtArr = impStackTop(1).val;
+
+ if (gtDim->IsIntegralConst())
+ {
+ bool isExact = false;
+ bool isNonNull = false;
+ CORINFO_CLASS_HANDLE arrCls = gtGetClassHandle(gtArr, &isExact, &isNonNull);
+ if (arrCls != NO_CLASS_HANDLE)
+ {
+ unsigned rank = info.compCompHnd->getArrayRank(arrCls);
+ if ((rank > 1) && !info.compCompHnd->isSDArray(arrCls))
+ {
+ // `rank` is guaranteed to be <=32 (see MAX_RANK in vm\array.h). Any constant argument
+ // is `int` sized.
+ INT64 dimValue = gtDim->AsIntConCommon()->IntegralValue();
+ assert((unsigned int)dimValue == dimValue);
+ unsigned dim = (unsigned int)dimValue;
+ if (dim < rank)
+ {
+ // This is now known to be a multi-dimension array with a constant dimension
+ // that is in range; we can expand it as an intrinsic.
+
+ impPopStack().val; // Pop the dim and array object; we already have a pointer to them.
+ impPopStack().val;
+
+ // Make sure there are no global effects in the array (such as it being a function
+ // call), so we can mark the generated indirection with GTF_IND_INVARIANT. In the
+ // GetUpperBound case we need the cloned object, since we refer to the array
+ // object twice. In the other cases, we don't need to clone.
+ GenTree* gtArrClone = nullptr;
+ if (((gtArr->gtFlags & GTF_GLOB_EFFECT) != 0) || (ni == NI_System_Array_GetUpperBound))
+ {
+ gtArr = impCloneExpr(gtArr, >ArrClone, NO_CLASS_HANDLE, CHECK_SPILL_ALL,
+ nullptr DEBUGARG("MD intrinsics array"));
+ }
+
+ switch (ni)
+ {
+ case NI_System_Array_GetLength:
+ {
+ // Generate *(array + offset-to-length-array + sizeof(int) * dim)
+ unsigned offs = eeGetMDArrayLengthOffset(rank, dim);
+ GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL);
+ GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs);
+ retNode = gtNewIndir(TYP_INT, gtAddr);
+ retNode->gtFlags |= GTF_IND_INVARIANT;
+ break;
+ }
+ case NI_System_Array_GetLowerBound:
+ {
+ // Generate *(array + offset-to-bounds-array + sizeof(int) * dim)
+ unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim);
+ GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL);
+ GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs);
+ retNode = gtNewIndir(TYP_INT, gtAddr);
+ retNode->gtFlags |= GTF_IND_INVARIANT;
+ break;
+ }
+ case NI_System_Array_GetUpperBound:
+ {
+ assert(gtArrClone != nullptr);
+
+ // Generate:
+ // *(array + offset-to-length-array + sizeof(int) * dim) +
+ // *(array + offset-to-bounds-array + sizeof(int) * dim) - 1
+ unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim);
+ GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL);
+ GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs);
+ GenTree* gtLowerBound = gtNewIndir(TYP_INT, gtAddr);
+ gtLowerBound->gtFlags |= GTF_IND_INVARIANT;
+
+ offs = eeGetMDArrayLengthOffset(rank, dim);
+ gtOffs = gtNewIconNode(offs, TYP_I_IMPL);
+ gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArrClone, gtOffs);
+ GenTree* gtLength = gtNewIndir(TYP_INT, gtAddr);
+ gtLength->gtFlags |= GTF_IND_INVARIANT;
+
+ GenTree* gtSum = gtNewOperNode(GT_ADD, TYP_INT, gtLowerBound, gtLength);
+ GenTree* gtOne = gtNewIconNode(1, TYP_INT);
+ retNode = gtNewOperNode(GT_SUB, TYP_INT, gtSum, gtOne);
+ break;
+ }
+ default:
+ unreached();
+ }
+ }
+ }
+ }
+ }
+ break;
+ }
+
+ case NI_System_Buffers_Binary_BinaryPrimitives_ReverseEndianness:
+ {
+ assert(sig->numArgs == 1);
+
+ // We expect the return type of the ReverseEndianness routine to match the type of the
+ // one and only argument to the method. We use a special instruction for 16-bit
+ // BSWAPs since on x86 processors this is implemented as ROR <16-bit reg>, 8. Additionally,
+ // we only emit 64-bit BSWAP instructions on 64-bit archs; if we're asked to perform a
+ // 64-bit byte swap on a 32-bit arch, we'll fall to the default case in the switch block below.
+
+ switch (sig->retType)
+ {
+ case CorInfoType::CORINFO_TYPE_SHORT:
+ case CorInfoType::CORINFO_TYPE_USHORT:
+ retNode = gtNewCastNode(TYP_INT, gtNewOperNode(GT_BSWAP16, TYP_INT, impPopStack().val), false,
+ callType);
+ break;
+
+ case CorInfoType::CORINFO_TYPE_INT:
+ case CorInfoType::CORINFO_TYPE_UINT:
+#ifdef TARGET_64BIT
+ case CorInfoType::CORINFO_TYPE_LONG:
+ case CorInfoType::CORINFO_TYPE_ULONG:
+#endif // TARGET_64BIT
+ retNode = gtNewOperNode(GT_BSWAP, callType, impPopStack().val);
+ break;
+
+ default:
+ // This default case gets hit on 32-bit archs when a call to a 64-bit overload
+ // of ReverseEndianness is encountered. In that case we'll let JIT treat this as a standard
+ // method call, where the implementation decomposes the operation into two 32-bit
+ // bswap routines. If the input to the 64-bit function is a constant, then we rely
+ // on inlining + constant folding of 32-bit bswaps to effectively constant fold
+ // the 64-bit call site.
+ break;
+ }
+
+ break;
+ }
+
+ // Fold PopCount for constant input
+ case NI_System_Numerics_BitOperations_PopCount:
+ {
+ assert(sig->numArgs == 1);
+ if (impStackTop().val->IsIntegralConst())
+ {
+ typeInfo argType = verParseArgSigToTypeInfo(sig, sig->args).NormaliseForStack();
+ INT64 cns = impPopStack().val->AsIntConCommon()->IntegralValue();
+ if (argType.IsType(TI_LONG))
+ {
+ retNode = gtNewIconNode(genCountBits(cns), callType);
+ }
+ else
+ {
+ assert(argType.IsType(TI_INT));
+ retNode = gtNewIconNode(genCountBits(static_cast<unsigned>(cns)), callType);
+ }
+ }
+ break;
+ }
+
+ case NI_System_GC_KeepAlive:
+ {
+ retNode = impKeepAliveIntrinsic(impPopStack().val);
+ break;
+ }
+
+ case NI_System_BitConverter_DoubleToInt64Bits:
+ {
+ GenTree* op1 = impStackTop().val;
+ assert(varTypeIsFloating(op1));
+
+ if (op1->IsCnsFltOrDbl())
+ {
+ impPopStack();
+
+ double f64Cns = op1->AsDblCon()->DconValue();
+ retNode = gtNewLconNode(*reinterpret_cast<int64_t*>(&f64Cns));
+ }
+#if TARGET_64BIT
+ else
+ {
+ // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work
+ impPopStack();
+
+ if (op1->TypeGet() != TYP_DOUBLE)
+ {
+ op1 = gtNewCastNode(TYP_DOUBLE, op1, false, TYP_DOUBLE);
+ }
+ retNode = gtNewBitCastNode(TYP_LONG, op1);
+ }
+#endif
+ break;
+ }
+
+ case NI_System_BitConverter_Int32BitsToSingle:
+ {
+ GenTree* op1 = impPopStack().val;
+ assert(varTypeIsInt(op1));
+
+ if (op1->IsIntegralConst())
+ {
+ int32_t i32Cns = (int32_t)op1->AsIntConCommon()->IconValue();
+ retNode = gtNewDconNode(*reinterpret_cast<float*>(&i32Cns), TYP_FLOAT);
+ }
+ else
+ {
+ retNode = gtNewBitCastNode(TYP_FLOAT, op1);
+ }
+ break;
+ }
+
+ case NI_System_BitConverter_Int64BitsToDouble:
+ {
+ GenTree* op1 = impStackTop().val;
+ assert(varTypeIsLong(op1));
+
+ if (op1->IsIntegralConst())
+ {
+ impPopStack();
+
+ int64_t i64Cns = op1->AsIntConCommon()->LngValue();
+ retNode = gtNewDconNode(*reinterpret_cast<double*>(&i64Cns));
+ }
+#if TARGET_64BIT
+ else
+ {
+ // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work
+ impPopStack();
+
+ retNode = gtNewBitCastNode(TYP_DOUBLE, op1);
+ }
+#endif
+ break;
+ }
+
+ case NI_System_BitConverter_SingleToInt32Bits:
+ {
+ GenTree* op1 = impPopStack().val;
+ assert(varTypeIsFloating(op1));
+
+ if (op1->IsCnsFltOrDbl())
+ {
+ float f32Cns = (float)op1->AsDblCon()->DconValue();
+ retNode = gtNewIconNode(*reinterpret_cast<int32_t*>(&f32Cns));
+ }
+ else
+ {
+ if (op1->TypeGet() != TYP_FLOAT)
+ {
+ op1 = gtNewCastNode(TYP_FLOAT, op1, false, TYP_FLOAT);
+ }
+ retNode = gtNewBitCastNode(TYP_INT, op1);
+ }
+ break;
+ }
+
+ default:
+ break;
+ }
+ }
+
+ if (mustExpand && (retNode == nullptr))
+ {
+ assert(!"Unhandled must expand intrinsic, throwing PlatformNotSupportedException");
+ return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand);
+ }
+
+ // Optionally report if this intrinsic is special
+ // (that is, potentially re-optimizable during morph).
+ if (isSpecialIntrinsic != nullptr)
+ {
+ *isSpecialIntrinsic = isSpecial;
+ }
+
+ return retNode;
+}
+
+GenTree* Compiler::impSRCSUnsafeIntrinsic(NamedIntrinsic intrinsic,
+ CORINFO_CLASS_HANDLE clsHnd,
+ CORINFO_METHOD_HANDLE method,
+ CORINFO_SIG_INFO* sig)
+{
+ // NextCallRetAddr requires a CALL, so return nullptr.
+ if (info.compHasNextCallRetAddr)
+ {
+ return nullptr;
+ }
+
+ assert(sig->sigInst.classInstCount == 0);
+
+ switch (intrinsic)
+ {
+ case NI_SRCS_UNSAFE_Add:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldarg.1
+ // sizeof !!T
+ // conv.i
+ // mul
+ // add
+ // ret
+
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+ impBashVarAddrsToI(op1, op2);
+
+ op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL);
+
+ unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]);
+
+ if (classSize != 1)
+ {
+ GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL);
+ op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size);
+ }
+
+ var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2);
+ return gtNewOperNode(GT_ADD, type, op1, op2);
+ }
+
+ case NI_SRCS_UNSAFE_AddByteOffset:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldarg.1
+ // add
+ // ret
+
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+ impBashVarAddrsToI(op1, op2);
+
+ var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2);
+ return gtNewOperNode(GT_ADD, type, op1, op2);
+ }
+
+ case NI_SRCS_UNSAFE_AreSame:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldarg.1
+ // ceq
+ // ret
+
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+
+ GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, op2);
+ return gtFoldExpr(tmp);
+ }
+
+ case NI_SRCS_UNSAFE_As:
+ {
+ assert((sig->sigInst.methInstCount == 1) || (sig->sigInst.methInstCount == 2));
+
+ // ldarg.0
+ // ret
+
+ return impPopStack().val;
+ }
+
+ case NI_SRCS_UNSAFE_AsPointer:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // conv.u
+ // ret
+
+ GenTree* op1 = impPopStack().val;
+ impBashVarAddrsToI(op1);
+
+ return gtNewCastNode(TYP_I_IMPL, op1, /* uns */ false, TYP_I_IMPL);
+ }
+
+ case NI_SRCS_UNSAFE_AsRef:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ret
+
+ return impPopStack().val;
+ }
+
+ case NI_SRCS_UNSAFE_ByteOffset:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.1
+ // ldarg.0
+ // sub
+ // ret
+
+ impSpillSideEffect(true, verCurrentState.esStackDepth -
+ 2 DEBUGARG("Spilling op1 side effects for Unsafe.ByteOffset"));
+
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+ impBashVarAddrsToI(op1, op2);
+
+ var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op2, &op1);
+ return gtNewOperNode(GT_SUB, type, op2, op1);
+ }
+
+ case NI_SRCS_UNSAFE_Copy:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldarg.1
+ // ldobj !!T
+ // stobj !!T
+ // ret
+
+ return nullptr;
+ }
+
+ case NI_SRCS_UNSAFE_CopyBlock:
+ {
+ assert(sig->sigInst.methInstCount == 0);
+
+ // ldarg.0
+ // ldarg.1
+ // ldarg.2
+ // cpblk
+ // ret
+
+ return nullptr;
+ }
+
+ case NI_SRCS_UNSAFE_CopyBlockUnaligned:
+ {
+ assert(sig->sigInst.methInstCount == 0);
+
+ // ldarg.0
+ // ldarg.1
+ // ldarg.2
+ // unaligned. 0x1
+ // cpblk
+ // ret
+
+ return nullptr;
+ }
+
+ case NI_SRCS_UNSAFE_InitBlock:
+ {
+ assert(sig->sigInst.methInstCount == 0);
+
+ // ldarg.0
+ // ldarg.1
+ // ldarg.2
+ // initblk
+ // ret
+
+ return nullptr;
+ }
+
+ case NI_SRCS_UNSAFE_InitBlockUnaligned:
+ {
+ assert(sig->sigInst.methInstCount == 0);
+
+ // ldarg.0
+ // ldarg.1
+ // ldarg.2
+ // unaligned. 0x1
+ // initblk
+ // ret
+
+ return nullptr;
+ }
+
+ case NI_SRCS_UNSAFE_IsAddressGreaterThan:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldarg.1
+ // cgt.un
+ // ret
+
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+
+ GenTree* tmp = gtNewOperNode(GT_GT, TYP_INT, op1, op2);
+ tmp->gtFlags |= GTF_UNSIGNED;
+ return gtFoldExpr(tmp);
+ }
+
+ case NI_SRCS_UNSAFE_IsAddressLessThan:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldarg.1
+ // clt.un
+ // ret
+
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+
+ GenTree* tmp = gtNewOperNode(GT_LT, TYP_INT, op1, op2);
+ tmp->gtFlags |= GTF_UNSIGNED;
+ return gtFoldExpr(tmp);
+ }
+
+ case NI_SRCS_UNSAFE_IsNullRef:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldc.i4.0
+ // conv.u
+ // ceq
+ // ret
+
+ GenTree* op1 = impPopStack().val;
+ GenTree* cns = gtNewIconNode(0, TYP_BYREF);
+ GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, cns);
+ return gtFoldExpr(tmp);
+ }
+
+ case NI_SRCS_UNSAFE_NullRef:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldc.i4.0
+ // conv.u
+ // ret
+
+ return gtNewIconNode(0, TYP_BYREF);
+ }
+
+ case NI_SRCS_UNSAFE_Read:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldobj !!T
+ // ret
+
+ return nullptr;
+ }
+
+ case NI_SRCS_UNSAFE_ReadUnaligned:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // unaligned. 0x1
+ // ldobj !!T
+ // ret
+
+ return nullptr;
+ }
+
+ case NI_SRCS_UNSAFE_SizeOf:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // sizeof !!T
+ // ret
+
+ unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]);
+ return gtNewIconNode(classSize, TYP_INT);
+ }
+
+ case NI_SRCS_UNSAFE_SkipInit:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ret
+
+ GenTree* op1 = impPopStack().val;
+
+ if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0)
+ {
+ return gtUnusedValNode(op1);
+ }
+ else
+ {
+ return gtNewNothingNode();
+ }
+ }
+
+ case NI_SRCS_UNSAFE_Subtract:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldarg.1
+ // sizeof !!T
+ // conv.i
+ // mul
+ // sub
+ // ret
+
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+ impBashVarAddrsToI(op1, op2);
+
+ op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL);
+
+ unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]);
+
+ if (classSize != 1)
+ {
+ GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL);
+ op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size);
+ }
+
+ var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2);
+ return gtNewOperNode(GT_SUB, type, op1, op2);
+ }
+
+ case NI_SRCS_UNSAFE_SubtractByteOffset:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldarg.1
+ // sub
+ // ret
+
+ GenTree* op2 = impPopStack().val;
+ GenTree* op1 = impPopStack().val;
+ impBashVarAddrsToI(op1, op2);
+
+ var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2);
+ return gtNewOperNode(GT_SUB, type, op1, op2);
+ }
+
+ case NI_SRCS_UNSAFE_Unbox:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // unbox !!T
+ // ret
+
+ return nullptr;
+ }
+
+ case NI_SRCS_UNSAFE_Write:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldarg.1
+ // stobj !!T
+ // ret
+
+ return nullptr;
+ }
+
+ case NI_SRCS_UNSAFE_WriteUnaligned:
+ {
+ assert(sig->sigInst.methInstCount == 1);
+
+ // ldarg.0
+ // ldarg.1
+ // unaligned. 0x01
+ // stobj !!T
+ // ret
+
+ return nullptr;
+ }
+
+ default:
+ {
+ unreached();
+ }
+ }
+}
projects / platform / upstream / dotnet / runtime.git / commitdiff