From: Tanner Gooding Date: Fri, 27 Jan 2023 20:46:04 +0000 (-0800) Subject: Ensure various scalar cross platform helper APIs are handled directly as intrinsic... X-Git-Tag: accepted/tizen/unified/riscv/20231226.055536~4376 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=9ad75b4a9fe87cab338439c96cd187e5791b6735;p=platform%2Fupstream%2Fdotnet%2Fruntime.git Ensure various scalar cross platform helper APIs are handled directly as intrinsic (#80789) * Ensure various scalar cross platform helper APIs are handled directly as intrinsic * Small refactoring to lookupNamedIntrinsic and impIntrinsic to improve TP --- diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index e4258c0..64eaf8e 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3873,6 +3873,7 @@ protected: NamedIntrinsic intrinsicName, bool tailCall); NamedIntrinsic lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method); + NamedIntrinsic lookupPrimitiveNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName); GenTree* impUnsupportedNamedIntrinsic(unsigned helper, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig, @@ -3883,6 +3884,11 @@ protected: CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig); + GenTree* impPrimitiveNamedIntrinsic(NamedIntrinsic intrinsic, + CORINFO_CLASS_HANDLE clsHnd, + CORINFO_METHOD_HANDLE method, + CORINFO_SIG_INFO* sig); + #ifdef FEATURE_HW_INTRINSICS GenTree* impHWIntrinsic(NamedIntrinsic intrinsic, CORINFO_CLASS_HANDLE clsHnd, diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index e31a1e6..dff9583 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -1168,6 +1168,10 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed break; // These are foldable if the first argument is a constant + case NI_PRIMITIVE_LeadingZeroCount: + case NI_PRIMITIVE_Log2: + case NI_PRIMITIVE_PopCount: + case NI_PRIMITIVE_TrailingZeroCount: case NI_System_Type_get_IsEnum: case NI_System_Type_GetEnumUnderlyingType: case NI_System_Type_get_IsValueType: @@ -1175,9 +1179,12 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed case NI_System_Type_GetTypeFromHandle: case NI_System_String_get_Length: case NI_System_Buffers_Binary_BinaryPrimitives_ReverseEndianness: - case NI_System_Numerics_BitOperations_PopCount: #if defined(FEATURE_HW_INTRINSICS) #if defined(TARGET_ARM64) + case NI_ArmBase_Arm64_LeadingZeroCount: + case NI_ArmBase_Arm64_ReverseElementBits: + case NI_ArmBase_LeadingZeroCount: + case NI_ArmBase_ReverseElementBits: case NI_Vector64_Create: case NI_Vector64_CreateScalar: case NI_Vector64_CreateScalarUnsafe: @@ -1196,10 +1203,20 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed case NI_Vector128_CreateScalarUnsafe: case NI_VectorT128_CreateBroadcast: #if defined(TARGET_XARCH) + case NI_BMI1_TrailingZeroCount: + case NI_BMI1_X64_TrailingZeroCount: + case NI_LZCNT_LeadingZeroCount: + case NI_LZCNT_X64_LeadingZeroCount: + case NI_POPCNT_PopCount: + case NI_POPCNT_X64_PopCount: case NI_Vector256_Create: case NI_Vector256_CreateScalar: case NI_Vector256_CreateScalarUnsafe: case NI_VectorT256_CreateBroadcast: + case NI_X86Base_BitScanForward: + case NI_X86Base_X64_BitScanForward: + case NI_X86Base_BitScanReverse: + case NI_X86Base_X64_BitScanReverse: #endif // TARGET_XARCH #endif // FEATURE_HW_INTRINSICS { @@ -1212,6 +1229,8 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed } // These are foldable if two arguments are constants + case NI_PRIMITIVE_RotateLeft: + case NI_PRIMITIVE_RotateRight: case NI_System_Type_op_Equality: case NI_System_Type_op_Inequality: case NI_System_String_get_Chars: diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 8479ee4..6a77c9a 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -2345,146 +2345,199 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, // We specially support the following on all platforms to allow for dead // code optimization and to more generally support recursive intrinsics. - if (isIntrinsic) + if (isIntrinsic && (ni > NI_SPECIAL_IMPORT_START) && (ni < NI_PRIMITIVE_END)) { - if (ni == NI_IsSupported_True) - { - assert(sig->numArgs == 0); - return gtNewIconNode(true); - } + static_assert_no_msg(NI_SPECIAL_IMPORT_START < NI_SPECIAL_IMPORT_END); + static_assert_no_msg(NI_SRCS_UNSAFE_START < NI_SRCS_UNSAFE_END); + static_assert_no_msg(NI_PRIMITIVE_START < NI_PRIMITIVE_END); - if (ni == NI_IsSupported_False) - { - assert(sig->numArgs == 0); - return gtNewIconNode(false); - } + static_assert_no_msg((NI_SPECIAL_IMPORT_END + 1) == NI_SRCS_UNSAFE_START); + static_assert_no_msg((NI_SRCS_UNSAFE_END + 1) == NI_PRIMITIVE_START); - if (ni == NI_IsSupported_Type) + if (ni < NI_SPECIAL_IMPORT_END) { - CORINFO_CLASS_HANDLE typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(clsHnd, 0); - CorInfoType simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); + assert(ni > NI_SPECIAL_IMPORT_START); - switch (simdBaseJitType) + switch (ni) { - case CORINFO_TYPE_BYTE: - case CORINFO_TYPE_UBYTE: - case CORINFO_TYPE_SHORT: - case CORINFO_TYPE_USHORT: - case CORINFO_TYPE_INT: - case CORINFO_TYPE_UINT: - case CORINFO_TYPE_LONG: - case CORINFO_TYPE_ULONG: - case CORINFO_TYPE_FLOAT: - case CORINFO_TYPE_DOUBLE: - case CORINFO_TYPE_NATIVEINT: - case CORINFO_TYPE_NATIVEUINT: + case NI_IsSupported_True: { + assert(sig->numArgs == 0); return gtNewIconNode(true); } - default: + case NI_IsSupported_False: { + assert(sig->numArgs == 0); return gtNewIconNode(false); } - } - } - if (ni == NI_Vector_GetCount) - { - CORINFO_CLASS_HANDLE typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(clsHnd, 0); - CorInfoType simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); - unsigned simdSize = info.compCompHnd->getClassSize(clsHnd); + case NI_IsSupported_Dynamic: + { + break; + } - switch (simdBaseJitType) - { - case CORINFO_TYPE_BYTE: - case CORINFO_TYPE_UBYTE: - case CORINFO_TYPE_SHORT: - case CORINFO_TYPE_USHORT: - case CORINFO_TYPE_INT: - case CORINFO_TYPE_UINT: - case CORINFO_TYPE_LONG: - case CORINFO_TYPE_ULONG: - case CORINFO_TYPE_FLOAT: - case CORINFO_TYPE_DOUBLE: - case CORINFO_TYPE_NATIVEINT: - case CORINFO_TYPE_NATIVEUINT: + case NI_IsSupported_Type: + { + CORINFO_CLASS_HANDLE typeArgHnd; + CorInfoType simdBaseJitType; + + typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(clsHnd, 0); + simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); + + switch (simdBaseJitType) + { + case CORINFO_TYPE_BYTE: + case CORINFO_TYPE_UBYTE: + case CORINFO_TYPE_SHORT: + case CORINFO_TYPE_USHORT: + case CORINFO_TYPE_INT: + case CORINFO_TYPE_UINT: + case CORINFO_TYPE_LONG: + case CORINFO_TYPE_ULONG: + case CORINFO_TYPE_FLOAT: + case CORINFO_TYPE_DOUBLE: + case CORINFO_TYPE_NATIVEINT: + case CORINFO_TYPE_NATIVEUINT: + { + return gtNewIconNode(true); + } + + default: + { + return gtNewIconNode(false); + } + } + } + + case NI_Throw_PlatformNotSupportedException: + { + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, + mustExpand); + } + + case NI_Vector_GetCount: { - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - unsigned elementSize = genTypeSize(simdBaseType); - GenTreeIntCon* countNode = gtNewIconNode(simdSize / elementSize, TYP_INT); + CORINFO_CLASS_HANDLE typeArgHnd; + CorInfoType simdBaseJitType; + unsigned simdSize; + + typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(clsHnd, 0); + simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); + simdSize = info.compCompHnd->getClassSize(clsHnd); + + switch (simdBaseJitType) + { + case CORINFO_TYPE_BYTE: + case CORINFO_TYPE_UBYTE: + case CORINFO_TYPE_SHORT: + case CORINFO_TYPE_USHORT: + case CORINFO_TYPE_INT: + case CORINFO_TYPE_UINT: + case CORINFO_TYPE_LONG: + case CORINFO_TYPE_ULONG: + case CORINFO_TYPE_FLOAT: + case CORINFO_TYPE_DOUBLE: + case CORINFO_TYPE_NATIVEINT: + case CORINFO_TYPE_NATIVEUINT: + { + var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); + unsigned elementSize = genTypeSize(simdBaseType); + GenTreeIntCon* countNode = gtNewIconNode(simdSize / elementSize, TYP_INT); #if defined(FEATURE_SIMD) - countNode->gtFlags |= GTF_ICON_SIMD_COUNT; + countNode->gtFlags |= GTF_ICON_SIMD_COUNT; #endif // FEATURE_SIMD - return countNode; + return countNode; + } + + default: + { + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, + mustExpand); + } + } } default: { - ni = NI_Throw_PlatformNotSupportedException; - break; + unreached(); } } } - - if (ni == NI_Throw_PlatformNotSupportedException) + else if (ni < NI_SRCS_UNSAFE_END) { - return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); + assert(ni > NI_SRCS_UNSAFE_START); + assert(!mustExpand); + return impSRCSUnsafeIntrinsic(ni, clsHnd, method, sig); } - - if ((ni > NI_SRCS_UNSAFE_START) && (ni < NI_SRCS_UNSAFE_END)) + else { + assert((ni > NI_PRIMITIVE_START) && (ni < NI_PRIMITIVE_END)); assert(!mustExpand); - return impSRCSUnsafeIntrinsic(ni, clsHnd, method, sig); + return impPrimitiveNamedIntrinsic(ni, clsHnd, method, sig); } } #ifdef FEATURE_HW_INTRINSICS - if ((ni > NI_HW_INTRINSIC_START) && (ni < NI_HW_INTRINSIC_END)) + if ((ni > NI_HW_INTRINSIC_START) && (ni < NI_SIMD_AS_HWINTRINSIC_END)) { - if (!isIntrinsic) + static_assert_no_msg(NI_HW_INTRINSIC_START < NI_HW_INTRINSIC_END); + static_assert_no_msg(NI_SIMD_AS_HWINTRINSIC_START < NI_SIMD_AS_HWINTRINSIC_END); + + static_assert_no_msg((NI_HW_INTRINSIC_END + 1) == NI_SIMD_AS_HWINTRINSIC_START); + + if (ni < NI_HW_INTRINSIC_END) { + assert(ni > NI_HW_INTRINSIC_START); + + if (!isIntrinsic) + { #if defined(TARGET_XARCH) - // We can't guarantee that all overloads for the xplat intrinsics can be - // handled by the AltJit, so limit only the platform specific intrinsics - assert((NI_Vector256_Xor + 1) == NI_X86Base_BitScanForward); + // We can't guarantee that all overloads for the xplat intrinsics can be + // handled by the AltJit, so limit only the platform specific intrinsics + assert((NI_Vector256_Xor + 1) == NI_X86Base_BitScanForward); - if (ni < NI_Vector256_Xor) + if (ni < NI_Vector256_Xor) #elif defined(TARGET_ARM64) - // We can't guarantee that all overloads for the xplat intrinsics can be - // handled by the AltJit, so limit only the platform specific intrinsics - assert((NI_Vector128_Xor + 1) == NI_AdvSimd_Abs); + // We can't guarantee that all overloads for the xplat intrinsics can be + // handled by the AltJit, so limit only the platform specific intrinsics + assert((NI_Vector128_Xor + 1) == NI_AdvSimd_Abs); - if (ni < NI_Vector128_Xor) + if (ni < NI_Vector128_Xor) #else #error Unsupported platform #endif - { - // Several of the NI_Vector64/128/256 APIs do not have - // all overloads as intrinsic today so they will assert - return nullptr; + { + // Several of the NI_Vector64/128/256 APIs do not have + // all overloads as intrinsic today so they will assert + return nullptr; + } } - } - GenTree* hwintrinsic = impHWIntrinsic(ni, clsHnd, method, sig, mustExpand); + GenTree* hwintrinsic = impHWIntrinsic(ni, clsHnd, method, sig, mustExpand); - if (mustExpand && (hwintrinsic == nullptr)) - { - return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_NOT_IMPLEMENTED, method, sig, mustExpand); - } + if (mustExpand && (hwintrinsic == nullptr)) + { + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_NOT_IMPLEMENTED, method, sig, mustExpand); + } - return hwintrinsic; - } + return hwintrinsic; + } + else + { + assert((ni > NI_SIMD_AS_HWINTRINSIC_START) && (ni < NI_SIMD_AS_HWINTRINSIC_END)); - if (isIntrinsic && (ni > NI_SIMD_AS_HWINTRINSIC_START) && (ni < NI_SIMD_AS_HWINTRINSIC_END)) - { - // These intrinsics aren't defined recursively and so they will never be mustExpand - // Instead, they provide software fallbacks that will be executed instead. + if (isIntrinsic) + { + // These intrinsics aren't defined recursively and so they will never be mustExpand + // Instead, they provide software fallbacks that will be executed instead. - assert(!mustExpand); - return impSimdAsHWIntrinsic(ni, clsHnd, method, sig, newobjThis); + assert(!mustExpand); + return impSimdAsHWIntrinsic(ni, clsHnd, method, sig, newobjThis); + } + } } #endif // FEATURE_HW_INTRINSICS @@ -3671,27 +3724,6 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, break; } - // Fold PopCount for constant input - case NI_System_Numerics_BitOperations_PopCount: - { - assert(sig->numArgs == 1); - if (impStackTop().val->IsIntegralConst()) - { - typeInfo argType = verParseArgSigToTypeInfo(sig, sig->args).NormaliseForStack(); - INT64 cns = impPopStack().val->AsIntConCommon()->IntegralValue(); - if (argType.IsType(TI_LONG)) - { - retNode = gtNewIconNode(genCountBits(cns), callType); - } - else - { - assert(argType.IsType(TI_INT)); - retNode = gtNewIconNode(genCountBits(static_cast(cns)), callType); - } - } - break; - } - case NI_System_GC_KeepAlive: { retNode = impKeepAliveIntrinsic(impPopStack().val); @@ -4206,160 +4238,716 @@ GenTree* Compiler::impSRCSUnsafeIntrinsic(NamedIntrinsic intrinsic, } //------------------------------------------------------------------------ -// impPopCallArgs: -// Pop the given number of values from the stack and return a list node with -// their values. +// impPrimitiveNamedIntrinsic: import a NamedIntrinsic representing a primitive operation // -// Parameters: -// sig - Signature used to figure out classes the runtime must load, and -// also to record exact receiving argument types that may be needed for ABI -// purposes later. -// call - The call to pop arguments into. +// Arguments: +// intrinsic - the intrinsic being imported +// clsHnd - handle for the intrinsic method's class +// method - handle for the intrinsic method +// sig - signature of the intrinsic method // -void Compiler::impPopCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call) +// Returns: +// IR tree to use in place of the call, or nullptr if the jit should treat +// the intrinsic call like a normal call. +// +GenTree* Compiler::impPrimitiveNamedIntrinsic(NamedIntrinsic intrinsic, + CORINFO_CLASS_HANDLE clsHnd, + CORINFO_METHOD_HANDLE method, + CORINFO_SIG_INFO* sig) { - assert(call->gtArgs.IsEmpty()); + assert(sig->sigInst.classInstCount == 0); - if (impStackHeight() < sig->numArgs) - { - BADCODE("not enough arguments for call"); - } + var_types retType = JITtype2varType(sig->retType); + assert(varTypeIsArithmetic(retType)); - struct SigParamInfo - { - CorInfoType CorType; - CORINFO_CLASS_HANDLE ClassHandle; - }; + NamedIntrinsic hwintrinsic = NI_Illegal; - SigParamInfo inlineParams[16]; - SigParamInfo* params = sig->numArgs <= 16 ? inlineParams : new (this, CMK_CallArgs) SigParamInfo[sig->numArgs]; + CORINFO_ARG_LIST_HANDLE args = sig->args; - // We will iterate and pop the args in reverse order as we sometimes need - // to spill some args. However, we need signature information and the - // JIT-EE interface only allows us to iterate the signature forwards. We - // will collect the needed information here and at the same time notify the - // EE that the signature types need to be loaded. - CORINFO_ARG_LIST_HANDLE sigArg = sig->args; - for (unsigned i = 0; i < sig->numArgs; i++) - { - params[i].CorType = strip(info.compCompHnd->getArgType(sig, sigArg, ¶ms[i].ClassHandle)); + assert((sig->numArgs == 1) || (sig->numArgs == 2)); - if (params[i].CorType != CORINFO_TYPE_CLASS && params[i].CorType != CORINFO_TYPE_BYREF && - params[i].CorType != CORINFO_TYPE_PTR && params[i].CorType != CORINFO_TYPE_VAR) + CORINFO_CLASS_HANDLE op1ClsHnd; + CorInfoType baseJitType = strip(info.compCompHnd->getArgType(sig, args, &op1ClsHnd)); + var_types baseType = JITtype2varType(baseJitType); + + GenTree* result = nullptr; + + switch (intrinsic) + { + case NI_PRIMITIVE_Crc32C: { - CORINFO_CLASS_HANDLE argRealClass = info.compCompHnd->getArgClass(sig, sigArg); - if (argRealClass != nullptr) + assert(sig->numArgs == 2); + assert(retType == TYP_INT); + + // Crc32 needs the base type from op2 + + CORINFO_CLASS_HANDLE op2ClsHnd; + args = info.compCompHnd->getArgNext(args); + + baseJitType = strip(info.compCompHnd->getArgType(sig, args, &op2ClsHnd)); + baseType = JITtype2varType(baseJitType); + +#if !defined(TARGET_64BIT) + if (varTypeIsLong(baseType)) { - // Make sure that all valuetypes (including enums) that we push are loaded. - // This is to guarantee that if a GC is triggered from the prestub of this methods, - // all valuetypes in the method signature are already loaded. - // We need to be able to find the size of the valuetypes, but we cannot - // do a class-load from within GC. - info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(argRealClass); + // TODO-CQ: Adding long decomposition support is more complex + // and not supported today so early exit if we have a long and + // either input is not a constant. + + break; } - } +#endif // !TARGET_64BIT - sigArg = info.compCompHnd->getArgNext(sigArg); - } +#if defined(FEATURE_HW_INTRINSICS) +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_SSE42)) + { + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; - if ((sig->retTypeSigClass != nullptr) && (sig->retType != CORINFO_TYPE_CLASS) && - (sig->retType != CORINFO_TYPE_BYREF) && (sig->retType != CORINFO_TYPE_PTR) && - (sig->retType != CORINFO_TYPE_VAR)) - { - // Make sure that all valuetypes (including enums) that we push are loaded. - // This is to guarantee that if a GC is triggered from the prestub of this methods, - // all valuetypes in the method signature are already loaded. - // We need to be able to find the size of the valuetypes, but we cannot - // do a class-load from within GC. - info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(sig->retTypeSigClass); - } + if (varTypeIsLong(baseType)) + { + hwintrinsic = NI_SSE42_X64_Crc32; + op1 = gtFoldExpr(gtNewCastNode(baseType, op1, /* unsigned */ true, baseType)); + } + else + { + hwintrinsic = NI_SSE42_Crc32; + baseType = genActualType(baseType); + } - // Now create the arguments in reverse. - for (unsigned i = sig->numArgs; i > 0; i--) - { - StackEntry se = impPopStack(); - typeInfo ti = se.seTypeInfo; - GenTree* argNode = se.val; + result = gtNewScalarHWIntrinsicNode(baseType, op1, op2, hwintrinsic); - var_types jitSigType = JITtype2varType(params[i - 1].CorType); - CORINFO_CLASS_HANDLE classHnd = params[i - 1].ClassHandle; + // We use the simdBaseJitType to bring the type of the second argument to codegen + result->AsHWIntrinsic()->SetSimdBaseJitType(baseJitType); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Crc32)) + { + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; - if (!impCheckImplicitArgumentCoercion(jitSigType, argNode->TypeGet())) - { - BADCODE("the call argument has a type that can't be implicitly converted to the signature type"); + hwintrinsic = varTypeIsLong(baseType) ? NI_Crc32_Arm64_ComputeCrc32C : NI_Crc32_ComputeCrc32C; + result = gtNewScalarHWIntrinsicNode(TYP_INT, op1, op2, hwintrinsic); + baseType = TYP_INT; + + // We use the simdBaseJitType to bring the type of the second argument to codegen + result->AsHWIntrinsic()->SetSimdBaseJitType(baseJitType); + } +#endif // TARGET_* +#endif // FEATURE_HW_INTRINSICS + + break; } - if (varTypeIsStruct(argNode)) + case NI_PRIMITIVE_LeadingZeroCount: { - // Morph trees that aren't already OBJs or MKREFANY to be OBJs - assert(ti.IsType(TI_STRUCT)); + assert(sig->numArgs == 1); + assert(!varTypeIsSmall(retType) && !varTypeIsSmall(baseType)); - JITDUMP("Calling impNormStructVal on:\n"); - DISPTREE(argNode); + GenTree* op1 = impStackTop().val; - argNode = impNormStructVal(argNode, classHnd, CHECK_SPILL_ALL); - // For SIMD types the normalization can normalize TYP_STRUCT to - // e.g. TYP_SIMD16 which we keep (along with the class handle) in - // the CallArgs. - jitSigType = argNode->TypeGet(); + if (op1->IsIntegralConst()) + { + // Pop the value from the stack + impPopStack(); - JITDUMP("resulting tree:\n"); - DISPTREE(argNode); - } - else - { - // Insert implied casts (from float to double or double to float). - argNode = impImplicitR4orR8Cast(argNode, jitSigType); - // insert any widening or narrowing casts for backwards compatibility - argNode = impImplicitIorI4Cast(argNode, jitSigType); - } + if (varTypeIsLong(baseType)) + { + uint64_t cns = static_cast(op1->AsIntConCommon()->LngValue()); + result = gtNewLconNode(BitOperations::LeadingZeroCount(cns)); + } + else + { + uint32_t cns = static_cast(op1->AsIntConCommon()->IconValue()); + result = gtNewIconNode(BitOperations::LeadingZeroCount(cns), baseType); + } + break; + } - NewCallArg arg; - if (varTypeIsStruct(jitSigType)) - { - arg = NewCallArg::Struct(argNode, jitSigType, classHnd); - } - else - { - arg = NewCallArg::Primitive(argNode, jitSigType); - } +#if !defined(TARGET_64BIT) + if (varTypeIsLong(baseType)) + { + // TODO-CQ: Adding long decomposition support is more complex + // and not supported today so early exit if we have a long and + // either input is not a constant. - call->gtArgs.PushFront(this, arg); - call->gtFlags |= argNode->gtFlags & GTF_GLOB_EFFECT; - } -} + break; + } +#endif // !TARGET_64BIT -/***************************************************************************** - * - * Pop the given number of values from the stack in reverse order (STDCALL/CDECL etc.) - * The first "skipReverseCount" items are not reversed. - */ +#if defined(FEATURE_HW_INTRINSICS) +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_LZCNT)) + { + // Pop the value from the stack + impPopStack(); -void Compiler::impPopReverseCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call, unsigned skipReverseCount) -{ - assert(skipReverseCount <= sig->numArgs); + hwintrinsic = varTypeIsLong(baseType) ? NI_LZCNT_X64_LeadingZeroCount : NI_LZCNT_LeadingZeroCount; + result = gtNewScalarHWIntrinsicNode(baseType, op1, hwintrinsic); + } + else if (compOpportunisticallyDependsOn(InstructionSet_X86Base)) + { + // Pop the value from the stack + impPopStack(); - impPopCallArgs(sig, call); + // We're importing this as the following... + // * 32-bit lzcnt: (value == 0) ? 32 : (31 ^ BSR(value)) + // * 64-bit lzcnt: (value == 0) ? 64 : (63 ^ BSR(value)) - call->gtArgs.Reverse(skipReverseCount, sig->numArgs - skipReverseCount); -} + GenTree* op1Dup; + op1 = impCloneExpr(op1, &op1Dup, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("Cloning op1 for LeadingZeroCount")); -GenTree* Compiler::impTransformThis(GenTree* thisPtr, - CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, - CORINFO_THIS_TRANSFORM transform) -{ - switch (transform) - { - case CORINFO_DEREF_THIS: - { - GenTree* obj = thisPtr; + hwintrinsic = varTypeIsLong(baseType) ? NI_X86Base_X64_BitScanReverse : NI_X86Base_BitScanReverse; + op1Dup = gtNewScalarHWIntrinsicNode(baseType, op1Dup, hwintrinsic); - // This does a LDIND on the obj, which should be a byref. pointing to a ref - impBashVarAddrsToI(obj); - assert(genActualType(obj->gtType) == TYP_I_IMPL || obj->gtType == TYP_BYREF); - CorInfoType constraintTyp = info.compCompHnd->asCorInfoType(pConstrainedResolvedToken->hClass); + GenTree* cond = gtFoldExpr(gtNewOperNode(GT_EQ, TYP_INT, op1, gtNewZeroConNode(baseType))); - obj = gtNewOperNode(GT_IND, JITtype2varType(constraintTyp), obj); + GenTree* trueRes; + GenTree* icon; + + if (varTypeIsLong(baseType)) + { + trueRes = gtNewLconNode(64); + icon = gtNewLconNode(63); + } + else + { + trueRes = gtNewIconNode(32, baseType); + icon = gtNewIconNode(31, baseType); + } + + GenTree* falseRes = gtNewOperNode(GT_XOR, baseType, op1Dup, icon); + GenTreeColon* colon = gtNewColonNode(baseType, trueRes, falseRes); + + result = gtNewQmarkNode(baseType, cond, colon); + + unsigned tmp = lvaGrabTemp(true DEBUGARG("Grabbing temp for LeadingZeroCount Qmark")); + impAssignTempGen(tmp, result, CHECK_SPILL_NONE); + result = gtNewLclvNode(tmp, baseType); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_ArmBase)) + { + // Pop the value from the stack + impPopStack(); + + hwintrinsic = varTypeIsLong(baseType) ? NI_ArmBase_Arm64_LeadingZeroCount : NI_ArmBase_LeadingZeroCount; + result = gtNewScalarHWIntrinsicNode(TYP_INT, op1, hwintrinsic); + baseType = TYP_INT; + } +#endif // TARGET_* +#endif // FEATURE_HW_INTRINSICS + + break; + } + + case NI_PRIMITIVE_Log2: + { + assert(sig->numArgs == 1); + assert(!varTypeIsSmall(retType) && !varTypeIsSmall(baseType)); + + GenTree* op1 = impStackTop().val; + + if (op1->IsIntegralConst()) + { + // Pop the value from the stack + impPopStack(); + + if (varTypeIsLong(baseType)) + { + uint64_t cns = static_cast(op1->AsIntConCommon()->LngValue()); + + if (varTypeIsUnsigned(JitType2PreciseVarType(baseJitType)) || (static_cast(cns) >= 0)) + { + result = gtNewLconNode(BitOperations::Log2(cns)); + } + } + else + { + uint32_t cns = static_cast(op1->AsIntConCommon()->IconValue()); + + if (varTypeIsUnsigned(JitType2PreciseVarType(baseJitType)) || (static_cast(cns) >= 0)) + { + result = gtNewIconNode(BitOperations::Log2(cns), baseType); + } + } + break; + } + +#if !defined(TARGET_64BIT) + if (varTypeIsLong(baseType)) + { + // TODO-CQ: Adding long decomposition support is more complex + // and not supported today so early exit if we have a long and + // either input is not a constant. + + break; + } +#endif // !TARGET_64BIT + + if (varTypeIsSigned(baseType)) + { + // TODO-CQ: We should insert the `if (value < 0) { throw }` handling + break; + } + +#if defined(FEATURE_HW_INTRINSICS) + GenTree* lzcnt = impPrimitiveNamedIntrinsic(NI_PRIMITIVE_LeadingZeroCount, clsHnd, method, sig); + + if (lzcnt != nullptr) + { + GenTree* icon; + + if (varTypeIsLong(retType)) + { + icon = gtNewLconNode(63); + } + else + { + icon = gtNewIconNode(31, retType); + } + + result = gtNewOperNode(GT_XOR, retType, lzcnt, icon); + baseType = retType; + } +#endif // FEATURE_HW_INTRINSICS + + break; + } + + case NI_PRIMITIVE_PopCount: + { + assert(sig->numArgs == 1); + assert(!varTypeIsSmall(retType) && !varTypeIsSmall(baseType)); + + GenTree* op1 = impStackTop().val; + + if (op1->IsIntegralConst()) + { + // Pop the value from the stack + impPopStack(); + + if (varTypeIsLong(baseType)) + { + uint64_t cns = static_cast(op1->AsIntConCommon()->LngValue()); + result = gtNewLconNode(BitOperations::PopCount(cns)); + } + else + { + uint32_t cns = static_cast(op1->AsIntConCommon()->IconValue()); + result = gtNewIconNode(BitOperations::PopCount(cns), baseType); + } + break; + } + +#if !defined(TARGET_64BIT) + if (varTypeIsLong(baseType)) + { + // TODO-CQ: Adding long decomposition support is more complex + // and not supported today so early exit if we have a long and + // either input is not a constant. + + break; + } +#endif // !TARGET_64BIT + +#if defined(FEATURE_HW_INTRINSICS) +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_POPCNT)) + { + // Pop the value from the stack + impPopStack(); + + hwintrinsic = varTypeIsLong(baseType) ? NI_POPCNT_X64_PopCount : NI_POPCNT_PopCount; + result = gtNewScalarHWIntrinsicNode(baseType, op1, hwintrinsic); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_AdvSimd)) + { + // TODO-ARM64-CQ: PopCount should be handled as an intrinsic for non-constant cases + } +#endif // TARGET_* +#endif // FEATURE_HW_INTRINSICS + + break; + } + + case NI_PRIMITIVE_RotateLeft: + { + assert(sig->numArgs == 2); + assert(!varTypeIsSmall(retType) && !varTypeIsSmall(baseType)); + + GenTree* op2 = impStackTop().val; + + if (!op2->IsIntegralConst()) + { + // TODO-CQ: ROL currently expects op2 to be a constant + break; + } + + // Pop the value from the stack + impPopStack(); + + GenTree* op1 = impPopStack().val; + uint32_t cns2 = static_cast(op2->AsIntConCommon()->IconValue()); + + // Mask the offset to ensure deterministic xplat behavior for overshifting + cns2 &= varTypeIsLong(baseType) ? 0x3F : 0x1F; + + if (cns2 == 0) + { + // No rotation is a nop + return op1; + } + + if (op1->IsIntegralConst()) + { + if (varTypeIsLong(baseType)) + { + uint64_t cns1 = static_cast(op1->AsIntConCommon()->LngValue()); + result = gtNewLconNode(BitOperations::RotateLeft(cns1, cns2)); + } + else + { + uint32_t cns1 = static_cast(op1->AsIntConCommon()->IconValue()); + result = gtNewIconNode(BitOperations::RotateLeft(cns1, cns2), baseType); + } + break; + } + + op2->AsIntConCommon()->SetIconValue(cns2); + result = gtFoldExpr(gtNewOperNode(GT_ROL, baseType, op1, op2)); + + break; + } + + case NI_PRIMITIVE_RotateRight: + { + assert(sig->numArgs == 2); + assert(!varTypeIsSmall(retType) && !varTypeIsSmall(baseType)); + + GenTree* op2 = impStackTop().val; + + if (!op2->IsIntegralConst()) + { + // TODO-CQ: ROR currently expects op2 to be a constant + break; + } + + // Pop the value from the stack + impPopStack(); + + GenTree* op1 = impPopStack().val; + uint32_t cns2 = static_cast(op2->AsIntConCommon()->IconValue()); + + // Mask the offset to ensure deterministic xplat behavior for overshifting + cns2 &= varTypeIsLong(baseType) ? 0x3F : 0x1F; + + if (cns2 == 0) + { + // No rotation is a nop + return op1; + } + + if (op1->IsIntegralConst()) + { + if (varTypeIsLong(baseType)) + { + uint64_t cns1 = static_cast(op1->AsIntConCommon()->LngValue()); + result = gtNewLconNode(BitOperations::RotateRight(cns1, cns2)); + } + else + { + uint32_t cns1 = static_cast(op1->AsIntConCommon()->IconValue()); + result = gtNewIconNode(BitOperations::RotateRight(cns1, cns2), baseType); + } + break; + } + + op2->AsIntConCommon()->SetIconValue(cns2); + result = gtFoldExpr(gtNewOperNode(GT_ROR, baseType, op1, op2)); + + break; + } + + case NI_PRIMITIVE_TrailingZeroCount: + { + assert(sig->numArgs == 1); + assert(!varTypeIsSmall(baseType)); + + GenTree* op1 = impStackTop().val; + + if (op1->IsIntegralConst()) + { + // Pop the value from the stack + impPopStack(); + + if (varTypeIsLong(baseType)) + { + uint64_t cns = static_cast(op1->AsIntConCommon()->LngValue()); + result = gtNewLconNode(BitOperations::TrailingZeroCount(cns)); + } + else + { + uint32_t cns = static_cast(op1->AsIntConCommon()->IconValue()); + result = gtNewIconNode(BitOperations::TrailingZeroCount(cns), baseType); + } + + baseType = retType; + break; + } + +#if !defined(TARGET_64BIT) + if (varTypeIsLong(baseType)) + { + // TODO-CQ: Adding long decomposition support is more complex + // and not supported today so early exit if we have a long and + // either input is not a constant. + + break; + } +#endif // !TARGET_64BIT + +#if defined(FEATURE_HW_INTRINSICS) +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_BMI1)) + { + // Pop the value from the stack + impPopStack(); + + hwintrinsic = varTypeIsLong(baseType) ? NI_BMI1_X64_TrailingZeroCount : NI_BMI1_TrailingZeroCount; + result = gtNewScalarHWIntrinsicNode(baseType, op1, hwintrinsic); + } + else if (compOpportunisticallyDependsOn(InstructionSet_X86Base)) + { + // Pop the value from the stack + impPopStack(); + + // We're importing this as the following... + // * 32-bit tzcnt: (value == 0) ? 32 : BSF(value) + // * 64-bit tzcnt: (value == 0) ? 64 : BSF(value) + + GenTree* op1Dup; + op1 = impCloneExpr(op1, &op1Dup, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("Cloning op1 for TrailingZeroCount")); + + hwintrinsic = varTypeIsLong(baseType) ? NI_X86Base_X64_BitScanForward : NI_X86Base_BitScanForward; + op1Dup = gtNewScalarHWIntrinsicNode(baseType, op1Dup, hwintrinsic); + + GenTree* cond = gtFoldExpr(gtNewOperNode(GT_EQ, TYP_INT, op1, gtNewZeroConNode(baseType))); + + GenTree* trueRes; + + if (varTypeIsLong(baseType)) + { + trueRes = gtNewLconNode(64); + } + else + { + trueRes = gtNewIconNode(32, baseType); + } + + GenTree* falseRes = op1Dup; + GenTreeColon* colon = gtNewColonNode(baseType, trueRes, falseRes); + + result = gtNewQmarkNode(baseType, cond, colon); + + unsigned tmp = lvaGrabTemp(true DEBUGARG("Grabbing temp for TrailingZeroCount Qmark")); + impAssignTempGen(tmp, result, CHECK_SPILL_NONE); + result = gtNewLclvNode(tmp, baseType); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_ArmBase)) + { + // Pop the value from the stack + impPopStack(); + + hwintrinsic = + varTypeIsLong(baseType) ? NI_ArmBase_Arm64_ReverseElementBits : NI_ArmBase_ReverseElementBits; + op1 = gtNewScalarHWIntrinsicNode(baseType, op1, hwintrinsic); + + hwintrinsic = varTypeIsLong(baseType) ? NI_ArmBase_Arm64_LeadingZeroCount : NI_ArmBase_LeadingZeroCount; + result = gtNewScalarHWIntrinsicNode(baseType, op1, hwintrinsic); + } +#endif // TARGET_* +#endif // FEATURE_HW_INTRINSICS + + break; + } + + default: + { + unreached(); + } + } + + if ((result != nullptr) && (retType != baseType)) + { + // We're either LONG->INT or INT->LONG + assert(!varTypeIsSmall(retType) && !varTypeIsSmall(baseType)); + result = gtFoldExpr(gtNewCastNode(retType, result, /* unsigned */ true, retType)); + } + + return result; +} + +//------------------------------------------------------------------------ +// impPopCallArgs: +// Pop the given number of values from the stack and return a list node with +// their values. +// +// Parameters: +// sig - Signature used to figure out classes the runtime must load, and +// also to record exact receiving argument types that may be needed for ABI +// purposes later. +// call - The call to pop arguments into. +// +void Compiler::impPopCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call) +{ + assert(call->gtArgs.IsEmpty()); + + if (impStackHeight() < sig->numArgs) + { + BADCODE("not enough arguments for call"); + } + + struct SigParamInfo + { + CorInfoType CorType; + CORINFO_CLASS_HANDLE ClassHandle; + }; + + SigParamInfo inlineParams[16]; + SigParamInfo* params = sig->numArgs <= 16 ? inlineParams : new (this, CMK_CallArgs) SigParamInfo[sig->numArgs]; + + // We will iterate and pop the args in reverse order as we sometimes need + // to spill some args. However, we need signature information and the + // JIT-EE interface only allows us to iterate the signature forwards. We + // will collect the needed information here and at the same time notify the + // EE that the signature types need to be loaded. + CORINFO_ARG_LIST_HANDLE sigArg = sig->args; + for (unsigned i = 0; i < sig->numArgs; i++) + { + params[i].CorType = strip(info.compCompHnd->getArgType(sig, sigArg, ¶ms[i].ClassHandle)); + + if (params[i].CorType != CORINFO_TYPE_CLASS && params[i].CorType != CORINFO_TYPE_BYREF && + params[i].CorType != CORINFO_TYPE_PTR && params[i].CorType != CORINFO_TYPE_VAR) + { + CORINFO_CLASS_HANDLE argRealClass = info.compCompHnd->getArgClass(sig, sigArg); + if (argRealClass != nullptr) + { + // Make sure that all valuetypes (including enums) that we push are loaded. + // This is to guarantee that if a GC is triggered from the prestub of this methods, + // all valuetypes in the method signature are already loaded. + // We need to be able to find the size of the valuetypes, but we cannot + // do a class-load from within GC. + info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(argRealClass); + } + } + + sigArg = info.compCompHnd->getArgNext(sigArg); + } + + if ((sig->retTypeSigClass != nullptr) && (sig->retType != CORINFO_TYPE_CLASS) && + (sig->retType != CORINFO_TYPE_BYREF) && (sig->retType != CORINFO_TYPE_PTR) && + (sig->retType != CORINFO_TYPE_VAR)) + { + // Make sure that all valuetypes (including enums) that we push are loaded. + // This is to guarantee that if a GC is triggered from the prestub of this methods, + // all valuetypes in the method signature are already loaded. + // We need to be able to find the size of the valuetypes, but we cannot + // do a class-load from within GC. + info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(sig->retTypeSigClass); + } + + // Now create the arguments in reverse. + for (unsigned i = sig->numArgs; i > 0; i--) + { + StackEntry se = impPopStack(); + typeInfo ti = se.seTypeInfo; + GenTree* argNode = se.val; + + var_types jitSigType = JITtype2varType(params[i - 1].CorType); + CORINFO_CLASS_HANDLE classHnd = params[i - 1].ClassHandle; + + if (!impCheckImplicitArgumentCoercion(jitSigType, argNode->TypeGet())) + { + BADCODE("the call argument has a type that can't be implicitly converted to the signature type"); + } + + if (varTypeIsStruct(argNode)) + { + // Morph trees that aren't already OBJs or MKREFANY to be OBJs + assert(ti.IsType(TI_STRUCT)); + + JITDUMP("Calling impNormStructVal on:\n"); + DISPTREE(argNode); + + argNode = impNormStructVal(argNode, classHnd, CHECK_SPILL_ALL); + // For SIMD types the normalization can normalize TYP_STRUCT to + // e.g. TYP_SIMD16 which we keep (along with the class handle) in + // the CallArgs. + jitSigType = argNode->TypeGet(); + + JITDUMP("resulting tree:\n"); + DISPTREE(argNode); + } + else + { + // Insert implied casts (from float to double or double to float). + argNode = impImplicitR4orR8Cast(argNode, jitSigType); + // insert any widening or narrowing casts for backwards compatibility + argNode = impImplicitIorI4Cast(argNode, jitSigType); + } + + NewCallArg arg; + if (varTypeIsStruct(jitSigType)) + { + arg = NewCallArg::Struct(argNode, jitSigType, classHnd); + } + else + { + arg = NewCallArg::Primitive(argNode, jitSigType); + } + + call->gtArgs.PushFront(this, arg); + call->gtFlags |= argNode->gtFlags & GTF_GLOB_EFFECT; + } +} + +/***************************************************************************** + * + * Pop the given number of values from the stack in reverse order (STDCALL/CDECL etc.) + * The first "skipReverseCount" items are not reversed. + */ + +void Compiler::impPopReverseCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call, unsigned skipReverseCount) +{ + assert(skipReverseCount <= sig->numArgs); + + impPopCallArgs(sig, call); + + call->gtArgs.Reverse(skipReverseCount, sig->numArgs - skipReverseCount); +} + +GenTree* Compiler::impTransformThis(GenTree* thisPtr, + CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, + CORINFO_THIS_TRANSFORM transform) +{ + switch (transform) + { + case CORINFO_DEREF_THIS: + { + GenTree* obj = thisPtr; + + // This does a LDIND on the obj, which should be a byref. pointing to a ref + impBashVarAddrsToI(obj); + assert(genActualType(obj->gtType) == TYP_I_IMPL || obj->gtType == TYP_BYREF); + CorInfoType constraintTyp = info.compCompHnd->asCorInfoType(pConstrainedResolvedToken->hClass); + + obj = gtNewOperNode(GT_IND, JITtype2varType(constraintTyp), obj); // ldind could point anywhere, example a boxed class static int obj->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF); @@ -7012,351 +7600,421 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) } JITDUMP(": "); - - NamedIntrinsic result = NI_Illegal; - - if (strncmp(namespaceName, "System", 6) == 0) - { - namespaceName += 6; - - if (namespaceName[0] == '\0') - { - if (strcmp(className, "Activator") == 0) - { - if (strcmp(methodName, "AllocatorOf") == 0) - { - result = NI_System_Activator_AllocatorOf; - } - else if (strcmp(methodName, "DefaultConstructorOf") == 0) - { - result = NI_System_Activator_DefaultConstructorOf; - } - } - else if (strcmp(className, "Array") == 0) - { - if (strcmp(methodName, "Clone") == 0) - { - result = NI_System_Array_Clone; - } - else if (strcmp(methodName, "GetLength") == 0) - { - result = NI_System_Array_GetLength; - } - else if (strcmp(methodName, "GetLowerBound") == 0) - { - result = NI_System_Array_GetLowerBound; - } - else if (strcmp(methodName, "GetUpperBound") == 0) - { - result = NI_System_Array_GetUpperBound; - } - } - else if (strcmp(className, "BitConverter") == 0) - { - if (strcmp(methodName, "DoubleToInt64Bits") == 0) - { - result = NI_System_BitConverter_DoubleToInt64Bits; - } - else if (strcmp(methodName, "DoubleToUInt64Bits") == 0) - { - result = NI_System_BitConverter_DoubleToInt64Bits; - } - else if (strcmp(methodName, "Int32BitsToSingle") == 0) - { - result = NI_System_BitConverter_Int32BitsToSingle; - } - else if (strcmp(methodName, "Int64BitsToDouble") == 0) - { - result = NI_System_BitConverter_Int64BitsToDouble; - } - else if (strcmp(methodName, "SingleToInt32Bits") == 0) - { - result = NI_System_BitConverter_SingleToInt32Bits; - } - else if (strcmp(methodName, "SingleToUInt32Bits") == 0) - { - result = NI_System_BitConverter_SingleToInt32Bits; - } - else if (strcmp(methodName, "UInt32BitsToSingle") == 0) - { - result = NI_System_BitConverter_Int32BitsToSingle; - } - else if (strcmp(methodName, "UInt64BitsToDouble") == 0) - { - result = NI_System_BitConverter_Int64BitsToDouble; - } - } - else if (strcmp(className, "Enum") == 0) - { - if (strcmp(methodName, "HasFlag") == 0) - { - result = NI_System_Enum_HasFlag; - } - } - else if (strcmp(className, "EETypePtr") == 0) - { - if (strcmp(methodName, "EETypePtrOf") == 0) - { - result = NI_System_EETypePtr_EETypePtrOf; - } - } - else if (strcmp(className, "GC") == 0) - { - if (strcmp(methodName, "KeepAlive") == 0) - { - result = NI_System_GC_KeepAlive; - } - } - else if (strcmp(className, "Math") == 0 || strcmp(className, "MathF") == 0) - { - if (strcmp(methodName, "Abs") == 0) - { - result = NI_System_Math_Abs; - } - else if (strcmp(methodName, "Acos") == 0) - { - result = NI_System_Math_Acos; - } - else if (strcmp(methodName, "Acosh") == 0) - { - result = NI_System_Math_Acosh; - } - else if (strcmp(methodName, "Asin") == 0) - { - result = NI_System_Math_Asin; - } - else if (strcmp(methodName, "Asinh") == 0) - { - result = NI_System_Math_Asinh; - } - else if (strcmp(methodName, "Atan") == 0) - { - result = NI_System_Math_Atan; - } - else if (strcmp(methodName, "Atanh") == 0) - { - result = NI_System_Math_Atanh; - } - else if (strcmp(methodName, "Atan2") == 0) - { - result = NI_System_Math_Atan2; - } - else if (strcmp(methodName, "Cbrt") == 0) - { - result = NI_System_Math_Cbrt; - } - else if (strcmp(methodName, "Ceiling") == 0) - { - result = NI_System_Math_Ceiling; - } - else if (strcmp(methodName, "Cos") == 0) - { - result = NI_System_Math_Cos; - } - else if (strcmp(methodName, "Cosh") == 0) - { - result = NI_System_Math_Cosh; - } - else if (strcmp(methodName, "Exp") == 0) - { - result = NI_System_Math_Exp; - } - else if (strcmp(methodName, "Floor") == 0) - { - result = NI_System_Math_Floor; - } - else if (strcmp(methodName, "FMod") == 0) - { - result = NI_System_Math_FMod; - } - else if (strcmp(methodName, "FusedMultiplyAdd") == 0) - { - result = NI_System_Math_FusedMultiplyAdd; - } - else if (strcmp(methodName, "ILogB") == 0) - { - result = NI_System_Math_ILogB; - } - else if (strcmp(methodName, "Log") == 0) - { - result = NI_System_Math_Log; - } - else if (strcmp(methodName, "Log2") == 0) - { - result = NI_System_Math_Log2; - } - else if (strcmp(methodName, "Log10") == 0) - { - result = NI_System_Math_Log10; - } - else if (strcmp(methodName, "Max") == 0) - { - result = NI_System_Math_Max; - } - else if (strcmp(methodName, "Min") == 0) - { - result = NI_System_Math_Min; - } - else if (strcmp(methodName, "Pow") == 0) - { - result = NI_System_Math_Pow; - } - else if (strcmp(methodName, "Round") == 0) - { - result = NI_System_Math_Round; - } - else if (strcmp(methodName, "Sin") == 0) - { - result = NI_System_Math_Sin; - } - else if (strcmp(methodName, "Sinh") == 0) - { - result = NI_System_Math_Sinh; - } - else if (strcmp(methodName, "Sqrt") == 0) - { - result = NI_System_Math_Sqrt; - } - else if (strcmp(methodName, "Tan") == 0) - { - result = NI_System_Math_Tan; - } - else if (strcmp(methodName, "Tanh") == 0) - { - result = NI_System_Math_Tanh; - } - else if (strcmp(methodName, "Truncate") == 0) - { - result = NI_System_Math_Truncate; - } - } - else if (strcmp(className, "MemoryExtensions") == 0) - { - if (strcmp(methodName, "AsSpan") == 0) - { - result = NI_System_MemoryExtensions_AsSpan; - } - else if (strcmp(methodName, "Equals") == 0) - { - result = NI_System_MemoryExtensions_Equals; - } - else if (strcmp(methodName, "SequenceEqual") == 0) - { - result = NI_System_MemoryExtensions_SequenceEqual; - } - else if (strcmp(methodName, "StartsWith") == 0) - { - result = NI_System_MemoryExtensions_StartsWith; - } - } - else if (strcmp(className, "Object") == 0) - { - if (strcmp(methodName, "GetType") == 0) - { - result = NI_System_Object_GetType; - } - else if (strcmp(methodName, "MemberwiseClone") == 0) - { - result = NI_System_Object_MemberwiseClone; - } - } - else if (strcmp(className, "ReadOnlySpan`1") == 0) - { - if (strcmp(methodName, "get_Item") == 0) - { - result = NI_System_ReadOnlySpan_get_Item; - } - else if (strcmp(methodName, "get_Length") == 0) - { - result = NI_System_ReadOnlySpan_get_Length; - } - } - else if (strcmp(className, "RuntimeType") == 0) - { - if (strcmp(methodName, "get_IsActualEnum") == 0) - { - result = NI_System_Type_get_IsEnum; - } - } - else if (strcmp(className, "RuntimeTypeHandle") == 0) - { - if (strcmp(methodName, "GetValueInternal") == 0) - { - result = NI_System_RuntimeTypeHandle_GetValueInternal; - } - } - else if (strcmp(className, "Span`1") == 0) - { - if (strcmp(methodName, "get_Item") == 0) - { - result = NI_System_Span_get_Item; - } - else if (strcmp(methodName, "get_Length") == 0) - { - result = NI_System_Span_get_Length; - } - } - else if (strcmp(className, "String") == 0) + + NamedIntrinsic result = NI_Illegal; + + if (strncmp(namespaceName, "System", 6) == 0) + { + namespaceName += 6; + + if (namespaceName[0] == '\0') + { + switch (className[0]) { - if (strcmp(methodName, "Equals") == 0) - { - result = NI_System_String_Equals; - } - else if (strcmp(methodName, "get_Chars") == 0) - { - result = NI_System_String_get_Chars; - } - else if (strcmp(methodName, "get_Length") == 0) - { - result = NI_System_String_get_Length; - } - else if (strcmp(methodName, "op_Implicit") == 0) + case 'A': { - result = NI_System_String_op_Implicit; + if (strcmp(className, "Activator") == 0) + { + if (strcmp(methodName, "AllocatorOf") == 0) + { + result = NI_System_Activator_AllocatorOf; + } + else if (strcmp(methodName, "DefaultConstructorOf") == 0) + { + result = NI_System_Activator_DefaultConstructorOf; + } + } + else if (strcmp(className, "Array") == 0) + { + if (strcmp(methodName, "Clone") == 0) + { + result = NI_System_Array_Clone; + } + else if (strcmp(methodName, "GetLength") == 0) + { + result = NI_System_Array_GetLength; + } + else if (strcmp(methodName, "GetLowerBound") == 0) + { + result = NI_System_Array_GetLowerBound; + } + else if (strcmp(methodName, "GetUpperBound") == 0) + { + result = NI_System_Array_GetUpperBound; + } + } + break; } - else if (strcmp(methodName, "StartsWith") == 0) + + case 'B': { - result = NI_System_String_StartsWith; + if (strcmp(className, "BitConverter") == 0) + { + if (strcmp(methodName, "DoubleToInt64Bits") == 0) + { + result = NI_System_BitConverter_DoubleToInt64Bits; + } + else if (strcmp(methodName, "DoubleToUInt64Bits") == 0) + { + result = NI_System_BitConverter_DoubleToInt64Bits; + } + else if (strcmp(methodName, "Int32BitsToSingle") == 0) + { + result = NI_System_BitConverter_Int32BitsToSingle; + } + else if (strcmp(methodName, "Int64BitsToDouble") == 0) + { + result = NI_System_BitConverter_Int64BitsToDouble; + } + else if (strcmp(methodName, "SingleToInt32Bits") == 0) + { + result = NI_System_BitConverter_SingleToInt32Bits; + } + else if (strcmp(methodName, "SingleToUInt32Bits") == 0) + { + result = NI_System_BitConverter_SingleToInt32Bits; + } + else if (strcmp(methodName, "UInt32BitsToSingle") == 0) + { + result = NI_System_BitConverter_Int32BitsToSingle; + } + else if (strcmp(methodName, "UInt64BitsToDouble") == 0) + { + result = NI_System_BitConverter_Int64BitsToDouble; + } + } + break; } - } - else if (strcmp(className, "Type") == 0) - { - if (strcmp(methodName, "get_IsEnum") == 0) + + case 'E': { - result = NI_System_Type_get_IsEnum; + if (strcmp(className, "Enum") == 0) + { + if (strcmp(methodName, "HasFlag") == 0) + { + result = NI_System_Enum_HasFlag; + } + } + else if (strcmp(className, "EETypePtr") == 0) + { + if (strcmp(methodName, "EETypePtrOf") == 0) + { + result = NI_System_EETypePtr_EETypePtrOf; + } + } + break; } - else if (strcmp(methodName, "get_IsValueType") == 0) + + case 'G': { - result = NI_System_Type_get_IsValueType; + if (strcmp(className, "GC") == 0) + { + if (strcmp(methodName, "KeepAlive") == 0) + { + result = NI_System_GC_KeepAlive; + } + } + break; } - else if (strcmp(methodName, "get_IsByRefLike") == 0) + + case 'I': { - result = NI_System_Type_get_IsByRefLike; + if ((strcmp(className, "Int32") == 0) || (strcmp(className, "Int64") == 0) || + (strcmp(className, "IntPtr") == 0)) + { + result = lookupPrimitiveNamedIntrinsic(method, methodName); + } + break; } - else if (strcmp(methodName, "GetEnumUnderlyingType") == 0) + + case 'M': { - result = NI_System_Type_GetEnumUnderlyingType; + if ((strcmp(className, "Math") == 0) || (strcmp(className, "MathF") == 0)) + { + if (strcmp(methodName, "Abs") == 0) + { + result = NI_System_Math_Abs; + } + else if (strcmp(methodName, "Acos") == 0) + { + result = NI_System_Math_Acos; + } + else if (strcmp(methodName, "Acosh") == 0) + { + result = NI_System_Math_Acosh; + } + else if (strcmp(methodName, "Asin") == 0) + { + result = NI_System_Math_Asin; + } + else if (strcmp(methodName, "Asinh") == 0) + { + result = NI_System_Math_Asinh; + } + else if (strcmp(methodName, "Atan") == 0) + { + result = NI_System_Math_Atan; + } + else if (strcmp(methodName, "Atanh") == 0) + { + result = NI_System_Math_Atanh; + } + else if (strcmp(methodName, "Atan2") == 0) + { + result = NI_System_Math_Atan2; + } + else if (strcmp(methodName, "Cbrt") == 0) + { + result = NI_System_Math_Cbrt; + } + else if (strcmp(methodName, "Ceiling") == 0) + { + result = NI_System_Math_Ceiling; + } + else if (strcmp(methodName, "Cos") == 0) + { + result = NI_System_Math_Cos; + } + else if (strcmp(methodName, "Cosh") == 0) + { + result = NI_System_Math_Cosh; + } + else if (strcmp(methodName, "Exp") == 0) + { + result = NI_System_Math_Exp; + } + else if (strcmp(methodName, "Floor") == 0) + { + result = NI_System_Math_Floor; + } + else if (strcmp(methodName, "FMod") == 0) + { + result = NI_System_Math_FMod; + } + else if (strcmp(methodName, "FusedMultiplyAdd") == 0) + { + result = NI_System_Math_FusedMultiplyAdd; + } + else if (strcmp(methodName, "ILogB") == 0) + { + result = NI_System_Math_ILogB; + } + else if (strcmp(methodName, "Log") == 0) + { + result = NI_System_Math_Log; + } + else if (strcmp(methodName, "Log2") == 0) + { + result = NI_System_Math_Log2; + } + else if (strcmp(methodName, "Log10") == 0) + { + result = NI_System_Math_Log10; + } + else if (strcmp(methodName, "Max") == 0) + { + result = NI_System_Math_Max; + } + else if (strcmp(methodName, "Min") == 0) + { + result = NI_System_Math_Min; + } + else if (strcmp(methodName, "Pow") == 0) + { + result = NI_System_Math_Pow; + } + else if (strcmp(methodName, "Round") == 0) + { + result = NI_System_Math_Round; + } + else if (strcmp(methodName, "Sin") == 0) + { + result = NI_System_Math_Sin; + } + else if (strcmp(methodName, "Sinh") == 0) + { + result = NI_System_Math_Sinh; + } + else if (strcmp(methodName, "Sqrt") == 0) + { + result = NI_System_Math_Sqrt; + } + else if (strcmp(methodName, "Tan") == 0) + { + result = NI_System_Math_Tan; + } + else if (strcmp(methodName, "Tanh") == 0) + { + result = NI_System_Math_Tanh; + } + else if (strcmp(methodName, "Truncate") == 0) + { + result = NI_System_Math_Truncate; + } + } + else if (strcmp(className, "MemoryExtensions") == 0) + { + if (strcmp(methodName, "AsSpan") == 0) + { + result = NI_System_MemoryExtensions_AsSpan; + } + else if (strcmp(methodName, "Equals") == 0) + { + result = NI_System_MemoryExtensions_Equals; + } + else if (strcmp(methodName, "SequenceEqual") == 0) + { + result = NI_System_MemoryExtensions_SequenceEqual; + } + else if (strcmp(methodName, "StartsWith") == 0) + { + result = NI_System_MemoryExtensions_StartsWith; + } + } + break; } - else if (strcmp(methodName, "GetTypeFromHandle") == 0) + + case 'O': { - result = NI_System_Type_GetTypeFromHandle; + if (strcmp(className, "Object") == 0) + { + if (strcmp(methodName, "GetType") == 0) + { + result = NI_System_Object_GetType; + } + else if (strcmp(methodName, "MemberwiseClone") == 0) + { + result = NI_System_Object_MemberwiseClone; + } + } + break; } - else if (strcmp(methodName, "IsAssignableFrom") == 0) + + case 'R': { - result = NI_System_Type_IsAssignableFrom; + if (strcmp(className, "ReadOnlySpan`1") == 0) + { + if (strcmp(methodName, "get_Item") == 0) + { + result = NI_System_ReadOnlySpan_get_Item; + } + else if (strcmp(methodName, "get_Length") == 0) + { + result = NI_System_ReadOnlySpan_get_Length; + } + } + else if (strcmp(className, "RuntimeType") == 0) + { + if (strcmp(methodName, "get_IsActualEnum") == 0) + { + result = NI_System_Type_get_IsEnum; + } + } + else if (strcmp(className, "RuntimeTypeHandle") == 0) + { + if (strcmp(methodName, "GetValueInternal") == 0) + { + result = NI_System_RuntimeTypeHandle_GetValueInternal; + } + } + break; } - else if (strcmp(methodName, "IsAssignableTo") == 0) + + case 'S': { - result = NI_System_Type_IsAssignableTo; + if (strcmp(className, "Span`1") == 0) + { + if (strcmp(methodName, "get_Item") == 0) + { + result = NI_System_Span_get_Item; + } + else if (strcmp(methodName, "get_Length") == 0) + { + result = NI_System_Span_get_Length; + } + } + else if (strcmp(className, "String") == 0) + { + if (strcmp(methodName, "Equals") == 0) + { + result = NI_System_String_Equals; + } + else if (strcmp(methodName, "get_Chars") == 0) + { + result = NI_System_String_get_Chars; + } + else if (strcmp(methodName, "get_Length") == 0) + { + result = NI_System_String_get_Length; + } + else if (strcmp(methodName, "op_Implicit") == 0) + { + result = NI_System_String_op_Implicit; + } + else if (strcmp(methodName, "StartsWith") == 0) + { + result = NI_System_String_StartsWith; + } + } + break; } - else if (strcmp(methodName, "op_Equality") == 0) + + case 'T': { - result = NI_System_Type_op_Equality; + if (strcmp(className, "Type") == 0) + { + if (strcmp(methodName, "get_IsEnum") == 0) + { + result = NI_System_Type_get_IsEnum; + } + else if (strcmp(methodName, "get_IsValueType") == 0) + { + result = NI_System_Type_get_IsValueType; + } + else if (strcmp(methodName, "get_IsByRefLike") == 0) + { + result = NI_System_Type_get_IsByRefLike; + } + else if (strcmp(methodName, "GetEnumUnderlyingType") == 0) + { + result = NI_System_Type_GetEnumUnderlyingType; + } + else if (strcmp(methodName, "GetTypeFromHandle") == 0) + { + result = NI_System_Type_GetTypeFromHandle; + } + else if (strcmp(methodName, "IsAssignableFrom") == 0) + { + result = NI_System_Type_IsAssignableFrom; + } + else if (strcmp(methodName, "IsAssignableTo") == 0) + { + result = NI_System_Type_IsAssignableTo; + } + else if (strcmp(methodName, "op_Equality") == 0) + { + result = NI_System_Type_op_Equality; + } + else if (strcmp(methodName, "op_Inequality") == 0) + { + result = NI_System_Type_op_Inequality; + } + } + break; } - else if (strcmp(methodName, "op_Inequality") == 0) + + case 'U': { - result = NI_System_Type_op_Inequality; + if ((strcmp(className, "UInt32") == 0) || (strcmp(className, "UInt64") == 0) || + (strcmp(className, "UIntPtr") == 0)) + { + result = lookupPrimitiveNamedIntrinsic(method, methodName); + } + break; } + + default: + break; } } else if (namespaceName[0] == '.') @@ -7397,10 +8055,7 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { if (strcmp(className, "BitOperations") == 0) { - if (strcmp(methodName, "PopCount") == 0) - { - result = NI_System_Numerics_BitOperations_PopCount; - } + result = lookupPrimitiveNamedIntrinsic(method, methodName); } else { @@ -7760,6 +8415,55 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) } //------------------------------------------------------------------------ +// lookupPrimitiveNamedIntrinsic: map method to jit named intrinsic value +// +// Arguments: +// method -- method handle for method +// +// Return Value: +// Id for the named intrinsic, or Illegal if none. +// +// Notes: +// method should have CORINFO_FLG_INTRINSIC set in its attributes, +// otherwise it is not a named jit intrinsic. +// +NamedIntrinsic Compiler::lookupPrimitiveNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName) +{ + NamedIntrinsic result = NI_Illegal; + + if (strcmp(methodName, "Crc32C") == 0) + { + result = NI_PRIMITIVE_Crc32C; + } + else if (strcmp(methodName, "LeadingZeroCount") == 0) + { + result = NI_PRIMITIVE_LeadingZeroCount; + } + else if (strcmp(methodName, "Log2") == 0) + { + result = NI_PRIMITIVE_Log2; + } + else if (strcmp(methodName, "PopCount") == 0) + { + result = NI_PRIMITIVE_PopCount; + } + else if (strcmp(methodName, "RotateLeft") == 0) + { + result = NI_PRIMITIVE_RotateLeft; + } + else if (strcmp(methodName, "RotateRight") == 0) + { + result = NI_PRIMITIVE_RotateRight; + } + else if (strcmp(methodName, "TrailingZeroCount") == 0) + { + result = NI_PRIMITIVE_TrailingZeroCount; + } + + return result; +} + +//------------------------------------------------------------------------ // impUnsupportedNamedIntrinsic: Throws an exception for an unsupported named intrinsic // // Arguments: diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index a58e0b7..4f922bc 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -56,7 +56,7 @@ enum NamedIntrinsic : unsigned short NI_System_Collections_Generic_Comparer_get_Default, NI_System_Collections_Generic_EqualityComparer_get_Default, NI_System_Buffers_Binary_BinaryPrimitives_ReverseEndianness, - NI_System_Numerics_BitOperations_PopCount, + NI_System_GC_KeepAlive, NI_System_Threading_Thread_get_CurrentThread, NI_System_Threading_Thread_get_ManagedThreadId, @@ -110,16 +110,6 @@ enum NamedIntrinsic : unsigned short NI_System_MemoryExtensions_SequenceEqual, NI_System_MemoryExtensions_StartsWith, - // These are used by HWIntrinsics but are defined more generally - // to allow dead code optimization and handle the recursion case - - NI_IsSupported_True, - NI_IsSupported_False, - NI_IsSupported_Dynamic, - NI_IsSupported_Type, - NI_Throw_PlatformNotSupportedException, - NI_Vector_GetCount, - NI_System_Threading_Interlocked_And, NI_System_Threading_Interlocked_Or, NI_System_Threading_Interlocked_CompareExchange, @@ -159,6 +149,28 @@ enum NamedIntrinsic : unsigned short NI_SIMD_UpperSave, #endif // FEATURE_SIMD + // + // Special Import Intrinsics + // + + NI_SPECIAL_IMPORT_START, + + // These are used by HWIntrinsics but are defined more generally + // to allow dead code optimization and handle the recursion case + + NI_IsSupported_True, + NI_IsSupported_False, + NI_IsSupported_Dynamic, + NI_IsSupported_Type, + NI_Throw_PlatformNotSupportedException, + NI_Vector_GetCount, + + NI_SPECIAL_IMPORT_END, + + // + // System.Runtime.CompilerServices.Unsafe Intrinsics + // + NI_SRCS_UNSAFE_START, NI_SRCS_UNSAFE_Add, @@ -188,6 +200,22 @@ enum NamedIntrinsic : unsigned short NI_SRCS_UNSAFE_WriteUnaligned, NI_SRCS_UNSAFE_END, + + // + // Primitive Intrinsics + // + + NI_PRIMITIVE_START, + + NI_PRIMITIVE_Crc32C, + NI_PRIMITIVE_LeadingZeroCount, + NI_PRIMITIVE_Log2, + NI_PRIMITIVE_PopCount, + NI_PRIMITIVE_RotateLeft, + NI_PRIMITIVE_RotateRight, + NI_PRIMITIVE_TrailingZeroCount, + + NI_PRIMITIVE_END, }; #endif // _NAMEDINTRINSICLIST_H_ diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 4a663f8..bf5181b 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -2517,6 +2517,469 @@ double FloatingPointUtils::normalize(double value) #endif } +//------------------------------------------------------------------------ +// BitOperations::BitScanForward: Search the mask data from least significant bit (LSB) to the most significant bit +// (MSB) for a set bit (1) +// +// Arguments: +// value - the value +// +// Return Value: +// 0 if the mask is zero; nonzero otherwise. +// +uint32_t BitOperations::BitScanForward(uint32_t value) +{ + assert(value != 0); + +#if defined(_MSC_VER) + unsigned long result; + ::_BitScanForward(&result, value); + return static_cast(result); +#else + int32_t result = __builtin_ctz(value); + return static_cast(result); +#endif +} + +//------------------------------------------------------------------------ +// BitOperations::BitScanForward: Search the mask data from least significant bit (LSB) to the most significant bit +// (MSB) for a set bit (1) +// +// Arguments: +// value - the value +// +// Return Value: +// 0 if the mask is zero; nonzero otherwise. +// +uint32_t BitOperations::BitScanForward(uint64_t value) +{ + assert(value != 0); + +#if defined(_MSC_VER) +#if defined(HOST_64BIT) + unsigned long result; + ::_BitScanForward64(&result, value); + return static_cast(result); +#else + uint32_t lower = static_cast(value); + + if (lower == 0) + { + uint32_t upper = static_cast(value >> 32); + return 32 + BitScanForward(upper); + } + + return BitScanForward(lower); +#endif // HOST_64BIT +#else + int32_t result = __builtin_ctzll(value); + return static_cast(result); +#endif +} + +//------------------------------------------------------------------------ +// BitOperations::BitScanReverse: Search the mask data from most significant bit (MSB) to least significant bit +// (LSB) for a set bit (1). +// +// Arguments: +// value - the value +// +// Return Value: +// 0 if the mask is zero; nonzero otherwise. +// +uint32_t BitOperations::BitScanReverse(uint32_t value) +{ + assert(value != 0); + +#if defined(_MSC_VER) + unsigned long result; + ::_BitScanReverse(&result, value); + return static_cast(result); +#else + // LZCNT returns index starting from MSB, whereas BSR gives the index from LSB. + // 31 ^ BSR here is equivalent to 31 - BSR since the BSR result is always between 0 and 31. + // This saves an instruction, as subtraction from constant requires either MOV/SUB or NEG/ADD. + + int32_t result = __builtin_clz(value); + return static_cast(31 ^ result); +#endif +} + +//------------------------------------------------------------------------ +// BitOperations::BitScanReverse: Search the mask data from most significant bit (MSB) to least significant bit +// (LSB) for a set bit (1). +// +// Arguments: +// value - the value +// +// Return Value: +// 0 if the mask is zero; nonzero otherwise. +// +uint32_t BitOperations::BitScanReverse(uint64_t value) +{ + assert(value != 0); + +#if defined(_MSC_VER) +#if defined(HOST_64BIT) + unsigned long result; + ::_BitScanReverse64(&result, value); + return static_cast(result); +#else + uint32_t upper = static_cast(value >> 32); + + if (upper == 0) + { + uint32_t lower = static_cast(value); + return BitScanReverse(lower); + } + + return 32 + BitScanReverse(upper); +#endif // HOST_64BIT +#else + // LZCNT returns index starting from MSB, whereas BSR gives the index from LSB. + // 63 ^ BSR here is equivalent to 63 - BSR since the BSR result is always between 0 and 63. + // This saves an instruction, as subtraction from constant requires either MOV/SUB or NEG/ADD. + + int32_t result = __builtin_clzll(value); + return static_cast(63 ^ result); +#endif +} + +//------------------------------------------------------------------------ +// BitOperations::LeadingZeroCount: Count the number of leading zero bits in a mask. +// +// Arguments: +// value - the value +// +// Return Value: +// The number of leading bits in value +// +uint32_t BitOperations::LeadingZeroCount(uint32_t value) +{ + if (value == 0) + { + return 32; + } + +#if defined(_MSC_VER) + // LZCNT returns index starting from MSB, whereas BSR gives the index from LSB. + // 31 ^ BSR here is equivalent to 31 - BSR since the BSR result is always between 0 and 31. + // This saves an instruction, as subtraction from constant requires either MOV/SUB or NEG/ADD. + + uint32_t result = BitOperations::BitScanReverse(value); + return 31 ^ result; +#else + int32_t result = __builtin_clz(value); + return static_cast(result); +#endif +} + +//------------------------------------------------------------------------ +// BitOperations::LeadingZeroCount: Count the number of leading zero bits in a mask. +// +// Arguments: +// value - the value +// +// Return Value: +// The number of leading bits in value +// +uint32_t BitOperations::LeadingZeroCount(uint64_t value) +{ + if (value == 0) + { + return 64; + } + +#if defined(_MSC_VER) + // LZCNT returns index starting from MSB, whereas BSR gives the index from LSB. + // 63 ^ BSR here is equivalent to 63 - BSR since the BSR result is always between 0 and 63. + // This saves an instruction, as subtraction from constant requires either MOV/SUB or NEG/ADD. + + uint32_t result = BitOperations::BitScanReverse(value); + return 63 ^ result; +#else + int32_t result = __builtin_clzll(value); + return static_cast(result); +#endif +} + +//------------------------------------------------------------------------ +// BitOperations::Log2: Returns the integer (floor) log of the specified value, base 2. +// +// Arguments: +// value - the value +// +// Return Value: +// The integer (floor) log of value, base 2 +// +uint32_t BitOperations::Log2(uint32_t value) +{ + // The 0->0 contract is fulfilled by setting the LSB to 1. + // Log(1) is 0, and setting the LSB for values > 1 does not change the log2 result. + return 31 ^ BitOperations::LeadingZeroCount(value | 1); +} + +//------------------------------------------------------------------------ +// BitOperations::Log2: Returns the integer (floor) log of the specified value, base 2. +// +// Arguments: +// value - the value +// +// Return Value: +// The integer (floor) log of value, base 2 +// +uint32_t BitOperations::Log2(uint64_t value) +{ + // The 0->0 contract is fulfilled by setting the LSB to 1. + // Log(1) is 0, and setting the LSB for values > 1 does not change the log2 result. + return 63 ^ BitOperations::LeadingZeroCount(value | 1); +} + +//------------------------------------------------------------------------ +// BitOperations::PopCount: Returns the population count (number of bits set) of a mask. +// +// Arguments: +// value - the value +// +// Return Value: +// The population count (number of bits set) of value +// +uint32_t BitOperations::PopCount(uint32_t value) +{ +#if defined(_MSC_VER) + // Inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson: + // http://graphics.stanford.edu/~seander/bithacks.html + + const uint32_t c1 = 0x55555555u; + const uint32_t c2 = 0x33333333u; + const uint32_t c3 = 0x0F0F0F0Fu; + const uint32_t c4 = 0x01010101u; + + value -= (value >> 1) & c1; + value = (value & c2) + ((value >> 2) & c2); + value = (((value + (value >> 4)) & c3) * c4) >> 24; + + return value; +#else + int32_t result = __builtin_popcount(value); + return static_cast(result); +#endif +} + +//------------------------------------------------------------------------ +// BitOperations::PopCount: Returns the population count (number of bits set) of a mask. +// +// Arguments: +// value - the value +// +// Return Value: +// The population count (number of bits set) of value +// +uint32_t BitOperations::PopCount(uint64_t value) +{ +#if defined(_MSC_VER) + // Inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson: + // http://graphics.stanford.edu/~seander/bithacks.html + + const uint64_t c1 = 0x5555555555555555ull; + const uint64_t c2 = 0x3333333333333333ull; + const uint64_t c3 = 0x0F0F0F0F0F0F0F0Full; + const uint64_t c4 = 0x0101010101010101ull; + + value -= (value >> 1) & c1; + value = (value & c2) + ((value >> 2) & c2); + value = (((value + (value >> 4)) & c3) * c4) >> 56; + + return static_cast(value); +#else + int32_t result = __builtin_popcountll(value); + return static_cast(result); +#endif +} + +//------------------------------------------------------------------------ +// BitOperations::ReverseBits: Reverses the bits in an integer value +// +// Arguments: +// value - the value +// +// Return Value: +// The reversed bits of value +// +uint32_t BitOperations::ReverseBits(uint32_t value) +{ + // Inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson: + // http://graphics.stanford.edu/~seander/bithacks.html + + uint32_t result = value; + + // swap odd and even bits + result = ((result >> 1) & 0x55555555) | ((result & 0x55555555) << 1); + + // swap consecutive pairs + result = ((result >> 2) & 0x33333333) | ((result & 0x33333333) << 2); + + // swap nibbles ... + result = ((result >> 4) & 0x0F0F0F0F) | ((result & 0x0F0F0F0F) << 4); + + // swap bytes + result = ((result >> 8) & 0x00FF00FF) | ((result & 0x00FF00FF) << 8); + + // swap 2-byte pairs + result = (result >> 16) | (result << 16); + + return result; +} + +//------------------------------------------------------------------------ +// BitOperations::ReverseBits: Reverses the bits in an integer value +// +// Arguments: +// value - the value +// +// Return Value: +// The reversed bits of value +// +uint64_t BitOperations::ReverseBits(uint64_t value) +{ + // Inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson: + // http://graphics.stanford.edu/~seander/bithacks.html + + uint64_t result = value; + + // swap odd and even bits + result = ((result >> 1) & 0x5555555555555555ull) | ((result & 0x5555555555555555ull) << 1); + + // swap consecutive pairs + result = ((result >> 2) & 0x3333333333333333ull) | ((result & 0x3333333333333333ull) << 2); + + // swap nibbles ... + result = ((result >> 4) & 0x0F0F0F0F0F0F0F0Full) | ((result & 0x0F0F0F0F0F0F0F0Full) << 4); + + // swap bytes + result = ((result >> 8) & 0x00FF00FF00FF00FFull) | ((result & 0x00FF00FF00FF00FFull) << 8); + + // swap 2-byte pairs + result = ((result >> 16) & 0x0000FFFF0000FFFFull) | ((result & 0x0000FFFF0000FFFFull) << 16); + + // swap 4-byte pairs + result = (result >> 32) | (result << 32); + + return result; +} + +//------------------------------------------------------------------------ +// BitOperations::RotateLeft: Rotates the specified value left by the specified number of bits. +// +// Arguments: +// value - the value to rotate +// offset - the number of bits to rotate by +// +// Return Value: +// The rotated value +// +uint32_t BitOperations::RotateLeft(uint32_t value, uint32_t offset) +{ + // Mask the offset to ensure deterministic xplat behavior for overshifting + return (value << (offset & 0x1F)) | (value >> ((32 - offset) & 0x1F)); +} + +//------------------------------------------------------------------------ +// BitOperations::RotateLeft: Rotates the specified value left by the specified number of bits. +// +// Arguments: +// value - the value to rotate +// offset - the number of bits to rotate by +// +// Return Value: +// The rotated value +// +uint64_t BitOperations::RotateLeft(uint64_t value, uint32_t offset) +{ + // Mask the offset to ensure deterministic xplat behavior for overshifting + return (value << (offset & 0x3F)) | (value >> ((64 - offset) & 0x3F)); +} + +//------------------------------------------------------------------------ +// BitOperations::RotateRight: Rotates the specified value right by the specified number of bits. +// +// Arguments: +// value - the value to rotate +// offset - the number of bits to rotate by +// +// Return Value: +// The rotated value +// +uint32_t BitOperations::RotateRight(uint32_t value, uint32_t offset) +{ + // Mask the offset to ensure deterministic xplat behavior for overshifting + return (value >> (offset & 0x1F)) | (value << ((32 - offset) & 0x1F)); +} + +//------------------------------------------------------------------------ +// BitOperations::RotateRight: Rotates the specified value right by the specified number of bits. +// +// Arguments: +// value - the value to rotate +// offset - the number of bits to rotate by +// +// Return Value: +// The rotated value +// +uint64_t BitOperations::RotateRight(uint64_t value, uint32_t offset) +{ + // Mask the offset to ensure deterministic xplat behavior for overshifting + return (value >> (offset & 0x3F)) | (value << ((64 - offset) & 0x3F)); +} + +//------------------------------------------------------------------------ +// BitOperations::TrailingZeroCount: Count the number of trailing zero bits in an integer value. +// +// Arguments: +// value - the value +// +// Return Value: +// The number of trailing zero bits in value +// +uint32_t BitOperations::TrailingZeroCount(uint32_t value) +{ + if (value == 0) + { + return 32; + } + +#if defined(_MSC_VER) + return BitOperations::BitScanForward(value); +#else + int32_t result = __builtin_ctz(value); + return static_cast(result); +#endif +} + +//------------------------------------------------------------------------ +// BitOperations::TrailingZeroCount: Count the number of trailing zero bits in an integer value. +// +// Arguments: +// value - the value +// +// Return Value: +// The number of trailing zero bits in value +// +uint32_t BitOperations::TrailingZeroCount(uint64_t value) +{ + if (value == 0) + { + return 64; + } + +#if defined(_MSC_VER) + return BitOperations::BitScanForward(value); +#else + int32_t result = __builtin_ctzll(value); + return static_cast(result); +#endif +} + namespace MagicDivide { template diff --git a/src/coreclr/jit/utils.h b/src/coreclr/jit/utils.h index 0e129f1..19083f3 100644 --- a/src/coreclr/jit/utils.h +++ b/src/coreclr/jit/utils.h @@ -730,6 +730,46 @@ public: static double normalize(double x); }; +class BitOperations +{ +public: + static uint32_t BitScanForward(uint32_t value); + + static uint32_t BitScanForward(uint64_t value); + + static uint32_t BitScanReverse(uint32_t value); + + static uint32_t BitScanReverse(uint64_t value); + + static uint32_t LeadingZeroCount(uint32_t value); + + static uint32_t LeadingZeroCount(uint64_t value); + + static uint32_t Log2(uint32_t value); + + static uint32_t Log2(uint64_t value); + + static uint32_t PopCount(uint32_t value); + + static uint32_t PopCount(uint64_t value); + + static uint32_t ReverseBits(uint32_t value); + + static uint64_t ReverseBits(uint64_t value); + + static uint32_t RotateLeft(uint32_t value, uint32_t offset); + + static uint64_t RotateLeft(uint64_t value, uint32_t offset); + + static uint32_t RotateRight(uint32_t value, uint32_t offset); + + static uint64_t RotateRight(uint64_t value, uint32_t offset); + + static uint32_t TrailingZeroCount(uint32_t value); + + static uint32_t TrailingZeroCount(uint64_t value); +}; + // The CLR requires that critical section locks be initialized via its ClrCreateCriticalSection API...but // that can't be called until the CLR is initialized. If we have static data that we'd like to protect by a // lock, and we have a statically allocated lock to protect that data, there's an issue in how to initialize diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index aa6d86b..8995eaa 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -6025,32 +6025,140 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunUnary( case NI_LZCNT_LeadingZeroCount: #endif { - UINT32 cns = (UINT32)GetConstantInt32(arg0VN); - int lzc = 0; - while (cns != 0) - { - cns = cns >> 1; - lzc++; - } - return VNForIntCon(32 - lzc); + assert(!varTypeIsSmall(type) && !varTypeIsLong(type)); + + int32_t value = GetConstantInt32(arg0VN); + uint32_t result = BitOperations::LeadingZeroCount(static_cast(value)); + + return VNForIntCon(static_cast(result)); } #ifdef TARGET_ARM64 case NI_ArmBase_Arm64_LeadingZeroCount: + { + assert(varTypeIsInt(type)); + + int64_t value = GetConstantInt64(arg0VN); + uint32_t result = BitOperations::LeadingZeroCount(static_cast(value)); + + return VNForIntCon(static_cast(result)); + } #else case NI_LZCNT_X64_LeadingZeroCount: + { + assert(varTypeIsLong(type)); + + int64_t value = GetConstantInt64(arg0VN); + uint32_t result = BitOperations::LeadingZeroCount(static_cast(value)); + + return VNForLongCon(static_cast(result)); + } #endif + +#if defined(TARGET_ARM64) + case NI_ArmBase_ReverseElementBits: { - UINT64 cns = (UINT64)GetConstantInt64(arg0VN); - int lzc = 0; - while (cns != 0) - { - cns = cns >> 1; - lzc++; - } - return VNForIntCon(64 - lzc); + assert(!varTypeIsSmall(type) && !varTypeIsLong(type)); + + int32_t value = GetConstantInt32(arg0VN); + uint32_t result = BitOperations::ReverseBits(static_cast(value)); + + return VNForIntCon(static_cast(result)); + } + + case NI_ArmBase_Arm64_ReverseElementBits: + { + assert(varTypeIsLong(type)); + + int64_t value = GetConstantInt64(arg0VN); + uint64_t result = BitOperations::ReverseBits(static_cast(value)); + + return VNForLongCon(static_cast(result)); + } +#endif // TARGET_ARM64 + +#if defined(TARGET_XARCH) + case NI_BMI1_TrailingZeroCount: + { + assert(!varTypeIsSmall(type) && !varTypeIsLong(type)); + + int32_t value = GetConstantInt32(arg0VN); + uint32_t result = BitOperations::TrailingZeroCount(static_cast(value)); + + return VNForIntCon(static_cast(result)); + } + + case NI_BMI1_X64_TrailingZeroCount: + { + assert(varTypeIsLong(type)); + + int64_t value = GetConstantInt64(arg0VN); + uint32_t result = BitOperations::TrailingZeroCount(static_cast(value)); + + return VNForLongCon(static_cast(result)); + } + + case NI_POPCNT_PopCount: + { + assert(!varTypeIsSmall(type) && !varTypeIsLong(type)); + + int32_t value = GetConstantInt32(arg0VN); + uint32_t result = BitOperations::PopCount(static_cast(value)); + + return VNForIntCon(static_cast(result)); + } + + case NI_POPCNT_X64_PopCount: + { + assert(varTypeIsLong(type)); + + int64_t value = GetConstantInt64(arg0VN); + uint32_t result = BitOperations::PopCount(static_cast(value)); + + return VNForLongCon(static_cast(result)); + } + + case NI_X86Base_BitScanForward: + { + assert(!varTypeIsSmall(type) && !varTypeIsLong(type)); + + int32_t value = GetConstantInt32(arg0VN); + uint32_t result = BitOperations::BitScanForward(static_cast(value)); + + return VNForIntCon(static_cast(result)); } + case NI_X86Base_X64_BitScanForward: + { + assert(varTypeIsLong(type)); + + int64_t value = GetConstantInt64(arg0VN); + uint32_t result = BitOperations::BitScanForward(static_cast(value)); + + return VNForLongCon(static_cast(result)); + } + + case NI_X86Base_BitScanReverse: + { + assert(!varTypeIsSmall(type) && !varTypeIsLong(type)); + + int32_t value = GetConstantInt32(arg0VN); + uint32_t result = BitOperations::BitScanReverse(static_cast(value)); + + return VNForIntCon(static_cast(result)); + } + + case NI_X86Base_X64_BitScanReverse: + { + assert(varTypeIsLong(type)); + + int64_t value = GetConstantInt64(arg0VN); + uint32_t result = BitOperations::BitScanReverse(static_cast(value)); + + return VNForLongCon(static_cast(result)); + } +#endif // TARGET_XARCH + default: break; } diff --git a/src/libraries/System.Private.CoreLib/src/System/Int32.cs b/src/libraries/System.Private.CoreLib/src/System/Int32.cs index 07c07bf..56dc9a7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Int32.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Int32.cs @@ -318,18 +318,23 @@ namespace System public static (int Quotient, int Remainder) DivRem(int left, int right) => Math.DivRem(left, right); /// + [Intrinsic] public static int LeadingZeroCount(int value) => BitOperations.LeadingZeroCount((uint)value); /// + [Intrinsic] public static int PopCount(int value) => BitOperations.PopCount((uint)value); /// + [Intrinsic] public static int RotateLeft(int value, int rotateAmount) => (int)BitOperations.RotateLeft((uint)value, rotateAmount); /// + [Intrinsic] public static int RotateRight(int value, int rotateAmount) => (int)BitOperations.RotateRight((uint)value, rotateAmount); /// + [Intrinsic] public static int TrailingZeroCount(int value) => BitOperations.TrailingZeroCount(value); /// @@ -564,6 +569,8 @@ namespace System public static bool IsPow2(int value) => BitOperations.IsPow2(value); /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Log2(int value) { if (value < 0) diff --git a/src/libraries/System.Private.CoreLib/src/System/Int64.cs b/src/libraries/System.Private.CoreLib/src/System/Int64.cs index 464ebb4..de49923 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Int64.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Int64.cs @@ -305,18 +305,23 @@ namespace System public static (long Quotient, long Remainder) DivRem(long left, long right) => Math.DivRem(left, right); /// + [Intrinsic] public static long LeadingZeroCount(long value) => BitOperations.LeadingZeroCount((ulong)value); /// + [Intrinsic] public static long PopCount(long value) => BitOperations.PopCount((ulong)value); /// + [Intrinsic] public static long RotateLeft(long value, int rotateAmount) => (long)BitOperations.RotateLeft((ulong)value, rotateAmount); /// + [Intrinsic] public static long RotateRight(long value, int rotateAmount) => (long)BitOperations.RotateRight((ulong)value, rotateAmount); /// + [Intrinsic] public static long TrailingZeroCount(long value) => BitOperations.TrailingZeroCount(value); /// @@ -551,6 +556,8 @@ namespace System public static bool IsPow2(long value) => BitOperations.IsPow2(value); /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long Log2(long value) { if (value < 0) diff --git a/src/libraries/System.Private.CoreLib/src/System/IntPtr.cs b/src/libraries/System.Private.CoreLib/src/System/IntPtr.cs index 27f1289..cac1718 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IntPtr.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IntPtr.cs @@ -284,18 +284,23 @@ namespace System public static (nint Quotient, nint Remainder) DivRem(nint left, nint right) => Math.DivRem(left, right); /// + [Intrinsic] public static nint LeadingZeroCount(nint value) => BitOperations.LeadingZeroCount((nuint)value); /// + [Intrinsic] public static nint PopCount(nint value) => BitOperations.PopCount((nuint)value); /// + [Intrinsic] public static nint RotateLeft(nint value, int rotateAmount) => (nint)BitOperations.RotateLeft((nuint)value, rotateAmount); /// + [Intrinsic] public static nint RotateRight(nint value, int rotateAmount) => (nint)BitOperations.RotateRight((nuint)value, rotateAmount); /// + [Intrinsic] public static nint TrailingZeroCount(nint value) => BitOperations.TrailingZeroCount(value); /// @@ -540,6 +545,8 @@ namespace System public static bool IsPow2(nint value) => BitOperations.IsPow2(value); /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static nint Log2(nint value) { if (value < 0) diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs index 0ecc5dd..e2fd653 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs @@ -96,7 +96,7 @@ namespace System.Numerics [CLSCompliant(false)] public static uint RoundUpToPowerOf2(uint value) { - if (Lzcnt.IsSupported || ArmBase.IsSupported || X86Base.IsSupported || WasmBase.IsSupported) + if (X86Base.IsSupported || ArmBase.IsSupported || WasmBase.IsSupported) { #if TARGET_64BIT return (uint)(0x1_0000_0000ul >> LeadingZeroCount(value - 1)); @@ -128,7 +128,7 @@ namespace System.Numerics [CLSCompliant(false)] public static ulong RoundUpToPowerOf2(ulong value) { - if (Lzcnt.X64.IsSupported || ArmBase.Arm64.IsSupported || WasmBase.IsSupported) + if (X86Base.X64.IsSupported || ArmBase.Arm64.IsSupported || WasmBase.IsSupported) { int shift = 64 - LeadingZeroCount(value - 1); return (1ul ^ (ulong)(shift >> 6)) << shift; @@ -278,6 +278,7 @@ namespace System.Numerics /// Similar in behavior to the x86 instruction LZCNT. /// /// The value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static int LeadingZeroCount(nuint value) @@ -294,6 +295,7 @@ namespace System.Numerics /// Note that by convention, input value 0 returns 0 since log(0) is undefined. /// /// The value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static int Log2(uint value) @@ -339,6 +341,7 @@ namespace System.Numerics /// Note that by convention, input value 0 returns 0 since log(0) is undefined. /// /// The value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static int Log2(ulong value) @@ -380,6 +383,7 @@ namespace System.Numerics /// Note that by convention, input value 0 returns 0 since log(0) is undefined. /// /// The value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static int Log2(nuint value) @@ -534,6 +538,7 @@ namespace System.Numerics /// Similar in behavior to the x86 instruction POPCNT. /// /// The value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static int PopCount(nuint value) @@ -604,6 +609,7 @@ namespace System.Numerics /// Similar in behavior to the x86 instruction TZCNT. /// /// The value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int TrailingZeroCount(long value) => TrailingZeroCount((ulong)value); @@ -655,15 +661,23 @@ namespace System.Numerics /// Similar in behavior to the x86 instruction TZCNT. /// /// The value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int TrailingZeroCount(nint value) - => TrailingZeroCount((nuint)value); + { +#if TARGET_64BIT + return TrailingZeroCount((ulong)(nuint)value); +#else + return TrailingZeroCount((uint)(nuint)value); +#endif + } /// /// Count the number of trailing zero bits in a mask. /// Similar in behavior to the x86 instruction TZCNT. /// /// The value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static int TrailingZeroCount(nuint value) @@ -683,6 +697,7 @@ namespace System.Numerics /// The number of bits to rotate by. /// Any value outside the range [0..31] is treated as congruent mod 32. /// The rotated value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static uint RotateLeft(uint value, int offset) @@ -696,6 +711,7 @@ namespace System.Numerics /// The number of bits to rotate by. /// Any value outside the range [0..63] is treated as congruent mod 64. /// The rotated value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static ulong RotateLeft(ulong value, int offset) @@ -710,6 +726,7 @@ namespace System.Numerics /// Any value outside the range [0..31] is treated as congruent mod 32 on a 32-bit process, /// and any value outside the range [0..63] is treated as congruent mod 64 on a 64-bit process. /// The rotated value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static nuint RotateLeft(nuint value, int offset) @@ -729,6 +746,7 @@ namespace System.Numerics /// The number of bits to rotate by. /// Any value outside the range [0..31] is treated as congruent mod 32. /// The rotated value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static uint RotateRight(uint value, int offset) @@ -742,6 +760,7 @@ namespace System.Numerics /// The number of bits to rotate by. /// Any value outside the range [0..63] is treated as congruent mod 64. /// The rotated value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static ulong RotateRight(ulong value, int offset) @@ -756,6 +775,7 @@ namespace System.Numerics /// Any value outside the range [0..31] is treated as congruent mod 32 on a 32-bit process, /// and any value outside the range [0..63] is treated as congruent mod 64 on a 64-bit process. /// The rotated value. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] [CLSCompliant(false)] public static nuint RotateRight(nuint value, int offset) @@ -773,6 +793,7 @@ namespace System.Numerics /// The base value to calculate checksum on /// The data for which to compute the checksum /// The CRC-checksum + [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static uint Crc32C(uint crc, byte data) @@ -796,6 +817,7 @@ namespace System.Numerics /// The base value to calculate checksum on /// The data for which to compute the checksum /// The CRC-checksum + [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static uint Crc32C(uint crc, ushort data) @@ -819,6 +841,7 @@ namespace System.Numerics /// The base value to calculate checksum on /// The data for which to compute the checksum /// The CRC-checksum + [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static uint Crc32C(uint crc, uint data) @@ -842,6 +865,7 @@ namespace System.Numerics /// The base value to calculate checksum on /// The data for which to compute the checksum /// The CRC-checksum + [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static uint Crc32C(uint crc, ulong data) @@ -852,12 +876,6 @@ namespace System.Numerics return (uint)Sse42.X64.Crc32(crc, data); } - if (Sse42.IsSupported) - { - uint result = Sse42.Crc32(crc, (uint)(data)); - return Sse42.Crc32(result, (uint)(data >> 32)); - } - if (Crc32.Arm64.IsSupported) { return Crc32.Arm64.ComputeCrc32C(crc, data); @@ -901,13 +919,8 @@ namespace System.Numerics internal static uint Crc32C(uint crc, ulong data) { - ref uint lookupTable = ref MemoryMarshal.GetArrayDataReference(s_crcTable); - - crc = Crc32CCore(ref lookupTable, crc, (uint)data); - data >>= 32; - crc = Crc32CCore(ref lookupTable, crc, (uint)data); - - return crc; + uint result = BitOperations.Crc32C(crc, (uint)(data)); + return BitOperations.Crc32C(result, (uint)(data >> 32)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/UInt32.cs b/src/libraries/System.Private.CoreLib/src/System/UInt32.cs index a07509d..c84612f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/UInt32.cs +++ b/src/libraries/System.Private.CoreLib/src/System/UInt32.cs @@ -299,18 +299,23 @@ namespace System public static (uint Quotient, uint Remainder) DivRem(uint left, uint right) => Math.DivRem(left, right); /// + [Intrinsic] public static uint LeadingZeroCount(uint value) => (uint)BitOperations.LeadingZeroCount(value); /// + [Intrinsic] public static uint PopCount(uint value) => (uint)BitOperations.PopCount(value); /// + [Intrinsic] public static uint RotateLeft(uint value, int rotateAmount) => BitOperations.RotateLeft(value, rotateAmount); /// + [Intrinsic] public static uint RotateRight(uint value, int rotateAmount) => BitOperations.RotateRight(value, rotateAmount); /// + [Intrinsic] public static uint TrailingZeroCount(uint value) => (uint)BitOperations.TrailingZeroCount(value); /// @@ -481,6 +486,7 @@ namespace System public static bool IsPow2(uint value) => BitOperations.IsPow2(value); /// + [Intrinsic] public static uint Log2(uint value) => (uint)BitOperations.Log2(value); // diff --git a/src/libraries/System.Private.CoreLib/src/System/UInt64.cs b/src/libraries/System.Private.CoreLib/src/System/UInt64.cs index 9c7467f..91b569a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/UInt64.cs +++ b/src/libraries/System.Private.CoreLib/src/System/UInt64.cs @@ -298,18 +298,23 @@ namespace System public static (ulong Quotient, ulong Remainder) DivRem(ulong left, ulong right) => Math.DivRem(left, right); /// + [Intrinsic] public static ulong LeadingZeroCount(ulong value) => (ulong)BitOperations.LeadingZeroCount(value); /// + [Intrinsic] public static ulong PopCount(ulong value) => (ulong)BitOperations.PopCount(value); /// + [Intrinsic] public static ulong RotateLeft(ulong value, int rotateAmount) => BitOperations.RotateLeft(value, rotateAmount); /// + [Intrinsic] public static ulong RotateRight(ulong value, int rotateAmount) => BitOperations.RotateRight(value, rotateAmount); /// + [Intrinsic] public static ulong TrailingZeroCount(ulong value) => (ulong)BitOperations.TrailingZeroCount(value); /// @@ -480,6 +485,7 @@ namespace System public static bool IsPow2(ulong value) => BitOperations.IsPow2(value); /// + [Intrinsic] public static ulong Log2(ulong value) => (ulong)BitOperations.Log2(value); // diff --git a/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs b/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs index 9cce3df..2c24851 100644 --- a/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs +++ b/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs @@ -280,18 +280,23 @@ namespace System public static (nuint Quotient, nuint Remainder) DivRem(nuint left, nuint right) => Math.DivRem(left, right); /// + [Intrinsic] public static nuint LeadingZeroCount(nuint value) => (nuint)BitOperations.LeadingZeroCount(value); /// + [Intrinsic] public static nuint PopCount(nuint value) => (nuint)BitOperations.PopCount(value); /// + [Intrinsic] public static nuint RotateLeft(nuint value, int rotateAmount) => BitOperations.RotateLeft(value, rotateAmount); /// + [Intrinsic] public static nuint RotateRight(nuint value, int rotateAmount) => BitOperations.RotateRight(value, rotateAmount); /// + [Intrinsic] public static nuint TrailingZeroCount(nuint value) => (nuint)BitOperations.TrailingZeroCount(value); /// @@ -481,6 +486,7 @@ namespace System public static bool IsPow2(nuint value) => BitOperations.IsPow2(value); /// + [Intrinsic] public static nuint Log2(nuint value) => (nuint)BitOperations.Log2(value); //