From 6854d3702cb0f645dbb72f1d26dbe0b2be75f326 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Mar 2023 07:06:49 -0800 Subject: [PATCH] Massively simplify getBaseJitTypeAndSizeOfSIMDType and the general SIMD handle caching/lookup (#83228) --- src/coreclr/jit/compiler.h | 250 ++++---- src/coreclr/jit/hwintrinsic.cpp | 144 ----- src/coreclr/jit/hwintrinsicarm64.cpp | 4 +- src/coreclr/jit/simd.cpp | 1026 +++++++-------------------------- src/coreclr/jit/simdashwintrinsic.cpp | 4 +- 5 files changed, 336 insertions(+), 1092 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 8f3e498..b772d40 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -2797,7 +2797,6 @@ public: NamedIntrinsic hwIntrinsicID); GenTreeHWIntrinsic* gtNewScalarHWIntrinsicNode( var_types type, GenTree* op1, GenTree* op2, GenTree* op3, NamedIntrinsic hwIntrinsicID); - CORINFO_CLASS_HANDLE gtGetStructHandleForHWSIMD(var_types simdType, CorInfoType simdBaseJitType); CorInfoType getBaseJitTypeFromArgIfNeeded(NamedIntrinsic intrinsic, CORINFO_CLASS_HANDLE clsHnd, CORINFO_SIG_INFO* sig, @@ -8408,80 +8407,28 @@ private: struct SIMDHandlesCache { + // BYTE, UBYTE, SHORT, USHORT, INT, UINT, LONG, ULONG + // NATIVEINT, NATIVEUINT, FLOAT, and DOUBLE + static const uint32_t SupportedTypeCount = 12; + // SIMD Types - CORINFO_CLASS_HANDLE SIMDFloatHandle; - CORINFO_CLASS_HANDLE SIMDDoubleHandle; - CORINFO_CLASS_HANDLE SIMDIntHandle; - CORINFO_CLASS_HANDLE SIMDUShortHandle; - CORINFO_CLASS_HANDLE SIMDUByteHandle; - CORINFO_CLASS_HANDLE SIMDShortHandle; - CORINFO_CLASS_HANDLE SIMDByteHandle; - CORINFO_CLASS_HANDLE SIMDLongHandle; - CORINFO_CLASS_HANDLE SIMDUIntHandle; - CORINFO_CLASS_HANDLE SIMDULongHandle; - CORINFO_CLASS_HANDLE SIMDNIntHandle; - CORINFO_CLASS_HANDLE SIMDNUIntHandle; - - CORINFO_CLASS_HANDLE SIMDPlaneHandle; - CORINFO_CLASS_HANDLE SIMDQuaternionHandle; - CORINFO_CLASS_HANDLE SIMDVector2Handle; - CORINFO_CLASS_HANDLE SIMDVector3Handle; - CORINFO_CLASS_HANDLE SIMDVector4Handle; - CORINFO_CLASS_HANDLE SIMDVectorHandle; + CORINFO_CLASS_HANDLE VectorTHandles[SupportedTypeCount]; + + CORINFO_CLASS_HANDLE PlaneHandle; + CORINFO_CLASS_HANDLE QuaternionHandle; + CORINFO_CLASS_HANDLE Vector2Handle; + CORINFO_CLASS_HANDLE Vector3Handle; + CORINFO_CLASS_HANDLE Vector4Handle; + CORINFO_CLASS_HANDLE VectorHandle; #ifdef FEATURE_HW_INTRINSICS #if defined(TARGET_ARM64) - CORINFO_CLASS_HANDLE Vector64FloatHandle; - CORINFO_CLASS_HANDLE Vector64DoubleHandle; - CORINFO_CLASS_HANDLE Vector64IntHandle; - CORINFO_CLASS_HANDLE Vector64UShortHandle; - CORINFO_CLASS_HANDLE Vector64UByteHandle; - CORINFO_CLASS_HANDLE Vector64ShortHandle; - CORINFO_CLASS_HANDLE Vector64ByteHandle; - CORINFO_CLASS_HANDLE Vector64LongHandle; - CORINFO_CLASS_HANDLE Vector64UIntHandle; - CORINFO_CLASS_HANDLE Vector64ULongHandle; - CORINFO_CLASS_HANDLE Vector64NIntHandle; - CORINFO_CLASS_HANDLE Vector64NUIntHandle; + CORINFO_CLASS_HANDLE Vector64THandles[SupportedTypeCount]; #endif // defined(TARGET_ARM64) - CORINFO_CLASS_HANDLE Vector128FloatHandle; - CORINFO_CLASS_HANDLE Vector128DoubleHandle; - CORINFO_CLASS_HANDLE Vector128IntHandle; - CORINFO_CLASS_HANDLE Vector128UShortHandle; - CORINFO_CLASS_HANDLE Vector128UByteHandle; - CORINFO_CLASS_HANDLE Vector128ShortHandle; - CORINFO_CLASS_HANDLE Vector128ByteHandle; - CORINFO_CLASS_HANDLE Vector128LongHandle; - CORINFO_CLASS_HANDLE Vector128UIntHandle; - CORINFO_CLASS_HANDLE Vector128ULongHandle; - CORINFO_CLASS_HANDLE Vector128NIntHandle; - CORINFO_CLASS_HANDLE Vector128NUIntHandle; + CORINFO_CLASS_HANDLE Vector128THandles[SupportedTypeCount]; #if defined(TARGET_XARCH) - CORINFO_CLASS_HANDLE Vector256FloatHandle; - CORINFO_CLASS_HANDLE Vector256DoubleHandle; - CORINFO_CLASS_HANDLE Vector256IntHandle; - CORINFO_CLASS_HANDLE Vector256UShortHandle; - CORINFO_CLASS_HANDLE Vector256UByteHandle; - CORINFO_CLASS_HANDLE Vector256ShortHandle; - CORINFO_CLASS_HANDLE Vector256ByteHandle; - CORINFO_CLASS_HANDLE Vector256LongHandle; - CORINFO_CLASS_HANDLE Vector256UIntHandle; - CORINFO_CLASS_HANDLE Vector256ULongHandle; - CORINFO_CLASS_HANDLE Vector256NIntHandle; - CORINFO_CLASS_HANDLE Vector256NUIntHandle; - - CORINFO_CLASS_HANDLE Vector512FloatHandle; - CORINFO_CLASS_HANDLE Vector512DoubleHandle; - CORINFO_CLASS_HANDLE Vector512IntHandle; - CORINFO_CLASS_HANDLE Vector512UShortHandle; - CORINFO_CLASS_HANDLE Vector512UByteHandle; - CORINFO_CLASS_HANDLE Vector512ShortHandle; - CORINFO_CLASS_HANDLE Vector512ByteHandle; - CORINFO_CLASS_HANDLE Vector512LongHandle; - CORINFO_CLASS_HANDLE Vector512UIntHandle; - CORINFO_CLASS_HANDLE Vector512ULongHandle; - CORINFO_CLASS_HANDLE Vector512NIntHandle; - CORINFO_CLASS_HANDLE Vector512NUIntHandle; + CORINFO_CLASS_HANDLE Vector256THandles[SupportedTypeCount]; + CORINFO_CLASS_HANDLE Vector512THandles[SupportedTypeCount]; #endif // defined(TARGET_XARCH) #endif // FEATURE_HW_INTRINSICS @@ -8492,23 +8439,21 @@ private: SIMDHandlesCache() { + assert(SupportedTypeCount == static_cast(CORINFO_TYPE_DOUBLE - CORINFO_TYPE_BYTE + 1)); memset(this, 0, sizeof(*this)); } }; SIMDHandlesCache* m_simdHandleCache; - // Get the handle for a SIMD type. +#if defined(FEATURE_HW_INTRINSICS) CORINFO_CLASS_HANDLE gtGetStructHandleForSIMD(var_types simdType, CorInfoType simdBaseJitType) { assert(varTypeIsSIMD(simdType)); + assert((simdBaseJitType >= CORINFO_TYPE_BYTE) && (simdBaseJitType <= CORINFO_TYPE_DOUBLE)); - if (m_simdHandleCache == nullptr) - { - // This may happen if the JIT generates SIMD node on its own, without importing them. - // Otherwise getBaseJitTypeAndSizeOfSIMDType should have created the cache. - return NO_CLASS_HANDLE; - } + // We should only be called from gtGetStructHandleForSimdOrHW and this should've been checked already + assert(m_simdHandleCache != nullptr); if (simdBaseJitType == CORINFO_TYPE_FLOAT) { @@ -8516,95 +8461,133 @@ private: { case TYP_SIMD8: { - return m_simdHandleCache->SIMDVector2Handle; + return m_simdHandleCache->Vector2Handle; } case TYP_SIMD12: { - return m_simdHandleCache->SIMDVector3Handle; + return m_simdHandleCache->Vector3Handle; } case TYP_SIMD16: { // We order the checks roughly by expected hit count so early exits are possible - if (simdBaseJitType != CORINFO_TYPE_FLOAT) + if (m_simdHandleCache->Vector4Handle != NO_CLASS_HANDLE) { - // We could be Vector, so handle below - assert(getSIMDVectorType() == TYP_SIMD16); - break; + return m_simdHandleCache->Vector4Handle; } - if (m_simdHandleCache->SIMDVector4Handle != NO_CLASS_HANDLE) + if (m_simdHandleCache->QuaternionHandle != NO_CLASS_HANDLE) { - return m_simdHandleCache->SIMDVector4Handle; + return m_simdHandleCache->QuaternionHandle; } - if (m_simdHandleCache->SIMDQuaternionHandle != NO_CLASS_HANDLE) + if (m_simdHandleCache->PlaneHandle != NO_CLASS_HANDLE) { - return m_simdHandleCache->SIMDQuaternionHandle; + return m_simdHandleCache->PlaneHandle; } - if (m_simdHandleCache->SIMDPlaneHandle != NO_CLASS_HANDLE) - { - return m_simdHandleCache->SIMDPlaneHandle; - } - - return NO_CLASS_HANDLE; + break; } #if defined(TARGET_XARCH) case TYP_SIMD32: case TYP_SIMD64: + { + // This should be handled by the Vector path below break; + } #endif // TARGET_XARCH default: + { unreached(); + } } } - assert(emitTypeSize(simdType) <= largestEnregisterableStructSize()); - - switch (simdBaseJitType) - { - case CORINFO_TYPE_FLOAT: - return m_simdHandleCache->SIMDFloatHandle; - case CORINFO_TYPE_DOUBLE: - return m_simdHandleCache->SIMDDoubleHandle; - case CORINFO_TYPE_INT: - return m_simdHandleCache->SIMDIntHandle; - case CORINFO_TYPE_USHORT: - return m_simdHandleCache->SIMDUShortHandle; - case CORINFO_TYPE_UBYTE: - return m_simdHandleCache->SIMDUByteHandle; - case CORINFO_TYPE_SHORT: - return m_simdHandleCache->SIMDShortHandle; - case CORINFO_TYPE_BYTE: - return m_simdHandleCache->SIMDByteHandle; - case CORINFO_TYPE_LONG: - return m_simdHandleCache->SIMDLongHandle; - case CORINFO_TYPE_UINT: - return m_simdHandleCache->SIMDUIntHandle; - case CORINFO_TYPE_ULONG: - return m_simdHandleCache->SIMDULongHandle; - case CORINFO_TYPE_NATIVEINT: - return m_simdHandleCache->SIMDNIntHandle; - case CORINFO_TYPE_NATIVEUINT: - return m_simdHandleCache->SIMDNUIntHandle; - default: - assert(!"Didn't find a class handle for simdType"); + if (emitTypeSize(simdType) != getSIMDVectorRegisterByteLength()) + { + // We have scenarios, such as shifting Vector by a non-constant + // which may introduce different sized vectors that are marked as + // isSimdAsHWIntrinsic. + + return NO_CLASS_HANDLE; } - return NO_CLASS_HANDLE; + uint32_t handleIndex = static_cast(simdBaseJitType - CORINFO_TYPE_BYTE); + assert(handleIndex < SIMDHandlesCache::SupportedTypeCount); + + return m_simdHandleCache->VectorTHandles[handleIndex]; + } + + CORINFO_CLASS_HANDLE gtGetStructHandleForHWSIMD(var_types simdType, CorInfoType simdBaseJitType) + { + assert(varTypeIsSIMD(simdType)); + assert((simdBaseJitType >= CORINFO_TYPE_BYTE) && (simdBaseJitType <= CORINFO_TYPE_DOUBLE)); + + // We should only be called from gtGetStructHandleForSimdOrHW and this should've been checked already + assert(m_simdHandleCache != nullptr); + + uint32_t handleIndex = static_cast(simdBaseJitType - CORINFO_TYPE_BYTE); + assert(handleIndex < SIMDHandlesCache::SupportedTypeCount); + + switch (simdType) + { + case TYP_SIMD8: + { +#if defined(TARGET_ARM64) + return m_simdHandleCache->Vector64THandles[handleIndex]; +#else + // This can only be Vector2 and should've been handled by gtGetStructHandleForSIMD + return NO_CLASS_HANDLE; +#endif + } + + case TYP_SIMD12: + { + // This can only be Vector3 and should've been handled by gtGetStructHandleForSIMD + return NO_CLASS_HANDLE; + } + + case TYP_SIMD16: + { + return m_simdHandleCache->Vector128THandles[handleIndex]; + } + +#if defined(TARGET_XARCH) + case TYP_SIMD32: + { + return m_simdHandleCache->Vector256THandles[handleIndex]; + } + + case TYP_SIMD64: + { + return m_simdHandleCache->Vector512THandles[handleIndex]; + } +#endif // TARGET_XARCH + + default: + { + unreached(); + } + } } -#if defined(FEATURE_HW_INTRINSICS) CORINFO_CLASS_HANDLE gtGetStructHandleForSimdOrHW(var_types simdType, CorInfoType simdBaseJitType, bool isSimdAsHWIntrinsic = false) { assert(varTypeIsSIMD(simdType)); + assert((simdBaseJitType >= CORINFO_TYPE_BYTE) && (simdBaseJitType <= CORINFO_TYPE_DOUBLE)); + + if (m_simdHandleCache == nullptr) + { + // This may happen if the JIT generates SIMD node on its own, without importing them. + // Otherwise getBaseJitTypeAndSizeOfSIMDType should have created the cache. + return NO_CLASS_HANDLE; + } CORINFO_CLASS_HANDLE clsHnd = NO_CLASS_HANDLE; @@ -8612,16 +8595,15 @@ private: { clsHnd = gtGetStructHandleForSIMD(simdType, simdBaseJitType); } - else + + if (clsHnd == NO_CLASS_HANDLE) { clsHnd = gtGetStructHandleForHWSIMD(simdType, simdBaseJitType); } - // Currently there are cases where isSimdAsHWIntrinsic is passed - // incorrectly. Fall back to the canonical SIMD handle in that case. - // TODO-cleanup: We can probably just always use the canonical handle. if (clsHnd == NO_CLASS_HANDLE) { + // TODO-cleanup: We can probably just always use the canonical handle. clsHnd = gtGetCanonicalStructHandleForSIMD(simdType); } @@ -8657,7 +8639,7 @@ private: case TYP_SIMD8: return m_simdHandleCache->CanonicalSimd8Handle; case TYP_SIMD12: - return m_simdHandleCache->SIMDVector3Handle; + return m_simdHandleCache->Vector3Handle; case TYP_SIMD16: return m_simdHandleCache->CanonicalSimd16Handle; #if defined(TARGET_XARCH) @@ -8685,27 +8667,27 @@ private: return false; } - if (structHandle == m_simdHandleCache->SIMDVector4Handle) + if (structHandle == m_simdHandleCache->Vector4Handle) { return false; } - if (structHandle == m_simdHandleCache->SIMDVector3Handle) + if (structHandle == m_simdHandleCache->Vector3Handle) { return false; } - if (structHandle == m_simdHandleCache->SIMDVector2Handle) + if (structHandle == m_simdHandleCache->Vector2Handle) { return false; } - if (structHandle == m_simdHandleCache->SIMDQuaternionHandle) + if (structHandle == m_simdHandleCache->QuaternionHandle) { return false; } - if (structHandle == m_simdHandleCache->SIMDPlaneHandle) + if (structHandle == m_simdHandleCache->PlaneHandle) { return false; } diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 7e85335..17fd44b 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -86,150 +86,6 @@ CorInfoType Compiler::getBaseJitTypeFromArgIfNeeded(NamedIntrinsic intrins return simdBaseJitType; } -CORINFO_CLASS_HANDLE Compiler::gtGetStructHandleForHWSIMD(var_types simdType, CorInfoType simdBaseJitType) -{ - assert(varTypeIsSIMD(simdType)); - - if (m_simdHandleCache == nullptr) - { - return NO_CLASS_HANDLE; - } - if (simdType == TYP_SIMD16) - { - switch (simdBaseJitType) - { - case CORINFO_TYPE_FLOAT: - return m_simdHandleCache->Vector128FloatHandle; - case CORINFO_TYPE_DOUBLE: - return m_simdHandleCache->Vector128DoubleHandle; - case CORINFO_TYPE_INT: - return m_simdHandleCache->Vector128IntHandle; - case CORINFO_TYPE_USHORT: - return m_simdHandleCache->Vector128UShortHandle; - case CORINFO_TYPE_UBYTE: - return m_simdHandleCache->Vector128UByteHandle; - case CORINFO_TYPE_SHORT: - return m_simdHandleCache->Vector128ShortHandle; - case CORINFO_TYPE_BYTE: - return m_simdHandleCache->Vector128ByteHandle; - case CORINFO_TYPE_LONG: - return m_simdHandleCache->Vector128LongHandle; - case CORINFO_TYPE_UINT: - return m_simdHandleCache->Vector128UIntHandle; - case CORINFO_TYPE_ULONG: - return m_simdHandleCache->Vector128ULongHandle; - case CORINFO_TYPE_NATIVEINT: - return m_simdHandleCache->Vector128NIntHandle; - case CORINFO_TYPE_NATIVEUINT: - return m_simdHandleCache->Vector128NUIntHandle; - default: - assert(!"Didn't find a class handle for simdType"); - } - } -#if defined(TARGET_XARCH) - else if (simdType == TYP_SIMD32) - { - switch (simdBaseJitType) - { - case CORINFO_TYPE_FLOAT: - return m_simdHandleCache->Vector256FloatHandle; - case CORINFO_TYPE_DOUBLE: - return m_simdHandleCache->Vector256DoubleHandle; - case CORINFO_TYPE_INT: - return m_simdHandleCache->Vector256IntHandle; - case CORINFO_TYPE_USHORT: - return m_simdHandleCache->Vector256UShortHandle; - case CORINFO_TYPE_UBYTE: - return m_simdHandleCache->Vector256UByteHandle; - case CORINFO_TYPE_SHORT: - return m_simdHandleCache->Vector256ShortHandle; - case CORINFO_TYPE_BYTE: - return m_simdHandleCache->Vector256ByteHandle; - case CORINFO_TYPE_LONG: - return m_simdHandleCache->Vector256LongHandle; - case CORINFO_TYPE_UINT: - return m_simdHandleCache->Vector256UIntHandle; - case CORINFO_TYPE_ULONG: - return m_simdHandleCache->Vector256ULongHandle; - case CORINFO_TYPE_NATIVEINT: - return m_simdHandleCache->Vector256NIntHandle; - case CORINFO_TYPE_NATIVEUINT: - return m_simdHandleCache->Vector256NUIntHandle; - default: - assert(!"Didn't find a class handle for simdType"); - } - } - else if (simdType == TYP_SIMD64) - { - switch (simdBaseJitType) - { - case CORINFO_TYPE_FLOAT: - return m_simdHandleCache->Vector512FloatHandle; - case CORINFO_TYPE_DOUBLE: - return m_simdHandleCache->Vector512DoubleHandle; - case CORINFO_TYPE_INT: - return m_simdHandleCache->Vector512IntHandle; - case CORINFO_TYPE_USHORT: - return m_simdHandleCache->Vector512UShortHandle; - case CORINFO_TYPE_UBYTE: - return m_simdHandleCache->Vector512UByteHandle; - case CORINFO_TYPE_SHORT: - return m_simdHandleCache->Vector512ShortHandle; - case CORINFO_TYPE_BYTE: - return m_simdHandleCache->Vector512ByteHandle; - case CORINFO_TYPE_LONG: - return m_simdHandleCache->Vector512LongHandle; - case CORINFO_TYPE_UINT: - return m_simdHandleCache->Vector512UIntHandle; - case CORINFO_TYPE_ULONG: - return m_simdHandleCache->Vector512ULongHandle; - case CORINFO_TYPE_NATIVEINT: - return m_simdHandleCache->Vector512NIntHandle; - case CORINFO_TYPE_NATIVEUINT: - return m_simdHandleCache->Vector512NUIntHandle; - default: - assert(!"Didn't find a class handle for simdType"); - } - } -#endif // TARGET_XARCH -#ifdef TARGET_ARM64 - else if (simdType == TYP_SIMD8) - { - switch (simdBaseJitType) - { - case CORINFO_TYPE_FLOAT: - return m_simdHandleCache->Vector64FloatHandle; - case CORINFO_TYPE_DOUBLE: - return m_simdHandleCache->Vector64DoubleHandle; - case CORINFO_TYPE_INT: - return m_simdHandleCache->Vector64IntHandle; - case CORINFO_TYPE_USHORT: - return m_simdHandleCache->Vector64UShortHandle; - case CORINFO_TYPE_UBYTE: - return m_simdHandleCache->Vector64UByteHandle; - case CORINFO_TYPE_SHORT: - return m_simdHandleCache->Vector64ShortHandle; - case CORINFO_TYPE_BYTE: - return m_simdHandleCache->Vector64ByteHandle; - case CORINFO_TYPE_UINT: - return m_simdHandleCache->Vector64UIntHandle; - case CORINFO_TYPE_LONG: - return m_simdHandleCache->Vector64LongHandle; - case CORINFO_TYPE_ULONG: - return m_simdHandleCache->Vector64ULongHandle; - case CORINFO_TYPE_NATIVEINT: - return m_simdHandleCache->Vector64NIntHandle; - case CORINFO_TYPE_NATIVEUINT: - return m_simdHandleCache->Vector64NUIntHandle; - default: - assert(!"Didn't find a class handle for simdType"); - } - } -#endif // TARGET_ARM64 - - return NO_CLASS_HANDLE; -} - //------------------------------------------------------------------------ // vnEncodesResultTypeForHWIntrinsic(NamedIntrinsic hwIntrinsicID): // diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 179f19b..57f3498 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -1056,7 +1056,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, if (varTypeIsByte(simdBaseType) && (simdSize == 16)) { - CORINFO_CLASS_HANDLE simdClsHnd = gtGetStructHandleForSIMD(simdType, simdBaseJitType); + CORINFO_CLASS_HANDLE simdClsHnd = gtGetStructHandleForSimdOrHW(simdType, simdBaseJitType); op1 = impCloneExpr(op1, &op2, simdClsHnd, CHECK_SPILL_ALL, nullptr DEBUGARG("Clone op1 for vector extractmostsignificantbits")); @@ -1091,7 +1091,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, { if ((simdSize == 8) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT))) { - CORINFO_CLASS_HANDLE simdClsHnd = gtGetStructHandleForSIMD(simdType, simdBaseJitType); + CORINFO_CLASS_HANDLE simdClsHnd = gtGetStructHandleForSimdOrHW(simdType, simdBaseJitType); op1 = impCloneExpr(op1, &op2, simdClsHnd, CHECK_SPILL_ALL, nullptr DEBUGARG("Clone op1 for vector extractmostsignificantbits")); diff --git a/src/coreclr/jit/simd.cpp b/src/coreclr/jit/simd.cpp index 599d3c6..4adf444 100644 --- a/src/coreclr/jit/simd.cpp +++ b/src/coreclr/jit/simd.cpp @@ -188,861 +188,265 @@ CorInfoType Compiler::getBaseJitTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeH if (isNumericsNamespace(namespaceName)) { - // The most likely to be used type handles are looked up first followed by - // less likely to be used type handles - if (typeHnd == m_simdHandleCache->SIMDFloatHandle) + switch (className[0]) { - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDIntHandle) - { - simdBaseJitType = CORINFO_TYPE_INT; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDVector2Handle) - { - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = 2 * genTypeSize(TYP_FLOAT); - assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); - JITDUMP(" Known type Vector2\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDVector3Handle) - { - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = 3 * genTypeSize(TYP_FLOAT); - assert(size == info.compCompHnd->getClassSize(typeHnd)); - JITDUMP(" Known type Vector3\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDVector4Handle) - { - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = 4 * genTypeSize(TYP_FLOAT); - assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); - JITDUMP(" Known type Vector4\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDVectorHandle) - { - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDUShortHandle) - { - simdBaseJitType = CORINFO_TYPE_USHORT; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDUByteHandle) - { - simdBaseJitType = CORINFO_TYPE_UBYTE; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDDoubleHandle) - { - simdBaseJitType = CORINFO_TYPE_DOUBLE; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDLongHandle) - { - simdBaseJitType = CORINFO_TYPE_LONG; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDShortHandle) - { - simdBaseJitType = CORINFO_TYPE_SHORT; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDByteHandle) - { - simdBaseJitType = CORINFO_TYPE_BYTE; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDUIntHandle) - { - simdBaseJitType = CORINFO_TYPE_UINT; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDULongHandle) - { - simdBaseJitType = CORINFO_TYPE_ULONG; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDNIntHandle) - { - simdBaseJitType = CORINFO_TYPE_NATIVEINT; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDNUIntHandle) - { - simdBaseJitType = CORINFO_TYPE_NATIVEUINT; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Known type SIMD Vector\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDQuaternionHandle) - { - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = 4 * genTypeSize(TYP_FLOAT); - assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); - JITDUMP(" Known type Quaternion\n"); - } - else if (typeHnd == m_simdHandleCache->SIMDPlaneHandle) - { - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = 4 * genTypeSize(TYP_FLOAT); - assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); - JITDUMP(" Known type Plane\n"); - } + case 'P': + { + if (strcmp(className, "Plane") != 0) + { + return CORINFO_TYPE_UNDEF; + } - // slow path search - if (simdBaseJitType == CORINFO_TYPE_UNDEF) - { - JITDUMP("SIMD Candidate Type %s\n", className); + JITDUMP(" Known type Plane\n"); + m_simdHandleCache->PlaneHandle = typeHnd; - if (strcmp(className, "Vector`1") == 0) + simdBaseJitType = CORINFO_TYPE_FLOAT; + size = 4 * genTypeSize(TYP_FLOAT); + break; + } + + case 'Q': { - size = getSIMDVectorRegisterByteLength(); + if (strcmp(className, "Quaternion") != 0) + { + return CORINFO_TYPE_UNDEF; + } - CORINFO_CLASS_HANDLE typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(typeHnd, 0); - simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); + JITDUMP(" Known type Quaternion\n"); + m_simdHandleCache->QuaternionHandle = typeHnd; + + simdBaseJitType = CORINFO_TYPE_FLOAT; + size = 4 * genTypeSize(TYP_FLOAT); + break; + } - switch (simdBaseJitType) + case 'V': + { + if (strncmp(className, "Vector", 6) != 0) { - case CORINFO_TYPE_FLOAT: - m_simdHandleCache->SIMDFloatHandle = typeHnd; - break; - case CORINFO_TYPE_INT: - m_simdHandleCache->SIMDIntHandle = typeHnd; - break; - case CORINFO_TYPE_USHORT: - m_simdHandleCache->SIMDUShortHandle = typeHnd; - break; - case CORINFO_TYPE_UBYTE: - m_simdHandleCache->SIMDUByteHandle = typeHnd; - break; - case CORINFO_TYPE_DOUBLE: - m_simdHandleCache->SIMDDoubleHandle = typeHnd; - break; - case CORINFO_TYPE_LONG: - m_simdHandleCache->SIMDLongHandle = typeHnd; - break; - case CORINFO_TYPE_SHORT: - m_simdHandleCache->SIMDShortHandle = typeHnd; - break; - case CORINFO_TYPE_BYTE: - m_simdHandleCache->SIMDByteHandle = typeHnd; + return CORINFO_TYPE_UNDEF; + } + + switch (className[6]) + { + case '\0': + { + JITDUMP(" Found type Vector\n"); + m_simdHandleCache->VectorHandle = typeHnd; + + size = getSIMDVectorRegisterByteLength(); break; - case CORINFO_TYPE_UINT: - m_simdHandleCache->SIMDUIntHandle = typeHnd; + } + + case '2': + { + if (className[7] != '\0') + { + return CORINFO_TYPE_UNDEF; + } + + JITDUMP(" Found Vector2\n"); + m_simdHandleCache->Vector2Handle = typeHnd; + + simdBaseJitType = CORINFO_TYPE_FLOAT; + size = 2 * genTypeSize(TYP_FLOAT); break; - case CORINFO_TYPE_ULONG: - m_simdHandleCache->SIMDULongHandle = typeHnd; + } + + case '3': + { + if (className[7] != '\0') + { + return CORINFO_TYPE_UNDEF; + } + + JITDUMP(" Found Vector3\n"); + m_simdHandleCache->Vector3Handle = typeHnd; + + simdBaseJitType = CORINFO_TYPE_FLOAT; + size = 3 * genTypeSize(TYP_FLOAT); break; - case CORINFO_TYPE_NATIVEINT: - m_simdHandleCache->SIMDNIntHandle = typeHnd; + } + + case '4': + { + if (className[7] != '\0') + { + return CORINFO_TYPE_UNDEF; + } + + JITDUMP(" Found Vector4\n"); + m_simdHandleCache->Vector4Handle = typeHnd; + + simdBaseJitType = CORINFO_TYPE_FLOAT; + size = 4 * genTypeSize(TYP_FLOAT); break; - case CORINFO_TYPE_NATIVEUINT: - m_simdHandleCache->SIMDNUIntHandle = typeHnd; + } + + case '`': + { + if ((className[7] != '1') || (className[8] != '\0')) + { + return CORINFO_TYPE_UNDEF; + } + + CORINFO_CLASS_HANDLE typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(typeHnd, 0); + simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); + + if ((simdBaseJitType < CORINFO_TYPE_BYTE) || (simdBaseJitType > CORINFO_TYPE_DOUBLE)) + { + return CORINFO_TYPE_UNDEF; + } + + JITDUMP(" Found Vector<%s>\n", varTypeName(JitType2PreciseVarType(simdBaseJitType))); + + size = getSIMDVectorRegisterByteLength(); + + uint32_t handleIndex = static_cast(simdBaseJitType - CORINFO_TYPE_BYTE); + assert(handleIndex < SIMDHandlesCache::SupportedTypeCount); + + m_simdHandleCache->VectorTHandles[handleIndex] = typeHnd; break; + } + default: - simdBaseJitType = CORINFO_TYPE_UNDEF; - break; + { + return CORINFO_TYPE_UNDEF; + } } + break; + } - if (simdBaseJitType != CORINFO_TYPE_UNDEF) + default: + { + return CORINFO_TYPE_UNDEF; + } + } + } +#ifdef FEATURE_HW_INTRINSICS + else + { + size = info.compCompHnd->getClassSize(typeHnd); + + switch (size) + { +#if defined(TARGET_ARM64) + case 8: + { + if (strcmp(className, "Vector64`1") != 0) { - JITDUMP(" Found type SIMD Vector<%s>\n", varTypeName(JitType2PreciseVarType(simdBaseJitType))); + return CORINFO_TYPE_UNDEF; } - else + + CORINFO_CLASS_HANDLE typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(typeHnd, 0); + simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); + + if ((simdBaseJitType < CORINFO_TYPE_BYTE) || (simdBaseJitType > CORINFO_TYPE_DOUBLE)) { - JITDUMP(" Unknown SIMD Vector\n"); + return CORINFO_TYPE_UNDEF; } - } - else if (strcmp(className, "Vector2") == 0) - { - m_simdHandleCache->SIMDVector2Handle = typeHnd; - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = 2 * genTypeSize(TYP_FLOAT); - assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); - JITDUMP(" Found Vector2\n"); - } - else if (strcmp(className, "Vector3") == 0) - { - m_simdHandleCache->SIMDVector3Handle = typeHnd; + JITDUMP(" Found Vector64<%s>\n", varTypeName(JitType2PreciseVarType(simdBaseJitType))); - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = 3 * genTypeSize(TYP_FLOAT); - assert(size == info.compCompHnd->getClassSize(typeHnd)); - JITDUMP(" Found Vector3\n"); - } - else if (strcmp(className, "Vector4") == 0) - { - m_simdHandleCache->SIMDVector4Handle = typeHnd; + uint32_t handleIndex = static_cast(simdBaseJitType - CORINFO_TYPE_BYTE); + assert(handleIndex < SIMDHandlesCache::SupportedTypeCount); - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = 4 * genTypeSize(TYP_FLOAT); - assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); - JITDUMP(" Found Vector4\n"); - } - else if (strcmp(className, "Vector") == 0) - { - m_simdHandleCache->SIMDVectorHandle = typeHnd; - size = getSIMDVectorRegisterByteLength(); - JITDUMP(" Found type Vector\n"); + m_simdHandleCache->Vector64THandles[handleIndex] = typeHnd; + break; } - else if (strcmp(className, "Quaternion") == 0) - { - m_simdHandleCache->SIMDQuaternionHandle = typeHnd; +#endif // TARGET_ARM64 - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = 4 * genTypeSize(TYP_FLOAT); - assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); - JITDUMP(" Found Quaternion\n"); - } - else if (strcmp(className, "Plane") == 0) + case 16: { - m_simdHandleCache->SIMDPlaneHandle = typeHnd; + if (strcmp(className, "Vector128`1") != 0) + { + return CORINFO_TYPE_UNDEF; + } - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = 4 * genTypeSize(TYP_FLOAT); - assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); - JITDUMP(" Found Plane\n"); - } - } - } -#ifdef FEATURE_HW_INTRINSICS - else - { - const size_t Vector64SizeBytes = 64 / 8; - const size_t Vector128SizeBytes = 128 / 8; - const size_t Vector256SizeBytes = 256 / 8; - const size_t Vector512SizeBytes = 512 / 8; + CORINFO_CLASS_HANDLE typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(typeHnd, 0); + simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); -#if defined(TARGET_XARCH) - static_assert_no_msg(ZMM_REGSIZE_BYTES == Vector512SizeBytes); - static_assert_no_msg(YMM_REGSIZE_BYTES == Vector256SizeBytes); - static_assert_no_msg(XMM_REGSIZE_BYTES == Vector128SizeBytes); + if ((simdBaseJitType < CORINFO_TYPE_BYTE) || (simdBaseJitType > CORINFO_TYPE_DOUBLE)) + { + return CORINFO_TYPE_UNDEF; + } - if (typeHnd == m_simdHandleCache->Vector512FloatHandle) - { - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512DoubleHandle) - { - simdBaseJitType = CORINFO_TYPE_DOUBLE; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512IntHandle) - { - simdBaseJitType = CORINFO_TYPE_INT; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512UIntHandle) - { - simdBaseJitType = CORINFO_TYPE_UINT; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512ShortHandle) - { - simdBaseJitType = CORINFO_TYPE_SHORT; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512UShortHandle) - { - simdBaseJitType = CORINFO_TYPE_USHORT; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512ByteHandle) - { - simdBaseJitType = CORINFO_TYPE_BYTE; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512UByteHandle) - { - simdBaseJitType = CORINFO_TYPE_UBYTE; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512LongHandle) - { - simdBaseJitType = CORINFO_TYPE_LONG; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512ULongHandle) - { - simdBaseJitType = CORINFO_TYPE_ULONG; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512NIntHandle) - { - simdBaseJitType = CORINFO_TYPE_NATIVEINT; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } - else if (typeHnd == m_simdHandleCache->Vector512NUIntHandle) - { - simdBaseJitType = CORINFO_TYPE_NATIVEUINT; - size = Vector512SizeBytes; - JITDUMP(" Known type Vector512\n"); - } + JITDUMP(" Found Vector128<%s>\n", varTypeName(JitType2PreciseVarType(simdBaseJitType))); - else if (typeHnd == m_simdHandleCache->Vector256FloatHandle) - { - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256DoubleHandle) - { - simdBaseJitType = CORINFO_TYPE_DOUBLE; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256IntHandle) - { - simdBaseJitType = CORINFO_TYPE_INT; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256UIntHandle) - { - simdBaseJitType = CORINFO_TYPE_UINT; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256ShortHandle) - { - simdBaseJitType = CORINFO_TYPE_SHORT; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256UShortHandle) - { - simdBaseJitType = CORINFO_TYPE_USHORT; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256ByteHandle) - { - simdBaseJitType = CORINFO_TYPE_BYTE; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256UByteHandle) - { - simdBaseJitType = CORINFO_TYPE_UBYTE; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256LongHandle) - { - simdBaseJitType = CORINFO_TYPE_LONG; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256ULongHandle) - { - simdBaseJitType = CORINFO_TYPE_ULONG; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256NIntHandle) - { - simdBaseJitType = CORINFO_TYPE_NATIVEINT; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else if (typeHnd == m_simdHandleCache->Vector256NUIntHandle) - { - simdBaseJitType = CORINFO_TYPE_NATIVEUINT; - size = Vector256SizeBytes; - JITDUMP(" Known type Vector256\n"); - } - else -#endif // defined(TARGET_XARCH) - if (typeHnd == m_simdHandleCache->Vector128FloatHandle) - { - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128DoubleHandle) - { - simdBaseJitType = CORINFO_TYPE_DOUBLE; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128IntHandle) - { - simdBaseJitType = CORINFO_TYPE_INT; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128UIntHandle) - { - simdBaseJitType = CORINFO_TYPE_UINT; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128ShortHandle) - { - simdBaseJitType = CORINFO_TYPE_SHORT; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128UShortHandle) - { - simdBaseJitType = CORINFO_TYPE_USHORT; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128ByteHandle) - { - simdBaseJitType = CORINFO_TYPE_BYTE; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128UByteHandle) - { - simdBaseJitType = CORINFO_TYPE_UBYTE; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128LongHandle) - { - simdBaseJitType = CORINFO_TYPE_LONG; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128ULongHandle) - { - simdBaseJitType = CORINFO_TYPE_ULONG; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128NIntHandle) - { - simdBaseJitType = CORINFO_TYPE_NATIVEINT; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else if (typeHnd == m_simdHandleCache->Vector128NUIntHandle) - { - simdBaseJitType = CORINFO_TYPE_NATIVEUINT; - size = Vector128SizeBytes; - JITDUMP(" Known type Vector128\n"); - } - else -#if defined(TARGET_ARM64) - if (typeHnd == m_simdHandleCache->Vector64FloatHandle) - { - simdBaseJitType = CORINFO_TYPE_FLOAT; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64DoubleHandle) - { - simdBaseJitType = CORINFO_TYPE_DOUBLE; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64IntHandle) - { - simdBaseJitType = CORINFO_TYPE_INT; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64UIntHandle) - { - simdBaseJitType = CORINFO_TYPE_UINT; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64ShortHandle) - { - simdBaseJitType = CORINFO_TYPE_SHORT; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64UShortHandle) - { - simdBaseJitType = CORINFO_TYPE_USHORT; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64ByteHandle) - { - simdBaseJitType = CORINFO_TYPE_BYTE; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64UByteHandle) - { - simdBaseJitType = CORINFO_TYPE_UBYTE; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64LongHandle) - { - simdBaseJitType = CORINFO_TYPE_LONG; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64ULongHandle) - { - simdBaseJitType = CORINFO_TYPE_ULONG; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64NIntHandle) - { - simdBaseJitType = CORINFO_TYPE_NATIVEINT; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } - else if (typeHnd == m_simdHandleCache->Vector64NUIntHandle) - { - simdBaseJitType = CORINFO_TYPE_NATIVEUINT; - size = Vector64SizeBytes; - JITDUMP(" Known type Vector64\n"); - } -#endif // defined(TARGET_ARM64) + uint32_t handleIndex = static_cast(simdBaseJitType - CORINFO_TYPE_BYTE); + assert(handleIndex < SIMDHandlesCache::SupportedTypeCount); - // slow path search - if (simdBaseJitType == CORINFO_TYPE_UNDEF) - { - // Doesn't match with any of the cached type handles. - CORINFO_CLASS_HANDLE baseTypeHnd = getTypeInstantiationArgument(typeHnd, 0); + m_simdHandleCache->Vector128THandles[handleIndex] = typeHnd; + break; + } - if (baseTypeHnd != nullptr) +#if defined(TARGET_XARCH) + case 32: { - CorInfoType type = info.compCompHnd->getTypeForPrimitiveNumericClass(baseTypeHnd); + if (strcmp(className, "Vector256`1") != 0) + { + return CORINFO_TYPE_UNDEF; + } - JITDUMP("HW Intrinsic SIMD Candidate Type %s with Base Type %s\n", className, - getClassNameFromMetadata(baseTypeHnd, nullptr)); + CORINFO_CLASS_HANDLE typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(typeHnd, 0); + simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); -#if defined(TARGET_XARCH) - if (strcmp(className, "Vector512`1") == 0) + if ((simdBaseJitType < CORINFO_TYPE_BYTE) || (simdBaseJitType > CORINFO_TYPE_DOUBLE)) { - size = Vector512SizeBytes; - switch (type) - { - case CORINFO_TYPE_FLOAT: - m_simdHandleCache->Vector512FloatHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_FLOAT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_DOUBLE: - m_simdHandleCache->Vector512DoubleHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_DOUBLE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_INT: - m_simdHandleCache->Vector512IntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_INT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_UINT: - m_simdHandleCache->Vector512UIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_UINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_SHORT: - m_simdHandleCache->Vector512ShortHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_SHORT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_USHORT: - m_simdHandleCache->Vector512UShortHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_USHORT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_LONG: - m_simdHandleCache->Vector512LongHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_LONG; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_ULONG: - m_simdHandleCache->Vector512ULongHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_ULONG; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_UBYTE: - m_simdHandleCache->Vector512UByteHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_UBYTE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_BYTE: - m_simdHandleCache->Vector512ByteHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_BYTE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_NATIVEINT: - m_simdHandleCache->Vector512NIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_NATIVEINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - case CORINFO_TYPE_NATIVEUINT: - m_simdHandleCache->Vector512NUIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_NATIVEUINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector512\n"); - break; - - default: - JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector512\n"); - } + return CORINFO_TYPE_UNDEF; } - else if (strcmp(className, "Vector256`1") == 0) + + if (!compExactlyDependsOn(InstructionSet_AVX)) { - size = Vector256SizeBytes; - switch (type) - { - case CORINFO_TYPE_FLOAT: - m_simdHandleCache->Vector256FloatHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_FLOAT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_DOUBLE: - m_simdHandleCache->Vector256DoubleHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_DOUBLE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_INT: - m_simdHandleCache->Vector256IntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_INT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_UINT: - m_simdHandleCache->Vector256UIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_UINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_SHORT: - m_simdHandleCache->Vector256ShortHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_SHORT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_USHORT: - m_simdHandleCache->Vector256UShortHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_USHORT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_LONG: - m_simdHandleCache->Vector256LongHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_LONG; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_ULONG: - m_simdHandleCache->Vector256ULongHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_ULONG; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_UBYTE: - m_simdHandleCache->Vector256UByteHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_UBYTE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_BYTE: - m_simdHandleCache->Vector256ByteHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_BYTE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_NATIVEINT: - m_simdHandleCache->Vector256NIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_NATIVEINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - case CORINFO_TYPE_NATIVEUINT: - m_simdHandleCache->Vector256NUIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_NATIVEUINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector256\n"); - break; - - default: - JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector256\n"); - } + // We must treat as a regular struct if AVX isn't supported + return CORINFO_TYPE_UNDEF; } - else -#endif // defined(TARGET_XARCH) - if (strcmp(className, "Vector128`1") == 0) + + JITDUMP(" Found Vector256<%s>\n", varTypeName(JitType2PreciseVarType(simdBaseJitType))); + + uint32_t handleIndex = static_cast(simdBaseJitType - CORINFO_TYPE_BYTE); + assert(handleIndex < SIMDHandlesCache::SupportedTypeCount); + + m_simdHandleCache->Vector256THandles[handleIndex] = typeHnd; + break; + } + + case 64: + { + if (strcmp(className, "Vector512`1") != 0) { - size = Vector128SizeBytes; - switch (type) - { - case CORINFO_TYPE_FLOAT: - m_simdHandleCache->Vector128FloatHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_FLOAT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_DOUBLE: - m_simdHandleCache->Vector128DoubleHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_DOUBLE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_INT: - m_simdHandleCache->Vector128IntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_INT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_UINT: - m_simdHandleCache->Vector128UIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_UINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_SHORT: - m_simdHandleCache->Vector128ShortHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_SHORT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_USHORT: - m_simdHandleCache->Vector128UShortHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_USHORT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_LONG: - m_simdHandleCache->Vector128LongHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_LONG; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_ULONG: - m_simdHandleCache->Vector128ULongHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_ULONG; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_UBYTE: - m_simdHandleCache->Vector128UByteHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_UBYTE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_BYTE: - m_simdHandleCache->Vector128ByteHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_BYTE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_NATIVEINT: - m_simdHandleCache->Vector128NIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_NATIVEINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - case CORINFO_TYPE_NATIVEUINT: - m_simdHandleCache->Vector128NUIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_NATIVEUINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector128\n"); - break; - - default: - JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector128\n"); - } + return CORINFO_TYPE_UNDEF; } -#if defined(TARGET_ARM64) - else if (strcmp(className, "Vector64`1") == 0) + + CORINFO_CLASS_HANDLE typeArgHnd = info.compCompHnd->getTypeInstantiationArgument(typeHnd, 0); + simdBaseJitType = info.compCompHnd->getTypeForPrimitiveNumericClass(typeArgHnd); + + if ((simdBaseJitType < CORINFO_TYPE_BYTE) || (simdBaseJitType > CORINFO_TYPE_DOUBLE)) { - size = Vector64SizeBytes; - switch (type) - { - case CORINFO_TYPE_FLOAT: - m_simdHandleCache->Vector64FloatHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_FLOAT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_DOUBLE: - m_simdHandleCache->Vector64DoubleHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_DOUBLE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_INT: - m_simdHandleCache->Vector64IntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_INT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_UINT: - m_simdHandleCache->Vector64UIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_UINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_SHORT: - m_simdHandleCache->Vector64ShortHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_SHORT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_USHORT: - m_simdHandleCache->Vector64UShortHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_USHORT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_LONG: - m_simdHandleCache->Vector64LongHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_LONG; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_ULONG: - m_simdHandleCache->Vector64ULongHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_ULONG; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_UBYTE: - m_simdHandleCache->Vector64UByteHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_UBYTE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_BYTE: - m_simdHandleCache->Vector64ByteHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_BYTE; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_NATIVEINT: - m_simdHandleCache->Vector64NIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_NATIVEINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - case CORINFO_TYPE_NATIVEUINT: - m_simdHandleCache->Vector64NUIntHandle = typeHnd; - simdBaseJitType = CORINFO_TYPE_NATIVEUINT; - JITDUMP(" Found type Hardware Intrinsic SIMD Vector64\n"); - break; - - default: - JITDUMP(" Unknown Hardware Intrinsic SIMD Type Vector64\n"); - } + return CORINFO_TYPE_UNDEF; } -#endif // defined(TARGET_ARM64) + + if (!compExactlyDependsOn(InstructionSet_AVX512F)) + { + // We must treat as a regular struct if AVX512F isn't supported + return CORINFO_TYPE_UNDEF; + } + + JITDUMP(" Found Vector512<%s>\n", varTypeName(JitType2PreciseVarType(simdBaseJitType))); + + uint32_t handleIndex = static_cast(simdBaseJitType - CORINFO_TYPE_BYTE); + assert(handleIndex < SIMDHandlesCache::SupportedTypeCount); + + m_simdHandleCache->Vector512THandles[handleIndex] = typeHnd; + break; } - } +#endif // TARGET_XARCH -#if defined(TARGET_XARCH) - // Even though Vector256 is TYP_SIMD32, if AVX isn't supported, then it must - // be treated as a regular struct - if (size == YMM_REGSIZE_BYTES && (simdBaseJitType != CORINFO_TYPE_UNDEF) && - !compExactlyDependsOn(InstructionSet_AVX)) - { - simdBaseJitType = CORINFO_TYPE_UNDEF; - } - if (size == ZMM_REGSIZE_BYTES && (simdBaseJitType != CORINFO_TYPE_UNDEF) && - !compExactlyDependsOn(InstructionSet_AVX512F)) - { - simdBaseJitType = CORINFO_TYPE_UNDEF; + default: + { + return CORINFO_TYPE_UNDEF; + } } -#endif // TARGET_XARCH } #endif // FEATURE_HW_INTRINSICS @@ -1053,9 +457,11 @@ CorInfoType Compiler::getBaseJitTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeH if (simdBaseJitType != CORINFO_TYPE_UNDEF) { + assert(size == info.compCompHnd->getClassSize(typeHnd)); setUsesSIMDTypes(true); CORINFO_CLASS_HANDLE* pCanonicalHnd = nullptr; + switch (size) { case 8: diff --git a/src/coreclr/jit/simdashwintrinsic.cpp b/src/coreclr/jit/simdashwintrinsic.cpp index 2ce3f54..e6d2bc5 100644 --- a/src/coreclr/jit/simdashwintrinsic.cpp +++ b/src/coreclr/jit/simdashwintrinsic.cpp @@ -240,7 +240,7 @@ GenTree* Compiler::impSimdAsHWIntrinsic(NamedIntrinsic intrinsic, // if it isn't the basis for anything carried on the node. simdBaseJitType = getBaseJitTypeAndSizeOfSIMDType(clsHnd, &simdSize); - if ((clsHnd != m_simdHandleCache->SIMDVectorHandle) && + if ((clsHnd != m_simdHandleCache->VectorHandle) && ((simdBaseJitType == CORINFO_TYPE_UNDEF) || !varTypeIsArithmetic(JitType2PreciseVarType(simdBaseJitType)))) { // We want to exit early if the clsHnd should have a base type and it isn't one @@ -286,7 +286,7 @@ GenTree* Compiler::impSimdAsHWIntrinsic(NamedIntrinsic intrinsic, simdBaseJitType = getBaseJitTypeAndSizeOfSIMDType(clsHnd, &simdSize); } } - else if ((clsHnd == m_simdHandleCache->SIMDVectorHandle) && (numArgs != 0) && + else if ((clsHnd == m_simdHandleCache->VectorHandle) && (numArgs != 0) && !SimdAsHWIntrinsicInfo::KeepBaseTypeFromRet(intrinsic)) { // We need to fixup the clsHnd in the case we are an intrinsic on Vector -- 2.7.4