From 20dbfee9dc7bea590c359aaa8ebdd5cf47c46186 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 21 Dec 2018 13:13:21 -0800 Subject: [PATCH] Ensure that the S.N.Vector methods are marked as [Intrinsic] and that the attribute is respected by the runtime (dotnet/coreclr#21601) * Updating the SIMD intrinsics to check for the `[Intrinsic]` attribute * Ensure that the existing Vector/Vector intrinsic methods are marked as such. * Fixing Vector.DotProduct and Vector.AndNot to be treated as intrinsic * Fixing `SIMDIntrinsicBitwiseAndNot` to swap the operands for XARCH Commit migrated from https://github.com/dotnet/coreclr/commit/61c8abe982bc1936112ea843524b0921e373f40f --- src/coreclr/src/jit/compiler.h | 1 + src/coreclr/src/jit/importer.cpp | 2 +- src/coreclr/src/jit/simd.cpp | 17 +++++++++- src/coreclr/src/jit/simdintrinsiclist.h | 2 +- src/coreclr/src/vm/methodtablebuilder.cpp | 2 +- .../src/System/Numerics/Vector.cs | 18 +++++++++-- .../src/System/Numerics/Vector.tt | 18 +++++++++-- .../src/System/Numerics/Vector_Operations.cs | 36 +++++++++++++++++++++- 8 files changed, 87 insertions(+), 9 deletions(-) diff --git a/src/coreclr/src/jit/compiler.h b/src/coreclr/src/jit/compiler.h index 45c2223..3b72774 100644 --- a/src/coreclr/src/jit/compiler.h +++ b/src/coreclr/src/jit/compiler.h @@ -7896,6 +7896,7 @@ private: CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig, + unsigned methodFlags, int memberRef); GenTree* getOp1ForConstructor(OPCODE opcode, GenTree* newobjThis, CORINFO_CLASS_HANDLE clsHnd); diff --git a/src/coreclr/src/jit/importer.cpp b/src/coreclr/src/jit/importer.cpp index 5601200..512521d 100644 --- a/src/coreclr/src/jit/importer.cpp +++ b/src/coreclr/src/jit/importer.cpp @@ -7838,7 +7838,7 @@ var_types Compiler::impImportCall(OPCODE opcode, #ifdef FEATURE_SIMD if (featureSIMD) { - call = impSIMDIntrinsic(opcode, newobjThis, clsHnd, methHnd, sig, pResolvedToken->token); + call = impSIMDIntrinsic(opcode, newobjThis, clsHnd, methHnd, sig, mflags, pResolvedToken->token); if (call != nullptr) { bIntrinsicImported = true; diff --git a/src/coreclr/src/jit/simd.cpp b/src/coreclr/src/jit/simd.cpp index ccebc78..a302c63 100644 --- a/src/coreclr/src/jit/simd.cpp +++ b/src/coreclr/src/jit/simd.cpp @@ -2349,11 +2349,14 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE methodHnd, CORINFO_SIG_INFO* sig, + unsigned methodFlags, int memberRef) { assert(featureSIMD); - if (!isSIMDClass(clsHnd)) + // Exit early if we are either not in one of the SIMD types or if the method + // is not a JIT Intrinsic (which requires the [Intrinsic] attribute). + if (!isSIMDClass(clsHnd) || ((methodFlags & CORINFO_FLG_JIT_INTRINSIC) == 0)) { return nullptr; } @@ -2879,6 +2882,18 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, op2 = impSIMDPopStack(simdType); op1 = impSIMDPopStack(simdType, instMethod); +#ifdef _TARGET_XARCH_ + if (simdIntrinsicID == SIMDIntrinsicBitwiseAndNot) + { + // XARCH implements SIMDIntrinsicBitwiseAndNot as ~op1 & op2, while the + // software implementation does op1 & ~op2, so we need to swap the operands + + GenTree* tmp = op2; + op2 = op1; + op1 = tmp; + } +#endif // _TARGET_XARCH_ + simdTree = gtNewSIMDNode(simdType, op1, op2, simdIntrinsicID, baseType, size); retVal = simdTree; } diff --git a/src/coreclr/src/jit/simdintrinsiclist.h b/src/coreclr/src/jit/simdintrinsiclist.h index 19f0543..500cac1 100644 --- a/src/coreclr/src/jit/simdintrinsiclist.h +++ b/src/coreclr/src/jit/simdintrinsiclist.h @@ -112,7 +112,7 @@ SIMD_INTRINSIC("GreaterThanOrEqual", false, GreaterThanOrEqual, // Bitwise operations SIMD_INTRINSIC("op_BitwiseAnd", false, BitwiseAnd, "&", TYP_STRUCT, 2, {TYP_STRUCT, TYP_STRUCT, TYP_UNDEF}, {TYP_INT, TYP_FLOAT, TYP_DOUBLE, TYP_LONG, TYP_USHORT, TYP_UBYTE, TYP_BYTE, TYP_SHORT, TYP_UINT, TYP_ULONG}) -SIMD_INTRINSIC("BitwiseAndNot", false, BitwiseAndNot, "&~", TYP_STRUCT, 2, {TYP_STRUCT, TYP_STRUCT, TYP_UNDEF}, {TYP_INT, TYP_FLOAT, TYP_DOUBLE, TYP_LONG, TYP_USHORT, TYP_UBYTE, TYP_BYTE, TYP_SHORT, TYP_UINT, TYP_ULONG}) +SIMD_INTRINSIC("AndNot", false, BitwiseAndNot, "&~", TYP_STRUCT, 2, {TYP_STRUCT, TYP_STRUCT, TYP_UNDEF}, {TYP_INT, TYP_FLOAT, TYP_DOUBLE, TYP_LONG, TYP_USHORT, TYP_UBYTE, TYP_BYTE, TYP_SHORT, TYP_UINT, TYP_ULONG}) SIMD_INTRINSIC("op_BitwiseOr", false, BitwiseOr, "|", TYP_STRUCT, 2, {TYP_STRUCT, TYP_STRUCT, TYP_UNDEF}, {TYP_INT, TYP_FLOAT, TYP_DOUBLE, TYP_LONG, TYP_USHORT, TYP_UBYTE, TYP_BYTE, TYP_SHORT, TYP_UINT, TYP_ULONG}) SIMD_INTRINSIC("op_ExclusiveOr", false, BitwiseXor, "^", TYP_STRUCT, 2, {TYP_STRUCT, TYP_STRUCT, TYP_UNDEF}, {TYP_INT, TYP_FLOAT, TYP_DOUBLE, TYP_LONG, TYP_USHORT, TYP_UBYTE, TYP_BYTE, TYP_SHORT, TYP_UINT, TYP_ULONG}) diff --git a/src/coreclr/src/vm/methodtablebuilder.cpp b/src/coreclr/src/vm/methodtablebuilder.cpp index 3e30f38..c33c2db 100644 --- a/src/coreclr/src/vm/methodtablebuilder.cpp +++ b/src/coreclr/src/vm/methodtablebuilder.cpp @@ -5120,7 +5120,7 @@ MethodTableBuilder::InitNewMethodDesc( } // Check for methods marked as [Intrinsic] - if (GetModule()->IsSystem()) + if (GetModule()->IsSystem() || GetAssembly()->IsSIMDVectorAssembly()) { HRESULT hr = GetMDImport()->GetCustomAttributeByName(pMethod->GetMethodSignature().GetToken(), g_CompilerServicesIntrinsicAttribute, diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs index 25a80ed..0c1787d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs @@ -89,7 +89,14 @@ namespace System.Numerics } private static readonly Vector s_one = new Vector(GetOneValue()); - internal static Vector AllOnes { get { return s_allOnes; } } + internal static Vector AllOnes + { + [Intrinsic] + get + { + return s_allOnes; + } + } private static readonly Vector s_allOnes = new Vector(GetAllBitsSetValue()); #endregion Static Members @@ -380,6 +387,7 @@ namespace System.Numerics /// Constructs a vector from the given array, starting from the given index. /// The array must contain at least Vector'T.Count from the given index. /// + [Intrinsic] public unsafe Vector(T[] values, int index) : this() { @@ -1591,6 +1599,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The summed vector. + [Intrinsic] public static unsafe Vector operator +(Vector left, Vector right) { unchecked @@ -1802,6 +1811,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The difference vector. + [Intrinsic] public static unsafe Vector operator -(Vector left, Vector right) { unchecked @@ -2014,6 +2024,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The product vector. + [Intrinsic] public static unsafe Vector operator *(Vector left, Vector right) { unchecked @@ -2250,6 +2261,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The vector resulting from the division. + [Intrinsic] public static unsafe Vector operator /(Vector left, Vector right) { unchecked @@ -2579,6 +2591,7 @@ namespace System.Numerics /// The first vector to compare. /// The first vector to compare. /// True if all elements are equal; False otherwise. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static bool operator ==(Vector left, Vector right) { @@ -2591,6 +2604,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// True if left and right are not equal; False otherwise. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static bool operator !=(Vector left, Vector right) { @@ -3969,7 +3983,7 @@ namespace System.Numerics } [Intrinsic] - internal static T DotProduct(Vector left, Vector right) + internal static T Dot(Vector left, Vector right) { if (Vector.IsHardwareAccelerated) { diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.tt b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.tt index a3bc25c..4b0fc4f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.tt +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.tt @@ -94,7 +94,14 @@ namespace System.Numerics } private static readonly Vector s_one = new Vector(GetOneValue()); - internal static Vector AllOnes { get { return s_allOnes; } } + internal static Vector AllOnes + { + [Intrinsic] + get + { + return s_allOnes; + } + } private static readonly Vector s_allOnes = new Vector(GetAllBitsSetValue()); #endregion Static Members @@ -195,6 +202,7 @@ namespace System.Numerics /// Constructs a vector from the given array, starting from the given index. /// The array must contain at least Vector'T.Count from the given index. /// + [Intrinsic] public unsafe Vector(T[] values, int index) : this() { @@ -620,6 +628,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The summed vector. + [Intrinsic] public static unsafe Vector operator +(Vector left, Vector right) { unchecked @@ -679,6 +688,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The difference vector. + [Intrinsic] public static unsafe Vector operator -(Vector left, Vector right) { unchecked @@ -739,6 +749,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The product vector. + [Intrinsic] public static unsafe Vector operator *(Vector left, Vector right) { unchecked @@ -823,6 +834,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The vector resulting from the division. + [Intrinsic] public static unsafe Vector operator /(Vector left, Vector right) { unchecked @@ -1000,6 +1012,7 @@ namespace System.Numerics /// The first vector to compare. /// The first vector to compare. /// True if all elements are equal; False otherwise. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static bool operator ==(Vector left, Vector right) { @@ -1012,6 +1025,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// True if left and right are not equal; False otherwise. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static bool operator !=(Vector left, Vector right) { @@ -1416,7 +1430,7 @@ namespace System.Numerics } [Intrinsic] - internal static T DotProduct(Vector left, Vector right) + internal static T Dot(Vector left, Vector right) { if (Vector.IsHardwareAccelerated) { diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_Operations.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_Operations.cs index d3ed250..567a0a7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_Operations.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_Operations.cs @@ -21,6 +21,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The new vector with elements selected based on the mask. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector ConditionalSelect(Vector condition, Vector left, Vector right) { @@ -34,6 +35,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The new vector with elements selected based on the mask. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector ConditionalSelect(Vector condition, Vector left, Vector right) { @@ -47,6 +49,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The new vector with elements selected based on the mask. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector ConditionalSelect(Vector condition, Vector left, Vector right) where T : struct { @@ -62,6 +65,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector Equals(Vector left, Vector right) where T : struct { @@ -74,6 +78,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector Equals(Vector left, Vector right) { @@ -86,6 +91,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector Equals(Vector left, Vector right) { @@ -98,6 +104,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector Equals(Vector left, Vector right) { @@ -110,6 +117,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector Equals(Vector left, Vector right) { @@ -149,6 +157,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector LessThan(Vector left, Vector right) where T : struct { @@ -162,6 +171,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant integral vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector LessThan(Vector left, Vector right) { @@ -175,6 +185,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector LessThan(Vector left, Vector right) { @@ -188,6 +199,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant integral vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector LessThan(Vector left, Vector right) { @@ -201,6 +213,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector LessThan(Vector left, Vector right) { @@ -242,6 +255,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector LessThanOrEqual(Vector left, Vector right) where T : struct { @@ -255,6 +269,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant integral vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector LessThanOrEqual(Vector left, Vector right) { @@ -268,6 +283,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector LessThanOrEqual(Vector left, Vector right) { @@ -281,6 +297,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector LessThanOrEqual(Vector left, Vector right) { @@ -294,6 +311,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant integral vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector LessThanOrEqual(Vector left, Vector right) { @@ -335,6 +353,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector GreaterThan(Vector left, Vector right) where T : struct { @@ -348,6 +367,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant integral vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector GreaterThan(Vector left, Vector right) { @@ -361,6 +381,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector GreaterThan(Vector left, Vector right) { @@ -374,6 +395,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant integral vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector GreaterThan(Vector left, Vector right) { @@ -387,6 +409,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector GreaterThan(Vector left, Vector right) { @@ -429,6 +452,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector GreaterThanOrEqual(Vector left, Vector right) where T : struct { @@ -442,6 +466,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant integral vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector GreaterThanOrEqual(Vector left, Vector right) { @@ -455,6 +480,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector GreaterThanOrEqual(Vector left, Vector right) { @@ -468,6 +494,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector GreaterThanOrEqual(Vector left, Vector right) { @@ -481,6 +508,7 @@ namespace System.Numerics /// The first vector to compare. /// The second vector to compare. /// The resultant integral vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector GreaterThanOrEqual(Vector left, Vector right) { @@ -542,6 +570,7 @@ namespace System.Numerics /// /// The source vector. /// The absolute value vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector Abs(Vector value) where T : struct { @@ -554,6 +583,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The minimum vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector Min(Vector left, Vector right) where T : struct { @@ -566,6 +596,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The maximum vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector Max(Vector left, Vector right) where T : struct { @@ -580,10 +611,11 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The dot product. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static T Dot(Vector left, Vector right) where T : struct { - return Vector.DotProduct(left, right); + return Vector.Dot(left, right); } /// @@ -591,6 +623,7 @@ namespace System.Numerics /// /// The source vector. /// The square root vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector SquareRoot(Vector value) where T : struct { @@ -738,6 +771,7 @@ namespace System.Numerics /// The first source vector. /// The second source vector. /// The resultant vector. + [Intrinsic] [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public static Vector AndNot(Vector left, Vector right) where T : struct { -- 2.7.4