GenTree* fgOptimizeCast(GenTreeCast* cast);
GenTree* fgOptimizeEqualityComparisonWithConst(GenTreeOp* cmp);
GenTree* fgOptimizeRelationalComparisonWithConst(GenTreeOp* cmp);
+#ifdef FEATURE_HW_INTRINSICS
+ GenTree* fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node);
+#endif
GenTree* fgOptimizeCommutativeArithmetic(GenTreeOp* tree);
GenTree* fgOptimizeAddition(GenTreeOp* add);
GenTree* fgOptimizeMultiply(GenTreeOp* mul);
//
inline bool GenTree::IsFloatPositiveZero() const
{
- return IsCnsFltOrDbl() && !IsCnsNonZeroFltOrDbl();
+ if (IsCnsFltOrDbl())
+ {
+ // This implementation is almost identical to IsCnsNonZeroFltOrDbl
+ // but it is easier to parse out
+ // rather than using !IsCnsNonZeroFltOrDbl.
+ double constValue = AsDblCon()->gtDconVal;
+ return *(__int64*)&constValue == 0;
+ }
+
+ return false;
}
//-------------------------------------------------------------------
-// IsVectorZero: returns true if this is an integral or floating-point (SIMD or HW intrinsic) vector
-// with all its elements equal to zero.
+// IsVectorZero: returns true if this node is a HWIntrinsic that is Vector*_get_Zero.
//
// Returns:
-// True if this represents an integral or floating-point const (SIMD or HW intrinsic) vector with all its elements
-// equal to zero.
+// True if this represents a HWIntrinsic node that is Vector*_get_Zero.
//
// TODO: We already have IsSIMDZero() and IsIntegralConstVector(0),
// however, IsSIMDZero() does not cover hardware intrinsics, and IsIntegralConstVector(0) does not cover floating
// separate ones; preferably this one.
inline bool GenTree::IsVectorZero() const
{
-#ifdef FEATURE_SIMD
- if (gtOper == GT_SIMD)
- {
- const GenTreeSIMD* node = AsSIMD();
-
- if (node->GetSIMDIntrinsicId() == SIMDIntrinsicInit)
- {
- return (node->Op(1)->IsIntegralConst(0) || node->Op(1)->IsFloatPositiveZero());
- }
- }
-#endif
-
#ifdef FEATURE_HW_INTRINSICS
if (gtOper == GT_HWINTRINSIC)
{
- const GenTreeHWIntrinsic* node = AsHWIntrinsic();
- const var_types simdBaseType = node->GetSimdBaseType();
-
- if (varTypeIsIntegral(simdBaseType) || varTypeIsFloating(simdBaseType))
- {
- const NamedIntrinsic intrinsicId = node->GetHWIntrinsicId();
+ const GenTreeHWIntrinsic* node = AsHWIntrinsic();
+ const NamedIntrinsic intrinsicId = node->GetHWIntrinsicId();
- if (node->GetOperandCount() == 0)
- {
-#if defined(TARGET_XARCH)
- return (intrinsicId == NI_Vector128_get_Zero) || (intrinsicId == NI_Vector256_get_Zero);
-#elif defined(TARGET_ARM64)
- return (intrinsicId == NI_Vector64_get_Zero) || (intrinsicId == NI_Vector128_get_Zero);
-#endif // !TARGET_XARCH && !TARGET_ARM64
- }
- else if ((node->GetOperandCount() == 1) &&
- (node->Op(1)->IsIntegralConst(0) || node->Op(1)->IsFloatPositiveZero()))
- {
#if defined(TARGET_XARCH)
- return (intrinsicId == NI_Vector128_Create) || (intrinsicId == NI_Vector256_Create);
+ return (intrinsicId == NI_Vector128_get_Zero) || (intrinsicId == NI_Vector256_get_Zero);
#elif defined(TARGET_ARM64)
- return (intrinsicId == NI_Vector64_Create) || (intrinsicId == NI_Vector128_Create);
+ return (intrinsicId == NI_Vector64_get_Zero) || (intrinsicId == NI_Vector128_get_Zero);
#endif // !TARGET_XARCH && !TARGET_ARM64
- }
- }
}
#endif // FEATURE_HW_INTRINSICS
return cmp;
}
+#ifdef FEATURE_HW_INTRINSICS
+
+//------------------------------------------------------------------------
+// fgOptimizeHWIntrinsic: optimize a HW intrinsic node
+//
+// Arguments:
+// node - HWIntrinsic node to examine
+//
+// Returns:
+// The original node if no optimization happened or if tree bashing occured.
+// An alternative tree if an optimization happened.
+//
+// Notes:
+// Checks for HWIntrinsic nodes: Vector64.Create/Vector128.Create/Vector256.Create,
+// and if the call is one of these, attempt to optimize.
+// This is post-order, meaning that it will not morph the children.
+//
+GenTree* Compiler::fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node)
+{
+ assert(!optValnumCSE_phase);
+
+ if (opts.OptimizationDisabled())
+ {
+ return node;
+ }
+
+ switch (node->GetHWIntrinsicId())
+ {
+ case NI_Vector128_Create:
+#if defined(TARGET_XARCH)
+ case NI_Vector256_Create:
+#elif defined(TARGET_ARM64)
+ case NI_Vector64_Create:
+#endif
+ {
+ bool hwAllArgsAreConstZero = true;
+ for (GenTree* arg : node->Operands())
+ {
+ if (!arg->IsIntegralConst(0) && !arg->IsFloatPositiveZero())
+ {
+ hwAllArgsAreConstZero = false;
+ break;
+ }
+ }
+
+ if (hwAllArgsAreConstZero)
+ {
+ switch (node->GetHWIntrinsicId())
+ {
+ case NI_Vector128_Create:
+ {
+ node->ResetHWIntrinsicId(NI_Vector128_get_Zero);
+ break;
+ }
+#if defined(TARGET_XARCH)
+ case NI_Vector256_Create:
+ {
+ node->ResetHWIntrinsicId(NI_Vector256_get_Zero);
+ break;
+ }
+#elif defined(TARGET_ARM64)
+ case NI_Vector64_Create:
+ {
+ node->ResetHWIntrinsicId(NI_Vector64_get_Zero);
+ break;
+ }
+#endif
+ default:
+ unreached();
+ }
+ }
+ break;
+ }
+
+ default:
+ break;
+ }
+
+ return node;
+}
+
+#endif
+
//------------------------------------------------------------------------
// fgOptimizeCommutativeArithmetic: Optimizes commutative operations.
//
}
#endif // defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH)
+#ifdef FEATURE_HW_INTRINSICS
+ if (multiOp->OperIsHWIntrinsic() && !optValnumCSE_phase)
+ {
+ return fgOptimizeHWIntrinsic(multiOp->AsHWIntrinsic());
+ }
+#endif
+
return multiOp;
}
#endif // defined(FEATURE_SIMD) || defined(FEATURE_HW_INTRINSICS)
}
[MethodImpl(MethodImplOptions.NoInlining)]
+ static Vector64<int> AdvSimd_CompareEqual_Vector64_Int32_CreateZero(Vector64<int> left)
+ {
+ return AdvSimd.CompareEqual(left, Vector64.Create(0));
+ }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ static Vector64<int> AdvSimd_CompareEqual_Vector64_Int32_CreateZeroZero(Vector64<int> left)
+ {
+ return AdvSimd.CompareEqual(left, Vector64.Create(0, 0));
+ }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ static Vector64<float> AdvSimd_CompareEqual_Vector64_Single_CreateZero(Vector64<float> left)
+ {
+ return AdvSimd.CompareEqual(left, Vector64.Create(0f));
+ }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ static Vector64<float> AdvSimd_CompareEqual_Vector64_Single_CreateZeroZero(Vector64<float> left)
+ {
+ return AdvSimd.CompareEqual(left, Vector64.Create(0f, 0f));
+ }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
static Vector128<byte> AdvSimd_CompareEqual_Vector128_Byte_Zero(Vector128<byte> left)
{
return AdvSimd.CompareEqual(left, Vector128<byte>.Zero);
}
[MethodImpl(MethodImplOptions.NoInlining)]
+ static Vector128<int> AdvSimd_CompareEqual_Vector128_Int32_CreateZero(Vector128<int> left)
+ {
+ return AdvSimd.CompareEqual(left, Vector128.Create(0));
+ }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ static Vector128<int> AdvSimd_CompareEqual_Vector128_Int32_CreateZeroZeroZeroZero(Vector128<int> left)
+ {
+ return AdvSimd.CompareEqual(left, Vector128.Create(0, 0, 0, 0));
+ }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ static Vector128<float> AdvSimd_CompareEqual_Vector128_Single_CreateZero(Vector128<float> left)
+ {
+ return AdvSimd.CompareEqual(left, Vector128.Create(0f));
+ }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ static Vector128<float> AdvSimd_CompareEqual_Vector128_Single_CreateZeroZeroZeroZero(Vector128<float> left)
+ {
+ return AdvSimd.CompareEqual(left, Vector128.Create(0f, 0f, 0f, 0f));
+ }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ static Vector128<float> AdvSimd_CompareEqual_Vector128_Single_CreateZeroZeroZeroZero_AsVariable(Vector128<float> left)
+ {
+ var asVar = Vector128.Create(0f, 0f, 0f, 0f);
+ return AdvSimd.CompareEqual(left, asVar);
+ }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
static Vector128<double> AdvSimd_Arm64_CompareEqual_Vector128_Double_Zero(Vector128<double> left)
{
return AdvSimd.Arm64.CompareEqual(left, Vector128<double>.Zero);
// Begin CompareEqual Tests
+ // Vector64
+
if (!ValidateResult_Vector64<byte>(AdvSimd_CompareEqual_Vector64_Byte_Zero(Vector64<byte>.Zero), Byte.MaxValue))
result = -1;
if (!ValidateResult_Vector64<float>(AdvSimd_CompareEqual_Vector64_Single_Zero(Vector64<float>.Zero), Single.NaN))
result = -1;
+ // Vector64.Create
+
+ if (!ValidateResult_Vector64<int>(AdvSimd_CompareEqual_Vector64_Int32_CreateZero(Vector64<int>.Zero), -1))
+ result = -1;
+
+ if (!ValidateResult_Vector64<float>(AdvSimd_CompareEqual_Vector64_Single_CreateZero(Vector64<float>.Zero), Single.NaN))
+ result = -1;
+
+ if (!ValidateResult_Vector64<int>(AdvSimd_CompareEqual_Vector64_Int32_CreateZeroZero(Vector64<int>.Zero), -1))
+ result = -1;
+
+ if (!ValidateResult_Vector64<float>(AdvSimd_CompareEqual_Vector64_Single_CreateZeroZero(Vector64<float>.Zero), Single.NaN))
+ result = -1;
+
+ // Vector128
+
if (!ValidateResult_Vector128<byte>(AdvSimd_CompareEqual_Vector128_Byte_Zero(Vector128<byte>.Zero), Byte.MaxValue))
result = -1;
if (!ValidateResult_Vector128<float>(AdvSimd_CompareEqual_Vector128_Single_Zero(Vector128<float>.Zero), Single.NaN))
result = -1;
+ // Vector128.Create
+
+ if (!ValidateResult_Vector128<int>(AdvSimd_CompareEqual_Vector128_Int32_CreateZero(Vector128<int>.Zero), -1))
+ result = -1;
+
+ if (!ValidateResult_Vector128<float>(AdvSimd_CompareEqual_Vector128_Single_CreateZero(Vector128<float>.Zero), Single.NaN))
+ result = -1;
+
+ if (!ValidateResult_Vector128<int>(AdvSimd_CompareEqual_Vector128_Int32_CreateZeroZeroZeroZero(Vector128<int>.Zero), -1))
+ result = -1;
+
+ if (!ValidateResult_Vector128<float>(AdvSimd_CompareEqual_Vector128_Single_CreateZeroZeroZeroZero(Vector128<float>.Zero), Single.NaN))
+ result = -1;
+
+ if (!ValidateResult_Vector128<float>(AdvSimd_CompareEqual_Vector128_Single_CreateZeroZeroZeroZero_AsVariable(Vector128<float>.Zero), Single.NaN))
+ result = -1;
+
// End CompareEqual Tests
// Begin CompareGreaterThan Tests
// Begin CompareEqual Tests
+ // Vector64
+
if (!ValidateResult_Vector64<byte>(AdvSimd_CompareEqual_Vector64_Byte_Zero_Swapped(Vector64<byte>.Zero), Byte.MaxValue))
result = -1;
if (!ValidateResult_Vector64<float>(AdvSimd_CompareEqual_Vector64_Single_Zero_Swapped(Vector64<float>.Zero), Single.NaN))
result = -1;
+ // Vector128
+
if (!ValidateResult_Vector128<byte>(AdvSimd_CompareEqual_Vector128_Byte_Zero_Swapped(Vector128<byte>.Zero), Byte.MaxValue))
result = -1;
// Begin CompareEqual Tests
+ // Vector128
+
if (!ValidateResult_Vector128<double>(AdvSimd_Arm64_CompareEqual_Vector128_Double_Zero(Vector128<double>.Zero), Double.NaN))
result = -1;
if (!ValidateResult_Vector128<long>(AdvSimd_Arm64_CompareEqual_Vector128_Int64_Zero(Vector128<long>.Zero), -1))
result = -1;
+ // Vector64
+
if (!ValidateResult_Vector64<float>(AdvSimd_Arm64_CompareEqualScalar_Vector64_Single_Zero(Vector64<float>.Zero), Vector64.CreateScalar(Single.NaN)))
result = -1;
// Begin CompareEqual Tests
+ // Vector128
+
if (!ValidateResult_Vector128<double>(AdvSimd_Arm64_CompareEqual_Vector128_Double_Zero_Swapped(Vector128<double>.Zero), Double.NaN))
result = -1;
if (!ValidateResult_Vector128<long>(AdvSimd_Arm64_CompareEqual_Vector128_Int64_Zero_Swapped(Vector128<long>.Zero), -1))
result = -1;
+ // Vector64
+
if (!ValidateResult_Vector64<float>(AdvSimd_Arm64_CompareEqualScalar_Vector64_Single_Zero_Swapped(Vector64<float>.Zero), Vector64.CreateScalar(Single.NaN)))
result = -1;