assert(op2->TypeIs(simdType));
var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
-
- // We support the return type being a SIMD for floating-point as a special optimization
- assert(varTypeIsArithmetic(type) || (varTypeIsSIMD(type) && varTypeIsFloating(simdBaseType)));
+ assert(varTypeIsSIMD(type));
NamedIntrinsic intrinsic = NI_Illegal;
op2 = impSIMDPopStack();
op1 = impSIMDPopStack();
- retNode = gtNewSimdDotProdNode(retType, op1, op2, simdBaseJitType, simdSize);
+ retNode = gtNewSimdDotProdNode(simdType, op1, op2, simdBaseJitType, simdSize);
+ retNode = gtNewSimdGetElementNode(retType, retNode, gtNewIconNode(0), simdBaseJitType, simdSize);
}
break;
}
op2 = impSIMDPopStack();
op1 = impSIMDPopStack();
- retNode = gtNewSimdDotProdNode(retType, op1, op2, simdBaseJitType, simdSize);
+ retNode = gtNewSimdDotProdNode(simdType, op1, op2, simdBaseJitType, simdSize);
+ retNode = gtNewSimdGetElementNode(retType, retNode, gtNewIconNode(0), simdBaseJitType, simdSize);
break;
}
assert(varTypeIsSIMD(simdType));
assert(varTypeIsArithmetic(simdBaseType));
assert(simdSize != 0);
-
- // We support the return type being a SIMD for floating-point as a special optimization
- assert(varTypeIsArithmetic(node) || (varTypeIsSIMD(node) && varTypeIsFloating(simdBaseType)));
+ assert(varTypeIsSIMD(node));
GenTree* op1 = node->Op(1);
GenTree* op2 = node->Op(2);
// For 12 byte SIMD, we need to clear the upper 4 bytes:
// idx = CNS_INT int 0x03
- // tmp1 = * CNS_DLB float 0.0
+ // tmp1 = * CNS_DBL float 0.0
// /--* op1 simd16
// +--* idx int
// +--* tmp1 simd16
}
}
- if (varTypeIsSIMD(node->gtType))
- {
- // We're producing a vector result, so just return the result directly
-
- LIR::Use use;
-
- if (BlockRange().TryGetUse(node, &use))
- {
- use.ReplaceWith(tmp2);
- }
+ // We're producing a vector result, so just return the result directly
+ LIR::Use use;
- BlockRange().Remove(node);
- return tmp2->gtNext;
- }
- else
+ if (BlockRange().TryGetUse(node, &use))
{
- // We will be constructing the following parts:
- // ...
- // /--* tmp2 simd16
- // node = * HWINTRINSIC simd16 T ToScalar
-
- // This is roughly the following managed code:
- // ...
- // return tmp2.ToScalar();
-
- node->ResetHWIntrinsicId((simdSize == 8) ? NI_Vector64_ToScalar : NI_Vector128_ToScalar, tmp2);
- return LowerNode(node);
+ use.ReplaceWith(tmp2);
}
+
+ BlockRange().Remove(node);
+ return tmp2->gtNext;
}
#endif // FEATURE_HW_INTRINSICS
assert(varTypeIsSIMD(simdType));
assert(varTypeIsArithmetic(simdBaseType));
assert(simdSize != 0);
-
- // We support the return type being a SIMD for floating-point as a special optimization
- assert(varTypeIsArithmetic(node) || (varTypeIsSIMD(node) && varTypeIsFloating(simdBaseType)));
+ assert(varTypeIsSIMD(node));
GenTree* op1 = node->Op(1);
GenTree* op2 = node->Op(2);
// tmp2 = * HWINTRINSIC simd16 T GetUpper
// /--* tmp1 simd16
// +--* tmp2 simd16
- // tmp3 = * HWINTRINSIC simd16 T Add
- // /--* tmp3 simd16
- // node = * HWINTRINSIC simd16 T ToScalar
+ // node = * HWINTRINSIC simd16 T Add
// This is roughly the following managed code:
// var tmp1 = Avx.DotProduct(op1, op2, 0xFF);
// var tmp2 = tmp1.GetUpper();
- // var tmp3 = Sse.Add(tmp1, tmp2);
- // return tmp3.ToScalar();
+ // return Sse.Add(tmp1, tmp2);
idx = comp->gtNewIconNode(0xF1, TYP_INT);
BlockRange().InsertBefore(node, idx);
tmp2 = comp->gtNewSimdBinOpNode(GT_ADD, TYP_SIMD16, tmp3, tmp1, simdBaseJitType, 16);
BlockRange().InsertAfter(tmp1, tmp2);
- LowerNode(tmp2);
- node->SetSimdSize(16);
+ // We're producing a vector result, so just return the result directly
+ LIR::Use use;
- node->ResetHWIntrinsicId(NI_Vector128_ToScalar, tmp2);
+ if (BlockRange().TryGetUse(node, &use))
+ {
+ use.ReplaceWith(tmp2);
+ }
- return LowerNode(node);
+ BlockRange().Remove(node);
+ return LowerNode(tmp2);
}
case TYP_DOUBLE:
tmp1 = tmp2;
}
- if (varTypeIsSIMD(node->gtType))
- {
- // We're producing a vector result, so just return the result directly
-
- LIR::Use use;
-
- if (BlockRange().TryGetUse(node, &use))
- {
- use.ReplaceWith(tmp1);
- }
+ // We're producing a vector result, so just return the result directly
+ LIR::Use use;
- BlockRange().Remove(node);
- return tmp1->gtNext;
- }
- else
+ if (BlockRange().TryGetUse(node, &use))
{
- // We will be constructing the following parts:
- // ...
- // /--* tmp1 simd16
- // node = * HWINTRINSIC simd16 T ToScalar
-
- // This is roughly the following managed code:
- // ...
- // return tmp1.ToScalar();
-
- node->ResetHWIntrinsicId(NI_Vector128_ToScalar, tmp1);
- return LowerNode(node);
+ use.ReplaceWith(tmp1);
}
+
+ BlockRange().Remove(node);
+ return tmp1->gtNext;
}
//----------------------------------------------------------------------------------------------
#endif // TARGET_ARM64
case NI_Vector128_Create:
{
- // The `Dot` API returns a scalar. However, many common usages require it to
- // be then immediately broadcast back to a vector so that it can be used in
- // a subsequent operation. One of the most common is normalizing a vector
+ // The managed `Dot` API returns a scalar. However, many common usages require
+ // it to be then immediately broadcast back to a vector so that it can be used
+ // in a subsequent operation. One of the most common is normalizing a vector
// which is effectively `value / value.Length` where `Length` is
- // `Sqrt(Dot(value, value))`
+ // `Sqrt(Dot(value, value))`. Because of this, and because of how a lot of
+ // hardware works, we treat `NI_Vector_Dot` as returning a SIMD type and then
+ // also wrap it in `ToScalar` where required.
//
// In order to ensure that developers can still utilize this efficiently, we
- // will look for two common patterns:
+ // then look for four common patterns:
// * Create(Dot(..., ...))
// * Create(Sqrt(Dot(..., ...)))
+ // * Create(ToScalar(Dot(..., ...)))
+ // * Create(ToScalar(Sqrt(Dot(..., ...))))
//
- // When these exist, we'll avoid converting to a scalar at all and just
- // keep everything as a vector. However, we only do this for Vector64/Vector128
- // and only for float/double.
+ // When these exist, we'll avoid converting to a scalar and hence, avoid broadcasting
+ // the value back into a vector. Instead we'll just keep everything as a vector.
//
- // We don't do this for Vector256 since that is xarch only and doesn't trivially
- // support operations which cross the upper and lower 128-bit lanes
+ // We only do this for Vector64/Vector128 today. We could expand this more in
+ // the future but it would require additional hand handling for Vector256
+ // (since a 256-bit result requires more work). We do some integer handling
+ // when the value is trivially replicated to all elements without extra work.
if (node->GetOperandCount() != 1)
{
break;
}
- if (!varTypeIsFloating(node->GetSimdBaseType()))
- {
- break;
- }
-
- GenTree* op1 = node->Op(1);
- GenTree* sqrt = nullptr;
+ GenTree* op1 = node->Op(1);
+ GenTree* sqrt = nullptr;
+ GenTree* toScalar = nullptr;
if (op1->OperIs(GT_INTRINSIC))
{
+ if (!varTypeIsFloating(node->GetSimdBaseType()))
+ {
+ break;
+ }
+
if (op1->AsIntrinsic()->gtIntrinsicName != NI_System_Math_Sqrt)
{
break;
GenTreeHWIntrinsic* hwop1 = op1->AsHWIntrinsic();
#if defined(TARGET_ARM64)
+ if ((hwop1->GetHWIntrinsicId() == NI_Vector64_ToScalar) ||
+ (hwop1->GetHWIntrinsicId() == NI_Vector128_ToScalar))
+#else
+ if (hwop1->GetHWIntrinsicId() == NI_Vector128_ToScalar)
+#endif
+ {
+ op1 = hwop1->Op(1);
+
+ if (!op1->OperIs(GT_HWINTRINSIC))
+ {
+ break;
+ }
+
+ toScalar = hwop1;
+ hwop1 = op1->AsHWIntrinsic();
+ }
+
+#if defined(TARGET_ARM64)
if ((hwop1->GetHWIntrinsicId() != NI_Vector64_Dot) && (hwop1->GetHWIntrinsicId() != NI_Vector128_Dot))
#else
if (hwop1->GetHWIntrinsicId() != NI_Vector128_Dot)
break;
}
- unsigned simdSize = node->GetSimdSize();
- var_types simdType = getSIMDTypeForSize(simdSize);
-
- hwop1->gtType = simdType;
+ if (toScalar != nullptr)
+ {
+ DEBUG_DESTROY_NODE(toScalar);
+ }
if (sqrt != nullptr)
{
+ unsigned simdSize = node->GetSimdSize();
+ var_types simdType = getSIMDTypeForSize(simdSize);
+
node = gtNewSimdSqrtNode(simdType, hwop1, node->GetSimdBaseJitType(), simdSize)->AsHWIntrinsic();
DEBUG_DESTROY_NODE(sqrt);
}
vecCon->gtSimdVal.f32[3] = +1.0f;
GenTree* conjugate = gtNewSimdBinOpNode(GT_MUL, retType, op1, vecCon, simdBaseJitType, simdSize);
-
- op1 = gtNewSimdDotProdNode(retType, clonedOp1, clonedOp2, simdBaseJitType, simdSize);
+ op1 = gtNewSimdDotProdNode(retType, clonedOp1, clonedOp2, simdBaseJitType, simdSize);
return gtNewSimdBinOpNode(GT_DIV, retType, conjugate, op1, simdBaseJitType, simdSize);
}
op1 = impCloneExpr(op1, &clonedOp1, CHECK_SPILL_ALL,
nullptr DEBUGARG("Clone op1 for vector length squared"));
- return gtNewSimdDotProdNode(retType, op1, clonedOp1, simdBaseJitType, simdSize);
+ op1 = gtNewSimdDotProdNode(simdType, op1, clonedOp1, simdBaseJitType, simdSize);
+ return gtNewSimdGetElementNode(retType, op1, gtNewIconNode(0), simdBaseJitType, simdSize);
}
case NI_VectorT128_Load:
nullptr DEBUGARG("Clone op1 for vector normalize (2)"));
op1 = gtNewSimdDotProdNode(retType, op1, clonedOp1, simdBaseJitType, simdSize);
-
op1 = gtNewSimdSqrtNode(retType, op1, simdBaseJitType, simdSize);
return gtNewSimdBinOpNode(GT_DIV, retType, clonedOp2, op1, simdBaseJitType, simdSize);
op1 = impCloneExpr(op1, &clonedOp1, CHECK_SPILL_ALL,
nullptr DEBUGARG("Clone diff for vector distance squared"));
- return gtNewSimdDotProdNode(retType, op1, clonedOp1, simdBaseJitType, simdSize);
+ op1 = gtNewSimdDotProdNode(simdType, op1, clonedOp1, simdBaseJitType, simdSize);
+ return gtNewSimdGetElementNode(retType, op1, gtNewIconNode(0), simdBaseJitType, simdSize);
}
case NI_Quaternion_Divide:
case NI_VectorT256_Dot:
#endif // TARGET_XARCH
{
- return gtNewSimdDotProdNode(retType, op1, op2, simdBaseJitType, simdSize);
+ op1 = gtNewSimdDotProdNode(simdType, op1, op2, simdBaseJitType, simdSize);
+ return gtNewSimdGetElementNode(retType, op1, gtNewIconNode(0), simdBaseJitType, simdSize);
}
case NI_VectorT128_Equals:
projects / platform / upstream / dotnet / runtime.git / commitdiff