GenTreePtr gtNewOneConNode(var_types type);
+#ifdef FEATURE_SIMD
+ GenTreePtr gtNewSIMDVectorZero(var_types simdType, var_types baseType, unsigned size);
+ GenTreePtr gtNewSIMDVectorOne(var_types simdType, var_types baseType, unsigned size);
+#endif
+
GenTreeBlk* gtNewBlkOpNode(
genTreeOps oper, GenTreePtr dst, GenTreePtr srcOrFillVal, GenTreePtr sizeOrClsTok, bool isVolatile);
GenTree** op1,
GenTree** op2);
+ // Creates a GT_SIMD tree for Abs intrinsic.
+ GenTreePtr impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned simdVectorSize, GenTree* op1);
+
#if defined(_TARGET_XARCH_) && !defined(LEGACY_BACKEND)
// Transforms operands and returns the SIMD intrinsic to be applied on
// transformed operands to obtain == comparison result.
}
}
+#ifdef FEATURE_SIMD
+//---------------------------------------------------------------------
+// gtNewSIMDVectorZero: create a GT_SIMD node for Vector<T>.Zero
+//
+// Arguments:
+// simdType - simd vector type
+// baseType - element type of vector
+// size - size of vector in bytes
+GenTreePtr Compiler::gtNewSIMDVectorZero(var_types simdType, var_types baseType, unsigned size)
+{
+ baseType = genActualType(baseType);
+ GenTree* initVal = gtNewZeroConNode(baseType);
+ initVal->gtType = baseType;
+ return gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, baseType, size);
+}
+
+//---------------------------------------------------------------------
+// gtNewSIMDVectorOne: create a GT_SIMD node for Vector<T>.One
+//
+// Arguments:
+// simdType - simd vector type
+// baseType - element type of vector
+// size - size of vector in bytes
+GenTreePtr Compiler::gtNewSIMDVectorOne(var_types simdType, var_types baseType, unsigned size)
+{
+ GenTree* initVal;
+ if (varTypeIsSmallInt(baseType))
+ {
+ unsigned baseSize = genTypeSize(baseType);
+ int val;
+ if (baseSize == 1)
+ {
+ val = 0x01010101;
+ }
+ else
+ {
+ val = 0x00010001;
+ }
+ initVal = gtNewIconNode(val);
+ }
+ else
+ {
+ initVal = gtNewOneConNode(baseType);
+ }
+
+ baseType = genActualType(baseType);
+ initVal->gtType = baseType;
+ return gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, baseType, size);
+}
+#endif // FEATURE_SIMD
+
GenTreeCall* Compiler::gtNewIndCallNode(GenTreePtr addr, var_types type, GenTreeArgList* args, IL_OFFSETX ilOffset)
{
return gtNewCallNode(CT_INDIRECT, (CORINFO_METHOD_HANDLE)addr, type, args, ilOffset);
case GT_COMMA:
{
- // The second thing is the block node.
+ // The second thing could either be a block node or a GT_SIMD or a GT_COMMA node.
GenTree* blockNode = structVal->gtOp.gtOp2;
assert(blockNode->gtType == structType);
- // It had better be a block node - any others should not occur here.
- assert(blockNode->OperIsBlk());
-
- // Sink the GT_COMMA below the blockNode addr.
- GenTree* blockNodeAddr = blockNode->gtOp.gtOp1;
- assert(blockNodeAddr->gtType == TYP_BYREF);
- GenTree* commaNode = structVal;
- commaNode->gtType = TYP_BYREF;
- commaNode->gtOp.gtOp2 = blockNodeAddr;
- blockNode->gtOp.gtOp1 = commaNode;
- structVal = blockNode;
- alreadyNormalized = true;
+
+ // Is this GT_COMMA(op1, GT_COMMA())?
+ GenTree* parent = structVal;
+ if (blockNode->OperGet() == GT_COMMA)
+ {
+ // Find the last node in the comma chain.
+ do
+ {
+ assert(blockNode->gtType == structType);
+ parent = blockNode;
+ blockNode = blockNode->gtOp.gtOp2;
+ } while (blockNode->OperGet() == GT_COMMA);
+ }
+
+#ifdef FEATURE_SIMD
+ if (blockNode->OperGet() == GT_SIMD)
+ {
+ parent->gtOp.gtOp2 = impNormStructVal(blockNode, structHnd, curLevel, forceNormalization);
+ alreadyNormalized = true;
+ }
+ else
+#endif
+ {
+ assert(blockNode->OperIsBlk());
+
+ // Sink the GT_COMMA below the blockNode addr.
+ // That is GT_COMMA(op1, op2=blockNode) is tranformed into
+ // blockNode(GT_COMMA(TYP_BYREF, op1, op2's op1)).
+ //
+ // In case of a chained GT_COMMA case, we sink the last
+ // GT_COMMA below the blockNode addr.
+ GenTree* blockNodeAddr = blockNode->gtOp.gtOp1;
+ assert(blockNodeAddr->gtType == TYP_BYREF);
+ GenTree* commaNode = parent;
+ commaNode->gtType = TYP_BYREF;
+ commaNode->gtOp.gtOp2 = blockNodeAddr;
+ blockNode->gtOp.gtOp1 = commaNode;
+ if (parent == structVal)
+ {
+ structVal = blockNode;
+ }
+ alreadyNormalized = true;
+ }
}
break;
addr->gtType = simdType;
use.ReplaceWith(comp, addr);
}
-#if defined(_TARGET_X86_)
- // For x86, if we have GT_IND(GT_ADDR(GT_SIMD)), remove the GT_IND(GT_ADDR()), leaving just
- // the GT_SIMD.
else if ((addr->OperGet() == GT_ADDR) && (addr->gtGetOp1()->OperGet() == GT_SIMD))
{
+ // if we have GT_IND(GT_ADDR(GT_SIMD)), remove the GT_IND(GT_ADDR()), leaving just the GT_SIMD.
BlockRange().Remove(tree);
BlockRange().Remove(addr);
use.ReplaceWith(comp, addr->gtGetOp1());
}
-#endif // defined(_TARGET_X86_)
else if (!keepBlk)
{
tree->SetOper(GT_IND);
case GT_IND:
// Clear the `GTF_IND_ASG_LHS` flag, which overlaps with `GTF_IND_REQ_ADDR_IN_REG`.
node->gtFlags &= ~GTF_IND_ASG_LHS;
+
+ if (varTypeIsSIMD(node))
+ {
+ RewriteSIMDOperand(use, false);
+ }
break;
case GT_NOP:
#endif // !_TARGET_XARCH_
}
+//-------------------------------------------------------------------------
+// impSIMDAbs: creates GT_SIMD node to compute Abs value of a given vector.
+//
+// Arguments:
+// typeHnd - type handle of SIMD vector
+// baseType - base type of vector
+// size - vector size in bytes
+// op1 - operand of Abs intrinsic
+//
+GenTreePtr Compiler::impSIMDAbs(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1)
+{
+ assert(varTypeIsSIMD(op1));
+
+ var_types simdType = op1->TypeGet();
+ GenTreePtr retVal = nullptr;
+
+#ifdef _TARGET_XARCH_
+ // When there is no direct support, Abs(v) could be computed
+ // on integer vectors as follows:
+ // BitVector = v < vector.Zero
+ // result = ConditionalSelect(BitVector, vector.Zero - v, v)
+
+ bool useConditionalSelect = false;
+ if (getSIMDInstructionSet() == InstructionSet_SSE2)
+ {
+ // SSE2 doesn't support abs on signed integer type vectors.
+ if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
+ {
+ useConditionalSelect = true;
+ }
+ }
+ else
+ {
+ assert(getSIMDInstructionSet() >= InstructionSet_SSE3_4);
+ if (baseType == TYP_LONG)
+ {
+ // SSE3_4/AVX2 don't support abs on long type vector.
+ useConditionalSelect = true;
+ }
+ }
+
+ if (useConditionalSelect)
+ {
+ // This works only on integer vectors not on float/double vectors.
+ assert(varTypeIsIntegral(baseType));
+
+ GenTreePtr op1Assign;
+ unsigned op1LclNum;
+
+ if (op1->OperGet() == GT_LCL_VAR)
+ {
+ op1LclNum = op1->gtLclVarCommon.gtLclNum;
+ op1Assign = nullptr;
+ }
+ else
+ {
+ op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs op1"));
+ lvaSetStruct(op1LclNum, typeHnd, false);
+ op1Assign = gtNewTempAssign(op1LclNum, op1);
+ op1 = gtNewLclvNode(op1LclNum, op1->TypeGet());
+ }
+
+ // Assign Vector.Zero to a temp since it is needed more than once
+ GenTreePtr vecZero = gtNewSIMDVectorZero(simdType, baseType, size);
+ unsigned vecZeroLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs VecZero"));
+ lvaSetStruct(vecZeroLclNum, typeHnd, false);
+ GenTreePtr vecZeroAssign = gtNewTempAssign(vecZeroLclNum, vecZero);
+
+ // Construct BitVector = v < vector.Zero
+ GenTreePtr bitVecOp1 = op1;
+ GenTreePtr bitVecOp2 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
+ var_types relOpBaseType = baseType;
+ SIMDIntrinsicID relOpIntrinsic =
+ impSIMDRelOp(SIMDIntrinsicLessThan, typeHnd, size, &relOpBaseType, &bitVecOp1, &bitVecOp2);
+ GenTreePtr bitVec = gtNewSIMDNode(simdType, bitVecOp1, bitVecOp2, relOpIntrinsic, relOpBaseType, size);
+ unsigned bitVecLclNum = lvaGrabTemp(true DEBUGARG("SIMD Abs bitVec"));
+ lvaSetStruct(bitVecLclNum, typeHnd, false);
+ GenTreePtr bitVecAssign = gtNewTempAssign(bitVecLclNum, bitVec);
+ bitVec = gtNewLclvNode(bitVecLclNum, bitVec->TypeGet());
+
+ // Construct condSelectOp1 = vector.Zero - v
+ GenTreePtr subOp1 = gtNewLclvNode(vecZeroLclNum, vecZero->TypeGet());
+ GenTreePtr subOp2 = gtNewLclvNode(op1LclNum, op1->TypeGet());
+ GenTreePtr negVec = gtNewSIMDNode(simdType, subOp1, subOp2, SIMDIntrinsicSub, baseType, size);
+
+ // Construct ConditionalSelect(bitVec, vector.Zero - v, v)
+ GenTreePtr vec = gtNewLclvNode(op1LclNum, op1->TypeGet());
+ retVal = impSIMDSelect(typeHnd, baseType, size, bitVec, negVec, vec);
+
+ // Prepend bitVec assignment to retVal.
+ // retVal = (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
+ retVal = gtNewOperNode(GT_COMMA, simdType, bitVecAssign, retVal);
+
+ // Prepend vecZero assignment to retVal.
+ // retVal = (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
+ retVal = gtNewOperNode(GT_COMMA, simdType, vecZeroAssign, retVal);
+
+ // If op1 was assigned to a temp, prepend that to retVal.
+ if (op1Assign != nullptr)
+ {
+ // retVal = (v=op1), (tmp1 = vector.Zero), (tmp2 = v < tmp1), CondSelect(tmp2, tmp1 - v, v)
+ retVal = gtNewOperNode(GT_COMMA, simdType, op1Assign, retVal);
+ }
+ }
+ else if (varTypeIsFloating(baseType))
+ {
+ // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
+ // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
+ GenTree* bitMask = nullptr;
+ if (baseType == TYP_FLOAT)
+ {
+ float f;
+ static_assert_no_msg(sizeof(float) == sizeof(int));
+ *((int*)&f) = 0x7fffffff;
+ bitMask = gtNewDconNode(f);
+ }
+ else if (baseType == TYP_DOUBLE)
+ {
+ double d;
+ static_assert_no_msg(sizeof(double) == sizeof(__int64));
+ *((__int64*)&d) = 0x7fffffffffffffffLL;
+ bitMask = gtNewDconNode(d);
+ }
+
+ assert(bitMask != nullptr);
+ bitMask->gtType = baseType;
+ GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
+ retVal = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
+ }
+ else if (baseType == TYP_CHAR || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
+ {
+ // Abs is a no-op on unsigned integer type vectors
+ retVal = op1;
+ }
+ else
+ {
+ assert(getSIMDInstructionSet() >= InstructionSet_SSE3_4);
+ assert(baseType != TYP_LONG);
+
+ retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
+ }
+#else // !_TARGET_XARCH_
+ assert(!"Abs intrinsic on non-xarch target not implemented");
+#endif // !_TARGET_XARCH_
+
+ return retVal;
+}
+
// Creates a GT_SIMD tree for Select operation
//
// Argumens:
break;
case SIMDIntrinsicGetZero:
- {
- baseType = genActualType(baseType);
- GenTree* initVal = gtNewZeroConNode(baseType);
- initVal->gtType = baseType;
- simdTree = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, baseType, size);
- retVal = simdTree;
- }
- break;
+ retVal = gtNewSIMDVectorZero(simdType, baseType, size);
+ break;
case SIMDIntrinsicGetOne:
- {
- GenTree* initVal;
- if (varTypeIsSmallInt(baseType))
- {
- unsigned baseSize = genTypeSize(baseType);
- int val;
- if (baseSize == 1)
- {
- val = 0x01010101;
- }
- else
- {
- val = 0x00010001;
- }
- initVal = gtNewIconNode(val);
- }
- else
- {
- initVal = gtNewOneConNode(baseType);
- }
-
- baseType = genActualType(baseType);
- initVal->gtType = baseType;
- simdTree = gtNewSIMDNode(simdType, initVal, nullptr, SIMDIntrinsicInit, baseType, size);
- retVal = simdTree;
- }
- break;
+ retVal = gtNewSIMDVectorOne(simdType, baseType, size);
+ break;
case SIMDIntrinsicGetAllOnes:
{
break;
case SIMDIntrinsicAbs:
- {
-#ifdef _TARGET_XARCH_
- // TODO-CQ-XARCH: we should be able to use conditional select
- // to compute abs(v) as follows when there is no direct support:
- // BitVector = v < vector.Zero
- // result = ConditionalSelect(BitVector, vector.Zero - v, v)
- if (getSIMDInstructionSet() == InstructionSet_SSE2)
- {
- // SSE2 doesn't support abs on signed integer type vectors.
- if (baseType == TYP_LONG || baseType == TYP_INT || baseType == TYP_SHORT || baseType == TYP_BYTE)
- {
- return nullptr;
- }
- }
- else
- {
- assert(getSIMDInstructionSet() >= InstructionSet_SSE3_4);
- if (baseType == TYP_LONG)
- {
- // SSE3_4/AVX2 don't support abs on long type vector.
- return nullptr;
- }
- }
-
- op1 = impSIMDPopStack(simdType);
-
- if (varTypeIsFloating(baseType))
- {
- // Abs(vf) = vf & new SIMDVector<float>(0x7fffffff);
- // Abs(vd) = vf & new SIMDVector<double>(0x7fffffffffffffff);
- GenTree* bitMask = nullptr;
- if (baseType == TYP_FLOAT)
- {
- float f;
- static_assert_no_msg(sizeof(float) == sizeof(int));
- *((int*)&f) = 0x7fffffff;
- bitMask = gtNewDconNode(f);
- }
- else if (baseType == TYP_DOUBLE)
- {
- double d;
- static_assert_no_msg(sizeof(double) == sizeof(__int64));
- *((__int64*)&d) = 0x7fffffffffffffffLL;
- bitMask = gtNewDconNode(d);
- }
-
- assert(bitMask != nullptr);
- bitMask->gtType = baseType;
- GenTree* bitMaskVector = gtNewSIMDNode(simdType, bitMask, SIMDIntrinsicInit, baseType, size);
- retVal = gtNewSIMDNode(simdType, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size);
- }
- else if (baseType == TYP_CHAR || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG)
- {
- // Abs is a no-op on unsigned integer type vectors
- retVal = op1;
- }
- else
- {
- assert(getSIMDInstructionSet() >= InstructionSet_SSE3_4);
- assert(baseType != TYP_LONG);
-
- retVal = gtNewSIMDNode(simdType, op1, SIMDIntrinsicAbs, baseType, size);
- }
-
-#else // !_TARGET_XARCH_
- assert(!"Abs intrinsic on non-xarch target not implemented");
- unreached();
-#endif // !_TARGET_XARCH_
- }
- break;
+ op1 = impSIMDPopStack(simdType);
+ retVal = impSIMDAbs(clsHnd, baseType, size, op1);
+ break;
case SIMDIntrinsicGetW:
retVal = impSIMDGetFixed(simdType, baseType, size, 3);
SIMD_INTRINSIC("op_Multiply", false, Mul, "*", TYP_STRUCT, 2, {TYP_STRUCT, TYP_STRUCT, TYP_UNDEF}, {TYP_INT, TYP_FLOAT, TYP_DOUBLE, TYP_SHORT,TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF})
SIMD_INTRINSIC("op_Division", false, Div, "/", TYP_STRUCT, 2, {TYP_STRUCT, TYP_STRUCT, TYP_UNDEF}, {TYP_FLOAT, TYP_DOUBLE, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF})
-// Abs and SquareRoot are recognized as intrinsics only in case of float or double vectors
-SIMD_INTRINSIC("Abs", false, Abs, "abs", TYP_STRUCT, 1, {TYP_STRUCT, TYP_UNDEF, TYP_UNDEF}, {TYP_FLOAT, TYP_DOUBLE, TYP_CHAR, TYP_UBYTE, TYP_UINT, TYP_ULONG, TYP_INT, TYP_SHORT, TYP_BYTE, TYP_UNDEF})
+// SquareRoot is recognized as an intrinsic only for float or double vectors
SIMD_INTRINSIC("SquareRoot", false, Sqrt, "sqrt", TYP_STRUCT, 1, {TYP_STRUCT, TYP_UNDEF, TYP_UNDEF}, {TYP_FLOAT, TYP_DOUBLE, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF, TYP_UNDEF})
-// Min and max methods are recognized as intrinsics only in case of float or double vectors
SIMD_INTRINSIC("Min", false, Min, "min", TYP_STRUCT, 2, {TYP_STRUCT, TYP_STRUCT, TYP_UNDEF}, {TYP_INT, TYP_FLOAT, TYP_DOUBLE, TYP_LONG, TYP_CHAR, TYP_UBYTE, TYP_BYTE, TYP_SHORT, TYP_UINT, TYP_ULONG})
SIMD_INTRINSIC("Max", false, Max, "max", TYP_STRUCT, 2, {TYP_STRUCT, TYP_STRUCT, TYP_UNDEF}, {TYP_INT, TYP_FLOAT, TYP_DOUBLE, TYP_LONG, TYP_CHAR, TYP_UBYTE, TYP_BYTE, TYP_SHORT, TYP_UINT, TYP_ULONG})
+SIMD_INTRINSIC("Abs", false, Abs, "abs", TYP_STRUCT, 1, {TYP_STRUCT, TYP_UNDEF, TYP_UNDEF }, {TYP_INT, TYP_FLOAT, TYP_DOUBLE, TYP_LONG, TYP_CHAR, TYP_UBYTE, TYP_BYTE, TYP_SHORT, TYP_UINT, TYP_ULONG})
// Vector Relational operators
SIMD_INTRINSIC("Equals", false, Equal, "eq", TYP_STRUCT, 2, {TYP_STRUCT, TYP_STRUCT, TYP_UNDEF}, {TYP_INT, TYP_FLOAT, TYP_DOUBLE, TYP_LONG, TYP_CHAR, TYP_UBYTE, TYP_BYTE, TYP_SHORT, TYP_UINT, TYP_ULONG})
JitLog jitLog = new JitLog();
if (!jitLog.Check("Abs", "Single")) returnVal = Fail;
if (!jitLog.Check("Abs", "Double")) returnVal = Fail;
- // SSE2: Abs is not an intrinsic for Int32 and Int64, but IS for UInt32 and UInt64
- // SSE3_4: Abs is not an intrinsic for Int64 alone.
- // Since right now there is no way to know SIMD instruction set used by JIT,
- // we will check conservatively on SSE3_4 targets.
+ if (!jitLog.Check("Abs", "Int64")) returnVal = Fail;
+ if (!jitLog.Check("Abs", "Int32")) returnVal = Fail;
+ if (!jitLog.Check("Abs", "Int16")) returnVal = Fail;
+ if (!jitLog.Check("Abs", "SByte")) returnVal = Fail;
if (!jitLog.Check("System.Numerics.Vector4:Abs")) returnVal = Fail;
if (!jitLog.Check("System.Numerics.Vector3:Abs")) returnVal = Fail;
if (!jitLog.Check("System.Numerics.Vector2:Abs")) returnVal = Fail;
if (!jitLog.Check("Abs", "UInt32")) returnVal = Fail;
if (!jitLog.Check("Abs", "UInt64")) returnVal = Fail;
- // AVX: Abs is not an intrinsic for Int64 alone.
- if (Vector<int>.Count == 8)
- {
- if (!jitLog.Check("Abs", "Int32")) returnVal = Fail;
- if (!jitLog.Check("Abs", "Int16")) returnVal = Fail;
- if (!jitLog.Check("Abs", "SByte")) returnVal = Fail;
- }
jitLog.Dispose();
return returnVal;
projects / platform / upstream / dotnet / runtime.git / commitdiff