void genCkfinite(GenTree* treeNode);
void genCodeForCompare(GenTreeOp* tree);
#ifdef TARGET_ARM64
- void genCodeForConditionalCompare(GenTreeOp* tree, GenCondition prevCond);
- void genCodeForContainedCompareChain(GenTree* tree, bool* inchain, GenCondition* prevCond);
+ void genCodeForCCMP(GenTreeCCMP* ccmp);
#endif
void genCodeForSelect(GenTreeOp* select);
void genIntrinsic(GenTreeIntrinsic* treeNode);
#endif // TARGET_XARCH
#if defined(TARGET_ARM64)
- static insCflags InsCflagsForCcmp(GenCondition cond);
static insCond JumpKindToInsCond(emitJumpKind condition);
#elif defined(TARGET_XARCH)
static instruction JumpKindToCmov(emitJumpKind condition);
return;
}
- if (tree->isContainedCompareChainSegment(op2))
- {
- GenCondition cond;
- bool chain = false;
-
- JITDUMP("Generating compare chain:\n");
- if (op1->isContained())
- {
- // Generate Op1 into flags.
- genCodeForContainedCompareChain(op1, &chain, &cond);
- assert(chain);
- }
- else
- {
- // Op1 is not contained, move it from a register into flags.
- emit->emitIns_R_I(INS_cmp, emitActualTypeSize(op1), op1->GetRegNum(), 0);
- cond = GenCondition::NE;
- chain = true;
- }
- // Gen Op2 into flags.
- genCodeForContainedCompareChain(op2, &chain, &cond);
- assert(chain);
-
- // Move the result from flags into a register.
- inst_SETCC(cond, tree->TypeGet(), targetReg);
- genProduceReg(tree);
- return;
- }
-
instruction ins = genGetInsForOper(tree->OperGet(), targetType);
if ((tree->gtFlags & GTF_SET_FLAGS) != 0)
// tree - a compare node (GT_EQ etc)
// cond - the condition of the previous generated compare.
//
-void CodeGen::genCodeForConditionalCompare(GenTreeOp* tree, GenCondition prevCond)
+void CodeGen::genCodeForCCMP(GenTreeCCMP* ccmp)
{
emitter* emit = GetEmitter();
- GenTree* op1 = tree->gtGetOp1();
- GenTree* op2 = tree->gtGetOp2();
- var_types op1Type = genActualType(op1->TypeGet());
- var_types op2Type = genActualType(op2->TypeGet());
- emitAttr cmpSize = EA_ATTR(genTypeSize(op1Type));
- regNumber targetReg = tree->GetRegNum();
- regNumber srcReg1 = op1->GetRegNum();
+ genConsumeOperands(ccmp);
+ GenTree* op1 = ccmp->gtGetOp1();
+ GenTree* op2 = ccmp->gtGetOp2();
+ var_types op1Type = genActualType(op1->TypeGet());
+ var_types op2Type = genActualType(op2->TypeGet());
+ emitAttr cmpSize = emitActualTypeSize(op1Type);
+ regNumber srcReg1 = op1->GetRegNum();
// No float support or swapping op1 and op2 to generate cmp reg, imm.
assert(!varTypeIsFloating(op2Type));
assert(!op1->isContainedIntOrIImmed());
- // Should only be called on contained nodes.
- assert(targetReg == REG_NA);
-
- // Should not be called for test conditionals (Arm64 does not have a ctst).
- assert(tree->OperIsCmpCompare());
-
// For the ccmp flags, invert the condition of the compare.
- insCflags cflags = InsCflagsForCcmp(GenCondition::FromRelop(tree));
-
// For the condition, use the previous compare.
- const GenConditionDesc& prevDesc = GenConditionDesc::Get(prevCond);
- insCond prevInsCond = JumpKindToInsCond(prevDesc.jumpKind1);
+ const GenConditionDesc& condDesc = GenConditionDesc::Get(ccmp->gtCondition);
+ insCond insCond = JumpKindToInsCond(condDesc.jumpKind1);
if (op2->isContainedIntOrIImmed())
{
GenTreeIntConCommon* intConst = op2->AsIntConCommon();
- emit->emitIns_R_I_FLAGS_COND(INS_ccmp, cmpSize, srcReg1, (int)intConst->IconValue(), cflags, prevInsCond);
+ emit->emitIns_R_I_FLAGS_COND(INS_ccmp, cmpSize, srcReg1, (int)intConst->IconValue(), ccmp->gtFlagsVal, insCond);
}
else
{
regNumber srcReg2 = op2->GetRegNum();
- emit->emitIns_R_R_FLAGS_COND(INS_ccmp, cmpSize, srcReg1, srcReg2, cflags, prevInsCond);
- }
-}
-
-//------------------------------------------------------------------------
-// genCodeForContainedCompareChain: Produce code for a chain of conditional compares.
-//
-// Only generates for contained nodes. Nodes that are not contained are assumed to be
-// generated as part of standard tree generation.
-//
-// Arguments:
-// tree - the node. Either a compare or a tree of compares connected by ANDs.
-// inChain - whether a contained chain is in progress.
-// prevCond - If a chain is in progress, the condition of the previous compare.
-// Return:
-// The last compare node generated.
-//
-void CodeGen::genCodeForContainedCompareChain(GenTree* tree, bool* inChain, GenCondition* prevCond)
-{
- assert(tree->isContained());
-
- if (tree->OperIs(GT_AND))
- {
- GenTree* op1 = tree->gtGetOp1();
- GenTree* op2 = tree->gtGetOp2();
-
- assert(op2->isContained());
-
- // If Op1 is contained, generate into flags. Otherwise, move the result into flags.
- if (op1->isContained())
- {
- genCodeForContainedCompareChain(op1, inChain, prevCond);
- assert(*inChain);
- }
- else
- {
- emitter* emit = GetEmitter();
- emit->emitIns_R_I(INS_cmp, emitActualTypeSize(op1), op1->GetRegNum(), 0);
- *prevCond = GenCondition::NE;
- *inChain = true;
- }
-
- // Generate Op2 based on Op1.
- genCodeForContainedCompareChain(op2, inChain, prevCond);
- assert(*inChain);
- }
- else
- {
- assert(tree->OperIsCmpCompare());
-
- // Generate the compare, putting the result in the flags register.
- if (!*inChain)
- {
- // First item in a chain. Use a standard compare.
- genCodeForCompare(tree->AsOp());
- }
- else
- {
- // Within the chain. Use a conditional compare (which is
- // dependent on the previous emitted compare).
- genCodeForConditionalCompare(tree->AsOp(), *prevCond);
- }
-
- *inChain = true;
- *prevCond = GenCondition::FromRelop(tree);
+ emit->emitIns_R_R_FLAGS_COND(INS_ccmp, cmpSize, srcReg1, srcReg2, ccmp->gtFlagsVal, insCond);
}
}
//
void CodeGen::genCodeForSelect(GenTreeOp* tree)
{
- assert(tree->OperIs(GT_SELECT));
- GenTreeConditional* select = tree->AsConditional();
- emitter* emit = GetEmitter();
+ assert(tree->OperIs(GT_SELECT, GT_SELECTCC));
+ GenTree* opcond = nullptr;
+ if (tree->OperIs(GT_SELECT))
+ {
+ opcond = tree->AsConditional()->gtCond;
+ genConsumeRegs(opcond);
+ }
- GenTree* opcond = select->gtCond;
- GenTree* op1 = select->gtOp1;
- GenTree* op2 = select->gtOp2;
- var_types op1Type = genActualType(op1->TypeGet());
- var_types op2Type = genActualType(op2->TypeGet());
- emitAttr attr = emitActualTypeSize(select->TypeGet());
+ emitter* emit = GetEmitter();
+
+ GenTree* op1 = tree->gtOp1;
+ GenTree* op2 = tree->gtOp2;
+ var_types op1Type = genActualType(op1);
+ var_types op2Type = genActualType(op2);
+ emitAttr attr = emitActualTypeSize(tree);
assert(!op1->isUsedFromMemory());
assert(genTypeSize(op1Type) == genTypeSize(op2Type));
- GenCondition prevCond;
- genConsumeRegs(opcond);
- if (opcond->isContained())
+ GenCondition cond;
+
+ if (opcond != nullptr)
{
- // Generate the contained condition.
- if (opcond->OperIsCompare())
- {
- genCodeForCompare(opcond->AsOp());
- prevCond = GenCondition::FromRelop(opcond);
- }
- else
- {
- // Condition is a compare chain. Try to contain it.
- assert(opcond->OperIs(GT_AND));
- bool chain = false;
- JITDUMP("Generating compare chain:\n");
- genCodeForContainedCompareChain(opcond, &chain, &prevCond);
- assert(chain);
- }
+ // Condition has been generated into a register - move it into flags.
+ emit->emitIns_R_I(INS_cmp, emitActualTypeSize(opcond), opcond->GetRegNum(), 0);
+ cond = GenCondition::NE;
}
else
{
- // Condition has been generated into a register - move it into flags.
- emit->emitIns_R_I(INS_cmp, emitActualTypeSize(opcond), opcond->GetRegNum(), 0);
- prevCond = GenCondition::NE;
+ assert(tree->OperIs(GT_SELECTCC));
+ cond = tree->AsOpCC()->gtCondition;
}
assert(!op1->isContained() || op1->IsIntegralConst(0));
regNumber targetReg = tree->GetRegNum();
regNumber srcReg1 = op1->IsIntegralConst(0) ? REG_ZR : genConsumeReg(op1);
regNumber srcReg2 = op2->IsIntegralConst(0) ? REG_ZR : genConsumeReg(op2);
- const GenConditionDesc& prevDesc = GenConditionDesc::Get(prevCond);
+ const GenConditionDesc& prevDesc = GenConditionDesc::Get(cond);
emit->emitIns_R_R_R_COND(INS_csel, attr, targetReg, srcReg1, srcReg2, JumpKindToInsCond(prevDesc.jumpKind1));
genProduceReg(tree);
}
-//------------------------------------------------------------------------
-// InsCflagsForCcmp: Get the Cflags for a required for a CCMP instruction.
-//
-// Consider:
-// cmp w, x
-// ccmp y, z, A, COND
-// This is: compare w and x, if this matches condition COND, then compare y and z.
-// Otherwise set flags to A - this should match the case where cmp failed.
-// Given COND, this function returns A.
-//
-// Arguments:
-// cond - the GenCondition.
-//
-insCflags CodeGen::InsCflagsForCcmp(GenCondition cond)
-{
- GenCondition inverted = GenCondition::Reverse(cond);
- switch (inverted.GetCode())
- {
- case GenCondition::EQ:
- return INS_FLAGS_Z;
- case GenCondition::NE:
- return INS_FLAGS_NONE;
- case GenCondition::SGE:
- return INS_FLAGS_Z;
- case GenCondition::SGT:
- return INS_FLAGS_NONE;
- case GenCondition::SLT:
- return INS_FLAGS_NC;
- case GenCondition::SLE:
- return INS_FLAGS_NZC;
- case GenCondition::UGE:
- return INS_FLAGS_C;
- case GenCondition::UGT:
- return INS_FLAGS_C;
- case GenCondition::ULT:
- return INS_FLAGS_NONE;
- case GenCondition::ULE:
- return INS_FLAGS_Z;
- default:
- NO_WAY("unexpected condition type");
- return INS_FLAGS_NONE;
- }
-}
-
//------------------------------------------------------------------------
// JumpKindToInsCond: Convert a Jump Kind to a condition.
//
case GT_SELECT:
genCodeForSelect(treeNode->AsConditional());
break;
+
+ case GT_SELECTCC:
+ genCodeForSelect(treeNode->AsOp());
+ break;
#endif
#ifdef TARGET_ARM64
case GT_JCMP:
genCodeForJumpCompare(treeNode->AsOp());
break;
+
+ case GT_CCMP:
+ genCodeForCCMP(treeNode->AsCCMP());
+ break;
#endif // TARGET_ARM64
case GT_JCC:
static_assert_no_msg(sizeof(GenTreeLclFld) <= TREE_NODE_SZ_SMALL);
static_assert_no_msg(sizeof(GenTreeCC) <= TREE_NODE_SZ_SMALL);
static_assert_no_msg(sizeof(GenTreeOpCC) <= TREE_NODE_SZ_SMALL);
+#ifdef TARGET_ARM64
+ static_assert_no_msg(sizeof(GenTreeCCMP) <= TREE_NODE_SZ_SMALL);
+#endif
+ static_assert_no_msg(sizeof(GenTreeConditional) <= TREE_NODE_SZ_SMALL);
static_assert_no_msg(sizeof(GenTreeCast) <= TREE_NODE_SZ_LARGE); // *** large node
static_assert_no_msg(sizeof(GenTreeBox) <= TREE_NODE_SZ_LARGE); // *** large node
static_assert_no_msg(sizeof(GenTreeField) <= TREE_NODE_SZ_LARGE); // *** large node
}
}
+#if defined(DEBUG) && defined(TARGET_ARM64)
+static const char* InsCflagsToString(insCflags flags)
+{
+ const static char* s_table[16] = {"0", "v", "c", "cv", "z", "zv", "zc", "zcv",
+ "n", "nv", "nc", "ncv", "nz", "nzv", "nzc", "nzcv"};
+ unsigned index = (unsigned)flags;
+ assert((0 <= index) && (index < ArrLen(s_table)));
+ return s_table[index];
+}
+#endif
+
//------------------------------------------------------------------------
// gtDispSsaName: Display the SSA use/def for a given local.
//
{
printf(" cond=%s", tree->AsOpCC()->gtCondition.Name());
}
+#ifdef TARGET_ARM64
+ else if (tree->OperIs(GT_CCMP))
+ {
+ printf(" cond=%s flags=%s", tree->AsCCMP()->gtCondition.Name(),
+ InsCflagsToString(tree->AsCCMP()->gtFlagsVal));
+ }
+#endif
gtDispCommonEndLine(tree);
}
#endif
#elif defined(TARGET_ARM64)
- if (OperIs(GT_AND, GT_ADD, GT_SUB))
+ if (OperIs(GT_AND))
+ {
+ return true;
+ }
+
+ // We do not support setting zero flag for madd/msub.
+ if (OperIs(GT_ADD, GT_SUB) && (!gtGetOp2()->OperIs(GT_MUL) || !gtGetOp2()->isContained()))
{
return true;
}
return isContained() && IsCnsIntOrI() && !isUsedFromSpillTemp();
}
- // Node and its child in isolation form a contained compare chain.
- bool isContainedCompareChainSegment(GenTree* child) const
- {
- return (OperIs(GT_AND) && child->isContained() && (child->OperIs(GT_AND) || child->OperIsCmpCompare()));
- }
-
bool isContainedFltOrDblImmed() const
{
return isContained() && OperIs(GT_CNS_DBL);
{
#if !defined(TARGET_64BIT)
if (OperIs(GT_ADD_HI, GT_SUB_HI))
+ {
+ return true;
+ }
+#endif
+#if defined(TARGET_ARM64)
+ if (OperIs(GT_CCMP))
+ {
return true;
+ }
#endif
return OperIs(GT_JCC, GT_SETCC, GT_SELECTCC);
}
};
// Represents a node with two operands and a condition.
-struct GenTreeOpCC final : public GenTreeOp
+struct GenTreeOpCC : public GenTreeOp
{
GenCondition gtCondition;
#endif // DEBUGGABLE_GENTREE
};
+#ifdef TARGET_ARM64
+enum insCflags : unsigned
+{
+ INS_FLAGS_NONE,
+ INS_FLAGS_V,
+ INS_FLAGS_C,
+ INS_FLAGS_CV,
+
+ INS_FLAGS_Z,
+ INS_FLAGS_ZV,
+ INS_FLAGS_ZC,
+ INS_FLAGS_ZCV,
+
+ INS_FLAGS_N,
+ INS_FLAGS_NV,
+ INS_FLAGS_NC,
+ INS_FLAGS_NCV,
+
+ INS_FLAGS_NZ,
+ INS_FLAGS_NZV,
+ INS_FLAGS_NZC,
+ INS_FLAGS_NZCV,
+};
+
+struct GenTreeCCMP final : public GenTreeOpCC
+{
+ insCflags gtFlagsVal;
+
+ GenTreeCCMP(var_types type, GenCondition condition, GenTree* op1, GenTree* op2, insCflags flagsVal)
+ : GenTreeOpCC(GT_CCMP, type, condition, op1, op2), gtFlagsVal(flagsVal)
+ {
+ }
+
+#if DEBUGGABLE_GENTREE
+ GenTreeCCMP() : GenTreeOpCC()
+ {
+ }
+#endif // DEBUGGABLE_GENTREE
+};
+#endif
+
//------------------------------------------------------------------------
// Deferred inline functions of GenTree -- these need the subtypes above to
// be defined already.
// Checks the condition flags and produces 1 if the condition specified by GenTreeCC::gtCondition is true and 0 otherwise.
GTNODE(SETCC , GenTreeCC ,0,GTK_LEAF|DBK_NOTHIR)
// Variant of SELECT that reuses flags computed by a previous node with the specified condition.
-GTNODE(SELECTCC , GenTreeCC ,0,GTK_BINOP|DBK_NOTHIR)
+GTNODE(SELECTCC , GenTreeOpCC ,0,GTK_BINOP|DBK_NOTHIR)
+#ifdef TARGET_ARM64
+// The arm64 ccmp instruction. If the specified condition is true, compares two
+// operands and sets the condition flags according to the result. Otherwise
+// sets the condition flags to the specified immediate value.
+GTNODE(CCMP , GenTreeCCMP ,0,GTK_BINOP|GTK_NOVALUE|DBK_NOTHIR)
+#endif
//-----------------------------------------------------------------------------
GTSTRUCT_1(ArrAddr , GT_ARR_ADDR)
GTSTRUCT_2(CC , GT_JCC, GT_SETCC)
GTSTRUCT_1(OpCC , GT_SELECTCC)
+#ifdef TARGET_ARM64
+GTSTRUCT_1(CCMP , GT_CCMP)
+#endif
#if defined(TARGET_X86)
GTSTRUCT_1(MultiRegOp , GT_MUL_LONG)
#elif defined (TARGET_ARM)
INS_COND_LE,
};
-enum insCflags : unsigned
-{
- INS_FLAGS_NONE,
- INS_FLAGS_V,
- INS_FLAGS_C,
- INS_FLAGS_CV,
-
- INS_FLAGS_Z,
- INS_FLAGS_ZV,
- INS_FLAGS_ZC,
- INS_FLAGS_ZCV,
-
- INS_FLAGS_N,
- INS_FLAGS_NV,
- INS_FLAGS_NC,
- INS_FLAGS_NCV,
-
- INS_FLAGS_NZ,
- INS_FLAGS_NZV,
- INS_FLAGS_NZC,
- INS_FLAGS_NZCV,
-};
-
enum insBarrier : unsigned
{
INS_BARRIER_OSHLD = 1,
return true;
}
+//------------------------------------------------------------------------
+// IsRangeInvariantInRange: Check if a range of nodes are invariant in the
+// specified range.
+//
+// Arguments:
+// rangeStart - The first node.
+// rangeEnd - The last node.
+// endExclusive - The exclusive end of the range to check invariance for.
+// ignoreNode - A node to ignore interference checks with, for example
+// because it will retain its relative order with 'node'.
+//
+// Returns:
+// True if the range can be evaluated at any point between its current location
+// and 'endExclusive' without giving a different result; otherwise false.
+//
+// Remarks:
+// Note that the range is treated as a unit and no pairwise interference
+// checks between nodes in the range are performed.
+//
+bool Lowering::IsRangeInvariantInRange(GenTree* rangeStart,
+ GenTree* rangeEnd,
+ GenTree* endExclusive,
+ GenTree* ignoreNode) const
+{
+ assert((rangeStart != nullptr) && (rangeEnd != nullptr));
+
+ if ((rangeEnd->gtNext == endExclusive) ||
+ ((ignoreNode != nullptr) && (rangeEnd->gtNext == ignoreNode) && (rangeEnd->gtNext->gtNext == endExclusive)))
+ {
+ return true;
+ }
+
+ if (rangeStart->OperConsumesFlags())
+ {
+ return false;
+ }
+
+ m_scratchSideEffects.Clear();
+ GenTree* cur = rangeStart;
+ while (true)
+ {
+ m_scratchSideEffects.AddNode(comp, cur);
+
+ if (cur == rangeEnd)
+ {
+ break;
+ }
+
+ cur = cur->gtNext;
+ assert((cur != nullptr) && "Expected rangeStart to precede rangeEnd");
+ }
+
+ for (GenTree* cur = rangeEnd->gtNext; cur != endExclusive; cur = cur->gtNext)
+ {
+ assert((cur != nullptr) && "Expected first node to precede end node");
+ if (cur == ignoreNode)
+ {
+ continue;
+ }
+
+ const bool strict = true;
+ if (m_scratchSideEffects.InterferesWith(comp, cur, strict))
+ {
+ return false;
+ }
+ }
+
+ return true;
+}
+
//------------------------------------------------------------------------
// IsSafeToContainMem: Checks for conflicts between childNode and parentNode,
// and returns 'true' iff memory operand childNode can be contained in parentNode.
GenTreeIntCon* op2 = cmp->gtGetOp2()->AsIntCon();
ssize_t op2Value = op2->IconValue();
-#ifdef TARGET_ARM64
- // Do not optimise further if op1 has a contained chain.
- if (op1->OperIs(GT_AND) &&
- (op1->isContainedCompareChainSegment(op1->gtGetOp1()) || op1->isContainedCompareChainSegment(op1->gtGetOp2())))
- {
- return cmp;
- }
-#endif
-
#ifdef TARGET_XARCH
var_types op1Type = op1->TypeGet();
if (IsContainableMemoryOp(op1) && varTypeIsSmall(op1Type) && FitsIn(op1Type, op2Value))
}
}
-#ifdef TARGET_XARCH
// Do not transform GT_SELECT with GTF_SET_FLAGS into GT_SELECTCC; this
// node is used by decomposition on x86.
// TODO-CQ: If we allowed multiple nodes to consume the same CPU flags then
ContainCheckSelect(newSelect);
return newSelect->gtNext;
}
-#endif
ContainCheckSelect(select);
return select->gtNext;
return true;
}
- // TODO-Cleanup: Avoid creating these SETCC nodes in the first place.
if (condition->OperIs(GT_SETCC))
{
assert((condition->gtPrev->gtFlags & GTF_SET_FLAGS) != 0);
- GenTree* flagsProducer = condition->gtPrev;
- if (!IsInvariantInRange(flagsProducer, parent, condition))
+ GenTree* flagsDef = condition->gtPrev;
+#ifdef TARGET_ARM64
+ // CCMP is a flag producing node that also consumes flags, so find the
+ // "root" of the flags producers and move the entire range.
+ // We limit this to 10 nodes look back to avoid quadratic behavior.
+ for (int i = 0; i < 10 && flagsDef->OperIs(GT_CCMP); i++)
+ {
+ assert((flagsDef->gtPrev != nullptr) && ((flagsDef->gtPrev->gtFlags & GTF_SET_FLAGS) != 0));
+ flagsDef = flagsDef->gtPrev;
+ }
+#endif
+ if (!IsRangeInvariantInRange(flagsDef, condition->gtPrev, parent, condition))
{
return false;
}
*cond = condition->AsCC()->gtCondition;
+ LIR::Range range = BlockRange().Remove(flagsDef, condition->gtPrev);
+ BlockRange().InsertBefore(parent, std::move(range));
BlockRange().Remove(condition);
- BlockRange().Remove(flagsProducer);
- BlockRange().InsertBefore(parent, flagsProducer);
return true;
}
void ContainCheckLclHeap(GenTreeOp* node);
void ContainCheckRet(GenTreeUnOp* ret);
#ifdef TARGET_ARM64
- bool IsValidCompareChain(GenTree* child, GenTree* parent);
- bool ContainCheckCompareChain(GenTree* child, GenTree* parent, GenTree** earliestValid);
- void ContainCheckCompareChainForAnd(GenTree* tree);
- void ContainCheckConditionalCompare(GenTreeOp* cmp);
+ GenTree* TryLowerAndToCCMP(GenTreeOp* tree);
+ insCflags TruthifyingFlags(GenCondition cond);
+ void ContainCheckConditionalCompare(GenTreeCCMP* ccmp);
void ContainCheckNeg(GenTreeOp* neg);
#endif
void ContainCheckSelect(GenTreeOp* select);
bool IsInvariantInRange(GenTree* node, GenTree* endExclusive) const;
bool IsInvariantInRange(GenTree* node, GenTree* endExclusive, GenTree* ignoreNode) const;
+ bool IsRangeInvariantInRange(GenTree* rangeStart,
+ GenTree* rangeEnd,
+ GenTree* endExclusive,
+ GenTree* ignoreNode) const;
// Checks for memory conflicts in the instructions between childNode and parentNode, and returns true if childNode
// can be contained.
#ifdef TARGET_ARM64
else
{
- ContainCheckCompareChainForAnd(binOp);
+ GenTree* next = TryLowerAndToCCMP(binOp);
+ if (next != nullptr)
+ {
+ return next;
+ }
}
#endif
}
#ifdef TARGET_ARM64
//------------------------------------------------------------------------
-// IsValidCompareChain : Determine if the node contains a valid chain of ANDs and CMPs.
+// TryLowerAndToCCMP : Lower an and of two conditions into test + CCMP + SETCC nodes.
//
// Arguments:
-// child - pointer to the node being checked.
-// parent - parent node of the child.
-//
-// Return value:
-// True if a valid chain is found.
+// tree - pointer to the node
//
-// Notes:
-// A compare chain is a sequence of CMP nodes connected by AND nodes.
-// For example: AND (AND (CMP A B) (CMP C D)) (CMP E F)
-// The chain can just be a single compare node, however it's parent
-// must always be an AND or SELECT node.
-// If a CMP or AND node is contained then it and all it's children are
-// considered to be in a valid chain.
-// Chains are built up during the lowering of each successive parent.
-//
-bool Lowering::IsValidCompareChain(GenTree* child, GenTree* parent)
+GenTree* Lowering::TryLowerAndToCCMP(GenTreeOp* tree)
{
- assert(parent->OperIs(GT_AND) || parent->OperIs(GT_SELECT));
+ assert(tree->OperIs(GT_AND));
- if (parent->isContainedCompareChainSegment(child))
+ if (!comp->opts.OptimizationEnabled())
{
- // Already have a chain.
- return true;
+ return nullptr;
}
- else if (child->OperIs(GT_AND))
+
+ GenTree* op1 = tree->gtGetOp1();
+ GenTree* op2 = tree->gtGetOp2();
+
+ // Find out whether op2 is eligible to be converted to a conditional
+ // compare. It must be a normal integral relop; for example, we cannot
+ // conditionally perform a floating point comparison and there is no "ctst"
+ // instruction that would allow us to conditionally implement
+ // TEST_EQ/TEST_NE.
+ //
+ if (!op2->OperIsCmpCompare() || !varTypeIsIntegral(op2->gtGetOp1()))
{
- // Count both sides.
- return IsValidCompareChain(child->AsOp()->gtGetOp2(), child) &&
- IsValidCompareChain(child->AsOp()->gtGetOp1(), child);
+ return nullptr;
}
- else if (child->OperIsCmpCompare() && varTypeIsIntegral(child->gtGetOp1()) && varTypeIsIntegral(child->gtGetOp2()))
+
+ // For op1 we can allow more arbitrary operations that set the condition
+ // flags; the final transformation into the flags def is done by
+ // TryLowerConditionToFlagsNode below, but we have a quick early out here
+ // too.
+ //
+ if (!op1->OperIsCompare() && !op1->OperIs(GT_SETCC))
{
- // Can the child compare be contained.
- return IsInvariantInRange(child, parent);
+ return nullptr;
}
- return false;
-}
-
-//------------------------------------------------------------------------
-// ContainCheckCompareChain : Determine if a chain of ANDs and CMPs can be contained.
-//
-// Arguments:
-// child - pointer to the node being checked.
-// parent - parent node of the child.
-// startOfChain - If found, returns the earliest valid op in the chain.
-//
-// Return value:
-// True if a valid chain is was contained.
-//
-// Notes:
-// Assumes the chain was checked via IsValidCompareChain.
-//
-bool Lowering::ContainCheckCompareChain(GenTree* child, GenTree* parent, GenTree** startOfChain)
-{
- assert(parent->OperIs(GT_AND) || parent->OperIs(GT_SELECT));
- *startOfChain = nullptr; // Nothing found yet.
-
- if (parent->isContainedCompareChainSegment(child))
+ // We leave checking invariance of op1 to tree to TryLowerConditionToFlagsNode.
+ if (!IsInvariantInRange(op2, tree))
{
- // Already have a contained chain.
- return true;
+ return nullptr;
}
- // Can the child be contained.
- else if (IsInvariantInRange(child, parent))
+
+ GenCondition cond1;
+ if (!TryLowerConditionToFlagsNode(tree, op1, &cond1))
{
- if (child->OperIs(GT_AND))
- {
- // If Op2 is not contained, then try to contain it.
- if (!child->isContainedCompareChainSegment(child->AsOp()->gtGetOp2()))
- {
- if (!ContainCheckCompareChain(child->gtGetOp2(), child, startOfChain))
- {
- // Op2 must be contained in order to contain Op1 or the AND.
- return false;
- }
- }
+ return nullptr;
+ }
- // If Op1 is not contained, then try to contain it.
- if (!child->isContainedCompareChainSegment(child->AsOp()->gtGetOp1()))
- {
- if (!ContainCheckCompareChain(child->gtGetOp1(), child, startOfChain))
- {
- return false;
- }
- }
+ BlockRange().Remove(op2);
+ BlockRange().InsertBefore(tree, op2);
- // Contain the AND.
- child->SetContained();
- return true;
- }
- else if (child->OperIsCmpCompare())
- {
- child->AsOp()->SetContained();
+ GenCondition cond2 = GenCondition::FromRelop(op2);
+ op2->SetOper(GT_CCMP);
+ op2->gtType = TYP_VOID;
+ op2->gtFlags |= GTF_SET_FLAGS;
- // Ensure the children of the compare are contained correctly.
- child->AsOp()->gtGetOp1()->ClearContained();
- child->AsOp()->gtGetOp2()->ClearContained();
- ContainCheckConditionalCompare(child->AsOp());
- *startOfChain = child;
- return true;
- }
- }
+ op2->gtGetOp1()->ClearContained();
+ op2->gtGetOp2()->ClearContained();
- return false;
+ GenTreeCCMP* ccmp = op2->AsCCMP();
+ ccmp->gtCondition = cond1;
+ // If the first comparison fails, set the condition flags to something that
+ // makes the second one fail as well so that the overall AND failed.
+ ccmp->gtFlagsVal = TruthifyingFlags(GenCondition::Reverse(cond2));
+ ContainCheckConditionalCompare(ccmp);
+
+ tree->SetOper(GT_SETCC);
+ tree->AsCC()->gtCondition = cond2;
+
+ return tree->gtNext;
}
//------------------------------------------------------------------------
-// ContainCheckCompareChainForAnd : Determine if an AND is a containable chain
+// TruthifyingFlags: Get a flags immediate that will make a specified condition true.
//
// Arguments:
-// node - pointer to the node
+// condition - the condition.
+//
+// Returns:
+// A flags immediate that, if those flags were set, would cause the specified condition to be true.
//
-void Lowering::ContainCheckCompareChainForAnd(GenTree* tree)
+insCflags Lowering::TruthifyingFlags(GenCondition condition)
{
- assert(tree->OperIs(GT_AND));
-
- if (!comp->opts.OptimizationEnabled())
- {
- return;
- }
-
- // First check there is a valid chain.
- if (IsValidCompareChain(tree->AsOp()->gtGetOp2(), tree) && IsValidCompareChain(tree->AsOp()->gtGetOp1(), tree))
- {
- GenTree* startOfChain = nullptr;
-
- // To ensure ordering at code generation, Op1 and the parent can
- // only be contained if Op2 is contained.
- if (ContainCheckCompareChain(tree->AsOp()->gtGetOp2(), tree, &startOfChain))
- {
- if (ContainCheckCompareChain(tree->AsOp()->gtGetOp1(), tree, &startOfChain))
- {
- // If op1 is the start of a chain, then it'll be generated as a standard compare.
- if (startOfChain != nullptr)
- {
- // The earliest node in the chain will be generated as a standard compare.
- assert(startOfChain->OperIsCmpCompare());
- startOfChain->AsOp()->gtGetOp1()->ClearContained();
- startOfChain->AsOp()->gtGetOp2()->ClearContained();
- ContainCheckCompare(startOfChain->AsOp());
- }
- }
- }
-
- JITDUMP("Lowered `AND` chain:\n");
- DISPTREE(tree);
+ switch (condition.GetCode())
+ {
+ case GenCondition::EQ:
+ return INS_FLAGS_Z;
+ case GenCondition::NE:
+ return INS_FLAGS_NONE;
+ case GenCondition::SGE:
+ return INS_FLAGS_Z;
+ case GenCondition::SGT:
+ return INS_FLAGS_NONE;
+ case GenCondition::SLT:
+ return INS_FLAGS_NC;
+ case GenCondition::SLE:
+ return INS_FLAGS_NZC;
+ case GenCondition::UGE:
+ return INS_FLAGS_C;
+ case GenCondition::UGT:
+ return INS_FLAGS_C;
+ case GenCondition::ULT:
+ return INS_FLAGS_NONE;
+ case GenCondition::ULE:
+ return INS_FLAGS_Z;
+ default:
+ NO_WAY("unexpected condition type");
+ return INS_FLAGS_NONE;
}
}
// Arguments:
// node - pointer to the node
//
-void Lowering::ContainCheckConditionalCompare(GenTreeOp* cmp)
+void Lowering::ContainCheckConditionalCompare(GenTreeCCMP* cmp)
{
- assert(cmp->OperIsCmpCompare());
GenTree* op2 = cmp->gtOp2;
if (op2->IsCnsIntOrI() && !op2->AsIntCon()->ImmedValNeedsReloc(comp))
#ifdef TARGET_ARM
noway_assert(!"GT_SELECT nodes are not supported on arm32");
#else
- if (!comp->opts.OptimizationEnabled())
- {
- return;
- }
-
- GenTree* cond = node->AsConditional()->gtCond;
- GenTree* op1 = node->gtOp1;
- GenTree* op2 = node->gtOp2;
-
- if (cond->OperIsCompare())
- {
- // All compare node types (including TEST_) are containable.
- if (IsInvariantInRange(cond, node))
- {
- cond->AsOp()->SetContained();
- }
- }
- else
- {
- // Check for a compare chain and try to contain it.
- GenTree* startOfChain = nullptr;
- ContainCheckCompareChain(cond, node, &startOfChain);
-
- if (startOfChain != nullptr)
- {
- // The earliest node in the chain will be generated as a standard compare.
- assert(startOfChain->OperIsCmpCompare());
- startOfChain->AsOp()->gtGetOp1()->ClearContained();
- startOfChain->AsOp()->gtGetOp2()->ClearContained();
- ContainCheckCompare(startOfChain->AsOp());
- }
- }
+ GenTree* op1 = node->gtOp1;
+ GenTree* op2 = node->gtOp2;
if (op1->IsIntegralConst(0))
{
case GT_TEST_NE:
case GT_CMP:
case GT_TEST:
+ case GT_CCMP:
case GT_JCMP:
srcCount = BuildCmp(tree);
break;
assert(dstCount == 1);
srcCount = BuildSelect(tree->AsConditional());
break;
+ case GT_SELECTCC:
+ assert(dstCount == 1);
+ srcCount = BuildSelect(tree->AsOp());
+ break;
} // end switch (tree->OperGet())
//
int LinearScan::BuildSelect(GenTreeOp* select)
{
- assert(select->OperIs(GT_SELECT));
+ assert(select->OperIs(GT_SELECT, GT_SELECTCC));
+
+ int srcCount = 0;
+ if (select->OperIs(GT_SELECT))
+ {
+ srcCount += BuildOperandUses(select->AsConditional()->gtCond);
+ }
- int srcCount = BuildOperandUses(select->AsConditional()->gtCond);
srcCount += BuildOperandUses(select->gtOp1);
srcCount += BuildOperandUses(select->gtOp2);
BuildDef(select);
//
int LinearScan::BuildCmp(GenTree* tree)
{
-#ifdef TARGET_XARCH
+#if defined(TARGET_XARCH)
assert(tree->OperIsCompare() || tree->OperIs(GT_CMP, GT_TEST, GT_JCMP, GT_BT));
+#elif defined(TARGET_ARM64)
+ assert(tree->OperIsCompare() || tree->OperIs(GT_CMP, GT_TEST, GT_JCMP, GT_CCMP));
#else
assert(tree->OperIsCompare() || tree->OperIs(GT_CMP, GT_TEST, GT_JCMP));
#endif
projects / platform / upstream / dotnet / runtime.git / commitdiff