#ifdef DEBUG
// Verify caller of this method checked safety.
//
- const bool isSafeToContainMem = IsSafeToContainMem(parentNode, childNode);
+ const bool isSafeToMarkRegOptional = IsSafeToMarkRegOptional(parentNode, childNode);
- if (!isSafeToContainMem)
+ if (!isSafeToMarkRegOptional)
{
JITDUMP("** Unsafe regOptional of [%06u] in [%06u}\n", comp->dspTreeID(childNode), comp->dspTreeID(parentNode));
- assert(isSafeToContainMem);
+ assert(isSafeToMarkRegOptional);
}
#endif
}
// HWIntrinsic nodes should use TryGetContainableHWIntrinsicOp and its relevant handling
assert(!parentNode->OperIsHWIntrinsic());
- if (!IsSafeToContainMem(parentNode, childNode))
- {
- return;
- }
-
- if (IsContainableMemoryOp(childNode))
+ if (IsContainableMemoryOp(childNode) && IsSafeToContainMem(parentNode, childNode))
{
MakeSrcContained(parentNode, childNode);
}
- else
+ else if (IsSafeToMarkRegOptional(parentNode, childNode))
{
MakeSrcRegOptional(parentNode, childNode);
}
}
//------------------------------------------------------------------------
-// IsSafeToContainMem: Checks for conflicts between childNode and parentNode,
-// and returns 'true' iff memory operand childNode can be contained in parentNode.
+// IsInvariantInRange: Check if a node is invariant in the specified range. In
+// other words, can 'node' be moved to right before 'endExclusive' without its
+// computation changing values?
//
// Arguments:
-// parentNode - any non-leaf node
-// childNode - some node that is an input to `parentNode`
+// node - The node.
+// endExclusive - The exclusive end of the range to check invariance for.
//
-// Return value:
-// true if it is safe to make childNode a contained memory operand.
+// Returns:
+// True if 'node' can be evaluated at any point between its current
+// location and 'endExclusive' without giving a different result; otherwise
+// false.
//
-bool Lowering::IsSafeToContainMem(GenTree* parentNode, GenTree* childNode) const
+bool Lowering::IsInvariantInRange(GenTree* node, GenTree* endExclusive) const
{
+ assert((node != nullptr) && (endExclusive != nullptr));
+
// Quick early-out for unary cases
//
- if (childNode->gtNext == parentNode)
+ if (node->gtNext == endExclusive)
{
return true;
}
m_scratchSideEffects.Clear();
- m_scratchSideEffects.AddNode(comp, childNode);
+ m_scratchSideEffects.AddNode(comp, node);
- for (GenTree* node = childNode->gtNext; node != parentNode; node = node->gtNext)
+ for (GenTree* cur = node->gtNext; cur != endExclusive; cur = cur->gtNext)
{
+ assert((cur != nullptr) && "Expected first node to precede end node");
const bool strict = true;
- if (m_scratchSideEffects.InterferesWith(comp, node, strict))
+ if (m_scratchSideEffects.InterferesWith(comp, cur, strict))
{
return false;
}
}
//------------------------------------------------------------------------
-// IsSafeToContainMem: Checks for conflicts between childNode and grandParentNode
-// and returns 'true' iff memory operand childNode can be contained in ancestorNode
+// IsInvariantInRange: Check if a node is invariant in the specified range,
+// ignoring conflicts with one particular node.
//
// Arguments:
-// grandParentNode - any non-leaf node
-// parentNode - parent of `childNode` and an input to `grandParentNode`
-// childNode - some node that is an input to `parentNode`
+// node - The node.
+// endExclusive - The exclusive end of the range to check invariance for.
+// ignoreNode - A node to ignore interference checks with, for example
+// because it will retain its relative order with 'node'.
//
-// Return value:
-// true if it is safe to make childNode a contained memory operand.
+// Returns:
+// True if 'node' can be evaluated at any point between its current location
+// and 'endExclusive' without giving a different result; otherwise false.
//
-bool Lowering::IsSafeToContainMem(GenTree* grandparentNode, GenTree* parentNode, GenTree* childNode) const
+bool Lowering::IsInvariantInRange(GenTree* node, GenTree* endExclusive, GenTree* ignoreNode) const
{
+ assert((node != nullptr) && (endExclusive != nullptr));
+
m_scratchSideEffects.Clear();
- m_scratchSideEffects.AddNode(comp, childNode);
+ m_scratchSideEffects.AddNode(comp, node);
- for (GenTree* node = childNode->gtNext; node != grandparentNode; node = node->gtNext)
+ for (GenTree* cur = node->gtNext; cur != endExclusive; cur = cur->gtNext)
{
- if (node == parentNode)
+ assert((cur != nullptr) && "Expected first node to precede end node");
+ if (cur == ignoreNode)
{
continue;
}
const bool strict = true;
- if (m_scratchSideEffects.InterferesWith(comp, node, strict))
+ if (m_scratchSideEffects.InterferesWith(comp, cur, strict))
{
return false;
}
}
//------------------------------------------------------------------------
+// IsSafeToContainMem: Checks for conflicts between childNode and parentNode,
+// and returns 'true' iff memory operand childNode can be contained in parentNode.
+//
+// Arguments:
+// parentNode - any non-leaf node
+// childNode - some node that is an input to `parentNode`
+//
+// Return value:
+// true if it is safe to make childNode a contained memory operand.
+//
+bool Lowering::IsSafeToContainMem(GenTree* parentNode, GenTree* childNode) const
+{
+ return IsInvariantInRange(childNode, parentNode);
+}
+
+//------------------------------------------------------------------------
+// IsSafeToContainMem: Checks for conflicts between childNode and grandParentNode
+// and returns 'true' iff memory operand childNode can be contained in grandParentNode.
+//
+// Arguments:
+// grandParentNode - any non-leaf node
+// parentNode - parent of `childNode` and an input to `grandParentNode`
+// childNode - some node that is an input to `parentNode`
+//
+// Return value:
+// true if it is safe to make childNode a contained memory operand.
+//
+bool Lowering::IsSafeToContainMem(GenTree* grandparentNode, GenTree* parentNode, GenTree* childNode) const
+{
+ return IsInvariantInRange(childNode, grandparentNode, parentNode);
+}
+
+//------------------------------------------------------------------------
+// IsSafeToMarkRegOptional: Check whether it is safe to mark 'childNode' as
+// reg-optional in 'parentNode'.
+//
+// Arguments:
+// parentNode - parent of 'childNode'
+// childNode - some node that is an input to `parentNode`
+//
+// Return value:
+// True if it is safe to mark childNode as reg-optional; otherwise false.
+//
+// Remarks:
+// Unlike containment, reg-optionality can only rarely introduce new
+// conflicts, because reg-optionality mostly does not cause the child node
+// to be evaluated at a new point in time:
+//
+// 1. For LIR edges (i.e. anything that isn't GT_LCL_VAR) reg-optionality
+// indicates that if the edge was spilled to a temp at its def, the parent
+// node can use it directly from its spill location without reloading it
+// into a register first. This is always safe as as spill temps cannot
+// interfere.
+//
+// For example, an indirection can be marked reg-optional even if there
+// is interference between it and its parent; the indirection will still
+// be evaluated at its original position, but if the value is spilled to
+// stack, then reg-optionality can allow using the value from the spill
+// location directly. Similarly, GT_LCL_FLD nodes are never register
+// candidates and can be handled the same way.
+//
+// 2. For GT_LCL_VAR reg-optionality indicates that the node can use the
+// local directly from its home location. IR invariants guarantee that the
+// local is not defined between its LIR location and the parent node (see
+// CheckLclVarSemanticsHelper). That means the only case where it could
+// interfere is due to it being address exposed. So this is the only unsafe
+// case.
+//
+bool Lowering::IsSafeToMarkRegOptional(GenTree* parentNode, GenTree* childNode) const
+{
+ if (!childNode->OperIs(GT_LCL_VAR))
+ {
+ // LIR edges never interfere. This includes GT_LCL_FLD, see the remarks above.
+ return true;
+ }
+
+ LclVarDsc* dsc = comp->lvaGetDesc(childNode->AsLclVarCommon());
+ if (!dsc->IsAddressExposed())
+ {
+ // Safe by IR invariants (no assignments occur between parent and node).
+ return true;
+ }
+
+ // We expect this to have interference as otherwise we could have marked it
+ // contained instead of reg-optional.
+ return false;
+}
+
+//------------------------------------------------------------------------
// LowerNode: this is the main entry point for Lowering.
//
// Arguments:
}
//------------------------------------------------------------------------
-// IsInvariantInRange: Check if a node is invariant in the specified range. In
-// other words, can 'node' be moved to right before 'endExclusive' without its
-// computation changing values?
+// IsCFGCallArgInvariantInRange: A cheap version of IsInvariantInRange to check
+// if a node is invariant in the specified range. In other words, can 'node' be
+// moved to right before 'endExclusive' without its computation changing
+// values?
//
// Arguments:
// node - The node.
// endExclusive - The exclusive end of the range to check invariance for.
//
-bool Lowering::IsInvariantInRange(GenTree* node, GenTree* endExclusive)
+bool Lowering::IsCFGCallArgInvariantInRange(GenTree* node, GenTree* endExclusive)
{
assert(node->Precedes(endExclusive));
GenTree* operand = node->AsOp()->gtGetOp1();
JITDUMP("Checking if we can move operand of GT_PUTARG_* node:\n");
DISPTREE(operand);
- if (((operand->gtFlags & GTF_ALL_EFFECT) == 0) && IsInvariantInRange(operand, call))
+ if (((operand->gtFlags & GTF_ALL_EFFECT) == 0) && IsCFGCallArgInvariantInRange(operand, call))
{
JITDUMP("...yes, moving to after validator call\n");
BlockRange().Remove(operand);
{
if (index->OperIs(GT_CAST) && (scale == 1) && (offset == 0) && varTypeIsByte(targetType))
{
- if (IsSafeToContainMem(parent, index))
+ if (IsInvariantInRange(index, parent))
{
// Check containment safety against the parent node - this will ensure that LEA with the contained
// index will itself always be contained. We do not support uncontained LEAs with contained indices.
if (cast->CastOp()->TypeIs(TYP_INT) && cast->TypeIs(TYP_LONG) &&
(genTypeSize(targetType) == (1U << shiftBy)) && (scale == 1) && (offset == 0))
{
- if (IsSafeToContainMem(parent, index))
+ if (IsInvariantInRange(index, parent))
{
// Check containment safety against the parent node - this will ensure that LEA with the contained
// index will itself always be contained. We do not support uncontained LEAs with contained indices.
#if defined(TARGET_ARM64)
// Verify containment safety before creating an LEA that must be contained.
//
- const bool isContainable = IsSafeToContainMem(ind, ind->Addr());
+ const bool isContainable = IsInvariantInRange(ind->Addr(), ind);
#else
const bool isContainable = true;
#endif
#if defined(TARGET_ARM64)
// Verify containment safety before creating an LEA that must be contained.
//
- const bool isContainable = (ind->Addr() != nullptr) && IsSafeToContainMem(ind, ind->Addr());
+ const bool isContainable = (ind->Addr() != nullptr) && IsInvariantInRange(ind->Addr(), ind);
#else
const bool isContainable = true;
#endif
void LowerBlock(BasicBlock* block);
GenTree* LowerNode(GenTree* node);
- bool IsInvariantInRange(GenTree* node, GenTree* endExclusive);
+ bool IsCFGCallArgInvariantInRange(GenTree* node, GenTree* endExclusive);
// ------------------------------
// Call Lowering
// Checks and makes 'childNode' contained in the 'parentNode'
bool CheckImmedAndMakeContained(GenTree* parentNode, GenTree* childNode);
+ bool IsInvariantInRange(GenTree* node, GenTree* endExclusive) const;
+ bool IsInvariantInRange(GenTree* node, GenTree* endExclusive, GenTree* ignoreNode) const;
+
// Checks for memory conflicts in the instructions between childNode and parentNode, and returns true if childNode
// can be contained.
bool IsSafeToContainMem(GenTree* parentNode, GenTree* childNode) const;
// Similar to above, but allows bypassing a "transparent" parent.
bool IsSafeToContainMem(GenTree* grandparentNode, GenTree* parentNode, GenTree* childNode) const;
+ // Check if marking an operand of a node as reg-optional is safe.
+ bool IsSafeToMarkRegOptional(GenTree* parentNode, GenTree* node) const;
+
inline LIR::Range& BlockRange() const
{
return LIR::AsRange(m_block);
if (parentNode->OperIs(GT_ADD))
{
// Find "c + (a * b)" or "(a * b) + c"
- return IsSafeToContainMem(parentNode, childNode);
+ return IsInvariantInRange(childNode, parentNode);
}
if (parentNode->OperIs(GT_SUB))
{
// Find "c - (a * b)"
assert(childNode == parentNode->gtGetOp2());
- return IsSafeToContainMem(parentNode, childNode);
+ return IsInvariantInRange(childNode, parentNode);
}
return false;
{
// These operations can still report flags
- if (IsSafeToContainMem(parentNode, childNode))
+ if (IsInvariantInRange(childNode, parentNode))
{
assert(shiftAmountNode->isContained());
return true;
if (parentNode->OperIs(GT_CMP, GT_OR, GT_XOR))
{
- if (IsSafeToContainMem(parentNode, childNode))
+ if (IsInvariantInRange(childNode, parentNode))
{
assert(shiftAmountNode->isContained());
return true;
{
// These operations can still report flags
- if (IsSafeToContainMem(parentNode, childNode))
+ if (IsInvariantInRange(childNode, parentNode))
{
return true;
}
if (parentNode->OperIs(GT_CMP))
{
- if (IsSafeToContainMem(parentNode, childNode))
+ if (IsInvariantInRange(childNode, parentNode))
{
return true;
}
}
#endif // !TARGET_ARM
- if (!IsSafeToContainMem(blkNode, addrParent, addr))
+ if (!IsInvariantInRange(addr, blkNode, addrParent))
{
return;
}
GenTree* addr = indirNode->Addr();
- if ((addr->OperGet() == GT_LEA) && IsSafeToContainMem(indirNode, addr))
+ if ((addr->OperGet() == GT_LEA) && IsInvariantInRange(addr, indirNode))
{
bool makeContained = true;
srcIsContainable = true;
}
- if (srcIsContainable && IsSafeToContainMem(node, castOp))
+ if (srcIsContainable)
{
- if (IsContainableMemoryOp(castOp))
+ if (IsContainableMemoryOp(castOp) && IsSafeToContainMem(node, castOp))
{
MakeSrcContained(node, castOp);
}
- else
+ else if (IsSafeToMarkRegOptional(node, castOp))
{
castOp->SetRegOptional();
}
else if (child->OperIsCmpCompare() && varTypeIsIntegral(child->gtGetOp1()) && varTypeIsIntegral(child->gtGetOp2()))
{
// Can the child compare be contained.
- return IsSafeToContainMem(parent, child);
+ return IsInvariantInRange(child, parent);
}
return false;
return true;
}
// Can the child be contained.
- else if (IsSafeToContainMem(parent, child))
+ else if (IsInvariantInRange(child, parent))
{
if (child->OperIs(GT_AND))
{
if (cond->OperIsCompare())
{
// All compare node types (including TEST_) are containable.
- if (IsSafeToContainMem(node, cond))
+ if (IsInvariantInRange(cond, node))
{
cond->AsOp()->SetContained();
}
if ((childNode->gtFlags & GTF_SET_FLAGS))
return;
- if (IsSafeToContainMem(neg, childNode))
+ if (IsInvariantInRange(childNode, neg))
{
MakeSrcContained(neg, childNode);
}
return;
}
- if (!IsSafeToContainMem(blkNode, addrParent, addr))
+ if (!IsInvariantInRange(addr, blkNode, addrParent))
{
return;
}
#endif // FEATURE_SIMD
GenTree* addr = indirNode->Addr();
- if ((addr->OperGet() == GT_LEA) && IsSafeToContainMem(indirNode, addr))
+ if ((addr->OperGet() == GT_LEA) && IsInvariantInRange(addr, indirNode))
{
MakeSrcContained(indirNode, addr);
}
// Note that the parentNode is always the block node, even if we're dealing with the source address.
// The source address is not directly used by the block node but by an IND node and that IND node is
// always contained.
- if (!IsSafeToContainMem(blkNode, addrParent, addrMode))
+ if (!IsInvariantInRange(addrMode, blkNode, addrParent))
{
return;
}
break;
}
- if (!IsSafeToContainMem(node, op1))
+ if (!IsInvariantInRange(op1, node))
{
// What we're doing here is effectively similar to containment,
// except for we're deleting the node entirely, so don't we have
MakeSrcContained(node, addr);
}
}
- else if ((addr->OperGet() == GT_LEA) && IsSafeToContainMem(node, addr))
+ else if ((addr->OperGet() == GT_LEA) && IsInvariantInRange(addr, node))
{
MakeSrcContained(node, addr);
}
{
unsigned swapSize = src->OperIs(GT_BSWAP16) ? 2 : genTypeSize(src);
- if ((swapSize == genTypeSize(node)) && IsSafeToContainMem(node, src))
+ if ((swapSize == genTypeSize(node)) && IsInvariantInRange(src, node))
{
// Prefer containing in the store in case the load has been contained.
src->gtGetOp1()->ClearContained();
}
}
- if (isContainable && IsSafeToContainMem(node, src))
+ if (isContainable && IsInvariantInRange(src, node))
{
MakeSrcContained(node, src);
}
else
{
- // IsSafeToContainMem is expensive so we call it at most once for each operand
- // in this method. If we already called IsSafeToContainMem, it must have returned false;
- // otherwise, memOp would be set to the corresponding operand (op1 or op2).
if (imm != nullptr)
{
// Has a contained immediate operand.
// Only 'other' operand can be marked as reg optional.
assert(other != nullptr);
- isSafeToContainOp1 = ((other == op1) && isSafeToContainOp1 && IsSafeToContainMem(node, op1));
- isSafeToContainOp2 = ((other == op2) && isSafeToContainOp2 && IsSafeToContainMem(node, op2));
+ isSafeToContainOp1 = ((other == op1) && IsSafeToMarkRegOptional(node, op1));
+ isSafeToContainOp2 = ((other == op2) && IsSafeToMarkRegOptional(node, op2));
}
else if (hasImpliedFirstOperand)
{
// Only op2 can be marked as reg optional.
isSafeToContainOp1 = false;
- isSafeToContainOp2 = isSafeToContainOp2 && IsSafeToContainMem(node, op2);
+ isSafeToContainOp2 = isSafeToContainOp2 && IsSafeToMarkRegOptional(node, op2);
}
else
{
// If there are no containable operands, we can make either of op1 or op2
// as reg optional.
- isSafeToContainOp1 = isSafeToContainOp1 && IsSafeToContainMem(node, op1);
- isSafeToContainOp2 = isSafeToContainOp2 && IsSafeToContainMem(node, op2);
+ isSafeToContainOp1 = isSafeToContainOp1 && IsSafeToMarkRegOptional(node, op1);
+ isSafeToContainOp2 = isSafeToContainOp2 && IsSafeToMarkRegOptional(node, op2);
}
SetRegOptionalForBinOp(node, isSafeToContainOp1, isSafeToContainOp2);
}
#endif
// divisor can be an r/m, but the memory indirection must be of the same size as the divide
- if (IsContainableMemoryOp(divisor) && (divisor->TypeGet() == node->TypeGet()) && IsSafeToContainMem(node, divisor))
+ if (IsContainableMemoryOp(divisor) && (divisor->TypeGet() == node->TypeGet()) && IsInvariantInRange(divisor, node))
{
MakeSrcContained(node, divisor);
}
- else if (divisorCanBeRegOptional)
+ else if (divisorCanBeRegOptional && IsSafeToMarkRegOptional(node, divisor))
{
// If there are no containable operands, we can make an operand reg optional.
// Div instruction allows only divisor to be a memory op.
}
}
- if (!otherOp->isContained() && isSafeToContainOtherOp && IsSafeToContainMem(cmp, otherOp))
+ if (!otherOp->isContained() && IsSafeToMarkRegOptional(cmp, otherOp))
{
// SSE2 allows only otherOp to be a memory-op. Since otherOp is not
// contained, we can mark it reg-optional.
// if one of them is on stack.
GenTree* regOptionalCandidate = op1->IsCnsIntOrI() ? op2 : PreferredRegOptionalOperand(cmp);
- // IsSafeToContainMem is expensive so we call it at most once for each operand
- // in this method. If we already called IsSafeToContainMem, it must have returned false;
- // otherwise, the corresponding operand (op1 or op2) would be contained.
- bool setRegOptional = (regOptionalCandidate == op1) ? isSafeToContainOp1 && IsSafeToContainMem(cmp, op1)
- : isSafeToContainOp2 && IsSafeToContainMem(cmp, op2);
+ bool setRegOptional =
+ (regOptionalCandidate == op1) ? IsSafeToMarkRegOptional(cmp, op1) : IsSafeToMarkRegOptional(cmp, op2);
if (setRegOptional)
{
MakeSrcRegOptional(cmp, regOptionalCandidate);
if (genTypeSize(op1) == operSize)
{
- if (IsContainableMemoryOp(op1))
+ if (IsContainableMemoryOp(op1) && IsSafeToContainMem(select, op1))
{
- if (IsSafeToContainMem(select, op1))
- {
- MakeSrcContained(select, op1);
- }
+ MakeSrcContained(select, op1);
}
- else if (IsSafeToContainMem(select, op1))
+ else if (IsSafeToMarkRegOptional(select, op1))
{
MakeSrcRegOptional(select, op1);
}
if (genTypeSize(op2) == operSize)
{
- if (IsContainableMemoryOp(op2))
+ if (IsContainableMemoryOp(op2) && IsSafeToContainMem(select, op2))
{
- if (IsSafeToContainMem(select, op2))
- {
- MakeSrcContained(select, op2);
- }
+ MakeSrcContained(select, op2);
}
- else if (IsSafeToContainMem(select, op2))
+ else if (IsSafeToMarkRegOptional(select, op2))
{
MakeSrcRegOptional(select, op2);
}
// Read-Modify-Write (RMW) operation, we can mark its operands
// as reg optional.
- // IsSafeToContainMem is expensive so we call it at most once for each operand
- // in this method. If we already called IsSafeToContainMem, it must have returned false;
- // otherwise, directlyEncodable would be true.
- isSafeToContainOp1 = isSafeToContainOp1 && IsSafeToContainMem(node, op1);
- isSafeToContainOp2 = isSafeToContainOp2 && IsSafeToContainMem(node, op2);
+ isSafeToContainOp1 = IsSafeToMarkRegOptional(node, op1);
+ isSafeToContainOp2 = IsSafeToMarkRegOptional(node, op2);
SetRegOptionalForBinOp(node, isSafeToContainOp1, isSafeToContainOp2);
}
}
}
- bool isSafeToContainMem = IsSafeToContainMem(parentNode, childNode);
-
- *supportsRegOptional = isSafeToContainMem && supportsGeneralLoads;
+ *supportsRegOptional = supportsGeneralLoads && IsSafeToMarkRegOptional(parentNode, childNode);
if (!childNode->OperIsHWIntrinsic())
{
{
if (IsContainableMemoryOp(childNode))
{
- canBeContained = isSafeToContainMem;
+ canBeContained = IsSafeToContainMem(parentNode, childNode);
}
else if (childNode->IsCnsNonZeroFltOrDbl())
{
TryCreateAddrMode(addr, true, node);
if ((addr->OperIs(GT_CLS_VAR_ADDR, GT_LCL_VAR_ADDR, GT_LCL_FLD_ADDR, GT_LEA) ||
(addr->IsCnsIntOrI() && addr->AsIntConCommon()->FitsInAddrBase(comp))) &&
- IsSafeToContainMem(node, addr))
+ IsInvariantInRange(addr, node))
{
MakeSrcContained(node, addr);
}
if (!op1->isContained() && !op2->isContained())
{
// If there are no containable operands, we can make an operand reg optional.
- // IsSafeToContainMem is expensive so we call it at most once for each operand
- // in this method. If we already called IsSafeToContainMem, it must have returned false;
- // otherwise, the corresponding operand (op1 or op2) would be contained.
- isSafeToContainOp1 = isSafeToContainOp1 && IsSafeToContainMem(node, op1);
- isSafeToContainOp2 = isSafeToContainOp2 && IsSafeToContainMem(node, op2);
+ isSafeToContainOp1 = IsSafeToMarkRegOptional(node, op1);
+ isSafeToContainOp2 = IsSafeToMarkRegOptional(node, op2);
SetRegOptionalForBinOp(node, isSafeToContainOp1, isSafeToContainOp2);
}
}
projects / platform / upstream / dotnet / runtime.git / commitdiff