// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
// loResult - the decomposed low part
-// hiResult - the decomposed high part. This must follow loResult in the linear order,
-// as the new GT_LONG node will be inserted immediately after it.
+// hiResult - the decomposed high part
+// insertResultAfter - the node that the GT_LONG should be inserted after
//
// Return Value:
// The next node to process.
//
-GenTree* DecomposeLongs::FinalizeDecomposition(LIR::Use& use, GenTree* loResult, GenTree* hiResult)
+GenTree* DecomposeLongs::FinalizeDecomposition(LIR::Use& use, GenTree* loResult, GenTree* hiResult, GenTree* insertResultAfter)
{
assert(use.IsInitialized());
assert(loResult != nullptr);
assert(hiResult != nullptr);
assert(Range().Contains(loResult));
assert(Range().Contains(hiResult));
- assert(loResult->Precedes(hiResult));
GenTree* gtLong = new (m_compiler, GT_LONG) GenTreeOp(GT_LONG, TYP_LONG, loResult, hiResult);
- Range().InsertAfter(hiResult, gtLong);
+ Range().InsertAfter(insertResultAfter, gtLong);
use.ReplaceWith(m_compiler, gtLong);
m_compiler->lvaIncRefCnts(loResult);
m_compiler->lvaIncRefCnts(hiResult);
- return FinalizeDecomposition(use, loResult, hiResult);
+ return FinalizeDecomposition(use, loResult, hiResult, hiResult);
}
//------------------------------------------------------------------------
GenTree* hiResult = m_compiler->gtNewLclFldNode(loResult->gtLclNum, TYP_INT, loResult->gtLclOffs + 4);
Range().InsertAfter(loResult, hiResult);
- return FinalizeDecomposition(use, loResult, hiResult);
+ return FinalizeDecomposition(use, loResult, hiResult, hiResult);
}
//------------------------------------------------------------------------
NYI("Unimplemented cast decomposition");
}
- return FinalizeDecomposition(use, loResult, hiResult);
+ return FinalizeDecomposition(use, loResult, hiResult, hiResult);
}
//------------------------------------------------------------------------
GenTree* hiResult = new (m_compiler, GT_CNS_INT) GenTreeIntCon(TYP_INT, hiVal);
Range().InsertAfter(loResult, hiResult);
- return FinalizeDecomposition(use, loResult, hiResult);
+ return FinalizeDecomposition(use, loResult, hiResult, hiResult);
}
//------------------------------------------------------------------------
Range().InsertAfter(indLow, addrBaseHigh, addrHigh, indHigh);
- return FinalizeDecomposition(use, indLow, indHigh);
+ return FinalizeDecomposition(use, indLow, indHigh, indHigh);
}
//------------------------------------------------------------------------
GenTree* hiResult = new (m_compiler, GT_NOT) GenTreeOp(GT_NOT, TYP_INT, hiOp1, nullptr);
Range().InsertAfter(loResult, hiResult);
- return FinalizeDecomposition(use, loResult, hiResult);
+ return FinalizeDecomposition(use, loResult, hiResult, hiResult);
}
//------------------------------------------------------------------------
Range().InsertAfter(loResult, zero, hiAdjust, hiResult);
- return FinalizeDecomposition(use, loResult, hiResult);
+ return FinalizeDecomposition(use, loResult, hiResult, hiResult);
}
//------------------------------------------------------------------------
}
}
- return FinalizeDecomposition(use, loResult, hiResult);
+ return FinalizeDecomposition(use, loResult, hiResult, hiResult);
}
//------------------------------------------------------------------------
-// DecomposeShift: Decompose GT_LSH, GT_RSH, GT_RSZ. For shift nodes, we need to use
-// the shift helper functions, so we here convert the shift into a helper call by
-// pulling its arguments out of linear order and making them the args to a call, then
-// replacing the original node with the new call.
+// DecomposeShift: Decompose GT_LSH, GT_RSH, GT_RSZ. For shift nodes being shifted
+// by a constant int, we can inspect the shift amount and decompose to the appropriate
+// node types, generating a shl/shld pattern for GT_LSH, a shrd/shr pattern for GT_RSZ,
+// and a shrd/sar pattern for GT_SHR for most shift amounts. Shifting by 0, >= 32 and
+// >= 64 are special cased to produce better code patterns.
+//
+// For all other shift nodes, we need to use the shift helper functions, so we here convert
+// the shift into a helper call by pulling its arguments out of linear order and making
+// them the args to a call, then replacing the original node with the new call.
//
// Arguments:
// use - the LIR::Use object for the def that needs to be decomposed.
GenTree* tree = use.Def();
GenTree* gtLong = tree->gtGetOp1();
+ GenTree* oldShiftByOp = tree->gtGetOp2();
+
genTreeOps oper = tree->OperGet();
+ genTreeOps shiftByOper = oldShiftByOp->OperGet();
assert((oper == GT_LSH) || (oper == GT_RSH) || (oper == GT_RSZ));
- LIR::Use loOp1Use(Range(), >Long->gtOp.gtOp1, gtLong);
- loOp1Use.ReplaceWithLclVar(m_compiler, m_blockWeight);
+ unsigned loOp1LclNum;
+ unsigned hiOp1LclNum;
- LIR::Use hiOp1Use(Range(), >Long->gtOp.gtOp2, gtLong);
- hiOp1Use.ReplaceWithLclVar(m_compiler, m_blockWeight);
+ if (gtLong->gtOp.gtOp1->OperGet() != GT_LCL_VAR)
+ {
+ LIR::Use loOp1Use(Range(), >Long->gtOp.gtOp1, gtLong);
+ loOp1LclNum = loOp1Use.ReplaceWithLclVar(m_compiler, m_blockWeight);
+ }
+ else
+ {
+ loOp1LclNum = gtLong->gtOp.gtOp1->AsLclVarCommon()->gtLclNum;
+ }
- LIR::Use shiftWidthUse(Range(), &tree->gtOp.gtOp2, tree);
- shiftWidthUse.ReplaceWithLclVar(m_compiler, m_blockWeight);
+ if (gtLong->gtOp.gtOp2->OperGet() != GT_LCL_VAR)
+ {
+ LIR::Use hiOp1Use(Range(), >Long->gtOp.gtOp2, gtLong);
+ hiOp1LclNum = hiOp1Use.ReplaceWithLclVar(m_compiler, m_blockWeight);
+ }
+ else
+ {
+ hiOp1LclNum = gtLong->gtOp.gtOp2->AsLclVarCommon()->gtLclNum;
+ }
GenTree* loOp1 = gtLong->gtGetOp1();
GenTree* hiOp1 = gtLong->gtGetOp2();
- GenTree* shiftWidthOp = tree->gtGetOp2();
-
Range().Remove(gtLong);
Range().Remove(loOp1);
Range().Remove(hiOp1);
- Range().Remove(shiftWidthOp);
+ // If we are shifting by a constant int, we do not want to use a helper, instead, we decompose.
+ if (shiftByOper == GT_CNS_INT)
+ {
+ unsigned int count = oldShiftByOp->gtIntCon.gtIconVal;
+ Range().Remove(oldShiftByOp);
- // TODO-X86-CQ: If the shift operand is a GT_CNS_INT, we should pipe the instructions through to codegen
- // and generate the shift instructions ourselves there, rather than replacing it with a helper call.
+ GenTree* loResult;
+ GenTree* hiResult;
- unsigned helper;
+ GenTree* insertAfter;
- switch (oper)
- {
- case GT_LSH:
- helper = CORINFO_HELP_LLSH;
+ switch (oper)
+ {
+ case GT_LSH:
+ {
+ if (count == 0)
+ {
+ // Do nothing.
+ loResult = loOp1;
+ hiResult = hiOp1;
+
+ Range().InsertBefore(tree, loResult, hiResult);
+
+ insertAfter = hiResult;
+ }
+ else if (count < 32)
+ {
+ // Hi is a GT_LSH_HI, lo is a GT_LSH. Will produce:
+ // reg1 = lo
+ // shl lo, shift
+ // shld hi, reg1, shift
+
+ GenTree* shiftByHi = m_compiler->gtNewIconNode(count, TYP_INT);
+ GenTree* shiftByLo = m_compiler->gtNewIconNode(count, TYP_INT);
+
+ loResult = m_compiler->gtNewOperNode(GT_LSH, TYP_INT, loOp1, shiftByLo);
+
+ // Create a GT_LONG that contains loCopy and hiOp1. This will be used in codegen to
+ // generate the shld instruction
+ GenTree* loCopy = m_compiler->gtNewLclvNode(loOp1LclNum, TYP_INT);
+ GenTree* hiOp = new (m_compiler, GT_LONG) GenTreeOp(GT_LONG, TYP_LONG, loCopy, hiOp1);
+ hiResult = m_compiler->gtNewOperNode(GT_LSH_HI, TYP_INT, hiOp, shiftByHi);
+
+ m_compiler->lvaIncRefCnts(loCopy);
+
+ Range().InsertBefore(tree, loCopy, hiOp1, hiOp);
+ Range().InsertBefore(tree, shiftByHi, hiResult);
+ Range().InsertBefore(tree, loOp1, shiftByLo, loResult);
+
+ insertAfter = loResult;
+ }
+ else
+ {
+ assert(count >= 32);
+
+ // Zero out loResult (shift of >= 32 bits shifts all lo bits to hiResult)
+ loResult = m_compiler->gtNewZeroConNode(TYP_INT);
+ Range().InsertBefore(tree, loResult);
+
+ if (count < 64)
+ {
+ if (count == 32)
+ {
+ // Move loOp1 into hiResult (shift of 32 bits is just a mov of lo to hi)
+ hiResult = loOp1;
+ Range().InsertBefore(tree, hiResult);
+ }
+ else
+ {
+ assert(count > 32 && count < 64);
+
+ // Move loOp1 into hiResult, do a GT_LSH with count - 32.
+ GenTree* shiftBy = m_compiler->gtNewIconNode(count - 32, TYP_INT);
+ hiResult = m_compiler->gtNewOperNode(oper, TYP_INT, loOp1, shiftBy);
+ Range().InsertBefore(tree, loOp1, shiftBy, hiResult);
+ }
+ }
+ else
+ {
+ assert(count >= 64);
+
+ // Zero out hi (shift of >= 64 bits moves all the bits out of the two registers)
+ hiResult = m_compiler->gtNewZeroConNode(TYP_INT);
+ Range().InsertBefore(tree, hiResult);
+ }
+
+ insertAfter = hiResult;
+ }
+ }
break;
- case GT_RSH:
- helper = CORINFO_HELP_LRSH;
+ case GT_RSZ:
+ {
+ if (count == 0)
+ {
+ // Do nothing.
+ loResult = loOp1;
+ hiResult = hiOp1;
+ Range().InsertBefore(tree, loResult, hiResult);
+ }
+ else if (count < 32)
+ {
+ // Hi is a GT_RSZ, lo is a GT_RSH_LO. Will produce:
+ // reg1 = hi
+ // shrd lo, reg1, shift
+ // shr hi, shift
+
+ GenTree* shiftByHi = m_compiler->gtNewIconNode(count, TYP_INT);
+ GenTree* shiftByLo = m_compiler->gtNewIconNode(count, TYP_INT);
+ GenTree* hiCopy = m_compiler->gtNewLclvNode(hiOp1LclNum, TYP_INT);
+ m_compiler->lvaIncRefCnts(hiCopy);
+
+ hiResult = m_compiler->gtNewOperNode(GT_RSZ, TYP_INT, hiOp1, shiftByHi);
+
+ // Create a GT_LONG that contains loOp1 and hiCopy. This will be used in codegen to
+ // generate the shrd instruction
+ GenTree* loOp = new (m_compiler, GT_LONG) GenTreeOp(GT_LONG, TYP_LONG, loOp1, hiCopy);
+ loResult = m_compiler->gtNewOperNode(GT_RSH_LO, TYP_INT, loOp, shiftByLo);
+
+ Range().InsertBefore(tree, loOp1, hiCopy, loOp);
+ Range().InsertBefore(tree, shiftByLo, loResult);
+ Range().InsertBefore(tree, hiOp1, shiftByHi, hiResult);
+ }
+ else
+ {
+ assert(count >= 32);
+ if (count < 64)
+ {
+ if (count == 32)
+ {
+ // Move hiOp1 into loResult.
+ loResult = hiOp1;
+ Range().InsertBefore(tree, loResult);
+ }
+ else
+ {
+ assert(count > 32 && count < 64);
+
+ // Move hiOp1 into loResult, do a GT_RSZ with count - 32.
+ GenTree* shiftBy = m_compiler->gtNewIconNode(count - 32, TYP_INT);
+ loResult = m_compiler->gtNewOperNode(oper, TYP_INT, hiOp1, shiftBy);
+ Range().InsertBefore(tree, hiOp1, shiftBy, loResult);
+ }
+ }
+ else
+ {
+ assert(count >= 64);
+
+ // Zero out lo
+ loResult = m_compiler->gtNewZeroConNode(TYP_INT);
+ Range().InsertBefore(tree, loResult);
+ }
+
+ // Zero out hi
+ hiResult = m_compiler->gtNewZeroConNode(TYP_INT);
+ Range().InsertBefore(tree, hiResult);
+ }
+
+ insertAfter = hiResult;
+ }
break;
- case GT_RSZ:
- helper = CORINFO_HELP_LRSZ;
+ case GT_RSH:
+ {
+ if (count == 0)
+ {
+ // Do nothing.
+ loResult = loOp1;
+ hiResult = hiOp1;
+ Range().InsertBefore(tree, loResult, hiResult);
+ }
+ else if (count < 32)
+ {
+ // Hi is a GT_RSH, lo is a GT_RSH_LO. Will produce:
+ // reg1 = hi
+ // shrd lo, reg1, shift
+ // sar hi, shift
+
+ GenTree* shiftByHi = m_compiler->gtNewIconNode(count, TYP_INT);
+ GenTree* shiftByLo = m_compiler->gtNewIconNode(count, TYP_INT);
+ GenTree* hiCopy = m_compiler->gtNewLclvNode(hiOp1LclNum, TYP_INT);
+ m_compiler->lvaIncRefCnts(hiCopy);
+
+ hiResult = m_compiler->gtNewOperNode(GT_RSH, TYP_INT, hiOp1, shiftByHi);
+
+ // Create a GT_LONG that contains loOp1 and hiCopy. This will be used in codegen to
+ // generate the shrd instruction
+ GenTree* loOp = new (m_compiler, GT_LONG) GenTreeOp(GT_LONG, TYP_LONG, loOp1, hiCopy);
+ loResult = m_compiler->gtNewOperNode(GT_RSH_LO, TYP_INT, loOp, shiftByLo);
+
+ Range().InsertBefore(tree, loOp1, hiCopy, loOp);
+ Range().InsertBefore(tree, shiftByLo, loResult);
+ Range().InsertBefore(tree, shiftByHi, hiOp1, hiResult);
+ }
+ else
+ {
+ assert(count >= 32);
+ if (count < 64)
+ {
+ if (count == 32)
+ {
+ // Move hiOp1 into loResult.
+ loResult = hiOp1;
+ Range().InsertBefore(tree, loResult);
+ }
+ else
+ {
+ assert(count > 32 && count < 64);
+
+ // Move hiOp1 into loResult, do a GT_RSH with count - 32.
+ GenTree* shiftBy = m_compiler->gtNewIconNode(count - 32, TYP_INT);
+ loResult = m_compiler->gtNewOperNode(oper, TYP_INT, hiOp1, shiftBy);
+ Range().InsertBefore(tree, hiOp1, shiftBy, loResult);
+ }
+
+ // Propagate sign bit in hiResult
+ GenTree* shiftBy = m_compiler->gtNewIconNode(31, TYP_INT);
+ GenTree* hiCopy = m_compiler->gtNewLclvNode(hiOp1LclNum, TYP_INT);
+ hiResult = m_compiler->gtNewOperNode(GT_RSH, TYP_INT, hiCopy, shiftBy);
+ Range().InsertBefore(tree, shiftBy, hiCopy, hiResult);
+
+ m_compiler->lvaIncRefCnts(hiCopy);
+ }
+ else
+ {
+ assert(count >= 64);
+
+ // Propagate sign bit in loResult
+ GenTree* hiCopy = m_compiler->gtNewLclvNode(hiOp1LclNum, TYP_INT);
+ GenTree* loShiftBy = m_compiler->gtNewIconNode(31, TYP_INT);
+ loResult = m_compiler->gtNewOperNode(GT_RSH, TYP_INT, hiCopy, loShiftBy);
+ Range().InsertBefore(tree, hiCopy, loShiftBy, loResult);
+
+ // Propagate sign bit in hiResult
+ GenTree* shiftBy = m_compiler->gtNewIconNode(31, TYP_INT);
+ hiResult = m_compiler->gtNewOperNode(GT_RSH, TYP_INT, hiOp1, shiftBy);
+ Range().InsertBefore(tree, shiftBy, hiOp1, hiResult);
+
+ m_compiler->lvaIncRefCnts(hiCopy);
+ }
+ }
+
+ insertAfter = hiResult;
+ }
break;
- default:
- unreached();
+ default:
+ unreached();
+ }
+
+ // Remove tree from Range
+ Range().Remove(tree);
+
+ return FinalizeDecomposition(use, loResult, hiResult, insertAfter);
}
+ else
+ {
+ GenTree* shiftByOp = oldShiftByOp;
+ if (shiftByOp->OperGet() != GT_LCL_VAR)
+ {
+ LIR::Use shiftByUse(Range(), &tree->gtOp.gtOp2, tree);
+ shiftByUse.ReplaceWithLclVar(m_compiler, m_blockWeight);
+ shiftByOp = tree->gtGetOp2();
+ }
+
+ Range().Remove(shiftByOp);
- GenTreeArgList* argList = m_compiler->gtNewArgList(loOp1, hiOp1, shiftWidthOp);
+ unsigned helper;
- GenTree* call = m_compiler->gtNewHelperCallNode(helper, TYP_LONG, 0, argList);
+ switch (oper)
+ {
+ case GT_LSH:
+ helper = CORINFO_HELP_LLSH;
+ break;
+ case GT_RSH:
+ helper = CORINFO_HELP_LRSH;
+ break;
+ case GT_RSZ:
+ helper = CORINFO_HELP_LRSZ;
+ break;
+ default:
+ unreached();
+ }
+
+ GenTreeArgList* argList = m_compiler->gtNewArgList(loOp1, hiOp1, shiftByOp);
- GenTreeCall* callNode = call->AsCall();
- ReturnTypeDesc* retTypeDesc = callNode->GetReturnTypeDesc();
- retTypeDesc->InitializeLongReturnType(m_compiler);
+ GenTree* call = m_compiler->gtNewHelperCallNode(helper, TYP_LONG, 0, argList);
- call = m_compiler->fgMorphArgs(callNode);
- Range().InsertAfter(tree, LIR::SeqTree(m_compiler, call));
+ GenTreeCall* callNode = call->AsCall();
+ ReturnTypeDesc* retTypeDesc = callNode->GetReturnTypeDesc();
+ retTypeDesc->InitializeLongReturnType(m_compiler);
- Range().Remove(tree);
- use.ReplaceWith(m_compiler, call);
- return call;
+ call = m_compiler->fgMorphArgs(callNode);
+ Range().InsertAfter(tree, LIR::SeqTree(m_compiler, call));
+
+ Range().Remove(tree);
+ use.ReplaceWith(m_compiler, call);
+ return call;
+ }
}
//------------------------------------------------------------------------
Range().InsertAfter(loResult, hiResult);
- return FinalizeDecomposition(use, loResult, hiResult);
+ return FinalizeDecomposition(use, loResult, hiResult, hiResult);
}
//------------------------------------------------------------------------
projects / platform / upstream / coreclr.git / commitdiff