case CEE_STARG:
case CEE_STARG_S:
{
- if (makeInlineObservations)
- {
- // The inliner keeps the args as trees and clones
- // them. Storing the arguments breaks that
- // simplification. To allow this, flag the argument
- // as written to and spill it before inlining. That
- // way the STARG in the inlinee is trivial.
- //
- // Arguably this should be NoteFatal, but the legacy behavior is
- // to ignore this for the prejit root.
- compInlineResult->Note(InlineObservation::CALLEE_STORES_TO_ARGUMENT);
-
- // Fail fast, if inlining
- if (isInlining)
- {
- assert(compInlineResult->IsFailure());
- JITDUMP("INLINER: Inline expansion aborted; opcode at offset [%02u]"
- " writes to an argument\n", codeAddr-codeBegp-1);
- return;
- }
- }
+ noway_assert(sz == sizeof(BYTE) || sz == sizeof(WORD));
- // In non-inline cases, note written-to locals.
- if (!isInlining)
+ if (codeAddr > codeEndp - sz)
{
- noway_assert(sz == sizeof(BYTE) || sz == sizeof(WORD));
-
- if (codeAddr > codeEndp - sz)
- {
- goto TOO_FAR;
- }
-
- varNum = (sz == sizeof(BYTE)) ? getU1LittleEndian(codeAddr)
- : getU2LittleEndian(codeAddr);
- varNum = compMapILargNum(varNum); // account for possible hidden param
-
- // This check is only intended to prevent an AV. Bad varNum values will later
- // be handled properly by the verifier.
- if (varNum < lvaTableCnt)
- {
- lvaTable[varNum].lvArgWrite = 1;
- }
+ goto TOO_FAR;
}
+
+ varNum = (sz == sizeof(BYTE)) ? getU1LittleEndian(codeAddr)
+ : getU2LittleEndian(codeAddr);
+ varNum = compMapILargNum(varNum); // account for possible hidden param
+
+ // This check is only intended to prevent an AV. Bad varNum values will later
+ // be handled properly by the verifier.
+ if (varNum < lvaTableCnt)
+ {
+ if (isInlining)
+ {
+ impInlineInfo->inlArgInfo[varNum].argHasStargOp = true;
+ }
+ else
+ {
+ // In non-inline cases, note written-to locals.
+ lvaTable[varNum].lvArgWrite = 1;
+ }
+ }
}
break;
if (compIsForInlining())
{
+ bool hasReturnBlocks = false;
+ bool hasMoreThanOneReturnBlock = false;
+
+ for (BasicBlock* block = fgFirstBB; block != nullptr; block = block->bbNext)
+ {
+ if (block->bbJumpKind == BBJ_RETURN)
+ {
+ if (hasReturnBlocks)
+ {
+ hasMoreThanOneReturnBlock = true;
+ break;
+ }
+
+ hasReturnBlocks = true;
+ }
+ }
+
+ if (!hasReturnBlocks && !compInlineResult->UsesLegacyPolicy())
+ {
+ //
+ // Mark the call node as "no return". The inliner might ignore CALLEE_DOES_NOT_RETURN and
+ // fail inline for a different reasons. In that case we still want to make the "no return"
+ // information available to the caller as it can impact caller's code quality.
+ //
+
+ impInlineInfo->iciCall->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN;
+ }
+
+ compInlineResult->NoteBool(InlineObservation::CALLEE_DOES_NOT_RETURN, !hasReturnBlocks);
+
+ if (compInlineResult->IsFailure())
+ {
+ return;
+ }
+
noway_assert(info.compXcptnsCount == 0);
compHndBBtab = impInlineInfo->InlinerCompiler->compHndBBtab;
compHndBBtabAllocCount = impInlineInfo->InlinerCompiler->compHndBBtabAllocCount; // we probably only use the table, not add to it.
compHndBBtabCount = impInlineInfo->InlinerCompiler->compHndBBtabCount;
info.compXcptnsCount = impInlineInfo->InlinerCompiler->info.compXcptnsCount;
- if (info.compRetNativeType != TYP_VOID &&
- fgMoreThanOneReturnBlock())
+ if (info.compRetNativeType != TYP_VOID && hasMoreThanOneReturnBlock)
{
// The lifetime of this var might expand multiple BBs. So it is a long lifetime compiler temp.
lvaInlineeReturnSpillTemp = lvaGrabTemp(false DEBUGARG("Inline candidate multiple BBJ_RETURN spill temp"));
if (chkFlags & ~treeFlags)
{
// Print the tree so we can see it in the log.
- printf("Missing flags on tree [%X]: ", tree);
+ printf("Missing flags on tree [%06d]: ", dspTreeID(tree));
GenTree::gtDispFlags(chkFlags & ~treeFlags, GTF_DEBUG_NONE);
printf("\n");
gtDispTree(tree);
noway_assert(!"Missing flags on tree");
// Print the tree again so we can see it right after we hook up the debugger.
- printf("Missing flags on tree [%X]: ", tree);
+ printf("Missing flags on tree [%06d]: ", dspTreeID(tree));
GenTree::gtDispFlags(chkFlags & ~treeFlags, GTF_DEBUG_NONE);
printf("\n");
gtDispTree(tree);
if (argSingleUseNode &&
!(argSingleUseNode->gtFlags & GTF_VAR_CLONED) &&
- !inlArgInfo[argNum].argHasLdargaOp)
+ !inlArgInfo[argNum].argHasLdargaOp &&
+ !inlArgInfo[argNum].argHasStargOp)
{
- /* Change the temp in-place to the actual argument */
-
- argSingleUseNode->CopyFrom(inlArgInfo[argNum].argNode, this);
+ // Change the temp in-place to the actual argument.
+ // We currently do not support this for struct arguments, so it must not be a GT_OBJ.
+ GenTree* argNode = inlArgInfo[argNum].argNode;
+ assert(argNode->gtOper != GT_OBJ);
+ argSingleUseNode->CopyFrom(argNode, this);
continue;
}
else
}
static
+ bool OperIsLocalField(genTreeOps gtOper)
+ {
+ return (gtOper == GT_LCL_FLD ||
+ gtOper == GT_LCL_FLD_ADDR ||
+ gtOper == GT_STORE_LCL_FLD );
+ }
+
+ inline bool OperIsLocalField() const
+ {
+ return OperIsLocalField(gtOper);
+ }
+
+ static
bool OperIsScalarLocal(genTreeOps gtOper)
{
return (gtOper == GT_LCL_VAR ||
// know when these flags are set.
#define GTF_CALL_M_R2R_REL_INDIRECT 0x2000 // GT_CALL -- ready to run call is indirected through a relative address
+ #define GTF_CALL_M_DOES_NOT_RETURN 0x4000 // GT_CALL -- call does not return
bool IsUnmanaged() const { return (gtFlags & GTF_CALL_UNMANAGED) != 0; }
bool NeedsNullCheck() const { return (gtFlags & GTF_CALL_NULLCHECK) != 0; }
bool IsVarargs() const { return (gtCallMoreFlags & GTF_CALL_M_VARARGS) != 0; }
+ bool IsNoReturn() const { return (gtCallMoreFlags & GTF_CALL_M_DOES_NOT_RETURN) != 0; }
+
unsigned short gtCallMoreFlags; // in addition to gtFlags
unsigned char gtCallType :3; // value from the gtCallTypes enumeration
INLINE_OBSERVATION(BEGIN_OPCODE_SCAN, bool, "prepare to look at opcodes", INFORMATION, CALLEE)
INLINE_OBSERVATION(BELOW_ALWAYS_INLINE_SIZE, bool, "below ALWAYS_INLINE size", INFORMATION, CALLEE)
INLINE_OBSERVATION(CLASS_PROMOTABLE, bool, "promotable value class", INFORMATION, CALLEE)
+ INLINE_OBSERVATION(DOES_NOT_RETURN, bool, "does not return", INFORMATION, CALLEE)
INLINE_OBSERVATION(END_OPCODE_SCAN, bool, "done looking at opcodes", INFORMATION, CALLEE)
INLINE_OBSERVATION(HAS_SIMD, bool, "has SIMD arg, local, or ret", INFORMATION, CALLEE)
INLINE_OBSERVATION(HAS_SWITCH, bool, "has switch", INFORMATION, CALLEE)
INLINE_OBSERVATION(NUMBER_OF_BASIC_BLOCKS, int, "number of basic blocks", INFORMATION, CALLEE)
INLINE_OBSERVATION(NUMBER_OF_LOCALS, int, "number of locals", INFORMATION, CALLEE)
INLINE_OBSERVATION(RANDOM_ACCEPT, bool, "random accept", INFORMATION, CALLEE)
-INLINE_OBSERVATION(STORES_TO_ARGUMENT, bool, "stores to argument", INFORMATION, CALLEE)
INLINE_OBSERVATION(UNSUPPORTED_OPCODE, bool, "unsupported opcode", INFORMATION, CALLEE)
// ------ Caller Corectness -------
unsigned argHasTmp :1; // the argument will be evaluated to a temp
unsigned argIsByRefToStructLocal:1; // Is this arg an address of a struct local or a normed struct local or a field in them?
unsigned argHasLdargaOp:1; // Is there LDARGA(s) operation on this argument?
+ unsigned argHasStargOp :1; // Is there STARG(s) operation on this argument?
unsigned argTmpNum; // the argument tmp number
GenTreePtr argNode;
bool hasSIMDTypeArgLocalOrReturn;
#endif // FEATURE_SIMD
- GenTree * iciCall; // The GT_CALL node to be inlined.
+ GenTreeCall * iciCall; // The GT_CALL node to be inlined.
GenTree * iciStmt; // The statement iciCall is in.
BasicBlock * iciBlock; // The basic block iciStmt is in.
};
#endif // defined(DEBUG) || defined(INLINE_DATA)
- InlinePolicy* policy = nullptr;
+ // Optionally install the ModelPolicy.
bool useModelPolicy = JitConfig.JitInlinePolicyModel() != 0;
if (useModelPolicy)
{
- // Optionally install the ModelPolicy.
- policy = new (compiler, CMK_Inlining) ModelPolicy(compiler, isPrejitRoot);
+ return new (compiler, CMK_Inlining) ModelPolicy(compiler, isPrejitRoot);
}
- else
+
+ // Optionally fallback to the original legacy policy
+ bool useLegacyPolicy = JitConfig.JitInlinePolicyLegacy() != 0;
+
+ if (useLegacyPolicy)
{
- // Use the legacy policy
- policy = new (compiler, CMK_Inlining) LegacyPolicy(compiler, isPrejitRoot);
+ return new (compiler, CMK_Inlining) LegacyPolicy(compiler, isPrejitRoot);
}
- return policy;
+ // Use the enhanced legacy policy by default
+ return new (compiler, CMK_Inlining) EnhancedLegacyPolicy(compiler, isPrejitRoot);
}
//------------------------------------------------------------------------
case InlineObservation::CALLEE_HAS_SWITCH:
case InlineObservation::CALLEE_UNSUPPORTED_OPCODE:
- case InlineObservation::CALLEE_STORES_TO_ARGUMENT:
// LegacyPolicy ignores these for prejit roots.
if (!m_IsPrejitRoot)
{
}
}
+ //------------------------------------------------------------------------
+ // NoteBool: handle a boolean observation with non-fatal impact
+ //
+ // Arguments:
+ // obs - the current obsevation
+ // value - the value of the observation
+
+ void EnhancedLegacyPolicy::NoteBool(InlineObservation obs, bool value)
+ {
+ switch (obs)
+ {
+ case InlineObservation::CALLEE_DOES_NOT_RETURN:
+ m_IsNoReturn = value;
+ m_IsNoReturnKnown = true;
+ break;
+
+ default:
+ // Pass all other information to the legacy policy
+ LegacyPolicy::NoteBool(obs, value);
+ break;
+ }
+ }
+
+ //------------------------------------------------------------------------
+ // NoteInt: handle an observed integer value
+ //
+ // Arguments:
+ // obs - the current obsevation
+ // value - the value being observed
+
+ void EnhancedLegacyPolicy::NoteInt(InlineObservation obs, int value)
+ {
+ switch (obs)
+ {
+ case InlineObservation::CALLEE_NUMBER_OF_BASIC_BLOCKS:
+ {
+ assert(value != 0);
+ assert(m_IsNoReturnKnown);
+
+ //
+ // Let's be conservative for now and reject inlining of "no return" methods only
+ // if the callee contains a single basic block. This covers most of the use cases
+ // (typical throw helpers simply do "throw new X();" and so they have a single block)
+ // without affecting more exotic cases (loops that do actual work for example) where
+ // failure to inline could negatively impact code quality.
+ //
+
+ unsigned basicBlockCount = static_cast<unsigned>(value);
+
+ if (m_IsNoReturn && (basicBlockCount == 1))
+ {
+ SetNever(InlineObservation::CALLEE_DOES_NOT_RETURN);
+ }
+ else
+ {
+ LegacyPolicy::NoteInt(obs, value);
+ }
+
+ break;
+ }
+
+ default:
+ // Pass all other information to the legacy policy
+ LegacyPolicy::NoteInt(obs, value);
+ break;
+ }
+ }
+
+ //------------------------------------------------------------------------
+ // PropagateNeverToRuntime: determine if a never result should cause the
+ // method to be marked as un-inlinable.
+
+ bool EnhancedLegacyPolicy::PropagateNeverToRuntime() const
+ {
+ //
+ // Do not propagate the "no return" observation. If we do this then future inlining
+ // attempts will fail immediately without marking the call node as "no return".
+ // This can have an adverse impact on caller's code quality as it may have to preserve
+ // registers across the call.
+ // TODO-Throughput: We should persist the "no return" information in the runtime
+ // so we don't need to re-analyze the inlinee all the time.
+ //
+
+ bool propagate = (m_Observation != InlineObservation::CALLEE_DOES_NOT_RETURN);
+
+ propagate &= LegacyPolicy::PropagateNeverToRuntime();
+
+ return propagate;
+ }
+
#ifdef DEBUG
//------------------------------------------------------------------------
case InlineObservation::CALLEE_HAS_SWITCH:
case InlineObservation::CALLEE_UNSUPPORTED_OPCODE:
- case InlineObservation::CALLEE_STORES_TO_ARGUMENT:
// Pass these on, they should cause inlining to fail.
propagate = true;
break;
arg = gtNewOperNode(GT_ADDR, TYP_BYREF, arg);
addrNode = arg;
- // Get a new Obj node temp to use it as a call argument
+ // Get a new Obj node temp to use it as a call argument.
+ // gtNewObjNode will set the GTF_EXCEPT flag if this is not a local stack object.
arg = gtNewObjNode(lvaGetStruct(tmpVarNum), arg);
- arg->gtFlags |= GTF_EXCEPT;
#endif // not (_TARGET_AMD64_ or _TARGET_ARM64_)
// Create an Obj of the temp to use it as a call argument.
arg = new (this, GT_OBJ) GenTreeObj(originalType, arg, lvaGetStruct(lclCommon->gtLclNum));
- arg->gtFlags |= GTF_EXCEPT;
- flagsSummary |= GTF_EXCEPT;
}
}
}
return result;
}
+ if (call->IsNoReturn())
+ {
+ //
+ // If we know that the call does not return then we can set fgRemoveRestOfBlock
+ // to remove all subsequent statements and change the call's basic block to BBJ_THROW.
+ // As a result the compiler won't need to preserve live registers across the call.
+ //
+ // This isn't need for tail calls as there shouldn't be any code after the call anyway.
+ // Besides, the tail call code is part of the epilog and converting the block to
+ // BBJ_THROW would result in the tail call being dropped as the epilog is generated
+ // only for BBJ_RETURN blocks.
+ //
+ // Currently this doesn't work for non-void callees. Some of the code that handles
+ // fgRemoveRestOfBlock expects the tree to have GTF_EXCEPT flag set but call nodes
+ // do not have this flag by default. We could add the flag here but the proper solution
+ // would be to replace the return expression with a local var node during inlining
+ // so the rest of the call tree stays in a separate statement. That statement can then
+ // be removed by fgRemoveRestOfBlock without needing to add GTF_EXCEPT anywhere.
+ //
+
+ if (!call->IsTailCall() && call->TypeGet() == TYP_VOID)
+ {
+ fgRemoveRestOfBlock = true;
+ }
+ }
return call;
}
}
}
- /* Indirect the dest node */
+ // Indirect the dest node.
dest = gtNewOperNode(GT_IND, type, dest);
- /* As long as we don't have more information about the destination we
- have to assume it could live anywhere (not just in the GC heap). Mark
- the GT_IND node so that we use the correct write barrier helper in case
- the field is a GC ref.
- */
+ // If we have no information about the destination, we have to assume it could
+ // live anywhere (not just in the GC heap).
+ // Mark the GT_IND node so that we use the correct write barrier helper in case
+ // the field is a GC ref.
- dest->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF | GTF_IND_TGTANYWHERE);
+ if (!fgIsIndirOfAddrOfLocal(dest))
+ {
+ dest->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF | GTF_IND_TGTANYWHERE);
+ }
_DoneDest:;
}
}
- /* Indirect the src node */
+ // Indirect the src node.
src = gtNewOperNode(GT_IND, type, src);
- src->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF | GTF_IND_TGTANYWHERE);
+
+ // If we have no information about the src, we have to assume it could
+ // live anywhere (not just in the GC heap).
+ // Mark the GT_IND node so that we use the correct write barrier helper in case
+ // the field is a GC ref.
+
+ if (!fgIsIndirOfAddrOfLocal(src))
+ {
+ src->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF | GTF_IND_TGTANYWHERE);
+ }
_DoneSrc:;
fgAddCodeRef(compCurBB, bbThrowIndex(compCurBB), SCK_ARITH_EXCPN, fgPtrArgCntCur);
break;
+ case GT_OBJ:
+ // If we have GT_OBJ(GT_ADDR(X)) and X has GTF_GLOB_REF, we must set GTF_GLOB_REF on
+ // the GT_OBJ. Note that the GTF_GLOB_REF will have been cleared on ADDR(X) where X
+ // is a local or clsVar, even if it has been address-exposed.
+ if (op1->OperGet() == GT_ADDR)
+ {
+ tree->gtFlags |= (op1->gtGetOp1()->gtFlags & GTF_GLOB_REF);
+ }
+ break;
+
case GT_IND:
// Can not remove a GT_IND if it is currently a CSE candidate.
projects / platform / upstream / coreclr.git / commitdiff