InlineResult* inlineResult);
void impCheckCanInline(GenTreeCall* call,
+ uint8_t candidateIndex,
CORINFO_METHOD_HANDLE fncHandle,
unsigned methAttr,
CORINFO_CONTEXT_HANDLE exactContextHnd,
IL_OFFSET ilOffset);
void impMarkInlineCandidateHelper(GenTreeCall* call,
+ uint8_t candidateIndex,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo,
- IL_OFFSET ilOffset);
+ IL_OFFSET ilOffset,
+ InlineResult* inlineResult);
bool impTailCallRetTypeCompatible(bool allowWidening,
var_types callerRetType,
{
bool inliningFailed = false;
- InlineCandidateInfo* inlCandInfo = call->GetInlineCandidateInfo();
+ InlineCandidateInfo* inlCandInfo = call->GetSingleInlineCandidateInfo();
// Is this call an inline candidate?
if (call->IsInlineCandidate())
{
#ifdef DEBUG
// In debug we always put all inline attempts into the inline tree.
- InlineContext* ctx =
- m_inlineStrategy->NewContext(call->GetInlineCandidateInfo()->inlinersContext, fgMorphStmt, call);
+ InlineContext* ctx = m_inlineStrategy->NewContext(call->GetSingleInlineCandidateInfo()->inlinersContext,
+ fgMorphStmt, call);
ctx->SetFailed(inlineResult);
#endif
}
inlineInfo.hasSIMDTypeArgLocalOrReturn = false;
#endif // FEATURE_SIMD
- InlineCandidateInfo* inlineCandidateInfo = call->GetInlineCandidateInfo();
+ InlineCandidateInfo* inlineCandidateInfo = call->GetSingleInlineCandidateInfo();
noway_assert(inlineCandidateInfo);
// Store the link to inlineCandidateInfo into inlineInfo
inlineInfo.inlineCandidateInfo = inlineCandidateInfo;
}
//-------------------------------------------------------------------------
-// SetSingleInlineCadidateInfo: set a single inline candidate info in the current call.
+// SetSingleInlineCandidateInfo: set a single inline candidate info in the current call.
//
// Arguments:
// candidateInfo - inline candidate info
//
-void GenTreeCall::SetSingleInlineCadidateInfo(InlineCandidateInfo* candidateInfo)
+void GenTreeCall::SetSingleInlineCandidateInfo(InlineCandidateInfo* candidateInfo)
{
if (candidateInfo != nullptr)
{
{
gtInlineInfoCount = 0;
gtFlags &= ~GTF_CALL_INLINE_CANDIDATE;
- gtCallMoreFlags &= ~GTF_CALL_M_GUARDED_DEVIRT;
}
gtInlineCandidateInfo = candidateInfo;
+ ClearGuardedDevirtualizationCandidate();
}
//-------------------------------------------------------------------------
//-------------------------------------------------------------------------
// AddGDVCandidateInfo: Record a guarded devirtualization (GDV) candidate info
-// for this call. For now, we only support one GDV candidate per call.
+// for this call. A call can't have more than MAX_GDV_TYPE_CHECKS number of candidates
//
// Arguments:
+// comp - Compiler instance
// candidateInfo - GDV candidate info
//
-void GenTreeCall::AddGDVCandidateInfo(InlineCandidateInfo* candidateInfo)
+void GenTreeCall::AddGDVCandidateInfo(Compiler* comp, InlineCandidateInfo* candidateInfo)
{
+ assert(gtInlineInfoCount < MAX_GDV_TYPE_CHECKS);
assert(candidateInfo != nullptr);
+
if (gtInlineInfoCount == 0)
{
gtInlineCandidateInfo = candidateInfo;
}
+ else if (gtInlineInfoCount == 1)
+ {
+ gtInlineCandidateInfo =
+ new (comp, CMK_Inlining) InlineCandidateInfo[MAX_GDV_TYPE_CHECKS]{*gtInlineCandidateInfo, *candidateInfo};
+ }
else
{
- // Allocate a fixed list of InlineCandidateInfo structs
- assert(!"multiple GDV candidates are not implemented yet");
+ gtInlineCandidateInfo[gtInlineInfoCount] = *candidateInfo;
}
gtCallMoreFlags |= GTF_CALL_M_GUARDED_DEVIRT;
node->gtCallType = callType;
node->gtCallMethHnd = callHnd;
INDEBUG(node->callSig = nullptr;)
- node->tailCallInfo = nullptr;
- node->gtRetClsHnd = nullptr;
- node->gtControlExpr = nullptr;
- node->gtCallMoreFlags = GTF_CALL_M_EMPTY;
+ node->tailCallInfo = nullptr;
+ node->gtRetClsHnd = nullptr;
+ node->gtControlExpr = nullptr;
+ node->gtCallMoreFlags = GTF_CALL_M_EMPTY;
+ node->gtInlineInfoCount = 0;
if (callType == CT_INDIRECT)
{
printf(" (FramesRoot last use)");
}
- if (((call->gtFlags & GTF_CALL_INLINE_CANDIDATE) != 0) && (call->GetInlineCandidateInfo() != nullptr) &&
- (call->GetInlineCandidateInfo()->exactContextHnd != nullptr))
+ if (((call->gtFlags & GTF_CALL_INLINE_CANDIDATE) != 0) &&
+ (call->GetSingleInlineCandidateInfo() != nullptr) &&
+ (call->GetSingleInlineCandidateInfo()->exactContextHnd != nullptr))
{
- printf(" (exactContextHnd=0x%p)", dspPtr(call->GetInlineCandidateInfo()->exactContextHnd));
+ printf(" (exactContextHnd=0x%p)", dspPtr(call->GetSingleInlineCandidateInfo()->exactContextHnd));
}
gtDispCommonEndLine(tree);
// type class handle in the inline info (for GDV candidates,
// this data is valid only for a correct guess, so we cannot
// use it).
- InlineCandidateInfo* inlInfo = call->GetInlineCandidateInfo();
+ InlineCandidateInfo* inlInfo = call->GetSingleInlineCandidateInfo();
assert(inlInfo != nullptr);
// Grab it as our first cut at a return type.
return (gtCallMoreFlags & GTF_CALL_M_RETBUFFARG_LCLOPT) != 0;
}
- InlineCandidateInfo* GetInlineCandidateInfo()
+ InlineCandidateInfo* GetSingleInlineCandidateInfo()
{
+ // gtInlineInfoCount can be 0 (not an inline candidate) or 1
+ if (gtInlineInfoCount == 0)
+ {
+ assert(!IsInlineCandidate());
+ assert(gtInlineCandidateInfo == nullptr);
+ return nullptr;
+ }
+ else if (gtInlineInfoCount > 1)
+ {
+ assert(!"Call has multiple inline candidates");
+ }
return gtInlineCandidateInfo;
}
- void SetSingleInlineCadidateInfo(InlineCandidateInfo* candidateInfo);
+ void SetSingleInlineCandidateInfo(InlineCandidateInfo* candidateInfo);
- InlineCandidateInfo* GetGDVCandidateInfo(uint8_t index = 0);
+ InlineCandidateInfo* GetGDVCandidateInfo(uint8_t index);
- void AddGDVCandidateInfo(InlineCandidateInfo* candidateInfo);
+ void AddGDVCandidateInfo(Compiler* comp, InlineCandidateInfo* candidateInfo);
void ClearInlineInfo()
{
- SetSingleInlineCadidateInfo(nullptr);
+ SetSingleInlineCandidateInfo(nullptr);
+ }
+
+ uint8_t GetInlineCandidatesCount()
+ {
+ return gtInlineInfoCount;
}
//-----------------------------------------------------------------------------------------
if (tree->OperGet() == GT_RET_EXPR)
{
JITDUMP("\n*** see V%02u = GT_RET_EXPR, noting temp\n", tnum);
- GenTree* call = tree->AsRetExpr()->gtInlineCandidate;
- InlineCandidateInfo* ici = call->AsCall()->GetInlineCandidateInfo();
- ici->preexistingSpillTemp = tnum;
+ GenTreeCall* call = tree->AsRetExpr()->gtInlineCandidate->AsCall();
+ if (call->IsGuardedDevirtualizationCandidate())
+ {
+ for (uint8_t i = 0; i < call->GetInlineCandidatesCount(); i++)
+ {
+ call->GetGDVCandidateInfo(i)->preexistingSpillTemp = tnum;
+ }
+ }
+ else
+ {
+ call->AsCall()->GetSingleInlineCandidateInfo()->preexistingSpillTemp = tnum;
+ }
}
}
GenTreeRetExpr* retExpr = gtNewInlineCandidateReturnExpr(call->AsCall(), genActualType(callRetTyp));
// Link the retExpr to the call so if necessary we can manipulate it later.
- origCall->GetInlineCandidateInfo()->retExpr = retExpr;
+ if (origCall->IsGuardedDevirtualizationCandidate())
+ {
+ for (uint8_t i = 0; i < origCall->GetInlineCandidatesCount(); i++)
+ {
+ origCall->GetGDVCandidateInfo(i)->retExpr = retExpr;
+ }
+ }
+ else
+ {
+ origCall->GetSingleInlineCandidateInfo()->retExpr = retExpr;
+ }
// Propagate retExpr as the placeholder for the call.
call = retExpr;
{
JITDUMP("Considering guarded devirtualization at IL offset %u (0x%x)\n", ilOffset, ilOffset);
+ bool hasPgoData = true;
+
+ CORINFO_CLASS_HANDLE likelyClass = NO_CLASS_HANDLE;
+ CORINFO_METHOD_HANDLE likelyMethod = NO_METHOD_HANDLE;
+ unsigned likelihood = 0;
+
// We currently only get likely class guesses when there is PGO data
// with class profiles.
//
if ((fgPgoClassProfiles == 0) && (fgPgoMethodProfiles == 0))
{
- JITDUMP("Not guessing for class or method: no GDV profile pgo data, or pgo disabled\n");
- return;
+ hasPgoData = false;
+ }
+ else
+ {
+ pickGDV(call, ilOffset, isInterface, &likelyClass, &likelyMethod, &likelihood);
+ if ((likelyClass == NO_CLASS_HANDLE) && (likelyMethod == NO_METHOD_HANDLE))
+ {
+ hasPgoData = false;
+ }
}
- CORINFO_CLASS_HANDLE likelyClass;
- CORINFO_METHOD_HANDLE likelyMethod;
- unsigned likelihood;
- pickGDV(call, ilOffset, isInterface, &likelyClass, &likelyMethod, &likelihood);
+ // NativeAOT is the only target that currently supports getExactClasses-based GDV
+ // where we know the exact number of classes implementing the given base in compile-time.
+ // For now, let's only do this when we don't have any PGO data. In future, we should be able to benefit
+ // from both.
+ if (!hasPgoData && (baseClass != NO_CLASS_HANDLE) && JitConfig.JitEnableExactDevirtualization())
+ {
+ int maxTypeChecks = min(JitConfig.JitGuardedDevirtualizationMaxTypeChecks(), MAX_GDV_TYPE_CHECKS);
- if ((likelyClass == NO_CLASS_HANDLE) && (likelyMethod == NO_METHOD_HANDLE))
+ CORINFO_CLASS_HANDLE exactClasses[MAX_GDV_TYPE_CHECKS];
+ int numExactClasses = info.compCompHnd->getExactClasses(baseClass, MAX_GDV_TYPE_CHECKS, exactClasses);
+ if (numExactClasses == 0)
+ {
+ JITDUMP("No exact classes implementing %s\n", eeGetClassName(baseClass))
+ }
+ else if (numExactClasses > maxTypeChecks)
+ {
+ JITDUMP("Too many exact classes implementing %s (%d > %d)\n", eeGetClassName(baseClass), numExactClasses,
+ maxTypeChecks)
+ }
+ else
+ {
+ assert((numExactClasses > 0) && (numExactClasses <= maxTypeChecks));
+ JITDUMP("We have exactly %d classes implementing %s:\n", numExactClasses, eeGetClassName(baseClass));
+
+ int skipped = 0;
+ for (int exactClsIdx = 0; exactClsIdx < numExactClasses; exactClsIdx++)
+ {
+ CORINFO_CLASS_HANDLE exactCls = exactClasses[exactClsIdx];
+ assert(exactCls != NO_CLASS_HANDLE);
+
+ uint32_t clsAttrs = info.compCompHnd->getClassAttribs(exactCls);
+
+ // The getExactClasses method is expected to return precise data, thus eliminating the need
+ // to check if it is stale.
+ //
+ assert((clsAttrs & CORINFO_FLG_ABSTRACT) == 0);
+
+ JITDUMP(" %d) %s\n", exactClsIdx, eeGetClassName(exactCls));
+
+ // Figure out which method will be called.
+ //
+ CORINFO_DEVIRTUALIZATION_INFO dvInfo;
+ dvInfo.virtualMethod = baseMethod;
+ dvInfo.objClass = exactCls;
+ dvInfo.context = *pContextHandle;
+ dvInfo.exactContext = *pContextHandle;
+ dvInfo.pResolvedTokenVirtualMethod = nullptr;
+
+ if (!info.compCompHnd->resolveVirtualMethod(&dvInfo))
+ {
+ JITDUMP("Can't figure out which method would be invoked, sorry\n");
+ return;
+ }
+
+ CORINFO_METHOD_HANDLE exactMethod = dvInfo.devirtualizedMethod;
+ uint32_t exactMethodAttrs = info.compCompHnd->getMethodAttribs(exactMethod);
+
+ // NOTE: This is currently used only with NativeAOT. In theory, we could also check if we
+ // have static PGO data to decide which class to guess first. Presumably, this is a rare case.
+ //
+ int likelyHood = 100 / numExactClasses;
+
+ // If numExactClasses is 3, then likelyHood is 33 and 33*3=99.
+ // Apply the error to the first guess, so we'll have [34,33,33]
+ if (exactClsIdx == 0)
+ {
+ likelyHood += 100 - likelyHood * numExactClasses;
+ }
+
+ addGuardedDevirtualizationCandidate(call, exactMethod, exactCls, exactMethodAttrs, clsAttrs,
+ likelyHood);
+ }
+ return;
+ }
+ }
+
+ if (!hasPgoData)
{
+ JITDUMP("Not guessing; no PGO and no exact classes\n");
return;
}
}
}
- call->AddGDVCandidateInfo(pInfo);
+ call->AddGDVCandidateInfo(this, pInfo);
}
//------------------------------------------------------------------------
{
GenTreeCall* call = callNode->AsCall();
- // Do the actual evaluation
- impMarkInlineCandidateHelper(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, ilOffset);
+ // Call might not have an inline candidate info yet (will be set by impMarkInlineCandidateHelper)
+ // so we assume there is always a least one candidate:
+ //
+
+ if (call->IsGuardedDevirtualizationCandidate())
+ {
+ const uint8_t candidatesCount = call->GetInlineCandidatesCount();
+ assert(candidatesCount > 0);
+ for (uint8_t candidateId = 0; candidateId < candidatesCount; candidateId++)
+ {
+ InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate", true);
+
+ // Do the actual evaluation
+ impMarkInlineCandidateHelper(call, candidateId, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
+ ilOffset, &inlineResult);
+
+ if (!inlineResult.IsCandidate())
+ {
+ if (candidatesCount > 1)
+ {
+ // TODO: we should not give up if one of the candidates fails to inline while others succeed.
+ //
+ JITDUMP(
+ "We had multiple inline candidates but have to give up on them since one of them didn't pass"
+ "inline checks")
+ call->ClearInlineInfo();
+ call->ClearGuardedDevirtualizationCandidate();
+ }
+ break;
+ }
+ }
+ }
+ else
+ {
+ const uint8_t candidatesCount = call->GetInlineCandidatesCount();
+ assert(candidatesCount <= 1);
+ InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate", true);
+ impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, ilOffset,
+ &inlineResult);
+ }
// If this call is an inline candidate or is not a guarded devirtualization
// candidate, we're done.
//
// Arguments:
// callNode -- call under scrutiny
+// candidateIndex -- index of the inline candidate to evaluate
// exactContextHnd -- context handle for inlining
// exactContextNeedsRuntimeLookup -- true if context required runtime lookup
// callInfo -- call info from VM
// filled in the associated InlineCandidateInfo.
//
// If callNode is not an inline candidate, and the reason is
-// something that is inherent to the method being called, the
// method may be marked as "noinline" to short-circuit any
// future assessments of calls to this method.
+//
void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call,
+ uint8_t candidateIndex,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo,
- IL_OFFSET ilOffset)
+ IL_OFFSET ilOffset,
+ InlineResult* inlineResult)
{
// Let the strategy know there's another call
impInlineRoot()->m_inlineStrategy->NoteCall();
return;
}
- InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate");
-
// Don't inline if not optimizing root method
if (opts.compDbgCode)
{
- inlineResult.NoteFatal(InlineObservation::CALLER_DEBUG_CODEGEN);
+ inlineResult->NoteFatal(InlineObservation::CALLER_DEBUG_CODEGEN);
return;
}
// Don't inline if inlining into this method is disabled.
if (impInlineRoot()->m_inlineStrategy->IsInliningDisabled())
{
- inlineResult.NoteFatal(InlineObservation::CALLER_IS_JIT_NOINLINE);
+ inlineResult->NoteFatal(InlineObservation::CALLER_IS_JIT_NOINLINE);
return;
}
// Don't inline into callers that use the NextCallReturnAddress intrinsic.
if (info.compHasNextCallRetAddr)
{
- inlineResult.NoteFatal(InlineObservation::CALLER_USES_NEXT_CALL_RET_ADDR);
+ inlineResult->NoteFatal(InlineObservation::CALLER_USES_NEXT_CALL_RET_ADDR);
return;
}
// Inlining takes precedence over implicit tail call optimization (if the call is not directly recursive).
if (call->IsTailPrefixedCall())
{
- inlineResult.NoteFatal(InlineObservation::CALLSITE_EXPLICIT_TAIL_PREFIX);
+ inlineResult->NoteFatal(InlineObservation::CALLSITE_EXPLICIT_TAIL_PREFIX);
return;
}
// Don't even bother trying to inline it.
if (call->IsDelegateInvoke() && !call->IsGuardedDevirtualizationCandidate())
{
- inlineResult.NoteFatal(InlineObservation::CALLEE_HAS_NO_BODY);
+ inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NO_BODY);
return;
}
// as a fast tail call or turned into a loop.
if (gtIsRecursiveCall(call) && call->IsImplicitTailCall())
{
- inlineResult.NoteFatal(InlineObservation::CALLSITE_IMPLICIT_REC_TAIL_CALL);
+ inlineResult->NoteFatal(InlineObservation::CALLSITE_IMPLICIT_REC_TAIL_CALL);
return;
}
// but reject all other virtual calls.
if (!call->IsGuardedDevirtualizationCandidate())
{
- inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_NOT_DIRECT);
+ inlineResult->NoteFatal(InlineObservation::CALLSITE_IS_NOT_DIRECT);
return;
}
}
if (call->gtCallType == CT_HELPER)
{
assert(!call->IsGuardedDevirtualizationCandidate());
- inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_CALL_TO_HELPER);
+ inlineResult->NoteFatal(InlineObservation::CALLSITE_IS_CALL_TO_HELPER);
return;
}
/* Ignore indirect calls */
if (call->gtCallType == CT_INDIRECT)
{
- inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_NOT_DIRECT_MANAGED);
+ inlineResult->NoteFatal(InlineObservation::CALLSITE_IS_NOT_DIRECT_MANAGED);
return;
}
if (call->IsGuardedDevirtualizationCandidate())
{
- if (call->GetGDVCandidateInfo()->guardedMethodUnboxedEntryHandle != nullptr)
+ if (call->GetGDVCandidateInfo(candidateIndex)->guardedMethodUnboxedEntryHandle != nullptr)
{
- fncHandle = call->GetGDVCandidateInfo()->guardedMethodUnboxedEntryHandle;
+ fncHandle = call->GetGDVCandidateInfo(candidateIndex)->guardedMethodUnboxedEntryHandle;
}
else
{
- fncHandle = call->GetGDVCandidateInfo()->guardedMethodHandle;
+ fncHandle = call->GetGDVCandidateInfo(candidateIndex)->guardedMethodHandle;
}
methAttr = info.compCompHnd->getMethodAttribs(fncHandle);
}
#endif
- inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_CATCH);
+ inlineResult->NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_CATCH);
return;
}
}
#endif
- inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_FILTER);
+ inlineResult->NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_FILTER);
return;
}
}
if (methAttr & CORINFO_FLG_DONT_INLINE)
{
- inlineResult.NoteFatal(InlineObservation::CALLEE_IS_NOINLINE);
+ inlineResult->NoteFatal(InlineObservation::CALLEE_IS_NOINLINE);
return;
}
if (methAttr & CORINFO_FLG_SYNCH)
{
- inlineResult.NoteFatal(InlineObservation::CALLEE_IS_SYNCHRONIZED);
+ inlineResult->NoteFatal(InlineObservation::CALLEE_IS_SYNCHRONIZED);
return;
}
BasicBlock* block = compIsForInlining() ? impInlineInfo->iciBlock : compCurBB;
if (!impCanPInvokeInlineCallSite(block))
{
- inlineResult.NoteFatal(InlineObservation::CALLSITE_PINVOKE_EH);
+ inlineResult->NoteFatal(InlineObservation::CALLSITE_PINVOKE_EH);
return;
}
}
InlineCandidateInfo* inlineCandidateInfo = nullptr;
- impCheckCanInline(call, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, &inlineResult);
+ impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, inlineResult);
- if (inlineResult.IsFailure())
+ if (inlineResult->IsFailure())
{
return;
}
// The old value should be null OR this call should be a guarded devirtualization candidate.
- assert((call->GetInlineCandidateInfo() == nullptr) || call->IsGuardedDevirtualizationCandidate());
+ assert(call->IsGuardedDevirtualizationCandidate() || (call->GetSingleInlineCandidateInfo() == nullptr));
// The new value should not be null.
assert(inlineCandidateInfo != nullptr);
inlineCandidateInfo->exactContextNeedsRuntimeLookup = exactContextNeedsRuntimeLookup;
inlineCandidateInfo->ilOffset = ilOffset;
- call->SetSingleInlineCadidateInfo(inlineCandidateInfo);
// If we're in an inlinee compiler, and have a return spill temp, and this inline candidate
// is also a tail call candidate, it can use the same return spill temp.
inlineCandidateInfo->preexistingSpillTemp);
}
+ if (call->IsGuardedDevirtualizationCandidate())
+ {
+ assert(call->GetGDVCandidateInfo(candidateIndex) == inlineCandidateInfo);
+ call->gtFlags |= GTF_CALL_INLINE_CANDIDATE;
+ }
+ else
+ {
+ assert(candidateIndex == 0);
+ call->SetSingleInlineCandidateInfo(inlineCandidateInfo);
+ }
+
// Let the strategy know there's another candidate.
impInlineRoot()->m_inlineStrategy->NoteCandidate();
// Since we're not actually inlining yet, and this call site is
// still just an inline candidate, there's nothing to report.
- inlineResult.SetSuccessResult(INLINE_CHECK_CAN_INLINE_SUCCESS);
+ inlineResult->SetSuccessResult(INLINE_CHECK_CAN_INLINE_SUCCESS);
}
/******************************************************************************/
//
// Arguments:
// call - inline candidate
+// candidateIndex - index of inline candidate in the call's inline candidate list
// fncHandle - method that will be called
// methAttr - attributes for the method
// exactContextHnd - exact context for the method
// status (if method cannot be inlined)
//
void Compiler::impCheckCanInline(GenTreeCall* call,
+ uint8_t candidateIndex,
CORINFO_METHOD_HANDLE fncHandle,
unsigned methAttr,
CORINFO_CONTEXT_HANDLE exactContextHnd,
{
Compiler* pThis;
GenTreeCall* call;
+ uint8_t candidateIndex;
CORINFO_METHOD_HANDLE fncHandle;
unsigned methAttr;
CORINFO_CONTEXT_HANDLE exactContextHnd;
param.pThis = this;
param.call = call;
+ param.candidateIndex = candidateIndex;
param.fncHandle = fncHandle;
param.methAttr = methAttr;
param.exactContextHnd = (exactContextHnd != nullptr) ? exactContextHnd : MAKE_METHODCONTEXT(fncHandle);
if (pParam->call->IsGuardedDevirtualizationCandidate())
{
- pInfo = pParam->call->GetInlineCandidateInfo();
+ pInfo = pParam->call->GetGDVCandidateInfo(pParam->candidateIndex);
}
else
{
FixupRetExpr();
ClearFlag();
CreateRemainder();
- CreateCheck();
- CreateThen();
+ assert(GetChecksCount() > 0);
+ for (uint8_t i = 0; i < GetChecksCount(); i++)
+ {
+ CreateCheck(i);
+ CreateThen(i);
+ }
CreateElse();
RemoveOldStatement();
SetWeights();
remainderBlock->bbFlags |= BBF_INTERNAL;
}
- virtual void CreateCheck() = 0;
+ virtual void CreateCheck(uint8_t checkIdx) = 0;
//------------------------------------------------------------------------
// CreateAndInsertBasicBlock: ask compiler to create new basic block.
return block;
}
- virtual void CreateThen() = 0;
- virtual void CreateElse() = 0;
+ virtual void CreateThen(uint8_t checkIdx) = 0;
+ virtual void CreateElse() = 0;
+
+ //------------------------------------------------------------------------
+ // GetChecksCount: Get number of Check-Then pairs
+ //
+ virtual UINT8 GetChecksCount()
+ {
+ return 1;
+ }
//------------------------------------------------------------------------
// RemoveOldStatement: remove original stmt from current block.
//------------------------------------------------------------------------
// CreateCheck: create check block, that checks fat pointer bit set.
//
- virtual void CreateCheck()
+ virtual void CreateCheck(uint8_t checkIdx)
{
+ assert(checkIdx == 0);
+
checkBlock = CreateAndInsertBasicBlock(BBJ_COND, currBlock);
GenTree* fatPointerMask = new (compiler, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, FAT_POINTER_MASK);
GenTree* fptrAddressCopy = compiler->gtCloneExpr(fptrAddress);
// CreateThen: create then block, that is executed if the check succeeds.
// This simply executes the original call.
//
- virtual void CreateThen()
+ virtual void CreateThen(uint8_t checkIdx)
{
thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock);
Statement* copyOfOriginalStmt = compiler->gtCloneStmt(stmt);
return;
}
- likelihood = origCall->GetGDVCandidateInfo()->likelihood;
+ likelihood = origCall->GetGDVCandidateInfo(0)->likelihood;
assert((likelihood >= 0) && (likelihood <= 100));
JITDUMP("Likelihood of correct guess is %u\n", likelihood);
- const bool isChainedGdv = (origCall->gtCallMoreFlags & GTF_CALL_M_GUARDED_DEVIRT_CHAIN) != 0;
-
- if (isChainedGdv)
+ // TODO: implement chaining for multiple GDV candidates
+ const bool canChainGdv = GetChecksCount() == 1;
+ if (canChainGdv)
{
- JITDUMP("Expansion will chain to the previous GDV\n");
- }
+ const bool isChainedGdv = (origCall->gtCallMoreFlags & GTF_CALL_M_GUARDED_DEVIRT_CHAIN) != 0;
- Transform();
+ if (isChainedGdv)
+ {
+ JITDUMP("Expansion will chain to the previous GDV\n");
+ }
- if (isChainedGdv)
+ Transform();
+
+ if (isChainedGdv)
+ {
+ TransformForChainedGdv();
+ }
+
+ // Look ahead and see if there's another Gdv we might chain to this one.
+ //
+ ScoutForChainedGdv();
+ }
+ else
{
- TransformForChainedGdv();
+ JITDUMP("Expansion will not chain to the previous GDV due to multiple type checks\n");
+ Transform();
}
-
- // Look ahead and see if there's another Gdv we might chain to this one.
- //
- ScoutForChainedGdv();
}
protected:
origCall->ClearGuardedDevirtualizationCandidate();
}
+ virtual UINT8 GetChecksCount()
+ {
+ return origCall->GetInlineCandidatesCount();
+ }
+
+ virtual void ChainFlow()
+ {
+ assert(compiler->fgPredsComputed);
+
+ // currBlock
+ compiler->fgRemoveRefPred(remainderBlock, currBlock);
+
+ // The rest of chaining is done in-place.
+ }
+
+ virtual void SetWeights()
+ {
+ // remainderBlock has the same weight as the original block.
+ remainderBlock->inheritWeight(currBlock);
+
+ // The rest of the weights are assigned in-place.
+ }
+
//------------------------------------------------------------------------
// CreateCheck: create check block and check method table
//
- virtual void CreateCheck()
+ virtual void CreateCheck(uint8_t checkIdx)
{
- // There's no need for a new block here. We can just append to currBlock.
- //
- checkBlock = currBlock;
- checkBlock->bbJumpKind = BBJ_COND;
+ if (checkIdx == 0)
+ {
+ // There's no need for a new block here. We can just append to currBlock.
+ //
+ checkBlock = currBlock;
+ checkBlock->bbJumpKind = BBJ_COND;
+ }
+ else
+ {
+ // In case of multiple checks, append to the previous thenBlock block
+ BasicBlock* prevCheckBlock = checkBlock;
+ checkBlock = CreateAndInsertBasicBlock(BBJ_COND, thenBlock);
+
+ // prevCheckBlock is expected to jump to this new check (if its type check doesn't succeed)
+ prevCheckBlock->bbJumpDest = checkBlock;
+ compiler->fgAddRefPred(checkBlock, prevCheckBlock);
+
+ // Weight for the new secondary check is the difference between the previous check and the thenBlock.
+ checkBlock->inheritWeightPercentage(prevCheckBlock,
+ 100 - origCall->GetGDVCandidateInfo(checkIdx)->likelihood);
+ }
// Find last arg with a side effect. All args with any effect
// before that will need to be spilled.
//
lastStmt = checkBlock->lastStmt();
- InlineCandidateInfo* guardedInfo = origCall->GetGDVCandidateInfo();
+ InlineCandidateInfo* guardedInfo = origCall->GetGDVCandidateInfo(checkIdx);
// Create comparison. On success we will jump to do the indirect call.
GenTree* compare;
//
// Note implicit by-ref returns should have already been converted
// so any struct copy we induce here should be cheap.
- InlineCandidateInfo* const inlineInfo = origCall->GetInlineCandidateInfo();
+ InlineCandidateInfo* const inlineInfo = origCall->GetGDVCandidateInfo(0);
if (!origCall->TypeIs(TYP_VOID))
{
}
//------------------------------------------------------------------------
- // CreateThen: create then block with direct call to method
+ // Devirtualize origCall using the given inline candidate
//
- virtual void CreateThen()
+ void DevirtualizeCall(BasicBlock* block, uint8_t candidateId)
{
- thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock);
- thenBlock->bbFlags |= currBlock->bbFlags & BBF_SPLIT_GAINED;
- InlineCandidateInfo* inlineInfo = origCall->GetInlineCandidateInfo();
+ InlineCandidateInfo* inlineInfo = origCall->GetGDVCandidateInfo(candidateId);
CORINFO_CLASS_HANDLE clsHnd = inlineInfo->guardedClassHandle;
//
compiler->info.compCompHnd->getMethodClass(inlineInfo->guardedMethodHandle));
}
- compiler->fgNewStmtAtEnd(thenBlock, store);
+ compiler->fgNewStmtAtEnd(block, store);
// Clone call for the devirtualized case. Note we must use the
// special candidate helper and we need to use the new 'this'.
call->gtArgs.GetThisArg()->SetEarlyNode(compiler->gtNewLclvNode(thisTemp, TYP_REF));
call->SetIsGuarded();
- JITDUMP("Direct call [%06u] in block " FMT_BB "\n", compiler->dspTreeID(call), thenBlock->bbNum);
+ JITDUMP("Direct call [%06u] in block " FMT_BB "\n", compiler->dspTreeID(call), block->bbNum);
CORINFO_METHOD_HANDLE methodHnd = call->gtCallMethHnd;
CORINFO_CONTEXT_HANDLE context = inlineInfo->exactContextHnd;
if (returnTemp != BAD_VAR_NUM)
{
GenTree* const store = compiler->gtNewTempStore(returnTemp, call);
- compiler->fgNewStmtAtEnd(thenBlock, store);
+ compiler->fgNewStmtAtEnd(block, store);
}
else
{
- compiler->fgNewStmtAtEnd(thenBlock, call, stmt->GetDebugInfo());
+ compiler->fgNewStmtAtEnd(block, call, stmt->GetDebugInfo());
}
}
else
{
// Add the call.
//
- compiler->fgNewStmtAtEnd(thenBlock, call, stmt->GetDebugInfo());
+ compiler->fgNewStmtAtEnd(block, call, stmt->GetDebugInfo());
// Re-establish this call as an inline candidate.
//
inlineInfo->clsHandle = compiler->info.compCompHnd->getMethodClass(methodHnd);
inlineInfo->exactContextHnd = context;
inlineInfo->preexistingSpillTemp = returnTemp;
- call->SetSingleInlineCadidateInfo(inlineInfo);
+ call->SetSingleInlineCandidateInfo(inlineInfo);
// If there was a ret expr for this call, we need to create a new one
// and append it just after the call.
// We should always have a return temp if we return results by value
assert(origCall->TypeGet() == TYP_VOID);
}
- compiler->fgNewStmtAtEnd(thenBlock, newRetExpr);
+ compiler->fgNewStmtAtEnd(block, newRetExpr);
}
}
}
//------------------------------------------------------------------------
+ // CreateThen: create then block with direct call to method
+ //
+ virtual void CreateThen(uint8_t checkIdx)
+ {
+ thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock);
+ thenBlock->bbFlags |= currBlock->bbFlags & BBF_SPLIT_GAINED;
+ thenBlock->bbJumpDest = remainderBlock;
+ thenBlock->inheritWeightPercentage(currBlock, origCall->GetGDVCandidateInfo(checkIdx)->likelihood);
+
+ // thenBlock always jumps to remainderBlock. Also, it has a single pred - last checkBlock
+ thenBlock->bbJumpDest = remainderBlock;
+ compiler->fgAddRefPred(thenBlock, checkBlock);
+ compiler->fgAddRefPred(remainderBlock, thenBlock);
+
+ DevirtualizeCall(thenBlock, checkIdx);
+ }
+
+ //------------------------------------------------------------------------
// CreateElse: create else block. This executes the original indirect call.
//
virtual void CreateElse()
{
elseBlock = CreateAndInsertBasicBlock(BBJ_NONE, thenBlock);
elseBlock->bbFlags |= currBlock->bbFlags & BBF_SPLIT_GAINED;
+
+ // elseBlock always flows into remainderBlock
+ checkBlock->bbJumpDest = elseBlock;
+ compiler->fgAddRefPred(remainderBlock, elseBlock);
+ compiler->fgAddRefPred(elseBlock, checkBlock);
+
+ // Calculate the likelihood of the else block as a remainder of the sum
+ // of all the other likelihoods.
+ unsigned elseLikelihood = 100;
+ for (uint8_t i = 0; i < origCall->GetInlineCandidatesCount(); i++)
+ {
+ elseLikelihood -= origCall->GetGDVCandidateInfo(i)->likelihood;
+ }
+ // Make sure it didn't overflow
+ assert(elseLikelihood <= 100);
+
+ elseBlock->inheritWeightPercentage(currBlock, elseLikelihood);
+
GenTreeCall* call = origCall;
Statement* newStmt = compiler->gtNewStmt(call, stmt->GetDebugInfo());
GenTreeCall* const call = root->AsCall();
if (call->IsGuardedDevirtualizationCandidate() &&
- (call->GetGDVCandidateInfo()->likelihood >= gdvChainLikelihood))
+ (call->GetGDVCandidateInfo(0)->likelihood >= gdvChainLikelihood))
{
JITDUMP("GDV call at [%06u] has likelihood %u >= %u; chaining (%u stmts, %u nodes to dup).\n",
- compiler->dspTreeID(call), call->GetGDVCandidateInfo()->likelihood, gdvChainLikelihood,
+ compiler->dspTreeID(call), call->GetGDVCandidateInfo(0)->likelihood, gdvChainLikelihood,
chainStatementDup, chainNodeDup);
call->gtCallMoreFlags |= GTF_CALL_M_GUARDED_DEVIRT_CHAIN;
if (IsFailure() && (m_Call != nullptr))
{
// compiler should have revoked candidacy on the call by now
- assert((m_Call->gtFlags & GTF_CALL_INLINE_CANDIDATE) == 0);
+ // Unless it's a call has both failed and successful candidates (GDV candidates)
+ assert(((m_Call->gtFlags & GTF_CALL_INLINE_CANDIDATE) == 0) || m_Call->IsGuardedDevirtualizationCandidate());
if (m_Call->gtInlineObservation == InlineObservation::CALLEE_UNUSED_INITIAL)
{
if (call->IsInlineCandidate())
{
- InlineCandidateInfo* info = call->GetInlineCandidateInfo();
+ InlineCandidateInfo* info = call->GetSingleInlineCandidateInfo();
context->m_Code = info->methInfo.ILCode;
context->m_ILSize = info->methInfo.ILCodeSize;
context->m_ActualCallOffset = info->ilOffset;
// Overall master enable for Guarded Devirtualization.
CONFIG_INTEGER(JitEnableGuardedDevirtualization, W("JitEnableGuardedDevirtualization"), 1)
+#define MAX_GDV_TYPE_CHECKS 5
+// Number of types to probe for polymorphic virtual call-sites to devirtualize them,
+// Max number is MAX_GDV_TYPE_CHECKS defined above ^
+CONFIG_INTEGER(JitGuardedDevirtualizationMaxTypeChecks, W("JitGuardedDevirtualizationMaxTypeChecks"), 3)
+
// Various policies for GuardedDevirtualization
CONFIG_INTEGER(JitGuardedDevirtualizationChainLikelihood, W("JitGuardedDevirtualizationChainLikelihood"), 0x4B) // 75
CONFIG_INTEGER(JitGuardedDevirtualizationChainStatements, W("JitGuardedDevirtualizationChainStatements"), 4)
projects / platform / upstream / dotnet / runtime.git / commitdiff