JIT: fix layering of profile and model inlining policies (#46612)

Both these policies delegate to the discretionary policy. When that policy
makes a decision, the overlaying policies shouldn't take further action.

In particular, in some test cases we were failing an inline twice for
recursion depth, once via the discretionary policy, and once via the
overlay policy; this triggered an assert.

diff --git a/src/coreclr/jit/inlinepolicy.cpp b/src/coreclr/jit/inlinepolicy.cpp
index 090283d..bd8b363 100644 (file)
--- a/src/coreclr/jit/inlinepolicy.cpp
+++ b/src/coreclr/jit/inlinepolicy.cpp
@@ -2087,6 +2087,11 @@ void ModelPolicy::NoteInt(InlineObservation obs, int value)
     // Let underlying policy do its thing.
     DiscretionaryPolicy::NoteInt(obs, value);
 
+    if (InlDecisionIsFailure(m_Decision))
+    {
+        return;
+    }
+
     // Fail fast for inlinees that are too large to ever inline.
     // The value of 120 is model-dependent; see notes above.
     if (!m_IsForceInline && (obs == InlineObservation::CALLEE_IL_CODE_SIZE) && (value >= 120))
@@ -2095,18 +2100,6 @@ void ModelPolicy::NoteInt(InlineObservation obs, int value)
         SetNever(InlineObservation::CALLEE_TOO_MUCH_IL);
         return;
     }
-
-    // Safeguard against overly deep inlines
-    if (obs == InlineObservation::CALLSITE_DEPTH)
-    {
-        unsigned depthLimit = m_RootCompiler->m_inlineStrategy->GetMaxInlineDepth();
-
-        if (m_CallsiteDepth > depthLimit)
-        {
-            SetFailure(InlineObservation::CALLSITE_IS_TOO_DEEP);
-            return;
-        }
-    }
 }
 
 //------------------------------------------------------------------------
@@ -2271,6 +2264,11 @@ void ProfilePolicy::NoteInt(InlineObservation obs, int value)
     // Let underlying policy do its thing.
     DiscretionaryPolicy::NoteInt(obs, value);
 
+    if (InlDecisionIsFailure(m_Decision))
+    {
+        return;
+    }
+
     // Fail fast for inlinees that are too large to ever inline.
     //
     if (!m_IsForceInline && (obs == InlineObservation::CALLEE_IL_CODE_SIZE) && (value >= 1000))
@@ -2280,18 +2278,6 @@ void ProfilePolicy::NoteInt(InlineObservation obs, int value)
         return;
     }
 
-    // Safeguard against overly deep inlines
-    if (obs == InlineObservation::CALLSITE_DEPTH)
-    {
-        unsigned depthLimit = m_RootCompiler->m_inlineStrategy->GetMaxInlineDepth();
-
-        if (m_CallsiteDepth > depthLimit)
-        {
-            SetFailure(InlineObservation::CALLSITE_IS_TOO_DEEP);
-            return;
-        }
-    }
-
     // This observation happens after we determine profitability
     // so we need to special case it here.
     //