JIT: enable implicit tail calls from inlined code (#9405)

Inlines of calls from implicit tail call sites should allow recognition
of inlinee implicit tail call sites.

The jit recognizes implicit tail call sites during importation,
but the inlinee compiler instance did not have compTailCallOpt set
and so never recognized these instances. Fix this and update the logic
to detect the transitively implicit tail calls.

Now that these sites are recognized, morph needs a fix to tunnel through
repeated casts for tail calls, since each level of inlining might add a
cast. All these casts should be identical.

Note under R2R tail calls are not yet recognized (see ZapInfo::canTailCall).

Enable only under FEATURE_TAILCALL_OPT_SHARED_RETURN since the
inline tail call sites are not likely to be in BBJ_RETURN blocks.

Closes #9349.

diff --git a/src/jit/compiler.cpp b/src/jit/compiler.cpp
index 01c7f8d6a733baee8bf9a8e0c65449fe66c7a057..ec6338e540a642294bf86d2c1d4db98afe4b18b9 100644 (file)
--- a/src/jit/compiler.cpp
+++ b/src/jit/compiler.cpp
@@ -3031,13 +3031,31 @@ void Compiler::compInitOptions(JitFlags* jitFlags)
     setUsesSIMDTypes(false);
 #endif // FEATURE_SIMD
 
-    if (compIsForInlining() || compIsForImportOnly())
+    if (compIsForImportOnly())
     {
         return;
     }
+
+#if FEATURE_TAILCALL_OPT
+    // By default opportunistic tail call optimization is enabled.
+    // Recognition is done in the importer so this must be set for
+    // inlinees as well.
+    opts.compTailCallOpt = true;
+#endif // FEATURE_TAILCALL_OPT
+
+    if (compIsForInlining())
+    {
+        return;
+    }
+
     // The rest of the opts fields that we initialize here
     // should only be used when we generate code for the method
     // They should not be used when importing or inlining
+    CLANG_FORMAT_COMMENT_ANCHOR;
+
+#if FEATURE_TAILCALL_OPT
+    opts.compTailCallLoopOpt = true;
+#endif // FEATURE_TAILCALL_OPT
 
     opts.genFPorder = true;
     opts.genFPopt   = true;
@@ -3045,12 +3063,6 @@ void Compiler::compInitOptions(JitFlags* jitFlags)
     opts.instrCount = 0;
     opts.lvRefCount = 0;
 
-#if FEATURE_TAILCALL_OPT
-    // By default opportunistic tail call optimization is enabled
-    opts.compTailCallOpt     = true;
-    opts.compTailCallLoopOpt = true;
-#endif
-
 #ifdef PROFILING_SUPPORTED
     opts.compJitELTHookEnabled = false;
 #endif // PROFILING_SUPPORTED

diff --git a/src/jit/importer.cpp b/src/jit/importer.cpp
index b1e0f487ef1b363ee205666bb111f6655879f027..23bf90513b1215c38926d7f2aa249c54aab4216f 100644 (file)
--- a/src/jit/importer.cpp
+++ b/src/jit/importer.cpp
@@ -12770,11 +12770,31 @@ void Compiler::impImportBlockCode(BasicBlock* block)
                             prefixFlags |= PREFIX_TAILCALL_EXPLICIT;
                         }
                     }
+                }
+
+                // This is split up to avoid goto flow warnings.
+                bool isRecursive;
+                isRecursive = !compIsForInlining() && (callInfo.hMethod == info.compMethodHnd);
 
-                    // Note that when running under tail call stress, a call will be marked as explicit tail prefixed
-                    // hence will not be considered for implicit tail calling.
-                    bool isRecursive = (callInfo.hMethod == info.compMethodHnd);
-                    if (impIsImplicitTailCallCandidate(opcode, codeAddr + sz, codeEndp, prefixFlags, isRecursive))
+                // Note that when running under tail call stress, a call will be marked as explicit tail prefixed
+                // hence will not be considered for implicit tail calling.
+                if (impIsImplicitTailCallCandidate(opcode, codeAddr + sz, codeEndp, prefixFlags, isRecursive))
+                {
+                    if (compIsForInlining())
+                    {
+#if FEATURE_TAILCALL_OPT_SHARED_RETURN
+                        // Are we inlining at an implicit tail call site? If so the we can flag
+                        // implicit tail call sites in the inline body. These call sites
+                        // often end up in non BBJ_RETURN blocks, so only flag them when
+                        // we're able to handle shared returns.
+                        if (impInlineInfo->iciCall->IsImplicitTailCall())
+                        {
+                            JITDUMP(" (Inline Implicit Tail call: prefixFlags |= PREFIX_TAILCALL_IMPLICIT)");
+                            prefixFlags |= PREFIX_TAILCALL_IMPLICIT;
+                        }
+#endif // FEATURE_TAILCALL_OPT_SHARED_RETURN
+                    }
+                    else
                     {
                         JITDUMP(" (Implicit Tail call: prefixFlags |= PREFIX_TAILCALL_IMPLICIT)");
                         prefixFlags |= PREFIX_TAILCALL_IMPLICIT;

diff --git a/src/jit/morph.cpp b/src/jit/morph.cpp
index dabca577102e293cb11e0f0e198f76e514062538..929c302b363e5a563a490c0b0ece46d3faa834bb 100644 (file)
--- a/src/jit/morph.cpp
+++ b/src/jit/morph.cpp
@@ -6735,8 +6735,10 @@ void Compiler::fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result)
         printTreeID(fgMorphStmt);
         printf(" in BB%02u:\n", compCurBB->bbNum);
         gtDispTree(fgMorphStmt);
-
-        // printf("startVars=%d.\n", startVars);
+        if (call->IsImplicitTailCall())
+        {
+            printf("Note: candidate is implicit tail call\n");
+        }
     }
 #endif
 
@@ -7882,14 +7884,15 @@ GenTreePtr Compiler::fgMorphCall(GenTreeCall* call)
             {
                 treeWithCall = stmtExpr->gtGetOp2();
             }
-            if (treeWithCall->gtOper == GT_CAST)
-            {
-                noway_assert(treeWithCall->gtGetOp1() == call && !treeWithCall->gtOverflow());
-            }
-            else
+
+            // Peel off casts
+            while (treeWithCall->gtOper == GT_CAST)
             {
-                noway_assert(treeWithCall == call);
+                noway_assert(!treeWithCall->gtOverflow());
+                treeWithCall = treeWithCall->gtGetOp1();
             }
+
+            noway_assert(treeWithCall == call);
         }
 #endif