From 372ff23a6cccdd98c2abdd02fdd22a2dff56d7de Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 10 Jul 2020 03:33:06 +0200 Subject: [PATCH] Move tailcall dispatcher into corelib (#38938) Fix #35559 --- .../CompilerServices/RuntimeHelpers.CoreCLR.cs | 36 +++- src/coreclr/src/vm/dllimport.h | 5 +- src/coreclr/src/vm/ilstubcache.cpp | 5 - src/coreclr/src/vm/ilstubresolver.cpp | 1 - src/coreclr/src/vm/ilstubresolver.h | 1 - src/coreclr/src/vm/jitinterface.cpp | 2 +- src/coreclr/src/vm/mscorlib.h | 5 +- src/coreclr/src/vm/tailcallhelp.cpp | 227 ++------------------- src/coreclr/src/vm/tailcallhelp.h | 2 +- 9 files changed, 56 insertions(+), 228 deletions(-) diff --git a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index 1e516e8..5c9a38e 100644 --- a/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -292,6 +292,40 @@ namespace System.Runtime.CompilerServices [MethodImpl(MethodImplOptions.InternalCall)] private static unsafe extern TailCallTls* GetTailCallInfo(IntPtr retAddrSlot, IntPtr* retAddr); + private static unsafe void DispatchTailCalls( + IntPtr callersRetAddrSlot, + delegate* callTarget, + IntPtr retVal) + { + IntPtr callersRetAddr; + TailCallTls* tls = GetTailCallInfo(callersRetAddrSlot, &callersRetAddr); + PortableTailCallFrame* prevFrame = tls->Frame; + if (callersRetAddr == prevFrame->TailCallAwareReturnAddress) + { + prevFrame->NextCall = callTarget; + return; + } + + PortableTailCallFrame newFrame; + newFrame.Prev = prevFrame; + + try + { + tls->Frame = &newFrame; + + do + { + newFrame.NextCall = null; + callTarget(tls->ArgBuffer, retVal, &newFrame.TailCallAwareReturnAddress); + callTarget = newFrame.NextCall; + } while (callTarget != null); + } + finally + { + tls->Frame = prevFrame; + } + } + [MethodImpl(MethodImplOptions.InternalCall)] internal static extern long GetILBytesJitted(); @@ -439,7 +473,7 @@ namespace System.Runtime.CompilerServices { public PortableTailCallFrame* Prev; public IntPtr TailCallAwareReturnAddress; - public IntPtr NextCall; + public delegate* NextCall; } [StructLayout(LayoutKind.Sequential)] diff --git a/src/coreclr/src/vm/dllimport.h b/src/coreclr/src/vm/dllimport.h index b7a1435..b24ab6e 100644 --- a/src/coreclr/src/vm/dllimport.h +++ b/src/coreclr/src/vm/dllimport.h @@ -191,7 +191,6 @@ enum ILStubTypes ILSTUB_WRAPPERDELEGATE_INVOKE = 0x80000007, ILSTUB_TAILCALL_STOREARGS = 0x80000008, ILSTUB_TAILCALL_CALLTARGET = 0x80000009, - ILSTUB_TAILCALL_DISPATCH = 0x8000000A, }; #ifdef FEATURE_COMINTEROP @@ -231,7 +230,6 @@ inline bool SF_IsInstantiatingStub (DWORD dwStubFlags) { LIMITED_METHOD_CON #endif inline bool SF_IsTailCallStoreArgsStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags == ILSTUB_TAILCALL_STOREARGS); } inline bool SF_IsTailCallCallTargetStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags == ILSTUB_TAILCALL_CALLTARGET); } -inline bool SF_IsTailCallDispatcherStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags == ILSTUB_TAILCALL_DISPATCH); } inline bool SF_IsCOMStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return COM_ONLY(dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_COM)); } inline bool SF_IsCOMLateBoundStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return COM_ONLY(dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_COMLATEBOUND)); } @@ -243,8 +241,7 @@ inline bool SF_IsSharedStub(DWORD dwStubFlags) { WRAPPER_NO_CONTRACT; - if (SF_IsTailCallStoreArgsStub(dwStubFlags) || SF_IsTailCallCallTargetStub(dwStubFlags) || - SF_IsTailCallDispatcherStub(dwStubFlags)) + if (SF_IsTailCallStoreArgsStub(dwStubFlags) || SF_IsTailCallCallTargetStub(dwStubFlags)) { return false; } diff --git a/src/coreclr/src/vm/ilstubcache.cpp b/src/coreclr/src/vm/ilstubcache.cpp index 4d24f19..fb9e0ea 100644 --- a/src/coreclr/src/vm/ilstubcache.cpp +++ b/src/coreclr/src/vm/ilstubcache.cpp @@ -256,11 +256,6 @@ MethodDesc* ILStubCache::CreateNewMethodDesc(LoaderHeap* pCreationHeap, MethodTa pMD->GetILStubResolver()->SetStubType(ILStubResolver::TailCallCallTargetStub); } else - if (SF_IsTailCallDispatcherStub(dwStubFlags)) - { - pMD->GetILStubResolver()->SetStubType(ILStubResolver::TailCallDispatcherStub); - } - else #ifdef FEATURE_COMINTEROP if (SF_IsCOMStub(dwStubFlags)) { diff --git a/src/coreclr/src/vm/ilstubresolver.cpp b/src/coreclr/src/vm/ilstubresolver.cpp index 4dfd747..2df8f54 100644 --- a/src/coreclr/src/vm/ilstubresolver.cpp +++ b/src/coreclr/src/vm/ilstubresolver.cpp @@ -88,7 +88,6 @@ LPCUTF8 ILStubResolver::GetStubMethodName() case WrapperDelegateStub: return "IL_STUB_WrapperDelegate_Invoke"; case TailCallStoreArgsStub: return "IL_STUB_StoreTailCallArgs"; case TailCallCallTargetStub: return "IL_STUB_CallTailCallTarget"; - case TailCallDispatcherStub: return "IL_STUB_DispatchTailCalls"; default: UNREACHABLE_MSG("Unknown stub type"); } diff --git a/src/coreclr/src/vm/ilstubresolver.h b/src/coreclr/src/vm/ilstubresolver.h index 1c46a3a..b60721d 100644 --- a/src/coreclr/src/vm/ilstubresolver.h +++ b/src/coreclr/src/vm/ilstubresolver.h @@ -93,7 +93,6 @@ public: #endif TailCallStoreArgsStub, TailCallCallTargetStub, - TailCallDispatcherStub, }; ILStubType GetStubType(); diff --git a/src/coreclr/src/vm/jitinterface.cpp b/src/coreclr/src/vm/jitinterface.cpp index 9692938..99073be 100644 --- a/src/coreclr/src/vm/jitinterface.cpp +++ b/src/coreclr/src/vm/jitinterface.cpp @@ -13939,7 +13939,7 @@ bool CEEInfo::getTailCallHelpersInternal(CORINFO_RESOLVED_TOKEN* callToken, pResult->flags = (CORINFO_TAILCALL_HELPERS_FLAGS)outFlags; pResult->hStoreArgs = (CORINFO_METHOD_HANDLE)pStoreArgsMD; pResult->hCallTarget = (CORINFO_METHOD_HANDLE)pCallTargetMD; - pResult->hDispatcher = (CORINFO_METHOD_HANDLE)TailCallHelp::GetOrCreateTailCallDispatcherMD(); + pResult->hDispatcher = (CORINFO_METHOD_HANDLE)TailCallHelp::GetOrLoadTailCallDispatcherMD(); return true; } diff --git a/src/coreclr/src/vm/mscorlib.h b/src/coreclr/src/vm/mscorlib.h index d6ee220..fd2578f 100644 --- a/src/coreclr/src/vm/mscorlib.h +++ b/src/coreclr/src/vm/mscorlib.h @@ -706,9 +706,10 @@ DEFINE_METHOD(RUNTIME_HELPERS, GET_RAW_ARRAY_DATA, GetRawArrayData, No DEFINE_METHOD(RUNTIME_HELPERS, GET_UNINITIALIZED_OBJECT, GetUninitializedObject, NoSig) DEFINE_METHOD(RUNTIME_HELPERS, ENUM_EQUALS, EnumEquals, NoSig) DEFINE_METHOD(RUNTIME_HELPERS, ENUM_COMPARE_TO, EnumCompareTo, NoSig) -DEFINE_METHOD(RUNTIME_HELPERS, ALLOC_TAILCALL_ARG_BUFFER, AllocTailCallArgBuffer, SM_Int_IntPtr_RetIntPtr) +DEFINE_METHOD(RUNTIME_HELPERS, ALLOC_TAILCALL_ARG_BUFFER, AllocTailCallArgBuffer, SM_Int_IntPtr_RetIntPtr) DEFINE_METHOD(RUNTIME_HELPERS, GET_TAILCALL_INFO, GetTailCallInfo, NoSig) -DEFINE_METHOD(RUNTIME_HELPERS, FREE_TAILCALL_ARG_BUFFER, FreeTailCallArgBuffer, SM_RetVoid) +DEFINE_METHOD(RUNTIME_HELPERS, FREE_TAILCALL_ARG_BUFFER, FreeTailCallArgBuffer, SM_RetVoid) +DEFINE_METHOD(RUNTIME_HELPERS, DISPATCH_TAILCALLS, DispatchTailCalls, NoSig) DEFINE_CLASS(UNSAFE, InternalCompilerServices, Unsafe) DEFINE_METHOD(UNSAFE, AS_POINTER, AsPointer, NoSig) diff --git a/src/coreclr/src/vm/tailcallhelp.cpp b/src/coreclr/src/vm/tailcallhelp.cpp index a732d45..70dee8e 100644 --- a/src/coreclr/src/vm/tailcallhelp.cpp +++ b/src/coreclr/src/vm/tailcallhelp.cpp @@ -108,226 +108,29 @@ struct TailCallInfo }; static MethodDesc* s_tailCallDispatcherMD; -MethodDesc* TailCallHelp::GetTailCallDispatcherMD() +MethodDesc* TailCallHelp::GetOrLoadTailCallDispatcherMD() { - LIMITED_METHOD_CONTRACT; - + CONTRACTL + { + THROWS; + GC_TRIGGERS; + INJECT_FAULT(ThrowOutOfMemory()); + } + CONTRACTL_END; + + if (s_tailCallDispatcherMD == NULL) + s_tailCallDispatcherMD = MscorlibBinder::GetMethod(METHOD__RUNTIME_HELPERS__DISPATCH_TAILCALLS); + return s_tailCallDispatcherMD; } - -// This creates the dispatcher used to dispatch sequences of tailcalls. In C# -// code it is the following function. Once C# gets function pointer support this -// function can be put in System.Private.CoreLib. -// private static unsafe void DispatchTailCalls( -// IntPtr callersRetAddrSlot, IntPtr callTarget, IntPtr retVal) -// { -// IntPtr callersRetAddr; -// TailCallTls* tls = GetTailCallInfo(callersRetAddrSlot, &callersRetAddr); -// PortableTailCallFrame* prevFrame = tls->Frame; -// if (callersRetAddr == prevFrame->TailCallAwareReturnAddress) -// { -// prevFrame->NextCall = callTarget; -// return; -// } -// -// PortableTailCallFrame newFrame; -// newFrame.Prev = prevFrame; -// -// try -// { -// tls->Frame = &newFrame; -// -// do -// { -// newFrame.NextCall = IntPtr.Zero; -// var fptr = (func* void(IntPtr, IntPtr, void*))callTarget; -// fptr(tls->ArgBuffer, retVal, &newFrame.TailCallAwareReturnAddress); -// callTarget = newFrame.NextCall; -// } while (callTarget != IntPtr.Zero); -// } -// finally -// { -// tls->Frame = prevFrame; -// } -// } -MethodDesc* TailCallHelp::GetOrCreateTailCallDispatcherMD() +MethodDesc* TailCallHelp::GetTailCallDispatcherMD() { - STANDARD_VM_CONTRACT; - - if (s_tailCallDispatcherMD != NULL) - return s_tailCallDispatcherMD; - - SigBuilder sigBuilder; - sigBuilder.AppendByte(IMAGE_CEE_CS_CALLCONV_DEFAULT); - - sigBuilder.AppendData(3); - sigBuilder.AppendElementType(ELEMENT_TYPE_VOID); - - sigBuilder.AppendElementType(ELEMENT_TYPE_I); - sigBuilder.AppendElementType(ELEMENT_TYPE_I); - sigBuilder.AppendElementType(ELEMENT_TYPE_I); - - const int ARG_CALLERS_RET_ADDR_SLOT = 0; - const int ARG_CALL_TARGET = 1; - const int ARG_RET_VAL = 2; - - DWORD cbSig; - PCCOR_SIGNATURE pSig = AllocateSignature( - MscorlibBinder::GetModule()->GetLoaderAllocator(), sigBuilder, &cbSig); - - SigTypeContext emptyCtx; - - ILStubLinker sl(MscorlibBinder::GetModule(), - Signature(pSig, cbSig), - &emptyCtx, - NULL, - FALSE, - FALSE); - - ILCodeStream* pCode = sl.NewCodeStream(ILStubLinker::kDispatch); - - DWORD retAddrLcl = pCode->NewLocal(ELEMENT_TYPE_I); - DWORD tlsLcl = pCode->NewLocal(ELEMENT_TYPE_I); - DWORD prevFrameLcl = pCode->NewLocal(ELEMENT_TYPE_I); - TypeHandle frameTyHnd = MscorlibBinder::GetClass(CLASS__PORTABLE_TAIL_CALL_FRAME); - DWORD newFrameEntryLcl = pCode->NewLocal(LocalDesc(frameTyHnd)); - DWORD argsLcl = pCode->NewLocal(ELEMENT_TYPE_I); - ILCodeLabel* noUnwindLbl = pCode->NewCodeLabel(); - ILCodeLabel* loopStart = pCode->NewCodeLabel(); - ILCodeLabel* afterTryFinally = pCode->NewCodeLabel(); - - // tls = RuntimeHelpers.GetTailcallInfo(callersRetAddrSlot, &retAddr); - pCode->EmitLDARG(ARG_CALLERS_RET_ADDR_SLOT); - pCode->EmitLDLOCA(retAddrLcl); - pCode->EmitCALL(METHOD__RUNTIME_HELPERS__GET_TAILCALL_INFO, 2, 1); - pCode->EmitSTLOC(tlsLcl); - - // prevFrame = tls.Frame; - pCode->EmitLDLOC(tlsLcl); - pCode->EmitLDFLD(FIELD__TAIL_CALL_TLS__FRAME); - pCode->EmitSTLOC(prevFrameLcl); - - // if (retAddr != prevFrame.TailCallAwareReturnAddress) goto noUnwindLbl; - pCode->EmitLDLOC(retAddrLcl); - pCode->EmitLDLOC(prevFrameLcl); - pCode->EmitLDFLD(FIELD__PORTABLE_TAIL_CALL_FRAME__TAILCALL_AWARE_RETURN_ADDRESS); - pCode->EmitBNE_UN(noUnwindLbl); - - // prevFrame->NextCall = callTarget; - pCode->EmitLDLOC(prevFrameLcl); - pCode->EmitLDARG(ARG_CALL_TARGET); - pCode->EmitSTFLD(FIELD__PORTABLE_TAIL_CALL_FRAME__NEXT_CALL); - - // return; - pCode->EmitRET(); - - // Ok, we are the "first" dispatcher. - pCode->EmitLabel(noUnwindLbl); - - // newFrameEntry.Prev = prevFrame; - pCode->EmitLDLOCA(newFrameEntryLcl); - pCode->EmitLDLOC(prevFrameLcl); - pCode->EmitSTFLD(FIELD__PORTABLE_TAIL_CALL_FRAME__PREV); - - // try { - pCode->BeginTryBlock(); - - // tls->Frame = &newFrameEntry; - pCode->EmitLDLOC(tlsLcl); - pCode->EmitLDLOCA(newFrameEntryLcl); - pCode->EmitSTFLD(FIELD__TAIL_CALL_TLS__FRAME); - - // do { - pCode->EmitLabel(loopStart); - - // newFrameEntry.NextCall = 0 - pCode->EmitLDLOCA(newFrameEntryLcl); - pCode->EmitLDC(0); - pCode->EmitCONV_I(); - pCode->EmitSTFLD(FIELD__PORTABLE_TAIL_CALL_FRAME__NEXT_CALL); - - SigBuilder calliSig; - calliSig.AppendByte(IMAGE_CEE_CS_CALLCONV_DEFAULT); - calliSig.AppendData(3); - calliSig.AppendElementType(ELEMENT_TYPE_VOID); - calliSig.AppendElementType(ELEMENT_TYPE_I); - calliSig.AppendElementType(ELEMENT_TYPE_I); - calliSig.AppendElementType(ELEMENT_TYPE_I); - - DWORD cbCalliSig; - PCCOR_SIGNATURE pCalliSig = (PCCOR_SIGNATURE)calliSig.GetSignature(&cbCalliSig); - - // callTarget(tls->ArgBuffer, retVal, &newFrameEntry.TailCallAwareReturnAddress) - // arg buffer - pCode->EmitLDLOC(tlsLcl); - pCode->EmitLDFLD(FIELD__TAIL_CALL_TLS__ARG_BUFFER); - - // ret val - pCode->EmitLDARG(ARG_RET_VAL); - - // TailCallAwareReturnAddress - pCode->EmitLDLOCA(newFrameEntryLcl); - pCode->EmitLDFLDA(FIELD__PORTABLE_TAIL_CALL_FRAME__TAILCALL_AWARE_RETURN_ADDRESS); - - // callTarget - pCode->EmitLDARG(ARG_CALL_TARGET); - - pCode->EmitCALLI(pCode->GetSigToken(pCalliSig, cbCalliSig), 2, 0); - - // callTarget = newFrameEntry.NextCall; - pCode->EmitLDLOC(newFrameEntryLcl); - pCode->EmitLDFLD(FIELD__PORTABLE_TAIL_CALL_FRAME__NEXT_CALL); - pCode->EmitSTARG(ARG_CALL_TARGET); - - // } while (callTarget != IntPtr.Zero); - pCode->EmitLDARG(ARG_CALL_TARGET); - pCode->EmitBRTRUE(loopStart); - - // } - pCode->EmitLEAVE(afterTryFinally); - pCode->EndTryBlock(); - - // finally { - pCode->BeginFinallyBlock(); - - // tls->Frame = prevFrame; - pCode->EmitLDLOC(tlsLcl); - pCode->EmitLDLOC(prevFrameLcl); - pCode->EmitSTFLD(FIELD__TAIL_CALL_TLS__FRAME); - - // } - pCode->EmitENDFINALLY(); - pCode->EndFinallyBlock(); - - // afterTryFinally: - pCode->EmitLabel(afterTryFinally); - - // return; - pCode->EmitRET(); - - Module* mscorlib = MscorlibBinder::GetModule(); - MethodDesc* pDispatchTailCallsMD = - ILStubCache::CreateAndLinkNewILStubMethodDesc( - MscorlibBinder::GetModule()->GetLoaderAllocator(), - mscorlib->GetILStubCache()->GetOrCreateStubMethodTable(mscorlib), - ILSTUB_TAILCALL_DISPATCH, - mscorlib, - pSig, cbSig, - &emptyCtx, - &sl); - -#ifdef _DEBUG - LOG((LF_STUBS, LL_INFO1000, "TAILCALLHELP: DispatchTailCalls IL created\n")); - sl.LogILStub(CORJIT_FLAGS()); -#endif - - // We might waste a MethodDesc here if we lose the race, but that is very - // unlikely and since this initialization only happens once not a big deal. - InterlockedCompareExchangeT(&s_tailCallDispatcherMD, pDispatchTailCallsMD, NULL); + LIMITED_METHOD_CONTRACT; return s_tailCallDispatcherMD; } + void TailCallHelp::CreateTailCallHelperStubs( MethodDesc* pCallerMD, MethodDesc* pCalleeMD, MetaSig& callSiteSig, bool virt, bool thisArgByRef, diff --git a/src/coreclr/src/vm/tailcallhelp.h b/src/coreclr/src/vm/tailcallhelp.h index 6b9b79f..327dc3d 100644 --- a/src/coreclr/src/vm/tailcallhelp.h +++ b/src/coreclr/src/vm/tailcallhelp.h @@ -23,7 +23,7 @@ public: MethodDesc** storeArgsStub, bool* storeArgsNeedsTarget, MethodDesc** callTargetStub); - static MethodDesc* GetOrCreateTailCallDispatcherMD(); + static MethodDesc* GetOrLoadTailCallDispatcherMD(); static MethodDesc* GetTailCallDispatcherMD(); private: -- 2.7.4