[NativeAOT] Simplifying access to thread static variables (#84566)
authorVladimir Sadov <vsadov@microsoft.com>
Thu, 4 May 2023 16:45:24 +0000 (09:45 -0700)
committerGitHub <noreply@github.com>
Thu, 4 May 2023 16:45:24 +0000 (09:45 -0700)
Fixes: https://github.com/dotnet/runtime/issues/84373
- [x] separate "fast" inlinable case . (used for singlemodule, not dynamic cases, when optimizing)
- [x] make the storage for fast threadstatics a single "combo" instance instead of array of instances.

33 files changed:
src/coreclr/nativeaot/Runtime/AsmOffsets.h
src/coreclr/nativeaot/Runtime/RuntimeInstance.cpp
src/coreclr/nativeaot/Runtime/RuntimeInstance.h
src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc
src/coreclr/nativeaot/Runtime/amd64/MiscStubs.S
src/coreclr/nativeaot/Runtime/amd64/MiscStubs.asm
src/coreclr/nativeaot/Runtime/arm64/MiscStubs.S
src/coreclr/nativeaot/Runtime/arm64/MiscStubs.asm
src/coreclr/nativeaot/Runtime/gcrhscan.cpp
src/coreclr/nativeaot/Runtime/thread.cpp
src/coreclr/nativeaot/Runtime/thread.h
src/coreclr/nativeaot/Runtime/threadstore.cpp
src/coreclr/nativeaot/Runtime/threadstore.inl
src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/ThreadStatics.cs
src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs
src/coreclr/tools/Common/Compiler/DependencyAnalysis/Target_ARM64/ARM64Emitter.cs
src/coreclr/tools/Common/TypeSystem/Common/Utilities/GCPointerMap.Algorithm.cs
src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilationBuilder.Aot.cs
src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ILScanNodeFactory.cs
src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/NodeFactory.cs
src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ReflectionFieldMapNode.cs
src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_ARM64/ARM64ReadyToRunHelperNode.cs
src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X64/X64ReadyToRunHelperNode.cs
src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ThreadStaticsNode.cs
src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/TypeThreadStaticIndexNode.cs
src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ILScanner.cs
src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/InlinedThreadStatics.cs [new file with mode: 0644]
src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj
src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/DependencyAnalysis/RyuJitNodeFactory.cs
src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/RyuJitCompilationBuilder.cs
src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs
src/coreclr/tools/aot/ILCompiler/Program.cs
src/tests/nativeaot/SmokeTests/DwarfDump/Program.cs

index e28386e1d3b5b84a95d9f2be7e0b2974319fc593..203fee38bf15ffb081382772c1fdd9a998bd7beb 100644 (file)
@@ -58,8 +58,6 @@ ASM_OFFSET(    0,    78, Thread, m_uHijackedReturnValueFlags)
 ASM_OFFSET(   48,    80, Thread, m_pExInfoStackHead)
 ASM_OFFSET(   4c,    88, Thread, m_threadAbortException)
 
-ASM_OFFSET(   50,    90, Thread, m_pThreadLocalModuleStatics)
-
 ASM_SIZEOF(   14,    20, EHEnum)
 
 ASM_OFFSET(    0,     0, gc_alloc_context, alloc_ptr)
index 581c27cbb163d48e11cb895494dd142e77ee9925..146c15aeff192cf831f47a7796ed3969124dbff5 100644 (file)
@@ -275,6 +275,25 @@ RuntimeInstance::TypeManagerList& RuntimeInstance::GetTypeManagerList()
     return m_TypeManagerList;
 }
 
+TypeManager* RuntimeInstance::GetSingleTypeManager()
+{
+    auto head = m_TypeManagerList.GetHead();
+    if (head != NULL && head->m_pNext == NULL)
+    {
+        return head->m_pTypeManager;
+    }
+
+    return NULL;
+}
+
+COOP_PINVOKE_HELPER(TypeManagerHandle, RhGetSingleTypeManager, ())
+{
+    TypeManager* typeManager = GetRuntimeInstance()->GetSingleTypeManager();
+    ASSERT(typeManager != NULL);
+
+    return TypeManagerHandle::Create(typeManager);
+}
+
 // static
 bool RuntimeInstance::Initialize(HANDLE hPalInstance)
 {
index 2de7c220f09128c2af0f17243815ae576f79cf5e..06db50839eebacaa6177b254cb82ff40480ae941 100644 (file)
@@ -99,6 +99,7 @@ public:
 
     bool RegisterTypeManager(TypeManager * pTypeManager);
     TypeManagerList& GetTypeManagerList();
+    TypeManager* GetSingleTypeManager();
     OsModuleList* GetOsModuleList();
 
     bool RegisterUnboxingStubs(PTR_VOID pvStartRange, uint32_t cbRange);
index 55ca399b4e405c22ce6b34f7c8b47c1b47b79660..5c2376955c70c4b62f4ceb7f0cc7405851096a68 100644 (file)
@@ -235,11 +235,11 @@ Name    dq      offset AddressToExport
 _tls_array     equ 58h     ;; offsetof(TEB, ThreadLocalStoragePointer)
 
 ;;
-;; __declspec(thread) version
+;; __declspec(thread) variable
 ;;
-INLINE_GETTHREAD macro destReg, trashReg
+INLINE_GET_TLS_VAR macro destReg, trashReg, variable
     EXTERN _tls_index : DWORD
-    EXTERN tls_CurrentThread:DWORD
+    EXTERN variable:DWORD
 
 ;;
 ;; construct 'eax' from 'rax' so that the register size and data size match
@@ -255,11 +255,18 @@ endif
     mov         destRegDWORD, [_tls_index]
     mov         trashReg, gs:[_tls_array]
     mov         trashReg, [trashReg + destReg * 8]
-    mov         destRegDWORD, SECTIONREL tls_CurrentThread
+    mov         destRegDWORD, SECTIONREL variable
     add         destReg, trashReg
 
 endm
 
+;;
+;; __declspec(thread) tls_CurrentThread
+;;
+INLINE_GETTHREAD macro destReg, trashReg
+    INLINE_GET_TLS_VAR destReg, trashReg, tls_CurrentThread
+endm
+
 INLINE_THREAD_UNHIJACK macro threadReg, trashReg1, trashReg2
         ;;
         ;; Thread::Unhijack()
index e2d0d91fce4bfc7160e9b1e8fda8e06836ccceb7..7faf58c75c45e9368e8d8daf998618c046c46730 100644 (file)
@@ -45,57 +45,20 @@ LOCAL_LABEL(ProbeLoop):
         ret
 NESTED_END RhpStackProbe, _TEXT
 
-NESTED_ENTRY RhpGetThreadStaticBaseForType, _TEXT, NoHandler
-        // On entry:
-        //   rdi - TypeManagerSlot*
-        //   rsi - type index
+NESTED_ENTRY RhpGetInlinedThreadStaticBase, _TEXT, NoHandler
         // On exit:
         //   rax - the thread static base for the given type
 
-        push_nonvol_reg rbx
-        push_nonvol_reg r12
-
-        mov     rbx, rdi               // Save TypeManagerSlot*
-        mov     r12, rsi               // Save type index
-
-        // rax = GetThread()
-        INLINE_GETTHREAD
-
-        mov     r8d, [rbx + 8]         // Get ModuleIndex out of the TypeManagerSlot
+        // rdi = &tls_InlinedThreadStatics
+        INLINE_GET_TLS_VAR tls_InlinedThreadStatics
+        mov     rdi, rax
 
         // get per-thread storage
-        mov     rax, [rax + OFFSETOF__Thread__m_pThreadLocalModuleStatics]
-
-        // get per-module storage
+        mov     rax, [rdi]
         test    rax, rax
-        jz      LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath)
-        cmp     r8d, [rax + OFFSETOF__Array__m_Length]
-        jae     LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath)
-        mov     rax, [rax + r8 * 8 + 0x10]
+        jz      C_FUNC(RhpGetInlinedThreadStaticBaseSlow)    // rdi contains the storage ref
 
-        // get the actual per-type storage
-        test    rax, rax
-        jz      LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath)
-        cmp     r12d, [rax + OFFSETOF__Array__m_Length]
-        jae     LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath)
-        mov     rax, [rax + r12 * 8 + 0x10]
-
-        // if have storage, return it
-        test    rax, rax
-        jz      LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath)
-
-        .cfi_remember_state
-        pop_nonvol_reg r12
-        pop_nonvol_reg rbx
+        // return it
         ret
+NESTED_END RhpGetInlinedThreadStaticBase, _TEXT
 
-        .cfi_restore_state
-        .cfi_def_cfa_offset 24  // workaround cfi_restore_state bug
-LOCAL_LABEL(RhpGetThreadStaticBaseForType_RarePath):
-        mov         rdi, rbx    // restore TypeManagerSlot*
-        mov         rsi, r12    // restore type index
-
-        pop_nonvol_reg r12
-        pop_nonvol_reg rbx
-        jmp C_FUNC(RhpGetThreadStaticBaseForTypeSlow)
-NESTED_END RhpGetThreadStaticBaseForType, _TEXT
index 9f11b73c4444dada21821fe25db75e3b8234594c..c4f39f5134558544f6907b83662d60258c55a4a0 100644 (file)
@@ -3,7 +3,7 @@
 
 include AsmMacros.inc
 
-EXTERN RhpGetThreadStaticBaseForTypeSlow : PROC
+EXTERN RhpGetInlinedThreadStaticBaseSlow : PROC
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ; The following helper will access ("probe") a word on each page of the stack
@@ -39,41 +39,20 @@ ProbeLoop:
 
 LEAF_END RhpStackProbe, _TEXT
 
-LEAF_ENTRY RhpGetThreadStaticBaseForType, _TEXT
-        ; On entry and thorough the procedure:
-        ;   rcx - TypeManagerSlot*
-        ;   rdx - type index
+LEAF_ENTRY RhpGetInlinedThreadStaticBase, _TEXT
         ; On exit:
         ;   rax - the thread static base for the given type
 
-        ;; rax = GetThread(), TRASHES r8
-        INLINE_GETTHREAD rax, r8
-
-        mov     r8d, [rcx + 8]         ; Get ModuleIndex out of the TypeManagerSlot
+        ;; rcx = &tls_InlinedThreadStatics, TRASHES r8
+        INLINE_GET_TLS_VAR rcx, r8, tls_InlinedThreadStatics
 
         ;; get per-thread storage
-        mov     rax, [rax + OFFSETOF__Thread__m_pThreadLocalModuleStatics]
-
-        ;; get per-module storage
-        test    rax, rax
-        jz      RhpGetThreadStaticBaseForTypeSlow
-        cmp     r8d, [rax + OFFSETOF__Array__m_Length]
-        jae     RhpGetThreadStaticBaseForTypeSlow
-        mov     rax, [rax + r8 * 8 + 10h]
-
-        ;; get the actual per-type storage
+        mov     rax, [rcx]
         test    rax, rax
-        jz      RhpGetThreadStaticBaseForTypeSlow
-        cmp     edx, [rax + OFFSETOF__Array__m_Length]
-        jae     RhpGetThreadStaticBaseForTypeSlow
-        mov     rax, [rax + rdx * 8 + 10h]
-
-        ;; if have storage, return it
-        test    rax, rax
-        jz      RhpGetThreadStaticBaseForTypeSlow
+        jz      RhpGetInlinedThreadStaticBaseSlow   ;; rcx contains the storage ref
 
+        ;; return it
         ret
-
-LEAF_END RhpGetThreadStaticBaseForType, _TEXT
+LEAF_END RhpGetInlinedThreadStaticBase, _TEXT
 
 end
index fc1f6d465ca2de7547c489ff81145954963f7060..6c70614f38136dfecf1961a56a9666c6a8280db9 100644 (file)
@@ -4,3 +4,21 @@
 #include <unixasmmacros.inc>
 #include "AsmOffsets.inc"
 
+NESTED_ENTRY RhpGetInlinedThreadStaticBase, _TEXT, NoHandler
+        // On exit:
+        //   x0 - the thread static base for the given type
+
+        // x1 = GetThread()
+        INLINE_GET_TLS_VAR x1, C_FUNC(tls_InlinedThreadStatics)
+
+        // get per-thread storage
+        ldr     x0, [x1]
+        cbnz    x0, HaveValue
+        mov     x0, x1
+        b       C_FUNC(RhpGetInlinedThreadStaticBaseSlow)
+
+HaveValue:
+        // return it
+        ret
+
+NESTED_END RhpGetInlinedThreadStaticBase, _TEXT
index 49baea4977259b99add88c086ae4b4f10f49bcda..cdb076b3d56b26f6f81b3802c470fad42d48b543 100644 (file)
@@ -3,6 +3,25 @@
 
 #include "AsmMacros.h"
 
+    EXTERN RhpGetInlinedThreadStaticBaseSlow
+
     TEXTAREA
 
+;; On exit:
+;;   x0 - the thread static base for the given type
+    LEAF_ENTRY RhpGetInlinedThreadStaticBase
+        ;; x1 = &tls_InlinedThreadStatics, TRASHES x2
+        INLINE_GET_TLS_VAR x1, x2, tls_InlinedThreadStatics
+
+        ;; get per-thread storage
+        ldr     x0, [x1]
+        cbnz    x0, HaveValue
+        mov     x0, x1
+        b       RhpGetInlinedThreadStaticBaseSlow
+
+HaveValue
+        ;; return it
+        ret
+    LEAF_END RhpGetInlinedThreadStaticBase
+
     end
index 641e55ec983711527ffb934fc13c9b1a82109ff4..e6c815094d29b9921d84cdcdf389873e9edff441 100644 (file)
@@ -54,6 +54,14 @@ void GCToEEInterface::GcScanRoots(EnumGcRefCallbackFunc * fn,  int condemned, in
         else
 #endif
         {
+            InlinedThreadStaticRoot* pRoot = pThread->GetInlinedThreadStaticList();
+            while (pRoot != NULL)
+            {
+                STRESS_LOG2(LF_GC | LF_GCROOTS, LL_INFO100, "{ Scanning Thread's %p inline thread statics root %p. \n", pThread, pRoot);
+                GcEnumObject(&pRoot->m_threadStaticsBase, 0 /*flags*/, fn, sc);
+                pRoot = pRoot->m_next;
+            }
+
             STRESS_LOG1(LF_GC | LF_GCROOTS, LL_INFO100, "{ Scanning Thread's %p thread statics root. \n", pThread);
             GcEnumObject(pThread->GetThreadStaticStorage(), 0 /*flags*/, fn, sc);
 
index 6f599b97c9eb25bf9c3e849765a8d831e9c920c6..4d8c2c29d1a780786cbe0ea09070c1b4b8c598ec 100644 (file)
@@ -284,7 +284,8 @@ void Thread::Construct()
 
     // Everything else should be initialized to 0 via the static initialization of tls_CurrentThread.
 
-    ASSERT(m_pThreadLocalModuleStatics == NULL);
+    ASSERT(m_pThreadLocalStatics == NULL);
+    ASSERT(m_pInlinedThreadLocalStatics == NULL);
 
     ASSERT(m_pGCFrameRegistrations == NULL);
 
@@ -1266,15 +1267,33 @@ COOP_PINVOKE_HELPER(Object *, RhpGetThreadAbortException, ())
 
 Object** Thread::GetThreadStaticStorage()
 {
-    return &m_pThreadLocalModuleStatics;
+    return &m_pThreadLocalStatics;
 }
 
 COOP_PINVOKE_HELPER(Object**, RhGetThreadStaticStorage, ())
 {
-    Thread * pCurrentThread = ThreadStore::RawGetCurrentThread();
+    Thread* pCurrentThread = ThreadStore::RawGetCurrentThread();
     return pCurrentThread->GetThreadStaticStorage();
 }
 
+InlinedThreadStaticRoot* Thread::GetInlinedThreadStaticList()
+{
+    return m_pInlinedThreadLocalStatics;
+}
+
+void Thread::RegisterInlinedThreadStaticRoot(InlinedThreadStaticRoot* newRoot)
+{
+    ASSERT(newRoot->m_next == NULL);
+    newRoot->m_next = m_pInlinedThreadLocalStatics;
+    m_pInlinedThreadLocalStatics = newRoot;
+}
+
+COOP_PINVOKE_HELPER(void, RhRegisterInlinedThreadStaticRoot, (Object** root))
+{
+    Thread* pCurrentThread = ThreadStore::RawGetCurrentThread();
+    pCurrentThread->RegisterInlinedThreadStaticRoot((InlinedThreadStaticRoot*)root);
+}
+
 // This is function is used to quickly query a value that can uniquely identify a thread
 COOP_PINVOKE_HELPER(uint8_t*, RhCurrentNativeThreadId, ())
 {
index 0dd855cbe968d07739fb4bc6d00deae2727d9543..39310955e388cf271b86598c9fc205d0efb2885b 100644 (file)
@@ -74,6 +74,12 @@ struct GCFrameRegistration
     int m_MaybeInterior;
 };
 
+struct InlinedThreadStaticRoot
+{
+    Object* m_threadStaticsBase;
+    InlinedThreadStaticRoot* m_next;
+};
+
 struct ThreadBuffer
 {
     uint8_t                 m_rgbAllocContextBuffer[SIZEOF_ALLOC_CONTEXT];
@@ -88,7 +94,8 @@ struct ThreadBuffer
     uintptr_t               m_uHijackedReturnValueFlags;            
     PTR_ExInfo              m_pExInfoStackHead;
     Object*                 m_threadAbortException;                 // ThreadAbortException instance -set only during thread abort
-    Object*                 m_pThreadLocalModuleStatics;
+    Object*                 m_pThreadLocalStatics;
+    InlinedThreadStaticRoot* m_pInlinedThreadLocalStatics;
     GCFrameRegistration*    m_pGCFrameRegistrations;
     PTR_VOID                m_pStackLow;
     PTR_VOID                m_pStackHigh;
@@ -288,6 +295,9 @@ public:
 
     Object** GetThreadStaticStorage();
 
+    InlinedThreadStaticRoot* GetInlinedThreadStaticList();
+    void RegisterInlinedThreadStaticRoot(InlinedThreadStaticRoot* newRoot);
+
     NATIVE_CONTEXT* GetInterruptedContext();
 
     void PushGCFrameRegistration(GCFrameRegistration* pRegistration);
index 12cdea592ce4987aa530aa8fbbdf54b97d3dbca7..c65d95770956657ae7afec0e39f05fa8837a746c 100644 (file)
@@ -430,6 +430,11 @@ C_ASSERT(sizeof(Thread) == sizeof(ThreadBuffer));
 
 #ifndef _MSC_VER
 __thread ThreadBuffer tls_CurrentThread;
+
+// the root of inlined threadstatics storage
+// there is only one now,
+// eventually this will be emitted by ILC and we may have more than one such variable
+__thread InlinedThreadStaticRoot tls_InlinedThreadStatics;
 #endif
 
 EXTERN_C ThreadBuffer* RhpGetThread()
@@ -437,6 +442,11 @@ EXTERN_C ThreadBuffer* RhpGetThread()
     return &tls_CurrentThread;
 }
 
+COOP_PINVOKE_HELPER(Object**, RhGetInlinedThreadStaticStorage, ())
+{
+    return &tls_InlinedThreadStatics.m_threadStaticsBase;
+}
+
 #endif // !DACCESS_COMPILE
 
 #ifdef _WIN32
@@ -505,4 +515,4 @@ void ThreadStore::SaveCurrentThreadOffsetForDAC()
 {
 }
 
-#endif // _WIN32
\ No newline at end of file
+#endif // _WIN32
index 29495046a98272936c4c20d078f6d69f90d5c216..6fe750f4b01c01508a1c523f076618cd7889d643 100644 (file)
@@ -4,8 +4,14 @@
 #ifdef _MSC_VER
 // a workaround to prevent tls_CurrentThread from becoming dynamically checked/initialized.
 EXTERN_C __declspec(selectany) __declspec(thread) ThreadBuffer tls_CurrentThread;
+
+// the root of inlined threadstatics storage
+// there is only one now,
+// eventually this will be emitted by ILC and we may have more than one such variable
+EXTERN_C __declspec(selectany) __declspec(thread) InlinedThreadStaticRoot tls_InlinedThreadStatics;
 #else
 EXTERN_C __thread ThreadBuffer tls_CurrentThread;
+EXTERN_C __thread InlinedThreadStaticRoot tls_InlinedThreadStatics;
 #endif
 
 // static
index b8dd4c0971e4188635a2bf7b2118810a3527764f..9734e2a0a2632f6a017366e606eb8fe6e288927b 100644 (file)
@@ -4,7 +4,7 @@
 using System;
 using System.Runtime;
 using System.Runtime.CompilerServices;
-
+using System.Runtime.InteropServices;
 using Internal.Runtime.CompilerHelpers;
 
 using Debug = System.Diagnostics.Debug;
@@ -22,6 +22,34 @@ namespace Internal.Runtime
         /// static storage for the given type.
         /// </summary>
         internal static unsafe object GetThreadStaticBaseForType(TypeManagerSlot* pModuleData, int typeTlsIndex)
+        {
+            if (typeTlsIndex >= 0)
+                return GetUninlinedThreadStaticBaseForType(pModuleData, typeTlsIndex);
+
+            ref object? threadStorage = ref RuntimeImports.RhGetInlinedThreadStaticStorage();
+            if (threadStorage != null)
+                return threadStorage;
+
+            return GetInlinedThreadStaticBaseSlow(ref threadStorage);
+        }
+
+        [RuntimeExport("RhpGetInlinedThreadStaticBaseSlow")]
+        internal static unsafe object GetInlinedThreadStaticBaseSlow(ref object? threadStorage)
+        {
+            Debug.Assert(threadStorage == null);
+            // Allocate an object that will represent a memory block for all thread static fields
+            TypeManagerHandle typeManager = RuntimeImports.RhGetSingleTypeManager();
+            object threadStaticBase = AllocateThreadStaticStorageForType(typeManager, 0);
+
+            // register the storage location with the thread for GC reporting.
+            RuntimeImports.RhRegisterInlinedThreadStaticRoot(ref threadStorage);
+
+            // assign the storage block to the storage variable and return
+            threadStorage = threadStaticBase;
+            return threadStaticBase;
+        }
+
+        internal static unsafe object GetUninlinedThreadStaticBaseForType(TypeManagerSlot* pModuleData, int typeTlsIndex)
         {
             Debug.Assert(typeTlsIndex >= 0);
             int moduleIndex = pModuleData->ModuleIndex;
@@ -41,16 +69,15 @@ namespace Internal.Runtime
                 }
             }
 
-            return GetThreadStaticBaseForTypeSlow(pModuleData, typeTlsIndex);
+            return GetUninlinedThreadStaticBaseForTypeSlow(pModuleData, typeTlsIndex);
         }
 
-        [RuntimeExport("RhpGetThreadStaticBaseForTypeSlow")]
         [MethodImpl(MethodImplOptions.NoInlining)]
-        internal static unsafe object GetThreadStaticBaseForTypeSlow(TypeManagerSlot* pModuleData, int typeTlsIndex)
+        internal static unsafe object GetUninlinedThreadStaticBaseForTypeSlow(TypeManagerSlot* pModuleData, int typeTlsIndex)
         {
             Debug.Assert(typeTlsIndex >= 0);
             int moduleIndex = pModuleData->ModuleIndex;
-            Debug.Assert(typeTlsIndex >= 0);
+            Debug.Assert(moduleIndex >= 0);
 
             // Get the array that holds thread statics for the current thread, if none present
             // allocate a new one big enough to hold the current module data
index f3b4cca51b052feb7aa7d4bc0651149fffff7245..9fe023ee52f8846a4bb03603d7dc401c115b241b 100644 (file)
@@ -557,6 +557,14 @@ namespace System.Runtime
         [RuntimeImport(RuntimeLibrary, "RhGetThreadStaticStorage")]
         internal static extern ref object[][] RhGetThreadStaticStorage();
 
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        [RuntimeImport(RuntimeLibrary, "RhGetInlinedThreadStaticStorage")]
+        internal static extern ref object? RhGetInlinedThreadStaticStorage();
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        [RuntimeImport(RuntimeLibrary, "RhRegisterInlinedThreadStaticRoot")]
+        internal static extern void RhRegisterInlinedThreadStaticRoot(ref object? root);
+
         [MethodImplAttribute(MethodImplOptions.InternalCall)]
         [RuntimeImport(RuntimeLibrary, "RhCurrentNativeThreadId")]
         internal static extern unsafe IntPtr RhCurrentNativeThreadId();
@@ -581,6 +589,10 @@ namespace System.Runtime
         [RuntimeImport(RuntimeLibrary, "RhGetTargetOfUnboxingAndInstantiatingStub")]
         public static extern IntPtr RhGetTargetOfUnboxingAndInstantiatingStub(IntPtr pCode);
 
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        [RuntimeImport(RuntimeLibrary, "RhGetSingleTypeManager")]
+        public static extern TypeManagerHandle RhGetSingleTypeManager();
+
         //
         // EH helpers
         //
index 84f67052d691d745c3f56f584f4394000fc06d1f..45b5018b58fa8b70f3d045a864f03a67f80ec073 100644 (file)
@@ -31,6 +31,13 @@ namespace ILCompiler.DependencyAnalysis.ARM64
             Builder.EmitUInt(instruction);
         }
 
+        public void EmitMVN(Register regDst, ushort imm16)
+        {
+            Debug.Assert((uint)regDst <= 0x1f);
+            uint instruction = 0x92800000u | ((uint)imm16 << 5) | (uint)regDst;
+            Builder.EmitUInt(instruction);
+        }
+
         public void EmitMOV(Register regDst, ISymbolNode symbol)
         {
             // ADRP regDst, [symbol (21bit ADRP thing)]
index a03addd6632b3ab8dfdfae6e9c67860946d04678..fd21b72a066a820e58bc6510c265fbe2de0954f3 100644 (file)
@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections.Generic;
 using Debug = System.Diagnostics.Debug;
 
 namespace Internal.TypeSystem
@@ -100,13 +101,8 @@ namespace Internal.TypeSystem
             return builder.ToGCMap();
         }
 
-        /// <summary>
-        /// Computes the GC pointer map of the thread static region of the type.
-        /// </summary>
-        public static GCPointerMap FromThreadStaticLayout(MetadataType type)
+        private static void MapThreadStaticsForType(GCPointerMapBuilder builder, MetadataType type, int baseOffset)
         {
-            GCPointerMapBuilder builder = new GCPointerMapBuilder(type.ThreadGcStaticFieldSize.AsInt, type.Context.Target.PointerSize);
-
             foreach (FieldDesc field in type.GetFields())
             {
                 if (!field.IsStatic || field.HasRva || field.IsLiteral || !field.IsThreadStatic || !field.HasGCStaticBase)
@@ -115,7 +111,7 @@ namespace Internal.TypeSystem
                 TypeDesc fieldType = field.FieldType;
                 if (fieldType.IsGCPointer)
                 {
-                    builder.MarkGCPointer(field.Offset.AsInt);
+                    builder.MarkGCPointer(field.Offset.AsInt + baseOffset);
                 }
                 else if (fieldType.IsValueType)
                 {
@@ -123,14 +119,39 @@ namespace Internal.TypeSystem
                     if (fieldDefType.ContainsGCPointers)
                     {
                         GCPointerMapBuilder innerBuilder =
-                            builder.GetInnerBuilder(field.Offset.AsInt, fieldDefType.InstanceByteCount.AsInt);
+                            builder.GetInnerBuilder(field.Offset.AsInt + baseOffset, fieldDefType.InstanceByteCount.AsInt);
                         FromInstanceLayoutHelper(ref innerBuilder, fieldDefType);
                     }
                 }
             }
+        }
+
+        /// <summary>
+        /// Computes the GC pointer map of the thread static region of the type.
+        /// </summary>
+        public static GCPointerMap FromThreadStaticLayout(MetadataType type)
+        {
+            GCPointerMapBuilder builder = new GCPointerMapBuilder(type.ThreadGcStaticFieldSize.AsInt, type.Context.Target.PointerSize);
+
+            MapThreadStaticsForType(builder, type, baseOffset: 0);
 
             Debug.Assert(builder.ToGCMap().Size * type.Context.Target.PointerSize >= type.ThreadGcStaticFieldSize.AsInt);
             return builder.ToGCMap();
         }
+
+        public static GCPointerMap FromInlinedThreadStatics(
+            List<MetadataType> types,
+            Dictionary<MetadataType, int> offsets,
+            int threadStaticSize,
+            int pointerSize)
+        {
+            GCPointerMapBuilder builder = new GCPointerMapBuilder(threadStaticSize, pointerSize);
+            foreach (var type in types)
+            {
+                MapThreadStaticsForType(builder, type, offsets[type]);
+            }
+
+            return builder.ToGCMap();
+        }
     }
 }
index 48d66ff85b26b6364ac674acd0f831f5b401c724..0f28af6440bf7e5c73eb6f4e4fae6ab219fbc82c 100644 (file)
@@ -17,6 +17,7 @@ namespace ILCompiler
         protected DictionaryLayoutProvider _dictionaryLayoutProvider = new LazyDictionaryLayoutProvider();
         protected DebugInformationProvider _debugInformationProvider = new DebugInformationProvider();
         protected DevirtualizationManager _devirtualizationManager = new DevirtualizationManager();
+        protected InlinedThreadStatics _inlinedThreadStatics = new InlinedThreadStatics();
         protected MethodImportationErrorProvider _methodImportationErrorProvider = new MethodImportationErrorProvider();
         protected IInliningPolicy _inliningPolicy;
         protected bool _methodBodyFolding;
@@ -109,6 +110,12 @@ namespace ILCompiler
             return this;
         }
 
+        public CompilationBuilder UseInlinedThreadStatics(InlinedThreadStatics inlinedThreadStatics)
+        {
+            _inlinedThreadStatics = inlinedThreadStatics;
+            return this;
+        }
+
         public CompilationBuilder UseDwarf5(bool value)
         {
             _useDwarf5 = value;
index 90e29d20bcc2d4c91302c425657fcf00d3f003ea..738f76f27f11d4af18af1d25cc74298c2970e28c 100644 (file)
@@ -13,7 +13,7 @@ namespace ILCompiler.DependencyAnalysis
     public sealed class ILScanNodeFactory : NodeFactory
     {
         public ILScanNodeFactory(CompilerTypeSystemContext context, CompilationModuleGroup compilationModuleGroup, MetadataManager metadataManager, InteropStubManager interopStubManager, NameMangler nameMangler, PreinitializationManager preinitManager)
-            : base(context, compilationModuleGroup, metadataManager, interopStubManager, nameMangler, new LazyGenericsDisabledPolicy(), new LazyVTableSliceProvider(), new LazyDictionaryLayoutProvider(), new ExternSymbolsImportedNodeProvider(), preinitManager)
+            : base(context, compilationModuleGroup, metadataManager, interopStubManager, nameMangler, new LazyGenericsDisabledPolicy(), new LazyVTableSliceProvider(), new LazyDictionaryLayoutProvider(), new InlinedThreadStatics(), new ExternSymbolsImportedNodeProvider(), preinitManager)
         {
         }
 
index c726a9dc0a66d55e6c6a45da4af451d2ef0978d0..53b44d32b142c76c6e963ebc6f76110d508cfae1 100644 (file)
@@ -24,6 +24,7 @@ namespace ILCompiler.DependencyAnalysis
         private CompilationModuleGroup _compilationModuleGroup;
         private VTableSliceProvider _vtableSliceProvider;
         private DictionaryLayoutProvider _dictionaryLayoutProvider;
+        private InlinedThreadStatics _inlinedThreadStatics;
         protected readonly ImportedNodeProvider _importedNodeProvider;
         private bool _markingComplete;
 
@@ -36,6 +37,7 @@ namespace ILCompiler.DependencyAnalysis
             LazyGenericsPolicy lazyGenericsPolicy,
             VTableSliceProvider vtableSliceProvider,
             DictionaryLayoutProvider dictionaryLayoutProvider,
+            InlinedThreadStatics inlinedThreadStatics,
             ImportedNodeProvider importedNodeProvider,
             PreinitializationManager preinitializationManager)
         {
@@ -44,6 +46,7 @@ namespace ILCompiler.DependencyAnalysis
             _compilationModuleGroup = compilationModuleGroup;
             _vtableSliceProvider = vtableSliceProvider;
             _dictionaryLayoutProvider = dictionaryLayoutProvider;
+            _inlinedThreadStatics = inlinedThreadStatics;
             NameMangler = nameMangler;
             InteropStubManager = interoptStubManager;
             CreateNodeCaches();
@@ -207,8 +210,21 @@ namespace ILCompiler.DependencyAnalysis
 
             _threadStatics = new NodeCache<MetadataType, ISymbolDefinitionNode>(CreateThreadStaticsNode);
 
+            TypeThreadStaticIndexNode inlinedThreadStatiscIndexNode = null;
+            if (_inlinedThreadStatics.IsComputed())
+            {
+                _inlinedThreadStatiscNode = new ThreadStaticsNode(_inlinedThreadStatics, this);
+                inlinedThreadStatiscIndexNode = new TypeThreadStaticIndexNode(_inlinedThreadStatiscNode);
+            }
+
             _typeThreadStaticIndices = new NodeCache<MetadataType, TypeThreadStaticIndexNode>(type =>
             {
+                if (inlinedThreadStatiscIndexNode != null &&
+                _inlinedThreadStatics.GetOffsets().ContainsKey(type))
+                {
+                    return inlinedThreadStatiscIndexNode;
+                }
+
                 return new TypeThreadStaticIndexNode(type);
             });
 
@@ -646,6 +662,7 @@ namespace ILCompiler.DependencyAnalysis
         }
 
         private NodeCache<MetadataType, ISymbolDefinitionNode> _threadStatics;
+        private ThreadStaticsNode _inlinedThreadStatiscNode;
 
         public ISymbolDefinitionNode TypeThreadStaticsSymbol(MetadataType type)
         {
@@ -844,6 +861,17 @@ namespace ILCompiler.DependencyAnalysis
             return _stringAllocators.GetOrAdd(stringConstructor);
         }
 
+        public uint ThreadStaticBaseOffset(MetadataType type)
+        {
+            if (_inlinedThreadStatics.IsComputed() &&
+                _inlinedThreadStatics.GetOffsets().TryGetValue(type, out var offset))
+            {
+                return (uint)offset;
+            }
+
+            return 0;
+        }
+
         private sealed class MethodEntrypointHashtable : LockFreeReaderHashtable<MethodDesc, IMethodNode>
         {
             private readonly NodeFactory _factory;
@@ -1263,6 +1291,11 @@ namespace ILCompiler.DependencyAnalysis
             graph.AddRoot(InterfaceDispatchCellSection, "Interface dispatch cell section is always generated");
             graph.AddRoot(ModuleInitializerList, "Module initializer list is always generated");
 
+            if (_inlinedThreadStatics.IsComputed())
+            {
+                graph.AddRoot(_inlinedThreadStatiscNode, "Inlined threadstatics are used if present");
+            }
+
             ReadyToRunHeader.Add(ReadyToRunSectionType.GCStaticRegion, GCStaticsRegion, GCStaticsRegion.StartSymbol, GCStaticsRegion.EndSymbol);
             ReadyToRunHeader.Add(ReadyToRunSectionType.ThreadStaticRegion, ThreadStaticsRegion, ThreadStaticsRegion.StartSymbol, ThreadStaticsRegion.EndSymbol);
             ReadyToRunHeader.Add(ReadyToRunSectionType.EagerCctor, EagerCctorTable, EagerCctorTable.StartSymbol, EagerCctorTable.EndSymbol);
index ec2c6ed59d1518a09c93f50152740e98a54599c8..3044fd4d979d077af4a40db7809729d71abad39b 100644 (file)
@@ -128,9 +128,15 @@ namespace ILCompiler.DependencyAnalysis
                         case FieldTableFlags.GCStatic:
                         case FieldTableFlags.NonGCStatic:
                             {
+                                uint fieldOffset = (uint)field.Offset.AsInt;
+                                if (field.IsThreadStatic && field.OwningType is MetadataType mt)
+                                {
+                                    fieldOffset += factory.ThreadStaticBaseOffset(mt);
+                                }
+
                                 if (field.OwningType.HasInstantiation)
                                 {
-                                    vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant((uint)(field.Offset.AsInt)));
+                                    vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant(fieldOffset));
                                 }
                                 else
                                 {
@@ -138,22 +144,28 @@ namespace ILCompiler.DependencyAnalysis
 
                                     ISymbolNode staticsNode;
                                     if (field.IsThreadStatic)
+                                    {
                                         staticsNode = factory.TypeThreadStaticIndex(metadataType);
+                                    }
                                     else if (field.HasGCStaticBase)
+                                    {
                                         staticsNode = factory.TypeGCStaticsSymbol(metadataType);
+                                    }
                                     else
+                                    {
                                         staticsNode = factory.TypeNonGCStaticsSymbol(metadataType);
+                                    }
 
                                     if (!field.IsThreadStatic && !field.HasGCStaticBase)
                                     {
-                                        uint index = _externalReferences.GetIndex(staticsNode, field.Offset.AsInt);
+                                        uint index = _externalReferences.GetIndex(staticsNode, (int)fieldOffset);
                                         vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant(index));
                                     }
                                     else
                                     {
                                         uint index = _externalReferences.GetIndex(staticsNode);
                                         vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant(index));
-                                        vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant((uint)(field.Offset.AsInt)));
+                                        vertex = writer.GetTuple(vertex, writer.GetUnsignedConstant(fieldOffset));
                                     }
                                 }
                             }
index c61a3ebf7b9eaed842830812f14e2ad5904efefd..8e43f126e0cc74c5055ccc0e96676e238ba67205 100644 (file)
@@ -71,30 +71,59 @@ namespace ILCompiler.DependencyAnalysis
                 case ReadyToRunHelperId.GetThreadStaticBase:
                     {
                         MetadataType target = (MetadataType)Target;
-                        encoder.EmitMOV(encoder.TargetRegister.Arg2, factory.TypeThreadStaticIndex(target));
-
-                        // First arg: address of the TypeManager slot that provides the helper with
-                        // information about module index and the type manager instance (which is used
-                        // for initialization on first access).
-                        encoder.EmitLDR(encoder.TargetRegister.Arg0, encoder.TargetRegister.Arg2);
-
-                        // Second arg: index of the type in the ThreadStatic section of the modules
-                        encoder.EmitLDR(encoder.TargetRegister.Arg1, encoder.TargetRegister.Arg2, factory.Target.PointerSize);
-
-                        if (!factory.PreinitializationManager.HasLazyStaticConstructor(target))
+                        ISortableSymbolNode index = factory.TypeThreadStaticIndex(target);
+                        if (index is TypeThreadStaticIndexNode ti && ti.Type == null)
                         {
-                            encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.GetThreadStaticBaseForType));
+                            ISymbolNode helper = factory.ExternSymbol("RhpGetInlinedThreadStaticBase");
+
+                            if (!factory.PreinitializationManager.HasLazyStaticConstructor(target))
+                            {
+                                encoder.EmitJMP(helper);
+                            }
+                            else
+                            {
+                                encoder.EmitMOV(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target));
+                                encoder.EmitSUB(encoder.TargetRegister.Arg2, NonGCStaticsNode.GetClassConstructorContextSize(factory.Target));
+
+                                encoder.EmitLDR(encoder.TargetRegister.Arg3, encoder.TargetRegister.Arg2);
+                                encoder.EmitCMP(encoder.TargetRegister.Arg3, 0);
+                                encoder.EmitJE(helper);
+
+                                // First arg: unused address of the TypeManager
+                                encoder.EmitMOV(encoder.TargetRegister.Arg0, (ushort)0);
+                                // Second arg: ~0 (index of inlined storage)
+                                encoder.EmitMVN(encoder.TargetRegister.Arg1, 0);
+                                encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase));
+                            }
                         }
                         else
                         {
-                            encoder.EmitMOV(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target));
-                            encoder.EmitSUB(encoder.TargetRegister.Arg2, NonGCStaticsNode.GetClassConstructorContextSize(factory.Target));
-
-                            encoder.EmitLDR(encoder.TargetRegister.Arg3, encoder.TargetRegister.Arg2);
-                            encoder.EmitCMP(encoder.TargetRegister.Arg3, 0);
-                            encoder.EmitJE(factory.HelperEntrypoint(HelperEntrypoint.GetThreadStaticBaseForType));
-
-                            encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase));
+                            encoder.EmitMOV(encoder.TargetRegister.Arg2, index);
+
+                            // First arg: address of the TypeManager slot that provides the helper with
+                            // information about module index and the type manager instance (which is used
+                            // for initialization on first access).
+                            encoder.EmitLDR(encoder.TargetRegister.Arg0, encoder.TargetRegister.Arg2);
+
+                            // Second arg: index of the type in the ThreadStatic section of the modules
+                            encoder.EmitLDR(encoder.TargetRegister.Arg1, encoder.TargetRegister.Arg2, factory.Target.PointerSize);
+
+                            ISymbolNode helper = factory.HelperEntrypoint(HelperEntrypoint.GetThreadStaticBaseForType);
+                            if (!factory.PreinitializationManager.HasLazyStaticConstructor(target))
+                            {
+                                encoder.EmitJMP(helper);
+                            }
+                            else
+                            {
+                                encoder.EmitMOV(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target));
+                                encoder.EmitSUB(encoder.TargetRegister.Arg2, NonGCStaticsNode.GetClassConstructorContextSize(factory.Target));
+
+                                encoder.EmitLDR(encoder.TargetRegister.Arg3, encoder.TargetRegister.Arg2);
+                                encoder.EmitCMP(encoder.TargetRegister.Arg3, 0);
+                                encoder.EmitJE(helper);
+
+                                encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase));
+                            }
                         }
                     }
                     break;
index a091fdfc943640fd9c4b99df1e9ab17dee9b481d..15c0a98e3dfeba6ff98e0375776abf2ef12a85d6 100644 (file)
@@ -71,32 +71,59 @@ namespace ILCompiler.DependencyAnalysis
                 case ReadyToRunHelperId.GetThreadStaticBase:
                     {
                         MetadataType target = (MetadataType)Target;
-
-                        encoder.EmitLEAQ(encoder.TargetRegister.Arg2, factory.TypeThreadStaticIndex(target));
-
-                        // First arg: address of the TypeManager slot that provides the helper with
-                        // information about module index and the type manager instance (which is used
-                        // for initialization on first access).
-                        AddrMode loadFromArg2 = new AddrMode(encoder.TargetRegister.Arg2, null, 0, 0, AddrModeSize.Int64);
-                        encoder.EmitMOV(encoder.TargetRegister.Arg0, ref loadFromArg2);
-
-                        // Second arg: index of the type in the ThreadStatic section of the modules
-                        AddrMode loadFromArg2AndDelta = new AddrMode(encoder.TargetRegister.Arg2, null, factory.Target.PointerSize, 0, AddrModeSize.Int32);
-                        encoder.EmitMOV(encoder.TargetRegister.Arg1, ref loadFromArg2AndDelta);
-
-                        if (!factory.PreinitializationManager.HasLazyStaticConstructor(target))
+                        ISortableSymbolNode index = factory.TypeThreadStaticIndex(target);
+                        if (index is TypeThreadStaticIndexNode ti && ti.Type == null)
                         {
-                            encoder.EmitJMP(factory.ExternSymbol("RhpGetThreadStaticBaseForType"));
+                            ISymbolNode helper = factory.ExternSymbol("RhpGetInlinedThreadStaticBase");
+
+                            if (!factory.PreinitializationManager.HasLazyStaticConstructor(target))
+                            {
+                                encoder.EmitJMP(helper);
+                            }
+                            else
+                            {
+                                encoder.EmitLEAQ(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target), -NonGCStaticsNode.GetClassConstructorContextSize(factory.Target));
+
+                                AddrMode initialized = new AddrMode(encoder.TargetRegister.Arg2, null, 0, 0, AddrModeSize.Int64);
+                                encoder.EmitCMP(ref initialized, 0);
+                                encoder.EmitJE(helper);
+
+                                // First arg: unused address of the TypeManager
+                                encoder.EmitMOV(encoder.TargetRegister.Arg0, 0);
+                                // Second arg: -1 (index of inlined storage)
+                                encoder.EmitMOV(encoder.TargetRegister.Arg1, -1);
+                                encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase));
+                            }
                         }
                         else
                         {
-                            encoder.EmitLEAQ(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target), - NonGCStaticsNode.GetClassConstructorContextSize(factory.Target));
-
-                            AddrMode initialized = new AddrMode(encoder.TargetRegister.Arg2, null, 0, 0, AddrModeSize.Int64);
-                            encoder.EmitCMP(ref initialized, 0);
-                            encoder.EmitJE(factory.ExternSymbol("RhpGetThreadStaticBaseForType"));
-
-                            encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase));
+                            encoder.EmitLEAQ(encoder.TargetRegister.Arg2, index);
+
+                            // First arg: address of the TypeManager slot that provides the helper with
+                            // information about module index and the type manager instance (which is used
+                            // for initialization on first access).
+                            AddrMode loadFromArg2 = new AddrMode(encoder.TargetRegister.Arg2, null, 0, 0, AddrModeSize.Int64);
+                            encoder.EmitMOV(encoder.TargetRegister.Arg0, ref loadFromArg2);
+
+                            // Second arg: index of the type in the ThreadStatic section of the modules
+                            AddrMode loadFromArg2AndDelta = new AddrMode(encoder.TargetRegister.Arg2, null, factory.Target.PointerSize, 0, AddrModeSize.Int32);
+                            encoder.EmitMOV(encoder.TargetRegister.Arg1, ref loadFromArg2AndDelta);
+
+                            ISymbolNode helper = factory.HelperEntrypoint(HelperEntrypoint.GetThreadStaticBaseForType);
+                            if (!factory.PreinitializationManager.HasLazyStaticConstructor(target))
+                            {
+                                encoder.EmitJMP(helper);
+                            }
+                            else
+                            {
+                                encoder.EmitLEAQ(encoder.TargetRegister.Arg2, factory.TypeNonGCStaticsSymbol(target), -NonGCStaticsNode.GetClassConstructorContextSize(factory.Target));
+
+                                AddrMode initialized = new AddrMode(encoder.TargetRegister.Arg2, null, 0, 0, AddrModeSize.Int64);
+                                encoder.EmitCMP(ref initialized, 0);
+                                encoder.EmitJE(helper);
+
+                                encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase));
+                            }
                         }
                     }
                     break;
index 4c48b63430a506bc44946f8e2a4e36caaecf5d30..6fa2a7a70cc114ab7cfc6f6255b7307ae5ca9b29 100644 (file)
@@ -16,6 +16,7 @@ namespace ILCompiler.DependencyAnalysis
     public class ThreadStaticsNode : EmbeddedObjectNode, ISymbolDefinitionNode
     {
         private MetadataType _type;
+        private InlinedThreadStatics _inlined;
 
         public ThreadStaticsNode(MetadataType type, NodeFactory factory)
         {
@@ -24,6 +25,11 @@ namespace ILCompiler.DependencyAnalysis
             _type = type;
         }
 
+        public ThreadStaticsNode(InlinedThreadStatics inlined, NodeFactory factory)
+        {
+            _inlined = inlined;
+        }
+
         protected override string GetName(NodeFactory factory) => this.GetMangledName(factory.NameMangler);
 
         protected override void OnMarked(NodeFactory factory)
@@ -42,12 +48,20 @@ namespace ILCompiler.DependencyAnalysis
 
         public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb)
         {
-            sb.Append(GetMangledName(_type, nameMangler));
+            string mangledName = _type == null ? "_inlinedThreadStatics" : GetMangledName(_type, nameMangler);
+            sb.Append(mangledName);
         }
 
         private ISymbolNode GetGCStaticEETypeNode(NodeFactory factory)
         {
-            GCPointerMap map = GCPointerMap.FromThreadStaticLayout(_type);
+            GCPointerMap map = _type != null ?
+                GCPointerMap.FromThreadStaticLayout(_type) :
+                GCPointerMap.FromInlinedThreadStatics(
+                    _inlined.GetTypes(),
+                    _inlined.GetOffsets(),
+                    _inlined.GetSize(),
+                    factory.Target.PointerSize);
+
             return factory.GCStaticEEType(map);
         }
 
@@ -57,20 +71,41 @@ namespace ILCompiler.DependencyAnalysis
 
             result.Add(new DependencyListEntry(GetGCStaticEETypeNode(factory), "ThreadStatic MethodTable"));
 
-            if (factory.PreinitializationManager.HasEagerStaticConstructor(_type))
+            if (_type != null)
             {
-                result.Add(new DependencyListEntry(factory.EagerCctorIndirection(_type.GetStaticConstructor()), "Eager .cctor"));
-            }
 
-            ModuleUseBasedDependencyAlgorithm.AddDependenciesDueToModuleUse(ref result, factory, _type.Module);
+                if (factory.PreinitializationManager.HasEagerStaticConstructor(_type))
+                {
+                    result.Add(new DependencyListEntry(factory.EagerCctorIndirection(_type.GetStaticConstructor()), "Eager .cctor"));
+                }
+
+                ModuleUseBasedDependencyAlgorithm.AddDependenciesDueToModuleUse(ref result, factory, _type.Module);
+            }
+            else
+            {
+                foreach (var type in _inlined.GetTypes())
+                {
+                    if (factory.PreinitializationManager.HasEagerStaticConstructor(type))
+                    {
+                        result.Add(new DependencyListEntry(factory.EagerCctorIndirection(type.GetStaticConstructor()), "Eager .cctor"));
+                    }
+
+                    ModuleUseBasedDependencyAlgorithm.AddDependenciesDueToModuleUse(ref result, factory, type.Module);
+                }
+            }
 
             return result;
         }
 
-        public override bool HasConditionalStaticDependencies => _type.ConvertToCanonForm(CanonicalFormKind.Specific) != _type;
+        public override bool HasConditionalStaticDependencies =>
+            _type != null ?
+                _type.ConvertToCanonForm(CanonicalFormKind.Specific) != _type:
+                false;
 
         public override IEnumerable<CombinedDependencyListEntry> GetConditionalStaticDependencies(NodeFactory factory)
         {
+            Debug.Assert(_type != null);
+
             // If we have a type loader template for this type, we need to keep track of the generated
             // bases in the type info hashtable. The type symbol node does such accounting.
             return new CombinedDependencyListEntry[]
@@ -91,10 +126,21 @@ namespace ILCompiler.DependencyAnalysis
             builder.EmitPointerReloc(GetGCStaticEETypeNode(factory));
         }
 
+        public MetadataType Type => _type;
+
         public override int ClassCode => 2091208431;
 
         public override int CompareToImpl(ISortableNode other, CompilerComparer comparer)
         {
+            // force the type of the storage block for inlined threadstatics to be "less"
+            // than other storage blocks, - to ensure it is serialized as the item #0
+            if (_type == null)
+            {
+                // there should only be at most one inlined storage type.
+                Debug.Assert(other != null);
+                return -1;
+            }
+
             return comparer.Compare(_type, ((ThreadStaticsNode)other)._type);
         }
     }
index f9200bef525b93fce5db8bf04602e9cd50d2faa8..772970bf145caedcc311a769c46295a6a3040823 100644 (file)
@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Diagnostics;
 using Internal.Text;
 using Internal.TypeSystem;
 
@@ -12,18 +13,26 @@ namespace ILCompiler.DependencyAnalysis
     public class TypeThreadStaticIndexNode : DehydratableObjectNode, ISymbolDefinitionNode, ISortableSymbolNode
     {
         private MetadataType _type;
+        private ThreadStaticsNode _inlinedThreadStatics;
 
         public TypeThreadStaticIndexNode(MetadataType type)
         {
             _type = type;
         }
 
+        public TypeThreadStaticIndexNode(ThreadStaticsNode inlinedThreadStatics)
+        {
+            _inlinedThreadStatics = inlinedThreadStatics;
+        }
+
         public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb)
         {
-            sb.Append(nameMangler.NodeMangler.ThreadStaticsIndex(_type));
+            sb.Append(_type != null ? nameMangler.NodeMangler.ThreadStaticsIndex(_type) : "_inlinedThreadStaticsIndex");
         }
+
         public int Offset => 0;
         protected override string GetName(NodeFactory factory) => this.GetMangledName(factory.NameMangler);
+
         protected override ObjectNodeSection GetDehydratedSection(NodeFactory factory)
         {
             if (factory.Target.IsWindows)
@@ -31,14 +40,19 @@ namespace ILCompiler.DependencyAnalysis
             else
                 return ObjectNodeSection.DataSection;
         }
+
         public override bool IsShareable => true;
         public override bool StaticDependenciesAreComputed => true;
 
         protected override DependencyList ComputeNonRelocationBasedDependencies(NodeFactory factory)
         {
+            ISymbolDefinitionNode node = _type != null ?
+                        factory.TypeThreadStaticsSymbol(_type) :
+                        _inlinedThreadStatics;
+
             return new DependencyList
             {
-                new DependencyListEntry(factory.TypeThreadStaticsSymbol(_type), "Thread static storage")
+                new DependencyListEntry(node, "Thread static storage")
             };
         }
 
@@ -52,8 +66,21 @@ namespace ILCompiler.DependencyAnalysis
             int typeTlsIndex = 0;
             if (!relocsOnly)
             {
-                var node = factory.TypeThreadStaticsSymbol(_type);
-                typeTlsIndex = ((ThreadStaticsNode)node).IndexFromBeginningOfArray;
+                if (_type != null)
+                {
+                    ISymbolDefinitionNode node = factory.TypeThreadStaticsSymbol(_type);
+                    typeTlsIndex = ((ThreadStaticsNode)node).IndexFromBeginningOfArray;
+                }
+                else
+                {
+                    // we use -1 to specify the index of inlined threadstatics,
+                    // which are stored separately from uninlined ones.
+                    typeTlsIndex = -1;
+
+                    // the type of the storage block for inlined threadstatics, if present,
+                    // is serialized as the item #0 among other storage block types.
+                    Debug.Assert(_inlinedThreadStatics.IndexFromBeginningOfArray == 0);
+                }
             }
 
             objData.EmitPointerReloc(factory.TypeManagerIndirection);
index 8f1bcca2e545dbb12a58a45b90a17851d58106ea..953c059649ebb90ef7413dabeb86e712de4e07b7 100644 (file)
@@ -252,6 +252,11 @@ namespace ILCompiler
             return new ScannedPreinitializationPolicy(_factory.PreinitializationManager, MarkedNodes);
         }
 
+        public InlinedThreadStatics GetInlinedThreadStatics()
+        {
+            return new ScannedInlinedThreadStatics(_factory, MarkedNodes);
+        }
+
         private sealed class ScannedVTableProvider : VTableSliceProvider
         {
             private Dictionary<TypeDesc, IReadOnlyList<MethodDesc>> _vtableSlices = new Dictionary<TypeDesc, IReadOnlyList<MethodDesc>>();
@@ -674,6 +679,75 @@ namespace ILCompiler
                 => _importationErrors.TryGetValue(method, out var exception) ? exception : null;
         }
 
+        private sealed class ScannedInlinedThreadStatics : InlinedThreadStatics
+        {
+            private readonly List<MetadataType> _types;
+            private readonly Dictionary<MetadataType, int> _offsets;
+            private readonly int _size;
+
+            internal override bool IsComputed() => true;
+            internal override List<MetadataType> GetTypes() => _types;
+            internal override Dictionary<MetadataType, int> GetOffsets() => _offsets;
+            internal override int GetSize() => _size;
+
+            public ScannedInlinedThreadStatics(NodeFactory factory, ImmutableArray<DependencyNodeCore<NodeFactory>> markedNodes)
+            {
+                List<ThreadStaticsNode> threadStaticNodes = new List<ThreadStaticsNode>();
+                foreach (var markedNode in markedNodes)
+                {
+                    if (markedNode is ThreadStaticsNode threadStaticNode)
+                    {
+                        threadStaticNodes.Add(threadStaticNode);
+                    }
+                }
+
+                // skip MT pointer
+                int nextDataOffset = factory.Target.PointerSize;
+
+                List<MetadataType> types = new List<MetadataType>();
+                Dictionary<MetadataType, int> offsets = new Dictionary<MetadataType, int>();
+
+                if (threadStaticNodes.Count > 0)
+                {
+                    threadStaticNodes.Sort(CompilerComparer.Instance);
+                    for (int i = 0; i < threadStaticNodes.Count; i++)
+                    {
+                        ThreadStaticsNode threadStaticNode = threadStaticNodes[i];
+                        MetadataType t = threadStaticNode.Type;
+
+                        // do not inline storage for shared generics
+                        if (t.ConvertToCanonForm(CanonicalFormKind.Specific) != t)
+                            continue;
+
+#if DEBUG
+                        // do not inline storage for some types in debug - for test coverage
+                        if (i % 8 == 0)
+                            continue;
+#endif
+
+                        types.Add(t);
+
+                        // N.B. for ARM32, we would need to deal with > PointerSize alignments.
+                        //      GCStaticEEType does not currently set RequiresAlign8Flag
+                        Debug.Assert(t.ThreadGcStaticFieldAlignment.AsInt <= factory.Target.PointerSize);
+                        nextDataOffset = nextDataOffset.AlignUp(t.ThreadGcStaticFieldAlignment.AsInt);
+
+                        // reported offset is from the MT pointer, adjust for that
+                        offsets.Add(t, nextDataOffset - factory.Target.PointerSize);
+
+                        // ThreadGcStaticFieldSize includes MT pointer, we will not need space for it
+                        int dataSize = t.ThreadGcStaticFieldSize.AsInt - factory.Target.PointerSize;
+                        nextDataOffset += dataSize;
+                    }
+                }
+
+                _types = types;
+                _offsets = offsets;
+                // the size is at least MIN_OBJECT_SIZE
+                _size = Math.Max(nextDataOffset, factory.Target.PointerSize * 3);
+            }
+        }
+
         private sealed class ScannedPreinitializationPolicy : TypePreinit.TypePreinitializationPolicy
         {
             private readonly HashSet<TypeDesc> _canonFormsWithCctorChecks = new HashSet<TypeDesc>();
diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/InlinedThreadStatics.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/InlinedThreadStatics.cs
new file mode 100644 (file)
index 0000000..d746962
--- /dev/null
@@ -0,0 +1,18 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using Internal.TypeSystem;
+
+namespace ILCompiler
+{
+    public class InlinedThreadStatics
+    {
+        internal virtual bool IsComputed() => false;
+
+        internal virtual int GetSize() => throw new InvalidOperationException();
+        internal virtual List<MetadataType> GetTypes() => throw new InvalidOperationException();
+        internal virtual Dictionary<MetadataType, int> GetOffsets() => throw new InvalidOperationException();
+    }
+}
index fc41ba97ec0f0d5f40e6cc9353f0dc112a461530..ad9980639e2594fb2a6ebb7c0aa0ca986a318942 100644 (file)
     <Compile Include="Compiler\Logging\NativeAotFatalErrorException.cs" />
     <Compile Include="Compiler\ManifestResourceBlockingPolicy.cs" />
     <Compile Include="Compiler\MethodImportationErrorProvider.cs" />
+    <Compile Include="Compiler\InlinedThreadStatics.cs" />
     <Compile Include="Compiler\MstatObjectDumper.cs" />
     <Compile Include="Compiler\NoMetadataBlockingPolicy.cs" />
     <Compile Include="Compiler\DependencyAnalysis\FrozenObjectNode.cs" />
index a51508005f53624b6a63a4c1d68f2dfbd4b7dac5..abae360824375f998daedc4c8ddcce950ad58da4 100644 (file)
@@ -10,8 +10,8 @@ namespace ILCompiler.DependencyAnalysis
     public sealed class RyuJitNodeFactory : NodeFactory
     {
         public RyuJitNodeFactory(CompilerTypeSystemContext context, CompilationModuleGroup compilationModuleGroup, MetadataManager metadataManager,
-            InteropStubManager interopStubManager, NameMangler nameMangler, VTableSliceProvider vtableSliceProvider, DictionaryLayoutProvider dictionaryLayoutProvider, PreinitializationManager preinitializationManager)
-            : base(context, compilationModuleGroup, metadataManager, interopStubManager, nameMangler, new LazyGenericsDisabledPolicy(), vtableSliceProvider, dictionaryLayoutProvider, new ExternSymbolsImportedNodeProvider(), preinitializationManager)
+            InteropStubManager interopStubManager, NameMangler nameMangler, VTableSliceProvider vtableSliceProvider, DictionaryLayoutProvider dictionaryLayoutProvider, InlinedThreadStatics inlinedThreadStatics, PreinitializationManager preinitializationManager)
+            : base(context, compilationModuleGroup, metadataManager, interopStubManager, nameMangler, new LazyGenericsDisabledPolicy(), vtableSliceProvider, dictionaryLayoutProvider, inlinedThreadStatics, new ExternSymbolsImportedNodeProvider(), preinitializationManager)
         {
         }
 
index 3c20598ffb099040fb8c4e0bf19677481023aa1d..ea29f7ff342c95ec9fe2c6072326fa00ff6eed62 100644 (file)
@@ -123,7 +123,7 @@ namespace ILCompiler
             if (_resilient)
                 options |= RyuJitCompilationOptions.UseResilience;
 
-            var factory = new RyuJitNodeFactory(_context, _compilationGroup, _metadataManager, _interopStubManager, _nameMangler, _vtableSliceProvider, _dictionaryLayoutProvider, GetPreinitializationManager());
+            var factory = new RyuJitNodeFactory(_context, _compilationGroup, _metadataManager, _interopStubManager, _nameMangler, _vtableSliceProvider, _dictionaryLayoutProvider, _inlinedThreadStatics, GetPreinitializationManager());
 
             JitConfigProvider.Initialize(_context.Target, jitFlagBuilder.ToArray(), _ryujitOptions, _jitPath);
             DependencyAnalyzerBase<NodeFactory> graph = CreateDependencyGraph(factory, new ObjectNode.ObjectNodeComparer(CompilerComparer.Instance));
index b8a56bb708cb0831b764ad5d1ec147013cfeb687..cdbf7fe492e273795fd37dbab67ab42cb61f6200 100644 (file)
@@ -13,7 +13,6 @@ using Internal.ReadyToRunConstants;
 
 using ILCompiler;
 using ILCompiler.DependencyAnalysis;
-using Internal.TypeSystem.Ecma;
 
 #if SUPPORT_JIT
 using MethodCodeNode = Internal.Runtime.JitSupport.JitMethodCodeNode;
@@ -2057,6 +2056,11 @@ namespace Internal.JitInterface
             CORINFO_FIELD_FLAGS fieldFlags = (CORINFO_FIELD_FLAGS)0;
             uint fieldOffset = (field.IsStatic && field.HasRva ? 0xBAADF00D : (uint)field.Offset.AsInt);
 
+            if (field.IsThreadStatic && field.OwningType is MetadataType mt)
+            {
+                fieldOffset += _compilation.NodeFactory.ThreadStaticBaseOffset(mt);
+            }
+
             if (field.IsStatic)
             {
                 fieldFlags |= CORINFO_FIELD_FLAGS.CORINFO_FLG_FIELD_STATIC;
index 56689af570bf01d97b1312dc083173420ec43e3e..2a9aff55141987201d86c63ba589d8b5f8a434b4 100644 (file)
@@ -485,6 +485,15 @@ namespace ILCompiler
                     preinitManager = new PreinitializationManager(typeSystemContext, compilationGroup, ilProvider, scanResults.GetPreinitializationPolicy());
                     builder.UsePreinitializationManager(preinitManager);
                 }
+
+                // If we have a scanner, we can inline threadstatics storage using the information
+                // we collected at scanning time.
+                // Inlined storage implies a single type manager, thus we do not do it in multifile case.
+                // This could be a command line switch if we really wanted to.
+                if (!multiFile)
+                {
+                    builder.UseInlinedThreadStatics(scanResults.GetInlinedThreadStatics());
+                }
             }
 
             string ilDump = Get(_command.IlDump);
index a6b7612c4895a7eee87080c56b1d7d73a4bf5805..f14de76be76442a2f059f8654e202c6e3d41af24 100644 (file)
@@ -51,7 +51,7 @@ public class Program
 
         // Just count the number of warnings and errors. There are so many right now that it's not worth enumerating the list
 #if DEBUG
-        const int MinWarnings = 16500;
+        const int MinWarnings = 12000;
         const int MaxWarnings = 20000;
 #else
         const int MinWarnings = 12000;