From: Mike McLaughlin Date: Sat, 17 Apr 2021 05:53:14 +0000 (-0700) Subject: Use MacOS coredump special thread info (#2187) X-Git-Tag: submit/tizen/20210909.063632~15^2~98 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=e39c070cf0b603ed47b6541adc863c14c1186f8a;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Use MacOS coredump special thread info (#2187) * Use MacOS coredump special thread info Disable dotnet-dump tests; enable lldb tests on MacOS Bump the SOS test runner time outs to 10min 'setsostid' with no parameters displays the thread info list. * Code review feedback * Update runtime version * Update aspnetcore versions --- diff --git a/diagnostics.sln b/diagnostics.sln index dda545db5..df6bf029e 100644 --- a/diagnostics.sln +++ b/diagnostics.sln @@ -186,6 +186,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "inc", "inc", "{BE45F03E-D70 src\SOS\inc\lldbservices.h = src\SOS\inc\lldbservices.h src\SOS\inc\remotememoryservice.h = src\SOS\inc\remotememoryservice.h src\SOS\inc\runtime.h = src\SOS\inc\runtime.h + src\SOS\inc\specialthreadinfo.h = src\SOS\inc\specialthreadinfo.h src\SOS\inc\symbolservice.h = src\SOS\inc\symbolservice.h src\SOS\inc\target.h = src\SOS\inc\target.h EndProjectSection diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2fe5802f7..7b1ff751f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -32,21 +32,21 @@ 7f13798e5f567b72ffe63205bf49839245f0f8c1 - + https://github.com/dotnet/aspnetcore - 2d6e985887068528980b5037b945232f35ef8019 + cede6c4fbb3ab6d7c42cc3e9a0fcf5b802a36af2 - + https://github.com/dotnet/aspnetcore - 2d6e985887068528980b5037b945232f35ef8019 + cede6c4fbb3ab6d7c42cc3e9a0fcf5b802a36af2 - + https://github.com/dotnet/runtime - 2588311215b3e9b49c695369941698f333f52fe9 + d25620b3fe5fba7b8fd7065f2d5947d0a20c6a30 - + https://github.com/dotnet/runtime - 2588311215b3e9b49c695369941698f333f52fe9 + d25620b3fe5fba7b8fd7065f2d5947d0a20c6a30 diff --git a/eng/Versions.props b/eng/Versions.props index 33a423bcc..9b287febe 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -17,11 +17,11 @@ 5.0.0 $(MicrosoftNETCoreApp50Version) - 6.0.0-preview.4.21214.9 - 6.0.0-preview.4.21214.9 + 6.0.0-preview.4.21215.13 + 6.0.0-preview.4.21215.13 - 6.0.0-preview.4.21215.16 - 6.0.0-preview.4.21215.16 + 6.0.0-preview.5.21216.8 + 6.0.0-preview.5.21216.8 6.0.100-preview.1.21103.13 diff --git a/src/SOS/SOS.UnitTests/SOS.cs b/src/SOS/SOS.UnitTests/SOS.cs index 45744bdf4..20a80e334 100644 --- a/src/SOS/SOS.UnitTests/SOS.cs +++ b/src/SOS/SOS.UnitTests/SOS.cs @@ -69,7 +69,7 @@ public class SOS // Test against a crash dump. if (information.TestConfiguration.DebuggeeDumpInputRootDir() != null) { - if (!SOSRunner.IsAlpine() && OS.Kind != OSKind.OSX) + if (!SOSRunner.IsAlpine()) { // With cdb (Windows) or lldb (Linux) using (SOSRunner runner = await SOSRunner.StartDebugger(information, SOSRunner.DebuggerAction.LoadDump)) @@ -79,7 +79,8 @@ public class SOS } // Using the dotnet-dump analyze tool if the path exists in the config file. - if (information.TestConfiguration.DotNetDumpPath() != null) + // TODO: dotnet-dump currently doesn't support macho core dumps that the MacOS createdump generates + if (information.TestConfiguration.DotNetDumpPath() != null && OS.Kind != OSKind.OSX) { // Don't test dotnet-dump on triage dumps when running on desktop CLR. if (information.TestConfiguration.IsNETCore || information.DumpType != SOSRunner.DumpType.Triage) diff --git a/src/SOS/SOS.UnitTests/SOSRunner.cs b/src/SOS/SOS.UnitTests/SOSRunner.cs index 7f29f0beb..53a08befd 100644 --- a/src/SOS/SOS.UnitTests/SOSRunner.cs +++ b/src/SOS/SOS.UnitTests/SOSRunner.cs @@ -253,7 +253,7 @@ public class SOSRunner : IDisposable ProcessRunner processRunner = new ProcessRunner(exePath, ReplaceVariables(variables, arguments.ToString())). WithEnvironmentVariable("COMPlus_DbgEnableElfDumpOnMacOS", "1"). WithLog(new TestRunner.TestLogger(outputHelper.IndentedOutput)). - WithTimeout(TimeSpan.FromMinutes(5)); + WithTimeout(TimeSpan.FromMinutes(10)); if (dumpGeneration == DumpGenerator.CreateDump) { @@ -320,7 +320,7 @@ public class SOSRunner : IDisposable } ProcessRunner dotnetDumpRunner = new ProcessRunner(config.DotNetDumpHost(), ReplaceVariables(variables, dotnetDumpArguments.ToString())). WithLog(new TestRunner.TestLogger(dotnetDumpOutputHelper)). - WithTimeout(TimeSpan.FromMinutes(5)). + WithTimeout(TimeSpan.FromMinutes(10)). WithExpectedExitCode(0); dotnetDumpRunner.Start(); diff --git a/src/SOS/inc/specialthreadinfo.h b/src/SOS/inc/specialthreadinfo.h new file mode 100644 index 000000000..7e661089e --- /dev/null +++ b/src/SOS/inc/specialthreadinfo.h @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// ****************************************************************************** +// WARNING!!!: This code is also used by createdump in the runtime repo. +// See: https://github.com/dotnet/runtime/blob/main/src/coreclr/debug/createdump/specialthreadinfo.h +// ****************************************************************************** + +// This defines a workaround to the MacOS dump format not having the OS process +// and thread ids that SOS needs to map thread "indexes" to thread "ids". The MacOS +// createdump adds this special memory region at this specific address that is not +// in the user or kernel address spaces. lldb is fine with it. + +#define SPECIAL_THREADINFO_SIGNATURE "THREADINFO" + +const uint64_t SpecialThreadInfoAddress = 0x7fffffff00000000; + +struct SpecialThreadInfoHeader +{ + char signature[16]; + uint32_t pid; + uint32_t numThreads; // The number of SpecialThreadInfoEntry's after this header +}; + +struct SpecialThreadInfoEntry +{ + uint32_t tid; + uint64_t sp; +}; diff --git a/src/SOS/lldbplugin/services.cpp b/src/SOS/lldbplugin/services.cpp index c9a584952..28e6ee1fd 100644 --- a/src/SOS/lldbplugin/services.cpp +++ b/src/SOS/lldbplugin/services.cpp @@ -16,18 +16,18 @@ #define InvalidTimeStamp 0xFFFFFFFE; #define InvalidChecksum 0xFFFFFFFF; -ULONG g_currentThreadIndex = (ULONG)-1; -ULONG g_currentThreadSystemId = (ULONG)-1; char *g_coreclrDirectory = nullptr; char *g_pluginModuleDirectory = nullptr; -LLDBServices::LLDBServices(lldb::SBDebugger debugger) : +LLDBServices::LLDBServices(lldb::SBDebugger debugger) : m_ref(1), m_debugger(debugger), m_interpreter(debugger.GetCommandInterpreter()), m_currentProcess(nullptr), m_currentThread(nullptr), - m_currentStopId(0) + m_currentStopId(0), + m_processId(0), + m_threadInfoInitialized(false) { ClearCache(); @@ -505,8 +505,8 @@ LLDBServices::GetLastEventInformation( return E_FAIL; } - *processId = process.GetProcessID(); - *threadId = thread.GetThreadID(); + *processId = GetProcessId(process); + *threadId = GetThreadId(thread); // Enumerate each stack frame at the special "throw" // breakpoint and find the raise exception function @@ -1381,7 +1381,7 @@ LLDBServices::GetCurrentProcessSystemId( return E_FAIL; } - *sysId = process.GetProcessID(); + *sysId = GetProcessId(process); return S_OK; } @@ -1401,14 +1401,6 @@ LLDBServices::GetCurrentThreadId( return E_FAIL; } - // This is allow the a valid current TID to be returned to - // workaround a bug in lldb on core dumps. - if (g_currentThreadIndex != (ULONG)-1) - { - *id = g_currentThreadIndex; - return S_OK; - } - *id = thread.GetIndexID(); return S_OK; } @@ -1447,15 +1439,7 @@ LLDBServices::GetCurrentThreadSystemId( return E_FAIL; } - // This is allow the a valid current TID to be returned to - // workaround a bug in lldb on core dumps. - if (g_currentThreadSystemId != (ULONG)-1) - { - *sysId = g_currentThreadSystemId; - return S_OK; - } - - *sysId = thread.GetThreadID(); + *sysId = GetThreadId(thread); return S_OK; } @@ -1464,44 +1448,21 @@ LLDBServices::GetThreadIdBySystemId( ULONG sysId, PULONG threadId) { - HRESULT hr = E_FAIL; - ULONG id = 0; - - lldb::SBProcess process; - lldb::SBThread thread; if (threadId == NULL) { return E_INVALIDARG; } - process = GetCurrentProcess(); - if (!process.IsValid()) - { - goto exit; - } - - // If we have a "fake" thread OS (system) id and a fake thread index, - // we need to return fake thread index. - if (g_currentThreadSystemId == sysId && g_currentThreadIndex != (ULONG)-1) - { - id = g_currentThreadIndex; - } - else + lldb::SBThread thread = GetThreadBySystemId(sysId); + if (!thread.IsValid()) { - thread = process.GetThreadByID(sysId); - if (!thread.IsValid()) - { - goto exit; - } - - id = thread.GetIndexID(); + *threadId = 0; + return E_FAIL; } - hr = S_OK; -exit: - *threadId = id; - return hr; + *threadId = thread.GetIndexID(); + return S_OK; } HRESULT @@ -1523,23 +1484,7 @@ LLDBServices::GetThreadContextBySystemId( } memset(context, 0, contextSize); - process = GetCurrentProcess(); - if (!process.IsValid()) - { - goto exit; - } - - // If we have a "fake" thread OS (system) id and a fake thread index, - // use the fake thread index to get the context. - if (g_currentThreadSystemId == sysId && g_currentThreadIndex != (ULONG)-1) - { - thread = process.GetThreadByIndexID(g_currentThreadIndex); - } - else - { - thread = process.GetThreadByID(sysId); - } - + thread = GetThreadBySystemId(sysId); if (!thread.IsValid()) { goto exit; @@ -2234,7 +2179,7 @@ LLDBServices::GetThreadIdsByIndex( } if (sysIds != nullptr) { - sysIds[index] = thread.GetThreadID(); + sysIds[index] = GetThreadId(thread); } } return S_OK; @@ -2244,12 +2189,12 @@ HRESULT LLDBServices::SetCurrentThreadSystemId( ULONG sysId) { - lldb::SBProcess process = GetCurrentProcess(); - if (!process.IsValid()) + lldb::SBThread thread = GetThreadBySystemId(sysId); + if (!thread.IsValid()) { - return E_UNEXPECTED; + return E_FAIL; } - if (!process.SetSelectedThreadByID(sysId)) + if (!thread.GetProcess().SetSelectedThread(thread)) { return E_FAIL; } @@ -2334,6 +2279,150 @@ exit: // Helper functions //---------------------------------------------------------------------------- +void +LLDBServices::InitializeThreadInfo(lldb::SBProcess process) +{ +#ifdef __APPLE__ + if (m_threadInfoInitialized) + { + return; + } + m_threadInfoInitialized = true; + + // Only attempt to read the special thread info block if MacOS core dump + const char* pluginName = process.GetPluginName(); + if (strcmp(pluginName, "mach-o-core") != 0) + { + return; + } + SpecialThreadInfoHeader header; + lldb::SBError error; + size_t read = process.ReadMemory(SpecialThreadInfoAddress, &header, sizeof(SpecialThreadInfoHeader), error); + if (error.Fail() || read != sizeof(header)) + { + return; + } + if (strncmp(header.signature, SPECIAL_THREADINFO_SIGNATURE, sizeof(SPECIAL_THREADINFO_SIGNATURE)) != 0) + { + Output(DEBUG_OUTPUT_WARNING, "Special thread info signature invalid\n"); + return; + } + uint32_t number = process.GetNumThreads(); + if (number != header.numThreads) + { + Output(DEBUG_OUTPUT_WARNING, "Special thread info number of threads mismatched - lldb: %d header: %d\n", number, header.numThreads); + return; + } + m_processId = header.pid; + m_threadInfos.clear(); + + uint64_t address = SpecialThreadInfoAddress + sizeof(header); + for (int index = 0; index < number; index++) + { + SpecialThreadInfoEntry entry; + read = process.ReadMemory(address, &entry, sizeof(SpecialThreadInfoEntry), error); + if (error.Fail() || read != sizeof(entry)) { + Output(DEBUG_OUTPUT_WARNING, "Special thread info entry %d read failed\n", index); + break; + } + m_threadInfos.push_back(entry); + address += sizeof(SpecialThreadInfoEntry); + + // Validate that the thread stack pointer matches the thread info's. + lldb::SBThread thread = process.GetThreadAtIndex(index); + if (thread.IsValid()) + { + lldb::SBFrame frame = thread.GetFrameAtIndex(0); + if (frame.IsValid()) + { + if (frame.GetSP() != entry.sp) + { + Output(DEBUG_OUTPUT_WARNING, "Special thread info SP (%p) doesn't match %p\n", (void*)entry.sp, (void*)frame.GetSP()); + } + } + else + { + Output(DEBUG_OUTPUT_WARNING, "Invalid stack frame for thread %d\n", index); + } + } + else + { + Output(DEBUG_OUTPUT_WARNING, "Invalid thread %d\n", index); + } + } +#endif +} + +lldb::SBThread +LLDBServices::GetThreadBySystemId( + ULONG sysId) +{ + lldb::SBProcess process; + lldb::SBThread thread; + + if (sysId == 0) + { + goto exit; + } + + process = GetCurrentProcess(); + if (!process.IsValid()) + { + goto exit; + } + + for (int index = 0; index < process.GetNumThreads(); index++) + { + if (m_threadInfos.size() <= index) + { + break; + } + if (sysId == m_threadInfos[index].tid) + { + thread = process.GetThreadAtIndex(index); + goto exit; + } + } + + thread = process.GetThreadByID(sysId); + +exit: + return thread; +} + +void +LLDBServices::AddThreadInfoEntry(uint32_t tid, uint32_t index) +{ + // Make sure there is room in the thread infos vector + if (m_threadInfos.empty()) + { + uint32_t number; + GetNumberThreads(&number); + m_threadInfos.assign(number, SpecialThreadInfoEntry{ 0, 0 }); + } + m_threadInfos[index - 1] = SpecialThreadInfoEntry{ tid, 0 }; +} + +uint32_t +LLDBServices::GetProcessId(lldb::SBProcess process) +{ + return m_processId != 0 ? m_processId : process.GetProcessID(); +} + +uint32_t +LLDBServices::GetThreadId(lldb::SBThread thread) +{ + uint32_t index = thread.GetIndexID() - 1; + if (m_threadInfos.size() > index && m_threadInfos[index].tid != 0) + { + return m_threadInfos[index].tid; + } + else + { + return thread.GetThreadID(); + } +} + lldb::SBProcess LLDBServices::GetCurrentProcess() { @@ -2565,8 +2654,10 @@ LLDBServices::FlushCheck() lldb::SBProcess process = GetCurrentProcess(); if (process.IsValid()) { + InitializeThreadInfo(process); + // Has the process changed since the last commmand? - Extensions::GetInstance()->UpdateTarget(process.GetProcessID()); + Extensions::GetInstance()->UpdateTarget(GetProcessId(process)); // Has the target "moved" (been continued) since the last command? Flush the target. uint32_t stopId = process.GetStopID(); @@ -2579,6 +2670,8 @@ LLDBServices::FlushCheck() else { Extensions::GetInstance()->DestroyTarget(); + m_threadInfoInitialized = false; + m_processId = 0; } } diff --git a/src/SOS/lldbplugin/services.h b/src/SOS/lldbplugin/services.h index 14912afd1..cabbf9fa0 100644 --- a/src/SOS/lldbplugin/services.h +++ b/src/SOS/lldbplugin/services.h @@ -17,7 +17,10 @@ private: lldb::SBProcess *m_currentProcess; lldb::SBThread *m_currentThread; uint32_t m_currentStopId; + uint32_t m_processId; std::set m_commands; + std::vector m_threadInfos; + bool m_threadInfoInitialized; BYTE m_cache[CACHE_SIZE]; ULONG64 m_startCache; @@ -42,6 +45,10 @@ private: void LoadNativeSymbols(lldb::SBTarget target, lldb::SBModule module, PFN_MODULE_LOAD_CALLBACK callback); + void InitializeThreadInfo(lldb::SBProcess process); + uint32_t GetProcessId(lldb::SBProcess process); + uint32_t GetThreadId(lldb::SBThread thread); + lldb::SBThread GetThreadBySystemId(ULONG sysId); lldb::SBProcess GetCurrentProcess(); lldb::SBThread GetCurrentThread(); lldb::SBFrame GetCurrentFrame(); @@ -50,6 +57,10 @@ public: LLDBServices(lldb::SBDebugger debugger); ~LLDBServices(); + std::vector& ThreadInfos() { return m_threadInfos; } + + void AddThreadInfoEntry(uint32_t tid, uint32_t index); + lldb::SBProcess* SetCurrentProcess(lldb::SBProcess* process) { return (lldb::SBProcess*)InterlockedExchangePointer(&m_currentProcess, process); diff --git a/src/SOS/lldbplugin/setsostidcommand.cpp b/src/SOS/lldbplugin/setsostidcommand.cpp index 9955be041..ffa729cd2 100644 --- a/src/SOS/lldbplugin/setsostidcommand.cpp +++ b/src/SOS/lldbplugin/setsostidcommand.cpp @@ -25,33 +25,44 @@ public: if (arguments == nullptr || arguments[0] == nullptr) { - if (g_currentThreadSystemId == (ULONG)-1 || g_currentThreadIndex == (ULONG)-1) + int index = 1; + result.Printf("OS TID -> lldb index\n"); + for (const SpecialThreadInfoEntry& entry: g_services->ThreadInfos()) { - result.Printf("sos OS tid not mapped\n"); + if (entry.tid != 0) + { + result.Printf("0x%08x -> %d\n", entry.tid, index); + } + index++; } - else { - result.Printf("sos OS tid 0x%x mapped to lldb thread index %d\n", - g_currentThreadSystemId, g_currentThreadIndex); - } - } - else if (strcmp(arguments[0], "-clear") == 0) { - g_currentThreadIndex = (ULONG)-1; - g_currentThreadSystemId = (ULONG)-1; - result.Printf("Cleared sos OS tid/index\n"); - } + } else if (arguments[1] == nullptr) { result.Printf("Need thread index parameter that maps to the OS tid. setsostid \n"); } else { - ULONG tid = strtoul(arguments[0], nullptr, 16); - g_currentThreadSystemId = tid; - - ULONG index = strtoul(arguments[1], nullptr, 16); - g_currentThreadIndex = index; - - result.Printf("Mapped sos OS tid 0x%x to lldb thread index %d\n", tid, index); + ULONG tid = 0; + if (strcmp(arguments[0], "-c") != 0 && strcmp(arguments[0], "--clear") != 0) + { + tid = strtoul(arguments[0], nullptr, 16); + } + ULONG index = strtoul(arguments[1], nullptr, 10); + if (index <= 0) + { + result.Printf("Invalid thread index parameter\n"); + } + else + { + g_services->AddThreadInfoEntry(tid, index); + if (tid == 0) + { + result.Printf("Cleared lldb thread index %d\n", index); + } + else { + result.Printf("Mapped SOS OS tid 0x%x to lldb thread index %d\n", tid, index); + } + } } return result.Succeeded(); } diff --git a/src/SOS/lldbplugin/sosplugin.h b/src/SOS/lldbplugin/sosplugin.h index d3a03e3a2..84b4a006b 100644 --- a/src/SOS/lldbplugin/sosplugin.h +++ b/src/SOS/lldbplugin/sosplugin.h @@ -10,6 +10,7 @@ #include "lldbservices.h" #include "extensions.h" #include "dbgtargetcontext.h" +#include "specialthreadinfo.h" #include "services.h" #define SOSInitialize "SOSInitializeByHost" @@ -18,8 +19,6 @@ typedef HRESULT (*CommandFunc)(ILLDBServices* services, const char* args); typedef HRESULT (*InitializeFunc)(IUnknown* punk); extern char *g_coreclrDirectory; -extern ULONG g_currentThreadIndex; -extern ULONG g_currentThreadSystemId; extern LLDBServices* g_services; bool