From: Mike McLaughlin Date: Tue, 20 Aug 2019 03:49:45 +0000 (-0700) Subject: Add Windows PE reading to dotnet-dump (#435) X-Git-Tag: submit/tizen/20191015.063341~12^2^2~7 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=0c7ba4bc9b05101921f102e3eef781a01af02529;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Add Windows PE reading to dotnet-dump (#435) Add Windows PE reading to dotnet-dump Finds or downloads the native modules like coreclr.dll or managed assemblies into the virtual address space of the core dump. Emulates what the Windows debugger do for minidumps. Added "logging" command that enables/disables internal logging --- diff --git a/src/SOS/SOS.Hosting/SOSHost.cs b/src/SOS/SOS.Hosting/SOSHost.cs index 6425ec1db..e4f3f2816 100644 --- a/src/SOS/SOS.Hosting/SOSHost.cs +++ b/src/SOS/SOS.Hosting/SOSHost.cs @@ -6,12 +6,17 @@ using Microsoft.Diagnostics.DebugServices; using Microsoft.Diagnostics.Runtime; using Microsoft.Diagnostics.Runtime.Interop; using Microsoft.Diagnostics.Runtime.Utilities; +using Microsoft.SymbolStore; +using Microsoft.SymbolStore.KeyGenerators; using System; +using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; using System.Runtime.InteropServices; using System.Text; @@ -54,18 +59,19 @@ namespace SOS bool logging, bool msdl, bool symweb, + string tempDirectory, string symbolServerPath, string symbolCachePath, string windowsSymbolPath); - private delegate void DisplaySymbolStoreDelegate(); + private delegate void DisplaySymbolStoreDelegate( + SymbolReader.WriteLine writeLine); private delegate void DisableSymbolStoreDelegate(); private delegate void LoadNativeSymbolsDelegate( SymbolReader.SymbolFileCallback callback, IntPtr parameter, - string tempDirectory, string moduleFilePath, ulong address, int size, @@ -158,6 +164,7 @@ namespace SOS private readonly COMCallableIUnknown _ccw; private readonly IntPtr _interface; private IntPtr _sosLibrary = IntPtr.Zero; + private Dictionary _pathToPeReader = new Dictionary(); /// /// Enable the assembly resolver to get the right SOS.NETCore version (the one @@ -186,6 +193,7 @@ namespace SOS /// /// Create an instance of the hosting class /// + /// Service provider public SOSHost(IServiceProvider serviceProvider) { DataTarget dataTarget = serviceProvider.GetService(); @@ -251,7 +259,7 @@ namespace SOS // SOS depends on that the temp directory ends with "/". if (!string.IsNullOrEmpty(tempDirectory) && tempDirectory[tempDirectory.Length - 1] != Path.DirectorySeparatorChar) { - tempDirectory = tempDirectory + Path.DirectorySeparatorChar; + tempDirectory += Path.DirectorySeparatorChar; } int result = initializeFunc( @@ -266,6 +274,7 @@ namespace SOS { throw new InvalidOperationException($"SOS initialization FAILED 0x{result:X8}"); } + Trace.TraceInformation("SOS initialized: tempDirectory '{0}' dacFilePath '{1}' sosPath '{2}'", tempDirectory, dacFilePath, sosPath); } } @@ -455,6 +464,106 @@ namespace SOS return E_FAIL; } + internal unsafe int ReadVirtualForWindows( + IntPtr self, + ulong address, + IntPtr buffer, + uint bytesRequested, + uint* pbytesRead) + { + if (DataReader.ReadMemory(address, buffer, unchecked((int)bytesRequested), out int bytesRead)) + { + Write(pbytesRead, (uint)bytesRead); + return S_OK; + } + + // The memory read failed. Check if there is a module that contains the + // address range being read and map it into the virtual address space. + foreach (ModuleInfo module in DataReader.EnumerateModules()) + { + ulong start = module.ImageBase; + ulong end = start + module.FileSize; + if (start <= address && end > address) + { + Trace.TraceInformation("ReadVirtualForWindows: address {0:X16} size {1:X8} found module {2}", address, bytesRequested, module.FileName); + + // We found a module that contains the memory requested. Now find or download the PE image. + PEReader reader = GetPEReader(module); + if (reader != null) + { + // Read the memory from the PE image. There are a few limitions: + // 1) Fix ups are NOT applied to the sections + // 2) Memory regions that cross/contain heap memory into module image memory + int rva = (int)(address - start); + try + { + PEMemoryBlock block = reader.GetSectionData(rva); + if (block.Pointer == null) + { + Trace.TraceInformation("ReadVirtualForWindows: rva {0:X8} not in any section; reading from entire image", rva); + + // If the address isn't contained in one of the sections, assume that SOS is reader the PE headers directly. + block = reader.GetEntireImage(); + } + BlobReader blob = block.GetReader(); + byte[] data = blob.ReadBytes((int)bytesRequested); + + Marshal.Copy(data, 0, buffer, data.Length); + Write(pbytesRead, (uint)data.Length); + return S_OK; + } + catch (Exception ex) when (ex is BadImageFormatException || ex is InvalidOperationException || ex is IOException) + { + Trace.TraceError("ReadVirtualForWindows: exception {0}", ex); + } + } + break; + } + } + + return E_FAIL; + } + + private PEReader GetPEReader(ModuleInfo module) + { + if (!_pathToPeReader.TryGetValue(module.FileName, out PEReader reader)) + { + Stream stream = null; + + string downloadFilePath = module.FileName; + if (!File.Exists(downloadFilePath)) + { + if (SymbolReader.IsSymbolStoreEnabled()) + { + SymbolStoreKey key = PEFileKeyGenerator.GetKey(Path.GetFileName(downloadFilePath), module.TimeStamp, module.FileSize); + if (key != null) + { + // Now download the module from the symbol server + downloadFilePath = SymbolReader.GetSymbolFile(key); + } + } + } + + if (!string.IsNullOrEmpty(downloadFilePath)) + { + try + { + stream = File.OpenRead(downloadFilePath); + } + catch (Exception ex) when (ex is DirectoryNotFoundException || ex is FileNotFoundException || ex is UnauthorizedAccessException || ex is IOException) + { + Trace.TraceError("GetPEReader: exception {0}", ex); + } + if (stream != null) + { + reader = new PEReader(stream); + _pathToPeReader.Add(module.FileName, reader); + } + } + } + return reader; + } + internal unsafe int WriteVirtual( IntPtr self, ulong address, diff --git a/src/SOS/SOS.Hosting/dbgeng/DebugDataSpaces.cs b/src/SOS/SOS.Hosting/dbgeng/DebugDataSpaces.cs index 253d27f11..ee6c82b11 100644 --- a/src/SOS/SOS.Hosting/dbgeng/DebugDataSpaces.cs +++ b/src/SOS/SOS.Hosting/dbgeng/DebugDataSpaces.cs @@ -24,7 +24,7 @@ namespace SOS private static void AddDebugDataSpaces(VTableBuilder builder, SOSHost soshost) { - builder.AddMethod(new ReadVirtualDelegate(soshost.ReadVirtual)); + builder.AddMethod(new ReadVirtualDelegate(soshost.ReadVirtualForWindows)); builder.AddMethod(new WriteVirtualDelegate(soshost.WriteVirtual)); builder.AddMethod(new SearchVirtualDelegate((self, offset, length, pattern, patternSize, patternGranularity, matchOffset) => DebugClient.NotImplemented)); builder.AddMethod(new ReadVirtualUncachedDelegate((self, offset, buffer, bufferSize, bytesRead) => DebugClient.NotImplemented)); diff --git a/src/SOS/SOS.NETCore/SymbolReader.cs b/src/SOS/SOS.NETCore/SymbolReader.cs index 28d24c627..ed42d7776 100644 --- a/src/SOS/SOS.NETCore/SymbolReader.cs +++ b/src/SOS/SOS.NETCore/SymbolReader.cs @@ -159,25 +159,37 @@ namespace SOS /// symbol file name and path public delegate void SymbolFileCallback(IntPtr parameter, [MarshalAs(UnmanagedType.LPStr)] string moduleFileName, [MarshalAs(UnmanagedType.LPStr)] string symbolFileName); + /// + /// Temporary directory for dac/symbols + /// + public static string TempDirectory { get; private set; } + + static readonly ITracer s_tracer = new Tracer(); static SymbolStore s_symbolStore = null; static bool s_symbolCacheAdded = false; - static ITracer s_tracer = null; /// /// Initializes symbol loading. Adds the symbol server and/or the cache path (if not null) to the list of /// symbol servers. This API can be called more than once to add more servers to search. /// - /// if true, logging diagnostics to console + /// if true, enable logging diagnostics to console /// if true, use the public microsoft server /// if true, use symweb internal server and protocol (file.ptr) + /// temp directory unique to this instance of SOS /// symbol server url (optional) /// symbol cache directory path (optional) /// windows symbol path (optional) - /// - public static bool InitializeSymbolStore(bool logging, bool msdl, bool symweb, string symbolServerPath, string symbolCachePath, string windowsSymbolPath) + /// if false, failure + public static bool InitializeSymbolStore(bool logging, bool msdl, bool symweb, string tempDirectory, string symbolServerPath, string symbolCachePath, string windowsSymbolPath) { - if (s_tracer == null) { - s_tracer = new Tracer(enabled: logging, enabledVerbose: logging, Console.WriteLine); + if (logging) { + // Uses the standard console to do the logging instead of sending it to the hosting debugger console + // because windbg/cdb can only output on the client thread without dead locking. Microsoft.SymbolStore + // can log on any thread. + Trace.Listeners.Add(new TextWriterTraceListener(Console.OpenStandardOutput())); + } + if (TempDirectory == null) { + TempDirectory = tempDirectory; } SymbolStore store = s_symbolStore; @@ -202,24 +214,21 @@ namespace SOS /// /// Displays the symbol server and cache configuration /// - public static void DisplaySymbolStore() + public static void DisplaySymbolStore(WriteLine writeLine) { - if (s_tracer != null) + SymbolStore symbolStore = s_symbolStore; + while (symbolStore != null) { - SymbolStore symbolStore = s_symbolStore; - while (symbolStore != null) - { - if (symbolStore is CacheSymbolStore cache) { - s_tracer.WriteLine("Cache: {0}", cache.CacheDirectory); - } - else if (symbolStore is HttpSymbolStore http) { - s_tracer.WriteLine("Server: {0}", http.Uri); - } - else { - s_tracer.WriteLine("Unknown symbol store"); - } - symbolStore = symbolStore.BackingStore; + if (symbolStore is CacheSymbolStore cache) { + writeLine($"Cache: {cache.CacheDirectory}"); } + else if (symbolStore is HttpSymbolStore http) { + writeLine($"Server: {http.Uri}"); + } + else { + writeLine("Unknown symbol store"); + } + symbolStore = symbolStore.BackingStore; } } @@ -228,7 +237,6 @@ namespace SOS /// public static void DisableSymbolStore() { - s_tracer = null; s_symbolStore = null; s_symbolCacheAdded = false; } @@ -238,16 +246,14 @@ namespace SOS /// /// called back for each symbol file loaded /// callback parameter - /// temp directory unique to this instance of SOS /// module path /// module base address /// module size /// read memory callback delegate - public static void LoadNativeSymbols(SymbolFileCallback callback, IntPtr parameter, string tempDirectory, string moduleFilePath, ulong address, int size, ReadMemoryDelegate readMemory) + public static void LoadNativeSymbols(SymbolFileCallback callback, IntPtr parameter, string moduleFilePath, ulong address, int size, ReadMemoryDelegate readMemory) { if (IsSymbolStoreEnabled()) { - Debug.Assert(s_tracer != null); Stream stream = new TargetStream(address, size, readMemory); KeyTypeFlags flags = KeyTypeFlags.SymbolKey | KeyTypeFlags.ClrKeys; KeyGenerator generator = null; @@ -282,7 +288,7 @@ namespace SOS // Don't download the sos binaries that come with the runtime if (moduleFileName != "SOS.NETCore.dll" && !moduleFileName.StartsWith("libsos.")) { - string downloadFilePath = GetSymbolFile(key, tempDirectory); + string downloadFilePath = GetSymbolFile(key); if (downloadFilePath != null) { s_tracer.Information("{0}: {1}", moduleFileName, downloadFilePath); @@ -302,10 +308,8 @@ namespace SOS /// Download a symbol from the symbol stores/server. /// /// index of the file to download - /// temp directory to put the file. This directory is only created if - /// the file is NOT already in the cache and is downloaded. /// Path to the downloaded file either in the cache or in the temp directory - public static string GetSymbolFile(SymbolStoreKey key, string tempDirectory) + public static string GetSymbolFile(SymbolStoreKey key) { string downloadFilePath = null; @@ -322,8 +326,13 @@ namespace SOS // If the downloaded doesn't already exists on disk in the cache, then write it to a temporary location. if (!File.Exists(downloadFilePath)) { - Directory.CreateDirectory(tempDirectory); - downloadFilePath = Path.Combine(tempDirectory, Path.GetFileName(key.FullPathName)); + if (TempDirectory == null) + { + int processId = Process.GetCurrentProcess().Id; + TempDirectory = Path.Combine(Path.GetTempPath(), "sos" + processId.ToString()); + Directory.CreateDirectory(TempDirectory); + } + downloadFilePath = Path.Combine(TempDirectory, Path.GetFileName(key.FullPathName)); using (Stream destinationStream = File.OpenWrite(downloadFilePath)) { file.Stream.CopyTo(destinationStream); diff --git a/src/SOS/SOS.NETCore/Tracer.cs b/src/SOS/SOS.NETCore/Tracer.cs index ac046d0ed..b7a1acb68 100644 --- a/src/SOS/SOS.NETCore/Tracer.cs +++ b/src/SOS/SOS.NETCore/Tracer.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Diagnostics; namespace SOS { @@ -11,22 +12,14 @@ namespace SOS /// internal sealed class Tracer : Microsoft.SymbolStore.ITracer { - readonly bool m_enabled; - readonly bool m_enabledVerbose; - readonly SymbolReader.WriteLine m_output; - - public Tracer(bool enabled, bool enabledVerbose, SymbolReader.WriteLine output) + public Tracer() { - m_enabled = enabled; - m_enabledVerbose = enabledVerbose; - m_output = output; } public void WriteLine(string message) { - lock (this) { - m_output?.Invoke(message); - } + Trace.WriteLine(message); + Trace.Flush(); } public void WriteLine(string format, params object[] arguments) @@ -36,66 +29,48 @@ namespace SOS public void Information(string message) { - if (m_enabled) - { - WriteLine(message); - } + Trace.TraceInformation(message); + Trace.Flush(); } public void Information(string format, params object[] arguments) { - if (m_enabled) - { - WriteLine(format, arguments); - } + Trace.TraceInformation(format, arguments); + Trace.Flush(); } public void Warning(string message) { - if (m_enabled) - { - WriteLine("WARNING: " + message); - } + Trace.TraceWarning(message); + Trace.Flush(); } public void Warning(string format, params object[] arguments) { - if (m_enabled) - { - WriteLine("WARNING: " + format, arguments); - } + Trace.TraceWarning(format, arguments); + Trace.Flush(); } public void Error(string message) { - if (m_enabled) - { - WriteLine("ERROR: " + message); - } + Trace.TraceError(message); + Trace.Flush(); } public void Error(string format, params object[] arguments) { - if (m_enabled) - { - WriteLine("ERROR: " + format, arguments); - } + Trace.TraceError(format, arguments); + Trace.Flush(); } public void Verbose(string message) { - if (m_enabledVerbose) - { - WriteLine(message); - } + Information(message); } public void Verbose(string format, params object[] arguments) { - if (m_enabledVerbose) - { - WriteLine(format, arguments); - } + Information(format, arguments); } } } diff --git a/src/SOS/Strike/hostcoreclr.cpp b/src/SOS/Strike/hostcoreclr.cpp index 05c90b4d9..670617dca 100644 --- a/src/SOS/Strike/hostcoreclr.cpp +++ b/src/SOS/Strike/hostcoreclr.cpp @@ -783,7 +783,7 @@ HRESULT InitializeSymbolStore(BOOL logging, BOOL msdl, BOOL symweb, const char* IfFailRet(InitializeHosting()); _ASSERTE(g_SOSNetCoreCallbacks.InitializeSymbolStoreDelegate != nullptr); - if (!g_SOSNetCoreCallbacks.InitializeSymbolStoreDelegate(logging, msdl, symweb, symbolServer, cacheDirectory, nullptr)) + if (!g_SOSNetCoreCallbacks.InitializeSymbolStoreDelegate(logging, msdl, symweb, GetTempDirectory(), symbolServer, cacheDirectory, nullptr)) { ExtErr("Error initializing symbol server support\n"); return E_FAIL; @@ -809,7 +809,7 @@ void InitializeSymbolStore() { if (strlen(symbolPath) > 0) { - if (!g_SOSNetCoreCallbacks.InitializeSymbolStoreDelegate(false, false, false, nullptr, nullptr, symbolPath)) + if (!g_SOSNetCoreCallbacks.InitializeSymbolStoreDelegate(false, false, false, GetTempDirectory(), nullptr, nullptr, symbolPath)) { ExtErr("Windows symbol path parsing FAILED\n"); } @@ -856,7 +856,7 @@ static void LoadNativeSymbolsCallback(void* param, const char* moduleFilePath, U { _ASSERTE(g_hostingInitialized); _ASSERTE(g_SOSNetCoreCallbacks.LoadNativeSymbolsDelegate != nullptr); - g_SOSNetCoreCallbacks.LoadNativeSymbolsDelegate(SymbolFileCallback, param, GetTempDirectory(), moduleFilePath, moduleAddress, moduleSize, ReadMemoryForSymbols); + g_SOSNetCoreCallbacks.LoadNativeSymbolsDelegate(SymbolFileCallback, param, moduleFilePath, moduleAddress, moduleSize, ReadMemoryForSymbols); } /**********************************************************************\ @@ -902,6 +902,7 @@ HRESULT LoadNativeSymbols(bool runtimeOnly) return hr; } + /**********************************************************************\ * Displays the symbol server and cache status. \**********************************************************************/ @@ -910,7 +911,10 @@ void DisplaySymbolStore() if (g_symbolStoreInitialized) { _ASSERTE(g_SOSNetCoreCallbacks.DisplaySymbolStoreDelegate != nullptr); - g_SOSNetCoreCallbacks.DisplaySymbolStoreDelegate(); + g_SOSNetCoreCallbacks.DisplaySymbolStoreDelegate([] (const char* message) { + ExtOut(message); + ExtOut("\n"); + }); } } diff --git a/src/SOS/Strike/hostcoreclr.h b/src/SOS/Strike/hostcoreclr.h index 3e92486b4..a5fe423cb 100644 --- a/src/SOS/Strike/hostcoreclr.h +++ b/src/SOS/Strike/hostcoreclr.h @@ -12,14 +12,14 @@ struct SymbolModuleInfo; -typedef void (*OutputDelegate)(const char*); +typedef void (*WriteLineDelegate)(const char*); typedef int (*ReadMemoryDelegate)(ULONG64, uint8_t*, int); typedef void (*SymbolFileCallbackDelegate)(void*, const char* moduleFileName, const char* symbolFilePath); -typedef BOOL (*InitializeSymbolStoreDelegate)(BOOL, BOOL, BOOL, const char*, const char*, const char*); -typedef void (*DisplaySymbolStoreDelegate)(); +typedef BOOL (*InitializeSymbolStoreDelegate)(BOOL, BOOL, BOOL, const char*, const char*, const char*, const char*); +typedef void (*DisplaySymbolStoreDelegate)(WriteLineDelegate); typedef void (*DisableSymbolStoreDelegate)(); -typedef void (*LoadNativeSymbolsDelegate)(SymbolFileCallbackDelegate, void*, const char*, const char*, ULONG64, int, ReadMemoryDelegate); +typedef void (*LoadNativeSymbolsDelegate)(SymbolFileCallbackDelegate, void*, const char*, ULONG64, int, ReadMemoryDelegate); typedef PVOID (*LoadSymbolsForModuleDelegate)(const char*, BOOL, ULONG64, int, ULONG64, int, ReadMemoryDelegate); typedef void (*DisposeDelegate)(PVOID); typedef BOOL (*ResolveSequencePointDelegate)(PVOID, const char*, unsigned int, unsigned int*, unsigned int*); diff --git a/src/Tools/dotnet-dump/Analyzer.cs b/src/Tools/dotnet-dump/Analyzer.cs index 839f734cc..d462f0269 100644 --- a/src/Tools/dotnet-dump/Analyzer.cs +++ b/src/Tools/dotnet-dump/Analyzer.cs @@ -28,8 +28,6 @@ namespace Microsoft.Diagnostics.Tools.Dump private readonly CommandProcessor _commandProcessor; private string _dacFilePath; - private static string s_tempDirectory; - /// /// Enable the assembly resolver to get the right SOS.NETCore version (the one /// in the same directory as this assembly). @@ -73,7 +71,7 @@ namespace Microsoft.Diagnostics.Tools.Dump AddServices(target); // Automatically enable symbol server support - SymbolReader.InitializeSymbolStore(logging: false, msdl: true, symweb: false, symbolServerPath: null, symbolCachePath: null, windowsSymbolPath: null); + SymbolReader.InitializeSymbolStore(logging: false, msdl: true, symweb: false, tempDirectory: null, symbolServerPath: null, symbolCachePath: null, windowsSymbolPath: null); // Run the commands from the dotnet-dump command line if (command != null) @@ -132,7 +130,7 @@ namespace Microsoft.Diagnostics.Tools.Dump _serviceProvider.AddServiceFactory(typeof(SOSHost), () => { var sosHost = new SOSHost(_serviceProvider); - sosHost.InitializeSOSHost(s_tempDirectory, _dacFilePath, dbiFilePath: null); + sosHost.InitializeSOSHost(SymbolReader.TempDirectory, _dacFilePath, dbiFilePath: null); return sosHost; }); } @@ -199,13 +197,8 @@ namespace Microsoft.Diagnostics.Tools.Dump if (key != null) { - if (s_tempDirectory == null) - { - int processId = Process.GetCurrentProcess().Id; - s_tempDirectory = Path.Combine(Path.GetTempPath(), "analyze" + processId.ToString()); - } // Now download the DAC module from the symbol server - _dacFilePath = SymbolReader.GetSymbolFile(key, s_tempDirectory); + _dacFilePath = SymbolReader.GetSymbolFile(key); } } } diff --git a/src/Tools/dotnet-dump/Commands/LoggingCommand.cs b/src/Tools/dotnet-dump/Commands/LoggingCommand.cs new file mode 100644 index 000000000..d505423ee --- /dev/null +++ b/src/Tools/dotnet-dump/Commands/LoggingCommand.cs @@ -0,0 +1,58 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.Repl; +using System; +using System.CommandLine; +using System.Diagnostics; + +namespace Microsoft.Diagnostics.Tools.Dump +{ + [Command(Name = "logging", Help = "Enable/disable internal logging")] + public class LoggingCommand : CommandBase + { + [Option(Name = "enable", Help = "Enable internal loggging.")] + public bool Enable { get; set; } + + [Option(Name = "disable", Help = "Disable internal loggging.")] + public bool Disable { get; set; } + + private const string ListenerName = "Analyze.LoggingListener"; + + public override void Invoke() + { + if (Enable) { + if (Trace.Listeners[ListenerName] == null) { + Trace.Listeners.Add(new LoggingListener(Console)); + } + } + else if (Disable) { + Trace.Listeners.Remove(ListenerName); + } + WriteLine("Logging is {0}", Trace.Listeners[ListenerName] != null ? "enabled" : "disabled"); + } + + class LoggingListener : TraceListener + { + private readonly IConsoleService _console; + + internal LoggingListener(IConsoleService console) + : base(ListenerName) + { + _console = console; + } + + public override void Write(string message) + { + _console.Write(message); + } + + public override void WriteLine(string message) + { + _console.Write(message + Environment.NewLine); + } + } + } +}