From: Mike McLaughlin Date: Mon, 1 Nov 2021 20:55:10 +0000 (-0700) Subject: Enable dotnet-dump tests on OSX (#2718) X-Git-Tag: submit/tizen/20220302.040122~18^2^2~27 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=7ac9f93faed5deecb03b017f7c5c5fefb91ff1ac;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Enable dotnet-dump tests on OSX (#2718) Enable dotnet-dump tests on OSX Fix crashreport test on MacOS Add metadata mapping memory service for OSX Fix CentOS build break and MacOS module size --- diff --git a/src/Microsoft.Diagnostics.DebugServices.Implementation/Module.cs b/src/Microsoft.Diagnostics.DebugServices.Implementation/Module.cs index f75f3e1dd..ab0e2ff38 100644 --- a/src/Microsoft.Diagnostics.DebugServices.Implementation/Module.cs +++ b/src/Microsoft.Diagnostics.DebugServices.Implementation/Module.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using Microsoft.Diagnostics.Runtime; using Microsoft.Diagnostics.Runtime.Utilities; using Microsoft.FileFormats; using Microsoft.FileFormats.ELF; @@ -39,7 +38,7 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation private readonly IDisposable _onChangeEvent; private Flags _flags; private PdbFileInfo _pdbFileInfo; - private ImmutableArray _buildId; + protected ImmutableArray _buildId; private PEImage _peImage; public readonly ServiceProvider ServiceProvider; @@ -141,13 +140,13 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation } } - public ImmutableArray BuildId + public virtual ImmutableArray BuildId { get { if (_buildId.IsDefault) { - byte[] id = ModuleService.GetBuildId(ImageBase, ImageSize); + byte[] id = ModuleService.GetBuildId(ImageBase); if (id != null) { _buildId = id.ToImmutableArray(); diff --git a/src/Microsoft.Diagnostics.DebugServices.Implementation/ModuleService.cs b/src/Microsoft.Diagnostics.DebugServices.Implementation/ModuleService.cs index 2ddf823d8..1d85c0deb 100644 --- a/src/Microsoft.Diagnostics.DebugServices.Implementation/ModuleService.cs +++ b/src/Microsoft.Diagnostics.DebugServices.Implementation/ModuleService.cs @@ -518,18 +518,16 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation /// Returns the ELF module build id or the MachO module uuid /// /// module base address - /// module size /// build id or null - internal byte[] GetBuildId(ulong address, ulong size) + internal byte[] GetBuildId(ulong address) { - Debug.Assert(size > 0); - Stream stream = MemoryService.CreateMemoryStream(address, size); + Stream stream = MemoryService.CreateMemoryStream(); byte[] buildId = null; try { if (Target.OperatingSystem == OSPlatform.Linux) { - var elfFile = new ELFFile(new StreamAddressSpace(stream), 0, true); + var elfFile = new ELFFile(new StreamAddressSpace(stream), address, true); if (elfFile.IsValid()) { buildId = elfFile.BuildID; @@ -537,7 +535,7 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation } else if (Target.OperatingSystem == OSPlatform.OSX) { - var machOFile = new MachOFile(new StreamAddressSpace(stream), 0, true); + var machOFile = new MachOFile(new StreamAddressSpace(stream), address, true); if (machOFile.IsValid()) { buildId = machOFile.Uuid; @@ -555,16 +553,15 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation /// Get the version string from a Linux or MacOS image /// /// image base - /// image size /// version string or null - protected string GetVersionString(ulong address, ulong size) + protected string GetVersionString(ulong address) { - Stream stream = MemoryService.CreateMemoryStream(address, size); + Stream stream = MemoryService.CreateMemoryStream(); try { if (Target.OperatingSystem == OSPlatform.Linux) { - var elfFile = new ELFFile(new StreamAddressSpace(stream), 0, true); + var elfFile = new ELFFile(new StreamAddressSpace(stream), address, true); if (elfFile.IsValid()) { foreach (ELFProgramHeader programHeader in elfFile.Segments.Select((segment) => segment.Header)) @@ -585,7 +582,7 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation } else if (Target.OperatingSystem == OSPlatform.OSX) { - var machOFile = new MachOFile(new StreamAddressSpace(stream), 0, true); + var machOFile = new MachOFile(new StreamAddressSpace(stream), address, true); if (machOFile.IsValid()) { foreach (MachSegmentLoadCommand loadCommand in machOFile.Segments.Select((segment) => segment.LoadCommand)) @@ -594,9 +591,9 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation (loadCommand.InitProt & VmProtWrite) != 0 && loadCommand.SegName.ToString() != "__LINKEDIT") { - ulong loadAddress = loadCommand.VMAddress; + ulong loadAddress = loadCommand.VMAddress + machOFile.PreferredVMBaseAddress; long loadSize = (long)loadCommand.VMSize; - if (SearchVersionString(address + loadAddress, loadSize, out string productVersion)) + if (SearchVersionString(loadAddress, loadSize, out string productVersion)) { return productVersion; } diff --git a/src/Microsoft.Diagnostics.DebugServices.Implementation/ModuleServiceFromDataReader.cs b/src/Microsoft.Diagnostics.DebugServices.Implementation/ModuleServiceFromDataReader.cs index 939ad8f66..8480ca4e6 100644 --- a/src/Microsoft.Diagnostics.DebugServices.Implementation/ModuleServiceFromDataReader.cs +++ b/src/Microsoft.Diagnostics.DebugServices.Implementation/ModuleServiceFromDataReader.cs @@ -5,8 +5,8 @@ using Microsoft.Diagnostics.Runtime; using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Diagnostics; -using System.IO; using System.Linq; using System.Runtime.InteropServices; @@ -22,7 +22,7 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation // This is what clrmd returns for non-PE modules that don't have a timestamp private const uint InvalidTimeStamp = 0; - private static readonly Microsoft.Diagnostics.Runtime.VersionInfo EmptyVersionInfo = new (0, 0, 0, 0); + private static readonly VersionInfo EmptyVersionInfo = new (0, 0, 0, 0); private readonly ModuleServiceFromDataReader _moduleService; private readonly IExportReader _exportReader; private readonly ModuleInfo _moduleInfo; @@ -54,6 +54,20 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation public override uint? IndexTimeStamp => _moduleInfo.IndexTimeStamp == InvalidTimeStamp ? null : (uint)_moduleInfo.IndexTimeStamp; + public override ImmutableArray BuildId + { + get + { + if (_buildId.IsDefault) + { + ImmutableArray buildId = _moduleService._dataReader.GetBuildId(ImageBase); + // If the data reader can't get the build id, it returns a empty (instead of default) immutable array. + _buildId = buildId.IsEmpty ? base.BuildId : buildId; + } + return _buildId; + } + } + public override VersionData VersionData { get @@ -84,7 +98,7 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation { if (_moduleService.Target.OperatingSystem != OSPlatform.Windows && !IsPEImage) { - _versionString = _moduleService.GetVersionString(ImageBase, ImageSize); + _versionString = _moduleService.GetVersionString(ImageBase); } } return _versionString; diff --git a/src/Microsoft.Diagnostics.DebugServices.Implementation/TargetFromDataReader.cs b/src/Microsoft.Diagnostics.DebugServices.Implementation/TargetFromDataReader.cs index 493415975..3679c460a 100644 --- a/src/Microsoft.Diagnostics.DebugServices.Implementation/TargetFromDataReader.cs +++ b/src/Microsoft.Diagnostics.DebugServices.Implementation/TargetFromDataReader.cs @@ -51,9 +51,15 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation ServiceProvider.AddServiceFactory(() => new ModuleServiceFromDataReader(this, _dataReader)); ServiceProvider.AddServiceFactory(() => { IMemoryService memoryService = new MemoryServiceFromDataReader(_dataReader); - if (IsDump && Host.HostType == HostType.DotnetDump) + if (IsDump) { memoryService = new ImageMappingMemoryService(this, memoryService); + // Any dump created for a MacOS target does not have managed assemblies in the module service so + // we need to use the metadata mapping memory service to make sure the metadata is available. + if (targetOS == OSPlatform.OSX) + { + memoryService = new MetadataMappingMemoryService(this, memoryService); + } } return memoryService; }); diff --git a/src/Microsoft.Diagnostics.DebugServices/MemoryServiceExtensions.cs b/src/Microsoft.Diagnostics.DebugServices/MemoryServiceExtensions.cs index 2fba2bb1c..2674d567a 100644 --- a/src/Microsoft.Diagnostics.DebugServices/MemoryServiceExtensions.cs +++ b/src/Microsoft.Diagnostics.DebugServices/MemoryServiceExtensions.cs @@ -104,15 +104,27 @@ namespace Microsoft.Diagnostics.DebugServices return false; } + /// + /// Create a stream for all of memory. + /// + /// memory service instance + /// Stream of all of memory + public static Stream CreateMemoryStream(this IMemoryService memoryService) + { + return new TargetStream(memoryService, 0, long.MaxValue); + } + /// /// Create a stream for the address range. /// /// memory service instance /// address to read /// size of stream - /// + /// memory range Stream public static Stream CreateMemoryStream(this IMemoryService memoryService, ulong address, ulong size) { + Debug.Assert(address != 0); + Debug.Assert(size != 0); return new TargetStream(memoryService, address, size); } @@ -133,8 +145,6 @@ namespace Microsoft.Diagnostics.DebugServices public TargetStream(IMemoryService memoryService, ulong address, ulong size) : base() { - Debug.Assert(address != 0); - Debug.Assert(size != 0); _memoryService = memoryService; _address = address; Length = (long)size; diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Host/ModulesCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Host/ModulesCommand.cs index 87c39f69d..b2e1b274e 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/Host/ModulesCommand.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Host/ModulesCommand.cs @@ -3,8 +3,16 @@ // See the LICENSE file in the project root for more information. using Microsoft.Diagnostics.DebugServices; +using Microsoft.FileFormats; +using Microsoft.FileFormats.ELF; +using Microsoft.FileFormats.MachO; +using System; +using System.Diagnostics; using System.Diagnostics.Tracing; +using System.IO; using System.Linq; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -14,38 +22,107 @@ namespace Microsoft.Diagnostics.ExtensionCommands [Option(Name = "--verbose", Aliases = new string[] { "-v" }, Help = "Displays more details.")] public bool Verbose { get; set; } + [Option(Name = "--segments", Aliases = new string[] { "-s" }, Help = "Displays the module segments.")] + public bool Segment { get; set; } + + + [Option(Name = "--name", Aliases = new string[] { "-n" }, Help = "RegEx filter on module name (path not included).")] + public string ModuleName { get; set; } + public IModuleService ModuleService { get; set; } public override void Invoke() { + Regex regex = ModuleName is not null ? new Regex(ModuleName, RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant) : null; ulong totalSize = 0; + foreach (IModule module in ModuleService.EnumerateModules().OrderBy((m) => m.ModuleIndex)) { totalSize += module.ImageSize; - if (Verbose) + if (regex is null || regex.IsMatch(Path.GetFileName(module.FileName))) { - WriteLine("{0} {1}", module.ModuleIndex, module.FileName); - WriteLine(" Address: {0:X16}", module.ImageBase); - WriteLine(" ImageSize: {0:X8}", module.ImageSize); - WriteLine(" IsPEImage: {0}", module.IsPEImage); - WriteLine(" IsManaged: {0}", module.IsManaged); - WriteLine(" IsFileLayout: {0}", module.IsFileLayout?.ToString() ?? ""); - WriteLine(" IndexFileSize: {0}", module.IndexFileSize?.ToString("X8") ?? ""); - WriteLine(" IndexTimeStamp: {0}", module.IndexTimeStamp?.ToString("X8") ?? ""); - WriteLine(" Version: {0}", module.VersionData?.ToString() ?? ""); - string versionString = module.VersionString; - if (!string.IsNullOrEmpty(versionString)) { - WriteLine(" {0}", versionString); + if (Verbose) + { + WriteLine("{0} {1}", module.ModuleIndex, module.FileName); + WriteLine(" Address: {0:X16}", module.ImageBase); + WriteLine(" ImageSize: {0:X8}", module.ImageSize); + WriteLine(" IsPEImage: {0}", module.IsPEImage); + WriteLine(" IsManaged: {0}", module.IsManaged); + WriteLine(" IsFileLayout: {0}", module.IsFileLayout?.ToString() ?? ""); + WriteLine(" IndexFileSize: {0}", module.IndexFileSize?.ToString("X8") ?? ""); + WriteLine(" IndexTimeStamp: {0}", module.IndexTimeStamp?.ToString("X8") ?? ""); + WriteLine(" Version: {0}", module.VersionData?.ToString() ?? ""); + string versionString = module.VersionString; + if (!string.IsNullOrEmpty(versionString)) + { + WriteLine(" {0}", versionString); + } + WriteLine(" PdbInfo: {0}", module.PdbFileInfo?.ToString() ?? ""); + WriteLine(" BuildId: {0}", !module.BuildId.IsDefaultOrEmpty ? string.Concat(module.BuildId.Select((b) => b.ToString("x2"))) : ""); + } + else + { + WriteLine("{0:X16} {1:X8} {2}", module.ImageBase, module.ImageSize, module.FileName); + } + if (Segment) + { + DisplaySegments(module.ImageBase); } - WriteLine(" PdbInfo: {0}", module.PdbFileInfo?.ToString() ?? ""); - WriteLine(" BuildId: {0}", !module.BuildId.IsDefaultOrEmpty ? string.Concat(module.BuildId.Select((b) => b.ToString("x2"))) : ""); } - else + } + WriteLine("Total image size: {0}", totalSize); + } + + public ITarget Target { get; set; } + + public IMemoryService MemoryService { get; set; } + + void DisplaySegments(ulong address) + { + try + { + if (Target.OperatingSystem == OSPlatform.Linux) + { + Stream stream = MemoryService.CreateMemoryStream(); + var elfFile = new ELFFile(new StreamAddressSpace(stream), address, true); + if (elfFile.IsValid()) + { + foreach (ELFProgramHeader programHeader in elfFile.Segments.Select((segment) => segment.Header)) + { + uint flags = MemoryService.PointerSize == 8 ? programHeader.Flags : programHeader.Flags32; + ulong loadAddress = programHeader.VirtualAddress; + ulong loadSize = programHeader.VirtualSize; + ulong fileOffset = programHeader.FileOffset; + string type = programHeader.Type.ToString(); + WriteLine($" Segment: {loadAddress:X16} {loadSize:X16} {fileOffset:X16} {flags:x2} {type}"); + } + } + } + else if (Target.OperatingSystem == OSPlatform.OSX) { - WriteLine("{0:X16} {1:X8} {2}", module.ImageBase, module.ImageSize, module.FileName); + Stream stream = MemoryService.CreateMemoryStream(); + MachOFile machOFile = new(new StreamAddressSpace(stream), address, true); + if (machOFile.IsValid()) + { + WriteLine(" LoadAddress: {0:X16}", machOFile.LoadAddress); + WriteLine(" LoadBias: {0:X16}", machOFile.PreferredVMBaseAddress); + for (int i = 0; i < machOFile.Segments.Length; i++) + { + MachSegment segment = machOFile.Segments[i]; + ulong loadAddress = segment.LoadCommand.VMAddress; + ulong loadSize = segment.LoadCommand.VMSize; + ulong fileOffset = segment.LoadCommand.FileOffset; + uint prot = segment.LoadCommand.InitProt; + string name = segment.LoadCommand.SegName.ToString(); + WriteLine($" Segment {i}: {loadAddress:X16} {loadSize:X16} {fileOffset:X16} {prot:x2} {name}"); + } + } } } - WriteLine("Total image size: {0}", totalSize); + catch (Exception ex) when (ex is InvalidVirtualAddressException || ex is BadInputFormatException) + { + Trace.TraceError($"Exception displaying module segments: {ex}"); + } } } } diff --git a/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs b/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs index 29b04ed03..0e91804e1 100644 --- a/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs +++ b/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs @@ -99,7 +99,7 @@ namespace SOS.Extensions { if (_moduleService.Target.OperatingSystem != OSPlatform.Windows && !IsPEImage) { - _versionString = _moduleService.GetVersionString(ImageBase, ImageSize); + _versionString = _moduleService.GetVersionString(ImageBase); } } } diff --git a/src/SOS/SOS.Hosting/SymbolServiceWrapper.cs b/src/SOS/SOS.Hosting/SymbolServiceWrapper.cs index ea90410b6..9b5f62dc0 100644 --- a/src/SOS/SOS.Hosting/SymbolServiceWrapper.cs +++ b/src/SOS/SOS.Hosting/SymbolServiceWrapper.cs @@ -200,20 +200,22 @@ namespace SOS.Hosting { try { - Stream stream = MemoryService.CreateMemoryStream(address, size); KeyGenerator generator = null; if (config == RuntimeConfiguration.UnixCore) { - var elfFile = new ELFFile(new StreamAddressSpace(stream), 0, true); + Stream stream = MemoryService.CreateMemoryStream(); + var elfFile = new ELFFile(new StreamAddressSpace(stream), address, true); generator = new ELFFileKeyGenerator(Tracer.Instance, elfFile, moduleFilePath); } else if (config == RuntimeConfiguration.OSXCore) { - var machOFile = new MachOFile(new StreamAddressSpace(stream), 0, true); + Stream stream = MemoryService.CreateMemoryStream(); + var machOFile = new MachOFile(new StreamAddressSpace(stream), address, true); generator = new MachOFileKeyGenerator(Tracer.Instance, machOFile, moduleFilePath); } else if (config == RuntimeConfiguration.WindowsCore || config == RuntimeConfiguration.WindowsDesktop) { + Stream stream = MemoryService.CreateMemoryStream(address, size); var peFile = new PEFile(new StreamAddressSpace(stream), true); generator = new PEFileKeyGenerator(Tracer.Instance, peFile, moduleFilePath); } diff --git a/src/SOS/SOS.UnitTests/SOS.cs b/src/SOS/SOS.UnitTests/SOS.cs index ea11b0e71..e2883e1c9 100644 --- a/src/SOS/SOS.UnitTests/SOS.cs +++ b/src/SOS/SOS.UnitTests/SOS.cs @@ -71,8 +71,7 @@ public class SOS } // Using the dotnet-dump analyze tool if the path exists in the config file. - // TODO: dotnet-dump currently doesn't support macho core dumps that the MacOS createdump generates - if (information.TestConfiguration.DotNetDumpPath() != null && OS.Kind != OSKind.OSX) + if (information.TestConfiguration.DotNetDumpPath() != null) { // Don't test dotnet-dump on triage dumps when running on desktop CLR. if (information.TestConfiguration.IsNETCore || information.DumpType != SOSRunner.DumpType.Triage) @@ -128,7 +127,7 @@ public class SOS Assert.NotNull(parameters); Assert.NotNull(parameters.ExceptionType); Assert.NotNull(parameters.OSVersion); - Assert.Equal(parameters.SystemManufacturer, "apple"); + Assert.Equal("apple", (string)parameters.SystemManufacturer); } } catch (Exception ex) diff --git a/src/SOS/lldbplugin/services.cpp b/src/SOS/lldbplugin/services.cpp index f812c0767..fbbf40958 100644 --- a/src/SOS/lldbplugin/services.cpp +++ b/src/SOS/lldbplugin/services.cpp @@ -7,6 +7,9 @@ #include #include #include +#if defined(__APPLE__) +#include +#endif #include "sosplugin.h" #include "arrayholder.h" @@ -1342,10 +1345,20 @@ LLDBServices::GetModuleBase( ULONG64 LLDBServices::GetModuleSize( + ULONG64 baseAddress, /* const */ lldb::SBModule& module) { ULONG64 size = 0; - +#if defined(__APPLE__) + mach_header_64 header; + ULONG read; + HRESULT hr = ReadVirtual(baseAddress, &header, sizeof(mach_header_64), &read) == S_OK; + if (SUCCEEDED(hr)) + { + // Since MachO segments are not contiguous the image size is just the headers/commands + size = sizeof(mach_header_64) + header.sizeofcmds; + } +#else // Find the first section with an valid base address int numSections = module.GetNumSections(); for (int si = 0; si < numSections; si++) @@ -1356,6 +1369,7 @@ LLDBServices::GetModuleSize( size += section.GetByteSize(); } } +#endif // For core dumps lldb doesn't return the section sizes when it // doesn't have access to the actual module file, but SOS (like // the SymbolReader code) still needs a non-zero module size. @@ -1744,7 +1758,7 @@ LLDBServices::LoadNativeSymbols( path.append("/"); path.append(filename); - int moduleSize = GetModuleSize(module); + int moduleSize = GetModuleSize(moduleAddress, module); callback(&module, path.c_str(), moduleAddress, moduleSize); } @@ -1822,9 +1836,9 @@ HRESULT LLDBServices::GetModuleInfo( { return E_INVALIDARG; } + ULONG64 moduleBase = GetModuleBase(target, module); if (pBase) { - ULONG64 moduleBase = GetModuleBase(target, module); if (moduleBase == UINT64_MAX) { return E_INVALIDARG; @@ -1833,7 +1847,7 @@ HRESULT LLDBServices::GetModuleInfo( } if (pSize) { - *pSize = GetModuleSize(module); + *pSize = GetModuleSize(moduleBase, module); } if (pTimestamp) { diff --git a/src/SOS/lldbplugin/services.h b/src/SOS/lldbplugin/services.h index cabbf9fa0..03eb4d308 100644 --- a/src/SOS/lldbplugin/services.h +++ b/src/SOS/lldbplugin/services.h @@ -28,7 +28,7 @@ private: ULONG m_cacheSize; ULONG64 GetModuleBase(lldb::SBTarget& target, lldb::SBModule& module); - ULONG64 GetModuleSize(lldb::SBModule& module); + ULONG64 GetModuleSize(ULONG64 baseAddress, lldb::SBModule& module); ULONG64 GetExpression(lldb::SBFrame& frame, lldb::SBError& error, PCSTR exp); void GetContextFromFrame(lldb::SBFrame& frame, DT_CONTEXT *dtcontext); DWORD_PTR GetRegister(lldb::SBFrame& frame, const char *name);