From: Lee Culver Date: Tue, 14 Feb 2023 17:42:26 +0000 (-0800) Subject: Add native memory analysis helper commands (#3647) X-Git-Tag: accepted/tizen/unified/riscv/20231226.055542~43^2~2^2~99 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=22e135e59ace69d48e2787359d40a70f0c374e47;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Add native memory analysis helper commands (#3647) * Define and implement IMemoryRegionService * Add !maddress Added !maddress and the helpers used to implement it. * Add MAddressCommand help and helpers * Add !gctonative * Add argument checks * Fix Width * Remove empty line * Add !findpointersin * Initial code review feedback * Use CommandBase instead of ExtensionCommandBase * Split MAddressCommand into two clases - One class for the command. - One class for shared components. * Fix help invoke * Command fixes - Added MAddressCommand's missing property. - Added sealed to a few classes * Fix !findpointersin perf * Add IModuleSymbol.GetSymbolStatus - Remove reference to SOSHosting from ExtensionCommands. - No longer provide IDebug* services. * Don't use IDataReader Code review feedback, use built in services. * Use DebuggerServices.DebugClient Avoid double COM startup. * Add missing file from last commit DebuggerServices was not saved. * Use ServiceExport/Import for NativeAddressHelper --- diff --git a/src/Microsoft.Diagnostics.DebugServices/IMemoryRegion.cs b/src/Microsoft.Diagnostics.DebugServices/IMemoryRegion.cs new file mode 100644 index 000000000..bfb0b2147 --- /dev/null +++ b/src/Microsoft.Diagnostics.DebugServices/IMemoryRegion.cs @@ -0,0 +1,98 @@ +namespace Microsoft.Diagnostics.DebugServices +{ + public enum MemoryRegionType + { + MEM_UNKNOWN = 0, + MEM_IMAGE = 0x1000000, + MEM_MAPPED = 0x40000, + MEM_PRIVATE = 0x20000 + } + + public enum MemoryRegionState + { + MEM_UNKNOWN = 0, + MEM_COMMIT = 0x1000, + MEM_FREE = 0x10000, + MEM_RESERVE = 0x2000 + } + + public enum MemoryRegionProtection + { + PAGE_UNKNOWN = 0, + PAGE_EXECUTE = 0x00000010, + PAGE_EXECUTE_READ = 0x00000020, + PAGE_EXECUTE_READWRITE = 0x00000040, + PAGE_EXECUTE_WRITECOPY = 0x00000080, + PAGE_NOACCESS = 0x00000001, + PAGE_READONLY = 0x00000002, + PAGE_READWRITE = 0x00000004, + PAGE_WRITECOPY = 0x00000008, + PAGE_GUARD = 0x00000100, + PAGE_NOCACHE = 0x00000200, + PAGE_WRITECOMBINE = 0x00000400 + } + + public enum MemoryRegionUsage + { + Unknown, + Free, + Image, + Peb, + Teb, + Stack, + Heap, + PageHeap, + FileMapping, + CLR, + Other, + } + + /// + /// Represents a single virtual address region in the target process. + /// + public interface IMemoryRegion + { + /// + /// The start address of the region. + /// + ulong Start { get; } + + /// + /// The end address of the region. + /// + ulong End { get; } + + /// + /// The size of the region. + /// + ulong Size { get; } + + /// + /// The type of the region. (Image/Private/Mapped) + /// + MemoryRegionType Type { get; } + + /// + /// The state of the region. (Commit/Free/Reserve) + /// + MemoryRegionState State { get; } + + /// + /// The protection of the region. + /// + MemoryRegionProtection Protection { get; } + + /// + /// What this memory is being used for. + /// This field is a best attempt at determining what the memory is being used for, + /// and may be marked as Unknown if certain debugging symbols are not available. + /// + MemoryRegionUsage Usage { get; } + + /// + /// If this file is an image or mapped file, this property may be non-null and + /// contain its path. + /// + public string Image { get; } + } +} diff --git a/src/Microsoft.Diagnostics.DebugServices/IMemoryRegionService.cs b/src/Microsoft.Diagnostics.DebugServices/IMemoryRegionService.cs new file mode 100644 index 000000000..37457612b --- /dev/null +++ b/src/Microsoft.Diagnostics.DebugServices/IMemoryRegionService.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; + +namespace Microsoft.Diagnostics.DebugServices +{ + /// + /// Enumerate virtual address regions, their protections, and usage. + /// + public interface IMemoryRegionService + { + IEnumerable EnumerateRegions(); + } +} diff --git a/src/Microsoft.Diagnostics.DebugServices/IModuleSymbols.cs b/src/Microsoft.Diagnostics.DebugServices/IModuleSymbols.cs index 1cde6e6a5..8f184621f 100644 --- a/src/Microsoft.Diagnostics.DebugServices/IModuleSymbols.cs +++ b/src/Microsoft.Diagnostics.DebugServices/IModuleSymbols.cs @@ -5,6 +5,34 @@ namespace Microsoft.Diagnostics.DebugServices { + /// + /// The status of the symbols for a module. + /// + public enum SymbolStatus + { + /// + /// The status of the symbols is unknown. The symbol may be + /// loaded or unloaded. + /// + Unknown, + + /// + /// The debugger has successfully loaded symbols for this module. + /// + Loaded, + + /// + /// The debugger does not have symbols loaded for this module. + /// + NotLoaded, + + /// + /// The debugger does not have symbols loaded for this module, but + /// it is able to report addresses of exported functions. + /// + ExportOnly, + } + /// /// Module symbol lookup /// @@ -34,5 +62,12 @@ namespace Microsoft.Diagnostics.DebugServices /// returned type if found /// true if type found bool TryGetType(string typeName, out IType type); + + /// + /// Returns the status of the symbols for this module. This function may cause + /// the debugger to load symbols for this module, which may take a long time. + /// + /// The status of symbols for this module. + SymbolStatus GetSymbolStatus(); } } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/ClrMemoryPointer.cs b/src/Microsoft.Diagnostics.ExtensionCommands/ClrMemoryPointer.cs new file mode 100644 index 000000000..8cad05657 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/ClrMemoryPointer.cs @@ -0,0 +1,117 @@ +using Microsoft.Diagnostics.Runtime; +using Microsoft.Diagnostics.Runtime.DacInterface; +using System; +using System.Collections.Generic; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + internal sealed class ClrMemoryPointer + { + public ulong Address { get; } + + public ClrMemoryKind Kind { get; } + + public ClrMemoryPointer(ulong address, ClrMemoryKind kind) + { + Address = address; + Kind = kind; + } + + /// + /// Enumerates pointers to various CLR heaps in memory. + /// + public static IEnumerable EnumerateClrMemoryAddresses(ClrRuntime runtime) + { + SOSDac sos = runtime.DacLibrary.SOSDacInterface; + foreach (JitManagerInfo jitMgr in sos.GetJitManagers()) + { + foreach (var handle in runtime.EnumerateHandles()) + yield return new ClrMemoryPointer(handle.Address, ClrMemoryKind.HandleTable); + + foreach (var mem in sos.GetCodeHeapList(jitMgr.Address)) + yield return new ClrMemoryPointer(mem.Address, mem.Type switch + { + CodeHeapType.Loader => ClrMemoryKind.LoaderHeap, + CodeHeapType.Host => ClrMemoryKind.Host, + _ => ClrMemoryKind.UnknownCodeHeap + }); + + Console.WriteLine("GC Segments:"); + foreach (var seg in runtime.Heap.Segments) + { + if (seg.CommittedMemory.Length > 0) + yield return new ClrMemoryPointer(seg.CommittedMemory.Start, ClrMemoryKind.GCHeapSegment); + + if (seg.ReservedMemory.Length > 0) + yield return new ClrMemoryPointer(seg.ReservedMemory.Start, ClrMemoryKind.GCHeapReserve); + } + + HashSet seen = new(); + + List heaps = new(); + if (runtime.SystemDomain is not null) + AddAppDomainHeaps(sos, runtime.SystemDomain.Address, heaps); + + if (runtime.SharedDomain is not null) + AddAppDomainHeaps(sos, runtime.SharedDomain.Address, heaps); + + foreach (var heap in heaps) + if (seen.Add(heap.Address)) + yield return heap; + + foreach (ClrDataAddress address in sos.GetAppDomainList()) + { + heaps.Clear(); + AddAppDomainHeaps(sos, address, heaps); + + foreach (var heap in heaps) + if (seen.Add(heap.Address)) + yield return heap; + } + } + } + + private enum VCSHeapType + { + IndcellHeap, + LookupHeap, + ResolveHeap, + DispatchHeap, + CacheEntryHeap + } + + private static void AddAppDomainHeaps(SOSDac sos, ClrDataAddress address, List heaps) + { + if (sos.GetAppDomainData(address, out AppDomainData domain)) + { + sos.TraverseLoaderHeap(domain.StubHeap, (address, size, isCurrent) => heaps.Add(new ClrMemoryPointer(address, ClrMemoryKind.StubHeap))); + sos.TraverseLoaderHeap(domain.HighFrequencyHeap, (address, size, isCurrent) => heaps.Add(new ClrMemoryPointer(address, ClrMemoryKind.HighFrequencyHeap))); + sos.TraverseLoaderHeap(domain.LowFrequencyHeap, (address, size, isCurrent) => heaps.Add(new ClrMemoryPointer(address, ClrMemoryKind.LowFrequencyHeap))); + sos.TraverseStubHeap(address, (int)VCSHeapType.IndcellHeap, (address, size, isCurrent) => heaps.Add(new ClrMemoryPointer(address, ClrMemoryKind.IndcellHeap))); + sos.TraverseStubHeap(address, (int)VCSHeapType.LookupHeap, (address, size, isCurrent) => heaps.Add(new ClrMemoryPointer(address, ClrMemoryKind.LookupHeap))); + sos.TraverseStubHeap(address, (int)VCSHeapType.ResolveHeap, (address, size, isCurrent) => heaps.Add(new ClrMemoryPointer(address, ClrMemoryKind.ResolveHeap))); + sos.TraverseStubHeap(address, (int)VCSHeapType.DispatchHeap, (address, size, isCurrent) => heaps.Add(new ClrMemoryPointer(address, ClrMemoryKind.DispatchHeap))); + sos.TraverseStubHeap(address, (int)VCSHeapType.CacheEntryHeap, (address, size, isCurrent) => heaps.Add(new ClrMemoryPointer(address, ClrMemoryKind.CacheEntryHeap))); + } + } + } + + internal enum ClrMemoryKind + { + None, + LoaderHeap, + Host, + UnknownCodeHeap, + GCHeapSegment, + GCHeapReserve, + StubHeap, + HighFrequencyHeap, + LowFrequencyHeap, + IndcellHeap, + LookupHeap, + ResolveHeap, + DispatchHeap, + CacheEntryHeap, + HandleTable, + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs b/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs new file mode 100644 index 000000000..ec091214b --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + internal static class ExtensionMethodHelpers + { + public static string ConvertToHumanReadable(this ulong totalBytes) => ConvertToHumanReadable((double)totalBytes); + public static string ConvertToHumanReadable(this long totalBytes) => ConvertToHumanReadable((double)totalBytes); + + public static string ConvertToHumanReadable(this double totalBytes) + { + double updated = totalBytes; + + updated /= 1024; + if (updated < 1024) + return $"{updated:0.00}kb"; + + updated /= 1024; + if (updated < 1024) + return $"{updated:0.00}mb"; + + updated /= 1024; + return $"{updated:0.00}gb"; + } + + internal static ulong FindMostCommonPointer(this IEnumerable enumerable) + => (from ptr in enumerable + group ptr by ptr into g + orderby g.Count() descending + select g.First()).First(); + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/FindPointersInCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/FindPointersInCommand.cs new file mode 100644 index 000000000..287114ae6 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/FindPointersInCommand.cs @@ -0,0 +1,497 @@ +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.Runtime; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using static Microsoft.Diagnostics.ExtensionCommands.NativeAddressHelper; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "findpointersin", Help="Finds pointers to the GC heap within the given memory regions.")] + public sealed class FindPointersInCommand : CommandBase + { + [ServiceImport] + public IModuleService ModuleService { get; set; } + + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + [ServiceImport] + public NativeAddressHelper AddressHelper { get; set; } + + [Option(Name = "--showAllObjects", Aliases = new string[] { "-a", "--all" }, Help = "Show all objects instead of only objects Pinned on the heap.")] + public bool ShowAllObjects { get; set; } + + [Argument(Help = "The types of memory to search for pointers to the GC heap.")] + public string[] Regions { get; set; } + + private const string VtableConst = "vtable for "; + + private int Width + { + get + { + int width = Console.WindowWidth; + if (width == 0) + width = 120; + if (width > 256) + width = 256; + + return width; + } + } + + public override void Invoke() + { + if (Regions is null || Regions.Length == 0) + throw new DiagnosticsException("Must specify at least one memory region type to search for."); + + PrintPointers(!ShowAllObjects, Regions); + } + + private void PrintPointers(bool pinnedOnly, params string[] memTypes) + { + DescribedRegion[] allRegions = AddressHelper.EnumerateAddressSpace(tagClrMemoryRanges: true, includeReserveMemory: false, tagReserveMemoryHeuristically: false).ToArray(); + + WriteLine("Scanning for pinned objects..."); + var ctx = CreateMemoryWalkContext(); + + foreach (string type in memTypes) + { + DescribedRegion[] matchingRanges = allRegions.Where(r => r.Name == type).ToArray(); + if (matchingRanges.Length == 0) + { + WriteLine($"Found not memory regions matching '{type}'."); + continue; + } + + RegionPointers totals = new(); + + foreach (DescribedRegion mem in matchingRanges.OrderBy(r => r.Start)) + { + var pointersFound = AddressHelper.EnumerateRegionPointers(mem.Start, mem.End, allRegions).Select(r => (r.Pointer, r.MemoryRange)); + RegionPointers result = ProcessOneRegion(pinnedOnly, pointersFound, ctx); + + WriteMemoryHeaderLine(mem); + + WriteLine($"Type: {mem.Name}"); + if (mem.Image is not null) + WriteLine($"Image: {mem.Image}"); + + WriteTables(ctx, result, false); + + WriteLine(""); + WriteLine("END REGION".PadLeft((Width - 10) / 2, '=').PadRight(Width, '=')); + WriteLine(""); + + result.AddTo(totals); + } + + if (matchingRanges.Length > 1) + { + WriteLine(" TOTALS ".PadLeft(Width / 2).PadRight(Width)); + + WriteTables(ctx, totals, truncate: true); + + WriteLine(new string('=', Width)); + } + } + } + + private void WriteTables(MemoryWalkContext ctx, RegionPointers result, bool truncate) + { + if (result.PointersToGC > 0) + { + WriteLine(""); + WriteLine("Pointers to GC heap:"); + + PrintGCPointerTable(result); + } + + if (result.ResolvablePointers.Count > 0) + { + WriteLine(""); + WriteLine("Pointers to images with symbols:"); + + WriteResolvablePointerTable(ctx, result, truncate); + } + + if (result.UnresolvablePointers.Count > 0) + { + WriteLine(""); + WriteLine("Other pointers:"); + + WriteUnresolvablePointerTable(result, truncate); + } + } + + private void WriteMemoryHeaderLine(DescribedRegion mem) + { + string header = $"REGION [{mem.Start:x}-{mem.End:x}] {mem.Type} {mem.State} {mem.Protection}"; + int lpad = (Width - header.Length) / 2; + if (lpad > 0) + header = header.PadLeft(Width - lpad, '='); + WriteLine(header.PadRight(Width, '=')); + } + + private void PrintGCPointerTable(RegionPointers result) + { + if (result.PinnedPointers.Count == 0) + { + WriteLine($"{result.PointersToGC:n0} pointers to the GC heap, but none pointed to a pinned object."); + } + else + { + var gcResult = from obj in result.PinnedPointers + let name = obj.Type?.Name ?? "" + group obj.Address by name into g + let Count = g.Count() + orderby Count descending + select + ( + g.Key, + Count, + new HashSet(g).Count, + g.AsEnumerable() + ); + + if (result.NonPinnedGCPointers.Count > 0) + { + var v = new (string, int, int, IEnumerable)[] { ("[Pointers to non-pinned objects]", result.NonPinnedGCPointers.Count, new HashSet(result.NonPinnedGCPointers).Count, result.NonPinnedGCPointers) }; + gcResult = v.Concat(gcResult); + } + + PrintPointerTable("Type", "[Other Pinned Object Pointers]", forceTruncate: false, gcResult); + } + } + + private void WriteUnresolvablePointerTable(RegionPointers result, bool forceTruncate) + { + var unresolvedQuery = from item in result.UnresolvablePointers + let Name = item.Key.Image ?? item.Key.Name + group item.Value by Name into g + let All = g.SelectMany(r => r).ToArray() + let Count = All.Length + orderby Count descending + select + ( + g.Key, + Count, + new HashSet(All).Count, + All.AsEnumerable() + ); + + + PrintPointerTable("Region", "[Unique Pointers to Unique Regions]", forceTruncate, unresolvedQuery); + } + + private void WriteResolvablePointerTable(MemoryWalkContext ctx, RegionPointers result, bool forceTruncate) + { + var resolvedQuery = from ptr in result.ResolvablePointers.SelectMany(r => r.Value) + let r = ctx.ResolveSymbol(ModuleService, ptr) + let name = r.Symbol ?? "" + group (ptr, r.Offset) by name into g + let Count = g.Count() + let UniqueOffsets = new HashSet(g.Select(g => g.Offset)) + orderby Count descending + select + ( + FixTypeName(g.Key, UniqueOffsets), + Count, + UniqueOffsets.Count, + g.Select(r => r.ptr) + ); + + PrintPointerTable("Symbol", "[Unique Pointers]", forceTruncate, resolvedQuery); + } + + private void PrintPointerTable(string nameColumn, string truncatedName, bool forceTruncate, IEnumerable<(string Name, int Count, int Unique, IEnumerable Pointers)> query) + { + var resolved = query.ToArray(); + if (resolved.Length == 0) + return; + + int single = resolved.Count(r => r.Count == 1); + int multi = resolved.Length - single; + bool truncate = forceTruncate || (single + multi > 75 && single > multi); + truncate = false; + + int maxNameLen = multi > 0 ? resolved.Where(r => !truncate || r.Count > 1).Max(r => r.Name.Length) : resolved.Max(r => r.Name.Length); + int nameLen = Math.Min(80, maxNameLen); + nameLen = Math.Max(nameLen, truncatedName.Length); + + TableOutput table = new(Console, (nameLen, ""), (12, "n0"), (12, "n0"), (12, "x")); + table.Divider = " "; + table.WriteRowWithSpacing('-', nameColumn, "Unique", "Count", "RndPtr"); + + var items = truncate ? resolved.Take(multi) : resolved; + foreach (var (Name, Count, Unique, Pointers) in items) + table.WriteRow(Name, Unique, Count, Pointers.FindMostCommonPointer()); + + if (truncate) + table.WriteRow(truncatedName, single, single); + + table.WriteRowWithSpacing('-', " [ TOTALS ] ", resolved.Sum(r => r.Unique), resolved.Sum(r => r.Count), ""); + } + + private static string FixTypeName(string typeName, HashSet offsets) + { + if (typeName.EndsWith("!") && typeName.Count(r => r == '!') == 1) + typeName = typeName.Substring(0, typeName.Length - 1); + + int vtableIdx = typeName.IndexOf(VtableConst); + if (vtableIdx > 0) + typeName = typeName.Replace(VtableConst, "") + "::vtable"; + + if (offsets.Count == 1 && offsets.Single() > 0) + typeName = $"{typeName}+{offsets.Single():x}"; + else if (offsets.Count > 1) + typeName = $"{typeName}+..."; + + return typeName; + } + + private RegionPointers ProcessOneRegion(bool pinnedOnly, IEnumerable<(ulong Pointer, DescribedRegion Range)> pointersFound, MemoryWalkContext ctx) + { + RegionPointers result = new(); + + foreach ((ulong Pointer, DescribedRegion Range) found in pointersFound) + { + if (found.Range.ClrMemoryKind == ClrMemoryKind.GCHeapSegment) + { + if (pinnedOnly) + { + if (ctx.IsPinnedObject(found.Pointer, out ClrObject obj)) + + result.AddGCPointer(obj); + else + result.AddGCPointer(found.Pointer); + } + else + { + ClrObject obj = Runtime.Heap.GetObject(found.Pointer); + if (obj.IsValid) + result.AddGCPointer(obj); + } + } + else if (found.Range.Type == MemoryRegionType.MEM_IMAGE) + { + bool hasSymbols = false; + IModuleSymbols symbols = found.Range.Module?.Services.GetService(); + if (symbols is not null) + hasSymbols = symbols.GetSymbolStatus() == SymbolStatus.Loaded; + + result.AddRegionPointer(found.Range, found.Pointer, hasSymbols); + } + else + { + result.AddRegionPointer(found.Range, found.Pointer, hasSymbols: false); + } + } + + return result; + } + + private MemoryWalkContext CreateMemoryWalkContext() + { + HashSet seen = new(); + List pinned = new(); + + foreach (var root in Runtime.Heap.EnumerateRoots().Where(r => r.IsPinned)) + { + if (root.Object.IsValid && !root.Object.IsFree) + if (seen.Add(root.Object)) + pinned.Add(root.Object); + } + + foreach (ClrSegment seg in Runtime.Heap.Segments.Where(s => s.IsPinnedObjectSegment || s.IsLargeObjectSegment)) + { + foreach (ClrObject obj in seg.EnumerateObjects().Where(o => seen.Add(o))) + { + if (!obj.IsFree && obj.IsValid) + pinned.Add(obj); + } + } + + return new MemoryWalkContext(pinned); + } + + private class RegionPointers + { + public Dictionary> ResolvablePointers { get; } = new(); + public Dictionary> UnresolvablePointers { get; } = new(); + public List PinnedPointers { get; } = new(); + public List NonPinnedGCPointers { get; } = new(); + public long PointersToGC => PinnedPointers.Count + NonPinnedGCPointers.Count; + + public RegionPointers() + { + } + + public void AddGCPointer(ulong address) + { + NonPinnedGCPointers.Add(address); + } + + public void AddGCPointer(ClrObject obj) + { + PinnedPointers.Add(obj); + } + + internal void AddRegionPointer(DescribedRegion range, ulong pointer, bool hasSymbols) + { + var pointerMap = hasSymbols ? ResolvablePointers : UnresolvablePointers; + + if (!pointerMap.TryGetValue(range, out List pointers)) + pointers = pointerMap[range] = new(); + + pointers.Add(pointer); + } + + public void AddTo(RegionPointers destination) + { + AddTo(ResolvablePointers, destination.ResolvablePointers); + AddTo(UnresolvablePointers, destination.UnresolvablePointers); + destination.PinnedPointers.AddRange(PinnedPointers); + destination.NonPinnedGCPointers.AddRange(NonPinnedGCPointers); + } + + private static void AddTo(Dictionary> sourceDict, Dictionary> destDict) + { + foreach (var item in sourceDict) + { + if (destDict.TryGetValue(item.Key, out List values)) + values.AddRange(item.Value); + else + destDict[item.Key] = new(item.Value); + } + } + } + + private class MemoryWalkContext + { + private readonly Dictionary _resolved = new(); + private readonly ClrObject[] _pinned; + + public MemoryWalkContext(IEnumerable pinnedObjects) + { + _pinned = pinnedObjects.Where(o => o.IsValid && !o.IsFree).OrderBy(o => o.Address).ToArray(); + } + + public bool IsPinnedObject(ulong address, out ClrObject found) + { + ClrObject last = _pinned.LastOrDefault(); + if (_pinned.Length == 0 || address < _pinned[0].Address || address >= last.Address + last.Size) + { + found = default; + return false; + } + + int low = 0; + int high = _pinned.Length - 1; + while (low <= high) + { + int mid = (low + high) >> 1; + if (_pinned[mid].Address + _pinned[mid].Size <= address) + { + low = mid + 1; + } + else if (address < _pinned[mid].Address) + { + high = mid - 1; + } + else + { + found = _pinned[mid]; + return true; + } + } + + found = default; + return false; + } + + public (string Symbol, int Offset) ResolveSymbol(IModuleService modules, ulong ptr) + { + if (_resolved.TryGetValue(ptr, out (string, int) result)) + return result; + + // _resolved is just a cache. Don't let it get so big we eat all of the memory. + if (_resolved.Count > 16 * 1024) + _resolved.Clear(); + + IModule module = modules.GetModuleFromAddress(ptr); + IModuleSymbols symbols = module?.Services.GetService(); + + if (symbols is not null && symbols.TryGetSymbolName(ptr, out string symbolName, out ulong displacement)) + { + string moduleName = module.FileName; + if (!string.IsNullOrWhiteSpace(moduleName)) + symbolName = Path.GetFileName(moduleName) + "!" + symbolName; + + return _resolved[ptr] = (symbolName, displacement > int.MaxValue ? int.MaxValue : (int)displacement); + } + + return (null, -1); + } + } + + [HelpInvoke] + public void HelpInvoke() + { + WriteLine( +@"------------------------------------------------------------------------------- +The findpointersin command will search the regions of memory given by MADDRESS_TYPE_LIST +to find all pointers to other memory regions and display them. By default, pointers +to the GC heap are only displayed if the object it points to is pinned. (As any +random pointer to the GC heap to a non-pinned object is either an old/leftover +pointer, or a stray pointer that should be ignored.) If --all is set, +then this command print out ALL objects that are pointed to instead of collapsing +them into one entry. + +usage: !findpointersin [--all] MADDRESS_TYPE_LIST + +Note: The MADDRESS_TYPE_LIST must be a memory type as printed by !maddress. + +Example: ""!findpointersin PAGE_READWRITE"" will only search for regions of memory that +!maddress marks as ""PAGE_READWRITE"" and not every page of memory that's +marked with PAGE_READWRITE protection. + +Example: Running the command ""!findpointersin Stack PAGE_READWRITE"" will find all pointers +on any ""Stack"" and ""PAGE_READWRITE"" memory segments and summarize those contents into +three tables: One table for pointers to the GC heap, one table for pointers where +symbols could be resolved, and one table of pointers where we couldn't resolve symbols. + + +Sample Output: + + Pointers to GC heap: + -------------------------------Type---------Unique----------Count---------RndPtr + [Pointers to non-pinned objects] 3,168 16,765 7f05b80d5b60 + System.Object[] 3 58 7f07380f3120 + System.Byte[] 7 22 7f07380f00d8 + Microsoft.Caching.ClrMD.RawResult[] 2 14 7f063822ae58 + ----------------------- [ TOTALS ] ----------3,180---------16,859--------------- + + Pointers to images: + --------------------------------------------------------------------------Symbol---------Unique----------Count---------RndPtr + libcoreclr+... 34 457 7f08c66ff776 + libcoreclr!JIT_GetSharedGCThreadStaticBase+33 1 260 7f08c637b453 + libcoreclr!COMInterlocked::ExchangeObject+17 1 258 7f08c6336597 + + ... + -------------------------------------------------------------------- [ TOTALS ] ------------740---------10,361--------------- + + Other pointers: + ---------------------------------------------------------------Region---------Unique----------Count---------RndPtr + Stack 25,229 37,656 7f05297f4738 + PAGE_READWRITE 1,696 7,882 7f0500000000 + LowFrequencyHeap 2,618 7,347 7f084d1868e0 + + ... + --------------------------------------------------------- [ TOTALS ] ---------33,360---------72,029--------------- +"); + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/GCToNativeCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/GCToNativeCommand.cs new file mode 100644 index 000000000..fbbee8dc0 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/GCToNativeCommand.cs @@ -0,0 +1,561 @@ +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.Runtime; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using static Microsoft.Diagnostics.ExtensionCommands.NativeAddressHelper; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "gctonative", Help = "Finds GC objects which point to the given native memory ranges.")] + public sealed class GCToNativeCommand : CommandBase + { + [Argument(Help ="The types of memory to search the GC heap for.")] + public string[] MemoryTypes { get; set; } + + [Option(Name = "--all", Aliases = new string[] { "-a" }, Help = "Show the complete list of objects and not just a summary.")] + public bool ShowAll { get; set; } + + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + [ServiceImport] + public NativeAddressHelper AddressHelper { get; set; } + + private int Width + { + get + { + int width = Console.WindowWidth; + if (width == 0) + width = 120; + if (width > 256) + width = 256; + + return width; + } + } + + public override void Invoke() + { + if (MemoryTypes is null || MemoryTypes.Length == 0) + throw new DiagnosticsException("Must specify at least one memory region type to search for."); + + PrintGCPointersToMemory(ShowAll, MemoryTypes); + } + + public void PrintGCPointersToMemory(bool showAll, params string[] memoryTypes) + { + // Strategy: + // 1. Use ClrMD to get the bounds of the GC heap where objects are allocated. + // 2. Manually read that memory and check every pointer-aligned address for pointers to the heap regions requested + // while recording pointers in a list as we go (along with which ClrSegment they came from). + // 3. Walk each GC segment which has pointers to the target regions to find objects: + // a. Annotate each target pointer so we know what object points to the region (and throw away any pointers + // that aren't in an object...those are pointers from dead or relocated objects). + // b. We have some special knowledge of "well known types" here that contain pointers. These types point to + // native memory and contain a size of the region they point to. Record that information as we go. + // 4. Use information from "well known types" about regions of memory to annotate other pointers that do not have + // size information. + // 5. Display all of this to the user. + + if (memoryTypes.Length == 0) + return; + + IEnumerable rangeEnum = AddressHelper.EnumerateAddressSpace(tagClrMemoryRanges: true, includeReserveMemory: false, tagReserveMemoryHeuristically: false); + rangeEnum = rangeEnum.Where(r => memoryTypes.Any(memType => r.Name.Equals(memType, StringComparison.OrdinalIgnoreCase))); + rangeEnum = rangeEnum.OrderBy(r => r.Start); + + DescribedRegion[] ranges = rangeEnum.ToArray(); + + if (ranges.Length == 0) + { + Console.WriteLine($"No matching memory ranges."); + Console.WriteLine(""); + return; + } + + Console.WriteLine("Walking GC heap to find pointers..."); + Dictionary> segmentLists = new(); + + var items = Runtime.Heap.Segments + .SelectMany(Segment => AddressHelper + .EnumerateRegionPointers(Segment.ObjectRange.Start, Segment.ObjectRange.End, ranges) + .Select(regionPointer => (Segment, regionPointer.Address, regionPointer.Pointer, regionPointer.MemoryRange))); + + foreach (var item in items) + { + if (!segmentLists.TryGetValue(item.Segment, out List list)) + list = segmentLists[item.Segment] = new(); + + list.Add(new GCObjectToRange(item.Address, item.Pointer, item.MemoryRange)); + } + + Console.WriteLine("Resolving object names..."); + foreach (string type in memoryTypes) + { + WriteHeader($" {type} Regions "); + + List addressesNotInObjects = new(); + List<(ulong Pointer, ClrObject Object)> unknownObjPointers = new(); + Dictionary knownMemory = new(); + Dictionary sizeHints = new(); + + foreach (KeyValuePair> segEntry in segmentLists) + { + ClrSegment seg = segEntry.Key; + List pointers = segEntry.Value; + pointers.Sort((x, y) => x.GCPointer.CompareTo(y.GCPointer)); + + int index = 0; + foreach (ClrObject obj in seg.EnumerateObjects()) + { + if (index >= pointers.Count) + break; + + while (index < pointers.Count && pointers[index].GCPointer < obj.Address) + { + // If we "missed" the pointer then it's outside of an object range. + addressesNotInObjects.Add(pointers[index].GCPointer); + + Trace.WriteLine($"Skipping {pointers[index].GCPointer:x} lastObj={obj.Address:x}-{obj.Address + obj.Size:x} {obj.Type?.Name}"); + + index++; + } + + if (index == pointers.Count) + break; + + while (index < pointers.Count && obj.Address <= pointers[index].GCPointer && pointers[index].GCPointer < obj.Address + obj.Size) + { + string typeName = obj.Type?.Name ?? $""; + + if (obj.IsFree) + { + // This is free space, if we found a pointer here then it was likely just relocated and we'll mark it elsewhere + } + else if (pointers[index].NativeMemoryRange.Name != type) + { + // This entry is for a different memory type, we'll get it on another pass + } + else if (knownMemory.ContainsKey(obj)) + { + // do nothing, we already marked this memory + } + else if (KnownClrMemoryPointer.ContainsKnownClrMemoryPointers(obj)) + { + foreach (KnownClrMemoryPointer knownMem in KnownClrMemoryPointer.EnumerateKnownClrMemoryPointers(obj, sizeHints)) + knownMemory.Add(obj, knownMem); + } + else + { + if (typeName.Contains('>')) + typeName = CollapseGenerics(typeName); + + unknownObjPointers.Add((pointers[index].TargetSegmentPointer, obj)); + } + + index++; + } + } + } + + Console.WriteLine(""); + if (knownMemory.Count == 0 && unknownObjPointers.Count == 0) + { + Console.WriteLine($"No GC heap pointers to '{type}' regions."); + } + else + { + if (showAll) + { + Console.WriteLine($"All memory pointers:"); + + IEnumerable<(ulong Pointer, ulong Size, ulong Object, string Type)> allPointers = unknownObjPointers.Select(unknown => (unknown.Pointer, 0ul, unknown.Object.Address, unknown.Object.Type?.Name ?? "")); + allPointers = allPointers.Concat(knownMemory.Values.Select(k => (k.Pointer, GetSize(sizeHints, k), k.Object.Address, k.Name))); + + TableOutput allOut = new(Console, (16, "x"), (16, "x"), (16, "x")) + { + Divider = " | " + }; + + allOut.WriteRowWithSpacing('-', "Pointer", "Size", "Object", "Type"); + foreach (var entry in allPointers) + if (entry.Size == 0) + allOut.WriteRow(entry.Pointer, "", entry.Object, entry.Type); + else + allOut.WriteRow(entry.Pointer, entry.Size, entry.Object, entry.Type); + + Console.WriteLine(""); + } + + if (knownMemory.Count > 0) + { + Console.WriteLine($"Well-known memory pointer summary:"); + + // totals + var knownMemorySummary = from known in knownMemory.Values + group known by known.Name into g + let Name = g.Key + let Count = g.Count() + let TotalSize = g.Sum(k => (long)GetSize(sizeHints, k)) + orderby TotalSize descending, Name ascending + select new + { + Name, + Count, + TotalSize, + Pointer = g.Select(p => p.Pointer).FindMostCommonPointer() + }; + + int maxNameLen = Math.Min(80, knownMemory.Values.Max(r => r.Name.Length)); + + TableOutput summary = new(Console, (-maxNameLen, ""), (8, "n0"), (12, "n0"), (12, "n0"), (12, "x")) + { + Divider = " | " + }; + + summary.WriteRowWithSpacing('-', "Type", "Count", "Size", "Size (bytes)", "RndPointer"); + + foreach (var item in knownMemorySummary) + summary.WriteRow(item.Name, item.Count, item.TotalSize.ConvertToHumanReadable(), item.TotalSize, item.Pointer); + + (int totalRegions, ulong totalBytes) = GetSizes(knownMemory, sizeHints); + + summary.WriteSpacer('-'); + summary.WriteRow("[TOTAL]", totalRegions, totalBytes.ConvertToHumanReadable(), totalBytes); + + Console.WriteLine(""); + } + + + if (unknownObjPointers.Count > 0) + { + Console.WriteLine($"Other memory pointer summary:"); + + var unknownMemQuery = from known in unknownObjPointers + let name = CollapseGenerics(known.Object.Type?.Name ?? "") + group known by name into g + let Name = g.Key + let Count = g.Count() + orderby Count descending + select new + { + Name, + Count, + Pointer = g.Select(p => p.Pointer).FindMostCommonPointer() + }; + + var unknownMem = unknownMemQuery.ToArray(); + int maxNameLen = Math.Min(80, unknownMem.Max(r => r.Name.Length)); + + TableOutput summary = new(Console, (-maxNameLen, ""), (8, "n0"), (12, "x")) + { + Divider = " | " + }; + + summary.WriteRowWithSpacing('-', "Type", "Count", "RndPointer"); + + foreach (var item in unknownMem) + summary.WriteRow(item.Name, item.Count, item.Pointer); + } + } + } + } + + private static (int Regions, ulong Bytes) GetSizes(Dictionary knownMemory, Dictionary sizeHints) + { + var ordered = from item in knownMemory.Values + orderby item.Pointer ascending, item.Size descending + select item; + + int totalRegions = 0; + ulong totalBytes = 0; + ulong prevEnd = 0; + + foreach (var item in ordered) + { + ulong size = GetSize(sizeHints, item); + + // overlapped pointer + if (item.Pointer < prevEnd) + { + if (item.Pointer + size <= prevEnd) + continue; + + ulong diff = prevEnd - item.Pointer; + if (diff >= size) + continue; + + size -= diff; + prevEnd += size; + } + else + { + totalRegions++; + prevEnd = item.Pointer + size; + } + + totalBytes += size; + } + + return (totalRegions, totalBytes); + } + + private void WriteHeader(string header) + { + int lpad = (Width - header.Length) / 2; + if (lpad > 0) + header = header.PadLeft(Width - lpad, '='); + Console.WriteLine(header.PadRight(Width, '=')); + } + + private static string CollapseGenerics(string typeName) + { + StringBuilder result = new(typeName.Length + 16); + int nest = 0; + for (int i = 0; i < typeName.Length; i++) + { + if (typeName[i] == '<') + { + if (nest++ == 0) + { + if (i < typeName.Length - 1 && typeName[i + 1] == '>') + result.Append("<>"); + else + result.Append("<...>"); + } + } + else if (typeName[i] == '>') + { + nest--; + } + else if (nest == 0) + { + result.Append(typeName[i]); + } + } + + return result.ToString(); + } + + private static ulong GetSize(Dictionary sizeHints, KnownClrMemoryPointer k) + { + if (sizeHints.TryGetValue(k.Pointer, out int hint)) + if ((ulong)hint > k.Size) + return (ulong)hint; + + return k.Size; + } + + private class GCObjectToRange + { + public ulong GCPointer { get; } + public ulong TargetSegmentPointer { get; } + public ClrObject Object { get; set; } + public DescribedRegion NativeMemoryRange { get; } + + public GCObjectToRange(ulong gcaddr, ulong pointer, DescribedRegion nativeMemory) + { + GCPointer = gcaddr; + TargetSegmentPointer = pointer; + NativeMemoryRange = nativeMemory; + } + } + + private class KnownClrMemoryPointer + { + private const string NativeHeapMemoryBlock = "System.Reflection.Internal.NativeHeapMemoryBlock"; + private const string MetadataReader = "System.Reflection.Metadata.MetadataReader"; + private const string NativeHeapMemoryBlockDisposableData = "System.Reflection.Internal.NativeHeapMemoryBlock+DisposableData"; + private const string ExternalMemoryBlockProvider = "System.Reflection.Internal.ExternalMemoryBlockProvider"; + private const string ExternalMemoryBlock = "System.Reflection.Internal.ExternalMemoryBlock"; + private const string RuntimeParameterInfo = "System.Reflection.RuntimeParameterInfo"; + + public string Name => Object.Type?.Name ?? ""; + public ClrObject Object { get; } + public ulong Pointer { get; } + public ulong Size { get; } + + public KnownClrMemoryPointer(ClrObject obj, nint pointer, int size) + { + Object = obj; + Pointer = (ulong)pointer; + Size = (ulong)size; + } + + public static bool ContainsKnownClrMemoryPointers(ClrObject obj) + { + string typeName = obj.Type?.Name; + return typeName == NativeHeapMemoryBlock + || typeName == MetadataReader + || typeName == NativeHeapMemoryBlockDisposableData + || typeName == ExternalMemoryBlockProvider + || typeName == ExternalMemoryBlock + || typeName == RuntimeParameterInfo + ; + } + + public static IEnumerable EnumerateKnownClrMemoryPointers(ClrObject obj, Dictionary sizeHints) + { + switch (obj.Type?.Name) + { + case RuntimeParameterInfo: + { + const int MDInternalROSize = 0x5f8; // Doesn't have to be exact + nint pointer = obj.ReadValueTypeField("m_scope").ReadField("m_metadataImport2"); + AddSizeHint(sizeHints, pointer, MDInternalROSize); + + yield return new KnownClrMemoryPointer(obj, pointer, MDInternalROSize); + } + break; + case ExternalMemoryBlock: + { + nint pointer = obj.ReadField("_buffer"); + int size = obj.ReadField("_size"); + + if (pointer != 0 && size > 0) + AddSizeHint(sizeHints, pointer, size); + + yield return new KnownClrMemoryPointer(obj, pointer, size); + } + break; + + case ExternalMemoryBlockProvider: + { + nint pointer = obj.ReadField("_memory"); + int size = obj.ReadField("_size"); + + if (pointer != 0 && size > 0) + AddSizeHint(sizeHints, pointer, size); + + yield return new KnownClrMemoryPointer(obj, pointer, size); + } + break; + + case NativeHeapMemoryBlockDisposableData: + { + nint pointer = obj.ReadField("_pointer"); + sizeHints.TryGetValue((ulong)pointer, out int size); + yield return new KnownClrMemoryPointer(obj, pointer, size); + } + break; + + case NativeHeapMemoryBlock: + { + // Just here for size hints + + ClrObject pointerObject = obj.ReadObjectField("_data"); + nint pointer = pointerObject.ReadField("_pointer"); + int size = obj.ReadField("_size"); + + if (pointer != 0 && size > 0) + AddSizeHint(sizeHints, pointer, size); + } + + break; + + case MetadataReader: + { + MemoryBlockImpl block = obj.ReadField("Block"); + if (block.Pointer != 0 && block.Size > 0) + yield return new KnownClrMemoryPointer(obj, block.Pointer, block.Size); + } + break; + } + } + + private static void AddSizeHint(Dictionary sizeHints, nint pointer, int size) + { + if (pointer != 0 && size != 0) + { + ulong ptr = (ulong)pointer; + + if (sizeHints.TryGetValue(ptr, out int hint)) + { + if (hint < size) + sizeHints[ptr] = size; + } + else + { + sizeHints[ptr] = size; + } + } + } + + private readonly struct MemoryBlockImpl + { + public readonly nint Pointer { get; } + public readonly int Size { get; } + } + } + + [HelpInvoke] + public void HelpInvoke() + { + WriteLine( +@"------------------------------------------------------------------------------- +!gctonative searches the GC heap for pointers to native memory. This is used +to help locate regions of native memory that are referenced (or possibly held +alive) by objects on the GC heap. + +usage: !gctonative [--all] MADDRESS_TYPE_LIST + +Note: The MADDRESS_TYPE_LIST must be a memory type as printed by !maddress. + +If --all is set, a full list of every pointer from the GC heap to the +specified memory will be displayed instead of just a summary table. + +Sample Output: + + 0:000> !gctonative PAGE_READWRITE + Walking GC heap to find pointers... + Resolving object names... + ================================================ PAGE_READWRITE Regions ================================================ + + Well-known memory pointer summary: + Type-----------------------------------------------------------------Count-----------Size---Size (bytes)-----RndPointer + System.Reflection.Internal.ExternalMemoryBlockProvider | 1,956 | 571.39mb | 599,145,088 | 7f0478747cf0 + System.Reflection.Internal.NativeHeapMemoryBlock+DisposableData | 1,956 | 571.39mb | 599,145,088 | 7f0478747cf0 + System.Reflection.Internal.ExternalMemoryBlock | 1,956 | 161.63mb | 169,483,352 | 7f04898e06a0 + System.Reflection.Metadata.MetadataReader | 1,956 | 161.63mb | 169,483,352 | 7f04898e06a0 + System.Reflection.RuntimeParameterInfo | 176 | 262.63kb | 268,928 | 7f058000c220 + ----------------------------------------------------------------------------------------------------------------------- + [TOTAL] | 1,963 | 571.40mb | 599,155,784 + + Other memory pointer summary: + Type----------------------------------------------------------------------------------Count-----RndPointer + System.SByte[] | 1,511 | 7f0500000000 + System.Byte[] | 539 | 7f0500000000 + System.Reflection.RuntimeAssembly | 135 | 7f05a0000ce0 + System.Char[] | 121 | 7f0500000000 + System.Threading.UnmanagedThreadPoolWorkItem | 113 | 7f05800120e0 + System.Diagnostics.Tracing.EventSource+EventMetadata[] | 75 | 7f0564001a20 + Microsoft.Win32.SafeHandles.SafeEvpMdCtxHandle | 56 | 7f044013c170 + System.Threading.Thread | 40 | 7f05741d7bd0 + System.Security.Cryptography.SafeEvpPKeyHandle | 40 | 7f051400cca0 + Microsoft.Win32.SafeHandles.SafeBioHandle | 38 | 7f05400078d0 + Microsoft.Win32.SafeHandles.SafeX509Handle | 37 | 7f04beab9c30 + Microsoft.Win32.SafeHandles.SafeSslHandle | 20 | 7f0540007af0 + System.Text.RegularExpressions.RegexCache+Node | 19 | 7f0500000001 + System.Collections.Concurrent.ConcurrentDictionary<...>+Node | 19 | 7f0500000001 + System.Diagnostics.Tracing.EventPipeEventProvider | 15 | 7f0580002240 + System.IntPtr[] | 15 | 7f0578000d00 + Microsoft.Extensions.Logging.LoggerMessage+LogValues<...> | 12 | 7f0500000002 + Microsoft.Extensions.Logging.LoggerMessage+LogValues<...>+<...>d__9 | 12 | 7f0500000002 + Microsoft.Win32.SafeHandles.SafeX509StackHandle | 10 | 7f0524179e50 + Microsoft.CodeAnalysis.CSharp.Symbols.MethodSymbol+<...>d__32 | 10 | 7f0500000002 + Microsoft.CodeAnalysis.ModuleMetadata[] | 5 | 7f052a7f7050 + System.Threading.TimerQueue+AppDomainTimerSafeHandle | 2 | 7f05a0006d30 + System.Net.Security.SafeFreeCertContext | 2 | 7f04beab9c30 + System.Threading.LowLevelLock | 1 | 7f05a0003420 + Microsoft.CodeAnalysis.CSharp.CSharpCompilation+ReferenceManager+AssemblyData... | 1 | 7f05800120e0 + System.Net.Sockets.SocketAsyncEngine | 1 | 7f059800edd0 + Microsoft.Extensions.Caching.Memory.CacheEntry | 1 | 7f05241e0000 + System.Runtime.CompilerServices.AsyncTaskMethodBuilder<...>+AsyncStateMachine... | 1 | 7f0500000004 +"); + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/MAddressCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/MAddressCommand.cs new file mode 100644 index 000000000..5742b0e53 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/MAddressCommand.cs @@ -0,0 +1,174 @@ +using Microsoft.Diagnostics.DebugServices; +using System; +using System.Collections.Generic; +using System.Linq; +using static Microsoft.Diagnostics.ExtensionCommands.NativeAddressHelper; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "maddress", Help = "Displays a breakdown of the virtual address space.")] + public sealed class MAddressCommand : CommandBase + { + [Option(Name = "--list", Aliases = new string[] { "-l", "--all", }, Help = "Prints the full list of annotated memory regions.")] + public bool ListAll { get; set; } + + [Option(Name = "--images", Aliases = new string[] { "-i", "-image", "-img", "--img" }, Help = "Prints the table of image memory usage.")] + public bool ShowImageTable { get; set; } + + [Option(Name = "--includeReserve", Help = "Include MEM_RESERVE regions in the output.")] + public bool IncludeReserveMemory { get; set; } + + [Option(Name = "--tagReserve", Help = "Heuristically tag MEM_RESERVE regions based on adjacent memory regions.")] + public bool TagReserveMemoryHeuristically { get; set; } + + [ServiceImport] + public NativeAddressHelper AddressHelper { get; set; } + + public override void Invoke() + { + if (TagReserveMemoryHeuristically && !IncludeReserveMemory) + throw new DiagnosticsException("Cannot use --tagReserve without --includeReserve"); + + PrintMemorySummary(ListAll, ShowImageTable, IncludeReserveMemory, TagReserveMemoryHeuristically); + } + + public void PrintMemorySummary(bool printAllMemory, bool showImageTable, bool includeReserveMemory, bool tagReserveMemoryHeuristically) + { + IEnumerable memoryRanges = AddressHelper.EnumerateAddressSpace(tagClrMemoryRanges: true, includeReserveMemory, tagReserveMemoryHeuristically); + if (!includeReserveMemory) + memoryRanges = memoryRanges.Where(m => m.State != MemoryRegionState.MEM_RESERVE); + + DescribedRegion[] ranges = memoryRanges.ToArray(); + + int nameSizeMax = ranges.Max(r => r.Name.Length); + + // Tag reserved memory based on what's adjacent. + if (tagReserveMemoryHeuristically) + CollapseReserveRegions(ranges); + + if (printAllMemory) + { + int kindSize = ranges.Max(r => r.Type.ToString().Length); + int stateSize = ranges.Max(r => r.State.ToString().Length); + int protectSize = ranges.Max(r => r.Protection.ToString().Length); + + TableOutput output = new(Console, (nameSizeMax, ""), (12, "x"), (12, "x"), (12, ""), (kindSize, ""), (stateSize, ""), (protectSize, "")) + { + AlignLeft = true, + Divider = " | " + }; + + output.WriteRowWithSpacing('-', "Memory Kind", "StartAddr", "EndAddr-1", "Size", "Type", "State", "Protect", "Image"); + foreach (DescribedRegion mem in ranges) + output.WriteRow(mem.Name, mem.Start, mem.End, mem.Length.ConvertToHumanReadable(), mem.Type, mem.State, mem.Protection, mem.Image); + + output.WriteSpacer('-'); + } + + if (showImageTable) + { + var imageGroups = from mem in ranges.Where(r => r.State != MemoryRegionState.MEM_RESERVE && r.Image != null) + group mem by mem.Image into g + let Size = g.Sum(k => (long)(k.End - k.Start)) + orderby Size descending + select new + { + Image = g.Key, + Count = g.Count(), + Size + }; + + int moduleLen = Math.Max(80, ranges.Max(r => r.Image?.Length ?? 0)); + + TableOutput output = new(Console, (moduleLen, ""), (8, "n0"), (12, ""), (24, "n0")) + { + Divider = " | " + }; + + output.WriteRowWithSpacing('-', "Image", "Regions", "Size", "Size (bytes)"); + + int count = 0; + long size = 0; + foreach (var item in imageGroups) + { + output.WriteRow(item.Image, item.Count, item.Size.ConvertToHumanReadable(), item.Size); + count += item.Count; + size += item.Size; + } + + output.WriteSpacer('-'); + output.WriteRow("[TOTAL]", count, size.ConvertToHumanReadable(), size); + WriteLine(""); + } + + + // Print summary table unconditionally + { + var grouped = from mem in ranges + let name = mem.Name + group mem by name into g + let Count = g.Count() + let Size = g.Sum(f => (long)(f.End - f.Start)) + orderby Size descending + select new + { + Name = g.Key, + Count, + Size + }; + + TableOutput output = new(Console, (-nameSizeMax, ""), (8, "n0"), (12, ""), (24, "n0")) + { + Divider = " | " + }; + + output.WriteRowWithSpacing('-', "Region Type", "Count", "Size", "Size (bytes)"); + + int count = 0; + long size = 0; + foreach (var item in grouped) + { + output.WriteRow(item.Name, item.Count, item.Size.ConvertToHumanReadable(), item.Size); + count += item.Count; + size += item.Size; + } + + output.WriteSpacer('-'); + output.WriteRow("[TOTAL]", count, size.ConvertToHumanReadable(), size); + } + } + + + [HelpInvoke] + public void HelpInvoke() + { + WriteLine( +@"------------------------------------------------------------------------------- +!maddress is a managed version of !address, which attempts to annotate all memory +with information about CLR's heaps. + +usage: !maddress [--list] [--images] [--includeReserve [--tagReserve]] + +Flags: + --list + Shows the full list of annotated memory regions and not just the statistics + table. + + --images + Summarizes the memory ranges consumed by images in the process. + + --includeReserve + Include reserved memory (MEM_RESERVE) in the output. This is usually only + useful if there is virtual address exhaustion. + + --tagReserve + If this flag is set, then !maddress will attempt to ""blame"" reserve segments + on the region that immediately proceeded it. For example, if a ""Heap"" + memory segment is immediately followed by a MEM_RESERVE region, we will call + that reserve region HeapReserve. Note that this is a heuristic and NOT + intended to be completely accurate. This can be useful to try to figure out + what is creating large amount of MEM_RESERVE regions. +"); + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/NativeAddressHelper.cs b/src/Microsoft.Diagnostics.ExtensionCommands/NativeAddressHelper.cs new file mode 100644 index 000000000..f13f13a83 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/NativeAddressHelper.cs @@ -0,0 +1,296 @@ +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.Runtime; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.InteropServices; +using System; +using System.Buffers; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [ServiceExport(Scope = ServiceScope.Target)] + public sealed class NativeAddressHelper + { + [ServiceImport] + public ITarget Target { get; set; } + + [ServiceImport] + public IMemoryService MemoryService { get; set; } + + [ServiceImport] + public IThreadService ThreadService { get; set; } + + [ServiceImport] + public IRuntimeService RuntimeService { get; set; } + + [ServiceImport] + public IModuleService ModuleService { get; set; } + + [ServiceImport] + public IMemoryRegionService MemoryRegionService { get; set; } + + /// + /// Enumerates the entire address space, optionally tagging special CLR heaps, and optionally "collapsing" + /// MEM_RESERVE regions with a heuristic to blame them on the MEM_COMMIT region that came before it. + /// See for more info. + /// + /// Whether to "tag" regions with CLR memory for more details. + /// Whether to include MEM_RESERVE memory or not in the enumeration. + /// Whether to heuristically "blame" MEM_RESERVE regions on what + /// lives before it in the address space. For example, if there is a MEM_COMMIT region followed by a MEM_RESERVE + /// region in the address space, this function will "blame" the MEM_RESERVE region on whatever type of memory + /// the MEM_COMMIT region happens to be. Usually this will be correct (e.g. the native heap will reserve a + /// large chunk of memory and commit the beginning of it as it allocates more and more memory...the RESERVE + /// region was actually "caused" by the Heap space before it). Sometimes this will simply be wrong when + /// a MEM_COMMIT region is next to an unrelated MEM_RESERVE region. + /// + /// This is a heuristic, so use it accordingly. + /// If !address fails we will throw InvalidOperationException. This is usually + /// because symbols for ntdll couldn't be found. + /// An enumerable of memory ranges. + internal IEnumerable EnumerateAddressSpace(bool tagClrMemoryRanges, bool includeReserveMemory, bool tagReserveMemoryHeuristically) + { + var addressResult = from region in MemoryRegionService.EnumerateRegions() + where region.State != MemoryRegionState.MEM_FREE + select new DescribedRegion(region, ModuleService.GetModuleFromAddress(region.Start)); + + if (!includeReserveMemory) + addressResult = addressResult.Where(m => m.State != MemoryRegionState.MEM_RESERVE); + + DescribedRegion[] ranges = addressResult.OrderBy(r => r.Start).ToArray(); + if (tagClrMemoryRanges) + { + foreach (IRuntime runtime in RuntimeService.EnumerateRuntimes()) + { + ClrRuntime clrRuntime = runtime.Services.GetService(); + if (clrRuntime is not null) + { + foreach (ClrMemoryPointer mem in ClrMemoryPointer.EnumerateClrMemoryAddresses(clrRuntime)) + { + var found = ranges.Where(m => m.Start <= mem.Address && mem.Address < m.End).ToArray(); + + if (found.Length == 0) + Trace.WriteLine($"Warning: Could not find a memory range for {mem.Address:x} - {mem.Kind}."); + else if (found.Length > 1) + Trace.WriteLine($"Warning: Found multiple memory ranges for entry {mem.Address:x} - {mem.Kind}."); + + foreach (var entry in found) + { + if (entry.ClrMemoryKind != ClrMemoryKind.None && entry.ClrMemoryKind != mem.Kind) + Trace.WriteLine($"Warning: Overwriting range {entry.Start:x} {entry.ClrMemoryKind} -> {mem.Kind}."); + + entry.ClrMemoryKind = mem.Kind; + } + } + } + } + } + + if (tagReserveMemoryHeuristically) + { + foreach (DescribedRegion mem in ranges) + { + string memName = mem.Name; + if (memName == "RESERVED") + TagMemoryRecursive(mem, ranges); + } + } + + // On Linux, !address doesn't mark stack space. Go do that. + if (Target.OperatingSystem == OSPlatform.Linux) + MarkStackSpace(ranges); + + return ranges; + } + + private void MarkStackSpace(DescribedRegion[] ranges) + { + foreach (IThread thread in ThreadService.EnumerateThreads()) + { + if (thread.TryGetRegisterValue(ThreadService.StackPointerIndex, out ulong sp) && sp != 0) + { + DescribedRegion range = FindMemory(ranges, sp); + if (range is not null) + range.Description = "Stack"; + } + } + } + + private static DescribedRegion FindMemory(DescribedRegion[] ranges, ulong ptr) + { + if (ptr < ranges[0].Start || ptr >= ranges.Last().End) + return null; + + int low = 0; + int high = ranges.Length - 1; + while (low <= high) + { + int mid = (low + high) >> 1; + if (ranges[mid].End <= ptr) + { + low = mid + 1; + } + else if (ptr < ranges[mid].Start) + { + high = mid - 1; + } + else + { + return ranges[mid]; + } + } + + return null; + } + + /// + /// This method heuristically tries to "blame" MEM_RESERVE regions on what lives before it on the heap. + /// For example, if there is a MEM_COMMIT region followed by a MEM_RESERVE region in the address space, + /// this function will "blame" the MEM_RESERVE region on whatever type of memory the MEM_COMMIT region + /// happens to be. Usually this will be correct (e.g. the native heap will reserve a large chunk of + /// memory and commit the beginning of it as it allocates more and more memory...the RESERVE region + /// was actually "caused" by the Heap space before it). Sometimes this will simply be wrong when + /// a MEM_COMMIT region is next to an unrelated MEM_RESERVE region. + /// + /// This is a heuristic, so use it accordingly. + /// + internal static void CollapseReserveRegions(DescribedRegion[] ranges) + { + foreach (DescribedRegion mem in ranges) + { + string memName = mem.Name; + if (memName == "RESERVED") + TagMemoryRecursive(mem, ranges); + } + } + + private static DescribedRegion TagMemoryRecursive(DescribedRegion mem, DescribedRegion[] ranges) + { + if (mem.Name != "RESERVED") + return mem; + + DescribedRegion found = ranges.SingleOrDefault(r => r.End == mem.Start); + if (found is null) + return null; + + DescribedRegion nonReserved = TagMemoryRecursive(found, ranges); + if (nonReserved is null) + return null; + + mem.Description = nonReserved.Name; + return nonReserved; + } + + internal IEnumerable<(ulong Address, ulong Pointer, DescribedRegion MemoryRange)> EnumerateRegionPointers(ulong start, ulong end, DescribedRegion[] ranges) + { + ulong[] array = ArrayPool.Shared.Rent(4096); + int arrayBytes = array.Length * sizeof(ulong); + try + { + ulong curr = start; + ulong remaining = end - start; + + while (remaining > 0) + { + int size = Math.Min(remaining > int.MaxValue ? int.MaxValue : (int)remaining, arrayBytes); + bool res = ReadMemory(curr, array, size, out int bytesRead); + if (!res || bytesRead <= 0) + break; + + for (int i = 0; i < bytesRead / sizeof(ulong); i++) + { + ulong ptr = array[i]; + + DescribedRegion found = FindMemory(ranges, ptr); + if (found is not null) + yield return (curr + (uint)i * sizeof(ulong), ptr, found); + } + + curr += (uint)bytesRead; + remaining -= (uint)bytesRead; ; + } + + } + finally + { + ArrayPool.Shared.Return(array); + } + } + + private unsafe bool ReadMemory(ulong start, ulong[] array, int size, out int bytesRead) + { + fixed (ulong* ptr = array) + { + Span buffer = new(ptr, size); + MemoryService.ReadMemory(start, buffer, out bytesRead); + return bytesRead == size; + } + } + + internal class DescribedRegion : IMemoryRegion + { + private readonly IMemoryRegion _region; + + public IModule Module { get; } + + public DescribedRegion(IMemoryRegion region, IModule module) + { + _region = region; + Module = module; + } + + public ulong Start => _region.Start; + + public ulong End => _region.End; + + public ulong Size => _region.Size; + + public MemoryRegionType Type => _region.Type; + + public MemoryRegionState State => _region.State; + + public MemoryRegionProtection Protection => _region.Protection; + + public MemoryRegionUsage Usage => _region.Usage; + + public string Image => _region.Image; + + public string Description { get; internal set; } + + public ClrMemoryKind ClrMemoryKind { get; internal set; } + public ulong Length => End <= Start ? 0 : End - Start; + + public string Name + { + get + { + if (ClrMemoryKind != ClrMemoryKind.None) + return ClrMemoryKind.ToString(); + + if (!string.IsNullOrWhiteSpace(Description)) + return Description; + + if (State == MemoryRegionState.MEM_RESERVE) + return "RESERVED"; + else if (State == MemoryRegionState.MEM_FREE) + return "FREE"; + + if (Type == MemoryRegionType.MEM_IMAGE || !string.IsNullOrWhiteSpace(Image)) + return "IMAGE"; + + string result = Protection.ToString(); + if (Type == MemoryRegionType.MEM_MAPPED) + { + if (string.IsNullOrWhiteSpace(result)) + result = Type.ToString(); + else + result = result.Replace("PAGE", "MAPPED"); + } + + return result; + } + } + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/TableOutput.cs b/src/Microsoft.Diagnostics.ExtensionCommands/TableOutput.cs new file mode 100644 index 000000000..60c9c71a1 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/TableOutput.cs @@ -0,0 +1,132 @@ +using Microsoft.Diagnostics.DebugServices; +using System; +using System.Linq; +using System.Text; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + internal sealed class TableOutput + { + private readonly char _spacing = ' '; + public string Divider { get; set; } = " "; + public bool AlignLeft { get; set; } = false; + + public IConsoleService Console { get; } + + private readonly (int width, string format)[] _formats; + + public TableOutput(IConsoleService console, params (int width, string format)[] columns) + { + _formats = columns.ToArray(); + Console = console; + } + + public void WriteRow(params object[] columns) + { + StringBuilder sb = new(Divider.Length * columns.Length + _formats.Sum(c => Math.Abs(c.width)) + 32); + + for (int i = 0; i < columns.Length; i++) + { + if (i != 0) + sb.Append(Divider); + + (int width, string format) = i < _formats.Length ? _formats[i] : default; + + string value; + if (string.IsNullOrWhiteSpace(format)) + value = columns[i]?.ToString(); + else + value = Format(columns[i], format); + + AddValue(_spacing, sb, width, value ?? ""); + } + + Console.WriteLine(sb.ToString()); + } + + public void WriteRowWithSpacing(char spacing, params object[] columns) + { + StringBuilder sb = new(columns.Length + _formats.Sum(c => Math.Abs(c.width))); + + for (int i = 0; i < columns.Length; i++) + { + if (i != 0) + sb.Append(spacing, Divider.Length); + + (int width, string format) = i < _formats.Length ? _formats[i] : default; + + string value; + if (string.IsNullOrWhiteSpace(format)) + value = columns[i]?.ToString(); + else + value = Format(columns[i], format); + + AddValue(spacing, sb, width, value ?? ""); + } + + Console.WriteLine(sb.ToString()); + } + + + public void WriteSpacer(char spacer) + { + Console.WriteLine(new string(spacer, Divider.Length * (_formats.Length - 1) + _formats.Sum(c => Math.Abs(c.width)))); + } + + private void AddValue(char spacing, StringBuilder sb, int width, string value) + { + bool leftAlign = AlignLeft ? width > 0 : width < 0; + width = Math.Abs(width); + + if (width == 0) + { + sb.Append(value); + } + else if (value.Length > width) + { + if (width <= 3) + { + sb.Append(value, 0, width); + } + else if (leftAlign) + { + value = value.Substring(0, width - 3); + sb.Append(value); + sb.Append("..."); + } + else + { + value = value.Substring(value.Length - (width - 3)); + sb.Append("..."); + sb.Append(value); + } + + } + else if (leftAlign) + { + sb.Append(value.PadRight(width, spacing)); + } + else + { + sb.Append(value.PadLeft(width, spacing)); + } + } + + private static string Format(object obj, string format) + { + if (obj is null) + return null; + + return obj switch + { + nint ni => ni.ToString(format), + ulong ul => ul.ToString(format), + long l => l.ToString(format), + uint ui => ui.ToString(format), + int i => i.ToString(format), + string s => s, + _ => throw new NotImplementedException(obj.GetType().ToString()), + }; + } + } +} diff --git a/src/SOS/SOS.Extensions/DbgEngOutputHolder.cs b/src/SOS/SOS.Extensions/DbgEngOutputHolder.cs new file mode 100644 index 000000000..b23831475 --- /dev/null +++ b/src/SOS/SOS.Extensions/DbgEngOutputHolder.cs @@ -0,0 +1,49 @@ +using Microsoft.Diagnostics.Runtime.Utilities; +using SOS.Hosting.DbgEng.Interop; +using System; +using System.Runtime.InteropServices; + +namespace SOS.Extensions +{ + /// + /// A helper class to capture output of DbgEng commands that will restore the previous output callbacks + /// when disposed. + /// + internal class DbgEngOutputHolder : IDebugOutputCallbacksWide, IDisposable + { + private readonly IDebugClient5 _client; + private readonly IDebugOutputCallbacksWide _previous; + + public DEBUG_OUTPUT InterestMask { get; set; } + + /// + /// Event fired when we receive output from the debugger. + /// + public Action OutputReceived; + + public DbgEngOutputHolder(IDebugClient5 client, DEBUG_OUTPUT interestMask = DEBUG_OUTPUT.NORMAL) + { + _client = client; + InterestMask = interestMask; + + _client.GetOutputCallbacksWide(out _previous); + HResult hr = _client.SetOutputCallbacksWide(this); + if (!hr) + throw Marshal.GetExceptionForHR(hr); + } + + public void Dispose() + { + if (_previous is not null) + _client.SetOutputCallbacksWide(_previous); + } + + public int Output(DEBUG_OUTPUT Mask, [In, MarshalAs(UnmanagedType.LPStr)] string Text) + { + if ((InterestMask & Mask) != 0 && Text is not null) + OutputReceived?.Invoke(Mask, Text); + + return 0; + } + } +} diff --git a/src/SOS/SOS.Extensions/DebuggerServices.cs b/src/SOS/SOS.Extensions/DebuggerServices.cs index 4cc700bf3..7f83c1702 100644 --- a/src/SOS/SOS.Extensions/DebuggerServices.cs +++ b/src/SOS/SOS.Extensions/DebuggerServices.cs @@ -31,10 +31,23 @@ namespace SOS private readonly HostType _hostType; + /// + /// A pointer to the underlying IDebugClient interface if the host is DbgEng. + /// + public IDebugClient5 DebugClient { get; } + internal DebuggerServices(IntPtr punk, HostType hostType) : base(new RefCountedFreeLibrary(IntPtr.Zero), IID_IDebuggerServices, punk) { _hostType = hostType; + + // This uses COM marshalling code, so we also check that the OSPlatform is Windows. + if (hostType == HostType.DbgEng && RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + object obj = Marshal.GetObjectForIUnknown(punk); + if (obj is IDebugClient5 client) + DebugClient = client; + } } public HResult GetOperatingSystem(out DebuggerServices.OperatingSystem operatingSystem) diff --git a/src/SOS/SOS.Extensions/HostServices.cs b/src/SOS/SOS.Extensions/HostServices.cs index dd1fc44d8..560841530 100644 --- a/src/SOS/SOS.Extensions/HostServices.cs +++ b/src/SOS/SOS.Extensions/HostServices.cs @@ -8,6 +8,7 @@ using Microsoft.Diagnostics.ExtensionCommands; using Microsoft.Diagnostics.Runtime; using Microsoft.Diagnostics.Runtime.Utilities; using SOS.Hosting; +using SOS.Hosting.DbgEng.Interop; using System; using System.Collections.Generic; using System.Diagnostics; @@ -238,6 +239,12 @@ namespace SOS.Extensions Trace.TraceWarning($"Cannot add extension command {hr:X8} {name} - {help}"); } } + + if (DebuggerServices.DebugClient is IDebugControl5 control) + { + MemoryRegionServiceFromDebuggerServices memRegions = new(DebuggerServices.DebugClient, control); + _serviceContainer.AddService(memRegions); + } } catch (Exception ex) { diff --git a/src/SOS/SOS.Extensions/MemoryRegionServiceFromDebuggerServices.cs b/src/SOS/SOS.Extensions/MemoryRegionServiceFromDebuggerServices.cs new file mode 100644 index 000000000..98b2b05ee --- /dev/null +++ b/src/SOS/SOS.Extensions/MemoryRegionServiceFromDebuggerServices.cs @@ -0,0 +1,199 @@ +using Microsoft.Diagnostics.DebugServices; +using SOS.Hosting.DbgEng.Interop; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; + +namespace SOS.Extensions +{ + internal class MemoryRegionServiceFromDebuggerServices : IMemoryRegionService + { + private readonly IDebugClient5 _client; + private readonly IDebugControl5 _control; + + public MemoryRegionServiceFromDebuggerServices(IDebugClient5 client, IDebugControl5 control) + { + _client = client; + _control = control; + } + + public IEnumerable EnumerateRegions() + { + bool foundHeader = false; + bool skipped = false; + + (int hr, string text) = RunCommandWithOutput("!address"); + if (hr < 0) + throw new InvalidOperationException($"!address failed with hresult={hr:x}"); + + foreach (string line in text.Split('\n')) + { + if (line.Length == 0) + continue; + + if (!foundHeader) + { + // find the !address header + string[] split = line.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + if (split.Length > 0) + foundHeader = split[0] == "BaseAddress" && split.Last() == "Usage"; + } + else if (!skipped) + { + // skip the ---------- line + skipped = true; + } + else + { + string[] parts = ((line[0] == '+') ? line.Substring(1) : line).Split(new char[] { ' ' }, 6, StringSplitOptions.RemoveEmptyEntries); + ulong start = ulong.Parse(parts[0].Replace("`", ""), System.Globalization.NumberStyles.HexNumber); + ulong end = ulong.Parse(parts[1].Replace("`", ""), System.Globalization.NumberStyles.HexNumber); + + int index = 3; + if (Enum.TryParse(parts[index], ignoreCase: true, out MemoryRegionType type)) + index++; + + if (Enum.TryParse(parts[index], ignoreCase: true, out MemoryRegionState state)) + index++; + + StringBuilder sbRemainder = new(); + for (int i = index; i < parts.Length; i++) + { + if (i != index) + sbRemainder.Append(' '); + + sbRemainder.Append(parts[i]); + } + + string remainder = sbRemainder.ToString(); + parts = remainder.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + MemoryRegionProtection protect = default; + index = 0; + while (index < parts.Length - 1) + { + if (Enum.TryParse(parts[index], ignoreCase: true, out MemoryRegionProtection result)) + { + protect |= result; + if (parts[index + 1] == "|") + index++; + } + else + { + break; + } + + index++; + } + + string description = parts[index++].Trim(); + + // On Linux, !address is reporting this as MEM_PRIVATE or MEM_UNKNOWN + if (description == "Image") + type = MemoryRegionType.MEM_IMAGE; + + // On Linux, !address is reporting this as nothing + if (type == MemoryRegionType.MEM_UNKNOWN && state == MemoryRegionState.MEM_UNKNOWN && protect == MemoryRegionProtection.PAGE_UNKNOWN) + { + state = MemoryRegionState.MEM_FREE; + protect = MemoryRegionProtection.PAGE_NOACCESS; + } + + string image = null; + if (type == MemoryRegionType.MEM_IMAGE) + { + image = parts[index].Substring(1, parts[index].Length - 2); + index++; + } + + if (description.Equals("", StringComparison.OrdinalIgnoreCase)) + description = ""; + + MemoryRegionUsage usage = description switch + { + "" => MemoryRegionUsage.Unknown, + + "Free" => MemoryRegionUsage.Free, + "Image" => MemoryRegionUsage.Image, + + "PEB" => MemoryRegionUsage.Peb, + "PEB32" => MemoryRegionUsage.Peb, + "PEB64" => MemoryRegionUsage.Peb, + + "TEB" => MemoryRegionUsage.Teb, + "TEB32" => MemoryRegionUsage.Teb, + "TEB64" => MemoryRegionUsage.Teb, + + "Stack" => MemoryRegionUsage.Stack, + "Stack32" => MemoryRegionUsage.Stack, + "Stack64" => MemoryRegionUsage.Stack, + + "Heap" => MemoryRegionUsage.Heap, + "Heap32" => MemoryRegionUsage.Heap, + "Heap64" => MemoryRegionUsage.Heap, + + + "PageHeap" => MemoryRegionUsage.PageHeap, + "PageHeap64" => MemoryRegionUsage.PageHeap, + "PageHeap32" => MemoryRegionUsage.PageHeap, + + "MappedFile" => MemoryRegionUsage.FileMapping, + "CLR" => MemoryRegionUsage.CLR, + + "Other" => MemoryRegionUsage.Other, + "Other32" => MemoryRegionUsage.Other, + "Other64" => MemoryRegionUsage.Other, + + _ => MemoryRegionUsage.Unknown + }; + + yield return new AddressMemoryRange() + { + Start = start, + End = end, + Type = type, + State = state, + Protection = protect, + Usage = usage, + Image = image + }; + } + } + + if (!foundHeader) + throw new InvalidOperationException($"!address did not produce a standard header.\nThis may mean symbols could not be resolved for ntdll.\nPlease run !address and make sure the output looks correct."); + + } + + private (int hresult, string output) RunCommandWithOutput(string command) + { + using DbgEngOutputHolder dbgengOutput = new(_client); + StringBuilder sb = new(1024); + dbgengOutput.OutputReceived += (mask, text) => sb.Append(text); + + int hr = _control.ExecuteWide(DEBUG_OUTCTL.THIS_CLIENT, command, DEBUG_EXECUTE.DEFAULT); + + return (hr, sb.ToString()); + } + + private class AddressMemoryRange : IMemoryRegion + { + public ulong Start { get; internal set; } + + public ulong End { get; internal set; } + + public ulong Size { get; internal set; } + + public MemoryRegionType Type { get; internal set; } + + public MemoryRegionState State { get; internal set; } + + public MemoryRegionProtection Protection { get; internal set; } + + public MemoryRegionUsage Usage { get; internal set; } + + public string Image { get; internal set; } + } + } +} diff --git a/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs b/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs index 04e9f807c..16cd71243 100644 --- a/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs +++ b/src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs @@ -9,7 +9,7 @@ using SOS.Hosting.DbgEng.Interop; using System; using System.Collections.Generic; using System.Diagnostics; -using System.Runtime.InteropServices; +using System.IO; namespace SOS.Extensions { @@ -73,6 +73,7 @@ namespace SOS.Extensions private readonly ModuleServiceFromDebuggerServices _moduleService; private Version _version; private string _versionString; + private SymbolStatus _symbolStatus = SymbolStatus.Unknown; public ModuleFromDebuggerServices( ModuleServiceFromDebuggerServices moduleService, @@ -173,6 +174,18 @@ namespace SOS.Extensions return true; } + SymbolStatus IModuleSymbols.GetSymbolStatus() + { + if (_symbolStatus != SymbolStatus.Unknown) + return _symbolStatus; + + // GetSymbolStatus is not implemented for anything other than DbgEng for now. + IDebugClient client = _moduleService._debuggerServices.DebugClient; + if (client is null || client is not IDebugSymbols5 symbols) + return SymbolStatus.Unknown; + + return _symbolStatus = GetSymbolStatusFromDbgEng(symbols); + } #endregion protected override bool TryGetSymbolAddressInner(string name, out ulong address) @@ -181,6 +194,60 @@ namespace SOS.Extensions } protected override ModuleService ModuleService => _moduleService; + + private SymbolStatus GetSymbolStatusFromDbgEng(IDebugSymbols5 symbols) + { + // First, see if the symbol is already loaded. Note that getting the symbol type + // from DbgEng won't force a symbol load, it will only tell us if it's already + // been loaded or not. + DEBUG_SYMTYPE symType = GetSymType(symbols, ImageBase); + if (symType != DEBUG_SYMTYPE.NONE && symType != DEBUG_SYMTYPE.DEFERRED) + return DebugToSymbolStatus(symType); + + // At this point, the symbol type is DEFERRED or NONE and we haven't tried reloading + // the symbol yet. Try a reload, and then ask one last time what the symbol is. + if (!string.IsNullOrWhiteSpace(FileName)) + { + string module = Path.GetFileName(FileName); + module = module.Replace('+', '_'); // Reload doesn't like '+' in module names + HResult hr = symbols.Reload(module); + if (!hr) + { + // Ugh, Reload might not like the module name that GetModuleName gives us. + // Instead, force DbgEng to look up the base address as a symbol which will + // force symbol load as well. + symbols.GetNameByOffset(ImageBase, null, 0, out _, out _); + } + } + + // Whether we successfully reloaded or not, get the final symbol type. + symType = GetSymType(symbols, ImageBase); + return DebugToSymbolStatus(symType); + } + + private static SymbolStatus DebugToSymbolStatus(DEBUG_SYMTYPE symType) + { + // By the time we get here, we've already tried forcing a symbol load. + // If it's NONE or DEFERRED at this point then we can't load it. We + // will never return SymbolStatus.Unknown, so GetSymbolStatusFromDbgEng + // will only ever be called once per module. + return symType switch + { + DEBUG_SYMTYPE.NONE => SymbolStatus.NotLoaded, + DEBUG_SYMTYPE.DEFERRED => SymbolStatus.NotLoaded, + DEBUG_SYMTYPE.EXPORT => SymbolStatus.ExportOnly, + _ => SymbolStatus.Loaded, + }; + } + + private static DEBUG_SYMTYPE GetSymType(IDebugSymbols symbols, ulong imageBase) + { + DEBUG_MODULE_PARAMETERS[] moduleParams = new DEBUG_MODULE_PARAMETERS[1]; + HResult hr = symbols.GetModuleParameters(1, new ulong[] { imageBase }, 0, moduleParams); + + var symType = hr ? moduleParams[0].SymbolType : DEBUG_SYMTYPE.NONE; + return symType; + } } private readonly DebuggerServices _debuggerServices; diff --git a/src/SOS/Strike/dbgengservices.cpp b/src/SOS/Strike/dbgengservices.cpp index 540e1cc9f..51a31c394 100644 --- a/src/SOS/Strike/dbgengservices.cpp +++ b/src/SOS/Strike/dbgengservices.cpp @@ -126,12 +126,16 @@ DbgEngServices::QueryInterface( AddRef(); return S_OK; } - if (InterfaceId == __uuidof(IDebugEventCallbacks)) + else if (InterfaceId == __uuidof(IDebugEventCallbacks)) { *Interface = static_cast(this); AddRef(); return S_OK; } + else if (m_client != nullptr) + { + return m_client->QueryInterface(InterfaceId, Interface); + } else { *Interface = nullptr;