From 14b2ea1cbe00633f5efb1e688fa1a8edfbeda7af Mon Sep 17 00:00:00 2001 From: JP Durot Date: Tue, 15 Dec 2020 00:38:49 +0100 Subject: [PATCH] Add DumpGen command to dotnet-dump (#1816) * Add DumpGen command in dotnet-dump * Fix tests on Windows x86 * Bump ClrMD version to get the fix needed for the previous commit --- eng/Versions.props | 2 +- .../Debuggees/DotnetDumpCommands/Program.cs | 26 +++ src/SOS/SOS.UnitTests/SOS.cs | 13 ++ src/SOS/SOS.UnitTests/Scripts/DumpGen.script | 55 ++++++ .../ExtensionCommands/ClrMDHelper.cs | 65 ++++++- .../dotnet-dump/ExtensionCommands/DumpGen.cs | 59 +++++++ .../ExtensionCommands/DumpGenCommand.cs | 167 ++++++++++++++++++ .../ExtensionCommands/DumpGenStats.cs | 15 ++ .../ExtensionCommands/GCGeneration.cs | 15 ++ 9 files changed, 415 insertions(+), 2 deletions(-) create mode 100644 src/SOS/SOS.UnitTests/Scripts/DumpGen.script create mode 100644 src/Tools/dotnet-dump/ExtensionCommands/DumpGen.cs create mode 100644 src/Tools/dotnet-dump/ExtensionCommands/DumpGenCommand.cs create mode 100644 src/Tools/dotnet-dump/ExtensionCommands/DumpGenStats.cs create mode 100644 src/Tools/dotnet-dump/ExtensionCommands/GCGeneration.cs diff --git a/eng/Versions.props b/eng/Versions.props index 15f9f6520..312f95180 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -43,7 +43,7 @@ 2.1.1 1.1.0 1.1.0 - 2.0.156101 + 2.0.161401 2.0.156101 2.0.1 1.7.0 diff --git a/src/SOS/SOS.UnitTests/Debuggees/DotnetDumpCommands/Program.cs b/src/SOS/SOS.UnitTests/Debuggees/DotnetDumpCommands/Program.cs index 3ac1b9dea..120cddffe 100644 --- a/src/SOS/SOS.UnitTests/Debuggees/DotnetDumpCommands/Program.cs +++ b/src/SOS/SOS.UnitTests/Debuggees/DotnetDumpCommands/Program.cs @@ -39,6 +39,10 @@ namespace DotnetDumpCommands { _objectsToKeepInMemory.AddRange(CreateConcurrentDictionaries()); } + else if ("dumpgen".Equals(args[1])) + { + _objectsToKeepInMemory.AddRange(CreateObjectsInDifferentGenerations()); + } else { Console.WriteLine($"Action parameter {args[1]} is not valid"); @@ -118,6 +122,28 @@ namespace DotnetDumpCommands yield return arrayDictionary; } + private static IEnumerable CreateObjectsInDifferentGenerations() + { + // This object should go into LOH + yield return new DumpSampleClass[50000]; + + for (var i = 0; i < 5; i++) + { + yield return new DumpSampleClass(); + } + GC.Collect(); + + for (var i = 0; i < 3; i++) + { + yield return new DumpSampleClass(); + } + GC.Collect(); + + for (var i = 0; i < 10; i++) + { + yield return new DumpSampleClass(); + } + } public class DumpSampleClass { diff --git a/src/SOS/SOS.UnitTests/SOS.cs b/src/SOS/SOS.UnitTests/SOS.cs index 9b974825d..681fb3f7a 100644 --- a/src/SOS/SOS.UnitTests/SOS.cs +++ b/src/SOS/SOS.UnitTests/SOS.cs @@ -271,6 +271,19 @@ public class SOS }); ; } + [SkippableTheory, MemberData(nameof(GetConfigurations), "TestName", "DotnetDumpCommands")] + public async Task DumpGen(TestConfiguration config) + { + await RunTest("DumpGen.script", testLive: false, information: new SOSRunner.TestInformation + { + TestConfiguration = config, + DebuggeeName = "DotnetDumpCommands", + DebuggeeArguments = "dumpgen", + UsePipeSync = true, + DumpGenerator = SOSRunner.DumpGenerator.DotNetDump, + }); ; + } + [SkippableTheory, MemberData(nameof(Configurations))] public async Task LLDBPluginTests(TestConfiguration config) { diff --git a/src/SOS/SOS.UnitTests/Scripts/DumpGen.script b/src/SOS/SOS.UnitTests/Scripts/DumpGen.script new file mode 100644 index 000000000..0c8617705 --- /dev/null +++ b/src/SOS/SOS.UnitTests/Scripts/DumpGen.script @@ -0,0 +1,55 @@ +# Concurrent dictionaries dump command +# 1) Load the executable +# 2) Run the executable +# 3) Take a dump of the executable before it exits +# 4) Open the dump, find objects in different generations and compare the output + +!IFDEF:CDB +!IFDEF:LLDB +IFDEF:NETCORE_OR_DOTNETDUMP + +COMMAND: dumpgen +VERIFY: Generation argument is missing + +COMMAND: dumpgen invalid +VERIFY: invalid is not a supported generation + +COMMAND: dumpgen gen0 -mt +VERIFY: Required argument missing for option: -mt + +COMMAND: dumpgen gen1 -mt zzzzz +VERIFY: Hexadecimal address expected for -mt option + +COMMAND: dumpgen gen0 +VERIFY: ^\s+MT\s+Count\s+TotalSize\s+Class Name +VERIFY:^\s+10\s+\s+DotnetDumpCommands\.Program\+DumpSampleClass + +COMMAND: dumpgen gen0 -type DotnetDumpCommands +VERIFY: ^\s+10\s+\s+DotnetDumpCommands\.Program\+DumpSampleClass +VERIFY: Total 10 objects + +COMMAND: dumpgen gen0 -mt ^() +VERIFY: ^\s+Address\s+MT\s+Size +VERIFY: Total 10 objects +VERIFY: (\s+\s+){10} + +COMMAND: dumpgen gen0 -mt 00000001 +VERIFY: Total 0 objects + +COMMAND: dumpgen gen0 -type NoMatchingType +VERIFY: Total 0 objects + +COMMAND: dumpgen gen1 +VERIFY: ^\s+3\s+\s+DotnetDumpCommands\.Program\+DumpSampleClass + +COMMAND: dumpgen gen2 +VERIFY: ^\s+5\s+\s+DotnetDumpCommands\.Program\+DumpSampleClass + +COMMAND: dumpgen loh +VERIFY: ^\s+1\s+\s+DotnetDumpCommands\.Program\+DumpSampleClass\[\] + +SOSCOMMAND: dumpheap -stat + +ENDIF:NETCORE_OR_DOTNETDUMP +ENDIF:LLDB +ENDIF:CDB \ No newline at end of file diff --git a/src/Tools/dotnet-dump/ExtensionCommands/ClrMDHelper.cs b/src/Tools/dotnet-dump/ExtensionCommands/ClrMDHelper.cs index 728d0ebe0..786a73459 100644 --- a/src/Tools/dotnet-dump/ExtensionCommands/ClrMDHelper.cs +++ b/src/Tools/dotnet-dump/ExtensionCommands/ClrMDHelper.cs @@ -692,6 +692,65 @@ namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands } } + public IEnumerable EnumerateObjectsInGeneration(GCGeneration generation) + { + foreach (var segment in _heap.Segments) + { + if (!TryGetSegmentMemoryRange(segment, generation, out var start, out var end)) + continue; + + var currentObjectAddress = start; + ClrObject currentObject; + do + { + currentObject = _heap.GetObject(currentObjectAddress); + if (currentObject.Type != null) + yield return currentObject; + + currentObjectAddress = segment.GetNextObjectAddress(currentObject); + } while (currentObjectAddress > 0 && currentObjectAddress < end); + } + } + + private bool TryGetSegmentMemoryRange(ClrSegment segment, GCGeneration generation, out ulong start, out ulong end) + { + start = 0; + end = 0; + switch (generation) + { + case GCGeneration.Generation0: + if (segment.IsEphemeralSegment) + { + start = segment.Generation0.Start; + end = segment.Generation0.End; + } + return start != end; + case GCGeneration.Generation1: + if (segment.IsEphemeralSegment) + { + start = segment.Generation1.Start; + end = segment.Generation1.End; + } + return start != end; + case GCGeneration.Generation2: + if (!segment.IsLargeObjectSegment) + { + start = segment.Generation2.Start; + end = segment.Generation2.End; + } + return start != end; + case GCGeneration.LargeObjectHeap: + if (segment.IsLargeObjectSegment) + { + start = segment.Start; + end = segment.End; + } + return start != end; + default: + return false; + } + } + public IEnumerable EnumerateConcurrentQueue(ulong address) { return IsNetCore() ? EnumerateConcurrentQueueCore(address) : EnumerateConcurrentQueueFramework(address); @@ -1061,7 +1120,6 @@ namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands return true; } - public ClrModule GetMscorlib() { var bclModule = _clr.BaseClassLibrary; @@ -1076,5 +1134,10 @@ namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands return (coreLib.Name.ToLower().Contains("corelib")); } + + public bool Is64Bits() + { + return _clr.DataTarget.DataReader.PointerSize == 8; + } } } diff --git a/src/Tools/dotnet-dump/ExtensionCommands/DumpGen.cs b/src/Tools/dotnet-dump/ExtensionCommands/DumpGen.cs new file mode 100644 index 000000000..81eb74b34 --- /dev/null +++ b/src/Tools/dotnet-dump/ExtensionCommands/DumpGen.cs @@ -0,0 +1,59 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Diagnostics.Runtime; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands +{ + + public class DumpGen + { + private readonly ClrMDHelper _helper; + private readonly GCGeneration _generation; + + public DumpGen(ClrMDHelper helper, GCGeneration generation) + { + _helper = helper; + _generation = generation; + } + + public IEnumerable GetStats(string typeNameFilter) + { + var types = new Dictionary(); + + foreach (var obj in _helper.EnumerateObjectsInGeneration(_generation) + .Where(obj => typeNameFilter == null || IsTypeNameMatching(obj.Type.Name, typeNameFilter))) + { + var objectType = obj.Type; + if (types.TryGetValue(objectType, out var type)) + { + type.NumberOfOccurences++; + type.TotalSize += obj.Size; + } + else + { + types.Add(objectType, new DumpGenStats { Type = objectType, NumberOfOccurences = 1, TotalSize = obj.Size }); + } + } + return types.Values.OrderBy(v => v.TotalSize); + } + + public IEnumerable GetInstances(ulong methodTableAddress) + { + return _helper.EnumerateObjectsInGeneration(_generation) + .Where(obj => obj.Type.MethodTable == methodTableAddress); + } + + + private static bool IsTypeNameMatching(string typeName, string typeNameFilter) + { + return typeName.Contains(typeNameFilter, StringComparison.OrdinalIgnoreCase); + } + + } + +} diff --git a/src/Tools/dotnet-dump/ExtensionCommands/DumpGenCommand.cs b/src/Tools/dotnet-dump/ExtensionCommands/DumpGenCommand.cs new file mode 100644 index 000000000..54d2d7a66 --- /dev/null +++ b/src/Tools/dotnet-dump/ExtensionCommands/DumpGenCommand.cs @@ -0,0 +1,167 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Diagnostic.Tools.Dump.ExtensionCommands; +using Microsoft.Diagnostics.Repl; +using Microsoft.Diagnostics.Runtime; +using System; +using System.Collections.Generic; +using System.CommandLine; + +namespace Microsoft.Diagnostics.Tools.Dump.ExtensionCommands +{ + [Command(Name = "dumpgen", Help = "Displays heap content for the specified generation.")] + [CommandAlias(Name = "dg")] + public class DumpGenCommand : ExtensionCommandBase + { + private const string statsHeader32bits = " MT Count TotalSize Class Name"; + private const string statsHeader64bits = " MT Count TotalSize Class Name"; + private const string methodTableHeader32bits = " Address MT Size"; + private const string methodTableHeader64bits = " Address MT Size"; + + [Argument(Name = "generation", Help = "The GC generation to get heap data from.")] + public string Generation { get; set; } + + [Option(Name = "-type", Help = "List only those objects whose type name is a substring match of the provided string.")] + public string FilterByTypeName { get; set; } + + [Option(Name = "-mt", Help = "The address pointing on a Method table.")] + public string MethodTableAddress { get; set; } + + public override void Invoke() + { + var generation = ParseGenerationArgument(Generation); + if (generation != GCGeneration.NotSet) + { + var dumpGen = new DumpGen(Helper, generation); + + if (string.IsNullOrEmpty(MethodTableAddress)) + { + var dumpGenResult = dumpGen.GetStats(FilterByTypeName); + WriteStatistics(dumpGenResult); + } + else if (TryParseAddress(MethodTableAddress, out var address)) + { + var objects = dumpGen.GetInstances(address); + WriteInstances(objects); + } + else + { + WriteLine("Hexadecimal address expected for -mt option"); + } + } + WriteLine(string.Empty); + } + + private void WriteInstances(IEnumerable objects) + { + var objectsCount = 0UL; + WriteLine(Helper.Is64Bits() ? methodTableHeader64bits : methodTableHeader32bits); + foreach (var obj in objects) + { + objectsCount++; + if (Helper.Is64Bits()) + { + WriteLine($"{obj.Address:x16} {obj.Type.MethodTable:x16} {obj.Size,8}"); + } + else + { + WriteLine($"{obj.Address:x8} {obj.Type.MethodTable:x8} {obj.Size,8}"); + } + } + WriteLine($"Total {objectsCount} objects"); + } + + private void WriteStatistics(IEnumerable dumpGenResult) + { + var objectsCount = 0UL; + WriteLine("Statistics:"); + WriteLine(Helper.Is64Bits() ? statsHeader64bits : statsHeader32bits); + foreach (var typeStats in dumpGenResult) + { + objectsCount += typeStats.NumberOfOccurences; + if (Helper.Is64Bits()) + { + WriteLine($"{typeStats.Type.MethodTable:x16} {typeStats.NumberOfOccurences,8} {typeStats.TotalSize,12} {typeStats.Type.Name}"); + } + else + { + WriteLine($"{typeStats.Type.MethodTable:x8} {typeStats.NumberOfOccurences,8} {typeStats.TotalSize,12} {typeStats.Type.Name}"); + } + } + WriteLine($"Total {objectsCount} objects"); + } + + private GCGeneration ParseGenerationArgument(string generation) + { + if (string.IsNullOrEmpty(generation)) + { + WriteLine("Generation argument is missing"); + return GCGeneration.NotSet; + } + var lowerString = generation.ToLowerInvariant(); + switch (lowerString) + { + case "gen0": + return GCGeneration.Generation0; + case "gen1": + return GCGeneration.Generation1; + case "gen2": + return GCGeneration.Generation2; + case "loh": + return GCGeneration.LargeObjectHeap; + default: + WriteLine($"{generation} is not a supported generation (gen0, gen1, gen2, loh)"); + return GCGeneration.NotSet; + } + } + + + protected override string GetDetailedHelp() + { + return +@"------------------------------------------------------------------------------- +DumpGen +This command can be used for 2 use cases: +- Lists number of objects and total size for every objects on the heap, for a specified generation + Acts like the 'dumpheap -stat' command for a specified generation and return data in the same format + +- Lists object addresses corresponding to the method table passed in parameter (by providing the '-mt' option), for a specified generation + Acts like the 'dumpheap -mt' command for a specified generation and return data in the same format + +Generation number can take the following values (case insensitive): +- gen0 +- gen1 +- gen2 +- loh + +> dumpgen gen0 +Statistics: + MT Count TotalSize Class Name +00007ff9ea6601c8 1 24 System.Collections.Generic.GenericEqualityComparer +00007ff9ea660338 1 24 System.Collections.Generic.NonRandomizedStringEqualityComparer +... +00007ff9ea69b268 7 33612 System.Char[] +00007ff9ea651e18 204 41154 System.String +Total 651 objects + +As the original dumpheap command, we can pass an additional '-type' parameter to filter out on type name +> dumpgen gen2 -type Object +Statistics: + MT Count TotalSize Class Name +00007ff9ea590af0 26 624 System.Object +00007ff9ea66f4e0 3 720 System.Collections.Generic.Dictionary+Entry[] +00007ff9ea596618 17 2080 System.Object[] +Total 46 objects + +> dumpgen gen0 -mt 00007ff9ea6e75b8 + Address MT Size +00000184aa23e8a0 00007ff9ea6e75b8 40 +00000184aa23e8f0 00007ff9ea6e75b8 40 +00000184aa23e918 00007ff9ea6e75b8 40 +Total 3 objects +"; + } + } +} diff --git a/src/Tools/dotnet-dump/ExtensionCommands/DumpGenStats.cs b/src/Tools/dotnet-dump/ExtensionCommands/DumpGenStats.cs new file mode 100644 index 000000000..f52030231 --- /dev/null +++ b/src/Tools/dotnet-dump/ExtensionCommands/DumpGenStats.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Diagnostics.Runtime; + +namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands +{ + public class DumpGenStats + { + public ClrType Type { get; set; } + public ulong NumberOfOccurences { get; set; } + public ulong TotalSize { get; set; } + } +} diff --git a/src/Tools/dotnet-dump/ExtensionCommands/GCGeneration.cs b/src/Tools/dotnet-dump/ExtensionCommands/GCGeneration.cs new file mode 100644 index 000000000..dc37fc9aa --- /dev/null +++ b/src/Tools/dotnet-dump/ExtensionCommands/GCGeneration.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands +{ + public enum GCGeneration + { + NotSet = 0, + Generation0 = 1, + Generation1 = 2, + Generation2 = 3, + LargeObjectHeap = 4 + } +} -- 2.34.1