Add DumpGen command to dotnet-dump (#1816)
authorJP Durot <jp.durot@criteo.com>
Mon, 14 Dec 2020 23:38:49 +0000 (00:38 +0100)
committerGitHub <noreply@github.com>
Mon, 14 Dec 2020 23:38:49 +0000 (15:38 -0800)
* Add DumpGen command in dotnet-dump

* Fix tests on Windows x86

* Bump ClrMD version to get the fix needed for the previous commit

eng/Versions.props
src/SOS/SOS.UnitTests/Debuggees/DotnetDumpCommands/Program.cs
src/SOS/SOS.UnitTests/SOS.cs
src/SOS/SOS.UnitTests/Scripts/DumpGen.script [new file with mode: 0644]
src/Tools/dotnet-dump/ExtensionCommands/ClrMDHelper.cs
src/Tools/dotnet-dump/ExtensionCommands/DumpGen.cs [new file with mode: 0644]
src/Tools/dotnet-dump/ExtensionCommands/DumpGenCommand.cs [new file with mode: 0644]
src/Tools/dotnet-dump/ExtensionCommands/DumpGenStats.cs [new file with mode: 0644]
src/Tools/dotnet-dump/ExtensionCommands/GCGeneration.cs [new file with mode: 0644]

index 15f9f6520d1abfdc24fb972bc6b108cec5ee1bf4..312f951802a0fd07ba52c0ef36d9ee02745681df 100644 (file)
@@ -43,7 +43,7 @@
     <MicrosoftAspNetCoreResponseCompressionVersion>2.1.1</MicrosoftAspNetCoreResponseCompressionVersion>
     <MicrosoftBclHashCodeVersion>1.1.0</MicrosoftBclHashCodeVersion>
     <MicrosoftBclAsyncInterfacesVersion>1.1.0</MicrosoftBclAsyncInterfacesVersion>
-    <MicrosoftDiagnosticsRuntimeVersion>2.0.156101</MicrosoftDiagnosticsRuntimeVersion>
+    <MicrosoftDiagnosticsRuntimeVersion>2.0.161401</MicrosoftDiagnosticsRuntimeVersion>
     <MicrosoftDiagnosticsRuntimeUtilitiesVersion>2.0.156101</MicrosoftDiagnosticsRuntimeUtilitiesVersion>
     <ParallelStacksRuntimeVersion>2.0.1</ParallelStacksRuntimeVersion>
     <MicrosoftDiaSymReaderNativePackageVersion>1.7.0</MicrosoftDiaSymReaderNativePackageVersion>
index 3ac1b9dea79f1f31cdc7394553257b8780d3fbe1..120cddffe1f71e3df2d212a3853afe38327d804d 100644 (file)
@@ -39,6 +39,10 @@ namespace DotnetDumpCommands
             {
                 _objectsToKeepInMemory.AddRange(CreateConcurrentDictionaries());
             }
+            else if ("dumpgen".Equals(args[1]))
+            {
+                _objectsToKeepInMemory.AddRange(CreateObjectsInDifferentGenerations());
+            }
             else
             {
                 Console.WriteLine($"Action parameter {args[1]} is not valid");
@@ -118,6 +122,28 @@ namespace DotnetDumpCommands
             yield return arrayDictionary;
         }
 
+        private static IEnumerable<object> CreateObjectsInDifferentGenerations()
+        {
+            // This object should go into LOH
+            yield return new DumpSampleClass[50000];
+
+            for (var i = 0; i < 5; i++)
+            {
+                yield return new DumpSampleClass();
+            }
+            GC.Collect();
+
+            for (var i = 0; i < 3; i++)
+            {
+                yield return new DumpSampleClass();
+            }
+            GC.Collect();
+
+            for (var i = 0; i < 10; i++)
+            {
+                yield return new DumpSampleClass();
+            }
+        }
 
         public class DumpSampleClass
         {
index 9b974825d760f57fceec6fc495d275880a47ed28..681fb3f7ac68c00b0fd730bba02eb01df7c441ab 100644 (file)
@@ -271,6 +271,19 @@ public class SOS
         }); ;
     }
 
+    [SkippableTheory, MemberData(nameof(GetConfigurations), "TestName", "DotnetDumpCommands")]
+    public async Task DumpGen(TestConfiguration config)
+    {
+        await RunTest("DumpGen.script", testLive: false, information: new SOSRunner.TestInformation
+        {
+            TestConfiguration = config,
+            DebuggeeName = "DotnetDumpCommands",
+            DebuggeeArguments = "dumpgen",
+            UsePipeSync = true,
+            DumpGenerator = SOSRunner.DumpGenerator.DotNetDump,
+        }); ;
+    }
+
     [SkippableTheory, MemberData(nameof(Configurations))]
     public async Task LLDBPluginTests(TestConfiguration config)
     {
diff --git a/src/SOS/SOS.UnitTests/Scripts/DumpGen.script b/src/SOS/SOS.UnitTests/Scripts/DumpGen.script
new file mode 100644 (file)
index 0000000..0c86177
--- /dev/null
@@ -0,0 +1,55 @@
+# Concurrent dictionaries dump command
+# 1) Load the executable
+# 2) Run the executable
+# 3) Take a dump of the executable before it exits
+# 4) Open the dump, find objects in different generations and compare the output
+
+!IFDEF:CDB
+!IFDEF:LLDB
+IFDEF:NETCORE_OR_DOTNETDUMP
+
+COMMAND: dumpgen
+VERIFY: Generation argument is missing
+
+COMMAND: dumpgen invalid
+VERIFY: invalid is not a supported generation
+
+COMMAND: dumpgen gen0 -mt
+VERIFY: Required argument missing for option: -mt
+
+COMMAND: dumpgen gen1 -mt zzzzz
+VERIFY: Hexadecimal address expected for -mt option
+
+COMMAND: dumpgen gen0
+VERIFY: ^\s+MT\s+Count\s+TotalSize\s+Class Name
+VERIFY:^<HEXVAL>\s+10\s+<DECVAL>\s+DotnetDumpCommands\.Program\+DumpSampleClass
+
+COMMAND: dumpgen gen0 -type DotnetDumpCommands
+VERIFY: ^<HEXVAL>\s+10\s+<DECVAL>\s+DotnetDumpCommands\.Program\+DumpSampleClass
+VERIFY: Total 10 objects
+
+COMMAND: dumpgen gen0 -mt <POUT>^(<HEXVAL>)<POUT>
+VERIFY: ^\s+Address\s+MT\s+Size
+VERIFY: Total 10 objects
+VERIFY: (<HEXVAL>\s+<HEXVAL>\s+<DECVAL>){10}
+
+COMMAND: dumpgen gen0 -mt 00000001
+VERIFY: Total 0 objects
+
+COMMAND: dumpgen gen0 -type NoMatchingType
+VERIFY: Total 0 objects
+
+COMMAND: dumpgen gen1
+VERIFY: ^<HEXVAL>\s+3\s+<DECVAL>\s+DotnetDumpCommands\.Program\+DumpSampleClass
+
+COMMAND: dumpgen gen2
+VERIFY: ^<HEXVAL>\s+5\s+<DECVAL>\s+DotnetDumpCommands\.Program\+DumpSampleClass
+
+COMMAND: dumpgen loh
+VERIFY: ^<HEXVAL>\s+1\s+<DECVAL>\s+DotnetDumpCommands\.Program\+DumpSampleClass\[\]
+
+SOSCOMMAND: dumpheap -stat
+
+ENDIF:NETCORE_OR_DOTNETDUMP
+ENDIF:LLDB
+ENDIF:CDB
\ No newline at end of file
index 728d0ebe08c07569e611b248583aa2e49d338267..786a734597f069c49d7eb96dc47b3947b8bc59a7 100644 (file)
@@ -692,6 +692,65 @@ namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands
             }
         }
 
+        public IEnumerable<ClrObject> EnumerateObjectsInGeneration(GCGeneration generation)
+        {
+            foreach (var segment in _heap.Segments)
+            {
+                if (!TryGetSegmentMemoryRange(segment, generation, out var start, out var end))
+                    continue;
+
+                var currentObjectAddress = start;
+                ClrObject currentObject;
+                do
+                {
+                    currentObject = _heap.GetObject(currentObjectAddress);
+                    if (currentObject.Type != null)
+                        yield return currentObject;
+
+                    currentObjectAddress = segment.GetNextObjectAddress(currentObject);
+                } while (currentObjectAddress > 0 && currentObjectAddress < end);
+            }
+        }
+
+        private bool TryGetSegmentMemoryRange(ClrSegment segment, GCGeneration generation, out ulong start, out ulong end)
+        {
+            start = 0;
+            end = 0;
+            switch (generation)
+            {
+                case GCGeneration.Generation0:
+                    if (segment.IsEphemeralSegment)
+                    {
+                        start = segment.Generation0.Start;
+                        end = segment.Generation0.End;
+                    }
+                    return start != end;
+                case GCGeneration.Generation1:
+                    if (segment.IsEphemeralSegment)
+                    {
+                        start = segment.Generation1.Start;
+                        end = segment.Generation1.End;
+                    }
+                    return start != end;
+                case GCGeneration.Generation2:
+                    if (!segment.IsLargeObjectSegment)
+                    {
+                        start = segment.Generation2.Start;
+                        end = segment.Generation2.End;
+                    }
+                    return start != end;
+                case GCGeneration.LargeObjectHeap:
+                    if (segment.IsLargeObjectSegment)
+                    {
+                        start = segment.Start;
+                        end = segment.End;
+                    }
+                    return start != end;
+                default:
+                    return false;
+            }
+        }
+
         public IEnumerable<string> EnumerateConcurrentQueue(ulong address)
         {
             return IsNetCore() ? EnumerateConcurrentQueueCore(address) : EnumerateConcurrentQueueFramework(address);
@@ -1061,7 +1120,6 @@ namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands
             return true;
         }
 
-
         public ClrModule GetMscorlib()
         {
             var bclModule = _clr.BaseClassLibrary;
@@ -1076,5 +1134,10 @@ namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands
 
             return (coreLib.Name.ToLower().Contains("corelib"));
         }
+
+        public bool Is64Bits()
+        {
+            return _clr.DataTarget.DataReader.PointerSize == 8;
+        }
     }
 }
diff --git a/src/Tools/dotnet-dump/ExtensionCommands/DumpGen.cs b/src/Tools/dotnet-dump/ExtensionCommands/DumpGen.cs
new file mode 100644 (file)
index 0000000..81eb74b
--- /dev/null
@@ -0,0 +1,59 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Diagnostics.Runtime;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands
+{
+
+    public class DumpGen
+    {
+        private readonly ClrMDHelper _helper;
+        private readonly GCGeneration _generation;
+
+        public DumpGen(ClrMDHelper helper, GCGeneration generation)
+        {
+            _helper = helper;
+            _generation = generation;
+        }
+
+        public IEnumerable<DumpGenStats> GetStats(string typeNameFilter)
+        {
+            var types = new Dictionary<ClrType, DumpGenStats>();
+
+            foreach (var obj in _helper.EnumerateObjectsInGeneration(_generation)
+                .Where(obj => typeNameFilter == null || IsTypeNameMatching(obj.Type.Name, typeNameFilter)))
+            {
+                var objectType = obj.Type;
+                if (types.TryGetValue(objectType, out var type))
+                {
+                    type.NumberOfOccurences++;
+                    type.TotalSize += obj.Size;
+                }
+                else
+                {
+                    types.Add(objectType, new DumpGenStats { Type = objectType, NumberOfOccurences = 1, TotalSize = obj.Size });
+                }
+            }
+            return types.Values.OrderBy(v => v.TotalSize);
+        }
+
+        public IEnumerable<ClrObject> GetInstances(ulong methodTableAddress)
+        {
+            return _helper.EnumerateObjectsInGeneration(_generation)
+                .Where(obj => obj.Type.MethodTable == methodTableAddress);
+        }
+
+
+        private static bool IsTypeNameMatching(string typeName, string typeNameFilter)
+        {
+            return typeName.Contains(typeNameFilter, StringComparison.OrdinalIgnoreCase);
+        }
+
+    }
+
+}
diff --git a/src/Tools/dotnet-dump/ExtensionCommands/DumpGenCommand.cs b/src/Tools/dotnet-dump/ExtensionCommands/DumpGenCommand.cs
new file mode 100644 (file)
index 0000000..54d2d7a
--- /dev/null
@@ -0,0 +1,167 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Diagnostic.Tools.Dump.ExtensionCommands;
+using Microsoft.Diagnostics.Repl;
+using Microsoft.Diagnostics.Runtime;
+using System;
+using System.Collections.Generic;
+using System.CommandLine;
+
+namespace Microsoft.Diagnostics.Tools.Dump.ExtensionCommands
+{
+    [Command(Name = "dumpgen", Help = "Displays heap content for the specified generation.")]
+    [CommandAlias(Name = "dg")]
+    public class DumpGenCommand : ExtensionCommandBase
+    {
+        private const string statsHeader32bits = "      MT    Count    TotalSize Class Name";
+        private const string statsHeader64bits = "              MT    Count    TotalSize Class Name";
+        private const string methodTableHeader32bits = " Address       MT     Size";
+        private const string methodTableHeader64bits = "         Address               MT     Size";
+
+        [Argument(Name = "generation", Help = "The GC generation to get heap data from.")]
+        public string Generation { get; set; }
+
+        [Option(Name = "-type", Help = "List only those objects whose type name is a substring match of the provided string.")]
+        public string FilterByTypeName { get; set; }
+
+        [Option(Name = "-mt", Help = "The address pointing on a Method table.")]
+        public string MethodTableAddress { get; set; }
+
+        public override void Invoke()
+        {
+            var generation = ParseGenerationArgument(Generation);
+            if (generation != GCGeneration.NotSet)
+            {
+                var dumpGen = new DumpGen(Helper, generation);
+
+                if (string.IsNullOrEmpty(MethodTableAddress))
+                {
+                    var dumpGenResult = dumpGen.GetStats(FilterByTypeName);
+                    WriteStatistics(dumpGenResult);
+                }
+                else if (TryParseAddress(MethodTableAddress, out var address))
+                {
+                    var objects = dumpGen.GetInstances(address);
+                    WriteInstances(objects);
+                }
+                else
+                {
+                    WriteLine("Hexadecimal address expected for -mt option");
+                }
+            }
+            WriteLine(string.Empty);
+        }
+
+        private void WriteInstances(IEnumerable<ClrObject> objects)
+        {
+            var objectsCount = 0UL;
+            WriteLine(Helper.Is64Bits() ? methodTableHeader64bits : methodTableHeader32bits);
+            foreach (var obj in objects)
+            {
+                objectsCount++;
+                if (Helper.Is64Bits())
+                {
+                    WriteLine($"{obj.Address:x16} {obj.Type.MethodTable:x16} {obj.Size,8}");
+                }
+                else
+                {
+                    WriteLine($"{obj.Address:x8} {obj.Type.MethodTable:x8} {obj.Size,8}");
+                }
+            }
+            WriteLine($"Total {objectsCount} objects");
+        }
+
+        private void WriteStatistics(IEnumerable<DumpGenStats> dumpGenResult)
+        {
+            var objectsCount = 0UL;
+            WriteLine("Statistics:");
+            WriteLine(Helper.Is64Bits() ? statsHeader64bits : statsHeader32bits);
+            foreach (var typeStats in dumpGenResult)
+            {
+                objectsCount += typeStats.NumberOfOccurences;
+                if (Helper.Is64Bits())
+                {
+                    WriteLine($"{typeStats.Type.MethodTable:x16} {typeStats.NumberOfOccurences,8} {typeStats.TotalSize,12} {typeStats.Type.Name}");
+                }
+                else
+                {
+                    WriteLine($"{typeStats.Type.MethodTable:x8} {typeStats.NumberOfOccurences,8} {typeStats.TotalSize,12} {typeStats.Type.Name}");
+                }
+            }
+            WriteLine($"Total {objectsCount} objects");
+        }
+
+        private GCGeneration ParseGenerationArgument(string generation)
+        {
+            if (string.IsNullOrEmpty(generation))
+            {
+                WriteLine("Generation argument is missing");
+                return GCGeneration.NotSet;
+            }
+            var lowerString = generation.ToLowerInvariant();
+            switch (lowerString)
+            {
+                case "gen0":
+                    return GCGeneration.Generation0;
+                case "gen1":
+                    return GCGeneration.Generation1;
+                case "gen2":
+                    return GCGeneration.Generation2;
+                case "loh":
+                    return GCGeneration.LargeObjectHeap;
+                default:
+                    WriteLine($"{generation} is not a supported generation (gen0, gen1, gen2, loh)");
+                    return GCGeneration.NotSet;
+            }
+        }
+
+
+        protected override string GetDetailedHelp()
+        {
+            return
+@"-------------------------------------------------------------------------------
+DumpGen
+This command can be used for 2 use cases:
+- Lists number of objects and total size for every objects on the heap, for a specified generation
+  Acts like the 'dumpheap -stat' command for a specified generation and return data in the same format
+
+- Lists object addresses corresponding to the method table passed in parameter (by providing the '-mt' option), for a specified generation
+  Acts like the 'dumpheap -mt' command for a specified generation and return data in the same format
+
+Generation number can take the following values (case insensitive):
+- gen0
+- gen1
+- gen2
+- loh
+
+> dumpgen gen0
+Statistics:
+              MT    Count    TotalSize Class Name
+00007ff9ea6601c8        1           24 System.Collections.Generic.GenericEqualityComparer<System.String>
+00007ff9ea660338        1           24 System.Collections.Generic.NonRandomizedStringEqualityComparer
+...
+00007ff9ea69b268        7        33612 System.Char[]
+00007ff9ea651e18      204        41154 System.String
+Total 651 objects
+
+As the original dumpheap command, we can pass an additional '-type' parameter to filter out on type name
+> dumpgen gen2 -type Object
+Statistics:
+              MT    Count    TotalSize Class Name
+00007ff9ea590af0       26          624 System.Object
+00007ff9ea66f4e0        3          720 System.Collections.Generic.Dictionary<System.String, System.Object>+Entry[]
+00007ff9ea596618       17         2080 System.Object[]
+Total 46 objects
+
+> dumpgen gen0 -mt 00007ff9ea6e75b8
+         Address               MT     Size
+00000184aa23e8a0 00007ff9ea6e75b8       40
+00000184aa23e8f0 00007ff9ea6e75b8       40
+00000184aa23e918 00007ff9ea6e75b8       40
+Total 3 objects
+";
+        }
+    }
+}
diff --git a/src/Tools/dotnet-dump/ExtensionCommands/DumpGenStats.cs b/src/Tools/dotnet-dump/ExtensionCommands/DumpGenStats.cs
new file mode 100644 (file)
index 0000000..f520302
--- /dev/null
@@ -0,0 +1,15 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Diagnostics.Runtime;
+
+namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands
+{
+    public class DumpGenStats
+    {
+        public ClrType Type { get; set; }
+        public ulong NumberOfOccurences { get; set; }
+        public ulong TotalSize { get; set; }
+    }
+}
diff --git a/src/Tools/dotnet-dump/ExtensionCommands/GCGeneration.cs b/src/Tools/dotnet-dump/ExtensionCommands/GCGeneration.cs
new file mode 100644 (file)
index 0000000..dc37fc9
--- /dev/null
@@ -0,0 +1,15 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace Microsoft.Diagnostic.Tools.Dump.ExtensionCommands
+{
+    public enum GCGeneration
+    {
+        NotSet = 0,
+        Generation0 = 1,
+        Generation1 = 2,
+        Generation2 = 3,
+        LargeObjectHeap = 4
+    }
+}