From 362c6545ce6e3c53a499ffac1dfff449212a3b5a Mon Sep 17 00:00:00 2001 From: Lee Culver Date: Thu, 30 Mar 2023 10:44:47 -0700 Subject: [PATCH] Add GC inter-generational analysis commands (#3790) * Add GC inter-generational analysis commands Added SOS commands requested by the GC team: !sos FindEphemeralReferencesToLOH - Finds references between gen0/gen1 objects and the Large Object Heap. !sos FindReferencesToEphemeral Finds all object references which point to a lower generation than the current object !sos SizeStats - Prints statistics about objects sizes * Code review feedback - shorten command names - check for cancellation - change a couple of object checks to filter better --- .../ExtensionMethodHelpers.cs | 35 ++++ .../FindEphemeralReferencesToLOHCommand.cs | 187 ++++++++++++++++++ .../FindReferencesToEphemeralCommand.cs | 174 ++++++++++++++++ .../Generation.cs | 16 ++ .../SizeStatsCommand.cs | 118 +++++++++++ 5 files changed, 530 insertions(+) create mode 100644 src/Microsoft.Diagnostics.ExtensionCommands/FindEphemeralReferencesToLOHCommand.cs create mode 100644 src/Microsoft.Diagnostics.ExtensionCommands/FindReferencesToEphemeralCommand.cs create mode 100644 src/Microsoft.Diagnostics.ExtensionCommands/Generation.cs create mode 100644 src/Microsoft.Diagnostics.ExtensionCommands/SizeStatsCommand.cs diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs b/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs index 7f67c6de7..15f81d59f 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs +++ b/src/Microsoft.Diagnostics.ExtensionCommands/ExtensionMethodHelpers.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; +using Microsoft.Diagnostics.Runtime; namespace Microsoft.Diagnostics.ExtensionCommands { @@ -36,5 +37,39 @@ namespace Microsoft.Diagnostics.ExtensionCommands group ptr by ptr into g orderby g.Count() descending select g.First()).First(); + + internal static Generation GetGeneration(this ClrObject obj, ClrSegment knownSegment) + { + if (knownSegment is null) + { + knownSegment = obj.Type.Heap.GetSegmentByAddress(obj); + if (knownSegment is null) + { + return Generation.Error; + } + } + + if (knownSegment.Kind == GCSegmentKind.Ephemeral) + { + return knownSegment.GetGeneration(obj) switch + { + 0 => Generation.Gen0, + 1 => Generation.Gen1, + 2 => Generation.Gen2, + _ => Generation.Error + }; + } + + return knownSegment.Kind switch + { + GCSegmentKind.Generation0 => Generation.Gen0, + GCSegmentKind.Generation1 => Generation.Gen1, + GCSegmentKind.Generation2 => Generation.Gen2, + GCSegmentKind.Large => Generation.Large, + GCSegmentKind.Pinned => Generation.Pinned, + GCSegmentKind.Frozen => Generation.Frozen, + _ => Generation.Error + }; + } } } diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/FindEphemeralReferencesToLOHCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/FindEphemeralReferencesToLOHCommand.cs new file mode 100644 index 000000000..d28166bad --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/FindEphemeralReferencesToLOHCommand.cs @@ -0,0 +1,187 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.Runtime; +using static Microsoft.Diagnostics.ExtensionCommands.TableOutput; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "ephtoloh", Help = "Finds ephemeral objects which reference the large object heap.")] + public class FindEphemeralReferencesToLOHCommand : CommandBase + { + // IComparer for binary search + private readonly IComparer<(ClrObject, ClrObject)> _firstObjectComparer = Comparer<(ClrObject, ClrObject)>.Create((x, y) => x.Item1.Address.CompareTo(y.Item1.Address)); + + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + public override void Invoke() + { + int segments = Runtime.Heap.Segments.Count(seg => seg.Kind is not GCSegmentKind.Frozen or GCSegmentKind.Pinned); + if (segments > 50) + { + string gcSegKind = Runtime.Heap.SubHeaps[0].HasRegions ? "regions" : "segments"; + Console.WriteLineWarning($"Walking {segments:n0} {gcSegKind}, this may take a moment..."); + } + + TableOutput output = new(Console, (16, "x12"), (64, ""), (16, "x12")); + + // Ephemeral -> Large + List<(ClrObject From, ClrObject To)> ephToLoh = FindEphemeralToLOH().OrderBy(i => i.From.Address).ThenBy(i => i.To.Address).ToList(); + if (ephToLoh.Count == 0) + { + Console.WriteLine("No Ephemeral objects pointing to the Large objects."); + } + else + { + Console.WriteLine("Ephemeral objects pointing to the Large objects:"); + Console.WriteLine(); + output.WriteRow("Ephemeral", "Ephemeral Type", "Large Object", "Large Object Type"); + + foreach ((ClrObject from, ClrObject to) in ephToLoh) + { + output.WriteRow(new DmlDumpObj(from), from.Type?.Name, new DmlDumpObj(to), to.Type?.Name); + } + + Console.WriteLine(); + } + + // Large -> Ephemeral + List<(ClrObject From, ClrObject To)> lohToEph = FindLOHToEphemeral().OrderBy(i => i.From.Address).ThenBy(i => i.To.Address).ToList(); + if (lohToEph.Count == 0) + { + Console.WriteLine("No Large objects pointing to Ephemeral objects."); + } + else + { + Console.WriteLine("Large objects pointing to Ephemeral objects:"); + Console.WriteLine(); + output.WriteRow("Ephemeral", "Ephemeral Type", "Large Object", "Large Object Type"); + + foreach ((ClrObject from, ClrObject to) in lohToEph) + { + output.WriteRow(new DmlDumpObj(from), from.Type?.Name, new DmlDumpObj(to), to.Type?.Name); + } + + Console.WriteLine(); + } + + // Ephemeral -> Large -> Ephemeral + if (ephToLoh.Count != 0 && lohToEph.Count != 0) + { + // We'll use output to signify if we need to print a header or not. + output = null; + foreach ((ClrObject from, ClrObject to) in ephToLoh) + { + int index = lohToEph.BinarySearch((to, to), _firstObjectComparer); + if (index < 0) + { + continue; + } + + Debug.Assert(to == lohToEph[index].From); + ClrObject ephEnd = lohToEph[index].To; + + if (output is null) + { + Console.WriteLine($"Ephemeral objects which point to Large objects which point to Ephemeral objects:"); + Console.WriteLine(); + output = new(Console, (16, "x12"), (64, ""), (16, "x12"), (64, ""), (16, "x12")); + output.WriteRow(new DmlDumpObj(from), from.Type?.Name, new DmlDumpObj(to), to.Type?.Name, new DmlDumpObj(ephEnd), ephEnd.Type?.Name); + } + } + + if (output is null) + { + Console.WriteLine("No Ephemeral objects which point to Large objects which point to Ephemeral objects."); + } + else + { + Console.WriteLine(); + } + } + + foreach ((ClrObject From, ClrObject To) item in ephToLoh) + { + if (lohToEph.Any(r => item.To.Address == r.From.Address)) + { + Console.WriteLine("error!"); + } + } + } + + private IEnumerable<(ClrObject From, ClrObject To)> FindEphemeralToLOH() + { + foreach (ClrSegment seg in Runtime.Heap.Segments) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (seg.Kind is GCSegmentKind.Frozen or GCSegmentKind.Large or GCSegmentKind.Generation2 or GCSegmentKind.Pinned) + { + continue; + } + + foreach (ClrObject obj in seg.EnumerateObjects().Where(obj => obj.IsValid && obj.ContainsPointers)) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + // This handles both regions and segments + Generation gen = obj.GetGeneration(seg); + if (gen is not Generation.Gen0 or Generation.Gen1) + { + continue; + } + + foreach (ClrObject objRef in obj.EnumerateReferences(carefully: true, considerDependantHandles: false)) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (!objRef.IsValid || objRef.IsFree) // heap corruption + { + continue; + } + + Generation refGen = objRef.GetGeneration(null); + if (refGen == Generation.Large) + { + yield return (obj, objRef); + } + } + } + } + } + + private IEnumerable<(ClrObject From, ClrObject To)> FindLOHToEphemeral() + { + foreach (ClrSegment seg in Runtime.Heap.Segments.Where(seg => seg.Kind == GCSegmentKind.Large)) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + foreach (ClrObject obj in seg.EnumerateObjects().Where(obj => obj.IsValid && obj.ContainsPointers)) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + foreach (ClrObject objRef in obj.EnumerateReferences(carefully: true, considerDependantHandles: false)) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (!objRef.IsValid || objRef.IsFree) // heap corruption + { + continue; + } + + Generation refGen = objRef.GetGeneration(null); + if (refGen is Generation.Gen0 or Generation.Gen1) + { + yield return (obj, objRef); + } + } + } + } + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/FindReferencesToEphemeralCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/FindReferencesToEphemeralCommand.cs new file mode 100644 index 000000000..e77660dbd --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/FindReferencesToEphemeralCommand.cs @@ -0,0 +1,174 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.Runtime; +using static Microsoft.Diagnostics.ExtensionCommands.TableOutput; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "ephrefs", Help = "Finds older generation objects which reference objects in the ephemeral segment.")] + public class FindReferencesToEphemeralCommand : CommandBase + { + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + private readonly HashSet _referenced = new(); + private ulong _referencedSize; + + public override void Invoke() + { + TableOutput output = new(Console, (16, "x12"), (16, "x12"), (10, "n0"), (8, ""), (8, ""), (12, "n0"), (12, "n0")); + + var generationGroup = from item in FindObjectsWithEphemeralReferences() + group item by (item.ObjectGeneration, item.ReferenceGeneration) into g + select new + { + g.Key.ObjectGeneration, + g.Key.ReferenceGeneration, + Objects = g.OrderBy(x => x.Object.Address), + }; + + long objCount = 0; + + (Generation, Generation) last = default; + foreach (var item in generationGroup) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + string objGen = item.ObjectGeneration.ToString().ToLowerInvariant(); + string refGen = item.ReferenceGeneration.ToString().ToLowerInvariant(); + + // Print a new header every time we hit a different combination of object/reference generations + (Generation, Generation) curr = (item.ObjectGeneration, item.ReferenceGeneration); + if (last != curr) + { + if (last != default) + { + Console.WriteLine(); + } + + last = curr; + + Console.WriteLine($"References from {objGen} to {refGen}:"); + Console.WriteLine(); + output.WriteRow("Object", "MethodTable", "Size", "Obj Gen", "Ref Gen", "Obj Count", "Obj Size", "Type"); + } + + foreach (EphemeralRefCount erc in item.Objects) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + objCount++; + output.WriteRow(new DmlDumpObj(erc.Object), erc.Object.Type.MethodTable, erc.Object.Size, erc.ObjectGeneration, erc.ReferenceGeneration, erc.Count, erc.Size, erc.Object.Type.Name); + } + } + + Console.WriteLine(); + Console.WriteLine($"{objCount:n0} older generation objects referenced {_referenced.Count:n0} younger objects ({_referencedSize:n0} bytes)"); + } + + + private IEnumerable FindObjectsWithEphemeralReferences() + { + foreach (ClrSegment seg in Runtime.Heap.Segments) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + // Only skip Gen0 and Frozen regions entirely + if (seg.Kind is GCSegmentKind.Generation0 or GCSegmentKind.Frozen) + { + continue; + } + + foreach (ClrObject obj in seg.EnumerateObjects().Where(obj => obj.IsValid && obj.ContainsPointers)) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + // Skip this object if it's gen0 or we hit an error + Generation objGen = obj.GetGeneration(seg); + if (objGen is Generation.Gen0 or Generation.Error) + { + continue; + } + + // Keep track of whether we've enumerated Gen0/Gen1 references already + EphemeralRefCount gen0 = null; + EphemeralRefCount gen1 = null; + foreach (ClrObject objRef in obj.EnumerateReferences(considerDependantHandles: false)) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (!objRef.IsValid || objRef.IsFree) + { + continue; + } + + ulong refObjSize = objRef.Size; + + Generation refGen = objRef.GetGeneration(null); + switch (refGen) + { + case Generation.Gen0: + gen0 ??= new EphemeralRefCount() + { + Object = obj, + ObjectGeneration = objGen, + ReferenceGeneration = refGen, + }; + + gen0.Count++; + gen0.Size += refObjSize; + if (_referenced.Add(objRef)) + { + _referencedSize += refObjSize; + } + + break; + + case Generation.Gen1: + if (objGen > Generation.Gen1) + { + gen1 ??= new EphemeralRefCount() + { + Object = obj, + ObjectGeneration = objGen, + ReferenceGeneration = refGen, + }; + + gen1.Count++; + gen1.Size += refObjSize; + if (_referenced.Add(objRef)) + { + _referencedSize += refObjSize; + } + } + break; + } + } + + if (gen0 is not null) + { + yield return gen0; + } + + if (gen1 is not null) + { + yield return gen1; + } + } + } + } + + private sealed class EphemeralRefCount + { + public ClrObject Object { get; set; } + public Generation ObjectGeneration { get; set; } + public Generation ReferenceGeneration { get; set; } + public int Count { get; set; } + public ulong Size { get; set; } + } + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Generation.cs b/src/Microsoft.Diagnostics.ExtensionCommands/Generation.cs new file mode 100644 index 000000000..990ef71a2 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Generation.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + internal enum Generation + { + Gen0, + Gen1, + Gen2, + Large, + Pinned, + Frozen, + Error, + } +} diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/SizeStatsCommand.cs b/src/Microsoft.Diagnostics.ExtensionCommands/SizeStatsCommand.cs new file mode 100644 index 000000000..61e88f7a3 --- /dev/null +++ b/src/Microsoft.Diagnostics.ExtensionCommands/SizeStatsCommand.cs @@ -0,0 +1,118 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Diagnostics.DebugServices; +using Microsoft.Diagnostics.Runtime; + +namespace Microsoft.Diagnostics.ExtensionCommands +{ + [Command(Name = "sizestats", Help = "Size statistics for the GC heap.")] + public sealed class SizeStatsCommand : CommandBase + { + [ServiceImport] + public ClrRuntime Runtime { get; set; } + + public override void Invoke() + { + SizeStats(Generation.Gen0, isFree: false); + SizeStats(Generation.Gen1, isFree: false); + SizeStats(Generation.Gen2, isFree: false); + SizeStats(Generation.Large, isFree: false); + + bool hasPinned = Runtime.Heap.Segments.Any(seg => seg.Kind == GCSegmentKind.Pinned); + if (hasPinned) + { + SizeStats(Generation.Pinned, isFree: false); + } + + if (Runtime.Heap.Segments.Any(r => r.Kind == GCSegmentKind.Frozen)) + { + SizeStats(Generation.Frozen, isFree: false); + } + + SizeStats(Generation.Gen0, isFree: true); + SizeStats(Generation.Gen1, isFree: true); + SizeStats(Generation.Gen2, isFree: true); + SizeStats(Generation.Large, isFree: true); + + if (hasPinned) + { + SizeStats(Generation.Pinned, isFree: true); + } + } + + private void SizeStats(Generation requestedGen, bool isFree) + { + Dictionary stats = new(); + foreach (ClrSegment seg in Runtime.Heap.Segments.Where(seg => FilterByGeneration(seg, requestedGen))) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + foreach (ClrObject obj in seg.EnumerateObjects()) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + if (!obj.IsValid || obj.IsFree != isFree) + { + continue; + } + + // If Kind == Ephemeral, we have to further filter by object generation + if (seg.Kind == GCSegmentKind.Ephemeral) + { + if (obj.GetGeneration(seg) != requestedGen) + { + continue; + } + } + + ulong size = (obj.Size + 7u) & ~7u; + stats.TryGetValue(size, out ulong count); + stats[size] = count + 1; + } + } + + string freeStr = isFree ? "free " : ""; + Console.WriteLine($"Size Statistics for {requestedGen.ToString().ToLowerInvariant()} {freeStr}objects"); + Console.WriteLine(); + + TableOutput output = new(Console, (16, "n0"), (16, "n0"), (16, "n0"), (16, "n0")); + output.WriteRow("Size", "Count", "Cumulative Size", "Cumulative Count"); + + + IEnumerable<(ulong Size, ulong Count)> sorted = from i in stats + orderby i.Key ascending + select (i.Key, i.Value); + + ulong cumulativeSize = 0; + ulong cumulativeCount = 0; + foreach ((ulong size, ulong count) in sorted) + { + Console.CancellationToken.ThrowIfCancellationRequested(); + + cumulativeSize += size * count; + cumulativeCount += count; + output.WriteRow(size, count, cumulativeSize, cumulativeCount); + } + + Console.WriteLine(); + } + + private static bool FilterByGeneration(ClrSegment seg, Generation gen) + { + return seg.Kind switch + { + GCSegmentKind.Ephemeral => gen <= Generation.Gen2, + GCSegmentKind.Generation0 => gen == Generation.Gen0, + GCSegmentKind.Generation1 => gen == Generation.Gen1, + GCSegmentKind.Generation2 => gen == Generation.Gen2, + GCSegmentKind.Frozen => gen == Generation.Frozen, + GCSegmentKind.Pinned => gen == Generation.Pinned, + GCSegmentKind.Large => gen == Generation.Large, + _ => false + }; + } + } +} -- 2.34.1