From: David Mason Date: Thu, 5 Jan 2023 07:05:28 +0000 (-0800) Subject: Port GCDump fixes from PerfView (#3475) X-Git-Tag: accepted/tizen/unified/riscv/20231226.055542~44^2^2~30 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=839361d2b6b3fc91f49091506224dddf6b7d9c4b;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Port GCDump fixes from PerfView (#3475) * GCDump updates * bump traceevent version --- diff --git a/eng/Versions.props b/eng/Versions.props index ebe0dce08..0f06e1b50 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -47,7 +47,7 @@ 1.1.0 2.3.405304 16.9.0-beta1.21055.5 - 2.0.64 + 2.0.76 2.1.1 diff --git a/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapDumpGraphReader.cs b/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapDumpGraphReader.cs index af5b45ebd..8d7836c4b 100644 --- a/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapDumpGraphReader.cs +++ b/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapDumpGraphReader.cs @@ -182,6 +182,12 @@ public class DotNetHeapDumpGraphReader } }; + source.Clr.GCGenAwareStart += delegate (GenAwareBeginTraceData data) + { + m_seenStart = true; + m_ignoreEvents = false; + }; + source.Clr.GCStart += delegate (GCStartTraceData data) { // If this GC is not part of a heap dump, ignore it. @@ -231,8 +237,6 @@ public class DotNetHeapDumpGraphReader } }; - - source.Clr.GCStop += delegate (GCEndTraceData data) { if (m_ignoreEvents || data.ProcessID != m_processId) @@ -262,6 +266,17 @@ public class DotNetHeapDumpGraphReader } }; + source.Clr.GCGenAwareEnd += delegate (GenAwareEndTraceData data) + { + m_ignoreEvents = true; + if (m_nodeBlocks.Count == 0 && m_typeBlocks.Count == 0 && m_edgeBlocks.Count == 0) + { + m_log.WriteLine("Found no node events, looking for another GC"); + m_seenStart = false; + return; + } + }; + source.Clr.TypeBulkType += delegate (GCBulkTypeTraceData data) { // Don't check m_ignoreEvents here, as BulkType events can be emitted by other events...such as the GC allocation event. @@ -474,6 +489,9 @@ public class DotNetHeapDumpGraphReader case 3: segment.Gen3End = end; break; + case 4: + segment.Gen4End = end; + break; default: throw new Exception("Invalid generation in GCGenerationRangeTraceData"); } @@ -488,8 +506,6 @@ public class DotNetHeapDumpGraphReader /// internal unsafe void ConvertHeapDataToGraph() { - int maxNodeCount = 10_000_000; - if (m_converted) { return; @@ -696,14 +712,6 @@ public class DotNetHeapDumpGraphReader Debug.Assert(!m_graph.IsDefined(nodeIdx)); m_graph.SetNode(nodeIdx, typeIdx, objSize, m_children); - - if (m_graph.NodeCount >= maxNodeCount) - { - doCompletionCheck = false; - var userMessage = string.Format("Exceeded max node count {0}", maxNodeCount); - m_log.WriteLine("[WARNING: ]", userMessage); - break; - } } if (doCompletionCheck && m_curEdgeBlock != null && m_curEdgeBlock.Count != m_curEdgeIdx) diff --git a/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapInfo.cs b/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapInfo.cs index 27f01b9c1..8f71b0808 100644 --- a/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapInfo.cs +++ b/src/Tools/dotnet-gcdump/DotNetHeapDump/DotNetHeapInfo.cs @@ -52,6 +52,11 @@ public class DotNetHeapInfo : IFastSerializable } } + if (obj < m_lastSegment.Gen4End) + { + return 4; + } + if (obj < m_lastSegment.Gen3End) { return 3; @@ -107,7 +112,7 @@ public class DotNetHeapInfo : IFastSerializable #endregion } -public class GCHeapDumpSegment : IFastSerializable +public class GCHeapDumpSegment : IFastSerializable, IFastSerializableVersion { public Address Start { get; internal set; } public Address End { get; internal set; } @@ -115,6 +120,13 @@ public class GCHeapDumpSegment : IFastSerializable public Address Gen1End { get; internal set; } public Address Gen2End { get; internal set; } public Address Gen3End { get; internal set; } + public Address Gen4End { get; internal set; } + + public int Version => 1; + + public int MinimumVersionCanRead => 0; + + public int MinimumReaderVersion => 1; #region private void IFastSerializable.ToStream(Serializer serializer) @@ -125,6 +137,7 @@ public class GCHeapDumpSegment : IFastSerializable serializer.Write((long)Gen1End); serializer.Write((long)Gen2End); serializer.Write((long)Gen3End); + serializer.Write((long)Gen4End); } void IFastSerializable.FromStream(Deserializer deserializer) @@ -135,6 +148,10 @@ public class GCHeapDumpSegment : IFastSerializable Gen1End = (Address)deserializer.ReadInt64(); Gen2End = (Address)deserializer.ReadInt64(); Gen3End = (Address)deserializer.ReadInt64(); + if (deserializer.VersionBeingRead >= 1) + { + Gen4End = (Address)deserializer.ReadInt64(); + } } #endregion -} \ No newline at end of file +} diff --git a/src/Tools/dotnet-gcdump/DotNetHeapDump/GCHeapDump.cs b/src/Tools/dotnet-gcdump/DotNetHeapDump/GCHeapDump.cs index 424be99da..b93d94c8f 100644 --- a/src/Tools/dotnet-gcdump/DotNetHeapDump/GCHeapDump.cs +++ b/src/Tools/dotnet-gcdump/DotNetHeapDump/GCHeapDump.cs @@ -18,11 +18,11 @@ using Address = System.UInt64; public class GCHeapDump : IFastSerializable, IFastSerializableVersion { public GCHeapDump(string inputFileName) : - this(new Deserializer(inputFileName)) + this(new Deserializer(inputFileName, new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes })) { } public GCHeapDump(Stream inputStream, string streamName) : - this(new Deserializer(inputStream, streamName)) + this(new Deserializer(inputStream, streamName, new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes })) { } /// @@ -193,7 +193,7 @@ public class GCHeapDump : IFastSerializable, IFastSerializableVersion private void Write(string outputFileName) { Debug.Assert(MemoryGraph != null); - var serializer = new Serializer(outputFileName, this); + var serializer = new Serializer(new IOStreamStreamWriter(outputFileName, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes }), this); serializer.Close(); } diff --git a/src/Tools/dotnet-gcdump/DotNetHeapDump/Graph.cs b/src/Tools/dotnet-gcdump/DotNetHeapDump/Graph.cs index 7280bbdb3..7447a90d2 100644 --- a/src/Tools/dotnet-gcdump/DotNetHeapDump/Graph.cs +++ b/src/Tools/dotnet-gcdump/DotNetHeapDump/Graph.cs @@ -164,8 +164,11 @@ namespace Graphs /// /// TODO I can eliminate the need for AllowReading. /// - public Graph(int expectedNodeCount) + /// if isVeryLargeGraph argument is true, then StreamLabels will be serialized as longs + /// too acommodate for the extra size of the graph's stream representation. + public Graph(int expectedNodeCount, bool isVeryLargeGraph = false) { + m_isVeryLargeGraph = isVeryLargeGraph; m_expectedNodeCount = expectedNodeCount; m_types = new GrowableArray(Math.Max(expectedNodeCount / 100, 2000)); m_nodes = new SegmentedList(SegmentSize, m_expectedNodeCount); @@ -465,7 +468,8 @@ namespace Graphs RootIndex = NodeIndex.Invalid; if (m_writer == null) { - m_writer = new SegmentedMemoryStreamWriter(m_expectedNodeCount * 8); + m_writer = new SegmentedMemoryStreamWriter(m_expectedNodeCount * 8, + m_isVeryLargeGraph ? new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes } : null); } m_totalSize = 0; @@ -513,7 +517,15 @@ namespace Graphs } // Write out the Nodes - serializer.Write(m_nodes.Count); + if (m_isVeryLargeGraph) + { + serializer.Write(m_nodes.Count); + } + else + { + serializer.Write((int)m_nodes.Count); + } + for (int i = 0; i < m_nodes.Count; i++) { serializer.Write((int)m_nodes[i]); @@ -551,7 +563,7 @@ namespace Graphs // You can place tagged values in here always adding right before the WriteTaggedEnd // for any new fields added after version 1 - serializer.WriteTaggedEnd(); // This insures tagged things don't read junk after the region. + serializer.WriteTaggedEnd(); // This ensures tagged things don't read junk after the region. }); } } @@ -574,10 +586,10 @@ namespace Graphs } // Read in the Nodes - int nodeCount = deserializer.ReadInt(); + long nodeCount = m_isVeryLargeGraph ? deserializer.ReadInt64() : deserializer.ReadInt(); m_nodes = new SegmentedList(SegmentSize, nodeCount); - for (int i = 0; i < nodeCount; i++) + for (long i = 0; i < nodeCount; i++) { m_nodes.Add((StreamLabel)(uint)deserializer.ReadInt()); } @@ -585,7 +597,9 @@ namespace Graphs // Read in the Blob stream. // TODO be lazy about reading in the blobs. int blobCount = deserializer.ReadInt(); - SegmentedMemoryStreamWriter writer = new SegmentedMemoryStreamWriter(blobCount); + SegmentedMemoryStreamWriter writer = new SegmentedMemoryStreamWriter(blobCount, + m_isVeryLargeGraph ? new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes } : null); + while (8 <= blobCount) { writer.Write(deserializer.ReadInt64()); @@ -644,7 +658,7 @@ namespace Graphs } } - private int m_expectedNodeCount; // Initial guess at graph Size. + private long m_expectedNodeCount; // Initial guess at graph Size. private long m_totalSize; // Total Size of all the nodes in the graph. internal int m_totalRefs; // Total Number of references in the graph internal GrowableArray m_types; // We expect only thousands of these @@ -656,6 +670,7 @@ namespace Graphs // There should not be any of these left as long as every node referenced // by another node has a definition. internal SegmentedMemoryStreamWriter m_writer; // Used only during construction to serialize the nodes. + protected bool m_isVeryLargeGraph; #endregion } @@ -1065,7 +1080,7 @@ namespace Graphs /// public DateTime BuildTime; // From in the PE header /// - /// The name of hte PDB file assoicated with this module. Ma bye null if unknown + /// The name of the PDB file associated with this module. May be null if unknown /// public string PdbName; /// @@ -1376,7 +1391,7 @@ public class RefGraph /// Given an arbitrary code:NodeIndex that identifies the node, Get a code:Node object. /// /// This routine does not allocated but uses the space passed in by 'storage. - /// 'storage' should be allocated with coode:AllocNodeStorage, and should be agressively reused. + /// 'storage' should be allocated with coode:AllocNodeStorage, and should be aggressively reused. /// public RefNode GetNode(NodeIndex nodeIndex, RefNode storage) { @@ -1829,8 +1844,8 @@ public class SpanningTree /// /// A helper for AddOrphansToQueue, so we only add orphans that are not reachable from other orphans. /// - /// Mark all decendents (but not nodeIndex itself) as being visited. Any arcs that form - /// cycles are ignored, so nodeIndex is guarenteed to NOT be marked. + /// Mark all descendants (but not nodeIndex itself) as being visited. Any arcs that form + /// cycles are ignored, so nodeIndex is guaranteed to NOT be marked. /// private void MarkDecendentsIgnoringCycles(NodeIndex nodeIndex, int recursionCount) { @@ -2393,7 +2408,7 @@ public class GraphSampler stats.TotalMetric += node.Size; } - // Also insure that if there are a large number of types, that we sample them at least some. + // Also ensure that if there are a large number of types, that we sample them at least some. if (stats.SampleCount == 0 && !mustAdd && (m_numDistictTypesWithSamples + .5F) * m_filteringRatio <= m_numDistictTypes) { mustAdd = true; @@ -2626,7 +2641,7 @@ public class GraphSampler /// /// This value goes in the m_newIndex[]. If we accept the node into the sampled graph, we put the node - /// index in the NET graph in m_newIndex. If we reject the node we use the special RegjectedNode value + /// index in the NET graph in m_newIndex. If we reject the node we use the special RejectedNode value /// below /// private const NodeIndex RejectedNode = (NodeIndex)(-2); diff --git a/src/Tools/dotnet-gcdump/DotNetHeapDump/MemoryGraph.cs b/src/Tools/dotnet-gcdump/DotNetHeapDump/MemoryGraph.cs index 98e833596..bb60ef3bf 100644 --- a/src/Tools/dotnet-gcdump/DotNetHeapDump/MemoryGraph.cs +++ b/src/Tools/dotnet-gcdump/DotNetHeapDump/MemoryGraph.cs @@ -9,10 +9,20 @@ namespace Graphs { public class MemoryGraph : Graph, IFastSerializable { - public MemoryGraph(int expectedSize) - : base(expectedSize) + public MemoryGraph(int expectedSize, bool isVeryLargeGraph = false) + : base(expectedSize, isVeryLargeGraph) + { + // If we have too many addresses we will reach the Dictionary's internal array's size limit and throw. + // Therefore use a new implementation of it that is similar in performance but that can handle the extra load. + if (isVeryLargeGraph) + { + m_addressToNodeIndex = new SegmentedDictionary(expectedSize); + } + else { m_addressToNodeIndex = new Dictionary(expectedSize); + } + m_nodeAddresses = new SegmentedList
(SegmentSize, expectedSize); } @@ -113,7 +123,7 @@ namespace Graphs /// THis table maps the ID that CLRProfiler uses (an address), to the NodeIndex we have assigned to it. /// It is only needed while the file is being read in. /// - protected Dictionary m_addressToNodeIndex; // This field is only used during construction + protected IDictionary m_addressToNodeIndex; // This field is only used during construction #endregion #region private @@ -121,7 +131,15 @@ namespace Graphs { base.ToStream(serializer); // Write out the Memory addresses of each object + if (m_isVeryLargeGraph) + { serializer.Write(m_nodeAddresses.Count); + } + else + { + serializer.Write((int)m_nodeAddresses.Count); + } + for (int i = 0; i < m_nodeAddresses.Count; i++) { serializer.Write((long)m_nodeAddresses[i]); @@ -134,10 +152,10 @@ namespace Graphs { base.FromStream(deserializer); // Read in the Memory addresses of each object - int addressCount = deserializer.ReadInt(); + long addressCount = m_isVeryLargeGraph ? deserializer.ReadInt64() : deserializer.ReadInt(); m_nodeAddresses = new SegmentedList
(SegmentSize, addressCount); - for (int i = 0; i < addressCount; i++) + for (long i = 0; i < addressCount; i++) { m_nodeAddresses.Add((Address)deserializer.ReadInt64()); }