From 4e53638a21b2cde04f94a506a3dee51e4bdff9f8 Mon Sep 17 00:00:00 2001 From: David Mason Date: Thu, 30 Jul 2020 16:20:24 -0700 Subject: [PATCH] Report POH objects in RootReferences and ObjectReferences (#39992) --- src/coreclr/src/gc/gc.cpp | 23 ++-- src/coreclr/tests/issues.targets | 11 +- src/tests/profiler/gc/gc.cs | 90 ++++++++------ src/tests/profiler/gc/gcbasic.cs | 83 +++++++++++++ src/tests/profiler/gc/gcbasic.csproj | 16 +++ src/tests/profiler/native/CMakeLists.txt | 2 + src/tests/profiler/native/classfactory.cpp | 4 +- .../profiler/native/gcprofiler/gcprofiler.cpp | 136 +++++++++++++++++++++ src/tests/profiler/native/gcprofiler/gcprofiler.h | 44 +++++++ 9 files changed, 351 insertions(+), 58 deletions(-) create mode 100644 src/tests/profiler/gc/gcbasic.cs create mode 100644 src/tests/profiler/gc/gcbasic.csproj create mode 100644 src/tests/profiler/native/gcprofiler/gcprofiler.cpp create mode 100644 src/tests/profiler/native/gcprofiler/gcprofiler.h diff --git a/src/coreclr/src/gc/gc.cpp b/src/coreclr/src/gc/gc.cpp index ea08dd1..e20daab 100644 --- a/src/coreclr/src/gc/gc.cpp +++ b/src/coreclr/src/gc/gc.cpp @@ -38609,8 +38609,8 @@ void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOO generation_allocation_start (gen)); uint8_t* end = heap_segment_allocated (seg); - BOOL small_object_segments = TRUE; - int align_const = get_alignment_constant (small_object_segments); + int align_const = get_alignment_constant (TRUE); + BOOL walk_pinned_object_heap = walk_large_object_heap_p; while (1) @@ -38625,20 +38625,25 @@ void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOO } else { - if (small_object_segments && walk_large_object_heap_p) - + if (walk_large_object_heap_p) { - small_object_segments = FALSE; - align_const = get_alignment_constant (small_object_segments); + walk_large_object_heap_p = FALSE; seg = generation_start_segment (large_object_generation); - x = heap_segment_mem (seg); - end = heap_segment_allocated (seg); - continue; + } + else if (walk_pinned_object_heap) + { + walk_pinned_object_heap = FALSE; + seg = generation_start_segment (pinned_object_generation); } else { break; } + + align_const = get_alignment_constant (FALSE); + x = heap_segment_mem (seg); + end = heap_segment_allocated (seg); + continue; } } diff --git a/src/coreclr/tests/issues.targets b/src/coreclr/tests/issues.targets index 3a96607..61fad7c 100644 --- a/src/coreclr/tests/issues.targets +++ b/src/coreclr/tests/issues.targets @@ -1625,16 +1625,7 @@ https://github.com/dotnet/runtime/issues/34072 - - needs triage - - - needs triage - - - needs triage - - + needs triage diff --git a/src/tests/profiler/gc/gc.cs b/src/tests/profiler/gc/gc.cs index e0840ef5..955ee84 100644 --- a/src/tests/profiler/gc/gc.cs +++ b/src/tests/profiler/gc/gc.cs @@ -6,53 +6,67 @@ using System.Threading; namespace Profiler.Tests { - class GCBasicTests //: ProfilerTest + class AllocObject { - static readonly Guid GcBasicEventsProfilerGuid = new Guid("A040B953-EDE7-42D9-9077-AA69BB2BE024"); + public static readonly int ArraySize = 100; + int[] m_array = null; - public static void DoWork() { - int k=0; - while(k<3) { - Console.WriteLine("{0}: Restarting run {1}",Thread.CurrentThread.Name,k); - int[] largeArray = new int[1000000]; - for (int i=0;i<=100;i++){ - int[] saveArray = largeArray; - largeArray = new int[largeArray.Length + 100000]; - saveArray = null; - //Console.WriteLine("{0} at size {1}",Thread.CurrentThread.Name,largeArray.Length.ToString()); - } - k++; - } - } - - public static int RunTest(String[] args) { - long Threads = 1; - - if(args.Length > 2) + public AllocObject(bool poh) + { + if (poh) { - Console.WriteLine("usage: LargeObjectAlloc runtest "); - return 1; + m_array = GC.AllocateArray(ArraySize, true); } - else if(args.Length == 2) + else { - Threads = Int64.Parse(args[1]); + m_array = new int[ArraySize]; } + } + } - Console.WriteLine("LargeObjectAlloc started with {0} threads. Control-C to exit", - Threads.ToString()); + class GCTests + { + static readonly Guid GCProfilerGuid = new Guid("BCD8186F-1EEC-47E9-AFA7-396F879382C3"); + + public static int RunTest(String[] args) + { + int numAllocators = 1024; + int[] root1 = GC.AllocateArray(AllocObject.ArraySize, true); + int[] root2 = GC.AllocateArray(AllocObject.ArraySize, true); + AllocObject[] objs = new AllocObject[numAllocators]; - Thread myThread = null; - for(long i = 0; i < Threads; i++) + Random rand = new Random(); + int numPoh = 0; + int numReg = 0; + for (int i = 0; i < 10000; ++i) { - myThread = new Thread(new ThreadStart(DoWork)); - myThread.Name = i.ToString(); - myThread.Start(); - } + int pos = rand.Next(0, numAllocators); + + bool poh = rand.NextDouble() > 0.5; + objs[pos] = new AllocObject(poh); + + if (poh) + { + ++numPoh; + } + else + { + ++numReg; + } - Console.WriteLine("All threads started"); - myThread.Join(); + if (i % 1000 == 0) + { + GC.Collect(); + Console.WriteLine ($"Did {i} iterations Allocated={GC.GetAllocatedBytesForCurrentThread()}"); + } + + + int[] m_array = new int[100]; + } - Console.WriteLine("Test Passed"); + Console.WriteLine($"{numPoh} POH allocs and {numReg} normal allocs."); + GC.KeepAlive(root1); + GC.KeepAlive(root2); return 100; } @@ -64,8 +78,8 @@ namespace Profiler.Tests } return ProfilerTestRunner.Run(profileePath: System.Reflection.Assembly.GetExecutingAssembly().Location, - testName: "GCCallbacksBasic", - profilerClsid: GcBasicEventsProfilerGuid); + testName: "GCTests", + profilerClsid: GCProfilerGuid); } } } diff --git a/src/tests/profiler/gc/gcbasic.cs b/src/tests/profiler/gc/gcbasic.cs new file mode 100644 index 0000000..61b3f14 --- /dev/null +++ b/src/tests/profiler/gc/gcbasic.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; + +namespace Profiler.Tests +{ + class GCBasicTests + { + static readonly Guid GcBasicEventsProfilerGuid = new Guid("A040B953-EDE7-42D9-9077-AA69BB2BE024"); + + public static void DoWork() + { + // Allocate POH objects + var arr0 = GC.AllocateUninitializedArray(100000, true); + var arr1 = GC.AllocateArray(200000, true); + + int k = 0; + while(k < 3) + { + Console.WriteLine("{0}: Restarting run {1}",Thread.CurrentThread.Name,k); + int[] largeArray = new int[1000000]; + for (int i = 0; i <= 100; i++) + { + int[] saveArray = largeArray; + largeArray = new int[largeArray.Length + 100000]; + saveArray = null; + //Console.WriteLine("{0} at size {1}",Thread.CurrentThread.Name,largeArray.Length.ToString()); + } + + k++; + } + + GC.KeepAlive(arr0); + GC.KeepAlive(arr1); + } + + public static int RunTest(String[] args) + { + long Threads = 1; + + if(args.Length > 2) + { + Console.WriteLine("usage: LargeObjectAlloc runtest "); + return 1; + } + else if(args.Length == 2) + { + Threads = Int64.Parse(args[1]); + } + + Console.WriteLine("LargeObjectAlloc started with {0} threads. Control-C to exit", + Threads.ToString()); + + Thread myThread = null; + for(long i = 0; i < Threads; i++) + { + myThread = new Thread(new ThreadStart(DoWork)); + myThread.Name = i.ToString(); + myThread.Start(); + } + + Console.WriteLine("All threads started"); + myThread.Join(); + + Console.WriteLine("Test Passed"); + return 100; + } + + public static int Main(string[] args) + { + if (args.Length > 0 && args[0].Equals("RunTest", StringComparison.OrdinalIgnoreCase)) + { + return RunTest(args); + } + + return ProfilerTestRunner.Run(profileePath: System.Reflection.Assembly.GetExecutingAssembly().Location, + testName: "GCCallbacksBasic", + profilerClsid: GcBasicEventsProfilerGuid); + } + } +} diff --git a/src/tests/profiler/gc/gcbasic.csproj b/src/tests/profiler/gc/gcbasic.csproj new file mode 100644 index 0000000..7a525ce --- /dev/null +++ b/src/tests/profiler/gc/gcbasic.csproj @@ -0,0 +1,16 @@ + + + .NETCoreApp + exe + BuildAndRun + true + 0 + true + + + + + + + + diff --git a/src/tests/profiler/native/CMakeLists.txt b/src/tests/profiler/native/CMakeLists.txt index a742c92..b58bb64 100644 --- a/src/tests/profiler/native/CMakeLists.txt +++ b/src/tests/profiler/native/CMakeLists.txt @@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 3.14.5) project(Profiler) set(GCBASIC_SOURCES gcbasicprofiler/gcbasicprofiler.cpp) +set(GC_SOURCES gcprofiler/gcprofiler.cpp) set(REJIT_SOURCES rejitprofiler/rejitprofiler.cpp rejitprofiler/ilrewriter.cpp rejitprofiler/sigparse.cpp) set(EVENTPIPE_SOURCES eventpipeprofiler/eventpipereadingprofiler.cpp @@ -14,6 +15,7 @@ set(ELT_SOURCES eltprofiler/slowpatheltprofiler.cpp) set(SOURCES ${GCBASIC_SOURCES} + ${GC_SOURCES} ${REJIT_SOURCES} ${EVENTPIPE_SOURCES} ${METADATAGETDISPENSER_SOURCES} diff --git a/src/tests/profiler/native/classfactory.cpp b/src/tests/profiler/native/classfactory.cpp index 301e91d..19de051 100644 --- a/src/tests/profiler/native/classfactory.cpp +++ b/src/tests/profiler/native/classfactory.cpp @@ -9,6 +9,7 @@ #include "metadatagetdispenser/metadatagetdispenser.h" #include "getappdomainstaticaddress/getappdomainstaticaddress.h" #include "eltprofiler/slowpatheltprofiler.h" +#include "gcprofiler/gcprofiler.h" ClassFactory::ClassFactory(REFCLSID clsid) : refCount(0), clsid(clsid) { @@ -63,7 +64,8 @@ HRESULT STDMETHODCALLTYPE ClassFactory::CreateInstance(IUnknown *pUnkOuter, REFI new EventPipeWritingProfiler(), new MetaDataGetDispenser(), new GetAppDomainStaticAddress(), - new SlowPathELTProfiler() + new SlowPathELTProfiler(), + new GCProfiler() // add new profilers here }; diff --git a/src/tests/profiler/native/gcprofiler/gcprofiler.cpp b/src/tests/profiler/native/gcprofiler/gcprofiler.cpp new file mode 100644 index 0000000..e6d3edc --- /dev/null +++ b/src/tests/profiler/native/gcprofiler/gcprofiler.cpp @@ -0,0 +1,136 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "gcprofiler.h" + +GUID GCProfiler::GetClsid() +{ + // {BCD8186F-1EEC-47E9-AFA7-396F879382C3} + GUID clsid = { 0xBCD8186F, 0x1EEC, 0x47E9, { 0xAF, 0xA7, 0x39, 0x6F, 0x87, 0x93, 0x82, 0xC3 } }; + return clsid; +} + +HRESULT GCProfiler::Initialize(IUnknown* pICorProfilerInfoUnk) +{ + Profiler::Initialize(pICorProfilerInfoUnk); + + HRESULT hr = S_OK; + if (FAILED(hr = pCorProfilerInfo->SetEventMask2(COR_PRF_MONITOR_GC, 0))) + { + _failures++; + printf("FAIL: ICorProfilerInfo::SetEventMask2() failed hr=0x%x", hr); + return hr; + } + + return S_OK; +} + +HRESULT GCProfiler::Shutdown() +{ + Profiler::Shutdown(); + + if (_gcStarts == 0) + { + printf("GCProfiler::Shutdown: FAIL: Expected GarbaseCollectionStarted to be called\n"); + } + else if (_gcFinishes == 0) + { + printf("GCProfiler::Shutdown: FAIL: Expected GarbageCollectionFinished to be called\n"); + } + else if (_pohObjectsSeenRootReferences == 0 || _pohObjectsSeenObjectReferences == 0) + { + printf("GCProfiler::Shutdown: FAIL: no POH objects seen. root references=%d object references=%d\n", + _pohObjectsSeenRootReferences.load(), _pohObjectsSeenObjectReferences.load()); + } + else if(_failures == 0) + { + printf("PROFILER TEST PASSES\n"); + } + else + { + // failures were printed earlier when _failures was incremented + } + fflush(stdout); + + return S_OK; +} + +HRESULT GCProfiler::GarbageCollectionStarted(int cGenerations, BOOL generationCollected[], COR_PRF_GC_REASON reason) +{ + _gcStarts++; + if (_gcStarts - _gcFinishes > 2) + { + _failures++; + printf("GCProfiler::GarbageCollectionStarted: FAIL: Expected GCStart <= GCFinish+2. GCStart=%d, GCFinish=%d\n", (int)_gcStarts, (int)_gcFinishes); + } + + return S_OK; +} + +HRESULT GCProfiler::GarbageCollectionFinished() +{ + _gcFinishes++; + if (_gcStarts < _gcFinishes) + { + _failures++; + printf("GCProfiler::GarbageCollectionFinished: FAIL: Expected GCStart >= GCFinish. Start=%d, Finish=%d\n", (int)_gcStarts, (int)_gcFinishes); + } + + _pohObjectsSeenObjectReferences += NumPOHObjectsSeen(_objectReferencesSeen); + _pohObjectsSeenRootReferences += NumPOHObjectsSeen(_rootReferencesSeen); + + return S_OK; +} + +HRESULT GCProfiler::ObjectReferences(ObjectID objectId, ClassID classId, ULONG cObjectRefs, ObjectID objectRefIds[]) +{ + HRESULT hr = S_OK; + for (ULONG i = 0; i < cObjectRefs; ++i) + { + ObjectID obj = objectRefIds[i]; + if (obj != NULL) + { + _objectReferencesSeen.insert(obj); + } + } + + return S_OK; +} + +HRESULT GCProfiler::RootReferences(ULONG cRootRefs, ObjectID rootRefIds[]) +{ + for (ULONG i = 0; i < cRootRefs; ++i) + { + ObjectID obj = rootRefIds[i]; + if (obj != NULL) + { + _rootReferencesSeen.insert(obj); + } + } + + return S_OK; +} + +int GCProfiler::NumPOHObjectsSeen(std::unordered_set objects) +{ + int count = 0; + for (auto it = objects.begin(); it != objects.end(); ++it) + { + COR_PRF_GC_GENERATION_RANGE gen; + ObjectID obj = *it; + HRESULT hr = pCorProfilerInfo->GetObjectGeneration(obj, &gen); + if (FAILED(hr)) + { + printf("GetObjectGeneration failed hr=0x%x\n", hr); + return hr; + } + + if (gen.generation == COR_PRF_GC_PINNED_OBJECT_HEAP) + { + count++; + } + + } + + return count; +} diff --git a/src/tests/profiler/native/gcprofiler/gcprofiler.h b/src/tests/profiler/native/gcprofiler/gcprofiler.h new file mode 100644 index 0000000..8e0fbeb --- /dev/null +++ b/src/tests/profiler/native/gcprofiler/gcprofiler.h @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma once + +#include "../profiler.h" +#include + + +// TODO: this class is intended to be the heavyweight GC API verification (i.e. COR_PRF_MONITOR_GC) +// vs GCBasicProfiler which verifies COR_PRF_HIGH_BASIC_GC. Right now the only thing it does +// is verify we see POH objects in the RootReferences callback, but it should be fleshed out. +class GCProfiler : public Profiler +{ +public: + GCProfiler() : Profiler(), + _gcStarts(0), + _gcFinishes(0), + _failures(0), + _pohObjectsSeenRootReferences(0), + _pohObjectsSeenObjectReferences(0), + _rootReferencesSeen(), + _objectReferencesSeen() + {} + + virtual GUID GetClsid(); + virtual HRESULT STDMETHODCALLTYPE Initialize(IUnknown* pICorProfilerInfoUnk); + virtual HRESULT STDMETHODCALLTYPE Shutdown(); + virtual HRESULT STDMETHODCALLTYPE GarbageCollectionStarted(int cGenerations, BOOL generationCollected[], COR_PRF_GC_REASON reason); + virtual HRESULT STDMETHODCALLTYPE GarbageCollectionFinished(); + virtual HRESULT STDMETHODCALLTYPE ObjectReferences(ObjectID objectId, ClassID classId, ULONG cObjectRefs, ObjectID objectRefIds[]); + virtual HRESULT STDMETHODCALLTYPE RootReferences(ULONG cRootRefs, ObjectID rootRefIds[]); + +private: + std::atomic _gcStarts; + std::atomic _gcFinishes; + std::atomic _failures; + std::atomic _pohObjectsSeenRootReferences; + std::atomic _pohObjectsSeenObjectReferences; + std::unordered_set _rootReferencesSeen; + std::unordered_set _objectReferencesSeen; + + int NumPOHObjectsSeen(std::unordered_set objects); +}; -- 2.7.4