Report POH objects in RootReferences and ObjectReferences (#39992)
authorDavid Mason <davmason@microsoft.com>
Thu, 30 Jul 2020 23:20:24 +0000 (16:20 -0700)
committerGitHub <noreply@github.com>
Thu, 30 Jul 2020 23:20:24 +0000 (16:20 -0700)
src/coreclr/src/gc/gc.cpp
src/coreclr/tests/issues.targets
src/tests/profiler/gc/gc.cs
src/tests/profiler/gc/gcbasic.cs [new file with mode: 0644]
src/tests/profiler/gc/gcbasic.csproj [new file with mode: 0644]
src/tests/profiler/native/CMakeLists.txt
src/tests/profiler/native/classfactory.cpp
src/tests/profiler/native/gcprofiler/gcprofiler.cpp [new file with mode: 0644]
src/tests/profiler/native/gcprofiler/gcprofiler.h [new file with mode: 0644]

index ea08dd1..e20daab 100644 (file)
@@ -38609,8 +38609,8 @@ void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOO
                      generation_allocation_start (gen));
 
     uint8_t*       end = heap_segment_allocated (seg);
-    BOOL small_object_segments = TRUE;
-    int align_const = get_alignment_constant (small_object_segments);
+    int align_const = get_alignment_constant (TRUE);
+    BOOL walk_pinned_object_heap = walk_large_object_heap_p;
 
     while (1)
 
@@ -38625,20 +38625,25 @@ void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOO
             }
             else
             {
-                if (small_object_segments && walk_large_object_heap_p)
-
+                if (walk_large_object_heap_p)
                 {
-                    small_object_segments = FALSE;
-                    align_const = get_alignment_constant (small_object_segments);
+                    walk_large_object_heap_p = FALSE;
                     seg = generation_start_segment (large_object_generation);
-                    x = heap_segment_mem (seg);
-                    end = heap_segment_allocated (seg);
-                    continue;
+                }
+                else if (walk_pinned_object_heap)
+                {
+                    walk_pinned_object_heap = FALSE;
+                    seg = generation_start_segment (pinned_object_generation);
                 }
                 else
                 {
                     break;
                 }
+
+                align_const = get_alignment_constant (FALSE);
+                x = heap_segment_mem (seg);
+                end = heap_segment_allocated (seg);
+                continue;
             }
         }
 
index 3a96607..61fad7c 100644 (file)
         <ExcludeList Include="$(XunitTestBinBase)/Loader/ContextualReflection/ContextualReflection/**">
             <Issue>https://github.com/dotnet/runtime/issues/34072</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/profiler/unittest/getappdomainstaticaddress/**">
-            <Issue>needs triage</Issue>
-        </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/profiler/eventpipe/eventpipe/*">
-            <Issue>needs triage</Issue>
-        </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/profiler/eventpipe/eventpipe_readevents/*">
-            <Issue>needs triage</Issue>
-        </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/profiler/gc/gc/**">
+        <ExcludeList Include="$(XunitTestBinBase)/profiler/**">
             <Issue>needs triage</Issue>
         </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/profiler/elt/slowpatheltenter/*">
index e0840ef..955ee84 100644 (file)
@@ -6,53 +6,67 @@ using System.Threading;
 
 namespace Profiler.Tests
 {
-    class GCBasicTests //: ProfilerTest
+    class AllocObject
     {
-        static readonly Guid GcBasicEventsProfilerGuid = new Guid("A040B953-EDE7-42D9-9077-AA69BB2BE024");
+        public static readonly int ArraySize = 100;
+        int[] m_array = null;
 
-        public static void DoWork() {
-            int k=0;
-            while(k<3) {
-                Console.WriteLine("{0}: Restarting run {1}",Thread.CurrentThread.Name,k);
-                int[] largeArray = new int[1000000];
-                for (int i=0;i<=100;i++){
-                    int[] saveArray = largeArray;
-                    largeArray = new int[largeArray.Length + 100000];
-                    saveArray = null;
-                    //Console.WriteLine("{0} at size {1}",Thread.CurrentThread.Name,largeArray.Length.ToString());
-                }
-                k++;
-           }
-        }
-
-        public static int RunTest(String[] args) {
-            long Threads = 1;
-
-            if(args.Length > 2)
+        public AllocObject(bool poh)
+        {
+            if (poh)
             {
-                Console.WriteLine("usage: LargeObjectAlloc runtest <number of threads>");
-                return 1;
+                m_array = GC.AllocateArray<int>(ArraySize, true);
             }
-            else if(args.Length == 2)
+            else
             {
-                Threads = Int64.Parse(args[1]);
+                m_array = new int[ArraySize];
             }
+        }
+    }
 
-            Console.WriteLine("LargeObjectAlloc started with {0} threads. Control-C to exit",
-                Threads.ToString());
+    class GCTests
+    {
+        static readonly Guid GCProfilerGuid = new Guid("BCD8186F-1EEC-47E9-AFA7-396F879382C3");
+
+        public static int RunTest(String[] args) 
+        {
+            int numAllocators = 1024;
+            int[] root1 = GC.AllocateArray<int>(AllocObject.ArraySize, true);
+            int[] root2 = GC.AllocateArray<int>(AllocObject.ArraySize, true);
+            AllocObject[] objs = new AllocObject[numAllocators];
 
-            Thread myThread = null;
-            for(long i = 0; i < Threads; i++)
+            Random rand = new Random();
+            int numPoh = 0;
+            int numReg = 0;
+            for (int i = 0; i < 10000; ++i)
             {
-                myThread = new Thread(new ThreadStart(DoWork));
-                myThread.Name = i.ToString();
-                myThread.Start();
-            }
+                int pos = rand.Next(0, numAllocators);
+
+                bool poh = rand.NextDouble() > 0.5;
+                objs[pos] = new AllocObject(poh);
+
+                if (poh)
+                {
+                    ++numPoh;
+                }
+                else
+                {
+                    ++numReg;
+                }
 
-            Console.WriteLine("All threads started");
-            myThread.Join();
+                if (i % 1000 == 0)
+                {
+                    GC.Collect();
+                    Console.WriteLine ($"Did {i} iterations Allocated={GC.GetAllocatedBytesForCurrentThread()}");
+                }
+
+
+                int[] m_array = new int[100];
+            }
 
-            Console.WriteLine("Test Passed");
+            Console.WriteLine($"{numPoh} POH allocs and {numReg} normal allocs.");
+            GC.KeepAlive(root1);
+            GC.KeepAlive(root2);
             return 100;
         }
 
@@ -64,8 +78,8 @@ namespace Profiler.Tests
             }
 
             return ProfilerTestRunner.Run(profileePath: System.Reflection.Assembly.GetExecutingAssembly().Location,
-                                          testName: "GCCallbacksBasic",
-                                          profilerClsid: GcBasicEventsProfilerGuid);
+                                          testName: "GCTests",
+                                          profilerClsid: GCProfilerGuid);
         }
     }
 }
diff --git a/src/tests/profiler/gc/gcbasic.cs b/src/tests/profiler/gc/gcbasic.cs
new file mode 100644 (file)
index 0000000..61b3f14
--- /dev/null
@@ -0,0 +1,83 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Threading;
+
+namespace Profiler.Tests
+{
+    class GCBasicTests
+    {
+        static readonly Guid GcBasicEventsProfilerGuid = new Guid("A040B953-EDE7-42D9-9077-AA69BB2BE024");
+
+        public static void DoWork() 
+        {
+            // Allocate POH objects
+            var arr0 = GC.AllocateUninitializedArray<int>(100000, true);
+            var arr1 = GC.AllocateArray<int>(200000, true);
+
+            int k = 0;
+            while(k < 3) 
+            {
+                Console.WriteLine("{0}: Restarting run {1}",Thread.CurrentThread.Name,k);
+                int[] largeArray = new int[1000000];
+                for (int i = 0; i <= 100; i++)
+                {
+                    int[] saveArray = largeArray;
+                    largeArray = new int[largeArray.Length + 100000];
+                    saveArray = null;
+                    //Console.WriteLine("{0} at size {1}",Thread.CurrentThread.Name,largeArray.Length.ToString());
+                }
+                
+                k++;
+           }
+
+            GC.KeepAlive(arr0);
+            GC.KeepAlive(arr1);
+        }
+
+        public static int RunTest(String[] args) 
+        {
+            long Threads = 1;
+
+            if(args.Length > 2)
+            {
+                Console.WriteLine("usage: LargeObjectAlloc runtest <number of threads>");
+                return 1;
+            }
+            else if(args.Length == 2)
+            {
+                Threads = Int64.Parse(args[1]);
+            }
+
+            Console.WriteLine("LargeObjectAlloc started with {0} threads. Control-C to exit",
+                Threads.ToString());
+
+            Thread myThread = null;
+            for(long i = 0; i < Threads; i++)
+            {
+                myThread = new Thread(new ThreadStart(DoWork));
+                myThread.Name = i.ToString();
+                myThread.Start();
+            }
+
+            Console.WriteLine("All threads started");
+            myThread.Join();
+
+            Console.WriteLine("Test Passed");
+            return 100;
+        }
+
+        public static int Main(string[] args)
+        {
+            if (args.Length > 0 && args[0].Equals("RunTest", StringComparison.OrdinalIgnoreCase))
+            {
+                return RunTest(args);
+            }
+
+            return ProfilerTestRunner.Run(profileePath: System.Reflection.Assembly.GetExecutingAssembly().Location,
+                                          testName: "GCCallbacksBasic",
+                                          profilerClsid: GcBasicEventsProfilerGuid);
+        }
+    }
+}
diff --git a/src/tests/profiler/gc/gcbasic.csproj b/src/tests/profiler/gc/gcbasic.csproj
new file mode 100644 (file)
index 0000000..7a525ce
--- /dev/null
@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworkIdentifier>.NETCoreApp</TargetFrameworkIdentifier>
+    <OutputType>exe</OutputType>
+    <CLRTestKind>BuildAndRun</CLRTestKind>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <CLRTestPriority>0</CLRTestPriority>
+    <Optimize>true</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+    <ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
+    <ProjectReference Include="../common/profiler_common.csproj" />
+    <ProjectReference Include="$(MSBuildThisFileDirectory)/../native/CMakeLists.txt" />
+  </ItemGroup>
+</Project>
index a742c92..b58bb64 100644 (file)
@@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 3.14.5)
 project(Profiler)
 
 set(GCBASIC_SOURCES gcbasicprofiler/gcbasicprofiler.cpp)
+set(GC_SOURCES gcprofiler/gcprofiler.cpp)
 set(REJIT_SOURCES rejitprofiler/rejitprofiler.cpp rejitprofiler/ilrewriter.cpp rejitprofiler/sigparse.cpp)
 set(EVENTPIPE_SOURCES
     eventpipeprofiler/eventpipereadingprofiler.cpp
@@ -14,6 +15,7 @@ set(ELT_SOURCES eltprofiler/slowpatheltprofiler.cpp)
 
 set(SOURCES
     ${GCBASIC_SOURCES}
+    ${GC_SOURCES}
     ${REJIT_SOURCES}
     ${EVENTPIPE_SOURCES}
     ${METADATAGETDISPENSER_SOURCES}
index 301e91d..19de051 100644 (file)
@@ -9,6 +9,7 @@
 #include "metadatagetdispenser/metadatagetdispenser.h"
 #include "getappdomainstaticaddress/getappdomainstaticaddress.h"
 #include "eltprofiler/slowpatheltprofiler.h"
+#include "gcprofiler/gcprofiler.h"
 
 ClassFactory::ClassFactory(REFCLSID clsid) : refCount(0), clsid(clsid)
 {
@@ -63,7 +64,8 @@ HRESULT STDMETHODCALLTYPE ClassFactory::CreateInstance(IUnknown *pUnkOuter, REFI
         new EventPipeWritingProfiler(),
         new MetaDataGetDispenser(),
         new GetAppDomainStaticAddress(),
-        new SlowPathELTProfiler()
+        new SlowPathELTProfiler(),
+        new GCProfiler()
                // add new profilers here
        };
 
diff --git a/src/tests/profiler/native/gcprofiler/gcprofiler.cpp b/src/tests/profiler/native/gcprofiler/gcprofiler.cpp
new file mode 100644 (file)
index 0000000..e6d3edc
--- /dev/null
@@ -0,0 +1,136 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "gcprofiler.h"
+
+GUID GCProfiler::GetClsid()
+{
+    // {BCD8186F-1EEC-47E9-AFA7-396F879382C3}
+    GUID clsid = { 0xBCD8186F, 0x1EEC, 0x47E9, { 0xAF, 0xA7, 0x39, 0x6F, 0x87, 0x93, 0x82, 0xC3 } };
+    return clsid;
+}
+
+HRESULT GCProfiler::Initialize(IUnknown* pICorProfilerInfoUnk)
+{
+    Profiler::Initialize(pICorProfilerInfoUnk);
+
+    HRESULT hr = S_OK;
+    if (FAILED(hr = pCorProfilerInfo->SetEventMask2(COR_PRF_MONITOR_GC, 0)))
+    {
+        _failures++;
+        printf("FAIL: ICorProfilerInfo::SetEventMask2() failed hr=0x%x", hr);
+        return hr;
+    }
+
+    return S_OK;
+}
+
+HRESULT GCProfiler::Shutdown()
+{
+    Profiler::Shutdown();
+
+    if (_gcStarts == 0)
+    {
+        printf("GCProfiler::Shutdown: FAIL: Expected GarbaseCollectionStarted to be called\n");
+    }
+    else if (_gcFinishes == 0)
+    {
+        printf("GCProfiler::Shutdown: FAIL: Expected GarbageCollectionFinished to be called\n");
+    }
+    else if (_pohObjectsSeenRootReferences == 0 || _pohObjectsSeenObjectReferences == 0)
+    {
+        printf("GCProfiler::Shutdown: FAIL: no POH objects seen. root references=%d object references=%d\n",
+            _pohObjectsSeenRootReferences.load(), _pohObjectsSeenObjectReferences.load());
+    }
+    else if(_failures == 0)
+    {
+        printf("PROFILER TEST PASSES\n");
+    }
+    else
+    {
+        // failures were printed earlier when _failures was incremented
+    }
+    fflush(stdout);
+
+    return S_OK;
+}
+
+HRESULT GCProfiler::GarbageCollectionStarted(int cGenerations, BOOL generationCollected[], COR_PRF_GC_REASON reason)
+{
+    _gcStarts++;
+    if (_gcStarts - _gcFinishes > 2)
+    {
+        _failures++;
+        printf("GCProfiler::GarbageCollectionStarted: FAIL: Expected GCStart <= GCFinish+2. GCStart=%d, GCFinish=%d\n", (int)_gcStarts, (int)_gcFinishes);
+    }
+
+    return S_OK;
+}
+
+HRESULT GCProfiler::GarbageCollectionFinished()
+{
+    _gcFinishes++;
+    if (_gcStarts < _gcFinishes)
+    {
+        _failures++;
+        printf("GCProfiler::GarbageCollectionFinished: FAIL: Expected GCStart >= GCFinish. Start=%d, Finish=%d\n", (int)_gcStarts, (int)_gcFinishes);
+    }
+
+    _pohObjectsSeenObjectReferences += NumPOHObjectsSeen(_objectReferencesSeen);
+    _pohObjectsSeenRootReferences += NumPOHObjectsSeen(_rootReferencesSeen);
+    
+    return S_OK;
+}
+
+HRESULT GCProfiler::ObjectReferences(ObjectID objectId, ClassID classId, ULONG cObjectRefs, ObjectID objectRefIds[])
+{
+    HRESULT hr = S_OK;
+    for (ULONG i = 0; i < cObjectRefs; ++i)
+    {
+        ObjectID obj = objectRefIds[i];
+        if (obj != NULL)
+        {
+            _objectReferencesSeen.insert(obj);
+        }
+    }
+
+    return S_OK;
+}
+
+HRESULT GCProfiler::RootReferences(ULONG cRootRefs, ObjectID rootRefIds[])
+{    
+    for (ULONG i = 0; i < cRootRefs; ++i)
+    {
+        ObjectID obj = rootRefIds[i];
+        if (obj != NULL)
+        {
+            _rootReferencesSeen.insert(obj);
+        }
+    }
+
+    return S_OK;
+}
+
+int GCProfiler::NumPOHObjectsSeen(std::unordered_set<ObjectID> objects)
+{
+    int count = 0;
+    for (auto it = objects.begin(); it != objects.end(); ++it)
+    {
+        COR_PRF_GC_GENERATION_RANGE gen;
+        ObjectID obj = *it;
+        HRESULT hr = pCorProfilerInfo->GetObjectGeneration(obj, &gen);
+        if (FAILED(hr))
+        {
+            printf("GetObjectGeneration failed hr=0x%x\n", hr);
+            return hr;
+        }
+
+        if (gen.generation == COR_PRF_GC_PINNED_OBJECT_HEAP)
+        {
+            count++;
+        }
+
+    }
+
+    return count;
+}
diff --git a/src/tests/profiler/native/gcprofiler/gcprofiler.h b/src/tests/profiler/native/gcprofiler/gcprofiler.h
new file mode 100644 (file)
index 0000000..8e0fbeb
--- /dev/null
@@ -0,0 +1,44 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#pragma once
+
+#include "../profiler.h"
+#include <unordered_set>
+
+
+// TODO: this class is intended to be the heavyweight GC API verification (i.e. COR_PRF_MONITOR_GC)
+// vs GCBasicProfiler which verifies COR_PRF_HIGH_BASIC_GC. Right now the only thing it does
+// is verify we see POH objects in the RootReferences callback, but it should be fleshed out.
+class GCProfiler : public Profiler
+{
+public:
+    GCProfiler() : Profiler(),
+        _gcStarts(0),
+        _gcFinishes(0),
+        _failures(0),
+        _pohObjectsSeenRootReferences(0),
+        _pohObjectsSeenObjectReferences(0),
+        _rootReferencesSeen(),
+        _objectReferencesSeen()
+    {}
+
+    virtual GUID GetClsid();
+    virtual HRESULT STDMETHODCALLTYPE Initialize(IUnknown* pICorProfilerInfoUnk);
+    virtual HRESULT STDMETHODCALLTYPE Shutdown();
+    virtual HRESULT STDMETHODCALLTYPE GarbageCollectionStarted(int cGenerations, BOOL generationCollected[], COR_PRF_GC_REASON reason);
+    virtual HRESULT STDMETHODCALLTYPE GarbageCollectionFinished();
+    virtual HRESULT STDMETHODCALLTYPE ObjectReferences(ObjectID objectId, ClassID classId, ULONG cObjectRefs, ObjectID objectRefIds[]);
+    virtual HRESULT STDMETHODCALLTYPE RootReferences(ULONG cRootRefs, ObjectID rootRefIds[]);
+
+private:
+    std::atomic<int> _gcStarts;
+    std::atomic<int> _gcFinishes;
+    std::atomic<int> _failures;
+    std::atomic<int> _pohObjectsSeenRootReferences;
+    std::atomic<int> _pohObjectsSeenObjectReferences;
+    std::unordered_set<ObjectID> _rootReferencesSeen;
+    std::unordered_set<ObjectID> _objectReferencesSeen;
+
+    int NumPOHObjectsSeen(std::unordered_set<ObjectID> objects);
+};