From ac1af50ee7a32dabccbe744c99ac7a78ca865cad Mon Sep 17 00:00:00 2001 From: Andrew Au Date: Wed, 29 Jan 2020 16:16:59 -0800 Subject: [PATCH] Improve GC stress tests (#2166) --- docs/project/garbage-collector-guidelines.md | 2 +- .../testing/coreclr/gc-stress-run-readme.md | 46 ++++++++++---------- src/coreclr/tests/src/GC/Stress/Tests/573277.cs | 2 +- .../tests/src/GC/Stress/Tests/DirectedGraph.cs | 2 +- .../tests/src/GC/Stress/Tests/ExpandHeap.cs | 2 +- src/coreclr/tests/src/GC/Stress/Tests/GCQueue.cs | 10 ++--- src/coreclr/tests/src/GC/Stress/Tests/GCVariant.cs | 16 +++---- .../tests/src/GC/Stress/Tests/LargeObjectAlloc.cs | 2 +- .../tests/src/GC/Stress/Tests/LargeObjectAlloc2.cs | 2 +- .../tests/src/GC/Stress/Tests/LargeObjectAlloc3.cs | 2 +- .../tests/src/GC/Stress/Tests/LargeObjectAlloc4.cs | 2 +- .../src/GC/Stress/Tests/LargeObjectAllocPinned.cs | 2 +- .../tests/src/GC/Stress/Tests/LeakGenThrd.cs | 10 ++--- .../tests/src/GC/Stress/Tests/MulDimJagAry.cs | 2 +- src/coreclr/tests/src/GC/Stress/Tests/PlugGaps.cs | 2 +- .../tests/src/GC/Stress/Tests/SingLinkStay.cs | 10 ++--- .../tests/src/GC/Stress/Tests/ThdTreeGrowingObj.cs | 6 +-- .../src/GC/Stress/Tests/allocationwithpins.cs | 2 +- src/coreclr/tests/src/GC/Stress/Tests/b115557.cs | 2 +- .../tests/src/GC/Stress/Tests/bestfit-finalize.cs | 2 +- .../tests/src/GC/Stress/Tests/doubLinkStay.cs | 10 ++--- src/coreclr/tests/src/GC/Stress/testmix_gc.config | 49 +++++++++++++++++++++- 22 files changed, 117 insertions(+), 68 deletions(-) rename src/coreclr/tests/src/GC/Stress/stress_run_readme.txt => docs/workflow/testing/coreclr/gc-stress-run-readme.md (53%) diff --git a/docs/project/garbage-collector-guidelines.md b/docs/project/garbage-collector-guidelines.md index 193758e..dfffdd8 100644 --- a/docs/project/garbage-collector-guidelines.md +++ b/docs/project/garbage-collector-guidelines.md @@ -26,7 +26,7 @@ Required Testing: Validation of the behavior of the affected APIs. ## Stress Testing ## Stress testing must run for at least **48 hours** against a debug build. -Stress testing for checked and release builds can be run locally. Please following the instructions described in tests\src\GC\Stress\stress_run_readme.txt. You can also request it on pull requests with The .NET CI infrastructure with the trigger phrase: +Stress testing for checked and release builds can be run locally. Please following the instructions described in [gc-stress-run-readme.md](https://github.com/dotnet/runtime/blob/master/docs/workflow/testing/coreclr/gc-stress-run-readme.md). You can also request it on pull requests with The .NET CI infrastructure with the trigger phrase: ``` @dotnet_bot test gc_reliability_framework diff --git a/src/coreclr/tests/src/GC/Stress/stress_run_readme.txt b/docs/workflow/testing/coreclr/gc-stress-run-readme.md similarity index 53% rename from src/coreclr/tests/src/GC/Stress/stress_run_readme.txt rename to docs/workflow/testing/coreclr/gc-stress-run-readme.md index f0efa71..ea0c574 100644 --- a/src/coreclr/tests/src/GC/Stress/stress_run_readme.txt +++ b/docs/workflow/testing/coreclr/gc-stress-run-readme.md @@ -1,62 +1,64 @@ -0. Deciding whether you should run stress for GC changes +# Deciding whether you should run stress for GC changes -Most of the code in gc.cpp is intricate - unless you are touching something that you are very confident will not have any adverse effect, you should run stress. When in doubt, feel free to ask. +Most of the code in `gc.cpp` is intricate - unless you are touching something that you are very confident will not have any adverse effect, you should run stress. When in doubt, feel free to ask. -1. What is this stress run? +# What is this stress run? -This was adapted from one of the ways we run stress internally. The idea is that it takes a config that specifies the tests to run and runs them all in one process to stress random combinations of allocation and survival patterns. +This was adapted from one of the ways we run stress internally. The idea is that it takes a config that specifies the tests to run and runs them all in one process to stress random combinations of allocation and survival patterns. Note that these tests were picked from functional tests so some of them could have failures because some condition it checks for is not met. In stress runs, we only care about AVs so we want to run them as long as possible and don't care about failures indicated by the tests themselves. This is a pretty crude implementation. Feel free to improve it! -2. Setting up stress +# Setting up stress It has 3 parts: -The stress framework is built from \tests\src\GC\Stress\Framework +- The stress framework is built from `\src\coreclr\tests\src\GC\Stress\Framework` -The tests are built from \tests\src\GC\Stress\Tests +- The tests are built from `\src\coreclr\tests\src\GC\Stress\Tests` -The config is at \tests\src\GC\Stress\testmix_gc.config, this will be copied to the output folder of Framework +- The config is at `\src\coreclr\tests\src\GC\Stress\testmix_gc.config`, this will be copied to the output folder of Framework -The easiest way to build the Framework+Tests is by building all tests - "\build_test[.bat|.sh]" +The easiest way to build the Framework+Tests is by building all tests - `\src\coreclr\build_test[.bat|.sh]` Sometimes there is a need (after initial build) to rebuild Framework+Tests. For example when modifying the Framework to add a new scenario or when investigating a failure. -In such case it is possible to go directly into the the Framework directory and build manually- Ex: "dotnet build -c:debug". +In such case it is possible to go directly into the the Framework directory and build manually- Ex: `dotnet build -c:debug`. -3. Running stress +# Running stress -The test .exe's need to be in a directory called Tests next to ReliabilityFramework.exe. So if you keep ReliabilityFramework.exe where it is, you should see the test binaries copied to the \GC\Stress\Framework\ReliabilityFramework\Tests. +The test binaries need to be in a directory called Tests next to `ReliabilityFramework.dll`. So if you keep `ReliabilityFramework.dll` where it is, you should see the test binaries copied to the `\GC\Stress\Framework\ReliabilityFramework\Tests`. To run stress: -%CORE_ROOT%\corerun ReliabilityFramework.exe testmix_gc.config +`%CORE_ROOT%\corerun ReliabilityFramework.dll testmix_gc.config` -(or if you copied testmix_gc.config somewhere else you need to tell it so, eg, c:\TestConfigs\testmix_gc.config) +(or if you copied `testmix_gc.config` somewhere else you need to tell it so, eg, `c:\TestConfigs\testmix_gc.config`) -We recommend to run it for 48 hours (see the comments below on maximumExecutionTime in test config for more detail). +We recommend to run it for 48 hours (see the comments below on `maximumExecutionTime` in test config for more detail). -4. Test config +# Test config There are a few interesting things in this config: - suppressConsoleOutputFromTests + `suppressConsoleOutputFromTests` -Set this to true if you want to see the console output from tests. +Set this to true if you want to see the console output from tests. - concurrentCopies + `concurrentCopies` If you specify this to something >1 it will load that many concurrent copies. - maximumExecutionTime + `maximumExecutionTime` Right now I set it to about 15 hours (instead of 48) because some of the tests will keep growing in memory usage. If you don't have a machine with a decent amount of memory you can change this to a smaller amount. You can always run this in a loop in a .cmd file for 48 hours: +``` :test -%CORE_ROOT%\corerun ReliabilityFramework.exe testmix_gc.config +%CORE_ROOT%\corerun ReliabilityFramework.dll testmix_gc.config goto test +``` Feel free to write your own tests and put them in the Tests directory and specify them in the config to run. -If you get an AV, often you would see it in different tests. But if you do get one that's consistently in one test, it's a good sign - it means it's likely that running that one test will give you the AV sooner because it exhibits a certain pattern that causes the AV. +If you get an AV, often you would see it in different tests. But if you do get one that's consistently in one test, it's a good sign - it means it's likely that running that one test will give you the AV sooner because it exhibits a certain pattern that causes the AV. diff --git a/src/coreclr/tests/src/GC/Stress/Tests/573277.cs b/src/coreclr/tests/src/GC/Stress/Tests/573277.cs index d36397e..8b73da0 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/573277.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/573277.cs @@ -15,7 +15,7 @@ public class Test public static bool fail = false; [System.Security.SecuritySafeCritical] - public static int Main(String[] args) + public static int Main(string[] args) { Thread[] threads = new Thread[Math.Max(Environment.ProcessorCount * 2, 64)]; for (int i = 0; i < threads.Length; i++) diff --git a/src/coreclr/tests/src/GC/Stress/Tests/DirectedGraph.cs b/src/coreclr/tests/src/GC/Stress/Tests/DirectedGraph.cs index cb3a0c0..ff32afc 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/DirectedGraph.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/DirectedGraph.cs @@ -368,7 +368,7 @@ namespace DefaultNamespace public class Test { - public static int Main() + public static int Main(string[] args) { TestLibrary.Logging.WriteLine("Building Graph with 800 vertices..."); Graph MyGraph = new Graph(800); // graph with 800 nodes diff --git a/src/coreclr/tests/src/GC/Stress/Tests/ExpandHeap.cs b/src/coreclr/tests/src/GC/Stress/Tests/ExpandHeap.cs index 7731ce5..0ff8201 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/ExpandHeap.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/ExpandHeap.cs @@ -106,7 +106,7 @@ public class GCUtil public class Test { - public static int Main(System.String[] Args) + public static int Main(string[] args) { Console.WriteLine("First Alloc"); GCUtil.Alloc(1024 * 1024 * 4, 30); diff --git a/src/coreclr/tests/src/GC/Stress/Tests/GCQueue.cs b/src/coreclr/tests/src/GC/Stress/Tests/GCQueue.cs index 0089632..7ec0565 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/GCQueue.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/GCQueue.cs @@ -40,26 +40,26 @@ namespace DefaultNamespace internal class GCQueue { internal static Queue G_Queue; - public static int Main(String[] Args) + public static int Main(string[] args) { int iRep = 0; int iObj = 0; Console.Out.WriteLine("Test should return with ExitCode 100 ..."); - switch (Args.Length) + switch (args.Length) { case 1: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 5; } break; case 2: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 5; } - if (!Int32.TryParse(Args[1], out iObj)) + if (!Int32.TryParse(args[1], out iObj)) { iObj = 5000; } diff --git a/src/coreclr/tests/src/GC/Stress/Tests/GCVariant.cs b/src/coreclr/tests/src/GC/Stress/Tests/GCVariant.cs index f88f8d6..96d28d2 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/GCVariant.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/GCVariant.cs @@ -15,17 +15,17 @@ namespace GCVariant { internal static object[] G_Vart; - public static int Main(String[] Args) + public static int Main(string[] args) { int iRep = 0; int iObj = 0; int iNum = 0; Console.WriteLine("Test should return with ExitCode 100 ..."); - switch (Args.Length) + switch (args.Length) { case 1: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 5; } @@ -34,11 +34,11 @@ namespace GCVariant break; case 2: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 5; } - if (!Int32.TryParse(Args[1], out iObj)) + if (!Int32.TryParse(args[1], out iObj)) { iObj = 100; } @@ -46,15 +46,15 @@ namespace GCVariant break; case 3: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 5; } - if (!Int32.TryParse(Args[1], out iObj)) + if (!Int32.TryParse(args[1], out iObj)) { iObj = 100; } - if (!Int32.TryParse(Args[2], out iNum)) + if (!Int32.TryParse(args[2], out iNum)) { iNum = 10; } diff --git a/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc.cs b/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc.cs index d273016..7607a19 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc.cs @@ -29,7 +29,7 @@ internal class Mainy } } - public static int Main(String[] args) + public static int Main(string[] args) { long Threads = 1; diff --git a/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc2.cs b/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc2.cs index 86a27d9..e6a8c75 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc2.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc2.cs @@ -75,7 +75,7 @@ namespace LargeObjectTest return true; } - public static int Main() + public static int Main(string[] args) { int loop = 0; diff --git a/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc3.cs b/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc3.cs index 522a753..cb90bda 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc3.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc3.cs @@ -49,7 +49,7 @@ namespace LargeObjectTest { public static int ExitCode = 1; - public static int Main() + public static int Main(string[] args) { int size = 1; int loop = 1; diff --git a/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc4.cs b/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc4.cs index 9a940f1..d690823 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc4.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAlloc4.cs @@ -9,7 +9,7 @@ using System; public class Test { - public static int Main() + public static int Main(string[] args) { Int32 basesize; Int32[] largeobjarr; diff --git a/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAllocPinned.cs b/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAllocPinned.cs index 4f255fa..9e28caf 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAllocPinned.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/LargeObjectAllocPinned.cs @@ -8,7 +8,7 @@ using System; internal class LargePinned { [System.Security.SecuritySafeCritical] - unsafe public static int Main(String[] args) + unsafe public static int Main(string[] args) { for (int i = 0; i < 25; i++) { diff --git a/src/coreclr/tests/src/GC/Stress/Tests/LeakGenThrd.cs b/src/coreclr/tests/src/GC/Stress/Tests/LeakGenThrd.cs index f01f8e5..b22313e 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/LeakGenThrd.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/LeakGenThrd.cs @@ -19,25 +19,25 @@ namespace LGen internal int Cv_iCounter = 0; internal int Cv_iRep; - public static int Main(System.String[] Args) + public static int Main(string[] args) { int iRep = 2; int iObj = 15; //the number of MB memory will be allocted in MakeLeak() - switch (Args.Length) + switch (args.Length) { case 1: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 2; } break; case 2: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 2; } - if (!Int32.TryParse(Args[1], out iObj)) + if (!Int32.TryParse(args[1], out iObj)) { iObj = 15; } diff --git a/src/coreclr/tests/src/GC/Stress/Tests/MulDimJagAry.cs b/src/coreclr/tests/src/GC/Stress/Tests/MulDimJagAry.cs index 79d8435..97cb477 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/MulDimJagAry.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/MulDimJagAry.cs @@ -21,7 +21,7 @@ namespace DefaultNamespace internal class MulDimJagAry { - public static int Main(String[] args) + public static int Main(string[] args) { int iDim1 = 100; int iDim2 = 100; diff --git a/src/coreclr/tests/src/GC/Stress/Tests/PlugGaps.cs b/src/coreclr/tests/src/GC/Stress/Tests/PlugGaps.cs index 8ba0fc8..2c9a222 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/PlugGaps.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/PlugGaps.cs @@ -110,7 +110,7 @@ public class Test public static List gchList = new List(); public static List bList = new List(); - public static int Main(System.String[] Args) + public static int Main(string[] args) { Console.WriteLine("Beginning phase 1"); GCUtil.AllocWithGaps(); diff --git a/src/coreclr/tests/src/GC/Stress/Tests/SingLinkStay.cs b/src/coreclr/tests/src/GC/Stress/Tests/SingLinkStay.cs index d80c2c6..d45bda0 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/SingLinkStay.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/SingLinkStay.cs @@ -23,26 +23,26 @@ namespace SingLink { internal SingLink[] Mv_Sing; - public static int Main(System.String[] Args) + public static int Main(string[] args) { int iRep = 0; int iObj = 0; Console.WriteLine("Test should return with ExitCode 100 ..."); - switch (Args.Length) + switch (args.Length) { case 1: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 100; } break; case 2: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 100; } - if (!Int32.TryParse(Args[1], out iObj)) + if (!Int32.TryParse(args[1], out iObj)) { iObj = 10; } diff --git a/src/coreclr/tests/src/GC/Stress/Tests/ThdTreeGrowingObj.cs b/src/coreclr/tests/src/GC/Stress/Tests/ThdTreeGrowingObj.cs index 2562889..42ad6fc 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/ThdTreeGrowingObj.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/ThdTreeGrowingObj.cs @@ -230,13 +230,13 @@ namespace DefaultNamespace public class ThdTreeGrowingObj { - public static int Main(System.String[] Args) + public static int Main(string[] args) { int iNofThread = 0; - if (Args.Length == 1) + if (args.Length == 1) { - if (!Int32.TryParse(Args[0], out iNofThread)) + if (!Int32.TryParse(args[0], out iNofThread)) { iNofThread = 2; } diff --git a/src/coreclr/tests/src/GC/Stress/Tests/allocationwithpins.cs b/src/coreclr/tests/src/GC/Stress/Tests/allocationwithpins.cs index d23a1bd..2ce0475 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/allocationwithpins.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/allocationwithpins.cs @@ -492,7 +492,7 @@ internal class FreeListTest } } - public static int Main(String[] args) + public static int Main(string[] args) { if (GCSettings.IsServerGC == true) { diff --git a/src/coreclr/tests/src/GC/Stress/Tests/b115557.cs b/src/coreclr/tests/src/GC/Stress/Tests/b115557.cs index 5906b5c..ba0a46b 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/b115557.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/b115557.cs @@ -42,7 +42,7 @@ internal class B115557 { private static int s_allocPerThreadMB = 120; - public static int Main(String[] args) + public static int Main(string[] args) { //check if total allocation size is not too much for x86 //Allocate at most 1700MB on x86 to avoid the risk of getting OOM. diff --git a/src/coreclr/tests/src/GC/Stress/Tests/bestfit-finalize.cs b/src/coreclr/tests/src/GC/Stress/Tests/bestfit-finalize.cs index ad308a1..a4d5e7c 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/bestfit-finalize.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/bestfit-finalize.cs @@ -128,7 +128,7 @@ namespace Fragment } - static public int Main(String[] args) + static public int Main(string[] args) { int numIterations = 0; int randomSeed = 0; diff --git a/src/coreclr/tests/src/GC/Stress/Tests/doubLinkStay.cs b/src/coreclr/tests/src/GC/Stress/Tests/doubLinkStay.cs index 78e81f4..4aa264c 100644 --- a/src/coreclr/tests/src/GC/Stress/Tests/doubLinkStay.cs +++ b/src/coreclr/tests/src/GC/Stress/Tests/doubLinkStay.cs @@ -25,27 +25,27 @@ namespace DoubLink { internal DoubLink[] Mv_Doub; - public static int Main(System.String[] Args) + public static int Main(string[] args) { int iRep = 100; int iObj = 10; Console.WriteLine("Test should return with ExitCode 100 ..."); - switch (Args.Length) + switch (args.Length) { case 1: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 100; } break; case 2: - if (!Int32.TryParse(Args[0], out iRep)) + if (!Int32.TryParse(args[0], out iRep)) { iRep = 100; } - if (!Int32.TryParse(Args[1], out iObj)) + if (!Int32.TryParse(args[1], out iObj)) { iObj = 10; } diff --git a/src/coreclr/tests/src/GC/Stress/testmix_gc.config b/src/coreclr/tests/src/GC/Stress/testmix_gc.config index ce26d46..bc118b0 100644 --- a/src/coreclr/tests/src/GC/Stress/testmix_gc.config +++ b/src/coreclr/tests/src/GC/Stress/testmix_gc.config @@ -12,7 +12,54 @@ minimumTests="0" minMaxTestUseCPUCount="true" suppressConsoleOutputFromTests="false"> - + + + + + + + +