From 70e7fa10827d039dcf2f3c59284da85eac02ac09 Mon Sep 17 00:00:00 2001 From: Jan Vorlicek Date: Wed, 23 Jan 2019 22:59:27 +0100 Subject: [PATCH] Add unloadability testing tool (#22064) * Add unloadability testing tool This change adds the unloadability testing tool that can be used to run coreclr tests or any other .NET core app inside of unloadable AssemblyLoadContext, unload it after its execution completes and verify that the unload succeeded. It has also various additional testing options: * memory leak testing * running multiple iterations of the load/run/unload sequence * optional breaking into debugger at various interesting stages (before executing the test assembly, after executing it, on unload failure) * delegated load when the AssemblyLoadContext that loads the test assembly delegates loading of all the dependencies to another AssemblyLoadContext --- tests/scripts/runincontext.cmd | 16 + tests/src/runincontext/runincontext.cs | 568 +++++++++++++++++++++++++++++ tests/src/runincontext/runincontext.csproj | 15 + 3 files changed, 599 insertions(+) create mode 100644 tests/scripts/runincontext.cmd create mode 100644 tests/src/runincontext/runincontext.cs create mode 100644 tests/src/runincontext/runincontext.csproj diff --git a/tests/scripts/runincontext.cmd b/tests/scripts/runincontext.cmd new file mode 100644 index 0000000..5b1d70c --- /dev/null +++ b/tests/scripts/runincontext.cmd @@ -0,0 +1,16 @@ +@rem This script is a bridge that allows .cmd files of individual tests to run the respective test executables +@rem in an unloadable AssemblyLoadContext. +@rem +@rem To use this script, set the CLRCustomTestLauncher environment variable to the full path of this script. +@rem +@rem Additional command line arguments can be passed to the runincontext tool by setting the RunInContextExtraArgs +@rem environment variable +@rem +@rem The .cmd files of the individual tests will call this script to launch the test. +@rem This script gets the following arguments +@rem 1. Full path to the directory of the test binaries (the test .cmd file is in there) +@rem 2. Filename of the test executable +@rem 3. - n. Additional arguments that were passed to the test .cmd + +set CORE_LIBRARIES=%1 +%_DebuggerFullPath% "%CORE_ROOT%\corerun.exe" "%CORE_ROOT%\..\..\runincontext\runincontext\runincontext.exe" %RunInContextExtraArgs% /referencespath:%CORE_ROOT%\ %1%2 %3 %4 %5 %6 %7 %8 %9 diff --git a/tests/src/runincontext/runincontext.cs b/tests/src/runincontext/runincontext.cs new file mode 100644 index 0000000..06aff6a --- /dev/null +++ b/tests/src/runincontext/runincontext.cs @@ -0,0 +1,568 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +using System; +using System.Reflection; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Threading; +using System.Runtime.CompilerServices; +using System.Runtime.Loader; + +public class ArgInput +{ + public bool Verbose = false; + public string AsmName = null; + public string AssemblyPath = null; + public string[] EntryArgs; + public StringBuilder EntryArgStr = null; + public bool StressMode = false; + public int StressModeCount = 2000; + public int MaxRestarts = 10; + public int IterationCount = 1; + public bool MonitorMode = false; + public int IterationsToSkip = 500; + public string ReferencesPath = null; + public bool BreakBeforeRun; + public bool BreakAfterRun; + public bool DelegateLoad; + public bool BreakOnUnloadFailure; + + public static void DisplayUsage() + { + Console.WriteLine("Usage: RunInContext.exe [options ...] [assembly command line options]"); + Console.WriteLine(" /v Verbose mode"); + Console.WriteLine(" /collectstress: Emit in collectible assembly n times, checking for memory leaks(def:2000)"); + Console.WriteLine(" /maxstressrestarts: Maximum allowed stress run restarts when memory usage increases (def:10)"); + Console.WriteLine(" /iterationcount: Number of iterations in non-stress mode (def:1)"); + Console.WriteLine(" /memorymonitor: Monitor memory usage closely. skip: number of iterations to skip before monitoring memory."); + Console.WriteLine(" /referencespath: Path to resolve assemblies referenced by the main assembly"); + Console.WriteLine(" /breakbeforerun Break into debugger before executing the assembly"); + Console.WriteLine(" /breakafterrun Break into debugger after executing the assembly"); + Console.WriteLine(" /breakonunloadfailure Break into debugger on unload failure"); + Console.WriteLine(" /delegateload Delegate the AssemblyLoadContext.Load to a secondary AssemblyLoadContext"); + } + + public ArgInput(String[] args) + { + EntryArgStr = new StringBuilder(); + var assemblyArgs = new List(); + + for (int i = 0; i < args.Length; i++) + { + string option = args[i].ToLower(); + + if (option.StartsWith("/v")) + { + Verbose = true; + } + else if (option.StartsWith("/collectstress")) + { + StressMode = true; + if (option.Length > 14 && option[14] == ':') + { + StressModeCount = int.Parse(option.Substring(15)); + } + } + else if (option.StartsWith("/iterationcount:")) + { + IterationCount = int.Parse(option.Substring(16)); + } + else if (option.StartsWith("/breakbeforerun")) + { + BreakBeforeRun = true; + } + else if (option.StartsWith("/breakafterrun")) + { + BreakAfterRun = true; + } + else if (option.StartsWith("/breakonunloadfailure")) + { + BreakOnUnloadFailure = true; + } + else if (option.StartsWith("/delegateload")) + { + DelegateLoad = true; + } + else if (option.StartsWith("/maxstressrestarts:")) + { + MaxRestarts = int.Parse(option.Substring(19)); + } + else if (option.StartsWith("/memorymonitor:")) + { + MonitorMode = true; + IterationsToSkip = int.Parse(option.Substring(15)); + } + else if (option.StartsWith("/referencespath:")) + { + ReferencesPath = Path.GetFullPath(args[i].Substring(16)); + } + else + { + // The remaining arguments are the assembly name and its parameters + AsmName = args[i]; + AssemblyPath = Path.GetDirectoryName(Path.GetFullPath(AsmName)); + + for (i++; i < args.Length; i++) + { + assemblyArgs.Add(args[i]); + if (args[i].Contains(" ") || args[i].Contains("\t")) + { + EntryArgStr.Append($"\"{args[i]}\""); + } + else + { + EntryArgStr.Append(args[i]); + } + EntryArgStr.Append(" "); + } + } + } + EntryArgs = assemblyArgs.ToArray(); + + if (StressModeCount < 50) + { + Console.WriteLine("The number of stress runs is less that the minimum (50). Defaulting to 50"); + StressModeCount = 50; + } + if (!MonitorMode && (MaxRestarts < 5)) + { + Console.WriteLine("The number of stress run restarts is less that the minimum (5). Defaulting to 5"); + MaxRestarts = 5; + } + } +} + +abstract class TestAssemblyLoadContextBase : AssemblyLoadContext +{ + public TestAssemblyLoadContextBase() : base(true) + { + + } + public virtual void Cleanup() + { + + } +} + +class TestAssemblyLoadContext : TestAssemblyLoadContextBase +{ + public List _assemblyReferences; + string _assemblyDirectory; + string _referencesDirectory; + + public TestAssemblyLoadContext(string assemblyDirectory, string referencesDirectory, List assemblyReferences) + { + _assemblyDirectory = assemblyDirectory; + _referencesDirectory = referencesDirectory; + _assemblyReferences = assemblyReferences; + } + + protected override Assembly Load(AssemblyName name) + { + Assembly assembly = null; + try + { + assembly = LoadFromAssemblyPath(Path.Combine(_referencesDirectory, name.Name + ".dll")); + } + catch (Exception) + { + try + { + assembly = LoadFromAssemblyPath(Path.Combine(_assemblyDirectory, name.Name + ".dll")); + } + catch (Exception) + { + assembly = LoadFromAssemblyPath(Path.Combine(_assemblyDirectory, name.Name + ".exe")); + } + } + + lock(_assemblyReferences) + { + _assemblyReferences.Add(new WeakReference(assembly)); + } + return assembly; + } +} + +class TestAssemblyLoadContextDelegating : TestAssemblyLoadContextBase +{ + public TestAssemblyLoadContextBase _delegateContext; + + public TestAssemblyLoadContextDelegating(TestAssemblyLoadContextBase delegateContext) + { + _delegateContext = delegateContext; + } + + public override void Cleanup() + { + _delegateContext.Cleanup(); + _delegateContext = null; + } + + protected override Assembly Load(AssemblyName name) + { + Assembly asm = _delegateContext.LoadFromAssemblyName(name); + return asm; + } +} + +public class UnloadFailedException : Exception +{ + +} + +public class TestRunner +{ + ArgInput _input; + + public TestRunner(ArgInput input) + { + _input = input; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public int DoWorkNonStress() + { + int retVal = 0; + + for (int i = 0; i < _input.IterationCount; i++) + { + retVal = ExecuteAssembly(); + if (retVal != RunInContext.SuccessExitCode) + { + break; + } + } + + return retVal; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public int DoWorkStress() + { + //Stress mode + + Process p = Process.GetCurrentProcess(); + long startMemory = p.PrivateMemorySize64; + long currentMemory = startMemory; + long monitorStartMem = 0; + long startSpeed = 0; + long currentSpeed = 0; + int restarts = 0; + long leak = 0; + int i; + int lastProgressReport = 0; + int retVal = 0; + + for (i = 1; i <= _input.StressModeCount; i++) + { + if (!_input.MonitorMode && (((i * 1.0) / _input.StressModeCount) * 100 > lastProgressReport)) + { + Console.WriteLine("Completed: {0}%...", lastProgressReport); + lastProgressReport += 10; + } + + Stopwatch sw = new Stopwatch(); + sw.Start(); + retVal = ExecuteAssembly(); + sw.Stop(); + + if (retVal != RunInContext.SuccessExitCode) + { + break; + } + + currentSpeed = sw.ElapsedMilliseconds; + + p = Process.GetCurrentProcess(); + currentMemory = p.PrivateMemorySize64; + if (_input.MonitorMode) + { + if (i == _input.IterationsToSkip) + { + startMemory = monitorStartMem = currentMemory; + } + + if (currentMemory > startMemory) + { + Console.WriteLine($"\n\n +++ Memory usage increased by {currentMemory - startMemory} bytes at iteration {i}!! "); + if (i > _input.IterationsToSkip) + { + leak = (long)((currentMemory - monitorStartMem) / ((i - _input.IterationsToSkip) * 1.0)); + } + startMemory = currentMemory; + } + else if (currentMemory < startMemory) + { + Console.WriteLine($"\n\n --- Memory usage decreased by {startMemory - currentMemory} bytes at iteration {i} "); + startMemory = currentMemory; + } + + Console.Write($"Private Memory Size = {currentMemory / 1024}K after {i + 1} iterations."); + + if (i > _input.IterationsToSkip) + { + Console.Write($" Average leak: {leak} bytes/iteration. speed: {(int)currentSpeed} ms/type."); + } + + Console.WriteLine(); + } + else + { + if (currentMemory > startMemory) + { + leak = (currentMemory - startMemory) / i; + Console.WriteLine($"LOOP #{i}: Memory usage increased by {_input.MaxRestarts - restarts - 1} bytes! Restarting test... ({currentMemory - startMemory} restarts left)"); + Console.WriteLine($" + Average leak over the last {i} iterations: {leak} bytes\n"); + + restarts++; + if (restarts == _input.MaxRestarts) + { + break; + } + i = 0; + startMemory = currentMemory; + leak = 0; + lastProgressReport = 0; + continue; + } + } + + if ((i == 2) && (startSpeed == 0)) + { + startSpeed = currentSpeed; + } + } + + //sometimes this happens (no real reason, but it's not a failure, so let's not write a "negative" leak to the output) + if (currentMemory < startMemory) + { + startMemory = currentMemory; + leak = 0; + } + if (_input.MonitorMode) + { + leak = (long)((currentMemory - monitorStartMem) / ((_input.StressModeCount * 1.0) - _input.IterationsToSkip)); + startMemory = monitorStartMem; + } + + Console.WriteLine("\n=================================================="); + Console.WriteLine($"Starting memory size : {startMemory / 1024} KB"); + Console.WriteLine($"Ending memory size : {currentMemory / 1024} KB"); + Console.WriteLine(); + Console.WriteLine($"Starting emission speed : {startSpeed} milliseconds"); + Console.WriteLine($"Ending emission speed : {currentSpeed} milliseconds"); + Console.WriteLine(); + Console.WriteLine($"Memory leak : {currentMemory - startMemory} bytes ({leak} bytes per iteration)."); + if (currentMemory > startMemory) + { + throw new Exception("Memory leaked"); + } + + return retVal; + } + + public int ExecuteAssemblyEntryPoint(MethodInfo entryPoint) + { + int result = 0; + + object res; + object[] args = (entryPoint.GetParameters().Length != 0) ? new object[] { _input.EntryArgs } : null; + string argsStr = (args == null) ? "" : _input.EntryArgStr.ToString(); + + if (_input.Verbose) + { + Console.WriteLine($"Invoking Main({argsStr})\n"); + } + + res = entryPoint.Invoke(null, args); + + result = (entryPoint.ReturnType == typeof(void)) ? Environment.ExitCode : Convert.ToInt32(res); + + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public int ExecuteAndUnload(List assemblyReferences, out WeakReference testAlcWeakRef, out WeakReference testAlcWeakRefInner) + { + int result; + TestAssemblyLoadContextBase testAlc = new TestAssemblyLoadContext(_input.AssemblyPath, _input.ReferencesPath, assemblyReferences); + + if (_input.DelegateLoad) + { + testAlcWeakRefInner = new WeakReference(testAlc, trackResurrection: true); + testAlc = new TestAssemblyLoadContextDelegating(testAlc); + } + else + { + testAlcWeakRefInner = new WeakReference(null); + } + + testAlcWeakRef = new WeakReference(testAlc, trackResurrection: true); + + Assembly inputAssembly = null; + try + { + inputAssembly = testAlc.LoadFromAssemblyPath(_input.AsmName); + } + catch (Exception LoadEx) + { + Console.WriteLine($"Failed to load assembly <{_input.AsmName}>!"); + Console.WriteLine($"Exception: {LoadEx.ToString()}"); + throw; + } + + assemblyReferences.Add(new WeakReference(inputAssembly)); + + Stopwatch sw = new Stopwatch(); + sw.Start(); + result = ExecuteAssemblyEntryPoint(inputAssembly.EntryPoint); + sw.Stop(); + + if (_input.Verbose) + { + Console.WriteLine($"Execution time: {sw.Elapsed}"); + + foreach (WeakReference wr in assemblyReferences) + { + if (wr.Target != null) + { + Console.WriteLine("Unloading Assembly [" + wr.Target + "]"); + } + } + } + + testAlc.Cleanup(); + testAlc.Unload(); + + testAlc = null; + inputAssembly = null; + + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + bool VerifyAssembliesUnloaded(List assemblyReferences) + { + bool unloadSucceeded = true; + + foreach (WeakReference wr in assemblyReferences) + { + if (wr.Target != null) + { + if (_input.Verbose) + { + Console.WriteLine("FAILURE: Assembly [" + wr.Target + "] was not unloaded!"); + } + unloadSucceeded = false; + } + } + + return unloadSucceeded; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + int ExecuteAssembly() + { + List assemblyReferences = new List(); + WeakReference testAlcWeakRef; + WeakReference testAlcWeakRefInner; + + if (_input.BreakBeforeRun) + { + Debugger.Break(); + } + + int result = ExecuteAndUnload(assemblyReferences, out testAlcWeakRef, out testAlcWeakRefInner); + + for (int i = 0; (testAlcWeakRef.IsAlive || testAlcWeakRefInner.IsAlive) && (i < 100); i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + + if (_input.BreakAfterRun) + { + Debugger.Break(); + } + + bool unloadSucceeded = VerifyAssembliesUnloaded(assemblyReferences); + + if (!unloadSucceeded) + { + if (_input.BreakOnUnloadFailure) + { + Debugger.Break(); + } + + throw new UnloadFailedException(); + } + + return result; + } +} + +public class RunInContext +{ + public static int FailureExitCode = 213; + public static int SuccessExitCode = 100; + + public static int Main(String[] args) + { + if (args.Length == 0) + { + ArgInput.DisplayUsage(); + return FailureExitCode; + } + + ArgInput input = new ArgInput(args); + TestRunner runner = new TestRunner(input); + + AppDomain.CurrentDomain.UnhandledException += OnUnhandledException; + + int retVal = FailureExitCode; + try + { + if (!input.StressMode) + { + retVal = runner.DoWorkNonStress(); + } + else + { + retVal = runner.DoWorkStress(); + } + } + catch (UnloadFailedException) + { + Console.WriteLine($"FAILURE: Unload failed"); + } + catch (Exception ex) + { + if (input.Verbose) + { + Console.WriteLine($"FAILURE: Exception: {ex.ToString()}"); + } + else + { + Console.WriteLine($"FAILURE: Exception: {ex.Message}"); + } + } + + string status = (retVal == FailureExitCode) ? "FAIL" : "PASS"; + + Console.WriteLine(); + Console.WriteLine($"RunInContext {status}! Exiting with code {retVal}"); + + return retVal; + } + + private static void OnUnhandledException(object sender, UnhandledExceptionEventArgs e) + { + Environment.Exit(FailureExitCode); + } + +} diff --git a/tests/src/runincontext/runincontext.csproj b/tests/src/runincontext/runincontext.csproj new file mode 100644 index 0000000..b5329eb --- /dev/null +++ b/tests/src/runincontext/runincontext.csproj @@ -0,0 +1,15 @@ + + + + Exe + netcoreapp3.0 + $(MicrosoftNETCoreAppPackageVersion) + false + BuildOnly + + + + + + + -- 2.7.4