From fcbace6643b68782ce78c4d02e9d9fb57990e92c Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 16 Sep 2019 14:45:46 -0700 Subject: [PATCH] Enable minidumps on all helix runs. (dotnet/coreclr#26455) * Set COMPlus_DbgEnableMiniDump on all helix test runs. * Set dump output path for *nix builds. * Enable collecting dumps for timed-out tests on Windows. * Enable xunit wrapper to get dumps for timed-out tests on non-Windows. * Use sudo for createdump * Update coredump-on-crash pattern to use the $HELIX_WORKITEM_UPLOAD_ROOT environment variable. * Implement linux-specific path for finding a child process by name. * Use HELIX_DUMP_FOLDER instead of HELIX_WORKITEM_UPLOAD_ROOT * Remove empty entries in childrenPidAsStrings. * Look up createdump in Core_Root * Implement timeout macOS crash dumps. * Add ulimit -c unlimited to the Helix script on MacOS. * Get MacOS timeout dumps working (and enable assert/exception dumps) * Copy OSX aborted test dumps to crash dump folder. * Add missing return true. * Allow overwriting dump files on copy (on macOS). * Fix accidental shadow. * Allow multiple spaces in ps output. * Fix dump on windows hitting an assert on checked coreclr. * TEMPORARY: Add more logging in macOS case. * Fix change. * Fix macOS timeout crash dump creation (at least locally). * Fix Windows timeout dump collection. * Explicitly use WChar structures. * TEMPORARY: Test OSX dumps on 10.14 queue. * Fix __CrashDumpFolder on OSX * Don't try to copy the dump if it doesn't exist (10.13 Helix queue doesn't support it yet). * Remove temporary OSX 10.14 queue usage in prs. * Add tracking issue for empty HELIX_DUMP_FOLDER env var on OSX. * PR Feedback Commit migrated from https://github.com/dotnet/coreclr/commit/f5437ac095272f1ab5e8ca79960add974fa96f2d --- src/coreclr/tests/helixpublishwitharcade.proj | 10 +- .../Coreclr.TestWrapper/CoreclrTestWrapperLib.cs | 173 ++++++++++++++++++--- src/coreclr/tests/testenvironment.proj | 4 + 3 files changed, 166 insertions(+), 21 deletions(-) diff --git a/src/coreclr/tests/helixpublishwitharcade.proj b/src/coreclr/tests/helixpublishwitharcade.proj index 3548018..673c29e 100644 --- a/src/coreclr/tests/helixpublishwitharcade.proj +++ b/src/coreclr/tests/helixpublishwitharcade.proj @@ -193,6 +193,8 @@ + + @@ -204,11 +206,13 @@ + + + - - - + + diff --git a/src/coreclr/tests/src/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs b/src/coreclr/tests/src/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs index c946e2e..22c4fbf 100644 --- a/src/coreclr/tests/src/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs +++ b/src/coreclr/tests/src/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs @@ -9,6 +9,7 @@ using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; using System.Text; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; @@ -63,7 +64,8 @@ namespace CoreclrTestLib TH32CS_SNAPTHREAD = 0x00000004 }; - public unsafe struct ProcessEntry32 + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + public unsafe struct ProcessEntry32W { public int Size; public int Usage; @@ -84,10 +86,31 @@ namespace CoreclrTestLib public static extern IntPtr CreateToolhelp32Snapshot(Toolhelp32Flags flags, int processId); [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] - public static extern bool Process32First(IntPtr snapshot, ref ProcessEntry32 entry); + public static extern bool Process32FirstW(IntPtr snapshot, ref ProcessEntry32W entry); [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] - public static extern bool Process32Next(IntPtr snapshot, ref ProcessEntry32 entry); + public static extern bool Process32NextW(IntPtr snapshot, ref ProcessEntry32W entry); + } + + static class libSystem + { + [DllImport(nameof(libSystem))] + public static extern int kill(int pid, int signal); + + public const int SIGABRT = 0x6; + } + + static class libproc + { + [DllImport(nameof(libproc))] + private static extern int proc_listchildpids(int ppid, int[] buffer, int byteSize); + + public static unsafe bool ListChildPids(int ppid, out int[] buffer) + { + int n = proc_listchildpids(ppid, null, 0); + buffer = new int[n]; + return proc_listchildpids(ppid, buffer, buffer.Length * sizeof(int)) != -1; + } } public class CoreclrTestWrapperLib @@ -103,15 +126,65 @@ namespace CoreclrTestLib static bool CollectCrashDump(Process process, string path) { - using (var crashDump = File.OpenWrite(path)) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + using (var crashDump = File.OpenWrite(path)) + { + var flags = DbgHelp.MiniDumpType.MiniDumpWithFullMemory | DbgHelp.MiniDumpType.MiniDumpIgnoreInaccessibleMemory; + return DbgHelp.MiniDumpWriteDump(process.Handle, process.Id, crashDump.SafeFileHandle, flags, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero); + } + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { - var flags = DbgHelp.MiniDumpType.MiniDumpWithFullMemory | DbgHelp.MiniDumpType.MiniDumpIgnoreInaccessibleMemory; - return DbgHelp.MiniDumpWriteDump(process.Handle, process.Id, crashDump.SafeFileHandle, flags, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero); + string coreRoot = Environment.GetEnvironmentVariable("CORE_ROOT"); + ProcessStartInfo createdumpInfo = new ProcessStartInfo("sudo"); + createdumpInfo.Arguments = $"{Path.Combine(coreRoot, "createdump")} --name \"{path}\" {process.Id} -h"; + Process createdump = Process.Start(createdumpInfo); + return createdump.WaitForExit(DEFAULT_TIMEOUT) && createdump.ExitCode == 0; } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + int pid = process.Id; + Console.WriteLine($"Aborting process {pid} to generate dump"); + int status = libSystem.kill(pid, libSystem.SIGABRT); + + string defaultCoreDumpPath = $"/cores/core.{pid}"; + + if (status == 0 && File.Exists(defaultCoreDumpPath)) + { + Console.WriteLine($"Moving dump for {pid} to {path}."); + File.Move(defaultCoreDumpPath, path, true); + } + else + { + Console.WriteLine($"Unable to find OS-generated dump for {pid} at default path: {defaultCoreDumpPath}"); + } + return true; + } + + return false; } static unsafe bool TryFindChildProcessByName(Process process, string childName, out Process child) { + child = null; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return TryFindChildProcessByNameWindows(process, childName, out child); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return TryFindChildProcessByNameLinux(process, childName, out child); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return TryFindChildProcessByNameMacOS(process, childName, out child); + } + return false; + } + + static unsafe bool TryFindChildProcessByNameWindows(Process process, string childName, out Process child) + { IntPtr snapshot = Kernel32.CreateToolhelp32Snapshot(Kernel32.Toolhelp32Flags.TH32CS_SNAPPROCESS, 0); if (snapshot == IntPtr.Zero) { @@ -123,9 +196,9 @@ namespace CoreclrTestLib { int ppid = process.Id; - var processEntry = new Kernel32.ProcessEntry32 { Size = sizeof(Kernel32.ProcessEntry32) }; + var processEntry = new Kernel32.ProcessEntry32W { Size = sizeof(Kernel32.ProcessEntry32W) }; - bool success = Kernel32.Process32First(snapshot, ref processEntry); + bool success = Kernel32.Process32FirstW(snapshot, ref processEntry); while (success) { if (processEntry.ParentProcessID == ppid) @@ -143,7 +216,7 @@ namespace CoreclrTestLib catch {} } - success = Kernel32.Process32Next(snapshot, ref processEntry); + success = Kernel32.Process32NextW(snapshot, ref processEntry); } child = null; @@ -155,6 +228,76 @@ namespace CoreclrTestLib } } + static bool TryFindChildProcessByNameLinux(Process process, string childName, out Process child) + { + Queue childrenFilesToCheck = new Queue(); + + childrenFilesToCheck.Enqueue($"/proc/{process.Id}/task/{process.Id}/children"); + + while (childrenFilesToCheck.Count != 0) + { + string childrenFile = childrenFilesToCheck.Dequeue(); + + try + { + string[] childrenPidAsStrings = File.ReadAllText(childrenFile).Split(' ', StringSplitOptions.RemoveEmptyEntries); + foreach (var childPidAsString in childrenPidAsStrings) + { + int childPid = int.Parse(childPidAsString); + Process childProcess = Process.GetProcessById(childPid); + if (childProcess.ProcessName.Equals(childName, StringComparison.Ordinal)) + { + child = childProcess; + return true; + } + else + { + childrenFilesToCheck.Enqueue($"/proc/{childPid}/task/{childPid}/children"); + } + } + } + catch (IOException) + { + // Ignore failure to read process children data, the process may have exited. + } + } + + child = null; + return false; + } + + static bool TryFindChildProcessByNameMacOS(Process process, string childName, out Process child) + { + child = null; + Queue childrenPidsToCheck = new Queue(); + + childrenPidsToCheck.Enqueue(process.Id); + + while (childrenPidsToCheck.Count != 0) + { + int pid = childrenPidsToCheck.Dequeue(); + if (libproc.ListChildPids(pid, out int[] children)) + { + foreach (var childPid in children) + { + Process childProcess = Process.GetProcessById(childPid); + if (childProcess.ProcessName.Equals(childName, StringComparison.Ordinal)) + { + child = childProcess; + return true; + } + else + { + childrenPidsToCheck.Enqueue(childPid); + } + } + } + } + + child = null; + return false; + } + public int RunTest(string executable, string outputFile, string errorFile) { Debug.Assert(outputFile != errorFile); @@ -165,13 +308,8 @@ namespace CoreclrTestLib // timeout. string environmentVar = Environment.GetEnvironmentVariable(TIMEOUT_ENVIRONMENT_VAR); int timeout = environmentVar != null ? int.Parse(environmentVar) : DEFAULT_TIMEOUT; - - // Check if we are running in Windows - string operatingSystem = System.Environment.GetEnvironmentVariable("OS"); - bool runningInWindows = (operatingSystem != null && operatingSystem.StartsWith("Windows")); - - // We can't yet take crash dumps on non-Windows OSs for timed-out tests - bool collectCrashDumps = runningInWindows && Environment.GetEnvironmentVariable(COLLECT_DUMPS_ENVIRONMENT_VAR) != null; + bool collectCrashDumps = Environment.GetEnvironmentVariable(COLLECT_DUMPS_ENVIRONMENT_VAR) != null; + string crashDumpFolder = Environment.GetEnvironmentVariable(CRASH_DUMP_FOLDER_ENVIRONMENT_VAR); var outputStream = new FileStream(outputFile, FileMode.Create); var errorStream = new FileStream(errorFile, FileMode.Create); @@ -181,7 +319,7 @@ namespace CoreclrTestLib using (Process process = new Process()) { // Windows can run the executable implicitly - if (runningInWindows) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { process.StartInfo.FileName = executable; } @@ -222,7 +360,6 @@ namespace CoreclrTestLib if (collectCrashDumps) { - string crashDumpFolder = Environment.GetEnvironmentVariable(CRASH_DUMP_FOLDER_ENVIRONMENT_VAR); if (crashDumpFolder != null) { Process childProcess; diff --git a/src/coreclr/tests/testenvironment.proj b/src/coreclr/tests/testenvironment.proj index 7a0aa63..e3ddfe0 100644 --- a/src/coreclr/tests/testenvironment.proj +++ b/src/coreclr/tests/testenvironment.proj @@ -10,6 +10,8 @@ COMPlus_TieredCompilation; + COMPlus_DbgEnableMiniDump; + COMPlus_DbgMiniDumpName; COMPlus_EnableAES; COMPlus_EnableAVX; COMPlus_EnableAVX2; @@ -48,6 +50,8 @@ 0 + 1 + $HELIX_DUMP_FOLDER/coredump.%d.dmp -- 2.7.4