using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
public const string COLLECT_DUMPS_ENVIRONMENT_VAR = "__CollectDumps";
public const string CRASH_DUMP_FOLDER_ENVIRONMENT_VAR = "__CrashDumpFolder";
- static bool CollectCrashDump(Process process, string path)
+ static bool CollectCrashDump(Process process, string crashDumpPath, StreamWriter outputWriter)
{
string coreRoot = Environment.GetEnvironmentVariable("CORE_ROOT");
string createdumpPath = Path.Combine(coreRoot, "createdump");
- string arguments = $"--name \"{path}\" {process.Id} --withheap";
+ string arguments = $"--name \"{crashDumpPath}\" {process.Id} --withheap";
Process createdump = new Process();
+ bool crashReportPresent = false;
if (OperatingSystem.IsWindows())
{
else if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
{
createdump.StartInfo.FileName = "sudo";
- createdump.StartInfo.Arguments = $"{createdumpPath} " + arguments;
- createdump.StartInfo.EnvironmentVariables.Add("DOTNET_DbgEnableElfDumpOnMacOS", "1");
+ createdump.StartInfo.Arguments = $"{createdumpPath} --crashreport {arguments}";
+ crashReportPresent = true;
}
createdump.StartInfo.UseShellExecute = false;
Console.WriteLine(output);
Console.WriteLine("createdump stderr:");
Console.WriteLine(error);
+
+ if (crashReportPresent)
+ {
+ TryPrintStackTraceFromCrashReport(crashDumpPath + ".crashreport.json", outputWriter);
+ }
}
else
{
return fSuccess && createdump.ExitCode == 0;
}
+ private static List<string> knownNativeModules = new List<string>() { "libcoreclr.so", "libclrjit.so" };
+ private static string TO_BE_CONTINUE_TAG = "<TO_BE_CONTINUE>";
+ private static string SKIP_LINE_TAG = "# <SKIP_LINE>";
+
+
+ static bool RunProcess(string fileName, string arguments)
+ {
+ Process proc = new Process()
+ {
+ StartInfo = new ProcessStartInfo()
+ {
+ FileName = fileName,
+ Arguments = arguments,
+ UseShellExecute = false,
+ RedirectStandardOutput = true,
+ RedirectStandardError = true,
+ }
+ };
+
+ Console.WriteLine($"Invoking: {proc.StartInfo.FileName} {proc.StartInfo.Arguments}");
+ proc.Start();
+
+ Task<string> stdOut = proc.StandardOutput.ReadToEndAsync();
+ Task<string> stdErr = proc.StandardError.ReadToEndAsync();
+ if(!proc.WaitForExit(DEFAULT_TIMEOUT_MS))
+ {
+ proc.Kill(true);
+ Console.WriteLine($"Timedout: '{fileName} {arguments}");
+ return false;
+ }
+
+ Task.WaitAll(stdOut, stdErr);
+ string output = stdOut.Result;
+ string error = stdErr.Result;
+ if (!string.IsNullOrWhiteSpace(output))
+ {
+ Console.WriteLine($"stdout: {output}");
+ }
+ if (!string.IsNullOrWhiteSpace(error))
+ {
+ Console.WriteLine($"stderr: {error}");
+ }
+ return true;
+ }
+
+ /// <summary>
+ /// Parse crashreport.json file, use llvm-symbolizer to extract symbols
+ /// and recreate the stacktrace that is printed on the console.
+ /// </summary>
+ /// <param name="crashReportJsonFile">crash dump path</param>
+ /// <param name="outputWriter">Stream for writing logs</param>
+ /// <returns>true, if we can print the stack trace, otherwise false.</returns>
+ static bool TryPrintStackTraceFromCrashReport(string crashReportJsonFile, StreamWriter outputWriter)
+ {
+ if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
+ {
+ if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}"))
+ {
+ return false;
+ }
+
+ Console.WriteLine("=========================================");
+ string userName = Environment.GetEnvironmentVariable("USER");
+ if (!string.IsNullOrEmpty(userName))
+ {
+ if (!RunProcess("sudo", $"chown {userName} {crashReportJsonFile}"))
+ {
+ return false;
+ }
+
+ Console.WriteLine("=========================================");
+ if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}"))
+ {
+ return false;
+ }
+
+ Console.WriteLine("=========================================");
+ if (!RunProcess("ls", $"-l {crashReportJsonFile}"))
+ {
+ return false;
+ }
+ }
+ }
+
+ if (!File.Exists(crashReportJsonFile))
+ {
+ return false;
+ }
+ outputWriter.WriteLine($"Printing stacktrace from '{crashReportJsonFile}'");
+
+ string contents = File.ReadAllText(crashReportJsonFile);
+ dynamic crashReport = JsonSerializer.Deserialize<JsonObject>(contents);
+ var threads = crashReport["payload"]["threads"];
+
+ // The logic happens in 3 steps:
+ // 1. Read the crashReport.json file, locate all the addresses of interest and then build
+ // a string that will be passed to llvm-symbolizer. It is populated so that each address
+ // is in its separate line along with the file name, etc. Some TAGS are added in the
+ // string that is used in step 2.
+ // 2. llvm-symbolizer is ran and above string is passed as input.
+ // 3. After llvm-symbolizer completes, TAGS are used to format its output to print it in
+ // the way it will be printed by sos.
+
+ StringBuilder addrBuilder = new StringBuilder();
+ string coreRoot = Environment.GetEnvironmentVariable("CORE_ROOT");
+ foreach (var thread in threads)
+ {
+
+ if (thread["native_thread_id"] == null)
+ {
+ continue;
+ }
+
+ addrBuilder.AppendLine();
+ addrBuilder.AppendLine("----------------------------------");
+ addrBuilder.AppendLine($"Thread Id: {thread["native_thread_id"]}");
+ addrBuilder.AppendLine(" Child SP IP Call Site");
+ var stack_frames = thread["stack_frames"];
+ foreach (var frame in stack_frames)
+ {
+ addrBuilder.Append($"{SKIP_LINE_TAG} {frame["stack_pointer"]} {frame["native_address"]} ");
+ bool isNative = (string)frame["is_managed"] == "false";
+
+ if (isNative)
+ {
+ string nativeModuleName = (string)frame["native_module"];
+ string unmanagedName = (string)frame["unmanaged_name"];
+
+ if ((nativeModuleName != null) && (knownNativeModules.Contains(nativeModuleName)))
+ {
+ // Need to use llvm-symbolizer (only if module_address != 0)
+ AppendAddress(addrBuilder, coreRoot, nativeModuleName, (string)frame["native_address"], (string)frame["module_address"]);
+ }
+ else if ((nativeModuleName != null) || (unmanagedName != null))
+ {
+ if (nativeModuleName != null)
+ {
+ addrBuilder.Append($"{nativeModuleName}!");
+ }
+ if (unmanagedName != null)
+ {
+ addrBuilder.Append($"{unmanagedName}");
+ }
+ }
+ }
+ else
+ {
+ string fileName = (string)frame["filename"];
+ string methodName = (string)frame["method_name"];
+
+ if ((fileName != null) || (methodName != null))
+ {
+ // found the managed method name
+ if (fileName != null)
+ {
+ addrBuilder.Append($"{fileName}!");
+ }
+ if (methodName != null)
+ {
+ addrBuilder.Append($"{methodName}");
+ }
+ }
+ else
+ {
+ addrBuilder.Append($"{frame["native_address"]}");
+ }
+ }
+ addrBuilder.AppendLine();
+
+ }
+ }
+
+ string symbolizerOutput = null;
+
+ Process llvmSymbolizer = new Process()
+ {
+ StartInfo = {
+ FileName = "llvm-symbolizer",
+ Arguments = $"--pretty-print",
+ UseShellExecute = false,
+ RedirectStandardOutput = true,
+ RedirectStandardError = true,
+ RedirectStandardInput = true,
+ }
+ };
+
+ outputWriter.WriteLine($"Invoking {llvmSymbolizer.StartInfo.FileName} {llvmSymbolizer.StartInfo.Arguments}");
+
+ try
+ {
+ if (!llvmSymbolizer.Start())
+ {
+ outputWriter.WriteLine($"Unable to start {llvmSymbolizer.StartInfo.FileName}");
+ }
+
+ using (var symbolizerWriter = llvmSymbolizer.StandardInput)
+ {
+ symbolizerWriter.WriteLine(addrBuilder.ToString());
+ }
+
+ Task<string> stdout = llvmSymbolizer.StandardOutput.ReadToEndAsync();
+ Task<string> stderr = llvmSymbolizer.StandardError.ReadToEndAsync();
+ bool fSuccess = llvmSymbolizer.WaitForExit(DEFAULT_TIMEOUT_MS);
+
+ Task.WaitAll(stdout, stderr);
+
+ if (!fSuccess)
+ {
+ outputWriter.WriteLine("Errors while running llvm-symbolizer --pretty-print");
+ string output = stdout.Result;
+ string error = stderr.Result;
+
+ Console.WriteLine("llvm-symbolizer stdout:");
+ Console.WriteLine(output);
+ Console.WriteLine("llvm-symbolizer stderr:");
+ Console.WriteLine(error);
+
+ llvmSymbolizer.Kill(true);
+
+ return false;
+ }
+
+ symbolizerOutput = stdout.Result;
+
+ } catch (Exception e) {
+ outputWriter.WriteLine("Errors while running llvm-symbolizer --pretty-print");
+ outputWriter.WriteLine(e.ToString());
+ return false;
+ }
+
+ // Go through the output of llvm-symbolizer and strip all the markers we added initially.
+ string[] contentsToSantize = symbolizerOutput.Split(Environment.NewLine);
+ StringBuilder finalBuilder = new StringBuilder();
+ for (int lineNum = 0; lineNum < contentsToSantize.Length; lineNum++)
+ {
+ string line = contentsToSantize[lineNum].Replace(SKIP_LINE_TAG, string.Empty);
+ if (string.IsNullOrWhiteSpace(line)) continue;
+
+ if (line.EndsWith(TO_BE_CONTINUE_TAG))
+ {
+ finalBuilder.Append(line.Replace(TO_BE_CONTINUE_TAG, string.Empty));
+ continue;
+ }
+ finalBuilder.AppendLine(line);
+ }
+ outputWriter.WriteLine("Stack trace:");
+ outputWriter.WriteLine(finalBuilder.ToString());
+ return true;
+ }
+
+ private static void AppendAddress(StringBuilder sb, string coreRoot, string nativeModuleName, string native_address, string module_address)
+ {
+ if (module_address != "0x0")
+ {
+ sb.Append($"{nativeModuleName}!");
+ sb.Append(TO_BE_CONTINUE_TAG);
+ sb.AppendLine();
+ //addrBuilder.AppendLine(frame.native_image_offset);
+ ulong nativeAddress = ulong.Parse(native_address.Substring(2), System.Globalization.NumberStyles.HexNumber);
+ ulong moduleAddress = ulong.Parse(module_address.Substring(2), System.Globalization.NumberStyles.HexNumber);
+ string fullPathToModule = Path.Combine(coreRoot, nativeModuleName);
+ sb.AppendFormat("{0} 0x{1:x}", fullPathToModule, nativeAddress - moduleAddress);
+ }
+ }
+
// Finds all children processes starting with a process named childName
// The children are sorted in the order they should be dumped
static unsafe IEnumerable<Process> FindChildProcessesByName(Process process, string childName)
exitCode = process.ExitCode;
MobileAppHandler.CheckExitCode(exitCode, testBinaryBase, category, outputWriter);
Task.WaitAll(copyOutput, copyError);
+
+ if (!OperatingSystem.IsWindows())
+ {
+ // crashreport is only for non-windows.
+ if (exitCode != 0)
+ {
+ // Search for dump, if created.
+ if (Directory.Exists(crashDumpFolder))
+ {
+ outputWriter.WriteLine($"Test failed. Trying to see if dump file was created in {crashDumpFolder} since {startTime}");
+ DirectoryInfo crashDumpFolderInfo = new DirectoryInfo(crashDumpFolder);
+ var dmpFilesInfo = crashDumpFolderInfo.GetFiles("*.crashreport.json").OrderByDescending(f => f.CreationTime);
+ foreach (var dmpFile in dmpFilesInfo)
+ {
+ if (dmpFile.CreationTime < startTime)
+ {
+ // No new files since test started.
+ outputWriter.WriteLine("Finish looking for *.crashreport.json. No new files created.");
+ break;
+ }
+ outputWriter.WriteLine($"Processing {dmpFile.FullName}");
+ TryPrintStackTraceFromCrashReport(dmpFile.FullName, outputWriter);
+ }
+ }
+ }
+ }
}
else
{
{
string crashDumpPath = Path.Combine(Path.GetFullPath(crashDumpFolder), string.Format("crashdump_{0}.dmp", child.Id));
Console.WriteLine($"Attempting to collect crash dump: {crashDumpPath}");
- if (CollectCrashDump(child, crashDumpPath))
+ if (CollectCrashDump(child, crashDumpPath, outputWriter))
{
Console.WriteLine("Collected crash dump: {0}", crashDumpPath);
}
projects / platform / upstream / dotnet / runtime.git / commitdiff