From 9140342364120d34e72e12ea04fd1e5524ee126a Mon Sep 17 00:00:00 2001 From: Mike McLaughlin Date: Mon, 6 Aug 2018 09:49:46 -0700 Subject: [PATCH] Test on more Linux distros. (#55) Test on more Linux distros. Fix some other problems in the internal builds. Fixed problem in sos test runner that ignored sos commands failing. Added a file or directory exists condition for configuration file entries. Change the dump generation to use createdump for 2.1/2.0. Still uses gdb for 1.1. Fixed ReadVirtualCache::Read problem on lldb that caused the tests to fail on OSX. The lldb services Read/WriteVirtual needed to properly return a partial read or write. --- .vsts-dotnet.yml | 186 ++++++++++++- documentation/debugging-instructions.md | 10 +- eng/build-native.sh | 5 - eng/build.yml | 7 +- eng/cibuild.sh | 4 +- eng/docker-cibuild.sh | 1 + eng/docker-init.sh | 13 + .../TestConfiguration.cs | 131 ++++++--- .../TestRunner.cs | 6 +- .../Unix/Debugger.Tests.Config.txt | 13 +- .../Windows/Debugger.Tests.Config.txt | 10 +- src/SOS/SOS.UnitTests/SOS.cs | 66 +---- src/SOS/SOS.UnitTests/SOSRunner.cs | 259 +++++++++++++++--- src/SOS/SOS.UnitTests/Scripts/lldbhelper.py | 13 +- src/SOS/Strike/gcroot.cpp | 6 +- src/SOS/Strike/sos.cpp | 4 +- src/SOS/lldbplugin/services.cpp | 4 +- 17 files changed, 569 insertions(+), 169 deletions(-) create mode 100755 eng/docker-init.sh diff --git a/.vsts-dotnet.yml b/.vsts-dotnet.yml index 565024a26..e53c98680 100644 --- a/.vsts-dotnet.yml +++ b/.vsts-dotnet.yml @@ -1,13 +1,14 @@ phases: - template: /eng/build.yml parameters: - phaseName: Windows - agentOs: Windows_NT + phaseName: CentOS_7 + agentOs: Linux buildReason: Internal - queue: - name: DotNetCore-Build + dockerImage: microsoft/dotnet-buildtools-prereqs:centos-7-c103199-20180628120549 + queue: + name: DotNet-Build demands: - - agent.os -equals Windows_NT + - agent.os -equals Linux parallel: 2 matrix: Build_Debug: @@ -23,43 +24,173 @@ phases: - template: /eng/build.yml parameters: - phaseName: CentOS_7 + phaseName: Debian_Stretch agentOs: Linux buildReason: Internal - dockerImage: microsoft/dotnet-buildtools-prereqs:centos-7-c103199-20180628120549 + dockerImage: microsoft/dotnet-buildtools-prereqs:debian-stretch-c103199-20180628122423 queue: name: DotNet-Build demands: - agent.os -equals Linux - parallel: 2 matrix: Build_Debug: _BuildConfig: Debug _BuildArch: x64 _PublishType: none _SignType: test - Build_Release: - _BuildConfig: Release + +- template: /eng/build.yml + parameters: + phaseName: Fedora_24 + agentOs: Linux + buildReason: Internal + dockerImage: microsoft/dotnet-buildtools-prereqs:fedora-24-c103199-20180628122443 + queue: + name: DotNet-Build + demands: + - agent.os -equals Linux + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishType: none + _SignType: test + +- template: /eng/build.yml + parameters: + phaseName: Fedora_27 + agentOs: Linux + buildReason: Internal + dockerImage: microsoft/dotnet-buildtools-prereqs:fedora-27-c103199-20180628122443 + queue: + name: DotNet-Build + demands: + - agent.os -equals Linux + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishType: none + _SignType: test + +- template: /eng/build.yml + parameters: + phaseName: Fedora_28 + agentOs: Linux + buildReason: Internal + dockerImage: microsoft/dotnet-buildtools-prereqs:fedora-28-c103199-20180628122443 + queue: + name: DotNet-Build + demands: + - agent.os -equals Linux + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishType: none + _SignType: test + +- template: /eng/build.yml + parameters: + phaseName: OpenSuse_42_1 + agentOs: Linux + buildReason: Internal + dockerImage: microsoft/dotnet-buildtools-prereqs:opensuse-42.1-c103199-20180628122439 + queue: + name: DotNet-Build + demands: + - agent.os -equals Linux + matrix: + Build_Debug: + _BuildConfig: Debug _BuildArch: x64 _PublishType: none _SignType: real +- template: /eng/build.yml + parameters: + phaseName: OpenSuse_42_3 + agentOs: Linux + buildReason: Internal + dockerImage: microsoft/dotnet-buildtools-prereqs:opensuse-42.3-c103199-20180628122439 + queue: + name: DotNet-Build + demands: + - agent.os -equals Linux + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishType: none + _SignType: test + - template: /eng/build.yml parameters: phaseName: Ubuntu_14_04 agentOs: Linux buildReason: Internal + dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-c103199-20180628134413 + queue: + name: DotNet-Build + demands: + - agent.os -equals Linux + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishType: none + _SignType: test + +- template: /eng/build.yml + parameters: + phaseName: Ubuntu_16_04 + agentOs: Linux + buildReason: Internal + dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-c103199-20180628134544 + queue: + name: DotNet-Build + demands: + - agent.os -equals Linux + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishType: none + _SignType: test + +- template: /eng/build.yml + parameters: + phaseName: Ubuntu_17_10 + agentOs: Linux + buildReason: Internal + dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-17.10-c103199-20180628134544 + queue: + name: DotNet-Build + demands: + - agent.os -equals Linux + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishType: none + _SignType: test + +- template: /eng/build.yml + parameters: + phaseName: Ubuntu_18_04 + agentOs: Linux + buildReason: Internal dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-18.04-c103199-20180628134610 queue: name: DotNet-Build demands: - agent.os -equals Linux matrix: - Build_Release: - _BuildConfig: Release + Build_Debug: + _BuildConfig: Debug _BuildArch: x64 _PublishType: none - _SignType: real + _SignType: test - template: /eng/build.yml parameters: @@ -67,7 +198,7 @@ phases: agentOs: Darwin buildReason: Internal queue: - name: DotNetCore-Build + name: Hosted macOS Preview demands: - agent.os -equals Darwin parallel: 2 @@ -83,3 +214,30 @@ phases: _PublishType: none _SignType: real +- template: /eng/build.yml + parameters: + phaseName: Windows + agentOs: Windows_NT + buildReason: Internal + queue: + name: DotNetCore-Build + demands: + - agent.os -equals Windows_NT + parallel: 3 + matrix: + Build_Debug: + _BuildConfig: Debug + _BuildArch: x64 + _PublishType: none + _SignType: test + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + _PublishType: none + _SignType: real + Build_Release_x86: + _BuildConfig: Release + _BuildArch: x86 + _PublishType: none + _SignType: real + diff --git a/documentation/debugging-instructions.md b/documentation/debugging-instructions.md index ca6fb668d..0ece87556 100644 --- a/documentation/debugging-instructions.md +++ b/documentation/debugging-instructions.md @@ -139,10 +139,12 @@ It is also possible to debug .NET Core crash dumps using lldb and SOS. In order - The crash dump file. We have a service called "Dumpling" which collects, uploads, and archives crash dump files during all of our CI jobs and official builds. - On Linux, there is an utility called `createdump` (see [doc](https://github.com/dotnet/coreclr/blob/master/Documentation/botr/xplat-minidump-generation.md "doc")) that can be setup to generate core dumps when a managed app throws an unhandled exception or faults. -- Matching coreclr/corefx runtime bits from the crash. To get these, you should either: - - Download the matching Jenkins archive onto your repro machine. - - Check out the coreclr and corefx repositories at the appropriate commit and re-build the necessary portions. - - You can also download the matching "symbols" nuget package from myget.org. There is a "Download Symbols" button in the myget UI for this purpose. +- To get matching runtime and symbol binaries for the core dump use the symbol downloader CLI extension: + - Install the [.NET Core 2.1 SDK](https://www.microsoft.com/net/download/). + - Install the symbol downloader extension: `dotnet tool install -g dotnet-symbol`. + - Run `dotnet symbol coredump` to download the runtime binaries and symbols. + - Check out the coreclr and corefx repositories at the appropriate commit for the appropriate source. + - For more details see: [dotnet-symbol](https://github.com/dotnet/symstore/blob/master/src/dotnet-symbol/README.md). - lldb version 3.9. The SOS plugin (i.e. libsosplugin.so) provided is now built for lldb 3.9. In order to install lldb 3.9 just run the following commands: ``` ~$ echo "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.9 main" | sudo tee /etc/apt/sources.list.d/llvm.list diff --git a/eng/build-native.sh b/eng/build-native.sh index 8125be073..868e1e432 100755 --- a/eng/build-native.sh +++ b/eng/build-native.sh @@ -456,11 +456,6 @@ if [ $__Test == 1 ]; then export GDB_PATH="$(which gdb 2> /dev/null)" fi - if [ "$__HostOS" == "Linux" ]; then - # This is needed on some distros like centos 7 so gdb generate-core-file creates a dump that works - echo 0x37 > /proc/self/coredump_filter - fi - echo "lldb: '$LLDB_PATH' gdb: '$GDB_PATH'" # Run xunit SOS tests diff --git a/eng/build.yml b/eng/build.yml index a0b18ba6d..5230bc228 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -34,6 +34,7 @@ phases: queue: ${{ parameters.queue }} variables: _DockerImageName: ${{ parameters.dockerImage }} + _PhaseName : ${{ parameters.phaseName }} ${{ if notIn(parameters.buildReason, 'IndividualCI', 'BatchedCI', 'PullRequest') }}: _PublishBlobFeedUrl: https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json _TeamName: DotNetCore @@ -114,7 +115,7 @@ phases: Contents: '**/*log' TargetFolder: '$(Build.StagingDirectory)/BuildLogs' continueOnError: true - condition: succeededOrFailed() + condition: always() - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS @@ -126,6 +127,6 @@ phases: ${{ if eq(parameters.agentOs, 'Windows_NT') }}: ArtifactName: Windows_NT_$(Agent.JobName) ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - ArtifactName: Linux_$(parameters.phaseName)_$(Agent.JobName) + ArtifactName: Linux_$(_PhaseName)_$(Agent.JobName) continueOnError: true - condition: succeededOrFailed() + condition: always() diff --git a/eng/cibuild.sh b/eng/cibuild.sh index 9519e111e..1291238be 100755 --- a/eng/cibuild.sh +++ b/eng/cibuild.sh @@ -25,8 +25,8 @@ if [ "$__osname" == "Linux" ]; then if [[ $ID == "ubuntu" ]]; then if [[ $VERSION_ID == "18.04" ]]; then # Fix the CI lab's ubuntu 18.04 docker image: install curl. - apt-get update - apt-get install -y curl + sudo apt-get update + sudo apt-get install -y curl fi fi elif [ -e /etc/redhat-release ]; then diff --git a/eng/docker-cibuild.sh b/eng/docker-cibuild.sh index 73c8fd9e8..c06eb2820 100755 --- a/eng/docker-cibuild.sh +++ b/eng/docker-cibuild.sh @@ -60,6 +60,7 @@ echo "container user name: $container_user_name" $docker_bin exec $docker_id useradd -m -u $user_id $container_user_name $docker_bin exec $docker_id groupadd container_SUDO_user $docker_bin exec $docker_id usermod -a -G container_SUDO_user $container_user_name +$docker_bin exec $docker_id su -c "$source_directory/eng/docker-init.sh" $docker_bin exec $docker_id su -c "echo '%container_SUDO_user ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers" echo "Execute cibuild.sh $args" diff --git a/eng/docker-init.sh b/eng/docker-init.sh new file mode 100755 index 000000000..2be3f6e66 --- /dev/null +++ b/eng/docker-init.sh @@ -0,0 +1,13 @@ +# Install sudo on ubuntu 18.04 +os_name=$(uname -s) +if [ "$os_name" == "Linux" ]; then + if [ -e /etc/os-release ]; then + source /etc/os-release + if [[ $ID == "ubuntu" ]]; then + if [[ $VERSION_ID == "18.04" ]]; then + apt-get update + apt-get install sudo + fi + fi + fi +fi diff --git a/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs b/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs index 8cc3b2ace..1442ab1f3 100644 --- a/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs +++ b/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs @@ -77,6 +77,10 @@ namespace Microsoft.Diagnostic.TestHelpers ["TargetArchitecture"] = OS.TargetArchitecture.ToString().ToLowerInvariant(), ["NuGetPackageCacheDir"] = nugetPackages }; + if (OS.Kind == OSKind.Windows) + { + initialConfig["WinDir"] = Path.GetFullPath(Environment.GetEnvironmentVariable("WINDIR")); + } IEnumerable> configs = ParseConfigFile(path, new Dictionary[] { initialConfig }); Configurations = configs.Select(c => new TestConfiguration(c)); } @@ -168,30 +172,45 @@ namespace Microsoft.Diagnostic.TestHelpers { string conditionText = attr.Value; - // Only equals and not equals are supported - string[] parts = conditionText.Split("=="); - bool equal; - - if (parts.Length == 2) + // Check if Exists('') + const string existsKeyword = "Exists('"; + int existsStartIndex = conditionText.IndexOf(existsKeyword); + if (existsStartIndex != -1) { - equal = true; + bool not = (existsStartIndex > 0) && (conditionText[existsStartIndex - 1] == '!'); + + existsStartIndex += existsKeyword.Length; + int existsEndIndex = conditionText.IndexOf("')", existsStartIndex); + Assert.NotEqual(-1, existsEndIndex); + + string path = conditionText.Substring(existsStartIndex, existsEndIndex - existsStartIndex); + path = Path.GetFullPath(ResolveProperties(config, path)); + bool exists = Directory.Exists(path) || File.Exists(path); + return not ? !exists : exists; } else { - parts = conditionText.Split("!="); - if (parts.Length != 2) + // Check if equals and not equals + string[] parts = conditionText.Split("=="); + bool equal; + + if (parts.Length == 2) { - throw new ArgumentException("Invalid Condition attribute {0}", attr.Value); + equal = true; } - equal = false; - } - - // Resolve any config values in the condition - string leftValue = ResolveProperties(config, parts[0]).Trim(); - string rightValue = ResolveProperties(config, parts[1]).Trim(); + else + { + parts = conditionText.Split("!="); + Assert.NotEqual(2, parts.Length); + equal = false; + } + // Resolve any config values in the condition + string leftValue = ResolveProperties(config, parts[0]).Trim(); + string rightValue = ResolveProperties(config, parts[1]).Trim(); - // Now do the simple string comparsion of the left/right sides of the condition - return equal ? leftValue == rightValue : leftValue != rightValue; + // Now do the simple string comparsion of the left/right sides of the condition + return equal ? leftValue == rightValue : leftValue != rightValue; + } } return true; } @@ -218,7 +237,9 @@ namespace Microsoft.Diagnostic.TestHelpers { resolvedValue.Append(rawNodeValue.Substring(i, propStartIndex - i)); } - resolvedValue.Append(ResolveProperty(config, rawNodeValue.Substring(propStartIndex+2, propEndIndex - propStartIndex-2))); + // Now resolve the property name from the config dictionary + string propertyName = rawNodeValue.Substring(propStartIndex + 2, propEndIndex - propStartIndex - 2); + resolvedValue.Append(config.GetValueOrDefault(propertyName, "")); i = propEndIndex + 1; } } @@ -226,18 +247,11 @@ namespace Microsoft.Diagnostic.TestHelpers return resolvedValue.ToString(); } - private string ResolveProperty(Dictionary config, string propName) - { - return propName.Equals("WinDir", StringComparison.OrdinalIgnoreCase) - ? Path.GetFullPath(Environment.ExpandEnvironmentVariables("%WINDIR%")) - : config[propName] ?? ""; - } - public void Dispose() { } } - + /// /// Represents the current test configuration /// @@ -255,7 +269,7 @@ namespace Microsoft.Diagnostic.TestHelpers _settings = new Dictionary(); } - public TestConfiguration(Dictionary initialSettings) + public TestConfiguration(Dictionary initialSettings) { _settings = new Dictionary(initialSettings); } @@ -269,7 +283,7 @@ namespace Microsoft.Diagnostic.TestHelpers { Debug.Assert(!string.IsNullOrWhiteSpace(pdbType)); - var currentSettings = new Dictionary(_settings); + var currentSettings = new Dictionary(_settings); // Set or replace if the pdb debug type currentSettings[DebugTypeKey] = pdbType; @@ -298,6 +312,22 @@ namespace Microsoft.Diagnostic.TestHelpers get { return GetValue("TestProduct").ToLowerInvariant(); } } + /// + /// Returns true if running on .NET Core (based on TestProduct). + /// + public bool IsNETCore + { + get { return TestProduct.Equals("projectk"); } + } + + /// + /// Returns true if running on desktop framework (based on TestProduct). + /// + public bool IsDesktop + { + get { return TestProduct.Equals("desktop"); } + } + /// /// The test runner script directory /// @@ -433,6 +463,15 @@ namespace Microsoft.Diagnostic.TestHelpers get { return GetValue("BuildProjectRuntime"); } } + /// + /// The version of the Microsoft.NETCore.App package to reference when running the debuggee (i.e. + /// using the dotnet cli --fx-version option). + /// + public string RuntimeFrameworkVersion + { + get { return GetValue("RuntimeFrameworkVersion"); } + } + /// /// The type of PDB: "full" (Windows PDB) or "portable". /// @@ -505,6 +544,34 @@ namespace Microsoft.Diagnostic.TestHelpers get { return GetValue("LinkerPackageVersion"); } } + #region Runtime Features properties + + /// + /// Returns true if the "createdump" facility exists. + /// + public bool CreateDumpExists + { + get { return OS.Kind == OSKind.Linux && IsNETCore && !RuntimeFrameworkVersion.StartsWith("1."); } + } + + /// + /// Returns true if a stack overflow causes dump to be generated with createdump. Currently no .NET Core version does. + /// + public bool StackOverflowCreatesDump + { + get { return false; } + } + + /// + /// Returns true if a stack overflow causes a SIGSEGV exception instead of aborting. + /// + public bool StackOverflowSIGSEGV + { + get { return OS.Kind == OSKind.Linux && IsNETCore && RuntimeFrameworkVersion.StartsWith("1."); } + } + + #endregion + /// /// Returns the configuration value for the key or null. /// @@ -579,15 +646,15 @@ namespace Microsoft.Diagnostic.TestHelpers static OS() { #if CORE_CLR // Only core build can run on different OSes - if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { Kind = OSKind.Linux; } - else if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { Kind = OSKind.OSX; } - else if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { Kind = OSKind.Windows; } @@ -610,6 +677,6 @@ namespace Microsoft.Diagnostic.TestHelpers /// /// The architecture the tests are running. We are assuming that the test runner, the debugger and the debugger's target are all the same architecture. /// - public static Architecture TargetArchitecture { get { return System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture; } } + public static Architecture TargetArchitecture { get { return RuntimeInformation.ProcessArchitecture; } } } } diff --git a/src/Microsoft.Diagnostic.TestHelpers/TestRunner.cs b/src/Microsoft.Diagnostic.TestHelpers/TestRunner.cs index f6ba3eedb..ecef8d297 100644 --- a/src/Microsoft.Diagnostic.TestHelpers/TestRunner.cs +++ b/src/Microsoft.Diagnostic.TestHelpers/TestRunner.cs @@ -104,7 +104,7 @@ namespace Microsoft.Diagnostic.TestHelpers if (OS.Kind == OSKind.Windows) { - if (config.TestProduct.Equals("projectk")) + if (config.IsNETCore) { // Enabled after an official dotnet cli supports "embedded" PDBs - issue #244 // pdbTypes = new string[] { "portable", "full", "embedded" }; @@ -143,7 +143,7 @@ namespace Microsoft.Diagnostic.TestHelpers ConsoleTestOutputHelper consoleLogger = null; if (!string.IsNullOrEmpty(config.LogDirPath)) { - string logFileName = testName + "." + config.ToString() + ".txt"; + string logFileName = testName + "." + config.ToString() + ".log"; string logPath = Path.Combine(config.LogDirPath, logFileName); fileLogger = new FileTestOutputHelper(logPath, FileMode.Append); } @@ -190,7 +190,7 @@ namespace Microsoft.Diagnostic.TestHelpers } } - class TestLogger : TestOutputProcessLogger + public class TestLogger : TestOutputProcessLogger { readonly StringBuilder _standardOutput; readonly StringBuilder _standardError; diff --git a/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt b/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt index c69b2ca39..95b3a6d7c 100644 --- a/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt +++ b/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt @@ -34,28 +34,33 @@ - $(RepoRootDir)/.dotnet/dotnet - --fx-version $(BuildProjectMicrosoftNetCoreAppVersion) - $(RepoRootDir)/.dotnet/shared/Microsoft.NETCore.App/$(BuildProjectMicrosoftNetCoreAppVersion) + --fx-version $(RuntimeFrameworkVersion) + $(RepoRootDir)/.dotnet/shared/Microsoft.NETCore.App/$(RuntimeFrameworkVersion) $(ScriptRootDir)/lldbhelper.py $(RepoRootDir)\.dotnet\dotnet.exe - --fx-version $(BuildProjectMicrosoftNetCoreAppVersion) - $(RepoRootDir)\.dotnet\shared\Microsoft.NETCore.App\$(BuildProjectMicrosoftNetCoreAppVersion) + --fx-version $(RuntimeFrameworkVersion) + $(RepoRootDir)\.dotnet\shared\Microsoft.NETCore.App\$(RuntimeFrameworkVersion) $(InstallDir)\sos.dll - $(DumpDir)\$(TestProduct) + $(DumpDir)\$(TestProduct)\$(RuntimeFrameworkVersion)\$(BuildProjectFramework) $(DebuggeeDumpOutputRootDir) diff --git a/src/SOS/SOS.UnitTests/SOS.cs b/src/SOS/SOS.UnitTests/SOS.cs index 27fbf1f20..ac5a87a0f 100644 --- a/src/SOS/SOS.UnitTests/SOS.cs +++ b/src/SOS/SOS.UnitTests/SOS.cs @@ -3,9 +3,7 @@ // See the LICENSE file in the project root for more information. using Microsoft.Diagnostic.TestHelpers; -using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Threading.Tasks; using Xunit; @@ -41,52 +39,12 @@ public class SOS return config.DebuggeeDumpInputRootDir() != null; } - private async Task CreateDump(TestConfiguration config, string testName, string debuggeeName, string debuggeeArguments) + private Task RunTest(TestConfiguration config, string debuggeeName, string scriptName, bool useCreateDump = true) { - Directory.CreateDirectory(config.DebuggeeDumpOutputRootDir()); - - using (SOSRunner runner = await SOSRunner.StartDebugger(config, Output, testName, debuggeeName, debuggeeArguments, loadDump: false, generateDump: true)) - { - try - { - await runner.LoadSosExtension(); - - string command = null; - switch (runner.Debugger) - { - case SOSRunner.NativeDebugger.Cdb: - await runner.ContinueExecution(); - // On desktop create triage dump. On .NET Core, create full dump. - command = config.TestProduct.Equals("desktop") ? ".dump /o /mshuRp %DUMP_NAME%" : ".dump /o /ma %DUMP_NAME%"; - break; - case SOSRunner.NativeDebugger.Gdb: - command = "generate-core-file %DUMP_NAME%"; - break; - case SOSRunner.NativeDebugger.Lldb: - await runner.ContinueExecution(); - command = OS.Kind == OSKind.OSX ? "process save-core %DUMP_NAME%" : "sos CreateDump %DUMP_NAME%"; - break; - default: - throw new Exception(runner.Debugger.ToString() + " does not support creating dumps"); - } - - await runner.RunCommand(command); - await runner.QuitDebugger(); - } - catch (Exception ex) - { - runner.WriteLine(ex.ToString()); - throw; - } - } + return RunTest(config, "SOS." + debuggeeName, debuggeeName, scriptName, useCreateDump: useCreateDump); } - private Task RunTest(TestConfiguration config, string debuggeeName, string scriptName) - { - return RunTest(config, "SOS." + debuggeeName, debuggeeName, null, scriptName); - } - - private async Task RunTest(TestConfiguration config, string testName, string debuggeeName, string debuggeeArguments, string scriptName) + private async Task RunTest(TestConfiguration config, string testName, string debuggeeName, string scriptName, string debuggeeArguments = null, bool useCreateDump = true) { SkipIfArm(config); @@ -99,7 +57,7 @@ public class SOS // Against a crash dump. if (IsCreateDumpConfig(config)) { - await CreateDump(config, testName, debuggeeName, debuggeeArguments); + await SOSRunner.CreateDump(config, Output, testName, debuggeeName, debuggeeArguments, useCreateDump); } if (IsOpenDumpConfig(config)) @@ -120,12 +78,9 @@ public class SOS [SkippableTheory, MemberData(nameof(Configurations))] public async Task GCTests(TestConfiguration config) { - const string testName = "SOS.GCTests"; - const string debuggeeName = "GCWhere"; - // Live only SkipIfArm(config); - using (SOSRunner runner = await SOSRunner.StartDebugger(config, Output, testName, debuggeeName, debuggeeArguments: null)) + using (SOSRunner runner = await SOSRunner.StartDebugger(config, Output, testName: "SOS.GCTests", debuggeeName: "GCWhere")) { await runner.RunScript("GCTests.script"); } @@ -134,7 +89,8 @@ public class SOS [SkippableTheory, MemberData(nameof(Configurations))] public async Task Overflow(TestConfiguration config) { - await RunTest(config, "Overflow", "Overflow.script"); + // The .NET Core createdump facility may not catch stack overflow so use gdb to generate dump + await RunTest(config, "Overflow", "Overflow.script", useCreateDump: config.StackOverflowCreatesDump); } [SkippableTheory, MemberData(nameof(Configurations))] @@ -164,21 +120,21 @@ public class SOS [SkippableTheory, MemberData(nameof(Configurations))] public async Task StackTests(TestConfiguration config) { - await RunTest(config, "SOS.StackTests", "NestedExceptionTest", null, "StackTests.script"); + await RunTest(config, "SOS.StackTests", "NestedExceptionTest", "StackTests.script"); } [SkippableTheory, MemberData(nameof(Configurations))] public async Task StackAndOtherTests(TestConfiguration config) { SkipIfArm(config); - if (config.BuildProjectMicrosoftNetCoreAppVersion.StartsWith("1.1")) + if (config.RuntimeFrameworkVersion.StartsWith("1.1")) { - throw new SkipTestException("The debuggee (SymbolTestApp) doesn't work on .NET Core 1.1"); + throw new SkipTestException("The debuggee (SymbolTestApp) doesn't work on .NET Core 1.1 because of a AssemblyLoadContext problem"); } foreach (TestConfiguration currentConfig in TestRunner.EnumeratePdbTypeConfigs(config)) { // This debuggee needs the directory of the exes/dlls to load the SymbolTestDll assembly. - await RunTest(currentConfig, "SOS.StackAndOtherTests", "SymbolTestApp", "%DEBUG_ROOT%", "StackAndOtherTests.script"); + await RunTest(currentConfig, "SOS.StackAndOtherTests", "SymbolTestApp", "StackAndOtherTests.script", debuggeeArguments: "%DEBUG_ROOT%"); } } } diff --git a/src/SOS/SOS.UnitTests/SOSRunner.cs b/src/SOS/SOS.UnitTests/SOSRunner.cs index 2d6c183f3..8374e0ed4 100644 --- a/src/SOS/SOS.UnitTests/SOSRunner.cs +++ b/src/SOS/SOS.UnitTests/SOSRunner.cs @@ -55,8 +55,132 @@ public class SOSRunner : IDisposable _isDump = isDump; } - public static async Task StartDebugger(TestConfiguration config, ITestOutputHelper output, - string testName, string debuggeeName, string debuggeeArguments, bool loadDump = false, bool generateDump = false) + /// + /// Run a debuggee and create a dump. + /// + /// test configuration + /// output instance + /// name of test + /// debuggee name + /// optional args to pass to debuggee + /// if true, use "createdump" to generate core dump + public static async Task CreateDump(TestConfiguration config, ITestOutputHelper output, string testName, string debuggeeName, + string debuggeeArguments = null, bool useCreateDump = true) + { + Directory.CreateDirectory(config.DebuggeeDumpOutputRootDir()); + + if (!config.CreateDumpExists || !useCreateDump || config.GenerateDumpWithLLDB() || config.GenerateDumpWithGDB()) + { + using (SOSRunner runner = await SOSRunner.StartDebugger(config, output, testName, debuggeeName, debuggeeArguments, loadDump: false, generateDump: true)) + { + try + { + await runner.LoadSosExtension(); + + string command = null; + switch (runner.Debugger) + { + case SOSRunner.NativeDebugger.Cdb: + await runner.ContinueExecution(); + // On desktop create triage dump. On .NET Core, create full dump. + command = config.IsDesktop ? ".dump /o /mshuRp %DUMP_NAME%" : ".dump /o /ma %DUMP_NAME%"; + break; + case SOSRunner.NativeDebugger.Gdb: + command = "generate-core-file %DUMP_NAME%"; + break; + case SOSRunner.NativeDebugger.Lldb: + await runner.ContinueExecution(); + command = "sos CreateDump %DUMP_NAME%"; + break; + default: + throw new Exception(runner.Debugger.ToString() + " does not support creating dumps"); + } + + await runner.RunCommand(command); + await runner.QuitDebugger(); + } + catch (Exception ex) + { + runner.WriteLine(ex.ToString()); + throw; + } + } + } + else + { + TestRunner.OutputHelper outputHelper = null; + try + { + // Setup the logging from the options in the config file + outputHelper = TestRunner.ConfigureLogging(config, output, testName); + + // Restore and build the debuggee. The debuggee name is lower cased because the + // source directory name has been lowercased by the build system. + DebuggeeConfiguration debuggeeConfig = await DebuggeeCompiler.Execute(config, debuggeeName, outputHelper); + + outputHelper.WriteLine("Starting {0}", testName); + outputHelper.WriteLine("{"); + + // Get the full debuggee launch command line (includes the host if required) + string exePath = debuggeeConfig.BinaryExePath; + var arguments = new StringBuilder(); + if (!string.IsNullOrWhiteSpace(config.HostExe)) + { + exePath = config.HostExe; + if (!string.IsNullOrWhiteSpace(config.HostArgs)) + { + arguments.Append(config.HostArgs); + arguments.Append(" "); + } + arguments.Append(debuggeeConfig.BinaryExePath); + } + if (!string.IsNullOrWhiteSpace(debuggeeArguments)) + { + arguments.Append(" "); + arguments.Append(debuggeeArguments); + } + + // Run the debuggee with the createdump environment variables set to generate a coredump on unhandled exception + var testLogger = new TestRunner.TestLogger(outputHelper.IndentedOutput); + var variables = GenerateVariables(config, debuggeeConfig, generateDump: true); + ProcessRunner processRunner = new ProcessRunner(exePath, ReplaceVariables(variables, arguments.ToString())). + WithLog(testLogger). + WithTimeout(TimeSpan.FromMinutes(5)). + WithEnvironmentVariable("COMPlus_DbgEnableMiniDump", "1"). + WithEnvironmentVariable("COMPlus_DbgMiniDumpName", ReplaceVariables(variables,"%DUMP_NAME%")); + + processRunner.Start(); + + // Wait for the debuggee to finish + await processRunner.WaitForExit(); + } + catch (Exception ex) + { + // Log the exception + outputHelper?.WriteLine(ex.ToString()); + throw; + } + finally + { + outputHelper?.WriteLine("}"); + outputHelper?.Dispose(); + } + } + } + + /// + /// Start a debuggee under a native debugger returning a sos runner instance. + /// + /// test configuration + /// output instance + /// name of test + /// debuggee name + /// optional args to pass to debuggee + /// if true, generate dump with native debugger + /// if true, load dump with native debugger + /// sos runner instance + public static async Task StartDebugger(TestConfiguration config, ITestOutputHelper output, string testName, string debuggeeName, + string debuggeeArguments = null, bool loadDump = false, bool generateDump = false) { TestRunner.OutputHelper outputHelper = null; SOSRunner sosRunner = null; @@ -78,6 +202,14 @@ public class SOSRunner : IDisposable var variables = GenerateVariables(config, debuggeeConfig, generateDump); var scriptLogger = new ScriptLogger(debugger, outputHelper.IndentedOutput); + if (loadDump) + { + if (!variables.TryGetValue("%DUMP_NAME%", out string dumpName) || !File.Exists(dumpName)) + { + throw new Exception($"Dump file does not exist: {dumpName ?? ""}"); + } + } + // Get the full debuggee launch command line (includes the host if required) var debuggeeCommandLine = new StringBuilder(); if (!string.IsNullOrWhiteSpace(config.HostExe)) @@ -101,7 +233,7 @@ public class SOSRunner : IDisposable string debuggerPath = GetNativeDebuggerPath(debugger, config); if (string.IsNullOrWhiteSpace(debuggerPath) || !File.Exists(debuggerPath)) { - throw new Exception("Native debugger path not set or does not exist: " + debuggerPath); + throw new Exception($"Native debugger path not set or does not exist: {debuggerPath}"); } // Get the debugger arguments and commands to run initially @@ -142,9 +274,7 @@ public class SOSRunner : IDisposable { throw new Exception("LLDB helper script path not set or does not exist: " + lldbHelperScript); } - arguments = string.Format(@"--no-lldbinit -o ""settings set interpreter.prompt-on-quit false"" -o ""command script import {0}""", lldbHelperScript); - - initialCommands.Add("version"); + arguments = string.Format(@"--no-lldbinit -o ""settings set interpreter.prompt-on-quit false"" -o ""command script import {0}"" -o ""version""", lldbHelperScript); // Load the dump or launch the debuggee process if (loadDump) @@ -173,10 +303,18 @@ public class SOSRunner : IDisposable } initialCommands.Add($@"target create ""{config.HostExe}"""); initialCommands.Add(sb.ToString()); - initialCommands.Add("process launch -s"); + + // .NET Core 1.1 or less don't catch stack overflow and abort so need to catch SIGSEGV + if (config.StackOverflowSIGSEGV) + { + initialCommands.Add("process handle -s true -n true -p true SIGSEGV"); + } + else + { + initialCommands.Add("process handle -s false -n false -p true SIGSEGV"); + } initialCommands.Add("process handle -s false -n false -p true SIGFPE"); - initialCommands.Add("process handle -s false -n false -p true SIGSEGV"); initialCommands.Add("process handle -s true -n true -p true SIGABRT"); } break; @@ -186,10 +324,20 @@ public class SOSRunner : IDisposable throw new Exception("GDB not meant for loading core dumps"); } arguments = "--args " + debuggeeCommandLine; + + // .NET Core 1.1 or less don't catch stack overflow and abort so need to catch SIGSEGV + if (config.StackOverflowSIGSEGV) + { + initialCommands.Add("handle SIGSEGV stop print"); + } + else + { + initialCommands.Add("handle SIGSEGV nostop noprint"); + } initialCommands.Add("handle SIGFPE nostop noprint"); - initialCommands.Add("handle SIGSEGV nostop noprint"); initialCommands.Add("handle SIGABRT stop print"); initialCommands.Add("set startup-with-shell off"); + initialCommands.Add("set use-coredump-filter on"); initialCommands.Add("run"); break; } @@ -197,7 +345,7 @@ public class SOSRunner : IDisposable // Create the native debugger process running ProcessRunner processRunner = new ProcessRunner(debuggerPath, ReplaceVariables(variables, arguments)). WithLog(scriptLogger). - WithTimeout(TimeSpan.FromMinutes(5)); + WithTimeout(TimeSpan.FromMinutes(10)); // Create the sos runner instance sosRunner = new SOSRunner(debugger, config, outputHelper, variables, scriptLogger, processRunner, loadDump); @@ -205,6 +353,13 @@ public class SOSRunner : IDisposable // Start the native debugger processRunner.Start(); + // Set the coredump_filter flags on the gdb process so the coredump it + // takes of the target process contains everything the tests need. + if (debugger == NativeDebugger.Gdb) + { + initialCommands.Insert(0, string.Format("shell echo 0x3F > /proc/{0}/coredump_filter", processRunner.ProcessId)); + } + // Execute the initial debugger commands await sosRunner.RunCommands(initialCommands); @@ -284,12 +439,18 @@ public class SOSRunner : IDisposable else if (line.StartsWith("SOSCOMMAND:")) { string input = line.Substring("SOSCOMMAND:".Length).TrimStart(); - await RunSosCommand(input); + if (!await RunSosCommand(input)) + { + throw new Exception($"SOS command FAILED: {input}"); + } } else if (line.StartsWith("COMMAND:")) { string input = line.Substring("COMMAND:".Length).TrimStart(); - await RunCommand(input); + if (!await RunCommand(input)) + { + throw new Exception($"Debugger command FAILED: {input}"); + } } else if (line.StartsWith("VERIFY:")) { @@ -374,10 +535,13 @@ public class SOSRunner : IDisposable command = "continue"; break; } - await RunCommand(command); + if (!await RunCommand(command)) + { + throw new Exception($"'{command}' FAILED"); + } } - public async Task RunSosCommand(string command) + public async Task RunSosCommand(string command) { switch (Debugger) { @@ -397,11 +561,14 @@ public class SOSRunner : IDisposable { foreach (string command in commands) { - await RunCommand(command); + if (!await RunCommand(command)) + { + throw new Exception($"'{command}' FAILED"); + } } } - public async Task RunCommand(string command) + public async Task RunCommand(string command) { if (string.IsNullOrWhiteSpace(command)) { @@ -513,7 +680,7 @@ public class SOSRunner : IDisposable return null; } - private async Task HandleCommand(string input) + private async Task HandleCommand(string input) { if (!await _scriptLogger.WaitForCommandPrompt()) { @@ -569,10 +736,12 @@ public class SOSRunner : IDisposable input = input.Substring(0, firstPOUT) + poutMatchResult + input.Substring(secondPOUT + poutTag.Length); } } - _processRunner.StandardInputWriteLine(_scriptLogger.ProcessCommand(ReplaceVariables(input))); - _lastCommandOutput = await _scriptLogger.WaitForCommandOutput(); - return _lastCommandOutput; + + ScriptLogger.CommandResult result = await _scriptLogger.WaitForCommandOutput(); + _lastCommandOutput = result.CommandOutput; + + return result.CommandSucceeded; } private void LogProcessingReproInfo(string scriptFile, HashSet enabledDefines) @@ -676,10 +845,22 @@ public class SOSRunner : IDisposable class ScriptLogger : TestOutputProcessLogger { + public struct CommandResult + { + public readonly string CommandOutput; // Command output or null if process terminated. + public readonly bool CommandSucceeded; // If true, command succeeded. + + internal CommandResult(string commandOutput, bool commandSucceeded) + { + CommandOutput = commandOutput; + CommandSucceeded = commandSucceeded; + } + } + readonly NativeDebugger _debugger; - readonly List> _taskQueue; + readonly List> _taskQueue; readonly StringBuilder _lastCommandOutput; - TaskCompletionSource _taskSource; + TaskCompletionSource _taskSource; public bool HasProcessExited { get; private set; } @@ -690,31 +871,31 @@ public class SOSRunner : IDisposable { _debugger = debugger; _lastCommandOutput = new StringBuilder(); - _taskQueue = new List>(); + _taskQueue = new List>(); AddTask(); } } private void AddTask() { - _taskSource = new TaskCompletionSource(); + _taskSource = new TaskCompletionSource(); _taskQueue.Add(_taskSource.Task); } public async Task WaitForCommandPrompt() { - Task currentTask = null; + Task currentTask = null; lock (this) { currentTask = _taskQueue[0]; _taskQueue.RemoveAt(0); } - return (await currentTask) != null; + return (await currentTask).CommandOutput != null; } - public Task WaitForCommandOutput() + public Task WaitForCommandOutput() { - Task currentTask = null; + Task currentTask = null; lock (this) { currentTask = _taskQueue[0]; @@ -739,9 +920,11 @@ public class SOSRunner : IDisposable if (stream == ProcessStream.StandardOut) { _lastCommandOutput.Append(data); + string lastCommandOutput = _lastCommandOutput.ToString(); + bool commandError = false; + bool commandEnd = false; - string prompt; switch (_debugger) { case NativeDebugger.Cdb: @@ -752,22 +935,23 @@ public class SOSRunner : IDisposable { return; } - prompt = "> "; + commandEnd = lastCommandOutput.EndsWith("> "); break; case NativeDebugger.Lldb: - prompt = ""; + commandError = lastCommandOutput.EndsWith(""); + commandEnd = commandError || lastCommandOutput.EndsWith(""); break; case NativeDebugger.Gdb: - prompt = "(gdb) "; + commandEnd = lastCommandOutput.EndsWith("(gdb) "); break; default: throw new Exception("Debugger prompt not supported"); } - if (lastCommandOutput.EndsWith(prompt)) + if (commandEnd) { FlushOutput(); - _taskSource.TrySetResult(lastCommandOutput); + _taskSource.TrySetResult(new CommandResult(lastCommandOutput, !commandError)); _lastCommandOutput.Clear(); AddTask(); } @@ -794,7 +978,7 @@ public class SOSRunner : IDisposable base.ProcessExited(runner); FlushOutput(); HasProcessExited = true; - _taskSource.TrySetResult(null); + _taskSource.TrySetResult(new CommandResult(null, true)); } } } @@ -847,6 +1031,11 @@ public static class TestConfigurationExtensions return config.GetValue("GenerateDumpWithLLDB")?.ToLowerInvariant() == "true"; } + public static bool GenerateDumpWithGDB(this TestConfiguration config) + { + return config.GetValue("GenerateDumpWithGDB")?.ToLowerInvariant() == "true"; + } + public static string DebuggeeDumpInputRootDir(this TestConfiguration config) { return TestConfiguration.MakeCanonicalPath(config.GetValue("DebuggeeDumpInputRootDir")); diff --git a/src/SOS/SOS.UnitTests/Scripts/lldbhelper.py b/src/SOS/SOS.UnitTests/Scripts/lldbhelper.py index 3eb073757..47581bcdd 100644 --- a/src/SOS/SOS.UnitTests/Scripts/lldbhelper.py +++ b/src/SOS/SOS.UnitTests/Scripts/lldbhelper.py @@ -10,6 +10,13 @@ def runcommand(debugger, command, result, internal_dict): commandResult = lldb.SBCommandReturnObject() interpreter.HandleCommand(command, commandResult) - print commandResult.GetOutput() - print commandResult.GetError() - print "" + if commandResult.GetOutputSize() > 0: + print commandResult.GetOutput() + + if commandResult.GetErrorSize() > 0: + print commandResult.GetError() + + if commandResult.Succeeded(): + print "" + else: + print "" diff --git a/src/SOS/Strike/gcroot.cpp b/src/SOS/Strike/gcroot.cpp index e4262208e..ca2b88e88 100644 --- a/src/SOS/Strike/gcroot.cpp +++ b/src/SOS/Strike/gcroot.cpp @@ -792,7 +792,7 @@ int GCRootImpl::PrintRootsOnHandleTable(int gen) hr = pEnum->Next(_countof(handles), handles, &fetched); if (FAILED(hr)) { - ExtOut("Failed to request more handles."); + ExtOut("Failed to request more handles.\n"); return total; } @@ -1144,9 +1144,9 @@ GCRootImpl::MTInfo *GCRootImpl::GetMTInfo(TADDR mt) { int nEntries; - if (FAILED(MOVE(nEntries, mt-sizeof(TADDR)))) + if (FAILED(MOVE(nEntries, mt - sizeof(TADDR)))) { - ExtOut("Failed to request number of entries."); + ExtOut("Failed to request number of entries for MethodTable %p.\n", SOS_PTR(mt)); delete curr; return NULL; } diff --git a/src/SOS/Strike/sos.cpp b/src/SOS/Strike/sos.cpp index 64ee4b9a0..ba4611109 100644 --- a/src/SOS/Strike/sos.cpp +++ b/src/SOS/Strike/sos.cpp @@ -471,8 +471,10 @@ namespace sos { int entries = 0; - if (FAILED(MOVE(entries, mt-sizeof(TADDR)))) + if (FAILED(MOVE(entries, mt - sizeof(TADDR)))) + { Throw("Failed to request number of entries."); + } // array of vc? if (entries < 0) diff --git a/src/SOS/lldbplugin/services.cpp b/src/SOS/lldbplugin/services.cpp index 547bc2788..2e853e94e 100644 --- a/src/SOS/lldbplugin/services.cpp +++ b/src/SOS/lldbplugin/services.cpp @@ -788,7 +788,7 @@ exit: { *bytesRead = read; } - return error.Success() ? S_OK : E_FAIL; + return error.Success() || (read != 0) ? S_OK : E_FAIL; } HRESULT @@ -817,7 +817,7 @@ exit: { *bytesWritten = written; } - return error.Success() ? S_OK : E_FAIL; + return error.Success() || (written != 0) ? S_OK : E_FAIL; } //---------------------------------------------------------------------------- -- 2.34.1