From: Mike McLaughlin Date: Thu, 16 May 2019 00:10:53 +0000 (-0700) Subject: Add alpine/MUSL SOS build/packaging (#264) X-Git-Tag: submit/tizen/20190813.035844~6^2^2~29 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=42b168aa2e6b395893b04ab7a170651ae9397ce9;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Add alpine/MUSL SOS build/packaging (#264) Build on alpine docker container. Add new MUSL SOS binaries to dotnet-dump/dotnet-sos. Add new "linux-musl-x64" rid to SOS installer and dotnet-dump. Skip the "clrstack -i" tests that fail on alpine and 3.0.x. --- diff --git a/.vsts-dotnet.yml b/.vsts-dotnet.yml index ef01bcf30..949b610b9 100644 --- a/.vsts-dotnet.yml +++ b/.vsts-dotnet.yml @@ -41,6 +41,22 @@ phases: _BuildArch: x64 _PublishArtifacts: bin/Linux.x64.Release +- template: /eng/build.yml + parameters: + name: Alpine3_6 + agentOs: Linux + dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.6-WithNode-f4d3fe3-20181220200247 + queue: + name: DotNet-Build + demands: + - agent.os -equals Linux + parallel: 2 + matrix: + Build_Release: + _BuildConfig: Release + _BuildArch: x64 + _PublishArtifacts: bin/Linux.x64.Release + - template: /eng/build.yml parameters: name: MacOS @@ -207,6 +223,7 @@ phases: dependsOn: - Windows - CentOS_7 + - Alpine3_6 - MacOS condition: ne(variables['Build.Reason'], 'Schedule') queue: @@ -275,15 +292,6 @@ phases: targetFolder: '$(Build.SourcesDirectory)/artifacts/Windows_NT.x64.Release' condition: succeeded() - - task: ArchiveFiles@2 - displayName: Archive Windows x64 Artifacts - inputs: - archiveType: zip - includeRootFolder: false - rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/Windows_NT.x64.Release' - archiveFile: '$(Build.SourcesDirectory)/artifacts/packages/Release/sos-win-x64.zip' - condition: succeeded() - # Windows x86 download, copy and archive. - task: DownloadBuildArtifacts@0 @@ -310,22 +318,13 @@ phases: targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Windows_NT.x86.Release' condition: succeeded() - - task: ArchiveFiles@2 - displayName: Archive Windows x86 Artifacts - inputs: - archiveType: zip - includeRootFolder: false - rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/bin/Windows_NT.x86.Release' - archiveFile: '$(Build.SourcesDirectory)/artifacts/packages/Release/sos-win-x86.zip' - condition: succeeded() - # Copy and archive the files for windbg extension gallery. - powershell: $(Build.SourcesDirectory)\eng\create-gallery-zip.ps1 $(Build.SourcesDirectory) displayName: Create Extension Gallery Zip File condition: succeeded() - # Linux download, copy and archive + # Linux x64 download, copy and archive - task: DownloadBuildArtifacts@0 displayName: Download Linux Artifacts @@ -351,13 +350,30 @@ phases: targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Linux.x64.Release' condition: succeeded() - - task: ArchiveFiles@2 - displayName: Archive Linux Artifacts + # Linux MUSL x64 download, copy and archive + + - task: DownloadBuildArtifacts@0 + displayName: Download Linux Artifacts + inputs: + artifactName: Alpine3_6_x64_Release + downloadPath: '$(System.ArtifactsDirectory)' + condition: succeeded() + + - task: CopyFiles@2 + displayName: Copy Linux Artifacts + inputs: + sourceFolder: '$(System.ArtifactsDirectory)/Alpine3_6_x64_Release' + contents: ?(libsos.so|libsosplugin.so|sosdocsunix.txt) + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Linux-musl.x64.Release' + condition: succeeded() + + - task: CopyFiles@2 + displayName: Copy Linux Artifacts inputs: - archiveType: zip - includeRootFolder: false - rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/bin/Linux.x64.Release' - archiveFile: '$(Build.SourcesDirectory)/artifacts/packages/Release/sos-linux-x64.zip' + sourceFolder: '$(_SOSNETCorePath)' + contents: '*.dll' + overWrite: true + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Linux-musl.x64.Release' condition: succeeded() # MacOS download, copy and archive @@ -386,15 +402,6 @@ phases: targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/OSX.x64.Release' condition: succeeded() - - task: ArchiveFiles@2 - displayName: Archive MacOS Artifacts - inputs: - archiveType: zip - includeRootFolder: false - rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/bin/OSX.x64.Release' - archiveFile: '$(Build.SourcesDirectory)/artifacts/packages/Release/sos-osx-x64.zip' - condition: succeeded() - # Create nuget package and sign binaries - script: $(Build.SourcesDirectory)\eng\cipack.cmd diff --git a/eng/build-native.sh b/eng/build-native.sh index 7057a72ab..68082d46f 100755 --- a/eng/build-native.sh +++ b/eng/build-native.sh @@ -33,6 +33,7 @@ __CI=false __Verbosity=minimal __TestArgs= __UnprocessedBuildArgs= +__Alpine=false usage() { @@ -195,6 +196,10 @@ while :; do shift ;; + --portablebuild=false) + __PortableBuild=0 + ;; + --clang3.5) __ClangMajorVersion=3 __ClangMinorVersion=5 @@ -348,77 +353,22 @@ build_native() popd } -initHostDistroRid() -{ - __HostDistroRid="" - if [ "$__HostOS" == "Linux" ]; then - if [ -e /etc/os-release ]; then - source /etc/os-release - if [[ $ID == "rhel" ]]; then - # remove the last version digit - VERSION_ID=${VERSION_ID%.*} - fi - __HostDistroRid="$ID.$VERSION_ID-$__HostArch" - if [[ $ID == "alpine" ]]; then - __HostDistroRid="linux-musl-$__HostArch" - fi - elif [ -e /etc/redhat-release ]; then - local redhatRelease=$(> /etc/sudoers" +$docker_bin exec $docker_id su -c "echo '%sudouser ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers" echo "Execute $args" $docker_bin exec --workdir=$source_directory --user $container_user_name $docker_id $args diff --git a/eng/init-distro-rid.sh b/eng/init-distro-rid.sh new file mode 100755 index 000000000..2f8215ffe --- /dev/null +++ b/eng/init-distro-rid.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash + +# initNonPortableDistroRid +# +# Input: +# buildOs: (str) +# buildArch: (str) +# isPortable: (int) +# rootfsDir: (str) +# +# Return: +# None +# +# Notes: +# +# initNonPortableDistroRid will attempt to initialize a non portable rid. These +# rids are specific to distros need to build the product/package and consume +# them on the same platform. +# +# If -portablebuild=false is passed a non-portable rid will be created for any +# distro. +# +# Below is the list of current non-portable platforms. +# +# Builds from the following *must* be non-portable: +# +# | OS | Expected RID | +# ------------------------------------------------- +# | rhel6 | rhel.6-x64 | +# | alpine* | linux-musl-(arch) | +# | freeBSD | freebsd.(version)-x64 | +# +# It is important to note that the function does not return anything, but it +# will set __DistroRid if there is a non-portable distro rid to be used. +# +initNonPortableDistroRid() +{ + # Make sure out parameter is cleared. + __DistroRid= + + local buildOs=$1 + local buildArch=$2 + local isPortable=$3 + local rootfsDir=$4 + + if [ "$buildOs" = "Linux" ]; then + # RHEL 6 is the only distro we will check redHat release for. + if [ -e "${rootfsDir}/etc/os-release" ]; then + source "${rootfsDir}/etc/os-release" + + # We have forced __PortableBuild=0. This is because -portablebuld + # has been passed as false. + if (( ${isPortable} == 0 )); then + if [ "${ID}" == "rhel" ]; then + # remove the last version digit + VERSION_ID=${VERSION_ID%.*} + fi + + nonPortableBuildID="${ID}.${VERSION_ID}-${buildArch}" + fi + + if [ "${ID}" == "alpine" ]; then + export __Alpine=true + fi + + elif [ -e "${rootfsDir}/etc/redhat-release" ]; then + local redhatRelease=$(<${rootfsDir}/etc/redhat-release) + + if [[ "${redhatRelease}" == "CentOS release 6."* || "$redhatRelease" == "Red Hat Enterprise Linux Server release 6."* ]]; then + nonPortableBuildID="rhel.6-${buildArch}" + fi + elif [ -e "${rootfsDir}/android_platform" ]; then + source $rootfsDir/android_platform + nonPortableBuildID="$RID" + fi + fi + + if [ "$buildOs" = "FreeBSD" ]; then + __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'` + nonPortableBuildID="freebsd.$__freebsd_version-${buildArch}" + fi + + if [ "${nonPortableBuildID}" != "" ]; then + export __DistroRid=${nonPortableBuildID} + + # We are using a non-portable build rid. Force __PortableBuild to false. + export __PortableBuild=0 + fi +} + + +# initDistroRidGlobal +# +# Input: +# os: (str) +# arch: (str) +# isPortable: (int) +# rootfsDir?: (nullable:string) +# +# Return: +# None +# +# Notes: +# +# The following out parameters are returned +# +# __DistroRid +# __PortableBuild +# +initDistroRidGlobal() +{ + # __DistroRid must be set at the end of the function. + # Previously we would create a variable __HostDistroRid and/or __DistroRid. + # + # __HostDistroRid was used in the case of a non-portable build, it has been + # deprecated. Now only __DistroRid is supported. It will be used for both + # portable and non-portable rids and will be used in build-packages.sh + + local buildOs=$1 + local buildArch=$2 + local isPortable=$3 + local rootfsDir=$4 + + # Setup whether this is a crossbuild. We can find this out if rootfsDir + # is set. + local isCrossBuild=0 + + if [ -z "${rootfsDir}" ]; then + isCrossBuild=0 + else + # We may have a cross build. Check for the existance of the rootfsDir + if [ -e ${rootfsDir} ]; then + isCrossBuild=1 + else + echo "Error rootfsDir has been passed, but the location is not valid." + exit 1 + fi + fi + + initNonPortableDistroRid ${buildOs} ${buildArch} ${isPortable} ${rootfsDir} + + if [ -z "${__DistroRid}" ]; then + # The non-portable build rid was not set. Set the portable rid. + + export __PortableBuild=1 + local distroRid="" + + # Check for alpine. It is the only portable build that will will have + # its name in the portable build rid. + if [ -e "${rootfsDir}/etc/os-release" ]; then + source "${rootfsDir}/etc/os-release" + if [ "${ID}" = "alpine" ]; then + distroRid="linux-musl-${buildArch}" + fi + fi + + if [ "${distroRid}" == "" ]; then + if [ "$buildOs" = "Linux" ]; then + distroRid="linux-$buildArch" + elif [ "$buildOs" = "OSX" ]; then + distroRid="osx-$buildArch" + elif [ "$buildOs" = "FreeBSD" ]; then + distroRid="freebsd-$buildArch" + fi + fi + + export __DistroRid=${distroRid} + fi + + if [ -z "$__DistroRid" ]; then + echo "DistroRid is not set. This is almost certainly an error" + + exit 1 + else + echo "__DistroRid: ${__DistroRid}" + echo "__RuntimeId: ${__DistroRid}" + + export __RuntimeId=${__DistroRid} + fi +} diff --git a/src/SOS/SOS.Hosting/SOS.Hosting.csproj b/src/SOS/SOS.Hosting/SOS.Hosting.csproj index e6daa9da2..a3d158746 100644 --- a/src/SOS/SOS.Hosting/SOS.Hosting.csproj +++ b/src/SOS/SOS.Hosting/SOS.Hosting.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/SOS/SOS.Hosting/SOSHost.cs b/src/SOS/SOS.Hosting/SOSHost.cs index 579a2f72e..306ae73a7 100644 --- a/src/SOS/SOS.Hosting/SOSHost.cs +++ b/src/SOS/SOS.Hosting/SOSHost.cs @@ -152,20 +152,8 @@ namespace SOS /// public SOSHost(IDataReader dataReader, ISOSHostContext context) { - string os = null; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - os = "win"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { - os = "linux"; - } - if (os == null) { - throw new PlatformNotSupportedException($"Unsupported operating system: {RuntimeInformation.OSDescription}"); - } - string architecture = RuntimeInformation.OSArchitecture.ToString().ToLowerInvariant(); - string rid = os + "-" + architecture; + string rid = InstallHelper.GetRid(); SOSPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), rid); - _wrapper = new LLDBServicesWrapper(this, dataReader, context); } diff --git a/src/SOS/SOS.InstallHelper/InstallHelper.cs b/src/SOS/SOS.InstallHelper/InstallHelper.cs index 737c5ac5c..c0eb3e897 100644 --- a/src/SOS/SOS.InstallHelper/InstallHelper.cs +++ b/src/SOS/SOS.InstallHelper/InstallHelper.cs @@ -50,7 +50,7 @@ namespace SOS { m_writeLine = writeLine; string home = null; - string os = null; + string rid = GetRid(); if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { @@ -58,7 +58,6 @@ namespace SOS if (string.IsNullOrEmpty(home)) { throw new SOSInstallerException("USERPROFILE environment variable not found"); } - os = "win"; } else { @@ -67,21 +66,8 @@ namespace SOS throw new SOSInstallerException("HOME environment variable not found"); } LLDBInitFile = Path.Combine(home, ".lldbinit"); - - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - os = "osx"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { - os = "linux"; - } - } - if (os == null) { - throw new SOSInstallerException($"Unsupported operating system {RuntimeInformation.OSDescription}"); } InstallLocation = Path.GetFullPath(Path.Combine(home, ".dotnet", "sos")); - - string architecture = RuntimeInformation.OSArchitecture.ToString().ToLowerInvariant(); - string rid = os + "-" + architecture; SOSSourcePath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), rid); } @@ -311,6 +297,43 @@ namespace SOS } } + /// + /// Returns the RID + /// + public static string GetRid() + { + string os = null; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + os = "win"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + os = "osx"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + os = "linux"; + try + { + string ostype = File.ReadAllText("/etc/os-release"); + if (ostype.Contains("ID=alpine")) + { + os = "linux-musl"; + } + } + catch (Exception ex) when (ex is FileNotFoundException || ex is DirectoryNotFoundException || ex is IOException) + { + } + } + if (os == null) + { + throw new SOSInstallerException($"Unsupported operating system {RuntimeInformation.OSDescription}"); + } + string architecture = RuntimeInformation.OSArchitecture.ToString().ToLowerInvariant(); + return $"{os}-{architecture}"; + } + private void WriteLine(string format, params object[] args) { m_writeLine?.Invoke(string.Format(format, args)); diff --git a/src/SOS/SOS.UnitTests/SOS.cs b/src/SOS/SOS.UnitTests/SOS.cs index 01cf14562..fc8796df9 100644 --- a/src/SOS/SOS.UnitTests/SOS.cs +++ b/src/SOS/SOS.UnitTests/SOS.cs @@ -3,7 +3,9 @@ // See the LICENSE file in the project root for more information. using Microsoft.Diagnostic.TestHelpers; +using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; using Xunit; @@ -48,25 +50,35 @@ public class SOS { SkipIfArm(config); - // Live - using (SOSRunner runner = await SOSRunner.StartDebugger(config, Output, testName, debuggeeName, debuggeeArguments)) + if (!SOSRunner.IsAlpine()) { - await runner.RunScript(scriptName); + // Live + using (SOSRunner runner = await SOSRunner.StartDebugger(config, Output, testName, debuggeeName, debuggeeArguments)) + { + await runner.RunScript(scriptName); + } } // Generate a crash dump. if (IsCreateDumpConfig(config)) { + if (!useCreateDump && SOSRunner.IsAlpine()) + { + throw new SkipTestException("lldb tests not supported on Alpine"); + } await SOSRunner.CreateDump(config, Output, testName, debuggeeName, debuggeeArguments, useCreateDump); } // Test against a crash dump. if (IsOpenDumpConfig(config)) { - // With cdb (Windows) or lldb (Linux or OSX) - using (SOSRunner runner = await SOSRunner.StartDebugger(config, Output, testName, debuggeeName, debuggeeArguments, SOSRunner.Options.LoadDump)) + if (!SOSRunner.IsAlpine()) { - await runner.RunScript(scriptName); + // With cdb (Windows) or lldb (Linux or OSX) + using (SOSRunner runner = await SOSRunner.StartDebugger(config, Output, testName, debuggeeName, debuggeeArguments, SOSRunner.Options.LoadDump)) + { + await runner.RunScript(scriptName); + } } // With the dotnet-dump analyze tool @@ -91,6 +103,10 @@ public class SOS { // Live only SkipIfArm(config); + if (SOSRunner.IsAlpine()) + { + throw new SkipTestException("lldb tests not supported on Alpine"); + } using (SOSRunner runner = await SOSRunner.StartDebugger(config, Output, testName: "SOS.GCTests", debuggeeName: "GCWhere")) { await runner.RunScript("GCTests.script"); diff --git a/src/SOS/SOS.UnitTests/SOSRunner.cs b/src/SOS/SOS.UnitTests/SOSRunner.cs index b821c74dd..ef87e3b28 100644 --- a/src/SOS/SOS.UnitTests/SOSRunner.cs +++ b/src/SOS/SOS.UnitTests/SOSRunner.cs @@ -709,6 +709,22 @@ public class SOSRunner : IDisposable _outputHelper.Dispose(); } + public static bool IsAlpine() + { + if (OS.Kind == OSKind.Linux) + { + try + { + string ostype = File.ReadAllText("/etc/os-release"); + return ostype.Contains("ID=alpine"); + } + catch (Exception ex) when (ex is FileNotFoundException || ex is DirectoryNotFoundException || ex is IOException) + { + } + } + return false; + } + private static NativeDebugger GetNativeDebuggerToUse(TestConfiguration config, Options options) { switch (OS.Kind) @@ -863,6 +879,10 @@ public class SOSRunner : IDisposable { throw new NotSupportedException("TargetArchitecture " + _config.TargetArchitecture + " not supported"); } + if (IsAlpine()) + { + defines.Add("ALPINE"); + } return defines; } diff --git a/src/SOS/SOS.UnitTests/Scripts/StackAndOtherTests.script b/src/SOS/SOS.UnitTests/Scripts/StackAndOtherTests.script index becb47a11..e811167ec 100644 --- a/src/SOS/SOS.UnitTests/Scripts/StackAndOtherTests.script +++ b/src/SOS/SOS.UnitTests/Scripts/StackAndOtherTests.script @@ -64,9 +64,12 @@ VERIFY:.*\s+\s+\s+SymbolTestApp\.Program\.Foo1\(.*\)\s+\[(?i:.*[ VERIFY:.*\s+\s+\s+SymbolTestApp\.Program\.Main\(.*\)\s+\[(?i:.*[\\|/]SymbolTestApp\.cs) @ 19\]\s* ENDIF:PROJECTK +# Issue #https://github.com/dotnet/diagnostics/issues/266 +IFDEF:NEVER + # Verify that ClrStack with the ICorDebug options works -SOSCOMMAND:ClrStack -i IFDEF:PROJECTK +SOSCOMMAND:ClrStack -i VERIFY:.*\s+Dumping managed stack and managed variables using ICorDebug.\s+ VERIFY:.*\s+Child\s+SP\s+IP\s+Call Site\s+ VERIFY:.*\s+\s+\s+\[NativeStackFrame\]\s+ @@ -94,6 +97,10 @@ VERIFY:.*\s+\s+\s+\[DEFAULT\] Void SymbolTestApp\.Program\.Main\ VERIFY:.*\s+Stack walk complete.\s+ ENDIF:PROJECTK +ENDIF:NEVER + +!IFDEF:ALPINE + # Verify DumpStackObjects works IFDEF:PROJECTK SOSCOMMAND:DumpStackObjects @@ -112,6 +119,8 @@ VERIFY:.*\s+\s+\s+System\.String.* VERIFY:.*\s+\s+\s+System\.String\[\].* ENDIF:PROJECTK +ENDIF:ALPINE + !IFDEF:DOTNETDUMP IFDEF:PROJECTK diff --git a/src/SOS/SOS.UnitTests/Scripts/StackTests.script b/src/SOS/SOS.UnitTests/Scripts/StackTests.script index 4805c84f0..05ad94536 100644 --- a/src/SOS/SOS.UnitTests/Scripts/StackTests.script +++ b/src/SOS/SOS.UnitTests/Scripts/StackTests.script @@ -66,6 +66,9 @@ VERIFY:\s+[r|e]ax=\s+[r|e]bx=\s+[r|e]cx=\s+ ENDIF:64BIT ENDIF:PROJECTK +# Issue #https://github.com/dotnet/diagnostics/issues/266 +IFDEF:NEVER + # 5) Verifying that ClrStack with the ICorDebug options works SOSCOMMAND:ClrStack -i VERIFY:.*\s+Dumping managed stack and managed variables using ICorDebug.\s+ @@ -90,6 +93,8 @@ VERIFY:.*\s+\s+\s+\[DEFAULT\] Void NestedExceptionTest\.Program\ VERIFY:.*\s+Stack walk complete.\s+ ENDIF:PROJECTK +ENDIF:NEVER + # 7) Verify DumpStackObjects works IFDEF:PROJECTK SOSCOMMAND:DumpStackObjects diff --git a/src/Tools/dotnet-dump/dotnet-dump.csproj b/src/Tools/dotnet-dump/dotnet-dump.csproj index 11043ac00..d0ca38a8f 100644 --- a/src/Tools/dotnet-dump/dotnet-dump.csproj +++ b/src/Tools/dotnet-dump/dotnet-dump.csproj @@ -48,6 +48,18 @@ tools/$(TargetFramework)/any/linux-x64 + + tools/$(TargetFramework)/any/linux-musl-x64 + + + tools/$(TargetFramework)/any/linux-musl-x64 + + + tools/$(TargetFramework)/any/linux-musl-x64 + + + tools/$(TargetFramework)/any/linux-musl-x64 + tools/$(TargetFramework)/any/osx-x64 diff --git a/src/Tools/dotnet-sos/dotnet-sos.csproj b/src/Tools/dotnet-sos/dotnet-sos.csproj index e64fe33df..9c412476f 100644 --- a/src/Tools/dotnet-sos/dotnet-sos.csproj +++ b/src/Tools/dotnet-sos/dotnet-sos.csproj @@ -43,6 +43,18 @@ tools/$(TargetFramework)/any/linux-x64 + + tools/$(TargetFramework)/any/linux-musl-x64 + + + tools/$(TargetFramework)/any/linux-musl-x64 + + + tools/$(TargetFramework)/any/linux-musl-x64 + + + tools/$(TargetFramework)/any/linux-musl-x64 + tools/$(TargetFramework)/any/osx-x64