From: Mike McLaughlin Date: Tue, 16 Apr 2024 22:56:24 +0000 (-0700) Subject: Move the symstore repo to diagnostics (#4603) X-Git-Tag: accepted/tizen/unified/20241231.014852~40^2~111 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=326ccf39e5685a37de66edad08838994a751b732;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Move the symstore repo to diagnostics (#4603) This PR moves the main symstore assemblies, tests and dotnet-symbol tool to the diagnostics repo. It doesn't move the NugetSymbolServer or the SymClient test tool. Neither are necessary for symbol indexing and downloading. The docs/specs were not copied either. I really don't want two copies of the indexing and download protocol specs. There is a base line commit that is exactly what is currently in the symstore repro and several other commits that fix all the coding convention and analyzer changes required to live in the diagnostics repo. The only real functional change in dotnet-symbol/Microsoft.SymbolStore is that ConfigureAwait(false) was added to all the awaits (I think the default without ConfigureAwait is true). --- diff --git a/diagnostics.sln b/diagnostics.sln index c34b90620..4ffd68285 100644 --- a/diagnostics.sln +++ b/diagnostics.sln @@ -273,6 +273,18 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Diagnostics.WebSo EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestExtension", "src\tests\TestExtension\TestExtension.csproj", "{C6EB3C21-FDFF-4CF0-BE3A-3D1A3924408E}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.FileFormats", "src\Microsoft.FileFormats\Microsoft.FileFormats.csproj", "{830A70D3-E604-467A-9846-6C5DF5BD3976}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SymbolStore", "src\Microsoft.SymbolStore\Microsoft.SymbolStore.csproj", "{438A539E-6AF2-4402-BBA0-E2AAC71A28A1}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "dotnet-symbol", "src\Tools\dotnet-symbol\dotnet-symbol.csproj", "{B6C33C85-08A7-47D9-BEA8-36164BB3653B}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestHelpers", "src\tests\TestHelpers\TestHelpers.csproj", "{C32F2858-6B5F-4967-ABC4-852B6399C4AE}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.FileFormats.UnitTests", "src\tests\Microsoft.FileFormats.UnitTests\Microsoft.FileFormats.UnitTests.csproj", "{44F93947-8FD4-4946-8AE5-EF6D25970CC7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SymbolStore.UnitTests", "src\tests\Microsoft.SymbolStore.UnitTests\Microsoft.SymbolStore.UnitTests.csproj", "{C2422836-BA25-4751-9060-7C7890085869}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Checked|Any CPU = Checked|Any CPU @@ -1952,6 +1964,246 @@ Global {C6EB3C21-FDFF-4CF0-BE3A-3D1A3924408E}.RelWithDebInfo|x64.Build.0 = Release|Any CPU {C6EB3C21-FDFF-4CF0-BE3A-3D1A3924408E}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU {C6EB3C21-FDFF-4CF0-BE3A-3D1A3924408E}.RelWithDebInfo|x86.Build.0 = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Checked|Any CPU.Build.0 = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Checked|ARM.ActiveCfg = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Checked|ARM.Build.0 = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Checked|ARM64.ActiveCfg = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Checked|ARM64.Build.0 = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Checked|x64.ActiveCfg = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Checked|x64.Build.0 = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Checked|x86.ActiveCfg = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Checked|x86.Build.0 = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Debug|Any CPU.Build.0 = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Debug|ARM.ActiveCfg = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Debug|ARM.Build.0 = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Debug|ARM64.ActiveCfg = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Debug|ARM64.Build.0 = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Debug|x64.ActiveCfg = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Debug|x64.Build.0 = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Debug|x86.ActiveCfg = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Debug|x86.Build.0 = Debug|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Release|Any CPU.ActiveCfg = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Release|Any CPU.Build.0 = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Release|ARM.ActiveCfg = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Release|ARM.Build.0 = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Release|ARM64.ActiveCfg = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Release|ARM64.Build.0 = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Release|x64.ActiveCfg = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Release|x64.Build.0 = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Release|x86.ActiveCfg = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.Release|x86.Build.0 = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.RelWithDebInfo|ARM.ActiveCfg = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.RelWithDebInfo|ARM.Build.0 = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.RelWithDebInfo|ARM64.ActiveCfg = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.RelWithDebInfo|ARM64.Build.0 = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.RelWithDebInfo|x64.Build.0 = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU + {830A70D3-E604-467A-9846-6C5DF5BD3976}.RelWithDebInfo|x86.Build.0 = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Checked|Any CPU.Build.0 = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Checked|ARM.ActiveCfg = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Checked|ARM.Build.0 = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Checked|ARM64.ActiveCfg = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Checked|ARM64.Build.0 = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Checked|x64.ActiveCfg = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Checked|x64.Build.0 = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Checked|x86.ActiveCfg = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Checked|x86.Build.0 = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Debug|ARM.ActiveCfg = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Debug|ARM.Build.0 = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Debug|ARM64.ActiveCfg = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Debug|ARM64.Build.0 = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Debug|x64.ActiveCfg = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Debug|x64.Build.0 = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Debug|x86.ActiveCfg = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Debug|x86.Build.0 = Debug|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Release|Any CPU.Build.0 = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Release|ARM.ActiveCfg = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Release|ARM.Build.0 = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Release|ARM64.ActiveCfg = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Release|ARM64.Build.0 = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Release|x64.ActiveCfg = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Release|x64.Build.0 = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Release|x86.ActiveCfg = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.Release|x86.Build.0 = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.RelWithDebInfo|ARM.ActiveCfg = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.RelWithDebInfo|ARM.Build.0 = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.RelWithDebInfo|ARM64.ActiveCfg = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.RelWithDebInfo|ARM64.Build.0 = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.RelWithDebInfo|x64.Build.0 = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1}.RelWithDebInfo|x86.Build.0 = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Checked|Any CPU.Build.0 = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Checked|ARM.ActiveCfg = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Checked|ARM.Build.0 = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Checked|ARM64.ActiveCfg = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Checked|ARM64.Build.0 = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Checked|x64.ActiveCfg = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Checked|x64.Build.0 = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Checked|x86.ActiveCfg = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Checked|x86.Build.0 = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Debug|ARM.ActiveCfg = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Debug|ARM.Build.0 = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Debug|ARM64.ActiveCfg = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Debug|ARM64.Build.0 = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Debug|x64.ActiveCfg = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Debug|x64.Build.0 = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Debug|x86.ActiveCfg = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Debug|x86.Build.0 = Debug|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Release|Any CPU.Build.0 = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Release|ARM.ActiveCfg = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Release|ARM.Build.0 = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Release|ARM64.ActiveCfg = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Release|ARM64.Build.0 = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Release|x64.ActiveCfg = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Release|x64.Build.0 = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Release|x86.ActiveCfg = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.Release|x86.Build.0 = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.RelWithDebInfo|ARM.ActiveCfg = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.RelWithDebInfo|ARM.Build.0 = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.RelWithDebInfo|ARM64.ActiveCfg = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.RelWithDebInfo|ARM64.Build.0 = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.RelWithDebInfo|x64.Build.0 = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU + {B6C33C85-08A7-47D9-BEA8-36164BB3653B}.RelWithDebInfo|x86.Build.0 = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Checked|Any CPU.Build.0 = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Checked|ARM.ActiveCfg = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Checked|ARM.Build.0 = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Checked|ARM64.ActiveCfg = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Checked|ARM64.Build.0 = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Checked|x64.ActiveCfg = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Checked|x64.Build.0 = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Checked|x86.ActiveCfg = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Checked|x86.Build.0 = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Debug|ARM.ActiveCfg = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Debug|ARM.Build.0 = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Debug|ARM64.ActiveCfg = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Debug|ARM64.Build.0 = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Debug|x64.ActiveCfg = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Debug|x64.Build.0 = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Debug|x86.ActiveCfg = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Debug|x86.Build.0 = Debug|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Release|Any CPU.Build.0 = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Release|ARM.ActiveCfg = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Release|ARM.Build.0 = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Release|ARM64.ActiveCfg = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Release|ARM64.Build.0 = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Release|x64.ActiveCfg = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Release|x64.Build.0 = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Release|x86.ActiveCfg = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.Release|x86.Build.0 = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.RelWithDebInfo|ARM.ActiveCfg = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.RelWithDebInfo|ARM.Build.0 = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.RelWithDebInfo|ARM64.ActiveCfg = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.RelWithDebInfo|ARM64.Build.0 = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.RelWithDebInfo|x64.Build.0 = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU + {C32F2858-6B5F-4967-ABC4-852B6399C4AE}.RelWithDebInfo|x86.Build.0 = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Checked|Any CPU.Build.0 = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Checked|ARM.ActiveCfg = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Checked|ARM.Build.0 = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Checked|ARM64.ActiveCfg = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Checked|ARM64.Build.0 = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Checked|x64.ActiveCfg = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Checked|x64.Build.0 = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Checked|x86.ActiveCfg = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Checked|x86.Build.0 = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Debug|ARM.ActiveCfg = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Debug|ARM.Build.0 = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Debug|ARM64.ActiveCfg = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Debug|ARM64.Build.0 = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Debug|x64.ActiveCfg = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Debug|x64.Build.0 = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Debug|x86.ActiveCfg = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Debug|x86.Build.0 = Debug|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Release|Any CPU.Build.0 = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Release|ARM.ActiveCfg = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Release|ARM.Build.0 = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Release|ARM64.ActiveCfg = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Release|ARM64.Build.0 = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Release|x64.ActiveCfg = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Release|x64.Build.0 = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Release|x86.ActiveCfg = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.Release|x86.Build.0 = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.RelWithDebInfo|ARM.ActiveCfg = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.RelWithDebInfo|ARM.Build.0 = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.RelWithDebInfo|ARM64.ActiveCfg = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.RelWithDebInfo|ARM64.Build.0 = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.RelWithDebInfo|x64.Build.0 = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU + {44F93947-8FD4-4946-8AE5-EF6D25970CC7}.RelWithDebInfo|x86.Build.0 = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Checked|Any CPU.Build.0 = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Checked|ARM.ActiveCfg = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Checked|ARM.Build.0 = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Checked|ARM64.ActiveCfg = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Checked|ARM64.Build.0 = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Checked|x64.ActiveCfg = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Checked|x64.Build.0 = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Checked|x86.ActiveCfg = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Checked|x86.Build.0 = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Debug|ARM.ActiveCfg = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Debug|ARM.Build.0 = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Debug|ARM64.ActiveCfg = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Debug|ARM64.Build.0 = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Debug|x64.ActiveCfg = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Debug|x64.Build.0 = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Debug|x86.ActiveCfg = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Debug|x86.Build.0 = Debug|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Release|Any CPU.Build.0 = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Release|ARM.ActiveCfg = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Release|ARM.Build.0 = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Release|ARM64.ActiveCfg = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Release|ARM64.Build.0 = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Release|x64.ActiveCfg = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Release|x64.Build.0 = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Release|x86.ActiveCfg = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.Release|x86.Build.0 = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.RelWithDebInfo|ARM.ActiveCfg = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.RelWithDebInfo|ARM.Build.0 = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.RelWithDebInfo|ARM64.ActiveCfg = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.RelWithDebInfo|ARM64.Build.0 = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.RelWithDebInfo|x64.Build.0 = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU + {C2422836-BA25-4751-9060-7C7890085869}.RelWithDebInfo|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2012,6 +2264,12 @@ Global {E8F133F8-4D20-475D-9D16-2BA236DAB65F} = {03479E19-3F18-49A6-910A-F5041E27E7C0} {1043FA82-37CC-4809-80DC-C1EB06A55133} = {19FAB78C-3351-4911-8F0C-8C6056401740} {C6EB3C21-FDFF-4CF0-BE3A-3D1A3924408E} = {03479E19-3F18-49A6-910A-F5041E27E7C0} + {830A70D3-E604-467A-9846-6C5DF5BD3976} = {19FAB78C-3351-4911-8F0C-8C6056401740} + {438A539E-6AF2-4402-BBA0-E2AAC71A28A1} = {19FAB78C-3351-4911-8F0C-8C6056401740} + {B6C33C85-08A7-47D9-BEA8-36164BB3653B} = {B62728C8-1267-4043-B46F-5537BBAEC692} + {C32F2858-6B5F-4967-ABC4-852B6399C4AE} = {03479E19-3F18-49A6-910A-F5041E27E7C0} + {44F93947-8FD4-4946-8AE5-EF6D25970CC7} = {03479E19-3F18-49A6-910A-F5041E27E7C0} + {C2422836-BA25-4751-9060-7C7890085869} = {03479E19-3F18-49A6-910A-F5041E27E7C0} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {46465737-C938-44FC-BE1A-4CE139EBB5E0} diff --git a/documentation/symbols/Myget_symbol_server_extensions.md b/documentation/symbols/Myget_symbol_server_extensions.md new file mode 100644 index 000000000..7e7b9a994 --- /dev/null +++ b/documentation/symbols/Myget_symbol_server_extensions.md @@ -0,0 +1,42 @@ +# MyGet symbol server extensions # + +This is a feature request for MyGet to create an implementation of the [Zip Package Based Symbol Server](Package_Based_Symbol_Server.md). As far as I can tell MyGet already exposes the [Simple Symbol Query Protocol](Simple_Symbol_Query_Protocol.md), albeit never specified with that name and marketed only as being SymSrv-compatible. + +## Endpoints ## + +Myget would expose the same per-feed symbol server endpoints it does today, for example: + + https://www.myget.org/F/dev-feed/symbols + https://www.myget.org/F/dev-feed/auth/11111111-1111-1111-1111-11111111111/symbols + +If possible, it would also be nice to define an aggregate feed that can serve up any file from a set of feeds. We could then configure a larger organizational aggregate feed for customers to use. This probably needs some further discussion once Maarten is back from vacation. + +## Packages ## + +The service operates over all the packages that are active on that feed at a given time. Active packages are the packages that have been uploaded and not yet deleted either directly by the developer or implicitly by the configurable myget retention policies. + +## Other clientKey sources ## + +This feature doesn't preclude MyGet from continuing to satisfy requests using clientKeys there were automatically derived from package contents or any other source, but those keys should never take precedence to a key mapping provided explicitly by the client in symbol\_index.json. + +## Access Privileges ## + +The symbol service is expected to be equally accessible as reading the underlying feed by default. If the underlying feed requires authentication, so to should the symbol service end-point. We aren't requesting MyGet add any additional configurability, but its fine if they did. + +## Package Management ## + +We believe the existing mechanisms MyGet uses to upload and manage packages on a feed are sufficient to implicitly manage the content on the symbol service. No additional requests here. + +## Performance ## + +We don't yet know what loads to expect. Although the load will probably be far lower to start, here an initial guess at a load this service might need to scale to: + +- 100,000 packages per-feed +- 100 million aggregate clientKeys per-feed +- 1000 requests/sec (burst) +- 1 GB/s (burst) +- 1 million requests/day (sustained load) +- 1 TB/day (sustained load) +- Average response time: < 1 sec, 99% response time < 5 sec (measured from the time the request arrives at the myget server to the time file data begins streaming back) + +My hope is that this is still well within the scalability range of other aspects of the existing myget service and thus doesn't require any significant investment in new infrastructure or more complex service logic. \ No newline at end of file diff --git a/documentation/symbols/Package_Based_Symbol_Server.md b/documentation/symbols/Package_Based_Symbol_Server.md new file mode 100644 index 000000000..f003b3ada --- /dev/null +++ b/documentation/symbols/Package_Based_Symbol_Server.md @@ -0,0 +1,43 @@ +# Zip Package Based Symbol Server # + +A zip package based symbol server is a network service that implements the [Simple Symbol Query Protocol](Simple_Symbol_Query_Protocol.md) (SSQP) using a set of zip compressed files with specific contents to define the files that are available for download and the clientKey/filenames that address them. This specification defines the format of the zip packages. Although this format is intended to be fully compatible with NuGet package and NuGet symbol package specifications, the zip packages are not required to qualify under either standard. + +## The zip package format ## + +Each symbol package is a compressed container of files in the zip format. At the root of the container there must be one file named 'symbol\_index.json'. There may be an arbitrary number of other files in the container either at the root level or in arbitrarily nested sub-containers. The symbol\_index.json is a json array where each element identifies a clientKey and a blobPath, the corresponding file in the zip that should be returned by an SSQP request for the clientKey. The blobPath is a filename, preceded by 0 or more container names using '/' as the separator: + +```json +[ + { + "clientKey" : "12387532", + "blobPath" : "debug_info.txt" + }, + { + "clientKey" : "MyProgram.exe/09safnf82asddasdqwd998vds/MyProgram.exe", + "blobPath" : "MyProgram.exe" + }, + { + "clientKey" : "12-09", + "blobPath" : "Content/localized/en-us/data.xml" + }, + { + "clientKey" : "23456", + "blobPath" : "Content/localized/en-us/data.xml" + } +] +``` + +## Expected service behavior ## + +In order to implement the [Simple Symbol Query Protocol](Simple_Symbol_Query_Protocol.md) the service must identify a map entry in some package's symbol\_index.json which has the matching clientKey and then return the file pointed to by blobPath. If there is more than one entry in the same package which has the same clientKey value that is a bad package and the service may handle the error in an implementation specific way. If more than one package defines an entry with the same clientKey the service may choose one of the entries arbitrarily using implementation specific behavior. If the clientKey isn't present in any package the service may return a 404, or it may fallback to other implementation specific techniques to satisfy the request. The service must be prepared to handle having N different clientKeys all refer to the same blob. + + +## Combining with other sources of clientKeys ## + +It is possible to run an SSQP service that uses more than one data source to determine the total set of clientKey/filename requests it is able to respond to. For example most existing NuGet symbol service implementations compute their own mappings for files in specific portions of a NuGet symbol package if the files are one of a few well-known formats. This specification explicitly allows for these other data sources to be integrated but implementers should document what happens in the event two disparate sources of mapping information request different blobs to be returned for the same clientKey. + +## Usage notes ## + +SSQP and the package based symbol server are best suited for controlled settings in which all publishers agree on the same conventions for publishing content. For example a set of developers on a particular project or the employees of a small company. + +If there is disagreement (or outright malicious actors) these services do not intrinsically provide any way to determine who deserves to be more trusted. Publishers could easily submit packages with conflicting indexing information which will give undefined results to the SSQP clients. Running this service for a large group, such as worldwide unrestricted publishing access, is therefore not recommended without adding additional arbitration procedures. \ No newline at end of file diff --git a/documentation/symbols/README.md b/documentation/symbols/README.md new file mode 100644 index 000000000..bd607a0e7 --- /dev/null +++ b/documentation/symbols/README.md @@ -0,0 +1,11 @@ +# Table of Contents # + +[Simple Symbol Server Protocol (SSQP)](Simple_Symbol_Query_Protocol.md) - Describes the protocol used by clients to download files from a symbol server + +[Package Based Symbol Server](Package_Based_Symbol_Server.md) - Describes a particular subset of SSQP servers that use specially formatted .zip or nuget packages to represent the files that will be made available. + +[Myget symbol server feature request](Myget_symbol_server_extensions.md) - A feature request to the myget team to implement a Package Based Symbol Server + +[SSQP Key Conventions](SSQP_Key_Conventions.md) - A description of the standard key formats intended for use with SSQP. + +[CLR Private SSQP Key Conventions](SSQP_CLR_Private_Key_Conventions.md) - A description of non-standard key formats produced solely by the CLR team \ No newline at end of file diff --git a/documentation/symbols/SSQP_CLR_Private_Key_Conventions.md b/documentation/symbols/SSQP_CLR_Private_Key_Conventions.md new file mode 100644 index 000000000..77fc491fc --- /dev/null +++ b/documentation/symbols/SSQP_CLR_Private_Key_Conventions.md @@ -0,0 +1,66 @@ +# SSQP CLR Private Key conventions # + +These conventions are private extensions to the normal [SSQP conventions](SSQP_Key_Conventions.md). They fulfill niche scenarios specific to the CLR product and are not expected to be used within any general purpose index generating tool. + +## Basic rules ## + +The private conventions use the same basic rules for bytes, bytes sequences, integers, strings, etc as described in the standard conventions. + +## Key formats ## + + +### PE-filesize-timestamp-coreclr + +This key indexes an sos\*.dll or mscordaccore\*.dll file that should be used to debug a given coreclr.dll. The lookup key is computed similar to PE-timestamp-filesize except the timestamp and filesize values are taken from coreclr.dll rather than the file being indexed. +Example: + +**File names:** `mscordaccore.dll, sos.dll or SOS.NETCore.dll` + +**CoreCLR’s COFF header Timestamp field:** `0x542d5742` + +**CoreCLR’s COFF header SizeOfImage field:** `0x32000` + +**Lookup keys:** + + mscordaccore.dll/542d574200032000/mscordaccore.dll + sos.dll/542d574200032000/sos.dll + SOS.NETCore.dll/542d574200032000/SOS.NETCore.dll + + +### ELF-buildid-coreclr + +This applies to any file named libmscordaccore.so or libsos.so that should be used to debug a given libcoreclr.so. The key is computed similarly to ELF-buildid except the note bytes is retrieved from the libcoreclr.so file and prefixed with 'elf-buildid-coreclr-': + +`/elf-buildid-coreclr-/` + +Example: + +**File names:** `libmscordaccore.so, libsos.so or SOS.NETCore.dll` + +**libcoreclr.so’s build note bytes:** `0x18, 0x0a, 0x37, 0x3d, 0x6a, 0xfb, 0xab, 0xf0, 0xeb, 0x1f, 0x09, 0xbe, 0x1b, 0xc4, 0x5b, 0xd7, 0x96, 0xa7, 0x10, 0x85` + +**Lookup keys:** + + libmscordaccore.so/elf-buildid-coreclr-180a373d6afbabf0eb1f09be1bc45bd796a71085/libmscordaccore.so + libsos.so/elf-buildid-coreclr-180a373d6afbabf0eb1f09be1bc45bd796a71085/libsos.so + sos-netcore.dll/elf-buildid-coreclr-180a373d6afbabf0eb1f09be1bc45bd796a71085/sos-netcore.dll + + +### Mach-uuid-coreclr + +This applies to any file named libmscordaccore.dylib or libsos.dylib that should be used to debug a given libcoreclr.dylib. The key is computed similarly to Mach-uuid except the uuid is retrieved from the libcoreclr.dylib file and prefixed with 'mach-uuid-coreclr-': + +`/mach-uuid-coreclr-/` + +Example: + +**File names:** `libmscordaccore.dylb, libsos.dylib or SOS.NETCore.dll` + +**libcoreclr.dylib’s uuid bytes:** `0x49, 0x7B, 0x72, 0xF6, 0x39, 0x0A, 0x44, 0xFC, 0x87, 0x8E, 0x5A, 0x2D, 0x63, 0xB6, 0xCC, 0x4B` + +**Lookup keys:** + + libmscordaccore.dylib/mach-uuid-coreclr-497b72f6390a44fc878e5a2d63b6cc4b/libmscordaccore.dylib + libsos.dylib/mach-uuid-coreclr-497b72f6390a44fc878e5a2d63b6cc4b/libsos.dylib + sos.netcore.dll/mach-uuid-coreclr-497b72f6390a44fc878e5a2d63b6cc4b/sos.netcore.dll + diff --git a/documentation/symbols/SSQP_Key_Conventions.md b/documentation/symbols/SSQP_Key_Conventions.md new file mode 100644 index 000000000..34d5e4f75 --- /dev/null +++ b/documentation/symbols/SSQP_Key_Conventions.md @@ -0,0 +1,159 @@ +# SSQP Key conventions # + +When using [SSQP](Simple_Symbol_Query_Protocol.md) it is critical that the content publishers and content consumers agree what keys should correspond to which files. Although any publisher-consumer pair is free to create private agreements, using a standard key format offers the widest compatibility. + + +## Key formatting basic rules +Unless otherwise specified: + +- Bytes: Convert to characters by splitting the byte into the most significant 4 bits and 4 least significant bits, each of which has value 0-15. Convert each of those chunks to the corresponding lower case hexadecimal character. Last concatenate the two characters putting the most significant bit chunk first. For example 0 => '00', 1 => '01', 45 => '2d', 185 => 'b9' +- Byte sequences: Convert to characters by converting each byte as above and then concatenating the characters. For example 2,45,4 => '022d04' +- Multi-byte integers: Convert to characters by first converting it to a big-endian byte sequence next convert the sequence as above and finally trim all leading '0' characters. Example 3,559,453,162 => 'd428f1ea', 114 => '72' +- strings: Convert all the characters to lower-case +- guid: The guid consists of a 4 byte integer, two 2 byte integers, and a sequence of 8 bytes. It is formatted by converting each portion to hex characters without trimming leading '0' characters on the integers, then concatenate the results. Example: { 0x097B72F6, 0x390A, 0x04FC, { 0x87, 0x8E, 0x5A, 0x2D, 0x63, 0xB6, 0xCC, 0x4B } } => '097b72f6390a04fc878e5a2d63b6cc4b' + +## Key formats + + +### PE-timestamp-filesize +This key references Windows Portable Executable format files which commonly have .dll or .exe suffixes. The key is computed by extracting the Timestamp (4 byte integer) and SizeOfImage (4 byte integer) fields from the COFF header in PE image. The key is formatted: + +`//` + +Note that the timeStamp is always printed as eight digits (with leading zeroes as needed) using upper-case for ‘A’ to ‘F’ (important if your symbol server is case sensitive), whereas the image size is printed using as few digits as needed, in lower-case. + +Example: + +**File name:** `Foo.exe` + +**COFF header Timestamp field:** `0x542d574e` + +**COFF header SizeOfImage field:** `0xc2000` + +**Lookup key:** `foo.exe/542D574Ec2000/foo.exe` + + +### PDB-Signature-Age + +This applies to Microsoft C++ Symbol Format, commonly called PDB and using files with the .pdb file extension. The key is computed by extracting the Signature (guid) and Age (4 byte integer) values from the guid stream within MSF container. The final key is formatted: + +`//` + +Example: + +**File name:** `Foo.pdb` + +**Signature field:** `{ 0x497B72F6, 0x390A, 0x44FC, { 0x87, 0x8E, 0x5A, 0x2D, 0x63, 0xB6, 0xCC, 0x4B } }` + +**Age field:** `0x1` + +**Lookup key**: `foo.pdb/497b72f6390a44fc878e5a2d63b6cc4b1/foo.pdb` + + +### Portable-Pdb-Signature + +This applies to Microsoft .Net portable PDB format files, commonly using the suffix .pdb. The Portable PDB format uses the same key format as Windows PDBs, except that 0xFFFFFFFF (UInt32.MaxValue) is used for the age. In other words, the key is computed by extracting the Signature (guid) from debug metadata header and combining it with 'FFFFFFFF'. The final key is formatted: + +`/FFFFFFFF/` + +Example: + +**File name:** `Foo.pdb` + +**Signature field:** `{ 0x497B72F6, 0x390A, 0x44FC { 0x87, 0x8E, 0x5A, 0x2D, 0x63, 0xB6, 0xCC, 0x4B } }` + +**Lookup key:** `foo.pdb/497b72f6390a44fc878e5a2d63b6cc4bFFFFFFFF/foo.pdb` + + +### ELF-buildid + +The ELF format files indexed with the ELF-buildid suffix are expected to be the exact image that is loaded in a process or core dump, doesn’t require the module to be stripped of its symbols (although it usually is), and commonly uses the .so suffix or no suffix. The key is computed by reading the 20 byte sequence of the ELF Note section that is named “GNU” and that has note type GNU Build Id (3). If byte sequence is smaller than 20 bytes, bytes of value 0x00 should be added until the byte sequence is 20 bytes long. The final key is formatted: + +`/elf-buildid-/` + +Example: + +**File name:** `foo.so` + +**Build note bytes:** `0x18, 0x0a, 0x37, 0x3d, 0x6a, 0xfb, 0xab, 0xf0, 0xeb, 0x1f, 0x09, 0xbe, 0x1b, 0xc4, 0x5b, 0xd7, 0x96, 0xa7, 0x10, 0x85` + +**Lookup key:** `foo.so/elf-buildid-180a373d6afbabf0eb1f09be1bc45bd796a71085/foo.so` + + +### ELF-buildid-sym + +The ELF format files indexed with the ELF-buildid-sym suffix are the result of the stripping process and contain only the symbols from the ELF-buildid indexed module. They commonly end in ‘.debug’, ‘.so.dbg’ or ‘.dbg’. The key is computed by reading the 20 byte sequence of the ELF Note section that is named “GNU” and that has note type GNU Build Id (3). If byte sequence is smaller than 20 bytes, bytes of value 0x00 should be added until the byte sequence is 20 bytes long. The file name is not used in the index because there are cases where all we have is the build id. The final key is formatted: + +`_.debug/elf-buildid-sym-/_.debug` + +Example: + +**File name:** `foo.so.dbg` + +**Build note bytes:** `0x18, 0x0a, 0x37, 0x3d, 0x6a, 0xfb, 0xab, 0xf0, 0xeb, 0x1f, 0x09, 0xbe, 0x1b, 0xc4, 0x5b, 0xd7, 0x96, 0xa7, 0x10, 0x85` + +**Lookup key:** `_.debug/elf-buildid-sym-180a373d6afbabf0eb1f09be1bc45bd796a71085/_.debug` + +Example: + +**File name:** `bar.so.dbg` + +**Build note bytes:** `0x18, 0x0a, 0x37, 0x3d, 0x6a, 0xfb, 0xab, 0xf0, 0xeb, 0x1f, 0x09, 0xbe, 0x1b, 0xc4, 0x5b, 0xd7` + +**Lookup key:** `_.debug/elf-buildid-sym-180a373d6afbabf0eb1f09be1bc45bd700000000/_.debug` + + +### Mach-uuid +This applies to any MachO format files that have been stripped of debugging information, commonly ending in 'dylib'. The key is computed by reading the uuid byte sequence of the MachO LC_UUID load command. The final key is formatted: + +`/mach-uuid-/` + +Example: + +**File name:** `foo.dylib` + +**Uuid bytes:** `0x49, 0x7B, 0x72, 0xF6, 0x39, 0x0A, 0x44, 0xFC, 0x87, 0x8E, 0x5A, 0x2D, 0x63, 0xB6, 0xCC, 0x4B` + +**Lookup key:** `foo.dylib/mach-uuid-497b72f6390a44fc878e5a2d63b6cc4b/foo.dylib` + + +### Mach-uuid-sym + +This applies to any MachO format files that have not been stripped of debugging information, commonly ending in '.dylib.dwarf'. The key is computed by reading the uuid byte sequence of the MachO LC_UUID load command. The final key is formatted: + +`_.dwarf/mach-uuid-sym-/_.dwarf` + +Example: + +**File name:** `foo.dylib.dwarf` + +**Uuid bytes:** `0x49, 0x7B, 0x72, 0xF6, 0x39, 0x0A, 0x44, 0xFC, 0x87, 0x8E, 0x5A, 0x2D, 0x63, 0xB6, 0xCC, 0x4B` + +**Lookup key:** `_.dwarf/mach-uuid-sym-497b72f6390a44fc878e5a2d63b6cc4b/_.dwarf` + + +### SHA1 + +This applies to any file, but is commonly used on sources. The key is computed by calculating a SHA1 hash, then formatting the 20 byte hash sequence prepended with “sha1-“ + +Example: + +**File name:** `Foo.cs` + +**Sha1 hash bytes:** `0x49, 0x7B, 0x72, 0xF6, 0x39, 0x0A, 0x44, 0xFC, 0x87, 0x8E, 0x5A, 0x2D, 0x63, 0xB6, 0xCC, 0x4B, 0x0C, 0x2D, 0x99, 0x84` + +**Lookup key:** `foo.cs/sha1-497b72f6390a44fc878e5a2d63b6cc4b0c2d9984/foo.cs` + +### R2R PerfMap v1 + +This applies to v1 PerfMap files produced by CrossGen2 , commonly having extensions `.ni.r2rmap`. The key is formed by formatting the signature, the file name, and the version in the following manner: + +Example: + +**File name:** `System.Private.CoreLib.ni.r2rmap` + +**Signature at pseudo-rva 0xFFFFFFFF:** `f5fddf60efb0bee79ef02a19c3decba9` + +**Version at pseudo-rva 0xFFFFFFFE:** `1` + +**Lookup key:** `system.private.corelib.ni.r2rmap/r2rmap-v1-f5fddf60efb0bee79ef02a19c3decba9/system.private.corelib.ni.r2rmap` diff --git a/documentation/symbols/Simple_Symbol_Query_Protocol.md b/documentation/symbols/Simple_Symbol_Query_Protocol.md new file mode 100644 index 000000000..d8577aef0 --- /dev/null +++ b/documentation/symbols/Simple_Symbol_Query_Protocol.md @@ -0,0 +1,45 @@ +# Simple Symbol Query Protocol (SSQP)# + +Frequently when diagnosing computer programs there is a need to retrieve additional information about the program beyond what was required for a computer to run it. This is accomplished by having a network service, the 'symbol server', which provides the additional information on demand. A diagnostic tool such as a debugger or profiler act as symbol server clients, submitting requests for additional information they need. + +The protocol solely consists of a mechanism for the client to provide a key, the 'clientKey', identifying the file it wants. The server then sends back that file. The publisher that created the content and client that downloads it may understand the semantics of the key and the resulting file, but from the perspective of the protocol and the server these items are merely character sequences and binary blobs. + +## Request ## +All requests are [HTTP/1.1](https://tools.ietf.org/html/rfc2616 "HTTP/1.1") messages using the 'get' verb. The client is configured with a valid URI for the service endpoint, and a clientKey it wants to retrieve. The client should request URI /. + + For example: + +**service endpoint URI:** http://www.contuso.com/symbol/server + +**clientKey:** debug\_info.txt/12345abcdefg/debug\_info.txt + + GET http://www.contuso.com/symbol/server/debug\_info.txt/12345abcdefg/debug\_info.txt + +### Encoding and case sensitivity ### +The clientKey needs to be URL encoded as it may contain characters outside the [URI unreserved character set](https://tools.ietf.org/html/rfc3986#Section-2.3). Any '/' character should not be escaped, but all other characters are URL encoded as normal. clientKey is not case-sensitive. Clients are encouraged to normalize the encoding to lower-case in the request URI, but the server should perform any clientKey comparisons in a case-insensitive manner regardless. + +## Response ## + +The response is a standard [HTTP/1.1](https://tools.ietf.org/html/rfc2616 "HTTP/1.1") response including the content of the requested file. Content-Type should be application/octet-stream. + +An http 304 redirection will be honored by the client. + +## Relation to the SymSrv protocol ## + +This protocol has been deliberately designed so that when used with the [recommended key conventions](SSQP_Key_Conventions.md) for windows file types it is also compatible with windows SymSrv clients. + +## Security Warnings ## +### ClientKeys ### +Be careful about assuming that client keys can be interpreted as relative file system paths or other meaningful storage paths. Although all the key conventions may appear safe to use this way an attacker might create keys which have special meaning to the filesystem in an attempt to read or write to sensitive areas on the client or server. If you do use the key this way, carefully validate it first. + +### Trusting downloaded files ### +Clients consuming the diagnostic files downloaded via SSQP often expect that there is a certain relationship between the key and the data that is downloaded. For example a client may expect requesting key 'sha-1-123878123871238712378519847' guarantees that the data which comes back has the given hash. The server is not required to do any verification of this however. A client that wants to be certain must either: + + - verify that invariants hold after the data has been downloaded + - trust the author of the mapping data, every intermediary that handled the data at rest or in transit, and know that none of the trusted parties merge in any clientKey mappings from untrusted parties + +There are internet hosted symbol server implementations that DO merge mappings from untrusted and potentially anonymous 3rd parties. In this case it is trivial for an attacker to pose as such a 3rd party and add mappings that remap legitimate clientKeys to arbitrary attacker chosen files. Clients need to assume that ANY data returned by such a service could have been tampered with, regardless of whether the particular key being requested was originally derived from a trusted source. + +For tools that allow the end-user to specify the SSQP endpoint to connect to, consider the risk that the end-user may change configuration from a service that hosts only trusted content to a service that hosts untrusted content without realizing the security implications of this choice. Some guides on the web from reputable and independent sources explicitly tell developers to make such configuration changes without any mention of the risks. + +Clients are strongly recommended to only work with trusted services, harden against using malicious downloaded files, or implement a scheme to independently verify the integrity and authenticity of downloaded files before using them. \ No newline at end of file diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f42f1b1cf..2ce2e0e38 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,5 @@ - - https://github.com/dotnet/symstore - 550601c12a227c87ded32316345934101a8a2422 - https://github.com/microsoft/clrmd 303bb23040392c845533c9d79911e9535d4ad1e4 diff --git a/eng/Versions.props b/eng/Versions.props index 72815ac05..a06e01e97 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -15,8 +15,6 @@ true - - 1.0.517501 9.0.0-preview.4.24215.19 9.0.0-preview.4.24215.19 diff --git a/eng/release/tool-list.json b/eng/release/tool-list.json index 8d9983f96..5a80692ad 100644 --- a/eng/release/tool-list.json +++ b/eng/release/tool-list.json @@ -29,6 +29,10 @@ { "name": "dotnet-dsrouter", "rids": ["win-x64", "win-x86", "win-arm", "win-arm64", "linux-x64", "linux-musl-arm64", "osx-x64", "linux-arm64", "linux-musl-x64", "linux-arm"] + }, + { + "name": "dotnet-symbol", + "rids": ["win-x64", "win-x86", "win-arm", "win-arm64", "linux-x64", "linux-musl-arm64", "osx-x64", "linux-arm64", "linux-musl-x64", "linux-arm"] } ] }, @@ -40,6 +44,7 @@ "dotnet-sos", "dotnet-trace", "dotnet-stack", + "dotnet-symbol", "dotnet-dsrouter", "Microsoft.Diagnostics.NETCore.Client" ] diff --git a/src/Microsoft.Diagnostics.DebugServices.Implementation/Microsoft.Diagnostics.DebugServices.Implementation.csproj b/src/Microsoft.Diagnostics.DebugServices.Implementation/Microsoft.Diagnostics.DebugServices.Implementation.csproj index 63facc266..5f17fb0a4 100644 --- a/src/Microsoft.Diagnostics.DebugServices.Implementation/Microsoft.Diagnostics.DebugServices.Implementation.csproj +++ b/src/Microsoft.Diagnostics.DebugServices.Implementation/Microsoft.Diagnostics.DebugServices.Implementation.csproj @@ -16,7 +16,6 @@ - @@ -26,5 +25,6 @@ + diff --git a/src/Microsoft.Diagnostics.ExtensionCommands/Microsoft.Diagnostics.ExtensionCommands.csproj b/src/Microsoft.Diagnostics.ExtensionCommands/Microsoft.Diagnostics.ExtensionCommands.csproj index a7c8e842c..b2b3caece 100644 --- a/src/Microsoft.Diagnostics.ExtensionCommands/Microsoft.Diagnostics.ExtensionCommands.csproj +++ b/src/Microsoft.Diagnostics.ExtensionCommands/Microsoft.Diagnostics.ExtensionCommands.csproj @@ -16,10 +16,10 @@ - - + + diff --git a/src/Microsoft.FileFormats/AddressSpaces.cs b/src/Microsoft.FileFormats/AddressSpaces.cs new file mode 100644 index 000000000..62eafff69 --- /dev/null +++ b/src/Microsoft.FileFormats/AddressSpaces.cs @@ -0,0 +1,143 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// An address space that starts at a fixed offset relative to another space + /// + public class RelativeAddressSpace : IAddressSpace + { + private IAddressSpace _baseAddressSpace; + private ulong _baseStart; + private ulong _length; + private long _baseToRelativeShift; + + + public RelativeAddressSpace(IAddressSpace baseAddressSpace, ulong startOffset, ulong length) : + this(baseAddressSpace, startOffset, length, -(long)startOffset) + { } + + public RelativeAddressSpace(IAddressSpace baseAddressSpace, ulong startOffset, ulong length, long baseToRelativeShift) + { + /* + if (startOffset < 0 || startOffset >= baseAddressSpace.Length) + { + throw new BadInputFormatException("Invalid startOffset"); + } + if (length < 0 || startOffset + length > baseAddressSpace.Length) + { + throw new BadInputFormatException("Invalid length"); + } + if((long)startOffset + baseToRelativeShift < 0) + { + throw new BadInputFormatException("Invalid baseToRelativeShift"); + }*/ + _baseAddressSpace = baseAddressSpace; + _baseStart = startOffset; + _length = length; + _baseToRelativeShift = baseToRelativeShift; + } + + /// + /// Reads a range of bytes from the address space + /// + /// The position in the address space to begin reading from + /// The buffer that will receive the bytes that are read + /// The offset in the output buffer to begin writing the bytes + /// The number of bytes to read into the buffer + /// The number of bytes read + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + ulong basePosition = (ulong)((long)position - _baseToRelativeShift); + if (basePosition < _baseStart) + { + return 0; + } + count = (uint)Math.Min(count, _length); + return _baseAddressSpace.Read(basePosition, buffer, bufferOffset, count); + } + + /// + /// The upper bound (non-inclusive) of readable addresses + /// + public ulong Length { get { return unchecked(_baseStart + _length + (ulong)_baseToRelativeShift); } } + } + + public class ZeroAddressSpace : IAddressSpace + { + public ZeroAddressSpace(ulong length) + { + Length = length; + } + + public ulong Length { get; private set; } + + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + if (position >= Length) + { + return 0; + } + count = (uint)Math.Min(Length - position, count); + Array.Clear(buffer, (int)bufferOffset, (int)count); + return count; + } + } + + public struct PiecewiseAddressSpaceRange + { + public ulong Start; + public ulong Length; + public IAddressSpace AddressSpace; + } + + public class PiecewiseAddressSpace : IAddressSpace + { + private PiecewiseAddressSpaceRange[] _ranges; + + public PiecewiseAddressSpace(params PiecewiseAddressSpaceRange[] ranges) + { + _ranges = ranges; + Length = _ranges.Max(r => r.Start + r.Length); + } + + public ulong Length { get; private set; } + + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + uint bytesRead = 0; + while (bytesRead != count) + { + int i = 0; + for (; i < _ranges.Length; i++) + { + ulong upper = _ranges[i].Start + _ranges[i].Length; + if (_ranges[i].Start <= position && position < upper) + { + uint bytesToReadRange = (uint)Math.Min(count - bytesRead, upper - position); + uint bytesReadRange = _ranges[i].AddressSpace.Read(position, buffer, bufferOffset, bytesToReadRange); + if (bytesReadRange == 0) + { + return bytesRead; + } + position += bytesReadRange; + bufferOffset += bytesReadRange; + bytesRead += bytesReadRange; + break; + } + } + if (i == _ranges.Length) + { + return bytesRead; + } + } + return bytesRead; + } + } +} diff --git a/src/Microsoft.FileFormats/ArrayHelper.cs b/src/Microsoft.FileFormats/ArrayHelper.cs new file mode 100644 index 000000000..2fc9af98d --- /dev/null +++ b/src/Microsoft.FileFormats/ArrayHelper.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + public static class ArrayHelper + { + /// + /// Safe array allocator - turns OverFlows and OutOfMemory into BIF's. + /// + public static E[] New(uint count) + { + E[] a; + try + { + a = new E[count]; + } + catch (Exception) + { + throw new BadInputFormatException("Internal overflow attempting to allocate an array of size " + count + "."); + } + return a; + } + } +} diff --git a/src/Microsoft.FileFormats/ArrayLayout.cs b/src/Microsoft.FileFormats/ArrayLayout.cs new file mode 100644 index 000000000..bcedfa24b --- /dev/null +++ b/src/Microsoft.FileFormats/ArrayLayout.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + internal sealed class ArrayLayout : LayoutBase + { + public ArrayLayout(Type arrayType, ILayout elementLayout, uint numElements) : + base(arrayType, numElements * elementLayout.Size, elementLayout.NaturalAlignment) + { + _elementLayout = elementLayout; + _numElements = numElements; + } + + public override object Read(IAddressSpace dataSource, ulong position) + { + ulong src = position; + uint elementSize = _elementLayout.Size; + Array a = Array.CreateInstance(_elementLayout.Type, (int)_numElements); + for (uint i = 0; i < _numElements; i++) + { + a.SetValue(_elementLayout.Read(dataSource, src), (int)i); + src += elementSize; + } + return a; + } + + private uint _numElements; + private ILayout _elementLayout; + } +} diff --git a/src/Microsoft.FileFormats/Attributes.cs b/src/Microsoft.FileFormats/Attributes.cs new file mode 100644 index 000000000..1c363d879 --- /dev/null +++ b/src/Microsoft.FileFormats/Attributes.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.FileFormats +{ + /// + /// Attach to an array-typed targeted field to indicate the number of elements + /// + [AttributeUsage(AttributeTargets.Field)] + public sealed class ArraySizeAttribute : Attribute + { + public ArraySizeAttribute(uint numElements) + { + NumElements = numElements; + } + + public uint NumElements { get; private set; } + } + + /// + /// Attach to a field to indicate that it should be only be included in the type + /// if a particular define has been enabled + /// + [AttributeUsage(AttributeTargets.Field)] + public sealed class IfAttribute : Attribute + { + public IfAttribute(string defineName) + { + DefineName = defineName; + } + + public string DefineName { get; private set; } + } +} diff --git a/src/Microsoft.FileFormats/ELF/ELFCoreFile.cs b/src/Microsoft.FileFormats/ELF/ELFCoreFile.cs new file mode 100644 index 000000000..6afb8d745 --- /dev/null +++ b/src/Microsoft.FileFormats/ELF/ELFCoreFile.cs @@ -0,0 +1,167 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.FileFormats.ELF +{ + public class ELFCoreFile + { + private readonly ELFFile _elf; + private readonly Lazy _fileTable; + private readonly Lazy _images; + + public ELFCoreFile(IAddressSpace dataSource) + { + _elf = new ELFFile(dataSource); + _fileTable = new Lazy(ReadFileTable); + _images = new Lazy(ReadLoadedImages); + } + + public ELFFileTable FileTable { get { return _fileTable.Value; } } + public ELFLoadedImage[] LoadedImages { get { return _images.Value; } } + public IAddressSpace DataSource { get { return _elf.VirtualAddressReader.DataSource; } } + + public bool IsValid() + { + return _elf.IsValid() && _elf.Header.Type == ELFHeaderType.Core; + } + + public bool Is64Bit => _elf.Is64Bit; + + public IEnumerable Segments => _elf.Segments; + + private ELFFileTable ReadFileTable() + { + foreach (ELFProgramSegment seg in _elf.Segments) + { + if (seg.Header.Type == ELFProgramHeaderType.Note) + { + ELFNoteList noteList = new(seg.Contents); + foreach (ELFNote note in noteList.Notes) + { + if (note.Header.Type == ELFNoteType.File) + { + return new ELFFileTable(note.Contents); + } + } + } + } + + throw new BadInputFormatException("No ELF file table found"); + } + + private ELFLoadedImage[] ReadLoadedImages() + { + Dictionary lookup = new(); + + foreach (ELFFileTableEntry fte in FileTable.Files.Where(fte => !fte.Path.StartsWith("/dev/zero") && !fte.Path.StartsWith("/run/shm"))) + { + string path = fte.Path; + if (!lookup.TryGetValue(path, out ELFLoadedImage image)) + { + image = lookup[path] = new ELFLoadedImage(path); + } + image.AddTableEntryPointers(fte); + } + + List result = new(); + foreach (ELFLoadedImage image in lookup.Values) + { + image.Image = new ELFFile(_elf.VirtualAddressReader.DataSource, image.LoadAddress, isDataSourceVirtualAddressSpace: true); + result.Add(image); + } + + return result.ToArray(); + } + } + + public class ELFLoadedImage + { + private ulong _loadAddress; + private ulong _minimumPointer = ulong.MaxValue; + + public ELFLoadedImage(ELFFile image, ELFFileTableEntry entry) + { + Image = image; + Path = entry.Path; + _loadAddress = entry.LoadAddress; + } + + public ELFLoadedImage(string path) + { + Path = path; + } + + public ulong LoadAddress => _loadAddress == 0 ? _minimumPointer : _loadAddress; + public string Path { get; } + public ELFFile Image { get; internal set; } + + internal void AddTableEntryPointers(ELFFileTableEntry entry) + { + // There are cases (like .NET single-file modules) where the first NT_FILE entry isn't the ELF + // or PE header (i.e the base address). The header is the first entry with PageOffset == 0. For + // ELF modules there should only be one PageOffset == 0 entry but with the memory mapped PE + // assemblies, there can be more than one PageOffset == 0 entry and the first one is the base + // address. + if (_loadAddress == 0 && entry.PageOffset == 0) + { + _loadAddress = entry.LoadAddress; + } + // If no load address was found, will use the lowest start address. There has to be at least one + // entry. This fixes the .NET 5.0 MacOS ELF dumps which have modules with no PageOffset == 0 entries. + _minimumPointer = Math.Min(entry.LoadAddress, _minimumPointer); + } + } + + public class ELFFileTableEntry + { + private readonly ELFFileTableEntryPointers _ptrs; + + public ELFFileTableEntry(string path, ELFFileTableEntryPointers ptrs) + { + Path = path; + _ptrs = ptrs; + } + + public ulong PageOffset => _ptrs.PageOffset; + public ulong LoadAddress => _ptrs.Start; + public string Path { get; private set; } + } + + public class ELFFileTable + { + private readonly Reader _noteReader; + private readonly Lazy> _files; + + public ELFFileTable(Reader noteReader) + { + _noteReader = noteReader; + _files = new Lazy>(ReadFiles); + } + + public IEnumerable Files { get { return _files.Value; } } + + private IEnumerable ReadFiles() + { + List files = new(); + ulong readPosition = 0; + ELFFileTableHeader header = _noteReader.Read(ref readPosition); + + //TODO: sanity check the entryCount + ELFFileTableEntryPointers[] ptrs = _noteReader.ReadArray(ref readPosition, (uint)(ulong)header.EntryCount); + for (int i = 0; i < (int)(ulong)header.EntryCount; i++) + { + string path = _noteReader.Read(ref readPosition); + + // This substitution is for unloaded modules for which Linux appends " (deleted)" to the module name. + path = path.Replace(" (deleted)", ""); + + files.Add(new ELFFileTableEntry(path, ptrs[i])); + } + return files; + } + } +} diff --git a/src/Microsoft.FileFormats/ELF/ELFFile.cs b/src/Microsoft.FileFormats/ELF/ELFFile.cs new file mode 100644 index 000000000..94373dda4 --- /dev/null +++ b/src/Microsoft.FileFormats/ELF/ELFFile.cs @@ -0,0 +1,513 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Microsoft.FileFormats.ELF +{ + public class ELFFile : IDisposable + { + private readonly ulong _position; + private readonly bool _isDataSourceVirtualAddressSpace; + private readonly Reader _reader; + private readonly Lazy _ident; + private readonly Lazy _dataSourceReader; + private readonly Lazy _header; + private readonly Lazy> _segments; + private readonly Lazy _sections; + private readonly Lazy _virtualAddressReader; + private readonly Lazy _buildId; + private readonly Lazy _sectionNameTable; + + public ELFFile(IAddressSpace dataSource, ulong position = 0, bool isDataSourceVirtualAddressSpace = false) + { + _position = position; + _reader = new Reader(dataSource); + _isDataSourceVirtualAddressSpace = isDataSourceVirtualAddressSpace; + _ident = new Lazy(() => _reader.Read(_position)); + _dataSourceReader = new Lazy(() => new Reader(dataSource, new LayoutManager().AddELFTypes(IsBigEndian, Is64Bit))); + _header = new Lazy(() => DataSourceReader.Read(_position)); + _segments = new Lazy>(ReadSegments); + _sections = new Lazy(ReadSections); + _virtualAddressReader = new Lazy(CreateVirtualAddressReader); + _buildId = new Lazy(ReadBuildId); + _sectionNameTable = new Lazy(ReadSectionNameTable); + } + + public ELFHeaderIdent Ident { get { return _ident.Value; } } + public ELFHeader Header { get { return _header.Value; } } + private Reader DataSourceReader { get { return _dataSourceReader.Value; } } + public IEnumerable Segments { get { return _segments.Value; } } + public ELFSection[] Sections { get { return _sections.Value; } } + public Reader VirtualAddressReader { get { return _virtualAddressReader.Value; } } + public byte[] BuildID { get { return _buildId.Value; } } + public byte[] SectionNameTable { get { return _sectionNameTable.Value; } } + + public void Dispose() + { + if (_reader.DataSource is IDisposable disposable) + { + disposable.Dispose(); + } + } + + public bool IsValid() + { + if (_reader.Length > (_position + _reader.SizeOf())) + { + try + { + return Ident.IsIdentMagicValid.Check(); + } + catch (Exception ex) when (ex is InvalidVirtualAddressException || ex is BadInputFormatException) + { + } + } + return false; + } + + public bool IsBigEndian + { + get + { + Ident.IsIdentMagicValid.CheckThrowing(); + Ident.IsDataValid.CheckThrowing(); + return Ident.Data == ELFData.BigEndian; + } + } + + public bool Is64Bit + { + get + { + Ident.IsIdentMagicValid.CheckThrowing(); + Ident.IsClassValid.CheckThrowing(); + return (Ident.Class == ELFClass.Class64); + } + } + + public ulong PreferredVMBaseAddress + { + get + { + ulong minAddr = ulong.MaxValue; + + foreach (ELFProgramSegment segment in Segments) + { + if (segment.Header.Type == ELFProgramHeaderType.Load) + { + minAddr = Math.Min(minAddr, segment.Header.VirtualAddress); + } + } + + return minAddr; + } + } + + public ELFSection FindSectionByName(string name) + { + foreach (ELFSection section in Sections) + { + if (string.Equals(section.Name, name)) + { + return section; + } + } + return null; + } + + private IEnumerable ReadSegments() + { + Header.IsProgramHeaderCountReasonable.CheckThrowing(); + IsHeaderProgramHeaderOffsetValid.CheckThrowing(); + IsHeaderProgramHeaderEntrySizeValid.CheckThrowing(); + + // Calculate the loadBias. It is usually just the base address except for some executable modules. + ulong loadBias = _position; + if (loadBias > 0) + { + for (uint i = 0; i < Header.ProgramHeaderCount; i++) + { + ulong programHeaderOffset = _position + Header.ProgramHeaderOffset + i * Header.ProgramHeaderEntrySize; + ELFProgramHeader header = DataSourceReader.Read(programHeaderOffset); + if (header.Type == ELFProgramHeaderType.Load && header.FileOffset == 0) + { + loadBias -= header.VirtualAddress; + } + } + } + + // Build the program segments + List segments = new(); + for (uint i = 0; i < Header.ProgramHeaderCount; i++) + { + ulong programHeaderOffset = _position + Header.ProgramHeaderOffset + i * Header.ProgramHeaderEntrySize; + segments.Add(new ELFProgramSegment(DataSourceReader, loadBias, programHeaderOffset, _isDataSourceVirtualAddressSpace)); + } + return segments; + } + + private ELFSection[] ReadSections() + { + Header.IsSectionHeaderCountReasonable.CheckThrowing(); + IsHeaderSectionHeaderOffsetValid.CheckThrowing(); + IsHeaderSectionHeaderEntrySizeValid.CheckThrowing(); + + List sections = new(); + for (uint i = 0; i < Header.SectionHeaderCount; i++) + { + sections.Add(new ELFSection(this, DataSourceReader, _position, _position + Header.SectionHeaderOffset + i * Header.SectionHeaderEntrySize)); + } + return sections.ToArray(); + } + + private Reader CreateVirtualAddressReader() + { + if (_isDataSourceVirtualAddressSpace) + { + return DataSourceReader; + } + else + { + return DataSourceReader.WithAddressSpace(new ELFVirtualAddressSpace(Segments)); + } + } + + private byte[] ReadBuildId() + { + byte[] buildId = null; + + if (Header.ProgramHeaderOffset > 0 && Header.ProgramHeaderEntrySize > 0 && Header.ProgramHeaderCount > 0) + { + try + { + foreach (ELFProgramSegment segment in Segments) + { + if (segment.Header.Type == ELFProgramHeaderType.Note) + { + buildId = ReadBuildIdNote(segment.Contents); + if (buildId != null) + { + break; + } + } + } + } + catch (Exception ex) when (ex is InvalidVirtualAddressException || ex is BadInputFormatException || ex is OverflowException) + { + } + } + + if (buildId == null) + { + // Use sections to find build id if there isn't any program headers (i.e. some FreeBSD .dbg files) + try + { + foreach (ELFSection section in Sections) + { + if (section.Header.Type == ELFSectionHeaderType.Note) + { + buildId = ReadBuildIdNote(section.Contents); + if (buildId != null) + { + break; + } + } + } + } + catch (Exception ex) when (ex is InvalidVirtualAddressException || ex is BadInputFormatException || ex is OverflowException) + { + } + } + + return buildId; + } + + private static byte[] ReadBuildIdNote(Reader noteReader) + { + if (noteReader != null) + { + ELFNoteList noteList = new(noteReader); + foreach (ELFNote note in noteList.Notes) + { + ELFNoteType type = note.Header.Type; + if (type == ELFNoteType.GnuBuildId && note.Name.Equals("GNU")) + { + return note.Contents.Read(0, (uint)note.Contents.Length); + } + } + } + return null; + } + + private byte[] ReadSectionNameTable() + { + try + { + int nameTableIndex = Header.SectionHeaderStringIndex; + if (Header.SectionHeaderOffset != 0 && Header.SectionHeaderCount > 0 && nameTableIndex != 0) + { + ELFSection nameTableSection = Sections[nameTableIndex]; + if (nameTableSection.Header.FileOffset > 0 && nameTableSection.Header.FileSize > 0) + { + return nameTableSection.Contents.Read(0, (uint)nameTableSection.Contents.Length); + } + } + } + catch (Exception ex) when (ex is InvalidVirtualAddressException || ex is BadInputFormatException || ex is OverflowException) + { + } + return null; + } + + #region Validation Rules + + public ValidationRule IsHeaderProgramHeaderOffsetValid + { + get + { + return new ValidationRule("ELF Header ProgramHeaderOffset is invalid or elf file is incomplete", () => + { + return Header.ProgramHeaderOffset < _reader.Length && + Header.ProgramHeaderOffset + (ulong)(Header.ProgramHeaderEntrySize * Header.ProgramHeaderCount) <= _reader.Length; + }, + IsHeaderProgramHeaderEntrySizeValid, + Header.IsProgramHeaderCountReasonable); + + } + } + + public ValidationRule IsHeaderProgramHeaderEntrySizeValid + { + get { return new ValidationRule("ELF Header ProgramHeaderEntrySize is invalid", () => Header.ProgramHeaderEntrySize == DataSourceReader.SizeOf()); } + } + + public ValidationRule IsHeaderSectionHeaderOffsetValid + { + get + { + return new ValidationRule("ELF Header SectionHeaderOffset is invalid or elf file is incomplete", () => { + return Header.SectionHeaderOffset < _reader.Length && + Header.SectionHeaderOffset + (ulong)(Header.SectionHeaderEntrySize * Header.SectionHeaderCount) <= _reader.Length; + }, + IsHeaderSectionHeaderEntrySizeValid, + Header.IsSectionHeaderCountReasonable); + } + } + + public ValidationRule IsHeaderSectionHeaderEntrySizeValid + { + get { return new ValidationRule("ELF Header SectionHeaderEntrySize is invalid", () => Header.SectionHeaderEntrySize == DataSourceReader.SizeOf()); } + } + + #endregion + } + + public class ELFVirtualAddressSpace : IAddressSpace + { + private readonly ELFProgramSegment[] _segments; + + public ELFVirtualAddressSpace(IEnumerable segments) + { + _segments = segments.Where((programHeader) => programHeader.Header.FileSize > 0).ToArray(); + Length = _segments.Max(s => s.Header.VirtualAddress + s.Header.VirtualSize); + } + + public ulong Length { get; private set; } + + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + uint bytesRead = 0; + while (bytesRead != count) + { + int i = 0; + for (; i < _segments.Length; i++) + { + ELFProgramHeader header = _segments[i].Header; + + ulong upperAddress = header.VirtualAddress + header.VirtualSize; + if (header.VirtualAddress <= position && position < upperAddress) + { + uint bytesToReadRange = (uint)Math.Min(count - bytesRead, upperAddress - position); + ulong segmentOffset = position - header.VirtualAddress; + uint bytesReadRange = _segments[i].Contents.Read(segmentOffset, buffer, bufferOffset, bytesToReadRange); + if (bytesReadRange == 0) { + goto done; + } + position += bytesReadRange; + bufferOffset += bytesReadRange; + bytesRead += bytesReadRange; + break; + } + } + if (i == _segments.Length) { + break; + } + } + done: + if (bytesRead == 0) { + throw new InvalidVirtualAddressException(string.Format("Virtual address range is not mapped {0:X16} {1}", position, count)); + } + // Zero the rest of the buffer if read less than requested + Array.Clear(buffer, (int)bufferOffset, (int)(count - bytesRead)); + return bytesRead; + } + } + + public class ELFProgramSegment + { + private readonly Lazy _contents; + + public ELFProgramSegment(Reader dataSourceReader, ulong elfOffset, ulong programHeaderOffset, bool isDataSourceVirtualAddressSpace) + { + Header = dataSourceReader.Read(programHeaderOffset); + if (isDataSourceVirtualAddressSpace) + { + _contents = new Lazy(() => dataSourceReader.WithRelativeAddressSpace(elfOffset + Header.VirtualAddress, Header.VirtualSize)); + } + else + { + _contents = new Lazy(() => dataSourceReader.WithRelativeAddressSpace(elfOffset + Header.FileOffset, Header.FileSize)); + } + } + + public ELFProgramHeader Header { get; } + public Reader Contents { get { return _contents.Value; } } + + public override string ToString() + { + return "Segment@[" + Header.VirtualAddress.ToString() + "-" + (Header.VirtualAddress + Header.VirtualSize).ToString("x") + ")"; + } + } + + public class ELFSection + { + private readonly ELFFile _elfFile; + private readonly Reader _dataSourceReader; + private readonly Lazy _header; + private readonly Lazy _name; + private readonly Lazy _contents; + + private static readonly ASCIIEncoding _decoder = new(); + + public ELFSection(ELFFile elfFile, Reader dataSourceReader, ulong elfOffset, ulong sectionHeaderOffset) + { + _elfFile = elfFile; + _dataSourceReader = dataSourceReader; + _header = new Lazy(() => _dataSourceReader.Read(sectionHeaderOffset)); + _name = new Lazy(ReadName); + _contents = new Lazy(() => _dataSourceReader.WithRelativeAddressSpace(elfOffset + Header.FileOffset, Header.FileSize)); + } + + public ELFSectionHeader Header { get { return _header.Value; } } + public string Name { get { return _name.Value; } } + public Reader Contents { get { return _contents.Value; } } + + private string ReadName() + { + if (Header.Type == ELFSectionHeaderType.Null) + { + return string.Empty; + } + byte[] sectionNameTable = _elfFile.SectionNameTable; + if (sectionNameTable == null || sectionNameTable.Length == 0) + { + return string.Empty; + } + if (Header.NameIndex > sectionNameTable.Length) + { + return string.Empty; + } + int index = (int)Header.NameIndex; + if (index == 0) + { + return string.Empty; + } + int count = 0; + for (; (index + count) < sectionNameTable.Length; count++) + { + if (sectionNameTable[index + count] == 0) + { + break; + } + } + return _decoder.GetString(sectionNameTable, index, count); + } + } + + public class ELFNoteList + { + private readonly Reader _elfSegmentReader; + private readonly Lazy> _notes; + + public ELFNoteList(Reader elfSegmentReader) + { + _elfSegmentReader = elfSegmentReader; + _notes = new Lazy>(ReadNotes); + } + + public IEnumerable Notes { get { return _notes.Value; } } + + private IEnumerable ReadNotes() + { + List notes = new(); + ulong position = 0; + while (position < _elfSegmentReader.Length) + { + ELFNote note = new(_elfSegmentReader, position); + notes.Add(note); + position += note.Size; + } + return notes; + } + } + + public class ELFNote + { + private readonly Reader _elfSegmentReader; + private readonly ulong _noteHeaderOffset; + private readonly Lazy _header; + private readonly Lazy _name; + private readonly Lazy _contents; + + public ELFNote(Reader elfSegmentReader, ulong offset) + { + _elfSegmentReader = elfSegmentReader; + _noteHeaderOffset = offset; + _header = new Lazy(() => _elfSegmentReader.Read(_noteHeaderOffset)); + _name = new Lazy(ReadName); + _contents = new Lazy(CreateContentsReader); + } + + public ELFNoteHeader Header { get { return _header.Value; } } + //TODO: validate these fields + public uint Size { get { return HeaderSize + Align4(Header.NameSize) + Align4(Header.ContentSize); } } + public string Name { get { return _name.Value; } } + public Reader Contents { get { return _contents.Value; } } + + private uint HeaderSize + { + get { return _elfSegmentReader.LayoutManager.GetLayout().Size; } + } + + private string ReadName() + { + ulong nameOffset = _noteHeaderOffset + HeaderSize; + return _elfSegmentReader.WithRelativeAddressSpace(nameOffset, Align4(Header.NameSize)).Read(0); + } + + private Reader CreateContentsReader() + { + ulong contentsOffset = _noteHeaderOffset + HeaderSize + Align4(Header.NameSize); + return _elfSegmentReader.WithRelativeAddressSpace(contentsOffset, Align4(Header.ContentSize)); + } + + private static uint Align4(uint x) + { + return (x + 3U) & ~3U; + } + } +} diff --git a/src/Microsoft.FileFormats/ELF/ELFNoteHeader.cs b/src/Microsoft.FileFormats/ELF/ELFNoteHeader.cs new file mode 100644 index 000000000..80ad20185 --- /dev/null +++ b/src/Microsoft.FileFormats/ELF/ELFNoteHeader.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.FileFormats.ELF +{ + public enum ELFNoteType + { + PrpsInfo = 3, // NT_PRPSINFO + GnuBuildId = 3, // NT_GNU_BUILD_ID + File = 0x46494c45 // "FILE" in ascii + } + + public class ELFNoteHeader : TStruct + { + public uint NameSize; + public uint ContentSize; + public ELFNoteType Type; + } +} diff --git a/src/Microsoft.FileFormats/ELF/ELFProgramHeader.cs b/src/Microsoft.FileFormats/ELF/ELFProgramHeader.cs new file mode 100644 index 000000000..cf7030ddf --- /dev/null +++ b/src/Microsoft.FileFormats/ELF/ELFProgramHeader.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.FileFormats.ELF +{ + public enum ELFProgramHeaderType : uint + { + Null = 0, + Load = 1, + Dynamic = 2, + Interp = 3, + Note = 4, + Shlib = 5, + Phdr = 6, + GnuEHFrame = 0x6474e550, + } + + [Flags] + public enum ELFProgramHeaderFlags : uint + { + Executable = 1, // PF_X + Writable = 2, // PF_W + Readable = 4, // PF_R + ReadWriteExecute = Executable | Writable | Readable, + OSMask = 0x0FF00000, // PF_MASKOS + ProcessorMask = 0xF0000000, // PF_MASKPROC + } + + public class ELFProgramHeader : TStruct + { + public ELFProgramHeaderType Type; // p_type + [If("64BIT")] + public uint Flags; // p_flags + public FileOffset FileOffset; // p_offset + public VirtualAddress VirtualAddress; // p_vaddr + public SizeT PhysicalAddress; // p_paddr + public SizeT FileSize; // p_filesz + public SizeT VirtualSize; // p_memsz + [If("32BIT")] + public uint Flags32; // p_flags + public SizeT Alignment; // p_align + } +} diff --git a/src/Microsoft.FileFormats/ELF/ELFSectionHeader.cs b/src/Microsoft.FileFormats/ELF/ELFSectionHeader.cs new file mode 100644 index 000000000..6436811ef --- /dev/null +++ b/src/Microsoft.FileFormats/ELF/ELFSectionHeader.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.FileFormats.ELF +{ + public enum ELFSectionHeaderType : uint + { + Null = 0, + ProgBits = 1, + SymTab = 2, + StrTab = 3, + Rela = 4, + Hash = 5, + Dynamic = 6, + Note = 7, + NoBits = 8, + Rel = 9, + ShLib = 10, + DynSym = 11, + InitArray = 14, + FiniArray = 15, + PreInitArray = 16, + Group = 17, + SymTabIndexes = 18, + Num = 19, + GnuAttributes = 0x6ffffff5, + GnuHash = 0x6ffffff6, + GnuLibList = 0x6ffffff7, + CheckSum = 0x6ffffff8, + GnuVerDef = 0x6ffffffd, + GnuVerNeed = 0x6ffffffe, + GnuVerSym = 0x6fffffff, + } + + public class ELFSectionHeader : TStruct + { + public uint NameIndex; // sh_name + public ELFSectionHeaderType Type; // sh_type + public SizeT Flags; // sh_flags + public VirtualAddress VirtualAddress; // sh_addr + public FileOffset FileOffset; // sh_offset + public SizeT FileSize; // sh_size + public uint Link; // sh_link + public uint Info; // sh_info + public SizeT Alignment; // sh_addralign + public SizeT EntrySize; // sh_entsize + } +} diff --git a/src/Microsoft.FileFormats/ELF/ELFStructures.cs b/src/Microsoft.FileFormats/ELF/ELFStructures.cs new file mode 100644 index 000000000..5c5f34c27 --- /dev/null +++ b/src/Microsoft.FileFormats/ELF/ELFStructures.cs @@ -0,0 +1,165 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.FileFormats.ELF +{ + public class FileOffset : Pointer { } + public class FileOffset : FileOffset { } + public class VirtualAddress : Pointer { } + public class VirtualAddress : VirtualAddress { } + + public static class ELFLayoutManagerExtensions + { + public static LayoutManager AddELFTypes(this LayoutManager layouts, bool isBigEndian, bool is64Bit) + { + return layouts + .AddPrimitives(isBigEndian) + .AddEnumTypes() + .AddSizeT(is64Bit ? 8 : 4) + .AddPointerTypes() + .AddNullTerminatedString() + .AddTStructTypes(is64Bit ? new string[] { "64BIT" } : new string[] { "32BIT" }); + } + } + + public enum ELFClass : byte + { + None = 0, + Class32 = 1, + Class64 = 2 + } + + public enum ELFData : byte + { + None = 0, + LittleEndian = 1, + BigEndian = 2 + } + + /// + /// The leading 16 bytes of the ELF file format + /// + /// + /// Although normally this is described as being part of the ELFHeader, its + /// useful to parse this independently. The endianess and bitness + /// described in the identity bytes are needed to calculate the size of and + /// offset of fields in the remainder of the header + /// + public class ELFHeaderIdent : TStruct + { + [ArraySize(16)] + public byte[] Ident; + + public ELFClass Class + { + get + { + return (ELFClass)Ident[4]; + } + } + + public ELFData Data + { + get + { + return (ELFData)Ident[5]; + } + } + + #region Validation Rules + public ValidationRule IsIdentMagicValid + { + get + { + return new ValidationRule("Invalid ELFHeader Ident magic", () => + { + return Ident[0] == 0x7f && + Ident[1] == 0x45 && + Ident[2] == 0x4c && + Ident[3] == 0x46; + }); + } + } + + public ValidationRule IsClassValid + { + get + { + return new ValidationRule("Invalid ELFHeader Ident Class", () => + { + return Class == ELFClass.Class32 || Class == ELFClass.Class64; + }); + } + } + + public ValidationRule IsDataValid + { + get + { + return new ValidationRule("Invalid ELFHeader Ident Data", () => + { + return Data == ELFData.BigEndian || Data == ELFData.LittleEndian; + }); + } + } + #endregion + } + + public enum ELFHeaderType : ushort + { + Relocatable = 1, + Executable = 2, + Shared = 3, + Core = 4 + } + + public class ELFHeader : ELFHeaderIdent + { + public ELFHeaderType Type; + public ushort Machine; + public uint Version; + public VirtualAddress Entry; + public FileOffset ProgramHeaderOffset; + public FileOffset SectionHeaderOffset; + public uint Flags; + public ushort EHSize; + public ushort ProgramHeaderEntrySize; + public ushort ProgramHeaderCount; + public ushort SectionHeaderEntrySize; + public ushort SectionHeaderCount; + public ushort SectionHeaderStringIndex; + + #region Validation Rules + + public ValidationRule IsProgramHeaderCountReasonable + { + get + { + return new ValidationRule("Unreasonably large ELFHeader ProgramHeaderCount", () => ProgramHeaderCount <= 30000); + } + } + + public ValidationRule IsSectionHeaderCountReasonable + { + get + { + return new ValidationRule("Unreasonably large ELFHeader SectionHeaderCount", () => SectionHeaderCount <= 30000); + } + } + + #endregion + } + + public class ELFFileTableHeader : TStruct + { + public SizeT EntryCount; + public SizeT PageSize; + } + + public class ELFFileTableEntryPointers : TStruct + { + public VirtualAddress Start; + public VirtualAddress Stop; + public SizeT PageOffset; + } +} diff --git a/src/Microsoft.FileFormats/EnumLayout.cs b/src/Microsoft.FileFormats/EnumLayout.cs new file mode 100644 index 000000000..ef79e463c --- /dev/null +++ b/src/Microsoft.FileFormats/EnumLayout.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + public class EnumLayout : LayoutBase + { + public EnumLayout(Type enumType, ILayout underlyingIntegralLayout) : + base(enumType, underlyingIntegralLayout.Size, underlyingIntegralLayout.NaturalAlignment) + { + _underlyingIntegralLayout = underlyingIntegralLayout; + } + + public override object Read(IAddressSpace dataSource, ulong position) + { + return _underlyingIntegralLayout.Read(dataSource, position); + } + + private ILayout _underlyingIntegralLayout; + } + + public static partial class LayoutManagerExtensions + { + public static LayoutManager AddEnumTypes(this LayoutManager layoutManager) + { + layoutManager.AddLayoutProvider(GetEnumLayout); + return layoutManager; + } + + private static ILayout GetEnumLayout(Type enumType, LayoutManager layoutManager) + { + if (!enumType.GetTypeInfo().IsEnum) + { + return null; + } + Type elementType = enumType.GetTypeInfo().GetEnumUnderlyingType(); + return new EnumLayout(enumType, layoutManager.GetLayout(elementType)); + } + } +} diff --git a/src/Microsoft.FileFormats/Exceptions.cs b/src/Microsoft.FileFormats/Exceptions.cs new file mode 100644 index 000000000..3f6fb428d --- /dev/null +++ b/src/Microsoft.FileFormats/Exceptions.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// Exception thrown to indicate that bits in the input cannot be parsed for whatever reason. + /// + public abstract class InputParsingException : Exception + { + public InputParsingException(string message) + : base(message) + { + } + } + + /// + /// Exception thrown to indicate unparsable bits found in the input data being parsed. + /// + public class BadInputFormatException : InputParsingException + { + public BadInputFormatException(string message) + : base(message) + { + } + } + + /// + /// Exception thrown to the virtual address/position is invalid + /// + public class InvalidVirtualAddressException : InputParsingException + { + public InvalidVirtualAddressException(string message) + : base(message) + { + } + } + + /// + /// Exception thrown to indicate errors during Layout construction. These errors are usually + /// attributable to bugs in the parsing code, not errors in the input data. + /// + public class LayoutException : Exception + { + public LayoutException(string message) + : base(message) + { + } + } +} diff --git a/src/Microsoft.FileFormats/Helpers.cs b/src/Microsoft.FileFormats/Helpers.cs new file mode 100644 index 000000000..f216483b3 --- /dev/null +++ b/src/Microsoft.FileFormats/Helpers.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.FileFormats +{ + internal static class Helpers + { + // Based on Convert.FromHexString, as we don't yet target TFMs where it's available. + public static bool TryConvertHexStringToBytes(string hexString, out byte[] bytes) + { + bytes = null; + if (hexString is null) + { + return false; + } + if (hexString.Length % 2 != 0) + { + return false; + } + if (hexString.Length == 0) + { + bytes = Array.Empty(); + return true; + } + bytes = new byte[hexString.Length >> 1]; + for (int i = 0; i < hexString.Length; i += 2) + { + byte byteHigh = byte.MaxValue >= (int)hexString[i] ? CharToHexLookup[hexString[i]] : (byte)0xFF; + byte byteLow = byte.MaxValue >= (int)hexString[i+1] ? CharToHexLookup[hexString[i+1]] : (byte)0xFF; + + if ((byteLow | byteHigh) == 0xFF) + { + return false; + } + bytes[i >> 1] = (byte)(byteHigh << 4 | byteLow); + } + return true; + } + + // From System.HexConverter, FF means not valid + private static byte[] CharToHexLookup => new byte[] + { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 15 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 31 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 47 + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 63 + 0xFF, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 79 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 95 + 0xFF, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 111 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 127 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 143 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 159 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 175 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 191 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 207 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 223 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 239 + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 255 + }; + } +} diff --git a/src/Microsoft.FileFormats/IAddressSpace.cs b/src/Microsoft.FileFormats/IAddressSpace.cs new file mode 100644 index 000000000..96bc329b2 --- /dev/null +++ b/src/Microsoft.FileFormats/IAddressSpace.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// Abstracts a flat randomly-accessible array of bytes + /// + public interface IAddressSpace + { + /// + /// Reads a range of bytes from the address space + /// + /// The position in the address space to begin reading from + /// The buffer that will receive the bytes that are read + /// The offset in the output buffer to begin writing the bytes + /// The number of bytes to read into the buffer + /// The number of bytes read + uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count); + + /// + /// The upper bound (non-inclusive) of readable addresses + /// + /// + /// Some address spaces may be sparse, there is no guarantee reads will succeed even + /// at addresses less than the Length + /// + ulong Length { get; } + } + + public static class AddressSpaceExtensions + { + /// + /// Read the specified number of bytes. + /// + /// The address space to read from + /// The position in the address space to start reading from + /// The number of bytes to read + /// + /// Returns an array of exactly "count" bytes or throw an exception. + /// + /// + /// BadInputFormatException to indicate an "unexpected end of stream" condition + /// + public static byte[] Read(this IAddressSpace addressSpace, ulong position, uint count) + { + byte[] bytes = ArrayHelper.New(count); + if (count != addressSpace.Read(position, bytes, 0, count)) + { + throw new BadInputFormatException("Unable to read bytes at offset 0x" + position.ToString("x")); + } + return bytes; + } + } +} diff --git a/src/Microsoft.FileFormats/IField.cs b/src/Microsoft.FileFormats/IField.cs new file mode 100644 index 000000000..5c258d731 --- /dev/null +++ b/src/Microsoft.FileFormats/IField.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + public interface IField + { + string Name { get; } + ILayout Layout { get; } + ILayout DeclaringLayout { get; } + uint Offset { get; } + + object GetValue(TStruct tStruct); + void SetValue(TStruct tStruct, object fieldValue); + } +} diff --git a/src/Microsoft.FileFormats/ILayout.cs b/src/Microsoft.FileFormats/ILayout.cs new file mode 100644 index 000000000..99f3e7827 --- /dev/null +++ b/src/Microsoft.FileFormats/ILayout.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// Information about a type that allows it to be parsed from a byte sequence either on its own + /// or as a sub-component of another type + /// + public interface ILayout + { + /// + /// The type being layed out + /// + Type Type { get; } + + /// + /// Size in bytes for the serialized representation of the type. + /// + /// InvalidOperationException if IsFixedSize == false + uint Size { get; } + + /// + /// Returns true if all instances of the type serialize to the same number of bytes + /// + bool IsFixedSize { get; } + + /// + /// The preferred alignment of this type + /// + uint NaturalAlignment { get; } + + /// + /// The set of fields that compose the type + /// + IEnumerable Fields { get; } + + /// + /// Size in bytes for the serialized representation of all the fields in this type. + /// This may be less than Size because it does not account for trailing padding bytes + /// after the last field. + /// + /// InvalidOperationException if IsFixedSize == false + uint SizeAsBaseType { get; } + + /// + /// Parse an instance from the dataSource starting at position + /// + object Read(IAddressSpace dataSource, ulong position); + + /// + /// Parse an instance from the dataSource starting at position and report the number of bytes + /// that were read + /// + object Read(IAddressSpace dataSource, ulong position, out uint bytesRead); + } +} diff --git a/src/Microsoft.FileFormats/LayoutBase.cs b/src/Microsoft.FileFormats/LayoutBase.cs new file mode 100644 index 000000000..8acb0088d --- /dev/null +++ b/src/Microsoft.FileFormats/LayoutBase.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// A common base class to assist in implementing ILayout + /// + public abstract class LayoutBase : ILayout + { + public LayoutBase(Type type) : this(type, 0) { } + public LayoutBase(Type type, uint size) : this(type, size, size) { } + public LayoutBase(Type type, uint size, uint naturalAlignment) : this(type, size, naturalAlignment, size) { } + public LayoutBase(Type type, uint size, uint naturalAlignment, uint sizeAsBaseType) : this(type, size, naturalAlignment, sizeAsBaseType, Array.Empty()) { } + public LayoutBase(Type type, uint size, uint naturalAlignment, uint sizeAsBaseType, IField[] fields) + { + Type = type; + IsFixedSize = true; + Size = size; + SizeAsBaseType = sizeAsBaseType; + NaturalAlignment = naturalAlignment; + Fields = fields; + } + + public IEnumerable Fields { get; private set; } + + public uint NaturalAlignment { get; private set; } + + public bool IsFixedSize { get; private set; } + + public uint Size { get; private set; } + + public uint SizeAsBaseType { get; private set; } + + public Type Type { get; private set; } + + public virtual object Read(IAddressSpace dataSource, ulong position, out uint bytesRead) + { + bytesRead = Size; + return Read(dataSource, position); + } + + public virtual object Read(IAddressSpace dataSource, ulong position) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/Microsoft.FileFormats/LayoutManager.cs b/src/Microsoft.FileFormats/LayoutManager.cs new file mode 100644 index 000000000..54940bf07 --- /dev/null +++ b/src/Microsoft.FileFormats/LayoutManager.cs @@ -0,0 +1,88 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// A container that can provide ILayout instances for Types. + /// + public class LayoutManager + { + private Dictionary _layouts = new(); + private List> _layoutProviders = new(); + private Dictionary, ILayout> _arrayLayouts = new(); + + public LayoutManager() { } + + public void AddLayout(ILayout layout) + { + _layouts.Add(layout.Type, layout); + } + + public void AddLayoutProvider(Func layoutProvider) + { + _layoutProviders.Add(layoutProvider); + } + + public ILayout GetArrayLayout(Type arrayType, uint numElements) + { + if (!arrayType.IsArray) + { + throw new ArgumentException("The type parameter must be an array"); + } + if (arrayType.GetArrayRank() != 1) + { + throw new ArgumentException("Multidimensional arrays are not supported"); + } + + ILayout layout; + Tuple key = new(arrayType, numElements); + if (!_arrayLayouts.TryGetValue(key, out layout)) + { + Type elemType = arrayType.GetElementType(); + layout = new ArrayLayout(arrayType, GetLayout(elemType), numElements); + _arrayLayouts.Add(key, layout); + } + return layout; + } + + public ILayout GetArrayLayout(uint numElements) + { + return GetArrayLayout(typeof(T), numElements); + } + + public ILayout GetLayout() + { + return GetLayout(typeof(T)); + } + + public ILayout GetLayout(Type t) + { + ILayout layout; + if (!_layouts.TryGetValue(t, out layout)) + { + foreach (Func provider in _layoutProviders) + { + layout = provider(t, this); + if (layout != null) + { + break; + } + } + if (layout == null) + { + throw new LayoutException("Unable to create layout for type " + t.FullName); + } + _layouts.Add(t, layout); + } + return layout; + } + } +} diff --git a/src/Microsoft.FileFormats/MachO/MachCore.cs b/src/Microsoft.FileFormats/MachO/MachCore.cs new file mode 100644 index 000000000..a17e2e456 --- /dev/null +++ b/src/Microsoft.FileFormats/MachO/MachCore.cs @@ -0,0 +1,172 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats.MachO +{ + public class MachCore + { + private readonly MachOFile _machO; + private readonly ulong _dylinkerHintAddress; + private readonly Lazy _dylinkerAddress; + private readonly Lazy _dylinker; + private readonly Lazy _loadedImages; + + public MachCore(IAddressSpace dataSource, ulong dylinkerHintAddress = 0) + { + _machO = new MachOFile(dataSource); + _dylinkerHintAddress = dylinkerHintAddress; + _dylinkerAddress = new Lazy(FindDylinker); + _dylinker = new Lazy(() => new MachDyld(new MachOFile(VirtualAddressReader.DataSource, DylinkerAddress, true))); + _loadedImages = new Lazy(ReadImages); + } + + public Reader VirtualAddressReader { get { return _machO.VirtualAddressReader; } } + public ulong DylinkerAddress { get { return _dylinkerAddress.Value; } } + public MachDyld Dylinker { get { return _dylinker.Value; } } + public IEnumerable LoadedImages { get { return _loadedImages.Value; } } + + public bool IsValid() + { + return _machO.IsValid() && _machO.Header.FileType == MachHeaderFileType.Core; + } + + private ulong FindDylinker() + { + if (_dylinkerHintAddress != 0 && IsValidDylinkerAddress(_dylinkerHintAddress)) + { + return _dylinkerHintAddress; + } + if (TryFindDylinker(firstPass: true, out ulong position)) + { + return position; + } + if (TryFindDylinker(firstPass: false, out position)) + { + return position; + } + throw new BadInputFormatException("No dylinker module found"); + } + + private bool TryFindDylinker(bool firstPass, out ulong position) + { + const uint skip = 0x1000; + const uint firstPassAttemptCount = 8; + foreach (MachSegment segment in _machO.Segments) + { + ulong start = 0; + ulong end = segment.LoadCommand.FileSize; + if (firstPass) + { + end = skip * firstPassAttemptCount; + } + else + { + start = skip * firstPassAttemptCount; + } + for (ulong offset = start; offset < end; offset += skip) + { + ulong possibleDylinker = segment.LoadCommand.VMAddress + offset; + if (IsValidDylinkerAddress(possibleDylinker)) + { + position = possibleDylinker; + return true; + } + } + } + position = 0; + return false; + } + + private bool IsValidDylinkerAddress(ulong possibleDylinkerAddress) + { + MachOFile dylinker = new(VirtualAddressReader.DataSource, possibleDylinkerAddress, true); + return dylinker.IsValid() && dylinker.Header.FileType == MachHeaderFileType.Dylinker; + } + + private MachLoadedImage[] ReadImages() + { + return Dylinker.Images.Select(i => new MachLoadedImage(new MachOFile(VirtualAddressReader.DataSource, i.LoadAddress, true), i)).ToArray(); + } + } + + public class MachLoadedImage + { + private readonly DyldLoadedImage _dyldLoadedImage; + + public MachLoadedImage(MachOFile image, DyldLoadedImage dyldLoadedImage) + { + Image = image; + _dyldLoadedImage = dyldLoadedImage; + } + + public MachOFile Image { get; private set; } + public ulong LoadAddress { get { return _dyldLoadedImage.LoadAddress; } } + public string Path { get { return _dyldLoadedImage.Path; } } + } + + public class MachDyld + { + private readonly MachOFile _dyldImage; + private readonly Lazy _dyldAllImageInfosAddress; + private readonly Lazy _dyldAllImageInfos; + private readonly Lazy _imageInfos; + private readonly Lazy _images; + + public MachDyld(MachOFile dyldImage) + { + _dyldImage = dyldImage; + _dyldAllImageInfosAddress = new Lazy(FindAllImageInfosAddress); + _dyldAllImageInfos = new Lazy(ReadAllImageInfos); + _imageInfos = new Lazy(ReadImageInfos); + _images = new Lazy(ReadLoadedImages); + } + + public ulong AllImageInfosAddress { get { return _dyldAllImageInfosAddress.Value; } } + public DyldImageAllInfosV2 AllImageInfos { get { return _dyldAllImageInfos.Value; } } + public IEnumerable ImageInfos { get { return _imageInfos.Value; } } + public IEnumerable Images { get { return _images.Value; } } + + private ulong FindAllImageInfosAddress() + { + if (!_dyldImage.Symtab.TryLookupSymbol("dyld_all_image_infos", out ulong offset)) + { + throw new BadInputFormatException("Can not find dyld_all_image_infos"); + } + return offset + _dyldImage.PreferredVMBaseAddress; + } + + private DyldImageAllInfosV2 ReadAllImageInfos() + { + return _dyldImage.VirtualAddressReader.Read(AllImageInfosAddress); + } + + private DyldImageInfo[] ReadImageInfos() + { + return _dyldImage.VirtualAddressReader.ReadArray(AllImageInfos.InfoArray, AllImageInfos.InfoArrayCount); + } + + private DyldLoadedImage[] ReadLoadedImages() + { + return ImageInfos.Select(i => new DyldLoadedImage(_dyldImage.VirtualAddressReader.Read(i.PathAddress), i)).ToArray(); + } + } + + public class DyldLoadedImage + { + private readonly DyldImageInfo _imageInfo; + + public DyldLoadedImage(string path, DyldImageInfo imageInfo) + { + Path = path; + _imageInfo = imageInfo; + } + + public string Path; + public ulong LoadAddress { get { return _imageInfo.Address; } } + } +} diff --git a/src/Microsoft.FileFormats/MachO/MachOFatHeaderStructures.cs b/src/Microsoft.FileFormats/MachO/MachOFatHeaderStructures.cs new file mode 100644 index 000000000..02c253e13 --- /dev/null +++ b/src/Microsoft.FileFormats/MachO/MachOFatHeaderStructures.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.FileFormats.MachO +{ + public static class MachFatHeaderLayoutManagerExtensions + { + public static LayoutManager AddMachFatHeaderTypes(this LayoutManager layoutManager, bool isBigEndian) + { + layoutManager + .AddPrimitives(isBigEndian) + .AddEnumTypes() + .AddTStructTypes(); + return layoutManager; + } + } + + public enum MachFatHeaderMagicKind : uint + { + LittleEndian = 0xcafebabe, + BigEndian = 0xbebafeca + } + + public class MachFatHeaderMagic : TStruct + { + public MachFatHeaderMagicKind Magic; + + #region Validation Rules + public ValidationRule IsMagicValid + { + get + { + return new ValidationRule("Invalid MachO Fat Header Magic", () => + { + return Magic == MachFatHeaderMagicKind.BigEndian || + Magic == MachFatHeaderMagicKind.LittleEndian; + }); + } + } + #endregion + } + + public class MachFatHeader : MachFatHeaderMagic + { + public uint CountFatArches; + + #region Validation Rules + public ValidationRule IsCountFatArchesReasonable + { + get + { + return new ValidationRule("Unreasonable MachO Fat Header CountFatArches", + () => CountFatArches <= 20); + } + } + #endregion + } + + public class MachFatArch : TStruct + { + public uint CpuType; + public uint CpuSubType; + public uint Offset; + public uint Size; + public uint Align; + } +} diff --git a/src/Microsoft.FileFormats/MachO/MachOFile.cs b/src/Microsoft.FileFormats/MachO/MachOFile.cs new file mode 100644 index 000000000..1cdf607c6 --- /dev/null +++ b/src/Microsoft.FileFormats/MachO/MachOFile.cs @@ -0,0 +1,554 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.FileFormats.MachO +{ + public class MachOFatFile : IDisposable + { + private readonly Reader _reader; + private readonly Lazy _headerMagic; + private readonly Lazy _headerReader; + private readonly Lazy _header; + private readonly Lazy _arches; + private readonly Lazy _archSpecificFiles; + + public MachOFatFile(IAddressSpace dataSource) + { + _reader = new Reader(dataSource); + _headerMagic = new Lazy(() => _reader.Read(0)); + _headerReader = new Lazy(() => new Reader(dataSource, new LayoutManager().AddMachFatHeaderTypes(IsBigEndian))); + _header = new Lazy(() => _headerReader.Value.Read(0)); + _arches = new Lazy(ReadArches); + _archSpecificFiles = new Lazy(ReadArchSpecificFiles); + } + + public MachFatHeaderMagic HeaderMagic { get { return _headerMagic.Value; } } + public MachFatHeader Header { get { return _header.Value; } } + public MachFatArch[] Arches { get { return _arches.Value; } } + public MachOFile[] ArchSpecificFiles { get { return _archSpecificFiles.Value; } } + + public void Dispose() + { + if (_reader.DataSource is IDisposable disposable) + { + disposable.Dispose(); + } + } + + public bool IsValid() + { + if (_reader.Length > (_reader.SizeOf())) + { + try + { + return HeaderMagic.IsMagicValid.Check(); + } + catch (Exception ex) when (ex is InvalidVirtualAddressException || ex is BadInputFormatException) + { + } + } + return false; + } + + public bool IsBigEndian + { + get + { + HeaderMagic.IsMagicValid.CheckThrowing(); + return HeaderMagic.Magic == MachFatHeaderMagicKind.BigEndian; + } + } + + private MachFatArch[] ReadArches() + { + Header.IsCountFatArchesReasonable.CheckThrowing(); + ulong position = _headerReader.Value.SizeOf(); + return _headerReader.Value.ReadArray(position, Header.CountFatArches); + } + + private MachOFile[] ReadArchSpecificFiles() + { + return Arches.Select(a => new MachOFile(new RelativeAddressSpace(_reader.DataSource, a.Offset, a.Size))).ToArray(); + } + } + + public class MachOFile : IDisposable + { + private readonly ulong _position; + private readonly bool _dataSourceIsVirtualAddressSpace; + private readonly Reader _reader; + private readonly Lazy _headerMagic; + private readonly Lazy _dataSourceReader; + private readonly Lazy _header; + private readonly Lazy[]> _loadCommands; + private readonly Lazy _segments; + private readonly Lazy _sections; + private readonly Lazy _virtualAddressReader; + private readonly Lazy _physicalAddressReader; + private readonly Lazy _uuid; + private readonly Lazy _symtab; + + public MachOFile(IAddressSpace dataSource, ulong position = 0, bool dataSourceIsVirtualAddressSpace = false) + { + _position = position; + _dataSourceIsVirtualAddressSpace = dataSourceIsVirtualAddressSpace; + _reader = new Reader(dataSource); + _headerMagic = new Lazy(() => _reader.Read(_position)); + _dataSourceReader = new Lazy(CreateDataSourceReader); + _header = new Lazy(() => DataSourceReader.Read(_position)); + _loadCommands = new Lazy[]>(ReadLoadCommands); + _segments = new Lazy(ReadSegments); + _sections = new Lazy(() => Segments.SelectMany(seg => seg.Sections).ToArray()); + _virtualAddressReader = new Lazy(CreateVirtualReader); + _physicalAddressReader = new Lazy(CreatePhysicalReader); + _uuid = new Lazy(ReadUuid); + _symtab = new Lazy(ReadSymtab); + } + + public MachHeaderMagic HeaderMagic { get { return _headerMagic.Value; } } + public MachHeader Header { get { return _header.Value; } } + public byte[] Uuid { get { return _uuid.Value; } } + public MachSegment[] Segments { get { return _segments.Value; } } + public MachSection[] Sections { get { return _sections.Value; } } + public Reader VirtualAddressReader { get { return _virtualAddressReader.Value; } } + public Reader PhysicalAddressReader { get { return _physicalAddressReader.Value; } } + public MachSymtab Symtab { get { return _symtab.Value; } } + private Reader DataSourceReader { get { return _dataSourceReader.Value; } } + + public void Dispose() + { + if (_reader.DataSource is IDisposable disposable) + { + disposable.Dispose(); + } + } + public bool IsValid() + { + if (_reader.Length > (_position + _reader.SizeOf())) + { + try + { + return HeaderMagic.IsMagicValid.Check(); + } + catch (Exception ex) when (ex is InvalidVirtualAddressException || ex is BadInputFormatException) + { + } + } + return false; + } + + public bool IsBigEndian + { + get + { + HeaderMagic.IsMagicValid.CheckThrowing(); + return (HeaderMagic.Magic == MachHeaderMagicType.BigEndian32Bit || + HeaderMagic.Magic == MachHeaderMagicType.BigEndian64Bit); + } + } + + public bool Is64Bit + { + get + { + HeaderMagic.IsMagicValid.CheckThrowing(); + return (HeaderMagic.Magic == MachHeaderMagicType.LittleEndian64Bit || + HeaderMagic.Magic == MachHeaderMagicType.BigEndian64Bit); + } + } + + public ulong PreferredVMBaseAddress + { + get + { + MachSegment first = Segments.Where(s => s.LoadCommand.SegName.ToString() == "__TEXT").FirstOrDefault(); + return first != null ? _position - first.LoadCommand.VMAddress : 0; + } + } + + public ulong LoadAddress + { + get + { + if (_dataSourceIsVirtualAddressSpace) + { + return _position; + } + else + { + return PreferredVMBaseAddress; + } + } + } + + private Reader CreateDataSourceReader() + { + return new Reader(_reader.DataSource, new LayoutManager().AddMachTypes(IsBigEndian, Is64Bit)); + } + + private Reader CreateVirtualReader() + { + if (_dataSourceIsVirtualAddressSpace) + { + return DataSourceReader; + } + else + { + return DataSourceReader.WithAddressSpace(new MachVirtualAddressSpace(Segments)); + } + } + + private Reader CreatePhysicalReader() + { + if (!_dataSourceIsVirtualAddressSpace) + { + return DataSourceReader; + } + else + { + return DataSourceReader.WithAddressSpace(new MachPhysicalAddressSpace(_reader.DataSource, PreferredVMBaseAddress, Segments)); + } + } + + private Tuple[] ReadLoadCommands() + { + Header.IsNumberCommandsReasonable.CheckThrowing(); + ulong position = _position + DataSourceReader.SizeOf(); + //TODO: do this more cleanly + if (Is64Bit) + { + position += 4; // the 64 bit version has an extra padding field to align at an + // 8 byte boundary + } + List> cmds = new(); + for (uint i = 0; i < Header.NumberCommands; i++) + { + MachLoadCommand cmd = DataSourceReader.Read(position); + cmd.IsCmdSizeReasonable.CheckThrowing(); + cmds.Add(new Tuple(cmd, position)); + position += cmd.CommandSize; + } + + return cmds.ToArray(); + } + + private byte[] ReadUuid() + { + IsAtLeastOneUuidLoadCommand.CheckThrowing(); + IsAtMostOneUuidLoadCommand.CheckThrowing(); + Tuple cmdAndPos = _loadCommands.Value.Where(c => c.Item1.Command == LoadCommandType.Uuid).First(); + MachUuidLoadCommand uuidCmd = DataSourceReader.Read(cmdAndPos.Item2); + uuidCmd.IsCommandSizeValid.CheckThrowing(); + return uuidCmd.Uuid; + } + + private MachSegment[] ReadSegments() + { + List segs = new(); + foreach (Tuple cmdAndPos in _loadCommands.Value) + { + LoadCommandType segType = Is64Bit ? LoadCommandType.Segment64 : LoadCommandType.Segment; + if (cmdAndPos.Item1.Command != segType) + { + continue; + } + MachSegment seg = new(DataSourceReader, cmdAndPos.Item2, _dataSourceIsVirtualAddressSpace); + segs.Add(seg); + } + + return segs.ToArray(); + } + + private MachSymtab ReadSymtab() + { + IsAtLeastOneSymtabLoadCommand.CheckThrowing(); + IsAtMostOneSymtabLoadCommand.CheckThrowing(); + ulong symtabPosition = 0; + ulong dysymtabPosition = 0; + foreach (Tuple cmdAndPos in _loadCommands.Value) + { + switch (cmdAndPos.Item1.Command) + { + case LoadCommandType.Symtab: + symtabPosition = cmdAndPos.Item2; + break; + case LoadCommandType.DySymtab: + dysymtabPosition = cmdAndPos.Item2; + break; + } + } + if (symtabPosition == 0 || dysymtabPosition == 0) + { + return null; + } + return new MachSymtab(DataSourceReader, symtabPosition, dysymtabPosition, PhysicalAddressReader); + } + + #region Validation Rules + public ValidationRule IsAtMostOneUuidLoadCommand + { + get + { + return new ValidationRule("Mach load command sequence has too many uuid elements", + () => _loadCommands.Value.Count(c => c.Item1.Command == LoadCommandType.Uuid) <= 1); + } + } + public ValidationRule IsAtLeastOneUuidLoadCommand + { + get + { + return new ValidationRule("Mach load command sequence has no uuid elements", + () => _loadCommands.Value.Any(c => c.Item1.Command == LoadCommandType.Uuid)); + } + } + public ValidationRule IsAtMostOneSymtabLoadCommand + { + get + { + return new ValidationRule("Mach load command sequence has too many symtab elements", + () => _loadCommands.Value.Count(c => c.Item1.Command == LoadCommandType.Symtab) <= 1); + } + } + public ValidationRule IsAtLeastOneSymtabLoadCommand + { + get + { + return new ValidationRule("Mach load command sequence has no symtab elements", + () => _loadCommands.Value.Any(c => c.Item1.Command == LoadCommandType.Symtab)); + } + } + public ValidationRule IsAtLeastOneSegmentAtFileOffsetZero + { + get + { + return new ValidationRule("Mach load command sequence has no segments which contain file offset zero", + () => Segments.Where(s => s.LoadCommand.FileOffset == 0 && + s.LoadCommand.FileSize != 0).Any()); + } + } + #endregion + } + + public class MachSegment + { + private readonly Reader _dataSourceReader; + private readonly ulong _position; + private readonly bool _readerIsVirtualAddressSpace; + private readonly Lazy _loadCommand; + private readonly Lazy _sections; + private readonly Lazy _physicalContents; + private readonly Lazy _virtualContents; + + public MachSegment(Reader machReader, ulong position, bool readerIsVirtualAddressSpace = false) + { + _dataSourceReader = machReader; + _position = position; + _readerIsVirtualAddressSpace = readerIsVirtualAddressSpace; + _loadCommand = new Lazy(() => _dataSourceReader.Read(_position)); + _sections = new Lazy(ReadSections); + _physicalContents = new Lazy(CreatePhysicalSegmentAddressSpace); + _virtualContents = new Lazy(CreateVirtualSegmentAddressSpace); + } + + public MachSegmentLoadCommand LoadCommand { get { return _loadCommand.Value; } } + public IEnumerable Sections { get { return _sections.Value; } } + public Reader PhysicalContents { get { return _physicalContents.Value; } } + public Reader VirtualContents { get { return _virtualContents.Value; } } + + private MachSection[] ReadSections() + { + ulong sectionStartOffset = _position + _dataSourceReader.SizeOf(); + return _dataSourceReader.ReadArray(sectionStartOffset, _loadCommand.Value.CountSections); + } + + private Reader CreatePhysicalSegmentAddressSpace() + { + if (!_readerIsVirtualAddressSpace) + { + return _dataSourceReader.WithRelativeAddressSpace(LoadCommand.FileOffset, LoadCommand.FileSize, 0); + } + else + { + return _dataSourceReader.WithRelativeAddressSpace(LoadCommand.VMAddress, LoadCommand.FileSize, + (long)(LoadCommand.FileOffset - LoadCommand.VMAddress)); + } + } + + private Reader CreateVirtualSegmentAddressSpace() + { + if (_readerIsVirtualAddressSpace) + { + return _dataSourceReader.WithRelativeAddressSpace(LoadCommand.VMAddress, LoadCommand.VMSize, 0); + } + else + { + return _dataSourceReader.WithAddressSpace( + new PiecewiseAddressSpace( + new PiecewiseAddressSpaceRange() + { + AddressSpace = new RelativeAddressSpace(_dataSourceReader.DataSource, LoadCommand.FileOffset, LoadCommand.FileSize, + (long)(LoadCommand.VMAddress - LoadCommand.FileOffset)), + Start = LoadCommand.VMAddress, + Length = LoadCommand.FileSize + }, + new PiecewiseAddressSpaceRange() + { + AddressSpace = new ZeroAddressSpace(LoadCommand.VMAddress + LoadCommand.VMSize), + Start = LoadCommand.VMAddress + LoadCommand.FileSize, + Length = LoadCommand.VMSize - LoadCommand.FileSize + })); + } + } + } + + + public class MachVirtualAddressSpace : PiecewiseAddressSpace + { + public MachVirtualAddressSpace(IEnumerable segments) : base(segments.Select(s => ToRange(s)).ToArray()) + { + } + + private static PiecewiseAddressSpaceRange ToRange(MachSegment segment) + { + return new PiecewiseAddressSpaceRange() + { + AddressSpace = segment.VirtualContents.DataSource, + Start = segment.LoadCommand.VMAddress, + Length = segment.LoadCommand.VMSize + }; + } + } + + public class MachPhysicalAddressSpace : PiecewiseAddressSpace + { + public MachPhysicalAddressSpace(IAddressSpace virtualAddressSpace, ulong preferredVMBaseAddress, IEnumerable segments) : + base(segments.Select(s => ToRange(virtualAddressSpace, preferredVMBaseAddress, s)).ToArray()) + { + } + + private static PiecewiseAddressSpaceRange ToRange(IAddressSpace virtualAddressSpace, ulong preferredVMBaseAddress, MachSegment segment) + { + ulong actualSegmentLoadAddress = preferredVMBaseAddress + segment.LoadCommand.VMAddress - segment.LoadCommand.FileOffset; + return new PiecewiseAddressSpaceRange() + { + AddressSpace = new RelativeAddressSpace(virtualAddressSpace, actualSegmentLoadAddress, segment.LoadCommand.FileSize), + Start = segment.LoadCommand.FileOffset, + Length = segment.LoadCommand.FileSize + }; + } + } + + public class MachSymbol + { + public string Name; + public ulong Value { get { return Raw.Value; } } + public NList Raw; + + public override string ToString() + { + return Name + "@0x" + Value.ToString("x"); + } + } + + public class MachSymtab + { + private readonly Reader _machReader; + private readonly Reader _physicalAddressSpace; + private readonly Lazy _symtabLoadCommand; + private readonly Lazy _dysymtabLoadCommand; + private readonly Lazy _symbols; + private readonly Lazy _stringReader; + private readonly Lazy _symbolTable; + + public MachSymtab(Reader machReader, ulong symtabPosition, ulong dysymtabPosition, Reader physicalAddressSpace) + { + _machReader = machReader; + _physicalAddressSpace = physicalAddressSpace; + _symtabLoadCommand = new Lazy(() => _machReader.Read(symtabPosition)); + _dysymtabLoadCommand = new Lazy(() => _machReader.Read(dysymtabPosition)); + _stringReader = new Lazy(GetStringReader); + _symbolTable = new Lazy(ReadSymbolTable); + _symbols = new Lazy(ReadSymbols); + } + + public IEnumerable Symbols { get { return _symbols.Value; } } + + public bool TryLookupSymbol(string symbol, out ulong offset) + { + if (symbol is null) + { + throw new ArgumentNullException(nameof(symbol)); + } + MachSymtabLoadCommand symtabLoadCommand = _symtabLoadCommand.Value; + MachDySymtabLoadCommand dysymtabLoadCommand = _dysymtabLoadCommand.Value; + + // First, search just the "external" export symbols + if (TryLookupSymbol(dysymtabLoadCommand.IExtDefSym, dysymtabLoadCommand.NextDefSym, symbol, out offset)) + { + return true; + } + + // If not found in external symbols, search all of them + if (TryLookupSymbol(0, symtabLoadCommand.SymCount, symbol, out offset)) + { + return true; + } + + offset = 0; + return false; + } + + private bool TryLookupSymbol(uint start, uint nsyms, string symbol, out ulong offset) + { + NList[] symTable = _symbolTable.Value; + if (symTable is not null) + { + for (uint i = 0; i < nsyms && start + i < symTable.Length; i++) + { + string name = _stringReader.Value.Read(symTable[start + i].StringIndex); + if (name.Length > 0) + { + // Skip the leading underscores to match Linux externs + if (name[0] == '_') + { + name = name.Substring(1); + } + if (name == symbol) + { + offset = symTable[start + i].Value; + return true; + } + } + } + } + offset = 0; + return false; + } + + private MachSymbol[] ReadSymbols() + { + Reader stringReader = _stringReader.Value; + return _symbolTable.Value?.Select(n => new MachSymbol() { Name = stringReader.Read(n.StringIndex), Raw = n }).ToArray(); + } + + private Reader GetStringReader() + { + MachSymtabLoadCommand symtabLoadCommand = _symtabLoadCommand.Value; + return _physicalAddressSpace.WithRelativeAddressSpace(symtabLoadCommand.StringOffset, symtabLoadCommand.StringSize); + } + + private NList[] ReadSymbolTable() + { + MachSymtabLoadCommand symtabLoadCommand = _symtabLoadCommand.Value; + if (symtabLoadCommand.IsNSymsReasonable.Check() && symtabLoadCommand.SymOffset > 0) + { + return _physicalAddressSpace.ReadArray(symtabLoadCommand.SymOffset, symtabLoadCommand.SymCount); + } + return null; + } + } +} diff --git a/src/Microsoft.FileFormats/MachO/MachOStructures.cs b/src/Microsoft.FileFormats/MachO/MachOStructures.cs new file mode 100644 index 000000000..d376691ef --- /dev/null +++ b/src/Microsoft.FileFormats/MachO/MachOStructures.cs @@ -0,0 +1,307 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq; +using System.Text; + +namespace Microsoft.FileFormats.MachO +{ + public static class MachLayoutManagerExtensions + { + public static LayoutManager AddMachTypes(this LayoutManager layoutManager, bool isBigEndian, bool is64Bit) + { + layoutManager + .AddPrimitives(isBigEndian) + .AddSizeT(is64Bit ? 8 : 4) + .AddEnumTypes() + .AddNullTerminatedString() + .AddTStructTypes(); + return layoutManager; + } + } + + public enum MachHeaderMagicType : uint + { + LittleEndian64Bit = 0xfeedfacf, + LittleEndian32Bit = 0xfeedface, + BigEndian64Bit = 0xcffaedfe, + BigEndian32Bit = 0xcefaedfe + } + + public class MachHeaderMagic : TStruct + { + public MachHeaderMagicType Magic; + #region Validation Rules + public ValidationRule IsMagicValid + { + get + { + return new ValidationRule("Invalid MachO Header Magic", () => + { + return Magic == MachHeaderMagicType.LittleEndian32Bit || + Magic == MachHeaderMagicType.LittleEndian64Bit; + }); + } + } + #endregion + } + + public enum MachHeaderFileType : uint + { + Object = 1, + Execute = 2, + FvmLib = 3, + Core = 4, + Preload = 5, + Dylib = 6, + Dylinker = 7, + Bundle = 8, + DylibStub = 9, + Dsym = 10, + KextBundle = 11 + } + + public class MachHeader : MachHeaderMagic + { + public uint CpuType; + public uint CpuSubType; + public MachHeaderFileType FileType; + public uint NumberCommands; + public uint SizeOfCommands; + public uint Flags; + + #region Validation Rules + public ValidationRule IsFileTypeValid + { + get + { + return new ValidationRule("Mach Header FileType is invalid", + () => Enum.IsDefined(typeof(MachHeaderFileType), FileType)); + } + } + + public ValidationRule IsNumberCommandsReasonable + { + get + { + return new ValidationRule("Mach Header NumberCommands is unreasonable", + () => NumberCommands <= 20000); + } + } + #endregion + } + + public enum LoadCommandType + { + Segment = 1, + Symtab = 2, + Thread = 4, + DySymtab = 11, + Segment64 = 25, + Uuid = 27, + } + + public class MachLoadCommand : TStruct + { + public LoadCommandType Command; + public uint CommandSize; + + public override string ToString() + { + return "LoadCommand {" + Command + ", 0x" + CommandSize.ToString("x") + "}"; + } + + #region Validation Rules + public ValidationRule IsCmdSizeReasonable + { + get + { + return new ValidationRule("Mach Load Command Size is unreasonable", + () => CommandSize < 0x1000); + } + } + + public ValidationRule IsCommandRecognized + { + get + { + return new ValidationRule("Mach Load Command is not recognized", + () => Enum.IsDefined(typeof(LoadCommandType), Command)); + } + } + #endregion + } + + public class MachFixedLengthString16 : TStruct + { + [ArraySize(16)] + public byte[] Bytes; + + public override string ToString() + { + try + { + return Encoding.UTF8.GetString(Bytes.TakeWhile(b => b != 0).ToArray()); + } + catch (FormatException) + { + throw new BadInputFormatException("Bytes could not be parsed with UTF8 encoding"); + } + } + } + + public class MachSegmentLoadCommand : MachLoadCommand + { + public MachFixedLengthString16 SegName; + public SizeT VMAddress; + public SizeT VMSize; + public SizeT FileOffset; + public SizeT FileSize; + public uint MaxProt; + public uint InitProt; + public uint CountSections; + public uint Flags; + + #region Validation Rules + private ValidationRule IsCommandValid + { + get + { + return new ValidationRule("Mach Segment Command has invalid Command field", + () => Command == LoadCommandType.Segment || Command == LoadCommandType.Segment64); + } + } + #endregion + } + + public class MachSection : TStruct + { + public MachFixedLengthString16 SectionName; + public MachFixedLengthString16 SegmentName; + public SizeT Address; + public SizeT Size; + public uint Offset; + public uint Align; + public uint RelativeOffset; + public uint CountRelocs; + public uint Flags; + public uint Reserved1; + public uint Reserved2; + } + + public class MachUuidLoadCommand : MachLoadCommand + { + [ArraySize(16)] + public byte[] Uuid; + + #region Validation Rules + public ValidationRule IsCommandValid + { + get + { + return new ValidationRule("Mach UUID LoadCommand has invalid command id", + () => Command == LoadCommandType.Uuid); + } + } + + public ValidationRule IsCommandSizeValid + { + get + { + return new ValidationRule("Mach UUID LoadCommand has invalid size", + () => CommandSize == 24); + } + } + #endregion + } + + public class MachSymtabLoadCommand : MachLoadCommand + { + public uint SymOffset; + public uint SymCount; + public uint StringOffset; + public uint StringSize; + + #region Validation Rules + public ValidationRule IsCommandValid + { + get + { + return new ValidationRule("Mach Symtab LoadCommand has invalid command id", + () => Command == LoadCommandType.Symtab); + } + } + + public ValidationRule IsCommandSizeValid + { + get + { + return new ValidationRule("Mach Symtab LoadCommand has invalid size", + () => CommandSize == 24); + } + } + + public ValidationRule IsNSymsReasonable + { + get + { + return new ValidationRule("Mach symtab LoadCommand has unreasonable SymCount", + () => SymCount <= 0x100000); + } + } + #endregion + } + + public class MachDySymtabLoadCommand : MachLoadCommand + { + public uint ILocalSym; + public uint NLocalSym; + public uint IExtDefSym; + public uint NextDefSym; + public uint IUndefSym; + public uint NUndefSym; + public uint ToCoff; + public uint NToc; + public uint ModTabOff; + public uint MModTab; + public uint ExtrefSymOff; + public uint NextrefSyms; + public uint IndirectSymOff; + public uint NindirectSyms; + public uint ExtrelOff; + public uint Nextrel; + public uint LocrelOff; + public uint NLocrel; + } + + public class NList : TStruct + { + public uint StringIndex; + public byte Type; + public byte Section; + public ushort Desc; + public SizeT Value; + } + + public class DyldImageAllInfosVersion : TStruct + { + public uint Version; + } + + public class DyldImageAllInfosV2 : DyldImageAllInfosVersion + { + public uint InfoArrayCount; + public SizeT InfoArray; + public SizeT Notification; + public SizeT Undetermined; // there are some fields here but I haven't determined their size and purpose + public SizeT ImageLoadAddress; + } + + public class DyldImageInfo : TStruct + { + public SizeT Address; + public SizeT PathAddress; + public SizeT ModDate; + } +} diff --git a/src/Microsoft.FileFormats/MemoryBufferAddressSpace.cs b/src/Microsoft.FileFormats/MemoryBufferAddressSpace.cs new file mode 100644 index 000000000..36a49b1cc --- /dev/null +++ b/src/Microsoft.FileFormats/MemoryBufferAddressSpace.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// Creates an address space that reads from a byte array in memory. + /// + public sealed class MemoryBufferAddressSpace : IAddressSpace + { + public MemoryBufferAddressSpace(IEnumerable bytes) + { + _bytes = bytes.ToArray(); + Length = (ulong)_bytes.Length; + } + + /// + /// The upper bound (non-inclusive) of readable addresses + /// + public ulong Length { get; private set; } + + /// + /// Reads a range of bytes from the address space + /// + /// The position in the address space to begin reading from + /// The buffer that will receive the bytes that are read + /// The offset in the output buffer to begin writing the bytes + /// The number of bytes to read into the buffer + /// The number of bytes read + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + if (position >= Length || position + count > Length) + { + throw new BadInputFormatException("Unexpected end of data: Expected " + count + " bytes."); + } + Array.Copy(_bytes, (int)position, buffer, (int)bufferOffset, (int)count); + return count; + } + + private byte[] _bytes; + } +} diff --git a/src/Microsoft.FileFormats/Microsoft.FileFormats.csproj b/src/Microsoft.FileFormats/Microsoft.FileFormats.csproj new file mode 100644 index 000000000..acbd447d4 --- /dev/null +++ b/src/Microsoft.FileFormats/Microsoft.FileFormats.csproj @@ -0,0 +1,12 @@ + + + net462;netstandard2.0 + ;1591;1701 + true + File format readers + $(Description) + File formats + true + true + + diff --git a/src/Microsoft.FileFormats/Minidump/Minidump.cs b/src/Microsoft.FileFormats/Minidump/Minidump.cs new file mode 100644 index 000000000..641cbde10 --- /dev/null +++ b/src/Microsoft.FileFormats/Minidump/Minidump.cs @@ -0,0 +1,175 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.Linq; + +namespace Microsoft.FileFormats.Minidump +{ + /// + /// A class which represents a Minidump (Microsoft "crash dump"). + /// + public class Minidump + { + private readonly ulong _position; + private readonly Reader _dataSourceReader; + private readonly MinidumpHeader _header; + private readonly MinidumpDirectory[] _directory; + private readonly MinidumpSystemInfo _systemInfo; + private readonly int _moduleListStream = -1; + private readonly Lazy> _loadedImages; + private readonly Lazy> _memoryRanges; + private readonly Lazy _virtualAddressReader; + + /// + /// Returns true if the given address space is a minidump. + /// + /// The address space to check. + /// The position of the minidump. + /// True if the address space is a minidump, false otherwise. + public static bool IsValid(IAddressSpace addressSpace, ulong position = 0) + { + Reader headerReader = new(addressSpace); + return headerReader.TryRead(position, out MinidumpHeader header) && header.IsSignatureValid.Check(); + } + + /// + /// Constructor. This constructor will throw exceptions if the file is not a minidump or contains corrupted data + /// which interferes with parsing it. + /// + /// The memory which backs this object. + /// The offset within addressSpace this minidump is located at. + public Minidump(IAddressSpace dataSource, ulong position = 0) + { + _position = position; + + Reader headerReader = new(dataSource); + _header = headerReader.Read(_position); + _header.IsSignatureValid.CheckThrowing(); + + int systemIndex = -1; + _directory = new MinidumpDirectory[_header.NumberOfStreams]; + ulong streamPos = _position + _header.StreamDirectoryRva; + for (int i = 0; i < _directory.Length; i++) + { + _directory[i] = headerReader.Read(ref streamPos); + + MinidumpStreamType streamType = _directory[i].StreamType; + if (streamType == MinidumpStreamType.SystemInfoStream) + { + Debug.Assert(systemIndex == -1); + systemIndex = i; + } + else if (streamType == MinidumpStreamType.ModuleListStream) + { + Debug.Assert(_moduleListStream == -1); + _moduleListStream = i; + } + } + + if (systemIndex == -1) + { + throw new BadInputFormatException("Minidump does not contain a MINIDUMP_SYSTEM_INFO stream"); + } + _systemInfo = headerReader.Read(_position + _directory[systemIndex].Rva); + + _dataSourceReader = new Reader(dataSource, new LayoutManager().AddCrashDumpTypes(false, Is64Bit)); + _loadedImages = new Lazy>(CreateLoadedImageList); + _memoryRanges = new Lazy>(CreateSegmentList); + _virtualAddressReader = new Lazy(CreateVirtualAddressReader); + } + + /// + /// Returns the architecture of the target process. + /// + public ProcessorArchitecture Architecture { get { return _systemInfo.ProcessorArchitecture; } } + + /// + /// A raw data reader for the underlying minidump file itself. + /// + public Reader DataSourceReader { get { return _dataSourceReader; } } + + /// + /// A raw data reader for the memory in virtual address space of this minidump. + /// + public Reader VirtualAddressReader { get { return _virtualAddressReader.Value; } } + + /// + /// A collection of loaded images in the minidump. This does NOT contain unloaded modules. + /// + public ReadOnlyCollection LoadedImages { get { return _loadedImages.Value.AsReadOnly(); } } + + /// + /// A collection of all the memory segments in minidump. + /// + public ReadOnlyCollection Segments { get { return _memoryRanges.Value.AsReadOnly(); } } + + /// + /// Returns true if the original process represented by this minidump was running as an x64 process or not. + /// + public bool Is64Bit + { + get + { + ProcessorArchitecture arch = _systemInfo.ProcessorArchitecture; + return arch == ProcessorArchitecture.Alpha64 || arch == ProcessorArchitecture.Amd64 || arch == ProcessorArchitecture.Ia64; + } + } + + private Reader CreateVirtualAddressReader() + { + return _dataSourceReader.WithAddressSpace(new MinidumpVirtualAddressSpace(Segments, _dataSourceReader.DataSource)); + } + + private List CreateLoadedImageList() + { + if (_moduleListStream == -1) + { + throw new BadInputFormatException("Minidump does not contain a ModuleStreamList in its directory."); + } + MinidumpModule[] modules = _dataSourceReader.ReadCountedArray(_position + _directory[_moduleListStream].Rva); + return new List(modules.Select(module => new MinidumpLoadedImage(this, module))); + } + + private List CreateSegmentList() + { + List ranges = new(); + + foreach (MinidumpDirectory item in _directory) + { + if (item.StreamType == MinidumpStreamType.MemoryListStream) + { + MinidumpMemoryDescriptor[] memoryRegions = _dataSourceReader.ReadCountedArray(_position + item.Rva); + + foreach (MinidumpMemoryDescriptor region in memoryRegions) + { + MinidumpSegment range = new(region); + ranges.Add(range); + } + + } + else if (item.StreamType == MinidumpStreamType.Memory64ListStream) + { + ulong position = item.Rva; + ulong count = _dataSourceReader.Read(ref position); + ulong rva = _dataSourceReader.Read(ref position); + + MinidumpMemoryDescriptor64[] memoryRegions = _dataSourceReader.ReadArray(position + _position, checked((uint)count)); + foreach (MinidumpMemoryDescriptor64 region in memoryRegions) + { + MinidumpSegment range = new(region, rva); + ranges.Add(range); + + rva += region.DataSize; + } + } + } + + ranges.Sort((MinidumpSegment a, MinidumpSegment b) => a.VirtualAddress.CompareTo(b.VirtualAddress)); + return ranges; + } + } +} diff --git a/src/Microsoft.FileFormats/Minidump/MinidumpLoadedImage.cs b/src/Microsoft.FileFormats/Minidump/MinidumpLoadedImage.cs new file mode 100644 index 000000000..f218a8543 --- /dev/null +++ b/src/Microsoft.FileFormats/Minidump/MinidumpLoadedImage.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Text; +using Microsoft.FileFormats.PE; + +namespace Microsoft.FileFormats.Minidump +{ + public class MinidumpLoadedImage + { + private readonly MinidumpModule _module; + private readonly Lazy _peFile; + private readonly Lazy _moduleName; + + /// + /// The minidump containing this loaded image. + /// + public Minidump Minidump { get; private set; } + + /// + /// The base address in the minidump's virtual address space that this image is mapped. + /// + public ulong BaseAddress { get { return _module.Baseofimage; } } + + /// + /// The checksum of this image. + /// + public uint CheckSum { get { return _module.CheckSum; } } + + /// + /// The TimeDateStamp of this image, as baked into the PE header. This value is used + /// for symbol sever requests to obtain a PE image. + /// + public uint TimeDateStamp { get { return _module.TimeDateStamp; } } + + /// + /// The compile time size of this PE image as it is baked into the PE header. This + /// value is used for simple server requests to obtain a PE image. + /// + public uint ImageSize { get { return _module.SizeOfImage; } } + + + /// + /// The full name of this module (including path it was originally loaded from on disk). + /// + public string ModuleName { get { return _moduleName.Value; } } + + /// + /// A PEFile representing this image. + /// + public PEFile Image { get { return _peFile.Value; } } + + public uint Major { get { return _module.VersionInfo.FileVersionMS >> 16; } } + public uint Minor { get { return _module.VersionInfo.FileVersionMS & 0xffff; } } + public uint Revision { get { return _module.VersionInfo.FileVersionLS >> 16; } } + public uint Patch { get { return _module.VersionInfo.FileVersionLS & 0xffff; } } + + internal MinidumpLoadedImage(Minidump minidump, MinidumpModule module) + { + Minidump = minidump; + _module = module; + _peFile = new Lazy(CreatePEFile); + _moduleName = new Lazy(GetModuleName); + } + + private PEFile CreatePEFile() + { + return new PEFile(new RelativeAddressSpace(Minidump.VirtualAddressReader.DataSource, BaseAddress, Minidump.VirtualAddressReader.Length), true); + } + + private string GetModuleName() + { + return Minidump.DataSourceReader.ReadCountedString(_module.ModuleNameRva, Encoding.Unicode); + } + } +} diff --git a/src/Microsoft.FileFormats/Minidump/MinidumpSegment.cs b/src/Microsoft.FileFormats/Minidump/MinidumpSegment.cs new file mode 100644 index 000000000..0fb72aae7 --- /dev/null +++ b/src/Microsoft.FileFormats/Minidump/MinidumpSegment.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.FileFormats.Minidump +{ + /// + /// Represents a segment of memory in the minidump's virtual address space. + /// + public class MinidumpSegment + { + /// + /// The physical location in the minidump file where this memory segment resides. + /// + public ulong FileOffset { get; private set; } + + /// + /// The base address of this chunk of virtual memory in the original process. + /// + public ulong VirtualAddress { get; private set; } + + /// + /// The size of this chunk of memory. Note that this is both the size of the physical + /// memory in the minidump as well as the virtual memory in the original process. + /// + public ulong Size { get; private set; } + + /// + /// Returns whether the given address is contained in this region of virtual memory. + /// + /// A virtual address in the original process's address space. + /// True if this segment contains the address, false otherwise. + public bool Contains(ulong address) + { + return VirtualAddress <= address && address < VirtualAddress + Size; + } + + internal MinidumpSegment(MinidumpMemoryDescriptor region) + { + FileOffset = region.Memory.Rva; + Size = region.Memory.DataSize; + VirtualAddress = region.StartOfMemoryRange; + } + + internal MinidumpSegment(MinidumpMemoryDescriptor64 region, ulong rva) + { + FileOffset = rva; + Size = region.DataSize; + VirtualAddress = region.StartOfMemoryRange; + } + } +} diff --git a/src/Microsoft.FileFormats/Minidump/MinidumpStructures.cs b/src/Microsoft.FileFormats/Minidump/MinidumpStructures.cs new file mode 100644 index 000000000..9b1a3c07a --- /dev/null +++ b/src/Microsoft.FileFormats/Minidump/MinidumpStructures.cs @@ -0,0 +1,168 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats.Minidump +{ + public static class CrashDumpLayoutManagerExtensions + { + public static LayoutManager AddCrashDumpTypes(this LayoutManager layouts, bool isBigEndian, bool is64Bit) + { + return layouts + .AddPrimitives(isBigEndian) + .AddEnumTypes() + .AddSizeT(is64Bit ? 8 : 4) + .AddPointerTypes() + .AddNullTerminatedString() + .AddTStructTypes(); + } + } + +#pragma warning disable 0649 +#pragma warning disable 0169 + + internal sealed class MinidumpHeader : TStruct + { + public const int MinidumpVersion = 0x504d444d; + + public uint Signature; + public uint Version; + public uint NumberOfStreams; + public uint StreamDirectoryRva; + public uint CheckSum; + public uint TimeDateStamp; + public ulong Flags; + + public ValidationRule IsSignatureValid + { + get + { + return new ValidationRule("Invalid minidump header signature", () => + { + return Signature == MinidumpVersion; + }); + } + } + } + + internal sealed class MinidumpDirectory : TStruct + { + public MinidumpStreamType StreamType; + public uint DataSize; + public uint Rva; + } + + internal enum MinidumpStreamType + { + UnusedStream = 0, + ReservedStream0 = 1, + ReservedStream1 = 2, + ThreadListStream = 3, + ModuleListStream = 4, + MemoryListStream = 5, + ExceptionStream = 6, + SystemInfoStream = 7, + ThreadExListStream = 8, + Memory64ListStream = 9, + CommentStreamA = 10, + CommentStreamW = 11, + HandleDataStream = 12, + FunctionTableStream = 13, + UnloadedModuleListStream = 14, + MiscInfoStream = 15, + MemoryInfoListStream = 16, + ThreadInfoListStream = 17, + LastReservedStream = 0xffff, + } + + + internal sealed class MinidumpSystemInfo : TStruct + { + public ProcessorArchitecture ProcessorArchitecture; + public ushort ProcessorLevel; + public ushort ProcessorRevision; + public byte NumberOfProcessors; + public byte ProductType; + public uint MajorVersion; + public uint MinorVersion; + public uint BuildNumber; + public uint PlatformId; + public uint CSDVersionRva; + } + + public enum ProcessorArchitecture : ushort + { + Intel = 0, + Mips = 1, + Alpha = 2, + Ppc = 3, + Shx = 4, + Arm = 5, + Ia64 = 6, + Alpha64 = 7, + Msil = 8, + Amd64 = 9, + Ia32OnWin64 = 10, + } + + internal sealed class FixedFileInfo : TStruct + { + public uint Signature; /* e.g. 0xfeef04bd */ + public uint StrucVersion; /* e.g. 0x00000042 = "0.42" */ + public uint FileVersionMS; /* e.g. 0x00030075 = "3.75" */ + public uint FileVersionLS; /* e.g. 0x00000031 = "0.31" */ + public uint ProductVersionMS; /* e.g. 0x00030010 = "3.10" */ + public uint ProductVersionLS; /* e.g. 0x00000031 = "0.31" */ + public uint FileFlagsMask; /* = 0x3F for version "0.42" */ + public uint FileFlags; /* e.g. VFF_DEBUG | VFF_PRERELEASE */ + public uint FileOS; /* e.g. VOS_DOS_WINDOWS16 */ + public uint FileType; /* e.g. VFT_DRIVER */ + public uint FileSubtype; /* e.g. VFT2_DRV_KEYBOARD */ + + // Timestamps would be useful, but they're generally missing (0). + public uint FileDateMS; /* e.g. 0 */ + public uint FileDateLS; /* e.g. 0 */ + } + + + internal sealed class MinidumpLocationDescriptor : TStruct + { + public uint DataSize; + public uint Rva; + } + + [Pack(4)] + internal sealed class MinidumpModule : TStruct + { + public ulong Baseofimage; + public uint SizeOfImage; + public uint CheckSum; + public uint TimeDateStamp; + public uint ModuleNameRva; + public FixedFileInfo VersionInfo; + public MinidumpLocationDescriptor CvRecord; + public MinidumpLocationDescriptor MiscRecord; +#pragma warning disable CA1823 // Avoid unused private fields + private ulong _reserved0; + private ulong _reserved1; +#pragma warning restore CA1823 // Avoid unused private fields + } + + internal sealed class MinidumpMemoryDescriptor : TStruct + { + public ulong StartOfMemoryRange; + public MinidumpLocationDescriptor Memory; + + } + + internal sealed class MinidumpMemoryDescriptor64 : TStruct + { + public ulong StartOfMemoryRange; + public ulong DataSize; + } +} diff --git a/src/Microsoft.FileFormats/Minidump/MinidumpVirtualAddressSpace.cs b/src/Microsoft.FileFormats/Minidump/MinidumpVirtualAddressSpace.cs new file mode 100644 index 000000000..ecd8ef1fc --- /dev/null +++ b/src/Microsoft.FileFormats/Minidump/MinidumpVirtualAddressSpace.cs @@ -0,0 +1,80 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats.Minidump +{ + public class MinidumpVirtualAddressSpace : IAddressSpace + { + private readonly IAddressSpace _addressSpace; + private readonly ReadOnlyCollection _segments; + private readonly ulong _length; + + public ulong Length + { + get + { + return _length; + } + } + + public MinidumpVirtualAddressSpace(ReadOnlyCollection segments, IAddressSpace addressSpace) + { + _addressSpace = addressSpace; + _segments = segments; + MinidumpSegment last = segments.Last(); + _length = last.VirtualAddress + last.Size; + } + + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + if (count == 0) + { + return 0; + } + MinidumpSegment seg = FindSegment(position); + if (seg == null) + { + return 0; + } + // TODO: What if they read past the end of the segment? + Debug.Assert(position >= seg.VirtualAddress); + ulong offset = position - seg.VirtualAddress + seg.FileOffset; + return _addressSpace.Read(offset, buffer, bufferOffset, count); + } + + private MinidumpSegment FindSegment(ulong position) + { + int min = 0; + int max = _segments.Count - 1; + + while (min <= max) + { + int mid = (min + max) / 2; + MinidumpSegment current = _segments[mid]; + + if (position < current.VirtualAddress) + { + max = mid - 1; + } + else if (position >= current.VirtualAddress + current.Size) + { + min = mid + 1; + } + else + { + Debug.Assert(current.Contains(position)); + return current; + } + } + + return null; + } + } +} diff --git a/src/Microsoft.FileFormats/Minidump/ReaderExtensions.cs b/src/Microsoft.FileFormats/Minidump/ReaderExtensions.cs new file mode 100644 index 000000000..180f7907c --- /dev/null +++ b/src/Microsoft.FileFormats/Minidump/ReaderExtensions.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; + +namespace Microsoft.FileFormats.Minidump +{ + internal static class MinidumpReaderExtensions + { + public static string ReadCountedString(this Reader self, ulong position, Encoding encoding) + { + uint elementCount = self.Read(ref position); + byte[] buffer = self.Read(position, elementCount); + return encoding.GetString(buffer); + } + + public static T[] ReadCountedArray(this Reader self, ulong position) + { + uint elementCount = self.Read(ref position); + return (T[])self.LayoutManager.GetArrayLayout(elementCount).Read(self.DataSource, position); + } + } +} diff --git a/src/Microsoft.FileFormats/NullTerminatedString.cs b/src/Microsoft.FileFormats/NullTerminatedString.cs new file mode 100644 index 000000000..cedf0b9ec --- /dev/null +++ b/src/Microsoft.FileFormats/NullTerminatedString.cs @@ -0,0 +1,111 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// Parses a string object from a standard null-terminated byte sequence. + /// + /// + /// Currently this type only supports ASCII or UTF8 encoding. + /// + public class NullTerminatedStringLayout : ILayout + { + private Encoding _encoding; + + /// + /// Create a new NullTerminatedStringLayout + /// + /// The encoding used to parse the string characters. Currently on ASCII or UTF8 is supported + public NullTerminatedStringLayout(Encoding encoding) + { + // Make sure we are scanning something where a single 0 byte means end of string. + // Although UTF8 does have code points that encode with more than one byte, + // byte 0 is never used except to encode code point 0. + if (encoding != Encoding.UTF8 && encoding != Encoding.ASCII) + { + throw new NotSupportedException("PEReader.ReadNullTerminatedString: Only UTF8 or ascii are supported for now"); + } + _encoding = encoding; + + // We could implement this for multi-byte encodings, there hasn't been a + // need yet. If you do change it, make sure to adjust NaturalAlignment too + } + + public IEnumerable Fields { get { return Array.Empty(); } } + public uint NaturalAlignment { get { return 1U; } } + + public bool IsFixedSize { get { return false; } } + + public uint Size + { + get + { + throw new InvalidOperationException("Size is invalid for variable sized layouts"); + } + } + + public uint SizeAsBaseType + { + get + { + throw new InvalidOperationException("Size is invalid for variable sized layouts"); + } + } + + public Type Type { get { return typeof(string); } } + + public object Read(IAddressSpace dataSource, ulong position) + { + return Read(dataSource, position, out uint _); + } + + public object Read(IAddressSpace dataSource, ulong position, out uint bytesRead) + { + List stringBytes = new(); + uint offset = 0; + for (; ; offset++) + { + byte[] nextByte = dataSource.Read(position + offset, 1); + if (nextByte[0] == 0) + { + break; + } + else + { + stringBytes.Add(nextByte[0]); + } + } + bytesRead = offset + 1; + return _encoding.GetString(stringBytes.ToArray(), 0, stringBytes.Count); + } + } + + public static partial class LayoutManagerExtensions + { + /// + /// Add support for parsing null terminated strings as System.String + /// + public static LayoutManager AddNullTerminatedString(this LayoutManager layouts) + { + return AddNullTerminatedString(layouts, Encoding.UTF8); + } + + /// + /// Add support for parsing null terminated strings as System.String + /// + /// The layout manager that will hold the new layout + /// The encoding used to parse string characters. Currently only UTF8 and ASCII are supported + public static LayoutManager AddNullTerminatedString(this LayoutManager layouts, Encoding encoding) + { + layouts.AddLayout(new NullTerminatedStringLayout(encoding)); + return layouts; + } + } +} diff --git a/src/Microsoft.FileFormats/PDB/PDBFile.cs b/src/Microsoft.FileFormats/PDB/PDBFile.cs new file mode 100644 index 000000000..0ccdda087 --- /dev/null +++ b/src/Microsoft.FileFormats/PDB/PDBFile.cs @@ -0,0 +1,187 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; + +namespace Microsoft.FileFormats.PDB +{ + public class PDBFile : IDisposable + { + private readonly Reader _reader; + private readonly Lazy _header; + private readonly Lazy _streams; + private readonly Lazy _nameStream; + private readonly Lazy _dbiStream; + + public PDBFile(IAddressSpace dataSource) + { + _reader = new Reader(dataSource); + _header = new Lazy(() => _reader.Read(0)); + _streams = new Lazy(ReadDirectory); + _nameStream = new Lazy(() => new PDBNameStream(Streams[1])); + _dbiStream = new Lazy(() => new DbiStream(Streams[3])); + } + + public PDBFileHeader Header { get { return _header.Value; } } + public IList Streams { get { return _streams.Value; } } + public PDBNameStream NameStream { get { return _nameStream.Value; } } + public DbiStream DbiStream { get { return _dbiStream.Value; } } + public uint Age { get { return NameStream.Header.Age; } } + public uint DbiAge { get { return DbiStream.Header.Age; } } + public Guid Signature { get { return new Guid(NameStream.Header.Guid); } } + + public void Dispose() + { + if (_reader.DataSource is IDisposable disposable) + { + disposable.Dispose(); + } + } + + public bool IsValid() + { + if (_reader.Length > _reader.SizeOf()) { + return Header.IsMagicValid.Check(); + } + return false; + } + + private Reader[] ReadDirectory() + { + Header.IsMagicValid.CheckThrowing(); + uint secondLevelPageCount = ToPageCount(Header.DirectorySize); + ulong pageIndicesOffset = _reader.SizeOf(); + PDBPagedAddressSpace secondLevelPageList = CreatePagedAddressSpace(_reader.DataSource, pageIndicesOffset, secondLevelPageCount * sizeof(uint)); + PDBPagedAddressSpace directoryContent = CreatePagedAddressSpace(secondLevelPageList, 0, Header.DirectorySize); + + Reader directoryReader = new(directoryContent); + ulong position = 0; + uint countStreams = directoryReader.Read(ref position); + uint[] streamSizes = directoryReader.ReadArray(ref position, countStreams); + Reader[] streams = new Reader[countStreams]; + for (uint i = 0; i < streamSizes.Length; i++) + { + streams[i] = new Reader(CreatePagedAddressSpace(directoryContent, position, streamSizes[i])); + position += ToPageCount(streamSizes[i]) * sizeof(uint); + } + return streams; + } + + private PDBPagedAddressSpace CreatePagedAddressSpace(IAddressSpace indicesData, ulong offset, uint length) + { + uint[] indices = new Reader(indicesData).ReadArray(offset, ToPageCount(length)); + return new PDBPagedAddressSpace(_reader.DataSource, indices, Header.PageSize, length); + } + + private uint ToPageCount(uint size) + { + return unchecked((Header.PageSize + size - 1) / Header.PageSize); + } + } + + /// + /// Defines a virtual address paged address space that maps to an underlying physical + /// paged address space with a different set of page Indices. + /// + /// + /// A paged address space is an address space where any address A can be converted + /// to a page index and a page offset. A = index*page_size + offset. + /// + /// This paged address space maps each virtual address to a physical address by + /// remapping each virtual page to potentially different physical page. If V is + /// the virtual page index then pageIndices[V] is the physical page index. + /// + /// For example if pageSize is 0x100 and pageIndices is { 0x7, 0x9 } then + /// virtual address 0x156 is: + /// virtual page index 0x1, virtual offset 0x56 + /// physical page index 0x9, physical offset 0x56 + /// physical address is 0x956 + /// + internal sealed class PDBPagedAddressSpace : IAddressSpace + { + private readonly IAddressSpace _physicalAddresses; + private readonly uint[] _pageIndices; + private readonly uint _pageSize; + + public PDBPagedAddressSpace(IAddressSpace physicalAddresses, uint[] pageIndices, uint pageSize, ulong length) + { + _physicalAddresses = physicalAddresses; + _pageIndices = pageIndices; + _pageSize = pageSize; + Length = length; + } + + public ulong Length { get; private set; } + + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + if (position + count > Length) + { + throw new BadInputFormatException("Unexpected end of data: Expected " + count + " bytes."); + } + + uint bytesRead = 0; + while (bytesRead != count) + { + uint virtualPageOffset; + ulong physicalPosition = GetPhysicalAddress(position, out virtualPageOffset); + uint pageBytesToRead = Math.Min(_pageSize - virtualPageOffset, count - bytesRead); + uint pageBytesRead = _physicalAddresses.Read(physicalPosition, buffer, bufferOffset + bytesRead, pageBytesToRead); + bytesRead += pageBytesRead; + position += pageBytesRead; + if (pageBytesToRead != pageBytesRead) + { + break; + } + } + return bytesRead; + } + + private ulong GetPhysicalAddress(ulong virtualAddress, out uint virtualOffset) + { + uint virtualPageIndex = (uint)(virtualAddress / _pageSize); + virtualOffset = (uint)(virtualAddress - (virtualPageIndex * _pageSize)); + uint physicalPageIndex = _pageIndices[(int)virtualPageIndex]; + return (ulong)physicalPageIndex * _pageSize + virtualOffset; + } + } + + public class PDBNameStream + { + private readonly Reader _streamReader; + private readonly Lazy _header; + + + public PDBNameStream(Reader streamReader) + { + _streamReader = streamReader; + _header = new Lazy(() => _streamReader.Read(0)); + } + + public NameIndexStreamHeader Header { get { return _header.Value; } } + } + + public class DbiStream + { + private readonly Reader _streamReader; + private readonly Lazy _header; + + + public DbiStream(Reader streamReader) + { + _streamReader = streamReader; + _header = new Lazy(() => _streamReader.Read(0)); + } + + public bool IsValid() + { + if (_streamReader.Length >= _streamReader.SizeOf()) { + return _header.Value.IsHeaderValid.Check(); + } + return false; + } + + public DbiStreamHeader Header { get { _header.Value.IsHeaderValid.CheckThrowing(); return _header.Value; } } + } +} diff --git a/src/Microsoft.FileFormats/PDB/PDBStructures.cs b/src/Microsoft.FileFormats/PDB/PDBStructures.cs new file mode 100644 index 000000000..7c082a6be --- /dev/null +++ b/src/Microsoft.FileFormats/PDB/PDBStructures.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats.PDB +{ + public class PDBFileHeader : TStruct + { + private static byte[] ExpectedMagic + { + get + { + return new byte[] + { + 0x4D, 0x69, 0x63, 0x72, 0x6F, 0x73, 0x6F, 0x66, // "Microsof" + 0x74, 0x20, 0x43, 0x2F, 0x43, 0x2B, 0x2B, 0x20, // "t C/C++ " + 0x4D, 0x53, 0x46, 0x20, 0x37, 0x2E, 0x30, 0x30, // "MSF 7.00" + 0x0D, 0x0A, 0x1A, 0x44, 0x53, 0x00, 0x00, 0x00 // "^^^DS^^^" + }; + } + } + + [ArraySize(32)] + public byte[] Magic; + public uint PageSize; + public uint FreePageMap; + public uint PagesUsed; + public uint DirectorySize; + public uint Reserved; + + #region Validation Rules + public ValidationRule IsMagicValid + { + get { return new ValidationRule("PDB header magic is invalid", () => Magic.SequenceEqual(ExpectedMagic)); } + } + #endregion + } + + public class NameIndexStreamHeader : TStruct + { + public uint Version; + public uint Signature; + public uint Age; + [ArraySize(16)] + public byte[] Guid; + public uint CountStringBytes; + } + + public class DbiStreamHeader : TStruct + { + private const uint CurrentSignature = uint.MaxValue; + private const uint CurrentVersion = 19990903; // DBIImpvV70 + + public uint Signature; + public uint Version; + public uint Age; + + // This is not the complete DBI header, but it is enough to get the Age. + + #region Validation Rules + public ValidationRule IsHeaderValid + { + get { return new ValidationRule("DBI header is invalid", () => Signature == CurrentSignature && Version == CurrentVersion); } + } + #endregion + } +} diff --git a/src/Microsoft.FileFormats/PE/PEFile.cs b/src/Microsoft.FileFormats/PE/PEFile.cs new file mode 100644 index 000000000..228a601b6 --- /dev/null +++ b/src/Microsoft.FileFormats/PE/PEFile.cs @@ -0,0 +1,435 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; + +namespace Microsoft.FileFormats.PE +{ + /// + /// A very basic PE reader that can extract a few useful pieces of information + /// + public class PEFile : IDisposable + { + // PE file + private readonly bool _isDataSourceVirtualAddressSpace; + private readonly Reader _headerReader; + private readonly Lazy _dosHeaderMagic; + private readonly Lazy _fileHeader; + private readonly Lazy _peHeaderOffset; + private readonly Lazy _peSignature; + private readonly Lazy _optionalHeaderMagic; + private readonly Lazy _fileReader; + private readonly Lazy _optionalHeader; + private readonly Lazy> _imageDataDirectory; + private readonly Lazy> _pdb; + private readonly Lazy> _segments; + private readonly Lazy _vsFixedFileInfo; + private readonly Lazy> _pdbChecksum; + private readonly Lazy> _perfMapsV1; + private readonly Lazy _virtualAddressReader; + private readonly Lazy _exportDirectory; + + private const ushort ExpectedDosHeaderMagic = 0x5A4D; // MZ + private const int PESignatureOffsetLocation = 0x3C; + private const uint ExpectedPESignature = 0x00004550; // PE00 + private const int ImageDataDirectoryCount = 15; + + public const uint ChecksumLength = 4; + public const uint CertDirectoryLength = 8; + public const int CertDirectoryIndex = 4; + + public PEFile(IAddressSpace dataSource, bool isDataSourceVirtualAddressSpace = false) + { + _isDataSourceVirtualAddressSpace = isDataSourceVirtualAddressSpace; + _headerReader = new Reader(dataSource); + _dosHeaderMagic = new Lazy(() => _headerReader.Read(0)); + _peHeaderOffset = new Lazy(ReadPEHeaderOffset); + _peSignature = new Lazy(() => _headerReader.Read(PEHeaderOffset)); + _fileHeader = new Lazy(ReadFileHeader); + _optionalHeaderMagic = new Lazy(ReadOptionalHeaderMagic); + _fileReader = new Lazy(CreateFileReader); + _optionalHeader = new Lazy(ReadOptionalHeader); + _imageDataDirectory = new Lazy>(ReadImageDataDirectory); + _pdb = new Lazy>(ReadPdbInfo); + _segments = new Lazy>(ReadSectionHeaders); + _vsFixedFileInfo = new Lazy(ReadVersionResource); + _pdbChecksum = new Lazy>(ReadPdbChecksum); + _perfMapsV1 = new Lazy>(ReadPerfMapV1Entries); + _virtualAddressReader = new Lazy(CreateVirtualAddressReader); + _exportDirectory = new Lazy(ReadExportDirectory); + } + + public ushort DosHeaderMagic { get { return _dosHeaderMagic.Value; } } + public uint PEHeaderOffset { get { return _peHeaderOffset.Value; } } + public uint PESignature { get { return _peSignature.Value; } } + public ImageFileHeader FileHeader { get { return _fileHeader.Value; } } + public uint Timestamp { get { return FileHeader.TimeDateStamp; } } + public ImageOptionalHeaderMagic OptionalHeaderMagic { get { return _optionalHeaderMagic.Value; } } + public Reader FileReader { get { return _fileReader.Value; } } + public ImageOptionalHeader OptionalHeader { get { return _optionalHeader.Value; } } + public uint SizeOfImage { get { return OptionalHeader.SizeOfImage; } } + public ReadOnlyCollection ImageDataDirectory { get { return _imageDataDirectory.Value.AsReadOnly(); } } + public IEnumerable Pdbs { get { return _pdb.Value; } } + public Reader RelativeVirtualAddressReader { get { return _virtualAddressReader.Value; } } + public ReadOnlyCollection Segments { get { return _segments.Value.AsReadOnly(); } } + public VsFixedFileInfo VersionInfo { get { return _vsFixedFileInfo.Value; } } + public IEnumerable PerfMapsV1 { get { return _perfMapsV1.Value; } } + public IEnumerable PdbChecksums { get { return _pdbChecksum.Value; } } + + public void Dispose() + { + if (_headerReader.DataSource is IDisposable disposable) + { + disposable.Dispose(); + } + } + + public bool IsValid() + { + if (_headerReader.Length > sizeof(ushort)) + { + try + { + if (HasValidDosSignature.Check()) + { + if (_headerReader.Length > PESignatureOffsetLocation) + { + return HasValidPESignature.Check(); + } + } + } + catch (Exception ex) when (ex is InvalidVirtualAddressException || ex is BadInputFormatException) + { + } + } + return false; + } + + public bool IsILImage { get { return ComDataDirectory.VirtualAddress != 0; } } + + /// + /// The COM data directory. In practice this is the metadata of an IL image. + /// + public ImageDataDirectory ComDataDirectory { get { return ImageDataDirectory[(int)ImageDirectoryEntry.ComDescriptor]; } } + + /// + /// Returns the address of a module export symbol if found + /// + /// symbol name (without the module name prepended) + /// symbol offset returned + /// true if found + public bool TryGetExportSymbol(string symbolName, out ulong offset) + { + try + { + ImageExportDirectory exportDirectory = _exportDirectory.Value; + if (exportDirectory is not null) + { + for (int nameIndex = 0; nameIndex < exportDirectory.NumberOfNames; nameIndex++) + { + uint namePointerRVA = RelativeVirtualAddressReader.Read((ulong)(exportDirectory.AddressOfNames + (sizeof(uint) * nameIndex))); + if (namePointerRVA != 0) + { + string name = RelativeVirtualAddressReader.Read(namePointerRVA); + if (name == symbolName) + { + ushort ordinalForNamedExport = RelativeVirtualAddressReader.Read((ulong)(exportDirectory.AddressOfNameOrdinals + (sizeof(ushort) * nameIndex))); + offset = RelativeVirtualAddressReader.Read((ulong)(exportDirectory.AddressOfFunctions + (sizeof(uint) * ordinalForNamedExport))); + return true; + } + } + } + } + } + catch (Exception ex) when (ex is InvalidVirtualAddressException || ex is BadInputFormatException) + { + } + offset = 0; + return false; + } + + private ImageExportDirectory ReadExportDirectory() + { + if (IsValid()) + { + ImageDataDirectory exportTableDirectory = ImageDataDirectory[(int)ImageDirectoryEntry.Export]; + if (exportTableDirectory is not null) + { + return RelativeVirtualAddressReader.Read(exportTableDirectory.VirtualAddress); + } + } + return null; + } + + private uint ReadPEHeaderOffset() + { + HasValidDosSignature.CheckThrowing(); + return _headerReader.Read(PESignatureOffsetLocation); + } + + private uint PEOptionalHeaderOffset + { + get { return _headerReader.SizeOf() + PEHeaderOffset + 0x4; } + } + + public uint PEChecksumOffset + { + get { return PEOptionalHeaderOffset + 0x40; } + } + + public uint CertificateTableOffset + { + get { return PEOptionalHeaderOffset + FileReader.SizeOf() + 0x20; } + } + + private ImageFileHeader ReadFileHeader() + { + HasValidPESignature.CheckThrowing(); + return _headerReader.Read(PEHeaderOffset + 0x4); + } + + private ImageOptionalHeaderMagic ReadOptionalHeaderMagic() + { + ulong offset = PEOptionalHeaderOffset; + return _headerReader.Read(offset); + } + + private Reader CreateFileReader() + { + OptionalHeaderMagic.IsMagicValid.CheckThrowing(); + bool is64Bit = OptionalHeaderMagic.Magic == ImageMagic.Magic64; + return new Reader(_headerReader.DataSource, new LayoutManager().AddPETypes(is64Bit)); + } + + private ImageOptionalHeader ReadOptionalHeader() + { + ulong offset = PEOptionalHeaderOffset; + return FileReader.Read(offset); + } + + private List ReadImageDataDirectory() + { + ulong offset = PEOptionalHeaderOffset + FileReader.SizeOf(); + + ImageDataDirectory[] result = _headerReader.ReadArray(offset, ImageDataDirectoryCount); + return new List(result); + } + + private List ReadSectionHeaders() + { + ulong offset = PEOptionalHeaderOffset + FileHeader.SizeOfOptionalHeader; + List result = new(_headerReader.ReadArray(offset, FileHeader.NumberOfSections)); + return result; + } + + private IEnumerable ReadPdbInfo() + { + ImageDataDirectory imageDebugDirectory = ImageDataDirectory[(int)ImageDirectoryEntry.Debug]; + uint count = imageDebugDirectory.Size / FileReader.SizeOf(); + ImageDebugDirectory[] debugDirectories = RelativeVirtualAddressReader.ReadArray(imageDebugDirectory.VirtualAddress, count); + + foreach (ImageDebugDirectory directory in debugDirectories) + { + if (directory.Type == ImageDebugType.Codeview) + { + ulong position = directory.AddressOfRawData; + CvInfoPdb70 pdb = RelativeVirtualAddressReader.Read(ref position); + if (pdb.CvSignature == CvInfoPdb70.PDB70CvSignature) + { + bool isPortablePDB = directory.MinorVersion == ImageDebugDirectory.PortablePDBMinorVersion; + string fileName = RelativeVirtualAddressReader.Read(position); + yield return new PEPdbRecord(isPortablePDB, fileName, new Guid(pdb.Signature), pdb.Age); + } + } + } + } + + private IEnumerable ReadPdbChecksum() + { + ImageDataDirectory imageDebugDirectory = ImageDataDirectory[(int)ImageDirectoryEntry.Debug]; + uint count = imageDebugDirectory.Size / FileReader.SizeOf(); + ImageDebugDirectory[] debugDirectories = RelativeVirtualAddressReader.ReadArray(imageDebugDirectory.VirtualAddress, count); + + foreach (ImageDebugDirectory directory in debugDirectories) + { + if (directory.Type == ImageDebugType.PdbChecksum) + { + uint sizeOfData = directory.SizeOfData; + ulong position = directory.AddressOfRawData; + string algorithmName = RelativeVirtualAddressReader.Read(position); + uint algorithmLength = (uint)algorithmName.Length; + uint length = sizeOfData - algorithmLength - 1; // -1 for null terminator + byte[] checksum = RelativeVirtualAddressReader.ReadArray(position + algorithmLength + 1 /* +1 for null terminator */, length); + yield return new PdbChecksum(algorithmName, checksum); + } + } + } + + private IEnumerable ReadPerfMapV1Entries() + { + ImageDataDirectory imageDebugDirectory = ImageDataDirectory[(int)ImageDirectoryEntry.Debug]; + uint count = imageDebugDirectory.Size / FileReader.SizeOf(); + ImageDebugDirectory[] debugDirectories = RelativeVirtualAddressReader.ReadArray(imageDebugDirectory.VirtualAddress, count); + + foreach (ImageDebugDirectory directory in debugDirectories) + { + if (directory.Type == ImageDebugType.PerfMap && directory.MajorVersion == 1 && directory.MinorVersion == 0) + { + ulong position = directory.AddressOfRawData; + PerfMapIdV1 perfmapEntryHeader = RelativeVirtualAddressReader.Read(ref position); + if (perfmapEntryHeader.Magic == PerfMapIdV1.PerfMapEntryMagic) + { + string fileName = RelativeVirtualAddressReader.Read(position); + yield return new PEPerfMapRecord(fileName, perfmapEntryHeader.Signature, perfmapEntryHeader.Version); + } + } + } + } + + private const uint VersionResourceType = 16; + private const uint VersionResourceName = 1; + private const uint VersionResourceLanguage = 0x409; + + private VsFixedFileInfo ReadVersionResource() + { + ImageResourceDataEntry dataEntry = GetResourceDataEntry(VersionResourceType, VersionResourceName, VersionResourceLanguage); + // If the version resource can't be found under the 0x409 language, try as language "neutral" (0) + dataEntry ??= GetResourceDataEntry(VersionResourceType, VersionResourceName, 0); + if (dataEntry != null) + { + VsVersionInfo info = RelativeVirtualAddressReader.Read(dataEntry.OffsetToData); + if (info.Value.Signature == VsFixedFileInfo.FixedFileInfoSignature) + { + return info.Value; + } + } + return null; + } + + private ImageResourceDataEntry GetResourceDataEntry(uint type, uint name, uint language) + { + uint resourceSectionRva = ImageDataDirectory[(int)ImageDirectoryEntry.Resource].VirtualAddress; + ImageResourceDirectory resourceDirectory = RelativeVirtualAddressReader.Read(resourceSectionRva); + + if (GetNextLevelResourceEntryRva(resourceDirectory, type, resourceSectionRva, out uint nameTableRva)) + { + if (GetNextLevelResourceEntryRva(resourceDirectory, name, resourceSectionRva + nameTableRva, out uint langTableRva)) + { + if (GetNextLevelResourceEntryRva(resourceDirectory, language, resourceSectionRva + langTableRva, out uint resourceDataEntryRva)) + { + return RelativeVirtualAddressReader.Read(resourceSectionRva + resourceDataEntryRva); + } + } + } + return null; + } + + private bool GetNextLevelResourceEntryRva(ImageResourceDirectory resourceDirectory, uint id, uint rva, out uint nextLevelRva) + { + ushort numNameEntries = resourceDirectory.NumberOfNamedEntries; + ushort numIDEntries = resourceDirectory.NumberOfIdEntries; + + if (numNameEntries == ushort.MaxValue) + { + numNameEntries = 0; + } + + if (numIDEntries == ushort.MaxValue) + { + numIDEntries = 0; + } + + uint directorySize = RelativeVirtualAddressReader.SizeOf(); + uint entrySize = RelativeVirtualAddressReader.SizeOf(); + + for (ushort i = numNameEntries; i < numNameEntries + numIDEntries; i++) + { + ImageResourceDirectoryEntry entry = RelativeVirtualAddressReader.Read(rva + directorySize + (i * entrySize)); + if (entry.Id == id) + { + nextLevelRva = entry.OffsetToData & 0x7FFFFFFF; + return true; + } + } + + nextLevelRva = 0; + return false; + } + + private Reader CreateVirtualAddressReader() + { + if (_isDataSourceVirtualAddressSpace) + { + return _fileReader.Value; + } + else + { + return _fileReader.Value.WithAddressSpace(new PEAddressSpace(_headerReader.DataSource, 0, Segments)); + } + } + + #region Validation Rules + public ValidationRule HasValidDosSignature + { + get + { + return new ValidationRule("PE file does not have valid DOS header", () => + DosHeaderMagic == ExpectedDosHeaderMagic); + } + } + + public ValidationRule HasValidPESignature + { + get + { + return new ValidationRule("PE file does not have a valid PE signature", () => + PESignature == ExpectedPESignature); + } + } + #endregion + } + + public class PEAddressSpace : IAddressSpace + { + private Lazy _length; + private ReadOnlyCollection _segments; + private ulong _baseAddress; + private IAddressSpace _addressSpace; + + public ulong Length + { + get + { + return _length.Value; + } + } + + public PEAddressSpace(IAddressSpace addressSpace, ulong baseAddress, ReadOnlyCollection segments) + { + _length = new Lazy(GetLength); + _segments = segments; + _baseAddress = baseAddress; + _addressSpace = addressSpace; + } + + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + ImageSectionHeader segment = _segments.Where(header => header.VirtualAddress <= position && position <= header.VirtualAddress + header.VirtualSize).FirstOrDefault(); + if (segment == null) + { + return 0; + } + ulong offset = _baseAddress + position - segment.VirtualAddress + segment.PointerToRawData; + uint result = _addressSpace.Read(offset, buffer, bufferOffset, count); + return result; + } + + private ulong GetLength() + { + return _segments.Max(seg => seg.VirtualAddress + seg.VirtualSize); + } + } +} diff --git a/src/Microsoft.FileFormats/PE/PEPdbRecord.cs b/src/Microsoft.FileFormats/PE/PEPdbRecord.cs new file mode 100644 index 000000000..de7bdfbd7 --- /dev/null +++ b/src/Microsoft.FileFormats/PE/PEPdbRecord.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +using System; + +namespace Microsoft.FileFormats.PE +{ + public sealed class PEPdbRecord + { + public bool IsPortablePDB { get; private set; } + public string Path { get; private set; } + public Guid Signature { get; private set; } + public int Age { get; private set; } + + public PEPdbRecord(bool isPortablePDB, string path, Guid sig, int age) + { + IsPortablePDB = isPortablePDB; + Path = path; + Signature = sig; + Age = age; + } + } +} diff --git a/src/Microsoft.FileFormats/PE/PEPerfMapRecord.cs b/src/Microsoft.FileFormats/PE/PEPerfMapRecord.cs new file mode 100644 index 000000000..953d99548 --- /dev/null +++ b/src/Microsoft.FileFormats/PE/PEPerfMapRecord.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +using System; + +namespace Microsoft.FileFormats.PE +{ + public sealed class PEPerfMapRecord + { + public string Path { get; private set; } + public byte[] Signature { get; private set; } + public uint Version { get; private set; } + + public PEPerfMapRecord(string path, byte[] sig, uint version) + { + Path = path; + Signature = sig; + Version = version; + } + } +} diff --git a/src/Microsoft.FileFormats/PE/PEStructures.cs b/src/Microsoft.FileFormats/PE/PEStructures.cs new file mode 100644 index 000000000..087831d91 --- /dev/null +++ b/src/Microsoft.FileFormats/PE/PEStructures.cs @@ -0,0 +1,392 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq; + +namespace Microsoft.FileFormats.PE +{ + public static class LayoutManagerExtensions + { + public static LayoutManager AddPETypes(this LayoutManager layouts, bool is64Bit) + { + return layouts + .AddPrimitives(false) + .AddEnumTypes() + .AddSizeT(is64Bit ? 8 : 4) + .AddNullTerminatedString() + .AddTStructTypes(is64Bit ? new string[] { "PE32+" } : new string[] { "PE32" }); + } + } + + /// + /// IMAGE_NT_OPTIONAL_HDR32_MAGIC/IMAGE_NT_OPTIONAL_HDR64_MAGIC values + /// + public enum ImageMagic : ushort + { + Magic32 = 0x10b, + Magic64 = 0x20b + } + + /// + /// IMAGE_OPTIONAL_HEADER + /// + public class ImageOptionalHeaderMagic : TStruct + { + public ImageMagic Magic; + + #region Validation Rules + public ValidationRule IsMagicValid + { + get + { + return new ValidationRule("PE Optional Header has invalid magic field", () => Enum.IsDefined(typeof(ImageMagic), Magic)); + } + } + #endregion + } + + /// + /// IMAGE_OPTIONAL_HEADER + /// + public class ImageOptionalHeader : ImageOptionalHeaderMagic + { + // Standard fields + public byte MajorLinkerVersion; + public byte MinorLinkerVersion; + public uint SizeOfCode; + public uint SizeOfInitializedData; + public uint SizeOfUninitializedData; + public uint RVAOfEntryPoint; + public uint BaseOfCode; + [If("PE32")] + public uint BaseOfData; + + // NT additional fields + public SizeT ImageBase; + public uint SectionAlignment; + public uint FileAlignment; + public ushort MajorOperatingSystemVersion; + public ushort MinorOperatingSystemVersion; + public ushort MajorImageVersion; + public ushort MinorImageVersion; + public ushort MajorSubsystemVersion; + public ushort MinorSubsystemVersion; + public uint Win32VersionValue; + public uint SizeOfImage; + public uint SizeOfHeaders; + public uint CheckSum; + public ushort Subsystem; + public ushort DllCharacteristics; + public SizeT SizeOfStackReserve; + public SizeT SizeOfStackCommit; + public SizeT SizeOfHeapReserve; + public SizeT SizeOfHeapCommit; + public uint LoaderFlags; + public uint NumberOfRvaAndSizes; + } + + /// + /// IMAGE_FILE_MACHINE_* values for ImageFileHeader.Machine + /// + public enum ImageFileMachine + { + Unknown = 0, + Amd64 = 0x8664, // AMD64 (K8) + I386 = 0x014c, // Intel 386. + Arm = 0x01c0, // ARM Little-Endian + Thumb = 0x01c2, + ArmNT = 0x01c4, // ARM Thumb-2 Little-Endian + Arm64 = 0xAA64 + } + + /// + /// Characteristics (IMAGE_FILE) + /// + [Flags] + public enum ImageFile : ushort + { + RelocsStripped = 0x0001, + ExecutableImage = 0x0002, + LargeAddressAware = 0x0020, + System = 0x1000, + Dll = 0x2000, + } + + /// + /// IMAGE_FILE_HEADER struct + /// + public class ImageFileHeader : TStruct + { + public ushort Machine; + public ushort NumberOfSections; + public uint TimeDateStamp; + public uint PointerToSymbolTable; + public uint NumberOfSymbols; + public ushort SizeOfOptionalHeader; + public ushort Characteristics; + } + + #region Section Header + + /// + /// IMAGE_SECTION_HEADER + /// + public class ImageSectionHeader : TStruct + { + [ArraySize(8)] + public byte[] Name; + public uint VirtualSize; + public uint VirtualAddress; + public uint SizeOfRawData; + public uint PointerToRawData; + public uint PointerToRelocations; + public uint PointerToLinenumbers; + public ushort NumberOfRelocations; + public ushort NumberOfLinenumbers; + public uint Characteristics; + } + + #endregion + + #region Directories + + /// + /// IMAGE_DIRECTORY_ENTRY_* defines + /// + public enum ImageDirectoryEntry + { + Export = 0, + Import = 1, + Resource = 2, + Exception = 3, + Certificates = 4, + BaseRelocation = 5, + Debug = 6, + Architecture = 7, + GlobalPointers = 8, + ThreadStorage = 9, + LoadConfiguration = 10, + BoundImport = 11, + ImportAddress = 12, + DelayImport = 13, + ComDescriptor = 14 + } + + /// + /// IMAGE_DATA_DIRECTORY struct + /// + public class ImageDataDirectory : TStruct + { + public uint VirtualAddress; + public uint Size; + } + + #endregion + + #region Debug Directory + + /// + /// IMAGE_DEBUG_TYPE_* defines + /// + public enum ImageDebugType + { + Unknown = 0, + Coff = 1, + Codeview = 2, + Fpo = 3, + Misc = 4, + Bbt = 10, + Reproducible = 16, + EmbeddedPortablePdb = 17, + PdbChecksum = 19, + PerfMap = 21 + }; + + /// + /// IMAGE_DEBUG_DIRECTORY struct + /// + public class ImageDebugDirectory : TStruct + { + public const ushort PortablePDBMinorVersion = 0x504d; + + public uint Characteristics; + public uint TimeDateStamp; + public ushort MajorVersion; + public ushort MinorVersion; + public ImageDebugType Type; + public uint SizeOfData; + public uint AddressOfRawData; + public uint PointerToRawData; + }; + + public class CvInfoPdb70 : TStruct + { + public const int PDB70CvSignature = 0x53445352; // RSDS in ascii + + public int CvSignature; + [ArraySize(16)] + public byte[] Signature; + public int Age; + } + + public class PerfMapIdV1 : TStruct + { + public const int PerfMapEntryMagic = 0x4D523252; // R2RM in ascii + + public int Magic; + + [ArraySize(16)] + public byte[] Signature; + public uint Version; + } + + #endregion + + #region Resource Directory + + /// + /// IMAGE_RESOURCE_DIRECTORY struct + /// + public class ImageResourceDirectory : TStruct + { + public uint Characteristics; + public uint TimeDateStamp; + public ushort MajorVersion; + public ushort MinorVersion; + public ushort NumberOfNamedEntries; + public ushort NumberOfIdEntries; + }; + + /// + /// IMAGE_RESOURCE_DIRECTORY_ENTRY for the resources by id + /// + public class ImageResourceDirectoryEntry : TStruct + { + // Resource id or name offset. Currently doesn't supported named entries/resources. + public uint Id; + + // High bit 0. Address of a Resource Data entry (a leaf). + // High bit 1. The lower 31 bits are the address of another resource directory table (the next level down). + public uint OffsetToData; + } + + /// + /// IMAGE_RESOURCE_DATA_ENTRY struct + /// + public class ImageResourceDataEntry : TStruct + { + public uint OffsetToData; + public uint Size; + public uint CodePage; + public uint Reserved; + } + + /// + /// VS_FIXEDFILEINFO.FileFlags + /// + [Flags] + public enum FileInfoFlags : uint + { + Debug = 0x00000001, + SpecialBuild = 0x00000020, + } + + /// + /// VS_FIXEDFILEINFO struct + /// + public class VsFixedFileInfo : TStruct + { + public const uint FixedFileInfoSignature = 0xFEEF04BD; + + public uint Signature; // e.g. 0xfeef04bd + public uint StrucVersion; // e.g. 0x00000042 = "0.42" + public ushort FileVersionMinor; + public ushort FileVersionMajor; + public ushort FileVersionRevision; + public ushort FileVersionBuild; + public ushort ProductVersionMinor; + public ushort ProductVersionMajor; + public ushort ProductVersionRevision; + public ushort ProductVersionBuild; + public uint FileFlagsMask; // = 0x3F for version "0.42" + public FileInfoFlags FileFlags; + public uint FileOS; // e.g. VOS_DOS_WINDOWS16 + public uint FileType; // e.g. VFT_DRIVER + public uint FileSubtype; // e.g. VFT2_DRV_KEYBOARD + public uint FileDateMS; // e.g. 0 + public uint FileDateLS; // e.g. 0 + } + + /// + /// VS_VERSIONINFO struct + /// + public class VsVersionInfo : TStruct + { + public ushort Length; + public ushort ValueLength; + public ushort Type; + [ArraySize(16)] + public char[] Key; + public ushort Padding1; + public VsFixedFileInfo Value; + } + + #endregion + + #region IMAGE_EXPORT_DIRECTORY + + /// + /// IMAGE_EXPORT_DIRECTORY struct + /// + public class ImageExportDirectory : TStruct + { + public uint Characteristics; + public uint TimeDateStamp; + public ushort MajorVersion; + public ushort MinorVersion; + public uint Name; + public uint Base; + public uint NumberOfFunctions; + public uint NumberOfNames; + public uint AddressOfFunctions; // RVA from base of image + public uint AddressOfNames; // RVA from base of image + public uint AddressOfNameOrdinals; // RVA from base of image + }; + + #endregion + + /// + /// Pair of a checksum algorithm name (ex: "SHA256") and the bytes of the checksum. + /// + public class PdbChecksum : TStruct + { + public PdbChecksum(string algorithmName, byte[] checksum) + { + AlgorithmName = algorithmName; + Checksum = checksum; + } + + public string AlgorithmName { get; } + public byte[] Checksum { get; } + + public override string ToString() + { + return $"{AlgorithmName}:{ToHexString(Checksum)}"; + } + + /// + /// Convert an array of bytes to a lower case hex string. + /// + /// array of bytes + /// hex string + public static string ToHexString(byte[] bytes) + { + if (bytes == null) + { + throw new ArgumentNullException(nameof(bytes)); + } + return string.Concat(bytes.Select(b => b.ToString("x2"))); + } + } +} diff --git a/src/Microsoft.FileFormats/PerfMap/PerfMapFile.cs b/src/Microsoft.FileFormats/PerfMap/PerfMapFile.cs new file mode 100644 index 000000000..1663d1317 --- /dev/null +++ b/src/Microsoft.FileFormats/PerfMap/PerfMapFile.cs @@ -0,0 +1,225 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace Microsoft.FileFormats.PerfMap +{ + public sealed class PerfMapFile + { + // See format of the perfmap file at https://github.com/dotnet/runtime/blob/main/docs/design/coreclr/botr/r2r-perfmap-format.md + private const int PerfMapV1SigLength = 16; + + private const int PerfMapV1HeaderRecordCount = 5; + + public const int MaxKnownPerfMapVersion = 1; + + private const int HeaderRecordPseudoLength = 0; + + private enum PerfMapPseudoRVAToken : uint + { + OutputSignature = 0xFFFFFFFF, + FormatVersion = 0xFFFFFFFE, + TargetOS = 0xFFFFFFFD, + TargetArchitecture = 0xFFFFFFFC, + TargetABI = 0xFFFFFFFB, + } + + public enum PerfMapArchitectureToken : uint + { + Unknown = 0, + ARM = 1, + ARM64 = 2, + X64 = 3, + X86 = 4, + } + + public enum PerfMapOSToken : uint + { + Unknown = 0, + Windows = 1, + Linux = 2, + OSX = 3, + FreeBSD = 4, + NetBSD = 5, + SunOS = 6, + } + + public enum PerfMapAbiToken : uint + { + Unknown = 0, + Default = 1, + Armel = 2, + } + + private readonly Stream _stream; + private readonly Lazy _header; + + public PerfMapHeader Header { get => _header.Value; } + + public bool IsValid => Header is not null; + + public IEnumerable PerfRecords + { + get + { + ThrowIfInvalid(); + + if (Header.Version > MaxKnownPerfMapVersion) + { + throw new NotImplementedException($"Format version {Header.Version} unknown. Max known format is {MaxKnownPerfMapVersion}"); + } + using StreamReader reader = new(_stream, Encoding.UTF8, false, 1024, leaveOpen: true); + + // Skip over the header. + // For now this is V1, the length will need to be a lookup on the version. + for (int i = 0; i < PerfMapV1HeaderRecordCount; ++i) + { + _ = reader.ReadLine(); + } + while (true) + { + PerfMapFile.PerfMapRecord cur = ReadRecord(reader); + if (cur is null) + { + yield break; + } + yield return cur; + } + } + } + + private void ThrowIfInvalid() + { + if (!IsValid) + { + throw new BadInputFormatException("The PerfMap is not valid"); + } + } + + public PerfMapFile(Stream stream) + { + System.Diagnostics.Debug.Assert(stream.CanSeek); + _stream = stream; + _header = new Lazy(ReadHeader, System.Threading.LazyThreadSafetyMode.ExecutionAndPublication); + } + + private PerfMapHeader ReadHeader() + { + static bool IsValidHeaderRecord(PerfMapPseudoRVAToken expectedToken, PerfMapRecord record) + => record is not null && (uint)expectedToken == record.Rva + && record.Length == HeaderRecordPseudoLength; + + long prevPosition = _stream.Position; + try + { + _stream.Position = 0; + // Headers don't need much of a buffer. + using StreamReader reader = new(_stream, Encoding.UTF8, false, 256, leaveOpen: true); + + PerfMapRecord sigRecord = ReadRecord(reader); + if (!IsValidHeaderRecord(PerfMapPseudoRVAToken.OutputSignature, sigRecord) || + !Helpers.TryConvertHexStringToBytes(sigRecord.Name, out byte[] sigBytes) || + sigBytes?.Length != PerfMapV1SigLength) + { + return null; + } + PerfMapRecord versionRecord = ReadRecord(reader); + if (!IsValidHeaderRecord(PerfMapPseudoRVAToken.FormatVersion, versionRecord) || + !uint.TryParse(versionRecord.Name, out uint version)) + { + return null; + } + PerfMapRecord osRecord = ReadRecord(reader); + if (!IsValidHeaderRecord(PerfMapPseudoRVAToken.TargetOS, osRecord) || + !uint.TryParse(osRecord.Name, out uint os)) + { + return null; + } + PerfMapRecord archRecord = ReadRecord(reader); + if (!IsValidHeaderRecord(PerfMapPseudoRVAToken.TargetArchitecture, archRecord) || + !uint.TryParse(archRecord.Name, out uint arch)) + { + return null; + } + PerfMapRecord abiRecord = ReadRecord(reader); + if (!IsValidHeaderRecord(PerfMapPseudoRVAToken.TargetABI, abiRecord) || + !uint.TryParse(abiRecord.Name, out uint abi)) + { + return null; + } + return new PerfMapHeader(sigBytes, version, os, arch, abi); + // Append as necessary as revisions get added here. + // We don't return null on a higher versioned heder than the max known as they are backwards compatible and they are not necessary for indexing. + } + catch (Exception ex) when (ex is BadInputFormatException || ex is EndOfStreamException) + { + + } + finally + { + _stream.Position = prevPosition; + } + return null; + } + + private static PerfMapRecord ReadRecord(StreamReader reader) + { + string[] segments = reader.ReadLine()?.Split(); + + if (segments is null) + { + return null; + } + if (segments.Length != 3) + { + throw new BadInputFormatException("Entry on perfmap record doesn't have 3 segments."); + } + if (!uint.TryParse(segments[0], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out uint rva)) + { + throw new BadInputFormatException("Record's RVA is not a valid hex unsigned int."); + } + if (!ushort.TryParse(segments[1], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out ushort length)) + { + throw new BadInputFormatException("Record's Length is not a valid hex unsigned int."); + } + return new PerfMapRecord(rva, length, segments[2]); + } + + public sealed class PerfMapRecord + { + public PerfMapRecord(uint rva, ushort length, string entryName) + { + Rva = rva; + Length = length; + Name = entryName; + } + + public uint Rva { get; } + public ushort Length { get; } + public string Name { get; } + } + + public sealed class PerfMapHeader + { + public PerfMapHeader(byte[] signature, uint version, uint operatingSystem, uint architecture, uint abi) + { + Signature = signature; + Version = version; + OperatingSystem = (PerfMapOSToken) operatingSystem; + Architecture = (PerfMapArchitectureToken) architecture; + Abi = (PerfMapAbiToken) abi; + } + + public byte[] Signature { get; } + public uint Version { get; } + public PerfMapOSToken OperatingSystem { get; } + public PerfMapArchitectureToken Architecture { get; } + public PerfMapAbiToken Abi { get; } + } + } +} diff --git a/src/Microsoft.FileFormats/Pointer.cs b/src/Microsoft.FileFormats/Pointer.cs new file mode 100644 index 000000000..cdb68ce9b --- /dev/null +++ b/src/Microsoft.FileFormats/Pointer.cs @@ -0,0 +1,223 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// A pointer layout that can create pointers from integral storage types + /// + public class PointerLayout : LayoutBase + { + protected readonly ILayout _storageLayout; + private readonly LayoutManager _layoutManager; + private readonly Type _targetType; + private ILayout _targetLayout; + + public ILayout TargetLayout => _targetLayout ??= _layoutManager.GetLayout(_targetType); + + public PointerLayout(LayoutManager layoutManager, Type pointerType, ILayout storageLayout, Type targetType) : + base(pointerType, storageLayout.Size, storageLayout.NaturalAlignment) + { + _layoutManager = layoutManager; + _storageLayout = storageLayout; + _targetType = targetType; + } + } + + /// + /// A pointer layout that can create pointers from the System.UInt64 storage type + /// + public class UInt64PointerLayout : PointerLayout + { + public UInt64PointerLayout(LayoutManager layoutManager, Type pointerType, ILayout storageLayout, Type targetType) : + base(layoutManager, pointerType, storageLayout, targetType) + { + if (storageLayout.Type != typeof(ulong)) + { + throw new ArgumentException("storageLayout must have System.UInt64 type"); + } + } + + public override object Read(IAddressSpace dataSource, ulong position) + { + ulong value = (ulong)_storageLayout.Read(dataSource, position); + Pointer p = (Pointer)Activator.CreateInstance(Type); + p.Init(TargetLayout, value); + return p; + } + } + + /// + /// A pointer layout that can create pointers from the System.UInt32 storage type + /// + public class UInt32PointerLayout : PointerLayout + { + public UInt32PointerLayout(LayoutManager layoutManager, Type pointerType, ILayout storageLayout, Type targetType) : + base(layoutManager, pointerType, storageLayout, targetType) + { + if (storageLayout.Type != typeof(uint)) + { + throw new ArgumentException("storageLayout must have System.UInt32 type"); + } + } + + public override object Read(IAddressSpace dataSource, ulong position) + { + ulong value = (uint)_storageLayout.Read(dataSource, position); + Pointer p = (Pointer)Activator.CreateInstance(Type); + p.Init(TargetLayout, value); + return p; + } + } + + /// + /// A pointer layout that can create pointers from the SizeT storage type + /// + public class SizeTPointerLayout : PointerLayout + { + public SizeTPointerLayout(LayoutManager layoutManager, Type pointerType, ILayout storageLayout, Type targetType) : + base(layoutManager, pointerType, storageLayout, targetType) + { + if (storageLayout.Type != typeof(SizeT)) + { + throw new ArgumentException("storageLayout must have SizeT type"); + } + } + + public override object Read(IAddressSpace dataSource, ulong position) + { + ulong value = (SizeT)_storageLayout.Read(dataSource, position); + Pointer p = (Pointer)Activator.CreateInstance(Type); + p.Init(TargetLayout, value); + return p; + } + } + + public class Pointer + { + public ulong Value; + public bool IsNull + { + get { return Value == 0; } + } + + public override string ToString() + { + return "0x" + Value.ToString("x"); + } + + public static implicit operator ulong (Pointer instance) + { + return instance.Value; + } + + internal void Init(ILayout targetLayout, ulong value) + { + _targetLayout = targetLayout; + Value = value; + } + + protected ILayout _targetLayout; + } + + /// + /// A pointer that can be dereferenced to produce another object + /// + /// The type of object that is produced by dereferencing the pointer + /// The type that determines how the pointer's underlying address value is parsed + public class Pointer : Pointer + { + /// + /// Read an object of _TargetType_ from the _addressSpace_ + /// + public TargetType Dereference(IAddressSpace addressSpace) + { + return Element(addressSpace, 0); + } + + /// + /// Read the array element _index_ from an array in _addressSpace_ + /// + public TargetType Element(IAddressSpace addressSpace, uint index) + { + if (Value != 0) + { + return (TargetType)_targetLayout.Read(addressSpace, Value + index * _targetLayout.Size); + } + return default; + } + } + + public static partial class LayoutManagerExtensions + { + /// + /// Adds support for reading types derived from Pointer<,> + /// + public static LayoutManager AddPointerTypes(this LayoutManager layouts) + { + layouts.AddLayoutProvider(GetPointerLayout); + return layouts; + } + + private static ILayout GetPointerLayout(Type pointerType, LayoutManager layoutManager) + { + if (!typeof(Pointer).GetTypeInfo().IsAssignableFrom(pointerType)) + { + return null; + } + Type curPointerType = pointerType; + TypeInfo genericPointerTypeInfo = null; + while (curPointerType != typeof(Pointer)) + { + TypeInfo curPointerTypeInfo = curPointerType.GetTypeInfo(); + if (curPointerTypeInfo.IsGenericType && curPointerTypeInfo.GetGenericTypeDefinition() == typeof(Pointer<,>)) + { + genericPointerTypeInfo = curPointerTypeInfo; + break; + } + curPointerType = curPointerTypeInfo.BaseType; + } + if (genericPointerTypeInfo == null) + { + throw new LayoutException("Pointer types must be derived from Pointer<,,>"); + } + Type targetType = genericPointerTypeInfo.GetGenericArguments()[0]; + Type storageType = genericPointerTypeInfo.GetGenericArguments()[1]; + ILayout storageLayout = layoutManager.GetLayout(storageType); + + // Unfortunately the storageLayout.Read returns a boxed object that can't be + // casted to a ulong without first being unboxed. These three Pointer layout + // types are identical other than unboxing to a different type. Generics + // doesn't work, there is no constraint that ensures the type parameter defines + // a casting operator to ulong. Specifying a Func parameter + // would work, but I opted to write each class separately so that we don't + // pay the cost of an extra delegate invocation for each pointer read. It + // may be premature optimization, but the complexity of it should be relatively + // constrained within this file at least. + + if (storageLayout.Type == typeof(SizeT)) + { + return new SizeTPointerLayout(layoutManager, pointerType, storageLayout, targetType); + } + else if (storageLayout.Type == typeof(ulong)) + { + return new UInt64PointerLayout(layoutManager, pointerType, storageLayout, targetType); + } + else if (storageLayout.Type == typeof(uint)) + { + return new UInt32PointerLayout(layoutManager, pointerType, storageLayout, targetType); + } + else + { + throw new LayoutException("Pointer types must have a storage type of SizeT, ulong, or uint"); + } + } + } +} diff --git a/src/Microsoft.FileFormats/PrimitiveTypes.cs b/src/Microsoft.FileFormats/PrimitiveTypes.cs new file mode 100644 index 000000000..bd49e1991 --- /dev/null +++ b/src/Microsoft.FileFormats/PrimitiveTypes.cs @@ -0,0 +1,263 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + public abstract class PrimitiveTypeLayout : LayoutBase + { + public PrimitiveTypeLayout(Type type, bool isBigEndian, uint size) : base(type, size) + { + IsBigEndian = isBigEndian; + } + public bool IsBigEndian { get; private set; } + } + + /// + /// TypeParser for System.Int8 + /// + public class BoolLayout : PrimitiveTypeLayout + { + public BoolLayout(Type type, bool isBigEndian) : base(type, isBigEndian, 1) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 1); + return buffer[0] != 0; + } + } + + /// + /// TypeParser for System.Int8 + /// + public class Int8Layout : PrimitiveTypeLayout + { + public Int8Layout(bool isBigEndian) : this(typeof(sbyte), isBigEndian) { } + public Int8Layout(Type type, bool isBigEndian) : base(type, isBigEndian, 1) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 1); + return unchecked((sbyte)buffer[0]); + } + } + + /// + /// TypeParser for System.UInt8 + /// + public class UInt8Layout : PrimitiveTypeLayout + { + public UInt8Layout(bool isBigEndian) : this(typeof(byte), isBigEndian) { } + public UInt8Layout(Type type, bool isBigEndian) : base(type, isBigEndian, 1) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 1); + return buffer[0]; + } + } + + /// + /// TypeParser for System.Char. + /// + public class CharLayout : PrimitiveTypeLayout + { + public CharLayout(bool isBigEndian) : this(typeof(char), isBigEndian) { } + public CharLayout(Type type, bool isBigEndian) : base(type, isBigEndian, 2) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 2); + if (IsBigEndian == BitConverter.IsLittleEndian) + { + Array.Reverse(buffer); + } + return BitConverter.ToChar(buffer, 0); + } + } + + /// + /// TypeParser for System.Int16 + /// + public class Int16Layout : PrimitiveTypeLayout + { + public Int16Layout(bool isBigEndian) : this(typeof(short), isBigEndian) { } + public Int16Layout(Type type, bool isBigEndian) : base(type, isBigEndian, 2) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 2); + if (IsBigEndian == BitConverter.IsLittleEndian) + { + Array.Reverse(buffer); + } + return BitConverter.ToInt16(buffer, 0); + } + } + + /// + /// TypeParser for System.UInt16 + /// + public class UInt16Layout : PrimitiveTypeLayout + { + public UInt16Layout(bool isBigEndian) : this(typeof(ushort), isBigEndian) { } + public UInt16Layout(Type type, bool isBigEndian) : base(type, isBigEndian, 2) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 2); + if (IsBigEndian == BitConverter.IsLittleEndian) + { + Array.Reverse(buffer); + } + return BitConverter.ToUInt16(buffer, 0); + } + } + + /// + /// TypeParser for System.Int32 + /// + public class Int32Layout : PrimitiveTypeLayout + { + public Int32Layout(bool isBigEndian) : this(typeof(int), isBigEndian) { } + public Int32Layout(Type type, bool isBigEndian) : base(type, isBigEndian, 4) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 4); + if (IsBigEndian == BitConverter.IsLittleEndian) + { + Array.Reverse(buffer); + } + return BitConverter.ToInt32(buffer, 0); + } + } + + /// + /// TypeParser for System.UInt32 + /// + public class UInt32Layout : PrimitiveTypeLayout + { + public UInt32Layout(bool isBigEndian) : this(typeof(uint), isBigEndian) { } + public UInt32Layout(Type type, bool isBigEndian) : base(type, isBigEndian, 4) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 4); + if (IsBigEndian == BitConverter.IsLittleEndian) + { + Array.Reverse(buffer); + } + return BitConverter.ToUInt32(buffer, 0); + } + } + + /// + /// TypeParser for System.Single + /// + public class SingleLayout : PrimitiveTypeLayout + { + public SingleLayout(bool isBigEndian) : this(typeof(float), isBigEndian) { } + public SingleLayout(Type type, bool isBigEndian) : base(type, isBigEndian, 4) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 4); + if (IsBigEndian == BitConverter.IsLittleEndian) + { + byte temp = buffer[0]; + buffer[0] = buffer[3]; + buffer[3] = temp; + temp = buffer[1]; + buffer[1] = buffer[2]; + buffer[2] = temp; + } + return BitConverter.ToSingle(buffer, 0); + } + } + + /// + /// TypeParser for System.Int64 + /// + public class Int64Layout : PrimitiveTypeLayout + { + public Int64Layout(bool isBigEndian) : this(typeof(long), isBigEndian) { } + public Int64Layout(Type type, bool isBigEndian) : base(type, isBigEndian, 8) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 8); + if (IsBigEndian == BitConverter.IsLittleEndian) + { + Array.Reverse(buffer); + } + return BitConverter.ToInt64(buffer, 0); + } + } + + /// + /// TypeParser for System.UInt64 + /// + public class UInt64Layout : PrimitiveTypeLayout + { + public UInt64Layout(bool isBigEndian) : this(typeof(ulong), isBigEndian) { } + public UInt64Layout(Type type, bool isBigEndian) : base(type, isBigEndian, 8) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 8); + if (IsBigEndian == BitConverter.IsLittleEndian) + { + Array.Reverse(buffer); + } + return BitConverter.ToUInt64(buffer, 0); + } + } + + /// + /// TypeParser for System.Double + /// + public class DoubleLayout : PrimitiveTypeLayout + { + public DoubleLayout(bool isBigEndian) : this(typeof(double), isBigEndian) { } + public DoubleLayout(Type type, bool isBigEndian) : base(type, isBigEndian, 8) { } + public override object Read(IAddressSpace dataSource, ulong position) + { + byte[] buffer = dataSource.Read(position, 4); + if (IsBigEndian == BitConverter.IsLittleEndian) + { + byte temp = buffer[0]; + buffer[0] = buffer[7]; + buffer[7] = temp; + temp = buffer[1]; + buffer[1] = buffer[6]; + buffer[6] = temp; + temp = buffer[2]; + buffer[2] = buffer[5]; + buffer[5] = temp; + temp = buffer[3]; + buffer[3] = buffer[4]; + buffer[4] = temp; + } + return BitConverter.ToDouble(buffer, 0); + } + } + + public static partial class LayoutManagerExtensions + { + /// + /// Adds supports for reading bool, sbyte, byte, char, short, ushort, int, uint, long, ulong, float, and double + /// + /// The layout manager that will hold the new layout + /// True if the primitives should be read in big endian byte order, otherwise little endian + public static LayoutManager AddPrimitives(this LayoutManager layouts, bool isBigEndian = false) + { + layouts.AddLayout(new BoolLayout(typeof(bool), isBigEndian)); + layouts.AddLayout(new Int8Layout(typeof(sbyte), isBigEndian)); + layouts.AddLayout(new UInt8Layout(typeof(byte), isBigEndian)); + layouts.AddLayout(new CharLayout(typeof(char), isBigEndian)); + layouts.AddLayout(new Int16Layout(typeof(short), isBigEndian)); + layouts.AddLayout(new UInt16Layout(typeof(ushort), isBigEndian)); + layouts.AddLayout(new Int32Layout(typeof(int), isBigEndian)); + layouts.AddLayout(new UInt32Layout(typeof(uint), isBigEndian)); + layouts.AddLayout(new Int64Layout(typeof(long), isBigEndian)); + layouts.AddLayout(new UInt64Layout(typeof(ulong), isBigEndian)); + layouts.AddLayout(new SingleLayout(typeof(float), isBigEndian)); + layouts.AddLayout(new DoubleLayout(typeof(double), isBigEndian)); + return layouts; + } + } +} diff --git a/src/Microsoft.FileFormats/Reader.cs b/src/Microsoft.FileFormats/Reader.cs new file mode 100644 index 000000000..384fa1b25 --- /dev/null +++ b/src/Microsoft.FileFormats/Reader.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + + +namespace Microsoft.FileFormats +{ + public class Reader + { + public Reader(IAddressSpace dataSource, bool isBigEndian = false) : + this(dataSource, new LayoutManager().AddPrimitives(isBigEndian).AddEnumTypes().AddTStructTypes()) + { } + + public Reader(IAddressSpace dataSource, LayoutManager layoutManager) + { + DataSource = dataSource; + LayoutManager = layoutManager; + } + + public LayoutManager LayoutManager { get; private set; } + public IAddressSpace DataSource { get; private set; } + + + public T[] ReadArray(ulong position, uint elementCount) + { + return (T[])LayoutManager.GetArrayLayout(elementCount).Read(DataSource, position); + } + + public T[] ReadArray(ref ulong position, uint elementCount) + { + uint bytesRead; + T[] ret = (T[])LayoutManager.GetArrayLayout(elementCount).Read(DataSource, position, out bytesRead); + position += bytesRead; + return ret; + } + + public bool TryRead(ulong position, out T value) + { + if (DataSource.Length > (position + SizeOf())) + { + value = Read(position); + return true; + } + value = default(T); + return false; + } + + public T Read(ulong position) + { + return (T)LayoutManager.GetLayout().Read(DataSource, position); + } + + public T Read(ref ulong position) + { + uint bytesRead; + T ret = (T)LayoutManager.GetLayout().Read(DataSource, position, out bytesRead); + position += bytesRead; + return ret; + } + + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + return DataSource.Read(position, buffer, bufferOffset, count); + } + + public byte[] Read(ulong position, uint count) + { + return DataSource.Read(position, count); + } + + public byte[] Read(ref ulong position, uint count) + { + byte[] ret = DataSource.Read(position, count); + position += count; + return ret; + } + + public ulong Length { get { return DataSource.Length; } } + + public uint SizeOf() + { + return LayoutManager.GetLayout().Size; + } + + public Reader WithRelativeAddressSpace(ulong startOffset, ulong length) + { + return WithAddressSpace(new RelativeAddressSpace(DataSource, startOffset, length)); + } + + public Reader WithRelativeAddressSpace(ulong startOffset, ulong length, long baseToRelativeShift) + { + return WithAddressSpace(new RelativeAddressSpace(DataSource, startOffset, length, baseToRelativeShift)); + } + + public Reader WithAddressSpace(IAddressSpace addressSpace) + { + return new Reader(addressSpace, LayoutManager); + } + } +} diff --git a/src/Microsoft.FileFormats/SizeT.cs b/src/Microsoft.FileFormats/SizeT.cs new file mode 100644 index 000000000..6b5e99a6a --- /dev/null +++ b/src/Microsoft.FileFormats/SizeT.cs @@ -0,0 +1,107 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// An integral type that can be configured to parse as a 4 byte uint or 8 byte ulong + /// + public struct SizeT + { + private ulong _value; + internal SizeT(ulong value) + { + _value = value; + } + + public static implicit operator ulong (SizeT instance) + { + return instance._value; + } + + public static explicit operator long (SizeT instance) + { + return (long)instance._value; + } + + public static explicit operator uint (SizeT instance) + { + return (uint)instance._value; + } + + public override string ToString() + { + return "0x" + _value.ToString("x"); + } + } + + public class UInt64SizeTLayout : LayoutBase + { + private ILayout _storageLayout; + public UInt64SizeTLayout(ILayout storageLayout) : base(typeof(SizeT), storageLayout.Size) + { + if (storageLayout.Type != typeof(ulong)) + { + throw new ArgumentException("storageLayout must be for the System.UInt64 type"); + } + _storageLayout = storageLayout; + } + + public override object Read(IAddressSpace dataSource, ulong position) + { + return new SizeT((ulong)_storageLayout.Read(dataSource, position)); + } + } + + public class UInt32SizeTLayout : LayoutBase + { + private ILayout _storageLayout; + public UInt32SizeTLayout(ILayout storageLayout) : base(typeof(SizeT), storageLayout.Size) + { + if (storageLayout.Type != typeof(uint)) + { + throw new ArgumentException("storageLayout must be for the System.UInt32 type"); + } + _storageLayout = storageLayout; + } + + public override object Read(IAddressSpace dataSource, ulong position) + { + return new SizeT((uint)_storageLayout.Read(dataSource, position)); + } + } + + public static partial class LayoutManagerExtensions + { + /// + /// Adds support for parsing the SizeT type + /// + /// The number of bytes that should be parsed for SizeT, either 4 or 8 + /// The layout manager that will hold the new layout + /// + /// SizeT reuses the existing parsing logic for either uint or ulong depending on size. The ILayoutManager + /// is expected to already have the relevant type's layout defined before calling this method. + /// + public static LayoutManager AddSizeT(this LayoutManager layouts, int size) + { + if (size == 4) + { + layouts.AddLayout(new UInt32SizeTLayout(layouts.GetLayout())); + } + else if (size == 8) + { + layouts.AddLayout(new UInt64SizeTLayout(layouts.GetLayout())); + } + else + { + throw new ArgumentException("Size must be 4 or 8"); + } + return layouts; + } + } +} diff --git a/src/Microsoft.FileFormats/StreamAddressSpace.cs b/src/Microsoft.FileFormats/StreamAddressSpace.cs new file mode 100644 index 000000000..773c411dd --- /dev/null +++ b/src/Microsoft.FileFormats/StreamAddressSpace.cs @@ -0,0 +1,59 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// Creates an address space that reads from a stream. + /// + public sealed class StreamAddressSpace : IAddressSpace, IDisposable + { + private Stream _stream; + + public StreamAddressSpace(Stream stream) + { + System.Diagnostics.Debug.Assert(stream.CanSeek); + _stream = stream; + Length = (ulong)stream.Length; + } + + /// + /// The upper bound (non-inclusive) of readable addresses + /// + public ulong Length { get; private set; } + + /// + /// Reads a range of bytes from the address space + /// + /// The position in the address space to begin reading from + /// The buffer that will receive the bytes that are read + /// The offset in the output buffer to begin writing the bytes + /// The number of bytes to read into the buffer + /// The number of bytes read + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + if (_stream is null) + { + throw new ObjectDisposedException(nameof(_stream), "StreamAddressSpace instance has been disposed"); + } + if (position + count > Length) + { + throw new BadInputFormatException("Unexpected end of data: Expected " + count + " bytes."); + } + _stream.Position = (long)position; + return (uint)_stream.Read(buffer, (int)bufferOffset, (int)count); + } + + public void Dispose() + { + _stream?.Dispose(); + _stream = null; + } + } +} diff --git a/src/Microsoft.FileFormats/TStruct.cs b/src/Microsoft.FileFormats/TStruct.cs new file mode 100644 index 000000000..4df5468a9 --- /dev/null +++ b/src/Microsoft.FileFormats/TStruct.cs @@ -0,0 +1,286 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + /// + /// A TStruct acts as a target-independent description of a structured sequence of bytes in + /// the input being parsed. + /// + /// + /// TStructs primarily declare public instance fields. The type of each field must be a type + /// for which a ILayout exists within the LayoutManager used to read it. + /// + /// Although a LayoutManager isn't required to have any particular types, it is often configured to support + /// at least these types: + /// + /// Byte/SByte/[U]Int[16/32/64] - One of the fixed-size integral types. (In particular, *not* IntPtr) + /// Enum - Any enum whose underlying type is one of the fixed-size integral type. + /// TStruct - Another TStruct. Note that this describes a *nested* struct, not a pointer + /// to another struct. + /// + /// Binding to a LayoutManager object: + /// -------------------------- + /// A TStruct is not actually parsable until a LayoutManager produces an ILayout for it. The LayoutManager + /// provides the extra information (e.g. pointer size) that permit the parser to compute the final + /// offsets and size of the TStruct fields. + /// + /// Non-instance fields: + /// -------------------- + /// TStructs can contain methods, static or private fields and even nested classes if convenient. The parsing code + /// ignores them. + /// + /// + public abstract class TStruct + { + } + + [AttributeUsage(AttributeTargets.Class)] + public class PackAttribute : Attribute + { + public uint Pack { get; private set; } + + public PackAttribute(uint pack) + { + Pack = pack; + } + } + + /// + /// TLayouts expose one of these for each field that's mapped to the input + /// + public sealed class TField : IField + { + public TField(FieldInfo fieldInfo, ILayout layout, uint offset) + { + FieldInfo = fieldInfo; + Layout = layout; + Offset = offset; + } + + public FieldInfo FieldInfo { get; private set; } + public uint Offset { get; private set; } + public uint Size { get { return Layout.Size; } } + public uint NaturalAlignment { get { return Layout.NaturalAlignment; } } + public ILayout Layout { get; private set; } + public ILayout DeclaringLayout { get; set; } + public string Name { get { return FieldInfo.Name; } } + + public object GetValue(TStruct tStruct) + { + return FieldInfo.GetValue(tStruct); + } + + public override string ToString() + { + return DeclaringLayout.Type.FullName + "." + FieldInfo.Name + " [+0x" + Offset.ToString("x") + "]"; + } + + public void SetValue(TStruct tStruct, object newValue) + { + FieldInfo.SetValue(tStruct, newValue); + } + } + + /// + /// The layout for a TStruct derived type + /// + public class TLayout : LayoutBase + { + public TLayout(Type type, uint size, uint naturalAlignment, uint sizeAsBaseType, IField[] fields) : + base(type, size, naturalAlignment, sizeAsBaseType, fields) + { } + + public override object Read(IAddressSpace dataTarget, ulong position) + { + TStruct blank = (TStruct)Activator.CreateInstance(Type); + foreach (IField field in Fields) + { + object fieldValue = field.Layout.Read(dataTarget, position + field.Offset); + field.SetValue(blank, fieldValue); + } + return blank; + } + } + + public static partial class LayoutManagerExtensions + { + /// + /// Adds support for parsing types derived from TStruct. All the field types used within the TStruct types + /// must also have layouts available from the LayoutManager. + /// + public static LayoutManager AddTStructTypes(this LayoutManager layouts) + { + return AddTStructTypes(layouts, null); + } + + /// + /// Adds support for parsing types derived from TStruct. All the field types used within the TStruct types + /// must also have layouts available from the LayoutManager. + /// + /// The layout manager that will hold the new layout + /// + /// The set of defines that can be used to enabled optional fields decorated with the IfAttribute + /// + public static LayoutManager AddTStructTypes(this LayoutManager layouts, IEnumerable enabledDefines) + { + return AddReflectionTypes(layouts, enabledDefines, typeof(TStruct)); + } + + /// + /// Adds support for parsing types derived from _requiredBaseType_ by using reflection to interpret their fields. + /// All field types used within these types must also have layouts available from the LayoutManager. + /// + /// + /// + /// The set of defines that can be used to enabled optional fields decorated with the IfAttribute + /// + /// + public static LayoutManager AddReflectionTypes(this LayoutManager layouts, IEnumerable enabledDefines, Type requiredBaseType) + { + return layouts.AddReflectionTypes(enabledDefines, typeFilter: (type) => requiredBaseType.GetTypeInfo().IsAssignableFrom(type)); + } + + /// + /// Adds support for parsing types filtered by typeFilter from by using reflection to interpret their fields. + /// All field types used within these types must also have layouts available from the LayoutManager. + /// + /// + /// + /// The set of defines that can be used to enabled optional fields decorated with the IfAttribute + /// + /// return true if reflection should be used to layout the type + public static LayoutManager AddReflectionTypes(this LayoutManager layouts, IEnumerable enabledDefines, Func typeFilter) + { + layouts.AddLayoutProvider((type, layoutManager) => + { + if (!typeFilter(type)) + { + return null; + } + return GetTStructLayout(type, layoutManager, enabledDefines); + }); + return layouts; + } + + private static ILayout GetTStructLayout(Type tStructType, LayoutManager layoutManager, IEnumerable enabledDefines) + { + enabledDefines ??= Array.Empty(); + + TypeInfo typeInfo = tStructType.GetTypeInfo(); + + PackAttribute pack = typeInfo.GetCustomAttributes().Where(attr => attr is PackAttribute).Cast().SingleOrDefault(); + + FieldInfo[] reflectionFields = typeInfo.GetFields(BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + reflectionFields = reflectionFields.OrderBy(f => f.MetadataToken).ToArray(); + reflectionFields = reflectionFields.Where(f => !f.DeclaringType.Equals(typeof(TStruct))).ToArray(); + reflectionFields = reflectionFields.Where(f => IsFieldIncludedInDefines(f, enabledDefines)).ToArray(); + TField[] tFields = new TField[reflectionFields.Length]; + + uint alignCeiling = pack?.Pack ?? 8; + uint biggestAlignmentSoFar = 1; + uint curOffset = 0; + + ILayout parentLayout = null; + Type baseType = typeInfo.BaseType; + if (!baseType.Equals(typeof(TStruct))) + { + // Treat base type as first member. + parentLayout = layoutManager.GetLayout(baseType); + uint align = Math.Min(parentLayout.NaturalAlignment, alignCeiling); + biggestAlignmentSoFar = Math.Max(biggestAlignmentSoFar, align); + curOffset += parentLayout.SizeAsBaseType; + } + + // build the field list + for (int i = 0; i < reflectionFields.Length; i++) + { + ILayout fieldLayout = GetFieldLayout(reflectionFields[i], layoutManager); + uint fieldSize = fieldLayout.Size; + uint align = fieldLayout.NaturalAlignment; + align = Math.Min(align, alignCeiling); + biggestAlignmentSoFar = Math.Max(biggestAlignmentSoFar, align); + curOffset = AlignUp(curOffset, align); + tFields[i] = new TField(reflectionFields[i], fieldLayout, curOffset); + curOffset += fieldSize; + } + curOffset = AlignUp(curOffset, biggestAlignmentSoFar); + + uint sizeAsBaseType = curOffset; + if (curOffset == 0) + { + curOffset = 1; // As with C++, zero-length struct not allowed (except as parent of another struct). + } + IField[] totalFields; + if (parentLayout != null) + { + totalFields = parentLayout.Fields.Concat(tFields).ToArray(); + } + else + { + totalFields = tFields; + } + TLayout layout = new(tStructType, curOffset, biggestAlignmentSoFar, sizeAsBaseType, totalFields); + foreach (TField field in tFields) + { + field.DeclaringLayout = layout; + } + return layout; + } + + private static bool IsFieldIncludedInDefines(FieldInfo fieldInfo, IEnumerable enabledDefines) + { + IEnumerable attrs = fieldInfo.GetCustomAttributes(); + foreach (IfAttribute attr in attrs) + { + if (!enabledDefines.Contains(attr.DefineName)) + { + return false; + } + } + return true; + } + + private static ILayout GetFieldLayout(FieldInfo fieldInfo, LayoutManager layoutManager) + { + ILayout fieldLayout; + Type fieldType = fieldInfo.FieldType; + if (fieldType.IsArray) + { + ArraySizeAttribute ca = (ArraySizeAttribute)fieldInfo.GetCustomAttributes(typeof(ArraySizeAttribute)).FirstOrDefault(); + if (ca == null) + { + throw new LayoutException("Array typed fields must use an ArraySize attribute to indicate their size"); + } + fieldLayout = layoutManager.GetArrayLayout(fieldType, ca.NumElements); + } + else + { + fieldLayout = layoutManager.GetLayout(fieldType); + } + + if (!fieldLayout.IsFixedSize) + { + throw new LayoutException(fieldInfo.Name + " is not a fixed size field. Only fixed size fields are supported in structures"); + } + + return fieldLayout; + } + + private static uint AlignUp(uint p, uint align) + { + uint remainder = (p % align); + if (remainder != 0) + { + p += (align - remainder); + } + return p; + } + } +} diff --git a/src/Microsoft.FileFormats/ValidationRule.cs b/src/Microsoft.FileFormats/ValidationRule.cs new file mode 100644 index 000000000..4b99b1363 --- /dev/null +++ b/src/Microsoft.FileFormats/ValidationRule.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.FileFormats +{ + public class ValidationRule + { + private Func _checkFunc; + private ValidationRule[] _prereqs; + + public ValidationRule(string errorMessage, Func checkFunc) : this(errorMessage, checkFunc, null) { } + + public ValidationRule(string errorMessage, Func checkFunc, params ValidationRule[] prerequisiteValidations) + { + ErrorMessage = errorMessage; + _checkFunc = checkFunc; + _prereqs = prerequisiteValidations; + } + + public string ErrorMessage { get; private set; } + + public bool CheckPrerequisites() + { + return _prereqs == null || _prereqs.All(v => v.Check()); + } + + public bool Check() + { + return CheckPrerequisites() && _checkFunc(); + } + + public void CheckThrowing() + { + if (!Check()) + { + throw new BadInputFormatException(ErrorMessage); + } + } + } +} diff --git a/src/Microsoft.SymbolStore/ChecksumValidator.cs b/src/Microsoft.SymbolStore/ChecksumValidator.cs new file mode 100644 index 000000000..f19ce2da8 --- /dev/null +++ b/src/Microsoft.SymbolStore/ChecksumValidator.cs @@ -0,0 +1,144 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Microsoft.FileFormats.PE; + +namespace Microsoft.SymbolStore +{ + internal sealed class ChecksumValidator + { + private const string pdbStreamName = "#Pdb"; + private const uint pdbIdSize = 20; + + internal static void Validate(ITracer tracer, Stream pdbStream, IEnumerable pdbChecksums) + { + uint offset = 0; + + byte[] bytes = new byte[pdbStream.Length]; + byte[] pdbId = new byte[pdbIdSize]; + if (pdbStream.Read(bytes, offset: 0, count: bytes.Length) != bytes.Length) + { + throw new InvalidChecksumException("Unexpected stream length"); + } + + try + { + offset = GetPdbStreamOffset(pdbStream); + } + catch (Exception ex) + { + tracer.Error(ex.Message); + throw; + } + + // Make a copy of the pdb Id + Array.Copy(bytes, offset, pdbId, 0, pdbIdSize); + + // Zero out the pdb Id + for (int i = 0; i < pdbIdSize; i++) + { + bytes[i + offset] = 0; + } + + bool algorithmNameKnown = false; + foreach (PdbChecksum checksum in pdbChecksums) + { + tracer.Information($"Testing checksum: {checksum}"); + + HashAlgorithm algorithm = HashAlgorithm.Create(checksum.AlgorithmName); + if (algorithm != null) + { + algorithmNameKnown = true; + byte[] hash = algorithm.ComputeHash(bytes); + if (hash.SequenceEqual(checksum.Checksum)) + { + // If any of the checksums are OK, we're good + tracer.Information($"Found checksum match {checksum}"); + // Restore the pdb Id + Array.Copy(pdbId, 0, bytes, offset, pdbIdSize); + // Restore the steam position + pdbStream.Seek(0, SeekOrigin.Begin); + + return; + } + } + } + + if (!algorithmNameKnown) + { + string algorithmNames = string.Join(" ", pdbChecksums.Select(c => c.AlgorithmName)); + throw new InvalidChecksumException($"Unknown hash algorithm: {algorithmNames}"); + } + + throw new InvalidChecksumException("PDB checksum mismatch"); + } + + private static uint GetPdbStreamOffset(Stream pdbStream) + { + pdbStream.Position = 0; + using (BinaryReader reader = new(pdbStream, Encoding.UTF8, leaveOpen: true)) + { + pdbStream.Seek(4 + // Signature + 2 + // Version Major + 2 + // Version Minor + 4, // Reserved) + SeekOrigin.Begin); + + // skip the version string + uint versionStringSize = reader.ReadUInt32(); + + pdbStream.Seek(versionStringSize, SeekOrigin.Current); + + // storage header + pdbStream.Seek(2, SeekOrigin.Current); + + // read the stream headers + ushort streamCount = reader.ReadUInt16(); + uint streamOffset; + string streamName; + + for (int i = 0; i < streamCount; i++) + { + streamOffset = reader.ReadUInt32(); + // stream size + pdbStream.Seek(4, SeekOrigin.Current); + streamName = reader.ReadNullTerminatedString(); + + if (streamName == pdbStreamName) + { + // We found it! + return streamOffset; + } + + // streams headers are on a four byte alignment + if (pdbStream.Position % 4 != 0) + { + pdbStream.Seek(4 - pdbStream.Position % 4, SeekOrigin.Current); + } + } + } + + throw new ArgumentException("We have a file with a metadata pdb signature but no pdb stream"); + } + } + + public static class BinaryReaderExtensions + { + public static string ReadNullTerminatedString(this BinaryReader stream) + { + StringBuilder builder = new(); + char ch; + while ((ch = stream.ReadChar()) != 0) + { + builder.Append(ch); + } + return builder.ToString(); + } + } +} diff --git a/src/Microsoft.SymbolStore/ITracer.cs b/src/Microsoft.SymbolStore/ITracer.cs new file mode 100644 index 000000000..3a0ab9c5b --- /dev/null +++ b/src/Microsoft.SymbolStore/ITracer.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.SymbolStore +{ + /// + /// A simple trace/logging interface. + /// + public interface ITracer + { + void WriteLine(string message); + + void WriteLine(string format, params object[] arguments); + + void Information(string message); + + void Information(string format, params object[] arguments); + + void Warning(string message); + + void Warning(string format, params object[] arguments); + + void Error(string message); + + void Error(string format, params object[] arguments); + + void Verbose(string message); + + void Verbose(string format, params object[] arguments); + } +} diff --git a/src/Microsoft.SymbolStore/InvalidChecksumException.cs b/src/Microsoft.SymbolStore/InvalidChecksumException.cs new file mode 100644 index 000000000..cdf886ca5 --- /dev/null +++ b/src/Microsoft.SymbolStore/InvalidChecksumException.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.SymbolStore +{ + public class InvalidChecksumException : Exception + { + public InvalidChecksumException(string message) : base(message) + { + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/ELFCoreKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/ELFCoreKeyGenerator.cs new file mode 100644 index 000000000..69250a93d --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/ELFCoreKeyGenerator.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.FileFormats; +using Microsoft.FileFormats.ELF; +using Microsoft.FileFormats.MachO; +using Microsoft.FileFormats.PE; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class ELFCoreKeyGenerator : KeyGenerator + { + private readonly ELFCoreFile _core; + + public ELFCoreKeyGenerator(ITracer tracer, SymbolStoreFile file) + : base(tracer) + { + StreamAddressSpace dataSource = new(file.Stream); + _core = new ELFCoreFile(dataSource); + } + + public override bool IsValid() + { + return _core.IsValid(); + } + + public override bool IsDump() + { + return true; + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if (IsValid()) + { + return _core.LoadedImages + .Select((ELFLoadedImage loadedImage) => CreateGenerator(loadedImage)) + .Where((KeyGenerator generator) => generator != null) + .SelectMany((KeyGenerator generator) => generator.GetKeys(flags)); + } + return SymbolStoreKey.EmptyArray; + } + + private KeyGenerator CreateGenerator(ELFLoadedImage loadedImage) + { + try + { + if (loadedImage.Image.IsValid()) + { + return new ELFFileKeyGenerator(Tracer, loadedImage.Image, loadedImage.Path); + } + // TODO - mikem 7/1/17 - need to figure out a better way to determine the file vs loaded layout + bool layout = loadedImage.Path.StartsWith("/"); + RelativeAddressSpace reader = new(_core.DataSource, loadedImage.LoadAddress, _core.DataSource.Length); + PEFile peFile = new(reader, layout); + if (peFile.IsValid()) + { + return new PEFileKeyGenerator(Tracer, peFile, loadedImage.Path); + } + // Check if this is a macho module in a ELF 5.0.x MacOS dump + MachOFile machOFile = new(reader, 0, true); + if (machOFile.IsValid()) + { + return new MachOFileKeyGenerator(Tracer, machOFile, loadedImage.Path); + } + Tracer.Warning("Unknown ELF core image {0:X16} {1}", loadedImage.LoadAddress, loadedImage.Path); + } + catch (InvalidVirtualAddressException ex) + { + Tracer.Error("{0}: {1:X16} {2}", ex.Message, loadedImage.LoadAddress, loadedImage.Path); + } + return null; + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/ELFFileKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/ELFFileKeyGenerator.cs new file mode 100644 index 000000000..4dd81a3f2 --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/ELFFileKeyGenerator.cs @@ -0,0 +1,212 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using Microsoft.FileFormats; +using Microsoft.FileFormats.ELF; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class ELFFileKeyGenerator : KeyGenerator + { + private const string IdentityPrefix = "elf-buildid"; + private const string SymbolPrefix = "elf-buildid-sym"; + private const string CoreClrPrefix = "elf-buildid-coreclr"; + private const string CoreClrFileName = "libcoreclr.so"; + + /// + /// Symbol file extensions. The first one is the default symbol file extension used by .NET Core. + /// + private static readonly string[] s_symbolFileExtensions = { ".dbg", ".debug" }; + + /// + /// List of special clr files that are also indexed with libcoreclr.so's key. + /// + private static readonly string[] s_specialFiles = new string[] { "libmscordaccore.so", "libmscordbi.so", "mscordaccore.dll", "mscordbi.dll" }; + private static readonly string[] s_sosSpecialFiles = new string[] { "libsos.so", "SOS.NETCore.dll" }; + + private static readonly HashSet s_coreClrSpecialFiles = new(s_specialFiles.Concat(s_sosSpecialFiles)); + private static readonly HashSet s_dacdbiSpecialFiles = new(s_specialFiles); + + private readonly ELFFile _elfFile; + private readonly string _path; + + public ELFFileKeyGenerator(ITracer tracer, ELFFile elfFile, string path) + : base(tracer) + { + _elfFile = elfFile; + _path = path; + } + + public ELFFileKeyGenerator(ITracer tracer, SymbolStoreFile file) + : this(tracer, new ELFFile(new StreamAddressSpace(file.Stream)), file.FileName) + { + } + + public override bool IsValid() + { + return _elfFile.IsValid() && + (_elfFile.Header.Type == ELFHeaderType.Executable || _elfFile.Header.Type == ELFHeaderType.Shared || _elfFile.Header.Type == ELFHeaderType.Relocatable); + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if (IsValid()) + { + byte[] buildId = _elfFile.BuildID; + if (NormalizeBuildId(ref buildId)) + { + bool symbolFile = false; + try + { + symbolFile = Array.Exists(_elfFile.Sections, section => (section.Name.StartsWith(".debug_info") || section.Name.StartsWith(".zdebug_info"))); + } + catch (Exception ex) when + (ex is InvalidVirtualAddressException || + ex is ArgumentOutOfRangeException || + ex is IndexOutOfRangeException || + ex is BadInputFormatException) + { + // This could occur when trying to read sections for an ELF image grabbed from a core dump + // In that case, fallback to checking the file extension + symbolFile = Array.IndexOf(s_symbolFileExtensions, Path.GetExtension(_path)) != -1; + } + + string symbolFileName = GetSymbolFileName(); + foreach (SymbolStoreKey key in GetKeys(flags, _path, buildId, symbolFile, symbolFileName)) + { + yield return key; + } + if ((flags & KeyTypeFlags.HostKeys) != 0) + { + if (_elfFile.Header.Type == ELFHeaderType.Executable) + { + // The host program as itself (usually dotnet) + yield return BuildKey(_path, IdentityPrefix, buildId); + + // apphost downloaded as the host program name + yield return BuildKey(_path, IdentityPrefix, buildId, "apphost"); + } + } + } + else + { + Tracer.Error("Invalid ELF BuildID '{0}' for {1}", buildId == null ? "" : ToHexString(buildId), _path); + } + } + } + + /// + /// Creates the ELF file symbol store keys. + /// + /// type of keys to return + /// file name and path + /// ELF file uuid bytes + /// if true, use the symbol file tag + /// name of symbol file (from .gnu_debuglink) or null + /// symbol store keys + public static IEnumerable GetKeys(KeyTypeFlags flags, string path, byte[] buildId, bool symbolFile, string symbolFileName) + { + Debug.Assert(path != null); + if (NormalizeBuildId(ref buildId)) + { + string fileName = GetFileName(path); + + if ((flags & KeyTypeFlags.IdentityKey) != 0) + { + if (symbolFile) + { + yield return BuildKey(path, SymbolPrefix, buildId, "_.debug"); + } + else + { + bool clrSpecialFile = s_coreClrSpecialFiles.Contains(fileName); + yield return BuildKey(path, IdentityPrefix, buildId, clrSpecialFile); + } + } + if (!symbolFile) + { + // This is a workaround for 5.0 where the ELF file type of dotnet isn't Executable but + // Shared. It doesn't work for self-contained apps (apphost renamed to host program). + if ((flags & KeyTypeFlags.HostKeys) != 0 && fileName == "dotnet") + { + yield return BuildKey(path, IdentityPrefix, buildId, clrSpecialFile: false); + } + if ((flags & KeyTypeFlags.RuntimeKeys) != 0 && fileName == CoreClrFileName) + { + yield return BuildKey(path, IdentityPrefix, buildId); + } + if ((flags & KeyTypeFlags.SymbolKey) != 0) + { + if (string.IsNullOrEmpty(symbolFileName)) + { + symbolFileName = path + s_symbolFileExtensions[0]; + } + yield return BuildKey(symbolFileName, SymbolPrefix, buildId, "_.debug"); + } + if ((flags & (KeyTypeFlags.ClrKeys | KeyTypeFlags.DacDbiKeys)) != 0) + { + // Creates all the special CLR keys if the path is the coreclr module for this platform + if (fileName == CoreClrFileName) + { + foreach (string specialFileName in (flags & KeyTypeFlags.ClrKeys) != 0 ? s_coreClrSpecialFiles : s_dacdbiSpecialFiles) + { + yield return BuildKey(specialFileName, CoreClrPrefix, buildId); + } + } + } + } + } + else + { + Debug.Fail($"Invalid ELF BuildId '{(buildId == null ? "" : ToHexString(buildId))}' for {path}"); + } + } + + private string GetSymbolFileName() + { + try + { + ELFSection section = _elfFile.FindSectionByName(".gnu_debuglink"); + if (section != null) + { + return section.Contents.Read(0); + } + } + catch (Exception ex) when + (ex is InvalidVirtualAddressException || + ex is ArgumentOutOfRangeException || + ex is IndexOutOfRangeException || + ex is BadInputFormatException) + { + Tracer.Verbose("ELF .gnu_debuglink section in {0}: {1}", _path, ex.Message); + } + return null; + } + + /// + /// Extends build-ids of 8-20 bytes (created by MD5 or UUID build ids) to 20 bytes with proper padding + /// using a zero extension + /// + /// Reference to ELF build-id. This build-id must be between 8 and 20 bytes in length. + /// True if the build-id is compliant and could be resized and padded. False otherwise. + private static bool NormalizeBuildId(ref byte[] buildId) + { + if (buildId == null || buildId.Length > 20 || buildId.Length < 8) + { + return false; + } + int oldLength = buildId.Length; + Array.Resize(ref buildId, 20); + for (int i = oldLength; i < buildId.Length; i++) + { + buildId[i] = 0; + } + return true; + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/FileKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/FileKeyGenerator.cs new file mode 100644 index 000000000..ebab6cdfa --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/FileKeyGenerator.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + /// + /// Generates a key for any kind of file (ELF core/MachO core/Minidump, + /// ELF/MachO/PE binary, PDB, etc). + /// + public class FileKeyGenerator : KeyGenerator + { + private readonly SymbolStoreFile _file; + + public FileKeyGenerator(ITracer tracer, SymbolStoreFile file) + : base(tracer) + { + _file = file; + } + + public override bool IsValid() + { + return GetGenerators().Any((generator) => generator.IsValid()); + } + + public override bool IsDump() + { + return GetGenerators().Any((generator) => generator.IsValid() && generator.IsDump()); + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + foreach (KeyGenerator generator in GetGenerators()) + { + _file.Stream.Position = 0; + if (generator.IsValid()) + { + return generator.GetKeys(flags); + } + } + Tracer.Verbose("Unknown file type: {0}", _file.FileName); + return SymbolStoreKey.EmptyArray; + } + + private IEnumerable GetGenerators() + { + if (_file.Stream.Length > 0) + { + yield return new ELFCoreKeyGenerator(Tracer, _file); + yield return new MachCoreKeyGenerator(Tracer, _file); + yield return new MinidumpKeyGenerator(Tracer, _file); + yield return new ELFFileKeyGenerator(Tracer, _file); + yield return new PEFileKeyGenerator(Tracer, _file); + yield return new MachOFatHeaderKeyGenerator(Tracer, _file); + yield return new MachOFileKeyGenerator(Tracer, _file); + yield return new PDBFileKeyGenerator(Tracer, _file); + yield return new PortablePDBFileKeyGenerator(Tracer, _file); + yield return new PerfMapFileKeyGenerator(Tracer, _file); + } + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/KeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/KeyGenerator.cs new file mode 100644 index 000000000..e2f51f2a9 --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/KeyGenerator.cs @@ -0,0 +1,202 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.FileFormats.PE; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + /// + /// Type of keys to generate + /// + [Flags] + public enum KeyTypeFlags + { + /// + /// No keys. + /// + None = 0x00, + + /// + /// Generate the key of the binary or file itself. + /// + IdentityKey = 0x01, + + /// + /// Generate the symbol key of the binary (if one). + /// + SymbolKey = 0x02, + + /// + /// Generate the keys for the DAC/SOS modules for a coreclr module. + /// + ClrKeys = 0x04, + + /// + /// Forces the key generators to create a Windows Pdb key even when + /// the DLL debug record entry is marked as portable. Used when both + /// the Portable and Windows PDBs are available on the symbol server. + /// + ForceWindowsPdbs = 0x08, + + /// + /// Generate keys for the host program. This includes the exe or main + /// module key (usually "dotnet") and an "apphost" symbol index using + /// the exe or main module's build id for self-contained apps. + /// + HostKeys = 0x10, + + /// + /// Return only the DAC (including any cross-OS DACs) and DBI module + /// keys. Does not including any SOS binaries. + /// + DacDbiKeys = 0x20, + + /// + /// Include the runtime modules (coreclr.dll, clrjit.dll, clrgc.dll, + /// libcoreclr.so, libclrjit.so, libcoreclr.dylib, etc.) + /// + RuntimeKeys = 0x40, + + /// + /// Generate the r2r perfmap key of the binary (if one exists). + /// + PerfMapKeys = 0x80 + } + + /// + /// The base class for all the key generators. They can be for individual files + /// or a group of file types. + /// + public abstract class KeyGenerator + { + /// + /// Trace/logging source + /// + protected readonly ITracer Tracer; + + /// + /// Key generator base class. + /// + /// logging + public KeyGenerator(ITracer tracer) + { + Tracer = tracer; + } + + /// + /// Returns true if the key generator can get keys for this file or binary. + /// + public abstract bool IsValid(); + + /// + /// Returns true if file is a mini or core dump. + /// + public virtual bool IsDump() + { + return false; + } + + /// + /// Returns the symbol store keys for this file or binary. + /// + /// what keys to get + public abstract IEnumerable GetKeys(KeyTypeFlags flags); + + /// + /// Key building helper for "file_name/string_id/file_name" formats. + /// + /// full path of file or binary + /// id string + /// if true, the file is one the clr special files + /// Checksums of pdb file. May be null. + /// key + protected static SymbolStoreKey BuildKey(string path, string id, bool clrSpecialFile = false, IEnumerable pdbChecksums = null) + { + string file = GetFileName(path).ToLowerInvariant(); + return BuildKey(path, null, id, file, clrSpecialFile, pdbChecksums); + } + + /// + /// Key building helper for "prefix/string_id/file_name" formats. + /// + /// full path of file or binary + /// optional id prefix + /// build id or uuid + /// if true, the file is one the clr special files + /// Checksums of pdb file. May be null. + /// key + protected static SymbolStoreKey BuildKey(string path, string prefix, byte[] id, bool clrSpecialFile = false, IEnumerable pdbChecksums = null) + { + string file = GetFileName(path).ToLowerInvariant(); + return BuildKey(path, prefix, id, file, clrSpecialFile, pdbChecksums); + } + + /// + /// Key building helper for "prefix/byte_sequence_id/file_name" formats. + /// + /// full path of file or binary + /// optional id prefix + /// build id or uuid + /// file name only + /// if true, the file is one the clr special files + /// Checksums of pdb file. May be null. + /// key + protected static SymbolStoreKey BuildKey(string path, string prefix, byte[] id, string file, bool clrSpecialFile = false, IEnumerable pdbChecksums = null) + { + return BuildKey(path, prefix, ToHexString(id), file, clrSpecialFile, pdbChecksums); + } + + /// + /// Key building helper for "prefix/byte_sequence_id/file_name" formats. + /// + /// full path of file or binary + /// optional id prefix + /// build id or uuid + /// file name only + /// if true, the file is one the clr special files + /// Checksums of pdb file. May be null. + /// key + protected static SymbolStoreKey BuildKey(string path, string prefix, string id, string file, bool clrSpecialFile = false, IEnumerable pdbChecksums = null) + { + StringBuilder key = new(); + key.Append(file); + key.Append('/'); + if (prefix != null) + { + key.Append(prefix); + key.Append('-'); + } + key.Append(id); + key.Append('/'); + key.Append(file); + return new SymbolStoreKey(key.ToString(), path, clrSpecialFile, pdbChecksums); + } + + /// /// Convert an array of bytes to a lower case hex string. /// /// array of bytes + /// hex string + public static string ToHexString(byte[] bytes) + { + if (bytes == null) + { + throw new ArgumentNullException(nameof(bytes)); + } + return string.Concat(bytes.Select(b => b.ToString("x2"))); + } + + /// + /// The back slashes are changed to forward slashes because Path.GetFileName doesn't work + /// on Linux /MacOS if there are backslashes. Both back and forward slashes work on Windows. + /// + /// possible windows path + /// just the file name + internal static string GetFileName(string path) + { + return Path.GetFileName(path.Replace('\\', '/')); + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/MachCoreKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/MachCoreKeyGenerator.cs new file mode 100644 index 000000000..0f088a0e6 --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/MachCoreKeyGenerator.cs @@ -0,0 +1,70 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.FileFormats; +using Microsoft.FileFormats.MachO; +using Microsoft.FileFormats.PE; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class MachCoreKeyGenerator : KeyGenerator + { + private readonly MachCore _core; + + public MachCoreKeyGenerator(ITracer tracer, SymbolStoreFile file) + : base(tracer) + { + StreamAddressSpace dataSource = new(file.Stream); + _core = new MachCore(dataSource); + } + + public override bool IsValid() + { + return _core.IsValid(); + } + + public override bool IsDump() + { + return true; + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if (IsValid()) + { + return _core.LoadedImages + .Select((MachLoadedImage loadedImage) => CreateGenerator(loadedImage)) + .Where((KeyGenerator generator) => generator != null) + .SelectMany((KeyGenerator generator) => generator.GetKeys(flags)); + } + return SymbolStoreKey.EmptyArray; + } + + private KeyGenerator CreateGenerator(MachLoadedImage loadedImage) + { + try + { + if (loadedImage.Image.IsValid()) + { + return new MachOFileKeyGenerator(Tracer, loadedImage.Image, loadedImage.Path); + } + // TODO - mikem 7/1/17 - need to figure out a better way to determine the file vs loaded layout + bool layout = loadedImage.Path.StartsWith("/"); + IAddressSpace dataSource = _core.VirtualAddressReader.DataSource; + PEFile peFile = new(new RelativeAddressSpace(dataSource, loadedImage.LoadAddress, dataSource.Length), layout); + if (peFile.IsValid()) + { + return new PEFileKeyGenerator(Tracer, peFile, loadedImage.Path); + } + Tracer.Warning("Unknown Mach core image {0:X16} {1}", loadedImage.LoadAddress, loadedImage.Path); + } + catch (InvalidVirtualAddressException ex) + { + Tracer.Error("{0}: {1:X16} {2}", ex.Message, loadedImage.LoadAddress, loadedImage.Path); + } + return null; + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/MachOFatHeaderKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/MachOFatHeaderKeyGenerator.cs new file mode 100644 index 000000000..ba6ac9239 --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/MachOFatHeaderKeyGenerator.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.FileFormats; +using Microsoft.FileFormats.MachO; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class MachOFatHeaderKeyGenerator : KeyGenerator + { + private readonly MachOFatFile _machoFatFile; + private readonly string _path; + + public MachOFatHeaderKeyGenerator(ITracer tracer, SymbolStoreFile file) + : base(tracer) + { + _machoFatFile = new MachOFatFile(new StreamAddressSpace(file.Stream)); + _path = file.FileName; + } + + public override bool IsValid() + { + return _machoFatFile.IsValid(); + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if (IsValid()) + { + return _machoFatFile.ArchSpecificFiles.Select((file) => new MachOFileKeyGenerator(Tracer, file, _path)).SelectMany((generator) => generator.GetKeys(flags)); + } + return SymbolStoreKey.EmptyArray; + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/MachOKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/MachOKeyGenerator.cs new file mode 100644 index 000000000..02a89b953 --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/MachOKeyGenerator.cs @@ -0,0 +1,142 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.FileFormats; +using Microsoft.FileFormats.MachO; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class MachOFileKeyGenerator : KeyGenerator + { + /// + /// The default symbol file extension used by .NET Core. + /// + private const string SymbolFileExtension = ".dwarf"; + + private const string IdentityPrefix = "mach-uuid"; + private const string SymbolPrefix = "mach-uuid-sym"; + private const string CoreClrPrefix = "mach-uuid-coreclr"; + private const string CoreClrFileName = "libcoreclr.dylib"; + + private static readonly string[] s_specialFiles = new string[] { "libmscordaccore.dylib", "libmscordbi.dylib" }; + private static readonly string[] s_sosSpecialFiles = new string[] { "libsos.dylib", "SOS.NETCore.dll" }; + + private static readonly HashSet s_coreClrSpecialFiles = new(s_specialFiles.Concat(s_sosSpecialFiles)); + private static readonly HashSet s_dacdbiSpecialFiles = new(s_specialFiles); + + private readonly MachOFile _machoFile; + private readonly string _path; + + public MachOFileKeyGenerator(ITracer tracer, MachOFile machoFile, string path) + : base(tracer) + { + _machoFile = machoFile; + _path = path; + } + + public MachOFileKeyGenerator(ITracer tracer, SymbolStoreFile file) + : this(tracer, new MachOFile(new StreamAddressSpace(file.Stream)), file.FileName) + { + } + + public override bool IsValid() + { + return _machoFile.IsValid() && + (_machoFile.Header.FileType == MachHeaderFileType.Execute || + _machoFile.Header.FileType == MachHeaderFileType.Dylib || + _machoFile.Header.FileType == MachHeaderFileType.Dsym || + _machoFile.Header.FileType == MachHeaderFileType.Bundle); + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if (IsValid()) + { + byte[] uuid = _machoFile.Uuid; + if (uuid != null && uuid.Length == 16) + { + bool symbolFile = _machoFile.Header.FileType == MachHeaderFileType.Dsym; + // TODO - mikem 1/23/18 - is there a way to get the name of the "linked" dwarf symbol file + foreach (SymbolStoreKey key in GetKeys(flags, _path, uuid, symbolFile, symbolFileName: null)) + { + yield return key; + } + if ((flags & KeyTypeFlags.HostKeys) != 0) + { + if (_machoFile.Header.FileType == MachHeaderFileType.Execute) + { + // The host program as itself (usually dotnet) + yield return BuildKey(_path, IdentityPrefix, uuid); + + // apphost downloaded as the host program name + yield return BuildKey(_path, IdentityPrefix, uuid, "apphost"); + } + } + } + else + { + Tracer.Error("Invalid MachO uuid {0}", _path); + } + } + } + + /// + /// Creates the MachO file symbol store keys. + /// + /// type of keys to return + /// file name and path + /// macho file uuid bytes + /// if true, use the symbol file tag + /// name of symbol file or null + /// symbol store keys + public static IEnumerable GetKeys(KeyTypeFlags flags, string path, byte[] uuid, bool symbolFile, string symbolFileName) + { + Debug.Assert(path != null); + Debug.Assert(uuid != null && uuid.Length == 16); + + string fileName = GetFileName(path); + + if ((flags & KeyTypeFlags.IdentityKey) != 0) + { + if (symbolFile) + { + yield return BuildKey(path, SymbolPrefix, uuid, "_.dwarf"); + } + else + { + bool clrSpecialFile = s_coreClrSpecialFiles.Contains(fileName); + yield return BuildKey(path, IdentityPrefix, uuid, clrSpecialFile); + } + } + if (!symbolFile) + { + if ((flags & KeyTypeFlags.RuntimeKeys) != 0 && fileName == CoreClrFileName) + { + yield return BuildKey(path, IdentityPrefix, uuid); + } + if ((flags & KeyTypeFlags.SymbolKey) != 0) + { + if (string.IsNullOrEmpty(symbolFileName)) + { + symbolFileName = path + SymbolFileExtension; + } + yield return BuildKey(symbolFileName, SymbolPrefix, uuid, "_.dwarf"); + } + if ((flags & (KeyTypeFlags.ClrKeys | KeyTypeFlags.DacDbiKeys)) != 0) + { + /// Creates all the special CLR keys if the path is the coreclr module for this platform + if (fileName == CoreClrFileName) + { + foreach (string specialFileName in (flags & KeyTypeFlags.ClrKeys) != 0 ? s_coreClrSpecialFiles : s_dacdbiSpecialFiles) + { + yield return BuildKey(specialFileName, CoreClrPrefix, uuid); + } + } + } + } + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/MinidumpKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/MinidumpKeyGenerator.cs new file mode 100644 index 000000000..324bdfa65 --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/MinidumpKeyGenerator.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.FileFormats; +using Microsoft.FileFormats.Minidump; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class MinidumpKeyGenerator : KeyGenerator + { + private readonly IAddressSpace _dataSource; + + public MinidumpKeyGenerator(ITracer tracer, SymbolStoreFile file) + : base(tracer) + { + _dataSource = new StreamAddressSpace(file.Stream); + } + + public override bool IsValid() + { + return Minidump.IsValid(_dataSource); + } + + public override bool IsDump() + { + return true; + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if (IsValid()) + { + try + { + Minidump dump = new(_dataSource); + return dump.LoadedImages + .Select((MinidumpLoadedImage loadedImage) => new PEFileKeyGenerator(Tracer, loadedImage.Image, loadedImage.ModuleName)) + .SelectMany((KeyGenerator generator) => generator.GetKeys(flags)); + } + catch (InvalidVirtualAddressException ex) + { + Tracer.Error("Minidump {0}", ex.Message); + } + } + return SymbolStoreKey.EmptyArray; + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/PDBFileKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/PDBFileKeyGenerator.cs new file mode 100644 index 000000000..79aa08ce8 --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/PDBFileKeyGenerator.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using Microsoft.FileFormats; +using Microsoft.FileFormats.PDB; +using Microsoft.FileFormats.PE; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class PDBFileKeyGenerator : KeyGenerator + { + private readonly PDBFile _pdbFile; + private readonly string _path; + + public PDBFileKeyGenerator(ITracer tracer, SymbolStoreFile file) + : base(tracer) + { + StreamAddressSpace dataSource = new(file.Stream); + _pdbFile = new PDBFile(dataSource); + _path = file.FileName; + } + + public override bool IsValid() + { + return _pdbFile.IsValid(); + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if (IsValid()) + { + if ((flags & KeyTypeFlags.IdentityKey) != 0) + { + if (_pdbFile.DbiStream.IsValid()) + { + yield return GetKey(_path, _pdbFile.Signature, unchecked((int)_pdbFile.DbiAge)); + } + else + { + yield return GetKey(_path, _pdbFile.Signature, unchecked((int)_pdbFile.Age)); + } + } + } + } + + /// + /// Create a symbol store key for a Windows PDB. + /// + /// file name and path + /// mvid guid + /// pdb age + /// symbol store key + public static SymbolStoreKey GetKey(string path, Guid signature, int age, IEnumerable pdbChecksums = null) + { + Debug.Assert(path != null); + Debug.Assert(signature != null); + return BuildKey(path, string.Format("{0}{1:x}", signature.ToString("N"), age)); + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/PEFileKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/PEFileKeyGenerator.cs new file mode 100644 index 000000000..e196e2051 --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/PEFileKeyGenerator.cs @@ -0,0 +1,219 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using Microsoft.FileFormats; +using Microsoft.FileFormats.PE; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class PEFileKeyGenerator : KeyGenerator + { + private const string CoreClrFileName = "coreclr.dll"; + + private static readonly HashSet s_longNameBinaryPrefixes = new(new string[] { "mscordaccore_", "sos_" }); + private static readonly HashSet s_daclongNameBinaryPrefixes = new(new string[] { "mscordaccore_" }); + + private static readonly string[] s_specialFiles = new string[] { "mscordaccore.dll", "mscordbi.dll" }; + private static readonly string[] s_sosSpecialFiles = new string[] { "sos.dll", "SOS.NETCore.dll" }; + + private static readonly HashSet s_coreClrSpecialFiles = new(s_specialFiles.Concat(s_sosSpecialFiles)); + private static readonly HashSet s_dacdbiSpecialFiles = new(s_specialFiles); + + private readonly PEFile _peFile; + private readonly string _path; + + public PEFileKeyGenerator(ITracer tracer, PEFile peFile, string path) + : base(tracer) + { + _peFile = peFile; + _path = path; + } + + public PEFileKeyGenerator(ITracer tracer, SymbolStoreFile file) + : this(tracer, new PEFile(new StreamAddressSpace(file.Stream)), file.FileName) + { + } + + public override bool IsValid() + { + return _peFile.IsValid(); + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if (IsValid()) + { + if ((flags & KeyTypeFlags.IdentityKey) != 0) + { + yield return GetKey(_path, _peFile.Timestamp, _peFile.SizeOfImage); + } + if ((flags & KeyTypeFlags.RuntimeKeys) != 0 && GetFileName(_path) == CoreClrFileName) + { + yield return GetKey(_path, _peFile.Timestamp, _peFile.SizeOfImage); + } + if ((flags & KeyTypeFlags.SymbolKey) != 0) + { + PEPdbRecord[] pdbs = System.Array.Empty(); + try + { + pdbs = _peFile.Pdbs.ToArray(); + } + catch (InvalidVirtualAddressException ex) + { + Tracer.Error("Reading PDB records for {0}: {1}", _path, ex.Message); + } + + foreach (PEPdbRecord pdb in pdbs) + { + if (((flags & KeyTypeFlags.ForceWindowsPdbs) == 0) && pdb.IsPortablePDB) + { + yield return PortablePDBFileKeyGenerator.GetKey(pdb.Path, pdb.Signature, _peFile.PdbChecksums); + } + else + { + yield return PDBFileKeyGenerator.GetKey(pdb.Path, pdb.Signature, pdb.Age, _peFile.PdbChecksums); + } + } + } + + if ((flags & KeyTypeFlags.PerfMapKeys) != 0) + { + foreach (PEPerfMapRecord perfmapRecord in _peFile.PerfMapsV1) + { + if (perfmapRecord.Version > FileFormats.PerfMap.PerfMapFile.MaxKnownPerfMapVersion) + { + Tracer.Warning("Trying to get key for PerfmapFile {0} associated with PE {1} with version {2}, higher than max known version {3}", + perfmapRecord.Path, _path, perfmapRecord.Version, FileFormats.PerfMap.PerfMapFile.MaxKnownPerfMapVersion); + } + yield return PerfMapFileKeyGenerator.GetKey(perfmapRecord.Path, perfmapRecord.Signature, perfmapRecord.Version); + } + } + + if ((flags & (KeyTypeFlags.ClrKeys | KeyTypeFlags.DacDbiKeys)) != 0) + { + if (GetFileName(_path) == CoreClrFileName) + { + string coreclrId = string.Format("{0:X8}{1:x}", _peFile.Timestamp, _peFile.SizeOfImage); + foreach (string specialFileName in GetSpecialFiles(flags)) + { + yield return BuildKey(specialFileName, coreclrId); + } + } + } + if ((flags & KeyTypeFlags.HostKeys) != 0) + { + if ((_peFile.FileHeader.Characteristics & (ushort)ImageFile.Dll) == 0 && !_peFile.IsILImage) + { + string id = string.Format("{0:X8}{1:x}", _peFile.Timestamp, _peFile.SizeOfImage); + + // The host program as itself (usually dotnet.exe) + yield return BuildKey(_path, id); + + // apphost.exe downloaded as the host program name + yield return BuildKey(_path, prefix: null, id, "apphost.exe"); + } + } + } + } + + private IEnumerable GetSpecialFiles(KeyTypeFlags flags) + { + List specialFiles = new((flags & KeyTypeFlags.ClrKeys) != 0 ? s_coreClrSpecialFiles : s_dacdbiSpecialFiles); + + VsFixedFileInfo fileVersion = _peFile.VersionInfo; + if (fileVersion != null) + { + ushort major = fileVersion.ProductVersionMajor; + ushort minor = fileVersion.ProductVersionMinor; + ushort build = fileVersion.ProductVersionBuild; + ushort revision = fileVersion.ProductVersionRevision; + + List hostArchitectures = new(); + string targetArchitecture = null; + + ImageFileMachine machine = (ImageFileMachine)_peFile.FileHeader.Machine; + switch (machine) + { + case ImageFileMachine.Amd64: + targetArchitecture = "amd64"; + break; + + case ImageFileMachine.I386: + targetArchitecture = "x86"; + break; + + case ImageFileMachine.ArmNT: + targetArchitecture = "arm"; + hostArchitectures.Add("x86"); + break; + + case ImageFileMachine.Arm64: + targetArchitecture = "arm64"; + hostArchitectures.Add("amd64"); + break; + } + + if (targetArchitecture != null) + { + hostArchitectures.Add(targetArchitecture); + + foreach (string hostArchitecture in hostArchitectures) + { + string buildFlavor = ""; + + if ((fileVersion.FileFlags & FileInfoFlags.Debug) != 0) + { + if ((fileVersion.FileFlags & FileInfoFlags.SpecialBuild) != 0) + { + buildFlavor = ".dbg"; + } + else + { + buildFlavor = ".chk"; + } + } + + foreach (string name in (flags & KeyTypeFlags.ClrKeys) != 0 ? s_longNameBinaryPrefixes : s_daclongNameBinaryPrefixes) + { + // The name prefixes include the trailing "_". + string longName = string.Format("{0}{1}_{2}_{3}.{4}.{5}.{6:00}{7}.dll", + name, hostArchitecture, targetArchitecture, major, minor, build, revision, buildFlavor); + specialFiles.Add(longName); + } + } + } + } + else + { + Tracer.Warning("{0} has no version resource", _path); + } + + return specialFiles; + } + + /// + /// Creates a PE file symbol store key identity key. + /// + /// file name and path + /// time stamp of pe image + /// size of pe image + /// symbol store keys (or empty enumeration) + public static SymbolStoreKey GetKey(string path, uint timestamp, uint sizeOfImage) + { + Debug.Assert(path != null); + + // The clr special file flag can not be based on the GetSpecialFiles() list because + // that is only valid when "path" is the coreclr.dll. + string fileName = GetFileName(path); + bool clrSpecialFile = s_coreClrSpecialFiles.Contains(fileName) || + (s_longNameBinaryPrefixes.Any((prefix) => fileName.StartsWith(prefix)) && Path.GetExtension(fileName) == ".dll"); + + string id = string.Format("{0:X8}{1:x}", timestamp, sizeOfImage); + return BuildKey(path, id, clrSpecialFile); + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/PerfMapFileKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/PerfMapFileKeyGenerator.cs new file mode 100644 index 000000000..9ecc355fb --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/PerfMapFileKeyGenerator.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.FileFormats.PerfMap; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class PerfMapFileKeyGenerator : KeyGenerator + { + private readonly SymbolStoreFile _file; + private readonly PerfMapFile _perfmapFile; + + public PerfMapFileKeyGenerator(ITracer tracer, SymbolStoreFile file) + : base(tracer) + { + _file = file; + _perfmapFile = new PerfMapFile(_file.Stream); + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if (!IsValid() || (flags & KeyTypeFlags.IdentityKey) == 0) + { + yield break; + } + Debug.Assert(_perfmapFile.Header is not null); + + PerfMapFile.PerfMapHeader header = _perfmapFile.Header; + + if (header.Version > PerfMapFile.MaxKnownPerfMapVersion) + { + Tracer.Warning("Trying to get key for PerfMap {0} with version {1}, higher than max known version {2}.", + _file.FileName, header.Version, PerfMapFile.MaxKnownPerfMapVersion); + } + yield return PerfMapFileKeyGenerator.GetKey(_file.FileName, header.Signature, header.Version); + } + + public override bool IsValid() => _perfmapFile.IsValid; + + internal static SymbolStoreKey GetKey(string path, byte[] signature, uint version) + { + Debug.Assert(path != null); + Debug.Assert(signature != null); + + string stringSignature = string.Concat(signature.Select(b => b.ToString("x2")));; + string idComponent = $"r2rmap-v{version}-{stringSignature}"; + return BuildKey(path, idComponent, clrSpecialFile: false, pdbChecksums: null); + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/PortablePDBFileKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/PortablePDBFileKeyGenerator.cs new file mode 100644 index 000000000..7bc3536ed --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/PortablePDBFileKeyGenerator.cs @@ -0,0 +1,88 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Reflection.Metadata; +using Microsoft.FileFormats.PE; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class PortablePDBFileKeyGenerator : KeyGenerator + { + private readonly SymbolStoreFile _file; + + public PortablePDBFileKeyGenerator(ITracer tracer, SymbolStoreFile file) + : base(tracer) + { + _file = file; + } + + public override bool IsValid() + { + try + { + _file.Stream.Position = 0; + using (MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbStream(_file.Stream, MetadataStreamOptions.LeaveOpen)) + { + MetadataReader reader = provider.GetMetadataReader(); + return true; + } + } + catch (BadImageFormatException) + { + } + return false; + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if ((flags & KeyTypeFlags.IdentityKey) != 0) + { + SymbolStoreKey key = null; + + try + { + _file.Stream.Position = 0; + using (MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbStream(_file.Stream, MetadataStreamOptions.LeaveOpen)) + { + MetadataReader reader = provider.GetMetadataReader(); + BlobContentId blob = new(reader.DebugMetadataHeader.Id); + if ((flags & KeyTypeFlags.ForceWindowsPdbs) == 0) + { + key = GetKey(_file.FileName, blob.Guid); + } + else + { + // Force the Windows PDB index + key = PDBFileKeyGenerator.GetKey(_file.FileName, blob.Guid, 1); + } + } + } + catch (BadImageFormatException ex) + { + Tracer.Warning("PortablePDBFileKeyGenerator {0}", ex.Message); + } + + if (key != null) + { + yield return key; + } + } + } + + /// + /// Create a symbol store key for a Portable PDB. + /// + /// file name and path + /// pdb guid + /// symbol store key + public static SymbolStoreKey GetKey(string path, Guid pdbId, IEnumerable pdbChecksums = null) + { + Debug.Assert(path != null); + Debug.Assert(pdbId != null); + return BuildKey(path, pdbId.ToString("N") + "FFFFFFFF", clrSpecialFile: false, pdbChecksums); + } + } +} diff --git a/src/Microsoft.SymbolStore/KeyGenerators/SourceFileKeyGenerator.cs b/src/Microsoft.SymbolStore/KeyGenerators/SourceFileKeyGenerator.cs new file mode 100644 index 000000000..c9095b8ac --- /dev/null +++ b/src/Microsoft.SymbolStore/KeyGenerators/SourceFileKeyGenerator.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Security.Cryptography; + +namespace Microsoft.SymbolStore.KeyGenerators +{ + public class SourceFileKeyGenerator : KeyGenerator + { + private readonly SymbolStoreFile _file; + + public SourceFileKeyGenerator(ITracer tracer, SymbolStoreFile file) + : base(tracer) + { + _file = file; + } + + public override bool IsValid() + { + return true; + } + + public override IEnumerable GetKeys(KeyTypeFlags flags) + { + if ((flags & KeyTypeFlags.IdentityKey) != 0) + { +#pragma warning disable CA5350 // Do Not Use Weak Cryptographic Algorithms + byte[] hash = SHA1.Create().ComputeHash(_file.Stream); +#pragma warning restore CA5350 // Do Not Use Weak Cryptographic Algorithms + yield return GetKey(_file.FileName, hash); + } + } + + /// + /// Create a symbol store key for a source file + /// + /// file name and path + /// sha1 hash of the source file + /// symbol store key + public static SymbolStoreKey GetKey(string path, byte[] hash) + { + Debug.Assert(path != null); + Debug.Assert(hash != null); + return BuildKey(path, "sha1", hash); + } + } +} diff --git a/src/Microsoft.SymbolStore/Microsoft.SymbolStore.csproj b/src/Microsoft.SymbolStore/Microsoft.SymbolStore.csproj new file mode 100644 index 000000000..f4c5d8a65 --- /dev/null +++ b/src/Microsoft.SymbolStore/Microsoft.SymbolStore.csproj @@ -0,0 +1,28 @@ + + + net462;netstandard2.0 + ;1591;1701 + true + Symbol server key generation and access protocol + $(Description) + Symbol Indexing + true + true + + + + + + + + + + + + + + + diff --git a/src/Microsoft.SymbolStore/SymbolStoreFile.cs b/src/Microsoft.SymbolStore/SymbolStoreFile.cs new file mode 100644 index 000000000..3c6493d09 --- /dev/null +++ b/src/Microsoft.SymbolStore/SymbolStoreFile.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics; +using System.IO; + +namespace Microsoft.SymbolStore +{ + /// + /// Symbol store file. + /// + /// Key generation: input file stream and file name/path. + /// Symbol store: output file stream and the file name/path it came. + /// + public sealed class SymbolStoreFile : IDisposable + { + /// + /// The input file stream to generate the key or the output file stream + /// for the symbol stores to write. + /// + public readonly Stream Stream; + + /// + /// The name of the input file for key generation or the name of where + /// the output file came for symbol stores i.e. cached file name, file.ptr + /// UNC path or http request URL. + /// + public readonly string FileName; + + /// + /// Create a symbol file instance + /// + /// stream of the file contents + /// name of the file + public SymbolStoreFile(Stream stream, string fileName) + { + Debug.Assert(stream != null); + Debug.Assert(stream.CanSeek); + Debug.Assert(fileName != null); + + Stream = stream; + FileName = fileName; + } + + public void Dispose() + { + Stream.Dispose(); + } + } +} diff --git a/src/Microsoft.SymbolStore/SymbolStoreKey.cs b/src/Microsoft.SymbolStore/SymbolStoreKey.cs new file mode 100644 index 000000000..8cb60f71f --- /dev/null +++ b/src/Microsoft.SymbolStore/SymbolStoreKey.cs @@ -0,0 +1,125 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using Microsoft.FileFormats.PE; + +namespace Microsoft.SymbolStore +{ + /// + /// Symbol store key information + /// + public sealed class SymbolStoreKey + { + /// + /// Symbol server index + /// + public readonly string Index; + + /// + /// Full path name + /// + public readonly string FullPathName; + + /// + /// If true, this file is one of the clr special files like the DAC or SOS, but + /// the key is the normal identity key for this file. + /// + public readonly bool IsClrSpecialFile; + + /// + /// Empty array of keys + /// + public static SymbolStoreKey[] EmptyArray = Array.Empty(); + + /// + /// The checksums of the pdb file (if any) + /// + public readonly IEnumerable PdbChecksums; + + /// + /// Create key instance. + /// + /// index to lookup on symbol server + /// the full path name of the file + /// if true, the file is one the clr special files + /// if true, the file is one the clr special files + public SymbolStoreKey(string index, string fullPathName, bool clrSpecialFile = false, IEnumerable pdbChecksums = null) + { + Debug.Assert(index != null && fullPathName != null); + Index = index; + FullPathName = fullPathName; + IsClrSpecialFile = clrSpecialFile; + PdbChecksums = pdbChecksums ?? Enumerable.Empty(); + } + + /// + /// Returns the first two parts of the index tuple. Allows a different file name + /// to be appended to this symbol key. Includes the trailing "/". + /// + public string IndexPrefix + { + get { return Index.Substring(0, Index.LastIndexOf("/") + 1); } + } + + /// + /// Returns the hash of the index. + /// + public override int GetHashCode() + { + return Index.GetHashCode(); + } + + /// + /// Only the index is compared or hashed. The FileName is already + /// part of the index. + /// + public override bool Equals(object obj) + { + SymbolStoreKey right = (SymbolStoreKey)obj; + return string.Equals(Index, right.Index); + } + + private static HashSet s_invalidChars = new(Path.GetInvalidFileNameChars()); + + /// + /// Validates a symbol index. + /// + /// SSQP theoretically supports a broader set of keys, but in order to ensure that all the keys + /// play well with the caching scheme we enforce additional requirements (that all current key + /// conventions also meet). + /// + /// symbol key index + /// true if valid + public static bool IsKeyValid(string index) + { + string[] parts = index.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 3) { + return false; + } + for (int i = 0; i < 3; i++) + { + foreach (char c in parts[i]) + { + if (char.IsLetterOrDigit(c)) { + continue; + } + if (!s_invalidChars.Contains(c)) { + continue; + } + return false; + } + // We need to support files with . in the name, but we don't want identifiers that + // are meaningful to the filesystem + if (parts[i] == "." || parts[i] == "..") { + return false; + } + } + return true; + } + } +} diff --git a/src/Microsoft.SymbolStore/SymbolStores/CacheSymbolStore.cs b/src/Microsoft.SymbolStore/SymbolStores/CacheSymbolStore.cs new file mode 100644 index 000000000..82b3f8d77 --- /dev/null +++ b/src/Microsoft.SymbolStore/SymbolStores/CacheSymbolStore.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.SymbolStore.SymbolStores +{ + public sealed class CacheSymbolStore : SymbolStore + { + public string CacheDirectory { get; } + + public CacheSymbolStore(ITracer tracer, SymbolStore backingStore, string cacheDirectory) + : base(tracer, backingStore) + { + CacheDirectory = cacheDirectory ?? throw new ArgumentNullException(nameof(cacheDirectory)); + } + + protected override Task GetFileInner(SymbolStoreKey key, CancellationToken token) + { + SymbolStoreFile result = null; + string cacheFile = GetCacheFilePath(key); + if (File.Exists(cacheFile)) + { + Stream fileStream = File.OpenRead(cacheFile); + result = new SymbolStoreFile(fileStream, cacheFile); + } + return Task.FromResult(result); + } + + protected override async Task WriteFileInner(SymbolStoreKey key, SymbolStoreFile file) + { + string cacheFile = GetCacheFilePath(key); + if (cacheFile != null && !File.Exists(cacheFile)) + { + try + { + Directory.CreateDirectory(Path.GetDirectoryName(cacheFile)); + using (Stream destinationStream = File.OpenWrite(cacheFile)) + { + await file.Stream.CopyToAsync(destinationStream).ConfigureAwait(false); + Tracer.Verbose("Cached: {0}", cacheFile); + } + } + catch (Exception ex) when (ex is ArgumentException || ex is UnauthorizedAccessException || ex is IOException) + { + } + } + } + + private string GetCacheFilePath(SymbolStoreKey key) + { + if (SymbolStoreKey.IsKeyValid(key.Index)) { + return Path.Combine(CacheDirectory, key.Index); + } + Tracer.Error("CacheSymbolStore: invalid key index {0}", key.Index); + return null; + } + + public override bool Equals(object obj) + { + if (obj is CacheSymbolStore store) + { + return IsPathEqual(CacheDirectory, store.CacheDirectory); + } + return false; + } + + public override int GetHashCode() + { + return HashPath(CacheDirectory); + } + + public override string ToString() + { + return $"Cache: {CacheDirectory}"; + } + } +} diff --git a/src/Microsoft.SymbolStore/SymbolStores/DirectorySymbolStore.cs b/src/Microsoft.SymbolStore/SymbolStores/DirectorySymbolStore.cs new file mode 100644 index 000000000..57c203ef1 --- /dev/null +++ b/src/Microsoft.SymbolStore/SymbolStores/DirectorySymbolStore.cs @@ -0,0 +1,91 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SymbolStore; +using Microsoft.SymbolStore.KeyGenerators; +using Microsoft.SymbolStore.SymbolStores; + +namespace SOS +{ + /// + /// Basic http symbol store. The request can be authentication with a PAT for VSTS symbol stores. + /// + public class DirectorySymbolStore : SymbolStore + { + /// + /// Directory to search symbols + /// + public string Directory { get; } + + /// + /// Create an instance of a directory symbol store + /// + /// next symbol store or null + /// symbol search path + public DirectorySymbolStore(ITracer tracer, SymbolStore backingStore, string directory) + : base(tracer, backingStore) + { + Directory = directory ?? throw new ArgumentNullException(nameof(directory)); + } + + protected override Task GetFileInner(SymbolStoreKey key, CancellationToken token) + { + SymbolStoreFile result = null; + + if (SymbolStoreKey.IsKeyValid(key.Index)) + { + string filePath = Path.Combine(Directory, Path.GetFileName(key.FullPathName)); + if (File.Exists(filePath)) + { + try + { + Stream fileStream = File.OpenRead(filePath); + SymbolStoreFile file = new(fileStream, filePath); + FileKeyGenerator generator = new(Tracer, file); + + foreach (SymbolStoreKey targetKey in generator.GetKeys(KeyTypeFlags.IdentityKey)) + { + if (key.Equals(targetKey)) + { + result = file; + break; + } + } + } + catch (Exception ex) when (ex is UnauthorizedAccessException || ex is IOException) + { + } + } + } + else + { + Tracer.Error("DirectorySymbolStore: invalid key index {0}", key.Index); + } + + return Task.FromResult(result); + } + + public override bool Equals(object obj) + { + if (obj is DirectorySymbolStore store) + { + return IsPathEqual(Directory, store.Directory); + } + return false; + } + + public override int GetHashCode() + { + return HashPath(Directory); + } + + public override string ToString() + { + return $"Directory: {Directory}"; + } + } +} diff --git a/src/Microsoft.SymbolStore/SymbolStores/HttpSymbolStore.cs b/src/Microsoft.SymbolStore/SymbolStores/HttpSymbolStore.cs new file mode 100644 index 000000000..9409babc9 --- /dev/null +++ b/src/Microsoft.SymbolStore/SymbolStores/HttpSymbolStore.cs @@ -0,0 +1,347 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Sockets; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.SymbolStore.SymbolStores +{ + /// + /// Basic http symbol store. The request can be authentication with a PAT for VSTS symbol stores. + /// + public class HttpSymbolStore : SymbolStore + { + private readonly HttpClient _client; + private readonly HttpClient _authenticatedClient; + private bool _clientFailure; + + /// + /// For example, https://dotnet.myget.org/F/dev-feed/symbols. + /// + public Uri Uri { get; } + + /// + /// Get or set the request timeout. Default 4 minutes. + /// + public TimeSpan Timeout + { + get + { + return _client.Timeout; + } + set + { + _client.Timeout = value; + if (_authenticatedClient != null) + { + _authenticatedClient.Timeout = value; + } + } + } + + /// + /// The number of retries to do on a retryable status or socket error + /// + public int RetryCount { get; set; } + + /// + /// Setups the underlying fields for HttpSymbolStore + /// + /// logger + /// next symbol store or null + /// symbol server url + /// flag to indicate to create an authenticatedClient if there is a PAT + private HttpSymbolStore(ITracer tracer, SymbolStore backingStore, Uri symbolServerUri, bool hasPAT) + : base(tracer, backingStore) + { + Uri = symbolServerUri ?? throw new ArgumentNullException(nameof(symbolServerUri)); + if (!symbolServerUri.IsAbsoluteUri || symbolServerUri.IsFile) + { + throw new ArgumentException(nameof(symbolServerUri)); + } + + // Normal unauthenticated client + _client = new HttpClient + { + Timeout = TimeSpan.FromMinutes(4) + }; + + if (hasPAT) + { + HttpClientHandler handler = new() + { + AllowAutoRedirect = false + }; + HttpClient client = new(handler) + { + Timeout = TimeSpan.FromMinutes(4) + }; + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + // Authorization is set in associated constructors. + _authenticatedClient = client; + } + } + + /// + /// Create an instance of a http symbol store + /// + /// logger + /// next symbol store or null + /// symbol server url + /// optional Basic Auth PAT or null if no authentication + public HttpSymbolStore(ITracer tracer, SymbolStore backingStore, Uri symbolServerUri, string personalAccessToken = null) + : this(tracer, backingStore, symbolServerUri, !string.IsNullOrEmpty(personalAccessToken)) + { + // If PAT, create authenticated client with Basic Auth + if (!string.IsNullOrEmpty(personalAccessToken)) + { + _authenticatedClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format("{0}:{1}", "", personalAccessToken)))); + } + } + + /// + /// Create an instance of a http symbol store with an authenticated client + /// + /// logger + /// next symbol store or null + /// symbol server url + /// The scheme information to use for the AuthenticationHeaderValue + /// The parameter information to use for the AuthenticationHeaderValue + public HttpSymbolStore(ITracer tracer, SymbolStore backingStore, Uri symbolServerUri, string scheme, string parameter) + : this(tracer, backingStore, symbolServerUri, true) + { + if (string.IsNullOrEmpty(scheme)) + { + throw new ArgumentNullException(nameof(scheme)); + } + + if (string.IsNullOrEmpty(parameter)) + { + throw new ArgumentNullException(nameof(parameter)); + } + + // Create authenticated header with given SymbolAuthHeader + _authenticatedClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(scheme, parameter); + // Force redirect logins to fail. + _authenticatedClient.DefaultRequestHeaders.Add("X-TFS-FedAuthRedirect", "Suppress"); + } + + /// + /// Resets the sticky client failure flag. This client instance will now + /// attempt to download again instead of automatically failing. + /// + public void ResetClientFailure() + { + _clientFailure = false; + } + + protected override async Task GetFileInner(SymbolStoreKey key, CancellationToken token) + { + Uri uri = GetRequestUri(key.Index); + + bool needsChecksumMatch = key.PdbChecksums.Any(); + + if (needsChecksumMatch) + { + string checksumHeader = string.Join(";", key.PdbChecksums); + HttpClient client = _authenticatedClient ?? _client; + Tracer.Information($"SymbolChecksum: {checksumHeader}"); + client.DefaultRequestHeaders.Add("SymbolChecksum", checksumHeader); + } + + Stream stream = await GetFileStream(key.FullPathName, uri, token).ConfigureAwait(false); + if (stream != null) + { + if (needsChecksumMatch) + { + ChecksumValidator.Validate(Tracer, stream, key.PdbChecksums); + } + return new SymbolStoreFile(stream, uri.ToString()); + } + return null; + } + + protected Uri GetRequestUri(string index) + { + // Escape everything except the forward slashes (/) in the index + index = string.Join("/", index.Split('/').Select(part => Uri.EscapeDataString(part))); + if (!Uri.TryCreate(Uri, index, out Uri requestUri)) + { + throw new ArgumentException(message: null, paramName: nameof(index)); + } + if (requestUri.IsFile) + { + throw new ArgumentException(message: null, paramName: nameof(index)); + } + return requestUri; + } + + protected async Task GetFileStream(string path, Uri requestUri, CancellationToken token) + { + // Just return if previous failure + if (_clientFailure) + { + return null; + } + string fileName = Path.GetFileName(path); + HttpClient client = _authenticatedClient ?? _client; + int retries = 0; + while (true) + { + bool retryable; + string message; + try + { + // Can not dispose the response (like via using) on success because then the content stream + // is disposed and it is returned by this function. + HttpResponseMessage response = await client.GetAsync(requestUri, token).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.OK) + { + return await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + } + if (response.StatusCode == HttpStatusCode.Found) + { + Uri location = response.Headers.Location; + response.Dispose(); + + response = await _client.GetAsync(location, token).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.OK) + { + return await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + } + } + HttpStatusCode statusCode = response.StatusCode; + string reasonPhrase = response.ReasonPhrase; + response.Dispose(); + + // The GET failed + + if (statusCode == HttpStatusCode.NotFound) + { + Tracer.Error("Not Found: {0} - '{1}'", fileName, requestUri.AbsoluteUri); + break; + } + + retryable = IsRetryableStatus(statusCode); + + // Build the status code error message + message = string.Format("{0} {1}: {2} - '{3}'", (int)statusCode, reasonPhrase, fileName, requestUri.AbsoluteUri); + } + catch (HttpRequestException ex) + { + SocketError socketError = SocketError.Success; + retryable = false; + + Exception innerException = ex.InnerException; + while (innerException != null) + { + if (innerException is SocketException se) + { + socketError = se.SocketErrorCode; + retryable = IsRetryableSocketError(socketError); + break; + } + + innerException = innerException.InnerException; + } + + // Build the socket error message + message = string.Format($"HttpSymbolStore: {fileName} retryable {retryable} socketError {socketError} '{requestUri.AbsoluteUri}' {ex}"); + } + + // If the status code or socket error isn't some temporary or retryable condition, mark failure + if (!retryable) + { + MarkClientFailure(); + Tracer.Error(message); + break; + } + else + { + Tracer.Warning(message); + } + + // Retry the operation? + if (retries++ >= RetryCount) + { + break; + } + + Tracer.Information($"HttpSymbolStore: retry #{retries}"); + + // Delay for a while before doing the retry + await Task.Delay(TimeSpan.FromMilliseconds((Math.Pow(2, retries) * 100) + new Random().Next(200)), token).ConfigureAwait(false); + } + return null; + } + + public override void Dispose() + { + _client.Dispose(); + _authenticatedClient?.Dispose(); + base.Dispose(); + } + + private HashSet s_retryableStatusCodes = new() + { + HttpStatusCode.RequestTimeout, + HttpStatusCode.InternalServerError, + HttpStatusCode.BadGateway, + HttpStatusCode.ServiceUnavailable, + HttpStatusCode.GatewayTimeout, + }; + + /// + /// Returns true if the http status code is temporary or retryable condition. + /// + protected bool IsRetryableStatus(HttpStatusCode status) => s_retryableStatusCodes.Contains(status); + + private HashSet s_retryableSocketErrors = new() + { + SocketError.ConnectionReset, + SocketError.ConnectionAborted, + SocketError.Shutdown, + SocketError.TimedOut, + SocketError.TryAgain, + }; + + protected bool IsRetryableSocketError(SocketError se) => s_retryableSocketErrors.Contains(se); + + /// + /// Marks this client as a failure where any subsequent calls to + /// GetFileStream() will return null. + /// + protected void MarkClientFailure() + { + _clientFailure = true; + } + + public override bool Equals(object obj) + { + if (obj is HttpSymbolStore store) + { + return Uri.Equals(store.Uri); + } + return false; + } + + public override int GetHashCode() + { + return Uri.GetHashCode(); + } + + public override string ToString() + { + return $"Server: {Uri}"; + } + } +} diff --git a/src/Microsoft.SymbolStore/SymbolStores/SymbolStore.cs b/src/Microsoft.SymbolStore/SymbolStores/SymbolStore.cs new file mode 100644 index 000000000..170a1e77d --- /dev/null +++ b/src/Microsoft.SymbolStore/SymbolStores/SymbolStore.cs @@ -0,0 +1,107 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.SymbolStore.SymbolStores +{ + public abstract class SymbolStore : IDisposable + { + /// + /// Next symbol store to chain if this store refuses the request + /// + public SymbolStore BackingStore { get; } + + /// + /// Trace/logging source + /// + protected readonly ITracer Tracer; + + public SymbolStore(ITracer tracer) + { + Tracer = tracer; + } + + public SymbolStore(ITracer tracer, SymbolStore backingStore) + : this(tracer) + { + BackingStore = backingStore; + } + + /// + /// Downloads the file or retrieves it from a cache from the symbol store chain. + /// + /// symbol index to retrieve + /// to cancel requests + /// + /// Thrown for a pdb file when its checksum + /// does not match the expected value. + /// + /// file or null if not found + public async Task GetFile(SymbolStoreKey key, CancellationToken token) + { + SymbolStoreFile file = await GetFileInner(key, token).ConfigureAwait(false); + if (file == null) + { + if (BackingStore != null) + { + file = await BackingStore.GetFile(key, token).ConfigureAwait(false); + if (file != null) + { + await WriteFileInner(key, file).ConfigureAwait(false); + } + } + } + if (file != null) + { + // Reset stream to the beginning because the stream may have + // been read or written by the symbol store implementation. + file.Stream.Position = 0; + } + return file; + } + + protected virtual Task GetFileInner(SymbolStoreKey key, CancellationToken token) + { + return Task.FromResult(null); + } + + protected virtual Task WriteFileInner(SymbolStoreKey key, SymbolStoreFile file) + { + return Task.FromResult(0); + } + + public virtual void Dispose() + { + BackingStore?.Dispose(); + } + + /// + /// Compares two file paths using OS specific casing. + /// + internal static bool IsPathEqual(string path1, string path2) + { +#if !NET462 + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return string.Equals(path1, path2); + } +#endif + return StringComparer.OrdinalIgnoreCase.Equals(path1, path2); + } + + internal static int HashPath(string path) + { +#if !NET462 + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return path.GetHashCode(); + } +#endif + return StringComparer.OrdinalIgnoreCase.GetHashCode(path); + } + } +} diff --git a/src/Microsoft.SymbolStore/SymbolStores/SymwebSymbolStore.cs b/src/Microsoft.SymbolStore/SymbolStores/SymwebSymbolStore.cs new file mode 100644 index 000000000..858ea96fd --- /dev/null +++ b/src/Microsoft.SymbolStore/SymbolStores/SymwebSymbolStore.cs @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.SymbolStore.SymbolStores +{ + /// + /// The symbol store for the internal symweb symbol server that handles the "file.ptr" support. + /// + public sealed class SymwebHttpSymbolStore : HttpSymbolStore + { + /// + /// Create an instance of a http symbol store + /// + /// trace source for logging + /// next symbol store or null + /// symbol server url + /// PAT or null if no authentication + public SymwebHttpSymbolStore(ITracer tracer, SymbolStore backingStore, Uri symbolServerUri, string personalAccessToken = null) + : base(tracer, backingStore, symbolServerUri, personalAccessToken) + { + } + + protected override async Task GetFileInner(SymbolStoreKey key, CancellationToken token) + { + SymbolStoreFile file = await base.GetFileInner(key, token).ConfigureAwait(false); + if (file != null) + { + return file; + } + Uri filePtrUri = GetRequestUri(key.IndexPrefix + "file.ptr"); + Stream filePtrStream = await GetFileStream(key.FullPathName, filePtrUri, token).ConfigureAwait(false); + if (filePtrStream != null) + { + using (filePtrStream) + { + try + { + using (TextReader reader = new StreamReader(filePtrStream)) + { + string filePtr = await reader.ReadToEndAsync().ConfigureAwait(false); + Tracer.Verbose("SymwebHttpSymbolStore: file.ptr '{0}'", filePtr); + if (filePtr.StartsWith("PATH:")) + { + filePtr = filePtr.Replace("PATH:", ""); + Stream stream = File.OpenRead(filePtr); + return new SymbolStoreFile(stream, filePtr); + } + } + } + catch (Exception ex) when (ex is InvalidOperationException || ex is IOException) + { + Tracer.Error("SymwebHttpSymbolStore: {0}", ex.Message); + MarkClientFailure(); + } + } + } + return null; + } + } +} diff --git a/src/SOS/SOS.Extensions/SOS.Extensions.csproj b/src/SOS/SOS.Extensions/SOS.Extensions.csproj index 444196d7b..a1ecf2ce3 100644 --- a/src/SOS/SOS.Extensions/SOS.Extensions.csproj +++ b/src/SOS/SOS.Extensions/SOS.Extensions.csproj @@ -11,16 +11,16 @@ - - + + diff --git a/src/SOS/SOS.Hosting/SOS.Hosting.csproj b/src/SOS/SOS.Hosting/SOS.Hosting.csproj index df9961fb6..645c0ec26 100644 --- a/src/SOS/SOS.Hosting/SOS.Hosting.csproj +++ b/src/SOS/SOS.Hosting/SOS.Hosting.csproj @@ -11,11 +11,11 @@ - + diff --git a/src/Tools/dotnet-dump/dotnet-dump.csproj b/src/Tools/dotnet-dump/dotnet-dump.csproj index 169996540..4926daf5c 100644 --- a/src/Tools/dotnet-dump/dotnet-dump.csproj +++ b/src/Tools/dotnet-dump/dotnet-dump.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -18,7 +18,6 @@ - @@ -36,6 +35,7 @@ + diff --git a/src/Tools/dotnet-symbol/Program.cs b/src/Tools/dotnet-symbol/Program.cs new file mode 100644 index 000000000..19a9bbebe --- /dev/null +++ b/src/Tools/dotnet-symbol/Program.cs @@ -0,0 +1,598 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Diagnostic.Tools.Symbol.Properties; +using Microsoft.FileFormats; +using Microsoft.FileFormats.ELF; +using Microsoft.FileFormats.MachO; +using Microsoft.FileFormats.PE; +using Microsoft.SymbolStore; +using Microsoft.SymbolStore.KeyGenerators; +using Microsoft.SymbolStore.SymbolStores; + +namespace Microsoft.Diagnostics.Tools.Symbol +{ + public class Program + { + private struct ServerInfo + { + public Uri Uri; + public string PersonalAccessToken; + public bool InternalSymwebServer; + } + + private readonly List InputFilePaths = new(); + private readonly List CacheDirectories = new(); + private readonly List SymbolServers = new(); + private string OutputDirectory; + private TimeSpan? Timeout; + private bool Overwrite; + private bool Subdirectories; + private bool Symbols; + private bool Debugging; + private bool Modules; + private bool ForceWindowsPdbs; + private bool HostOnly; + private bool VerifyCore; + private Tracer Tracer; + + public static void Main(string[] args) + { + if (args.Length == 0) + { + goto usage; + } + Program program = new(); + Tracer tracer = new(); + program.Tracer = tracer; + + for (int i = 0; i < args.Length; i++) + { + string personalAccessToken = null; + Uri uri; + switch (args[i]) + { + case "--microsoft-symbol-server": + Uri.TryCreate("https://msdl.microsoft.com/download/symbols/", UriKind.Absolute, out uri); + program.SymbolServers.Add(new ServerInfo { Uri = uri, PersonalAccessToken = null }); + break; + + case "--internal-server": + Uri.TryCreate("https://symweb/", UriKind.Absolute, out uri); + program.SymbolServers.Add(new ServerInfo { Uri = uri, PersonalAccessToken = null, InternalSymwebServer = true }); + break; + + case "--authenticated-server-path": + if (++i < args.Length) + { + personalAccessToken = args[i]; + } + else + { + goto usage; + } + if (string.IsNullOrEmpty(personalAccessToken)) + { + tracer.Error("No personal access token option"); + goto usage; + } + goto case "--server-path"; + + case "--server-path": + if (++i < args.Length) + { + // Make sure the server Uri ends with "/" + string serverPath = args[i].TrimEnd('/') + '/'; + if (!Uri.TryCreate(serverPath, UriKind.Absolute, out uri) || uri.IsFile) + { + tracer.Error(Resources.InvalidServerPath, args[i]); + goto usage; + } + Uri.TryCreate(serverPath, UriKind.Absolute, out uri); + program.SymbolServers.Add(new ServerInfo { Uri = uri, PersonalAccessToken = personalAccessToken }); + } + else + { + goto usage; + } + break; + + case "-o": + case "--output": + if (++i < args.Length) + { + program.OutputDirectory = args[i]; + } + else + { + goto usage; + } + break; + + case "--overwrite": + program.Overwrite = true; + break; + + case "--timeout": + if (++i < args.Length) + { + double timeoutInMinutes = double.Parse(args[i]); + program.Timeout = TimeSpan.FromMinutes(timeoutInMinutes); + } + else + { + goto usage; + } + break; + + case "--cache-directory": + if (++i < args.Length) + { + program.CacheDirectories.Add(args[i]); + } + else + { + goto usage; + } + break; + + case "--recurse-subdirectories": + program.Subdirectories = true; + break; + + case "--modules": + program.Modules = true; + break; + + case "--symbols": + program.Symbols = true; + break; + + case "--debugging": + program.Debugging = true; + break; + + case "--windows-pdbs": + program.ForceWindowsPdbs = true; + break; + + case "--host-only": + program.HostOnly = true; + break; + + case "--verifycore": + program.VerifyCore = true; + break; + + case "-d": + case "--diagnostics": + tracer.Enabled = true; + tracer.EnabledVerbose = true; + break; + + case "-h": + case "-?": + case "--help": + goto usage; + + default: + string inputFile = args[i]; + if (inputFile.StartsWith("-") || inputFile.StartsWith("--")) + { + tracer.Error(Resources.InvalidCommandLineOption, inputFile); + goto usage; + } + program.InputFilePaths.Add(inputFile); + break; + } + } + if (program.VerifyCore) + { + program.VerifyCoreDump(); + } + else + { + // Default to public Microsoft symbol server + if (program.SymbolServers.Count == 0) + { + Uri.TryCreate("https://msdl.microsoft.com/download/symbols/", UriKind.Absolute, out Uri uri); + program.SymbolServers.Add(new ServerInfo { Uri = uri, PersonalAccessToken = null }); + } + foreach (ServerInfo server in program.SymbolServers) + { + tracer.WriteLine(Resources.DownloadFromUri, server.Uri); + } + if (program.OutputDirectory != null) + { + Directory.CreateDirectory(program.OutputDirectory); + tracer.WriteLine(Resources.WritingFilesToOutput, program.OutputDirectory); + } + try + { + program.DownloadFiles().GetAwaiter().GetResult(); + } + catch (Exception ex) + { + tracer.Error("{0}{1}", ex.Message, ex.InnerException != null ? " -> " + ex.InnerException.Message : ""); + } + } + return; + + usage: + PrintUsage(); + } + + private static void PrintUsage() + { + Console.WriteLine(); + Console.WriteLine(Resources.UsageOptions); + } + + internal async Task DownloadFiles() + { + using (Microsoft.SymbolStore.SymbolStores.SymbolStore symbolStore = BuildSymbolStore()) + { + foreach (SymbolStoreKeyWrapper wrapper in GetKeys().Distinct()) + { + SymbolStoreKey key = wrapper.Key; + if (symbolStore != null) + { + using (SymbolStoreFile symbolFile = await symbolStore.GetFile(key, CancellationToken.None).ConfigureAwait(false)) + { + if (symbolFile != null) + { + await WriteFile(symbolFile, wrapper).ConfigureAwait(false); + } + } + } + } + } + } + + private Microsoft.SymbolStore.SymbolStores.SymbolStore BuildSymbolStore() + { + Microsoft.SymbolStore.SymbolStores.SymbolStore store = null; + + foreach (ServerInfo server in ((IEnumerable)SymbolServers).Reverse()) + { + if (server.InternalSymwebServer) + { + store = new SymwebHttpSymbolStore(Tracer, store, server.Uri, server.PersonalAccessToken); + } + else + { + store = new HttpSymbolStore(Tracer, store, server.Uri, server.PersonalAccessToken); + } + if (Timeout.HasValue && store is HttpSymbolStore http) + { + http.Timeout = Timeout.Value; + } + } + + // Add default symbol cache if one wasn't set by the command line + if (CacheDirectories.Count == 0) + { + CacheDirectories.Add(GetDefaultSymbolCache()); + } + + foreach (string cache in ((IEnumerable)CacheDirectories).Reverse()) + { + store = new CacheSymbolStore(Tracer, store, cache); + } + + return store; + } + + private sealed class SymbolStoreKeyWrapper + { + public readonly SymbolStoreKey Key; + public readonly string InputFile; + + internal SymbolStoreKeyWrapper(SymbolStoreKey key, string inputFile) + { + Key = key; + InputFile = inputFile; + } + + /// + /// Returns the hash of the index. + /// + public override int GetHashCode() + { + return Key.GetHashCode(); + } + + /// + /// Only the index is compared or hashed. The FileName is already + /// part of the index. + /// + public override bool Equals(object obj) + { + SymbolStoreKeyWrapper wrapper = (SymbolStoreKeyWrapper)obj; + return Key.Equals(wrapper.Key); + } + } + + private IEnumerable GetKeys() + { + IEnumerable inputFiles = GetInputFiles(); + + foreach (string inputFile in inputFiles) + { + foreach (KeyGenerator generator in GetKeyGenerators(inputFile)) + { + KeyTypeFlags flags = KeyTypeFlags.None; + if (HostOnly) + { + flags |= KeyTypeFlags.HostKeys; + } + if (Symbols) + { + flags |= KeyTypeFlags.SymbolKey | KeyTypeFlags.PerfMapKeys; + } + if (Modules) + { + flags |= KeyTypeFlags.IdentityKey; + } + if (Debugging) + { + flags |= KeyTypeFlags.RuntimeKeys | KeyTypeFlags.ClrKeys; + } + if (flags == KeyTypeFlags.None) + { + if (generator.IsDump()) + { + // The default for dumps is to download everything + flags = KeyTypeFlags.IdentityKey | KeyTypeFlags.SymbolKey | KeyTypeFlags.ClrKeys | KeyTypeFlags.HostKeys; + } + else + { + // Otherwise the default is just the symbol files + flags = KeyTypeFlags.SymbolKey | KeyTypeFlags.PerfMapKeys; + } + } + if (ForceWindowsPdbs) + { + flags |= KeyTypeFlags.ForceWindowsPdbs; + } + foreach (SymbolStoreKeyWrapper wrapper in generator.GetKeys(flags).Select((key) => new SymbolStoreKeyWrapper(key, inputFile))) + { + yield return wrapper; + } + } + } + } + + private IEnumerable GetKeyGenerators(string inputFile) + { + using (Stream inputStream = File.Open(inputFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + SymbolStoreFile file = new(inputStream, inputFile); + string extension = Path.GetExtension(inputFile); + yield return new FileKeyGenerator(Tracer, file); + } + } + + private async Task WriteFile(SymbolStoreFile file, SymbolStoreKeyWrapper wrapper) + { + if (OutputDirectory != null) + { + await WriteFileToDirectory(file.Stream, wrapper.Key.FullPathName, OutputDirectory).ConfigureAwait(false); + } + else + { + await WriteFileToDirectory(file.Stream, wrapper.Key.FullPathName, Path.GetDirectoryName(wrapper.InputFile)).ConfigureAwait(false); + } + } + + private async Task WriteFileToDirectory(Stream stream, string fileName, string destinationDirectory) + { + stream.Position = 0; + string destination = Path.Combine(destinationDirectory, Path.GetFileName(fileName.Replace('\\', '/'))); + if (!Overwrite && File.Exists(destination)) + { + Tracer.WriteLine(Resources.FileAlreadyExists, destination); + } + else + { + Tracer.WriteLine(Resources.WritingFile, destination); + using (Stream destinationStream = File.OpenWrite(destination)) + { + await stream.CopyToAsync(destinationStream).ConfigureAwait(false); + } + } + } + + private static string GetDefaultSymbolCache() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return Path.Combine(Path.GetTempPath(), "SymbolCache"); + } + else + { + return Path.Combine(Environment.GetEnvironmentVariable("HOME"), ".dotnet", "symbolcache"); + } + } + + internal void VerifyCoreDump() + { + foreach (string inputFile in GetInputFiles()) + { + Console.WriteLine($"{inputFile}"); + + using Stream inputStream = File.Open(inputFile, FileMode.Open, FileAccess.Read, FileShare.Read); + StreamAddressSpace dataSource = new(inputStream); + ELFCoreFile core = new(dataSource); + + if (Tracer.Enabled) + { + foreach (ELFProgramSegment segment in core.Segments) + { + Tracer.Information("{0:X16}-{1:X16} {2:X8} {3:X8} {4}", + segment.Header.VirtualAddress.Value, + segment.Header.VirtualAddress + segment.Header.VirtualSize, + segment.Header.FileOffset.Value, + (ulong)segment.Header.FileSize, + segment.Header.Type); + } + } + + foreach (ELFLoadedImage image in core.LoadedImages) + { + Console.WriteLine("{0:X16} {1}", image.LoadAddress, image.Path); + Exception elfException = null; + Exception machoException = null; + Exception peException = null; + try + { + ELFFile elfFile = image.Image; + if (elfFile.IsValid()) + { + try + { + byte[] buildid = elfFile.BuildID; + } + catch (Exception ex) + { + Console.WriteLine(" ELF file invalid build id - {0}", ex.Message); + } + foreach (ELFProgramSegment segment in elfFile.Segments) + { + Tracer.Verbose(" {0:X16}-{1:X16} file off {2:X8} file size {3:X8} {4}", + segment.Header.VirtualAddress.Value, + segment.Header.VirtualAddress + segment.Header.VirtualSize, + segment.Header.FileOffset.Value, + (ulong)segment.Header.FileSize, + segment.Header.Type); + + if (segment.Header.Type == ELFProgramHeaderType.Note || + segment.Header.Type == ELFProgramHeaderType.Dynamic || + segment.Header.Type == ELFProgramHeaderType.GnuEHFrame) + { + try + { + byte[] data = segment.Contents.Read(0, (uint)segment.Header.VirtualSize); + } + catch (Exception ex) + { + Console.WriteLine(" ELF file segment {0} virt addr {1:X16} virt size {2:X8} INVALID - {3}", + segment.Header.Type, segment.Header.VirtualAddress, segment.Header.VirtualSize, ex.Message); + } + } + } + + // The ELF module was valid try next module + continue; + } + } + catch (Exception ex) + { + elfException = ex; + } + + IAddressSpace addressSpace = new RelativeAddressSpace(core.DataSource, image.LoadAddress, core.DataSource.Length); + try + { + MachOFile machoFile = new(addressSpace); + if (machoFile.IsValid()) + { + try + { + byte[] uuid = machoFile.Uuid; + } + catch (Exception ex) + { + Console.WriteLine(" MachO file invalid uuid - {0}", ex.Message); + } + foreach (MachSegment segment in machoFile.Segments) + { + Tracer.Verbose(" {0:X16}-{1:X16} offset {2:X16} size {3:X16} {4} {5}", + (ulong)segment.LoadCommand.VMAddress, + segment.LoadCommand.VMAddress + segment.LoadCommand.VMSize, + (ulong)segment.LoadCommand.FileOffset, + (ulong)segment.LoadCommand.FileSize, + segment.LoadCommand.Command, + segment.LoadCommand.SegName); + + foreach (MachSection section in segment.Sections) + { + Tracer.Verbose(" addr {0:X16} size {1:X16} offset {2:X8} {3}", + (ulong)section.Address, + (ulong)section.Size, + section.Offset, + section.SectionName); + } + } + + // The MachO module was valid try next module + continue; + } + } + catch (Exception ex) + { + machoException = ex; + } + + try + { + PEFile peFile = new(addressSpace, true); + if (peFile.IsValid()) + { + // The PE module was valid try next module + continue; + } + } + catch (Exception ex) + { + peException = ex; + } + + Console.WriteLine("{0:X16} invalid image - {1}", image.LoadAddress, image.Path); + if (elfException != null) + { + Tracer.Verbose("ELF {0}", elfException.Message); + } + if (machoException != null) + { + Tracer.Verbose("MachO {0}", machoException.Message); + } + if (peException != null) + { + Tracer.Verbose("PE {0}", peException.Message); + } + } + + ulong segmentsTotal = core.Segments.Max(s => s.Header.FileOffset + s.Header.FileSize); + if (segmentsTotal > dataSource.Length) + { + Console.WriteLine($"ERROR: Core file not complete: file size 0x{dataSource.Length:X8} segments total 0x{segmentsTotal:X8}"); + } + } + } + + private IEnumerable GetInputFiles() + { + IEnumerable inputFiles = InputFilePaths.SelectMany((string file) => + { + string directory = Path.GetDirectoryName(file); + string pattern = Path.GetFileName(file); + return Directory.EnumerateFiles(string.IsNullOrWhiteSpace(directory) ? "." : directory, pattern, + Subdirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly); + }); + + if (!inputFiles.Any()) + { + throw new ArgumentException("Input files not found"); + } + return inputFiles; + } + } +} diff --git a/src/Tools/dotnet-symbol/Properties/Resources.Designer.cs b/src/Tools/dotnet-symbol/Properties/Resources.Designer.cs new file mode 100644 index 000000000..ef6e9daad --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/Resources.Designer.cs @@ -0,0 +1,144 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.Diagnostic.Tools.Symbol.Properties { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Microsoft.Diagnostic.Tools.Symbol.Properties.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to Downloading from {0}. + /// + internal static string DownloadFromUri { + get { + return ResourceManager.GetString("DownloadFromUri", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {0} already exists, file not written. + /// + internal static string FileAlreadyExists { + get { + return ResourceManager.GetString("FileAlreadyExists", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Invalid option {0}. + /// + internal static string InvalidCommandLineOption { + get { + return ResourceManager.GetString("InvalidCommandLineOption", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Invalid server path '{0}'. + /// + internal static string InvalidServerPath { + get { + return ResourceManager.GetString("InvalidServerPath", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Input files not found. + /// + internal static string NoInputFiles { + get { + return ResourceManager.GetString("NoInputFiles", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Usage: dotnet symbol [options] <FILES> + /// + ///Arguments: + /// <FILES> List of files. Can contain wildcards. + /// + ///Options: + /// --microsoft-symbol-server Add 'https://msdl.microsoft.com/download/symbols' symbol server path (default). + /// --internal-server Add 'https://symweb' internal symbol server path. + /// --server-path <symbol server path> Add a http server path. + /// --authenticated-server-path <pat> <server path> Add a http PAT authenticated s [rest of string was truncated]";. + /// + internal static string UsageOptions { + get { + return ResourceManager.GetString("UsageOptions", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Writing: {0}. + /// + internal static string WritingFile { + get { + return ResourceManager.GetString("WritingFile", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Writing files to {0}. + /// + internal static string WritingFilesToOutput { + get { + return ResourceManager.GetString("WritingFilesToOutput", resourceCulture); + } + } + } +} diff --git a/src/Tools/dotnet-symbol/Properties/Resources.resx b/src/Tools/dotnet-symbol/Properties/Resources.resx new file mode 100644 index 000000000..cba536031 --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/Resources.resx @@ -0,0 +1,166 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Downloading from {0} + + + {0} already exists, file not written + + + Invalid option {0} + + + Invalid server path '{0}' + + + Input files not found + + + Usage: dotnet symbol [options] <FILES> + +Arguments: + <FILES> List of files. Can contain wildcards. + +Options: + --microsoft-symbol-server Add 'https://msdl.microsoft.com/download/symbols' symbol server path (default). + --internal-server Add 'https://symweb' internal symbol server path. + --server-path <symbol server path> Add a http server path. + --authenticated-server-path <pat> <server path> Add a http PAT authenticated server path. + --cache-directory <file cache directory> Add a cache directory. + --timeout <minutes> Change http timeout in minutes (default: 4). + --recurse-subdirectories Process input files in all subdirectories. + --host-only Download only the host program (i.e. dotnet) that lldb needs for loading coredumps. + --symbols Download the symbol files (.pdb, .dbg, .dwarf). + --modules Download the module files (.dll, .so, .dylib). + --debugging Download the special debugging modules (DAC, DBI, SOS). + --windows-pdbs Force the downloading of the Windows PDBs when Portable PDBs are also available. + --overwrite Overwrite existing files in output directory. + --verifycore Verify ELF core dump. + -o, --output <output directory> Set the output directory. Otherwise, write next to the input file (default). + -d, --diagnostics Enable diagnostic output. + -h, --help Show help information. + + + Writing: {0} + + + Writing files to {0} + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.cs.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.cs.xlf new file mode 100644 index 000000000..f1c6d53b2 --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.cs.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.de.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.de.xlf new file mode 100644 index 000000000..aa9c55af8 --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.de.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.es.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.es.xlf new file mode 100644 index 000000000..0cd0e95bd --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.es.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.fr.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.fr.xlf new file mode 100644 index 000000000..2bc067b5f --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.fr.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.it.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.it.xlf new file mode 100644 index 000000000..965c242c9 --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.it.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.ja.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.ja.xlf new file mode 100644 index 000000000..7d95ef807 --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.ja.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.ko.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.ko.xlf new file mode 100644 index 000000000..da107ac18 --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.ko.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.pl.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.pl.xlf new file mode 100644 index 000000000..7eadfc3e0 --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.pl.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.pt-BR.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.pt-BR.xlf new file mode 100644 index 000000000..4c343afc6 --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.pt-BR.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.ru.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.ru.xlf new file mode 100644 index 000000000..0d3618015 --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.ru.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.tr.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.tr.xlf new file mode 100644 index 000000000..e4bf0b86f --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.tr.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.zh-Hans.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.zh-Hans.xlf new file mode 100644 index 000000000..a78e3c27c --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.zh-Hans.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/Properties/xlf/Resources.zh-Hant.xlf b/src/Tools/dotnet-symbol/Properties/xlf/Resources.zh-Hant.xlf new file mode 100644 index 000000000..fa7de62c0 --- /dev/null +++ b/src/Tools/dotnet-symbol/Properties/xlf/Resources.zh-Hant.xlf @@ -0,0 +1,42 @@ + + + + + + "Downloading from {0}" + "Downloading from {0}" + + + + "{0} already exists, file not written" + "{0} already exists, file not written" + + + + "Invalid option {0}" + "Invalid option {0}" + + + + "Invalid server path '{0}'" + "Invalid server path '{0}'" + + + + "Input files not found" + "Input files not found" + + + + "Writing: {0}" + "Writing: {0}" + + + + "Writing files to {0}" + "Writing files to {0}" + + + + + \ No newline at end of file diff --git a/src/Tools/dotnet-symbol/README.md b/src/Tools/dotnet-symbol/README.md new file mode 100644 index 000000000..110c0c3ba --- /dev/null +++ b/src/Tools/dotnet-symbol/README.md @@ -0,0 +1,90 @@ +# Symbol downloader dotnet cli extension # + +This tool can download all the files needed for debugging (symbols, modules, SOS and DAC for the coreclr module given) for any given core dump, minidump or any supported platform's file formats like ELF, MachO, Windows DLLs, PDBs and portable PDBs. See [debugging coredumps](https://github.com/dotnet/diagnostics/blob/main/documentation/debugging-coredump.md) for more details. + + Usage: dotnet symbol [options] + + Arguments: + List of files. Can contain wildcards. + + Options: + --microsoft-symbol-server Add 'https://msdl.microsoft.com/download/symbols' symbol server path (default). + --internal-server Add 'https://symweb' internal symbol server path. + --server-path Add a http server path. + --authenticated-server-path Add a http PAT authenticated server path. + --cache-directory Add a cache directory. + --recurse-subdirectories Process input files in all subdirectories. + --host-only Download only the host program (i.e. dotnet) that lldb needs for loading coredumps. + --symbols Download the symbol files (.pdb, .dbg, .dwarf). + --modules Download the module files (.dll, .so, .dylib). + --debugging Download the special debugging modules (DAC, DBI, SOS). + --windows-pdbs Force the downloading of the Windows PDBs when Portable PDBs are also available. + -o, --output Set the output directory. Otherwise, write next to the input file (default). + -d, --diagnostics Enable diagnostic output. + -h, --help Show help information. + +## Install ## + +This is a dotnet global tool "extension" supported only by [.NET Core 2.1](https://www.microsoft.com/net/download/) or greater. The latest version of the downloader can be installed with the following command. Make sure you are not in any project directory with a NuGet.Config that doesn't include nuget.org as a source. See the Notes section about any errors. + + dotnet tool install -g dotnet-symbol + +If you already have dotnet-symbol installed you can update it with: + + dotnet tool update -g dotnet-symbol + +## Examples ## + +This will attempt to download all the modules, symbols and DAC/DBI files needed to debug the core dump including the managed assemblies and their PDBs if Linux/ELF core dump or Windows minidump: + + dotnet-symbol coredump.4507 + +This downloads just the host program needed to load a core dump on Linux or macOS under lldb. SOS under lldb can download the rest of the symbols and modules needed on demand or with the "loadsymbols" command. See [debugging coredumps](https://github.com/dotnet/diagnostics/blob/main/documentation/debugging-coredump.md) for more details. + + dotnet-symbol --host-only coredump.4507 + +To download the symbol files for a specific assembly: + + dotnet-symbol --symbols --cache-directory c:\temp\symcache --server-path https://symweb --output c:\temp\symout System.Threading.dll + +Downloads all the symbol files for the shared runtime: + + dotnet-symbol --symbols --output /tmp/symbols /usr/share/dotnet/shared/Microsoft.NETCore.App/2.0.3/* + +After the symbols are downloaded to `/tmp/symbols` they can be copied back to the above runtime directory so the native debuggers like lldb or gdb can find them, but the copy needs to be superuser: + + sudo cp /tmp/symbols/* /usr/share/dotnet/shared/Microsoft.NETCore.App/2.0.3 + +To verify a symbol package on a local VSTS symbol server: + + dotnet-symbol --authenticated-server-path x349x9dfkdx33333livjit4wcvaiwc3v4wjyvnq https://mikemvsts.artifacts.visualstudio.com/defaultcollection/_apis/Symbol/symsrv coredump.45634 + +## Notes ## + +Symbol download is only supported for official .NET Core runtime versions acquired through official channels such as [the official web site](https://dotnet.microsoft.com/download/dotnet-core) and the [default sources in the dotnet installation scripts](https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-scripts). Runtimes obtained from community sites like [archlinux](https://www.archlinux.org/packages/community/x86_64/dotnet-runtime/) are not supported. + +Core dumps generated with gdb (generate-core-file command) or gcore (utility that comes with gdb) do not currently work with this utility (issue [#47](https://github.com/dotnet/symstore/issues/47)). + +The best way to generate core dumps on Linux (not supported on Windows or MacOS) is to use the [createdump](https://github.com/dotnet/runtime/blob/main/docs/design/coreclr/botr/xplat-minidump-generation.md) facility that is part of .NET Core 2.0 and greater. It can be setup (see [createdump](https://github.com/dotnet/runtime/blob/main/docs/design/coreclr/botr/xplat-minidump-generation.md#configurationpolicy) for the details) to automatically generate a "minidump" like ELF core dump when your .NET Core app crashes. The coredump will contain all the necessary managed state to analyze with SOS or dotnet-dump. + + Linux system core generation (enabled with `ulimit -c unlimited`) also works if the coredump_filter flags are set (see [core](http://man7.org/linux/man-pages/man5/core.5.html)) to at least 0x3f but they are usually a lot larger than necessary. +``` +echo 0x3f > /proc/self/coredump_filter +ulimit -c unlimited +``` + + +If you receive the below error when installing the extension, you are in a project or directory that contains a NuGet.Config that doesn't contain nuget.org. + + error NU1101: Unable to find package dotnet-symbol. No packages exist with this id in source(s): ... + The tool package could not be restored. + Tool 'dotnet-symbol' failed to install. This failure may have been caused by: + + * You are attempting to install a preview release and did not use the --version option to specify the version. + * A package by this name was found, but it was not a .NET Core tool. + * The required NuGet feed cannot be accessed, perhaps because of an Internet connection problem. + * You mistyped the name of the tool. + +You can either run the install command from your $HOME or %HOME% directory or override this behavior with the `--add-source` option: + +`dotnet tool install -g --add-source https://api.nuget.org/v3/index.json dotnet-symbol` diff --git a/src/Tools/dotnet-symbol/Tracer.cs b/src/Tools/dotnet-symbol/Tracer.cs new file mode 100644 index 000000000..bb5578723 --- /dev/null +++ b/src/Tools/dotnet-symbol/Tracer.cs @@ -0,0 +1,84 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Diagnostics.Tools.Symbol +{ + /// + /// Simple trace/logging support. + /// + internal sealed class Tracer : Microsoft.SymbolStore.ITracer + { + public bool Enabled; + public bool EnabledVerbose; + + public void WriteLine(string message) + { + Console.WriteLine(message); + } + + public void WriteLine(string format, params object[] arguments) + { + Console.WriteLine(format, arguments); + } + + public void Information(string message) + { + if (Enabled) + { + Console.WriteLine(message); + } + } + + public void Information(string format, params object[] arguments) + { + if (Enabled) + { + Console.WriteLine(format, arguments); + } + } + + public void Warning(string message) + { + if (Enabled) + { + Console.WriteLine("WARNING: " + message); + } + } + + public void Warning(string format, params object[] arguments) + { + if (Enabled) + { + Console.WriteLine("WARNING: " + format, arguments); + } + } + + public void Error(string message) + { + Console.WriteLine("ERROR: " + message); + } + + public void Error(string format, params object[] arguments) + { + Console.WriteLine("ERROR: " + format, arguments); + } + + public void Verbose(string message) + { + if (EnabledVerbose) + { + Console.WriteLine(message); + } + } + + public void Verbose(string format, params object[] arguments) + { + if (EnabledVerbose) + { + Console.WriteLine(format, arguments); + } + } + } +} diff --git a/src/Tools/dotnet-symbol/dotnet-symbol.csproj b/src/Tools/dotnet-symbol/dotnet-symbol.csproj new file mode 100644 index 000000000..86275cc92 --- /dev/null +++ b/src/Tools/dotnet-symbol/dotnet-symbol.csproj @@ -0,0 +1,32 @@ + + + net6.0 + dotnet-symbol + Microsoft.Diagnostic.Tools.Symbol + Symbols download utility + Symbols + $(Description) + ;1591;1701 + + + + + All + + + + + + True + True + Resources.resx + + + + + + ResXFileCodeGenerator + Resources.Designer.cs + + + diff --git a/src/Tools/dotnet-symbol/runtimeconfig.template.json b/src/Tools/dotnet-symbol/runtimeconfig.template.json new file mode 100644 index 000000000..f022b7ffc --- /dev/null +++ b/src/Tools/dotnet-symbol/runtimeconfig.template.json @@ -0,0 +1,3 @@ +{ + "rollForwardOnNoCandidateFx": 2 +} \ No newline at end of file diff --git a/src/tests/Microsoft.FileFormats.UnitTests/AddressSpace.cs b/src/tests/Microsoft.FileFormats.UnitTests/AddressSpace.cs new file mode 100644 index 000000000..472a4c11c --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/AddressSpace.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq; +using Xunit; + +namespace Microsoft.FileFormats.Tests +{ + public class AddressSpace + { + [Fact] + public void GoodReads() + { + MemoryBufferAddressSpace buffer = new MemoryBufferAddressSpace(new byte[] { 1, 2, 3, 4, 5 }); + Assert.True(Enumerable.SequenceEqual(new byte[] { 1 }, buffer.Read(0, 1))); + Assert.True(Enumerable.SequenceEqual(new byte[] { 3, 4 }, buffer.Read(2, 2))); + Assert.True(Enumerable.SequenceEqual(new byte[] { 1, 2, 3, 4, 5 }, buffer.Read(0, 5))); + Assert.True(Enumerable.SequenceEqual(new byte[0], buffer.Read(0, 0))); + Assert.True(Enumerable.SequenceEqual(new byte[0], buffer.Read(4, 0))); + } + + [Fact] + public void BadReads() + { + MemoryBufferAddressSpace buffer = new MemoryBufferAddressSpace(new byte[] { 1, 2, 3, 4, 5 }); + Assert.Throws(() => + { + buffer.Read(5, 1); + }); + Assert.Throws(() => + { + buffer.Read(5, 0); + }); + Assert.Throws(() => + { + buffer.Read(3, 3); + }); + } + } +} diff --git a/src/tests/Microsoft.FileFormats.UnitTests/ELF/Tests.cs b/src/tests/Microsoft.FileFormats.UnitTests/ELF/Tests.cs new file mode 100644 index 000000000..fa2c32115 --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/ELF/Tests.cs @@ -0,0 +1,118 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; +using System.Linq; +using TestHelpers; +using Xunit; + +namespace Microsoft.FileFormats.ELF.Tests +{ + public class Tests + { + [Fact] + public void CheckIndexingInfo() + { + using (Stream libcoreclr = TestUtilities.OpenCompressedFile("TestBinaries/libcoreclr.so.gz")) + { + StreamAddressSpace dataSource = new(libcoreclr); + ELFFile elf = new(dataSource); + Assert.True(elf.IsValid()); + Assert.True(elf.Header.Type == ELFHeaderType.Shared); + string buildId = TestUtilities.ToHexString(elf.BuildID); + + //this is the build id for libcoreclr.so from package: + // https://dotnet.myget.org/feed/dotnet-core/package/nuget/runtime.ubuntu.14.04-x64.Microsoft.NETCore.Runtime.CoreCLR/2.0.0-preview3-25428-01 + Assert.Equal("ef8f58a0b402d11c68f78342ef4fcc7d23798d4c", buildId); + } + + // 32 bit arm ELF binary + using (Stream apphost = TestUtilities.OpenCompressedFile("TestBinaries/apphost.gz")) + { + StreamAddressSpace dataSource = new(apphost); + ELFFile elf = new(dataSource); + Assert.True(elf.IsValid()); + Assert.True(elf.Header.Type == ELFHeaderType.Executable); + string buildId = TestUtilities.ToHexString(elf.BuildID); + + //this is the build id for apphost from package: + // https://dotnet.myget.org/F/dotnet-core/symbols/runtime.linux-arm.Microsoft.NETCore.DotNetAppHost/2.1.0-preview2-25512-03 + Assert.Equal("316d55471a8d5ebd6f2cb0631f0020518ab13dc0", buildId); + } + } + + [Fact] + public void CheckDbgIndexingInfo() + { + using (Stream stream = TestUtilities.OpenCompressedFile("TestBinaries/libcoreclrtraceptprovider.so.dbg.gz")) + { + StreamAddressSpace dataSource = new(stream); + ELFFile elf = new(dataSource); + Assert.True(elf.IsValid()); + Assert.True(elf.Header.Type == ELFHeaderType.Shared); + string buildId = TestUtilities.ToHexString(elf.BuildID); + Assert.Equal("ce4ce0558d878a05754dff246ccea2a70a1db3a8", buildId); + } + } + + [Fact] + public void CheckFreeBSDIndexingInfo() + { + using (Stream stream = File.OpenRead("TestBinaries/ilasm.dbg")) + { + StreamAddressSpace dataSource = new(stream); + ELFFile elf = new(dataSource); + Assert.True(elf.IsValid()); + Assert.True(elf.Header.Type == ELFHeaderType.Executable); + string buildId = TestUtilities.ToHexString(elf.BuildID); + Assert.Equal("4a91e41002a1307ef4097419d7875df001969daa", buildId); + } + } + + [Fact] + public void CheckCustomNamedBuildIdSection() + { + using (Stream stream = File.OpenRead("TestBinaries/renamed_build_id_section")) + { + StreamAddressSpace dataSource = new(stream); + ELFFile elf = new(dataSource); + Assert.True(elf.IsValid()); + Assert.True(elf.Header.Type == ELFHeaderType.Shared); + string buildId = TestUtilities.ToHexString(elf.BuildID); + Assert.Equal("1bd6a199dcb6f234558d9439cfcbba2727f1e1d9", buildId); + } + } + + [Fact] + public void ParseCore() + { + using (Stream core = TestUtilities.OpenCompressedFile("TestBinaries/core.gz")) + { + StreamAddressSpace dataSource = new(core); + ELFCoreFile coreReader = new(dataSource); + Assert.True(coreReader.IsValid()); + ELFLoadedImage loadedImage = coreReader.LoadedImages.Where(i => i.Path.EndsWith("librt-2.17.so")).First(); + Assert.True(loadedImage.Image.IsValid()); + Assert.True(loadedImage.Image.Header.Type == ELFHeaderType.Shared); + string buildId = TestUtilities.ToHexString(loadedImage.Image.BuildID); + Assert.Equal("1d2ad4eaa62bad560685a4b8dccc8d9aa95e22ce", buildId); + } + } + + [Fact] + public void ParseTriageDump() + { + using (Stream core = TestUtilities.OpenCompressedFile("TestBinaries/triagedump.gz")) + { + StreamAddressSpace dataSource = new(core); + ELFCoreFile coreReader = new(dataSource); + Assert.True(coreReader.IsValid()); + ELFLoadedImage loadedImage = coreReader.LoadedImages.Where(i => i.Path.EndsWith("libcoreclr.so")).First(); + Assert.True(loadedImage.Image.IsValid()); + Assert.True(loadedImage.Image.Header.Type == ELFHeaderType.Shared); + string buildId = TestUtilities.ToHexString(loadedImage.Image.BuildID); + Assert.Equal("8f39a52a756311ab365090bfe9edef7ee8c44503", buildId); + } + } + } +} diff --git a/src/tests/Microsoft.FileFormats.UnitTests/Layouts.cs b/src/tests/Microsoft.FileFormats.UnitTests/Layouts.cs new file mode 100644 index 000000000..b5f37e1c9 --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/Layouts.cs @@ -0,0 +1,145 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace Microsoft.FileFormats.Tests +{ + public class Layouts + { + [Fact] + public void ReadPrimitives() + { + MemoryBufferAddressSpace dataSource = new(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }); + Reader reader = new Reader(dataSource); + Assert.Equal(0x0201, reader.Read(0)); + Assert.Equal(0x5, reader.Read(4)); + Assert.Equal((uint)0x08070605, reader.Read(4)); + } + +#pragma warning disable 0649 + private class SimpleStruct : TStruct + { + public int X; + public short Y; + } + + [Fact] + public void ReadTStruct() + { + MemoryBufferAddressSpace dataSource = new(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }); + Reader reader = new(dataSource); + SimpleStruct s = reader.Read(1); + Assert.Equal(0x05040302, s.X); + Assert.Equal(0x0706, s.Y); + } + + private class DerivedStruct : SimpleStruct + { + public int Z; + } + + [Fact] + public void ReadDerivedTStruct() + { + MemoryBufferAddressSpace dataSource = new(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }); + Reader reader = new(dataSource); + DerivedStruct s = reader.Read(1); + Assert.Equal(0x05040302, s.X); + Assert.Equal(0x0706, s.Y); + Assert.Equal(0x0d0c0b0a, s.Z); + } + + private class ArrayStruct : TStruct + { + [ArraySize(3)] + public short[] array; + public int X; + } + + [Fact] + public void ReadArrayTStructTest() + { + MemoryBufferAddressSpace dataSource = new(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }); + Reader reader = new(dataSource); + ArrayStruct s = reader.Read(1); + Assert.Equal(3, s.array.Length); + Assert.Equal(0x0302, s.array[0]); + Assert.Equal(0x0504, s.array[1]); + Assert.Equal(0x0706, s.array[2]); + Assert.Equal(0x0d0c0b0a, s.X); + } + + private enum FooEnum : ushort + { + ThreeTwo = 0x0302 + } + + private class EnumStruct : TStruct + { + public FooEnum E; + public int X; + } + + [Fact] + public void EnumTStructTest() + { + MemoryBufferAddressSpace dataSource = new(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }); + Reader reader = new(dataSource); + EnumStruct s = reader.Read(1); + Assert.Equal(FooEnum.ThreeTwo, s.E); + Assert.Equal(0x09080706, s.X); + } + + private class VariableSizedPointer : Pointer { } + private class UInt32Pointer : Pointer { } + private class UInt64Pointer : Pointer { } + private class PointerStruct : TStruct + { + public VariableSizedPointer P; + public UInt32Pointer P32; + public UInt64Pointer P64; + } + + [Fact] + public void PointerTStructTest() + { + MemoryBufferAddressSpace dataSource = new(new byte[] { 4, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0 }); + LayoutManager mgr = new LayoutManager().AddPrimitives().AddSizeT(4).AddPointerTypes().AddTStructTypes(); + Reader reader = new(dataSource, mgr); + PointerStruct s = reader.Read(0); + Assert.Equal((ulong)0x4, s.P.Value); + Assert.False(s.P.IsNull); + Assert.Equal((uint)0x1, s.P.Dereference(dataSource)); + Assert.Equal((ulong)0x1, s.P32.Value); + Assert.Equal((byte)0x0, s.P32.Dereference(dataSource)); + Assert.Equal((byte)0x1, s.P32.Element(dataSource, 3)); + Assert.Equal((ulong)0x2, s.P64.Value); + Assert.Equal((ulong)0x0002000000010000, s.P64.Dereference(dataSource)); + } + + public class OptionalField : TStruct + { + public int X; + [If("A")] + public int Y; + public int Z; + } + + [Fact] + public void DefineTest() + { + MemoryBufferAddressSpace dataSource = new(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }); + LayoutManager a = new LayoutManager().AddPrimitives().AddTStructTypes(new string[] { "A" }); + Reader readerA = new(dataSource, a); + OptionalField fA = readerA.Read(0); + Assert.Equal(0x08070605, fA.Y); + Assert.Equal(0x0c0b0a09, fA.Z); + + Reader readerB = new(dataSource); + OptionalField fB = readerB.Read(0); + Assert.Equal(0x0, fB.Y); + Assert.Equal(0x08070605, fB.Z); + } + } +} diff --git a/src/tests/Microsoft.FileFormats.UnitTests/MachO/Tests.cs b/src/tests/Microsoft.FileFormats.UnitTests/MachO/Tests.cs new file mode 100644 index 000000000..5a256fc71 --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/MachO/Tests.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Linq; +using TestHelpers; +using Xunit; + +namespace Microsoft.FileFormats.MachO.Tests +{ + public class Tests + { + [Fact] + public void CheckIndexingInfo() + { + // https://dotnet.myget.org/feed/dotnet-core/package/nuget/runtime.osx.10.12-x64.Microsoft.NETCore.Runtime.CoreCLR/1.1.2 + using (Stream dylib = TestUtilities.OpenCompressedFile("TestBinaries/libcoreclr.dylib.gz")) + { + StreamAddressSpace dataSource = new(dylib); + MachOFile machO = new(dataSource); + Assert.True(machO.IsValid()); + Assert.Equal(Guid.Parse("da2b37b5-cdbc-f838-899b-6a782ceca847"), new Guid(machO.Uuid)); + } + } + + [Fact] + public void CheckDwarfIndexingInfo() + { + // From a local build + using (Stream dwarf = TestUtilities.OpenCompressedFile("TestBinaries/libclrjit.dylib.dwarf.gz")) + { + StreamAddressSpace dataSource = new(dwarf); + MachOFile machO = new(dataSource); + Assert.True(machO.IsValid()); + Assert.Equal(Guid.Parse("0c235eb3-e98e-ef32-b6e6-e6ed18a604a8"), new Guid(machO.Uuid)); + } + } + + [Fact(Skip = "Need an alternate scheme to acquire the binary this test was reading")] + public void ParseCore() + { + using (Stream core = TestUtilities.DecompressFile("TestBinaries/core.gz", "TestBinaries/core")) + { + StreamAddressSpace dataSource = new(core); + // hard-coding the dylinker position so we don't pay to search for it each time + // the code is capable of finding it by brute force search even if we don't provide the hint + MachCore coreReader = new(dataSource, 0x000000010750c000); + Assert.True(coreReader.IsValid()); + MachLoadedImage[] images = coreReader.LoadedImages.Where(i => i.Path.EndsWith("libcoreclr.dylib")).ToArray(); + MachOFile libCoreclr = images[0].Image; + Assert.True(libCoreclr.IsValid()); + Assert.Equal(Guid.Parse("c5660f3e-7352-b138-8141-e9d63b8ab415"), new Guid(libCoreclr.Uuid)); + } + } + } +} diff --git a/src/tests/Microsoft.FileFormats.UnitTests/Microsoft.FileFormats.UnitTests.csproj b/src/tests/Microsoft.FileFormats.UnitTests/Microsoft.FileFormats.UnitTests.csproj new file mode 100644 index 000000000..22139cfa2 --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/Microsoft.FileFormats.UnitTests.csproj @@ -0,0 +1,62 @@ + + + net6.0 + ;1591;1701 + + + + + + + + PreserveNewest + + + Always + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + + diff --git a/src/tests/Microsoft.FileFormats.UnitTests/Minidump/Tests.cs b/src/tests/Microsoft.FileFormats.UnitTests/Minidump/Tests.cs new file mode 100644 index 000000000..b44fba2a5 --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/Minidump/Tests.cs @@ -0,0 +1,201 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.FileFormats.PE; +using System; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using TestHelpers; +using Xunit; + +namespace Microsoft.FileFormats.Minidump +{ + public class Tests + { + const string x86Dump = "TestBinaries/minidump_x86.dmp.gz"; + const string x64Dump = "TestBinaries/minidump_x64.dmp.gz"; + + readonly Guid x64ClrGuid = new Guid("e18d6461-eb4f-49a6-b418-e9af91007a21"); + readonly Guid x86ClrGuid = new Guid("df1e3528-29be-4d0e-9457-4c8ccfdc278a"); + const int ClrAge = 2; + const string ClrPdb = "clr.pdb"; + + [Fact] + public void CheckIsMinidump() + { + using (Stream stream = TestUtilities.OpenCompressedFile(x86Dump)) + { + Assert.True(Minidump.IsValid(new StreamAddressSpace(stream))); + Assert.False(Minidump.IsValid(new StreamAddressSpace(stream), 1)); + } + + using (Stream stream = TestUtilities.OpenCompressedFile(x64Dump)) + { + Assert.True(Minidump.IsValid(new StreamAddressSpace(stream))); + Assert.False(Minidump.IsValid(new StreamAddressSpace(stream), 1)); + } + + // These are GZiped files, they should not be minidumps. + using (FileStream stream = File.OpenRead(x86Dump)) + Assert.False(Minidump.IsValid(new StreamAddressSpace(stream))); + + using (FileStream stream = File.OpenRead(x64Dump)) + Assert.False(Minidump.IsValid(new StreamAddressSpace(stream))); + } + + [Fact] + public void CheckPdbInfo() + { + using (Stream stream = TestUtilities.OpenCompressedFile(x86Dump)) + { + CheckPdbInfoInternal(GetMinidumpFromStream(stream), x86ClrGuid); + } + using (Stream stream = TestUtilities.OpenCompressedFile(x64Dump)) + { + CheckPdbInfoInternal(GetMinidumpFromStream(stream), x64ClrGuid); + } + } + + private void CheckPdbInfoInternal(Minidump minidump, Guid guid) + { + PEFile image = minidump.LoadedImages.Where(i => i.ModuleName.EndsWith(@"\clr.dll")).Single().Image; + foreach (PEPdbRecord pdb in image.Pdbs) + { + Assert.NotNull(pdb); + Assert.Equal(ClrPdb, pdb.Path); + Assert.Equal(ClrAge, pdb.Age); + Assert.Equal(guid, pdb.Signature); + } + } + + [Fact] + public void CheckModuleNames() + { + using (Stream stream = TestUtilities.OpenCompressedFile(x86Dump)) + { + CheckModuleNamesInternal(GetMinidumpFromStream(stream)); + } + using (Stream stream = TestUtilities.OpenCompressedFile(x64Dump)) + { + CheckModuleNamesInternal(GetMinidumpFromStream(stream)); + } + } + + private void CheckModuleNamesInternal(Minidump minidump) + { + Assert.Single(minidump.LoadedImages.Where(i => i.ModuleName.EndsWith(@"\clr.dll"))); + + foreach (var module in minidump.LoadedImages) + Assert.NotNull(module.ModuleName); + } + + [Fact] + public void CheckNestedPEImages() + { + using (Stream stream = TestUtilities.OpenCompressedFile(x86Dump)) + { + CheckNestedPEImagesInternal(GetMinidumpFromStream(stream)); + } + using (Stream stream = TestUtilities.OpenCompressedFile(x64Dump)) + { + CheckNestedPEImagesInternal(GetMinidumpFromStream(stream)); + } + } + + private void CheckNestedPEImagesInternal(Minidump minidump) + { + foreach (var loadedImage in minidump.LoadedImages) + { + Assert.True(loadedImage.Image.HasValidDosSignature.Check()); + Assert.True(loadedImage.Image.HasValidPESignature.Check()); + } + } + + [Fact] + public void CheckMemoryRanges() + { + using (Stream stream = TestUtilities.OpenCompressedFile(x86Dump)) + { + CheckMemoryRangesInternal(GetMinidumpFromStream(stream)); + } + using (Stream stream = TestUtilities.OpenCompressedFile(x64Dump)) + { + CheckMemoryRangesInternal(GetMinidumpFromStream(stream)); + } + } + + private void CheckMemoryRangesInternal(Minidump minidump) + { + ReadOnlyCollection images = minidump.LoadedImages; + ReadOnlyCollection memory = minidump.Segments; + + // Ensure that all of our images actually correspond to memory in the crash dump. Note that our minidumps used + // for this test are all full dumps with all memory (including images) in them. + foreach (var image in images) + { + int count = memory.Where(m => m.VirtualAddress <= image.BaseAddress && image.BaseAddress < m.VirtualAddress + m.Size).Count(); + Assert.Equal(1, count); + + // Check the start of each image for the PE header 'MZ' + byte[] header = minidump.VirtualAddressReader.Read(image.BaseAddress, 2); + Assert.Equal((byte)'M', header[0]); + Assert.Equal((byte)'Z', header[1]); + } + } + + [Fact] + public void CheckLoadedModules() + { + using (Stream stream = TestUtilities.OpenCompressedFile(x86Dump)) + { + CheckLoadedModulesInternal(stream); + } + using (Stream stream = TestUtilities.OpenCompressedFile(x64Dump)) + { + CheckLoadedModulesInternal(stream); + } + } + + private static void CheckLoadedModulesInternal(Stream stream) + { + Minidump minidump = GetMinidumpFromStream(stream); + + var modules = minidump.LoadedImages; + Assert.True(modules.Count > 0); + } + + [Fact] + public void CheckStartupMemoryRead() + { + using (Stream stream = TestUtilities.OpenCompressedFile(x86Dump)) + { + CheckStartupMemoryReadInternal(stream); + } + using (Stream stream = TestUtilities.OpenCompressedFile(x64Dump)) + { + CheckStartupMemoryReadInternal(stream); + } + } + + private static void CheckStartupMemoryReadInternal(Stream stream) + { + IAddressSpace sas = new StreamAddressSpace(stream); + MaxStreamReadHelper readHelper = new MaxStreamReadHelper(sas); + + Minidump minidump = new Minidump(readHelper); + + // We should have read the header of a minidump, so we cannot have read nothing. + Assert.True(readHelper.Max > 0); + + // We should only read the header and not too far into the dump file, 1k should be plenty. + Assert.True(readHelper.Max <= 1024); + } + + private static Minidump GetMinidumpFromStream(Stream stream) + { + StreamAddressSpace sas = new(stream); + return new(sas); + } + } +} diff --git a/src/tests/Microsoft.FileFormats.UnitTests/PDB/Tests.cs b/src/tests/Microsoft.FileFormats.UnitTests/PDB/Tests.cs new file mode 100644 index 000000000..b82c4db4b --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/PDB/Tests.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.FileFormats.PDB.Tests +{ + public class Tests + { + [Fact] + public void CheckIndexingInfo() + { + using (Stream s = File.OpenRead("TestBinaries/HelloWorld.pdb")) + { + StreamAddressSpace fileContent = new(s); + PDBFile pdb = new(fileContent); + Assert.True(pdb.Header.IsMagicValid.Check()); + Assert.True(pdb.IsValid()); + Assert.Equal((uint)1, pdb.Age); + Assert.Equal(Guid.Parse("99891B3E-D7AE-4C3B-ABFF-8A2B4A9B0C43"), pdb.Signature); + } + } + } +} diff --git a/src/tests/Microsoft.FileFormats.UnitTests/PE/Tests.cs b/src/tests/Microsoft.FileFormats.UnitTests/PE/Tests.cs new file mode 100644 index 000000000..fd63b2bc2 --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/PE/Tests.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.FileFormats.PE.Tests +{ + public class Tests + { + [Fact] + public void CheckExeIndexingInfo() + { + using (Stream s = File.OpenRead("TestBinaries/HelloWorld.exe")) + { + StreamAddressSpace fileContent = new(s); + PEFile pe = new(fileContent); + Assert.True(pe.IsValid()); + Assert.Equal((uint)0x8000, pe.SizeOfImage); + Assert.Equal((uint)0x577F5919, pe.Timestamp); + } + } + + [Fact] + public void CheckExePdbInfo() + { + using (Stream s = File.OpenRead("TestBinaries/HelloWorld.exe")) + { + StreamAddressSpace fileContent = new(s); + PEFile pe = new(fileContent); + + // There should only be one pdb record entry + foreach (PEPdbRecord pdb in pe.Pdbs) + { + Assert.Equal(new Guid("99891b3e-d7ae-4c3b-abff-8a2b4a9b0c43"), pdb.Signature); + Assert.Equal(1, pdb.Age); + Assert.Equal(@"c:\users\noahfalk\documents\visual studio 2015\Projects\HelloWorld\HelloWorld\obj\Debug\HelloWorld.pdb", pdb.Path); + } + } + } + + [Fact] + public void CheckDllIndexingInfo() + { + using (Stream s = File.OpenRead("TestBinaries/System.Diagnostics.StackTrace.dll")) + { + StreamAddressSpace fileContent = new(s); + PEFile pe = new(fileContent); + Assert.True(pe.IsValid()); + Assert.Equal((uint)0x35a00, pe.SizeOfImage); + Assert.Equal((uint)0x595cd91b, pe.Timestamp); + } + } + + [Fact] + public void CheckDllPdbInfo() + { + using (Stream s = File.OpenRead("TestBinaries/System.Diagnostics.StackTrace.dll")) + { + StreamAddressSpace fileContent = new(s); + PEFile pe = new(fileContent); + + bool first = true; + foreach (PEPdbRecord pdb in pe.Pdbs) + { + // Skip the first entry (ngen pdb) + if (!first) + { + Assert.Equal(new Guid("8B2E8CF4-4314-4806-982A-B7D904876A50"), pdb.Signature); + Assert.Equal(1, pdb.Age); + Assert.Equal(@"/root/corefx/bin/obj/Unix.AnyCPU.Release/System.Diagnostics.StackTrace/netcoreapp/System.Diagnostics.StackTrace.pdb", pdb.Path); + } + first = false; + } + } + } + } +} diff --git a/src/tests/Microsoft.FileFormats.UnitTests/PerfMap/Tests.cs b/src/tests/Microsoft.FileFormats.UnitTests/PerfMap/Tests.cs new file mode 100644 index 000000000..c6ec43bf2 --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/PerfMap/Tests.cs @@ -0,0 +1,186 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.FileFormats.PerfMap; +using Xunit; + +namespace Microsoft.FileFormats.PerfMap.Tests +{ + public class Tests + { + public static MemoryStream GenerateStreamFromString(string value) + { + return new MemoryStream(Encoding.UTF8.GetBytes(value ?? "")); + } + + public const string s_validV1PerfMap = +@"FFFFFFFF 00 734D59D6DE0E96AA3C77B3E2ED498097 +FFFFFFFE 00 1 +FFFFFFFD 00 2 +FFFFFFFC 00 3 +FFFFFFFB 00 1 +000115D0 0D Microsoft.CodeAnalysis.EmbeddedAttribute::.ctor()"; + + [Fact] + public void CheckIndexingInfo() + { + using (var s = new MemoryStream(Encoding.UTF8.GetBytes(s_validV1PerfMap))) + { + PerfMapFile perfMap = new(s); + Assert.True(perfMap.IsValid); + Assert.True(perfMap.Header is not null); + Assert.True(TestHelpers.TestUtilities.ToHexString(perfMap.Header.Signature) == "734d59d6de0e96aa3c77b3e2ed498097"); + Assert.True(perfMap.Header.Version == 1); + } + } + + [Fact] + public void CheckFields() + { + using (var s = new MemoryStream(Encoding.UTF8.GetBytes(s_validV1PerfMap))) + { + PerfMapFile perfMap = new(s); + Assert.True(perfMap.IsValid); + Assert.True(perfMap.Header is not null); + Assert.True(TestHelpers.TestUtilities.ToHexString(perfMap.Header.Signature) == "734d59d6de0e96aa3c77b3e2ed498097"); + Assert.True(perfMap.Header.Version == 1); + Assert.True(perfMap.Header.OperatingSystem == PerfMapFile.PerfMapOSToken.Linux); + Assert.True(perfMap.Header.Architecture == PerfMapFile.PerfMapArchitectureToken.X64); + Assert.True(perfMap.Header.Abi == PerfMapFile.PerfMapAbiToken.Default); + } + } + + [Fact] + public void CheckRecords() + { + using (var s = new MemoryStream(Encoding.UTF8.GetBytes(s_validV1PerfMap))) + { + PerfMapFile perfMap = new(s); + PerfMapFile.PerfMapRecord record = perfMap.PerfRecords.Single(); + Assert.True(record.Rva == 0x115D0); + Assert.True(record.Length == 0x0D); + Assert.True(record.Name == "Microsoft.CodeAnalysis.EmbeddedAttribute::.ctor()"); + } + } + + public const string s_VNextPerfMapValid = +@"FFFFFFFF 00 734D59D6DE0E96AA3C77B3E2ED498097 +FFFFFFFE 00 99 +FFFFFFFD 00 2 +FFFFFFFC 00 3 +FFFFFFFB 00 1 +000115D0 0D Microsoft.CodeAnalysis.EmbeddedAttribute::.ctor()"; + + [Fact] + public void CheckHeaderVNext() + { + // Reading the vNext header is valid as long as the signature and fields remain compatible. + using (var s = new MemoryStream(Encoding.UTF8.GetBytes(s_VNextPerfMapValid))) + { + PerfMapFile perfMap = new(s); + Assert.True(perfMap.IsValid); + Assert.True(perfMap.Header is not null); + Assert.True(TestHelpers.TestUtilities.ToHexString(perfMap.Header.Signature) == "734d59d6de0e96aa3c77b3e2ed498097"); + Assert.True(perfMap.Header.Version == 99); + Assert.True(perfMap.Header.OperatingSystem == PerfMapFile.PerfMapOSToken.Linux); + Assert.True(perfMap.Header.Architecture == PerfMapFile.PerfMapArchitectureToken.X64); + Assert.True(perfMap.Header.Abi == PerfMapFile.PerfMapAbiToken.Default); + } + } + + [Fact] + public void CheckRecordsVNextFail() + { + // Reading the vNext records is invalid as . + using (var s = new MemoryStream(Encoding.UTF8.GetBytes(s_VNextPerfMapValid))) + { + PerfMapFile perfMap = new(s); + Assert.True(perfMap.IsValid); + Assert.True(perfMap.Header is not null); + Assert.True(perfMap.Header.Version == 99); + Assert.Throws(perfMap.PerfRecords.First); + } + } + + public static IEnumerable InvalidSigPerfMaps() => + new object[][] { +// Too short +new object[]{@"FFFFFFFF 00 734D59D6DE0E96AA3C77B3E2ED4980 +FFFFFFFE 00 1 +FFFFFFFD 00 2 +FFFFFFFC 00 3 +FFFFFFFB 00 1"}, +// Not HexString +new object[]{@"FFFFFFFF 00 734D59D6DE0E96AA3C77B3E2ED4980CG +FFFFFFFE 00 1 +FFFFFFFD 00 2 +FFFFFFFC 00 3 +FFFFFFFB 00 1"}, +// Too long +new object[]{@"FFFFFFFF 00 734D59D6DE0E96AA3C77B3E2ED49809701 +FFFFFFFE 00 1 +FFFFFFFD 00 2 +FFFFFFFC 00 3 +FFFFFFFB 00 1"}}; + + [Theory] + [MemberData(nameof(InvalidSigPerfMaps))] + public void CheckInvalidSigsFail(string doc) + { + using (var s = new MemoryStream(Encoding.UTF8.GetBytes(doc))) + { + var perfMap = new PerfMapFile(s); + Assert.True(!perfMap.IsValid); + } + } + + public static IEnumerable InvalidHeaders() => + new object[][]{ +// Wrong token for sig +new object[]{ @"FFFFFFFA 00 734D59D6DE0E96AA3C77B3E2ED4980 +FFFFFFFE 00 1 +FFFFFFFD 00 2 +FFFFFFFC 00 3 +FFFFFFFB 00 1" }, +// Out of order +new object[]{ @"FFFFFFFF 00 734D59D6DE0E96AA3C77B3E2ED4980CG +FFFFFFFE 00 1 +FFFFFFFC 00 3 +FFFFFFFD 00 2 +FFFFFFFB 00 1"}, +// Missing Entry +new object[]{ @"FFFFFFFF 00 734D59D6DE0E96AA3C77B3E2ED498097 +FFFFFFFE 00 1 +FFFFFFFC 00 3 +FFFFFFFB 00 1"}, +// Repeated pseudo RVA +new object[]{ @"FFFFFFFF 00 734D59D6DE0E96AA3C77B3E2ED498097 +FFFFFFFE 00 1 +FFFFFFFD 00 2 +FFFFFFFD 00 2 +FFFFFFFC 00 3 +FFFFFFFB 00 1"}, +// Wrong pseudo offset +new object[]{ @"FFFFFFFF 02 734D59D6DE0E96AA3C77B3E2ED498097 +FFFFFFFE 00 1 +FFFFFFFD 00 2 +FFFFFFFC 00 3 +FFFFFFFB 00 1"}}; + + [Theory] + [MemberData(nameof(InvalidHeaders))] + public void CheckInvalidHeadersFail(string doc) + { + using (var s = new MemoryStream(Encoding.UTF8.GetBytes(doc))) + { + var perfMap = new PerfMapFile(s); + Assert.True(!perfMap.IsValid); + } + } + } +} diff --git a/src/tests/Microsoft.FileFormats.UnitTests/PrimitiveTypes.cs b/src/tests/Microsoft.FileFormats.UnitTests/PrimitiveTypes.cs new file mode 100644 index 000000000..c98a4b4ef --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/PrimitiveTypes.cs @@ -0,0 +1,102 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.FileFormats.Tests +{ + public class PrimitiveTypes + { + [Fact] + public void ByteTest() + { + MemoryBufferAddressSpace dt = new MemoryBufferAddressSpace(new byte[] { 200, 12, 0 }); + Assert.Equal(200, (byte)new UInt8Layout(false).Read(dt, 0)); + Assert.Equal(12, (byte)new UInt8Layout(false).Read(dt, 1)); + Assert.Equal(0, (byte)new UInt8Layout(false).Read(dt, 2)); + Assert.Equal(200, (byte)new UInt8Layout(true).Read(dt, 0)); + Assert.Equal(12, (byte)new UInt8Layout(true).Read(dt, 1)); + Assert.Equal(0, (byte)new UInt8Layout(true).Read(dt, 2)); + } + + [Fact] + public void SByteTest() + { + MemoryBufferAddressSpace dt = new MemoryBufferAddressSpace(new byte[] { 200, 12, 0 }); + Assert.Equal(-56, (sbyte)new Int8Layout(false).Read(dt, 0)); + Assert.Equal(12, (sbyte)new Int8Layout(false).Read(dt, 1)); + Assert.Equal(0, (sbyte)new Int8Layout(false).Read(dt, 2)); + Assert.Equal(-56, (sbyte)new Int8Layout(true).Read(dt, 0)); + Assert.Equal(12, (sbyte)new Int8Layout(true).Read(dt, 1)); + Assert.Equal(0, (sbyte)new Int8Layout(true).Read(dt, 2)); + } + + [Fact] + public void UShortTest() + { + MemoryBufferAddressSpace dt = new MemoryBufferAddressSpace(new byte[] { 200, 12, 0, 0 }); + Assert.Equal(12 * 256 + 200, (ushort)new UInt16Layout(false).Read(dt, 0)); + Assert.Equal(0, (ushort)new UInt16Layout(false).Read(dt, 2)); + Assert.Equal(200 * 256 + 12, (ushort)new UInt16Layout(true).Read(dt, 0)); + Assert.Equal(0, (ushort)new UInt16Layout(true).Read(dt, 2)); + } + + [Fact] + public void ShortTest() + { + MemoryBufferAddressSpace dt = new MemoryBufferAddressSpace(new byte[] { 200, 12, 0, 0 }); + Assert.Equal(12 * 256 + 200, (short)new Int16Layout(false).Read(dt, 0)); + Assert.Equal(0, (short)new Int16Layout(false).Read(dt, 2)); + Assert.Equal(-56 * 256 + 12, (short)new Int16Layout(true).Read(dt, 0)); + Assert.Equal(0, (short)new Int16Layout(true).Read(dt, 2)); + } + + [Fact] + public void UIntTest() + { + MemoryBufferAddressSpace dt = new MemoryBufferAddressSpace(new byte[] { 200, 12, 19, 139, 0, 0, 0, 0 }); + Assert.Equal((uint)139 * 256 * 256 * 256 + 19 * 256 * 256 + 12 * 256 + 200, new UInt32Layout(false).Read(dt, 0)); + Assert.Equal((uint)0, new UInt32Layout(false).Read(dt, 4)); + Assert.Equal((uint)200 * 256 * 256 * 256 + 12 * 256 * 256 + 19 * 256 + 139, new UInt32Layout(true).Read(dt, 0)); + Assert.Equal((uint)0, new UInt32Layout(true).Read(dt, 4)); + } + + [Fact] + public void IntTest() + { + MemoryBufferAddressSpace dt = new MemoryBufferAddressSpace(new byte[] { 200, 12, 19, 139, 0, 0, 0, 0 }); + Assert.Equal((139 - 256) * 256 * 256 * 256 + 19 * 256 * 256 + 12 * 256 + 200, new Int32Layout(false).Read(dt, 0)); + Assert.Equal(0, new Int32Layout(false).Read(dt, 4)); + Assert.Equal((200 - 256) * 256 * 256 * 256 + 12 * 256 * 256 + 19 * 256 + 139, new Int32Layout(true).Read(dt, 0)); + Assert.Equal(0, new Int32Layout(true).Read(dt, 4)); + } + + [Fact] + public void ULongTest() + { + MemoryBufferAddressSpace dt = new MemoryBufferAddressSpace(new byte[] { 200, 12, 19, 139, 192, 7, 1, 40, 0, 0, 0, 0, 0, 0, 0, 0 }); + Assert.Equal((40UL << 56) + (1UL << 48) + (7UL << 40) + (192UL << 32) + (139UL << 24) + (19UL << 16) + (12UL << 8) + 200UL, + new UInt64Layout(false).Read(dt, 0)); + Assert.Equal(0UL, new UInt64Layout(false).Read(dt, 8)); + Assert.Equal((200UL << 56) + (12UL << 48) + (19UL << 40) + (139UL << 32) + (192UL << 24) + (7UL << 16) + (1UL << 8) + 40UL, + new UInt64Layout(true).Read(dt, 0)); + Assert.Equal(0UL, new UInt64Layout(true).Read(dt, 8)); + } + + [Fact] + public void LongTest() + { + MemoryBufferAddressSpace dt = new MemoryBufferAddressSpace(new byte[] { 200, 12, 19, 139, 192, 7, 1, 40, 0, 0, 0, 0, 0, 0, 0, 0 }); + Assert.Equal((40L << 56) + (1L << 48) + (7L << 40) + (192L << 32) + (139L << 24) + (19L << 16) + (12L << 8) + 200L, + new Int64Layout(false).Read(dt, 0)); + Assert.Equal(0L, new Int64Layout(false).Read(dt, 8)); + Assert.Equal((-56L << 56) + (12L << 48) + (19L << 40) + (139L << 32) + (192L << 24) + (7L << 16) + (1L << 8) + 40L, + new Int64Layout(true).Read(dt, 0)); + Assert.Equal(0L, new Int64Layout(true).Read(dt, 8)); + } + } +} diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/HelloWorld.exe b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/HelloWorld.exe new file mode 100644 index 000000000..b02a4570a Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/HelloWorld.exe differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/HelloWorld.pdb b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/HelloWorld.pdb new file mode 100644 index 000000000..adea403b4 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/HelloWorld.pdb differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/PerfMapEnabled/System.ComponentModel.EventBasedAsync.dll b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/PerfMapEnabled/System.ComponentModel.EventBasedAsync.dll new file mode 100644 index 000000000..6e61957a9 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/PerfMapEnabled/System.ComponentModel.EventBasedAsync.dll differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/PerfMapEnabled/System.ComponentModel.EventBasedAsync.ni.r2rmap b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/PerfMapEnabled/System.ComponentModel.EventBasedAsync.ni.r2rmap new file mode 100644 index 000000000..ac7f913a8 --- /dev/null +++ b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/PerfMapEnabled/System.ComponentModel.EventBasedAsync.ni.r2rmap @@ -0,0 +1,76 @@ +FFFFFFFF 00 734D59D6DE0E96AA3C77B3E2ED498097 +FFFFFFFE 00 1 +FFFFFFFD 00 2 +FFFFFFFC 00 3 +FFFFFFFB 00 1 +000115D0 0D Microsoft.CodeAnalysis.EmbeddedAttribute::.ctor() +000115E0 3F System.Runtime.CompilerServices.NullableAttribute::.ctor(System.Byte) +00011620 26 System.Runtime.CompilerServices.NullableContextAttribute::.ctor(System.Byte) +00011650 26 System.Runtime.CompilerServices.NullablePublicOnlyAttribute::.ctor(System.Boolean) +00011680 10 System.Boolean System.SR::UsingResourceKeys() +00011690 74 System.String System.SR::GetResourceString(System.String) +00011710 50 System.Resources.ResourceManager System.SR::get_ResourceManager() +00011760 17 System.String System.SR::get_Async_NullDelegate() +00011780 17 System.String System.SR::get_Async_OperationAlreadyCompleted() +000117A0 17 System.String System.SR::get_Async_OperationCancelled() +000117C0 17 System.String System.SR::get_Async_ExceptionOccurred() +000117E0 17 System.String System.SR::get_BackgroundWorker_WorkerAlreadyRunning() +00011800 17 System.String System.SR::get_BackgroundWorker_WorkerDoesntReportProgress() +00011820 17 System.String System.SR::get_BackgroundWorker_WorkerDoesntSupportCancellation() +00011840 3A System.SR::.cctor() +00011880 50 System.ComponentModel.AsyncCompletedEventArgs::.ctor(System.Exception, System.Boolean, System.Object) +000118D0 9A System.ComponentModel.AsyncCompletedEventArgs::RaiseExceptionIfNecessary() +00011970 0D System.Boolean System.ComponentModel.AsyncCompletedEventArgs::get_Cancelled() +00011980 0C System.Exception System.ComponentModel.AsyncCompletedEventArgs::get_Error() +00011990 0D System.Object System.ComponentModel.AsyncCompletedEventArgs::get_UserState() +000119A0 43 System.ComponentModel.AsyncOperation::.ctor(System.Object, System.Threading.SynchronizationContext) +000119F0 25 System.ComponentModel.AsyncOperation::Finalize() +00011A20 05 System.Object System.ComponentModel.AsyncOperation::get_UserSuppliedState() +00011A30 05 System.Threading.SynchronizationContext System.ComponentModel.AsyncOperation::get_SynchronizationContext() +00011A40 4F System.ComponentModel.AsyncOperation::Post(System.Threading.SendOrPostCallback, System.Object) +00011A90 5C System.ComponentModel.AsyncOperation::PostOperationCompleted(System.Threading.SendOrPostCallback, System.Object) +00011AF0 29 System.ComponentModel.AsyncOperation::OperationCompleted() +00011B20 5A System.ComponentModel.AsyncOperation::PostCore(System.Threading.SendOrPostCallback, System.Object, System.Boolean) +00011B80 58 System.ComponentModel.AsyncOperation::OperationCompletedCore() +00011BE0 46 System.ComponentModel.AsyncOperation::VerifyNotCompleted() +00011C30 4F System.ComponentModel.AsyncOperation::VerifyDelegateNotNull(System.Threading.SendOrPostCallback) +00011C80 5B System.ComponentModel.AsyncOperation System.ComponentModel.AsyncOperation::CreateOperation(System.Object, System.Threading.SynchronizationContext) +00011CE0 61 System.ComponentModel.AsyncOperation System.ComponentModel.AsyncOperationManager::CreateOperation(System.Object) +00011D50 3C System.Threading.SynchronizationContext System.ComponentModel.AsyncOperationManager::get_SynchronizationContext() +00011D90 0D System.ComponentModel.AsyncOperationManager::set_SynchronizationContext(System.Threading.SynchronizationContext) +00011DA0 37 System.ComponentModel.ProgressChangedEventArgs::.ctor(System.Int32, System.Object) +00011DE0 0C System.Int32 System.ComponentModel.ProgressChangedEventArgs::get_ProgressPercentage() +00011DF0 0C System.Object System.ComponentModel.ProgressChangedEventArgs::get_UserState() +00011E00 65 System.ComponentModel.BackgroundWorker::.ctor() +00011E70 42 System.ComponentModel.BackgroundWorker::AsyncOperationCompleted(System.Object) +00011EC0 0D System.Boolean System.ComponentModel.BackgroundWorker::get_CancellationPending() +00011ED0 53 System.ComponentModel.BackgroundWorker::CancelAsync() +00011F30 62 System.ComponentModel.BackgroundWorker::add_DoWork(System.ComponentModel.DoWorkEventHandler) +00011FA0 62 System.ComponentModel.BackgroundWorker::remove_DoWork(System.ComponentModel.DoWorkEventHandler) +00012010 0D System.Boolean System.ComponentModel.BackgroundWorker::get_IsBusy() +00012020 2F System.ComponentModel.BackgroundWorker::OnDoWork(System.ComponentModel.DoWorkEventArgs) +00012050 2F System.ComponentModel.BackgroundWorker::OnRunWorkerCompleted(System.ComponentModel.RunWorkerCompletedEventArgs) +00012080 2F System.ComponentModel.BackgroundWorker::OnProgressChanged(System.ComponentModel.ProgressChangedEventArgs) +000120B0 62 System.ComponentModel.BackgroundWorker::add_ProgressChanged(System.ComponentModel.ProgressChangedEventHandler) +00012120 62 System.ComponentModel.BackgroundWorker::remove_ProgressChanged(System.ComponentModel.ProgressChangedEventHandler) +00012190 31 System.ComponentModel.BackgroundWorker::ProgressReporter(System.Object) +000121D0 0F System.ComponentModel.BackgroundWorker::ReportProgress(System.Int32) +000121E0 F7 System.ComponentModel.BackgroundWorker::ReportProgress(System.Int32, System.Object) +000122E0 0F System.ComponentModel.BackgroundWorker::RunWorkerAsync() +000122F0 11A System.ComponentModel.BackgroundWorker::RunWorkerAsync(System.Object) +00012410 62 System.ComponentModel.BackgroundWorker::add_RunWorkerCompleted(System.ComponentModel.RunWorkerCompletedEventHandler) +00012480 62 System.ComponentModel.BackgroundWorker::remove_RunWorkerCompleted(System.ComponentModel.RunWorkerCompletedEventHandler) +000124F0 0D System.Boolean System.ComponentModel.BackgroundWorker::get_WorkerReportsProgress() +00012500 0D System.ComponentModel.BackgroundWorker::set_WorkerReportsProgress(System.Boolean) +00012510 0D System.Boolean System.ComponentModel.BackgroundWorker::get_WorkerSupportsCancellation() +00012520 0D System.ComponentModel.BackgroundWorker::set_WorkerSupportsCancellation(System.Boolean) +00012530 15B System.ComponentModel.BackgroundWorker::WorkerThreadStart(System.Object) +00012690 01 System.ComponentModel.BackgroundWorker::Dispose(System.Boolean) +000126A0 0D System.ComponentModel.BackgroundWorker::b__27_0(System.Object) +000126B0 30 System.ComponentModel.DoWorkEventArgs::.ctor(System.Object) +000126E0 0C System.Object System.ComponentModel.DoWorkEventArgs::get_Argument() +000126F0 0D System.Object System.ComponentModel.DoWorkEventArgs::get_Result() +00012700 14 System.ComponentModel.DoWorkEventArgs::set_Result(System.Object) +00012720 5D System.ComponentModel.RunWorkerCompletedEventArgs::.ctor(System.Object, System.Exception, System.Boolean) +00012780 1A System.Object System.ComponentModel.RunWorkerCompletedEventArgs::get_Result() +000127A0 0D System.Object System.ComponentModel.RunWorkerCompletedEventArgs::get_UserState() diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/System.Diagnostics.StackTrace.dll b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/System.Diagnostics.StackTrace.dll new file mode 100644 index 000000000..4f8bf9f35 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/System.Diagnostics.StackTrace.dll differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/apphost.gz b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/apphost.gz new file mode 100644 index 000000000..096f38a0a Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/apphost.gz differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/core.gz b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/core.gz new file mode 100644 index 000000000..bb50f8c89 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/core.gz differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/ilasm.dbg b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/ilasm.dbg new file mode 100644 index 000000000..0be104ca5 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/ilasm.dbg differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libclrjit.dylib.dwarf.gz b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libclrjit.dylib.dwarf.gz new file mode 100644 index 000000000..d3e4fa4a8 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libclrjit.dylib.dwarf.gz differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libcoreclr.dylib.gz b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libcoreclr.dylib.gz new file mode 100644 index 000000000..9990ca08d Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libcoreclr.dylib.gz differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libcoreclr.so.gz b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libcoreclr.so.gz new file mode 100644 index 000000000..4c30ca692 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libcoreclr.so.gz differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libcoreclrtraceptprovider.so.dbg.gz b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libcoreclrtraceptprovider.so.dbg.gz new file mode 100644 index 000000000..4ede1a815 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/libcoreclrtraceptprovider.so.dbg.gz differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/minidump_x64.dmp.gz b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/minidump_x64.dmp.gz new file mode 100644 index 000000000..4afe7408a Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/minidump_x64.dmp.gz differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/minidump_x86.dmp.gz b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/minidump_x86.dmp.gz new file mode 100644 index 000000000..d289689e7 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/minidump_x86.dmp.gz differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/renamed_build_id_section b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/renamed_build_id_section new file mode 100644 index 000000000..ad5423dc5 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/renamed_build_id_section differ diff --git a/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/triagedump.gz b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/triagedump.gz new file mode 100644 index 000000000..749849f88 Binary files /dev/null and b/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/triagedump.gz differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/KeyGeneratorTests.cs b/src/tests/Microsoft.SymbolStore.UnitTests/KeyGeneratorTests.cs new file mode 100644 index 000000000..f2ed6baf8 --- /dev/null +++ b/src/tests/Microsoft.SymbolStore.UnitTests/KeyGeneratorTests.cs @@ -0,0 +1,524 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.SymbolStore.KeyGenerators; +using TestHelpers; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.SymbolStore.Tests +{ + public class KeyGenerationTests + { + readonly ITracer _tracer; + + public KeyGenerationTests(ITestOutputHelper output) + { + _tracer = new Tracer(output); + } + + [Fact] + public void FileKeyGenerator() + { + ELFCoreKeyGeneratorInternal(fileGenerator: true); + ELFFileKeyGeneratorInternal(fileGenerator: true); + //MachCoreKeyGeneratorInternal(fileGenerator: true); + MachOFileKeyGeneratorInternal(fileGenerator: true); + MinidumpKeyGeneratorInternal(fileGenerator: true); + PDBFileKeyGeneratorInternal(fileGenerator: true); + PEFileKeyGeneratorInternal(fileGenerator: true); + PortablePDBFileKeyGeneratorInternal(fileGenerator: true); + PerfMapFileKeyGeneratorInternal(fileGenerator: true); + } + + + [Fact] + public void PerfMapFileKeyGenerator() + { + PerfMapFileKeyGeneratorInternal(fileGenerator: false); + } + + private void PerfMapFileKeyGeneratorInternal(bool fileGenerator) + { + const string LinuxPerfMapPath = "TestBinaries/PerfMapEnabled/System.ComponentModel.EventBasedAsync.ni.r2rmap"; + using (Stream linuxPerfMapStream = File.OpenRead(LinuxPerfMapPath)) + { + var file = new SymbolStoreFile(linuxPerfMapStream, LinuxPerfMapPath); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new PerfMapFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Single().Index == "system.componentmodel.eventbasedasync.ni.r2rmap/r2rmap-v1-734d59d6de0e96aa3c77b3e2ed498097/system.componentmodel.eventbasedasync.ni.r2rmap"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(!symbolKey.Any()); + + IEnumerable perfMapKey = generator.GetKeys(KeyTypeFlags.PerfMapKeys); + Assert.True(!symbolKey.Any()); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(!symbolKey.Any()); + } + } + + [Fact] + public void ELFCoreKeyGenerator() + { + ELFCoreKeyGeneratorInternal(fileGenerator: false); + } + + private void ELFCoreKeyGeneratorInternal(bool fileGenerator) + { + using (Stream core = TestUtilities.OpenCompressedFile("TestBinaries/triagedump.gz")) + { + var file = new SymbolStoreFile(core, "triagedump"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new ELFCoreKeyGenerator(_tracer, file); + + Dictionary identityKeys = generator.GetKeys(KeyTypeFlags.IdentityKey).ToDictionary((key) => key.Index); + Dictionary symbolKeys = generator.GetKeys(KeyTypeFlags.SymbolKey).ToDictionary((key) => key.Index); + Dictionary clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys).ToDictionary((key) => key.Index); + Dictionary dacdbiKeys = generator.GetKeys(KeyTypeFlags.DacDbiKeys).ToDictionary((key) => key.Index); + Dictionary runtimeKeys = generator.GetKeys(KeyTypeFlags.RuntimeKeys).ToDictionary((key) => key.Index); + + // Program (SymbolTestApp2) + Assert.True(identityKeys.ContainsKey("symboltestapp2.dll/DD52998F8000/symboltestapp2.dll")); + Assert.True(symbolKeys.ContainsKey("symboltestapp2.pdb/ed4317cbcab24c1fa06d93f8164c74ddFFFFFFFF/symboltestapp2.pdb")); + + // System.IO.dll + Assert.True(identityKeys.ContainsKey("system.io.dll/595CD90631400/system.io.dll")); + Assert.True(symbolKeys.ContainsKey("system.io.pdb/5e949d2065c746a1b510de28f35d114cFFFFFFFF/system.io.pdb")); + + // System.Native.so + Assert.True(identityKeys.ContainsKey("system.native.so/elf-buildid-3c22124b073eeb90746d6f6eab1ae2bf4097eb70/system.native.so")); + Assert.True(symbolKeys.ContainsKey("_.debug/elf-buildid-sym-3c22124b073eeb90746d6f6eab1ae2bf4097eb70/_.debug")); + + // libcoreclr.so + Assert.True(identityKeys.ContainsKey("libcoreclr.so/elf-buildid-8f39a52a756311ab365090bfe9edef7ee8c44503/libcoreclr.so")); + Assert.True(symbolKeys.ContainsKey("_.debug/elf-buildid-sym-8f39a52a756311ab365090bfe9edef7ee8c44503/_.debug")); + Assert.True(runtimeKeys.ContainsKey("libcoreclr.so/elf-buildid-8f39a52a756311ab365090bfe9edef7ee8c44503/libcoreclr.so")); + + Assert.True(clrKeys.ContainsKey("libmscordaccore.so/elf-buildid-coreclr-8f39a52a756311ab365090bfe9edef7ee8c44503/libmscordaccore.so")); + Assert.True(clrKeys.ContainsKey("libsos.so/elf-buildid-coreclr-8f39a52a756311ab365090bfe9edef7ee8c44503/libsos.so")); + Assert.True(clrKeys.ContainsKey("sos.netcore.dll/elf-buildid-coreclr-8f39a52a756311ab365090bfe9edef7ee8c44503/sos.netcore.dll")); + + Assert.True(dacdbiKeys.ContainsKey("libmscordaccore.so/elf-buildid-coreclr-8f39a52a756311ab365090bfe9edef7ee8c44503/libmscordaccore.so")); + Assert.False(dacdbiKeys.ContainsKey("libsos.so/elf-buildid-coreclr-8f39a52a756311ab365090bfe9edef7ee8c44503/libsos.so")); + Assert.False(dacdbiKeys.ContainsKey("sos.netcore.dll/elf-buildid-coreclr-8f39a52a756311ab365090bfe9edef7ee8c44503/sos.netcore.dll")); + } + } + + [Fact] + public void ELFFileKeyGenerator() + { + ELFFileKeyGeneratorInternal(fileGenerator: false); + } + + private void ELFFileKeyGeneratorInternal(bool fileGenerator) + { + using (Stream stream = TestUtilities.OpenCompressedFile("TestBinaries/libcoreclr.so.gz")) + { + var file = new SymbolStoreFile(stream, "libcoreclr.so"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new ELFFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "libcoreclr.so/elf-buildid-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/libcoreclr.so"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 1); + Assert.True(symbolKey.First().Index == "_.debug/elf-buildid-sym-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/_.debug"); + + Dictionary clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys).ToDictionary((key) => key.Index); + Assert.True(clrKeys.ContainsKey("libmscordaccore.so/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/libmscordaccore.so")); + Assert.True(clrKeys.ContainsKey("libmscordbi.so/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/libmscordbi.so")); + Assert.True(clrKeys.ContainsKey("mscordaccore.dll/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/mscordaccore.dll")); + Assert.True(clrKeys.ContainsKey("mscordbi.dll/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/mscordbi.dll")); + Assert.True(clrKeys.ContainsKey("libsos.so/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/libsos.so")); + Assert.True(clrKeys.ContainsKey("sos.netcore.dll/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/sos.netcore.dll")); + + Dictionary dacdbiKeys = generator.GetKeys(KeyTypeFlags.DacDbiKeys).ToDictionary((key) => key.Index); + Assert.True(dacdbiKeys.ContainsKey("libmscordaccore.so/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/libmscordaccore.so")); + Assert.True(dacdbiKeys.ContainsKey("libmscordbi.so/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/libmscordbi.so")); + Assert.True(dacdbiKeys.ContainsKey("mscordaccore.dll/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/mscordaccore.dll")); + Assert.True(dacdbiKeys.ContainsKey("mscordbi.dll/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/mscordbi.dll")); + Assert.False(dacdbiKeys.ContainsKey("libsos.so/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/libsos.so")); + Assert.False(dacdbiKeys.ContainsKey("sos.netcore.dll/elf-buildid-coreclr-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/sos.netcore.dll")); + + Dictionary runtimeKeys = generator.GetKeys(KeyTypeFlags.RuntimeKeys).ToDictionary((key) => key.Index); + Assert.True(runtimeKeys.ContainsKey("libcoreclr.so/elf-buildid-ef8f58a0b402d11c68f78342ef4fcc7d23798d4c/libcoreclr.so")); + } + + using (Stream stream = TestUtilities.OpenCompressedFile("TestBinaries/libcoreclrtraceptprovider.so.dbg.gz")) + { + var file = new SymbolStoreFile(stream, "libcoreclrtraceptprovider.so.dbg"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new ELFFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "_.debug/elf-buildid-sym-ce4ce0558d878a05754dff246ccea2a70a1db3a8/_.debug"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 0); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + + using (Stream stream = File.OpenRead("TestBinaries/symbolized_executable")) + { + var file = new SymbolStoreFile(stream, "symbolized_executable"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new ELFFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "_.debug/elf-buildid-sym-126ba1461caf6644cfdd124bfcceeffa81b18897/_.debug"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 0); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + + using (Stream stream = File.OpenRead("TestBinaries/stripped_executable")) + { + var file = new SymbolStoreFile(stream, "stripped_executable"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new ELFFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "stripped_executable/elf-buildid-126ba1461caf6644cfdd124bfcceeffa81b18897/stripped_executable"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 1); + Assert.True(symbolKey.First().Index == "_.debug/elf-buildid-sym-126ba1461caf6644cfdd124bfcceeffa81b18897/_.debug"); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + + using (Stream stream = File.OpenRead("TestBinaries/md5_build_id")) + { + var file = new SymbolStoreFile(stream, "md5_build_id"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new ELFFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "md5_build_id/elf-buildid-001ba81f23966cf77e40bcbb0701cd3400000000/md5_build_id"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 1); + Assert.True(symbolKey.First().Index == "_.debug/elf-buildid-sym-001ba81f23966cf77e40bcbb0701cd3400000000/_.debug"); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + } + + [Fact(Skip = "Need an alternate scheme to acquire the binary this test was reading")] + public void MachCoreKeyGenerator() + { + MachCoreKeyGeneratorInternal(fileGenerator: false); + } + + private void MachCoreKeyGeneratorInternal(bool fileGenerator) + { + using (Stream core = TestUtilities.DecompressFile("TestBinaries/core.gz", "TestBinaries/core")) + { + var file = new SymbolStoreFile(core, "core"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new MachCoreKeyGenerator(_tracer, file); + + Dictionary identityKeys = generator.GetKeys(KeyTypeFlags.IdentityKey).ToDictionary((key) => key.Index); + Dictionary symbolKeys = generator.GetKeys(KeyTypeFlags.SymbolKey).ToDictionary((key) => key.Index); + Dictionary clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys).ToDictionary((key) => key.Index); + Dictionary dacdbiKeys = generator.GetKeys(KeyTypeFlags.DacDbiKeys).ToDictionary((key) => key.Index); + Dictionary runtimeKeys = generator.GetKeys(KeyTypeFlags.RuntimeKeys).ToDictionary((key) => key.Index); + + // System.Native.dylib + Assert.True(identityKeys.ContainsKey("system.native.dylib/mach-uuid-f7c77509e13a3da18099a2b97e90fade/system.native.dylib")); + Assert.True(symbolKeys.ContainsKey("_.dwarf/mach-uuid-sym-f7c77509e13a3da18099a2b97e90fade/_.dwarf")); + + // libcoreclr.dylib + Assert.True(identityKeys.ContainsKey("libcoreclr.dylib/mach-uuid-3e0f66c5527338b18141e9d63b8ab415/libcoreclr.dylib")); + Assert.True(symbolKeys.ContainsKey("_.dwarf/mach-uuid-sym-3e0f66c5527338b18141e9d63b8ab415/_.dwarf")); + Assert.True(runtimeKeys.ContainsKey("libcoreclr.dylib/mach-uuid-3e0f66c5527338b18141e9d63b8ab415/libcoreclr.dylib")); + + Assert.True(clrKeys.ContainsKey("libmscordaccore.dylib/mach-uuid-coreclr-3e0f66c5527338b18141e9d63b8ab415/libmscordaccore.dylib")); + Assert.True(clrKeys.ContainsKey("libmscordbi.dylib/mach-uuid-coreclr-3e0f66c5527338b18141e9d63b8ab415/libmscordbi.dylib")); + Assert.True(clrKeys.ContainsKey("libsos.dylib/mach-uuid-coreclr-3e0f66c5527338b18141e9d63b8ab415/libsos.dylib")); + Assert.True(clrKeys.ContainsKey("sos.netcore.dll/mach-uuid-coreclr-3e0f66c5527338b18141e9d63b8ab415/sos.netcore.dll")); + + Assert.True(dacdbiKeys.ContainsKey("libmscordaccore.dylib/mach-uuid-coreclr-3e0f66c5527338b18141e9d63b8ab415/libmscordaccore.dylib")); + Assert.True(dacdbiKeys.ContainsKey("libmscordbi.dylib/mach-uuid-coreclr-3e0f66c5527338b18141e9d63b8ab415/libmscordbi.dylib")); + Assert.False(dacdbiKeys.ContainsKey("libsos.dylib/mach-uuid-coreclr-3e0f66c5527338b18141e9d63b8ab415/libsos.dylib")); + Assert.False(dacdbiKeys.ContainsKey("sos.netcore.dll/mach-uuid-coreclr-3e0f66c5527338b18141e9d63b8ab415/sos.netcore.dll")); + + } + } + + [Fact] + public void MachOFileKeyGenerator() + { + MachOFileKeyGeneratorInternal(fileGenerator: false); + } + + private void MachOFileKeyGeneratorInternal(bool fileGenerator) + { + using (Stream dylib = TestUtilities.OpenCompressedFile("TestBinaries/libcoreclr.dylib.gz")) + { + var file = new SymbolStoreFile(dylib, "libcoreclr.dylib"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new MachOFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "libcoreclr.dylib/mach-uuid-b5372bdabccd38f8899b6a782ceca847/libcoreclr.dylib"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 1); + Assert.True(symbolKey.First().Index == "_.dwarf/mach-uuid-sym-b5372bdabccd38f8899b6a782ceca847/_.dwarf"); + + Dictionary clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys).ToDictionary((key) => key.Index); + Assert.True(clrKeys.ContainsKey("libmscordaccore.dylib/mach-uuid-coreclr-b5372bdabccd38f8899b6a782ceca847/libmscordaccore.dylib")); + Assert.True(clrKeys.ContainsKey("libsos.dylib/mach-uuid-coreclr-b5372bdabccd38f8899b6a782ceca847/libsos.dylib")); + Assert.True(clrKeys.ContainsKey("sos.netcore.dll/mach-uuid-coreclr-b5372bdabccd38f8899b6a782ceca847/sos.netcore.dll")); + + Dictionary runtimeKeys = generator.GetKeys(KeyTypeFlags.RuntimeKeys).ToDictionary((key) => key.Index); + Assert.True(runtimeKeys.ContainsKey("libcoreclr.dylib/mach-uuid-b5372bdabccd38f8899b6a782ceca847/libcoreclr.dylib")); + } + + using (Stream dwarf = TestUtilities.OpenCompressedFile("TestBinaries/libclrjit.dylib.dwarf.gz")) + { + var file = new SymbolStoreFile(dwarf, "libclrjit.dylib.dwarf"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new MachOFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "_.dwarf/mach-uuid-sym-b35e230c8ee932efb6e6e6ed18a604a8/_.dwarf"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 0); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + + using (Stream machofat = TestUtilities.OpenCompressedFile("TestBinaries/libSystem.Security.Cryptography.Native.Apple.dylib.gz")) + { + var file = new SymbolStoreFile(machofat, "libsystem.security.cryptography.native.apple.dylib"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new MachOFatHeaderKeyGenerator(_tracer, file); + + Dictionary identityKeys = generator.GetKeys(KeyTypeFlags.IdentityKey).ToDictionary((key) => key.Index); + Assert.True(identityKeys.ContainsKey("libsystem.security.cryptography.native.apple.dylib/mach-uuid-fad93e41f2e23d11aab75e98d7fe66d6/libsystem.security.cryptography.native.apple.dylib")); + Assert.True(identityKeys.ContainsKey("libsystem.security.cryptography.native.apple.dylib/mach-uuid-e5bf8b935f393806a20933aa98adf5b7/libsystem.security.cryptography.native.apple.dylib")); + + Dictionary symbolKeys = generator.GetKeys(KeyTypeFlags.SymbolKey).ToDictionary((key) => key.Index); + Assert.True(symbolKeys.ContainsKey("_.dwarf/mach-uuid-sym-fad93e41f2e23d11aab75e98d7fe66d6/_.dwarf")); + Assert.True(symbolKeys.ContainsKey("_.dwarf/mach-uuid-sym-e5bf8b935f393806a20933aa98adf5b7/_.dwarf")); + } + } + + [Fact] + public void MinidumpKeyGenerator() + { + MinidumpKeyGeneratorInternal(fileGenerator: false); + } + + private void MinidumpKeyGeneratorInternal(bool fileGenerator) + { + using (Stream core = TestUtilities.OpenCompressedFile("TestBinaries/minidump_x64.dmp.gz")) + { + var file = new SymbolStoreFile(core, "minidump_x64.dmp"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new MinidumpKeyGenerator(_tracer, file); + + Dictionary identityKeys = generator.GetKeys(KeyTypeFlags.IdentityKey).ToDictionary((key) => key.Index); + Dictionary symbolKeys = generator.GetKeys(KeyTypeFlags.SymbolKey).ToDictionary((key) => key.Index); + + // Program (exception.exe) + Assert.True(identityKeys.ContainsKey("exception.exe/57B39FFA6000/exception.exe")); + Assert.True(symbolKeys.ContainsKey("exception.pdb/df85e94d63ae4d8992fbf81730a7ac911/exception.pdb")); + + // mscoree.dll + Assert.True(identityKeys.ContainsKey("mscoree.dll/57A5832766000/mscoree.dll")); + Assert.True(symbolKeys.ContainsKey("mscoree.pdb/4a348372fdff448ab6a1bfc8b93ffb6b1/mscoree.pdb")); + } + } + + [Fact] + public void PDBFileKeyGenerator() + { + PDBFileKeyGeneratorInternal(fileGenerator: false); + } + + private void PDBFileKeyGeneratorInternal(bool fileGenerator) + { + const string TestBinary = "TestBinaries/HelloWorld.pdb"; + using (Stream pdb = File.OpenRead(TestBinary)) + { + var file = new SymbolStoreFile(pdb, TestBinary); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new PDBFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "helloworld.pdb/99891b3ed7ae4c3babff8a2b4a9b0c431/helloworld.pdb"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 0); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + } + + [Fact] + public void PEFileKeyGenerator() + { + PEFileKeyGeneratorInternal(fileGenerator: false); + } + + private void PEFileKeyGeneratorInternal(bool fileGenerator) + { + const string TestBinaryExe = "TestBinaries/HelloWorld.exe"; + using (Stream exe = File.OpenRead(TestBinaryExe)) + { + var file = new SymbolStoreFile(exe, TestBinaryExe); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new PEFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "helloworld.exe/577F59198000/helloworld.exe"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 1); + Assert.True(symbolKey.First().Index == "helloworld.pdb/99891b3ed7ae4c3babff8a2b4a9b0c431/helloworld.pdb"); + + IEnumerable perfMapKey = generator.GetKeys(KeyTypeFlags.PerfMapKeys); + Assert.True(perfMapKey.Count() == 0); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + + const string LinuxPePath = "TestBinaries/PerfMapEnabled/System.ComponentModel.EventBasedAsync.dll"; + using (Stream linuxPeStream = File.OpenRead(LinuxPePath)) + { + var file = new SymbolStoreFile(linuxPeStream, LinuxPePath); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new PEFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "system.componentmodel.eventbasedasync.dll/9757F3A636c00/system.componentmodel.eventbasedasync.dll"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Single().Index == "system.componentmodel.eventbasedasync.pdb/99d3f272c6a8429ba694647a7912d178FFFFFFFF/system.componentmodel.eventbasedasync.pdb"); + + IEnumerable perfMapKey = generator.GetKeys(KeyTypeFlags.PerfMapKeys); + Assert.True(perfMapKey.Single().Index == "system.componentmodel.eventbasedasync.ni.r2rmap/r2rmap-v1-734d59d6de0e96aa3c77b3e2ed498097/system.componentmodel.eventbasedasync.ni.r2rmap"); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + + const string TestBinaryDll = "TestBinaries/System.Diagnostics.StackTrace.dll"; + using (Stream dll = File.OpenRead(TestBinaryDll)) + { + var file = new SymbolStoreFile(dll, TestBinaryDll); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new PEFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "system.diagnostics.stacktrace.dll/595CD91B35a00/system.diagnostics.stacktrace.dll"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 2); + Assert.True(symbolKey.First().Index == "system.diagnostics.stacktrace.ni.pdb/3cd5a68a9f2cd99b169d074e6e956d4fFFFFFFFF/system.diagnostics.stacktrace.ni.pdb"); + Assert.True(symbolKey.Last().Index == "system.diagnostics.stacktrace.pdb/8b2e8cf443144806982ab7d904876a50FFFFFFFF/system.diagnostics.stacktrace.pdb"); + + IEnumerable perfMapKey = generator.GetKeys(KeyTypeFlags.PerfMapKeys); + Assert.True(perfMapKey.Count() == 0); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + + using (Stream coreclr = TestUtilities.OpenCompressedFile("TestBinaries/coreclr.dll.gz")) + { + var file = new SymbolStoreFile(coreclr, "coreclr.dll"); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new PEFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "coreclr.dll/595EBCD5538000/coreclr.dll"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 1); + Assert.True(symbolKey.First().Index == "coreclr.pdb/3f3d5a3258e64ae8b86b31ff776949351/coreclr.pdb"); + + Dictionary clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys).ToDictionary((key) => key.Index); + Assert.True(clrKeys.ContainsKey("mscordaccore.dll/595EBCD5538000/mscordaccore.dll")); + Assert.True(clrKeys.ContainsKey("mscordbi.dll/595EBCD5538000/mscordbi.dll")); + Assert.True(clrKeys.ContainsKey("sos.dll/595EBCD5538000/sos.dll")); + Assert.True(clrKeys.ContainsKey("sos.netcore.dll/595EBCD5538000/sos.netcore.dll")); + + Dictionary dacdbiKeys = generator.GetKeys(KeyTypeFlags.DacDbiKeys).ToDictionary((key) => key.Index); + Assert.True(dacdbiKeys.ContainsKey("mscordaccore.dll/595EBCD5538000/mscordaccore.dll")); + Assert.True(dacdbiKeys.ContainsKey("mscordbi.dll/595EBCD5538000/mscordbi.dll")); + Assert.False(dacdbiKeys.ContainsKey("sos.dll/595EBCD5538000/sos.dll")); + Assert.False(dacdbiKeys.ContainsKey("sos.netcore.dll/595EBCD5538000/sos.netcore.dll")); + + IEnumerable perfMapKey = generator.GetKeys(KeyTypeFlags.PerfMapKeys); + Assert.True(perfMapKey.Count() == 0); + + Dictionary runtimeKeys = generator.GetKeys(KeyTypeFlags.RuntimeKeys).ToDictionary((key) => key.Index); + Assert.True(runtimeKeys.ContainsKey("coreclr.dll/595EBCD5538000/coreclr.dll")); + } + } + + [Fact] + public void PortablePDBFileKeyGenerator() + { + PortablePDBFileKeyGeneratorInternal(fileGenerator: false); + } + + private void PortablePDBFileKeyGeneratorInternal(bool fileGenerator) + { + const string TestBinary = "TestBinaries/dir1/System.Threading.Thread.pdb"; + using (Stream pdb = File.OpenRead(TestBinary)) + { + var file = new SymbolStoreFile(pdb, TestBinary); + KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new PortablePDBFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "system.threading.thread.pdb/a43b38726e6a4b3cb1691f35f0d6cc48FFFFFFFF/system.threading.thread.pdb"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 0); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + } + + [Fact] + public void SourceFileKeyGenerator() + { + using (Stream source = TestUtilities.OpenCompressedFile("TestBinaries/StackTraceSymbols.CoreCLR.cs.gz")) + { + var file = new SymbolStoreFile(source, "StackTraceSymbols.CoreCLR.cs"); + var generator = new SourceFileKeyGenerator(_tracer, file); + + IEnumerable identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(identityKey.Count() == 1); + Assert.True(identityKey.First().Index == "stacktracesymbols.coreclr.cs/sha1-da39a3ee5e6b4b0d3255bfef95601890afd80709/stacktracesymbols.coreclr.cs"); + + IEnumerable symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey); + Assert.True(symbolKey.Count() == 0); + + IEnumerable clrKeys = generator.GetKeys(KeyTypeFlags.ClrKeys); + Assert.True(clrKeys.Count() == 0); + } + } + } +} diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/Microsoft.SymbolStore.UnitTests.csproj b/src/tests/Microsoft.SymbolStore.UnitTests/Microsoft.SymbolStore.UnitTests.csproj new file mode 100644 index 000000000..60f3e2213 --- /dev/null +++ b/src/tests/Microsoft.SymbolStore.UnitTests/Microsoft.SymbolStore.UnitTests.csproj @@ -0,0 +1,95 @@ + + + net6.0 + ;1591;1701 + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/SymbolStoreTests.cs b/src/tests/Microsoft.SymbolStore.UnitTests/SymbolStoreTests.cs new file mode 100644 index 000000000..e93ab2933 --- /dev/null +++ b/src/tests/Microsoft.SymbolStore.UnitTests/SymbolStoreTests.cs @@ -0,0 +1,198 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SymbolStore.KeyGenerators; +using Microsoft.SymbolStore.SymbolStores; +using SOS; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.SymbolStore.Tests +{ + public class SymbolStoreTests + { + readonly ITracer _tracer; + + public SymbolStoreTests(ITestOutputHelper output) + { + _tracer = new Tracer(output); + } + + [Fact] + public async Task CacheSymbolStore() + { + using (Stream pdb = File.OpenRead("TestBinaries/HelloWorld.pdb")) { + // Clean up any previous cache directories + string cacheDirectory = "TestSymbolCache"; + try { + Directory.Delete(cacheDirectory, recursive: true); + } + catch (DirectoryNotFoundException) { + } + var inputFile = new SymbolStoreFile(pdb, "HelloWorld.pdb"); + var generator = new PDBFileKeyGenerator(_tracer, inputFile); + + IEnumerable keys = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(keys.Count() == 1); + SymbolStoreKey key = keys.First(); + + var backingStore = new TestSymbolStore(_tracer, key, inputFile); + var cacheSymbolStore = new CacheSymbolStore(_tracer, backingStore, cacheDirectory); + + // This should put HelloWorld.pdb into the cache + SymbolStoreFile outputFile = await cacheSymbolStore.GetFile(key, CancellationToken.None); + Assert.True(outputFile != null); + + // Should be the exact same instance given to TestSymbolStore + Assert.True(inputFile == outputFile); + + // This should get it from the cache and not the backingStore + backingStore.Dispose(); + outputFile = await cacheSymbolStore.GetFile(key, CancellationToken.None); + Assert.True(outputFile != null); + + // Should NOT be the exact same SymbolStoreFile instance given to TestSymbolStore + Assert.True(inputFile != outputFile); + + // Now make sure the output file from the cache is the same as the pdb we opened above + CompareStreams(pdb, outputFile.Stream); + } + } + + [Fact] + public async Task DirectorySymbolStore() + { + using (Stream pdb = File.OpenRead("TestBinaries/dir1/System.Threading.Thread.pdb")) + { + var inputFile = new SymbolStoreFile(pdb, "System.Threading.Thread.pdb"); + var generator = new PortablePDBFileKeyGenerator(_tracer, inputFile); + + IEnumerable keys = generator.GetKeys(KeyTypeFlags.IdentityKey); + Assert.True(keys.Count() == 1); + SymbolStoreKey key = keys.First(); + + var dir1store = new DirectorySymbolStore(_tracer, null, "TestBinaries/dir1"); + var dir2store = new DirectorySymbolStore(_tracer, dir1store, "TestBinaries/dir2"); + + SymbolStoreFile outputFile = await dir2store.GetFile(key, CancellationToken.None); + Assert.True(outputFile != null); + + // Should NOT be the exact same SymbolStoreFile instance + Assert.True(inputFile != outputFile); + + CompareStreams(pdb, outputFile.Stream); + } + } + + [Fact] + public async Task HttpSymbolStore() + { + using (FileStream downloadStream = File.OpenRead("TestBinaries/dir1/System.Threading.Thread.dll")) + { + using (Stream compareStream = File.OpenRead("TestBinaries/dir1/System.Threading.Thread.pdb")) + { + await DownloadFile(downloadStream, compareStream, ms: true, mi: false, KeyTypeFlags.SymbolKey); + } + } + } + + [Fact] + public async Task SymwebHttpSymbolStore() + { + using (FileStream downloadStream = File.OpenRead("TestBinaries/dir2/System.Threading.Thread.dll")) + { + await DownloadFile(downloadStream, downloadStream, ms: false, mi: true, KeyTypeFlags.IdentityKey); + } + } + + private async Task DownloadFile(FileStream downloadStream, Stream compareStream, bool ms, bool mi, KeyTypeFlags flags) + { + SymbolStoreFile file = new SymbolStoreFile(downloadStream, downloadStream.Name); + SymbolStores.SymbolStore store = null; + if (ms) + { + Uri.TryCreate("https://msdl.microsoft.com/download/symbols/", UriKind.Absolute, out Uri uri); + store = new HttpSymbolStore(_tracer, store, uri); + } + if (mi) + { + Uri.TryCreate("https://symweb/", UriKind.Absolute, out Uri uri); + store = new SymwebHttpSymbolStore(_tracer, store, uri); + } + var generator = new FileKeyGenerator(_tracer, file); + + IEnumerable keys = generator.GetKeys(flags); + Assert.True(keys.Count() > 0); + + foreach (SymbolStoreKey key in keys) + { + if (key.FullPathName.Contains(".ni.pdb")) { + continue; + } + using (SymbolStoreFile symbolFile = await store.GetFile(key, CancellationToken.None)) + { + if (symbolFile != null) + { + Assert.True(downloadStream != symbolFile.Stream); + Assert.True(compareStream != symbolFile.Stream); + + compareStream.Seek(0, SeekOrigin.Begin); + CompareStreams(compareStream, symbolFile.Stream); + } + } + } + } + + private void CompareStreams(Stream stream1, Stream stream2) + { + Assert.True(stream1.Length == stream2.Length); + + stream1.Position = 0; + stream2.Position = 0; + + for (int i = 0; i < stream1.Length; i++) { + int b1 = stream1.ReadByte(); + int b2 = stream2.ReadByte(); + Assert.True(b1 == b2); + if (b1 != b2) { + break; + } + } + } + + sealed class TestSymbolStore : Microsoft.SymbolStore.SymbolStores.SymbolStore + { + readonly SymbolStoreKey _key; + SymbolStoreFile _file; + + public TestSymbolStore(ITracer tracer, SymbolStoreKey key, SymbolStoreFile file) + : base(tracer) + { + _key = key; + _file = file; + } + + protected override Task GetFileInner(SymbolStoreKey key, CancellationToken token) + { + if (_file != null && key.Equals(_key)) + { + _file.Stream.Position = 0; + return Task.FromResult(_file); + } + return Task.FromResult(null); + } + + public override void Dispose() + { + _file = null; + base.Dispose(); + } + } + } +} diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/StackTraceSymbols.CoreCLR.cs.gz b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/StackTraceSymbols.CoreCLR.cs.gz new file mode 100644 index 000000000..a47096bb6 Binary files /dev/null and b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/StackTraceSymbols.CoreCLR.cs.gz differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/coreclr.dll.gz b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/coreclr.dll.gz new file mode 100644 index 000000000..84af0dc7d Binary files /dev/null and b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/coreclr.dll.gz differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir1/System.Threading.Thread.dll b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir1/System.Threading.Thread.dll new file mode 100644 index 000000000..4a9307681 Binary files /dev/null and b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir1/System.Threading.Thread.dll differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir1/System.Threading.Thread.pdb b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir1/System.Threading.Thread.pdb new file mode 100644 index 000000000..57e63f5ab Binary files /dev/null and b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir1/System.Threading.Thread.pdb differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir2/System.Threading.Thread.dll b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir2/System.Threading.Thread.dll new file mode 100644 index 000000000..b9a6924e4 Binary files /dev/null and b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir2/System.Threading.Thread.dll differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir2/System.Threading.Thread.pdb b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir2/System.Threading.Thread.pdb new file mode 100644 index 000000000..2fe974307 Binary files /dev/null and b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/dir2/System.Threading.Thread.pdb differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/libSystem.Security.Cryptography.Native.Apple.dylib.gz b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/libSystem.Security.Cryptography.Native.Apple.dylib.gz new file mode 100644 index 000000000..f2aad5f95 Binary files /dev/null and b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/libSystem.Security.Cryptography.Native.Apple.dylib.gz differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/md5_build_id b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/md5_build_id new file mode 100644 index 000000000..6fbb979ba Binary files /dev/null and b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/md5_build_id differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/stripped_executable b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/stripped_executable new file mode 100644 index 000000000..d1cde58bb Binary files /dev/null and b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/stripped_executable differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/symbolized_executable b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/symbolized_executable new file mode 100644 index 000000000..50865caa1 Binary files /dev/null and b/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/symbolized_executable differ diff --git a/src/tests/Microsoft.SymbolStore.UnitTests/Tracer.cs b/src/tests/Microsoft.SymbolStore.UnitTests/Tracer.cs new file mode 100644 index 000000000..0adf3d85d --- /dev/null +++ b/src/tests/Microsoft.SymbolStore.UnitTests/Tracer.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit.Abstractions; + +namespace Microsoft.SymbolStore.Tests +{ + /// + /// Simple trace/logging support. + /// + internal sealed class Tracer : ITracer + { + private readonly ITestOutputHelper _output; + + public Tracer(ITestOutputHelper output) + { + _output = output; + } + + public void WriteLine(string message) + { + _output.WriteLine(message); + } + + public void WriteLine(string format, params object[] arguments) + { + _output.WriteLine(format, arguments); + } + + public void Information(string message) + { + _output.WriteLine(message); + } + + public void Information(string format, params object[] arguments) + { + _output.WriteLine(format, arguments); + } + + public void Warning(string message) + { + _output.WriteLine("WARNING: " + message); + } + + public void Warning(string format, params object[] arguments) + { + _output.WriteLine("WARNING: " + format, arguments); + } + + public void Error(string message) + { + _output.WriteLine("ERROR: " + message); + } + + public void Error(string format, params object[] arguments) + { + _output.WriteLine("ERROR: " + format, arguments); + } + + public void Verbose(string message) + { + } + + public void Verbose(string format, params object[] arguments) + { + } + } +} diff --git a/src/tests/TestHelpers/MaxStreamReadHelper.cs b/src/tests/TestHelpers/MaxStreamReadHelper.cs new file mode 100644 index 000000000..d5299ed0a --- /dev/null +++ b/src/tests/TestHelpers/MaxStreamReadHelper.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.FileFormats; + +namespace TestHelpers +{ + public class MaxStreamReadHelper : IAddressSpace + { + private readonly IAddressSpace _addressSpace; + + public ulong Max { get; private set; } + + public MaxStreamReadHelper(IAddressSpace address) + { + _addressSpace = address; + } + + public ulong Length + { + get + { + return _addressSpace.Length; + } + } + + public uint Read(ulong position, byte[] buffer, uint bufferOffset, uint count) + { + ulong max = position + count; + if (max > Max) + { + Max = max; + } + return _addressSpace.Read(position, buffer, bufferOffset, count); + } + } +} diff --git a/src/tests/TestHelpers/TestHelpers.cs b/src/tests/TestHelpers/TestHelpers.cs new file mode 100644 index 000000000..d54717fdb --- /dev/null +++ b/src/tests/TestHelpers/TestHelpers.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; +using System.IO.Compression; +using System.Linq; + +namespace TestHelpers +{ + public static class TestUtilities + { + public static Stream OpenCompressedFile(string path) + { + MemoryStream ms = new(); + using (FileStream fs = File.OpenRead(path)) + { + using (GZipStream gs = new(fs, CompressionMode.Decompress)) + { + gs.CopyTo(ms); + } + } + return ms; + } + + public static Stream DecompressFile(string source, string destination) + { + bool fileExists = File.Exists(destination); + FileStream destStream = File.Open(destination, FileMode.OpenOrCreate, FileAccess.ReadWrite); + if (!fileExists || destStream.Length == 0) + { + using (FileStream s = File.OpenRead(source)) + { + using (GZipStream gs = new(s, CompressionMode.Decompress)) + { + gs.CopyTo(destStream); + } + destStream.Position = 0; + } + } + return destStream; + } + + /// + /// Convert an array of bytes to a lower case hex string. + /// + /// array of bytes + /// hex string + public static string ToHexString(byte[] bytes) + { + return string.Concat(bytes.Select(b => b.ToString("x2"))); + } + } +} diff --git a/src/tests/TestHelpers/TestHelpers.csproj b/src/tests/TestHelpers/TestHelpers.csproj new file mode 100644 index 000000000..3ad5a576f --- /dev/null +++ b/src/tests/TestHelpers/TestHelpers.csproj @@ -0,0 +1,10 @@ + + + netstandard2.0 + ;1591;1701 + + + + + +