Add PDZ indexing to symbol store (#4907)
authorMike McLaughlin <mikem@microsoft.com>
Fri, 6 Sep 2024 17:27:26 +0000 (10:27 -0700)
committerGitHub <noreply@github.com>
Fri, 6 Sep 2024 17:27:26 +0000 (10:27 -0700)
Change debug asserts to release argument exceptions for better parameter
validation

---------

Co-authored-by: Juan Hoyos <19413848+hoyosjs@users.noreply.github.com>
15 files changed:
documentation/symbols/SSQP_Key_Conventions.md
src/Microsoft.FileFormats/StreamAddressSpace.cs
src/Microsoft.SymbolStore/KeyGenerators/ELFFileKeyGenerator.cs
src/Microsoft.SymbolStore/KeyGenerators/KeyGenerator.cs
src/Microsoft.SymbolStore/KeyGenerators/MachOKeyGenerator.cs
src/Microsoft.SymbolStore/KeyGenerators/PDBFileKeyGenerator.cs
src/Microsoft.SymbolStore/KeyGenerators/PEFileKeyGenerator.cs
src/Microsoft.SymbolStore/KeyGenerators/PerfMapFileKeyGenerator.cs
src/Microsoft.SymbolStore/KeyGenerators/PortablePDBFileKeyGenerator.cs
src/Microsoft.SymbolStore/KeyGenerators/SourceFileKeyGenerator.cs
src/Microsoft.SymbolStore/SymbolStoreFile.cs
src/Microsoft.SymbolStore/SymbolStoreKey.cs
src/tests/Microsoft.FileFormats.UnitTests/Microsoft.FileFormats.UnitTests.csproj
src/tests/Microsoft.SymbolStore.UnitTests/KeyGeneratorTests.cs
src/tests/Microsoft.SymbolStore.UnitTests/Microsoft.SymbolStore.UnitTests.csproj

index 34d5e4f753c53022179e0de21811d756e1ea82c6..e5a720547401319328fdcbf4653dfdd233674304 100644 (file)
@@ -49,6 +49,25 @@ Example:
 
 **Lookup key**: `foo.pdb/497b72f6390a44fc878e5a2d63b6cc4b1/foo.pdb`
 
+### PDZ-Signature-Age
+
+This applies to the Microsoft C++ Symbol Format with compressed streams, known as PDZ or msfz, also commonly saved with the pdb extension. 
+
+Like regular C++ PDBs, the key also uses values extracted from the GUID stream which is uncompressed. Additionally, the index contains a marker for the type ('msfz') and version (currently only '0'):
+
+`<filename>/<Signature><Age>/msfz<version>/<filename>`
+
+Example:
+
+**File name:** `Foo.pdb`
+
+**Signature field:** `{ 0x497B72F6, 0x390A, 0x44FC, { 0x87, 0x8E, 0x5A, 0x2D, 0x63, 0xB6, 0xCC, 0x4B } }`
+
+**Age field:** `0x1`
+
+**Format version:** `0`
+
+**Lookup key**: `foo.pdb/497b72f6390a44fc878e5a2d63b6cc4b1/msfz0/foo.pdb`
 
 ### Portable-Pdb-Signature
 
index 773c411dd3eb41bc3ea4f644bf5c92fdc1a9d9b7..278a2115e44f33f17984873af1de05de20a7daf1 100644 (file)
@@ -18,7 +18,10 @@ namespace Microsoft.FileFormats
 
         public StreamAddressSpace(Stream stream)
         {
-            System.Diagnostics.Debug.Assert(stream.CanSeek);
+            if (stream is null || !stream.CanSeek)
+            {
+                throw new ArgumentException("Stream null or not seekable", nameof(stream));
+            }
             _stream = stream;
             Length = (ulong)stream.Length;
         }
index 4dd81a3f213b28741d238e42149a31d3237b7f15..c7de8cc2c3a5a23a3e389940e2fdb75f305a2423 100644 (file)
@@ -38,8 +38,8 @@ namespace Microsoft.SymbolStore.KeyGenerators
         public ELFFileKeyGenerator(ITracer tracer, ELFFile elfFile, string path)
             : base(tracer)
         {
-            _elfFile = elfFile;
-            _path = path;
+            _elfFile = elfFile ?? throw new ArgumentNullException(nameof(elfFile));
+            _path = path ?? throw new ArgumentNullException(nameof(path));
         }
 
         public ELFFileKeyGenerator(ITracer tracer, SymbolStoreFile file)
index e2f51f2a9bf57b467c2e762ae7cd85aa6b083c1a..03307b909acf887e6602f0b0f8278f84c764531b 100644 (file)
@@ -118,7 +118,7 @@ namespace Microsoft.SymbolStore.KeyGenerators
         protected static SymbolStoreKey BuildKey(string path, string id, bool clrSpecialFile = false, IEnumerable<PdbChecksum> pdbChecksums = null)
         {
             string file = GetFileName(path).ToLowerInvariant();
-            return BuildKey(path, null, id, file, clrSpecialFile, pdbChecksums);
+            return BuildKey(path, prefix: null, id, type: null, file, clrSpecialFile, pdbChecksums);
         }
 
         /// <summary>
@@ -148,7 +148,7 @@ namespace Microsoft.SymbolStore.KeyGenerators
         /// <returns>key</returns>
         protected static SymbolStoreKey BuildKey(string path, string prefix, byte[] id, string file, bool clrSpecialFile = false, IEnumerable<PdbChecksum> pdbChecksums = null)
         {
-            return BuildKey(path, prefix, ToHexString(id), file, clrSpecialFile, pdbChecksums);
+            return BuildKey(path, prefix, ToHexString(id), type: null, file, clrSpecialFile, pdbChecksums);
         }
 
         /// <summary>
@@ -163,15 +163,44 @@ namespace Microsoft.SymbolStore.KeyGenerators
         /// <returns>key</returns>
         protected static SymbolStoreKey BuildKey(string path, string prefix, string id, string file, bool clrSpecialFile = false, IEnumerable<PdbChecksum> pdbChecksums = null)
         {
+            return BuildKey(path, prefix, id, type: null, file, clrSpecialFile, pdbChecksums);
+        }
+
+        /// <summary>
+        /// Base key building helper for "file/{prefix}-id/{type}/file" indexes.
+        /// </summary>
+        /// <param name="path">full path of file or binary</param>
+        /// <param name="prefix">optional id prefix</param>
+        /// <param name="id">build id or uuid</param>
+        /// <param name="type">optional type or format piece</param>
+        /// <param name="file">file name only</param>
+        /// <param name="clrSpecialFile">if true, the file is one the clr special files</param>
+        /// <param name="pdbChecksums">Checksums of pdb file. May be null.</param>
+        /// <returns>key</returns>
+        internal static SymbolStoreKey BuildKey(string path, string prefix, string id, string type, string file, bool clrSpecialFile, IEnumerable<PdbChecksum> pdbChecksums)
+        {
+            if (id is null or "")
+            {
+                throw new ArgumentNullException(nameof(id));
+            }
+            if (file is null or "")
+            {
+                throw new ArgumentNullException(nameof(file));
+            }
             StringBuilder key = new();
             key.Append(file);
             key.Append('/');
-            if (prefix != null)
+            if (prefix is not null)
             {
                 key.Append(prefix);
                 key.Append('-');
             }
             key.Append(id);
+            if (type is not null)
+            {
+                key.Append('/');
+                key.Append(type);
+            }
             key.Append('/');
             key.Append(file);
             return new SymbolStoreKey(key.ToString(), path, clrSpecialFile, pdbChecksums);
@@ -181,7 +210,7 @@ namespace Microsoft.SymbolStore.KeyGenerators
         /// <returns>hex string</returns>
         public static string ToHexString(byte[] bytes)
         {
-            if (bytes == null)
+            if (bytes is null)
             {
                 throw new ArgumentNullException(nameof(bytes));
             }
@@ -196,6 +225,10 @@ namespace Microsoft.SymbolStore.KeyGenerators
         /// <returns>just the file name</returns>
         internal static string GetFileName(string path)
         {
+            if (path is null or "")
+            {
+                throw new ArgumentNullException(nameof(path));
+            }
             return Path.GetFileName(path.Replace('\\', '/'));
         }
     }
index 02a89b9538d44d9e689c8f0267cfeea32ca5844f..ae4f9106d180918bf8ff676da16a3680c7fa91c5 100644 (file)
@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
@@ -33,8 +34,8 @@ namespace Microsoft.SymbolStore.KeyGenerators
         public MachOFileKeyGenerator(ITracer tracer, MachOFile machoFile, string path)
             : base(tracer)
         {
-            _machoFile = machoFile;
-            _path = path;
+            _machoFile = machoFile ?? throw new ArgumentNullException(nameof(machoFile));
+            _path = path ?? throw new ArgumentNullException(nameof(path));
         }
 
         public MachOFileKeyGenerator(ITracer tracer, SymbolStoreFile file)
index 79aa08ce8bc9227a3ae1a0582c0553d91b4d631d..1e0e62765082e35d5a0c11e4a8171bad39809d5e 100644 (file)
@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using Microsoft.FileFormats;
 using Microsoft.FileFormats.PDB;
 using Microsoft.FileFormats.PE;
@@ -18,6 +19,10 @@ namespace Microsoft.SymbolStore.KeyGenerators
         public PDBFileKeyGenerator(ITracer tracer, SymbolStoreFile file)
             : base(tracer)
         {
+            if (file is null)
+            {
+                throw new ArgumentNullException(nameof(file));
+            }
             StreamAddressSpace dataSource = new(file.Stream);
             _pdbFile = new PDBFile(dataSource);
             _path = file.FileName;
@@ -34,14 +39,10 @@ namespace Microsoft.SymbolStore.KeyGenerators
             {
                 if ((flags & KeyTypeFlags.IdentityKey) != 0)
                 {
-                    if (_pdbFile.DbiStream.IsValid())
-                    {
-                        yield return GetKey(_path, _pdbFile.Signature, unchecked((int)_pdbFile.DbiAge));
-                    }
-                    else
-                    {
-                        yield return GetKey(_path, _pdbFile.Signature, unchecked((int)_pdbFile.Age));
-                    }
+                    uint age = _pdbFile.DbiStream.IsValid() ? _pdbFile.DbiAge : _pdbFile.Age;
+                    // No format type if legacy Windows PDB (MSF), otherwise, pass container type string (i.e. msfz0)
+                    string type = _pdbFile.ContainerKind == PDBContainerKind.MSF ? null : _pdbFile.ContainerKindSpecString;
+                    yield return GetKey(_path, _pdbFile.Signature, unchecked((int)age), type, pdbChecksums: null);
                 }
             }
         }
@@ -52,12 +53,26 @@ namespace Microsoft.SymbolStore.KeyGenerators
         /// <param name="path">file name and path</param>
         /// <param name="signature">mvid guid</param>
         /// <param name="age">pdb age</param>
+        /// <param name="pdbChecksums">Checksums of pdb file. May be null.</param>
         /// <returns>symbol store key</returns>
         public static SymbolStoreKey GetKey(string path, Guid signature, int age, IEnumerable<PdbChecksum> pdbChecksums = null)
         {
-            Debug.Assert(path != null);
-            Debug.Assert(signature != null);
-            return BuildKey(path, string.Format("{0}{1:x}", signature.ToString("N"), age));
+            return GetKey(path, signature, age, type: null, pdbChecksums);
+        }
+
+        /// <summary>
+        /// Create a symbol store key for a Windows PDB or PDZ.
+        /// </summary>
+        /// <param name="path">file name and path</param>
+        /// <param name="signature">mvid guid</param>
+        /// <param name="age">pdb age</param>
+        /// <param name="type">PDB format type like msfz0 or null</param>
+        /// <param name="pdbChecksums">Checksums of pdb file. May be null.</param>
+        /// <returns>symbol store key</returns>
+        public static SymbolStoreKey GetKey(string path, Guid signature, int age, string type, IEnumerable<PdbChecksum> pdbChecksums = null)
+        {
+            string file = GetFileName(path).ToLowerInvariant();
+            return BuildKey(path, prefix: null, string.Format("{0}{1:x}", signature.ToString("N"), age), type, file, clrSpecialFile: false, pdbChecksums);
         }
     }
 }
index ed8b2190e6f6b88a31b6577cad8a465c23b020a2..5ccfd355a3459685bc35dfd007b28a9f844bfd47 100644 (file)
@@ -29,8 +29,8 @@ namespace Microsoft.SymbolStore.KeyGenerators
         public PEFileKeyGenerator(ITracer tracer, PEFile peFile, string path)
             : base(tracer)
         {
-            _peFile = peFile;
-            _path = path;
+            _peFile = peFile ?? throw new ArgumentNullException(nameof(peFile));
+            _path = path ?? throw new ArgumentNullException(nameof(path));
         }
 
         public PEFileKeyGenerator(ITracer tracer, SymbolStoreFile file)
index 9ecc355fbf8ac23f8894ef7c3e0ad434e05f0c96..f7efd4fab68dd8264d41bda25a9336e10481a87b 100644 (file)
@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
@@ -16,7 +17,7 @@ namespace Microsoft.SymbolStore.KeyGenerators
         public PerfMapFileKeyGenerator(ITracer tracer, SymbolStoreFile file)
             : base(tracer)
         {
-            _file = file;
+            _file = file ?? throw new ArgumentNullException(nameof(file));
             _perfmapFile = new PerfMapFile(_file.Stream);
         }
 
index 7bc3536ed5601fe8d7eb9ce54a8f0537d2d8a596..c11fb42b4e1bbc1982f86ac8303955c0f9237e1e 100644 (file)
@@ -16,7 +16,7 @@ namespace Microsoft.SymbolStore.KeyGenerators
         public PortablePDBFileKeyGenerator(ITracer tracer, SymbolStoreFile file)
             : base(tracer)
         {
-            _file = file;
+            _file = file ?? throw new ArgumentNullException(nameof(file));
         }
 
         public override bool IsValid()
@@ -56,7 +56,7 @@ namespace Microsoft.SymbolStore.KeyGenerators
                         else
                         {
                             // Force the Windows PDB index
-                            key = PDBFileKeyGenerator.GetKey(_file.FileName, blob.Guid, 1);
+                            key = PDBFileKeyGenerator.GetKey(_file.FileName, blob.Guid, age: 1);
                         }
                     }
                 }
index c9095b8aceae085e7e1dda3885e8f59990557c06..cb0631149eb0eeeb02cf76e3b117c3216a5ee20f 100644 (file)
@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Security.Cryptography;
@@ -14,7 +15,7 @@ namespace Microsoft.SymbolStore.KeyGenerators
         public SourceFileKeyGenerator(ITracer tracer, SymbolStoreFile file)
             : base(tracer)
         {
-            _file = file;
+            _file = file ?? throw new ArgumentNullException(nameof(file));
         }
 
         public override bool IsValid()
index 3c6493d098366ef45b2feab063bee9a4fd9aa5f0..2e4f01e518432c38ca92d7d8320c416e1ab6bfbc 100644 (file)
@@ -35,10 +35,14 @@ namespace Microsoft.SymbolStore
         /// <param name="fileName">name of the file</param>
         public SymbolStoreFile(Stream stream, string fileName)
         {
-            Debug.Assert(stream != null);
-            Debug.Assert(stream.CanSeek);
-            Debug.Assert(fileName != null);
-
+            if (stream is null || !stream.CanSeek)
+            {
+                throw new ArgumentException("Stream null or not seekable", nameof(stream));
+            }
+            if (fileName is null or "")
+            {
+                throw new ArgumentNullException(nameof(fileName));
+            }
             Stream = stream;
             FileName = fileName;
         }
index 8cb60f71ff61017d4ec8e753867f42ac55cb8744..a7f57ff56422aa5624deb81b39435cc61f2ff04a 100644 (file)
@@ -50,17 +50,17 @@ namespace Microsoft.SymbolStore
         /// <param name="pdbChecksums">if true, the file is one the clr special files</param>
         public SymbolStoreKey(string index, string fullPathName, bool clrSpecialFile = false, IEnumerable<PdbChecksum> pdbChecksums = null)
         {
-            Debug.Assert(index != null && fullPathName != null);
-            Index = index;
-            FullPathName = fullPathName;
+            Index = index ?? throw new ArgumentNullException(nameof(index));
+            FullPathName = fullPathName ?? throw new ArgumentNullException(nameof(fullPathName));
             IsClrSpecialFile = clrSpecialFile;
             PdbChecksums = pdbChecksums ?? Enumerable.Empty<PdbChecksum>();
         }
 
         /// <summary>
-        /// Returns the first two parts of the index tuple. Allows a different file name
+        /// Returns the first two or three parts of the index. Allows a different file name
         /// to be appended to this symbol key. Includes the trailing "/".
         /// </summary>
+        [Obsolete]
         public string IndexPrefix
         {
             get { return Index.Substring(0, Index.LastIndexOf("/") + 1); }
@@ -98,24 +98,28 @@ namespace Microsoft.SymbolStore
         public static bool IsKeyValid(string index)
         {
             string[] parts = index.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
-            if (parts.Length != 3) {
+            if (parts.Length < 3 || parts.Length > 4)
+            {
                 return false;
             }
-            for (int i = 0; i < 3; i++)
+            for (int i = 0; i < parts.Length; i++)
             {
                 foreach (char c in parts[i])
                 {
-                    if (char.IsLetterOrDigit(c)) {
+                    if (char.IsLetterOrDigit(c))
+                    {
                         continue;
                     }
-                    if (!s_invalidChars.Contains(c)) {
+                    if (!s_invalidChars.Contains(c))
+                    {
                         continue;
                     }
                     return false;
                 }
                 // We need to support files with . in the name, but we don't want identifiers that
                 // are meaningful to the filesystem
-                if (parts[i] == "." || parts[i] == "..") {
+                if (parts[i] == "." || parts[i] == "..")
+                {
                     return false;
                 }
             }
index f46b5d9310671b60c49d076e631c39fce1e7cfcc..59a73891df73efa2279f79811b537a46731c11f3 100644 (file)
@@ -23,6 +23,9 @@
     <Content Include="$(MSBuildThisFileDirectory)TestBinaries\HelloWorld.pdb">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="$(MSBuildThisFileDirectory)TestBinaries\HelloWorld.pdz">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="$(MSBuildThisFileDirectory)TestBinaries\libclrjit.dylib.dwarf.gz">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
@@ -60,9 +63,4 @@
     <ProjectReference Include="$(MSBuildThisFileDirectory)..\TestHelpers\TestHelpers.csproj" />
   </ItemGroup>
 
-  <ItemGroup>
-    <None Update="TestBinaries\HelloWorld.pdz">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
 </Project>
index 9da934c6b7b6007858d9792509c9f0a870826d93..85509b5b96f8dc6bfea689040f880c8d55069b73 100644 (file)
@@ -28,7 +28,8 @@ namespace Microsoft.SymbolStore.Tests
             //MachCoreKeyGeneratorInternal(fileGenerator: true);
             MachOFileKeyGeneratorInternal(fileGenerator: true);
             MinidumpKeyGeneratorInternal(fileGenerator: true);
-            PDBFileKeyGeneratorInternal(fileGenerator: true);
+            PDBFileKeyGeneratorInternal(fileGenerator: true, pdz: false);
+            PDBFileKeyGeneratorInternal(fileGenerator: true, pdz: true);
             PEFileKeyGeneratorInternal(fileGenerator: true);
             PortablePDBFileKeyGeneratorInternal(fileGenerator: true);
             PerfMapFileKeyGeneratorInternal(fileGenerator: true);
@@ -348,20 +349,30 @@ namespace Microsoft.SymbolStore.Tests
         [Fact]
         public void PDBFileKeyGenerator()
         {
-            PDBFileKeyGeneratorInternal(fileGenerator: false);
+            PDBFileKeyGeneratorInternal(fileGenerator: false, pdz: false);
         }
 
-        private void PDBFileKeyGeneratorInternal(bool fileGenerator)
+        [Fact]
+        public void PDZFileKeyGenerator()
         {
-            const string TestBinary = "TestBinaries/HelloWorld.pdb";
-            using (Stream pdb = File.OpenRead(TestBinary))
+            PDBFileKeyGeneratorInternal(fileGenerator: false, pdz: true);
+        }
+
+        private void PDBFileKeyGeneratorInternal(bool fileGenerator, bool pdz)
+        {
+            using (Stream pdb = File.OpenRead(pdz ? "TestBinaries/HelloWorld.pdz" : "TestBinaries/HelloWorld.pdb"))
             {
-                var file = new SymbolStoreFile(pdb, TestBinary);
+                var file = new SymbolStoreFile(pdb, "TestBinaries/HelloWorld.pdb");
                 KeyGenerator generator = fileGenerator ? (KeyGenerator)new FileKeyGenerator(_tracer, file) : new PDBFileKeyGenerator(_tracer, file);
 
                 IEnumerable<SymbolStoreKey> identityKey = generator.GetKeys(KeyTypeFlags.IdentityKey);
                 Assert.True(identityKey.Count() == 1);
-                Assert.True(identityKey.First().Index == "helloworld.pdb/99891b3ed7ae4c3babff8a2b4a9b0c431/helloworld.pdb");
+
+                string index = pdz ?
+                    "helloworld.pdb/99891b3ed7ae4c3babff8a2b4a9b0c431/msfz0/helloworld.pdb" :
+                    "helloworld.pdb/99891b3ed7ae4c3babff8a2b4a9b0c431/helloworld.pdb";
+
+                Assert.True(identityKey.First().Index == index);
 
                 IEnumerable<SymbolStoreKey> symbolKey = generator.GetKeys(KeyTypeFlags.SymbolKey);
                 Assert.True(symbolKey.Count() == 0);
index a603d622ca2aeab795ad2fadf67a0ae51cd8c24b..04396b193c97ac0b8b785567691f33a46a001a52 100644 (file)
@@ -55,6 +55,9 @@
     <Content Include="$(MSBuildThisFileDirectory)..\Microsoft.FileFormats.UnitTests\TestBinaries\HelloWorld.pdb" Link="TestBinaries\HelloWorld.pdb">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="$(MSBuildThisFileDirectory)..\Microsoft.FileFormats.UnitTests\TestBinaries\HelloWorld.pdz" Link="TestBinaries\HelloWorld.pdz">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="$(MSBuildThisFileDirectory)..\Microsoft.FileFormats.UnitTests\TestBinaries\HelloWorld.exe" Link="TestBinaries\HelloWorld.exe">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>