From: Zachary Turner Date: Fri, 23 Mar 2018 18:43:39 +0000 (+0000) Subject: [PDB] Make our PDBs look more like MS PDBs. X-Git-Tag: llvmorg-7.0.0-rc1~9802 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=a6fb536e5b5c98d3ca46fbd7ea21893f633d21d8;p=platform%2Fupstream%2Fllvm.git [PDB] Make our PDBs look more like MS PDBs. When investigating bugs in PDB generation, the first step is often to do the same link with link.exe and then compare PDBs. But comparing PDBs is hard because two completely different byte sequences can both be correct, so it hampers the investigation when you also have to spend time figuring out not just which bytes are different, but also if the difference is meaningful. This patch fixes a couple of cases related to string table emission, hash table emission, and the order in which we emit strings that makes more of our bytes the same as the bytes generated by MS PDBs. Differential Revision: https://reviews.llvm.org/D44810 llvm-svn: 328348 --- diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp index a13b900..2287dda 100644 --- a/lld/COFF/PDB.cpp +++ b/lld/COFF/PDB.cpp @@ -85,7 +85,12 @@ class PDBLinker { public: PDBLinker(SymbolTable *Symtab) : Alloc(), Symtab(Symtab), Builder(Alloc), TypeTable(Alloc), - IDTable(Alloc), GlobalTypeTable(Alloc), GlobalIDTable(Alloc) {} + IDTable(Alloc), GlobalTypeTable(Alloc), GlobalIDTable(Alloc) { + // This isn't strictly necessary, but link.exe usually puts an empty string + // as the first "valid" string in the string table, so we do the same in + // order to maintain as much byte-for-byte compatibility as possible. + PDBStrTab.insert(""); + } /// Emit the basic PDB structure: initial streams, headers, etc. void initialize(const llvm::codeview::DebugInfo &BuildId); @@ -1066,7 +1071,6 @@ void PDBLinker::initialize(const llvm::codeview::DebugInfo &BuildId) { pdb::DbiStreamBuilder &DbiBuilder = Builder.getDbiBuilder(); DbiBuilder.setAge(BuildId.PDB70.Age); DbiBuilder.setVersionHeader(pdb::PdbDbiV70); - ExitOnErr(DbiBuilder.addDbgStream(pdb::DbgHeaderType::NewFPO, {})); } void PDBLinker::addSectionContrib(pdb::DbiModuleDescriptorBuilder &LinkerModule, diff --git a/lld/test/COFF/pdb-diff.test b/lld/test/COFF/pdb-diff.test deleted file mode 100644 index 17d26b6..0000000 --- a/lld/test/COFF/pdb-diff.test +++ /dev/null @@ -1,215 +0,0 @@ -This test verifies that we produce PDBs compatible with MSVC in various ways. -We check in a cl-generated object file, PDB, and original source which serve -as the "baseline" for us to measure against. Then we link the same object -file with LLD and compare the two PDBs. Since the baseline object file and -PDB are already checked in, we just run LLD on the object file. - -RUN: rm -f %T/pdb-diff-lld.pdb %T/pdb-diff-lld.exe -RUN: lld-link /debug /pdb:%T/pdb-diff-lld.pdb /out:%T/pdb-diff-lld.exe /nodefaultlib \ -RUN: /entry:main %S/Inputs/pdb-diff.obj -RUN: llvm-pdbutil diff -result -values=false -left-bin-root=%S -right-bin-root=D:/src/llvm-mono/lld/test/COFF/ \ -RUN: %T/pdb-diff-lld.pdb %S/Inputs/pdb-diff-cl.pdb | FileCheck %s - -CHECK: ---------------------- -CHECK-NEXT: | MSF Super Block | -CHECK-NEXT: |----------------+---| -CHECK-NEXT: | File | | -CHECK-NEXT: |----------------+---| -CHECK-NEXT: | Block Size | I | -CHECK-NEXT: |----------------+---| -CHECK-NEXT: | Block Count | -CHECK-NEXT: |----------------+---| -CHECK-NEXT: | Unknown 1 | I | -CHECK-NEXT: |----------------+---| -CHECK-NEXT: | Directory Size | -CHECK-NEXT: |----------------+---| -CHECK-NEXT: ------------------------------------ -CHECK-NEXT: | Stream Directory | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | File | | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Stream Count | I | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Old MSF Directory | I | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | PDB Stream | I | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | TPI Stream | I | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | DBI Stream | I | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | IPI Stream | I | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | New FPO Data | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Section Header Data | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Named Stream "/names" | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Named Stream "/LinkInfo" | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Module "Inputs\pdb-diff.obj" | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Module "* Linker *" | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | TPI Hash | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | IPI Hash | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Public Symbol Hash | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Global Symbol Hash | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Symbol Records | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: ------------------------------------ -CHECK-NEXT: | String Table | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | File | | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Number of Strings | D | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Hash Version | I | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Byte Size | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Signature | I | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | Empty Strings | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | {{.*}}pdb-diff.cpp | {{[EI]}} | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | $T0 $ebp = $...p $T0 8 + = | D | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: | d:\src\llvm-...er internal) | D | -CHECK-NEXT: |------------------------------+---| -CHECK-NEXT: ---------------------------- -CHECK-NEXT: | PDB Stream | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | File | | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | Stream Size | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | Age | I | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | Guid | D | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | Signature | D | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | Version | I | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | Features (set) | I | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | Feature | I | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | Named Stream Size | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | Named Streams (map) | {{[EI]}} | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | /names | {{[EI]}} | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: | /LinkInfo | {{[EI]}} | -CHECK-NEXT: |----------------------+---| -CHECK-NEXT: ---------------------------------------------- -CHECK-NEXT: | DBI Stream | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | File | | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Dbi Version | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Age | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Machine | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Flags | D | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Build Major | D | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Build Minor | D | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Build Number | D | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | PDB DLL Version | D | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | PDB DLL RBLD | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (FPO) | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (Exception) | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (Fixup) | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (OmapToSrc) | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (OmapFromSrc) | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (SectionHdr) | {{[EI]}} | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (TokenRidMap) | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (Xdata) | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (Pdata) | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (NewFPO) | {{[EI]}} | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | DBG (SectionHdrOrig) | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Globals Stream | {{[EI]}} | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Publics Stream | {{[EI]}} | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Symbol Records | {{[EI]}} | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Has CTypes | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Is Incrementally Linked | D | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Is Stripped | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Module Count | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Source File Count | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Module "Inputs\pdb-diff.obj" | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Modi | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Obj File Name | {{[EI]}} | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Debug Stream | {{[EI]}} | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - C11 Byte Size | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - C13 Byte Size | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - # of files | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Pdb File Path Index | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Source File Name Index | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Symbol Byte Size | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | Module "* Linker *" | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Modi | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Obj File Name | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Debug Stream | {{[EI]}} | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - C11 Byte Size | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - C13 Byte Size | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - # of files | I | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Pdb File Path Index | {{[EI]}} | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Source File Name Index | {{[EI]}} | -CHECK-NEXT: |----------------------------------------+---| -CHECK-NEXT: | - Symbol Byte Size | -CHECK-NEXT: |----------------------------------------+---| - - diff --git a/lld/test/COFF/pdb-file-static.test b/lld/test/COFF/pdb-file-static.test index 1d09823..f08f717 100644 --- a/lld/test/COFF/pdb-file-static.test +++ b/lld/test/COFF/pdb-file-static.test @@ -43,9 +43,9 @@ # CHECK: ============================================================ # CHECK-LABEL: Mod 0000 | `{{.*}}a.obj`: # CHECK: 232 | S_FILESTATIC [size = 16] `x` -# CHECK-NEXT: type = 0x0074 (int), file name = 1 (D:\src\llvmbuild\cl\Debug\x64\a.obj), flags = enreg global | enreg static +# CHECK-NEXT: type = 0x0074 (int), file name = 2 (D:\src\llvmbuild\cl\Debug\x64\a.obj), flags = enreg global | enreg static # CHECK: Mod 0001 | `{{.*}}b.obj`: # CHECK: 232 | S_FILESTATIC [size = 16] `y` -# CHECK-NEXT: type = 0x0074 (int), file name = 73 (D:\src\llvmbuild\cl\Debug\x64\b.obj), flags = enreg global | enreg static +# CHECK-NEXT: type = 0x0074 (int), file name = 74 (D:\src\llvmbuild\cl\Debug\x64\b.obj), flags = enreg global | enreg static # CHECK-LABEL: Mod 0002 | `* Linker *`: diff --git a/lld/test/COFF/pdb-lib.s b/lld/test/COFF/pdb-lib.s index 319d4bc..c970f0b 100644 --- a/lld/test/COFF/pdb-lib.s +++ b/lld/test/COFF/pdb-lib.s @@ -13,15 +13,15 @@ # CHECK-NEXT: ============================================================ # CHECK-NEXT: Mod 0000 | `{{.*pdb-lib.s.tmp[/\\]foo.obj}}`: # CHECK-NEXT: Obj: `{{.*pdb-lib.s.tmp[/\\]foo.obj}}`: -# CHECK-NEXT: debug stream: 9, # files: 0, has ec info: false +# CHECK-NEXT: debug stream: 10, # files: 0, has ec info: false # CHECK-NEXT: pdb file ni: 0 ``, src file ni: 0 `` # CHECK-NEXT: Mod 0001 | `bar.obj`: # CHECK-NEXT: Obj: `{{.*pdb-lib.s.tmp[/\\]bar.lib}}`: -# CHECK-NEXT: debug stream: 10, # files: 0, has ec info: false +# CHECK-NEXT: debug stream: 11, # files: 0, has ec info: false # CHECK-NEXT: pdb file ni: 0 ``, src file ni: 0 `` # CHECK-NEXT: Mod 0002 | `* Linker *`: # CHECK-NEXT: Obj: ``: -# CHECK-NEXT: debug stream: 11, # files: 0, has ec info: false +# CHECK-NEXT: debug stream: 12, # files: 0, has ec info: false # CHECK-NEXT: pdb file ni: 1 `{{.*foo.pdb}}`, src file ni: 0 `` .def _main; diff --git a/lld/test/COFF/pdb-linker-module.test b/lld/test/COFF/pdb-linker-module.test index 1bb5729..96ca1b4 100644 --- a/lld/test/COFF/pdb-linker-module.test +++ b/lld/test/COFF/pdb-linker-module.test @@ -4,7 +4,7 @@ RUN: llvm-pdbutil dump -symbols %t.pdb | FileCheck --check-prefix=SYMS %s MODS: Mod 0001 | `* Linker *` MODS-NEXT: Obj: ``: -MODS-NEXT: debug stream: 10, # files: 0, has ec info: false +MODS-NEXT: debug stream: 12, # files: 0, has ec info: false MODS-NEXT: pdb file ni: 1 `{{.*}}pdb-linker-module.test.tmp.pdb`, src file ni: 0 `` SYMS: Mod 0001 | `* Linker *` diff --git a/lld/test/COFF/pdb-same-name.test b/lld/test/COFF/pdb-same-name.test index 76db69f..352bfc9 100644 --- a/lld/test/COFF/pdb-same-name.test +++ b/lld/test/COFF/pdb-same-name.test @@ -15,9 +15,9 @@ RAW: Modules RAW-NEXT: ============================================================ RAW-NEXT: Mod 0000 | `foo.obj`: RAW-NEXT: Obj: `{{.*}}1\foo.lib`: -RAW-NEXT: debug stream: 9, # files: 1, has ec info: false +RAW-NEXT: debug stream: 11, # files: 1, has ec info: false RAW-NEXT: pdb file ni: 0 ``, src file ni: 0 `` RAW-NEXT: Mod 0001 | `foo.obj`: RAW-NEXT: Obj: `{{.*}}2\foo.lib`: -RAW-NEXT: debug stream: 10, # files: 1, has ec info: false +RAW-NEXT: debug stream: 12, # files: 1, has ec info: false RAW-NEXT: pdb file ni: 0 ``, src file ni: 0 `` diff --git a/lld/test/COFF/pdb.test b/lld/test/COFF/pdb.test index dad6de2..0d9f128 100644 --- a/lld/test/COFF/pdb.test +++ b/lld/test/COFF/pdb.test @@ -121,15 +121,15 @@ RAW: Modules RAW-NEXT: ============================================================ RAW-NEXT: Mod 0000 | `{{.*}}pdb.test.tmp1.obj`: RAW-NEXT: Obj: `{{.*}}pdb.test.tmp1.obj`: -RAW-NEXT: debug stream: 9, # files: 1, has ec info: false +RAW-NEXT: debug stream: 11, # files: 1, has ec info: false RAW-NEXT: pdb file ni: 0 ``, src file ni: 0 `` RAW-NEXT: Mod 0001 | `{{.*}}pdb.test.tmp2.obj`: RAW-NEXT: Obj: `{{.*}}pdb.test.tmp2.obj`: -RAW-NEXT: debug stream: 10, # files: 1, has ec info: false +RAW-NEXT: debug stream: 12, # files: 1, has ec info: false RAW-NEXT: pdb file ni: 0 ``, src file ni: 0 `` RAW-NEXT: Mod 0002 | `* Linker *`: RAW-NEXT: Obj: ``: -RAW-NEXT: debug stream: 11, # files: 0, has ec info: false +RAW-NEXT: debug stream: 13, # files: 0, has ec info: false RAW-NEXT: pdb file ni: 1 `{{.*pdb.test.tmp.pdb}}`, src file ni: 0 `` RAW: Types (TPI Stream) RAW-NEXT: ============================================================ diff --git a/llvm/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h b/llvm/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h index 9469c06..bebc960 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h +++ b/llvm/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h @@ -82,6 +82,8 @@ public: StringMap::const_iterator end() const { return StringToId.end(); } + std::vector sortedIds() const; + private: DenseMap IdToString; StringMap StringToId; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h b/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h index ad4a0d1..daea062 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h @@ -121,7 +121,7 @@ private: MutableBinaryByteStream FileInfoBuffer; std::vector SectionContribs; ArrayRef SectionMap; - llvm::SmallVector DbgStreams; + std::array, (int)DbgHeaderType::Max> DbgStreams; }; } } diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h b/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h index 4b7aa0d..34cc617 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h @@ -304,12 +304,12 @@ private: void grow() { uint32_t S = size(); + uint32_t MaxLoad = maxLoad(capacity()); if (S < maxLoad(capacity())) return; assert(capacity() != UINT32_MAX && "Can't grow Hash table!"); - uint32_t NewCapacity = - (capacity() <= INT32_MAX) ? capacity() * 2 : UINT32_MAX; + uint32_t NewCapacity = (capacity() <= INT32_MAX) ? MaxLoad * 2 : UINT32_MAX; // Growing requires rebuilding the table and re-hashing every item. Make a // copy with a larger capacity, insert everything into the copy, then swap diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h b/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h index 7ed164b..2604306 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h @@ -54,10 +54,11 @@ public: Error commit(StringRef Filename); Expected getNamedStreamIndex(StringRef Name) const; - Error addNamedStream(StringRef Name, uint32_t Size); + Error addNamedStream(StringRef Name, StringRef Data); private: Expected finalizeMsfLayout(); + Expected allocateNamedStream(StringRef Name, uint32_t Size); void commitFpm(WritableBinaryStream &MsfBuffer, const msf::MSFLayout &Layout); @@ -72,6 +73,7 @@ private: PDBStringTableBuilder Strings; NamedStreamMap NamedStreams; + DenseMap NamedStreamData; }; } } diff --git a/llvm/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp b/llvm/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp index c731b68..1278680 100644 --- a/llvm/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp +++ b/llvm/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp @@ -86,6 +86,15 @@ Error DebugStringTableSubsection::commit(BinaryStreamWriter &Writer) const { uint32_t DebugStringTableSubsection::size() const { return StringToId.size(); } +std::vector DebugStringTableSubsection::sortedIds() const { + std::vector Result; + Result.reserve(IdToString.size()); + for (const auto &Entry : IdToString) + Result.push_back(Entry.first); + std::sort(Result.begin(), Result.end()); + return Result; +} + uint32_t DebugStringTableSubsection::getIdForString(StringRef S) const { auto Iter = StringToId.find(S); assert(Iter != StringToId.end()); diff --git a/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp index c96553f..f17e081 100644 --- a/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp @@ -27,7 +27,7 @@ using namespace llvm::pdb; DbiStreamBuilder::DbiStreamBuilder(msf::MSFBuilder &Msf) : Msf(Msf), Allocator(Msf.getAllocator()), Age(1), BuildNumber(0), PdbDllVersion(0), PdbDllRbld(0), Flags(0), MachineType(PDB_Machine::x86), - Header(nullptr), DbgStreams((int)DbgHeaderType::Max) {} + Header(nullptr) {} DbiStreamBuilder::~DbiStreamBuilder() {} @@ -63,15 +63,8 @@ void DbiStreamBuilder::setPublicsStreamIndex(uint32_t Index) { Error DbiStreamBuilder::addDbgStream(pdb::DbgHeaderType Type, ArrayRef Data) { - if (DbgStreams[(int)Type].StreamNumber != kInvalidStreamIndex) - return make_error(raw_error_code::duplicate_entry, - "The specified stream type already exists"); - auto ExpectedIndex = Msf.addStream(Data.size()); - if (!ExpectedIndex) - return ExpectedIndex.takeError(); - uint32_t Index = std::move(*ExpectedIndex); - DbgStreams[(int)Type].Data = Data; - DbgStreams[(int)Type].StreamNumber = Index; + DbgStreams[(int)Type].emplace(); + DbgStreams[(int)Type]->Data = Data; return Error::success(); } @@ -266,6 +259,15 @@ Error DbiStreamBuilder::finalize() { } Error DbiStreamBuilder::finalizeMsfLayout() { + for (auto &S : DbgStreams) { + if (!S.hasValue()) + continue; + auto ExpectedIndex = Msf.addStream(S->Data.size()); + if (!ExpectedIndex) + return ExpectedIndex.takeError(); + S->StreamNumber = *ExpectedIndex; + } + for (auto &MI : ModiList) { if (auto EC = MI->finalizeMsfLayout()) return EC; @@ -375,17 +377,23 @@ Error DbiStreamBuilder::commit(const msf::MSFLayout &Layout, if (auto EC = ECNamesBuilder.commit(Writer)) return EC; - for (auto &Stream : DbgStreams) - if (auto EC = Writer.writeInteger(Stream.StreamNumber)) + for (auto &Stream : DbgStreams) { + uint16_t StreamNumber = kInvalidStreamIndex; + if (Stream.hasValue()) + StreamNumber = Stream->StreamNumber; + if (auto EC = Writer.writeInteger(StreamNumber)) return EC; + } for (auto &Stream : DbgStreams) { - if (Stream.StreamNumber == kInvalidStreamIndex) + if (!Stream.hasValue()) continue; + assert(Stream->StreamNumber != kInvalidStreamIndex); + auto WritableStream = WritableMappedBlockStream::createIndexedStream( - Layout, MsfBuffer, Stream.StreamNumber, Allocator); + Layout, MsfBuffer, Stream->StreamNumber, Allocator); BinaryStreamWriter DbgStreamWriter(*WritableStream); - if (auto EC = DbgStreamWriter.writeArray(Stream.Data)) + if (auto EC = DbgStreamWriter.writeArray(Stream->Data)) return EC; } diff --git a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp index e84f25d..63d63c1 100644 --- a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp @@ -150,14 +150,14 @@ Error GSIStreamBuilder::finalizeMsfLayout() { PSH->finalizeBuckets(PSHZero); GSH->finalizeBuckets(GSHZero); - Expected Idx = Msf.addStream(calculatePublicsHashStreamSize()); + Expected Idx = Msf.addStream(calculateGlobalsHashStreamSize()); if (!Idx) return Idx.takeError(); - PSH->StreamIndex = *Idx; - Idx = Msf.addStream(calculateGlobalsHashStreamSize()); + GSH->StreamIndex = *Idx; + Idx = Msf.addStream(calculatePublicsHashStreamSize()); if (!Idx) return Idx.takeError(); - GSH->StreamIndex = *Idx; + PSH->StreamIndex = *Idx; uint32_t RecordBytes = GSH->calculateRecordByteSize() + PSH->calculateRecordByteSize(); diff --git a/llvm/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp index a20b451..54d6835 100644 --- a/llvm/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp @@ -73,5 +73,6 @@ Error InfoStreamBuilder::commit(const msf::MSFLayout &Layout, if (auto EC = Writer.writeEnum(E)) return EC; } + assert(Writer.bytesRemaining() == 0); return Error::success(); } diff --git a/llvm/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp b/llvm/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp index 6076b10..a4eaed9 100644 --- a/llvm/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp @@ -47,7 +47,7 @@ uint32_t NamedStreamMapTraits::lookupKeyToStorageKey(StringRef S) { } NamedStreamMap::NamedStreamMap() - : HashTraits(*this), OffsetIndexMap(HashTraits) {} + : HashTraits(*this), OffsetIndexMap(1, HashTraits) {} Error NamedStreamMap::load(BinaryStreamReader &Stream) { uint32_t StringBufferSize; diff --git a/llvm/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp index 1cb890f..38bba9e 100644 --- a/llvm/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp @@ -80,11 +80,20 @@ GSIStreamBuilder &PDBFileBuilder::getGsiBuilder() { return *Gsi; } -Error PDBFileBuilder::addNamedStream(StringRef Name, uint32_t Size) { +Expected PDBFileBuilder::allocateNamedStream(StringRef Name, + uint32_t Size) { auto ExpectedStream = Msf->addStream(Size); - if (!ExpectedStream) - return ExpectedStream.takeError(); - NamedStreams.set(Name, *ExpectedStream); + if (ExpectedStream) + NamedStreams.set(Name, *ExpectedStream); + return ExpectedStream; +} + +Error PDBFileBuilder::addNamedStream(StringRef Name, StringRef Data) { + Expected ExpectedIndex = allocateNamedStream(Name, Data.size()); + if (!ExpectedIndex) + return ExpectedIndex.takeError(); + assert(NamedStreamData.count(*ExpectedIndex) == 0); + NamedStreamData[*ExpectedIndex] = Data; return Error::success(); } @@ -101,35 +110,41 @@ Expected PDBFileBuilder::finalizeMsfLayout() { uint32_t StringsLen = Strings.calculateSerializedSize(); - if (auto EC = addNamedStream("/names", StringsLen)) - return std::move(EC); - if (auto EC = addNamedStream("/LinkInfo", 0)) - return std::move(EC); + Expected SN = allocateNamedStream("/LinkInfo", 0); + if (!SN) + return SN.takeError(); - if (Info) { - if (auto EC = Info->finalizeMsfLayout()) - return std::move(EC); - } - if (Dbi) { - if (auto EC = Dbi->finalizeMsfLayout()) + if (Gsi) { + if (auto EC = Gsi->finalizeMsfLayout()) return std::move(EC); + if (Dbi) { + Dbi->setPublicsStreamIndex(Gsi->getPublicsStreamIndex()); + Dbi->setGlobalsStreamIndex(Gsi->getGlobalsStreamIndex()); + Dbi->setSymbolRecordStreamIndex(Gsi->getRecordStreamIdx()); + } } if (Tpi) { if (auto EC = Tpi->finalizeMsfLayout()) return std::move(EC); } + if (Dbi) { + if (auto EC = Dbi->finalizeMsfLayout()) + return std::move(EC); + } + SN = allocateNamedStream("/names", StringsLen); + if (!SN) + return SN.takeError(); + if (Ipi) { if (auto EC = Ipi->finalizeMsfLayout()) return std::move(EC); } - if (Gsi) { - if (auto EC = Gsi->finalizeMsfLayout()) + + // Do this last, since it relies on the named stream map being complete, and + // that can be updated by previous steps in the finalization. + if (Info) { + if (auto EC = Info->finalizeMsfLayout()) return std::move(EC); - if (Dbi) { - Dbi->setPublicsStreamIndex(Gsi->getPublicsStreamIndex()); - Dbi->setGlobalsStreamIndex(Gsi->getGlobalsStreamIndex()); - Dbi->setSymbolRecordStreamIndex(Gsi->getRecordStreamIdx()); - } } return Msf->build(); @@ -219,6 +234,17 @@ Error PDBFileBuilder::commit(StringRef Filename) { if (auto EC = Strings.commit(NSWriter)) return EC; + for (const auto &NSE : NamedStreamData) { + if (NSE.second.empty()) + continue; + + auto NS = WritableMappedBlockStream::createIndexedStream( + Layout, Buffer, NSE.first, Allocator); + BinaryStreamWriter NSW(*NS); + if (auto EC = NSW.writeBytes(arrayRefFromStringRef(NSE.second))) + return EC; + } + if (Info) { if (auto EC = Info->commit(Layout, Buffer)) return EC; diff --git a/llvm/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp index 5020423..0975d9e 100644 --- a/llvm/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp @@ -15,6 +15,8 @@ #include "llvm/Support/BinaryStreamWriter.h" #include "llvm/Support/Endian.h" +#include + using namespace llvm; using namespace llvm::msf; using namespace llvm::support; @@ -33,13 +35,66 @@ StringRef PDBStringTableBuilder::getStringForId(uint32_t Id) const { return Strings.getStringForId(Id); } +// This is a precomputed list of Buckets given the specified number of +// strings. Matching the reference algorithm exactly is not strictly +// necessary for correctness, but it helps when comparing LLD's PDBs with +// Microsoft's PDBs so as to eliminate superfluous differences. +static std::map StringsToBuckets = { + {1, 2}, + {2, 4}, + {4, 7}, + {6, 11}, + {9, 17}, + {13, 26}, + {20, 40}, + {31, 61}, + {46, 92}, + {70, 139}, + {105, 209}, + {157, 314}, + {236, 472}, + {355, 709}, + {532, 1064}, + {799, 1597}, + {1198, 2396}, + {1798, 3595}, + {2697, 5393}, + {4045, 8090}, + {6068, 12136}, + {9103, 18205}, + {13654, 27308}, + {20482, 40963}, + {30723, 61445}, + {46084, 92168}, + {69127, 138253}, + {103690, 207380}, + {155536, 311071}, + {233304, 466607}, + {349956, 699911}, + {524934, 1049867}, + {787401, 1574801}, + {1181101, 2362202}, + {1771652, 3543304}, + {2657479, 5314957}, + {3986218, 7972436}, + {5979328, 11958655}, + {8968992, 17937983}, + {13453488, 26906975}, + {20180232, 40360463}, + {30270348, 60540695}, + {45405522, 90811043}, + {68108283, 136216565}, + {102162424, 204324848}, + {153243637, 306487273}, + {229865455, 459730910}, + {344798183, 689596366}, + {517197275, 1034394550}, + {775795913, 1551591826}}; + static uint32_t computeBucketCount(uint32_t NumStrings) { - // The /names stream is basically an on-disk open-addressing hash table. - // Hash collisions are resolved by linear probing. We cannot make - // utilization 100% because it will make the linear probing extremely - // slow. But lower utilization wastes disk space. As a reasonable - // load factor, we choose 80%. We need +1 because slot 0 is reserved. - return (NumStrings + 1) * 1.25; + auto Entry = StringsToBuckets.lower_bound(NumStrings); + assert(Entry != StringsToBuckets.end()); + return Entry->second; } uint32_t PDBStringTableBuilder::calculateHashTableSize() const { diff --git a/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp b/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp index 365386f..6694a21 100644 --- a/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp +++ b/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp @@ -90,7 +90,13 @@ Error DumpOutputStyle::dump() { P.NewLine(); } - if (opts::dump::DumpStringTable) { + if (opts::dump::DumpNamedStreams) { + if (auto EC = dumpNamedStreams()) + return EC; + P.NewLine(); + } + + if (opts::dump::DumpStringTable || opts::dump::DumpStringTableDetails) { if (auto EC = dumpStringTable()) return EC; P.NewLine(); @@ -857,33 +863,64 @@ Error DumpOutputStyle::dumpStringTableFromPdb() { return Error::success(); } - if (IS->name_ids().empty()) { - P.formatLine("Empty"); - return Error::success(); + if (opts::dump::DumpStringTable) { + if (IS->name_ids().empty()) + P.formatLine("Empty"); + else { + auto MaxID = + std::max_element(IS->name_ids().begin(), IS->name_ids().end()); + uint32_t Digits = NumDigits(*MaxID); + + P.formatLine("{0} | {1}", fmt_align("ID", AlignStyle::Right, Digits), + "String"); + + std::vector SortedIDs(IS->name_ids().begin(), + IS->name_ids().end()); + std::sort(SortedIDs.begin(), SortedIDs.end()); + for (uint32_t I : SortedIDs) { + auto ES = IS->getStringForID(I); + llvm::SmallString<32> Str; + if (!ES) { + consumeError(ES.takeError()); + Str = "Error reading string"; + } else if (!ES->empty()) { + Str.append("'"); + Str.append(*ES); + Str.append("'"); + } + + if (!Str.empty()) + P.formatLine("{0} | {1}", fmt_align(I, AlignStyle::Right, Digits), + Str); + } + } } - auto MaxID = std::max_element(IS->name_ids().begin(), IS->name_ids().end()); - uint32_t Digits = NumDigits(*MaxID); - - P.formatLine("{0} | {1}", fmt_align("ID", AlignStyle::Right, Digits), - "String"); - - std::vector SortedIDs(IS->name_ids().begin(), IS->name_ids().end()); - std::sort(SortedIDs.begin(), SortedIDs.end()); - for (uint32_t I : SortedIDs) { - auto ES = IS->getStringForID(I); - llvm::SmallString<32> Str; - if (!ES) { - consumeError(ES.takeError()); - Str = "Error reading string"; - } else if (!ES->empty()) { - Str.append("'"); - Str.append(*ES); - Str.append("'"); + if (opts::dump::DumpStringTableDetails) { + P.NewLine(); + { + P.printLine("String Table Header:"); + AutoIndent Indent(P); + P.formatLine("Signature: {0}", IS->getSignature()); + P.formatLine("Hash Version: {0}", IS->getHashVersion()); + P.formatLine("Name Buffer Size: {0}", IS->getByteSize()); + P.NewLine(); } - if (!Str.empty()) - P.formatLine("{0} | {1}", fmt_align(I, AlignStyle::Right, Digits), Str); + BinaryStreamRef NameBuffer = IS->getStringTable().getBuffer(); + ArrayRef Contents; + cantFail(NameBuffer.readBytes(0, NameBuffer.getLength(), Contents)); + P.formatBinary("Name Buffer", Contents, 0); + P.NewLine(); + { + P.printLine("Hash Table:"); + AutoIndent Indent(P); + P.formatLine("Bucket Count: {0}", IS->name_ids().size()); + for (const auto &Entry : enumerate(IS->name_ids())) + P.formatLine("Bucket[{0}] : {1}", Entry.index(), + uint32_t(Entry.value())); + P.formatLine("Name Count: {0}", IS->getNameCount()); + } } return Error::success(); } @@ -909,6 +946,29 @@ Error DumpOutputStyle::dumpStringTableFromObj() { return Error::success(); } +Error DumpOutputStyle::dumpNamedStreams() { + printHeader(P, "Named Streams"); + AutoIndent Indent(P, 2); + + if (File.isObj()) { + P.formatLine("Dumping Named Streams is only supported for PDB files."); + return Error::success(); + } + ExitOnError Err("Invalid PDB File: "); + + auto &IS = Err(File.pdb().getPDBInfoStream()); + const NamedStreamMap &NS = IS.getNamedStreams(); + for (const auto &Entry : NS.entries()) { + P.printLine(Entry.getKey()); + AutoIndent Indent2(P, 2); + P.formatLine("Index: {0}", Entry.getValue()); + P.formatLine("Size in bytes: {0}", + File.pdb().getStreamByteSize(Entry.getValue())); + } + + return Error::success(); +} + Error DumpOutputStyle::dumpStringTable() { printHeader(P, "String Table"); diff --git a/llvm/tools/llvm-pdbutil/DumpOutputStyle.h b/llvm/tools/llvm-pdbutil/DumpOutputStyle.h index fad304c..36d8b13 100644 --- a/llvm/tools/llvm-pdbutil/DumpOutputStyle.h +++ b/llvm/tools/llvm-pdbutil/DumpOutputStyle.h @@ -74,6 +74,7 @@ private: Error dumpStreamSummary(); Error dumpSymbolStats(); Error dumpUdtStats(); + Error dumpNamedStreams(); Error dumpStringTable(); Error dumpStringTableFromPdb(); Error dumpStringTableFromObj(); diff --git a/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp b/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp index 25e0ddf..cdc01d8 100644 --- a/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp +++ b/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp @@ -534,8 +534,16 @@ cl::opt JustMyCode("jmc", cl::Optional, cl::cat(FileOptions), cl::sub(DumpSubcommand)); // MISCELLANEOUS OPTIONS +cl::opt DumpNamedStreams("named-streams", + cl::desc("dump PDB named stream table"), + cl::cat(MiscOptions), cl::sub(DumpSubcommand)); + cl::opt DumpStringTable("string-table", cl::desc("dump PDB String Table"), cl::cat(MiscOptions), cl::sub(DumpSubcommand)); +cl::opt DumpStringTableDetails("string-table-details", + cl::desc("dump PDB String Table Details"), + cl::cat(MiscOptions), + cl::sub(DumpSubcommand)); cl::opt DumpSectionContribs("section-contribs", cl::desc("dump section contributions"), @@ -1199,6 +1207,7 @@ int main(int argc_, const char *argv_[]) { opts::dump::DumpStreams = true; opts::dump::DumpStreamBlocks = true; opts::dump::DumpStringTable = true; + opts::dump::DumpStringTableDetails = true; opts::dump::DumpSummary = true; opts::dump::DumpSymbols = true; opts::dump::DumpSymbolStats = true; diff --git a/llvm/tools/llvm-pdbutil/llvm-pdbutil.h b/llvm/tools/llvm-pdbutil/llvm-pdbutil.h index 3ce03d5..7003b8c 100644 --- a/llvm/tools/llvm-pdbutil/llvm-pdbutil.h +++ b/llvm/tools/llvm-pdbutil/llvm-pdbutil.h @@ -142,7 +142,9 @@ extern llvm::cl::opt DumpLines; extern llvm::cl::opt DumpInlineeLines; extern llvm::cl::opt DumpXmi; extern llvm::cl::opt DumpXme; +extern llvm::cl::opt DumpNamedStreams; extern llvm::cl::opt DumpStringTable; +extern llvm::cl::opt DumpStringTableDetails; extern llvm::cl::opt DumpTypes; extern llvm::cl::opt DumpTypeData; extern llvm::cl::opt DumpTypeExtras;