From e4eb6216c2ebeac5203ff9cab8fe96f092d08c0f Mon Sep 17 00:00:00 2001 From: Chris Davis Date: Mon, 6 Dec 2021 18:11:34 -0500 Subject: [PATCH] Enable pdbpagesize to allow support for PDB file sizes > 4GB Enable the pdbpagesize flag to allow linking of PDB files > 4GB. Also includes a couple small fixes to change to uint64_t to support the larger file sizes. I updated the max file size check in MSFBuilder.cpp to take into account the page size. Differential Revision: https://reviews.llvm.org/D115051 --- lld/COFF/DriverUtils.cpp | 6 ------ lld/test/COFF/pdbpagesize.test | 14 ++++++++++--- llvm/include/llvm/DebugInfo/MSF/MSFCommon.h | 20 ++++++++++++++++++ llvm/include/llvm/DebugInfo/MSF/MSFError.h | 5 ++++- llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h | 2 +- llvm/lib/DebugInfo/MSF/MSFBuilder.cpp | 26 ++++++++++++++++-------- llvm/lib/DebugInfo/MSF/MSFError.cpp | 8 +++++++- llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp | 2 +- llvm/tools/llvm-pdbutil/PdbYaml.h | 2 +- 9 files changed, 63 insertions(+), 22 deletions(-) diff --git a/lld/COFF/DriverUtils.cpp b/lld/COFF/DriverUtils.cpp index cac254e..0921c8e 100644 --- a/lld/COFF/DriverUtils.cpp +++ b/lld/COFF/DriverUtils.cpp @@ -186,12 +186,6 @@ void parsePDBPageSize(StringRef s) { return; } - // FIXME: Remove this once other page sizes work. - if (v != 4096) { - warn("/pdbpagesize: page sizes != 4096 not yet implemented, ignoring flag"); - v = 4096; - } - config->pdbPageSize = v; } diff --git a/lld/test/COFF/pdbpagesize.test b/lld/test/COFF/pdbpagesize.test index 1f9df65..d6efc44 100644 --- a/lld/test/COFF/pdbpagesize.test +++ b/lld/test/COFF/pdbpagesize.test @@ -10,6 +10,14 @@ # RUN: llvm-pdbutil pdb2yaml %t.pdb | FileCheck --check-prefix=PAGE4096 %s # PAGE4096: BlockSize: 4096 -# RUN: lld-link /entry:main %t.obj /out:%t.exe /debug /pdbpagesize:8192 2>&1 \ -# RUN: | FileCheck --check-prefix=TODO %s -# TODO: warning: /pdbpagesize: page sizes != 4096 not yet implemented, ignoring flag +# RUN: lld-link /entry:main %t.obj /out:%t.exe /debug /pdbpagesize:8192 +# RUN: llvm-pdbutil pdb2yaml %t.pdb | FileCheck --check-prefix=PAGE8192 %s +# PAGE8192: BlockSize: 8192 + +# RUN: lld-link /entry:main %t.obj /out:%t.exe /debug /pdbpagesize:16384 +# RUN: llvm-pdbutil pdb2yaml %t.pdb | FileCheck --check-prefix=PAGE16384 %s +# PAGE16384: BlockSize: 16384 + +# RUN: lld-link /entry:main %t.obj /out:%t.exe /debug /pdbpagesize:32768 +# RUN: llvm-pdbutil pdb2yaml %t.pdb | FileCheck --check-prefix=PAGE32768 %s +# PAGE32768: BlockSize: 32768 diff --git a/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h b/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h index a922839..0520b94 100644 --- a/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h +++ b/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h @@ -101,6 +101,26 @@ inline bool isValidBlockSize(uint32_t Size) { return false; } +/// Given the specified block size, returns the maximum possible file size. +/// Block Size | Max File Size +/// <= 4096 | 4GB +/// 8192 | 8GB +/// 16384 | 16GB +/// 32768 | 32GB +/// \p Size - the block size of the MSF +inline uint64_t getMaxFileSizeFromBlockSize(uint32_t Size) { + switch (Size) { + case 8192: + return (uint64_t)UINT32_MAX * 2ULL; + case 16384: + return (uint64_t)UINT32_MAX * 3ULL; + case 32768: + return (uint64_t)UINT32_MAX * 4ULL; + default: + return (uint64_t)UINT32_MAX; + } +} + // Super Block, Fpm0, Fpm1, and Block Map inline uint32_t getMinimumBlockCount() { return 4; } diff --git a/llvm/include/llvm/DebugInfo/MSF/MSFError.h b/llvm/include/llvm/DebugInfo/MSF/MSFError.h index 0ef30f1..b84f9d7 100644 --- a/llvm/include/llvm/DebugInfo/MSF/MSFError.h +++ b/llvm/include/llvm/DebugInfo/MSF/MSFError.h @@ -16,7 +16,10 @@ namespace msf { enum class msf_error_code { unspecified = 1, insufficient_buffer, - size_overflow, + size_overflow_4096, + size_overflow_8192, + size_overflow_16384, + size_overflow_32768, not_writable, no_stream, invalid_format, diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h b/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h index 2124e6a..c5ee732 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h @@ -65,7 +65,7 @@ public: uint32_t getStreamByteSize(uint32_t StreamIndex) const override; ArrayRef getStreamBlockList(uint32_t StreamIndex) const override; - uint32_t getFileSize() const; + uint64_t getFileSize() const; Expected> getBlockData(uint32_t BlockIndex, uint32_t NumBytes) const override; diff --git a/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp b/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp index 1a92e2c..437d9e1 100644 --- a/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp +++ b/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp @@ -343,15 +343,25 @@ Expected MSFBuilder::commit(StringRef Path, Layout = std::move(*L); uint64_t FileSize = uint64_t(Layout.SB->BlockSize) * Layout.SB->NumBlocks; - if (FileSize > UINT32_MAX) { - // FIXME: Changing the BinaryStream classes to use 64-bit numbers lets - // us create PDBs larger than 4 GiB successfully. The file format is - // block-based and as long as each stream is small enough, PDBs larger than - // 4 GiB might work. Check if tools can handle these large PDBs, and if so - // add support for writing them. + // Ensure that the file size is under the limit for the specified block size. + if (FileSize > getMaxFileSizeFromBlockSize(Layout.SB->BlockSize)) { + msf_error_code error_code; + switch (Layout.SB->BlockSize) { + case 8192: + error_code = msf_error_code::size_overflow_8192; + case 16384: + error_code = msf_error_code::size_overflow_16384; + case 32768: + error_code = msf_error_code::size_overflow_32768; + default: + error_code = msf_error_code::size_overflow_4096; + } + return make_error( - msf_error_code::size_overflow, - formatv("File size would have been {0,1:N}", FileSize)); + error_code, + formatv("File size would have been {0,1:N} which exceeds the maximum \ + file size for the current page size {1}", + FileSize, Layout.SB->BlockSize)); } auto OutFileOrError = FileOutputBuffer::create(Path, FileSize); diff --git a/llvm/lib/DebugInfo/MSF/MSFError.cpp b/llvm/lib/DebugInfo/MSF/MSFError.cpp index e42157e..9df2158 100644 --- a/llvm/lib/DebugInfo/MSF/MSFError.cpp +++ b/llvm/lib/DebugInfo/MSF/MSFError.cpp @@ -28,8 +28,14 @@ public: case msf_error_code::insufficient_buffer: return "The buffer is not large enough to read the requested number of " "bytes."; - case msf_error_code::size_overflow: + case msf_error_code::size_overflow_4096: return "Output data is larger than 4 GiB."; + case msf_error_code::size_overflow_8192: + return "Output data is larger than 8 GiB."; + case msf_error_code::size_overflow_16384: + return "Output data is larger than 16 GiB."; + case msf_error_code::size_overflow_32768: + return "Output data is larger than 32 GiB."; case msf_error_code::not_writable: return "The specified stream is not writable."; case msf_error_code::no_stream: diff --git a/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp b/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp index cde6452..5c61530 100644 --- a/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp @@ -100,7 +100,7 @@ PDBFile::getStreamBlockList(uint32_t StreamIndex) const { return ContainerLayout.StreamMap[StreamIndex]; } -uint32_t PDBFile::getFileSize() const { return Buffer->getLength(); } +uint64_t PDBFile::getFileSize() const { return Buffer->getLength(); } Expected> PDBFile::getBlockData(uint32_t BlockIndex, uint32_t NumBytes) const { diff --git a/llvm/tools/llvm-pdbutil/PdbYaml.h b/llvm/tools/llvm-pdbutil/PdbYaml.h index ed6346c..2c2878c 100644 --- a/llvm/tools/llvm-pdbutil/PdbYaml.h +++ b/llvm/tools/llvm-pdbutil/PdbYaml.h @@ -40,7 +40,7 @@ struct MSFHeaders { uint32_t NumDirectoryBlocks = 0; std::vector DirectoryBlocks; uint32_t NumStreams = 0; - uint32_t FileSize = 0; + uint64_t FileSize = 0; }; struct StreamBlockList { -- 2.7.4