[ELF] Add --compress-debug-sections=zstd

`clang -gz=zstd a.o` passes this option to the linker. This option compresses output
debug sections with zstd and sets ch_type to ELFCOMPRESS_ZSTD. As of today, very
few DWARF consumers recognize ELFCOMPRESS_ZSTD.

Use the llvm::zstd::compress API with level llvm::zstd::DefaultCompression (5),
which we may tune after we have more experience with zstd output.
zstd has built-in parallel compression support (so we don't need to do D117853
for zlib), which is not leveraged yet.

Reviewed By: peter.smith

Differential Revision: https://reviews.llvm.org/D133548

diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index accf1fd..1f8ff93 100644 (file)
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -19,6 +19,7 @@
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Support/CachePruning.h"
 #include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Compression.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/GlobPattern.h"
 #include "llvm/Support/PrettyStackTrace.h"
@@ -164,7 +165,7 @@ struct Configuration {
   bool callGraphProfileSort;
   bool checkSections;
   bool checkDynamicRelocs;
-  bool compressDebugSections;
+  llvm::DebugCompressionType compressDebugSections;
   bool cref;
   llvm::SmallVector<std::pair<llvm::GlobPattern, uint64_t>, 0>
       deadRelocInNonAlloc;

diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 650b3cc..369ef02 100644 (file)
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -947,15 +947,21 @@ template <class ELFT> static void readCallGraphsFromObjectFiles() {
   }
 }
 
-static bool getCompressDebugSections(opt::InputArgList &args) {
+static DebugCompressionType getCompressDebugSections(opt::InputArgList &args) {
   StringRef s = args.getLastArgValue(OPT_compress_debug_sections, "none");
-  if (s == "none")
-    return false;
-  if (s != "zlib")
+  if (s == "zlib") {
+    if (!compression::zlib::isAvailable())
+      error("--compress-debug-sections: zlib is not available");
+    return DebugCompressionType::Zlib;
+  }
+  if (s == "zstd") {
+    if (!compression::zstd::isAvailable())
+      error("--compress-debug-sections: zstd is not available");
+    return DebugCompressionType::Zstd;
+  }
+  if (s != "none")
     error("unknown --compress-debug-sections value: " + s);
-  if (!compression::zlib::isAvailable())
-    error("--compress-debug-sections: zlib is not available");
-  return true;
+  return DebugCompressionType::None;
 }
 
 static StringRef getAliasSpelling(opt::Arg *arg) {

diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index d9266e5..72d5b6e 100644 (file)
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -60,7 +60,7 @@ defm check_sections: B<"check-sections",
 
 defm compress_debug_sections:
   Eq<"compress-debug-sections", "Compress DWARF debug sections">,
-  MetaVarName<"[none,zlib]">;
+  MetaVarName<"[none,zlib,zstd]">;
 
 defm defsym: Eq<"defsym", "Define a symbol alias">, MetaVarName<"<symbol>=<value>">;
 

diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 48515b7..517e815 100644 (file)
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -17,6 +17,7 @@
 #include "lld/Common/Memory.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Config/llvm-config.h" // LLVM_ENABLE_ZLIB
+#include "llvm/Support/Compression.h"
 #include "llvm/Support/Parallel.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TimeProfiler.h"
@@ -320,18 +321,31 @@ static SmallVector<uint8_t, 0> deflateShard(ArrayRef<uint8_t> in, int level,
 
 // Compress section contents if this section contains debug info.
 template <class ELFT> void OutputSection::maybeCompress() {
-#if LLVM_ENABLE_ZLIB
   using Elf_Chdr = typename ELFT::Chdr;
 
   // Compress only DWARF debug sections.
-  if (!config->compressDebugSections || (flags & SHF_ALLOC) ||
-      !name.startswith(".debug_") || size == 0)
+  if (config->compressDebugSections == DebugCompressionType::None ||
+      (flags & SHF_ALLOC) || !name.startswith(".debug_") || size == 0)
     return;
 
   llvm::TimeTraceScope timeScope("Compress debug sections");
+  compressed.uncompressedSize = size;
+  auto buf = std::make_unique<uint8_t[]>(size);
+  if (config->compressDebugSections == DebugCompressionType::Zstd) {
+    {
+      parallel::TaskGroup tg;
+      writeTo<ELFT>(buf.get(), tg);
+    }
+    compressed.shards = std::make_unique<SmallVector<uint8_t, 0>[]>(1);
+    compression::zstd::compress(makeArrayRef(buf.get(), size),
+                                compressed.shards[0]);
+    size = sizeof(Elf_Chdr) + compressed.shards[0].size();
+    flags |= SHF_COMPRESSED;
+    return;
+  }
 
+#if LLVM_ENABLE_ZLIB
   // Write uncompressed data to a temporary zero-initialized buffer.
-  auto buf = std::make_unique<uint8_t[]>(size);
   {
     parallel::TaskGroup tg;
     writeTo<ELFT>(buf.get(), tg);
@@ -361,7 +375,6 @@ template <class ELFT> void OutputSection::maybeCompress() {
 
   // Update section size and combine Alder-32 checksums.
   uint32_t checksum = 1;       // Initial Adler-32 value
-  compressed.uncompressedSize = size;
   size = sizeof(Elf_Chdr) + 2; // Elf_Chdir and zlib header
   for (size_t i = 0; i != numShards; ++i) {
     size += shardsOut[i].size();
@@ -400,10 +413,15 @@ void OutputSection::writeTo(uint8_t *buf, parallel::TaskGroup &tg) {
   // just write it down.
   if (compressed.shards) {
     auto *chdr = reinterpret_cast<typename ELFT::Chdr *>(buf);
-    chdr->ch_type = ELFCOMPRESS_ZLIB;
     chdr->ch_size = compressed.uncompressedSize;
     chdr->ch_addralign = alignment;
     buf += sizeof(*chdr);
+    if (config->compressDebugSections == DebugCompressionType::Zstd) {
+      chdr->ch_type = ELFCOMPRESS_ZSTD;
+      memcpy(buf, compressed.shards[0].data(), compressed.shards[0].size());
+      return;
+    }
+    chdr->ch_type = ELFCOMPRESS_ZLIB;
 
     // Compute shard offsets.
     auto offsets = std::make_unique<size_t[]>(compressed.numShards);

diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index af8b555..2aad9b8 100644 (file)
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -28,6 +28,9 @@ ELF Improvements
 
 * ``ELFCOMPRESS_ZSTD`` compressed input sections are now supported.
   (`D129406 <https://reviews.llvm.org/D129406>`_)
+* ``--compress-debug-sections=zstd`` is now available to compress debug
+  sections with zstd (``ELFCOMPRESS_ZSTD``).
+  (`D133548 <https://reviews.llvm.org/D133548>`_)
 
 Breaking changes
 ----------------

diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index b81eeb2..7a4aa21 100644 (file)
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -130,16 +130,22 @@ Alias for
 .Fl -color-diagnostics Ns = Ns Cm auto .
 .It Fl -compress-debug-sections Ns = Ns Ar value
 Compress DWARF debug sections.
-.Ar value
+.Cm value
 may be
-.Cm none
-or
-.Cm zlib .
+.Pp
+.Bl -tag -width 2n -compact
+.It Cm none
+No compression.
+.It Cm zlib
 The default compression level is 1 (fastest) as the debug info usually
-compresses well at that level, but if you want to compress it more,
+compresses well at that level. If you want to compress it more,
 you can specify
 .Fl O2
 to set the compression level to 6.
+.It Cm zstd
+The compression level is 5.
+.El
+.Pp
 .It Fl -cref
 Output cross reference table. If
 .Fl Map

diff --git a/lld/test/ELF/compress-debug-sections-zstd.s b/lld/test/ELF/compress-debug-sections-zstd.s
index 59e4644..cf029c1 100644 (file)
--- a/lld/test/ELF/compress-debug-sections-zstd.s
+++ b/lld/test/ELF/compress-debug-sections-zstd.s
@@ -13,6 +13,13 @@
 # CHECK-NEXT: 0x00000030 6e74006c 6f6e6720 756e7369 676e6564 nt.long unsigned
 # CHECK-NEXT: 0x00000040 20696e74 00                          int.
 
+# RUN: ld.lld %t.o -o %t.so -shared --compress-debug-sections=zstd
+# RUN: llvm-readelf -S %t.so | FileCheck %s --check-prefix=OUTPUT-SEC
+# RUN: llvm-objcopy --decompress-debug-sections %t.so
+# RUN: llvm-readelf -S -x .debug_str %t.so | FileCheck %s
+
+# OUTPUT-SEC: .debug_str    PROGBITS [[#%x,]] [[#%x,]] [[#%x,]] 01 MSC 0 0  1
+
 .section .debug_str,"MS",@progbits,1
 .LASF2:
  .string "short unsigned int"