From 52536d7d213846a3122d4dd40f6268f231109990 Mon Sep 17 00:00:00 2001 From: Mark Wielaard Date: Fri, 18 Sep 2020 12:49:29 +0200 Subject: [PATCH] libdwfl: Add ZSTD support. Newer kernels might be compressed using ZSTD add support to libdwfl open so we can can automatically read ZSTD compressed files and kernel images. The support is very similar to the bzip2 and lzma support, but slightly different. With a bit more macros it could maybe have used the gzip.c USE_INFLATE code path. But I felt that the many macros didn't really help understand the code. So the unzip routine has a slightly different code path for ZSTD. https://sourceware.org/bugzilla/show_bug.cgi?id=26632 Signed-off-by: Mark Wielaard --- ChangeLog | 4 ++ config/ChangeLog | 5 +++ config/elfutils.spec.in | 2 + config/libdw.pc.in | 4 +- configure.ac | 13 +++++- libdwfl/ChangeLog | 12 ++++++ libdwfl/Makefile.am | 3 ++ libdwfl/gzip.c | 79 +++++++++++++++++++++++++++++++++++- libdwfl/libdwflP.h | 5 +++ libdwfl/linux-kernel-modules.c | 3 ++ libdwfl/open.c | 6 +++ libdwfl/zstd.c | 4 ++ tests/ChangeLog | 6 +++ tests/Makefile.am | 5 +++ tests/run-readelf-compressed-zstd.sh | 39 ++++++++++++++++++ 15 files changed, 184 insertions(+), 6 deletions(-) create mode 100644 libdwfl/zstd.c create mode 100755 tests/run-readelf-compressed-zstd.sh diff --git a/ChangeLog b/ChangeLog index 094a798..021b06f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2020-09-18 Mark Wielaard + + * configure.ac: Check availability of libzstd and zstd. + 2020-09-08 Mark Wielaard * configure.ac: Set version to 0.181. diff --git a/config/ChangeLog b/config/ChangeLog index 1cb3d20..c8e4fcd 100644 --- a/config/ChangeLog +++ b/config/ChangeLog @@ -1,3 +1,8 @@ +2020-09-18 Mark Wielaard + + * elfutils.spec.in: Add BuildRequires for libzstd-devel and zstd. + * libdw.pc.in: Requires.private libzstd. + 2020-09-08 Mark Wielaard * elfutils.spec.in: Update for 0.181. diff --git a/config/elfutils.spec.in b/config/elfutils.spec.in index 95f63f5..37af1b0 100644 --- a/config/elfutils.spec.in +++ b/config/elfutils.spec.in @@ -24,6 +24,7 @@ BuildRequires: flex BuildRequires: zlib-devel BuildRequires: bzip2-devel BuildRequires: xz-devel +BuildRequires: libzstd-devel # For debuginfod BuildRequires: pkgconfig(libmicrohttpd) >= 0.9.33 @@ -33,6 +34,7 @@ BuildRequires: pkgconfig(libarchive) >= 3.1.2 # For tests need to bunzip2 test files. BuildRequires: bzip2 +BuildRequires: zstd # For the run-debuginfod-find.sh test case in %check for /usr/sbin/ss BuildRequires: iproute BuildRequires: bsdtar diff --git a/config/libdw.pc.in b/config/libdw.pc.in index 3fc283d..2e83a43 100644 --- a/config/libdw.pc.in +++ b/config/libdw.pc.in @@ -17,6 +17,6 @@ Requires: libelf = @VERSION@ # We support various compressed ELF images, but don't export any of the # data structures or functions. zlib (gz) is always required, bzip2 (bz2) -# and lzma (xz) are optional. But bzip2 doesn't have a pkg-config file. -Requires.private: zlib @LIBLZMA@ +# lzma (xz) and zstd () are optional. But bzip2 doesn't have a pkg-config file. +Requires.private: zlib @LIBLZMA@ @LIBZSTD@ Libs.private: @BZ2_LIB@ diff --git a/configure.ac b/configure.ac index bf83387..1b794df 100644 --- a/configure.ac +++ b/configure.ac @@ -397,8 +397,8 @@ eu_ZIPLIB(zlib,ZLIB,z,gzdirect,gzip) AS_IF([test "x$with_zlib" = xno], [AC_MSG_ERROR([zlib not found but is required])]) LIBS="$save_LIBS" -dnl Test for bzlib and xz/lzma, gives BZLIB/LZMALIB .am -dnl conditional and config.h USE_BZLIB/USE_LZMALIB #define. +dnl Test for bzlib and xz/lzma/zstd, gives BZLIB/LZMALIB/ZSTD .am +dnl conditional and config.h USE_BZLIB/USE_LZMALIB/USE_ZSTD #define. save_LIBS="$LIBS" LIBS= eu_ZIPLIB(bzlib,BZLIB,bz2,BZ2_bzdopen,bzip2) @@ -408,6 +408,9 @@ AC_SUBST([BZ2_LIB]) eu_ZIPLIB(lzma,LZMA,lzma,lzma_auto_decoder,[LZMA (xz)]) AS_IF([test "x$with_lzma" = xyes], [LIBLZMA="liblzma"], [LIBLZMA=""]) AC_SUBST([LIBLZMA]) +eu_ZIPLIB(zstd,ZSTD,zstd,ZSTD_decompress,[ZSTD (zst)]) +AS_IF([test "x$with_zstd" = xyes], [LIBZSTD="libzstd"], [LIBLZSTD=""]) +AC_SUBST([LIBZSTD]) zip_LIBS="$LIBS" LIBS="$save_LIBS" AC_SUBST([zip_LIBS]) @@ -677,6 +680,10 @@ if test "$HAVE_BUNZIP2" = "no"; then AC_MSG_WARN([No bunzip2, needed to run make check]) fi +# For tests that need to use zstd compression +AC_CHECK_PROG(HAVE_ZSTD, zstd, yes, no) +AM_CONDITIONAL([HAVE_ZSTD],[test "x$HAVE_ZSTD" = "xyes"]) + # Look for libcurl for libdebuginfod minimum version as per rhel7. AC_ARG_ENABLE([libdebuginfod],AC_HELP_STRING([--enable-libdebuginfod], [Build debuginfod client library (can be =dummy)])) AS_IF([test "x$enable_libdebuginfod" != "xno"], [ @@ -742,6 +749,7 @@ AC_MSG_NOTICE([ gzip support : ${with_zlib} bzip2 support : ${with_bzlib} lzma/xz support : ${with_lzma} + zstd support : ${with_zstd} libstdc++ demangle support : ${enable_demangler} File textrel check : ${enable_textrelcheck} Symbol versioning : ${enable_symbol_versioning} @@ -759,6 +767,7 @@ AC_MSG_NOTICE([ EXTRA TEST FEATURES (used with make check) have bunzip2 installed (required) : ${HAVE_BUNZIP2} + have zstd installed : ${HAVE_ZSTD} debug branch prediction : ${use_debugpred} gprof support : ${use_gprof} gcov support : ${use_gcov} diff --git a/libdwfl/ChangeLog b/libdwfl/ChangeLog index ca10ce8..344db7c 100644 --- a/libdwfl/ChangeLog +++ b/libdwfl/ChangeLog @@ -1,3 +1,15 @@ +2020-09-18 Mark Wielaard + + * zstd.c: New file. + * libdwflP.h: Add DWFL_E_ZSTD and __libdw_unzstd. + * Makefile.am (libdwfl_a_SOURCES): add zstd.c if ZSTD. + * gzip.c: Add defines and includes for ZSTD. + (zlib_fail): Don't define for ZSTD. + (unzip): Change pread_retry failure from zlib_fail to fail. + Add ZSTD support. + * open.c (decompress): Also try __libdw_unzstd. + * linux-kernel-modules.c (check_suffix): Also TRY ".ko.zst". + 2020-08-20 Dmitry V. Levin * Makefile.am (libdwfl_a_SOURCES): Conditionalize diff --git a/libdwfl/Makefile.am b/libdwfl/Makefile.am index 1de0549..a0013e4 100644 --- a/libdwfl/Makefile.am +++ b/libdwfl/Makefile.am @@ -78,6 +78,9 @@ endif if LZMA libdwfl_a_SOURCES += lzma.c endif +if ZSTD +libdwfl_a_SOURCES += zstd.c +endif if LIBDEBUGINFOD libdwfl_a_SOURCES += debuginfod-client.c endif diff --git a/libdwfl/gzip.c b/libdwfl/gzip.c index e9988cc..ba8ecfb 100644 --- a/libdwfl/gzip.c +++ b/libdwfl/gzip.c @@ -48,6 +48,12 @@ # define inflateInit(z) lzma_auto_decoder (z, 1 << 30, 0) # define do_inflate(z) lzma_code (z, LZMA_RUN) # define inflateEnd(z) lzma_end (z) +#elif defined ZSTD +# define USE_INFLATE 1 +# include +# define unzip __libdw_unzstd +# define DWFL_E_ZLIB DWFL_E_ZSTD +# define MAGIC "\x28\xb5\x2f\xfd" #elif defined BZLIB # define USE_INFLATE 1 # include @@ -119,6 +125,7 @@ fail (struct unzip_state *state, Dwfl_Error failure) return failure; } +#ifndef ZSTD static inline Dwfl_Error zlib_fail (struct unzip_state *state, int result) { @@ -132,6 +139,7 @@ zlib_fail (struct unzip_state *state, int result) return fail (state, DWFL_E_ZLIB); } } +#endif #if !USE_INFLATE static Dwfl_Error @@ -197,7 +205,7 @@ unzip (int fd, off_t start_offset, ssize_t n = pread_retry (fd, state.input_buffer, READ_SIZE, start_offset); if (unlikely (n < 0)) - return zlib_fail (&state, Z (ERRNO)); + return fail (&state, DWFL_E_ERRNO); state.input_pos = n; mapped = state.input_buffer; @@ -223,7 +231,74 @@ unzip (int fd, off_t start_offset, /* Not a compressed file. */ return DWFL_E_BADELF; -#if USE_INFLATE +#ifdef ZSTD + /* special case for libzstd since it is slightly different from the + API provided by bzlib and liblzma. */ + + void *next_in = mapped; + size_t avail_in = state.mapped_size; + void *next_out = NULL; + size_t avail_out = 0; + size_t total_out = 0; + + size_t result; + ZSTD_DCtx *dctx = ZSTD_createDCtx(); + if (dctx == NULL) + return fail (&state, DWFL_E_NOMEM); + + do + { + if (avail_in == 0 && state.input_buffer != NULL) + { + ssize_t n = pread_retry (fd, state.input_buffer, READ_SIZE, + start_offset + state.input_pos); + if (unlikely (n < 0)) + { + ZSTD_freeDCtx (dctx); + return fail (&state, DWFL_E_ERRNO); + } + next_in = state.input_buffer; + avail_in = n; + state.input_pos += n; + } + if (avail_out == 0) + { + ptrdiff_t pos = (void *) next_out - state.buffer; + if (!bigger_buffer (&state, avail_in)) + { + ZSTD_freeDCtx (dctx); + return fail (&state, DWFL_E_NOMEM); + } + next_out = state.buffer + pos; + avail_out = state.size - pos; + } + + ZSTD_inBuffer input = { next_in, avail_in, 0 }; + ZSTD_outBuffer output = { next_out, avail_out, 0 }; + result = ZSTD_decompressStream (dctx, &output, &input); + + if (! ZSTD_isError (result)) + { + total_out += output.pos; + next_out += output.pos; + avail_out -= output.pos; + next_in += input.pos; + avail_in -= input.pos; + } + + if (result == 0) + break; + } + while (avail_in > 0 && ! ZSTD_isError (result)); + + ZSTD_freeDCtx (dctx); + + if (ZSTD_isError (result)) + return fail (&state, DWFL_E_ZSTD); + + smaller_buffer (&state, total_out); + +#elif USE_INFLATE /* This style actually only works with bzlib and liblzma. The stupid zlib interface has nothing to grok the diff --git a/libdwfl/libdwflP.h b/libdwfl/libdwflP.h index ad6779a..4c6fcb2 100644 --- a/libdwfl/libdwflP.h +++ b/libdwfl/libdwflP.h @@ -61,6 +61,7 @@ typedef struct Dwfl_Process Dwfl_Process; DWFL_ERROR (ZLIB, N_("gzip decompression failed")) \ DWFL_ERROR (BZLIB, N_("bzip2 decompression failed")) \ DWFL_ERROR (LZMA, N_("LZMA decompression failed")) \ + DWFL_ERROR (ZSTD, N_("zstd decompression failed")) \ DWFL_ERROR (UNKNOWN_MACHINE, N_("no support library found for machine")) \ DWFL_ERROR (NOREL, N_("Callbacks missing for ET_REL file")) \ DWFL_ERROR (BADRELTYPE, N_("Unsupported relocation type")) \ @@ -612,6 +613,10 @@ extern Dwfl_Error __libdw_unlzma (int fd, off_t start_offset, void *mapped, size_t mapped_size, void **whole, size_t *whole_size) internal_function; +extern Dwfl_Error __libdw_unzstd (int fd, off_t start_offset, + void *mapped, size_t mapped_size, + void **whole, size_t *whole_size) + internal_function; /* Skip the image header before a file image: updates *START_OFFSET. */ extern Dwfl_Error __libdw_image_header (int fd, off_t *start_offset, diff --git a/libdwfl/linux-kernel-modules.c b/libdwfl/linux-kernel-modules.c index 548cb56..6edb27f 100644 --- a/libdwfl/linux-kernel-modules.c +++ b/libdwfl/linux-kernel-modules.c @@ -357,6 +357,9 @@ check_suffix (const FTSENT *f, size_t namelen) #if USE_LZMA TRY (".ko.xz"); #endif +#if USE_ZSTD + TRY (".ko.zst"); +#endif return 0; diff --git a/libdwfl/open.c b/libdwfl/open.c index 35fc528..77bd2bd 100644 --- a/libdwfl/open.c +++ b/libdwfl/open.c @@ -44,6 +44,10 @@ # define __libdw_unlzma(...) DWFL_E_BADELF #endif +#if !USE_ZSTD +# define __libdw_unzstd(...) DWFL_E_BADELF +#endif + /* Consumes and replaces *ELF only on success. */ static Dwfl_Error decompress (int fd __attribute__ ((unused)), Elf **elf) @@ -64,6 +68,8 @@ decompress (int fd __attribute__ ((unused)), Elf **elf) error = __libdw_bunzip2 (fd, offset, mapped, mapped_size, &buffer, &size); if (error == DWFL_E_BADELF) error = __libdw_unlzma (fd, offset, mapped, mapped_size, &buffer, &size); + if (error == DWFL_E_BADELF) + error = __libdw_unzstd (fd, offset, mapped, mapped_size, &buffer, &size); if (error == DWFL_E_NOERROR) { diff --git a/libdwfl/zstd.c b/libdwfl/zstd.c new file mode 100644 index 0000000..dc4d523 --- /dev/null +++ b/libdwfl/zstd.c @@ -0,0 +1,4 @@ +/* libzstd is pretty close to zlib and bzlib. */ + +#define ZSTD +#include "gzip.c" diff --git a/tests/ChangeLog b/tests/ChangeLog index 5f2b144..5a8b589 100644 --- a/tests/ChangeLog +++ b/tests/ChangeLog @@ -1,3 +1,9 @@ +2020-09-18 Mark Wielaard + + * run-readelf-compressed-zstd.sh: New test. + * Makefile.am (EXTRA_DISTS): Add run-readelf-compressed-zstd.sh. + (TESTS): Add run-readelf-compressed-zstd.sh if HAVE_ZSTD. + 2020-09-03 Mark Wielaard * run-readelf-frames.sh: New test. diff --git a/tests/Makefile.am b/tests/Makefile.am index 4629ce6..9d0707d 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -204,6 +204,10 @@ if LZMA TESTS += run-readelf-s.sh run-dwflsyms.sh endif +if HAVE_ZSTD +TESTS += run-readelf-compressed-zstd.sh +endif + if HAVE_LIBASM check_PROGRAMS += $(asm_TESTS) TESTS += $(asm_TESTS) run-disasm-bpf.sh @@ -256,6 +260,7 @@ EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \ run-nm-syms.sh testfilesyms32.bz2 testfilesyms64.bz2 \ run-nm-self.sh run-readelf-self.sh run-readelf-info-plus.sh \ run-readelf-compressed.sh \ + run-readelf-compressed-zstd.sh \ run-readelf-const-values.sh testfile-const-values.debug.bz2 \ run-addrcfi.sh run-dwarfcfi.sh \ testfile11-debugframe.bz2 testfile12-debugframe.bz2 \ diff --git a/tests/run-readelf-compressed-zstd.sh b/tests/run-readelf-compressed-zstd.sh new file mode 100755 index 0000000..9620809 --- /dev/null +++ b/tests/run-readelf-compressed-zstd.sh @@ -0,0 +1,39 @@ +#! /bin/sh +# Copyright (C) 2018 Red Hat, Inc. +# This file is part of elfutils. +# +# This file is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# elfutils is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. $srcdir/test-subr.sh + +if ! grep -q -F '#define USE_ZSTD' ${abs_top_builddir}/config.h; then + echo "elfutils built without zstd support" + exit 77 +fi + +# See run-strip-reloc.sh +testfiles hello_i386.ko + +tempfiles hello_i386.ko.zst readelf.out.1 readelf.out.2 + +testrun ${abs_top_builddir}/src/readelf -a hello_i386.ko > readelf.out.1 +zstd hello_i386.ko +testrun ${abs_top_builddir}/src/readelf -a hello_i386.ko.zst > readelf.out.2 + +diff -u readelf.out.1 readelf.out.2 +if [ $? != 0 ]; then + exit 1; +fi + +exit 0 -- 2.7.4