erofs-utils: fix reproducible builds for multi-threaded libdeflate

`last_uncompressed_size` should be reset on the basis of segments.

Fixes: 830b27bc2334 ("erofs-utils: mkfs: introduce inner-file multi-threaded compression")
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Link: https://lore.kernel.org/r/20240712093808.2986196-1-hsiangkao@linux.alibaba.com

diff --git a/lib/compress.c b/lib/compress.c
index 1f1d4cf02938db2e06932a43c06c553c80ba76f1..794f7144636c37916f6d2e9090d5a3115e160f6c 100644 (file)
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1245,7 +1245,7 @@ void z_erofs_mt_workfn(struct erofs_work *work, void *tlsp)
        sctx->queue = tls->queue;
        sctx->destbuf = tls->destbuf;
        sctx->chandle = &tls->ccfg[cwork->alg_id].handle;
-
+       erofs_compressor_reset(sctx->chandle);
        sctx->membuf = malloc(round_up(sctx->remaining, erofs_blksiz(sbi)));
        if (!sctx->membuf) {
                ret = -ENOMEM;

diff --git a/lib/compressor.c b/lib/compressor.c
index 24c99acecc54e6b918748888626ec2082dc3d1cf..41f49fff249c6208b6808e9c306ad60a89660e97 100644 (file)
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -155,3 +155,9 @@ int erofs_compressor_exit(struct erofs_compress *c)
                return c->alg->c->exit(c);
        return 0;
 }
+
+void erofs_compressor_reset(struct erofs_compress *c)
+{
+       if (c->alg && c->alg->c->reset)
+               c->alg->c->reset(c);
+}

diff --git a/lib/compressor.h b/lib/compressor.h
index 59d525d3c24a0338327de6174476d06e926f8593..8d322d594fb2d3fe3de5045ad37b09e60bca52e0 100644 (file)
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -19,6 +19,7 @@ struct erofs_compressor {
 
        int (*init)(struct erofs_compress *c);
        int (*exit)(struct erofs_compress *c);
+       void (*reset)(struct erofs_compress *c);
        int (*setlevel)(struct erofs_compress *c, int compression_level);
        int (*setdictsize)(struct erofs_compress *c, u32 dict_size);
 
@@ -63,5 +64,6 @@ int erofs_compress_destsize(const struct erofs_compress *c,
 int erofs_compressor_init(struct erofs_sb_info *sbi, struct erofs_compress *c,
                          char *alg_name, int compression_level, u32 dict_size);
 int erofs_compressor_exit(struct erofs_compress *c);
+void erofs_compressor_reset(struct erofs_compress *c);
 
 #endif

diff --git a/lib/compressor_libdeflate.c b/lib/compressor_libdeflate.c
index 14cbce4b7c36bde8e34807801a102f5b12ddfced..aaf4684885a9e51ac49b32761aae88d8947cd1c5 100644 (file)
--- a/lib/compressor_libdeflate.c
+++ b/lib/compressor_libdeflate.c
@@ -3,22 +3,28 @@
 #include "erofs/print.h"
 #include "erofs/config.h"
 #include <libdeflate.h>
+#include <stdlib.h>
 #include "compressor.h"
 #include "erofs/atomic.h"
 
+struct erofs_libdeflate_context {
+       struct libdeflate_compressor *strm;
+       size_t last_uncompressed_size;
+};
+
 static int libdeflate_compress_destsize(const struct erofs_compress *c,
                                        const void *src, unsigned int *srcsize,
                                        void *dst, unsigned int dstsize)
 {
-       static size_t last_uncompressed_size = 0;
+       struct erofs_libdeflate_context *ctx = c->private_data;
        size_t l = 0; /* largest input that fits so far */
        size_t l_csize = 0;
        size_t r = *srcsize + 1; /* smallest input that doesn't fit so far */
        size_t m;
        u8 tmpbuf[dstsize + 9];
 
-       if (last_uncompressed_size)
-               m = last_uncompressed_size * 15 / 16;
+       if (ctx->last_uncompressed_size)
+               m = ctx->last_uncompressed_size * 15 / 16;
        else
                m = dstsize * 4;
        for (;;) {
@@ -27,7 +33,7 @@ static int libdeflate_compress_destsize(const struct erofs_compress *c,
                m = max(m, l + 1);
                m = min(m, r - 1);
 
-               csize = libdeflate_deflate_compress(c->private_data, src, m,
+               csize = libdeflate_deflate_compress(ctx->strm, src, m,
                                                    tmpbuf, dstsize + 9);
                /*printf("Tried %zu => %zu\n", m, csize);*/
                if (csize > 0 && csize <= dstsize) {
@@ -68,40 +74,55 @@ static int libdeflate_compress_destsize(const struct erofs_compress *c,
 
        /*printf("Choosing %zu => %zu\n", l, l_csize);*/
        *srcsize = l;
-       last_uncompressed_size = l;
+       ctx->last_uncompressed_size = l;
        return l_csize;
 }
 
 static int compressor_libdeflate_exit(struct erofs_compress *c)
 {
-       if (!c->private_data)
-               return -EINVAL;
+       struct erofs_libdeflate_context *ctx = c->private_data;
 
-       libdeflate_free_compressor(c->private_data);
+       if (!ctx)
+               return -EINVAL;
+       libdeflate_free_compressor(ctx->strm);
+       free(ctx);
        return 0;
 }
 
 static int compressor_libdeflate_init(struct erofs_compress *c)
 {
        static erofs_atomic_bool_t __warnonce;
+       struct erofs_libdeflate_context *ctx;
 
-       libdeflate_free_compressor(c->private_data);
-       c->private_data = libdeflate_alloc_compressor(c->compression_level);
-       if (!c->private_data)
+       DBG_BUGON(c->private_data);
+       ctx = calloc(1, sizeof(struct erofs_libdeflate_context));
+       if (!ctx)
                return -ENOMEM;
-
+       ctx->strm = libdeflate_alloc_compressor(c->compression_level);
+       if (!ctx->strm) {
+               free(ctx);
+               return -ENOMEM;
+       }
+       c->private_data = ctx;
        if (!erofs_atomic_test_and_set(&__warnonce))
                erofs_warn("EXPERIMENTAL libdeflate compressor in use. Use at your own risk!");
        return 0;
 }
 
+static void compressor_libdeflate_reset(struct erofs_compress *c)
+{
+       struct erofs_libdeflate_context *ctx = c->private_data;
+
+       ctx->last_uncompressed_size = 0;
+}
+
 static int erofs_compressor_libdeflate_setlevel(struct erofs_compress *c,
                                                int compression_level)
 {
        if (compression_level < 0)
-               compression_level = erofs_compressor_deflate.default_level;
+               compression_level = erofs_compressor_libdeflate.default_level;
 
-       if (compression_level > erofs_compressor_deflate.best_level) {
+       if (compression_level > erofs_compressor_libdeflate.best_level) {
                erofs_err("invalid compression level %d", compression_level);
                return -EINVAL;
        }
@@ -114,6 +135,7 @@ const struct erofs_compressor erofs_compressor_libdeflate = {
        .best_level = 12,
        .init = compressor_libdeflate_init,
        .exit = compressor_libdeflate_exit,
+       .reset = compressor_libdeflate_reset,
        .setlevel = erofs_compressor_libdeflate_setlevel,
        .compress_destsize = libdeflate_compress_destsize,
 };