erofs-utils: use external xxhash library if possible
authorGao Xiang <hsiangkao@linux.alibaba.com>
Fri, 20 Dec 2024 14:41:39 +0000 (22:41 +0800)
committerGao Xiang <hsiangkao@linux.alibaba.com>
Fri, 20 Dec 2024 14:46:01 +0000 (22:46 +0800)
It's expected to be faster than the internal one.

Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Link: https://lore.kernel.org/r/20241220144139.648210-1-hsiangkao@linux.alibaba.com
configure.ac
dump/Makefile.am
fsck/Makefile.am
fuse/Makefile.am
lib/Makefile.am
lib/dedupe.c
lib/liberofs_xxhash.h [new file with mode: 0644]
lib/xattr.c
lib/xxhash.c
lib/xxhash.h [deleted file]
mkfs/Makefile.am

index 45a7d3357d810a6e89eace30bba33e14d7d6f4d9..0a069c5197cb5971405e0972c41b3a72a40cea64 100644 (file)
@@ -148,6 +148,10 @@ AC_ARG_WITH(qpl,
       [Enable and build with Intel QPL support @<:@default=disabled@:>@])], [],
       [with_qpl="no"])
 
+AC_ARG_WITH(xxhash,
+   [AS_HELP_STRING([--with-xxhash],
+      [Enable and build with libxxhash support @<:@default=auto@:>@])])
+
 AC_ARG_ENABLE(fuse,
    [AS_HELP_STRING([--enable-fuse], [enable erofsfuse @<:@default=no@:>@])],
    [enable_fuse="$enableval"], [enable_fuse="no"])
@@ -531,6 +535,31 @@ AS_IF([test "x$with_qpl" != "xno"], [
   ])
 ])
 
+# Configure libxxhash
+have_xxhash="no"
+AS_IF([test "x$with_xxhash" != "xno"], [
+  PKG_CHECK_MODULES([libxxhash], [libxxhash], [
+    # Paranoia: don't trust the result reported by pkgconfig before trying out
+    saved_LIBS="$LIBS"
+    saved_CPPFLAGS=${CPPFLAGS}
+    CPPFLAGS="${libxxhash_CFLAGS} ${CPPFLAGS}"
+    LIBS="${libxxhash_LIBS} $LIBS"
+    AC_CHECK_HEADERS([xxhash.h],[
+      AC_CHECK_LIB(xxhash, XXH32, [], [
+        AC_MSG_ERROR([libxxhash doesn't work properly])])
+      AC_CHECK_DECL(XXH32, [have_xxhash="yes"],
+        [AC_MSG_ERROR([libxxhash doesn't work properly])], [[
+#include <xxhash.h>
+      ]])
+    ])
+    LIBS="${saved_LIBS}"
+    CPPFLAGS="${saved_CPPFLAGS}"], [
+    AS_IF([test "x$with_xxhash" = "xyes"], [
+      AC_MSG_ERROR([Cannot find proper libxxhash])
+    ])
+  ])
+])
+
 # Enable 64-bit off_t
 CFLAGS+=" -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64"
 
@@ -553,6 +582,7 @@ AM_CONDITIONAL([ENABLE_LIBLZMA], [test "x${have_liblzma}" = "xyes"])
 AM_CONDITIONAL([ENABLE_LIBDEFLATE], [test "x${have_libdeflate}" = "xyes"])
 AM_CONDITIONAL([ENABLE_LIBZSTD], [test "x${have_libzstd}" = "xyes"])
 AM_CONDITIONAL([ENABLE_QPL], [test "x${have_qpl}" = "xyes"])
+AM_CONDITIONAL([ENABLE_XXHASH], [test "x${have_xxhash}" = "xyes"])
 AM_CONDITIONAL([ENABLE_STATIC_FUSE], [test "x${enable_static_fuse}" = "xyes"])
 
 if test "x$have_uuid" = "xyes"; then
@@ -600,6 +630,10 @@ if test "x$have_qpl" = "xyes"; then
   AC_SUBST([libqpl_CFLAGS])
 fi
 
+if test "x$have_xxhash" = "xyes"; then
+  AC_DEFINE([HAVE_XXHASH], 1, [Define to 1 if xxhash is found])
+fi
+
 # Dump maximum block size
 AS_IF([test "x$erofs_cv_max_block_size" = "x"],
       [$erofs_cv_max_block_size = 4096], [])
index 2a4f67ae0921a19a44dde17c13f68587f9750268..2cf7fe87b63df1b2e9abbd699b5aad805cef9bcd 100644 (file)
@@ -8,4 +8,4 @@ dump_erofs_SOURCES = main.c
 dump_erofs_CFLAGS = -Wall -I$(top_srcdir)/include
 dump_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \
        ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} \
-       ${libzstd_LIBS} ${libqpl_LIBS}
+       ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS}
index 5bdee4d9f9b2342c987204c94d87f501143b0e20..3b7b59146225e0b5f7f41a2062ffb7110664a3be 100644 (file)
@@ -8,7 +8,7 @@ fsck_erofs_SOURCES = main.c
 fsck_erofs_CFLAGS = -Wall -I$(top_srcdir)/include
 fsck_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \
        ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} \
-       ${libzstd_LIBS} ${libqpl_LIBS}
+       ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS}
 
 if ENABLE_FUZZING
 noinst_PROGRAMS   = fuzz_erofsfsck
@@ -17,5 +17,5 @@ fuzz_erofsfsck_CFLAGS = -Wall -I$(top_srcdir)/include -DFUZZING
 fuzz_erofsfsck_LDFLAGS = -fsanitize=address,fuzzer
 fuzz_erofsfsck_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \
        ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} \
-       ${libzstd_LIBS} ${libqpl_LIBS}
+       ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS}
 endif
index 50186dadd395d07e87721f3ed912b5d8b384ecb1..55fb61fc1d29101bb7f0c6871b7fcb579adec89c 100644 (file)
@@ -8,7 +8,7 @@ erofsfuse_CFLAGS = -Wall -I$(top_srcdir)/include
 erofsfuse_CFLAGS += ${libfuse2_CFLAGS} ${libfuse3_CFLAGS} ${libselinux_CFLAGS}
 erofsfuse_LDADD = $(top_builddir)/lib/liberofs.la ${libfuse2_LIBS} ${libfuse3_LIBS} ${liblz4_LIBS} \
        ${libselinux_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} ${libzstd_LIBS} \
-       ${libqpl_LIBS}
+       ${libqpl_LIBS} ${libxxhash_LIBS}
 
 if ENABLE_STATIC_FUSE
 lib_LTLIBRARIES = liberofsfuse.la
index 9c0604dc0e3f55654615ffdda757ef2923ea3c8e..ef98377b9e637b16fdd34fb4e7b02d4bd800f142 100644 (file)
@@ -27,14 +27,14 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/fragments.h \
       $(top_srcdir)/include/erofs/rebuild.h \
       $(top_srcdir)/lib/liberofs_private.h \
-      $(top_srcdir)/lib/xxhash.h
+      $(top_srcdir)/lib/liberofs_xxhash.h
 
 noinst_HEADERS += compressor.h
 liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
                      namei.c data.c compress.c compressor.c zmap.c decompress.c \
                      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
                      fragments.c dedupe.c uuid_unparse.c uuid.c tar.c \
-                     block_list.c xxhash.c rebuild.c diskbuf.c
+                     block_list.c rebuild.c diskbuf.c
 
 liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
 if ENABLE_LZ4
@@ -58,6 +58,11 @@ if ENABLE_LIBZSTD
 liberofs_la_CFLAGS += ${libzstd_CFLAGS}
 liberofs_la_SOURCES += compressor_libzstd.c
 endif
+if ENABLE_XXHASH
+liberofs_la_CFLAGS += ${libxxhash_CFLAGS}
+else
+liberofs_la_SOURCES += xxhash.c
+endif
 if ENABLE_EROFS_MT
 liberofs_la_LDFLAGS = -lpthread
 liberofs_la_SOURCES += workqueue.c
index 665915a135f178c48bda267ac9a261a36747e486..85ff3c982c4d51c72bc18c9c68ca273df8b7912f 100644 (file)
@@ -6,7 +6,7 @@
 #include "erofs/dedupe.h"
 #include "erofs/print.h"
 #include "rolling_hash.h"
-#include "xxhash.h"
+#include "liberofs_xxhash.h"
 #include "sha256.h"
 
 unsigned long erofs_memcmp2(const u8 *s1, const u8 *s2,
diff --git a/lib/liberofs_xxhash.h b/lib/liberofs_xxhash.h
new file mode 100644 (file)
index 0000000..a0f8367
--- /dev/null
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0+ */
+#ifndef __EROFS_LIB_XXHASH_H
+#define __EROFS_LIB_XXHASH_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include <stdint.h>
+#ifdef HAVE_XXHASH_H
+#include <xxhash.h>
+#endif
+
+#ifdef HAVE_XXHASH
+static inline uint32_t xxh32(const void *input, size_t length, uint32_t seed)
+{
+       return XXH32(input, length, seed);
+}
+
+static inline uint64_t xxh64(const void *input, const size_t len, const uint64_t seed)
+{
+       return XXH64(input, len, seed);
+}
+#else
+/*
+ * xxh32() - calculate the 32-bit hash of the input with a given seed.
+ *
+ * @input:  The data to hash.
+ * @length: The length of the data to hash.
+ * @seed:   The seed can be used to alter the result predictably.
+ *
+ * Return:  The 32-bit hash of the data.
+ */
+uint32_t xxh32(const void *input, size_t length, uint32_t seed);
+
+/*
+ * xxh64() - calculate the 64-bit hash of the input with a given seed.
+ *
+ * @input:  The data to hash.
+ * @length: The length of the data to hash.
+ * @seed:   The seed can be used to alter the result predictably.
+ *
+ * This function runs 2x faster on 64-bit systems, but slower on 32-bit systems.
+ *
+ * Return:  The 64-bit hash of the data.
+ */
+uint64_t xxh64(const void *input, const size_t len, const uint64_t seed);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
index e4207758fa62151e4f1057896272e51a08f745f3..dc919abbccd2502b33bb38e3a5d22b34f14a70d5 100644 (file)
@@ -17,7 +17,7 @@
 #include "erofs/xattr.h"
 #include "erofs/cache.h"
 #include "erofs/fragments.h"
-#include "xxhash.h"
+#include "liberofs_xxhash.h"
 #include "liberofs_private.h"
 
 #ifndef XATTR_SYSTEM_PREFIX
index 2768375da31a458211f8721ed2487f19f286642f..ee78ebf1f70de388279d55221ce5e73f09018aea 100644 (file)
@@ -44,7 +44,7 @@
  * - xxHash source repository: https://github.com/Cyan4973/xxHash
  */
 #include "erofs/defs.h"
-#include "xxhash.h"
+#include "liberofs_xxhash.h"
 
 /*-*************************************
  * Macros
diff --git a/lib/xxhash.h b/lib/xxhash.h
deleted file mode 100644 (file)
index 723c3a5..0000000
+++ /dev/null
@@ -1,40 +0,0 @@
-/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0+ */
-#ifndef __EROFS_LIB_XXHASH_H
-#define __EROFS_LIB_XXHASH_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif
-
-#include <stdint.h>
-
-/*
- * xxh32() - calculate the 32-bit hash of the input with a given seed.
- *
- * @input:  The data to hash.
- * @length: The length of the data to hash.
- * @seed:   The seed can be used to alter the result predictably.
- *
- * Return:  The 32-bit hash of the data.
- */
-uint32_t xxh32(const void *input, size_t length, uint32_t seed);
-
-/*
- * xxh64() - calculate the 64-bit hash of the input with a given seed.
- *
- * @input:  The data to hash.
- * @length: The length of the data to hash.
- * @seed:   The seed can be used to alter the result predictably.
- *
- * This function runs 2x faster on 64-bit systems, but slower on 32-bit systems.
- *
- * Return:  The 64-bit hash of the data.
- */
-uint64_t xxh64(const void *input, const size_t len, const uint64_t seed);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
index 6354712b2191cc1fa3b4c0381e05834e266fd72e..249924211aeccedcdc05ad5fb4140d211ed70572 100644 (file)
@@ -7,4 +7,4 @@ mkfs_erofs_SOURCES = main.c
 mkfs_erofs_CFLAGS = -Wall -I$(top_srcdir)/include
 mkfs_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \
        ${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} \
-       ${libdeflate_LIBS} ${libzstd_LIBS} ${libqpl_LIBS}
+       ${libdeflate_LIBS} ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS}