Support for threaded XZ encoding (Issue #53)

Added new CMAKE (compile-time) option ``-DENABLE_THREADED_XZ_ENCODER=ON``
which enable use of threaded ``lzma_stream_encoder_mt`` encoder instead
of single-threaded ``lzma_easy_encoder``.

Note: This option is disabled by default because Createrepo_c
parallelize a lot of tasks (including compression) by default, this
only adds extra threads on XZ library level which causes thread bloat
and doesn't bring any performance boost for most use-cases.
On regular hardware (e.g. less-or-equal 4 cores) this option may even
cause degradation of performance.

Threaded decoding isn't supported yet as it isn't supported
in the xz library [1].

[1] http://comments.gmane.org/gmane.comp.compression.xz.devel/225

Signed-off-by: Tomas Mlcoch <tmlcoch@redhat.com>

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7757066fff259507c922f28529a4305648a13029..46d5c16017d22fffe433f85a1c5f4908c991b571 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -98,6 +98,18 @@ IF (ENABLE_DRPM)
     ENDIF (CR_DELTA_RPM_SUPPORT)
 ENDIF (ENABLE_DRPM)
 
+# Threaded XZ Compression
+# Note: This option is disabled by default, because Createrepo_c
+# parallelize a lot of tasks (including compression) by default, this
+# only adds extra threads on XZ library level which causes thread bloat
+# and for most usecases doesn't bring any performance boost.
+# On regular hardware (e.g. less-or-equal 4 cores) this option may even
+# cause degradation of performance.
+OPTION(ENABLE_THREADED_XZ_ENCODER "Enable threaded XZ encoder?" OFF)
+IF (ENABLE_THREADED_XZ_ENCODER)
+    ADD_DEFINITIONS("-DENABLE_THREADED_XZ_ENCODER=1")
+ENDIF (ENABLE_THREADED_XZ_ENCODER)
+
 # Get package version
 INCLUDE (${CMAKE_SOURCE_DIR}/VERSION.cmake)
 SET (VERSION "${CR_MAJOR}.${CR_MINOR}.${CR_PATCH}")

diff --git a/README.md b/README.md
index 175ef2c1d63ea46e0575fb9c8a0c547627b2c642..11f781891375f2e4e269a1dc87d383e0b19192fe 100644 (file)
--- a/README.md
+++ b/README.md
@@ -77,6 +77,24 @@ By default, cmake should set up things to build for Python 2, but you can do a b
 
     cmake -DPYTHON_DESIRED=3 .
 
+## Other build options
+
+### ``-DENABLE_LEGACY_WEAKDEPS=ON``
+
+Enable old SUSE weaktags (Default: OFF)
+
+### ``-DENABLE_THREADED_XZ_ENCODER=ON``
+
+Threaded XZ encoding (Default: OFF)
+
+Note: This option is disabled by default, because Createrepo_c
+parallelize a lot of tasks (including compression) by default, this
+only adds extra threads on XZ library level which causes thread bloat
+and for most usecases doesn't bring any performance boost.
+On regular hardware (e.g. less-or-equal 4 cores) this option may even
+cause degradation of performance.
+
+
 ## Build tarball
 
     utils/make_tarball.sh [git revision]

diff --git a/src/compression_wrapper.c b/src/compression_wrapper.c
index db35929fc4d20111904099cee90d651922a02de5..aacaf90f343f641284567fce6eef994b357be29e 100644 (file)
--- a/src/compression_wrapper.c
+++ b/src/compression_wrapper.c
@@ -443,14 +443,65 @@ cr_sopen(const char *filename,
 
             // Prepare coder/decoder
 
-            if (mode == CR_CW_MODE_WRITE)
-                ret = lzma_easy_encoder(stream,
-                                        CR_CW_XZ_COMPRESSION_LEVEL,
-                                        XZ_CHECK);
-            else
+            if (mode == CR_CW_MODE_WRITE) {
+
+#ifdef ENABLE_THREADED_XZ_ENCODER
+                // The threaded encoder takes the options as pointer to
+                // a lzma_mt structure.
+                lzma_mt mt = {
+                    // No flags are needed.
+                    .flags = 0,
+
+                    // Let liblzma determine a sane block size.
+                    .block_size = 0,
+
+                    // Use no timeout for lzma_code() calls by setting timeout
+                    // to zero. That is, sometimes lzma_code() might block for
+                    // a long time (from several seconds to even minutes).
+                    // If this is not OK, for example due to progress indicator
+                    // needing updates, specify a timeout in milliseconds here.
+                    // See the documentation of lzma_mt in lzma/container.h for
+                    // information how to choose a reasonable timeout.
+                    .timeout = 0,
+
+                    // Use the default preset (6) for LZMA2.
+                    // To use a preset, filters must be set to NULL.
+                    .preset = LZMA_PRESET_DEFAULT,
+                    .filters = NULL,
+
+                    // Integrity checking.
+                    .check = XZ_CHECK,
+                };
+
+                // Detect how many threads the CPU supports.
+                mt.threads = lzma_cputhreads();
+
+                // If the number of CPU cores/threads cannot be detected,
+                // use one thread.
+                if (mt.threads == 0)
+                    mt.threads = 1;
+
+                // If the number of CPU cores/threads exceeds threads_max,
+                // limit the number of threads to keep memory usage lower.
+                const uint32_t threads_max = 2;
+                if (mt.threads > threads_max)
+                    mt.threads = threads_max;
+
+                if (mt.threads > 1)
+                    // Initialize the threaded encoder
+                    ret = lzma_stream_encoder_mt(stream, &mt);
+                else
+#endif
+                    // Initialize the single-threaded encoder
+                    ret = lzma_easy_encoder(stream,
+                                            CR_CW_XZ_COMPRESSION_LEVEL,
+                                            XZ_CHECK);
+
+            } else {
                 ret = lzma_auto_decoder(stream,
                                         XZ_MEMORY_USAGE_LIMIT,
                                         XZ_DECODER_FLAGS);
+            }
 
             if (ret != LZMA_OK) {
                 const char *err_msg;

diff --git a/src/misc.c b/src/misc.c
index 8644e311841e8ad75fe9bc5ba7d5ccb5411d60e4..99374809fd85fb03b91f96889a7b86c624eb5b7f 100644 (file)
--- a/src/misc.c
+++ b/src/misc.c
@@ -1502,6 +1502,9 @@ cr_version_string_with_features(void)
 #endif
 #ifdef ENABLE_LEGACY_WEAKDEPS
             "LegacyWeakdeps "
+#endif
+#ifdef ENABLE_THREADED_XZ_ENCODER
+            "ThreadedXzEncoder "
 #endif
             ")");
 }