# This is needed to know if assembler optimizations can be used.
AC_CANONICAL_HOST
+
echo
echo "Configure options:"
-# Enable/disable debugging code:
+
+#############
+# Debugging #
+#############
+
AC_MSG_CHECKING([if debugging code should be compiled])
AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug], [Enable debugging code.]),
[], enable_debug=no)
AC_MSG_RESULT([no])
fi
-# Enable/disable the encoder components:
+
+###########
+# Encoder #
+###########
+
AC_MSG_CHECKING([if encoder components should be built])
AC_ARG_ENABLE(encoder, AC_HELP_STRING([--disable-encoder],
[Do not build the encoder components.]),
fi
AM_CONDITIONAL(COND_MAIN_ENCODER, test "x$enable_encoder" = xyes)
-# Enable/disable the decoder components:
+
+###########
+# Decoder #
+###########
+
AC_MSG_CHECKING([if decoder components should be built])
AC_ARG_ENABLE(decoder, AC_HELP_STRING([--disable-decoder],
[Do not build the decoder components.]),
fi
AM_CONDITIONAL(COND_MAIN_DECODER, test "x$enable_decoder" = xyes)
-# Filters
-AC_MSG_CHECKING([which filters to build])
-AC_ARG_ENABLE(filters, AC_HELP_STRING([--enable-filters=LIST],
- [Comma-separated list of filters to build. Default=all.
- Filters used in encoding are needed also in decoding.
- Available filters: copy subblock x86 powerpc ia64
- arm armthumb sparc delta lzma]),
- [], [enable_filters=copy,subblock,x86,powerpc,ia64,arm,armthumb,sparc,delta,lzma])
-enable_filters=`echo "$enable_filters" | sed 's/,/ /g'`
-enable_filters_copy=no
-enable_filters_subblock=no
-enable_filters_x86=no
-enable_filters_powerpc=no
-enable_filters_ia64=no
-enable_filters_arm=no
-enable_filters_armthumb=no
-enable_filters_sparc=no
-enable_filters_delta=no
-enable_filters_lzma=no
-enable_simple_filters=no
-if test "x$enable_filters" = xno || test "x$enable_filters" = x; then
- AC_MSG_RESULT([])
- AC_MSG_ERROR([Please enable at least one filter.])
+
+###########
+# Filters #
+###########
+
+m4_define([SUPPORTED_FILTERS], [lzma,lzma2,subblock,delta,x86,powerpc,ia64,arm,armthumb,sparc])dnl
+m4_define([SIMPLE_FILTERS], [x86,powerpc,ia64,arm,armthumb,sparc])
+m4_define([LZ_FILTERS], [lzma,lzma2])
+
+m4_foreach([NAME], [SUPPORTED_FILTERS],
+[enable_filter_[]NAME=no
+enable_encoder_[]NAME=no
+enable_decoder_[]NAME=no
+])dnl
+
+AC_MSG_CHECKING([which encoders to build])
+AC_ARG_ENABLE([encoders], AC_HELP_STRING([--enable-encoders=LIST],
+ [Comma-separated list of encoders to build. Default=all.
+ Available encoders:]
+ m4_translit(m4_defn([SUPPORTED_FILTERS]), [,], [ ])),
+ [], [enable_encoders=SUPPORTED_FILTERS])
+enable_encoders=`echo "$enable_encoders" | sed 's/,/ /g'`
+if test "x$enable_encoders" = xno || test "x$enable_encoders" = x; then
+ AC_MSG_RESULT([(none)])
else
- for arg in $enable_filters
+ for arg in $enable_encoders
do
- case $arg in
- copy)
- enable_filters_copy=yes
- AC_DEFINE([HAVE_FILTER_COPY], 1,
- [Define to 1 if support for the
- Copy filter is enabled.])
- ;;
- subblock)
- enable_filters_subblock=yes
- AC_DEFINE([HAVE_FILTER_SUBBLOCK], 1,
- [Define to 1 if support for the
- Subblock filter is enabled.])
- ;;
- x86)
- enable_filters_x86=yes
- enable_simple_filters=yes
- AC_DEFINE([HAVE_FILTER_X86], 1,
- [Define to 1 if support for the
- x86 (BCJ) filter is enabled.])
- ;;
- powerpc)
- enable_filters_powerpc=yes
- enable_simple_filters=yes
- AC_DEFINE([HAVE_FILTER_POWERPC], 1,
- [Define to 1 if support for the
- PowerPC filter is enabled.])
- ;;
- ia64)
- enable_filters_ia64=yes
- enable_simple_filters=yes
- AC_DEFINE([HAVE_FILTER_IA64], 1,
- [Define to 1 if support for the
- IA64 filter is enabled.])
- ;;
- arm)
- enable_filters_arm=yes
- enable_simple_filters=yes
- AC_DEFINE([HAVE_FILTER_ARM], 1,
- [Define to 1 if support for the
- ARM filter is enabled.])
- ;;
- armthumb)
- enable_filters_armthumb=yes
- enable_simple_filters=yes
- AC_DEFINE([HAVE_FILTER_ARMTHUMB], 1,
- [Define to 1 if support for the
- ARMThumb filter is enabled.])
- ;;
- sparc)
- enable_filters_sparc=yes
- enable_simple_filters=yes
- AC_DEFINE([HAVE_FILTER_SPARC], 1,
- [Define to 1 if support for the
- SPARC filter is enabled.])
- ;;
- delta)
- enable_filters_delta=yes
- AC_DEFINE([HAVE_FILTER_DELTA], 1,
- [Define to 1 if support for the
- Delta filter is enabled.])
- ;;
- lzma)
- enable_filters_lzma=yes
- AC_DEFINE([HAVE_FILTER_LZMA], 1,
- [Define to 1 if support for the
- LZMA filter is enabled.])
+ case $arg in m4_foreach([NAME], [SUPPORTED_FILTERS], [
+ NAME)
+ enable_filter_[]NAME=yes
+ enable_encoder_[]NAME=yes
+ AC_DEFINE(HAVE_ENCODER_[]m4_toupper(NAME), [1],
+ [Define to 1 if] NAME [encoder is enabled.])
+ ;;])
+ *)
+ AC_MSG_RESULT([])
+ AC_MSG_ERROR([unknown filter: $arg])
;;
+ esac
+ done
+ AC_MSG_RESULT([$enable_encoders])
+fi
+
+AC_MSG_CHECKING([which decoders to build])
+AC_ARG_ENABLE([decoders], AC_HELP_STRING([--enable-decoders=LIST],
+ [Comma-separated list of decoders to build. Default=all.
+ Available decoders are the same as available encoders.]),
+ [], [enable_decoders=SUPPORTED_FILTERS])
+enable_decoders=`echo "$enable_decoders" | sed 's/,/ /g'`
+if test "x$enable_decoders" = xno || test "x$enable_decoders" = x; then
+ AC_MSG_RESULT([(none)])
+else
+ for arg in $enable_decoders
+ do
+ case $arg in m4_foreach([NAME], [SUPPORTED_FILTERS], [
+ NAME)
+ enable_filter_[]NAME=yes
+ enable_decoder_[]NAME=yes
+ AC_DEFINE(HAVE_DECODER_[]m4_toupper(NAME), [1],
+ [Define to 1 if] NAME [decoder is enabled.])
+ ;;])
*)
AC_MSG_RESULT([])
AC_MSG_ERROR([unknown filter: $arg])
;;
esac
done
- AC_MSG_RESULT([$enable_filters])
+
+ # LZMA2 requires that LZMA is enabled.
+ test "x$enable_encoder_lzma2" = xyes && enable_encoder_lzma=yes
+ test "x$enable_decoder_lzma2" = xyes && enable_decoder_lzma=yes
+
+ AC_MSG_RESULT([$enable_decoders])
fi
-if test "x$enable_simple_filters" = xyes ; then
- AC_DEFINE([HAVE_FILTER_SIMPLE], 1, [Define to 1 if support for any
- of the so called simple filters is enabled.])
+
+if test "x$enable_encoder_lzma2$enable_encoder_lzma" = xyesno \
+ || test "x$enable_decoder_lzma2$enable_decoder_lzma" = xyesno; then
+ AC_MSG_ERROR([LZMA2 requires that LZMA is also enabled.])
fi
-AM_CONDITIONAL(COND_FILTER_COPY, test "x$enable_filters_copy" = xyes)
-AM_CONDITIONAL(COND_FILTER_SUBBLOCK, test "x$enable_filters_subblock" = xyes)
-AM_CONDITIONAL(COND_FILTER_X86, test "x$enable_filters_x86" = xyes)
-AM_CONDITIONAL(COND_FILTER_POWERPC, test "x$enable_filters_powerpc" = xyes)
-AM_CONDITIONAL(COND_FILTER_IA64, test "x$enable_filters_ia64" = xyes)
-AM_CONDITIONAL(COND_FILTER_ARM, test "x$enable_filters_arm" = xyes)
-AM_CONDITIONAL(COND_FILTER_ARMTHUMB, test "x$enable_filters_armthumb" = xyes)
-AM_CONDITIONAL(COND_FILTER_SPARC, test "x$enable_filters_sparc" = xyes)
-AM_CONDITIONAL(COND_FILTER_DELTA, test "x$enable_filters_delta" = xyes)
-AM_CONDITIONAL(COND_FILTER_LZMA, test "x$enable_filters_lzma" = xyes)
-AM_CONDITIONAL(COND_MAIN_SIMPLE, test "x$enable_simple_filters" = xyes)
-
-# Which match finders should be enabled:
+
+m4_foreach([NAME], [SUPPORTED_FILTERS],
+[AM_CONDITIONAL(COND_FILTER_[]m4_toupper(NAME), test "x$enable_filter_[]NAME" = xyes)
+AM_CONDITIONAL(COND_ENCODER_[]m4_toupper(NAME), test "x$enable_encoder_[]NAME" = xyes)
+AM_CONDITIONAL(COND_DECODER_[]m4_toupper(NAME), test "x$enable_decoder_[]NAME" = xyes)
+])dnl
+
+# The so called "simple filters" share common code.
+enable_filter_simple=no
+enable_encoder_simple=no
+enable_decoder_simple=no
+m4_foreach([NAME], [SIMPLE_FILTERS],
+[test "x$enable_filter_[]NAME" = xyes && enable_filter_simple=yes
+test "x$enable_encoder_[]NAME" = xyes && enable_encoder_simple=yes
+test "x$enable_decoder_[]NAME" = xyes && enable_decoder_simple=yes
+])dnl
+AM_CONDITIONAL(COND_FILTER_SIMPLE, test "x$enable_filter_simple" = xyes)
+AM_CONDITIONAL(COND_ENCODER_SIMPLE, test "x$enable_encoder_simple" = xyes)
+AM_CONDITIONAL(COND_DECODER_SIMPLE, test "x$enable_decoder_simple" = xyes)
+
+# LZ-based filters share common code.
+enable_filter_lz=no
+enable_encoder_lz=no
+enable_decoder_lz=no
+m4_foreach([NAME], [LZ_FILTERS],
+[test "x$enable_filter_[]NAME" = xyes && enable_filter_lz=yes
+test "x$enable_encoder_[]NAME" = xyes && enable_encoder_lz=yes
+test "x$enable_decoder_[]NAME" = xyes && enable_decoder_lz=yes
+])dnl
+AM_CONDITIONAL(COND_FILTER_LZ, test "x$enable_filter_lz" = xyes)
+AM_CONDITIONAL(COND_ENCODER_LZ, test "x$enable_encoder_lz" = xyes)
+AM_CONDITIONAL(COND_DECODER_LZ, test "x$enable_decoder_lz" = xyes)
+
+
+#################
+# Match finders #
+#################
+
+m4_define([SUPPORTED_MATCH_FINDERS], [hc3,hc4,bt2,bt3,bt4])
+
+m4_foreach([NAME], [SUPPORTED_MATCH_FINDERS],
+[enable_match_finder_[]NAME=no
+])
+
AC_MSG_CHECKING([which match finders to build])
AC_ARG_ENABLE(match-finders, AC_HELP_STRING([--enable-match-finders=LIST],
[Comma-separated list of match finders to build. Default=all.
At least one match finder is required for encoding with
- the LZMA filter.
- Available match finders: hc3 hc4 bt2 bt3 bt4]), [],
- [enable_match_finders=hc3,hc4,bt2,bt3,bt4])
+ the LZMA filter. Available match finders:]
+ m4_translit(m4_defn([SUPPORTED_MATCH_FINDERS]), [,], [ ])), [],
+ [enable_match_finders=SUPPORTED_MATCH_FINDERS])
enable_match_finders=`echo "$enable_match_finders" | sed 's/,/ /g'`
-enable_match_finders_hc3=no
-enable_match_finders_hc4=no
-enable_match_finders_bt2=no
-enable_match_finders_bt3=no
-enable_match_finders_bt4=no
-if test "x$enable_encoder" = xyes && test "x$enable_filters_lzma" = xyes ; then
+if test "x$enable_encoder" = xyes && test "x$enable_encoder_lz" = xyes ; then
for arg in $enable_match_finders
do
- case $arg in
- hc3) enable_match_finders_hc3=yes ;;
- hc4) enable_match_finders_hc4=yes ;;
- bt2) enable_match_finders_bt2=yes ;;
- bt3) enable_match_finders_bt3=yes ;;
- bt4) enable_match_finders_bt4=yes ;;
+ case $arg in m4_foreach([NAME], [SUPPORTED_MATCH_FINDERS], [
+ NAME)
+ enable_match_finder_[]NAME=yes
+ AC_DEFINE(HAVE_MF_[]m4_toupper(NAME), [1],
+ [Define to 1 to enable] NAME [match finder.])
+ ;;])
*)
AC_MSG_RESULT([])
AC_MSG_ERROR([unknown match finder: $arg])
done
AC_MSG_RESULT([$enable_match_finders])
else
- AC_MSG_RESULT([(none because not building the LZMA encoder)])
+ AC_MSG_RESULT([(none because not building any LZ-based encoder)])
fi
-AM_CONDITIONAL(COND_MF_HC3, test "x$enable_match_finders_hc3" = xyes)
-AM_CONDITIONAL(COND_MF_HC4, test "x$enable_match_finders_hc4" = xyes)
-AM_CONDITIONAL(COND_MF_BT2, test "x$enable_match_finders_bt2" = xyes)
-AM_CONDITIONAL(COND_MF_BT3, test "x$enable_match_finders_bt3" = xyes)
-AM_CONDITIONAL(COND_MF_BT4, test "x$enable_match_finders_bt4" = xyes)
-# Which integrity checks to build
+
+####################
+# Integrity checks #
+####################
+
+m4_define([SUPPORTED_CHECKS], [crc32,crc64,sha256])
+
+m4_foreach([NAME], [SUPPORTED_FILTERS],
+[enable_check_[]NAME=no
+])dnl
+
AC_MSG_CHECKING([which integrity checks to build])
AC_ARG_ENABLE(checks, AC_HELP_STRING([--enable-checks=LIST],
[Comma-separated list of integrity checks to build.
- Default=all. Available integrity checks: crc32 crc64 sha256]),
- [], [enable_checks=crc32,crc64,sha256])
+ Default=all. Available integrity checks:]
+ m4_translit(m4_defn([SUPPORTED_CHECKS]), [,], [ ])),
+ [], [enable_checks=SUPPORTED_CHECKS])
enable_checks=`echo "$enable_checks" | sed 's/,/ /g'`
-enable_checks_crc32=no
-enable_checks_crc64=no
-enable_checks_sha256=no
if test "x$enable_checks" = xno || test "x$enable_checks" = x; then
AC_MSG_RESULT([(none)])
else
for arg in $enable_checks
do
- case $arg in
- crc32)
- enable_checks_crc32=yes
- AC_DEFINE([HAVE_CHECK_CRC32], 1,
- [Define to 1 if CRC32 support
- is enabled.])
- ;;
- crc64)
- enable_checks_crc64=yes
- AC_DEFINE([HAVE_CHECK_CRC64], 1,
- [Define to 1 if CRC64 support
- is enabled.])
- ;;
- sha256)
- enable_checks_sha256=yes
- AC_DEFINE([HAVE_CHECK_SHA256], 1,
- [Define to 1 if SHA256 support
- is enabled.])
- ;;
+ case $arg in m4_foreach([NAME], [SUPPORTED_CHECKS], [
+ NAME)
+ enable_check_[]NAME=yes
+ AC_DEFINE(HAVE_CHECK_[]m4_toupper(NAME), [1],
+ [Define to 1 if] NAME
+ [integrity check is enabled.])
+ ;;])
*)
AC_MSG_RESULT([])
AC_MSG_ERROR([unknown integrity check: $arg])
if test "x$enable_checks_crc32" = xno ; then
AC_MSG_ERROR([For now, the CRC32 check must always be enabled.])
fi
-AM_CONDITIONAL(COND_CHECK_CRC32, test "x$enable_checks_crc32" = xyes)
-AM_CONDITIONAL(COND_CHECK_CRC64, test "x$enable_checks_crc64" = xyes)
-AM_CONDITIONAL(COND_CHECK_SHA256, test "x$enable_checks_sha256" = xyes)
-# Assembler optimizations
+m4_foreach([NAME], [SUPPORTED_CHECKS],
+[AM_CONDITIONAL(COND_CHECK_[]m4_toupper(NAME), test "x$enable_check_[]NAME" = xyes)
+])dnl
+
+
+###########################
+# Assembler optimizations #
+###########################
+
AC_MSG_CHECKING([if assembler optimizations should be used])
AC_ARG_ENABLE(assembler, AC_HELP_STRING([--disable-assembler],
[Do not use assembler optimizations even if such exist
;;
*)
AC_MSG_RESULT([])
- AC_MSG_ERROR([--enable-assembler accepts only \`yes', \`no', or \`x86'.])
+ AC_MSG_ERROR([--enable-assembler accepts only \`yes', \`no', \`x86', or \`x86_64'.])
;;
esac
AC_MSG_RESULT([$enable_assembler])
AM_CONDITIONAL(COND_ASM_X86, test "x$enable_assembler" = xx86)
+AM_CONDITIONAL(COND_ASM_X86_64, test "x$enable_assembler" = xx86_64)
+
+
+################################
+# Fast unaligned memory access #
+################################
-# Fast unaligned memory access
AC_MSG_CHECKING([if unaligned memory access should be used])
AC_ARG_ENABLE(unaligned-access, AC_HELP_STRING([--enable-unaligned-access],
[Enable if the system supports *fast* unaligned memory access
AC_MSG_RESULT([no])
fi
-# Size optimization
+
+#####################
+# Size optimization #
+#####################
+
AC_MSG_CHECKING([if small size is preferred over speed])
AC_ARG_ENABLE(small, AC_HELP_STRING([--enable-small],
- [Omit precomputed tables to make liblzma a few kilobytes
- smaller. This will increase startup time of applications
- slightly, because the tables need to be computed first.]),
+ [Make liblzma smaller and a little slower.
+ This is disabled by default to optimize for speed.]),
[], [enable_small=no])
if test "x$enable_small" = xyes; then
AC_DEFINE([HAVE_SMALL], 1, [Define to 1 if optimizing for size.])
AC_MSG_RESULT([$enable_small])
AM_CONDITIONAL(COND_SMALL, test "x$enable_small" = xyes)
+
###############################################################################
# Checks for programs.
###############################################################################
AC_TYPE_UINT64_T
AC_TYPE_UINTPTR_T
-AC_CHECK_SIZEOF([unsigned long])
AC_CHECK_SIZEOF([size_t])
# The command line tool can copy high resolution timestamps if such
src/liblzma/api/Makefile
src/liblzma/common/Makefile
src/liblzma/check/Makefile
+ src/liblzma/rangecoder/Makefile
src/liblzma/lz/Makefile
src/liblzma/lzma/Makefile
- src/liblzma/simple/Makefile
src/liblzma/subblock/Makefile
- src/liblzma/rangecoder/Makefile
+ src/liblzma/delta/Makefile
+ src/liblzma/simple/Makefile
src/lzma/Makefile
src/lzmadec/Makefile
src/scripts/Makefile
repeat \
sync_flush \
full_flush \
- memusage
+ memusage \
+ crc32 \
+ known_sizes \
+ hex2bin
AM_CPPFLAGS = \
-I@top_srcdir@/src/common \
///////////////////////////////////////////////////////////////////////////////
//
-/// \file lz_encoder_private.h
-/// \brief Private definitions for LZ encoder
+/// \file crc32.c
+/// \brief Primitive CRC32 calculation tool
//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
+// Copyright (C) 2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
//
///////////////////////////////////////////////////////////////////////////////
-#ifndef LZMA_LZ_ENCODER_PRIVATE_H
-#define LZMA_LZ_ENCODER_PRIVATE_H
+#include "sysdefs.h"
+#include <stdio.h>
-#include "lz_encoder.h"
-/// Value used to indicate unused slot
-#define EMPTY_HASH_VALUE 0
+int
+main(void)
+{
+ uint32_t crc = 0;
-/// When the dictionary and hash variables need to be adjusted to prevent
-/// integer overflows. Since we use uint32_t to store the offsets, half
-/// of it is the biggest safe limit.
-#define MAX_VAL_FOR_NORMALIZE (UINT32_MAX / 2)
+ do {
+ uint8_t buf[BUFSIZ];
+ const size_t size = fread(buf, 1, sizeof(buf), stdin);
+ crc = lzma_crc32(buf, size, crc);
+ } while (!ferror(stdin) && !feof(stdin));
+ //printf("%08" PRIX32 "\n", crc);
-struct lzma_coder_s {
- lzma_next_coder next;
- lzma_lz_encoder lz;
-};
+ // I want it little endian so it's easy to work with hex editor.
+ printf("%02" PRIX32 " ", crc & 0xFF);
+ printf("%02" PRIX32 " ", (crc >> 8) & 0xFF);
+ printf("%02" PRIX32 " ", (crc >> 16) & 0xFF);
+ printf("%02" PRIX32 " ", crc >> 24);
+ printf("\n");
-#endif
+ return 0;
+}
file_in = argc > 1 ? fopen(argv[1], "rb") : stdin;
+
// Config
- lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
- filters[0].id = LZMA_FILTER_SUBBLOCK;
- filters[0].options = NULL;
+ lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
+ filters[0].id = LZMA_FILTER_LZMA2;
+ filters[0].options = (void *)&lzma_preset_lzma[0];
filters[1].id = LZMA_VLI_VALUE_UNKNOWN;
// Init
- if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC32) != LZMA_OK) {
+ if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_SHA256) != LZMA_OK) {
fprintf(stderr, "init failed\n");
exit(1);
}
+// if (lzma_easy_encoder(&strm, 1)) {
+// fprintf(stderr, "init failed\n");
+// exit(1);
+// }
+
// Encoding
encode(0, LZMA_FULL_FLUSH);
encode(6, LZMA_FULL_FLUSH);
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file hex2bin.c
+/// \brief Converts hexadecimal input strings to binary
+//
+// This code has been put into the public domain.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "sysdefs.h"
+#include <stdio.h>
+#include <ctype.h>
+
+
+static int
+getbin(int x)
+{
+ if (x >= '0' && x <= '9')
+ return x - '0';
+
+ if (x >= 'A' && x <= 'F')
+ return x - 'A' + 10;
+
+ return x - 'a' + 10;
+}
+
+
+int
+main(void)
+{
+ while (true) {
+ int byte = getchar();
+ if (byte == EOF)
+ return 0;
+ if (!isxdigit(byte))
+ continue;
+
+ const int digit = getchar();
+ if (digit == EOF || !isxdigit(digit)) {
+ fprintf(stderr, "Invalid input\n");
+ return 1;
+ }
+
+ byte = (getbin(byte) << 4) | getbin(digit);
+ if (putchar(byte) == EOF) {
+ perror(NULL);
+ return 1;
+ }
+ }
+}
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file known_sizes.c
+/// \brief Encodes .lzma Stream with sizes known in Block Header
+///
+/// The input file is encoded in RAM, and the known Compressed Size
+/// and/or Uncompressed Size values are stored in the Block Header.
+/// As of writing there's no such Stream encoder in liblzma.
+//
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "sysdefs.h"
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/unistd.h>
+#include <stdio.h>
+
+
+// Support file sizes up to 1 MiB. We use this for output space too, so files
+// close to 1 MiB had better compress at least a little or we have a buffer
+// overflow.
+#define BUFFER_SIZE (1U << 20)
+
+
+int
+main(void)
+{
+ // Allocate the buffers.
+ uint8_t *in = malloc(BUFFER_SIZE);
+ uint8_t *out = malloc(BUFFER_SIZE);
+ if (in == NULL || out == NULL)
+ return 1;
+
+ // Fill the input buffer.
+ const size_t in_size = fread(in, 1, BUFFER_SIZE, stdin);
+
+ // Filter setup
+ lzma_filter filters[] = {
+ {
+ .id = LZMA_FILTER_LZMA2,
+ .options = (void *)(&lzma_preset_lzma[0])
+ },
+ {
+ .id = LZMA_VLI_VALUE_UNKNOWN
+ }
+ };
+
+ lzma_block block = {
+ .check = LZMA_CHECK_CRC32,
+ .compressed_size = BUFFER_SIZE, // Worst case reserve
+ .uncompressed_size = in_size,
+ .filters = filters,
+ };
+
+ // FIXME Insane paranoia in liblzma.
+ if (lzma_block_header_size(&block) != LZMA_OK)
+ return 1;
+
+ // We don't actually know the compressed size, so don't tell it to
+ // Block encoder.
+ block.compressed_size = LZMA_VLI_VALUE_UNKNOWN;
+
+ lzma_stream strm = LZMA_STREAM_INIT;
+ if (lzma_block_encoder(&strm, &block) != LZMA_OK)
+ return 1;
+
+ // Reserve space for Stream Header and Block Header.
+ size_t out_size = LZMA_STREAM_HEADER_SIZE + block.header_size;
+
+ strm.next_in = in;
+ strm.avail_in = in_size;
+ strm.next_out = out + out_size;
+ strm.avail_out = BUFFER_SIZE - out_size;
+
+ if (lzma_code(&strm, LZMA_FINISH) != LZMA_STREAM_END)
+ return 1;
+
+ out_size += strm.total_out;
+
+ if (lzma_block_header_encode(&block, out + LZMA_STREAM_HEADER_SIZE)
+ != LZMA_OK)
+ return 1;
+
+ lzma_index *idx = lzma_index_init(NULL, NULL);
+ if (idx == NULL)
+ return 1;
+
+ if (lzma_index_append(idx, NULL, block.header_size + strm.total_out,
+ strm.total_in) != LZMA_OK)
+ return 1;
+
+ if (lzma_index_encoder(&strm, idx) != LZMA_OK)
+ return 1;
+
+ if (lzma_code(&strm, LZMA_RUN) != LZMA_STREAM_END)
+ return 1;
+
+ out_size += strm.total_out;
+
+ lzma_end(&strm);
+
+ lzma_index_end(idx, NULL);
+
+ // Encode the Stream Header and Stream Footer. backwards_size is
+ // needed only for the Stream Footer.
+ lzma_stream_flags sf = {
+ .backward_size = strm.total_out,
+ .check = block.check,
+ };
+
+ if (lzma_stream_header_encode(&sf, out) != LZMA_OK)
+ return 1;
+
+ if (lzma_stream_footer_encode(&sf, out + out_size) != LZMA_OK)
+ return 1;
+
+ out_size += LZMA_STREAM_HEADER_SIZE;
+
+ // Write out the file.
+ fwrite(out, 1, out_size, stdout);
+
+ return 0;
+}
int
main(void)
{
+ lzma_init();
lzma_options_lzma lzma = {
.dictionary_size = (1 << 27) + (1 << 26),
.pos_bits = 2,
.preset_dictionary = NULL,
.preset_dictionary_size = 0,
- .mode = LZMA_MODE_BEST,
+ .mode = LZMA_MODE_NORMAL,
.fast_bytes = 48,
.match_finder = LZMA_MF_BT4,
.match_finder_cycles = 0,
{ UINT64_MAX, NULL }
};
*/
- lzma_options_filter filters[] = {
+ lzma_filter filters[] = {
{ LZMA_FILTER_LZMA, &lzma },
{ UINT64_MAX, NULL }
};
- printf("%u MiB\n", lzma_memory_usage(filters, true));
+ printf("Encoder: %10" PRIu64 " B\n", lzma_memusage_encoder(filters));
+ printf("Decoder: %10" PRIu64 " B\n", lzma_memusage_decoder(filters));
return 0;
}
.literal_pos_bits = LZMA_LITERAL_POS_BITS_DEFAULT,
.pos_bits = LZMA_POS_BITS_DEFAULT,
.preset_dictionary = NULL,
- .mode = LZMA_MODE_BEST,
+ .persistent = true,
+ .mode = LZMA_MODE_NORMAL,
.fast_bytes = 32,
- .match_finder = LZMA_MF_BT3,
+ .match_finder = LZMA_MF_HC3,
.match_finder_cycles = 0,
};
opt_subblock.subfilter_options.id = LZMA_FILTER_DELTA;
opt_subblock.subfilter_options.options = &opt_delta;
- lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
- filters[0].id = LZMA_FILTER_LZMA;
+ lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
+ filters[0].id = LZMA_FILTER_LZMA2;
filters[0].options = &opt_lzma;
filters[1].id = LZMA_VLI_VALUE_UNKNOWN;
// Init
- if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_NONE) != LZMA_OK) {
+ if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC32) != LZMA_OK) {
fprintf(stderr, "init failed\n");
exit(1);
}
// Encoding
+/*
encode(0, LZMA_SYNC_FLUSH);
encode(6, LZMA_SYNC_FLUSH);
encode(0, LZMA_SYNC_FLUSH);
- encode(6, LZMA_SYNC_FLUSH);
+ encode(7, LZMA_SYNC_FLUSH);
encode(0, LZMA_SYNC_FLUSH);
encode(0, LZMA_FINISH);
+*/
+ encode(53, LZMA_SYNC_FLUSH);
+// opt_lzma.literal_context_bits = 2;
+// opt_lzma.literal_pos_bits = 1;
+// opt_lzma.pos_bits = 0;
+ encode(404, LZMA_FINISH);
// Clean up
lzma_end(&strm);
#define LZMA_INTEGER_H
// I'm aware of AC_CHECK_ALIGNED_ACCESS_REQUIRED from Autoconf archive, but
-// it's not useful for us. We don't care if unaligned access is supported,
+// it's not useful here. We don't care if unaligned access is supported,
// we care if it is fast. Some systems can emulate unaligned access in
// software, which is horribly slow; we want to use byte-by-byte access on
// such systems but the Autoconf test would detect such a system as
// that also allow unaligned access. Inline assembler could be OK for that.
#ifdef WORDS_BIGENDIAN
# include "bswap.h"
-# define integer_convert_16(n) bswap_16(n)
-# define integer_convert_32(n) bswap_32(n)
-# define integer_convert_64(n) bswap_64(n)
+# define integer_le_16(n) bswap_16(n)
+# define integer_le_32(n) bswap_32(n)
+# define integer_le_64(n) bswap_64(n)
#else
-# define integer_convert_16(n) (n)
-# define integer_convert_32(n) (n)
-# define integer_convert_64(n) (n)
+# define integer_le_16(n) (n)
+# define integer_le_32(n) (n)
+# define integer_le_64(n) (n)
#endif
integer_read_16(const uint8_t buf[static 2])
{
uint16_t ret = *(const uint16_t *)(buf);
- return integer_convert_16(ret);
+ return integer_le_16(ret);
}
integer_read_32(const uint8_t buf[static 4])
{
uint32_t ret = *(const uint32_t *)(buf);
- return integer_convert_32(ret);
+ return integer_le_32(ret);
}
integer_read_64(const uint8_t buf[static 8])
{
uint64_t ret = *(const uint64_t *)(buf);
- return integer_convert_64(ret);
+ return integer_le_64(ret);
}
*/
static inline void
integer_write_16(uint8_t buf[static 2], uint16_t num)
{
- *(uint16_t *)(buf) = integer_convert_16(num);
+ *(uint16_t *)(buf) = integer_le_16(num);
}
static inline void
integer_write_32(uint8_t buf[static 4], uint32_t num)
{
- *(uint32_t *)(buf) = integer_convert_32(num);
+ *(uint32_t *)(buf) = integer_le_32(num);
}
static inline void
integer_write_64(uint8_t buf[static 8], uint64_t num)
{
- *(uint64_t *)(buf) = integer_convert_64(num);
+ *(uint64_t *)(buf) = integer_le_64(num);
}
*/
# include <config.h>
#endif
-#include <sys/types.h>
+// size_t and NULL
+#include <stddef.h>
#ifdef HAVE_INTTYPES_H
# include <inttypes.h>
#endif
+// C99 says that inttypes.h always includes stdint.h, but some systems
+// don't do that, and require including stdint.h separately.
+#ifdef HAVE_STDINT_H
+# include <stdint.h>
+#endif
+
+// Some pre-C99 systems have SIZE_MAX in limits.h instead of stdint.h. The
+// limits are also used to figure out some macros missing from pre-C99 systems.
#ifdef HAVE_LIMITS_H
# include <limits.h>
#endif
// Be more compatible with systems that have non-conforming inttypes.h.
// We assume that int is 32-bit and that long is either 32-bit or 64-bit.
// Full Autoconf test could be more correct, but this should work well enough.
+// Note that this duplicates some code from lzma.h, but this is better since
+// we can work without inttypes.h thanks to Autoconf tests.
#ifndef UINT32_C
+# if UINT_MAX != 4294967295U
+# error UINT32_C is not defined and unsiged int is not 32-bit.
+# endif
# define UINT32_C(n) n ## U
#endif
#ifndef UINT32_MAX
#ifndef PRIX32
# define PRIX32 "X"
#endif
-#if SIZEOF_UNSIGNED_LONG == 4
+
+#if ULONG_MAX == 4294967295UL
# ifndef UINT64_C
# define UINT64_C(n) n ## ULL
# endif
#ifndef UINT64_MAX
# define UINT64_MAX UINT64_C(18446744073709551615)
#endif
+
+// The code currently assumes that size_t is either 32-bit or 64-bit.
#ifndef SIZE_MAX
# if SIZEOF_SIZE_T == 4
# define SIZE_MAX UINT32_MAX
-# else
+# elif SIZEOF_SIZE_T == 8
# define SIZE_MAX UINT64_MAX
+# else
+# error sizeof(size_t) is not 32-bit or 64-bit
# endif
#endif
+#if SIZE_MAX != UINT32_MAX && SIZE_MAX != UINT64_MAX
+# error sizeof(size_t) is not 32-bit or 64-bit
+#endif
#include <stdlib.h>
+// Pre-C99 systems lack stdbool.h. All the code in LZMA Utils must be written
+// so that it works with fake bool type, for example:
+//
+// bool foo = (flags & 0x100) != 0;
+// bool bar = !!(flags & 0x100);
+//
+// This works with the real C99 bool but breaks with fake bool:
+//
+// bool baz = (flags & 0x100);
+//
#ifdef HAVE_STDBOOL_H
# include <stdbool.h>
#else
# ifdef NDEBUG
# define assert(x)
# else
- // TODO: Pretty bad assert() macro.
+ // TODO: Pretty bad assert macro.
# define assert(x) (!(x) && abort())
# endif
#endif
+// string.h should be enough but let's include strings.h and memory.h too if
+// they exists, since that shouldn't do any harm, but may improve portability.
#ifdef HAVE_STRING_H
# include <string.h>
#endif
common/libcommon.la \
check/libcheck.la
+if COND_FILTER_LZ
+SUBDIRS += lz
+liblzma_la_LIBADD += lz/liblz.la
+endif
+
if COND_FILTER_LZMA
-SUBDIRS += lz lzma rangecoder
+SUBDIRS += lzma rangecoder
liblzma_la_LIBADD += \
- lz/liblz.la \
- lzma/liblzma4.la \
+ lzma/liblzma2.la \
rangecoder/librangecoder.la
endif
liblzma_la_LIBADD += subblock/libsubblock.la
endif
-if COND_MAIN_SIMPLE
+if COND_FILTER_DELTA
+SUBDIRS += delta
+liblzma_la_LIBADD += delta/libdelta.la
+endif
+
+if COND_FILTER_SIMPLE
SUBDIRS += simple
liblzma_la_LIBADD += simple/libsimple.la
endif
nobase_include_HEADERS = \
lzma.h \
lzma/alignment.h \
- lzma/alone.h \
- lzma/auto.h \
lzma/base.h \
lzma/block.h \
lzma/check.h \
+ lzma/container.h \
lzma/delta.h \
- lzma/easy.h \
lzma/filter.h \
lzma/index.h \
lzma/index_hash.h \
lzma/init.h \
lzma/lzma.h \
lzma/memlimit.h \
- lzma/raw.h \
lzma/simple.h \
- lzma/stream.h \
lzma/stream_flags.h \
lzma/subblock.h \
lzma/version.h \
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
+ */
+
+#ifndef LZMA_H
+#define LZMA_H
+
+/*****************************
+ * Required standard headers *
+ *****************************/
+
+/**
+ * liblzma API headers need some standard types and macros. To allow
+ * including lzma.h without requiring the application to include other
+ * headers first, lzma.h includes the required standard headers unless
+ * they already seem to be included.
*
- * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Here's what types and macros are needed and from which headers:
+ * - stddef.h: size_t, NULL
+ * - stdint.h: uint8_t, uint32_t, uint64_t, UINT32_C(n), uint64_C(n),
+ * UINT32_MAX, UINT64_MAX
*
- * Before #including this file, you must make the following types available:
- * - size_t
- * - uint8_t
- * - int32_t
- * - uint32_t
- * - int64_t
- * - uint64_t
+ * However, inttypes.h is a little more portable than stdint.h, although
+ * inttypes.h declares some unneeded things compared to plain stdint.h.
*
- * Before #including this file, you must make the following macros available:
- * - UINT32_C(n)
- * - UINT64_C(n)
- * - UINT32_MAX
- * - UINT64_MAX
+ * The hacks below aren't perfect, specifically they assume that inttypes.h
+ * exists and that it typedefs at least uint8_t, uint32_t, and uint64_t,
+ * and that unsigned int is 32-bit. If your application already takes care
+ * of setting up all the types properly (for example by using gnulib's
+ * stdint.h or inttypes.h), feel free to define LZMA_MANUAL_HEADERS before
+ * including lzma.h.
*
- * Easiest way to achieve the above is to #include sys/types.h and inttypes.h
- * before #including lzma.h. However, some pre-C99 libc headers don't provide
- * all the required types in inttypes.h (that file may even be missing).
- * Portable applications need to provide these types themselves. This way
- * liblzma API can use the standard types instead of defining its own
- * (e.g. lzma_uint32).
+ * Some could argue that liblzma API should provide all the required types,
+ * for example lzma_uint64, LZMA_UINT64_C(n), and LZMA_UINT64_MAX. This was
+ * seen unnecessary mess, since most systems already provide all the necessary
+ * types and macros in the standard headers.
*
- * Note that the API still has lzma_bool, because using stdbool.h would
+ * Note that liblzma API still has lzma_bool, because using stdbool.h would
* break C89 and C++ programs on many systems.
*/
-#ifndef LZMA_H
-#define LZMA_H
+/* stddef.h even in C++ so that we get size_t in global namespace. */
+#include <stddef.h>
+
+#if !defined(UINT32_C) || !defined(UINT64_C) \
+ || !defined(UINT32_MAX) || !defined(UINT64_MAX)
+# ifdef __cplusplus
+ /*
+ * C99 sections 7.18.2 and 7.18.4 specify that in C++
+ * implementations define the limit and constant macros only
+ * if specifically requested. Note that if you want the
+ * format macros too, you need to define __STDC_FORMAT_MACROS
+ * before including lzma.h, since re-including inttypes.h
+ * with __STDC_FORMAT_MACROS defined doesn't necessarily work.
+ */
+# ifndef __STDC_LIMIT_MACROS
+# define __STDC_LIMIT_MACROS 1
+# endif
+# ifndef __STDC_CONSTANT_MACROS
+# define __STDC_CONSTANT_MACROS 1
+# endif
+# endif
+
+# include <inttypes.h>
+
+ /*
+ * Some old systems have only the typedefs in inttypes.h, and lack
+ * all the macros. For those systems, we need a few more hacks.
+ * We assume that unsigned int is 32-bit and unsigned long is either
+ * 32-bit or 64-bit. If these hacks aren't enough, the application
+ * has to use setup the types manually before including lzma.h.
+ */
+# ifndef UINT32_C
+# define UINT32_C(n) n # U
+# endif
+
+# ifndef UINT64_C
+ /* Get ULONG_MAX. */
+# ifndef __cplusplus
+# include <limits.h>
+# else
+# include <climits>
+# endif
+# if ULONG_MAX == 4294967295UL
+# define UINT64_C(n) n ## ULL
+# else
+# define UINT64_C(n) n ## UL
+# endif
+# endif
+
+# ifndef UINT32_MAX
+# define UINT32_MAX (UINT32_C(4294967295))
+# endif
+
+# ifndef UINT64_MAX
+# define UINT64_MAX (UINT64_C(18446744073709551615))
+# endif
+#endif
+
/******************
* GCC extensions *
* break anything if these are sometimes enabled and sometimes not, only
* affects warnings and optimizations.
*/
-#if defined(__GNUC__) && __GNUC__ >= 3
+#if __GNUC__ >= 3
# ifndef lzma_attribute
# define lzma_attribute(attr) __attribute__(attr)
# endif
+
# ifndef lzma_restrict
# define lzma_restrict __restrict__
# endif
+
+ /* warn_unused_result was added in GCC 3.4. */
+# ifndef lzma_attr_warn_unused_result
+# if __GNUC__ == 3 && __GNUC_MINOR__ < 4
+# define lzma_attr_warn_unused_result
+# endif
+# endif
+
#else
# ifndef lzma_attribute
# define lzma_attribute(attr)
# endif
+
# ifndef lzma_restrict
-# define lzma_restrict
+# if __STDC_VERSION__ >= 199901L
+# define lzma_restrict restrict
+# else
+# define lzma_restrict
+# endif
# endif
+
+# define lzma_attr_warn_unused_result
+#endif
+
+
+#ifndef lzma_attr_pure
+# define lzma_attr_pure lzma_attribute((__pure__))
+#endif
+
+#ifndef lzma_attr_const
+# define lzma_attr_const lzma_attribute((__const__))
+#endif
+
+#ifndef lzma_attr_warn_unused_result
+# define lzma_attr_warn_unused_result \
+ lzma_attribute((__warn_unused_result__))
#endif
#define LZMA_H_INTERNAL 1
/* Basic features */
+#include "lzma/version.h"
#include "lzma/init.h"
#include "lzma/base.h"
#include "lzma/vli.h"
-#include "lzma/filter.h"
#include "lzma/check.h"
/* Filters */
+#include "lzma/filter.h"
#include "lzma/subblock.h"
#include "lzma/simple.h"
#include "lzma/delta.h"
#include "lzma/lzma.h"
/* Container formats */
-#include "lzma/block.h"
-#include "lzma/stream.h"
-#include "lzma/alone.h"
-#include "lzma/raw.h"
-#include "lzma/auto.h"
-#include "lzma/easy.h"
+#include "lzma/container.h"
/* Advanced features */
+#include "lzma/alignment.h" /* FIXME */
+#include "lzma/block.h"
#include "lzma/index.h"
#include "lzma/index_hash.h"
-#include "lzma/alignment.h"
#include "lzma/stream_flags.h"
#include "lzma/memlimit.h"
-/* Version number */
-#include "lzma/version.h"
-
/*
* All subheaders included. Undefine LZMA_H_INTERNAL to prevent applications
* re-including the subheaders.
* FIXME desc
*/
extern uint32_t lzma_alignment_input(
- const lzma_options_filter *filters, uint32_t guess);
+ const lzma_filter *filters, uint32_t guess);
/**
* Knowing the alignment of the output data is useful e.g. in the Block
* encoder which tries to align the Compressed Data field optimally.
*
- * \param filters Pointer to lzma_options_filter array, whose last
+ * \param filters Pointer to lzma_filter array, whose last
* member must have .id = LZMA_VLI_VALUE_UNKNOWN.
* \param guess The value to return if the alignment of the output
* is the same as the alignment of the input data.
* options), UINT32_MAX is returned.
*/
extern uint32_t lzma_alignment_output(
- const lzma_options_filter *filters, uint32_t guess);
+ const lzma_filter *filters, uint32_t guess);
+++ /dev/null
-/**
- * \file lzma/alone.h
- * \brief Handling of the legacy LZMA_Alone format
- *
- * \author Copyright (C) 1999-2006 Igor Pavlov
- * \author Copyright (C) 2007 Lasse Collin
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- */
-
-#ifndef LZMA_H_INTERNAL
-# error Never include this file directly. Use <lzma.h> instead.
-#endif
-
-
-/**
- * \brief Initializes LZMA_Alone encoder
- *
- * LZMA_Alone files have the suffix .lzma like the .lzma Stream files.
- * LZMA_Alone format supports only one filter, the LZMA filter. There is
- * no support for integrity checks like CRC32.
- *
- * Use this format if and only if you need to create files readable by
- * legacy LZMA tools.
- *
- * LZMA_Alone encoder doesn't support LZMA_SYNC_FLUSH or LZMA_FULL_FLUSH.
- *
- * \return - LZMA_OK
- * - LZMA_MEM_ERROR
- * - LZMA_PROG_ERROR
- */
-extern lzma_ret lzma_alone_encoder(
- lzma_stream *strm, const lzma_options_lzma *options);
-
-
-/**
- * \brief Initializes decoder for LZMA_Alone file
- *
- * The LZMA_Alone decoder supports LZMA_SYNC_FLUSH.
- *
- * \return - LZMA_OK
- * - LZMA_MEM_ERROR
- */
-extern lzma_ret lzma_alone_decoder(lzma_stream *strm);
+++ /dev/null
-/**
- * \file lzma/auto.h
- * \brief Decoder with automatic file format detection
- *
- * \author Copyright (C) 1999-2006 Igor Pavlov
- * \author Copyright (C) 2007 Lasse Collin
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- */
-
-#ifndef LZMA_H_INTERNAL
-# error Never include this file directly. Use <lzma.h> instead.
-#endif
-
-
-/**
- * \brief Decode .lzma Streams and LZMA_Alone files with autodetection
- *
- * Autodetects between the .lzma Stream and LZMA_Alone formats, and
- * calls lzma_stream_decoder_init() or lzma_alone_decoder_init() once
- * the type of the file has been detected.
- *
- * \param strm Pointer to propertily prepared lzma_stream
- *
- * \return - LZMA_OK: Initialization was successful.
- * - LZMA_MEM_ERROR: Cannot allocate memory.
- */
-extern lzma_ret lzma_auto_decoder(lzma_stream *strm);
* \brief Unknown file format
*/
- LZMA_MEMLIMIT_ERROR = -9
+ LZMA_MEMLIMIT_ERROR = -9,
/**
* \brief Memory usage limit was reached
*
* the memory usage limit has to be increased. See functions
* lzma_memlimit_get() and lzma_memlimit_set().
*/
+
+ LZMA_NO_CHECK = -10,
+ LZMA_SEE_CHECK = -11
} lzma_ret;
/**
* \brief Pointer to custom memory allocation function
*
- * Set this to point to your custom memory allocation function.
- * It can be useful for example if you want to limit how much
- * memory liblzma is allowed to use: for this, you may use
- * a pointer to lzma_memory_alloc().
- *
* If you don't want a custom allocator, but still want
* custom free(), set this to NULL and liblzma will use
* the standard malloc().
* size nmemb * size, or NULL if allocation fails
* for some reason. When allocation fails, functions
* of liblzma return LZMA_MEM_ERROR.
+ *
+ * For performance reasons, the allocator should not waste time
+ * zeroing the allocated buffers. This is not only about speed, but
+ * also memory usage, since the operating system kernel doesn't
+ * necessarily allocate the requested memory until it is actually
+ * used. With small input files liblzma may actually need only a
+ * fraction of the memory that it requested for allocation.
*/
void *(*alloc)(void *opaque, size_t nmemb, size_t size);
/**
* \brief Pointer to custom memory freeing function
*
- * Set this to point to your custom memory freeing function.
- * If lzma_memory_alloc() is used as allocator, this should
- * be set to lzma_memory_free().
- *
* If you don't want a custom freeing function, but still
* want a custom allocator, set this to NULL and liblzma
* will use the standard free().
* and lzma_allocator.free(). This intended to ease implementing
* custom memory allocation functions for use with liblzma.
*
- * When using lzma_memory_alloc() and lzma_memory_free(), opaque
- * must point to lzma_memory_limiter structure allocated and
- * initialized with lzma_memory_limiter_create().
- *
* If you don't need this, you should set it to NULL.
*/
void *opaque;
/** Internal state is not visible to outsiders. */
lzma_internal *internal;
+ /**
+ * Reserved space to allow possible future extensions without
+ * breaking the ABI. Excluding the initialization of this structure,
+ * you should not touch these, because the names of these variables
+ * may change.
+ */
+ void *reserved_ptr1;
+ void *reserved_ptr2;
+ uint64_t reserved_int1;
+ uint64_t reserved_int2;
+
} lzma_stream;
* has been allocated yet:
*
* lzma_stream strm = LZMA_STREAM_INIT;
- */
-#define LZMA_STREAM_INIT { NULL, 0, 0, NULL, 0, 0, NULL, NULL }
-
-
-/**
- * \brief Initialization for lzma_stream
*
- * This is like LZMA_STREAM_INIT, but this can be used when the lzma_stream
- * has already been allocated:
+ * If you need to initialize a dynamically allocatedlzma_stream, you can use
+ * memset(strm_pointer, 0, sizeof(lzma_stream)). Strictly speaking, this
+ * violates the C standard since NULL may have different internal
+ * representation than zero, but it should be portable enough in practice.
+ * Anyway, for maximum portability, you could use this:
*
- * lzma_stream *strm = malloc(sizeof(lzma_stream));
- * if (strm == NULL)
- * return LZMA_MEM_ERROR;
- * *strm = LZMA_STREAM_INIT_VAR;
+ * lzma_stream tmp = LZMA_STREAM_INIT;
+ * *strm = tmp;
*/
-extern const lzma_stream LZMA_STREAM_INIT_VAR;
+#define LZMA_STREAM_INIT \
+ { NULL, 0, 0, NULL, 0, 0, NULL, NULL, NULL, NULL, 0, 0 }
/**
* - LZMA_PROG_ERROR: Invalid arguments or the internal state
* of the coder is corrupt.
*/
-extern lzma_ret lzma_code(lzma_stream *strm, lzma_action action);
+extern lzma_ret lzma_code(lzma_stream *strm, lzma_action action)
+ lzma_attr_warn_unused_result;
/**
* \brief Size of the Block Header
*
* Read by:
+ * - lzma_block_header_encode()
+ * - lzma_block_header_decode()
* - lzma_block_encoder()
* - lzma_block_decoder()
*
* Written by:
* - lzma_block_header_size()
- * - lzma_block_header_decode()
*/
uint32_t header_size;
# define LZMA_BLOCK_HEADER_SIZE_MIN 8
* Header, thus its value must be provided also when decoding.
*
* Read by:
+ * - lzma_block_header_encode()
+ * - lzma_block_header_decode()
* - lzma_block_encoder()
* - lzma_block_decoder()
*/
- lzma_check_type check;
+ lzma_check check;
/**
* \brief Size of the Compressed Data in bytes
* have LZMA_BLOCK_FILTERS_MAX + 1 members or the Block
* Header decoder will overflow the buffer.
*/
- lzma_options_filter *filters;
+ lzma_filter *filters;
# define LZMA_BLOCK_FILTERS_MAX 4
-} lzma_options_block;
+} lzma_block;
/**
* \brief Decodes the Block Header Size field
*
* To decode Block Header using lzma_block_header_decode(), the size of the
- * Block Header has to be known and stored into lzma_options_block.header_size.
+ * Block Header has to be known and stored into lzma_block.header_size.
* The size can be calculated from the first byte of a Block using this macro.
* Note that if the first byte is 0x00, it indicates beginning of Index; use
* this macro only when the byte is not 0x00.
* may return LZMA_OK even if lzma_block_header_encode() or
* lzma_block_encoder() would fail.
*/
-extern lzma_ret lzma_block_header_size(lzma_options_block *options);
+extern lzma_ret lzma_block_header_size(lzma_block *options)
+ lzma_attr_warn_unused_result;
/**
* - LZMA_PROG_ERROR
*/
extern lzma_ret lzma_block_header_encode(
- const lzma_options_block *options, uint8_t *out);
+ const lzma_block *options, uint8_t *out)
+ lzma_attr_warn_unused_result;
/**
* - LZMA_HEADER_ERROR: Invalid or unsupported options.
* - LZMA_PROG_ERROR
*/
-extern lzma_ret lzma_block_header_decode(lzma_options_block *options,
- lzma_allocator *allocator, const uint8_t *in);
+extern lzma_ret lzma_block_header_decode(lzma_block *options,
+ lzma_allocator *allocator, const uint8_t *in)
+ lzma_attr_warn_unused_result;
/**
* options->header_size between 8 and 1024 inclusive.
*/
extern lzma_ret lzma_block_total_size_set(
- lzma_options_block *options, lzma_vli total_size);
+ lzma_block *options, lzma_vli total_size)
+ lzma_attr_warn_unused_result;
/**
*
* \return Total Size on success, or zero on error.
*/
-extern lzma_vli lzma_block_total_size_get(const lzma_options_block *options);
+extern lzma_vli lzma_block_total_size_get(const lzma_block *options)
+ lzma_attr_pure;
/**
*
* lzma_code() can return FIXME
*/
-extern lzma_ret lzma_block_encoder(
- lzma_stream *strm, lzma_options_block *options);
+extern lzma_ret lzma_block_encoder(lzma_stream *strm, lzma_block *options)
+ lzma_attr_warn_unused_result;
/**
* - LZMA_PROG_ERROR
* - LZMA_MEM_ERROR
*/
-extern lzma_ret lzma_block_decoder(
- lzma_stream *strm, lzma_options_block *options);
+extern lzma_ret lzma_block_decoder(lzma_stream *strm, lzma_block *options)
+ lzma_attr_warn_unused_result;
*
* Size of the Check field: 32 bytes
*/
-} lzma_check_type;
+} lzma_check;
/**
/**
- * \brief Check IDs supported by this liblzma build
- *
- * If lzma_available_checks[n] is true, the Check ID n is supported by this
- * liblzma build. You can assume that LZMA_CHECK_NONE and LZMA_CHECK_CRC32
- * are always available.
+ * \brief Maximum size of a Check field
*/
-extern const lzma_bool lzma_available_checks[LZMA_CHECK_ID_MAX + 1];
+#define LZMA_CHECK_SIZE_MAX 64
/**
- * \brief Size of the Check field with different Check IDs
+ * \brief Test if the given Check ID is supported
*
- * Although not all Check IDs have a check algorithm associated, the size of
- * every Check is already frozen. This array contains the size (in bytes) of
- * the Check field with specified Check ID. The values are taken from the
- * section 2.1.1.2 of the .lzma file format specification:
- * { 0, 4, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 32, 64, 64, 64 }
+ * Returns true if the given Check ID is supported by this liblzma build.
+ * Otherwise false is returned. It is safe to call this with a value that
+ * is not in the range [0, 15]; in that case the return value is always false.
*/
-extern const uint32_t lzma_check_sizes[LZMA_CHECK_ID_MAX + 1];
+extern lzma_bool lzma_check_is_supported(lzma_check check)
+ lzma_attr_const;
/**
- * \brief Maximum size of a Check field
+ * \brief Get the size of the Check field with given Check ID
+ *
+ * Although not all Check IDs have a check algorithm associated, the size of
+ * every Check is already frozen. This function returns the size (in bytes) of
+ * the Check field with the specified Check ID. The values are taken from the
+ * section 2.1.1.2 of the .lzma file format specification:
+ * { 0, 4, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 32, 64, 64, 64 }
+ *
+ * If the argument is not in the range [0, 15], UINT32_MAX is returned.
*/
-#define LZMA_CHECK_SIZE_MAX 64
+extern uint32_t lzma_check_size(lzma_check check) lzma_attr_const;
/**
* \return Updated CRC value, which can be passed to this function
* again to continue CRC calculation.
*/
-extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc);
+extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc)
+ lzma_attr_pure;
/**
*
* This function is used similarly to lzma_crc32(). See its documentation.
*/
-extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc);
+extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc)
+ lzma_attr_pure;
/*
--- /dev/null
+/**
+ * \file lzma/FIXME.h
+ * \brief File formats
+ *
+ * \author Copyright (C) 1999-2008 Igor Pavlov
+ * \author Copyright (C) 2007-2008 Lasse Collin
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ */
+
+#ifndef LZMA_H_INTERNAL
+# error Never include this file directly. Use <lzma.h> instead.
+#endif
+
+
+/************
+ * Encoding *
+ ************/
+
+/**
+ * \brief Compression level names for lzma_easy_* functions
+ *
+ * At the moment, all the compression levels support LZMA_SYNC_FLUSH.
+ * In future there may be levels that don't support LZMA_SYNC_FLUSH.
+ * However, the LZMA_SYNC_FLUSH support won't be removed from the
+ * existing compression levels.
+ *
+ * \note If liblzma is built without encoder support, or with some
+ * filters disabled, some of the compression levels may be
+ * unsupported. In that case, the initialization functions
+ * will return LZMA_HEADER_ERROR.
+ */
+typedef enum {
+ LZMA_EASY_COPY = 0,
+ /**<
+ * No compression; the data is just wrapped into .lzma
+ * container.
+ */
+
+ LZMA_EASY_LZMA2_1 = 1,
+ /**<
+ * LZMA2 filter with fast compression (fast in terms of LZMA2).
+ * If you are interested in the exact options used, see
+ * lzma_preset_lzma[0]. Note that the exact options may
+ * change between liblzma versions.
+ *
+ * At the moment, the command line tool uses these settings
+ * when `lzma -1' is used. In future, the command line tool
+ * may default to some more complex way to determine the
+ * settings used e.g. the type of files being compressed.
+ *
+ * LZMA_EASY_LZMA_2 is equivalent to lzma_preset_lzma[1]
+ * and so on.
+ */
+
+ LZMA_EASY_LZMA_2 = 2,
+ LZMA_EASY_LZMA_3 = 3,
+ LZMA_EASY_LZMA_4 = 4,
+ LZMA_EASY_LZMA_5 = 5,
+ LZMA_EASY_LZMA_6 = 6,
+ LZMA_EASY_LZMA_7 = 7,
+ LZMA_EASY_LZMA_8 = 8,
+ LZMA_EASY_LZMA_9 = 9,
+} lzma_easy_level;
+
+
+/**
+ * \brief Default compression level
+ *
+ * Data Blocks contain the actual compressed data. It's not straightforward
+ * to recommend a default level, because in some cases keeping the resource
+ * usage relatively low is more important that getting the maximum
+ * compression ratio.
+ */
+#define LZMA_EASY_DEFAULT LZMA_EASY_LZMA2_7
+
+
+/**
+ * \brief Calculates rough memory requirements of a compression level
+ *
+ * This function is a wrapper for lzma_memory_usage(), which is declared
+ * in filter.h.
+ *
+ * \return Approximate memory usage of the encoder with the given
+ * compression level in mebibytes (value * 1024 * 1024 bytes).
+ * On error (e.g. compression level is not supported),
+ * UINT32_MAX is returned.
+ */
+extern uint32_t lzma_easy_memory_usage(lzma_easy_level level)
+ lzma_attr_pure;
+
+
+/**
+ * \brief Initializes .lzma Stream encoder
+ *
+ * This function is intended for those who just want to use the basic features
+ * if liblzma (that is, most developers out there). Lots of assumptions are
+ * made, which are correct or at least good enough for most situations.
+ *
+ * \param strm Pointer to lzma_stream that is at least initialized
+ * with LZMA_STREAM_INIT.
+ * \param level Compression level to use. This selects a set of
+ * compression settings from a list of compression
+ * presets.
+ *
+ * \return - LZMA_OK: Initialization succeeded. Use lzma_code() to
+ * encode your data.
+ * - LZMA_MEM_ERROR: Memory allocation failed. All memory
+ * previously allocated for *strm is now freed.
+ * - LZMA_HEADER_ERROR: The given compression level is not
+ * supported by this build of liblzma.
+ *
+ * If initialization succeeds, use lzma_code() to do the actual encoding.
+ * Valid values for `action' (the second argument of lzma_code()) are
+ * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future,
+ * there may be compression levels that don't support LZMA_SYNC_FLUSH.
+ */
+extern lzma_ret lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level)
+ lzma_attr_warn_unused_result;
+
+
+/**
+ * \brief Initializes .lzma Stream encoder
+ *
+ * \param strm Pointer to properly prepared lzma_stream
+ * \param filters Array of filters. This must be terminated with
+ * filters[n].id = LZMA_VLI_VALUE_UNKNOWN. There must
+ * be 1-4 filters, but there are restrictions on how
+ * multiple filters can be combined. FIXME Tell where
+ * to find more information.
+ * \param check Type of the integrity check to calculate from
+ * uncompressed data.
+ *
+ * \return - LZMA_OK: Initialization was successful.
+ * - LZMA_MEM_ERROR
+ * - LZMA_HEADER_ERROR
+ * - LZMA_PROG_ERROR
+ */
+extern lzma_ret lzma_stream_encoder(lzma_stream *strm,
+ const lzma_filter *filters, lzma_check check)
+ lzma_attr_warn_unused_result;
+
+
+/**
+ * \brief Initializes LZMA_Alone (deprecated file format) encoder
+ *
+ * LZMA_Alone files have the suffix .lzma like the .lzma Stream files.
+ * LZMA_Alone format supports only one filter, the LZMA filter. There is
+ * no support for integrity checks like CRC32.
+ *
+ * Use this format if and only if you need to create files readable by
+ * legacy LZMA tools such as LZMA Utils 4.32.x.
+ *
+ * LZMA_Alone encoder doesn't support LZMA_SYNC_FLUSH or LZMA_FULL_FLUSH.
+ *
+ * \return - LZMA_OK
+ * - LZMA_MEM_ERROR
+ * - LZMA_PROG_ERROR
+ */
+extern lzma_ret lzma_alone_encoder(
+ lzma_stream *strm, const lzma_options_lzma *options)
+ lzma_attr_warn_unused_result;
+
+
+/************
+ * Decoding *
+ ************/
+
+/**
+ * This flag makes lzma_code() return LZMA_NO_CHECK if the input stream
+ * being decoded has no integrity check. Note that when used with
+ * lzma_auto_decoder(), all LZMA_Alone files will cause trigger LZMA_NO_CHECK
+ * if LZMA_WARN_NO_CHECK is used.
+ */
+#define LZMA_WARN_NO_CHECK UINT32_C(0x01)
+
+
+/**
+ * This flag makes lzma_code() return LZMA_UNSUPPORTED_CHECK if the input
+ * stream has an integrity check, but the type of the integrity check is not
+ * supported by this liblzma version or build. Such files can still be
+ * decoded, but the integrity check cannot be verified.
+ */
+#define LZMA_WARN_UNSUPPORTED_CHECK UINT32_C(0x02)
+
+
+/**
+ * This flag makes lzma_code() return LZMA_READ_CHECK as soon as the type
+ * of the integrity check is known. The type can then be read with
+ * lzma_check_get().
+ */
+#define LZMA_TELL_CHECK UINT32_C(0x04)
+
+
+/**
+ * This flag makes lzma_code() decode concatenated .lzma files.
+ * FIXME Explain the changed API.
+ */
+#define LZMA_CONCATENATED UINT32_C(0x08)
+
+
+/**
+ * \brief Initializes decoder for .lzma Stream
+ *
+ * \param strm Pointer to propertily prepared lzma_stream
+ * \param memlimit Rough memory usage limit as bytes
+ *
+ * \return - LZMA_OK: Initialization was successful.
+ * - LZMA_MEM_ERROR: Cannot allocate memory.
+ */
+extern lzma_ret lzma_stream_decoder(
+ lzma_stream *strm, uint64_t memlimit, uint32_t flags)
+ lzma_attr_warn_unused_result;
+
+
+/**
+ * \brief Decode .lzma Streams and LZMA_Alone files with autodetection
+ *
+ * Autodetects between the .lzma Stream and LZMA_Alone formats, and
+ * calls lzma_stream_decoder_init() or lzma_alone_decoder_init() once
+ * the type of the file has been detected.
+ *
+ * \param strm Pointer to propertily prepared lzma_stream
+ * \param memlimit Rough memory usage limit as bytes
+ * \param flags Bitwise-or of flags, or zero for no flags.
+ *
+ * \return - LZMA_OK: Initialization was successful.
+ * - LZMA_MEM_ERROR: Cannot allocate memory.
+ */
+extern lzma_ret lzma_auto_decoder(
+ lzma_stream *strm, uint64_t memlimit, uint32_t flags)
+ lzma_attr_warn_unused_result;
+
+
+/**
+ * \brief Initializes decoder for LZMA_Alone file
+ *
+ * The LZMA_Alone decoder supports LZMA_SYNC_FLUSH. FIXME
+ *
+ * \return - LZMA_OK
+ * - LZMA_MEM_ERROR
+ */
+extern lzma_ret lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit)
+ lzma_attr_warn_unused_result;
/**
* \brief Filter ID
*
- * Filter ID of the Delta filter. This is used as lzma_options_filter.id.
+ * Filter ID of the Delta filter. This is used as lzma_filter.id.
*/
-#define LZMA_FILTER_DELTA LZMA_VLI_C(0x20)
+#define LZMA_FILTER_DELTA LZMA_VLI_C(0x03)
+
+
+/**
+ * \brief Type of the delta calculation
+ *
+ * Currently only byte-wise delta is supported. Other possible types could
+ * be, for example, delta of 16/32/64-bit little/big endian integers, but
+ * these are not currently planned since byte-wise delta is almost as good.
+ */
+typedef enum {
+ LZMA_DELTA_TYPE_BYTE
+} lzma_delta_type;
/**
* These options are needed by both encoder and decoder.
*/
typedef struct {
+ /** For now, this must always be LZMA_DELTA_TYPE_BYTE. */
+ lzma_delta_type type;
+
/**
- * \brief Delta distance as bytes
+ * \brief Delta distance
+ *
+ * With the only currently supported type, LZMA_DELTA_TYPE_BYTE,
+ * the distance is as bytes.
*
* Examples:
* - 16-bit stereo audio: distance = 4 bytes
# define LZMA_DELTA_DISTANCE_MIN 1
# define LZMA_DELTA_DISTANCE_MAX 256
+ /**
+ * \brief Reserved space for possible future extensions
+ *
+ * You should not touch these, because the names of these variables
+ * may change. These are and will never be used when type is
+ * LZMA_DELTA_TYPE_BYTE, so it is safe to leave these uninitialized.
+ */
+ uint32_t reserved_int1;
+ uint32_t reserved_int2;
+ void *reserved_ptr1;
+ void *reserved_ptr2;
+
} lzma_options_delta;
+++ /dev/null
-/**
- * \file lzma/easy.h
- * \brief Easy to use encoder initialization
- *
- * \author Copyright (C) 1999-2006 Igor Pavlov
- * \author Copyright (C) 2008 Lasse Collin
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- */
-
-#ifndef LZMA_H_INTERNAL
-# error Never include this file directly. Use <lzma.h> instead.
-#endif
-
-
-/**
- * \brief Compression level names for lzma_easy_* functions
- *
- * At the moment, all the compression levels support LZMA_SYNC_FLUSH.
- * In future there may be levels that don't support LZMA_SYNC_FLUSH.
- * However, the LZMA_SYNC_FLUSH support won't be removed from the
- * existing compression levels.
- *
- * \note If liblzma is built without encoder support, or with some
- * filters disabled, some of the compression levels may be
- * unsupported. In that case, the initialization functions
- * will return LZMA_HEADER_ERROR.
- */
-typedef enum {
- LZMA_EASY_COPY,
- /**<
- * No compression; the data is just wrapped into .lzma
- * container.
- */
-
- LZMA_EASY_LZMA_1,
- /**<
- * LZMA filter with fast compression (fast in terms of LZMA).
- * If you are interested in the exact options used, see
- * lzma_preset_lzma[0]. Note that the exact options may
- * change between liblzma versions.
- *
- * At the moment, the command line tool uses these settings
- * when `lzma -1' is used. In future, the command line tool
- * may default to some more complex way to determine the
- * settings used e.g. the type of files being compressed.
- *
- * LZMA_EASY_LZMA_2 is equivalent to lzma_preset_lzma[1]
- * and so on.
- */
-
- LZMA_EASY_LZMA_2,
- LZMA_EASY_LZMA_3,
- LZMA_EASY_LZMA_4,
- LZMA_EASY_LZMA_5,
- LZMA_EASY_LZMA_6,
- LZMA_EASY_LZMA_7,
- LZMA_EASY_LZMA_8,
- LZMA_EASY_LZMA_9,
-} lzma_easy_level;
-
-
-/**
- * \brief Default compression level
- *
- * Data Blocks contain the actual compressed data. It's not straightforward
- * to recommend a default level, because in some cases keeping the resource
- * usage relatively low is more important that getting the maximum
- * compression ratio.
- */
-#define LZMA_EASY_DEFAULT LZMA_EASY_LZMA_7
-
-
-/**
- * \brief Calculates rough memory requirements of a compression level
- *
- * This function is a wrapper for lzma_memory_usage(), which is declared
- * in lzma/filter.h.
- *
- * \return Approximate memory usage of the encoder with the given
- * compression level in mebibytes (value * 1024 * 1024 bytes).
- * On error (e.g. compression level is not supported),
- * UINT32_MAX is returned.
- */
-extern uint32_t lzma_easy_memory_usage(lzma_easy_level level);
-
-
-/**
- * \brief Initializes .lzma Stream encoder
- *
- * This function is intended for those who just want to use the basic LZMA
- * features (that is, most developers out there). Lots of assumptions are
- * made, which are correct or at least good enough for most situations.
- *
- * \param strm Pointer to lzma_stream that is at least initialized
- * with LZMA_STREAM_INIT.
- * \param level Compression level to use. This selects a set of
- * compression settings from a list of compression
- * presets.
- *
- * \return - LZMA_OK: Initialization succeeded. Use lzma_code() to
- * encode your data.
- * - LZMA_MEM_ERROR: Memory allocation failed. All memory
- * previously allocated for *strm is now freed.
- * - LZMA_HEADER_ERROR: The given compression level is not
- * supported by this build of liblzma.
- *
- * If initialization succeeds, use lzma_code() to do the actual encoding.
- * Valid values for `action' (the second argument of lzma_code()) are
- * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future,
- * there may be compression levels that don't support LZMA_SYNC_FLUSH.
- */
-extern lzma_ret lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level);
*/
void *options;
-} lzma_options_filter;
+} lzma_filter;
/**
* encoding-specific functions are probably missing from the library
* API/ABI completely.
*/
-extern const lzma_vli *const lzma_available_filter_encoders;
+extern const lzma_vli *const lzma_filter_encoders;
/**
* decoding-specific functions are probably missing from the library
* API/ABI completely.
*/
-extern const lzma_vli *const lzma_available_filter_decoders;
+extern const lzma_vli *const lzma_filter_decoders;
/**
*
* \param filters Array of filters terminated with
* .id == LZMA_VLI_VALUE_UNKNOWN.
- * \param is_encoder Set to true when calculating memory requirements
- * of an encoder; false for decoder.
*
* \return Number of mebibytes (MiB i.e. 2^20) required for the given
* encoder or decoder filter chain.
* if calculating memory requirements of decoder, lzma_init() or
* lzma_init_decoder() must have been called earlier.
*/
-extern uint32_t lzma_memory_usage(
- const lzma_options_filter *filters, lzma_bool is_encoder);
+// extern uint32_t lzma_memory_usage(
+// const lzma_filter *filters, lzma_bool is_encoder);
+
+extern uint64_t lzma_memusage_encoder(const lzma_filter *filters)
+ lzma_attr_pure;
+
+extern uint64_t lzma_memusage_decoder(const lzma_filter *filters)
+ lzma_attr_pure;
+
+
+/**
+ * \brief Initializes raw encoder
+ *
+ * This function may be useful when implementing custom file formats.
+ *
+ * \param strm Pointer to properly prepared lzma_stream
+ * \param options Array of lzma_filter structures.
+ * The end of the array must be marked with
+ * .id = LZMA_VLI_VALUE_UNKNOWN. The minimum
+ * number of filters is one and the maximum is four.
+ *
+ * The `action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the
+ * filter chain supports it), or LZMA_FINISH.
+ *
+ * \return - LZMA_OK
+ * - LZMA_MEM_ERROR
+ * - LZMA_HEADER_ERROR
+ * - LZMA_PROG_ERROR
+ */
+extern lzma_ret lzma_raw_encoder(
+ lzma_stream *strm, const lzma_filter *options)
+ lzma_attr_warn_unused_result;
+
+
+/**
+ * \brief Initializes raw decoder
+ *
+ * The initialization of raw decoder goes similarly to raw encoder.
+ *
+ * The `action' with lzma_code() can be LZMA_RUN or LZMA_SYNC_FLUSH.
+ *
+ * \return - LZMA_OK
+ * - LZMA_MEM_ERROR
+ * - LZMA_HEADER_ERROR
+ * - LZMA_PROG_ERROR
+ */
+extern lzma_ret lzma_raw_decoder(
+ lzma_stream *strm, const lzma_filter *options)
+ lzma_attr_warn_unused_result;
/**
* - LZMA_PROG_ERROR: Invalid options
*
* \note If you need to calculate size of List of Filter Flags,
- * you need to loop over every lzma_options_filter entry.
+ * you need to loop over every lzma_filter entry.
*/
extern lzma_ret lzma_filter_flags_size(
- uint32_t *size, const lzma_options_filter *options);
+ uint32_t *size, const lzma_filter *options)
+ lzma_attr_warn_unused_result;
/**
* buffer space (you should have checked it with
* lzma_filter_flags_size()).
*/
-extern lzma_ret lzma_filter_flags_encode(uint8_t *out, size_t *out_pos,
- size_t out_size, const lzma_options_filter *options);
+extern lzma_ret lzma_filter_flags_encode(const lzma_filter *options,
+ uint8_t *out, size_t *out_pos, size_t out_size)
+ lzma_attr_warn_unused_result;
/**
* - LZMA_PROG_ERROR
*/
extern lzma_ret lzma_filter_flags_decode(
- lzma_options_filter *options, lzma_allocator *allocator,
- const uint8_t *in, size_t *in_pos, size_t in_size);
+ lzma_filter *options, lzma_allocator *allocator,
+ const uint8_t *in, size_t *in_pos, size_t in_size)
+ lzma_attr_warn_unused_result;
* In this case, return value cannot be NULL or a different pointer than
* the i given as argument.
*/
-extern lzma_index *lzma_index_init(lzma_index *i, lzma_allocator *allocator);
+extern lzma_index *lzma_index_init(lzma_index *i, lzma_allocator *allocator)
+ lzma_attr_warn_unused_result;
/**
* - LZMA_PROG_ERROR
*/
extern lzma_ret lzma_index_append(lzma_index *i, lzma_allocator *allocator,
- lzma_vli total_size, lzma_vli uncompressed_size);
+ lzma_vli total_size, lzma_vli uncompressed_size)
+ lzma_attr_warn_unused_result;
/**
* \brief Get the number of Records
*/
-extern lzma_vli lzma_index_count(const lzma_index *i);
+extern lzma_vli lzma_index_count(const lzma_index *i) lzma_attr_pure;
/**
*
* This is needed to verify the Index Size field from the Stream Footer.
*/
-extern lzma_vli lzma_index_size(const lzma_index *i);
+extern lzma_vli lzma_index_size(const lzma_index *i) lzma_attr_pure;
/**
* This doesn't include the Stream Header, Stream Footer, Stream Padding,
* or Index fields.
*/
-extern lzma_vli lzma_index_total_size(const lzma_index *i);
+extern lzma_vli lzma_index_total_size(const lzma_index *i) lzma_attr_pure;
/**
* If multiple Indexes have been combined, this works as if the Blocks
* were in a single Stream.
*/
-extern lzma_vli lzma_index_stream_size(const lzma_index *i);
+extern lzma_vli lzma_index_stream_size(const lzma_index *i) lzma_attr_pure;
/**
* identical to lzma_index_stream_size(). If multiple Indexes have been
* combined, this includes also the possible Stream Padding fields.
*/
-extern lzma_vli lzma_index_file_size(const lzma_index *i);
+extern lzma_vli lzma_index_file_size(const lzma_index *i) lzma_attr_pure;
/**
* \brief Get the uncompressed size of the Stream
*/
-extern lzma_vli lzma_index_uncompressed_size(const lzma_index *i);
+extern lzma_vli lzma_index_uncompressed_size(const lzma_index *i)
+ lzma_attr_pure;
/**
* \brief Get the next Record from the Index
*/
-extern lzma_bool lzma_index_read(lzma_index *i, lzma_index_record *record);
+extern lzma_bool lzma_index_read(lzma_index *i, lzma_index_record *record)
+ lzma_attr_warn_unused_result;
/**
* and the read position are not modified, and this function returns true.
*/
extern lzma_bool lzma_index_locate(
- lzma_index *i, lzma_index_record *record, lzma_vli target);
+ lzma_index *i, lzma_index_record *record, lzma_vli target)
+ lzma_attr_warn_unused_result;
/**
*/
extern lzma_ret lzma_index_cat(lzma_index *lzma_restrict dest,
lzma_index *lzma_restrict src,
- lzma_allocator *allocator, lzma_vli padding);
+ lzma_allocator *allocator, lzma_vli padding)
+ lzma_attr_warn_unused_result;
/**
* \return A copy of the Index, or NULL if memory allocation failed.
*/
extern lzma_index *lzma_index_dup(
- const lzma_index *i, lzma_allocator *allocator);
+ const lzma_index *i, lzma_allocator *allocator)
+ lzma_attr_warn_unused_result;
/**
* \brief Compares if two Index lists are identical
*/
-extern lzma_bool lzma_index_equal(const lzma_index *a, const lzma_index *b);
+extern lzma_bool lzma_index_equal(const lzma_index *a, const lzma_index *b)
+ lzma_attr_pure;
/**
* \brief Initializes Index encoder
*/
-extern lzma_ret lzma_index_encoder(lzma_stream *strm, lzma_index *i);
+extern lzma_ret lzma_index_encoder(lzma_stream *strm, lzma_index *i)
+ lzma_attr_warn_unused_result;
/**
* \brief Initializes Index decoder
*/
-extern lzma_ret lzma_index_decoder(lzma_stream *strm, lzma_index **i);
+extern lzma_ret lzma_index_decoder(lzma_stream *strm, lzma_index **i)
+ lzma_attr_warn_unused_result;
* pointer than the index_hash given as argument.
*/
extern lzma_index_hash *lzma_index_hash_init(
- lzma_index_hash *index_hash, lzma_allocator *allocator);
+ lzma_index_hash *index_hash, lzma_allocator *allocator)
+ lzma_attr_warn_unused_result;
/**
* used when lzma_index_hash_decode() has already been used.
*/
extern lzma_ret lzma_index_hash_append(lzma_index_hash *index_hash,
- lzma_vli total_size, lzma_vli uncompressed_size);
+ lzma_vli total_size, lzma_vli uncompressed_size)
+ lzma_attr_warn_unused_result;
/**
* Records can be added using lzma_index_hash_append().
*/
extern lzma_ret lzma_index_hash_decode(lzma_index_hash *index_hash,
- const uint8_t *in, size_t *in_pos, size_t in_size);
+ const uint8_t *in, size_t *in_pos, size_t in_size)
+ lzma_attr_warn_unused_result;
/**
*
* This is needed to verify the Index Size field from the Stream Footer.
*/
-extern lzma_vli lzma_index_hash_size(const lzma_index_hash *index_hash);
+extern lzma_vli lzma_index_hash_size(const lzma_index_hash *index_hash)
+ lzma_attr_pure;
/**
* \brief Filter ID
*
- * Filter ID of the LZMA filter. This is used as lzma_options_filter.id.
+ * Filter ID of the LZMA filter. This is used as lzma_filter.id.
*/
#define LZMA_FILTER_LZMA LZMA_VLI_C(0x40)
-
-/**
- * \brief LZMA compression modes
- *
- * Currently there are only two modes. Earlier LZMA SDKs had also third
- * mode between fast and best.
- */
-typedef enum {
- LZMA_MODE_INVALID = -1,
- /**<
- * \brief Invalid mode
- *
- * Used as array terminator in lzma_available_modes.
- */
-
-
- LZMA_MODE_FAST = 0,
- /**<
- * \brief Fast compression
- *
- * Fast mode is usually at its best when combined with
- * a hash chain match finder.
- */
-
- LZMA_MODE_BEST = 2
- /**<
- * \brief Best compression ratio
- *
- * This is usually notably slower than fast mode. Use this
- * together with binary tree match finders to expose the
- * full potential of the LZMA encoder.
- */
-} lzma_mode;
+#define LZMA_FILTER_LZMA2 LZMA_VLI_C(0x21)
/**
/**
+ * \brief Test if given match finder is supported
+ *
+ * Returns true if the given match finder is supported by this liblzma build.
+ * Otherwise false is returned. It is safe to call this with a value that
+ * isn't listed in lzma_match_finder enumeration; the return value will be
+ * false.
+ *
+ * There is no way to list which match finders are available in this
+ * particular liblzma version and build. It would be useless, because
+ * a new match finder, which the application developer wasn't aware,
+ * could require giving additional options to the encoder that the older
+ * match finders don't need.
+ */
+extern lzma_bool lzma_mf_is_supported(lzma_match_finder match_finder)
+ lzma_attr_const;
+
+
+/**
+ * \brief LZMA compression modes
+ *
+ * This selects the function used to analyze the data produced by the match
+ * finder.
+ */
+typedef enum {
+ LZMA_MODE_INVALID = -1,
+ /**<
+ * \brief Invalid mode
+ *
+ * Used as array terminator in lzma_available_modes.
+ */
+
+ LZMA_MODE_FAST = 0,
+ /**<
+ * \brief Fast compression
+ *
+ * Fast mode is usually at its best when combined with
+ * a hash chain match finder.
+ */
+
+ LZMA_MODE_NORMAL = 1
+ /**<
+ * \brief Normal compression
+ *
+ * This is usually notably slower than fast mode. Use this
+ * together with binary tree match finders to expose the
+ * full potential of the LZMA encoder.
+ */
+} lzma_mode;
+
+
+/**
+ * \brief Test if given compression mode is supported
+ *
+ * Returns true if the given compression mode is supported by this liblzma
+ * build. Otherwise false is returned. It is safe to call this with a value
+ * that isn't listed in lzma_mode enumeration; the return value will be false.
+ *
+ * There is no way to list which modes are available in this particular
+ * liblzma version and build. It would be useless, because a new compression
+ * mode, which the application developer wasn't aware, could require giving
+ * additional options to the encoder that the older modes don't need.
+ */
+extern lzma_bool lzma_mode_is_available(lzma_mode mode) lzma_attr_const;
+
+
+/**
* \brief Options specific to the LZMA method handler
*/
typedef struct {
# define LZMA_DICTIONARY_SIZE_DEFAULT (UINT32_C(1) << 23)
/**
+ * \brief Pointer to an initial dictionary
+ *
+ * It is possible to initialize the LZ77 history window using
+ * a preset dictionary. Here is a good quote from zlib's
+ * documentation; this applies to LZMA as is:
+ *
+ * "The dictionary should consist of strings (byte sequences) that
+ * are likely to be encountered later in the data to be compressed,
+ * with the most commonly used strings preferably put towards the
+ * end of the dictionary. Using a dictionary is most useful when
+ * the data to be compressed is short and can be predicted with
+ * good accuracy; the data can then be compressed better than
+ * with the default empty dictionary."
+ * (From deflateSetDictionary() in zlib.h of zlib version 1.2.3)
+ *
+ * This feature should be used only in special situations.
+ * It works correctly only with raw encoding and decoding.
+ * Currently none of the container formats supported by
+ * liblzma allow preset dictionary when decoding, thus if
+ * you create a .lzma file with preset dictionary, it cannot
+ * be decoded with the regular .lzma decoder functions.
+ *
+ * \todo This feature is not implemented yet.
+ */
+ const uint8_t *preset_dictionary;
+
+ /**
+ * \brief Size of the preset dictionary
+ *
+ * Specifies the size of the preset dictionary. If the size is
+ * bigger than dictionary_size, only the last dictionary_size
+ * bytes are processed.
+ *
+ * This variable is read only when preset_dictionary is not NULL.
+ */
+ uint32_t preset_dictionary_size;
+
+ /**
* \brief Number of literal context bits
*
* How many of the highest bits of the previous uncompressed
# define LZMA_POS_BITS_MAX 4
# define LZMA_POS_BITS_DEFAULT 2
- /**
- * \brief Pointer to an initial dictionary
- *
- * It is possible to initialize the LZ77 history window using
- * a preset dictionary. Here is a good quote from zlib's
- * documentation; this applies to LZMA as is:
- *
- * "The dictionary should consist of strings (byte sequences) that
- * are likely to be encountered later in the data to be compressed,
- * with the most commonly used strings preferably put towards the
- * end of the dictionary. Using a dictionary is most useful when
- * the data to be compressed is short and can be predicted with
- * good accuracy; the data can then be compressed better than
- * with the default empty dictionary."
- * (From deflateSetDictionary() in zlib.h of zlib version 1.2.3)
- *
- * This feature should be used only in special situations.
- * It works correctly only with raw encoding and decoding.
- * Currently none of the container formats supported by
- * liblzma allow preset dictionary when decoding, thus if
- * you create a .lzma file with preset dictionary, it cannot
- * be decoded with the regular .lzma decoder functions.
- *
- * \todo This feature is not implemented yet.
- */
- const uint8_t *preset_dictionary;
+ /******************************************
+ * LZMA options needed only when encoding *
+ ******************************************/
/**
- * \brief Size of the preset dictionary
+ * \brief Indicate if the options structure is persistent
*
- * Specifies the size of the preset dictionary. If the size is
- * bigger than dictionary_size, only the last dictionary_size
- * bytes are processed.
+ * If this is true, the application must keep this options structure
+ * available after the LZMA2 encoder has been initialized. With
+ * persistent structure it is possible to change some encoder options
+ * in the middle of the encoding process without resetting the encoder.
*
- * This variable is read only when preset_dictionary is not NULL.
+ * This option is used only by LZMA2. LZMA1 ignores this and it is
+ * safeto not initialize this when encoding with LZMA1.
*/
- uint32_t preset_dictionary_size;
-
- /******************************************
- * LZMA options needed only when encoding *
- ******************************************/
+ lzma_bool persistent;
/** LZMA compression mode */
lzma_mode mode;
*/
uint32_t match_finder_cycles;
+ /**
+ * \brief Reserved space for possible future extensions
+ *
+ * You should not touch these, because the names of these variables
+ * may change. These are and will never be used with the currently
+ * supported options, so it is safe to leave these uninitialized.
+ */
+ uint32_t reserved_int1;
+ uint32_t reserved_int2;
+ uint32_t reserved_int3;
+ uint32_t reserved_int4;
+ void *reserved_ptr1;
+ void *reserved_ptr2;
+
} lzma_options_lzma;
/**
- * \brief Available LZMA encoding modes
- *
- * Pointer to an array containing the list of available encoding modes.
- *
- * This variable is available only if LZMA encoder has been enabled.
- */
-extern const lzma_mode *const lzma_available_modes;
-
-
-/**
- * \brief Available match finders
- *
- * Pointer to an array containing the list of available match finders.
- * The last element is LZMA_MF_INVALID.
- *
- * This variable is available only if LZMA encoder has been enabled.
- */
-extern const lzma_match_finder *const lzma_available_match_finders;
-
-
-/**
* \brief Table of presets for the LZMA filter
*
* lzma_preset_lzma[0] is the fastest and lzma_preset_lzma[8] is the slowest.
* lzma_memlimit_ can be used even if lzma_init() hasn't been
* called.
*/
-extern lzma_memlimit *lzma_memlimit_create(size_t limit);
+extern lzma_memlimit *lzma_memlimit_create(size_t limit)
+ lzma_attr_warn_unused_result;
/**
/**
* \brief Gets the current memory usage limit
*/
-extern size_t lzma_memlimit_get(const lzma_memlimit *mem);
+extern size_t lzma_memlimit_get(const lzma_memlimit *mem)
+ lzma_attr_pure;
/**
* thus it will always be larger than the total number of
* bytes allocated via lzma_memlimit_alloc().
*/
-extern size_t lzma_memlimit_used(const lzma_memlimit *mem);
+extern size_t lzma_memlimit_used(const lzma_memlimit *mem)
+ lzma_attr_pure;
/**
* been allocated with lzma_memlimit_alloc() or all memory allocated
* has been freed or detached, this will return zero.
*/
-extern size_t lzma_memlimit_count(const lzma_memlimit *mem);
+extern size_t lzma_memlimit_count(const lzma_memlimit *mem)
+ lzma_attr_pure;
/**
* invalid amount of memory being allocated.
*/
extern void *lzma_memlimit_alloc(
- lzma_memlimit *mem, size_t nmemb, size_t size);
+ lzma_memlimit *mem, size_t nmemb, size_t size)
+ lzma_attr_warn_unused_result;
/**
+++ /dev/null
-/**
- * \file lzma/raw.h
- * \brief Raw encoder and decoder
- *
- * \author Copyright (C) 1999-2006 Igor Pavlov
- * \author Copyright (C) 2007 Lasse Collin
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- */
-
-#ifndef LZMA_H_INTERNAL
-# error Never include this file directly. Use <lzma.h> instead.
-#endif
-
-
-/**
- * \brief Initializes raw encoder
- *
- * This function may be useful when implementing custom file formats.
- *
- * \param strm Pointer to properly prepared lzma_stream
- * \param options Array of lzma_options_filter structures.
- * The end of the array must be marked with
- * .id = LZMA_VLI_VALUE_UNKNOWN. The minimum
- * number of filters is one and the maximum is four.
- *
- * The `action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the
- * filter chain supports it), or LZMA_FINISH.
- *
- * \return - LZMA_OK
- * - LZMA_MEM_ERROR
- * - LZMA_HEADER_ERROR
- * - LZMA_PROG_ERROR
- */
-extern lzma_ret lzma_raw_encoder(
- lzma_stream *strm, const lzma_options_filter *options);
-
-
-/**
- * \brief Initializes raw decoder
- *
- * The initialization of raw decoder goes similarly to raw encoder.
- *
- * The `action' with lzma_code() can be LZMA_RUN or LZMA_SYNC_FLUSH.
- *
- * \return - LZMA_OK
- * - LZMA_MEM_ERROR
- * - LZMA_HEADER_ERROR
- * - LZMA_PROG_ERROR
- */
-extern lzma_ret lzma_raw_decoder(
- lzma_stream *strm, const lzma_options_filter *options);
#endif
-/* Filter IDs for lzma_options_filter.id */
+/* Filter IDs for lzma_filter.id */
#define LZMA_FILTER_X86 LZMA_VLI_C(0x04)
/**<
+++ /dev/null
-/**
- * \file lzma/stream.h
- * \brief .lzma Stream handling
- *
- * \author Copyright (C) 1999-2006 Igor Pavlov
- * \author Copyright (C) 2007 Lasse Collin
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- */
-
-#ifndef LZMA_H_INTERNAL
-# error Never include this file directly. Use <lzma.h> instead.
-#endif
-
-
-/**
- * \brief Initializes .lzma Stream encoder
- *
- * \param strm Pointer to properly prepared lzma_stream
- * \param filters Array of filters. This must be terminated with
- * filters[n].id = LZMA_VLI_VALUE_UNKNOWN. There must
- * be 1-4 filters, but there are restrictions on how
- * multiple filters can be combined. FIXME Tell where
- * to find more information.
- * \param check Type of the integrity check to calculate from
- * uncompressed data.
- *
- * \return - LZMA_OK: Initialization was successful.
- * - LZMA_MEM_ERROR
- * - LZMA_HEADER_ERROR
- * - LZMA_PROG_ERROR
- */
-extern lzma_ret lzma_stream_encoder(lzma_stream *strm,
- const lzma_options_filter *filters, lzma_check_type check);
-
-
-/**
- * \brief Initializes decoder for .lzma Stream
- *
- * \param strm Pointer to propertily prepared lzma_stream
- *
- * \return - LZMA_OK: Initialization was successful.
- * - LZMA_MEM_ERROR: Cannot allocate memory.
- */
-extern lzma_ret lzma_stream_decoder(lzma_stream *strm);
/**
* Type of the Check calculated from uncompressed data
*/
- lzma_check_type check;
+ lzma_check check;
} lzma_stream_flags;
* - LZMA_PROG_ERROR: Invalid options.
*/
extern lzma_ret lzma_stream_header_encode(
- const lzma_stream_flags *options, uint8_t *out);
+ const lzma_stream_flags *options, uint8_t *out)
+ lzma_attr_warn_unused_result;
/**
* - LZMA_PROG_ERROR: Invalid options.
*/
extern lzma_ret lzma_stream_footer_encode(
- const lzma_stream_flags *options, uint8_t *out);
+ const lzma_stream_flags *options, uint8_t *out)
+ lzma_attr_warn_unused_result;
/**
* in the header.
*/
extern lzma_ret lzma_stream_header_decode(
- lzma_stream_flags *options, const uint8_t *in);
+ lzma_stream_flags *options, const uint8_t *in)
+ lzma_attr_warn_unused_result;
/**
* in the footer.
*/
extern lzma_ret lzma_stream_footer_decode(
- lzma_stream_flags *options, const uint8_t *in);
+ lzma_stream_flags *options, const uint8_t *in)
+ lzma_attr_warn_unused_result;
/**
* \return true if both structures are considered equal; false otherwise.
*/
extern lzma_bool lzma_stream_flags_equal(
- const lzma_stream_flags *a, lzma_stream_flags *b);
+ const lzma_stream_flags *a, const lzma_stream_flags *b)
+ lzma_attr_pure;
/**
* \brief Filter ID
*
- * Filter ID of the Subblock filter. This is used as lzma_options_filter.id.
+ * Filter ID of the Subblock filter. This is used as lzma_filter.id.
*/
#define LZMA_FILTER_SUBBLOCK LZMA_VLI_C(0x01)
*
* \note This variable is ignored if allow_subfilters is false.
*/
- lzma_options_filter subfilter_options;
+ lzma_filter subfilter_options;
} lzma_options_subblock;
/**
* \brief liblzma version number as an integer
*
- * This is the value of LZMA_VERSION macro at the compile time of liblzma.
+ * Returns the value of LZMA_VERSION macro at the compile time of liblzma.
* This allows the application to compare if it was built against the same,
* older, or newer version of liblzma that is currently running.
*/
-extern const uint32_t lzma_version_number;
+extern uint32_t lzma_version_number(void) lzma_attr_const;
/**
- * \brief Returns versions number of liblzma as a string
+ * \brief Version number of liblzma as a string
*
* This function may be useful if you want to display which version of
- * libilzma your application is currently using.
+ * liblzma your application is currently using.
*/
-extern const char *const lzma_version_string;
+extern const char *lzma_version_string(void) lzma_attr_const;
/**
- * \brief Sets VLI to given value with error checking
- *
- * \param dest Target variable which must have type of lzma_vli.
- * \param src New value to be stored to dest.
- * \param limit Maximum allowed value for src.
- *
- * \return False on success, true on error. If an error occurred,
- * dest is left in undefined state (i.e. it's possible that
- * it will be different in newer liblzma versions).
- */
-#define lzma_vli_set_lim(dest, src, limit) \
- ((src) > (limit) || ((dest) = (src)) > (limit))
-
-/**
- * \brief
- */
-#define lzma_vli_add_lim(dest, src, limit) \
- ((src) > (limit) || ((dest) += (src)) > (limit))
-
-#define lzma_vli_add2_lim(dest, src1, src2, limit) \
- (lzma_vli_add_lim(dest, src1, limit) \
- || lzma_vli_add_lim(dest, src2, limit))
-
-#define lzma_vli_add3_lim(dest, src1, src2, src3, limit) \
- (lzma_vli_add_lim(dest, src1, limit) \
- || lzma_vli_add_lim(dest, src2, limit) \
- || lzma_vli_add_lim(dest, src3, limit))
-
-#define lzma_vli_add4_lim(dest, src1, src2, src3, src4, limit) \
- (lzma_vli_add_lim(dest, src1, limit) \
- || lzma_vli_add_lim(dest, src2, limit) \
- || lzma_vli_add_lim(dest, src3, limit) \
- || lzma_vli_add_lim(dest, src4, limit))
-
-#define lzma_vli_sum_lim(dest, src1, src2, limit) \
- (lzma_vli_set_lim(dest, src1, limit) \
- || lzma_vli_add_lim(dest, src2, limit))
-
-#define lzma_vli_sum3_lim(dest, src1, src2, src3, limit) \
- (lzma_vli_set_lim(dest, src1, limit) \
- || lzma_vli_add_lim(dest, src2, limit) \
- || lzma_vli_add_lim(dest, src3, limit))
-
-#define lzma_vli_sum4_lim(dest, src1, src2, src3, src4, limit) \
- (lzma_vli_set_lim(dest, src1, limit) \
- || lzma_vli_add_lim(dest, src2, limit) \
- || lzma_vli_add_lim(dest, src3, limit) \
- || lzma_vli_add_lim(dest, src4, limit))
-
-#define lzma_vli_set(dest, src) lzma_vli_set_lim(dest, src, LZMA_VLI_VALUE_MAX)
-
-#define lzma_vli_add(dest, src) lzma_vli_add_lim(dest, src, LZMA_VLI_VALUE_MAX)
-
-#define lzma_vli_add2(dest, src1, src2) \
- lzma_vli_add2_lim(dest, src1, src2, LZMA_VLI_VALUE_MAX)
-
-#define lzma_vli_add3(dest, src1, src2, src3) \
- lzma_vli_add3_lim(dest, src1, src2, src3, LZMA_VLI_VALUE_MAX)
-
-#define lzma_vli_add4(dest, src1, src2, src3, src4) \
- lzma_vli_add4_lim(dest, src1, src2, src3, src4, LZMA_VLI_VALUE_MAX)
-
-#define lzma_vli_sum(dest, src1, src2) \
- lzma_vli_sum_lim(dest, src1, src2, LZMA_VLI_VALUE_MAX)
-
-#define lzma_vli_sum3(dest, src1, src2, src3) \
- lzma_vli_sum3_lim(dest, src1, src2, src3, LZMA_VLI_VALUE_MAX)
-
-#define lzma_vli_sum4(dest, src1, src2, src3, src4) \
- lzma_vli_sum4_lim(dest, src1, src2, src3, src4, LZMA_VLI_VALUE_MAX)
-
-
-/**
* \brief Encodes variable-length integer
*
- * In the new .lzma format, most integers are encoded in variable-length
+ * In the .lzma format, most integers are encoded in variable-length
* representation. This saves space when smaller values are more likely
* than bigger values.
*
* The encoding scheme encodes seven bits to every byte, using minimum
- * number of bytes required to represent the given value. In other words,
- * it puts 7-63 bits into 1-9 bytes. This implementation limits the number
- * of bits used to 63, thus num must be at maximum of UINT64_MAX / 2. You
- * may use LZMA_VLI_VALUE_MAX for clarity.
+ * number of bytes required to represent the given value. Encodings that use
+ * non-minimum number of bytes are invalid, thus every integer has exactly
+ * one encoded representation. The maximum number of bits in a VLI is 63,
+ * thus the vli argument must be at maximum of UINT64_MAX / 2. You should
+ * use LZMA_VLI_VALUE_MAX for clarity.
+ *
+ * This function has two modes: single-call and multi-call. Single-call mode
+ * encodes the whole integer at once; it is an error if the output buffer is
+ * too small. Multi-call mode saves the position in *vli_pos, and thus it is
+ * possible to continue encoding if the buffer becomes full before the whole
+ * integer has been encoded.
*
* \param vli Integer to be encoded
* \param vli_pos How many VLI-encoded bytes have already been written
* \return Slightly different return values are used in multi-call and
* single-call modes.
*
+ * Single-call (vli_pos == NULL):
+ * - LZMA_OK: Integer successfully encoded.
+ * - LZMA_PROG_ERROR: Arguments are not sane. This can be due
+ * to too little output space; single-call mode doesn't use
+ * LZMA_BUF_ERROR, since the application should have checked
+ * the encoded size with lzma_vli_size().
+ *
* Multi-call (vli_pos != NULL):
* - LZMA_OK: So far all OK, but the integer is not
* completely written out yet.
* - LZMA_STREAM_END: Integer successfully encoded.
- * - LZMA_PROG_ERROR: Arguments are not sane. This can be due
- * to no *out_pos == out_size; this function doesn't use
- * LZMA_BUF_ERROR.
- *
- * Single-call (vli_pos == NULL):
- * - LZMA_OK: Integer successfully encoded.
- * - LZMA_PROG_ERROR: Arguments are not sane. This can be due
- * to too little output space; this function doesn't use
- * LZMA_BUF_ERROR.
+ * - LZMA_BUF_ERROR: No output space was provided.
+ * - LZMA_PROG_ERROR: Arguments are not sane.
*/
extern lzma_ret lzma_vli_encode(
lzma_vli vli, size_t *lzma_restrict vli_pos,
/**
* \brief Decodes variable-length integer
*
+ * Like lzma_vli_encode(), this function has single-call and multi-call modes.
+ *
* \param vli Pointer to decoded integer. The decoder will
* initialize it to zero when *vli_pos == 0, so
* application isn't required to initialize *vli.
* \return Slightly different return values are used in multi-call and
* single-call modes.
*
+ * Single-call (vli_pos == NULL):
+ * - LZMA_OK: Integer successfully decoded.
+ * - LZMA_DATA_ERROR: Integer is corrupt. This includes hitting
+ * the end of the input buffer before the whole integer was
+ * decoded; providing no input at all will use LZMA_DATA_ERROR.
+ * - LZMA_PROG_ERROR: Arguments are not sane.
+ *
* Multi-call (vli_pos != NULL):
* - LZMA_OK: So far all OK, but the integer is not
* completely decoded yet.
* - LZMA_STREAM_END: Integer successfully decoded.
* - LZMA_DATA_ERROR: Integer is corrupt.
- * - LZMA_PROG_ERROR: Arguments are not sane. This can be
- * due to *in_pos == in_size; this function doesn't use
- * LZMA_BUF_ERROR.
- *
- * Single-call (vli_pos == NULL):
- * - LZMA_OK: Integer successfully decoded.
- * - LZMA_DATA_ERROR: Integer is corrupt.
- * - LZMA_PROG_ERROR: Arguments are not sane. This can be due to
- * too little input; this function doesn't use LZMA_BUF_ERROR.
+ * - LZMA_BUF_ERROR: No input was provided.
+ * - LZMA_PROG_ERROR: Arguments are not sane.
*/
extern lzma_ret lzma_vli_decode(lzma_vli *lzma_restrict vli,
size_t *lzma_restrict vli_pos, const uint8_t *lzma_restrict in,
* \return Number of bytes on success (1-9). If vli isn't valid,
* zero is returned.
*/
-extern uint32_t lzma_vli_size(lzma_vli vli);
+extern uint32_t lzma_vli_size(lzma_vli vli)
+ lzma_attr_pure;
#include "check.h"
-// See the .lzma header format specification section 2.1.1.2.
-LZMA_API const uint32_t lzma_check_sizes[LZMA_CHECK_ID_MAX + 1] = {
- 0,
- 4, 4, 4,
- 8, 8, 8,
- 16, 16, 16,
- 32, 32, 32,
- 64, 64, 64
-};
+extern LZMA_API lzma_bool
+lzma_check_is_supported(lzma_check type)
+{
+ if ((unsigned)(type) > LZMA_CHECK_ID_MAX)
+ return false;
-LZMA_API const lzma_bool lzma_available_checks[LZMA_CHECK_ID_MAX + 1] = {
- true, // LZMA_CHECK_NONE
+ static const lzma_bool available_checks[LZMA_CHECK_ID_MAX + 1] = {
+ true, // LZMA_CHECK_NONE
#ifdef HAVE_CHECK_CRC32
- true,
+ true,
#else
- false,
+ false,
#endif
- false, // Reserved
- false, // Reserved
+ false, // Reserved
+ false, // Reserved
#ifdef HAVE_CHECK_CRC64
- true,
+ true,
#else
- false,
+ false,
#endif
- false, // Reserved
- false, // Reserved
- false, // Reserved
- false, // Reserved
- false, // Reserved
+ false, // Reserved
+ false, // Reserved
+ false, // Reserved
+ false, // Reserved
+ false, // Reserved
#ifdef HAVE_CHECK_SHA256
- true,
+ true,
#else
- false,
+ false,
#endif
- false, // Reserved
- false, // Reserved
- false, // Reserved
- false, // Reserved
- false, // Reserved
-};
+ false, // Reserved
+ false, // Reserved
+ false, // Reserved
+ false, // Reserved
+ false, // Reserved
+ };
+
+ return available_checks[(unsigned)(type)];
+}
-extern lzma_ret
-lzma_check_init(lzma_check *check, lzma_check_type type)
+extern LZMA_API uint32_t
+lzma_check_size(lzma_check type)
{
- lzma_ret ret = LZMA_OK;
+ if ((unsigned)(type) > LZMA_CHECK_ID_MAX)
+ return UINT32_MAX;
+
+ // See file-format.txt section 2.1.1.2.
+ static const uint8_t check_sizes[LZMA_CHECK_ID_MAX + 1] = {
+ 0,
+ 4, 4, 4,
+ 8, 8, 8,
+ 16, 16, 16,
+ 32, 32, 32,
+ 64, 64, 64
+ };
+
+ return check_sizes[(unsigned)(type)];
+}
+
+extern void
+lzma_check_init(lzma_check_state *check, lzma_check type)
+{
switch (type) {
case LZMA_CHECK_NONE:
break;
#endif
default:
- if ((unsigned)(type) <= LZMA_CHECK_ID_MAX)
- ret = LZMA_UNSUPPORTED_CHECK;
- else
- ret = LZMA_PROG_ERROR;
break;
}
- return ret;
+ return;
}
extern void
-lzma_check_update(lzma_check *check, lzma_check_type type,
+lzma_check_update(lzma_check_state *check, lzma_check type,
const uint8_t *buf, size_t size)
{
switch (type) {
extern void
-lzma_check_finish(lzma_check *check, lzma_check_type type)
+lzma_check_finish(lzma_check_state *check, lzma_check type)
{
switch (type) {
#ifdef HAVE_CHECK_CRC32
case LZMA_CHECK_CRC32:
- *(uint32_t *)(check->buffer) = check->state.crc32;
+ check->buffer.u32[0] = integer_le_32(check->state.crc32);
break;
#endif
#ifdef HAVE_CHECK_CRC64
case LZMA_CHECK_CRC64:
- *(uint64_t *)(check->buffer) = check->state.crc64;
+ check->buffer.u64[0] = integer_le_64(check->state.crc64);
break;
#endif
return;
}
-
-
-/*
-extern bool
-lzma_check_compare(
- lzma_check *check1, lzma_check *check2, lzma_check_type type)
-{
- bool ret;
-
- switch (type) {
- case LZMA_CHECK_NONE:
- break;
-
- case LZMA_CHECK_CRC32:
- ret = check1->crc32 != check2->crc32;
- break;
-
- case LZMA_CHECK_CRC64:
- ret = check1->crc64 != check2->crc64;
- break;
-
- default:
- // Unsupported check
- assert(type <= 7);
- ret = false;
- break;
- }
-
- return ret;
-}
-*/
///////////////////////////////////////////////////////////////////////////////
//
/// \file check.h
-/// \brief Prototypes for different check functions
+/// \brief Internal API to different integrity check functions
//
// This code has been put into the public domain.
//
#include "common.h"
-// Index hashing used to verify the Index with O(1) memory usage needs
-// a good hash function.
+// Index hashing needs the best possible hash function (preferably
+// a cryptographic hash) for maximum reliability.
#if defined(HAVE_CHECK_SHA256)
# define LZMA_CHECK_BEST LZMA_CHECK_SHA256
#elif defined(HAVE_CHECK_CRC64)
#endif
+/// \brief Structure to hold internal state of the check being calculated
+///
+/// \note This is not in the public API because this structure may
+/// change in future if new integrity check algorithms are added.
typedef struct {
- /// Internal state
- uint32_t state[8];
-
- /// Size of the message excluding padding
- uint64_t size;
-
-} lzma_sha256;
-
-
-/// \note This is not in the public API because this structure will
-/// change in future.
-typedef struct {
- // FIXME Guarantee 8-byte alignment
-
- /// Buffer to hold the final result; this is also used as a temporary
- /// buffer in SHA256. Note that this buffer must be 8-byte aligned.
- uint8_t buffer[64];
+ /// Buffer to hold the final result and a temporary buffer for SHA256.
+ union {
+ uint8_t u8[64];
+ uint32_t u32[16];
+ uint64_t u64[8];
+ } buffer;
/// Check-specific data
union {
} sha256;
} state;
-} lzma_check;
+} lzma_check_state;
#ifdef HAVE_SMALL
extern const uint64_t lzma_crc64_table[4][256];
#endif
-// Generic
/// \brief Initializes *check depending on type
///
/// supported by the current version or build of liblzma.
/// LZMA_PROG_ERROR if type > LZMA_CHECK_ID_MAX.
///
-extern lzma_ret lzma_check_init(lzma_check *check, lzma_check_type type);
+extern void lzma_check_init(lzma_check_state *check, lzma_check type);
+
/// \brief Updates *check
///
-extern void lzma_check_update(lzma_check *check, lzma_check_type type,
+extern void lzma_check_update(lzma_check_state *check, lzma_check type,
const uint8_t *buf, size_t size);
-/// \brief Finishes *check
-///
-extern void lzma_check_finish(lzma_check *check, lzma_check_type type);
-
-/*
-/// \brief Compare two checks
-///
-/// \return false if the checks are identical; true if they differ.
+/// \brief Finishes *check
///
-extern bool lzma_check_compare(
- lzma_check *check1, lzma_check *check2, lzma_check_type type);
-*/
+extern void lzma_check_finish(lzma_check_state *check, lzma_check type);
-// CRC32
-
extern void lzma_crc32_init(void);
-// CRC64
-
extern void lzma_crc64_init(void);
-// SHA256
-
-extern void lzma_sha256_init(lzma_check *check);
+extern void lzma_sha256_init(lzma_check_state *check);
extern void lzma_sha256_update(
- const uint8_t *buf, size_t size, lzma_check *check);
-
-extern void lzma_sha256_finish(lzma_check *check);
+ const uint8_t *buf, size_t size, lzma_check_state *check);
+extern void lzma_sha256_finish(lzma_check_state *check);
#endif
static void
-process(lzma_check *check)
+process(lzma_check_state *check)
{
#ifdef WORDS_BIGENDIAN
- transform(check->state.sha256.state, (uint32_t *)(check->buffer));
+ transform(check->state.sha256.state, check->buffer.u32);
#else
uint32_t data[16];
for (size_t i = 0; i < 16; ++i)
- data[i] = bswap_32(*((uint32_t*)(check->buffer) + i));
+ data[i] = bswap_32(check->buffer.u32[i]);
transform(check->state.sha256.state, data);
#endif
extern void
-lzma_sha256_init(lzma_check *check)
+lzma_sha256_init(lzma_check_state *check)
{
static const uint32_t s[8] = {
0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A,
extern void
-lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check *check)
+lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check)
{
// Copy the input data into a properly aligned temporary buffer.
// This way we can be called with arbitrarily sized buffers
if (copy_size > size)
copy_size = size;
- memcpy(check->buffer + copy_start, buf, copy_size);
+ memcpy(check->buffer.u8 + copy_start, buf, copy_size);
buf += copy_size;
size -= copy_size;
extern void
-lzma_sha256_finish(lzma_check *check)
+lzma_sha256_finish(lzma_check_state *check)
{
// Add padding as described in RFC 3174 (it describes SHA-1 but
// the same padding style is used for SHA-256 too).
size_t pos = check->state.sha256.size & 0x3F;
- check->buffer[pos++] = 0x80;
+ check->buffer.u8[pos++] = 0x80;
while (pos != 64 - 8) {
if (pos == 64) {
pos = 0;
}
- check->buffer[pos++] = 0x00;
+ check->buffer.u8[pos++] = 0x00;
}
// Convert the message size from bytes to bits.
check->state.sha256.size *= 8;
#ifdef WORDS_BIGENDIAN
- *(uint64_t *)(check->buffer + 64 - 8) = check->state.sha256.size;
+ check->buffer.u64[(64 - 8) / 8] = check->state.sha256.size;
#else
- *(uint64_t *)(check->buffer + 64 - 8)
- = bswap_64(check->state.sha256.size);
+ check->buffer.u64[(64 - 8) / 8] = bswap_64(check->state.sha256.size);
#endif
process(check);
for (size_t i = 0; i < 8; ++i)
#ifdef WORDS_BIGENDIAN
- ((uint32_t *)(check->buffer))[i]
- = check->state.sha256.state[i];
+ check->buffer.u32[i] = check->state.sha256.state[i];
#else
- ((uint32_t *)(check->buffer))[i]
- = bswap_32(check->state.sha256.state[i]);
+ check->buffer.u32[i] = bswap_32(check->state.sha256.state[i]);
#endif
return;
libcommon_la_CPPFLAGS = \
-I@top_srcdir@/src/liblzma/api \
-I@top_srcdir@/src/liblzma/check \
+ -I@top_srcdir@/src/liblzma/rangecoder \
-I@top_srcdir@/src/liblzma/lz \
-I@top_srcdir@/src/liblzma/lzma \
- -I@top_srcdir@/src/liblzma/simple \
-I@top_srcdir@/src/liblzma/subblock \
- -I@top_srcdir@/src/liblzma/rangecoder
+ -I@top_srcdir@/src/liblzma/delta \
+ -I@top_srcdir@/src/liblzma/simple
+
libcommon_la_SOURCES = \
+ common.c \
common.h \
bsr.h \
- allocator.c \
block_util.c \
block_private.h \
- features.c \
+ filter_common.c \
+ filter_common.h \
index.c \
+ index.h \
init.c \
memory_limiter.c \
- memory_usage.c \
- next_coder.c \
- raw_common.c \
- raw_common.h \
- stream_flags_equal.c \
- code.c \
- version.c
-
-if COND_FILTER_DELTA
-libcommon_la_SOURCES += \
- delta_common.c \
- delta_common.h
-if COND_MAIN_ENCODER
-libcommon_la_SOURCES += \
- delta_encoder.c \
- delta_encoder.h
-endif
-if COND_MAIN_DECODER
-libcommon_la_SOURCES += \
- delta_decoder.c \
- delta_decoder.h
-endif
-endif
+ stream_flags_common.c \
+ stream_flags_common.h \
+ vli_size.c
if COND_MAIN_ENCODER
libcommon_la_SOURCES += \
alignment.c \
- auto_decoder.c \
alone_encoder.c \
block_encoder.c \
block_encoder.h \
block_header_encoder.c \
easy.c \
+ filter_encoder.c \
+ filter_encoder.h \
filter_flags_encoder.c \
index_encoder.c \
index_encoder.h \
init_encoder.c \
- raw_encoder.c \
- raw_encoder.h \
- stream_common.c \
- stream_common.h \
stream_encoder.c \
stream_encoder.h \
stream_flags_encoder.c \
libcommon_la_SOURCES += \
alone_decoder.c \
alone_decoder.h \
+ auto_decoder.c \
block_decoder.c \
block_decoder.h \
block_header_decoder.c \
+ filter_decoder.c \
+ filter_decoder.h \
filter_flags_decoder.c \
index_decoder.c \
index_hash.c \
init_decoder.c \
- raw_decoder.c \
- raw_decoder.h \
stream_decoder.c \
+ stream_decoder.h \
stream_flags_decoder.c \
stream_flags_decoder.h \
vli_decoder.c
extern LZMA_API uint32_t
-lzma_alignment_input(const lzma_options_filter *filters, uint32_t guess)
+lzma_alignment_input(const lzma_filter *filters, uint32_t guess)
{
for (size_t i = 0; filters[i].id != LZMA_VLI_VALUE_UNKNOWN; ++i) {
switch (filters[i].id) {
extern LZMA_API uint32_t
-lzma_alignment_output(const lzma_options_filter *filters, uint32_t guess)
+lzma_alignment_output(const lzma_filter *filters, uint32_t guess)
{
if (filters[0].id == LZMA_VLI_VALUE_UNKNOWN)
return UINT32_MAX;
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file allocator.c
-/// \brief Allocating and freeing memory
-//
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "common.h"
-
-#undef lzma_free
-
-extern void * lzma_attribute((malloc))
-lzma_alloc(size_t size, lzma_allocator *allocator)
-{
- // Some malloc() variants return NULL if called with size == 0.
- if (size == 0)
- size = 1;
-
- void *ptr;
-
- if (allocator != NULL && allocator->alloc != NULL)
- ptr = allocator->alloc(allocator->opaque, 1, size);
- else
- ptr = malloc(size);
-
-#ifndef NDEBUG
- // This helps to catch some stupid mistakes, but also hides them from
- // Valgrind. Uncomment when useful.
-// if (ptr != NULL)
-// memset(ptr, 0xFD, size);
-#endif
-
- return ptr;
-}
-
-
-extern void
-lzma_free(void *ptr, lzma_allocator *allocator)
-{
- if (allocator != NULL && allocator->free != NULL)
- allocator->free(allocator->opaque, ptr);
- else
- free(ptr);
-
- return;
-}
#include "alone_decoder.h"
#include "lzma_decoder.h"
+#include "lz_decoder.h"
struct lzma_coder_s {
/// Uncompressed size decoded from the header
lzma_vli uncompressed_size;
+ /// Memory usage limit
+ uint64_t memlimit;
+
/// Options decoded from the header needed to initialize
/// the LZMA decoder
lzma_options_lzma options;
&& (coder->sequence == SEQ_CODE || *in_pos < in_size))
switch (coder->sequence) {
case SEQ_PROPERTIES:
- if (lzma_lzma_decode_properties(&coder->options, in[*in_pos]))
+ if (lzma_lzma_lclppb_decode(&coder->options, in[*in_pos]))
return LZMA_FORMAT_ERROR;
coder->sequence = SEQ_DICTIONARY_SIZE;
if (++coder->pos == 4) {
if (coder->options.dictionary_size
- < LZMA_DICTIONARY_SIZE_MIN
- || coder->options.dictionary_size
> LZMA_DICTIONARY_SIZE_MAX)
return LZMA_FORMAT_ERROR;
break;
case SEQ_CODER_INIT: {
- // Two is enough because there won't be implicit filters.
+ // FIXME It is unfair that this doesn't add a fixed amount
+ // like lzma_memusage_common() does.
+ const uint64_t memusage
+ = lzma_lzma_decoder_memusage(&coder->options);
+
+ // Use LZMA_PROG_ERROR since LZMA_Alone decoder cannot be
+ // built without LZMA support.
+ // FIXME TODO Make the above comment true.
+ if (memusage == UINT64_MAX)
+ return LZMA_PROG_ERROR;
+
+ if (memusage > coder->memlimit)
+ return LZMA_MEMLIMIT_ERROR;
+
lzma_filter_info filters[2] = {
{
.init = &lzma_lzma_decoder_init,
return ret;
// Use a hack to set the uncompressed size.
- lzma_lzma_decoder_uncompressed_size(&coder->next,
+ lzma_lz_decoder_uncompressed(coder->next.coder,
coder->uncompressed_size);
coder->sequence = SEQ_CODE;
static void
alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->next, allocator);
+ lzma_next_end(&coder->next, allocator);
lzma_free(coder, allocator);
return;
}
-static lzma_ret
-alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator)
+extern lzma_ret
+lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ uint64_t memlimit)
{
+ lzma_next_coder_init(lzma_alone_decoder_init, next, allocator);
+
if (next->coder == NULL) {
next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
if (next->coder == NULL)
next->coder->pos = 0;
next->coder->options.dictionary_size = 0;
next->coder->uncompressed_size = 0;
+ next->coder->memlimit = memlimit;
return LZMA_OK;
}
-extern lzma_ret
-lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator)
-{
- lzma_next_coder_init0(alone_decoder_init, next, allocator);
-}
-
-
extern LZMA_API lzma_ret
-lzma_alone_decoder(lzma_stream *strm)
+lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit)
{
- lzma_next_strm_init0(strm, alone_decoder_init);
+ lzma_next_strm_init(lzma_alone_decoder_init, strm, memlimit);
strm->internal->supported_actions[LZMA_RUN] = true;
- strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
+// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; FIXME
+ strm->internal->supported_actions[LZMA_FINISH] = true;
return LZMA_OK;
}
//
///////////////////////////////////////////////////////////////////////////////
+#ifndef LZMA_ALONE_DECODER_H
+#define LZMA_ALONE_DECODER_H
+
#include "common.h"
-extern lzma_ret lzma_alone_decoder_init(
- lzma_next_coder *next, lzma_allocator *allocator);
+extern lzma_ret lzma_alone_decoder_init(lzma_next_coder *next,
+ lzma_allocator *allocator, uint64_t memlimit);
+
+#endif
while (*out_pos < out_size)
switch (coder->sequence) {
case SEQ_HEADER:
- bufcpy(coder->header, &coder->header_pos,
+ lzma_bufcpy(coder->header, &coder->header_pos,
ALONE_HEADER_SIZE,
out, out_pos, out_size);
if (coder->header_pos < ALONE_HEADER_SIZE)
static void
alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->next, allocator);
+ lzma_next_end(&coder->next, allocator);
lzma_free(coder, allocator);
return;
}
alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
const lzma_options_lzma *options)
{
+ lzma_next_coder_init(alone_encoder_init, next, allocator);
+
if (next->coder == NULL) {
next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
if (next->coder == NULL)
// Encode the header:
// - Properties (1 byte)
- if (lzma_lzma_encode_properties(options, next->coder->header))
+ if (lzma_lzma_lclppb_encode(options, next->coder->header))
return LZMA_PROG_ERROR;
// - Dictionary size (4 bytes)
// one is the next. While the header would allow any 32-bit integer,
// we do this to keep the decoder of liblzma accepting the resulting
// files.
+ //
+ // FIXME Maybe LZMA_Alone needs some lower limit for maximum
+ // dictionary size? Must check decoders from old LZMA SDK version.
uint32_t d = options->dictionary_size - 1;
d |= d >> 2;
d |= d >> 3;
extern LZMA_API lzma_ret
lzma_alone_encoder(lzma_stream *strm, const lzma_options_lzma *options)
{
- lzma_next_strm_init(strm, alone_encoder_init, options);
+ lzma_next_strm_init(alone_encoder_init, strm, options);
strm->internal->supported_actions[LZMA_RUN] = true;
strm->internal->supported_actions[LZMA_FINISH] = true;
struct lzma_coder_s {
lzma_next_coder next;
+ uint64_t memlimit;
+ uint32_t flags;
bool initialized;
};
if (in[*in_pos] == 0xFF)
ret = lzma_stream_decoder_init(
- &coder->next, allocator);
+ &coder->next, allocator,
+ coder->memlimit, coder->flags);
else
- ret = lzma_alone_decoder_init(&coder->next, allocator);
+ ret = lzma_alone_decoder_init(&coder->next,
+ allocator, coder->memlimit);
if (ret != LZMA_OK)
return ret;
static void
auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->next, allocator);
+ lzma_next_end(&coder->next, allocator);
lzma_free(coder, allocator);
return;
}
static lzma_ret
-auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator)
+auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ uint64_t memlimit, uint32_t flags)
{
+ lzma_next_coder_init(auto_decoder_init, next, allocator);
+
+ if (flags & ~LZMA_SUPPORTED_FLAGS)
+ return LZMA_HEADER_ERROR;
+
if (next->coder == NULL) {
next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
if (next->coder == NULL)
next->coder->next = LZMA_NEXT_CODER_INIT;
}
+ next->coder->memlimit = memlimit;
+ next->coder->flags = flags;
next->coder->initialized = false;
return LZMA_OK;
}
-/*
-extern lzma_ret
-lzma_auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- lzma_extra **header, lzma_extra **footer)
-{
- lzma_next_coder_init(
- auto_decoder_init, next, allocator, header, footer);
-}
-*/
-
-
extern LZMA_API lzma_ret
-lzma_auto_decoder(lzma_stream *strm)
+lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags)
{
- lzma_next_strm_init0(strm, auto_decoder_init);
+ lzma_next_strm_init(auto_decoder_init, strm, memlimit, flags);
strm->internal->supported_actions[LZMA_RUN] = true;
- strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
+// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; FIXME
+ strm->internal->supported_actions[LZMA_FINISH] = true;
return LZMA_OK;
}
#include "block_decoder.h"
#include "block_private.h"
-#include "raw_decoder.h"
+#include "filter_decoder.h"
#include "check.h"
/// Decoding options; we also write Compressed Size and Uncompressed
/// Size back to this structure when the encoding has been finished.
- lzma_options_block *options;
+ lzma_block *options;
/// Compressed Size calculated while encoding
lzma_vli compressed_size;
size_t check_pos;
/// Check of the uncompressed data
- lzma_check check;
+ lzma_check_state check;
};
{
switch (coder->sequence) {
case SEQ_CODE: {
- if (*out_pos >= out_size)
- return LZMA_OK;
-
const size_t in_start = *in_pos;
const size_t out_start = *out_pos;
// Fall through
case SEQ_PADDING:
- // If Compressed Data is padded to a multiple of four bytes.
+ // Compressed Data is padded to a multiple of four bytes.
while (coder->compressed_size & 3) {
if (*in_pos >= in_size)
return LZMA_OK;
// Fall through
- case SEQ_CHECK:
+ case SEQ_CHECK: {
+ const bool chksup = lzma_check_is_supported(
+ coder->options->check);
+
while (*in_pos < in_size) {
- if (in[(*in_pos)++] != coder->check.buffer[
- coder->check_pos])
+ // coder->check.buffer[] may be uninitialized when
+ // the Check ID is not supported.
+ if (chksup && coder->check.buffer.u8[coder->check_pos]
+ != in[*in_pos]) {
+ ++*in_pos;
return LZMA_DATA_ERROR;
+ }
- if (++coder->check_pos == lzma_check_sizes[
- coder->options->check])
+ ++*in_pos;
+
+ if (++coder->check_pos == lzma_check_size(
+ coder->options->check))
return LZMA_STREAM_END;
}
return LZMA_OK;
}
+ }
return LZMA_PROG_ERROR;
}
static void
block_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->next, allocator);
+ lzma_next_end(&coder->next, allocator);
lzma_free(coder, allocator);
return;
}
-static lzma_ret
-block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- lzma_options_block *options)
+extern lzma_ret
+lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ lzma_block *options)
{
+ lzma_next_coder_init(lzma_block_decoder_init, next, allocator);
+
// While lzma_block_total_size_get() is meant to calculate the Total
// Size, it also validates the options excluding the filters.
if (lzma_block_total_size_get(options) == 0)
return LZMA_PROG_ERROR;
+ // options->check is used for array indexing so we need to know that
+ // it is in the valid range.
+ if ((unsigned)(options->check) > LZMA_CHECK_ID_MAX)
+ return LZMA_PROG_ERROR;
+
// Allocate and initialize *next->coder if needed.
if (next->coder == NULL) {
next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
= options->compressed_size == LZMA_VLI_VALUE_UNKNOWN
? (LZMA_VLI_VALUE_MAX & ~LZMA_VLI_C(3))
- options->header_size
- - lzma_check_sizes[options->check]
+ - lzma_check_size(options->check)
: options->compressed_size;
- // Initialize the check
+ // Initialize the check. It's caller's problem if the Check ID is not
+ // supported, and the Block decoder cannot verify the Check field.
+ // Caller can test lzma_checks[options->check].
next->coder->check_pos = 0;
- return_if_error(lzma_check_init(&next->coder->check, options->check));
+ lzma_check_init(&next->coder->check, options->check);
+ // Initialize the filter chain.
return lzma_raw_decoder_init(&next->coder->next, allocator,
options->filters);
}
-extern lzma_ret
-lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- lzma_options_block *options)
-{
- lzma_next_coder_init(block_decoder_init, next, allocator, options);
-}
-
-
extern LZMA_API lzma_ret
-lzma_block_decoder(lzma_stream *strm, lzma_options_block *options)
+lzma_block_decoder(lzma_stream *strm, lzma_block *options)
{
- lzma_next_strm_init(strm, block_decoder_init, options);
+ lzma_next_strm_init(lzma_block_decoder_init, strm, options);
strm->internal->supported_actions[LZMA_RUN] = true;
strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
extern lzma_ret lzma_block_decoder_init(lzma_next_coder *next,
- lzma_allocator *allocator, lzma_options_block *options);
+ lzma_allocator *allocator, lzma_block *options);
#endif
#include "block_encoder.h"
#include "block_private.h"
-#include "raw_encoder.h"
+#include "filter_encoder.h"
#include "check.h"
/// Encoding options; we also write Total Size, Compressed Size, and
/// Uncompressed Size back to this structure when the encoding has
/// been finished.
- lzma_options_block *options;
+ lzma_block *options;
enum {
SEQ_CODE,
size_t check_pos;
/// Check of the uncompressed data
- lzma_check check;
+ lzma_check_state check;
};
// Fall through
case SEQ_CHECK:
- out[*out_pos] = coder->check.buffer[coder->check_pos];
+ out[*out_pos] = coder->check.buffer.u8[coder->check_pos];
++*out_pos;
if (++coder->check_pos
- == lzma_check_sizes[coder->options->check])
+ == lzma_check_size(coder->options->check))
return LZMA_STREAM_END;
break;
static void
block_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->next, allocator);
+ lzma_next_end(&coder->next, allocator);
lzma_free(coder, allocator);
return;
}
-static lzma_ret
-block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- lzma_options_block *options)
+extern lzma_ret
+lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ lzma_block *options)
{
+ lzma_next_coder_init(lzma_block_encoder_init, next, allocator);
+
// While lzma_block_total_size_get() is meant to calculate the Total
// Size, it also validates the options excluding the filters.
if (lzma_block_total_size_get(options) == 0)
return LZMA_PROG_ERROR;
+ // If the Check ID is not supported, we cannot calculate the check and
+ // thus not create a proper Block.
+ if ((unsigned)(options->check) > LZMA_CHECK_ID_MAX)
+ return LZMA_PROG_ERROR;
+
+ if (!lzma_check_is_supported(options->check))
+ return LZMA_UNSUPPORTED_CHECK;
+
// Allocate and initialize *next->coder if needed.
if (next->coder == NULL) {
next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
// Initialize the check
next->coder->check_pos = 0;
- return_if_error(lzma_check_init(&next->coder->check, options->check));
+ lzma_check_init(&next->coder->check, options->check);
// Initialize the requested filters.
return lzma_raw_encoder_init(&next->coder->next, allocator,
}
-extern lzma_ret
-lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- lzma_options_block *options)
-{
- lzma_next_coder_init(block_encoder_init, next, allocator, options);
-}
-
-
extern LZMA_API lzma_ret
-lzma_block_encoder(lzma_stream *strm, lzma_options_block *options)
+lzma_block_encoder(lzma_stream *strm, lzma_block *options)
{
- lzma_next_strm_init(strm, block_encoder_init, options);
+ lzma_next_strm_init(lzma_block_encoder_init, strm, options);
strm->internal->supported_actions[LZMA_RUN] = true;
strm->internal->supported_actions[LZMA_FINISH] = true;
extern lzma_ret lzma_block_encoder_init(lzma_next_coder *next,
- lzma_allocator *allocator, lzma_options_block *options);
+ lzma_allocator *allocator, lzma_block *options);
#endif
static void
-free_properties(lzma_options_block *options, lzma_allocator *allocator)
+free_properties(lzma_block *options, lzma_allocator *allocator)
{
// Free allocated filter options. The last array member is not
// touched after the initialization in the beginning of
extern LZMA_API lzma_ret
-lzma_block_header_decode(lzma_options_block *options,
+lzma_block_header_decode(lzma_block *options,
lzma_allocator *allocator, const uint8_t *in)
{
// NOTE: We consider the header to be corrupt not only when the
// CRC32 doesn't match, but also when variable-length integers
- // are invalid or not over 63 bits, or if the header is too small
+ // are invalid or over 63 bits, or if the header is too small
// to contain the claimed information.
// Initialize the filter options array. This way the caller can
extern LZMA_API lzma_ret
-lzma_block_header_size(lzma_options_block *options)
+lzma_block_header_size(lzma_block *options)
{
// Block Header Size + Block Flags + CRC32.
size_t size = 1 + 1 + 4;
extern LZMA_API lzma_ret
-lzma_block_header_encode(const lzma_options_block *options, uint8_t *out)
+lzma_block_header_encode(const lzma_block *options, uint8_t *out)
{
if ((options->header_size & 3)
|| options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN
if (filter_count == 4)
return LZMA_PROG_ERROR;
- return_if_error(lzma_filter_flags_encode(out, &out_pos,
- out_size, options->filters + filter_count));
+ return_if_error(lzma_filter_flags_encode(
+ options->filters + filter_count,
+ out, &out_pos, out_size));
} while (options->filters[++filter_count].id
!= LZMA_VLI_VALUE_UNKNOWN);
///////////////////////////////////////////////////////////////////////////////
//
/// \file block_header.c
-/// \brief Utility functions to handle lzma_options_block
+/// \brief Utility functions to handle lzma_block
//
// Copyright (C) 2008 Lasse Collin
//
extern LZMA_API lzma_ret
-lzma_block_total_size_set(lzma_options_block *options, lzma_vli total_size)
+lzma_block_total_size_set(lzma_block *options, lzma_vli total_size)
{
// Validate.
if (options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN
return LZMA_PROG_ERROR;
const uint32_t container_size = options->header_size
- + lzma_check_sizes[options->check];
+ + lzma_check_size(options->check);
// Validate that Compressed Size will be greater than zero.
if (container_size <= total_size)
extern LZMA_API lzma_vli
-lzma_block_total_size_get(const lzma_options_block *options)
+lzma_block_total_size_get(const lzma_block *options)
{
// Validate the values that we are interested in.
if (options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN
const lzma_vli total_size = options->compressed_size
+ options->header_size
- + lzma_check_sizes[options->check];
+ + lzma_check_size(options->check);
// Validate the calculated Total Size.
if (options->compressed_size > LZMA_VLI_VALUE_MAX
///////////////////////////////////////////////////////////////////////////////
//
-/// \file code.c
-/// \brief zlib-like API wrapper for liblzma's internal API
+/// \file common.h
+/// \brief Common functions needed in many places in liblzma
//
-// Copyright (C) 2007 Lasse Collin
+// Copyright (C) 2007-2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
#include "common.h"
-LZMA_API const lzma_stream LZMA_STREAM_INIT_VAR = {
- .next_in = NULL,
- .avail_in = 0,
- .total_in = 0,
- .next_out = NULL,
- .avail_out = 0,
- .total_out = 0,
- .allocator = NULL,
- .internal = NULL,
-};
+/////////////
+// Version //
+/////////////
+
+extern LZMA_API uint32_t
+lzma_version_number(void)
+{
+ return LZMA_VERSION;
+}
+
+
+extern LZMA_API const char *
+lzma_version_string(void)
+{
+ return PACKAGE_VERSION;
+}
+
+
+///////////////////////
+// Memory allocation //
+///////////////////////
+
+extern void * lzma_attribute((malloc))
+lzma_alloc(size_t size, lzma_allocator *allocator)
+{
+ // Some malloc() variants return NULL if called with size == 0.
+ if (size == 0)
+ size = 1;
+
+ void *ptr;
+
+ if (allocator != NULL && allocator->alloc != NULL)
+ ptr = allocator->alloc(allocator->opaque, 1, size);
+ else
+ ptr = malloc(size);
+
+ return ptr;
+}
+
+
+extern void
+lzma_free(void *ptr, lzma_allocator *allocator)
+{
+ if (allocator != NULL && allocator->free != NULL)
+ allocator->free(allocator->opaque, ptr);
+ else
+ free(ptr);
+
+ return;
+}
+
+
+//////////
+// Misc //
+//////////
+
+extern size_t
+lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos,
+ size_t in_size, uint8_t *restrict out,
+ size_t *restrict out_pos, size_t out_size)
+{
+ const size_t in_avail = in_size - *in_pos;
+ const size_t out_avail = out_size - *out_pos;
+ const size_t copy_size = MIN(in_avail, out_avail);
+
+ memcpy(out + *out_pos, in + *in_pos, copy_size);
+
+ *in_pos += copy_size;
+ *out_pos += copy_size;
+
+ return copy_size;
+}
extern lzma_ret
+lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter_info *filters)
+{
+ lzma_next_coder_init(filters[0].init, next, allocator);
+
+ return filters[0].init == NULL
+ ? LZMA_OK : filters[0].init(next, allocator, filters);
+}
+
+
+extern void
+lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator)
+{
+ if (next->init != (uintptr_t)(NULL)) {
+ // To avoid tiny end functions that simply call
+ // lzma_free(coder, allocator), we allow leaving next->end
+ // NULL and call lzma_free() here.
+ if (next->end != NULL)
+ next->end(next->coder, allocator);
+ else
+ lzma_free(next->coder, allocator);
+
+ // Reset the variables so the we don't accidentally think
+ // that it is an already initialized coder.
+ *next = LZMA_NEXT_CODER_INIT;
+ }
+
+ return;
+}
+
+
+//////////////////////////////////////
+// External to internal API wrapper //
+//////////////////////////////////////
+
+extern lzma_ret
lzma_strm_init(lzma_stream *strm)
{
if (strm == NULL)
lzma_end(lzma_stream *strm)
{
if (strm != NULL && strm->internal != NULL) {
- if (strm->internal->next.end != NULL)
- strm->internal->next.end(strm->internal->next.coder,
- strm->allocator);
-
+ lzma_next_end(&strm->internal->next, strm->allocator);
lzma_free(strm->internal, strm->allocator);
strm->internal = NULL;
}
/// \file common.h
/// \brief Definitions common to the whole liblzma library
//
-// Copyright (C) 2007 Lasse Collin
+// Copyright (C) 2007-2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
#define LZMA_BUFFER_SIZE 4096
+/// Start of internal Filter ID space. These IDs must never be used
+/// in Streams.
+#define LZMA_FILTER_RESERVED_START (LZMA_VLI_C(1) << 62)
+
+
/// Internal helper filter used by Subblock decoder. It is mapped to an
/// otherwise invalid Filter ID, which is impossible to get from any input
/// file (even if malicious file).
-#define LZMA_FILTER_SUBBLOCK_HELPER (UINT64_MAX - 2)
+#define LZMA_FILTER_SUBBLOCK_HELPER (LZMA_FILTER_RESERVED_START + 1)
+
+
+/// Supported flags that can be passed to lzma_stream_decoder()
+/// or lzma_auto_decoder().
+#define LZMA_SUPPORTED_FLAGS \
+ (LZMA_WARN_NO_CHECK \
+ | LZMA_WARN_UNSUPPORTED_CHECK \
+ | LZMA_TELL_CHECK \
+ | LZMA_CONCATENATED)
///////////
// Types //
///////////
+/// Type of encoder/decoder specific data; the actual structure is defined
+/// differently in different coders.
typedef struct lzma_coder_s lzma_coder;
typedef struct lzma_next_coder_s lzma_next_coder;
typedef struct lzma_filter_info_s lzma_filter_info;
+/// Type of a function used to initialize a filter encoder or decoder
typedef lzma_ret (*lzma_init_function)(
lzma_next_coder *next, lzma_allocator *allocator,
const lzma_filter_info *filters);
+/// Type of a function to do some kind of coding work (filters, Stream,
+/// Block encoders/decoders etc.). Some special coders use don't use both
+/// input and output buffers, but for simplicity they still use this same
+/// function prototype.
typedef lzma_ret (*lzma_code_function)(
lzma_coder *coder, lzma_allocator *allocator,
const uint8_t *restrict in, size_t *restrict in_pos,
size_t *restrict out_pos, size_t out_size,
lzma_action action);
+/// Type of a function to free the memory allocated for the coder
typedef void (*lzma_end_function)(
lzma_coder *coder, lzma_allocator *allocator);
+/// Raw coder validates and converts an array of lzma_filter structures to
+/// an array of lzma_filter_info structures. This array is used with
+/// lzma_next_filter_init to initialize the filter chain.
+struct lzma_filter_info_s {
+ /// Pointer to function used to initialize the filter.
+ /// This is NULL to indicate end of array.
+ lzma_init_function init;
+
+ /// Pointer to filter's options structure
+ void *options;
+};
+
+
/// Hold data and function pointers of the next filter in the chain.
struct lzma_next_coder_s {
/// Pointer to coder-specific data
/// "Pointer" to init function. This is never called here.
/// We need only to detect if we are initializing a coder
- /// that was allocated earlier. See code.c and next_coder.c.
+ /// that was allocated earlier. See lzma_next_coder_init and
+ /// lzma_next_strm_init macros in this file.
uintptr_t init;
/// Pointer to function to do the actual coding
lzma_code_function code;
- /// Pointer to function to free lzma_next_coder.coder
+ /// Pointer to function to free lzma_next_coder.coder. This can
+ /// be NULL; in that case, lzma_free is called to free
+ /// lzma_next_coder.coder.
lzma_end_function end;
+
+ /// Pointer to function to return the type of the integrity check.
+ /// Most coders won't support this.
+ lzma_check (*see_check)(const lzma_coder *coder);
+
+// uint64_t (*memconfig)(
+// lzma_coder *coder, uint64_t memlimit, bool change);
};
+
+/// Macro to initialize lzma_next_coder structure
#define LZMA_NEXT_CODER_INIT \
(lzma_next_coder){ \
.coder = NULL, \
- .init = 0, \
+ .init = (uintptr_t)(NULL), \
.code = NULL, \
.end = NULL, \
+ .see_check = NULL, \
}
+/// Internal data for lzma_strm_init, lzma_code, and lzma_end. A pointer to
+/// this is stored in lzma_stream.
struct lzma_internal_s {
+ /// The actual coder that should do something useful
lzma_next_coder next;
+ /// Track the state of the coder. This is used to validate arguments
+ /// so that the actual coders can rely on e.g. that LZMA_SYNC_FLUSH
+ /// is used on every call to lzma_code until next.code has returned
+ /// LZMA_STREAM_END.
enum {
ISEQ_RUN,
ISEQ_SYNC_FLUSH,
ISEQ_ERROR,
} sequence;
- bool supported_actions[4];
- bool allow_buf_error;
+ /// A copy of lzma_stream avail_in. This is used to verify that the
+ /// amount of input doesn't change once e.g. LZMA_FINISH has been
+ /// used.
size_t avail_in;
-};
-
-struct lzma_filter_info_s {
- /// Pointer to function used to initialize the filter.
- /// This is NULL to indicate end of array.
- lzma_init_function init;
+ /// Indicates which lzma_action values are allowed by next.code.
+ bool supported_actions[4];
- /// Pointer to filter's options structure
- void *options;
+ /// If true, lzma_code will return LZMA_BUF_ERROR if no progress was
+ /// made (no input consumed and no output produced by next.code).
+ bool allow_buf_error;
};
-/*
-typedef struct {
- lzma_init_function init;
- uint32_t (*input_alignment)(lzma_vli id, const void *options);
- uint32_t (*output_alignment)(lzma_vli id, const void *options);
- bool changes_uncompressed_size;
- bool supports_eopm;
-} lzma_filter_hook;
-*/
-
-
///////////////
// Functions //
///////////////
/// Frees memory
extern void lzma_free(void *ptr, lzma_allocator *allocator);
-/// Initializes lzma_stream FIXME desc
+
+/// Allocates strm->internal if it is NULL, and initializes *strm and
+/// strm->internal. This function is only called via lzma_next_strm_init macro.
extern lzma_ret lzma_strm_init(lzma_stream *strm);
-///
+/// Initializes the next filter in the chain, if any. This takes care of
+/// freeing the memory of previously initialized filter if it is different
+/// than the filter being initialized now. This way the actual filter
+/// initialization functions don't need to use lzma_next_coder_init macro.
extern lzma_ret lzma_next_filter_init(lzma_next_coder *next,
lzma_allocator *allocator, const lzma_filter_info *filters);
-///
-extern void lzma_next_coder_end(lzma_next_coder *next,
- lzma_allocator *allocator);
+/// Frees the memory allocated for next->coder either using next->end or,
+/// if next->end is NULL, using lzma_free.
+extern void lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator);
-/// \brief Wrapper for memcpy()
-///
-/// This function copies as much data as possible from in[] to out[] and
-/// updates *in_pos and *out_pos accordingly.
-///
-static inline size_t
-bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size,
- uint8_t *restrict out, size_t *restrict out_pos,
- size_t out_size)
-{
- const size_t in_avail = in_size - *in_pos;
- const size_t out_avail = out_size - *out_pos;
- const size_t copy_size = MIN(in_avail, out_avail);
-
- memcpy(out + *out_pos, in + *in_pos, copy_size);
-
- *in_pos += copy_size;
- *out_pos += copy_size;
-
- return copy_size;
-}
+/// Copy as much data as possible from in[] to out[] and update *in_pos
+/// and *out_pos accordingly. Returns the number of bytes copied.
+extern size_t lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos,
+ size_t in_size, uint8_t *restrict out,
+ size_t *restrict out_pos, size_t out_size);
-/// \brief Initializing the next coder
-///
-/// lzma_next_coder can point to different types of coders. The existing
-/// coder may be different than what we are initializing now. In that case
-/// we must git rid of the old coder first. Otherwise we reuse the existing
-/// coder structure.
+/// \brief Return if expression doesn't evaluate to LZMA_OK
///
-#define lzma_next_coder_init2(next, allocator, cmpfunc, func, ...) \
+/// There are several situations where we want to return immediatelly
+/// with the value of expr if it isn't LZMA_OK. This macro shortens
+/// the code a little.
+#define return_if_error(expr) \
do { \
- if ((uintptr_t)(&cmpfunc) != (next)->init) \
- lzma_next_coder_end(next, allocator); \
- const lzma_ret ret = func(next, __VA_ARGS__); \
- if (ret == LZMA_OK) { \
- (next)->init = (uintptr_t)(&cmpfunc); \
- assert((next)->code != NULL); \
- assert((next)->end != NULL); \
- } else { \
- lzma_next_coder_end(next, allocator); \
- } \
- return ret; \
+ const lzma_ret ret_ = (expr); \
+ if (ret_ != LZMA_OK) \
+ return ret_; \
} while (0)
-/// \brief Initializing lzma_next_coder
-///
-/// Call the initialization function, which must take at least one
-/// argument in addition to lzma_next_coder and lzma_allocator.
-#define lzma_next_coder_init(func, next, allocator, ...) \
- lzma_next_coder_init2(next, allocator, \
- func, func, allocator, __VA_ARGS__)
-
-/// \brief Initializing lzma_next_coder
-///
-/// Call the initialization function, which takes no other arguments than
-/// lzma_next_coder and lzma_allocator.
-#define lzma_next_coder_init0(func, next, allocator) \
- lzma_next_coder_init2(next, allocator, func, func, allocator)
-
-/// \brief Initializing lzma_stream
-///
-/// lzma_strm initialization with more detailed options.
-#define lzma_next_strm_init2(strm, cmpfunc, func, ...) \
+/// If next isn't already initialized, free the previous coder. Then mark
+/// that next is _possibly_ initialized for the coder using this macro.
+/// "Possibly" means that if e.g. allocation of next->coder fails, the
+/// structure isn't actually initialized for this coder, but leaving
+/// next->init to func is still OK.
+#define lzma_next_coder_init(func, next, allocator) \
do { \
- lzma_ret ret = lzma_strm_init(strm); \
- if (ret != LZMA_OK) \
- return ret; \
- if ((uintptr_t)(&cmpfunc) != (strm)->internal->next.init) \
- lzma_next_coder_end(\
- &(strm)->internal->next, (strm)->allocator); \
- ret = func(&(strm)->internal->next, __VA_ARGS__); \
- if (ret != LZMA_OK) { \
- lzma_end(strm); \
- return ret; \
- } \
- (strm)->internal->next.init = (uintptr_t)(&cmpfunc); \
- assert((strm)->internal->next.code != NULL); \
- assert((strm)->internal->next.end != NULL); \
+ if ((uintptr_t)(&func) != (next)->init) \
+ lzma_next_end(next, allocator); \
+ (next)->init = (uintptr_t)(&func); \
} while (0)
-/// \brief Initializing lzma_stream
-///
-/// Call the initialization function, which must take at least one
-/// argument in addition to lzma_next_coder and lzma_allocator.
-#define lzma_next_strm_init(strm, func, ...) \
- lzma_next_strm_init2(strm, func, func, (strm)->allocator, __VA_ARGS__)
-/// \brief Initializing lzma_stream
-///
-/// Call the initialization function, which takes no other arguments than
-/// lzma_next_coder and lzma_allocator.
-#define lzma_next_strm_init0(strm, func) \
- lzma_next_strm_init2(strm, func, func, (strm)->allocator)
-
-
-/// \brief Return if expression doesn't evaluate to LZMA_OK
-///
-/// There are several situations where we want to return immediatelly
-/// with the value of expr if it isn't LZMA_OK. This macro shortens
-/// the code a bit.
-///
-#define return_if_error(expr) \
+/// Initializes lzma_strm and calls func() to initialize strm->internal->next.
+/// (The function being called will use lzma_next_coder_init()). If
+/// initialization fails, memory that wasn't freed by func() is freed
+/// along strm->internal.
+#define lzma_next_strm_init(func, strm, ...) \
do { \
- const lzma_ret ret_ = expr; \
- if (ret_ != LZMA_OK) \
+ return_if_error(lzma_strm_init(strm)); \
+ const lzma_ret ret_ = func(&(strm)->internal->next, \
+ (strm)->allocator, __VA_ARGS__); \
+ if (ret_ != LZMA_OK) { \
+ lzma_end(strm); \
return ret_; \
+ } \
} while (0)
#endif
/// We need to keep the filters array available in case
/// LZMA_FULL_FLUSH is used.
- lzma_options_filter filters[5];
+ lzma_filter filters[5];
};
static bool
-easy_set_filters(lzma_options_filter *filters, uint32_t level)
+easy_set_filters(lzma_filter *filters, uint32_t level)
{
bool error = false;
// TODO FIXME Use Subblock or LZMA2 with no compression.
error = true;
-#ifdef HAVE_FILTER_LZMA
+#ifdef HAVE_ENCODER_LZMA2
} else if (level <= 9) {
- filters[0].id = LZMA_FILTER_LZMA;
+ filters[0].id = LZMA_FILTER_LZMA2;
filters[0].options = (void *)(&lzma_preset_lzma[level - 1]);
filters[1].id = LZMA_VLI_VALUE_UNKNOWN;
#endif
static void
easy_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->stream_encoder, allocator);
+ lzma_next_end(&coder->stream_encoder, allocator);
lzma_free(coder, allocator);
return;
}
easy_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
lzma_easy_level level)
{
+ lzma_next_coder_init(easy_encoder_init, next, allocator);
+
if (next->coder == NULL) {
next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
if (next->coder == NULL)
extern LZMA_API lzma_ret
lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level)
{
- lzma_next_strm_init(strm, easy_encoder_init, level);
+ lzma_next_strm_init(easy_encoder_init, strm, level);
strm->internal->supported_actions[LZMA_RUN] = true;
strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
extern LZMA_API uint32_t
lzma_easy_memory_usage(lzma_easy_level level)
{
- lzma_options_filter filters[5];
+ lzma_filter filters[5];
if (easy_set_filters(filters, level))
return UINT32_MAX;
- return lzma_memory_usage(filters, true);
+ return lzma_memusage_encoder(filters);
}
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file features.c
-/// \brief Information about features enabled at compile time
-//
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "common.h"
-
-
-static const lzma_vli filters[] = {
-#ifdef HAVE_FILTER_SUBBLOCK
- LZMA_FILTER_SUBBLOCK,
-#endif
-
-#ifdef HAVE_FILTER_X86
- LZMA_FILTER_X86,
-#endif
-
-#ifdef HAVE_FILTER_POWERPC
- LZMA_FILTER_POWERPC,
-#endif
-
-#ifdef HAVE_FILTER_IA64
- LZMA_FILTER_IA64,
-#endif
-
-#ifdef HAVE_FILTER_ARM
- LZMA_FILTER_ARM,
-#endif
-
-#ifdef HAVE_FILTER_ARMTHUMB
- LZMA_FILTER_ARMTHUMB,
-#endif
-
-#ifdef HAVE_FILTER_SPARC
- LZMA_FILTER_SPARC,
-#endif
-
-#ifdef HAVE_FILTER_DELTA
- LZMA_FILTER_DELTA,
-#endif
-
-#ifdef HAVE_FILTER_LZMA
- LZMA_FILTER_LZMA,
-#endif
-
- LZMA_VLI_VALUE_UNKNOWN
-};
-
-
-LZMA_API const lzma_vli *const lzma_available_filter_encoders = filters;
-
-LZMA_API const lzma_vli *const lzma_available_filter_decoders = filters;
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file filter_common.c
+/// \brief Filter-specific stuff common for both encoder and decoder
+//
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "filter_common.h"
+
+
+static const struct {
+ /// Filter ID
+ lzma_vli id;
+
+ /// True if it is OK to use this filter as non-last filter in
+ /// the chain.
+ bool non_last_ok;
+
+ /// True if it is OK to use this filter as the last filter in
+ /// the chain.
+ bool last_ok;
+
+ /// True if the filter may change the size of the data (that is, the
+ /// amount of encoded output can be different than the amount of
+ /// uncompressed input).
+ bool changes_size;
+
+} features[] = {
+#if defined (HAVE_ENCODER_LZMA) || defined(HAVE_DECODER_LZMA)
+ {
+ .id = LZMA_FILTER_LZMA,
+ .non_last_ok = false,
+ .last_ok = true,
+ .changes_size = true,
+ },
+#endif
+#ifdef HAVE_DECODER_LZMA2
+ {
+ .id = LZMA_FILTER_LZMA2,
+ .non_last_ok = false,
+ .last_ok = true,
+ .changes_size = true,
+ },
+#endif
+#if defined(HAVE_ENCODER_SUBBLOCK) || defined(HAVE_DECODER_SUBBLOCK)
+ {
+ .id = LZMA_FILTER_SUBBLOCK,
+ .non_last_ok = true,
+ .last_ok = true,
+ .changes_size = true,
+ },
+#endif
+#ifdef HAVE_DECODER_X86
+ {
+ .id = LZMA_FILTER_X86,
+ .non_last_ok = true,
+ .last_ok = false,
+ .changes_size = false,
+ },
+#endif
+#if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC)
+ {
+ .id = LZMA_FILTER_POWERPC,
+ .non_last_ok = true,
+ .last_ok = false,
+ .changes_size = false,
+ },
+#endif
+#ifdef HAVE_DECODER_IA64
+ {
+ .id = LZMA_FILTER_IA64,
+ .non_last_ok = true,
+ .last_ok = false,
+ .changes_size = false,
+ },
+#endif
+#if defined(HAVE_ENCODER_ARM) || defined(HAVE_DECODER_ARM)
+ {
+ .id = LZMA_FILTER_ARM,
+ .non_last_ok = true,
+ .last_ok = false,
+ .changes_size = false,
+ },
+#endif
+#if defined(HAVE_ENCODER_ARMTHUMB) || defined(HAVE_DECODER_ARMTHUMB)
+ {
+ .id = LZMA_FILTER_ARMTHUMB,
+ .non_last_ok = true,
+ .last_ok = false,
+ .changes_size = false,
+ },
+#endif
+#if defined(HAVE_ENCODER_SPARC) || defined(HAVE_DECODER_SPARC)
+ {
+ .id = LZMA_FILTER_SPARC,
+ .non_last_ok = true,
+ .last_ok = false,
+ .changes_size = false,
+ },
+#endif
+#if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA)
+ {
+ .id = LZMA_FILTER_DELTA,
+ .non_last_ok = true,
+ .last_ok = false,
+ .changes_size = false,
+ },
+#endif
+ {
+ .id = LZMA_VLI_VALUE_UNKNOWN
+ }
+};
+
+
+static lzma_ret
+validate_chain(const lzma_filter *filters, size_t *count)
+{
+ // There must be at least one filter.
+ if (filters == NULL || filters[0].id == LZMA_VLI_VALUE_UNKNOWN)
+ return LZMA_PROG_ERROR;
+
+ // Number of non-last filters that may change the size of the data
+ // significantly (that is, more than 1-2 % or so).
+ size_t changes_size_count = 0;
+
+ // True if it is OK to add a new filter after the current filter.
+ bool non_last_ok = true;
+
+ // True if the last filter in the given chain is actually usable as
+ // the last filter. Only filters that support embedding End of Payload
+ // Marker can be used as the last filter in the chain.
+ bool last_ok = false;
+
+ size_t i = 0;
+ do {
+ size_t j;
+ for (j = 0; filters[i].id != features[j].id; ++j)
+ if (features[j].id == LZMA_VLI_VALUE_UNKNOWN)
+ return LZMA_HEADER_ERROR;
+
+ // If the previous filter in the chain cannot be a non-last
+ // filter, the chain is invalid.
+ if (!non_last_ok)
+ return LZMA_HEADER_ERROR;
+
+ non_last_ok = features[j].non_last_ok;
+ last_ok = features[j].last_ok;
+ changes_size_count += features[j].changes_size;
+
+ } while (filters[++i].id != LZMA_VLI_VALUE_UNKNOWN);
+
+ // There must be 1-4 filters. The last filter must be usable as
+ // the last filter in the chain. At maximum of three filters are
+ // allowed to change the size of the data.
+ if (i > LZMA_BLOCK_FILTERS_MAX || !last_ok || changes_size_count > 3)
+ return LZMA_HEADER_ERROR;
+
+ *count = i;
+ return LZMA_OK;
+}
+
+
+extern lzma_ret
+lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter *options,
+ lzma_filter_find coder_find, bool is_encoder)
+{
+ // Do some basic validation and get the number of filters.
+ size_t count;
+ return_if_error(validate_chain(options, &count));
+
+ // Set the filter functions and copy the options pointer.
+ lzma_filter_info filters[LZMA_BLOCK_FILTERS_MAX + 1];
+ if (is_encoder) {
+ for (size_t i = 0; i < count; ++i) {
+ // The order of the filters is reversed in the
+ // encoder. It allows more efficient handling
+ // of the uncompressed data.
+ const size_t j = count - i - 1;
+
+ const lzma_filter_coder *const fc
+ = coder_find(options[i].id);
+ if (fc == NULL || fc->init == NULL)
+ return LZMA_HEADER_ERROR;
+
+ filters[j].init = fc->init;
+ filters[j].options = options[i].options;
+ }
+ } else {
+ for (size_t i = 0; i < count; ++i) {
+ const lzma_filter_coder *const fc
+ = coder_find(options[i].id);
+ if (fc == NULL || fc->init == NULL)
+ return LZMA_HEADER_ERROR;
+
+ filters[i].init = fc->init;
+ filters[i].options = options[i].options;
+ }
+ }
+
+ // Terminate the array.
+ filters[count].init = NULL;
+
+ // Initialize the filters.
+ const lzma_ret ret = lzma_next_filter_init(next, allocator, filters);
+ if (ret != LZMA_OK)
+ lzma_next_end(next, allocator);
+
+ return ret;
+}
+
+
+extern uint64_t
+lzma_memusage_coder(lzma_filter_find coder_find,
+ const lzma_filter *filters)
+{
+ // The chain has to have at least one filter.
+ if (filters[0].id == LZMA_VLI_VALUE_UNKNOWN)
+ return UINT64_MAX;
+
+ uint64_t total = 0;
+ size_t i = 0;
+
+ do {
+ const lzma_filter_coder *const fc
+ = coder_find(filters[i].id);
+ if (fc == NULL)
+ return UINT64_MAX; // Unsupported Filter ID
+
+ if (fc->memusage == NULL) {
+ // This filter doesn't have a function to calculate
+ // the memory usage. Such filters need only little
+ // memory, so we use 1 KiB as a good estimate.
+ total += 1024;
+ } else {
+ // Call the filter-specific memory usage calculation
+ // function.
+ const uint64_t usage
+ = fc->memusage(filters[i].options);
+ if (usage == UINT64_MAX)
+ return UINT64_MAX; // Invalid options
+
+ total += usage;
+ }
+ } while (filters[++i].id != LZMA_VLI_VALUE_UNKNOWN);
+
+ // Add some fixed amount of extra. It's to compensate memory usage
+ // of Stream, Block etc. coders, malloc() overhead, stack etc.
+ return total + (1U << 15);
+}
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file filter_common.c
+/// \brief Filter-specific stuff common for both encoder and decoder
+//
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef LZMA_FILTER_COMMON_H
+#define LZMA_FILTER_COMMON_H
+
+#include "common.h"
+
+
+/// Both lzma_filter_encoder and lzma_filter_decoder begin with these members.
+typedef struct {
+ /// Initializes the filter encoder and calls lzma_next_filter_init()
+ /// for filters + 1.
+ lzma_init_function init;
+
+ /// Calculates memory usage of the encoder. If the options are
+ /// invalid, UINT64_MAX is returned.
+ uint64_t (*memusage)(const void *options);
+
+} lzma_filter_coder;
+
+
+typedef const lzma_filter_coder *(*lzma_filter_find)(lzma_vli id);
+
+
+extern lzma_ret lzma_raw_coder_init(
+ lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter *filters,
+ lzma_filter_find coder_find, bool is_encoder);
+
+
+extern uint64_t lzma_memusage_coder(lzma_filter_find coder_find,
+ const lzma_filter *filters);
+
+
+#endif
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file filter_decoder.c
+/// \brief Filter ID mapping to filter-specific functions
+//
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "filter_decoder.h"
+#include "filter_common.h"
+#include "lzma_decoder.h"
+#include "lzma2_decoder.h"
+#include "subblock_decoder.h"
+#include "subblock_decoder_helper.h"
+#include "simple_decoder.h"
+#include "delta_decoder.h"
+
+
+typedef struct {
+ /// Initializes the filter encoder and calls lzma_next_filter_init()
+ /// for filters + 1.
+ lzma_init_function init;
+
+ /// Calculates memory usage of the encoder. If the options are
+ /// invalid, UINT64_MAX is returned.
+ uint64_t (*memusage)(const void *options);
+
+ /// Decodes Filter Properties.
+ ///
+ /// \return - LZMA_OK: Properties decoded successfully.
+ /// - LZMA_HEADER_ERROR: Unsupported properties
+ /// - LZMA_MEM_ERROR: Memory allocation failed.
+ lzma_ret (*props_decode)(void **options, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size);
+
+} lzma_filter_decoder;
+
+
+static const lzma_vli ids[] = {
+#ifdef HAVE_DECODER_LZMA
+ LZMA_FILTER_LZMA,
+#endif
+
+#ifdef HAVE_DECODER_LZMA2
+ LZMA_FILTER_LZMA2,
+#endif
+
+#ifdef HAVE_DECODER_SUBBLOCK
+ LZMA_FILTER_SUBBLOCK,
+ LZMA_FILTER_SUBBLOCK_HELPER,
+#endif
+
+#ifdef HAVE_DECODER_X86
+ LZMA_FILTER_X86,
+#endif
+
+#ifdef HAVE_DECODER_POWERPC
+ LZMA_FILTER_POWERPC,
+#endif
+
+#ifdef HAVE_DECODER_IA64
+ LZMA_FILTER_IA64,
+#endif
+
+#ifdef HAVE_DECODER_ARM
+ LZMA_FILTER_ARM,
+#endif
+
+#ifdef HAVE_DECODER_ARMTHUMB
+ LZMA_FILTER_ARMTHUMB,
+#endif
+
+#ifdef HAVE_DECODER_SPARC
+ LZMA_FILTER_SPARC,
+#endif
+
+#ifdef HAVE_DECODER_DELTA
+ LZMA_FILTER_DELTA,
+#endif
+
+ LZMA_VLI_VALUE_UNKNOWN
+};
+
+
+// Using a pointer to avoid putting the size of the array to API/ABI.
+LZMA_API const lzma_vli *const lzma_filter_decoders = ids;
+
+
+// These must be in the same order as ids[].
+static const lzma_filter_decoder funcs[] = {
+#ifdef HAVE_DECODER_LZMA
+ {
+ .init = &lzma_lzma_decoder_init,
+ .memusage = &lzma_lzma_decoder_memusage,
+ .props_decode = &lzma_lzma_props_decode,
+ },
+#endif
+#ifdef HAVE_DECODER_LZMA2
+ {
+ .init = &lzma_lzma2_decoder_init,
+ .memusage = &lzma_lzma2_decoder_memusage,
+ .props_decode = &lzma_lzma2_props_decode,
+ },
+#endif
+#ifdef HAVE_DECODER_SUBBLOCK
+ {
+ .init = &lzma_subblock_decoder_init,
+// .memusage = &lzma_subblock_decoder_memusage,
+ .props_decode = NULL,
+ },
+ {
+ .init = &lzma_subblock_decoder_helper_init,
+ .memusage = NULL,
+ .props_decode = NULL,
+ },
+#endif
+#ifdef HAVE_DECODER_X86
+ {
+ .init = &lzma_simple_x86_decoder_init,
+ .memusage = NULL,
+ .props_decode = &lzma_simple_props_decode,
+ },
+#endif
+#ifdef HAVE_DECODER_POWERPC
+ {
+ .init = &lzma_simple_powerpc_decoder_init,
+ .memusage = NULL,
+ .props_decode = &lzma_simple_props_decode,
+ },
+#endif
+#ifdef HAVE_DECODER_IA64
+ {
+ .init = &lzma_simple_ia64_decoder_init,
+ .memusage = NULL,
+ .props_decode = &lzma_simple_props_decode,
+ },
+#endif
+#ifdef HAVE_DECODER_ARM
+ {
+ .init = &lzma_simple_arm_decoder_init,
+ .memusage = NULL,
+ .props_decode = &lzma_simple_props_decode,
+ },
+#endif
+#ifdef HAVE_DECODER_ARMTHUMB
+ {
+ .init = &lzma_simple_armthumb_decoder_init,
+ .memusage = NULL,
+ .props_decode = &lzma_simple_props_decode,
+ },
+#endif
+#ifdef HAVE_DECODER_SPARC
+ {
+ .init = &lzma_simple_sparc_decoder_init,
+ .memusage = NULL,
+ .props_decode = &lzma_simple_props_decode,
+ },
+#endif
+#ifdef HAVE_DECODER_DELTA
+ {
+ .init = &lzma_delta_decoder_init,
+ .memusage = NULL,
+ .props_decode = &lzma_delta_props_decode,
+ },
+#endif
+};
+
+
+static const lzma_filter_decoder *
+decoder_find(lzma_vli id)
+{
+ for (size_t i = 0; ids[i] != LZMA_VLI_VALUE_UNKNOWN; ++i)
+ if (ids[i] == id)
+ return funcs + i;
+
+ return NULL;
+}
+
+
+extern lzma_ret
+lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter *options)
+{
+ return lzma_raw_coder_init(next, allocator,
+ options, (lzma_filter_find)(&decoder_find), false);
+}
+
+
+extern LZMA_API lzma_ret
+lzma_raw_decoder(lzma_stream *strm, const lzma_filter *options)
+{
+ lzma_next_strm_init(lzma_raw_decoder_init, strm, options);
+
+ strm->internal->supported_actions[LZMA_RUN] = true;
+ strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
+
+ return LZMA_OK;
+}
+
+
+extern LZMA_API uint64_t
+lzma_memusage_decoder(const lzma_filter *filters)
+{
+ return lzma_memusage_coder(
+ (lzma_filter_find)(&decoder_find), filters);
+}
+
+
+extern LZMA_API lzma_ret
+lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size)
+{
+ // Make it always NULL so that the caller can always safely free() it.
+ filter->options = NULL;
+
+ const lzma_filter_decoder *const fd = decoder_find(filter->id);
+ if (fd == NULL)
+ return LZMA_HEADER_ERROR;
+
+ if (fd->props_decode == NULL)
+ return props_size == 0 ? LZMA_OK : LZMA_HEADER_ERROR;
+
+ return fd->props_decode(
+ &filter->options, allocator, props, props_size);
+}
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file filter_decoder.c
+/// \brief Filter ID mapping to filter-specific functions
+//
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef LZMA_FILTER_DECODER_H
+#define LZMA_FILTER_DECODER_H
+
+#include "common.h"
+
+// FIXME !!! Public API
+extern lzma_ret lzma_properties_decode(
+ lzma_filter *filter, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size);
+
+
+extern lzma_ret lzma_raw_decoder_init(
+ lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter *options);
+
+#endif
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file filter_decoder.c
+/// \brief Filter ID mapping to filter-specific functions
+//
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "filter_encoder.h"
+#include "filter_common.h"
+#include "lzma_encoder.h"
+#include "lzma2_encoder.h"
+#include "subblock_encoder.h"
+#include "simple_encoder.h"
+#include "delta_encoder.h"
+
+
+typedef struct {
+ /// Initializes the filter encoder and calls lzma_next_filter_init()
+ /// for filters + 1.
+ lzma_init_function init;
+
+ /// Calculates memory usage of the encoder. If the options are
+ /// invalid, UINT64_MAX is returned.
+ uint64_t (*memusage)(const void *options);
+
+ /// Calculates the minimum sane size for Blocks (or other types of
+ /// chunks) to which the input data can be splitted to make
+ /// multithreaded encoding possible. If this is NULL, it is assumed
+ /// that the encoder is fast enough with single thread.
+ lzma_vli (*chunk_size)(const void *options);
+
+ /// Tells the size of the Filter Properties field. If options are
+ /// invalid, UINT32_MAX is returned. If this is NULL, props_size_fixed
+ /// is used.
+ lzma_ret (*props_size_get)(uint32_t *size, const void *options);
+ uint32_t props_size_fixed;
+
+ /// Encodes Filter Properties.
+ ///
+ /// \return - LZMA_OK: Properties encoded sucessfully.
+ /// - LZMA_HEADER_ERROR: Unsupported options
+ /// - LZMA_PROG_ERROR: Invalid options or not enough
+ /// output space
+ lzma_ret (*props_encode)(const void *options, uint8_t *out);
+
+} lzma_filter_encoder;
+
+
+static const lzma_vli ids[] = {
+#ifdef HAVE_ENCODER_LZMA
+ LZMA_FILTER_LZMA,
+#endif
+
+#ifdef HAVE_ENCODER_LZMA2
+ LZMA_FILTER_LZMA2,
+#endif
+
+#ifdef HAVE_ENCODER_SUBBLOCK
+ LZMA_FILTER_SUBBLOCK,
+#endif
+
+#ifdef HAVE_ENCODER_X86
+ LZMA_FILTER_X86,
+#endif
+
+#ifdef HAVE_ENCODER_POWERPC
+ LZMA_FILTER_POWERPC,
+#endif
+
+#ifdef HAVE_ENCODER_IA64
+ LZMA_FILTER_IA64,
+#endif
+
+#ifdef HAVE_ENCODER_ARM
+ LZMA_FILTER_ARM,
+#endif
+
+#ifdef HAVE_ENCODER_ARMTHUMB
+ LZMA_FILTER_ARMTHUMB,
+#endif
+
+#ifdef HAVE_ENCODER_SPARC
+ LZMA_FILTER_SPARC,
+#endif
+
+#ifdef HAVE_ENCODER_DELTA
+ LZMA_FILTER_DELTA,
+#endif
+
+ LZMA_VLI_VALUE_UNKNOWN
+};
+
+
+// Using a pointer to avoid putting the size of the array to API/ABI.
+LZMA_API const lzma_vli *const lzma_filter_encoders = ids;
+
+
+// These must be in the same order as ids[].
+static const lzma_filter_encoder funcs[] = {
+#ifdef HAVE_ENCODER_LZMA
+ {
+ .init = &lzma_lzma_encoder_init,
+ .memusage = &lzma_lzma_encoder_memusage,
+ .chunk_size = NULL, // FIXME
+ .props_size_get = NULL,
+ .props_size_fixed = 5,
+ .props_encode = &lzma_lzma_props_encode,
+ },
+#endif
+#ifdef HAVE_ENCODER_LZMA2
+ {
+ .init = &lzma_lzma2_encoder_init,
+ .memusage = &lzma_lzma2_encoder_memusage,
+ .chunk_size = NULL, // FIXME
+ .props_size_get = NULL,
+ .props_size_fixed = 1,
+ .props_encode = &lzma_lzma2_props_encode,
+ },
+#endif
+#ifdef HAVE_ENCODER_SUBBLOCK
+ {
+ .init = &lzma_subblock_encoder_init,
+// .memusage = &lzma_subblock_encoder_memusage,
+ .chunk_size = NULL,
+ .props_size_get = NULL,
+ .props_size_fixed = 0,
+ .props_encode = NULL,
+ },
+#endif
+#ifdef HAVE_ENCODER_X86
+ {
+ .init = &lzma_simple_x86_encoder_init,
+ .memusage = NULL,
+ .chunk_size = NULL,
+ .props_size_get = &lzma_simple_props_size,
+ .props_encode = &lzma_simple_props_encode,
+ },
+#endif
+#ifdef HAVE_ENCODER_POWERPC
+ {
+ .init = &lzma_simple_powerpc_encoder_init,
+ .memusage = NULL,
+ .chunk_size = NULL,
+ .props_size_get = &lzma_simple_props_size,
+ .props_encode = &lzma_simple_props_encode,
+ },
+#endif
+#ifdef HAVE_ENCODER_IA64
+ {
+ .init = &lzma_simple_ia64_encoder_init,
+ .memusage = NULL,
+ .chunk_size = NULL,
+ .props_size_get = &lzma_simple_props_size,
+ .props_encode = &lzma_simple_props_encode,
+ },
+#endif
+#ifdef HAVE_ENCODER_ARM
+ {
+ .init = &lzma_simple_arm_encoder_init,
+ .memusage = NULL,
+ .chunk_size = NULL,
+ .props_size_get = &lzma_simple_props_size,
+ .props_encode = &lzma_simple_props_encode,
+ },
+#endif
+#ifdef HAVE_ENCODER_ARMTHUMB
+ {
+ .init = &lzma_simple_armthumb_encoder_init,
+ .memusage = NULL,
+ .chunk_size = NULL,
+ .props_size_get = &lzma_simple_props_size,
+ .props_encode = &lzma_simple_props_encode,
+ },
+#endif
+#ifdef HAVE_ENCODER_SPARC
+ {
+ .init = &lzma_simple_sparc_encoder_init,
+ .memusage = NULL,
+ .chunk_size = NULL,
+ .props_size_get = &lzma_simple_props_size,
+ .props_encode = &lzma_simple_props_encode,
+ },
+#endif
+#ifdef HAVE_ENCODER_DELTA
+ {
+ .init = &lzma_delta_encoder_init,
+ .memusage = NULL,
+ .chunk_size = NULL,
+ .props_size_get = NULL,
+ .props_size_fixed = 1,
+ .props_encode = &lzma_delta_props_encode,
+ },
+#endif
+};
+
+
+static const lzma_filter_encoder *
+encoder_find(lzma_vli id)
+{
+ for (size_t i = 0; ids[i] != LZMA_VLI_VALUE_UNKNOWN; ++i)
+ if (ids[i] == id)
+ return funcs + i;
+
+ return NULL;
+}
+
+
+extern lzma_ret
+lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter *options)
+{
+ return lzma_raw_coder_init(next, allocator,
+ options, (lzma_filter_find)(&encoder_find), true);
+}
+
+
+extern LZMA_API lzma_ret
+lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options)
+{
+ lzma_next_strm_init(lzma_raw_coder_init, strm, options,
+ (lzma_filter_find)(&encoder_find), true);
+
+ strm->internal->supported_actions[LZMA_RUN] = true;
+ strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
+ strm->internal->supported_actions[LZMA_FINISH] = true;
+
+ return LZMA_OK;
+}
+
+
+extern LZMA_API uint64_t
+lzma_memusage_encoder(const lzma_filter *filters)
+{
+ return lzma_memusage_coder(
+ (lzma_filter_find)(&encoder_find), filters);
+}
+
+
+extern LZMA_API lzma_vli
+lzma_chunk_size(const lzma_filter *filters)
+{
+ uint64_t max = 0;
+
+ for (size_t i = 0; filters[i].id != LZMA_VLI_VALUE_UNKNOWN; ++i) {
+ const lzma_filter_encoder *const fe
+ = encoder_find(filters[i].id);
+ if (fe->chunk_size != NULL) {
+ const lzma_vli size
+ = fe->chunk_size(filters[i].options);
+ if (size == LZMA_VLI_VALUE_UNKNOWN)
+ return LZMA_VLI_VALUE_UNKNOWN;
+
+ if (size > max)
+ max = size;
+ }
+ }
+
+ return max;
+}
+
+
+extern LZMA_API lzma_ret
+lzma_properties_size(uint32_t *size, const lzma_filter *filter)
+{
+ const lzma_filter_encoder *const fe = encoder_find(filter->id);
+ if (fe == NULL) {
+ // Unknown filter - if the Filter ID is a proper VLI,
+ // return LZMA_HEADER_ERROR instead of LZMA_PROG_ERROR,
+ // because it's possible that we just don't have support
+ // compiled in for the requested filter.
+ return filter->id <= LZMA_VLI_VALUE_MAX
+ ? LZMA_HEADER_ERROR : LZMA_PROG_ERROR;
+ }
+
+ if (fe->props_size_get == NULL) {
+ // No props_size() function, use props_size_fixed.
+ *size = fe->props_size_fixed;
+ return LZMA_OK;
+ }
+
+ return fe->props_size_get(size, filter->options);
+}
+
+
+extern LZMA_API lzma_ret
+lzma_properties_encode(const lzma_filter *filter, uint8_t *props)
+{
+ const lzma_filter_encoder *const fe = encoder_find(filter->id);
+ if (fe == NULL)
+ return LZMA_PROG_ERROR;
+
+ if (fe->props_encode == NULL)
+ return LZMA_OK;
+
+ return fe->props_encode(filter->options, props);
+}
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file filter_encoder.c
+/// \brief Filter ID mapping to filter-specific functions
+//
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef LZMA_FILTER_ENCODER_H
+#define LZMA_FILTER_ENCODER_H
+
+#include "common.h"
+
+
+// FIXME !!! Public API
+extern lzma_vli lzma_chunk_size(const lzma_filter *filters);
+extern lzma_ret lzma_properties_size(
+ uint32_t *size, const lzma_filter *filter);
+extern lzma_ret lzma_properties_encode(
+ const lzma_filter *filter, uint8_t *props);
+
+
+extern lzma_ret lzma_raw_encoder_init(
+ lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter *options);
+
+#endif
//
///////////////////////////////////////////////////////////////////////////////
-#include "common.h"
-#include "lzma_decoder.h"
-
-
-#ifdef HAVE_FILTER_SUBBLOCK
-static lzma_ret
-properties_subblock(lzma_options_filter *options, lzma_allocator *allocator,
- const uint8_t *props lzma_attribute((unused)),
- size_t prop_size lzma_attribute((unused)))
-{
- if (prop_size != 0)
- return LZMA_HEADER_ERROR;
-
- options->options = lzma_alloc(
- sizeof(lzma_options_subblock), allocator);
- if (options->options == NULL)
- return LZMA_MEM_ERROR;
-
- ((lzma_options_subblock *)(options->options))->allow_subfilters = true;
- return LZMA_OK;
-}
-#endif
-
-
-#ifdef HAVE_FILTER_SIMPLE
-static lzma_ret
-properties_simple(lzma_options_filter *options, lzma_allocator *allocator,
- const uint8_t *props, size_t prop_size)
-{
- if (prop_size == 0)
- return LZMA_OK;
-
- if (prop_size != 4)
- return LZMA_HEADER_ERROR;
-
- lzma_options_simple *simple = lzma_alloc(
- sizeof(lzma_options_simple), allocator);
- if (simple == NULL)
- return LZMA_MEM_ERROR;
-
- simple->start_offset = integer_read_32(props);
-
- // Don't leave an options structure allocated if start_offset is zero.
- if (simple->start_offset == 0)
- lzma_free(simple, allocator);
- else
- options->options = simple;
-
- return LZMA_OK;
-}
-#endif
-
-
-#ifdef HAVE_FILTER_DELTA
-static lzma_ret
-properties_delta(lzma_options_filter *options, lzma_allocator *allocator,
- const uint8_t *props, size_t prop_size)
-{
- if (prop_size != 1)
- return LZMA_HEADER_ERROR;
-
- options->options = lzma_alloc(sizeof(lzma_options_delta), allocator);
- if (options->options == NULL)
- return LZMA_MEM_ERROR;
-
- ((lzma_options_delta *)(options->options))->distance
- = (uint32_t)(props[0]) + 1;
-
- return LZMA_OK;
-}
-#endif
-
-
-#ifdef HAVE_FILTER_LZMA
-static lzma_ret
-properties_lzma(lzma_options_filter *options, lzma_allocator *allocator,
- const uint8_t *props, size_t prop_size)
-{
- // LZMA properties are always two bytes (at least for now).
- if (prop_size != 2)
- return LZMA_HEADER_ERROR;
-
- lzma_options_lzma *lzma = lzma_alloc(
- sizeof(lzma_options_lzma), allocator);
- if (lzma == NULL)
- return LZMA_MEM_ERROR;
-
- // Decode lc, lp, and pb.
- if (lzma_lzma_decode_properties(lzma, props[0]))
- goto error;
-
- // Check that reserved bits are unset.
- if (props[1] & 0xC0)
- goto error;
-
- // Decode the dictionary size.
- // FIXME The specification says that maximum is 4 GiB.
- if (props[1] > 36)
- goto error;
-#if LZMA_DICTIONARY_SIZE_MAX != UINT32_C(1) << 30
-# error Update the if()-condition a few lines
-# error above to match LZMA_DICTIONARY_SIZE_MAX.
-#endif
-
- lzma->dictionary_size = 2 | (props[1] & 1);
- lzma->dictionary_size <<= props[1] / 2 + 11;
-
- options->options = lzma;
- return LZMA_OK;
-
-error:
- lzma_free(lzma, allocator);
- return LZMA_HEADER_ERROR;
-}
-#endif
+#include "filter_decoder.h"
extern LZMA_API lzma_ret
lzma_filter_flags_decode(
- lzma_options_filter *options, lzma_allocator *allocator,
+ lzma_filter *filter, lzma_allocator *allocator,
const uint8_t *in, size_t *in_pos, size_t in_size)
{
// Set the pointer to NULL so the caller can always safely free it.
- options->options = NULL;
+ filter->options = NULL;
// Filter ID
- return_if_error(lzma_vli_decode(&options->id, NULL,
+ return_if_error(lzma_vli_decode(&filter->id, NULL,
in, in_pos, in_size));
+ if (filter->id >= LZMA_FILTER_RESERVED_START)
+ return LZMA_DATA_ERROR;
+
// Size of Properties
- lzma_vli prop_size;
- return_if_error(lzma_vli_decode(&prop_size, NULL,
+ lzma_vli props_size;
+ return_if_error(lzma_vli_decode(&props_size, NULL,
in, in_pos, in_size));
- // Check that we have enough input.
- if (prop_size > in_size - *in_pos)
+ // Filter Properties
+ if (in_size - *in_pos < props_size)
return LZMA_DATA_ERROR;
- // Determine the function to decode the properties.
- lzma_ret (*get_properties)(lzma_options_filter *options,
- lzma_allocator *allocator, const uint8_t *props,
- size_t prop_size);
+ const lzma_ret ret = lzma_properties_decode(
+ filter, allocator, in + *in_pos, props_size);
- switch (options->id) {
-#ifdef HAVE_FILTER_SUBBLOCK
- case LZMA_FILTER_SUBBLOCK:
- get_properties = &properties_subblock;
- break;
-#endif
-#ifdef HAVE_FILTER_SIMPLE
-# ifdef HAVE_FILTER_X86
- case LZMA_FILTER_X86:
-# endif
-# ifdef HAVE_FILTER_POWERPC
- case LZMA_FILTER_POWERPC:
-# endif
-# ifdef HAVE_FILTER_IA64
- case LZMA_FILTER_IA64:
-# endif
-# ifdef HAVE_FILTER_ARM
- case LZMA_FILTER_ARM:
-# endif
-# ifdef HAVE_FILTER_ARMTHUMB
- case LZMA_FILTER_ARMTHUMB:
-# endif
-# ifdef HAVE_FILTER_SPARC
- case LZMA_FILTER_SPARC:
-# endif
- get_properties = &properties_simple;
- break;
-#endif
-#ifdef HAVE_FILTER_DELTA
- case LZMA_FILTER_DELTA:
- get_properties = &properties_delta;
- break;
-#endif
-#ifdef HAVE_FILTER_LZMA
- case LZMA_FILTER_LZMA:
- get_properties = &properties_lzma;
- break;
-#endif
- default:
- return LZMA_HEADER_ERROR;
- }
+ *in_pos += props_size;
- const uint8_t *props = in + *in_pos;
- *in_pos += prop_size;
- return get_properties(options, allocator, props, prop_size);
+ return ret;
}
//
///////////////////////////////////////////////////////////////////////////////
-#include "common.h"
-#include "lzma_encoder.h"
-#include "fastpos.h"
-
-
-/// Calculate the size of the Filter Properties field
-static lzma_ret
-get_properties_size(uint32_t *size, const lzma_options_filter *options)
-{
- lzma_ret ret = LZMA_OK;
-
- switch (options->id) {
-#ifdef HAVE_FILTER_SUBBLOCK
- case LZMA_FILTER_SUBBLOCK:
- *size = 0;
- break;
-#endif
-
-#ifdef HAVE_FILTER_SIMPLE
-# ifdef HAVE_FILTER_X86
- case LZMA_FILTER_X86:
-# endif
-# ifdef HAVE_FILTER_POWERPC
- case LZMA_FILTER_POWERPC:
-# endif
-# ifdef HAVE_FILTER_IA64
- case LZMA_FILTER_IA64:
-# endif
-# ifdef HAVE_FILTER_ARM
- case LZMA_FILTER_ARM:
-# endif
-# ifdef HAVE_FILTER_ARMTHUMB
- case LZMA_FILTER_ARMTHUMB:
-# endif
-# ifdef HAVE_FILTER_SPARC
- case LZMA_FILTER_SPARC:
-# endif
- if (options->options == NULL || ((const lzma_options_simple *)(
- options->options))->start_offset == 0)
- *size = 0;
- else
- *size = 4;
- break;
-#endif
-
-#ifdef HAVE_FILTER_DELTA
- case LZMA_FILTER_DELTA:
- *size = 1;
- break;
-#endif
-
-#ifdef HAVE_FILTER_LZMA
- case LZMA_FILTER_LZMA:
- *size = 2;
- break;
-#endif
-
- default:
- // Unknown filter - if the Filter ID is a proper VLI,
- // return LZMA_HEADER_ERROR instead of LZMA_PROG_ERROR,
- // because it's possible that we just don't have support
- // compiled in for the requested filter.
- ret = options->id <= LZMA_VLI_VALUE_MAX
- ? LZMA_HEADER_ERROR : LZMA_PROG_ERROR;
- break;
- }
-
- return ret;
-}
+#include "filter_encoder.h"
extern LZMA_API lzma_ret
-lzma_filter_flags_size(uint32_t *size, const lzma_options_filter *options)
-{
- // Get size of Filter Properties. This also validates the Filter ID.
- uint32_t prop_size;
- return_if_error(get_properties_size(&prop_size, options));
-
- // Calculate the size of the Filter ID and Size of Properties fields.
- // These cannot fail since get_properties_size() already succeeded.
- *size = lzma_vli_size(options->id) + lzma_vli_size(prop_size)
- + prop_size;
-
- return LZMA_OK;
-}
-
-
-#ifdef HAVE_FILTER_SIMPLE
-/// Encodes Filter Properties of the so called simple filters
-static lzma_ret
-properties_simple(uint8_t *out, size_t *out_pos, size_t out_size,
- const lzma_options_simple *options)
-{
- if (options == NULL || options->start_offset == 0)
- return LZMA_OK;
-
- if (out_size - *out_pos < 4)
- return LZMA_PROG_ERROR;
-
- integer_write_32(out + *out_pos, options->start_offset);
- *out_pos += 4;
-
- return LZMA_OK;
-}
-#endif
-
-
-#ifdef HAVE_FILTER_DELTA
-/// Encodes Filter Properties of the Delta filter
-static lzma_ret
-properties_delta(uint8_t *out, size_t *out_pos, size_t out_size,
- const lzma_options_delta *options)
-{
- if (options == NULL)
- return LZMA_PROG_ERROR;
-
- // It's possible that newer liblzma versions will support larger
- // distance values.
- if (options->distance < LZMA_DELTA_DISTANCE_MIN
- || options->distance > LZMA_DELTA_DISTANCE_MAX)
- return LZMA_HEADER_ERROR;
-
- if (out_size - *out_pos < 1)
- return LZMA_PROG_ERROR;
-
- out[*out_pos] = options->distance - LZMA_DELTA_DISTANCE_MIN;
- ++*out_pos;
-
- return LZMA_OK;
-}
-#endif
-
-
-#ifdef HAVE_FILTER_LZMA
-/// Encodes LZMA Properties and Dictionary Flags (two bytes)
-static lzma_ret
-properties_lzma(uint8_t *out, size_t *out_pos, size_t out_size,
- const lzma_options_lzma *options)
+lzma_filter_flags_size(uint32_t *size, const lzma_filter *filter)
{
- if (options == NULL)
- return LZMA_PROG_ERROR;
-
- if (out_size - *out_pos < 2)
- return LZMA_PROG_ERROR;
-
- // LZMA Properties
- if (lzma_lzma_encode_properties(options, out + *out_pos))
- return LZMA_HEADER_ERROR;
-
- ++*out_pos;
+ return_if_error(lzma_properties_size(size, filter));
- // Dictionary flags
- //
- // Dictionary size is encoded using similar encoding that is used
- // internally by LZMA.
- //
- // This won't work if dictionary size can be zero:
-# if LZMA_DICTIONARY_SIZE_MIN < 1
-# error LZMA_DICTIONARY_SIZE_MIN cannot be zero.
-# endif
-
- uint32_t d = options->dictionary_size;
-
- // Validate it:
- if (d < LZMA_DICTIONARY_SIZE_MIN || d > LZMA_DICTIONARY_SIZE_MAX)
- return LZMA_HEADER_ERROR;
-
- // Round up to to the next 2^n or 2^n + 2^(n - 1) depending on which
- // one is the next:
- --d;
- d |= d >> 2;
- d |= d >> 3;
- d |= d >> 4;
- d |= d >> 8;
- d |= d >> 16;
- ++d;
-
- // Get the highest two bits using the proper encoding:
- out[*out_pos] = get_pos_slot(d) - 24;
- ++*out_pos;
+ // lzma_properties_size() validates the Filter ID as a side-effect,
+ // so we know that it is a valid VLI.
+ *size += lzma_vli_size(filter->id) + lzma_vli_size(*size);
return LZMA_OK;
}
-#endif
extern LZMA_API lzma_ret
-lzma_filter_flags_encode(uint8_t *out, size_t *out_pos, size_t out_size,
- const lzma_options_filter *options)
+lzma_filter_flags_encode(const lzma_filter *filter,
+ uint8_t *out, size_t *out_pos, size_t out_size)
{
- // Minimum output is one byte (everything fits into Misc).
- // The caller should have checked that there is enough output space,
- // so we return LZMA_PROG_ERROR instead of LZMA_BUF_ERROR.
- if (*out_pos >= out_size)
- return LZMA_PROG_ERROR;
-
- // Get size of Filter Properties.
- uint32_t prop_size;
- return_if_error(get_properties_size(&prop_size, options));
-
// Filter ID
- return_if_error(lzma_vli_encode(options->id, NULL,
+ if (filter->id >= LZMA_FILTER_RESERVED_START)
+ return LZMA_HEADER_ERROR;
+
+ return_if_error(lzma_vli_encode(filter->id, NULL,
out, out_pos, out_size));
// Size of Properties
- return_if_error(lzma_vli_encode(prop_size, NULL,
+ uint32_t props_size;
+ return_if_error(lzma_properties_size(&props_size, filter));
+ return_if_error(lzma_vli_encode(props_size, NULL,
out, out_pos, out_size));
// Filter Properties
- lzma_ret ret;
- switch (options->id) {
-#ifdef HAVE_FILTER_SUBBLOCK
- case LZMA_FILTER_SUBBLOCK:
- assert(prop_size == 0);
- ret = LZMA_OK;
- break;
-#endif
-
-#ifdef HAVE_FILTER_SIMPLE
-# ifdef HAVE_FILTER_X86
- case LZMA_FILTER_X86:
-# endif
-# ifdef HAVE_FILTER_POWERPC
- case LZMA_FILTER_POWERPC:
-# endif
-# ifdef HAVE_FILTER_IA64
- case LZMA_FILTER_IA64:
-# endif
-# ifdef HAVE_FILTER_ARM
- case LZMA_FILTER_ARM:
-# endif
-# ifdef HAVE_FILTER_ARMTHUMB
- case LZMA_FILTER_ARMTHUMB:
-# endif
-# ifdef HAVE_FILTER_SPARC
- case LZMA_FILTER_SPARC:
-# endif
- ret = properties_simple(out, out_pos, out_size,
- options->options);
- break;
-#endif
-
-#ifdef HAVE_FILTER_DELTA
- case LZMA_FILTER_DELTA:
- ret = properties_delta(out, out_pos, out_size,
- options->options);
- break;
-#endif
+ if (out_size - *out_pos < props_size)
+ return LZMA_PROG_ERROR;
-#ifdef HAVE_FILTER_LZMA
- case LZMA_FILTER_LZMA:
- ret = properties_lzma(out, out_pos, out_size,
- options->options);
- break;
-#endif
+ return_if_error(lzma_properties_encode(filter, out + *out_pos));
- default:
- assert(0);
- ret = LZMA_PROG_ERROR;
- break;
- }
+ *out_pos += props_size;
- return ret;
+ return LZMA_OK;
}
index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
lzma_index **i)
{
+ lzma_next_coder_init(index_decoder_init, next, allocator);
+
if (i == NULL)
return LZMA_PROG_ERROR;
}
-/*
-extern lzma_ret
-lzma_index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- lzma_index **i)
-{
- lzma_next_coder_init(index_decoder_init, next, allocator, i);
-}
-*/
-
-
extern LZMA_API lzma_ret
lzma_index_decoder(lzma_stream *strm, lzma_index **i)
{
- lzma_next_strm_init(strm, index_decoder_init, i);
+ lzma_next_strm_init(index_decoder_init, strm, i);
strm->internal->supported_actions[LZMA_RUN] = true;
}
-static lzma_ret
-index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+extern lzma_ret
+lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
lzma_index *i)
{
+ lzma_next_coder_init(lzma_index_encoder_init, next, allocator);
+
if (i == NULL)
return LZMA_PROG_ERROR;
}
-extern lzma_ret
-lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- lzma_index *i)
-{
- lzma_next_coder_init(index_encoder_init, next, allocator, i);
-}
-
-
extern LZMA_API lzma_ret
lzma_index_encoder(lzma_stream *strm, lzma_index *i)
{
- lzma_next_strm_init(strm, index_encoder_init, i);
+ lzma_next_strm_init(lzma_index_encoder_init, strm, i);
strm->internal->supported_actions[LZMA_RUN] = true;
lzma_vli index_list_size;
/// Check calculated from Total Sizes and Uncompressed Sizes.
- lzma_check check;
+ lzma_check_state check;
} lzma_index_hash_info;
// Finish the hashes and compare them.
lzma_check_finish(&index_hash->blocks.check, LZMA_CHECK_BEST);
lzma_check_finish(&index_hash->records.check, LZMA_CHECK_BEST);
- if (memcmp(index_hash->blocks.check.buffer,
- index_hash->records.check.buffer,
- lzma_check_sizes[LZMA_CHECK_BEST]) != 0)
+ if (memcmp(index_hash->blocks.check.buffer.u8,
+ index_hash->records.check.buffer.u8,
+ lzma_check_size(LZMA_CHECK_BEST)) != 0)
return LZMA_DATA_ERROR;
// Finish the CRC32 calculation.
lzma_init_check();
-#if defined(HAVE_SMALL) && defined(HAVE_ENCODER) && defined(HAVE_FILTER_LZMA)
+#if defined(HAVE_SMALL) && defined(HAVE_ENCODER_LZMA)
lzma_rc_init();
#endif
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file memory_usage.c
-/// \brief Calculate rough amount of memory required by filters
-//
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "common.h"
-#include "lz_encoder.h"
-#include "lzma_literal.h"
-
-
-static uint64_t
-get_usage(const lzma_options_filter *filter, bool is_encoder)
-{
- uint64_t ret;
-
- switch (filter->id) {
- case LZMA_FILTER_X86:
- case LZMA_FILTER_POWERPC:
- case LZMA_FILTER_IA64:
- case LZMA_FILTER_ARM:
- case LZMA_FILTER_ARMTHUMB:
- case LZMA_FILTER_SPARC:
- case LZMA_FILTER_DELTA:
- // These don't require any significant amount of memory.
- ret = 0;
- break;
-
- case LZMA_FILTER_SUBBLOCK:
- if (is_encoder) {
- const lzma_options_subblock *options = filter->options;
- ret = options->subblock_data_size;
- } else {
- ret = 0;
- }
- break;
-
-#ifdef HAVE_FILTER_LZMA
- case LZMA_FILTER_LZMA: {
- const lzma_options_lzma *options = filter->options;
-
- // Literal coder - this can be signficant if both values are
- // big, or if sizeof(probability) is big.
- ret = literal_states(options->literal_context_bits,
- options->literal_pos_bits) * LIT_SIZE
- * sizeof(probability);
-
- // Dictionary base size
- ret += options->dictionary_size;
-
- if (is_encoder) {
-# ifdef HAVE_ENCODER
- // This is rough, but should be accurate enough
- // in practice.
- ret += options->dictionary_size / 2;
-
- uint32_t dummy1;
- uint32_t dummy2;
- uint32_t num_items;
- if (lzma_lz_encoder_hash_properties(
- options->match_finder,
- options->dictionary_size,
- &dummy1, &dummy2, &num_items))
- return UINT64_MAX;
-
- ret += (uint64_t)(num_items) * sizeof(uint32_t);
-# else
- return UINT64_MAX;
-# endif
- }
-
- break;
- }
-#endif
-
- default:
- return UINT64_MAX;
- }
-
- return ret;
-}
-
-
-extern LZMA_API uint32_t
-lzma_memory_usage(const lzma_options_filter *filters, lzma_bool is_encoder)
-{
- uint64_t usage = 0;
-
- for (size_t i = 0; filters[i].id != UINT64_MAX; ++i) {
- const uint64_t ret = get_usage(filters + i, is_encoder);
- if (ret == UINT64_MAX)
- return UINT32_MAX;
-
- usage += ret;
- }
-
- // Convert to mebibytes with rounding.
- return usage / (1024 * 1024) + (usage % (1024 * 1024) >= 512 ? 1 : 0);
-}
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file next_coder.c
-/// \brief Initializing and freeing the next coder in the chain
-//
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "common.h"
-
-extern lzma_ret
-lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator,
- const lzma_filter_info *filters)
-{
- lzma_ret ret = LZMA_OK;
-
- // Free the existing coder if it is different than the current one.
- if ((uintptr_t)(filters[0].init) != next->init)
- lzma_next_coder_end(next, allocator);
-
- if (filters[0].init != NULL) {
- // Initialize the new coder.
- ret = filters[0].init(next, allocator, filters);
-
- // Set the init function pointer if initialization was
- // successful. next->code and next->end are set by the
- // initialization function itself.
- if (ret == LZMA_OK) {
- next->init = (uintptr_t)(filters[0].init);
- assert(next->code != NULL);
- assert(next->end != NULL);
- } else {
- lzma_next_coder_end(next, allocator);
- }
- }
-
- return ret;
-}
-
-
-extern void
-lzma_next_coder_end(lzma_next_coder *next, lzma_allocator *allocator)
-{
- if (next != NULL) {
- if (next->end != NULL)
- next->end(next->coder, allocator);
-
- // Reset the variables so the we don't accidentally think
- // that it is an already initialized coder.
- *next = LZMA_NEXT_CODER_INIT;
- }
-
- return;
-}
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file raw_common.c
-/// \brief Stuff shared between raw encoder and raw decoder
-//
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "raw_common.h"
-
-
-static lzma_ret
-validate_options(const lzma_options_filter *options, size_t *count)
-{
- if (options == NULL)
- return LZMA_PROG_ERROR;
-
- // Number of non-last filters that may change the size of the data
- // significantly (that is, more than 1-2 % or so).
- size_t change = 0;
-
- // True if the last filter in the given chain is actually usable as
- // the last filter. Only filters that support embedding End of Payload
- // Marker can be used as the last filter in the chain.
- bool last_ok = false;
-
- size_t i;
- for (i = 0; options[i].id != LZMA_VLI_VALUE_UNKNOWN; ++i) {
- switch (options[i].id) {
- // Not #ifdeffing these for simplicity.
- case LZMA_FILTER_X86:
- case LZMA_FILTER_POWERPC:
- case LZMA_FILTER_IA64:
- case LZMA_FILTER_ARM:
- case LZMA_FILTER_ARMTHUMB:
- case LZMA_FILTER_SPARC:
- case LZMA_FILTER_DELTA:
- // These don't change the size of the data and cannot
- // be used as the last filter in the chain.
- last_ok = false;
- break;
-
-#ifdef HAVE_FILTER_SUBBLOCK
- case LZMA_FILTER_SUBBLOCK:
- last_ok = true;
- ++change;
- break;
-#endif
-
-#ifdef HAVE_FILTER_LZMA
- case LZMA_FILTER_LZMA:
- last_ok = true;
- break;
-#endif
-
- default:
- return LZMA_HEADER_ERROR;
- }
- }
-
- // There must be 1-4 filters and the last filter must be usable as
- // the last filter in the chain.
- if (i == 0 || i > 4 || !last_ok)
- return LZMA_HEADER_ERROR;
-
- // At maximum of two non-last filters are allowed to change the
- // size of the data.
- if (change > 2)
- return LZMA_HEADER_ERROR;
-
- *count = i;
- return LZMA_OK;
-}
-
-
-extern lzma_ret
-lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
- const lzma_options_filter *options,
- lzma_init_function (*get_function)(lzma_vli id),
- bool is_encoder)
-{
- // Do some basic validation and get the number of filters.
- size_t count;
- return_if_error(validate_options(options, &count));
-
- // Set the filter functions and copy the options pointer.
- lzma_filter_info filters[count + 1];
- if (is_encoder) {
- for (size_t i = 0; i < count; ++i) {
- // The order of the filters is reversed in the
- // encoder. It allows more efficient handling
- // of the uncompressed data.
- const size_t j = count - i - 1;
-
- filters[j].init = get_function(options[i].id);
- if (filters[j].init == NULL)
- return LZMA_HEADER_ERROR;
-
- filters[j].options = options[i].options;
- }
- } else {
- for (size_t i = 0; i < count; ++i) {
- filters[i].init = get_function(options[i].id);
- if (filters[i].init == NULL)
- return LZMA_HEADER_ERROR;
-
- filters[i].options = options[i].options;
- }
- }
-
- // Terminate the array.
- filters[count].init = NULL;
-
- // Initialize the filters.
- return lzma_next_filter_init(next, allocator, filters);
-}
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file raw_decoder.c
-/// \brief Raw decoder initialization API
-//
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "raw_decoder.h"
-#include "simple_coder.h"
-#include "subblock_decoder.h"
-#include "subblock_decoder_helper.h"
-#include "delta_decoder.h"
-#include "lzma_decoder.h"
-
-
-static lzma_init_function
-get_function(lzma_vli id)
-{
- switch (id) {
-#ifdef HAVE_FILTER_SUBBLOCK
- case LZMA_FILTER_SUBBLOCK:
- return &lzma_subblock_decoder_init;
-#endif
-
-#ifdef HAVE_FILTER_X86
- case LZMA_FILTER_X86:
- return &lzma_simple_x86_decoder_init;
-#endif
-
-#ifdef HAVE_FILTER_POWERPC
- case LZMA_FILTER_POWERPC:
- return &lzma_simple_powerpc_decoder_init;
-#endif
-
-#ifdef HAVE_FILTER_IA64
- case LZMA_FILTER_IA64:
- return &lzma_simple_ia64_decoder_init;
-#endif
-
-#ifdef HAVE_FILTER_ARM
- case LZMA_FILTER_ARM:
- return &lzma_simple_arm_decoder_init;
-#endif
-
-#ifdef HAVE_FILTER_ARMTHUMB
- case LZMA_FILTER_ARMTHUMB:
- return &lzma_simple_armthumb_decoder_init;
-#endif
-
-#ifdef HAVE_FILTER_SPARC
- case LZMA_FILTER_SPARC:
- return &lzma_simple_sparc_decoder_init;
-#endif
-
-#ifdef HAVE_FILTER_DELTA
- case LZMA_FILTER_DELTA:
- return &lzma_delta_decoder_init;
-#endif
-
-#ifdef HAVE_FILTER_LZMA
- case LZMA_FILTER_LZMA:
- return &lzma_lzma_decoder_init;
-#endif
-
-#ifdef HAVE_FILTER_SUBBLOCK
- case LZMA_FILTER_SUBBLOCK_HELPER:
- return &lzma_subblock_decoder_helper_init;
-#endif
- }
-
- return NULL;
-}
-
-
-extern lzma_ret
-lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- const lzma_options_filter *options)
-{
- const lzma_ret ret = lzma_raw_coder_init(next, allocator,
- options, &get_function, false);
-
- if (ret != LZMA_OK)
- lzma_next_coder_end(next, allocator);
-
- return ret;
-}
-
-
-extern LZMA_API lzma_ret
-lzma_raw_decoder(lzma_stream *strm, const lzma_options_filter *options)
-{
- return_if_error(lzma_strm_init(strm));
-
- strm->internal->supported_actions[LZMA_RUN] = true;
- strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
-
- const lzma_ret ret = lzma_raw_coder_init(&strm->internal->next,
- strm->allocator, options, &get_function, false);
-
- if (ret != LZMA_OK)
- lzma_end(strm);
-
- return ret;
-}
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file raw_encoder.c
-/// \brief Raw encoder initialization API
-//
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "raw_encoder.h"
-#include "simple_coder.h"
-#include "subblock_encoder.h"
-#include "delta_encoder.h"
-#include "lzma_encoder.h"
-
-
-static lzma_init_function
-get_function(lzma_vli id)
-{
- switch (id) {
-#ifdef HAVE_FILTER_SUBBLOCK
- case LZMA_FILTER_SUBBLOCK:
- return &lzma_subblock_encoder_init;
-#endif
-
-#ifdef HAVE_FILTER_X86
- case LZMA_FILTER_X86:
- return &lzma_simple_x86_encoder_init;
-#endif
-
-#ifdef HAVE_FILTER_POWERPC
- case LZMA_FILTER_POWERPC:
- return &lzma_simple_powerpc_encoder_init;
-#endif
-
-#ifdef HAVE_FILTER_IA64
- case LZMA_FILTER_IA64:
- return &lzma_simple_ia64_encoder_init;
-#endif
-
-#ifdef HAVE_FILTER_ARM
- case LZMA_FILTER_ARM:
- return &lzma_simple_arm_encoder_init;
-#endif
-
-#ifdef HAVE_FILTER_ARMTHUMB
- case LZMA_FILTER_ARMTHUMB:
- return &lzma_simple_armthumb_encoder_init;
-#endif
-
-#ifdef HAVE_FILTER_SPARC
- case LZMA_FILTER_SPARC:
- return &lzma_simple_sparc_encoder_init;
-#endif
-
-#ifdef HAVE_FILTER_DELTA
- case LZMA_FILTER_DELTA:
- return &lzma_delta_encoder_init;
-#endif
-
-#ifdef HAVE_FILTER_LZMA
- case LZMA_FILTER_LZMA:
- return &lzma_lzma_encoder_init;
-#endif
- }
-
- return NULL;
-}
-
-
-extern lzma_ret
-lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- const lzma_options_filter *options)
-{
- const lzma_ret ret = lzma_raw_coder_init(next, allocator,
- options, &get_function, true);
-
- if (ret != LZMA_OK)
- lzma_next_coder_end(next, allocator);
-
- return ret;
-}
-
-
-extern LZMA_API lzma_ret
-lzma_raw_encoder(lzma_stream *strm, const lzma_options_filter *options)
-{
- return_if_error(lzma_strm_init(strm));
-
- strm->internal->supported_actions[LZMA_RUN] = true;
- strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
- strm->internal->supported_actions[LZMA_FINISH] = true;
-
- const lzma_ret ret = lzma_raw_coder_init(&strm->internal->next,
- strm->allocator, options, &get_function, true);
-
- if (ret != LZMA_OK)
- lzma_end(strm);
-
- return ret;
-}
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file stream_common.c
-/// \brief Common stuff for Stream coders
-//
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "stream_common.h"
-
-const uint8_t lzma_header_magic[6] = { 0xFF, 0x4C, 0x5A, 0x4D, 0x41, 0x00 };
-const uint8_t lzma_footer_magic[2] = { 0x59, 0x5A };
//
///////////////////////////////////////////////////////////////////////////////
-#include "stream_common.h"
#include "stream_decoder.h"
+#include "stream_flags_common.h"
#include "check.h"
#include "stream_flags_decoder.h"
#include "block_decoder.h"
SEQ_BLOCK,
SEQ_INDEX,
SEQ_STREAM_FOOTER,
+ SEQ_STREAM_PADDING,
} sequence;
/// Block or Metadata decoder. This takes little memory and the same
/// Block options decoded by the Block Header decoder and used by
/// the Block decoder.
- lzma_options_block block_options;
+ lzma_block block_options;
/// Stream Flags from Stream Header
lzma_stream_flags stream_flags;
/// with O(1) memory usage.
lzma_index_hash *index_hash;
- /// Write position in buffer[]
- size_t buffer_pos;
+ /// Memory usage limit
+ uint64_t memlimit;
+
+ /// If true, LZMA_NO_CHECK is returned if the Stream has
+ /// no integrity check.
+ bool warn_no_check;
+
+ /// If true, LZMA_UNSUPPORTED_CHECK is returned if the Stream has
+ /// an integrity check that isn't supported by this liblzma build.
+ bool warn_unsupported_check;
+
+ /// If true, LZMA_SEE_CHECK is returned after decoding Stream Header.
+ bool tell_check;
+
+ /// If true, we will decode concatenated Streams that possibly have
+ /// Stream Padding between or after them. LZMA_STREAM_END is returned
+ /// once the application isn't giving us any new input and we aren't
+ /// in the middle of a Stream and possible Stream Padding is a
+ /// multiple of four bytes. FIXME
+ bool concatenated;
+
+ /// When decoding concatenated Streams, this is true as long as we
+ /// are decoding the first Stream. This is needed to avoid misleading
+ /// LZMA_FORMAT_ERROR in case the later Streams don't have valid magic
+ /// bytes.
+ bool first_stream;
+
+ /// Write position in buffer[] and position in Stream Padding
+ size_t pos;
/// Buffer to hold Stream Header, Block Header, and Stream Footer.
/// Block Header has biggest maximum size.
static lzma_ret
+stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator)
+{
+ // Initialize the Index hash used to verify the Index.
+ coder->index_hash = lzma_index_hash_init(coder->index_hash, allocator);
+ if (coder->index_hash == NULL)
+ return LZMA_MEM_ERROR;
+
+ // Reset the rest of the variables.
+ coder->sequence = SEQ_STREAM_HEADER;
+ coder->block_options.filters = NULL;
+ coder->pos = 0;
+
+ return LZMA_OK;
+}
+
+
+static lzma_ret
stream_decode(lzma_coder *coder, lzma_allocator *allocator,
const uint8_t *restrict in, size_t *restrict in_pos,
size_t in_size, uint8_t *restrict out,
{
// When decoding the actual Block, it may be able to produce more
// output even if we don't give it any new input.
- while (*out_pos < out_size && (*in_pos < in_size
- || coder->sequence == SEQ_BLOCK))
+ while (true)
switch (coder->sequence) {
case SEQ_STREAM_HEADER: {
// Copy the Stream Header to the internal buffer.
- bufcpy(in, in_pos, in_size, coder->buffer, &coder->buffer_pos,
+ lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos,
LZMA_STREAM_HEADER_SIZE);
// Return if we didn't get the whole Stream Header yet.
- if (coder->buffer_pos < LZMA_STREAM_HEADER_SIZE)
+ if (coder->pos < LZMA_STREAM_HEADER_SIZE)
return LZMA_OK;
- coder->buffer_pos = 0;
+ coder->pos = 0;
// Decode the Stream Header.
- return_if_error(lzma_stream_header_decode(
- &coder->stream_flags, coder->buffer));
+ const lzma_ret ret = lzma_stream_header_decode(
+ &coder->stream_flags, coder->buffer);
+ if (ret != LZMA_OK)
+ return ret == LZMA_FORMAT_ERROR && !coder->first_stream
+ ? LZMA_DATA_ERROR : ret;
// Copy the type of the Check so that Block Header and Block
// decoders see it.
coder->block_options.check = coder->stream_flags.check;
- // Even if we return LZMA_UNSUPPORTED_CHECK below, we want
+ // Even if we return LZMA_*_CHECK below, we want
// to continue from Block Header decoding.
coder->sequence = SEQ_BLOCK_HEADER;
- // Detect if the Check type is supported and give appropriate
- // warning if it isn't. We don't warn every time a new Block
- // is started.
- if (!lzma_available_checks[coder->block_options.check])
+ // Detect if there's no integrity check or if it is
+ // unsupported if those were requested by the application.
+ if (coder->warn_no_check && coder->stream_flags.check
+ == LZMA_CHECK_NONE)
+ return LZMA_NO_CHECK;
+
+ if (coder->warn_unsupported_check
+ && !lzma_check_is_supported(
+ coder->stream_flags.check))
return LZMA_UNSUPPORTED_CHECK;
+ if (coder->tell_check)
+ return LZMA_SEE_CHECK;
+
break;
}
case SEQ_BLOCK_HEADER: {
- if (coder->buffer_pos == 0) {
+ if (*in_pos >= in_size)
+ return LZMA_OK;
+
+ if (coder->pos == 0) {
// Detect if it's Index.
if (in[*in_pos] == 0x00) {
coder->sequence = SEQ_INDEX;
}
// Copy the Block Header to the internal buffer.
- bufcpy(in, in_pos, in_size, coder->buffer, &coder->buffer_pos,
+ lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos,
coder->block_options.header_size);
// Return if we didn't get the whole Block Header yet.
- if (coder->buffer_pos < coder->block_options.header_size)
+ if (coder->pos < coder->block_options.header_size)
return LZMA_OK;
- coder->buffer_pos = 0;
+ coder->pos = 0;
// Set up a buffer to hold the filter chain. Block Header
// decoder will initialize all members of this array so
// we don't need to do it here.
- lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
+ lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
coder->block_options.filters = filters;
// Decode the Block Header.
return_if_error(lzma_block_header_decode(&coder->block_options,
allocator, coder->buffer));
- // Initialize the Block decoder.
- const lzma_ret ret = lzma_block_decoder_init(
- &coder->block_decoder,
- allocator, &coder->block_options);
+ // Check the memory usage limit.
+ const uint64_t memusage = lzma_memusage_decoder(filters);
+ lzma_ret ret;
+
+ if (memusage == UINT64_MAX) {
+ // One or more unknown Filter IDs.
+ ret = LZMA_HEADER_ERROR;
+ } else if (memusage > coder->memlimit) {
+ // The chain would need too much memory.
+ ret = LZMA_MEMLIMIT_ERROR;
+ } else {
+ // Memory usage is OK. Initialize the Block decoder.
+ ret = lzma_block_decoder_init(
+ &coder->block_decoder,
+ allocator, &coder->block_options);
+ }
// Free the allocated filter options since they are needed
// only to initialize the Block decoder.
coder->block_options.filters = NULL;
- // Check if Block enocoder initialization succeeded. Don't
- // warn about unsupported check anymore since we did it
- // earlier if it was needed.
- if (ret != LZMA_OK && ret != LZMA_UNSUPPORTED_CHECK)
+ // Check if memory usage calculation and Block enocoder
+ // initialization succeeded.
+ if (ret != LZMA_OK)
return ret;
coder->sequence = SEQ_BLOCK;
}
case SEQ_BLOCK: {
- lzma_ret ret = coder->block_decoder.code(
+ const lzma_ret ret = coder->block_decoder.code(
coder->block_decoder.coder, allocator,
in, in_pos, in_size, out, out_pos, out_size,
action);
}
case SEQ_INDEX: {
+ // If we don't have any input, don't call
+ // lzma_index_hash_decode() since it would return
+ // LZMA_BUF_ERROR, which we must not do here.
+ if (*in_pos >= in_size)
+ return LZMA_OK;
+
// Decode the Index and compare it to the hash calculated
// from the sizes of the Blocks (if any).
const lzma_ret ret = lzma_index_hash_decode(coder->index_hash,
case SEQ_STREAM_FOOTER:
// Copy the Stream Footer to the internal buffer.
- bufcpy(in, in_pos, in_size, coder->buffer, &coder->buffer_pos,
+ lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos,
LZMA_STREAM_HEADER_SIZE);
// Return if we didn't get the whole Stream Footer yet.
- if (coder->buffer_pos < LZMA_STREAM_HEADER_SIZE)
+ if (coder->pos < LZMA_STREAM_HEADER_SIZE)
return LZMA_OK;
+ coder->pos = 0;
+
// Decode the Stream Footer.
+ // FIXME LZMA_FORMAT_ERROR doesn't make sense here.
lzma_stream_flags footer_flags;
return_if_error(lzma_stream_footer_decode(
&footer_flags, coder->buffer));
&footer_flags))
return LZMA_DATA_ERROR;
- return LZMA_STREAM_END;
+ if (!coder->concatenated)
+ return LZMA_STREAM_END;
+
+ coder->sequence = SEQ_STREAM_PADDING;
+ break;
+
+ case SEQ_STREAM_PADDING:
+ assert(coder->concatenated);
+
+ while (true) {
+ if (*in_pos >= in_size) {
+ // Unless LZMA_FINISH was used, we cannot
+ // know if there's more input coming later.
+ if (action != LZMA_FINISH)
+ return LZMA_OK;
+
+ // Stream Padding must be a multiple of
+ // four bytes.
+ return coder->pos == 0
+ ? LZMA_STREAM_END
+ : LZMA_DATA_ERROR;
+ }
+
+ if (in[*in_pos] != 0x00) {
+ if (coder->pos != 0) {
+ // Stream Padding is not a multiple of
+ // four bytes.
+ ++*in_pos;
+ return LZMA_DATA_ERROR;
+ }
+
+ // Prepare to decode the next Stream.
+ return_if_error(stream_decoder_reset(
+ coder, allocator));
+ break;
+ }
+
+ ++*in_pos;
+ coder->pos = (coder->pos + 1) & 3;
+ }
+
+ break;
default:
assert(0);
static void
stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->block_decoder, allocator);
+ lzma_next_end(&coder->block_decoder, allocator);
lzma_index_hash_end(coder->index_hash, allocator);
lzma_free(coder, allocator);
return;
}
-static lzma_ret
-stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator)
+static lzma_check
+stream_decoder_see_check(const lzma_coder *coder)
{
+ return coder->stream_flags.check;
+}
+
+
+extern lzma_ret
+lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ uint64_t memlimit, uint32_t flags)
+{
+ lzma_next_coder_init(lzma_stream_decoder_init, next, allocator);
+
+ if (flags & ~LZMA_SUPPORTED_FLAGS)
+ return LZMA_HEADER_ERROR;
+
if (next->coder == NULL) {
next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
if (next->coder == NULL)
next->code = &stream_decode;
next->end = &stream_decoder_end;
+ next->see_check = &stream_decoder_see_check;
next->coder->block_decoder = LZMA_NEXT_CODER_INIT;
next->coder->index_hash = NULL;
}
- // Initialize the Index hash used to verify the Index.
- next->coder->index_hash = lzma_index_hash_init(
- next->coder->index_hash, allocator);
- if (next->coder->index_hash == NULL)
- return LZMA_MEM_ERROR;
-
- // Reset the rest of the variables.
- next->coder->sequence = SEQ_STREAM_HEADER;
- next->coder->block_options.filters = NULL;
- next->coder->buffer_pos = 0;
+ next->coder->memlimit = memlimit;
+ next->coder->warn_no_check = (flags & LZMA_WARN_NO_CHECK) != 0;
+ next->coder->warn_unsupported_check
+ = (flags & LZMA_WARN_UNSUPPORTED_CHECK) != 0;
+ next->coder->tell_check = (flags & LZMA_TELL_CHECK) != 0;
+ next->coder->concatenated
+ = (flags & LZMA_CONCATENATED) != 0;
- return LZMA_OK;
-}
-
-
-extern lzma_ret
-lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator)
-{
- lzma_next_coder_init0(stream_decoder_init, next, allocator);
+ return stream_decoder_reset(next->coder, allocator);
}
extern LZMA_API lzma_ret
-lzma_stream_decoder(lzma_stream *strm)
+lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags)
{
- lzma_next_strm_init0(strm, stream_decoder_init);
+ lzma_next_strm_init(lzma_stream_decoder_init, strm, memlimit, flags);
strm->internal->supported_actions[LZMA_RUN] = true;
- strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
+// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; // FIXME
+ strm->internal->supported_actions[LZMA_FINISH] = true;
return LZMA_OK;
}
#include "common.h"
-extern lzma_ret lzma_stream_decoder_init(
- lzma_next_coder *next, lzma_allocator *allocator);
+extern lzma_ret lzma_stream_decoder_init(lzma_next_coder *next,
+ lzma_allocator *allocator, uint64_t memlimit, uint32_t flags);
#endif
//
///////////////////////////////////////////////////////////////////////////////
-#include "stream_common.h"
#include "stream_encoder.h"
+#include "stream_flags_common.h"
#include "block_encoder.h"
#include "index_encoder.h"
lzma_next_coder block_encoder;
/// Options for the Block encoder
- lzma_options_block block_options;
+ lzma_block block_options;
/// Index encoder. This is separate from Block encoder, because this
/// doesn't take much memory, and when encoding multiple Streams
case SEQ_STREAM_HEADER:
case SEQ_BLOCK_HEADER:
case SEQ_STREAM_FOOTER:
- bufcpy(coder->buffer, &coder->buffer_pos, coder->buffer_size,
- out, out_pos, out_size);
+ lzma_bufcpy(coder->buffer, &coder->buffer_pos,
+ coder->buffer_size, out, out_pos, out_size);
if (coder->buffer_pos < coder->buffer_size)
return LZMA_OK;
static void
stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->block_encoder, allocator);
- lzma_next_coder_end(&coder->index_encoder, allocator);
+ lzma_next_end(&coder->block_encoder, allocator);
+ lzma_next_end(&coder->index_encoder, allocator);
lzma_index_end(coder->index, allocator);
lzma_free(coder, allocator);
return;
}
-static lzma_ret
-stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- const lzma_options_filter *filters, lzma_check_type check)
+extern lzma_ret
+lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter *filters, lzma_check check)
{
+ lzma_next_coder_init(lzma_stream_encoder_init, next, allocator);
+
if (filters == NULL)
return LZMA_PROG_ERROR;
// Basic initializations
next->coder->sequence = SEQ_STREAM_HEADER;
next->coder->block_options.check = check;
- next->coder->block_options.filters = (lzma_options_filter *)(filters);
+ next->coder->block_options.filters = (lzma_filter *)(filters);
// Initialize the Index
next->coder->index = lzma_index_init(next->coder->index, allocator);
}
-extern lzma_ret
-lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- const lzma_options_filter *filters, lzma_check_type check)
-{
- lzma_next_coder_init(stream_encoder_init, next, allocator,
- filters, check);
-}
-
-
extern LZMA_API lzma_ret
lzma_stream_encoder(lzma_stream *strm,
- const lzma_options_filter *filters, lzma_check_type check)
+ const lzma_filter *filters, lzma_check check)
{
- lzma_next_strm_init(strm, stream_encoder_init, filters, check);
+ lzma_next_strm_init(lzma_stream_encoder_init, strm, filters, check);
strm->internal->supported_actions[LZMA_RUN] = true;
strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
extern lzma_ret lzma_stream_encoder_init(
lzma_next_coder *next, lzma_allocator *allocator,
- const lzma_options_filter *filters, lzma_check_type check);
+ const lzma_filter *filters, lzma_check check);
#endif
///////////////////////////////////////////////////////////////////////////////
//
-/// \file stream_flags_equal.c
-/// \brief Compare Stream Header and Stream Footer
+/// \file stream_flags_common.c
+/// \brief Common stuff for Stream flags coders
//
-// Copyright (C) 2008 Lasse Collin
+// Copyright (C) 2007-2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
//
///////////////////////////////////////////////////////////////////////////////
-#include "common.h"
+#include "stream_flags_common.h"
+
+
+const uint8_t lzma_header_magic[6] = { 0xFF, 0x4C, 0x5A, 0x4D, 0x41, 0x00 };
+const uint8_t lzma_footer_magic[2] = { 0x59, 0x5A };
extern LZMA_API lzma_bool
-lzma_stream_flags_equal(const lzma_stream_flags *a, lzma_stream_flags *b)
+lzma_stream_flags_equal(const lzma_stream_flags *a, const lzma_stream_flags *b)
{
if (a->check != b->check)
return false;
///////////////////////////////////////////////////////////////////////////////
//
-/// \file stream_common.h
-/// \brief Common stuff for Stream coders
+/// \file stream_flags_common.h
+/// \brief Common stuff for Stream flags coders
//
// Copyright (C) 2007 Lasse Collin
//
//
///////////////////////////////////////////////////////////////////////////////
-#ifndef LZMA_STREAM_COMMON_H
-#define LZMA_STREAM_COMMON_H
+#ifndef LZMA_STREAM_FLAGS_COMMON_H
+#define LZMA_STREAM_FLAGS_COMMON_H
#include "common.h"
//
///////////////////////////////////////////////////////////////////////////////
-#include "stream_common.h"
+#include "stream_flags_common.h"
static bool
//
///////////////////////////////////////////////////////////////////////////////
-#include "stream_common.h"
+#include "stream_flags_common.h"
static bool
{
// If we haven't been given vli_pos, work in single-call mode.
size_t vli_pos_internal = 0;
- if (vli_pos == NULL)
+ if (vli_pos == NULL) {
vli_pos = &vli_pos_internal;
-
- // Initialize *vli when starting to decode a new integer.
- if (*vli_pos == 0)
*vli = 0;
- // Validate the arguments.
- if (*vli_pos >= LZMA_VLI_BYTES_MAX || *in_pos >= in_size
- || (*vli >> (*vli_pos * 7)) != 0)
- return LZMA_PROG_ERROR;;
+ // If there's no input, use LZMA_DATA_ERROR. This way it is
+ // easy to decode VLIs from buffers that have known size,
+ // and get the correct error code in case the buffer is
+ // too short.
+ if (*in_pos >= in_size)
+ return LZMA_DATA_ERROR;
+
+ } else {
+ // Initialize *vli when starting to decode a new integer.
+ if (*vli_pos == 0)
+ *vli = 0;
+
+ // Validate the arguments.
+ if (*vli_pos >= LZMA_VLI_BYTES_MAX
+ || (*vli >> (*vli_pos * 7)) != 0)
+ return LZMA_PROG_ERROR;;
+
+ if (*in_pos >= in_size)
+ return LZMA_BUF_ERROR;
+ }
do {
// Read the next byte.
vli_pos = &vli_pos_internal;
// Validate the arguments.
- if (*vli_pos >= LZMA_VLI_BYTES_MAX || *out_pos >= out_size
- || vli > LZMA_VLI_VALUE_MAX)
+ if (*vli_pos >= LZMA_VLI_BYTES_MAX || vli > LZMA_VLI_VALUE_MAX)
return LZMA_PROG_ERROR;
+ if (*out_pos >= out_size)
+ return LZMA_BUF_ERROR;
+
// Write the non-last bytes in a loop.
while ((vli >> (*vli_pos * 7)) >= 0x80) {
out[*out_pos] = (uint8_t)(vli >> (*vli_pos * 7)) | 0x80;
return vli_pos == &vli_pos_internal ? LZMA_OK : LZMA_STREAM_END;
}
-
-
-extern LZMA_API uint32_t
-lzma_vli_size(lzma_vli vli)
-{
- if (vli > LZMA_VLI_VALUE_MAX)
- return 0;
-
- uint32_t i = 0;
- do {
- vli >>= 7;
- ++i;
- } while (vli != 0);
-
- assert(i <= LZMA_VLI_BYTES_MAX);
- return i;
-}
///////////////////////////////////////////////////////////////////////////////
//
-/// \file version.c
-/// \brief liblzma version number
+/// \file vli_size.c
+/// \brief Calculates the encoded size of a variable-length integer
//
-// Copyright (C) 2007 Lasse Collin
+// Copyright (C) 2007-2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
#include "common.h"
-LZMA_API const uint32_t lzma_version_number = LZMA_VERSION;
+extern LZMA_API uint32_t
+lzma_vli_size(lzma_vli vli)
+{
+ if (vli > LZMA_VLI_VALUE_MAX)
+ return 0;
-LZMA_API const char *const lzma_version_string = PACKAGE_VERSION;
+ uint32_t i = 0;
+ do {
+ vli >>= 7;
+ ++i;
+ } while (vli != 0);
+
+ assert(i <= LZMA_VLI_BYTES_MAX);
+ return i;
+}
--- /dev/null
+##
+## Copyright (C) 2008 Lasse Collin
+##
+## This library is free software; you can redistribute it and/or
+## modify it under the terms of the GNU Lesser General Public
+## License as published by the Free Software Foundation; either
+## version 2.1 of the License, or (at your option) any later version.
+##
+## This library is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## Lesser General Public License for more details.
+##
+
+noinst_LTLIBRARIES = libdelta.la
+libdelta_la_CPPFLAGS = \
+ -I@top_srcdir@/src/liblzma/api \
+ -I@top_srcdir@/src/liblzma/common
+
+libdelta_la_SOURCES = \
+ delta_common.c \
+ delta_common.h
+
+if COND_ENCODER_DELTA
+libdelta_la_SOURCES += \
+ delta_encoder.c \
+ delta_encoder.h
+endif
+
+if COND_DECODER_DELTA
+libdelta_la_SOURCES += \
+ delta_decoder.c \
+ delta_decoder.h
+endif
static void
delta_coder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->next, allocator);
+ lzma_next_end(&coder->next, allocator);
lzma_free(coder, allocator);
return;
}
{
return lzma_delta_coder_init(next, allocator, filters, &delta_decode);
}
+
+
+extern lzma_ret
+lzma_delta_props_decode(void **options, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size)
+{
+ if (props_size != 1)
+ return LZMA_HEADER_ERROR;
+
+ lzma_options_delta *opt
+ = lzma_alloc(sizeof(lzma_options_delta), allocator);
+ if (opt == NULL)
+ return LZMA_MEM_ERROR;
+
+ opt->type = LZMA_DELTA_TYPE_BYTE;
+ opt->distance = props[0] + 1;
+
+ *options = opt;
+
+ return LZMA_OK;
+}
extern lzma_ret lzma_delta_decoder_init(lzma_next_coder *next,
lzma_allocator *allocator, const lzma_filter_info *filters);
+extern lzma_ret lzma_delta_props_decode(
+ void **options, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size);
+
#endif
{
return lzma_delta_coder_init(next, allocator, filters, &delta_encode);
}
+
+
+extern lzma_ret
+lzma_delta_props_encode(const void *options, uint8_t *out)
+{
+ if (options == NULL)
+ return LZMA_PROG_ERROR;
+
+ const lzma_options_delta *opt = options;
+
+ // It's possible that newer liblzma versions will support larger
+ // distance values.
+ if (opt->type != LZMA_DELTA_TYPE_BYTE
+ || opt->distance < LZMA_DELTA_DISTANCE_MIN
+ || opt->distance > LZMA_DELTA_DISTANCE_MAX)
+ return LZMA_HEADER_ERROR;
+
+ out[0] = opt->distance - LZMA_DELTA_DISTANCE_MIN;
+
+ return LZMA_OK;
+}
extern lzma_ret lzma_delta_encoder_init(lzma_next_coder *next,
lzma_allocator *allocator, const lzma_filter_info *filters);
+extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out);
+
#endif
liblz_la_SOURCES =
-if COND_MAIN_ENCODER
+if COND_ENCODER_LZ
liblz_la_SOURCES += \
lz_encoder.c \
lz_encoder.h \
- lz_encoder_private.h \
- match_c.h \
- match_h.h
-
-if COND_MF_HC3
-liblz_la_SOURCES += hc3.c hc3.h
-liblz_la_CPPFLAGS += -DHAVE_HC3
-endif
-
-if COND_MF_HC4
-liblz_la_SOURCES += hc4.c hc4.h
-liblz_la_CPPFLAGS += -DHAVE_HC4
-endif
-
-if COND_MF_BT2
-liblz_la_SOURCES += bt2.c bt2.h
-liblz_la_CPPFLAGS += -DHAVE_BT2
-endif
-
-if COND_MF_BT3
-liblz_la_SOURCES += bt3.c bt3.h
-liblz_la_CPPFLAGS += -DHAVE_BT3
-endif
-
-if COND_MF_BT4
-liblz_la_SOURCES += bt4.c bt4.h
-liblz_la_CPPFLAGS += -DHAVE_BT4
-endif
-
+ lz_encoder_hash.h \
+ lz_encoder_mf.c
endif
-if COND_MAIN_DECODER
+if COND_DECODER_LZ
liblz_la_SOURCES += \
lz_decoder.c \
lz_decoder.h
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file bt2.c
-/// \brief Binary Tree 2
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "bt2.h"
-
-#undef IS_HASH_CHAIN
-#undef HASH_ARRAY_2
-#undef HASH_ARRAY_3
-
-#include "match_c.h"
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file bt2.h
-/// \brief Binary Tree 2
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#ifndef LZMA_BT2_H
-#define LZMA_BT2_H
-
-#undef LZMA_MATCH_FINDER_NAME_LOWER
-#undef LZMA_MATCH_FINDER_NAME_UPPER
-#define LZMA_MATCH_FINDER_NAME_LOWER bt2
-#define LZMA_MATCH_FINDER_NAME_UPPER BT2
-
-#include "match_h.h"
-
-#endif
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file bt3.c
-/// \brief Binary Tree 3
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "bt3.h"
-
-#undef IS_HASH_CHAIN
-#undef HASH_ARRAY_2
-#undef HASH_ARRAY_3
-
-#define HASH_ARRAY_2
-
-#include "match_c.h"
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file bt3.h
-/// \brief Binary Tree 3
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#ifndef LZMA_BT3_H
-#define LZMA_BT3_H
-
-#undef LZMA_MATCH_FINDER_NAME_LOWER
-#undef LZMA_MATCH_FINDER_NAME_UPPER
-#define LZMA_MATCH_FINDER_NAME_LOWER bt3
-#define LZMA_MATCH_FINDER_NAME_UPPER BT3
-
-#include "match_h.h"
-
-#endif
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file bt4.c
-/// \brief Binary Tree 4
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "bt4.h"
-
-#undef IS_HASH_CHAIN
-#undef HASH_ARRAY_2
-#undef HASH_ARRAY_3
-
-#define HASH_ARRAY_2
-#define HASH_ARRAY_3
-
-#include "match_c.h"
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file bt4.h
-/// \brief Binary Tree 4
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#ifndef LZMA_BT4_H
-#define LZMA_BT4_H
-
-#undef LZMA_MATCH_FINDER_NAME_LOWER
-#undef LZMA_MATCH_FINDER_NAME_UPPER
-#define LZMA_MATCH_FINDER_NAME_LOWER bt4
-#define LZMA_MATCH_FINDER_NAME_UPPER BT4
-
-#include "match_h.h"
-
-#endif
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file hc3.c
-/// \brief Hash Chain 3
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "hc3.h"
-
-#undef IS_HASH_CHAIN
-#undef HASH_ARRAY_2
-#undef HASH_ARRAY_3
-
-#define IS_HASH_CHAIN
-#define HASH_ARRAY_2
-
-#include "match_c.h"
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file hc3.h
-/// \brief Hash Chain 3
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#ifndef LZMA_HC3_H
-#define LZMA_HC3_H
-
-#undef LZMA_MATCH_FINDER_NAME_LOWER
-#undef LZMA_MATCH_FINDER_NAME_UPPER
-#define LZMA_MATCH_FINDER_NAME_LOWER hc3
-#define LZMA_MATCH_FINDER_NAME_UPPER HC3
-
-#include "match_h.h"
-
-#endif
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file hc4.c
-/// \brief Hash Chain 4
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "hc4.h"
-
-#undef IS_HASH_CHAIN
-#undef HASH_ARRAY_2
-#undef HASH_ARRAY_3
-
-#define IS_HASH_CHAIN
-#define HASH_ARRAY_2
-#define HASH_ARRAY_3
-
-#include "match_c.h"
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file hc4.h
-/// \brief Hash Chain 4
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#ifndef LZMA_HC4_H
-#define LZMA_HC4_H
-
-#undef LZMA_MATCH_FINDER_NAME_LOWER
-#undef LZMA_MATCH_FINDER_NAME_UPPER
-#define LZMA_MATCH_FINDER_NAME_LOWER hc4
-#define LZMA_MATCH_FINDER_NAME_UPPER HC4
-
-#include "match_h.h"
-
-#endif
//
///////////////////////////////////////////////////////////////////////////////
-#include "lz_decoder.h"
+// liblzma supports multiple LZ77-based filters. The LZ part is shared
+// between these filters. The LZ code takes care of dictionary handling
+// and passing the data between filters in the chain. The filter-specific
+// part decodes from the input buffer to the dictionary.
-/// Minimum size of allocated dictionary
-#define DICT_SIZE_MIN 8192
+#include "lz_decoder.h"
-/// When there is less than this amount of data available for decoding,
-/// it is moved to the temporary buffer which
-/// - protects from reads past the end of the buffer; and
-/// - stored the incomplete data between lzma_code() calls.
-///
-/// \note TEMP_LIMIT must be at least as much as
-/// REQUIRED_IN_BUFFER_SIZE defined in lzma_decoder.c.
-#define TEMP_LIMIT 32
-// lzma_lz_decoder.dict[] must be three times the size of TEMP_LIMIT.
-// 2 * TEMP_LIMIT is used for the actual data, and the third TEMP_LIMIT
-// bytes is needed for safety to allow decode_dummy() in lzma_decoder.c
-// to read past end of the buffer. This way it should be both fast and simple.
-#if LZMA_BUFFER_SIZE < 3 * TEMP_LIMIT
-# error LZMA_BUFFER_SIZE < 3 * TEMP_LIMIT
-#endif
+struct lzma_coder_s {
+ /// Dictionary (history buffer)
+ lzma_dict dict;
+ /// The actual LZ-based decoder e.g. LZMA
+ lzma_lz_decoder lz;
-struct lzma_coder_s {
+ /// Next filter in the chain, if any. Note that LZMA and LZMA2 are
+ /// only allowed as the last filter, but the long-range filter in
+ /// future can be in the middle of the chain.
lzma_next_coder next;
- lzma_lz_decoder lz;
- // There are more members in this structure but they are not
- // visible in LZ coder.
+ /// True if the next filter in the chain has returned LZMA_STREAM_END.
+ bool next_finished;
+
+ /// True if the LZ decoder (e.g. LZMA) has detected end of payload
+ /// marker. This may become true before next_finished becomes true.
+ bool this_finished;
+
+ /// Temporary buffer needed when the LZ-based filter is not the last
+ /// filter in the chain. The output of the next filter is first
+ /// decoded into buffer[], which is then used as input for the actual
+ /// LZ-based decoder.
+ struct {
+ size_t pos;
+ size_t size;
+ uint8_t buffer[LZMA_BUFFER_SIZE];
+ } temp;
};
-/// - Copy as much data as possible from lz->dict[] to out[].
-/// - Update *out_pos, lz->start, and lz->end accordingly.
-/// - Wrap lz-pos to the beginning of lz->dict[] if there is a danger that
-/// it may go past the end of the buffer (lz->pos >= lz->must_flush_pos).
-static inline bool
-flush(lzma_lz_decoder *restrict lz, uint8_t *restrict out,
- size_t *restrict out_pos, size_t out_size)
-{
- // Flush uncompressed data from the history buffer to
- // the output buffer. This is done in two phases.
-
- assert(lz->start <= lz->end);
-
- // Flush if pos < start < end.
- if (lz->pos < lz->start && lz->start < lz->end) {
- bufcpy(lz->dict, &lz->start, lz->end, out, out_pos, out_size);
-
- // If we reached end of the data in history buffer,
- // wrap to the beginning.
- if (lz->start == lz->end)
- lz->start = 0;
- }
-
- // Flush if start start < pos <= end. This is not as `else' for
- // previous `if' because the previous one may make this one true.
- if (lz->start < lz->pos) {
- bufcpy(lz->dict, &lz->start,
- lz->pos, out, out_pos, out_size);
-
- if (lz->pos >= lz->must_flush_pos) {
- // Wrap the flushing position if we have
- // flushed the whole history buffer.
- if (lz->pos == lz->start)
- lz->start = 0;
-
- // Wrap the write position and store to lz.end
- // how much there is new data available.
- lz->end = lz->pos;
- lz->pos = 0;
- lz->is_full = true;
- }
- }
-
- assert(lz->pos < lz->must_flush_pos);
-
- return *out_pos == out_size;
-}
-
-
-/// Calculate safe value for lz->limit. If no safe value can be found,
-/// set lz->limit to zero. When flushing, only as little data will be
-/// decoded as is needed to fill the output buffer (lowers both latency
-/// and throughput).
-///
-/// \return true if there is no space for new uncompressed data.
-///
-static inline bool
-set_limit(lzma_lz_decoder *lz, size_t out_avail, bool flushing)
-{
- // Set the limit so that writing to dict[limit + match_max_len - 1]
- // doesn't overwrite any unflushed data and doesn't write past the
- // end of the dict buffer.
- if (lz->start <= lz->pos) {
- // We can fill the buffer from pos till the end
- // of the dict buffer.
- lz->limit = lz->must_flush_pos;
- } else if (lz->pos + lz->match_max_len < lz->start) {
- // There's some unflushed data between pos and end of the
- // buffer. Limit so that we don't overwrite the unflushed data.
- lz->limit = lz->start - lz->match_max_len;
- } else {
- // Buffer is too full.
- lz->limit = 0;
- return true;
- }
-
- // Finetune the limit a bit if it isn't zero.
-
- assert(lz->limit > lz->pos);
- const size_t dict_avail = lz->limit - lz->pos;
-
- if (lz->uncompressed_size < dict_avail) {
- // Finishing a stream that doesn't have
- // an end of stream marker.
- lz->limit = lz->pos + lz->uncompressed_size;
-
- } else if (flushing && out_avail < dict_avail) {
- // Flushing enabled, decoding only as little as needed to
- // fill the out buffer (if there's enough input, of course).
- lz->limit = lz->pos + out_avail;
- }
-
- return lz->limit == lz->pos;
-}
-
-
-/// Takes care of wrapping the data into temporary buffer when needed,
-/// and calls the actual decoder.
-///
-/// \return true if error occurred
-///
-static inline bool
-call_process(lzma_coder *restrict coder, const uint8_t *restrict in,
- size_t *restrict in_pos, size_t in_size)
-{
- // It would be nice and simple if we could just give in[] to the
- // decoder, but the requirement of zlib-like API forces us to be
- // able to make *in_pos == in_size whenever there is enough output
- // space. If needed, we will append a few bytes from in[] to
- // a temporary buffer and decode enough to reach the part that
- // was copied from in[]. Then we can continue with the real in[].
-
- bool error;
- const size_t dict_old_pos = coder->lz.pos;
- const size_t in_avail = in_size - *in_pos;
-
- if (coder->lz.temp_size + in_avail < 2 * TEMP_LIMIT) {
- // Copy all the available input from in[] to temp[].
- memcpy(coder->lz.temp + coder->lz.temp_size,
- in + *in_pos, in_avail);
- coder->lz.temp_size += in_avail;
- *in_pos += in_avail;
- assert(*in_pos == in_size);
-
- // Decode as much as possible.
- size_t temp_used = 0;
- error = coder->lz.process(coder, coder->lz.temp, &temp_used,
- coder->lz.temp_size, true);
- assert(temp_used <= coder->lz.temp_size);
-
- // Move the remaining data to the beginning of temp[].
- coder->lz.temp_size -= temp_used;
- memmove(coder->lz.temp, coder->lz.temp + temp_used,
- coder->lz.temp_size);
-
- } else if (coder->lz.temp_size > 0) {
- // Fill temp[] unless it is already full because we aren't
- // the last filter in the chain.
- size_t copy_size = 0;
- if (coder->lz.temp_size < 2 * TEMP_LIMIT) {
- assert(*in_pos < in_size);
- copy_size = 2 * TEMP_LIMIT - coder->lz.temp_size;
- memcpy(coder->lz.temp + coder->lz.temp_size,
- in + *in_pos, copy_size);
- // NOTE: We don't update lz.temp_size or *in_pos yet.
- }
-
- size_t temp_used = 0;
- error = coder->lz.process(coder, coder->lz.temp, &temp_used,
- coder->lz.temp_size + copy_size, false);
-
- if (temp_used < coder->lz.temp_size) {
- // Only very little input data was consumed. Move
- // the unprocessed data to the beginning temp[].
- coder->lz.temp_size += copy_size - temp_used;
- memmove(coder->lz.temp, coder->lz.temp + temp_used,
- coder->lz.temp_size);
- *in_pos += copy_size;
- assert(*in_pos <= in_size);
-
- } else {
- // We were able to decode so much data that next time
- // we can decode directly from in[]. That is, we can
- // consider temp[] to be empty now.
- *in_pos += temp_used - coder->lz.temp_size;
- coder->lz.temp_size = 0;
- assert(*in_pos <= in_size);
- }
-
- } else {
- // Decode directly from in[].
- error = coder->lz.process(coder, in, in_pos, in_size, false);
- assert(*in_pos <= in_size);
- }
-
- assert(coder->lz.pos >= dict_old_pos);
- if (coder->lz.uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) {
- // Update uncompressed size.
- coder->lz.uncompressed_size -= coder->lz.pos - dict_old_pos;
-
- // Check that End of Payload Marker hasn't been detected
- // since it must not be present because uncompressed size
- // is known.
- if (coder->lz.eopm_detected)
- error = true;
- }
-
- return error;
-}
-
-
static lzma_ret
decode_buffer(lzma_coder *coder,
const uint8_t *restrict in, size_t *restrict in_pos,
size_t in_size, uint8_t *restrict out,
- size_t *restrict out_pos, size_t out_size,
- bool flushing)
+ size_t *restrict out_pos, size_t out_size)
{
- bool stop = false;
-
while (true) {
- // Flush from coder->lz.dict to out[].
- flush(&coder->lz, out, out_pos, out_size);
-
- // All done?
- if (*out_pos == out_size
- || stop
- || coder->lz.eopm_detected
- || coder->lz.uncompressed_size == 0)
- break;
-
- // Set write limit in the dictionary.
- if (set_limit(&coder->lz, out_size - *out_pos, flushing))
- break;
-
- // Decode more data.
- if (call_process(coder, in, in_pos, in_size))
- return LZMA_DATA_ERROR;
-
- // Set stop to true if we must not call call_process() again
- // during this function call.
- // FIXME: Can this make the loop exist too early? It wouldn't
- // cause data corruption so not a critical problem. It can
- // happen if dictionary gets full and lz.temp still contains
- // a few bytes data that we could decode right now.
- if (*in_pos == in_size && coder->lz.temp_size <= TEMP_LIMIT
- && coder->lz.pos < coder->lz.limit)
- stop = true;
+ // Wrap the dictionary if needed.
+ if (coder->dict.pos == coder->dict.size)
+ coder->dict.pos = 0;
+
+ // Store the current dictionary position. It is needed to know
+ // where to start copying to the out[] buffer.
+ const size_t dict_start = coder->dict.pos;
+
+ // Calculate how much we allow the process() function to
+ // decode. It must not decode past the end of the dictionary
+ // buffer, and we don't want it to decode more than is
+ // actually needed to fill the out[] buffer.
+ coder->dict.limit = coder->dict.pos + MIN(out_size - *out_pos,
+ coder->dict.size - coder->dict.pos);
+
+ // Call the process() function to do the actual decoding.
+ const lzma_ret ret = coder->lz.code(
+ coder->lz.coder, &coder->dict,
+ in, in_pos, in_size);
+
+ // Copy the decoded data from the dictionary to the out[]
+ // buffer.
+ const size_t copy_size = coder->dict.pos - dict_start;
+ assert(copy_size <= out_size - *out_pos);
+ memcpy(out + *out_pos, coder->dict.buf + dict_start,
+ copy_size);
+ *out_pos += copy_size;
+
+ // Return if everything got decoded or an error occurred, or
+ // if there's no more data to decode.
+ if (ret != LZMA_OK || *out_pos == out_size
+ || coder->dict.pos < coder->dict.size)
+ return ret;
}
-
- // If we have decoded everything (EOPM detected or uncompressed_size
- // bytes were processed) to the history buffer, and also flushed
- // everything from the history buffer, our job is done.
- if ((coder->lz.eopm_detected
- || coder->lz.uncompressed_size == 0)
- && coder->lz.start == coder->lz.pos)
- return LZMA_STREAM_END;
-
- return LZMA_OK;
}
-extern lzma_ret
-lzma_lz_decode(lzma_coder *coder,
+static lzma_ret
+lz_decode(lzma_coder *coder,
lzma_allocator *allocator lzma_attribute((unused)),
const uint8_t *restrict in, size_t *restrict in_pos,
size_t in_size, uint8_t *restrict out,
size_t *restrict out_pos, size_t out_size,
lzma_action action)
{
- if (coder->next.code == NULL) {
- const lzma_ret ret = decode_buffer(coder, in, in_pos, in_size,
- out, out_pos, out_size,
- action == LZMA_SYNC_FLUSH);
-
- if (*out_pos == out_size || ret == LZMA_STREAM_END) {
- // Unread to make coder->temp[] empty. This is easy,
- // because we know that all the data currently in
- // coder->temp[] has been copied form in[] during this
- // call to the decoder.
- //
- // If we didn't do this, we could have data left in
- // coder->temp[] when end of stream is reached. That
- // data could be left there from *previous* call to
- // the decoder; in that case we wouldn't know where
- // to put that data.
- assert(*in_pos >= coder->lz.temp_size);
- *in_pos -= coder->lz.temp_size;
- coder->lz.temp_size = 0;
- }
-
- return ret;
- }
+ if (coder->next.code == NULL)
+ return decode_buffer(coder, in, in_pos, in_size,
+ out, out_pos, out_size);
// We aren't the last coder in the chain, we need to decode
// our input to a temporary buffer.
- const bool flushing = action == LZMA_SYNC_FLUSH;
while (*out_pos < out_size) {
- if (!coder->lz.next_finished
- && coder->lz.temp_size < LZMA_BUFFER_SIZE) {
+ // Fill the temporary buffer if it is empty.
+ if (!coder->next_finished
+ && coder->temp.pos == coder->temp.size) {
+ coder->temp.pos = 0;
+ coder->temp.size = 0;
+
const lzma_ret ret = coder->next.code(
coder->next.coder,
allocator, in, in_pos, in_size,
- coder->lz.temp, &coder->lz.temp_size,
+ coder->temp.buffer, &coder->temp.size,
LZMA_BUFFER_SIZE, action);
if (ret == LZMA_STREAM_END)
- coder->lz.next_finished = true;
- else if (coder->lz.temp_size < LZMA_BUFFER_SIZE
- || ret != LZMA_OK)
+ coder->next_finished = true;
+ else if (ret != LZMA_OK || coder->temp.size == 0)
return ret;
}
- if (coder->lz.this_finished) {
- if (coder->lz.temp_size != 0)
+ if (coder->this_finished) {
+ if (coder->temp.size != 0)
return LZMA_DATA_ERROR;
- if (coder->lz.next_finished)
+ if (coder->next_finished)
return LZMA_STREAM_END;
return LZMA_OK;
}
- size_t dummy = 0;
- const lzma_ret ret = decode_buffer(coder, NULL, &dummy, 0,
- out, out_pos, out_size, flushing);
+ const lzma_ret ret = decode_buffer(coder, coder->temp.buffer,
+ &coder->temp.pos, coder->temp.size,
+ out, out_pos, out_size);
if (ret == LZMA_STREAM_END)
- coder->lz.this_finished = true;
+ coder->this_finished = true;
else if (ret != LZMA_OK)
return ret;
- else if (coder->lz.next_finished && *out_pos < out_size)
+ else if (coder->next_finished && *out_pos < out_size)
return LZMA_DATA_ERROR;
}
}
-/// \brief Initializes LZ part of the LZMA decoder or Inflate
-///
-/// \param history_size Number of bytes the LZ out window is
-/// supposed keep available from the output
-/// history.
-/// \param match_max_len Number of bytes a single decoding loop
-/// can advance the write position (lz->pos)
-/// in the history buffer (lz->dict).
-///
-/// \note This function is called by LZMA decoder and Inflate init()s.
-/// It's up to those functions allocate *lz and initialize it
-/// with LZMA_LZ_DECODER_INIT.
+static void
+lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
+{
+ lzma_next_end(&coder->next, allocator);
+ lzma_free(coder->dict.buf, allocator);
+
+ if (coder->lz.end != NULL)
+ coder->lz.end(coder->lz.coder, allocator);
+ else
+ lzma_free(coder->lz.coder, allocator);
+
+ lzma_free(coder, allocator);
+ return;
+}
+
+
extern lzma_ret
-lzma_lz_decoder_reset(lzma_lz_decoder *lz, lzma_allocator *allocator,
- bool (*process)(lzma_coder *restrict coder,
- const uint8_t *restrict in, size_t *restrict in_pos,
- size_t in_size, bool has_safe_buffer),
- size_t history_size, size_t match_max_len)
+lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter_info *filters,
+ lzma_ret (*lz_init)(lzma_lz_decoder *lz,
+ lzma_allocator *allocator, const void *options,
+ size_t *dict_size))
{
- // Known uncompressed size is used only with LZMA_Alone files so we
- // set it always to unknown by default.
- lz->uncompressed_size = LZMA_VLI_VALUE_UNKNOWN;
-
- // Limit the history size to roughly sane values. This is primarily
- // to prevent integer overflows.
- if (history_size > UINT32_MAX / 2)
- return LZMA_HEADER_ERROR;
-
- // Store the value actually requested. We use it for sanity checks
- // when repeating data from the history buffer.
- lz->requested_size = history_size;
-
- // Avoid tiny history buffer sizes for performance reasons.
- // TODO: Test if this actually helps...
- if (history_size < DICT_SIZE_MIN)
- history_size = DICT_SIZE_MIN;
-
- // The real size of the history buffer is a bit bigger than
- // requested by our caller. This allows us to do some optimizations,
- // which help not only speed but simplicity of the code; specifically,
- // we can make sure that there is always at least match_max_len
- // bytes immediatelly available for writing without a need to wrap
- // the history buffer.
- const size_t dict_real_size = history_size + 2 * match_max_len + 1;
-
- // Reallocate memory if needed.
- if (history_size != lz->size || match_max_len != lz->match_max_len) {
- // Destroy the old buffer.
- lzma_lz_decoder_end(lz, allocator);
-
- lz->size = history_size;
- lz->match_max_len = match_max_len;
- lz->must_flush_pos = history_size + match_max_len + 1;
-
- lz->dict = lzma_alloc(dict_real_size, allocator);
- if (lz->dict == NULL)
+ // Allocate the base structure if it isn't already allocated.
+ if (next->coder == NULL) {
+ next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
+ if (next->coder == NULL)
return LZMA_MEM_ERROR;
+
+ next->code = &lz_decode;
+ next->end = &lz_decoder_end;
+
+ next->coder->dict.buf = NULL;
+ next->coder->dict.size = 0;
+ next->coder->lz = LZMA_LZ_DECODER_INIT;
+ next->coder->next = LZMA_NEXT_CODER_INIT;
}
- // Reset the variables so that lz_get_byte(lz, 0) will return '\0'.
- lz->pos = 0;
- lz->start = 0;
- lz->end = dict_real_size;
- lz->dict[dict_real_size - 1] = 0;
- lz->is_full = false;
- lz->eopm_detected = false;
- lz->next_finished = false;
- lz->this_finished = false;
- lz->temp_size = 0;
-
- // Clean up the temporary buffer to make it very sure that there are
- // no information leaks when multiple steams are decoded with the
- // same decoder structures.
- memzero(lz->temp, LZMA_BUFFER_SIZE);
-
- // Set the process function pointer.
- lz->process = process;
+ // Allocate and initialize the LZ-based decoder. It will also give
+ // us the dictionary size.
+ size_t dict_size;
+ return_if_error(lz_init(&next->coder->lz, allocator,
+ filters[0].options, &dict_size));
+
+ // If the dictionary size is very small, increase it to 4096 bytes.
+ // This is to prevent constant wrapping of the dictionary, which
+ // would slow things down. The downside is that since we don't check
+ // separately for the real dictionary size, we may happily accept
+ // corrupt files.
+ if (dict_size < 4096)
+ dict_size = 4096;
+
+ // Make dictionary size a multipe of 16. Some LZ-based decoders like
+ // LZMA use the lowest bits lzma_dict.pos to know the alignment of the
+ // data. Aligned buffer is also good when memcpying from the
+ // dictionary to the output buffer, since applications are
+ // recommended to give aligned buffers to liblzma.
+ //
+ // Avoid integer overflow. FIXME Should the return value be
+ // LZMA_HEADER_ERROR or LZMA_MEM_ERROR?
+ if (dict_size > SIZE_MAX - 15)
+ return LZMA_MEM_ERROR;
+
+ dict_size = (dict_size + 15) & (SIZE_MAX - 15);
+
+ // Allocate and initialize the dictionary.
+ if (next->coder->dict.size != dict_size) {
+ lzma_free(next->coder->dict.buf, allocator);
+ next->coder->dict.buf = lzma_alloc(dict_size, allocator);
+ if (next->coder->dict.buf == NULL)
+ return LZMA_MEM_ERROR;
- return LZMA_OK;
+ next->coder->dict.size = dict_size;
+ }
+
+ dict_reset(&next->coder->dict);
+
+ // Miscellaneous initializations
+ next->coder->next_finished = false;
+ next->coder->this_finished = false;
+ next->coder->temp.pos = 0;
+ next->coder->temp.size = 0;
+
+ // Initialize the next filter in the chain, if any.
+ return lzma_next_filter_init(&next->coder->next, allocator,
+ filters + 1);
+}
+
+
+extern uint64_t
+lzma_lz_decoder_memusage(size_t dictionary_size)
+{
+ return sizeof(lzma_coder) + (uint64_t)(dictionary_size);
}
extern void
-lzma_lz_decoder_end(lzma_lz_decoder *lz, lzma_allocator *allocator)
+lzma_lz_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size)
{
- lzma_free(lz->dict, allocator);
- lz->dict = NULL;
- lz->size = 0;
- lz->match_max_len = 0;
- return;
+ coder->lz.set_uncompressed(coder->lz.coder, uncompressed_size);
}
//
///////////////////////////////////////////////////////////////////////////////
-#ifndef LZMA_LZ_OUT_H
-#define LZMA_LZ_OUT_H
+#ifndef LZMA_LZ_DECODER_H
+#define LZMA_LZ_DECODER_H
#include "common.h"
-/// Get a byte from the history buffer.
-#define lz_get_byte(lz, distance) \
- ((distance) < (lz).pos \
- ? (lz).dict[(lz).pos - (distance) - 1] \
- : (lz).dict[(lz).pos - (distance) - 1 + (lz).end])
-
-
-/// Test if dictionary is empty.
-#define lz_is_empty(lz) \
- ((lz).pos == 0 && !(lz).is_full)
-
-
-#define LZMA_LZ_DECODER_INIT \
- (lzma_lz_decoder){ .dict = NULL, .size = 0, .match_max_len = 0 }
-
-
typedef struct {
- /// Function to do the actual decoding (LZMA or Inflate)
- bool (*process)(lzma_coder *restrict coder, const uint8_t *restrict in,
- size_t *restrict in_pos, size_t size_in,
- bool has_safe_buffer);
+ /// Pointer to the dictionary buffer. It can be an allocated buffer
+ /// internal to liblzma, or it can a be a buffer given by the
+ /// application when in single-call mode (not implemented yet).
+ uint8_t *buf;
- /// Pointer to dictionary (history) buffer.
- /// \note Not 'restrict' because can alias next_out.
- uint8_t *dict;
-
- /// Next write goes to dict[pos].
+ /// Write position in dictionary. The next byte will be written to
+ /// buf[pos].
size_t pos;
- /// Next byte to flush is buffer[start].
- size_t start;
-
- /// First byte to not flush is buffer[end].
- size_t end;
+ /// Indicates how full the dictionary is. This is used by
+ /// dict_is_distance_valid() to detect corrupt files that would
+ /// read beyond the beginning of the dictionary.
+ size_t full;
- /// First position to which data must not be written.
+ /// Write limit
size_t limit;
- /// True if dictionary has needed wrapping.
- bool is_full;
-
- /// True if process() has detected End of Payload Marker.
- bool eopm_detected;
+ /// Size of the dictionary
+ size_t size;
- /// True if the next coder in the chain has returned LZMA_STREAM_END.
- bool next_finished;
+} lzma_dict;
- /// True if the LZ decoder (e.g. LZMA) has detected End of Payload
- /// Marker. This may become true before next_finished becomes true.
- bool this_finished;
- /// When pos >= must_flush_pos, we must not call process().
- size_t must_flush_pos;
+typedef struct {
+ /// Data specific to the LZ-based decoder
+ lzma_coder *coder;
- /// Maximum number of bytes that a single decoding loop inside
- /// process() can produce data into dict. This amount is kept
- /// always available at dict + pos i.e. it is safe to write a byte
- /// to dict[pos + match_max_len - 1].
- size_t match_max_len;
+ /// Function to decode from in[] to *dict
+ lzma_ret (*code)(lzma_coder *restrict coder,
+ lzma_dict *restrict dict, const uint8_t *restrict in,
+ size_t *restrict in_pos, size_t in_size);
- /// Number of bytes allocated to dict.
- size_t size;
+ void (*reset)(lzma_coder *coder, const void *options);
- /// Requested size of the dictionary. This is needed because we avoid
- /// using extremely tiny history buffers.
- size_t requested_size;
+ /// Set the uncompressed size
+ void (*set_uncompressed)(lzma_coder *coder,
+ lzma_vli uncompressed_size);
- /// Uncompressed Size or LZMA_VLI_VALUE_UNKNOWN if unknown.
- lzma_vli uncompressed_size;
+ /// Free allocated resources
+ void (*end)(lzma_coder *coder, lzma_allocator *allocator);
- /// Number of bytes currently in temp[].
- size_t temp_size;
+} lzma_lz_decoder;
- /// Temporary buffer needed when
- /// 1) we cannot make the input buffer completely empty; or
- /// 2) we are not the last filter in the chain.
- uint8_t temp[LZMA_BUFFER_SIZE];
-} lzma_lz_decoder;
+#define LZMA_LZ_DECODER_INIT \
+ (lzma_lz_decoder){ \
+ .coder = NULL, \
+ .code = NULL, \
+ .reset = NULL, \
+ .set_uncompressed = NULL, \
+ .end = NULL, \
+ }
-/////////////////////////
-// Function prototypes //
-/////////////////////////
+extern lzma_ret lzma_lz_decoder_init(lzma_next_coder *next,
+ lzma_allocator *allocator, const lzma_filter_info *filters,
+ lzma_ret (*lz_init)(lzma_lz_decoder *lz,
+ lzma_allocator *allocator, const void *options,
+ size_t *dict_size));
-extern lzma_ret lzma_lz_decoder_reset(lzma_lz_decoder *lz,
- lzma_allocator *allocator, bool (*process)(
- lzma_coder *restrict coder, const uint8_t *restrict in,
- size_t *restrict in_pos, size_t in_size,
- bool has_safe_buffer),
- size_t history_size, size_t match_max_len);
+extern uint64_t lzma_lz_decoder_memusage(size_t dictionary_size);
-extern lzma_ret lzma_lz_decode(lzma_coder *coder, lzma_allocator *allocator,
- const uint8_t *restrict in, size_t *restrict in_pos,
- size_t in_size, uint8_t *restrict out,
- size_t *restrict out_pos, size_t out_size,
- lzma_action action);
+extern void lzma_lz_decoder_uncompressed(
+ lzma_coder *coder, lzma_vli uncompressed_size);
-/// Deallocates the history buffer if one exists.
-extern void lzma_lz_decoder_end(
- lzma_lz_decoder *lz, lzma_allocator *allocator);
//////////////////////
// Inline functions //
//////////////////////
-// Repeat a block of data from the history. Because memcpy() is faster
-// than copying byte by byte in a loop, the copying process gets split
-// into three cases:
-// 1. distance < length
-// Source and target areas overlap, thus we can't use memcpy()
-// (nor memmove()) safely.
-// TODO: If this is common enough, it might be worth optimizing this
-// more e.g. by checking if distance > sizeof(uint8_t*) and using
-// memcpy in small chunks.
-// 2. distance < pos
-// This is the easiest and the fastest case. The block being copied
-// is a contiguous piece in the history buffer. The buffer offset
-// doesn't need wrapping.
-// 3. distance >= pos
-// We need to wrap the position, because otherwise we would try copying
-// behind the first byte of the allocated buffer. It is possible that
-// the block is fragmeneted into two pieces, thus we might need to call
-// memcpy() twice.
-// NOTE: The function using this macro must ensure that length is positive
-// and that distance is FIXME
+/// Get a byte from the history buffer.
+static inline uint8_t
+dict_get(const lzma_dict *const dict, const uint32_t distance)
+{
+ return dict->buf[dict->pos - distance - 1
+ + (distance < dict->pos ? 0 : dict->size)];
+}
+
+
+/// Test if dictionary is empty.
+static inline bool
+dict_is_empty(const lzma_dict *const dict)
+{
+ return dict->full == 0;
+}
+
+
+/// Validate the match distance
+static inline bool
+dict_is_distance_valid(const lzma_dict *const dict, const size_t distance)
+{
+ return dict->full >= distance;
+}
+
+
+/// Repeat *len bytes at distance.
static inline bool
-lzma_lz_out_repeat(lzma_lz_decoder *lz, size_t distance, size_t length)
+dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len)
{
- // Validate offset of the block to be repeated. It doesn't
- // make sense to copy data behind the beginning of the stream.
- // Leaving this check away would lead to a security problem,
- // in which e.g. the data of the previously decoded file(s)
- // would be leaked (or whatever happens to be in unused
- // part of the dictionary buffer).
- if (unlikely(distance >= lz->pos && !lz->is_full))
- return false;
-
- // It also doesn't make sense to copy data farer than
- // the dictionary size.
- if (unlikely(distance >= lz->requested_size))
- return false;
-
- // The caller must have checked these!
- assert(distance <= lz->size);
- assert(length > 0);
- assert(length <= lz->match_max_len);
-
- // Copy the amount of data requested by the decoder.
- if (distance < length) {
+ // Don't write past the end of the dictionary.
+ const size_t dict_avail = dict->limit - dict->pos;
+ uint32_t left = MIN(dict_avail, *len);
+ *len -= left;
+
+ // Repeat a block of data from the history. Because memcpy() is faster
+ // than copying byte by byte in a loop, the copying process gets split
+ // into three cases.
+ if (distance < left) {
// Source and target areas overlap, thus we can't use
- // memcpy() nor even memmove() safely. :-(
- // TODO: Copying byte by byte is slow. It might be
- // worth optimizing this more if this case is common.
+ // memcpy() nor even memmove() safely.
do {
- lz->dict[lz->pos] = lz_get_byte(*lz, distance);
- ++lz->pos;
- } while (--length > 0);
+ dict->buf[dict->pos] = dict_get(dict, distance);
+ ++dict->pos;
+ } while (--left > 0);
- } else if (distance < lz->pos) {
+ } else if (distance < dict->pos) {
// The easiest and fastest case
- memcpy(lz->dict + lz->pos,
- lz->dict + lz->pos - distance - 1,
- length);
- lz->pos += length;
+ memcpy(dict->buf + dict->pos,
+ dict->buf + dict->pos - distance - 1,
+ left);
+ dict->pos += left;
} else {
// The bigger the dictionary, the more rare this
// case occurs. We need to "wrap" the dict, thus
// we might need two memcpy() to copy all the data.
- assert(lz->is_full);
- const uint32_t copy_pos = lz->pos - distance - 1 + lz->end;
- uint32_t copy_size = lz->end - copy_pos;
+ assert(dict->full == dict->size);
+ const uint32_t copy_pos
+ = dict->pos - distance - 1 + dict->size;
+ uint32_t copy_size = dict->size - copy_pos;
- if (copy_size < length) {
- memcpy(lz->dict + lz->pos, lz->dict + copy_pos,
+ if (copy_size < left) {
+ memcpy(dict->buf + dict->pos, dict->buf + copy_pos,
copy_size);
- lz->pos += copy_size;
- copy_size = length - copy_size;
- memcpy(lz->dict + lz->pos, lz->dict, copy_size);
- lz->pos += copy_size;
+ dict->pos += copy_size;
+ copy_size = left - copy_size;
+ memcpy(dict->buf + dict->pos, dict->buf, copy_size);
+ dict->pos += copy_size;
} else {
- memcpy(lz->dict + lz->pos, lz->dict + copy_pos,
- length);
- lz->pos += length;
+ memcpy(dict->buf + dict->pos, dict->buf + copy_pos,
+ left);
+ dict->pos += left;
}
}
- return true;
+ // Update how full the dictionary is.
+ if (dict->full < dict->pos)
+ dict->full = dict->pos;
+
+ return unlikely(*len != 0);
+}
+
+
+/// Puts one byte into the dictionary. Returns true if the dictionary was
+/// already full and the byte couldn't be added.
+static inline bool
+dict_put(lzma_dict *dict, uint8_t byte)
+{
+ if (unlikely(dict->pos == dict->limit))
+ return true;
+
+ dict->buf[dict->pos++] = byte;
+
+ if (dict->pos > dict->full)
+ dict->full = dict->pos;
+
+ return false;
+}
+
+
+/// Copies arbitrary amount of data into the dictionary.
+static inline void
+dict_write(lzma_dict *restrict dict, const uint8_t *restrict in,
+ size_t *restrict in_pos, size_t in_size,
+ size_t *restrict left)
+{
+ // NOTE: If we are being given more data than the size of the
+ // dictionary, it could be possible to optimize the LZ decoder
+ // so that not everything needs to go through the dictionary.
+ // This shouldn't be very common thing in practice though, and
+ // the slowdown of one extra memcpy() isn't bad compared to how
+ // much time it would have taken if the data were compressed.
+
+ if (in_size - *in_pos > *left)
+ in_size = *in_pos + *left;
+
+ *left -= lzma_bufcpy(in, in_pos, in_size,
+ dict->buf, &dict->pos, dict->limit);
+
+ if (dict->pos > dict->full)
+ dict->full = dict->pos;
+
+ return;
+}
+
+
+static inline void
+dict_reset(lzma_dict *dict)
+{
+ dict->pos = 0;
+ dict->full = 0;
+ dict->buf[dict->size - 1] = '\0';
}
#endif
/// \file lz_encoder.c
/// \brief LZ in window
//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
+// Copyright (C) 1999-2008 Igor Pavlov
+// Copyright (C) 2007-2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
//
///////////////////////////////////////////////////////////////////////////////
-#include "lz_encoder_private.h"
+#include "lz_encoder.h"
+#include "lz_encoder_hash.h"
-// Hash Chains
-#ifdef HAVE_HC3
-# include "hc3.h"
-#endif
-#ifdef HAVE_HC4
-# include "hc4.h"
-#endif
-// Binary Trees
-#ifdef HAVE_BT2
-# include "bt2.h"
-#endif
-#ifdef HAVE_BT3
-# include "bt3.h"
-#endif
-#ifdef HAVE_BT4
-# include "bt4.h"
-#endif
+struct lzma_coder_s {
+ /// LZ-based encoder e.g. LZMA
+ lzma_lz_encoder lz;
+ /// History buffer and match finder
+ lzma_mf mf;
-/// This is needed in two places so provide a macro.
-#define get_cyclic_buffer_size(history_size) ((history_size) + 1)
+ /// Next coder in the chain
+ lzma_next_coder next;
+};
-/// Calculate certain match finder properties and validate the calculated
-/// values. This is as its own function, because *num_items is needed to
-/// calculate memory requirements in common/memory.c.
-extern bool
-lzma_lz_encoder_hash_properties(lzma_match_finder match_finder,
- uint32_t history_size, uint32_t *restrict hash_mask,
- uint32_t *restrict hash_size_sum, uint32_t *restrict num_items)
+/// \brief Moves the data in the input window to free space for new data
+///
+/// mf->buffer is a sliding input window, which keeps mf->keep_size_before
+/// bytes of input history available all the time. Now and then we need to
+/// "slide" the buffer to make space for the new data to the end of the
+/// buffer. At the same time, data older than keep_size_before is dropped.
+///
+static void
+move_window(lzma_mf *mf)
{
- uint32_t fix_hash_size;
- uint32_t sons;
+ // Align the move to a multiple of 16 bytes. Some LZ-based encoders
+ // like LZMA use the lowest bits of mf->read_pos to know the
+ // alignment of the uncompressed data. We also get better speed
+ // for memmove() with aligned buffers.
+ assert(mf->read_pos > mf->keep_size_before);
+ const uint32_t move_offset
+ = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15);
- switch (match_finder) {
-#ifdef HAVE_HC3
- case LZMA_MF_HC3:
- fix_hash_size = LZMA_HC3_FIX_HASH_SIZE;
- sons = 1;
- break;
-#endif
-#ifdef HAVE_HC4
- case LZMA_MF_HC4:
- fix_hash_size = LZMA_HC4_FIX_HASH_SIZE;
- sons = 1;
- break;
-#endif
-#ifdef HAVE_BT2
- case LZMA_MF_BT2:
- fix_hash_size = LZMA_BT2_FIX_HASH_SIZE;
- sons = 2;
- break;
-#endif
-#ifdef HAVE_BT3
- case LZMA_MF_BT3:
- fix_hash_size = LZMA_BT3_FIX_HASH_SIZE;
- sons = 2;
- break;
-#endif
-#ifdef HAVE_BT4
- case LZMA_MF_BT4:
- fix_hash_size = LZMA_BT4_FIX_HASH_SIZE;
- sons = 2;
- break;
-#endif
- default:
- return true;
- }
+ assert(mf->write_pos > move_offset);
+ const size_t move_size = mf->write_pos - move_offset;
- uint32_t hs;
+ assert(move_offset + move_size <= mf->size);
-#ifdef HAVE_LZMA_BT2
- if (match_finder == LZMA_BT2) {
- // NOTE: hash_mask is not used by the BT2 match finder,
- // but it is initialized just in case.
- hs = LZMA_BT2_HASH_SIZE;
- *hash_mask = 0;
- } else
-#endif
- {
- hs = history_size - 1;
- hs |= (hs >> 1);
- hs |= (hs >> 2);
- hs |= (hs >> 4);
- hs |= (hs >> 8);
- hs >>= 1;
- hs |= 0xFFFF;
+ memmove(mf->buffer, mf->buffer + move_offset, move_size);
- if (hs > (UINT32_C(1) << 24)) {
- if (match_finder == LZMA_MF_HC4
- || match_finder == LZMA_MF_BT4)
- hs >>= 1;
- else
- hs = (1 << 24) - 1;
- }
+ mf->offset += move_offset;
+ mf->read_pos -= move_offset;
+ mf->read_limit -= move_offset;
+ mf->write_pos -= move_offset;
+
+ return;
+}
- *hash_mask = hs;
- ++hs;
- }
- *hash_size_sum = hs + fix_hash_size;
+/// \brief Tries to fill the input window (mf->buffer)
+///
+/// If we are the last encoder in the chain, our input data is in in[].
+/// Otherwise we call the next filter in the chain to process in[] and
+/// write its output to mf->buffer.
+///
+/// This function must not be called once it has returned LZMA_STREAM_END.
+///
+static lzma_ret
+fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in,
+ size_t *in_pos, size_t in_size, lzma_action action)
+{
+ assert(coder->mf.read_pos <= coder->mf.write_pos);
- *num_items = *hash_size_sum
- + get_cyclic_buffer_size(history_size) * sons;
+ // Move the sliding window if needed.
+ if (coder->mf.read_pos >= coder->mf.size - coder->mf.keep_size_after)
+ move_window(&coder->mf);
- return false;
-}
+ size_t in_used;
+ lzma_ret ret;
+ if (coder->next.code == NULL) {
+ // Not using a filter, simply memcpy() as much as possible.
+ in_used = lzma_bufcpy(in, in_pos, in_size, coder->mf.buffer,
+ &coder->mf.write_pos, coder->mf.size);
+ ret = action != LZMA_RUN && *in_pos == in_size
+ ? LZMA_STREAM_END : LZMA_OK;
-extern lzma_ret
-lzma_lz_encoder_reset(lzma_lz_encoder *lz, lzma_allocator *allocator,
- bool (*process)(lzma_coder *coder, uint8_t *restrict out,
- size_t *restrict out_pos, size_t out_size),
- size_t history_size, size_t additional_buffer_before,
- size_t match_max_len, size_t additional_buffer_after,
- lzma_match_finder match_finder, uint32_t match_finder_cycles,
- const uint8_t *preset_dictionary,
- size_t preset_dictionary_size)
-{
- lz->sequence = SEQ_RUN;
+ } else {
+ const size_t in_start = *in_pos;
+ ret = coder->next.code(coder->next.coder, allocator,
+ in, in_pos, in_size,
+ coder->mf.buffer, &coder->mf.write_pos,
+ coder->mf.size, action);
+ in_used = *in_pos - in_start;
+ }
- ///////////////
- // In Window //
- ///////////////
+ // If end of stream has been reached or flushing completed, we allow
+ // the encoder to process all the input (that is, read_pos is allowed
+ // to reach write_pos). Otherwise we keep keep_size_after bytes
+ // available as prebuffer.
+ if (ret == LZMA_STREAM_END) {
+ assert(*in_pos == in_size);
+ ret = LZMA_OK;
+ coder->mf.action = action;
+ coder->mf.read_limit = coder->mf.write_pos;
- // Validate history size.
- if (history_size < LZMA_DICTIONARY_SIZE_MIN
- || history_size > LZMA_DICTIONARY_SIZE_MAX) {
- lzma_lz_encoder_end(lz, allocator);
- return LZMA_HEADER_ERROR;
+ } else if (coder->mf.write_pos > coder->mf.keep_size_after) {
+ // This needs to be done conditionally, because if we got
+ // only little new input, there may be too little input
+ // to do any encoding yet.
+ coder->mf.read_limit = coder->mf.write_pos
+ - coder->mf.keep_size_after;
}
- assert(history_size <= MAX_VAL_FOR_NORMALIZE - 256);
- assert(LZMA_DICTIONARY_SIZE_MAX <= MAX_VAL_FOR_NORMALIZE - 256);
+ // Restart the match finder after finished LZMA_SYNC_FLUSH.
+ if (coder->mf.pending > 0
+ && coder->mf.read_pos < coder->mf.read_limit) {
+ // Match finder may update coder->pending and expects it to
+ // start from zero, so use a temporary variable.
+ const size_t pending = coder->mf.pending;
+ coder->mf.pending = 0;
- // Calculate the size of the history buffer to allocate.
- // TODO: Get a reason for magic constant of 256.
- const size_t size_reserv = (history_size + additional_buffer_before
- + match_max_len + additional_buffer_after) / 2 + 256;
+ // Rewind read_pos so that the match finder can hash
+ // the pending bytes.
+ assert(coder->mf.read_pos >= pending);
+ coder->mf.read_pos -= pending;
- lz->keep_size_before = history_size + additional_buffer_before;
- lz->keep_size_after = match_max_len + additional_buffer_after;
+ // Call the skip function directly instead of using
+ // lz_dict_skip(), since we don't want to touch
+ // mf->read_ahead.
+ coder->mf.skip(&coder->mf, pending);
+ }
- const size_t buffer_size = lz->keep_size_before + lz->keep_size_after
- + size_reserv;
+ return ret;
+}
- // Allocate history buffer if its size has changed.
- if (buffer_size != lz->size) {
- lzma_free(lz->buffer, allocator);
- lz->buffer = lzma_alloc(buffer_size, allocator);
- if (lz->buffer == NULL) {
- lzma_lz_encoder_end(lz, allocator);
- return LZMA_MEM_ERROR;
+
+static lzma_ret
+lz_encode(lzma_coder *coder, lzma_allocator *allocator,
+ const uint8_t *restrict in, size_t *restrict in_pos,
+ size_t in_size,
+ uint8_t *restrict out, size_t *restrict out_pos,
+ size_t out_size, lzma_action action)
+{
+ while (*out_pos < out_size
+ && (*in_pos < in_size || action != LZMA_RUN)) {
+ // Read more data to coder->mf.buffer if needed.
+ if (coder->mf.action == LZMA_RUN && coder->mf.read_pos
+ >= coder->mf.read_limit)
+ return_if_error(fill_window(coder, allocator,
+ in, in_pos, in_size, action));
+
+ // Encode
+ const lzma_ret ret = coder->lz.code(coder->lz.coder,
+ &coder->mf, out, out_pos, out_size);
+ if (ret != LZMA_OK) {
+ // Setting this to LZMA_RUN for cases when we are
+ // flushing. It doesn't matter when finishing or if
+ // an error occurred.
+ coder->mf.action = LZMA_RUN;
+ return ret;
}
}
- // Allocation successful. Store the new size.
- lz->size = buffer_size;
+ return LZMA_OK;
+}
+
+
+static bool
+lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator,
+ const lzma_lz_options *lz_options)
+{
+ if (lz_options->dictionary_size < LZMA_DICTIONARY_SIZE_MIN
+ || lz_options->dictionary_size
+ > LZMA_DICTIONARY_SIZE_MAX
+ || lz_options->find_len_max
+ > lz_options->match_len_max)
+ return true;
+
+ mf->keep_size_before = lz_options->before_size
+ + lz_options->dictionary_size;
- // Reset in window variables.
- lz->offset = 0;
- lz->read_pos = 0;
- lz->read_limit = 0;
- lz->write_pos = 0;
- lz->pending = 0;
+ mf->keep_size_after = lz_options->after_size
+ + lz_options->match_len_max;
+ // To avoid constant memmove()s, allocate some extra space. Since
+ // memmove()s become more expensive when the size of the buffer
+ // increases, we reserve more space when a large dictionary is
+ // used to make the memmove() calls rarer.
+ uint32_t reserve = lz_options->dictionary_size / 2;
+ if (reserve > (UINT32_C(1) << 30))
+ reserve /= 2;
- //////////////////
- // Match Finder //
- //////////////////
+ reserve += (lz_options->before_size + lz_options->match_len_max
+ + lz_options->after_size) / 2 + (UINT32_C(1) << 19);
- // Validate match_finder, set function pointers and a few match
- // finder specific variables.
- switch (match_finder) {
-#ifdef HAVE_HC3
+ const uint32_t old_size = mf->size;
+ mf->size = mf->keep_size_before + reserve + mf->keep_size_after;
+
+ // FIXME Integer overflows
+
+ // Deallocate the old history buffer if it exists but has different
+ // size than what is needed now.
+ if (mf->buffer != NULL && old_size != mf->size) {
+ lzma_free(mf->buffer, allocator);
+ mf->buffer = NULL;
+ }
+
+ // Match finder options
+ mf->match_len_max = lz_options->match_len_max;
+ mf->find_len_max = lz_options->find_len_max;
+ mf->cyclic_buffer_size = lz_options->dictionary_size + 1;
+
+ // Validate the match finder ID and setup the function pointers.
+ switch (lz_options->match_finder) {
+#ifdef HAVE_MF_HC3
case LZMA_MF_HC3:
- lz->get_matches = &lzma_hc3_get_matches;
- lz->skip = &lzma_hc3_skip;
- lz->cut_value = 8 + (match_max_len >> 2);
+ mf->find = &lzma_mf_hc3_find;
+ mf->skip = &lzma_mf_hc3_skip;
break;
#endif
-#ifdef HAVE_HC4
+#ifdef HAVE_MF_HC4
case LZMA_MF_HC4:
- lz->get_matches = &lzma_hc4_get_matches;
- lz->skip = &lzma_hc4_skip;
- lz->cut_value = 8 + (match_max_len >> 2);
+ mf->find = &lzma_mf_hc4_find;
+ mf->skip = &lzma_mf_hc4_skip;
break;
#endif
-#ifdef HAVE_BT2
+#ifdef HAVE_MF_BT2
case LZMA_MF_BT2:
- lz->get_matches = &lzma_bt2_get_matches;
- lz->skip = &lzma_bt2_skip;
- lz->cut_value = 16 + (match_max_len >> 1);
+ mf->find = &lzma_mf_bt2_find;
+ mf->skip = &lzma_mf_bt2_skip;
break;
#endif
-#ifdef HAVE_BT3
+#ifdef HAVE_MF_BT3
case LZMA_MF_BT3:
- lz->get_matches = &lzma_bt3_get_matches;
- lz->skip = &lzma_bt3_skip;
- lz->cut_value = 16 + (match_max_len >> 1);
+ mf->find = &lzma_mf_bt3_find;
+ mf->skip = &lzma_mf_bt3_skip;
break;
#endif
-#ifdef HAVE_BT4
+#ifdef HAVE_MF_BT4
case LZMA_MF_BT4:
- lz->get_matches = &lzma_bt4_get_matches;
- lz->skip = &lzma_bt4_skip;
- lz->cut_value = 16 + (match_max_len >> 1);
+ mf->find = &lzma_mf_bt4_find;
+ mf->skip = &lzma_mf_bt4_skip;
break;
#endif
+
default:
- lzma_lz_encoder_end(lz, allocator);
- return LZMA_HEADER_ERROR;
+ return true;
}
- // Check if we have been requested to use a non-default cut_value.
- if (match_finder_cycles > 0)
- lz->cut_value = match_finder_cycles;
-
- lz->match_max_len = match_max_len;
- lz->cyclic_buffer_size = get_cyclic_buffer_size(history_size);
+ // Calculate the sizes of mf->hash and mf->son.
+ const uint32_t hash_bytes = lz_options->match_finder & 0x0F;
+ const bool is_bt = (lz_options->match_finder & 0x10) != 0;
+ uint32_t hs;
- uint32_t hash_size_sum;
- uint32_t num_items;
- if (lzma_lz_encoder_hash_properties(match_finder, history_size,
- &lz->hash_mask, &hash_size_sum, &num_items)) {
- lzma_lz_encoder_end(lz, allocator);
- return LZMA_HEADER_ERROR;
- }
+ if (hash_bytes == 2) {
+ hs = 0xFFFF;
+ } else {
+ // Round dictionary size up to the next 2^n - 1 so it can
+ // be used as a hash mask.
+ hs = lz_options->dictionary_size - 1;
+ hs |= hs >> 1;
+ hs |= hs >> 2;
+ hs |= hs >> 4;
+ hs |= hs >> 8;
+ hs >>= 1;
+ hs |= 0xFFFF;
- if (num_items != lz->num_items) {
-#if UINT32_MAX >= SIZE_MAX / 4
- // Check for integer overflow. (Huge dictionaries are not
- // possible on 32-bit CPU.)
- if (num_items > SIZE_MAX / sizeof(uint32_t)) {
- lzma_lz_encoder_end(lz, allocator);
- return LZMA_MEM_ERROR;
+ if (hs > (UINT32_C(1) << 24)) {
+ if (hash_bytes == 3)
+ hs = (UINT32_C(1) << 24) - 1;
+ else
+ hs >>= 1;
}
-#endif
-
- const size_t size_in_bytes
- = (size_t)(num_items) * sizeof(uint32_t);
+ }
- lzma_free(lz->hash, allocator);
- lz->hash = lzma_alloc(size_in_bytes, allocator);
- if (lz->hash == NULL) {
- lzma_lz_encoder_end(lz, allocator);
- return LZMA_MEM_ERROR;
- }
+ mf->hash_mask = hs;
+
+ ++hs;
+ if (hash_bytes > 2)
+ hs += HASH_2_SIZE;
+ if (hash_bytes > 3)
+ hs += HASH_3_SIZE;
+/*
+ No match finder uses this at the moment.
+ if (mf->hash_bytes > 4)
+ hs += HASH_4_SIZE;
+*/
+
+ const uint32_t old_count = mf->hash_size_sum + mf->sons_count;
+ mf->hash_size_sum = hs;
+ mf->sons_count = mf->cyclic_buffer_size;
+ if (is_bt)
+ mf->sons_count *= 2;
+
+ const uint32_t new_count = mf->hash_size_sum + mf->sons_count;
+
+ // Deallocate the old hash array if it exists and has different size
+ // than what is needed now.
+ if (mf->hash != NULL && old_count != new_count) {
+ lzma_free(mf->hash, allocator);
+ mf->hash = NULL;
+ }
- lz->num_items = num_items;
+ // Maximum number of match finder cycles
+ mf->loops = lz_options->match_finder_cycles;
+ if (mf->loops == 0) {
+ mf->loops = 16 + (lz_options->find_len_max / 2);
+ if (!is_bt)
+ mf->loops /= 2;
}
- lz->son = lz->hash + hash_size_sum;
+ return false;
+}
- // Reset the hash table to empty hash values.
- {
- uint32_t *restrict items = lz->hash;
- for (uint32_t i = 0; i < hash_size_sum; ++i)
- items[i] = EMPTY_HASH_VALUE;
+static bool
+lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator)
+{
+ // Allocate the history buffer.
+ if (mf->buffer == NULL) {
+ mf->buffer = lzma_alloc(mf->size, allocator);
+ if (mf->buffer == NULL)
+ return true;
}
- lz->cyclic_buffer_pos = 0;
+ // Use cyclic_buffer_size as initial mf->offset. This allows
+ // avoiding a few branches in the match finders. The downside is
+ // that match finder needs to be normalized more often, which may
+ // hurt performance with huge dictionaries.
+ mf->offset = mf->cyclic_buffer_size;
+ mf->read_pos = 0;
+ mf->read_ahead = 0;
+ mf->read_limit = 0;
+ mf->write_pos = 0;
+ mf->pending = 0;
- // Because zero is used as empty hash value, make the first byte
- // appear at buffer[1 - offset].
- ++lz->offset;
+ // Allocate match finder's hash array.
+ const size_t alloc_count = mf->hash_size_sum + mf->sons_count;
- // If we are using a preset dictionary, read it now.
- // TODO: This isn't implemented yet so return LZMA_HEADER_ERROR.
- if (preset_dictionary != NULL && preset_dictionary_size > 0) {
- lzma_lz_encoder_end(lz, allocator);
- return LZMA_HEADER_ERROR;
+#if UINT32_MAX >= SIZE_MAX / 4
+ // Check for integer overflow. (Huge dictionaries are not
+ // possible on 32-bit CPU.)
+ if (alloc_count > SIZE_MAX / sizeof(uint32_t))
+ return true;
+#endif
+
+ if (mf->hash == NULL) {
+ mf->hash = lzma_alloc(alloc_count * sizeof(uint32_t),
+ allocator);
+ if (mf->hash == NULL)
+ return true;
}
- // Set the process function pointer.
- lz->process = process;
+ mf->son = mf->hash + mf->hash_size_sum;
+ mf->cyclic_buffer_pos = 0;
+
+ // Initialize the hash table. Since EMPTY_HASH_VALUE is zero, we
+ // can use memset().
+/*
+ for (uint32_t i = 0; i < hash_size_sum; ++i)
+ mf->hash[i] = EMPTY_HASH_VALUE;
+*/
+ memzero(mf->hash, (size_t)(mf->hash_size_sum) * sizeof(uint32_t));
+
+ // We don't need to initialize mf->son, but not doing that will
+ // make Valgrind complain in normalization (see normalize() in
+ // lz_encoder_mf.c).
+ //
+ // Skipping this initialization is *very* good when big dictionary is
+ // used but only small amount of data gets actually compressed: most
+ // of the mf->hash won't get actually allocated by the kernel, so
+ // we avoid wasting RAM and improve initialization speed a lot.
+ //memzero(mf->son, (size_t)(mf->sons_count) * sizeof(uint32_t));
+
+ mf->action = LZMA_RUN;
- return LZMA_OK;
+ return false;
}
-extern void
-lzma_lz_encoder_end(lzma_lz_encoder *lz, lzma_allocator *allocator)
+extern uint64_t
+lzma_lz_encoder_memusage(const lzma_lz_options *lz_options)
{
- lzma_free(lz->hash, allocator);
- lz->hash = NULL;
- lz->num_items = 0;
-
- lzma_free(lz->buffer, allocator);
- lz->buffer = NULL;
- lz->size = 0;
-
- return;
+ // Old buffers must not exist when calling lz_encoder_prepare().
+ lzma_mf mf = {
+ .buffer = NULL,
+ .hash = NULL,
+ };
+
+ // Setup the size information into mf.
+ if (lz_encoder_prepare(&mf, NULL, lz_options))
+ return UINT64_MAX;
+
+ // Calculate the memory usage.
+ return (uint64_t)(mf.hash_size_sum + mf.sons_count)
+ * sizeof(uint32_t)
+ + (uint64_t)(mf.size) + sizeof(lzma_coder);
}
-/// \brief Moves the data in the input window to free space for new data
-///
-/// lz->buffer is a sliding input window, which keeps lz->keep_size_before
-/// bytes of input history available all the time. Now and then we need to
-/// "slide" the buffer to make space for the new data to the end of the
-/// buffer. At the same time, data older than keep_size_before is dropped.
-///
static void
-move_window(lzma_lz_encoder *lz)
+lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- // buffer[move_offset] will become buffer[0].
- assert(lz->read_pos > lz->keep_size_after);
- size_t move_offset = lz->read_pos - lz->keep_size_before;
-
- // We need one additional byte, since move_pos() moves on 1 byte.
- // TODO: Clean up? At least document more.
- if (move_offset > 0)
- --move_offset;
-
- assert(lz->write_pos > move_offset);
- const size_t move_size = lz->write_pos - move_offset;
+ lzma_next_end(&coder->next, allocator);
- assert(move_offset + move_size <= lz->size);
+ lzma_free(coder->mf.hash, allocator);
+ lzma_free(coder->mf.buffer, allocator);
- memmove(lz->buffer, lz->buffer + move_offset, move_size);
-
- lz->offset += move_offset;
- lz->read_pos -= move_offset;
- lz->read_limit -= move_offset;
- lz->write_pos -= move_offset;
+ if (coder->lz.end != NULL)
+ coder->lz.end(coder->lz.coder, allocator);
+ else
+ lzma_free(coder->lz.coder, allocator);
+ lzma_free(coder, allocator);
return;
}
-/// \brief Tries to fill the input window (lz->buffer)
-///
-/// If we are the last encoder in the chain, our input data is in in[].
-/// Otherwise we call the next filter in the chain to process in[] and
-/// write its output to lz->buffer.
-///
-/// This function must not be called once it has returned LZMA_STREAM_END.
-///
-static lzma_ret
-fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in,
- size_t *in_pos, size_t in_size, lzma_action action)
+extern lzma_ret
+lzma_lz_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter_info *filters,
+ lzma_ret (*lz_init)(lzma_lz_encoder *lz,
+ lzma_allocator *allocator, const void *options,
+ lzma_lz_options *lz_options))
{
- assert(coder->lz.read_pos <= coder->lz.write_pos);
+ // Allocate and initialize the base data structure.
+ if (next->coder == NULL) {
+ next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
+ if (next->coder == NULL)
+ return LZMA_MEM_ERROR;
- // Move the sliding window if needed.
- if (coder->lz.read_pos >= coder->lz.size - coder->lz.keep_size_after)
- move_window(&coder->lz);
+ next->code = &lz_encode;
+ next->end = &lz_encoder_end;
- size_t in_used;
- lzma_ret ret;
- if (coder->next.code == NULL) {
- // Not using a filter, simply memcpy() as much as possible.
- in_used = bufcpy(in, in_pos, in_size, coder->lz.buffer,
- &coder->lz.write_pos, coder->lz.size);
+ next->coder->lz.coder = NULL;
+ next->coder->lz.code = NULL;
+ next->coder->lz.end = NULL;
- if (action != LZMA_RUN && *in_pos == in_size)
- ret = LZMA_STREAM_END;
- else
- ret = LZMA_OK;
+ next->coder->mf.buffer = NULL;
+ next->coder->mf.hash = NULL;
- } else {
- const size_t in_start = *in_pos;
- ret = coder->next.code(coder->next.coder, allocator,
- in, in_pos, in_size,
- coder->lz.buffer, &coder->lz.write_pos,
- coder->lz.size, action);
- in_used = *in_pos - in_start;
+ next->coder->next = LZMA_NEXT_CODER_INIT;
}
- // If end of stream has been reached or flushing completed, we allow
- // the encoder to process all the input (that is, read_pos is allowed
- // to reach write_pos). Otherwise we keep keep_size_after bytes
- // available as prebuffer.
- if (ret == LZMA_STREAM_END) {
- assert(*in_pos == in_size);
- coder->lz.read_limit = coder->lz.write_pos;
- ret = LZMA_OK;
+ // Initialize the LZ-based encoder.
+ lzma_lz_options lz_options;
+ return_if_error(lz_init(&next->coder->lz, allocator,
+ filters[0].options, &lz_options));
- switch (action) {
- case LZMA_SYNC_FLUSH:
- coder->lz.sequence = SEQ_FLUSH;
- break;
-
- case LZMA_FINISH:
- coder->lz.sequence = SEQ_FINISH;
- break;
-
- default:
- assert(0);
- ret = LZMA_PROG_ERROR;
- break;
- }
-
- } else if (coder->lz.write_pos > coder->lz.keep_size_after) {
- // This needs to be done conditionally, because if we got
- // only little new input, there may be too little input
- // to do any encoding yet.
- coder->lz.read_limit = coder->lz.write_pos
- - coder->lz.keep_size_after;
- }
-
- // Restart the match finder after finished LZMA_SYNC_FLUSH.
- if (coder->lz.pending > 0
- && coder->lz.read_pos < coder->lz.read_limit) {
- // Match finder may update coder->pending and expects it to
- // start from zero, so use a temporary variable.
- const size_t pending = coder->lz.pending;
- coder->lz.pending = 0;
+ // Setup the size information into next->coder->mf and deallocate
+ // old buffers if they have wrong size.
+ if (lz_encoder_prepare(&next->coder->mf, allocator, &lz_options))
+ return LZMA_HEADER_ERROR;
- // Rewind read_pos so that the match finder can hash
- // the pending bytes.
- assert(coder->lz.read_pos >= pending);
- coder->lz.read_pos -= pending;
- coder->lz.skip(&coder->lz, pending);
- }
+ // Allocate new buffers if needed, and do the rest of
+ // the initialization.
+ if (lz_encoder_init(&next->coder->mf, allocator))
+ return LZMA_MEM_ERROR;
- return ret;
+ // Initialize the next filter in the chain, if any.
+ return lzma_next_filter_init(&next->coder->next, allocator,
+ filters + 1);
}
-extern lzma_ret
-lzma_lz_encode(lzma_coder *coder, lzma_allocator *allocator,
- const uint8_t *restrict in, size_t *restrict in_pos,
- size_t in_size,
- uint8_t *restrict out, size_t *restrict out_pos,
- size_t out_size, lzma_action action)
+extern LZMA_API lzma_bool
+lzma_mf_is_supported(lzma_match_finder mf)
{
- while (*out_pos < out_size
- && (*in_pos < in_size || action != LZMA_RUN)) {
- // Read more data to coder->lz.buffer if needed.
- if (coder->lz.sequence == SEQ_RUN
- && coder->lz.read_pos >= coder->lz.read_limit)
- return_if_error(fill_window(coder, allocator,
- in, in_pos, in_size, action));
+ bool ret = false;
- // Encode
- if (coder->lz.process(coder, out, out_pos, out_size)) {
- // Setting this to SEQ_RUN for cases when we are
- // flushing. It doesn't matter when finishing.
- coder->lz.sequence = SEQ_RUN;
- return action != LZMA_RUN ? LZMA_STREAM_END : LZMA_OK;
- }
- }
+#ifdef HAVE_MF_HC3
+ if (mf == LZMA_MF_HC3)
+ ret = true;
+#endif
- return LZMA_OK;
-}
+#ifdef HAVE_MF_HC4
+ if (mf == LZMA_MF_HC4)
+ ret = true;
+#endif
+#ifdef HAVE_MF_BT2
+ if (mf == LZMA_MF_BT2)
+ ret = true;
+#endif
-/// \brief Normalizes hash values
-///
-/// lzma_lz_normalize is called when lz->pos hits MAX_VAL_FOR_NORMALIZE,
-/// which currently happens once every 2 GiB of input data (to be exact,
-/// after the first 2 GiB it happens once every 2 GiB minus dictionary_size
-/// bytes). lz->pos is incremented by lzma_lz_move_pos().
-///
-/// lz->hash contains big amount of offsets relative to lz->buffer.
-/// The offsets are stored as uint32_t, which is the only reasonable
-/// datatype for these offsets; uint64_t would waste far too much RAM
-/// and uint16_t would limit the dictionary to 64 KiB (far too small).
-///
-/// When compressing files over 2 GiB, lz->buffer needs to be moved forward
-/// to avoid integer overflows. We scan the lz->hash array and fix every
-/// value to match the updated lz->buffer.
-extern void
-lzma_lz_encoder_normalize(lzma_lz_encoder *lz)
-{
- const uint32_t subvalue = lz->read_pos - lz->cyclic_buffer_size;
- assert(subvalue <= INT32_MAX);
-
- {
- const uint32_t num_items = lz->num_items;
- uint32_t *restrict items = lz->hash;
-
- for (uint32_t i = 0; i < num_items; ++i) {
- // If the distance is greater than the dictionary
- // size, we can simply mark the item as empty.
- if (items[i] <= subvalue)
- items[i] = EMPTY_HASH_VALUE;
- else
- items[i] -= subvalue;
- }
- }
+#ifdef HAVE_MF_BT3
+ if (mf == LZMA_MF_BT3)
+ ret = true;
+#endif
- // Update offset to match the new locations.
- lz->offset -= subvalue;
+#ifdef HAVE_MF_BT4
+ if (mf == LZMA_MF_BT4)
+ ret = true;
+#endif
- return;
+ return ret;
}
/// \file lz_encoder.h
/// \brief LZ in window and match finder API
//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
+// Copyright (C) 1999-2008 Igor Pavlov
+// Copyright (C) 2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
#include "common.h"
-typedef struct lzma_lz_encoder_s lzma_lz_encoder;
-struct lzma_lz_encoder_s {
- enum {
- SEQ_RUN,
- SEQ_FLUSH,
- SEQ_FINISH,
- } sequence;
+/// A table of these is used by the LZ-based encoder to hold
+/// the length-distance pairs found by the match finder.
+typedef struct {
+ uint32_t len;
+ uint32_t dist;
+} lzma_match;
- /// Function to do the actual encoding from the sliding input window
- /// to the output stream.
- bool (*process)(lzma_coder *coder, uint8_t *restrict out,
- size_t *restrict out_pos, size_t out_size);
+typedef struct lzma_mf_s lzma_mf;
+struct lzma_mf_s {
///////////////
// In Window //
///////////////
/// Total size of the allocated buffer (that is, including all
/// the extra space)
- size_t size;
+ uint32_t size;
+
+ /// Number of bytes that must be kept available in our input history.
+ /// That is, once keep_size_before bytes have been processed,
+ /// buffer[read_pos - keep_size_before] is the oldest byte that
+ /// must be available for reading.
+ uint32_t keep_size_before;
+
+ /// Number of bytes that must be kept in buffer after read_pos.
+ /// That is, read_pos <= write_pos - keep_size_after as long as
+ /// stream_end_was_reached is false (once it is true, read_pos
+ /// is allowed to reach write_pos).
+ uint32_t keep_size_after;
/// Match finders store locations of matches using 32-bit integers.
/// To avoid adjusting several megabytes of integers every time the
/// input window is moved with move_window(), we only adjust the
/// offset of the buffer. Thus, buffer[match_finder_pos - offset]
/// is the byte pointed by match_finder_pos.
- size_t offset;
+ uint32_t offset;
/// buffer[read_pos] is the current byte.
- size_t read_pos;
+ uint32_t read_pos;
+
+ /// Number of bytes that have been ran through the match finder, but
+ /// which haven't been encoded by the LZ-based encoder yet.
+ uint32_t read_ahead;
/// As long as read_pos is less than read_limit, there is enough
/// input available in buffer for at least one encoding loop.
/// Because of the stateful API, read_limit may and will get greater
/// than read_pos quite often. This is taken into account when
/// calculating the value for keep_size_after.
- size_t read_limit;
+ uint32_t read_limit;
/// buffer[write_pos] is the first byte that doesn't contain valid
/// uncompressed data; that is, the next input byte will be copied
/// to buffer[write_pos].
- size_t write_pos;
+ uint32_t write_pos;
/// Number of bytes not hashed before read_pos. This is needed to
/// restart the match finder after LZMA_SYNC_FLUSH.
- size_t pending;
-
- /// Number of bytes that must be kept available in our input history.
- /// That is, once keep_size_before bytes have been processed,
- /// buffer[read_pos - keep_size_before] is the oldest byte that
- /// must be available for reading.
- size_t keep_size_before;
-
- /// Number of bytes that must be kept in buffer after read_pos.
- /// That is, read_pos <= write_pos - keep_size_after as long as
- /// stream_end_was_reached is false (once it is true, read_pos
- /// is allowed to reach write_pos).
- size_t keep_size_after;
+ uint32_t pending;
//////////////////
// Match Finder //
//////////////////
- // Pointers to match finder functions
- void (*get_matches)(lzma_lz_encoder *restrict lz,
- uint32_t *restrict distances);
- void (*skip)(lzma_lz_encoder *restrict lz, uint32_t num);
+ /// Find matches. Returns the number of distance-length pairs written
+ /// to the matches array. This is called only via lzma_mf_find.
+ uint32_t (*find)(lzma_mf *mf, lzma_match *matches);
+
+ /// Skips num bytes. This is like find() but doesn't make the
+ /// distance-length pairs available, thus being a little faster.
+ /// This is called only via mf_skip function.
+ void (*skip)(lzma_mf *mf, uint32_t num);
- // Match finder data
- uint32_t *hash; // TODO: Check if hash aliases son
- uint32_t *son; // and add 'restrict' if possible.
+ uint32_t *hash;
+ uint32_t *son;
uint32_t cyclic_buffer_pos;
uint32_t cyclic_buffer_size; // Must be dictionary_size + 1.
uint32_t hash_mask;
- uint32_t cut_value;
+
+ /// Maximum number of loops in the match finder
+ uint32_t loops;
+
+ /// Maximum length of a match that the match finder will try to find.
+ uint32_t find_len_max;
+
+ /// Maximum length of a match supported by the LZ-based encoder.
+ /// If the longest match found by the match finder is find_len_max,
+ /// lz_dict_find() tries to expand it up to match_len_max bytes.
+ uint32_t match_len_max;
+
+ /// When running out of input, binary tree match finders need to know
+ /// if it is due to flushing or finishing. The action is used also
+ /// by the LZ-based encoders themselves.
+ lzma_action action;
+
+ /// Number of elements in hash[]
uint32_t hash_size_sum;
- uint32_t num_items;
- uint32_t match_max_len;
+
+ /// Number of elements in son[]
+ uint32_t sons_count;
};
-#define LZMA_LZ_ENCODER_INIT \
- (lzma_lz_encoder){ \
- .buffer = NULL, \
- .size = 0, \
- .hash = NULL, \
- .num_items = 0, \
+typedef struct {
+ /// Extra amount of data to keep available before the "actual"
+ /// dictionary.
+ size_t before_size;
+
+ /// Size of the history buffer
+ size_t dictionary_size;
+
+ /// Extra amount of data to keep available after the "actual"
+ /// dictionary.
+ size_t after_size;
+
+ /// Maximum length of a match that the LZ-based encoder can accept.
+ /// This is used to extend matches of length find_len_max to the
+ /// maximum possible length.
+ size_t match_len_max;
+
+ /// Match finder will search matches of at maximum of this length.
+ /// This must be less than or equal to match_len_max.
+ size_t find_len_max;
+
+ /// Type of the match finder to use
+ lzma_match_finder match_finder;
+
+ /// TODO: Comment
+ uint32_t match_finder_cycles;
+
+ /// TODO: Comment
+ const uint8_t *preset_dictionary;
+
+ uint32_t preset_dictionary_size;
+
+} lzma_lz_options;
+
+
+// The total usable buffer space at any moment outside the match finder:
+// before_size + dictionary_size + after_size + match_len_max
+//
+// In reality, there's some extra space allocated to prevent the number of
+// memmove() calls reasonable. The bigger the dictionary_size is, the bigger
+// this extra buffer will be since with bigger dictionaries memmove() would
+// also take longer.
+//
+// A single encoder loop in the LZ-based encoder may call the match finder
+// (lz_dict_find() or lz_dict_skip()) at maximum of after_size times.
+// In other words, a single encoder loop may advance lz_dict.read_pos at
+// maximum of after_size times. Since matches are looked up to
+// lz_dict.buffer[lz_dict.read_pos + match_len_max - 1], the total
+// amount of extra buffer needed after dictionary_size becomes
+// after_size + match_len_max.
+//
+// before_size has two uses. The first one is to keep literals available
+// in cases when the LZ-based encoder has made some read ahead.
+// TODO: Maybe this could be changed by making the LZ-based encoders to
+// store the actual literals as they do with length-distance pairs.
+//
+// Alrogithms such as LZMA2 first try to compress a chunk, and then check
+// if the encoded result is smaller than the uncompressed one. If the chunk
+// was uncompressible, it is better to store it in uncompressed form in
+// the output stream. To do this, the whole uncompressed chunk has to be
+// still available in the history buffer. before_size achieves that.
+
+
+typedef struct {
+ /// Data specific to the LZ-based encoder
+ lzma_coder *coder;
+
+ /// Function to encode from *dict to out[]
+ lzma_ret (*code)(lzma_coder *restrict coder,
+ lzma_mf *restrict mf, uint8_t *restrict out,
+ size_t *restrict out_pos, size_t out_size);
+
+ /// Free allocated resources
+ void (*end)(lzma_coder *coder, lzma_allocator *allocator);
+
+} lzma_lz_encoder;
+
+
+// Basic steps:
+// 1. Input gets copied into the dictionary.
+// 2. Data in dictionary gets run through the match finder byte by byte.
+// 3. The literals and matches are encoded using e.g. LZMA.
+//
+// The bytes that have been ran through the match finder, but not encoded yet,
+// are called `read ahead'.
+
+
+/// Get pointer to the first byte not ran through the match finder
+static inline const uint8_t *
+mf_ptr(const lzma_mf *mf)
+{
+ return mf->buffer + mf->read_pos;
+}
+
+
+/// Get the number of bytes that haven't been ran through the match finder yet.
+static inline uint32_t
+mf_avail(const lzma_mf *mf)
+{
+ return mf->write_pos - mf->read_pos;
+}
+
+
+/// Get the number of bytes that haven't been encoded yet (some of these
+/// bytes may have been ran through the match finder though).
+static inline uint32_t
+mf_unencoded(const lzma_mf *mf)
+{
+ return mf->write_pos - mf->read_pos - mf->read_ahead;
+}
+
+
+/// Calculate the absolute offset from the beginning of the most recent
+/// dictionary reset. Only the lowest four bits are important, so there's no
+/// problem that we don't know the 64-bit size of the data encoded so far.
+///
+/// NOTE: When moving the input window, we need to do it so that the lowest
+/// bits of dict->read_pos are not modified to keep this macro working
+/// as intended.
+static inline uint32_t
+mf_position(const lzma_mf *mf)
+{
+ return mf->read_pos - mf->read_ahead;
+}
+
+
+/// Since everything else begins with mf_, use it also for lzma_mf_find().
+#define mf_find lzma_mf_find
+
+
+/// Skip the given number of bytes. This is used when a good match was found.
+/// For example, if mf_find() finds a match of 200 bytes long, the first byte
+/// of that match was already consumed by mf_find(), and the rest 199 bytes
+/// have to be skipped with mf_skip(mf, 199).
+static inline void
+mf_skip(lzma_mf *mf, uint32_t amount)
+{
+ if (amount != 0) {
+ mf->skip(mf, amount);
+ mf->read_ahead += amount;
}
+}
+
+
+/// Copies at maximum of *left amount of bytes from the history buffer
+/// to out[]. This is needed by LZMA2 to encode uncompressed chunks.
+static inline void
+mf_read(lzma_mf *mf, uint8_t *out, size_t *out_pos, size_t out_size,
+ size_t *left)
+{
+ const size_t out_avail = out_size - *out_pos;
+ const size_t copy_size = MIN(out_avail, *left);
+
+ assert(mf->read_ahead == 0);
+ assert(mf->read_pos >= *left);
+
+ memcpy(out + *out_pos, mf->buffer + mf->read_pos - *left,
+ copy_size);
+
+ *out_pos += copy_size;
+ *left -= copy_size;
+ return;
+}
+
+
+extern lzma_ret lzma_lz_encoder_init(
+ lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter_info *filters,
+ lzma_ret (*lz_init)(lzma_lz_encoder *lz,
+ lzma_allocator *allocator, const void *options,
+ lzma_lz_options *lz_options));
+
+
+extern uint64_t lzma_lz_encoder_memusage(const lzma_lz_options *lz_options);
+
+
+// These are only for LZ encoder's internal use.
+extern uint32_t lzma_mf_find(
+ lzma_mf *mf, uint32_t *count, lzma_match *matches);
+
+extern uint32_t lzma_mf_hc3_find(lzma_mf *dict, lzma_match *matches);
+extern void lzma_mf_hc3_skip(lzma_mf *dict, uint32_t amount);
+
+extern uint32_t lzma_mf_hc4_find(lzma_mf *dict, lzma_match *matches);
+extern void lzma_mf_hc4_skip(lzma_mf *dict, uint32_t amount);
+
+extern uint32_t lzma_mf_bt2_find(lzma_mf *dict, lzma_match *matches);
+extern void lzma_mf_bt2_skip(lzma_mf *dict, uint32_t amount);
+extern uint32_t lzma_mf_bt3_find(lzma_mf *dict, lzma_match *matches);
+extern void lzma_mf_bt3_skip(lzma_mf *dict, uint32_t amount);
-/// Calculates
-extern bool lzma_lz_encoder_hash_properties(lzma_match_finder match_finder,
- uint32_t history_size, uint32_t *restrict hash_mask,
- uint32_t *restrict hash_size_sum,
- uint32_t *restrict num_items);
-
-// NOTE: liblzma doesn't use callback API like LZMA SDK does. The caller
-// must make sure that keep_size_after is big enough for single encoding pass
-// i.e. keep_size_after >= maximum number of bytes possibly needed after
-// the current position between calls to lzma_lz_read().
-extern lzma_ret lzma_lz_encoder_reset(lzma_lz_encoder *lz,
- lzma_allocator *allocator,
- bool (*process)(lzma_coder *coder, uint8_t *restrict out,
- size_t *restrict out_pos, size_t out_size),
- size_t history_size, size_t additional_buffer_before,
- size_t match_max_len, size_t additional_buffer_after,
- lzma_match_finder match_finder, uint32_t match_finder_cycles,
- const uint8_t *preset_dictionary,
- size_t preset_dictionary_size);
-
-/// Frees memory allocated for in window and match finder buffers.
-extern void lzma_lz_encoder_end(
- lzma_lz_encoder *lz, lzma_allocator *allocator);
-
-extern lzma_ret lzma_lz_encode(lzma_coder *coder,
- lzma_allocator *allocator lzma_attribute((unused)),
- const uint8_t *restrict in, size_t *restrict in_pos,
- size_t in_size, uint8_t *restrict out,
- size_t *restrict out_pos, size_t out_size,
- lzma_action action);
-
-/// This should not be called directly, but only via move_pos() macro.
-extern void lzma_lz_encoder_normalize(lzma_lz_encoder *lz);
+extern uint32_t lzma_mf_bt4_find(lzma_mf *dict, lzma_match *matches);
+extern void lzma_mf_bt4_skip(lzma_mf *dict, uint32_t amount);
#endif
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file lz_encoder_hash.h
+/// \brief Hash macros for match finders
+//
+// Copyright (C) 1999-2008 Igor Pavlov
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef LZMA_LZ_ENCODER_HASH_H
+#define LZMA_LZ_ENCODER_HASH_H
+
+#define HASH_2_SIZE (UINT32_C(1) << 10)
+#define HASH_3_SIZE (UINT32_C(1) << 16)
+#define HASH_4_SIZE (UINT32_C(1) << 20)
+
+#define HASH_2_MASK (HASH_2_SIZE - 1)
+#define HASH_3_MASK (HASH_3_SIZE - 1)
+#define HASH_4_MASK (HASH_4_SIZE - 1)
+
+#define FIX_3_HASH_SIZE (HASH_2_SIZE)
+#define FIX_4_HASH_SIZE (HASH_2_SIZE + HASH_3_SIZE)
+#define FIX_5_HASH_SIZE (HASH_2_SIZE + HASH_3_SIZE + HASH_4_SIZE)
+
+// TODO Benchmark, and probably doesn't need to be endian dependent.
+#if !defined(WORDS_BIGENDIAN) && defined(HAVE_FAST_UNALIGNED_ACCESS)
+# define hash_2_calc() \
+ const uint32_t hash_value = *(const uint16_t *)(cur);
+#else
+# define hash_2_calc() \
+ const uint32_t hash_value \
+ = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8)
+#endif
+
+#define hash_3_calc() \
+ const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \
+ const uint32_t hash_2_value = temp & HASH_2_MASK; \
+ const uint32_t hash_value \
+ = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask
+
+#define hash_4_calc() \
+ const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \
+ const uint32_t hash_2_value = temp & HASH_2_MASK; \
+ const uint32_t hash_3_value \
+ = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \
+ const uint32_t hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \
+ ^ (lzma_crc32_table[0][cur[3]] << 5)) & mf->hash_mask
+
+
+// The following are not currently used.
+
+#define hash_5_calc() \
+ const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \
+ const uint32_t hash_2_value = temp & HASH_2_MASK; \
+ const uint32_t hash_3_value \
+ = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \
+ uint32_t hash_4_value = (temp ^ ((uint32_t)(cur[2]) << 8) ^ \
+ ^ lzma_crc32_table[0][cur[3]] << 5); \
+ const uint32_t hash_value \
+ = (hash_4_value ^ (lzma_crc32_table[0][cur[4]] << 3)) \
+ & mf->hash_mask; \
+ hash_4_value &= HASH_4_MASK
+
+/*
+#define hash_zip_calc() \
+ const uint32_t hash_value \
+ = (((uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8)) \
+ ^ lzma_crc32_table[0][cur[2]]) & 0xFFFF
+*/
+
+#define hash_zip_calc() \
+ const uint32_t hash_value \
+ = (((uint32_t)(cur[2]) | ((uint32_t)(cur[0]) << 8)) \
+ ^ lzma_crc32_table[0][cur[1]]) & 0xFFFF
+
+#define mt_hash_2_calc() \
+ const uint32_t hash_2_value \
+ = (lzma_crc32_table[0][cur[0]] ^ cur[1]) & HASH_2_MASK
+
+#define mt_hash_3_calc() \
+ const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \
+ const uint32_t hash_2_value = temp & HASH_2_MASK; \
+ const uint32_t hash_3_value \
+ = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK
+
+#define mt_hash_4_calc() \
+ const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \
+ const uint32_t hash_2_value = temp & HASH_2_MASK; \
+ const uint32_t hash_3_value \
+ = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \
+ const uint32_t hash_4_value = (temp ^ ((uint32_t)(cur[2]) << 8) ^ \
+ (lzma_crc32_table[0][cur[3]] << 5)) & HASH_4_MASK
+
+#endif
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file lz_encoder_mf.c
+/// \brief Match finders
+//
+// Copyright (C) 1999-2008 Igor Pavlov
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "lz_encoder.h"
+#include "lz_encoder_hash.h"
+#include "check.h"
+
+
+/// \brief Find matches starting from the current byte
+///
+/// \return The length of the longest match found
+extern uint32_t
+lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches)
+{
+ // Call the match finder. It returns the number of length-distance
+ // pairs found.
+ // FIXME: Minimum count is zero, what _exactly_ is the maximum?
+ const uint32_t count = mf->find(mf, matches);
+
+ // Length of the longest match; assume that no matches were found
+ // and thus the maximum length is zero.
+ uint32_t len_best = 0;
+
+ if (count > 0) {
+#ifndef NDEBUG
+ // Validate the matches.
+ for (uint32_t i = 0; i < count; ++i) {
+ assert(matches[i].len <= mf->find_len_max);
+ assert(matches[i].dist < mf->read_pos);
+ assert(memcmp(mf_ptr(mf) - 1,
+ mf_ptr(mf) - matches[i].dist - 2,
+ matches[i].len) == 0);
+ }
+#endif
+
+ // The last used element in the array contains
+ // the longest match.
+ len_best = matches[count - 1].len;
+
+ // If a match of maximum search length was found, try to
+ // extend the match to maximum possible length.
+ if (len_best == mf->find_len_max) {
+ // The limit for the match length is either the
+ // maximum match length supported by the LZ-based
+ // encoder or the number of bytes left in the
+ // dictionary, whichever is smaller.
+ uint32_t limit = mf_avail(mf) + 1;
+ if (limit > mf->match_len_max)
+ limit = mf->match_len_max;
+
+ // Pointer to the byte we just ran through
+ // the match finder.
+ const uint8_t *p1 = mf_ptr(mf) - 1;
+
+ // Pointer to the beginning of the match. We need -1
+ // here because the match distances are zero based.
+ const uint8_t *p2 = p1 - matches[count - 1].dist - 1;
+
+ while (len_best < limit
+ && p1[len_best] == p2[len_best])
+ ++len_best;
+ }
+ }
+
+ *count_ptr = count;
+
+ // Finally update the read position to indicate that match finder was
+ // run for this dictionary offset.
+ ++mf->read_ahead;
+
+ return len_best;
+}
+
+
+/// Hash value to indicate unused element in the hash. Since we start the
+/// positions from dictionary_size + 1, zero is always too far to qualify
+/// as usable match position.
+#define EMPTY_HASH_VALUE 0
+
+
+/// Normalization must be done when lzma_mf.offset + lzma_mf.read_pos
+/// reaches MUST_NORMALIZE_POS.
+#define MUST_NORMALIZE_POS UINT32_MAX
+
+
+/// \brief Normalizes hash values
+///
+/// The hash arrays store positions of match candidates. The positions are
+/// relative to an arbitrary offset that is not the same as the absolute
+/// offset in the input stream. The relative position of the current byte
+/// is lzma_mf.offset + lzma_mf.read_pos. The distances of the matches are
+/// the differences of the current read position and the position found from
+/// the hash.
+///
+/// To prevent integer overflows of the offsets stored in the hash arrays,
+/// we need to "normalize" the stored values now and then. During the
+/// normalization, we drop values that indicate distance greater than the
+/// dictionary size, thus making space for new values.
+static void
+normalize(lzma_mf *mf)
+{
+ assert(mf->read_pos + mf->offset == MUST_NORMALIZE_POS);
+
+ // In future we may not want to touch the lowest bits, because there
+ // may be match finders that use larger resolution than one byte.
+ const uint32_t subvalue
+ = (MUST_NORMALIZE_POS - mf->cyclic_buffer_size);
+ // & (~(UINT32_C(1) << 10) - 1);
+
+ const uint32_t count = mf->hash_size_sum + mf->sons_count;
+ uint32_t *hash = mf->hash;
+
+ for (uint32_t i = 0; i < count; ++i) {
+ // If the distance is greater than the dictionary size,
+ // we can simply mark the hash element as empty.
+ //
+ // NOTE: Only the first mf->hash_size_sum elements are
+ // initialized for sure. There may be uninitialized elements
+ // in mf->son. Since we go through both mf->hash and
+ // mf->son here in normalization, Valgrind may complain
+ // that the "if" below depends on uninitialized value. In
+ // this case it is safe to ignore the warning. See also the
+ // comments in lz_encoder_init() in lz_encoder.c.
+ if (hash[i] <= subvalue)
+ hash[i] = EMPTY_HASH_VALUE;
+ else
+ hash[i] -= subvalue;
+ }
+
+ // Update offset to match the new locations.
+ mf->offset -= subvalue;
+
+ return;
+}
+
+
+/// Mark the current byte as processed from point of view of the match finder.
+static void
+move_pos(lzma_mf *mf)
+{
+ if (++mf->cyclic_buffer_pos == mf->cyclic_buffer_size)
+ mf->cyclic_buffer_pos = 0;
+
+ ++mf->read_pos;
+ assert(mf->read_pos <= mf->write_pos);
+
+ if (unlikely(mf->read_pos + mf->offset == UINT32_MAX))
+ normalize(mf);
+}
+
+
+/// When flushing, we cannot run the match finder unless there is find_len_max
+/// bytes available in the dictionary. Instead, we skip running the match
+/// finder (indicating that no match was found), and count how many bytes we
+/// have ignored this way.
+///
+/// When new data is given after the flushing was completed, the match finder
+/// is restarted by rewinding mf->read_pos backwards by mf->pending. Then
+/// the missed bytes are added to the hash using the match finder's skip
+/// function (with small amount of input, it may start using mf->pending
+/// again if flushing).
+///
+/// Due to this rewinding, we don't touch cyclic_buffer_pos or test for
+/// normalization. It will be done when the match finder's skip function
+/// catches up after a flush.
+static void
+move_pending(lzma_mf *mf)
+{
+ ++mf->read_pos;
+ assert(mf->read_pos <= mf->write_pos);
+ ++mf->pending;
+}
+
+
+/// Calculate len_limit and determine if there is enough input to run
+/// the actual match finder code. Sets up "cur" and "pos". This macro
+/// is used by all find functions and binary tree skip functions. Hash
+/// chain skip function doesn't need len_limit so a simpler code is used
+/// in them.
+#define header(is_bt, len_min, ret_op) \
+ uint32_t len_limit = mf_avail(mf); \
+ if (mf->find_len_max <= len_limit) { \
+ len_limit = mf->find_len_max; \
+ } else if (len_limit < (len_min) \
+ || (is_bt && mf->action == LZMA_SYNC_FLUSH)) { \
+ assert(mf->action != LZMA_RUN); \
+ move_pending(mf); \
+ ret_op; \
+ } \
+ const uint8_t *cur = mf_ptr(mf); \
+ const uint32_t pos = mf->read_pos + mf->offset
+
+
+/// Header for find functions. "return 0" indicates that zero matches
+/// were found.
+#define header_find(is_bt, len_min) \
+ header(is_bt, len_min, return 0); \
+ uint32_t matches_count = 0
+
+
+/// Header for a loop in a skip function. "continue" tells to skip the rest
+/// of the code in the loop.
+#define header_skip(is_bt, len_min) \
+ header(is_bt, len_min, continue)
+
+
+/// Calls hc_find_func() or bt_find_func() and calculates the total number
+/// of matches found. Updates the dictionary position and returns the number
+/// of matches found.
+#define call_find(func, len_best) \
+do { \
+ matches_count = func(len_limit, pos, cur, cur_match, mf->loops, \
+ mf->son, mf->cyclic_buffer_pos, \
+ mf->cyclic_buffer_size, \
+ matches + matches_count, len_best) \
+ - matches; \
+ move_pos(mf); \
+ return matches_count; \
+} while (0)
+
+
+////////////////
+// Hash Chain //
+////////////////
+
+#if defined(HAVE_MF_HC3) || defined(HAVE_MF_HC4)
+///
+///
+/// \param len_limit Don't look for matches longer than len_limit.
+/// \param pos lzma_mf.read_pos + lzma_mf.offset
+/// \param cur Pointer to current byte (lzma_dict_ptr(mf))
+/// \param cur_match Start position of the current match candidate
+/// \param loops Maximum length of the hash chain
+/// \param son lzma_mf.son (contains the hash chain)
+/// \param cyclic_buffer_pos
+/// \param cyclic_buffer_size
+/// \param matches Array to hold the matches.
+/// \param len_best The length of the longest match found so far.
+static lzma_match *
+hc_find_func(
+ const uint32_t len_limit,
+ const uint32_t pos,
+ const uint8_t *const cur,
+ uint32_t cur_match,
+ uint32_t loops,
+ uint32_t *const son,
+ const uint32_t cyclic_buffer_pos,
+ const uint32_t cyclic_buffer_size,
+ lzma_match *matches,
+ uint32_t len_best)
+{
+ son[cyclic_buffer_pos] = cur_match;
+
+ while (true) {
+ const uint32_t delta = pos - cur_match;
+ if (loops-- == 0 || delta >= cyclic_buffer_size)
+ return matches;
+
+ const uint8_t *const pb = cur - delta;
+ cur_match = son[cyclic_buffer_pos - delta
+ + (delta > cyclic_buffer_pos
+ ? cyclic_buffer_size : 0)];
+
+ if (pb[len_best] == cur[len_best] && pb[0] == cur[0]) {
+ uint32_t len = 0;
+ while (++len != len_limit)
+ if (pb[len] != cur[len])
+ break;
+
+ if (len_best < len) {
+ len_best = len;
+ matches->len = len;
+ matches->dist = delta - 1;
+ ++matches;
+
+ if (len == len_limit)
+ return matches;
+ }
+ }
+ }
+}
+
+/*
+#define hc_header_find(len_min, ret_op) \
+ uint32_t len_limit = mf_avail(mf); \
+ if (mf->find_len_max <= len_limit) { \
+ len_limit = mf->find_len_max; \
+ } else if (len_limit < (len_min)) { \
+ move_pending(mf); \
+ ret_op; \
+ } \
+#define header_hc(len_min, ret_op) \
+do { \
+ if (mf_avail(mf) < (len_min)) { \
+ move_pending(mf); \
+ ret_op; \
+ } \
+} while (0)
+*/
+
+#define hc_find(len_best) \
+ call_find(hc_find_func, len_best)
+
+
+#define hc_skip() \
+do { \
+ mf->son[mf->cyclic_buffer_pos] = cur_match; \
+ move_pos(mf); \
+} while (0)
+
+#endif
+
+
+#ifdef HAVE_MF_HC3
+extern uint32_t
+lzma_mf_hc3_find(lzma_mf *mf, lzma_match *matches)
+{
+ header_find(false, 3);
+
+ hash_3_calc();
+
+ const uint32_t delta2 = pos - mf->hash[hash_2_value];
+ const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value];
+
+ mf->hash[hash_2_value] = pos;
+ mf->hash[FIX_3_HASH_SIZE + hash_value] = pos;
+
+ uint32_t len_best = 2;
+
+ if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) {
+ for ( ; len_best != len_limit; ++len_best)
+ if (*(cur + len_best - delta2) != cur[len_best])
+ break;
+
+ matches[0].len = len_best;
+ matches[0].dist = delta2 - 1;
+ matches_count = 1;
+
+ if (len_best == len_limit) {
+ hc_skip();
+ return 1; // matches_count
+ }
+ }
+
+ hc_find(len_best);
+}
+
+
+extern void
+lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount)
+{
+ do {
+ if (mf_avail(mf) < 3) {
+ move_pending(mf);
+ continue;
+ }
+
+ const uint8_t *cur = mf_ptr(mf);
+ const uint32_t pos = mf->read_pos + mf->offset;
+
+ hash_3_calc();
+
+ const uint32_t cur_match
+ = mf->hash[FIX_3_HASH_SIZE + hash_value];
+
+ mf->hash[hash_2_value] = pos;
+ mf->hash[FIX_3_HASH_SIZE + hash_value] = pos;
+
+ hc_skip();
+
+ } while (--amount != 0);
+}
+#endif
+
+
+#ifdef HAVE_MF_HC4
+extern uint32_t
+lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches)
+{
+ header_find(false, 4);
+
+ hash_4_calc();
+
+ uint32_t delta2 = pos - mf->hash[hash_2_value];
+ const uint32_t delta3
+ = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value];
+ const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value];
+
+ mf->hash[hash_2_value ] = pos;
+ mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos;
+ mf->hash[FIX_4_HASH_SIZE + hash_value] = pos;
+
+ uint32_t len_best = 1;
+
+ if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) {
+ len_best = 2;
+ matches[0].len = 2;
+ matches[0].dist = delta2 - 1;
+ matches_count = 1;
+ }
+
+ if (delta2 != delta3 && delta3 < mf->cyclic_buffer_size
+ && *(cur - delta3) == *cur) {
+ len_best = 3;
+ matches[matches_count++].dist = delta3 - 1;
+ delta2 = delta3;
+ }
+
+ if (matches_count != 0) {
+ for ( ; len_best != len_limit; ++len_best)
+ if (*(cur + len_best - delta2) != cur[len_best])
+ break;
+
+ matches[matches_count - 1].len = len_best;
+
+ if (len_best == len_limit) {
+ hc_skip();
+ return matches_count;
+ }
+ }
+
+ if (len_best < 3)
+ len_best = 3;
+
+ hc_find(len_best);
+}
+
+
+extern void
+lzma_mf_hc4_skip(lzma_mf *mf, uint32_t amount)
+{
+ do {
+ if (mf_avail(mf) < 4) {
+ move_pending(mf);
+ continue;
+ }
+
+ const uint8_t *cur = mf_ptr(mf);
+ const uint32_t pos = mf->read_pos + mf->offset;
+
+ hash_4_calc();
+
+ const uint32_t cur_match
+ = mf->hash[FIX_4_HASH_SIZE + hash_value];
+
+ mf->hash[hash_2_value] = pos;
+ mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos;
+ mf->hash[FIX_4_HASH_SIZE + hash_value] = pos;
+
+ hc_skip();
+
+ } while (--amount != 0);
+}
+#endif
+
+
+/////////////////
+// Binary Tree //
+/////////////////
+
+#if defined(HAVE_MF_BT2) || defined(HAVE_MF_BT3) || defined(HAVE_MF_BT4)
+static lzma_match *
+bt_find_func(
+ const uint32_t len_limit,
+ const uint32_t pos,
+ const uint8_t *const cur,
+ uint32_t cur_match,
+ uint32_t loops,
+ uint32_t *const son,
+ const uint32_t cyclic_buffer_pos,
+ const uint32_t cyclic_buffer_size,
+ lzma_match *matches,
+ uint32_t len_best)
+{
+ uint32_t *ptr0 = son + (cyclic_buffer_pos << 1) + 1;
+ uint32_t *ptr1 = son + (cyclic_buffer_pos << 1);
+
+ uint32_t len0 = 0;
+ uint32_t len1 = 0;
+
+ while (true) {
+ const uint32_t delta = pos - cur_match;
+ if (loops-- == 0 || delta >= cyclic_buffer_size) {
+ *ptr0 = EMPTY_HASH_VALUE;
+ *ptr1 = EMPTY_HASH_VALUE;
+ return matches;
+ }
+
+ uint32_t *const pair = son + ((cyclic_buffer_pos - delta
+ + (delta > cyclic_buffer_pos
+ ? cyclic_buffer_size : 0)) << 1);
+
+ const uint8_t *const pb = cur - delta;
+ uint32_t len = MIN(len0, len1);
+
+ if (pb[len] == cur[len]) {
+ while (++len != len_limit)
+ if (pb[len] != cur[len])
+ break;
+
+ if (len_best < len) {
+ len_best = len;
+ matches->len = len;
+ matches->dist = delta - 1;
+ ++matches;
+
+ if (len == len_limit) {
+ *ptr1 = pair[0];
+ *ptr0 = pair[1];
+ return matches;
+ }
+ }
+ }
+
+ if (pb[len] < cur[len]) {
+ *ptr1 = cur_match;
+ ptr1 = pair + 1;
+ cur_match = *ptr1;
+ len1 = len;
+ } else {
+ *ptr0 = cur_match;
+ ptr0 = pair;
+ cur_match = *ptr0;
+ len0 = len;
+ }
+ }
+}
+
+
+static void
+bt_skip_func(
+ const uint32_t len_limit,
+ const uint32_t pos,
+ const uint8_t *const cur,
+ uint32_t cur_match,
+ uint32_t loops,
+ uint32_t *const son,
+ const uint32_t cyclic_buffer_pos,
+ const uint32_t cyclic_buffer_size)
+{
+ uint32_t *ptr0 = son + (cyclic_buffer_pos << 1) + 1;
+ uint32_t *ptr1 = son + (cyclic_buffer_pos << 1);
+
+ uint32_t len0 = 0;
+ uint32_t len1 = 0;
+
+ while (true) {
+ const uint32_t delta = pos - cur_match;
+ if (loops-- == 0 || delta >= cyclic_buffer_size) {
+ *ptr0 = EMPTY_HASH_VALUE;
+ *ptr1 = EMPTY_HASH_VALUE;
+ return;
+ }
+
+ uint32_t *pair = son + ((cyclic_buffer_pos - delta
+ + (delta > cyclic_buffer_pos
+ ? cyclic_buffer_size : 0)) << 1);
+ const uint8_t *pb = cur - delta;
+ uint32_t len = MIN(len0, len1);
+
+ if (pb[len] == cur[len]) {
+ while (++len != len_limit)
+ if (pb[len] != cur[len])
+ break;
+
+ if (len == len_limit) {
+ *ptr1 = pair[0];
+ *ptr0 = pair[1];
+ return;
+ }
+ }
+
+ if (pb[len] < cur[len]) {
+ *ptr1 = cur_match;
+ ptr1 = pair + 1;
+ cur_match = *ptr1;
+ len1 = len;
+ } else {
+ *ptr0 = cur_match;
+ ptr0 = pair;
+ cur_match = *ptr0;
+ len0 = len;
+ }
+ }
+}
+
+
+#define bt_find(len_best) \
+ call_find(bt_find_func, len_best)
+
+#define bt_skip() \
+do { \
+ bt_skip_func(len_limit, pos, cur, cur_match, mf->loops, \
+ mf->son, mf->cyclic_buffer_pos, \
+ mf->cyclic_buffer_size); \
+ move_pos(mf); \
+} while (0)
+
+#endif
+
+
+#ifdef HAVE_MF_BT2
+extern uint32_t
+lzma_mf_bt2_find(lzma_mf *mf, lzma_match *matches)
+{
+ header_find(true, 2);
+
+ hash_2_calc();
+
+ const uint32_t cur_match = mf->hash[hash_value];
+ mf->hash[hash_value] = pos;
+
+ bt_find(1);
+}
+
+
+extern void
+lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount)
+{
+ do {
+ header_skip(true, 2);
+
+ hash_2_calc();
+
+ const uint32_t cur_match = mf->hash[hash_value];
+ mf->hash[hash_value] = pos;
+
+ bt_skip();
+
+ } while (--amount != 0);
+}
+#endif
+
+
+#ifdef HAVE_MF_BT3
+extern uint32_t
+lzma_mf_bt3_find(lzma_mf *mf, lzma_match *matches)
+{
+ header_find(true, 3);
+
+ hash_3_calc();
+
+ const uint32_t delta2 = pos - mf->hash[hash_2_value];
+ const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value];
+
+ mf->hash[hash_2_value] = pos;
+ mf->hash[FIX_3_HASH_SIZE + hash_value] = pos;
+
+ uint32_t len_best = 2;
+
+ if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) {
+ for ( ; len_best != len_limit; ++len_best)
+ if (*(cur + len_best - delta2) != cur[len_best])
+ break;
+
+ matches[0].len = len_best;
+ matches[0].dist = delta2 - 1;
+ matches_count = 1;
+
+ if (len_best == len_limit) {
+ bt_skip();
+ return 1; // matches_count
+ }
+ }
+
+ bt_find(len_best);
+}
+
+
+extern void
+lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount)
+{
+ do {
+ header_skip(true, 3);
+
+ hash_3_calc();
+
+ const uint32_t cur_match
+ = mf->hash[FIX_3_HASH_SIZE + hash_value];
+
+ mf->hash[hash_2_value] = pos;
+ mf->hash[FIX_3_HASH_SIZE + hash_value] = pos;
+
+ bt_skip();
+
+ } while (--amount != 0);
+}
+#endif
+
+
+#ifdef HAVE_MF_BT4
+extern uint32_t
+lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches)
+{
+ header_find(true, 4);
+
+ hash_4_calc();
+
+ uint32_t delta2 = pos - mf->hash[hash_2_value];
+ const uint32_t delta3
+ = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value];
+ const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value];
+
+ mf->hash[hash_2_value] = pos;
+ mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos;
+ mf->hash[FIX_4_HASH_SIZE + hash_value] = pos;
+
+ uint32_t len_best = 1;
+
+ if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) {
+ len_best = 2;
+ matches[0].len = 2;
+ matches[0].dist = delta2 - 1;
+ matches_count = 1;
+ }
+
+ if (delta2 != delta3 && delta3 < mf->cyclic_buffer_size
+ && *(cur - delta3) == *cur) {
+ len_best = 3;
+ matches[matches_count++].dist = delta3 - 1;
+ delta2 = delta3;
+ }
+
+ if (matches_count != 0) {
+ for ( ; len_best != len_limit; ++len_best)
+ if (*(cur + len_best - delta2) != cur[len_best])
+ break;
+
+ matches[matches_count - 1].len = len_best;
+
+ if (len_best == len_limit) {
+ bt_skip();
+ return matches_count;
+ }
+ }
+
+ if (len_best < 3)
+ len_best = 3;
+
+ bt_find(len_best);
+}
+
+
+extern void
+lzma_mf_bt4_skip(lzma_mf *mf, uint32_t amount)
+{
+ do {
+ header_skip(true, 4);
+
+ hash_4_calc();
+
+ const uint32_t cur_match
+ = mf->hash[FIX_4_HASH_SIZE + hash_value];
+
+ mf->hash[hash_2_value] = pos;
+ mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos;
+ mf->hash[FIX_4_HASH_SIZE + hash_value] = pos;
+
+ bt_skip();
+
+ } while (--amount != 0);
+}
+#endif
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file match_c.h
-/// \brief Template for different match finders
-///
-/// This file is included by hc3.c, hc4, bt2.c, bt3.c and bt4.c. Each file
-/// sets slighly different #defines, resulting the different match finders.
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-//////////////
-// Includes //
-//////////////
-
-#include "check.h"
-
-
-///////////////
-// Constants //
-///////////////
-
-#define START_MAX_LEN 1
-
-#ifdef HASH_ARRAY_2
-# define NUM_HASH_DIRECT_BYTES 0
-# define HASH_2_SIZE (1 << 10)
-# ifdef HASH_ARRAY_3
-# define NUM_HASH_BYTES 4
-# define HASH_3_SIZE (1 << 16)
-# define HASH_3_OFFSET HASH_2_SIZE
-# define FIX_HASH_SIZE (HASH_2_SIZE + HASH_3_SIZE)
-# else
-# define NUM_HASH_BYTES 3
-# define FIX_HASH_SIZE HASH_2_SIZE
-# endif
-# define HASH_SIZE 0
-# define MIN_MATCH_CHECK NUM_HASH_BYTES
-#else
-# define NUM_HASH_DIRECT_BYTES 2
-# define NUM_HASH_BYTES 2
-# define HASH_SIZE (1 << (8 * NUM_HASH_BYTES))
-# define MIN_MATCH_CHECK (NUM_HASH_BYTES + 1)
-# define FIX_HASH_SIZE 0
-#endif
-
-
-////////////
-// Macros //
-////////////
-
-#ifdef HASH_ARRAY_2
-# ifdef HASH_ARRAY_3
-# define HASH_CALC() \
- do { \
- const uint32_t temp = lzma_crc32_table[0][ \
- cur[0]] ^ cur[1]; \
- hash_2_value = temp & (HASH_2_SIZE - 1); \
- hash_3_value = (temp ^ ((uint32_t)(cur[2]) << 8)) \
- & (HASH_3_SIZE - 1); \
- hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \
- ^ (lzma_crc32_table[0][cur[3]] << 5)) \
- & lz->hash_mask; \
- } while (0)
-# else
-# define HASH_CALC() \
- do { \
- const uint32_t temp = lzma_crc32_table[0][ \
- cur[0]] ^ cur[1]; \
- hash_2_value = temp & (HASH_2_SIZE - 1); \
- hash_value = (temp ^ ((uint32_t)(cur[2]) << 8)) \
- & lz->hash_mask; \
- } while (0)
-# endif
-#else
-# define HASH_CALC() hash_value = cur[0] ^ ((uint32_t)(cur[1]) << 8)
-#endif
-
-
-// Moves the current read position forward by one byte. In LZMA SDK,
-// CLZInWindow::MovePos() can read more input data if needed, because of
-// the callback style API. In liblzma we must have ensured earlier, that
-// there is enough data available in lz->buffer.
-#define move_pos() \
-do { \
- if (++lz->cyclic_buffer_pos == lz->cyclic_buffer_size) \
- lz->cyclic_buffer_pos = 0; \
- ++lz->read_pos; \
- assert(lz->read_pos <= lz->write_pos); \
- if (lz->read_pos == MAX_VAL_FOR_NORMALIZE) \
- lzma_lz_encoder_normalize(lz); \
-} while (0)
-
-
-#define move_pending() \
-do { \
- ++lz->read_pos; \
- assert(lz->read_pos <= lz->write_pos); \
- ++lz->pending; \
-} while (0)
-
-
-//////////////////////
-// Global constants //
-//////////////////////
-
-LZMA_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER) = HASH_SIZE;
-LZMA_FIX_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER) = FIX_HASH_SIZE;
-
-
-///////////////////
-// API functions //
-///////////////////
-
-LZMA_GET_MATCHES(LZMA_MATCH_FINDER_NAME_LOWER)
-{
- uint32_t len_limit;
- if (lz->read_pos + lz->match_max_len <= lz->write_pos) {
- len_limit = lz->match_max_len;
- } else {
- len_limit = lz->write_pos - lz->read_pos;
- if (len_limit < MIN_MATCH_CHECK || lz->sequence == SEQ_FLUSH) {
- distances[0] = 0;
- move_pending();
- return;
- }
- }
-
- assert(lz->pending == 0);
-
- int32_t offset = 1;
- const uint32_t match_min_pos
- = lz->read_pos + lz->offset > lz->cyclic_buffer_size
- ? lz->read_pos + lz->offset - lz->cyclic_buffer_size
- : 0;
- const uint8_t *cur = lz->buffer + lz->read_pos;
- uint32_t max_len = START_MAX_LEN; // to avoid items for len < hash_size
-
-#ifdef HASH_ARRAY_2
- uint32_t hash_2_value;
-# ifdef HASH_ARRAY_3
- uint32_t hash_3_value;
-# endif
-#endif
- uint32_t hash_value;
- HASH_CALC();
-
- uint32_t cur_match = lz->hash[FIX_HASH_SIZE + hash_value];
-#ifdef HASH_ARRAY_2
- uint32_t cur_match2 = lz->hash[hash_2_value];
-# ifdef HASH_ARRAY_3
- uint32_t cur_match3 = lz->hash[HASH_3_OFFSET + hash_3_value];
-# endif
- lz->hash[hash_2_value] = lz->read_pos + lz->offset;
-
- if (cur_match2 > match_min_pos) {
- if (lz->buffer[cur_match2 - lz->offset] == cur[0]) {
- max_len = 2;
- distances[offset++] = 2;
- distances[offset++] = lz->read_pos + lz->offset
- - cur_match2 - 1;
- }
- }
-
-# ifdef HASH_ARRAY_3
- lz->hash[HASH_3_OFFSET + hash_3_value] = lz->read_pos + lz->offset;
- if (cur_match3 > match_min_pos) {
- if (lz->buffer[cur_match3 - lz->offset] == cur[0]) {
- if (cur_match3 == cur_match2)
- offset -= 2;
-
- max_len = 3;
- distances[offset++] = 3;
- distances[offset++] = lz->read_pos + lz->offset
- - cur_match3 - 1;
- cur_match2 = cur_match3;
- }
- }
-# endif
-
- if (offset != 1 && cur_match2 == cur_match) {
- offset -= 2;
- max_len = START_MAX_LEN;
- }
-#endif
-
- lz->hash[FIX_HASH_SIZE + hash_value] = lz->read_pos + lz->offset;
-
-#ifdef IS_HASH_CHAIN
- lz->son[lz->cyclic_buffer_pos] = cur_match;
-#else
- uint32_t *ptr0 = lz->son + (lz->cyclic_buffer_pos << 1) + 1;
- uint32_t *ptr1 = lz->son + (lz->cyclic_buffer_pos << 1);
-
- uint32_t len0 = NUM_HASH_DIRECT_BYTES;
- uint32_t len1 = NUM_HASH_DIRECT_BYTES;
-#endif
-
-#if NUM_HASH_DIRECT_BYTES != 0
- if (cur_match > match_min_pos) {
- if (lz->buffer[cur_match + NUM_HASH_DIRECT_BYTES - lz->offset]
- != cur[NUM_HASH_DIRECT_BYTES]) {
- max_len = NUM_HASH_DIRECT_BYTES;
- distances[offset++] = NUM_HASH_DIRECT_BYTES;
- distances[offset++] = lz->read_pos + lz->offset
- - cur_match - 1;
- }
- }
-#endif
-
- uint32_t count = lz->cut_value;
-
- while (true) {
- if (cur_match <= match_min_pos || count-- == 0) {
-#ifndef IS_HASH_CHAIN
- *ptr0 = EMPTY_HASH_VALUE;
- *ptr1 = EMPTY_HASH_VALUE;
-#endif
- break;
- }
-
- const uint32_t delta = lz->read_pos + lz->offset - cur_match;
- const uint32_t cyclic_pos = delta <= lz->cyclic_buffer_pos
- ? lz->cyclic_buffer_pos - delta
- : lz->cyclic_buffer_pos - delta
- + lz->cyclic_buffer_size;
- uint32_t *pair = lz->son +
-#ifdef IS_HASH_CHAIN
- cyclic_pos;
-#else
- (cyclic_pos << 1);
-#endif
-
- const uint8_t *pb = lz->buffer + cur_match - lz->offset;
- uint32_t len =
-#ifdef IS_HASH_CHAIN
- NUM_HASH_DIRECT_BYTES;
- if (pb[max_len] == cur[max_len])
-#else
- MIN(len0, len1);
-#endif
-
- if (pb[len] == cur[len]) {
- while (++len != len_limit)
- if (pb[len] != cur[len])
- break;
-
- if (max_len < len) {
- max_len = len;
- distances[offset++] = len;
- distances[offset++] = delta - 1;
- if (len == len_limit) {
-#ifndef IS_HASH_CHAIN
- *ptr1 = pair[0];
- *ptr0 = pair[1];
-#endif
- break;
- }
- }
- }
-
-#ifdef IS_HASH_CHAIN
- cur_match = *pair;
-#else
- if (pb[len] < cur[len]) {
- *ptr1 = cur_match;
- ptr1 = pair + 1;
- cur_match = *ptr1;
- len1 = len;
- } else {
- *ptr0 = cur_match;
- ptr0 = pair;
- cur_match = *ptr0;
- len0 = len;
- }
-#endif
- }
-
- distances[0] = offset - 1;
-
- move_pos();
-
- return;
-}
-
-
-LZMA_SKIP(LZMA_MATCH_FINDER_NAME_LOWER)
-{
- do {
-#ifdef IS_HASH_CHAIN
- if (lz->write_pos - lz->read_pos < NUM_HASH_BYTES) {
- move_pending();
- continue;
- }
-#else
- uint32_t len_limit;
- if (lz->read_pos + lz->match_max_len <= lz->write_pos) {
- len_limit = lz->match_max_len;
- } else {
- len_limit = lz->write_pos - lz->read_pos;
- if (len_limit < MIN_MATCH_CHECK
- || lz->sequence == SEQ_FLUSH) {
- move_pending();
- continue;
- }
- }
- const uint32_t match_min_pos
- = lz->read_pos + lz->offset > lz->cyclic_buffer_size
- ? lz->read_pos + lz->offset - lz->cyclic_buffer_size
- : 0;
-#endif
-
- assert(lz->pending == 0);
-
- const uint8_t *cur = lz->buffer + lz->read_pos;
-
-#ifdef HASH_ARRAY_2
- uint32_t hash_2_value;
-# ifdef HASH_ARRAY_3
- uint32_t hash_3_value;
- uint32_t hash_value;
- HASH_CALC();
- lz->hash[HASH_3_OFFSET + hash_3_value]
- = lz->read_pos + lz->offset;
-# else
- uint32_t hash_value;
- HASH_CALC();
-# endif
- lz->hash[hash_2_value] = lz->read_pos + lz->offset;
-#else
- uint32_t hash_value;
- HASH_CALC();
-#endif
-
- uint32_t cur_match = lz->hash[FIX_HASH_SIZE + hash_value];
- lz->hash[FIX_HASH_SIZE + hash_value]
- = lz->read_pos + lz->offset;
-
-#ifdef IS_HASH_CHAIN
- lz->son[lz->cyclic_buffer_pos] = cur_match;
-#else
- uint32_t *ptr0 = lz->son + (lz->cyclic_buffer_pos << 1) + 1;
- uint32_t *ptr1 = lz->son + (lz->cyclic_buffer_pos << 1);
-
- uint32_t len0 = NUM_HASH_DIRECT_BYTES;
- uint32_t len1 = NUM_HASH_DIRECT_BYTES;
- uint32_t count = lz->cut_value;
-
- while (true) {
- if (cur_match <= match_min_pos || count-- == 0) {
- *ptr0 = EMPTY_HASH_VALUE;
- *ptr1 = EMPTY_HASH_VALUE;
- break;
- }
-
- const uint32_t delta = lz->read_pos
- + lz->offset - cur_match;
- const uint32_t cyclic_pos
- = delta <= lz->cyclic_buffer_pos
- ? lz->cyclic_buffer_pos - delta
- : lz->cyclic_buffer_pos - delta
- + lz->cyclic_buffer_size;
- uint32_t *pair = lz->son + (cyclic_pos << 1);
-
- const uint8_t *pb = lz->buffer + cur_match
- - lz->offset;
- uint32_t len = MIN(len0, len1);
-
- if (pb[len] == cur[len]) {
- while (++len != len_limit)
- if (pb[len] != cur[len])
- break;
-
- if (len == len_limit) {
- *ptr1 = pair[0];
- *ptr0 = pair[1];
- break;
- }
- }
-
- if (pb[len] < cur[len]) {
- *ptr1 = cur_match;
- ptr1 = pair + 1;
- cur_match = *ptr1;
- len1 = len;
- } else {
- *ptr0 = cur_match;
- ptr0 = pair;
- cur_match = *ptr0;
- len0 = len;
- }
- }
-#endif
-
- move_pos();
-
- } while (--num != 0);
-
- return;
-}
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file match_h.h
-/// \brief Header template for different match finders
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "lz_encoder_private.h"
-
-
-//////////////////////
-// Global constants //
-//////////////////////
-
-#undef LZMA_HASH_SIZE
-#undef LZMA_FIX_HASH_SIZE
-#undef LZMA_HASH_SIZE_C
-#undef LZMA_FIX_HASH_SIZE_C
-
-#define LZMA_HASH_SIZE(mf_name) LZMA_HASH_SIZE_C(mf_name)
-#define LZMA_FIX_HASH_SIZE(mf_name) LZMA_FIX_HASH_SIZE_C(mf_name)
-
-#define LZMA_HASH_SIZE_C(mf_name) \
- const uint32_t LZMA_ ## mf_name ## _HASH_SIZE
-
-#define LZMA_FIX_HASH_SIZE_C(mf_name) \
- const uint32_t LZMA_ ## mf_name ## _FIX_HASH_SIZE
-
-extern LZMA_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER);
-extern LZMA_FIX_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER);
-
-
-///////////////
-// Functions //
-///////////////
-
-#undef LZMA_GET_MATCHES
-#undef LZMA_SKIP
-#undef LZMA_GET_MATCHES_C
-#undef LZMA_SKIP_C
-
-#define LZMA_GET_MATCHES(mf_name) LZMA_GET_MATCHES_C(mf_name)
-#define LZMA_SKIP(mf_name) LZMA_SKIP_C(mf_name)
-
-#define LZMA_GET_MATCHES_C(mf_name) \
- extern void lzma_ ## mf_name ## _get_matches( \
- lzma_lz_encoder *restrict lz, \
- uint32_t *restrict distances)
-
-#define LZMA_SKIP_C(mf_name) \
- extern void lzma_ ## mf_name ## _skip( \
- lzma_lz_encoder *lz, uint32_t num)
-
-LZMA_GET_MATCHES(LZMA_MATCH_FINDER_NAME_LOWER);
-
-LZMA_SKIP(LZMA_MATCH_FINDER_NAME_LOWER);
EXTRA_DIST = fastpos_tablegen.c
-noinst_LTLIBRARIES = liblzma4.la
-liblzma4_la_CPPFLAGS = \
+## Using liblzma2 since liblzma is already used for the final library.
+noinst_LTLIBRARIES = liblzma2.la
+liblzma2_la_CPPFLAGS = \
-I@top_srcdir@/src/liblzma/api \
-I@top_srcdir@/src/liblzma/common \
-I@top_srcdir@/src/liblzma/lz \
-I@top_srcdir@/src/liblzma/rangecoder
-liblzma4_la_SOURCES = \
- lzma_common.h \
- lzma_literal.c \
- lzma_literal.h
+liblzma2_la_SOURCES = lzma_common.h
-if COND_MAIN_ENCODER
-liblzma4_la_SOURCES += \
+if COND_ENCODER_LZMA
+liblzma2_la_SOURCES += \
fastpos.h \
lzma_encoder.h \
lzma_encoder.c \
lzma_encoder_presets.c \
lzma_encoder_private.h \
- lzma_encoder_init.c \
lzma_encoder_features.c \
- lzma_encoder_getoptimum.c \
- lzma_encoder_getoptimumfast.c
+ lzma_encoder_optimum_fast.c \
+ lzma_encoder_optimum_normal.c
if !COND_SMALL
-liblzma4_la_SOURCES += fastpos_table.c
+liblzma2_la_SOURCES += fastpos_table.c
endif
endif
-if COND_MAIN_DECODER
-liblzma4_la_SOURCES += \
+if COND_DECODER_LZMA
+liblzma2_la_SOURCES += \
lzma_decoder.c \
lzma_decoder.h
endif
+
+if COND_ENCODER_LZMA2
+liblzma2_la_SOURCES += \
+ lzma2_encoder.c \
+ lzma2_encoder.h
+endif
+
+if COND_DECODER_LZMA2
+liblzma2_la_SOURCES += \
+ lzma2_decoder.c \
+ lzma2_decoder.h
+endif
// I'm making the table version the default, because that has good speed
// on all systems I have tried. The size optimized version is sometimes
// slightly faster, but sometimes it is a lot slower.
-//
-// Finally, this code isn't a major bottle neck in LZMA encoding anyway.
#ifdef HAVE_SMALL
# include "bsr.h"
static inline uint32_t
get_pos_slot_2(uint32_t pos)
{
- // FIXME: This assert() cannot be enabled at the moment, because
- // lzma_getoptimum.c calls this function so that this assertion
- // fails; however, it ignores the result of this function when
- // this assert() would have failed.
- // assert(pos >= FULL_DISTANCES);
+ assert(pos >= FULL_DISTANCES);
if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 0))
return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 0);
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file lzma2_decoder.c
+/// \brief LZMA2 decoder
+//
+// Copyright (C) 1999-2008 Igor Pavlov
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "lzma2_decoder.h"
+#include "lz_decoder.h"
+#include "lzma_decoder.h"
+
+
+struct lzma_coder_s {
+ enum sequence {
+ SEQ_CONTROL,
+ SEQ_UNCOMPRESSED_1,
+ SEQ_UNCOMPRESSED_2,
+ SEQ_COMPRESSED_0,
+ SEQ_COMPRESSED_1,
+ SEQ_PROPERTIES,
+ SEQ_LZMA,
+ SEQ_COPY,
+ } sequence;
+
+ /// Sequence after the size fields have been decoded.
+ enum sequence next_sequence;
+
+ /// LZMA decoder
+ lzma_lz_decoder lzma;
+
+ /// Uncompressed size of LZMA chunk
+ size_t uncompressed_size;
+
+ /// Compressed size of the chunk (naturally equals to uncompressed
+ /// size of uncompressed chunk)
+ size_t compressed_size;
+
+ /// True if properties are needed. This is false before the
+ /// first LZMA chunk.
+ bool need_properties;
+
+ /// True if dictionary reset is needed. This is false before the
+ /// first chunk (LZMA or uncompressed).
+ bool need_dictionary_reset;
+
+ lzma_options_lzma options;
+};
+
+
+static lzma_ret
+lzma2_decode(lzma_coder *restrict coder, lzma_dict *restrict dict,
+ const uint8_t *restrict in, size_t *restrict in_pos,
+ size_t in_size)
+{
+ // With SEQ_LZMA it is possible that no new input is needed to do
+ // some progress. The rest of the sequences assume that there is
+ // at least one byte of input.
+ while (*in_pos < in_size || coder->sequence == SEQ_LZMA)
+ switch (coder->sequence) {
+ case SEQ_CONTROL:
+ if (in[*in_pos] & 0x80) {
+ // Get the highest five bits of uncompressed size.
+ coder->uncompressed_size
+ = (uint32_t)(in[*in_pos] & 0x1F) << 16;
+ coder->sequence = SEQ_UNCOMPRESSED_1;
+
+ // See if we need to reset dictionary or state.
+ switch ((in[(*in_pos)++] >> 5) & 3) {
+ case 3:
+ dict_reset(dict);
+ coder->need_dictionary_reset = false;
+
+ // Fall through
+
+ case 2:
+ if (coder->need_dictionary_reset)
+ return LZMA_DATA_ERROR;
+
+ coder->need_properties = false;
+ coder->next_sequence = SEQ_PROPERTIES;
+ break;
+
+ case 1:
+ if (coder->need_properties)
+ return LZMA_DATA_ERROR;
+
+ coder->lzma.reset(coder->lzma.coder,
+ &coder->options);
+
+ coder->next_sequence = SEQ_LZMA;
+ break;
+
+ case 0:
+ if (coder->need_properties)
+ return LZMA_DATA_ERROR;
+
+ coder->next_sequence = SEQ_LZMA;
+ break;
+ }
+
+ } else {
+ switch (in[(*in_pos)++]) {
+ case 0:
+ // End of payload marker
+ return LZMA_STREAM_END;
+
+ case 1:
+ // Dictionary reset
+ dict_reset(dict);
+ coder->need_dictionary_reset = false;
+
+ // Fall through
+
+ case 2:
+ if (coder->need_dictionary_reset)
+ return LZMA_DATA_ERROR;
+
+ // Uncompressed chunk; we need to read total
+ // size first.
+ coder->sequence = SEQ_COMPRESSED_0;
+ coder->next_sequence = SEQ_COPY;
+ break;
+
+ default:
+ return LZMA_DATA_ERROR;
+ }
+ }
+
+ break;
+
+ case SEQ_UNCOMPRESSED_1:
+ coder->uncompressed_size += (uint32_t)(in[(*in_pos)++]) << 8;
+ coder->sequence = SEQ_UNCOMPRESSED_2;
+ break;
+
+ case SEQ_UNCOMPRESSED_2:
+ coder->uncompressed_size += in[(*in_pos)++] + 1;
+ coder->sequence = SEQ_COMPRESSED_0;
+ coder->lzma.set_uncompressed(coder->lzma.coder,
+ coder->uncompressed_size);
+ break;
+
+ case SEQ_COMPRESSED_0:
+ coder->compressed_size = (uint32_t)(in[(*in_pos)++]) << 8;
+ coder->sequence = SEQ_COMPRESSED_1;
+ break;
+
+ case SEQ_COMPRESSED_1:
+ coder->compressed_size += in[(*in_pos)++] + 1;
+ coder->sequence = coder->next_sequence;
+ break;
+
+ case SEQ_PROPERTIES:
+ if (lzma_lzma_lclppb_decode(&coder->options, in[(*in_pos)++]))
+ return LZMA_DATA_ERROR;
+
+ coder->lzma.reset(coder->lzma.coder, &coder->options);
+
+ coder->sequence = SEQ_LZMA;
+ break;
+
+ case SEQ_LZMA: {
+ // Store the start offset so that we can update
+ // coder->compressed_size later.
+ const size_t in_start = *in_pos;
+
+ // Decode from in[] to *dict.
+ const lzma_ret ret = coder->lzma.code(coder->lzma.coder,
+ dict, in, in_pos, in_size);
+
+ // Validate and update coder->compressed_size.
+ const size_t in_used = *in_pos - in_start;
+ if (in_used > coder->compressed_size)
+ return LZMA_DATA_ERROR;
+
+ coder->compressed_size -= in_used;
+
+ // Return if we didn't finish the chunk, or an error occurred.
+ if (ret != LZMA_STREAM_END)
+ return ret;
+
+ // The LZMA decoder must have consumed the whole chunk now.
+ // We don't need to worry about uncompressed size since it
+ // is checked by the LZMA decoder.
+ if (coder->compressed_size != 0)
+ return LZMA_DATA_ERROR;
+
+ coder->sequence = SEQ_CONTROL;
+ break;
+ }
+
+ case SEQ_COPY: {
+ // Copy from input to the dictionary as is.
+ // FIXME Can copy too much?
+ dict_write(dict, in, in_pos, in_size, &coder->compressed_size);
+ if (coder->compressed_size != 0)
+ return LZMA_OK;
+
+ coder->sequence = SEQ_CONTROL;
+ break;
+ }
+
+ default:
+ assert(0);
+ return LZMA_PROG_ERROR;
+ }
+
+ return LZMA_OK;
+}
+
+
+static void
+lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
+{
+ assert(coder->lzma.end == NULL);
+ lzma_free(coder->lzma.coder, allocator);
+
+ lzma_free(coder, allocator);
+
+ return;
+}
+
+
+static lzma_ret
+lzma2_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator,
+ const void *options, size_t *dict_size)
+{
+ if (lz->coder == NULL) {
+ lz->coder = lzma_alloc(sizeof(lzma_coder), allocator);
+ if (lz->coder == NULL)
+ return LZMA_MEM_ERROR;
+
+ lz->code = &lzma2_decode;
+ lz->end = &lzma2_decoder_end;
+
+ lz->coder->lzma = LZMA_LZ_DECODER_INIT;
+ }
+
+ lz->coder->sequence = SEQ_CONTROL;
+ lz->coder->need_properties = true;
+ lz->coder->need_dictionary_reset = true;
+
+ return lzma_lzma_decoder_create(&lz->coder->lzma,
+ allocator, options, dict_size);
+}
+
+
+extern lzma_ret
+lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter_info *filters)
+{
+ // LZMA2 can only be the last filter in the chain. This is enforced
+ // by the raw_decoder initialization.
+ assert(filters[1].init == NULL);
+
+ return lzma_lz_decoder_init(next, allocator, filters,
+ &lzma2_decoder_init);
+}
+
+
+extern uint64_t
+lzma_lzma2_decoder_memusage(const void *options)
+{
+ const uint64_t lzma_memusage = lzma_lzma_decoder_memusage(options);
+ if (lzma_memusage == UINT64_MAX)
+ return UINT64_MAX;
+
+ return sizeof(lzma_coder) + lzma_memusage;
+}
+
+
+extern lzma_ret
+lzma_lzma2_props_decode(void **options, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size)
+{
+ if (props_size != 1)
+ return LZMA_HEADER_ERROR;
+
+ // Check that reserved bits are unset.
+ if (props[0] & 0xC0)
+ return LZMA_HEADER_ERROR;
+
+ // Decode the dictionary size.
+ if (props[0] > 40)
+ return LZMA_HEADER_ERROR;
+
+ lzma_options_lzma *opt = lzma_alloc(
+ sizeof(lzma_options_lzma), allocator);
+ if (opt == NULL)
+ return LZMA_MEM_ERROR;
+
+ if (props[0] == 40) {
+ opt->dictionary_size = UINT32_MAX;
+ } else {
+ opt->dictionary_size = 2 | (props[0] & 1);
+ opt->dictionary_size <<= props[0] / 2 + 11;
+ }
+
+ opt->preset_dictionary = NULL;
+ opt->preset_dictionary_size = 0;
+
+ *options = opt;
+
+ return LZMA_OK;
+}
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file lzma2_decoder.h
+/// \brief LZMA2 decoder
+//
+// Copyright (C) 1999-2008 Igor Pavlov
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef LZMA_LZMA2_DECODER_H
+#define LZMA_LZMA2_DECODER_H
+
+#include "common.h"
+
+extern lzma_ret lzma_lzma2_decoder_init(lzma_next_coder *next,
+ lzma_allocator *allocator, const lzma_filter_info *filters);
+
+extern uint64_t lzma_lzma2_decoder_memusage(const void *options);
+
+extern lzma_ret lzma_lzma2_props_decode(
+ void **options, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size);
+
+#endif
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file lzma2_encoder.c
+/// \brief LZMA2 encoder
+//
+// Copyright (C) 1999-2008 Igor Pavlov
+// Copyright (C) 2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "lz_encoder.h"
+#include "lzma_encoder.h"
+#include "fastpos.h"
+#include "lzma2_encoder.h"
+
+
+/// Maximum number of bytes of actual data per chunk (no headers)
+#define LZMA2_CHUNK_MAX (UINT32_C(1) << 16)
+
+/// Maximum uncompressed size of LZMA chunk (no headers)
+#define LZMA2_UNCOMPRESSED_MAX (UINT32_C(1) << 21)
+
+/// Maximum size of LZMA2 headers
+#define LZMA2_HEADER_MAX 6
+
+/// Size of a header for uncompressed chunk
+#define LZMA2_HEADER_UNCOMPRESSED 3
+
+
+struct lzma_coder_s {
+ enum {
+ SEQ_INIT,
+ SEQ_LZMA_ENCODE,
+ SEQ_LZMA_COPY,
+ SEQ_UNCOMPRESSED_HEADER,
+ SEQ_UNCOMPRESSED_COPY,
+ } sequence;
+
+ /// LZMA encoder
+ lzma_coder *lzma;
+
+ /// If this is not NULL, we will check new options from this
+ /// structure when starting a new chunk.
+ const lzma_options_lzma *opt_new;
+
+ /// LZMA options currently in use.
+ lzma_options_lzma opt_cur;
+
+ bool need_properties;
+ bool need_state_reset;
+ bool need_dictionary_reset;
+
+ /// Uncompressed size of a chunk
+ size_t uncompressed_size;
+
+ /// Compressed size of a chunk (excluding headers); this is also used
+ /// to indicate the end of buf[] in SEQ_LZMA_COPY.
+ size_t compressed_size;
+
+ /// Read position in buf[]
+ size_t buf_pos;
+
+ /// Buffer to hold the chunk header and LZMA compressed data
+ uint8_t buf[LZMA2_HEADER_MAX + LZMA2_CHUNK_MAX];
+};
+
+
+static void
+lzma2_header_lzma(lzma_coder *coder)
+{
+ assert(coder->uncompressed_size > 0);
+ assert(coder->uncompressed_size <= LZMA2_UNCOMPRESSED_MAX);
+ assert(coder->compressed_size > 0);
+ assert(coder->compressed_size <= LZMA2_CHUNK_MAX);
+
+ size_t pos;
+
+ if (coder->need_properties) {
+ pos = 0;
+
+ if (coder->need_dictionary_reset)
+ coder->buf[pos] = 0x80 + (3 << 5);
+ else
+ coder->buf[pos] = 0x80 + (2 << 5);
+ } else {
+ pos = 1;
+
+ if (coder->need_state_reset)
+ coder->buf[pos] = 0x80 + (1 << 5);
+ else
+ coder->buf[pos] = 0x80;
+ }
+
+ // Set the start position for copying.
+ coder->buf_pos = pos;
+
+ // Uncompressed size
+ size_t size = coder->uncompressed_size - 1;
+ coder->buf[pos++] += size >> 16;
+ coder->buf[pos++] = (size >> 8) & 0xFF;
+ coder->buf[pos++] = size & 0xFF;
+
+ // Compressed size
+ size = coder->compressed_size - 1;
+ coder->buf[pos++] = size >> 8;
+ coder->buf[pos++] = size & 0xFF;
+
+ // Properties, if needed
+ if (coder->need_properties)
+ lzma_lzma_lclppb_encode(&coder->opt_cur, coder->buf + pos);
+
+ coder->need_properties = false;
+ coder->need_state_reset = false;
+ coder->need_dictionary_reset = false;
+
+ // The copying code uses coder->compressed_size to indicate the end
+ // of coder->buf[], so we need add the maximum size of the header here.
+ coder->compressed_size += LZMA2_HEADER_MAX;
+
+ return;
+}
+
+
+static void
+lzma2_header_uncompressed(lzma_coder *coder)
+{
+ assert(coder->uncompressed_size > 0);
+ assert(coder->uncompressed_size <= LZMA2_CHUNK_MAX);
+
+ // If this is the first chunk, we need to include dictionary
+ // reset indicator.
+ if (coder->need_dictionary_reset)
+ coder->buf[0] = 1;
+ else
+ coder->buf[0] = 2;
+
+ coder->need_dictionary_reset = false;
+
+ // "Compressed" size
+ coder->buf[1] = (coder->uncompressed_size - 1) >> 8;
+ coder->buf[2] = (coder->uncompressed_size - 1) & 0xFF;
+
+ // Set the start position for copying.
+ coder->buf_pos = 0;
+ return;
+}
+
+
+static lzma_ret
+lzma2_encode(lzma_coder *restrict coder, lzma_mf *restrict mf,
+ uint8_t *restrict out, size_t *restrict out_pos,
+ size_t out_size)
+{
+ while (*out_pos < out_size)
+ switch (coder->sequence) {
+ case SEQ_INIT:
+ // If there's no input left and we are flushing or finishing,
+ // don't start a new chunk.
+ if (mf_unencoded(mf) == 0) {
+ // Write end of payload marker if finishing.
+ if (mf->action == LZMA_FINISH)
+ out[(*out_pos)++] = 0;
+
+ return mf->action == LZMA_RUN
+ ? LZMA_OK : LZMA_STREAM_END;
+ }
+
+ // Look if there are new options. At least for now,
+ // only lc/lp/pb can be changed.
+ if (coder->opt_new != NULL
+ && (coder->opt_cur.literal_context_bits
+ != coder->opt_new->literal_context_bits
+ || coder->opt_cur.literal_pos_bits
+ != coder->opt_new->literal_pos_bits
+ || coder->opt_cur.pos_bits
+ != coder->opt_new->pos_bits)) {
+ // Options have been changed, copy them to opt_cur.
+ coder->opt_cur.literal_context_bits
+ = coder->opt_new->literal_context_bits;
+ coder->opt_cur.literal_pos_bits
+ = coder->opt_new->literal_pos_bits;
+ coder->opt_cur.pos_bits
+ = coder->opt_new->pos_bits;
+
+ // We need to write the new options and reset
+ // the encoder state.
+ coder->need_properties = true;
+ coder->need_state_reset = true;
+ }
+
+ if (coder->need_state_reset)
+ lzma_lzma_encoder_reset(coder->lzma, &coder->opt_cur);
+
+ coder->uncompressed_size = 0;
+ coder->compressed_size = 0;
+ coder->sequence = SEQ_LZMA_ENCODE;
+
+ // Fall through
+
+ case SEQ_LZMA_ENCODE: {
+ // Calculate how much more uncompressed data this chunk
+ // could accept.
+ const uint32_t left = LZMA2_UNCOMPRESSED_MAX
+ - coder->uncompressed_size;
+ uint32_t limit;
+
+ if (left < mf->match_len_max) {
+ // Must flush immediatelly since the next LZMA symbol
+ // could make the uncompressed size of the chunk too
+ // big.
+ limit = 0;
+ } else {
+ // Calculate maximum read_limit that is OK from point
+ // of view of LZMA2 chunk size.
+ limit = mf->read_pos - mf->read_ahead
+ + left - mf->match_len_max;
+ }
+
+ // Save the start position so that we can update
+ // coder->uncompressed_size.
+ const uint32_t read_start = mf->read_pos - mf->read_ahead;
+
+ // Call the LZMA encoder until the chunk is finished.
+ const lzma_ret ret = lzma_lzma_encode(coder->lzma, mf,
+ coder->buf + LZMA2_HEADER_MAX,
+ &coder->compressed_size,
+ LZMA2_CHUNK_MAX, limit);
+
+ coder->uncompressed_size += mf->read_pos - mf->read_ahead
+ - read_start;
+
+ assert(coder->compressed_size <= LZMA2_CHUNK_MAX);
+ assert(coder->uncompressed_size <= LZMA2_UNCOMPRESSED_MAX);
+
+ if (ret != LZMA_STREAM_END)
+ return LZMA_OK;
+
+ // See if the chunk compressed. If it didn't, we encode it
+ // as uncompressed chunk. This saves a few bytes of space
+ // and makes decoding faster.
+ if (coder->compressed_size >= coder->uncompressed_size) {
+ coder->uncompressed_size += mf->read_ahead;
+ assert(coder->uncompressed_size
+ <= LZMA2_UNCOMPRESSED_MAX);
+ mf->read_ahead = 0;
+ lzma2_header_uncompressed(coder);
+ coder->need_state_reset = true;
+ coder->sequence = SEQ_UNCOMPRESSED_HEADER;
+ break;
+ }
+
+ // The chunk did compress at least by one byte, so we store
+ // the chunk as LZMA.
+ lzma2_header_lzma(coder);
+
+ coder->sequence = SEQ_LZMA_COPY;
+ }
+
+ // Fall through
+
+ case SEQ_LZMA_COPY:
+ // Copy the compressed chunk along its headers to the
+ // output buffer.
+ lzma_bufcpy(coder->buf, &coder->buf_pos,
+ coder->compressed_size,
+ out, out_pos, out_size);
+ if (coder->buf_pos != coder->compressed_size)
+ return LZMA_OK;
+
+ coder->sequence = SEQ_INIT;
+ break;
+
+ case SEQ_UNCOMPRESSED_HEADER:
+ // Copy the three-byte header to indicate uncompressed chunk.
+ lzma_bufcpy(coder->buf, &coder->buf_pos,
+ LZMA2_HEADER_UNCOMPRESSED,
+ out, out_pos, out_size);
+ if (coder->buf_pos != LZMA2_HEADER_UNCOMPRESSED)
+ return LZMA_OK;
+
+ coder->sequence = SEQ_UNCOMPRESSED_COPY;
+
+ // Fall through
+
+ case SEQ_UNCOMPRESSED_COPY:
+ // Copy the uncompressed data as is from the dictionary
+ // to the output buffer.
+ mf_read(mf, out, out_pos, out_size, &coder->uncompressed_size);
+ if (coder->uncompressed_size != 0)
+ return LZMA_OK;
+
+ coder->sequence = SEQ_INIT;
+ break;
+ }
+
+ return LZMA_OK;
+}
+
+
+static void
+lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
+{
+ lzma_free(coder->lzma, allocator);
+ lzma_free(coder, allocator);
+ return;
+}
+
+
+static lzma_ret
+lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator,
+ const void *options, lzma_lz_options *lz_options)
+{
+ if (options == NULL)
+ return LZMA_PROG_ERROR;
+
+ if (lz->coder == NULL) {
+ lz->coder = lzma_alloc(sizeof(lzma_coder), allocator);
+ if (lz->coder == NULL)
+ return LZMA_MEM_ERROR;
+
+ lz->code = &lzma2_encode;
+ lz->end = &lzma2_encoder_end;
+
+ lz->coder->lzma = NULL;
+ }
+
+ lz->coder->sequence = SEQ_INIT;
+ lz->coder->need_properties = true;
+ lz->coder->need_state_reset = false;
+ lz->coder->need_dictionary_reset = true;
+
+ lz->coder->opt_cur = *(const lzma_options_lzma *)(options);
+ lz->coder->opt_new = lz->coder->opt_cur.persistent
+ ? options : NULL;
+
+ // Initialize LZMA encoder
+ return_if_error(lzma_lzma_encoder_create(&lz->coder->lzma, allocator,
+ &lz->coder->opt_cur, lz_options));
+
+ // Make sure that we will always have enough history available in
+ // case we need to use uncompressed chunks. They are used when the
+ // compressed size of a chunk is not smaller than the uncompressed
+ // size, so we need to have at least LZMA2_COMPRESSED_MAX bytes
+ // history available.
+ if (lz_options->before_size + lz_options->dictionary_size
+ < LZMA2_CHUNK_MAX)
+ lz_options->before_size = LZMA2_CHUNK_MAX
+ - lz_options->dictionary_size;
+
+ return LZMA_OK;
+}
+
+
+extern lzma_ret
+lzma_lzma2_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter_info *filters)
+{
+ return lzma_lz_encoder_init(
+ next, allocator, filters, &lzma2_encoder_init);
+}
+
+
+extern uint64_t
+lzma_lzma2_encoder_memusage(const void *options)
+{
+ const uint64_t lzma_memusage = lzma_lzma_encoder_memusage(options);
+ if (lzma_memusage == UINT64_MAX)
+ return UINT64_MAX;
+
+ return sizeof(lzma_coder) + lzma_memusage;
+}
+
+
+extern lzma_ret
+lzma_lzma2_props_encode(const void *options, uint8_t *out)
+{
+ const lzma_options_lzma *const opt = options;
+ uint32_t d = MAX(opt->dictionary_size, LZMA_DICTIONARY_SIZE_MIN);
+
+ // Round up to to the next 2^n - 1 or 2^n + 2^(n - 1) - 1 depending
+ // on which one is the next:
+ --d;
+ d |= d >> 2;
+ d |= d >> 3;
+ d |= d >> 4;
+ d |= d >> 8;
+ d |= d >> 16;
+
+ // Get the highest two bits using the proper encoding:
+ if (d == UINT32_MAX)
+ out[0] = 40;
+ else
+ out[0] = get_pos_slot(d + 1) - 24;
+
+ return LZMA_OK;
+}
///////////////////////////////////////////////////////////////////////////////
//
-/// \file raw_common.h
-/// \brief Stuff shared between raw encoder and raw decoder
+/// \file lzma2_encoder.h
+/// \brief LZMA2 encoder
//
-// Copyright (C) 2007 Lasse Collin
+// Copyright (C) 1999-2008 Igor Pavlov
+// Copyright (C) 2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
//
///////////////////////////////////////////////////////////////////////////////
-#ifndef LZMA_RAW_COMMON_H
-#define LZMA_RAW_COMMON_H
+#ifndef LZMA_LZMA2_ENCODER_H
+#define LZMA_LZMA2_ENCODER_H
#include "common.h"
-extern lzma_ret lzma_raw_coder_init(lzma_next_coder *next,
- lzma_allocator *allocator, const lzma_options_filter *options,
- lzma_init_function (*get_function)(lzma_vli id),
- bool is_encoder);
+extern lzma_ret lzma_lzma2_encoder_init(
+ lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter_info *filters);
+
+extern uint64_t lzma_lzma2_encoder_memusage(const void *options);
+
+extern lzma_ret lzma_lzma2_props_encode(const void *options, uint8_t *out);
#endif
#define LZMA_LZMA_COMMON_H
#include "common.h"
-#include "lzma_literal.h"
#include "range_common.h"
-///////////////
-// Constants //
-///////////////
-
-#define REP_DISTANCES 4
-
-#define POS_SLOT_BITS 6
-#define DICT_LOG_SIZE_MAX 30
-#define DIST_TABLE_SIZE_MAX (DICT_LOG_SIZE_MAX * 2)
-#if (UINT32_C(1) << DICT_LOG_SIZE_MAX) != LZMA_DICTIONARY_SIZE_MAX
-# error DICT_LOG_SIZE_MAX is inconsistent with LZMA_DICTIONARY_SIZE_MAX
-#endif
-
-// 2 is for speed optimization
-#define LEN_TO_POS_STATES_BITS 2
-#define LEN_TO_POS_STATES (1 << LEN_TO_POS_STATES_BITS)
-
-#define MATCH_MIN_LEN 2
-
-#define ALIGN_BITS 4
-#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS)
-#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1)
-
-#define START_POS_MODEL_INDEX 4
-#define END_POS_MODEL_INDEX 14
-#define POS_MODELS (END_POS_MODEL_INDEX - START_POS_MODEL_INDEX)
-
-#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2)
-#define FULL_DISTANCES (1 << FULL_DISTANCES_BITS)
+///////////////////
+// Miscellaneous //
+///////////////////
+/// Maximum number of position states. A position state is the lowest pos bits
+/// number of bits of the current uncompressed offset. In some places there
+/// are different sets of probabilities for different pos states.
#define POS_STATES_MAX (1 << LZMA_POS_BITS_MAX)
-// Length coder & Length price table encoder
-#define LEN_LOW_BITS 3
-#define LEN_LOW_SYMBOLS (1 << LEN_LOW_BITS)
-#define LEN_MID_BITS 3
-#define LEN_MID_SYMBOLS (1 << LEN_MID_BITS)
-#define LEN_HIGH_BITS 8
-#define LEN_HIGH_SYMBOLS (1 << LEN_HIGH_BITS)
-#define LEN_SYMBOLS (LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS + LEN_HIGH_SYMBOLS)
-#define LEN_SPEC_SYMBOLS (LOW_LOW_SYMBOLS + LEN_MID_LEN_SYMBOLS)
-#define MATCH_MAX_LEN (MATCH_MIN_LEN + LEN_SYMBOLS - 1)
-
-// Total number of probs in a Len Encoder
-#define LEN_CODER_TOTAL_PROBS (LEN_HIGH_CODER + LEN_HIGH_SYMBOLS)
-
-// Price table size of Len Encoder
-#define LEN_PRICES (LEN_SYMBOLS << LZMA_POS_BITS_MAX)
-
-// Special lengths used together with distance == UINT32_MAX
-#define LEN_SPECIAL_EOPM MATCH_MIN_LEN
-#define LEN_SPECIAL_FLUSH (LEN_SPECIAL_EOPM + 1)
-
-
-// Optimal - Number of entries in the optimum array.
-#define OPTS (1 << 12)
-
-
-// Miscellaneous
-#define INFINITY_PRICE 0x0FFFFFFF
-
-
-////////////
-// Macros //
-////////////
-
-#define get_len_to_pos_state(len) \
- ((len) < LEN_TO_POS_STATES + MATCH_MIN_LEN \
- ? (len) - MATCH_MIN_LEN \
- : LEN_TO_POS_STATES - 1)
+/// Validates literal_context_bits, literal_pos_bits, and pos_bits.
+static inline bool
+is_lclppb_valid(const lzma_options_lzma *options)
+{
+ return options->literal_context_bits <= LZMA_LITERAL_CONTEXT_BITS_MAX
+ && options->literal_pos_bits
+ <= LZMA_LITERAL_POS_BITS_MAX
+ && options->literal_context_bits
+ + options->literal_pos_bits
+ <= LZMA_LITERAL_BITS_MAX
+ && options->pos_bits <= LZMA_POS_BITS_MAX;
+}
///////////
#define is_literal_state(state) \
((state) < LIT_STATES)
+
+/////////////
+// Literal //
+/////////////
+
+/// Each literal coder is divided in three sections:
+/// - 0x001-0x0FF: Without match byte
+/// - 0x101-0x1FF: With match byte; match bit is 0
+/// - 0x201-0x2FF: With match byte; match bit is 1
+///
+/// Match byte is used when the previous LZMA symbol was something else than
+/// a literal (that is, it was some kind of match).
+#define LITERAL_CODER_SIZE 0x300
+
+/// Maximum number of literal coders
+#define LITERAL_CODERS_MAX (1 << LZMA_LITERAL_BITS_MAX)
+
+/// Locate the literal coder for the next literal byte. The choice depends on
+/// - the lowest literal_pos_bits bits of the position of the current
+/// byte; and
+/// - the highest literal_context_bits bits of the previous byte.
+#define literal_subcoder(probs, lc, lp_mask, pos, prev_byte) \
+ ((probs)[(((pos) & lp_mask) << lc) + ((prev_byte) >> (8 - lc))])
+
+
+static inline void
+literal_init(probability (*probs)[LITERAL_CODER_SIZE],
+ uint32_t literal_context_bits, uint32_t literal_pos_bits)
+{
+ assert(literal_context_bits + literal_pos_bits
+ <= LZMA_LITERAL_BITS_MAX);
+
+ const uint32_t coders
+ = 1U << (literal_context_bits + literal_pos_bits);
+
+ for (uint32_t i = 0; i < coders; ++i)
+ for (uint32_t j = 0; j < LITERAL_CODER_SIZE; ++j)
+ bit_reset(probs[i][j]);
+
+ return;
+}
+
+
+//////////////////
+// Match length //
+//////////////////
+
+// Minimum length of a match is two bytes.
+#define MATCH_LEN_MIN 2
+
+// Match length is encoded with 4, 5, or 10 bits.
+//
+// Length Bits
+// 2-9 4 = Choice=0 + 3 bits
+// 10-17 5 = Choice=1 + Choice2=0 + 3 bits
+// 18-273 10 = Choice=1 + Choice2=1 + 8 bits
+#define LEN_LOW_BITS 3
+#define LEN_LOW_SYMBOLS (1 << LEN_LOW_BITS)
+#define LEN_MID_BITS 3
+#define LEN_MID_SYMBOLS (1 << LEN_MID_BITS)
+#define LEN_HIGH_BITS 8
+#define LEN_HIGH_SYMBOLS (1 << LEN_HIGH_BITS)
+#define LEN_SYMBOLS (LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS + LEN_HIGH_SYMBOLS)
+
+// Maximum length of a match is 273 which is a result of the encoding
+// described above.
+#define MATCH_LEN_MAX (MATCH_LEN_MIN + LEN_SYMBOLS - 1)
+
+
+////////////////////
+// Match distance //
+////////////////////
+
+// Different set of probabilities is used for match distances that have very
+// short match length: Lengths of 2, 3, and 4 bytes have a separate set of
+// probabilities for each length. The matches with longer length use a shared
+// set of probabilities.
+#define LEN_TO_POS_STATES 4
+
+// Macro to get the index of the appropriate probability array.
+#define get_len_to_pos_state(len) \
+ ((len) < LEN_TO_POS_STATES + MATCH_LEN_MIN \
+ ? (len) - MATCH_LEN_MIN \
+ : LEN_TO_POS_STATES - 1)
+
+// The highest two bits of a match distance (pos slot) are encoded using six
+// bits. See fastpos.h for more explanation.
+#define POS_SLOT_BITS 6
+#define POS_SLOTS (1 << POS_SLOT_BITS)
+
+// Match distances up to 127 are fully encoded using probabilities. Since
+// the highest two bits (pos slot) are always encoded using six bits, the
+// distances 0-3 don't need any additional bits to encode, since the pos
+// slot itself is the same as the actual distance. START_POS_MODEL_INDEX
+// indicates the first pos slot where at least one additional bit is needed.
+#define START_POS_MODEL_INDEX 4
+
+// Match distances greater than 127 are encoded in three pieces:
+// - pos slot: the highest two bits
+// - direct bits: 2-26 bits below the highest two bits
+// - alignment bits: four lowest bits
+//
+// Direct bits don't use any probabilities.
+//
+// The pos slot value of 14 is for distances 128-191 (see the table in
+// fastpos.h to understand why).
+#define END_POS_MODEL_INDEX 14
+
+// Seven-bit distances use the full FIXME
+#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2)
+#define FULL_DISTANCES (1 << FULL_DISTANCES_BITS)
+
+// For match distances greater than 127, only the highest two bits and the
+// lowest four bits (alignment) is encoded using probabilities.
+#define ALIGN_BITS 4
+#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS)
+#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1)
+
+// LZMA remembers the four most recent match distances. Reusing these distances
+// tends to take less space than re-encoding the actual distance value.
+#define REP_DISTANCES 4
+
#endif
/// \brief LZMA decoder
//
// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
+// Copyright (C) 2007-2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
//
///////////////////////////////////////////////////////////////////////////////
-// NOTE: If you want to keep the line length in 80 characters, set
-// tab width to 4 or less in your editor when editing this file.
-
+#include "lz_decoder.h"
#include "lzma_common.h"
#include "lzma_decoder.h"
-#include "lz_decoder.h"
#include "range_decoder.h"
-/// REQUIRED_IN_BUFFER_SIZE is the number of required input bytes
-/// for the worst case: longest match with longest distance.
-/// LZMA_IN_BUFFER_SIZE must be larger than REQUIRED_IN_BUFFER_SIZE.
-/// 23 bits = 2 (match select) + 10 (len) + 6 (distance) + 4 (align)
-/// + 1 (rc_normalize)
-///
-/// \todo Is this correct for sure?
-///
-#define REQUIRED_IN_BUFFER_SIZE \
- ((23 * (BIT_MODEL_TOTAL_BITS - MOVE_BITS + 1) + 26 + 9) / 8)
+#ifdef HAVE_SMALL
+// Macros for (somewhat) size-optimized code.
+#define seq_4(seq) seq
-// Length decoders are easiest to have as macros, because they use range
-// decoder macros, which use local variables rc_range and rc_code.
+#define seq_6(seq) seq
-#define length_decode(target, len_decoder, pos_state) \
+#define seq_8(seq) seq
+
+#define seq_len(seq) \
+ seq ## _CHOICE, \
+ seq ## _CHOICE2, \
+ seq ## _BITTREE
+
+#define len_decode(target, ld, pos_state, seq) \
do { \
- if_bit_0(len_decoder.choice) { \
- update_bit_0(len_decoder.choice); \
- target = MATCH_MIN_LEN; \
- bittree_decode(target, len_decoder.low[pos_state], LEN_LOW_BITS); \
+case seq ## _CHOICE: \
+ rc_if_0(ld.choice, seq ## _CHOICE) { \
+ rc_update_0(ld.choice); \
+ probs = ld.low[pos_state];\
+ limit = LEN_LOW_SYMBOLS; \
+ target = MATCH_LEN_MIN; \
} else { \
- update_bit_1(len_decoder.choice); \
- if_bit_0(len_decoder.choice2) { \
- update_bit_0(len_decoder.choice2); \
- target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \
- bittree_decode(target, len_decoder.mid[pos_state], LEN_MID_BITS); \
+ rc_update_1(ld.choice); \
+case seq ## _CHOICE2: \
+ rc_if_0(ld.choice2, seq ## _CHOICE2) { \
+ rc_update_0(ld.choice2); \
+ probs = ld.mid[pos_state]; \
+ limit = LEN_MID_SYMBOLS; \
+ target = MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \
} else { \
- update_bit_1(len_decoder.choice2); \
- target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \
- bittree_decode(target, len_decoder.high, LEN_HIGH_BITS); \
+ rc_update_1(ld.choice2); \
+ probs = ld.high; \
+ limit = LEN_HIGH_SYMBOLS; \
+ target = MATCH_LEN_MIN + LEN_LOW_SYMBOLS \
+ + LEN_MID_SYMBOLS; \
} \
} \
+ symbol = 1; \
+case seq ## _BITTREE: \
+ do { \
+ rc_bit(probs[symbol], , , seq ## _BITTREE); \
+ } while (symbol < limit); \
+ target += symbol - limit; \
} while (0)
-
-#define length_decode_dummy(target, len_decoder, pos_state) \
+#else // HAVE_SMALL
+
+// Unrolled versions
+#define seq_4(seq) \
+ seq ## 0, \
+ seq ## 1, \
+ seq ## 2, \
+ seq ## 3
+
+#define seq_6(seq) \
+ seq ## 0, \
+ seq ## 1, \
+ seq ## 2, \
+ seq ## 3, \
+ seq ## 4, \
+ seq ## 5
+
+#define seq_8(seq) \
+ seq ## 0, \
+ seq ## 1, \
+ seq ## 2, \
+ seq ## 3, \
+ seq ## 4, \
+ seq ## 5, \
+ seq ## 6, \
+ seq ## 7
+
+#define seq_len(seq) \
+ seq ## _CHOICE, \
+ seq ## _LOW0, \
+ seq ## _LOW1, \
+ seq ## _LOW2, \
+ seq ## _CHOICE2, \
+ seq ## _MID0, \
+ seq ## _MID1, \
+ seq ## _MID2, \
+ seq ## _HIGH0, \
+ seq ## _HIGH1, \
+ seq ## _HIGH2, \
+ seq ## _HIGH3, \
+ seq ## _HIGH4, \
+ seq ## _HIGH5, \
+ seq ## _HIGH6, \
+ seq ## _HIGH7
+
+#define len_decode(target, ld, pos_state, seq) \
do { \
- if_bit_0(len_decoder.choice) { \
- update_bit_0_dummy(); \
- target = MATCH_MIN_LEN; \
- bittree_decode_dummy(target, \
- len_decoder.low[pos_state], LEN_LOW_BITS); \
+ symbol = 1; \
+case seq ## _CHOICE: \
+ rc_if_0(ld.choice, seq ## _CHOICE) { \
+ rc_update_0(ld.choice); \
+ rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW0); \
+ rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW1); \
+ rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW2); \
+ target = symbol - LEN_LOW_SYMBOLS + MATCH_LEN_MIN; \
} else { \
- update_bit_1_dummy(); \
- if_bit_0(len_decoder.choice2) { \
- update_bit_0_dummy(); \
- target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \
- bittree_decode_dummy(target, len_decoder.mid[pos_state], \
- LEN_MID_BITS); \
+ rc_update_1(ld.choice); \
+case seq ## _CHOICE2: \
+ rc_if_0(ld.choice2, seq ## _CHOICE2) { \
+ rc_update_0(ld.choice2); \
+ rc_bit_case(ld.mid[pos_state][symbol], , , \
+ seq ## _MID0); \
+ rc_bit_case(ld.mid[pos_state][symbol], , , \
+ seq ## _MID1); \
+ rc_bit_case(ld.mid[pos_state][symbol], , , \
+ seq ## _MID2); \
+ target = symbol - LEN_MID_SYMBOLS \
+ + MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \
} else { \
- update_bit_1_dummy(); \
- target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \
- bittree_decode_dummy(target, len_decoder.high, LEN_HIGH_BITS); \
+ rc_update_1(ld.choice2); \
+ rc_bit_case(ld.high[symbol], , , seq ## _HIGH0); \
+ rc_bit_case(ld.high[symbol], , , seq ## _HIGH1); \
+ rc_bit_case(ld.high[symbol], , , seq ## _HIGH2); \
+ rc_bit_case(ld.high[symbol], , , seq ## _HIGH3); \
+ rc_bit_case(ld.high[symbol], , , seq ## _HIGH4); \
+ rc_bit_case(ld.high[symbol], , , seq ## _HIGH5); \
+ rc_bit_case(ld.high[symbol], , , seq ## _HIGH6); \
+ rc_bit_case(ld.high[symbol], , , seq ## _HIGH7); \
+ target = symbol - LEN_HIGH_SYMBOLS \
+ + MATCH_LEN_MIN \
+ + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \
} \
} \
} while (0)
+#endif // HAVE_SMALL
+
+/// Length decoder probabilities; see comments in lzma_common.h.
typedef struct {
probability choice;
probability choice2;
struct lzma_coder_s {
- /// Data of the next coder, if any.
- lzma_next_coder next;
-
- /// Sliding output window a.k.a. dictionary a.k.a. history buffer.
- lzma_lz_decoder lz;
-
- // Range coder
- lzma_range_decoder rc;
-
- // State
- lzma_lzma_state state;
- uint32_t rep0; ///< Distance of the latest match
- uint32_t rep1; ///< Distance of second latest match
- uint32_t rep2; ///< Distance of third latest match
- uint32_t rep3; ///< Distance of fourth latest match
- uint32_t pos_bits;
- uint32_t pos_mask;
- uint32_t now_pos; // Lowest 32-bits are enough here.
+ ///////////////////
+ // Probabilities //
+ ///////////////////
- lzma_literal_coder literal_coder;
+ /// Literals; see comments in lzma_common.h.
+ probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE];
/// If 1, it's a match. Otherwise it's a single 8-bit literal.
probability is_match[STATES][POS_STATES_MAX];
/// the length is decoded from rep_len_decoder.
probability is_rep0_long[STATES][POS_STATES_MAX];
- probability pos_slot_decoder[LEN_TO_POS_STATES][1 << POS_SLOT_BITS];
- probability pos_decoders[FULL_DISTANCES - END_POS_MODEL_INDEX];
- probability pos_align_decoder[1 << ALIGN_BITS];
-
- /// Length of a match
- lzma_length_decoder match_len_decoder;
-
- /// Length of a repeated match.
- lzma_length_decoder rep_len_decoder;
-
- /// True when we have produced at least one byte of output since the
- /// beginning of the stream or the latest flush marker.
- bool has_produced_output;
-};
-
-
-/// \brief Check if the next iteration of the decoder loop can be run.
-///
-/// \note There must always be REQUIRED_IN_BUFFER_SIZE bytes
-/// readable space!
-///
-static bool lzma_attribute((pure))
-decode_dummy(const lzma_coder *restrict coder,
- const uint8_t *restrict in, size_t in_pos_local,
- const size_t in_size, lzma_range_decoder rc,
- uint32_t state, uint32_t rep0, const uint32_t now_pos)
-{
- uint32_t rc_bound;
-
- do {
- const uint32_t pos_state = now_pos & coder->pos_mask;
-
- if_bit_0(coder->is_match[state][pos_state]) {
- // It's a literal i.e. a single 8-bit byte.
-
- update_bit_0_dummy();
-
- const probability *subcoder = literal_get_subcoder(
- coder->literal_coder, now_pos, lz_get_byte(coder->lz, 0));
- uint32_t symbol = 1;
-
- if (is_literal_state(state)) {
- // Decode literal without match byte.
- do {
- if_bit_0(subcoder[symbol]) {
- update_bit_0_dummy();
- symbol <<= 1;
- } else {
- update_bit_1_dummy();
- symbol = (symbol << 1) | 1;
- }
- } while (symbol < 0x100);
-
- } else {
- // Decode literal with match byte.
- uint32_t match_byte = lz_get_byte(coder->lz, rep0);
- uint32_t subcoder_offset = 0x100;
-
- do {
- match_byte <<= 1;
- const uint32_t match_bit = match_byte & subcoder_offset;
- const uint32_t subcoder_index
- = subcoder_offset + match_bit + symbol;
-
- if_bit_0(subcoder[subcoder_index]) {
- update_bit_0_dummy();
- symbol <<= 1;
- subcoder_offset &= ~match_bit;
- } else {
- update_bit_1_dummy();
- symbol = (symbol << 1) | 1;
- subcoder_offset &= match_bit;
- }
- } while (symbol < 0x100);
- }
-
- break;
- }
-
- update_bit_1_dummy();
- uint32_t len;
-
- if_bit_0(coder->is_rep[state]) {
- update_bit_0_dummy();
- length_decode_dummy(len, coder->match_len_decoder, pos_state);
-
- const uint32_t len_to_pos_state = get_len_to_pos_state(len);
- uint32_t pos_slot = 0;
- bittree_decode_dummy(pos_slot,
- coder->pos_slot_decoder[len_to_pos_state], POS_SLOT_BITS);
- assert(pos_slot <= 63);
-
- if (pos_slot >= START_POS_MODEL_INDEX) {
- uint32_t direct_bits = (pos_slot >> 1) - 1;
- assert(direct_bits >= 1 && direct_bits <= 31);
- rep0 = 2 | (pos_slot & 1);
-
- if (pos_slot < END_POS_MODEL_INDEX) {
- assert(direct_bits <= 5);
- rep0 <<= direct_bits;
- assert(rep0 <= 96);
- // -1 is fine, because bittree_reverse_decode()
- // starts from table index [1] (not [0]).
- assert((int32_t)(rep0 - pos_slot - 1) >= -1);
- assert((int32_t)(rep0 - pos_slot - 1) <= 82);
- // We add the result to rep0, so rep0
- // must not be part of second argument
- // of the macro.
- const int32_t offset = rep0 - pos_slot - 1;
- bittree_reverse_decode_dummy(coder->pos_decoders + offset,
- direct_bits);
- } else {
- assert(pos_slot >= 14);
- assert(direct_bits >= 6);
- direct_bits -= ALIGN_BITS;
- assert(direct_bits >= 2);
- rc_decode_direct_dummy(direct_bits);
-
- bittree_reverse_decode_dummy(coder->pos_align_decoder,
- ALIGN_BITS);
- }
- }
+ /// Probability tree for the highest two bits of the match distance.
+ /// There is a separate probability tree for match lengths of
+ /// 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273].
+ probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS];
- } else {
- update_bit_1_dummy();
+ /// Probility trees for additional bits for match distance when the
+ /// distance is in the range [4, 127].
+ probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX];
- if_bit_0(coder->is_rep0[state]) {
- update_bit_0_dummy();
+ /// Probability tree for the lowest four bits of a match distance
+ /// that is equal to or greater than 128.
+ probability pos_align[ALIGN_TABLE_SIZE];
- if_bit_0(coder->is_rep0_long[state][pos_state]) {
- update_bit_0_dummy();
- break;
- } else {
- update_bit_1_dummy();
- }
+ /// Length of a normal match
+ lzma_length_decoder match_len_decoder;
- } else {
- update_bit_1_dummy();
+ /// Length of a repeated match
+ lzma_length_decoder rep_len_decoder;
- if_bit_0(coder->is_rep1[state]) {
- update_bit_0_dummy();
- } else {
- update_bit_1_dummy();
+ ///////////////////
+ // Decoder state //
+ ///////////////////
- if_bit_0(coder->is_rep2[state]) {
- update_bit_0_dummy();
- } else {
- update_bit_1_dummy();
- }
- }
- }
+ // Range coder
+ lzma_range_decoder rc;
- length_decode_dummy(len, coder->rep_len_decoder, pos_state);
- }
- } while (0);
+ // Types of the most recently seen LZMA symbols
+ lzma_lzma_state state;
- rc_normalize();
+ uint32_t rep0; ///< Distance of the latest match
+ uint32_t rep1; ///< Distance of second latest match
+ uint32_t rep2; ///< Distance of third latest match
+ uint32_t rep3; ///< Distance of fourth latest match
- return in_pos_local <= in_size;
-}
+ uint32_t pos_mask; // (1U << pos_bits) - 1
+ uint32_t literal_context_bits;
+ uint32_t literal_pos_mask;
+
+ /// Uncompressed size as bytes, or LZMA_VLI_VALUE_UNKNOWN if end of
+ /// payload marker is expected.
+ lzma_vli uncompressed_size;
+
+ ////////////////////////////////
+ // State of incomplete symbol //
+ ////////////////////////////////
+
+ /// Position where to continue the decoder loop
+ enum {
+ SEQ_NORMALIZE,
+ SEQ_IS_MATCH,
+ seq_8(SEQ_LITERAL),
+ seq_8(SEQ_LITERAL_MATCHED),
+ SEQ_LITERAL_WRITE,
+ SEQ_IS_REP,
+ seq_len(SEQ_MATCH_LEN),
+ seq_6(SEQ_POS_SLOT),
+ SEQ_POS_MODEL,
+ SEQ_DIRECT,
+ seq_4(SEQ_ALIGN),
+ SEQ_EOPM,
+ SEQ_IS_REP0,
+ SEQ_SHORTREP,
+ SEQ_IS_REP0_LONG,
+ SEQ_IS_REP1,
+ SEQ_IS_REP2,
+ seq_len(SEQ_REP_LEN),
+ SEQ_COPY,
+ } sequence;
+
+ /// Base of the current probability tree
+ probability *probs;
+
+ /// Symbol being decoded. This is also used as an index variable in
+ /// bittree decoders: probs[symbol]
+ uint32_t symbol;
+
+ /// Used as a loop termination condition on bittree decoders and
+ /// direct bits decoder.
+ uint32_t limit;
+
+ /// Matched literal decoder: 0x100 or 0 to help avoiding branches.
+ /// Bittree reverse decoders: Offset of the next bit: 1 << offset
+ uint32_t offset;
+
+ /// If decoding a literal: match byte.
+ /// If decoding a match: length of the match.
+ uint32_t len;
+};
-static bool
-decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
- size_t *restrict in_pos, size_t in_size, bool has_safe_buffer)
+static lzma_ret
+lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr,
+ const uint8_t *restrict in,
+ size_t *restrict in_pos, size_t in_size)
{
////////////////////
// Initialization //
////////////////////
if (!rc_read_init(&coder->rc, in, in_pos, in_size))
- return false;
+ return LZMA_OK;
///////////////
// Variables //
// Making local copies of often-used variables improves both
// speed and readability.
+ lzma_dict dict = *dictptr;
+
+ const size_t dict_start = dict.pos;
+
// Range decoder
- rc_to_local(coder->rc);
+ rc_to_local(coder->rc, *in_pos);
// State
uint32_t state = coder->state;
uint32_t rep2 = coder->rep2;
uint32_t rep3 = coder->rep3;
- // Misc
- uint32_t now_pos = coder->now_pos;
- bool has_produced_output = coder->has_produced_output;
-
- // Variables derived from decoder settings
const uint32_t pos_mask = coder->pos_mask;
- size_t in_pos_local = *in_pos; // Local copy
- size_t in_limit;
- if (in_size <= REQUIRED_IN_BUFFER_SIZE)
- in_limit = 0;
- else
- in_limit = in_size - REQUIRED_IN_BUFFER_SIZE;
-
-
- while (coder->lz.pos < coder->lz.limit
- && (in_pos_local < in_limit || (has_safe_buffer
- && decode_dummy(coder, in, in_pos_local, in_size,
- rc, state, rep0, now_pos)))) {
-
- /////////////////////
- // Actual decoding //
- /////////////////////
-
- const uint32_t pos_state = now_pos & pos_mask;
+ // These variables are actually needed only if we last time ran
+ // out of input in the middle of the decoder loop.
+ probability *probs = coder->probs;
+ uint32_t symbol = coder->symbol;
+ uint32_t limit = coder->limit;
+ uint32_t offset = coder->offset;
+ uint32_t len = coder->len;
+
+ const uint32_t literal_pos_mask = coder->literal_pos_mask;
+ const uint32_t literal_context_bits = coder->literal_context_bits;
+
+ // Temporary variables
+ uint32_t pos_state = dict.pos & pos_mask;
+
+ lzma_ret ret = LZMA_OK;
+
+ // If uncompressed size is known, there must be no end of payload
+ // marker.
+ const bool no_eopm = coder->uncompressed_size
+ != LZMA_VLI_VALUE_UNKNOWN;
+ if (no_eopm && coder->uncompressed_size < dict.limit - dict.pos)
+ dict.limit = dict.pos + (size_t)(coder->uncompressed_size);
+
+ // The main decoder loop. The "switch" is used to restart the decoder at
+ // correct location. Once restarted, the "switch" is no longer used.
+ switch (coder->sequence)
+ while (true) {
+ // Calculate new pos_state. This is skipped on the first loop
+ // since we already calculated it when setting up the local
+ // variables.
+ pos_state = dict.pos & pos_mask;
+
+ case SEQ_NORMALIZE:
+ case SEQ_IS_MATCH:
+ if (unlikely(no_eopm && dict.pos == dict.limit))
+ break;
- if_bit_0(coder->is_match[state][pos_state]) {
- update_bit_0(coder->is_match[state][pos_state]);
+ rc_if_0(coder->is_match[state][pos_state], SEQ_IS_MATCH) {
+ rc_update_0(coder->is_match[state][pos_state]);
// It's a literal i.e. a single 8-bit byte.
- probability *subcoder = literal_get_subcoder(coder->literal_coder,
- now_pos, lz_get_byte(coder->lz, 0));
- uint32_t symbol = 1;
+ probs = literal_subcoder(coder->literal,
+ literal_context_bits, literal_pos_mask,
+ dict.pos, dict_get(&dict, 0));
+ symbol = 1;
if (is_literal_state(state)) {
// Decode literal without match byte.
+#ifdef HAVE_SMALL
+ case SEQ_LITERAL:
do {
- if_bit_0(subcoder[symbol]) {
- update_bit_0(subcoder[symbol]);
- symbol <<= 1;
- } else {
- update_bit_1(subcoder[symbol]);
- symbol = (symbol << 1) | 1;
- }
- } while (symbol < 0x100);
-
+ rc_bit(probs[symbol], , , SEQ_LITERAL);
+ } while (symbol < (1 << 8));
+#else
+ rc_bit_case(probs[symbol], , , SEQ_LITERAL0);
+ rc_bit_case(probs[symbol], , , SEQ_LITERAL1);
+ rc_bit_case(probs[symbol], , , SEQ_LITERAL2);
+ rc_bit_case(probs[symbol], , , SEQ_LITERAL3);
+ rc_bit_case(probs[symbol], , , SEQ_LITERAL4);
+ rc_bit_case(probs[symbol], , , SEQ_LITERAL5);
+ rc_bit_case(probs[symbol], , , SEQ_LITERAL6);
+ rc_bit_case(probs[symbol], , , SEQ_LITERAL7);
+#endif
} else {
// Decode literal with match byte.
//
- // The usage of subcoder_offset allows omitting some
- // branches, which should give tiny speed improvement on
- // some CPUs. subcoder_offset gets set to zero if match_bit
- // didn't match.
- uint32_t match_byte = lz_get_byte(coder->lz, rep0);
- uint32_t subcoder_offset = 0x100;
-
+ // We store the byte we compare against
+ // ("match byte") to "len" to minimize the
+ // number of variables we need to store
+ // between decoder calls.
+ len = dict_get(&dict, rep0) << 1;
+
+ // The usage of "offset" allows omitting some
+ // branches, which should give tiny speed
+ // improvement on some CPUs. "offset" gets
+ // set to zero if match_bit didn't match.
+ offset = 0x100;
+
+#ifdef HAVE_SMALL
+ case SEQ_LITERAL_MATCHED:
do {
- match_byte <<= 1;
- const uint32_t match_bit = match_byte & subcoder_offset;
+ const uint32_t match_bit
+ = len & offset;
const uint32_t subcoder_index
- = subcoder_offset + match_bit + symbol;
+ = offset + match_bit
+ + symbol;
+
+ rc_bit(probs[subcoder_index],
+ offset &= ~match_bit,
+ offset &= match_bit,
+ SEQ_LITERAL_MATCHED);
+
+ // It seems to be faster to do this
+ // here instead of putting it to the
+ // beginning of the loop and then
+ // putting the "case" in the middle
+ // of the loop.
+ len <<= 1;
+
+ } while (symbol < (1 << 8));
+#else
+ // Unroll the loop.
+ uint32_t match_bit;
+ uint32_t subcoder_index;
+
+# define d(seq) \
+ case seq: \
+ match_bit = len & offset; \
+ subcoder_index = offset + match_bit + symbol; \
+ rc_bit(probs[subcoder_index], \
+ offset &= ~match_bit, \
+ offset &= match_bit, \
+ seq)
+
+ d(SEQ_LITERAL_MATCHED0);
+ len <<= 1;
+ d(SEQ_LITERAL_MATCHED1);
+ len <<= 1;
+ d(SEQ_LITERAL_MATCHED2);
+ len <<= 1;
+ d(SEQ_LITERAL_MATCHED3);
+ len <<= 1;
+ d(SEQ_LITERAL_MATCHED4);
+ len <<= 1;
+ d(SEQ_LITERAL_MATCHED5);
+ len <<= 1;
+ d(SEQ_LITERAL_MATCHED6);
+ len <<= 1;
+ d(SEQ_LITERAL_MATCHED7);
+# undef d
+#endif
+ }
- if_bit_0(subcoder[subcoder_index]) {
- update_bit_0(subcoder[subcoder_index]);
- symbol <<= 1;
- subcoder_offset &= ~match_bit;
- } else {
- update_bit_1(subcoder[subcoder_index]);
- symbol = (symbol << 1) | 1;
- subcoder_offset &= match_bit;
- }
- } while (symbol < 0x100);
+ //update_literal(state);
+ // Use a lookup table to update to literal state,
+ // since compared to other state updates, this would
+ // need two branches.
+ static const lzma_lzma_state next_state[] = {
+ STATE_LIT_LIT,
+ STATE_LIT_LIT,
+ STATE_LIT_LIT,
+ STATE_LIT_LIT,
+ STATE_MATCH_LIT_LIT,
+ STATE_REP_LIT_LIT,
+ STATE_SHORTREP_LIT_LIT,
+ STATE_MATCH_LIT,
+ STATE_REP_LIT,
+ STATE_SHORTREP_LIT,
+ STATE_MATCH_LIT,
+ STATE_REP_LIT
+ };
+ state = next_state[state];
+
+ case SEQ_LITERAL_WRITE:
+ if (unlikely(dict_put(&dict, symbol))) {
+ coder->sequence = SEQ_LITERAL_WRITE;
+ goto out;
}
- // Put the decoded byte to the dictionary, update the
- // decoder state, and start a new decoding loop.
- coder->lz.dict[coder->lz.pos++] = (uint8_t)(symbol);
- ++now_pos;
- update_literal(state);
- has_produced_output = true;
continue;
}
// (distance and length) which will be repeated from our
// output history.
- update_bit_1(coder->is_match[state][pos_state]);
- uint32_t len;
-
- if_bit_0(coder->is_rep[state]) {
- update_bit_0(coder->is_rep[state]);
+ rc_update_1(coder->is_match[state][pos_state]);
+ case SEQ_IS_REP:
+ rc_if_0(coder->is_rep[state], SEQ_IS_REP) {
// Not a repeated match
- //
- // We will decode a new distance and store
- // the value to distance.
-
- // Decode the length of the match.
- length_decode(len, coder->match_len_decoder, pos_state);
-
+ rc_update_0(coder->is_rep[state]);
update_match(state);
- const uint32_t len_to_pos_state = get_len_to_pos_state(len);
- uint32_t pos_slot = 0;
- bittree_decode(pos_slot,
- coder->pos_slot_decoder[len_to_pos_state], POS_SLOT_BITS);
- assert(pos_slot <= 63);
-
- if (pos_slot >= START_POS_MODEL_INDEX) {
- uint32_t direct_bits = (pos_slot >> 1) - 1;
- assert(direct_bits >= 1 && direct_bits <= 30);
- uint32_t distance = 2 | (pos_slot & 1);
-
- if (pos_slot < END_POS_MODEL_INDEX) {
- assert(direct_bits <= 5);
- distance <<= direct_bits;
- assert(distance <= 96);
- // -1 is fine, because
- // bittree_reverse_decode()
- // starts from table index [1]
- // (not [0]).
- assert((int32_t)(distance - pos_slot - 1) >= -1);
- assert((int32_t)(distance - pos_slot - 1) <= 82);
- // We add the result to distance, so distance
- // must not be part of second argument
- // of the macro.
- const int32_t offset = distance - pos_slot - 1;
- bittree_reverse_decode(distance, coder->pos_decoders + offset,
- direct_bits);
+ // The latest three match distances are kept in
+ // memory in case there are repeated matches.
+ rep3 = rep2;
+ rep2 = rep1;
+ rep1 = rep0;
+
+ // Decode the length of the match.
+ len_decode(len, coder->match_len_decoder,
+ pos_state, SEQ_MATCH_LEN);
+
+ // Prepare to decode the highest two bits of the
+ // match distance.
+ probs = coder->pos_slot[get_len_to_pos_state(len)];
+ symbol = 1;
+
+#ifdef HAVE_SMALL
+ case SEQ_POS_SLOT:
+ do {
+ rc_bit(probs[symbol], , , SEQ_POS_SLOT);
+ } while (symbol < POS_SLOTS);
+#else
+ rc_bit_case(probs[symbol], , , SEQ_POS_SLOT0);
+ rc_bit_case(probs[symbol], , , SEQ_POS_SLOT1);
+ rc_bit_case(probs[symbol], , , SEQ_POS_SLOT2);
+ rc_bit_case(probs[symbol], , , SEQ_POS_SLOT3);
+ rc_bit_case(probs[symbol], , , SEQ_POS_SLOT4);
+ rc_bit_case(probs[symbol], , , SEQ_POS_SLOT5);
+#endif
+ // Get rid of the highest bit that was needed for
+ // indexing of the probability array.
+ symbol -= POS_SLOTS;
+ assert(symbol <= 63);
+
+ if (symbol < START_POS_MODEL_INDEX) {
+ // Match distances [0, 3] have only two bits.
+ rep0 = symbol;
+ } else {
+ // Decode the lowest [1, 29] bits of
+ // the match distance.
+ limit = (symbol >> 1) - 1;
+ assert(limit >= 1 && limit <= 30);
+ rep0 = 2 + (symbol & 1);
+
+ if (symbol < END_POS_MODEL_INDEX) {
+ // Prepare to decode the low bits for
+ // a distance of [4, 127].
+ assert(limit <= 5);
+ rep0 <<= limit;
+ assert(rep0 <= 96);
+ // -1 is fine, because we start
+ // decoding at probs[1], not probs[0].
+ // NOTE: This violates the C standard,
+ // since we are doing pointer
+ // arithmetic past the beginning of
+ // the array.
+ assert((int32_t)(rep0 - symbol - 1)
+ >= -1);
+ assert((int32_t)(rep0 - symbol - 1)
+ <= 82);
+ probs = coder->pos_special + rep0
+ - symbol - 1;
+ symbol = 1;
+ offset = 0;
+ case SEQ_POS_MODEL:
+#ifdef HAVE_SMALL
+ do {
+ rc_bit(probs[symbol], ,
+ rep0 += 1 << offset,
+ SEQ_POS_MODEL);
+ } while (++offset < limit);
+#else
+ switch (limit) {
+ case 5:
+ assert(offset == 0);
+ rc_bit(probs[symbol], ,
+ rep0 += 1,
+ SEQ_POS_MODEL);
+ ++offset;
+ --limit;
+ case 4:
+ rc_bit(probs[symbol], ,
+ rep0 += 1 << offset,
+ SEQ_POS_MODEL);
+ ++offset;
+ --limit;
+ case 3:
+ rc_bit(probs[symbol], ,
+ rep0 += 1 << offset,
+ SEQ_POS_MODEL);
+ ++offset;
+ --limit;
+ case 2:
+ rc_bit(probs[symbol], ,
+ rep0 += 1 << offset,
+ SEQ_POS_MODEL);
+ ++offset;
+ --limit;
+ case 1:
+ // We need "symbol" only for
+ // indexing the probability
+ // array, thus we can use
+ // rc_bit_last() here to omit
+ // the unneeded updating of
+ // "symbol".
+ rc_bit_last(probs[symbol], ,
+ rep0 += 1 << offset,
+ SEQ_POS_MODEL);
+ }
+#endif
} else {
- assert(pos_slot >= 14);
- assert(direct_bits >= 6);
- direct_bits -= ALIGN_BITS;
- assert(direct_bits >= 2);
- rc_decode_direct(distance, direct_bits);
- distance <<= ALIGN_BITS;
-
- bittree_reverse_decode(distance, coder->pos_align_decoder,
- ALIGN_BITS);
-
- if (distance == UINT32_MAX) {
- if (len == LEN_SPECIAL_EOPM) {
- // End of Payload Marker found.
- coder->lz.eopm_detected = true;
- break;
-
- } else if (len == LEN_SPECIAL_FLUSH) {
- // Flush marker detected. We must have produced
- // at least one byte of output since the previous
- // flush marker or the beginning of the stream.
- // This is to prevent hanging the decoder with
- // malicious input files.
- if (!has_produced_output)
- return true;
-
- has_produced_output = false;
-
- // We know that we have enough input to call
- // this macro, because it is tested at the
- // end of decode_dummy().
- rc_normalize();
-
- rc_reset(rc);
-
- // If we don't have enough input here, we jump
- // out of the loop. Note that while there is a
- // useless call to rc_normalize(), it does nothing
- // since we have just reset the range decoder.
- if (!rc_read_init(&rc, in, &in_pos_local, in_size))
- break;
-
- continue;
-
- } else {
- return true;
+ // The distace is >= 128. Decode the
+ // lower bits without probabilities
+ // except the lowest four bits.
+ assert(symbol >= 14);
+ assert(limit >= 6);
+ limit -= ALIGN_BITS;
+ assert(limit >= 2);
+ case SEQ_DIRECT:
+ // Not worth manual unrolling
+ do {
+ rc_direct(rep0, SEQ_DIRECT);
+ } while (--limit > 0);
+
+ // Decode the lowest four bits using
+ // probabilities.
+ rep0 <<= ALIGN_BITS;
+ symbol = 1;
+#ifdef HAVE_SMALL
+ offset = 0;
+ case SEQ_ALIGN:
+ do {
+ rc_bit(coder->pos_align[
+ symbol], ,
+ rep0 += 1 << offset,
+ SEQ_ALIGN);
+ } while (++offset < ALIGN_BITS);
+#else
+ case SEQ_ALIGN0:
+ rc_bit(coder->pos_align[symbol], ,
+ rep0 += 1, SEQ_ALIGN0);
+ case SEQ_ALIGN1:
+ rc_bit(coder->pos_align[symbol], ,
+ rep0 += 2, SEQ_ALIGN1);
+ case SEQ_ALIGN2:
+ rc_bit(coder->pos_align[symbol], ,
+ rep0 += 4, SEQ_ALIGN2);
+ case SEQ_ALIGN3:
+ // Like in SEQ_POS_MODEL, we don't
+ // need "symbol" for anything else
+ // than indexing the probability array.
+ rc_bit_last(coder->pos_align[symbol], ,
+ rep0 += 8, SEQ_ALIGN3);
+#endif
+
+ if (rep0 == UINT32_MAX) {
+ // End of payload marker was
+ // found. It must not be
+ // present if uncompressed
+ // size is known.
+ if (coder->uncompressed_size
+ != LZMA_VLI_VALUE_UNKNOWN) {
+ ret = LZMA_DATA_ERROR;
+ goto out;
}
+
+ case SEQ_EOPM:
+ // TODO Comment
+ rc_normalize(SEQ_EOPM);
+ ret = LZMA_STREAM_END;
+ goto out;
}
}
+ }
- // The latest three match distances are kept in
- // memory in case there are repeated matches.
- rep3 = rep2;
- rep2 = rep1;
- rep1 = rep0;
- rep0 = distance;
-
- } else {
- rep3 = rep2;
- rep2 = rep1;
- rep1 = rep0;
- rep0 = pos_slot;
+ // Validate the distance we just decoded.
+ if (unlikely(!dict_is_distance_valid(&dict, rep0))) {
+ ret = LZMA_DATA_ERROR;
+ goto out;
}
} else {
- update_bit_1(coder->is_rep[state]);
+ rc_update_1(coder->is_rep[state]);
// Repeated match
//
// earlier. The latest four match distances are
// available as rep0, rep1, rep2 and rep3. We will
// now decode which of them is the new distance.
+ //
+ // There cannot be a match if we haven't produced
+ // any output, so check that first.
+ if (unlikely(!dict_is_distance_valid(&dict, 0))) {
+ ret = LZMA_DATA_ERROR;
+ goto out;
+ }
- if_bit_0(coder->is_rep0[state]) {
- update_bit_0(coder->is_rep0[state]);
-
+ case SEQ_IS_REP0:
+ rc_if_0(coder->is_rep0[state], SEQ_IS_REP0) {
+ rc_update_0(coder->is_rep0[state]);
// The distance is rep0.
- if_bit_0(coder->is_rep0_long[state][pos_state]) {
- update_bit_0(coder->is_rep0_long[state][pos_state]);
+ case SEQ_IS_REP0_LONG:
+ rc_if_0(coder->is_rep0_long[state][pos_state],
+ SEQ_IS_REP0_LONG) {
+ rc_update_0(coder->is_rep0_long[
+ state][pos_state]);
update_short_rep(state);
- // Repeat exactly one byte and start a new decoding loop.
- // Note that rep0 is known to have a safe value, thus we
- // don't need to check if we are wrapping the dictionary
- // when it isn't full yet.
- if (unlikely(lz_is_empty(coder->lz)))
- return true;
-
- coder->lz.dict[coder->lz.pos]
- = lz_get_byte(coder->lz, rep0);
- ++coder->lz.pos;
- ++now_pos;
- has_produced_output = true;
- continue;
-
- } else {
- update_bit_1(coder->is_rep0_long[state][pos_state]);
+ case SEQ_SHORTREP:
+ if (unlikely(dict_put(&dict, dict_get(
+ &dict, rep0)))) {
+ coder->sequence = SEQ_SHORTREP;
+ goto out;
+ }
- // Repeating more than one byte at
- // distance of rep0.
+ continue;
}
+ // Repeating more than one byte at
+ // distance of rep0.
+ rc_update_1(coder->is_rep0_long[
+ state][pos_state]);
+
} else {
- update_bit_1(coder->is_rep0[state]);
+ rc_update_1(coder->is_rep0[state]);
+ case SEQ_IS_REP1:
// The distance is rep1, rep2 or rep3. Once
// we find out which one of these three, it
// is stored to rep0 and rep1, rep2 and rep3
// are updated accordingly.
+ rc_if_0(coder->is_rep1[state], SEQ_IS_REP1) {
+ rc_update_0(coder->is_rep1[state]);
- uint32_t distance;
+ const uint32_t distance = rep1;
+ rep1 = rep0;
+ rep0 = distance;
- if_bit_0(coder->is_rep1[state]) {
- update_bit_0(coder->is_rep1[state]);
- distance = rep1;
} else {
- update_bit_1(coder->is_rep1[state]);
+ rc_update_1(coder->is_rep1[state]);
+ case SEQ_IS_REP2:
+ rc_if_0(coder->is_rep2[state],
+ SEQ_IS_REP2) {
+ rc_update_0(coder->is_rep2[
+ state]);
+
+ const uint32_t distance = rep2;
+ rep2 = rep1;
+ rep1 = rep0;
+ rep0 = distance;
- if_bit_0(coder->is_rep2[state]) {
- update_bit_0(coder->is_rep2[state]);
- distance = rep2;
} else {
- update_bit_1(coder->is_rep2[state]);
- distance = rep3;
+ rc_update_1(coder->is_rep2[
+ state]);
+
+ const uint32_t distance = rep3;
rep3 = rep2;
+ rep2 = rep1;
+ rep1 = rep0;
+ rep0 = distance;
}
-
- rep2 = rep1;
}
-
- rep1 = rep0;
- rep0 = distance;
}
update_long_rep(state);
// Decode the length of the repeated match.
- length_decode(len, coder->rep_len_decoder, pos_state);
+ len_decode(len, coder->rep_len_decoder,
+ pos_state, SEQ_REP_LEN);
}
-
/////////////////////////////////
// Repeat from history buffer. //
/////////////////////////////////
// The length is always between these limits. There is no way
// to trigger the algorithm to set len outside this range.
- assert(len >= MATCH_MIN_LEN);
- assert(len <= MATCH_MAX_LEN);
-
- now_pos += len;
- has_produced_output = true;
+ assert(len >= MATCH_LEN_MIN);
+ assert(len <= MATCH_LEN_MAX);
+ case SEQ_COPY:
// Repeat len bytes from distance of rep0.
- if (!lzma_lz_out_repeat(&coder->lz, rep0, len))
- return true;
+ if (unlikely(dict_repeat(&dict, rep0, &len))) {
+ coder->sequence = SEQ_COPY;
+ goto out;
+ }
}
- rc_normalize();
+ rc_normalize(SEQ_NORMALIZE);
+ coder->sequence = SEQ_IS_MATCH;
+out:
+ // Save state
- /////////////////////////////////
- // Update the *data structure. //
- /////////////////////////////////
+ // NOTE: Must not copy dict.limit.
+ dictptr->pos = dict.pos;
+ dictptr->full = dict.full;
- // Range decoder
- rc_from_local(coder->rc);
+ rc_from_local(coder->rc, *in_pos);
- // State
coder->state = state;
coder->rep0 = rep0;
coder->rep1 = rep1;
coder->rep2 = rep2;
coder->rep3 = rep3;
- // Misc
- coder->now_pos = now_pos;
- coder->has_produced_output = has_produced_output;
- *in_pos = in_pos_local;
+ coder->probs = probs;
+ coder->symbol = symbol;
+ coder->limit = limit;
+ coder->offset = offset;
+ coder->len = len;
+
+ // Update the remaining amount of uncompressed data if uncompressed
+ // size was known.
+ if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) {
+ coder->uncompressed_size -= dict.pos - dict_start;
+
+ // Since there cannot be end of payload marker if the
+ // uncompressed size was known, we check here if we
+ // finished decoding.
+ if (coder->uncompressed_size == 0 && ret == LZMA_OK
+ && coder->sequence != SEQ_NORMALIZE)
+ ret = coder->sequence == SEQ_IS_MATCH
+ ? LZMA_STREAM_END : LZMA_DATA_ERROR;
+ }
+
+ // We can do an additional check in the range decoder to catch some
+ // corrupted files.
+ if (ret == LZMA_STREAM_END) {
+ if (!rc_is_finished(coder->rc))
+ ret = LZMA_DATA_ERROR;
- return false;
+ // Reset the range decoder so that it is ready to reinitialize
+ // for a new LZMA2 chunk.
+ rc_reset(coder->rc);
+ }
+
+ return ret;
}
+
static void
-lzma_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
+lzma_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size)
{
- lzma_next_coder_end(&coder->next, allocator);
- lzma_lz_decoder_end(&coder->lz, allocator);
- lzma_free(coder, allocator);
- return;
+ coder->uncompressed_size = uncompressed_size;
}
-
-extern lzma_ret
-lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- const lzma_filter_info *filters)
+/*
+extern void
+lzma_lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size)
{
- // LZMA can only be the last filter in the chain.
- assert(filters[1].init == NULL);
-
- // Validate pos_bits. Other options are validated by the
- // respective initialization functions.
- const lzma_options_lzma *options = filters[0].options;
- if (options->pos_bits > LZMA_POS_BITS_MAX)
- return LZMA_HEADER_ERROR;
+ // This is hack.
+ (*(lzma_coder **)(coder))->uncompressed_size = uncompressed_size;
+}
+*/
- // Allocate memory for the decoder if needed.
- if (next->coder == NULL) {
- next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
- if (next->coder == NULL)
- return LZMA_MEM_ERROR;
+static void
+lzma_decoder_reset(lzma_coder *coder, const void *opt)
+{
+ const lzma_options_lzma *options = opt;
- next->code = &lzma_lz_decode;
- next->end = &lzma_decoder_end;
- next->coder->next = LZMA_NEXT_CODER_INIT;
- next->coder->lz = LZMA_LZ_DECODER_INIT;
- }
+ // NOTE: We assume that lc/lp/pb are valid since they were
+ // successfully decoded with lzma_lzma_decode_properties().
+ // FIXME?
- // Store the pos_bits and calculate pos_mask.
- next->coder->pos_bits = options->pos_bits;
- next->coder->pos_mask = (1U << next->coder->pos_bits) - 1;
+ // Calculate pos_mask. We don't need pos_bits as is for anything.
+ coder->pos_mask = (1U << options->pos_bits) - 1;
// Initialize the literal decoder.
- return_if_error(lzma_literal_init(&next->coder->literal_coder,
- options->literal_context_bits,
- options->literal_pos_bits));
+ literal_init(coder->literal, options->literal_context_bits,
+ options->literal_pos_bits);
- // Allocate and initialize the LZ decoder.
- return_if_error(lzma_lz_decoder_reset(&next->coder->lz, allocator,
- &decode_real, options->dictionary_size,
- MATCH_MAX_LEN));
+ coder->literal_context_bits = options->literal_context_bits;
+ coder->literal_pos_mask = (1 << options->literal_pos_bits) - 1;
// State
- next->coder->state = 0;
- next->coder->rep0 = 0;
- next->coder->rep1 = 0;
- next->coder->rep2 = 0;
- next->coder->rep3 = 0;
- next->coder->pos_bits = options->pos_bits;
- next->coder->pos_mask = (1 << next->coder->pos_bits) - 1;
- next->coder->now_pos = 0;
+ coder->state = STATE_LIT_LIT;
+ coder->rep0 = 0;
+ coder->rep1 = 0;
+ coder->rep2 = 0;
+ coder->rep3 = 0;
+ coder->pos_mask = (1 << options->pos_bits) - 1;
// Range decoder
- rc_reset(next->coder->rc);
+ rc_reset(coder->rc);
// Bit and bittree decoders
for (uint32_t i = 0; i < STATES; ++i) {
- for (uint32_t j = 0; j <= next->coder->pos_mask; ++j) {
- bit_reset(next->coder->is_match[i][j]);
- bit_reset(next->coder->is_rep0_long[i][j]);
+ for (uint32_t j = 0; j <= coder->pos_mask; ++j) {
+ bit_reset(coder->is_match[i][j]);
+ bit_reset(coder->is_rep0_long[i][j]);
}
- bit_reset(next->coder->is_rep[i]);
- bit_reset(next->coder->is_rep0[i]);
- bit_reset(next->coder->is_rep1[i]);
- bit_reset(next->coder->is_rep2[i]);
+ bit_reset(coder->is_rep[i]);
+ bit_reset(coder->is_rep0[i]);
+ bit_reset(coder->is_rep1[i]);
+ bit_reset(coder->is_rep2[i]);
}
for (uint32_t i = 0; i < LEN_TO_POS_STATES; ++i)
- bittree_reset(next->coder->pos_slot_decoder[i], POS_SLOT_BITS);
+ bittree_reset(coder->pos_slot[i], POS_SLOT_BITS);
for (uint32_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i)
- bit_reset(next->coder->pos_decoders[i]);
+ bit_reset(coder->pos_special[i]);
- bittree_reset(next->coder->pos_align_decoder, ALIGN_BITS);
+ bittree_reset(coder->pos_align, ALIGN_BITS);
// Len decoders (also bit/bittree)
- const uint32_t num_pos_states = 1 << next->coder->pos_bits;
- bit_reset(next->coder->match_len_decoder.choice);
- bit_reset(next->coder->match_len_decoder.choice2);
- bit_reset(next->coder->rep_len_decoder.choice);
- bit_reset(next->coder->rep_len_decoder.choice2);
+ const uint32_t num_pos_states = 1 << options->pos_bits;
+ bit_reset(coder->match_len_decoder.choice);
+ bit_reset(coder->match_len_decoder.choice2);
+ bit_reset(coder->rep_len_decoder.choice);
+ bit_reset(coder->rep_len_decoder.choice2);
for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) {
- bittree_reset(next->coder->match_len_decoder.low[pos_state],
+ bittree_reset(coder->match_len_decoder.low[pos_state],
LEN_LOW_BITS);
- bittree_reset(next->coder->match_len_decoder.mid[pos_state],
+ bittree_reset(coder->match_len_decoder.mid[pos_state],
LEN_MID_BITS);
- bittree_reset(next->coder->rep_len_decoder.low[pos_state],
+ bittree_reset(coder->rep_len_decoder.low[pos_state],
LEN_LOW_BITS);
- bittree_reset(next->coder->rep_len_decoder.mid[pos_state],
+ bittree_reset(coder->rep_len_decoder.mid[pos_state],
LEN_MID_BITS);
}
- bittree_reset(next->coder->match_len_decoder.high, LEN_HIGH_BITS);
- bittree_reset(next->coder->rep_len_decoder.high, LEN_HIGH_BITS);
+ bittree_reset(coder->match_len_decoder.high, LEN_HIGH_BITS);
+ bittree_reset(coder->rep_len_decoder.high, LEN_HIGH_BITS);
+
+ coder->sequence = SEQ_IS_MATCH;
+ coder->probs = NULL;
+ coder->symbol = 0;
+ coder->limit = 0;
+ coder->offset = 0;
+ coder->len = 0;
+
+ return;
+}
+
+
+extern lzma_ret
+lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator,
+ const void *opt, size_t *dict_size)
+{
+ if (lz->coder == NULL) {
+ lz->coder = lzma_alloc(sizeof(lzma_coder), allocator);
+ if (lz->coder == NULL)
+ return LZMA_MEM_ERROR;
+
+ lz->code = &lzma_decode;
+ lz->reset = &lzma_decoder_reset;
+ lz->set_uncompressed = &lzma_decoder_uncompressed;
+ }
- next->coder->has_produced_output = false;
+ // All dictionary sizes are OK here. LZ decoder will take care of
+ // the special cases.
+ const lzma_options_lzma *options = opt;
+ *dict_size = options->dictionary_size;
return LZMA_OK;
}
-extern void
-lzma_lzma_decoder_uncompressed_size(
- lzma_next_coder *next, lzma_vli uncompressed_size)
+/// Allocate and initialize LZMA decoder. This is used only via LZ
+/// initialization (lzma_lzma_decoder_init() passes function pointer to
+/// the LZ initialization).
+static lzma_ret
+lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator,
+ const void *options, size_t *dict_size)
{
- next->coder->lz.uncompressed_size = uncompressed_size;
- return;
+ if (!is_lclppb_valid(options))
+ return LZMA_PROG_ERROR;
+
+ return_if_error(lzma_lzma_decoder_create(
+ lz, allocator, options, dict_size));
+
+ lzma_decoder_reset(lz->coder, options);
+ lzma_decoder_uncompressed(lz->coder, LZMA_VLI_VALUE_UNKNOWN);
+
+ return LZMA_OK;
+}
+
+
+extern lzma_ret
+lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter_info *filters)
+{
+ // LZMA can only be the last filter in the chain. This is enforced
+ // by the raw_decoder initialization.
+ assert(filters[1].init == NULL);
+
+ return lzma_lz_decoder_init(next, allocator, filters,
+ &lzma_decoder_init);
}
extern bool
-lzma_lzma_decode_properties(lzma_options_lzma *options, uint8_t byte)
+lzma_lzma_lclppb_decode(lzma_options_lzma *options, uint8_t byte)
{
if (byte > (4 * 5 + 4) * 9 + 8)
return true;
return options->literal_context_bits + options->literal_pos_bits
> LZMA_LITERAL_BITS_MAX;
}
+
+
+extern uint64_t
+lzma_lzma_decoder_memusage(const void *options)
+{
+ const lzma_options_lzma *const opt = options;
+ const uint64_t lz_memusage
+ = lzma_lz_decoder_memusage(opt->dictionary_size);
+ if (lz_memusage == UINT64_MAX)
+ return UINT64_MAX;
+
+ return sizeof(lzma_coder) + lz_memusage;
+}
+
+
+extern lzma_ret
+lzma_lzma_props_decode(void **options, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size)
+{
+ if (props_size != 5)
+ return LZMA_HEADER_ERROR;
+
+ lzma_options_lzma *opt
+ = lzma_alloc(sizeof(lzma_options_lzma), allocator);
+ if (opt == NULL)
+ return LZMA_MEM_ERROR;
+
+ if (lzma_lzma_lclppb_decode(opt, props[0]))
+ goto error;
+
+ // All dictionary sizes are accepted, including zero. LZ decoder
+ // will automatically use a dictionary at least a few KiB even if
+ // a smaller dictionary is requested.
+ opt->dictionary_size = integer_read_32(props + 1);
+
+ opt->preset_dictionary = NULL;
+ opt->preset_dictionary_size = 0;
+
+ *options = opt;
+
+ return LZMA_OK;
+
+error:
+ lzma_free(opt, allocator);
+ return LZMA_HEADER_ERROR;
+}
extern lzma_ret lzma_lzma_decoder_init(lzma_next_coder *next,
lzma_allocator *allocator, const lzma_filter_info *filters);
-/// Set known uncompressed size. This is a hack needed to support
-/// LZMA_Alone files that don't have EOPM.
-extern void lzma_lzma_decoder_uncompressed_size(
- lzma_next_coder *next, lzma_vli uncompressed_size);
+extern uint64_t lzma_lzma_decoder_memusage(const void *options);
+
+extern lzma_ret lzma_lzma_props_decode(
+ void **options, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size);
+
/// \brief Decodes the LZMA Properties byte (lc/lp/pb)
///
/// \return true if error occorred, false on success
///
-extern bool lzma_lzma_decode_properties(
+extern bool lzma_lzma_lclppb_decode(
lzma_options_lzma *options, uint8_t byte);
+
+#ifdef LZMA_LZ_DECODER_H
+/// Allocate and setup function pointers only. This is used by LZMA1 and
+/// LZMA2 decoders.
+extern lzma_ret lzma_lzma_decoder_create(
+ lzma_lz_decoder *lz, lzma_allocator *allocator,
+ const void *opt, size_t *dict_size);
+#endif
+
#endif
literal_matched(lzma_range_encoder *rc, probability *subcoder,
uint32_t match_byte, uint32_t symbol)
{
- uint32_t context = 1;
- uint32_t bit_count = 8;
+ uint32_t offset = 0x100;
+ symbol += UINT32_C(1) << 8;
do {
- uint32_t bit = (symbol >> --bit_count) & 1;
- const uint32_t match_bit = (match_byte >> bit_count) & 1;
- rc_bit(rc, &subcoder[(0x100 << match_bit) + context], bit);
- context = (context << 1) | bit;
-
- if (match_bit != bit) {
- // The bit from the literal being encoded and the bit
- // from the previous match differ. Finish encoding
- // as a normal literal.
- while (bit_count != 0) {
- bit = (symbol >> --bit_count) & 1;
- rc_bit(rc, &subcoder[context], bit);
- context = (context << 1) | bit;
- }
+ match_byte <<= 1;
+ const uint32_t match_bit = match_byte & offset;
+ const uint32_t subcoder_index
+ = offset + match_bit + (symbol >> 8);
+ const uint32_t bit = (symbol >> 7) & 1;
+ rc_bit(rc, &subcoder[subcoder_index], bit);
- break;
- }
+ symbol <<= 1;
+ offset &= ~(match_byte ^ symbol);
- } while (bit_count != 0);
+ } while (symbol < (UINT32_C(1) << 16));
}
static inline void
-literal(lzma_coder *coder)
+literal(lzma_coder *coder, lzma_mf *mf, uint32_t position)
{
// Locate the literal byte to be encoded and the subcoder.
- const uint8_t cur_byte = coder->lz.buffer[
- coder->lz.read_pos - coder->additional_offset];
- probability *subcoder = literal_get_subcoder(coder->literal_coder,
- coder->now_pos, coder->previous_byte);
+ const uint8_t cur_byte = mf->buffer[
+ mf->read_pos - mf->read_ahead];
+ probability *subcoder = literal_subcoder(coder->literal,
+ coder->literal_context_bits, coder->literal_pos_mask,
+ position, mf->buffer[mf->read_pos - mf->read_ahead - 1]);
if (is_literal_state(coder->state)) {
// Previous LZMA-symbol was a literal. Encode a normal
// Previous LZMA-symbol was a match. Use the last byte of
// the match as a "match byte". That is, compare the bits
// of the current literal and the match byte.
- const uint8_t match_byte = coder->lz.buffer[
- coder->lz.read_pos - coder->reps[0] - 1
- - coder->additional_offset];
+ const uint8_t match_byte = mf->buffer[
+ mf->read_pos - coder->reps[0] - 1
+ - mf->read_ahead];
literal_matched(&coder->rc, subcoder, match_byte, cur_byte);
}
update_literal(coder->state);
- coder->previous_byte = cur_byte;
}
// Match length //
//////////////////
+static void
+length_update_prices(lzma_length_encoder *lc, const uint32_t pos_state)
+{
+ const uint32_t table_size = lc->table_size;
+ lc->counters[pos_state] = table_size;
+
+ const uint32_t a0 = rc_bit_0_price(lc->choice);
+ const uint32_t a1 = rc_bit_1_price(lc->choice);
+ const uint32_t b0 = a1 + rc_bit_0_price(lc->choice2);
+ const uint32_t b1 = a1 + rc_bit_1_price(lc->choice2);
+ uint32_t *const prices = lc->prices[pos_state];
+
+ uint32_t i;
+ for (i = 0; i < table_size && i < LEN_LOW_SYMBOLS; ++i)
+ prices[i] = a0 + rc_bittree_price(lc->low[pos_state],
+ LEN_LOW_BITS, i);
+
+ for (; i < table_size && i < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; ++i)
+ prices[i] = b0 + rc_bittree_price(lc->mid[pos_state],
+ LEN_MID_BITS, i - LEN_LOW_SYMBOLS);
+
+ for (; i < table_size; ++i)
+ prices[i] = b1 + rc_bittree_price(lc->high, LEN_HIGH_BITS,
+ i - LEN_LOW_SYMBOLS - LEN_MID_SYMBOLS);
+
+ return;
+}
+
+
static inline void
length(lzma_range_encoder *rc, lzma_length_encoder *lc,
- const uint32_t pos_state, uint32_t len)
+ const uint32_t pos_state, uint32_t len, const bool fast_mode)
{
- assert(len <= MATCH_MAX_LEN);
- len -= MATCH_MIN_LEN;
+ assert(len <= MATCH_LEN_MAX);
+ len -= MATCH_LEN_MIN;
if (len < LEN_LOW_SYMBOLS) {
rc_bit(rc, &lc->choice, 0);
rc_bittree(rc, lc->high, LEN_HIGH_BITS, len);
}
}
+
+ // Only getoptimum uses the prices so don't update the table when
+ // in fast mode.
+ if (!fast_mode)
+ if (--lc->counters[pos_state] == 0)
+ length_update_prices(lc, pos_state);
}
{
update_match(coder->state);
- length(&coder->rc, &coder->match_len_encoder, pos_state, len);
- coder->prev_len_encoder = &coder->match_len_encoder;
+ length(&coder->rc, &coder->match_len_encoder, pos_state, len,
+ coder->fast_mode);
const uint32_t pos_slot = get_pos_slot(distance);
const uint32_t len_to_pos_state = get_len_to_pos_state(len);
- rc_bittree(&coder->rc, coder->pos_slot_encoder[len_to_pos_state],
+ rc_bittree(&coder->rc, coder->pos_slot[len_to_pos_state],
POS_SLOT_BITS, pos_slot);
if (pos_slot >= START_POS_MODEL_INDEX) {
if (pos_slot < END_POS_MODEL_INDEX) {
rc_bittree_reverse(&coder->rc,
- &coder->pos_encoders[base - pos_slot - 1],
+ &coder->pos_special[base - pos_slot - 1],
footer_bits, pos_reduced);
} else {
rc_direct(&coder->rc, pos_reduced >> ALIGN_BITS,
footer_bits - ALIGN_BITS);
rc_bittree_reverse(
- &coder->rc, coder->pos_align_encoder,
+ &coder->rc, coder->pos_align,
ALIGN_BITS, pos_reduced & ALIGN_MASK);
++coder->align_price_count;
}
if (len == 1) {
update_short_rep(coder->state);
} else {
- length(&coder->rc, &coder->rep_len_encoder, pos_state, len);
- coder->prev_len_encoder = &coder->rep_len_encoder;
+ length(&coder->rc, &coder->rep_len_encoder, pos_state, len,
+ coder->fast_mode);
update_long_rep(coder->state);
}
}
//////////
static void
-encode_symbol(lzma_coder *coder, uint32_t pos, uint32_t len)
+encode_symbol(lzma_coder *coder, lzma_mf *mf,
+ uint32_t back, uint32_t len, uint32_t position)
{
- const uint32_t pos_state = coder->now_pos & coder->pos_mask;
+ const uint32_t pos_state = position & coder->pos_mask;
- if (len == 1 && pos == UINT32_MAX) {
+ if (back == UINT32_MAX) {
// Literal i.e. eight-bit byte
+ assert(len == 1);
rc_bit(&coder->rc,
&coder->is_match[coder->state][pos_state], 0);
- literal(coder);
+ literal(coder, mf, position);
} else {
// Some type of match
rc_bit(&coder->rc,
&coder->is_match[coder->state][pos_state], 1);
- if (pos < REP_DISTANCES) {
+ if (back < REP_DISTANCES) {
// It's a repeated match i.e. the same distance
// has been used earlier.
rc_bit(&coder->rc, &coder->is_rep[coder->state], 1);
- rep_match(coder, pos_state, pos, len);
+ rep_match(coder, pos_state, back, len);
} else {
// Normal match
rc_bit(&coder->rc, &coder->is_rep[coder->state], 0);
- match(coder, pos_state, pos - REP_DISTANCES, len);
+ match(coder, pos_state, back - REP_DISTANCES, len);
}
+ }
+
+ assert(mf->read_ahead >= len);
+ mf->read_ahead -= len;
+}
+
+
+static bool
+encode_init(lzma_coder *coder, lzma_mf *mf)
+{
+ if (mf->read_pos == mf->read_limit) {
+ if (mf->action == LZMA_RUN)
+ return false; // We cannot do anything.
- coder->previous_byte = coder->lz.buffer[
- coder->lz.read_pos + len - 1
- - coder->additional_offset];
+ // We are finishing (we cannot get here when flushing).
+ assert(mf->write_pos == mf->read_pos);
+ assert(mf->action == LZMA_FINISH);
+ } else {
+ // Do the actual initialization. The first LZMA symbol must
+ // always be a literal.
+ mf_skip(mf, 1);
+ mf->read_ahead = 0;
+ rc_bit(&coder->rc, &coder->is_match[0][0], 0);
+ rc_bittree(&coder->rc, coder->literal[0], 8, mf->buffer[0]);
}
- assert(coder->additional_offset >= len);
- coder->additional_offset -= len;
- coder->now_pos += len;
+ // Initialization is done (except if empty file).
+ coder->is_initialized = true;
+
+ return true;
}
static void
-encode_eopm(lzma_coder *coder)
+encode_eopm(lzma_coder *coder, uint32_t position)
{
- const uint32_t pos_state = coder->now_pos & coder->pos_mask;
+ const uint32_t pos_state = position & coder->pos_mask;
rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1);
rc_bit(&coder->rc, &coder->is_rep[coder->state], 0);
- match(coder, pos_state, UINT32_MAX, MATCH_MIN_LEN);
+ match(coder, pos_state, UINT32_MAX, MATCH_LEN_MIN);
}
-/**
- * \brief LZMA encoder
- *
- * \return true if end of stream was reached, false otherwise.
- */
-extern bool
-lzma_lzma_encode(lzma_coder *coder, uint8_t *restrict out,
- size_t *restrict out_pos, size_t out_size)
+/// Number of bytes that a single encoding loop in lzma_lzma_encode() can
+/// consume from the dictionary. This limit comes from lzma_lzma_optimum()
+/// and may need to be updated if that function is significantly modified.
+#define LOOP_INPUT_MAX (OPTS + 1)
+
+
+extern lzma_ret
+lzma_lzma_encode(lzma_coder *restrict coder, lzma_mf *restrict mf,
+ uint8_t *restrict out, size_t *restrict out_pos,
+ size_t out_size, uint32_t limit)
{
// Initialize the stream if no data has been encoded yet.
- if (!coder->is_initialized) {
- if (coder->lz.read_pos == coder->lz.read_limit) {
- if (coder->lz.sequence == SEQ_RUN)
- return false; // We cannot do anything.
-
- // We are finishing (we cannot get here when flushing).
- assert(coder->lz.write_pos == coder->lz.read_pos);
- assert(coder->lz.sequence == SEQ_FINISH);
- } else {
- // Do the actual initialization.
- uint32_t len;
- uint32_t num_distance_pairs;
- lzma_read_match_distances(coder,
- &len, &num_distance_pairs);
+ if (!coder->is_initialized && !encode_init(coder, mf))
+ return LZMA_OK;
- encode_symbol(coder, UINT32_MAX, 1);
+ // Get the lowest bits of the uncompressed offset from the LZ layer.
+ uint32_t position = mf_position(mf);
- assert(coder->additional_offset == 0);
+ while (true) {
+ // Encode pending bits, if any. Calling this before encoding
+ // the next symbol is needed only with plain LZMA, since
+ // LZMA2 always provides big enough buffer to flush
+ // everything out from the range encoder. For the same reason,
+ // rc_encode() never returns true when this function is used
+ // as part of LZMA2 encoder.
+ if (rc_encode(&coder->rc, out, out_pos, out_size)) {
+ assert(limit == UINT32_MAX);
+ return LZMA_OK;
}
- // Initialization is done (except if empty file).
- coder->is_initialized = true;
- }
-
- // Encoding loop
- while (true) {
- // Encode pending bits, if any.
- if (rc_encode(&coder->rc, out, out_pos, out_size))
- return false;
+ // With LZMA2 we need to take care that compressed size of
+ // a chunk doesn't get too big.
+ // TODO
+ if (limit != UINT32_MAX
+ && (mf->read_pos - mf->read_ahead >= limit
+ || *out_pos + rc_pending(&coder->rc)
+ >= (UINT32_C(1) << 16)
+ - LOOP_INPUT_MAX))
+ break;
// Check that there is some input to process.
- if (coder->lz.read_pos >= coder->lz.read_limit) {
- // If flushing or finishing, we must keep encoding
- // until additional_offset becomes zero to make
- // all the input available at output.
- if (coder->lz.sequence == SEQ_RUN)
- return false;
-
- if (coder->additional_offset == 0)
- break;
- }
-
- assert(coder->lz.read_pos <= coder->lz.write_pos);
+ if (mf->read_pos >= mf->read_limit) {
+ if (mf->action == LZMA_RUN)
+ return LZMA_OK;
-#ifndef NDEBUG
- if (coder->lz.sequence != SEQ_RUN) {
- assert(coder->lz.read_limit == coder->lz.write_pos);
- } else {
- assert(coder->lz.read_limit + coder->lz.keep_size_after
- == coder->lz.write_pos);
+ if (mf->read_ahead == 0)
+ break;
}
-#endif
-
- uint32_t pos;
- uint32_t len;
// Get optimal match (repeat position and length).
// Value ranges for pos:
// match at (pos - REP_DISTANCES)
// - UINT32_MAX: not a match but a literal
// Value ranges for len:
- // - [MATCH_MIN_LEN, MATCH_MAX_LEN]
- if (coder->best_compression)
- lzma_get_optimum(coder, &pos, &len);
+ // - [MATCH_LEN_MIN, MATCH_LEN_MAX]
+ uint32_t len;
+ uint32_t back;
+
+ if (coder->fast_mode)
+ lzma_lzma_optimum_fast(coder, mf, &back, &len);
else
- lzma_get_optimum_fast(coder, &pos, &len);
+ lzma_lzma_optimum_normal(
+ coder, mf, &back, &len, position);
+
+ encode_symbol(coder, mf, back, len, position);
+
+ position += len;
+ }
+
+ if (!coder->is_flushed) {
+ coder->is_flushed = true;
- encode_symbol(coder, pos, len);
+ // We don't support encoding plain LZMA streams without EOPM,
+ // and LZMA2 doesn't use EOPM at LZMA level.
+ if (limit == UINT32_MAX)
+ encode_eopm(coder, position);
+
+ // Flush the remaining bytes from the range encoder.
+ rc_flush(&coder->rc);
+
+ // Copy the remaining bytes to the output buffer. If there
+ // isn't enough output space, we will copy out the remaining
+ // bytes on the next call to this function by using
+ // the rc_encode() call in the encoding loop above.
+ if (rc_encode(&coder->rc, out, out_pos, out_size)) {
+ assert(limit == UINT32_MAX);
+ return LZMA_OK;
+ }
}
- assert(!coder->longest_match_was_found);
+ // Make it ready for the next LZMA2 chunk.
+ coder->is_flushed = false;
+
+ return LZMA_STREAM_END;
+}
+
+
+static lzma_ret
+lzma_encode(lzma_coder *restrict coder, lzma_mf *restrict mf,
+ uint8_t *restrict out, size_t *restrict out_pos,
+ size_t out_size)
+{
+ // Plain LZMA has no support for sync-flushing.
+ if (unlikely(mf->action == LZMA_SYNC_FLUSH))
+ return LZMA_HEADER_ERROR;
+
+ return lzma_lzma_encode(coder, mf, out, out_pos, out_size, UINT32_MAX);
+}
+
- if (coder->is_flushed) {
- coder->is_flushed = false;
+////////////////////
+// Initialization //
+////////////////////
+
+static bool
+set_lz_options(lzma_lz_options *lz_options, const lzma_options_lzma *options)
+{
+ if (!is_lclppb_valid(options)
+ || options->fast_bytes < LZMA_FAST_BYTES_MIN
+ || options->fast_bytes > LZMA_FAST_BYTES_MAX)
return true;
+
+ // FIXME validation
+
+ lz_options->before_size = OPTS;
+ lz_options->dictionary_size = options->dictionary_size;
+ lz_options->after_size = LOOP_INPUT_MAX;
+ lz_options->match_len_max = MATCH_LEN_MAX;
+ lz_options->find_len_max = options->fast_bytes;
+ lz_options->match_finder = options->match_finder;
+ lz_options->match_finder_cycles = options->match_finder_cycles;
+ lz_options->preset_dictionary = options->preset_dictionary;
+ lz_options->preset_dictionary_size = options->preset_dictionary_size;
+
+ return false;
+}
+
+
+static void
+length_encoder_reset(lzma_length_encoder *lencoder,
+ const uint32_t num_pos_states, const bool fast_mode)
+{
+ bit_reset(lencoder->choice);
+ bit_reset(lencoder->choice2);
+
+ for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) {
+ bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS);
+ bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS);
}
- // We don't support encoding old LZMA streams without EOPM, and LZMA2
- // doesn't use EOPM at LZMA level.
- if (coder->write_eopm)
- encode_eopm(coder);
+ bittree_reset(lencoder->high, LEN_HIGH_BITS);
- rc_flush(&coder->rc);
+ if (!fast_mode)
+ for (size_t pos_state = 0; pos_state < num_pos_states;
+ ++pos_state)
+ length_update_prices(lencoder, pos_state);
- if (rc_encode(&coder->rc, out, out_pos, out_size)) {
- coder->is_flushed = true;
- return false;
+ return;
+}
+
+
+extern void
+lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options)
+{
+ assert(!coder->is_flushed);
+
+ coder->pos_mask = (1U << options->pos_bits) - 1;
+ coder->literal_context_bits = options->literal_context_bits;
+ coder->literal_pos_mask = (1 << options->literal_pos_bits) - 1;
+
+
+ // Range coder
+ rc_reset(&coder->rc);
+
+ // State
+ coder->state = 0;
+ for (size_t i = 0; i < REP_DISTANCES; ++i)
+ coder->reps[i] = 0;
+
+ literal_init(coder->literal, options->literal_context_bits,
+ options->literal_pos_bits);
+
+ // Bit encoders
+ for (size_t i = 0; i < STATES; ++i) {
+ for (size_t j = 0; j <= coder->pos_mask; ++j) {
+ bit_reset(coder->is_match[i][j]);
+ bit_reset(coder->is_rep0_long[i][j]);
+ }
+
+ bit_reset(coder->is_rep[i]);
+ bit_reset(coder->is_rep0[i]);
+ bit_reset(coder->is_rep1[i]);
+ bit_reset(coder->is_rep2[i]);
}
- return true;
+ for (size_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i)
+ bit_reset(coder->pos_special[i]);
+
+ // Bit tree encoders
+ for (size_t i = 0; i < LEN_TO_POS_STATES; ++i)
+ bittree_reset(coder->pos_slot[i], POS_SLOT_BITS);
+
+ bittree_reset(coder->pos_align, ALIGN_BITS);
+
+ // Length encoders
+ length_encoder_reset(&coder->match_len_encoder,
+ 1U << options->pos_bits, coder->fast_mode);
+
+ length_encoder_reset(&coder->rep_len_encoder,
+ 1U << options->pos_bits, coder->fast_mode);
+
+ // FIXME: Too big or too small won't work when resetting in the middle of LZMA2.
+ coder->match_price_count = UINT32_MAX / 2;
+ coder->align_price_count = UINT32_MAX / 2;
+
+ coder->opts_end_index = 0;
+ coder->opts_current_index = 0;
+}
+
+
+extern lzma_ret
+lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator,
+ const lzma_options_lzma *options, lzma_lz_options *lz_options)
+{
+ if (*coder_ptr == NULL) {
+ *coder_ptr = lzma_alloc(sizeof(lzma_coder), allocator);
+ if (*coder_ptr == NULL)
+ return LZMA_MEM_ERROR;
+ }
+
+ lzma_coder *coder = *coder_ptr;
+
+ // Validate options that aren't validated elsewhere.
+ if (!is_lclppb_valid(options)
+ || options->fast_bytes < LZMA_FAST_BYTES_MIN
+ || options->fast_bytes > LZMA_FAST_BYTES_MAX)
+ return LZMA_HEADER_ERROR;
+
+ // Set compression mode.
+ switch (options->mode) {
+ case LZMA_MODE_FAST:
+ coder->fast_mode = true;
+ break;
+
+ case LZMA_MODE_NORMAL: {
+ coder->fast_mode = false;
+
+ // Set dist_table_size.
+ // Round the dictionary size up to next 2^n.
+ uint32_t log_size = 0;
+ while ((UINT32_C(1) << log_size)
+ < options->dictionary_size)
+ ++log_size;
+
+ coder->dist_table_size = log_size * 2;
+
+ // Length encoders' price table size
+ coder->match_len_encoder.table_size
+ = options->fast_bytes + 1 - MATCH_LEN_MIN;
+ coder->rep_len_encoder.table_size
+ = options->fast_bytes + 1 - MATCH_LEN_MIN;
+ break;
+ }
+
+ default:
+ return LZMA_HEADER_ERROR;
+ }
+
+ coder->is_initialized = false;
+ coder->is_flushed = false;
+
+ lzma_lzma_encoder_reset(coder, options);
+
+ // LZ encoder options FIXME validation
+ if (set_lz_options(lz_options, options))
+ return LZMA_HEADER_ERROR;
+
+ return LZMA_OK;
+}
+
+
+static lzma_ret
+lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator,
+ const void *options, lzma_lz_options *lz_options)
+{
+ lz->code = &lzma_encode;
+ return lzma_lzma_encoder_create(
+ &lz->coder, allocator, options, lz_options);
+}
+
+
+extern lzma_ret
+lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ const lzma_filter_info *filters)
+{
+ // Initialization call chain:
+ //
+ // lzma_lzma_encoder_init()
+ // `-- lzma_lz_encoder_init()
+ // `-- lzma_encoder_init()
+ // `-- lzma_encoder_init2()
+ //
+ // The above complexity is to let LZ encoder store the pointer to
+ // the LZMA encoder structure. Encoding call tree:
+ //
+ // lz_encode()
+ // |-- fill_window()
+ // | `-- Next coder in the chain, if any
+ // `-- lzma_encode()
+ // |-- lzma_dict_find()
+ // `-- lzma_dict_skip()
+ //
+ // FIXME ^
+ //
+ return lzma_lz_encoder_init(
+ next, allocator, filters, &lzma_encoder_init);
+}
+
+
+extern uint64_t
+lzma_lzma_encoder_memusage(const void *options)
+{
+ lzma_lz_options lz_options;
+ if (set_lz_options(&lz_options, options))
+ return UINT64_MAX;
+
+ const uint64_t lz_memusage = lzma_lz_encoder_memusage(&lz_options);
+ if (lz_memusage == UINT64_MAX)
+ return UINT64_MAX;
+
+ return (uint64_t)(sizeof(lzma_coder)) + lz_memusage;
+}
+
+
+extern bool
+lzma_lzma_lclppb_encode(const lzma_options_lzma *options, uint8_t *byte)
+{
+ if (options->literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX
+ || options->literal_pos_bits
+ > LZMA_LITERAL_POS_BITS_MAX
+ || options->pos_bits > LZMA_POS_BITS_MAX
+ || options->literal_context_bits
+ + options->literal_pos_bits
+ > LZMA_LITERAL_BITS_MAX)
+ return true;
+
+ *byte = (options->pos_bits * 5 + options->literal_pos_bits) * 9
+ + options->literal_context_bits;
+ assert(*byte <= (4 * 5 + 4) * 9 + 8);
+
+ return false;
+}
+
+
+#ifdef HAVE_ENCODER_LZMA
+extern lzma_ret
+lzma_lzma_props_encode(const void *options, uint8_t *out)
+{
+ const lzma_options_lzma *const opt = options;
+
+ if (lzma_lzma_lclppb_encode(opt, out))
+ return LZMA_PROG_ERROR;
+
+ integer_write_32(out + 1, opt->dictionary_size);
+
+ return LZMA_OK;
+}
+#endif
+
+
+extern LZMA_API lzma_bool
+lzma_mode_is_available(lzma_mode mode)
+{
+ return mode == LZMA_MODE_FAST || mode == LZMA_MODE_NORMAL;
}
///////////////////////////////////////////////////////////////////////////////
//
/// \file lzma_encoder.h
-/// \brief LZMA method handler API
+/// \brief LZMA encoder API
//
// Copyright (C) 1999-2006 Igor Pavlov
// Copyright (C) 2007 Lasse Collin
#include "common.h"
+
extern lzma_ret lzma_lzma_encoder_init(lzma_next_coder *next,
lzma_allocator *allocator, const lzma_filter_info *filters);
-extern bool lzma_lzma_encode_properties(
+
+extern uint64_t lzma_lzma_encoder_memusage(const void *options);
+
+extern lzma_ret lzma_lzma_props_encode(const void *options, uint8_t *out);
+
+
+/// Encodes lc/lp/pb into one byte. Returns false on success and true on error.
+extern bool lzma_lzma_lclppb_encode(
const lzma_options_lzma *options, uint8_t *byte);
+
+#ifdef HAVE_SMALL
+
/// Initializes the lzma_fastpos[] array.
extern void lzma_fastpos_init(void);
#endif
+
+
+#ifdef LZMA_LZ_ENCODER_H
+
+/// Initializes raw LZMA encoder; this is used by LZMA2.
+extern lzma_ret lzma_lzma_encoder_create(
+ lzma_coder **coder_ptr, lzma_allocator *allocator,
+ const lzma_options_lzma *options, lzma_lz_options *lz_options);
+
+
+/// Resets an already initialized LZMA encoder; this is used by LZMA2.
+extern void lzma_lzma_encoder_reset(
+ lzma_coder *coder, const lzma_options_lzma *options);
+
+
+extern lzma_ret lzma_lzma_encode(lzma_coder *restrict coder,
+ lzma_mf *restrict mf, uint8_t *restrict out,
+ size_t *restrict out_pos, size_t out_size,
+ uint32_t read_limit);
+
+#endif
+
+#endif
static lzma_mode modes[] = {
LZMA_MODE_FAST,
- LZMA_MODE_BEST,
+ LZMA_MODE_NORMAL,
LZMA_MODE_INVALID
};
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file lzma_encoder_getoptimum.c
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-// NOTE: If you want to keep the line length in 80 characters, set
-// tab width to 4 or less in your editor when editing this file.
-
-
-// "Would you love the monster code?
-// Could you understand beauty of the beast?"
-// --Adapted from Lordi's "Would you love a monster man".
-
-
-#include "lzma_encoder_private.h"
-#include "fastpos.h"
-
-
-#define length_get_price(length_encoder, symbol, pos_state) \
- (length_encoder).prices[pos_state][symbol]
-
-
-#define get_rep_len_1_price(state, pos_state) \
- bit_get_price_0(coder->is_rep0[state]) \
- + bit_get_price_0(coder->is_rep0_long[state][pos_state])
-
-
-// Adds to price_target.
-#define get_pure_rep_price(price_target, rep_index, state, pos_state) \
-do { \
- if ((rep_index) == 0) { \
- price_target += bit_get_price_0(coder->is_rep0[state]); \
- price_target += bit_get_price_1( \
- coder->is_rep0_long[state][pos_state]); \
- } else { \
- price_target += bit_get_price_1(coder->is_rep0[state]); \
- if ((rep_index) == 1) { \
- price_target += bit_get_price_0(coder->is_rep1[state]); \
- } else { \
- price_target += bit_get_price_1(coder->is_rep1[state]); \
- price_target += bit_get_price( \
- coder->is_rep2[state], (rep_index) - 2); \
- } \
- } \
-} while (0)
-
-
-// Adds to price_target.
-#define get_rep_price(price_target, rep_index, len, state, pos_state) \
-do { \
- get_pure_rep_price(price_target, rep_index, state, pos_state); \
- price_target += length_get_price(coder->rep_len_encoder, \
- (len) - MATCH_MIN_LEN, pos_state); \
-} while (0)
-
-
-// Adds to price_target.
-#define get_pos_len_price(price_target, pos, len, pos_state) \
-do { \
- const uint32_t len_to_pos_state_tmp = get_len_to_pos_state(len); \
- if ((pos) < FULL_DISTANCES) { \
- price_target += distances_prices[len_to_pos_state_tmp][pos]; \
- } else { \
- price_target \
- += pos_slot_prices[len_to_pos_state_tmp][get_pos_slot_2(pos)] \
- + align_prices[(pos) & ALIGN_MASK]; \
- } \
- price_target += length_get_price( \
- coder->match_len_encoder, (len) - MATCH_MIN_LEN, pos_state); \
-} while (0)
-
-
-// Three macros to manipulate lzma_optimal structures:
-#define make_as_char(opt) \
-do { \
- (opt).back_prev = UINT32_MAX; \
- (opt).prev_1_is_char = false; \
-} while (0)
-
-
-#define make_as_short_rep(opt) \
-do { \
- (opt).back_prev = 0; \
- (opt).prev_1_is_char = false; \
-} while (0)
-
-
-#define is_short_rep(opt) \
- ((opt).back_prev == 0)
-
-
-static void
-fill_length_prices(lzma_length_encoder *lc, uint32_t pos_state)
-{
- const uint32_t num_symbols = lc->table_size;
- const uint32_t a0 = bit_get_price_0(lc->choice);
- const uint32_t a1 = bit_get_price_1(lc->choice);
- const uint32_t b0 = a1 + bit_get_price_0(lc->choice2);
- const uint32_t b1 = a1 + bit_get_price_1(lc->choice2);
-
- uint32_t *prices = lc->prices[pos_state];
- uint32_t i = 0;
-
- for (i = 0; i < num_symbols && i < LEN_LOW_SYMBOLS; ++i)
- prices[i] = a0 + bittree_get_price(lc->low[pos_state],
- LEN_LOW_BITS, i);
-
- for (; i < num_symbols && i < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; ++i)
- prices[i] = b0 + bittree_get_price(lc->mid[pos_state],
- LEN_MID_BITS, i - LEN_LOW_SYMBOLS);
-
- for (; i < num_symbols; ++i)
- prices[i] = b1 + bittree_get_price(lc->high, LEN_HIGH_BITS,
- i - LEN_LOW_SYMBOLS - LEN_MID_SYMBOLS);
-
- lc->counters[pos_state] = num_symbols;
-
- return;
-}
-
-
-static void
-fill_distances_prices(lzma_coder *coder)
-{
- uint32_t temp_prices[FULL_DISTANCES];
-
- for (uint32_t i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) {
- const uint32_t pos_slot = get_pos_slot(i);
- const uint32_t footer_bits = ((pos_slot >> 1) - 1);
- const uint32_t base = (2 | (pos_slot & 1)) << footer_bits;
- temp_prices[i] = bittree_reverse_get_price(
- coder->pos_encoders + base - pos_slot - 1,
- footer_bits, i - base);
- }
-
- const uint32_t dist_table_size = coder->dist_table_size;
-
- for (uint32_t len_to_pos_state = 0;
- len_to_pos_state < LEN_TO_POS_STATES;
- ++len_to_pos_state) {
-
- const probability *encoder = coder->pos_slot_encoder[len_to_pos_state];
- uint32_t *pos_slot_prices = coder->pos_slot_prices[len_to_pos_state];
-
- for (uint32_t pos_slot = 0;
- pos_slot < dist_table_size;
- ++pos_slot) {
- pos_slot_prices[pos_slot] = bittree_get_price(encoder,
- POS_SLOT_BITS, pos_slot);
- }
-
- for (uint32_t pos_slot = END_POS_MODEL_INDEX;
- pos_slot < dist_table_size;
- ++pos_slot)
- pos_slot_prices[pos_slot] += (((pos_slot >> 1) - 1)
- - ALIGN_BITS) << BIT_PRICE_SHIFT_BITS;
-
-
- uint32_t *distances_prices
- = coder->distances_prices[len_to_pos_state];
-
- uint32_t i;
- for (i = 0; i < START_POS_MODEL_INDEX; ++i)
- distances_prices[i] = pos_slot_prices[i];
-
- for (; i < FULL_DISTANCES; ++i)
- distances_prices[i] = pos_slot_prices[get_pos_slot(i)]
- + temp_prices[i];
- }
-
- coder->match_price_count = 0;
-
- return;
-}
-
-
-static void
-fill_align_prices(lzma_coder *coder)
-{
- for (uint32_t i = 0; i < ALIGN_TABLE_SIZE; ++i)
- coder->align_prices[i] = bittree_reverse_get_price(
- coder->pos_align_encoder, ALIGN_BITS, i);
-
- coder->align_price_count = 0;
- return;
-}
-
-
-// The first argument is a pointer returned by literal_get_subcoder().
-static uint32_t
-literal_get_price(const probability *encoders, const bool match_mode,
- const uint8_t match_byte, const uint8_t symbol)
-{
- uint32_t price = 0;
- uint32_t context = 1;
- int i = 8;
-
- if (match_mode) {
- do {
- --i;
- const uint32_t match_bit = (match_byte >> i) & 1;
- const uint32_t bit = (symbol >> i) & 1;
- const uint32_t subcoder_index
- = 0x100 + (match_bit << 8) + context;
-
- price += bit_get_price(encoders[subcoder_index], bit);
- context = (context << 1) | bit;
-
- if (match_bit != bit)
- break;
-
- } while (i != 0);
- }
-
- while (i != 0) {
- --i;
- const uint32_t bit = (symbol >> i) & 1;
- price += bit_get_price(encoders[context], bit);
- context = (context << 1) | bit;
- }
-
- return price;
-}
-
-
-static void
-backward(lzma_coder *restrict coder, uint32_t *restrict len_res,
- uint32_t *restrict back_res, uint32_t cur)
-{
- coder->optimum_end_index = cur;
-
- uint32_t pos_mem = coder->optimum[cur].pos_prev;
- uint32_t back_mem = coder->optimum[cur].back_prev;
-
- do {
- if (coder->optimum[cur].prev_1_is_char) {
- make_as_char(coder->optimum[pos_mem]);
- coder->optimum[pos_mem].pos_prev = pos_mem - 1;
-
- if (coder->optimum[cur].prev_2) {
- coder->optimum[pos_mem - 1].prev_1_is_char = false;
- coder->optimum[pos_mem - 1].pos_prev
- = coder->optimum[cur].pos_prev_2;
- coder->optimum[pos_mem - 1].back_prev
- = coder->optimum[cur].back_prev_2;
- }
- }
-
- uint32_t pos_prev = pos_mem;
- uint32_t back_cur = back_mem;
-
- back_mem = coder->optimum[pos_prev].back_prev;
- pos_mem = coder->optimum[pos_prev].pos_prev;
-
- coder->optimum[pos_prev].back_prev = back_cur;
- coder->optimum[pos_prev].pos_prev = cur;
- cur = pos_prev;
-
- } while (cur != 0);
-
- coder->optimum_current_index = coder->optimum[0].pos_prev;
- *len_res = coder->optimum[0].pos_prev;
- *back_res = coder->optimum[0].back_prev;
-
- return;
-}
-
-
-extern void
-lzma_get_optimum(lzma_coder *restrict coder,
- uint32_t *restrict back_res, uint32_t *restrict len_res)
-{
- uint32_t position = coder->now_pos;
- uint32_t pos_state = position & coder->pos_mask;
-
- // Update the price tables. In the C++ LZMA SDK 4.42 this was done in both
- // initialization function and in the main loop. In liblzma they were
- // moved into this single place.
- if (coder->additional_offset == 0) {
- if (coder->match_price_count >= (1 << 7))
- fill_distances_prices(coder);
-
- if (coder->align_price_count >= ALIGN_TABLE_SIZE)
- fill_align_prices(coder);
- }
-
- if (coder->prev_len_encoder != NULL) {
- if (--coder->prev_len_encoder->counters[pos_state] == 0)
- fill_length_prices(coder->prev_len_encoder, pos_state);
-
- coder->prev_len_encoder = NULL;
- }
-
-
- if (coder->optimum_end_index != coder->optimum_current_index) {
- *len_res = coder->optimum[coder->optimum_current_index].pos_prev
- - coder->optimum_current_index;
- *back_res = coder->optimum[coder->optimum_current_index].back_prev;
- coder->optimum_current_index = coder->optimum[
- coder->optimum_current_index].pos_prev;
- return;
- }
-
- coder->optimum_current_index = 0;
- coder->optimum_end_index = 0;
-
-
- const uint32_t fast_bytes = coder->fast_bytes;
- uint32_t *match_distances = coder->match_distances;
-
- uint32_t len_main;
- uint32_t num_distance_pairs;
-
- if (!coder->longest_match_was_found) {
- lzma_read_match_distances(coder, &len_main, &num_distance_pairs);
- } else {
- len_main = coder->longest_match_length;
- num_distance_pairs = coder->num_distance_pairs;
- coder->longest_match_was_found = false;
- }
-
-
- const uint8_t *buf = coder->lz.buffer + coder->lz.read_pos - 1;
- uint32_t num_available_bytes
- = coder->lz.write_pos - coder->lz.read_pos + 1;
- if (num_available_bytes < 2) {
- *back_res = UINT32_MAX;
- *len_res = 1;
- return;
- }
-
- if (num_available_bytes > MATCH_MAX_LEN)
- num_available_bytes = MATCH_MAX_LEN;
-
-
- uint32_t reps[REP_DISTANCES];
- uint32_t rep_lens[REP_DISTANCES];
- uint32_t rep_max_index = 0;
-
- for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
- reps[i] = coder->reps[i];
- const uint32_t back_offset = reps[i] + 1;
-
- if (buf[0] != *(buf - back_offset)
- || buf[1] != *(buf + 1 - back_offset)) {
- rep_lens[i] = 0;
- continue;
- }
-
- uint32_t len_test;
- for (len_test = 2; len_test < num_available_bytes
- && buf[len_test] == *(buf + len_test - back_offset);
- ++len_test) ;
-
- rep_lens[i] = len_test;
- if (len_test > rep_lens[rep_max_index])
- rep_max_index = i;
- }
-
- if (rep_lens[rep_max_index] >= fast_bytes) {
- *back_res = rep_max_index;
- *len_res = rep_lens[rep_max_index];
- move_pos(*len_res - 1);
- return;
- }
-
-
- if (len_main >= fast_bytes) {
- *back_res = match_distances[num_distance_pairs] + REP_DISTANCES;
- *len_res = len_main;
- move_pos(len_main - 1);
- return;
- }
-
- uint8_t current_byte = *buf;
- uint8_t match_byte = *(buf - reps[0] - 1);
-
- if (len_main < 2 && current_byte != match_byte
- && rep_lens[rep_max_index] < 2) {
- *back_res = UINT32_MAX;
- *len_res = 1;
- return;
- }
-
- coder->optimum[0].state = coder->state;
-
- coder->optimum[1].price = bit_get_price_0(
- coder->is_match[coder->state][pos_state])
- + literal_get_price(
- literal_get_subcoder(coder->literal_coder,
- position, coder->previous_byte),
- !is_literal_state(coder->state), match_byte, current_byte);
-
- make_as_char(coder->optimum[1]);
-
- uint32_t match_price
- = bit_get_price_1(coder->is_match[coder->state][pos_state]);
- uint32_t rep_match_price
- = match_price + bit_get_price_1(coder->is_rep[coder->state]);
-
-
- if (match_byte == current_byte) {
- const uint32_t short_rep_price = rep_match_price
- + get_rep_len_1_price(coder->state, pos_state);
-
- if (short_rep_price < coder->optimum[1].price) {
- coder->optimum[1].price = short_rep_price;
- make_as_short_rep(coder->optimum[1]);
- }
- }
-
- uint32_t len_end = (len_main >= rep_lens[rep_max_index])
- ? len_main
- : rep_lens[rep_max_index];
-
- if (len_end < 2) {
- *back_res = coder->optimum[1].back_prev;
- *len_res = 1;
- return;
- }
-
- coder->optimum[1].pos_prev = 0;
-
- for (uint32_t i = 0; i < REP_DISTANCES; ++i)
- coder->optimum[0].backs[i] = reps[i];
-
- uint32_t len = len_end;
- do {
- coder->optimum[len].price = INFINITY_PRICE;
- } while (--len >= 2);
-
-
- uint32_t (*distances_prices)[FULL_DISTANCES] = coder->distances_prices;
- uint32_t (*pos_slot_prices)[DIST_TABLE_SIZE_MAX] = coder->pos_slot_prices;
- uint32_t *align_prices = coder->align_prices;
-
- for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
- uint32_t rep_len = rep_lens[i];
- if (rep_len < 2)
- continue;
-
- uint32_t price = rep_match_price;
- get_pure_rep_price(price, i, coder->state, pos_state);
-
- do {
- const uint32_t cur_and_len_price = price
- + length_get_price(
- coder->rep_len_encoder,
- rep_len - 2, pos_state);
-
- if (cur_and_len_price < coder->optimum[rep_len].price) {
- coder->optimum[rep_len].price = cur_and_len_price;
- coder->optimum[rep_len].pos_prev = 0;
- coder->optimum[rep_len].back_prev = i;
- coder->optimum[rep_len].prev_1_is_char = false;
- }
- } while (--rep_len >= 2);
- }
-
-
- uint32_t normal_match_price = match_price
- + bit_get_price_0(coder->is_rep[coder->state]);
-
- len = (rep_lens[0] >= 2) ? rep_lens[0] + 1 : 2;
-
- if (len <= len_main) {
- uint32_t offs = 0;
-
- while (len > match_distances[offs + 1])
- offs += 2;
-
- for(; ; ++len) {
- const uint32_t distance = match_distances[offs + 2];
- uint32_t cur_and_len_price = normal_match_price;
- get_pos_len_price(cur_and_len_price, distance, len, pos_state);
-
- if (cur_and_len_price < coder->optimum[len].price) {
- coder->optimum[len].price = cur_and_len_price;
- coder->optimum[len].pos_prev = 0;
- coder->optimum[len].back_prev = distance + REP_DISTANCES;
- coder->optimum[len].prev_1_is_char = false;
- }
-
- if (len == match_distances[offs + 1]) {
- offs += 2;
- if (offs == num_distance_pairs)
- break;
- }
- }
- }
-
-
- //////////////////
- // Big loop ;-) //
- //////////////////
-
- uint32_t cur = 0;
-
- // The rest of this function is a huge while-loop. To avoid extreme
- // indentation, the indentation level is not increased here.
- while (true) {
-
- ++cur;
-
- assert(cur < OPTS);
-
- if (cur == len_end) {
- backward(coder, len_res, back_res, cur);
- return;
- }
-
- uint32_t new_len;
-
- lzma_read_match_distances(coder, &new_len, &num_distance_pairs);
-
- if (new_len >= fast_bytes) {
- coder->num_distance_pairs = num_distance_pairs;
- coder->longest_match_length = new_len;
- coder->longest_match_was_found = true;
- backward(coder, len_res, back_res, cur);
- return;
- }
-
-
- ++position;
-
- uint32_t pos_prev = coder->optimum[cur].pos_prev;
- uint32_t state;
-
- if (coder->optimum[cur].prev_1_is_char) {
- --pos_prev;
-
- if (coder->optimum[cur].prev_2) {
- state = coder->optimum[coder->optimum[cur].pos_prev_2].state;
-
- if (coder->optimum[cur].back_prev_2 < REP_DISTANCES)
- update_long_rep(state);
- else
- update_match(state);
-
- } else {
- state = coder->optimum[pos_prev].state;
- }
-
- update_literal(state);
-
- } else {
- state = coder->optimum[pos_prev].state;
- }
-
- if (pos_prev == cur - 1) {
- if (is_short_rep(coder->optimum[cur]))
- update_short_rep(state);
- else
- update_literal(state);
- } else {
- uint32_t pos;
- if (coder->optimum[cur].prev_1_is_char && coder->optimum[cur].prev_2) {
- pos_prev = coder->optimum[cur].pos_prev_2;
- pos = coder->optimum[cur].back_prev_2;
- update_long_rep(state);
- } else {
- pos = coder->optimum[cur].back_prev;
- if (pos < REP_DISTANCES)
- update_long_rep(state);
- else
- update_match(state);
- }
-
- if (pos < REP_DISTANCES) {
- reps[0] = coder->optimum[pos_prev].backs[pos];
-
- uint32_t i;
- for (i = 1; i <= pos; ++i)
- reps[i] = coder->optimum[pos_prev].backs[i - 1];
-
- for (; i < REP_DISTANCES; ++i)
- reps[i] = coder->optimum[pos_prev].backs[i];
-
- } else {
- reps[0] = pos - REP_DISTANCES;
-
- for (uint32_t i = 1; i < REP_DISTANCES; ++i)
- reps[i] = coder->optimum[pos_prev].backs[i - 1];
- }
- }
-
- coder->optimum[cur].state = state;
-
- for (uint32_t i = 0; i < REP_DISTANCES; ++i)
- coder->optimum[cur].backs[i] = reps[i];
-
- const uint32_t cur_price = coder->optimum[cur].price;
-
- buf = coder->lz.buffer + coder->lz.read_pos - 1;
- current_byte = *buf;
- match_byte = *(buf - reps[0] - 1);
-
- pos_state = position & coder->pos_mask;
-
- const uint32_t cur_and_1_price = cur_price
- + bit_get_price_0(coder->is_match[state][pos_state])
- + literal_get_price(
- literal_get_subcoder(coder->literal_coder,
- position, buf[-1]),
- !is_literal_state(state), match_byte, current_byte);
-
- bool next_is_char = false;
-
- if (cur_and_1_price < coder->optimum[cur + 1].price) {
- coder->optimum[cur + 1].price = cur_and_1_price;
- coder->optimum[cur + 1].pos_prev = cur;
- make_as_char(coder->optimum[cur + 1]);
- next_is_char = true;
- }
-
- match_price = cur_price
- + bit_get_price_1(coder->is_match[state][pos_state]);
- rep_match_price = match_price
- + bit_get_price_1(coder->is_rep[state]);
-
- if (match_byte == current_byte
- && !(coder->optimum[cur + 1].pos_prev < cur
- && coder->optimum[cur + 1].back_prev == 0)) {
-
- const uint32_t short_rep_price = rep_match_price
- + get_rep_len_1_price(state, pos_state);
-
- if (short_rep_price <= coder->optimum[cur + 1].price) {
- coder->optimum[cur + 1].price = short_rep_price;
- coder->optimum[cur + 1].pos_prev = cur;
- make_as_short_rep(coder->optimum[cur + 1]);
- next_is_char = true;
- }
- }
-
- uint32_t num_available_bytes_full
- = coder->lz.write_pos - coder->lz.read_pos + 1;
- num_available_bytes_full = MIN(OPTS - 1 - cur, num_available_bytes_full);
- num_available_bytes = num_available_bytes_full;
-
- if (num_available_bytes < 2)
- continue;
-
- if (num_available_bytes > fast_bytes)
- num_available_bytes = fast_bytes;
-
- if (!next_is_char && match_byte != current_byte) { // speed optimization
- // try literal + rep0
- const uint32_t back_offset = reps[0] + 1;
- const uint32_t limit = MIN(num_available_bytes_full, fast_bytes + 1);
-
- uint32_t temp;
- for (temp = 1; temp < limit
- && buf[temp] == *(buf + temp - back_offset);
- ++temp) ;
-
- const uint32_t len_test_2 = temp - 1;
-
- if (len_test_2 >= 2) {
- uint32_t state_2 = state;
- update_literal(state_2);
-
- const uint32_t pos_state_next = (position + 1) & coder->pos_mask;
- const uint32_t next_rep_match_price = cur_and_1_price
- + bit_get_price_1(coder->is_match[state_2][pos_state_next])
- + bit_get_price_1(coder->is_rep[state_2]);
-
- // for (; len_test_2 >= 2; --len_test_2) {
- const uint32_t offset = cur + 1 + len_test_2;
-
- while (len_end < offset)
- coder->optimum[++len_end].price = INFINITY_PRICE;
-
- uint32_t cur_and_len_price = next_rep_match_price;
- get_rep_price(cur_and_len_price,
- 0, len_test_2, state_2, pos_state_next);
-
- if (cur_and_len_price < coder->optimum[offset].price) {
- coder->optimum[offset].price = cur_and_len_price;
- coder->optimum[offset].pos_prev = cur + 1;
- coder->optimum[offset].back_prev = 0;
- coder->optimum[offset].prev_1_is_char = true;
- coder->optimum[offset].prev_2 = false;
- }
-// }
- }
- }
-
-
- uint32_t start_len = 2; // speed optimization
-
- for (uint32_t rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) {
- const uint32_t back_offset = reps[rep_index] + 1;
-
- if (buf[0] != *(buf - back_offset) || buf[1] != *(buf + 1 - back_offset))
- continue;
-
- uint32_t len_test;
- for (len_test = 2; len_test < num_available_bytes
- && buf[len_test] == *(buf + len_test - back_offset);
- ++len_test) ;
-
- while (len_end < cur + len_test)
- coder->optimum[++len_end].price = INFINITY_PRICE;
-
- const uint32_t len_test_temp = len_test;
- uint32_t price = rep_match_price;
- get_pure_rep_price(price, rep_index, state, pos_state);
-
- do {
- const uint32_t cur_and_len_price = price
- + length_get_price(coder->rep_len_encoder,
- len_test - 2, pos_state);
-
- if (cur_and_len_price < coder->optimum[cur + len_test].price) {
- coder->optimum[cur + len_test].price = cur_and_len_price;
- coder->optimum[cur + len_test].pos_prev = cur;
- coder->optimum[cur + len_test].back_prev = rep_index;
- coder->optimum[cur + len_test].prev_1_is_char = false;
- }
- } while (--len_test >= 2);
-
- len_test = len_test_temp;
-
- if (rep_index == 0)
- start_len = len_test + 1;
-
-
- uint32_t len_test_2 = len_test + 1;
- const uint32_t limit = MIN(num_available_bytes_full,
- len_test_2 + fast_bytes);
- for (; len_test_2 < limit
- && buf[len_test_2] == *(buf + len_test_2 - back_offset);
- ++len_test_2) ;
-
- len_test_2 -= len_test + 1;
-
- if (len_test_2 >= 2) {
- uint32_t state_2 = state;
- update_long_rep(state_2);
-
- uint32_t pos_state_next = (position + len_test) & coder->pos_mask;
-
- const uint32_t cur_and_len_char_price = price
- + length_get_price(coder->rep_len_encoder,
- len_test - 2, pos_state)
- + bit_get_price_0(coder->is_match[state_2][pos_state_next])
- + literal_get_price(
- literal_get_subcoder(coder->literal_coder,
- position + len_test, buf[len_test - 1]),
- true, *(buf + len_test - back_offset), buf[len_test]);
-
- update_literal(state_2);
-
- pos_state_next = (position + len_test + 1) & coder->pos_mask;
-
- const uint32_t next_rep_match_price = cur_and_len_char_price
- + bit_get_price_1(coder->is_match[state_2][pos_state_next])
- + bit_get_price_1(coder->is_rep[state_2]);
-
-// for(; len_test_2 >= 2; len_test_2--) {
- const uint32_t offset = cur + len_test + 1 + len_test_2;
-
- while (len_end < offset)
- coder->optimum[++len_end].price = INFINITY_PRICE;
-
- uint32_t cur_and_len_price = next_rep_match_price;
- get_rep_price(cur_and_len_price,
- 0, len_test_2, state_2, pos_state_next);
-
- if (cur_and_len_price < coder->optimum[offset].price) {
- coder->optimum[offset].price = cur_and_len_price;
- coder->optimum[offset].pos_prev = cur + len_test + 1;
- coder->optimum[offset].back_prev = 0;
- coder->optimum[offset].prev_1_is_char = true;
- coder->optimum[offset].prev_2 = true;
- coder->optimum[offset].pos_prev_2 = cur;
- coder->optimum[offset].back_prev_2 = rep_index;
- }
-// }
- }
- }
-
-
-// for (uint32_t len_test = 2; len_test <= new_len; ++len_test)
- if (new_len > num_available_bytes) {
- new_len = num_available_bytes;
-
- for (num_distance_pairs = 0;
- new_len > match_distances[num_distance_pairs + 1];
- num_distance_pairs += 2) ;
-
- match_distances[num_distance_pairs + 1] = new_len;
- num_distance_pairs += 2;
- }
-
-
- if (new_len >= start_len) {
- normal_match_price = match_price
- + bit_get_price_0(coder->is_rep[state]);
-
- while (len_end < cur + new_len)
- coder->optimum[++len_end].price = INFINITY_PRICE;
-
- uint32_t offs = 0;
- while (start_len > match_distances[offs + 1])
- offs += 2;
-
- uint32_t cur_back = match_distances[offs + 2];
- uint32_t pos_slot = get_pos_slot_2(cur_back);
-
- for (uint32_t len_test = start_len; ; ++len_test) {
- uint32_t cur_and_len_price = normal_match_price;
- const uint32_t len_to_pos_state = get_len_to_pos_state(len_test);
-
- if (cur_back < FULL_DISTANCES)
- cur_and_len_price += distances_prices[
- len_to_pos_state][cur_back];
- else
- cur_and_len_price += pos_slot_prices[
- len_to_pos_state][pos_slot]
- + align_prices[cur_back & ALIGN_MASK];
-
- cur_and_len_price += length_get_price(coder->match_len_encoder,
- len_test - MATCH_MIN_LEN, pos_state);
-
- if (cur_and_len_price < coder->optimum[cur + len_test].price) {
- coder->optimum[cur + len_test].price = cur_and_len_price;
- coder->optimum[cur + len_test].pos_prev = cur;
- coder->optimum[cur + len_test].back_prev
- = cur_back + REP_DISTANCES;
- coder->optimum[cur + len_test].prev_1_is_char = false;
- }
-
- if (len_test == match_distances[offs + 1]) {
- // Try Match + Literal + Rep0
- const uint32_t back_offset = cur_back + 1;
- uint32_t len_test_2 = len_test + 1;
- const uint32_t limit = MIN(num_available_bytes_full,
- len_test_2 + fast_bytes);
-
- for (; len_test_2 < limit &&
- buf[len_test_2] == *(buf + len_test_2 - back_offset);
- ++len_test_2) ;
-
- len_test_2 -= len_test + 1;
-
- if (len_test_2 >= 2) {
- uint32_t state_2 = state;
- update_match(state_2);
- uint32_t pos_state_next
- = (position + len_test) & coder->pos_mask;
-
- const uint32_t cur_and_len_char_price = cur_and_len_price
- + bit_get_price_0(
- coder->is_match[state_2][pos_state_next])
- + literal_get_price(
- literal_get_subcoder(
- coder->literal_coder,
- position + len_test,
- buf[len_test - 1]),
- true,
- *(buf + len_test - back_offset),
- buf[len_test]);
-
- update_literal(state_2);
- pos_state_next = (pos_state_next + 1) & coder->pos_mask;
-
- const uint32_t next_rep_match_price
- = cur_and_len_char_price
- + bit_get_price_1(
- coder->is_match[state_2][pos_state_next])
- + bit_get_price_1(coder->is_rep[state_2]);
-
- // for(; len_test_2 >= 2; --len_test_2) {
- const uint32_t offset = cur + len_test + 1 + len_test_2;
-
- while (len_end < offset)
- coder->optimum[++len_end].price = INFINITY_PRICE;
-
- cur_and_len_price = next_rep_match_price;
- get_rep_price(cur_and_len_price,
- 0, len_test_2, state_2, pos_state_next);
-
- if (cur_and_len_price < coder->optimum[offset].price) {
- coder->optimum[offset].price = cur_and_len_price;
- coder->optimum[offset].pos_prev = cur + len_test + 1;
- coder->optimum[offset].back_prev = 0;
- coder->optimum[offset].prev_1_is_char = true;
- coder->optimum[offset].prev_2 = true;
- coder->optimum[offset].pos_prev_2 = cur;
- coder->optimum[offset].back_prev_2
- = cur_back + REP_DISTANCES;
- }
-// }
- }
-
- offs += 2;
- if (offs == num_distance_pairs)
- break;
-
- cur_back = match_distances[offs + 2];
- if (cur_back >= FULL_DISTANCES)
- pos_slot = get_pos_slot_2(cur_back);
- }
- }
- }
-
- } // Closes: while (true)
-}
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file lzma_encoder_getoptimumfast.c
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-// NOTE: If you want to keep the line length in 80 characters, set
-// tab width to 4 or less in your editor when editing this file.
-
-
-#include "lzma_encoder_private.h"
-
-
-#define change_pair(small_dist, big_dist) \
- (((big_dist) >> 7) > (small_dist))
-
-
-extern void
-lzma_get_optimum_fast(lzma_coder *restrict coder,
- uint32_t *restrict back_res, uint32_t *restrict len_res)
-{
- // Local copies
- const uint32_t fast_bytes = coder->fast_bytes;
-
- uint32_t len_main;
- uint32_t num_distance_pairs;
- if (!coder->longest_match_was_found) {
- lzma_read_match_distances(coder, &len_main, &num_distance_pairs);
- } else {
- len_main = coder->longest_match_length;
- num_distance_pairs = coder->num_distance_pairs;
- coder->longest_match_was_found = false;
- }
-
- const uint8_t *buf = coder->lz.buffer + coder->lz.read_pos - 1;
- uint32_t num_available_bytes
- = coder->lz.write_pos - coder->lz.read_pos + 1;
-
- if (num_available_bytes < 2) {
- // There's not enough input left to encode a match.
- *back_res = UINT32_MAX;
- *len_res = 1;
- return;
- }
-
- if (num_available_bytes > MATCH_MAX_LEN)
- num_available_bytes = MATCH_MAX_LEN;
-
-
- // Look for repetitive matches; scan the previous four match distances
- uint32_t rep_lens[REP_DISTANCES];
- uint32_t rep_max_index = 0;
-
- for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
- const uint32_t back_offset = coder->reps[i] + 1;
-
- // If the first two bytes (2 == MATCH_MIN_LEN) do not match,
- // this rep_distance[i] is not useful. This is indicated
- // using zero as the length of the repetitive match.
- if (buf[0] != *(buf - back_offset)
- || buf[1] != *(buf + 1 - back_offset)) {
- rep_lens[i] = 0;
- continue;
- }
-
- // The first two bytes matched.
- // Calculate the length of the match.
- uint32_t len;
- for (len = 2; len < num_available_bytes
- && buf[len] == *(buf + len - back_offset);
- ++len) ;
-
- // If we have found a repetitive match that is at least
- // as long as fast_bytes, return it immediatelly.
- if (len >= fast_bytes) {
- *back_res = i;
- *len_res = len;
- move_pos(len - 1);
- return;
- }
-
- rep_lens[i] = len;
-
- // After this loop, rep_lens[rep_max_index] is the biggest
- // value of all values in rep_lens[].
- if (len > rep_lens[rep_max_index])
- rep_max_index = i;
- }
-
-
- if (len_main >= fast_bytes) {
- *back_res = coder->match_distances[num_distance_pairs]
- + REP_DISTANCES;
- *len_res = len_main;
- move_pos(len_main - 1);
- return;
- }
-
- uint32_t back_main = 0;
- if (len_main >= 2) {
- back_main = coder->match_distances[num_distance_pairs];
-
- while (num_distance_pairs > 2 && len_main ==
- coder->match_distances[num_distance_pairs - 3] + 1) {
- if (!change_pair(coder->match_distances[
- num_distance_pairs - 2], back_main))
- break;
-
- num_distance_pairs -= 2;
- len_main = coder->match_distances[num_distance_pairs - 1];
- back_main = coder->match_distances[num_distance_pairs];
- }
-
- if (len_main == 2 && back_main >= 0x80)
- len_main = 1;
- }
-
- if (rep_lens[rep_max_index] >= 2) {
- if (rep_lens[rep_max_index] + 1 >= len_main
- || (rep_lens[rep_max_index] + 2 >= len_main
- && (back_main > (1 << 9)))
- || (rep_lens[rep_max_index] + 3 >= len_main
- && (back_main > (1 << 15)))) {
- *back_res = rep_max_index;
- *len_res = rep_lens[rep_max_index];
- move_pos(*len_res - 1);
- return;
- }
- }
-
- if (len_main >= 2 && num_available_bytes > 2) {
- lzma_read_match_distances(coder, &coder->longest_match_length,
- &coder->num_distance_pairs);
-
- if (coder->longest_match_length >= 2) {
- const uint32_t new_distance = coder->match_distances[
- coder->num_distance_pairs];
-
- if ((coder->longest_match_length >= len_main
- && new_distance < back_main)
- || (coder->longest_match_length == len_main + 1
- && !change_pair(back_main, new_distance))
- || (coder->longest_match_length > len_main + 1)
- || (coder->longest_match_length + 1 >= len_main
- && len_main >= 3
- && change_pair(new_distance, back_main))) {
- coder->longest_match_was_found = true;
- *back_res = UINT32_MAX;
- *len_res = 1;
- return;
- }
- }
-
- ++buf;
- --num_available_bytes;
-
- for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
- const uint32_t back_offset = coder->reps[i] + 1;
-
- if (buf[1] != *(buf + 1 - back_offset)
- || buf[2] != *(buf + 2 - back_offset)) {
- rep_lens[i] = 0;
- continue;
- }
-
- uint32_t len;
- for (len = 2; len < num_available_bytes
- && buf[len] == *(buf + len - back_offset);
- ++len) ;
-
- if (len + 1 >= len_main) {
- coder->longest_match_was_found = true;
- *back_res = UINT32_MAX;
- *len_res = 1;
- return;
- }
- }
-
- *back_res = back_main + REP_DISTANCES;
- *len_res = len_main;
- move_pos(len_main - 2);
- return;
- }
-
- *back_res = UINT32_MAX;
- *len_res = 1;
- return;
-}
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file lzma_encoder_init.c
-/// \brief Creating, resetting and destroying the LZMA encoder
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "lzma_encoder_private.h"
-
-
-/// \brief Initializes the length encoder
-static void
-length_encoder_reset(lzma_length_encoder *lencoder,
- const uint32_t num_pos_states, const uint32_t table_size)
-{
- // NLength::CPriceTableEncoder::SetTableSize()
- lencoder->table_size = table_size;
-
- // NLength::CEncoder::Init()
- bit_reset(lencoder->choice);
- bit_reset(lencoder->choice2);
-
- for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) {
- bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS);
- bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS);
- }
-
- bittree_reset(lencoder->high, LEN_HIGH_BITS);
-
- // NLength::CPriceTableEncoder::UpdateTables()
- for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state)
- lencoder->counters[pos_state] = 1;
-
- return;
-}
-
-
-static void
-lzma_lzma_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
-{
- lzma_lz_encoder_end(&coder->lz, allocator);
- lzma_free(coder, allocator);
- return;
-}
-
-
-extern lzma_ret
-lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
- const lzma_filter_info *filters)
-{
- if (next->coder == NULL) {
- next->coder = lzma_alloc(sizeof(lzma_coder), allocator);
- if (next->coder == NULL)
- return LZMA_MEM_ERROR;
-
- next->coder->next = LZMA_NEXT_CODER_INIT;
- next->coder->lz = LZMA_LZ_ENCODER_INIT;
- }
-
- // Validate options that aren't validated elsewhere.
- const lzma_options_lzma *options = filters[0].options;
- if (options->pos_bits > LZMA_POS_BITS_MAX
- || options->fast_bytes < LZMA_FAST_BYTES_MIN
- || options->fast_bytes > LZMA_FAST_BYTES_MAX) {
- lzma_lzma_encoder_end(next->coder, allocator);
- return LZMA_HEADER_ERROR;
- }
-
- // Set compression mode.
- switch (options->mode) {
- case LZMA_MODE_FAST:
- next->coder->best_compression = false;
- break;
-
- case LZMA_MODE_BEST:
- next->coder->best_compression = true;
- break;
-
- default:
- lzma_lzma_encoder_end(next->coder, allocator);
- return LZMA_HEADER_ERROR;
- }
-
- // Initialize literal coder.
- {
- const lzma_ret ret = lzma_literal_init(
- &next->coder->literal_coder,
- options->literal_context_bits,
- options->literal_pos_bits);
- if (ret != LZMA_OK)
- return ret;
- }
-
- // Initialize LZ encoder.
- {
- const lzma_ret ret = lzma_lz_encoder_reset(
- &next->coder->lz, allocator, &lzma_lzma_encode,
- options->dictionary_size, OPTS,
- options->fast_bytes, MATCH_MAX_LEN + 1 + OPTS,
- options->match_finder,
- options->match_finder_cycles,
- options->preset_dictionary,
- options->preset_dictionary_size);
- if (ret != LZMA_OK) {
- lzma_lzma_encoder_end(next->coder, allocator);
- return ret;
- }
- }
-
- // Set dist_table_size.
- {
- // Round the dictionary size up to next 2^n.
- uint32_t log_size;
- for (log_size = 0; (UINT32_C(1) << log_size)
- < options->dictionary_size; ++log_size) ;
-
- next->coder->dist_table_size = log_size * 2;
- }
-
- // Misc FIXME desc
- next->coder->align_price_count = UINT32_MAX;
- next->coder->match_price_count = UINT32_MAX;
- next->coder->dictionary_size = options->dictionary_size;
- next->coder->pos_mask = (1U << options->pos_bits) - 1;
- next->coder->fast_bytes = options->fast_bytes;
-
- // Range coder
- rc_reset(&next->coder->rc);
-
- // State
- next->coder->state = 0;
- next->coder->previous_byte = 0;
- for (size_t i = 0; i < REP_DISTANCES; ++i)
- next->coder->reps[i] = 0;
-
- // Bit encoders
- for (size_t i = 0; i < STATES; ++i) {
- for (size_t j = 0; j <= next->coder->pos_mask; ++j) {
- bit_reset(next->coder->is_match[i][j]);
- bit_reset(next->coder->is_rep0_long[i][j]);
- }
-
- bit_reset(next->coder->is_rep[i]);
- bit_reset(next->coder->is_rep0[i]);
- bit_reset(next->coder->is_rep1[i]);
- bit_reset(next->coder->is_rep2[i]);
- }
-
- for (size_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i)
- bit_reset(next->coder->pos_encoders[i]);
-
- // Bit tree encoders
- for (size_t i = 0; i < LEN_TO_POS_STATES; ++i)
- bittree_reset(next->coder->pos_slot_encoder[i], POS_SLOT_BITS);
-
- bittree_reset(next->coder->pos_align_encoder, ALIGN_BITS);
-
- // Length encoders
- length_encoder_reset(&next->coder->match_len_encoder,
- 1U << options->pos_bits,
- options->fast_bytes + 1 - MATCH_MIN_LEN);
-
- length_encoder_reset(&next->coder->rep_len_encoder,
- 1U << options->pos_bits,
- next->coder->fast_bytes + 1 - MATCH_MIN_LEN);
-
- next->coder->prev_len_encoder = NULL;
-
- // Misc
- next->coder->longest_match_was_found = false;
- next->coder->optimum_end_index = 0;
- next->coder->optimum_current_index = 0;
- next->coder->additional_offset = 0;
-
- next->coder->now_pos = 0;
- next->coder->is_initialized = false;
- next->coder->is_flushed = false,
- next->coder->write_eopm = true;
-
- // Initialize the next decoder in the chain, if any.
- {
- const lzma_ret ret = lzma_next_filter_init(&next->coder->next,
- allocator, filters + 1);
- if (ret != LZMA_OK) {
- lzma_lzma_encoder_end(next->coder, allocator);
- return ret;
- }
- }
-
- // Initialization successful. Set the function pointers.
- next->code = &lzma_lz_encode;
- next->end = &lzma_lzma_encoder_end;
-
- return LZMA_OK;
-}
-
-
-extern bool
-lzma_lzma_encode_properties(const lzma_options_lzma *options, uint8_t *byte)
-{
- if (options->literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX
- || options->literal_pos_bits
- > LZMA_LITERAL_POS_BITS_MAX
- || options->pos_bits > LZMA_POS_BITS_MAX
- || options->literal_context_bits
- + options->literal_pos_bits
- > LZMA_LITERAL_BITS_MAX)
- return true;
-
- *byte = (options->pos_bits * 5 + options->literal_pos_bits) * 9
- + options->literal_context_bits;
- assert(*byte <= (4 * 5 + 4) * 9 + 8);
-
- return false;
-}
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file lzma_encoder_optimum_fast.c
+//
+// Copyright (C) 1999-2008 Igor Pavlov
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "lzma_encoder_private.h"
+
+
+#define change_pair(small_dist, big_dist) \
+ (((big_dist) >> 7) > (small_dist))
+
+
+static inline void
+literal(const lzma_coder *restrict coder, const uint8_t *restrict buf,
+ uint32_t *restrict back_res, uint32_t *restrict len_res)
+{
+ // Try short rep0 instead of always coding it as a literal.
+ *back_res = *buf == *(buf - coder->reps[0] - 1) ? 0 : UINT32_MAX;
+ *len_res = 1;
+ return;
+}
+
+
+extern void
+lzma_lzma_optimum_fast(lzma_coder *restrict coder, lzma_mf *restrict mf,
+ uint32_t *restrict back_res, uint32_t *restrict len_res)
+{
+ const uint32_t fast_bytes = mf->find_len_max;
+
+ uint32_t len_main;
+ uint32_t matches_count;
+ if (mf->read_ahead == 0) {
+ len_main = mf_find(mf, &matches_count, coder->matches);
+ } else {
+ assert(mf->read_ahead == 1);
+ len_main = coder->longest_match_length;
+ matches_count = coder->matches_count;
+ }
+
+ const uint8_t *buf = mf_ptr(mf) - 1;
+ const uint32_t buf_avail = MIN(mf_avail(mf) + 1, MATCH_LEN_MAX);
+
+ if (buf_avail < 2) {
+ // There's not enough input left to encode a match.
+ literal(coder, buf, back_res, len_res);
+ return;
+ }
+
+ // Look for repeated matches; scan the previous four match distances
+ uint32_t rep_len = 0;
+ uint32_t rep_index = 0;
+
+ for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
+ // Pointer to the beginning of the match candidate
+ const uint8_t *const buf_back = buf - coder->reps[i] - 1;
+
+ // If the first two bytes (2 == MATCH_LEN_MIN) do not match,
+ // this rep is not useful.
+ if (not_equal_16(buf, buf_back))
+ continue;
+
+ // The first two bytes matched.
+ // Calculate the length of the match.
+ uint32_t len;
+ for (len = 2; len < buf_avail
+ && buf[len] == buf_back[len]; ++len) ;
+
+ // If we have found a repeated match that is at least
+ // fast_bytes long, return it immediatelly.
+ if (len >= fast_bytes) {
+ *back_res = i;
+ *len_res = len;
+ mf_skip(mf, len - 1);
+ return;
+ }
+
+ if (len > rep_len) {
+ rep_index = i;
+ rep_len = len;
+ }
+ }
+
+ // We didn't find a long enough repeated match. Encode it as a normal
+ // match if the match length is at least fast_bytes.
+ if (len_main >= fast_bytes) {
+ *back_res = coder->matches[matches_count - 1].dist
+ + REP_DISTANCES;
+ *len_res = len_main;
+ mf_skip(mf, len_main - 1);
+ return;
+ }
+
+ uint32_t back_main = 0;
+ if (len_main >= 2) {
+ back_main = coder->matches[matches_count - 1].dist;
+
+ while (matches_count > 1 && len_main ==
+ coder->matches[matches_count - 2].len + 1) {
+ if (!change_pair(coder->matches[
+ matches_count - 2].dist,
+ back_main))
+ break;
+
+ --matches_count;
+ len_main = coder->matches[matches_count - 1].len;
+ back_main = coder->matches[matches_count - 1].dist;
+ }
+
+ if (len_main == 2 && back_main >= 0x80)
+ len_main = 1;
+ }
+
+ if (rep_len >= 2) {
+ if (rep_len + 1 >= len_main
+ || (rep_len + 2 >= len_main
+ && back_main > (UINT32_C(1) << 9))
+ || (rep_len + 3 >= len_main
+ && back_main > (UINT32_C(1) << 15))) {
+ *back_res = rep_index;
+ *len_res = rep_len;
+ mf_skip(mf, rep_len - 1);
+ return;
+ }
+ }
+
+ if (len_main < 2 || buf_avail <= 2) {
+ literal(coder, buf, back_res, len_res);
+ return;
+ }
+
+ // Get the matches for the next byte. If we find a better match,
+ // the current byte is encoded as a literal.
+ coder->longest_match_length = mf_find(mf,
+ &coder->matches_count, coder->matches);
+
+ if (coder->longest_match_length >= 2) {
+ const uint32_t new_dist = coder->matches[
+ coder->matches_count - 1].dist;
+
+ if ((coder->longest_match_length >= len_main
+ && new_dist < back_main)
+ || (coder->longest_match_length == len_main + 1
+ && !change_pair(back_main, new_dist))
+ || (coder->longest_match_length > len_main + 1)
+ || (coder->longest_match_length + 1 >= len_main
+ && len_main >= 3
+ && change_pair(new_dist, back_main))) {
+ literal(coder, buf, back_res, len_res);
+ return;
+ }
+ }
+
+ // In contrast to LZMA SDK, dictionary could not have been moved
+ // between mf_find() calls, thus it is safe to just increment
+ // the old buf pointer instead of recalculating it with mf_ptr().
+ ++buf;
+
+ const uint32_t limit = len_main - 1;
+
+ for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
+ const uint8_t *const buf_back = buf - coder->reps[i] - 1;
+
+ if (not_equal_16(buf, buf_back))
+ continue;
+
+ uint32_t len;
+ for (len = 2; len < limit
+ && buf[len] == buf_back[len]; ++len) ;
+
+ if (len >= limit) {
+ literal(coder, buf - 1, back_res, len_res);
+ return;
+ }
+ }
+
+ *back_res = back_main + REP_DISTANCES;
+ *len_res = len_main;
+ mf_skip(mf, len_main - 2);
+ return;
+}
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file lzma_encoder_optimum_normal.c
+//
+// Copyright (C) 1999-2008 Igor Pavlov
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "lzma_encoder_private.h"
+#include "fastpos.h"
+
+
+////////////
+// Prices //
+////////////
+
+static uint32_t
+get_literal_price(const lzma_coder *const coder, const uint32_t pos,
+ const uint32_t prev_byte, const bool match_mode,
+ uint32_t match_byte, uint32_t symbol)
+{
+ const probability *const subcoder = literal_subcoder(coder->literal,
+ coder->literal_context_bits, coder->literal_pos_mask,
+ pos, prev_byte);
+
+ uint32_t price = 0;
+
+ if (!match_mode) {
+ price = rc_bittree_price(subcoder, 8, symbol);
+ } else {
+ uint32_t offset = 0x100;
+ symbol += UINT32_C(1) << 8;
+
+ do {
+ match_byte <<= 1;
+
+ const uint32_t match_bit = match_byte & offset;
+ const uint32_t subcoder_index
+ = offset + match_bit + (symbol >> 8);
+ const uint32_t bit = (symbol >> 7) & 1;
+ price += rc_bit_price(subcoder[subcoder_index], bit);
+
+ symbol <<= 1;
+ offset &= ~(match_byte ^ symbol);
+
+ } while (symbol < (UINT32_C(1) << 16));
+ }
+
+ return price;
+}
+
+
+static inline uint32_t
+get_len_price(const lzma_length_encoder *const lencoder,
+ const uint32_t len, const uint32_t pos_state)
+{
+ // NOTE: Unlike the other price tables, length prices are updated
+ // in lzma_encoder.c
+ return lencoder->prices[pos_state][len - MATCH_LEN_MIN];
+}
+
+
+static inline uint32_t
+get_short_rep_price(const lzma_coder *const coder,
+ const lzma_lzma_state state, const uint32_t pos_state)
+{
+ return rc_bit_0_price(coder->is_rep0[state])
+ + rc_bit_0_price(coder->is_rep0_long[state][pos_state]);
+}
+
+
+static inline uint32_t
+get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index,
+ const lzma_lzma_state state, uint32_t pos_state)
+{
+ uint32_t price;
+
+ if (rep_index == 0) {
+ price = rc_bit_0_price(coder->is_rep0[state]);
+ price += rc_bit_1_price(coder->is_rep0_long[state][pos_state]);
+ } else {
+ price = rc_bit_1_price(coder->is_rep0[state]);
+
+ if (rep_index == 1) {
+ price += rc_bit_0_price(coder->is_rep1[state]);
+ } else {
+ price += rc_bit_1_price(coder->is_rep1[state]);
+ price += rc_bit_price(coder->is_rep2[state],
+ rep_index - 2);
+ }
+ }
+
+ return price;
+}
+
+
+static inline uint32_t
+get_rep_price(const lzma_coder *const coder, const uint32_t rep_index,
+ const uint32_t len, const lzma_lzma_state state,
+ const uint32_t pos_state)
+{
+ return get_len_price(&coder->rep_len_encoder, len, pos_state)
+ + get_pure_rep_price(coder, rep_index, state, pos_state);
+}
+
+
+static inline uint32_t
+get_pos_len_price(const lzma_coder *const coder, const uint32_t pos,
+ const uint32_t len, const uint32_t pos_state)
+{
+ const uint32_t len_to_pos_state = get_len_to_pos_state(len);
+ uint32_t price;
+
+ if (pos < FULL_DISTANCES) {
+ price = coder->distances_prices[len_to_pos_state][pos];
+ } else {
+ const uint32_t pos_slot = get_pos_slot_2(pos);
+ price = coder->pos_slot_prices[len_to_pos_state][pos_slot]
+ + coder->align_prices[pos & ALIGN_MASK];
+ }
+
+ price += get_len_price(&coder->match_len_encoder, len, pos_state);
+
+ return price;
+}
+
+
+static void
+fill_distances_prices(lzma_coder *coder)
+{
+ for (uint32_t len_to_pos_state = 0;
+ len_to_pos_state < LEN_TO_POS_STATES;
+ ++len_to_pos_state) {
+
+ uint32_t *const pos_slot_prices
+ = coder->pos_slot_prices[len_to_pos_state];
+
+ // Price to encode the pos_slot.
+ for (uint32_t pos_slot = 0;
+ pos_slot < coder->dist_table_size; ++pos_slot)
+ pos_slot_prices[pos_slot] = rc_bittree_price(
+ coder->pos_slot[len_to_pos_state],
+ POS_SLOT_BITS, pos_slot);
+
+ // For matches with distance >= FULL_DISTANCES, add the price
+ // of the direct bits part of the match distance. (Align bits
+ // are handled by fill_align_prices()).
+ for (uint32_t pos_slot = END_POS_MODEL_INDEX;
+ pos_slot < coder->dist_table_size; ++pos_slot)
+ pos_slot_prices[pos_slot] += rc_direct_price(
+ ((pos_slot >> 1) - 1) - ALIGN_BITS);
+
+ // Distances in the range [0, 3] are fully encoded with
+ // pos_slot, so they are used for coder->distances_prices
+ // as is.
+ for (uint32_t i = 0; i < START_POS_MODEL_INDEX; ++i)
+ coder->distances_prices[len_to_pos_state][i]
+ = pos_slot_prices[i];
+ }
+
+ // Distances in the range [4, 127] depend on pos_slot and pos_special.
+ // We do this in a loop separate from the above loop to avoid
+ // redundant calls to get_pos_slot().
+ for (uint32_t i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) {
+ const uint32_t pos_slot = get_pos_slot(i);
+ const uint32_t footer_bits = ((pos_slot >> 1) - 1);
+ const uint32_t base = (2 | (pos_slot & 1)) << footer_bits;
+ const uint32_t price = rc_bittree_reverse_price(
+ coder->pos_special + base - pos_slot - 1,
+ footer_bits, i - base);
+
+ for (uint32_t len_to_pos_state = 0;
+ len_to_pos_state < LEN_TO_POS_STATES;
+ ++len_to_pos_state)
+ coder->distances_prices[len_to_pos_state][i]
+ = price + coder->pos_slot_prices[
+ len_to_pos_state][pos_slot];
+ }
+
+ coder->match_price_count = 0;
+ return;
+}
+
+
+static void
+fill_align_prices(lzma_coder *coder)
+{
+ for (uint32_t i = 0; i < ALIGN_TABLE_SIZE; ++i)
+ coder->align_prices[i] = rc_bittree_reverse_price(
+ coder->pos_align, ALIGN_BITS, i);
+
+ coder->align_price_count = 0;
+ return;
+}
+
+
+/////////////
+// Optimal //
+/////////////
+
+static inline void
+make_literal(lzma_optimal *optimal)
+{
+ optimal->back_prev = UINT32_MAX;
+ optimal->prev_1_is_literal = false;
+}
+
+
+static inline void
+make_short_rep(lzma_optimal *optimal)
+{
+ optimal->back_prev = 0;
+ optimal->prev_1_is_literal = false;
+}
+
+
+#define is_short_rep(optimal) \
+ ((optimal).back_prev == 0)
+
+
+static void
+backward(lzma_coder *restrict coder, uint32_t *restrict len_res,
+ uint32_t *restrict back_res, uint32_t cur)
+{
+ coder->opts_end_index = cur;
+
+ uint32_t pos_mem = coder->opts[cur].pos_prev;
+ uint32_t back_mem = coder->opts[cur].back_prev;
+
+ do {
+ if (coder->opts[cur].prev_1_is_literal) {
+ make_literal(&coder->opts[pos_mem]);
+ coder->opts[pos_mem].pos_prev = pos_mem - 1;
+
+ if (coder->opts[cur].prev_2) {
+ coder->opts[pos_mem - 1].prev_1_is_literal
+ = false;
+ coder->opts[pos_mem - 1].pos_prev
+ = coder->opts[cur].pos_prev_2;
+ coder->opts[pos_mem - 1].back_prev
+ = coder->opts[cur].back_prev_2;
+ }
+ }
+
+ const uint32_t pos_prev = pos_mem;
+ const uint32_t back_cur = back_mem;
+
+ back_mem = coder->opts[pos_prev].back_prev;
+ pos_mem = coder->opts[pos_prev].pos_prev;
+
+ coder->opts[pos_prev].back_prev = back_cur;
+ coder->opts[pos_prev].pos_prev = cur;
+ cur = pos_prev;
+
+ } while (cur != 0);
+
+ coder->opts_current_index = coder->opts[0].pos_prev;
+ *len_res = coder->opts[0].pos_prev;
+ *back_res = coder->opts[0].back_prev;
+
+ return;
+}
+
+
+//////////
+// Main //
+//////////
+
+static inline uint32_t
+helper1(lzma_coder *restrict coder, lzma_mf *restrict mf,
+ uint32_t *restrict back_res, uint32_t *restrict len_res,
+ uint32_t position)
+{
+ const uint32_t fast_bytes = mf->find_len_max;
+
+ uint32_t len_main;
+ uint32_t matches_count;
+
+ if (mf->read_ahead == 0) {
+ len_main = mf_find(mf, &matches_count, coder->matches);
+ } else {
+ assert(mf->read_ahead == 1);
+ len_main = coder->longest_match_length;
+ matches_count = coder->matches_count;
+ }
+
+ const uint32_t buf_avail = MIN(mf_avail(mf) + 1, MATCH_LEN_MAX);
+ if (buf_avail < 2) {
+ *back_res = UINT32_MAX;
+ *len_res = 1;
+ return UINT32_MAX;
+ }
+
+ const uint8_t *const buf = mf_ptr(mf) - 1;
+
+ uint32_t rep_lens[REP_DISTANCES];
+ uint32_t rep_max_index = 0;
+
+ for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
+ const uint8_t *const buf_back = buf - coder->reps[i] - 1;
+
+ if (not_equal_16(buf, buf_back)) {
+ rep_lens[i] = 0;
+ continue;
+ }
+
+ uint32_t len_test;
+ for (len_test = 2; len_test < buf_avail
+ && buf[len_test] == buf_back[len_test];
+ ++len_test) ;
+
+ rep_lens[i] = len_test;
+ if (len_test > rep_lens[rep_max_index])
+ rep_max_index = i;
+ }
+
+ if (rep_lens[rep_max_index] >= fast_bytes) {
+ *back_res = rep_max_index;
+ *len_res = rep_lens[rep_max_index];
+ mf_skip(mf, *len_res - 1);
+ return UINT32_MAX;
+ }
+
+
+ if (len_main >= fast_bytes) {
+ *back_res = coder->matches[matches_count - 1].dist
+ + REP_DISTANCES;
+ *len_res = len_main;
+ mf_skip(mf, len_main - 1);
+ return UINT32_MAX;
+ }
+
+ const uint8_t current_byte = *buf;
+ const uint8_t match_byte = *(buf - coder->reps[0] - 1);
+
+ if (len_main < 2 && current_byte != match_byte
+ && rep_lens[rep_max_index] < 2) {
+ *back_res = UINT32_MAX;
+ *len_res = 1;
+ return UINT32_MAX;
+ }
+
+ coder->opts[0].state = coder->state;
+
+ const uint32_t pos_state = position & coder->pos_mask;
+
+ coder->opts[1].price = rc_bit_0_price(
+ coder->is_match[coder->state][pos_state])
+ + get_literal_price(coder, position, buf[-1],
+ !is_literal_state(coder->state),
+ match_byte, current_byte);
+
+ make_literal(&coder->opts[1]);
+
+ const uint32_t match_price = rc_bit_1_price(
+ coder->is_match[coder->state][pos_state]);
+ const uint32_t rep_match_price = match_price
+ + rc_bit_1_price(coder->is_rep[coder->state]);
+
+ if (match_byte == current_byte) {
+ const uint32_t short_rep_price = rep_match_price
+ + get_short_rep_price(
+ coder, coder->state, pos_state);
+
+ if (short_rep_price < coder->opts[1].price) {
+ coder->opts[1].price = short_rep_price;
+ make_short_rep(&coder->opts[1]);
+ }
+ }
+
+ const uint32_t len_end = MAX(len_main, rep_lens[rep_max_index]);
+
+ if (len_end < 2) {
+ *back_res = coder->opts[1].back_prev;
+ *len_res = 1;
+ return UINT32_MAX;
+ }
+
+ coder->opts[1].pos_prev = 0;
+
+ for (uint32_t i = 0; i < REP_DISTANCES; ++i)
+ coder->opts[0].backs[i] = coder->reps[i];
+
+ uint32_t len = len_end;
+ do {
+ coder->opts[len].price = RC_INFINITY_PRICE;
+ } while (--len >= 2);
+
+
+ for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
+ uint32_t rep_len = rep_lens[i];
+ if (rep_len < 2)
+ continue;
+
+ const uint32_t price = rep_match_price + get_pure_rep_price(
+ coder, i, coder->state, pos_state);
+
+ do {
+ const uint32_t cur_and_len_price = price
+ + get_len_price(
+ &coder->rep_len_encoder,
+ rep_len, pos_state);
+
+ if (cur_and_len_price < coder->opts[rep_len].price) {
+ coder->opts[rep_len].price = cur_and_len_price;
+ coder->opts[rep_len].pos_prev = 0;
+ coder->opts[rep_len].back_prev = i;
+ coder->opts[rep_len].prev_1_is_literal = false;
+ }
+ } while (--rep_len >= 2);
+ }
+
+
+ const uint32_t normal_match_price = match_price
+ + rc_bit_0_price(coder->is_rep[coder->state]);
+
+ len = rep_lens[0] >= 2 ? rep_lens[0] + 1 : 2;
+ if (len <= len_main) {
+ uint32_t i = 0;
+ while (len > coder->matches[i].len)
+ ++i;
+
+ for(; ; ++len) {
+ const uint32_t dist = coder->matches[i].dist;
+ const uint32_t cur_and_len_price = normal_match_price
+ + get_pos_len_price(coder,
+ dist, len, pos_state);
+
+ if (cur_and_len_price < coder->opts[len].price) {
+ coder->opts[len].price = cur_and_len_price;
+ coder->opts[len].pos_prev = 0;
+ coder->opts[len].back_prev
+ = dist + REP_DISTANCES;
+ coder->opts[len].prev_1_is_literal = false;
+ }
+
+ if (len == coder->matches[i].len)
+ if (++i == matches_count)
+ break;
+ }
+ }
+
+ return len_end;
+}
+
+
+static inline uint32_t
+helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf,
+ uint32_t len_end, uint32_t position, const uint32_t cur,
+ const uint32_t fast_bytes, const uint32_t buf_avail_full)
+{
+ uint32_t matches_count = coder->matches_count;
+ uint32_t new_len = coder->longest_match_length;
+ uint32_t pos_prev = coder->opts[cur].pos_prev;
+ uint32_t state;
+
+ if (coder->opts[cur].prev_1_is_literal) {
+ --pos_prev;
+
+ if (coder->opts[cur].prev_2) {
+ state = coder->opts[coder->opts[cur].pos_prev_2].state;
+
+ if (coder->opts[cur].back_prev_2 < REP_DISTANCES)
+ update_long_rep(state);
+ else
+ update_match(state);
+
+ } else {
+ state = coder->opts[pos_prev].state;
+ }
+
+ update_literal(state);
+
+ } else {
+ state = coder->opts[pos_prev].state;
+ }
+
+ if (pos_prev == cur - 1) {
+ if (is_short_rep(coder->opts[cur]))
+ update_short_rep(state);
+ else
+ update_literal(state);
+ } else {
+ uint32_t pos;
+ if (coder->opts[cur].prev_1_is_literal
+ && coder->opts[cur].prev_2) {
+ pos_prev = coder->opts[cur].pos_prev_2;
+ pos = coder->opts[cur].back_prev_2;
+ update_long_rep(state);
+ } else {
+ pos = coder->opts[cur].back_prev;
+ if (pos < REP_DISTANCES)
+ update_long_rep(state);
+ else
+ update_match(state);
+ }
+
+ if (pos < REP_DISTANCES) {
+ reps[0] = coder->opts[pos_prev].backs[pos];
+
+ uint32_t i;
+ for (i = 1; i <= pos; ++i)
+ reps[i] = coder->opts[pos_prev].backs[i - 1];
+
+ for (; i < REP_DISTANCES; ++i)
+ reps[i] = coder->opts[pos_prev].backs[i];
+
+ } else {
+ reps[0] = pos - REP_DISTANCES;
+
+ for (uint32_t i = 1; i < REP_DISTANCES; ++i)
+ reps[i] = coder->opts[pos_prev].backs[i - 1];
+ }
+ }
+
+ coder->opts[cur].state = state;
+
+ for (uint32_t i = 0; i < REP_DISTANCES; ++i)
+ coder->opts[cur].backs[i] = reps[i];
+
+ const uint32_t cur_price = coder->opts[cur].price;
+
+ const uint8_t current_byte = *buf;
+ const uint8_t match_byte = *(buf - reps[0] - 1);
+
+ const uint32_t pos_state = position & coder->pos_mask;
+
+ const uint32_t cur_and_1_price = cur_price
+ + rc_bit_0_price(coder->is_match[state][pos_state])
+ + get_literal_price(coder, position, buf[-1],
+ !is_literal_state(state), match_byte, current_byte);
+
+ bool next_is_literal = false;
+
+ if (cur_and_1_price < coder->opts[cur + 1].price) {
+ coder->opts[cur + 1].price = cur_and_1_price;
+ coder->opts[cur + 1].pos_prev = cur;
+ make_literal(&coder->opts[cur + 1]);
+ next_is_literal = true;
+ }
+
+ const uint32_t match_price = cur_price
+ + rc_bit_1_price(coder->is_match[state][pos_state]);
+ const uint32_t rep_match_price = match_price
+ + rc_bit_1_price(coder->is_rep[state]);
+
+ if (match_byte == current_byte
+ && !(coder->opts[cur + 1].pos_prev < cur
+ && coder->opts[cur + 1].back_prev == 0)) {
+
+ const uint32_t short_rep_price = rep_match_price
+ + get_short_rep_price(coder, state, pos_state);
+
+ if (short_rep_price <= coder->opts[cur + 1].price) {
+ coder->opts[cur + 1].price = short_rep_price;
+ coder->opts[cur + 1].pos_prev = cur;
+ make_short_rep(&coder->opts[cur + 1]);
+ next_is_literal = true;
+ }
+ }
+
+ if (buf_avail_full < 2)
+ return len_end;
+
+ const uint32_t buf_avail = MIN(buf_avail_full, fast_bytes);
+
+ if (!next_is_literal && match_byte != current_byte) { // speed optimization
+ // try literal + rep0
+ const uint8_t *const buf_back = buf - reps[0] - 1;
+ const uint32_t limit = MIN(buf_avail_full, fast_bytes + 1);
+
+ uint32_t len_test = 1;
+ while (len_test < limit && buf[len_test] == buf_back[len_test])
+ ++len_test;
+
+ --len_test;
+
+ if (len_test >= 2) {
+ uint32_t state_2 = state;
+ update_literal(state_2);
+
+ const uint32_t pos_state_next = (position + 1) & coder->pos_mask;
+ const uint32_t next_rep_match_price = cur_and_1_price
+ + rc_bit_1_price(coder->is_match[state_2][pos_state_next])
+ + rc_bit_1_price(coder->is_rep[state_2]);
+
+ //for (; len_test >= 2; --len_test) {
+ const uint32_t offset = cur + 1 + len_test;
+
+ while (len_end < offset)
+ coder->opts[++len_end].price = RC_INFINITY_PRICE;
+
+ const uint32_t cur_and_len_price = next_rep_match_price
+ + get_rep_price(coder, 0, len_test,
+ state_2, pos_state_next);
+
+ if (cur_and_len_price < coder->opts[offset].price) {
+ coder->opts[offset].price = cur_and_len_price;
+ coder->opts[offset].pos_prev = cur + 1;
+ coder->opts[offset].back_prev = 0;
+ coder->opts[offset].prev_1_is_literal = true;
+ coder->opts[offset].prev_2 = false;
+ }
+ //}
+ }
+ }
+
+
+ uint32_t start_len = 2; // speed optimization
+
+ for (uint32_t rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) {
+ const uint8_t *const buf_back = buf - reps[rep_index] - 1;
+ if (not_equal_16(buf, buf_back))
+ continue;
+
+ uint32_t len_test;
+ for (len_test = 2; len_test < buf_avail
+ && buf[len_test] == buf_back[len_test];
+ ++len_test) ;
+
+ while (len_end < cur + len_test)
+ coder->opts[++len_end].price = RC_INFINITY_PRICE;
+
+ const uint32_t len_test_temp = len_test;
+ const uint32_t price = rep_match_price + get_pure_rep_price(
+ coder, rep_index, state, pos_state);
+
+ do {
+ const uint32_t cur_and_len_price = price
+ + get_len_price(&coder->rep_len_encoder,
+ len_test, pos_state);
+
+ if (cur_and_len_price < coder->opts[cur + len_test].price) {
+ coder->opts[cur + len_test].price = cur_and_len_price;
+ coder->opts[cur + len_test].pos_prev = cur;
+ coder->opts[cur + len_test].back_prev = rep_index;
+ coder->opts[cur + len_test].prev_1_is_literal = false;
+ }
+ } while (--len_test >= 2);
+
+ len_test = len_test_temp;
+
+ if (rep_index == 0)
+ start_len = len_test + 1;
+
+
+ uint32_t len_test_2 = len_test + 1;
+ const uint32_t limit = MIN(buf_avail_full,
+ len_test_2 + fast_bytes);
+ for (; len_test_2 < limit
+ && buf[len_test_2] == buf_back[len_test_2];
+ ++len_test_2) ;
+
+ len_test_2 -= len_test + 1;
+
+ if (len_test_2 >= 2) {
+ uint32_t state_2 = state;
+ update_long_rep(state_2);
+
+ uint32_t pos_state_next = (position + len_test) & coder->pos_mask;
+
+ const uint32_t cur_and_len_literal_price = price
+ + get_len_price(&coder->rep_len_encoder,
+ len_test, pos_state)
+ + rc_bit_0_price(coder->is_match[state_2][pos_state_next])
+ + get_literal_price(coder, position + len_test,
+ buf[len_test - 1], true,
+ buf_back[len_test], buf[len_test]);
+
+ update_literal(state_2);
+
+ pos_state_next = (position + len_test + 1) & coder->pos_mask;
+
+ const uint32_t next_rep_match_price = cur_and_len_literal_price
+ + rc_bit_1_price(coder->is_match[state_2][pos_state_next])
+ + rc_bit_1_price(coder->is_rep[state_2]);
+
+ //for(; len_test_2 >= 2; len_test_2--) {
+ const uint32_t offset = cur + len_test + 1 + len_test_2;
+
+ while (len_end < offset)
+ coder->opts[++len_end].price = RC_INFINITY_PRICE;
+
+ const uint32_t cur_and_len_price = next_rep_match_price
+ + get_rep_price(coder, 0, len_test_2,
+ state_2, pos_state_next);
+
+ if (cur_and_len_price < coder->opts[offset].price) {
+ coder->opts[offset].price = cur_and_len_price;
+ coder->opts[offset].pos_prev = cur + len_test + 1;
+ coder->opts[offset].back_prev = 0;
+ coder->opts[offset].prev_1_is_literal = true;
+ coder->opts[offset].prev_2 = true;
+ coder->opts[offset].pos_prev_2 = cur;
+ coder->opts[offset].back_prev_2 = rep_index;
+ }
+ //}
+ }
+ }
+
+
+ //for (uint32_t len_test = 2; len_test <= new_len; ++len_test)
+ if (new_len > buf_avail) {
+ new_len = buf_avail;
+
+ matches_count = 0;
+ while (new_len > coder->matches[matches_count].len)
+ ++matches_count;
+
+ coder->matches[matches_count++].len = new_len;
+ }
+
+
+ if (new_len >= start_len) {
+ const uint32_t normal_match_price = match_price
+ + rc_bit_0_price(coder->is_rep[state]);
+
+ while (len_end < cur + new_len)
+ coder->opts[++len_end].price = RC_INFINITY_PRICE;
+
+ uint32_t i = 0;
+ while (start_len > coder->matches[i].len)
+ ++i;
+
+ for (uint32_t len_test = start_len; ; ++len_test) {
+ const uint32_t cur_back = coder->matches[i].dist;
+ uint32_t cur_and_len_price = normal_match_price
+ + get_pos_len_price(coder,
+ cur_back, len_test, pos_state);
+
+ if (cur_and_len_price < coder->opts[cur + len_test].price) {
+ coder->opts[cur + len_test].price = cur_and_len_price;
+ coder->opts[cur + len_test].pos_prev = cur;
+ coder->opts[cur + len_test].back_prev
+ = cur_back + REP_DISTANCES;
+ coder->opts[cur + len_test].prev_1_is_literal = false;
+ }
+
+ if (len_test == coder->matches[i].len) {
+ // Try Match + Literal + Rep0
+ const uint8_t *const buf_back = buf - cur_back - 1;
+ uint32_t len_test_2 = len_test + 1;
+ const uint32_t limit = MIN(buf_avail_full,
+ len_test_2 + fast_bytes);
+
+ for (; len_test_2 < limit &&
+ buf[len_test_2] == buf_back[len_test_2];
+ ++len_test_2) ;
+
+ len_test_2 -= len_test + 1;
+
+ if (len_test_2 >= 2) {
+ uint32_t state_2 = state;
+ update_match(state_2);
+ uint32_t pos_state_next
+ = (position + len_test) & coder->pos_mask;
+
+ const uint32_t cur_and_len_literal_price = cur_and_len_price
+ + rc_bit_0_price(
+ coder->is_match[state_2][pos_state_next])
+ + get_literal_price(coder,
+ position + len_test,
+ buf[len_test - 1],
+ true,
+ buf_back[len_test],
+ buf[len_test]);
+
+ update_literal(state_2);
+ pos_state_next = (pos_state_next + 1) & coder->pos_mask;
+
+ const uint32_t next_rep_match_price
+ = cur_and_len_literal_price
+ + rc_bit_1_price(
+ coder->is_match[state_2][pos_state_next])
+ + rc_bit_1_price(coder->is_rep[state_2]);
+
+ // for(; len_test_2 >= 2; --len_test_2) {
+ const uint32_t offset = cur + len_test + 1 + len_test_2;
+
+ while (len_end < offset)
+ coder->opts[++len_end].price = RC_INFINITY_PRICE;
+
+ cur_and_len_price = next_rep_match_price
+ + get_rep_price(coder, 0, len_test_2,
+ state_2, pos_state_next);
+
+ if (cur_and_len_price < coder->opts[offset].price) {
+ coder->opts[offset].price = cur_and_len_price;
+ coder->opts[offset].pos_prev = cur + len_test + 1;
+ coder->opts[offset].back_prev = 0;
+ coder->opts[offset].prev_1_is_literal = true;
+ coder->opts[offset].prev_2 = true;
+ coder->opts[offset].pos_prev_2 = cur;
+ coder->opts[offset].back_prev_2
+ = cur_back + REP_DISTANCES;
+ }
+ //}
+ }
+
+ if (++i == matches_count)
+ break;
+ }
+ }
+ }
+
+ return len_end;
+}
+
+
+extern void
+lzma_lzma_optimum_normal(lzma_coder *restrict coder, lzma_mf *restrict mf,
+ uint32_t *restrict back_res, uint32_t *restrict len_res,
+ uint32_t position)
+{
+ // If we have symbols pending, return the next pending symbol.
+ if (coder->opts_end_index != coder->opts_current_index) {
+ assert(mf->read_ahead > 0);
+ *len_res = coder->opts[coder->opts_current_index].pos_prev
+ - coder->opts_current_index;
+ *back_res = coder->opts[coder->opts_current_index].back_prev;
+ coder->opts_current_index = coder->opts[
+ coder->opts_current_index].pos_prev;
+ return;
+ }
+
+ // Update the price tables. In LZMA SDK <= 4.60 (and possibly later)
+ // this was done in both initialization function and in the main loop.
+ // In liblzma they were moved into this single place.
+ if (mf->read_ahead == 0) {
+ if (coder->match_price_count >= (1 << 7))
+ fill_distances_prices(coder);
+
+ if (coder->align_price_count >= ALIGN_TABLE_SIZE)
+ fill_align_prices(coder);
+ }
+
+ // TODO: This needs quite a bit of cleaning still. But splitting
+ // the oroginal function to two pieces makes it at least a little
+ // more readable, since those two parts don't share many variables.
+
+ uint32_t len_end = helper1(coder, mf, back_res, len_res, position);
+ if (len_end == UINT32_MAX)
+ return;
+
+ uint32_t reps[REP_DISTANCES];
+ memcpy(reps, coder->reps, sizeof(reps));
+
+ uint32_t cur;
+ for (cur = 1; cur < len_end; ++cur) {
+ assert(cur < OPTS);
+
+ coder->longest_match_length = mf_find(
+ mf, &coder->matches_count, coder->matches);
+
+ if (coder->longest_match_length >= mf->find_len_max)
+ break;
+
+ len_end = helper2(coder, reps, mf_ptr(mf) - 1, len_end,
+ position + cur, cur, mf->find_len_max,
+ MIN(mf_avail(mf) + 1, OPTS - 1 - cur));
+ }
+
+ backward(coder, len_res, back_res, cur);
+ return;
+}
#include "common.h"
+#define pow2(e) (UINT32_C(1) << (e))
+
+
LZMA_API const lzma_options_lzma lzma_preset_lzma[9] = {
-// dictionary_size lc lp pb mode fb mf mfc
-{ UINT32_C(1) << 16, 3, 0, 2, NULL, 0, LZMA_MODE_FAST, 64, LZMA_MF_HC3, 0 },
-{ UINT32_C(1) << 20, 3, 0, 2, NULL, 0, LZMA_MODE_FAST, 64, LZMA_MF_HC4, 0 },
-{ UINT32_C(1) << 19, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 64, LZMA_MF_BT4, 0 },
-{ UINT32_C(1) << 20, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 64, LZMA_MF_BT4, 0 },
-{ UINT32_C(1) << 21, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 },
-{ UINT32_C(1) << 22, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 },
-{ UINT32_C(1) << 23, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 },
-{ UINT32_C(1) << 24, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 273, LZMA_MF_BT4, 0 },
-{ UINT32_C(1) << 25, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 273, LZMA_MF_BT4, 0 },
+// dict lc lp pb mode fb mf mfc
+{ pow2(16), NULL, 0, 3, 0, 2, false, LZMA_MODE_FAST, 64, LZMA_MF_HC3, 0, 0, 0, 0, 0, NULL, NULL },
+{ pow2(20), NULL, 0, 3, 0, 0, false, LZMA_MODE_FAST, 64, LZMA_MF_HC4, 0, 0, 0, 0, 0, NULL, NULL },
+{ pow2(19), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 64, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL },
+{ pow2(20), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 64, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL },
+{ pow2(21), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 128, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL },
+{ pow2(22), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 128, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL },
+{ pow2(23), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 128, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL },
+{ pow2(24), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 273, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL },
+{ pow2(25), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 273, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL },
};
+
+
+/*
+extern LZMA_API lzma_bool
+lzma_preset_lzma(lzma_options_lzma *options, uint32_t level)
+{
+ *options = (lzma_options_lzma){
+
+ };
+
+ options->literal_context_bits = LZMA_LITERAL_CONTEXT_BITS_DEFAULT
+ options->literal_pos_bits = LZMA_LITERAL_POS_BITS_DEFAULT;
+ options->pos_bits = LZMA_POS_BITS_DEFAULT;
+ options->preset_dictionary = NULL;
+ options->preset_dictionary_size = 0;
+ options->persistent = false;
+
+ options->mode = level <= 2 ? LZMA_MODE_FAST : LZMA_MODE_NORMAL;
+ options->fast_bytes = level <=
+
+ options->match_finder = level == 1 ? LZMA_MF_HC3
+ : (level == 2 ? LZMA_MF_HC4 : LZMA_MF_BT4);
+ options->match_finder_cycles = 0;
+
+
+
+ options->dictionary_size =
+}
+*/
#ifndef LZMA_LZMA_ENCODER_PRIVATE_H
#define LZMA_LZMA_ENCODER_PRIVATE_H
-#include "lzma_encoder.h"
-#include "lzma_common.h"
#include "lz_encoder.h"
#include "range_encoder.h"
+#include "lzma_common.h"
+#include "lzma_encoder.h"
+
+// Macro to compare if the first two bytes in two buffers differ. This is
+// needed in lzma_lzma_optimum_*() to test if the match is at least
+// MATCH_LEN_MIN bytes. Unaligned access gives tiny gain so there's no
+// reason to not use it when it is supported.
+#ifdef HAVE_FAST_UNALIGNED_ACCESS
+# define not_equal_16(a, b) \
+ (*(const uint16_t *)(a) != *(const uint16_t *)(b))
+#else
+# define not_equal_16(a, b) \
+ ((a)[0] != (b)[0] || (a)[1] != (b)[1])
+#endif
-#define move_pos(num) \
-do { \
- assert((int32_t)(num) >= 0); \
- if ((num) != 0) { \
- coder->additional_offset += num; \
- coder->lz.skip(&coder->lz, num); \
- } \
-} while (0)
+
+// Optimal - Number of entries in the optimum array.
+#define OPTS (1 << 12)
typedef struct {
typedef struct {
lzma_lzma_state state;
- bool prev_1_is_char;
+ bool prev_1_is_literal;
bool prev_2;
uint32_t pos_prev_2;
struct lzma_coder_s {
- // Next coder in the chain
- lzma_next_coder next;
-
- // In window and match finder
- lzma_lz_encoder lz;
-
- // Range encoder
+ /// Range encoder
lzma_range_encoder rc;
- // State
+ /// State
lzma_lzma_state state;
- uint8_t previous_byte;
+
+ /// The four most recent match distances
uint32_t reps[REP_DISTANCES];
- // Misc
- uint32_t match_distances[MATCH_MAX_LEN * 2 + 2 + 1];
- uint32_t num_distance_pairs;
- uint32_t additional_offset;
- uint32_t now_pos; // Lowest 32 bits are enough here.
- bool best_compression; ///< True when LZMA_MODE_BEST is used
+ /// Array of match candidates
+ lzma_match matches[MATCH_LEN_MAX + 1];
+
+ /// Number of match candidates in matches[]
+ uint32_t matches_count;
+
+ /// Varibale to hold the length of the longest match between calls
+ /// to lzma_lzma_optimum_*().
+ uint32_t longest_match_length;
+
+ /// True if using getoptimumfast
+ bool fast_mode;
+
+ /// True if the encoder has been initialized by encoding the first
+ /// byte as a literal.
bool is_initialized;
+
+ /// True if the range encoder has been flushed, but not all bytes
+ /// have been written to the output buffer yet.
bool is_flushed;
- bool write_eopm;
- // Literal encoder
- lzma_literal_coder literal_coder;
+ uint32_t pos_mask; ///< (1 << pos_bits) - 1
+ uint32_t literal_context_bits;
+ uint32_t literal_pos_mask;
- // Bit encoders
+ // These are the same as in lzma_decoder.c. See comments there.
+ probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE];
probability is_match[STATES][POS_STATES_MAX];
probability is_rep[STATES];
probability is_rep0[STATES];
probability is_rep1[STATES];
probability is_rep2[STATES];
probability is_rep0_long[STATES][POS_STATES_MAX];
- probability pos_encoders[FULL_DISTANCES - END_POS_MODEL_INDEX];
+ probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS];
+ probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX];
+ probability pos_align[ALIGN_TABLE_SIZE];
- // Bit Tree Encoders
- probability pos_slot_encoder[LEN_TO_POS_STATES][1 << POS_SLOT_BITS];
- probability pos_align_encoder[1 << ALIGN_BITS];
-
- // Length encoders
+ // These are the same as in lzma_decoder.c except that the encoders
+ // include also price tables.
lzma_length_encoder match_len_encoder;
lzma_length_encoder rep_len_encoder;
- lzma_length_encoder *prev_len_encoder;
- // Optimal
- lzma_optimal optimum[OPTS];
- uint32_t optimum_end_index;
- uint32_t optimum_current_index;
- uint32_t longest_match_length;
- bool longest_match_was_found;
-
- // Prices
- uint32_t pos_slot_prices[LEN_TO_POS_STATES][DIST_TABLE_SIZE_MAX];
+ // Price tables
+ uint32_t pos_slot_prices[LEN_TO_POS_STATES][POS_SLOTS];
uint32_t distances_prices[LEN_TO_POS_STATES][FULL_DISTANCES];
- uint32_t align_prices[ALIGN_TABLE_SIZE];
- uint32_t align_price_count;
uint32_t dist_table_size;
uint32_t match_price_count;
- // LZMA specific settings
- uint32_t dictionary_size; ///< Size in bytes
- uint32_t fast_bytes;
- uint32_t pos_state_bits;
- uint32_t pos_mask; ///< (1 << pos_state_bits) - 1
-};
-
-
-extern void lzma_length_encoder_update_table(lzma_length_encoder *lencoder,
- const uint32_t pos_state);
+ uint32_t align_prices[ALIGN_TABLE_SIZE];
+ uint32_t align_price_count;
-extern bool lzma_lzma_encode(lzma_coder *coder, uint8_t *restrict out,
- size_t *restrict out_pos, size_t out_size);
+ // Optimal
+ uint32_t opts_end_index;
+ uint32_t opts_current_index;
+ lzma_optimal opts[OPTS];
+};
-extern void lzma_get_optimum(lzma_coder *restrict coder,
- uint32_t *restrict back_res, uint32_t *restrict len_res);
-extern void lzma_get_optimum_fast(lzma_coder *restrict coder,
+extern void lzma_lzma_optimum_fast(
+ lzma_coder *restrict coder, lzma_mf *restrict mf,
uint32_t *restrict back_res, uint32_t *restrict len_res);
-
-// NOTE: Don't add 'restrict'.
-static inline void
-lzma_read_match_distances(lzma_coder *coder,
- uint32_t *len_res, uint32_t *num_distance_pairs)
-{
- *len_res = 0;
-
- coder->lz.get_matches(&coder->lz, coder->match_distances);
-
- *num_distance_pairs = coder->match_distances[0];
-
- if (*num_distance_pairs > 0) {
- *len_res = coder->match_distances[*num_distance_pairs - 1];
- assert(*len_res <= MATCH_MAX_LEN);
-
- if (*len_res == coder->fast_bytes) {
- uint32_t offset = *len_res - 1;
- const uint32_t distance = coder->match_distances[
- *num_distance_pairs] + 1;
- uint32_t limit = MATCH_MAX_LEN - *len_res;
-
- assert(offset + limit < coder->lz.keep_size_after);
- assert(coder->lz.read_pos <= coder->lz.write_pos);
-
- // If we are close to end of the stream, we may need
- // to limit the length of the match.
- if (coder->lz.write_pos - coder->lz.read_pos
- < offset + limit)
- limit = coder->lz.write_pos
- - (coder->lz.read_pos + offset);
-
- offset += coder->lz.read_pos;
- uint32_t i = 0;
- while (i < limit && coder->lz.buffer[offset + i]
- == coder->lz.buffer[
- offset + i - distance])
- ++i;
-
- *len_res += i;
- }
- }
-
- ++coder->additional_offset;
-
- return;
-}
+extern void lzma_lzma_optimum_normal(lzma_coder *restrict coder,
+ lzma_mf *restrict mf, uint32_t *restrict back_res,
+ uint32_t *restrict len_res, uint32_t position);
#endif
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file lzma_literal.c
-/// \brief Literal Coder
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#include "lzma_literal.h"
-
-
-extern lzma_ret
-lzma_literal_init(lzma_literal_coder *coder,
- uint32_t literal_context_bits, uint32_t literal_pos_bits)
-{
- // Verify that arguments are sane.
- if (literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX
- || literal_pos_bits > LZMA_LITERAL_POS_BITS_MAX)
- return LZMA_HEADER_ERROR;
-
- // Calculate the number of states the literal coder must store.
- const uint32_t states = literal_states(
- literal_pos_bits, literal_context_bits);
-
- // Store the new settings.
- coder->literal_context_bits = literal_context_bits;
- coder->literal_pos_bits = literal_pos_bits;
-
- // Calculate also the literal_pos_mask. It's not changed
- // anywhere else than here.
- coder->literal_pos_mask = (1 << literal_pos_bits) - 1;
-
- // Reset the literal coder.
- for (uint32_t i = 0; i < states; ++i)
- for (uint32_t j = 0; j < LIT_SIZE; ++j)
- bit_reset(coder->coders[i][j]);
-
- return LZMA_OK;
-}
+++ /dev/null
-///////////////////////////////////////////////////////////////////////////////
-//
-/// \file lzma_literal.h
-/// \brief Literal Coder
-///
-/// This is used as is by both LZMA encoder and decoder.
-//
-// Copyright (C) 1999-2006 Igor Pavlov
-// Copyright (C) 2007 Lasse Collin
-//
-// This library is free software; you can redistribute it and/or
-// modify it under the terms of the GNU Lesser General Public
-// License as published by the Free Software Foundation; either
-// version 2.1 of the License, or (at your option) any later version.
-//
-// This library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// Lesser General Public License for more details.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-#ifndef LZMA_LITERAL_H
-#define LZMA_LITERAL_H
-
-#include "common.h"
-
-// We need typedef of `probability'.
-#include "range_common.h"
-
-
-/// Each literal coder is divided in three sections:
-/// - 0x001-0x0FF: Without match byte
-/// - 0x101-0x1FF: With match byte; match bit is 0
-/// - 0x201-0x2FF: With match byte; match bit is 1
-#define LIT_SIZE 0x300
-
-/// Calculate how many states are needed. Each state has
-/// LIT_SIZE `probability' variables.
-#define literal_states(literal_context_bits, literal_pos_bits) \
- (1U << ((literal_context_bits) + (literal_pos_bits)))
-
-/// Locate the literal coder for the next literal byte. The choice depends on
-/// - the lowest literal_pos_bits bits of the position of the current
-/// byte; and
-/// - the highest literal_context_bits bits of the previous byte.
-#define literal_get_subcoder(literal_coder, pos, prev_byte) \
- (literal_coder).coders[(((pos) & (literal_coder).literal_pos_mask) \
- << (literal_coder).literal_context_bits) \
- + ((prev_byte) >> (8 - (literal_coder).literal_context_bits))]
-
-
-typedef struct {
- uint32_t literal_context_bits;
- uint32_t literal_pos_bits;
-
- /// literal_pos_mask is always (1 << literal_pos_bits) - 1.
- uint32_t literal_pos_mask;
-
- /// There are (1 << (literal_pos_bits + literal_context_bits))
- /// literal coders.
- probability coders[1 << LZMA_LITERAL_BITS_MAX][LIT_SIZE];
-
-} lzma_literal_coder;
-
-
-extern lzma_ret lzma_literal_init(
- lzma_literal_coder *coder,
- uint32_t literal_context_bits, uint32_t literal_pos_bits);
-
-#endif
## Lesser General Public License for more details.
##
-EXTRA_DIST = price_table_gen.c
+EXTRA_DIST = price_tablegen.c
noinst_LTLIBRARIES = librangecoder.la
-I@top_srcdir@/src/liblzma/api \
-I@top_srcdir@/src/liblzma/common
-if COND_MAIN_ENCODER
-librangecoder_la_SOURCES += range_encoder.h
+if COND_ENCODER_LZMA
+librangecoder_la_SOURCES += \
+ range_encoder.h \
+ price.h
if COND_SMALL
librangecoder_la_SOURCES += price_table_init.c
else
endif
endif
-if COND_MAIN_DECODER
+if COND_DECODER_LZMA
librangecoder_la_SOURCES += range_decoder.h
endif
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file price.h
+/// \brief Probability price calculation
+//
+// Copyright (C) 1999-2008 Igor Pavlov
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef LZMA_PRICE_H
+#define LZMA_PRICE_H
+
+
+#define RC_MOVE_REDUCING_BITS 4
+#define RC_BIT_PRICE_SHIFT_BITS 4
+#define RC_PRICE_TABLE_SIZE (RC_BIT_MODEL_TOTAL >> RC_MOVE_REDUCING_BITS)
+
+#define RC_INFINITY_PRICE (UINT32_C(1) << 30)
+
+
+#if !defined(LZMA_RANGE_ENCODER_H) || defined(HAVE_SMALL)
+/// Probability prices used by *_get_price() macros. This is initialized
+/// by lzma_rc_init() and is not modified later.
+extern uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE];
+
+/// Initializes lzma_rc_prices[]. This needs to be called only once.
+extern void lzma_rc_init(void);
+
+#else
+// Not building a size optimized version, so we use a precomputed
+// constant table.
+extern const uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE];
+
+#endif
+
+
+static inline uint32_t
+rc_bit_price(const probability prob, const uint32_t bit)
+{
+ return lzma_rc_prices[(prob ^ ((UINT32_C(0) - bit)
+ & (RC_BIT_MODEL_TOTAL - 1))) >> RC_MOVE_REDUCING_BITS];
+}
+
+
+static inline uint32_t
+rc_bit_0_price(const probability prob)
+{
+ return lzma_rc_prices[prob >> RC_MOVE_REDUCING_BITS];
+}
+
+
+static inline uint32_t
+rc_bit_1_price(const probability prob)
+{
+ return lzma_rc_prices[(prob ^ (RC_BIT_MODEL_TOTAL - 1))
+ >> RC_MOVE_REDUCING_BITS];
+}
+
+
+static inline uint32_t
+rc_bittree_price(const probability *const probs,
+ const uint32_t bit_levels, uint32_t symbol)
+{
+ uint32_t price = 0;
+ symbol += UINT32_C(1) << bit_levels;
+
+ do {
+ const uint32_t bit = symbol & 1;
+ symbol >>= 1;
+ price += rc_bit_price(probs[symbol], bit);
+ } while (symbol != 1);
+
+ return price;
+}
+
+
+static inline uint32_t
+rc_bittree_reverse_price(const probability *const probs,
+ uint32_t bit_levels, uint32_t symbol)
+{
+ uint32_t price = 0;
+ uint32_t model_index = 1;
+
+ do {
+ const uint32_t bit = symbol & 1;
+ symbol >>= 1;
+ price += rc_bit_price(probs[model_index], bit);
+ model_index = (model_index << 1) + bit;
+ } while (--bit_levels != 0);
+
+ return price;
+}
+
+
+static inline uint32_t
+rc_direct_price(const uint32_t bits)
+{
+ return bits << RC_BIT_PRICE_SHIFT_BITS;
+}
+
+#endif
-/* This file has been automatically generated by price_table_gen.c. */
+/* This file has been automatically generated by price_tablegen.c. */
#include "range_encoder.h"
-const uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS] = {
- 0, 576, 512, 480, 448, 432, 416, 400,
- 384, 376, 368, 360, 352, 344, 336, 328,
- 320, 316, 312, 308, 304, 300, 296, 292,
- 288, 284, 280, 276, 272, 268, 264, 260,
- 256, 254, 252, 250, 248, 246, 244, 242,
- 240, 238, 236, 234, 232, 230, 228, 226,
- 224, 222, 220, 218, 216, 214, 212, 210,
- 208, 206, 204, 202, 200, 198, 196, 194,
- 192, 191, 190, 189, 188, 187, 186, 185,
- 184, 183, 182, 181, 180, 179, 178, 177,
- 176, 175, 174, 173, 172, 171, 170, 169,
- 168, 167, 166, 165, 164, 163, 162, 161,
- 160, 159, 158, 157, 156, 155, 154, 153,
- 152, 151, 150, 149, 148, 147, 146, 145,
- 144, 143, 142, 141, 140, 139, 138, 137,
- 136, 135, 134, 133, 132, 131, 130, 129,
- 128, 127, 127, 126, 126, 125, 125, 124,
- 124, 123, 123, 122, 122, 121, 121, 120,
- 120, 119, 119, 118, 118, 117, 117, 116,
- 116, 115, 115, 114, 114, 113, 113, 112,
- 112, 111, 111, 110, 110, 109, 109, 108,
- 108, 107, 107, 106, 106, 105, 105, 104,
- 104, 103, 103, 102, 102, 101, 101, 100,
- 100, 99, 99, 98, 98, 97, 97, 96,
- 96, 95, 95, 94, 94, 93, 93, 92,
- 92, 91, 91, 90, 90, 89, 89, 88,
- 88, 87, 87, 86, 86, 85, 85, 84,
- 84, 83, 83, 82, 82, 81, 81, 80,
- 80, 79, 79, 78, 78, 77, 77, 76,
- 76, 75, 75, 74, 74, 73, 73, 72,
- 72, 71, 71, 70, 70, 69, 69, 68,
- 68, 67, 67, 66, 66, 65, 65, 64,
- 64, 63, 63, 63, 63, 62, 62, 62,
- 62, 61, 61, 61, 61, 60, 60, 60,
- 60, 59, 59, 59, 59, 58, 58, 58,
- 58, 57, 57, 57, 57, 56, 56, 56,
- 56, 55, 55, 55, 55, 54, 54, 54,
- 54, 53, 53, 53, 53, 52, 52, 52,
- 52, 51, 51, 51, 51, 50, 50, 50,
- 50, 49, 49, 49, 49, 48, 48, 48,
- 48, 47, 47, 47, 47, 46, 46, 46,
- 46, 45, 45, 45, 45, 44, 44, 44,
- 44, 43, 43, 43, 43, 42, 42, 42,
- 42, 41, 41, 41, 41, 40, 40, 40,
- 40, 39, 39, 39, 39, 38, 38, 38,
- 38, 37, 37, 37, 37, 36, 36, 36,
- 36, 35, 35, 35, 35, 34, 34, 34,
- 34, 33, 33, 33, 33, 32, 32, 32,
- 32, 31, 31, 31, 31, 30, 30, 30,
- 30, 29, 29, 29, 29, 28, 28, 28,
- 28, 27, 27, 27, 27, 26, 26, 26,
- 26, 25, 25, 25, 25, 24, 24, 24,
- 24, 23, 23, 23, 23, 22, 22, 22,
- 22, 21, 21, 21, 21, 20, 20, 20,
- 20, 19, 19, 19, 19, 18, 18, 18,
- 18, 17, 17, 17, 17, 16, 16, 16,
- 16, 15, 15, 15, 15, 14, 14, 14,
- 14, 13, 13, 13, 13, 12, 12, 12,
- 12, 11, 11, 11, 11, 10, 10, 10,
- 10, 9, 9, 9, 9, 8, 8, 8,
- 8, 7, 7, 7, 7, 6, 6, 6,
- 6, 5, 5, 5, 5, 4, 4, 4,
- 4, 3, 3, 3, 3, 2, 2, 2,
- 2, 1, 1, 1, 1, 0, 0, 0
+const uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE] = {
+ 128, 103, 91, 84, 78, 73, 69, 66,
+ 63, 61, 58, 56, 54, 52, 51, 49,
+ 48, 46, 45, 44, 43, 42, 41, 40,
+ 39, 38, 37, 36, 35, 34, 34, 33,
+ 32, 31, 31, 30, 29, 29, 28, 28,
+ 27, 26, 26, 25, 25, 24, 24, 23,
+ 23, 22, 22, 22, 21, 21, 20, 20,
+ 19, 19, 19, 18, 18, 17, 17, 17,
+ 16, 16, 16, 15, 15, 15, 14, 14,
+ 14, 13, 13, 13, 12, 12, 12, 11,
+ 11, 11, 11, 10, 10, 10, 10, 9,
+ 9, 9, 9, 8, 8, 8, 8, 7,
+ 7, 7, 7, 6, 6, 6, 6, 5,
+ 5, 5, 5, 5, 4, 4, 4, 4,
+ 3, 3, 3, 3, 3, 2, 2, 2,
+ 2, 2, 2, 1, 1, 1, 1, 1
};
#endif
-#define NUM_BITS (BIT_MODEL_TOTAL_BITS - MOVE_REDUCING_BITS)
-
-
-uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS];
+uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE];
extern void
lzma_rc_init(void)
{
- // Initialize lzma_rc_prob_prices[].
- for (int i = NUM_BITS - 1; i >= 0; --i) {
- const uint32_t start = 1 << (NUM_BITS - i - 1);
- const uint32_t end = 1 << (NUM_BITS - i);
-
- for (uint32_t j = start; j < end; ++j) {
- lzma_rc_prob_prices[j] = (i << BIT_PRICE_SHIFT_BITS)
- + (((end - j) << BIT_PRICE_SHIFT_BITS)
- >> (NUM_BITS - i - 1));
+ for (uint32_t i = (UINT32_C(1) << RC_MOVE_REDUCING_BITS) / 2;
+ i < RC_BIT_MODEL_TOTAL;
+ i += (UINT32_C(1) << RC_MOVE_REDUCING_BITS)) {
+ const uint32_t cycles_bits = RC_BIT_PRICE_SHIFT_BITS;
+ uint32_t w = i;
+ uint32_t bit_count = 0;
+
+ for (uint32_t j = 0; j < cycles_bits; ++j) {
+ w *= w;
+ bit_count <<= 1;
+
+ while (w >= (UINT32_C(1) << 16)) {
+ w >>= 1;
+ ++bit_count;
+ }
}
+
+ lzma_rc_prices[i >> RC_MOVE_REDUCING_BITS]
+ = (RC_BIT_MODEL_TOTAL_BITS << cycles_bits)
+ - 15 - bit_count;
}
return;
///////////////////////////////////////////////////////////////////////////////
//
-/// \file price_table_gen.c
+/// \file price_tablegen.c
/// \brief Probability price table generator
///
-/// Compiling: gcc -std=c99 -o price_table_gen price_table_gen.c
+/// Compiling: gcc -std=c99 -o price_tablegen price_tablegen.c
//
// Copyright (C) 2007 Lasse Collin
//
//
///////////////////////////////////////////////////////////////////////////////
-#include <sys/types.h>
+#include <stddef.h>
#include <inttypes.h>
#include <stdio.h>
#include "range_common.h"
+#include "price.h"
#include "price_table_init.c"
lzma_rc_init();
printf("/* This file has been automatically generated by "
- "price_table_gen.c. */\n\n"
+ "price_tablegen.c. */\n\n"
"#include \"range_encoder.h\"\n\n"
- "const uint32_t lzma_rc_prob_prices["
- "BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS] = {");
+ "const uint32_t lzma_rc_prices["
+ "RC_PRICE_TABLE_SIZE] = {");
- const size_t array_size = sizeof(lzma_rc_prob_prices)
- / sizeof(lzma_rc_prob_prices[0]);
+ const size_t array_size = sizeof(lzma_rc_prices)
+ / sizeof(lzma_rc_prices[0]);
for (size_t i = 0; i < array_size; ++i) {
if (i % 8 == 0)
printf("\n\t");
- printf("%4" PRIu32, lzma_rc_prob_prices[i]);
+ printf("%4" PRIu32, lzma_rc_prices[i]);
if (i != array_size - 1)
printf(",");
// Constants //
///////////////
-#define SHIFT_BITS 8
-#define TOP_BITS 24
-#define TOP_VALUE (UINT32_C(1) << TOP_BITS)
-#define BIT_MODEL_TOTAL_BITS 11
-#define BIT_MODEL_TOTAL (UINT32_C(1) << BIT_MODEL_TOTAL_BITS)
-#define MOVE_BITS 5
-
-#define MOVE_REDUCING_BITS 2
-#define BIT_PRICE_SHIFT_BITS 6
+#define RC_SHIFT_BITS 8
+#define RC_TOP_BITS 24
+#define RC_TOP_VALUE (UINT32_C(1) << RC_TOP_BITS)
+#define RC_BIT_MODEL_TOTAL_BITS 11
+#define RC_BIT_MODEL_TOTAL (UINT32_C(1) << RC_BIT_MODEL_TOTAL_BITS)
+#define RC_MOVE_BITS 5
////////////
// Resets the probability so that both 0 and 1 have probability of 50 %
#define bit_reset(prob) \
- prob = BIT_MODEL_TOTAL >> 1
+ prob = RC_BIT_MODEL_TOTAL >> 1
// This does the same for a complete bit tree.
// (A tree represented as an array.)
} lzma_range_decoder;
+/// Reads the first five bytes to initialize the range decoder.
static inline bool
rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in,
size_t *restrict in_pos, size_t in_size)
}
-/// Makes local copies of range decoder variables.
-#define rc_to_local(range_decoder) \
+/// Makes local copies of range decoder and *in_pos variables. Doing this
+/// improves speed significantly. The range decoder macros expect also
+/// variables `in' and `in_size' to be defined.
+#define rc_to_local(range_decoder, in_pos) \
lzma_range_decoder rc = range_decoder; \
+ size_t rc_in_pos = (in_pos); \
uint32_t rc_bound
+
/// Stores the local copes back to the range decoder structure.
-#define rc_from_local(range_decoder) \
- range_decoder = rc
+#define rc_from_local(range_decoder, in_pos) \
+do { \
+ range_decoder = rc; \
+ in_pos = rc_in_pos; \
+} while (0)
+
/// Resets the range decoder structure.
#define rc_reset(range_decoder) \
} while (0)
-// All of the macros in this file expect the following variables being defined:
-// - lzma_range_decoder range_decoder;
-// - uint32_t rc_bound; // Temporary variable
-// - uint8_t *in;
-// - size_t in_pos_local; // Local alias for *in_pos
-
+/// When decoding has been properly finished, rc.code is always zero unless
+/// the input stream is corrupt. So checking this can catch some corrupt
+/// files especially if they don't have any other integrity check.
+#define rc_is_finished(range_decoder) \
+ ((range_decoder).code == 0)
-//////////////////
-// Buffer "I/O" //
-//////////////////
-// Read the next byte of compressed data from buffer_in, if needed.
-#define rc_normalize() \
+/// Read the next input byte if needed. If more input is needed but there is
+/// no more input available, "goto out" is used to jump out of the main
+/// decoder loop.
+#define rc_normalize(seq) \
do { \
- if (rc.range < TOP_VALUE) { \
- rc.range <<= SHIFT_BITS; \
- rc.code = (rc.code << SHIFT_BITS) | in[in_pos_local++]; \
+ if (rc.range < RC_TOP_VALUE) { \
+ if (unlikely(rc_in_pos == in_size)) { \
+ coder->sequence = seq; \
+ goto out; \
+ } \
+ rc.range <<= RC_SHIFT_BITS; \
+ rc.code = (rc.code << RC_SHIFT_BITS) | in[rc_in_pos++]; \
} \
} while (0)
-//////////////////
-// Bit decoding //
-//////////////////
-
-// Range decoder's DecodeBit() is splitted into three macros:
-// if_bit_0(prob) {
-// update_bit_0(prob)
-// ...
-// } else {
-// update_bit_1(prob)
-// ...
-// }
-
-#define if_bit_0(prob) \
- rc_normalize(); \
- rc_bound = (rc.range >> BIT_MODEL_TOTAL_BITS) * (prob); \
+/// Start decoding a bit. This must be used together with rc_update_0()
+/// and rc_update_1():
+///
+/// rc_if_0(prob, seq) {
+/// rc_update_0(prob);
+/// // Do something
+/// } else {
+/// rc_update_1(prob);
+/// // Do something else
+/// }
+///
+#define rc_if_0(prob, seq) \
+ rc_normalize(seq); \
+ rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \
if (rc.code < rc_bound)
-#define update_bit_0(prob) \
+/// Update the range decoder state and the used probability variable to
+/// match a decoded bit of 0.
+#define rc_update_0(prob) \
do { \
rc.range = rc_bound; \
- prob += (BIT_MODEL_TOTAL - (prob)) >> MOVE_BITS; \
+ prob += (RC_BIT_MODEL_TOTAL - (prob)) >> RC_MOVE_BITS; \
} while (0)
-#define update_bit_1(prob) \
+/// Update the range decoder state and the used probability variable to
+/// match a decoded bit of 1.
+#define rc_update_1(prob) \
do { \
rc.range -= rc_bound; \
rc.code -= rc_bound; \
- prob -= (prob) >> MOVE_BITS; \
+ prob -= (prob) >> RC_MOVE_BITS; \
} while (0)
-#define rc_decode_direct(dest, count) \
+/// Decodes one bit and runs action0 or action1 depending on the decoded bit.
+/// This macro is used as the last step in bittree reverse decoders since
+/// those don't use "symbol" for anything else than indexing the probability
+/// arrays.
+#define rc_bit_last(prob, action0, action1, seq) \
do { \
- rc_normalize(); \
- rc.range >>= 1; \
- rc.code -= rc.range; \
- rc_bound = UINT32_C(0) - (rc.code >> 31); \
- rc.code += rc.range & rc_bound; \
- dest = (dest << 1) + (rc_bound + 1); \
-} while (--count > 0)
+ rc_if_0(prob, seq) { \
+ rc_update_0(prob); \
+ action0; \
+ } else { \
+ rc_update_1(prob); \
+ action1; \
+ } \
+} while (0)
-// Dummy versions don't update prob or dest.
-#define update_bit_0_dummy() \
- rc.range = rc_bound
+/// Decodes one bit, updates "symbol", and runs action0 or action1 depending
+/// on the decoded bit.
+#define rc_bit(prob, action0, action1, seq) \
+ rc_bit_last(prob, \
+ symbol <<= 1; action0, \
+ symbol = (symbol << 1) + 1; action1, \
+ seq);
-#define update_bit_1_dummy() \
-do { \
- rc.range -= rc_bound; \
- rc.code -= rc_bound; \
-} while (0)
+/// Like rc_bit() but add "case seq:" as a prefix. This makes the unrolled
+/// loops more readable because the code isn't littered with "case"
+/// statements. On the other hand this also makes it less readable, since
+/// spotting the places where the decoder loop may be restarted is less
+/// obvious.
+#define rc_bit_case(prob, action0, action1, seq) \
+ case seq: rc_bit(prob, action0, action1, seq)
-#define rc_decode_direct_dummy(count) \
+/// Decode a bit without using a probability.
+#define rc_direct(dest, seq) \
do { \
- rc_normalize(); \
+ rc_normalize(seq); \
rc.range >>= 1; \
rc.code -= rc.range; \
- rc.code += rc.range & (UINT32_C(0) - (rc.code >> 31)); \
-} while (--count > 0)
-
-
-///////////////////////
-// Bit tree decoding //
-///////////////////////
-
-#define bittree_decode(target, probs, bit_levels) \
-do { \
- uint32_t model_index = 1; \
- for (uint32_t bit_index = (bit_levels); bit_index != 0; --bit_index) { \
- if_bit_0((probs)[model_index]) { \
- update_bit_0((probs)[model_index]); \
- model_index <<= 1; \
- } else { \
- update_bit_1((probs)[model_index]); \
- model_index = (model_index << 1) | 1; \
- } \
- } \
- target += model_index - (1 << bit_levels); \
-} while (0)
-
-
-#define bittree_reverse_decode(target, probs, bit_levels) \
-do { \
- uint32_t model_index = 1; \
- for (uint32_t bit_index = 0; bit_index < bit_levels; ++bit_index) { \
- if_bit_0((probs)[model_index]) { \
- update_bit_0((probs)[model_index]); \
- model_index <<= 1; \
- } else { \
- update_bit_1((probs)[model_index]); \
- model_index = (model_index << 1) | 1; \
- target += 1 << bit_index; \
- } \
- } \
-} while (0)
-
-
-// Dummy versions don't update prob.
-#define bittree_decode_dummy(target, probs, bit_levels) \
-do { \
- uint32_t model_index = 1; \
- for (uint32_t bit_index = (bit_levels); bit_index != 0; --bit_index) { \
- if_bit_0((probs)[model_index]) { \
- update_bit_0_dummy(); \
- model_index <<= 1; \
- } else { \
- update_bit_1_dummy(); \
- model_index = (model_index << 1) | 1; \
- } \
- } \
- target += model_index - (1 << bit_levels); \
+ rc_bound = UINT32_C(0) - (rc.code >> 31); \
+ rc.code += rc.range & rc_bound; \
+ dest = (dest << 1) + (rc_bound + 1); \
} while (0)
-#define bittree_reverse_decode_dummy(probs, bit_levels) \
-do { \
- uint32_t model_index = 1; \
- for (uint32_t bit_index = 0; bit_index < bit_levels; ++bit_index) { \
- if_bit_0((probs)[model_index]) { \
- update_bit_0_dummy(); \
- model_index <<= 1; \
- } else { \
- update_bit_1_dummy(); \
- model_index = (model_index << 1) | 1; \
- } \
- } \
-} while (0)
+// NOTE: No macros are provided for bittree decoding. It seems to be simpler
+// to just write them open in the code.
#endif
#define LZMA_RANGE_ENCODER_H
#include "range_common.h"
+#include "price.h"
/// Maximum number of symbols that can be put pending into lzma_range_encoder
do {
const uint32_t bit = (symbol >> --bit_count) & 1;
rc_bit(rc, &probs[model_index], bit);
- model_index = (model_index << 1) | bit;
+ model_index = (model_index << 1) + bit;
} while (bit_count != 0);
}
const uint32_t bit = symbol & 1;
symbol >>= 1;
rc_bit(rc, &probs[model_index], bit);
- model_index = (model_index << 1) | bit;
+ model_index = (model_index << 1) + bit;
} while (--bit_count != 0);
}
}
++rc->cache_size;
- rc->low = (rc->low & 0x00FFFFFF) << SHIFT_BITS;
+ rc->low = (rc->low & 0x00FFFFFF) << RC_SHIFT_BITS;
return false;
}
rc_encode(lzma_range_encoder *rc,
uint8_t *out, size_t *out_pos, size_t out_size)
{
+ assert(rc->count <= RC_SYMBOLS_MAX);
+
while (rc->pos < rc->count) {
// Normalize
- if (rc->range < TOP_VALUE) {
+ if (rc->range < RC_TOP_VALUE) {
if (rc_shift_low(rc, out, out_pos, out_size))
return true;
- rc->range <<= SHIFT_BITS;
+ rc->range <<= RC_SHIFT_BITS;
}
// Encode a bit
switch (rc->symbols[rc->pos]) {
case RC_BIT_0: {
probability prob = *rc->probs[rc->pos];
- rc->range = (rc->range >> BIT_MODEL_TOTAL_BITS) * prob;
- prob += (BIT_MODEL_TOTAL - prob) >> MOVE_BITS;
+ rc->range = (rc->range >> RC_BIT_MODEL_TOTAL_BITS)
+ * prob;
+ prob += (RC_BIT_MODEL_TOTAL - prob) >> RC_MOVE_BITS;
*rc->probs[rc->pos] = prob;
break;
}
case RC_BIT_1: {
probability prob = *rc->probs[rc->pos];
- const uint32_t bound = prob
- * (rc->range >> BIT_MODEL_TOTAL_BITS);
+ const uint32_t bound = prob * (rc->range
+ >> RC_BIT_MODEL_TOTAL_BITS);
rc->low += bound;
rc->range -= bound;
- prob -= prob >> MOVE_BITS;
+ prob -= prob >> RC_MOVE_BITS;
*rc->probs[rc->pos] = prob;
break;
}
return rc->cache_size + 5 - 1;
}
-
-////////////
-// Prices //
-////////////
-
-#ifdef HAVE_SMALL
-/// Probability prices used by *_get_price() macros. This is initialized
-/// by lzma_rc_init() and is not modified later.
-extern uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS];
-
-/// Initializes lzma_rc_prob_prices[]. This needs to be called only once.
-extern void lzma_rc_init(void);
-
-#else
-// Not building a size optimized version, so we use a precomputed
-// constant table.
-extern const uint32_t
-lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS];
-
-#endif
-
-
-#define bit_get_price(prob, symbol) \
- lzma_rc_prob_prices[((((prob) - (symbol)) ^ (-(symbol))) \
- & (BIT_MODEL_TOTAL - 1)) >> MOVE_REDUCING_BITS]
-
-
-#define bit_get_price_0(prob) \
- lzma_rc_prob_prices[(prob) >> MOVE_REDUCING_BITS]
-
-
-#define bit_get_price_1(prob) \
- lzma_rc_prob_prices[(BIT_MODEL_TOTAL - (prob)) >> MOVE_REDUCING_BITS]
-
-
-static inline uint32_t
-bittree_get_price(const probability *probs,
- uint32_t bit_levels, uint32_t symbol)
-{
- uint32_t price = 0;
- symbol |= UINT32_C(1) << bit_levels;
-
- do {
- price += bit_get_price(probs[symbol >> 1], symbol & 1);
- symbol >>= 1;
- } while (symbol != 1);
-
- return price;
-}
-
-
-static inline uint32_t
-bittree_reverse_get_price(const probability *probs,
- uint32_t bit_levels, uint32_t symbol)
-{
- uint32_t price = 0;
- uint32_t model_index = 1;
-
- do {
- const uint32_t bit = symbol & 1;
- symbol >>= 1;
- price += bit_get_price(probs[model_index], bit);
- model_index = (model_index << 1) | bit;
- } while (--bit_levels != 0);
-
- return price;
-}
-
#endif
simple_coder.h \
simple_private.h
+if COND_ENCODER_SIMPLE
+libsimple_la_SOURCES += \
+ simple_encoder.c \
+ simple_encoder.h
+endif
+
+if COND_DECODER_SIMPLE
+libsimple_la_SOURCES += \
+ simple_decoder.c \
+ simple_decoder.h
+endif
+
if COND_FILTER_X86
libsimple_la_SOURCES += x86.c
endif
assert(!coder->end_was_reached);
if (coder->next.code == NULL) {
- bufcpy(in, in_pos, in_size, out, out_pos, out_size);
+ lzma_bufcpy(in, in_pos, in_size, out, out_pos, out_size);
// Check if end of stream was reached.
if (coder->is_encoder && action == LZMA_FINISH
// Flush already filtered data from coder->buffer[] to out[].
if (coder->pos < coder->filtered) {
- bufcpy(coder->buffer, &coder->pos, coder->filtered,
+ lzma_bufcpy(coder->buffer, &coder->pos, coder->filtered,
out, out_pos, out_size);
// If we couldn't flush all the filtered data, return to
coder->filtered = coder->size;
// Flush as much as possible.
- bufcpy(coder->buffer, &coder->pos, coder->filtered,
+ lzma_bufcpy(coder->buffer, &coder->pos, coder->filtered,
out, out_pos, out_size);
}
static void
simple_coder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->next, allocator);
+ lzma_next_end(&coder->next, allocator);
lzma_free(coder->simple, allocator);
lzma_free(coder, allocator);
return;
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file simple_decoder.c
+/// \brief Properties decoder for simple filters
+//
+// Copyright (C) 2007-2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "simple_decoder.h"
+
+
+extern lzma_ret
+lzma_simple_props_decode(void **options, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size)
+{
+ if (props_size == 0)
+ return LZMA_OK;
+
+ if (props_size != 4)
+ return LZMA_HEADER_ERROR;
+
+ lzma_options_simple *opt = lzma_alloc(
+ sizeof(lzma_options_simple), allocator);
+ if (opt == NULL)
+ return LZMA_MEM_ERROR;
+
+ opt->start_offset = integer_read_32(props);
+
+ // Don't leave an options structure allocated if start_offset is zero.
+ if (opt->start_offset == 0)
+ lzma_free(opt, allocator);
+ else
+ *options = opt;
+
+ return LZMA_OK;
+}
///////////////////////////////////////////////////////////////////////////////
//
-/// \file raw_decoder.h
-/// \brief Raw decoder initialization API
+/// \file simple_decoder.h
+/// \brief Properties decoder for simple filters
//
-// Copyright (C) 2007 Lasse Collin
+// Copyright (C) 2007-2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
//
///////////////////////////////////////////////////////////////////////////////
-#ifndef LZMA_RAW_DECODER_H
-#define LZMA_RAW_DECODER_H
+#ifndef LZMA_SIMPLE_DECODER_H
+#define LZMA_SIMPLE_DECODER_H
-#include "raw_common.h"
+#include "simple_coder.h"
-
-extern lzma_ret lzma_raw_decoder_init(lzma_next_coder *next,
- lzma_allocator *allocator, const lzma_options_filter *options);
+extern lzma_ret lzma_simple_props_decode(
+ void **options, lzma_allocator *allocator,
+ const uint8_t *props, size_t props_size);
#endif
--- /dev/null
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file simple_encoder.c
+/// \brief Properties encoder for simple filters
+//
+// Copyright (C) 2007-2008 Lasse Collin
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "simple_encoder.h"
+
+
+extern lzma_ret
+lzma_simple_props_size(uint32_t *size, const void *options)
+{
+ const lzma_options_simple *const opt = options;
+ *size = (opt == NULL || opt->start_offset == 0) ? 0 : 4;
+ return LZMA_OK;
+}
+
+
+extern lzma_ret
+lzma_simple_props_encode(const void *options, uint8_t *out)
+{
+ const lzma_options_simple *const opt = options;
+
+ // The default start offset is zero, so we don't need to store any
+ // options unless the start offset is non-zero.
+ if (opt == NULL || opt->start_offset == 0)
+ return LZMA_OK;
+
+ integer_write_32(out, opt->start_offset);
+
+ return LZMA_OK;
+}
///////////////////////////////////////////////////////////////////////////////
//
-/// \file raw_encoder.h
-/// \brief Raw encoder initialization API
+/// \file simple_encoder.c
+/// \brief Properties encoder for simple filters
//
-// Copyright (C) 2007 Lasse Collin
+// Copyright (C) 2007-2008 Lasse Collin
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
//
///////////////////////////////////////////////////////////////////////////////
-#ifndef LZMA_RAW_ENCODER_H
-#define LZMA_RAW_ENCODER_H
+#ifndef LZMA_SIMPLE_ENCODER_H
+#define LZMA_SIMPLE_ENCODER_H
-#include "raw_common.h"
+#include "simple_coder.h"
-extern lzma_ret lzma_raw_encoder_init(lzma_next_coder *next,
- lzma_allocator *allocator, const lzma_options_filter *options);
+extern lzma_ret lzma_simple_props_size(uint32_t *size, const void *options);
+
+extern lzma_ret lzma_simple_props_encode(const void *options, uint8_t *out);
#endif
-I@top_srcdir@/src/liblzma/api \
-I@top_srcdir@/src/liblzma/common
-if COND_MAIN_ENCODER
+if COND_ENCODER_SUBBLOCK
libsubblock_la_SOURCES += \
subblock_encoder.c \
subblock_encoder.h
endif
-if COND_MAIN_DECODER
+if COND_DECODER_SUBBLOCK
libsubblock_la_SOURCES += \
subblock_decoder.c \
subblock_decoder.h \
#include "subblock_decoder.h"
#include "subblock_decoder_helper.h"
-#include "raw_decoder.h"
+#include "filter_decoder.h"
/// Maximum number of consecutive Subblocks with Subblock Type Padding
lzma_next_coder filter_flags_decoder;
/// The filter_flags_decoder stores its results here.
- lzma_options_filter filter_flags;
+ lzma_filter filter_flags;
/// Options for the Subblock decoder helper. This is used to tell
/// the helper when it should return LZMA_STREAM_END to the subfilter.
// if Subfilter isn't used again, we could leave
// a memory-hogging filter dangling until someone
// frees Subblock filter itself.
- lzma_next_coder_end(&coder->subfilter, allocator);
+ lzma_next_end(&coder->subfilter, allocator);
// Free memory used for subfilter options. This is
// safe, because we don't support any Subfilter that
coder->helper.end_was_reached = false;
- lzma_options_filter filters[3] = {
+ lzma_filter filters[3] = {
{
.id = coder->filter_flags.id,
.options = coder->filter_flags.options,
in_limit = in_size;
if (coder->subfilter.code == NULL) {
- const size_t copy_size = bufcpy(
+ const size_t copy_size = lzma_bufcpy(
in, in_pos, in_limit,
out, out_pos, out_size);
}
if (coder->subfilter.code == NULL) {
- bufcpy(coder->repeat.buffer,
+ lzma_bufcpy(coder->repeat.buffer,
&coder->repeat.pos,
coder->repeat.size,
out, out_pos, out_size);
static void
subblock_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->next, allocator);
- lzma_next_coder_end(&coder->subfilter, allocator);
- lzma_next_coder_end(&coder->filter_flags_decoder, allocator);
+ lzma_next_end(&coder->next, allocator);
+ lzma_next_end(&coder->subfilter, allocator);
+ lzma_next_end(&coder->filter_flags_decoder, allocator);
lzma_free(coder->filter_flags.options, allocator);
lzma_free(coder, allocator);
return;
next->coder->filter_flags_decoder = LZMA_NEXT_CODER_INIT;
} else {
- lzma_next_coder_end(&next->coder->subfilter, allocator);
+ lzma_next_end(&next->coder->subfilter, allocator);
lzma_free(next->coder->filter_flags.options, allocator);
}
// We can safely copy as much as possible, because we are never
// given more data than a single Subblock Data field.
- bufcpy(in, in_pos, in_size, out, out_pos, out_size);
+ lzma_bufcpy(in, in_pos, in_size, out, out_pos, out_size);
// Return LZMA_STREAM_END when instructed so by the Subblock decoder.
return coder->options->end_was_reached ? LZMA_STREAM_END : LZMA_OK;
///////////////////////////////////////////////////////////////////////////////
#include "subblock_encoder.h"
-#include "raw_encoder.h"
+#include "filter_encoder.h"
/// Maximum number of repeats that a single Repeating Data can indicate.
assert(coder->subfilter.subcoder.code == NULL);
// No Subfilter is enabled, just copy the data as is.
- coder->subblock.in_pending += bufcpy(
+ coder->subblock.in_pending += lzma_bufcpy(
in, in_pos, in_size,
coder->subblock.data,
&coder->subblock.size,
// otherwise the Subfilter's memory could be
// left allocated for long time, and would
// just waste memory.
- lzma_next_coder_end(&coder->subfilter.subcoder,
+ lzma_next_end(&coder->subfilter.subcoder,
allocator);
// We need to flush the currently buffered
break;
case SEQ_RLE_DATA:
- bufcpy(coder->rle.buffer, &coder->pos, coder->rle.size,
+ lzma_bufcpy(coder->rle.buffer, &coder->pos, coder->rle.size,
out, out_pos, out_size);
if (coder->pos < coder->rle.size)
return LZMA_OK;
break;
case SEQ_DATA:
- bufcpy(coder->subblock.data, &coder->pos,
+ lzma_bufcpy(coder->subblock.data, &coder->pos,
coder->subblock.size, out, out_pos, out_size);
if (coder->pos < coder->subblock.size)
return LZMA_OK;
return LZMA_HEADER_ERROR;
// Initialize a raw encoder to work as a Subfilter.
- lzma_options_filter options[2];
+ lzma_filter options[2];
options[0] = coder->options->subfilter_options;
options[1].id = LZMA_VLI_VALUE_UNKNOWN;
// Now we have a big-enough buffer. Encode the Filter Flags.
// Like above, this should never fail.
size_t dummy = 0;
- ret = lzma_filter_flags_encode(coder->subfilter.flags,
- &dummy, coder->subfilter.flags_size, options);
+ ret = lzma_filter_flags_encode(options, coder->subfilter.flags,
+ &dummy, coder->subfilter.flags_size);
assert(ret == LZMA_OK);
assert(dummy == coder->subfilter.flags_size);
if (ret != LZMA_OK || dummy != coder->subfilter.flags_size)
coder->sequence = SEQ_SUBFILTER_FLAGS;
// It is safe to fall through because SEQ_SUBFILTER_FLAGS
- // uses bufcpy() which doesn't write unless there is output
- // space.
+ // uses lzma_bufcpy() which doesn't write unless there is
+ // output space.
}
// Fall through
case SEQ_SUBFILTER_FLAGS:
// Copy the Filter Flags to the output stream.
- bufcpy(coder->subfilter.flags, &coder->pos,
+ lzma_bufcpy(coder->subfilter.flags, &coder->pos,
coder->subfilter.flags_size,
out, out_pos, out_size);
if (coder->pos < coder->subfilter.flags_size)
static void
subblock_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
{
- lzma_next_coder_end(&coder->next, allocator);
- lzma_next_coder_end(&coder->subfilter.subcoder, allocator);
+ lzma_next_end(&coder->next, allocator);
+ lzma_next_end(&coder->subfilter.subcoder, allocator);
lzma_free(coder->subblock.data, allocator);
lzma_free(coder->subfilter.flags, allocator);
lzma_free(coder, allocator);
next->coder->subblock.limit = 0;
next->coder->subfilter.subcoder = LZMA_NEXT_CODER_INIT;
} else {
- lzma_next_coder_end(&next->coder->subfilter.subcoder,
+ lzma_next_end(&next->coder->subfilter.subcoder,
allocator);
lzma_free(next->coder->subfilter.flags, allocator);
}
bool opt_keep_original = false;
bool opt_preserve_name = false;
-lzma_check_type opt_check = LZMA_CHECK_CRC64;
-lzma_options_filter opt_filters[8];
+lzma_check opt_check = LZMA_CHECK_CRC64;
+lzma_filter opt_filters[8];
// We don't modify or free() this, but we need to assign it in some
// non-const pointers.
OPT_SPARC,
OPT_DELTA,
OPT_LZMA,
+ OPT_LZMA2,
OPT_FILES,
OPT_FILES0,
{ "sparc", no_argument, NULL, OPT_SPARC },
{ "delta", optional_argument, NULL, OPT_DELTA },
{ "lzma", optional_argument, NULL, OPT_LZMA },
+ { "lzma2", optional_argument, NULL, OPT_LZMA2 },
// Other
{ "format", required_argument, NULL, 'F' },
break;
case LZMA_FILTER_LZMA:
+ case LZMA_FILTER_LZMA2:
opt_filters[filter_count].options
= parse_options_lzma(opt_str);
break;
add_filter(LZMA_FILTER_LZMA, optarg);
break;
+ case OPT_LZMA2:
+ add_filter(LZMA_FILTER_LZMA2, optarg);
+ break;
+
// Other
// --format
set_compression_settings(void)
{
if (filter_count == 0) {
- opt_filters[0].id = LZMA_FILTER_LZMA;
+ opt_filters[0].id = opt_header == HEADER_ALONE
+ ? LZMA_FILTER_LZMA : LZMA_FILTER_LZMA2;
opt_filters[0].options = (lzma_options_lzma *)(
lzma_preset_lzma + preset_number);
filter_count = 1;
my_exit(ERROR);
}
- const uint32_t memory_limit = opt_memory / (1024 * 1024) + 1;
- uint32_t memory_usage = lzma_memory_usage(opt_filters, true);
+ uint64_t memory_usage = lzma_memusage_encoder(opt_filters);
+ /* opt_mode == MODE_COMPRESS
+ ? lzma_memusage_encoder(opt_filters)
+ : lzma_memusage_decoder(opt_filters); */
// Don't go over the memory limits when the default
// setting is used.
if (preset_default) {
- while (memory_usage > memory_limit) {
+ while (memory_usage > opt_memory) {
if (preset_number == 0) {
errmsg(V_ERROR, _("Memory usage limit is too "
"small for any internal "
opt_filters[0].options = (lzma_options_lzma *)(
lzma_preset_lzma
+ preset_number);
- memory_usage = lzma_memory_usage(opt_filters,
- true);
+ memory_usage = lzma_memusage_encoder(opt_filters);
}
} else {
- if (memory_usage > memory_limit) {
+ if (memory_usage > opt_memory) {
errmsg(V_ERROR, _("Memory usage limit is too small "
"for the given filter setup"));
my_exit(ERROR);
// Limit the number of worked threads so that memory usage
// limit isn't exceeded.
- // FIXME: Probably should use bytes instead of mebibytes for
- // memory_usage and memory_limit.
- if (memory_usage == 0)
- memory_usage = 1;
-
- size_t thread_limit = memory_limit / memory_usage;
+ assert(memory_usage > 0);
+ size_t thread_limit = opt_memory / memory_usage;
if (thread_limit == 0)
thread_limit = 1;
extern enum tool_mode opt_mode;
extern enum header_type opt_header;
-extern lzma_check_type opt_check;
-extern lzma_options_filter opt_filters[8];
+extern lzma_check opt_check;
+extern lzma_filter opt_filters[8];
extern const char *stdin_filename;
parse_options_lzma(const char *str)
{
static const name_id_map modes[] = {
- { "fast", LZMA_MODE_FAST },
- { "best", LZMA_MODE_BEST },
- { NULL, 0 }
+ { "fast", LZMA_MODE_FAST },
+ { "normal", LZMA_MODE_NORMAL },
+ { NULL, 0 }
};
static const name_id_map mfs[] = {
{ "dict", NULL, LZMA_DICTIONARY_SIZE_MIN,
LZMA_DICTIONARY_SIZE_MAX },
{ "lc", NULL, LZMA_LITERAL_CONTEXT_BITS_MIN,
- LZMA_LITERAL_CONTEXT_BITS_MAX },
+ LZMA_LITERAL_CONTEXT_BITS_MAX },
{ "lp", NULL, LZMA_LITERAL_POS_BITS_MIN,
- LZMA_LITERAL_POS_BITS_MAX },
+ LZMA_LITERAL_POS_BITS_MAX },
{ "pb", NULL, LZMA_POS_BITS_MIN, LZMA_POS_BITS_MAX },
{ "mode", modes, 0, 0 },
{ "fb", NULL, LZMA_FAST_BYTES_MIN, LZMA_FAST_BYTES_MAX },
.literal_context_bits = LZMA_LITERAL_CONTEXT_BITS_DEFAULT,
.literal_pos_bits = LZMA_LITERAL_POS_BITS_DEFAULT,
.pos_bits = LZMA_POS_BITS_DEFAULT,
- .mode = LZMA_MODE_BEST,
+ .preset_dictionary = NULL,
+ .persistent = false,
+ .mode = LZMA_MODE_NORMAL,
.fast_bytes = LZMA_FAST_BYTES_DEFAULT,
.match_finder = LZMA_MF_BT4,
.match_finder_cycles = 0,
}
for (size_t i = 0; i < opt_threads; ++i)
- threads[i] = (thread_data){
- .strm = LZMA_STREAM_INIT_VAR,
- .options = NULL,
- .pair = NULL,
- .in_use = false,
- };
+ memzero(&threads[i], sizeof(threads[0]));
if (pthread_attr_init(&thread_attr)
|| pthread_attr_setdetachstate(
}
} else {
// TODO Restrict file format if requested on the command line.
- ret = lzma_auto_decoder(&t->strm);
+ ret = lzma_auto_decoder(&t->strm, opt_memory,
+ LZMA_WARN_UNSUPPORTED_CHECK
+ | LZMA_CONCATENATED);
}
if (ret != LZMA_OK) {
}
-static lzma_ret
-single_skip_padding(thread_data *t, uint8_t *in_buf)
-{
- // Handle decoding of concatenated Streams. There can be arbitrary
- // number of nul-byte padding between the Streams, which must be
- // ignored.
- //
- // NOTE: Concatenating LZMA_Alone files works only if at least
- // one of lc, lp, and pb is non-zero. Using the concatenation
- // on LZMA_Alone files is strongly discouraged.
- while (true) {
- while (t->strm.avail_in > 0) {
- if (*t->strm.next_in != '\0')
- return LZMA_OK;
-
- ++t->strm.next_in;
- --t->strm.avail_in;
- }
-
- if (t->pair->src_eof)
- return LZMA_STREAM_END;
-
- t->strm.next_in = in_buf;
- t->strm.avail_in = io_read(t->pair, in_buf, BUFSIZ);
- if (t->strm.avail_in == SIZE_MAX)
- return LZMA_DATA_ERROR;
- }
-}
-
-
static void *
single(thread_data *t)
{
uint8_t in_buf[BUFSIZ];
uint8_t out_buf[BUFSIZ];
lzma_action action = LZMA_RUN;
- lzma_ret ret;
bool success = false;
t->strm.avail_in = 0;
+ t->strm.next_out = out_buf;
+ t->strm.avail_out = BUFSIZ;
while (!user_abort) {
if (t->strm.avail_in == 0 && !t->pair->src_eof) {
if (t->strm.avail_in == SIZE_MAX)
break;
- else if (t->pair->src_eof
- && opt_mode == MODE_COMPRESS)
+
+ if (t->pair->src_eof)
action = LZMA_FINISH;
}
- t->strm.next_out = out_buf;
- t->strm.avail_out = BUFSIZ;
-
- ret = lzma_code(&t->strm, action);
+ const lzma_ret ret = lzma_code(&t->strm, action);
- if (opt_mode != MODE_TEST)
+ if ((t->strm.avail_out == 0 || ret != LZMA_OK)
+ && opt_mode != MODE_TEST) {
if (io_write(t->pair, out_buf,
BUFSIZ - t->strm.avail_out))
break;
+ t->strm.next_out = out_buf;
+ t->strm.avail_out = BUFSIZ;
+ }
+
if (ret != LZMA_OK) {
if (ret == LZMA_STREAM_END) {
- if (opt_mode == MODE_COMPRESS) {
- assert(t->pair->src_eof);
- success = true;
- break;
- }
-
- // Support decoding concatenated .lzma files.
- ret = single_skip_padding(t, in_buf);
-
- if (ret == LZMA_STREAM_END) {
- assert(t->pair->src_eof);
- success = true;
- break;
- }
-
- if (ret == LZMA_OK && !single_init(t))
- continue;
-
- break;
-
+ // FIXME !!! This doesn't work when decoding
+ // LZMA_Alone files, because LZMA_Alone decoder
+ // doesn't wait for LZMA_FINISH.
+ assert(t->pair->src_eof);
+ success = true;
} else {
errmsg(V_ERROR, "%s: %s", t->pair->src_name,
str_strm_error(ret));
- break;
}
+
+ break;
}
}
static lzma_stream strm = LZMA_STREAM_INIT;
/// Number of bytes to use memory at maximum
-static size_t mem_limit;
-
-/// Memory allocation hooks
-static lzma_allocator allocator = {
- .alloc = (void *(*)(void *, size_t, size_t))(&lzma_memlimit_alloc),
- .free = (void (*)(void *, void *))(&lzma_memlimit_free),
- .opaque = NULL,
-};
+static size_t memlimit;
/// Program name to be shown in error messages
static const char *argv0;
" MiB of memory at maximum.\n"
"\n"
"Report bugs to <" PACKAGE_BUGREPORT "> (in English or Finnish).\n",
- argv0, ((uint64_t)(mem_limit) + 512 * 1024) / (1024 * 1024));
+ argv0, ((uint64_t)(memlimit) + 512 * 1024) / (1024 * 1024));
// Using PRIu64 above instead of %zu to support pre-C99 libc.
exit(0);
}
/// Finds out the amount of physical memory in the system, and sets
/// a default memory usage limit.
static void
-set_default_mem_limit(void)
+set_default_memlimit(void)
{
uint64_t mem = physmem();
if (mem != 0) {
mem = SIZE_MAX;
#endif
- mem_limit = mem / 3;
+ memlimit = mem / 3;
} else {
// Cannot autodetect, use 10 MiB as the default limit.
- mem_limit = (1U << 23) + (1U << 21);
+ memlimit = (1U << 23) + (1U << 21);
}
return;
break;
case 'M':
- mem_limit = str_to_size(optarg);
+ memlimit = str_to_size(optarg);
break;
case 'h':
static void
init(void)
{
+ const uint32_t flags = LZMA_WARN_UNSUPPORTED_CHECK | LZMA_CONCATENATED;
lzma_ret ret;
switch (format_type) {
case FORMAT_AUTO:
- ret = lzma_auto_decoder(&strm);
+ ret = lzma_auto_decoder(&strm, memlimit, flags);
break;
case FORMAT_NATIVE:
- ret = lzma_stream_decoder(&strm);
+ ret = lzma_stream_decoder(&strm, memlimit, flags);
break;
case FORMAT_ALONE:
- ret = lzma_alone_decoder(&strm);
+ ret = lzma_alone_decoder(&strm, memlimit);
break;
default:
static void
-read_input(void)
-{
- strm.next_in = in_buf;
- strm.avail_in = fread(in_buf, 1, BUFSIZ, file);
-
- if (ferror(file)) {
- // POSIX says that fread() sets errno if an error occurred.
- // ferror() doesn't touch errno.
- fprintf(stderr, "%s: %s: Error reading input file: %s\n",
- argv0, filename, strerror(errno));
- exit(ERROR);
- }
-
- return;
-}
-
-
-static bool
-skip_padding(void)
-{
- // Handle concatenated Streams. There can be arbitrary number of
- // nul-byte padding between the Streams, which must be ignored.
- //
- // NOTE: Concatenating LZMA_Alone files works only if at least
- // one of lc, lp, and pb is non-zero. Using the concatenation
- // on LZMA_Alone files is strongly discouraged.
- while (true) {
- while (strm.avail_in > 0) {
- if (*strm.next_in != '\0')
- return true;
-
- ++strm.next_in;
- --strm.avail_in;
- }
-
- if (feof(file))
- return false;
-
- read_input();
- }
-}
-
-
-static void
uncompress(void)
{
if (file == stdin && !force && isatty(STDIN_FILENO)) {
}
init();
+
strm.avail_in = 0;
+ strm.next_out = out_buf;
+ strm.avail_out = BUFSIZ;
+
+ lzma_action action = LZMA_RUN;
while (true) {
- if (strm.avail_in == 0)
- read_input();
+ if (strm.avail_in == 0) {
+ strm.next_in = in_buf;
+ strm.avail_in = fread(in_buf, 1, BUFSIZ, file);
+
+ if (ferror(file)) {
+ // POSIX says that fread() sets errno if
+ // an error occurred. ferror() doesn't
+ // touch errno.
+ fprintf(stderr, "%s: %s: Error reading "
+ "input file: %s\n",
+ argv0, filename,
+ strerror(errno));
+ exit(ERROR);
+ }
- strm.next_out = out_buf;
- strm.avail_out = BUFSIZ;
+ if (feof(file))
+ action = LZMA_FINISH;
+ }
- const lzma_ret ret = lzma_code(&strm, LZMA_RUN);
+ const lzma_ret ret = lzma_code(&strm, action);
// Write and check write error before checking decoder error.
// This way as much data as possible gets written to output
- // even if decoder detected an error. Checking write error
- // needs to be done before checking decoder error due to
- // how concatenated Streams are handled a few lines later.
- const size_t write_size = BUFSIZ - strm.avail_out;
- if (fwrite(out_buf, 1, write_size, stdout) != write_size) {
- // Wouldn't be a surprise if writing to stderr would
- // fail too but at least try to show an error message.
- fprintf(stderr, "%s: Cannot write to "
- "standard output: %s\n", argv0,
- strerror(errno));
- exit(ERROR);
+ // even if decoder detected an error.
+ if (strm.avail_out == 0 || ret != LZMA_OK) {
+ const size_t write_size = BUFSIZ - strm.avail_out;
+
+ if (fwrite(out_buf, 1, write_size, stdout)
+ != write_size) {
+ // Wouldn't be a surprise if writing to stderr
+ // would fail too but at least try to show an
+ // error message.
+ fprintf(stderr, "%s: Cannot write to "
+ "standard output: %s\n", argv0,
+ strerror(errno));
+ exit(ERROR);
+ }
+
+ strm.next_out = out_buf;
+ strm.avail_out = BUFSIZ;
}
if (ret != LZMA_OK) {
- if (ret == LZMA_STREAM_END) {
- if (skip_padding()) {
- init();
- continue;
- }
-
+ // FIXME !!! Doesn't work with LZMA_Alone for the
+ // same reason as in process.c.
+ if (ret == LZMA_STREAM_END)
return;
- }
fprintf(stderr, "%s: %s: ", argv0, filename);
+ // FIXME Add LZMA_*_CHECK and LZMA_FORMAT_ERROR.
switch (ret) {
case LZMA_DATA_ERROR:
fprintf(stderr, "File is corrupt\n");
fprintf(stderr, "%s\n", strerror(ENOMEM));
exit(ERROR);
+ case LZMA_MEMLIMIT_ERROR:
+ fprintf(stderr, "Memory usage limit "
+ "reached\n");
+ exit(ERROR);
+
case LZMA_BUF_ERROR:
fprintf(stderr, "Unexpected end of input\n");
exit(ERROR);
{
argv0 = argv[0];
- set_default_mem_limit();
+ set_default_memlimit();
parse_options(argc, argv);
lzma_init_decoder();
- lzma_memlimit *mem_limiter = lzma_memlimit_create(mem_limit);
- if (mem_limiter == NULL) {
- fprintf(stderr, "%s: %s\n", argv0, strerror(ENOMEM));
- exit(ERROR);
- }
-
- assert(lzma_memlimit_count(mem_limiter) == 0);
-
- allocator.opaque = mem_limiter;
- strm.allocator = &allocator;
-
#ifdef WIN32
setmode(fileno(stdin), O_BINARY);
setmode(fileno(stdout), O_BINARY);
// Free the memory only when debugging. Freeing wastes some time,
// but allows detecting possible memory leaks with Valgrind.
lzma_end(&strm);
- assert(lzma_memlimit_count(mem_limiter) == 0);
- lzma_memlimit_end(mem_limiter, false);
#endif
return exit_status;
test_filter_flags \
test_block_header \
test_index \
+ test_files.sh \
test_compress.sh
clean-local:
1. File Types
Good files (good-*.lzma) must decode successfully without requiring
- a lot of CPU time or RAM. If the decoder supports only Single-Block
- Streams, then good-multi-*.lzma won't decode, of course.
+ a lot of CPU time or RAM.
+
+ Unsupported files (unsupported-*.lzma) are good files, but headers
+ indicate features not supported by the current file format
+ specification.
Bad files (bad-*.lzma) must cause the decoder to give an error. Like
with the good files, these files must not require a lot of CPU time
or RAM before they get detected to be broken.
- Malicious files (malicious-*.lzma) are good in terms of the file format
- specification, but try to trigger excessive CPU, RAM or disk usage in
- the decoder. To prevent malicious files from putting the decoder in
- inifinite loop (*), eating all available RAM or disk space, decoders
- should have internal limiters that catch these situations.
-
- (*) Strictly speaking not infinite, but if decoding of a small file
- would take a few weeks or even years, it's an infinite loop in
- practice.
-
2. Descriptions of Individual Files
2.1. Good Files
- good-single-none.lzma uses implicit Copy filter with known Uncompressed
- Size.
+ good-0-empty.lzma has one Stream with no Blocks.
+
+ good-0pad-empty.lzma has one Stream with no Blocks followed by
+ four-byte Stream Padding.
- good-single-none-pad.lzma is good-single-none.lzma with Footer Padding.
+ good-0cat-empty.lzma has two zero-Block Streams concatenated without
+ Stream Padding.
- good-cat-single-none-pad.lzma is two good-single-none-pad.lzma files
- concatenated as is. Fully decoding this file requires that the decoder
- supports decoding concatenated files.
+ good-0catpad-empty.lzma has two zero-Block Streams concatenated with
+ four-byte Stream Padding between the Streams.
- good-single-subblock_implicit.lzma uses implicit Subblock filter.
+ good-1-check-none.lzma has one Stream with one Block with two
+ uncompressed LZMA2 chunks and no integrity check.
- good-single-lzma.lzma is LZMA compressed file with EOPM.
+ good-1-check-crc32.lzma has one Stream with one Block with two
+ uncompressed LZMA2 chunks and CRC32 check.
- good-single-subblock-lzma.lzma has basic combination of Subblock and
- LZMA filters.
+ good-1-check-crc64.lzma is like good-1-check-crc32.lzma but with CRC64.
- good-single-none-empty_1.lzma is an empty file with implicit Copy
- filter and no integrity Check.
+ good-1-check-sha256.lzma is like good-1-check-crc32.lzma but with
+ SHA256.
- good-single-none-empty_2.lzma is an empty file with implicit Copy
- filter and CRC32 as Check.
+ good-2-lzma2.lzma has one Stream with two Blocks with one uncompressed
+ LZMA2 chunk in each Block.
- good-single-none-empty_3.lzma is an empty file with implicit Copy
- filter, known Compressed Size, and no integrity Check.
+ good-1-block_header-1.lzma has both Compressed Size and Uncompressed
+ Size in the Block Header. This has also four extra bytes of Header
+ Padding.
- good-single-lzma-empty.lzma is an empty file with LZMA filter and no
- integrity Check.
+ good-1-block_header-2.lzma has known Compressed Size.
- good-single-subblock_rle.lzma takes advantage of Subblock filter's
- run-length encoding.
+ good-1-block_header-3.lzma has known Uncompressed Size.
- good-single-delta-lzma.tiff.lzma is an image file that compresses
- better with Delta+LZMA than with plain LZMA.
+ good-1-delta-lzma2.tiff.lzma is an image file that compresses
+ better with Delta+LZMA2 than with plain LZMA2.
- good-single-x86-lzma.lzma uses the x86 filter (BCJ) and LZMA. The
+ good-1-x86-lzma2.lzma uses the x86 filter (BCJ) and LZMA2. The
uncompressed file is compress_prepared_bcj_x86 found from the tests
directory.
- good-single-sparc-lzma.lzma uses the SPARC filter and LZMA. The
+ good-1-sparc-lzma2.lzma uses the SPARC filter and LZMA. The
uncompressed file is compress_prepared_bcj_sparc found from the tests
directory.
- good-single-lzma-flush_1.lzma has a flush marker in the middle of
- the file, and no EOPM.
-
- good-single-lzma-flush_2.lzma has a flush marker in the middle of
- the file and just before EOPM.
-
- good-multi-none-1.lzma is a basic Multi-Block Stream with two Data
- Blocks and Footer Metadata Block.
-
- good-multi-none-2.lzma is good-multi-none-1.lzma with Total Size and
- Uncompressed Size added to the Footer Metadata Block.
-
- good-multi-none-extra_1.lzma has the `Extra is present' flag set but
- no actual Extra Records.
-
- good-multi-none-extra_2.lzma has two non-empty Extra Records.
-
- good-multi-none-extra_3.lzma has an Extra Record that has empty Data.
-
- good-multi-none-header_1.lzma has very minimal Header Metadata Block
- with only the Metadata Flags field.
-
- good-multi-none-header_2.lzma has all information in both Header and
- Footer Metadata Blocks. The Size of Header Metadata Block has wrong
- value in Header Metadata Block, but this value must be ignored by
- the decoder in case of Header Metadata Block.
-
- good-multi-none-header_3.lzma has Index only in the Header Metadata
- Block. Footer Metadata Block contains only Size of Header Metadata
- Block and Total Size.
-
- good-multi-none-block_1.lzma has Index in Header Metadata Block. The
- Compressed Size and Uncompressed Size fields are present in the Data
- Blocks. There is some Footer Padding between the Blocks.
-
- good-multi-none-block_2.lzma has Index in Header Metadata Block. The
- Uncompressed Size field is present in Data Blocks and no EOPM is used.
+ good-1-lzma2-1.lzma has two LZMA2 chunks, of which the second sets
+ new properties.
+ good-1-lzma2-2.lzma has two LZMA2 chunks, of which the second resets
+ the state without specifying new properties.
-2.2. Bad Files
+ good-1-lzma2-3.lzma has two LZMA2 chunks, of which the first is
+ uncompressed and the second is LZMA. The first chunk resets dictionary
+ and the second sets new properties.
- bad-single-none-truncated.lzma is good-single-none.lzma without the
- last byte of the file.
+ good-1-3delta-lzma2.lzma has three Delta filters and LZMA2.
- bad-cat-single-none-pad_garbage_1.lzma is good-cat-single-none-pad.lzma
- with 0xFE appended to the end of the file. 0xFE doesn't begin .lzma
- or LZMA_Alone format file.
- bad-cat-single-none-pad_garbage_2.lzma is good-cat-single-none-pad.lzma
- with 0xFF appended to the end of the file. 0xFF begins .lzma format
- file, thus the decoder has to detect that the file is incomplete.
+2.2. Unsupported Files
- bad-cat-single-none-pad_garbage_3.lzma is good-cat-single-none-pad.lzma
- with 0x5D appended to the end of the file. 0x5D is the most common
- first byte of LZMA_Alone format file.
+ unsupported-check.lzma uses Check ID 0x02 which isn't supported by
+ the current version of the file format. It is implementation-defined
+ how this file handled (it may reject it, or decode it possibly with
+ a warning).
- bad-single-none-footer_filter_flags.lzma has different Stream Flags
- in Stream Footer than in Stream Header.
+ unsupported-block_header.lzma has a non-nul byte in Header Padding,
+ which may indicate presence of a new unsupported field.
- bad-single-none-too_long_vli.lzma has 10-byte variable-length integer.
+ unsupported-filter_flags-1.lzma has unsupported Filter ID 0x7F.
- bad-single-none-empty.lzma is like good-single-none-empty_3.lzma but
- with non-zero value in the Compressed Size field.
+ unsupported-filter_flags-2.lzma specifies only Delta filter in the
+ List of Filter Flags, but Delta isn't allowed as the last filter in
+ the chain. It could be a little more correct to detect this file as
+ corrupt instead of unsupported, but saying it is unsupported is
+ simpler in case of liblzma.
- bad-single-data_after_eopm_1.lzma has LZMA+Subblock, where the Subblock
- filter gives one byte of data to LZMA after LZMA has detected EOPM.
+ unsupported-filter_flags-3.lzma specifies two LZMA2 filters in the
+ List of Filter Flags. LZMA2 is allowed only as the last filter in the
+ chain. It could be a little more correct to detect this file as
+ corrupt instead of unsupported, but saying it is unsupported is
+ simpler in case of liblzma.
- bad-single-data_after_eopm_2.lzma is like
- bad-single-data_after_eopm_1.lzma but Subblock gives 256 MiB of data
- to LZMA after LZMA has detected EOPM.
- bad-single-subblock_subblock.lzma has Subblock+Subblock, where the
- Subblock decoder is given End of Input in the middle of a Subblock.
+2.3. Bad Files
- bad-single-subblock-padding_loop.lzma contains huge amount of
- consecutive Padding bytes, which isn't allowed by the Subblock filter
- format. If it were allowed, this file would hang the decoder for very
- long time (weeks to years).
+ bad-0pad-empty.lzma has one Stream with no Blocks followed by
+ five-byte Stream Padding. Stream Padding must be a multiple of four
+ bytes, thus this file is corrupt.
- bad-single-subblock1023-slow.lzma is similar to
- malicious-single-subblock31-slow.lzma except that this uses 1023 bytes
- of Padding in every place instead of 31 bytes. The Subblock filter
- format specification allows only 31-byte Padings, thus this file must
- get detected as bad without producing any output. Allowing larger
- Padding than 31 bytes was considered (so this test file was created),
- but it seemed to be a bad idea since it would increase worst-case CPU
- usage.
+ bad-0catpad-empty.lzma has two zero-Block Streams concatenated with
+ five-byte Stream Padding between the Streams.
- bad-single-lzma-flush_beginning.lzma has flush marker in the beginning
- of the LZMA data.
+ bad-0cat-alone.lzma is good-0-empty.lzma concatenated with an empty
+ LZMA_Alone file.
- bad-single-lzma-flush_twice.lzma has two flush markers with no data
- between them.
+ bad-0-empty-truncated.lzma is good-0-empty.lzma without the last byte
+ of the file.
- bad-multi-none-1.lzma has data after the last field in the Metadata
- Block and the `Extra is present' flag is not set.
+ bad-0-nonempty_index.lzma has no Blocks but Index claims that there is
+ one Block.
- bad-multi-none-2.lzma has wrong Total Size in Footer Metadata Block.
+ bad-0-backward_size.lzma has wrong Backward Size in Stream Footer.
- bad-multi-none-3.lzma has wrong Uncompressed Size in Footer Metadata
- Block.
+ bad-1-stream_flags-1.lzma has different Stream Flags in Stream Header
+ and Stream Footer.
- bad-multi-none-index_1.lzma has wrong value in the Number of Data
- Blocks field.
+ bad-1-stream_flags-2.lzma has wrong CRC32 in Stream Header.
- bad-multi-none-index_2.lzma has too short Metadata to contain all
- the Index Records.
+ bad-1-stream_flags-3.lzma has wrong CRC32 in Stream Footer.
- bad-multi-none-index_3.lzma has wrong value in Total Size field in
- the Index.
+ bad-1-vli-1.lzma has two-byte variable-length integer in the
+ Uncompressed Size field in Block Header while one-byte would be enough
+ for that value. It's important that the file gets rejected due to too
+ big integer encoding instead of due to Uncompressed Size not matching
+ the value stored in the Block Header. That is, the decoder must not
+ try to decode the Compressed Data field.
- bad-multi-none-index_4.lzma has wrong value in Uncompressed Size field
- in the Index.
+ bad-1-vli-2.lzma has ten-byte variable-length integer as Uncompressed
+ Size in Block Header. It's important that the file gets rejected due
+ to too big integer encoding instead of due to Uncompressed Size not
+ matching the value stored in the Block Header. That is, the decoder
+ must not try to decode the Compressed Data field.
- bad-multi-none-extra_1.lzma has incomplete Extra Record at the end of
- the Metadata Block.
+ bad-1-block_header-1.lzma has Block Header that ends in the middle of
+ the Filter Flags field.
- bad-multi-none-extra_2.lzma has incomplete variable-length integer as
- Extra Record ID.
+ bad-1-block_header-2.lzma has Block Header that has Compressed Size and
+ Uncompressed Size but no List of Filter Flags field.
- bad-multi-none-extra_3.lzma has incomplete Extra Record at the end of
- the Metadata Block.
+ bad-1-block_header-3.lzma has wrong CRC32 in Block Header.
- bad-multi-none-header_1.lzma has empty Header Metadata Block (even
- the Metadata Flags field is not present).
+ bad-1-block_header-4.lzma has too big Compressed Size (2^63 bytes while
+ maximum is 2^63 - 4 bytes) in Block Header. It's important that the
+ file gets rejected due to invalid Compressed Size value; the decoder
+ must not try decoding the Compressed Data field.
- bad-multi-none-header_2.lzma has Index in the Header Metadata Block,
- which describes only one Data Block, while the Stream actually has
- two Data Blocks. A sophisticated decoder should give an error when
- it detects the second Data Block; all Multi-Block decoders must
- detect the file as corrupt at some point.
+ bad-2-index-1.lzma has wrong Total Sizes in Index.
- bad-multi-none-header_3.lzma contains too small Total Size in Header
- Metadata Block. A sophisticated decoder should abort decoding before
- the second Data Block, preferably before the first Data Block has
- been finished; all Multi-Block decoders must detect the file as
- corrupt at some point.
+ bad-2-index-2.lzma has wrong Uncompressed Sizes in Index.
- bad-multi-none-header_4.lzma is like bad-multi-none-header_3.lzma but
- with too small Uncompressed Size.
+ bad-2-index-3.lzma has non-nul byte in Index Padding.
- bad-multi-none-header_5.lzma has Index in the Header Metadata Block,
- but the Total Size field is missing from the Footer Metadata Block.
+ bad-2-index-4.lzma wrong CRC32 in Index.
- bad-multi-none-header_6.lzma has both Index and Total Size in Header
- Metadata Block, but Total Size doesn't match the Index. A sophisticated
- decoder should abort before decoding any Data Blocks; all Multi-Block
- decoders must detect the file as corrupt at some point.
+ bad-2-compressed_data_padding.lzma has non-nul byte in the padding of
+ the Compressed Data field of the first Block.
- bad-multi-none-header_7.lzma has zero as the Size of Header Metadata
- Block in the Header Metadata Block.
+ bad-1-check-crc32.lzma has wrong Check (CRC32).
- bad-multi-none-block_1.lzma has wrong Uncompressed Size in the first
- Data Block. A sophisticated decoder should detect this error before
- producing any output, because it can see that the Uncompressed Size
- doesn't match with the Index in Header Metadata Block; all Multi-Block
- decoders must detect the file as corrupt at some point.
+ bad-1-check-crc64.lzma has wrong Check (CRC64).
- bad-multi-none-block_2.lzma has too big Compressed Size in the first
- Data Block. A sophisticated decoder may be able to detect the file as
- corrupt before producing any output, because Comrpessed Size + size
- of Block Header exceed the Total Size stored in Index in Header
- Metadata Block. A sophisticated decoder should be able to detect the
- error before the end of the first Data Block; all Multi-Block decoders
- must detect the file as corrupt at some point.
+ bad-1-check-sha256.lzma has wrong Check (SHA-256).
- bad-multi-none-block_3.lzma has only the Compressed Size field in the
- Block Header of the second Data Block and EOPM isn't used.
+ bad-1-lzma2-1.lzma has LZMA2 stream whose first chunk (uncompressed)
+ doesn't reset the dictionary.
+ bad-1-lzma2-2.lzma has two LZMA2 chunks, of which the second chunk
+ indicates dictionary reset, but the LZMA compressed data tries to
+ repeat data from the previous chunk.
-2.3. Malicious Files
+ bad-1-lzma2-3.lzma sets new invalid properties (lc=8, lp=0, pb=0) in
+ the middle of Block.
- malicious-single-subblock31-slow.lzma requires quite a bit of CPU time
- per decoded byte. It contains LZMA compressed Subblock filter data that
- has as much Padding as the specification allows. LZMA is also used as
- a Subfilter, to further slowdown the decoder. Every Subfilter instance
- produces only one byte of output. If you can create a file that wastes
- notably more CPU cycles than this file, please contact Lasse Collin.
+ bad-1-lzma2-4.lzma has two LZMA2 chunks, of which the first is
+ uncompressed and the second is LZMA. The first chunk resets dictionary
+ as it should, but the second chunk tries to reset state without
+ specifying properties for LZMA.
- malicious-single-subblock-256MiB.lzma is a tiny file that produces
- 256 MiB of output. It uses Subblock filter's run-length encoding
- to achieve this.
+ bad-1-lzma2-5.lzma is like bad-1-lzma2-4.lzma but doesn't try to reset
+ anything in the header of the second chunk.
- malicious-single-subblock-64PiB.lzma is a tiny file that produces
- 64 PiB of output (if you have patience to wait). This is done by
- chaining two Subblock filters and using their run-length encoders.
+ bad-1-lzma2-6.lzma has reserved LZMA2 control byte value (0x03).
- malicious-multi-metadata-64PiB.lzma is like
- malicious-single-subblock-64PiB.lzma but the huge amount of output
- is in a Metadata Block. Trying to decode this file may take years
- unless the decoder catches that the Metadata has unreasonable size.
+ bad-1-lzma2-7.lzma has EOPM at LZMA level.
static uint8_t buf[LZMA_BLOCK_HEADER_SIZE_MAX];
-static lzma_options_block known_options;
-static lzma_options_block decoded_options;
+static lzma_block known_options;
+static lzma_block decoded_options;
-static lzma_options_filter filters_none[1] = {
+static lzma_filter filters_none[1] = {
{
.id = LZMA_VLI_VALUE_UNKNOWN,
},
};
-static lzma_options_filter filters_one[2] = {
+static lzma_filter filters_one[2] = {
{
- .id = LZMA_FILTER_LZMA,
+ .id = LZMA_FILTER_LZMA2,
.options = (void *)(&lzma_preset_lzma[0]),
}, {
.id = LZMA_VLI_VALUE_UNKNOWN,
};
-static lzma_options_filter filters_four[5] = {
+static lzma_filter filters_four[5] = {
{
.id = LZMA_FILTER_X86,
.options = NULL,
.id = LZMA_FILTER_X86,
.options = NULL,
}, {
- .id = LZMA_FILTER_LZMA,
+ .id = LZMA_FILTER_LZMA2,
.options = (void *)(&lzma_preset_lzma[0]),
}, {
.id = LZMA_VLI_VALUE_UNKNOWN,
};
-static lzma_options_filter filters_five[6] = {
+static lzma_filter filters_five[6] = {
{
.id = LZMA_FILTER_X86,
.options = NULL,
.id = LZMA_FILTER_X86,
.options = NULL,
}, {
- .id = LZMA_FILTER_LZMA,
+ .id = LZMA_FILTER_LZMA2,
.options = (void *)(&lzma_preset_lzma[0]),
}, {
.id = LZMA_VLI_VALUE_UNKNOWN,
{
expect(lzma_block_header_encode(&known_options, buf) == LZMA_OK);
- lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
+ lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
memcrap(filters, sizeof(filters));
memcrap(&decoded_options, sizeof(decoded_options));
static void
test1(void)
{
- known_options = (lzma_options_block){
+ known_options = (lzma_block){
.check = LZMA_CHECK_NONE,
.compressed_size = LZMA_VLI_VALUE_UNKNOWN,
.uncompressed_size = LZMA_VLI_VALUE_UNKNOWN,
static void
test2(void)
{
- known_options = (lzma_options_block){
+ known_options = (lzma_block){
.check = LZMA_CHECK_CRC32,
.compressed_size = LZMA_VLI_VALUE_UNKNOWN,
.uncompressed_size = LZMA_VLI_VALUE_UNKNOWN,
static void
test3(void)
{
- known_options = (lzma_options_block){
+ known_options = (lzma_block){
.check = LZMA_CHECK_CRC32,
.compressed_size = LZMA_VLI_VALUE_UNKNOWN,
.uncompressed_size = LZMA_VLI_VALUE_UNKNOWN,
known_options.header_size += 4;
expect(lzma_block_header_encode(&known_options, buf) == LZMA_OK);
- lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
+ lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1];
decoded_options.header_size = known_options.header_size;
decoded_options.check = known_options.check;
decoded_options.filters = filters;
--armthumb \
--sparc
do
- test_lzma $ARGS --lzma=dict=64KiB,fb=32,mode=fast
- test_lzma --subblock $ARGS --lzma=dict=64KiB,fb=32,mode=fast
+ test_lzma $ARGS --lzma2=dict=64KiB,fb=32,mode=fast
+ test_lzma --subblock $ARGS --lzma2=dict=64KiB,fb=32,mode=fast
done
echo
static uint8_t buffer[4096];
-static lzma_options_filter known_flags;
-static lzma_options_filter decoded_flags;
+static lzma_filter known_flags;
+static lzma_filter decoded_flags;
static lzma_stream strm = LZMA_STREAM_INIT;
return true;
size_t out_pos = 0;
- if (lzma_filter_flags_encode(buffer, &out_pos, known_size,
- &known_flags) != LZMA_OK)
+ if (lzma_filter_flags_encode(&known_flags,
+ buffer, &out_pos, known_size) != LZMA_OK)
return true;
if (out_pos != known_size)
}
-#ifdef HAVE_FILTER_SUBBLOCK
+#if defined(HAVE_ENCODER_SUBBLOCK) && defined(HAVE_DECODER_SUBBLOCK)
static void
test_subblock(void)
{
// Test 1
known_flags.id = LZMA_FILTER_SUBBLOCK;
known_flags.options = NULL;
-
expect(!encode(2));
expect(!decode(2));
- expect(decoded_flags.options != NULL);
- expect(((lzma_options_subblock *)(decoded_flags.options))
- ->allow_subfilters);
+ expect(decoded_flags.options == NULL);
// Test 2
- known_flags.options = decoded_flags.options;
- expect(!encode(2));
- expect(!decode(2));
- expect(decoded_flags.options != NULL);
- expect(((lzma_options_subblock *)(decoded_flags.options))
- ->allow_subfilters);
-
- free(decoded_flags.options);
- free(known_flags.options);
-
- // Test 3
buffer[0] = LZMA_FILTER_SUBBLOCK;
buffer[1] = 1;
buffer[2] = 0;
#endif
-#ifdef HAVE_FILTER_SIMPLE
+#if defined(HAVE_ENCODER_X86) && defined(HAVE_DECODER_X86)
static void
test_simple(void)
{
#endif
-#ifdef HAVE_FILTER_DELTA
+#if defined(HAVE_ENCODER_DELTA) && defined(HAVE_DECODER_DELTA)
static void
test_delta(void)
{
expect(encode(99));
// Test 2
- lzma_options_delta options = { 0 };
+ lzma_options_delta options = {
+ .type = LZMA_DELTA_TYPE_BYTE,
+ .distance = 0
+ };
known_flags.options = &options;
expect(encode(99));
}
#endif
-
+/*
#ifdef HAVE_FILTER_LZMA
static void
validate_lzma(void)
}
}
#endif
-
+*/
int
main(void)
{
lzma_init();
-#ifdef HAVE_FILTER_SUBBLOCK
+#if defined(HAVE_ENCODER_SUBBLOCK) && defined(HAVE_DECODER_SUBBLOCK)
test_subblock();
#endif
-#ifdef HAVE_FILTER_SIMPLE
+#if defined(HAVE_ENCODER_X86) && defined(HAVE_DECODER_X86)
test_simple();
#endif
-#ifdef HAVE_FILTER_DELTA
+#if defined(HAVE_ENCODER_DELTA) && defined(HAVE_DECODER_DELTA)
test_delta();
#endif
-#ifdef HAVE_FILTER_LZMA
- test_lzma();
-#endif
+// #ifdef HAVE_FILTER_LZMA
+// test_lzma();
+// #endif
lzma_end(&strm);
expect(lzma_stream_footer_encode(&known_flags, buffer)
== LZMA_PROG_ERROR);
- known_flags.check = (lzma_check_type)(-1);
+ known_flags.check = (lzma_check)(-1);
expect(lzma_stream_header_encode(&known_flags, buffer)
== LZMA_PROG_ERROR);
// Valid headers
known_flags.backward_size = 1024;
- for (lzma_check_type check = LZMA_CHECK_NONE;
+ for (lzma_check check = LZMA_CHECK_NONE;
check <= LZMA_CHECK_ID_MAX; ++check) {
test_header();
test_footer();
str = "LZMA_HEADER_ERROR";
break;
+ case LZMA_NO_CHECK:
+ str = "LZMA_NO_CHECK";
+ break;
+
case LZMA_UNSUPPORTED_CHECK:
str = "LZMA_UNSUPPORTED_CHECK";
break;
+ case LZMA_SEE_CHECK:
+ str = "LZMA_SEE_CHECK";
+ break;
+
case LZMA_FORMAT_ERROR:
str = "LZMA_FORMAT_ERROR";
break;
projects / platform / upstream / xz.git / commitdiff