2 * libdpkg - Debian packaging suite library routines
3 * compress.c - compression support functions
5 * Copyright © 2000 Wichert Akkerman <wakkerma@debian.org>
6 * Copyright © 2004 Scott James Remnant <scott@netsplit.com>
7 * Copyright © 2006-2012 Guillem Jover <guillem@debian.org>
9 * This is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
14 * This is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with this program. If not, see <http://www.gnu.org/licenses/>.
42 #include <dpkg/i18n.h>
43 #include <dpkg/dpkg.h>
44 #include <dpkg/error.h>
45 #include <dpkg/varbuf.h>
46 #include <dpkg/fdio.h>
47 #include <dpkg/buffer.h>
48 #include <dpkg/command.h>
49 #include <dpkg/compress.h>
50 #if !defined(WITH_ZLIB) || !defined(WITH_LIBLZMA) || !defined(WITH_BZ2)
51 #include <dpkg/subproc.h>
53 static void DPKG_ATTR_SENTINEL
54 fd_fd_filter(int fd_in, int fd_out, const char *desc, const char *file, ...)
71 command_init(&cmd, file, desc);
72 command_add_arg(&cmd, file);
74 command_add_argv(&cmd, args);
79 subproc_wait_check(pid, desc, 0);
85 const char *extension;
87 void (*fixup_params)(struct compress_params *params);
88 void (*compress)(int fd_in, int fd_out, struct compress_params *params,
90 void (*decompress)(int fd_in, int fd_out, const char *desc);
94 * No compressor (pass-through).
98 fixup_none_params(struct compress_params *params)
103 decompress_none(int fd_in, int fd_out, const char *desc)
105 struct dpkg_error err;
107 if (fd_fd_copy(fd_in, fd_out, -1, &err) < 0)
108 ohshit(_("%s: pass-through copy error: %s"), desc, err.str);
112 compress_none(int fd_in, int fd_out, struct compress_params *params, const char *desc)
114 struct dpkg_error err;
116 if (fd_fd_copy(fd_in, fd_out, -1, &err) < 0)
117 ohshit(_("%s: pass-through copy error: %s"), desc, err.str);
120 static const struct compressor compressor_none = {
124 .fixup_params = fixup_none_params,
125 .compress = compress_none,
126 .decompress = decompress_none,
136 fixup_gzip_params(struct compress_params *params)
138 /* Normalize compression level. */
139 if (params->level == 0)
140 params->type = compressor_type_none;
145 decompress_gzip(int fd_in, int fd_out, const char *desc)
147 char buffer[DPKG_BUFFER_SIZE];
148 gzFile gzfile = gzdopen(fd_in, "r");
151 ohshit(_("%s: error binding input to gzip stream"), desc);
154 int actualread, actualwrite;
156 actualread = gzread(gzfile, buffer, sizeof(buffer));
157 if (actualread < 0) {
159 const char *errmsg = gzerror(gzfile, &z_errnum);
161 if (z_errnum == Z_ERRNO)
162 errmsg = strerror(errno);
163 ohshit(_("%s: internal gzip read error: '%s'"), desc,
166 if (actualread == 0) /* EOF. */
169 actualwrite = fd_write(fd_out, buffer, actualread);
170 if (actualwrite != actualread)
171 ohshite(_("%s: internal gzip write error"), desc);
175 ohshite(_("%s: internal gzip write error"), desc);
179 compress_gzip(int fd_in, int fd_out, struct compress_params *params, const char *desc)
181 char buffer[DPKG_BUFFER_SIZE];
186 snprintf(combuf, sizeof(combuf), "w%d", params->level);
187 gzfile = gzdopen(fd_out, combuf);
189 ohshit(_("%s: error binding output to gzip stream"), desc);
192 int actualread, actualwrite;
194 actualread = fd_read(fd_in, buffer, sizeof(buffer));
196 ohshite(_("%s: internal gzip read error"), desc);
197 if (actualread == 0) /* EOF. */
200 actualwrite = gzwrite(gzfile, buffer, actualread);
201 if (actualwrite != actualread) {
202 const char *errmsg = gzerror(gzfile, &z_errnum);
204 if (z_errnum == Z_ERRNO)
205 errmsg = strerror(errno);
206 ohshit(_("%s: internal gzip write error: '%s'"), desc,
211 z_errnum = gzclose(gzfile);
215 if (z_errnum == Z_ERRNO)
216 errmsg = strerror(errno);
218 errmsg = zError(z_errnum);
219 ohshit(_("%s: internal gzip write error: %s"), desc, errmsg);
224 decompress_gzip(int fd_in, int fd_out, const char *desc)
226 fd_fd_filter(fd_in, fd_out, desc, GZIP, "-dc", NULL);
230 compress_gzip(int fd_in, int fd_out, struct compress_params *params, const char *desc)
234 snprintf(combuf, sizeof(combuf), "-c%d", params->level);
235 fd_fd_filter(fd_in, fd_out, desc, GZIP, combuf, NULL);
239 static const struct compressor compressor_gzip = {
243 .fixup_params = fixup_gzip_params,
244 .compress = compress_gzip,
245 .decompress = decompress_gzip,
252 #define BZIP2 "bzip2"
255 fixup_bzip2_params(struct compress_params *params)
257 /* Normalize compression level. */
258 if (params->level == 0)
264 decompress_bzip2(int fd_in, int fd_out, const char *desc)
266 char buffer[DPKG_BUFFER_SIZE];
267 BZFILE *bzfile = BZ2_bzdopen(fd_in, "r");
270 ohshit(_("%s: error binding input to bzip2 stream"), desc);
273 int actualread, actualwrite;
275 actualread = BZ2_bzread(bzfile, buffer, sizeof(buffer));
276 if (actualread < 0) {
278 const char *errmsg = BZ2_bzerror(bzfile, &bz_errnum);
280 if (bz_errnum == BZ_IO_ERROR)
281 errmsg = strerror(errno);
282 ohshit(_("%s: internal bzip2 read error: '%s'"), desc,
285 if (actualread == 0) /* EOF. */
288 actualwrite = fd_write(fd_out, buffer, actualread);
289 if (actualwrite != actualread)
290 ohshite(_("%s: internal bzip2 write error"), desc);
294 ohshite(_("%s: internal bzip2 write error"), desc);
298 compress_bzip2(int fd_in, int fd_out, struct compress_params *params, const char *desc)
300 char buffer[DPKG_BUFFER_SIZE];
305 snprintf(combuf, sizeof(combuf), "w%d", params->level);
306 bzfile = BZ2_bzdopen(fd_out, combuf);
308 ohshit(_("%s: error binding output to bzip2 stream"), desc);
311 int actualread, actualwrite;
313 actualread = fd_read(fd_in, buffer, sizeof(buffer));
315 ohshite(_("%s: internal bzip2 read error"), desc);
316 if (actualread == 0) /* EOF. */
319 actualwrite = BZ2_bzwrite(bzfile, buffer, actualread);
320 if (actualwrite != actualread) {
321 const char *errmsg = BZ2_bzerror(bzfile, &bz_errnum);
323 if (bz_errnum == BZ_IO_ERROR)
324 errmsg = strerror(errno);
325 ohshit(_("%s: internal bzip2 write error: '%s'"), desc,
330 BZ2_bzWriteClose(&bz_errnum, bzfile, 0, NULL, NULL);
331 if (bz_errnum != BZ_OK) {
332 const char *errmsg = _("unexpected bzip2 error");
334 if (bz_errnum == BZ_IO_ERROR)
335 errmsg = strerror(errno);
336 ohshit(_("%s: internal bzip2 write error: '%s'"), desc,
340 /* Because BZ2_bzWriteClose has done a fflush on the file handle,
341 * doing a close on the file descriptor associated with it should
344 ohshite(_("%s: internal bzip2 write error"), desc);
348 decompress_bzip2(int fd_in, int fd_out, const char *desc)
350 fd_fd_filter(fd_in, fd_out, desc, BZIP2, "-dc", NULL);
354 compress_bzip2(int fd_in, int fd_out, struct compress_params *params, const char *desc)
358 snprintf(combuf, sizeof(combuf), "-c%d", params->level);
359 fd_fd_filter(fd_in, fd_out, desc, BZIP2, combuf, NULL);
363 static const struct compressor compressor_bzip2 = {
367 .fixup_params = fixup_bzip2_params,
368 .compress = compress_bzip2,
369 .decompress = decompress_bzip2,
379 enum dpkg_stream_status {
380 DPKG_STREAM_INIT = DPKG_BIT(1),
381 DPKG_STREAM_RUN = DPKG_BIT(2),
382 DPKG_STREAM_COMPRESS = DPKG_BIT(3),
383 DPKG_STREAM_DECOMPRESS = DPKG_BIT(4),
384 DPKG_STREAM_FILTER = DPKG_STREAM_COMPRESS | DPKG_STREAM_DECOMPRESS,
387 /* XXX: liblzma does not expose error messages. */
389 dpkg_lzma_strerror(lzma_ret code, enum dpkg_stream_status status)
391 const char *const impossible = _("internal error (bug)");
395 return strerror(ENOMEM);
396 case LZMA_MEMLIMIT_ERROR:
397 if (status & DPKG_STREAM_RUN)
398 return _("memory usage limit reached");
400 case LZMA_OPTIONS_ERROR:
401 if (status == (DPKG_STREAM_INIT | DPKG_STREAM_COMPRESS))
402 return _("unsupported compression preset");
403 if (status == (DPKG_STREAM_RUN | DPKG_STREAM_DECOMPRESS))
404 return _("unsupported options in file header");
406 case LZMA_DATA_ERROR:
407 if (status & DPKG_STREAM_RUN)
408 return _("compressed data is corrupt");
411 if (status & DPKG_STREAM_RUN)
412 return _("unexpected end of input");
414 case LZMA_FORMAT_ERROR:
415 if (status == (DPKG_STREAM_RUN | DPKG_STREAM_DECOMPRESS))
416 return _("file format not recognized");
418 case LZMA_UNSUPPORTED_CHECK:
419 if (status == (DPKG_STREAM_INIT | DPKG_STREAM_COMPRESS))
420 return _("unsupported type of integrity check");
430 struct compress_params *params;
431 enum dpkg_stream_status status;
434 void (*init)(struct io_lzma *io, lzma_stream *s);
435 int (*code)(struct io_lzma *io, lzma_stream *s);
436 void (*done)(struct io_lzma *io, lzma_stream *s);
440 filter_lzma(struct io_lzma *io, int fd_in, int fd_out)
442 uint8_t buf_in[DPKG_BUFFER_SIZE];
443 uint8_t buf_out[DPKG_BUFFER_SIZE];
444 lzma_stream s = LZMA_STREAM_INIT;
447 s.next_out = buf_out;
448 s.avail_out = sizeof(buf_out);
450 io->action = LZMA_RUN;
451 io->status = DPKG_STREAM_INIT;
453 io->status = (io->status & DPKG_STREAM_FILTER) | DPKG_STREAM_RUN;
458 if (s.avail_in == 0 && io->action != LZMA_FINISH) {
459 len = fd_read(fd_in, buf_in, sizeof(buf_in));
461 ohshite(_("%s: lzma read error"), io->desc);
463 io->action = LZMA_FINISH;
468 ret = io->code(io, &s);
470 if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
471 len = fd_write(fd_out, buf_out, s.next_out - buf_out);
473 ohshite(_("%s: lzma write error"), io->desc);
474 s.next_out = buf_out;
475 s.avail_out = sizeof(buf_out);
477 } while (ret != LZMA_STREAM_END);
482 ohshite(_("%s: lzma close error"), io->desc);
486 filter_lzma_error(struct io_lzma *io, lzma_ret ret)
488 ohshit(_("%s: lzma error: %s"), io->desc,
489 dpkg_lzma_strerror(ret, io->status));
493 filter_unxz_init(struct io_lzma *io, lzma_stream *s)
495 uint64_t memlimit = UINT64_MAX;
498 io->status |= DPKG_STREAM_DECOMPRESS;
500 ret = lzma_stream_decoder(s, memlimit, 0);
502 filter_lzma_error(io, ret);
506 filter_xz_init(struct io_lzma *io, lzma_stream *s)
511 io->status |= DPKG_STREAM_COMPRESS;
513 preset = io->params->level;
514 if (io->params->strategy == compressor_strategy_extreme)
515 preset |= LZMA_PRESET_EXTREME;
516 ret = lzma_easy_encoder(s, preset, LZMA_CHECK_CRC32);
518 filter_lzma_error(io, ret);
522 filter_lzma_code(struct io_lzma *io, lzma_stream *s)
526 ret = lzma_code(s, io->action);
527 if (ret != LZMA_OK && ret != LZMA_STREAM_END)
528 filter_lzma_error(io, ret);
534 filter_lzma_done(struct io_lzma *io, lzma_stream *s)
540 decompress_xz(int fd_in, int fd_out, const char *desc)
544 io.init = filter_unxz_init;
545 io.code = filter_lzma_code;
546 io.done = filter_lzma_done;
549 filter_lzma(&io, fd_in, fd_out);
553 compress_xz(int fd_in, int fd_out, struct compress_params *params, const char *desc)
557 io.init = filter_xz_init;
558 io.code = filter_lzma_code;
559 io.done = filter_lzma_done;
563 filter_lzma(&io, fd_in, fd_out);
567 decompress_xz(int fd_in, int fd_out, const char *desc)
569 fd_fd_filter(fd_in, fd_out, desc, XZ, "-dc", NULL);
573 compress_xz(int fd_in, int fd_out, struct compress_params *params, const char *desc)
576 const char *strategy;
578 if (params->strategy == compressor_strategy_extreme)
583 snprintf(combuf, sizeof(combuf), "-c%d", params->level);
584 fd_fd_filter(fd_in, fd_out, desc, XZ, combuf, strategy, NULL);
588 static const struct compressor compressor_xz = {
592 .fixup_params = fixup_none_params,
593 .compress = compress_xz,
594 .decompress = decompress_xz,
603 filter_unlzma_init(struct io_lzma *io, lzma_stream *s)
605 uint64_t memlimit = UINT64_MAX;
608 io->status |= DPKG_STREAM_DECOMPRESS;
610 ret = lzma_alone_decoder(s, memlimit);
612 filter_lzma_error(io, ret);
616 filter_lzma_init(struct io_lzma *io, lzma_stream *s)
619 lzma_options_lzma options;
622 io->status |= DPKG_STREAM_COMPRESS;
624 preset = io->params->level;
625 if (io->params->strategy == compressor_strategy_extreme)
626 preset |= LZMA_PRESET_EXTREME;
627 if (lzma_lzma_preset(&options, preset))
628 filter_lzma_error(io, LZMA_OPTIONS_ERROR);
630 ret = lzma_alone_encoder(s, &options);
632 filter_lzma_error(io, ret);
636 decompress_lzma(int fd_in, int fd_out, const char *desc)
640 io.init = filter_unlzma_init;
641 io.code = filter_lzma_code;
642 io.done = filter_lzma_done;
645 filter_lzma(&io, fd_in, fd_out);
649 compress_lzma(int fd_in, int fd_out, struct compress_params *params, const char *desc)
653 io.init = filter_lzma_init;
654 io.code = filter_lzma_code;
655 io.done = filter_lzma_done;
659 filter_lzma(&io, fd_in, fd_out);
663 decompress_lzma(int fd_in, int fd_out, const char *desc)
665 fd_fd_filter(fd_in, fd_out, desc, XZ, "-dc", "--format=lzma", NULL);
669 compress_lzma(int fd_in, int fd_out, struct compress_params *params, const char *desc)
673 snprintf(combuf, sizeof(combuf), "-c%d", params->level);
674 fd_fd_filter(fd_in, fd_out, desc, XZ, combuf, "--format=lzma", NULL);
678 static const struct compressor compressor_lzma = {
680 .extension = ".lzma",
682 .fixup_params = fixup_none_params,
683 .compress = compress_lzma,
684 .decompress = decompress_lzma,
688 * Generic compressor filter.
691 static const struct compressor *compressor_array[] = {
692 [compressor_type_none] = &compressor_none,
693 [compressor_type_gzip] = &compressor_gzip,
694 [compressor_type_xz] = &compressor_xz,
695 [compressor_type_bzip2] = &compressor_bzip2,
696 [compressor_type_lzma] = &compressor_lzma,
699 static const struct compressor *
700 compressor(enum compressor_type type)
702 const enum compressor_type max_type = array_count(compressor_array);
704 if (type < 0 || type >= max_type)
705 internerr("compressor_type %d is out of range", type);
707 return compressor_array[type];
711 compressor_get_extension(enum compressor_type type)
713 return compressor(type)->extension;
717 compressor_find_by_name(const char *name)
721 for (i = 0; i < array_count(compressor_array); i++)
722 if (strcmp(compressor_array[i]->name, name) == 0)
725 return compressor_type_unknown;
729 compressor_find_by_extension(const char *extension)
733 for (i = 0; i < array_count(compressor_array); i++)
734 if (strcmp(compressor_array[i]->extension, extension) == 0)
737 return compressor_type_unknown;
740 enum compressor_strategy
741 compressor_get_strategy(const char *name)
743 if (strcmp(name, "none") == 0)
744 return compressor_strategy_none;
745 if (strcmp(name, "extreme") == 0)
746 return compressor_strategy_extreme;
748 return compressor_strategy_unknown;
752 compressor_check_params(struct compress_params *params, struct dpkg_error *err)
754 if (params->strategy == compressor_strategy_none)
757 if (params->type == compressor_type_xz &&
758 params->strategy == compressor_strategy_extreme)
761 dpkg_put_error(err, _("unknown compression strategy"));
766 compressor_fixup_params(struct compress_params *params)
768 compressor(params->type)->fixup_params(params);
772 decompress_filter(enum compressor_type type, int fd_in, int fd_out,
773 const char *desc_fmt, ...)
776 struct varbuf desc = VARBUF_INIT;
778 va_start(args, desc_fmt);
779 varbuf_vprintf(&desc, desc_fmt, args);
782 compressor(type)->decompress(fd_in, fd_out, desc.buf);
786 compress_filter(struct compress_params *params, int fd_in, int fd_out,
787 const char *desc_fmt, ...)
790 struct varbuf desc = VARBUF_INIT;
792 va_start(args, desc_fmt);
793 varbuf_vprintf(&desc, desc_fmt, args);
796 compressor_fixup_params(params);
798 if (params->level < 0)
799 params->level = compressor(params->type)->default_level;
801 compressor(params->type)->compress(fd_in, fd_out, params, desc.buf);