2 * Copyright (c) 2011, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
15 #include "solv_xfopen.h"
19 static FILE *cookieopen(void *cookie, const char *mode,
20 ssize_t (*cread)(void *, char *, size_t),
21 ssize_t (*cwrite)(void *, const char *, size_t),
22 int (*cclose)(void *))
27 return funopen(cookie,
28 (int (*)(void *, char *, int))(*mode == 'r' ? cread : NULL), /* readfn */
29 (int (*)(void *, const char *, int))(*mode == 'w' ? cwrite : NULL), /* writefn */
30 (fpos_t (*)(void *, fpos_t, int))NULL, /* seekfn */
33 #elif defined(HAVE_FOPENCOOKIE)
34 cookie_io_functions_t cio;
38 memset(&cio, 0, sizeof(cio));
41 else if (*mode == 'w')
44 return fopencookie(cookie, *mode == 'w' ? "w" : "r", cio);
46 # error Need to implement custom I/O
51 #ifdef ENABLE_ZLIB_COMPRESSION
53 /* gzip compression */
57 static ssize_t cookie_gzread(void *cookie, char *buf, size_t nbytes)
59 return gzread((gzFile)cookie, buf, nbytes);
62 static ssize_t cookie_gzwrite(void *cookie, const char *buf, size_t nbytes)
64 return gzwrite((gzFile)cookie, buf, nbytes);
67 static int cookie_gzclose(void *cookie)
69 return gzclose((gzFile)cookie);
72 static inline FILE *mygzfopen(const char *fn, const char *mode)
74 gzFile gzf = gzopen(fn, mode);
75 return cookieopen(gzf, mode, cookie_gzread, cookie_gzwrite, cookie_gzclose);
78 static inline FILE *mygzfdopen(int fd, const char *mode)
80 gzFile gzf = gzdopen(fd, mode);
81 return cookieopen(gzf, mode, cookie_gzread, cookie_gzwrite, cookie_gzclose);
87 #ifdef ENABLE_BZIP2_COMPRESSION
89 /* bzip2 compression */
93 static ssize_t cookie_bzread(void *cookie, char *buf, size_t nbytes)
95 return BZ2_bzread((BZFILE *)cookie, buf, nbytes);
98 static ssize_t cookie_bzwrite(void *cookie, const char *buf, size_t nbytes)
100 return BZ2_bzwrite((BZFILE *)cookie, (char *)buf, nbytes);
103 static int cookie_bzclose(void *cookie)
105 BZ2_bzclose((BZFILE *)cookie);
109 static inline FILE *mybzfopen(const char *fn, const char *mode)
111 BZFILE *bzf = BZ2_bzopen(fn, mode);
112 return cookieopen(bzf, mode, cookie_bzread, cookie_bzwrite, cookie_bzclose);
115 static inline FILE *mybzfdopen(int fd, const char *mode)
117 BZFILE *bzf = BZ2_bzdopen(fd, mode);
118 return cookieopen(bzf, mode, cookie_bzread, cookie_bzwrite, cookie_bzclose);
124 #ifdef ENABLE_LZMA_COMPRESSION
126 /* lzma code written by me in 2008 for rpm's rpmio.c */
130 typedef struct lzfile {
131 unsigned char buf[1 << 15];
138 static inline lzma_ret setup_alone_encoder(lzma_stream *strm, int level)
140 lzma_options_lzma options;
141 lzma_lzma_preset(&options, level);
142 return lzma_alone_encoder(strm, &options);
145 static lzma_stream stream_init = LZMA_STREAM_INIT;
147 static LZFILE *lzopen(const char *path, const char *mode, int fd, int isxz)
157 for (; *mode; mode++)
161 else if (*mode == 'r')
163 else if (*mode >= '1' && *mode <= '9')
167 fp = fdopen(fd, encoding ? "w" : "r");
169 fp = fopen(path, encoding ? "w" : "r");
172 lzfile = calloc(1, sizeof(*lzfile));
179 lzfile->encoding = encoding;
181 lzfile->strm = stream_init;
185 ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256);
187 ret = setup_alone_encoder(&lzfile->strm, level);
190 ret = lzma_auto_decoder(&lzfile->strm, 100 << 20, 0);
200 static int lzclose(void *cookie)
202 LZFILE *lzfile = cookie;
209 if (lzfile->encoding)
213 lzfile->strm.avail_out = sizeof(lzfile->buf);
214 lzfile->strm.next_out = lzfile->buf;
215 ret = lzma_code(&lzfile->strm, LZMA_FINISH);
216 if (ret != LZMA_OK && ret != LZMA_STREAM_END)
218 n = sizeof(lzfile->buf) - lzfile->strm.avail_out;
219 if (n && fwrite(lzfile->buf, 1, n, lzfile->file) != n)
221 if (ret == LZMA_STREAM_END)
225 lzma_end(&lzfile->strm);
226 rc = fclose(lzfile->file);
231 static ssize_t lzread(void *cookie, char *buf, size_t len)
233 LZFILE *lzfile = cookie;
237 if (!lzfile || lzfile->encoding)
241 lzfile->strm.next_out = (unsigned char *)buf;
242 lzfile->strm.avail_out = len;
245 if (!lzfile->strm.avail_in)
247 lzfile->strm.next_in = lzfile->buf;
248 lzfile->strm.avail_in = fread(lzfile->buf, 1, sizeof(lzfile->buf), lzfile->file);
249 if (!lzfile->strm.avail_in)
252 ret = lzma_code(&lzfile->strm, LZMA_RUN);
253 if (ret == LZMA_STREAM_END)
256 return len - lzfile->strm.avail_out;
260 if (!lzfile->strm.avail_out)
267 static ssize_t lzwrite(void *cookie, const char *buf, size_t len)
269 LZFILE *lzfile = cookie;
272 if (!lzfile || !lzfile->encoding)
276 lzfile->strm.next_in = (unsigned char *)buf;
277 lzfile->strm.avail_in = len;
280 lzfile->strm.next_out = lzfile->buf;
281 lzfile->strm.avail_out = sizeof(lzfile->buf);
282 ret = lzma_code(&lzfile->strm, LZMA_RUN);
285 n = sizeof(lzfile->buf) - lzfile->strm.avail_out;
286 if (n && fwrite(lzfile->buf, 1, n, lzfile->file) != n)
288 if (!lzfile->strm.avail_in)
293 static inline FILE *myxzfopen(const char *fn, const char *mode)
295 LZFILE *lzf = lzopen(fn, mode, -1, 1);
296 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
299 static inline FILE *myxzfdopen(int fd, const char *mode)
301 LZFILE *lzf = lzopen(0, mode, fd, 1);
302 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
305 static inline FILE *mylzfopen(const char *fn, const char *mode)
307 LZFILE *lzf = lzopen(fn, mode, -1, 0);
308 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
311 static inline FILE *mylzfdopen(int fd, const char *mode)
313 LZFILE *lzf = lzopen(0, mode, fd, 0);
314 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
317 #endif /* ENABLE_LZMA_COMPRESSION */
319 #ifdef ENABLE_ZSTD_COMPRESSION
323 typedef struct zstdfile {
324 ZSTD_CStream *cstream;
325 ZSTD_DStream *dstream;
331 unsigned char buf[1 << 15];
334 static ZSTDFILE *zstdopen(const char *path, const char *mode, int fd)
343 for (; *mode; mode++)
347 else if (*mode == 'r')
349 else if (*mode >= '1' && *mode <= '9')
353 fp = fdopen(fd, encoding ? "w" : "r");
355 fp = fopen(path, encoding ? "w" : "r");
358 zstdfile = solv_calloc(1, sizeof(*zstdfile));
359 zstdfile->encoding = encoding;
362 zstdfile->cstream = ZSTD_createCStream();
363 zstdfile->encoding = 1;
364 if (!zstdfile->cstream)
370 if (ZSTD_isError(ZSTD_initCStream(zstdfile->cstream, level)))
372 ZSTD_freeCStream(zstdfile->cstream);
377 zstdfile->out.dst = zstdfile->buf;
378 zstdfile->out.pos = 0;
379 zstdfile->out.size = sizeof(zstdfile->buf);
383 zstdfile->dstream = ZSTD_createDStream();
384 if (ZSTD_isError(ZSTD_initDStream(zstdfile->dstream)))
386 ZSTD_freeDStream(zstdfile->dstream);
391 zstdfile->in.src = zstdfile->buf;
392 zstdfile->in.pos = 0;
393 zstdfile->in.size = 0;
399 static int zstdclose(void *cookie)
401 ZSTDFILE *zstdfile = cookie;
406 if (zstdfile->encoding)
411 zstdfile->out.pos = 0;
412 ret = ZSTD_endStream(zstdfile->cstream, &zstdfile->out);
413 if (ZSTD_isError(ret))
415 if (zstdfile->out.pos && fwrite(zstdfile->buf, 1, zstdfile->out.pos, zstdfile->file) != zstdfile->out.pos)
420 ZSTD_freeCStream(zstdfile->cstream);
424 ZSTD_freeDStream(zstdfile->dstream);
426 rc = fclose(zstdfile->file);
431 static ssize_t zstdread(void *cookie, char *buf, size_t len)
433 ZSTDFILE *zstdfile = cookie;
436 if (!zstdfile || zstdfile->encoding)
440 zstdfile->out.dst = buf;
441 zstdfile->out.pos = 0;
442 zstdfile->out.size = len;
445 if (!eof && zstdfile->in.pos == zstdfile->in.size)
447 zstdfile->in.pos = 0;
448 zstdfile->in.size = fread(zstdfile->buf, 1, sizeof(zstdfile->buf), zstdfile->file);
449 if (!zstdfile->in.size)
453 ret = ZSTD_decompressStream(zstdfile->dstream, &zstdfile->out, &zstdfile->in);
457 return zstdfile->out.pos;
459 if (ZSTD_isError(ret))
461 if (zstdfile->out.pos == len)
466 static ssize_t zstdwrite(void *cookie, const char *buf, size_t len)
468 ZSTDFILE *zstdfile = cookie;
469 if (!zstdfile || !zstdfile->encoding)
473 zstdfile->in.src = buf;
474 zstdfile->in.pos = 0;
475 zstdfile->in.size = len;
480 zstdfile->out.pos = 0;
481 ret = ZSTD_compressStream(zstdfile->cstream, &zstdfile->out, &zstdfile->in);
482 if (ZSTD_isError(ret))
484 if (zstdfile->out.pos && fwrite(zstdfile->buf, 1, zstdfile->out.pos, zstdfile->file) != zstdfile->out.pos)
486 if (zstdfile->in.pos == len)
491 static inline FILE *myzstdfopen(const char *fn, const char *mode)
493 ZSTDFILE *zstdfile = zstdopen(fn, mode, -1);
494 return cookieopen(zstdfile, mode, zstdread, zstdwrite, zstdclose);
497 static inline FILE *myzstdfdopen(int fd, const char *mode)
499 ZSTDFILE *zstdfile = zstdopen(0, mode, fd);
500 return cookieopen(zstdfile, mode, zstdread, zstdwrite, zstdclose);
505 #ifdef ENABLE_ZCHUNK_COMPRESSION
507 #ifdef WITH_SYSTEM_ZCHUNK
508 /* use the system's zchunk library that supports reading and writing of zchunk files */
512 static ssize_t cookie_zckread(void *cookie, char *buf, size_t nbytes)
514 return zck_read((zckCtx *)cookie, buf, nbytes);
517 static ssize_t cookie_zckwrite(void *cookie, const char *buf, size_t nbytes)
519 return zck_write((zckCtx *)cookie, buf, nbytes);
522 static int cookie_zckclose(void *cookie)
524 zckCtx *zck = (zckCtx *)cookie;
525 int fd = zck_get_fd(zck);
532 static void *zchunkopen(const char *path, const char *mode, int fd)
541 fd = open(path, O_RDONLY);
543 fd = open(path, O_WRONLY | O_CREAT, 0666);
555 if(!zck_init_read(f, fd))
560 if(!zck_init_write(f, fd))
563 return cookieopen(f, mode, cookie_zckread, cookie_zckwrite, cookie_zckclose);
568 #include "solv_zchunk.h"
569 /* use the libsolv's limited zchunk implementation that only supports reading of zchunk files */
571 static void *zchunkopen(const char *path, const char *mode, int fd)
578 fp = fdopen(fd, mode);
580 fp = fopen(path, mode);
583 if (strcmp(mode, "r") != 0)
585 f = solv_zchunk_open(fp, 1);
588 return cookieopen(f, mode, (ssize_t (*)(void *, char *, size_t))solv_zchunk_read, 0, (int (*)(void *))solv_zchunk_close);
593 static inline FILE *myzchunkfopen(const char *fn, const char *mode)
595 return zchunkopen(fn, mode, -1);
598 static inline FILE *myzchunkfdopen(int fd, const char *mode)
600 return zchunkopen(0, mode, fd);
603 #endif /* ENABLE_ZCHUNK_COMPRESSION */
607 solv_xfopen(const char *fn, const char *mode)
615 suf = strrchr(fn, '.');
616 #ifdef ENABLE_ZLIB_COMPRESSION
617 if (suf && !strcmp(suf, ".gz"))
618 return mygzfopen(fn, mode);
620 if (suf && !strcmp(suf, ".gz"))
623 #ifdef ENABLE_LZMA_COMPRESSION
624 if (suf && !strcmp(suf, ".xz"))
625 return myxzfopen(fn, mode);
626 if (suf && !strcmp(suf, ".lzma"))
627 return mylzfopen(fn, mode);
629 if (suf && !strcmp(suf, ".xz"))
631 if (suf && !strcmp(suf, ".lzma"))
634 #ifdef ENABLE_BZIP2_COMPRESSION
635 if (suf && !strcmp(suf, ".bz2"))
636 return mybzfopen(fn, mode);
638 if (suf && !strcmp(suf, ".bz2"))
641 #ifdef ENABLE_ZSTD_COMPRESSION
642 if (suf && !strcmp(suf, ".zst"))
643 return myzstdfopen(fn, mode);
645 if (suf && !strcmp(suf, ".zst"))
648 #ifdef ENABLE_ZCHUNK_COMPRESSION
649 if (suf && !strcmp(suf, ".zck"))
650 return myzchunkfopen(fn, mode);
652 if (suf && !strcmp(suf, ".zck"))
655 return fopen(fn, mode);
659 solv_xfopen_fd(const char *fn, int fd, const char *mode)
661 const char *simplemode = mode;
664 suf = fn ? strrchr(fn, '.') : 0;
667 int fl = fcntl(fd, F_GETFL, 0);
670 fl &= O_RDONLY|O_WRONLY|O_RDWR;
672 mode = simplemode = "w";
673 else if (fl == O_RDWR)
679 mode = simplemode = "r";
681 #ifdef ENABLE_ZLIB_COMPRESSION
682 if (suf && !strcmp(suf, ".gz"))
683 return mygzfdopen(fd, simplemode);
685 if (suf && !strcmp(suf, ".gz"))
688 #ifdef ENABLE_LZMA_COMPRESSION
689 if (suf && !strcmp(suf, ".xz"))
690 return myxzfdopen(fd, simplemode);
691 if (suf && !strcmp(suf, ".lzma"))
692 return mylzfdopen(fd, simplemode);
694 if (suf && !strcmp(suf, ".xz"))
696 if (suf && !strcmp(suf, ".lzma"))
699 #ifdef ENABLE_BZIP2_COMPRESSION
700 if (suf && !strcmp(suf, ".bz2"))
701 return mybzfdopen(fd, simplemode);
703 if (suf && !strcmp(suf, ".bz2"))
706 #ifdef ENABLE_ZSTD_COMPRESSION
707 if (suf && !strcmp(suf, ".zst"))
708 return myzstdfdopen(fd, simplemode);
710 if (suf && !strcmp(suf, ".zst"))
713 #ifdef ENABLE_ZCHUNK_COMPRESSION
714 if (suf && !strcmp(suf, ".zck"))
715 return myzchunkfdopen(fd, simplemode);
717 if (suf && !strcmp(suf, ".zck"))
720 return fdopen(fd, mode);
724 solv_xfopen_iscompressed(const char *fn)
726 const char *suf = fn ? strrchr(fn, '.') : 0;
729 #ifdef ENABLE_ZLIB_COMPRESSION
730 if (!strcmp(suf, ".gz"))
735 if (!strcmp(suf, ".xz") || !strcmp(suf, ".lzma"))
736 #ifdef ENABLE_LZMA_COMPRESSION
741 if (!strcmp(suf, ".bz2"))
742 #ifdef ENABLE_BZIP2_COMPRESSION
747 if (!strcmp(suf, ".zst"))
748 #ifdef ENABLE_ZSTD_COMPRESSION
753 if (!strcmp(suf, ".zck"))
754 #ifdef ENABLE_ZCHUNK_COMPRESSION
769 static ssize_t cookie_bufread(void *cookie, char *buf, size_t nbytes)
771 struct bufcookie *bc = cookie;
772 size_t n = *bc->buflp > nbytes ? nbytes : *bc->buflp;
775 memcpy(buf, *bc->bufp, n);
782 static ssize_t cookie_bufwrite(void *cookie, const char *buf, size_t nbytes)
784 struct bufcookie *bc = cookie;
785 int n = nbytes > 0x40000000 ? 0x40000000 : nbytes;
788 *bc->bufp = solv_extend(*bc->bufp, *bc->buflp, n + 1, 1, 4095);
789 memcpy(*bc->bufp, buf, n);
790 (*bc->bufp)[n] = 0; /* zero-terminate */
796 static int cookie_bufclose(void *cookie)
798 struct bufcookie *bc = cookie;
800 solv_free(bc->freemem);
806 solv_xfopen_buf(const char *fn, char **bufp, size_t *buflp, const char *mode)
808 struct bufcookie *bc;
810 if (*mode != 'r' && *mode != 'w')
812 bc = solv_calloc(1, sizeof(*bc));
817 bc->bufl_int = *mode == 'w' ? 0 : strlen(*bufp);
818 buflp = &bc->bufl_int;
823 *bc->bufp = solv_extend(0, 0, 1, 1, 4095); /* always zero-terminate */
827 fp = cookieopen(bc, mode, cookie_bufread, cookie_bufwrite, cookie_bufclose);
828 if (!strcmp(mode, "rf")) /* auto-free */
833 *bc->bufp = solv_free(*bc->bufp);