2 * Copyright (c) 2011, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
19 #include "solv_xfopen.h"
22 #ifndef WITHOUT_COOKIEOPEN
24 static FILE *cookieopen(void *cookie, const char *mode,
25 ssize_t (*cread)(void *, char *, size_t),
26 ssize_t (*cwrite)(void *, const char *, size_t),
27 int (*cclose)(void *))
32 return funopen(cookie,
33 (int (*)(void *, char *, int))(*mode == 'r' ? cread : NULL), /* readfn */
34 (int (*)(void *, const char *, int))(*mode == 'w' ? cwrite : NULL), /* writefn */
35 (fpos_t (*)(void *, fpos_t, int))NULL, /* seekfn */
38 #elif defined(HAVE_FOPENCOOKIE)
39 cookie_io_functions_t cio;
43 memset(&cio, 0, sizeof(cio));
46 else if (*mode == 'w')
49 return fopencookie(cookie, *mode == 'w' ? "w" : "r", cio);
51 # error Need to implement custom I/O
56 #ifdef ENABLE_ZLIB_COMPRESSION
58 /* gzip compression */
62 static ssize_t cookie_gzread(void *cookie, char *buf, size_t nbytes)
64 ssize_t r = gzread((gzFile)cookie, buf, nbytes);
68 gzerror((gzFile)cookie, &err);
69 if (err == Z_BUF_ERROR)
75 static ssize_t cookie_gzwrite(void *cookie, const char *buf, size_t nbytes)
77 return gzwrite((gzFile)cookie, buf, nbytes);
80 static int cookie_gzclose(void *cookie)
82 return gzclose((gzFile)cookie);
85 static inline FILE *mygzfopen(const char *fn, const char *mode)
87 gzFile gzf = gzopen(fn, mode);
88 return cookieopen(gzf, mode, cookie_gzread, cookie_gzwrite, cookie_gzclose);
91 static inline FILE *mygzfdopen(int fd, const char *mode)
93 gzFile gzf = gzdopen(fd, mode);
94 return cookieopen(gzf, mode, cookie_gzread, cookie_gzwrite, cookie_gzclose);
100 #ifdef ENABLE_BZIP2_COMPRESSION
102 /* bzip2 compression */
106 static ssize_t cookie_bzread(void *cookie, char *buf, size_t nbytes)
108 return BZ2_bzread((BZFILE *)cookie, buf, nbytes);
111 static ssize_t cookie_bzwrite(void *cookie, const char *buf, size_t nbytes)
113 return BZ2_bzwrite((BZFILE *)cookie, (char *)buf, nbytes);
116 static int cookie_bzclose(void *cookie)
118 BZ2_bzclose((BZFILE *)cookie);
122 static inline FILE *mybzfopen(const char *fn, const char *mode)
124 BZFILE *bzf = BZ2_bzopen(fn, mode);
125 return cookieopen(bzf, mode, cookie_bzread, cookie_bzwrite, cookie_bzclose);
128 static inline FILE *mybzfdopen(int fd, const char *mode)
130 BZFILE *bzf = BZ2_bzdopen(fd, mode);
131 return cookieopen(bzf, mode, cookie_bzread, cookie_bzwrite, cookie_bzclose);
137 #ifdef ENABLE_LZMA_COMPRESSION
139 /* lzma code written by me in 2008 for rpm's rpmio.c */
143 typedef struct lzfile {
144 unsigned char buf[1 << 15];
151 static inline lzma_ret setup_alone_encoder(lzma_stream *strm, int level)
153 lzma_options_lzma options;
154 lzma_lzma_preset(&options, level);
155 return lzma_alone_encoder(strm, &options);
158 static lzma_stream stream_init = LZMA_STREAM_INIT;
160 static LZFILE *lzopen(const char *path, const char *mode, int fd, int isxz)
168 if ((!path && fd < 0) || (path && fd >= 0))
170 for (; *mode; mode++)
174 else if (*mode == 'r')
176 else if (*mode >= '1' && *mode <= '9')
179 lzfile = solv_calloc(1, sizeof(*lzfile));
180 lzfile->encoding = encoding;
182 lzfile->strm = stream_init;
186 ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256);
188 ret = setup_alone_encoder(&lzfile->strm, level);
191 ret = lzma_auto_decoder(&lzfile->strm, 100 << 20, 0);
198 fp = fdopen(fd, encoding ? "w" : "r");
200 fp = fopen(path, encoding ? "w" : "r");
203 lzma_end(&lzfile->strm);
211 static int lzclose(void *cookie)
213 LZFILE *lzfile = cookie;
220 if (lzfile->encoding)
224 lzfile->strm.avail_out = sizeof(lzfile->buf);
225 lzfile->strm.next_out = lzfile->buf;
226 ret = lzma_code(&lzfile->strm, LZMA_FINISH);
227 if (ret != LZMA_OK && ret != LZMA_STREAM_END)
229 n = sizeof(lzfile->buf) - lzfile->strm.avail_out;
230 if (n && fwrite(lzfile->buf, 1, n, lzfile->file) != n)
232 if (ret == LZMA_STREAM_END)
236 lzma_end(&lzfile->strm);
237 rc = fclose(lzfile->file);
242 static ssize_t lzread(void *cookie, char *buf, size_t len)
244 LZFILE *lzfile = cookie;
248 if (!lzfile || lzfile->encoding)
252 lzfile->strm.next_out = (unsigned char *)buf;
253 lzfile->strm.avail_out = len;
256 if (!lzfile->strm.avail_in)
258 lzfile->strm.next_in = lzfile->buf;
259 lzfile->strm.avail_in = fread(lzfile->buf, 1, sizeof(lzfile->buf), lzfile->file);
260 if (!lzfile->strm.avail_in)
263 ret = lzma_code(&lzfile->strm, LZMA_RUN);
264 if (ret == LZMA_STREAM_END)
267 return len - lzfile->strm.avail_out;
271 if (!lzfile->strm.avail_out)
278 static ssize_t lzwrite(void *cookie, const char *buf, size_t len)
280 LZFILE *lzfile = cookie;
283 if (!lzfile || !lzfile->encoding)
287 lzfile->strm.next_in = (unsigned char *)buf;
288 lzfile->strm.avail_in = len;
291 lzfile->strm.next_out = lzfile->buf;
292 lzfile->strm.avail_out = sizeof(lzfile->buf);
293 ret = lzma_code(&lzfile->strm, LZMA_RUN);
296 n = sizeof(lzfile->buf) - lzfile->strm.avail_out;
297 if (n && fwrite(lzfile->buf, 1, n, lzfile->file) != n)
299 if (!lzfile->strm.avail_in)
304 static inline FILE *myxzfopen(const char *fn, const char *mode)
306 LZFILE *lzf = lzopen(fn, mode, -1, 1);
307 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
310 static inline FILE *myxzfdopen(int fd, const char *mode)
312 LZFILE *lzf = lzopen(0, mode, fd, 1);
313 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
316 static inline FILE *mylzfopen(const char *fn, const char *mode)
318 LZFILE *lzf = lzopen(fn, mode, -1, 0);
319 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
322 static inline FILE *mylzfdopen(int fd, const char *mode)
324 LZFILE *lzf = lzopen(0, mode, fd, 0);
325 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
328 #endif /* ENABLE_LZMA_COMPRESSION */
330 #ifdef ENABLE_ZSTD_COMPRESSION
334 typedef struct zstdfile {
335 ZSTD_CStream *cstream;
336 ZSTD_DStream *dstream;
342 unsigned char buf[1 << 15];
345 static ZSTDFILE *zstdopen(const char *path, const char *mode, int fd)
352 if ((!path && fd < 0) || (path && fd >= 0))
354 for (; *mode; mode++)
358 else if (*mode == 'r')
360 else if (*mode >= '1' && *mode <= '9')
363 zstdfile = solv_calloc(1, sizeof(*zstdfile));
364 zstdfile->encoding = encoding;
367 zstdfile->cstream = ZSTD_createCStream();
368 zstdfile->encoding = 1;
369 if (!zstdfile->cstream)
374 if (ZSTD_isError(ZSTD_initCStream(zstdfile->cstream, level)))
376 ZSTD_freeCStream(zstdfile->cstream);
380 zstdfile->out.dst = zstdfile->buf;
381 zstdfile->out.pos = 0;
382 zstdfile->out.size = sizeof(zstdfile->buf);
386 zstdfile->dstream = ZSTD_createDStream();
387 if (ZSTD_isError(ZSTD_initDStream(zstdfile->dstream)))
389 ZSTD_freeDStream(zstdfile->dstream);
393 zstdfile->in.src = zstdfile->buf;
394 zstdfile->in.pos = 0;
395 zstdfile->in.size = 0;
398 fp = fdopen(fd, encoding ? "w" : "r");
400 fp = fopen(path, encoding ? "w" : "r");
404 ZSTD_freeCStream(zstdfile->cstream);
406 ZSTD_freeDStream(zstdfile->dstream);
414 static int zstdclose(void *cookie)
416 ZSTDFILE *zstdfile = cookie;
421 if (zstdfile->encoding)
426 zstdfile->out.pos = 0;
427 ret = ZSTD_endStream(zstdfile->cstream, &zstdfile->out);
428 if (ZSTD_isError(ret))
430 if (zstdfile->out.pos && fwrite(zstdfile->buf, 1, zstdfile->out.pos, zstdfile->file) != zstdfile->out.pos)
435 ZSTD_freeCStream(zstdfile->cstream);
439 ZSTD_freeDStream(zstdfile->dstream);
441 rc = fclose(zstdfile->file);
446 static ssize_t zstdread(void *cookie, char *buf, size_t len)
448 ZSTDFILE *zstdfile = cookie;
451 if (!zstdfile || zstdfile->encoding)
455 zstdfile->out.dst = buf;
456 zstdfile->out.pos = 0;
457 zstdfile->out.size = len;
460 if (!eof && zstdfile->in.pos == zstdfile->in.size)
462 zstdfile->in.pos = 0;
463 zstdfile->in.size = fread(zstdfile->buf, 1, sizeof(zstdfile->buf), zstdfile->file);
464 if (!zstdfile->in.size)
468 ret = ZSTD_decompressStream(zstdfile->dstream, &zstdfile->out, &zstdfile->in);
472 return zstdfile->out.pos;
474 if (ZSTD_isError(ret))
476 if (zstdfile->out.pos == len)
481 static ssize_t zstdwrite(void *cookie, const char *buf, size_t len)
483 ZSTDFILE *zstdfile = cookie;
484 if (!zstdfile || !zstdfile->encoding)
488 zstdfile->in.src = buf;
489 zstdfile->in.pos = 0;
490 zstdfile->in.size = len;
495 zstdfile->out.pos = 0;
496 ret = ZSTD_compressStream(zstdfile->cstream, &zstdfile->out, &zstdfile->in);
497 if (ZSTD_isError(ret))
499 if (zstdfile->out.pos && fwrite(zstdfile->buf, 1, zstdfile->out.pos, zstdfile->file) != zstdfile->out.pos)
501 if (zstdfile->in.pos == len)
506 static inline FILE *myzstdfopen(const char *fn, const char *mode)
508 ZSTDFILE *zstdfile = zstdopen(fn, mode, -1);
509 return cookieopen(zstdfile, mode, zstdread, zstdwrite, zstdclose);
512 static inline FILE *myzstdfdopen(int fd, const char *mode)
514 ZSTDFILE *zstdfile = zstdopen(0, mode, fd);
515 return cookieopen(zstdfile, mode, zstdread, zstdwrite, zstdclose);
520 #ifdef ENABLE_ZCHUNK_COMPRESSION
522 #ifdef WITH_SYSTEM_ZCHUNK
523 /* use the system's zchunk library that supports reading and writing of zchunk files */
527 static ssize_t cookie_zckread(void *cookie, char *buf, size_t nbytes)
529 return zck_read((zckCtx *)cookie, buf, nbytes);
532 static ssize_t cookie_zckwrite(void *cookie, const char *buf, size_t nbytes)
534 return zck_write((zckCtx *)cookie, buf, nbytes);
537 static int cookie_zckclose(void *cookie)
539 zckCtx *zck = (zckCtx *)cookie;
540 int fd = zck_get_fd(zck);
547 static void *zchunkopen(const char *path, const char *mode, int fd)
551 if ((!path && fd < 0) || (path && fd >= 0))
556 fd = open(path, O_RDONLY);
558 fd = open(path, O_WRONLY | O_CREAT, 0666);
571 if(!zck_init_read(f, fd))
581 if(!zck_init_write(f, fd))
589 return cookieopen(f, mode, cookie_zckread, cookie_zckwrite, cookie_zckclose);
594 #include "solv_zchunk.h"
595 /* use the libsolv's limited zchunk implementation that only supports reading of zchunk files */
597 static void *zchunkopen(const char *path, const char *mode, int fd)
601 if ((!path && fd < 0) || (path && fd >= 0))
603 if (strcmp(mode, "r") != 0)
606 fp = fdopen(fd, mode);
608 fp = fopen(path, mode);
611 f = solv_zchunk_open(fp, 1);
616 /* The fd passed by user must not be closed! */
617 /* Dup (save) the original fd to a temporary variable and then back. */
618 /* It is ugly and thread unsafe hack (non atomical sequence fclose dup2). */
629 return cookieopen(f, mode, (ssize_t (*)(void *, char *, size_t))solv_zchunk_read, 0, (int (*)(void *))solv_zchunk_close);
634 static inline FILE *myzchunkfopen(const char *fn, const char *mode)
636 return zchunkopen(fn, mode, -1);
639 static inline FILE *myzchunkfdopen(int fd, const char *mode)
641 return zchunkopen(0, mode, fd);
644 #endif /* ENABLE_ZCHUNK_COMPRESSION */
647 /* no cookies no compression */
648 #undef ENABLE_ZLIB_COMPRESSION
649 #undef ENABLE_LZMA_COMPRESSION
650 #undef ENABLE_BZIP2_COMPRESSION
651 #undef ENABLE_ZSTD_COMPRESSION
652 #undef ENABLE_ZCHUNK_COMPRESSION
658 solv_xfopen(const char *fn, const char *mode)
666 suf = strrchr(fn, '.');
667 #ifdef ENABLE_ZLIB_COMPRESSION
668 if (suf && !strcmp(suf, ".gz"))
669 return mygzfopen(fn, mode);
671 if (suf && !strcmp(suf, ".gz"))
674 #ifdef ENABLE_LZMA_COMPRESSION
675 if (suf && !strcmp(suf, ".xz"))
676 return myxzfopen(fn, mode);
677 if (suf && !strcmp(suf, ".lzma"))
678 return mylzfopen(fn, mode);
680 if (suf && !strcmp(suf, ".xz"))
682 if (suf && !strcmp(suf, ".lzma"))
685 #ifdef ENABLE_BZIP2_COMPRESSION
686 if (suf && !strcmp(suf, ".bz2"))
687 return mybzfopen(fn, mode);
689 if (suf && !strcmp(suf, ".bz2"))
692 #ifdef ENABLE_ZSTD_COMPRESSION
693 if (suf && !strcmp(suf, ".zst"))
694 return myzstdfopen(fn, mode);
696 if (suf && !strcmp(suf, ".zst"))
699 #ifdef ENABLE_ZCHUNK_COMPRESSION
700 if (suf && !strcmp(suf, ".zck"))
701 return myzchunkfopen(fn, mode);
703 if (suf && !strcmp(suf, ".zck"))
706 return fopen(fn, mode);
710 solv_xfopen_fd(const char *fn, int fd, const char *mode)
712 const char *simplemode = mode;
715 suf = fn ? strrchr(fn, '.') : 0;
719 int fl = fcntl(fd, F_GETFL, 0);
721 HANDLE handle = (HANDLE) _get_osfhandle(fd);
722 BY_HANDLE_FILE_INFORMATION file_info;
723 if (!GetFileInformationByHandle(handle, &file_info))
725 int fl = file_info.dwFileAttributes;
729 fl &= O_RDONLY|O_WRONLY|O_RDWR;
731 mode = simplemode = "w";
732 else if (fl == O_RDWR)
738 mode = simplemode = "r";
740 #ifdef ENABLE_ZLIB_COMPRESSION
741 if (suf && !strcmp(suf, ".gz"))
742 return mygzfdopen(fd, simplemode);
744 if (suf && !strcmp(suf, ".gz"))
747 #ifdef ENABLE_LZMA_COMPRESSION
748 if (suf && !strcmp(suf, ".xz"))
749 return myxzfdopen(fd, simplemode);
750 if (suf && !strcmp(suf, ".lzma"))
751 return mylzfdopen(fd, simplemode);
753 if (suf && !strcmp(suf, ".xz"))
755 if (suf && !strcmp(suf, ".lzma"))
758 #ifdef ENABLE_BZIP2_COMPRESSION
759 if (suf && !strcmp(suf, ".bz2"))
760 return mybzfdopen(fd, simplemode);
762 if (suf && !strcmp(suf, ".bz2"))
765 #ifdef ENABLE_ZSTD_COMPRESSION
766 if (suf && !strcmp(suf, ".zst"))
767 return myzstdfdopen(fd, simplemode);
769 if (suf && !strcmp(suf, ".zst"))
772 #ifdef ENABLE_ZCHUNK_COMPRESSION
773 if (suf && !strcmp(suf, ".zck"))
774 return myzchunkfdopen(fd, simplemode);
776 if (suf && !strcmp(suf, ".zck"))
779 return fdopen(fd, mode);
783 solv_xfopen_iscompressed(const char *fn)
785 const char *suf = fn ? strrchr(fn, '.') : 0;
788 #ifdef ENABLE_ZLIB_COMPRESSION
789 if (!strcmp(suf, ".gz"))
794 if (!strcmp(suf, ".xz") || !strcmp(suf, ".lzma"))
795 #ifdef ENABLE_LZMA_COMPRESSION
800 if (!strcmp(suf, ".bz2"))
801 #ifdef ENABLE_BZIP2_COMPRESSION
806 if (!strcmp(suf, ".zst"))
807 #ifdef ENABLE_ZSTD_COMPRESSION
812 if (!strcmp(suf, ".zck"))
813 #ifdef ENABLE_ZCHUNK_COMPRESSION
822 #ifndef WITHOUT_COOKIEOPEN
832 static ssize_t cookie_bufread(void *cookie, char *buf, size_t nbytes)
834 struct bufcookie *bc = cookie;
835 size_t n = *bc->buflp > nbytes ? nbytes : *bc->buflp;
838 memcpy(buf, *bc->bufp, n);
845 static ssize_t cookie_bufwrite(void *cookie, const char *buf, size_t nbytes)
847 struct bufcookie *bc = cookie;
848 int n = nbytes > 0x40000000 ? 0x40000000 : nbytes;
851 *bc->bufp = solv_extend(*bc->bufp, *bc->buflp, n + 1, 1, 4095);
852 memcpy(*bc->bufp, buf, n);
853 (*bc->bufp)[n] = 0; /* zero-terminate */
859 static int cookie_bufclose(void *cookie)
861 struct bufcookie *bc = cookie;
863 solv_free(bc->freemem);
869 solv_xfopen_buf(const char *fn, char **bufp, size_t *buflp, const char *mode)
871 struct bufcookie *bc;
873 if (*mode != 'r' && *mode != 'w')
875 bc = solv_calloc(1, sizeof(*bc));
880 bc->bufl_int = *mode == 'w' ? 0 : strlen(*bufp);
881 buflp = &bc->bufl_int;
886 *bc->bufp = solv_extend(0, 0, 1, 1, 4095); /* always zero-terminate */
890 fp = cookieopen(bc, mode, cookie_bufread, cookie_bufwrite, cookie_bufclose);
891 if (!strcmp(mode, "rf")) /* auto-free */
896 *bc->bufp = solv_free(*bc->bufp);
903 solv_fmemopen(const char *buf, size_t bufl, const char *mode)
905 struct bufcookie *bc;
909 bc = solv_calloc(1, sizeof(*bc));
910 bc->buf_int = (char *)buf;
912 bc->bufp = &bc->buf_int;
913 bc->buflp = &bc->bufl_int;
914 fp = cookieopen(bc, mode, cookie_bufread, cookie_bufwrite, cookie_bufclose);
915 if (!strcmp(mode, "rf")) /* auto-free */
916 bc->freemem = bc->buf_int;
925 solv_fmemopen(const char *buf, size_t bufl, const char *mode)
930 if (!strcmp(mode, "rf"))
932 if (!(fp = fmemopen(0, bufl, "r+")))
934 if (bufl && fwrite(buf, bufl, 1, fp) != 1)
939 solv_free((char *)buf);
943 fp = fmemopen((char *)buf, bufl, "r");