2 * Copyright (c) 2011, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
19 #include "solv_xfopen.h"
22 #ifndef WITHOUT_COOKIEOPEN
24 static FILE *cookieopen(void *cookie, const char *mode,
25 ssize_t (*cread)(void *, char *, size_t),
26 ssize_t (*cwrite)(void *, const char *, size_t),
27 int (*cclose)(void *))
32 return funopen(cookie,
33 (int (*)(void *, char *, int))(*mode == 'r' ? cread : NULL), /* readfn */
34 (int (*)(void *, const char *, int))(*mode == 'w' ? cwrite : NULL), /* writefn */
35 (fpos_t (*)(void *, fpos_t, int))NULL, /* seekfn */
38 #elif defined(HAVE_FOPENCOOKIE)
39 cookie_io_functions_t cio;
43 memset(&cio, 0, sizeof(cio));
46 else if (*mode == 'w')
49 return fopencookie(cookie, *mode == 'w' ? "w" : "r", cio);
51 # error Need to implement custom I/O
56 #ifdef ENABLE_ZLIB_COMPRESSION
58 /* gzip compression */
62 static ssize_t cookie_gzread(void *cookie, char *buf, size_t nbytes)
64 ssize_t r = gzread((gzFile)cookie, buf, nbytes);
68 gzerror((gzFile)cookie, &err);
69 if (err == Z_BUF_ERROR)
75 static ssize_t cookie_gzwrite(void *cookie, const char *buf, size_t nbytes)
77 return gzwrite((gzFile)cookie, buf, nbytes);
80 static int cookie_gzclose(void *cookie)
82 return gzclose((gzFile)cookie);
85 static inline FILE *mygzfopen(const char *fn, const char *mode)
87 gzFile gzf = gzopen(fn, mode);
88 return cookieopen(gzf, mode, cookie_gzread, cookie_gzwrite, cookie_gzclose);
91 static inline FILE *mygzfdopen(int fd, const char *mode)
93 gzFile gzf = gzdopen(fd, mode);
94 return cookieopen(gzf, mode, cookie_gzread, cookie_gzwrite, cookie_gzclose);
100 #ifdef ENABLE_BZIP2_COMPRESSION
102 /* bzip2 compression */
106 static ssize_t cookie_bzread(void *cookie, char *buf, size_t nbytes)
108 return BZ2_bzread((BZFILE *)cookie, buf, nbytes);
111 static ssize_t cookie_bzwrite(void *cookie, const char *buf, size_t nbytes)
113 return BZ2_bzwrite((BZFILE *)cookie, (char *)buf, nbytes);
116 static int cookie_bzclose(void *cookie)
118 BZ2_bzclose((BZFILE *)cookie);
122 static inline FILE *mybzfopen(const char *fn, const char *mode)
124 BZFILE *bzf = BZ2_bzopen(fn, mode);
125 return cookieopen(bzf, mode, cookie_bzread, cookie_bzwrite, cookie_bzclose);
128 static inline FILE *mybzfdopen(int fd, const char *mode)
130 BZFILE *bzf = BZ2_bzdopen(fd, mode);
131 return cookieopen(bzf, mode, cookie_bzread, cookie_bzwrite, cookie_bzclose);
137 #ifdef ENABLE_LZMA_COMPRESSION
139 /* lzma code written by me in 2008 for rpm's rpmio.c */
143 typedef struct lzfile {
144 unsigned char buf[1 << 15];
151 static inline lzma_ret setup_alone_encoder(lzma_stream *strm, int level)
153 lzma_options_lzma options;
154 lzma_lzma_preset(&options, level);
155 return lzma_alone_encoder(strm, &options);
158 static lzma_stream stream_init = LZMA_STREAM_INIT;
160 static LZFILE *lzopen(const char *path, const char *mode, int fd, int isxz)
170 for (; *mode; mode++)
174 else if (*mode == 'r')
176 else if (*mode >= '1' && *mode <= '9')
180 fp = fdopen(fd, encoding ? "w" : "r");
182 fp = fopen(path, encoding ? "w" : "r");
185 lzfile = calloc(1, sizeof(*lzfile));
192 lzfile->encoding = encoding;
194 lzfile->strm = stream_init;
198 ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256);
200 ret = setup_alone_encoder(&lzfile->strm, level);
203 ret = lzma_auto_decoder(&lzfile->strm, 100 << 20, 0);
213 static int lzclose(void *cookie)
215 LZFILE *lzfile = cookie;
222 if (lzfile->encoding)
226 lzfile->strm.avail_out = sizeof(lzfile->buf);
227 lzfile->strm.next_out = lzfile->buf;
228 ret = lzma_code(&lzfile->strm, LZMA_FINISH);
229 if (ret != LZMA_OK && ret != LZMA_STREAM_END)
231 n = sizeof(lzfile->buf) - lzfile->strm.avail_out;
232 if (n && fwrite(lzfile->buf, 1, n, lzfile->file) != n)
234 if (ret == LZMA_STREAM_END)
238 lzma_end(&lzfile->strm);
239 rc = fclose(lzfile->file);
244 static ssize_t lzread(void *cookie, char *buf, size_t len)
246 LZFILE *lzfile = cookie;
250 if (!lzfile || lzfile->encoding)
254 lzfile->strm.next_out = (unsigned char *)buf;
255 lzfile->strm.avail_out = len;
258 if (!lzfile->strm.avail_in)
260 lzfile->strm.next_in = lzfile->buf;
261 lzfile->strm.avail_in = fread(lzfile->buf, 1, sizeof(lzfile->buf), lzfile->file);
262 if (!lzfile->strm.avail_in)
265 ret = lzma_code(&lzfile->strm, LZMA_RUN);
266 if (ret == LZMA_STREAM_END)
269 return len - lzfile->strm.avail_out;
273 if (!lzfile->strm.avail_out)
280 static ssize_t lzwrite(void *cookie, const char *buf, size_t len)
282 LZFILE *lzfile = cookie;
285 if (!lzfile || !lzfile->encoding)
289 lzfile->strm.next_in = (unsigned char *)buf;
290 lzfile->strm.avail_in = len;
293 lzfile->strm.next_out = lzfile->buf;
294 lzfile->strm.avail_out = sizeof(lzfile->buf);
295 ret = lzma_code(&lzfile->strm, LZMA_RUN);
298 n = sizeof(lzfile->buf) - lzfile->strm.avail_out;
299 if (n && fwrite(lzfile->buf, 1, n, lzfile->file) != n)
301 if (!lzfile->strm.avail_in)
306 static inline FILE *myxzfopen(const char *fn, const char *mode)
308 LZFILE *lzf = lzopen(fn, mode, -1, 1);
309 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
312 static inline FILE *myxzfdopen(int fd, const char *mode)
314 LZFILE *lzf = lzopen(0, mode, fd, 1);
315 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
318 static inline FILE *mylzfopen(const char *fn, const char *mode)
320 LZFILE *lzf = lzopen(fn, mode, -1, 0);
321 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
324 static inline FILE *mylzfdopen(int fd, const char *mode)
326 LZFILE *lzf = lzopen(0, mode, fd, 0);
327 return cookieopen(lzf, mode, lzread, lzwrite, lzclose);
330 #endif /* ENABLE_LZMA_COMPRESSION */
332 #ifdef ENABLE_ZSTD_COMPRESSION
336 typedef struct zstdfile {
337 ZSTD_CStream *cstream;
338 ZSTD_DStream *dstream;
344 unsigned char buf[1 << 15];
347 static ZSTDFILE *zstdopen(const char *path, const char *mode, int fd)
356 for (; *mode; mode++)
360 else if (*mode == 'r')
362 else if (*mode >= '1' && *mode <= '9')
366 fp = fdopen(fd, encoding ? "w" : "r");
368 fp = fopen(path, encoding ? "w" : "r");
371 zstdfile = solv_calloc(1, sizeof(*zstdfile));
372 zstdfile->encoding = encoding;
375 zstdfile->cstream = ZSTD_createCStream();
376 zstdfile->encoding = 1;
377 if (!zstdfile->cstream)
383 if (ZSTD_isError(ZSTD_initCStream(zstdfile->cstream, level)))
385 ZSTD_freeCStream(zstdfile->cstream);
390 zstdfile->out.dst = zstdfile->buf;
391 zstdfile->out.pos = 0;
392 zstdfile->out.size = sizeof(zstdfile->buf);
396 zstdfile->dstream = ZSTD_createDStream();
397 if (ZSTD_isError(ZSTD_initDStream(zstdfile->dstream)))
399 ZSTD_freeDStream(zstdfile->dstream);
404 zstdfile->in.src = zstdfile->buf;
405 zstdfile->in.pos = 0;
406 zstdfile->in.size = 0;
412 static int zstdclose(void *cookie)
414 ZSTDFILE *zstdfile = cookie;
419 if (zstdfile->encoding)
424 zstdfile->out.pos = 0;
425 ret = ZSTD_endStream(zstdfile->cstream, &zstdfile->out);
426 if (ZSTD_isError(ret))
428 if (zstdfile->out.pos && fwrite(zstdfile->buf, 1, zstdfile->out.pos, zstdfile->file) != zstdfile->out.pos)
433 ZSTD_freeCStream(zstdfile->cstream);
437 ZSTD_freeDStream(zstdfile->dstream);
439 rc = fclose(zstdfile->file);
444 static ssize_t zstdread(void *cookie, char *buf, size_t len)
446 ZSTDFILE *zstdfile = cookie;
449 if (!zstdfile || zstdfile->encoding)
453 zstdfile->out.dst = buf;
454 zstdfile->out.pos = 0;
455 zstdfile->out.size = len;
458 if (!eof && zstdfile->in.pos == zstdfile->in.size)
460 zstdfile->in.pos = 0;
461 zstdfile->in.size = fread(zstdfile->buf, 1, sizeof(zstdfile->buf), zstdfile->file);
462 if (!zstdfile->in.size)
466 ret = ZSTD_decompressStream(zstdfile->dstream, &zstdfile->out, &zstdfile->in);
470 return zstdfile->out.pos;
472 if (ZSTD_isError(ret))
474 if (zstdfile->out.pos == len)
479 static ssize_t zstdwrite(void *cookie, const char *buf, size_t len)
481 ZSTDFILE *zstdfile = cookie;
482 if (!zstdfile || !zstdfile->encoding)
486 zstdfile->in.src = buf;
487 zstdfile->in.pos = 0;
488 zstdfile->in.size = len;
493 zstdfile->out.pos = 0;
494 ret = ZSTD_compressStream(zstdfile->cstream, &zstdfile->out, &zstdfile->in);
495 if (ZSTD_isError(ret))
497 if (zstdfile->out.pos && fwrite(zstdfile->buf, 1, zstdfile->out.pos, zstdfile->file) != zstdfile->out.pos)
499 if (zstdfile->in.pos == len)
504 static inline FILE *myzstdfopen(const char *fn, const char *mode)
506 ZSTDFILE *zstdfile = zstdopen(fn, mode, -1);
507 return cookieopen(zstdfile, mode, zstdread, zstdwrite, zstdclose);
510 static inline FILE *myzstdfdopen(int fd, const char *mode)
512 ZSTDFILE *zstdfile = zstdopen(0, mode, fd);
513 return cookieopen(zstdfile, mode, zstdread, zstdwrite, zstdclose);
518 #ifdef ENABLE_ZCHUNK_COMPRESSION
520 #ifdef WITH_SYSTEM_ZCHUNK
521 /* use the system's zchunk library that supports reading and writing of zchunk files */
525 static ssize_t cookie_zckread(void *cookie, char *buf, size_t nbytes)
527 return zck_read((zckCtx *)cookie, buf, nbytes);
530 static ssize_t cookie_zckwrite(void *cookie, const char *buf, size_t nbytes)
532 return zck_write((zckCtx *)cookie, buf, nbytes);
535 static int cookie_zckclose(void *cookie)
537 zckCtx *zck = (zckCtx *)cookie;
538 int fd = zck_get_fd(zck);
545 static void *zchunkopen(const char *path, const char *mode, int fd)
554 fd = open(path, O_RDONLY);
556 fd = open(path, O_WRONLY | O_CREAT, 0666);
568 if(!zck_init_read(f, fd))
573 if(!zck_init_write(f, fd))
576 return cookieopen(f, mode, cookie_zckread, cookie_zckwrite, cookie_zckclose);
581 #include "solv_zchunk.h"
582 /* use the libsolv's limited zchunk implementation that only supports reading of zchunk files */
584 static void *zchunkopen(const char *path, const char *mode, int fd)
591 fp = fdopen(fd, mode);
593 fp = fopen(path, mode);
596 if (strcmp(mode, "r") != 0)
598 f = solv_zchunk_open(fp, 1);
601 return cookieopen(f, mode, (ssize_t (*)(void *, char *, size_t))solv_zchunk_read, 0, (int (*)(void *))solv_zchunk_close);
606 static inline FILE *myzchunkfopen(const char *fn, const char *mode)
608 return zchunkopen(fn, mode, -1);
611 static inline FILE *myzchunkfdopen(int fd, const char *mode)
613 return zchunkopen(0, mode, fd);
616 #endif /* ENABLE_ZCHUNK_COMPRESSION */
619 /* no cookies no compression */
620 #undef ENABLE_ZLIB_COMPRESSION
621 #undef ENABLE_LZMA_COMPRESSION
622 #undef ENABLE_BZIP2_COMPRESSION
623 #undef ENABLE_ZSTD_COMPRESSION
624 #undef ENABLE_ZCHUNK_COMPRESSION
630 solv_xfopen(const char *fn, const char *mode)
638 suf = strrchr(fn, '.');
639 #ifdef ENABLE_ZLIB_COMPRESSION
640 if (suf && !strcmp(suf, ".gz"))
641 return mygzfopen(fn, mode);
643 if (suf && !strcmp(suf, ".gz"))
646 #ifdef ENABLE_LZMA_COMPRESSION
647 if (suf && !strcmp(suf, ".xz"))
648 return myxzfopen(fn, mode);
649 if (suf && !strcmp(suf, ".lzma"))
650 return mylzfopen(fn, mode);
652 if (suf && !strcmp(suf, ".xz"))
654 if (suf && !strcmp(suf, ".lzma"))
657 #ifdef ENABLE_BZIP2_COMPRESSION
658 if (suf && !strcmp(suf, ".bz2"))
659 return mybzfopen(fn, mode);
661 if (suf && !strcmp(suf, ".bz2"))
664 #ifdef ENABLE_ZSTD_COMPRESSION
665 if (suf && !strcmp(suf, ".zst"))
666 return myzstdfopen(fn, mode);
668 if (suf && !strcmp(suf, ".zst"))
671 #ifdef ENABLE_ZCHUNK_COMPRESSION
672 if (suf && !strcmp(suf, ".zck"))
673 return myzchunkfopen(fn, mode);
675 if (suf && !strcmp(suf, ".zck"))
678 return fopen(fn, mode);
682 solv_xfopen_fd(const char *fn, int fd, const char *mode)
684 const char *simplemode = mode;
687 suf = fn ? strrchr(fn, '.') : 0;
691 int fl = fcntl(fd, F_GETFL, 0);
693 HANDLE handle = (HANDLE) _get_osfhandle(fd);
694 BY_HANDLE_FILE_INFORMATION file_info;
695 if (!GetFileInformationByHandle(handle, &file_info))
697 int fl = file_info.dwFileAttributes;
701 fl &= O_RDONLY|O_WRONLY|O_RDWR;
703 mode = simplemode = "w";
704 else if (fl == O_RDWR)
710 mode = simplemode = "r";
712 #ifdef ENABLE_ZLIB_COMPRESSION
713 if (suf && !strcmp(suf, ".gz"))
714 return mygzfdopen(fd, simplemode);
716 if (suf && !strcmp(suf, ".gz"))
719 #ifdef ENABLE_LZMA_COMPRESSION
720 if (suf && !strcmp(suf, ".xz"))
721 return myxzfdopen(fd, simplemode);
722 if (suf && !strcmp(suf, ".lzma"))
723 return mylzfdopen(fd, simplemode);
725 if (suf && !strcmp(suf, ".xz"))
727 if (suf && !strcmp(suf, ".lzma"))
730 #ifdef ENABLE_BZIP2_COMPRESSION
731 if (suf && !strcmp(suf, ".bz2"))
732 return mybzfdopen(fd, simplemode);
734 if (suf && !strcmp(suf, ".bz2"))
737 #ifdef ENABLE_ZSTD_COMPRESSION
738 if (suf && !strcmp(suf, ".zst"))
739 return myzstdfdopen(fd, simplemode);
741 if (suf && !strcmp(suf, ".zst"))
744 #ifdef ENABLE_ZCHUNK_COMPRESSION
745 if (suf && !strcmp(suf, ".zck"))
746 return myzchunkfdopen(fd, simplemode);
748 if (suf && !strcmp(suf, ".zck"))
751 return fdopen(fd, mode);
755 solv_xfopen_iscompressed(const char *fn)
757 const char *suf = fn ? strrchr(fn, '.') : 0;
760 #ifdef ENABLE_ZLIB_COMPRESSION
761 if (!strcmp(suf, ".gz"))
766 if (!strcmp(suf, ".xz") || !strcmp(suf, ".lzma"))
767 #ifdef ENABLE_LZMA_COMPRESSION
772 if (!strcmp(suf, ".bz2"))
773 #ifdef ENABLE_BZIP2_COMPRESSION
778 if (!strcmp(suf, ".zst"))
779 #ifdef ENABLE_ZSTD_COMPRESSION
784 if (!strcmp(suf, ".zck"))
785 #ifdef ENABLE_ZCHUNK_COMPRESSION
794 #ifndef WITHOUT_COOKIEOPEN
803 static ssize_t cookie_bufread(void *cookie, char *buf, size_t nbytes)
805 struct bufcookie *bc = cookie;
806 size_t n = *bc->buflp > nbytes ? nbytes : *bc->buflp;
809 memcpy(buf, *bc->bufp, n);
816 static ssize_t cookie_bufwrite(void *cookie, const char *buf, size_t nbytes)
818 struct bufcookie *bc = cookie;
819 int n = nbytes > 0x40000000 ? 0x40000000 : nbytes;
822 *bc->bufp = solv_extend(*bc->bufp, *bc->buflp, n + 1, 1, 4095);
823 memcpy(*bc->bufp, buf, n);
824 (*bc->bufp)[n] = 0; /* zero-terminate */
830 static int cookie_bufclose(void *cookie)
832 struct bufcookie *bc = cookie;
834 solv_free(bc->freemem);
840 solv_xfopen_buf(const char *fn, char **bufp, size_t *buflp, const char *mode)
842 struct bufcookie *bc;
844 if (*mode != 'r' && *mode != 'w')
846 bc = solv_calloc(1, sizeof(*bc));
851 bc->bufl_int = *mode == 'w' ? 0 : strlen(*bufp);
852 buflp = &bc->bufl_int;
857 *bc->bufp = solv_extend(0, 0, 1, 1, 4095); /* always zero-terminate */
861 fp = cookieopen(bc, mode, cookie_bufread, cookie_bufwrite, cookie_bufclose);
862 if (!strcmp(mode, "rf")) /* auto-free */
867 *bc->bufp = solv_free(*bc->bufp);
876 solv_xfopen_buf(const char *fn, char **bufp, size_t *buflp, const char *mode)
882 l = buflp ? *buflp : strlen(*bufp);
883 if (!strcmp(mode, "rf"))
885 if (!(fp = fmemopen(0, l, "r+")))
887 if (l && fwrite(*bufp, l, 1, fp) != 1)
896 fp = fmemopen(*bufp, l, "r");