2 * Copyright (c) 2018, SUSE LLC.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
16 #include "solv_zchunk.h"
18 #define MAX_HDR_SIZE 0xffffff00
19 #define MAX_CHUNK_CNT 0x0fffffff
21 #undef VERIFY_DATA_CHKSUM
26 unsigned char *hdr_end;
28 unsigned int flags; /* header flags */
29 unsigned int comp; /* compression type */
31 unsigned int hdr_chk_type; /* header + data checksum */
32 unsigned int hdr_chk_len;
35 unsigned int chunk_chk_type; /* chunk checksum */
36 unsigned int chunk_chk_len;
39 Chksum *data_chk; /* for data checksum verification */
40 unsigned char *data_chk_ptr;
42 unsigned int streamid; /* stream we are reading */
43 unsigned int nchunks; /* chunks left */
44 unsigned char *chunks;
51 unsigned int buf_used;
52 unsigned int buf_avail;
55 /* return 32bit compressed integer. returns NULL on overflow. */
56 static unsigned char *
57 getuint(unsigned char *p, unsigned char *endp, unsigned int *dp)
61 if (p < endp && (*p & 0x80) != 0)
66 if (++p < endp && (*p & 0x80) != 0)
68 *dp = p[-1] ^ ((p[0] ^ 0x80) << 7);
71 if (++p < endp && (*p & 0x80) != 0)
73 *dp = p[-2] ^ (p[-1] << 7) ^ ((p[0] ^ 0x80) << 14);
76 if (++p < endp && (*p & 0x80) != 0)
78 *dp = p[-3] ^ (p[-2] << 7) ^ (p[-1] << 14) ^ ((p[0] ^ 0x80) << 21);
81 if (++p < endp && (*p & 0xf0) == 0x80)
83 *dp = p[-4] ^ (p[-3] << 7) ^ (p[-2] << 14) ^ (p[-1] << 21) ^ ((p[0] ^ 0x80) << 28);
89 static unsigned char *
90 getchksum(unsigned char *p, unsigned char *endp, unsigned int *typep, unsigned int *lenp, Id *idp)
92 if ((p = getuint(p, endp, typep)) == 0)
98 *idp = REPOKEY_TYPE_SHA1;
102 *idp = REPOKEY_TYPE_SHA256;
106 *idp = REPOKEY_TYPE_SHA512;
110 *idp = REPOKEY_TYPE_SHA512;
119 skip_bytes(FILE *fp, size_t skip, Chksum *chk)
121 unsigned char buf[4096];
124 size_t bite = skip > sizeof(buf) ? sizeof(buf) : skip;
125 if (fread(buf, bite, 1, fp) != 1)
128 solv_chksum_add(chk, buf, bite);
135 nextchunk(struct solv_zchunk *zck, unsigned int streamid)
137 unsigned char *p = zck->chunks;
138 unsigned char *chunk_chk_ptr;
139 unsigned int sid, chunk_len, uncompressed_len;
142 /* free old buffer */
143 zck->buf = solv_free(zck->buf);
149 if (zck->nchunks == 0)
152 return 1; /* EOF reached */
154 if (p >= zck->hdr_end)
156 sid = streamid ? 1 : 0;
157 /* check if this is the correct stream */
158 if ((zck->flags & 1) != 0 && (p = getuint(p, zck->hdr_end, &sid)) == 0)
160 chunk_chk_ptr = p; /* remember for verification */
161 p += zck->chunk_chk_len;
162 if (p >= zck->hdr_end)
164 if ((p = getuint(p, zck->hdr_end, &chunk_len)) == 0)
166 if ((p = getuint(p, zck->hdr_end, &uncompressed_len)) == 0)
171 /* skip the chunk, but the dict chunk must come first */
172 if (streamid == 0 || skip_bytes(zck->fp, chunk_len, zck->data_chk) == 0)
177 /* ok, read the compressed chunk */
179 return uncompressed_len ? 0 : 1;
180 cbuf = solv_malloc(chunk_len);
181 if (fread(cbuf, chunk_len, 1, zck->fp) != 1)
187 solv_chksum_add(zck->data_chk, cbuf, chunk_len);
189 /* verify the chunk checksum */
190 if (zck->chunk_chk_id)
192 Chksum *chk = solv_chksum_create(zck->chunk_chk_id);
198 solv_chksum_add(chk, cbuf, chunk_len);
199 if (memcmp(solv_chksum_get(chk, 0), chunk_chk_ptr, zck->chunk_chk_len) != 0)
201 solv_chksum_free(chk, 0);
205 solv_chksum_free(chk, 0);
212 if (chunk_len != uncompressed_len)
218 zck->buf_avail = uncompressed_len;
223 /* zstd compressed */
225 zck->buf = solv_malloc(uncompressed_len + 1); /* +1 so we can detect too large frames */
227 r = ZSTD_decompress_usingDDict(zck->dctx, zck->buf, uncompressed_len + 1, cbuf, chunk_len, zck->ddict);
229 r = ZSTD_decompressDCtx(zck->dctx, zck->buf, uncompressed_len + 1, cbuf, chunk_len);
231 if (r != uncompressed_len)
233 zck->buf_avail = uncompressed_len;
240 static inline struct solv_zchunk *
241 open_error(struct solv_zchunk *zck)
243 solv_zchunk_close(zck);
248 solv_zchunk_open(FILE *fp, unsigned int streamid)
250 struct solv_zchunk *zck;
252 unsigned int hdr_size; /* preface + index + signatures */
253 unsigned int lead_size;
254 unsigned int preface_size;
255 unsigned int index_size;
257 zck = solv_calloc(1, sizeof(*zck));
259 /* read and parse the lead, read the complete header */
260 zck->hdr = solv_calloc(15, 1);
261 zck->hdr_end = zck->hdr + 15;
262 if (fread(zck->hdr, 15, 1, fp) != 1 || memcmp(zck->hdr, "\000ZCK1", 5) != 0)
263 return open_error(zck);
265 if ((p = getchksum(p, zck->hdr_end, &zck->hdr_chk_type, &zck->hdr_chk_len, &zck->hdr_chk_id)) == 0)
266 return open_error(zck);
267 if ((p = getuint(p, zck->hdr_end, &hdr_size)) == 0 || hdr_size > MAX_HDR_SIZE)
268 return open_error(zck);
269 lead_size = p - zck->hdr + zck->hdr_chk_len;
270 zck->hdr = solv_realloc(zck->hdr, lead_size + hdr_size);
271 zck->hdr_end = zck->hdr + lead_size + hdr_size;
272 if (fread(zck->hdr + 15, lead_size + hdr_size - 15, 1, fp) != 1)
273 return open_error(zck);
275 /* verify header checksum to guard against corrupt files */
278 Chksum *chk = solv_chksum_create(zck->hdr_chk_id);
280 return open_error(zck);
281 solv_chksum_add(chk, zck->hdr, lead_size - zck->hdr_chk_len);
282 solv_chksum_add(chk, zck->hdr + lead_size, hdr_size);
283 if (memcmp(solv_chksum_get(chk, 0), zck->hdr + (lead_size - zck->hdr_chk_len), zck->hdr_chk_len) != 0)
285 solv_chksum_free(chk, 0);
286 return open_error(zck);
288 solv_chksum_free(chk, 0);
291 /* parse preface: data chksum, flags, compression */
292 p = zck->hdr + lead_size;
293 if (p + zck->hdr_chk_len > zck->hdr_end)
294 return open_error(zck);
295 zck->data_chk_ptr = p;
296 p += zck->hdr_chk_len;
297 #ifdef VERIFY_DATA_CHKSUM
298 if (zck->hdr_chk_id && (zck->data_chk = solv_chksum_create(zck->hdr_chk_id)) == 0)
299 return open_error(zck);
301 if ((p = getuint(p, zck->hdr_end, &zck->flags)) == 0)
302 return open_error(zck);
303 if ((zck->flags & ~(3)) != 0)
304 return open_error(zck);
305 if ((p = getuint(p, zck->hdr_end, &zck->comp)) == 0 || (zck->comp != 0 && zck->comp != 2))
306 return open_error(zck); /* only uncompressed + zstd supported */
307 /* skip all optional elements if present */
308 if ((zck->flags & 2) != 0)
310 unsigned int nopt, lopt;
311 if ((p = getuint(p, zck->hdr_end, &nopt)) == 0)
312 return open_error(zck);
313 for (; nopt != 0; nopt--)
315 if ((p = getuint(p, zck->hdr_end, &lopt)) == 0)
316 return open_error(zck);
317 if ((p = getuint(p, zck->hdr_end, &lopt)) == 0)
318 return open_error(zck);
319 if (p + lopt > zck->hdr_end)
320 return open_error(zck);
325 preface_size = p - (zck->hdr + lead_size);
327 /* parse index: index size, index chksum type, num chunks, chunk data */
328 if ((p = getuint(p, zck->hdr_end, &index_size)) == 0)
329 return open_error(zck);
330 if (hdr_size < preface_size + index_size)
331 return open_error(zck);
332 if ((p = getchksum(p, zck->hdr_end, &zck->chunk_chk_type, &zck->chunk_chk_len, &zck->chunk_chk_id)) == 0)
333 return open_error(zck);
334 if ((p = getuint(p, zck->hdr_end, &zck->nchunks)) == 0 || zck->nchunks > MAX_CHUNK_CNT)
335 return open_error(zck);
337 /* setup decompressor */
340 if ((zck->dctx = ZSTD_createDCtx()) == 0)
341 return open_error(zck);
346 zck->streamid = streamid;
349 zck->nchunks = zck->nchunks ? 1 : 0; /* limit to dict chunk */
353 /* setup dictionary */
354 if (!nextchunk(zck, 0))
357 return open_error(zck);
359 if (zck->comp == 2 && zck->buf_avail)
361 if ((zck->ddict = ZSTD_createDDict(zck->buf, zck->buf_avail)) == 0)
364 return open_error(zck);
367 zck->buf = solv_free(zck->buf);
371 /* ready to read the rest of the chunks */
376 solv_zchunk_read(struct solv_zchunk *zck, char *buf, size_t len)
379 if (!zck || zck->eof == 2)
381 while (n < len && !zck->eof)
384 while (!zck->buf_avail)
388 /* verify data checksum if requested */
389 if (zck->streamid != 0 && zck->data_chk && memcmp(solv_chksum_get(zck->data_chk, 0), zck->data_chk_ptr, zck->hdr_chk_len) != 0) {
396 if (!nextchunk(zck, zck->streamid))
402 bite = len - n > zck->buf_avail ? zck->buf_avail : len - n;
403 memcpy(buf + n, zck->buf + zck->buf_used, bite);
405 zck->buf_used += bite;
406 zck->buf_avail -= bite;
412 solv_zchunk_close(struct solv_zchunk *zck)
415 solv_chksum_free(zck->data_chk, 0);
417 ZSTD_freeDDict(zck->ddict);
419 ZSTD_freeDCtx(zck->dctx);