2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Read the binary dump of a Repo and create a Repo * from it
14 * Repo *pool_addrepo_solv(Pool *pool, FILE *fp)
26 #include "repo_solv.h"
28 #include "attr_store_p.h"
30 #define INTERESTED_START SOLVABLE_NAME
31 #define INTERESTED_END SOLVABLE_FRESHENS
33 #define SOLV_ERROR_NOT_SOLV 1
34 #define SOLV_ERROR_UNSUPPORTED 2
35 #define SOLV_ERROR_EOF 3
36 #define SOLV_ERROR_ID_RANGE 4
37 #define SOLV_ERROR_OVERFLOW 5
38 #define SOLV_ERROR_CORRUPT 6
40 static Pool *mypool; /* for pool_debug... */
42 /*-----------------------------------------------------------------*/
43 /* .solv read functions */
50 read_u32(Repodata *data)
57 for (i = 0; i < 4; i++)
62 pool_debug(mypool, SAT_ERROR, "unexpected EOF\n");
63 data->error = SOLV_ERROR_EOF;
77 read_u8(Repodata *data)
86 pool_debug(mypool, SAT_ERROR, "unexpected EOF\n");
87 data->error = SOLV_ERROR_EOF;
99 read_id(Repodata *data, Id max)
106 for (i = 0; i < 5; i++)
111 pool_debug(mypool, SAT_ERROR, "unexpected EOF\n");
112 data->error = SOLV_ERROR_EOF;
120 pool_debug(mypool, SAT_ERROR, "read_id: id too large (%u/%u)\n", x, max);
121 data->error = SOLV_ERROR_ID_RANGE;
126 x = (x << 7) ^ c ^ 128;
128 pool_debug(mypool, SAT_ERROR, "read_id: id too long\n");
129 data->error = SOLV_ERROR_CORRUPT;
139 read_idarray(Repodata *data, Id max, Id *map, Id *store, Id *end, int relative)
152 pool_debug(mypool, SAT_ERROR, "unexpected EOF\n");
153 data->error = SOLV_ERROR_EOF;
158 x = (x << 6) | (c & 63);
161 if (x == 0 && c == 0x40)
166 pool_debug(mypool, SAT_ERROR, "read_idarray: array overflow\n");
167 data->error = SOLV_ERROR_OVERFLOW;
170 *store++ = SOLVABLE_PREREQMARKER;
180 pool_debug(mypool, SAT_ERROR, "read_idarray: id too large (%u/%u)\n", x, max);
181 data->error = SOLV_ERROR_ID_RANGE;
188 pool_debug(mypool, SAT_ERROR, "read_idarray: array overflow\n");
194 if (x == 0) /* already have trailing zero? */
198 pool_debug(mypool, SAT_ERROR, "read_idarray: array overflow\n");
199 data->error = SOLV_ERROR_OVERFLOW;
208 x = (x << 7) ^ c ^ 128;
213 read_str(Repodata *data, char **inbuf, unsigned *len)
215 unsigned char *buf = (unsigned char*)*inbuf;
218 buf = xmalloc (1024);
223 while((c = getc(data->fp)) != 0)
227 pool_debug (mypool, SAT_ERROR, "unexpected EOF\n");
228 data->error = SOLV_ERROR_EOF;
231 /* Plus 1 as we also want to add the 0. */
235 /* Don't realloc on the inbuf, it might be on the stack. */
236 if (buf == (unsigned char*)*inbuf)
239 memcpy(buf, *inbuf, *len - 256);
242 buf = xrealloc(buf, *len);
251 skip_item (Repodata *data, unsigned type, unsigned numid, unsigned numrel)
258 read_id(data, numid + numrel); /* just check Id */
263 case TYPE_ATTR_STRING:
265 while(read_u8(data) != 0)
269 case TYPE_IDVALUEARRAY:
270 case TYPE_IDVALUEVALUEARRAY:
271 case TYPE_REL_IDARRAY:
272 while ((read_u8(data) & 0xc0) != 0)
275 case TYPE_COUNT_NAMED:
277 unsigned count = read_id(data, 0);
280 read_id(data, numid); /* Name */
281 unsigned t = read_id(data, TYPE_ATTR_TYPE_MAX + 1);
282 skip_item(data, t, numid, numrel);
288 unsigned count = read_id(data, 0);
289 unsigned t = read_id(data, TYPE_ATTR_TYPE_MAX + 1);
291 skip_item(data, t, numid, numrel);
294 case TYPE_ATTR_CHUNK:
300 case TYPE_ATTR_INTLIST:
301 case TYPE_ATTR_LOCALIDS:
302 while (read_id(data, 0) != 0)
306 pool_debug(mypool, SAT_ERROR, "unknown type %d\n", type);
307 data->error = SOLV_ERROR_CORRUPT;
313 key_cmp (const void *pa, const void *pb)
315 Repokey *a = (Repokey *)pa;
316 Repokey *b = (Repokey *)pb;
317 return a->name - b->name;
321 parse_repodata(Repodata *maindata, Id *keyp, Repokey *keys, Id *idmap, unsigned numid, unsigned numrel, Repo *repo)
328 repo->repodata = xrealloc(repo->repodata, (repo->nrepodata + 1) * sizeof (*data));
329 data = repo->repodata + repo->nrepodata++;
331 memset(data, 0, sizeof(*data));
333 while ((key = *keyp++) != 0)
336 switch (keys[key].type)
338 case TYPE_IDVALUEARRAY:
339 if (id != REPODATA_KEYS)
341 skip_item(maindata, TYPE_IDVALUEARRAY, numid, numrel);
344 ida = xcalloc(keys[key].size, sizeof(Id));
345 ide = read_idarray(maindata, 0, 0, ida, ida + keys[key].size, 0);
349 data->error = SOLV_ERROR_CORRUPT;
352 data->nkeys = 1 + (n >> 1);
353 data->keys = xmalloc2(data->nkeys, sizeof(data->keys[0]));
354 memset(data->keys, 0, sizeof(Repokey));
355 for (i = 1, ide = ida; i < data->nkeys; i++)
359 data->error = SOLV_ERROR_CORRUPT;
360 pool_debug (mypool, SAT_ERROR, "invalid attribute data\n");
363 data->keys[i].name = idmap ? idmap[*ide++] : *ide++;
364 data->keys[i].type = *ide++;
365 data->keys[i].size = 0;
366 data->keys[i].storage = 0;
370 qsort(data->keys + 1, data->nkeys - 1, sizeof(data->keys[0]), key_cmp);
373 if (id != REPODATA_LOCATION)
374 skip_item(maindata, TYPE_STR, numid, numrel);
378 unsigned len = sizeof (buf);
379 char *filename = buf;
380 read_str(maindata, &filename, &len);
381 data->location = strdup(filename);
387 skip_item(maindata, keys[key].type, numid, numrel);
393 /*-----------------------------------------------------------------*/
397 skip_schema(Repodata *data, Id *keyp, Repokey *keys, unsigned int numid, unsigned int numrel)
400 while ((key = *keyp++) != 0)
401 skip_item(data, keys[key].type, numid, numrel);
404 /*-----------------------------------------------------------------*/
407 incore_add_id(Repodata *data, Id x)
410 /* make sure we have at least 5 bytes free */
411 if (data->incoredatafree < 5)
413 data->incoredata = xrealloc(data->incoredata, data->incoredatalen + 1024);
414 data->incoredatafree = 1024;
416 dp = data->incoredata + data->incoredatalen;
422 *dp++ = (x >> 28) | 128;
424 *dp++ = (x >> 21) | 128;
425 *dp++ = (x >> 14) | 128;
428 *dp++ = (x >> 7) | 128;
430 data->incoredatafree -= dp - (data->incoredata + data->incoredatalen);
431 data->incoredatalen = dp - data->incoredata;
435 incore_add_u32(Repodata *data, unsigned int x)
438 /* make sure we have at least 4 bytes free */
439 if (data->incoredatafree < 4)
441 data->incoredata = xrealloc(data->incoredata, data->incoredatalen + 1024);
442 data->incoredatafree = 1024;
444 dp = data->incoredata + data->incoredatalen;
449 data->incoredatafree -= 4;
450 data->incoredatalen += 4;
454 incore_add_u8(Repodata *data, unsigned int x)
457 /* make sure we have at least 1 byte free */
458 if (data->incoredatafree < 1)
460 data->incoredata = xrealloc(data->incoredata, data->incoredatalen + 1024);
461 data->incoredatafree = 1024;
463 dp = data->incoredata + data->incoredatalen;
465 data->incoredatafree--;
466 data->incoredatalen++;
469 // ----------------------------------------------
472 * read repo from .solv file
477 repo_add_solv(Repo *repo, FILE *fp)
479 Pool *pool = repo->pool;
481 unsigned int numid, numrel, numsolv;
482 unsigned int numkeys, numschemata, numinfo;
484 Attrstore *embedded_store = 0;
488 Offset *str; /* map Id -> Offset into string space */
489 char *strsp; /* repo string space */
490 char *sp; /* pointer into string space */
491 Id *idmap; /* map of repo Ids to pool Ids */
493 unsigned int hashmask, h;
499 unsigned int size_idarray;
500 Id *idarraydatap, *idarraydataend;
503 unsigned int solvflags;
504 unsigned int solvversion;
506 Id *schemadata, *schemadatap, *schemadataend;
511 memset(&data, 0, sizeof(data));
517 if (read_u32(&data) != ('S' << 24 | 'O' << 16 | 'L' << 8 | 'V'))
519 pool_debug(pool, SAT_ERROR, "not a SOLV file\n");
520 return SOLV_ERROR_NOT_SOLV;
522 solvversion = read_u32(&data);
530 pool_debug(pool, SAT_ERROR, "unsupported SOLV version\n");
531 return SOLV_ERROR_UNSUPPORTED;
534 pool_freeidhashes(pool);
536 numid = read_u32(&data);
537 numrel = read_u32(&data);
538 numsolv = read_u32(&data);
539 numkeys = read_u32(&data);
540 numschemata = read_u32(&data);
541 numinfo = read_u32(&data);
542 solvflags = read_u32(&data);
544 if (solvversion < SOLV_VERSION_3
547 pool_debug(pool, SAT_ERROR, "unsupported SOLV format (has info)\n");
548 return SOLV_ERROR_UNSUPPORTED;
551 /******* Part 1: string IDs *****************************************/
553 sizeid = read_u32(&data); /* size of string+Id space */
556 * read strings and Ids
565 /* alloc string buffer */
566 pool->ss.stringspace = xrealloc(pool->ss.stringspace, pool->ss.sstrings + sizeid + 1);
567 /* alloc string offsets (Id -> Offset into string space) */
568 pool->ss.strings = xrealloc2(pool->ss.strings, pool->ss.nstrings + numid, sizeof(Offset));
570 strsp = pool->ss.stringspace;
571 str = pool->ss.strings; /* array of offsets into strsp, indexed by Id */
573 /* point to _BEHIND_ already allocated string/Id space */
574 strsp += pool->ss.sstrings;
578 * read new repo at end of pool
581 if ((solvflags & SOLV_FLAG_PREFIX_POOL) == 0)
583 if (fread(strsp, sizeid, 1, fp) != 1)
585 pool_debug(pool, SAT_ERROR, "read error while reading strings\n");
586 return SOLV_ERROR_EOF;
591 unsigned int pfsize = read_u32(&data);
592 char *prefix = xmalloc(pfsize);
596 if (fread(prefix, pfsize, 1, fp) != 1)
598 pool_debug(pool, SAT_ERROR, "read error while reading strings\n");
600 return SOLV_ERROR_EOF;
602 for (i = 1; i < numid; i++)
604 int same = (unsigned char)*pp++;
605 size_t len = strlen (pp) + 1;
607 memcpy(dest, old_str, same);
608 memcpy(dest + same, pp, len);
615 strsp[sizeid] = 0; /* make string space \0 terminated */
619 /* alloc id map for name and rel Ids. this maps ids in the solv files
620 * to the ids in our pool */
621 idmap = (Id *)xcalloc(numid + numrel, sizeof(Id));
624 * build hashes for all read strings
628 hashmask = mkmask(pool->ss.nstrings + numid);
631 POOL_DEBUG(SAT_DEBUG_STATS, "read %d strings\n", numid);
632 POOL_DEBUG(SAT_DEBUG_STATS, "string hash buckets: %d\n", hashmask + 1);
636 * create hashtable with strings already in pool
639 hashtbl = (Id *)xcalloc(hashmask + 1, sizeof(Id));
640 for (i = 1; i < pool->ss.nstrings; i++) /* leave out our dummy zero id */
642 h = strhash(pool->ss.stringspace + pool->ss.strings[i]) & hashmask;
643 hh = HASHCHAIN_START;
645 h = HASHCHAIN_NEXT(h, hh, hashmask);
650 * run over string space, calculate offsets
652 * build id map (maps solv Id -> pool Id)
655 for (i = 1; i < numid; i++)
657 if (sp >= strsp + sizeid)
661 pool_debug(pool, SAT_ERROR, "not enough strings\n");
662 return SOLV_ERROR_OVERFLOW;
664 if (!*sp) /* empty string */
672 h = strhash(sp) & hashmask;
673 hh = HASHCHAIN_START;
679 if (!strcmp(pool->ss.stringspace + pool->ss.strings[id], sp))
680 break; /* existing string */
681 h = HASHCHAIN_NEXT(h, hh, hashmask);
684 /* length == offset to next string */
686 if (id == ID_NULL) /* end of hash chain -> new string */
688 id = pool->ss.nstrings++;
690 str[id] = pool->ss.sstrings; /* save Offset */
691 if (sp != pool->ss.stringspace + pool->ss.sstrings) /* not at end-of-buffer */
692 memmove(pool->ss.stringspace + pool->ss.sstrings, sp, l); /* append to pool buffer */
693 pool->ss.sstrings += l;
695 idmap[i] = id; /* repo relative -> pool relative */
696 sp += l; /* next string */
699 pool_shrink_strings(pool); /* vacuum */
702 /******* Part 2: Relation IDs ***************************************/
712 pool->rels = xrealloc2(pool->rels, pool->nrels + numrel, sizeof(Reldep));
715 hashmask = mkmask(pool->nrels + numrel);
717 POOL_DEBUG(SAT_DEBUG_STATS, "read %d rels\n", numrel);
718 POOL_DEBUG(SAT_DEBUG_STATS, "rel hash buckets: %d\n", hashmask + 1);
721 * prep hash table with already existing RelDeps
724 hashtbl = xcalloc(hashmask + 1, sizeof(Id));
725 for (i = 1; i < pool->nrels; i++)
727 h = relhash(ran[i].name, ran[i].evr, ran[i].flags) & hashmask;
728 hh = HASHCHAIN_START;
730 h = HASHCHAIN_NEXT(h, hh, hashmask);
735 * read RelDeps from repo
738 for (i = 0; i < numrel; i++)
740 name = read_id(&data, i + numid); /* read (repo relative) Ids */
741 evr = read_id(&data, i + numid);
742 flags = read_u8(&data);
743 name = idmap[name]; /* map to (pool relative) Ids */
745 h = relhash(name, evr, flags) & hashmask;
746 hh = HASHCHAIN_START;
750 if (id == ID_NULL) /* end of hash chain */
752 if (ran[id].name == name && ran[id].evr == evr && ran[id].flags == flags)
754 h = HASHCHAIN_NEXT(h, hh, hashmask);
756 if (id == ID_NULL) /* new RelDep */
762 ran[id].flags = flags;
764 idmap[i + numid] = MAKERELDEP(id); /* fill Id map */
767 pool_shrink_rels(pool); /* vacuum */
771 /******* Part 3: Keys ***********************************************/
773 keys = xcalloc(numkeys, sizeof(*keys));
774 /* keys start at 1 */
775 for (i = 1; i < numkeys; i++)
777 id = read_id(&data, numid);
781 keys[i].type = read_id(&data, 0);
782 keys[i].size = read_id(&data, 0);
783 keys[i].storage = KEY_STORAGE_DROPPED;
784 switch (keys[i].type)
792 case SOLVABLE_VENDOR:
793 keys[i].storage = KEY_STORAGE_SOLVABLE;
796 keys[i].storage = KEY_STORAGE_INCORE;
801 case TYPE_REL_IDARRAY:
802 if (id >= INTERESTED_START && id <= INTERESTED_END)
803 keys[i].storage = KEY_STORAGE_SOLVABLE;
805 keys[i].storage = KEY_STORAGE_INCORE;
808 keys[i].storage = KEY_STORAGE_INCORE;
811 if (id == RPM_RPMDBID)
812 keys[i].storage = KEY_STORAGE_SOLVABLE;
814 keys[i].storage = KEY_STORAGE_INCORE;
822 data.nkeys = numkeys;
824 /******* Part 4: Schemata ********************************************/
826 id = read_id(&data, 0);
827 schemadata = xcalloc(id, sizeof(Id));
828 schemadatap = schemadata;
829 schemadataend = schemadata + id;
830 schemata = xcalloc(numschemata, sizeof(Id));
831 for (i = 0; i < numschemata; i++)
833 schemata[i] = schemadatap - schemadata;
834 schemadatap = read_idarray(&data, numid, 0, schemadatap, schemadataend, 0);
836 data.schemata = schemata;
837 data.nschemata = numschemata;
838 data.schemadata = schemadata;
840 /******* Part 5: Info ***********************************************/
841 for (i = 0; i < numinfo; i++)
843 /* for now we're just interested in data that starts with
844 * the repodata_external id
846 Id *keyp = schemadata + schemata[read_id(&data, numschemata)];
848 if (keys[key].name == REPODATA_EXTERNAL && keys[key].type == TYPE_VOID)
850 /* external data for some ids */
851 parse_repodata(&data, keyp, keys, idmap, numid, numrel, repo);
854 skip_schema(&data, keyp, keys, numid, numrel);
857 /******* Part 6: packed sizes (optional) ****************************/
859 if ((solvflags & SOLV_FLAG_PACKEDSIZES) != 0)
861 exists = xmalloc (numsolv);
862 for (i = 0; i < numsolv; i++)
863 exists[i] = read_id(&data, 0) != 0;
866 /******* Part 7: item data *******************************************/
868 /* calculate idarray size */
870 for (i = 1; i < numkeys; i++)
873 if ((keys[i].type == TYPE_IDARRAY || keys[i].type == TYPE_REL_IDARRAY)
874 && id >= INTERESTED_START && id <= INTERESTED_END)
875 size_idarray += keys[i].size;
878 /* allocate needed space in repo */
881 repo_reserve_ids(repo, 0, size_idarray);
882 idarraydatap = repo->idarraydata + repo->idarraysize;
883 repo->idarraysize += size_idarray;
884 idarraydataend = idarraydatap + size_idarray;
894 s = pool_id2solvable(pool, repo_add_solvable_block(repo, numsolv));
896 /* store start and end of our id block */
897 data.start = s - pool->solvables;
898 data.end = data.start + numsolv;
900 for (i = 0; i < numsolv; i++, s++)
904 if (exists && !exists[i])
906 Id *keyp = schemadata + schemata[read_id(&data, numschemata)];
907 while ((key = *keyp++) != 0)
910 switch (keys[key].type)
913 did = read_id(&data, numid + numrel);
916 if (id == SOLVABLE_NAME)
918 else if (id == SOLVABLE_ARCH)
920 else if (id == SOLVABLE_EVR)
922 else if (id == SOLVABLE_VENDOR)
924 else if (keys[key].storage == KEY_STORAGE_INCORE)
925 incore_add_id(&data, did);
927 POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %s\n", id2str(pool, id), id2str(pool, did));
933 POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %u\n", id2str(pool, id), h);
935 if (id == RPM_RPMDBID)
938 repo->rpmdbid = (Id *)xcalloc(numsolv, sizeof(Id));
939 repo->rpmdbid[i] = h;
941 else if (keys[key].storage == KEY_STORAGE_INCORE)
942 incore_add_u32(&data, h);
945 if (keys[key].storage == KEY_STORAGE_INCORE)
947 while ((h = read_u8(&data)) != 0)
948 incore_add_u8(&data, h);
949 incore_add_u8(&data, 0);
953 while (read_u8(&data) != 0)
958 case TYPE_REL_IDARRAY:
959 if (id < INTERESTED_START || id > INTERESTED_END)
961 if (keys[key].storage == KEY_STORAGE_INCORE)
965 Id old = 0, rel = keys[key].type == TYPE_REL_IDARRAY ? SOLVABLE_PREREQMARKER : 0;
968 did = read_id(&data, 0);
970 did = (did & 0x3f) | ((did >> 1) & ~0x3f);
984 if (did >= numid + numrel)
987 did = ((did & ~0x3f) << 1) | h;
988 incore_add_id(&data, did);
994 while (((h = read_u8(&data)) & 0xc0) != 0)
995 incore_add_u8(&data, h);
1001 while ((read_u8(&data) & 0xc0) != 0)
1007 ido = idarraydatap - repo->idarraydata;
1008 idarraydatap = read_idarray(&data, numid + numrel, idmap, idarraydatap, idarraydataend, keys[key].type == TYPE_REL_IDARRAY);
1009 if (id == SOLVABLE_PROVIDES)
1011 else if (id == SOLVABLE_OBSOLETES)
1013 else if (id == SOLVABLE_CONFLICTS)
1015 else if (id == SOLVABLE_REQUIRES)
1017 else if (id == SOLVABLE_RECOMMENDS)
1019 else if (id == SOLVABLE_SUPPLEMENTS)
1020 s->supplements = ido;
1021 else if (id == SOLVABLE_SUGGESTS)
1023 else if (id == SOLVABLE_ENHANCES)
1025 else if (id == SOLVABLE_FRESHENS)
1028 POOL_DEBUG(SAT_DEBUG_STATS, "%s ->\n", id2str(pool, id));
1029 for (; repo->idarraydata[ido]; ido++)
1030 POOL_DEBUG(SAT_DEBUG_STATS," %s\n", dep2str(pool, repo->idarraydata[ido]));
1037 case TYPE_ATTR_CHUNK:
1038 case TYPE_ATTR_STRING:
1039 case TYPE_ATTR_INTLIST:
1040 case TYPE_ATTR_LOCALIDS:
1041 if (!embedded_store)
1042 embedded_store = new_store (pool);
1043 add_attr_from_file (embedded_store, i, id, keys[key].type, idmap, numid, &data);
1047 skip_item(&data, keys[key].type, numid, numrel);
1054 /* free solvables */
1055 repo_free_solvable_block(repo, data.start, data.end - data.start, 1);
1057 repo->idarraysize -= size_idarray;
1058 /* free incore data */
1059 data.incoredata = xfree(data.incoredata);
1060 data.incoredatalen = data.incoredatafree = 0;
1063 if (data.incoredatafree)
1065 /* shrink excess size */
1066 data.incoredata = xrealloc(data.incoredata, data.incoredatalen);
1067 data.incoredatafree = 0;
1070 for (i = 1; i < numkeys; i++)
1071 if (keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
1073 if (i < numkeys && !data.error)
1075 /* we have vertical data, make it available */
1076 data.verticaloffset = ftell(fp);
1080 /* no longer needed */
1084 if (data.incoredatalen || data.fp)
1086 /* we got some data, make it available */
1087 repo->repodata = xrealloc(repo->repodata, (repo->nrepodata + 1) * sizeof(data));
1088 repo->repodata[repo->nrepodata++] = data;
1101 attr_store_pack (embedded_store);
1102 /* If we have any attributes we also have pages. */
1103 read_or_setup_pages (fp, embedded_store);
1104 /* The NULL name here means embedded attributes. */
1105 repo_add_attrstore (repo, embedded_store, NULL);