2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Read the binary dump of a Repo and create a Repo * from it
14 * Repo *pool_addrepo_solv(Pool *pool, FILE *fp)
26 #include "repo_solv.h"
28 #include "attr_store_p.h"
30 #define INTERESTED_START SOLVABLE_NAME
31 #define INTERESTED_END SOLVABLE_FRESHENS
33 static Pool *mypool; /* for pool_debug... */
35 /*-----------------------------------------------------------------*/
36 /* .solv read functions */
48 for (i = 0; i < 4; i++)
53 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
73 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
85 read_id(FILE *fp, Id max)
90 for (i = 0; i < 5; i++)
95 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
103 pool_debug(mypool, SAT_FATAL, "read_id: id too large (%u/%u)\n", x, max);
108 x = (x << 7) ^ c ^ 128;
110 pool_debug(mypool, SAT_FATAL, "read_id: id too long\n");
120 read_idarray(FILE *fp, Id max, Id *map, Id *store, Id *end, int relative)
130 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
135 x = (x << 6) | (c & 63);
138 if (x == 0 && c == 0x40)
143 pool_debug(mypool, SAT_FATAL, "read_idarray: array overflow\n");
146 *store++ = SOLVABLE_PREREQMARKER;
156 pool_debug(mypool, SAT_FATAL, "read_idarray: id too large (%u/%u)\n", x, max);
163 pool_debug(mypool, SAT_FATAL, "read_idarray: array overflow\n");
169 if (x == 0) /* already have trailing zero? */
173 pool_debug(mypool, SAT_FATAL, "read_idarray: array overflow\n");
182 x = (x << 7) ^ c ^ 128;
187 read_str (FILE *fp, char **inbuf, unsigned *len)
189 unsigned char *buf = (unsigned char*)*inbuf;
192 buf = xmalloc (1024);
197 while((c = getc (fp)) != 0)
201 pool_debug (mypool, SAT_FATAL, "unexpected EOF\n");
204 /* Plus 1 as we also want to add the 0. */
208 /* Don't realloc on the inbuf, it might be on the stack. */
209 if (buf == (unsigned char*)*inbuf)
211 buf = xmalloc (*len);
212 memcpy (buf, *inbuf, *len - 256);
215 buf = xrealloc (buf, *len);
224 skip_item (FILE *fp, unsigned type, unsigned numid, unsigned numrel)
231 read_id(fp, numid + numrel); /* just check Id */
236 case TYPE_ATTR_STRING:
238 while(read_u8(fp) != 0)
242 case TYPE_IDVALUEARRAY:
243 case TYPE_IDVALUEVALUEARRAY:
244 case TYPE_REL_IDARRAY:
245 while ((read_u8(fp) & 0xc0) != 0)
248 case TYPE_COUNT_NAMED:
250 unsigned count = read_id (fp, 0);
253 read_id (fp, numid); /* Name */
254 unsigned t = read_id (fp, TYPE_ATTR_TYPE_MAX + 1);
255 skip_item (fp, t, numid, numrel);
261 unsigned count = read_id (fp, 0);
262 unsigned t = read_id (fp, TYPE_ATTR_TYPE_MAX + 1);
264 skip_item (fp, t, numid, numrel);
267 case TYPE_ATTR_CHUNK:
273 case TYPE_ATTR_INTLIST:
274 case TYPE_ATTR_LOCALIDS:
275 while (read_id(fp, 0) != 0)
279 pool_debug(mypool, SAT_FATAL, "unknown type %d\n", type);
285 key_cmp (const void *pa, const void *pb)
287 Repokey *a = (Repokey *)pa;
288 Repokey *b = (Repokey *)pb;
289 return a->name - b->name;
293 parse_repodata (FILE *fp, Id *keyp, Repokey *keys, Id *idmap, unsigned numid, unsigned numrel, Repo *repo)
300 repo->repodata = xrealloc(repo->repodata, (repo->nrepodata + 1) * sizeof (*data));
301 data = repo->repodata + repo->nrepodata++;
303 memset(data, 0, sizeof(*data));
305 while ((key = *keyp++) != 0)
308 switch (keys[key].type)
310 case TYPE_IDVALUEARRAY:
311 if (id != REPODATA_KEYS)
313 skip_item(fp, TYPE_IDVALUEARRAY, numid, numrel);
316 ida = xcalloc(keys[key].size, sizeof(Id));
317 ide = read_idarray(fp, 0, 0, ida, ida + keys[key].size, 0);
321 pool_debug (mypool, SAT_FATAL, "invalid attribute data\n");
324 data->nkeys = 1 + (n >> 1);
325 data->keys = xmalloc2(data->nkeys, sizeof(data->keys[0]));
326 memset(data->keys, 0, sizeof(Repokey));
327 for (i = 1, ide = ida; i < data->nkeys; i++)
331 pool_debug (mypool, SAT_FATAL, "invalid attribute data\n");
334 data->keys[i].name = idmap[*ide++];
335 data->keys[i].type = *ide++;
336 data->keys[i].size = 0;
337 data->keys[i].storage = 0;
341 qsort(data->keys + 1, data->nkeys - 1, sizeof(data->keys[0]), key_cmp);
344 if (id != REPODATA_LOCATION)
345 skip_item(fp, TYPE_STR, numid, numrel);
349 unsigned len = sizeof (buf);
350 char *filename = buf;
351 read_str(fp, &filename, &len);
352 data->location = strdup(filename);
358 skip_item(fp, keys[key].type, numid, numrel);
364 /*-----------------------------------------------------------------*/
368 skip_schema(FILE *fp, Id *keyp, Repokey *keys, unsigned int numid, unsigned int numrel)
371 while ((key = *keyp++) != 0)
372 skip_item(fp, keys[key].type, numid, numrel);
375 /*-----------------------------------------------------------------*/
378 incore_add_id(Repodata *data, Id x)
381 /* make sure we have at least 5 bytes free */
382 if (data->incoredatafree < 5)
384 data->incoredata = xrealloc(data->incoredata, data->incoredatalen + 1024);
385 data->incoredatafree = 1024;
387 dp = data->incoredata + data->incoredatalen;
393 *dp++ = (x >> 28) | 128;
395 *dp++ = (x >> 21) | 128;
396 *dp++ = (x >> 14) | 128;
399 *dp++ = (x >> 7) | 128;
401 data->incoredatafree -= dp - (data->incoredata + data->incoredatalen);
402 data->incoredatalen = dp - data->incoredata;
406 incore_add_u32(Repodata *data, unsigned int x)
409 /* make sure we have at least 4 bytes free */
410 if (data->incoredatafree < 4)
412 data->incoredata = xrealloc(data->incoredata, data->incoredatalen + 1024);
413 data->incoredatafree = 1024;
415 dp = data->incoredata + data->incoredatalen;
420 data->incoredatafree -= 4;
421 data->incoredatalen += 4;
425 incore_add_u8(Repodata *data, unsigned int x)
428 /* make sure we have at least 1 byte free */
429 if (data->incoredatafree < 1)
431 data->incoredata = xrealloc(data->incoredata, data->incoredatalen + 1024);
432 data->incoredatafree = 1024;
434 dp = data->incoredata + data->incoredatalen;
436 data->incoredatafree--;
437 data->incoredatalen++;
440 // ----------------------------------------------
443 * read repo from .solv file
448 repo_add_solv(Repo *repo, FILE *fp)
450 Pool *pool = repo->pool;
452 unsigned int numid, numrel, numsolv;
453 unsigned int numkeys, numschemata, numinfo;
455 Attrstore *embedded_store = 0;
459 Offset *str; /* map Id -> Offset into string space */
460 char *strsp; /* repo string space */
461 char *sp; /* pointer into string space */
462 Id *idmap; /* map of repo Ids to pool Ids */
464 unsigned int hashmask, h;
470 unsigned int size_idarray;
471 Id *idarraydatap, *idarraydataend;
474 unsigned int solvflags;
475 unsigned int solvversion;
477 Id *schemadata, *schemadatap, *schemadataend;
484 if (read_u32(fp) != ('S' << 24 | 'O' << 16 | 'L' << 8 | 'V'))
486 pool_debug(pool, SAT_FATAL, "not a SOLV file\n");
489 solvversion = read_u32(fp);
497 pool_debug(pool, SAT_FATAL, "unsupported SOLV version\n");
501 pool_freeidhashes(pool);
503 numid = read_u32(fp);
504 numrel = read_u32(fp);
505 numsolv = read_u32(fp);
506 numkeys = read_u32(fp);
507 numschemata = read_u32(fp);
508 numinfo = read_u32(fp);
509 solvflags = read_u32(fp);
511 memset(&data, 0, sizeof(data));
514 if (solvversion < SOLV_VERSION_3
517 pool_debug(pool, SAT_FATAL, "unsupported SOLV format (has info)\n");
521 /******* Part 1: string IDs *****************************************/
523 sizeid = read_u32(fp); /* size of string+Id space */
526 * read strings and Ids
535 /* alloc string buffer */
536 strsp = (char *)xrealloc(pool->ss.stringspace, pool->ss.sstrings + sizeid + 1);
537 /* alloc string offsets (Id -> Offset into string space) */
538 str = (Offset *)xrealloc(pool->ss.strings, (pool->ss.nstrings + numid) * sizeof(Offset));
540 pool->ss.stringspace = strsp;
541 pool->ss.strings = str; /* array of offsets into strsp, indexed by Id */
543 /* point to _BEHIND_ already allocated string/Id space */
544 strsp += pool->ss.sstrings;
546 /* alloc id map for name and rel Ids. this maps ids in the solv files
547 * to the ids in our pool */
548 idmap = (Id *)xcalloc(numid + numrel, sizeof(Id));
551 * read new repo at end of pool
554 if ((solvflags & SOLV_FLAG_PREFIX_POOL) == 0)
556 if (fread(strsp, sizeid, 1, fp) != 1)
558 pool_debug(pool, SAT_FATAL, "read error while reading strings\n");
564 unsigned int pfsize = read_u32 (fp);
565 char *prefix = xmalloc (pfsize);
569 if (fread (prefix, pfsize, 1, fp) != 1)
571 pool_debug(pool, SAT_FATAL, "read error while reading strings\n");
574 for (i = 1; i < numid; i++)
576 int same = (unsigned char)*pp++;
577 size_t len = strlen (pp) + 1;
579 memcpy (dest, old_str, same);
580 memcpy (dest + same, pp, len);
587 strsp[sizeid] = 0; /* make string space \0 terminated */
591 * build hashes for all read strings
595 hashmask = mkmask(pool->ss.nstrings + numid);
598 POOL_DEBUG(SAT_DEBUG_STATS, "read %d strings\n", numid);
599 POOL_DEBUG(SAT_DEBUG_STATS, "string hash buckets: %d\n", hashmask + 1);
603 * ensure sufficient hash size
606 hashtbl = (Id *)xcalloc(hashmask + 1, sizeof(Id));
609 * fill hashtable with strings already in pool
612 for (i = 1; i < pool->ss.nstrings; i++) /* leave out our dummy zero id */
614 h = strhash(pool->ss.stringspace + pool->ss.strings[i]) & hashmask;
615 hh = HASHCHAIN_START;
617 h = HASHCHAIN_NEXT(h, hh, hashmask);
622 * run over string space, calculate offsets
624 * build id map (maps solv Id -> pool Id)
627 for (i = 1; i < numid; i++)
629 if (sp >= strsp + sizeid)
631 pool_debug(pool, SAT_FATAL, "not enough strings\n");
634 if (!*sp) /* empty string */
642 h = strhash(sp) & hashmask;
643 hh = HASHCHAIN_START;
649 if (!strcmp(pool->ss.stringspace + pool->ss.strings[id], sp))
650 break; /* existing string */
651 h = HASHCHAIN_NEXT(h, hh, hashmask);
654 /* length == offset to next string */
656 if (id == ID_NULL) /* end of hash chain -> new string */
658 id = pool->ss.nstrings++;
660 str[id] = pool->ss.sstrings; /* save Offset */
661 if (sp != pool->ss.stringspace + pool->ss.sstrings) /* not at end-of-buffer */
662 memmove(pool->ss.stringspace + pool->ss.sstrings, sp, l); /* append to pool buffer */
663 pool->ss.sstrings += l;
665 idmap[i] = id; /* repo relative -> pool relative */
666 sp += l; /* next string */
669 pool_shrink_strings(pool); /* vacuum */
672 /******* Part 2: Relation IDs ***************************************/
682 ran = (Reldep *)xrealloc(pool->rels, (pool->nrels + numrel) * sizeof(Reldep));
685 pool_debug(pool, SAT_FATAL, "no mem for rel space\n");
688 pool->rels = ran; /* extended rel space */
690 hashmask = mkmask(pool->nrels + numrel);
692 POOL_DEBUG(SAT_DEBUG_STATS, "read %d rels\n", numrel);
693 POOL_DEBUG(SAT_DEBUG_STATS, "rel hash buckets: %d\n", hashmask + 1);
696 * prep hash table with already existing RelDeps
699 hashtbl = (Id *)xcalloc(hashmask + 1, sizeof(Id));
700 for (i = 1; i < pool->nrels; i++)
702 h = relhash(ran[i].name, ran[i].evr, ran[i].flags) & hashmask;
703 hh = HASHCHAIN_START;
705 h = HASHCHAIN_NEXT(h, hh, hashmask);
710 * read RelDeps from repo
713 for (i = 0; i < numrel; i++)
715 name = read_id(fp, i + numid); /* read (repo relative) Ids */
716 evr = read_id(fp, i + numid);
718 name = idmap[name]; /* map to (pool relative) Ids */
720 h = relhash(name, evr, flags) & hashmask;
721 hh = HASHCHAIN_START;
725 if (id == ID_NULL) /* end of hash chain */
727 if (ran[id].name == name && ran[id].evr == evr && ran[id].flags == flags)
729 h = HASHCHAIN_NEXT(h, hh, hashmask);
731 if (id == ID_NULL) /* new RelDep */
737 ran[id].flags = flags;
739 idmap[i + numid] = MAKERELDEP(id); /* fill Id map */
742 pool_shrink_rels(pool); /* vacuum */
746 /******* Part 3: Keys ***********************************************/
748 keys = xcalloc(numkeys, sizeof(*keys));
749 /* keys start at 1 */
750 for (i = 1; i < numkeys; i++)
752 keys[i].name = id = idmap[read_id(fp, numid)];
753 keys[i].type = read_id(fp, 0);
754 keys[i].size = read_id(fp, 0);
755 keys[i].storage = KEY_STORAGE_DROPPED;
756 switch (keys[i].type)
764 case SOLVABLE_VENDOR:
765 keys[i].storage = KEY_STORAGE_SOLVABLE;
768 keys[i].storage = KEY_STORAGE_INCORE;
773 case TYPE_REL_IDARRAY:
774 if (id >= INTERESTED_START && id <= INTERESTED_END)
775 keys[i].storage = KEY_STORAGE_SOLVABLE;
777 keys[i].storage = KEY_STORAGE_INCORE;
780 keys[i].storage = KEY_STORAGE_INCORE;
783 if (id == RPM_RPMDBID)
784 keys[i].storage = KEY_STORAGE_SOLVABLE;
786 keys[i].storage = KEY_STORAGE_INCORE;
794 data.nkeys = numkeys;
796 /******* Part 4: Schemata ********************************************/
799 schemadata = xcalloc(id, sizeof(Id));
800 schemadatap = schemadata;
801 schemadataend = schemadata + id;
802 schemata = xcalloc(numschemata, sizeof(Id));
803 for (i = 0; i < numschemata; i++)
805 schemata[i] = schemadatap - schemadata;
806 schemadatap = read_idarray(fp, numid, 0, schemadatap, schemadataend, 0);
808 data.schemata = schemata;
809 data.nschemata = numschemata;
810 data.schemadata = schemadata;
812 /******* Part 5: Info ***********************************************/
813 for (i = 0; i < numinfo; i++)
815 /* for now we're just interested in data that starts with
816 * the repodata_external id
818 Id *keyp = schemadata + schemata[read_id(fp, numschemata)];
820 if (keys[key].name == REPODATA_EXTERNAL && keys[key].type == TYPE_VOID)
822 /* external data for some ids */
823 parse_repodata(fp, keyp, keys, idmap, numid, numrel, repo);
826 skip_schema(fp, keyp, keys, numid, numrel);
829 /******* Part 6: packed sizes (optional) ****************************/
831 if ((solvflags & SOLV_FLAG_PACKEDSIZES) != 0)
833 exists = xmalloc (numsolv);
834 for (i = 0; i < numsolv; i++)
835 exists[i] = read_id(fp, 0) != 0;
838 /******* Part 7: item data *******************************************/
840 /* calculate idarray size */
842 for (i = 1; i < numkeys; i++)
845 if ((keys[i].type == TYPE_IDARRAY || keys[i].type == TYPE_REL_IDARRAY)
846 && id >= INTERESTED_START && id <= INTERESTED_END)
847 size_idarray += keys[i].size;
850 /* allocate needed space in repo */
853 repo_reserve_ids(repo, 0, size_idarray);
854 idarraydatap = repo->idarraydata + repo->idarraysize;
855 repo->idarraysize += size_idarray;
856 idarraydataend = repo->idarraydata + repo->idarraysize;
866 s = pool_id2solvable(pool, repo_add_solvable_block(repo, numsolv));
868 if ((solvflags & SOLV_FLAG_VERTICAL) != 0)
870 Id *solvschema = xcalloc(numsolv, sizeof(Id));
871 unsigned char *used = xmalloc(numschemata);
872 Solvable *sstart = s;
875 /* XXX BROKEN CODE */
877 for (i = 0; i < numsolv; i++)
878 solvschema[i] = read_id(fp, numschemata);
879 for (key = 1; key < numkeys; key++)
882 type = keys[key].type;
883 memset(used, 0, numschemata);
884 for (i = 0; i < numschemata; i++)
886 Id *keyp = schemadata + schemata[i];
894 for (i = 0, s = sstart; i < numsolv; i++, s++)
896 if (!used[solvschema[i]])
901 did = idmap[read_id(fp, numid + numrel)];
902 if (id == SOLVABLE_NAME)
904 else if (id == SOLVABLE_ARCH)
906 else if (id == SOLVABLE_EVR)
908 else if (id == SOLVABLE_VENDOR)
911 incore_add_id(&data, did);
915 if (id == RPM_RPMDBID)
918 repo->rpmdbid = (Id *)xcalloc(numsolv, sizeof(Id));
919 repo->rpmdbid[i] = h;
923 incore_add_u32(&data, h);
927 while(read_u8(fp) != 0)
931 case TYPE_REL_IDARRAY:
932 if (id < INTERESTED_START || id > INTERESTED_END)
934 /* not interested in array */
935 while ((read_u8(fp) & 0xc0) != 0)
939 ido = idarraydatap - repo->idarraydata;
940 idarraydatap = read_idarray(fp, numid + numrel, idmap, idarraydatap, idarraydataend, type == TYPE_REL_IDARRAY);
941 if (id == SOLVABLE_PROVIDES)
943 else if (id == SOLVABLE_OBSOLETES)
945 else if (id == SOLVABLE_CONFLICTS)
947 else if (id == SOLVABLE_REQUIRES)
949 else if (id == SOLVABLE_RECOMMENDS)
951 else if (id == SOLVABLE_SUPPLEMENTS)
952 s->supplements = ido;
953 else if (id == SOLVABLE_SUGGESTS)
955 else if (id == SOLVABLE_ENHANCES)
957 else if (id == SOLVABLE_FRESHENS)
964 case TYPE_ATTR_CHUNK:
965 case TYPE_ATTR_STRING:
966 case TYPE_ATTR_INTLIST:
967 case TYPE_ATTR_LOCALIDS:
969 embedded_store = new_store (pool);
970 add_attr_from_file (embedded_store, i, id, type, idmap, numid, fp);
988 for (i = 0; i < numsolv; i++, s++)
990 if (exists && !exists[i])
992 Id *keyp = schemadata + schemata[read_id(fp, numschemata)];
993 while ((key = *keyp++) != 0)
996 switch (keys[key].type)
999 did = idmap[read_id(fp, numid + numrel)];
1000 if (id == SOLVABLE_NAME)
1002 else if (id == SOLVABLE_ARCH)
1004 else if (id == SOLVABLE_EVR)
1006 else if (id == SOLVABLE_VENDOR)
1008 else if (keys[key].storage == KEY_STORAGE_INCORE)
1009 incore_add_id(&data, did);
1011 POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %s\n", id2str(pool, id), id2str(pool, did));
1017 POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %u\n", id2str(pool, id), h);
1019 if (id == RPM_RPMDBID)
1022 repo->rpmdbid = (Id *)xcalloc(numsolv, sizeof(Id));
1023 repo->rpmdbid[i] = h;
1025 else if (keys[key].storage == KEY_STORAGE_INCORE)
1026 incore_add_u32(&data, h);
1029 if (keys[key].storage == KEY_STORAGE_INCORE)
1031 while ((h = read_u8(fp)) != 0)
1032 incore_add_u8(&data, h);
1033 incore_add_u8(&data, 0);
1037 while (read_u8(fp) != 0)
1042 case TYPE_REL_IDARRAY:
1043 if (id < INTERESTED_START || id > INTERESTED_END)
1045 if (keys[key].storage == KEY_STORAGE_INCORE)
1047 Id old = 0, rel = keys[key].type == TYPE_REL_IDARRAY ? SOLVABLE_PREREQMARKER : 0;
1050 did = read_id(fp, 0);
1052 did = (did & 0x3f) | ((did >> 1) & ~0x3f);
1066 if (did >= numid + numrel)
1069 did = ((did & ~0x3f) << 1) | h;
1070 incore_add_id(&data, did);
1076 while ((read_u8(fp) & 0xc0) != 0)
1081 ido = idarraydatap - repo->idarraydata;
1082 idarraydatap = read_idarray(fp, numid + numrel, idmap, idarraydatap, idarraydataend, keys[key].type == TYPE_REL_IDARRAY);
1083 if (id == SOLVABLE_PROVIDES)
1085 else if (id == SOLVABLE_OBSOLETES)
1087 else if (id == SOLVABLE_CONFLICTS)
1089 else if (id == SOLVABLE_REQUIRES)
1091 else if (id == SOLVABLE_RECOMMENDS)
1093 else if (id == SOLVABLE_SUPPLEMENTS)
1094 s->supplements = ido;
1095 else if (id == SOLVABLE_SUGGESTS)
1097 else if (id == SOLVABLE_ENHANCES)
1099 else if (id == SOLVABLE_FRESHENS)
1102 POOL_DEBUG(SAT_DEBUG_STATS, "%s ->\n", id2str(pool, id));
1103 for (; repo->idarraydata[ido]; ido++)
1104 POOL_DEBUG(SAT_DEBUG_STATS," %s\n", dep2str(pool, repo->idarraydata[ido]));
1111 case TYPE_ATTR_CHUNK:
1112 case TYPE_ATTR_STRING:
1113 case TYPE_ATTR_INTLIST:
1114 case TYPE_ATTR_LOCALIDS:
1115 if (!embedded_store)
1116 embedded_store = new_store (pool);
1117 add_attr_from_file (embedded_store, i, id, keys[key].type, idmap, numid, fp);
1121 skip_item(fp, keys[key].type, numid, numrel);
1126 if (data.incoredatafree)
1128 /* shrink excess size */
1129 data.incoredata = xrealloc(data.incoredata, data.incoredatalen);
1130 data.incoredatafree = 0;
1133 for (i = 1; i < numkeys; i++)
1134 if (keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
1138 /* we have vertical data, make it available */
1140 data.verticaloffset = ftell(fp);
1143 if (data.incoredatalen || data.fp)
1145 /* we got some data, make it available */
1146 repo->repodata = xrealloc(repo->repodata, (repo->nrepodata + 1) * sizeof(data));
1147 repo->repodata[repo->nrepodata++] = data;
1160 attr_store_pack (embedded_store);
1161 /* If we have any attributes we also have pages. */
1162 read_or_setup_pages (fp, embedded_store);
1163 /* The NULL name here means embedded attributes. */
1164 repo_add_attrstore (repo, embedded_store, NULL);