2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Read the binary dump of a Repo and create a Repo * from it
14 * Repo *pool_addrepo_solv(Pool *pool, FILE *fp)
26 #include "repo_solv.h"
28 #include "attr_store_p.h"
30 #define INTERESTED_START SOLVABLE_NAME
31 #define INTERESTED_END SOLVABLE_FRESHENS
33 static Pool *mypool; /* for pool_debug... */
35 /*-----------------------------------------------------------------*/
36 /* .solv read functions */
48 for (i = 0; i < 4; i++)
53 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
73 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
85 read_id(FILE *fp, Id max)
90 for (i = 0; i < 5; i++)
95 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
103 pool_debug(mypool, SAT_FATAL, "read_id: id too large (%u/%u)\n", x, max);
108 x = (x << 7) ^ c ^ 128;
110 pool_debug(mypool, SAT_FATAL, "read_id: id too long\n");
120 read_idarray(FILE *fp, Id max, Id *map, Id *store, Id *end, int relative)
130 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
135 x = (x << 6) | (c & 63);
138 if (x == 0 && c == 0x40)
143 pool_debug(mypool, SAT_FATAL, "read_idarray: array overflow\n");
146 *store++ = SOLVABLE_PREREQMARKER;
156 pool_debug(mypool, SAT_FATAL, "read_idarray: id too large (%u/%u)\n", x, max);
163 pool_debug(mypool, SAT_FATAL, "read_idarray: array overflow\n");
169 if (x == 0) /* already have trailing zero? */
173 pool_debug(mypool, SAT_FATAL, "read_idarray: array overflow\n");
182 x = (x << 7) ^ c ^ 128;
187 read_str(FILE *fp, char **inbuf, unsigned *len)
189 unsigned char *buf = (unsigned char*)*inbuf;
192 buf = xmalloc (1024);
197 while((c = getc (fp)) != 0)
201 pool_debug (mypool, SAT_FATAL, "unexpected EOF\n");
204 /* Plus 1 as we also want to add the 0. */
208 /* Don't realloc on the inbuf, it might be on the stack. */
209 if (buf == (unsigned char*)*inbuf)
211 buf = xmalloc (*len);
212 memcpy (buf, *inbuf, *len - 256);
215 buf = xrealloc (buf, *len);
224 skip_item (FILE *fp, unsigned type, unsigned numid, unsigned numrel)
231 read_id(fp, numid + numrel); /* just check Id */
236 case TYPE_ATTR_STRING:
238 while(read_u8(fp) != 0)
242 case TYPE_IDVALUEARRAY:
243 case TYPE_IDVALUEVALUEARRAY:
244 case TYPE_REL_IDARRAY:
245 while ((read_u8(fp) & 0xc0) != 0)
248 case TYPE_COUNT_NAMED:
250 unsigned count = read_id (fp, 0);
253 read_id (fp, numid); /* Name */
254 unsigned t = read_id (fp, TYPE_ATTR_TYPE_MAX + 1);
255 skip_item (fp, t, numid, numrel);
261 unsigned count = read_id (fp, 0);
262 unsigned t = read_id (fp, TYPE_ATTR_TYPE_MAX + 1);
264 skip_item (fp, t, numid, numrel);
267 case TYPE_ATTR_CHUNK:
273 case TYPE_ATTR_INTLIST:
274 case TYPE_ATTR_LOCALIDS:
275 while (read_id(fp, 0) != 0)
279 pool_debug(mypool, SAT_FATAL, "unknown type %d\n", type);
285 key_cmp (const void *pa, const void *pb)
287 Repokey *a = (Repokey *)pa;
288 Repokey *b = (Repokey *)pb;
289 return a->name - b->name;
293 parse_repodata (FILE *fp, Id *keyp, Repokey *keys, Id *idmap, unsigned numid, unsigned numrel, Repo *repo)
300 repo->repodata = xrealloc(repo->repodata, (repo->nrepodata + 1) * sizeof (*data));
301 data = repo->repodata + repo->nrepodata++;
303 memset(data, 0, sizeof(*data));
305 while ((key = *keyp++) != 0)
308 switch (keys[key].type)
310 case TYPE_IDVALUEARRAY:
311 if (id != REPODATA_KEYS)
313 skip_item(fp, TYPE_IDVALUEARRAY, numid, numrel);
316 ida = xcalloc(keys[key].size, sizeof(Id));
317 ide = read_idarray(fp, 0, 0, ida, ida + keys[key].size, 0);
321 pool_debug (mypool, SAT_FATAL, "invalid attribute data\n");
324 data->nkeys = 1 + (n >> 1);
325 data->keys = xmalloc2(data->nkeys, sizeof(data->keys[0]));
326 memset(data->keys, 0, sizeof(Repokey));
327 for (i = 1, ide = ida; i < data->nkeys; i++)
331 pool_debug (mypool, SAT_FATAL, "invalid attribute data\n");
334 data->keys[i].name = idmap ? idmap[*ide++] : *ide++;
335 data->keys[i].type = *ide++;
336 data->keys[i].size = 0;
337 data->keys[i].storage = 0;
341 qsort(data->keys + 1, data->nkeys - 1, sizeof(data->keys[0]), key_cmp);
344 if (id != REPODATA_LOCATION)
345 skip_item(fp, TYPE_STR, numid, numrel);
349 unsigned len = sizeof (buf);
350 char *filename = buf;
351 read_str(fp, &filename, &len);
352 data->location = strdup(filename);
358 skip_item(fp, keys[key].type, numid, numrel);
364 /*-----------------------------------------------------------------*/
368 skip_schema(FILE *fp, Id *keyp, Repokey *keys, unsigned int numid, unsigned int numrel)
371 while ((key = *keyp++) != 0)
372 skip_item(fp, keys[key].type, numid, numrel);
375 /*-----------------------------------------------------------------*/
378 incore_add_id(Repodata *data, Id x)
381 /* make sure we have at least 5 bytes free */
382 if (data->incoredatafree < 5)
384 data->incoredata = xrealloc(data->incoredata, data->incoredatalen + 1024);
385 data->incoredatafree = 1024;
387 dp = data->incoredata + data->incoredatalen;
393 *dp++ = (x >> 28) | 128;
395 *dp++ = (x >> 21) | 128;
396 *dp++ = (x >> 14) | 128;
399 *dp++ = (x >> 7) | 128;
401 data->incoredatafree -= dp - (data->incoredata + data->incoredatalen);
402 data->incoredatalen = dp - data->incoredata;
406 incore_add_u32(Repodata *data, unsigned int x)
409 /* make sure we have at least 4 bytes free */
410 if (data->incoredatafree < 4)
412 data->incoredata = xrealloc(data->incoredata, data->incoredatalen + 1024);
413 data->incoredatafree = 1024;
415 dp = data->incoredata + data->incoredatalen;
420 data->incoredatafree -= 4;
421 data->incoredatalen += 4;
425 incore_add_u8(Repodata *data, unsigned int x)
428 /* make sure we have at least 1 byte free */
429 if (data->incoredatafree < 1)
431 data->incoredata = xrealloc(data->incoredata, data->incoredatalen + 1024);
432 data->incoredatafree = 1024;
434 dp = data->incoredata + data->incoredatalen;
436 data->incoredatafree--;
437 data->incoredatalen++;
440 // ----------------------------------------------
443 * read repo from .solv file
448 repo_add_solv(Repo *repo, FILE *fp)
450 Pool *pool = repo->pool;
452 unsigned int numid, numrel, numsolv;
453 unsigned int numkeys, numschemata, numinfo;
455 Attrstore *embedded_store = 0;
459 Offset *str; /* map Id -> Offset into string space */
460 char *strsp; /* repo string space */
461 char *sp; /* pointer into string space */
462 Id *idmap; /* map of repo Ids to pool Ids */
464 unsigned int hashmask, h;
470 unsigned int size_idarray;
471 Id *idarraydatap, *idarraydataend;
474 unsigned int solvflags;
475 unsigned int solvversion;
477 Id *schemadata, *schemadatap, *schemadataend;
484 if (read_u32(fp) != ('S' << 24 | 'O' << 16 | 'L' << 8 | 'V'))
486 pool_debug(pool, SAT_FATAL, "not a SOLV file\n");
489 solvversion = read_u32(fp);
497 pool_debug(pool, SAT_FATAL, "unsupported SOLV version\n");
501 pool_freeidhashes(pool);
503 numid = read_u32(fp);
504 numrel = read_u32(fp);
505 numsolv = read_u32(fp);
506 numkeys = read_u32(fp);
507 numschemata = read_u32(fp);
508 numinfo = read_u32(fp);
509 solvflags = read_u32(fp);
511 memset(&data, 0, sizeof(data));
514 if (solvversion < SOLV_VERSION_3
517 pool_debug(pool, SAT_FATAL, "unsupported SOLV format (has info)\n");
521 /******* Part 1: string IDs *****************************************/
523 sizeid = read_u32(fp); /* size of string+Id space */
526 * read strings and Ids
535 /* alloc string buffer */
536 strsp = (char *)xrealloc(pool->ss.stringspace, pool->ss.sstrings + sizeid + 1);
537 /* alloc string offsets (Id -> Offset into string space) */
538 str = (Offset *)xrealloc(pool->ss.strings, (pool->ss.nstrings + numid) * sizeof(Offset));
540 pool->ss.stringspace = strsp;
541 pool->ss.strings = str; /* array of offsets into strsp, indexed by Id */
543 /* point to _BEHIND_ already allocated string/Id space */
544 strsp += pool->ss.sstrings;
546 /* alloc id map for name and rel Ids. this maps ids in the solv files
547 * to the ids in our pool */
548 idmap = (Id *)xcalloc(numid + numrel, sizeof(Id));
551 * read new repo at end of pool
554 if ((solvflags & SOLV_FLAG_PREFIX_POOL) == 0)
556 if (fread(strsp, sizeid, 1, fp) != 1)
558 pool_debug(pool, SAT_FATAL, "read error while reading strings\n");
564 unsigned int pfsize = read_u32 (fp);
565 char *prefix = xmalloc (pfsize);
569 if (fread (prefix, pfsize, 1, fp) != 1)
571 pool_debug(pool, SAT_FATAL, "read error while reading strings\n");
574 for (i = 1; i < numid; i++)
576 int same = (unsigned char)*pp++;
577 size_t len = strlen (pp) + 1;
579 memcpy (dest, old_str, same);
580 memcpy (dest + same, pp, len);
587 strsp[sizeid] = 0; /* make string space \0 terminated */
591 * build hashes for all read strings
595 hashmask = mkmask(pool->ss.nstrings + numid);
598 POOL_DEBUG(SAT_DEBUG_STATS, "read %d strings\n", numid);
599 POOL_DEBUG(SAT_DEBUG_STATS, "string hash buckets: %d\n", hashmask + 1);
603 * ensure sufficient hash size
606 hashtbl = (Id *)xcalloc(hashmask + 1, sizeof(Id));
609 * fill hashtable with strings already in pool
612 for (i = 1; i < pool->ss.nstrings; i++) /* leave out our dummy zero id */
614 h = strhash(pool->ss.stringspace + pool->ss.strings[i]) & hashmask;
615 hh = HASHCHAIN_START;
617 h = HASHCHAIN_NEXT(h, hh, hashmask);
622 * run over string space, calculate offsets
624 * build id map (maps solv Id -> pool Id)
627 for (i = 1; i < numid; i++)
629 if (sp >= strsp + sizeid)
631 pool_debug(pool, SAT_FATAL, "not enough strings\n");
634 if (!*sp) /* empty string */
642 h = strhash(sp) & hashmask;
643 hh = HASHCHAIN_START;
649 if (!strcmp(pool->ss.stringspace + pool->ss.strings[id], sp))
650 break; /* existing string */
651 h = HASHCHAIN_NEXT(h, hh, hashmask);
654 /* length == offset to next string */
656 if (id == ID_NULL) /* end of hash chain -> new string */
658 id = pool->ss.nstrings++;
660 str[id] = pool->ss.sstrings; /* save Offset */
661 if (sp != pool->ss.stringspace + pool->ss.sstrings) /* not at end-of-buffer */
662 memmove(pool->ss.stringspace + pool->ss.sstrings, sp, l); /* append to pool buffer */
663 pool->ss.sstrings += l;
665 idmap[i] = id; /* repo relative -> pool relative */
666 sp += l; /* next string */
669 pool_shrink_strings(pool); /* vacuum */
672 /******* Part 2: Relation IDs ***************************************/
682 ran = (Reldep *)xrealloc(pool->rels, (pool->nrels + numrel) * sizeof(Reldep));
685 pool_debug(pool, SAT_FATAL, "no mem for rel space\n");
688 pool->rels = ran; /* extended rel space */
690 hashmask = mkmask(pool->nrels + numrel);
692 POOL_DEBUG(SAT_DEBUG_STATS, "read %d rels\n", numrel);
693 POOL_DEBUG(SAT_DEBUG_STATS, "rel hash buckets: %d\n", hashmask + 1);
696 * prep hash table with already existing RelDeps
699 hashtbl = (Id *)xcalloc(hashmask + 1, sizeof(Id));
700 for (i = 1; i < pool->nrels; i++)
702 h = relhash(ran[i].name, ran[i].evr, ran[i].flags) & hashmask;
703 hh = HASHCHAIN_START;
705 h = HASHCHAIN_NEXT(h, hh, hashmask);
710 * read RelDeps from repo
713 for (i = 0; i < numrel; i++)
715 name = read_id(fp, i + numid); /* read (repo relative) Ids */
716 evr = read_id(fp, i + numid);
718 name = idmap[name]; /* map to (pool relative) Ids */
720 h = relhash(name, evr, flags) & hashmask;
721 hh = HASHCHAIN_START;
725 if (id == ID_NULL) /* end of hash chain */
727 if (ran[id].name == name && ran[id].evr == evr && ran[id].flags == flags)
729 h = HASHCHAIN_NEXT(h, hh, hashmask);
731 if (id == ID_NULL) /* new RelDep */
737 ran[id].flags = flags;
739 idmap[i + numid] = MAKERELDEP(id); /* fill Id map */
742 pool_shrink_rels(pool); /* vacuum */
746 /******* Part 3: Keys ***********************************************/
748 keys = xcalloc(numkeys, sizeof(*keys));
749 /* keys start at 1 */
750 for (i = 1; i < numkeys; i++)
752 id = read_id(fp, numid);
756 keys[i].type = read_id(fp, 0);
757 keys[i].size = read_id(fp, 0);
758 keys[i].storage = KEY_STORAGE_DROPPED;
759 switch (keys[i].type)
767 case SOLVABLE_VENDOR:
768 keys[i].storage = KEY_STORAGE_SOLVABLE;
771 keys[i].storage = KEY_STORAGE_INCORE;
776 case TYPE_REL_IDARRAY:
777 if (id >= INTERESTED_START && id <= INTERESTED_END)
778 keys[i].storage = KEY_STORAGE_SOLVABLE;
780 keys[i].storage = KEY_STORAGE_INCORE;
783 keys[i].storage = KEY_STORAGE_INCORE;
786 if (id == RPM_RPMDBID)
787 keys[i].storage = KEY_STORAGE_SOLVABLE;
789 keys[i].storage = KEY_STORAGE_INCORE;
797 data.nkeys = numkeys;
799 /******* Part 4: Schemata ********************************************/
802 schemadata = xcalloc(id, sizeof(Id));
803 schemadatap = schemadata;
804 schemadataend = schemadata + id;
805 schemata = xcalloc(numschemata, sizeof(Id));
806 for (i = 0; i < numschemata; i++)
808 schemata[i] = schemadatap - schemadata;
809 schemadatap = read_idarray(fp, numid, 0, schemadatap, schemadataend, 0);
811 data.schemata = schemata;
812 data.nschemata = numschemata;
813 data.schemadata = schemadata;
815 /******* Part 5: Info ***********************************************/
816 for (i = 0; i < numinfo; i++)
818 /* for now we're just interested in data that starts with
819 * the repodata_external id
821 Id *keyp = schemadata + schemata[read_id(fp, numschemata)];
823 if (keys[key].name == REPODATA_EXTERNAL && keys[key].type == TYPE_VOID)
825 /* external data for some ids */
826 parse_repodata(fp, keyp, keys, idmap, numid, numrel, repo);
829 skip_schema(fp, keyp, keys, numid, numrel);
832 /******* Part 6: packed sizes (optional) ****************************/
834 if ((solvflags & SOLV_FLAG_PACKEDSIZES) != 0)
836 exists = xmalloc (numsolv);
837 for (i = 0; i < numsolv; i++)
838 exists[i] = read_id(fp, 0) != 0;
841 /******* Part 7: item data *******************************************/
843 /* calculate idarray size */
845 for (i = 1; i < numkeys; i++)
848 if ((keys[i].type == TYPE_IDARRAY || keys[i].type == TYPE_REL_IDARRAY)
849 && id >= INTERESTED_START && id <= INTERESTED_END)
850 size_idarray += keys[i].size;
853 /* allocate needed space in repo */
856 repo_reserve_ids(repo, 0, size_idarray);
857 idarraydatap = repo->idarraydata + repo->idarraysize;
858 repo->idarraysize += size_idarray;
859 idarraydataend = repo->idarraydata + repo->idarraysize;
869 s = pool_id2solvable(pool, repo_add_solvable_block(repo, numsolv));
871 /* store start and end of our id block */
872 data.start = s - pool->solvables;
873 data.end = data.start + numsolv;
875 for (i = 0; i < numsolv; i++, s++)
877 if (exists && !exists[i])
879 Id *keyp = schemadata + schemata[read_id(fp, numschemata)];
880 while ((key = *keyp++) != 0)
883 switch (keys[key].type)
886 did = read_id(fp, numid + numrel);
889 if (id == SOLVABLE_NAME)
891 else if (id == SOLVABLE_ARCH)
893 else if (id == SOLVABLE_EVR)
895 else if (id == SOLVABLE_VENDOR)
897 else if (keys[key].storage == KEY_STORAGE_INCORE)
898 incore_add_id(&data, did);
900 POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %s\n", id2str(pool, id), id2str(pool, did));
906 POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %u\n", id2str(pool, id), h);
908 if (id == RPM_RPMDBID)
911 repo->rpmdbid = (Id *)xcalloc(numsolv, sizeof(Id));
912 repo->rpmdbid[i] = h;
914 else if (keys[key].storage == KEY_STORAGE_INCORE)
915 incore_add_u32(&data, h);
918 if (keys[key].storage == KEY_STORAGE_INCORE)
920 while ((h = read_u8(fp)) != 0)
921 incore_add_u8(&data, h);
922 incore_add_u8(&data, 0);
926 while (read_u8(fp) != 0)
931 case TYPE_REL_IDARRAY:
932 if (id < INTERESTED_START || id > INTERESTED_END)
934 if (keys[key].storage == KEY_STORAGE_INCORE)
938 Id old = 0, rel = keys[key].type == TYPE_REL_IDARRAY ? SOLVABLE_PREREQMARKER : 0;
941 did = read_id(fp, 0);
943 did = (did & 0x3f) | ((did >> 1) & ~0x3f);
957 if (did >= numid + numrel)
960 did = ((did & ~0x3f) << 1) | h;
961 incore_add_id(&data, did);
967 while (((h = read_u8(fp)) & 0xc0) != 0)
968 incore_add_u8(&data, h);
974 while ((read_u8(fp) & 0xc0) != 0)
980 ido = idarraydatap - repo->idarraydata;
981 idarraydatap = read_idarray(fp, numid + numrel, idmap, idarraydatap, idarraydataend, keys[key].type == TYPE_REL_IDARRAY);
982 if (id == SOLVABLE_PROVIDES)
984 else if (id == SOLVABLE_OBSOLETES)
986 else if (id == SOLVABLE_CONFLICTS)
988 else if (id == SOLVABLE_REQUIRES)
990 else if (id == SOLVABLE_RECOMMENDS)
992 else if (id == SOLVABLE_SUPPLEMENTS)
993 s->supplements = ido;
994 else if (id == SOLVABLE_SUGGESTS)
996 else if (id == SOLVABLE_ENHANCES)
998 else if (id == SOLVABLE_FRESHENS)
1001 POOL_DEBUG(SAT_DEBUG_STATS, "%s ->\n", id2str(pool, id));
1002 for (; repo->idarraydata[ido]; ido++)
1003 POOL_DEBUG(SAT_DEBUG_STATS," %s\n", dep2str(pool, repo->idarraydata[ido]));
1010 case TYPE_ATTR_CHUNK:
1011 case TYPE_ATTR_STRING:
1012 case TYPE_ATTR_INTLIST:
1013 case TYPE_ATTR_LOCALIDS:
1014 if (!embedded_store)
1015 embedded_store = new_store (pool);
1016 add_attr_from_file (embedded_store, i, id, keys[key].type, idmap, numid, fp);
1020 skip_item(fp, keys[key].type, numid, numrel);
1025 if (data.incoredatafree)
1027 /* shrink excess size */
1028 data.incoredata = xrealloc(data.incoredata, data.incoredatalen);
1029 data.incoredatafree = 0;
1032 for (i = 1; i < numkeys; i++)
1033 if (keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
1037 /* we have vertical data, make it available */
1039 data.verticaloffset = ftell(fp);
1042 if (data.incoredatalen || data.fp)
1044 /* we got some data, make it available */
1045 repo->repodata = xrealloc(repo->repodata, (repo->nrepodata + 1) * sizeof(data));
1046 repo->repodata[repo->nrepodata++] = data;
1059 attr_store_pack (embedded_store);
1060 /* If we have any attributes we also have pages. */
1061 read_or_setup_pages (fp, embedded_store);
1062 /* The NULL name here means embedded attributes. */
1063 repo_add_attrstore (repo, embedded_store, NULL);