2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Read the binary dump of a Repo and create a Repo * from it
14 * Repo *pool_addrepo_solv(Pool *pool, FILE *fp)
26 #include "repo_solv.h"
28 #include "attr_store_p.h"
30 #define INTERESTED_START SOLVABLE_NAME
31 #define INTERESTED_END SOLVABLE_FRESHENS
33 static Pool *mypool; /* for pool_debug... */
35 /*-----------------------------------------------------------------*/
36 /* .solv read functions */
48 for (i = 0; i < 4; i++)
53 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
73 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
85 read_id(FILE *fp, Id max)
90 for (i = 0; i < 5; i++)
95 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
103 pool_debug(mypool, SAT_FATAL, "read_id: id too large (%u/%u)\n", x, max);
108 x = (x << 7) ^ c ^ 128;
110 pool_debug(mypool, SAT_FATAL, "read_id: id too long\n");
120 read_idarray(FILE *fp, Id max, Id *map, Id *store, Id *end, int relative)
130 pool_debug(mypool, SAT_FATAL, "unexpected EOF\n");
135 x = (x << 6) | (c & 63);
138 if (x == 0 && c == 0x40)
143 pool_debug(mypool, SAT_FATAL, "read_idarray: array overflow\n");
146 *store++ = SOLVABLE_PREREQMARKER;
156 pool_debug(mypool, SAT_FATAL, "read_idarray: id too large (%u/%u)\n", x, max);
163 pool_debug(mypool, SAT_FATAL, "read_idarray: array overflow\n");
169 if (x == 0) /* already have trailing zero? */
173 pool_debug(mypool, SAT_FATAL, "read_idarray: array overflow\n");
182 x = (x << 7) ^ c ^ 128;
187 read_str (FILE *fp, char **inbuf, unsigned *len)
189 unsigned char *buf = (unsigned char*)*inbuf;
192 buf = xmalloc (1024);
197 while((c = getc (fp)) != 0)
201 pool_debug (mypool, SAT_FATAL, "unexpected EOF\n");
204 /* Plus 1 as we also want to add the 0. */
208 /* Don't realloc on the inbuf, it might be on the stack. */
209 if (buf == (unsigned char*)*inbuf)
211 buf = xmalloc (*len);
212 memcpy (buf, *inbuf, *len - 256);
215 buf = xrealloc (buf, *len);
224 skip_item (FILE *fp, unsigned type, unsigned numid, unsigned numrel)
231 read_id(fp, numid + numrel); /* just check Id */
236 case TYPE_ATTR_STRING:
238 while(read_u8(fp) != 0)
242 case TYPE_IDVALUEARRAY:
243 case TYPE_IDVALUEVALUEARRAY:
244 case TYPE_REL_IDARRAY:
245 while ((read_u8(fp) & 0xc0) != 0)
248 case TYPE_COUNT_NAMED:
250 unsigned count = read_id (fp, 0);
253 read_id (fp, numid); /* Name */
254 unsigned t = read_id (fp, TYPE_ATTR_TYPE_MAX + 1);
255 skip_item (fp, t, numid, numrel);
261 unsigned count = read_id (fp, 0);
262 unsigned t = read_id (fp, TYPE_ATTR_TYPE_MAX + 1);
264 skip_item (fp, t, numid, numrel);
267 case TYPE_ATTR_CHUNK:
273 case TYPE_ATTR_INTLIST:
274 case TYPE_ATTR_LOCALIDS:
275 while (read_id(fp, 0) != 0)
279 pool_debug(mypool, SAT_FATAL, "unknown type %d\n", type);
285 key_cmp (const void *pa, const void *pb)
287 struct key { Id name; unsigned type; };
288 struct key *a = (struct key*)pa;
289 struct key *b = (struct key*)pb;
290 return a->name - b->name;
300 parse_repodata (FILE *fp, Id *keyp, struct key *keys, Id *idmap, unsigned numid, unsigned numrel, Repo *repo)
307 repo->repodata = xrealloc(repo->repodata, (repo->nrepodata + 1) * sizeof (*data));
308 data = repo->repodata + repo->nrepodata++;
309 memset(data, 0, sizeof(*data));
311 while ((key = *keyp++) != 0)
314 switch (keys[key].type)
316 case TYPE_IDVALUEARRAY:
317 if (id != REPODATA_KEYS)
319 skip_item(fp, TYPE_IDVALUEARRAY, numid, numrel);
322 ida = xcalloc(keys[key].size, sizeof(Id));
323 ide = read_idarray(fp, 0, 0, ida, ida + keys[key].size, 0);
327 pool_debug (mypool, SAT_FATAL, "invalid attribute data\n");
330 data->nkeys = n >> 1;
331 data->keys = xmalloc2(data->nkeys, sizeof(data->keys[0]));
332 for (i = 0, ide = ida; i < data->nkeys; i++)
336 pool_debug (mypool, SAT_FATAL, "invalid attribute data\n");
339 data->keys[i].name = idmap[*ide++];
340 data->keys[i].type = *ide++;
343 qsort(data->keys, data->nkeys, sizeof(data->keys[0]), key_cmp);
346 if (id != REPODATA_LOCATION)
347 skip_item(fp, TYPE_STR, numid, numrel);
351 unsigned len = sizeof (buf);
352 char *filename = buf;
353 read_str(fp, &filename, &len);
354 data->location = strdup(filename);
360 skip_item(fp, keys[key].type, numid, numrel);
366 /*-----------------------------------------------------------------*/
370 skip_schema(FILE *fp, Id *keyp, struct key *keys, unsigned int numid, unsigned int numrel)
373 while ((key = *keyp++) != 0)
374 skip_item(fp, keys[key].type, numid, numrel);
377 // ----------------------------------------------
380 * read repo from .solv file
385 repo_add_solv(Repo *repo, FILE *fp)
387 Pool *pool = repo->pool;
389 unsigned int numid, numrel, numsolv;
390 unsigned int numkeys, numschemata, numinfo;
391 Attrstore *embedded_store = 0;
394 Offset *str; /* map Id -> Offset into string space */
395 char *strsp; /* repo string space */
396 char *sp; /* pointer into string space */
397 Id *idmap; /* map of repo Ids to pool Ids */
399 unsigned int hashmask, h;
405 unsigned int size_idarray;
406 Id *idarraydatap, *idarraydataend;
409 unsigned int solvflags;
410 unsigned int solvversion;
412 Id *schemadata, *schemadatap, *schemadataend;
417 if (read_u32(fp) != ('S' << 24 | 'O' << 16 | 'L' << 8 | 'V'))
419 pool_debug(pool, SAT_FATAL, "not a SOLV file\n");
422 solvversion = read_u32(fp);
430 pool_debug(pool, SAT_FATAL, "unsupported SOLV version\n");
434 pool_freeidhashes(pool);
436 numid = read_u32(fp);
437 numrel = read_u32(fp);
438 numsolv = read_u32(fp);
439 numkeys = read_u32(fp);
440 numschemata = read_u32(fp);
441 numinfo = read_u32(fp);
442 solvflags = read_u32(fp);
444 if (solvversion < SOLV_VERSION_3
447 pool_debug(pool, SAT_FATAL, "unsupported SOLV format (has info)\n");
451 /******* Part 1: string IDs *****************************************/
453 sizeid = read_u32(fp); /* size of string+Id space */
456 * read strings and Ids
465 /* alloc string buffer */
466 strsp = (char *)xrealloc(pool->ss.stringspace, pool->ss.sstrings + sizeid + 1);
467 /* alloc string offsets (Id -> Offset into string space) */
468 str = (Offset *)xrealloc(pool->ss.strings, (pool->ss.nstrings + numid) * sizeof(Offset));
470 pool->ss.stringspace = strsp;
471 pool->ss.strings = str; /* array of offsets into strsp, indexed by Id */
473 /* point to _BEHIND_ already allocated string/Id space */
474 strsp += pool->ss.sstrings;
476 /* alloc id map for name and rel Ids. this maps ids in the solv files
477 * to the ids in our pool */
478 idmap = (Id *)xcalloc(numid + numrel, sizeof(Id));
481 * read new repo at end of pool
484 if ((solvflags & SOLV_FLAG_PREFIX_POOL) == 0)
486 if (fread(strsp, sizeid, 1, fp) != 1)
488 pool_debug(pool, SAT_FATAL, "read error while reading strings\n");
494 unsigned int pfsize = read_u32 (fp);
495 char *prefix = xmalloc (pfsize);
499 if (fread (prefix, pfsize, 1, fp) != 1)
501 pool_debug(pool, SAT_FATAL, "read error while reading strings\n");
504 for (i = 1; i < numid; i++)
506 int same = (unsigned char)*pp++;
507 size_t len = strlen (pp) + 1;
509 memcpy (dest, old_str, same);
510 memcpy (dest + same, pp, len);
517 strsp[sizeid] = 0; /* make string space \0 terminated */
521 * build hashes for all read strings
525 hashmask = mkmask(pool->ss.nstrings + numid);
528 POOL_DEBUG(SAT_DEBUG_STATS, "read %d strings\n", numid);
529 POOL_DEBUG(SAT_DEBUG_STATS, "string hash buckets: %d\n", hashmask + 1);
533 * ensure sufficient hash size
536 hashtbl = (Id *)xcalloc(hashmask + 1, sizeof(Id));
539 * fill hashtable with strings already in pool
542 for (i = 1; i < pool->ss.nstrings; i++) /* leave out our dummy zero id */
544 h = strhash(pool->ss.stringspace + pool->ss.strings[i]) & hashmask;
545 hh = HASHCHAIN_START;
547 h = HASHCHAIN_NEXT(h, hh, hashmask);
552 * run over string space, calculate offsets
554 * build id map (maps solv Id -> pool Id)
557 for (i = 1; i < numid; i++)
559 if (sp >= strsp + sizeid)
561 pool_debug(pool, SAT_FATAL, "not enough strings\n");
564 if (!*sp) /* empty string */
572 h = strhash(sp) & hashmask;
573 hh = HASHCHAIN_START;
579 if (!strcmp(pool->ss.stringspace + pool->ss.strings[id], sp))
580 break; /* existing string */
581 h = HASHCHAIN_NEXT(h, hh, hashmask);
584 /* length == offset to next string */
586 if (id == ID_NULL) /* end of hash chain -> new string */
588 id = pool->ss.nstrings++;
590 str[id] = pool->ss.sstrings; /* save Offset */
591 if (sp != pool->ss.stringspace + pool->ss.sstrings) /* not at end-of-buffer */
592 memmove(pool->ss.stringspace + pool->ss.sstrings, sp, l); /* append to pool buffer */
593 pool->ss.sstrings += l;
595 idmap[i] = id; /* repo relative -> pool relative */
596 sp += l; /* next string */
599 pool_shrink_strings(pool); /* vacuum */
602 /******* Part 2: Relation IDs ***************************************/
612 ran = (Reldep *)xrealloc(pool->rels, (pool->nrels + numrel) * sizeof(Reldep));
615 pool_debug(pool, SAT_FATAL, "no mem for rel space\n");
618 pool->rels = ran; /* extended rel space */
620 hashmask = mkmask(pool->nrels + numrel);
622 POOL_DEBUG(SAT_DEBUG_STATS, "read %d rels\n", numrel);
623 POOL_DEBUG(SAT_DEBUG_STATS, "rel hash buckets: %d\n", hashmask + 1);
626 * prep hash table with already existing RelDeps
629 hashtbl = (Id *)xcalloc(hashmask + 1, sizeof(Id));
630 for (i = 1; i < pool->nrels; i++)
632 h = relhash(ran[i].name, ran[i].evr, ran[i].flags) & hashmask;
633 hh = HASHCHAIN_START;
635 h = HASHCHAIN_NEXT(h, hh, hashmask);
640 * read RelDeps from repo
643 for (i = 0; i < numrel; i++)
645 name = read_id(fp, i + numid); /* read (repo relative) Ids */
646 evr = read_id(fp, i + numid);
648 name = idmap[name]; /* map to (pool relative) Ids */
650 h = relhash(name, evr, flags) & hashmask;
651 hh = HASHCHAIN_START;
655 if (id == ID_NULL) /* end of hash chain */
657 if (ran[id].name == name && ran[id].evr == evr && ran[id].flags == flags)
659 h = HASHCHAIN_NEXT(h, hh, hashmask);
661 if (id == ID_NULL) /* new RelDep */
667 ran[id].flags = flags;
669 idmap[i + numid] = MAKERELDEP(id); /* fill Id map */
672 pool_shrink_rels(pool); /* vacuum */
676 /******* Part 3: Keys ***********************************************/
678 keys = xcalloc(numkeys, sizeof(*keys));
679 /* keys start at 1 */
680 for (i = 1; i < numkeys; i++)
682 keys[i].name = idmap[read_id(fp, numid)];
683 keys[i].type = read_id(fp, 0);
684 keys[i].size = read_id(fp, 0);
687 /******* Part 4: Schemata ********************************************/
690 schemadata = xcalloc(id, sizeof(Id));
691 schemadatap = schemadata;
692 schemadataend = schemadata + id;
693 schemata = xcalloc(numschemata, sizeof(Id));
694 for (i = 0; i < numschemata; i++)
696 schemata[i] = schemadatap - schemadata;
697 schemadatap = read_idarray(fp, numid, 0, schemadatap, schemadataend, 0);
700 /******* Part 5: Info ***********************************************/
701 for (i = 0; i < numinfo; i++)
703 /* for now we're just interested in data that starts with
704 * the repodata_external id
706 Id *keyp = schemadata + schemata[read_id(fp, numschemata)];
708 if (keys[key].name == REPODATA_EXTERNAL && keys[key].type == TYPE_VOID)
710 /* external data for some ids */
711 parse_repodata(fp, keyp, keys, idmap, numid, numrel, repo);
714 skip_schema(fp, keyp, keys, numid, numrel);
717 /******* Part 6: packed sizes (optional) ****************************/
719 if ((solvflags & SOLV_FLAG_PACKEDSIZES) != 0)
721 exists = xmalloc (numsolv);
722 for (i = 0; i < numsolv; i++)
723 exists[i] = read_id(fp, 0) != 0;
726 /******* Part 7: item data *******************************************/
728 /* calculate idarray size */
730 for (i = 1; i < numkeys; i++)
733 if ((keys[i].type == TYPE_IDARRAY || keys[i].type == TYPE_REL_IDARRAY)
734 && id >= INTERESTED_START && id <= INTERESTED_END)
735 size_idarray += keys[i].size;
738 /* allocate needed space in repo */
741 repo_reserve_ids(repo, 0, size_idarray);
742 idarraydatap = repo->idarraydata + repo->idarraysize;
743 repo->idarraysize += size_idarray;
744 idarraydataend = repo->idarraydata + repo->idarraysize;
754 s = pool_id2solvable(pool, repo_add_solvable_block(repo, numsolv));
756 if ((solvflags & SOLV_FLAG_VERTICAL) != 0)
758 Id *solvschema = xcalloc(numsolv, sizeof(Id));
759 unsigned char *used = xmalloc(numschemata);
760 Solvable *sstart = s;
763 for (i = 0; i < numsolv; i++)
764 solvschema[i] = read_id(fp, numschemata);
765 for (key = 1; key < numkeys; key++)
768 type = keys[key].type;
769 memset(used, 0, numschemata);
770 for (i = 0; i < numschemata; i++)
772 Id *keyp = schemadata + schemata[i];
780 for (i = 0, s = sstart; i < numsolv; i++, s++)
782 if (!used[solvschema[i]])
787 did = idmap[read_id(fp, numid + numrel)];
788 if (id == SOLVABLE_NAME)
790 else if (id == SOLVABLE_ARCH)
792 else if (id == SOLVABLE_EVR)
794 else if (id == SOLVABLE_VENDOR)
799 if (id == RPM_RPMDBID)
802 repo->rpmdbid = (Id *)xcalloc(numsolv, sizeof(Id));
803 repo->rpmdbid[i] = h;
807 while(read_u8(fp) != 0)
811 case TYPE_REL_IDARRAY:
812 if (id < INTERESTED_START || id > INTERESTED_END)
814 /* not interested in array */
815 while ((read_u8(fp) & 0xc0) != 0)
819 ido = idarraydatap - repo->idarraydata;
820 idarraydatap = read_idarray(fp, numid + numrel, idmap, idarraydatap, idarraydataend, type == TYPE_REL_IDARRAY);
821 if (id == SOLVABLE_PROVIDES)
823 else if (id == SOLVABLE_OBSOLETES)
825 else if (id == SOLVABLE_CONFLICTS)
827 else if (id == SOLVABLE_REQUIRES)
829 else if (id == SOLVABLE_RECOMMENDS)
831 else if (id == SOLVABLE_SUPPLEMENTS)
832 s->supplements = ido;
833 else if (id == SOLVABLE_SUGGESTS)
835 else if (id == SOLVABLE_ENHANCES)
837 else if (id == SOLVABLE_FRESHENS)
841 case TYPE_ATTR_CHUNK:
842 case TYPE_ATTR_STRING:
843 case TYPE_ATTR_INTLIST:
844 case TYPE_ATTR_LOCALIDS:
846 embedded_store = new_store (pool);
847 add_attr_from_file (embedded_store, i, id, type, idmap, numid, fp);
861 for (i = 0; i < numsolv; i++, s++)
863 if (exists && !exists[i])
865 Id *keyp = schemadata + schemata[read_id(fp, numschemata)];
866 while ((key = *keyp++) != 0)
869 switch (keys[key].type)
872 did = idmap[read_id(fp, numid + numrel)];
873 if (id == SOLVABLE_NAME)
875 else if (id == SOLVABLE_ARCH)
877 else if (id == SOLVABLE_EVR)
879 else if (id == SOLVABLE_VENDOR)
882 POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %s\n", id2str(pool, id), id2str(pool, did));
888 POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %u\n", id2str(pool, id), h);
890 if (id == RPM_RPMDBID)
893 repo->rpmdbid = (Id *)xcalloc(numsolv, sizeof(Id));
894 repo->rpmdbid[i] = h;
898 while(read_u8(fp) != 0)
902 case TYPE_REL_IDARRAY:
903 if (id < INTERESTED_START || id > INTERESTED_END)
905 /* not interested in array */
906 while ((read_u8(fp) & 0xc0) != 0)
910 ido = idarraydatap - repo->idarraydata;
911 idarraydatap = read_idarray(fp, numid + numrel, idmap, idarraydatap, idarraydataend, keys[key].type == TYPE_REL_IDARRAY);
912 if (id == SOLVABLE_PROVIDES)
914 else if (id == SOLVABLE_OBSOLETES)
916 else if (id == SOLVABLE_CONFLICTS)
918 else if (id == SOLVABLE_REQUIRES)
920 else if (id == SOLVABLE_RECOMMENDS)
922 else if (id == SOLVABLE_SUPPLEMENTS)
923 s->supplements = ido;
924 else if (id == SOLVABLE_SUGGESTS)
926 else if (id == SOLVABLE_ENHANCES)
928 else if (id == SOLVABLE_FRESHENS)
931 POOL_DEBUG(SAT_DEBUG_STATS, "%s ->\n", id2str(pool, id));
932 for (; repo->idarraydata[ido]; ido++)
933 POOL_DEBUG(SAT_DEBUG_STATS," %s\n", dep2str(pool, repo->idarraydata[ido]));
937 case TYPE_ATTR_CHUNK:
938 case TYPE_ATTR_STRING:
939 case TYPE_ATTR_INTLIST:
940 case TYPE_ATTR_LOCALIDS:
942 embedded_store = new_store (pool);
943 add_attr_from_file (embedded_store, i, id, keys[key].type, idmap, numid, fp);
946 skip_item(fp, keys[key].type, numid, numrel);
953 attr_store_pack (embedded_store);
954 /* If we have any attributes we also have pages. */
955 read_or_setup_pages (fp, embedded_store);
956 /* The NULL name here means embedded attributes. */
957 repo_add_attrstore (repo, embedded_store, NULL);