2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Add a repo in solv format
22 #include "repo_solv.h"
28 #include "poolid_private.h" /* WHATPROVIDES_BLOCK */
30 #define INTERESTED_START SOLVABLE_NAME
31 #define INTERESTED_END SOLVABLE_ENHANCES
33 #define SOLV_ERROR_NOT_SOLV 1
34 #define SOLV_ERROR_UNSUPPORTED 2
35 #define SOLV_ERROR_EOF 3
36 #define SOLV_ERROR_ID_RANGE 4
37 #define SOLV_ERROR_OVERFLOW 5
38 #define SOLV_ERROR_CORRUPT 6
42 /*******************************************************************************
43 * functions to extract data from a file handle
51 read_u32(Repodata *data)
58 for (i = 0; i < 4; i++)
63 data->error = pool_error(data->repo->pool, SOLV_ERROR_EOF, "unexpected EOF");
77 read_u8(Repodata *data)
86 data->error = pool_error(data->repo->pool, SOLV_ERROR_EOF, "unexpected EOF");
98 read_id(Repodata *data, Id max)
105 for (i = 0; i < 5; i++)
110 data->error = pool_error(data->repo->pool, SOLV_ERROR_EOF, "unexpected EOF");
116 if (max && x >= (unsigned int)max)
118 data->error = pool_error(data->repo->pool, SOLV_ERROR_ID_RANGE, "read_id: id too large (%u/%u)", x, max);
123 x = (x << 7) ^ c ^ 128;
125 data->error = pool_error(data->repo->pool, SOLV_ERROR_CORRUPT, "read_id: id too long");
131 read_idarray(Repodata *data, Id max, Id *map, Id *store, Id *end)
143 data->error = pool_error(data->repo->pool, SOLV_ERROR_EOF, "unexpected EOF");
148 x = (x << 7) ^ c ^ 128;
151 x = (x << 6) | (c & 63);
152 if (max && x >= (unsigned int)max)
154 data->error = pool_error(data->repo->pool, SOLV_ERROR_ID_RANGE, "read_idarray: id too large (%u/%u)", x, max);
161 data->error = pool_error(data->repo->pool, SOLV_ERROR_OVERFLOW, "read_idarray: array overflow");
167 if (x == 0) /* already have trailing zero? */
171 data->error = pool_error(data->repo->pool, SOLV_ERROR_OVERFLOW, "read_idarray: array overflow");
182 /*******************************************************************************
183 * functions to extract data from memory
190 static inline unsigned char *
191 data_read_id_max(unsigned char *dp, Id *ret, Id *map, int max, Repodata *data)
194 dp = data_read_id(dp, &x);
195 if (x < 0 || (max && x >= max))
197 data->error = pool_error(data->repo->pool, SOLV_ERROR_ID_RANGE, "data_read_id_max: id too large (%u/%u)", x, max);
200 *ret = map ? map[x] : x;
204 static unsigned char *
205 data_read_idarray(unsigned char *dp, Id **storep, Id *map, int max, Repodata *data)
216 x = (x << 7) ^ c ^ 128;
219 x = (x << 6) | (c & 63);
220 if (max && x >= (unsigned int)max)
222 data->error = pool_error(data->repo->pool, SOLV_ERROR_ID_RANGE, "data_read_idarray: id too large (%u/%u)", x, max);
223 data->error = SOLV_ERROR_ID_RANGE;
236 static unsigned char *
237 data_read_rel_idarray(unsigned char *dp, Id **storep, Id *map, int max, Repodata *data, Id marker)
249 x = (x << 7) ^ c ^ 128;
252 x = (x << 6) | (c & 63);
264 if (max && x >= (unsigned int)max)
266 data->error = pool_error(data->repo->pool, SOLV_ERROR_ID_RANGE, "data_read_rel_idarray: id too large (%u/%u)", x, max);
269 *store++ = map ? map[x] : x;
282 /*******************************************************************************
283 * functions to add data to our incore memory space
286 #define INCORE_ADD_CHUNK 8192
287 #define DATA_READ_CHUNK 8192
290 incore_add_id(Repodata *data, Id sx)
292 unsigned int x = (unsigned int)sx;
294 /* make sure we have at least 5 bytes free */
295 if (data->incoredatafree < 5)
297 data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK);
298 data->incoredatafree = INCORE_ADD_CHUNK;
300 dp = data->incoredata + data->incoredatalen;
304 *dp++ = (x >> 28) | 128;
306 *dp++ = (x >> 21) | 128;
307 *dp++ = (x >> 14) | 128;
310 *dp++ = (x >> 7) | 128;
312 data->incoredatafree -= dp - (data->incoredata + data->incoredatalen);
313 data->incoredatalen = dp - data->incoredata;
317 incore_add_sizek(Repodata *data, unsigned int sx)
320 incore_add_id(data, (Id)(sx << 10));
325 incore_add_id(data, (Id)(sx >> 25));
326 data->incoredata[data->incoredatalen - 1] |= 128;
328 incore_add_id(data, (Id)((sx << 10) | 0x80000000));
329 data->incoredata[data->incoredatalen - 5] = (sx >> 18) | 128;
334 incore_add_ideof(Repodata *data, Id sx, int eof)
336 unsigned int x = (unsigned int)sx;
338 /* make sure we have at least 5 bytes free */
339 if (data->incoredatafree < 5)
341 data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK);
342 data->incoredatafree = INCORE_ADD_CHUNK;
344 dp = data->incoredata + data->incoredatalen;
348 *dp++ = (x >> 27) | 128;
350 *dp++ = (x >> 20) | 128;
351 *dp++ = (x >> 13) | 128;
354 *dp++ = (x >> 6) | 128;
355 *dp++ = eof ? (x & 63) : (x & 63) | 64;
356 data->incoredatafree -= dp - (data->incoredata + data->incoredatalen);
357 data->incoredatalen = dp - data->incoredata;
361 incore_add_blob(Repodata *data, unsigned char *buf, int len)
363 if (data->incoredatafree < (unsigned int)len)
365 data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK + len);
366 data->incoredatafree = INCORE_ADD_CHUNK + len;
368 memcpy(data->incoredata + data->incoredatalen, buf, len);
369 data->incoredatafree -= len;
370 data->incoredatalen += len;
374 incore_map_idarray(Repodata *data, unsigned char *dp, Id *map, Id max)
376 /* We have to map the IDs, which might also change
377 the necessary number of bytes, so we can't just copy
378 over the blob and adjust it. */
383 dp = data_read_ideof(dp, &id, &eof);
384 if (id < 0 || (max && id >= max))
386 data->error = pool_error(data->repo->pool, SOLV_ERROR_ID_RANGE, "incore_map_idarray: id too large (%u/%u)", id, max);
390 incore_add_ideof(data, id, eof);
398 incore_add_u32(Repodata *data, unsigned int x)
401 /* make sure we have at least 4 bytes free */
402 if (data->incoredatafree < 4)
404 data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK);
405 data->incoredatafree = INCORE_ADD_CHUNK;
407 dp = data->incoredata + data->incoredatalen;
412 data->incoredatafree -= 4;
413 data->incoredatalen += 4;
417 incore_add_u8(Repodata *data, unsigned int x)
420 /* make sure we have at least 1 byte free */
421 if (data->incoredatafree < 1)
423 data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + 1024);
424 data->incoredatafree = 1024;
426 dp = data->incoredata + data->incoredatalen;
428 data->incoredatafree--;
429 data->incoredatalen++;
434 /*******************************************************************************
439 * read repo from .solv file and add it to pool
443 repo_add_solv(Repo *repo, FILE *fp, int flags)
445 Pool *pool = repo->pool;
447 int numid, numrel, numdir, numsolv;
448 int numkeys, numschemata;
451 Offset *str; /* map Id -> Offset into string space */
452 char *strsp; /* repo string space */
453 char *sp; /* pointer into string space */
454 Id *idmap; /* map of repo Ids to pool Ids */
456 Hashval hashmask, h, hh;
461 unsigned int size_idarray;
462 Id *idarraydatap, *idarraydataend;
465 unsigned int solvflags;
466 unsigned int solvversion;
468 Id *schemadata, *schemadatap, *schemadataend;
469 Id *schemata, key, *keyp;
472 int maxsize, allsize;
473 unsigned char *buf, *bufend, *dp, *dps;
476 int needchunk; /* need a new chunk of data */
478 int oldnstrings = pool->ss.nstrings;
479 int oldnrels = pool->nrels;
481 struct _Stringpool *spool;
483 Repodata *parent = 0;
486 int extendstart = 0, extendend = 0; /* set in case we're extending */
488 now = solv_timems(0);
490 if ((flags & REPO_USE_LOADING) != 0)
492 /* this is a stub replace operation */
493 flags |= REPO_EXTEND_SOLVABLES;
494 /* use REPO_REUSE_REPODATA hack so that the old repodata is kept */
495 parent = repo_add_repodata(repo, flags | REPO_REUSE_REPODATA);
496 extendstart = parent->start;
497 extendend = parent->end;
499 else if (flags & REPO_EXTEND_SOLVABLES)
501 /* extend all solvables of this repo */
502 extendstart = repo->start;
503 extendend = repo->end;
506 memset(&data, 0, sizeof(data));
509 repopagestore_init(&data.store);
511 if (read_u32(&data) != ('S' << 24 | 'O' << 16 | 'L' << 8 | 'V'))
512 return pool_error(pool, SOLV_ERROR_NOT_SOLV, "not a SOLV file");
513 solvversion = read_u32(&data);
519 return pool_error(pool, SOLV_ERROR_UNSUPPORTED, "unsupported SOLV version");
522 numid = (int)read_u32(&data);
523 numrel = (int)read_u32(&data);
524 numdir = (int)read_u32(&data);
525 numsolv = (int)read_u32(&data);
526 numkeys = (int)read_u32(&data);
527 numschemata = (int)read_u32(&data);
528 solvflags = read_u32(&data);
530 if (numid < 0 || numid >= 0x20000000)
531 return pool_error(pool, SOLV_ERROR_CORRUPT, "bad number of ids");
532 if (numrel < 0 || numrel >= 0x20000000)
533 return pool_error(pool, SOLV_ERROR_CORRUPT, "bad number of rels");
534 if (numdir && (numdir < 2 || numdir >= 0x20000000))
535 return pool_error(pool, SOLV_ERROR_CORRUPT, "bad number of dirs");
536 if (numsolv < 0 || numsolv >= 0x20000000)
537 return pool_error(pool, SOLV_ERROR_CORRUPT, "bad number of solvables");
538 if (numkeys < 0 || numkeys >= 0x20000000)
539 return pool_error(pool, SOLV_ERROR_CORRUPT, "bad number of keys");
540 if (numschemata < 0 || numschemata >= 0x20000000)
541 return pool_error(pool, SOLV_ERROR_CORRUPT, "bad number of schematas");
543 if (numrel && (flags & REPO_LOCALPOOL) != 0)
544 return pool_error(pool, SOLV_ERROR_CORRUPT, "relations are forbidden in a local pool");
545 if ((flags & REPO_EXTEND_SOLVABLES) && numsolv)
547 /* make sure that we exactly replace the stub repodata */
548 if (extendend - extendstart != numsolv)
549 return pool_error(pool, SOLV_ERROR_CORRUPT, "sub-repository solvable number does not match main repository (%d - %d)", extendend - extendstart, numsolv);
550 for (i = 0; i < numsolv; i++)
551 if (pool->solvables[extendstart + i].repo != repo)
552 return pool_error(pool, SOLV_ERROR_CORRUPT, "main repository contains holes, cannot extend");
555 /******* Part 1: string IDs *****************************************/
557 sizeid = read_u32(&data); /* size of string space */
560 * read strings and Ids
569 if (!(flags & REPO_LOCALPOOL))
572 /* alloc max needed string buffer and string pointers, will shrink again later */
574 spool->stringspace = solv_realloc(spool->stringspace, spool->sstrings + sizeid + 1);
575 spool->strings = solv_realloc2(spool->strings, spool->nstrings + numid, sizeof(Offset));
577 spool->sstrings += sizeid + 1;
578 spool->nstrings += numid;
579 stringpool_shrink(spool); /* we misuse stringpool_shrink so that the correct BLOCK factor is used */
580 spool->sstrings -= sizeid + 1;
581 spool->nstrings -= numid;
588 spool->stringspace = solv_malloc(7 + sizeid + 1);
589 spool->strings = solv_malloc2(numid < 2 ? 2 : numid, sizeof(Offset));
590 strcpy(spool->stringspace, "<NULL>");
593 spool->strings[0] = 0; /* <NULL> */
598 * read string data and append to old string space
601 strsp = spool->stringspace + spool->sstrings; /* append new entries */
602 if ((solvflags & SOLV_FLAG_PREFIX_POOL) == 0)
604 if (sizeid && fread(strsp, sizeid, 1, fp) != 1)
606 repodata_freedata(&data);
607 return pool_error(pool, SOLV_ERROR_EOF, "read error while reading strings");
612 unsigned int pfsize = read_u32(&data);
613 char *prefix = solv_malloc(pfsize);
619 if (pfsize && fread(prefix, pfsize, 1, fp) != 1)
622 repodata_freedata(&data);
623 return pool_error(pool, SOLV_ERROR_EOF, "read error while reading strings");
625 for (i = 1; i < numid; i++)
627 int same = (unsigned char)*pp++;
628 size_t len = strlen(pp) + 1;
629 freesp -= same + len;
633 repodata_freedata(&data);
634 return pool_error(pool, SOLV_ERROR_OVERFLOW, "overflow while expanding strings");
637 memcpy(dest, old_str, same);
638 memcpy(dest + same, pp, len);
646 repodata_freedata(&data);
647 return pool_error(pool, SOLV_ERROR_CORRUPT, "expanding strings size mismatch");
650 strsp[sizeid] = 0; /* make string space \0 terminated */
654 str = spool->strings; /* array of offsets into strsp, indexed by Id */
655 if ((flags & REPO_LOCALPOOL) != 0)
657 /* no shared pool, thus no idmap and no unification needed */
659 spool->nstrings = numid < 2 ? 2 : numid; /* make sure we have at least id 0 and 1 */
662 /* we need id 1 to be '' for directories */
663 repodata_freedata(&data);
664 return pool_error(pool, SOLV_ERROR_CORRUPT, "store strings don't start with an empty string");
666 for (i = 1; i < spool->nstrings; i++)
668 if (sp >= strsp + sizeid && numid >= 2)
670 repodata_freedata(&data);
671 return pool_error(pool, SOLV_ERROR_OVERFLOW, "not enough strings");
673 str[i] = sp - spool->stringspace;
674 sp += strlen(sp) + 1;
676 spool->sstrings = sp - spool->stringspace;
680 Offset oldsstrings = spool->sstrings;
682 /* alloc id map for name and rel Ids. this maps ids in the solv files
683 * to the ids in our pool */
684 idmap = solv_calloc(numid + numrel, sizeof(Id));
686 /* grow hash if needed, otherwise reuse */
687 hashmask = mkmask(spool->nstrings + numid);
689 POOL_DEBUG(SOLV_DEBUG_STATS, "read %d strings\n", numid);
690 POOL_DEBUG(SOLV_DEBUG_STATS, "string hash buckets: %d, old %d\n", hashmask + 1, spool->stringhashmask + 1);
692 if (hashmask > spool->stringhashmask)
694 spool->stringhashtbl = solv_free(spool->stringhashtbl);
695 spool->stringhashmask = hashmask;
696 spool->stringhashtbl = hashtbl = solv_calloc(hashmask + 1, sizeof(Id));
697 for (i = 1; i < spool->nstrings; i++)
699 h = strhash(spool->stringspace + spool->strings[i]) & hashmask;
700 hh = HASHCHAIN_START;
702 h = HASHCHAIN_NEXT(h, hh, hashmask);
708 hashtbl = spool->stringhashtbl;
709 hashmask = spool->stringhashmask;
713 * run over strings and merge with pool.
714 * also populate id map (maps solv Id -> pool Id)
716 for (i = 1; i < numid; i++)
718 if (sp >= strsp + sizeid)
721 spool->nstrings = oldnstrings;
722 spool->sstrings = oldsstrings;
723 stringpool_freehash(spool);
724 repodata_freedata(&data);
725 return pool_error(pool, SOLV_ERROR_OVERFLOW, "not enough strings %d %d", i, numid);
727 if (!*sp) /* empty string */
735 h = strhash(sp) & hashmask;
736 hh = HASHCHAIN_START;
742 if (!strcmp(spool->stringspace + spool->strings[id], sp))
743 break; /* already in pool */
744 h = HASHCHAIN_NEXT(h, hh, hashmask);
747 /* length == offset to next string */
749 if (!id) /* end of hash chain -> new string */
751 id = spool->nstrings++;
753 str[id] = spool->sstrings; /* save offset */
754 if (sp != spool->stringspace + spool->sstrings)
755 memmove(spool->stringspace + spool->sstrings, sp, l);
756 spool->sstrings += l;
758 idmap[i] = id; /* repo relative -> pool relative */
759 sp += l; /* next string */
761 if (hashmask > mkmask(spool->nstrings + 8192))
763 spool->stringhashtbl = solv_free(spool->stringhashtbl);
764 spool->stringhashmask = 0;
766 stringpool_shrink(spool); /* vacuum */
770 /******* Part 2: Relation IDs ***************************************/
780 pool->rels = solv_realloc2(pool->rels, pool->nrels + numrel, sizeof(Reldep));
783 /* grow hash if needed, otherwise reuse */
784 hashmask = mkmask(pool->nrels + numrel);
786 POOL_DEBUG(SOLV_DEBUG_STATS, "read %d rels\n", numrel);
787 POOL_DEBUG(SOLV_DEBUG_STATS, "rel hash buckets: %d, old %d\n", hashmask + 1, pool->relhashmask + 1);
789 if (hashmask > pool->relhashmask)
791 pool->relhashtbl = solv_free(pool->relhashtbl);
792 pool->relhashmask = hashmask;
793 pool->relhashtbl = hashtbl = solv_calloc(hashmask + 1, sizeof(Id));
794 for (i = 1; i < pool->nrels; i++)
796 h = relhash(ran[i].name, ran[i].evr, ran[i].flags) & hashmask;
797 hh = HASHCHAIN_START;
799 h = HASHCHAIN_NEXT(h, hh, hashmask);
805 hashtbl = pool->relhashtbl;
806 hashmask = pool->relhashmask;
810 * read RelDeps from repo
812 for (i = 0; i < numrel; i++)
814 name = read_id(&data, i + numid); /* read (repo relative) Ids */
815 evr = read_id(&data, i + numid);
816 relflags = read_u8(&data);
817 name = idmap[name]; /* map to (pool relative) Ids */
819 h = relhash(name, evr, relflags) & hashmask;
820 hh = HASHCHAIN_START;
824 if (!id) /* end of hash chain reached */
826 if (ran[id].name == name && ran[id].evr == evr && ran[id].flags == relflags)
828 h = HASHCHAIN_NEXT(h, hh, hashmask);
830 if (!id) /* new RelDep */
836 ran[id].flags = relflags;
838 idmap[i + numid] = MAKERELDEP(id); /* fill Id map */
840 if (hashmask > mkmask(pool->nrels + 4096))
842 pool->relhashtbl = solv_free(pool->relhashtbl);
843 pool->relhashmask = 0;
845 pool_shrink_rels(pool); /* vacuum */
848 /* if we added ids/rels, make room in our whatprovide arrays */
849 if (!(flags & REPO_LOCALPOOL))
851 if (pool->whatprovides && oldnstrings != pool->ss.nstrings)
853 int newlen = (pool->ss.nstrings + WHATPROVIDES_BLOCK) & ~WHATPROVIDES_BLOCK;
854 pool->whatprovides = solv_realloc2(pool->whatprovides, newlen, sizeof(Offset));
855 memset(pool->whatprovides + oldnstrings, 0, (newlen - oldnstrings) * sizeof(Offset));
857 if (pool->whatprovides_rel && oldnrels != pool->nrels)
859 int newlen = (pool->nrels + WHATPROVIDES_BLOCK) & ~WHATPROVIDES_BLOCK;
860 pool->whatprovides_rel = solv_realloc2(pool->whatprovides_rel, newlen, sizeof(Offset));
861 memset(pool->whatprovides_rel + oldnrels, 0, (newlen - oldnrels) * sizeof(Offset));
865 /******* Part 3: Dirs ***********************************************/
868 data.dirpool.dirs = solv_malloc2(numdir, sizeof(Id));
869 data.dirpool.ndirs = numdir;
870 data.dirpool.dirs[0] = 0; /* dir 0: virtual root */
871 data.dirpool.dirs[1] = 1; /* dir 1: / */
872 for (i = 2; i < numdir; i++)
874 id = read_id(&data, i + numid);
877 data.dirpool.dirs[i++] = -(id - numid);
880 data.error = pool_error(pool, SOLV_ERROR_CORRUPT, "last dir entry is not a component");
883 id = read_id(&data, numid);
887 data.dirpool.dirs[i] = id;
889 data.error = pool_error(pool, SOLV_ERROR_CORRUPT, "bad dir component");
893 /******* Part 4: Keys ***********************************************/
895 keys = solv_calloc(numkeys, sizeof(*keys));
896 /* keys start at 1 */
897 for (i = 1; i < numkeys; i++)
899 id = read_id(&data, numid);
902 else if ((flags & REPO_LOCALPOOL) != 0)
903 id = pool_str2id(pool, stringpool_id2str(spool, id), 1);
904 type = read_id(&data, numid);
907 else if ((flags & REPO_LOCALPOOL) != 0)
908 type = pool_str2id(pool, stringpool_id2str(spool, type), 1);
909 if (type < REPOKEY_TYPE_VOID || type > REPOKEY_TYPE_FLEXARRAY)
911 data.error = pool_error(pool, SOLV_ERROR_UNSUPPORTED, "unsupported data type '%s'", pool_id2str(pool, type));
912 type = REPOKEY_TYPE_VOID;
916 keys[i].size = read_id(&data, keys[i].type == REPOKEY_TYPE_CONSTANTID ? numid + numrel : 0);
917 keys[i].storage = read_id(&data, 0);
918 /* old versions used SOLVABLE for main solvable data */
919 if (keys[i].storage == KEY_STORAGE_SOLVABLE)
920 keys[i].storage = KEY_STORAGE_INCORE;
921 if (keys[i].storage != KEY_STORAGE_INCORE && keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
922 data.error = pool_error(pool, SOLV_ERROR_UNSUPPORTED, "unsupported storage type %d", keys[i].storage);
923 if (id >= SOLVABLE_NAME && id <= RPM_RPMDBID)
925 if (keys[i].storage != KEY_STORAGE_INCORE)
926 data.error = pool_error(pool, SOLV_ERROR_UNSUPPORTED, "main solvable data must use incore storage %d", keys[i].storage);
927 keys[i].storage = KEY_STORAGE_SOLVABLE;
929 /* cannot handle rel idarrays in incore/vertical */
930 if (type == REPOKEY_TYPE_REL_IDARRAY && keys[i].storage != KEY_STORAGE_SOLVABLE)
931 data.error = pool_error(pool, SOLV_ERROR_UNSUPPORTED, "type REL_IDARRAY is only supported for STORAGE_SOLVABLE");
932 /* cannot handle mapped ids in vertical */
933 if (!(flags & REPO_LOCALPOOL) && keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET && (type == REPOKEY_TYPE_ID || type == REPOKEY_TYPE_IDARRAY))
934 data.error = pool_error(pool, SOLV_ERROR_UNSUPPORTED, "mapped ids are not supported for STORAGE_VERTICAL_OFFSET");
936 if (keys[i].type == REPOKEY_TYPE_CONSTANTID && idmap)
937 keys[i].size = idmap[keys[i].size];
939 fprintf(stderr, "key %d %s %s %d %d\n", i, pool_id2str(pool,id), pool_id2str(pool, keys[i].type),
940 keys[i].size, keys[i].storage);
945 for (i = 1; i < numkeys; i++)
946 if (keys[i].storage == KEY_STORAGE_INCORE || keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
950 data.nkeys = numkeys;
951 for (i = 1; i < numkeys; i++)
954 data.keybits[(id >> 3) & (sizeof(data.keybits) - 1)] |= 1 << (id & 7);
957 /******* Part 5: Schemata ********************************************/
959 id = read_id(&data, 0);
960 schemadata = solv_calloc(id + 1, sizeof(Id));
961 schemadatap = schemadata + 1;
962 schemadataend = schemadatap + id;
963 schemata = solv_calloc(numschemata, sizeof(Id));
964 for (i = 1; i < numschemata; i++)
966 schemata[i] = schemadatap - schemadata;
967 schemadatap = read_idarray(&data, numid, 0, schemadatap, schemadataend);
969 Id *sp = schemadata + schemata[i];
970 fprintf(stderr, "schema %d:", i);
972 fprintf(stderr, " %d", *sp);
973 fprintf(stderr, "\n");
976 data.schemata = schemata;
977 data.nschemata = numschemata;
978 data.schemadata = schemadata;
979 data.schemadatalen = schemadataend - data.schemadata;
981 /******* Part 6: Data ********************************************/
983 idarraydatap = idarraydataend = 0;
986 maxsize = read_id(&data, 0);
987 allsize = read_id(&data, 0);
988 maxsize += 5; /* so we can read the next schema of an array */
989 if (maxsize > allsize)
992 buf = solv_calloc(maxsize + DATA_READ_CHUNK + 4, 1); /* 4 extra bytes to detect overflows */
997 if (l < DATA_READ_CHUNK)
1001 if (!l || fread(buf, l, 1, data.fp) != 1)
1003 data.error = pool_error(pool, SOLV_ERROR_EOF, "unexpected EOF");
1010 dp = data_read_id_max(dp, &id, 0, numschemata, &data);
1013 incore_add_id(&data, 0); /* so that incoreoffset 0 means schema 0 */
1014 incore_add_id(&data, id); /* main schema id */
1015 keyp = schemadata + schemata[id];
1016 data.mainschema = id;
1017 for (i = 0; keyp[i]; i++)
1020 data.mainschemaoffsets = solv_calloc(i, sizeof(Id));
1028 /* make sure we have enough room */
1029 if (keydepth == 0 || needchunk)
1031 int left = bufend - dp;
1032 /* read data chunk to dp */
1037 data.error = pool_error(pool, SOLV_ERROR_EOF, "buffer overrun");
1043 memmove(buf, dp, left);
1045 if (l < DATA_READ_CHUNK)
1046 l = DATA_READ_CHUNK;
1049 if (l && fread(buf + left, l, 1, data.fp) != 1)
1051 data.error = pool_error(pool, SOLV_ERROR_EOF, "unexpected EOF");
1056 bufend = buf + left;
1057 if (allsize + left < maxsize)
1058 maxsize = allsize + left;
1066 printf("key %d at %d\n", key, (int)(keyp - 1 - schemadata));
1074 if (s && keydepth == 3)
1076 s++; /* next solvable */
1077 if (have_incoredata)
1078 data.incoreoffset[(s - pool->solvables) - data.start] = data.incoredatalen;
1080 id = stack[keydepth - 1];
1083 dp = data_read_id_max(dp, &id, 0, numschemata, &data);
1084 incore_add_id(&data, id);
1086 keyp = schemadata + schemata[id];
1093 keyp = schemadata + stack[--keydepth];
1094 nentries = stack[--keydepth];
1096 printf("pop flexarray %d %d\n", keydepth, nentries);
1099 s = 0; /* back from solvables */
1104 data.mainschemaoffsets[keyp - 1 - (schemadata + schemata[data.mainschema])] = data.incoredatalen;
1107 printf("=> %s %s %p\n", pool_id2str(pool, keys[key].name), pool_id2str(pool, keys[key].type), s);
1109 id = keys[key].name;
1110 if (keys[key].storage == KEY_STORAGE_VERTICAL_OFFSET)
1113 dp = data_skip(dp, REPOKEY_TYPE_ID);
1114 dp = data_skip(dp, REPOKEY_TYPE_ID);
1115 incore_add_blob(&data, dps, dp - dps); /* just record offset/size */
1118 switch (keys[key].type)
1120 case REPOKEY_TYPE_ID:
1121 dp = data_read_id_max(dp, &did, idmap, numid + numrel, &data);
1122 if (s && id == SOLVABLE_NAME)
1124 else if (s && id == SOLVABLE_ARCH)
1126 else if (s && id == SOLVABLE_EVR)
1128 else if (s && id == SOLVABLE_VENDOR)
1130 else if (keys[key].storage == KEY_STORAGE_INCORE)
1131 incore_add_id(&data, did);
1133 POOL_DEBUG(SOLV_DEBUG_STATS, "%s -> %s\n", pool_id2str(pool, id), pool_id2str(pool, did));
1136 case REPOKEY_TYPE_IDARRAY:
1137 case REPOKEY_TYPE_REL_IDARRAY:
1138 if (!s || id < INTERESTED_START || id > INTERESTED_END)
1141 dp = data_skip(dp, REPOKEY_TYPE_IDARRAY);
1142 if (keys[key].storage != KEY_STORAGE_INCORE)
1145 incore_map_idarray(&data, dps, idmap, numid + numrel);
1147 incore_add_blob(&data, dps, dp - dps);
1150 ido = idarraydatap - repo->idarraydata;
1151 if (keys[key].type == REPOKEY_TYPE_IDARRAY)
1152 dp = data_read_idarray(dp, &idarraydatap, idmap, numid + numrel, &data);
1153 else if (id == SOLVABLE_REQUIRES)
1154 dp = data_read_rel_idarray(dp, &idarraydatap, idmap, numid + numrel, &data, SOLVABLE_PREREQMARKER);
1155 else if (id == SOLVABLE_PROVIDES)
1156 dp = data_read_rel_idarray(dp, &idarraydatap, idmap, numid + numrel, &data, SOLVABLE_FILEMARKER);
1158 dp = data_read_rel_idarray(dp, &idarraydatap, idmap, numid + numrel, &data, 0);
1159 if (idarraydatap > idarraydataend)
1161 data.error = pool_error(pool, SOLV_ERROR_OVERFLOW, "idarray overflow");
1164 if (id == SOLVABLE_PROVIDES)
1166 else if (id == SOLVABLE_OBSOLETES)
1168 else if (id == SOLVABLE_CONFLICTS)
1170 else if (id == SOLVABLE_REQUIRES)
1172 else if (id == SOLVABLE_RECOMMENDS)
1174 else if (id == SOLVABLE_SUPPLEMENTS)
1175 s->supplements = ido;
1176 else if (id == SOLVABLE_SUGGESTS)
1178 else if (id == SOLVABLE_ENHANCES)
1181 POOL_DEBUG(SOLV_DEBUG_STATS, "%s ->\n", pool_id2str(pool, id));
1182 for (; repo->idarraydata[ido]; ido++)
1183 POOL_DEBUG(SOLV_DEBUG_STATS," %s\n", pool_dep2str(pool, repo->idarraydata[ido]));
1186 case REPOKEY_TYPE_FIXARRAY:
1187 case REPOKEY_TYPE_FLEXARRAY:
1190 if (keydepth == sizeof(stack)/sizeof(*stack))
1192 data.error = pool_error(pool, SOLV_ERROR_OVERFLOW, "array stack overflow");
1195 stack[keydepth++] = nentries;
1196 stack[keydepth++] = keyp - schemadata;
1197 stack[keydepth++] = 0;
1198 dp = data_read_id_max(dp, &nentries, 0, 0, &data);
1199 incore_add_id(&data, nentries);
1202 /* zero size array? */
1204 nentries = stack[--keydepth];
1207 if (keydepth == 3 && id == REPOSITORY_SOLVABLES)
1209 /* horray! here come the solvables */
1210 if (nentries != numsolv)
1212 data.error = pool_error(pool, SOLV_ERROR_CORRUPT, "inconsistent number of solvables: %d %d", nentries, numsolv);
1217 data.error = pool_error(pool, SOLV_ERROR_CORRUPT, "more than one solvable block");
1220 if ((flags & REPO_EXTEND_SOLVABLES) != 0)
1221 s = pool_id2solvable(pool, extendstart);
1223 s = pool_id2solvable(pool, repo_add_solvable_block(repo, numsolv));
1224 data.start = s - pool->solvables;
1225 data.end = data.start + numsolv;
1226 repodata_extend_block(&data, data.start, numsolv);
1227 for (i = 1; i < numkeys; i++)
1230 if ((keys[i].type == REPOKEY_TYPE_IDARRAY || keys[i].type == REPOKEY_TYPE_REL_IDARRAY)
1231 && id >= INTERESTED_START && id <= INTERESTED_END)
1232 size_idarray += keys[i].size;
1234 /* allocate needed space in repo */
1235 /* we add maxsize because it is an upper limit for all idarrays, thus we can't overflow */
1236 repo_reserve_ids(repo, 0, size_idarray + maxsize + 1);
1237 idarraydatap = repo->idarraydata + repo->idarraysize;
1238 repo->idarraysize += size_idarray;
1239 idarraydataend = idarraydatap + size_idarray;
1241 if (have_incoredata)
1242 data.incoreoffset[(s - pool->solvables) - data.start] = data.incoredatalen;
1245 dp = data_read_id_max(dp, &id, 0, numschemata, &data);
1246 incore_add_id(&data, id);
1247 if (keys[key].type == REPOKEY_TYPE_FIXARRAY)
1250 data.error = pool_error(pool, SOLV_ERROR_CORRUPT, "illegal fixarray");
1251 stack[keydepth - 1] = id;
1253 keyp = schemadata + schemata[id];
1255 case REPOKEY_TYPE_NUM:
1256 if (!(solvflags & SOLV_FLAG_SIZE_BYTES) && keys[key].storage == KEY_STORAGE_INCORE &&
1257 (id == SOLVABLE_INSTALLSIZE || id == SOLVABLE_DOWNLOADSIZE || id == DELTA_DOWNLOADSIZE))
1259 /* old solv file with sizes in kilos. transcode. */
1260 dp = data_read_id(dp, &id);
1261 incore_add_sizek(&data, (unsigned int)id);
1266 if (id == RPM_RPMDBID && s && (keys[key].type == REPOKEY_TYPE_U32 || keys[key].type == REPOKEY_TYPE_NUM))
1268 if (keys[key].type == REPOKEY_TYPE_U32)
1269 dp = data_read_u32(dp, (unsigned int *)&id);
1271 dp = data_read_id_max(dp, &id, 0, 0, &data);
1273 repo->rpmdbid = repo_sidedata_create(repo, sizeof(Id));
1274 repo->rpmdbid[(s - pool->solvables) - repo->start] = id;
1278 dp = data_skip(dp, keys[key].type);
1279 if (keys[key].storage == KEY_STORAGE_INCORE)
1280 incore_add_blob(&data, dps, dp - dps);
1284 /* should shrink idarraydata again */
1287 data.error = pool_error(pool, SOLV_ERROR_EOF, "unexpected EOF, depth = %d", keydepth);
1291 data.error = pool_error(pool, SOLV_ERROR_EOF, "buffer overrun");
1297 /* free solvables */
1298 repo_free_solvable_block(repo, data.start, data.end - data.start, 1);
1300 repo->idarraysize -= size_idarray;
1301 /* free incore data */
1302 data.incoredata = solv_free(data.incoredata);
1303 data.incoredatalen = data.incoredatafree = 0;
1306 if (data.incoredatafree)
1308 /* shrink excess size */
1309 data.incoredata = solv_realloc(data.incoredata, data.incoredatalen);
1310 data.incoredatafree = 0;
1314 /* fixup the special idarray type */
1315 for (i = 1; i < numkeys; i++)
1316 if (keys[i].type == REPOKEY_TYPE_REL_IDARRAY)
1317 keys[i].type = REPOKEY_TYPE_IDARRAY;
1319 for (i = 1; i < numkeys; i++)
1320 if (keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
1322 if (i < numkeys && !data.error)
1325 unsigned int pagesize;
1327 /* we have vertical data, make it available */
1328 data.verticaloffset = solv_calloc(numkeys, sizeof(Id));
1329 for (i = 1; i < numkeys; i++)
1330 if (keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
1332 data.verticaloffset[i] = fileoffset;
1333 fileoffset += keys[i].size;
1335 data.lastverticaloffset = fileoffset;
1336 pagesize = read_u32(&data);
1339 data.error = repopagestore_read_or_setup_pages(&data.store, data.fp, pagesize, fileoffset);
1340 if (data.error == SOLV_ERROR_EOF)
1341 pool_error(pool, data.error, "repopagestore setup: unexpected EOF");
1342 else if (data.error)
1343 pool_error(pool, data.error, "repopagestore setup failed");
1346 data.fp = 0; /* no longer needed */
1351 repodata_freedata(&data);
1357 /* overwrite stub repodata */
1358 repodata_freedata(parent);
1359 data.repodataid = parent->repodataid;
1364 /* make it available as new repodata */
1365 if (!repo->nrepodata)
1367 repo->nrepodata = 1;
1368 repo->repodata = solv_calloc(2, sizeof(data));
1371 repo->repodata = solv_realloc2(repo->repodata, repo->nrepodata + 1, sizeof(data));
1372 data.repodataid = repo->nrepodata;
1373 repo->repodata[repo->nrepodata++] = data;
1376 /* create stub repodata entries for all external */
1377 if (!(flags & SOLV_ADD_NO_STUBS) && !parent)
1379 for (key = 1 ; key < data.nkeys; key++)
1380 if (data.keys[key].name == REPOSITORY_EXTERNAL && data.keys[key].type == REPOKEY_TYPE_FLEXARRAY)
1382 if (key < data.nkeys)
1383 repodata_create_stubs(repo->repodata + (repo->nrepodata - 1));
1386 POOL_DEBUG(SOLV_DEBUG_STATS, "repo_add_solv took %d ms\n", solv_timems(now));
1387 POOL_DEBUG(SOLV_DEBUG_STATS, "repo size: %d solvables\n", repo->nsolvables);
1388 POOL_DEBUG(SOLV_DEBUG_STATS, "repo memory used: %d K incore, %d K idarray\n", data.incoredatalen/1024, repo->idarraysize / (int)(1024/sizeof(Id)));