2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Add a repo in solv format
22 #include "repo_solv.h"
28 #include "poolid_private.h" /* WHATPROVIDES_BLOCK */
30 #define INTERESTED_START SOLVABLE_NAME
31 #define INTERESTED_END SOLVABLE_ENHANCES
33 #define SOLV_ERROR_NOT_SOLV 1
34 #define SOLV_ERROR_UNSUPPORTED 2
35 #define SOLV_ERROR_EOF 3
36 #define SOLV_ERROR_ID_RANGE 4
37 #define SOLV_ERROR_OVERFLOW 5
38 #define SOLV_ERROR_CORRUPT 6
42 /*******************************************************************************
43 * functions to extract data from a file handle
51 read_u32(Repodata *data)
58 for (i = 0; i < 4; i++)
63 pool_debug(data->repo->pool, SOLV_ERROR, "unexpected EOF\n");
64 data->error = SOLV_ERROR_EOF;
78 read_u8(Repodata *data)
87 pool_debug(data->repo->pool, SOLV_ERROR, "unexpected EOF\n");
88 data->error = SOLV_ERROR_EOF;
100 read_id(Repodata *data, Id max)
107 for (i = 0; i < 5; i++)
112 pool_debug(data->repo->pool, SOLV_ERROR, "unexpected EOF\n");
113 data->error = SOLV_ERROR_EOF;
121 pool_debug(data->repo->pool, SOLV_ERROR, "read_id: id too large (%u/%u)\n", x, max);
122 data->error = SOLV_ERROR_ID_RANGE;
127 x = (x << 7) ^ c ^ 128;
129 pool_debug(data->repo->pool, SOLV_ERROR, "read_id: id too long\n");
130 data->error = SOLV_ERROR_CORRUPT;
136 read_idarray(Repodata *data, Id max, Id *map, Id *store, Id *end)
148 pool_debug(data->repo->pool, SOLV_ERROR, "unexpected EOF\n");
149 data->error = SOLV_ERROR_EOF;
154 x = (x << 7) ^ c ^ 128;
157 x = (x << 6) | (c & 63);
160 pool_debug(data->repo->pool, SOLV_ERROR, "read_idarray: id too large (%u/%u)\n", x, max);
161 data->error = SOLV_ERROR_ID_RANGE;
168 pool_debug(data->repo->pool, SOLV_ERROR, "read_idarray: array overflow\n");
174 if (x == 0) /* already have trailing zero? */
178 pool_debug(data->repo->pool, SOLV_ERROR, "read_idarray: array overflow\n");
179 data->error = SOLV_ERROR_OVERFLOW;
190 /*******************************************************************************
191 * functions to extract data from memory
198 static inline unsigned char *
199 data_read_id_max(unsigned char *dp, Id *ret, Id *map, int max, Repodata *data)
202 dp = data_read_id(dp, &x);
203 if (x < 0 || (max && x >= max))
205 pool_debug(data->repo->pool, SOLV_ERROR, "data_read_id_max: id too large (%u/%u)\n", x, max);
206 data->error = SOLV_ERROR_ID_RANGE;
209 *ret = map ? map[x] : x;
213 static unsigned char *
214 data_read_idarray(unsigned char *dp, Id **storep, Id *map, int max, Repodata *data)
225 x = (x << 7) ^ c ^ 128;
228 x = (x << 6) | (c & 63);
231 pool_debug(data->repo->pool, SOLV_ERROR, "data_read_idarray: id too large (%u/%u)\n", x, max);
232 data->error = SOLV_ERROR_ID_RANGE;
245 static unsigned char *
246 data_read_rel_idarray(unsigned char *dp, Id **storep, Id *map, int max, Repodata *data, Id marker)
258 x = (x << 7) ^ c ^ 128;
261 x = (x << 6) | (c & 63);
275 pool_debug(data->repo->pool, SOLV_ERROR, "data_read_rel_idarray: id too large (%u/%u)\n", x, max);
276 data->error = SOLV_ERROR_ID_RANGE;
279 *store++ = map ? map[x] : x;
292 /*******************************************************************************
293 * functions to add data to our incore memory space
296 #define INCORE_ADD_CHUNK 8192
297 #define DATA_READ_CHUNK 8192
300 incore_add_id(Repodata *data, Id sx)
302 unsigned int x = (unsigned int)sx;
304 /* make sure we have at least 5 bytes free */
305 if (data->incoredatafree < 5)
307 data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK);
308 data->incoredatafree = INCORE_ADD_CHUNK;
310 dp = data->incoredata + data->incoredatalen;
314 *dp++ = (x >> 28) | 128;
316 *dp++ = (x >> 21) | 128;
317 *dp++ = (x >> 14) | 128;
320 *dp++ = (x >> 7) | 128;
322 data->incoredatafree -= dp - (data->incoredata + data->incoredatalen);
323 data->incoredatalen = dp - data->incoredata;
327 incore_add_sizek(Repodata *data, unsigned int sx)
330 incore_add_id(data, (Id)(sx << 10));
335 incore_add_id(data, (Id)(sx >> 25));
336 data->incoredata[data->incoredatalen - 1] |= 128;
338 incore_add_id(data, (Id)((sx << 10) | 0x80000000));
339 data->incoredata[data->incoredatalen - 5] = (sx >> 18) | 128;
344 incore_add_ideof(Repodata *data, Id sx, int eof)
346 unsigned int x = (unsigned int)sx;
348 /* make sure we have at least 5 bytes free */
349 if (data->incoredatafree < 5)
351 data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK);
352 data->incoredatafree = INCORE_ADD_CHUNK;
354 dp = data->incoredata + data->incoredatalen;
358 *dp++ = (x >> 27) | 128;
360 *dp++ = (x >> 20) | 128;
361 *dp++ = (x >> 13) | 128;
364 *dp++ = (x >> 6) | 128;
365 *dp++ = eof ? (x & 63) : (x & 63) | 64;
366 data->incoredatafree -= dp - (data->incoredata + data->incoredatalen);
367 data->incoredatalen = dp - data->incoredata;
371 incore_add_blob(Repodata *data, unsigned char *buf, int len)
373 if (data->incoredatafree < len)
375 data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK + len);
376 data->incoredatafree = INCORE_ADD_CHUNK + len;
378 memcpy(data->incoredata + data->incoredatalen, buf, len);
379 data->incoredatafree -= len;
380 data->incoredatalen += len;
384 incore_map_idarray(Repodata *data, unsigned char *dp, Id *map, Id max)
386 /* We have to map the IDs, which might also change
387 the necessary number of bytes, so we can't just copy
388 over the blob and adjust it. */
393 dp = data_read_ideof(dp, &id, &eof);
394 if (id < 0 || (max && id >= max))
396 pool_debug(data->repo->pool, SOLV_ERROR, "incore_map_idarray: id too large (%u/%u)\n", id, max);
397 data->error = SOLV_ERROR_ID_RANGE;
401 incore_add_ideof(data, id, eof);
409 incore_add_u32(Repodata *data, unsigned int x)
412 /* make sure we have at least 4 bytes free */
413 if (data->incoredatafree < 4)
415 data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK);
416 data->incoredatafree = INCORE_ADD_CHUNK;
418 dp = data->incoredata + data->incoredatalen;
423 data->incoredatafree -= 4;
424 data->incoredatalen += 4;
428 incore_add_u8(Repodata *data, unsigned int x)
431 /* make sure we have at least 1 byte free */
432 if (data->incoredatafree < 1)
434 data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + 1024);
435 data->incoredatafree = 1024;
437 dp = data->incoredata + data->incoredatalen;
439 data->incoredatafree--;
440 data->incoredatalen++;
445 /*******************************************************************************
450 * read repo from .solv file and add it to pool
454 repo_add_solv(Repo *repo, FILE *fp, int flags)
456 Pool *pool = repo->pool;
458 unsigned int numid, numrel, numdir, numsolv;
459 unsigned int numkeys, numschemata;
462 Offset *str; /* map Id -> Offset into string space */
463 char *strsp; /* repo string space */
464 char *sp; /* pointer into string space */
465 Id *idmap; /* map of repo Ids to pool Ids */
467 unsigned int hashmask, h;
473 unsigned int size_idarray;
474 Id *idarraydatap, *idarraydataend;
477 unsigned int solvflags;
478 unsigned int solvversion;
480 Id *schemadata, *schemadatap, *schemadataend;
481 Id *schemata, key, *keyp;
484 int maxsize, allsize;
485 unsigned char *buf, *bufend, *dp, *dps;
488 int needchunk; /* need a new chunk of data */
490 int oldnstrings = pool->ss.nstrings;
491 int oldnrels = pool->nrels;
493 struct _Stringpool *spool;
495 Repodata *parent = 0;
498 now = solv_timems(0);
500 if ((flags & REPO_USE_LOADING) != 0)
502 /* this is a stub replace operation */
503 flags |= REPO_EXTEND_SOLVABLES;
504 /* use REPO_REUSE_REPODATA hack so that the old repodata is kept */
505 parent = repo_add_repodata(repo, flags | REPO_REUSE_REPODATA);
508 memset(&data, 0, sizeof(data));
511 repopagestore_init(&data.store);
513 if (read_u32(&data) != ('S' << 24 | 'O' << 16 | 'L' << 8 | 'V'))
515 pool_debug(pool, SOLV_ERROR, "not a SOLV file\n");
516 return SOLV_ERROR_NOT_SOLV;
518 solvversion = read_u32(&data);
524 pool_debug(pool, SOLV_ERROR, "unsupported SOLV version\n");
525 return SOLV_ERROR_UNSUPPORTED;
528 numid = read_u32(&data);
529 numrel = read_u32(&data);
530 numdir = read_u32(&data);
531 numsolv = read_u32(&data);
532 numkeys = read_u32(&data);
533 numschemata = read_u32(&data);
534 solvflags = read_u32(&data);
536 if (numdir && numdir < 2)
538 pool_debug(pool, SOLV_ERROR, "bad number of dirs\n");
539 return SOLV_ERROR_CORRUPT;
542 if (numrel && (flags & REPO_LOCALPOOL) != 0)
544 pool_debug(pool, SOLV_ERROR, "relations are forbidden in a local pool\n");
545 return SOLV_ERROR_CORRUPT;
547 if (parent && numsolv)
549 /* make sure that we exactly replace the stub repodata */
550 if (parent->end - parent->start != numsolv)
552 pool_debug(pool, SOLV_ERROR, "sub-repository solvable number does not match main repository (%d - %d)\n", parent->end - parent->start, numsolv);
553 return SOLV_ERROR_CORRUPT;
555 for (i = 0; i < numsolv; i++)
556 if (pool->solvables[parent->start + i].repo != repo)
558 pool_debug(pool, SOLV_ERROR, "main repository contains holes\n");
559 return SOLV_ERROR_CORRUPT;
563 /******* Part 1: string IDs *****************************************/
565 sizeid = read_u32(&data); /* size of string+Id space */
568 * read strings and Ids
577 if (!(flags & REPO_LOCALPOOL))
583 spool->stringspace = solv_malloc(7);
584 strcpy(spool->stringspace, "<NULL>");
589 /* alloc string buffer */
590 spool->stringspace = solv_realloc(spool->stringspace, spool->sstrings + sizeid + 1);
591 /* alloc string offsets (Id -> Offset into string space) */
592 spool->strings = solv_realloc2(spool->strings, spool->nstrings + numid, sizeof(Offset));
594 strsp = spool->stringspace;
595 str = spool->strings; /* array of offsets into strsp, indexed by Id */
597 /* point to _BEHIND_ already allocated string/Id space */
598 strsp += spool->sstrings;
602 * read new repo at end of pool
605 if ((solvflags & SOLV_FLAG_PREFIX_POOL) == 0)
607 if (sizeid && fread(strsp, sizeid, 1, fp) != 1)
609 pool_debug(pool, SOLV_ERROR, "read error while reading strings\n");
610 return SOLV_ERROR_EOF;
615 unsigned int pfsize = read_u32(&data);
616 char *prefix = solv_malloc(pfsize);
622 if (pfsize && fread(prefix, pfsize, 1, fp) != 1)
624 pool_debug(pool, SOLV_ERROR, "read error while reading strings\n");
626 return SOLV_ERROR_EOF;
628 for (i = 1; i < numid; i++)
630 int same = (unsigned char)*pp++;
631 size_t len = strlen(pp) + 1;
632 freesp -= same + len;
635 pool_debug(pool, SOLV_ERROR, "overflow while expanding strings\n");
637 return SOLV_ERROR_OVERFLOW;
640 memcpy(dest, old_str, same);
641 memcpy(dest + same, pp, len);
649 pool_debug(pool, SOLV_ERROR, "expanding strings size mismatch\n");
650 return SOLV_ERROR_CORRUPT;
653 strsp[sizeid] = 0; /* make string space \0 terminated */
656 if ((flags & REPO_LOCALPOOL) != 0)
658 /* no shared pool, thus no idmap and no unification */
660 spool->nstrings = numid;
664 /* we need the '' for directories */
665 pool_debug(pool, SOLV_ERROR, "store strings don't start with ''\n");
666 return SOLV_ERROR_CORRUPT;
668 for (i = 1; i < spool->nstrings; i++)
670 if (sp >= strsp + sizeid)
672 pool_debug(pool, SOLV_ERROR, "not enough strings\n");
673 return SOLV_ERROR_OVERFLOW;
675 str[i] = sp - spool->stringspace;
676 sp += strlen(sp) + 1;
678 spool->sstrings = sp - spool->stringspace;
682 /* alloc id map for name and rel Ids. this maps ids in the solv files
683 * to the ids in our pool */
684 idmap = solv_calloc(numid + numrel, sizeof(Id));
686 /* grow hash if needed, otherwise reuse */
687 hashmask = mkmask(spool->nstrings + numid);
689 POOL_DEBUG(SOLV_DEBUG_STATS, "read %d strings\n", numid);
690 POOL_DEBUG(SOLV_DEBUG_STATS, "string hash buckets: %d, old %d\n", hashmask + 1, spool->stringhashmask + 1);
692 if (hashmask > spool->stringhashmask)
694 spool->stringhashtbl = solv_free(spool->stringhashtbl);
695 spool->stringhashmask = hashmask;
696 spool->stringhashtbl = hashtbl = solv_calloc(hashmask + 1, sizeof(Id));
697 for (i = 1; i < spool->nstrings; i++)
699 h = strhash(spool->stringspace + spool->strings[i]) & hashmask;
700 hh = HASHCHAIN_START;
702 h = HASHCHAIN_NEXT(h, hh, hashmask);
708 hashtbl = spool->stringhashtbl;
709 hashmask = spool->stringhashmask;
713 * run over strings and merge with pool.
714 * also populate id map (maps solv Id -> pool Id)
716 for (i = 1; i < numid; i++)
718 if (sp >= strsp + sizeid)
722 pool_debug(pool, SOLV_ERROR, "not enough strings %d %d\n", i, numid);
723 return SOLV_ERROR_OVERFLOW;
725 if (!*sp) /* empty string */
733 h = strhash(sp) & hashmask;
734 hh = HASHCHAIN_START;
740 if (!strcmp(spool->stringspace + spool->strings[id], sp))
741 break; /* already in pool */
742 h = HASHCHAIN_NEXT(h, hh, hashmask);
745 /* length == offset to next string */
747 if (!id) /* end of hash chain -> new string */
749 id = spool->nstrings++;
751 str[id] = spool->sstrings; /* save offset */
752 if (sp != spool->stringspace + spool->sstrings)
753 memmove(spool->stringspace + spool->sstrings, sp, l);
754 spool->sstrings += l;
756 idmap[i] = id; /* repo relative -> pool relative */
757 sp += l; /* next string */
759 if (hashmask > mkmask(spool->nstrings + 8192))
761 spool->stringhashtbl = solv_free(spool->stringhashtbl);
762 spool->stringhashmask = 0;
765 pool_shrink_strings(pool); /* vacuum */
768 /******* Part 2: Relation IDs ***************************************/
778 pool->rels = solv_realloc2(pool->rels, pool->nrels + numrel, sizeof(Reldep));
781 /* grow hash if needed, otherwise reuse */
782 hashmask = mkmask(pool->nrels + numrel);
784 POOL_DEBUG(SOLV_DEBUG_STATS, "read %d rels\n", numrel);
785 POOL_DEBUG(SOLV_DEBUG_STATS, "rel hash buckets: %d, old %d\n", hashmask + 1, pool->relhashmask + 1);
787 if (hashmask > pool->relhashmask)
789 pool->relhashtbl = solv_free(pool->relhashtbl);
790 pool->relhashmask = hashmask;
791 pool->relhashtbl = hashtbl = solv_calloc(hashmask + 1, sizeof(Id));
792 for (i = 1; i < pool->nrels; i++)
794 h = relhash(ran[i].name, ran[i].evr, ran[i].flags) & hashmask;
795 hh = HASHCHAIN_START;
797 h = HASHCHAIN_NEXT(h, hh, hashmask);
803 hashtbl = pool->relhashtbl;
804 hashmask = pool->relhashmask;
808 * read RelDeps from repo
810 for (i = 0; i < numrel; i++)
812 name = read_id(&data, i + numid); /* read (repo relative) Ids */
813 evr = read_id(&data, i + numid);
814 relflags = read_u8(&data);
815 name = idmap[name]; /* map to (pool relative) Ids */
817 h = relhash(name, evr, relflags) & hashmask;
818 hh = HASHCHAIN_START;
822 if (!id) /* end of hash chain reached */
824 if (ran[id].name == name && ran[id].evr == evr && ran[id].flags == relflags)
826 h = HASHCHAIN_NEXT(h, hh, hashmask);
828 if (!id) /* new RelDep */
834 ran[id].flags = relflags;
836 idmap[i + numid] = MAKERELDEP(id); /* fill Id map */
838 if (hashmask > mkmask(pool->nrels + 4096))
840 pool->relhashtbl = solv_free(pool->relhashtbl);
841 pool->relhashmask = 0;
843 pool_shrink_rels(pool); /* vacuum */
846 /* if we added ids/rels, make room in our whatprovide arrays */
847 if (!(flags & REPO_LOCALPOOL))
849 if (pool->whatprovides && oldnstrings != pool->ss.nstrings)
851 int newlen = (pool->ss.nstrings + WHATPROVIDES_BLOCK) & ~WHATPROVIDES_BLOCK;
852 pool->whatprovides = solv_realloc2(pool->whatprovides, newlen, sizeof(Offset));
853 memset(pool->whatprovides + oldnstrings, 0, (newlen - oldnstrings) * sizeof(Offset));
855 if (pool->whatprovides_rel && oldnrels != pool->nrels)
857 int newlen = (pool->nrels + WHATPROVIDES_BLOCK) & ~WHATPROVIDES_BLOCK;
858 pool->whatprovides_rel = solv_realloc2(pool->whatprovides_rel, newlen, sizeof(Offset));
859 memset(pool->whatprovides_rel + oldnrels, 0, (newlen - oldnrels) * sizeof(Offset));
863 /******* Part 3: Dirs ***********************************************/
866 data.dirpool.dirs = solv_malloc2(numdir, sizeof(Id));
867 data.dirpool.ndirs = numdir;
868 data.dirpool.dirs[0] = 0; /* dir 0: virtual root */
869 data.dirpool.dirs[1] = 1; /* dir 1: / */
870 for (i = 2; i < numdir; i++)
872 id = read_id(&data, i + numid);
874 data.dirpool.dirs[i] = -(id - numid);
876 data.dirpool.dirs[i] = idmap[id];
878 data.dirpool.dirs[i] = id;
882 /******* Part 4: Keys ***********************************************/
884 keys = solv_calloc(numkeys, sizeof(*keys));
885 /* keys start at 1 */
886 for (i = 1; i < numkeys; i++)
888 id = read_id(&data, numid);
891 else if ((flags & REPO_LOCALPOOL) != 0)
892 id = pool_str2id(pool, stringpool_id2str(spool, id), 1);
893 type = read_id(&data, numid);
896 else if ((flags & REPO_LOCALPOOL) != 0)
897 type = pool_str2id(pool, stringpool_id2str(spool, type), 1);
898 if (type < REPOKEY_TYPE_VOID || type > REPOKEY_TYPE_FLEXARRAY)
900 pool_debug(pool, SOLV_ERROR, "unsupported data type '%s'\n", pool_id2str(pool, type));
901 data.error = SOLV_ERROR_UNSUPPORTED;
902 type = REPOKEY_TYPE_VOID;
906 keys[i].size = read_id(&data, keys[i].type == REPOKEY_TYPE_CONSTANTID ? numid + numrel : 0);
907 keys[i].storage = read_id(&data, 0);
908 /* old versions used SOLVABLE for main solvable data */
909 if (keys[i].storage == KEY_STORAGE_SOLVABLE)
910 keys[i].storage = KEY_STORAGE_INCORE;
911 if (keys[i].storage != KEY_STORAGE_INCORE && keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
913 pool_debug(pool, SOLV_ERROR, "unsupported storage type %d\n", keys[i].storage);
914 data.error = SOLV_ERROR_UNSUPPORTED;
916 if (id >= SOLVABLE_NAME && id <= RPM_RPMDBID)
918 if (keys[i].storage != KEY_STORAGE_INCORE)
920 pool_debug(pool, SOLV_ERROR, "main solvable data must use incore storage%d\n", keys[i].storage);
921 data.error = SOLV_ERROR_UNSUPPORTED;
923 keys[i].storage = KEY_STORAGE_SOLVABLE;
925 /* cannot handle rel idarrays in incore/vertical */
926 if (type == REPOKEY_TYPE_REL_IDARRAY && keys[i].storage != KEY_STORAGE_SOLVABLE)
928 pool_debug(pool, SOLV_ERROR, "type REL_IDARRAY is only supported for STORAGE_SOLVABLE\n");
929 data.error = SOLV_ERROR_UNSUPPORTED;
931 /* cannot handle mapped ids in vertical */
932 if (!(flags & REPO_LOCALPOOL) && keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET && (type == REPOKEY_TYPE_ID || type == REPOKEY_TYPE_IDARRAY))
934 pool_debug(pool, SOLV_ERROR, "mapped ids are not supported for STORAGE_VERTICAL_OFFSET\n");
935 data.error = SOLV_ERROR_UNSUPPORTED;
938 if (keys[i].type == REPOKEY_TYPE_CONSTANTID && idmap)
939 keys[i].size = idmap[keys[i].size];
941 fprintf(stderr, "key %d %s %s %d %d\n", i, pool_id2str(pool,id), pool_id2str(pool, keys[i].type),
942 keys[i].size, keys[i].storage);
946 have_xdata = parent ? 1 : 0;
947 for (i = 1; i < numkeys; i++)
948 if (keys[i].storage == KEY_STORAGE_INCORE || keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
952 data.nkeys = numkeys;
953 for (i = 1; i < numkeys; i++)
956 data.keybits[(id >> 3) & (sizeof(data.keybits) - 1)] |= 1 << (id & 7);
959 /******* Part 5: Schemata ********************************************/
961 id = read_id(&data, 0);
962 schemadata = solv_calloc(id + 1, sizeof(Id));
963 schemadatap = schemadata + 1;
964 schemadataend = schemadatap + id;
965 schemata = solv_calloc(numschemata, sizeof(Id));
966 for (i = 1; i < numschemata; i++)
968 schemata[i] = schemadatap - schemadata;
969 schemadatap = read_idarray(&data, numid, 0, schemadatap, schemadataend);
971 Id *sp = schemadata + schemata[i];
972 fprintf(stderr, "schema %d:", i);
974 fprintf(stderr, " %d", *sp);
975 fprintf(stderr, "\n");
978 data.schemata = schemata;
979 data.nschemata = numschemata;
980 data.schemadata = schemadata;
981 data.schemadatalen = schemadataend - data.schemadata;
983 /******* Part 6: Data ********************************************/
985 idarraydatap = idarraydataend = 0;
988 maxsize = read_id(&data, 0);
989 allsize = read_id(&data, 0);
990 maxsize += 5; /* so we can read the next schema of an array */
991 if (maxsize > allsize)
994 buf = solv_calloc(maxsize + DATA_READ_CHUNK + 4, 1); /* 4 extra bytes to detect overflows */
999 if (l < DATA_READ_CHUNK)
1000 l = DATA_READ_CHUNK;
1003 if (!l || fread(buf, l, 1, data.fp) != 1)
1005 pool_debug(pool, SOLV_ERROR, "unexpected EOF\n");
1006 data.error = SOLV_ERROR_EOF;
1013 dp = data_read_id_max(dp, &id, 0, numschemata, &data);
1016 incore_add_id(&data, 0); /* XXX? */
1017 incore_add_id(&data, id);
1018 keyp = schemadata + schemata[id];
1019 data.mainschema = id;
1020 for (i = 0; keyp[i]; i++)
1023 data.mainschemaoffsets = solv_calloc(i, sizeof(Id));
1031 /* make sure we have enough room */
1032 if (keydepth == 0 || needchunk)
1034 int left = bufend - dp;
1035 /* read data chunk to dp */
1040 pool_debug(pool, SOLV_ERROR, "buffer overrun\n");
1041 data.error = SOLV_ERROR_EOF;
1047 memmove(buf, dp, left);
1049 if (l < DATA_READ_CHUNK)
1050 l = DATA_READ_CHUNK;
1053 if (l && fread(buf + left, l, 1, data.fp) != 1)
1055 pool_debug(pool, SOLV_ERROR, "unexpected EOF\n");
1056 data.error = SOLV_ERROR_EOF;
1061 bufend = buf + left;
1062 if (allsize + left < maxsize)
1063 maxsize = allsize + left;
1071 printf("key %d at %d\n", key, (int)(keyp - 1 - schemadata));
1079 if (s && keydepth == 3)
1081 s++; /* next solvable */
1083 data.incoreoffset[(s - pool->solvables) - data.start] = data.incoredatalen;
1085 id = stack[keydepth - 1];
1088 dp = data_read_id_max(dp, &id, 0, numschemata, &data);
1089 incore_add_id(&data, id);
1091 keyp = schemadata + schemata[id];
1098 keyp = schemadata + stack[--keydepth];
1099 nentries = stack[--keydepth];
1101 printf("pop flexarray %d %d\n", keydepth, nentries);
1104 s = 0; /* back from solvables */
1109 data.mainschemaoffsets[keyp - 1 - (schemadata + schemata[data.mainschema])] = data.incoredatalen;
1112 printf("=> %s %s %p\n", pool_id2str(pool, keys[key].name), pool_id2str(pool, keys[key].type), s);
1114 id = keys[key].name;
1115 if (keys[key].storage == KEY_STORAGE_VERTICAL_OFFSET)
1118 dp = data_skip(dp, REPOKEY_TYPE_ID);
1119 dp = data_skip(dp, REPOKEY_TYPE_ID);
1120 incore_add_blob(&data, dps, dp - dps); /* just record offset/size */
1123 switch (keys[key].type)
1125 case REPOKEY_TYPE_ID:
1126 dp = data_read_id_max(dp, &did, idmap, numid + numrel, &data);
1127 if (s && id == SOLVABLE_NAME)
1129 else if (s && id == SOLVABLE_ARCH)
1131 else if (s && id == SOLVABLE_EVR)
1133 else if (s && id == SOLVABLE_VENDOR)
1135 else if (keys[key].storage == KEY_STORAGE_INCORE)
1136 incore_add_id(&data, did);
1138 POOL_DEBUG(SOLV_DEBUG_STATS, "%s -> %s\n", pool_id2str(pool, id), pool_id2str(pool, did));
1141 case REPOKEY_TYPE_IDARRAY:
1142 case REPOKEY_TYPE_REL_IDARRAY:
1143 if (!s || id < INTERESTED_START || id > INTERESTED_END)
1146 dp = data_skip(dp, REPOKEY_TYPE_IDARRAY);
1147 if (keys[key].storage != KEY_STORAGE_INCORE)
1150 incore_map_idarray(&data, dps, idmap, numid + numrel);
1152 incore_add_blob(&data, dps, dp - dps);
1155 ido = idarraydatap - repo->idarraydata;
1156 if (keys[key].type == REPOKEY_TYPE_IDARRAY)
1157 dp = data_read_idarray(dp, &idarraydatap, idmap, numid + numrel, &data);
1158 else if (id == SOLVABLE_REQUIRES)
1159 dp = data_read_rel_idarray(dp, &idarraydatap, idmap, numid + numrel, &data, SOLVABLE_PREREQMARKER);
1160 else if (id == SOLVABLE_PROVIDES)
1161 dp = data_read_rel_idarray(dp, &idarraydatap, idmap, numid + numrel, &data, SOLVABLE_FILEMARKER);
1163 dp = data_read_rel_idarray(dp, &idarraydatap, idmap, numid + numrel, &data, 0);
1164 if (idarraydatap > idarraydataend)
1166 pool_debug(pool, SOLV_ERROR, "idarray overflow\n");
1167 data.error = SOLV_ERROR_OVERFLOW;
1170 if (id == SOLVABLE_PROVIDES)
1172 else if (id == SOLVABLE_OBSOLETES)
1174 else if (id == SOLVABLE_CONFLICTS)
1176 else if (id == SOLVABLE_REQUIRES)
1178 else if (id == SOLVABLE_RECOMMENDS)
1180 else if (id == SOLVABLE_SUPPLEMENTS)
1181 s->supplements = ido;
1182 else if (id == SOLVABLE_SUGGESTS)
1184 else if (id == SOLVABLE_ENHANCES)
1187 POOL_DEBUG(SOLV_DEBUG_STATS, "%s ->\n", pool_id2str(pool, id));
1188 for (; repo->idarraydata[ido]; ido++)
1189 POOL_DEBUG(SOLV_DEBUG_STATS," %s\n", pool_dep2str(pool, repo->idarraydata[ido]));
1192 case REPOKEY_TYPE_FIXARRAY:
1193 case REPOKEY_TYPE_FLEXARRAY:
1196 if (keydepth == sizeof(stack)/sizeof(*stack))
1198 pool_debug(pool, SOLV_ERROR, "array stack overflow\n");
1199 data.error = SOLV_ERROR_CORRUPT;
1202 stack[keydepth++] = nentries;
1203 stack[keydepth++] = keyp - schemadata;
1204 stack[keydepth++] = 0;
1205 dp = data_read_id_max(dp, &nentries, 0, 0, &data);
1206 incore_add_id(&data, nentries);
1209 /* zero size array? */
1211 nentries = stack[--keydepth];
1214 if (keydepth == 3 && id == REPOSITORY_SOLVABLES)
1216 /* horray! here come the solvables */
1217 if (nentries != numsolv)
1219 pool_debug(pool, SOLV_ERROR, "inconsistent number of solvables: %d %d\n", nentries, numsolv);
1220 data.error = SOLV_ERROR_CORRUPT;
1225 pool_debug(pool, SOLV_ERROR, "more than one solvable block\n");
1226 data.error = SOLV_ERROR_CORRUPT;
1230 s = pool_id2solvable(pool, parent->start);
1232 s = pool_id2solvable(pool, repo_add_solvable_block(repo, numsolv));
1233 data.start = s - pool->solvables;
1234 data.end = data.start + numsolv;
1235 repodata_extend_block(&data, data.start, numsolv);
1236 for (i = 1; i < numkeys; i++)
1239 if ((keys[i].type == REPOKEY_TYPE_IDARRAY || keys[i].type == REPOKEY_TYPE_REL_IDARRAY)
1240 && id >= INTERESTED_START && id <= INTERESTED_END)
1241 size_idarray += keys[i].size;
1243 /* allocate needed space in repo */
1244 /* we add maxsize because it is an upper limit for all idarrays, thus we can't overflow */
1245 repo_reserve_ids(repo, 0, size_idarray + maxsize + 1);
1246 idarraydatap = repo->idarraydata + repo->idarraysize;
1247 repo->idarraysize += size_idarray;
1248 idarraydataend = idarraydatap + size_idarray;
1251 data.incoreoffset[(s - pool->solvables) - data.start] = data.incoredatalen;
1254 dp = data_read_id_max(dp, &id, 0, numschemata, &data);
1255 incore_add_id(&data, id);
1256 if (keys[key].type == REPOKEY_TYPE_FIXARRAY)
1260 pool_debug(pool, SOLV_ERROR, "illegal fixarray\n");
1261 data.error = SOLV_ERROR_CORRUPT;
1263 stack[keydepth - 1] = id;
1265 keyp = schemadata + schemata[id];
1267 case REPOKEY_TYPE_NUM:
1268 if (!(solvflags & SOLV_FLAG_SIZE_BYTES) && keys[key].storage == KEY_STORAGE_INCORE &&
1269 (id == SOLVABLE_INSTALLSIZE || id == SOLVABLE_DOWNLOADSIZE || id == DELTA_DOWNLOADSIZE))
1271 /* old solv file with sizes in kilos. transcode. */
1272 dp = data_read_id(dp, &id);
1273 incore_add_sizek(&data, (unsigned int)id);
1278 if (id == RPM_RPMDBID && s && (keys[key].type == REPOKEY_TYPE_U32 || keys[key].type == REPOKEY_TYPE_NUM))
1280 if (keys[key].type == REPOKEY_TYPE_U32)
1281 dp = data_read_u32(dp, (unsigned int *)&id);
1283 dp = data_read_id_max(dp, &id, 0, 0, &data);
1285 repo->rpmdbid = repo_sidedata_create(repo, sizeof(Id));
1286 repo->rpmdbid[(s - pool->solvables) - repo->start] = id;
1290 dp = data_skip(dp, keys[key].type);
1291 if (keys[key].storage == KEY_STORAGE_INCORE)
1292 incore_add_blob(&data, dps, dp - dps);
1296 /* should shrink idarraydata again */
1300 pool_debug(pool, SOLV_ERROR, "unexpected EOF, depth = %d\n", keydepth);
1301 data.error = SOLV_ERROR_CORRUPT;
1307 pool_debug(pool, SOLV_ERROR, "buffer overrun\n");
1308 data.error = SOLV_ERROR_EOF;
1315 /* free solvables */
1316 repo_free_solvable_block(repo, data.start, data.end - data.start, 1);
1318 repo->idarraysize -= size_idarray;
1319 /* free incore data */
1320 data.incoredata = solv_free(data.incoredata);
1321 data.incoredatalen = data.incoredatafree = 0;
1324 if (data.incoredatafree)
1326 /* shrink excess size */
1327 data.incoredata = solv_realloc(data.incoredata, data.incoredatalen);
1328 data.incoredatafree = 0;
1331 for (i = 1; i < numkeys; i++)
1332 if (keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
1334 if (i < numkeys && !data.error)
1337 unsigned int pagesize;
1339 /* we have vertical data, make it available */
1340 data.verticaloffset = solv_calloc(numkeys, sizeof(Id));
1341 for (i = 1; i < numkeys; i++)
1342 if (keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
1344 data.verticaloffset[i] = fileoffset;
1345 fileoffset += keys[i].size;
1347 data.lastverticaloffset = fileoffset;
1348 pagesize = read_u32(&data);
1349 data.error = repopagestore_read_or_setup_pages(&data.store, data.fp, pagesize, fileoffset);
1353 /* no longer needed */
1360 /* XXX: free repodata? */
1366 /* overwrite stub repodata */
1367 repodata_freedata(parent);
1368 data.repodataid = parent->repodataid;
1373 /* make it available as new repodata */
1374 if (!repo->nrepodata)
1376 repo->nrepodata = 1;
1377 repo->repodata = solv_calloc(2, sizeof(data));
1380 repo->repodata = solv_realloc2(repo->repodata, repo->nrepodata + 1, sizeof(data));
1381 data.repodataid = repo->nrepodata;
1382 repo->repodata[repo->nrepodata++] = data;
1385 /* create stub repodata entries for all external */
1386 if (!(flags & SOLV_ADD_NO_STUBS) && !parent)
1388 for (key = 1 ; key < data.nkeys; key++)
1389 if (data.keys[key].name == REPOSITORY_EXTERNAL && data.keys[key].type == REPOKEY_TYPE_FLEXARRAY)
1391 if (key < data.nkeys)
1392 repodata_create_stubs(repo->repodata + (repo->nrepodata - 1));
1395 POOL_DEBUG(SOLV_DEBUG_STATS, "repo_add_solv took %d ms\n", solv_timems(now));
1396 POOL_DEBUG(SOLV_DEBUG_STATS, "repo size: %d solvables\n", repo->nsolvables);
1397 POOL_DEBUG(SOLV_DEBUG_STATS, "repo memory used: %d K incore, %d K idarray\n", data.incoredatalen/1024, repo->idarraysize / (int)(1024/sizeof(Id)));