2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
26 #include "poolid_private.h"
31 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
32 unsigned char *out, unsigned int out_len);
33 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
36 unsigned int out_len);
38 #define REPODATA_BLOCK 255
42 repodata_init(Repodata *data, Repo *repo, int localpool)
44 memset(data, 0, sizeof (*data));
46 data->localpool = localpool;
48 stringpool_init_empty(&data->spool);
49 data->keys = sat_calloc(1, sizeof(Repokey));
51 data->schemata = sat_calloc(1, sizeof(Id));
52 data->schemadata = sat_calloc(1, sizeof(Id));
54 data->schemadatalen = 1;
55 data->start = repo->start;
56 data->end = repo->end;
57 data->nextra = repo->nextra;
59 data->incoreoffset = sat_extend_resize(0, data->end - data->start, sizeof(Id), REPODATA_BLOCK);
60 data->extraoffset = sat_extend_resize(0, repo->nextra, sizeof(Id), REPODATA_BLOCK);
65 repodata_free(Repodata *data)
68 sat_free(data->schemata);
69 sat_free(data->schemadata);
71 sat_free(data->spool.strings);
72 sat_free(data->spool.stringspace);
73 sat_free(data->spool.stringhashtbl);
75 sat_free(data->dirpool.dirs);
76 sat_free(data->dirpool.dirtraverse);
78 sat_free(data->incoredata);
79 sat_free(data->incoreoffset);
80 sat_free(data->extraoffset);
81 sat_free(data->verticaloffset);
83 sat_free(data->blob_store);
84 sat_free(data->pages);
85 sat_free(data->mapped);
87 sat_free(data->vincore);
89 sat_free(data->attrs);
90 sat_free(data->extraattrs);
91 sat_free(data->attrdata);
92 sat_free(data->attriddata);
94 sat_free(data->location);
95 sat_free(data->addedfileprovides);
97 if (data->pagefd != -1)
102 data_skip_recursive(Repodata *data, unsigned char *dp, Repokey *key)
105 if (key->type != REPOKEY_TYPE_COUNTED)
106 return data_skip(dp, key->type);
107 dp = data_fetch(dp, &kv, key);
112 Id *keyp = data->schemadata + data->schemata[schema];
113 for (; *keyp; keyp++)
114 dp = data_skip_recursive(data, dp, data->keys + *keyp);
119 static unsigned char *
120 forward_to_key(Repodata *data, Id keyid, Id schemaid, unsigned char *dp)
124 keyp = data->schemadata + data->schemata[schemaid];
125 while ((k = *keyp++) != 0)
129 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
131 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip that offset */
132 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip that length */
135 if (data->keys[k].storage != KEY_STORAGE_INCORE)
137 dp = data_skip_recursive(data, dp, data->keys + k);
142 #define BLOB_PAGEBITS 15
143 #define BLOB_PAGESIZE (1 << BLOB_PAGEBITS)
145 static unsigned char *
146 load_page_range(Repodata *data, unsigned int pstart, unsigned int pend)
148 /* Make sure all pages from PSTART to PEND (inclusive) are loaded,
149 and are consecutive. Return a pointer to the mapping of PSTART. */
150 unsigned char buf[BLOB_PAGESIZE];
153 /* Quick check in case all pages are there already and consecutive. */
154 for (i = pstart; i <= pend; i++)
155 if (data->pages[i].mapped_at == -1
157 && data->pages[i].mapped_at
158 != data->pages[i-1].mapped_at + BLOB_PAGESIZE))
161 return data->blob_store + data->pages[pstart].mapped_at;
163 if (data->pagefd == -1)
166 /* Ensure that we can map the numbers of pages we need at all. */
167 if (pend - pstart + 1 > data->ncanmap)
169 unsigned int oldcan = data->ncanmap;
170 data->ncanmap = pend - pstart + 1;
171 if (data->ncanmap < 4)
173 data->mapped = sat_realloc2(data->mapped, data->ncanmap, sizeof(data->mapped[0]));
174 memset (data->mapped + oldcan, 0, (data->ncanmap - oldcan) * sizeof (data->mapped[0]));
175 data->blob_store = sat_realloc2(data->blob_store, data->ncanmap, BLOB_PAGESIZE);
177 fprintf (stderr, "PAGE: can map %d pages\n", data->ncanmap);
181 /* Now search for "cheap" space in our store. Space is cheap if it's either
182 free (very cheap) or contains pages we search for anyway. */
184 /* Setup cost array. */
185 unsigned int cost[data->ncanmap];
186 for (i = 0; i < data->ncanmap; i++)
188 unsigned int pnum = data->mapped[i];
194 Attrblobpage *p = data->pages + pnum;
195 assert (p->mapped_at != -1);
196 if (pnum >= pstart && pnum <= pend)
203 /* And search for cheapest space. */
204 unsigned int best_cost = -1;
205 unsigned int best = 0;
206 unsigned int same_cost = 0;
207 for (i = 0; i + pend - pstart < data->ncanmap; i++)
209 unsigned int c = cost[i];
211 for (j = 0; j < pend - pstart + 1; j++)
214 best_cost = c, best = i;
215 else if (c == best_cost)
217 /* A null cost won't become better. */
221 /* If all places have the same cost we would thrash on slot 0. Avoid
222 this by doing a round-robin strategy in this case. */
223 if (same_cost == data->ncanmap - pend + pstart - 1)
224 best = data->rr_counter++ % (data->ncanmap - pend + pstart);
226 /* So we want to map our pages from [best] to [best+pend-pstart].
227 Use a very simple strategy, which doesn't make the best use of
228 our resources, but works. Throw away all pages in that range
229 (even ours) then copy around ours (in case they were outside the
230 range) or read them in. */
231 for (i = best; i < best + pend - pstart + 1; i++)
233 unsigned int pnum = data->mapped[i];
235 /* If this page is exactly at the right place already,
236 no need to evict it. */
237 && pnum != pstart + i - best)
239 /* Evict this page. */
241 fprintf (stderr, "PAGE: evict page %d from %d\n", pnum, i);
245 data->pages[pnum].mapped_at = -1;
249 /* Everything is free now. Read in the pages we want. */
250 for (i = pstart; i <= pend; i++)
252 Attrblobpage *p = data->pages + i;
253 unsigned int pnum = i - pstart + best;
254 void *dest = data->blob_store + pnum * BLOB_PAGESIZE;
255 if (p->mapped_at != -1)
257 if (p->mapped_at != pnum * BLOB_PAGESIZE)
260 fprintf (stderr, "PAGECOPY: %d to %d\n", i, pnum);
262 /* Still mapped somewhere else, so just copy it from there. */
263 memcpy (dest, data->blob_store + p->mapped_at, BLOB_PAGESIZE);
264 data->mapped[p->mapped_at / BLOB_PAGESIZE] = 0;
269 unsigned int in_len = p->file_size;
270 unsigned int compressed = in_len & 1;
273 fprintf (stderr, "PAGEIN: %d to %d", i, pnum);
275 if (pread(data->pagefd, compressed ? buf : dest, in_len, p->file_offset) != in_len)
277 perror ("mapping pread");
282 unsigned int out_len;
283 out_len = unchecked_decompress_buf(buf, in_len,
284 dest, BLOB_PAGESIZE);
285 if (out_len != BLOB_PAGESIZE && i < data->num_pages - 1)
287 fprintf(stderr, "can't decompress\n");
291 fprintf (stderr, " (expand %d to %d)", in_len, out_len);
295 fprintf (stderr, "\n");
298 p->mapped_at = pnum * BLOB_PAGESIZE;
299 data->mapped[pnum] = i + 1;
301 return data->blob_store + best * BLOB_PAGESIZE;
304 static unsigned char *
305 make_vertical_available(Repodata *data, Repokey *key, Id off, Id len)
310 if (off >= data->lastverticaloffset)
312 off -= data->lastverticaloffset;
313 if (off + len > data->vincorelen)
315 return data->vincore + off;
317 if (off + len > key->size)
319 /* we now have the offset, go into vertical */
320 off += data->verticaloffset[key - data->keys];
321 /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */
322 dp = load_page_range(data, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
324 dp += off % BLOB_PAGESIZE;
328 static inline unsigned char *
329 get_data(Repodata *data, Repokey *key, unsigned char **dpp)
331 unsigned char *dp = *dpp;
335 if (key->storage == KEY_STORAGE_INCORE)
337 /* hmm, this is a bit expensive */
338 *dpp = data_skip_recursive(data, dp, key);
341 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
344 dp = data_read_id(dp, &off);
345 dp = data_read_id(dp, &len);
347 return make_vertical_available(data, key, off, len);
353 maybe_load_repodata(Repodata *data, Id *keyid)
355 if (data->state == REPODATA_STUB)
357 if (data->loadcallback)
361 /* key order may change when loading */
363 Id name = data->keys[*keyid].name;
364 Id type = data->keys[*keyid].type;
365 data->loadcallback(data);
366 if (data->state == REPODATA_AVAILABLE)
368 for (i = 1; i < data->nkeys; i++)
369 if (data->keys[i].name == name && data->keys[i].type == type)
378 data->loadcallback(data);
381 data->state = REPODATA_ERROR;
383 if (data->state == REPODATA_AVAILABLE)
385 data->state = REPODATA_ERROR;
389 static inline unsigned char*
390 entry2data(Repodata *data, Id entry)
393 return data->incoredata + data->extraoffset[-1 - entry];
395 return data->incoredata + data->incoreoffset[entry];
399 repodata_lookup_id(Repodata *data, Id entry, Id keyid)
406 if (!maybe_load_repodata(data, &keyid))
408 dp = entry2data(data, entry);
411 dp = data_read_id(dp, &schema);
412 /* make sure the schema of this solvable contains the key */
413 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
416 dp = forward_to_key(data, keyid, schema, dp);
417 key = data->keys + keyid;
418 dp = get_data(data, key, &dp);
421 if (key->type == REPOKEY_TYPE_CONSTANTID)
423 if (key->type != REPOKEY_TYPE_ID)
425 dp = data_read_id(dp, &id);
430 repodata_lookup_str(Repodata *data, Id entry, Id keyid)
437 if (!maybe_load_repodata(data, &keyid))
440 dp = entry2data(data, entry);
443 dp = data_read_id(dp, &schema);
444 /* make sure the schema of this solvable contains the key */
445 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
448 dp = forward_to_key(data, keyid, schema, dp);
449 key = data->keys + keyid;
450 dp = get_data(data, key, &dp);
453 if (key->type == REPOKEY_TYPE_STR)
454 return (const char *)dp;
455 if (key->type == REPOKEY_TYPE_CONSTANTID)
456 return id2str(data->repo->pool, key->size);
457 if (key->type == REPOKEY_TYPE_ID)
458 dp = data_read_id(dp, &id);
462 return data->spool.stringspace + data->spool.strings[id];
463 return id2str(data->repo->pool, id);
467 repodata_lookup_num(Repodata *data, Id entry, Id keyid, unsigned int *value)
477 if (!maybe_load_repodata(data, &keyid))
480 dp = entry2data(data, entry);
483 dp = data_read_id(dp, &schema);
484 /* make sure the schema of this solvable contains the key */
485 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
488 dp = forward_to_key(data, keyid, schema, dp);
489 key = data->keys + keyid;
490 dp = get_data(data, key, &dp);
493 if (key->type == REPOKEY_TYPE_NUM
494 || key->type == REPOKEY_TYPE_U32
495 || key->type == REPOKEY_TYPE_CONSTANT)
497 dp = data_fetch(dp, &kv, key);
505 repodata_lookup_void(Repodata *data, Id entry, Id keyid)
510 if (!maybe_load_repodata(data, &keyid))
512 dp = entry2data(data, entry);
515 dp = data_read_id(dp, &schema);
516 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
522 const unsigned char *
523 repodata_lookup_bin_checksum(Repodata *data, Id entry, Id keyid, Id *typep)
530 if (!maybe_load_repodata(data, &keyid))
532 dp = entry2data(data, entry);
535 dp = data_read_id(dp, &schema);
536 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
539 dp = forward_to_key(data, keyid, schema, dp);
540 key = data->keys + keyid;
542 return get_data(data, key, &dp);
546 repodata_search(Repodata *data, Id entry, Id keyname, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
550 Id k, keyid, *kp, *keyp;
551 unsigned char *dp, *ddp;
557 || !maybe_load_repodata(data, 0))
560 dp = entry2data(data, entry);
563 dp = data_read_id(dp, &schema);
564 keyp = data->schemadata + data->schemata[schema];
567 /* search in a specific key */
568 for (kp = keyp; (k = *kp++) != 0; )
569 if (data->keys[k].name == keyname)
573 dp = forward_to_key(data, k, schema, dp);
579 while ((keyid = *keyp++) != 0)
582 key = data->keys + keyid;
583 ddp = get_data(data, key, &dp);
586 ddp = data_fetch(ddp, &kv, key);
589 if (key->type == REPOKEY_TYPE_COUNTED)
592 int subschema = kv.id;
593 Repokey *countkey = key;
595 callback(cbdata, data->repo->pool->solvables + data->start + entry, data, countkey, &kv);
598 Id *kp = data->schemadata + data->schemata[subschema];
601 key = data->keys + *kp;
602 ddp = data_fetch(ddp, &kv, key);
605 callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv);
608 callback(cbdata, data->repo->pool->solvables + data->start + entry, data, countkey, &kv);
611 stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, countkey, &kv);
614 stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv);
616 while (!kv.eof && !stop);
617 if (onekey || stop > SEARCH_NEXT_KEY)
623 dataiterator_newdata(Dataiterator *di)
625 Id keyname = di->keyname;
626 Repodata *data = di->data;
629 if (data->state == REPODATA_STUB)
634 for (j = 1; j < data->nkeys; j++)
635 if (keyname == data->keys[j].name)
637 if (j == data->nkeys)
641 if (data->loadcallback)
642 data->loadcallback(data);
644 data->state = REPODATA_ERROR;
646 if (data->state == REPODATA_ERROR)
650 unsigned char *dp = data->incoredata;
654 dp += data->incoreoffset[di->solvid - data->start];
656 dp += data->extraoffset[-1 - di->solvid - data->extrastart];
657 dp = data_read_id(dp, &schema);
658 Id *keyp = data->schemadata + data->schemata[schema];
662 /* search in a specific key */
663 for (kp = keyp; (k = *kp++) != 0; )
664 if (data->keys[k].name == keyname)
668 dp = forward_to_key(data, k, schema, dp);
678 di->key = di->data->keys + keyid;
683 di->dp = get_data(di->data, di->key, &di->nextkeydp);
688 dataiterator_init(Dataiterator *di, Repo *repo, Id p, Id keyname,
689 const char *match, int flags)
695 di->flags |= __SEARCH_ONESOLVABLE;
696 di->data = repo->repodata - 1;
697 if (flags & SEARCH_NO_STORAGE_SOLVABLE)
704 di->solvid = repo->start - 1;
707 fprintf(stderr, "A repo contains the NULL solvable!\n");
710 di->data = repo->repodata + repo->nrepodata - 1;
715 if ((di->flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
719 /* We feed multiple lines eventually (e.g. authors or descriptions),
720 so set REG_NEWLINE. */
722 regcomp(&di->regex, di->match,
723 REG_EXTENDED | REG_NOSUB | REG_NEWLINE
724 | ((di->flags & SEARCH_NOCASE) ? REG_ICASE : 0));
726 if (di->regex_err != 0)
728 fprintf(stderr, "Given regex failed to compile: %s\n", di->match);
729 fprintf(stderr, "regcomp error code: %d\n", di->regex_err);
736 di->flags |= (di->flags & SEARCH_STRINGMASK) | SEARCH_STRING;
742 di->keyname = keyname;
743 static Id zeroid = 0;
751 /* FIXME factor and merge with repo_matchvalue */
753 dataiterator_match_int_real(Dataiterator *di, int flags, const void *vmatch)
755 KeyValue *kv = &di->kv;
756 const char *match = vmatch;
757 if ((flags & SEARCH_STRINGMASK) != 0)
759 switch (di->key->type)
761 case REPOKEY_TYPE_ID:
762 case REPOKEY_TYPE_IDARRAY:
763 if (di->data && di->data->localpool)
764 kv->str = stringpool_id2str(&di->data->spool, kv->id);
766 kv->str = id2str(di->repo->pool, kv->id);
768 case REPOKEY_TYPE_STR:
770 case REPOKEY_TYPE_DIRSTRARRAY:
771 if (!(flags & SEARCH_FILES))
773 /* Put the full filename into kv->str. */
774 kv->str = repodata_dir2str(di->data, kv->id, kv->str);
775 /* And to compensate for that put the "empty" directory into
776 kv->id, so that later calls to repodata_dir2str on this data
777 come up with the same filename again. */
783 /* Maybe skip the kind specifier. Do this only for SOLVABLE attributes,
784 for the others we can't know if a colon separates a kind or not. */
785 if ((flags & SEARCH_SKIP_KIND)
786 && di->key->storage == KEY_STORAGE_SOLVABLE)
788 const char *s = strchr(kv->str, ':');
792 switch ((flags & SEARCH_STRINGMASK))
794 case SEARCH_SUBSTRING:
795 if (flags & SEARCH_NOCASE)
797 if (!strcasestr(kv->str, match))
802 if (!strstr(kv->str, match))
807 if (flags & SEARCH_NOCASE)
809 if (strcasecmp(match, kv->str))
814 if (strcmp(match, kv->str))
819 if (fnmatch(match, kv->str, (flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
823 if (regexec((const regex_t *)vmatch, kv->str, 0, NULL, 0))
834 dataiterator_match_int(Dataiterator *di)
836 if ((di->flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
837 return dataiterator_match_int_real(di, di->flags, &di->regex);
839 return dataiterator_match_int_real(di, di->flags, di->match);
843 dataiterator_match(Dataiterator *di, int flags, const void *vmatch)
845 return dataiterator_match_int_real(di, flags, vmatch);
848 static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = {
849 { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
850 { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
851 { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
852 { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
853 { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
854 { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
855 { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
856 { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
857 { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
858 { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
859 { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
860 { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
861 { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE },
865 dataiterator_step(Dataiterator *di)
879 di->kv.eof = idp[1] ? 0 : 1;
885 Solvable *s = di->repo->pool->solvables + di->solvid;
886 int state = di->state;
887 di->key = solvablekeys + state - 1;
889 di->state = RPM_RPMDBID;
896 state = di->keyname - 1;
918 case SOLVABLE_VENDOR:
921 di->kv.id = s->vendor;
924 case SOLVABLE_PROVIDES:
925 di->idp = s->provides
926 ? di->repo->idarraydata + s->provides : 0;
928 case SOLVABLE_OBSOLETES:
929 di->idp = s->obsoletes
930 ? di->repo->idarraydata + s->obsoletes : 0;
932 case SOLVABLE_CONFLICTS:
933 di->idp = s->conflicts
934 ? di->repo->idarraydata + s->conflicts : 0;
936 case SOLVABLE_REQUIRES:
937 di->idp = s->requires
938 ? di->repo->idarraydata + s->requires : 0;
940 case SOLVABLE_RECOMMENDS:
941 di->idp = s->recommends
942 ? di->repo->idarraydata + s->recommends : 0;
944 case SOLVABLE_SUPPLEMENTS:
945 di->idp = s->supplements
946 ? di->repo->idarraydata + s->supplements : 0;
948 case SOLVABLE_SUGGESTS:
949 di->idp = s->suggests
950 ? di->repo->idarraydata + s->suggests : 0;
952 case SOLVABLE_ENHANCES:
953 di->idp = s->enhances
954 ? di->repo->idarraydata + s->enhances : 0;
957 if (!di->repo->rpmdbid)
959 di->kv.num = di->repo->rpmdbid[di->solvid - di->repo->start];
963 di->data = di->repo->repodata - 1;
969 else if (di->subkeyp)
974 /* Send end-of-substruct. We are here only when we saw a
975 _COUNTED key one level up. Since then we didn't increment
976 ->keyp, so it still can be found at keyp[-1]. */
978 di->key = di->data->keys + di->keyp[-1];
981 else if (!(keyid = *di->subkeyp++))
983 /* Send end-of-element. See above for keyp[-1]. */
985 di->key = di->data->keys + di->keyp[-1];
986 di->subkeyp = di->data->schemadata + di->data->schemata[di->subschema];
991 di->key = di->data->keys + keyid;
992 di->dp = data_fetch(di->dp, &di->kv, di->key);
1002 di->dp = data_fetch(di->dp, &di->kv, di->key);
1007 if (di->keyname || !(keyid = *di->keyp++))
1011 Repo *repo = di->repo;
1012 Repodata *data = ++di->data;
1013 if (data >= repo->repodata + repo->nrepodata)
1015 if (di->flags & __SEARCH_ONESOLVABLE)
1017 if (di->solvid >= 0)
1019 while (++di->solvid < repo->end)
1020 if (repo->pool->solvables[di->solvid].repo == repo)
1022 if (di->solvid >= repo->end)
1024 if (!(di->flags & SEARCH_EXTRA))
1027 if (di->solvid < -repo->nextra)
1034 if (di->solvid < -repo->nextra)
1037 Pool *pool = di->repo->pool;
1038 if (!(di->flags & SEARCH_ALL_REPOS)
1039 || di->repo == pool->repos[pool->nrepos - 1])
1042 for (i = 0; i < pool->nrepos; i++)
1043 if (di->repo == pool->repos[i])
1045 di->repo = pool->repos[i + 1];
1046 dataiterator_init(di, di->repo, 0, di->keyname, di->match, di->flags);
1050 di->data = repo->repodata - 1;
1052 || (di->flags & SEARCH_NO_STORAGE_SOLVABLE))
1054 static Id zeroid = 0;
1059 if ((di->solvid < 0 && (-1 - di->solvid) >= data->extrastart && (-1 - di->solvid) < (data->extrastart + data->nextra))
1060 || (di->solvid >= 0 && di->solvid >= data->start && di->solvid < data->end))
1062 dataiterator_newdata(di);
1070 di->key = di->data->keys + keyid;
1071 di->dp = get_data(di->data, di->key, &di->nextkeydp);
1073 di->dp = data_fetch(di->dp, &di->kv, di->key);
1075 if (di->key->type == REPOKEY_TYPE_COUNTED)
1077 di->subnum = di->kv.num;
1078 di->subschema = di->kv.id;
1080 di->subkeyp = di->data->schemadata + di->data->schemata[di->subschema];
1085 || dataiterator_match_int(di))
1092 dataiterator_skip_attribute(Dataiterator *di)
1096 /* This will make the next _step call to retrieve the next field. */
1101 dataiterator_skip_solvable(Dataiterator *di)
1103 /* We're done with this field. */
1105 /* And with solvable data. */
1107 /* And with all keys for this repodata and thing. */
1108 static Id zeroid = 0;
1110 /* And with all repodatas for this thing. */
1111 di->data = di->repo->repodata + di->repo->nrepodata - 1;
1112 /* Hence the next call to _step will retrieve the next thing. */
1116 dataiterator_skip_repo(Dataiterator *di)
1118 dataiterator_skip_solvable(di);
1119 /* We're done with all solvables and all extra things for this repo. */
1120 di->solvid = -1 - di->repo->nextra;
1124 dataiterator_jump_to_solvable(Dataiterator *di, Solvable *s)
1127 /* Simulate us being done with the solvable before the requested one. */
1128 dataiterator_skip_solvable(di);
1129 di->solvid = s - s->repo->pool->solvables;
1134 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1137 dataiterator_skip_solvable(di);
1138 di->solvid = repo->start - 1;
1141 /* extend repodata so that it includes solvables p */
1143 repodata_extend(Repodata *data, Id p)
1145 if (data->start == data->end)
1146 data->start = data->end = p;
1149 int old = data->end - data->start;
1150 int new = p - data->end + 1;
1153 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id), REPODATA_BLOCK);
1154 memset(data->attrs + old, 0, new * sizeof(Id));
1156 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
1157 memset(data->incoreoffset + old, 0, new * sizeof(Id));
1160 if (p < data->start)
1162 int old = data->end - data->start;
1163 int new = data->start - p;
1166 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id), REPODATA_BLOCK);
1167 memmove(data->attrs + new, data->attrs, old * sizeof(Id));
1168 memset(data->attrs, 0, new * sizeof(Id));
1170 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
1171 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
1172 memset(data->incoreoffset, 0, new * sizeof(Id));
1178 repodata_extend_extra(Repodata *data, int nextra)
1180 if (nextra <= data->nextra)
1182 if (data->extraattrs)
1184 data->extraattrs = sat_extend(data->extraattrs, data->nextra, nextra - data->nextra, sizeof(Id), REPODATA_BLOCK);
1185 memset(data->extraattrs + data->nextra, 0, (nextra - data->nextra) * sizeof (Id));
1187 data->extraoffset = sat_extend(data->extraoffset, data->nextra, nextra - data->nextra, sizeof(Id), REPODATA_BLOCK);
1188 memset(data->extraoffset + data->nextra, 0, (nextra - data->nextra) * sizeof(Id));
1189 data->nextra = nextra;
1193 repodata_extend_block(Repodata *data, Id start, Id num)
1197 if (!data->incoreoffset)
1199 data->incoreoffset = sat_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
1200 data->start = start;
1201 data->end = start + num;
1204 repodata_extend(data, start);
1206 repodata_extend(data, start + num - 1);
1209 /**********************************************************************/
1211 #define REPODATA_ATTRS_BLOCK 63
1212 #define REPODATA_ATTRDATA_BLOCK 1023
1213 #define REPODATA_ATTRIDDATA_BLOCK 63
1216 get_new_struct(Repodata *data)
1218 /* Make sure to never give out struct id 0. */
1221 data->structs = sat_extend(0, 0, 2, sizeof(Id *), REPODATA_BLOCK);
1222 data->structs[0] = 0;
1223 data->structs[1] = 0;
1227 data->structs = sat_extend(data->structs, data->nstructs, 1, sizeof(Id *), REPODATA_BLOCK);
1228 data->structs[data->nstructs] = 0;
1229 return data->nstructs++;
1233 repodata_get_handle_int(Repodata *data, Id entry)
1236 if (!data->attrs && entry >= 0)
1238 data->attrs = sat_calloc_block(data->end - data->start, sizeof(Id),
1241 else if (!data->extraattrs && entry < 0)
1242 data->extraattrs = sat_calloc_block(data->nextra, sizeof(Id), REPODATA_BLOCK);
1244 ap = &data->extraattrs[-1 - entry];
1246 ap = &data->attrs[entry];
1248 *ap = get_new_struct(data);
1253 repodata_get_handle(Repodata *data, Id entry)
1255 return repodata_get_handle_int(data, entry);
1259 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
1264 ap = data->structs[handle];
1268 for (pp = ap; *pp; pp += 2)
1269 /* Determine equality based on the name only, allows us to change
1270 type (when overwrite is set), and makes TYPE_CONSTANT work. */
1271 if (data->keys[*pp].name == data->keys[keyid].name)
1284 ap = sat_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
1285 data->structs[handle] = ap;
1293 repodata_set(Repodata *data, Id handle, Repokey *key, Id val)
1297 /* find key in keys */
1298 for (keyid = 1; keyid < data->nkeys; keyid++)
1299 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
1301 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
1305 if (keyid == data->nkeys)
1307 /* allocate new key */
1308 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
1309 data->keys[data->nkeys++] = *key;
1310 if (data->verticaloffset)
1312 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
1313 data->verticaloffset[data->nkeys - 1] = 0;
1316 repodata_insert_keyid(data, handle, keyid, val, 1);
1320 repodata_set_id(Repodata *data, Id handle, Id keyname, Id id)
1324 key.type = REPOKEY_TYPE_ID;
1326 key.storage = KEY_STORAGE_INCORE;
1327 repodata_set(data, handle, &key, id);
1331 repodata_set_num(Repodata *data, Id handle, Id keyname, unsigned int num)
1335 key.type = REPOKEY_TYPE_NUM;
1337 key.storage = KEY_STORAGE_INCORE;
1338 repodata_set(data, handle, &key, (Id)num);
1342 repodata_set_poolstr(Repodata *data, Id handle, Id keyname, const char *str)
1346 if (data->localpool)
1347 id = stringpool_str2id(&data->spool, str, 1);
1349 id = str2id(data->repo->pool, str, 1);
1351 key.type = REPOKEY_TYPE_ID;
1353 key.storage = KEY_STORAGE_INCORE;
1354 repodata_set(data, handle, &key, id);
1358 repodata_set_constant(Repodata *data, Id handle, Id keyname, unsigned int constant)
1362 key.type = REPOKEY_TYPE_CONSTANT;
1363 key.size = constant;
1364 key.storage = KEY_STORAGE_INCORE;
1365 repodata_set(data, handle, &key, 0);
1369 repodata_set_constantid(Repodata *data, Id handle, Id keyname, Id id)
1373 key.type = REPOKEY_TYPE_CONSTANTID;
1375 key.storage = KEY_STORAGE_INCORE;
1376 repodata_set(data, handle, &key, 0);
1380 repodata_set_void(Repodata *data, Id handle, Id keyname)
1384 key.type = REPOKEY_TYPE_VOID;
1386 key.storage = KEY_STORAGE_INCORE;
1387 repodata_set(data, handle, &key, 0);
1391 repodata_set_str(Repodata *data, Id handle, Id keyname, const char *str)
1396 l = strlen(str) + 1;
1398 key.type = REPOKEY_TYPE_STR;
1400 key.storage = KEY_STORAGE_INCORE;
1401 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1402 memcpy(data->attrdata + data->attrdatalen, str, l);
1403 repodata_set(data, handle, &key, data->attrdatalen);
1404 data->attrdatalen += l;
1408 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
1413 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
1415 /* great! just append the new data */
1416 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1417 data->attriddatalen--; /* overwrite terminating 0 */
1418 data->lastdatalen += entrysize;
1421 pp = data->structs[handle];
1423 for (; *pp; pp += 2)
1424 if (data->keys[*pp].name == keyname && data->keys[*pp].type == keytype)
1428 /* not found. allocate new key */
1433 key.storage = KEY_STORAGE_INCORE;
1434 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1435 repodata_set(data, handle, &key, data->attriddatalen);
1436 data->lasthandle = 0; /* next time... */
1440 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
1441 oldsize += entrysize;
1442 if (ida + 1 == data->attriddata + data->attriddatalen)
1444 /* this was the last entry, just append it */
1445 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1446 data->attriddatalen--; /* overwrite terminating 0 */
1450 /* too bad. move to back. */
1451 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1452 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
1453 pp[1] = data->attriddatalen;
1454 data->attriddatalen += oldsize;
1456 data->lasthandle = handle;
1457 data->lastkey = *pp;
1458 data->lastdatalen = data->attriddatalen + entrysize + 1;
1462 checksumtype2len(Id type)
1466 case REPOKEY_TYPE_MD5:
1468 case REPOKEY_TYPE_SHA1:
1470 case REPOKEY_TYPE_SHA256:
1471 return SIZEOF_SHA256;
1478 repodata_set_bin_checksum(Repodata *data, Id handle, Id keyname, Id type,
1479 const unsigned char *str)
1482 int l = checksumtype2len(type);
1489 key.storage = KEY_STORAGE_INCORE;
1490 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1491 memcpy(data->attrdata + data->attrdatalen, str, l);
1492 repodata_set(data, handle, &key, data->attrdatalen);
1493 data->attrdatalen += l;
1497 hexstr2bytes(unsigned char *buf, const char *str, int buflen)
1500 for (i = 0; i < buflen; i++)
1502 #define c2h(c) (((c)>='0' && (c)<='9') ? ((c)-'0') \
1503 : ((c)>='a' && (c)<='f') ? ((c)-'a'+10) \
1504 : ((c)>='A' && (c)<='F') ? ((c)-'A'+10) \
1515 buf[i] = (buf[i] << 4) | v;
1522 repodata_set_checksum(Repodata *data, Id handle, Id keyname, Id type,
1525 unsigned char buf[64];
1526 int l = checksumtype2len(type);
1530 if (hexstr2bytes(buf, str, l) != l)
1532 fprintf(stderr, "Invalid hex character in '%s'\n", str);
1535 repodata_set_bin_checksum(data, handle, keyname, type, buf);
1539 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
1544 l = checksumtype2len(type);
1547 s = str = pool_alloctmpspace(data->repo->pool, 2 * l + 1);
1548 for (i = 0; i < l; i++)
1550 unsigned char v = buf[i];
1551 unsigned char w = v >> 4;
1552 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
1554 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
1561 repodata_globalize_id(Repodata *data, Id id)
1563 if (!data || !data->localpool)
1565 return str2id(data->repo->pool, stringpool_id2str(&data->spool, id), 1);
1569 repodata_add_dirnumnum(Repodata *data, Id handle, Id keyname, Id dir, Id num, Id num2)
1573 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", handle, dir, num, num2, data->attriddatalen);
1575 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
1576 data->attriddata[data->attriddatalen++] = dir;
1577 data->attriddata[data->attriddatalen++] = num;
1578 data->attriddata[data->attriddatalen++] = num2;
1579 data->attriddata[data->attriddatalen++] = 0;
1583 repodata_add_dirstr(Repodata *data, Id handle, Id keyname, Id dir, const char *str)
1589 l = strlen(str) + 1;
1590 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1591 memcpy(data->attrdata + data->attrdatalen, str, l);
1592 stroff = data->attrdatalen;
1593 data->attrdatalen += l;
1596 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", handle, dir, str, data->attriddatalen);
1598 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
1599 data->attriddata[data->attriddatalen++] = dir;
1600 data->attriddata[data->attriddatalen++] = stroff;
1601 data->attriddata[data->attriddatalen++] = 0;
1605 repodata_add_idarray(Repodata *data, Id handle, Id keyname, Id id)
1608 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", handle, id, data->attriddatalen);
1610 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_IDARRAY, 1);
1611 data->attriddata[data->attriddatalen++] = id;
1612 data->attriddata[data->attriddatalen++] = 0;
1616 repodata_add_poolstr_array(Repodata *data, Id handle, Id keyname,
1620 if (data->localpool)
1621 id = stringpool_str2id(&data->spool, str, 1);
1623 id = str2id(data->repo->pool, str, 1);
1624 repodata_add_idarray(data, handle, keyname, id);
1628 repodata_create_struct(Repodata *data, Id handle, Id keyname)
1630 Id newhandle = get_new_struct(data);
1631 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_COUNTED, 1);
1632 data->attriddata[data->attriddatalen++] = newhandle;
1633 data->attriddata[data->attriddatalen++] = 0;
1638 repodata_merge_attrs(Repodata *data, Id dest, Id src)
1642 || !(keyp = data->structs[src < 0
1643 ? data->extraattrs[-1 - src]
1644 : data->attrs[src]]))
1646 dest = repodata_get_handle_int(data, dest);
1647 for (; *keyp; keyp += 2)
1648 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
1651 /*********************************/
1653 /* unify with repo_write! */
1655 #define EXTDATA_BLOCK 1023
1656 #define SCHEMATA_BLOCK 31
1657 #define SCHEMATADATA_BLOCK 255
1665 data_addid(struct extdata *xd, Id x)
1668 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
1669 dp = xd->buf + xd->len;
1674 *dp++ = (x >> 28) | 128;
1676 *dp++ = (x >> 21) | 128;
1677 *dp++ = (x >> 14) | 128;
1680 *dp++ = (x >> 7) | 128;
1682 xd->len = dp - xd->buf;
1686 data_addideof(struct extdata *xd, Id x, int eof)
1689 x = (x & 63) | ((x & ~63) << 1);
1690 data_addid(xd, (eof ? x: x | 64));
1694 data_addblob(struct extdata *xd, unsigned char *blob, int len)
1696 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
1697 memcpy(xd->buf + xd->len, blob, len);
1701 /*********************************/
1704 addschema_prepare(Repodata *data, Id *schematacache)
1709 memset(schematacache, 0, 256 * sizeof(Id));
1710 for (i = 0; i < data->nschemata; i++)
1712 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
1715 schematacache[h] = i + 1;
1717 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
1718 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
1722 addschema(Repodata *data, Id *schema, Id *schematacache)
1727 for (sp = schema, len = 0, h = 0; *sp; len++)
1732 cid = schematacache[h];
1736 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1738 /* cache conflict */
1739 for (cid = 0; cid < data->nschemata; cid++)
1740 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1743 /* a new one. make room. */
1744 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
1745 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
1747 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
1748 data->schemata[data->nschemata] = data->schemadatalen;
1749 data->schemadatalen += len;
1750 schematacache[h] = data->nschemata + 1;
1752 fprintf(stderr, "addschema: new schema\n");
1754 return data->nschemata++;
1758 repodata_serialize_key(Repodata *data, struct extdata *newincore,
1759 struct extdata *newvincore,
1760 Id *schema, Id *schematacache,
1761 Repokey *key, Id val)
1763 /* Otherwise we have a new value. Parse it into the internal
1767 unsigned int oldvincorelen = 0;
1771 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1774 oldvincorelen = xd->len;
1778 case REPOKEY_TYPE_VOID:
1779 case REPOKEY_TYPE_CONSTANT:
1780 case REPOKEY_TYPE_CONSTANTID:
1782 case REPOKEY_TYPE_STR:
1783 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
1785 case REPOKEY_TYPE_MD5:
1786 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
1788 case REPOKEY_TYPE_SHA1:
1789 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
1791 case REPOKEY_TYPE_ID:
1792 case REPOKEY_TYPE_NUM:
1793 case REPOKEY_TYPE_DIR:
1794 data_addid(xd, val);
1796 case REPOKEY_TYPE_IDARRAY:
1797 for (ida = data->attriddata + val; *ida; ida++)
1798 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
1800 case REPOKEY_TYPE_DIRNUMNUMARRAY:
1801 for (ida = data->attriddata + val; *ida; ida += 3)
1803 data_addid(xd, ida[0]);
1804 data_addid(xd, ida[1]);
1805 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
1808 case REPOKEY_TYPE_DIRSTRARRAY:
1809 for (ida = data->attriddata + val; *ida; ida += 2)
1811 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
1812 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
1815 case REPOKEY_TYPE_COUNTED:
1819 for (ida = data->attriddata + val; *ida; ida++)
1822 fprintf(stderr, "serialize struct %d\n", *ida);
1825 Id *kp = data->structs[*ida];
1832 fprintf(stderr, " %s:%d\n", id2str(data->repo->pool, data->keys[*kp].name), kp[1]);
1838 schemaid = addschema(data, schema, schematacache);
1839 else if (schemaid != addschema(data, schema, schematacache))
1841 fprintf(stderr, " not yet implemented: substructs with different schemas\n");
1845 fprintf(stderr, " schema %d\n", schemaid);
1850 data_addid(xd, num);
1851 data_addid(xd, schemaid);
1852 for (ida = data->attriddata + val; *ida; ida++)
1854 Id *kp = data->structs[*ida];
1859 repodata_serialize_key(data, newincore, newvincore,
1860 schema, schematacache,
1861 data->keys + *kp, kp[1]);
1867 fprintf(stderr, "don't know how to handle type %d\n", key->type);
1870 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1872 /* put offset/len in incore */
1873 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
1874 oldvincorelen = xd->len - oldvincorelen;
1875 data_addid(newincore, oldvincorelen);
1880 repodata_internalize(Repodata *data)
1884 Id schematacache[256];
1885 Id schemaid, *schema, *sp, oldschema, *keyp, *seen;
1886 unsigned char *dp, *ndp;
1887 int newschema, oldcount;
1888 struct extdata newincore;
1889 struct extdata newvincore;
1891 if (!data->attrs && !data->extraattrs)
1894 newvincore.buf = data->vincore;
1895 newvincore.len = data->vincorelen;
1897 schema = sat_malloc2(data->nkeys, sizeof(Id));
1898 seen = sat_malloc2(data->nkeys, sizeof(Id));
1900 /* Merge the data already existing (in data->schemata, ->incoredata and
1901 friends) with the new attributes in data->attrs[]. */
1902 nentry = data->end - data->start;
1903 addschema_prepare(data, schematacache);
1904 memset(&newincore, 0, sizeof(newincore));
1905 data_addid(&newincore, 0);
1908 for (entry = data->extraattrs ? -data->nextra : 0; entry < nentry; entry++)
1911 memset(seen, 0, data->nkeys * sizeof(Id));
1913 dp = entry2data(data, entry);
1914 if (data->incoredata)
1915 dp = data_read_id(dp, &oldschema);
1919 fprintf(stderr, "oldschema %d\n", oldschema);
1920 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
1921 fprintf(stderr, "schemadata %p\n", data->schemadata);
1923 /* seen: -1: old data 0: skipped >0: id + 1 */
1926 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
1930 fprintf(stderr, "Inconsistent old data (key occured twice).\n");
1937 handle = entry < 0 ? data->extraattrs[-1 - entry] : data->attrs[entry];
1938 keyp = data->structs[handle];
1940 for (; *keyp; keyp += 2)
1947 seen[*keyp] = keyp[1] + 1;
1951 /* Ideally we'd like to sort the new schema here, to ensure
1952 schema equality independend of the ordering. We can't do that
1953 yet. For once see below (old ids need to come before new ids).
1954 An additional difficulty is that we also need to move
1955 the values with the keys. */
1956 schemaid = addschema(data, schema, schematacache);
1958 schemaid = oldschema;
1961 /* Now create data blob. We walk through the (possibly new) schema
1962 and either copy over old data, or insert the new. */
1963 /* XXX Here we rely on the fact that the (new) schema has the form
1964 o1 o2 o3 o4 ... | n1 n2 n3 ...
1965 (oX being the old keyids (possibly overwritten), and nX being
1966 the new keyids). This rules out sorting the keyids in order
1967 to ensure a small schema count. */
1969 data->extraoffset[-1 - entry] = newincore.len;
1971 data->incoreoffset[entry] = newincore.len;
1972 data_addid(&newincore, schemaid);
1973 for (keyp = data->schemadata + data->schemata[schemaid]; *keyp; keyp++)
1975 key = data->keys + *keyp;
1977 fprintf(stderr, "internalize %d:%s:%s\n", entry, id2str(data->repo->pool, key->name), id2str(data->repo->pool, key->type));
1982 /* Skip the data associated with this old key. */
1983 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1985 ndp = data_skip(dp, REPOKEY_TYPE_ID);
1986 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
1988 else if (key->storage == KEY_STORAGE_INCORE)
1989 ndp = data_skip_recursive(data, dp, key);
1992 if (seen[*keyp] == -1)
1994 /* If this key was an old one _and_ was not overwritten with
1995 a different value copy over the old value (we skipped it
1998 data_addblob(&newincore, dp, ndp - dp);
2001 else if (seen[*keyp])
2003 /* Otherwise we have a new value. Parse it into the internal
2005 repodata_serialize_key(data, &newincore, &newvincore,
2006 schema, schematacache,
2007 key, seen[*keyp] - 1);
2011 if (data->structs[handle])
2012 data->structs[handle] = sat_free(data->structs[handle]);
2014 for (entry = 0; entry < data->nstructs; entry++)
2015 if (data->structs[entry])
2016 sat_free(data->structs[entry]);
2017 data->structs = sat_free(data->structs);
2018 data->lasthandle = 0;
2020 data->lastdatalen = 0;
2024 sat_free(data->incoredata);
2025 data->incoredata = newincore.buf;
2026 data->incoredatalen = newincore.len;
2027 data->incoredatafree = 0;
2029 sat_free(data->vincore);
2030 data->vincore = newvincore.buf;
2031 data->vincorelen = newvincore.len;
2033 data->attrs = sat_free(data->attrs);
2034 data->extraattrs = sat_free(data->extraattrs);
2035 data->attrdata = sat_free(data->attrdata);
2036 data->attriddata = sat_free(data->attriddata);
2037 data->attrdatalen = 0;
2038 data->attriddatalen = 0;
2042 repodata_str2dir(Repodata *data, const char *dir, int create)
2048 while (*dir == '/' && dir[1] == '/')
2050 if (*dir == '/' && !dir[1])
2054 dire = strchrnul(dir, '/');
2055 if (data->localpool)
2056 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
2058 id = strn2id(data->repo->pool, dir, dire - dir, create);
2061 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
2074 repodata_dir2str(Repodata *data, Id did, const char *suf)
2076 Pool *pool = data->repo->pool;
2083 return suf ? suf : "";
2087 comp = dirpool_compid(&data->dirpool, parent);
2088 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
2090 parent = dirpool_parent(&data->dirpool, parent);
2095 l += strlen(suf) + 1;
2096 p = pool_alloctmpspace(pool, l + 1) + l;
2107 comp = dirpool_compid(&data->dirpool, parent);
2108 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
2111 strncpy(p, comps, l);
2112 parent = dirpool_parent(&data->dirpool, parent);
2120 repodata_compress_page(unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max)
2122 return compress_buf(page, len, cpage, max);
2125 #define SOLV_ERROR_EOF 3
2127 static inline unsigned int
2133 for (i = 0; i < 4; i++)
2143 #define SOLV_ERROR_EOF 3
2144 #define SOLV_ERROR_CORRUPT 6
2146 /* Try to either setup on-demand paging (using FP as backing
2147 file), or in case that doesn't work (FP not seekable) slurps in
2148 all pages and deactivates paging. */
2150 repodata_read_or_setup_pages(Repodata *data, unsigned int pagesz, unsigned int blobsz)
2152 FILE *fp = data->fp;
2153 unsigned int npages;
2155 unsigned int can_seek;
2157 unsigned char buf[BLOB_PAGESIZE];
2159 if (pagesz != BLOB_PAGESIZE)
2161 /* We could handle this by slurping in everything. */
2162 data->error = SOLV_ERROR_CORRUPT;
2166 if ((cur_file_ofs = ftell(fp)) < 0)
2170 data->pagefd = dup(fileno(fp));
2171 if (data->pagefd == -1)
2175 fprintf (stderr, "can %sseek\n", can_seek ? "" : "NOT ");
2177 npages = (blobsz + BLOB_PAGESIZE - 1) / BLOB_PAGESIZE;
2179 data->num_pages = npages;
2180 data->pages = sat_malloc2(npages, sizeof(data->pages[0]));
2182 /* If we can't seek on our input we have to slurp in everything. */
2184 data->blob_store = sat_malloc(npages * BLOB_PAGESIZE);
2185 for (i = 0; i < npages; i++)
2187 unsigned int in_len = read_u32(fp);
2188 unsigned int compressed = in_len & 1;
2189 Attrblobpage *p = data->pages + i;
2192 fprintf (stderr, "page %d: len %d (%scompressed)\n",
2193 i, in_len, compressed ? "" : "not ");
2199 p->file_offset = cur_file_ofs;
2200 p->file_size = in_len * 2 + compressed;
2201 if (fseek(fp, in_len, SEEK_CUR) < 0)
2204 fprintf (stderr, "can't seek after we thought we can\n");
2205 /* We can't fall back to non-seeking behaviour as we already
2206 read over some data pages without storing them away. */
2207 data->error = SOLV_ERROR_EOF;
2208 close(data->pagefd);
2212 cur_file_ofs += in_len;
2216 unsigned int out_len;
2217 void *dest = data->blob_store + i * BLOB_PAGESIZE;
2218 p->mapped_at = i * BLOB_PAGESIZE;
2221 /* We can't seek, so suck everything in. */
2222 if (fread(compressed ? buf : dest, in_len, 1, fp) != 1)
2225 data->error = SOLV_ERROR_EOF;
2230 out_len = unchecked_decompress_buf(buf, in_len, dest, BLOB_PAGESIZE);
2231 if (out_len != BLOB_PAGESIZE && i < npages - 1)
2233 data->error = SOLV_ERROR_CORRUPT;
2242 repodata_disable_paging(Repodata *data)
2244 if (maybe_load_repodata(data, 0)
2246 load_page_range (data, 0, data->num_pages - 1);
2249 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: