2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
26 #include "poolid_private.h"
31 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
32 unsigned char *out, unsigned int out_len);
33 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
36 unsigned int out_len);
38 #define REPODATA_BLOCK 255
42 repodata_init(Repodata *data, Repo *repo, int localpool)
44 memset(data, 0, sizeof (*data));
46 data->localpool = localpool;
48 stringpool_init_empty(&data->spool);
49 data->keys = sat_calloc(1, sizeof(Repokey));
51 data->schemata = sat_calloc(1, sizeof(Id));
52 data->schemadata = sat_calloc(1, sizeof(Id));
54 data->schemadatalen = 1;
55 data->start = repo->start;
56 data->end = repo->end;
57 data->nextra = repo->nextra;
59 data->incoreoffset = sat_extend_resize(0, data->end - data->start, sizeof(Id), REPODATA_BLOCK);
60 data->extraoffset = sat_extend_resize(0, repo->nextra, sizeof(Id), REPODATA_BLOCK);
65 repodata_free(Repodata *data)
68 sat_free(data->schemata);
69 sat_free(data->schemadata);
71 sat_free(data->spool.strings);
72 sat_free(data->spool.stringspace);
73 sat_free(data->spool.stringhashtbl);
75 sat_free(data->dirpool.dirs);
76 sat_free(data->dirpool.dirtraverse);
78 sat_free(data->incoredata);
79 sat_free(data->incoreoffset);
80 sat_free(data->extraoffset);
81 sat_free(data->verticaloffset);
83 sat_free(data->blob_store);
84 sat_free(data->pages);
85 sat_free(data->mapped);
87 sat_free(data->vincore);
89 sat_free(data->attrs);
90 sat_free(data->extraattrs);
91 sat_free(data->attrdata);
92 sat_free(data->attriddata);
94 sat_free(data->location);
95 sat_free(data->addedfileprovides);
97 if (data->pagefd != -1)
102 data_skip_recursive(Repodata *data, unsigned char *dp, Repokey *key)
105 if (key->type != REPOKEY_TYPE_COUNTED)
106 return data_skip(dp, key->type);
107 dp = data_fetch(dp, &kv, key);
112 Id *keyp = data->schemadata + data->schemata[schema];
113 for (; *keyp; keyp++)
114 dp = data_skip_recursive(data, dp, data->keys + *keyp);
119 static unsigned char *
120 forward_to_key(Repodata *data, Id keyid, Id schemaid, unsigned char *dp)
124 keyp = data->schemadata + data->schemata[schemaid];
125 while ((k = *keyp++) != 0)
129 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
131 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip that offset */
132 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip that length */
135 if (data->keys[k].storage != KEY_STORAGE_INCORE)
137 dp = data_skip_recursive(data, dp, data->keys + k);
142 #define BLOB_PAGEBITS 15
143 #define BLOB_PAGESIZE (1 << BLOB_PAGEBITS)
145 static unsigned char *
146 load_page_range(Repodata *data, unsigned int pstart, unsigned int pend)
148 /* Make sure all pages from PSTART to PEND (inclusive) are loaded,
149 and are consecutive. Return a pointer to the mapping of PSTART. */
150 unsigned char buf[BLOB_PAGESIZE];
153 /* Quick check in case all pages are there already and consecutive. */
154 for (i = pstart; i <= pend; i++)
155 if (data->pages[i].mapped_at == -1
157 && data->pages[i].mapped_at
158 != data->pages[i-1].mapped_at + BLOB_PAGESIZE))
161 return data->blob_store + data->pages[pstart].mapped_at;
163 if (data->pagefd == -1)
166 /* Ensure that we can map the numbers of pages we need at all. */
167 if (pend - pstart + 1 > data->ncanmap)
169 unsigned int oldcan = data->ncanmap;
170 data->ncanmap = pend - pstart + 1;
171 if (data->ncanmap < 4)
173 data->mapped = sat_realloc2(data->mapped, data->ncanmap, sizeof(data->mapped[0]));
174 memset (data->mapped + oldcan, 0, (data->ncanmap - oldcan) * sizeof (data->mapped[0]));
175 data->blob_store = sat_realloc2(data->blob_store, data->ncanmap, BLOB_PAGESIZE);
177 fprintf (stderr, "PAGE: can map %d pages\n", data->ncanmap);
181 /* Now search for "cheap" space in our store. Space is cheap if it's either
182 free (very cheap) or contains pages we search for anyway. */
184 /* Setup cost array. */
185 unsigned int cost[data->ncanmap];
186 for (i = 0; i < data->ncanmap; i++)
188 unsigned int pnum = data->mapped[i];
194 Attrblobpage *p = data->pages + pnum;
195 assert (p->mapped_at != -1);
196 if (pnum >= pstart && pnum <= pend)
203 /* And search for cheapest space. */
204 unsigned int best_cost = -1;
205 unsigned int best = 0;
206 unsigned int same_cost = 0;
207 for (i = 0; i + pend - pstart < data->ncanmap; i++)
209 unsigned int c = cost[i];
211 for (j = 0; j < pend - pstart + 1; j++)
214 best_cost = c, best = i;
215 else if (c == best_cost)
217 /* A null cost won't become better. */
221 /* If all places have the same cost we would thrash on slot 0. Avoid
222 this by doing a round-robin strategy in this case. */
223 if (same_cost == data->ncanmap - pend + pstart - 1)
224 best = data->rr_counter++ % (data->ncanmap - pend + pstart);
226 /* So we want to map our pages from [best] to [best+pend-pstart].
227 Use a very simple strategy, which doesn't make the best use of
228 our resources, but works. Throw away all pages in that range
229 (even ours) then copy around ours (in case they were outside the
230 range) or read them in. */
231 for (i = best; i < best + pend - pstart + 1; i++)
233 unsigned int pnum = data->mapped[i];
235 /* If this page is exactly at the right place already,
236 no need to evict it. */
237 && pnum != pstart + i - best)
239 /* Evict this page. */
241 fprintf (stderr, "PAGE: evict page %d from %d\n", pnum, i);
245 data->pages[pnum].mapped_at = -1;
249 /* Everything is free now. Read in the pages we want. */
250 for (i = pstart; i <= pend; i++)
252 Attrblobpage *p = data->pages + i;
253 unsigned int pnum = i - pstart + best;
254 void *dest = data->blob_store + pnum * BLOB_PAGESIZE;
255 if (p->mapped_at != -1)
257 if (p->mapped_at != pnum * BLOB_PAGESIZE)
260 fprintf (stderr, "PAGECOPY: %d to %d\n", i, pnum);
262 /* Still mapped somewhere else, so just copy it from there. */
263 memcpy (dest, data->blob_store + p->mapped_at, BLOB_PAGESIZE);
264 data->mapped[p->mapped_at / BLOB_PAGESIZE] = 0;
269 unsigned int in_len = p->file_size;
270 unsigned int compressed = in_len & 1;
273 fprintf (stderr, "PAGEIN: %d to %d", i, pnum);
275 if (pread(data->pagefd, compressed ? buf : dest, in_len, p->file_offset) != in_len)
277 perror ("mapping pread");
282 unsigned int out_len;
283 out_len = unchecked_decompress_buf(buf, in_len,
284 dest, BLOB_PAGESIZE);
285 if (out_len != BLOB_PAGESIZE && i < data->num_pages - 1)
287 fprintf(stderr, "can't decompress\n");
291 fprintf (stderr, " (expand %d to %d)", in_len, out_len);
295 fprintf (stderr, "\n");
298 p->mapped_at = pnum * BLOB_PAGESIZE;
299 data->mapped[pnum] = i + 1;
301 return data->blob_store + best * BLOB_PAGESIZE;
304 static unsigned char *
305 make_vertical_available(Repodata *data, Repokey *key, Id off, Id len)
310 if (off >= data->lastverticaloffset)
312 off -= data->lastverticaloffset;
313 if (off + len > data->vincorelen)
315 return data->vincore + off;
317 if (off + len > key->size)
319 /* we now have the offset, go into vertical */
320 off += data->verticaloffset[key - data->keys];
321 /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */
322 dp = load_page_range(data, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
324 dp += off % BLOB_PAGESIZE;
328 static inline unsigned char *
329 get_data(Repodata *data, Repokey *key, unsigned char **dpp)
331 unsigned char *dp = *dpp;
335 if (key->storage == KEY_STORAGE_INCORE)
337 /* hmm, this is a bit expensive */
338 *dpp = data_skip_recursive(data, dp, key);
341 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
344 dp = data_read_id(dp, &off);
345 dp = data_read_id(dp, &len);
347 return make_vertical_available(data, key, off, len);
353 maybe_load_repodata(Repodata *data, Id *keyid)
355 if (data->state == REPODATA_STUB)
357 if (data->loadcallback)
361 /* key order may change when loading */
363 Id name = data->keys[*keyid].name;
364 Id type = data->keys[*keyid].type;
365 data->loadcallback(data);
366 if (data->state == REPODATA_AVAILABLE)
368 for (i = 1; i < data->nkeys; i++)
369 if (data->keys[i].name == name && data->keys[i].type == type)
378 data->loadcallback(data);
381 data->state = REPODATA_ERROR;
383 if (data->state == REPODATA_AVAILABLE)
385 data->state = REPODATA_ERROR;
389 static inline unsigned char*
390 entry2data(Repodata *data, Id entry)
393 return data->incoredata + data->extraoffset[-1 - entry];
395 return data->incoredata + data->incoreoffset[entry];
399 repodata_lookup_id(Repodata *data, Id entry, Id keyid)
406 if (!maybe_load_repodata(data, &keyid))
408 dp = entry2data(data, entry);
411 dp = data_read_id(dp, &schema);
412 /* make sure the schema of this solvable contains the key */
413 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
416 dp = forward_to_key(data, keyid, schema, dp);
417 key = data->keys + keyid;
418 dp = get_data(data, key, &dp);
421 if (key->type == REPOKEY_TYPE_CONSTANTID)
423 if (key->type != REPOKEY_TYPE_ID)
425 dp = data_read_id(dp, &id);
430 repodata_lookup_str(Repodata *data, Id entry, Id keyid)
437 if (!maybe_load_repodata(data, &keyid))
440 dp = entry2data(data, entry);
443 dp = data_read_id(dp, &schema);
444 /* make sure the schema of this solvable contains the key */
445 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
448 dp = forward_to_key(data, keyid, schema, dp);
449 key = data->keys + keyid;
450 dp = get_data(data, key, &dp);
453 if (key->type == REPOKEY_TYPE_STR)
454 return (const char *)dp;
455 if (key->type == REPOKEY_TYPE_CONSTANTID)
456 return id2str(data->repo->pool, key->size);
457 if (key->type == REPOKEY_TYPE_ID)
458 dp = data_read_id(dp, &id);
462 return data->spool.stringspace + data->spool.strings[id];
463 return id2str(data->repo->pool, id);
467 repodata_lookup_num(Repodata *data, Id entry, Id keyid, unsigned int *value)
477 if (!maybe_load_repodata(data, &keyid))
480 dp = entry2data(data, entry);
483 dp = data_read_id(dp, &schema);
484 /* make sure the schema of this solvable contains the key */
485 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
488 dp = forward_to_key(data, keyid, schema, dp);
489 key = data->keys + keyid;
490 dp = get_data(data, key, &dp);
493 if (key->type == REPOKEY_TYPE_NUM
494 || key->type == REPOKEY_TYPE_U32
495 || key->type == REPOKEY_TYPE_CONSTANT)
497 dp = data_fetch(dp, &kv, key);
505 repodata_lookup_void(Repodata *data, Id entry, Id keyid)
510 if (!maybe_load_repodata(data, &keyid))
512 dp = entry2data(data, entry);
515 dp = data_read_id(dp, &schema);
516 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
522 const unsigned char *
523 repodata_lookup_bin_checksum(Repodata *data, Id entry, Id keyid, Id *typep)
530 if (!maybe_load_repodata(data, &keyid))
532 dp = entry2data(data, entry);
535 dp = data_read_id(dp, &schema);
536 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
539 dp = forward_to_key(data, keyid, schema, dp);
540 key = data->keys + keyid;
542 return get_data(data, key, &dp);
546 repodata_search(Repodata *data, Id entry, Id keyname, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
550 Id k, keyid, *kp, *keyp;
551 unsigned char *dp, *ddp;
557 || !maybe_load_repodata(data, 0))
560 dp = entry2data(data, entry);
563 dp = data_read_id(dp, &schema);
564 keyp = data->schemadata + data->schemata[schema];
567 /* search in a specific key */
568 for (kp = keyp; (k = *kp++) != 0; )
569 if (data->keys[k].name == keyname)
573 dp = forward_to_key(data, k, schema, dp);
579 while ((keyid = *keyp++) != 0)
582 key = data->keys + keyid;
583 ddp = get_data(data, key, &dp);
586 ddp = data_fetch(ddp, &kv, key);
589 if (key->type == REPOKEY_TYPE_COUNTED)
592 int subschema = kv.id;
593 Repokey *countkey = key;
595 callback(cbdata, data->repo->pool->solvables + data->start + entry, data, countkey, &kv);
598 Id *kp = data->schemadata + data->schemata[subschema];
601 key = data->keys + *kp;
602 ddp = data_fetch(ddp, &kv, key);
605 callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv);
608 callback(cbdata, data->repo->pool->solvables + data->start + entry, data, countkey, &kv);
611 stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, countkey, &kv);
614 stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv);
616 while (!kv.eof && !stop);
617 if (onekey || stop > SEARCH_NEXT_KEY)
623 dataiterator_newdata(Dataiterator *di)
625 Id keyname = di->keyname;
626 Repodata *data = di->data;
629 if (data->state == REPODATA_STUB)
634 for (j = 1; j < data->nkeys; j++)
635 if (keyname == data->keys[j].name)
637 if (j == data->nkeys)
641 if (data->loadcallback)
642 data->loadcallback(data);
644 data->state = REPODATA_ERROR;
646 if (data->state == REPODATA_ERROR)
650 unsigned char *dp = data->incoredata;
654 dp += data->incoreoffset[di->solvid - data->start];
656 dp += data->extraoffset[-1 - di->solvid - data->extrastart];
657 dp = data_read_id(dp, &schema);
658 Id *keyp = data->schemadata + data->schemata[schema];
662 /* search in a specific key */
663 for (kp = keyp; (k = *kp++) != 0; )
664 if (data->keys[k].name == keyname)
668 dp = forward_to_key(data, k, schema, dp);
678 di->key = di->data->keys + keyid;
683 di->dp = get_data(di->data, di->key, &di->nextkeydp);
688 dataiterator_init(Dataiterator *di, Repo *repo, Id p, Id keyname,
689 const char *match, int flags)
695 di->flags |= __SEARCH_ONESOLVABLE;
696 di->data = repo->repodata - 1;
697 if (flags & SEARCH_NO_STORAGE_SOLVABLE)
704 di->solvid = repo->start - 1;
707 fprintf(stderr, "A repo contains the NULL solvable!\n");
710 di->data = repo->repodata + repo->nrepodata - 1;
715 if ((di->flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
719 /* We feed multiple lines eventually (e.g. authors or descriptions),
720 so set REG_NEWLINE. */
722 regcomp(&di->regex, di->match,
723 REG_EXTENDED | REG_NOSUB | REG_NEWLINE
724 | ((di->flags & SEARCH_NOCASE) ? REG_ICASE : 0));
726 if (di->regex_err != 0)
728 fprintf(stderr, "Given regex failed to compile: %s\n", di->match);
729 fprintf(stderr, "regcomp error code: %d\n", di->regex_err);
736 di->flags |= (di->flags & SEARCH_STRINGMASK) | SEARCH_STRING;
742 di->keyname = keyname;
743 static Id zeroid = 0;
751 /* FIXME factor and merge with repo_matchvalue */
753 dataiterator_match_int_real(Dataiterator *di, int flags, const void *vmatch)
755 KeyValue *kv = &di->kv;
756 const char *match = vmatch;
757 if ((flags & SEARCH_STRINGMASK) != 0)
759 switch (di->key->type)
761 case REPOKEY_TYPE_ID:
762 case REPOKEY_TYPE_IDARRAY:
763 if (di->data && di->data->localpool)
764 kv->str = stringpool_id2str(&di->data->spool, kv->id);
766 kv->str = id2str(di->repo->pool, kv->id);
768 case REPOKEY_TYPE_STR:
773 /* Maybe skip the kind specifier. Do this only for SOLVABLE attributes,
774 for the others we can't know if a colon separates a kind or not. */
775 if ((flags & SEARCH_SKIP_KIND)
776 && di->key->storage == KEY_STORAGE_SOLVABLE)
778 const char *s = strchr(kv->str, ':');
782 switch ((flags & SEARCH_STRINGMASK))
784 case SEARCH_SUBSTRING:
785 if (flags & SEARCH_NOCASE)
787 if (!strcasestr(kv->str, match))
792 if (!strstr(kv->str, match))
797 if (flags & SEARCH_NOCASE)
799 if (strcasecmp(match, kv->str))
804 if (strcmp(match, kv->str))
809 if (fnmatch(match, kv->str, (flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
813 if (regexec((const regex_t *)vmatch, kv->str, 0, NULL, 0))
824 dataiterator_match_int(Dataiterator *di)
826 if ((di->flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
827 return dataiterator_match_int_real(di, di->flags, &di->regex);
829 return dataiterator_match_int_real(di, di->flags, di->match);
833 dataiterator_match(Dataiterator *di, int flags, const void *vmatch)
835 return dataiterator_match_int_real(di, flags, vmatch);
838 static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = {
839 { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
840 { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
841 { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
842 { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
843 { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
844 { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
845 { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
846 { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
847 { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
848 { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
849 { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
850 { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
851 { SOLVABLE_FRESHENS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
852 { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE },
856 dataiterator_step(Dataiterator *di)
870 di->kv.eof = idp[1] ? 0 : 1;
876 Solvable *s = di->repo->pool->solvables + di->solvid;
877 int state = di->state;
878 di->key = solvablekeys + state - 1;
880 di->state = RPM_RPMDBID;
887 state = di->keyname - 1;
909 case SOLVABLE_VENDOR:
912 di->kv.id = s->vendor;
915 case SOLVABLE_PROVIDES:
916 di->idp = s->provides
917 ? di->repo->idarraydata + s->provides : 0;
919 case SOLVABLE_OBSOLETES:
920 di->idp = s->obsoletes
921 ? di->repo->idarraydata + s->obsoletes : 0;
923 case SOLVABLE_CONFLICTS:
924 di->idp = s->conflicts
925 ? di->repo->idarraydata + s->conflicts : 0;
927 case SOLVABLE_REQUIRES:
928 di->idp = s->requires
929 ? di->repo->idarraydata + s->requires : 0;
931 case SOLVABLE_RECOMMENDS:
932 di->idp = s->recommends
933 ? di->repo->idarraydata + s->recommends : 0;
935 case SOLVABLE_SUPPLEMENTS:
936 di->idp = s->supplements
937 ? di->repo->idarraydata + s->supplements : 0;
939 case SOLVABLE_SUGGESTS:
940 di->idp = s->suggests
941 ? di->repo->idarraydata + s->suggests : 0;
943 case SOLVABLE_ENHANCES:
944 di->idp = s->enhances
945 ? di->repo->idarraydata + s->enhances : 0;
947 case SOLVABLE_FRESHENS:
948 di->idp = s->freshens
949 ? di->repo->idarraydata + s->freshens : 0;
952 if (!di->repo->rpmdbid)
954 di->kv.num = di->repo->rpmdbid[di->solvid - di->repo->start];
958 di->data = di->repo->repodata - 1;
964 else if (di->subkeyp)
969 /* Send end-of-substruct. We are here only when we saw a
970 _COUNTED key one level up. Since then we didn't increment
971 ->keyp, so it still can be found at keyp[-1]. */
973 di->key = di->data->keys + di->keyp[-1];
976 else if (!(keyid = *di->subkeyp++))
978 /* Send end-of-element. See above for keyp[-1]. */
980 di->key = di->data->keys + di->keyp[-1];
981 di->subkeyp = di->data->schemadata + di->data->schemata[di->subschema];
986 di->key = di->data->keys + keyid;
987 di->dp = data_fetch(di->dp, &di->kv, di->key);
997 di->dp = data_fetch(di->dp, &di->kv, di->key);
1002 if (di->keyname || !(keyid = *di->keyp++))
1006 Repo *repo = di->repo;
1007 Repodata *data = ++di->data;
1008 if (data >= repo->repodata + repo->nrepodata)
1010 if (di->flags & __SEARCH_ONESOLVABLE)
1012 if (di->solvid >= 0)
1014 while (++di->solvid < repo->end)
1015 if (repo->pool->solvables[di->solvid].repo == repo)
1017 if (di->solvid >= repo->end)
1019 if (!(di->flags & SEARCH_EXTRA))
1022 if (di->solvid < -repo->nextra)
1029 if (di->solvid < -repo->nextra)
1032 Pool *pool = di->repo->pool;
1033 if (!(di->flags & SEARCH_ALL_REPOS)
1034 || di->repo == pool->repos[pool->nrepos - 1])
1037 for (i = 0; i < pool->nrepos; i++)
1038 if (di->repo == pool->repos[i])
1040 di->repo = pool->repos[i + 1];
1041 dataiterator_init(di, di->repo, 0, di->keyname, di->match, di->flags);
1045 di->data = repo->repodata - 1;
1047 || (di->flags & SEARCH_NO_STORAGE_SOLVABLE))
1049 static Id zeroid = 0;
1054 if ((di->solvid < 0 && (-1 - di->solvid) >= data->extrastart && (-1 - di->solvid) < (data->extrastart + data->nextra))
1055 || (di->solvid >= 0 && di->solvid >= data->start && di->solvid < data->end))
1057 dataiterator_newdata(di);
1065 di->key = di->data->keys + keyid;
1066 di->dp = get_data(di->data, di->key, &di->nextkeydp);
1068 di->dp = data_fetch(di->dp, &di->kv, di->key);
1070 if (di->key->type == REPOKEY_TYPE_COUNTED)
1072 di->subnum = di->kv.num;
1073 di->subschema = di->kv.id;
1075 di->subkeyp = di->data->schemadata + di->data->schemata[di->subschema];
1080 || dataiterator_match_int(di))
1087 dataiterator_skip_attribute(Dataiterator *di)
1091 /* This will make the next _step call to retrieve the next field. */
1096 dataiterator_skip_solvable(Dataiterator *di)
1098 /* We're done with this field. */
1100 /* And with solvable data. */
1102 /* And with all keys for this repodata and thing. */
1103 static Id zeroid = 0;
1105 /* And with all repodatas for this thing. */
1106 di->data = di->repo->repodata + di->repo->nrepodata - 1;
1107 /* Hence the next call to _step will retrieve the next thing. */
1111 dataiterator_skip_repo(Dataiterator *di)
1113 dataiterator_skip_solvable(di);
1114 /* We're done with all solvables and all extra things for this repo. */
1115 di->solvid = -1 - di->repo->nextra;
1119 dataiterator_jump_to_solvable(Dataiterator *di, Solvable *s)
1122 /* Simulate us being done with the solvable before the requested one. */
1123 dataiterator_skip_solvable(di);
1124 di->solvid = s - s->repo->pool->solvables;
1129 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1132 dataiterator_skip_solvable(di);
1133 di->solvid = repo->start - 1;
1136 /* extend repodata so that it includes solvables p */
1138 repodata_extend(Repodata *data, Id p)
1140 if (data->start == data->end)
1141 data->start = data->end = p;
1144 int old = data->end - data->start;
1145 int new = p - data->end + 1;
1148 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id), REPODATA_BLOCK);
1149 memset(data->attrs + old, 0, new * sizeof(Id));
1151 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
1152 memset(data->incoreoffset + old, 0, new * sizeof(Id));
1155 if (p < data->start)
1157 int old = data->end - data->start;
1158 int new = data->start - p;
1161 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id), REPODATA_BLOCK);
1162 memmove(data->attrs + new, data->attrs, old * sizeof(Id));
1163 memset(data->attrs, 0, new * sizeof(Id));
1165 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
1166 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
1167 memset(data->incoreoffset, 0, new * sizeof(Id));
1173 repodata_extend_extra(Repodata *data, int nextra)
1175 if (nextra <= data->nextra)
1177 if (data->extraattrs)
1179 data->extraattrs = sat_extend(data->extraattrs, data->nextra, nextra - data->nextra, sizeof(Id), REPODATA_BLOCK);
1180 memset(data->extraattrs + data->nextra, 0, (nextra - data->nextra) * sizeof (Id));
1182 data->extraoffset = sat_extend(data->extraoffset, data->nextra, nextra - data->nextra, sizeof(Id), REPODATA_BLOCK);
1183 memset(data->extraoffset + data->nextra, 0, (nextra - data->nextra) * sizeof(Id));
1184 data->nextra = nextra;
1188 repodata_extend_block(Repodata *data, Id start, Id num)
1192 if (!data->incoreoffset)
1194 data->incoreoffset = sat_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
1195 data->start = start;
1196 data->end = start + num;
1199 repodata_extend(data, start);
1201 repodata_extend(data, start + num - 1);
1204 /**********************************************************************/
1206 #define REPODATA_ATTRS_BLOCK 63
1207 #define REPODATA_ATTRDATA_BLOCK 1023
1208 #define REPODATA_ATTRIDDATA_BLOCK 63
1211 get_new_struct(Repodata *data)
1213 /* Make sure to never give out struct id 0. */
1216 data->structs = sat_extend(0, 0, 2, sizeof(Id *), REPODATA_BLOCK);
1217 data->structs[0] = 0;
1218 data->structs[1] = 0;
1222 data->structs = sat_extend(data->structs, data->nstructs, 1, sizeof(Id *), REPODATA_BLOCK);
1223 data->structs[data->nstructs] = 0;
1224 return data->nstructs++;
1228 repodata_get_handle_int(Repodata *data, Id entry)
1231 if (!data->attrs && entry >= 0)
1233 data->attrs = sat_calloc_block(data->end - data->start, sizeof(Id),
1236 else if (!data->extraattrs && entry < 0)
1237 data->extraattrs = sat_calloc_block(data->nextra, sizeof(Id), REPODATA_BLOCK);
1239 ap = &data->extraattrs[-1 - entry];
1241 ap = &data->attrs[entry];
1243 *ap = get_new_struct(data);
1248 repodata_get_handle(Repodata *data, Id entry)
1250 return repodata_get_handle_int(data, entry);
1254 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
1259 ap = data->structs[handle];
1263 for (pp = ap; *pp; pp += 2)
1264 /* Determine equality based on the name only, allows us to change
1265 type (when overwrite is set), and makes TYPE_CONSTANT work. */
1266 if (data->keys[*pp].name == data->keys[keyid].name)
1279 ap = sat_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
1280 data->structs[handle] = ap;
1288 repodata_set(Repodata *data, Id handle, Repokey *key, Id val)
1292 /* find key in keys */
1293 for (keyid = 1; keyid < data->nkeys; keyid++)
1294 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
1296 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
1300 if (keyid == data->nkeys)
1302 /* allocate new key */
1303 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
1304 data->keys[data->nkeys++] = *key;
1305 if (data->verticaloffset)
1307 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
1308 data->verticaloffset[data->nkeys - 1] = 0;
1311 repodata_insert_keyid(data, handle, keyid, val, 1);
1315 repodata_set_id(Repodata *data, Id handle, Id keyname, Id id)
1319 key.type = REPOKEY_TYPE_ID;
1321 key.storage = KEY_STORAGE_INCORE;
1322 repodata_set(data, handle, &key, id);
1326 repodata_set_num(Repodata *data, Id handle, Id keyname, unsigned int num)
1330 key.type = REPOKEY_TYPE_NUM;
1332 key.storage = KEY_STORAGE_INCORE;
1333 repodata_set(data, handle, &key, (Id)num);
1337 repodata_set_poolstr(Repodata *data, Id handle, Id keyname, const char *str)
1341 if (data->localpool)
1342 id = stringpool_str2id(&data->spool, str, 1);
1344 id = str2id(data->repo->pool, str, 1);
1346 key.type = REPOKEY_TYPE_ID;
1348 key.storage = KEY_STORAGE_INCORE;
1349 repodata_set(data, handle, &key, id);
1353 repodata_set_constant(Repodata *data, Id handle, Id keyname, unsigned int constant)
1357 key.type = REPOKEY_TYPE_CONSTANT;
1358 key.size = constant;
1359 key.storage = KEY_STORAGE_INCORE;
1360 repodata_set(data, handle, &key, 0);
1364 repodata_set_constantid(Repodata *data, Id handle, Id keyname, Id id)
1368 key.type = REPOKEY_TYPE_CONSTANTID;
1370 key.storage = KEY_STORAGE_INCORE;
1371 repodata_set(data, handle, &key, 0);
1375 repodata_set_void(Repodata *data, Id handle, Id keyname)
1379 key.type = REPOKEY_TYPE_VOID;
1381 key.storage = KEY_STORAGE_INCORE;
1382 repodata_set(data, handle, &key, 0);
1386 repodata_set_str(Repodata *data, Id handle, Id keyname, const char *str)
1391 l = strlen(str) + 1;
1393 key.type = REPOKEY_TYPE_STR;
1395 key.storage = KEY_STORAGE_INCORE;
1396 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1397 memcpy(data->attrdata + data->attrdatalen, str, l);
1398 repodata_set(data, handle, &key, data->attrdatalen);
1399 data->attrdatalen += l;
1403 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
1408 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
1410 /* great! just append the new data */
1411 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1412 data->attriddatalen--; /* overwrite terminating 0 */
1413 data->lastdatalen += entrysize;
1416 pp = data->structs[handle];
1418 for (; *pp; pp += 2)
1419 if (data->keys[*pp].name == keyname && data->keys[*pp].type == keytype)
1423 /* not found. allocate new key */
1428 key.storage = KEY_STORAGE_INCORE;
1429 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1430 repodata_set(data, handle, &key, data->attriddatalen);
1431 data->lasthandle = 0; /* next time... */
1435 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
1436 oldsize += entrysize;
1437 if (ida + 1 == data->attriddata + data->attriddatalen)
1439 /* this was the last entry, just append it */
1440 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1441 data->attriddatalen--; /* overwrite terminating 0 */
1445 /* too bad. move to back. */
1446 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1447 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
1448 pp[1] = data->attriddatalen;
1449 data->attriddatalen += oldsize;
1451 data->lasthandle = handle;
1452 data->lastkey = *pp;
1453 data->lastdatalen = data->attriddatalen + entrysize + 1;
1457 checksumtype2len(Id type)
1461 case REPOKEY_TYPE_MD5:
1463 case REPOKEY_TYPE_SHA1:
1465 case REPOKEY_TYPE_SHA256:
1466 return SIZEOF_SHA256;
1473 repodata_set_bin_checksum(Repodata *data, Id handle, Id keyname, Id type,
1474 const unsigned char *str)
1477 int l = checksumtype2len(type);
1484 key.storage = KEY_STORAGE_INCORE;
1485 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1486 memcpy(data->attrdata + data->attrdatalen, str, l);
1487 repodata_set(data, handle, &key, data->attrdatalen);
1488 data->attrdatalen += l;
1492 hexstr2bytes(unsigned char *buf, const char *str, int buflen)
1495 for (i = 0; i < buflen; i++)
1497 #define c2h(c) (((c)>='0' && (c)<='9') ? ((c)-'0') \
1498 : ((c)>='a' && (c)<='f') ? ((c)-'a'+10) \
1499 : ((c)>='A' && (c)<='F') ? ((c)-'A'+10) \
1510 buf[i] = (buf[i] << 4) | v;
1517 repodata_set_checksum(Repodata *data, Id handle, Id keyname, Id type,
1520 unsigned char buf[64];
1521 int l = checksumtype2len(type);
1525 if (hexstr2bytes(buf, str, l) != l)
1527 fprintf(stderr, "Invalid hex character in '%s'\n", str);
1530 repodata_set_bin_checksum(data, handle, keyname, type, buf);
1534 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
1539 l = checksumtype2len(type);
1542 s = str = pool_alloctmpspace(data->repo->pool, 2 * l + 1);
1543 for (i = 0; i < l; i++)
1545 unsigned char v = buf[i];
1546 unsigned char w = v >> 4;
1547 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
1549 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
1556 repodata_globalize_id(Repodata *data, Id id)
1558 if (!data || !data->localpool)
1560 return str2id(data->repo->pool, stringpool_id2str(&data->spool, id), 1);
1564 repodata_add_dirnumnum(Repodata *data, Id handle, Id keyname, Id dir, Id num, Id num2)
1568 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", handle, dir, num, num2, data->attriddatalen);
1570 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
1571 data->attriddata[data->attriddatalen++] = dir;
1572 data->attriddata[data->attriddatalen++] = num;
1573 data->attriddata[data->attriddatalen++] = num2;
1574 data->attriddata[data->attriddatalen++] = 0;
1578 repodata_add_dirstr(Repodata *data, Id handle, Id keyname, Id dir, const char *str)
1584 l = strlen(str) + 1;
1585 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1586 memcpy(data->attrdata + data->attrdatalen, str, l);
1587 stroff = data->attrdatalen;
1588 data->attrdatalen += l;
1591 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", handle, dir, str, data->attriddatalen);
1593 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
1594 data->attriddata[data->attriddatalen++] = dir;
1595 data->attriddata[data->attriddatalen++] = stroff;
1596 data->attriddata[data->attriddatalen++] = 0;
1600 repodata_add_idarray(Repodata *data, Id handle, Id keyname, Id id)
1603 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", handle, id, data->attriddatalen);
1605 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_IDARRAY, 1);
1606 data->attriddata[data->attriddatalen++] = id;
1607 data->attriddata[data->attriddatalen++] = 0;
1611 repodata_add_poolstr_array(Repodata *data, Id handle, Id keyname,
1615 if (data->localpool)
1616 id = stringpool_str2id(&data->spool, str, 1);
1618 id = str2id(data->repo->pool, str, 1);
1619 repodata_add_idarray(data, handle, keyname, id);
1623 repodata_create_struct(Repodata *data, Id handle, Id keyname)
1625 Id newhandle = get_new_struct(data);
1626 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_COUNTED, 1);
1627 data->attriddata[data->attriddatalen++] = newhandle;
1628 data->attriddata[data->attriddatalen++] = 0;
1633 repodata_merge_attrs(Repodata *data, Id dest, Id src)
1637 || !(keyp = data->structs[src < 0
1638 ? data->extraattrs[-1 - src]
1639 : data->attrs[src]]))
1641 dest = repodata_get_handle_int(data, dest);
1642 for (; *keyp; keyp += 2)
1643 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
1646 /*********************************/
1648 /* unify with repo_write! */
1650 #define EXTDATA_BLOCK 1023
1651 #define SCHEMATA_BLOCK 31
1652 #define SCHEMATADATA_BLOCK 255
1660 data_addid(struct extdata *xd, Id x)
1663 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
1664 dp = xd->buf + xd->len;
1669 *dp++ = (x >> 28) | 128;
1671 *dp++ = (x >> 21) | 128;
1672 *dp++ = (x >> 14) | 128;
1675 *dp++ = (x >> 7) | 128;
1677 xd->len = dp - xd->buf;
1681 data_addideof(struct extdata *xd, Id x, int eof)
1684 x = (x & 63) | ((x & ~63) << 1);
1685 data_addid(xd, (eof ? x: x | 64));
1689 data_addblob(struct extdata *xd, unsigned char *blob, int len)
1691 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
1692 memcpy(xd->buf + xd->len, blob, len);
1696 /*********************************/
1699 addschema_prepare(Repodata *data, Id *schematacache)
1704 memset(schematacache, 0, 256 * sizeof(Id));
1705 for (i = 0; i < data->nschemata; i++)
1707 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
1710 schematacache[h] = i + 1;
1712 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
1713 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
1717 addschema(Repodata *data, Id *schema, Id *schematacache)
1722 for (sp = schema, len = 0, h = 0; *sp; len++)
1727 cid = schematacache[h];
1731 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1733 /* cache conflict */
1734 for (cid = 0; cid < data->nschemata; cid++)
1735 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1738 /* a new one. make room. */
1739 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
1740 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
1742 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
1743 data->schemata[data->nschemata] = data->schemadatalen;
1744 data->schemadatalen += len;
1745 schematacache[h] = data->nschemata + 1;
1747 fprintf(stderr, "addschema: new schema\n");
1749 return data->nschemata++;
1753 repodata_serialize_key(Repodata *data, struct extdata *newincore,
1754 struct extdata *newvincore,
1755 Id *schema, Id *schematacache,
1756 Repokey *key, Id val)
1758 /* Otherwise we have a new value. Parse it into the internal
1762 unsigned int oldvincorelen = 0;
1766 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1769 oldvincorelen = xd->len;
1773 case REPOKEY_TYPE_VOID:
1774 case REPOKEY_TYPE_CONSTANT:
1775 case REPOKEY_TYPE_CONSTANTID:
1777 case REPOKEY_TYPE_STR:
1778 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
1780 case REPOKEY_TYPE_MD5:
1781 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
1783 case REPOKEY_TYPE_SHA1:
1784 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
1786 case REPOKEY_TYPE_ID:
1787 case REPOKEY_TYPE_NUM:
1788 case REPOKEY_TYPE_DIR:
1789 data_addid(xd, val);
1791 case REPOKEY_TYPE_IDARRAY:
1792 for (ida = data->attriddata + val; *ida; ida++)
1793 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
1795 case REPOKEY_TYPE_DIRNUMNUMARRAY:
1796 for (ida = data->attriddata + val; *ida; ida += 3)
1798 data_addid(xd, ida[0]);
1799 data_addid(xd, ida[1]);
1800 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
1803 case REPOKEY_TYPE_DIRSTRARRAY:
1804 for (ida = data->attriddata + val; *ida; ida += 2)
1806 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
1807 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
1810 case REPOKEY_TYPE_COUNTED:
1814 for (ida = data->attriddata + val; *ida; ida++)
1817 fprintf(stderr, "serialize struct %d\n", *ida);
1820 Id *kp = data->structs[*ida];
1827 fprintf(stderr, " %s:%d\n", id2str(data->repo->pool, data->keys[*kp].name), kp[1]);
1833 schemaid = addschema(data, schema, schematacache);
1834 else if (schemaid != addschema(data, schema, schematacache))
1836 fprintf(stderr, " not yet implemented: substructs with different schemas\n");
1840 fprintf(stderr, " schema %d\n", schemaid);
1845 data_addid(xd, num);
1846 data_addid(xd, schemaid);
1847 for (ida = data->attriddata + val; *ida; ida++)
1849 Id *kp = data->structs[*ida];
1854 repodata_serialize_key(data, newincore, newvincore,
1855 schema, schematacache,
1856 data->keys + *kp, kp[1]);
1862 fprintf(stderr, "don't know how to handle type %d\n", key->type);
1865 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1867 /* put offset/len in incore */
1868 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
1869 oldvincorelen = xd->len - oldvincorelen;
1870 data_addid(newincore, oldvincorelen);
1875 repodata_internalize(Repodata *data)
1879 Id schematacache[256];
1880 Id schemaid, *schema, *sp, oldschema, *keyp, *seen;
1881 unsigned char *dp, *ndp;
1882 int newschema, oldcount;
1883 struct extdata newincore;
1884 struct extdata newvincore;
1886 if (!data->attrs && !data->extraattrs)
1889 newvincore.buf = data->vincore;
1890 newvincore.len = data->vincorelen;
1892 schema = sat_malloc2(data->nkeys, sizeof(Id));
1893 seen = sat_malloc2(data->nkeys, sizeof(Id));
1895 /* Merge the data already existing (in data->schemata, ->incoredata and
1896 friends) with the new attributes in data->attrs[]. */
1897 nentry = data->end - data->start;
1898 addschema_prepare(data, schematacache);
1899 memset(&newincore, 0, sizeof(newincore));
1900 data_addid(&newincore, 0);
1903 for (entry = data->extraattrs ? -data->nextra : 0; entry < nentry; entry++)
1906 memset(seen, 0, data->nkeys * sizeof(Id));
1908 dp = entry2data(data, entry);
1909 if (data->incoredata)
1910 dp = data_read_id(dp, &oldschema);
1914 fprintf(stderr, "oldschema %d\n", oldschema);
1915 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
1916 fprintf(stderr, "schemadata %p\n", data->schemadata);
1918 /* seen: -1: old data 0: skipped >0: id + 1 */
1921 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
1925 fprintf(stderr, "Inconsistent old data (key occured twice).\n");
1932 handle = entry < 0 ? data->extraattrs[-1 - entry] : data->attrs[entry];
1933 keyp = data->structs[handle];
1935 for (; *keyp; keyp += 2)
1942 seen[*keyp] = keyp[1] + 1;
1946 /* Ideally we'd like to sort the new schema here, to ensure
1947 schema equality independend of the ordering. We can't do that
1948 yet. For once see below (old ids need to come before new ids).
1949 An additional difficulty is that we also need to move
1950 the values with the keys. */
1951 schemaid = addschema(data, schema, schematacache);
1953 schemaid = oldschema;
1956 /* Now create data blob. We walk through the (possibly new) schema
1957 and either copy over old data, or insert the new. */
1958 /* XXX Here we rely on the fact that the (new) schema has the form
1959 o1 o2 o3 o4 ... | n1 n2 n3 ...
1960 (oX being the old keyids (possibly overwritten), and nX being
1961 the new keyids). This rules out sorting the keyids in order
1962 to ensure a small schema count. */
1964 data->extraoffset[-1 - entry] = newincore.len;
1966 data->incoreoffset[entry] = newincore.len;
1967 data_addid(&newincore, schemaid);
1968 for (keyp = data->schemadata + data->schemata[schemaid]; *keyp; keyp++)
1970 key = data->keys + *keyp;
1972 fprintf(stderr, "internalize %d:%s:%s\n", entry, id2str(data->repo->pool, key->name), id2str(data->repo->pool, key->type));
1977 /* Skip the data associated with this old key. */
1978 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1980 ndp = data_skip(dp, REPOKEY_TYPE_ID);
1981 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
1983 else if (key->storage == KEY_STORAGE_INCORE)
1984 ndp = data_skip_recursive(data, dp, key);
1987 if (seen[*keyp] == -1)
1989 /* If this key was an old one _and_ was not overwritten with
1990 a different value copy over the old value (we skipped it
1993 data_addblob(&newincore, dp, ndp - dp);
1996 else if (seen[*keyp])
1998 /* Otherwise we have a new value. Parse it into the internal
2000 repodata_serialize_key(data, &newincore, &newvincore,
2001 schema, schematacache,
2002 key, seen[*keyp] - 1);
2006 if (data->structs[handle])
2007 data->structs[handle] = sat_free(data->structs[handle]);
2009 for (entry = 0; entry < data->nstructs; entry++)
2010 if (data->structs[entry])
2011 sat_free(data->structs[entry]);
2012 data->structs = sat_free(data->structs);
2013 data->lasthandle = 0;
2015 data->lastdatalen = 0;
2019 sat_free(data->incoredata);
2020 data->incoredata = newincore.buf;
2021 data->incoredatalen = newincore.len;
2022 data->incoredatafree = 0;
2024 sat_free(data->vincore);
2025 data->vincore = newvincore.buf;
2026 data->vincorelen = newvincore.len;
2028 data->attrs = sat_free(data->attrs);
2029 data->extraattrs = sat_free(data->extraattrs);
2030 data->attrdata = sat_free(data->attrdata);
2031 data->attriddata = sat_free(data->attriddata);
2032 data->attrdatalen = 0;
2033 data->attriddatalen = 0;
2037 repodata_str2dir(Repodata *data, const char *dir, int create)
2043 while (*dir == '/' && dir[1] == '/')
2045 if (*dir == '/' && !dir[1])
2049 dire = strchrnul(dir, '/');
2050 if (data->localpool)
2051 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
2053 id = strn2id(data->repo->pool, dir, dire - dir, create);
2056 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
2069 repodata_dir2str(Repodata *data, Id did, const char *suf)
2071 Pool *pool = data->repo->pool;
2078 return suf ? suf : "";
2082 comp = dirpool_compid(&data->dirpool, parent);
2083 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
2085 parent = dirpool_parent(&data->dirpool, parent);
2090 l += strlen(suf) + 1;
2091 p = pool_alloctmpspace(pool, l + 1) + l;
2102 comp = dirpool_compid(&data->dirpool, parent);
2103 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
2106 strncpy(p, comps, l);
2107 parent = dirpool_parent(&data->dirpool, parent);
2115 repodata_compress_page(unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max)
2117 return compress_buf(page, len, cpage, max);
2120 #define SOLV_ERROR_EOF 3
2122 static inline unsigned int
2128 for (i = 0; i < 4; i++)
2138 #define SOLV_ERROR_EOF 3
2139 #define SOLV_ERROR_CORRUPT 6
2141 /* Try to either setup on-demand paging (using FP as backing
2142 file), or in case that doesn't work (FP not seekable) slurps in
2143 all pages and deactivates paging. */
2145 repodata_read_or_setup_pages(Repodata *data, unsigned int pagesz, unsigned int blobsz)
2147 FILE *fp = data->fp;
2148 unsigned int npages;
2150 unsigned int can_seek;
2152 unsigned char buf[BLOB_PAGESIZE];
2154 if (pagesz != BLOB_PAGESIZE)
2156 /* We could handle this by slurping in everything. */
2157 data->error = SOLV_ERROR_CORRUPT;
2161 if ((cur_file_ofs = ftell(fp)) < 0)
2165 data->pagefd = dup(fileno(fp));
2166 if (data->pagefd == -1)
2170 fprintf (stderr, "can %sseek\n", can_seek ? "" : "NOT ");
2172 npages = (blobsz + BLOB_PAGESIZE - 1) / BLOB_PAGESIZE;
2174 data->num_pages = npages;
2175 data->pages = sat_malloc2(npages, sizeof(data->pages[0]));
2177 /* If we can't seek on our input we have to slurp in everything. */
2179 data->blob_store = sat_malloc(npages * BLOB_PAGESIZE);
2180 for (i = 0; i < npages; i++)
2182 unsigned int in_len = read_u32(fp);
2183 unsigned int compressed = in_len & 1;
2184 Attrblobpage *p = data->pages + i;
2187 fprintf (stderr, "page %d: len %d (%scompressed)\n",
2188 i, in_len, compressed ? "" : "not ");
2194 p->file_offset = cur_file_ofs;
2195 p->file_size = in_len * 2 + compressed;
2196 if (fseek(fp, in_len, SEEK_CUR) < 0)
2199 fprintf (stderr, "can't seek after we thought we can\n");
2200 /* We can't fall back to non-seeking behaviour as we already
2201 read over some data pages without storing them away. */
2202 data->error = SOLV_ERROR_EOF;
2203 close(data->pagefd);
2207 cur_file_ofs += in_len;
2211 unsigned int out_len;
2212 void *dest = data->blob_store + i * BLOB_PAGESIZE;
2213 p->mapped_at = i * BLOB_PAGESIZE;
2216 /* We can't seek, so suck everything in. */
2217 if (fread(compressed ? buf : dest, in_len, 1, fp) != 1)
2220 data->error = SOLV_ERROR_EOF;
2225 out_len = unchecked_decompress_buf(buf, in_len, dest, BLOB_PAGESIZE);
2226 if (out_len != BLOB_PAGESIZE && i < npages - 1)
2228 data->error = SOLV_ERROR_CORRUPT;
2237 repodata_disable_paging(Repodata *data)
2239 if (maybe_load_repodata(data, 0)
2241 load_page_range (data, 0, data->num_pages - 1);
2244 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: