2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
26 #include "poolid_private.h"
31 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
32 unsigned char *out, unsigned int out_len);
33 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
36 unsigned int out_len);
38 #define REPODATA_BLOCK 255
42 repodata_init(Repodata *data, Repo *repo, int localpool)
44 memset(data, 0, sizeof (*data));
46 data->localpool = localpool;
48 stringpool_init_empty(&data->spool);
49 data->keys = sat_calloc(1, sizeof(Repokey));
51 data->schemata = sat_calloc(1, sizeof(Id));
52 data->schemadata = sat_calloc(1, sizeof(Id));
54 data->schemadatalen = 1;
55 data->start = repo->start;
56 data->end = repo->end;
57 data->nextra = repo->nextra;
59 data->incoreoffset = sat_extend_resize(0, data->end - data->start, sizeof(Id), REPODATA_BLOCK);
60 data->extraoffset = sat_extend_resize(0, repo->nextra, sizeof(Id), REPODATA_BLOCK);
65 repodata_free(Repodata *data)
68 sat_free(data->schemata);
69 sat_free(data->schemadata);
71 sat_free(data->spool.strings);
72 sat_free(data->spool.stringspace);
73 sat_free(data->spool.stringhashtbl);
75 sat_free(data->dirpool.dirs);
76 sat_free(data->dirpool.dirtraverse);
78 sat_free(data->incoredata);
79 sat_free(data->incoreoffset);
80 sat_free(data->extraoffset);
81 sat_free(data->verticaloffset);
83 sat_free(data->blob_store);
84 sat_free(data->pages);
85 sat_free(data->mapped);
87 sat_free(data->vincore);
89 sat_free(data->attrs);
90 sat_free(data->extraattrs);
91 sat_free(data->attrdata);
92 sat_free(data->attriddata);
94 sat_free(data->location);
95 sat_free(data->addedfileprovides);
97 if (data->pagefd != -1)
102 data_skip_recursive(Repodata *data, unsigned char *dp, Repokey *key)
105 if (key->type != REPOKEY_TYPE_COUNTED)
106 return data_skip(dp, key->type);
107 dp = data_fetch(dp, &kv, key);
112 Id *keyp = data->schemadata + data->schemata[schema];
113 for (; *keyp; keyp++)
114 dp = data_skip_recursive(data, dp, data->keys + *keyp);
119 static unsigned char *
120 forward_to_key(Repodata *data, Id keyid, Id schemaid, unsigned char *dp)
124 keyp = data->schemadata + data->schemata[schemaid];
125 while ((k = *keyp++) != 0)
129 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
131 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip that offset */
132 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip that length */
135 if (data->keys[k].storage != KEY_STORAGE_INCORE)
137 dp = data_skip_recursive(data, dp, data->keys + k);
142 #define BLOB_PAGEBITS 15
143 #define BLOB_PAGESIZE (1 << BLOB_PAGEBITS)
145 static unsigned char *
146 load_page_range(Repodata *data, unsigned int pstart, unsigned int pend)
148 /* Make sure all pages from PSTART to PEND (inclusive) are loaded,
149 and are consecutive. Return a pointer to the mapping of PSTART. */
150 unsigned char buf[BLOB_PAGESIZE];
153 /* Quick check in case all pages are there already and consecutive. */
154 for (i = pstart; i <= pend; i++)
155 if (data->pages[i].mapped_at == -1
157 && data->pages[i].mapped_at
158 != data->pages[i-1].mapped_at + BLOB_PAGESIZE))
161 return data->blob_store + data->pages[pstart].mapped_at;
163 if (data->pagefd == -1)
166 /* Ensure that we can map the numbers of pages we need at all. */
167 if (pend - pstart + 1 > data->ncanmap)
169 unsigned int oldcan = data->ncanmap;
170 data->ncanmap = pend - pstart + 1;
171 if (data->ncanmap < 4)
173 data->mapped = sat_realloc2(data->mapped, data->ncanmap, sizeof(data->mapped[0]));
174 memset (data->mapped + oldcan, 0, (data->ncanmap - oldcan) * sizeof (data->mapped[0]));
175 data->blob_store = sat_realloc2(data->blob_store, data->ncanmap, BLOB_PAGESIZE);
177 fprintf (stderr, "PAGE: can map %d pages\n", data->ncanmap);
181 /* Now search for "cheap" space in our store. Space is cheap if it's either
182 free (very cheap) or contains pages we search for anyway. */
184 /* Setup cost array. */
185 unsigned int cost[data->ncanmap];
186 for (i = 0; i < data->ncanmap; i++)
188 unsigned int pnum = data->mapped[i];
194 Attrblobpage *p = data->pages + pnum;
195 assert (p->mapped_at != -1);
196 if (pnum >= pstart && pnum <= pend)
203 /* And search for cheapest space. */
204 unsigned int best_cost = -1;
205 unsigned int best = 0;
206 unsigned int same_cost = 0;
207 for (i = 0; i + pend - pstart < data->ncanmap; i++)
209 unsigned int c = cost[i];
211 for (j = 0; j < pend - pstart + 1; j++)
214 best_cost = c, best = i;
215 else if (c == best_cost)
217 /* A null cost won't become better. */
221 /* If all places have the same cost we would thrash on slot 0. Avoid
222 this by doing a round-robin strategy in this case. */
223 if (same_cost == data->ncanmap - pend + pstart - 1)
224 best = data->rr_counter++ % (data->ncanmap - pend + pstart);
226 /* So we want to map our pages from [best] to [best+pend-pstart].
227 Use a very simple strategy, which doesn't make the best use of
228 our resources, but works. Throw away all pages in that range
229 (even ours) then copy around ours (in case they were outside the
230 range) or read them in. */
231 for (i = best; i < best + pend - pstart + 1; i++)
233 unsigned int pnum = data->mapped[i];
235 /* If this page is exactly at the right place already,
236 no need to evict it. */
237 && pnum != pstart + i - best)
239 /* Evict this page. */
241 fprintf (stderr, "PAGE: evict page %d from %d\n", pnum, i);
245 data->pages[pnum].mapped_at = -1;
249 /* Everything is free now. Read in the pages we want. */
250 for (i = pstart; i <= pend; i++)
252 Attrblobpage *p = data->pages + i;
253 unsigned int pnum = i - pstart + best;
254 void *dest = data->blob_store + pnum * BLOB_PAGESIZE;
255 if (p->mapped_at != -1)
257 if (p->mapped_at != pnum * BLOB_PAGESIZE)
260 fprintf (stderr, "PAGECOPY: %d to %d\n", i, pnum);
262 /* Still mapped somewhere else, so just copy it from there. */
263 memcpy (dest, data->blob_store + p->mapped_at, BLOB_PAGESIZE);
264 data->mapped[p->mapped_at / BLOB_PAGESIZE] = 0;
269 unsigned int in_len = p->file_size;
270 unsigned int compressed = in_len & 1;
273 fprintf (stderr, "PAGEIN: %d to %d", i, pnum);
275 if (pread(data->pagefd, compressed ? buf : dest, in_len, p->file_offset) != in_len)
277 perror ("mapping pread");
282 unsigned int out_len;
283 out_len = unchecked_decompress_buf(buf, in_len,
284 dest, BLOB_PAGESIZE);
285 if (out_len != BLOB_PAGESIZE && i < data->num_pages - 1)
287 fprintf(stderr, "can't decompress\n");
291 fprintf (stderr, " (expand %d to %d)", in_len, out_len);
295 fprintf (stderr, "\n");
298 p->mapped_at = pnum * BLOB_PAGESIZE;
299 data->mapped[pnum] = i + 1;
301 return data->blob_store + best * BLOB_PAGESIZE;
304 static unsigned char *
305 make_vertical_available(Repodata *data, Repokey *key, Id off, Id len)
310 if (off >= data->lastverticaloffset)
312 off -= data->lastverticaloffset;
313 if (off + len > data->vincorelen)
315 return data->vincore + off;
317 if (off + len > key->size)
319 /* we now have the offset, go into vertical */
320 off += data->verticaloffset[key - data->keys];
321 /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */
322 dp = load_page_range(data, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
324 dp += off % BLOB_PAGESIZE;
328 static inline unsigned char *
329 get_data(Repodata *data, Repokey *key, unsigned char **dpp)
331 unsigned char *dp = *dpp;
335 if (key->storage == KEY_STORAGE_INCORE)
337 /* hmm, this is a bit expensive */
338 *dpp = data_skip_recursive(data, dp, key);
341 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
344 dp = data_read_id(dp, &off);
345 dp = data_read_id(dp, &len);
347 return make_vertical_available(data, key, off, len);
353 maybe_load_repodata(Repodata *data, Id *keyid)
355 if (data->state == REPODATA_STUB)
357 if (data->loadcallback)
361 /* key order may change when loading */
363 Id name = data->keys[*keyid].name;
364 Id type = data->keys[*keyid].type;
365 data->loadcallback(data);
366 if (data->state == REPODATA_AVAILABLE)
368 for (i = 1; i < data->nkeys; i++)
369 if (data->keys[i].name == name && data->keys[i].type == type)
378 data->loadcallback(data);
381 data->state = REPODATA_ERROR;
383 if (data->state == REPODATA_AVAILABLE)
385 data->state = REPODATA_ERROR;
389 static inline unsigned char*
390 entry2data(Repodata *data, Id entry)
393 return data->incoredata + data->extraoffset[-1 - entry];
395 return data->incoredata + data->incoreoffset[entry];
399 repodata_lookup_id(Repodata *data, Id entry, Id keyid)
406 if (!maybe_load_repodata(data, &keyid))
408 dp = entry2data(data, entry);
411 dp = data_read_id(dp, &schema);
412 /* make sure the schema of this solvable contains the key */
413 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
416 dp = forward_to_key(data, keyid, schema, dp);
417 key = data->keys + keyid;
418 dp = get_data(data, key, &dp);
421 if (key->type == REPOKEY_TYPE_CONSTANTID)
423 if (key->type != REPOKEY_TYPE_ID)
425 dp = data_read_id(dp, &id);
430 repodata_lookup_str(Repodata *data, Id entry, Id keyid)
437 if (!maybe_load_repodata(data, &keyid))
440 dp = entry2data(data, entry);
443 dp = data_read_id(dp, &schema);
444 /* make sure the schema of this solvable contains the key */
445 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
448 dp = forward_to_key(data, keyid, schema, dp);
449 key = data->keys + keyid;
450 dp = get_data(data, key, &dp);
453 if (key->type == REPOKEY_TYPE_STR)
454 return (const char *)dp;
455 if (key->type == REPOKEY_TYPE_CONSTANTID)
456 return id2str(data->repo->pool, key->size);
457 if (key->type == REPOKEY_TYPE_ID)
458 dp = data_read_id(dp, &id);
462 return data->spool.stringspace + data->spool.strings[id];
463 return id2str(data->repo->pool, id);
467 repodata_lookup_num(Repodata *data, Id entry, Id keyid, unsigned int *value)
477 if (!maybe_load_repodata(data, &keyid))
480 dp = entry2data(data, entry);
483 dp = data_read_id(dp, &schema);
484 /* make sure the schema of this solvable contains the key */
485 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
488 dp = forward_to_key(data, keyid, schema, dp);
489 key = data->keys + keyid;
490 dp = get_data(data, key, &dp);
493 if (key->type == REPOKEY_TYPE_NUM
494 || key->type == REPOKEY_TYPE_U32
495 || key->type == REPOKEY_TYPE_CONSTANT)
497 dp = data_fetch(dp, &kv, key);
505 repodata_lookup_void(Repodata *data, Id entry, Id keyid)
510 if (!maybe_load_repodata(data, &keyid))
512 dp = entry2data(data, entry);
515 dp = data_read_id(dp, &schema);
516 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
522 const unsigned char *
523 repodata_lookup_bin_checksum(Repodata *data, Id entry, Id keyid, Id *typep)
530 if (!maybe_load_repodata(data, &keyid))
532 dp = entry2data(data, entry);
535 dp = data_read_id(dp, &schema);
536 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
539 dp = forward_to_key(data, keyid, schema, dp);
540 key = data->keys + keyid;
542 return get_data(data, key, &dp);
546 repodata_search(Repodata *data, Id entry, Id keyname, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
550 Id k, keyid, *kp, *keyp;
551 unsigned char *dp, *ddp;
557 || !maybe_load_repodata(data, 0))
560 dp = entry2data(data, entry);
563 dp = data_read_id(dp, &schema);
564 keyp = data->schemadata + data->schemata[schema];
567 /* search in a specific key */
568 for (kp = keyp; (k = *kp++) != 0; )
569 if (data->keys[k].name == keyname)
573 dp = forward_to_key(data, k, schema, dp);
579 while ((keyid = *keyp++) != 0)
582 key = data->keys + keyid;
583 ddp = get_data(data, key, &dp);
586 ddp = data_fetch(ddp, &kv, key);
589 if (key->type == REPOKEY_TYPE_COUNTED)
592 int subschema = kv.id;
593 Repokey *countkey = key;
595 callback(cbdata, data->repo->pool->solvables + data->start + entry, data, countkey, &kv);
598 Id *kp = data->schemadata + data->schemata[subschema];
601 key = data->keys + *kp;
602 ddp = data_fetch(ddp, &kv, key);
605 callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv);
608 callback(cbdata, data->repo->pool->solvables + data->start + entry, data, countkey, &kv);
611 stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, countkey, &kv);
614 stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv);
616 while (!kv.eof && !stop);
617 if (onekey || stop > SEARCH_NEXT_KEY)
623 dataiterator_newdata(Dataiterator *di)
625 Id keyname = di->keyname;
626 Repodata *data = di->data;
629 if (data->state == REPODATA_STUB)
634 for (j = 1; j < data->nkeys; j++)
635 if (keyname == data->keys[j].name)
637 if (j == data->nkeys)
641 if (data->loadcallback)
642 data->loadcallback(data);
644 data->state = REPODATA_ERROR;
646 if (data->state == REPODATA_ERROR)
650 unsigned char *dp = data->incoredata;
654 dp += data->incoreoffset[di->solvid - data->start];
656 dp += data->extraoffset[-1 - di->solvid - data->extrastart];
657 dp = data_read_id(dp, &schema);
658 Id *keyp = data->schemadata + data->schemata[schema];
662 /* search in a specific key */
663 for (kp = keyp; (k = *kp++) != 0; )
664 if (data->keys[k].name == keyname)
668 dp = forward_to_key(data, k, schema, dp);
678 di->key = di->data->keys + keyid;
683 di->dp = get_data(di->data, di->key, &di->nextkeydp);
688 dataiterator_init(Dataiterator *di, Repo *repo, Id p, Id keyname,
689 const char *match, int flags)
695 di->flags |= __SEARCH_ONESOLVABLE;
696 di->data = repo->repodata - 1;
697 if (flags & SEARCH_NO_STORAGE_SOLVABLE)
704 di->solvid = repo->start - 1;
707 fprintf(stderr, "A repo contains the NULL solvable!\n");
710 di->data = repo->repodata + repo->nrepodata - 1;
715 if ((di->flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
719 /* We feed multiple lines eventually (e.g. authors or descriptions),
720 so set REG_NEWLINE. */
722 regcomp(&di->regex, di->match,
723 REG_EXTENDED | REG_NOSUB | REG_NEWLINE
724 | ((di->flags & SEARCH_NOCASE) ? REG_ICASE : 0));
726 if (di->regex_err != 0)
728 fprintf(stderr, "Given regex failed to compile: %s\n", di->match);
729 fprintf(stderr, "regcomp error code: %d\n", di->regex_err);
736 di->flags |= (di->flags & SEARCH_STRINGMASK) | SEARCH_STRING;
742 di->keyname = keyname;
743 static Id zeroid = 0;
751 /* FIXME factor and merge with repo_matchvalue */
753 dataiterator_match_int_real(Dataiterator *di, int flags, const void *vmatch)
755 KeyValue *kv = &di->kv;
756 const char *match = vmatch;
757 if ((flags & SEARCH_STRINGMASK) != 0)
759 switch (di->key->type)
761 case REPOKEY_TYPE_ID:
762 case REPOKEY_TYPE_IDARRAY:
763 if (di->data && di->data->localpool)
764 kv->str = stringpool_id2str(&di->data->spool, kv->id);
766 kv->str = id2str(di->repo->pool, kv->id);
768 case REPOKEY_TYPE_STR:
773 /* Maybe skip the kind specifier. Do this only for SOLVABLE attributes,
774 for the others we can't know if a colon separates a kind or not. */
775 if ((flags & SEARCH_SKIP_KIND)
776 && di->key->storage == KEY_STORAGE_SOLVABLE)
778 const char *s = strchr(kv->str, ':');
782 switch ((flags & SEARCH_STRINGMASK))
784 case SEARCH_SUBSTRING:
785 if (flags & SEARCH_NOCASE)
787 if (!strcasestr(kv->str, match))
792 if (!strstr(kv->str, match))
797 if (flags & SEARCH_NOCASE)
799 if (strcasecmp(match, kv->str))
804 if (strcmp(match, kv->str))
809 if (fnmatch(match, kv->str, (flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
813 if (regexec((const regex_t *)vmatch, kv->str, 0, NULL, 0))
824 dataiterator_match_int(Dataiterator *di)
826 if ((di->flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
827 return dataiterator_match_int_real(di, di->flags, &di->regex);
829 return dataiterator_match_int_real(di, di->flags, di->match);
833 dataiterator_match(Dataiterator *di, int flags, const void *vmatch)
835 return dataiterator_match_int_real(di, flags, vmatch);
838 static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = {
839 { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
840 { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
841 { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
842 { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
843 { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
844 { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
845 { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
846 { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
847 { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
848 { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
849 { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
850 { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
851 { SOLVABLE_FRESHENS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
852 { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE },
856 dataiterator_step(Dataiterator *di)
870 di->kv.eof = idp[1] ? 0 : 1;
876 Solvable *s = di->repo->pool->solvables + di->solvid;
877 int state = di->state;
878 di->key = solvablekeys + state - 1;
880 di->state = RPM_RPMDBID;
887 state = di->keyname - 1;
909 case SOLVABLE_VENDOR:
912 di->kv.id = s->vendor;
915 case SOLVABLE_PROVIDES:
916 di->idp = s->provides
917 ? di->repo->idarraydata + s->provides : 0;
919 case SOLVABLE_OBSOLETES:
920 di->idp = s->obsoletes
921 ? di->repo->idarraydata + s->obsoletes : 0;
923 case SOLVABLE_CONFLICTS:
924 di->idp = s->conflicts
925 ? di->repo->idarraydata + s->conflicts : 0;
927 case SOLVABLE_REQUIRES:
928 di->idp = s->requires
929 ? di->repo->idarraydata + s->requires : 0;
931 case SOLVABLE_RECOMMENDS:
932 di->idp = s->recommends
933 ? di->repo->idarraydata + s->recommends : 0;
935 case SOLVABLE_SUPPLEMENTS:
936 di->idp = s->supplements
937 ? di->repo->idarraydata + s->supplements : 0;
939 case SOLVABLE_SUGGESTS:
940 di->idp = s->suggests
941 ? di->repo->idarraydata + s->suggests : 0;
943 case SOLVABLE_ENHANCES:
944 di->idp = s->enhances
945 ? di->repo->idarraydata + s->enhances : 0;
947 case SOLVABLE_FRESHENS:
948 di->idp = s->freshens
949 ? di->repo->idarraydata + s->freshens : 0;
952 if (!di->repo->rpmdbid)
954 di->kv.num = di->repo->rpmdbid[di->solvid - di->repo->start];
958 di->data = di->repo->repodata - 1;
964 else if (di->subkeyp)
969 /* Send end-of-substruct. We are here only when we saw a
970 _COUNTED key one level up. Since then we didn't increment
971 ->keyp, so it still can be found at keyp[-1]. */
973 di->key = di->data->keys + di->keyp[-1];
976 else if (!(keyid = *di->subkeyp++))
978 /* Send end-of-element. See above for keyp[-1]. */
980 di->key = di->data->keys + di->keyp[-1];
981 di->subkeyp = di->data->schemadata + di->data->schemata[di->subschema];
986 di->key = di->data->keys + keyid;
987 di->dp = data_fetch(di->dp, &di->kv, di->key);
997 di->dp = data_fetch(di->dp, &di->kv, di->key);
1002 if (di->keyname || !(keyid = *di->keyp++))
1006 Repo *repo = di->repo;
1007 Repodata *data = ++di->data;
1008 if (data >= repo->repodata + repo->nrepodata)
1010 if (di->flags & __SEARCH_ONESOLVABLE)
1012 if (di->solvid >= 0)
1014 while (++di->solvid < repo->end)
1015 if (repo->pool->solvables[di->solvid].repo == repo)
1017 if (di->solvid >= repo->end)
1019 if (!(di->flags & SEARCH_EXTRA))
1022 if (di->solvid < -repo->nextra)
1029 if (di->solvid < -repo->nextra)
1032 Pool *pool = di->repo->pool;
1033 if (!(di->flags & SEARCH_ALL_REPOS)
1034 || di->repo == pool->repos[pool->nrepos - 1])
1037 for (i = 0; i < pool->nrepos; i++)
1038 if (di->repo == pool->repos[i])
1040 di->repo = pool->repos[i + 1];
1041 dataiterator_init(di, di->repo, 0, di->keyname, di->match, di->flags);
1045 di->data = repo->repodata - 1;
1047 || (di->flags & SEARCH_NO_STORAGE_SOLVABLE))
1049 static Id zeroid = 0;
1054 if ((di->solvid < 0 && (-1 - di->solvid) >= data->extrastart && (-1 - di->solvid) < (data->extrastart + data->nextra))
1055 || (di->solvid >= 0 && di->solvid >= data->start && di->solvid < data->end))
1057 dataiterator_newdata(di);
1065 di->key = di->data->keys + keyid;
1066 di->dp = get_data(di->data, di->key, &di->nextkeydp);
1068 di->dp = data_fetch(di->dp, &di->kv, di->key);
1070 if (di->key->type == REPOKEY_TYPE_COUNTED)
1072 di->subnum = di->kv.num;
1073 di->subschema = di->kv.id;
1075 di->subkeyp = di->data->schemadata + di->data->schemata[di->subschema];
1080 || dataiterator_match_int(di))
1087 dataiterator_skip_attribute(Dataiterator *di)
1091 /* This will make the next _step call to retrieve the next field. */
1096 dataiterator_skip_solvable(Dataiterator *di)
1098 /* We're done with this field. */
1100 /* And with solvable data. */
1102 /* And with all keys for this repodata and thing. */
1103 static Id zeroid = 0;
1105 /* And with all repodatas for this thing. */
1106 di->data = di->repo->repodata + di->repo->nrepodata - 1;
1107 /* Hence the next call to _step will retrieve the next thing. */
1111 dataiterator_skip_repo(Dataiterator *di)
1113 dataiterator_skip_solvable(di);
1114 /* We're done with all solvables and all extra things for this repo. */
1115 di->solvid = -1 - di->repo->nextra;
1119 dataiterator_jump_to_solvable(Dataiterator *di, Solvable *s)
1122 /* Simulate us being done with the solvable before the requested one. */
1123 dataiterator_skip_solvable(di);
1124 di->solvid = s - s->repo->pool->solvables;
1129 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1132 dataiterator_skip_solvable(di);
1133 di->solvid = repo->start - 1;
1136 /* extend repodata so that it includes solvables p */
1138 repodata_extend(Repodata *data, Id p)
1140 if (data->start == data->end)
1141 data->start = data->end = p;
1144 int old = data->end - data->start;
1145 int new = p - data->end + 1;
1148 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id), REPODATA_BLOCK);
1149 memset(data->attrs + old, 0, new * sizeof(Id));
1151 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
1152 memset(data->incoreoffset + old, 0, new * sizeof(Id));
1155 if (p < data->start)
1157 int old = data->end - data->start;
1158 int new = data->start - p;
1161 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id), REPODATA_BLOCK);
1162 memmove(data->attrs + new, data->attrs, old * sizeof(Id));
1163 memset(data->attrs, 0, new * sizeof(Id));
1165 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
1166 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
1167 memset(data->incoreoffset, 0, new * sizeof(Id));
1173 repodata_extend_extra(Repodata *data, int nextra)
1175 if (nextra <= data->nextra)
1177 if (data->extraattrs)
1179 data->extraattrs = sat_extend(data->extraattrs, data->nextra, nextra - data->nextra, sizeof(Id), REPODATA_BLOCK);
1180 memset(data->extraattrs + data->nextra, 0, (nextra - data->nextra) * sizeof (Id));
1182 data->extraoffset = sat_extend(data->extraoffset, data->nextra, nextra - data->nextra, sizeof(Id), REPODATA_BLOCK);
1183 memset(data->extraoffset + data->nextra, 0, (nextra - data->nextra) * sizeof(Id));
1184 data->nextra = nextra;
1188 repodata_extend_block(Repodata *data, Id start, Id num)
1192 if (!data->incoreoffset)
1194 data->incoreoffset = sat_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
1195 data->start = start;
1196 data->end = start + num;
1199 repodata_extend(data, start);
1201 repodata_extend(data, start + num - 1);
1204 /**********************************************************************/
1206 #define REPODATA_ATTRS_BLOCK 63
1207 #define REPODATA_ATTRDATA_BLOCK 1023
1208 #define REPODATA_ATTRIDDATA_BLOCK 63
1211 get_new_struct(Repodata *data)
1213 /* Make sure to never give out struct id 0. */
1216 data->structs = sat_extend(0, 0, 2, sizeof(Id *), REPODATA_BLOCK);
1217 data->structs[0] = 0;
1218 data->structs[1] = 0;
1222 data->structs = sat_extend(data->structs, data->nstructs, 1, sizeof(Id *), REPODATA_BLOCK);
1223 data->structs[data->nstructs] = 0;
1224 return data->nstructs++;
1228 repodata_get_handle_int(Repodata *data, Id entry)
1231 if (!data->attrs && entry >= 0)
1233 data->attrs = sat_calloc_block(data->end - data->start, sizeof(Id),
1236 else if (!data->extraattrs && entry < 0)
1237 data->extraattrs = sat_calloc_block(data->nextra, sizeof(Id), REPODATA_BLOCK);
1239 ap = &data->extraattrs[-1 - entry];
1241 ap = &data->attrs[entry];
1243 *ap = get_new_struct(data);
1248 repodata_get_handle(Repodata *data, Id entry)
1250 return repodata_get_handle_int(data, entry);
1254 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
1259 ap = data->structs[handle];
1263 for (pp = ap; *pp; pp += 2)
1264 /* Determine equality based on the name only, allows us to change
1265 type (when overwrite is set), and makes TYPE_CONSTANT work. */
1266 if (data->keys[*pp].name == data->keys[keyid].name)
1279 ap = sat_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
1280 data->structs[handle] = ap;
1288 repodata_set(Repodata *data, Id handle, Repokey *key, Id val)
1292 /* find key in keys */
1293 for (keyid = 1; keyid < data->nkeys; keyid++)
1294 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
1296 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
1300 if (keyid == data->nkeys)
1302 /* allocate new key */
1303 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
1304 data->keys[data->nkeys++] = *key;
1305 if (data->verticaloffset)
1307 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
1308 data->verticaloffset[data->nkeys - 1] = 0;
1311 repodata_insert_keyid(data, handle, keyid, val, 1);
1315 repodata_set_id(Repodata *data, Id handle, Id keyname, Id id)
1319 key.type = REPOKEY_TYPE_ID;
1321 key.storage = KEY_STORAGE_INCORE;
1322 repodata_set(data, handle, &key, id);
1326 repodata_set_num(Repodata *data, Id handle, Id keyname, unsigned int num)
1330 key.type = REPOKEY_TYPE_NUM;
1332 key.storage = KEY_STORAGE_INCORE;
1333 repodata_set(data, handle, &key, (Id)num);
1337 repodata_set_poolstr(Repodata *data, Id handle, Id keyname, const char *str)
1341 if (data->localpool)
1342 id = stringpool_str2id(&data->spool, str, 1);
1344 id = str2id(data->repo->pool, str, 1);
1346 key.type = REPOKEY_TYPE_ID;
1348 key.storage = KEY_STORAGE_INCORE;
1349 repodata_set(data, handle, &key, id);
1353 repodata_set_constant(Repodata *data, Id handle, Id keyname, unsigned int constant)
1357 key.type = REPOKEY_TYPE_CONSTANT;
1358 key.size = constant;
1359 key.storage = KEY_STORAGE_INCORE;
1360 repodata_set(data, handle, &key, 0);
1364 repodata_set_constantid(Repodata *data, Id handle, Id keyname, Id id)
1368 key.type = REPOKEY_TYPE_CONSTANTID;
1370 key.storage = KEY_STORAGE_INCORE;
1371 repodata_set(data, handle, &key, 0);
1375 repodata_set_void(Repodata *data, Id handle, Id keyname)
1379 key.type = REPOKEY_TYPE_VOID;
1381 key.storage = KEY_STORAGE_INCORE;
1382 repodata_set(data, handle, &key, 0);
1386 repodata_set_str(Repodata *data, Id handle, Id keyname, const char *str)
1391 l = strlen(str) + 1;
1393 key.type = REPOKEY_TYPE_STR;
1395 key.storage = KEY_STORAGE_INCORE;
1396 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1397 memcpy(data->attrdata + data->attrdatalen, str, l);
1398 repodata_set(data, handle, &key, data->attrdatalen);
1399 data->attrdatalen += l;
1403 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
1408 pp = data->structs[handle];
1410 for (; *pp; pp += 2)
1411 if (data->keys[*pp].name == keyname && data->keys[*pp].type == keytype)
1415 /* not found. allocate new key */
1420 key.storage = KEY_STORAGE_INCORE;
1421 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1422 repodata_set(data, handle, &key, data->attriddatalen);
1426 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
1427 oldsize += entrysize;
1428 if (ida + 1 == data->attriddata + data->attriddatalen)
1430 /* this was the last entry, just append it */
1431 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1432 data->attriddatalen--; /* overwrite terminating 0 */
1436 /* too bad. move to back. */
1437 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1438 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
1439 pp[1] = data->attriddatalen;
1440 data->attriddatalen += oldsize;
1445 checksumtype2len(Id type)
1449 case REPOKEY_TYPE_MD5:
1451 case REPOKEY_TYPE_SHA1:
1453 case REPOKEY_TYPE_SHA256:
1454 return SIZEOF_SHA256;
1461 repodata_set_bin_checksum(Repodata *data, Id handle, Id keyname, Id type,
1462 const unsigned char *str)
1465 int l = checksumtype2len(type);
1472 key.storage = KEY_STORAGE_INCORE;
1473 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1474 memcpy(data->attrdata + data->attrdatalen, str, l);
1475 repodata_set(data, handle, &key, data->attrdatalen);
1476 data->attrdatalen += l;
1480 hexstr2bytes(unsigned char *buf, const char *str, int buflen)
1483 for (i = 0; i < buflen; i++)
1485 #define c2h(c) (((c)>='0' && (c)<='9') ? ((c)-'0') \
1486 : ((c)>='a' && (c)<='f') ? ((c)-'a'+10) \
1487 : ((c)>='A' && (c)<='F') ? ((c)-'A'+10) \
1498 buf[i] = (buf[i] << 4) | v;
1505 repodata_set_checksum(Repodata *data, Id handle, Id keyname, Id type,
1508 unsigned char buf[64];
1509 int l = checksumtype2len(type);
1513 if (hexstr2bytes(buf, str, l) != l)
1515 fprintf(stderr, "Invalid hex character in '%s'\n", str);
1518 repodata_set_bin_checksum(data, handle, keyname, type, buf);
1522 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
1527 l = checksumtype2len(type);
1530 s = str = pool_alloctmpspace(data->repo->pool, 2 * l + 1);
1531 for (i = 0; i < l; i++)
1533 unsigned char v = buf[i];
1534 unsigned char w = v >> 4;
1535 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
1537 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
1544 repodata_globalize_id(Repodata *data, Id id)
1546 if (!data || !data->localpool)
1548 return str2id(data->repo->pool, stringpool_id2str(&data->spool, id), 1);
1552 repodata_add_dirnumnum(Repodata *data, Id handle, Id keyname, Id dir, Id num, Id num2)
1556 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", handle, dir, num, num2, data->attriddatalen);
1558 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
1559 data->attriddata[data->attriddatalen++] = dir;
1560 data->attriddata[data->attriddatalen++] = num;
1561 data->attriddata[data->attriddatalen++] = num2;
1562 data->attriddata[data->attriddatalen++] = 0;
1566 repodata_add_dirstr(Repodata *data, Id handle, Id keyname, Id dir, const char *str)
1572 l = strlen(str) + 1;
1573 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1574 memcpy(data->attrdata + data->attrdatalen, str, l);
1575 stroff = data->attrdatalen;
1576 data->attrdatalen += l;
1579 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", handle, dir, str, data->attriddatalen);
1581 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
1582 data->attriddata[data->attriddatalen++] = dir;
1583 data->attriddata[data->attriddatalen++] = stroff;
1584 data->attriddata[data->attriddatalen++] = 0;
1588 repodata_add_idarray(Repodata *data, Id handle, Id keyname, Id id)
1591 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", handle, id, data->attriddatalen);
1593 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_IDARRAY, 1);
1594 data->attriddata[data->attriddatalen++] = id;
1595 data->attriddata[data->attriddatalen++] = 0;
1599 repodata_add_poolstr_array(Repodata *data, Id handle, Id keyname,
1603 if (data->localpool)
1604 id = stringpool_str2id(&data->spool, str, 1);
1606 id = str2id(data->repo->pool, str, 1);
1607 repodata_add_idarray(data, handle, keyname, id);
1611 repodata_create_struct(Repodata *data, Id handle, Id keyname)
1613 Id newhandle = get_new_struct(data);
1614 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_COUNTED, 1);
1615 data->attriddata[data->attriddatalen++] = newhandle;
1616 data->attriddata[data->attriddatalen++] = 0;
1621 repodata_merge_attrs(Repodata *data, Id dest, Id src)
1625 || !(keyp = data->structs[src < 0
1626 ? data->extraattrs[-1 - src]
1627 : data->attrs[src]]))
1629 dest = repodata_get_handle_int(data, dest);
1630 for (; *keyp; keyp += 2)
1631 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
1634 /*********************************/
1636 /* unify with repo_write! */
1638 #define EXTDATA_BLOCK 1023
1639 #define SCHEMATA_BLOCK 31
1640 #define SCHEMATADATA_BLOCK 255
1648 data_addid(struct extdata *xd, Id x)
1651 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
1652 dp = xd->buf + xd->len;
1657 *dp++ = (x >> 28) | 128;
1659 *dp++ = (x >> 21) | 128;
1660 *dp++ = (x >> 14) | 128;
1663 *dp++ = (x >> 7) | 128;
1665 xd->len = dp - xd->buf;
1669 data_addideof(struct extdata *xd, Id x, int eof)
1672 x = (x & 63) | ((x & ~63) << 1);
1673 data_addid(xd, (eof ? x: x | 64));
1677 data_addblob(struct extdata *xd, unsigned char *blob, int len)
1679 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
1680 memcpy(xd->buf + xd->len, blob, len);
1684 /*********************************/
1687 addschema_prepare(Repodata *data, Id *schematacache)
1692 memset(schematacache, 0, 256 * sizeof(Id));
1693 for (i = 0; i < data->nschemata; i++)
1695 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
1698 schematacache[h] = i + 1;
1700 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
1701 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
1705 addschema(Repodata *data, Id *schema, Id *schematacache)
1710 for (sp = schema, len = 0, h = 0; *sp; len++)
1715 cid = schematacache[h];
1719 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1721 /* cache conflict */
1722 for (cid = 0; cid < data->nschemata; cid++)
1723 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1726 /* a new one. make room. */
1727 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
1728 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
1730 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
1731 data->schemata[data->nschemata] = data->schemadatalen;
1732 data->schemadatalen += len;
1733 schematacache[h] = data->nschemata + 1;
1735 fprintf(stderr, "addschema: new schema\n");
1737 return data->nschemata++;
1741 repodata_serialize_key(Repodata *data, struct extdata *newincore,
1742 struct extdata *newvincore,
1743 Id *schema, Id *schematacache,
1744 Repokey *key, Id val)
1746 /* Otherwise we have a new value. Parse it into the internal
1750 unsigned int oldvincorelen = 0;
1754 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1757 oldvincorelen = xd->len;
1761 case REPOKEY_TYPE_VOID:
1762 case REPOKEY_TYPE_CONSTANT:
1763 case REPOKEY_TYPE_CONSTANTID:
1765 case REPOKEY_TYPE_STR:
1766 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
1768 case REPOKEY_TYPE_MD5:
1769 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
1771 case REPOKEY_TYPE_SHA1:
1772 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
1774 case REPOKEY_TYPE_ID:
1775 case REPOKEY_TYPE_NUM:
1776 case REPOKEY_TYPE_DIR:
1777 data_addid(xd, val);
1779 case REPOKEY_TYPE_IDARRAY:
1780 for (ida = data->attriddata + val; *ida; ida++)
1781 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
1783 case REPOKEY_TYPE_DIRNUMNUMARRAY:
1784 for (ida = data->attriddata + val; *ida; ida += 3)
1786 data_addid(xd, ida[0]);
1787 data_addid(xd, ida[1]);
1788 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
1791 case REPOKEY_TYPE_DIRSTRARRAY:
1792 for (ida = data->attriddata + val; *ida; ida += 2)
1794 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
1795 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
1798 case REPOKEY_TYPE_COUNTED:
1802 for (ida = data->attriddata + val; *ida; ida++)
1805 fprintf(stderr, "serialize struct %d\n", *ida);
1808 Id *kp = data->structs[*ida];
1815 fprintf(stderr, " %s:%d\n", id2str(data->repo->pool, data->keys[*kp].name), kp[1]);
1821 schemaid = addschema(data, schema, schematacache);
1822 else if (schemaid != addschema(data, schema, schematacache))
1824 fprintf(stderr, " not yet implemented: substructs with different schemas\n");
1828 fprintf(stderr, " schema %d\n", schemaid);
1833 data_addid(xd, num);
1834 data_addid(xd, schemaid);
1835 for (ida = data->attriddata + val; *ida; ida++)
1837 Id *kp = data->structs[*ida];
1842 repodata_serialize_key(data, newincore, newvincore,
1843 schema, schematacache,
1844 data->keys + *kp, kp[1]);
1850 fprintf(stderr, "don't know how to handle type %d\n", key->type);
1853 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1855 /* put offset/len in incore */
1856 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
1857 oldvincorelen = xd->len - oldvincorelen;
1858 data_addid(newincore, oldvincorelen);
1863 repodata_internalize(Repodata *data)
1867 Id schematacache[256];
1868 Id schemaid, *schema, *sp, oldschema, *keyp, *seen;
1869 unsigned char *dp, *ndp;
1870 int newschema, oldcount;
1871 struct extdata newincore;
1872 struct extdata newvincore;
1874 if (!data->attrs && !data->extraattrs)
1877 newvincore.buf = data->vincore;
1878 newvincore.len = data->vincorelen;
1880 schema = sat_malloc2(data->nkeys, sizeof(Id));
1881 seen = sat_malloc2(data->nkeys, sizeof(Id));
1883 /* Merge the data already existing (in data->schemata, ->incoredata and
1884 friends) with the new attributes in data->attrs[]. */
1885 nentry = data->end - data->start;
1886 addschema_prepare(data, schematacache);
1887 memset(&newincore, 0, sizeof(newincore));
1888 data_addid(&newincore, 0);
1891 for (entry = data->extraattrs ? -data->nextra : 0; entry < nentry; entry++)
1894 memset(seen, 0, data->nkeys * sizeof(Id));
1896 dp = entry2data(data, entry);
1897 if (data->incoredata)
1898 dp = data_read_id(dp, &oldschema);
1902 fprintf(stderr, "oldschema %d\n", oldschema);
1903 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
1904 fprintf(stderr, "schemadata %p\n", data->schemadata);
1906 /* seen: -1: old data 0: skipped >0: id + 1 */
1909 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
1913 fprintf(stderr, "Inconsistent old data (key occured twice).\n");
1920 handle = entry < 0 ? data->extraattrs[-1 - entry] : data->attrs[entry];
1921 keyp = data->structs[handle];
1923 for (; *keyp; keyp += 2)
1930 seen[*keyp] = keyp[1] + 1;
1934 /* Ideally we'd like to sort the new schema here, to ensure
1935 schema equality independend of the ordering. We can't do that
1936 yet. For once see below (old ids need to come before new ids).
1937 An additional difficulty is that we also need to move
1938 the values with the keys. */
1939 schemaid = addschema(data, schema, schematacache);
1941 schemaid = oldschema;
1944 /* Now create data blob. We walk through the (possibly new) schema
1945 and either copy over old data, or insert the new. */
1946 /* XXX Here we rely on the fact that the (new) schema has the form
1947 o1 o2 o3 o4 ... | n1 n2 n3 ...
1948 (oX being the old keyids (possibly overwritten), and nX being
1949 the new keyids). This rules out sorting the keyids in order
1950 to ensure a small schema count. */
1952 data->extraoffset[-1 - entry] = newincore.len;
1954 data->incoreoffset[entry] = newincore.len;
1955 data_addid(&newincore, schemaid);
1956 for (keyp = data->schemadata + data->schemata[schemaid]; *keyp; keyp++)
1958 key = data->keys + *keyp;
1960 fprintf(stderr, "internalize %d:%s:%s\n", entry, id2str(data->repo->pool, key->name), id2str(data->repo->pool, key->type));
1965 /* Skip the data associated with this old key. */
1966 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1968 ndp = data_skip(dp, REPOKEY_TYPE_ID);
1969 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
1971 else if (key->storage == KEY_STORAGE_INCORE)
1972 ndp = data_skip_recursive(data, dp, key);
1975 if (seen[*keyp] == -1)
1977 /* If this key was an old one _and_ was not overwritten with
1978 a different value copy over the old value (we skipped it
1981 data_addblob(&newincore, dp, ndp - dp);
1984 else if (seen[*keyp])
1986 /* Otherwise we have a new value. Parse it into the internal
1988 repodata_serialize_key(data, &newincore, &newvincore,
1989 schema, schematacache,
1990 key, seen[*keyp] - 1);
1994 if (data->structs[handle])
1995 data->structs[handle] = sat_free(data->structs[handle]);
1997 for (entry = 0; entry < data->nstructs; entry++)
1998 if (data->structs[entry])
1999 sat_free(data->structs[entry]);
2000 data->structs = sat_free(data->structs);
2004 sat_free(data->incoredata);
2005 data->incoredata = newincore.buf;
2006 data->incoredatalen = newincore.len;
2007 data->incoredatafree = 0;
2009 sat_free(data->vincore);
2010 data->vincore = newvincore.buf;
2011 data->vincorelen = newvincore.len;
2013 data->attrs = sat_free(data->attrs);
2014 data->extraattrs = sat_free(data->extraattrs);
2015 data->attrdata = sat_free(data->attrdata);
2016 data->attriddata = sat_free(data->attriddata);
2017 data->attrdatalen = 0;
2018 data->attriddatalen = 0;
2022 repodata_str2dir(Repodata *data, const char *dir, int create)
2028 while (*dir == '/' && dir[1] == '/')
2030 if (*dir == '/' && !dir[1])
2034 dire = strchrnul(dir, '/');
2035 if (data->localpool)
2036 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
2038 id = strn2id(data->repo->pool, dir, dire - dir, create);
2041 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
2054 repodata_dir2str(Repodata *data, Id did, const char *suf)
2056 Pool *pool = data->repo->pool;
2063 return suf ? suf : "";
2067 comp = dirpool_compid(&data->dirpool, parent);
2068 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
2070 parent = dirpool_parent(&data->dirpool, parent);
2075 l += strlen(suf) + 1;
2076 p = pool_alloctmpspace(pool, l + 1) + l;
2087 comp = dirpool_compid(&data->dirpool, parent);
2088 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
2091 strncpy(p, comps, l);
2092 parent = dirpool_parent(&data->dirpool, parent);
2100 repodata_compress_page(unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max)
2102 return compress_buf(page, len, cpage, max);
2105 #define SOLV_ERROR_EOF 3
2107 static inline unsigned int
2113 for (i = 0; i < 4; i++)
2123 #define SOLV_ERROR_EOF 3
2124 #define SOLV_ERROR_CORRUPT 6
2126 /* Try to either setup on-demand paging (using FP as backing
2127 file), or in case that doesn't work (FP not seekable) slurps in
2128 all pages and deactivates paging. */
2130 repodata_read_or_setup_pages(Repodata *data, unsigned int pagesz, unsigned int blobsz)
2132 FILE *fp = data->fp;
2133 unsigned int npages;
2135 unsigned int can_seek;
2137 unsigned char buf[BLOB_PAGESIZE];
2139 if (pagesz != BLOB_PAGESIZE)
2141 /* We could handle this by slurping in everything. */
2142 data->error = SOLV_ERROR_CORRUPT;
2146 if ((cur_file_ofs = ftell(fp)) < 0)
2150 data->pagefd = dup(fileno(fp));
2151 if (data->pagefd == -1)
2155 fprintf (stderr, "can %sseek\n", can_seek ? "" : "NOT ");
2157 npages = (blobsz + BLOB_PAGESIZE - 1) / BLOB_PAGESIZE;
2159 data->num_pages = npages;
2160 data->pages = sat_malloc2(npages, sizeof(data->pages[0]));
2162 /* If we can't seek on our input we have to slurp in everything. */
2164 data->blob_store = sat_malloc(npages * BLOB_PAGESIZE);
2165 for (i = 0; i < npages; i++)
2167 unsigned int in_len = read_u32(fp);
2168 unsigned int compressed = in_len & 1;
2169 Attrblobpage *p = data->pages + i;
2172 fprintf (stderr, "page %d: len %d (%scompressed)\n",
2173 i, in_len, compressed ? "" : "not ");
2179 p->file_offset = cur_file_ofs;
2180 p->file_size = in_len * 2 + compressed;
2181 if (fseek(fp, in_len, SEEK_CUR) < 0)
2184 fprintf (stderr, "can't seek after we thought we can\n");
2185 /* We can't fall back to non-seeking behaviour as we already
2186 read over some data pages without storing them away. */
2187 data->error = SOLV_ERROR_EOF;
2188 close(data->pagefd);
2192 cur_file_ofs += in_len;
2196 unsigned int out_len;
2197 void *dest = data->blob_store + i * BLOB_PAGESIZE;
2198 p->mapped_at = i * BLOB_PAGESIZE;
2201 /* We can't seek, so suck everything in. */
2202 if (fread(compressed ? buf : dest, in_len, 1, fp) != 1)
2205 data->error = SOLV_ERROR_EOF;
2210 out_len = unchecked_decompress_buf(buf, in_len, dest, BLOB_PAGESIZE);
2211 if (out_len != BLOB_PAGESIZE && i < npages - 1)
2213 data->error = SOLV_ERROR_CORRUPT;
2222 repodata_disable_paging(Repodata *data)
2224 if (maybe_load_repodata(data, 0)
2226 load_page_range (data, 0, data->num_pages - 1);
2229 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: