2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
26 #include "poolid_private.h"
31 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
32 unsigned char *out, unsigned int out_len);
33 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
36 unsigned int out_len);
38 #define REPODATA_BLOCK 255
42 repodata_init(Repodata *data, Repo *repo, int localpool)
44 memset(data, 0, sizeof (*data));
46 data->localpool = localpool;
48 stringpool_init_empty(&data->spool);
49 data->keys = sat_calloc(1, sizeof(Repokey));
51 data->schemata = sat_calloc(1, sizeof(Id));
52 data->schemadata = sat_calloc(1, sizeof(Id));
54 data->schemadatalen = 1;
55 data->start = repo->start;
56 data->end = repo->end;
57 data->incoreoffset = sat_extend_resize(0, data->end - data->start, sizeof(Id), REPODATA_BLOCK);
62 repodata_free(Repodata *data)
65 sat_free(data->schemata);
66 sat_free(data->schemadata);
68 sat_free(data->spool.strings);
69 sat_free(data->spool.stringspace);
70 sat_free(data->spool.stringhashtbl);
72 sat_free(data->dirpool.dirs);
73 sat_free(data->dirpool.dirtraverse);
75 sat_free(data->incoredata);
76 sat_free(data->incoreoffset);
77 sat_free(data->verticaloffset);
79 sat_free(data->blob_store);
80 sat_free(data->pages);
81 sat_free(data->mapped);
83 sat_free(data->vincore);
85 sat_free(data->attrs);
86 sat_free(data->attrdata);
87 sat_free(data->attriddata);
89 sat_free(data->location);
91 if (data->pagefd != -1)
95 static unsigned char *
96 forward_to_key(Repodata *data, Id keyid, Id schemaid, unsigned char *dp)
100 keyp = data->schemadata + data->schemata[schemaid];
101 while ((k = *keyp++) != 0)
105 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
107 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip that offset */
108 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip that length */
111 if (data->keys[k].storage != KEY_STORAGE_INCORE)
113 dp = data_skip(dp, data->keys[k].type);
118 #define BLOB_PAGEBITS 15
119 #define BLOB_PAGESIZE (1 << BLOB_PAGEBITS)
121 static unsigned char *
122 load_page_range(Repodata *data, unsigned int pstart, unsigned int pend)
124 /* Make sure all pages from PSTART to PEND (inclusive) are loaded,
125 and are consecutive. Return a pointer to the mapping of PSTART. */
126 unsigned char buf[BLOB_PAGESIZE];
129 /* Quick check in case all pages are there already and consecutive. */
130 for (i = pstart; i <= pend; i++)
131 if (data->pages[i].mapped_at == -1
133 && data->pages[i].mapped_at
134 != data->pages[i-1].mapped_at + BLOB_PAGESIZE))
137 return data->blob_store + data->pages[pstart].mapped_at;
139 if (data->pagefd == -1)
142 /* Ensure that we can map the numbers of pages we need at all. */
143 if (pend - pstart + 1 > data->ncanmap)
145 unsigned int oldcan = data->ncanmap;
146 data->ncanmap = pend - pstart + 1;
147 if (data->ncanmap < 4)
149 data->mapped = sat_realloc2(data->mapped, data->ncanmap, sizeof(data->mapped[0]));
150 memset (data->mapped + oldcan, 0, (data->ncanmap - oldcan) * sizeof (data->mapped[0]));
151 data->blob_store = sat_realloc2(data->blob_store, data->ncanmap, BLOB_PAGESIZE);
153 fprintf (stderr, "PAGE: can map %d pages\n", data->ncanmap);
157 /* Now search for "cheap" space in our store. Space is cheap if it's either
158 free (very cheap) or contains pages we search for anyway. */
160 /* Setup cost array. */
161 unsigned int cost[data->ncanmap];
162 for (i = 0; i < data->ncanmap; i++)
164 unsigned int pnum = data->mapped[i];
170 Attrblobpage *p = data->pages + pnum;
171 assert (p->mapped_at != -1);
172 if (pnum >= pstart && pnum <= pend)
179 /* And search for cheapest space. */
180 unsigned int best_cost = -1;
181 unsigned int best = 0;
182 unsigned int same_cost = 0;
183 for (i = 0; i + pend - pstart < data->ncanmap; i++)
185 unsigned int c = cost[i];
187 for (j = 0; j < pend - pstart + 1; j++)
190 best_cost = c, best = i;
191 else if (c == best_cost)
193 /* A null cost won't become better. */
197 /* If all places have the same cost we would thrash on slot 0. Avoid
198 this by doing a round-robin strategy in this case. */
199 if (same_cost == data->ncanmap - pend + pstart - 1)
200 best = data->rr_counter++ % (data->ncanmap - pend + pstart);
202 /* So we want to map our pages from [best] to [best+pend-pstart].
203 Use a very simple strategy, which doesn't make the best use of
204 our resources, but works. Throw away all pages in that range
205 (even ours) then copy around ours (in case they were outside the
206 range) or read them in. */
207 for (i = best; i < best + pend - pstart + 1; i++)
209 unsigned int pnum = data->mapped[i];
211 /* If this page is exactly at the right place already,
212 no need to evict it. */
213 && pnum != pstart + i - best)
215 /* Evict this page. */
217 fprintf (stderr, "PAGE: evict page %d from %d\n", pnum, i);
221 data->pages[pnum].mapped_at = -1;
225 /* Everything is free now. Read in the pages we want. */
226 for (i = pstart; i <= pend; i++)
228 Attrblobpage *p = data->pages + i;
229 unsigned int pnum = i - pstart + best;
230 void *dest = data->blob_store + pnum * BLOB_PAGESIZE;
231 if (p->mapped_at != -1)
233 if (p->mapped_at != pnum * BLOB_PAGESIZE)
236 fprintf (stderr, "PAGECOPY: %d to %d\n", i, pnum);
238 /* Still mapped somewhere else, so just copy it from there. */
239 memcpy (dest, data->blob_store + p->mapped_at, BLOB_PAGESIZE);
240 data->mapped[p->mapped_at / BLOB_PAGESIZE] = 0;
245 unsigned int in_len = p->file_size;
246 unsigned int compressed = in_len & 1;
249 fprintf (stderr, "PAGEIN: %d to %d", i, pnum);
251 if (pread(data->pagefd, compressed ? buf : dest, in_len, p->file_offset) != in_len)
253 perror ("mapping pread");
258 unsigned int out_len;
259 out_len = unchecked_decompress_buf(buf, in_len,
260 dest, BLOB_PAGESIZE);
261 if (out_len != BLOB_PAGESIZE && i < data->num_pages - 1)
263 fprintf(stderr, "can't decompress\n");
267 fprintf (stderr, " (expand %d to %d)", in_len, out_len);
271 fprintf (stderr, "\n");
274 p->mapped_at = pnum * BLOB_PAGESIZE;
275 data->mapped[pnum] = i + 1;
277 return data->blob_store + best * BLOB_PAGESIZE;
280 static unsigned char *
281 make_vertical_available(Repodata *data, Repokey *key, Id off, Id len)
286 if (off >= data->lastverticaloffset)
288 off -= data->lastverticaloffset;
289 if (off + len > data->vincorelen)
291 return data->vincore + off;
293 if (off + len > key->size)
295 /* we now have the offset, go into vertical */
296 off += data->verticaloffset[key - data->keys];
297 /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */
298 dp = load_page_range(data, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
300 dp += off % BLOB_PAGESIZE;
304 static inline unsigned char *
305 get_data(Repodata *data, Repokey *key, unsigned char **dpp)
307 unsigned char *dp = *dpp;
311 if (key->storage == KEY_STORAGE_INCORE)
313 /* hmm, this is a bit expensive */
314 *dpp = data_skip(dp, key->type);
317 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
320 dp = data_read_id(dp, &off);
321 dp = data_read_id(dp, &len);
323 return make_vertical_available(data, key, off, len);
329 maybe_load_repodata(Repodata *data, Id *keyid)
331 if (data->state == REPODATA_STUB)
333 if (data->loadcallback)
337 /* key order may change when loading */
339 Id name = data->keys[*keyid].name;
340 Id type = data->keys[*keyid].type;
341 data->loadcallback(data);
342 if (data->state == REPODATA_AVAILABLE)
344 for (i = 1; i < data->nkeys; i++)
345 if (data->keys[i].name == name && data->keys[i].type == type)
354 data->loadcallback(data);
357 data->state = REPODATA_ERROR;
359 if (data->state == REPODATA_AVAILABLE)
361 data->state = REPODATA_ERROR;
366 repodata_lookup_str(Repodata *data, Id entry, Id keyid)
373 if (!maybe_load_repodata(data, &keyid))
376 dp = data->incoredata + data->incoreoffset[entry];
377 dp = data_read_id(dp, &schema);
378 /* make sure the schema of this solvable contains the key */
379 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
382 dp = forward_to_key(data, keyid, schema, dp);
383 key = data->keys + keyid;
384 dp = get_data(data, key, &dp);
387 if (key->type == REPOKEY_TYPE_STR)
388 return (const char *)dp;
389 if (key->type == REPOKEY_TYPE_CONSTANTID)
390 return id2str(data->repo->pool, key->size);
391 if (key->type == REPOKEY_TYPE_ID)
392 dp = data_read_id(dp, &id);
396 return data->spool.stringspace + data->spool.strings[id];
397 return id2str(data->repo->pool, id);
401 repodata_lookup_num(Repodata *data, Id entry, Id keyid, unsigned int *value)
411 if (!maybe_load_repodata(data, &keyid))
414 dp = data->incoredata + data->incoreoffset[entry];
415 dp = data_read_id(dp, &schema);
416 /* make sure the schema of this solvable contains the key */
417 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
420 dp = forward_to_key(data, keyid, schema, dp);
421 key = data->keys + keyid;
422 dp = get_data(data, key, &dp);
425 if (key->type == REPOKEY_TYPE_NUM
426 || key->type == REPOKEY_TYPE_U32
427 || key->type == REPOKEY_TYPE_CONSTANT)
429 dp = data_fetch(dp, &kv, key);
437 repodata_lookup_void(Repodata *data, Id entry, Id keyid)
442 if (!maybe_load_repodata(data, &keyid))
444 dp = data->incoredata + data->incoreoffset[entry];
445 dp = data_read_id(dp, &schema);
446 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
453 repodata_search(Repodata *data, Id entry, Id keyname, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
457 Id k, keyid, *kp, *keyp;
458 unsigned char *dp, *ddp;
463 if (!maybe_load_repodata(data, 0))
466 dp = data->incoredata + data->incoreoffset[entry];
467 dp = data_read_id(dp, &schema);
468 keyp = data->schemadata + data->schemata[schema];
471 /* search in a specific key */
472 for (kp = keyp; (k = *kp++) != 0; )
473 if (data->keys[k].name == keyname)
477 dp = forward_to_key(data, k, schema, dp);
483 while ((keyid = *keyp++) != 0)
486 key = data->keys + keyid;
487 ddp = get_data(data, key, &dp);
490 ddp = data_fetch(ddp, &kv, key);
493 stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv);
495 while (!kv.eof && !stop);
496 if (onekey || stop > SEARCH_NEXT_KEY)
502 dataiterator_newdata(Dataiterator *di)
504 Id keyname = di->keyname;
505 Repodata *data = di->data;
508 if (data->state == REPODATA_STUB)
513 for (j = 1; j < data->nkeys; j++)
514 if (keyname == data->keys[j].name)
516 if (j == data->nkeys)
520 if (data->loadcallback)
521 data->loadcallback(data);
523 data->state = REPODATA_ERROR;
525 if (data->state == REPODATA_ERROR)
529 unsigned char *dp = data->incoredata + data->incoreoffset[di->solvid - data->start];
530 dp = data_read_id(dp, &schema);
531 Id *keyp = data->schemadata + data->schemata[schema];
535 /* search in a specific key */
536 for (kp = keyp; (k = *kp++) != 0; )
537 if (data->keys[k].name == keyname)
541 dp = forward_to_key(data, k, schema, dp);
551 di->key = di->data->keys + keyid;
556 di->dp = get_data(di->data, di->key, &di->nextkeydp);
561 dataiterator_init(Dataiterator *di, Repo *repo, Id p, Id keyname,
562 const char *match, int flags)
568 di->flags |= __SEARCH_ONESOLVABLE;
569 di->data = repo->repodata - 1;
570 if (flags & SEARCH_NO_STORAGE_SOLVABLE)
577 di->solvid = repo->start - 1;
578 di->data = repo->repodata + repo->nrepodata - 1;
582 di->keyname = keyname;
583 static Id zeroid = 0;
590 /* FIXME factor and merge with repo_matchvalue */
592 dataiterator_match(Dataiterator *di, KeyValue *kv)
594 int flags = di->flags;
596 if ((flags & SEARCH_STRINGMASK) != 0)
598 switch (di->key->type)
600 case REPOKEY_TYPE_ID:
601 case REPOKEY_TYPE_IDARRAY:
602 if (di->data && di->data->localpool)
603 kv->str = stringpool_id2str(&di->data->spool, kv->id);
605 kv->str = id2str(di->repo->pool, kv->id);
607 case REPOKEY_TYPE_STR:
612 switch ((flags & SEARCH_STRINGMASK))
614 case SEARCH_SUBSTRING:
615 if (flags & SEARCH_NOCASE)
617 if (!strcasestr(kv->str, di->match))
622 if (!strstr(kv->str, di->match))
627 if (flags & SEARCH_NOCASE)
629 if (strcasecmp(di->match, kv->str))
634 if (strcmp(di->match, kv->str))
639 if (fnmatch(di->match, kv->str, (flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
644 if (regexec(&di->regexp, kv->str, 0, NULL, 0))
654 static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = {
655 { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
656 { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
657 { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
658 { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
659 { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
660 { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
661 { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
662 { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
663 { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
664 { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
665 { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
666 { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
667 { SOLVABLE_FRESHENS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
668 { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE },
672 dataiterator_step(Dataiterator *di)
686 di->kv.eof = idp[1] ? 0 : 1;
692 Solvable *s = di->repo->pool->solvables + di->solvid;
693 int state = di->state;
694 di->key = solvablekeys + state - 1;
696 di->state = RPM_RPMDBID;
703 state = di->keyname - 1;
722 case SOLVABLE_VENDOR:
725 di->kv.id = s->vendor;
727 case SOLVABLE_PROVIDES:
728 di->idp = s->provides
729 ? di->repo->idarraydata + s->provides : 0;
731 case SOLVABLE_OBSOLETES:
732 di->idp = s->obsoletes
733 ? di->repo->idarraydata + s->obsoletes : 0;
735 case SOLVABLE_CONFLICTS:
736 di->idp = s->conflicts
737 ? di->repo->idarraydata + s->conflicts : 0;
739 case SOLVABLE_REQUIRES:
740 di->idp = s->requires
741 ? di->repo->idarraydata + s->requires : 0;
743 case SOLVABLE_RECOMMENDS:
744 di->idp = s->recommends
745 ? di->repo->idarraydata + s->recommends : 0;
747 case SOLVABLE_SUPPLEMENTS:
748 di->idp = s->supplements
749 ? di->repo->idarraydata + s->supplements : 0;
751 case SOLVABLE_SUGGESTS:
752 di->idp = s->suggests
753 ? di->repo->idarraydata + s->suggests : 0;
755 case SOLVABLE_ENHANCES:
756 di->idp = s->enhances
757 ? di->repo->idarraydata + s->enhances : 0;
759 case SOLVABLE_FRESHENS:
760 di->idp = s->freshens
761 ? di->repo->idarraydata + s->freshens : 0;
764 if (!di->repo->rpmdbid)
766 di->kv.num = di->repo->rpmdbid[di->solvid - di->repo->start];
769 di->data = di->repo->repodata - 1;
780 di->dp = data_fetch(di->dp, &di->kv, di->key);
785 if (di->keyname || !(keyid = *di->keyp++))
789 Repo *repo = di->repo;
790 Repodata *data = ++di->data;
791 if (data >= repo->repodata + repo->nrepodata)
793 if (di->flags & __SEARCH_ONESOLVABLE)
795 while (++di->solvid < repo->end)
796 if (repo->pool->solvables[di->solvid].repo == repo)
798 if (di->solvid >= repo->end)
800 di->data = repo->repodata - 1;
801 if (di->flags & SEARCH_NO_STORAGE_SOLVABLE)
803 static Id zeroid = 0;
808 if (di->solvid >= data->start && di->solvid < data->end)
810 dataiterator_newdata(di);
818 di->key = di->data->keys + keyid;
819 di->dp = get_data(di->data, di->key, &di->nextkeydp);
821 di->dp = data_fetch(di->dp, &di->kv, di->key);
826 || dataiterator_match(di, &di->kv))
832 /* extend repodata so that it includes solvables p */
834 repodata_extend(Repodata *data, Id p)
836 if (data->start == data->end)
837 data->start = data->end = p;
840 int old = data->end - data->start;
841 int new = p - data->end + 1;
844 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
845 memset(data->attrs + old, 0, new * sizeof(Id *));
847 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
848 memset(data->incoreoffset + old, 0, new * sizeof(Id));
853 int old = data->end - data->start;
854 int new = data->start - p;
857 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
858 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
859 memset(data->attrs, 0, new * sizeof(Id *));
861 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
862 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
863 memset(data->incoreoffset, 0, new * sizeof(Id));
869 repodata_extend_block(Repodata *data, Id start, Id num)
873 if (!data->incoreoffset)
875 data->incoreoffset = sat_extend_resize(data->incoreoffset, num, sizeof(Id), REPODATA_BLOCK);
876 memset(data->incoreoffset, 0, num * sizeof(Id));
878 data->end = start + num;
881 repodata_extend(data, start);
883 repodata_extend(data, start + num - 1);
886 /**********************************************************************/
888 #define REPODATA_ATTRS_BLOCK 63
889 #define REPODATA_ATTRDATA_BLOCK 1023
890 #define REPODATA_ATTRIDDATA_BLOCK 63
893 repodata_insert_keyid(Repodata *data, Id entry, Id keyid, Id val, int overwrite)
899 data->attrs = sat_extend_resize(0, data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
900 memset(data->attrs, 0, (data->end - data->start) * sizeof(Id *));
903 if (data->attrs[entry])
905 for (pp = data->attrs[entry]; *pp; pp += 2)
906 /* Determine equality based on the name only, allows us to change
907 type (when overwrite is set), and makes TYPE_CONSTANT work. */
908 if (data->keys[*pp].name == data->keys[keyid].name)
919 i = pp - data->attrs[entry];
921 data->attrs[entry] = sat_extend(data->attrs[entry], i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
922 pp = data->attrs[entry] + i;
929 repodata_set(Repodata *data, Id entry, Repokey *key, Id val)
933 /* find key in keys */
934 for (keyid = 1; keyid < data->nkeys; keyid++)
935 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
937 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
941 if (keyid == data->nkeys)
943 /* allocate new key */
944 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
945 data->keys[data->nkeys++] = *key;
946 if (data->verticaloffset)
948 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
949 data->verticaloffset[data->nkeys - 1] = 0;
952 repodata_insert_keyid(data, entry, keyid, val, 1);
956 repodata_set_id(Repodata *data, Id entry, Id keyname, Id id)
960 key.type = REPOKEY_TYPE_ID;
962 key.storage = KEY_STORAGE_INCORE;
963 repodata_set(data, entry, &key, id);
967 repodata_set_num(Repodata *data, Id entry, Id keyname, Id num)
971 key.type = REPOKEY_TYPE_NUM;
973 key.storage = KEY_STORAGE_INCORE;
974 repodata_set(data, entry, &key, num);
978 repodata_set_poolstr(Repodata *data, Id entry, Id keyname, const char *str)
983 id = stringpool_str2id(&data->spool, str, 1);
985 id = str2id(data->repo->pool, str, 1);
987 key.type = REPOKEY_TYPE_ID;
989 key.storage = KEY_STORAGE_INCORE;
990 repodata_set(data, entry, &key, id);
994 repodata_set_constant(Repodata *data, Id entry, Id keyname, Id constant)
998 key.type = REPOKEY_TYPE_CONSTANT;
1000 key.storage = KEY_STORAGE_INCORE;
1001 repodata_set(data, entry, &key, 0);
1005 repodata_set_constantid(Repodata *data, Id entry, Id keyname, Id id)
1009 key.type = REPOKEY_TYPE_CONSTANTID;
1011 key.storage = KEY_STORAGE_INCORE;
1012 repodata_set(data, entry, &key, 0);
1016 repodata_set_void(Repodata *data, Id entry, Id keyname)
1020 key.type = REPOKEY_TYPE_VOID;
1022 key.storage = KEY_STORAGE_INCORE;
1023 repodata_set(data, entry, &key, 0);
1027 repodata_set_str(Repodata *data, Id entry, Id keyname, const char *str)
1032 l = strlen(str) + 1;
1034 key.type = REPOKEY_TYPE_STR;
1036 key.storage = KEY_STORAGE_INCORE;
1037 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1038 memcpy(data->attrdata + data->attrdatalen, str, l);
1039 repodata_set(data, entry, &key, data->attrdatalen);
1040 data->attrdatalen += l;
1044 repodata_add_dirnumnum(Repodata *data, Id entry, Id keyname, Id dir, Id num, Id num2)
1050 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", entry, dir, num, num2, data->attriddatalen);
1052 if (data->attrs && data->attrs[entry])
1054 for (pp = data->attrs[entry]; *pp; pp += 2)
1055 if (data->keys[*pp].name == keyname && data->keys[*pp].type == REPOKEY_TYPE_DIRNUMNUMARRAY)
1060 for (ida = data->attriddata + pp[1]; *ida; ida += 3)
1062 if (ida + 1 == data->attriddata + data->attriddatalen)
1064 /* this was the last entry, just append it */
1065 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 3, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1066 data->attriddatalen--; /* overwrite terminating 0 */
1070 /* too bad. move to back. */
1071 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + 4, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1072 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
1073 pp[1] = data->attriddatalen;
1074 data->attriddatalen += oldsize;
1076 data->attriddata[data->attriddatalen++] = dir;
1077 data->attriddata[data->attriddatalen++] = num;
1078 data->attriddata[data->attriddatalen++] = num2;
1079 data->attriddata[data->attriddatalen++] = 0;
1084 key.type = REPOKEY_TYPE_DIRNUMNUMARRAY;
1086 key.storage = KEY_STORAGE_INCORE;
1087 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 4, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1088 repodata_set(data, entry, &key, data->attriddatalen);
1089 data->attriddata[data->attriddatalen++] = dir;
1090 data->attriddata[data->attriddatalen++] = num;
1091 data->attriddata[data->attriddatalen++] = num2;
1092 data->attriddata[data->attriddatalen++] = 0;
1096 repodata_add_dirstr(Repodata *data, Id entry, Id keyname, Id dir, const char *str)
1098 Id *ida, *pp, stroff;
1102 l = strlen(str) + 1;
1103 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1104 memcpy(data->attrdata + data->attrdatalen, str, l);
1105 stroff = data->attrdatalen;
1106 data->attrdatalen += l;
1109 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", entry, dir, str, data->attriddatalen);
1111 if (data->attrs && data->attrs[entry])
1113 for (pp = data->attrs[entry]; *pp; pp += 2)
1114 if (data->keys[*pp].name == keyname && data->keys[*pp].type == REPOKEY_TYPE_DIRSTRARRAY)
1119 for (ida = data->attriddata + pp[1]; *ida; ida += 2)
1121 if (ida + 1 == data->attriddata + data->attriddatalen)
1123 /* this was the last entry, just append it */
1124 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 2, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1125 data->attriddatalen--; /* overwrite terminating 0 */
1129 /* too bad. move to back. */
1130 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + 3, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1131 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
1132 pp[1] = data->attriddatalen;
1133 data->attriddatalen += oldsize;
1135 data->attriddata[data->attriddatalen++] = dir;
1136 data->attriddata[data->attriddatalen++] = stroff;
1137 data->attriddata[data->attriddatalen++] = 0;
1142 key.type = REPOKEY_TYPE_DIRSTRARRAY;
1144 key.storage = KEY_STORAGE_INCORE;
1145 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 3, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1146 repodata_set(data, entry, &key, data->attriddatalen);
1147 data->attriddata[data->attriddatalen++] = dir;
1148 data->attriddata[data->attriddatalen++] = stroff;
1149 data->attriddata[data->attriddatalen++] = 0;
1153 repodata_merge_attrs(Repodata *data, Id dest, Id src)
1156 if (dest == src || !(keyp = data->attrs[src]))
1158 for (; *keyp; keyp += 2)
1159 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
1162 /*********************************/
1164 /* unify with repo_write! */
1166 #define EXTDATA_BLOCK 1023
1167 #define SCHEMATA_BLOCK 31
1168 #define SCHEMATADATA_BLOCK 255
1176 data_addid(struct extdata *xd, Id x)
1179 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
1180 dp = xd->buf + xd->len;
1185 *dp++ = (x >> 28) | 128;
1187 *dp++ = (x >> 21) | 128;
1188 *dp++ = (x >> 14) | 128;
1191 *dp++ = (x >> 7) | 128;
1193 xd->len = dp - xd->buf;
1197 data_addideof(struct extdata *xd, Id x, int eof)
1200 x = (x & 63) | ((x & ~63) << 1);
1201 data_addid(xd, (eof ? x: x | 64));
1205 data_addblob(struct extdata *xd, unsigned char *blob, int len)
1207 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
1208 memcpy(xd->buf + xd->len, blob, len);
1212 /*********************************/
1215 addschema_prepare(Repodata *data, Id *schematacache)
1220 memset(schematacache, 0, 256 * sizeof(Id));
1221 for (i = 0; i < data->nschemata; i++)
1223 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
1226 schematacache[h] = i + 1;
1228 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
1229 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
1233 addschema(Repodata *data, Id *schema, Id *schematacache)
1238 for (sp = schema, len = 0, h = 0; *sp; len++)
1243 cid = schematacache[h];
1247 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1249 /* cache conflict */
1250 for (cid = 0; cid < data->nschemata; cid++)
1251 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1254 /* a new one. make room. */
1255 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
1256 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
1258 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
1259 data->schemata[data->nschemata] = data->schemadatalen;
1260 data->schemadatalen += len;
1261 schematacache[h] = data->nschemata + 1;
1263 fprintf(stderr, "addschema: new schema\n");
1265 return data->nschemata++;
1270 repodata_internalize(Repodata *data)
1273 Id id, entry, nentry, *ida;
1274 Id schematacache[256];
1275 Id schemaid, *schema, *sp, oldschema, *keyp, *seen;
1276 unsigned char *dp, *ndp;
1277 int newschema, oldcount;
1278 struct extdata newincore;
1279 struct extdata newvincore;
1284 newvincore.buf = data->vincore;
1285 newvincore.len = data->vincorelen;
1287 schema = sat_malloc2(data->nkeys, sizeof(Id));
1288 seen = sat_malloc2(data->nkeys, sizeof(Id));
1290 /* Merge the data already existing (in data->schemata, ->incoredata and
1291 friends) with the new attributes in data->attrs[]. */
1292 nentry = data->end - data->start;
1293 addschema_prepare(data, schematacache);
1294 memset(&newincore, 0, sizeof(newincore));
1295 for (entry = 0; entry < nentry; entry++)
1297 memset(seen, 0, data->nkeys * sizeof(Id));
1299 dp = data->incoredata + data->incoreoffset[entry];
1300 if (data->incoredata)
1301 dp = data_read_id(dp, &oldschema);
1305 fprintf(stderr, "oldschema %d\n", oldschema);
1306 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
1307 fprintf(stderr, "schemadata %p\n", data->schemadata);
1309 /* seen: -1: old data 0: skipped >0: id + 1 */
1312 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
1316 fprintf(stderr, "Inconsistent old data (key occured twice).\n");
1323 if (data->attrs[entry])
1324 for (keyp = data->attrs[entry]; *keyp; keyp += 2)
1331 seen[*keyp] = keyp[1] + 1;
1335 /* Ideally we'd like to sort the new schema here, to ensure
1336 schema equality independend of the ordering. We can't do that
1337 yet. For once see below (old ids need to come before new ids).
1338 An additional difficulty is that we also need to move
1339 the values with the keys. */
1340 schemaid = addschema(data, schema, schematacache);
1342 schemaid = oldschema;
1345 /* Now create data blob. We walk through the (possibly new) schema
1346 and either copy over old data, or insert the new. */
1347 /* XXX Here we rely on the fact that the (new) schema has the form
1348 o1 o2 o3 o4 ... | n1 n2 n3 ...
1349 (oX being the old keyids (possibly overwritten), and nX being
1350 the new keyids). This rules out sorting the keyids in order
1351 to ensure a small schema count. */
1352 data->incoreoffset[entry] = newincore.len;
1353 data_addid(&newincore, schemaid);
1354 for (keyp = data->schemadata + data->schemata[schemaid]; *keyp; keyp++)
1356 key = data->keys + *keyp;
1360 /* Skip the data associated with this old key. */
1361 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1363 ndp = data_skip(dp, REPOKEY_TYPE_ID);
1364 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
1366 else if (key->storage == KEY_STORAGE_INCORE)
1367 ndp = data_skip(dp, key->type);
1370 if (seen[*keyp] == -1)
1372 /* If this key was an old one _and_ was not overwritten with
1373 a different value copy over the old value (we skipped it
1376 data_addblob(&newincore, dp, ndp - dp);
1379 else if (seen[*keyp])
1381 /* Otherwise we have a new value. Parse it into the internal
1384 unsigned int oldvincorelen = 0;
1387 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1390 oldvincorelen = xd->len;
1392 id = seen[*keyp] - 1;
1395 case REPOKEY_TYPE_VOID:
1396 case REPOKEY_TYPE_CONSTANT:
1397 case REPOKEY_TYPE_CONSTANTID:
1399 case REPOKEY_TYPE_STR:
1400 data_addblob(xd, data->attrdata + id, strlen((char *)(data->attrdata + id)) + 1);
1402 case REPOKEY_TYPE_ID:
1403 case REPOKEY_TYPE_NUM:
1404 case REPOKEY_TYPE_DIR:
1407 case REPOKEY_TYPE_DIRNUMNUMARRAY:
1408 for (ida = data->attriddata + id; *ida; ida += 3)
1410 data_addid(xd, ida[0]);
1411 data_addid(xd, ida[1]);
1412 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
1415 case REPOKEY_TYPE_DIRSTRARRAY:
1416 for (ida = data->attriddata + id; *ida; ida += 2)
1418 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
1419 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
1423 fprintf(stderr, "don't know how to handle type %d\n", key->type);
1426 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1428 /* put offset/len in incore */
1429 data_addid(&newincore, data->lastverticaloffset + oldvincorelen);
1430 oldvincorelen = xd->len - oldvincorelen;
1431 data_addid(&newincore, oldvincorelen);
1436 if (data->attrs[entry])
1437 sat_free(data->attrs[entry]);
1442 sat_free(data->incoredata);
1443 data->incoredata = newincore.buf;
1444 data->incoredatalen = newincore.len;
1445 data->incoredatafree = 0;
1447 sat_free(data->vincore);
1448 data->vincore = newvincore.buf;
1449 data->vincorelen = newvincore.len;
1451 data->attrs = sat_free(data->attrs);
1452 data->attrdata = sat_free(data->attrdata);
1453 data->attriddata = sat_free(data->attriddata);
1454 data->attrdatalen = 0;
1455 data->attriddatalen = 0;
1459 repodata_str2dir(Repodata *data, const char *dir, int create)
1465 while (*dir == '/' && dir[1] == '/')
1467 if (*dir == '/' && !dir[1])
1471 dire = strchrnul(dir, '/');
1472 if (data->localpool)
1473 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
1475 id = strn2id(data->repo->pool, dir, dire - dir, create);
1478 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
1491 repodata_dir2str(Repodata *data, Id did, const char *suf)
1493 Pool *pool = data->repo->pool;
1500 return suf ? suf : "";
1504 comp = dirpool_compid(&data->dirpool, parent);
1505 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
1507 parent = dirpool_parent(&data->dirpool, parent);
1512 l += strlen(suf) + 1;
1513 p = pool_alloctmpspace(pool, l + 1) + l;
1524 comp = dirpool_compid(&data->dirpool, parent);
1525 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
1528 strncpy(p, comps, l);
1529 parent = dirpool_parent(&data->dirpool, parent);
1537 repodata_compress_page(unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max)
1539 return compress_buf(page, len, cpage, max);
1542 #define SOLV_ERROR_EOF 3
1544 static inline unsigned int
1550 for (i = 0; i < 4; i++)
1560 #define SOLV_ERROR_EOF 3
1561 #define SOLV_ERROR_CORRUPT 6
1563 /* Try to either setup on-demand paging (using FP as backing
1564 file), or in case that doesn't work (FP not seekable) slurps in
1565 all pages and deactivates paging. */
1567 repodata_read_or_setup_pages(Repodata *data, unsigned int pagesz, unsigned int blobsz)
1569 FILE *fp = data->fp;
1570 unsigned int npages;
1572 unsigned int can_seek;
1574 unsigned char buf[BLOB_PAGESIZE];
1576 if (pagesz != BLOB_PAGESIZE)
1578 /* We could handle this by slurping in everything. */
1579 data->error = SOLV_ERROR_CORRUPT;
1583 if ((cur_file_ofs = ftell(fp)) < 0)
1587 data->pagefd = dup(fileno(fp));
1588 if (data->pagefd == -1)
1592 fprintf (stderr, "can %sseek\n", can_seek ? "" : "NOT ");
1594 npages = (blobsz + BLOB_PAGESIZE - 1) / BLOB_PAGESIZE;
1596 data->num_pages = npages;
1597 data->pages = sat_malloc2(npages, sizeof(data->pages[0]));
1599 /* If we can't seek on our input we have to slurp in everything. */
1601 data->blob_store = sat_malloc(npages * BLOB_PAGESIZE);
1602 for (i = 0; i < npages; i++)
1604 unsigned int in_len = read_u32(fp);
1605 unsigned int compressed = in_len & 1;
1606 Attrblobpage *p = data->pages + i;
1609 fprintf (stderr, "page %d: len %d (%scompressed)\n",
1610 i, in_len, compressed ? "" : "not ");
1616 p->file_offset = cur_file_ofs;
1617 p->file_size = in_len * 2 + compressed;
1618 if (fseek(fp, in_len, SEEK_CUR) < 0)
1621 fprintf (stderr, "can't seek after we thought we can\n");
1622 /* We can't fall back to non-seeking behaviour as we already
1623 read over some data pages without storing them away. */
1624 data->error = SOLV_ERROR_EOF;
1625 close(data->pagefd);
1629 cur_file_ofs += in_len;
1633 unsigned int out_len;
1634 void *dest = data->blob_store + i * BLOB_PAGESIZE;
1635 p->mapped_at = i * BLOB_PAGESIZE;
1638 /* We can't seek, so suck everything in. */
1639 if (fread(compressed ? buf : dest, in_len, 1, fp) != 1)
1642 data->error = SOLV_ERROR_EOF;
1647 out_len = unchecked_decompress_buf(buf, in_len, dest, BLOB_PAGESIZE);
1648 if (out_len != BLOB_PAGESIZE && i < npages - 1)
1650 data->error = SOLV_ERROR_CORRUPT;
1659 repodata_disable_paging(Repodata *data)
1661 if (maybe_load_repodata(data, 0)
1663 load_page_range (data, 0, data->num_pages - 1);
1666 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: