2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
26 #include "poolid_private.h"
29 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
30 unsigned char *out, unsigned int out_len);
31 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
34 unsigned int out_len);
36 #define REPODATA_BLOCK 255
39 repodata_free(Repodata *data)
42 sat_free(data->schemata);
43 sat_free(data->schemadata);
45 sat_free(data->spool.strings);
46 sat_free(data->spool.stringspace);
47 sat_free(data->spool.stringhashtbl);
49 sat_free(data->dirpool.dirs);
50 sat_free(data->dirpool.dirtraverse);
52 sat_free(data->incoredata);
53 sat_free(data->incoreoffset);
54 sat_free(data->verticaloffset);
56 sat_free(data->blob_store);
57 sat_free(data->pages);
58 sat_free(data->mapped);
60 sat_free(data->vincore);
62 sat_free(data->attrs);
63 sat_free(data->attrdata);
64 sat_free(data->attriddata);
70 static unsigned char *
71 data_read_id(unsigned char *dp, Id *idp)
83 x = (x << 7) ^ c ^ 128;
87 static unsigned char *
88 data_read_ideof(unsigned char *dp, Id *idp, int *eof)
107 x = (x << 7) ^ c ^ 128;
111 static unsigned char *
112 data_skip(unsigned char *dp, int type)
123 while ((*dp & 0x80) != 0)
127 while ((*dp & 0xc0) != 0)
134 case TYPE_DIRSTRARRAY:
137 while ((*dp & 0x80) != 0)
146 case TYPE_DIRNUMNUMARRAY:
149 while ((*dp & 0x80) != 0)
152 while ((*dp & 0x80) != 0)
155 while ((*dp & 0x80) != 0)
162 fprintf(stderr, "unknown type in data_skip\n");
167 static unsigned char *
168 data_fetch(unsigned char *dp, KeyValue *kv, Repokey *key)
181 kv->str = (const char *)dp;
182 return dp + strlen(kv->str) + 1;
184 return data_read_id(dp, &kv->id);
186 return data_read_id(dp, &kv->num);
188 return data_read_ideof(dp, &kv->id, &kv->eof);
190 return data_read_id(dp, &kv->id);
191 case TYPE_DIRSTRARRAY:
192 dp = data_read_ideof(dp, &kv->id, &kv->eof);
193 kv->str = (const char *)dp;
194 return dp + strlen(kv->str) + 1;
195 case TYPE_DIRNUMNUMARRAY:
196 dp = data_read_id(dp, &kv->id);
197 dp = data_read_id(dp, &kv->num);
198 return data_read_ideof(dp, &kv->num2, &kv->eof);
204 static unsigned char *
205 forward_to_key(Repodata *data, Id key, Id schemaid, unsigned char *dp)
209 keyp = data->schemadata + data->schemata[schemaid];
210 while ((k = *keyp++) != 0)
214 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
216 dp = data_skip(dp, TYPE_ID); /* skip that offset */
217 dp = data_skip(dp, TYPE_ID); /* skip that length */
220 if (data->keys[k].storage != KEY_STORAGE_INCORE)
222 dp = data_skip(dp, data->keys[k].type);
227 #define BLOB_PAGEBITS 15
228 #define BLOB_PAGESIZE (1 << BLOB_PAGEBITS)
230 static unsigned char *
231 load_page_range(Repodata *data, unsigned int pstart, unsigned int pend)
233 /* Make sure all pages from PSTART to PEND (inclusive) are loaded,
234 and are consecutive. Return a pointer to the mapping of PSTART. */
235 unsigned char buf[BLOB_PAGESIZE];
238 /* Quick check in case all pages are there already and consecutive. */
239 for (i = pstart; i <= pend; i++)
240 if (data->pages[i].mapped_at == -1
242 && data->pages[i].mapped_at
243 != data->pages[i-1].mapped_at + BLOB_PAGESIZE))
246 return data->blob_store + data->pages[pstart].mapped_at;
248 /* Ensure that we can map the numbers of pages we need at all. */
249 if (pend - pstart + 1 > data->ncanmap)
251 unsigned int oldcan = data->ncanmap;
252 data->ncanmap = pend - pstart + 1;
253 if (data->ncanmap < 4)
255 data->mapped = sat_realloc2(data->mapped, data->ncanmap, sizeof(data->mapped[0]));
256 memset (data->mapped + oldcan, 0, (data->ncanmap - oldcan) * sizeof (data->mapped[0]));
257 data->blob_store = sat_realloc2(data->blob_store, data->ncanmap, BLOB_PAGESIZE);
259 fprintf (stderr, "PAGE: can map %d pages\n", data->ncanmap);
263 /* Now search for "cheap" space in our store. Space is cheap if it's either
264 free (very cheap) or contains pages we search for anyway. */
266 /* Setup cost array. */
267 unsigned int cost[data->ncanmap];
268 for (i = 0; i < data->ncanmap; i++)
270 unsigned int pnum = data->mapped[i];
276 Attrblobpage *p = data->pages + pnum;
277 assert (p->mapped_at != -1);
278 if (pnum >= pstart && pnum <= pend)
285 /* And search for cheapest space. */
286 unsigned int best_cost = -1;
287 unsigned int best = 0;
288 unsigned int same_cost = 0;
289 for (i = 0; i + pend - pstart < data->ncanmap; i++)
291 unsigned int c = cost[i];
293 for (j = 0; j < pend - pstart + 1; j++)
296 best_cost = c, best = i;
297 else if (c == best_cost)
299 /* A null cost won't become better. */
303 /* If all places have the same cost we would thrash on slot 0. Avoid
304 this by doing a round-robin strategy in this case. */
305 if (same_cost == data->ncanmap - pend + pstart - 1)
306 best = data->rr_counter++ % (data->ncanmap - pend + pstart);
308 /* So we want to map our pages from [best] to [best+pend-pstart].
309 Use a very simple strategy, which doesn't make the best use of
310 our resources, but works. Throw away all pages in that range
311 (even ours) then copy around ours (in case they were outside the
312 range) or read them in. */
313 for (i = best; i < best + pend - pstart + 1; i++)
315 unsigned int pnum = data->mapped[i];
317 /* If this page is exactly at the right place already,
318 no need to evict it. */
319 && pnum != pstart + i - best)
321 /* Evict this page. */
323 fprintf (stderr, "PAGE: evict page %d from %d\n", pnum, i);
327 data->pages[pnum].mapped_at = -1;
331 /* Everything is free now. Read in the pages we want. */
332 for (i = pstart; i <= pend; i++)
334 Attrblobpage *p = data->pages + i;
335 unsigned int pnum = i - pstart + best;
336 void *dest = data->blob_store + pnum * BLOB_PAGESIZE;
337 if (p->mapped_at != -1)
339 if (p->mapped_at != pnum * BLOB_PAGESIZE)
342 fprintf (stderr, "PAGECOPY: %d to %d\n", i, pnum);
344 /* Still mapped somewhere else, so just copy it from there. */
345 memcpy (dest, data->blob_store + p->mapped_at, BLOB_PAGESIZE);
346 data->mapped[p->mapped_at / BLOB_PAGESIZE] = 0;
351 unsigned int in_len = p->file_size;
352 unsigned int compressed = in_len & 1;
355 fprintf (stderr, "PAGEIN: %d to %d", i, pnum);
357 /* Not mapped, so read in this page. */
358 if (fseek(data->fp, p->file_offset, SEEK_SET) < 0)
360 perror ("mapping fseek");
363 if (fread(compressed ? buf : dest, in_len, 1, data->fp) != 1)
365 perror ("mapping fread");
370 unsigned int out_len;
371 out_len = unchecked_decompress_buf(buf, in_len,
372 dest, BLOB_PAGESIZE);
373 if (out_len != BLOB_PAGESIZE
374 && i < data->num_pages - 1)
376 fprintf (stderr, "can't decompress\n");
380 fprintf (stderr, " (expand %d to %d)", in_len, out_len);
384 fprintf (stderr, "\n");
387 p->mapped_at = pnum * BLOB_PAGESIZE;
388 data->mapped[pnum] = i + 1;
390 return data->blob_store + best * BLOB_PAGESIZE;
393 static unsigned char *
394 make_vertical_available(Repodata *data, Repokey *key, Id off, Id len)
397 if (key->type == TYPE_VOID)
399 if (off >= data->lastverticaloffset)
401 off -= data->lastverticaloffset;
402 if (off + len > data->vincorelen)
404 return data->vincore + off;
408 if (off + len > key->size)
410 /* we now have the offset, go into vertical */
411 off += data->verticaloffset[key - data->keys];
412 dp = load_page_range(data, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
414 dp += off % BLOB_PAGESIZE;
418 static inline unsigned char *
419 get_data(Repodata *data, Repokey *key, unsigned char **dpp)
421 unsigned char *dp = *dpp;
425 if (key->storage == KEY_STORAGE_INCORE)
427 /* hmm, this is a bit expensive */
428 *dpp = data_skip(dp, key->type);
431 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
434 dp = data_read_id(dp, &off);
435 dp = data_read_id(dp, &len);
437 return make_vertical_available(data, key, off, len);
443 maybe_load_repodata(Repodata *data)
445 if (data->state == REPODATA_STUB)
447 if (data->loadcallback)
448 data->loadcallback(data);
450 data->state = REPODATA_ERROR;
452 if (data->state == REPODATA_AVAILABLE)
454 data->state = REPODATA_ERROR;
459 repodata_lookup_str(Repodata *data, Id entry, Id keyid)
466 if (!maybe_load_repodata (data))
469 dp = data->incoredata + data->incoreoffset[entry];
470 dp = data_read_id(dp, &schema);
471 /* make sure the schema of this solvable contains the key */
472 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
475 dp = forward_to_key(data, keyid, schema, dp);
476 key = data->keys + keyid;
477 dp = get_data(data, key, &dp);
480 if (key->type == TYPE_STR)
481 return (const char *)dp;
482 if (key->type != TYPE_ID)
484 /* id type, must either use global or local string store*/
485 dp = data_read_id(dp, &id);
487 return data->spool.stringspace + data->spool.strings[id];
488 return id2str(data->repo->pool, id);
492 repodata_lookup_num(Repodata *data, Id entry, Id keyid, unsigned *value)
502 if (!maybe_load_repodata (data))
505 dp = data->incoredata + data->incoreoffset[entry];
506 dp = data_read_id(dp, &schema);
507 /* make sure the schema of this solvable contains the key */
508 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
511 dp = forward_to_key(data, keyid, schema, dp);
512 key = data->keys + keyid;
513 dp = get_data(data, key, &dp);
516 if (key->type == TYPE_NUM
517 || key->type == TYPE_U32
518 || key->type == TYPE_CONSTANT)
520 dp = data_fetch(dp, &kv, key);
528 repodata_search(Repodata *data, Id entry, Id keyname, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
532 Id k, keyid, *kp, *keyp;
533 unsigned char *dp, *ddp;
538 if (!maybe_load_repodata (data))
541 dp = data->incoredata + data->incoreoffset[entry];
542 dp = data_read_id(dp, &schema);
543 keyp = data->schemadata + data->schemata[schema];
546 /* search in a specific key */
547 for (kp = keyp; (k = *kp++) != 0; )
548 if (data->keys[k].name == keyname)
552 dp = forward_to_key(data, k, schema, dp);
558 while ((keyid = *keyp++) != 0)
561 key = data->keys + keyid;
562 ddp = get_data(data, key, &dp);
565 ddp = data_fetch(ddp, &kv, key);
568 stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv);
570 while (!kv.eof && !stop);
571 if (onekey || stop > SEARCH_NEXT_KEY)
577 dataiterator_newdata(Dataiterator *di)
579 Id keyname = di->keyname;
580 Repodata *data = di->data;
583 if (data->state == REPODATA_STUB)
588 for (j = 1; j < data->nkeys; j++)
589 if (keyname == data->keys[j].name)
591 if (j == data->nkeys)
595 if (data->loadcallback)
596 data->loadcallback(data);
598 data->state = REPODATA_ERROR;
600 if (data->state == REPODATA_ERROR)
604 unsigned char *dp = data->incoredata + data->incoreoffset[di->solvid - data->start];
605 dp = data_read_id(dp, &schema);
606 Id *keyp = data->schemadata + data->schemata[schema];
610 /* search in a specific key */
611 for (kp = keyp; (k = *kp++) != 0; )
612 if (data->keys[k].name == keyname)
616 dp = forward_to_key(data, k, schema, dp);
626 di->key = di->data->keys + keyid;
631 di->dp = get_data(di->data, di->key, &di->nextkeydp);
636 dataiterator_init(Dataiterator *di, Repo *repo, Id p, Id keyname,
637 const char *match, int flags)
643 di->flags |= __SEARCH_ONESOLVABLE;
644 di->data = repo->repodata - 1;
648 di->solvid = repo->start - 1;
649 di->data = repo->repodata + repo->nrepodata - 1;
652 di->keyname = keyname;
653 static Id zeroid = 0;
659 /* FIXME factor and merge with repo_matchvalue */
661 dataiterator_match(Dataiterator *di, KeyValue *kv)
663 int flags = di->flags;
665 if ((flags & SEARCH_STRINGMASK) != 0)
667 switch (di->key->type)
671 if (di->data && di->data->localpool)
672 kv->str = stringpool_id2str(&di->data->spool, kv->id);
674 kv->str = id2str(di->repo->pool, kv->id);
681 switch ((flags & SEARCH_STRINGMASK))
683 case SEARCH_SUBSTRING:
684 if (flags & SEARCH_NOCASE)
686 if (!strcasestr(kv->str, di->match))
691 if (!strstr(kv->str, di->match))
696 if (flags & SEARCH_NOCASE)
698 if (strcasecmp(di->match, kv->str))
703 if (strcmp(di->match, kv->str))
708 if (fnmatch(di->match, kv->str, (flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
713 if (regexec(&di->regexp, kv->str, 0, NULL, 0))
724 dataiterator_step(Dataiterator *di)
726 /* FIXME add solvable data */
732 di->dp = data_fetch(di->dp, &di->kv, di->key);
737 if (di->keyname || !(keyid = *di->keyp++))
741 Repo *repo = di->repo;
742 Repodata *data = ++di->data;
743 if (data >= repo->repodata + repo->nrepodata)
745 if (di->flags & __SEARCH_ONESOLVABLE)
747 while (++di->solvid < repo->end)
748 if (repo->pool->solvables[di->solvid].repo == repo)
750 if (di->solvid >= repo->end)
752 di->data = repo->repodata - 1;
755 if (di->solvid >= data->start && di->solvid < data->end)
757 dataiterator_newdata(di);
765 di->key = di->data->keys + keyid;
766 di->dp = get_data(di->data, di->key, &di->nextkeydp);
768 di->dp = data_fetch(di->dp, &di->kv, di->key);
771 || dataiterator_match(di, &di->kv))
778 repodata_init(Repodata *data, Repo *repo, int localpool)
780 memset(data, 0, sizeof (*data));
782 data->localpool = localpool;
784 stringpool_init_empty(&data->spool);
785 data->keys = sat_calloc(1, sizeof(Repokey));
787 data->schemata = sat_calloc(1, sizeof(Id));
788 data->schemadata = sat_calloc(1, sizeof(Id));
790 data->schemadatalen = 1;
791 data->start = repo->start;
792 data->end = repo->end;
793 data->incoreoffset = sat_extend_resize(0, data->end - data->start, sizeof(Id), REPODATA_BLOCK);
796 /* extend repodata so that it includes solvables p */
798 repodata_extend(Repodata *data, Id p)
800 if (data->start == data->end)
801 data->start = data->end = p;
804 int old = data->end - data->start;
805 int new = p - data->end + 1;
808 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
809 memset(data->attrs + old, 0, new * sizeof(Id *));
811 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
812 memset(data->incoreoffset + old, 0, new * sizeof(Id));
817 int old = data->end - data->start;
818 int new = data->start - p;
821 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
822 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
823 memset(data->attrs, 0, new * sizeof(Id *));
825 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
826 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
827 memset(data->incoreoffset, 0, new * sizeof(Id));
833 repodata_extend_block(Repodata *data, Id start, Id num)
837 if (!data->incoreoffset)
839 data->incoreoffset = sat_extend_resize(data->incoreoffset, num, sizeof(Id), REPODATA_BLOCK);
840 memset(data->incoreoffset, 0, num * sizeof(Id));
842 data->end = start + num;
845 repodata_extend(data, start);
847 repodata_extend(data, start + num - 1);
850 #define REPODATA_ATTRS_BLOCK 63
851 #define REPODATA_ATTRDATA_BLOCK 1023
852 #define REPODATA_ATTRIDDATA_BLOCK 63
855 repodata_insert_keyid(Repodata *data, Id entry, Id keyid, Id val, int overwrite)
861 data->attrs = sat_extend_resize(0, data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
862 memset(data->attrs, 0, (data->end - data->start) * sizeof(Id *));
865 if (data->attrs[entry])
867 for (pp = data->attrs[entry]; *pp; pp += 2)
868 /* Determine equality based on the name only, allows us to change
869 type (when overwrite is set), and makes TYPE_CONSTANT work. */
870 if (data->keys[*pp].name == data->keys[keyid].name)
881 i = pp - data->attrs[entry];
883 data->attrs[entry] = sat_extend(data->attrs[entry], i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
884 pp = data->attrs[entry] + i;
891 repodata_set(Repodata *data, Id entry, Repokey *key, Id val)
895 /* find key in keys */
896 for (keyid = 1; keyid < data->nkeys; keyid++)
897 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
899 if (key->type == TYPE_CONSTANT && key->size != data->keys[keyid].size)
903 if (keyid == data->nkeys)
905 /* allocate new key */
906 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
907 data->keys[data->nkeys++] = *key;
908 if (data->verticaloffset)
910 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
911 data->verticaloffset[data->nkeys - 1] = 0;
914 repodata_insert_keyid(data, entry, keyid, val, 1);
918 repodata_set_id(Repodata *data, Id entry, Id keyname, Id id)
924 key.storage = KEY_STORAGE_INCORE;
925 repodata_set(data, entry, &key, id);
929 repodata_set_num(Repodata *data, Id entry, Id keyname, Id num)
935 key.storage = KEY_STORAGE_INCORE;
936 repodata_set(data, entry, &key, num);
940 repodata_set_poolstr(Repodata *data, Id entry, Id keyname, const char *str)
945 id = stringpool_str2id(&data->spool, str, 1);
947 id = str2id(data->repo->pool, str, 1);
951 key.storage = KEY_STORAGE_INCORE;
952 repodata_set(data, entry, &key, id);
956 repodata_set_constant(Repodata *data, Id entry, Id keyname, Id constant)
960 key.type = TYPE_CONSTANT;
962 key.storage = KEY_STORAGE_INCORE;
963 repodata_set(data, entry, &key, 0);
967 repodata_set_void(Repodata *data, Id entry, Id keyname)
971 key.type = TYPE_VOID;
973 key.storage = KEY_STORAGE_INCORE;
974 repodata_set(data, entry, &key, 0);
978 repodata_set_str(Repodata *data, Id entry, Id keyname, const char *str)
987 key.storage = KEY_STORAGE_INCORE;
988 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
989 memcpy(data->attrdata + data->attrdatalen, str, l);
990 repodata_set(data, entry, &key, data->attrdatalen);
991 data->attrdatalen += l;
995 repodata_add_dirnumnum(Repodata *data, Id entry, Id keyname, Id dir, Id num, Id num2)
1001 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", entry, dir, num, num2, data->attriddatalen);
1003 if (data->attrs && data->attrs[entry])
1005 for (pp = data->attrs[entry]; *pp; pp += 2)
1006 if (data->keys[*pp].name == keyname && data->keys[*pp].type == TYPE_DIRNUMNUMARRAY)
1011 for (ida = data->attriddata + pp[1]; *ida; ida += 3)
1013 if (ida + 1 == data->attriddata + data->attriddatalen)
1015 /* this was the last entry, just append it */
1016 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 3, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1017 data->attriddatalen--; /* overwrite terminating 0 */
1021 /* too bad. move to back. */
1022 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + 4, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1023 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
1024 pp[1] = data->attriddatalen;
1025 data->attriddatalen += oldsize;
1027 data->attriddata[data->attriddatalen++] = dir;
1028 data->attriddata[data->attriddatalen++] = num;
1029 data->attriddata[data->attriddatalen++] = num2;
1030 data->attriddata[data->attriddatalen++] = 0;
1035 key.type = TYPE_DIRNUMNUMARRAY;
1037 key.storage = KEY_STORAGE_INCORE;
1038 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 4, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1039 repodata_set(data, entry, &key, data->attriddatalen);
1040 data->attriddata[data->attriddatalen++] = dir;
1041 data->attriddata[data->attriddatalen++] = num;
1042 data->attriddata[data->attriddatalen++] = num2;
1043 data->attriddata[data->attriddatalen++] = 0;
1047 repodata_add_dirstr(Repodata *data, Id entry, Id keyname, Id dir, const char *str)
1049 Id *ida, *pp, stroff;
1053 l = strlen(str) + 1;
1054 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1055 memcpy(data->attrdata + data->attrdatalen, str, l);
1056 stroff = data->attrdatalen;
1057 data->attrdatalen += l;
1060 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", entry, dir, str, data->attriddatalen);
1062 if (data->attrs && data->attrs[entry])
1064 for (pp = data->attrs[entry]; *pp; pp += 2)
1065 if (data->keys[*pp].name == keyname && data->keys[*pp].type == TYPE_DIRSTRARRAY)
1070 for (ida = data->attriddata + pp[1]; *ida; ida += 2)
1072 if (ida + 1 == data->attriddata + data->attriddatalen)
1074 /* this was the last entry, just append it */
1075 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 2, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1076 data->attriddatalen--; /* overwrite terminating 0 */
1080 /* too bad. move to back. */
1081 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + 3, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1082 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
1083 pp[1] = data->attriddatalen;
1084 data->attriddatalen += oldsize;
1086 data->attriddata[data->attriddatalen++] = dir;
1087 data->attriddata[data->attriddatalen++] = stroff;
1088 data->attriddata[data->attriddatalen++] = 0;
1093 key.type = TYPE_DIRSTRARRAY;
1095 key.storage = KEY_STORAGE_INCORE;
1096 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 3, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1097 repodata_set(data, entry, &key, data->attriddatalen);
1098 data->attriddata[data->attriddatalen++] = dir;
1099 data->attriddata[data->attriddatalen++] = stroff;
1100 data->attriddata[data->attriddatalen++] = 0;
1104 repodata_merge_attrs (Repodata *data, Id dest, Id src)
1107 for (keyp = data->attrs[src]; *keyp; keyp += 2)
1108 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
1111 /*********************************/
1113 /* unify with repo_write! */
1115 #define EXTDATA_BLOCK 1023
1116 #define SCHEMATA_BLOCK 31
1117 #define SCHEMATADATA_BLOCK 255
1125 data_addid(struct extdata *xd, Id x)
1128 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
1129 dp = xd->buf + xd->len;
1134 *dp++ = (x >> 28) | 128;
1136 *dp++ = (x >> 21) | 128;
1137 *dp++ = (x >> 14) | 128;
1140 *dp++ = (x >> 7) | 128;
1142 xd->len = dp - xd->buf;
1146 data_addideof(struct extdata *xd, Id x, int eof)
1149 x = (x & 63) | ((x & ~63) << 1);
1150 data_addid(xd, (eof ? x: x | 64));
1154 data_addblob(struct extdata *xd, unsigned char *blob, int len)
1156 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
1157 memcpy(xd->buf + xd->len, blob, len);
1161 /*********************************/
1164 addschema_prepare(Repodata *data, Id *schematacache)
1169 memset(schematacache, 0, 256 * sizeof(Id));
1170 for (i = 0; i < data->nschemata; i++)
1172 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
1175 schematacache[h] = i + 1;
1177 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
1178 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
1182 addschema(Repodata *data, Id *schema, Id *schematacache)
1187 for (sp = schema, len = 0, h = 0; *sp; len++)
1192 cid = schematacache[h];
1196 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1198 /* cache conflict */
1199 for (cid = 0; cid < data->nschemata; cid++)
1200 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1203 /* a new one. make room. */
1204 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
1205 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
1207 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
1208 data->schemata[data->nschemata] = data->schemadatalen;
1209 data->schemadatalen += len;
1210 schematacache[h] = data->nschemata + 1;
1212 fprintf(stderr, "addschema: new schema\n");
1214 return data->nschemata++;
1219 repodata_internalize(Repodata *data)
1222 Id id, entry, nentry, *ida;
1223 Id schematacache[256];
1224 Id schemaid, *schema, *sp, oldschema, *keyp, *seen;
1225 unsigned char *dp, *ndp;
1226 int newschema, oldcount;
1227 struct extdata newincore;
1228 struct extdata newvincore;
1233 newvincore.buf = data->vincore;
1234 newvincore.len = data->vincorelen;
1236 schema = sat_malloc2(data->nkeys, sizeof(Id));
1237 seen = sat_malloc2(data->nkeys, sizeof(Id));
1239 /* Merge the data already existing (in data->schemata, ->incoredata and
1240 friends) with the new attributes in data->attrs[]. */
1241 nentry = data->end - data->start;
1242 addschema_prepare(data, schematacache);
1243 memset(&newincore, 0, sizeof(newincore));
1244 for (entry = 0; entry < nentry; entry++)
1246 memset(seen, 0, data->nkeys * sizeof(Id));
1248 dp = data->incoredata + data->incoreoffset[entry];
1249 if (data->incoredata)
1250 dp = data_read_id(dp, &oldschema);
1254 fprintf(stderr, "oldschema %d\n", oldschema);
1255 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
1256 fprintf(stderr, "schemadata %p\n", data->schemadata);
1258 /* seen: -1: old data 0: skipped >0: id + 1 */
1261 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
1265 fprintf(stderr, "Inconsistent old data (key occured twice).\n");
1272 if (data->attrs[entry])
1273 for (keyp = data->attrs[entry]; *keyp; keyp += 2)
1280 seen[*keyp] = keyp[1] + 1;
1284 /* Ideally we'd like to sort the new schema here, to ensure
1285 schema equality independend of the ordering. We can't do that
1286 yet. For once see below (old ids need to come before new ids).
1287 An additional difficulty is that we also need to move
1288 the values with the keys. */
1289 schemaid = addschema(data, schema, schematacache);
1291 schemaid = oldschema;
1294 /* Now create data blob. We walk through the (possibly new) schema
1295 and either copy over old data, or insert the new. */
1296 /* XXX Here we rely on the fact that the (new) schema has the form
1297 o1 o2 o3 o4 ... | n1 n2 n3 ...
1298 (oX being the old keyids (possibly overwritten), and nX being
1299 the new keyids). This rules out sorting the keyids in order
1300 to ensure a small schema count. */
1301 data->incoreoffset[entry] = newincore.len;
1302 data_addid(&newincore, schemaid);
1303 for (keyp = data->schemadata + data->schemata[schemaid]; *keyp; keyp++)
1305 key = data->keys + *keyp;
1309 /* Skip the data associated with this old key. */
1310 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1312 ndp = data_skip(dp, TYPE_ID);
1313 ndp = data_skip(ndp, TYPE_ID);
1315 else if (key->storage == KEY_STORAGE_INCORE)
1316 ndp = data_skip(dp, key->type);
1319 if (seen[*keyp] == -1)
1321 /* If this key was an old one _and_ was not overwritten with
1322 a different value copy over the old value (we skipped it
1325 data_addblob(&newincore, dp, ndp - dp);
1328 else if (seen[*keyp])
1330 /* Otherwise we have a new value. Parse it into the internal
1333 unsigned int oldvincorelen = 0;
1336 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1339 oldvincorelen = xd->len;
1341 id = seen[*keyp] - 1;
1348 data_addblob(xd, data->attrdata + id, strlen((char *)(data->attrdata + id)) + 1);
1355 case TYPE_DIRNUMNUMARRAY:
1356 for (ida = data->attriddata + id; *ida; ida += 3)
1358 data_addid(xd, ida[0]);
1359 data_addid(xd, ida[1]);
1360 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
1363 case TYPE_DIRSTRARRAY:
1364 for (ida = data->attriddata + id; *ida; ida += 2)
1366 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
1367 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
1371 fprintf(stderr, "don't know how to handle type %d\n", key->type);
1374 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1376 /* put offset/len in incore */
1377 data_addid(&newincore, data->lastverticaloffset + oldvincorelen);
1378 oldvincorelen = xd->len - oldvincorelen;
1379 data_addid(&newincore, oldvincorelen);
1384 if (data->attrs[entry])
1385 sat_free(data->attrs[entry]);
1390 sat_free(data->incoredata);
1391 data->incoredata = newincore.buf;
1392 data->incoredatalen = newincore.len;
1393 data->incoredatafree = 0;
1395 sat_free(data->vincore);
1396 data->vincore = newvincore.buf;
1397 data->vincorelen = newvincore.len;
1399 data->attrs = sat_free(data->attrs);
1400 data->attrdata = sat_free(data->attrdata);
1401 data->attriddata = sat_free(data->attriddata);
1402 data->attrdatalen = 0;
1403 data->attriddatalen = 0;
1407 repodata_str2dir(Repodata *data, const char *dir, int create)
1413 while (*dir == '/' && dir[1] == '/')
1417 dire = strchrnul(dir, '/');
1418 if (data->localpool)
1419 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
1421 id = strn2id(data->repo->pool, dir, dire - dir, create);
1424 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
1437 repodata_compress_page(unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max)
1439 return compress_buf(page, len, cpage, max);
1442 #define SOLV_ERROR_EOF 3
1444 static inline unsigned int
1450 for (i = 0; i < 4; i++)
1460 /* Try to either setup on-demand paging (using FP as backing
1461 file), or in case that doesn't work (FP not seekable) slurps in
1462 all pages and deactivates paging. */
1465 repodata_read_or_setup_pages(Repodata *data, unsigned int pagesz, unsigned int blobsz)
1467 FILE *fp = data->fp;
1468 unsigned int npages;
1470 unsigned int can_seek;
1472 unsigned char buf[BLOB_PAGESIZE];
1473 if (pagesz != BLOB_PAGESIZE)
1475 /* We could handle this by slurping in everything. */
1476 fprintf (stderr, "non matching page size\n");
1480 if ((cur_file_ofs = ftell(fp)) < 0)
1484 fprintf (stderr, "can %sseek\n", can_seek ? "" : "NOT ");
1486 npages = (blobsz + BLOB_PAGESIZE - 1) / BLOB_PAGESIZE;
1488 data->num_pages = npages;
1489 data->pages = sat_malloc2(npages, sizeof(data->pages[0]));
1491 /* If we can't seek on our input we have to slurp in everything. */
1493 data->blob_store = sat_malloc(npages * BLOB_PAGESIZE);
1494 for (i = 0; i < npages; i++)
1496 unsigned int in_len = read_u32(fp);
1497 unsigned int compressed = in_len & 1;
1498 Attrblobpage *p = data->pages + i;
1501 fprintf (stderr, "page %d: len %d (%scompressed)\n",
1502 i, in_len, compressed ? "" : "not ");
1508 p->file_offset = cur_file_ofs;
1509 p->file_size = in_len * 2 + compressed;
1510 if (fseek(fp, in_len, SEEK_CUR) < 0)
1513 fprintf (stderr, "can't seek after we thought we can\n");
1514 /* We can't fall back to non-seeking behaviour as we already
1515 read over some data pages without storing them away. */
1518 cur_file_ofs += in_len;
1522 unsigned int out_len;
1523 void *dest = data->blob_store + i * BLOB_PAGESIZE;
1524 p->mapped_at = i * BLOB_PAGESIZE;
1527 /* We can't seek, so suck everything in. */
1528 if (fread(compressed ? buf : dest, in_len, 1, fp) != 1)
1535 out_len = unchecked_decompress_buf(buf, in_len, dest, BLOB_PAGESIZE);
1536 if (out_len != BLOB_PAGESIZE
1539 fprintf (stderr, "can't decompress\n");
1548 /* If we are here we were able to seek to all page
1549 positions, so activate paging by copying FP into our structure.
1550 We dup() the file, so that our callers can fclose() it and we
1551 still have it open. But this means that we share file positions
1552 with the input filedesc. So in case our caller reads it after us,
1553 and calls back into us we might change the file position unexpectedly
1555 int fd = dup (fileno (fp));
1558 /* Jeez! What a bloody system, we can't dup() anymore. */
1562 /* XXX we don't close this yet anywhere. */
1563 data->fp = fdopen (fd, "r");
1566 /* My God! What happened now? */
1573 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: