2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
25 #include "poolid_private.h"
28 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
29 unsigned char *out, unsigned int out_len);
30 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
33 unsigned int out_len);
35 #define REPODATA_BLOCK 255
38 repodata_free(Repodata *data)
41 sat_free(data->schemata);
42 sat_free(data->schemadata);
44 sat_free(data->spool.strings);
45 sat_free(data->spool.stringspace);
46 sat_free(data->spool.stringhashtbl);
48 sat_free(data->dirpool.dirs);
49 sat_free(data->dirpool.dirtraverse);
51 sat_free(data->incoredata);
52 sat_free(data->incoreoffset);
53 sat_free(data->verticaloffset);
55 sat_free(data->blob_store);
56 sat_free(data->pages);
57 sat_free(data->mapped);
59 sat_free(data->vincore);
61 sat_free(data->attrs);
62 sat_free(data->attrdata);
63 sat_free(data->attriddata);
69 static unsigned char *
70 data_read_id(unsigned char *dp, Id *idp)
82 x = (x << 7) ^ c ^ 128;
86 static unsigned char *
87 data_read_ideof(unsigned char *dp, Id *idp, int *eof)
106 x = (x << 7) ^ c ^ 128;
110 static unsigned char *
111 data_skip(unsigned char *dp, int type)
122 while ((*dp & 0x80) != 0)
126 while ((*dp & 0xc0) != 0)
133 case TYPE_DIRSTRARRAY:
136 while ((*dp & 0x80) != 0)
145 case TYPE_DIRNUMNUMARRAY:
148 while ((*dp & 0x80) != 0)
151 while ((*dp & 0x80) != 0)
154 while ((*dp & 0x80) != 0)
161 fprintf(stderr, "unknown type in data_skip\n");
166 static unsigned char *
167 data_fetch(unsigned char *dp, KeyValue *kv, Repokey *key)
180 kv->str = (const char *)dp;
181 return dp + strlen(kv->str) + 1;
183 return data_read_id(dp, &kv->id);
185 return data_read_id(dp, &kv->num);
187 return data_read_ideof(dp, &kv->id, &kv->eof);
189 return data_read_id(dp, &kv->id);
190 case TYPE_DIRSTRARRAY:
191 dp = data_read_ideof(dp, &kv->id, &kv->eof);
192 kv->str = (const char *)dp;
193 return dp + strlen(kv->str) + 1;
194 case TYPE_DIRNUMNUMARRAY:
195 dp = data_read_id(dp, &kv->id);
196 dp = data_read_id(dp, &kv->num);
197 return data_read_ideof(dp, &kv->num2, &kv->eof);
203 static unsigned char *
204 forward_to_key(Repodata *data, Id key, Id schemaid, unsigned char *dp)
208 keyp = data->schemadata + data->schemata[schemaid];
209 while ((k = *keyp++) != 0)
213 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
215 dp = data_skip(dp, TYPE_ID); /* skip that offset */
216 dp = data_skip(dp, TYPE_ID); /* skip that length */
219 if (data->keys[k].storage != KEY_STORAGE_INCORE)
221 dp = data_skip(dp, data->keys[k].type);
226 #define BLOB_PAGEBITS 15
227 #define BLOB_PAGESIZE (1 << BLOB_PAGEBITS)
229 static unsigned char *
230 load_page_range(Repodata *data, unsigned int pstart, unsigned int pend)
232 /* Make sure all pages from PSTART to PEND (inclusive) are loaded,
233 and are consecutive. Return a pointer to the mapping of PSTART. */
234 unsigned char buf[BLOB_PAGESIZE];
237 /* Quick check in case all pages are there already and consecutive. */
238 for (i = pstart; i <= pend; i++)
239 if (data->pages[i].mapped_at == -1
241 && data->pages[i].mapped_at
242 != data->pages[i-1].mapped_at + BLOB_PAGESIZE))
245 return data->blob_store + data->pages[pstart].mapped_at;
247 /* Ensure that we can map the numbers of pages we need at all. */
248 if (pend - pstart + 1 > data->ncanmap)
250 unsigned int oldcan = data->ncanmap;
251 data->ncanmap = pend - pstart + 1;
252 if (data->ncanmap < 4)
254 data->mapped = sat_realloc2(data->mapped, data->ncanmap, sizeof(data->mapped[0]));
255 memset (data->mapped + oldcan, 0, (data->ncanmap - oldcan) * sizeof (data->mapped[0]));
256 data->blob_store = sat_realloc2(data->blob_store, data->ncanmap, BLOB_PAGESIZE);
258 fprintf (stderr, "PAGE: can map %d pages\n", data->ncanmap);
262 /* Now search for "cheap" space in our store. Space is cheap if it's either
263 free (very cheap) or contains pages we search for anyway. */
265 /* Setup cost array. */
266 unsigned int cost[data->ncanmap];
267 for (i = 0; i < data->ncanmap; i++)
269 unsigned int pnum = data->mapped[i];
275 Attrblobpage *p = data->pages + pnum;
276 assert (p->mapped_at != -1);
277 if (pnum >= pstart && pnum <= pend)
284 /* And search for cheapest space. */
285 unsigned int best_cost = -1;
286 unsigned int best = 0;
287 unsigned int same_cost = 0;
288 for (i = 0; i + pend - pstart < data->ncanmap; i++)
290 unsigned int c = cost[i];
292 for (j = 0; j < pend - pstart + 1; j++)
295 best_cost = c, best = i;
296 else if (c == best_cost)
298 /* A null cost won't become better. */
302 /* If all places have the same cost we would thrash on slot 0. Avoid
303 this by doing a round-robin strategy in this case. */
304 if (same_cost == data->ncanmap - pend + pstart - 1)
305 best = data->rr_counter++ % (data->ncanmap - pend + pstart);
307 /* So we want to map our pages from [best] to [best+pend-pstart].
308 Use a very simple strategy, which doesn't make the best use of
309 our resources, but works. Throw away all pages in that range
310 (even ours) then copy around ours (in case they were outside the
311 range) or read them in. */
312 for (i = best; i < best + pend - pstart + 1; i++)
314 unsigned int pnum = data->mapped[i];
316 /* If this page is exactly at the right place already,
317 no need to evict it. */
318 && pnum != pstart + i - best)
320 /* Evict this page. */
322 fprintf (stderr, "PAGE: evict page %d from %d\n", pnum, i);
326 data->pages[pnum].mapped_at = -1;
330 /* Everything is free now. Read in the pages we want. */
331 for (i = pstart; i <= pend; i++)
333 Attrblobpage *p = data->pages + i;
334 unsigned int pnum = i - pstart + best;
335 void *dest = data->blob_store + pnum * BLOB_PAGESIZE;
336 if (p->mapped_at != -1)
338 if (p->mapped_at != pnum * BLOB_PAGESIZE)
341 fprintf (stderr, "PAGECOPY: %d to %d\n", i, pnum);
343 /* Still mapped somewhere else, so just copy it from there. */
344 memcpy (dest, data->blob_store + p->mapped_at, BLOB_PAGESIZE);
345 data->mapped[p->mapped_at / BLOB_PAGESIZE] = 0;
350 unsigned int in_len = p->file_size;
351 unsigned int compressed = in_len & 1;
354 fprintf (stderr, "PAGEIN: %d to %d", i, pnum);
356 /* Not mapped, so read in this page. */
357 if (fseek(data->fp, p->file_offset, SEEK_SET) < 0)
359 perror ("mapping fseek");
362 if (fread(compressed ? buf : dest, in_len, 1, data->fp) != 1)
364 perror ("mapping fread");
369 unsigned int out_len;
370 out_len = unchecked_decompress_buf(buf, in_len,
371 dest, BLOB_PAGESIZE);
372 if (out_len != BLOB_PAGESIZE
373 && i < data->num_pages - 1)
375 fprintf (stderr, "can't decompress\n");
379 fprintf (stderr, " (expand %d to %d)", in_len, out_len);
383 fprintf (stderr, "\n");
386 p->mapped_at = pnum * BLOB_PAGESIZE;
387 data->mapped[pnum] = i + 1;
389 return data->blob_store + best * BLOB_PAGESIZE;
392 static unsigned char *
393 make_vertical_available(Repodata *data, Repokey *key, Id off, Id len)
396 if (key->type == TYPE_VOID)
398 if (off >= data->lastverticaloffset)
400 off -= data->lastverticaloffset;
401 if (off + len > data->vincorelen)
403 return data->vincore + off;
407 if (off + len > key->size)
409 /* we now have the offset, go into vertical */
410 off += data->verticaloffset[key - data->keys];
411 dp = load_page_range(data, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
413 dp += off % BLOB_PAGESIZE;
417 static inline unsigned char *
418 get_data(Repodata *data, Repokey *key, unsigned char **dpp)
420 unsigned char *dp = *dpp;
424 if (key->storage == KEY_STORAGE_INCORE)
426 /* hmm, this is a bit expensive */
427 *dpp = data_skip(dp, key->type);
430 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
433 dp = data_read_id(dp, &off);
434 dp = data_read_id(dp, &len);
436 return make_vertical_available(data, key, off, len);
442 maybe_load_repodata(Repodata *data)
444 if (data->state == REPODATA_STUB)
446 if (data->loadcallback)
447 data->loadcallback(data);
449 data->state = REPODATA_ERROR;
451 if (data->state == REPODATA_AVAILABLE)
453 data->state = REPODATA_ERROR;
458 repodata_lookup_str(Repodata *data, Id entry, Id keyid)
465 if (!maybe_load_repodata (data))
468 dp = data->incoredata + data->incoreoffset[entry];
469 dp = data_read_id(dp, &schema);
470 /* make sure the schema of this solvable contains the key */
471 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
474 dp = forward_to_key(data, keyid, schema, dp);
475 key = data->keys + keyid;
476 dp = get_data(data, key, &dp);
479 if (key->type == TYPE_STR)
480 return (const char *)dp;
481 if (key->type != TYPE_ID)
483 /* id type, must either use global or local string store*/
484 dp = data_read_id(dp, &id);
486 return data->spool.stringspace + data->spool.strings[id];
487 return id2str(data->repo->pool, id);
491 repodata_lookup_num(Repodata *data, Id entry, Id keyid, unsigned *value)
501 if (!maybe_load_repodata (data))
504 dp = data->incoredata + data->incoreoffset[entry];
505 dp = data_read_id(dp, &schema);
506 /* make sure the schema of this solvable contains the key */
507 for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++)
510 dp = forward_to_key(data, keyid, schema, dp);
511 key = data->keys + keyid;
512 dp = get_data(data, key, &dp);
515 if (key->type == TYPE_NUM
516 || key->type == TYPE_U32
517 || key->type == TYPE_CONSTANT)
519 dp = data_fetch(dp, &kv, key);
527 repodata_search(Repodata *data, Id entry, Id keyname, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
531 Id k, keyid, *kp, *keyp;
532 unsigned char *dp, *ddp;
537 if (!maybe_load_repodata (data))
540 dp = data->incoredata + data->incoreoffset[entry];
541 dp = data_read_id(dp, &schema);
542 keyp = data->schemadata + data->schemata[schema];
545 /* search in a specific key */
546 for (kp = keyp; (k = *kp++) != 0; )
547 if (data->keys[k].name == keyname)
551 dp = forward_to_key(data, k, schema, dp);
557 while ((keyid = *keyp++) != 0)
560 key = data->keys + keyid;
561 ddp = get_data(data, key, &dp);
564 ddp = data_fetch(ddp, &kv, key);
567 stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv);
569 while (!kv.eof && !stop);
570 if (onekey || stop > SEARCH_NEXT_KEY)
576 repodata_init(Repodata *data, Repo *repo, int localpool)
578 memset(data, 0, sizeof (*data));
580 data->localpool = localpool;
582 stringpool_init_empty(&data->spool);
583 data->keys = sat_calloc(1, sizeof(Repokey));
585 data->schemata = sat_calloc(1, sizeof(Id));
586 data->schemadata = sat_calloc(1, sizeof(Id));
588 data->schemadatalen = 1;
589 data->start = repo->start;
590 data->end = repo->end;
591 data->incoreoffset = sat_extend_resize(0, data->end - data->start, sizeof(Id), REPODATA_BLOCK);
594 /* extend repodata so that it includes solvables p */
596 repodata_extend(Repodata *data, Id p)
598 if (data->start == data->end)
599 data->start = data->end = p;
602 int old = data->end - data->start;
603 int new = p - data->end + 1;
606 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
607 memset(data->attrs + old, 0, new * sizeof(Id *));
609 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
610 memset(data->incoreoffset + old, 0, new * sizeof(Id));
615 int old = data->end - data->start;
616 int new = data->start - p;
619 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
620 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
621 memset(data->attrs, 0, new * sizeof(Id *));
623 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
624 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
625 memset(data->incoreoffset, 0, new * sizeof(Id));
631 repodata_extend_block(Repodata *data, Id start, Id num)
635 if (!data->incoreoffset)
637 data->incoreoffset = sat_extend_resize(data->incoreoffset, num, sizeof(Id), REPODATA_BLOCK);
638 memset(data->incoreoffset, 0, num * sizeof(Id));
640 data->end = start + num;
643 repodata_extend(data, start);
645 repodata_extend(data, start + num - 1);
648 #define REPODATA_ATTRS_BLOCK 63
649 #define REPODATA_ATTRDATA_BLOCK 1023
650 #define REPODATA_ATTRIDDATA_BLOCK 63
653 repodata_insert_keyid(Repodata *data, Id entry, Id keyid, Id val, int overwrite)
659 data->attrs = sat_extend_resize(0, data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
660 memset(data->attrs, 0, (data->end - data->start) * sizeof(Id *));
663 if (data->attrs[entry])
665 for (pp = data->attrs[entry]; *pp; pp += 2)
666 /* Determine equality based on the name only, allows us to change
667 type (when overwrite is set), and makes TYPE_CONSTANT work. */
668 if (data->keys[*pp].name == data->keys[keyid].name)
679 i = pp - data->attrs[entry];
681 data->attrs[entry] = sat_extend(data->attrs[entry], i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
682 pp = data->attrs[entry] + i;
689 repodata_set(Repodata *data, Id entry, Repokey *key, Id val)
693 /* find key in keys */
694 for (keyid = 1; keyid < data->nkeys; keyid++)
695 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
697 if (key->type == TYPE_CONSTANT && key->size != data->keys[keyid].size)
701 if (keyid == data->nkeys)
703 /* allocate new key */
704 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
705 data->keys[data->nkeys++] = *key;
706 if (data->verticaloffset)
708 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
709 data->verticaloffset[data->nkeys - 1] = 0;
712 repodata_insert_keyid(data, entry, keyid, val, 1);
716 repodata_set_id(Repodata *data, Id entry, Id keyname, Id id)
722 key.storage = KEY_STORAGE_INCORE;
723 repodata_set(data, entry, &key, id);
727 repodata_set_num(Repodata *data, Id entry, Id keyname, Id num)
733 key.storage = KEY_STORAGE_INCORE;
734 repodata_set(data, entry, &key, num);
738 repodata_set_poolstr(Repodata *data, Id entry, Id keyname, const char *str)
743 id = stringpool_str2id(&data->spool, str, 1);
745 id = str2id(data->repo->pool, str, 1);
749 key.storage = KEY_STORAGE_INCORE;
750 repodata_set(data, entry, &key, id);
754 repodata_set_constant(Repodata *data, Id entry, Id keyname, Id constant)
758 key.type = TYPE_CONSTANT;
760 key.storage = KEY_STORAGE_INCORE;
761 repodata_set(data, entry, &key, 0);
765 repodata_set_void(Repodata *data, Id entry, Id keyname)
769 key.type = TYPE_VOID;
771 key.storage = KEY_STORAGE_INCORE;
772 repodata_set(data, entry, &key, 0);
776 repodata_set_str(Repodata *data, Id entry, Id keyname, const char *str)
785 key.storage = KEY_STORAGE_INCORE;
786 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
787 memcpy(data->attrdata + data->attrdatalen, str, l);
788 repodata_set(data, entry, &key, data->attrdatalen);
789 data->attrdatalen += l;
793 repodata_add_dirnumnum(Repodata *data, Id entry, Id keyname, Id dir, Id num, Id num2)
799 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", entry, dir, num, num2, data->attriddatalen);
801 if (data->attrs && data->attrs[entry])
803 for (pp = data->attrs[entry]; *pp; pp += 2)
804 if (data->keys[*pp].name == keyname && data->keys[*pp].type == TYPE_DIRNUMNUMARRAY)
809 for (ida = data->attriddata + pp[1]; *ida; ida += 3)
811 if (ida + 1 == data->attriddata + data->attriddatalen)
813 /* this was the last entry, just append it */
814 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 3, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
815 data->attriddatalen--; /* overwrite terminating 0 */
819 /* too bad. move to back. */
820 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + 4, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
821 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
822 pp[1] = data->attriddatalen;
823 data->attriddatalen += oldsize;
825 data->attriddata[data->attriddatalen++] = dir;
826 data->attriddata[data->attriddatalen++] = num;
827 data->attriddata[data->attriddatalen++] = num2;
828 data->attriddata[data->attriddatalen++] = 0;
833 key.type = TYPE_DIRNUMNUMARRAY;
835 key.storage = KEY_STORAGE_INCORE;
836 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 4, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
837 repodata_set(data, entry, &key, data->attriddatalen);
838 data->attriddata[data->attriddatalen++] = dir;
839 data->attriddata[data->attriddatalen++] = num;
840 data->attriddata[data->attriddatalen++] = num2;
841 data->attriddata[data->attriddatalen++] = 0;
845 repodata_add_dirstr(Repodata *data, Id entry, Id keyname, Id dir, const char *str)
847 Id *ida, *pp, stroff;
852 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
853 memcpy(data->attrdata + data->attrdatalen, str, l);
854 stroff = data->attrdatalen;
855 data->attrdatalen += l;
858 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", entry, dir, str, data->attriddatalen);
860 if (data->attrs && data->attrs[entry])
862 for (pp = data->attrs[entry]; *pp; pp += 2)
863 if (data->keys[*pp].name == keyname && data->keys[*pp].type == TYPE_DIRSTRARRAY)
868 for (ida = data->attriddata + pp[1]; *ida; ida += 2)
870 if (ida + 1 == data->attriddata + data->attriddatalen)
872 /* this was the last entry, just append it */
873 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 2, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
874 data->attriddatalen--; /* overwrite terminating 0 */
878 /* too bad. move to back. */
879 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + 3, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
880 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
881 pp[1] = data->attriddatalen;
882 data->attriddatalen += oldsize;
884 data->attriddata[data->attriddatalen++] = dir;
885 data->attriddata[data->attriddatalen++] = stroff;
886 data->attriddata[data->attriddatalen++] = 0;
891 key.type = TYPE_DIRSTRARRAY;
893 key.storage = KEY_STORAGE_INCORE;
894 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, 3, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
895 repodata_set(data, entry, &key, data->attriddatalen);
896 data->attriddata[data->attriddatalen++] = dir;
897 data->attriddata[data->attriddatalen++] = stroff;
898 data->attriddata[data->attriddatalen++] = 0;
902 repodata_merge_attrs (Repodata *data, Id dest, Id src)
905 for (keyp = data->attrs[src]; *keyp; keyp += 2)
906 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
909 /*********************************/
911 /* unify with repo_write! */
913 #define EXTDATA_BLOCK 1023
914 #define SCHEMATA_BLOCK 31
915 #define SCHEMATADATA_BLOCK 255
923 data_addid(struct extdata *xd, Id x)
926 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
927 dp = xd->buf + xd->len;
932 *dp++ = (x >> 28) | 128;
934 *dp++ = (x >> 21) | 128;
935 *dp++ = (x >> 14) | 128;
938 *dp++ = (x >> 7) | 128;
940 xd->len = dp - xd->buf;
944 data_addideof(struct extdata *xd, Id x, int eof)
947 x = (x & 63) | ((x & ~63) << 1);
948 data_addid(xd, (eof ? x: x | 64));
952 data_addblob(struct extdata *xd, unsigned char *blob, int len)
954 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
955 memcpy(xd->buf + xd->len, blob, len);
959 /*********************************/
962 addschema_prepare(Repodata *data, Id *schematacache)
967 memset(schematacache, 0, 256 * sizeof(Id));
968 for (i = 0; i < data->nschemata; i++)
970 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
973 schematacache[h] = i + 1;
975 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
976 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
980 addschema(Repodata *data, Id *schema, Id *schematacache)
985 for (sp = schema, len = 0, h = 0; *sp; len++)
990 cid = schematacache[h];
994 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
997 for (cid = 0; cid < data->nschemata; cid++)
998 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
1001 /* a new one. make room. */
1002 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
1003 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
1005 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
1006 data->schemata[data->nschemata] = data->schemadatalen;
1007 data->schemadatalen += len;
1008 schematacache[h] = data->nschemata + 1;
1010 fprintf(stderr, "addschema: new schema\n");
1012 return data->nschemata++;
1017 repodata_internalize(Repodata *data)
1020 Id id, entry, nentry, *ida;
1021 Id schematacache[256];
1022 Id schemaid, *schema, *sp, oldschema, *keyp, *seen;
1023 unsigned char *dp, *ndp;
1024 int newschema, oldcount;
1025 struct extdata newincore;
1026 struct extdata newvincore;
1031 newvincore.buf = data->vincore;
1032 newvincore.len = data->vincorelen;
1034 schema = sat_malloc2(data->nkeys, sizeof(Id));
1035 seen = sat_malloc2(data->nkeys, sizeof(Id));
1037 /* Merge the data already existing (in data->schemata, ->incoredata and
1038 friends) with the new attributes in data->attrs[]. */
1039 nentry = data->end - data->start;
1040 addschema_prepare(data, schematacache);
1041 memset(&newincore, 0, sizeof(newincore));
1042 for (entry = 0; entry < nentry; entry++)
1044 memset(seen, 0, data->nkeys * sizeof(Id));
1046 dp = data->incoredata + data->incoreoffset[entry];
1047 if (data->incoredata)
1048 dp = data_read_id(dp, &oldschema);
1052 fprintf(stderr, "oldschema %d\n", oldschema);
1053 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
1054 fprintf(stderr, "schemadata %p\n", data->schemadata);
1056 /* seen: -1: old data 0: skipped >0: id + 1 */
1059 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
1063 fprintf(stderr, "Inconsistent old data (key occured twice).\n");
1070 if (data->attrs[entry])
1071 for (keyp = data->attrs[entry]; *keyp; keyp += 2)
1078 seen[*keyp] = keyp[1] + 1;
1082 /* Ideally we'd like to sort the new schema here, to ensure
1083 schema equality independend of the ordering. We can't do that
1084 yet. For once see below (old ids need to come before new ids).
1085 An additional difficulty is that we also need to move
1086 the values with the keys. */
1087 schemaid = addschema(data, schema, schematacache);
1089 schemaid = oldschema;
1092 /* Now create data blob. We walk through the (possibly new) schema
1093 and either copy over old data, or insert the new. */
1094 /* XXX Here we rely on the fact that the (new) schema has the form
1095 o1 o2 o3 o4 ... | n1 n2 n3 ...
1096 (oX being the old keyids (possibly overwritten), and nX being
1097 the new keyids). This rules out sorting the keyids in order
1098 to ensure a small schema count. */
1099 data->incoreoffset[entry] = newincore.len;
1100 data_addid(&newincore, schemaid);
1101 for (keyp = data->schemadata + data->schemata[schemaid]; *keyp; keyp++)
1103 key = data->keys + *keyp;
1107 /* Skip the data associated with this old key. */
1108 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1110 ndp = data_skip(dp, TYPE_ID);
1111 ndp = data_skip(ndp, TYPE_ID);
1113 else if (key->storage == KEY_STORAGE_INCORE)
1114 ndp = data_skip(dp, key->type);
1117 if (seen[*keyp] == -1)
1119 /* If this key was an old one _and_ was not overwritten with
1120 a different value copy over the old value (we skipped it
1123 data_addblob(&newincore, dp, ndp - dp);
1126 else if (seen[*keyp])
1128 /* Otherwise we have a new value. Parse it into the internal
1131 unsigned int oldvincorelen = 0;
1134 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1137 oldvincorelen = xd->len;
1139 id = seen[*keyp] - 1;
1146 data_addblob(xd, data->attrdata + id, strlen((char *)(data->attrdata + id)) + 1);
1153 case TYPE_DIRNUMNUMARRAY:
1154 for (ida = data->attriddata + id; *ida; ida += 3)
1156 data_addid(xd, ida[0]);
1157 data_addid(xd, ida[1]);
1158 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
1161 case TYPE_DIRSTRARRAY:
1162 for (ida = data->attriddata + id; *ida; ida += 2)
1164 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
1165 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
1169 fprintf(stderr, "don't know how to handle type %d\n", key->type);
1172 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1174 /* put offset/len in incore */
1175 data_addid(&newincore, data->lastverticaloffset + oldvincorelen);
1176 oldvincorelen = xd->len - oldvincorelen;
1177 data_addid(&newincore, oldvincorelen);
1182 if (data->attrs[entry])
1183 sat_free(data->attrs[entry]);
1188 sat_free(data->incoredata);
1189 data->incoredata = newincore.buf;
1190 data->incoredatalen = newincore.len;
1191 data->incoredatafree = 0;
1193 sat_free(data->vincore);
1194 data->vincore = newvincore.buf;
1195 data->vincorelen = newvincore.len;
1197 data->attrs = sat_free(data->attrs);
1198 data->attrdata = sat_free(data->attrdata);
1199 data->attriddata = sat_free(data->attriddata);
1200 data->attrdatalen = 0;
1201 data->attriddatalen = 0;
1205 repodata_str2dir(Repodata *data, const char *dir, int create)
1211 while (*dir == '/' && dir[1] == '/')
1215 dire = strchrnul(dir, '/');
1216 if (data->localpool)
1217 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
1219 id = strn2id(data->repo->pool, dir, dire - dir, create);
1222 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
1235 repodata_compress_page(unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max)
1237 return compress_buf(page, len, cpage, max);
1240 #define SOLV_ERROR_EOF 3
1242 static inline unsigned int
1248 for (i = 0; i < 4; i++)
1258 /* Try to either setup on-demand paging (using FP as backing
1259 file), or in case that doesn't work (FP not seekable) slurps in
1260 all pages and deactivates paging. */
1263 repodata_read_or_setup_pages(Repodata *data, unsigned int pagesz, unsigned int blobsz)
1265 FILE *fp = data->fp;
1266 unsigned int npages;
1268 unsigned int can_seek;
1270 unsigned char buf[BLOB_PAGESIZE];
1271 if (pagesz != BLOB_PAGESIZE)
1273 /* We could handle this by slurping in everything. */
1274 fprintf (stderr, "non matching page size\n");
1278 if ((cur_file_ofs = ftell(fp)) < 0)
1282 fprintf (stderr, "can %sseek\n", can_seek ? "" : "NOT ");
1284 npages = (blobsz + BLOB_PAGESIZE - 1) / BLOB_PAGESIZE;
1286 data->num_pages = npages;
1287 data->pages = sat_malloc2(npages, sizeof(data->pages[0]));
1289 /* If we can't seek on our input we have to slurp in everything. */
1291 data->blob_store = sat_malloc(npages * BLOB_PAGESIZE);
1292 for (i = 0; i < npages; i++)
1294 unsigned int in_len = read_u32(fp);
1295 unsigned int compressed = in_len & 1;
1296 Attrblobpage *p = data->pages + i;
1299 fprintf (stderr, "page %d: len %d (%scompressed)\n",
1300 i, in_len, compressed ? "" : "not ");
1306 p->file_offset = cur_file_ofs;
1307 p->file_size = in_len * 2 + compressed;
1308 if (fseek(fp, in_len, SEEK_CUR) < 0)
1311 fprintf (stderr, "can't seek after we thought we can\n");
1312 /* We can't fall back to non-seeking behaviour as we already
1313 read over some data pages without storing them away. */
1316 cur_file_ofs += in_len;
1320 unsigned int out_len;
1321 void *dest = data->blob_store + i * BLOB_PAGESIZE;
1322 p->mapped_at = i * BLOB_PAGESIZE;
1325 /* We can't seek, so suck everything in. */
1326 if (fread(compressed ? buf : dest, in_len, 1, fp) != 1)
1333 out_len = unchecked_decompress_buf(buf, in_len, dest, BLOB_PAGESIZE);
1334 if (out_len != BLOB_PAGESIZE
1337 fprintf (stderr, "can't decompress\n");
1346 /* If we are here we were able to seek to all page
1347 positions, so activate paging by copying FP into our structure.
1348 We dup() the file, so that our callers can fclose() it and we
1349 still have it open. But this means that we share file positions
1350 with the input filedesc. So in case our caller reads it after us,
1351 and calls back into us we might change the file position unexpectedly
1353 int fd = dup (fileno (fp));
1356 /* Jeez! What a bloody system, we can't dup() anymore. */
1360 /* XXX we don't close this yet anywhere. */
1361 data->fp = fdopen (fd, "r");
1364 /* My God! What happened now? */