2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
26 #include "poolid_private.h"
32 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
33 unsigned char *out, unsigned int out_len);
34 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
37 unsigned int out_len);
39 #define REPODATA_BLOCK 255
43 repodata_init(Repodata *data, Repo *repo, int localpool)
45 memset(data, 0, sizeof (*data));
47 data->localpool = localpool;
49 stringpool_init_empty(&data->spool);
50 data->keys = sat_calloc(1, sizeof(Repokey));
52 data->schemata = sat_calloc(1, sizeof(Id));
53 data->schemadata = sat_calloc(1, sizeof(Id));
55 data->schemadatalen = 1;
60 repodata_free(Repodata *data)
66 sat_free(data->schemata);
67 sat_free(data->schemadata);
68 sat_free(data->schematahash);
70 stringpool_free(&data->spool);
71 dirpool_free(&data->dirpool);
73 sat_free(data->mainschemaoffsets);
74 sat_free(data->incoredata);
75 sat_free(data->incoreoffset);
76 sat_free(data->verticaloffset);
78 sat_free(data->blob_store);
79 sat_free(data->pages);
80 sat_free(data->mapped);
82 sat_free(data->vincore);
85 for (i = 0; i < data->end - data->start; i++)
86 sat_free(data->attrs[i]);
87 sat_free(data->attrs);
89 for (i = 0; i < data->nxattrs; i++)
90 sat_free(data->xattrs[i]);
91 sat_free(data->xattrs);
93 sat_free(data->attrdata);
94 sat_free(data->attriddata);
96 if (data->pagefd != -1)
101 /***************************************************************
102 * key pool management
105 /* this is not so time critical that we need a hash, so we do a simple
108 repodata_key2id(Repodata *data, Repokey *key, int create)
112 for (keyid = 1; keyid < data->nkeys; keyid++)
113 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
115 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
119 if (keyid == data->nkeys)
123 /* allocate new key */
124 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
125 data->keys[data->nkeys++] = *key;
126 if (data->verticaloffset)
128 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
129 data->verticaloffset[data->nkeys - 1] = 0;
131 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
137 /***************************************************************
138 * schema pool management
141 #define SCHEMATA_BLOCK 31
142 #define SCHEMATADATA_BLOCK 255
145 repodata_schema2id(Repodata *data, Id *schema, int create)
151 if ((schematahash = data->schematahash) == 0)
153 data->schematahash = schematahash = sat_calloc(256, sizeof(Id));
154 for (i = 0; i < data->nschemata; i++)
156 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
159 schematahash[h] = i + 1;
161 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
162 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
165 for (sp = schema, len = 0, h = 0; *sp; len++)
170 cid = schematahash[h];
174 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
177 for (cid = 0; cid < data->nschemata; cid++)
178 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
184 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
185 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
187 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
188 data->schemata[data->nschemata] = data->schemadatalen;
189 data->schemadatalen += len;
190 schematahash[h] = data->nschemata + 1;
192 fprintf(stderr, "schema2id: new schema\n");
194 return data->nschemata++;
198 repodata_free_schemahash(Repodata *data)
200 data->schematahash = sat_free(data->schematahash);
202 data->schemata = sat_realloc2(data->schemata, data->nschemata, sizeof(Id));
203 data->schemadata = sat_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
207 /***************************************************************
208 * dir pool management
212 repodata_str2dir(Repodata *data, const char *dir, int create)
218 while (*dir == '/' && dir[1] == '/')
220 if (*dir == '/' && !dir[1])
224 dire = strchrnul(dir, '/');
226 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
228 id = strn2id(data->repo->pool, dir, dire - dir, create);
231 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
244 repodata_dir2str(Repodata *data, Id did, const char *suf)
246 Pool *pool = data->repo->pool;
253 return suf ? suf : "";
257 comp = dirpool_compid(&data->dirpool, parent);
258 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
260 parent = dirpool_parent(&data->dirpool, parent);
265 l += strlen(suf) + 1;
266 p = pool_alloctmpspace(pool, l + 1) + l;
277 comp = dirpool_compid(&data->dirpool, parent);
278 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
281 strncpy(p, comps, l);
282 parent = dirpool_parent(&data->dirpool, parent);
290 /***************************************************************
294 static inline unsigned char *
295 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
297 Id *keyp = data->schemadata + data->schemata[schema];
298 for (; *keyp; keyp++)
299 dp = data_skip_key(data, dp, data->keys + *keyp);
304 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
306 int nentries, schema;
309 case REPOKEY_TYPE_FIXARRAY:
310 dp = data_read_id(dp, &nentries);
313 dp = data_read_id(dp, &schema);
315 dp = data_skip_schema(data, dp, schema);
317 case REPOKEY_TYPE_FLEXARRAY:
318 dp = data_read_id(dp, &nentries);
321 dp = data_read_id(dp, &schema);
322 dp = data_skip_schema(data, dp, schema);
326 if (key->storage == KEY_STORAGE_INCORE)
327 dp = data_skip(dp, key->type);
328 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
330 dp = data_skip(dp, REPOKEY_TYPE_ID);
331 dp = data_skip(dp, REPOKEY_TYPE_ID);
337 static unsigned char *
338 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
344 while ((k = *keyp++) != 0)
348 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
350 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
351 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
354 if (data->keys[k].storage != KEY_STORAGE_INCORE)
356 dp = data_skip_key(data, dp, data->keys + k);
361 static unsigned char *
362 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
367 if (off >= data->lastverticaloffset)
369 off -= data->lastverticaloffset;
370 if (off + len > data->vincorelen)
372 return data->vincore + off;
374 if (off + len > key->size)
376 /* we now have the offset, go into vertical */
377 off += data->verticaloffset[key - data->keys];
378 /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */
379 dp = repodata_load_page_range(data, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
381 dp += off % BLOB_PAGESIZE;
385 static inline unsigned char *
386 get_data(Repodata *data, Repokey *key, unsigned char **dpp)
388 unsigned char *dp = *dpp;
392 if (key->storage == KEY_STORAGE_INCORE)
394 /* hmm, this is a bit expensive */
395 *dpp = data_skip_key(data, dp, key);
398 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
401 dp = data_read_id(dp, &off);
402 dp = data_read_id(dp, &len);
404 return get_vertical_data(data, key, off, len);
410 load_repodata(Repodata *data)
412 if (data->loadcallback)
414 data->loadcallback(data);
415 if (data->state == REPODATA_AVAILABLE)
418 data->state = REPODATA_ERROR;
423 maybe_load_repodata(Repodata *data, Id keyname)
425 if (keyname && !repodata_precheck_keyname(data, keyname))
426 return 0; /* do not bother... */
433 for (i = 0; i < data->nkeys; i++)
434 if (keyname == data->keys[i].name)
436 if (i == data->nkeys)
439 return load_repodata(data);
442 case REPODATA_AVAILABLE:
445 data->state = REPODATA_ERROR;
450 static inline unsigned char*
451 entry2data(Repodata *data, Id entry, Id *schemap)
453 unsigned char *dp = data->incoredata;
456 if (entry == REPOENTRY_META) /* META */
458 else if (entry == REPOENTRY_POS) /* META */
460 Pool *pool = data->repo->pool;
461 if (data->repo != pool->pos.repo)
463 if (data != data->repo->repodata + pool->pos.repodataid)
465 *schemap = pool->pos.schema;
466 return data->incoredata + pool->pos.dp;
470 if (entry < data->start || entry >= data->end)
472 dp += data->incoreoffset[entry - data->start];
474 return data_read_id(dp, schemap);
477 /************************************************************************
482 find_schema_key(Repodata *data, Id schema, Id keyname)
485 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
486 if (data->keys[*keyp].name == keyname)
491 static inline unsigned char *
492 find_key_data(Repodata *data, Id entry, Id keyname, Repokey **keyp)
494 unsigned char *dp, *ddp;
498 if (!maybe_load_repodata(data, keyname))
500 dp = entry2data(data, entry, &schema);
503 keyid = find_schema_key(data, schema, keyname);
506 key = data->keys + keyid;
508 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
510 dp = forward_to_key(data, keyid, data->schemadata + data->schemata[schema], dp);
513 ddp = get_data(data, key, &dp);
519 repodata_lookup_id(Repodata *data, Id entry, Id keyname)
525 dp = find_key_data(data, entry, keyname, &key);
528 if (key->type == REPOKEY_TYPE_CONSTANTID)
530 if (key->type != REPOKEY_TYPE_ID)
532 dp = data_read_id(dp, &id);
537 repodata_lookup_str(Repodata *data, Id entry, Id keyname)
543 dp = find_key_data(data, entry, keyname, &key);
546 if (key->type == REPOKEY_TYPE_STR)
547 return (const char *)dp;
548 if (key->type == REPOKEY_TYPE_CONSTANTID)
549 return id2str(data->repo->pool, key->size);
550 if (key->type == REPOKEY_TYPE_ID)
551 dp = data_read_id(dp, &id);
555 return data->spool.stringspace + data->spool.strings[id];
556 return id2str(data->repo->pool, id);
560 repodata_lookup_num(Repodata *data, Id entry, Id keyname, unsigned int *value)
567 dp = find_key_data(data, entry, keyname, &key);
570 if (key->type == REPOKEY_TYPE_NUM
571 || key->type == REPOKEY_TYPE_U32
572 || key->type == REPOKEY_TYPE_CONSTANT)
574 dp = data_fetch(dp, &kv, key);
582 repodata_lookup_void(Repodata *data, Id entry, Id keyname)
588 if (!maybe_load_repodata(data, keyname))
590 dp = entry2data(data, entry, &schema);
593 /* can't use find_schema_key as we need to test the type */
594 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
595 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
600 const unsigned char *
601 repodata_lookup_bin_checksum(Repodata *data, Id entry, Id keyname, Id *typep)
606 dp = find_key_data(data, entry, keyname, &key);
614 /************************************************************************
618 struct subschema_data {
624 /* search in a specific entry */
626 repodata_search(Repodata *data, Id entry, Id keyname, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
630 Id k, keyid, *kp, *keyp;
631 unsigned char *dp, *ddp;
637 if (!maybe_load_repodata(data, keyname))
639 if (entry == REPOENTRY_SUBSCHEMA)
641 struct subschema_data *subd = cbdata;
642 cbdata = subd->cbdata;
644 schema = subd->parent->id;
645 dp = (unsigned char *)subd->parent->str;
646 kv.parent = subd->parent;
651 dp = entry2data(data, entry, &schema);
654 s = data->repo->pool->solvables + entry;
657 keyp = data->schemadata + data->schemata[schema];
660 /* search for a specific key */
661 for (kp = keyp; (k = *kp++) != 0; )
662 if (data->keys[k].name == keyname)
666 dp = forward_to_key(data, k, data->schemadata + data->schemata[schema], dp);
672 while ((keyid = *keyp++) != 0)
675 key = data->keys + keyid;
676 ddp = get_data(data, key, &dp);
678 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
680 struct subschema_data subd;
684 subd.cbdata = cbdata;
687 ddp = data_read_id(ddp, &nentries);
690 while (ddp && nentries > 0)
692 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
693 ddp = data_read_id(ddp, &schema);
695 kv.str = (char *)ddp;
696 stop = callback(cbdata, s, data, key, &kv);
697 if (stop > SEARCH_NEXT_KEY)
702 repodata_search(data, REPOENTRY_SUBSCHEMA, 0, callback, &subd);
703 ddp = data_skip_schema(data, ddp, schema);
711 kv.str = (char *)ddp;
712 stop = callback(cbdata, s, data, key, &kv);
713 if (stop > SEARCH_NEXT_KEY)
723 ddp = data_fetch(ddp, &kv, key);
726 stop = callback(cbdata, s, data, key, &kv);
729 while (!kv.eof && !stop);
730 if (onekey || stop > SEARCH_NEXT_KEY)
736 repodata_setpos_kv(Repodata *data, KeyValue *kv)
738 Pool *pool = data->repo->pool;
742 pool->pos.repodataid = 0;
744 pool->pos.schema = 0;
749 pool->pos.repodataid = data - data->repo->repodata;
750 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
751 pool->pos.schema = kv->id;
755 /************************************************************************/
757 static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = {
758 { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
759 { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
760 { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
761 { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
762 { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
763 { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
764 { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
765 { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
766 { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
767 { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
768 { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
769 { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
770 { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE },
775 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
789 case SOLVABLE_VENDOR:
792 case SOLVABLE_PROVIDES:
794 return s->provides ? s->repo->idarraydata + s->provides : 0;
795 case SOLVABLE_OBSOLETES:
797 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
798 case SOLVABLE_CONFLICTS:
800 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
801 case SOLVABLE_REQUIRES:
803 return s->requires ? s->repo->idarraydata + s->requires : 0;
804 case SOLVABLE_RECOMMENDS:
806 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
807 case SOLVABLE_SUPPLEMENTS:
809 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
810 case SOLVABLE_SUGGESTS:
812 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
813 case SOLVABLE_ENHANCES:
815 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
818 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
825 datamatcher_init(Datamatcher *ma, Pool *pool, const char *match, int flags)
828 ma->match = (void *)match;
831 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
833 ma->match = sat_calloc(1, sizeof(regex_t));
834 ma->error = regcomp((regex_t *)ma->match, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
838 ma->match = (void *)match;
839 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
845 datamatcher_free(Datamatcher *ma)
847 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->match)
850 ma->match = sat_free(ma->match);
855 datamatcher_match(Datamatcher *ma, Repodata *data, Repokey *key, KeyValue *kv)
859 case REPOKEY_TYPE_ID:
860 case REPOKEY_TYPE_IDARRAY:
861 if (data && data->localpool)
862 kv->str = stringpool_id2str(&data->spool, kv->id);
864 kv->str = id2str(ma->pool, kv->id);
866 case REPOKEY_TYPE_STR:
868 case REPOKEY_TYPE_DIRSTRARRAY:
869 if (!(ma->flags & SEARCH_FILES))
871 /* Put the full filename into kv->str. */
872 kv->str = repodata_dir2str(data, kv->id, kv->str);
873 /* And to compensate for that put the "empty" directory into
874 kv->id, so that later calls to repodata_dir2str on this data
875 come up with the same filename again. */
881 /* Maybe skip the kind specifier. Do this only for SOLVABLE attributes,
882 for the others we can't know if a colon separates a kind or not. */
883 if ((ma->flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
885 const char *s = strchr(kv->str, ':');
889 switch ((ma->flags & SEARCH_STRINGMASK))
891 case SEARCH_SUBSTRING:
892 if (ma->flags & SEARCH_NOCASE)
894 if (!strcasestr(kv->str, (const char *)ma->match))
899 if (!strstr(kv->str, (const char *)ma->match))
904 if (ma->flags & SEARCH_NOCASE)
906 if (strcasecmp((const char *)ma->match, kv->str))
911 if (strcmp((const char *)ma->match, kv->str))
916 if (fnmatch((const char *)ma->match, kv->str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
920 if (regexec((const regex_t *)ma->match, kv->str, 0, NULL, 0))
953 dataiterator_init(Dataiterator *di, Repo *repo, Id p, Id keyname, const char *match, int flags)
955 memset(di, 0, sizeof(*di));
957 di->keyname = keyname;
959 di->pool = repo->pool;
961 flags |= SEARCH_THISENTRY;
966 datamatcher_init(&di->matcher, di->pool, match, flags);
967 if (p == REPOENTRY_POS)
969 di->repo = di->pool->pos.repo;
970 di->data = di->repo->repodata + di->pool->pos.repodataid;
974 di->state = di_enterrepo;
978 dataiterator_free(Dataiterator *di)
980 if (di->matcher.match)
981 datamatcher_free(&di->matcher);
985 dataiterator_step(Dataiterator *di)
993 case di_nextattr: di_nextattr:
995 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
997 di->state = di_nextkey;
999 di->state = di_nextattr;
1002 case di_nextkey: di_nextkey:
1008 else if ((di->flags & SEARCH_SUB) != 0)
1010 Id *keyp = di->keyp;
1011 for (keyp++; *keyp; keyp++)
1012 if (di->data->keys[*keyp].name == di->keyname ||
1013 di->data->keys[*keyp].type == REPOKEY_TYPE_FIXARRAY ||
1014 di->data->keys[*keyp].type == REPOKEY_TYPE_FLEXARRAY)
1016 if (*keyp && (di->dp = forward_to_key(di->data, *keyp, di->keyp, di->dp)) != 0)
1027 case di_nextrepodata: di_nextrepodata:
1028 if (di->repodataid >= 0 && ++di->repodataid < di->repo->nrepodata)
1029 goto di_enterrepodata;
1032 case di_nextsolvable:
1033 if (!(di->flags & SEARCH_THISENTRY))
1036 di->entry = di->repo->start;
1039 for (; di->entry < di->repo->end; di->entry++)
1041 if (di->pool->solvables[di->entry].repo == di->repo)
1042 goto di_entersolvable;
1048 if (di->repoid >= 0)
1051 if (di->repoid < di->pool->nrepos)
1053 di->repo = di->pool->repos[di->repoid];
1063 case di_enterrepo: di_enterrepo:
1064 if (!(di->flags & SEARCH_THISENTRY))
1065 di->entry = di->repo->start;
1068 case di_entersolvable: di_entersolvable:
1069 if (di->repodataid >= 0)
1072 if (di->entry > 0 && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)))
1074 di->key = solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1076 goto di_entersolvablekey;
1080 case di_enterrepodata: di_enterrepodata:
1081 if (di->repodataid >= 0)
1082 di->data = di->repo->repodata + di->repodataid;
1083 if (!maybe_load_repodata(di->data, di->keyname))
1084 goto di_nextrepodata;
1085 di->dp = entry2data(di->data, di->entry, &schema);
1087 goto di_nextrepodata;
1088 di->keyp = di->data->schemadata + di->data->schemata[schema];
1092 if ((di->flags & SEARCH_SUB) != 0)
1097 for (keyp = di->keyp; *keyp; keyp++)
1098 if (di->data->keys[*keyp].name == di->keyname)
1101 goto di_nextrepodata;
1102 di->dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1105 goto di_nextrepodata;
1108 case di_enterkey: di_enterkey:
1110 di->key = di->data->keys + *di->keyp;
1111 di->ddp = get_data(di->data, di->key, &di->dp);
1114 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1116 di->ddp = data_read_id(di->ddp, &di->kv.num);
1119 goto di_nextarrayelement;
1123 case di_nextarrayelement: di_nextarrayelement:
1126 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1127 if (di->kv.entry == di->kv.num)
1129 if (di->keyname && di->key->name != di->keyname)
1131 di->kv.str = (char *)di->ddp;
1133 di->state = di_nextkey;
1136 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1137 di->ddp = data_read_id(di->ddp, &di->kv.id);
1138 di->kv.str = (char *)di->ddp;
1139 if (di->keyname && di->key->name != di->keyname)
1141 if ((di->flags & SEARCH_SUB) != 0)
1142 di->state = di_entersub;
1144 di->state = di_nextarrayelement;
1147 case di_entersub: di_entersub:
1148 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1149 goto di_nextarrayelement; /* sorry, full */
1150 di->parents[di->nparents].kv = di->kv;
1151 di->parents[di->nparents].dp = di->dp;
1152 di->parents[di->nparents].keyp = di->keyp;
1153 di->dp = (unsigned char *)di->kv.str;
1154 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1155 memset(&di->kv, 0, sizeof(di->kv));
1156 di->kv.parent = &di->parents[di->nparents].kv;
1161 case di_leavesub: di_leavesub:
1163 di->dp = di->parents[di->nparents].dp;
1164 di->kv = di->parents[di->nparents].kv;
1165 di->keyp = di->parents[di->nparents].keyp;
1166 di->key = di->data->keys + *di->keyp;
1167 di->ddp = (unsigned char *)di->kv.str;
1168 goto di_nextarrayelement;
1170 /* special solvable attr handling follows */
1172 case di_nextsolvableattr:
1173 di->kv.id = *di->idp++;
1178 di->state = di_nextsolvablekey;
1182 case di_nextsolvablekey: di_nextsolvablekey:
1183 if (di->keyname || di->key->name == RPM_RPMDBID)
1184 goto di_enterrepodata;
1188 case di_entersolvablekey: di_entersolvablekey:
1189 di->idp = solvabledata_fetch(di->pool->solvables + di->entry, &di->kv, di->key->name);
1190 if (!di->idp || !di->idp[0])
1191 goto di_nextsolvablekey;
1192 di->kv.id = di->idp[0];
1193 di->kv.num = di->idp[0];
1194 if (!di->kv.eof && !di->idp[1])
1198 di->state = di_nextsolvablekey;
1200 di->state = di_nextsolvableattr;
1204 if (di->matcher.match)
1205 if (!datamatcher_match(&di->matcher, di->data, di->key, &di->kv))
1207 /* found something! */
1213 dataiterator_setpos(Dataiterator *di)
1215 di->pool->pos.repo = di->repo;
1216 di->pool->pos.repodataid = di->data - di->repo->repodata;
1217 di->pool->pos.schema = di->kv.id;
1218 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1222 dataiterator_skip_attribute(Dataiterator *di)
1224 if (di->state == di_nextsolvableattr)
1225 di->state = di_nextsolvablekey;
1227 di->state = di_nextkey;
1231 dataiterator_skip_solvable(Dataiterator *di)
1233 di->state = di_nextsolvable;
1237 dataiterator_skip_repo(Dataiterator *di)
1239 di->state = di_nextrepo;
1243 dataiterator_jump_to_solvable(Dataiterator *di, Solvable *s)
1247 di->entry = s - di->pool->solvables;
1248 di->state = di_entersolvable;
1252 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1256 di->state = di_enterrepo;
1261 /************************************************************************
1262 * data search iterator
1266 dataiterator_newdata(Dataiterator *di)
1268 Id keyname = di->keyname;
1269 Repodata *data = di->data;
1272 if (data->state == REPODATA_STUB)
1277 for (j = 1; j < data->nkeys; j++)
1278 if (keyname == data->keys[j].name)
1280 if (j == data->nkeys)
1284 if (data->loadcallback)
1285 data->loadcallback(data);
1287 data->state = REPODATA_ERROR;
1289 if (data->state == REPODATA_ERROR)
1293 unsigned char *dp = data->incoredata;
1296 if (di->solvid >= 0)
1297 dp += data->incoreoffset[di->solvid - data->start];
1298 dp = data_read_id(dp, &schema);
1299 Id *keyp = data->schemadata + data->schemata[schema];
1303 /* search in a specific key */
1304 for (kp = keyp; (k = *kp++) != 0; )
1305 if (data->keys[k].name == keyname)
1309 dp = forward_to_key(data, k, keyp, dp);
1319 di->key = di->data->keys + keyid;
1324 di->dp = get_data(di->data, di->key, &di->nextkeydp);
1329 dataiterator_init(Dataiterator *di, Repo *repo, Id p, Id keyname,
1330 const char *match, int flags)
1336 di->flags |= __SEARCH_ONESOLVABLE;
1337 di->data = repo->repodata - 1;
1338 if (flags & SEARCH_NO_STORAGE_SOLVABLE)
1345 di->solvid = repo->start - 1;
1348 fprintf(stderr, "A repo contains the NULL solvable!\n");
1351 di->data = repo->repodata + repo->nrepodata - 1;
1356 if ((di->flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
1360 /* We feed multiple lines eventually (e.g. authors or descriptions),
1361 so set REG_NEWLINE. */
1363 regcomp(&di->regex, di->match,
1364 REG_EXTENDED | REG_NOSUB | REG_NEWLINE
1365 | ((di->flags & SEARCH_NOCASE) ? REG_ICASE : 0));
1367 if (di->regex_err != 0)
1369 fprintf(stderr, "Given regex failed to compile: %s\n", di->match);
1370 fprintf(stderr, "regcomp error code: %d\n", di->regex_err);
1377 di->flags |= (di->flags & SEARCH_STRINGMASK) | SEARCH_STRING;
1383 di->keyname = keyname;
1384 static Id zeroid = 0;
1392 /* FIXME factor and merge with repo_matchvalue */
1394 dataiterator_match_int_real(Dataiterator *di, int flags, const void *vmatch)
1396 KeyValue *kv = &di->kv;
1397 const char *match = vmatch;
1398 if ((flags & SEARCH_STRINGMASK) != 0)
1400 switch (di->key->type)
1402 case REPOKEY_TYPE_ID:
1403 case REPOKEY_TYPE_IDARRAY:
1404 if (di->data && di->data->localpool)
1405 kv->str = stringpool_id2str(&di->data->spool, kv->id);
1407 kv->str = id2str(di->repo->pool, kv->id);
1409 case REPOKEY_TYPE_STR:
1411 case REPOKEY_TYPE_DIRSTRARRAY:
1412 if (!(flags & SEARCH_FILES))
1414 /* Put the full filename into kv->str. */
1415 kv->str = repodata_dir2str(di->data, kv->id, kv->str);
1416 /* And to compensate for that put the "empty" directory into
1417 kv->id, so that later calls to repodata_dir2str on this data
1418 come up with the same filename again. */
1424 /* Maybe skip the kind specifier. Do this only for SOLVABLE attributes,
1425 for the others we can't know if a colon separates a kind or not. */
1426 if ((flags & SEARCH_SKIP_KIND)
1427 && di->key->storage == KEY_STORAGE_SOLVABLE)
1429 const char *s = strchr(kv->str, ':');
1433 switch ((flags & SEARCH_STRINGMASK))
1435 case SEARCH_SUBSTRING:
1436 if (flags & SEARCH_NOCASE)
1438 if (!strcasestr(kv->str, match))
1443 if (!strstr(kv->str, match))
1448 if (flags & SEARCH_NOCASE)
1450 if (strcasecmp(match, kv->str))
1455 if (strcmp(match, kv->str))
1460 if (fnmatch(match, kv->str, (flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
1464 if (regexec((const regex_t *)vmatch, kv->str, 0, NULL, 0))
1475 dataiterator_match_int(Dataiterator *di)
1477 if ((di->flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
1478 return dataiterator_match_int_real(di, di->flags, &di->regex);
1480 return dataiterator_match_int_real(di, di->flags, di->match);
1484 dataiterator_match(Dataiterator *di, int flags, const void *vmatch)
1486 return dataiterator_match_int_real(di, flags, vmatch);
1490 dataiterator_step(Dataiterator *di)
1497 /* we're stepping through solvable data, 1 -> SOLVABLE_NAME... */
1500 /* we're stepping through an id array */
1506 di->kv.eof = idp[1] ? 0 : 1;
1512 Solvable *s = di->repo->pool->solvables + di->solvid;
1513 int state = di->state;
1514 di->key = solvablekeys + state - 1;
1516 di->state = RPM_RPMDBID;
1523 state = di->keyname - 1;
1530 di->kv.id = s->name;
1536 di->kv.id = s->arch;
1545 case SOLVABLE_VENDOR:
1548 di->kv.id = s->vendor;
1551 case SOLVABLE_PROVIDES:
1552 di->idp = s->provides
1553 ? di->repo->idarraydata + s->provides : 0;
1555 case SOLVABLE_OBSOLETES:
1556 di->idp = s->obsoletes
1557 ? di->repo->idarraydata + s->obsoletes : 0;
1559 case SOLVABLE_CONFLICTS:
1560 di->idp = s->conflicts
1561 ? di->repo->idarraydata + s->conflicts : 0;
1563 case SOLVABLE_REQUIRES:
1564 di->idp = s->requires
1565 ? di->repo->idarraydata + s->requires : 0;
1567 case SOLVABLE_RECOMMENDS:
1568 di->idp = s->recommends
1569 ? di->repo->idarraydata + s->recommends : 0;
1571 case SOLVABLE_SUPPLEMENTS:
1572 di->idp = s->supplements
1573 ? di->repo->idarraydata + s->supplements : 0;
1575 case SOLVABLE_SUGGESTS:
1576 di->idp = s->suggests
1577 ? di->repo->idarraydata + s->suggests : 0;
1579 case SOLVABLE_ENHANCES:
1580 di->idp = s->enhances
1581 ? di->repo->idarraydata + s->enhances : 0;
1584 if (!di->repo->rpmdbid)
1586 di->kv.num = di->repo->rpmdbid[di->solvid - di->repo->start];
1590 di->data = di->repo->repodata - 1;
1596 else if (di->subkeyp)
1601 /* Send end-of-substruct. We are here only when we saw a
1602 _COUNTED key one level up. Since then we didn't increment
1603 ->keyp, so it still can be found at keyp[-1]. */
1605 di->key = di->data->keys + di->keyp[-1];
1608 else if (!(keyid = *di->subkeyp++))
1610 /* Send end-of-element. See above for keyp[-1]. */
1612 di->key = di->data->keys + di->keyp[-1];
1614 di->subkeyp = di->data->schemadata + di->data->schemata[di->subschema];
1617 di->dp = data_read_id(di->dp, &di->subschema);
1618 di->subkeyp = di->data->schemadata + di->data->schemata[di->subschema];
1625 di->key = di->data->keys + keyid;
1626 di->dp = data_fetch(di->dp, &di->kv, di->key);
1636 di->dp = data_fetch(di->dp, &di->kv, di->key);
1641 if (di->keyname || !(keyid = *di->keyp++))
1645 Repo *repo = di->repo;
1646 Repodata *data = ++di->data;
1647 if (data >= repo->repodata + repo->nrepodata)
1649 if (di->flags & __SEARCH_ONESOLVABLE)
1651 if (di->solvid >= 0)
1653 while (++di->solvid < repo->end)
1654 if (repo->pool->solvables[di->solvid].repo == repo)
1656 if (di->solvid >= repo->end)
1658 if (!(di->flags & SEARCH_EXTRA))
1667 Pool *pool = di->repo->pool;
1668 if (!(di->flags & SEARCH_ALL_REPOS)
1669 || di->repo == pool->repos[pool->nrepos - 1])
1672 for (i = 0; i < pool->nrepos; i++)
1673 if (di->repo == pool->repos[i])
1675 di->repo = pool->repos[i + 1];
1676 dataiterator_init(di, di->repo, 0, di->keyname, di->match, di->flags);
1680 di->data = repo->repodata - 1;
1681 if ((di->flags & SEARCH_NO_STORAGE_SOLVABLE))
1683 static Id zeroid = 0;
1688 if ((di->solvid >= 0 && di->solvid >= data->start && di->solvid < data->end))
1690 dataiterator_newdata(di);
1698 di->key = di->data->keys + keyid;
1699 di->dp = get_data(di->data, di->key, &di->nextkeydp);
1701 di->dp = data_fetch(di->dp, &di->kv, di->key);
1703 if (di->key->type == REPOKEY_TYPE_FIXARRAY)
1705 di->subnum = di->kv.num;
1706 di->subschema = di->kv.id;
1708 di->subkeyp = di->data->schemadata + di->data->schemata[di->subschema];
1710 if (di->key->type == REPOKEY_TYPE_FLEXARRAY)
1712 di->subnum = di->kv.num;
1714 di->dp = data_read_id(di->dp, &di->subschema);
1715 di->subkeyp = di->data->schemadata + di->data->schemata[di->subschema];
1721 || dataiterator_match_int(di))
1728 dataiterator_skip_attribute(Dataiterator *di)
1732 /* This will make the next _step call to retrieve the next field. */
1737 dataiterator_skip_solvable(Dataiterator *di)
1739 /* We're done with this field. */
1741 /* And with solvable data. */
1743 /* And with all keys for this repodata and thing. */
1744 static Id zeroid = 0;
1746 /* And with all repodatas for this thing. */
1747 di->data = di->repo->repodata + di->repo->nrepodata - 1;
1748 /* Hence the next call to _step will retrieve the next thing. */
1752 dataiterator_skip_repo(Dataiterator *di)
1754 dataiterator_skip_solvable(di);
1755 /* We're done with all solvables and all extra things for this repo. */
1760 dataiterator_jump_to_solvable(Dataiterator *di, Solvable *s)
1763 /* Simulate us being done with the solvable before the requested one. */
1764 dataiterator_skip_solvable(di);
1765 di->solvid = s - s->repo->pool->solvables;
1770 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1773 dataiterator_skip_solvable(di);
1774 di->solvid = repo->start - 1;
1779 /************************************************************************
1780 * data modify functions
1783 /* extend repodata so that it includes solvables p */
1785 repodata_extend(Repodata *data, Id p)
1787 if (data->start == data->end)
1788 data->start = data->end = p;
1791 int old = data->end - data->start;
1792 int new = p - data->end + 1;
1795 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id), REPODATA_BLOCK);
1796 memset(data->attrs + old, 0, new * sizeof(Id));
1798 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
1799 memset(data->incoreoffset + old, 0, new * sizeof(Id));
1802 if (p < data->start)
1804 int old = data->end - data->start;
1805 int new = data->start - p;
1808 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id), REPODATA_BLOCK);
1809 memmove(data->attrs + new, data->attrs, old * sizeof(Id));
1810 memset(data->attrs, 0, new * sizeof(Id));
1812 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
1813 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
1814 memset(data->incoreoffset, 0, new * sizeof(Id));
1820 repodata_extend_block(Repodata *data, Id start, Id num)
1824 if (!data->incoreoffset)
1826 data->incoreoffset = sat_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
1827 data->start = start;
1828 data->end = start + num;
1831 repodata_extend(data, start);
1833 repodata_extend(data, start + num - 1);
1836 /**********************************************************************/
1838 #define REPODATA_ATTRS_BLOCK 63
1839 #define REPODATA_ATTRDATA_BLOCK 1023
1840 #define REPODATA_ATTRIDDATA_BLOCK 63
1844 repodata_new_handle(Repodata *data)
1848 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1851 data->xattrs = sat_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
1852 data->xattrs[data->nxattrs] = 0;
1853 return -(data->nxattrs++);
1857 repodata_get_attrp(Repodata *data, Id handle)
1859 if (handle == REPOENTRY_META)
1863 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1868 return data->xattrs - handle;
1869 if (handle < data->start || handle >= data->end)
1870 repodata_extend(data, handle);
1872 data->attrs = sat_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
1873 return data->attrs + (handle - data->start);
1877 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
1883 app = repodata_get_attrp(data, handle);
1888 for (pp = ap; *pp; pp += 2)
1889 /* Determine equality based on the name only, allows us to change
1890 type (when overwrite is set), and makes TYPE_CONSTANT work. */
1891 if (data->keys[*pp].name == data->keys[keyid].name)
1904 ap = sat_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
1914 repodata_set(Repodata *data, Id handle, Repokey *key, Id val)
1918 keyid = repodata_key2id(data, key, 1);
1919 repodata_insert_keyid(data, handle, keyid, val, 1);
1923 repodata_set_id(Repodata *data, Id handle, Id keyname, Id id)
1927 key.type = REPOKEY_TYPE_ID;
1929 key.storage = KEY_STORAGE_INCORE;
1930 repodata_set(data, handle, &key, id);
1934 repodata_set_num(Repodata *data, Id handle, Id keyname, unsigned int num)
1938 key.type = REPOKEY_TYPE_NUM;
1940 key.storage = KEY_STORAGE_INCORE;
1941 repodata_set(data, handle, &key, (Id)num);
1945 repodata_set_poolstr(Repodata *data, Id handle, Id keyname, const char *str)
1949 if (data->localpool)
1950 id = stringpool_str2id(&data->spool, str, 1);
1952 id = str2id(data->repo->pool, str, 1);
1954 key.type = REPOKEY_TYPE_ID;
1956 key.storage = KEY_STORAGE_INCORE;
1957 repodata_set(data, handle, &key, id);
1961 repodata_set_constant(Repodata *data, Id handle, Id keyname, unsigned int constant)
1965 key.type = REPOKEY_TYPE_CONSTANT;
1966 key.size = constant;
1967 key.storage = KEY_STORAGE_INCORE;
1968 repodata_set(data, handle, &key, 0);
1972 repodata_set_constantid(Repodata *data, Id handle, Id keyname, Id id)
1976 key.type = REPOKEY_TYPE_CONSTANTID;
1978 key.storage = KEY_STORAGE_INCORE;
1979 repodata_set(data, handle, &key, 0);
1983 repodata_set_void(Repodata *data, Id handle, Id keyname)
1987 key.type = REPOKEY_TYPE_VOID;
1989 key.storage = KEY_STORAGE_INCORE;
1990 repodata_set(data, handle, &key, 0);
1994 repodata_set_str(Repodata *data, Id handle, Id keyname, const char *str)
1999 l = strlen(str) + 1;
2001 key.type = REPOKEY_TYPE_STR;
2003 key.storage = KEY_STORAGE_INCORE;
2004 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2005 memcpy(data->attrdata + data->attrdatalen, str, l);
2006 repodata_set(data, handle, &key, data->attrdatalen);
2007 data->attrdatalen += l;
2011 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
2014 Id *ida, *pp, **ppp;
2016 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
2018 /* great! just append the new data */
2019 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2020 data->attriddatalen--; /* overwrite terminating 0 */
2021 data->lastdatalen += entrysize;
2024 ppp = repodata_get_attrp(data, handle);
2027 for (; *pp; pp += 2)
2028 if (data->keys[*pp].name == keyname && data->keys[*pp].type == keytype)
2032 /* not found. allocate new key */
2037 key.storage = KEY_STORAGE_INCORE;
2038 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2039 repodata_set(data, handle, &key, data->attriddatalen);
2040 data->lasthandle = 0; /* next time... */
2044 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2045 oldsize += entrysize;
2046 if (ida + 1 == data->attriddata + data->attriddatalen)
2048 /* this was the last entry, just append it */
2049 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2050 data->attriddatalen--; /* overwrite terminating 0 */
2054 /* too bad. move to back. */
2055 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2056 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2057 pp[1] = data->attriddatalen;
2058 data->attriddatalen += oldsize;
2060 data->lasthandle = handle;
2061 data->lastkey = *pp;
2062 data->lastdatalen = data->attriddatalen + entrysize + 1;
2066 checksumtype2len(Id type)
2070 case REPOKEY_TYPE_MD5:
2072 case REPOKEY_TYPE_SHA1:
2074 case REPOKEY_TYPE_SHA256:
2075 return SIZEOF_SHA256;
2082 repodata_set_bin_checksum(Repodata *data, Id handle, Id keyname, Id type,
2083 const unsigned char *str)
2086 int l = checksumtype2len(type);
2093 key.storage = KEY_STORAGE_INCORE;
2094 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2095 memcpy(data->attrdata + data->attrdatalen, str, l);
2096 repodata_set(data, handle, &key, data->attrdatalen);
2097 data->attrdatalen += l;
2101 hexstr2bytes(unsigned char *buf, const char *str, int buflen)
2104 for (i = 0; i < buflen; i++)
2106 #define c2h(c) (((c)>='0' && (c)<='9') ? ((c)-'0') \
2107 : ((c)>='a' && (c)<='f') ? ((c)-'a'+10) \
2108 : ((c)>='A' && (c)<='F') ? ((c)-'A'+10) \
2119 buf[i] = (buf[i] << 4) | v;
2126 repodata_set_checksum(Repodata *data, Id handle, Id keyname, Id type,
2129 unsigned char buf[64];
2130 int l = checksumtype2len(type);
2134 if (hexstr2bytes(buf, str, l) != l)
2136 fprintf(stderr, "Invalid hex character in '%s'\n", str);
2139 repodata_set_bin_checksum(data, handle, keyname, type, buf);
2143 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2148 l = checksumtype2len(type);
2151 s = str = pool_alloctmpspace(data->repo->pool, 2 * l + 1);
2152 for (i = 0; i < l; i++)
2154 unsigned char v = buf[i];
2155 unsigned char w = v >> 4;
2156 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2158 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2165 repodata_globalize_id(Repodata *data, Id id)
2167 if (!data || !data->localpool)
2169 return str2id(data->repo->pool, stringpool_id2str(&data->spool, id), 1);
2173 repodata_add_dirnumnum(Repodata *data, Id handle, Id keyname, Id dir, Id num, Id num2)
2177 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", handle, dir, num, num2, data->attriddatalen);
2179 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2180 data->attriddata[data->attriddatalen++] = dir;
2181 data->attriddata[data->attriddatalen++] = num;
2182 data->attriddata[data->attriddatalen++] = num2;
2183 data->attriddata[data->attriddatalen++] = 0;
2187 repodata_add_dirstr(Repodata *data, Id handle, Id keyname, Id dir, const char *str)
2193 l = strlen(str) + 1;
2194 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2195 memcpy(data->attrdata + data->attrdatalen, str, l);
2196 stroff = data->attrdatalen;
2197 data->attrdatalen += l;
2200 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", handle, dir, str, data->attriddatalen);
2202 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2203 data->attriddata[data->attriddatalen++] = dir;
2204 data->attriddata[data->attriddatalen++] = stroff;
2205 data->attriddata[data->attriddatalen++] = 0;
2209 repodata_add_idarray(Repodata *data, Id handle, Id keyname, Id id)
2212 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", handle, id, data->attriddatalen);
2214 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_IDARRAY, 1);
2215 data->attriddata[data->attriddatalen++] = id;
2216 data->attriddata[data->attriddatalen++] = 0;
2220 repodata_add_poolstr_array(Repodata *data, Id handle, Id keyname,
2224 if (data->localpool)
2225 id = stringpool_str2id(&data->spool, str, 1);
2227 id = str2id(data->repo->pool, str, 1);
2228 repodata_add_idarray(data, handle, keyname, id);
2232 repodata_add_fixarray(Repodata *data, Id handle, Id keyname, Id ghandle)
2234 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2235 data->attriddata[data->attriddatalen++] = ghandle;
2236 data->attriddata[data->attriddatalen++] = 0;
2240 repodata_add_flexarray(Repodata *data, Id handle, Id keyname, Id ghandle)
2242 repodata_add_array(data, handle, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2243 data->attriddata[data->attriddatalen++] = ghandle;
2244 data->attriddata[data->attriddatalen++] = 0;
2248 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2251 if (dest == src || !(keyp = data->attrs[src]))
2253 for (; *keyp; keyp += 2)
2254 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2260 /**********************************************************************/
2262 /* unify with repo_write! */
2264 #define EXTDATA_BLOCK 1023
2272 data_addid(struct extdata *xd, Id x)
2275 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2276 dp = xd->buf + xd->len;
2281 *dp++ = (x >> 28) | 128;
2283 *dp++ = (x >> 21) | 128;
2284 *dp++ = (x >> 14) | 128;
2287 *dp++ = (x >> 7) | 128;
2289 xd->len = dp - xd->buf;
2293 data_addideof(struct extdata *xd, Id x, int eof)
2296 x = (x & 63) | ((x & ~63) << 1);
2297 data_addid(xd, (eof ? x: x | 64));
2301 data_addblob(struct extdata *xd, unsigned char *blob, int len)
2303 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
2304 memcpy(xd->buf + xd->len, blob, len);
2308 /*********************************/
2311 repodata_serialize_key(Repodata *data, struct extdata *newincore,
2312 struct extdata *newvincore,
2314 Repokey *key, Id val)
2316 /* Otherwise we have a new value. Parse it into the internal
2320 unsigned int oldvincorelen = 0;
2324 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2327 oldvincorelen = xd->len;
2331 case REPOKEY_TYPE_VOID:
2332 case REPOKEY_TYPE_CONSTANT:
2333 case REPOKEY_TYPE_CONSTANTID:
2335 case REPOKEY_TYPE_STR:
2336 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
2338 case REPOKEY_TYPE_MD5:
2339 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
2341 case REPOKEY_TYPE_SHA1:
2342 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
2344 case REPOKEY_TYPE_SHA256:
2345 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
2347 case REPOKEY_TYPE_ID:
2348 case REPOKEY_TYPE_NUM:
2349 case REPOKEY_TYPE_DIR:
2350 data_addid(xd, val);
2352 case REPOKEY_TYPE_IDARRAY:
2353 for (ida = data->attriddata + val; *ida; ida++)
2354 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
2356 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2357 for (ida = data->attriddata + val; *ida; ida += 3)
2359 data_addid(xd, ida[0]);
2360 data_addid(xd, ida[1]);
2361 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
2364 case REPOKEY_TYPE_DIRSTRARRAY:
2365 for (ida = data->attriddata + val; *ida; ida += 2)
2367 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
2368 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
2371 case REPOKEY_TYPE_FIXARRAY:
2375 for (ida = data->attriddata + val; *ida; ida++)
2378 fprintf(stderr, "serialize struct %d\n", *ida);
2381 Id *kp = data->xattrs[-*ida];
2388 fprintf(stderr, " %s:%d\n", id2str(data->repo->pool, data->keys[*kp].name), kp[1]);
2394 schemaid = repodata_schema2id(data, schema, 1);
2395 else if (schemaid != repodata_schema2id(data, schema, 0))
2397 fprintf(stderr, " not yet implemented: substructs with different schemas\n");
2401 fprintf(stderr, " schema %d\n", schemaid);
2406 data_addid(xd, num);
2407 data_addid(xd, schemaid);
2408 for (ida = data->attriddata + val; *ida; ida++)
2410 Id *kp = data->xattrs[-*ida];
2415 repodata_serialize_key(data, newincore, newvincore,
2416 schema, data->keys + *kp, kp[1]);
2421 case REPOKEY_TYPE_FLEXARRAY:
2424 for (ida = data->attriddata + val; *ida; ida++)
2426 data_addid(xd, num);
2427 for (ida = data->attriddata + val; *ida; ida++)
2429 Id *kp = data->xattrs[-*ida];
2432 data_addid(xd, 0); /* XXX */
2439 schemaid = repodata_schema2id(data, schema, 1);
2440 data_addid(xd, schemaid);
2441 kp = data->xattrs[-*ida];
2444 repodata_serialize_key(data, newincore, newvincore,
2445 schema, data->keys + *kp, kp[1]);
2451 fprintf(stderr, "don't know how to handle type %d\n", key->type);
2454 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2456 /* put offset/len in incore */
2457 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
2458 oldvincorelen = xd->len - oldvincorelen;
2459 data_addid(newincore, oldvincorelen);
2464 repodata_internalize(Repodata *data)
2466 Repokey *key, solvkey;
2468 Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen;
2469 unsigned char *dp, *ndp;
2470 int newschema, oldcount;
2471 struct extdata newincore;
2472 struct extdata newvincore;
2475 if (!data->attrs && !data->xattrs)
2478 newvincore.buf = data->vincore;
2479 newvincore.len = data->vincorelen;
2481 /* find the solvables key, create if needed */
2482 memset(&solvkey, 0, sizeof(solvkey));
2483 solvkey.name = REPOSITORY_SOLVABLES;
2484 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
2486 solvkey.storage = KEY_STORAGE_INCORE;
2487 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
2489 schema = sat_malloc2(data->nkeys, sizeof(Id));
2490 seen = sat_malloc2(data->nkeys, sizeof(Id));
2492 /* Merge the data already existing (in data->schemata, ->incoredata and
2493 friends) with the new attributes in data->attrs[]. */
2494 nentry = data->end - data->start;
2495 memset(&newincore, 0, sizeof(newincore));
2496 data_addid(&newincore, 0); /* start data at offset 1 */
2498 data->mainschema = 0;
2499 data->mainschemaoffsets = sat_free(data->mainschemaoffsets);
2501 /* join entry data */
2502 /* we start with the meta data, entry -1 */
2503 for (entry = -1; entry < nentry; entry++)
2505 memset(seen, 0, data->nkeys * sizeof(Id));
2507 dp = data->incoredata;
2510 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
2511 dp = data_read_id(dp, &oldschema);
2514 fprintf(stderr, "oldschema %d\n", oldschema);
2515 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
2516 fprintf(stderr, "schemadata %p\n", data->schemadata);
2518 /* seen: -1: old data 0: skipped >0: id + 1 */
2522 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
2526 fprintf(stderr, "Inconsistent old data (key occured twice).\n");
2534 keyp = data->attrs ? data->attrs[entry] : 0;
2537 /* strip solvables key */
2539 for (sp = keyp = schema; *sp; sp++)
2540 if (*sp != solvkeyid)
2545 seen[solvkeyid] = 0;
2546 keyp = data->xattrs ? data->xattrs[1] : 0;
2549 for (; *keyp; keyp += 2)
2556 seen[*keyp] = keyp[1] + 1;
2558 if (entry < 0 && data->end != data->start)
2565 /* Ideally we'd like to sort the new schema here, to ensure
2566 schema equality independend of the ordering. We can't do that
2567 yet. For once see below (old ids need to come before new ids).
2568 An additional difficulty is that we also need to move
2569 the values with the keys. */
2570 schemaid = repodata_schema2id(data, schema, 1);
2572 schemaid = oldschema;
2575 /* Now create data blob. We walk through the (possibly new) schema
2576 and either copy over old data, or insert the new. */
2577 /* XXX Here we rely on the fact that the (new) schema has the form
2578 o1 o2 o3 o4 ... | n1 n2 n3 ...
2579 (oX being the old keyids (possibly overwritten), and nX being
2580 the new keyids). This rules out sorting the keyids in order
2581 to ensure a small schema count. */
2583 data->incoreoffset[entry] = newincore.len;
2584 data_addid(&newincore, schemaid);
2587 data->mainschema = schemaid;
2588 data->mainschemaoffsets = sat_calloc(sp - schema, sizeof(Id));
2590 keypstart = data->schemadata + data->schemata[schemaid];
2591 for (keyp = keypstart; *keyp; keyp++)
2594 data->mainschemaoffsets[keyp - keypstart] = newincore.len;
2595 if (*keyp == solvkeyid)
2597 /* add flexarray entry count */
2598 data_addid(&newincore, data->end - data->start);
2601 key = data->keys + *keyp;
2603 fprintf(stderr, "internalize %d:%s:%s\n", entry, id2str(data->repo->pool, key->name), id2str(data->repo->pool, key->type));
2608 /* Skip the data associated with this old key. */
2609 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2611 ndp = data_skip(dp, REPOKEY_TYPE_ID);
2612 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
2614 else if (key->storage == KEY_STORAGE_INCORE)
2615 ndp = data_skip_key(data, dp, key);
2618 if (seen[*keyp] == -1)
2620 /* If this key was an old one _and_ was not overwritten with
2621 a different value copy over the old value (we skipped it
2624 data_addblob(&newincore, dp, ndp - dp);
2627 else if (seen[*keyp])
2629 /* Otherwise we have a new value. Parse it into the internal
2631 repodata_serialize_key(data, &newincore, &newvincore,
2632 schema, key, seen[*keyp] - 1);
2636 if (entry >= 0 && data->attrs && data->attrs[entry])
2637 data->attrs[entry] = sat_free(data->attrs[entry]);
2639 /* free all xattrs */
2640 for (entry = 0; entry < data->nxattrs; entry++)
2641 if (data->xattrs[entry])
2642 sat_free(data->xattrs[entry]);
2643 data->xattrs = sat_free(data->xattrs);
2646 data->lasthandle = 0;
2648 data->lastdatalen = 0;
2651 repodata_free_schemahash(data);
2653 sat_free(data->incoredata);
2654 data->incoredata = newincore.buf;
2655 data->incoredatalen = newincore.len;
2656 data->incoredatafree = 0;
2658 sat_free(data->vincore);
2659 data->vincore = newvincore.buf;
2660 data->vincorelen = newvincore.len;
2662 data->attrs = sat_free(data->attrs);
2663 data->attrdata = sat_free(data->attrdata);
2664 data->attriddata = sat_free(data->attriddata);
2665 data->attrdatalen = 0;
2666 data->attriddatalen = 0;
2670 repodata_disable_paging(Repodata *data)
2672 if (maybe_load_repodata(data, 0)
2674 repodata_load_page_range(data, 0, data->num_pages - 1);
2678 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: