2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
13 * a repository can contain multiple repodata entries, consisting of
14 * different sets of keys and different sets of solvables
29 #include "poolid_private.h"
37 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
38 unsigned char *out, unsigned int out_len);
39 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
42 unsigned int out_len);
44 #define REPODATA_BLOCK 255
48 repodata_initdata(Repodata *data, Repo *repo, int localpool)
50 memset(data, 0, sizeof (*data));
52 data->localpool = localpool;
54 stringpool_init_empty(&data->spool);
55 data->keys = sat_calloc(1, sizeof(Repokey));
57 data->schemata = sat_calloc(1, sizeof(Id));
58 data->schemadata = sat_calloc(1, sizeof(Id));
60 data->schemadatalen = 1;
61 repopagestore_init(&data->store);
65 repodata_freedata(Repodata *data)
71 sat_free(data->schemata);
72 sat_free(data->schemadata);
73 sat_free(data->schematahash);
75 stringpool_free(&data->spool);
76 dirpool_free(&data->dirpool);
78 sat_free(data->mainschemaoffsets);
79 sat_free(data->incoredata);
80 sat_free(data->incoreoffset);
81 sat_free(data->verticaloffset);
83 repopagestore_free(&data->store);
85 sat_free(data->vincore);
88 for (i = 0; i < data->end - data->start; i++)
89 sat_free(data->attrs[i]);
90 sat_free(data->attrs);
92 for (i = 0; i < data->nxattrs; i++)
93 sat_free(data->xattrs[i]);
94 sat_free(data->xattrs);
96 sat_free(data->attrdata);
97 sat_free(data->attriddata);
101 repodata_create(Repo *repo, int localpool)
106 repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata, sizeof(*data));
107 data = repo->repodata + repo->nrepodata - 1;
108 repodata_initdata(data, repo, localpool);
113 repodata_free(Repodata *data)
115 Repo *repo = data->repo;
116 int i = data - repo->repodata;
117 repodata_freedata(data);
118 if (i < repo->nrepodata - 1)
119 memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
124 /***************************************************************
125 * key pool management
128 /* this is not so time critical that we need a hash, so we do a simple
131 repodata_key2id(Repodata *data, Repokey *key, int create)
135 for (keyid = 1; keyid < data->nkeys; keyid++)
136 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
138 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
142 if (keyid == data->nkeys)
146 /* allocate new key */
147 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
148 data->keys[data->nkeys++] = *key;
149 if (data->verticaloffset)
151 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
152 data->verticaloffset[data->nkeys - 1] = 0;
154 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
160 /***************************************************************
161 * schema pool management
164 #define SCHEMATA_BLOCK 31
165 #define SCHEMATADATA_BLOCK 255
168 repodata_schema2id(Repodata *data, Id *schema, int create)
174 if ((schematahash = data->schematahash) == 0)
176 data->schematahash = schematahash = sat_calloc(256, sizeof(Id));
177 for (i = 0; i < data->nschemata; i++)
179 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
182 schematahash[h] = i + 1;
184 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
185 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
188 for (sp = schema, len = 0, h = 0; *sp; len++)
193 cid = schematahash[h];
197 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
200 for (cid = 0; cid < data->nschemata; cid++)
201 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
207 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
208 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
210 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
211 data->schemata[data->nschemata] = data->schemadatalen;
212 data->schemadatalen += len;
213 schematahash[h] = data->nschemata + 1;
215 fprintf(stderr, "schema2id: new schema\n");
217 return data->nschemata++;
221 repodata_free_schemahash(Repodata *data)
223 data->schematahash = sat_free(data->schematahash);
225 data->schemata = sat_realloc2(data->schemata, data->nschemata, sizeof(Id));
226 data->schemadata = sat_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
230 /***************************************************************
231 * dir pool management
235 repodata_str2dir(Repodata *data, const char *dir, int create)
241 while (*dir == '/' && dir[1] == '/')
243 if (*dir == '/' && !dir[1])
245 if (data->dirpool.ndirs)
247 return dirpool_add_dir(&data->dirpool, 0, 1, create);
251 dire = strchrnul(dir, '/');
253 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
255 id = strn2id(data->repo->pool, dir, dire - dir, create);
258 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
271 repodata_dir2str(Repodata *data, Id did, const char *suf)
273 Pool *pool = data->repo->pool;
280 return suf ? suf : "";
284 comp = dirpool_compid(&data->dirpool, parent);
285 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
287 parent = dirpool_parent(&data->dirpool, parent);
292 l += strlen(suf) + 1;
293 p = pool_alloctmpspace(pool, l + 1) + l;
304 comp = dirpool_compid(&data->dirpool, parent);
305 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
308 strncpy(p, comps, l);
309 parent = dirpool_parent(&data->dirpool, parent);
317 /***************************************************************
321 static inline unsigned char *
322 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
324 Id *keyp = data->schemadata + data->schemata[schema];
325 for (; *keyp; keyp++)
326 dp = data_skip_key(data, dp, data->keys + *keyp);
331 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
333 int nentries, schema;
336 case REPOKEY_TYPE_FIXARRAY:
337 dp = data_read_id(dp, &nentries);
340 dp = data_read_id(dp, &schema);
342 dp = data_skip_schema(data, dp, schema);
344 case REPOKEY_TYPE_FLEXARRAY:
345 dp = data_read_id(dp, &nentries);
348 dp = data_read_id(dp, &schema);
349 dp = data_skip_schema(data, dp, schema);
353 if (key->storage == KEY_STORAGE_INCORE)
354 dp = data_skip(dp, key->type);
355 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
357 dp = data_skip(dp, REPOKEY_TYPE_ID);
358 dp = data_skip(dp, REPOKEY_TYPE_ID);
364 static unsigned char *
365 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
371 if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
374 for (i = 0; (k = *keyp++) != 0; i++)
376 return data->incoredata + data->mainschemaoffsets[i];
379 while ((k = *keyp++) != 0)
383 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
385 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
386 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
389 if (data->keys[k].storage != KEY_STORAGE_INCORE)
391 dp = data_skip_key(data, dp, data->keys + k);
396 static unsigned char *
397 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
402 if (off >= data->lastverticaloffset)
404 off -= data->lastverticaloffset;
405 if (off + len > data->vincorelen)
407 return data->vincore + off;
409 if (off + len > key->size)
411 /* we now have the offset, go into vertical */
412 off += data->verticaloffset[key - data->keys];
413 /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */
414 dp = repopagestore_load_page_range(&data->store, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
416 dp += off % BLOB_PAGESIZE;
420 static inline unsigned char *
421 get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
423 unsigned char *dp = *dpp;
427 if (key->storage == KEY_STORAGE_INCORE)
430 *dpp = data_skip_key(data, dp, key);
433 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
436 dp = data_read_id(dp, &off);
437 dp = data_read_id(dp, &len);
440 return get_vertical_data(data, key, off, len);
446 load_repodata(Repodata *data)
448 if (data->loadcallback)
450 data->loadcallback(data);
451 if (data->state == REPODATA_AVAILABLE)
454 data->state = REPODATA_ERROR;
459 maybe_load_repodata(Repodata *data, Id keyname)
461 if (keyname && !repodata_precheck_keyname(data, keyname))
462 return 0; /* do not bother... */
469 for (i = 0; i < data->nkeys; i++)
470 if (keyname == data->keys[i].name)
472 if (i == data->nkeys)
475 return load_repodata(data);
478 case REPODATA_AVAILABLE:
479 case REPODATA_LOADING:
482 data->state = REPODATA_ERROR;
487 static inline unsigned char *
488 solvid2data(Repodata *data, Id solvid, Id *schemap)
490 unsigned char *dp = data->incoredata;
493 if (solvid == SOLVID_META) /* META */
495 else if (solvid == SOLVID_POS) /* META */
497 Pool *pool = data->repo->pool;
498 if (data->repo != pool->pos.repo)
500 if (data != data->repo->repodata + pool->pos.repodataid)
502 *schemap = pool->pos.schema;
503 return data->incoredata + pool->pos.dp;
507 if (solvid < data->start || solvid >= data->end)
509 dp += data->incoreoffset[solvid - data->start];
511 return data_read_id(dp, schemap);
514 /************************************************************************
518 static inline unsigned char *
519 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
522 Id schema, *keyp, *kp;
525 if (!maybe_load_repodata(data, keyname))
527 dp = solvid2data(data, solvid, &schema);
530 keyp = data->schemadata + data->schemata[schema];
531 for (kp = keyp; *kp; kp++)
532 if (data->keys[*kp].name == keyname)
536 *keypp = key = data->keys + *kp;
537 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
538 return dp; /* no need to forward... */
539 dp = forward_to_key(data, *kp, keyp, dp);
542 return get_data(data, key, &dp, 0);
547 repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
553 dp = find_key_data(data, solvid, keyname, &key);
556 if (key->type == REPOKEY_TYPE_CONSTANTID)
558 if (key->type != REPOKEY_TYPE_ID)
560 dp = data_read_id(dp, &id);
565 repodata_globalize_id(Repodata *data, Id id, int create)
567 if (!id || !data || !data->localpool)
569 return str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
573 repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
579 dp = find_key_data(data, solvid, keyname, &key);
582 if (key->type == REPOKEY_TYPE_STR)
583 return (const char *)dp;
584 if (key->type == REPOKEY_TYPE_CONSTANTID)
585 return id2str(data->repo->pool, key->size);
586 if (key->type == REPOKEY_TYPE_ID)
587 dp = data_read_id(dp, &id);
591 return data->spool.stringspace + data->spool.strings[id];
592 return id2str(data->repo->pool, id);
596 repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned int *value)
603 dp = find_key_data(data, solvid, keyname, &key);
606 if (key->type == REPOKEY_TYPE_NUM
607 || key->type == REPOKEY_TYPE_U32
608 || key->type == REPOKEY_TYPE_CONSTANT)
610 dp = data_fetch(dp, &kv, key);
618 repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
624 if (!maybe_load_repodata(data, keyname))
626 dp = solvid2data(data, solvid, &schema);
629 /* can't use find_key_data as we need to test the type */
630 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
631 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
636 const unsigned char *
637 repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
642 dp = find_key_data(data, solvid, keyname, &key);
650 repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
658 dp = find_key_data(data, solvid, keyname, &key);
663 dp = data_read_ideof(dp, &id, &eof);
671 /************************************************************************
677 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
681 case REPOKEY_TYPE_ID:
682 case REPOKEY_TYPE_CONSTANTID:
683 case REPOKEY_TYPE_IDARRAY:
684 if (data && data->localpool)
685 kv->str = stringpool_id2str(&data->spool, kv->id);
687 kv->str = id2str(pool, kv->id);
688 if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
691 for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
693 if (*s == ':' && s > kv->str)
697 case REPOKEY_TYPE_STR:
699 case REPOKEY_TYPE_DIRSTRARRAY:
700 if (!(flags & SEARCH_FILES))
701 return 1; /* match just the basename */
702 /* Put the full filename into kv->str. */
703 kv->str = repodata_dir2str(data, kv->id, kv->str);
704 /* And to compensate for that put the "empty" directory into
705 kv->id, so that later calls to repodata_dir2str on this data
706 come up with the same filename again. */
709 case REPOKEY_TYPE_MD5:
710 case REPOKEY_TYPE_SHA1:
711 case REPOKEY_TYPE_SHA256:
712 if (!(flags & SEARCH_CHECKSUMS))
713 return 0; /* skip em */
714 kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
722 struct subschema_data {
728 /* search a specific repodata */
730 repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
734 Id keyid, *kp, *keyp;
735 unsigned char *dp, *ddp;
741 if (!maybe_load_repodata(data, keyname))
743 if (solvid == SOLVID_SUBSCHEMA)
745 struct subschema_data *subd = cbdata;
746 cbdata = subd->cbdata;
748 schema = subd->parent->id;
749 dp = (unsigned char *)subd->parent->str;
750 kv.parent = subd->parent;
755 dp = solvid2data(data, solvid, &schema);
758 s = data->repo->pool->solvables + solvid;
761 keyp = data->schemadata + data->schemata[schema];
764 /* search for a specific key */
765 for (kp = keyp; *kp; kp++)
766 if (data->keys[*kp].name == keyname)
770 dp = forward_to_key(data, *kp, keyp, dp);
776 while ((keyid = *keyp++) != 0)
779 key = data->keys + keyid;
780 ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
782 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
784 struct subschema_data subd;
788 subd.cbdata = cbdata;
791 ddp = data_read_id(ddp, &nentries);
795 while (ddp && nentries > 0)
799 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
800 ddp = data_read_id(ddp, &schema);
802 kv.str = (char *)ddp;
803 stop = callback(cbdata, s, data, key, &kv);
804 if (stop > SEARCH_NEXT_KEY)
806 if (stop && stop != SEARCH_ENTERSUB)
808 if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
809 repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
810 ddp = data_skip_schema(data, ddp, schema);
813 if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
817 kv.str = (char *)ddp;
818 stop = callback(cbdata, s, data, key, &kv);
819 if (stop > SEARCH_NEXT_KEY)
829 ddp = data_fetch(ddp, &kv, key);
832 stop = callback(cbdata, s, data, key, &kv);
835 while (!kv.eof && !stop);
836 if (onekey || stop > SEARCH_NEXT_KEY)
842 repodata_setpos_kv(Repodata *data, KeyValue *kv)
844 Pool *pool = data->repo->pool;
846 pool_clear_pos(pool);
849 pool->pos.repo = data->repo;
850 pool->pos.repodataid = data - data->repo->repodata;
851 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
852 pool->pos.schema = kv->id;
856 /************************************************************************
857 * data iterator functions
860 static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = {
861 { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
862 { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
863 { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
864 { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
865 { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
866 { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
867 { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
868 { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
869 { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
870 { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
871 { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
872 { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
873 { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE },
877 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
891 case SOLVABLE_VENDOR:
894 case SOLVABLE_PROVIDES:
896 return s->provides ? s->repo->idarraydata + s->provides : 0;
897 case SOLVABLE_OBSOLETES:
899 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
900 case SOLVABLE_CONFLICTS:
902 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
903 case SOLVABLE_REQUIRES:
905 return s->requires ? s->repo->idarraydata + s->requires : 0;
906 case SOLVABLE_RECOMMENDS:
908 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
909 case SOLVABLE_SUPPLEMENTS:
911 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
912 case SOLVABLE_SUGGESTS:
914 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
915 case SOLVABLE_ENHANCES:
917 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
920 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
927 datamatcher_init(Datamatcher *ma, const char *match, int flags)
933 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
935 ma->matchdata = sat_calloc(1, sizeof(regex_t));
936 ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
939 sat_free(ma->matchdata);
940 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
947 datamatcher_free(Datamatcher *ma)
949 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
951 regfree(ma->matchdata);
952 ma->matchdata = sat_free(ma->matchdata);
957 datamatcher_match(Datamatcher *ma, const char *str)
960 switch ((ma->flags & SEARCH_STRINGMASK))
962 case SEARCH_SUBSTRING:
963 if (ma->flags & SEARCH_NOCASE)
965 if (!strcasestr(str, ma->match))
970 if (!strstr(str, ma->match))
975 if (ma->flags & SEARCH_NOCASE)
977 if (strcasecmp(ma->match, str))
982 if (strcmp(ma->match, str))
986 case SEARCH_STRINGSTART:
987 if (ma->flags & SEARCH_NOCASE)
989 if (strncasecmp(ma->match, str, strlen(ma->match)))
994 if (strncmp(ma->match, str, strlen(ma->match)))
998 case SEARCH_STRINGEND:
999 l = strlen(str) - strlen(ma->match);
1002 if (ma->flags & SEARCH_NOCASE)
1004 if (strcasecmp(ma->match, str + l))
1009 if (strcmp(ma->match, str + l))
1014 if (fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
1018 if (regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0))
1028 repodata_filelistfilter_matches(Repodata *data, const char *str)
1030 /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
1031 /* for now hardcoded */
1032 if (strstr(str, "bin/"))
1034 if (!strncmp(str, "/etc/", 5))
1036 if (!strcmp(str, "/usr/lib/sendmail"))
1058 di_nextarrayelement,
1063 di_nextsolvableattr,
1068 /* see repo.h for documentation */
1070 dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
1072 memset(di, 0, sizeof(*di));
1074 di->flags = flags & ~SEARCH_THISSOLVID;
1075 if (!pool || (repo && repo->pool != pool))
1083 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1089 di->keyname = keyname;
1090 di->keynames[0] = keyname;
1091 dataiterator_set_search(di, repo, p);
1096 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
1099 memset(&di->matcher, 0, sizeof(di->matcher));
1100 if (from->matcher.match)
1101 datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1106 for (i = 1; i < di->nparents; i++)
1107 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1108 di->kv.parent = &di->parents[di->nparents - 1].kv;
1113 dataiterator_set_match(Dataiterator *di, const char *match, int flags)
1115 di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
1116 datamatcher_free(&di->matcher);
1117 memset(&di->matcher, 0, sizeof(di->matcher));
1121 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1131 dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
1135 di->flags &= ~SEARCH_THISSOLVID;
1139 if (!di->pool->nrepos)
1147 di->repo = di->pool->repos[0];
1149 di->state = di_enterrepo;
1151 dataiterator_jump_to_solvid(di, p);
1155 dataiterator_set_keyname(Dataiterator *di, Id keyname)
1158 di->keyname = keyname;
1159 di->keynames[0] = keyname;
1163 dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1167 if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1169 di->state = di_bye; /* sorry */
1172 for (i = di->nkeynames + 1; i > 0; i--)
1173 di->keynames[i] = di->keynames[i - 1];
1174 di->keynames[0] = di->keyname = keyname;
1179 dataiterator_free(Dataiterator *di)
1181 if (di->matcher.match)
1182 datamatcher_free(&di->matcher);
1185 static inline unsigned char *
1186 dataiterator_find_keyname(Dataiterator *di, Id keyname)
1188 Id *keyp = di->keyp;
1189 Repokey *keys = di->data->keys;
1192 for (keyp = di->keyp; *keyp; keyp++)
1193 if (keys[*keyp].name == keyname)
1197 dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1205 dataiterator_filelistcheck(Dataiterator *di)
1208 int needcomplete = 0;
1209 Repodata *data = di->data;
1211 if ((di->matcher.flags & SEARCH_COMPLETE_FILELIST) != 0)
1212 if (!di->matcher.match || (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING || !repodata_filelistfilter_matches(di->data, di->matcher.match))
1214 if (data->state != REPODATA_AVAILABLE)
1215 return needcomplete ? 1 : 0;
1216 for (j = 1; j < data->nkeys; j++)
1217 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1219 return j == data->nkeys && !needcomplete ? 0 : 1;
1223 dataiterator_step(Dataiterator *di)
1231 case di_enterrepo: di_enterrepo:
1234 if (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS))
1236 if (!(di->flags & SEARCH_THISSOLVID))
1238 di->solvid = di->repo->start - 1; /* reset solvid iterator */
1239 goto di_nextsolvable;
1243 case di_entersolvable: di_entersolvable:
1244 if (di->repodataid >= 0)
1246 di->repodataid = 0; /* reset repodata iterator */
1247 if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1249 di->key = solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1251 goto di_entersolvablekey;
1256 case di_enterrepodata: di_enterrepodata:
1257 if (di->repodataid >= 0)
1259 if (di->repodataid >= di->repo->nrepodata)
1260 goto di_nextsolvable;
1261 di->data = di->repo->repodata + di->repodataid;
1263 if (di->repodataid >= 0 && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1264 goto di_nextrepodata;
1265 if (!maybe_load_repodata(di->data, di->keyname))
1266 goto di_nextrepodata;
1267 di->dp = solvid2data(di->data, di->solvid, &schema);
1269 goto di_nextrepodata;
1270 if (di->solvid == SOLVID_POS)
1271 di->solvid = di->pool->pos.solvid;
1272 /* reset key iterator */
1273 di->keyp = di->data->schemadata + di->data->schemata[schema];
1276 case di_enterschema: di_enterschema:
1278 di->dp = dataiterator_find_keyname(di, di->keyname);
1279 if (!di->dp || !*di->keyp)
1283 goto di_nextrepodata;
1287 case di_enterkey: di_enterkey:
1289 di->key = di->data->keys + *di->keyp;
1290 di->ddp = get_data(di->data, di->key, &di->dp, di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0) ? 1 : 0);
1293 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1295 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1301 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1303 di->state = di_nextkey;
1305 di->state = di_nextattr;
1308 case di_nextkey: di_nextkey:
1309 if (!di->keyname && *++di->keyp)
1315 case di_nextrepodata: di_nextrepodata:
1316 if (di->repodataid >= 0 && ++di->repodataid < di->repo->nrepodata)
1317 goto di_enterrepodata;
1320 case di_nextsolvable: di_nextsolvable:
1321 if (!(di->flags & SEARCH_THISSOLVID))
1324 di->solvid = di->repo->start;
1327 for (; di->solvid < di->repo->end; di->solvid++)
1329 if (di->pool->solvables[di->solvid].repo == di->repo)
1330 goto di_entersolvable;
1335 case di_nextrepo: di_nextrepo:
1336 if (di->repoid >= 0)
1340 if (di->repoid < di->pool->nrepos)
1342 di->repo = di->pool->repos[di->repoid];
1348 case di_bye: di_bye:
1352 case di_enterarray: di_enterarray:
1353 if (di->key->name == REPOSITORY_SOLVABLES)
1355 di->ddp = data_read_id(di->ddp, &di->kv.num);
1360 case di_nextarrayelement: di_nextarrayelement:
1363 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1364 if (di->kv.entry == di->kv.num)
1366 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1368 if (!(di->flags & SEARCH_ARRAYSENTINEL))
1370 di->kv.str = (char *)di->ddp;
1372 di->state = di_nextkey;
1375 if (di->kv.entry == di->kv.num - 1)
1377 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1378 di->ddp = data_read_id(di->ddp, &di->kv.id);
1379 di->kv.str = (char *)di->ddp;
1380 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1382 if ((di->flags & SEARCH_SUB) != 0)
1383 di->state = di_entersub;
1385 di->state = di_nextarrayelement;
1388 case di_entersub: di_entersub:
1389 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1390 goto di_nextarrayelement; /* sorry, full */
1391 di->parents[di->nparents].kv = di->kv;
1392 di->parents[di->nparents].dp = di->dp;
1393 di->parents[di->nparents].keyp = di->keyp;
1394 di->dp = (unsigned char *)di->kv.str;
1395 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1396 memset(&di->kv, 0, sizeof(di->kv));
1397 di->kv.parent = &di->parents[di->nparents].kv;
1399 di->keyname = di->keynames[di->nparents - di->rootlevel];
1400 goto di_enterschema;
1402 case di_leavesub: di_leavesub:
1403 if (di->nparents - 1 < di->rootlevel)
1406 di->dp = di->parents[di->nparents].dp;
1407 di->kv = di->parents[di->nparents].kv;
1408 di->keyp = di->parents[di->nparents].keyp;
1409 di->key = di->data->keys + *di->keyp;
1410 di->ddp = (unsigned char *)di->kv.str;
1411 di->keyname = di->keynames[di->nparents - di->rootlevel];
1412 goto di_nextarrayelement;
1414 /* special solvable attr handling follows */
1416 case di_nextsolvableattr:
1417 di->kv.id = *di->idp++;
1422 di->state = di_nextsolvablekey;
1426 case di_nextsolvablekey: di_nextsolvablekey:
1427 if (di->keyname || di->key->name == RPM_RPMDBID)
1428 goto di_enterrepodata;
1432 case di_entersolvablekey: di_entersolvablekey:
1433 di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1434 if (!di->idp || !di->idp[0])
1435 goto di_nextsolvablekey;
1436 di->kv.id = di->idp[0];
1437 di->kv.num = di->idp[0];
1439 if (!di->kv.eof && !di->idp[0])
1443 di->state = di_nextsolvablekey;
1445 di->state = di_nextsolvableattr;
1449 if (di->matcher.match)
1451 /* simple pre-check so that we don't need to stringify */
1452 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && di->matcher.match && (di->matcher.flags & (SEARCH_FILES|SEARCH_NOCASE|SEARCH_STRINGMASK)) == (SEARCH_FILES|SEARCH_STRING))
1454 int l = strlen(di->matcher.match) - strlen(di->kv.str);
1455 if (l < 0 || strcmp(di->matcher.match + l, di->kv.str))
1458 if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
1460 if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
1464 if (!datamatcher_match(&di->matcher, di->kv.str))
1467 /* found something! */
1473 dataiterator_entersub(Dataiterator *di)
1475 if (di->state == di_nextarrayelement)
1476 di->state = di_entersub;
1480 dataiterator_setpos(Dataiterator *di)
1482 if (di->kv.eof == 2)
1484 pool_clear_pos(di->pool);
1487 di->pool->pos.solvid = di->solvid;
1488 di->pool->pos.repo = di->repo;
1489 di->pool->pos.repodataid = di->data - di->repo->repodata;
1490 di->pool->pos.schema = di->kv.id;
1491 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1495 dataiterator_setpos_parent(Dataiterator *di)
1497 if (!di->kv.parent || di->kv.parent->eof == 2)
1499 pool_clear_pos(di->pool);
1502 di->pool->pos.solvid = di->solvid;
1503 di->pool->pos.repo = di->repo;
1504 di->pool->pos.repodataid = di->data - di->repo->repodata;
1505 di->pool->pos.schema = di->kv.parent->id;
1506 di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1509 /* clones just the position, not the search keys/matcher */
1511 dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
1513 di->state = from->state;
1514 di->flags &= ~SEARCH_THISSOLVID;
1515 di->flags |= (from->flags & SEARCH_THISSOLVID);
1516 di->repo = from->repo;
1517 di->data = from->data;
1519 di->ddp = from->ddp;
1520 di->idp = from->idp;
1521 di->keyp = from->keyp;
1522 di->key = from->key;
1524 di->repodataid = from->repodataid;
1525 di->solvid = from->solvid;
1526 di->repoid = from->repoid;
1527 di->rootlevel = from->rootlevel;
1528 memcpy(di->parents, from->parents, sizeof(from->parents));
1529 di->nparents = from->nparents;
1533 for (i = 1; i < di->nparents; i++)
1534 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1535 di->kv.parent = &di->parents[di->nparents - 1].kv;
1540 dataiterator_seek(Dataiterator *di, int whence)
1542 if ((whence & DI_SEEK_STAY) != 0)
1543 di->rootlevel = di->nparents;
1544 switch (whence & ~DI_SEEK_STAY)
1547 if (di->state != di_nextarrayelement)
1549 if ((whence & DI_SEEK_STAY) != 0)
1550 di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1551 di->state = di_entersub;
1553 case DI_SEEK_PARENT:
1560 if (di->rootlevel > di->nparents)
1561 di->rootlevel = di->nparents;
1562 di->dp = di->parents[di->nparents].dp;
1563 di->kv = di->parents[di->nparents].kv;
1564 di->keyp = di->parents[di->nparents].keyp;
1565 di->key = di->data->keys + *di->keyp;
1566 di->ddp = (unsigned char *)di->kv.str;
1567 di->keyname = di->keynames[di->nparents - di->rootlevel];
1568 di->state = di_nextarrayelement;
1570 case DI_SEEK_REWIND:
1576 di->dp = (unsigned char *)di->kv.parent->str;
1577 di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1578 di->state = di_enterschema;
1586 dataiterator_skip_attribute(Dataiterator *di)
1588 if (di->state == di_nextsolvableattr)
1589 di->state = di_nextsolvablekey;
1591 di->state = di_nextkey;
1595 dataiterator_skip_solvable(Dataiterator *di)
1599 di->keyname = di->keynames[0];
1600 di->state = di_nextsolvable;
1604 dataiterator_skip_repo(Dataiterator *di)
1608 di->keyname = di->keynames[0];
1609 di->state = di_nextrepo;
1613 dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1617 di->keyname = di->keynames[0];
1618 if (solvid == SOLVID_POS)
1620 di->repo = di->pool->pos.repo;
1627 di->data = di->repo->repodata + di->pool->pos.repodataid;
1628 di->repodataid = -1;
1629 di->solvid = solvid;
1630 di->state = di_enterrepo;
1631 di->flags |= SEARCH_THISSOLVID;
1636 di->repo = di->pool->solvables[solvid].repo;
1639 else if (di->repoid >= 0)
1641 if (!di->pool->nrepos)
1646 di->repo = di->pool->repos[0];
1650 di->solvid = solvid;
1652 di->flags |= SEARCH_THISSOLVID;
1653 di->state = di_enterrepo;
1657 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1665 di->flags &= ~SEARCH_THISSOLVID;
1666 di->state = di_enterrepo;
1670 dataiterator_match(Dataiterator *di, Datamatcher *ma)
1672 if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
1676 return datamatcher_match(ma, di->kv.str);
1679 /************************************************************************
1680 * data modify functions
1683 /* extend repodata so that it includes solvables p */
1685 repodata_extend(Repodata *data, Id p)
1687 if (data->start == data->end)
1688 data->start = data->end = p;
1691 int old = data->end - data->start;
1692 int new = p - data->end + 1;
1695 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
1696 memset(data->attrs + old, 0, new * sizeof(Id *));
1698 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
1699 memset(data->incoreoffset + old, 0, new * sizeof(Id));
1702 if (p < data->start)
1704 int old = data->end - data->start;
1705 int new = data->start - p;
1708 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
1709 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
1710 memset(data->attrs, 0, new * sizeof(Id *));
1712 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
1713 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
1714 memset(data->incoreoffset, 0, new * sizeof(Id));
1719 /* shrink end of repodata */
1721 repodata_shrink(Repodata *data, int end)
1725 if (data->end <= end)
1727 if (data->start >= end)
1731 for (i = 0; i < data->end - data->start; i++)
1732 sat_free(data->attrs[i]);
1733 data->attrs = sat_free(data->attrs);
1735 data->incoreoffset = sat_free(data->incoreoffset);
1736 data->start = data->end = 0;
1741 for (i = end; i < data->end; i++)
1742 sat_free(data->attrs[i - data->start]);
1743 data->attrs = sat_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
1745 if (data->incoreoffset)
1746 data->incoreoffset = sat_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
1750 /* extend repodata so that it includes solvables from start to start + num - 1 */
1752 repodata_extend_block(Repodata *data, Id start, Id num)
1756 if (!data->incoreoffset)
1758 data->incoreoffset = sat_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
1759 data->start = start;
1760 data->end = start + num;
1763 repodata_extend(data, start);
1765 repodata_extend(data, start + num - 1);
1768 /**********************************************************************/
1771 #define REPODATA_ATTRS_BLOCK 31
1772 #define REPODATA_ATTRDATA_BLOCK 1023
1773 #define REPODATA_ATTRIDDATA_BLOCK 63
1777 repodata_new_handle(Repodata *data)
1781 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1784 data->xattrs = sat_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
1785 data->xattrs[data->nxattrs] = 0;
1786 return -(data->nxattrs++);
1790 repodata_get_attrp(Repodata *data, Id handle)
1792 if (handle == SOLVID_META)
1796 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1801 return data->xattrs - handle;
1802 if (handle < data->start || handle >= data->end)
1803 repodata_extend(data, handle);
1805 data->attrs = sat_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
1806 return data->attrs + (handle - data->start);
1810 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
1816 app = repodata_get_attrp(data, handle);
1821 /* Determine equality based on the name only, allows us to change
1822 type (when overwrite is set), and makes TYPE_CONSTANT work. */
1823 for (pp = ap; *pp; pp += 2)
1824 if (data->keys[*pp].name == data->keys[keyid].name)
1837 ap = sat_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
1847 repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
1851 keyid = repodata_key2id(data, key, 1);
1852 repodata_insert_keyid(data, solvid, keyid, val, 1);
1856 repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
1860 key.type = REPOKEY_TYPE_ID;
1862 key.storage = KEY_STORAGE_INCORE;
1863 repodata_set(data, solvid, &key, id);
1867 repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned int num)
1871 key.type = REPOKEY_TYPE_NUM;
1873 key.storage = KEY_STORAGE_INCORE;
1874 repodata_set(data, solvid, &key, (Id)num);
1878 repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
1882 if (data->localpool)
1883 id = stringpool_str2id(&data->spool, str, 1);
1885 id = str2id(data->repo->pool, str, 1);
1887 key.type = REPOKEY_TYPE_ID;
1889 key.storage = KEY_STORAGE_INCORE;
1890 repodata_set(data, solvid, &key, id);
1894 repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
1898 key.type = REPOKEY_TYPE_CONSTANT;
1899 key.size = constant;
1900 key.storage = KEY_STORAGE_INCORE;
1901 repodata_set(data, solvid, &key, 0);
1905 repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
1909 key.type = REPOKEY_TYPE_CONSTANTID;
1911 key.storage = KEY_STORAGE_INCORE;
1912 repodata_set(data, solvid, &key, 0);
1916 repodata_set_void(Repodata *data, Id solvid, Id keyname)
1920 key.type = REPOKEY_TYPE_VOID;
1922 key.storage = KEY_STORAGE_INCORE;
1923 repodata_set(data, solvid, &key, 0);
1927 repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
1932 l = strlen(str) + 1;
1934 key.type = REPOKEY_TYPE_STR;
1936 key.storage = KEY_STORAGE_INCORE;
1937 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1938 memcpy(data->attrdata + data->attrdatalen, str, l);
1939 repodata_set(data, solvid, &key, data->attrdatalen);
1940 data->attrdatalen += l;
1943 /* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
1944 * so that the caller can append the new element there */
1946 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
1949 Id *ida, *pp, **ppp;
1951 /* check if it is the same as last time, this speeds things up a lot */
1952 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
1954 /* great! just append the new data */
1955 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1956 data->attriddatalen--; /* overwrite terminating 0 */
1957 data->lastdatalen += entrysize;
1961 ppp = repodata_get_attrp(data, handle);
1964 for (; *pp; pp += 2)
1965 if (data->keys[*pp].name == keyname && data->keys[*pp].type == keytype)
1969 /* not found. allocate new key */
1974 key.storage = KEY_STORAGE_INCORE;
1975 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1976 repodata_set(data, handle, &key, data->attriddatalen);
1977 data->lasthandle = 0; /* next time... */
1981 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
1982 oldsize += entrysize;
1983 if (ida + 1 == data->attriddata + data->attriddatalen)
1985 /* this was the last entry, just append it */
1986 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1987 data->attriddatalen--; /* overwrite terminating 0 */
1991 /* too bad. move to back. */
1992 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1993 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
1994 pp[1] = data->attriddatalen;
1995 data->attriddatalen += oldsize;
1997 data->lasthandle = handle;
1998 data->lastkey = *pp;
1999 data->lastdatalen = data->attriddatalen + entrysize + 1;
2003 repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2004 const unsigned char *str)
2009 if (!(l = sat_chksum_len(type)))
2014 key.storage = KEY_STORAGE_INCORE;
2015 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2016 memcpy(data->attrdata + data->attrdatalen, str, l);
2017 repodata_set(data, solvid, &key, data->attrdatalen);
2018 data->attrdatalen += l;
2022 hexstr2bytes(unsigned char *buf, const char *str, int buflen)
2025 for (i = 0; i < buflen; i++)
2027 #define c2h(c) (((c)>='0' && (c)<='9') ? ((c)-'0') \
2028 : ((c)>='a' && (c)<='f') ? ((c)-('a'-10)) \
2029 : ((c)>='A' && (c)<='F') ? ((c)-('A'-10)) \
2040 buf[i] = (buf[i] << 4) | v;
2047 repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2050 unsigned char buf[64];
2053 if (!(l = sat_chksum_len(type)))
2055 if (hexstr2bytes(buf, str, l) != l)
2057 repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2061 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2066 if (!(l = sat_chksum_len(type)))
2068 s = str = pool_alloctmpspace(data->repo->pool, 2 * l + 1);
2069 for (i = 0; i < l; i++)
2071 unsigned char v = buf[i];
2072 unsigned char w = v >> 4;
2073 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2075 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2081 /* rpm filenames don't contain the epoch, so strip it */
2082 static inline const char *
2083 evrid2vrstr(Pool *pool, Id evrid)
2085 const char *p, *evr = id2str(pool, evrid);
2088 for (p = evr; *p >= '0' && *p <= '9'; p++)
2090 return p != evr && *p == ':' ? p + 1 : evr;
2094 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
2096 Pool *pool = data->repo->pool;
2098 const char *str, *fp;
2102 repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2105 if ((dir = strrchr(file, '/')) != 0)
2116 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2121 if (l == 1 && dir[0] == '.')
2123 s = pool->solvables + solvid;
2126 str = id2str(pool, s->arch);
2127 if (!strncmp(dir, str, l) && !str[l])
2128 repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2130 repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir);
2133 char *dir2 = strdup(dir);
2135 repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir2);
2140 str = id2str(pool, s->name);
2142 if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
2145 str = evrid2vrstr(pool, s->evr);
2147 if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
2150 str = id2str(pool, s->arch);
2152 if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2154 repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2159 repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2163 repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2167 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2169 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2170 data->attriddata[data->attriddatalen++] = dir;
2171 data->attriddata[data->attriddatalen++] = num;
2172 data->attriddata[data->attriddatalen++] = num2;
2173 data->attriddata[data->attriddatalen++] = 0;
2177 repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
2183 l = strlen(str) + 1;
2184 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2185 memcpy(data->attrdata + data->attrdatalen, str, l);
2186 stroff = data->attrdatalen;
2187 data->attrdatalen += l;
2190 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2192 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2193 data->attriddata[data->attriddatalen++] = dir;
2194 data->attriddata[data->attriddatalen++] = stroff;
2195 data->attriddata[data->attriddatalen++] = 0;
2199 repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2202 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2204 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2205 data->attriddata[data->attriddatalen++] = id;
2206 data->attriddata[data->attriddatalen++] = 0;
2210 repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2214 if (data->localpool)
2215 id = stringpool_str2id(&data->spool, str, 1);
2217 id = str2id(data->repo->pool, str, 1);
2218 repodata_add_idarray(data, solvid, keyname, id);
2222 repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2224 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2225 data->attriddata[data->attriddatalen++] = ghandle;
2226 data->attriddata[data->attriddatalen++] = 0;
2230 repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2232 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2233 data->attriddata[data->attriddatalen++] = ghandle;
2234 data->attriddata[data->attriddatalen++] = 0;
2237 /* add all attrs from src to dest */
2239 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2242 if (dest == src || !(keyp = data->attrs[src - data->start]))
2244 for (; *keyp; keyp += 2)
2245 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2249 repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
2252 if (dest == src || !(keyp = data->attrs[src - data->start]))
2254 for (; *keyp; keyp += 2)
2255 if (!keyidmap || MAPTST(keyidmap, keyp[0]))
2256 repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2261 /**********************************************************************/
2263 /* TODO: unify with repo_write! */
2265 #define EXTDATA_BLOCK 1023
2273 data_addid(struct extdata *xd, Id x)
2276 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2277 dp = xd->buf + xd->len;
2282 *dp++ = (x >> 28) | 128;
2284 *dp++ = (x >> 21) | 128;
2285 *dp++ = (x >> 14) | 128;
2288 *dp++ = (x >> 7) | 128;
2290 xd->len = dp - xd->buf;
2294 data_addideof(struct extdata *xd, Id x, int eof)
2297 x = (x & 63) | ((x & ~63) << 1);
2298 data_addid(xd, (eof ? x: x | 64));
2302 data_addblob(struct extdata *xd, unsigned char *blob, int len)
2304 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
2305 memcpy(xd->buf + xd->len, blob, len);
2309 /*********************************/
2312 repodata_serialize_key(Repodata *data, struct extdata *newincore,
2313 struct extdata *newvincore,
2315 Repokey *key, Id val)
2317 /* Otherwise we have a new value. Parse it into the internal
2321 unsigned int oldvincorelen = 0;
2325 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2328 oldvincorelen = xd->len;
2332 case REPOKEY_TYPE_VOID:
2333 case REPOKEY_TYPE_CONSTANT:
2334 case REPOKEY_TYPE_CONSTANTID:
2336 case REPOKEY_TYPE_STR:
2337 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
2339 case REPOKEY_TYPE_MD5:
2340 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
2342 case REPOKEY_TYPE_SHA1:
2343 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
2345 case REPOKEY_TYPE_SHA256:
2346 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
2348 case REPOKEY_TYPE_ID:
2349 case REPOKEY_TYPE_NUM:
2350 case REPOKEY_TYPE_DIR:
2351 data_addid(xd, val);
2353 case REPOKEY_TYPE_IDARRAY:
2354 for (ida = data->attriddata + val; *ida; ida++)
2355 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
2357 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2358 for (ida = data->attriddata + val; *ida; ida += 3)
2360 data_addid(xd, ida[0]);
2361 data_addid(xd, ida[1]);
2362 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
2365 case REPOKEY_TYPE_DIRSTRARRAY:
2366 for (ida = data->attriddata + val; *ida; ida += 2)
2368 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
2369 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
2372 case REPOKEY_TYPE_FIXARRAY:
2376 for (ida = data->attriddata + val; *ida; ida++)
2379 fprintf(stderr, "serialize struct %d\n", *ida);
2382 Id *kp = data->xattrs[-*ida];
2389 fprintf(stderr, " %s:%d\n", id2str(data->repo->pool, data->keys[*kp].name), kp[1]);
2395 schemaid = repodata_schema2id(data, schema, 1);
2396 else if (schemaid != repodata_schema2id(data, schema, 0))
2398 pool_debug(data->repo->pool, SAT_FATAL, "fixarray substructs with different schemas\n");
2402 fprintf(stderr, " schema %d\n", schemaid);
2407 data_addid(xd, num);
2408 data_addid(xd, schemaid);
2409 for (ida = data->attriddata + val; *ida; ida++)
2411 Id *kp = data->xattrs[-*ida];
2416 repodata_serialize_key(data, newincore, newvincore,
2417 schema, data->keys + *kp, kp[1]);
2422 case REPOKEY_TYPE_FLEXARRAY:
2425 for (ida = data->attriddata + val; *ida; ida++)
2427 data_addid(xd, num);
2428 for (ida = data->attriddata + val; *ida; ida++)
2430 Id *kp = data->xattrs[-*ida];
2433 data_addid(xd, 0); /* XXX */
2440 schemaid = repodata_schema2id(data, schema, 1);
2441 data_addid(xd, schemaid);
2442 kp = data->xattrs[-*ida];
2445 repodata_serialize_key(data, newincore, newvincore,
2446 schema, data->keys + *kp, kp[1]);
2452 pool_debug(data->repo->pool, SAT_FATAL, "don't know how to handle type %d\n", key->type);
2455 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2457 /* put offset/len in incore */
2458 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
2459 oldvincorelen = xd->len - oldvincorelen;
2460 data_addid(newincore, oldvincorelen);
2465 repodata_internalize(Repodata *data)
2467 Repokey *key, solvkey;
2469 Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen;
2470 unsigned char *dp, *ndp;
2471 int newschema, oldcount;
2472 struct extdata newincore;
2473 struct extdata newvincore;
2476 if (!data->attrs && !data->xattrs)
2479 newvincore.buf = data->vincore;
2480 newvincore.len = data->vincorelen;
2482 /* find the solvables key, create if needed */
2483 memset(&solvkey, 0, sizeof(solvkey));
2484 solvkey.name = REPOSITORY_SOLVABLES;
2485 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
2487 solvkey.storage = KEY_STORAGE_INCORE;
2488 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
2490 schema = sat_malloc2(data->nkeys, sizeof(Id));
2491 seen = sat_malloc2(data->nkeys, sizeof(Id));
2493 /* Merge the data already existing (in data->schemata, ->incoredata and
2494 friends) with the new attributes in data->attrs[]. */
2495 nentry = data->end - data->start;
2496 memset(&newincore, 0, sizeof(newincore));
2497 data_addid(&newincore, 0); /* start data at offset 1 */
2499 data->mainschema = 0;
2500 data->mainschemaoffsets = sat_free(data->mainschemaoffsets);
2502 /* join entry data */
2503 /* we start with the meta data, entry -1 */
2504 for (entry = -1; entry < nentry; entry++)
2506 memset(seen, 0, data->nkeys * sizeof(Id));
2508 dp = data->incoredata;
2511 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
2512 dp = data_read_id(dp, &oldschema);
2515 fprintf(stderr, "oldschema %d\n", oldschema);
2516 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
2517 fprintf(stderr, "schemadata %p\n", data->schemadata);
2519 /* seen: -1: old data 0: skipped >0: id + 1 */
2523 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
2527 pool_debug(data->repo->pool, SAT_FATAL, "Inconsistent old data (key occured twice).\n");
2535 keyp = data->attrs ? data->attrs[entry] : 0;
2538 /* strip solvables key */
2540 for (sp = keyp = schema; *sp; sp++)
2541 if (*sp != solvkeyid)
2546 seen[solvkeyid] = 0;
2547 keyp = data->xattrs ? data->xattrs[1] : 0;
2550 for (; *keyp; keyp += 2)
2557 seen[*keyp] = keyp[1] + 1;
2559 if (entry < 0 && data->end != data->start)
2566 /* Ideally we'd like to sort the new schema here, to ensure
2567 schema equality independend of the ordering. We can't do that
2568 yet. For once see below (old ids need to come before new ids).
2569 An additional difficulty is that we also need to move
2570 the values with the keys. */
2571 schemaid = repodata_schema2id(data, schema, 1);
2573 schemaid = oldschema;
2576 /* Now create data blob. We walk through the (possibly new) schema
2577 and either copy over old data, or insert the new. */
2578 /* XXX Here we rely on the fact that the (new) schema has the form
2579 o1 o2 o3 o4 ... | n1 n2 n3 ...
2580 (oX being the old keyids (possibly overwritten), and nX being
2581 the new keyids). This rules out sorting the keyids in order
2582 to ensure a small schema count. */
2584 data->incoreoffset[entry] = newincore.len;
2585 data_addid(&newincore, schemaid);
2588 data->mainschema = schemaid;
2589 data->mainschemaoffsets = sat_calloc(sp - schema, sizeof(Id));
2591 keypstart = data->schemadata + data->schemata[schemaid];
2592 for (keyp = keypstart; *keyp; keyp++)
2595 data->mainschemaoffsets[keyp - keypstart] = newincore.len;
2596 if (*keyp == solvkeyid)
2598 /* add flexarray entry count */
2599 data_addid(&newincore, data->end - data->start);
2602 key = data->keys + *keyp;
2604 fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, id2str(data->repo->pool, key->name), id2str(data->repo->pool, key->type));
2609 /* Skip the data associated with this old key. */
2610 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2612 ndp = data_skip(dp, REPOKEY_TYPE_ID);
2613 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
2615 else if (key->storage == KEY_STORAGE_INCORE)
2616 ndp = data_skip_key(data, dp, key);
2619 if (seen[*keyp] == -1)
2621 /* If this key was an old one _and_ was not overwritten with
2622 a different value copy over the old value (we skipped it
2625 data_addblob(&newincore, dp, ndp - dp);
2628 else if (seen[*keyp])
2630 /* Otherwise we have a new value. Parse it into the internal
2632 repodata_serialize_key(data, &newincore, &newvincore,
2633 schema, key, seen[*keyp] - 1);
2637 if (entry >= 0 && data->attrs && data->attrs[entry])
2638 data->attrs[entry] = sat_free(data->attrs[entry]);
2640 /* free all xattrs */
2641 for (entry = 0; entry < data->nxattrs; entry++)
2642 if (data->xattrs[entry])
2643 sat_free(data->xattrs[entry]);
2644 data->xattrs = sat_free(data->xattrs);
2647 data->lasthandle = 0;
2649 data->lastdatalen = 0;
2652 repodata_free_schemahash(data);
2654 sat_free(data->incoredata);
2655 data->incoredata = newincore.buf;
2656 data->incoredatalen = newincore.len;
2657 data->incoredatafree = 0;
2659 sat_free(data->vincore);
2660 data->vincore = newvincore.buf;
2661 data->vincorelen = newvincore.len;
2663 data->attrs = sat_free(data->attrs);
2664 data->attrdata = sat_free(data->attrdata);
2665 data->attriddata = sat_free(data->attriddata);
2666 data->attrdatalen = 0;
2667 data->attriddatalen = 0;
2671 repodata_disable_paging(Repodata *data)
2673 if (maybe_load_repodata(data, 0))
2674 repopagestore_disable_paging(&data->store);
2678 repodata_load_stub(Repodata *data)
2680 Repo *repo = data->repo;
2681 Pool *pool = repo->pool;
2684 if (!pool->loadcallback)
2686 data->state = REPODATA_ERROR;
2689 data->state = REPODATA_LOADING;
2690 r = pool->loadcallback(pool, data, pool->loadcallbackdata);
2692 data->state = REPODATA_ERROR;
2696 repodata_create_stubs(Repodata *data)
2698 Repo *repo = data->repo;
2699 Pool *pool = repo->pool;
2706 int datastart, dataend;
2708 repodataid = data - repo->repodata;
2709 datastart = data->start;
2710 dataend = data->end;
2711 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
2712 while (dataiterator_step(&di))
2714 if (di.data - repo->repodata != repodataid)
2718 dataiterator_free(&di);
2721 stubdataids = sat_calloc(cnt, sizeof(*stubdataids));
2722 for (i = 0; i < cnt; i++)
2724 sdata = repo_add_repodata(repo, 0);
2725 if (dataend > datastart)
2726 repodata_extend_block(sdata, datastart, dataend - datastart);
2727 stubdataids[i] = sdata - repo->repodata;
2728 sdata->state = REPODATA_STUB;
2729 sdata->loadcallback = repodata_load_stub;
2732 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
2734 while (dataiterator_step(&di))
2736 if (di.data - repo->repodata != repodataid)
2738 if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
2740 dataiterator_entersub(&di);
2741 sdata = repo->repodata + stubdataids[i++];
2745 switch (di.key->type)
2747 case REPOKEY_TYPE_ID:
2748 repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
2750 case REPOKEY_TYPE_CONSTANTID:
2751 repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
2753 case REPOKEY_TYPE_STR:
2754 repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
2756 case REPOKEY_TYPE_VOID:
2757 repodata_set_void(sdata, SOLVID_META, di.key->name);
2759 case REPOKEY_TYPE_NUM:
2760 repodata_set_num(sdata, SOLVID_META, di.key->name, di.kv.num);
2762 case REPOKEY_TYPE_MD5:
2763 case REPOKEY_TYPE_SHA1:
2764 case REPOKEY_TYPE_SHA256:
2765 repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
2767 case REPOKEY_TYPE_IDARRAY:
2768 repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
2769 if (di.key->name == REPOSITORY_KEYS)
2776 xkeyname = di.kv.id;
2779 xkey.name = xkeyname;
2780 xkey.type = di.kv.id;
2781 xkey.storage = KEY_STORAGE_INCORE;
2783 repodata_key2id(sdata, &xkey, 1);
2788 dataiterator_free(&di);
2789 for (i = 0; i < cnt; i++)
2790 repodata_internalize(repo->repodata + stubdataids[i]);
2791 sat_free(stubdataids);
2795 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: