2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
13 * a repository can contain multiple repodata entries, consisting of
14 * different sets of keys and different sets of solvables
29 #include "poolid_private.h"
37 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
38 unsigned char *out, unsigned int out_len);
39 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
42 unsigned int out_len);
44 #define REPODATA_BLOCK 255
48 repodata_initdata(Repodata *data, Repo *repo, int localpool)
50 memset(data, 0, sizeof (*data));
52 data->localpool = localpool;
54 stringpool_init_empty(&data->spool);
55 data->keys = sat_calloc(1, sizeof(Repokey));
57 data->schemata = sat_calloc(1, sizeof(Id));
58 data->schemadata = sat_calloc(1, sizeof(Id));
60 data->schemadatalen = 1;
61 repopagestore_init(&data->store);
65 repodata_freedata(Repodata *data)
71 sat_free(data->schemata);
72 sat_free(data->schemadata);
73 sat_free(data->schematahash);
75 stringpool_free(&data->spool);
76 dirpool_free(&data->dirpool);
78 sat_free(data->mainschemaoffsets);
79 sat_free(data->incoredata);
80 sat_free(data->incoreoffset);
81 sat_free(data->verticaloffset);
83 repopagestore_free(&data->store);
85 sat_free(data->vincore);
88 for (i = 0; i < data->end - data->start; i++)
89 sat_free(data->attrs[i]);
90 sat_free(data->attrs);
92 for (i = 0; i < data->nxattrs; i++)
93 sat_free(data->xattrs[i]);
94 sat_free(data->xattrs);
96 sat_free(data->attrdata);
97 sat_free(data->attriddata);
101 repodata_create(Repo *repo, int localpool)
106 repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata, sizeof(*data));
107 data = repo->repodata + repo->nrepodata - 1;
108 repodata_initdata(data, repo, localpool);
113 repodata_free(Repodata *data)
115 Repo *repo = data->repo;
116 int i = data - repo->repodata;
117 repodata_freedata(data);
118 if (i < repo->nrepodata - 1)
119 memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
124 /***************************************************************
125 * key pool management
128 /* this is not so time critical that we need a hash, so we do a simple
131 repodata_key2id(Repodata *data, Repokey *key, int create)
135 for (keyid = 1; keyid < data->nkeys; keyid++)
136 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
138 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
142 if (keyid == data->nkeys)
146 /* allocate new key */
147 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
148 data->keys[data->nkeys++] = *key;
149 if (data->verticaloffset)
151 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
152 data->verticaloffset[data->nkeys - 1] = 0;
154 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
160 /***************************************************************
161 * schema pool management
164 #define SCHEMATA_BLOCK 31
165 #define SCHEMATADATA_BLOCK 255
168 repodata_schema2id(Repodata *data, Id *schema, int create)
174 if ((schematahash = data->schematahash) == 0)
176 data->schematahash = schematahash = sat_calloc(256, sizeof(Id));
177 for (i = 0; i < data->nschemata; i++)
179 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
182 schematahash[h] = i + 1;
184 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
185 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
188 for (sp = schema, len = 0, h = 0; *sp; len++)
193 cid = schematahash[h];
197 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
200 for (cid = 0; cid < data->nschemata; cid++)
201 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
207 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
208 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
210 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
211 data->schemata[data->nschemata] = data->schemadatalen;
212 data->schemadatalen += len;
213 schematahash[h] = data->nschemata + 1;
215 fprintf(stderr, "schema2id: new schema\n");
217 return data->nschemata++;
221 repodata_free_schemahash(Repodata *data)
223 data->schematahash = sat_free(data->schematahash);
225 data->schemata = sat_realloc2(data->schemata, data->nschemata, sizeof(Id));
226 data->schemadata = sat_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
230 /***************************************************************
231 * dir pool management
235 repodata_str2dir(Repodata *data, const char *dir, int create)
241 while (*dir == '/' && dir[1] == '/')
243 if (*dir == '/' && !dir[1])
245 if (data->dirpool.ndirs)
247 return dirpool_add_dir(&data->dirpool, 0, 1, create);
251 dire = strchrnul(dir, '/');
253 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
255 id = strn2id(data->repo->pool, dir, dire - dir, create);
258 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
271 repodata_dir2str(Repodata *data, Id did, const char *suf)
273 Pool *pool = data->repo->pool;
280 return suf ? suf : "";
284 comp = dirpool_compid(&data->dirpool, parent);
285 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
287 parent = dirpool_parent(&data->dirpool, parent);
292 l += strlen(suf) + 1;
293 p = pool_alloctmpspace(pool, l + 1) + l;
304 comp = dirpool_compid(&data->dirpool, parent);
305 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
308 strncpy(p, comps, l);
309 parent = dirpool_parent(&data->dirpool, parent);
317 /***************************************************************
321 static inline unsigned char *
322 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
324 Id *keyp = data->schemadata + data->schemata[schema];
325 for (; *keyp; keyp++)
326 dp = data_skip_key(data, dp, data->keys + *keyp);
331 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
333 int nentries, schema;
336 case REPOKEY_TYPE_FIXARRAY:
337 dp = data_read_id(dp, &nentries);
340 dp = data_read_id(dp, &schema);
342 dp = data_skip_schema(data, dp, schema);
344 case REPOKEY_TYPE_FLEXARRAY:
345 dp = data_read_id(dp, &nentries);
348 dp = data_read_id(dp, &schema);
349 dp = data_skip_schema(data, dp, schema);
353 if (key->storage == KEY_STORAGE_INCORE)
354 dp = data_skip(dp, key->type);
355 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
357 dp = data_skip(dp, REPOKEY_TYPE_ID);
358 dp = data_skip(dp, REPOKEY_TYPE_ID);
364 static unsigned char *
365 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
371 if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
374 for (i = 0; (k = *keyp++) != 0; i++)
376 return data->incoredata + data->mainschemaoffsets[i];
379 while ((k = *keyp++) != 0)
383 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
385 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
386 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
389 if (data->keys[k].storage != KEY_STORAGE_INCORE)
391 dp = data_skip_key(data, dp, data->keys + k);
396 static unsigned char *
397 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
402 if (off >= data->lastverticaloffset)
404 off -= data->lastverticaloffset;
405 if (off + len > data->vincorelen)
407 return data->vincore + off;
409 if (off + len > key->size)
411 /* we now have the offset, go into vertical */
412 off += data->verticaloffset[key - data->keys];
413 /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */
414 dp = repopagestore_load_page_range(&data->store, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
416 dp += off % BLOB_PAGESIZE;
420 static inline unsigned char *
421 get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
423 unsigned char *dp = *dpp;
427 if (key->storage == KEY_STORAGE_INCORE)
430 *dpp = data_skip_key(data, dp, key);
433 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
436 dp = data_read_id(dp, &off);
437 dp = data_read_id(dp, &len);
440 return get_vertical_data(data, key, off, len);
446 load_repodata(Repodata *data)
448 if (data->loadcallback)
450 data->loadcallback(data);
451 if (data->state == REPODATA_AVAILABLE)
454 data->state = REPODATA_ERROR;
459 maybe_load_repodata(Repodata *data, Id keyname)
461 if (keyname && !repodata_precheck_keyname(data, keyname))
462 return 0; /* do not bother... */
469 for (i = 0; i < data->nkeys; i++)
470 if (keyname == data->keys[i].name)
472 if (i == data->nkeys)
475 return load_repodata(data);
478 case REPODATA_AVAILABLE:
479 case REPODATA_LOADING:
482 data->state = REPODATA_ERROR;
487 static inline unsigned char *
488 solvid2data(Repodata *data, Id solvid, Id *schemap)
490 unsigned char *dp = data->incoredata;
493 if (solvid == SOLVID_META) /* META */
495 else if (solvid == SOLVID_POS) /* META */
497 Pool *pool = data->repo->pool;
498 if (data->repo != pool->pos.repo)
500 if (data != data->repo->repodata + pool->pos.repodataid)
502 *schemap = pool->pos.schema;
503 return data->incoredata + pool->pos.dp;
507 if (solvid < data->start || solvid >= data->end)
509 dp += data->incoreoffset[solvid - data->start];
511 return data_read_id(dp, schemap);
514 /************************************************************************
518 static inline unsigned char *
519 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
522 Id schema, *keyp, *kp;
525 if (!maybe_load_repodata(data, keyname))
527 dp = solvid2data(data, solvid, &schema);
530 keyp = data->schemadata + data->schemata[schema];
531 for (kp = keyp; *kp; kp++)
532 if (data->keys[*kp].name == keyname)
536 *keypp = key = data->keys + *kp;
537 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
538 return dp; /* no need to forward... */
539 dp = forward_to_key(data, *kp, keyp, dp);
542 return get_data(data, key, &dp, 0);
547 repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
553 dp = find_key_data(data, solvid, keyname, &key);
556 if (key->type == REPOKEY_TYPE_CONSTANTID)
558 if (key->type != REPOKEY_TYPE_ID)
560 dp = data_read_id(dp, &id);
565 repodata_globalize_id(Repodata *data, Id id, int create)
567 if (!id || !data || !data->localpool)
569 return str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
573 repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
579 dp = find_key_data(data, solvid, keyname, &key);
582 if (key->type == REPOKEY_TYPE_STR)
583 return (const char *)dp;
584 if (key->type == REPOKEY_TYPE_CONSTANTID)
585 return id2str(data->repo->pool, key->size);
586 if (key->type == REPOKEY_TYPE_ID)
587 dp = data_read_id(dp, &id);
591 return data->spool.stringspace + data->spool.strings[id];
592 return id2str(data->repo->pool, id);
596 repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned int *value)
603 dp = find_key_data(data, solvid, keyname, &key);
606 if (key->type == REPOKEY_TYPE_NUM
607 || key->type == REPOKEY_TYPE_U32
608 || key->type == REPOKEY_TYPE_CONSTANT)
610 dp = data_fetch(dp, &kv, key);
618 repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
624 if (!maybe_load_repodata(data, keyname))
626 dp = solvid2data(data, solvid, &schema);
629 /* can't use find_key_data as we need to test the type */
630 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
631 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
636 const unsigned char *
637 repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
642 dp = find_key_data(data, solvid, keyname, &key);
650 repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
658 dp = find_key_data(data, solvid, keyname, &key);
663 dp = data_read_ideof(dp, &id, &eof);
671 /************************************************************************
677 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
681 case REPOKEY_TYPE_ID:
682 case REPOKEY_TYPE_CONSTANTID:
683 case REPOKEY_TYPE_IDARRAY:
684 if (data && data->localpool)
685 kv->str = stringpool_id2str(&data->spool, kv->id);
687 kv->str = id2str(pool, kv->id);
688 if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
691 for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
693 if (*s == ':' && s > kv->str)
697 case REPOKEY_TYPE_STR:
699 case REPOKEY_TYPE_DIRSTRARRAY:
700 if (!(flags & SEARCH_FILES))
701 return 1; /* match just the basename */
702 /* Put the full filename into kv->str. */
703 kv->str = repodata_dir2str(data, kv->id, kv->str);
704 /* And to compensate for that put the "empty" directory into
705 kv->id, so that later calls to repodata_dir2str on this data
706 come up with the same filename again. */
709 case REPOKEY_TYPE_MD5:
710 case REPOKEY_TYPE_SHA1:
711 case REPOKEY_TYPE_SHA256:
712 if (!(flags & SEARCH_CHECKSUMS))
713 return 0; /* skip em */
714 kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
722 struct subschema_data {
728 /* search a specific repodata */
730 repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
734 Id keyid, *kp, *keyp;
735 unsigned char *dp, *ddp;
741 if (!maybe_load_repodata(data, keyname))
743 if (solvid == SOLVID_SUBSCHEMA)
745 struct subschema_data *subd = cbdata;
746 cbdata = subd->cbdata;
748 schema = subd->parent->id;
749 dp = (unsigned char *)subd->parent->str;
750 kv.parent = subd->parent;
755 dp = solvid2data(data, solvid, &schema);
758 s = data->repo->pool->solvables + solvid;
761 keyp = data->schemadata + data->schemata[schema];
764 /* search for a specific key */
765 for (kp = keyp; *kp; kp++)
766 if (data->keys[*kp].name == keyname)
770 dp = forward_to_key(data, *kp, keyp, dp);
776 while ((keyid = *keyp++) != 0)
779 key = data->keys + keyid;
780 ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
782 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
784 struct subschema_data subd;
788 subd.cbdata = cbdata;
791 ddp = data_read_id(ddp, &nentries);
795 while (ddp && nentries > 0)
799 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
800 ddp = data_read_id(ddp, &schema);
802 kv.str = (char *)ddp;
803 stop = callback(cbdata, s, data, key, &kv);
804 if (stop > SEARCH_NEXT_KEY)
806 if (stop && stop != SEARCH_ENTERSUB)
808 if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
809 repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
810 ddp = data_skip_schema(data, ddp, schema);
813 if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
817 kv.str = (char *)ddp;
818 stop = callback(cbdata, s, data, key, &kv);
819 if (stop > SEARCH_NEXT_KEY)
829 ddp = data_fetch(ddp, &kv, key);
832 stop = callback(cbdata, s, data, key, &kv);
835 while (!kv.eof && !stop);
836 if (onekey || stop > SEARCH_NEXT_KEY)
842 repodata_setpos_kv(Repodata *data, KeyValue *kv)
844 Pool *pool = data->repo->pool;
846 pool_clear_pos(pool);
849 pool->pos.repo = data->repo;
850 pool->pos.repodataid = data - data->repo->repodata;
851 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
852 pool->pos.schema = kv->id;
856 /************************************************************************
857 * data iterator functions
860 static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = {
861 { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
862 { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
863 { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
864 { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
865 { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
866 { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
867 { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
868 { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
869 { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
870 { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
871 { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
872 { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
873 { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE },
877 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
891 case SOLVABLE_VENDOR:
894 case SOLVABLE_PROVIDES:
896 return s->provides ? s->repo->idarraydata + s->provides : 0;
897 case SOLVABLE_OBSOLETES:
899 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
900 case SOLVABLE_CONFLICTS:
902 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
903 case SOLVABLE_REQUIRES:
905 return s->requires ? s->repo->idarraydata + s->requires : 0;
906 case SOLVABLE_RECOMMENDS:
908 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
909 case SOLVABLE_SUPPLEMENTS:
911 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
912 case SOLVABLE_SUGGESTS:
914 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
915 case SOLVABLE_ENHANCES:
917 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
920 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
927 datamatcher_init(Datamatcher *ma, const char *match, int flags)
933 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
935 ma->matchdata = sat_calloc(1, sizeof(regex_t));
936 ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
939 sat_free(ma->matchdata);
940 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
947 datamatcher_free(Datamatcher *ma)
949 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
951 regfree(ma->matchdata);
952 ma->matchdata = sat_free(ma->matchdata);
957 datamatcher_match(Datamatcher *ma, const char *str)
960 switch ((ma->flags & SEARCH_STRINGMASK))
962 case SEARCH_SUBSTRING:
963 if (ma->flags & SEARCH_NOCASE)
965 if (!strcasestr(str, ma->match))
970 if (!strstr(str, ma->match))
975 if (ma->flags & SEARCH_NOCASE)
977 if (strcasecmp(ma->match, str))
982 if (strcmp(ma->match, str))
986 case SEARCH_STRINGSTART:
987 if (ma->flags & SEARCH_NOCASE)
989 if (strncasecmp(ma->match, str, strlen(ma->match)))
994 if (strncmp(ma->match, str, strlen(ma->match)))
998 case SEARCH_STRINGEND:
999 l = strlen(str) - strlen(ma->match);
1002 if (ma->flags & SEARCH_NOCASE)
1004 if (strcasecmp(ma->match, str + l))
1009 if (strcmp(ma->match, str + l))
1014 if (fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
1018 if (regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0))
1028 repodata_filelistfilter_matches(Repodata *data, const char *str)
1030 /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
1031 /* for now hardcoded */
1032 if (strstr(str, "bin/"))
1034 if (!strncmp(str, "/etc/", 5))
1036 if (!strcmp(str, "/usr/lib/sendmail"))
1058 di_nextarrayelement,
1063 di_nextsolvableattr,
1068 /* see repo.h for documentation */
1070 dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
1072 memset(di, 0, sizeof(*di));
1074 di->flags = flags & ~SEARCH_THISSOLVID;
1075 if (!pool || (repo && repo->pool != pool))
1083 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1089 di->keyname = keyname;
1090 di->keynames[0] = keyname;
1091 dataiterator_set_search(di, repo, p);
1096 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
1099 memset(&di->matcher, 0, sizeof(di->matcher));
1100 if (from->matcher.match)
1101 datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1106 for (i = 1; i < di->nparents; i++)
1107 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1108 di->kv.parent = &di->parents[di->nparents - 1].kv;
1113 dataiterator_set_match(Dataiterator *di, const char *match, int flags)
1115 di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
1116 datamatcher_free(&di->matcher);
1117 memset(&di->matcher, 0, sizeof(di->matcher));
1121 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1131 dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
1135 di->flags &= ~SEARCH_THISSOLVID;
1139 if (!di->pool->nrepos)
1147 di->repo = di->pool->repos[0];
1149 di->state = di_enterrepo;
1151 dataiterator_jump_to_solvid(di, p);
1155 dataiterator_set_keyname(Dataiterator *di, Id keyname)
1158 di->keyname = keyname;
1159 di->keynames[0] = keyname;
1163 dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1167 if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1169 di->state = di_bye; /* sorry */
1172 for (i = di->nkeynames + 1; i > 0; i--)
1173 di->keynames[i] = di->keynames[i - 1];
1174 di->keynames[0] = di->keyname = keyname;
1179 dataiterator_free(Dataiterator *di)
1181 if (di->matcher.match)
1182 datamatcher_free(&di->matcher);
1185 static inline unsigned char *
1186 dataiterator_find_keyname(Dataiterator *di, Id keyname)
1188 Id *keyp = di->keyp;
1189 Repokey *keys = di->data->keys;
1192 for (keyp = di->keyp; *keyp; keyp++)
1193 if (keys[*keyp].name == keyname)
1197 dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1205 dataiterator_filelistcheck(Dataiterator *di)
1208 int needcomplete = 0;
1209 Repodata *data = di->data;
1211 if ((di->matcher.flags & SEARCH_COMPLETE_FILELIST) != 0)
1212 if (!di->matcher.match || (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING || !repodata_filelistfilter_matches(di->data, di->matcher.match))
1214 if (data->state != REPODATA_AVAILABLE)
1215 return needcomplete ? 1 : 0;
1216 for (j = 1; j < data->nkeys; j++)
1217 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1219 return j == data->nkeys && !needcomplete ? 0 : 1;
1223 dataiterator_step(Dataiterator *di)
1231 case di_enterrepo: di_enterrepo:
1234 if (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS))
1236 if (!(di->flags & SEARCH_THISSOLVID))
1238 di->solvid = di->repo->start - 1; /* reset solvid iterator */
1239 goto di_nextsolvable;
1243 case di_entersolvable: di_entersolvable:
1244 if (di->repodataid >= 0)
1246 di->repodataid = 0; /* reset repodata iterator */
1247 if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1249 di->key = solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1251 goto di_entersolvablekey;
1256 case di_enterrepodata: di_enterrepodata:
1257 if (di->repodataid >= 0)
1259 if (di->repodataid >= di->repo->nrepodata)
1260 goto di_nextsolvable;
1261 di->data = di->repo->repodata + di->repodataid;
1263 if (di->repodataid >= 0 && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1264 goto di_nextrepodata;
1265 if (!maybe_load_repodata(di->data, di->keyname))
1266 goto di_nextrepodata;
1267 di->dp = solvid2data(di->data, di->solvid, &schema);
1269 goto di_nextrepodata;
1270 if (di->solvid == SOLVID_POS)
1271 di->solvid = di->pool->pos.solvid;
1272 /* reset key iterator */
1273 di->keyp = di->data->schemadata + di->data->schemata[schema];
1276 case di_enterschema: di_enterschema:
1278 di->dp = dataiterator_find_keyname(di, di->keyname);
1279 if (!di->dp || !*di->keyp)
1283 goto di_nextrepodata;
1287 case di_enterkey: di_enterkey:
1289 di->key = di->data->keys + *di->keyp;
1290 di->ddp = get_data(di->data, di->key, &di->dp, di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0) ? 1 : 0);
1293 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1295 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1301 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1303 di->state = di_nextkey;
1305 di->state = di_nextattr;
1308 case di_nextkey: di_nextkey:
1309 if (!di->keyname && *++di->keyp)
1315 case di_nextrepodata: di_nextrepodata:
1316 if (di->repodataid >= 0 && ++di->repodataid < di->repo->nrepodata)
1317 goto di_enterrepodata;
1320 case di_nextsolvable: di_nextsolvable:
1321 if (!(di->flags & SEARCH_THISSOLVID))
1324 di->solvid = di->repo->start;
1327 for (; di->solvid < di->repo->end; di->solvid++)
1329 if (di->pool->solvables[di->solvid].repo == di->repo)
1330 goto di_entersolvable;
1335 case di_nextrepo: di_nextrepo:
1336 if (di->repoid >= 0)
1340 if (di->repoid < di->pool->nrepos)
1342 di->repo = di->pool->repos[di->repoid];
1348 case di_bye: di_bye:
1352 case di_enterarray: di_enterarray:
1353 if (di->key->name == REPOSITORY_SOLVABLES)
1355 di->ddp = data_read_id(di->ddp, &di->kv.num);
1360 case di_nextarrayelement: di_nextarrayelement:
1363 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1364 if (di->kv.entry == di->kv.num)
1366 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1368 if (!(di->flags & SEARCH_ARRAYSENTINEL))
1370 di->kv.str = (char *)di->ddp;
1372 di->state = di_nextkey;
1375 if (di->kv.entry == di->kv.num - 1)
1377 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1378 di->ddp = data_read_id(di->ddp, &di->kv.id);
1379 di->kv.str = (char *)di->ddp;
1380 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1382 if ((di->flags & SEARCH_SUB) != 0)
1383 di->state = di_entersub;
1385 di->state = di_nextarrayelement;
1388 case di_entersub: di_entersub:
1389 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1390 goto di_nextarrayelement; /* sorry, full */
1391 di->parents[di->nparents].kv = di->kv;
1392 di->parents[di->nparents].dp = di->dp;
1393 di->parents[di->nparents].keyp = di->keyp;
1394 di->dp = (unsigned char *)di->kv.str;
1395 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1396 memset(&di->kv, 0, sizeof(di->kv));
1397 di->kv.parent = &di->parents[di->nparents].kv;
1399 di->keyname = di->keynames[di->nparents - di->rootlevel];
1400 goto di_enterschema;
1402 case di_leavesub: di_leavesub:
1403 if (di->nparents - 1 < di->rootlevel)
1406 di->dp = di->parents[di->nparents].dp;
1407 di->kv = di->parents[di->nparents].kv;
1408 di->keyp = di->parents[di->nparents].keyp;
1409 di->key = di->data->keys + *di->keyp;
1410 di->ddp = (unsigned char *)di->kv.str;
1411 di->keyname = di->keynames[di->nparents - di->rootlevel];
1412 goto di_nextarrayelement;
1414 /* special solvable attr handling follows */
1416 case di_nextsolvableattr:
1417 di->kv.id = *di->idp++;
1422 di->state = di_nextsolvablekey;
1426 case di_nextsolvablekey: di_nextsolvablekey:
1427 if (di->keyname || di->key->name == RPM_RPMDBID)
1428 goto di_enterrepodata;
1432 case di_entersolvablekey: di_entersolvablekey:
1433 di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1434 if (!di->idp || !di->idp[0])
1435 goto di_nextsolvablekey;
1436 di->kv.id = di->idp[0];
1437 di->kv.num = di->idp[0];
1439 if (!di->kv.eof && !di->idp[0])
1443 di->state = di_nextsolvablekey;
1445 di->state = di_nextsolvableattr;
1449 if (di->matcher.match)
1451 /* simple pre-check so that we don't need to stringify */
1452 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && di->matcher.match && (di->matcher.flags & (SEARCH_FILES|SEARCH_NOCASE|SEARCH_STRINGMASK)) == (SEARCH_FILES|SEARCH_STRING))
1454 int l = strlen(di->matcher.match) - strlen(di->kv.str);
1455 if (l < 0 || strcmp(di->matcher.match + l, di->kv.str))
1458 if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
1460 if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
1464 if (!datamatcher_match(&di->matcher, di->kv.str))
1467 /* found something! */
1473 dataiterator_entersub(Dataiterator *di)
1475 if (di->state == di_nextarrayelement)
1476 di->state = di_entersub;
1480 dataiterator_setpos(Dataiterator *di)
1482 if (di->kv.eof == 2)
1484 pool_clear_pos(di->pool);
1487 di->pool->pos.solvid = di->solvid;
1488 di->pool->pos.repo = di->repo;
1489 di->pool->pos.repodataid = di->data - di->repo->repodata;
1490 di->pool->pos.schema = di->kv.id;
1491 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1495 dataiterator_setpos_parent(Dataiterator *di)
1497 if (!di->kv.parent || di->kv.parent->eof == 2)
1499 pool_clear_pos(di->pool);
1502 di->pool->pos.solvid = di->solvid;
1503 di->pool->pos.repo = di->repo;
1504 di->pool->pos.repodataid = di->data - di->repo->repodata;
1505 di->pool->pos.schema = di->kv.parent->id;
1506 di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1509 /* clones just the position, not the search keys/matcher */
1511 dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
1513 di->state = from->state;
1514 di->flags &= ~SEARCH_THISSOLVID;
1515 di->flags |= (from->flags & SEARCH_THISSOLVID);
1516 di->repo = from->repo;
1517 di->data = from->data;
1519 di->ddp = from->ddp;
1520 di->idp = from->idp;
1521 di->keyp = from->keyp;
1522 di->key = from->key;
1524 di->repodataid = from->repodataid;
1525 di->solvid = from->solvid;
1526 di->repoid = from->repoid;
1527 di->rootlevel = from->rootlevel;
1528 memcpy(di->parents, from->parents, sizeof(from->parents));
1529 di->nparents = from->nparents;
1533 for (i = 1; i < di->nparents; i++)
1534 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1535 di->kv.parent = &di->parents[di->nparents - 1].kv;
1540 dataiterator_seek(Dataiterator *di, int whence)
1542 if ((whence & DI_SEEK_STAY) != 0)
1543 di->rootlevel = di->nparents;
1544 switch (whence & ~DI_SEEK_STAY)
1547 if (di->state != di_nextarrayelement)
1549 if ((whence & DI_SEEK_STAY) != 0)
1550 di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1551 di->state = di_entersub;
1553 case DI_SEEK_PARENT:
1560 if (di->rootlevel > di->nparents)
1561 di->rootlevel = di->nparents;
1562 di->dp = di->parents[di->nparents].dp;
1563 di->kv = di->parents[di->nparents].kv;
1564 di->keyp = di->parents[di->nparents].keyp;
1565 di->key = di->data->keys + *di->keyp;
1566 di->ddp = (unsigned char *)di->kv.str;
1567 di->keyname = di->keynames[di->nparents - di->rootlevel];
1568 di->state = di_nextarrayelement;
1570 case DI_SEEK_REWIND:
1576 di->dp = (unsigned char *)di->kv.parent->str;
1577 di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1578 di->state = di_enterschema;
1586 dataiterator_skip_attribute(Dataiterator *di)
1588 if (di->state == di_nextsolvableattr)
1589 di->state = di_nextsolvablekey;
1591 di->state = di_nextkey;
1595 dataiterator_skip_solvable(Dataiterator *di)
1600 di->keyname = di->keynames[0];
1601 di->state = di_nextsolvable;
1605 dataiterator_skip_repo(Dataiterator *di)
1610 di->keyname = di->keynames[0];
1611 di->state = di_nextrepo;
1615 dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1620 di->keyname = di->keynames[0];
1621 if (solvid == SOLVID_POS)
1623 di->repo = di->pool->pos.repo;
1630 di->data = di->repo->repodata + di->pool->pos.repodataid;
1631 di->repodataid = -1;
1632 di->solvid = solvid;
1633 di->state = di_enterrepo;
1634 di->flags |= SEARCH_THISSOLVID;
1639 di->repo = di->pool->solvables[solvid].repo;
1642 else if (di->repoid >= 0)
1644 if (!di->pool->nrepos)
1649 di->repo = di->pool->repos[0];
1653 di->solvid = solvid;
1655 di->flags |= SEARCH_THISSOLVID;
1656 di->state = di_enterrepo;
1660 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1669 di->flags &= ~SEARCH_THISSOLVID;
1670 di->state = di_enterrepo;
1674 dataiterator_match(Dataiterator *di, Datamatcher *ma)
1676 if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
1680 return datamatcher_match(ma, di->kv.str);
1683 /************************************************************************
1684 * data modify functions
1687 /* extend repodata so that it includes solvables p */
1689 repodata_extend(Repodata *data, Id p)
1691 if (data->start == data->end)
1692 data->start = data->end = p;
1695 int old = data->end - data->start;
1696 int new = p - data->end + 1;
1699 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
1700 memset(data->attrs + old, 0, new * sizeof(Id *));
1702 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
1703 memset(data->incoreoffset + old, 0, new * sizeof(Id));
1706 if (p < data->start)
1708 int old = data->end - data->start;
1709 int new = data->start - p;
1712 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
1713 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
1714 memset(data->attrs, 0, new * sizeof(Id *));
1716 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
1717 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
1718 memset(data->incoreoffset, 0, new * sizeof(Id));
1723 /* shrink end of repodata */
1725 repodata_shrink(Repodata *data, int end)
1729 if (data->end <= end)
1731 if (data->start >= end)
1735 for (i = 0; i < data->end - data->start; i++)
1736 sat_free(data->attrs[i]);
1737 data->attrs = sat_free(data->attrs);
1739 data->incoreoffset = sat_free(data->incoreoffset);
1740 data->start = data->end = 0;
1745 for (i = end; i < data->end; i++)
1746 sat_free(data->attrs[i - data->start]);
1747 data->attrs = sat_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
1749 if (data->incoreoffset)
1750 data->incoreoffset = sat_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
1754 /* extend repodata so that it includes solvables from start to start + num - 1 */
1756 repodata_extend_block(Repodata *data, Id start, Id num)
1760 if (!data->incoreoffset)
1762 data->incoreoffset = sat_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
1763 data->start = start;
1764 data->end = start + num;
1767 repodata_extend(data, start);
1769 repodata_extend(data, start + num - 1);
1772 /**********************************************************************/
1775 #define REPODATA_ATTRS_BLOCK 31
1776 #define REPODATA_ATTRDATA_BLOCK 1023
1777 #define REPODATA_ATTRIDDATA_BLOCK 63
1781 repodata_new_handle(Repodata *data)
1785 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1788 data->xattrs = sat_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
1789 data->xattrs[data->nxattrs] = 0;
1790 return -(data->nxattrs++);
1794 repodata_get_attrp(Repodata *data, Id handle)
1796 if (handle == SOLVID_META)
1800 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1805 return data->xattrs - handle;
1806 if (handle < data->start || handle >= data->end)
1807 repodata_extend(data, handle);
1809 data->attrs = sat_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
1810 return data->attrs + (handle - data->start);
1814 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
1820 app = repodata_get_attrp(data, handle);
1825 /* Determine equality based on the name only, allows us to change
1826 type (when overwrite is set), and makes TYPE_CONSTANT work. */
1827 for (pp = ap; *pp; pp += 2)
1828 if (data->keys[*pp].name == data->keys[keyid].name)
1841 ap = sat_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
1851 repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
1855 keyid = repodata_key2id(data, key, 1);
1856 repodata_insert_keyid(data, solvid, keyid, val, 1);
1860 repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
1864 key.type = REPOKEY_TYPE_ID;
1866 key.storage = KEY_STORAGE_INCORE;
1867 repodata_set(data, solvid, &key, id);
1871 repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned int num)
1875 key.type = REPOKEY_TYPE_NUM;
1877 key.storage = KEY_STORAGE_INCORE;
1878 repodata_set(data, solvid, &key, (Id)num);
1882 repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
1886 if (data->localpool)
1887 id = stringpool_str2id(&data->spool, str, 1);
1889 id = str2id(data->repo->pool, str, 1);
1891 key.type = REPOKEY_TYPE_ID;
1893 key.storage = KEY_STORAGE_INCORE;
1894 repodata_set(data, solvid, &key, id);
1898 repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
1902 key.type = REPOKEY_TYPE_CONSTANT;
1903 key.size = constant;
1904 key.storage = KEY_STORAGE_INCORE;
1905 repodata_set(data, solvid, &key, 0);
1909 repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
1913 key.type = REPOKEY_TYPE_CONSTANTID;
1915 key.storage = KEY_STORAGE_INCORE;
1916 repodata_set(data, solvid, &key, 0);
1920 repodata_set_void(Repodata *data, Id solvid, Id keyname)
1924 key.type = REPOKEY_TYPE_VOID;
1926 key.storage = KEY_STORAGE_INCORE;
1927 repodata_set(data, solvid, &key, 0);
1931 repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
1936 l = strlen(str) + 1;
1938 key.type = REPOKEY_TYPE_STR;
1940 key.storage = KEY_STORAGE_INCORE;
1941 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1942 memcpy(data->attrdata + data->attrdatalen, str, l);
1943 repodata_set(data, solvid, &key, data->attrdatalen);
1944 data->attrdatalen += l;
1948 repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len)
1954 key.type = REPOKEY_TYPE_BINARY;
1956 key.storage = KEY_STORAGE_INCORE;
1957 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK);
1958 dp = data->attrdata + data->attrdatalen;
1959 if (len >= (1 << 14))
1961 if (len >= (1 << 28))
1962 *dp++ = (len >> 28) | 128;
1963 if (len >= (1 << 21))
1964 *dp++ = (len >> 21) | 128;
1965 *dp++ = (len >> 14) | 128;
1967 if (len >= (1 << 7))
1968 *dp++ = (len >> 7) | 128;
1971 memcpy(dp, buf, len);
1972 repodata_set(data, solvid, &key, data->attrdatalen);
1973 data->attrdatalen = dp + len - data->attrdata;
1976 /* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
1977 * so that the caller can append the new element there */
1979 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
1982 Id *ida, *pp, **ppp;
1984 /* check if it is the same as last time, this speeds things up a lot */
1985 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
1987 /* great! just append the new data */
1988 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1989 data->attriddatalen--; /* overwrite terminating 0 */
1990 data->lastdatalen += entrysize;
1994 ppp = repodata_get_attrp(data, handle);
1997 for (; *pp; pp += 2)
1998 if (data->keys[*pp].name == keyname && data->keys[*pp].type == keytype)
2002 /* not found. allocate new key */
2007 key.storage = KEY_STORAGE_INCORE;
2008 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2009 repodata_set(data, handle, &key, data->attriddatalen);
2010 data->lasthandle = 0; /* next time... */
2014 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2015 oldsize += entrysize;
2016 if (ida + 1 == data->attriddata + data->attriddatalen)
2018 /* this was the last entry, just append it */
2019 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2020 data->attriddatalen--; /* overwrite terminating 0 */
2024 /* too bad. move to back. */
2025 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2026 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2027 pp[1] = data->attriddatalen;
2028 data->attriddatalen += oldsize;
2030 data->lasthandle = handle;
2031 data->lastkey = *pp;
2032 data->lastdatalen = data->attriddatalen + entrysize + 1;
2036 repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2037 const unsigned char *str)
2042 if (!(l = sat_chksum_len(type)))
2047 key.storage = KEY_STORAGE_INCORE;
2048 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2049 memcpy(data->attrdata + data->attrdatalen, str, l);
2050 repodata_set(data, solvid, &key, data->attrdatalen);
2051 data->attrdatalen += l;
2055 hexstr2bytes(unsigned char *buf, const char *str, int buflen)
2058 for (i = 0; i < buflen; i++)
2060 #define c2h(c) (((c)>='0' && (c)<='9') ? ((c)-'0') \
2061 : ((c)>='a' && (c)<='f') ? ((c)-('a'-10)) \
2062 : ((c)>='A' && (c)<='F') ? ((c)-('A'-10)) \
2073 buf[i] = (buf[i] << 4) | v;
2080 repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2083 unsigned char buf[64];
2086 if (!(l = sat_chksum_len(type)))
2088 if (hexstr2bytes(buf, str, l) != l)
2090 repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2094 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2099 if (!(l = sat_chksum_len(type)))
2101 s = str = pool_alloctmpspace(data->repo->pool, 2 * l + 1);
2102 for (i = 0; i < l; i++)
2104 unsigned char v = buf[i];
2105 unsigned char w = v >> 4;
2106 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2108 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2114 /* rpm filenames don't contain the epoch, so strip it */
2115 static inline const char *
2116 evrid2vrstr(Pool *pool, Id evrid)
2118 const char *p, *evr = id2str(pool, evrid);
2121 for (p = evr; *p >= '0' && *p <= '9'; p++)
2123 return p != evr && *p == ':' ? p + 1 : evr;
2127 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
2129 Pool *pool = data->repo->pool;
2131 const char *str, *fp;
2135 repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2138 if ((dir = strrchr(file, '/')) != 0)
2149 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2154 if (l == 1 && dir[0] == '.')
2156 s = pool->solvables + solvid;
2159 str = id2str(pool, s->arch);
2160 if (!strncmp(dir, str, l) && !str[l])
2161 repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2163 repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir);
2166 char *dir2 = strdup(dir);
2168 repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir2);
2173 str = id2str(pool, s->name);
2175 if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
2178 str = evrid2vrstr(pool, s->evr);
2180 if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
2183 str = id2str(pool, s->arch);
2185 if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2187 repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2192 repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2196 repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2200 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2202 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2203 data->attriddata[data->attriddatalen++] = dir;
2204 data->attriddata[data->attriddatalen++] = num;
2205 data->attriddata[data->attriddatalen++] = num2;
2206 data->attriddata[data->attriddatalen++] = 0;
2210 repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
2216 l = strlen(str) + 1;
2217 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2218 memcpy(data->attrdata + data->attrdatalen, str, l);
2219 stroff = data->attrdatalen;
2220 data->attrdatalen += l;
2223 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2225 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2226 data->attriddata[data->attriddatalen++] = dir;
2227 data->attriddata[data->attriddatalen++] = stroff;
2228 data->attriddata[data->attriddatalen++] = 0;
2232 repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2235 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2237 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2238 data->attriddata[data->attriddatalen++] = id;
2239 data->attriddata[data->attriddatalen++] = 0;
2243 repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2247 if (data->localpool)
2248 id = stringpool_str2id(&data->spool, str, 1);
2250 id = str2id(data->repo->pool, str, 1);
2251 repodata_add_idarray(data, solvid, keyname, id);
2255 repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2257 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2258 data->attriddata[data->attriddatalen++] = ghandle;
2259 data->attriddata[data->attriddatalen++] = 0;
2263 repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2265 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2266 data->attriddata[data->attriddatalen++] = ghandle;
2267 data->attriddata[data->attriddatalen++] = 0;
2271 repodata_delete_uninternalized(Repodata *data, Id solvid, Id keyname)
2274 app = repodata_get_attrp(data, solvid);
2278 for (; *ap; ap += 2)
2279 if (data->keys[*ap].name == keyname)
2285 for (; *ap; ap += 2)
2287 if (data->keys[*ap].name == keyname)
2295 /* add all attrs from src to dest */
2297 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2300 if (dest == src || !(keyp = data->attrs[src - data->start]))
2302 for (; *keyp; keyp += 2)
2303 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2307 repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
2310 if (dest == src || !(keyp = data->attrs[src - data->start]))
2312 for (; *keyp; keyp += 2)
2313 if (!keyidmap || MAPTST(keyidmap, keyp[0]))
2314 repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2319 /**********************************************************************/
2321 /* TODO: unify with repo_write and repo_solv! */
2323 #define EXTDATA_BLOCK 1023
2331 data_addid(struct extdata *xd, Id x)
2335 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2336 dp = xd->buf + xd->len;
2341 *dp++ = (x >> 28) | 128;
2343 *dp++ = (x >> 21) | 128;
2344 *dp++ = (x >> 14) | 128;
2347 *dp++ = (x >> 7) | 128;
2349 xd->len = dp - xd->buf;
2353 data_addideof(struct extdata *xd, Id x, int eof)
2356 x = (x & 63) | ((x & ~63) << 1);
2357 data_addid(xd, (eof ? x : x | 64));
2361 data_addblob(struct extdata *xd, unsigned char *blob, int len)
2363 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
2364 memcpy(xd->buf + xd->len, blob, len);
2368 /*********************************/
2371 repodata_serialize_key(Repodata *data, struct extdata *newincore,
2372 struct extdata *newvincore,
2374 Repokey *key, Id val)
2376 /* Otherwise we have a new value. Parse it into the internal
2380 unsigned int oldvincorelen = 0;
2384 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2387 oldvincorelen = xd->len;
2391 case REPOKEY_TYPE_VOID:
2392 case REPOKEY_TYPE_CONSTANT:
2393 case REPOKEY_TYPE_CONSTANTID:
2395 case REPOKEY_TYPE_STR:
2396 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
2398 case REPOKEY_TYPE_MD5:
2399 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
2401 case REPOKEY_TYPE_SHA1:
2402 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
2404 case REPOKEY_TYPE_SHA256:
2405 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
2407 case REPOKEY_TYPE_ID:
2408 case REPOKEY_TYPE_NUM:
2409 case REPOKEY_TYPE_DIR:
2410 data_addid(xd, val);
2412 case REPOKEY_TYPE_BINARY:
2415 unsigned char *dp = data_read_id(data->attrdata + val, &len);
2417 data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
2420 case REPOKEY_TYPE_IDARRAY:
2421 for (ida = data->attriddata + val; *ida; ida++)
2422 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
2424 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2425 for (ida = data->attriddata + val; *ida; ida += 3)
2427 data_addid(xd, ida[0]);
2428 data_addid(xd, ida[1]);
2429 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
2432 case REPOKEY_TYPE_DIRSTRARRAY:
2433 for (ida = data->attriddata + val; *ida; ida += 2)
2435 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
2436 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
2439 case REPOKEY_TYPE_FIXARRAY:
2443 for (ida = data->attriddata + val; *ida; ida++)
2446 fprintf(stderr, "serialize struct %d\n", *ida);
2449 Id *kp = data->xattrs[-*ida];
2456 fprintf(stderr, " %s:%d\n", id2str(data->repo->pool, data->keys[*kp].name), kp[1]);
2462 schemaid = repodata_schema2id(data, schema, 1);
2463 else if (schemaid != repodata_schema2id(data, schema, 0))
2465 pool_debug(data->repo->pool, SAT_FATAL, "fixarray substructs with different schemas\n");
2469 fprintf(stderr, " schema %d\n", schemaid);
2474 data_addid(xd, num);
2475 data_addid(xd, schemaid);
2476 for (ida = data->attriddata + val; *ida; ida++)
2478 Id *kp = data->xattrs[-*ida];
2483 repodata_serialize_key(data, newincore, newvincore,
2484 schema, data->keys + *kp, kp[1]);
2489 case REPOKEY_TYPE_FLEXARRAY:
2492 for (ida = data->attriddata + val; *ida; ida++)
2494 data_addid(xd, num);
2495 for (ida = data->attriddata + val; *ida; ida++)
2497 Id *kp = data->xattrs[-*ida];
2500 data_addid(xd, 0); /* XXX */
2507 schemaid = repodata_schema2id(data, schema, 1);
2508 data_addid(xd, schemaid);
2509 kp = data->xattrs[-*ida];
2512 repodata_serialize_key(data, newincore, newvincore,
2513 schema, data->keys + *kp, kp[1]);
2519 pool_debug(data->repo->pool, SAT_FATAL, "don't know how to handle type %d\n", key->type);
2522 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2524 /* put offset/len in incore */
2525 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
2526 oldvincorelen = xd->len - oldvincorelen;
2527 data_addid(newincore, oldvincorelen);
2532 repodata_internalize(Repodata *data)
2534 Repokey *key, solvkey;
2536 Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen;
2537 unsigned char *dp, *ndp;
2538 int newschema, oldcount;
2539 struct extdata newincore;
2540 struct extdata newvincore;
2543 if (!data->attrs && !data->xattrs)
2546 newvincore.buf = data->vincore;
2547 newvincore.len = data->vincorelen;
2549 /* find the solvables key, create if needed */
2550 memset(&solvkey, 0, sizeof(solvkey));
2551 solvkey.name = REPOSITORY_SOLVABLES;
2552 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
2554 solvkey.storage = KEY_STORAGE_INCORE;
2555 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
2557 schema = sat_malloc2(data->nkeys, sizeof(Id));
2558 seen = sat_malloc2(data->nkeys, sizeof(Id));
2560 /* Merge the data already existing (in data->schemata, ->incoredata and
2561 friends) with the new attributes in data->attrs[]. */
2562 nentry = data->end - data->start;
2563 memset(&newincore, 0, sizeof(newincore));
2564 data_addid(&newincore, 0); /* start data at offset 1 */
2566 data->mainschema = 0;
2567 data->mainschemaoffsets = sat_free(data->mainschemaoffsets);
2569 /* join entry data */
2570 /* we start with the meta data, entry -1 */
2571 for (entry = -1; entry < nentry; entry++)
2573 memset(seen, 0, data->nkeys * sizeof(Id));
2575 dp = data->incoredata;
2578 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
2579 dp = data_read_id(dp, &oldschema);
2582 fprintf(stderr, "oldschema %d\n", oldschema);
2583 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
2584 fprintf(stderr, "schemadata %p\n", data->schemadata);
2586 /* seen: -1: old data 0: skipped >0: id + 1 */
2590 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
2594 pool_debug(data->repo->pool, SAT_FATAL, "Inconsistent old data (key occured twice).\n");
2602 keyp = data->attrs ? data->attrs[entry] : 0;
2605 /* strip solvables key */
2607 for (sp = keyp = schema; *sp; sp++)
2608 if (*sp != solvkeyid)
2613 seen[solvkeyid] = 0;
2614 keyp = data->xattrs ? data->xattrs[1] : 0;
2617 for (; *keyp; keyp += 2)
2624 seen[*keyp] = keyp[1] + 1;
2626 if (entry < 0 && data->end != data->start)
2633 /* Ideally we'd like to sort the new schema here, to ensure
2634 schema equality independend of the ordering. We can't do that
2635 yet. For once see below (old ids need to come before new ids).
2636 An additional difficulty is that we also need to move
2637 the values with the keys. */
2638 schemaid = repodata_schema2id(data, schema, 1);
2640 schemaid = oldschema;
2643 /* Now create data blob. We walk through the (possibly new) schema
2644 and either copy over old data, or insert the new. */
2645 /* XXX Here we rely on the fact that the (new) schema has the form
2646 o1 o2 o3 o4 ... | n1 n2 n3 ...
2647 (oX being the old keyids (possibly overwritten), and nX being
2648 the new keyids). This rules out sorting the keyids in order
2649 to ensure a small schema count. */
2651 data->incoreoffset[entry] = newincore.len;
2652 data_addid(&newincore, schemaid);
2655 data->mainschema = schemaid;
2656 data->mainschemaoffsets = sat_calloc(sp - schema, sizeof(Id));
2658 keypstart = data->schemadata + data->schemata[schemaid];
2659 for (keyp = keypstart; *keyp; keyp++)
2662 data->mainschemaoffsets[keyp - keypstart] = newincore.len;
2663 if (*keyp == solvkeyid)
2665 /* add flexarray entry count */
2666 data_addid(&newincore, data->end - data->start);
2669 key = data->keys + *keyp;
2671 fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, id2str(data->repo->pool, key->name), id2str(data->repo->pool, key->type));
2676 /* Skip the data associated with this old key. */
2677 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2679 ndp = data_skip(dp, REPOKEY_TYPE_ID);
2680 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
2682 else if (key->storage == KEY_STORAGE_INCORE)
2683 ndp = data_skip_key(data, dp, key);
2686 if (seen[*keyp] == -1)
2688 /* If this key was an old one _and_ was not overwritten with
2689 a different value copy over the old value (we skipped it
2692 data_addblob(&newincore, dp, ndp - dp);
2695 else if (seen[*keyp])
2697 /* Otherwise we have a new value. Parse it into the internal
2699 repodata_serialize_key(data, &newincore, &newvincore,
2700 schema, key, seen[*keyp] - 1);
2704 if (entry >= 0 && data->attrs && data->attrs[entry])
2705 data->attrs[entry] = sat_free(data->attrs[entry]);
2707 /* free all xattrs */
2708 for (entry = 0; entry < data->nxattrs; entry++)
2709 if (data->xattrs[entry])
2710 sat_free(data->xattrs[entry]);
2711 data->xattrs = sat_free(data->xattrs);
2714 data->lasthandle = 0;
2716 data->lastdatalen = 0;
2719 repodata_free_schemahash(data);
2721 sat_free(data->incoredata);
2722 data->incoredata = newincore.buf;
2723 data->incoredatalen = newincore.len;
2724 data->incoredatafree = 0;
2726 sat_free(data->vincore);
2727 data->vincore = newvincore.buf;
2728 data->vincorelen = newvincore.len;
2730 data->attrs = sat_free(data->attrs);
2731 data->attrdata = sat_free(data->attrdata);
2732 data->attriddata = sat_free(data->attriddata);
2733 data->attrdatalen = 0;
2734 data->attriddatalen = 0;
2738 repodata_disable_paging(Repodata *data)
2740 if (maybe_load_repodata(data, 0))
2741 repopagestore_disable_paging(&data->store);
2745 repodata_load_stub(Repodata *data)
2747 Repo *repo = data->repo;
2748 Pool *pool = repo->pool;
2751 if (!pool->loadcallback)
2753 data->state = REPODATA_ERROR;
2756 data->state = REPODATA_LOADING;
2757 r = pool->loadcallback(pool, data, pool->loadcallbackdata);
2759 data->state = REPODATA_ERROR;
2763 repodata_create_stubs(Repodata *data)
2765 Repo *repo = data->repo;
2766 Pool *pool = repo->pool;
2773 int datastart, dataend;
2775 repodataid = data - repo->repodata;
2776 datastart = data->start;
2777 dataend = data->end;
2778 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
2779 while (dataiterator_step(&di))
2781 if (di.data - repo->repodata != repodataid)
2785 dataiterator_free(&di);
2788 stubdataids = sat_calloc(cnt, sizeof(*stubdataids));
2789 for (i = 0; i < cnt; i++)
2791 sdata = repo_add_repodata(repo, 0);
2792 if (dataend > datastart)
2793 repodata_extend_block(sdata, datastart, dataend - datastart);
2794 stubdataids[i] = sdata - repo->repodata;
2795 sdata->state = REPODATA_STUB;
2796 sdata->loadcallback = repodata_load_stub;
2799 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
2801 while (dataiterator_step(&di))
2803 if (di.data - repo->repodata != repodataid)
2805 if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
2807 dataiterator_entersub(&di);
2808 sdata = repo->repodata + stubdataids[i++];
2812 switch (di.key->type)
2814 case REPOKEY_TYPE_ID:
2815 repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
2817 case REPOKEY_TYPE_CONSTANTID:
2818 repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
2820 case REPOKEY_TYPE_STR:
2821 repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
2823 case REPOKEY_TYPE_VOID:
2824 repodata_set_void(sdata, SOLVID_META, di.key->name);
2826 case REPOKEY_TYPE_NUM:
2827 repodata_set_num(sdata, SOLVID_META, di.key->name, di.kv.num);
2829 case REPOKEY_TYPE_MD5:
2830 case REPOKEY_TYPE_SHA1:
2831 case REPOKEY_TYPE_SHA256:
2832 repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
2834 case REPOKEY_TYPE_IDARRAY:
2835 repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
2836 if (di.key->name == REPOSITORY_KEYS)
2843 xkeyname = di.kv.id;
2846 xkey.name = xkeyname;
2847 xkey.type = di.kv.id;
2848 xkey.storage = KEY_STORAGE_INCORE;
2850 repodata_key2id(sdata, &xkey, 1);
2855 dataiterator_free(&di);
2856 for (i = 0; i < cnt; i++)
2857 repodata_internalize(repo->repodata + stubdataids[i]);
2858 sat_free(stubdataids);
2862 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: