2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
13 * a repository can contain multiple repodata entries, consisting of
14 * different sets of keys and different sets of solvables
29 #include "poolid_private.h"
37 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
38 unsigned char *out, unsigned int out_len);
39 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
42 unsigned int out_len);
44 #define REPODATA_BLOCK 255
48 repodata_initdata(Repodata *data, Repo *repo, int localpool)
50 memset(data, 0, sizeof (*data));
52 data->localpool = localpool;
54 stringpool_init_empty(&data->spool);
55 data->keys = sat_calloc(1, sizeof(Repokey));
57 data->schemata = sat_calloc(1, sizeof(Id));
58 data->schemadata = sat_calloc(1, sizeof(Id));
60 data->schemadatalen = 1;
61 repopagestore_init(&data->store);
65 repodata_freedata(Repodata *data)
71 sat_free(data->schemata);
72 sat_free(data->schemadata);
73 sat_free(data->schematahash);
75 stringpool_free(&data->spool);
76 dirpool_free(&data->dirpool);
78 sat_free(data->mainschemaoffsets);
79 sat_free(data->incoredata);
80 sat_free(data->incoreoffset);
81 sat_free(data->verticaloffset);
83 repopagestore_free(&data->store);
85 sat_free(data->vincore);
88 for (i = 0; i < data->end - data->start; i++)
89 sat_free(data->attrs[i]);
90 sat_free(data->attrs);
92 for (i = 0; i < data->nxattrs; i++)
93 sat_free(data->xattrs[i]);
94 sat_free(data->xattrs);
96 sat_free(data->attrdata);
97 sat_free(data->attriddata);
101 repodata_create(Repo *repo, int localpool)
106 repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata, sizeof(*data));
107 data = repo->repodata + repo->nrepodata - 1;
108 repodata_initdata(data, repo, localpool);
113 repodata_free(Repodata *data)
115 Repo *repo = data->repo;
116 int i = data - repo->repodata;
117 repodata_freedata(data);
118 if (i < repo->nrepodata - 1)
119 memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
124 /***************************************************************
125 * key pool management
128 /* this is not so time critical that we need a hash, so we do a simple
131 repodata_key2id(Repodata *data, Repokey *key, int create)
135 for (keyid = 1; keyid < data->nkeys; keyid++)
136 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
138 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
142 if (keyid == data->nkeys)
146 /* allocate new key */
147 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
148 data->keys[data->nkeys++] = *key;
149 if (data->verticaloffset)
151 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
152 data->verticaloffset[data->nkeys - 1] = 0;
154 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
160 /***************************************************************
161 * schema pool management
164 #define SCHEMATA_BLOCK 31
165 #define SCHEMATADATA_BLOCK 255
168 repodata_schema2id(Repodata *data, Id *schema, int create)
174 if ((schematahash = data->schematahash) == 0)
176 data->schematahash = schematahash = sat_calloc(256, sizeof(Id));
177 for (i = 0; i < data->nschemata; i++)
179 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
182 schematahash[h] = i + 1;
184 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
185 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
188 for (sp = schema, len = 0, h = 0; *sp; len++)
193 cid = schematahash[h];
197 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
200 for (cid = 0; cid < data->nschemata; cid++)
201 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
207 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
208 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
210 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
211 data->schemata[data->nschemata] = data->schemadatalen;
212 data->schemadatalen += len;
213 schematahash[h] = data->nschemata + 1;
215 fprintf(stderr, "schema2id: new schema\n");
217 return data->nschemata++;
221 repodata_free_schemahash(Repodata *data)
223 data->schematahash = sat_free(data->schematahash);
225 data->schemata = sat_realloc2(data->schemata, data->nschemata, sizeof(Id));
226 data->schemadata = sat_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
230 /***************************************************************
231 * dir pool management
235 repodata_str2dir(Repodata *data, const char *dir, int create)
241 while (*dir == '/' && dir[1] == '/')
243 if (*dir == '/' && !dir[1])
245 if (data->dirpool.ndirs)
247 return dirpool_add_dir(&data->dirpool, 0, 1, create);
251 dire = strchrnul(dir, '/');
253 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
255 id = strn2id(data->repo->pool, dir, dire - dir, create);
258 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
271 repodata_dir2str(Repodata *data, Id did, const char *suf)
273 Pool *pool = data->repo->pool;
280 return suf ? suf : "";
284 comp = dirpool_compid(&data->dirpool, parent);
285 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
287 parent = dirpool_parent(&data->dirpool, parent);
292 l += strlen(suf) + 1;
293 p = pool_alloctmpspace(pool, l + 1) + l;
304 comp = dirpool_compid(&data->dirpool, parent);
305 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
308 strncpy(p, comps, l);
309 parent = dirpool_parent(&data->dirpool, parent);
317 /***************************************************************
321 static inline unsigned char *
322 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
324 Id *keyp = data->schemadata + data->schemata[schema];
325 for (; *keyp; keyp++)
326 dp = data_skip_key(data, dp, data->keys + *keyp);
331 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
333 int nentries, schema;
336 case REPOKEY_TYPE_FIXARRAY:
337 dp = data_read_id(dp, &nentries);
340 dp = data_read_id(dp, &schema);
342 dp = data_skip_schema(data, dp, schema);
344 case REPOKEY_TYPE_FLEXARRAY:
345 dp = data_read_id(dp, &nentries);
348 dp = data_read_id(dp, &schema);
349 dp = data_skip_schema(data, dp, schema);
353 if (key->storage == KEY_STORAGE_INCORE)
354 dp = data_skip(dp, key->type);
355 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
357 dp = data_skip(dp, REPOKEY_TYPE_ID);
358 dp = data_skip(dp, REPOKEY_TYPE_ID);
364 static unsigned char *
365 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
371 if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
374 for (i = 0; (k = *keyp++) != 0; i++)
376 return data->incoredata + data->mainschemaoffsets[i];
379 while ((k = *keyp++) != 0)
383 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
385 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
386 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
389 if (data->keys[k].storage != KEY_STORAGE_INCORE)
391 dp = data_skip_key(data, dp, data->keys + k);
396 static unsigned char *
397 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
402 if (off >= data->lastverticaloffset)
404 off -= data->lastverticaloffset;
405 if (off + len > data->vincorelen)
407 return data->vincore + off;
409 if (off + len > key->size)
411 /* we now have the offset, go into vertical */
412 off += data->verticaloffset[key - data->keys];
413 /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */
414 dp = repopagestore_load_page_range(&data->store, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
416 dp += off % BLOB_PAGESIZE;
420 static inline unsigned char *
421 get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
423 unsigned char *dp = *dpp;
427 if (key->storage == KEY_STORAGE_INCORE)
430 *dpp = data_skip_key(data, dp, key);
433 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
436 dp = data_read_id(dp, &off);
437 dp = data_read_id(dp, &len);
440 return get_vertical_data(data, key, off, len);
446 load_repodata(Repodata *data)
448 if (data->loadcallback)
450 data->loadcallback(data);
451 if (data->state == REPODATA_AVAILABLE)
454 data->state = REPODATA_ERROR;
459 maybe_load_repodata(Repodata *data, Id keyname)
461 if (keyname && !repodata_precheck_keyname(data, keyname))
462 return 0; /* do not bother... */
469 for (i = 0; i < data->nkeys; i++)
470 if (keyname == data->keys[i].name)
472 if (i == data->nkeys)
475 return load_repodata(data);
478 case REPODATA_AVAILABLE:
479 case REPODATA_LOADING:
482 data->state = REPODATA_ERROR;
487 static inline unsigned char *
488 solvid2data(Repodata *data, Id solvid, Id *schemap)
490 unsigned char *dp = data->incoredata;
493 if (solvid == SOLVID_META) /* META */
495 else if (solvid == SOLVID_POS) /* META */
497 Pool *pool = data->repo->pool;
498 if (data->repo != pool->pos.repo)
500 if (data != data->repo->repodata + pool->pos.repodataid)
502 *schemap = pool->pos.schema;
503 return data->incoredata + pool->pos.dp;
507 if (solvid < data->start || solvid >= data->end)
509 dp += data->incoreoffset[solvid - data->start];
511 return data_read_id(dp, schemap);
514 /************************************************************************
518 static inline unsigned char *
519 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
522 Id schema, *keyp, *kp;
525 if (!maybe_load_repodata(data, keyname))
527 dp = solvid2data(data, solvid, &schema);
530 keyp = data->schemadata + data->schemata[schema];
531 for (kp = keyp; *kp; kp++)
532 if (data->keys[*kp].name == keyname)
536 *keypp = key = data->keys + *kp;
537 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
538 return dp; /* no need to forward... */
539 dp = forward_to_key(data, *kp, keyp, dp);
542 return get_data(data, key, &dp, 0);
547 repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
553 dp = find_key_data(data, solvid, keyname, &key);
556 if (key->type == REPOKEY_TYPE_CONSTANTID)
558 if (key->type != REPOKEY_TYPE_ID)
560 dp = data_read_id(dp, &id);
565 repodata_globalize_id(Repodata *data, Id id, int create)
567 if (!id || !data || !data->localpool)
569 return str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
573 repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
579 dp = find_key_data(data, solvid, keyname, &key);
582 if (key->type == REPOKEY_TYPE_STR)
583 return (const char *)dp;
584 if (key->type == REPOKEY_TYPE_CONSTANTID)
585 return id2str(data->repo->pool, key->size);
586 if (key->type == REPOKEY_TYPE_ID)
587 dp = data_read_id(dp, &id);
591 return data->spool.stringspace + data->spool.strings[id];
592 return id2str(data->repo->pool, id);
596 repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned int *value)
603 dp = find_key_data(data, solvid, keyname, &key);
606 if (key->type == REPOKEY_TYPE_NUM
607 || key->type == REPOKEY_TYPE_U32
608 || key->type == REPOKEY_TYPE_CONSTANT)
610 dp = data_fetch(dp, &kv, key);
618 repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
624 if (!maybe_load_repodata(data, keyname))
626 dp = solvid2data(data, solvid, &schema);
629 /* can't use find_key_data as we need to test the type */
630 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
631 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
636 const unsigned char *
637 repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
642 dp = find_key_data(data, solvid, keyname, &key);
650 repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
658 dp = find_key_data(data, solvid, keyname, &key);
663 dp = data_read_ideof(dp, &id, &eof);
671 /************************************************************************
677 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
681 case REPOKEY_TYPE_ID:
682 case REPOKEY_TYPE_CONSTANTID:
683 case REPOKEY_TYPE_IDARRAY:
684 if (data && data->localpool)
685 kv->str = stringpool_id2str(&data->spool, kv->id);
687 kv->str = id2str(pool, kv->id);
688 if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
691 for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
693 if (*s == ':' && s > kv->str)
697 case REPOKEY_TYPE_STR:
699 case REPOKEY_TYPE_DIRSTRARRAY:
700 if (!(flags & SEARCH_FILES))
701 return 1; /* match just the basename */
702 /* Put the full filename into kv->str. */
703 kv->str = repodata_dir2str(data, kv->id, kv->str);
704 /* And to compensate for that put the "empty" directory into
705 kv->id, so that later calls to repodata_dir2str on this data
706 come up with the same filename again. */
709 case REPOKEY_TYPE_MD5:
710 case REPOKEY_TYPE_SHA1:
711 case REPOKEY_TYPE_SHA256:
712 if (!(flags & SEARCH_CHECKSUMS))
713 return 0; /* skip em */
714 kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
722 struct subschema_data {
728 /* search a specific repodata */
730 repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
734 Id keyid, *kp, *keyp;
735 unsigned char *dp, *ddp;
741 if (!maybe_load_repodata(data, keyname))
743 if (solvid == SOLVID_SUBSCHEMA)
745 struct subschema_data *subd = cbdata;
746 cbdata = subd->cbdata;
748 schema = subd->parent->id;
749 dp = (unsigned char *)subd->parent->str;
750 kv.parent = subd->parent;
755 dp = solvid2data(data, solvid, &schema);
758 s = data->repo->pool->solvables + solvid;
761 keyp = data->schemadata + data->schemata[schema];
764 /* search for a specific key */
765 for (kp = keyp; *kp; kp++)
766 if (data->keys[*kp].name == keyname)
770 dp = forward_to_key(data, *kp, keyp, dp);
776 while ((keyid = *keyp++) != 0)
779 key = data->keys + keyid;
780 ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
782 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
784 struct subschema_data subd;
788 subd.cbdata = cbdata;
791 ddp = data_read_id(ddp, &nentries);
795 while (ddp && nentries > 0)
799 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
800 ddp = data_read_id(ddp, &schema);
802 kv.str = (char *)ddp;
803 stop = callback(cbdata, s, data, key, &kv);
804 if (stop > SEARCH_NEXT_KEY)
806 if (stop && stop != SEARCH_ENTERSUB)
808 if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
809 repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
810 ddp = data_skip_schema(data, ddp, schema);
813 if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
817 kv.str = (char *)ddp;
818 stop = callback(cbdata, s, data, key, &kv);
819 if (stop > SEARCH_NEXT_KEY)
829 ddp = data_fetch(ddp, &kv, key);
832 stop = callback(cbdata, s, data, key, &kv);
835 while (!kv.eof && !stop);
836 if (onekey || stop > SEARCH_NEXT_KEY)
842 repodata_setpos_kv(Repodata *data, KeyValue *kv)
844 Pool *pool = data->repo->pool;
846 pool_clear_pos(pool);
849 pool->pos.repo = data->repo;
850 pool->pos.repodataid = data - data->repo->repodata;
851 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
852 pool->pos.schema = kv->id;
856 /************************************************************************
857 * data iterator functions
860 static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = {
861 { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
862 { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
863 { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
864 { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
865 { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
866 { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
867 { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
868 { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
869 { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
870 { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
871 { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
872 { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
873 { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE },
877 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
891 case SOLVABLE_VENDOR:
894 case SOLVABLE_PROVIDES:
896 return s->provides ? s->repo->idarraydata + s->provides : 0;
897 case SOLVABLE_OBSOLETES:
899 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
900 case SOLVABLE_CONFLICTS:
902 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
903 case SOLVABLE_REQUIRES:
905 return s->requires ? s->repo->idarraydata + s->requires : 0;
906 case SOLVABLE_RECOMMENDS:
908 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
909 case SOLVABLE_SUPPLEMENTS:
911 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
912 case SOLVABLE_SUGGESTS:
914 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
915 case SOLVABLE_ENHANCES:
917 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
920 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
927 datamatcher_init(Datamatcher *ma, const char *match, int flags)
933 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
935 ma->matchdata = sat_calloc(1, sizeof(regex_t));
936 ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
939 sat_free(ma->matchdata);
940 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
947 datamatcher_free(Datamatcher *ma)
949 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
951 regfree(ma->matchdata);
952 ma->matchdata = sat_free(ma->matchdata);
957 datamatcher_match(Datamatcher *ma, const char *str)
960 switch ((ma->flags & SEARCH_STRINGMASK))
962 case SEARCH_SUBSTRING:
963 if (ma->flags & SEARCH_NOCASE)
965 if (!strcasestr(str, ma->match))
970 if (!strstr(str, ma->match))
975 if (ma->flags & SEARCH_NOCASE)
977 if (strcasecmp(ma->match, str))
982 if (strcmp(ma->match, str))
986 case SEARCH_STRINGSTART:
987 if (ma->flags & SEARCH_NOCASE)
989 if (strncasecmp(ma->match, str, strlen(ma->match)))
994 if (strncmp(ma->match, str, strlen(ma->match)))
998 case SEARCH_STRINGEND:
999 l = strlen(str) - strlen(ma->match);
1002 if (ma->flags & SEARCH_NOCASE)
1004 if (strcasecmp(ma->match, str + l))
1009 if (strcmp(ma->match, str + l))
1014 if (fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
1018 if (regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0))
1028 repodata_filelistfilter_matches(Repodata *data, const char *str)
1030 /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
1031 /* for now hardcoded */
1032 if (strstr(str, "bin/"))
1034 if (!strncmp(str, "/etc/", 5))
1036 if (!strcmp(str, "/usr/lib/sendmail"))
1058 di_nextarrayelement,
1063 di_nextsolvableattr,
1068 /* see repo.h for documentation */
1070 dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
1072 memset(di, 0, sizeof(*di));
1074 di->flags = flags & ~SEARCH_THISSOLVID;
1075 if (!pool || (repo && repo->pool != pool))
1083 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1089 di->keyname = keyname;
1090 di->keynames[0] = keyname;
1091 dataiterator_set_search(di, repo, p);
1096 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
1099 memset(&di->matcher, 0, sizeof(di->matcher));
1100 if (from->matcher.match)
1101 datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1106 for (i = 1; i < di->nparents; i++)
1107 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1108 di->kv.parent = &di->parents[di->nparents - 1].kv;
1113 dataiterator_set_match(Dataiterator *di, const char *match, int flags)
1115 di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
1116 datamatcher_free(&di->matcher);
1117 memset(&di->matcher, 0, sizeof(di->matcher));
1121 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1131 dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
1135 di->flags &= ~SEARCH_THISSOLVID;
1139 if (!di->pool->nrepos)
1147 di->repo = di->pool->repos[0];
1149 di->state = di_enterrepo;
1151 dataiterator_jump_to_solvid(di, p);
1155 dataiterator_set_keyname(Dataiterator *di, Id keyname)
1158 di->keyname = keyname;
1159 di->keynames[0] = keyname;
1163 dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1167 if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1169 di->state = di_bye; /* sorry */
1172 for (i = di->nkeynames + 1; i > 0; i--)
1173 di->keynames[i] = di->keynames[i - 1];
1174 di->keynames[0] = di->keyname = keyname;
1179 dataiterator_free(Dataiterator *di)
1181 if (di->matcher.match)
1182 datamatcher_free(&di->matcher);
1185 static inline unsigned char *
1186 dataiterator_find_keyname(Dataiterator *di, Id keyname)
1188 Id *keyp = di->keyp;
1189 Repokey *keys = di->data->keys;
1192 for (keyp = di->keyp; *keyp; keyp++)
1193 if (keys[*keyp].name == keyname)
1197 dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1205 dataiterator_filelistcheck(Dataiterator *di)
1208 int needcomplete = 0;
1209 Repodata *data = di->data;
1211 if ((di->matcher.flags & SEARCH_COMPLETE_FILELIST) != 0)
1212 if (!di->matcher.match || (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING || !repodata_filelistfilter_matches(di->data, di->matcher.match))
1214 if (data->state != REPODATA_AVAILABLE)
1215 return needcomplete ? 1 : 0;
1216 for (j = 1; j < data->nkeys; j++)
1217 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1219 return j == data->nkeys && !needcomplete ? 0 : 1;
1223 dataiterator_step(Dataiterator *di)
1231 case di_enterrepo: di_enterrepo:
1234 if (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS))
1236 if (!(di->flags & SEARCH_THISSOLVID))
1238 di->solvid = di->repo->start - 1; /* reset solvid iterator */
1239 goto di_nextsolvable;
1243 case di_entersolvable: di_entersolvable:
1244 if (di->repodataid >= 0)
1246 di->repodataid = 0; /* reset repodata iterator */
1247 if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1249 di->key = solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1251 goto di_entersolvablekey;
1256 case di_enterrepodata: di_enterrepodata:
1257 if (di->repodataid >= 0)
1259 if (di->repodataid >= di->repo->nrepodata)
1260 goto di_nextsolvable;
1261 di->data = di->repo->repodata + di->repodataid;
1263 if (di->repodataid >= 0 && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1264 goto di_nextrepodata;
1265 if (!maybe_load_repodata(di->data, di->keyname))
1266 goto di_nextrepodata;
1267 di->dp = solvid2data(di->data, di->solvid, &schema);
1269 goto di_nextrepodata;
1270 if (di->solvid == SOLVID_POS)
1271 di->solvid = di->pool->pos.solvid;
1272 /* reset key iterator */
1273 di->keyp = di->data->schemadata + di->data->schemata[schema];
1276 case di_enterschema: di_enterschema:
1278 di->dp = dataiterator_find_keyname(di, di->keyname);
1279 if (!di->dp || !*di->keyp)
1283 goto di_nextrepodata;
1287 case di_enterkey: di_enterkey:
1289 di->key = di->data->keys + *di->keyp;
1290 di->ddp = get_data(di->data, di->key, &di->dp, di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0) ? 1 : 0);
1293 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1295 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1301 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1303 di->state = di_nextkey;
1305 di->state = di_nextattr;
1308 case di_nextkey: di_nextkey:
1309 if (!di->keyname && *++di->keyp)
1315 case di_nextrepodata: di_nextrepodata:
1316 if (di->repodataid >= 0 && ++di->repodataid < di->repo->nrepodata)
1317 goto di_enterrepodata;
1320 case di_nextsolvable: di_nextsolvable:
1321 if (!(di->flags & SEARCH_THISSOLVID))
1324 di->solvid = di->repo->start;
1327 for (; di->solvid < di->repo->end; di->solvid++)
1329 if (di->pool->solvables[di->solvid].repo == di->repo)
1330 goto di_entersolvable;
1335 case di_nextrepo: di_nextrepo:
1336 if (di->repoid >= 0)
1340 if (di->repoid < di->pool->nrepos)
1342 di->repo = di->pool->repos[di->repoid];
1348 case di_bye: di_bye:
1352 case di_enterarray: di_enterarray:
1353 if (di->key->name == REPOSITORY_SOLVABLES)
1355 di->ddp = data_read_id(di->ddp, &di->kv.num);
1360 case di_nextarrayelement: di_nextarrayelement:
1363 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1364 if (di->kv.entry == di->kv.num)
1366 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1368 if (!(di->flags & SEARCH_ARRAYSENTINEL))
1370 di->kv.str = (char *)di->ddp;
1372 di->state = di_nextkey;
1375 if (di->kv.entry == di->kv.num - 1)
1377 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1378 di->ddp = data_read_id(di->ddp, &di->kv.id);
1379 di->kv.str = (char *)di->ddp;
1380 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1382 if ((di->flags & SEARCH_SUB) != 0)
1383 di->state = di_entersub;
1385 di->state = di_nextarrayelement;
1388 case di_entersub: di_entersub:
1389 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1390 goto di_nextarrayelement; /* sorry, full */
1391 di->parents[di->nparents].kv = di->kv;
1392 di->parents[di->nparents].dp = di->dp;
1393 di->parents[di->nparents].keyp = di->keyp;
1394 di->dp = (unsigned char *)di->kv.str;
1395 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1396 memset(&di->kv, 0, sizeof(di->kv));
1397 di->kv.parent = &di->parents[di->nparents].kv;
1399 di->keyname = di->keynames[di->nparents - di->rootlevel];
1400 goto di_enterschema;
1402 case di_leavesub: di_leavesub:
1403 if (di->nparents - 1 < di->rootlevel)
1406 di->dp = di->parents[di->nparents].dp;
1407 di->kv = di->parents[di->nparents].kv;
1408 di->keyp = di->parents[di->nparents].keyp;
1409 di->key = di->data->keys + *di->keyp;
1410 di->ddp = (unsigned char *)di->kv.str;
1411 di->keyname = di->keynames[di->nparents - di->rootlevel];
1412 goto di_nextarrayelement;
1414 /* special solvable attr handling follows */
1416 case di_nextsolvableattr:
1417 di->kv.id = *di->idp++;
1422 di->state = di_nextsolvablekey;
1426 case di_nextsolvablekey: di_nextsolvablekey:
1427 if (di->keyname || di->key->name == RPM_RPMDBID)
1428 goto di_enterrepodata;
1432 case di_entersolvablekey: di_entersolvablekey:
1433 di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1434 if (!di->idp || !di->idp[0])
1435 goto di_nextsolvablekey;
1436 di->kv.id = di->idp[0];
1437 di->kv.num = di->idp[0];
1439 if (!di->kv.eof && !di->idp[0])
1443 di->state = di_nextsolvablekey;
1445 di->state = di_nextsolvableattr;
1449 if (di->matcher.match)
1451 /* simple pre-check so that we don't need to stringify */
1452 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && di->matcher.match && (di->matcher.flags & (SEARCH_FILES|SEARCH_NOCASE|SEARCH_STRINGMASK)) == (SEARCH_FILES|SEARCH_STRING))
1454 int l = strlen(di->matcher.match) - strlen(di->kv.str);
1455 if (l < 0 || strcmp(di->matcher.match + l, di->kv.str))
1458 if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
1460 if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
1464 if (!datamatcher_match(&di->matcher, di->kv.str))
1467 /* found something! */
1473 dataiterator_entersub(Dataiterator *di)
1475 if (di->state == di_nextarrayelement)
1476 di->state = di_entersub;
1480 dataiterator_setpos(Dataiterator *di)
1482 if (di->kv.eof == 2)
1484 pool_clear_pos(di->pool);
1487 di->pool->pos.solvid = di->solvid;
1488 di->pool->pos.repo = di->repo;
1489 di->pool->pos.repodataid = di->data - di->repo->repodata;
1490 di->pool->pos.schema = di->kv.id;
1491 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1495 dataiterator_setpos_parent(Dataiterator *di)
1497 if (!di->kv.parent || di->kv.parent->eof == 2)
1499 pool_clear_pos(di->pool);
1502 di->pool->pos.solvid = di->solvid;
1503 di->pool->pos.repo = di->repo;
1504 di->pool->pos.repodataid = di->data - di->repo->repodata;
1505 di->pool->pos.schema = di->kv.parent->id;
1506 di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1509 /* clones just the position, not the search keys/matcher */
1511 dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
1513 di->state = from->state;
1514 di->flags &= ~SEARCH_THISSOLVID;
1515 di->flags |= (from->flags & SEARCH_THISSOLVID);
1516 di->repo = from->repo;
1517 di->data = from->data;
1519 di->ddp = from->ddp;
1520 di->idp = from->idp;
1521 di->keyp = from->keyp;
1522 di->key = from->key;
1524 di->repodataid = from->repodataid;
1525 di->solvid = from->solvid;
1526 di->repoid = from->repoid;
1527 di->rootlevel = from->rootlevel;
1528 memcpy(di->parents, from->parents, sizeof(from->parents));
1529 di->nparents = from->nparents;
1533 for (i = 1; i < di->nparents; i++)
1534 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1535 di->kv.parent = &di->parents[di->nparents - 1].kv;
1540 dataiterator_seek(Dataiterator *di, int whence)
1542 if ((whence & DI_SEEK_STAY) != 0)
1543 di->rootlevel = di->nparents;
1544 switch (whence & ~DI_SEEK_STAY)
1547 if (di->state != di_nextarrayelement)
1549 if ((whence & DI_SEEK_STAY) != 0)
1550 di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1551 di->state = di_entersub;
1553 case DI_SEEK_PARENT:
1560 if (di->rootlevel > di->nparents)
1561 di->rootlevel = di->nparents;
1562 di->dp = di->parents[di->nparents].dp;
1563 di->kv = di->parents[di->nparents].kv;
1564 di->keyp = di->parents[di->nparents].keyp;
1565 di->key = di->data->keys + *di->keyp;
1566 di->ddp = (unsigned char *)di->kv.str;
1567 di->keyname = di->keynames[di->nparents - di->rootlevel];
1568 di->state = di_nextarrayelement;
1570 case DI_SEEK_REWIND:
1576 di->dp = (unsigned char *)di->kv.parent->str;
1577 di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1578 di->state = di_enterschema;
1586 dataiterator_skip_attribute(Dataiterator *di)
1588 if (di->state == di_nextsolvableattr)
1589 di->state = di_nextsolvablekey;
1591 di->state = di_nextkey;
1595 dataiterator_skip_solvable(Dataiterator *di)
1599 di->keyname = di->keynames[0];
1600 di->state = di_nextsolvable;
1604 dataiterator_skip_repo(Dataiterator *di)
1608 di->keyname = di->keynames[0];
1609 di->state = di_nextrepo;
1613 dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1617 di->keyname = di->keynames[0];
1618 if (solvid == SOLVID_POS)
1620 di->repo = di->pool->pos.repo;
1627 di->data = di->repo->repodata + di->pool->pos.repodataid;
1628 di->repodataid = -1;
1629 di->solvid = solvid;
1630 di->state = di_enterrepo;
1631 di->flags |= SEARCH_THISSOLVID;
1636 di->repo = di->pool->solvables[solvid].repo;
1639 else if (di->repoid >= 0)
1641 if (!di->pool->nrepos)
1646 di->repo = di->pool->repos[0];
1650 di->solvid = solvid;
1652 di->flags |= SEARCH_THISSOLVID;
1653 di->state = di_enterrepo;
1657 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1665 di->flags &= ~SEARCH_THISSOLVID;
1666 di->state = di_enterrepo;
1670 dataiterator_match(Dataiterator *di, Datamatcher *ma)
1672 if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
1676 return datamatcher_match(ma, di->kv.str);
1679 /************************************************************************
1680 * data modify functions
1683 /* extend repodata so that it includes solvables p */
1685 repodata_extend(Repodata *data, Id p)
1687 if (data->start == data->end)
1688 data->start = data->end = p;
1691 int old = data->end - data->start;
1692 int new = p - data->end + 1;
1695 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
1696 memset(data->attrs + old, 0, new * sizeof(Id *));
1698 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
1699 memset(data->incoreoffset + old, 0, new * sizeof(Id));
1702 if (p < data->start)
1704 int old = data->end - data->start;
1705 int new = data->start - p;
1708 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
1709 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
1710 memset(data->attrs, 0, new * sizeof(Id *));
1712 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
1713 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
1714 memset(data->incoreoffset, 0, new * sizeof(Id));
1719 /* shrink end of repodata */
1721 repodata_shrink(Repodata *data, int end)
1725 if (data->end <= end)
1727 if (data->start >= end)
1731 for (i = 0; i < data->end - data->start; i++)
1732 sat_free(data->attrs[i]);
1733 data->attrs = sat_free(data->attrs);
1735 data->incoreoffset = sat_free(data->incoreoffset);
1736 data->start = data->end = 0;
1741 for (i = end; i < data->end; i++)
1742 sat_free(data->attrs[i - data->start]);
1743 data->attrs = sat_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
1745 if (data->incoreoffset)
1746 data->incoreoffset = sat_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
1750 /* extend repodata so that it includes solvables from start to start + num - 1 */
1752 repodata_extend_block(Repodata *data, Id start, Id num)
1756 if (!data->incoreoffset)
1758 data->incoreoffset = sat_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
1759 data->start = start;
1760 data->end = start + num;
1763 repodata_extend(data, start);
1765 repodata_extend(data, start + num - 1);
1768 /**********************************************************************/
1771 #define REPODATA_ATTRS_BLOCK 31
1772 #define REPODATA_ATTRDATA_BLOCK 1023
1773 #define REPODATA_ATTRIDDATA_BLOCK 63
1777 repodata_new_handle(Repodata *data)
1781 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1784 data->xattrs = sat_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
1785 data->xattrs[data->nxattrs] = 0;
1786 return -(data->nxattrs++);
1790 repodata_get_attrp(Repodata *data, Id handle)
1792 if (handle == SOLVID_META)
1796 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1801 return data->xattrs - handle;
1802 if (handle < data->start || handle >= data->end)
1803 repodata_extend(data, handle);
1805 data->attrs = sat_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
1806 return data->attrs + (handle - data->start);
1810 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
1816 app = repodata_get_attrp(data, handle);
1821 /* Determine equality based on the name only, allows us to change
1822 type (when overwrite is set), and makes TYPE_CONSTANT work. */
1823 for (pp = ap; *pp; pp += 2)
1824 if (data->keys[*pp].name == data->keys[keyid].name)
1837 ap = sat_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
1847 repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
1851 keyid = repodata_key2id(data, key, 1);
1852 repodata_insert_keyid(data, solvid, keyid, val, 1);
1856 repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
1860 key.type = REPOKEY_TYPE_ID;
1862 key.storage = KEY_STORAGE_INCORE;
1863 repodata_set(data, solvid, &key, id);
1867 repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned int num)
1871 key.type = REPOKEY_TYPE_NUM;
1873 key.storage = KEY_STORAGE_INCORE;
1874 repodata_set(data, solvid, &key, (Id)num);
1878 repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
1882 if (data->localpool)
1883 id = stringpool_str2id(&data->spool, str, 1);
1885 id = str2id(data->repo->pool, str, 1);
1887 key.type = REPOKEY_TYPE_ID;
1889 key.storage = KEY_STORAGE_INCORE;
1890 repodata_set(data, solvid, &key, id);
1894 repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
1898 key.type = REPOKEY_TYPE_CONSTANT;
1899 key.size = constant;
1900 key.storage = KEY_STORAGE_INCORE;
1901 repodata_set(data, solvid, &key, 0);
1905 repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
1909 key.type = REPOKEY_TYPE_CONSTANTID;
1911 key.storage = KEY_STORAGE_INCORE;
1912 repodata_set(data, solvid, &key, 0);
1916 repodata_set_void(Repodata *data, Id solvid, Id keyname)
1920 key.type = REPOKEY_TYPE_VOID;
1922 key.storage = KEY_STORAGE_INCORE;
1923 repodata_set(data, solvid, &key, 0);
1927 repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
1932 l = strlen(str) + 1;
1934 key.type = REPOKEY_TYPE_STR;
1936 key.storage = KEY_STORAGE_INCORE;
1937 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1938 memcpy(data->attrdata + data->attrdatalen, str, l);
1939 repodata_set(data, solvid, &key, data->attrdatalen);
1940 data->attrdatalen += l;
1944 repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len)
1950 key.type = REPOKEY_TYPE_BINARY;
1952 key.storage = KEY_STORAGE_INCORE;
1953 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK);
1954 dp = data->attrdata + data->attrdatalen;
1955 if (len >= (1 << 14))
1957 if (len >= (1 << 28))
1958 *dp++ = (len >> 28) | 128;
1959 if (len >= (1 << 21))
1960 *dp++ = (len >> 21) | 128;
1961 *dp++ = (len >> 14) | 128;
1963 if (len >= (1 << 7))
1964 *dp++ = (len >> 7) | 128;
1967 memcpy(dp, buf, len);
1968 repodata_set(data, solvid, &key, data->attrdatalen);
1969 data->attrdatalen = dp + len - data->attrdata;
1972 /* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
1973 * so that the caller can append the new element there */
1975 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
1978 Id *ida, *pp, **ppp;
1980 /* check if it is the same as last time, this speeds things up a lot */
1981 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
1983 /* great! just append the new data */
1984 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1985 data->attriddatalen--; /* overwrite terminating 0 */
1986 data->lastdatalen += entrysize;
1990 ppp = repodata_get_attrp(data, handle);
1993 for (; *pp; pp += 2)
1994 if (data->keys[*pp].name == keyname && data->keys[*pp].type == keytype)
1998 /* not found. allocate new key */
2003 key.storage = KEY_STORAGE_INCORE;
2004 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2005 repodata_set(data, handle, &key, data->attriddatalen);
2006 data->lasthandle = 0; /* next time... */
2010 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2011 oldsize += entrysize;
2012 if (ida + 1 == data->attriddata + data->attriddatalen)
2014 /* this was the last entry, just append it */
2015 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2016 data->attriddatalen--; /* overwrite terminating 0 */
2020 /* too bad. move to back. */
2021 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2022 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2023 pp[1] = data->attriddatalen;
2024 data->attriddatalen += oldsize;
2026 data->lasthandle = handle;
2027 data->lastkey = *pp;
2028 data->lastdatalen = data->attriddatalen + entrysize + 1;
2032 repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2033 const unsigned char *str)
2038 if (!(l = sat_chksum_len(type)))
2043 key.storage = KEY_STORAGE_INCORE;
2044 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2045 memcpy(data->attrdata + data->attrdatalen, str, l);
2046 repodata_set(data, solvid, &key, data->attrdatalen);
2047 data->attrdatalen += l;
2051 hexstr2bytes(unsigned char *buf, const char *str, int buflen)
2054 for (i = 0; i < buflen; i++)
2056 #define c2h(c) (((c)>='0' && (c)<='9') ? ((c)-'0') \
2057 : ((c)>='a' && (c)<='f') ? ((c)-('a'-10)) \
2058 : ((c)>='A' && (c)<='F') ? ((c)-('A'-10)) \
2069 buf[i] = (buf[i] << 4) | v;
2076 repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2079 unsigned char buf[64];
2082 if (!(l = sat_chksum_len(type)))
2084 if (hexstr2bytes(buf, str, l) != l)
2086 repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2090 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2095 if (!(l = sat_chksum_len(type)))
2097 s = str = pool_alloctmpspace(data->repo->pool, 2 * l + 1);
2098 for (i = 0; i < l; i++)
2100 unsigned char v = buf[i];
2101 unsigned char w = v >> 4;
2102 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2104 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2110 /* rpm filenames don't contain the epoch, so strip it */
2111 static inline const char *
2112 evrid2vrstr(Pool *pool, Id evrid)
2114 const char *p, *evr = id2str(pool, evrid);
2117 for (p = evr; *p >= '0' && *p <= '9'; p++)
2119 return p != evr && *p == ':' ? p + 1 : evr;
2123 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
2125 Pool *pool = data->repo->pool;
2127 const char *str, *fp;
2131 repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2134 if ((dir = strrchr(file, '/')) != 0)
2145 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2150 if (l == 1 && dir[0] == '.')
2152 s = pool->solvables + solvid;
2155 str = id2str(pool, s->arch);
2156 if (!strncmp(dir, str, l) && !str[l])
2157 repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2159 repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir);
2162 char *dir2 = strdup(dir);
2164 repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir2);
2169 str = id2str(pool, s->name);
2171 if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
2174 str = evrid2vrstr(pool, s->evr);
2176 if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
2179 str = id2str(pool, s->arch);
2181 if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2183 repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2188 repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2192 repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2196 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2198 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2199 data->attriddata[data->attriddatalen++] = dir;
2200 data->attriddata[data->attriddatalen++] = num;
2201 data->attriddata[data->attriddatalen++] = num2;
2202 data->attriddata[data->attriddatalen++] = 0;
2206 repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
2212 l = strlen(str) + 1;
2213 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2214 memcpy(data->attrdata + data->attrdatalen, str, l);
2215 stroff = data->attrdatalen;
2216 data->attrdatalen += l;
2219 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2221 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2222 data->attriddata[data->attriddatalen++] = dir;
2223 data->attriddata[data->attriddatalen++] = stroff;
2224 data->attriddata[data->attriddatalen++] = 0;
2228 repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2231 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2233 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2234 data->attriddata[data->attriddatalen++] = id;
2235 data->attriddata[data->attriddatalen++] = 0;
2239 repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2243 if (data->localpool)
2244 id = stringpool_str2id(&data->spool, str, 1);
2246 id = str2id(data->repo->pool, str, 1);
2247 repodata_add_idarray(data, solvid, keyname, id);
2251 repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2253 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2254 data->attriddata[data->attriddatalen++] = ghandle;
2255 data->attriddata[data->attriddatalen++] = 0;
2259 repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2261 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2262 data->attriddata[data->attriddatalen++] = ghandle;
2263 data->attriddata[data->attriddatalen++] = 0;
2267 repodata_delete_uninternalized(Repodata *data, Id solvid, Id keyname)
2270 app = repodata_get_attrp(data, solvid);
2274 for (; *ap; ap += 2)
2275 if (data->keys[*ap].name == keyname)
2281 for (; *ap; ap += 2)
2283 if (data->keys[*ap].name == keyname)
2291 /* add all attrs from src to dest */
2293 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2296 if (dest == src || !(keyp = data->attrs[src - data->start]))
2298 for (; *keyp; keyp += 2)
2299 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2303 repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
2306 if (dest == src || !(keyp = data->attrs[src - data->start]))
2308 for (; *keyp; keyp += 2)
2309 if (!keyidmap || MAPTST(keyidmap, keyp[0]))
2310 repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2315 /**********************************************************************/
2317 /* TODO: unify with repo_write and repo_solv! */
2319 #define EXTDATA_BLOCK 1023
2327 data_addid(struct extdata *xd, Id x)
2331 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2332 dp = xd->buf + xd->len;
2337 *dp++ = (x >> 28) | 128;
2339 *dp++ = (x >> 21) | 128;
2340 *dp++ = (x >> 14) | 128;
2343 *dp++ = (x >> 7) | 128;
2345 xd->len = dp - xd->buf;
2349 data_addideof(struct extdata *xd, Id x, int eof)
2352 x = (x & 63) | ((x & ~63) << 1);
2353 data_addid(xd, (eof ? x : x | 64));
2357 data_addblob(struct extdata *xd, unsigned char *blob, int len)
2359 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
2360 memcpy(xd->buf + xd->len, blob, len);
2364 /*********************************/
2367 repodata_serialize_key(Repodata *data, struct extdata *newincore,
2368 struct extdata *newvincore,
2370 Repokey *key, Id val)
2372 /* Otherwise we have a new value. Parse it into the internal
2376 unsigned int oldvincorelen = 0;
2380 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2383 oldvincorelen = xd->len;
2387 case REPOKEY_TYPE_VOID:
2388 case REPOKEY_TYPE_CONSTANT:
2389 case REPOKEY_TYPE_CONSTANTID:
2391 case REPOKEY_TYPE_STR:
2392 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
2394 case REPOKEY_TYPE_MD5:
2395 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
2397 case REPOKEY_TYPE_SHA1:
2398 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
2400 case REPOKEY_TYPE_SHA256:
2401 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
2403 case REPOKEY_TYPE_ID:
2404 case REPOKEY_TYPE_NUM:
2405 case REPOKEY_TYPE_DIR:
2406 data_addid(xd, val);
2408 case REPOKEY_TYPE_BINARY:
2411 unsigned char *dp = data_read_id(data->attrdata + val, &len);
2413 data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
2416 case REPOKEY_TYPE_IDARRAY:
2417 for (ida = data->attriddata + val; *ida; ida++)
2418 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
2420 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2421 for (ida = data->attriddata + val; *ida; ida += 3)
2423 data_addid(xd, ida[0]);
2424 data_addid(xd, ida[1]);
2425 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
2428 case REPOKEY_TYPE_DIRSTRARRAY:
2429 for (ida = data->attriddata + val; *ida; ida += 2)
2431 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
2432 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
2435 case REPOKEY_TYPE_FIXARRAY:
2439 for (ida = data->attriddata + val; *ida; ida++)
2442 fprintf(stderr, "serialize struct %d\n", *ida);
2445 Id *kp = data->xattrs[-*ida];
2452 fprintf(stderr, " %s:%d\n", id2str(data->repo->pool, data->keys[*kp].name), kp[1]);
2458 schemaid = repodata_schema2id(data, schema, 1);
2459 else if (schemaid != repodata_schema2id(data, schema, 0))
2461 pool_debug(data->repo->pool, SAT_FATAL, "fixarray substructs with different schemas\n");
2465 fprintf(stderr, " schema %d\n", schemaid);
2470 data_addid(xd, num);
2471 data_addid(xd, schemaid);
2472 for (ida = data->attriddata + val; *ida; ida++)
2474 Id *kp = data->xattrs[-*ida];
2479 repodata_serialize_key(data, newincore, newvincore,
2480 schema, data->keys + *kp, kp[1]);
2485 case REPOKEY_TYPE_FLEXARRAY:
2488 for (ida = data->attriddata + val; *ida; ida++)
2490 data_addid(xd, num);
2491 for (ida = data->attriddata + val; *ida; ida++)
2493 Id *kp = data->xattrs[-*ida];
2496 data_addid(xd, 0); /* XXX */
2503 schemaid = repodata_schema2id(data, schema, 1);
2504 data_addid(xd, schemaid);
2505 kp = data->xattrs[-*ida];
2508 repodata_serialize_key(data, newincore, newvincore,
2509 schema, data->keys + *kp, kp[1]);
2515 pool_debug(data->repo->pool, SAT_FATAL, "don't know how to handle type %d\n", key->type);
2518 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2520 /* put offset/len in incore */
2521 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
2522 oldvincorelen = xd->len - oldvincorelen;
2523 data_addid(newincore, oldvincorelen);
2528 repodata_internalize(Repodata *data)
2530 Repokey *key, solvkey;
2532 Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen;
2533 unsigned char *dp, *ndp;
2534 int newschema, oldcount;
2535 struct extdata newincore;
2536 struct extdata newvincore;
2539 if (!data->attrs && !data->xattrs)
2542 newvincore.buf = data->vincore;
2543 newvincore.len = data->vincorelen;
2545 /* find the solvables key, create if needed */
2546 memset(&solvkey, 0, sizeof(solvkey));
2547 solvkey.name = REPOSITORY_SOLVABLES;
2548 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
2550 solvkey.storage = KEY_STORAGE_INCORE;
2551 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
2553 schema = sat_malloc2(data->nkeys, sizeof(Id));
2554 seen = sat_malloc2(data->nkeys, sizeof(Id));
2556 /* Merge the data already existing (in data->schemata, ->incoredata and
2557 friends) with the new attributes in data->attrs[]. */
2558 nentry = data->end - data->start;
2559 memset(&newincore, 0, sizeof(newincore));
2560 data_addid(&newincore, 0); /* start data at offset 1 */
2562 data->mainschema = 0;
2563 data->mainschemaoffsets = sat_free(data->mainschemaoffsets);
2565 /* join entry data */
2566 /* we start with the meta data, entry -1 */
2567 for (entry = -1; entry < nentry; entry++)
2569 memset(seen, 0, data->nkeys * sizeof(Id));
2571 dp = data->incoredata;
2574 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
2575 dp = data_read_id(dp, &oldschema);
2578 fprintf(stderr, "oldschema %d\n", oldschema);
2579 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
2580 fprintf(stderr, "schemadata %p\n", data->schemadata);
2582 /* seen: -1: old data 0: skipped >0: id + 1 */
2586 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
2590 pool_debug(data->repo->pool, SAT_FATAL, "Inconsistent old data (key occured twice).\n");
2598 keyp = data->attrs ? data->attrs[entry] : 0;
2601 /* strip solvables key */
2603 for (sp = keyp = schema; *sp; sp++)
2604 if (*sp != solvkeyid)
2609 seen[solvkeyid] = 0;
2610 keyp = data->xattrs ? data->xattrs[1] : 0;
2613 for (; *keyp; keyp += 2)
2620 seen[*keyp] = keyp[1] + 1;
2622 if (entry < 0 && data->end != data->start)
2629 /* Ideally we'd like to sort the new schema here, to ensure
2630 schema equality independend of the ordering. We can't do that
2631 yet. For once see below (old ids need to come before new ids).
2632 An additional difficulty is that we also need to move
2633 the values with the keys. */
2634 schemaid = repodata_schema2id(data, schema, 1);
2636 schemaid = oldschema;
2639 /* Now create data blob. We walk through the (possibly new) schema
2640 and either copy over old data, or insert the new. */
2641 /* XXX Here we rely on the fact that the (new) schema has the form
2642 o1 o2 o3 o4 ... | n1 n2 n3 ...
2643 (oX being the old keyids (possibly overwritten), and nX being
2644 the new keyids). This rules out sorting the keyids in order
2645 to ensure a small schema count. */
2647 data->incoreoffset[entry] = newincore.len;
2648 data_addid(&newincore, schemaid);
2651 data->mainschema = schemaid;
2652 data->mainschemaoffsets = sat_calloc(sp - schema, sizeof(Id));
2654 keypstart = data->schemadata + data->schemata[schemaid];
2655 for (keyp = keypstart; *keyp; keyp++)
2658 data->mainschemaoffsets[keyp - keypstart] = newincore.len;
2659 if (*keyp == solvkeyid)
2661 /* add flexarray entry count */
2662 data_addid(&newincore, data->end - data->start);
2665 key = data->keys + *keyp;
2667 fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, id2str(data->repo->pool, key->name), id2str(data->repo->pool, key->type));
2672 /* Skip the data associated with this old key. */
2673 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2675 ndp = data_skip(dp, REPOKEY_TYPE_ID);
2676 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
2678 else if (key->storage == KEY_STORAGE_INCORE)
2679 ndp = data_skip_key(data, dp, key);
2682 if (seen[*keyp] == -1)
2684 /* If this key was an old one _and_ was not overwritten with
2685 a different value copy over the old value (we skipped it
2688 data_addblob(&newincore, dp, ndp - dp);
2691 else if (seen[*keyp])
2693 /* Otherwise we have a new value. Parse it into the internal
2695 repodata_serialize_key(data, &newincore, &newvincore,
2696 schema, key, seen[*keyp] - 1);
2700 if (entry >= 0 && data->attrs && data->attrs[entry])
2701 data->attrs[entry] = sat_free(data->attrs[entry]);
2703 /* free all xattrs */
2704 for (entry = 0; entry < data->nxattrs; entry++)
2705 if (data->xattrs[entry])
2706 sat_free(data->xattrs[entry]);
2707 data->xattrs = sat_free(data->xattrs);
2710 data->lasthandle = 0;
2712 data->lastdatalen = 0;
2715 repodata_free_schemahash(data);
2717 sat_free(data->incoredata);
2718 data->incoredata = newincore.buf;
2719 data->incoredatalen = newincore.len;
2720 data->incoredatafree = 0;
2722 sat_free(data->vincore);
2723 data->vincore = newvincore.buf;
2724 data->vincorelen = newvincore.len;
2726 data->attrs = sat_free(data->attrs);
2727 data->attrdata = sat_free(data->attrdata);
2728 data->attriddata = sat_free(data->attriddata);
2729 data->attrdatalen = 0;
2730 data->attriddatalen = 0;
2734 repodata_disable_paging(Repodata *data)
2736 if (maybe_load_repodata(data, 0))
2737 repopagestore_disable_paging(&data->store);
2741 repodata_load_stub(Repodata *data)
2743 Repo *repo = data->repo;
2744 Pool *pool = repo->pool;
2747 if (!pool->loadcallback)
2749 data->state = REPODATA_ERROR;
2752 data->state = REPODATA_LOADING;
2753 r = pool->loadcallback(pool, data, pool->loadcallbackdata);
2755 data->state = REPODATA_ERROR;
2759 repodata_create_stubs(Repodata *data)
2761 Repo *repo = data->repo;
2762 Pool *pool = repo->pool;
2769 int datastart, dataend;
2771 repodataid = data - repo->repodata;
2772 datastart = data->start;
2773 dataend = data->end;
2774 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
2775 while (dataiterator_step(&di))
2777 if (di.data - repo->repodata != repodataid)
2781 dataiterator_free(&di);
2784 stubdataids = sat_calloc(cnt, sizeof(*stubdataids));
2785 for (i = 0; i < cnt; i++)
2787 sdata = repo_add_repodata(repo, 0);
2788 if (dataend > datastart)
2789 repodata_extend_block(sdata, datastart, dataend - datastart);
2790 stubdataids[i] = sdata - repo->repodata;
2791 sdata->state = REPODATA_STUB;
2792 sdata->loadcallback = repodata_load_stub;
2795 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
2797 while (dataiterator_step(&di))
2799 if (di.data - repo->repodata != repodataid)
2801 if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
2803 dataiterator_entersub(&di);
2804 sdata = repo->repodata + stubdataids[i++];
2808 switch (di.key->type)
2810 case REPOKEY_TYPE_ID:
2811 repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
2813 case REPOKEY_TYPE_CONSTANTID:
2814 repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
2816 case REPOKEY_TYPE_STR:
2817 repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
2819 case REPOKEY_TYPE_VOID:
2820 repodata_set_void(sdata, SOLVID_META, di.key->name);
2822 case REPOKEY_TYPE_NUM:
2823 repodata_set_num(sdata, SOLVID_META, di.key->name, di.kv.num);
2825 case REPOKEY_TYPE_MD5:
2826 case REPOKEY_TYPE_SHA1:
2827 case REPOKEY_TYPE_SHA256:
2828 repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
2830 case REPOKEY_TYPE_IDARRAY:
2831 repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
2832 if (di.key->name == REPOSITORY_KEYS)
2839 xkeyname = di.kv.id;
2842 xkey.name = xkeyname;
2843 xkey.type = di.kv.id;
2844 xkey.storage = KEY_STORAGE_INCORE;
2846 repodata_key2id(sdata, &xkey, 1);
2851 dataiterator_free(&di);
2852 for (i = 0; i < cnt; i++)
2853 repodata_internalize(repo->repodata + stubdataids[i]);
2854 sat_free(stubdataids);
2858 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: