2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
13 * a repository can contain multiple repodata entries, consisting of
14 * different sets of keys and different sets of solvables
29 #include "poolid_private.h"
37 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
38 unsigned char *out, unsigned int out_len);
39 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
42 unsigned int out_len);
44 #define REPODATA_BLOCK 255
48 repodata_initdata(Repodata *data, Repo *repo, int localpool)
50 memset(data, 0, sizeof (*data));
52 data->localpool = localpool;
54 stringpool_init_empty(&data->spool);
55 data->keys = sat_calloc(1, sizeof(Repokey));
57 data->schemata = sat_calloc(1, sizeof(Id));
58 data->schemadata = sat_calloc(1, sizeof(Id));
60 data->schemadatalen = 1;
61 repopagestore_init(&data->store);
65 repodata_freedata(Repodata *data)
71 sat_free(data->schemata);
72 sat_free(data->schemadata);
73 sat_free(data->schematahash);
75 stringpool_free(&data->spool);
76 dirpool_free(&data->dirpool);
78 sat_free(data->mainschemaoffsets);
79 sat_free(data->incoredata);
80 sat_free(data->incoreoffset);
81 sat_free(data->verticaloffset);
83 repopagestore_free(&data->store);
85 sat_free(data->vincore);
88 for (i = 0; i < data->end - data->start; i++)
89 sat_free(data->attrs[i]);
90 sat_free(data->attrs);
92 for (i = 0; i < data->nxattrs; i++)
93 sat_free(data->xattrs[i]);
94 sat_free(data->xattrs);
96 sat_free(data->attrdata);
97 sat_free(data->attriddata);
101 repodata_create(Repo *repo, int localpool)
106 repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata, sizeof(*data));
107 data = repo->repodata + repo->nrepodata - 1;
108 repodata_initdata(data, repo, localpool);
113 repodata_free(Repodata *data)
115 Repo *repo = data->repo;
116 int i = data - repo->repodata;
117 repodata_freedata(data);
118 if (i < repo->nrepodata - 1)
119 memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
124 /***************************************************************
125 * key pool management
128 /* this is not so time critical that we need a hash, so we do a simple
131 repodata_key2id(Repodata *data, Repokey *key, int create)
135 for (keyid = 1; keyid < data->nkeys; keyid++)
136 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
138 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
142 if (keyid == data->nkeys)
146 /* allocate new key */
147 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
148 data->keys[data->nkeys++] = *key;
149 if (data->verticaloffset)
151 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
152 data->verticaloffset[data->nkeys - 1] = 0;
154 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
160 /***************************************************************
161 * schema pool management
164 #define SCHEMATA_BLOCK 31
165 #define SCHEMATADATA_BLOCK 255
168 repodata_schema2id(Repodata *data, Id *schema, int create)
174 if ((schematahash = data->schematahash) == 0)
176 data->schematahash = schematahash = sat_calloc(256, sizeof(Id));
177 for (i = 0; i < data->nschemata; i++)
179 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
182 schematahash[h] = i + 1;
184 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
185 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
188 for (sp = schema, len = 0, h = 0; *sp; len++)
193 cid = schematahash[h];
197 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
200 for (cid = 0; cid < data->nschemata; cid++)
201 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
207 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
208 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
210 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
211 data->schemata[data->nschemata] = data->schemadatalen;
212 data->schemadatalen += len;
213 schematahash[h] = data->nschemata + 1;
215 fprintf(stderr, "schema2id: new schema\n");
217 return data->nschemata++;
221 repodata_free_schemahash(Repodata *data)
223 data->schematahash = sat_free(data->schematahash);
225 data->schemata = sat_realloc2(data->schemata, data->nschemata, sizeof(Id));
226 data->schemadata = sat_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
230 /***************************************************************
231 * dir pool management
235 repodata_str2dir(Repodata *data, const char *dir, int create)
241 while (*dir == '/' && dir[1] == '/')
243 if (*dir == '/' && !dir[1])
245 if (data->dirpool.ndirs)
247 return dirpool_add_dir(&data->dirpool, 0, 1, create);
251 dire = strchrnul(dir, '/');
253 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
255 id = strn2id(data->repo->pool, dir, dire - dir, create);
258 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
271 repodata_dir2str(Repodata *data, Id did, const char *suf)
273 Pool *pool = data->repo->pool;
280 return suf ? suf : "";
284 comp = dirpool_compid(&data->dirpool, parent);
285 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
287 parent = dirpool_parent(&data->dirpool, parent);
292 l += strlen(suf) + 1;
293 p = pool_alloctmpspace(pool, l + 1) + l;
304 comp = dirpool_compid(&data->dirpool, parent);
305 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
308 strncpy(p, comps, l);
309 parent = dirpool_parent(&data->dirpool, parent);
317 /***************************************************************
321 static inline unsigned char *
322 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
324 Id *keyp = data->schemadata + data->schemata[schema];
325 for (; *keyp; keyp++)
326 dp = data_skip_key(data, dp, data->keys + *keyp);
331 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
333 int nentries, schema;
336 case REPOKEY_TYPE_FIXARRAY:
337 dp = data_read_id(dp, &nentries);
340 dp = data_read_id(dp, &schema);
342 dp = data_skip_schema(data, dp, schema);
344 case REPOKEY_TYPE_FLEXARRAY:
345 dp = data_read_id(dp, &nentries);
348 dp = data_read_id(dp, &schema);
349 dp = data_skip_schema(data, dp, schema);
353 if (key->storage == KEY_STORAGE_INCORE)
354 dp = data_skip(dp, key->type);
355 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
357 dp = data_skip(dp, REPOKEY_TYPE_ID);
358 dp = data_skip(dp, REPOKEY_TYPE_ID);
364 static unsigned char *
365 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
371 if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
374 for (i = 0; (k = *keyp++) != 0; i++)
376 return data->incoredata + data->mainschemaoffsets[i];
379 while ((k = *keyp++) != 0)
383 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
385 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
386 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
389 if (data->keys[k].storage != KEY_STORAGE_INCORE)
391 dp = data_skip_key(data, dp, data->keys + k);
396 static unsigned char *
397 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
402 if (off >= data->lastverticaloffset)
404 off -= data->lastverticaloffset;
405 if (off + len > data->vincorelen)
407 return data->vincore + off;
409 if (off + len > key->size)
411 /* we now have the offset, go into vertical */
412 off += data->verticaloffset[key - data->keys];
413 /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */
414 dp = repopagestore_load_page_range(&data->store, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
416 dp += off % BLOB_PAGESIZE;
420 static inline unsigned char *
421 get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
423 unsigned char *dp = *dpp;
427 if (key->storage == KEY_STORAGE_INCORE)
430 *dpp = data_skip_key(data, dp, key);
433 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
436 dp = data_read_id(dp, &off);
437 dp = data_read_id(dp, &len);
440 return get_vertical_data(data, key, off, len);
446 load_repodata(Repodata *data)
448 if (data->loadcallback)
450 data->loadcallback(data);
451 if (data->state == REPODATA_AVAILABLE)
454 data->state = REPODATA_ERROR;
459 maybe_load_repodata(Repodata *data, Id keyname)
461 if (keyname && !repodata_precheck_keyname(data, keyname))
462 return 0; /* do not bother... */
469 for (i = 0; i < data->nkeys; i++)
470 if (keyname == data->keys[i].name)
472 if (i == data->nkeys)
475 return load_repodata(data);
478 case REPODATA_AVAILABLE:
479 case REPODATA_LOADING:
482 data->state = REPODATA_ERROR;
487 static inline unsigned char *
488 solvid2data(Repodata *data, Id solvid, Id *schemap)
490 unsigned char *dp = data->incoredata;
493 if (solvid == SOLVID_META) /* META */
495 else if (solvid == SOLVID_POS) /* META */
497 Pool *pool = data->repo->pool;
498 if (data->repo != pool->pos.repo)
500 if (data != data->repo->repodata + pool->pos.repodataid)
502 *schemap = pool->pos.schema;
503 return data->incoredata + pool->pos.dp;
507 if (solvid < data->start || solvid >= data->end)
509 dp += data->incoreoffset[solvid - data->start];
511 return data_read_id(dp, schemap);
514 /************************************************************************
518 static inline unsigned char *
519 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
522 Id schema, *keyp, *kp;
525 if (!maybe_load_repodata(data, keyname))
527 dp = solvid2data(data, solvid, &schema);
530 keyp = data->schemadata + data->schemata[schema];
531 for (kp = keyp; *kp; kp++)
532 if (data->keys[*kp].name == keyname)
536 *keypp = key = data->keys + *kp;
537 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
538 return dp; /* no need to forward... */
539 dp = forward_to_key(data, *kp, keyp, dp);
542 return get_data(data, key, &dp, 0);
547 repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
553 dp = find_key_data(data, solvid, keyname, &key);
556 if (key->type == REPOKEY_TYPE_CONSTANTID)
558 if (key->type != REPOKEY_TYPE_ID)
560 dp = data_read_id(dp, &id);
565 repodata_globalize_id(Repodata *data, Id id, int create)
567 if (!id || !data || !data->localpool)
569 return str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
573 repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
579 dp = find_key_data(data, solvid, keyname, &key);
582 if (key->type == REPOKEY_TYPE_STR)
583 return (const char *)dp;
584 if (key->type == REPOKEY_TYPE_CONSTANTID)
585 return id2str(data->repo->pool, key->size);
586 if (key->type == REPOKEY_TYPE_ID)
587 dp = data_read_id(dp, &id);
591 return data->spool.stringspace + data->spool.strings[id];
592 return id2str(data->repo->pool, id);
596 repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned int *value)
603 dp = find_key_data(data, solvid, keyname, &key);
606 if (key->type == REPOKEY_TYPE_NUM
607 || key->type == REPOKEY_TYPE_U32
608 || key->type == REPOKEY_TYPE_CONSTANT)
610 dp = data_fetch(dp, &kv, key);
618 repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
624 if (!maybe_load_repodata(data, keyname))
626 dp = solvid2data(data, solvid, &schema);
629 /* can't use find_key_data as we need to test the type */
630 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
631 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
636 const unsigned char *
637 repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
642 dp = find_key_data(data, solvid, keyname, &key);
650 repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
658 dp = find_key_data(data, solvid, keyname, &key);
663 dp = data_read_ideof(dp, &id, &eof);
671 /************************************************************************
677 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
681 case REPOKEY_TYPE_ID:
682 case REPOKEY_TYPE_CONSTANTID:
683 case REPOKEY_TYPE_IDARRAY:
684 if (data && data->localpool)
685 kv->str = stringpool_id2str(&data->spool, kv->id);
687 kv->str = id2str(pool, kv->id);
688 if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
691 for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
693 if (*s == ':' && s > kv->str)
697 case REPOKEY_TYPE_STR:
699 case REPOKEY_TYPE_DIRSTRARRAY:
700 if (!(flags & SEARCH_FILES))
701 return 1; /* match just the basename */
702 /* Put the full filename into kv->str. */
703 kv->str = repodata_dir2str(data, kv->id, kv->str);
704 /* And to compensate for that put the "empty" directory into
705 kv->id, so that later calls to repodata_dir2str on this data
706 come up with the same filename again. */
709 case REPOKEY_TYPE_MD5:
710 case REPOKEY_TYPE_SHA1:
711 case REPOKEY_TYPE_SHA256:
712 if (!(flags & SEARCH_CHECKSUMS))
713 return 0; /* skip em */
714 kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
722 struct subschema_data {
728 /* search a specific repodata */
730 repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
734 Id keyid, *kp, *keyp;
735 unsigned char *dp, *ddp;
741 if (!maybe_load_repodata(data, keyname))
743 if (solvid == SOLVID_SUBSCHEMA)
745 struct subschema_data *subd = cbdata;
746 cbdata = subd->cbdata;
748 schema = subd->parent->id;
749 dp = (unsigned char *)subd->parent->str;
750 kv.parent = subd->parent;
755 dp = solvid2data(data, solvid, &schema);
758 s = data->repo->pool->solvables + solvid;
761 keyp = data->schemadata + data->schemata[schema];
764 /* search for a specific key */
765 for (kp = keyp; *kp; kp++)
766 if (data->keys[*kp].name == keyname)
770 dp = forward_to_key(data, *kp, keyp, dp);
776 while ((keyid = *keyp++) != 0)
779 key = data->keys + keyid;
780 ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
782 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
784 struct subschema_data subd;
788 subd.cbdata = cbdata;
791 ddp = data_read_id(ddp, &nentries);
795 while (ddp && nentries > 0)
799 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
800 ddp = data_read_id(ddp, &schema);
802 kv.str = (char *)ddp;
803 stop = callback(cbdata, s, data, key, &kv);
804 if (stop > SEARCH_NEXT_KEY)
806 if (stop && stop != SEARCH_ENTERSUB)
808 if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
809 repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
810 ddp = data_skip_schema(data, ddp, schema);
813 if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
817 kv.str = (char *)ddp;
818 stop = callback(cbdata, s, data, key, &kv);
819 if (stop > SEARCH_NEXT_KEY)
829 ddp = data_fetch(ddp, &kv, key);
832 stop = callback(cbdata, s, data, key, &kv);
835 while (!kv.eof && !stop);
836 if (onekey || stop > SEARCH_NEXT_KEY)
842 repodata_setpos_kv(Repodata *data, KeyValue *kv)
844 Pool *pool = data->repo->pool;
846 pool_clear_pos(pool);
849 pool->pos.repo = data->repo;
850 pool->pos.repodataid = data - data->repo->repodata;
851 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
852 pool->pos.schema = kv->id;
856 /************************************************************************
857 * data iterator functions
860 static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = {
861 { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
862 { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
863 { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
864 { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
865 { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
866 { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
867 { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
868 { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
869 { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
870 { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
871 { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
872 { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
873 { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE },
877 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
891 case SOLVABLE_VENDOR:
894 case SOLVABLE_PROVIDES:
896 return s->provides ? s->repo->idarraydata + s->provides : 0;
897 case SOLVABLE_OBSOLETES:
899 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
900 case SOLVABLE_CONFLICTS:
902 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
903 case SOLVABLE_REQUIRES:
905 return s->requires ? s->repo->idarraydata + s->requires : 0;
906 case SOLVABLE_RECOMMENDS:
908 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
909 case SOLVABLE_SUPPLEMENTS:
911 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
912 case SOLVABLE_SUGGESTS:
914 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
915 case SOLVABLE_ENHANCES:
917 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
920 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
927 datamatcher_init(Datamatcher *ma, const char *match, int flags)
933 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
935 ma->matchdata = sat_calloc(1, sizeof(regex_t));
936 ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
939 sat_free(ma->matchdata);
940 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
947 datamatcher_free(Datamatcher *ma)
949 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
951 regfree(ma->matchdata);
952 ma->matchdata = sat_free(ma->matchdata);
957 datamatcher_match(Datamatcher *ma, const char *str)
960 switch ((ma->flags & SEARCH_STRINGMASK))
962 case SEARCH_SUBSTRING:
963 if (ma->flags & SEARCH_NOCASE)
965 if (!strcasestr(str, ma->match))
970 if (!strstr(str, ma->match))
975 if (ma->flags & SEARCH_NOCASE)
977 if (strcasecmp(ma->match, str))
982 if (strcmp(ma->match, str))
986 case SEARCH_STRINGSTART:
987 if (ma->flags & SEARCH_NOCASE)
989 if (strncasecmp(ma->match, str, strlen(ma->match)))
994 if (strncmp(ma->match, str, strlen(ma->match)))
998 case SEARCH_STRINGEND:
999 l = strlen(str) - strlen(ma->match);
1002 if (ma->flags & SEARCH_NOCASE)
1004 if (strcasecmp(ma->match, str + l))
1009 if (strcmp(ma->match, str + l))
1014 if (fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
1018 if (regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0))
1028 repodata_filelistfilter_matches(Repodata *data, const char *str)
1030 /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
1031 /* for now hardcoded */
1032 if (strstr(str, "bin/"))
1034 if (!strncmp(str, "/etc/", 5))
1036 if (!strcmp(str, "/usr/lib/sendmail"))
1058 di_nextarrayelement,
1063 di_nextsolvableattr,
1068 /* see repo.h for documentation */
1070 dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
1072 memset(di, 0, sizeof(*di));
1074 di->flags = flags & ~SEARCH_THISSOLVID;
1075 if (!pool || (repo && repo->pool != pool))
1083 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1089 di->keyname = keyname;
1090 di->keynames[0] = keyname;
1091 dataiterator_set_search(di, repo, p);
1096 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
1099 memset(&di->matcher, 0, sizeof(di->matcher));
1100 if (from->matcher.match)
1101 datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1106 for (i = 1; i < di->nparents; i++)
1107 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1108 di->kv.parent = &di->parents[di->nparents - 1].kv;
1113 dataiterator_set_match(Dataiterator *di, const char *match, int flags)
1115 di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
1116 datamatcher_free(&di->matcher);
1117 memset(&di->matcher, 0, sizeof(di->matcher));
1121 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1131 dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
1135 di->flags &= ~SEARCH_THISSOLVID;
1139 if (!di->pool->nrepos)
1147 di->repo = di->pool->repos[0];
1149 di->state = di_enterrepo;
1151 dataiterator_jump_to_solvid(di, p);
1155 dataiterator_set_keyname(Dataiterator *di, Id keyname)
1158 di->keyname = keyname;
1159 di->keynames[0] = keyname;
1163 dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1167 if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1169 di->state = di_bye; /* sorry */
1172 for (i = di->nkeynames + 1; i > 0; i--)
1173 di->keynames[i] = di->keynames[i - 1];
1174 di->keynames[0] = di->keyname = keyname;
1179 dataiterator_free(Dataiterator *di)
1181 if (di->matcher.match)
1182 datamatcher_free(&di->matcher);
1185 static inline unsigned char *
1186 dataiterator_find_keyname(Dataiterator *di, Id keyname)
1188 Id *keyp = di->keyp;
1189 Repokey *keys = di->data->keys;
1192 for (keyp = di->keyp; *keyp; keyp++)
1193 if (keys[*keyp].name == keyname)
1197 dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1205 dataiterator_filelistcheck(Dataiterator *di)
1208 int needcomplete = 0;
1209 Repodata *data = di->data;
1211 if ((di->matcher.flags & SEARCH_COMPLETE_FILELIST) != 0)
1212 if (!di->matcher.match || (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING || !repodata_filelistfilter_matches(di->data, di->matcher.match))
1214 if (data->state != REPODATA_AVAILABLE)
1215 return needcomplete ? 1 : 0;
1216 for (j = 1; j < data->nkeys; j++)
1217 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1219 return j == data->nkeys && !needcomplete ? 0 : 1;
1223 dataiterator_step(Dataiterator *di)
1231 case di_enterrepo: di_enterrepo:
1234 if (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS))
1236 if (!(di->flags & SEARCH_THISSOLVID))
1238 di->solvid = di->repo->start - 1; /* reset solvid iterator */
1239 goto di_nextsolvable;
1243 case di_entersolvable: di_entersolvable:
1244 if (di->repodataid >= 0)
1246 di->repodataid = 0; /* reset repodata iterator */
1247 if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1249 di->key = solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1251 goto di_entersolvablekey;
1256 case di_enterrepodata: di_enterrepodata:
1257 if (di->repodataid >= 0)
1259 if (di->repodataid >= di->repo->nrepodata)
1260 goto di_nextsolvable;
1261 di->data = di->repo->repodata + di->repodataid;
1263 if (di->repodataid >= 0 && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1264 goto di_nextrepodata;
1265 if (!maybe_load_repodata(di->data, di->keyname))
1266 goto di_nextrepodata;
1267 di->dp = solvid2data(di->data, di->solvid, &schema);
1269 goto di_nextrepodata;
1270 if (di->solvid == SOLVID_POS)
1271 di->solvid = di->pool->pos.solvid;
1272 /* reset key iterator */
1273 di->keyp = di->data->schemadata + di->data->schemata[schema];
1276 case di_enterschema: di_enterschema:
1278 di->dp = dataiterator_find_keyname(di, di->keyname);
1279 if (!di->dp || !*di->keyp)
1283 goto di_nextrepodata;
1287 case di_enterkey: di_enterkey:
1289 di->key = di->data->keys + *di->keyp;
1290 di->ddp = get_data(di->data, di->key, &di->dp, di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0) ? 1 : 0);
1293 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1295 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1301 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1303 di->state = di_nextkey;
1305 di->state = di_nextattr;
1308 case di_nextkey: di_nextkey:
1309 if (!di->keyname && *++di->keyp)
1315 case di_nextrepodata: di_nextrepodata:
1316 if (di->repodataid >= 0 && ++di->repodataid < di->repo->nrepodata)
1317 goto di_enterrepodata;
1320 case di_nextsolvable: di_nextsolvable:
1321 if (!(di->flags & SEARCH_THISSOLVID))
1324 di->solvid = di->repo->start;
1327 for (; di->solvid < di->repo->end; di->solvid++)
1329 if (di->pool->solvables[di->solvid].repo == di->repo)
1330 goto di_entersolvable;
1335 case di_nextrepo: di_nextrepo:
1336 if (di->repoid >= 0)
1340 if (di->repoid < di->pool->nrepos)
1342 di->repo = di->pool->repos[di->repoid];
1348 case di_bye: di_bye:
1352 case di_enterarray: di_enterarray:
1353 if (di->key->name == REPOSITORY_SOLVABLES)
1355 di->ddp = data_read_id(di->ddp, &di->kv.num);
1360 case di_nextarrayelement: di_nextarrayelement:
1363 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1364 if (di->kv.entry == di->kv.num)
1366 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1368 if (!(di->flags & SEARCH_ARRAYSENTINEL))
1370 di->kv.str = (char *)di->ddp;
1372 di->state = di_nextkey;
1375 if (di->kv.entry == di->kv.num - 1)
1377 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1378 di->ddp = data_read_id(di->ddp, &di->kv.id);
1379 di->kv.str = (char *)di->ddp;
1380 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1382 if ((di->flags & SEARCH_SUB) != 0)
1383 di->state = di_entersub;
1385 di->state = di_nextarrayelement;
1388 case di_entersub: di_entersub:
1389 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1390 goto di_nextarrayelement; /* sorry, full */
1391 di->parents[di->nparents].kv = di->kv;
1392 di->parents[di->nparents].dp = di->dp;
1393 di->parents[di->nparents].keyp = di->keyp;
1394 di->dp = (unsigned char *)di->kv.str;
1395 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1396 memset(&di->kv, 0, sizeof(di->kv));
1397 di->kv.parent = &di->parents[di->nparents].kv;
1399 di->keyname = di->keynames[di->nparents - di->rootlevel];
1400 goto di_enterschema;
1402 case di_leavesub: di_leavesub:
1403 if (di->nparents - 1 < di->rootlevel)
1406 di->dp = di->parents[di->nparents].dp;
1407 di->kv = di->parents[di->nparents].kv;
1408 di->keyp = di->parents[di->nparents].keyp;
1409 di->key = di->data->keys + *di->keyp;
1410 di->ddp = (unsigned char *)di->kv.str;
1411 di->keyname = di->keynames[di->nparents - di->rootlevel];
1412 goto di_nextarrayelement;
1414 /* special solvable attr handling follows */
1416 case di_nextsolvableattr:
1417 di->kv.id = *di->idp++;
1422 di->state = di_nextsolvablekey;
1426 case di_nextsolvablekey: di_nextsolvablekey:
1427 if (di->keyname || di->key->name == RPM_RPMDBID)
1428 goto di_enterrepodata;
1432 case di_entersolvablekey: di_entersolvablekey:
1433 di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1434 if (!di->idp || !di->idp[0])
1435 goto di_nextsolvablekey;
1436 di->kv.id = di->idp[0];
1437 di->kv.num = di->idp[0];
1439 if (!di->kv.eof && !di->idp[0])
1443 di->state = di_nextsolvablekey;
1445 di->state = di_nextsolvableattr;
1449 if (di->matcher.match)
1451 /* simple pre-check so that we don't need to stringify */
1452 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && di->matcher.match && (di->matcher.flags & (SEARCH_FILES|SEARCH_NOCASE|SEARCH_STRINGMASK)) == (SEARCH_FILES|SEARCH_STRING))
1454 int l = strlen(di->matcher.match) - strlen(di->kv.str);
1455 if (l < 0 || strcmp(di->matcher.match + l, di->kv.str))
1458 if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
1460 if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
1464 if (!datamatcher_match(&di->matcher, di->kv.str))
1467 /* found something! */
1473 dataiterator_entersub(Dataiterator *di)
1475 if (di->state == di_nextarrayelement)
1476 di->state = di_entersub;
1480 dataiterator_setpos(Dataiterator *di)
1482 if (di->kv.eof == 2)
1484 pool_clear_pos(di->pool);
1487 di->pool->pos.solvid = di->solvid;
1488 di->pool->pos.repo = di->repo;
1489 di->pool->pos.repodataid = di->data - di->repo->repodata;
1490 di->pool->pos.schema = di->kv.id;
1491 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1495 dataiterator_setpos_parent(Dataiterator *di)
1497 if (!di->kv.parent || di->kv.parent->eof == 2)
1499 pool_clear_pos(di->pool);
1502 di->pool->pos.solvid = di->solvid;
1503 di->pool->pos.repo = di->repo;
1504 di->pool->pos.repodataid = di->data - di->repo->repodata;
1505 di->pool->pos.schema = di->kv.parent->id;
1506 di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1509 /* clones just the position, not the search keys/matcher */
1511 dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
1513 di->state = from->state;
1514 di->flags &= ~SEARCH_THISSOLVID;
1515 di->flags |= (from->flags & SEARCH_THISSOLVID);
1516 di->repo = from->repo;
1517 di->data = from->data;
1519 di->ddp = from->ddp;
1520 di->idp = from->idp;
1521 di->keyp = from->keyp;
1522 di->key = from->key;
1524 di->repodataid = from->repodataid;
1525 di->solvid = from->solvid;
1526 di->repoid = from->repoid;
1527 di->rootlevel = from->rootlevel;
1528 memcpy(di->parents, from->parents, sizeof(from->parents));
1529 di->nparents = from->nparents;
1533 for (i = 1; i < di->nparents; i++)
1534 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1535 di->kv.parent = &di->parents[di->nparents - 1].kv;
1540 dataiterator_seek(Dataiterator *di, int whence)
1542 if ((whence & DI_SEEK_STAY) != 0)
1543 di->rootlevel = di->nparents;
1544 switch (whence & ~DI_SEEK_STAY)
1547 if (di->state != di_nextarrayelement)
1549 if ((whence & DI_SEEK_STAY) != 0)
1550 di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1551 di->state = di_entersub;
1553 case DI_SEEK_PARENT:
1560 if (di->rootlevel > di->nparents)
1561 di->rootlevel = di->nparents;
1562 di->dp = di->parents[di->nparents].dp;
1563 di->kv = di->parents[di->nparents].kv;
1564 di->keyp = di->parents[di->nparents].keyp;
1565 di->key = di->data->keys + *di->keyp;
1566 di->ddp = (unsigned char *)di->kv.str;
1567 di->keyname = di->keynames[di->nparents - di->rootlevel];
1568 di->state = di_nextarrayelement;
1570 case DI_SEEK_REWIND:
1576 di->dp = (unsigned char *)di->kv.parent->str;
1577 di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1578 di->state = di_enterschema;
1586 dataiterator_skip_attribute(Dataiterator *di)
1588 if (di->state == di_nextsolvableattr)
1589 di->state = di_nextsolvablekey;
1591 di->state = di_nextkey;
1595 dataiterator_skip_solvable(Dataiterator *di)
1599 di->keyname = di->keynames[0];
1600 di->state = di_nextsolvable;
1604 dataiterator_skip_repo(Dataiterator *di)
1608 di->keyname = di->keynames[0];
1609 di->state = di_nextrepo;
1613 dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1617 di->keyname = di->keynames[0];
1618 if (solvid == SOLVID_POS)
1620 di->repo = di->pool->pos.repo;
1627 di->data = di->repo->repodata + di->pool->pos.repodataid;
1628 di->repodataid = -1;
1629 di->solvid = solvid;
1630 di->state = di_enterrepo;
1631 di->flags |= SEARCH_THISSOLVID;
1636 di->repo = di->pool->solvables[solvid].repo;
1639 else if (di->repoid >= 0)
1641 if (!di->pool->nrepos)
1646 di->repo = di->pool->repos[0];
1650 di->solvid = solvid;
1652 di->flags |= SEARCH_THISSOLVID;
1653 di->state = di_enterrepo;
1657 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1665 di->flags &= ~SEARCH_THISSOLVID;
1666 di->state = di_enterrepo;
1670 dataiterator_match(Dataiterator *di, Datamatcher *ma)
1672 if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
1676 return datamatcher_match(ma, di->kv.str);
1679 /************************************************************************
1680 * data modify functions
1683 /* extend repodata so that it includes solvables p */
1685 repodata_extend(Repodata *data, Id p)
1687 if (data->start == data->end)
1688 data->start = data->end = p;
1691 int old = data->end - data->start;
1692 int new = p - data->end + 1;
1695 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
1696 memset(data->attrs + old, 0, new * sizeof(Id *));
1698 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
1699 memset(data->incoreoffset + old, 0, new * sizeof(Id));
1702 if (p < data->start)
1704 int old = data->end - data->start;
1705 int new = data->start - p;
1708 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
1709 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
1710 memset(data->attrs, 0, new * sizeof(Id *));
1712 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
1713 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
1714 memset(data->incoreoffset, 0, new * sizeof(Id));
1719 /* shrink end of repodata */
1721 repodata_shrink(Repodata *data, int end)
1725 if (data->end <= end)
1727 if (data->start >= end)
1731 for (i = 0; i < data->end - data->start; i++)
1732 sat_free(data->attrs[i]);
1733 data->attrs = sat_free(data->attrs);
1735 data->incoreoffset = sat_free(data->incoreoffset);
1736 data->start = data->end = 0;
1741 for (i = end; i < data->end; i++)
1742 sat_free(data->attrs[i - data->start]);
1743 data->attrs = sat_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
1745 if (data->incoreoffset)
1746 data->incoreoffset = sat_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
1750 /* extend repodata so that it includes solvables from start to start + num - 1 */
1752 repodata_extend_block(Repodata *data, Id start, Id num)
1756 if (!data->incoreoffset)
1758 data->incoreoffset = sat_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
1759 data->start = start;
1760 data->end = start + num;
1763 repodata_extend(data, start);
1765 repodata_extend(data, start + num - 1);
1768 /**********************************************************************/
1771 #define REPODATA_ATTRS_BLOCK 31
1772 #define REPODATA_ATTRDATA_BLOCK 1023
1773 #define REPODATA_ATTRIDDATA_BLOCK 63
1777 repodata_new_handle(Repodata *data)
1781 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1784 data->xattrs = sat_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
1785 data->xattrs[data->nxattrs] = 0;
1786 return -(data->nxattrs++);
1790 repodata_get_attrp(Repodata *data, Id handle)
1792 if (handle == SOLVID_META)
1796 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1801 return data->xattrs - handle;
1802 if (handle < data->start || handle >= data->end)
1803 repodata_extend(data, handle);
1805 data->attrs = sat_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
1806 return data->attrs + (handle - data->start);
1810 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
1816 app = repodata_get_attrp(data, handle);
1821 /* Determine equality based on the name only, allows us to change
1822 type (when overwrite is set), and makes TYPE_CONSTANT work. */
1823 for (pp = ap; *pp; pp += 2)
1824 if (data->keys[*pp].name == data->keys[keyid].name)
1837 ap = sat_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
1847 repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
1851 keyid = repodata_key2id(data, key, 1);
1852 repodata_insert_keyid(data, solvid, keyid, val, 1);
1856 repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
1860 key.type = REPOKEY_TYPE_ID;
1862 key.storage = KEY_STORAGE_INCORE;
1863 repodata_set(data, solvid, &key, id);
1867 repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned int num)
1871 key.type = REPOKEY_TYPE_NUM;
1873 key.storage = KEY_STORAGE_INCORE;
1874 repodata_set(data, solvid, &key, (Id)num);
1878 repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
1882 if (data->localpool)
1883 id = stringpool_str2id(&data->spool, str, 1);
1885 id = str2id(data->repo->pool, str, 1);
1887 key.type = REPOKEY_TYPE_ID;
1889 key.storage = KEY_STORAGE_INCORE;
1890 repodata_set(data, solvid, &key, id);
1894 repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
1898 key.type = REPOKEY_TYPE_CONSTANT;
1899 key.size = constant;
1900 key.storage = KEY_STORAGE_INCORE;
1901 repodata_set(data, solvid, &key, 0);
1905 repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
1909 key.type = REPOKEY_TYPE_CONSTANTID;
1911 key.storage = KEY_STORAGE_INCORE;
1912 repodata_set(data, solvid, &key, 0);
1916 repodata_set_void(Repodata *data, Id solvid, Id keyname)
1920 key.type = REPOKEY_TYPE_VOID;
1922 key.storage = KEY_STORAGE_INCORE;
1923 repodata_set(data, solvid, &key, 0);
1927 repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
1932 l = strlen(str) + 1;
1934 key.type = REPOKEY_TYPE_STR;
1936 key.storage = KEY_STORAGE_INCORE;
1937 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1938 memcpy(data->attrdata + data->attrdatalen, str, l);
1939 repodata_set(data, solvid, &key, data->attrdatalen);
1940 data->attrdatalen += l;
1944 repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len)
1950 key.type = REPOKEY_TYPE_BINARY;
1952 key.storage = KEY_STORAGE_INCORE;
1953 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK);
1954 dp = data->attrdata + data->attrdatalen;
1955 if (len >= (1 << 14))
1957 if (len >= (1 << 28))
1958 *dp++ = (len >> 28) | 128;
1959 if (len >= (1 << 21))
1960 *dp++ = (len >> 21) | 128;
1961 *dp++ = (len >> 14) | 128;
1963 if (len >= (1 << 7))
1964 *dp++ = (len >> 7) | 128;
1967 memcpy(dp, buf, len);
1968 repodata_set(data, solvid, &key, data->attrdatalen);
1969 data->attrdatalen = dp + len - data->attrdata;
1972 /* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
1973 * so that the caller can append the new element there */
1975 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
1978 Id *ida, *pp, **ppp;
1980 /* check if it is the same as last time, this speeds things up a lot */
1981 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
1983 /* great! just append the new data */
1984 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
1985 data->attriddatalen--; /* overwrite terminating 0 */
1986 data->lastdatalen += entrysize;
1990 ppp = repodata_get_attrp(data, handle);
1993 for (; *pp; pp += 2)
1994 if (data->keys[*pp].name == keyname && data->keys[*pp].type == keytype)
1998 /* not found. allocate new key */
2003 key.storage = KEY_STORAGE_INCORE;
2004 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2005 repodata_set(data, handle, &key, data->attriddatalen);
2006 data->lasthandle = 0; /* next time... */
2010 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2011 oldsize += entrysize;
2012 if (ida + 1 == data->attriddata + data->attriddatalen)
2014 /* this was the last entry, just append it */
2015 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2016 data->attriddatalen--; /* overwrite terminating 0 */
2020 /* too bad. move to back. */
2021 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2022 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2023 pp[1] = data->attriddatalen;
2024 data->attriddatalen += oldsize;
2026 data->lasthandle = handle;
2027 data->lastkey = *pp;
2028 data->lastdatalen = data->attriddatalen + entrysize + 1;
2032 repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2033 const unsigned char *str)
2038 if (!(l = sat_chksum_len(type)))
2043 key.storage = KEY_STORAGE_INCORE;
2044 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2045 memcpy(data->attrdata + data->attrdatalen, str, l);
2046 repodata_set(data, solvid, &key, data->attrdatalen);
2047 data->attrdatalen += l;
2051 hexstr2bytes(unsigned char *buf, const char *str, int buflen)
2054 for (i = 0; i < buflen; i++)
2056 #define c2h(c) (((c)>='0' && (c)<='9') ? ((c)-'0') \
2057 : ((c)>='a' && (c)<='f') ? ((c)-('a'-10)) \
2058 : ((c)>='A' && (c)<='F') ? ((c)-('A'-10)) \
2069 buf[i] = (buf[i] << 4) | v;
2076 repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2079 unsigned char buf[64];
2082 if (!(l = sat_chksum_len(type)))
2084 if (hexstr2bytes(buf, str, l) != l)
2086 repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2090 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2095 if (!(l = sat_chksum_len(type)))
2097 s = str = pool_alloctmpspace(data->repo->pool, 2 * l + 1);
2098 for (i = 0; i < l; i++)
2100 unsigned char v = buf[i];
2101 unsigned char w = v >> 4;
2102 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2104 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2110 /* rpm filenames don't contain the epoch, so strip it */
2111 static inline const char *
2112 evrid2vrstr(Pool *pool, Id evrid)
2114 const char *p, *evr = id2str(pool, evrid);
2117 for (p = evr; *p >= '0' && *p <= '9'; p++)
2119 return p != evr && *p == ':' ? p + 1 : evr;
2123 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
2125 Pool *pool = data->repo->pool;
2127 const char *str, *fp;
2131 repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2134 if ((dir = strrchr(file, '/')) != 0)
2145 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2150 if (l == 1 && dir[0] == '.')
2152 s = pool->solvables + solvid;
2155 str = id2str(pool, s->arch);
2156 if (!strncmp(dir, str, l) && !str[l])
2157 repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2159 repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir);
2162 char *dir2 = strdup(dir);
2164 repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir2);
2169 str = id2str(pool, s->name);
2171 if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
2174 str = evrid2vrstr(pool, s->evr);
2176 if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
2179 str = id2str(pool, s->arch);
2181 if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2183 repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2188 repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2192 repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2196 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2198 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2199 data->attriddata[data->attriddatalen++] = dir;
2200 data->attriddata[data->attriddatalen++] = num;
2201 data->attriddata[data->attriddatalen++] = num2;
2202 data->attriddata[data->attriddatalen++] = 0;
2206 repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
2212 l = strlen(str) + 1;
2213 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2214 memcpy(data->attrdata + data->attrdatalen, str, l);
2215 stroff = data->attrdatalen;
2216 data->attrdatalen += l;
2219 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2221 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2222 data->attriddata[data->attriddatalen++] = dir;
2223 data->attriddata[data->attriddatalen++] = stroff;
2224 data->attriddata[data->attriddatalen++] = 0;
2228 repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2231 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2233 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2234 data->attriddata[data->attriddatalen++] = id;
2235 data->attriddata[data->attriddatalen++] = 0;
2239 repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2243 if (data->localpool)
2244 id = stringpool_str2id(&data->spool, str, 1);
2246 id = str2id(data->repo->pool, str, 1);
2247 repodata_add_idarray(data, solvid, keyname, id);
2251 repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2253 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2254 data->attriddata[data->attriddatalen++] = ghandle;
2255 data->attriddata[data->attriddatalen++] = 0;
2259 repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2261 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2262 data->attriddata[data->attriddatalen++] = ghandle;
2263 data->attriddata[data->attriddatalen++] = 0;
2266 /* add all attrs from src to dest */
2268 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2271 if (dest == src || !(keyp = data->attrs[src - data->start]))
2273 for (; *keyp; keyp += 2)
2274 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2278 repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
2281 if (dest == src || !(keyp = data->attrs[src - data->start]))
2283 for (; *keyp; keyp += 2)
2284 if (!keyidmap || MAPTST(keyidmap, keyp[0]))
2285 repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2290 /**********************************************************************/
2292 /* TODO: unify with repo_write and repo_solv! */
2294 #define EXTDATA_BLOCK 1023
2302 data_addid(struct extdata *xd, Id x)
2306 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2307 dp = xd->buf + xd->len;
2312 *dp++ = (x >> 28) | 128;
2314 *dp++ = (x >> 21) | 128;
2315 *dp++ = (x >> 14) | 128;
2318 *dp++ = (x >> 7) | 128;
2320 xd->len = dp - xd->buf;
2324 data_addideof(struct extdata *xd, Id x, int eof)
2327 x = (x & 63) | ((x & ~63) << 1);
2328 data_addid(xd, (eof ? x : x | 64));
2332 data_addblob(struct extdata *xd, unsigned char *blob, int len)
2334 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
2335 memcpy(xd->buf + xd->len, blob, len);
2339 /*********************************/
2342 repodata_serialize_key(Repodata *data, struct extdata *newincore,
2343 struct extdata *newvincore,
2345 Repokey *key, Id val)
2347 /* Otherwise we have a new value. Parse it into the internal
2351 unsigned int oldvincorelen = 0;
2355 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2358 oldvincorelen = xd->len;
2362 case REPOKEY_TYPE_VOID:
2363 case REPOKEY_TYPE_CONSTANT:
2364 case REPOKEY_TYPE_CONSTANTID:
2366 case REPOKEY_TYPE_STR:
2367 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
2369 case REPOKEY_TYPE_MD5:
2370 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
2372 case REPOKEY_TYPE_SHA1:
2373 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
2375 case REPOKEY_TYPE_SHA256:
2376 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
2378 case REPOKEY_TYPE_ID:
2379 case REPOKEY_TYPE_NUM:
2380 case REPOKEY_TYPE_DIR:
2381 data_addid(xd, val);
2383 case REPOKEY_TYPE_BINARY:
2386 unsigned char *dp = data_read_id(data->attrdata + val, &len);
2388 data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
2391 case REPOKEY_TYPE_IDARRAY:
2392 for (ida = data->attriddata + val; *ida; ida++)
2393 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
2395 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2396 for (ida = data->attriddata + val; *ida; ida += 3)
2398 data_addid(xd, ida[0]);
2399 data_addid(xd, ida[1]);
2400 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
2403 case REPOKEY_TYPE_DIRSTRARRAY:
2404 for (ida = data->attriddata + val; *ida; ida += 2)
2406 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
2407 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
2410 case REPOKEY_TYPE_FIXARRAY:
2414 for (ida = data->attriddata + val; *ida; ida++)
2417 fprintf(stderr, "serialize struct %d\n", *ida);
2420 Id *kp = data->xattrs[-*ida];
2427 fprintf(stderr, " %s:%d\n", id2str(data->repo->pool, data->keys[*kp].name), kp[1]);
2433 schemaid = repodata_schema2id(data, schema, 1);
2434 else if (schemaid != repodata_schema2id(data, schema, 0))
2436 pool_debug(data->repo->pool, SAT_FATAL, "fixarray substructs with different schemas\n");
2440 fprintf(stderr, " schema %d\n", schemaid);
2445 data_addid(xd, num);
2446 data_addid(xd, schemaid);
2447 for (ida = data->attriddata + val; *ida; ida++)
2449 Id *kp = data->xattrs[-*ida];
2454 repodata_serialize_key(data, newincore, newvincore,
2455 schema, data->keys + *kp, kp[1]);
2460 case REPOKEY_TYPE_FLEXARRAY:
2463 for (ida = data->attriddata + val; *ida; ida++)
2465 data_addid(xd, num);
2466 for (ida = data->attriddata + val; *ida; ida++)
2468 Id *kp = data->xattrs[-*ida];
2471 data_addid(xd, 0); /* XXX */
2478 schemaid = repodata_schema2id(data, schema, 1);
2479 data_addid(xd, schemaid);
2480 kp = data->xattrs[-*ida];
2483 repodata_serialize_key(data, newincore, newvincore,
2484 schema, data->keys + *kp, kp[1]);
2490 pool_debug(data->repo->pool, SAT_FATAL, "don't know how to handle type %d\n", key->type);
2493 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2495 /* put offset/len in incore */
2496 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
2497 oldvincorelen = xd->len - oldvincorelen;
2498 data_addid(newincore, oldvincorelen);
2503 repodata_internalize(Repodata *data)
2505 Repokey *key, solvkey;
2507 Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen;
2508 unsigned char *dp, *ndp;
2509 int newschema, oldcount;
2510 struct extdata newincore;
2511 struct extdata newvincore;
2514 if (!data->attrs && !data->xattrs)
2517 newvincore.buf = data->vincore;
2518 newvincore.len = data->vincorelen;
2520 /* find the solvables key, create if needed */
2521 memset(&solvkey, 0, sizeof(solvkey));
2522 solvkey.name = REPOSITORY_SOLVABLES;
2523 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
2525 solvkey.storage = KEY_STORAGE_INCORE;
2526 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
2528 schema = sat_malloc2(data->nkeys, sizeof(Id));
2529 seen = sat_malloc2(data->nkeys, sizeof(Id));
2531 /* Merge the data already existing (in data->schemata, ->incoredata and
2532 friends) with the new attributes in data->attrs[]. */
2533 nentry = data->end - data->start;
2534 memset(&newincore, 0, sizeof(newincore));
2535 data_addid(&newincore, 0); /* start data at offset 1 */
2537 data->mainschema = 0;
2538 data->mainschemaoffsets = sat_free(data->mainschemaoffsets);
2540 /* join entry data */
2541 /* we start with the meta data, entry -1 */
2542 for (entry = -1; entry < nentry; entry++)
2544 memset(seen, 0, data->nkeys * sizeof(Id));
2546 dp = data->incoredata;
2549 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
2550 dp = data_read_id(dp, &oldschema);
2553 fprintf(stderr, "oldschema %d\n", oldschema);
2554 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
2555 fprintf(stderr, "schemadata %p\n", data->schemadata);
2557 /* seen: -1: old data 0: skipped >0: id + 1 */
2561 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
2565 pool_debug(data->repo->pool, SAT_FATAL, "Inconsistent old data (key occured twice).\n");
2573 keyp = data->attrs ? data->attrs[entry] : 0;
2576 /* strip solvables key */
2578 for (sp = keyp = schema; *sp; sp++)
2579 if (*sp != solvkeyid)
2584 seen[solvkeyid] = 0;
2585 keyp = data->xattrs ? data->xattrs[1] : 0;
2588 for (; *keyp; keyp += 2)
2595 seen[*keyp] = keyp[1] + 1;
2597 if (entry < 0 && data->end != data->start)
2604 /* Ideally we'd like to sort the new schema here, to ensure
2605 schema equality independend of the ordering. We can't do that
2606 yet. For once see below (old ids need to come before new ids).
2607 An additional difficulty is that we also need to move
2608 the values with the keys. */
2609 schemaid = repodata_schema2id(data, schema, 1);
2611 schemaid = oldschema;
2614 /* Now create data blob. We walk through the (possibly new) schema
2615 and either copy over old data, or insert the new. */
2616 /* XXX Here we rely on the fact that the (new) schema has the form
2617 o1 o2 o3 o4 ... | n1 n2 n3 ...
2618 (oX being the old keyids (possibly overwritten), and nX being
2619 the new keyids). This rules out sorting the keyids in order
2620 to ensure a small schema count. */
2622 data->incoreoffset[entry] = newincore.len;
2623 data_addid(&newincore, schemaid);
2626 data->mainschema = schemaid;
2627 data->mainschemaoffsets = sat_calloc(sp - schema, sizeof(Id));
2629 keypstart = data->schemadata + data->schemata[schemaid];
2630 for (keyp = keypstart; *keyp; keyp++)
2633 data->mainschemaoffsets[keyp - keypstart] = newincore.len;
2634 if (*keyp == solvkeyid)
2636 /* add flexarray entry count */
2637 data_addid(&newincore, data->end - data->start);
2640 key = data->keys + *keyp;
2642 fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, id2str(data->repo->pool, key->name), id2str(data->repo->pool, key->type));
2647 /* Skip the data associated with this old key. */
2648 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2650 ndp = data_skip(dp, REPOKEY_TYPE_ID);
2651 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
2653 else if (key->storage == KEY_STORAGE_INCORE)
2654 ndp = data_skip_key(data, dp, key);
2657 if (seen[*keyp] == -1)
2659 /* If this key was an old one _and_ was not overwritten with
2660 a different value copy over the old value (we skipped it
2663 data_addblob(&newincore, dp, ndp - dp);
2666 else if (seen[*keyp])
2668 /* Otherwise we have a new value. Parse it into the internal
2670 repodata_serialize_key(data, &newincore, &newvincore,
2671 schema, key, seen[*keyp] - 1);
2675 if (entry >= 0 && data->attrs && data->attrs[entry])
2676 data->attrs[entry] = sat_free(data->attrs[entry]);
2678 /* free all xattrs */
2679 for (entry = 0; entry < data->nxattrs; entry++)
2680 if (data->xattrs[entry])
2681 sat_free(data->xattrs[entry]);
2682 data->xattrs = sat_free(data->xattrs);
2685 data->lasthandle = 0;
2687 data->lastdatalen = 0;
2690 repodata_free_schemahash(data);
2692 sat_free(data->incoredata);
2693 data->incoredata = newincore.buf;
2694 data->incoredatalen = newincore.len;
2695 data->incoredatafree = 0;
2697 sat_free(data->vincore);
2698 data->vincore = newvincore.buf;
2699 data->vincorelen = newvincore.len;
2701 data->attrs = sat_free(data->attrs);
2702 data->attrdata = sat_free(data->attrdata);
2703 data->attriddata = sat_free(data->attriddata);
2704 data->attrdatalen = 0;
2705 data->attriddatalen = 0;
2709 repodata_disable_paging(Repodata *data)
2711 if (maybe_load_repodata(data, 0))
2712 repopagestore_disable_paging(&data->store);
2716 repodata_load_stub(Repodata *data)
2718 Repo *repo = data->repo;
2719 Pool *pool = repo->pool;
2722 if (!pool->loadcallback)
2724 data->state = REPODATA_ERROR;
2727 data->state = REPODATA_LOADING;
2728 r = pool->loadcallback(pool, data, pool->loadcallbackdata);
2730 data->state = REPODATA_ERROR;
2734 repodata_create_stubs(Repodata *data)
2736 Repo *repo = data->repo;
2737 Pool *pool = repo->pool;
2744 int datastart, dataend;
2746 repodataid = data - repo->repodata;
2747 datastart = data->start;
2748 dataend = data->end;
2749 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
2750 while (dataiterator_step(&di))
2752 if (di.data - repo->repodata != repodataid)
2756 dataiterator_free(&di);
2759 stubdataids = sat_calloc(cnt, sizeof(*stubdataids));
2760 for (i = 0; i < cnt; i++)
2762 sdata = repo_add_repodata(repo, 0);
2763 if (dataend > datastart)
2764 repodata_extend_block(sdata, datastart, dataend - datastart);
2765 stubdataids[i] = sdata - repo->repodata;
2766 sdata->state = REPODATA_STUB;
2767 sdata->loadcallback = repodata_load_stub;
2770 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
2772 while (dataiterator_step(&di))
2774 if (di.data - repo->repodata != repodataid)
2776 if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
2778 dataiterator_entersub(&di);
2779 sdata = repo->repodata + stubdataids[i++];
2783 switch (di.key->type)
2785 case REPOKEY_TYPE_ID:
2786 repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
2788 case REPOKEY_TYPE_CONSTANTID:
2789 repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
2791 case REPOKEY_TYPE_STR:
2792 repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
2794 case REPOKEY_TYPE_VOID:
2795 repodata_set_void(sdata, SOLVID_META, di.key->name);
2797 case REPOKEY_TYPE_NUM:
2798 repodata_set_num(sdata, SOLVID_META, di.key->name, di.kv.num);
2800 case REPOKEY_TYPE_MD5:
2801 case REPOKEY_TYPE_SHA1:
2802 case REPOKEY_TYPE_SHA256:
2803 repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
2805 case REPOKEY_TYPE_IDARRAY:
2806 repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
2807 if (di.key->name == REPOSITORY_KEYS)
2814 xkeyname = di.kv.id;
2817 xkey.name = xkeyname;
2818 xkey.type = di.kv.id;
2819 xkey.storage = KEY_STORAGE_INCORE;
2821 repodata_key2id(sdata, &xkey, 1);
2826 dataiterator_free(&di);
2827 for (i = 0; i < cnt; i++)
2828 repodata_internalize(repo->repodata + stubdataids[i]);
2829 sat_free(stubdataids);
2833 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: