2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
13 * a repository can contain multiple repodata entries, consisting of
14 * different sets of keys and different sets of solvables
29 #include "poolid_private.h"
37 extern unsigned int compress_buf (const unsigned char *in, unsigned int in_len,
38 unsigned char *out, unsigned int out_len);
39 extern unsigned int unchecked_decompress_buf (const unsigned char *in,
42 unsigned int out_len);
44 #define REPODATA_BLOCK 255
48 repodata_initdata(Repodata *data, Repo *repo, int localpool)
50 memset(data, 0, sizeof (*data));
52 data->localpool = localpool;
54 stringpool_init_empty(&data->spool);
55 data->keys = sat_calloc(1, sizeof(Repokey));
57 data->schemata = sat_calloc(1, sizeof(Id));
58 data->schemadata = sat_calloc(1, sizeof(Id));
60 data->schemadatalen = 1;
61 repopagestore_init(&data->store);
65 repodata_freedata(Repodata *data)
71 sat_free(data->schemata);
72 sat_free(data->schemadata);
73 sat_free(data->schematahash);
75 stringpool_free(&data->spool);
76 dirpool_free(&data->dirpool);
78 sat_free(data->mainschemaoffsets);
79 sat_free(data->incoredata);
80 sat_free(data->incoreoffset);
81 sat_free(data->verticaloffset);
83 repopagestore_free(&data->store);
85 sat_free(data->vincore);
88 for (i = 0; i < data->end - data->start; i++)
89 sat_free(data->attrs[i]);
90 sat_free(data->attrs);
92 for (i = 0; i < data->nxattrs; i++)
93 sat_free(data->xattrs[i]);
94 sat_free(data->xattrs);
96 sat_free(data->attrdata);
97 sat_free(data->attriddata);
101 repodata_create(Repo *repo, int localpool)
106 repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata, sizeof(*data));
107 data = repo->repodata + repo->nrepodata - 1;
108 repodata_initdata(data, repo, localpool);
113 repodata_free(Repodata *data)
115 Repo *repo = data->repo;
116 int i = data - repo->repodata;
117 repodata_freedata(data);
118 if (i < repo->nrepodata - 1)
119 memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
124 /***************************************************************
125 * key pool management
128 /* this is not so time critical that we need a hash, so we do a simple
131 repodata_key2id(Repodata *data, Repokey *key, int create)
135 for (keyid = 1; keyid < data->nkeys; keyid++)
136 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
138 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
142 if (keyid == data->nkeys)
146 /* allocate new key */
147 data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
148 data->keys[data->nkeys++] = *key;
149 if (data->verticaloffset)
151 data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
152 data->verticaloffset[data->nkeys - 1] = 0;
154 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
160 /***************************************************************
161 * schema pool management
164 #define SCHEMATA_BLOCK 31
165 #define SCHEMATADATA_BLOCK 255
168 repodata_schema2id(Repodata *data, Id *schema, int create)
174 if ((schematahash = data->schematahash) == 0)
176 data->schematahash = schematahash = sat_calloc(256, sizeof(Id));
177 for (i = 0; i < data->nschemata; i++)
179 for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
182 schematahash[h] = i + 1;
184 data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
185 data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
188 for (sp = schema, len = 0, h = 0; *sp; len++)
193 cid = schematahash[h];
197 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
200 for (cid = 0; cid < data->nschemata; cid++)
201 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
207 data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
208 data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
210 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
211 data->schemata[data->nschemata] = data->schemadatalen;
212 data->schemadatalen += len;
213 schematahash[h] = data->nschemata + 1;
215 fprintf(stderr, "schema2id: new schema\n");
217 return data->nschemata++;
221 repodata_free_schemahash(Repodata *data)
223 data->schematahash = sat_free(data->schematahash);
225 data->schemata = sat_realloc2(data->schemata, data->nschemata, sizeof(Id));
226 data->schemadata = sat_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
230 /***************************************************************
231 * dir pool management
234 #ifndef HAVE_STRCHRNUL
235 static inline const char *strchrnul(const char *str, char x)
237 const char *p = strchr(str, x);
238 return p ? p : str + strlen(str);
243 repodata_str2dir(Repodata *data, const char *dir, int create)
249 while (*dir == '/' && dir[1] == '/')
251 if (*dir == '/' && !dir[1])
253 if (data->dirpool.ndirs)
255 return dirpool_add_dir(&data->dirpool, 0, 1, create);
259 dire = strchrnul(dir, '/');
261 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
263 id = strn2id(data->repo->pool, dir, dire - dir, create);
266 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
279 repodata_dir2str(Repodata *data, Id did, const char *suf)
281 Pool *pool = data->repo->pool;
288 return suf ? suf : "";
292 comp = dirpool_compid(&data->dirpool, parent);
293 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
295 parent = dirpool_parent(&data->dirpool, parent);
300 l += strlen(suf) + 1;
301 p = pool_alloctmpspace(pool, l + 1) + l;
312 comp = dirpool_compid(&data->dirpool, parent);
313 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
316 strncpy(p, comps, l);
317 parent = dirpool_parent(&data->dirpool, parent);
325 /***************************************************************
329 static inline unsigned char *
330 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
332 Id *keyp = data->schemadata + data->schemata[schema];
333 for (; *keyp; keyp++)
334 dp = data_skip_key(data, dp, data->keys + *keyp);
339 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
341 int nentries, schema;
344 case REPOKEY_TYPE_FIXARRAY:
345 dp = data_read_id(dp, &nentries);
348 dp = data_read_id(dp, &schema);
350 dp = data_skip_schema(data, dp, schema);
352 case REPOKEY_TYPE_FLEXARRAY:
353 dp = data_read_id(dp, &nentries);
356 dp = data_read_id(dp, &schema);
357 dp = data_skip_schema(data, dp, schema);
361 if (key->storage == KEY_STORAGE_INCORE)
362 dp = data_skip(dp, key->type);
363 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
365 dp = data_skip(dp, REPOKEY_TYPE_ID);
366 dp = data_skip(dp, REPOKEY_TYPE_ID);
372 static unsigned char *
373 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
379 if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
382 for (i = 0; (k = *keyp++) != 0; i++)
384 return data->incoredata + data->mainschemaoffsets[i];
387 while ((k = *keyp++) != 0)
391 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
393 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
394 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
397 if (data->keys[k].storage != KEY_STORAGE_INCORE)
399 dp = data_skip_key(data, dp, data->keys + k);
404 static unsigned char *
405 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
410 if (off >= data->lastverticaloffset)
412 off -= data->lastverticaloffset;
413 if (off + len > data->vincorelen)
415 return data->vincore + off;
417 if (off + len > key->size)
419 /* we now have the offset, go into vertical */
420 off += data->verticaloffset[key - data->keys];
421 /* fprintf(stderr, "key %d page %d\n", key->name, off / BLOB_PAGESIZE); */
422 dp = repopagestore_load_page_range(&data->store, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE);
424 dp += off % BLOB_PAGESIZE;
428 static inline unsigned char *
429 get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
431 unsigned char *dp = *dpp;
435 if (key->storage == KEY_STORAGE_INCORE)
438 *dpp = data_skip_key(data, dp, key);
441 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
444 dp = data_read_id(dp, &off);
445 dp = data_read_id(dp, &len);
448 return get_vertical_data(data, key, off, len);
454 load_repodata(Repodata *data)
456 if (data->loadcallback)
458 data->loadcallback(data);
459 if (data->state == REPODATA_AVAILABLE)
462 data->state = REPODATA_ERROR;
467 maybe_load_repodata(Repodata *data, Id keyname)
469 if (keyname && !repodata_precheck_keyname(data, keyname))
470 return 0; /* do not bother... */
477 for (i = 0; i < data->nkeys; i++)
478 if (keyname == data->keys[i].name)
480 if (i == data->nkeys)
483 return load_repodata(data);
486 case REPODATA_AVAILABLE:
487 case REPODATA_LOADING:
490 data->state = REPODATA_ERROR;
495 static inline unsigned char *
496 solvid2data(Repodata *data, Id solvid, Id *schemap)
498 unsigned char *dp = data->incoredata;
501 if (solvid == SOLVID_META) /* META */
503 else if (solvid == SOLVID_POS) /* META */
505 Pool *pool = data->repo->pool;
506 if (data->repo != pool->pos.repo)
508 if (data != data->repo->repodata + pool->pos.repodataid)
510 *schemap = pool->pos.schema;
511 return data->incoredata + pool->pos.dp;
515 if (solvid < data->start || solvid >= data->end)
517 dp += data->incoreoffset[solvid - data->start];
519 return data_read_id(dp, schemap);
522 /************************************************************************
526 static inline unsigned char *
527 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
530 Id schema, *keyp, *kp;
533 if (!maybe_load_repodata(data, keyname))
535 dp = solvid2data(data, solvid, &schema);
538 keyp = data->schemadata + data->schemata[schema];
539 for (kp = keyp; *kp; kp++)
540 if (data->keys[*kp].name == keyname)
544 *keypp = key = data->keys + *kp;
545 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
546 return dp; /* no need to forward... */
547 dp = forward_to_key(data, *kp, keyp, dp);
550 return get_data(data, key, &dp, 0);
555 repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
561 dp = find_key_data(data, solvid, keyname, &key);
564 if (key->type == REPOKEY_TYPE_CONSTANTID)
566 if (key->type != REPOKEY_TYPE_ID)
568 dp = data_read_id(dp, &id);
573 repodata_globalize_id(Repodata *data, Id id, int create)
575 if (!id || !data || !data->localpool)
577 return str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
581 repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
587 dp = find_key_data(data, solvid, keyname, &key);
590 if (key->type == REPOKEY_TYPE_STR)
591 return (const char *)dp;
592 if (key->type == REPOKEY_TYPE_CONSTANTID)
593 return id2str(data->repo->pool, key->size);
594 if (key->type == REPOKEY_TYPE_ID)
595 dp = data_read_id(dp, &id);
599 return data->spool.stringspace + data->spool.strings[id];
600 return id2str(data->repo->pool, id);
604 repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned int *value)
611 dp = find_key_data(data, solvid, keyname, &key);
614 if (key->type == REPOKEY_TYPE_NUM
615 || key->type == REPOKEY_TYPE_U32
616 || key->type == REPOKEY_TYPE_CONSTANT)
618 dp = data_fetch(dp, &kv, key);
626 repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
632 if (!maybe_load_repodata(data, keyname))
634 dp = solvid2data(data, solvid, &schema);
637 /* can't use find_key_data as we need to test the type */
638 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
639 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
644 const unsigned char *
645 repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
650 dp = find_key_data(data, solvid, keyname, &key);
658 repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
666 dp = find_key_data(data, solvid, keyname, &key);
671 dp = data_read_ideof(dp, &id, &eof);
679 /************************************************************************
685 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
689 case REPOKEY_TYPE_ID:
690 case REPOKEY_TYPE_CONSTANTID:
691 case REPOKEY_TYPE_IDARRAY:
692 if (data && data->localpool)
693 kv->str = stringpool_id2str(&data->spool, kv->id);
695 kv->str = id2str(pool, kv->id);
696 if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
699 for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
701 if (*s == ':' && s > kv->str)
705 case REPOKEY_TYPE_STR:
707 case REPOKEY_TYPE_DIRSTRARRAY:
708 if (!(flags & SEARCH_FILES))
709 return 1; /* match just the basename */
710 /* Put the full filename into kv->str. */
711 kv->str = repodata_dir2str(data, kv->id, kv->str);
712 /* And to compensate for that put the "empty" directory into
713 kv->id, so that later calls to repodata_dir2str on this data
714 come up with the same filename again. */
717 case REPOKEY_TYPE_MD5:
718 case REPOKEY_TYPE_SHA1:
719 case REPOKEY_TYPE_SHA256:
720 if (!(flags & SEARCH_CHECKSUMS))
721 return 0; /* skip em */
722 kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
730 struct subschema_data {
736 /* search a specific repodata */
738 repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
742 Id keyid, *kp, *keyp;
743 unsigned char *dp, *ddp;
749 if (!maybe_load_repodata(data, keyname))
751 if (solvid == SOLVID_SUBSCHEMA)
753 struct subschema_data *subd = cbdata;
754 cbdata = subd->cbdata;
756 schema = subd->parent->id;
757 dp = (unsigned char *)subd->parent->str;
758 kv.parent = subd->parent;
763 dp = solvid2data(data, solvid, &schema);
766 s = data->repo->pool->solvables + solvid;
769 keyp = data->schemadata + data->schemata[schema];
772 /* search for a specific key */
773 for (kp = keyp; *kp; kp++)
774 if (data->keys[*kp].name == keyname)
778 dp = forward_to_key(data, *kp, keyp, dp);
784 while ((keyid = *keyp++) != 0)
787 key = data->keys + keyid;
788 ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
790 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
792 struct subschema_data subd;
796 subd.cbdata = cbdata;
799 ddp = data_read_id(ddp, &nentries);
803 while (ddp && nentries > 0)
807 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
808 ddp = data_read_id(ddp, &schema);
810 kv.str = (char *)ddp;
811 stop = callback(cbdata, s, data, key, &kv);
812 if (stop > SEARCH_NEXT_KEY)
814 if (stop && stop != SEARCH_ENTERSUB)
816 if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
817 repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
818 ddp = data_skip_schema(data, ddp, schema);
821 if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
825 kv.str = (char *)ddp;
826 stop = callback(cbdata, s, data, key, &kv);
827 if (stop > SEARCH_NEXT_KEY)
837 ddp = data_fetch(ddp, &kv, key);
840 stop = callback(cbdata, s, data, key, &kv);
843 while (!kv.eof && !stop);
844 if (onekey || stop > SEARCH_NEXT_KEY)
850 repodata_setpos_kv(Repodata *data, KeyValue *kv)
852 Pool *pool = data->repo->pool;
854 pool_clear_pos(pool);
857 pool->pos.repo = data->repo;
858 pool->pos.repodataid = data - data->repo->repodata;
859 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
860 pool->pos.schema = kv->id;
864 /************************************************************************
865 * data iterator functions
868 static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = {
869 { SOLVABLE_NAME, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
870 { SOLVABLE_ARCH, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
871 { SOLVABLE_EVR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
872 { SOLVABLE_VENDOR, REPOKEY_TYPE_ID, 0, KEY_STORAGE_SOLVABLE },
873 { SOLVABLE_PROVIDES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
874 { SOLVABLE_OBSOLETES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
875 { SOLVABLE_CONFLICTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
876 { SOLVABLE_REQUIRES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
877 { SOLVABLE_RECOMMENDS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
878 { SOLVABLE_SUGGESTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
879 { SOLVABLE_SUPPLEMENTS, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
880 { SOLVABLE_ENHANCES, REPOKEY_TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE },
881 { RPM_RPMDBID, REPOKEY_TYPE_U32, 0, KEY_STORAGE_SOLVABLE },
885 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
899 case SOLVABLE_VENDOR:
902 case SOLVABLE_PROVIDES:
904 return s->provides ? s->repo->idarraydata + s->provides : 0;
905 case SOLVABLE_OBSOLETES:
907 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
908 case SOLVABLE_CONFLICTS:
910 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
911 case SOLVABLE_REQUIRES:
913 return s->requires ? s->repo->idarraydata + s->requires : 0;
914 case SOLVABLE_RECOMMENDS:
916 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
917 case SOLVABLE_SUPPLEMENTS:
919 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
920 case SOLVABLE_SUGGESTS:
922 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
923 case SOLVABLE_ENHANCES:
925 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
928 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
935 datamatcher_init(Datamatcher *ma, const char *match, int flags)
941 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
943 ma->matchdata = sat_calloc(1, sizeof(regex_t));
944 ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
947 sat_free(ma->matchdata);
948 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
955 datamatcher_free(Datamatcher *ma)
957 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
959 regfree(ma->matchdata);
960 ma->matchdata = sat_free(ma->matchdata);
965 datamatcher_match(Datamatcher *ma, const char *str)
968 switch ((ma->flags & SEARCH_STRINGMASK))
970 case SEARCH_SUBSTRING:
971 if (ma->flags & SEARCH_NOCASE)
973 if (!strcasestr(str, ma->match))
978 if (!strstr(str, ma->match))
983 if (ma->flags & SEARCH_NOCASE)
985 if (strcasecmp(ma->match, str))
990 if (strcmp(ma->match, str))
994 case SEARCH_STRINGSTART:
995 if (ma->flags & SEARCH_NOCASE)
997 if (strncasecmp(ma->match, str, strlen(ma->match)))
1002 if (strncmp(ma->match, str, strlen(ma->match)))
1006 case SEARCH_STRINGEND:
1007 l = strlen(str) - strlen(ma->match);
1010 if (ma->flags & SEARCH_NOCASE)
1012 if (strcasecmp(ma->match, str + l))
1017 if (strcmp(ma->match, str + l))
1022 if (fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
1026 if (regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0))
1036 repodata_filelistfilter_matches(Repodata *data, const char *str)
1038 /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
1039 /* for now hardcoded */
1040 if (strstr(str, "bin/"))
1042 if (!strncmp(str, "/etc/", 5))
1044 if (!strcmp(str, "/usr/lib/sendmail"))
1066 di_nextarrayelement,
1071 di_nextsolvableattr,
1076 /* see repo.h for documentation */
1078 dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
1080 memset(di, 0, sizeof(*di));
1082 di->flags = flags & ~SEARCH_THISSOLVID;
1083 if (!pool || (repo && repo->pool != pool))
1091 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1097 di->keyname = keyname;
1098 di->keynames[0] = keyname;
1099 dataiterator_set_search(di, repo, p);
1104 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
1107 memset(&di->matcher, 0, sizeof(di->matcher));
1108 if (from->matcher.match)
1109 datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1114 for (i = 1; i < di->nparents; i++)
1115 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1116 di->kv.parent = &di->parents[di->nparents - 1].kv;
1121 dataiterator_set_match(Dataiterator *di, const char *match, int flags)
1123 di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
1124 datamatcher_free(&di->matcher);
1125 memset(&di->matcher, 0, sizeof(di->matcher));
1129 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1139 dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
1143 di->flags &= ~SEARCH_THISSOLVID;
1147 if (!di->pool->nrepos)
1155 di->repo = di->pool->repos[0];
1157 di->state = di_enterrepo;
1159 dataiterator_jump_to_solvid(di, p);
1163 dataiterator_set_keyname(Dataiterator *di, Id keyname)
1166 di->keyname = keyname;
1167 di->keynames[0] = keyname;
1171 dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1175 if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1177 di->state = di_bye; /* sorry */
1180 for (i = di->nkeynames + 1; i > 0; i--)
1181 di->keynames[i] = di->keynames[i - 1];
1182 di->keynames[0] = di->keyname = keyname;
1187 dataiterator_free(Dataiterator *di)
1189 if (di->matcher.match)
1190 datamatcher_free(&di->matcher);
1193 static inline unsigned char *
1194 dataiterator_find_keyname(Dataiterator *di, Id keyname)
1196 Id *keyp = di->keyp;
1197 Repokey *keys = di->data->keys;
1200 for (keyp = di->keyp; *keyp; keyp++)
1201 if (keys[*keyp].name == keyname)
1205 dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1213 dataiterator_filelistcheck(Dataiterator *di)
1216 int needcomplete = 0;
1217 Repodata *data = di->data;
1219 if ((di->matcher.flags & SEARCH_COMPLETE_FILELIST) != 0)
1220 if (!di->matcher.match
1221 || ((di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING
1222 && (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_GLOB)
1223 || !repodata_filelistfilter_matches(di->data, di->matcher.match))
1225 if (data->state != REPODATA_AVAILABLE)
1226 return needcomplete ? 1 : 0;
1227 for (j = 1; j < data->nkeys; j++)
1228 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1230 return j == data->nkeys && !needcomplete ? 0 : 1;
1234 dataiterator_step(Dataiterator *di)
1242 case di_enterrepo: di_enterrepo:
1245 if (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS))
1247 if (!(di->flags & SEARCH_THISSOLVID))
1249 di->solvid = di->repo->start - 1; /* reset solvid iterator */
1250 goto di_nextsolvable;
1254 case di_entersolvable: di_entersolvable:
1255 if (di->repodataid >= 0)
1257 di->repodataid = 0; /* reset repodata iterator */
1258 if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1260 di->key = solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1262 goto di_entersolvablekey;
1267 case di_enterrepodata: di_enterrepodata:
1268 if (di->repodataid >= 0)
1270 if (di->repodataid >= di->repo->nrepodata)
1271 goto di_nextsolvable;
1272 di->data = di->repo->repodata + di->repodataid;
1274 if (di->repodataid >= 0 && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1275 goto di_nextrepodata;
1276 if (!maybe_load_repodata(di->data, di->keyname))
1277 goto di_nextrepodata;
1278 di->dp = solvid2data(di->data, di->solvid, &schema);
1280 goto di_nextrepodata;
1281 if (di->solvid == SOLVID_POS)
1282 di->solvid = di->pool->pos.solvid;
1283 /* reset key iterator */
1284 di->keyp = di->data->schemadata + di->data->schemata[schema];
1287 case di_enterschema: di_enterschema:
1289 di->dp = dataiterator_find_keyname(di, di->keyname);
1290 if (!di->dp || !*di->keyp)
1294 goto di_nextrepodata;
1298 case di_enterkey: di_enterkey:
1300 di->key = di->data->keys + *di->keyp;
1301 di->ddp = get_data(di->data, di->key, &di->dp, di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0) ? 1 : 0);
1304 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1306 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1312 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1314 di->state = di_nextkey;
1316 di->state = di_nextattr;
1319 case di_nextkey: di_nextkey:
1320 if (!di->keyname && *++di->keyp)
1326 case di_nextrepodata: di_nextrepodata:
1327 if (di->repodataid >= 0 && ++di->repodataid < di->repo->nrepodata)
1328 goto di_enterrepodata;
1331 case di_nextsolvable: di_nextsolvable:
1332 if (!(di->flags & SEARCH_THISSOLVID))
1335 di->solvid = di->repo->start;
1338 for (; di->solvid < di->repo->end; di->solvid++)
1340 if (di->pool->solvables[di->solvid].repo == di->repo)
1341 goto di_entersolvable;
1346 case di_nextrepo: di_nextrepo:
1347 if (di->repoid >= 0)
1351 if (di->repoid < di->pool->nrepos)
1353 di->repo = di->pool->repos[di->repoid];
1359 case di_bye: di_bye:
1363 case di_enterarray: di_enterarray:
1364 if (di->key->name == REPOSITORY_SOLVABLES)
1366 di->ddp = data_read_id(di->ddp, &di->kv.num);
1371 case di_nextarrayelement: di_nextarrayelement:
1374 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1375 if (di->kv.entry == di->kv.num)
1377 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1379 if (!(di->flags & SEARCH_ARRAYSENTINEL))
1381 di->kv.str = (char *)di->ddp;
1383 di->state = di_nextkey;
1386 if (di->kv.entry == di->kv.num - 1)
1388 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1389 di->ddp = data_read_id(di->ddp, &di->kv.id);
1390 di->kv.str = (char *)di->ddp;
1391 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1393 if ((di->flags & SEARCH_SUB) != 0)
1394 di->state = di_entersub;
1396 di->state = di_nextarrayelement;
1399 case di_entersub: di_entersub:
1400 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1401 goto di_nextarrayelement; /* sorry, full */
1402 di->parents[di->nparents].kv = di->kv;
1403 di->parents[di->nparents].dp = di->dp;
1404 di->parents[di->nparents].keyp = di->keyp;
1405 di->dp = (unsigned char *)di->kv.str;
1406 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1407 memset(&di->kv, 0, sizeof(di->kv));
1408 di->kv.parent = &di->parents[di->nparents].kv;
1410 di->keyname = di->keynames[di->nparents - di->rootlevel];
1411 goto di_enterschema;
1413 case di_leavesub: di_leavesub:
1414 if (di->nparents - 1 < di->rootlevel)
1417 di->dp = di->parents[di->nparents].dp;
1418 di->kv = di->parents[di->nparents].kv;
1419 di->keyp = di->parents[di->nparents].keyp;
1420 di->key = di->data->keys + *di->keyp;
1421 di->ddp = (unsigned char *)di->kv.str;
1422 di->keyname = di->keynames[di->nparents - di->rootlevel];
1423 goto di_nextarrayelement;
1425 /* special solvable attr handling follows */
1427 case di_nextsolvableattr:
1428 di->kv.id = *di->idp++;
1433 di->state = di_nextsolvablekey;
1437 case di_nextsolvablekey: di_nextsolvablekey:
1438 if (di->keyname || di->key->name == RPM_RPMDBID)
1439 goto di_enterrepodata;
1443 case di_entersolvablekey: di_entersolvablekey:
1444 di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1445 if (!di->idp || !di->idp[0])
1446 goto di_nextsolvablekey;
1447 di->kv.id = di->idp[0];
1448 di->kv.num = di->idp[0];
1450 if (!di->kv.eof && !di->idp[0])
1454 di->state = di_nextsolvablekey;
1456 di->state = di_nextsolvableattr;
1460 if (di->matcher.match)
1462 /* simple pre-check so that we don't need to stringify */
1463 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && di->matcher.match && (di->matcher.flags & (SEARCH_FILES|SEARCH_NOCASE|SEARCH_STRINGMASK)) == (SEARCH_FILES|SEARCH_STRING))
1465 int l = strlen(di->matcher.match) - strlen(di->kv.str);
1466 if (l < 0 || strcmp(di->matcher.match + l, di->kv.str))
1469 if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
1471 if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
1475 if (!datamatcher_match(&di->matcher, di->kv.str))
1478 /* found something! */
1484 dataiterator_entersub(Dataiterator *di)
1486 if (di->state == di_nextarrayelement)
1487 di->state = di_entersub;
1491 dataiterator_setpos(Dataiterator *di)
1493 if (di->kv.eof == 2)
1495 pool_clear_pos(di->pool);
1498 di->pool->pos.solvid = di->solvid;
1499 di->pool->pos.repo = di->repo;
1500 di->pool->pos.repodataid = di->data - di->repo->repodata;
1501 di->pool->pos.schema = di->kv.id;
1502 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1506 dataiterator_setpos_parent(Dataiterator *di)
1508 if (!di->kv.parent || di->kv.parent->eof == 2)
1510 pool_clear_pos(di->pool);
1513 di->pool->pos.solvid = di->solvid;
1514 di->pool->pos.repo = di->repo;
1515 di->pool->pos.repodataid = di->data - di->repo->repodata;
1516 di->pool->pos.schema = di->kv.parent->id;
1517 di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1520 /* clones just the position, not the search keys/matcher */
1522 dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
1524 di->state = from->state;
1525 di->flags &= ~SEARCH_THISSOLVID;
1526 di->flags |= (from->flags & SEARCH_THISSOLVID);
1527 di->repo = from->repo;
1528 di->data = from->data;
1530 di->ddp = from->ddp;
1531 di->idp = from->idp;
1532 di->keyp = from->keyp;
1533 di->key = from->key;
1535 di->repodataid = from->repodataid;
1536 di->solvid = from->solvid;
1537 di->repoid = from->repoid;
1538 di->rootlevel = from->rootlevel;
1539 memcpy(di->parents, from->parents, sizeof(from->parents));
1540 di->nparents = from->nparents;
1544 for (i = 1; i < di->nparents; i++)
1545 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1546 di->kv.parent = &di->parents[di->nparents - 1].kv;
1551 dataiterator_seek(Dataiterator *di, int whence)
1553 if ((whence & DI_SEEK_STAY) != 0)
1554 di->rootlevel = di->nparents;
1555 switch (whence & ~DI_SEEK_STAY)
1558 if (di->state != di_nextarrayelement)
1560 if ((whence & DI_SEEK_STAY) != 0)
1561 di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1562 di->state = di_entersub;
1564 case DI_SEEK_PARENT:
1571 if (di->rootlevel > di->nparents)
1572 di->rootlevel = di->nparents;
1573 di->dp = di->parents[di->nparents].dp;
1574 di->kv = di->parents[di->nparents].kv;
1575 di->keyp = di->parents[di->nparents].keyp;
1576 di->key = di->data->keys + *di->keyp;
1577 di->ddp = (unsigned char *)di->kv.str;
1578 di->keyname = di->keynames[di->nparents - di->rootlevel];
1579 di->state = di_nextarrayelement;
1581 case DI_SEEK_REWIND:
1587 di->dp = (unsigned char *)di->kv.parent->str;
1588 di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1589 di->state = di_enterschema;
1597 dataiterator_skip_attribute(Dataiterator *di)
1599 if (di->state == di_nextsolvableattr)
1600 di->state = di_nextsolvablekey;
1602 di->state = di_nextkey;
1606 dataiterator_skip_solvable(Dataiterator *di)
1611 di->keyname = di->keynames[0];
1612 di->state = di_nextsolvable;
1616 dataiterator_skip_repo(Dataiterator *di)
1621 di->keyname = di->keynames[0];
1622 di->state = di_nextrepo;
1626 dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1631 di->keyname = di->keynames[0];
1632 if (solvid == SOLVID_POS)
1634 di->repo = di->pool->pos.repo;
1641 di->data = di->repo->repodata + di->pool->pos.repodataid;
1642 di->repodataid = -1;
1643 di->solvid = solvid;
1644 di->state = di_enterrepo;
1645 di->flags |= SEARCH_THISSOLVID;
1650 di->repo = di->pool->solvables[solvid].repo;
1653 else if (di->repoid >= 0)
1655 if (!di->pool->nrepos)
1660 di->repo = di->pool->repos[0];
1664 di->solvid = solvid;
1666 di->flags |= SEARCH_THISSOLVID;
1667 di->state = di_enterrepo;
1671 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1680 di->flags &= ~SEARCH_THISSOLVID;
1681 di->state = di_enterrepo;
1685 dataiterator_match(Dataiterator *di, Datamatcher *ma)
1687 if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
1691 return datamatcher_match(ma, di->kv.str);
1694 /************************************************************************
1695 * data modify functions
1698 /* extend repodata so that it includes solvables p */
1700 repodata_extend(Repodata *data, Id p)
1702 if (data->start == data->end)
1703 data->start = data->end = p;
1706 int old = data->end - data->start;
1707 int new = p - data->end + 1;
1710 data->attrs = sat_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
1711 memset(data->attrs + old, 0, new * sizeof(Id *));
1713 data->incoreoffset = sat_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
1714 memset(data->incoreoffset + old, 0, new * sizeof(Id));
1717 if (p < data->start)
1719 int old = data->end - data->start;
1720 int new = data->start - p;
1723 data->attrs = sat_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
1724 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
1725 memset(data->attrs, 0, new * sizeof(Id *));
1727 data->incoreoffset = sat_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
1728 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
1729 memset(data->incoreoffset, 0, new * sizeof(Id));
1734 /* shrink end of repodata */
1736 repodata_shrink(Repodata *data, int end)
1740 if (data->end <= end)
1742 if (data->start >= end)
1746 for (i = 0; i < data->end - data->start; i++)
1747 sat_free(data->attrs[i]);
1748 data->attrs = sat_free(data->attrs);
1750 data->incoreoffset = sat_free(data->incoreoffset);
1751 data->start = data->end = 0;
1756 for (i = end; i < data->end; i++)
1757 sat_free(data->attrs[i - data->start]);
1758 data->attrs = sat_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
1760 if (data->incoreoffset)
1761 data->incoreoffset = sat_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
1765 /* extend repodata so that it includes solvables from start to start + num - 1 */
1767 repodata_extend_block(Repodata *data, Id start, Id num)
1771 if (!data->incoreoffset)
1773 data->incoreoffset = sat_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
1774 data->start = start;
1775 data->end = start + num;
1778 repodata_extend(data, start);
1780 repodata_extend(data, start + num - 1);
1783 /**********************************************************************/
1786 #define REPODATA_ATTRS_BLOCK 31
1787 #define REPODATA_ATTRDATA_BLOCK 1023
1788 #define REPODATA_ATTRIDDATA_BLOCK 63
1792 repodata_new_handle(Repodata *data)
1796 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1799 data->xattrs = sat_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
1800 data->xattrs[data->nxattrs] = 0;
1801 return -(data->nxattrs++);
1805 repodata_get_attrp(Repodata *data, Id handle)
1807 if (handle == SOLVID_META)
1811 data->xattrs = sat_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
1816 return data->xattrs - handle;
1817 if (handle < data->start || handle >= data->end)
1818 repodata_extend(data, handle);
1820 data->attrs = sat_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
1821 return data->attrs + (handle - data->start);
1825 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
1831 app = repodata_get_attrp(data, handle);
1836 /* Determine equality based on the name only, allows us to change
1837 type (when overwrite is set), and makes TYPE_CONSTANT work. */
1838 for (pp = ap; *pp; pp += 2)
1839 if (data->keys[*pp].name == data->keys[keyid].name)
1852 ap = sat_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
1862 repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
1866 keyid = repodata_key2id(data, key, 1);
1867 repodata_insert_keyid(data, solvid, keyid, val, 1);
1871 repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
1875 key.type = REPOKEY_TYPE_ID;
1877 key.storage = KEY_STORAGE_INCORE;
1878 repodata_set(data, solvid, &key, id);
1882 repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned int num)
1886 key.type = REPOKEY_TYPE_NUM;
1888 key.storage = KEY_STORAGE_INCORE;
1889 repodata_set(data, solvid, &key, (Id)num);
1893 repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
1897 if (data->localpool)
1898 id = stringpool_str2id(&data->spool, str, 1);
1900 id = str2id(data->repo->pool, str, 1);
1902 key.type = REPOKEY_TYPE_ID;
1904 key.storage = KEY_STORAGE_INCORE;
1905 repodata_set(data, solvid, &key, id);
1909 repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
1913 key.type = REPOKEY_TYPE_CONSTANT;
1914 key.size = constant;
1915 key.storage = KEY_STORAGE_INCORE;
1916 repodata_set(data, solvid, &key, 0);
1920 repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
1924 key.type = REPOKEY_TYPE_CONSTANTID;
1926 key.storage = KEY_STORAGE_INCORE;
1927 repodata_set(data, solvid, &key, 0);
1931 repodata_set_void(Repodata *data, Id solvid, Id keyname)
1935 key.type = REPOKEY_TYPE_VOID;
1937 key.storage = KEY_STORAGE_INCORE;
1938 repodata_set(data, solvid, &key, 0);
1942 repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
1947 l = strlen(str) + 1;
1949 key.type = REPOKEY_TYPE_STR;
1951 key.storage = KEY_STORAGE_INCORE;
1952 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
1953 memcpy(data->attrdata + data->attrdatalen, str, l);
1954 repodata_set(data, solvid, &key, data->attrdatalen);
1955 data->attrdatalen += l;
1959 repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len)
1965 key.type = REPOKEY_TYPE_BINARY;
1967 key.storage = KEY_STORAGE_INCORE;
1968 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK);
1969 dp = data->attrdata + data->attrdatalen;
1970 if (len >= (1 << 14))
1972 if (len >= (1 << 28))
1973 *dp++ = (len >> 28) | 128;
1974 if (len >= (1 << 21))
1975 *dp++ = (len >> 21) | 128;
1976 *dp++ = (len >> 14) | 128;
1978 if (len >= (1 << 7))
1979 *dp++ = (len >> 7) | 128;
1982 memcpy(dp, buf, len);
1983 repodata_set(data, solvid, &key, data->attrdatalen);
1984 data->attrdatalen = dp + len - data->attrdata;
1987 /* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
1988 * so that the caller can append the new element there */
1990 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
1993 Id *ida, *pp, **ppp;
1995 /* check if it is the same as last time, this speeds things up a lot */
1996 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
1998 /* great! just append the new data */
1999 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2000 data->attriddatalen--; /* overwrite terminating 0 */
2001 data->lastdatalen += entrysize;
2005 ppp = repodata_get_attrp(data, handle);
2008 for (; *pp; pp += 2)
2009 if (data->keys[*pp].name == keyname && data->keys[*pp].type == keytype)
2013 /* not found. allocate new key */
2018 key.storage = KEY_STORAGE_INCORE;
2019 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2020 repodata_set(data, handle, &key, data->attriddatalen);
2021 data->lasthandle = 0; /* next time... */
2025 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2026 oldsize += entrysize;
2027 if (ida + 1 == data->attriddata + data->attriddatalen)
2029 /* this was the last entry, just append it */
2030 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2031 data->attriddatalen--; /* overwrite terminating 0 */
2035 /* too bad. move to back. */
2036 data->attriddata = sat_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2037 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2038 pp[1] = data->attriddatalen;
2039 data->attriddatalen += oldsize;
2041 data->lasthandle = handle;
2042 data->lastkey = *pp;
2043 data->lastdatalen = data->attriddatalen + entrysize + 1;
2047 repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2048 const unsigned char *str)
2053 if (!(l = sat_chksum_len(type)))
2058 key.storage = KEY_STORAGE_INCORE;
2059 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2060 memcpy(data->attrdata + data->attrdatalen, str, l);
2061 repodata_set(data, solvid, &key, data->attrdatalen);
2062 data->attrdatalen += l;
2066 hexstr2bytes(unsigned char *buf, const char *str, int buflen)
2069 for (i = 0; i < buflen; i++)
2071 #define c2h(c) (((c)>='0' && (c)<='9') ? ((c)-'0') \
2072 : ((c)>='a' && (c)<='f') ? ((c)-('a'-10)) \
2073 : ((c)>='A' && (c)<='F') ? ((c)-('A'-10)) \
2084 buf[i] = (buf[i] << 4) | v;
2091 repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2094 unsigned char buf[64];
2097 if (!(l = sat_chksum_len(type)))
2099 if (hexstr2bytes(buf, str, l) != l)
2101 repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2105 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2110 if (!(l = sat_chksum_len(type)))
2112 s = str = pool_alloctmpspace(data->repo->pool, 2 * l + 1);
2113 for (i = 0; i < l; i++)
2115 unsigned char v = buf[i];
2116 unsigned char w = v >> 4;
2117 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2119 *s++ = w >= 10 ? w + ('a' - 10) : w + '0';
2125 /* rpm filenames don't contain the epoch, so strip it */
2126 static inline const char *
2127 evrid2vrstr(Pool *pool, Id evrid)
2129 const char *p, *evr = id2str(pool, evrid);
2132 for (p = evr; *p >= '0' && *p <= '9'; p++)
2134 return p != evr && *p == ':' ? p + 1 : evr;
2138 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
2140 Pool *pool = data->repo->pool;
2142 const char *str, *fp;
2146 repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2149 if ((dir = strrchr(file, '/')) != 0)
2160 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2165 if (l == 1 && dir[0] == '.')
2167 s = pool->solvables + solvid;
2170 str = id2str(pool, s->arch);
2171 if (!strncmp(dir, str, l) && !str[l])
2172 repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2174 repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir);
2177 char *dir2 = strdup(dir);
2179 repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir2);
2184 str = id2str(pool, s->name);
2186 if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
2189 str = evrid2vrstr(pool, s->evr);
2191 if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
2194 str = id2str(pool, s->arch);
2196 if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2198 repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2203 repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2207 repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2211 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2213 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2214 data->attriddata[data->attriddatalen++] = dir;
2215 data->attriddata[data->attriddatalen++] = num;
2216 data->attriddata[data->attriddatalen++] = num2;
2217 data->attriddata[data->attriddatalen++] = 0;
2221 repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
2227 l = strlen(str) + 1;
2228 data->attrdata = sat_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2229 memcpy(data->attrdata + data->attrdatalen, str, l);
2230 stroff = data->attrdatalen;
2231 data->attrdatalen += l;
2234 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2236 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2237 data->attriddata[data->attriddatalen++] = dir;
2238 data->attriddata[data->attriddatalen++] = stroff;
2239 data->attriddata[data->attriddatalen++] = 0;
2243 repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2246 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2248 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2249 data->attriddata[data->attriddatalen++] = id;
2250 data->attriddata[data->attriddatalen++] = 0;
2254 repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2258 if (data->localpool)
2259 id = stringpool_str2id(&data->spool, str, 1);
2261 id = str2id(data->repo->pool, str, 1);
2262 repodata_add_idarray(data, solvid, keyname, id);
2266 repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2268 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2269 data->attriddata[data->attriddatalen++] = ghandle;
2270 data->attriddata[data->attriddatalen++] = 0;
2274 repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2276 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2277 data->attriddata[data->attriddatalen++] = ghandle;
2278 data->attriddata[data->attriddatalen++] = 0;
2282 repodata_delete_uninternalized(Repodata *data, Id solvid, Id keyname)
2285 app = repodata_get_attrp(data, solvid);
2289 for (; *ap; ap += 2)
2290 if (data->keys[*ap].name == keyname)
2296 for (; *ap; ap += 2)
2298 if (data->keys[*ap].name == keyname)
2306 /* add all attrs from src to dest */
2308 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2311 if (dest == src || !(keyp = data->attrs[src - data->start]))
2313 for (; *keyp; keyp += 2)
2314 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2318 repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
2321 if (dest == src || !(keyp = data->attrs[src - data->start]))
2323 for (; *keyp; keyp += 2)
2324 if (!keyidmap || MAPTST(keyidmap, keyp[0]))
2325 repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2330 /**********************************************************************/
2332 /* TODO: unify with repo_write and repo_solv! */
2334 #define EXTDATA_BLOCK 1023
2342 data_addid(struct extdata *xd, Id x)
2346 xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2347 dp = xd->buf + xd->len;
2352 *dp++ = (x >> 28) | 128;
2354 *dp++ = (x >> 21) | 128;
2355 *dp++ = (x >> 14) | 128;
2358 *dp++ = (x >> 7) | 128;
2360 xd->len = dp - xd->buf;
2364 data_addideof(struct extdata *xd, Id x, int eof)
2367 x = (x & 63) | ((x & ~63) << 1);
2368 data_addid(xd, (eof ? x : x | 64));
2372 data_addblob(struct extdata *xd, unsigned char *blob, int len)
2374 xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
2375 memcpy(xd->buf + xd->len, blob, len);
2379 /*********************************/
2382 repodata_serialize_key(Repodata *data, struct extdata *newincore,
2383 struct extdata *newvincore,
2385 Repokey *key, Id val)
2387 /* Otherwise we have a new value. Parse it into the internal
2391 unsigned int oldvincorelen = 0;
2395 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2398 oldvincorelen = xd->len;
2402 case REPOKEY_TYPE_VOID:
2403 case REPOKEY_TYPE_CONSTANT:
2404 case REPOKEY_TYPE_CONSTANTID:
2406 case REPOKEY_TYPE_STR:
2407 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
2409 case REPOKEY_TYPE_MD5:
2410 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
2412 case REPOKEY_TYPE_SHA1:
2413 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
2415 case REPOKEY_TYPE_SHA256:
2416 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
2418 case REPOKEY_TYPE_ID:
2419 case REPOKEY_TYPE_NUM:
2420 case REPOKEY_TYPE_DIR:
2421 data_addid(xd, val);
2423 case REPOKEY_TYPE_BINARY:
2426 unsigned char *dp = data_read_id(data->attrdata + val, &len);
2428 data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
2431 case REPOKEY_TYPE_IDARRAY:
2432 for (ida = data->attriddata + val; *ida; ida++)
2433 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
2435 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2436 for (ida = data->attriddata + val; *ida; ida += 3)
2438 data_addid(xd, ida[0]);
2439 data_addid(xd, ida[1]);
2440 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
2443 case REPOKEY_TYPE_DIRSTRARRAY:
2444 for (ida = data->attriddata + val; *ida; ida += 2)
2446 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
2447 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
2450 case REPOKEY_TYPE_FIXARRAY:
2454 for (ida = data->attriddata + val; *ida; ida++)
2457 fprintf(stderr, "serialize struct %d\n", *ida);
2460 Id *kp = data->xattrs[-*ida];
2467 fprintf(stderr, " %s:%d\n", id2str(data->repo->pool, data->keys[*kp].name), kp[1]);
2473 schemaid = repodata_schema2id(data, schema, 1);
2474 else if (schemaid != repodata_schema2id(data, schema, 0))
2476 pool_debug(data->repo->pool, SAT_FATAL, "fixarray substructs with different schemas\n");
2480 fprintf(stderr, " schema %d\n", schemaid);
2485 data_addid(xd, num);
2486 data_addid(xd, schemaid);
2487 for (ida = data->attriddata + val; *ida; ida++)
2489 Id *kp = data->xattrs[-*ida];
2494 repodata_serialize_key(data, newincore, newvincore,
2495 schema, data->keys + *kp, kp[1]);
2500 case REPOKEY_TYPE_FLEXARRAY:
2503 for (ida = data->attriddata + val; *ida; ida++)
2505 data_addid(xd, num);
2506 for (ida = data->attriddata + val; *ida; ida++)
2508 Id *kp = data->xattrs[-*ida];
2511 data_addid(xd, 0); /* XXX */
2518 schemaid = repodata_schema2id(data, schema, 1);
2519 data_addid(xd, schemaid);
2520 kp = data->xattrs[-*ida];
2523 repodata_serialize_key(data, newincore, newvincore,
2524 schema, data->keys + *kp, kp[1]);
2530 pool_debug(data->repo->pool, SAT_FATAL, "don't know how to handle type %d\n", key->type);
2533 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2535 /* put offset/len in incore */
2536 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
2537 oldvincorelen = xd->len - oldvincorelen;
2538 data_addid(newincore, oldvincorelen);
2543 repodata_internalize(Repodata *data)
2545 Repokey *key, solvkey;
2547 Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen;
2548 unsigned char *dp, *ndp;
2549 int newschema, oldcount;
2550 struct extdata newincore;
2551 struct extdata newvincore;
2554 if (!data->attrs && !data->xattrs)
2557 newvincore.buf = data->vincore;
2558 newvincore.len = data->vincorelen;
2560 /* find the solvables key, create if needed */
2561 memset(&solvkey, 0, sizeof(solvkey));
2562 solvkey.name = REPOSITORY_SOLVABLES;
2563 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
2565 solvkey.storage = KEY_STORAGE_INCORE;
2566 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
2568 schema = sat_malloc2(data->nkeys, sizeof(Id));
2569 seen = sat_malloc2(data->nkeys, sizeof(Id));
2571 /* Merge the data already existing (in data->schemata, ->incoredata and
2572 friends) with the new attributes in data->attrs[]. */
2573 nentry = data->end - data->start;
2574 memset(&newincore, 0, sizeof(newincore));
2575 data_addid(&newincore, 0); /* start data at offset 1 */
2577 data->mainschema = 0;
2578 data->mainschemaoffsets = sat_free(data->mainschemaoffsets);
2580 /* join entry data */
2581 /* we start with the meta data, entry -1 */
2582 for (entry = -1; entry < nentry; entry++)
2584 memset(seen, 0, data->nkeys * sizeof(Id));
2586 dp = data->incoredata;
2589 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
2590 dp = data_read_id(dp, &oldschema);
2593 fprintf(stderr, "oldschema %d\n", oldschema);
2594 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
2595 fprintf(stderr, "schemadata %p\n", data->schemadata);
2597 /* seen: -1: old data 0: skipped >0: id + 1 */
2601 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
2605 pool_debug(data->repo->pool, SAT_FATAL, "Inconsistent old data (key occured twice).\n");
2613 keyp = data->attrs ? data->attrs[entry] : 0;
2616 /* strip solvables key */
2618 for (sp = keyp = schema; *sp; sp++)
2619 if (*sp != solvkeyid)
2624 seen[solvkeyid] = 0;
2625 keyp = data->xattrs ? data->xattrs[1] : 0;
2628 for (; *keyp; keyp += 2)
2635 seen[*keyp] = keyp[1] + 1;
2637 if (entry < 0 && data->end != data->start)
2644 /* Ideally we'd like to sort the new schema here, to ensure
2645 schema equality independend of the ordering. We can't do that
2646 yet. For once see below (old ids need to come before new ids).
2647 An additional difficulty is that we also need to move
2648 the values with the keys. */
2649 schemaid = repodata_schema2id(data, schema, 1);
2651 schemaid = oldschema;
2654 /* Now create data blob. We walk through the (possibly new) schema
2655 and either copy over old data, or insert the new. */
2656 /* XXX Here we rely on the fact that the (new) schema has the form
2657 o1 o2 o3 o4 ... | n1 n2 n3 ...
2658 (oX being the old keyids (possibly overwritten), and nX being
2659 the new keyids). This rules out sorting the keyids in order
2660 to ensure a small schema count. */
2662 data->incoreoffset[entry] = newincore.len;
2663 data_addid(&newincore, schemaid);
2666 data->mainschema = schemaid;
2667 data->mainschemaoffsets = sat_calloc(sp - schema, sizeof(Id));
2669 keypstart = data->schemadata + data->schemata[schemaid];
2670 for (keyp = keypstart; *keyp; keyp++)
2673 data->mainschemaoffsets[keyp - keypstart] = newincore.len;
2674 if (*keyp == solvkeyid)
2676 /* add flexarray entry count */
2677 data_addid(&newincore, data->end - data->start);
2680 key = data->keys + *keyp;
2682 fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, id2str(data->repo->pool, key->name), id2str(data->repo->pool, key->type));
2687 /* Skip the data associated with this old key. */
2688 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2690 ndp = data_skip(dp, REPOKEY_TYPE_ID);
2691 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
2693 else if (key->storage == KEY_STORAGE_INCORE)
2694 ndp = data_skip_key(data, dp, key);
2697 if (seen[*keyp] == -1)
2699 /* If this key was an old one _and_ was not overwritten with
2700 a different value copy over the old value (we skipped it
2703 data_addblob(&newincore, dp, ndp - dp);
2706 else if (seen[*keyp])
2708 /* Otherwise we have a new value. Parse it into the internal
2710 repodata_serialize_key(data, &newincore, &newvincore,
2711 schema, key, seen[*keyp] - 1);
2715 if (entry >= 0 && data->attrs && data->attrs[entry])
2716 data->attrs[entry] = sat_free(data->attrs[entry]);
2718 /* free all xattrs */
2719 for (entry = 0; entry < data->nxattrs; entry++)
2720 if (data->xattrs[entry])
2721 sat_free(data->xattrs[entry]);
2722 data->xattrs = sat_free(data->xattrs);
2725 data->lasthandle = 0;
2727 data->lastdatalen = 0;
2730 repodata_free_schemahash(data);
2732 sat_free(data->incoredata);
2733 data->incoredata = newincore.buf;
2734 data->incoredatalen = newincore.len;
2735 data->incoredatafree = 0;
2737 sat_free(data->vincore);
2738 data->vincore = newvincore.buf;
2739 data->vincorelen = newvincore.len;
2741 data->attrs = sat_free(data->attrs);
2742 data->attrdata = sat_free(data->attrdata);
2743 data->attriddata = sat_free(data->attriddata);
2744 data->attrdatalen = 0;
2745 data->attriddatalen = 0;
2749 repodata_disable_paging(Repodata *data)
2751 if (maybe_load_repodata(data, 0))
2752 repopagestore_disable_paging(&data->store);
2756 repodata_load_stub(Repodata *data)
2758 Repo *repo = data->repo;
2759 Pool *pool = repo->pool;
2762 if (!pool->loadcallback)
2764 data->state = REPODATA_ERROR;
2767 data->state = REPODATA_LOADING;
2768 r = pool->loadcallback(pool, data, pool->loadcallbackdata);
2770 data->state = REPODATA_ERROR;
2774 repodata_create_stubs(Repodata *data)
2776 Repo *repo = data->repo;
2777 Pool *pool = repo->pool;
2784 int datastart, dataend;
2786 repodataid = data - repo->repodata;
2787 datastart = data->start;
2788 dataend = data->end;
2789 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
2790 while (dataiterator_step(&di))
2792 if (di.data - repo->repodata != repodataid)
2796 dataiterator_free(&di);
2799 stubdataids = sat_calloc(cnt, sizeof(*stubdataids));
2800 for (i = 0; i < cnt; i++)
2802 sdata = repo_add_repodata(repo, 0);
2803 if (dataend > datastart)
2804 repodata_extend_block(sdata, datastart, dataend - datastart);
2805 stubdataids[i] = sdata - repo->repodata;
2806 sdata->state = REPODATA_STUB;
2807 sdata->loadcallback = repodata_load_stub;
2810 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
2812 while (dataiterator_step(&di))
2814 if (di.data - repo->repodata != repodataid)
2816 if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
2818 dataiterator_entersub(&di);
2819 sdata = repo->repodata + stubdataids[i++];
2823 switch (di.key->type)
2825 case REPOKEY_TYPE_ID:
2826 repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
2828 case REPOKEY_TYPE_CONSTANTID:
2829 repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
2831 case REPOKEY_TYPE_STR:
2832 repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
2834 case REPOKEY_TYPE_VOID:
2835 repodata_set_void(sdata, SOLVID_META, di.key->name);
2837 case REPOKEY_TYPE_NUM:
2838 repodata_set_num(sdata, SOLVID_META, di.key->name, di.kv.num);
2840 case REPOKEY_TYPE_MD5:
2841 case REPOKEY_TYPE_SHA1:
2842 case REPOKEY_TYPE_SHA256:
2843 repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
2845 case REPOKEY_TYPE_IDARRAY:
2846 repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
2847 if (di.key->name == REPOSITORY_KEYS)
2854 xkeyname = di.kv.id;
2857 xkey.name = xkeyname;
2858 xkey.type = di.kv.id;
2859 xkey.storage = KEY_STORAGE_INCORE;
2861 repodata_key2id(sdata, &xkey, 1);
2866 dataiterator_free(&di);
2867 for (i = 0; i < cnt; i++)
2868 repodata_internalize(repo->repodata + stubdataids[i]);
2869 sat_free(stubdataids);
2873 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: