2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
13 * a repository can contain multiple repodata entries, consisting of
14 * different sets of keys and different sets of solvables
29 #include "poolid_private.h"
37 #define REPODATA_BLOCK 255
39 static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);
42 repodata_initdata(Repodata *data, Repo *repo, int localpool)
44 memset(data, 0, sizeof (*data));
45 data->repodataid = data - repo->repodata;
47 data->localpool = localpool;
49 stringpool_init_empty(&data->spool);
50 /* dirpool_init(&data->dirpool); just zeros out again */
51 data->keys = solv_calloc(1, sizeof(Repokey));
53 data->schemata = solv_calloc(1, sizeof(Id));
54 data->schemadata = solv_calloc(1, sizeof(Id));
56 data->schemadatalen = 1;
57 repopagestore_init(&data->store);
61 repodata_freedata(Repodata *data)
65 solv_free(data->keys);
67 solv_free(data->schemata);
68 solv_free(data->schemadata);
69 solv_free(data->schematahash);
71 stringpool_free(&data->spool);
72 dirpool_free(&data->dirpool);
74 solv_free(data->mainschemaoffsets);
75 solv_free(data->incoredata);
76 solv_free(data->incoreoffset);
77 solv_free(data->verticaloffset);
79 repopagestore_free(&data->store);
81 solv_free(data->vincore);
84 for (i = 0; i < data->end - data->start; i++)
85 solv_free(data->attrs[i]);
86 solv_free(data->attrs);
88 for (i = 0; i < data->nxattrs; i++)
89 solv_free(data->xattrs[i]);
90 solv_free(data->xattrs);
92 solv_free(data->attrdata);
93 solv_free(data->attriddata);
94 solv_free(data->attrnum64data);
96 solv_free(data->dircache);
100 repodata_free(Repodata *data)
102 Repo *repo = data->repo;
103 int i = data - repo->repodata;
106 repodata_freedata(data);
107 if (i < repo->nrepodata - 1)
109 /* whoa! this changes the repodataids! */
110 memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
111 for (; i < repo->nrepodata - 1; i++)
112 repo->repodata[i].repodataid = i;
115 if (repo->nrepodata == 1)
117 repo->repodata = solv_free(repo->repodata);
123 repodata_empty(Repodata *data, int localpool)
125 void (*loadcallback)(Repodata *) = data->loadcallback;
126 int state = data->state;
127 repodata_freedata(data);
128 repodata_initdata(data, data->repo, localpool);
130 data->loadcallback = loadcallback;
134 /***************************************************************
135 * key pool management
138 /* this is not so time critical that we need a hash, so we do a simple
141 repodata_key2id(Repodata *data, Repokey *key, int create)
145 for (keyid = 1; keyid < data->nkeys; keyid++)
146 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
148 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
152 if (keyid == data->nkeys)
156 /* allocate new key */
157 data->keys = solv_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
158 data->keys[data->nkeys++] = *key;
159 if (data->verticaloffset)
161 data->verticaloffset = solv_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
162 data->verticaloffset[data->nkeys - 1] = 0;
164 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
170 /***************************************************************
171 * schema pool management
174 #define SCHEMATA_BLOCK 31
175 #define SCHEMATADATA_BLOCK 255
178 repodata_schema2id(Repodata *data, Id *schema, int create)
185 return 0; /* XXX: allow empty schema? */
186 if ((schematahash = data->schematahash) == 0)
188 data->schematahash = schematahash = solv_calloc(256, sizeof(Id));
189 for (i = 1; i < data->nschemata; i++)
191 for (sp = data->schemadata + data->schemata[i], h = 0; *sp;)
196 data->schemadata = solv_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
197 data->schemata = solv_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
200 for (sp = schema, len = 0, h = 0; *sp; len++)
205 cid = schematahash[h];
208 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
210 /* cache conflict, do a slow search */
211 for (cid = 1; cid < data->nschemata; cid++)
212 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
218 data->schemadata = solv_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
219 data->schemata = solv_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
221 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
222 data->schemata[data->nschemata] = data->schemadatalen;
223 data->schemadatalen += len;
224 schematahash[h] = data->nschemata;
226 fprintf(stderr, "schema2id: new schema\n");
228 return data->nschemata++;
232 repodata_free_schemahash(Repodata *data)
234 data->schematahash = solv_free(data->schematahash);
236 data->schemata = solv_realloc2(data->schemata, data->nschemata, sizeof(Id));
237 data->schemadata = solv_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
241 /***************************************************************
242 * dir pool management
245 #ifndef HAVE_STRCHRNUL
246 static inline const char *strchrnul(const char *str, char x)
248 const char *p = strchr(str, x);
249 return p ? p : str + strlen(str);
253 #define DIRCACHE_SIZE 41 /* < 1k */
257 Id ids[DIRCACHE_SIZE];
258 char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
263 repodata_str2dir(Repodata *data, const char *dir, int create)
274 while (*dir == '/' && dir[1] == '/')
276 if (*dir == '/' && !dir[1])
278 if (data->dirpool.ndirs)
280 return dirpool_add_dir(&data->dirpool, 0, 1, create);
287 struct dircache *dircache = data->dircache;
291 if (l < DIRCACHE_SIZE && dircache->ids[l] && !memcmp(dircache->str + l * (l - 1) / 2, dir, l))
293 parent = dircache->ids[l];
309 dire = strchrnul(dir, '/');
311 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
313 id = pool_strn2id(data->repo->pool, dir, dire - dir, create);
316 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
321 data->dircache = solv_calloc(1, sizeof(struct dircache));
325 if (l < DIRCACHE_SIZE)
327 data->dircache->ids[l] = parent;
328 memcpy(data->dircache->str + l * (l - 1) / 2, dirs, l);
342 repodata_free_dircache(Repodata *data)
344 data->dircache = solv_free(data->dircache);
348 repodata_dir2str(Repodata *data, Id did, const char *suf)
350 Pool *pool = data->repo->pool;
357 return suf ? suf : "";
361 comp = dirpool_compid(&data->dirpool, parent);
362 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
364 parent = dirpool_parent(&data->dirpool, parent);
369 l += strlen(suf) + 1;
370 p = pool_alloctmpspace(pool, l + 1) + l;
381 comp = dirpool_compid(&data->dirpool, parent);
382 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
385 strncpy(p, comps, l);
386 parent = dirpool_parent(&data->dirpool, parent);
394 /***************************************************************
398 static inline unsigned char *
399 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
401 Id *keyp = data->schemadata + data->schemata[schema];
402 for (; *keyp; keyp++)
403 dp = data_skip_key(data, dp, data->keys + *keyp);
407 static unsigned char *
408 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
410 int nentries, schema;
413 case REPOKEY_TYPE_FIXARRAY:
414 dp = data_read_id(dp, &nentries);
417 dp = data_read_id(dp, &schema);
419 dp = data_skip_schema(data, dp, schema);
421 case REPOKEY_TYPE_FLEXARRAY:
422 dp = data_read_id(dp, &nentries);
425 dp = data_read_id(dp, &schema);
426 dp = data_skip_schema(data, dp, schema);
430 if (key->storage == KEY_STORAGE_INCORE)
431 dp = data_skip(dp, key->type);
432 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
434 dp = data_skip(dp, REPOKEY_TYPE_ID);
435 dp = data_skip(dp, REPOKEY_TYPE_ID);
441 static unsigned char *
442 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
448 if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
451 for (i = 0; (k = *keyp++) != 0; i++)
453 return data->incoredata + data->mainschemaoffsets[i];
456 while ((k = *keyp++) != 0)
460 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
462 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
463 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
466 if (data->keys[k].storage != KEY_STORAGE_INCORE)
468 dp = data_skip_key(data, dp, data->keys + k);
473 static unsigned char *
474 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
479 if (off >= data->lastverticaloffset)
481 off -= data->lastverticaloffset;
482 if ((unsigned int)off + len > data->vincorelen)
484 return data->vincore + off;
486 if ((unsigned int)off + len > key->size)
488 /* we now have the offset, go into vertical */
489 off += data->verticaloffset[key - data->keys];
490 /* fprintf(stderr, "key %d page %d\n", key->name, off / REPOPAGE_BLOBSIZE); */
491 dp = repopagestore_load_page_range(&data->store, off / REPOPAGE_BLOBSIZE, (off + len - 1) / REPOPAGE_BLOBSIZE);
494 dp += off % REPOPAGE_BLOBSIZE;
498 static inline unsigned char *
499 get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
501 unsigned char *dp = *dpp;
505 if (key->storage == KEY_STORAGE_INCORE)
508 *dpp = data_skip_key(data, dp, key);
511 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
514 dp = data_read_id(dp, &off);
515 dp = data_read_id(dp, &len);
518 return get_vertical_data(data, key, off, len);
524 load_repodata(Repodata *data)
526 if (data->loadcallback)
528 data->loadcallback(data);
529 if (data->state == REPODATA_AVAILABLE)
532 data->state = REPODATA_ERROR;
537 maybe_load_repodata(Repodata *data, Id keyname)
539 if (keyname && !repodata_precheck_keyname(data, keyname))
540 return 0; /* do not bother... */
547 for (i = 1; i < data->nkeys; i++)
548 if (keyname == data->keys[i].name)
550 if (i == data->nkeys)
553 return load_repodata(data);
556 case REPODATA_AVAILABLE:
557 case REPODATA_LOADING:
560 data->state = REPODATA_ERROR;
565 static inline unsigned char *
566 solvid2data(Repodata *data, Id solvid, Id *schemap)
568 unsigned char *dp = data->incoredata;
571 if (solvid == SOLVID_META)
572 dp += 1; /* offset of "meta" solvable */
573 else if (solvid == SOLVID_POS)
575 Pool *pool = data->repo->pool;
576 if (data->repo != pool->pos.repo)
578 if (data != data->repo->repodata + pool->pos.repodataid)
581 if (pool->pos.dp != 1)
583 *schemap = pool->pos.schema;
589 if (solvid < data->start || solvid >= data->end)
591 dp += data->incoreoffset[solvid - data->start];
593 return data_read_id(dp, schemap);
596 /************************************************************************
600 static unsigned char *
601 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
604 Id schema, *keyp, *kp;
607 if (!maybe_load_repodata(data, keyname))
609 dp = solvid2data(data, solvid, &schema);
612 keyp = data->schemadata + data->schemata[schema];
613 for (kp = keyp; *kp; kp++)
614 if (data->keys[*kp].name == keyname)
618 *keypp = key = data->keys + *kp;
619 if (key->type == REPOKEY_TYPE_DELETED)
621 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
622 return dp; /* no need to forward... */
623 if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
624 return 0; /* get_data will not work, no need to forward */
625 dp = forward_to_key(data, *kp, keyp, dp);
628 return get_data(data, key, &dp, 0);
632 repodata_lookup_type(Repodata *data, Id solvid, Id keyname)
634 Id schema, *keyp, *kp;
635 if (!maybe_load_repodata(data, keyname))
637 if (!solvid2data(data, solvid, &schema))
639 keyp = data->schemadata + data->schemata[schema];
640 for (kp = keyp; *kp; kp++)
641 if (data->keys[*kp].name == keyname)
642 return data->keys[*kp].type;
647 repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
653 dp = find_key_data(data, solvid, keyname, &key);
656 if (key->type == REPOKEY_TYPE_CONSTANTID)
658 if (key->type != REPOKEY_TYPE_ID)
660 dp = data_read_id(dp, &id);
665 repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
671 dp = find_key_data(data, solvid, keyname, &key);
674 if (key->type == REPOKEY_TYPE_STR)
675 return (const char *)dp;
676 if (key->type == REPOKEY_TYPE_CONSTANTID)
678 else if (key->type == REPOKEY_TYPE_ID)
679 dp = data_read_id(dp, &id);
683 return stringpool_id2str(&data->spool, id);
684 return pool_id2str(data->repo->pool, id);
688 repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long *value)
692 unsigned int high, low;
695 dp = find_key_data(data, solvid, keyname, &key);
700 case REPOKEY_TYPE_NUM:
701 data_read_num64(dp, &low, &high);
702 *value = (unsigned long long)high << 32 | low;
704 case REPOKEY_TYPE_U32:
705 data_read_u32(dp, &low);
708 case REPOKEY_TYPE_CONSTANT:
717 repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
723 if (!maybe_load_repodata(data, keyname))
725 dp = solvid2data(data, solvid, &schema);
728 /* can't use find_key_data as we need to test the type */
729 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
730 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
735 const unsigned char *
736 repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
741 dp = find_key_data(data, solvid, keyname, &key);
756 repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
764 dp = find_key_data(data, solvid, keyname, &key);
767 if (key->type != REPOKEY_TYPE_IDARRAY)
771 dp = data_read_ideof(dp, &id, &eof);
780 repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp)
786 dp = find_key_data(data, solvid, keyname, &key);
787 if (!dp || key->type != REPOKEY_TYPE_BINARY)
792 dp = data_read_id(dp, &len);
798 repodata_globalize_id(Repodata *data, Id id, int create)
800 if (!id || !data || !data->localpool)
802 return pool_str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
806 repodata_localize_id(Repodata *data, Id id, int create)
808 if (!id || !data || !data->localpool)
810 return stringpool_str2id(&data->spool, pool_id2str(data->repo->pool, id), create);
814 repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create)
816 if (!id || !data || !fromdata)
818 if (!data->localpool || !fromdata->localpool)
820 if (fromdata->localpool)
821 id = repodata_globalize_id(fromdata, id, create);
823 id = repodata_localize_id(data, id, create);
826 /* localpool is set in both data and fromdata */
827 return stringpool_str2id(&data->spool, stringpool_id2str(&fromdata->spool, id), create);
831 repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id voidid)
836 ap = data->attrs[solvid - data->start];
841 if (data->keys[*ap].name != keyname)
843 if (data->keys[*ap].type == REPOKEY_TYPE_VOID)
845 if (data->keys[*ap].type == REPOKEY_TYPE_ID)
853 /************************************************************************
859 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
863 case REPOKEY_TYPE_ID:
864 case REPOKEY_TYPE_CONSTANTID:
865 case REPOKEY_TYPE_IDARRAY:
866 if (data && data->localpool)
867 kv->str = stringpool_id2str(&data->spool, kv->id);
869 kv->str = pool_id2str(pool, kv->id);
870 if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
873 for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
875 if (*s == ':' && s > kv->str)
879 case REPOKEY_TYPE_STR:
881 case REPOKEY_TYPE_DIRSTRARRAY:
882 if (!(flags & SEARCH_FILES))
883 return kv->str; /* match just the basename */
885 return kv->str; /* already stringified */
886 /* Put the full filename into kv->str. */
887 kv->str = repodata_dir2str(data, kv->id, kv->str);
888 kv->num = 1; /* mark stringification */
891 if (!(flags & SEARCH_CHECKSUMS))
892 return 0; /* skip em */
894 return kv->str; /* already stringified */
895 kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
896 kv->num = 1; /* mark stringification */
904 struct subschema_data {
910 /* search a specific repodata */
912 repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
916 Id keyid, *kp, *keyp;
917 unsigned char *dp, *ddp;
923 if (!maybe_load_repodata(data, keyname))
925 if (solvid == SOLVID_SUBSCHEMA)
927 struct subschema_data *subd = cbdata;
928 cbdata = subd->cbdata;
930 schema = subd->parent->id;
931 dp = (unsigned char *)subd->parent->str;
932 kv.parent = subd->parent;
937 dp = solvid2data(data, solvid, &schema);
940 s = data->repo->pool->solvables + solvid;
943 keyp = data->schemadata + data->schemata[schema];
946 /* search for a specific key */
947 for (kp = keyp; *kp; kp++)
948 if (data->keys[*kp].name == keyname)
952 dp = forward_to_key(data, *kp, keyp, dp);
958 while ((keyid = *keyp++) != 0)
961 key = data->keys + keyid;
962 ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
964 if (key->type == REPOKEY_TYPE_DELETED)
966 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
968 struct subschema_data subd;
972 subd.cbdata = cbdata;
975 ddp = data_read_id(ddp, &nentries);
979 while (ddp && nentries > 0)
983 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
984 ddp = data_read_id(ddp, &schema);
986 kv.str = (char *)ddp;
987 stop = callback(cbdata, s, data, key, &kv);
988 if (stop > SEARCH_NEXT_KEY)
990 if (stop && stop != SEARCH_ENTERSUB)
992 if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
993 repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
994 ddp = data_skip_schema(data, ddp, schema);
997 if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
1001 kv.str = (char *)ddp;
1002 stop = callback(cbdata, s, data, key, &kv);
1003 if (stop > SEARCH_NEXT_KEY)
1013 ddp = data_fetch(ddp, &kv, key);
1016 stop = callback(cbdata, s, data, key, &kv);
1019 while (!kv.eof && !stop);
1020 if (onekey || stop > SEARCH_NEXT_KEY)
1026 repodata_setpos_kv(Repodata *data, KeyValue *kv)
1028 Pool *pool = data->repo->pool;
1030 pool_clear_pos(pool);
1033 pool->pos.repo = data->repo;
1034 pool->pos.repodataid = data - data->repo->repodata;
1035 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
1036 pool->pos.schema = kv->id;
1040 /************************************************************************
1041 * data iterator functions
1045 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
1059 case SOLVABLE_VENDOR:
1062 case SOLVABLE_PROVIDES:
1064 return s->provides ? s->repo->idarraydata + s->provides : 0;
1065 case SOLVABLE_OBSOLETES:
1067 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
1068 case SOLVABLE_CONFLICTS:
1070 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
1071 case SOLVABLE_REQUIRES:
1073 return s->requires ? s->repo->idarraydata + s->requires : 0;
1074 case SOLVABLE_RECOMMENDS:
1076 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
1077 case SOLVABLE_SUPPLEMENTS:
1079 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
1080 case SOLVABLE_SUGGESTS:
1082 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
1083 case SOLVABLE_ENHANCES:
1085 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
1088 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
1095 datamatcher_init(Datamatcher *ma, const char *match, int flags)
1097 match = match ? solv_strdup(match) : 0;
1102 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
1104 ma->matchdata = solv_calloc(1, sizeof(regex_t));
1105 ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
1108 solv_free(ma->matchdata);
1109 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
1112 if ((flags & SEARCH_FILES) != 0 && match)
1114 /* prepare basename check */
1115 if ((flags & SEARCH_STRINGMASK) == SEARCH_STRING || (flags & SEARCH_STRINGMASK) == SEARCH_STRINGEND)
1117 const char *p = strrchr(match, '/');
1118 ma->matchdata = (void *)(p ? p + 1 : match);
1120 else if ((flags & SEARCH_STRINGMASK) == SEARCH_GLOB)
1123 for (p = match + strlen(match) - 1; p >= match; p--)
1124 if (*p == '[' || *p == ']' || *p == '*' || *p == '?' || *p == '/')
1126 ma->matchdata = (void *)(p + 1);
1133 datamatcher_free(Datamatcher *ma)
1136 ma->match = solv_free((char *)ma->match);
1137 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
1139 regfree(ma->matchdata);
1140 solv_free(ma->matchdata);
1146 datamatcher_match(Datamatcher *ma, const char *str)
1149 switch ((ma->flags & SEARCH_STRINGMASK))
1151 case SEARCH_SUBSTRING:
1152 if (ma->flags & SEARCH_NOCASE)
1153 return strcasestr(str, ma->match) != 0;
1155 return strstr(str, ma->match) != 0;
1157 if (ma->flags & SEARCH_NOCASE)
1158 return !strcasecmp(ma->match, str);
1160 return !strcmp(ma->match, str);
1161 case SEARCH_STRINGSTART:
1162 if (ma->flags & SEARCH_NOCASE)
1163 return !strncasecmp(ma->match, str, strlen(ma->match));
1165 return !strncmp(ma->match, str, strlen(ma->match));
1166 case SEARCH_STRINGEND:
1167 l = strlen(str) - strlen(ma->match);
1170 if (ma->flags & SEARCH_NOCASE)
1171 return !strcasecmp(ma->match, str + l);
1173 return !strcmp(ma->match, str + l);
1175 return !fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0);
1177 return !regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0);
1183 /* check if the matcher can match the provides basename */
1186 datamatcher_checkbasename(Datamatcher *ma, const char *basename)
1189 const char *match = ma->matchdata;
1192 switch (ma->flags & SEARCH_STRINGMASK)
1196 case SEARCH_STRINGEND:
1197 if (match != ma->match)
1198 break; /* had slash, do exact match on basename */
1201 /* check if the basename ends with match */
1202 l = strlen(basename) - strlen(match);
1208 return 1; /* maybe matches */
1210 if ((ma->flags & SEARCH_NOCASE) != 0)
1211 return !strcasecmp(match, basename);
1213 return !strcmp(match, basename);
1217 repodata_filelistfilter_matches(Repodata *data, const char *str)
1219 /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
1220 /* for now hardcoded */
1221 if (strstr(str, "bin/"))
1223 if (!strncmp(str, "/etc/", 5))
1225 if (!strcmp(str, "/usr/lib/sendmail"))
1247 di_nextarrayelement,
1253 di_entersolvablekey,
1257 /* see dataiterator.h for documentation */
1259 dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
1261 memset(di, 0, sizeof(*di));
1263 di->flags = flags & ~SEARCH_THISSOLVID;
1264 if (!pool || (repo && repo->pool != pool))
1272 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1278 di->keyname = keyname;
1279 di->keynames[0] = keyname;
1280 dataiterator_set_search(di, repo, p);
1285 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
1290 if (di->dupstr == di->kv.str)
1291 di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
1298 memset(&di->matcher, 0, sizeof(di->matcher));
1299 if (from->matcher.match)
1300 datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1305 for (i = 1; i < di->nparents; i++)
1306 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1307 di->kv.parent = &di->parents[di->nparents - 1].kv;
1312 dataiterator_set_match(Dataiterator *di, const char *match, int flags)
1314 di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
1315 datamatcher_free(&di->matcher);
1316 memset(&di->matcher, 0, sizeof(di->matcher));
1320 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1330 dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
1334 di->flags &= ~SEARCH_THISSOLVID;
1338 if (!di->pool->urepos)
1346 di->repo = di->pool->repos[di->repoid];
1348 di->state = di_enterrepo;
1350 dataiterator_jump_to_solvid(di, p);
1354 dataiterator_set_keyname(Dataiterator *di, Id keyname)
1357 di->keyname = keyname;
1358 di->keynames[0] = keyname;
1362 dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1366 if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1368 di->state = di_bye; /* sorry */
1371 for (i = di->nkeynames + 1; i > 0; i--)
1372 di->keynames[i] = di->keynames[i - 1];
1373 di->keynames[0] = di->keyname = keyname;
1378 dataiterator_free(Dataiterator *di)
1380 if (di->matcher.match)
1381 datamatcher_free(&di->matcher);
1383 solv_free(di->dupstr);
1386 static unsigned char *
1387 dataiterator_find_keyname(Dataiterator *di, Id keyname)
1390 Repokey *keys = di->data->keys, *key;
1393 for (keyp = di->keyp; *keyp; keyp++)
1394 if (keys[*keyp].name == keyname)
1399 if (key->type == REPOKEY_TYPE_DELETED)
1401 if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
1402 return 0; /* get_data will not work, no need to forward */
1403 dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1411 is_filelist_extension(Repodata *data)
1414 if (!repodata_precheck_keyname(data, SOLVABLE_FILELIST))
1416 for (j = 1; j < data->nkeys; j++)
1417 if (data->keys[j].name == SOLVABLE_FILELIST)
1419 if (j == data->nkeys)
1421 if (data->state != REPODATA_AVAILABLE)
1423 for (j = 1; j < data->nkeys; j++)
1424 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1430 dataiterator_filelistcheck(Dataiterator *di)
1433 int needcomplete = 0;
1434 Repodata *data = di->data;
1436 if ((di->flags & SEARCH_COMPLETE_FILELIST) != 0)
1437 if (!di->matcher.match
1438 || ((di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING
1439 && (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_GLOB)
1440 || !repodata_filelistfilter_matches(data, di->matcher.match))
1442 if (data->state != REPODATA_AVAILABLE)
1443 return needcomplete ? 1 : 0;
1446 /* we don't need the complete filelist, so ignore all stubs */
1447 if (data->repo->nrepodata == 2)
1449 for (j = 1; j < data->nkeys; j++)
1450 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1456 /* we need the complete filelist. check if we habe a filtered filelist and there's
1457 * a extension with the complete filelist later on */
1458 for (j = 1; j < data->nkeys; j++)
1459 if (data->keys[j].name == SOLVABLE_FILELIST)
1461 if (j == data->nkeys)
1462 return 0; /* does not have filelist */
1463 for (j = 1; j < data->nkeys; j++)
1464 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1466 if (j == data->nkeys)
1467 return 1; /* this is the externsion */
1468 while (data - data->repo->repodata + 1 < data->repo->nrepodata)
1471 if (is_filelist_extension(data))
1479 dataiterator_step(Dataiterator *di)
1483 if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate) {
1484 unsigned int ddpoff = di->ddp - di->vert_ddp;
1485 di->vert_off += ddpoff;
1486 di->vert_len -= ddpoff;
1487 di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len);
1488 di->vert_storestate = di->data->storestate;
1490 di->state = di_nextkey;
1496 case di_enterrepo: di_enterrepo:
1497 if (!di->repo || (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS)))
1499 if (!(di->flags & SEARCH_THISSOLVID))
1501 di->solvid = di->repo->start - 1; /* reset solvid iterator */
1502 goto di_nextsolvable;
1506 case di_entersolvable: di_entersolvable:
1509 di->repodataid = 1; /* reset repodata iterator */
1510 if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1512 extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1];
1514 di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1516 goto di_entersolvablekey;
1521 case di_enterrepodata: di_enterrepodata:
1524 if (di->repodataid >= di->repo->nrepodata)
1525 goto di_nextsolvable;
1526 di->data = di->repo->repodata + di->repodataid;
1528 if (di->repodataid && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1529 goto di_nextrepodata;
1530 if (!maybe_load_repodata(di->data, di->keyname))
1531 goto di_nextrepodata;
1532 di->dp = solvid2data(di->data, di->solvid, &schema);
1534 goto di_nextrepodata;
1535 if (di->solvid == SOLVID_POS)
1536 di->solvid = di->pool->pos.solvid;
1537 /* reset key iterator */
1538 di->keyp = di->data->schemadata + di->data->schemata[schema];
1541 case di_enterschema: di_enterschema:
1543 di->dp = dataiterator_find_keyname(di, di->keyname);
1544 if (!di->dp || !*di->keyp)
1548 goto di_nextrepodata;
1552 case di_enterkey: di_enterkey:
1554 di->key = di->data->keys + *di->keyp;
1557 /* this is get_data() modified to store vert_ data */
1558 if (di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1561 di->dp = data_read_id(di->dp, &off);
1562 di->dp = data_read_id(di->dp, &len);
1563 di->vert_ddp = di->ddp = get_vertical_data(di->data, di->key, off, len);
1566 di->vert_storestate = di->data->storestate;
1568 else if (di->key->storage == KEY_STORAGE_INCORE)
1571 if (di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0))
1572 di->dp = data_skip_key(di->data, di->dp, di->key);
1578 if (di->key->type == REPOKEY_TYPE_DELETED)
1580 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1582 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1588 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1590 di->state = di_nextkey;
1592 di->state = di_nextattr;
1595 case di_nextkey: di_nextkey:
1596 if (!di->keyname && *++di->keyp)
1602 case di_nextrepodata: di_nextrepodata:
1603 if (di->repodataid && ++di->repodataid < di->repo->nrepodata)
1604 goto di_enterrepodata;
1607 case di_nextsolvable: di_nextsolvable:
1608 if (!(di->flags & SEARCH_THISSOLVID))
1611 di->solvid = di->repo->start;
1614 for (; di->solvid < di->repo->end; di->solvid++)
1616 if (di->pool->solvables[di->solvid].repo == di->repo)
1617 goto di_entersolvable;
1622 case di_nextrepo: di_nextrepo:
1627 if (di->repoid < di->pool->nrepos)
1629 di->repo = di->pool->repos[di->repoid];
1635 case di_bye: di_bye:
1639 case di_enterarray: di_enterarray:
1640 if (di->key->name == REPOSITORY_SOLVABLES)
1642 di->ddp = data_read_id(di->ddp, (Id *)&di->kv.num);
1647 case di_nextarrayelement: di_nextarrayelement:
1650 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1651 if (di->kv.entry == di->kv.num)
1653 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1655 if (!(di->flags & SEARCH_ARRAYSENTINEL))
1657 di->kv.str = (char *)di->ddp;
1659 di->state = di_nextkey;
1662 if (di->kv.entry == di->kv.num - 1)
1664 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1665 di->ddp = data_read_id(di->ddp, &di->kv.id);
1666 di->kv.str = (char *)di->ddp;
1667 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1669 if ((di->flags & SEARCH_SUB) != 0)
1670 di->state = di_entersub;
1672 di->state = di_nextarrayelement;
1675 case di_entersub: di_entersub:
1676 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1677 goto di_nextarrayelement; /* sorry, full */
1678 di->parents[di->nparents].kv = di->kv;
1679 di->parents[di->nparents].dp = di->dp;
1680 di->parents[di->nparents].keyp = di->keyp;
1681 di->dp = (unsigned char *)di->kv.str;
1682 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1683 memset(&di->kv, 0, sizeof(di->kv));
1684 di->kv.parent = &di->parents[di->nparents].kv;
1686 di->keyname = di->keynames[di->nparents - di->rootlevel];
1687 goto di_enterschema;
1689 case di_leavesub: di_leavesub:
1690 if (di->nparents - 1 < di->rootlevel)
1693 di->dp = di->parents[di->nparents].dp;
1694 di->kv = di->parents[di->nparents].kv;
1695 di->keyp = di->parents[di->nparents].keyp;
1696 di->key = di->data->keys + *di->keyp;
1697 di->ddp = (unsigned char *)di->kv.str;
1698 di->keyname = di->keynames[di->nparents - di->rootlevel];
1699 goto di_nextarrayelement;
1701 /* special solvable attr handling follows */
1703 case di_nextsolvablekey: di_nextsolvablekey:
1704 if (di->keyname || di->key->name == RPM_RPMDBID)
1705 goto di_enterrepodata;
1709 case di_entersolvablekey: di_entersolvablekey:
1710 di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1711 if (!di->idp || !*di->idp)
1712 goto di_nextsolvablekey;
1716 di->kv.id = *di->idp;
1717 di->kv.num = *di->idp; /* for rpmdbid */
1718 di->kv.num2 = 0; /* for rpmdbid */
1720 di->state = di_nextsolvablekey;
1726 case di_nextsolvableattr:
1727 di->state = di_nextsolvableattr;
1728 di->kv.id = *di->idp++;
1733 di->state = di_nextsolvablekey;
1739 if (di->matcher.match)
1742 /* simple pre-check so that we don't need to stringify */
1743 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0)
1744 if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
1746 if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
1748 if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
1752 if (!datamatcher_match(&di->matcher, str))
1757 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->flags & SEARCH_FILES) != 0)
1758 repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags);
1760 /* found something! */
1766 dataiterator_entersub(Dataiterator *di)
1768 if (di->state == di_nextarrayelement)
1769 di->state = di_entersub;
1773 dataiterator_setpos(Dataiterator *di)
1775 if (di->kv.eof == 2)
1777 pool_clear_pos(di->pool);
1780 di->pool->pos.solvid = di->solvid;
1781 di->pool->pos.repo = di->repo;
1782 di->pool->pos.repodataid = di->data - di->repo->repodata;
1783 di->pool->pos.schema = di->kv.id;
1784 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1788 dataiterator_setpos_parent(Dataiterator *di)
1790 if (!di->kv.parent || di->kv.parent->eof == 2)
1792 pool_clear_pos(di->pool);
1795 di->pool->pos.solvid = di->solvid;
1796 di->pool->pos.repo = di->repo;
1797 di->pool->pos.repodataid = di->data - di->repo->repodata;
1798 di->pool->pos.schema = di->kv.parent->id;
1799 di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1802 /* clones just the position, not the search keys/matcher */
1804 dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
1806 di->state = from->state;
1807 di->flags &= ~SEARCH_THISSOLVID;
1808 di->flags |= (from->flags & SEARCH_THISSOLVID);
1809 di->repo = from->repo;
1810 di->data = from->data;
1812 di->ddp = from->ddp;
1813 di->idp = from->idp;
1814 di->keyp = from->keyp;
1815 di->key = from->key;
1817 di->repodataid = from->repodataid;
1818 di->solvid = from->solvid;
1819 di->repoid = from->repoid;
1820 di->rootlevel = from->rootlevel;
1821 memcpy(di->parents, from->parents, sizeof(from->parents));
1822 di->nparents = from->nparents;
1826 for (i = 1; i < di->nparents; i++)
1827 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1828 di->kv.parent = &di->parents[di->nparents - 1].kv;
1832 if (from->dupstr && from->dupstr == from->kv.str)
1834 di->dupstrn = from->dupstrn;
1835 di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
1840 dataiterator_seek(Dataiterator *di, int whence)
1842 if ((whence & DI_SEEK_STAY) != 0)
1843 di->rootlevel = di->nparents;
1844 switch (whence & ~DI_SEEK_STAY)
1847 if (di->state != di_nextarrayelement)
1849 if ((whence & DI_SEEK_STAY) != 0)
1850 di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1851 di->state = di_entersub;
1853 case DI_SEEK_PARENT:
1860 if (di->rootlevel > di->nparents)
1861 di->rootlevel = di->nparents;
1862 di->dp = di->parents[di->nparents].dp;
1863 di->kv = di->parents[di->nparents].kv;
1864 di->keyp = di->parents[di->nparents].keyp;
1865 di->key = di->data->keys + *di->keyp;
1866 di->ddp = (unsigned char *)di->kv.str;
1867 di->keyname = di->keynames[di->nparents - di->rootlevel];
1868 di->state = di_nextarrayelement;
1870 case DI_SEEK_REWIND:
1876 di->dp = (unsigned char *)di->kv.parent->str;
1877 di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1878 di->state = di_enterschema;
1886 dataiterator_skip_attribute(Dataiterator *di)
1888 if (di->state == di_nextsolvableattr)
1889 di->state = di_nextsolvablekey;
1891 di->state = di_nextkey;
1895 dataiterator_skip_solvable(Dataiterator *di)
1900 di->keyname = di->keynames[0];
1901 di->state = di_nextsolvable;
1905 dataiterator_skip_repo(Dataiterator *di)
1910 di->keyname = di->keynames[0];
1911 di->state = di_nextrepo;
1915 dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1920 di->keyname = di->keynames[0];
1921 if (solvid == SOLVID_POS)
1923 di->repo = di->pool->pos.repo;
1930 di->data = di->repo->repodata + di->pool->pos.repodataid;
1932 di->solvid = solvid;
1933 di->state = di_enterrepo;
1934 di->flags |= SEARCH_THISSOLVID;
1939 di->repo = di->pool->solvables[solvid].repo;
1942 else if (di->repoid > 0)
1944 if (!di->pool->urepos)
1950 di->repo = di->pool->repos[di->repoid];
1953 di->solvid = solvid;
1955 di->flags |= SEARCH_THISSOLVID;
1956 di->state = di_enterrepo;
1960 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1966 di->repoid = 0; /* 0 means stay at repo */
1969 di->flags &= ~SEARCH_THISSOLVID;
1970 di->state = di_enterrepo;
1974 dataiterator_match(Dataiterator *di, Datamatcher *ma)
1977 if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
1979 return ma ? datamatcher_match(ma, str) : 1;
1983 dataiterator_strdup(Dataiterator *di)
1987 if (!di->kv.str || di->kv.str == di->dupstr)
1989 switch (di->key->type)
1992 case REPOKEY_TYPE_DIRSTRARRAY:
1993 if (di->kv.num) /* was it stringified into tmp space? */
1994 l = strlen(di->kv.str) + 1;
1999 if (l < 0 && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2001 switch (di->key->type)
2003 case REPOKEY_TYPE_STR:
2004 case REPOKEY_TYPE_DIRSTRARRAY:
2005 l = strlen(di->kv.str) + 1;
2008 l = solv_chksum_len(di->key->type);
2010 case REPOKEY_TYPE_BINARY:
2017 if (!di->dupstrn || di->dupstrn < l)
2019 di->dupstrn = l + 16;
2020 di->dupstr = solv_realloc(di->dupstr, di->dupstrn);
2023 memcpy(di->dupstr, di->kv.str, l);
2024 di->kv.str = di->dupstr;
2028 /************************************************************************
2029 * data modify functions
2032 /* extend repodata so that it includes solvables p */
2034 repodata_extend(Repodata *data, Id p)
2036 if (data->start == data->end)
2037 data->start = data->end = p;
2040 int old = data->end - data->start;
2041 int new = p - data->end + 1;
2044 data->attrs = solv_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
2045 memset(data->attrs + old, 0, new * sizeof(Id *));
2047 data->incoreoffset = solv_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
2048 memset(data->incoreoffset + old, 0, new * sizeof(Id));
2051 if (p < data->start)
2053 int old = data->end - data->start;
2054 int new = data->start - p;
2057 data->attrs = solv_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
2058 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
2059 memset(data->attrs, 0, new * sizeof(Id *));
2061 data->incoreoffset = solv_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
2062 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
2063 memset(data->incoreoffset, 0, new * sizeof(Id));
2068 /* shrink end of repodata */
2070 repodata_shrink(Repodata *data, int end)
2074 if (data->end <= end)
2076 if (data->start >= end)
2080 for (i = 0; i < data->end - data->start; i++)
2081 solv_free(data->attrs[i]);
2082 data->attrs = solv_free(data->attrs);
2084 data->incoreoffset = solv_free(data->incoreoffset);
2085 data->start = data->end = 0;
2090 for (i = end; i < data->end; i++)
2091 solv_free(data->attrs[i - data->start]);
2092 data->attrs = solv_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
2094 if (data->incoreoffset)
2095 data->incoreoffset = solv_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
2099 /* extend repodata so that it includes solvables from start to start + num - 1 */
2101 repodata_extend_block(Repodata *data, Id start, Id num)
2105 if (!data->incoreoffset)
2107 /* this also means that data->attrs is NULL */
2108 data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
2109 data->start = start;
2110 data->end = start + num;
2113 repodata_extend(data, start);
2115 repodata_extend(data, start + num - 1);
2118 /**********************************************************************/
2121 #define REPODATA_ATTRS_BLOCK 31
2122 #define REPODATA_ATTRDATA_BLOCK 1023
2123 #define REPODATA_ATTRIDDATA_BLOCK 63
2124 #define REPODATA_ATTRNUM64DATA_BLOCK 15
2128 repodata_new_handle(Repodata *data)
2132 data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
2133 data->nxattrs = 2; /* -1: SOLVID_META */
2135 data->xattrs = solv_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
2136 data->xattrs[data->nxattrs] = 0;
2137 return -(data->nxattrs++);
2141 repodata_get_attrp(Repodata *data, Id handle)
2145 if (handle == SOLVID_META && !data->xattrs)
2147 data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
2150 return data->xattrs - handle;
2152 if (handle < data->start || handle >= data->end)
2153 repodata_extend(data, handle);
2155 data->attrs = solv_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
2156 return data->attrs + (handle - data->start);
2160 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
2166 app = repodata_get_attrp(data, handle);
2171 /* Determine equality based on the name only, allows us to change
2172 type (when overwrite is set), and makes TYPE_CONSTANT work. */
2173 for (pp = ap; *pp; pp += 2)
2174 if (data->keys[*pp].name == data->keys[keyid].name)
2178 if (overwrite || data->keys[*pp].type == REPOKEY_TYPE_DELETED)
2187 ap = solv_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
2197 repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
2201 keyid = repodata_key2id(data, key, 1);
2202 repodata_insert_keyid(data, solvid, keyid, val, 1);
2206 repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
2210 key.type = REPOKEY_TYPE_ID;
2212 key.storage = KEY_STORAGE_INCORE;
2213 repodata_set(data, solvid, &key, id);
2217 repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned long long num)
2221 key.type = REPOKEY_TYPE_NUM;
2223 key.storage = KEY_STORAGE_INCORE;
2224 if (num >= 0x80000000)
2226 data->attrnum64data = solv_extend(data->attrnum64data, data->attrnum64datalen, 1, sizeof(unsigned long long), REPODATA_ATTRNUM64DATA_BLOCK);
2227 data->attrnum64data[data->attrnum64datalen] = num;
2228 num = 0x80000000 | data->attrnum64datalen++;
2230 repodata_set(data, solvid, &key, (Id)num);
2234 repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
2238 if (data->localpool)
2239 id = stringpool_str2id(&data->spool, str, 1);
2241 id = pool_str2id(data->repo->pool, str, 1);
2243 key.type = REPOKEY_TYPE_ID;
2245 key.storage = KEY_STORAGE_INCORE;
2246 repodata_set(data, solvid, &key, id);
2250 repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
2254 key.type = REPOKEY_TYPE_CONSTANT;
2255 key.size = constant;
2256 key.storage = KEY_STORAGE_INCORE;
2257 repodata_set(data, solvid, &key, 0);
2261 repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
2265 key.type = REPOKEY_TYPE_CONSTANTID;
2267 key.storage = KEY_STORAGE_INCORE;
2268 repodata_set(data, solvid, &key, 0);
2272 repodata_set_void(Repodata *data, Id solvid, Id keyname)
2276 key.type = REPOKEY_TYPE_VOID;
2278 key.storage = KEY_STORAGE_INCORE;
2279 repodata_set(data, solvid, &key, 0);
2283 repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
2288 l = strlen(str) + 1;
2290 key.type = REPOKEY_TYPE_STR;
2292 key.storage = KEY_STORAGE_INCORE;
2293 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2294 memcpy(data->attrdata + data->attrdatalen, str, l);
2295 repodata_set(data, solvid, &key, data->attrdatalen);
2296 data->attrdatalen += l;
2300 repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len)
2308 key.type = REPOKEY_TYPE_BINARY;
2310 key.storage = KEY_STORAGE_INCORE;
2311 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK);
2312 dp = data->attrdata + data->attrdatalen;
2313 if (len >= (1 << 14))
2315 if (len >= (1 << 28))
2316 *dp++ = (len >> 28) | 128;
2317 if (len >= (1 << 21))
2318 *dp++ = (len >> 21) | 128;
2319 *dp++ = (len >> 14) | 128;
2321 if (len >= (1 << 7))
2322 *dp++ = (len >> 7) | 128;
2325 memcpy(dp, buf, len);
2326 repodata_set(data, solvid, &key, data->attrdatalen);
2327 data->attrdatalen = dp + len - data->attrdata;
2330 /* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
2331 * so that the caller can append entrysize new elements plus the termination zero there */
2333 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
2336 Id *ida, *pp, **ppp;
2338 /* check if it is the same as last time, this speeds things up a lot */
2339 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
2341 /* great! just append the new data */
2342 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2343 data->attriddatalen--; /* overwrite terminating 0 */
2344 data->lastdatalen += entrysize;
2348 ppp = repodata_get_attrp(data, handle);
2352 for (; *pp; pp += 2)
2353 if (data->keys[*pp].name == keyname)
2356 if (!pp || !*pp || data->keys[*pp].type != keytype)
2358 /* not found. allocate new key */
2364 key.storage = KEY_STORAGE_INCORE;
2365 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2366 keyid = repodata_key2id(data, &key, 1);
2367 repodata_insert_keyid(data, handle, keyid, data->attriddatalen, 1);
2368 data->lasthandle = handle;
2369 data->lastkey = keyid;
2370 data->lastdatalen = data->attriddatalen + entrysize + 1;
2374 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2375 oldsize += entrysize;
2376 if (ida + 1 == data->attriddata + data->attriddatalen)
2378 /* this was the last entry, just append it */
2379 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2380 data->attriddatalen--; /* overwrite terminating 0 */
2384 /* too bad. move to back. */
2385 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2386 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2387 pp[1] = data->attriddatalen;
2388 data->attriddatalen += oldsize;
2390 data->lasthandle = handle;
2391 data->lastkey = *pp;
2392 data->lastdatalen = data->attriddatalen + entrysize + 1;
2396 repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2397 const unsigned char *str)
2402 if (!(l = solv_chksum_len(type)))
2407 key.storage = KEY_STORAGE_INCORE;
2408 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2409 memcpy(data->attrdata + data->attrdatalen, str, l);
2410 repodata_set(data, solvid, &key, data->attrdatalen);
2411 data->attrdatalen += l;
2415 repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2418 unsigned char buf[64];
2421 if (!(l = solv_chksum_len(type)))
2423 if (l > sizeof(buf) || solv_hex2bin(&str, buf, l) != l)
2425 repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2429 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2433 if (!(l = solv_chksum_len(type)))
2435 return pool_bin2hex(data->repo->pool, buf, l);
2438 /* rpm filenames don't contain the epoch, so strip it */
2439 static inline const char *
2440 evrid2vrstr(Pool *pool, Id evrid)
2442 const char *p, *evr = pool_id2str(pool, evrid);
2445 for (p = evr; *p >= '0' && *p <= '9'; p++)
2447 return p != evr && *p == ':' && p[1] ? p + 1 : evr;
2451 repodata_set_poolstrn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
2454 if (data->localpool)
2455 id = stringpool_strn2id(&data->spool, str, l, 1);
2457 id = pool_strn2id(data->repo->pool, str, l, 1);
2458 repodata_set_id(data, solvid, keyname, id);
2462 repodata_set_strn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
2465 repodata_set_str(data, solvid, keyname, str);
2468 char *s = solv_strdup(str);
2470 repodata_set_str(data, solvid, keyname, s);
2476 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
2478 Pool *pool = data->repo->pool;
2480 const char *str, *fp;
2484 repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2487 if ((dir = strrchr(file, '/')) != 0)
2498 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2503 if (l == 1 && dir[0] == '.')
2505 s = pool->solvables + solvid;
2508 str = pool_id2str(pool, s->arch);
2509 if (!strncmp(dir, str, l) && !str[l])
2510 repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2512 repodata_set_strn(data, solvid, SOLVABLE_MEDIADIR, dir, l);
2515 str = pool_id2str(pool, s->name);
2517 if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
2520 str = evrid2vrstr(pool, s->evr);
2522 if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
2525 str = pool_id2str(pool, s->arch);
2527 if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2529 repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2534 repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2537 /* XXX: medianr is currently not stored */
2539 repodata_set_deltalocation(Repodata *data, Id handle, int medianr, const char *dir, const char *file)
2542 const char *evr, *suf, *s;
2546 if ((dir = strrchr(file, '/')) != 0)
2557 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2562 if (l == 1 && dir[0] == '.')
2565 repodata_set_poolstrn(data, handle, DELTA_LOCATION_DIR, dir, l);
2566 evr = strchr(file, '-');
2569 for (s = evr - 1; s > file; s--)
2576 suf = strrchr(file, '.');
2579 for (s = suf - 1; s > file; s--)
2585 if (!strcmp(suf, ".delta.rpm") || !strcmp(suf, ".patch.rpm"))
2587 /* We accept one more item as suffix. */
2588 for (s = suf - 1; s > file; s--)
2598 if (suf && evr && suf < evr)
2600 repodata_set_poolstrn(data, handle, DELTA_LOCATION_NAME, file, evr ? evr - file : strlen(file));
2602 repodata_set_poolstrn(data, handle, DELTA_LOCATION_EVR, evr + 1, suf ? suf - evr - 1: strlen(evr + 1));
2604 repodata_set_poolstr(data, handle, DELTA_LOCATION_SUFFIX, suf + 1);
2608 repodata_set_sourcepkg(Repodata *data, Id solvid, const char *sourcepkg)
2610 Pool *pool = data->repo->pool;
2611 Solvable *s = pool->solvables + solvid;
2612 const char *p, *sevr, *sarch, *name, *evr;
2614 p = strrchr(sourcepkg, '.');
2615 if (!p || strcmp(p, ".rpm") != 0)
2618 repodata_set_str(data, solvid, SOLVABLE_SOURCENAME, sourcepkg);
2622 while (p > sourcepkg && *p != '.')
2624 if (*p != '.' || p == sourcepkg)
2627 while (p > sourcepkg && *p != '-')
2629 if (*p != '-' || p == sourcepkg)
2632 while (p > sourcepkg && *p != '-')
2634 if (*p != '-' || p == sourcepkg)
2637 pool = s->repo->pool;
2639 name = pool_id2str(pool, s->name);
2640 if (name && !strncmp(sourcepkg, name, sevr - sourcepkg - 1) && name[sevr - sourcepkg - 1] == 0)
2641 repodata_set_void(data, solvid, SOLVABLE_SOURCENAME);
2643 repodata_set_id(data, solvid, SOLVABLE_SOURCENAME, pool_strn2id(pool, sourcepkg, sevr - sourcepkg - 1, 1));
2645 evr = evrid2vrstr(pool, s->evr);
2646 if (evr && !strncmp(sevr, evr, sarch - sevr - 1) && evr[sarch - sevr - 1] == 0)
2647 repodata_set_void(data, solvid, SOLVABLE_SOURCEEVR);
2649 repodata_set_id(data, solvid, SOLVABLE_SOURCEEVR, pool_strn2id(pool, sevr, sarch - sevr - 1, 1));
2651 if (!strcmp(sarch, "src.rpm"))
2652 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_SRC);
2653 else if (!strcmp(sarch, "nosrc.rpm"))
2654 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_NOSRC);
2656 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, pool_strn2id(pool, sarch, strlen(sarch) - 4, 1));
2660 repodata_set_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
2666 key.type = REPOKEY_TYPE_IDARRAY;
2668 key.storage = KEY_STORAGE_INCORE;
2669 repodata_set(data, solvid, &key, data->attriddatalen);
2670 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, q->count + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2671 for (i = 0; i < q->count; i++)
2672 data->attriddata[data->attriddatalen++] = q->elements[i];
2673 data->attriddata[data->attriddatalen++] = 0;
2677 repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2681 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2683 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2684 data->attriddata[data->attriddatalen++] = dir;
2685 data->attriddata[data->attriddatalen++] = num;
2686 data->attriddata[data->attriddatalen++] = num2;
2687 data->attriddata[data->attriddatalen++] = 0;
2691 repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
2697 l = strlen(str) + 1;
2698 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2699 memcpy(data->attrdata + data->attrdatalen, str, l);
2700 stroff = data->attrdatalen;
2701 data->attrdatalen += l;
2704 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2706 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2707 data->attriddata[data->attriddatalen++] = dir;
2708 data->attriddata[data->attriddatalen++] = stroff;
2709 data->attriddata[data->attriddatalen++] = 0;
2713 repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2716 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2718 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2719 data->attriddata[data->attriddatalen++] = id;
2720 data->attriddata[data->attriddatalen++] = 0;
2724 repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2728 if (data->localpool)
2729 id = stringpool_str2id(&data->spool, str, 1);
2731 id = pool_str2id(data->repo->pool, str, 1);
2732 repodata_add_idarray(data, solvid, keyname, id);
2736 repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2738 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2739 data->attriddata[data->attriddatalen++] = ghandle;
2740 data->attriddata[data->attriddatalen++] = 0;
2744 repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2746 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2747 data->attriddata[data->attriddatalen++] = ghandle;
2748 data->attriddata[data->attriddatalen++] = 0;
2752 repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname)
2755 app = repodata_get_attrp(data, solvid);
2759 for (; *ap; ap += 2)
2760 if (data->keys[*ap].name == keyname)
2766 for (; *ap; ap += 2)
2768 if (data->keys[*ap].name == keyname)
2776 /* XXX: does not work correctly, needs fix in iterators! */
2778 repodata_unset(Repodata *data, Id solvid, Id keyname)
2782 key.type = REPOKEY_TYPE_DELETED;
2784 key.storage = KEY_STORAGE_INCORE;
2785 repodata_set(data, solvid, &key, 0);
2788 /* add all (uninternalized) attrs from src to dest */
2790 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2793 if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
2795 for (; *keyp; keyp += 2)
2796 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2799 /* add some (uninternalized) attrs from src to dest */
2801 repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
2804 if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
2806 for (; *keyp; keyp += 2)
2807 if (!keyidmap || MAPTST(keyidmap, keyp[0]))
2808 repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2811 /* swap (uninternalized) attrs from src and dest */
2813 repodata_swap_attrs(Repodata *data, Id dest, Id src)
2816 if (!data->attrs || dest == src)
2818 if (dest < data->start || dest >= data->end)
2819 repodata_extend(data, dest);
2820 if (src < data->start || src >= data->end)
2821 repodata_extend(data, src);
2822 tmpattrs = data->attrs[dest - data->start];
2823 data->attrs[dest - data->start] = data->attrs[src - data->start];
2824 data->attrs[src - data->start] = tmpattrs;
2828 /**********************************************************************/
2830 /* TODO: unify with repo_write and repo_solv! */
2832 #define EXTDATA_BLOCK 1023
2840 data_addid(struct extdata *xd, Id sx)
2842 unsigned int x = (unsigned int)sx;
2845 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2846 dp = xd->buf + xd->len;
2851 *dp++ = (x >> 28) | 128;
2853 *dp++ = (x >> 21) | 128;
2854 *dp++ = (x >> 14) | 128;
2857 *dp++ = (x >> 7) | 128;
2859 xd->len = dp - xd->buf;
2863 data_addid64(struct extdata *xd, unsigned long long x)
2865 if (x >= 0x100000000)
2869 data_addid(xd, (Id)(x >> 35));
2870 xd->buf[xd->len - 1] |= 128;
2872 data_addid(xd, (Id)((unsigned int)x | 0x80000000));
2873 xd->buf[xd->len - 5] = (x >> 28) | 128;
2876 data_addid(xd, (Id)x);
2880 data_addideof(struct extdata *xd, Id sx, int eof)
2882 unsigned int x = (unsigned int)sx;
2885 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2886 dp = xd->buf + xd->len;
2891 *dp++ = (x >> 27) | 128;
2893 *dp++ = (x >> 20) | 128;
2894 *dp++ = (x >> 13) | 128;
2897 *dp++ = (x >> 6) | 128;
2898 *dp++ = eof ? (x & 63) : (x & 63) | 64;
2899 xd->len = dp - xd->buf;
2903 data_addblob(struct extdata *xd, unsigned char *blob, int len)
2905 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
2906 memcpy(xd->buf + xd->len, blob, len);
2910 /*********************************/
2912 /* this is to reduct memory usage when internalizing oversized repos */
2914 compact_attrdata(Repodata *data, int entry, int nentry)
2917 unsigned int attrdatastart = data->attrdatalen;
2918 unsigned int attriddatastart = data->attriddatalen;
2919 if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
2921 for (i = entry; i < nentry; i++)
2923 Id v, *attrs = data->attrs[i];
2926 for (; *attrs; attrs += 2)
2928 switch (data->keys[*attrs].type)
2930 case REPOKEY_TYPE_STR:
2931 case REPOKEY_TYPE_BINARY:
2933 if ((unsigned int)attrs[1] < attrdatastart)
2934 attrdatastart = attrs[1];
2936 case REPOKEY_TYPE_DIRSTRARRAY:
2937 for (v = attrs[1]; data->attriddata[v] ; v += 2)
2938 if (data->attriddata[v + 1] < attrdatastart)
2939 attrdatastart = data->attriddata[v + 1];
2941 case REPOKEY_TYPE_IDARRAY:
2942 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2943 if ((unsigned int)attrs[1] < attriddatastart)
2944 attriddatastart = attrs[1];
2946 case REPOKEY_TYPE_FIXARRAY:
2947 case REPOKEY_TYPE_FLEXARRAY:
2955 printf("compact_attrdata %d %d\n", entry, nentry);
2956 printf("attrdatastart: %d\n", attrdatastart);
2957 printf("attriddatastart: %d\n", attriddatastart);
2959 if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
2961 for (i = entry; i < nentry; i++)
2963 Id v, *attrs = data->attrs[i];
2966 for (; *attrs; attrs += 2)
2968 switch (data->keys[*attrs].type)
2970 case REPOKEY_TYPE_STR:
2971 case REPOKEY_TYPE_BINARY:
2973 attrs[1] -= attrdatastart;
2975 case REPOKEY_TYPE_DIRSTRARRAY:
2976 for (v = attrs[1]; data->attriddata[v] ; v += 2)
2977 data->attriddata[v + 1] -= attrdatastart;
2979 case REPOKEY_TYPE_IDARRAY:
2980 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2981 attrs[1] -= attriddatastart;
2990 data->attrdatalen -= attrdatastart;
2991 memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
2992 data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
2994 if (attriddatastart)
2996 data->attriddatalen -= attriddatastart;
2997 memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
2998 data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
3002 /* internalalize some key into incore/vincore data */
3005 repodata_serialize_key(Repodata *data, struct extdata *newincore,
3006 struct extdata *newvincore,
3008 Repokey *key, Id val)
3012 unsigned int oldvincorelen = 0;
3016 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3019 oldvincorelen = xd->len;
3023 case REPOKEY_TYPE_VOID:
3024 case REPOKEY_TYPE_CONSTANT:
3025 case REPOKEY_TYPE_CONSTANTID:
3027 case REPOKEY_TYPE_STR:
3028 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
3030 case REPOKEY_TYPE_MD5:
3031 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
3033 case REPOKEY_TYPE_SHA1:
3034 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
3036 case REPOKEY_TYPE_SHA224:
3037 data_addblob(xd, data->attrdata + val, SIZEOF_SHA224);
3039 case REPOKEY_TYPE_SHA256:
3040 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
3042 case REPOKEY_TYPE_SHA384:
3043 data_addblob(xd, data->attrdata + val, SIZEOF_SHA384);
3045 case REPOKEY_TYPE_SHA512:
3046 data_addblob(xd, data->attrdata + val, SIZEOF_SHA512);
3048 case REPOKEY_TYPE_NUM:
3049 if (val & 0x80000000)
3051 data_addid64(xd, data->attrnum64data[val ^ 0x80000000]);
3055 case REPOKEY_TYPE_ID:
3056 case REPOKEY_TYPE_DIR:
3057 data_addid(xd, val);
3059 case REPOKEY_TYPE_BINARY:
3062 unsigned char *dp = data_read_id(data->attrdata + val, &len);
3063 dp += (unsigned int)len;
3064 data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
3067 case REPOKEY_TYPE_IDARRAY:
3068 for (ida = data->attriddata + val; *ida; ida++)
3069 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
3071 case REPOKEY_TYPE_DIRNUMNUMARRAY:
3072 for (ida = data->attriddata + val; *ida; ida += 3)
3074 data_addid(xd, ida[0]);
3075 data_addid(xd, ida[1]);
3076 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
3079 case REPOKEY_TYPE_DIRSTRARRAY:
3080 for (ida = data->attriddata + val; *ida; ida += 2)
3082 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
3083 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
3086 case REPOKEY_TYPE_FIXARRAY:
3090 for (ida = data->attriddata + val; *ida; ida++)
3094 kp = data->xattrs[-*ida];
3102 schemaid = repodata_schema2id(data, schema, 1);
3103 else if (schemaid != repodata_schema2id(data, schema, 0))
3105 pool_debug(data->repo->pool, SOLV_FATAL, "fixarray substructs with different schemas\n");
3111 data_addid(xd, num);
3112 data_addid(xd, schemaid);
3113 for (ida = data->attriddata + val; *ida; ida++)
3115 Id *kp = data->xattrs[-*ida];
3119 repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3123 case REPOKEY_TYPE_FLEXARRAY:
3126 for (ida = data->attriddata + val; *ida; ida++)
3128 data_addid(xd, num);
3129 for (ida = data->attriddata + val; *ida; ida++)
3131 Id *kp = data->xattrs[-*ida];
3134 data_addid(xd, 0); /* XXX */
3141 schemaid = repodata_schema2id(data, schema, 1);
3142 data_addid(xd, schemaid);
3143 kp = data->xattrs[-*ida];
3145 repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3150 pool_debug(data->repo->pool, SOLV_FATAL, "don't know how to handle type %d\n", key->type);
3153 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3155 /* put offset/len in incore */
3156 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
3157 oldvincorelen = xd->len - oldvincorelen;
3158 data_addid(newincore, oldvincorelen);
3163 repodata_internalize(Repodata *data)
3165 Repokey *key, solvkey;
3167 Id schemaid, keyid, *schema, *sp, oldschema, *keyp, *seen;
3169 unsigned char *dp, *ndp;
3170 int newschema, oldcount;
3171 struct extdata newincore;
3172 struct extdata newvincore;
3175 if (!data->attrs && !data->xattrs)
3179 printf("repodata_internalize %d\n", data->repodataid);
3180 printf(" attr data: %d K\n", data->attrdatalen / 1024);
3181 printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3183 newvincore.buf = data->vincore;
3184 newvincore.len = data->vincorelen;
3186 /* find the solvables key, create if needed */
3187 memset(&solvkey, 0, sizeof(solvkey));
3188 solvkey.name = REPOSITORY_SOLVABLES;
3189 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
3191 solvkey.storage = KEY_STORAGE_INCORE;
3192 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
3194 schema = solv_malloc2(data->nkeys, sizeof(Id));
3195 seen = solv_malloc2(data->nkeys, sizeof(Id));
3197 /* Merge the data already existing (in data->schemata, ->incoredata and
3198 friends) with the new attributes in data->attrs[]. */
3199 nentry = data->end - data->start;
3200 memset(&newincore, 0, sizeof(newincore));
3201 data_addid(&newincore, 0); /* start data at offset 1 */
3203 data->mainschema = 0;
3204 data->mainschemaoffsets = solv_free(data->mainschemaoffsets);
3206 /* join entry data */
3207 /* we start with the meta data, entry -1 */
3208 for (entry = -1; entry < nentry; entry++)
3210 memset(seen, 0, data->nkeys * sizeof(Id));
3212 dp = data->incoredata;
3215 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
3216 dp = data_read_id(dp, &oldschema);
3219 fprintf(stderr, "oldschema %d\n", oldschema);
3220 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
3221 fprintf(stderr, "schemadata %p\n", data->schemadata);
3223 /* seen: -1: old data 0: skipped >0: id + 1 */
3227 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
3231 pool_debug(data->repo->pool, SOLV_FATAL, "Inconsistent old data (key occured twice).\n");
3239 keyp = data->attrs ? data->attrs[entry] : 0;
3242 /* strip solvables key */
3244 for (sp = keyp = schema; *sp; sp++)
3245 if (*sp != solvkeyid)
3250 seen[solvkeyid] = 0;
3251 keyp = data->xattrs ? data->xattrs[1] : 0;
3254 for (; *keyp; keyp += 2)
3261 seen[*keyp] = keyp[1] + 1;
3263 if (entry < 0 && data->end != data->start)
3270 /* Ideally we'd like to sort the new schema here, to ensure
3271 schema equality independend of the ordering. We can't do that
3272 yet. For once see below (old ids need to come before new ids).
3273 An additional difficulty is that we also need to move
3274 the values with the keys. */
3275 schemaid = repodata_schema2id(data, schema, 1);
3277 schemaid = oldschema;
3280 /* Now create data blob. We walk through the (possibly new) schema
3281 and either copy over old data, or insert the new. */
3282 /* XXX Here we rely on the fact that the (new) schema has the form
3283 o1 o2 o3 o4 ... | n1 n2 n3 ...
3284 (oX being the old keyids (possibly overwritten), and nX being
3285 the new keyids). This rules out sorting the keyids in order
3286 to ensure a small schema count. */
3288 data->incoreoffset[entry] = newincore.len;
3289 data_addid(&newincore, schemaid);
3292 data->mainschema = schemaid;
3293 data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
3295 /* we don't use a pointer to the schemadata here as repodata_serialize_key
3296 * may call repodata_schema2id() which might realloc our schemadata */
3297 for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
3300 data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
3301 if (keyid == solvkeyid)
3303 /* add flexarray entry count */
3304 data_addid(&newincore, data->end - data->start);
3307 key = data->keys + keyid;
3309 fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, pool_id2str(data->repo->pool, key->name), pool_id2str(data->repo->pool, key->type));
3314 /* Skip the data associated with this old key. */
3315 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3317 ndp = data_skip(dp, REPOKEY_TYPE_ID);
3318 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
3320 else if (key->storage == KEY_STORAGE_INCORE)
3321 ndp = data_skip_key(data, dp, key);
3324 if (seen[keyid] == -1)
3326 /* If this key was an old one _and_ was not overwritten with
3327 a different value copy over the old value (we skipped it
3330 data_addblob(&newincore, dp, ndp - dp);
3333 else if (seen[keyid])
3335 /* Otherwise we have a new value. Parse it into the internal form. */
3336 repodata_serialize_key(data, &newincore, &newvincore, schema, key, seen[keyid] - 1);
3340 if (entry >= 0 && data->attrs)
3342 if (data->attrs[entry])
3343 data->attrs[entry] = solv_free(data->attrs[entry]);
3344 if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
3346 compact_attrdata(data, entry + 1, nentry); /* try to free some memory */
3348 printf(" attr data: %d K\n", data->attrdatalen / 1024);
3349 printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3350 printf(" incore data: %d K\n", newincore.len / 1024);
3351 printf(" sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
3352 /* malloc_stats(); */
3357 /* free all xattrs */
3358 for (entry = 0; entry < data->nxattrs; entry++)
3359 if (data->xattrs[entry])
3360 solv_free(data->xattrs[entry]);
3361 data->xattrs = solv_free(data->xattrs);
3364 data->lasthandle = 0;
3366 data->lastdatalen = 0;
3369 repodata_free_schemahash(data);
3371 solv_free(data->incoredata);
3372 data->incoredata = newincore.buf;
3373 data->incoredatalen = newincore.len;
3374 data->incoredatafree = 0;
3376 solv_free(data->vincore);
3377 data->vincore = newvincore.buf;
3378 data->vincorelen = newvincore.len;
3380 data->attrs = solv_free(data->attrs);
3381 data->attrdata = solv_free(data->attrdata);
3382 data->attriddata = solv_free(data->attriddata);
3383 data->attrnum64data = solv_free(data->attrnum64data);
3384 data->attrdatalen = 0;
3385 data->attriddatalen = 0;
3386 data->attrnum64datalen = 0;
3388 printf("repodata_internalize %d done\n", data->repodataid);
3389 printf(" incore data: %d K\n", data->incoredatalen / 1024);
3394 repodata_disable_paging(Repodata *data)
3396 if (maybe_load_repodata(data, 0))
3398 repopagestore_disable_paging(&data->store);
3404 repodata_load_stub(Repodata *data)
3406 Repo *repo = data->repo;
3407 Pool *pool = repo->pool;
3409 struct _Pool_tmpspace oldtmpspace;
3412 if (!pool->loadcallback)
3414 data->state = REPODATA_ERROR;
3417 data->state = REPODATA_LOADING;
3419 /* save tmp space and pos */
3420 oldtmpspace = pool->tmpspace;
3421 memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
3424 r = pool->loadcallback(pool, data, pool->loadcallbackdata);
3426 /* restore tmp space and pos */
3427 for (i = 0; i < POOL_TMPSPACEBUF; i++)
3428 solv_free(pool->tmpspace.buf[i]);
3429 pool->tmpspace = oldtmpspace;
3430 if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
3431 memset(&oldpos, 0, sizeof(oldpos));
3434 data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR;
3438 repodata_add_stubkey(Repodata *data, Id keyname, Id keytype)
3442 xkey.name = keyname;
3443 xkey.type = keytype;
3444 xkey.storage = KEY_STORAGE_INCORE;
3446 repodata_key2id(data, &xkey, 1);
3450 repodata_add_stub(Repodata **datap)
3452 Repodata *data = *datap;
3453 Repo *repo = data->repo;
3454 Id repodataid = data - repo->repodata;
3455 Repodata *sdata = repo_add_repodata(repo, 0);
3456 data = repo->repodata + repodataid;
3457 if (data->end > data->start)
3458 repodata_extend_block(sdata, data->start, data->end - data->start);
3459 sdata->state = REPODATA_STUB;
3460 sdata->loadcallback = repodata_load_stub;
3466 repodata_create_stubs(Repodata *data)
3468 Repo *repo = data->repo;
3469 Pool *pool = repo->pool;
3476 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
3477 while (dataiterator_step(&di))
3478 if (di.data == data)
3480 dataiterator_free(&di);
3483 stubdataids = solv_calloc(cnt, sizeof(*stubdataids));
3484 for (i = 0; i < cnt; i++)
3486 sdata = repodata_add_stub(&data);
3487 stubdataids[i] = sdata - repo->repodata;
3490 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
3492 while (dataiterator_step(&di))
3494 if (di.data != data)
3496 if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
3498 dataiterator_entersub(&di);
3499 sdata = repo->repodata + stubdataids[i++];
3503 switch (di.key->type)
3505 case REPOKEY_TYPE_ID:
3506 repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
3508 case REPOKEY_TYPE_CONSTANTID:
3509 repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
3511 case REPOKEY_TYPE_STR:
3512 repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
3514 case REPOKEY_TYPE_VOID:
3515 repodata_set_void(sdata, SOLVID_META, di.key->name);
3517 case REPOKEY_TYPE_NUM:
3518 repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv));
3521 repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
3523 case REPOKEY_TYPE_IDARRAY:
3524 repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
3525 if (di.key->name == REPOSITORY_KEYS)
3530 xkeyname = di.kv.id;
3534 repodata_add_stubkey(sdata, xkeyname, di.kv.id);
3543 dataiterator_free(&di);
3544 for (i = 0; i < cnt; i++)
3545 repodata_internalize(repo->repodata + stubdataids[i]);
3546 solv_free(stubdataids);
3551 repodata_memused(Repodata *data)
3553 return data->incoredatalen + data->vincorelen;