2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
13 * a repository can contain multiple repodata entries, consisting of
14 * different sets of keys and different sets of solvables
29 #include "poolid_private.h"
37 #define REPODATA_BLOCK 255
39 static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);
42 repodata_initdata(Repodata *data, Repo *repo, int localpool)
44 memset(data, 0, sizeof (*data));
45 data->repodataid = data - repo->repodata;
47 data->localpool = localpool;
49 stringpool_init_empty(&data->spool);
50 /* dirpool_init(&data->dirpool); just zeros out again */
51 data->keys = solv_calloc(1, sizeof(Repokey));
53 data->schemata = solv_calloc(1, sizeof(Id));
54 data->schemadata = solv_calloc(1, sizeof(Id));
56 data->schemadatalen = 1;
57 repopagestore_init(&data->store);
61 repodata_freedata(Repodata *data)
65 solv_free(data->keys);
67 solv_free(data->schemata);
68 solv_free(data->schemadata);
69 solv_free(data->schematahash);
71 stringpool_free(&data->spool);
72 dirpool_free(&data->dirpool);
74 solv_free(data->mainschemaoffsets);
75 solv_free(data->incoredata);
76 solv_free(data->incoreoffset);
77 solv_free(data->verticaloffset);
79 repopagestore_free(&data->store);
81 solv_free(data->vincore);
84 for (i = 0; i < data->end - data->start; i++)
85 solv_free(data->attrs[i]);
86 solv_free(data->attrs);
88 for (i = 0; i < data->nxattrs; i++)
89 solv_free(data->xattrs[i]);
90 solv_free(data->xattrs);
92 solv_free(data->attrdata);
93 solv_free(data->attriddata);
94 solv_free(data->attrnum64data);
96 solv_free(data->dircache);
100 repodata_free(Repodata *data)
102 Repo *repo = data->repo;
103 int i = data - repo->repodata;
106 repodata_freedata(data);
107 if (i < repo->nrepodata - 1)
109 /* whoa! this changes the repodataids! */
110 memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
111 for (; i < repo->nrepodata - 1; i++)
112 repo->repodata[i].repodataid = i;
115 if (repo->nrepodata == 1)
117 repo->repodata = solv_free(repo->repodata);
123 repodata_empty(Repodata *data, int localpool)
125 void (*loadcallback)(Repodata *) = data->loadcallback;
126 int state = data->state;
127 repodata_freedata(data);
128 repodata_initdata(data, data->repo, localpool);
130 data->loadcallback = loadcallback;
134 /***************************************************************
135 * key pool management
138 /* this is not so time critical that we need a hash, so we do a simple
141 repodata_key2id(Repodata *data, Repokey *key, int create)
145 for (keyid = 1; keyid < data->nkeys; keyid++)
146 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
148 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
152 if (keyid == data->nkeys)
156 /* allocate new key */
157 data->keys = solv_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
158 data->keys[data->nkeys++] = *key;
159 if (data->verticaloffset)
161 data->verticaloffset = solv_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
162 data->verticaloffset[data->nkeys - 1] = 0;
164 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
170 /***************************************************************
171 * schema pool management
174 #define SCHEMATA_BLOCK 31
175 #define SCHEMATADATA_BLOCK 255
178 repodata_schema2id(Repodata *data, Id *schema, int create)
185 return 0; /* XXX: allow empty schema? */
186 if ((schematahash = data->schematahash) == 0)
188 data->schematahash = schematahash = solv_calloc(256, sizeof(Id));
189 for (i = 1; i < data->nschemata; i++)
191 for (sp = data->schemadata + data->schemata[i], h = 0; *sp;)
196 data->schemadata = solv_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
197 data->schemata = solv_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
200 for (sp = schema, len = 0, h = 0; *sp; len++)
205 cid = schematahash[h];
208 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
210 /* cache conflict, do a slow search */
211 for (cid = 1; cid < data->nschemata; cid++)
212 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
218 data->schemadata = solv_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
219 data->schemata = solv_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
221 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
222 data->schemata[data->nschemata] = data->schemadatalen;
223 data->schemadatalen += len;
224 schematahash[h] = data->nschemata;
226 fprintf(stderr, "schema2id: new schema\n");
228 return data->nschemata++;
232 repodata_free_schemahash(Repodata *data)
234 data->schematahash = solv_free(data->schematahash);
236 data->schemata = solv_realloc2(data->schemata, data->nschemata, sizeof(Id));
237 data->schemadata = solv_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
241 /***************************************************************
242 * dir pool management
245 #ifndef HAVE_STRCHRNUL
246 static inline const char *strchrnul(const char *str, char x)
248 const char *p = strchr(str, x);
249 return p ? p : str + strlen(str);
253 #define DIRCACHE_SIZE 41 /* < 1k */
257 Id ids[DIRCACHE_SIZE];
258 char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
263 repodata_str2dir(Repodata *data, const char *dir, int create)
274 while (*dir == '/' && dir[1] == '/')
276 if (*dir == '/' && !dir[1])
278 if (data->dirpool.ndirs)
280 return dirpool_add_dir(&data->dirpool, 0, 1, create);
287 struct dircache *dircache = data->dircache;
291 if (l < DIRCACHE_SIZE && dircache->ids[l] && !memcmp(dircache->str + l * (l - 1) / 2, dir, l))
293 parent = dircache->ids[l];
309 dire = strchrnul(dir, '/');
311 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
313 id = pool_strn2id(data->repo->pool, dir, dire - dir, create);
316 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
321 data->dircache = solv_calloc(1, sizeof(struct dircache));
325 if (l < DIRCACHE_SIZE)
327 data->dircache->ids[l] = parent;
328 memcpy(data->dircache->str + l * (l - 1) / 2, dirs, l);
342 repodata_free_dircache(Repodata *data)
344 data->dircache = solv_free(data->dircache);
348 repodata_dir2str(Repodata *data, Id did, const char *suf)
350 Pool *pool = data->repo->pool;
357 return suf ? suf : "";
361 comp = dirpool_compid(&data->dirpool, parent);
362 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
364 parent = dirpool_parent(&data->dirpool, parent);
369 l += strlen(suf) + 1;
370 p = pool_alloctmpspace(pool, l + 1) + l;
381 comp = dirpool_compid(&data->dirpool, parent);
382 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
385 strncpy(p, comps, l);
386 parent = dirpool_parent(&data->dirpool, parent);
394 /***************************************************************
398 static inline unsigned char *
399 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
401 Id *keyp = data->schemadata + data->schemata[schema];
402 for (; *keyp; keyp++)
403 dp = data_skip_key(data, dp, data->keys + *keyp);
407 static unsigned char *
408 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
410 int nentries, schema;
413 case REPOKEY_TYPE_FIXARRAY:
414 dp = data_read_id(dp, &nentries);
417 dp = data_read_id(dp, &schema);
419 dp = data_skip_schema(data, dp, schema);
421 case REPOKEY_TYPE_FLEXARRAY:
422 dp = data_read_id(dp, &nentries);
425 dp = data_read_id(dp, &schema);
426 dp = data_skip_schema(data, dp, schema);
430 if (key->storage == KEY_STORAGE_INCORE)
431 dp = data_skip(dp, key->type);
432 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
434 dp = data_skip(dp, REPOKEY_TYPE_ID);
435 dp = data_skip(dp, REPOKEY_TYPE_ID);
441 static unsigned char *
442 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
448 if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
451 for (i = 0; (k = *keyp++) != 0; i++)
453 return data->incoredata + data->mainschemaoffsets[i];
456 while ((k = *keyp++) != 0)
460 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
462 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
463 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
466 if (data->keys[k].storage != KEY_STORAGE_INCORE)
468 dp = data_skip_key(data, dp, data->keys + k);
473 static unsigned char *
474 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
479 if (off >= data->lastverticaloffset)
481 off -= data->lastverticaloffset;
482 if (off + len > data->vincorelen)
484 return data->vincore + off;
486 if (off + len > key->size)
488 /* we now have the offset, go into vertical */
489 off += data->verticaloffset[key - data->keys];
490 /* fprintf(stderr, "key %d page %d\n", key->name, off / REPOPAGE_BLOBSIZE); */
491 dp = repopagestore_load_page_range(&data->store, off / REPOPAGE_BLOBSIZE, (off + len - 1) / REPOPAGE_BLOBSIZE);
494 dp += off % REPOPAGE_BLOBSIZE;
498 static inline unsigned char *
499 get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
501 unsigned char *dp = *dpp;
505 if (key->storage == KEY_STORAGE_INCORE)
508 *dpp = data_skip_key(data, dp, key);
511 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
514 dp = data_read_id(dp, &off);
515 dp = data_read_id(dp, &len);
518 return get_vertical_data(data, key, off, len);
524 load_repodata(Repodata *data)
526 if (data->loadcallback)
528 data->loadcallback(data);
529 if (data->state == REPODATA_AVAILABLE)
532 data->state = REPODATA_ERROR;
537 maybe_load_repodata(Repodata *data, Id keyname)
539 if (keyname && !repodata_precheck_keyname(data, keyname))
540 return 0; /* do not bother... */
547 for (i = 1; i < data->nkeys; i++)
548 if (keyname == data->keys[i].name)
550 if (i == data->nkeys)
553 return load_repodata(data);
556 case REPODATA_AVAILABLE:
557 case REPODATA_LOADING:
560 data->state = REPODATA_ERROR;
565 static inline unsigned char *
566 solvid2data(Repodata *data, Id solvid, Id *schemap)
568 unsigned char *dp = data->incoredata;
571 if (solvid == SOLVID_META)
572 dp += 1; /* offset of "meta" solvable */
573 else if (solvid == SOLVID_POS)
575 Pool *pool = data->repo->pool;
576 if (data->repo != pool->pos.repo)
578 if (data != data->repo->repodata + pool->pos.repodataid)
581 if (pool->pos.dp != 1)
583 *schemap = pool->pos.schema;
589 if (solvid < data->start || solvid >= data->end)
591 dp += data->incoreoffset[solvid - data->start];
593 return data_read_id(dp, schemap);
596 /************************************************************************
600 static unsigned char *
601 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
604 Id schema, *keyp, *kp;
607 if (!maybe_load_repodata(data, keyname))
609 dp = solvid2data(data, solvid, &schema);
612 keyp = data->schemadata + data->schemata[schema];
613 for (kp = keyp; *kp; kp++)
614 if (data->keys[*kp].name == keyname)
618 *keypp = key = data->keys + *kp;
619 if (key->type == REPOKEY_TYPE_DELETED)
621 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
622 return dp; /* no need to forward... */
623 if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
624 return 0; /* get_data will not work, no need to forward */
625 dp = forward_to_key(data, *kp, keyp, dp);
628 return get_data(data, key, &dp, 0);
632 repodata_lookup_type(Repodata *data, Id solvid, Id keyname)
634 Id schema, *keyp, *kp;
635 if (!maybe_load_repodata(data, keyname))
637 if (!solvid2data(data, solvid, &schema))
639 keyp = data->schemadata + data->schemata[schema];
640 for (kp = keyp; *kp; kp++)
641 if (data->keys[*kp].name == keyname)
642 return data->keys[*kp].type;
647 repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
653 dp = find_key_data(data, solvid, keyname, &key);
656 if (key->type == REPOKEY_TYPE_CONSTANTID)
658 if (key->type != REPOKEY_TYPE_ID)
660 dp = data_read_id(dp, &id);
665 repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
671 dp = find_key_data(data, solvid, keyname, &key);
674 if (key->type == REPOKEY_TYPE_STR)
675 return (const char *)dp;
676 if (key->type == REPOKEY_TYPE_CONSTANTID)
678 else if (key->type == REPOKEY_TYPE_ID)
679 dp = data_read_id(dp, &id);
683 return stringpool_id2str(&data->spool, id);
684 return pool_id2str(data->repo->pool, id);
688 repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long *value)
692 unsigned int high, low;
695 dp = find_key_data(data, solvid, keyname, &key);
700 case REPOKEY_TYPE_NUM:
701 data_read_num64(dp, &low, &high);
702 *value = (unsigned long long)high << 32 | low;
704 case REPOKEY_TYPE_U32:
705 data_read_u32(dp, &low);
708 case REPOKEY_TYPE_CONSTANT:
717 repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
723 if (!maybe_load_repodata(data, keyname))
725 dp = solvid2data(data, solvid, &schema);
728 /* can't use find_key_data as we need to test the type */
729 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
730 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
735 const unsigned char *
736 repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
741 dp = find_key_data(data, solvid, keyname, &key);
756 repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
764 dp = find_key_data(data, solvid, keyname, &key);
767 if (key->type != REPOKEY_TYPE_IDARRAY)
771 dp = data_read_ideof(dp, &id, &eof);
780 repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp)
786 dp = find_key_data(data, solvid, keyname, &key);
787 if (!dp || key->type != REPOKEY_TYPE_BINARY)
792 dp = data_read_id(dp, &len);
798 repodata_globalize_id(Repodata *data, Id id, int create)
800 if (!id || !data || !data->localpool)
802 return pool_str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
806 repodata_localize_id(Repodata *data, Id id, int create)
808 if (!id || !data || !data->localpool)
810 return stringpool_str2id(&data->spool, pool_id2str(data->repo->pool, id), create);
814 repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create)
816 if (!id || !data || !fromdata)
818 if (!data->localpool || !fromdata->localpool)
820 if (fromdata->localpool)
821 id = repodata_globalize_id(fromdata, id, create);
823 id = repodata_localize_id(data, id, create);
826 /* localpool is set in both data and fromdata */
827 return stringpool_str2id(&data->spool, stringpool_id2str(&fromdata->spool, id), create);
831 repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id voidid)
836 ap = data->attrs[solvid - data->start];
841 if (data->keys[*ap].name != keyname)
843 if (data->keys[*ap].type == REPOKEY_TYPE_VOID)
845 if (data->keys[*ap].type == REPOKEY_TYPE_ID)
853 /************************************************************************
859 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
863 case REPOKEY_TYPE_ID:
864 case REPOKEY_TYPE_CONSTANTID:
865 case REPOKEY_TYPE_IDARRAY:
866 if (data && data->localpool)
867 kv->str = stringpool_id2str(&data->spool, kv->id);
869 kv->str = pool_id2str(pool, kv->id);
870 if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
873 for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
875 if (*s == ':' && s > kv->str)
879 case REPOKEY_TYPE_STR:
881 case REPOKEY_TYPE_DIRSTRARRAY:
882 if (!(flags & SEARCH_FILES))
883 return kv->str; /* match just the basename */
885 return kv->str; /* already stringified */
886 /* Put the full filename into kv->str. */
887 kv->str = repodata_dir2str(data, kv->id, kv->str);
888 kv->num = 1; /* mark stringification */
891 if (!(flags & SEARCH_CHECKSUMS))
892 return 0; /* skip em */
894 return kv->str; /* already stringified */
895 kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
896 kv->num = 1; /* mark stringification */
904 struct subschema_data {
910 /* search a specific repodata */
912 repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
916 Id keyid, *kp, *keyp;
917 unsigned char *dp, *ddp;
923 if (!maybe_load_repodata(data, keyname))
925 if (solvid == SOLVID_SUBSCHEMA)
927 struct subschema_data *subd = cbdata;
928 cbdata = subd->cbdata;
930 schema = subd->parent->id;
931 dp = (unsigned char *)subd->parent->str;
932 kv.parent = subd->parent;
937 dp = solvid2data(data, solvid, &schema);
940 s = data->repo->pool->solvables + solvid;
943 keyp = data->schemadata + data->schemata[schema];
946 /* search for a specific key */
947 for (kp = keyp; *kp; kp++)
948 if (data->keys[*kp].name == keyname)
952 dp = forward_to_key(data, *kp, keyp, dp);
958 while ((keyid = *keyp++) != 0)
961 key = data->keys + keyid;
962 ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
964 if (key->type == REPOKEY_TYPE_DELETED)
966 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
968 struct subschema_data subd;
972 subd.cbdata = cbdata;
975 ddp = data_read_id(ddp, &nentries);
979 while (ddp && nentries > 0)
983 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
984 ddp = data_read_id(ddp, &schema);
986 kv.str = (char *)ddp;
987 stop = callback(cbdata, s, data, key, &kv);
988 if (stop > SEARCH_NEXT_KEY)
990 if (stop && stop != SEARCH_ENTERSUB)
992 if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
993 repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
994 ddp = data_skip_schema(data, ddp, schema);
997 if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
1001 kv.str = (char *)ddp;
1002 stop = callback(cbdata, s, data, key, &kv);
1003 if (stop > SEARCH_NEXT_KEY)
1013 ddp = data_fetch(ddp, &kv, key);
1016 stop = callback(cbdata, s, data, key, &kv);
1019 while (!kv.eof && !stop);
1020 if (onekey || stop > SEARCH_NEXT_KEY)
1026 repodata_setpos_kv(Repodata *data, KeyValue *kv)
1028 Pool *pool = data->repo->pool;
1030 pool_clear_pos(pool);
1033 pool->pos.repo = data->repo;
1034 pool->pos.repodataid = data - data->repo->repodata;
1035 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
1036 pool->pos.schema = kv->id;
1040 /************************************************************************
1041 * data iterator functions
1045 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
1059 case SOLVABLE_VENDOR:
1062 case SOLVABLE_PROVIDES:
1064 return s->provides ? s->repo->idarraydata + s->provides : 0;
1065 case SOLVABLE_OBSOLETES:
1067 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
1068 case SOLVABLE_CONFLICTS:
1070 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
1071 case SOLVABLE_REQUIRES:
1073 return s->requires ? s->repo->idarraydata + s->requires : 0;
1074 case SOLVABLE_RECOMMENDS:
1076 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
1077 case SOLVABLE_SUPPLEMENTS:
1079 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
1080 case SOLVABLE_SUGGESTS:
1082 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
1083 case SOLVABLE_ENHANCES:
1085 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
1088 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
1095 datamatcher_init(Datamatcher *ma, const char *match, int flags)
1097 match = match ? solv_strdup(match) : 0;
1102 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
1104 ma->matchdata = solv_calloc(1, sizeof(regex_t));
1105 ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
1108 solv_free(ma->matchdata);
1109 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
1112 if ((flags & SEARCH_FILES) != 0 && match)
1114 /* prepare basename check */
1115 if ((flags & SEARCH_STRINGMASK) == SEARCH_STRING || (flags & SEARCH_STRINGMASK) == SEARCH_STRINGEND)
1117 const char *p = strrchr(match, '/');
1118 ma->matchdata = (void *)(p ? p + 1 : match);
1120 else if ((flags & SEARCH_STRINGMASK) == SEARCH_GLOB)
1123 for (p = match + strlen(match) - 1; p >= match; p--)
1124 if (*p == '[' || *p == ']' || *p == '*' || *p == '?' || *p == '/')
1126 ma->matchdata = (void *)(p + 1);
1133 datamatcher_free(Datamatcher *ma)
1136 ma->match = solv_free((char *)ma->match);
1137 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
1139 regfree(ma->matchdata);
1140 solv_free(ma->matchdata);
1146 datamatcher_match(Datamatcher *ma, const char *str)
1149 switch ((ma->flags & SEARCH_STRINGMASK))
1151 case SEARCH_SUBSTRING:
1152 if (ma->flags & SEARCH_NOCASE)
1153 return strcasestr(str, ma->match) != 0;
1155 return strstr(str, ma->match) != 0;
1157 if (ma->flags & SEARCH_NOCASE)
1158 return !strcasecmp(ma->match, str);
1160 return !strcmp(ma->match, str);
1161 case SEARCH_STRINGSTART:
1162 if (ma->flags & SEARCH_NOCASE)
1163 return !strncasecmp(ma->match, str, strlen(ma->match));
1165 return !strncmp(ma->match, str, strlen(ma->match));
1166 case SEARCH_STRINGEND:
1167 l = strlen(str) - strlen(ma->match);
1170 if (ma->flags & SEARCH_NOCASE)
1171 return !strcasecmp(ma->match, str + l);
1173 return !strcmp(ma->match, str + l);
1175 return !fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0);
1177 return !regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0);
1183 /* check if the matcher can match the provides basename */
1186 datamatcher_checkbasename(Datamatcher *ma, const char *basename)
1189 const char *match = ma->matchdata;
1192 switch (ma->flags & SEARCH_STRINGMASK)
1196 case SEARCH_STRINGEND:
1197 if (match != ma->match)
1198 break; /* had slash, do exact match on basename */
1201 /* check if the basename ends with match */
1202 l = strlen(basename) - strlen(match);
1208 return 1; /* maybe matches */
1210 if ((ma->flags & SEARCH_NOCASE) != 0)
1211 return !strcasecmp(match, basename);
1213 return !strcmp(match, basename);
1217 repodata_filelistfilter_matches(Repodata *data, const char *str)
1219 /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
1220 /* for now hardcoded */
1221 if (strstr(str, "bin/"))
1223 if (!strncmp(str, "/etc/", 5))
1225 if (!strcmp(str, "/usr/lib/sendmail"))
1247 di_nextarrayelement,
1253 di_entersolvablekey,
1257 /* see dataiterator.h for documentation */
1259 dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
1261 memset(di, 0, sizeof(*di));
1263 di->flags = flags & ~SEARCH_THISSOLVID;
1264 if (!pool || (repo && repo->pool != pool))
1272 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1278 di->keyname = keyname;
1279 di->keynames[0] = keyname;
1280 dataiterator_set_search(di, repo, p);
1285 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
1290 if (di->dupstr == di->kv.str)
1291 di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
1298 memset(&di->matcher, 0, sizeof(di->matcher));
1299 if (from->matcher.match)
1300 datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1305 for (i = 1; i < di->nparents; i++)
1306 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1307 di->kv.parent = &di->parents[di->nparents - 1].kv;
1312 dataiterator_set_match(Dataiterator *di, const char *match, int flags)
1314 di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
1315 datamatcher_free(&di->matcher);
1316 memset(&di->matcher, 0, sizeof(di->matcher));
1320 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1330 dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
1334 di->flags &= ~SEARCH_THISSOLVID;
1338 if (!di->pool->urepos)
1346 di->repo = di->pool->repos[di->repoid];
1348 di->state = di_enterrepo;
1350 dataiterator_jump_to_solvid(di, p);
1354 dataiterator_set_keyname(Dataiterator *di, Id keyname)
1357 di->keyname = keyname;
1358 di->keynames[0] = keyname;
1362 dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1366 if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1368 di->state = di_bye; /* sorry */
1371 for (i = di->nkeynames + 1; i > 0; i--)
1372 di->keynames[i] = di->keynames[i - 1];
1373 di->keynames[0] = di->keyname = keyname;
1378 dataiterator_free(Dataiterator *di)
1380 if (di->matcher.match)
1381 datamatcher_free(&di->matcher);
1383 solv_free(di->dupstr);
1386 static unsigned char *
1387 dataiterator_find_keyname(Dataiterator *di, Id keyname)
1390 Repokey *keys = di->data->keys, *key;
1393 for (keyp = di->keyp; *keyp; keyp++)
1394 if (keys[*keyp].name == keyname)
1399 if (key->type == REPOKEY_TYPE_DELETED)
1401 if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
1402 return 0; /* get_data will not work, no need to forward */
1403 dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1411 is_filelist_extension(Repodata *data)
1414 if (!repodata_precheck_keyname(data, SOLVABLE_FILELIST))
1416 for (j = 1; j < data->nkeys; j++)
1417 if (data->keys[j].name == SOLVABLE_FILELIST)
1419 if (j == data->nkeys)
1421 if (data->state != REPODATA_AVAILABLE)
1423 for (j = 1; j < data->nkeys; j++)
1424 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1430 dataiterator_filelistcheck(Dataiterator *di)
1433 int needcomplete = 0;
1434 Repodata *data = di->data;
1436 if ((di->flags & SEARCH_COMPLETE_FILELIST) != 0)
1437 if (!di->matcher.match
1438 || ((di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING
1439 && (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_GLOB)
1440 || !repodata_filelistfilter_matches(data, di->matcher.match))
1442 if (data->state != REPODATA_AVAILABLE)
1443 return needcomplete ? 1 : 0;
1446 /* we don't need the complete filelist, so ignore all stubs */
1447 for (j = 1; j < data->nkeys; j++)
1448 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1454 /* we need the complete filelist. check if we habe a filtered filelist and there's
1455 * a extension with the complete filelist later on */
1456 for (j = 1; j < data->nkeys; j++)
1457 if (data->keys[j].name == SOLVABLE_FILELIST)
1459 if (j == data->nkeys)
1460 return 0; /* does not have filelist */
1461 for (j = 1; j < data->nkeys; j++)
1462 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1464 if (j == data->nkeys)
1465 return 1; /* this is the externsion */
1466 while (data - data->repo->repodata + 1 < data->repo->nrepodata)
1469 if (is_filelist_extension(data))
1477 dataiterator_step(Dataiterator *di)
1481 if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate) {
1482 unsigned int ddpoff = di->ddp - di->vert_ddp;
1483 di->vert_off += ddpoff;
1484 di->vert_len -= ddpoff;
1485 di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len);
1486 di->vert_storestate = di->data->storestate;
1488 di->state = di_nextkey;
1494 case di_enterrepo: di_enterrepo:
1495 if (!di->repo || (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS)))
1497 if (!(di->flags & SEARCH_THISSOLVID))
1499 di->solvid = di->repo->start - 1; /* reset solvid iterator */
1500 goto di_nextsolvable;
1504 case di_entersolvable: di_entersolvable:
1507 di->repodataid = 1; /* reset repodata iterator */
1508 if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1510 extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1];
1512 di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1514 goto di_entersolvablekey;
1519 case di_enterrepodata: di_enterrepodata:
1522 if (di->repodataid >= di->repo->nrepodata)
1523 goto di_nextsolvable;
1524 di->data = di->repo->repodata + di->repodataid;
1526 if (di->repodataid && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1527 goto di_nextrepodata;
1528 if (!maybe_load_repodata(di->data, di->keyname))
1529 goto di_nextrepodata;
1530 di->dp = solvid2data(di->data, di->solvid, &schema);
1532 goto di_nextrepodata;
1533 if (di->solvid == SOLVID_POS)
1534 di->solvid = di->pool->pos.solvid;
1535 /* reset key iterator */
1536 di->keyp = di->data->schemadata + di->data->schemata[schema];
1539 case di_enterschema: di_enterschema:
1541 di->dp = dataiterator_find_keyname(di, di->keyname);
1542 if (!di->dp || !*di->keyp)
1546 goto di_nextrepodata;
1550 case di_enterkey: di_enterkey:
1552 di->key = di->data->keys + *di->keyp;
1555 /* this is get_data() modified to store vert_ data */
1556 if (di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1559 di->dp = data_read_id(di->dp, &off);
1560 di->dp = data_read_id(di->dp, &len);
1561 di->vert_ddp = di->ddp = get_vertical_data(di->data, di->key, off, len);
1564 di->vert_storestate = di->data->storestate;
1566 else if (di->key->storage == KEY_STORAGE_INCORE)
1569 if (di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0))
1570 di->dp = data_skip_key(di->data, di->dp, di->key);
1576 if (di->key->type == REPOKEY_TYPE_DELETED)
1578 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1580 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1586 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1588 di->state = di_nextkey;
1590 di->state = di_nextattr;
1593 case di_nextkey: di_nextkey:
1594 if (!di->keyname && *++di->keyp)
1600 case di_nextrepodata: di_nextrepodata:
1601 if (di->repodataid && ++di->repodataid < di->repo->nrepodata)
1602 goto di_enterrepodata;
1605 case di_nextsolvable: di_nextsolvable:
1606 if (!(di->flags & SEARCH_THISSOLVID))
1609 di->solvid = di->repo->start;
1612 for (; di->solvid < di->repo->end; di->solvid++)
1614 if (di->pool->solvables[di->solvid].repo == di->repo)
1615 goto di_entersolvable;
1620 case di_nextrepo: di_nextrepo:
1625 if (di->repoid < di->pool->nrepos)
1627 di->repo = di->pool->repos[di->repoid];
1633 case di_bye: di_bye:
1637 case di_enterarray: di_enterarray:
1638 if (di->key->name == REPOSITORY_SOLVABLES)
1640 di->ddp = data_read_id(di->ddp, (Id *)&di->kv.num);
1645 case di_nextarrayelement: di_nextarrayelement:
1648 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1649 if (di->kv.entry == di->kv.num)
1651 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1653 if (!(di->flags & SEARCH_ARRAYSENTINEL))
1655 di->kv.str = (char *)di->ddp;
1657 di->state = di_nextkey;
1660 if (di->kv.entry == di->kv.num - 1)
1662 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1663 di->ddp = data_read_id(di->ddp, &di->kv.id);
1664 di->kv.str = (char *)di->ddp;
1665 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1667 if ((di->flags & SEARCH_SUB) != 0)
1668 di->state = di_entersub;
1670 di->state = di_nextarrayelement;
1673 case di_entersub: di_entersub:
1674 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1675 goto di_nextarrayelement; /* sorry, full */
1676 di->parents[di->nparents].kv = di->kv;
1677 di->parents[di->nparents].dp = di->dp;
1678 di->parents[di->nparents].keyp = di->keyp;
1679 di->dp = (unsigned char *)di->kv.str;
1680 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1681 memset(&di->kv, 0, sizeof(di->kv));
1682 di->kv.parent = &di->parents[di->nparents].kv;
1684 di->keyname = di->keynames[di->nparents - di->rootlevel];
1685 goto di_enterschema;
1687 case di_leavesub: di_leavesub:
1688 if (di->nparents - 1 < di->rootlevel)
1691 di->dp = di->parents[di->nparents].dp;
1692 di->kv = di->parents[di->nparents].kv;
1693 di->keyp = di->parents[di->nparents].keyp;
1694 di->key = di->data->keys + *di->keyp;
1695 di->ddp = (unsigned char *)di->kv.str;
1696 di->keyname = di->keynames[di->nparents - di->rootlevel];
1697 goto di_nextarrayelement;
1699 /* special solvable attr handling follows */
1701 case di_nextsolvablekey: di_nextsolvablekey:
1702 if (di->keyname || di->key->name == RPM_RPMDBID)
1703 goto di_enterrepodata;
1707 case di_entersolvablekey: di_entersolvablekey:
1708 di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1709 if (!di->idp || !*di->idp)
1710 goto di_nextsolvablekey;
1714 di->kv.id = *di->idp;
1715 di->kv.num = *di->idp; /* for rpmdbid */
1716 di->kv.num2 = 0; /* for rpmdbid */
1718 di->state = di_nextsolvablekey;
1724 case di_nextsolvableattr:
1725 di->state = di_nextsolvableattr;
1726 di->kv.id = *di->idp++;
1731 di->state = di_nextsolvablekey;
1737 if (di->matcher.match)
1740 /* simple pre-check so that we don't need to stringify */
1741 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0)
1742 if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
1744 if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
1746 if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
1750 if (!datamatcher_match(&di->matcher, str))
1755 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->flags & SEARCH_FILES) != 0)
1756 repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags);
1758 /* found something! */
1764 dataiterator_entersub(Dataiterator *di)
1766 if (di->state == di_nextarrayelement)
1767 di->state = di_entersub;
1771 dataiterator_setpos(Dataiterator *di)
1773 if (di->kv.eof == 2)
1775 pool_clear_pos(di->pool);
1778 di->pool->pos.solvid = di->solvid;
1779 di->pool->pos.repo = di->repo;
1780 di->pool->pos.repodataid = di->data - di->repo->repodata;
1781 di->pool->pos.schema = di->kv.id;
1782 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1786 dataiterator_setpos_parent(Dataiterator *di)
1788 if (!di->kv.parent || di->kv.parent->eof == 2)
1790 pool_clear_pos(di->pool);
1793 di->pool->pos.solvid = di->solvid;
1794 di->pool->pos.repo = di->repo;
1795 di->pool->pos.repodataid = di->data - di->repo->repodata;
1796 di->pool->pos.schema = di->kv.parent->id;
1797 di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1800 /* clones just the position, not the search keys/matcher */
1802 dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
1804 di->state = from->state;
1805 di->flags &= ~SEARCH_THISSOLVID;
1806 di->flags |= (from->flags & SEARCH_THISSOLVID);
1807 di->repo = from->repo;
1808 di->data = from->data;
1810 di->ddp = from->ddp;
1811 di->idp = from->idp;
1812 di->keyp = from->keyp;
1813 di->key = from->key;
1815 di->repodataid = from->repodataid;
1816 di->solvid = from->solvid;
1817 di->repoid = from->repoid;
1818 di->rootlevel = from->rootlevel;
1819 memcpy(di->parents, from->parents, sizeof(from->parents));
1820 di->nparents = from->nparents;
1824 for (i = 1; i < di->nparents; i++)
1825 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1826 di->kv.parent = &di->parents[di->nparents - 1].kv;
1830 if (from->dupstr && from->dupstr == from->kv.str)
1832 di->dupstrn = from->dupstrn;
1833 di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
1838 dataiterator_seek(Dataiterator *di, int whence)
1840 if ((whence & DI_SEEK_STAY) != 0)
1841 di->rootlevel = di->nparents;
1842 switch (whence & ~DI_SEEK_STAY)
1845 if (di->state != di_nextarrayelement)
1847 if ((whence & DI_SEEK_STAY) != 0)
1848 di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1849 di->state = di_entersub;
1851 case DI_SEEK_PARENT:
1858 if (di->rootlevel > di->nparents)
1859 di->rootlevel = di->nparents;
1860 di->dp = di->parents[di->nparents].dp;
1861 di->kv = di->parents[di->nparents].kv;
1862 di->keyp = di->parents[di->nparents].keyp;
1863 di->key = di->data->keys + *di->keyp;
1864 di->ddp = (unsigned char *)di->kv.str;
1865 di->keyname = di->keynames[di->nparents - di->rootlevel];
1866 di->state = di_nextarrayelement;
1868 case DI_SEEK_REWIND:
1874 di->dp = (unsigned char *)di->kv.parent->str;
1875 di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1876 di->state = di_enterschema;
1884 dataiterator_skip_attribute(Dataiterator *di)
1886 if (di->state == di_nextsolvableattr)
1887 di->state = di_nextsolvablekey;
1889 di->state = di_nextkey;
1893 dataiterator_skip_solvable(Dataiterator *di)
1898 di->keyname = di->keynames[0];
1899 di->state = di_nextsolvable;
1903 dataiterator_skip_repo(Dataiterator *di)
1908 di->keyname = di->keynames[0];
1909 di->state = di_nextrepo;
1913 dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1918 di->keyname = di->keynames[0];
1919 if (solvid == SOLVID_POS)
1921 di->repo = di->pool->pos.repo;
1928 di->data = di->repo->repodata + di->pool->pos.repodataid;
1930 di->solvid = solvid;
1931 di->state = di_enterrepo;
1932 di->flags |= SEARCH_THISSOLVID;
1937 di->repo = di->pool->solvables[solvid].repo;
1940 else if (di->repoid > 0)
1942 if (!di->pool->urepos)
1948 di->repo = di->pool->repos[di->repoid];
1951 di->solvid = solvid;
1953 di->flags |= SEARCH_THISSOLVID;
1954 di->state = di_enterrepo;
1958 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1964 di->repoid = 0; /* 0 means stay at repo */
1967 di->flags &= ~SEARCH_THISSOLVID;
1968 di->state = di_enterrepo;
1972 dataiterator_match(Dataiterator *di, Datamatcher *ma)
1975 if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
1977 return ma ? datamatcher_match(ma, str) : 1;
1981 dataiterator_strdup(Dataiterator *di)
1985 if (!di->kv.str || di->kv.str == di->dupstr)
1987 switch (di->key->type)
1990 case REPOKEY_TYPE_DIRSTRARRAY:
1991 if (di->kv.num) /* was it stringified into tmp space? */
1992 l = strlen(di->kv.str) + 1;
1997 if (l < 0 && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1999 switch (di->key->type)
2001 case REPOKEY_TYPE_STR:
2002 case REPOKEY_TYPE_DIRSTRARRAY:
2003 l = strlen(di->kv.str) + 1;
2006 l = solv_chksum_len(di->key->type);
2008 case REPOKEY_TYPE_BINARY:
2015 if (!di->dupstrn || di->dupstrn < l)
2017 di->dupstrn = l + 16;
2018 di->dupstr = solv_realloc(di->dupstr, di->dupstrn);
2021 memcpy(di->dupstr, di->kv.str, l);
2022 di->kv.str = di->dupstr;
2026 /************************************************************************
2027 * data modify functions
2030 /* extend repodata so that it includes solvables p */
2032 repodata_extend(Repodata *data, Id p)
2034 if (data->start == data->end)
2035 data->start = data->end = p;
2038 int old = data->end - data->start;
2039 int new = p - data->end + 1;
2042 data->attrs = solv_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
2043 memset(data->attrs + old, 0, new * sizeof(Id *));
2045 data->incoreoffset = solv_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
2046 memset(data->incoreoffset + old, 0, new * sizeof(Id));
2049 if (p < data->start)
2051 int old = data->end - data->start;
2052 int new = data->start - p;
2055 data->attrs = solv_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
2056 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
2057 memset(data->attrs, 0, new * sizeof(Id *));
2059 data->incoreoffset = solv_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
2060 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
2061 memset(data->incoreoffset, 0, new * sizeof(Id));
2066 /* shrink end of repodata */
2068 repodata_shrink(Repodata *data, int end)
2072 if (data->end <= end)
2074 if (data->start >= end)
2078 for (i = 0; i < data->end - data->start; i++)
2079 solv_free(data->attrs[i]);
2080 data->attrs = solv_free(data->attrs);
2082 data->incoreoffset = solv_free(data->incoreoffset);
2083 data->start = data->end = 0;
2088 for (i = end; i < data->end; i++)
2089 solv_free(data->attrs[i - data->start]);
2090 data->attrs = solv_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
2092 if (data->incoreoffset)
2093 data->incoreoffset = solv_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
2097 /* extend repodata so that it includes solvables from start to start + num - 1 */
2099 repodata_extend_block(Repodata *data, Id start, Id num)
2103 if (!data->incoreoffset)
2105 /* this also means that data->attrs is NULL */
2106 data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
2107 data->start = start;
2108 data->end = start + num;
2111 repodata_extend(data, start);
2113 repodata_extend(data, start + num - 1);
2116 /**********************************************************************/
2119 #define REPODATA_ATTRS_BLOCK 31
2120 #define REPODATA_ATTRDATA_BLOCK 1023
2121 #define REPODATA_ATTRIDDATA_BLOCK 63
2122 #define REPODATA_ATTRNUM64DATA_BLOCK 15
2126 repodata_new_handle(Repodata *data)
2130 data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
2131 data->nxattrs = 2; /* -1: SOLVID_META */
2133 data->xattrs = solv_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
2134 data->xattrs[data->nxattrs] = 0;
2135 return -(data->nxattrs++);
2139 repodata_get_attrp(Repodata *data, Id handle)
2143 if (handle == SOLVID_META && !data->xattrs)
2145 data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
2148 return data->xattrs - handle;
2150 if (handle < data->start || handle >= data->end)
2151 repodata_extend(data, handle);
2153 data->attrs = solv_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
2154 return data->attrs + (handle - data->start);
2158 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
2164 app = repodata_get_attrp(data, handle);
2169 /* Determine equality based on the name only, allows us to change
2170 type (when overwrite is set), and makes TYPE_CONSTANT work. */
2171 for (pp = ap; *pp; pp += 2)
2172 if (data->keys[*pp].name == data->keys[keyid].name)
2176 if (overwrite || data->keys[*pp].type == REPOKEY_TYPE_DELETED)
2185 ap = solv_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
2195 repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
2199 keyid = repodata_key2id(data, key, 1);
2200 repodata_insert_keyid(data, solvid, keyid, val, 1);
2204 repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
2208 key.type = REPOKEY_TYPE_ID;
2210 key.storage = KEY_STORAGE_INCORE;
2211 repodata_set(data, solvid, &key, id);
2215 repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned long long num)
2219 key.type = REPOKEY_TYPE_NUM;
2221 key.storage = KEY_STORAGE_INCORE;
2222 if (num >= 0x80000000)
2224 data->attrnum64data = solv_extend(data->attrnum64data, data->attrnum64datalen, 1, sizeof(unsigned long long), REPODATA_ATTRNUM64DATA_BLOCK);
2225 data->attrnum64data[data->attrnum64datalen] = num;
2226 num = 0x80000000 | data->attrnum64datalen++;
2228 repodata_set(data, solvid, &key, (Id)num);
2232 repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
2236 if (data->localpool)
2237 id = stringpool_str2id(&data->spool, str, 1);
2239 id = pool_str2id(data->repo->pool, str, 1);
2241 key.type = REPOKEY_TYPE_ID;
2243 key.storage = KEY_STORAGE_INCORE;
2244 repodata_set(data, solvid, &key, id);
2248 repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
2252 key.type = REPOKEY_TYPE_CONSTANT;
2253 key.size = constant;
2254 key.storage = KEY_STORAGE_INCORE;
2255 repodata_set(data, solvid, &key, 0);
2259 repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
2263 key.type = REPOKEY_TYPE_CONSTANTID;
2265 key.storage = KEY_STORAGE_INCORE;
2266 repodata_set(data, solvid, &key, 0);
2270 repodata_set_void(Repodata *data, Id solvid, Id keyname)
2274 key.type = REPOKEY_TYPE_VOID;
2276 key.storage = KEY_STORAGE_INCORE;
2277 repodata_set(data, solvid, &key, 0);
2281 repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
2286 l = strlen(str) + 1;
2288 key.type = REPOKEY_TYPE_STR;
2290 key.storage = KEY_STORAGE_INCORE;
2291 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2292 memcpy(data->attrdata + data->attrdatalen, str, l);
2293 repodata_set(data, solvid, &key, data->attrdatalen);
2294 data->attrdatalen += l;
2298 repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len)
2306 key.type = REPOKEY_TYPE_BINARY;
2308 key.storage = KEY_STORAGE_INCORE;
2309 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK);
2310 dp = data->attrdata + data->attrdatalen;
2311 if (len >= (1 << 14))
2313 if (len >= (1 << 28))
2314 *dp++ = (len >> 28) | 128;
2315 if (len >= (1 << 21))
2316 *dp++ = (len >> 21) | 128;
2317 *dp++ = (len >> 14) | 128;
2319 if (len >= (1 << 7))
2320 *dp++ = (len >> 7) | 128;
2323 memcpy(dp, buf, len);
2324 repodata_set(data, solvid, &key, data->attrdatalen);
2325 data->attrdatalen = dp + len - data->attrdata;
2328 /* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
2329 * so that the caller can append entrysize new elements plus the termination zero there */
2331 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
2334 Id *ida, *pp, **ppp;
2336 /* check if it is the same as last time, this speeds things up a lot */
2337 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
2339 /* great! just append the new data */
2340 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2341 data->attriddatalen--; /* overwrite terminating 0 */
2342 data->lastdatalen += entrysize;
2346 ppp = repodata_get_attrp(data, handle);
2350 for (; *pp; pp += 2)
2351 if (data->keys[*pp].name == keyname)
2354 if (!pp || !*pp || data->keys[*pp].type != keytype)
2356 /* not found. allocate new key */
2362 key.storage = KEY_STORAGE_INCORE;
2363 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2364 keyid = repodata_key2id(data, &key, 1);
2365 repodata_insert_keyid(data, handle, keyid, data->attriddatalen, 1);
2366 data->lasthandle = handle;
2367 data->lastkey = keyid;
2368 data->lastdatalen = data->attriddatalen + entrysize + 1;
2372 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2373 oldsize += entrysize;
2374 if (ida + 1 == data->attriddata + data->attriddatalen)
2376 /* this was the last entry, just append it */
2377 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2378 data->attriddatalen--; /* overwrite terminating 0 */
2382 /* too bad. move to back. */
2383 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2384 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2385 pp[1] = data->attriddatalen;
2386 data->attriddatalen += oldsize;
2388 data->lasthandle = handle;
2389 data->lastkey = *pp;
2390 data->lastdatalen = data->attriddatalen + entrysize + 1;
2394 repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2395 const unsigned char *str)
2400 if (!(l = solv_chksum_len(type)))
2405 key.storage = KEY_STORAGE_INCORE;
2406 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2407 memcpy(data->attrdata + data->attrdatalen, str, l);
2408 repodata_set(data, solvid, &key, data->attrdatalen);
2409 data->attrdatalen += l;
2413 repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2416 unsigned char buf[64];
2419 if (!(l = solv_chksum_len(type)))
2421 if (l > sizeof(buf) || solv_hex2bin(&str, buf, l) != l)
2423 repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2427 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2431 if (!(l = solv_chksum_len(type)))
2433 return pool_bin2hex(data->repo->pool, buf, l);
2436 /* rpm filenames don't contain the epoch, so strip it */
2437 static inline const char *
2438 evrid2vrstr(Pool *pool, Id evrid)
2440 const char *p, *evr = pool_id2str(pool, evrid);
2443 for (p = evr; *p >= '0' && *p <= '9'; p++)
2445 return p != evr && *p == ':' && p[1] ? p + 1 : evr;
2449 repodata_set_poolstrn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
2452 if (data->localpool)
2453 id = stringpool_strn2id(&data->spool, str, l, 1);
2455 id = pool_strn2id(data->repo->pool, str, l, 1);
2456 repodata_set_id(data, solvid, keyname, id);
2460 repodata_set_strn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
2463 repodata_set_str(data, solvid, keyname, str);
2466 char *s = solv_strdup(str);
2468 repodata_set_str(data, solvid, keyname, s);
2474 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
2476 Pool *pool = data->repo->pool;
2478 const char *str, *fp;
2482 repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2485 if ((dir = strrchr(file, '/')) != 0)
2496 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2501 if (l == 1 && dir[0] == '.')
2503 s = pool->solvables + solvid;
2506 str = pool_id2str(pool, s->arch);
2507 if (!strncmp(dir, str, l) && !str[l])
2508 repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2510 repodata_set_strn(data, solvid, SOLVABLE_MEDIADIR, dir, l);
2513 str = pool_id2str(pool, s->name);
2515 if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
2518 str = evrid2vrstr(pool, s->evr);
2520 if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
2523 str = pool_id2str(pool, s->arch);
2525 if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2527 repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2532 repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2535 /* XXX: medianr is currently not stored */
2537 repodata_set_deltalocation(Repodata *data, Id handle, int medianr, const char *dir, const char *file)
2540 const char *evr, *suf, *s;
2544 if ((dir = strrchr(file, '/')) != 0)
2555 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2560 if (l == 1 && dir[0] == '.')
2563 repodata_set_poolstrn(data, handle, DELTA_LOCATION_DIR, dir, l);
2564 evr = strchr(file, '-');
2567 for (s = evr - 1; s > file; s--)
2574 suf = strrchr(file, '.');
2577 for (s = suf - 1; s > file; s--)
2583 if (!strcmp(suf, ".delta.rpm") || !strcmp(suf, ".patch.rpm"))
2585 /* We accept one more item as suffix. */
2586 for (s = suf - 1; s > file; s--)
2596 if (suf && evr && suf < evr)
2598 repodata_set_poolstrn(data, handle, DELTA_LOCATION_NAME, file, evr ? evr - file : strlen(file));
2600 repodata_set_poolstrn(data, handle, DELTA_LOCATION_EVR, evr + 1, suf ? suf - evr - 1: strlen(evr + 1));
2602 repodata_set_poolstr(data, handle, DELTA_LOCATION_SUFFIX, suf + 1);
2606 repodata_set_sourcepkg(Repodata *data, Id solvid, const char *sourcepkg)
2608 Pool *pool = data->repo->pool;
2609 Solvable *s = pool->solvables + solvid;
2610 const char *p, *sevr, *sarch, *name, *evr;
2612 p = strrchr(sourcepkg, '.');
2613 if (!p || strcmp(p, ".rpm") != 0)
2616 repodata_set_str(data, solvid, SOLVABLE_SOURCENAME, sourcepkg);
2620 while (p > sourcepkg && *p != '.')
2622 if (*p != '.' || p == sourcepkg)
2625 while (p > sourcepkg && *p != '-')
2627 if (*p != '-' || p == sourcepkg)
2630 while (p > sourcepkg && *p != '-')
2632 if (*p != '-' || p == sourcepkg)
2635 pool = s->repo->pool;
2637 name = pool_id2str(pool, s->name);
2638 if (name && !strncmp(sourcepkg, name, sevr - sourcepkg - 1) && name[sevr - sourcepkg - 1] == 0)
2639 repodata_set_void(data, solvid, SOLVABLE_SOURCENAME);
2641 repodata_set_id(data, solvid, SOLVABLE_SOURCENAME, pool_strn2id(pool, sourcepkg, sevr - sourcepkg - 1, 1));
2643 evr = evrid2vrstr(pool, s->evr);
2644 if (evr && !strncmp(sevr, evr, sarch - sevr - 1) && evr[sarch - sevr - 1] == 0)
2645 repodata_set_void(data, solvid, SOLVABLE_SOURCEEVR);
2647 repodata_set_id(data, solvid, SOLVABLE_SOURCEEVR, pool_strn2id(pool, sevr, sarch - sevr - 1, 1));
2649 if (!strcmp(sarch, "src.rpm"))
2650 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_SRC);
2651 else if (!strcmp(sarch, "nosrc.rpm"))
2652 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_NOSRC);
2654 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, pool_strn2id(pool, sarch, strlen(sarch) - 4, 1));
2658 repodata_set_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
2664 key.type = REPOKEY_TYPE_IDARRAY;
2666 key.storage = KEY_STORAGE_INCORE;
2667 repodata_set(data, solvid, &key, data->attriddatalen);
2668 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, q->count + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2669 for (i = 0; i < q->count; i++)
2670 data->attriddata[data->attriddatalen++] = q->elements[i];
2671 data->attriddata[data->attriddatalen++] = 0;
2675 repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2679 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2681 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2682 data->attriddata[data->attriddatalen++] = dir;
2683 data->attriddata[data->attriddatalen++] = num;
2684 data->attriddata[data->attriddatalen++] = num2;
2685 data->attriddata[data->attriddatalen++] = 0;
2689 repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
2695 l = strlen(str) + 1;
2696 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2697 memcpy(data->attrdata + data->attrdatalen, str, l);
2698 stroff = data->attrdatalen;
2699 data->attrdatalen += l;
2702 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2704 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2705 data->attriddata[data->attriddatalen++] = dir;
2706 data->attriddata[data->attriddatalen++] = stroff;
2707 data->attriddata[data->attriddatalen++] = 0;
2711 repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2714 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2716 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2717 data->attriddata[data->attriddatalen++] = id;
2718 data->attriddata[data->attriddatalen++] = 0;
2722 repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2726 if (data->localpool)
2727 id = stringpool_str2id(&data->spool, str, 1);
2729 id = pool_str2id(data->repo->pool, str, 1);
2730 repodata_add_idarray(data, solvid, keyname, id);
2734 repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2736 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2737 data->attriddata[data->attriddatalen++] = ghandle;
2738 data->attriddata[data->attriddatalen++] = 0;
2742 repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2744 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2745 data->attriddata[data->attriddatalen++] = ghandle;
2746 data->attriddata[data->attriddatalen++] = 0;
2750 repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname)
2753 app = repodata_get_attrp(data, solvid);
2757 for (; *ap; ap += 2)
2758 if (data->keys[*ap].name == keyname)
2764 for (; *ap; ap += 2)
2766 if (data->keys[*ap].name == keyname)
2774 /* XXX: does not work correctly, needs fix in iterators! */
2776 repodata_unset(Repodata *data, Id solvid, Id keyname)
2780 key.type = REPOKEY_TYPE_DELETED;
2782 key.storage = KEY_STORAGE_INCORE;
2783 repodata_set(data, solvid, &key, 0);
2786 /* add all (uninternalized) attrs from src to dest */
2788 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2791 if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
2793 for (; *keyp; keyp += 2)
2794 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2797 /* add some (uninternalized) attrs from src to dest */
2799 repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
2802 if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
2804 for (; *keyp; keyp += 2)
2805 if (!keyidmap || MAPTST(keyidmap, keyp[0]))
2806 repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2809 /* swap (uninternalized) attrs from src and dest */
2811 repodata_swap_attrs(Repodata *data, Id dest, Id src)
2814 if (!data->attrs || dest == src)
2816 if (dest < data->start || dest >= data->end)
2817 repodata_extend(data, dest);
2818 if (src < data->start || src >= data->end)
2819 repodata_extend(data, src);
2820 tmpattrs = data->attrs[dest - data->start];
2821 data->attrs[dest - data->start] = data->attrs[src - data->start];
2822 data->attrs[src - data->start] = tmpattrs;
2826 /**********************************************************************/
2828 /* TODO: unify with repo_write and repo_solv! */
2830 #define EXTDATA_BLOCK 1023
2838 data_addid(struct extdata *xd, Id sx)
2840 unsigned int x = (unsigned int)sx;
2843 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2844 dp = xd->buf + xd->len;
2849 *dp++ = (x >> 28) | 128;
2851 *dp++ = (x >> 21) | 128;
2852 *dp++ = (x >> 14) | 128;
2855 *dp++ = (x >> 7) | 128;
2857 xd->len = dp - xd->buf;
2861 data_addid64(struct extdata *xd, unsigned long long x)
2863 if (x >= 0x100000000)
2867 data_addid(xd, (Id)(x >> 35));
2868 xd->buf[xd->len - 1] |= 128;
2870 data_addid(xd, (Id)((unsigned int)x | 0x80000000));
2871 xd->buf[xd->len - 5] = (x >> 28) | 128;
2874 data_addid(xd, (Id)x);
2878 data_addideof(struct extdata *xd, Id sx, int eof)
2880 unsigned int x = (unsigned int)sx;
2883 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2884 dp = xd->buf + xd->len;
2889 *dp++ = (x >> 27) | 128;
2891 *dp++ = (x >> 20) | 128;
2892 *dp++ = (x >> 13) | 128;
2895 *dp++ = (x >> 6) | 128;
2896 *dp++ = eof ? (x & 63) : (x & 63) | 64;
2897 xd->len = dp - xd->buf;
2901 data_addblob(struct extdata *xd, unsigned char *blob, int len)
2903 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
2904 memcpy(xd->buf + xd->len, blob, len);
2908 /*********************************/
2910 /* this is to reduct memory usage when internalizing oversized repos */
2912 compact_attrdata(Repodata *data, int entry, int nentry)
2915 unsigned int attrdatastart = data->attrdatalen;
2916 unsigned int attriddatastart = data->attriddatalen;
2917 if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
2919 for (i = entry; i < nentry; i++)
2921 Id v, *attrs = data->attrs[i];
2924 for (; *attrs; attrs += 2)
2926 switch (data->keys[*attrs].type)
2928 case REPOKEY_TYPE_STR:
2929 case REPOKEY_TYPE_BINARY:
2931 if (attrs[1] < attrdatastart)
2932 attrdatastart = attrs[1];
2934 case REPOKEY_TYPE_DIRSTRARRAY:
2935 for (v = attrs[1]; data->attriddata[v] ; v += 2)
2936 if (data->attriddata[v + 1] < attrdatastart)
2937 attrdatastart = data->attriddata[v + 1];
2939 case REPOKEY_TYPE_IDARRAY:
2940 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2941 if (attrs[1] < attriddatastart)
2942 attriddatastart = attrs[1];
2944 case REPOKEY_TYPE_FIXARRAY:
2945 case REPOKEY_TYPE_FLEXARRAY:
2953 printf("compact_attrdata %d %d\n", entry, nentry);
2954 printf("attrdatastart: %d\n", attrdatastart);
2955 printf("attriddatastart: %d\n", attriddatastart);
2957 if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
2959 for (i = entry; i < nentry; i++)
2961 Id v, *attrs = data->attrs[i];
2964 for (; *attrs; attrs += 2)
2966 switch (data->keys[*attrs].type)
2968 case REPOKEY_TYPE_STR:
2969 case REPOKEY_TYPE_BINARY:
2971 attrs[1] -= attrdatastart;
2973 case REPOKEY_TYPE_DIRSTRARRAY:
2974 for (v = attrs[1]; data->attriddata[v] ; v += 2)
2975 data->attriddata[v + 1] -= attrdatastart;
2977 case REPOKEY_TYPE_IDARRAY:
2978 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2979 attrs[1] -= attriddatastart;
2988 data->attrdatalen -= attrdatastart;
2989 memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
2990 data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
2992 if (attriddatastart)
2994 data->attriddatalen -= attriddatastart;
2995 memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
2996 data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
3000 /* internalalize some key into incore/vincore data */
3003 repodata_serialize_key(Repodata *data, struct extdata *newincore,
3004 struct extdata *newvincore,
3006 Repokey *key, Id val)
3010 unsigned int oldvincorelen = 0;
3014 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3017 oldvincorelen = xd->len;
3021 case REPOKEY_TYPE_VOID:
3022 case REPOKEY_TYPE_CONSTANT:
3023 case REPOKEY_TYPE_CONSTANTID:
3025 case REPOKEY_TYPE_STR:
3026 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
3028 case REPOKEY_TYPE_MD5:
3029 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
3031 case REPOKEY_TYPE_SHA1:
3032 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
3034 case REPOKEY_TYPE_SHA224:
3035 data_addblob(xd, data->attrdata + val, SIZEOF_SHA224);
3037 case REPOKEY_TYPE_SHA256:
3038 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
3040 case REPOKEY_TYPE_SHA384:
3041 data_addblob(xd, data->attrdata + val, SIZEOF_SHA384);
3043 case REPOKEY_TYPE_SHA512:
3044 data_addblob(xd, data->attrdata + val, SIZEOF_SHA512);
3046 case REPOKEY_TYPE_NUM:
3047 if (val & 0x80000000)
3049 data_addid64(xd, data->attrnum64data[val ^ 0x80000000]);
3053 case REPOKEY_TYPE_ID:
3054 case REPOKEY_TYPE_DIR:
3055 data_addid(xd, val);
3057 case REPOKEY_TYPE_BINARY:
3060 unsigned char *dp = data_read_id(data->attrdata + val, &len);
3061 dp += (unsigned int)len;
3062 data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
3065 case REPOKEY_TYPE_IDARRAY:
3066 for (ida = data->attriddata + val; *ida; ida++)
3067 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
3069 case REPOKEY_TYPE_DIRNUMNUMARRAY:
3070 for (ida = data->attriddata + val; *ida; ida += 3)
3072 data_addid(xd, ida[0]);
3073 data_addid(xd, ida[1]);
3074 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
3077 case REPOKEY_TYPE_DIRSTRARRAY:
3078 for (ida = data->attriddata + val; *ida; ida += 2)
3080 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
3081 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
3084 case REPOKEY_TYPE_FIXARRAY:
3088 for (ida = data->attriddata + val; *ida; ida++)
3092 kp = data->xattrs[-*ida];
3100 schemaid = repodata_schema2id(data, schema, 1);
3101 else if (schemaid != repodata_schema2id(data, schema, 0))
3103 pool_debug(data->repo->pool, SOLV_FATAL, "fixarray substructs with different schemas\n");
3109 data_addid(xd, num);
3110 data_addid(xd, schemaid);
3111 for (ida = data->attriddata + val; *ida; ida++)
3113 Id *kp = data->xattrs[-*ida];
3117 repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3121 case REPOKEY_TYPE_FLEXARRAY:
3124 for (ida = data->attriddata + val; *ida; ida++)
3126 data_addid(xd, num);
3127 for (ida = data->attriddata + val; *ida; ida++)
3129 Id *kp = data->xattrs[-*ida];
3132 data_addid(xd, 0); /* XXX */
3139 schemaid = repodata_schema2id(data, schema, 1);
3140 data_addid(xd, schemaid);
3141 kp = data->xattrs[-*ida];
3143 repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3148 pool_debug(data->repo->pool, SOLV_FATAL, "don't know how to handle type %d\n", key->type);
3151 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3153 /* put offset/len in incore */
3154 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
3155 oldvincorelen = xd->len - oldvincorelen;
3156 data_addid(newincore, oldvincorelen);
3161 repodata_internalize(Repodata *data)
3163 Repokey *key, solvkey;
3165 Id schemaid, keyid, *schema, *sp, oldschema, *keyp, *seen;
3167 unsigned char *dp, *ndp;
3168 int newschema, oldcount;
3169 struct extdata newincore;
3170 struct extdata newvincore;
3173 if (!data->attrs && !data->xattrs)
3177 printf("repodata_internalize %d\n", data->repodataid);
3178 printf(" attr data: %d K\n", data->attrdatalen / 1024);
3179 printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3181 newvincore.buf = data->vincore;
3182 newvincore.len = data->vincorelen;
3184 /* find the solvables key, create if needed */
3185 memset(&solvkey, 0, sizeof(solvkey));
3186 solvkey.name = REPOSITORY_SOLVABLES;
3187 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
3189 solvkey.storage = KEY_STORAGE_INCORE;
3190 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
3192 schema = solv_malloc2(data->nkeys, sizeof(Id));
3193 seen = solv_malloc2(data->nkeys, sizeof(Id));
3195 /* Merge the data already existing (in data->schemata, ->incoredata and
3196 friends) with the new attributes in data->attrs[]. */
3197 nentry = data->end - data->start;
3198 memset(&newincore, 0, sizeof(newincore));
3199 data_addid(&newincore, 0); /* start data at offset 1 */
3201 data->mainschema = 0;
3202 data->mainschemaoffsets = solv_free(data->mainschemaoffsets);
3204 /* join entry data */
3205 /* we start with the meta data, entry -1 */
3206 for (entry = -1; entry < nentry; entry++)
3208 memset(seen, 0, data->nkeys * sizeof(Id));
3210 dp = data->incoredata;
3213 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
3214 dp = data_read_id(dp, &oldschema);
3217 fprintf(stderr, "oldschema %d\n", oldschema);
3218 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
3219 fprintf(stderr, "schemadata %p\n", data->schemadata);
3221 /* seen: -1: old data 0: skipped >0: id + 1 */
3225 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
3229 pool_debug(data->repo->pool, SOLV_FATAL, "Inconsistent old data (key occured twice).\n");
3237 keyp = data->attrs ? data->attrs[entry] : 0;
3240 /* strip solvables key */
3242 for (sp = keyp = schema; *sp; sp++)
3243 if (*sp != solvkeyid)
3248 seen[solvkeyid] = 0;
3249 keyp = data->xattrs ? data->xattrs[1] : 0;
3252 for (; *keyp; keyp += 2)
3259 seen[*keyp] = keyp[1] + 1;
3261 if (entry < 0 && data->end != data->start)
3268 /* Ideally we'd like to sort the new schema here, to ensure
3269 schema equality independend of the ordering. We can't do that
3270 yet. For once see below (old ids need to come before new ids).
3271 An additional difficulty is that we also need to move
3272 the values with the keys. */
3273 schemaid = repodata_schema2id(data, schema, 1);
3275 schemaid = oldschema;
3278 /* Now create data blob. We walk through the (possibly new) schema
3279 and either copy over old data, or insert the new. */
3280 /* XXX Here we rely on the fact that the (new) schema has the form
3281 o1 o2 o3 o4 ... | n1 n2 n3 ...
3282 (oX being the old keyids (possibly overwritten), and nX being
3283 the new keyids). This rules out sorting the keyids in order
3284 to ensure a small schema count. */
3286 data->incoreoffset[entry] = newincore.len;
3287 data_addid(&newincore, schemaid);
3290 data->mainschema = schemaid;
3291 data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
3293 /* we don't use a pointer to the schemadata here as repodata_serialize_key
3294 * may call repodata_schema2id() which might realloc our schemadata */
3295 for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
3298 data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
3299 if (keyid == solvkeyid)
3301 /* add flexarray entry count */
3302 data_addid(&newincore, data->end - data->start);
3305 key = data->keys + keyid;
3307 fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, pool_id2str(data->repo->pool, key->name), pool_id2str(data->repo->pool, key->type));
3312 /* Skip the data associated with this old key. */
3313 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3315 ndp = data_skip(dp, REPOKEY_TYPE_ID);
3316 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
3318 else if (key->storage == KEY_STORAGE_INCORE)
3319 ndp = data_skip_key(data, dp, key);
3322 if (seen[keyid] == -1)
3324 /* If this key was an old one _and_ was not overwritten with
3325 a different value copy over the old value (we skipped it
3328 data_addblob(&newincore, dp, ndp - dp);
3331 else if (seen[keyid])
3333 /* Otherwise we have a new value. Parse it into the internal form. */
3334 repodata_serialize_key(data, &newincore, &newvincore, schema, key, seen[keyid] - 1);
3338 if (entry >= 0 && data->attrs)
3340 if (data->attrs[entry])
3341 data->attrs[entry] = solv_free(data->attrs[entry]);
3342 if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
3344 compact_attrdata(data, entry + 1, nentry); /* try to free some memory */
3346 printf(" attr data: %d K\n", data->attrdatalen / 1024);
3347 printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3348 printf(" incore data: %d K\n", newincore.len / 1024);
3349 printf(" sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
3350 /* malloc_stats(); */
3355 /* free all xattrs */
3356 for (entry = 0; entry < data->nxattrs; entry++)
3357 if (data->xattrs[entry])
3358 solv_free(data->xattrs[entry]);
3359 data->xattrs = solv_free(data->xattrs);
3362 data->lasthandle = 0;
3364 data->lastdatalen = 0;
3367 repodata_free_schemahash(data);
3369 solv_free(data->incoredata);
3370 data->incoredata = newincore.buf;
3371 data->incoredatalen = newincore.len;
3372 data->incoredatafree = 0;
3374 solv_free(data->vincore);
3375 data->vincore = newvincore.buf;
3376 data->vincorelen = newvincore.len;
3378 data->attrs = solv_free(data->attrs);
3379 data->attrdata = solv_free(data->attrdata);
3380 data->attriddata = solv_free(data->attriddata);
3381 data->attrnum64data = solv_free(data->attrnum64data);
3382 data->attrdatalen = 0;
3383 data->attriddatalen = 0;
3384 data->attrnum64datalen = 0;
3386 printf("repodata_internalize %d done\n", data->repodataid);
3387 printf(" incore data: %d K\n", data->incoredatalen / 1024);
3392 repodata_disable_paging(Repodata *data)
3394 if (maybe_load_repodata(data, 0))
3396 repopagestore_disable_paging(&data->store);
3402 repodata_load_stub(Repodata *data)
3404 Repo *repo = data->repo;
3405 Pool *pool = repo->pool;
3407 struct _Pool_tmpspace oldtmpspace;
3410 if (!pool->loadcallback)
3412 data->state = REPODATA_ERROR;
3415 data->state = REPODATA_LOADING;
3417 /* save tmp space and pos */
3418 oldtmpspace = pool->tmpspace;
3419 memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
3422 r = pool->loadcallback(pool, data, pool->loadcallbackdata);
3424 /* restore tmp space and pos */
3425 for (i = 0; i < POOL_TMPSPACEBUF; i++)
3426 solv_free(pool->tmpspace.buf[i]);
3427 pool->tmpspace = oldtmpspace;
3428 if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
3429 memset(&oldpos, 0, sizeof(oldpos));
3432 data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR;
3436 repodata_add_stubkey(Repodata *data, Id keyname, Id keytype)
3440 xkey.name = keyname;
3441 xkey.type = keytype;
3442 xkey.storage = KEY_STORAGE_INCORE;
3444 repodata_key2id(data, &xkey, 1);
3448 repodata_add_stub(Repodata **datap)
3450 Repodata *data = *datap;
3451 Repo *repo = data->repo;
3452 Id repodataid = data - repo->repodata;
3453 Repodata *sdata = repo_add_repodata(repo, 0);
3454 data = repo->repodata + repodataid;
3455 if (data->end > data->start)
3456 repodata_extend_block(sdata, data->start, data->end - data->start);
3457 sdata->state = REPODATA_STUB;
3458 sdata->loadcallback = repodata_load_stub;
3464 repodata_create_stubs(Repodata *data)
3466 Repo *repo = data->repo;
3467 Pool *pool = repo->pool;
3474 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
3475 while (dataiterator_step(&di))
3476 if (di.data == data)
3478 dataiterator_free(&di);
3481 stubdataids = solv_calloc(cnt, sizeof(*stubdataids));
3482 for (i = 0; i < cnt; i++)
3484 sdata = repodata_add_stub(&data);
3485 stubdataids[i] = sdata - repo->repodata;
3488 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
3490 while (dataiterator_step(&di))
3492 if (di.data != data)
3494 if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
3496 dataiterator_entersub(&di);
3497 sdata = repo->repodata + stubdataids[i++];
3501 switch (di.key->type)
3503 case REPOKEY_TYPE_ID:
3504 repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
3506 case REPOKEY_TYPE_CONSTANTID:
3507 repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
3509 case REPOKEY_TYPE_STR:
3510 repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
3512 case REPOKEY_TYPE_VOID:
3513 repodata_set_void(sdata, SOLVID_META, di.key->name);
3515 case REPOKEY_TYPE_NUM:
3516 repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv));
3519 repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
3521 case REPOKEY_TYPE_IDARRAY:
3522 repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
3523 if (di.key->name == REPOSITORY_KEYS)
3528 xkeyname = di.kv.id;
3532 repodata_add_stubkey(sdata, xkeyname, di.kv.id);
3541 dataiterator_free(&di);
3542 for (i = 0; i < cnt; i++)
3543 repodata_internalize(repo->repodata + stubdataids[i]);
3544 solv_free(stubdataids);
3549 repodata_memused(Repodata *data)
3551 return data->incoredatalen + data->vincorelen;