2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
13 * a repository can contain multiple repodata entries, consisting of
14 * different sets of keys and different sets of solvables
29 #include "poolid_private.h"
37 #define REPODATA_BLOCK 255
39 static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);
42 repodata_initdata(Repodata *data, Repo *repo, int localpool)
44 memset(data, 0, sizeof (*data));
45 data->repodataid = data - repo->repodata;
47 data->localpool = localpool;
49 stringpool_init_empty(&data->spool);
50 /* dirpool_init(&data->dirpool); just zeros out again */
51 data->keys = solv_calloc(1, sizeof(Repokey));
53 data->schemata = solv_calloc(1, sizeof(Id));
54 data->schemadata = solv_calloc(1, sizeof(Id));
56 data->schemadatalen = 1;
57 repopagestore_init(&data->store);
61 repodata_freedata(Repodata *data)
65 solv_free(data->keys);
67 solv_free(data->schemata);
68 solv_free(data->schemadata);
69 solv_free(data->schematahash);
71 stringpool_free(&data->spool);
72 dirpool_free(&data->dirpool);
74 solv_free(data->mainschemaoffsets);
75 solv_free(data->incoredata);
76 solv_free(data->incoreoffset);
77 solv_free(data->verticaloffset);
79 repopagestore_free(&data->store);
81 solv_free(data->vincore);
84 for (i = 0; i < data->end - data->start; i++)
85 solv_free(data->attrs[i]);
86 solv_free(data->attrs);
88 for (i = 0; i < data->nxattrs; i++)
89 solv_free(data->xattrs[i]);
90 solv_free(data->xattrs);
92 solv_free(data->attrdata);
93 solv_free(data->attriddata);
94 solv_free(data->attrnum64data);
96 solv_free(data->dircache);
100 repodata_free(Repodata *data)
102 Repo *repo = data->repo;
103 int i = data - repo->repodata;
106 repodata_freedata(data);
107 if (i < repo->nrepodata - 1)
109 /* whoa! this changes the repodataids! */
110 memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
111 for (; i < repo->nrepodata - 1; i++)
112 repo->repodata[i].repodataid = i;
115 if (repo->nrepodata == 1)
117 repo->repodata = solv_free(repo->repodata);
123 repodata_empty(Repodata *data, int localpool)
125 void (*loadcallback)(Repodata *) = data->loadcallback;
126 int state = data->state;
127 repodata_freedata(data);
128 repodata_initdata(data, data->repo, localpool);
130 data->loadcallback = loadcallback;
134 /***************************************************************
135 * key pool management
138 /* this is not so time critical that we need a hash, so we do a simple
141 repodata_key2id(Repodata *data, Repokey *key, int create)
145 for (keyid = 1; keyid < data->nkeys; keyid++)
146 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
148 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
152 if (keyid == data->nkeys)
156 /* allocate new key */
157 data->keys = solv_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
158 data->keys[data->nkeys++] = *key;
159 if (data->verticaloffset)
161 data->verticaloffset = solv_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
162 data->verticaloffset[data->nkeys - 1] = 0;
164 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
170 /***************************************************************
171 * schema pool management
174 #define SCHEMATA_BLOCK 31
175 #define SCHEMATADATA_BLOCK 255
178 repodata_schema2id(Repodata *data, Id *schema, int create)
185 return 0; /* XXX: allow empty schema? */
186 if ((schematahash = data->schematahash) == 0)
188 data->schematahash = schematahash = solv_calloc(256, sizeof(Id));
189 for (i = 1; i < data->nschemata; i++)
191 for (sp = data->schemadata + data->schemata[i], h = 0; *sp;)
196 data->schemadata = solv_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
197 data->schemata = solv_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
200 for (sp = schema, len = 0, h = 0; *sp; len++)
205 cid = schematahash[h];
208 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
210 /* cache conflict, do a slow search */
211 for (cid = 1; cid < data->nschemata; cid++)
212 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
218 data->schemadata = solv_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
219 data->schemata = solv_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
221 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
222 data->schemata[data->nschemata] = data->schemadatalen;
223 data->schemadatalen += len;
224 schematahash[h] = data->nschemata;
226 fprintf(stderr, "schema2id: new schema\n");
228 return data->nschemata++;
232 repodata_free_schemahash(Repodata *data)
234 data->schematahash = solv_free(data->schematahash);
236 data->schemata = solv_realloc2(data->schemata, data->nschemata, sizeof(Id));
237 data->schemadata = solv_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
241 /***************************************************************
242 * dir pool management
245 #ifndef HAVE_STRCHRNUL
246 static inline const char *strchrnul(const char *str, char x)
248 const char *p = strchr(str, x);
249 return p ? p : str + strlen(str);
253 #define DIRCACHE_SIZE 41 /* < 1k */
257 Id ids[DIRCACHE_SIZE];
258 char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
263 repodata_str2dir(Repodata *data, const char *dir, int create)
274 while (*dir == '/' && dir[1] == '/')
276 if (*dir == '/' && !dir[1])
278 if (data->dirpool.ndirs)
280 return dirpool_add_dir(&data->dirpool, 0, 1, create);
287 struct dircache *dircache = data->dircache;
291 if (l < DIRCACHE_SIZE && dircache->ids[l] && !memcmp(dircache->str + l * (l - 1) / 2, dir, l))
293 parent = dircache->ids[l];
309 dire = strchrnul(dir, '/');
311 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
313 id = pool_strn2id(data->repo->pool, dir, dire - dir, create);
316 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
321 data->dircache = solv_calloc(1, sizeof(struct dircache));
325 if (l < DIRCACHE_SIZE)
327 data->dircache->ids[l] = parent;
328 memcpy(data->dircache->str + l * (l - 1) / 2, dirs, l);
342 repodata_free_dircache(Repodata *data)
344 data->dircache = solv_free(data->dircache);
348 repodata_dir2str(Repodata *data, Id did, const char *suf)
350 Pool *pool = data->repo->pool;
357 return suf ? suf : "";
361 comp = dirpool_compid(&data->dirpool, parent);
362 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
364 parent = dirpool_parent(&data->dirpool, parent);
369 l += strlen(suf) + 1;
370 p = pool_alloctmpspace(pool, l + 1) + l;
381 comp = dirpool_compid(&data->dirpool, parent);
382 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
385 strncpy(p, comps, l);
386 parent = dirpool_parent(&data->dirpool, parent);
394 /***************************************************************
398 static inline unsigned char *
399 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
401 Id *keyp = data->schemadata + data->schemata[schema];
402 for (; *keyp; keyp++)
403 dp = data_skip_key(data, dp, data->keys + *keyp);
407 static unsigned char *
408 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
410 int nentries, schema;
413 case REPOKEY_TYPE_FIXARRAY:
414 dp = data_read_id(dp, &nentries);
417 dp = data_read_id(dp, &schema);
419 dp = data_skip_schema(data, dp, schema);
421 case REPOKEY_TYPE_FLEXARRAY:
422 dp = data_read_id(dp, &nentries);
425 dp = data_read_id(dp, &schema);
426 dp = data_skip_schema(data, dp, schema);
430 if (key->storage == KEY_STORAGE_INCORE)
431 dp = data_skip(dp, key->type);
432 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
434 dp = data_skip(dp, REPOKEY_TYPE_ID);
435 dp = data_skip(dp, REPOKEY_TYPE_ID);
441 static unsigned char *
442 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
448 if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
451 for (i = 0; (k = *keyp++) != 0; i++)
453 return data->incoredata + data->mainschemaoffsets[i];
456 while ((k = *keyp++) != 0)
460 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
462 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
463 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
466 if (data->keys[k].storage != KEY_STORAGE_INCORE)
468 dp = data_skip_key(data, dp, data->keys + k);
473 static unsigned char *
474 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
479 if (off >= data->lastverticaloffset)
481 off -= data->lastverticaloffset;
482 if (off + len > data->vincorelen)
484 return data->vincore + off;
486 if (off + len > key->size)
488 /* we now have the offset, go into vertical */
489 off += data->verticaloffset[key - data->keys];
490 /* fprintf(stderr, "key %d page %d\n", key->name, off / REPOPAGE_BLOBSIZE); */
491 dp = repopagestore_load_page_range(&data->store, off / REPOPAGE_BLOBSIZE, (off + len - 1) / REPOPAGE_BLOBSIZE);
494 dp += off % REPOPAGE_BLOBSIZE;
498 static inline unsigned char *
499 get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
501 unsigned char *dp = *dpp;
505 if (key->storage == KEY_STORAGE_INCORE)
508 *dpp = data_skip_key(data, dp, key);
511 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
514 dp = data_read_id(dp, &off);
515 dp = data_read_id(dp, &len);
518 return get_vertical_data(data, key, off, len);
524 load_repodata(Repodata *data)
526 if (data->loadcallback)
528 data->loadcallback(data);
529 if (data->state == REPODATA_AVAILABLE)
532 data->state = REPODATA_ERROR;
537 maybe_load_repodata(Repodata *data, Id keyname)
539 if (keyname && !repodata_precheck_keyname(data, keyname))
540 return 0; /* do not bother... */
547 for (i = 1; i < data->nkeys; i++)
548 if (keyname == data->keys[i].name)
550 if (i == data->nkeys)
553 return load_repodata(data);
556 case REPODATA_AVAILABLE:
557 case REPODATA_LOADING:
560 data->state = REPODATA_ERROR;
565 static inline unsigned char *
566 solvid2data(Repodata *data, Id solvid, Id *schemap)
568 unsigned char *dp = data->incoredata;
571 if (solvid == SOLVID_META)
572 dp += 1; /* offset of "meta" solvable */
573 else if (solvid == SOLVID_POS)
575 Pool *pool = data->repo->pool;
576 if (data->repo != pool->pos.repo)
578 if (data != data->repo->repodata + pool->pos.repodataid)
581 if (pool->pos.dp != 1)
583 *schemap = pool->pos.schema;
589 if (solvid < data->start || solvid >= data->end)
591 dp += data->incoreoffset[solvid - data->start];
593 return data_read_id(dp, schemap);
596 /************************************************************************
600 static unsigned char *
601 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
604 Id schema, *keyp, *kp;
607 if (!maybe_load_repodata(data, keyname))
609 dp = solvid2data(data, solvid, &schema);
612 keyp = data->schemadata + data->schemata[schema];
613 for (kp = keyp; *kp; kp++)
614 if (data->keys[*kp].name == keyname)
618 *keypp = key = data->keys + *kp;
619 if (key->type == REPOKEY_TYPE_DELETED)
621 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
622 return dp; /* no need to forward... */
623 if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
624 return 0; /* get_data will not work, no need to forward */
625 dp = forward_to_key(data, *kp, keyp, dp);
628 return get_data(data, key, &dp, 0);
632 repodata_lookup_type(Repodata *data, Id solvid, Id keyname)
634 Id schema, *keyp, *kp;
635 if (!maybe_load_repodata(data, keyname))
637 if (!solvid2data(data, solvid, &schema))
639 keyp = data->schemadata + data->schemata[schema];
640 for (kp = keyp; *kp; kp++)
641 if (data->keys[*kp].name == keyname)
642 return data->keys[*kp].type;
647 repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
653 dp = find_key_data(data, solvid, keyname, &key);
656 if (key->type == REPOKEY_TYPE_CONSTANTID)
658 if (key->type != REPOKEY_TYPE_ID)
660 dp = data_read_id(dp, &id);
665 repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
671 dp = find_key_data(data, solvid, keyname, &key);
674 if (key->type == REPOKEY_TYPE_STR)
675 return (const char *)dp;
676 if (key->type == REPOKEY_TYPE_CONSTANTID)
678 else if (key->type == REPOKEY_TYPE_ID)
679 dp = data_read_id(dp, &id);
683 return stringpool_id2str(&data->spool, id);
684 return pool_id2str(data->repo->pool, id);
688 repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long *value)
692 unsigned int high, low;
695 dp = find_key_data(data, solvid, keyname, &key);
700 case REPOKEY_TYPE_NUM:
701 data_read_num64(dp, &low, &high);
702 *value = (unsigned long long)high << 32 | low;
704 case REPOKEY_TYPE_U32:
705 data_read_u32(dp, &low);
708 case REPOKEY_TYPE_CONSTANT:
717 repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
723 if (!maybe_load_repodata(data, keyname))
725 dp = solvid2data(data, solvid, &schema);
728 /* can't use find_key_data as we need to test the type */
729 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
730 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
735 const unsigned char *
736 repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
741 dp = find_key_data(data, solvid, keyname, &key);
756 repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
764 dp = find_key_data(data, solvid, keyname, &key);
767 if (key->type != REPOKEY_TYPE_IDARRAY)
771 dp = data_read_ideof(dp, &id, &eof);
780 repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp)
786 dp = find_key_data(data, solvid, keyname, &key);
787 if (!dp || key->type != REPOKEY_TYPE_BINARY)
792 dp = data_read_id(dp, &len);
798 repodata_globalize_id(Repodata *data, Id id, int create)
800 if (!id || !data || !data->localpool)
802 return pool_str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
806 repodata_localize_id(Repodata *data, Id id, int create)
808 if (!id || !data || !data->localpool)
810 return stringpool_str2id(&data->spool, pool_id2str(data->repo->pool, id), create);
814 repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create)
816 if (!id || !data || !fromdata)
818 if (!data->localpool || !fromdata->localpool)
820 if (fromdata->localpool)
821 id = repodata_globalize_id(fromdata, id, create);
823 id = repodata_localize_id(data, id, create);
826 /* localpool is set in both data and fromdata */
827 return stringpool_str2id(&data->spool, stringpool_id2str(&fromdata->spool, id), create);
831 repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id voidid)
836 ap = data->attrs[solvid - data->start];
841 if (data->keys[*ap].name != keyname)
843 if (data->keys[*ap].type == REPOKEY_TYPE_VOID)
845 if (data->keys[*ap].type == REPOKEY_TYPE_ID)
853 /************************************************************************
859 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
863 case REPOKEY_TYPE_ID:
864 case REPOKEY_TYPE_CONSTANTID:
865 case REPOKEY_TYPE_IDARRAY:
866 if (data && data->localpool)
867 kv->str = stringpool_id2str(&data->spool, kv->id);
869 kv->str = pool_id2str(pool, kv->id);
870 if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
873 for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
875 if (*s == ':' && s > kv->str)
879 case REPOKEY_TYPE_STR:
881 case REPOKEY_TYPE_DIRSTRARRAY:
882 if (!(flags & SEARCH_FILES))
883 return kv->str; /* match just the basename */
885 return kv->str; /* already stringified */
886 /* Put the full filename into kv->str. */
887 kv->str = repodata_dir2str(data, kv->id, kv->str);
888 kv->num = 1; /* mark stringification */
891 if (!(flags & SEARCH_CHECKSUMS))
892 return 0; /* skip em */
894 return kv->str; /* already stringified */
895 kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
896 kv->num = 1; /* mark stringification */
904 struct subschema_data {
910 /* search a specific repodata */
912 repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
916 Id keyid, *kp, *keyp;
917 unsigned char *dp, *ddp;
923 if (!maybe_load_repodata(data, keyname))
925 if (solvid == SOLVID_SUBSCHEMA)
927 struct subschema_data *subd = cbdata;
928 cbdata = subd->cbdata;
930 schema = subd->parent->id;
931 dp = (unsigned char *)subd->parent->str;
932 kv.parent = subd->parent;
937 dp = solvid2data(data, solvid, &schema);
940 s = data->repo->pool->solvables + solvid;
943 keyp = data->schemadata + data->schemata[schema];
946 /* search for a specific key */
947 for (kp = keyp; *kp; kp++)
948 if (data->keys[*kp].name == keyname)
952 dp = forward_to_key(data, *kp, keyp, dp);
958 while ((keyid = *keyp++) != 0)
961 key = data->keys + keyid;
962 ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
964 if (key->type == REPOKEY_TYPE_DELETED)
966 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
968 struct subschema_data subd;
972 subd.cbdata = cbdata;
975 ddp = data_read_id(ddp, &nentries);
979 while (ddp && nentries > 0)
983 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
984 ddp = data_read_id(ddp, &schema);
986 kv.str = (char *)ddp;
987 stop = callback(cbdata, s, data, key, &kv);
988 if (stop > SEARCH_NEXT_KEY)
990 if (stop && stop != SEARCH_ENTERSUB)
992 if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
993 repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
994 ddp = data_skip_schema(data, ddp, schema);
997 if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
1001 kv.str = (char *)ddp;
1002 stop = callback(cbdata, s, data, key, &kv);
1003 if (stop > SEARCH_NEXT_KEY)
1013 ddp = data_fetch(ddp, &kv, key);
1016 stop = callback(cbdata, s, data, key, &kv);
1019 while (!kv.eof && !stop);
1020 if (onekey || stop > SEARCH_NEXT_KEY)
1026 repodata_setpos_kv(Repodata *data, KeyValue *kv)
1028 Pool *pool = data->repo->pool;
1030 pool_clear_pos(pool);
1033 pool->pos.repo = data->repo;
1034 pool->pos.repodataid = data - data->repo->repodata;
1035 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
1036 pool->pos.schema = kv->id;
1040 /************************************************************************
1041 * data iterator functions
1045 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
1059 case SOLVABLE_VENDOR:
1062 case SOLVABLE_PROVIDES:
1064 return s->provides ? s->repo->idarraydata + s->provides : 0;
1065 case SOLVABLE_OBSOLETES:
1067 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
1068 case SOLVABLE_CONFLICTS:
1070 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
1071 case SOLVABLE_REQUIRES:
1073 return s->requires ? s->repo->idarraydata + s->requires : 0;
1074 case SOLVABLE_RECOMMENDS:
1076 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
1077 case SOLVABLE_SUPPLEMENTS:
1079 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
1080 case SOLVABLE_SUGGESTS:
1082 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
1083 case SOLVABLE_ENHANCES:
1085 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
1088 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
1095 datamatcher_init(Datamatcher *ma, const char *match, int flags)
1097 match = match ? solv_strdup(match) : 0;
1102 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
1104 ma->matchdata = solv_calloc(1, sizeof(regex_t));
1105 ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
1108 solv_free(ma->matchdata);
1109 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
1112 if ((flags & SEARCH_FILES) != 0 && match)
1114 /* prepare basename check */
1115 if ((flags & SEARCH_STRINGMASK) == SEARCH_STRING || (flags & SEARCH_STRINGMASK) == SEARCH_STRINGEND)
1117 const char *p = strrchr(match, '/');
1118 ma->matchdata = (void *)(p ? p + 1 : match);
1120 else if ((flags & SEARCH_STRINGMASK) == SEARCH_GLOB)
1123 for (p = match + strlen(match) - 1; p >= match; p--)
1124 if (*p == '[' || *p == ']' || *p == '*' || *p == '?' || *p == '/')
1126 ma->matchdata = (void *)(p + 1);
1133 datamatcher_free(Datamatcher *ma)
1136 ma->match = solv_free((char *)ma->match);
1137 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
1139 regfree(ma->matchdata);
1140 solv_free(ma->matchdata);
1146 datamatcher_match(Datamatcher *ma, const char *str)
1149 switch ((ma->flags & SEARCH_STRINGMASK))
1151 case SEARCH_SUBSTRING:
1152 if (ma->flags & SEARCH_NOCASE)
1153 return strcasestr(str, ma->match) != 0;
1155 return strstr(str, ma->match) != 0;
1157 if (ma->flags & SEARCH_NOCASE)
1158 return !strcasecmp(ma->match, str);
1160 return !strcmp(ma->match, str);
1161 case SEARCH_STRINGSTART:
1162 if (ma->flags & SEARCH_NOCASE)
1163 return !strncasecmp(ma->match, str, strlen(ma->match));
1165 return !strncmp(ma->match, str, strlen(ma->match));
1166 case SEARCH_STRINGEND:
1167 l = strlen(str) - strlen(ma->match);
1170 if (ma->flags & SEARCH_NOCASE)
1171 return !strcasecmp(ma->match, str + l);
1173 return !strcmp(ma->match, str + l);
1175 return !fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0);
1177 return !regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0);
1183 /* check if the matcher can match the provides basename */
1186 datamatcher_checkbasename(Datamatcher *ma, const char *basename)
1189 const char *match = ma->matchdata;
1192 switch (ma->flags & SEARCH_STRINGMASK)
1196 case SEARCH_STRINGEND:
1197 if (match != ma->match)
1198 break; /* had slash, do exact match on basename */
1201 /* check if the basename ends with match */
1202 l = strlen(basename) - strlen(match);
1208 return 1; /* maybe matches */
1210 if ((ma->flags & SEARCH_NOCASE) != 0)
1211 return !strcasecmp(match, basename);
1213 return !strcmp(match, basename);
1217 repodata_filelistfilter_matches(Repodata *data, const char *str)
1219 /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
1220 /* for now hardcoded */
1221 if (strstr(str, "bin/"))
1223 if (!strncmp(str, "/etc/", 5))
1225 if (!strcmp(str, "/usr/lib/sendmail"))
1247 di_nextarrayelement,
1253 di_entersolvablekey,
1257 /* see dataiterator.h for documentation */
1259 dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
1261 memset(di, 0, sizeof(*di));
1263 di->flags = flags & ~SEARCH_THISSOLVID;
1264 if (!pool || (repo && repo->pool != pool))
1272 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1278 di->keyname = keyname;
1279 di->keynames[0] = keyname;
1280 dataiterator_set_search(di, repo, p);
1285 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
1290 if (di->dupstr == di->kv.str)
1291 di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
1298 memset(&di->matcher, 0, sizeof(di->matcher));
1299 if (from->matcher.match)
1300 datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1305 for (i = 1; i < di->nparents; i++)
1306 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1307 di->kv.parent = &di->parents[di->nparents - 1].kv;
1312 dataiterator_set_match(Dataiterator *di, const char *match, int flags)
1314 di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
1315 datamatcher_free(&di->matcher);
1316 memset(&di->matcher, 0, sizeof(di->matcher));
1320 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1330 dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
1334 di->flags &= ~SEARCH_THISSOLVID;
1338 if (!di->pool->urepos)
1346 di->repo = di->pool->repos[di->repoid];
1348 di->state = di_enterrepo;
1350 dataiterator_jump_to_solvid(di, p);
1354 dataiterator_set_keyname(Dataiterator *di, Id keyname)
1357 di->keyname = keyname;
1358 di->keynames[0] = keyname;
1362 dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1366 if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1368 di->state = di_bye; /* sorry */
1371 for (i = di->nkeynames + 1; i > 0; i--)
1372 di->keynames[i] = di->keynames[i - 1];
1373 di->keynames[0] = di->keyname = keyname;
1378 dataiterator_free(Dataiterator *di)
1380 if (di->matcher.match)
1381 datamatcher_free(&di->matcher);
1383 solv_free(di->dupstr);
1386 static unsigned char *
1387 dataiterator_find_keyname(Dataiterator *di, Id keyname)
1390 Repokey *keys = di->data->keys, *key;
1393 for (keyp = di->keyp; *keyp; keyp++)
1394 if (keys[*keyp].name == keyname)
1399 if (key->type == REPOKEY_TYPE_DELETED)
1401 if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
1402 return 0; /* get_data will not work, no need to forward */
1403 dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1411 is_filelist_extension(Repodata *data)
1414 if (!repodata_precheck_keyname(data, SOLVABLE_FILELIST))
1416 for (j = 1; j < data->nkeys; j++)
1417 if (data->keys[j].name == SOLVABLE_FILELIST)
1419 if (j == data->nkeys)
1421 if (data->state != REPODATA_AVAILABLE)
1423 for (j = 1; j < data->nkeys; j++)
1424 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1430 dataiterator_filelistcheck(Dataiterator *di)
1433 int needcomplete = 0;
1434 Repodata *data = di->data;
1436 if ((di->flags & SEARCH_COMPLETE_FILELIST) != 0)
1437 if (!di->matcher.match
1438 || ((di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING
1439 && (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_GLOB)
1440 || !repodata_filelistfilter_matches(data, di->matcher.match))
1442 if (data->state != REPODATA_AVAILABLE)
1443 return needcomplete ? 1 : 0;
1446 /* we don't need the complete filelist, so ignore all stubs */
1447 for (j = 1; j < data->nkeys; j++)
1448 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1454 /* we need the complete filelist. check if we habe a filtered filelist and there's
1455 * a extension with the complete filelist later on */
1456 for (j = 1; j < data->nkeys; j++)
1457 if (data->keys[j].name == SOLVABLE_FILELIST)
1459 if (j == data->nkeys)
1460 return 0; /* does not have filelist */
1461 for (j = 1; j < data->nkeys; j++)
1462 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1464 if (j == data->nkeys)
1465 return 1; /* this is the externsion */
1466 while (data - data->repo->repodata + 1 < data->repo->nrepodata)
1469 if (is_filelist_extension(data))
1477 dataiterator_step(Dataiterator *di)
1481 if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate) {
1482 unsigned int ddpoff = di->ddp - di->vert_ddp;
1483 di->vert_off += ddpoff;
1484 di->vert_len -= ddpoff;
1485 di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len);
1486 di->vert_storestate = di->data->storestate;
1488 di->state = di_nextkey;
1494 case di_enterrepo: di_enterrepo:
1495 if (!di->repo || (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS)))
1497 if (!(di->flags & SEARCH_THISSOLVID))
1499 di->solvid = di->repo->start - 1; /* reset solvid iterator */
1500 goto di_nextsolvable;
1504 case di_entersolvable: di_entersolvable:
1507 di->repodataid = 1; /* reset repodata iterator */
1508 if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1510 extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1];
1512 di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1514 goto di_entersolvablekey;
1519 case di_enterrepodata: di_enterrepodata:
1522 if (di->repodataid >= di->repo->nrepodata)
1523 goto di_nextsolvable;
1524 di->data = di->repo->repodata + di->repodataid;
1526 if (di->repodataid && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1527 goto di_nextrepodata;
1528 if (!maybe_load_repodata(di->data, di->keyname))
1529 goto di_nextrepodata;
1530 di->dp = solvid2data(di->data, di->solvid, &schema);
1532 goto di_nextrepodata;
1533 if (di->solvid == SOLVID_POS)
1534 di->solvid = di->pool->pos.solvid;
1535 /* reset key iterator */
1536 di->keyp = di->data->schemadata + di->data->schemata[schema];
1539 case di_enterschema: di_enterschema:
1541 di->dp = dataiterator_find_keyname(di, di->keyname);
1542 if (!di->dp || !*di->keyp)
1546 goto di_nextrepodata;
1550 case di_enterkey: di_enterkey:
1552 di->key = di->data->keys + *di->keyp;
1555 /* this is get_data() modified to store vert_ data */
1556 if (di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1559 di->dp = data_read_id(di->dp, &off);
1560 di->dp = data_read_id(di->dp, &len);
1561 di->vert_ddp = di->ddp = get_vertical_data(di->data, di->key, off, len);
1564 di->vert_storestate = di->data->storestate;
1566 else if (di->key->storage == KEY_STORAGE_INCORE)
1569 if (di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0))
1570 di->dp = data_skip_key(di->data, di->dp, di->key);
1576 if (di->key->type == REPOKEY_TYPE_DELETED)
1578 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1580 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1586 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1588 di->state = di_nextkey;
1590 di->state = di_nextattr;
1593 case di_nextkey: di_nextkey:
1594 if (!di->keyname && *++di->keyp)
1600 case di_nextrepodata: di_nextrepodata:
1601 if (di->repodataid && ++di->repodataid < di->repo->nrepodata)
1602 goto di_enterrepodata;
1605 case di_nextsolvable: di_nextsolvable:
1606 if (!(di->flags & SEARCH_THISSOLVID))
1609 di->solvid = di->repo->start;
1612 for (; di->solvid < di->repo->end; di->solvid++)
1614 if (di->pool->solvables[di->solvid].repo == di->repo)
1615 goto di_entersolvable;
1620 case di_nextrepo: di_nextrepo:
1625 if (di->repoid < di->pool->nrepos)
1627 di->repo = di->pool->repos[di->repoid];
1633 case di_bye: di_bye:
1637 case di_enterarray: di_enterarray:
1638 if (di->key->name == REPOSITORY_SOLVABLES)
1640 di->ddp = data_read_id(di->ddp, (Id *)&di->kv.num);
1645 case di_nextarrayelement: di_nextarrayelement:
1648 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1649 if (di->kv.entry == di->kv.num)
1651 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1653 if (!(di->flags & SEARCH_ARRAYSENTINEL))
1655 di->kv.str = (char *)di->ddp;
1657 di->state = di_nextkey;
1660 if (di->kv.entry == di->kv.num - 1)
1662 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1663 di->ddp = data_read_id(di->ddp, &di->kv.id);
1664 di->kv.str = (char *)di->ddp;
1665 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1667 if ((di->flags & SEARCH_SUB) != 0)
1668 di->state = di_entersub;
1670 di->state = di_nextarrayelement;
1673 case di_entersub: di_entersub:
1674 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1675 goto di_nextarrayelement; /* sorry, full */
1676 di->parents[di->nparents].kv = di->kv;
1677 di->parents[di->nparents].dp = di->dp;
1678 di->parents[di->nparents].keyp = di->keyp;
1679 di->dp = (unsigned char *)di->kv.str;
1680 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1681 memset(&di->kv, 0, sizeof(di->kv));
1682 di->kv.parent = &di->parents[di->nparents].kv;
1684 di->keyname = di->keynames[di->nparents - di->rootlevel];
1685 goto di_enterschema;
1687 case di_leavesub: di_leavesub:
1688 if (di->nparents - 1 < di->rootlevel)
1691 di->dp = di->parents[di->nparents].dp;
1692 di->kv = di->parents[di->nparents].kv;
1693 di->keyp = di->parents[di->nparents].keyp;
1694 di->key = di->data->keys + *di->keyp;
1695 di->ddp = (unsigned char *)di->kv.str;
1696 di->keyname = di->keynames[di->nparents - di->rootlevel];
1697 goto di_nextarrayelement;
1699 /* special solvable attr handling follows */
1701 case di_nextsolvablekey: di_nextsolvablekey:
1702 if (di->keyname || di->key->name == RPM_RPMDBID)
1703 goto di_enterrepodata;
1707 case di_entersolvablekey: di_entersolvablekey:
1708 di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1709 if (!di->idp || !*di->idp)
1710 goto di_nextsolvablekey;
1714 di->kv.id = *di->idp;
1715 di->kv.num = *di->idp; /* for rpmdbid */
1716 di->kv.num2 = 0; /* for rpmdbid */
1718 di->state = di_nextsolvablekey;
1724 case di_nextsolvableattr:
1725 di->state = di_nextsolvableattr;
1726 di->kv.id = *di->idp++;
1731 di->state = di_nextsolvablekey;
1737 if (di->matcher.match)
1740 /* simple pre-check so that we don't need to stringify */
1741 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0)
1742 if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
1744 if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
1746 if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
1750 if (!datamatcher_match(&di->matcher, str))
1755 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->flags & SEARCH_FILES) != 0)
1756 repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags);
1758 /* found something! */
1764 dataiterator_entersub(Dataiterator *di)
1766 if (di->state == di_nextarrayelement)
1767 di->state = di_entersub;
1771 dataiterator_setpos(Dataiterator *di)
1773 if (di->kv.eof == 2)
1775 pool_clear_pos(di->pool);
1778 di->pool->pos.solvid = di->solvid;
1779 di->pool->pos.repo = di->repo;
1780 di->pool->pos.repodataid = di->data - di->repo->repodata;
1781 di->pool->pos.schema = di->kv.id;
1782 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1786 dataiterator_setpos_parent(Dataiterator *di)
1788 if (!di->kv.parent || di->kv.parent->eof == 2)
1790 pool_clear_pos(di->pool);
1793 di->pool->pos.solvid = di->solvid;
1794 di->pool->pos.repo = di->repo;
1795 di->pool->pos.repodataid = di->data - di->repo->repodata;
1796 di->pool->pos.schema = di->kv.parent->id;
1797 di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1800 /* clones just the position, not the search keys/matcher */
1802 dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
1804 di->state = from->state;
1805 di->flags &= ~SEARCH_THISSOLVID;
1806 di->flags |= (from->flags & SEARCH_THISSOLVID);
1807 di->repo = from->repo;
1808 di->data = from->data;
1810 di->ddp = from->ddp;
1811 di->idp = from->idp;
1812 di->keyp = from->keyp;
1813 di->key = from->key;
1815 di->repodataid = from->repodataid;
1816 di->solvid = from->solvid;
1817 di->repoid = from->repoid;
1818 di->rootlevel = from->rootlevel;
1819 memcpy(di->parents, from->parents, sizeof(from->parents));
1820 di->nparents = from->nparents;
1824 for (i = 1; i < di->nparents; i++)
1825 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1826 di->kv.parent = &di->parents[di->nparents - 1].kv;
1830 if (from->dupstr && from->dupstr == from->kv.str)
1832 di->dupstrn = from->dupstrn;
1833 di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
1838 dataiterator_seek(Dataiterator *di, int whence)
1840 if ((whence & DI_SEEK_STAY) != 0)
1841 di->rootlevel = di->nparents;
1842 switch (whence & ~DI_SEEK_STAY)
1845 if (di->state != di_nextarrayelement)
1847 if ((whence & DI_SEEK_STAY) != 0)
1848 di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1849 di->state = di_entersub;
1851 case DI_SEEK_PARENT:
1858 if (di->rootlevel > di->nparents)
1859 di->rootlevel = di->nparents;
1860 di->dp = di->parents[di->nparents].dp;
1861 di->kv = di->parents[di->nparents].kv;
1862 di->keyp = di->parents[di->nparents].keyp;
1863 di->key = di->data->keys + *di->keyp;
1864 di->ddp = (unsigned char *)di->kv.str;
1865 di->keyname = di->keynames[di->nparents - di->rootlevel];
1866 di->state = di_nextarrayelement;
1868 case DI_SEEK_REWIND:
1874 di->dp = (unsigned char *)di->kv.parent->str;
1875 di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1876 di->state = di_enterschema;
1884 dataiterator_skip_attribute(Dataiterator *di)
1886 if (di->state == di_nextsolvableattr)
1887 di->state = di_nextsolvablekey;
1889 di->state = di_nextkey;
1893 dataiterator_skip_solvable(Dataiterator *di)
1898 di->keyname = di->keynames[0];
1899 di->state = di_nextsolvable;
1903 dataiterator_skip_repo(Dataiterator *di)
1908 di->keyname = di->keynames[0];
1909 di->state = di_nextrepo;
1913 dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1918 di->keyname = di->keynames[0];
1919 if (solvid == SOLVID_POS)
1921 di->repo = di->pool->pos.repo;
1928 di->data = di->repo->repodata + di->pool->pos.repodataid;
1930 di->solvid = solvid;
1931 di->state = di_enterrepo;
1932 di->flags |= SEARCH_THISSOLVID;
1937 di->repo = di->pool->solvables[solvid].repo;
1940 else if (di->repoid > 0)
1942 if (!di->pool->urepos)
1948 di->repo = di->pool->repos[di->repoid];
1951 di->solvid = solvid;
1953 di->flags |= SEARCH_THISSOLVID;
1954 di->state = di_enterrepo;
1958 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1964 di->repoid = 0; /* 0 means stay at repo */
1967 di->flags &= ~SEARCH_THISSOLVID;
1968 di->state = di_enterrepo;
1972 dataiterator_match(Dataiterator *di, Datamatcher *ma)
1975 if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
1977 return ma ? datamatcher_match(ma, str) : 1;
1981 dataiterator_strdup(Dataiterator *di)
1985 if (!di->kv.str || di->kv.str == di->dupstr)
1987 switch (di->key->type)
1990 case REPOKEY_TYPE_DIRSTRARRAY:
1991 if (di->kv.num) /* was it stringified into tmp space? */
1992 l = strlen(di->kv.str) + 1;
1997 if (l < 0 && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1999 switch (di->key->type)
2001 case REPOKEY_TYPE_STR:
2002 case REPOKEY_TYPE_DIRSTRARRAY:
2003 l = strlen(di->kv.str) + 1;
2006 l = solv_chksum_len(di->key->type);
2008 case REPOKEY_TYPE_BINARY:
2015 if (!di->dupstrn || di->dupstrn < l)
2017 di->dupstrn = l + 16;
2018 di->dupstr = solv_realloc(di->dupstr, di->dupstrn);
2021 memcpy(di->dupstr, di->kv.str, l);
2022 di->kv.str = di->dupstr;
2026 /************************************************************************
2027 * data modify functions
2030 /* extend repodata so that it includes solvables p */
2032 repodata_extend(Repodata *data, Id p)
2034 if (data->start == data->end)
2035 data->start = data->end = p;
2038 int old = data->end - data->start;
2039 int new = p - data->end + 1;
2042 data->attrs = solv_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
2043 memset(data->attrs + old, 0, new * sizeof(Id *));
2045 data->incoreoffset = solv_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
2046 memset(data->incoreoffset + old, 0, new * sizeof(Id));
2049 if (p < data->start)
2051 int old = data->end - data->start;
2052 int new = data->start - p;
2055 data->attrs = solv_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
2056 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
2057 memset(data->attrs, 0, new * sizeof(Id *));
2059 data->incoreoffset = solv_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
2060 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
2061 memset(data->incoreoffset, 0, new * sizeof(Id));
2066 /* shrink end of repodata */
2068 repodata_shrink(Repodata *data, int end)
2072 if (data->end <= end)
2074 if (data->start >= end)
2078 for (i = 0; i < data->end - data->start; i++)
2079 solv_free(data->attrs[i]);
2080 data->attrs = solv_free(data->attrs);
2082 data->incoreoffset = solv_free(data->incoreoffset);
2083 data->start = data->end = 0;
2088 for (i = end; i < data->end; i++)
2089 solv_free(data->attrs[i - data->start]);
2090 data->attrs = solv_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
2092 if (data->incoreoffset)
2093 data->incoreoffset = solv_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
2097 /* extend repodata so that it includes solvables from start to start + num - 1 */
2099 repodata_extend_block(Repodata *data, Id start, Id num)
2103 if (!data->incoreoffset)
2105 data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
2106 data->start = start;
2107 data->end = start + num;
2110 repodata_extend(data, start);
2112 repodata_extend(data, start + num - 1);
2115 /**********************************************************************/
2118 #define REPODATA_ATTRS_BLOCK 31
2119 #define REPODATA_ATTRDATA_BLOCK 1023
2120 #define REPODATA_ATTRIDDATA_BLOCK 63
2121 #define REPODATA_ATTRNUM64DATA_BLOCK 15
2125 repodata_new_handle(Repodata *data)
2129 data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
2130 data->nxattrs = 2; /* -1: SOLVID_META */
2132 data->xattrs = solv_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
2133 data->xattrs[data->nxattrs] = 0;
2134 return -(data->nxattrs++);
2138 repodata_get_attrp(Repodata *data, Id handle)
2142 if (handle == SOLVID_META && !data->xattrs)
2144 data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
2147 return data->xattrs - handle;
2149 if (handle < data->start || handle >= data->end)
2150 repodata_extend(data, handle);
2152 data->attrs = solv_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
2153 return data->attrs + (handle - data->start);
2157 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
2163 app = repodata_get_attrp(data, handle);
2168 /* Determine equality based on the name only, allows us to change
2169 type (when overwrite is set), and makes TYPE_CONSTANT work. */
2170 for (pp = ap; *pp; pp += 2)
2171 if (data->keys[*pp].name == data->keys[keyid].name)
2175 if (overwrite || data->keys[*pp].type == REPOKEY_TYPE_DELETED)
2184 ap = solv_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
2194 repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
2198 keyid = repodata_key2id(data, key, 1);
2199 repodata_insert_keyid(data, solvid, keyid, val, 1);
2203 repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
2207 key.type = REPOKEY_TYPE_ID;
2209 key.storage = KEY_STORAGE_INCORE;
2210 repodata_set(data, solvid, &key, id);
2214 repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned long long num)
2218 key.type = REPOKEY_TYPE_NUM;
2220 key.storage = KEY_STORAGE_INCORE;
2221 if (num >= 0x80000000)
2223 data->attrnum64data = solv_extend(data->attrnum64data, data->attrnum64datalen, 1, sizeof(unsigned long long), REPODATA_ATTRNUM64DATA_BLOCK);
2224 data->attrnum64data[data->attrnum64datalen] = num;
2225 num = 0x80000000 | data->attrnum64datalen++;
2227 repodata_set(data, solvid, &key, (Id)num);
2231 repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
2235 if (data->localpool)
2236 id = stringpool_str2id(&data->spool, str, 1);
2238 id = pool_str2id(data->repo->pool, str, 1);
2240 key.type = REPOKEY_TYPE_ID;
2242 key.storage = KEY_STORAGE_INCORE;
2243 repodata_set(data, solvid, &key, id);
2247 repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
2251 key.type = REPOKEY_TYPE_CONSTANT;
2252 key.size = constant;
2253 key.storage = KEY_STORAGE_INCORE;
2254 repodata_set(data, solvid, &key, 0);
2258 repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
2262 key.type = REPOKEY_TYPE_CONSTANTID;
2264 key.storage = KEY_STORAGE_INCORE;
2265 repodata_set(data, solvid, &key, 0);
2269 repodata_set_void(Repodata *data, Id solvid, Id keyname)
2273 key.type = REPOKEY_TYPE_VOID;
2275 key.storage = KEY_STORAGE_INCORE;
2276 repodata_set(data, solvid, &key, 0);
2280 repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
2285 l = strlen(str) + 1;
2287 key.type = REPOKEY_TYPE_STR;
2289 key.storage = KEY_STORAGE_INCORE;
2290 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2291 memcpy(data->attrdata + data->attrdatalen, str, l);
2292 repodata_set(data, solvid, &key, data->attrdatalen);
2293 data->attrdatalen += l;
2297 repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len)
2305 key.type = REPOKEY_TYPE_BINARY;
2307 key.storage = KEY_STORAGE_INCORE;
2308 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK);
2309 dp = data->attrdata + data->attrdatalen;
2310 if (len >= (1 << 14))
2312 if (len >= (1 << 28))
2313 *dp++ = (len >> 28) | 128;
2314 if (len >= (1 << 21))
2315 *dp++ = (len >> 21) | 128;
2316 *dp++ = (len >> 14) | 128;
2318 if (len >= (1 << 7))
2319 *dp++ = (len >> 7) | 128;
2322 memcpy(dp, buf, len);
2323 repodata_set(data, solvid, &key, data->attrdatalen);
2324 data->attrdatalen = dp + len - data->attrdata;
2327 /* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
2328 * so that the caller can append entrysize new elements plus the termination zero there */
2330 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
2333 Id *ida, *pp, **ppp;
2335 /* check if it is the same as last time, this speeds things up a lot */
2336 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
2338 /* great! just append the new data */
2339 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2340 data->attriddatalen--; /* overwrite terminating 0 */
2341 data->lastdatalen += entrysize;
2345 ppp = repodata_get_attrp(data, handle);
2349 for (; *pp; pp += 2)
2350 if (data->keys[*pp].name == keyname)
2353 if (!pp || !*pp || data->keys[*pp].type != keytype)
2355 /* not found. allocate new key */
2361 key.storage = KEY_STORAGE_INCORE;
2362 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2363 keyid = repodata_key2id(data, &key, 1);
2364 repodata_insert_keyid(data, handle, keyid, data->attriddatalen, 1);
2365 data->lasthandle = handle;
2366 data->lastkey = keyid;
2367 data->lastdatalen = data->attriddatalen + entrysize + 1;
2371 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2372 oldsize += entrysize;
2373 if (ida + 1 == data->attriddata + data->attriddatalen)
2375 /* this was the last entry, just append it */
2376 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2377 data->attriddatalen--; /* overwrite terminating 0 */
2381 /* too bad. move to back. */
2382 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2383 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2384 pp[1] = data->attriddatalen;
2385 data->attriddatalen += oldsize;
2387 data->lasthandle = handle;
2388 data->lastkey = *pp;
2389 data->lastdatalen = data->attriddatalen + entrysize + 1;
2393 repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2394 const unsigned char *str)
2399 if (!(l = solv_chksum_len(type)))
2404 key.storage = KEY_STORAGE_INCORE;
2405 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2406 memcpy(data->attrdata + data->attrdatalen, str, l);
2407 repodata_set(data, solvid, &key, data->attrdatalen);
2408 data->attrdatalen += l;
2412 repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2415 unsigned char buf[64];
2418 if (!(l = solv_chksum_len(type)))
2420 if (l > sizeof(buf) || solv_hex2bin(&str, buf, l) != l)
2422 repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2426 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2430 if (!(l = solv_chksum_len(type)))
2432 return pool_bin2hex(data->repo->pool, buf, l);
2435 /* rpm filenames don't contain the epoch, so strip it */
2436 static inline const char *
2437 evrid2vrstr(Pool *pool, Id evrid)
2439 const char *p, *evr = pool_id2str(pool, evrid);
2442 for (p = evr; *p >= '0' && *p <= '9'; p++)
2444 return p != evr && *p == ':' && p[1] ? p + 1 : evr;
2448 repodata_set_poolstrn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
2451 if (data->localpool)
2452 id = stringpool_strn2id(&data->spool, str, l, 1);
2454 id = pool_strn2id(data->repo->pool, str, l, 1);
2455 repodata_set_id(data, solvid, keyname, id);
2459 repodata_set_strn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
2462 repodata_set_str(data, solvid, keyname, str);
2465 char *s = solv_strdup(str);
2467 repodata_set_str(data, solvid, keyname, s);
2473 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
2475 Pool *pool = data->repo->pool;
2477 const char *str, *fp;
2481 repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2484 if ((dir = strrchr(file, '/')) != 0)
2495 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2500 if (l == 1 && dir[0] == '.')
2502 s = pool->solvables + solvid;
2505 str = pool_id2str(pool, s->arch);
2506 if (!strncmp(dir, str, l) && !str[l])
2507 repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2509 repodata_set_strn(data, solvid, SOLVABLE_MEDIADIR, dir, l);
2512 str = pool_id2str(pool, s->name);
2514 if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
2517 str = evrid2vrstr(pool, s->evr);
2519 if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
2522 str = pool_id2str(pool, s->arch);
2524 if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2526 repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2531 repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2534 /* XXX: medianr is currently not stored */
2536 repodata_set_deltalocation(Repodata *data, Id handle, int medianr, const char *dir, const char *file)
2539 const char *evr, *suf, *s;
2543 if ((dir = strrchr(file, '/')) != 0)
2554 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2559 if (l == 1 && dir[0] == '.')
2562 repodata_set_poolstrn(data, handle, DELTA_LOCATION_DIR, dir, l);
2563 evr = strchr(file, '-');
2566 for (s = evr - 1; s > file; s--)
2573 suf = strrchr(file, '.');
2576 for (s = suf - 1; s > file; s--)
2582 if (!strcmp(suf, ".delta.rpm") || !strcmp(suf, ".patch.rpm"))
2584 /* We accept one more item as suffix. */
2585 for (s = suf - 1; s > file; s--)
2595 if (suf && evr && suf < evr)
2597 repodata_set_poolstrn(data, handle, DELTA_LOCATION_NAME, file, evr ? evr - file : strlen(file));
2599 repodata_set_poolstrn(data, handle, DELTA_LOCATION_EVR, evr + 1, suf ? suf - evr - 1: strlen(evr + 1));
2601 repodata_set_poolstr(data, handle, DELTA_LOCATION_SUFFIX, suf + 1);
2605 repodata_set_sourcepkg(Repodata *data, Id solvid, const char *sourcepkg)
2607 Pool *pool = data->repo->pool;
2608 Solvable *s = pool->solvables + solvid;
2609 const char *p, *sevr, *sarch, *name, *evr;
2611 p = strrchr(sourcepkg, '.');
2612 if (!p || strcmp(p, ".rpm") != 0)
2615 repodata_set_str(data, solvid, SOLVABLE_SOURCENAME, sourcepkg);
2619 while (p > sourcepkg && *p != '.')
2621 if (*p != '.' || p == sourcepkg)
2624 while (p > sourcepkg && *p != '-')
2626 if (*p != '-' || p == sourcepkg)
2629 while (p > sourcepkg && *p != '-')
2631 if (*p != '-' || p == sourcepkg)
2634 pool = s->repo->pool;
2636 name = pool_id2str(pool, s->name);
2637 if (name && !strncmp(sourcepkg, name, sevr - sourcepkg - 1) && name[sevr - sourcepkg - 1] == 0)
2638 repodata_set_void(data, solvid, SOLVABLE_SOURCENAME);
2640 repodata_set_id(data, solvid, SOLVABLE_SOURCENAME, pool_strn2id(pool, sourcepkg, sevr - sourcepkg - 1, 1));
2642 evr = evrid2vrstr(pool, s->evr);
2643 if (evr && !strncmp(sevr, evr, sarch - sevr - 1) && evr[sarch - sevr - 1] == 0)
2644 repodata_set_void(data, solvid, SOLVABLE_SOURCEEVR);
2646 repodata_set_id(data, solvid, SOLVABLE_SOURCEEVR, pool_strn2id(pool, sevr, sarch - sevr - 1, 1));
2648 if (!strcmp(sarch, "src.rpm"))
2649 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_SRC);
2650 else if (!strcmp(sarch, "nosrc.rpm"))
2651 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_NOSRC);
2653 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, pool_strn2id(pool, sarch, strlen(sarch) - 4, 1));
2657 repodata_set_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
2663 key.type = REPOKEY_TYPE_IDARRAY;
2665 key.storage = KEY_STORAGE_INCORE;
2666 repodata_set(data, solvid, &key, data->attriddatalen);
2667 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, q->count + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2668 for (i = 0; i < q->count; i++)
2669 data->attriddata[data->attriddatalen++] = q->elements[i];
2670 data->attriddata[data->attriddatalen++] = 0;
2674 repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2678 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2680 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2681 data->attriddata[data->attriddatalen++] = dir;
2682 data->attriddata[data->attriddatalen++] = num;
2683 data->attriddata[data->attriddatalen++] = num2;
2684 data->attriddata[data->attriddatalen++] = 0;
2688 repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
2694 l = strlen(str) + 1;
2695 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2696 memcpy(data->attrdata + data->attrdatalen, str, l);
2697 stroff = data->attrdatalen;
2698 data->attrdatalen += l;
2701 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2703 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2704 data->attriddata[data->attriddatalen++] = dir;
2705 data->attriddata[data->attriddatalen++] = stroff;
2706 data->attriddata[data->attriddatalen++] = 0;
2710 repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2713 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2715 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2716 data->attriddata[data->attriddatalen++] = id;
2717 data->attriddata[data->attriddatalen++] = 0;
2721 repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2725 if (data->localpool)
2726 id = stringpool_str2id(&data->spool, str, 1);
2728 id = pool_str2id(data->repo->pool, str, 1);
2729 repodata_add_idarray(data, solvid, keyname, id);
2733 repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2735 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2736 data->attriddata[data->attriddatalen++] = ghandle;
2737 data->attriddata[data->attriddatalen++] = 0;
2741 repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2743 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2744 data->attriddata[data->attriddatalen++] = ghandle;
2745 data->attriddata[data->attriddatalen++] = 0;
2749 repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname)
2752 app = repodata_get_attrp(data, solvid);
2756 for (; *ap; ap += 2)
2757 if (data->keys[*ap].name == keyname)
2763 for (; *ap; ap += 2)
2765 if (data->keys[*ap].name == keyname)
2773 /* XXX: does not work correctly, needs fix in iterators! */
2775 repodata_unset(Repodata *data, Id solvid, Id keyname)
2779 key.type = REPOKEY_TYPE_DELETED;
2781 key.storage = KEY_STORAGE_INCORE;
2782 repodata_set(data, solvid, &key, 0);
2785 /* add all (uninternalized) attrs from src to dest */
2787 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2790 if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
2792 for (; *keyp; keyp += 2)
2793 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2796 /* add some (uninternalized) attrs from src to dest */
2798 repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
2801 if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
2803 for (; *keyp; keyp += 2)
2804 if (!keyidmap || MAPTST(keyidmap, keyp[0]))
2805 repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2808 /* swap (uninternalized) attrs from src and dest */
2810 repodata_swap_attrs(Repodata *data, Id dest, Id src)
2813 if (!data->attrs || dest == src)
2815 tmpattrs = data->attrs[dest - data->start];
2816 data->attrs[dest - data->start] = data->attrs[src - data->start];
2817 data->attrs[src - data->start] = tmpattrs;
2821 /**********************************************************************/
2823 /* TODO: unify with repo_write and repo_solv! */
2825 #define EXTDATA_BLOCK 1023
2833 data_addid(struct extdata *xd, Id sx)
2835 unsigned int x = (unsigned int)sx;
2838 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2839 dp = xd->buf + xd->len;
2844 *dp++ = (x >> 28) | 128;
2846 *dp++ = (x >> 21) | 128;
2847 *dp++ = (x >> 14) | 128;
2850 *dp++ = (x >> 7) | 128;
2852 xd->len = dp - xd->buf;
2856 data_addid64(struct extdata *xd, unsigned long long x)
2858 if (x >= 0x100000000)
2862 data_addid(xd, (Id)(x >> 35));
2863 xd->buf[xd->len - 1] |= 128;
2865 data_addid(xd, (Id)((unsigned int)x | 0x80000000));
2866 xd->buf[xd->len - 5] = (x >> 28) | 128;
2869 data_addid(xd, (Id)x);
2873 data_addideof(struct extdata *xd, Id sx, int eof)
2875 unsigned int x = (unsigned int)sx;
2878 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2879 dp = xd->buf + xd->len;
2884 *dp++ = (x >> 27) | 128;
2886 *dp++ = (x >> 20) | 128;
2887 *dp++ = (x >> 13) | 128;
2890 *dp++ = (x >> 6) | 128;
2891 *dp++ = eof ? (x & 63) : (x & 63) | 64;
2892 xd->len = dp - xd->buf;
2896 data_addblob(struct extdata *xd, unsigned char *blob, int len)
2898 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
2899 memcpy(xd->buf + xd->len, blob, len);
2903 /*********************************/
2905 /* this is to reduct memory usage when internalizing oversized repos */
2907 compact_attrdata(Repodata *data, int entry, int nentry)
2910 unsigned int attrdatastart = data->attrdatalen;
2911 unsigned int attriddatastart = data->attriddatalen;
2912 if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
2914 for (i = entry; i < nentry; i++)
2916 Id v, *attrs = data->attrs[i];
2919 for (; *attrs; attrs += 2)
2921 switch (data->keys[*attrs].type)
2923 case REPOKEY_TYPE_STR:
2924 case REPOKEY_TYPE_BINARY:
2926 if (attrs[1] < attrdatastart)
2927 attrdatastart = attrs[1];
2929 case REPOKEY_TYPE_DIRSTRARRAY:
2930 for (v = attrs[1]; data->attriddata[v] ; v += 2)
2931 if (data->attriddata[v + 1] < attrdatastart)
2932 attrdatastart = data->attriddata[v + 1];
2934 case REPOKEY_TYPE_IDARRAY:
2935 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2936 if (attrs[1] < attriddatastart)
2937 attriddatastart = attrs[1];
2939 case REPOKEY_TYPE_FIXARRAY:
2940 case REPOKEY_TYPE_FLEXARRAY:
2948 printf("compact_attrdata %d %d\n", entry, nentry);
2949 printf("attrdatastart: %d\n", attrdatastart);
2950 printf("attriddatastart: %d\n", attriddatastart);
2952 if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
2954 for (i = entry; i < nentry; i++)
2956 Id v, *attrs = data->attrs[i];
2959 for (; *attrs; attrs += 2)
2961 switch (data->keys[*attrs].type)
2963 case REPOKEY_TYPE_STR:
2964 case REPOKEY_TYPE_BINARY:
2966 attrs[1] -= attrdatastart;
2968 case REPOKEY_TYPE_DIRSTRARRAY:
2969 for (v = attrs[1]; data->attriddata[v] ; v += 2)
2970 data->attriddata[v + 1] -= attrdatastart;
2972 case REPOKEY_TYPE_IDARRAY:
2973 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2974 attrs[1] -= attriddatastart;
2983 data->attrdatalen -= attrdatastart;
2984 memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
2985 data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
2987 if (attriddatastart)
2989 data->attriddatalen -= attriddatastart;
2990 memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
2991 data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2995 /* internalalize some key into incore/vincore data */
2998 repodata_serialize_key(Repodata *data, struct extdata *newincore,
2999 struct extdata *newvincore,
3001 Repokey *key, Id val)
3005 unsigned int oldvincorelen = 0;
3009 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3012 oldvincorelen = xd->len;
3016 case REPOKEY_TYPE_VOID:
3017 case REPOKEY_TYPE_CONSTANT:
3018 case REPOKEY_TYPE_CONSTANTID:
3020 case REPOKEY_TYPE_STR:
3021 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
3023 case REPOKEY_TYPE_MD5:
3024 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
3026 case REPOKEY_TYPE_SHA1:
3027 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
3029 case REPOKEY_TYPE_SHA224:
3030 data_addblob(xd, data->attrdata + val, SIZEOF_SHA224);
3032 case REPOKEY_TYPE_SHA256:
3033 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
3035 case REPOKEY_TYPE_SHA384:
3036 data_addblob(xd, data->attrdata + val, SIZEOF_SHA384);
3038 case REPOKEY_TYPE_SHA512:
3039 data_addblob(xd, data->attrdata + val, SIZEOF_SHA512);
3041 case REPOKEY_TYPE_NUM:
3042 if (val & 0x80000000)
3044 data_addid64(xd, data->attrnum64data[val ^ 0x80000000]);
3048 case REPOKEY_TYPE_ID:
3049 case REPOKEY_TYPE_DIR:
3050 data_addid(xd, val);
3052 case REPOKEY_TYPE_BINARY:
3055 unsigned char *dp = data_read_id(data->attrdata + val, &len);
3056 dp += (unsigned int)len;
3057 data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
3060 case REPOKEY_TYPE_IDARRAY:
3061 for (ida = data->attriddata + val; *ida; ida++)
3062 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
3064 case REPOKEY_TYPE_DIRNUMNUMARRAY:
3065 for (ida = data->attriddata + val; *ida; ida += 3)
3067 data_addid(xd, ida[0]);
3068 data_addid(xd, ida[1]);
3069 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
3072 case REPOKEY_TYPE_DIRSTRARRAY:
3073 for (ida = data->attriddata + val; *ida; ida += 2)
3075 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
3076 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
3079 case REPOKEY_TYPE_FIXARRAY:
3083 for (ida = data->attriddata + val; *ida; ida++)
3087 kp = data->xattrs[-*ida];
3095 schemaid = repodata_schema2id(data, schema, 1);
3096 else if (schemaid != repodata_schema2id(data, schema, 0))
3098 pool_debug(data->repo->pool, SOLV_FATAL, "fixarray substructs with different schemas\n");
3104 data_addid(xd, num);
3105 data_addid(xd, schemaid);
3106 for (ida = data->attriddata + val; *ida; ida++)
3108 Id *kp = data->xattrs[-*ida];
3112 repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3116 case REPOKEY_TYPE_FLEXARRAY:
3119 for (ida = data->attriddata + val; *ida; ida++)
3121 data_addid(xd, num);
3122 for (ida = data->attriddata + val; *ida; ida++)
3124 Id *kp = data->xattrs[-*ida];
3127 data_addid(xd, 0); /* XXX */
3134 schemaid = repodata_schema2id(data, schema, 1);
3135 data_addid(xd, schemaid);
3136 kp = data->xattrs[-*ida];
3138 repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3143 pool_debug(data->repo->pool, SOLV_FATAL, "don't know how to handle type %d\n", key->type);
3146 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3148 /* put offset/len in incore */
3149 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
3150 oldvincorelen = xd->len - oldvincorelen;
3151 data_addid(newincore, oldvincorelen);
3156 repodata_internalize(Repodata *data)
3158 Repokey *key, solvkey;
3160 Id schemaid, keyid, *schema, *sp, oldschema, *keyp, *seen;
3162 unsigned char *dp, *ndp;
3163 int newschema, oldcount;
3164 struct extdata newincore;
3165 struct extdata newvincore;
3168 if (!data->attrs && !data->xattrs)
3172 printf("repodata_internalize %d\n", data->repodataid);
3173 printf(" attr data: %d K\n", data->attrdatalen / 1024);
3174 printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3176 newvincore.buf = data->vincore;
3177 newvincore.len = data->vincorelen;
3179 /* find the solvables key, create if needed */
3180 memset(&solvkey, 0, sizeof(solvkey));
3181 solvkey.name = REPOSITORY_SOLVABLES;
3182 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
3184 solvkey.storage = KEY_STORAGE_INCORE;
3185 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
3187 schema = solv_malloc2(data->nkeys, sizeof(Id));
3188 seen = solv_malloc2(data->nkeys, sizeof(Id));
3190 /* Merge the data already existing (in data->schemata, ->incoredata and
3191 friends) with the new attributes in data->attrs[]. */
3192 nentry = data->end - data->start;
3193 memset(&newincore, 0, sizeof(newincore));
3194 data_addid(&newincore, 0); /* start data at offset 1 */
3196 data->mainschema = 0;
3197 data->mainschemaoffsets = solv_free(data->mainschemaoffsets);
3199 /* join entry data */
3200 /* we start with the meta data, entry -1 */
3201 for (entry = -1; entry < nentry; entry++)
3203 memset(seen, 0, data->nkeys * sizeof(Id));
3205 dp = data->incoredata;
3208 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
3209 dp = data_read_id(dp, &oldschema);
3212 fprintf(stderr, "oldschema %d\n", oldschema);
3213 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
3214 fprintf(stderr, "schemadata %p\n", data->schemadata);
3216 /* seen: -1: old data 0: skipped >0: id + 1 */
3220 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
3224 pool_debug(data->repo->pool, SOLV_FATAL, "Inconsistent old data (key occured twice).\n");
3232 keyp = data->attrs ? data->attrs[entry] : 0;
3235 /* strip solvables key */
3237 for (sp = keyp = schema; *sp; sp++)
3238 if (*sp != solvkeyid)
3243 seen[solvkeyid] = 0;
3244 keyp = data->xattrs ? data->xattrs[1] : 0;
3247 for (; *keyp; keyp += 2)
3254 seen[*keyp] = keyp[1] + 1;
3256 if (entry < 0 && data->end != data->start)
3263 /* Ideally we'd like to sort the new schema here, to ensure
3264 schema equality independend of the ordering. We can't do that
3265 yet. For once see below (old ids need to come before new ids).
3266 An additional difficulty is that we also need to move
3267 the values with the keys. */
3268 schemaid = repodata_schema2id(data, schema, 1);
3270 schemaid = oldschema;
3273 /* Now create data blob. We walk through the (possibly new) schema
3274 and either copy over old data, or insert the new. */
3275 /* XXX Here we rely on the fact that the (new) schema has the form
3276 o1 o2 o3 o4 ... | n1 n2 n3 ...
3277 (oX being the old keyids (possibly overwritten), and nX being
3278 the new keyids). This rules out sorting the keyids in order
3279 to ensure a small schema count. */
3281 data->incoreoffset[entry] = newincore.len;
3282 data_addid(&newincore, schemaid);
3285 data->mainschema = schemaid;
3286 data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
3288 /* we don't use a pointer to the schemadata here as repodata_serialize_key
3289 * may call repodata_schema2id() which might realloc our schemadata */
3290 for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
3293 data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
3294 if (keyid == solvkeyid)
3296 /* add flexarray entry count */
3297 data_addid(&newincore, data->end - data->start);
3300 key = data->keys + keyid;
3302 fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, pool_id2str(data->repo->pool, key->name), pool_id2str(data->repo->pool, key->type));
3307 /* Skip the data associated with this old key. */
3308 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3310 ndp = data_skip(dp, REPOKEY_TYPE_ID);
3311 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
3313 else if (key->storage == KEY_STORAGE_INCORE)
3314 ndp = data_skip_key(data, dp, key);
3317 if (seen[keyid] == -1)
3319 /* If this key was an old one _and_ was not overwritten with
3320 a different value copy over the old value (we skipped it
3323 data_addblob(&newincore, dp, ndp - dp);
3326 else if (seen[keyid])
3328 /* Otherwise we have a new value. Parse it into the internal form. */
3329 repodata_serialize_key(data, &newincore, &newvincore, schema, key, seen[keyid] - 1);
3333 if (entry >= 0 && data->attrs)
3335 if (data->attrs[entry])
3336 data->attrs[entry] = solv_free(data->attrs[entry]);
3337 if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
3339 compact_attrdata(data, entry + 1, nentry); /* try to free some memory */
3341 printf(" attr data: %d K\n", data->attrdatalen / 1024);
3342 printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3343 printf(" incore data: %d K\n", newincore.len / 1024);
3344 printf(" sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
3345 /* malloc_stats(); */
3350 /* free all xattrs */
3351 for (entry = 0; entry < data->nxattrs; entry++)
3352 if (data->xattrs[entry])
3353 solv_free(data->xattrs[entry]);
3354 data->xattrs = solv_free(data->xattrs);
3357 data->lasthandle = 0;
3359 data->lastdatalen = 0;
3362 repodata_free_schemahash(data);
3364 solv_free(data->incoredata);
3365 data->incoredata = newincore.buf;
3366 data->incoredatalen = newincore.len;
3367 data->incoredatafree = 0;
3369 solv_free(data->vincore);
3370 data->vincore = newvincore.buf;
3371 data->vincorelen = newvincore.len;
3373 data->attrs = solv_free(data->attrs);
3374 data->attrdata = solv_free(data->attrdata);
3375 data->attriddata = solv_free(data->attriddata);
3376 data->attrnum64data = solv_free(data->attrnum64data);
3377 data->attrdatalen = 0;
3378 data->attriddatalen = 0;
3379 data->attrnum64datalen = 0;
3381 printf("repodata_internalize %d done\n", data->repodataid);
3382 printf(" incore data: %d K\n", data->incoredatalen / 1024);
3387 repodata_disable_paging(Repodata *data)
3389 if (maybe_load_repodata(data, 0))
3391 repopagestore_disable_paging(&data->store);
3397 repodata_load_stub(Repodata *data)
3399 Repo *repo = data->repo;
3400 Pool *pool = repo->pool;
3402 struct _Pool_tmpspace oldtmpspace;
3405 if (!pool->loadcallback)
3407 data->state = REPODATA_ERROR;
3410 data->state = REPODATA_LOADING;
3412 /* save tmp space and pos */
3413 oldtmpspace = pool->tmpspace;
3414 memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
3417 r = pool->loadcallback(pool, data, pool->loadcallbackdata);
3419 /* restore tmp space and pos */
3420 for (i = 0; i < POOL_TMPSPACEBUF; i++)
3421 solv_free(pool->tmpspace.buf[i]);
3422 pool->tmpspace = oldtmpspace;
3423 if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
3424 memset(&oldpos, 0, sizeof(oldpos));
3427 data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR;
3431 repodata_add_stubkey(Repodata *data, Id keyname, Id keytype)
3435 xkey.name = keyname;
3436 xkey.type = keytype;
3437 xkey.storage = KEY_STORAGE_INCORE;
3439 repodata_key2id(data, &xkey, 1);
3443 repodata_add_stub(Repodata **datap)
3445 Repodata *data = *datap;
3446 Repo *repo = data->repo;
3447 Id repodataid = data - repo->repodata;
3448 Repodata *sdata = repo_add_repodata(repo, 0);
3449 data = repo->repodata + repodataid;
3450 if (data->end > data->start)
3451 repodata_extend_block(sdata, data->start, data->end - data->start);
3452 sdata->state = REPODATA_STUB;
3453 sdata->loadcallback = repodata_load_stub;
3459 repodata_create_stubs(Repodata *data)
3461 Repo *repo = data->repo;
3462 Pool *pool = repo->pool;
3469 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
3470 while (dataiterator_step(&di))
3471 if (di.data == data)
3473 dataiterator_free(&di);
3476 stubdataids = solv_calloc(cnt, sizeof(*stubdataids));
3477 for (i = 0; i < cnt; i++)
3479 sdata = repodata_add_stub(&data);
3480 stubdataids[i] = sdata - repo->repodata;
3483 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
3485 while (dataiterator_step(&di))
3487 if (di.data != data)
3489 if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
3491 dataiterator_entersub(&di);
3492 sdata = repo->repodata + stubdataids[i++];
3496 switch (di.key->type)
3498 case REPOKEY_TYPE_ID:
3499 repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
3501 case REPOKEY_TYPE_CONSTANTID:
3502 repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
3504 case REPOKEY_TYPE_STR:
3505 repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
3507 case REPOKEY_TYPE_VOID:
3508 repodata_set_void(sdata, SOLVID_META, di.key->name);
3510 case REPOKEY_TYPE_NUM:
3511 repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv));
3514 repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
3516 case REPOKEY_TYPE_IDARRAY:
3517 repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
3518 if (di.key->name == REPOSITORY_KEYS)
3523 xkeyname = di.kv.id;
3527 repodata_add_stubkey(sdata, xkeyname, di.kv.id);
3536 dataiterator_free(&di);
3537 for (i = 0; i < cnt; i++)
3538 repodata_internalize(repo->repodata + stubdataids[i]);
3539 solv_free(stubdataids);
3544 repodata_memused(Repodata *data)
3546 return data->incoredatalen + data->vincorelen;