2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
13 * a repository can contain multiple repodata entries, consisting of
14 * different sets of keys and different sets of solvables
29 #include "poolid_private.h"
37 #define REPODATA_BLOCK 255
39 static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);
42 repodata_initdata(Repodata *data, Repo *repo, int localpool)
44 memset(data, 0, sizeof (*data));
45 data->repodataid = data - repo->repodata;
47 data->localpool = localpool;
49 stringpool_init_empty(&data->spool);
50 /* dirpool_init(&data->dirpool); just zeros out again */
51 data->keys = solv_calloc(1, sizeof(Repokey));
53 data->schemata = solv_calloc(1, sizeof(Id));
54 data->schemadata = solv_calloc(1, sizeof(Id));
56 data->schemadatalen = 1;
57 repopagestore_init(&data->store);
61 repodata_freedata(Repodata *data)
65 solv_free(data->keys);
67 solv_free(data->schemata);
68 solv_free(data->schemadata);
69 solv_free(data->schematahash);
71 stringpool_free(&data->spool);
72 dirpool_free(&data->dirpool);
74 solv_free(data->mainschemaoffsets);
75 solv_free(data->incoredata);
76 solv_free(data->incoreoffset);
77 solv_free(data->verticaloffset);
79 repopagestore_free(&data->store);
81 solv_free(data->vincore);
84 for (i = 0; i < data->end - data->start; i++)
85 solv_free(data->attrs[i]);
86 solv_free(data->attrs);
88 for (i = 0; i < data->nxattrs; i++)
89 solv_free(data->xattrs[i]);
90 solv_free(data->xattrs);
92 solv_free(data->attrdata);
93 solv_free(data->attriddata);
94 solv_free(data->attrnum64data);
96 solv_free(data->dircache);
100 repodata_free(Repodata *data)
102 Repo *repo = data->repo;
103 int i = data - repo->repodata;
106 repodata_freedata(data);
107 if (i < repo->nrepodata - 1)
109 /* whoa! this changes the repodataids! */
110 memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
111 for (; i < repo->nrepodata - 1; i++)
112 repo->repodata[i].repodataid = i;
115 if (repo->nrepodata == 1)
117 repo->repodata = solv_free(repo->repodata);
123 repodata_empty(Repodata *data, int localpool)
125 void (*loadcallback)(Repodata *) = data->loadcallback;
126 int state = data->state;
127 repodata_freedata(data);
128 repodata_initdata(data, data->repo, localpool);
130 data->loadcallback = loadcallback;
134 /***************************************************************
135 * key pool management
138 /* this is not so time critical that we need a hash, so we do a simple
141 repodata_key2id(Repodata *data, Repokey *key, int create)
145 for (keyid = 1; keyid < data->nkeys; keyid++)
146 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
148 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
152 if (keyid == data->nkeys)
156 /* allocate new key */
157 data->keys = solv_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
158 data->keys[data->nkeys++] = *key;
159 if (data->verticaloffset)
161 data->verticaloffset = solv_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
162 data->verticaloffset[data->nkeys - 1] = 0;
164 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
170 /***************************************************************
171 * schema pool management
174 #define SCHEMATA_BLOCK 31
175 #define SCHEMATADATA_BLOCK 255
178 repodata_schema2id(Repodata *data, Id *schema, int create)
185 return 0; /* XXX: allow empty schema? */
186 if ((schematahash = data->schematahash) == 0)
188 data->schematahash = schematahash = solv_calloc(256, sizeof(Id));
189 for (i = 1; i < data->nschemata; i++)
191 for (sp = data->schemadata + data->schemata[i], h = 0; *sp;)
196 data->schemadata = solv_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
197 data->schemata = solv_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
200 for (sp = schema, len = 0, h = 0; *sp; len++)
205 cid = schematahash[h];
208 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
210 /* cache conflict, do a slow search */
211 for (cid = 1; cid < data->nschemata; cid++)
212 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
218 data->schemadata = solv_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
219 data->schemata = solv_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
221 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
222 data->schemata[data->nschemata] = data->schemadatalen;
223 data->schemadatalen += len;
224 schematahash[h] = data->nschemata;
226 fprintf(stderr, "schema2id: new schema\n");
228 return data->nschemata++;
232 repodata_free_schemahash(Repodata *data)
234 data->schematahash = solv_free(data->schematahash);
236 data->schemata = solv_realloc2(data->schemata, data->nschemata, sizeof(Id));
237 data->schemadata = solv_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
241 /***************************************************************
242 * dir pool management
245 #ifndef HAVE_STRCHRNUL
246 static inline const char *strchrnul(const char *str, char x)
248 const char *p = strchr(str, x);
249 return p ? p : str + strlen(str);
253 #define DIRCACHE_SIZE 41 /* < 1k */
257 Id ids[DIRCACHE_SIZE];
258 char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
263 repodata_str2dir(Repodata *data, const char *dir, int create)
274 while (*dir == '/' && dir[1] == '/')
276 if (*dir == '/' && !dir[1])
278 if (data->dirpool.ndirs)
280 return dirpool_add_dir(&data->dirpool, 0, 1, create);
287 struct dircache *dircache = data->dircache;
291 if (l < DIRCACHE_SIZE && dircache->ids[l] && !memcmp(dircache->str + l * (l - 1) / 2, dir, l))
293 parent = dircache->ids[l];
309 dire = strchrnul(dir, '/');
311 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
313 id = pool_strn2id(data->repo->pool, dir, dire - dir, create);
316 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
321 data->dircache = solv_calloc(1, sizeof(struct dircache));
325 if (l < DIRCACHE_SIZE)
327 data->dircache->ids[l] = parent;
328 memcpy(data->dircache->str + l * (l - 1) / 2, dirs, l);
342 repodata_free_dircache(Repodata *data)
344 data->dircache = solv_free(data->dircache);
348 repodata_dir2str(Repodata *data, Id did, const char *suf)
350 Pool *pool = data->repo->pool;
357 return suf ? suf : "";
361 comp = dirpool_compid(&data->dirpool, parent);
362 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
364 parent = dirpool_parent(&data->dirpool, parent);
369 l += strlen(suf) + 1;
370 p = pool_alloctmpspace(pool, l + 1) + l;
381 comp = dirpool_compid(&data->dirpool, parent);
382 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
385 strncpy(p, comps, l);
386 parent = dirpool_parent(&data->dirpool, parent);
394 /***************************************************************
398 static inline unsigned char *
399 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
401 Id *keyp = data->schemadata + data->schemata[schema];
402 for (; *keyp; keyp++)
403 dp = data_skip_key(data, dp, data->keys + *keyp);
407 static unsigned char *
408 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
410 int nentries, schema;
413 case REPOKEY_TYPE_FIXARRAY:
414 dp = data_read_id(dp, &nentries);
417 dp = data_read_id(dp, &schema);
419 dp = data_skip_schema(data, dp, schema);
421 case REPOKEY_TYPE_FLEXARRAY:
422 dp = data_read_id(dp, &nentries);
425 dp = data_read_id(dp, &schema);
426 dp = data_skip_schema(data, dp, schema);
430 if (key->storage == KEY_STORAGE_INCORE)
431 dp = data_skip(dp, key->type);
432 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
434 dp = data_skip(dp, REPOKEY_TYPE_ID);
435 dp = data_skip(dp, REPOKEY_TYPE_ID);
441 static unsigned char *
442 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
448 if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
451 for (i = 0; (k = *keyp++) != 0; i++)
453 return data->incoredata + data->mainschemaoffsets[i];
456 while ((k = *keyp++) != 0)
460 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
462 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
463 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
466 if (data->keys[k].storage != KEY_STORAGE_INCORE)
468 dp = data_skip_key(data, dp, data->keys + k);
473 static unsigned char *
474 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
479 if (off >= data->lastverticaloffset)
481 off -= data->lastverticaloffset;
482 if ((unsigned int)off + len > data->vincorelen)
484 return data->vincore + off;
486 if ((unsigned int)off + len > key->size)
488 /* we now have the offset, go into vertical */
489 off += data->verticaloffset[key - data->keys];
490 /* fprintf(stderr, "key %d page %d\n", key->name, off / REPOPAGE_BLOBSIZE); */
491 dp = repopagestore_load_page_range(&data->store, off / REPOPAGE_BLOBSIZE, (off + len - 1) / REPOPAGE_BLOBSIZE);
494 dp += off % REPOPAGE_BLOBSIZE;
498 static inline unsigned char *
499 get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
501 unsigned char *dp = *dpp;
505 if (key->storage == KEY_STORAGE_INCORE)
508 *dpp = data_skip_key(data, dp, key);
511 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
514 dp = data_read_id(dp, &off);
515 dp = data_read_id(dp, &len);
518 return get_vertical_data(data, key, off, len);
524 load_repodata(Repodata *data)
526 if (data->loadcallback)
528 data->loadcallback(data);
529 if (data->state == REPODATA_AVAILABLE)
532 data->state = REPODATA_ERROR;
537 maybe_load_repodata(Repodata *data, Id keyname)
539 if (keyname && !repodata_precheck_keyname(data, keyname))
540 return 0; /* do not bother... */
547 for (i = 1; i < data->nkeys; i++)
548 if (keyname == data->keys[i].name)
550 if (i == data->nkeys)
553 return load_repodata(data);
556 case REPODATA_AVAILABLE:
557 case REPODATA_LOADING:
560 data->state = REPODATA_ERROR;
565 static inline unsigned char *
566 solvid2data(Repodata *data, Id solvid, Id *schemap)
568 unsigned char *dp = data->incoredata;
571 if (solvid == SOLVID_META)
572 dp += 1; /* offset of "meta" solvable */
573 else if (solvid == SOLVID_POS)
575 Pool *pool = data->repo->pool;
576 if (data->repo != pool->pos.repo)
578 if (data != data->repo->repodata + pool->pos.repodataid)
581 if (pool->pos.dp != 1)
583 *schemap = pool->pos.schema;
589 if (solvid < data->start || solvid >= data->end)
591 dp += data->incoreoffset[solvid - data->start];
593 return data_read_id(dp, schemap);
596 /************************************************************************
600 static unsigned char *
601 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
604 Id schema, *keyp, *kp;
607 if (!maybe_load_repodata(data, keyname))
609 dp = solvid2data(data, solvid, &schema);
612 keyp = data->schemadata + data->schemata[schema];
613 for (kp = keyp; *kp; kp++)
614 if (data->keys[*kp].name == keyname)
618 *keypp = key = data->keys + *kp;
619 if (key->type == REPOKEY_TYPE_DELETED)
621 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
622 return dp; /* no need to forward... */
623 if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
624 return 0; /* get_data will not work, no need to forward */
625 dp = forward_to_key(data, *kp, keyp, dp);
628 return get_data(data, key, &dp, 0);
632 repodata_lookup_type(Repodata *data, Id solvid, Id keyname)
634 Id schema, *keyp, *kp;
635 if (!maybe_load_repodata(data, keyname))
637 if (!solvid2data(data, solvid, &schema))
639 keyp = data->schemadata + data->schemata[schema];
640 for (kp = keyp; *kp; kp++)
641 if (data->keys[*kp].name == keyname)
642 return data->keys[*kp].type;
647 repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
653 dp = find_key_data(data, solvid, keyname, &key);
656 if (key->type == REPOKEY_TYPE_CONSTANTID)
658 if (key->type != REPOKEY_TYPE_ID)
660 dp = data_read_id(dp, &id);
665 repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
671 dp = find_key_data(data, solvid, keyname, &key);
674 if (key->type == REPOKEY_TYPE_STR)
675 return (const char *)dp;
676 if (key->type == REPOKEY_TYPE_CONSTANTID)
678 else if (key->type == REPOKEY_TYPE_ID)
679 dp = data_read_id(dp, &id);
683 return stringpool_id2str(&data->spool, id);
684 return pool_id2str(data->repo->pool, id);
688 repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long *value)
692 unsigned int high, low;
695 dp = find_key_data(data, solvid, keyname, &key);
700 case REPOKEY_TYPE_NUM:
701 data_read_num64(dp, &low, &high);
702 *value = (unsigned long long)high << 32 | low;
704 case REPOKEY_TYPE_U32:
705 data_read_u32(dp, &low);
708 case REPOKEY_TYPE_CONSTANT:
717 repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
723 if (!maybe_load_repodata(data, keyname))
725 dp = solvid2data(data, solvid, &schema);
728 /* can't use find_key_data as we need to test the type */
729 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
730 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
735 const unsigned char *
736 repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
741 dp = find_key_data(data, solvid, keyname, &key);
756 repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
764 dp = find_key_data(data, solvid, keyname, &key);
767 if (key->type != REPOKEY_TYPE_IDARRAY)
771 dp = data_read_ideof(dp, &id, &eof);
780 repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp)
786 dp = find_key_data(data, solvid, keyname, &key);
787 if (!dp || key->type != REPOKEY_TYPE_BINARY)
792 dp = data_read_id(dp, &len);
798 repodata_globalize_id(Repodata *data, Id id, int create)
800 if (!id || !data || !data->localpool)
802 return pool_str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
806 repodata_localize_id(Repodata *data, Id id, int create)
808 if (!id || !data || !data->localpool)
810 return stringpool_str2id(&data->spool, pool_id2str(data->repo->pool, id), create);
814 repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create)
816 if (!id || !data || !fromdata)
818 if (!data->localpool || !fromdata->localpool)
820 if (fromdata->localpool)
821 id = repodata_globalize_id(fromdata, id, create);
823 id = repodata_localize_id(data, id, create);
826 /* localpool is set in both data and fromdata */
827 return stringpool_str2id(&data->spool, stringpool_id2str(&fromdata->spool, id), create);
831 repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id voidid)
836 ap = data->attrs[solvid - data->start];
841 if (data->keys[*ap].name != keyname)
843 if (data->keys[*ap].type == REPOKEY_TYPE_VOID)
845 if (data->keys[*ap].type == REPOKEY_TYPE_ID)
853 /************************************************************************
859 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
863 case REPOKEY_TYPE_ID:
864 case REPOKEY_TYPE_CONSTANTID:
865 case REPOKEY_TYPE_IDARRAY:
866 if (data && data->localpool)
867 kv->str = stringpool_id2str(&data->spool, kv->id);
869 kv->str = pool_id2str(pool, kv->id);
870 if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
873 for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
875 if (*s == ':' && s > kv->str)
879 case REPOKEY_TYPE_STR:
881 case REPOKEY_TYPE_DIRSTRARRAY:
882 if (!(flags & SEARCH_FILES))
883 return kv->str; /* match just the basename */
885 return kv->str; /* already stringified */
886 /* Put the full filename into kv->str. */
887 kv->str = repodata_dir2str(data, kv->id, kv->str);
888 kv->num = 1; /* mark stringification */
891 if (!(flags & SEARCH_CHECKSUMS))
892 return 0; /* skip em */
894 return kv->str; /* already stringified */
895 kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
896 kv->num = 1; /* mark stringification */
904 struct subschema_data {
910 /* search a specific repodata */
912 repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
916 Id keyid, *kp, *keyp;
917 unsigned char *dp, *ddp;
923 if (!maybe_load_repodata(data, keyname))
925 if (solvid == SOLVID_SUBSCHEMA)
927 struct subschema_data *subd = cbdata;
928 cbdata = subd->cbdata;
930 schema = subd->parent->id;
931 dp = (unsigned char *)subd->parent->str;
932 kv.parent = subd->parent;
937 dp = solvid2data(data, solvid, &schema);
940 s = data->repo->pool->solvables + solvid;
943 keyp = data->schemadata + data->schemata[schema];
946 /* search for a specific key */
947 for (kp = keyp; *kp; kp++)
948 if (data->keys[*kp].name == keyname)
952 dp = forward_to_key(data, *kp, keyp, dp);
958 while ((keyid = *keyp++) != 0)
961 key = data->keys + keyid;
962 ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
964 if (key->type == REPOKEY_TYPE_DELETED)
966 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
968 struct subschema_data subd;
972 subd.cbdata = cbdata;
975 ddp = data_read_id(ddp, &nentries);
979 while (ddp && nentries > 0)
983 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
984 ddp = data_read_id(ddp, &schema);
986 kv.str = (char *)ddp;
987 stop = callback(cbdata, s, data, key, &kv);
988 if (stop > SEARCH_NEXT_KEY)
990 if (stop && stop != SEARCH_ENTERSUB)
992 if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
993 repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
994 ddp = data_skip_schema(data, ddp, schema);
997 if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
1001 kv.str = (char *)ddp;
1002 stop = callback(cbdata, s, data, key, &kv);
1003 if (stop > SEARCH_NEXT_KEY)
1013 ddp = data_fetch(ddp, &kv, key);
1016 stop = callback(cbdata, s, data, key, &kv);
1019 while (!kv.eof && !stop);
1020 if (onekey || stop > SEARCH_NEXT_KEY)
1026 repodata_setpos_kv(Repodata *data, KeyValue *kv)
1028 Pool *pool = data->repo->pool;
1030 pool_clear_pos(pool);
1033 pool->pos.repo = data->repo;
1034 pool->pos.repodataid = data - data->repo->repodata;
1035 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
1036 pool->pos.schema = kv->id;
1040 /************************************************************************
1041 * data iterator functions
1045 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
1059 case SOLVABLE_VENDOR:
1062 case SOLVABLE_PROVIDES:
1064 return s->provides ? s->repo->idarraydata + s->provides : 0;
1065 case SOLVABLE_OBSOLETES:
1067 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
1068 case SOLVABLE_CONFLICTS:
1070 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
1071 case SOLVABLE_REQUIRES:
1073 return s->requires ? s->repo->idarraydata + s->requires : 0;
1074 case SOLVABLE_RECOMMENDS:
1076 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
1077 case SOLVABLE_SUPPLEMENTS:
1079 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
1080 case SOLVABLE_SUGGESTS:
1082 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
1083 case SOLVABLE_ENHANCES:
1085 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
1088 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
1095 datamatcher_init(Datamatcher *ma, const char *match, int flags)
1097 match = match ? solv_strdup(match) : 0;
1102 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
1104 ma->matchdata = solv_calloc(1, sizeof(regex_t));
1105 ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
1108 solv_free(ma->matchdata);
1109 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
1112 if ((flags & SEARCH_FILES) != 0 && match)
1114 /* prepare basename check */
1115 if ((flags & SEARCH_STRINGMASK) == SEARCH_STRING || (flags & SEARCH_STRINGMASK) == SEARCH_STRINGEND)
1117 const char *p = strrchr(match, '/');
1118 ma->matchdata = (void *)(p ? p + 1 : match);
1120 else if ((flags & SEARCH_STRINGMASK) == SEARCH_GLOB)
1123 for (p = match + strlen(match) - 1; p >= match; p--)
1124 if (*p == '[' || *p == ']' || *p == '*' || *p == '?' || *p == '/')
1126 ma->matchdata = (void *)(p + 1);
1133 datamatcher_free(Datamatcher *ma)
1136 ma->match = solv_free((char *)ma->match);
1137 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
1139 regfree(ma->matchdata);
1140 solv_free(ma->matchdata);
1146 datamatcher_match(Datamatcher *ma, const char *str)
1149 switch ((ma->flags & SEARCH_STRINGMASK))
1151 case SEARCH_SUBSTRING:
1152 if (ma->flags & SEARCH_NOCASE)
1153 return strcasestr(str, ma->match) != 0;
1155 return strstr(str, ma->match) != 0;
1157 if (ma->flags & SEARCH_NOCASE)
1158 return !strcasecmp(ma->match, str);
1160 return !strcmp(ma->match, str);
1161 case SEARCH_STRINGSTART:
1162 if (ma->flags & SEARCH_NOCASE)
1163 return !strncasecmp(ma->match, str, strlen(ma->match));
1165 return !strncmp(ma->match, str, strlen(ma->match));
1166 case SEARCH_STRINGEND:
1167 l = strlen(str) - strlen(ma->match);
1170 if (ma->flags & SEARCH_NOCASE)
1171 return !strcasecmp(ma->match, str + l);
1173 return !strcmp(ma->match, str + l);
1175 return !fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0);
1177 return !regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0);
1183 /* check if the matcher can match the provides basename */
1186 datamatcher_checkbasename(Datamatcher *ma, const char *basename)
1189 const char *match = ma->matchdata;
1192 switch (ma->flags & SEARCH_STRINGMASK)
1196 case SEARCH_STRINGEND:
1197 if (match != ma->match)
1198 break; /* had slash, do exact match on basename */
1201 /* check if the basename ends with match */
1202 l = strlen(basename) - strlen(match);
1208 return 1; /* maybe matches */
1210 if ((ma->flags & SEARCH_NOCASE) != 0)
1211 return !strcasecmp(match, basename);
1213 return !strcmp(match, basename);
1217 repodata_filelistfilter_matches(Repodata *data, const char *str)
1219 /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
1220 /* for now hardcoded */
1221 if (strstr(str, "bin/"))
1223 if (!strncmp(str, "/etc/", 5))
1225 if (!strcmp(str, "/usr/lib/sendmail"))
1247 di_nextarrayelement,
1253 di_entersolvablekey,
1257 /* see dataiterator.h for documentation */
1259 dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
1261 memset(di, 0, sizeof(*di));
1263 di->flags = flags & ~SEARCH_THISSOLVID;
1264 if (!pool || (repo && repo->pool != pool))
1272 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1278 di->keyname = keyname;
1279 di->keynames[0] = keyname;
1280 dataiterator_set_search(di, repo, p);
1285 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
1290 if (di->dupstr == di->kv.str)
1291 di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
1298 memset(&di->matcher, 0, sizeof(di->matcher));
1299 if (from->matcher.match)
1300 datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1305 for (i = 1; i < di->nparents; i++)
1306 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1307 di->kv.parent = &di->parents[di->nparents - 1].kv;
1312 dataiterator_set_match(Dataiterator *di, const char *match, int flags)
1314 di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
1315 datamatcher_free(&di->matcher);
1316 memset(&di->matcher, 0, sizeof(di->matcher));
1320 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1330 dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
1334 di->flags &= ~SEARCH_THISSOLVID;
1338 if (!di->pool->urepos)
1346 di->repo = di->pool->repos[di->repoid];
1348 di->state = di_enterrepo;
1350 dataiterator_jump_to_solvid(di, p);
1354 dataiterator_set_keyname(Dataiterator *di, Id keyname)
1357 di->keyname = keyname;
1358 di->keynames[0] = keyname;
1362 dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1366 if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1368 di->state = di_bye; /* sorry */
1371 for (i = di->nkeynames + 1; i > 0; i--)
1372 di->keynames[i] = di->keynames[i - 1];
1373 di->keynames[0] = di->keyname = keyname;
1378 dataiterator_free(Dataiterator *di)
1380 if (di->matcher.match)
1381 datamatcher_free(&di->matcher);
1383 solv_free(di->dupstr);
1386 static unsigned char *
1387 dataiterator_find_keyname(Dataiterator *di, Id keyname)
1390 Repokey *keys = di->data->keys, *key;
1393 for (keyp = di->keyp; *keyp; keyp++)
1394 if (keys[*keyp].name == keyname)
1399 if (key->type == REPOKEY_TYPE_DELETED)
1401 if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
1402 return 0; /* get_data will not work, no need to forward */
1403 dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1411 is_filelist_extension(Repodata *data)
1414 if (!repodata_precheck_keyname(data, SOLVABLE_FILELIST))
1416 for (j = 1; j < data->nkeys; j++)
1417 if (data->keys[j].name == SOLVABLE_FILELIST)
1419 if (j == data->nkeys)
1421 if (data->state != REPODATA_AVAILABLE)
1423 for (j = 1; j < data->nkeys; j++)
1424 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1430 dataiterator_filelistcheck(Dataiterator *di)
1433 int needcomplete = 0;
1434 Repodata *data = di->data;
1436 if ((di->flags & SEARCH_COMPLETE_FILELIST) != 0)
1437 if (!di->matcher.match
1438 || ((di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING
1439 && (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_GLOB)
1440 || !repodata_filelistfilter_matches(data, di->matcher.match))
1442 if (data->state != REPODATA_AVAILABLE)
1443 return needcomplete ? 1 : 0;
1446 /* we don't need the complete filelist, so ignore all stubs */
1447 if (data->repo->nrepodata == 2)
1449 for (j = 1; j < data->nkeys; j++)
1450 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1456 /* we need the complete filelist. check if we habe a filtered filelist and there's
1457 * a extension with the complete filelist later on */
1458 for (j = 1; j < data->nkeys; j++)
1459 if (data->keys[j].name == SOLVABLE_FILELIST)
1461 if (j == data->nkeys)
1462 return 0; /* does not have filelist */
1463 for (j = 1; j < data->nkeys; j++)
1464 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1466 if (j == data->nkeys)
1467 return 1; /* this is the externsion */
1468 while (data - data->repo->repodata + 1 < data->repo->nrepodata)
1471 if (is_filelist_extension(data))
1479 dataiterator_step(Dataiterator *di)
1483 if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate) {
1484 unsigned int ddpoff = di->ddp - di->vert_ddp;
1485 di->vert_off += ddpoff;
1486 di->vert_len -= ddpoff;
1487 di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len);
1488 di->vert_storestate = di->data->storestate;
1490 di->state = di_nextkey;
1496 case di_enterrepo: di_enterrepo:
1497 if (!di->repo || (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS)))
1499 if (!(di->flags & SEARCH_THISSOLVID))
1501 di->solvid = di->repo->start - 1; /* reset solvid iterator */
1502 goto di_nextsolvable;
1506 case di_entersolvable: di_entersolvable:
1509 di->repodataid = 1; /* reset repodata iterator */
1510 if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1512 extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1];
1514 di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1516 goto di_entersolvablekey;
1521 case di_enterrepodata: di_enterrepodata:
1524 if (di->repodataid >= di->repo->nrepodata)
1525 goto di_nextsolvable;
1526 di->data = di->repo->repodata + di->repodataid;
1528 if (di->repodataid && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1529 goto di_nextrepodata;
1530 if (!maybe_load_repodata(di->data, di->keyname))
1531 goto di_nextrepodata;
1532 di->dp = solvid2data(di->data, di->solvid, &schema);
1534 goto di_nextrepodata;
1535 if (di->solvid == SOLVID_POS)
1536 di->solvid = di->pool->pos.solvid;
1537 /* reset key iterator */
1538 di->keyp = di->data->schemadata + di->data->schemata[schema];
1541 case di_enterschema: di_enterschema:
1543 di->dp = dataiterator_find_keyname(di, di->keyname);
1544 if (!di->dp || !*di->keyp)
1548 goto di_nextrepodata;
1552 case di_enterkey: di_enterkey:
1554 di->key = di->data->keys + *di->keyp;
1557 /* this is get_data() modified to store vert_ data */
1558 if (di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1561 di->dp = data_read_id(di->dp, &off);
1562 di->dp = data_read_id(di->dp, &len);
1563 di->vert_ddp = di->ddp = get_vertical_data(di->data, di->key, off, len);
1566 di->vert_storestate = di->data->storestate;
1568 else if (di->key->storage == KEY_STORAGE_INCORE)
1571 if (di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0))
1572 di->dp = data_skip_key(di->data, di->dp, di->key);
1578 if (di->key->type == REPOKEY_TYPE_DELETED)
1580 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1582 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1588 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1590 di->state = di_nextkey;
1592 di->state = di_nextattr;
1595 case di_nextkey: di_nextkey:
1596 if (!di->keyname && *++di->keyp)
1602 case di_nextrepodata: di_nextrepodata:
1603 if (di->repodataid && ++di->repodataid < di->repo->nrepodata)
1604 goto di_enterrepodata;
1607 case di_nextsolvable: di_nextsolvable:
1608 if (!(di->flags & SEARCH_THISSOLVID))
1611 di->solvid = di->repo->start;
1614 for (; di->solvid < di->repo->end; di->solvid++)
1616 if (di->pool->solvables[di->solvid].repo == di->repo)
1617 goto di_entersolvable;
1622 case di_nextrepo: di_nextrepo:
1627 if (di->repoid < di->pool->nrepos)
1629 di->repo = di->pool->repos[di->repoid];
1635 case di_bye: di_bye:
1639 case di_enterarray: di_enterarray:
1640 if (di->key->name == REPOSITORY_SOLVABLES)
1642 di->ddp = data_read_id(di->ddp, (Id *)&di->kv.num);
1647 case di_nextarrayelement: di_nextarrayelement:
1650 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1651 if (di->kv.entry == di->kv.num)
1653 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1655 if (!(di->flags & SEARCH_ARRAYSENTINEL))
1657 di->kv.str = (char *)di->ddp;
1659 di->state = di_nextkey;
1662 if (di->kv.entry == di->kv.num - 1)
1664 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1665 di->ddp = data_read_id(di->ddp, &di->kv.id);
1666 di->kv.str = (char *)di->ddp;
1667 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1669 if ((di->flags & SEARCH_SUB) != 0)
1670 di->state = di_entersub;
1672 di->state = di_nextarrayelement;
1675 case di_entersub: di_entersub:
1676 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1677 goto di_nextarrayelement; /* sorry, full */
1678 di->parents[di->nparents].kv = di->kv;
1679 di->parents[di->nparents].dp = di->dp;
1680 di->parents[di->nparents].keyp = di->keyp;
1681 di->dp = (unsigned char *)di->kv.str;
1682 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1683 memset(&di->kv, 0, sizeof(di->kv));
1684 di->kv.parent = &di->parents[di->nparents].kv;
1686 di->keyname = di->keynames[di->nparents - di->rootlevel];
1687 goto di_enterschema;
1689 case di_leavesub: di_leavesub:
1690 if (di->nparents - 1 < di->rootlevel)
1693 di->dp = di->parents[di->nparents].dp;
1694 di->kv = di->parents[di->nparents].kv;
1695 di->keyp = di->parents[di->nparents].keyp;
1696 di->key = di->data->keys + *di->keyp;
1697 di->ddp = (unsigned char *)di->kv.str;
1698 di->keyname = di->keynames[di->nparents - di->rootlevel];
1699 goto di_nextarrayelement;
1701 /* special solvable attr handling follows */
1703 case di_nextsolvablekey: di_nextsolvablekey:
1704 if (di->keyname || di->key->name == RPM_RPMDBID)
1705 goto di_enterrepodata;
1709 case di_entersolvablekey: di_entersolvablekey:
1710 di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1711 if (!di->idp || !*di->idp)
1712 goto di_nextsolvablekey;
1716 di->kv.id = *di->idp;
1717 di->kv.num = *di->idp; /* for rpmdbid */
1718 di->kv.num2 = 0; /* for rpmdbid */
1720 di->state = di_nextsolvablekey;
1726 case di_nextsolvableattr:
1727 di->state = di_nextsolvableattr;
1728 di->kv.id = *di->idp++;
1733 di->state = di_nextsolvablekey;
1739 if (di->matcher.match)
1742 /* simple pre-check so that we don't need to stringify */
1743 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0)
1744 if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
1746 if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
1748 if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
1752 if (!datamatcher_match(&di->matcher, str))
1757 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->flags & SEARCH_FILES) != 0)
1758 repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags);
1760 /* found something! */
1766 dataiterator_entersub(Dataiterator *di)
1768 if (di->state == di_nextarrayelement)
1769 di->state = di_entersub;
1773 dataiterator_setpos(Dataiterator *di)
1775 if (di->kv.eof == 2)
1777 pool_clear_pos(di->pool);
1780 di->pool->pos.solvid = di->solvid;
1781 di->pool->pos.repo = di->repo;
1782 di->pool->pos.repodataid = di->data - di->repo->repodata;
1783 di->pool->pos.schema = di->kv.id;
1784 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1788 dataiterator_setpos_parent(Dataiterator *di)
1790 if (!di->kv.parent || di->kv.parent->eof == 2)
1792 pool_clear_pos(di->pool);
1795 di->pool->pos.solvid = di->solvid;
1796 di->pool->pos.repo = di->repo;
1797 di->pool->pos.repodataid = di->data - di->repo->repodata;
1798 di->pool->pos.schema = di->kv.parent->id;
1799 di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1802 /* clones just the position, not the search keys/matcher */
1804 dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
1806 di->state = from->state;
1807 di->flags &= ~SEARCH_THISSOLVID;
1808 di->flags |= (from->flags & SEARCH_THISSOLVID);
1809 di->repo = from->repo;
1810 di->data = from->data;
1812 di->ddp = from->ddp;
1813 di->idp = from->idp;
1814 di->keyp = from->keyp;
1815 di->key = from->key;
1817 di->repodataid = from->repodataid;
1818 di->solvid = from->solvid;
1819 di->repoid = from->repoid;
1820 di->rootlevel = from->rootlevel;
1821 memcpy(di->parents, from->parents, sizeof(from->parents));
1822 di->nparents = from->nparents;
1826 for (i = 1; i < di->nparents; i++)
1827 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1828 di->kv.parent = &di->parents[di->nparents - 1].kv;
1832 if (from->dupstr && from->dupstr == from->kv.str)
1834 di->dupstrn = from->dupstrn;
1835 di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
1840 dataiterator_seek(Dataiterator *di, int whence)
1842 if ((whence & DI_SEEK_STAY) != 0)
1843 di->rootlevel = di->nparents;
1844 switch (whence & ~DI_SEEK_STAY)
1847 if (di->state != di_nextarrayelement)
1849 if ((whence & DI_SEEK_STAY) != 0)
1850 di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1851 di->state = di_entersub;
1853 case DI_SEEK_PARENT:
1860 if (di->rootlevel > di->nparents)
1861 di->rootlevel = di->nparents;
1862 di->dp = di->parents[di->nparents].dp;
1863 di->kv = di->parents[di->nparents].kv;
1864 di->keyp = di->parents[di->nparents].keyp;
1865 di->key = di->data->keys + *di->keyp;
1866 di->ddp = (unsigned char *)di->kv.str;
1867 di->keyname = di->keynames[di->nparents - di->rootlevel];
1868 di->state = di_nextarrayelement;
1870 case DI_SEEK_REWIND:
1876 di->dp = (unsigned char *)di->kv.parent->str;
1877 di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1878 di->state = di_enterschema;
1886 dataiterator_skip_attribute(Dataiterator *di)
1888 if (di->state == di_nextsolvableattr)
1889 di->state = di_nextsolvablekey;
1891 di->state = di_nextkey;
1895 dataiterator_skip_solvable(Dataiterator *di)
1900 di->keyname = di->keynames[0];
1901 di->state = di_nextsolvable;
1905 dataiterator_skip_repo(Dataiterator *di)
1910 di->keyname = di->keynames[0];
1911 di->state = di_nextrepo;
1915 dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1920 di->keyname = di->keynames[0];
1921 if (solvid == SOLVID_POS)
1923 di->repo = di->pool->pos.repo;
1930 if (!di->pool->pos.repodataid && di->pool->pos.solvid == SOLVID_META) {
1931 solvid = SOLVID_META; /* META pos hack */
1933 di->data = di->repo->repodata + di->pool->pos.repodataid;
1937 else if (solvid > 0)
1939 di->repo = di->pool->solvables[solvid].repo;
1944 if (!di->pool->urepos)
1950 di->repo = di->pool->repos[di->repoid];
1952 if (solvid != SOLVID_POS)
1954 di->solvid = solvid;
1956 di->flags |= SEARCH_THISSOLVID;
1957 di->state = di_enterrepo;
1961 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
1967 di->repoid = 0; /* 0 means stay at repo */
1970 di->flags &= ~SEARCH_THISSOLVID;
1971 di->state = di_enterrepo;
1975 dataiterator_match(Dataiterator *di, Datamatcher *ma)
1978 if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
1980 return ma ? datamatcher_match(ma, str) : 1;
1984 dataiterator_strdup(Dataiterator *di)
1988 if (!di->kv.str || di->kv.str == di->dupstr)
1990 switch (di->key->type)
1993 case REPOKEY_TYPE_DIRSTRARRAY:
1994 if (di->kv.num) /* was it stringified into tmp space? */
1995 l = strlen(di->kv.str) + 1;
2000 if (l < 0 && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2002 switch (di->key->type)
2004 case REPOKEY_TYPE_STR:
2005 case REPOKEY_TYPE_DIRSTRARRAY:
2006 l = strlen(di->kv.str) + 1;
2009 l = solv_chksum_len(di->key->type);
2011 case REPOKEY_TYPE_BINARY:
2018 if (!di->dupstrn || di->dupstrn < l)
2020 di->dupstrn = l + 16;
2021 di->dupstr = solv_realloc(di->dupstr, di->dupstrn);
2024 memcpy(di->dupstr, di->kv.str, l);
2025 di->kv.str = di->dupstr;
2029 /************************************************************************
2030 * data modify functions
2033 /* extend repodata so that it includes solvables p */
2035 repodata_extend(Repodata *data, Id p)
2037 if (data->start == data->end)
2038 data->start = data->end = p;
2041 int old = data->end - data->start;
2042 int new = p - data->end + 1;
2045 data->attrs = solv_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
2046 memset(data->attrs + old, 0, new * sizeof(Id *));
2048 data->incoreoffset = solv_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
2049 memset(data->incoreoffset + old, 0, new * sizeof(Id));
2052 if (p < data->start)
2054 int old = data->end - data->start;
2055 int new = data->start - p;
2058 data->attrs = solv_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
2059 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
2060 memset(data->attrs, 0, new * sizeof(Id *));
2062 data->incoreoffset = solv_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
2063 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
2064 memset(data->incoreoffset, 0, new * sizeof(Id));
2069 /* shrink end of repodata */
2071 repodata_shrink(Repodata *data, int end)
2075 if (data->end <= end)
2077 if (data->start >= end)
2081 for (i = 0; i < data->end - data->start; i++)
2082 solv_free(data->attrs[i]);
2083 data->attrs = solv_free(data->attrs);
2085 data->incoreoffset = solv_free(data->incoreoffset);
2086 data->start = data->end = 0;
2091 for (i = end; i < data->end; i++)
2092 solv_free(data->attrs[i - data->start]);
2093 data->attrs = solv_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
2095 if (data->incoreoffset)
2096 data->incoreoffset = solv_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
2100 /* extend repodata so that it includes solvables from start to start + num - 1 */
2102 repodata_extend_block(Repodata *data, Id start, Id num)
2106 if (!data->incoreoffset)
2108 /* this also means that data->attrs is NULL */
2109 data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
2110 data->start = start;
2111 data->end = start + num;
2114 repodata_extend(data, start);
2116 repodata_extend(data, start + num - 1);
2119 /**********************************************************************/
2122 #define REPODATA_ATTRS_BLOCK 31
2123 #define REPODATA_ATTRDATA_BLOCK 1023
2124 #define REPODATA_ATTRIDDATA_BLOCK 63
2125 #define REPODATA_ATTRNUM64DATA_BLOCK 15
2129 repodata_new_handle(Repodata *data)
2133 data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
2134 data->nxattrs = 2; /* -1: SOLVID_META */
2136 data->xattrs = solv_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
2137 data->xattrs[data->nxattrs] = 0;
2138 return -(data->nxattrs++);
2142 repodata_get_attrp(Repodata *data, Id handle)
2146 if (handle == SOLVID_META && !data->xattrs)
2148 data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
2151 return data->xattrs - handle;
2153 if (handle < data->start || handle >= data->end)
2154 repodata_extend(data, handle);
2156 data->attrs = solv_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
2157 return data->attrs + (handle - data->start);
2161 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
2167 app = repodata_get_attrp(data, handle);
2172 /* Determine equality based on the name only, allows us to change
2173 type (when overwrite is set), and makes TYPE_CONSTANT work. */
2174 for (pp = ap; *pp; pp += 2)
2175 if (data->keys[*pp].name == data->keys[keyid].name)
2179 if (overwrite || data->keys[*pp].type == REPOKEY_TYPE_DELETED)
2188 ap = solv_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
2198 repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
2202 keyid = repodata_key2id(data, key, 1);
2203 repodata_insert_keyid(data, solvid, keyid, val, 1);
2207 repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
2211 key.type = REPOKEY_TYPE_ID;
2213 key.storage = KEY_STORAGE_INCORE;
2214 repodata_set(data, solvid, &key, id);
2218 repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned long long num)
2222 key.type = REPOKEY_TYPE_NUM;
2224 key.storage = KEY_STORAGE_INCORE;
2225 if (num >= 0x80000000)
2227 data->attrnum64data = solv_extend(data->attrnum64data, data->attrnum64datalen, 1, sizeof(unsigned long long), REPODATA_ATTRNUM64DATA_BLOCK);
2228 data->attrnum64data[data->attrnum64datalen] = num;
2229 num = 0x80000000 | data->attrnum64datalen++;
2231 repodata_set(data, solvid, &key, (Id)num);
2235 repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
2239 if (data->localpool)
2240 id = stringpool_str2id(&data->spool, str, 1);
2242 id = pool_str2id(data->repo->pool, str, 1);
2244 key.type = REPOKEY_TYPE_ID;
2246 key.storage = KEY_STORAGE_INCORE;
2247 repodata_set(data, solvid, &key, id);
2251 repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
2255 key.type = REPOKEY_TYPE_CONSTANT;
2256 key.size = constant;
2257 key.storage = KEY_STORAGE_INCORE;
2258 repodata_set(data, solvid, &key, 0);
2262 repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
2266 key.type = REPOKEY_TYPE_CONSTANTID;
2268 key.storage = KEY_STORAGE_INCORE;
2269 repodata_set(data, solvid, &key, 0);
2273 repodata_set_void(Repodata *data, Id solvid, Id keyname)
2277 key.type = REPOKEY_TYPE_VOID;
2279 key.storage = KEY_STORAGE_INCORE;
2280 repodata_set(data, solvid, &key, 0);
2284 repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
2289 l = strlen(str) + 1;
2291 key.type = REPOKEY_TYPE_STR;
2293 key.storage = KEY_STORAGE_INCORE;
2294 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2295 memcpy(data->attrdata + data->attrdatalen, str, l);
2296 repodata_set(data, solvid, &key, data->attrdatalen);
2297 data->attrdatalen += l;
2301 repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len)
2309 key.type = REPOKEY_TYPE_BINARY;
2311 key.storage = KEY_STORAGE_INCORE;
2312 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK);
2313 dp = data->attrdata + data->attrdatalen;
2314 if (len >= (1 << 14))
2316 if (len >= (1 << 28))
2317 *dp++ = (len >> 28) | 128;
2318 if (len >= (1 << 21))
2319 *dp++ = (len >> 21) | 128;
2320 *dp++ = (len >> 14) | 128;
2322 if (len >= (1 << 7))
2323 *dp++ = (len >> 7) | 128;
2326 memcpy(dp, buf, len);
2327 repodata_set(data, solvid, &key, data->attrdatalen);
2328 data->attrdatalen = dp + len - data->attrdata;
2331 /* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
2332 * so that the caller can append entrysize new elements plus the termination zero there */
2334 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
2337 Id *ida, *pp, **ppp;
2339 /* check if it is the same as last time, this speeds things up a lot */
2340 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
2342 /* great! just append the new data */
2343 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2344 data->attriddatalen--; /* overwrite terminating 0 */
2345 data->lastdatalen += entrysize;
2349 ppp = repodata_get_attrp(data, handle);
2353 for (; *pp; pp += 2)
2354 if (data->keys[*pp].name == keyname)
2357 if (!pp || !*pp || data->keys[*pp].type != keytype)
2359 /* not found. allocate new key */
2365 key.storage = KEY_STORAGE_INCORE;
2366 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2367 keyid = repodata_key2id(data, &key, 1);
2368 repodata_insert_keyid(data, handle, keyid, data->attriddatalen, 1);
2369 data->lasthandle = handle;
2370 data->lastkey = keyid;
2371 data->lastdatalen = data->attriddatalen + entrysize + 1;
2375 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2376 oldsize += entrysize;
2377 if (ida + 1 == data->attriddata + data->attriddatalen)
2379 /* this was the last entry, just append it */
2380 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2381 data->attriddatalen--; /* overwrite terminating 0 */
2385 /* too bad. move to back. */
2386 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2387 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2388 pp[1] = data->attriddatalen;
2389 data->attriddatalen += oldsize;
2391 data->lasthandle = handle;
2392 data->lastkey = *pp;
2393 data->lastdatalen = data->attriddatalen + entrysize + 1;
2397 repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2398 const unsigned char *str)
2403 if (!(l = solv_chksum_len(type)))
2408 key.storage = KEY_STORAGE_INCORE;
2409 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2410 memcpy(data->attrdata + data->attrdatalen, str, l);
2411 repodata_set(data, solvid, &key, data->attrdatalen);
2412 data->attrdatalen += l;
2416 repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2419 unsigned char buf[64];
2422 if (!(l = solv_chksum_len(type)))
2424 if (l > sizeof(buf) || solv_hex2bin(&str, buf, l) != l)
2426 repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2430 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2434 if (!(l = solv_chksum_len(type)))
2436 return pool_bin2hex(data->repo->pool, buf, l);
2439 /* rpm filenames don't contain the epoch, so strip it */
2440 static inline const char *
2441 evrid2vrstr(Pool *pool, Id evrid)
2443 const char *p, *evr = pool_id2str(pool, evrid);
2446 for (p = evr; *p >= '0' && *p <= '9'; p++)
2448 return p != evr && *p == ':' && p[1] ? p + 1 : evr;
2452 repodata_set_poolstrn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
2455 if (data->localpool)
2456 id = stringpool_strn2id(&data->spool, str, l, 1);
2458 id = pool_strn2id(data->repo->pool, str, l, 1);
2459 repodata_set_id(data, solvid, keyname, id);
2463 repodata_set_strn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
2466 repodata_set_str(data, solvid, keyname, str);
2469 char *s = solv_strdup(str);
2471 repodata_set_str(data, solvid, keyname, s);
2477 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
2479 Pool *pool = data->repo->pool;
2481 const char *str, *fp;
2485 repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2488 if ((dir = strrchr(file, '/')) != 0)
2499 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2504 if (l == 1 && dir[0] == '.')
2506 s = pool->solvables + solvid;
2509 str = pool_id2str(pool, s->arch);
2510 if (!strncmp(dir, str, l) && !str[l])
2511 repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2513 repodata_set_strn(data, solvid, SOLVABLE_MEDIADIR, dir, l);
2516 str = pool_id2str(pool, s->name);
2518 if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
2521 str = evrid2vrstr(pool, s->evr);
2523 if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
2526 str = pool_id2str(pool, s->arch);
2528 if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2530 repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2535 repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2538 /* XXX: medianr is currently not stored */
2540 repodata_set_deltalocation(Repodata *data, Id handle, int medianr, const char *dir, const char *file)
2543 const char *evr, *suf, *s;
2547 if ((dir = strrchr(file, '/')) != 0)
2558 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2563 if (l == 1 && dir[0] == '.')
2566 repodata_set_poolstrn(data, handle, DELTA_LOCATION_DIR, dir, l);
2567 evr = strchr(file, '-');
2570 for (s = evr - 1; s > file; s--)
2577 suf = strrchr(file, '.');
2580 for (s = suf - 1; s > file; s--)
2586 if (!strcmp(suf, ".delta.rpm") || !strcmp(suf, ".patch.rpm"))
2588 /* We accept one more item as suffix. */
2589 for (s = suf - 1; s > file; s--)
2599 if (suf && evr && suf < evr)
2601 repodata_set_poolstrn(data, handle, DELTA_LOCATION_NAME, file, evr ? evr - file : strlen(file));
2603 repodata_set_poolstrn(data, handle, DELTA_LOCATION_EVR, evr + 1, suf ? suf - evr - 1: strlen(evr + 1));
2605 repodata_set_poolstr(data, handle, DELTA_LOCATION_SUFFIX, suf + 1);
2609 repodata_set_sourcepkg(Repodata *data, Id solvid, const char *sourcepkg)
2611 Pool *pool = data->repo->pool;
2612 Solvable *s = pool->solvables + solvid;
2613 const char *p, *sevr, *sarch, *name, *evr;
2615 p = strrchr(sourcepkg, '.');
2616 if (!p || strcmp(p, ".rpm") != 0)
2619 repodata_set_str(data, solvid, SOLVABLE_SOURCENAME, sourcepkg);
2623 while (p > sourcepkg && *p != '.')
2625 if (*p != '.' || p == sourcepkg)
2628 while (p > sourcepkg && *p != '-')
2630 if (*p != '-' || p == sourcepkg)
2633 while (p > sourcepkg && *p != '-')
2635 if (*p != '-' || p == sourcepkg)
2638 pool = s->repo->pool;
2640 name = pool_id2str(pool, s->name);
2641 if (name && !strncmp(sourcepkg, name, sevr - sourcepkg - 1) && name[sevr - sourcepkg - 1] == 0)
2642 repodata_set_void(data, solvid, SOLVABLE_SOURCENAME);
2644 repodata_set_id(data, solvid, SOLVABLE_SOURCENAME, pool_strn2id(pool, sourcepkg, sevr - sourcepkg - 1, 1));
2646 evr = evrid2vrstr(pool, s->evr);
2647 if (evr && !strncmp(sevr, evr, sarch - sevr - 1) && evr[sarch - sevr - 1] == 0)
2648 repodata_set_void(data, solvid, SOLVABLE_SOURCEEVR);
2650 repodata_set_id(data, solvid, SOLVABLE_SOURCEEVR, pool_strn2id(pool, sevr, sarch - sevr - 1, 1));
2652 if (!strcmp(sarch, "src.rpm"))
2653 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_SRC);
2654 else if (!strcmp(sarch, "nosrc.rpm"))
2655 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_NOSRC);
2657 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, pool_strn2id(pool, sarch, strlen(sarch) - 4, 1));
2661 repodata_set_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
2667 key.type = REPOKEY_TYPE_IDARRAY;
2669 key.storage = KEY_STORAGE_INCORE;
2670 repodata_set(data, solvid, &key, data->attriddatalen);
2671 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, q->count + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2672 for (i = 0; i < q->count; i++)
2673 data->attriddata[data->attriddatalen++] = q->elements[i];
2674 data->attriddata[data->attriddatalen++] = 0;
2678 repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2682 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2684 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2685 data->attriddata[data->attriddatalen++] = dir;
2686 data->attriddata[data->attriddatalen++] = num;
2687 data->attriddata[data->attriddatalen++] = num2;
2688 data->attriddata[data->attriddatalen++] = 0;
2692 repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
2698 l = strlen(str) + 1;
2699 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2700 memcpy(data->attrdata + data->attrdatalen, str, l);
2701 stroff = data->attrdatalen;
2702 data->attrdatalen += l;
2705 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2707 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2708 data->attriddata[data->attriddatalen++] = dir;
2709 data->attriddata[data->attriddatalen++] = stroff;
2710 data->attriddata[data->attriddatalen++] = 0;
2714 repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2717 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2719 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2720 data->attriddata[data->attriddatalen++] = id;
2721 data->attriddata[data->attriddatalen++] = 0;
2725 repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2729 if (data->localpool)
2730 id = stringpool_str2id(&data->spool, str, 1);
2732 id = pool_str2id(data->repo->pool, str, 1);
2733 repodata_add_idarray(data, solvid, keyname, id);
2737 repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2739 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2740 data->attriddata[data->attriddatalen++] = ghandle;
2741 data->attriddata[data->attriddatalen++] = 0;
2745 repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2747 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2748 data->attriddata[data->attriddatalen++] = ghandle;
2749 data->attriddata[data->attriddatalen++] = 0;
2753 repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname)
2756 app = repodata_get_attrp(data, solvid);
2760 for (; *ap; ap += 2)
2761 if (data->keys[*ap].name == keyname)
2767 for (; *ap; ap += 2)
2769 if (data->keys[*ap].name == keyname)
2777 /* XXX: does not work correctly, needs fix in iterators! */
2779 repodata_unset(Repodata *data, Id solvid, Id keyname)
2783 key.type = REPOKEY_TYPE_DELETED;
2785 key.storage = KEY_STORAGE_INCORE;
2786 repodata_set(data, solvid, &key, 0);
2789 /* add all (uninternalized) attrs from src to dest */
2791 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2794 if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
2796 for (; *keyp; keyp += 2)
2797 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2800 /* add some (uninternalized) attrs from src to dest */
2802 repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
2805 if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
2807 for (; *keyp; keyp += 2)
2808 if (!keyidmap || MAPTST(keyidmap, keyp[0]))
2809 repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2812 /* swap (uninternalized) attrs from src and dest */
2814 repodata_swap_attrs(Repodata *data, Id dest, Id src)
2817 if (!data->attrs || dest == src)
2819 if (dest < data->start || dest >= data->end)
2820 repodata_extend(data, dest);
2821 if (src < data->start || src >= data->end)
2822 repodata_extend(data, src);
2823 tmpattrs = data->attrs[dest - data->start];
2824 data->attrs[dest - data->start] = data->attrs[src - data->start];
2825 data->attrs[src - data->start] = tmpattrs;
2829 /**********************************************************************/
2831 /* TODO: unify with repo_write and repo_solv! */
2833 #define EXTDATA_BLOCK 1023
2841 data_addid(struct extdata *xd, Id sx)
2843 unsigned int x = (unsigned int)sx;
2846 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2847 dp = xd->buf + xd->len;
2852 *dp++ = (x >> 28) | 128;
2854 *dp++ = (x >> 21) | 128;
2855 *dp++ = (x >> 14) | 128;
2858 *dp++ = (x >> 7) | 128;
2860 xd->len = dp - xd->buf;
2864 data_addid64(struct extdata *xd, unsigned long long x)
2866 if (x >= 0x100000000)
2870 data_addid(xd, (Id)(x >> 35));
2871 xd->buf[xd->len - 1] |= 128;
2873 data_addid(xd, (Id)((unsigned int)x | 0x80000000));
2874 xd->buf[xd->len - 5] = (x >> 28) | 128;
2877 data_addid(xd, (Id)x);
2881 data_addideof(struct extdata *xd, Id sx, int eof)
2883 unsigned int x = (unsigned int)sx;
2886 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2887 dp = xd->buf + xd->len;
2892 *dp++ = (x >> 27) | 128;
2894 *dp++ = (x >> 20) | 128;
2895 *dp++ = (x >> 13) | 128;
2898 *dp++ = (x >> 6) | 128;
2899 *dp++ = eof ? (x & 63) : (x & 63) | 64;
2900 xd->len = dp - xd->buf;
2904 data_addblob(struct extdata *xd, unsigned char *blob, int len)
2906 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
2907 memcpy(xd->buf + xd->len, blob, len);
2911 /*********************************/
2913 /* this is to reduct memory usage when internalizing oversized repos */
2915 compact_attrdata(Repodata *data, int entry, int nentry)
2918 unsigned int attrdatastart = data->attrdatalen;
2919 unsigned int attriddatastart = data->attriddatalen;
2920 if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
2922 for (i = entry; i < nentry; i++)
2924 Id v, *attrs = data->attrs[i];
2927 for (; *attrs; attrs += 2)
2929 switch (data->keys[*attrs].type)
2931 case REPOKEY_TYPE_STR:
2932 case REPOKEY_TYPE_BINARY:
2934 if ((unsigned int)attrs[1] < attrdatastart)
2935 attrdatastart = attrs[1];
2937 case REPOKEY_TYPE_DIRSTRARRAY:
2938 for (v = attrs[1]; data->attriddata[v] ; v += 2)
2939 if (data->attriddata[v + 1] < attrdatastart)
2940 attrdatastart = data->attriddata[v + 1];
2942 case REPOKEY_TYPE_IDARRAY:
2943 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2944 if ((unsigned int)attrs[1] < attriddatastart)
2945 attriddatastart = attrs[1];
2947 case REPOKEY_TYPE_FIXARRAY:
2948 case REPOKEY_TYPE_FLEXARRAY:
2956 printf("compact_attrdata %d %d\n", entry, nentry);
2957 printf("attrdatastart: %d\n", attrdatastart);
2958 printf("attriddatastart: %d\n", attriddatastart);
2960 if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
2962 for (i = entry; i < nentry; i++)
2964 Id v, *attrs = data->attrs[i];
2967 for (; *attrs; attrs += 2)
2969 switch (data->keys[*attrs].type)
2971 case REPOKEY_TYPE_STR:
2972 case REPOKEY_TYPE_BINARY:
2974 attrs[1] -= attrdatastart;
2976 case REPOKEY_TYPE_DIRSTRARRAY:
2977 for (v = attrs[1]; data->attriddata[v] ; v += 2)
2978 data->attriddata[v + 1] -= attrdatastart;
2980 case REPOKEY_TYPE_IDARRAY:
2981 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2982 attrs[1] -= attriddatastart;
2991 data->attrdatalen -= attrdatastart;
2992 memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
2993 data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
2995 if (attriddatastart)
2997 data->attriddatalen -= attriddatastart;
2998 memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
2999 data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
3003 /* internalalize some key into incore/vincore data */
3006 repodata_serialize_key(Repodata *data, struct extdata *newincore,
3007 struct extdata *newvincore,
3009 Repokey *key, Id val)
3013 unsigned int oldvincorelen = 0;
3017 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3020 oldvincorelen = xd->len;
3024 case REPOKEY_TYPE_VOID:
3025 case REPOKEY_TYPE_CONSTANT:
3026 case REPOKEY_TYPE_CONSTANTID:
3028 case REPOKEY_TYPE_STR:
3029 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
3031 case REPOKEY_TYPE_MD5:
3032 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
3034 case REPOKEY_TYPE_SHA1:
3035 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
3037 case REPOKEY_TYPE_SHA224:
3038 data_addblob(xd, data->attrdata + val, SIZEOF_SHA224);
3040 case REPOKEY_TYPE_SHA256:
3041 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
3043 case REPOKEY_TYPE_SHA384:
3044 data_addblob(xd, data->attrdata + val, SIZEOF_SHA384);
3046 case REPOKEY_TYPE_SHA512:
3047 data_addblob(xd, data->attrdata + val, SIZEOF_SHA512);
3049 case REPOKEY_TYPE_NUM:
3050 if (val & 0x80000000)
3052 data_addid64(xd, data->attrnum64data[val ^ 0x80000000]);
3056 case REPOKEY_TYPE_ID:
3057 case REPOKEY_TYPE_DIR:
3058 data_addid(xd, val);
3060 case REPOKEY_TYPE_BINARY:
3063 unsigned char *dp = data_read_id(data->attrdata + val, &len);
3064 dp += (unsigned int)len;
3065 data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
3068 case REPOKEY_TYPE_IDARRAY:
3069 for (ida = data->attriddata + val; *ida; ida++)
3070 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
3072 case REPOKEY_TYPE_DIRNUMNUMARRAY:
3073 for (ida = data->attriddata + val; *ida; ida += 3)
3075 data_addid(xd, ida[0]);
3076 data_addid(xd, ida[1]);
3077 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
3080 case REPOKEY_TYPE_DIRSTRARRAY:
3081 for (ida = data->attriddata + val; *ida; ida += 2)
3083 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
3084 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
3087 case REPOKEY_TYPE_FIXARRAY:
3091 for (ida = data->attriddata + val; *ida; ida++)
3095 kp = data->xattrs[-*ida];
3103 schemaid = repodata_schema2id(data, schema, 1);
3104 else if (schemaid != repodata_schema2id(data, schema, 0))
3106 pool_debug(data->repo->pool, SOLV_FATAL, "fixarray substructs with different schemas\n");
3112 data_addid(xd, num);
3113 data_addid(xd, schemaid);
3114 for (ida = data->attriddata + val; *ida; ida++)
3116 Id *kp = data->xattrs[-*ida];
3120 repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3124 case REPOKEY_TYPE_FLEXARRAY:
3127 for (ida = data->attriddata + val; *ida; ida++)
3129 data_addid(xd, num);
3130 for (ida = data->attriddata + val; *ida; ida++)
3132 Id *kp = data->xattrs[-*ida];
3135 data_addid(xd, 0); /* XXX */
3142 schemaid = repodata_schema2id(data, schema, 1);
3143 data_addid(xd, schemaid);
3144 kp = data->xattrs[-*ida];
3146 repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3151 pool_debug(data->repo->pool, SOLV_FATAL, "don't know how to handle type %d\n", key->type);
3154 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3156 /* put offset/len in incore */
3157 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
3158 oldvincorelen = xd->len - oldvincorelen;
3159 data_addid(newincore, oldvincorelen);
3164 repodata_internalize(Repodata *data)
3166 Repokey *key, solvkey;
3168 Id schemaid, keyid, *schema, *sp, oldschema, *keyp, *seen;
3170 unsigned char *dp, *ndp;
3171 int newschema, oldcount;
3172 struct extdata newincore;
3173 struct extdata newvincore;
3176 if (!data->attrs && !data->xattrs)
3180 printf("repodata_internalize %d\n", data->repodataid);
3181 printf(" attr data: %d K\n", data->attrdatalen / 1024);
3182 printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3184 newvincore.buf = data->vincore;
3185 newvincore.len = data->vincorelen;
3187 /* find the solvables key, create if needed */
3188 memset(&solvkey, 0, sizeof(solvkey));
3189 solvkey.name = REPOSITORY_SOLVABLES;
3190 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
3192 solvkey.storage = KEY_STORAGE_INCORE;
3193 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
3195 schema = solv_malloc2(data->nkeys, sizeof(Id));
3196 seen = solv_malloc2(data->nkeys, sizeof(Id));
3198 /* Merge the data already existing (in data->schemata, ->incoredata and
3199 friends) with the new attributes in data->attrs[]. */
3200 nentry = data->end - data->start;
3201 memset(&newincore, 0, sizeof(newincore));
3202 data_addid(&newincore, 0); /* start data at offset 1 */
3204 data->mainschema = 0;
3205 data->mainschemaoffsets = solv_free(data->mainschemaoffsets);
3207 /* join entry data */
3208 /* we start with the meta data, entry -1 */
3209 for (entry = -1; entry < nentry; entry++)
3211 memset(seen, 0, data->nkeys * sizeof(Id));
3213 dp = data->incoredata;
3216 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
3217 dp = data_read_id(dp, &oldschema);
3220 fprintf(stderr, "oldschema %d\n", oldschema);
3221 fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
3222 fprintf(stderr, "schemadata %p\n", data->schemadata);
3224 /* seen: -1: old data 0: skipped >0: id + 1 */
3228 for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
3232 pool_debug(data->repo->pool, SOLV_FATAL, "Inconsistent old data (key occured twice).\n");
3240 keyp = data->attrs ? data->attrs[entry] : 0;
3243 /* strip solvables key */
3245 for (sp = keyp = schema; *sp; sp++)
3246 if (*sp != solvkeyid)
3251 seen[solvkeyid] = 0;
3252 keyp = data->xattrs ? data->xattrs[1] : 0;
3255 for (; *keyp; keyp += 2)
3262 seen[*keyp] = keyp[1] + 1;
3264 if (entry < 0 && data->end != data->start)
3271 /* Ideally we'd like to sort the new schema here, to ensure
3272 schema equality independend of the ordering. We can't do that
3273 yet. For once see below (old ids need to come before new ids).
3274 An additional difficulty is that we also need to move
3275 the values with the keys. */
3276 schemaid = repodata_schema2id(data, schema, 1);
3278 schemaid = oldschema;
3281 /* Now create data blob. We walk through the (possibly new) schema
3282 and either copy over old data, or insert the new. */
3283 /* XXX Here we rely on the fact that the (new) schema has the form
3284 o1 o2 o3 o4 ... | n1 n2 n3 ...
3285 (oX being the old keyids (possibly overwritten), and nX being
3286 the new keyids). This rules out sorting the keyids in order
3287 to ensure a small schema count. */
3289 data->incoreoffset[entry] = newincore.len;
3290 data_addid(&newincore, schemaid);
3293 data->mainschema = schemaid;
3294 data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
3296 /* we don't use a pointer to the schemadata here as repodata_serialize_key
3297 * may call repodata_schema2id() which might realloc our schemadata */
3298 for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
3301 data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
3302 if (keyid == solvkeyid)
3304 /* add flexarray entry count */
3305 data_addid(&newincore, data->end - data->start);
3308 key = data->keys + keyid;
3310 fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, pool_id2str(data->repo->pool, key->name), pool_id2str(data->repo->pool, key->type));
3315 /* Skip the data associated with this old key. */
3316 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3318 ndp = data_skip(dp, REPOKEY_TYPE_ID);
3319 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
3321 else if (key->storage == KEY_STORAGE_INCORE)
3322 ndp = data_skip_key(data, dp, key);
3325 if (seen[keyid] == -1)
3327 /* If this key was an old one _and_ was not overwritten with
3328 a different value copy over the old value (we skipped it
3331 data_addblob(&newincore, dp, ndp - dp);
3334 else if (seen[keyid])
3336 /* Otherwise we have a new value. Parse it into the internal form. */
3337 repodata_serialize_key(data, &newincore, &newvincore, schema, key, seen[keyid] - 1);
3341 if (entry >= 0 && data->attrs)
3343 if (data->attrs[entry])
3344 data->attrs[entry] = solv_free(data->attrs[entry]);
3345 if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
3347 compact_attrdata(data, entry + 1, nentry); /* try to free some memory */
3349 printf(" attr data: %d K\n", data->attrdatalen / 1024);
3350 printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3351 printf(" incore data: %d K\n", newincore.len / 1024);
3352 printf(" sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
3353 /* malloc_stats(); */
3358 /* free all xattrs */
3359 for (entry = 0; entry < data->nxattrs; entry++)
3360 if (data->xattrs[entry])
3361 solv_free(data->xattrs[entry]);
3362 data->xattrs = solv_free(data->xattrs);
3365 data->lasthandle = 0;
3367 data->lastdatalen = 0;
3370 repodata_free_schemahash(data);
3372 solv_free(data->incoredata);
3373 data->incoredata = newincore.buf;
3374 data->incoredatalen = newincore.len;
3375 data->incoredatafree = 0;
3377 solv_free(data->vincore);
3378 data->vincore = newvincore.buf;
3379 data->vincorelen = newvincore.len;
3381 data->attrs = solv_free(data->attrs);
3382 data->attrdata = solv_free(data->attrdata);
3383 data->attriddata = solv_free(data->attriddata);
3384 data->attrnum64data = solv_free(data->attrnum64data);
3385 data->attrdatalen = 0;
3386 data->attriddatalen = 0;
3387 data->attrnum64datalen = 0;
3389 printf("repodata_internalize %d done\n", data->repodataid);
3390 printf(" incore data: %d K\n", data->incoredatalen / 1024);
3395 repodata_disable_paging(Repodata *data)
3397 if (maybe_load_repodata(data, 0))
3399 repopagestore_disable_paging(&data->store);
3405 repodata_load_stub(Repodata *data)
3407 Repo *repo = data->repo;
3408 Pool *pool = repo->pool;
3410 struct _Pool_tmpspace oldtmpspace;
3413 if (!pool->loadcallback)
3415 data->state = REPODATA_ERROR;
3418 data->state = REPODATA_LOADING;
3420 /* save tmp space and pos */
3421 oldtmpspace = pool->tmpspace;
3422 memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
3425 r = pool->loadcallback(pool, data, pool->loadcallbackdata);
3427 /* restore tmp space and pos */
3428 for (i = 0; i < POOL_TMPSPACEBUF; i++)
3429 solv_free(pool->tmpspace.buf[i]);
3430 pool->tmpspace = oldtmpspace;
3431 if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
3432 memset(&oldpos, 0, sizeof(oldpos));
3435 data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR;
3439 repodata_add_stubkey(Repodata *data, Id keyname, Id keytype)
3443 xkey.name = keyname;
3444 xkey.type = keytype;
3445 xkey.storage = KEY_STORAGE_INCORE;
3447 repodata_key2id(data, &xkey, 1);
3451 repodata_add_stub(Repodata **datap)
3453 Repodata *data = *datap;
3454 Repo *repo = data->repo;
3455 Id repodataid = data - repo->repodata;
3456 Repodata *sdata = repo_add_repodata(repo, 0);
3457 data = repo->repodata + repodataid;
3458 if (data->end > data->start)
3459 repodata_extend_block(sdata, data->start, data->end - data->start);
3460 sdata->state = REPODATA_STUB;
3461 sdata->loadcallback = repodata_load_stub;
3467 repodata_create_stubs(Repodata *data)
3469 Repo *repo = data->repo;
3470 Pool *pool = repo->pool;
3477 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
3478 while (dataiterator_step(&di))
3479 if (di.data == data)
3481 dataiterator_free(&di);
3484 stubdataids = solv_calloc(cnt, sizeof(*stubdataids));
3485 for (i = 0; i < cnt; i++)
3487 sdata = repodata_add_stub(&data);
3488 stubdataids[i] = sdata - repo->repodata;
3491 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
3493 while (dataiterator_step(&di))
3495 if (di.data != data)
3497 if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
3499 dataiterator_entersub(&di);
3500 sdata = repo->repodata + stubdataids[i++];
3504 switch (di.key->type)
3506 case REPOKEY_TYPE_ID:
3507 repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
3509 case REPOKEY_TYPE_CONSTANTID:
3510 repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
3512 case REPOKEY_TYPE_STR:
3513 repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
3515 case REPOKEY_TYPE_VOID:
3516 repodata_set_void(sdata, SOLVID_META, di.key->name);
3518 case REPOKEY_TYPE_NUM:
3519 repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv));
3522 repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
3524 case REPOKEY_TYPE_IDARRAY:
3525 repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
3526 if (di.key->name == REPOSITORY_KEYS)
3531 xkeyname = di.kv.id;
3535 repodata_add_stubkey(sdata, xkeyname, di.kv.id);
3544 dataiterator_free(&di);
3545 for (i = 0; i < cnt; i++)
3546 repodata_internalize(repo->repodata + stubdataids[i]);
3547 solv_free(stubdataids);
3552 repodata_memused(Repodata *data)
3554 return data->incoredatalen + data->vincorelen;