2 * Copyright (c) 2007, Novell Inc.
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
11 * Manage data coming from one repository
13 * a repository can contain multiple repodata entries, consisting of
14 * different sets of keys and different sets of solvables
29 #include "poolid_private.h"
37 #define REPODATA_BLOCK 255
39 static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);
42 repodata_initdata(Repodata *data, Repo *repo, int localpool)
44 memset(data, 0, sizeof (*data));
45 data->repodataid = data - repo->repodata;
47 data->localpool = localpool;
49 stringpool_init_empty(&data->spool);
50 /* dirpool_init(&data->dirpool); just zeros out again */
51 data->keys = solv_calloc(1, sizeof(Repokey));
53 data->schemata = solv_calloc(1, sizeof(Id));
54 data->schemadata = solv_calloc(1, sizeof(Id));
56 data->schemadatalen = 1;
57 repopagestore_init(&data->store);
61 repodata_freedata(Repodata *data)
65 solv_free(data->keys);
67 solv_free(data->schemata);
68 solv_free(data->schemadata);
69 solv_free(data->schematahash);
71 stringpool_free(&data->spool);
72 dirpool_free(&data->dirpool);
74 solv_free(data->mainschemaoffsets);
75 solv_free(data->incoredata);
76 solv_free(data->incoreoffset);
77 solv_free(data->verticaloffset);
79 repopagestore_free(&data->store);
81 solv_free(data->vincore);
84 for (i = 0; i < data->end - data->start; i++)
85 solv_free(data->attrs[i]);
86 solv_free(data->attrs);
88 for (i = 0; i < data->nxattrs; i++)
89 solv_free(data->xattrs[i]);
90 solv_free(data->xattrs);
92 solv_free(data->attrdata);
93 solv_free(data->attriddata);
94 solv_free(data->attrnum64data);
96 solv_free(data->dircache);
100 repodata_free(Repodata *data)
102 Repo *repo = data->repo;
103 int i = data - repo->repodata;
106 repodata_freedata(data);
107 if (i < repo->nrepodata - 1)
109 /* whoa! this changes the repodataids! */
110 memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
111 for (; i < repo->nrepodata - 1; i++)
112 repo->repodata[i].repodataid = i;
115 if (repo->nrepodata == 1)
117 repo->repodata = solv_free(repo->repodata);
123 repodata_empty(Repodata *data, int localpool)
125 void (*loadcallback)(Repodata *) = data->loadcallback;
126 int state = data->state;
127 repodata_freedata(data);
128 repodata_initdata(data, data->repo, localpool);
130 data->loadcallback = loadcallback;
134 /***************************************************************
135 * key pool management
138 /* this is not so time critical that we need a hash, so we do a simple
141 repodata_key2id(Repodata *data, Repokey *key, int create)
145 for (keyid = 1; keyid < data->nkeys; keyid++)
146 if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
148 if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
152 if (keyid == data->nkeys)
156 /* allocate new key */
157 data->keys = solv_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
158 data->keys[data->nkeys++] = *key;
159 if (data->verticaloffset)
161 data->verticaloffset = solv_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
162 data->verticaloffset[data->nkeys - 1] = 0;
164 data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
170 /***************************************************************
171 * schema pool management
174 #define SCHEMATA_BLOCK 31
175 #define SCHEMATADATA_BLOCK 255
178 repodata_schema2id(Repodata *data, Id *schema, int create)
185 return 0; /* XXX: allow empty schema? */
186 if ((schematahash = data->schematahash) == 0)
188 data->schematahash = schematahash = solv_calloc(256, sizeof(Id));
189 for (i = 1; i < data->nschemata; i++)
191 for (sp = data->schemadata + data->schemata[i], h = 0; *sp;)
196 data->schemadata = solv_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
197 data->schemata = solv_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
200 for (sp = schema, len = 0, h = 0; *sp; len++)
205 cid = schematahash[h];
208 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
210 /* cache conflict, do a slow search */
211 for (cid = 1; cid < data->nschemata; cid++)
212 if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
218 data->schemadata = solv_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
219 data->schemata = solv_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
221 memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
222 data->schemata[data->nschemata] = data->schemadatalen;
223 data->schemadatalen += len;
224 schematahash[h] = data->nschemata;
226 fprintf(stderr, "schema2id: new schema\n");
228 return data->nschemata++;
232 repodata_free_schemahash(Repodata *data)
234 data->schematahash = solv_free(data->schematahash);
236 data->schemata = solv_realloc2(data->schemata, data->nschemata, sizeof(Id));
237 data->schemadata = solv_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
241 /***************************************************************
242 * dir pool management
245 #ifndef HAVE_STRCHRNUL
246 static inline const char *strchrnul(const char *str, char x)
248 const char *p = strchr(str, x);
249 return p ? p : str + strlen(str);
253 #define DIRCACHE_SIZE 41 /* < 1k */
257 Id ids[DIRCACHE_SIZE];
258 char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
263 repodata_str2dir(Repodata *data, const char *dir, int create)
272 return data->dirpool.ndirs ? 0 : dirpool_add_dir(&data->dirpool, 0, 0, create);
273 while (*dir == '/' && dir[1] == '/')
275 if (*dir == '/' && !dir[1])
276 return data->dirpool.ndirs ? 1 : dirpool_add_dir(&data->dirpool, 0, 1, create);
283 struct dircache *dircache = data->dircache;
287 if (l < DIRCACHE_SIZE && dircache->ids[l] && !memcmp(dircache->str + l * (l - 1) / 2, dir, l))
289 parent = dircache->ids[l];
305 dire = strchrnul(dir, '/');
307 id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
309 id = pool_strn2id(data->repo->pool, dir, dire - dir, create);
312 parent = dirpool_add_dir(&data->dirpool, parent, id, create);
317 data->dircache = solv_calloc(1, sizeof(struct dircache));
321 if (l < DIRCACHE_SIZE)
323 data->dircache->ids[l] = parent;
324 memcpy(data->dircache->str + l * (l - 1) / 2, dirs, l);
338 repodata_free_dircache(Repodata *data)
340 data->dircache = solv_free(data->dircache);
344 repodata_dir2str(Repodata *data, Id did, const char *suf)
346 Pool *pool = data->repo->pool;
353 return suf ? suf : "";
354 if (did == 1 && !suf)
359 comp = dirpool_compid(&data->dirpool, parent);
360 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
362 parent = dirpool_parent(&data->dirpool, parent);
367 l += strlen(suf) + 1;
368 p = pool_alloctmpspace(pool, l + 1) + l;
379 comp = dirpool_compid(&data->dirpool, parent);
380 comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
383 strncpy(p, comps, l);
384 parent = dirpool_parent(&data->dirpool, parent);
392 /***************************************************************
396 static inline unsigned char *
397 data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
399 Id *keyp = data->schemadata + data->schemata[schema];
400 for (; *keyp; keyp++)
401 dp = data_skip_key(data, dp, data->keys + *keyp);
405 static unsigned char *
406 data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
408 int nentries, schema;
411 case REPOKEY_TYPE_FIXARRAY:
412 dp = data_read_id(dp, &nentries);
415 dp = data_read_id(dp, &schema);
417 dp = data_skip_schema(data, dp, schema);
419 case REPOKEY_TYPE_FLEXARRAY:
420 dp = data_read_id(dp, &nentries);
423 dp = data_read_id(dp, &schema);
424 dp = data_skip_schema(data, dp, schema);
428 if (key->storage == KEY_STORAGE_INCORE)
429 dp = data_skip(dp, key->type);
430 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
432 dp = data_skip(dp, REPOKEY_TYPE_ID);
433 dp = data_skip(dp, REPOKEY_TYPE_ID);
439 static unsigned char *
440 forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
446 if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
449 for (i = 0; (k = *keyp++) != 0; i++)
451 return data->incoredata + data->mainschemaoffsets[i];
454 while ((k = *keyp++) != 0)
458 if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
460 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip offset */
461 dp = data_skip(dp, REPOKEY_TYPE_ID); /* skip length */
464 if (data->keys[k].storage != KEY_STORAGE_INCORE)
466 dp = data_skip_key(data, dp, data->keys + k);
471 static unsigned char *
472 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
477 if (off >= data->lastverticaloffset)
479 off -= data->lastverticaloffset;
480 if ((unsigned int)off + len > data->vincorelen)
482 return data->vincore + off;
484 if ((unsigned int)off + len > key->size)
486 /* we now have the offset, go into vertical */
487 off += data->verticaloffset[key - data->keys];
488 /* fprintf(stderr, "key %d page %d\n", key->name, off / REPOPAGE_BLOBSIZE); */
489 dp = repopagestore_load_page_range(&data->store, off / REPOPAGE_BLOBSIZE, (off + len - 1) / REPOPAGE_BLOBSIZE);
492 dp += off % REPOPAGE_BLOBSIZE;
496 static inline unsigned char *
497 get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
499 unsigned char *dp = *dpp;
503 if (key->storage == KEY_STORAGE_INCORE)
506 *dpp = data_skip_key(data, dp, key);
509 else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
512 dp = data_read_id(dp, &off);
513 dp = data_read_id(dp, &len);
516 return get_vertical_data(data, key, off, len);
522 load_repodata(Repodata *data)
524 if (data->loadcallback)
526 data->loadcallback(data);
527 if (data->state == REPODATA_AVAILABLE)
530 data->state = REPODATA_ERROR;
535 maybe_load_repodata(Repodata *data, Id keyname)
537 if (keyname && !repodata_precheck_keyname(data, keyname))
538 return 0; /* do not bother... */
545 for (i = 1; i < data->nkeys; i++)
546 if (keyname == data->keys[i].name)
548 if (i == data->nkeys)
551 return load_repodata(data);
554 case REPODATA_AVAILABLE:
555 case REPODATA_LOADING:
558 data->state = REPODATA_ERROR;
563 static inline unsigned char *
564 solvid2data(Repodata *data, Id solvid, Id *schemap)
566 unsigned char *dp = data->incoredata;
569 if (solvid == SOLVID_META)
570 dp += 1; /* offset of "meta" solvable */
571 else if (solvid == SOLVID_POS)
573 Pool *pool = data->repo->pool;
574 if (data->repo != pool->pos.repo)
576 if (data != data->repo->repodata + pool->pos.repodataid)
579 if (pool->pos.dp != 1)
581 *schemap = pool->pos.schema;
587 if (solvid < data->start || solvid >= data->end)
589 dp += data->incoreoffset[solvid - data->start];
591 return data_read_id(dp, schemap);
594 /************************************************************************
598 static unsigned char *
599 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
602 Id schema, *keyp, *kp;
605 if (!maybe_load_repodata(data, keyname))
607 dp = solvid2data(data, solvid, &schema);
610 keyp = data->schemadata + data->schemata[schema];
611 for (kp = keyp; *kp; kp++)
612 if (data->keys[*kp].name == keyname)
616 *keypp = key = data->keys + *kp;
617 if (key->type == REPOKEY_TYPE_DELETED)
619 if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
620 return dp; /* no need to forward... */
621 if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
622 return 0; /* get_data will not work, no need to forward */
623 dp = forward_to_key(data, *kp, keyp, dp);
626 return get_data(data, key, &dp, 0);
630 repodata_lookup_type(Repodata *data, Id solvid, Id keyname)
632 Id schema, *keyp, *kp;
633 if (!maybe_load_repodata(data, keyname))
635 if (!solvid2data(data, solvid, &schema))
637 keyp = data->schemadata + data->schemata[schema];
638 for (kp = keyp; *kp; kp++)
639 if (data->keys[*kp].name == keyname)
640 return data->keys[*kp].type;
645 repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
651 dp = find_key_data(data, solvid, keyname, &key);
654 if (key->type == REPOKEY_TYPE_CONSTANTID)
656 if (key->type != REPOKEY_TYPE_ID)
658 dp = data_read_id(dp, &id);
663 repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
669 dp = find_key_data(data, solvid, keyname, &key);
672 if (key->type == REPOKEY_TYPE_STR)
673 return (const char *)dp;
674 if (key->type == REPOKEY_TYPE_CONSTANTID)
676 else if (key->type == REPOKEY_TYPE_ID)
677 dp = data_read_id(dp, &id);
681 return stringpool_id2str(&data->spool, id);
682 return pool_id2str(data->repo->pool, id);
686 repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long *value)
690 unsigned int high, low;
693 dp = find_key_data(data, solvid, keyname, &key);
698 case REPOKEY_TYPE_NUM:
699 data_read_num64(dp, &low, &high);
700 *value = (unsigned long long)high << 32 | low;
702 case REPOKEY_TYPE_U32:
703 data_read_u32(dp, &low);
706 case REPOKEY_TYPE_CONSTANT:
715 repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
721 if (!maybe_load_repodata(data, keyname))
723 dp = solvid2data(data, solvid, &schema);
726 /* can't use find_key_data as we need to test the type */
727 for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
728 if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
733 const unsigned char *
734 repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
739 dp = find_key_data(data, solvid, keyname, &key);
754 repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
762 dp = find_key_data(data, solvid, keyname, &key);
765 if (key->type != REPOKEY_TYPE_IDARRAY)
769 dp = data_read_ideof(dp, &id, &eof);
778 repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp)
784 dp = find_key_data(data, solvid, keyname, &key);
785 if (!dp || key->type != REPOKEY_TYPE_BINARY)
790 dp = data_read_id(dp, &len);
796 repodata_globalize_id(Repodata *data, Id id, int create)
798 if (!id || !data || !data->localpool)
800 return pool_str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
804 repodata_localize_id(Repodata *data, Id id, int create)
806 if (!id || !data || !data->localpool)
808 return stringpool_str2id(&data->spool, pool_id2str(data->repo->pool, id), create);
812 repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create)
814 if (!id || !data || !fromdata)
816 if (!data->localpool || !fromdata->localpool)
818 if (fromdata->localpool)
819 id = repodata_globalize_id(fromdata, id, create);
821 id = repodata_localize_id(data, id, create);
824 /* localpool is set in both data and fromdata */
825 return stringpool_str2id(&data->spool, stringpool_id2str(&fromdata->spool, id), create);
829 repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id voidid)
834 ap = data->attrs[solvid - data->start];
839 if (data->keys[*ap].name != keyname)
841 if (data->keys[*ap].type == REPOKEY_TYPE_VOID)
843 if (data->keys[*ap].type == REPOKEY_TYPE_ID)
851 repodata_lookup_dirstrarray_uninternalized(Repodata *data, Id solvid, Id keyname, Id *didp, Id *iterp)
855 if (iter == 0) /* find key data */
859 ap = data->attrs[solvid - data->start];
863 if (data->keys[*ap].name == keyname && data->keys[*ap].type == REPOKEY_TYPE_DIRSTRARRAY)
870 for (ap = data->attriddata + iter; *ap; ap += 2)
872 if (did && ap[0] != did)
875 *iterp = ap - data->attriddata + 2;
876 return (const char *)data->attrdata + ap[1];
882 const unsigned char *
883 repodata_lookup_bin_checksum_uninternalized(Repodata *data, Id solvid, Id keyname, Id *typep)
888 ap = data->attrs[solvid - data->start];
893 if (data->keys[*ap].name != keyname)
895 switch (data->keys[*ap].type)
898 *typep = data->keys[*ap].type;
899 return (const unsigned char *)data->attrdata + ap[1];
907 /************************************************************************
913 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
917 case REPOKEY_TYPE_ID:
918 case REPOKEY_TYPE_CONSTANTID:
919 case REPOKEY_TYPE_IDARRAY:
920 if (data && data->localpool)
921 kv->str = stringpool_id2str(&data->spool, kv->id);
923 kv->str = pool_id2str(pool, kv->id);
924 if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
927 for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
929 if (*s == ':' && s > kv->str)
933 case REPOKEY_TYPE_STR:
935 case REPOKEY_TYPE_DIRSTRARRAY:
936 if (!(flags & SEARCH_FILES))
937 return kv->str; /* match just the basename */
939 return kv->str; /* already stringified */
940 /* Put the full filename into kv->str. */
941 kv->str = repodata_dir2str(data, kv->id, kv->str);
942 kv->num = 1; /* mark stringification */
945 if (!(flags & SEARCH_CHECKSUMS))
946 return 0; /* skip em */
948 return kv->str; /* already stringified */
949 kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
950 kv->num = 1; /* mark stringification */
958 struct subschema_data {
964 /* search a specific repodata */
966 repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
970 Id keyid, *kp, *keyp;
971 unsigned char *dp, *ddp;
977 if (!maybe_load_repodata(data, keyname))
979 if (solvid == SOLVID_SUBSCHEMA)
981 struct subschema_data *subd = cbdata;
982 cbdata = subd->cbdata;
984 schema = subd->parent->id;
985 dp = (unsigned char *)subd->parent->str;
986 kv.parent = subd->parent;
991 dp = solvid2data(data, solvid, &schema);
994 s = data->repo->pool->solvables + solvid;
997 keyp = data->schemadata + data->schemata[schema];
1000 /* search for a specific key */
1001 for (kp = keyp; *kp; kp++)
1002 if (data->keys[*kp].name == keyname)
1006 dp = forward_to_key(data, *kp, keyp, dp);
1012 while ((keyid = *keyp++) != 0)
1015 key = data->keys + keyid;
1016 ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
1018 if (key->type == REPOKEY_TYPE_DELETED)
1024 if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
1026 struct subschema_data subd;
1030 subd.cbdata = cbdata;
1033 ddp = data_read_id(ddp, &nentries);
1037 while (ddp && nentries > 0)
1041 if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
1042 ddp = data_read_id(ddp, &schema);
1044 kv.str = (char *)ddp;
1045 stop = callback(cbdata, s, data, key, &kv);
1046 if (stop > SEARCH_NEXT_KEY)
1048 if (stop && stop != SEARCH_ENTERSUB)
1050 if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
1051 repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
1052 ddp = data_skip_schema(data, ddp, schema);
1055 if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
1059 kv.str = (char *)ddp;
1060 stop = callback(cbdata, s, data, key, &kv);
1061 if (stop > SEARCH_NEXT_KEY)
1071 ddp = data_fetch(ddp, &kv, key);
1074 stop = callback(cbdata, s, data, key, &kv);
1077 while (!kv.eof && !stop);
1078 if (onekey || stop > SEARCH_NEXT_KEY)
1084 repodata_setpos_kv(Repodata *data, KeyValue *kv)
1086 Pool *pool = data->repo->pool;
1088 pool_clear_pos(pool);
1091 pool->pos.repo = data->repo;
1092 pool->pos.repodataid = data - data->repo->repodata;
1093 pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
1094 pool->pos.schema = kv->id;
1098 /************************************************************************
1099 * data iterator functions
1103 solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
1117 case SOLVABLE_VENDOR:
1120 case SOLVABLE_PROVIDES:
1122 return s->provides ? s->repo->idarraydata + s->provides : 0;
1123 case SOLVABLE_OBSOLETES:
1125 return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
1126 case SOLVABLE_CONFLICTS:
1128 return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
1129 case SOLVABLE_REQUIRES:
1131 return s->requires ? s->repo->idarraydata + s->requires : 0;
1132 case SOLVABLE_RECOMMENDS:
1134 return s->recommends ? s->repo->idarraydata + s->recommends : 0;
1135 case SOLVABLE_SUPPLEMENTS:
1137 return s->supplements ? s->repo->idarraydata + s->supplements : 0;
1138 case SOLVABLE_SUGGESTS:
1140 return s->suggests ? s->repo->idarraydata + s->suggests : 0;
1141 case SOLVABLE_ENHANCES:
1143 return s->enhances ? s->repo->idarraydata + s->enhances : 0;
1146 return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
1153 datamatcher_init(Datamatcher *ma, const char *match, int flags)
1155 match = match ? solv_strdup(match) : 0;
1160 if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
1162 ma->matchdata = solv_calloc(1, sizeof(regex_t));
1163 ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
1166 solv_free(ma->matchdata);
1167 ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
1170 if ((flags & SEARCH_FILES) != 0 && match)
1172 /* prepare basename check */
1173 if ((flags & SEARCH_STRINGMASK) == SEARCH_STRING || (flags & SEARCH_STRINGMASK) == SEARCH_STRINGEND)
1175 const char *p = strrchr(match, '/');
1176 ma->matchdata = (void *)(p ? p + 1 : match);
1178 else if ((flags & SEARCH_STRINGMASK) == SEARCH_GLOB)
1181 for (p = match + strlen(match) - 1; p >= match; p--)
1182 if (*p == '[' || *p == ']' || *p == '*' || *p == '?' || *p == '/')
1184 ma->matchdata = (void *)(p + 1);
1191 datamatcher_free(Datamatcher *ma)
1194 ma->match = solv_free((char *)ma->match);
1195 if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
1197 regfree(ma->matchdata);
1198 solv_free(ma->matchdata);
1204 datamatcher_match(Datamatcher *ma, const char *str)
1207 switch ((ma->flags & SEARCH_STRINGMASK))
1209 case SEARCH_SUBSTRING:
1210 if (ma->flags & SEARCH_NOCASE)
1211 return strcasestr(str, ma->match) != 0;
1213 return strstr(str, ma->match) != 0;
1215 if (ma->flags & SEARCH_NOCASE)
1216 return !strcasecmp(ma->match, str);
1218 return !strcmp(ma->match, str);
1219 case SEARCH_STRINGSTART:
1220 if (ma->flags & SEARCH_NOCASE)
1221 return !strncasecmp(ma->match, str, strlen(ma->match));
1223 return !strncmp(ma->match, str, strlen(ma->match));
1224 case SEARCH_STRINGEND:
1225 l = strlen(str) - strlen(ma->match);
1228 if (ma->flags & SEARCH_NOCASE)
1229 return !strcasecmp(ma->match, str + l);
1231 return !strcmp(ma->match, str + l);
1233 return !fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0);
1235 return !regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0);
1241 /* check if the matcher can match the provides basename */
1244 datamatcher_checkbasename(Datamatcher *ma, const char *basename)
1247 const char *match = ma->matchdata;
1250 switch (ma->flags & SEARCH_STRINGMASK)
1254 case SEARCH_STRINGEND:
1255 if (match != ma->match)
1256 break; /* had slash, do exact match on basename */
1259 /* check if the basename ends with match */
1260 l = strlen(basename) - strlen(match);
1266 return 1; /* maybe matches */
1268 if ((ma->flags & SEARCH_NOCASE) != 0)
1269 return !strcasecmp(match, basename);
1271 return !strcmp(match, basename);
1275 repodata_filelistfilter_matches(Repodata *data, const char *str)
1277 /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
1278 /* for now hardcoded */
1279 if (strstr(str, "bin/"))
1281 if (!strncmp(str, "/etc/", 5))
1283 if (!strcmp(str, "/usr/lib/sendmail"))
1305 di_nextarrayelement,
1311 di_entersolvablekey,
1315 /* see dataiterator.h for documentation */
1317 dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
1319 memset(di, 0, sizeof(*di));
1321 di->flags = flags & ~SEARCH_THISSOLVID;
1322 if (!pool || (repo && repo->pool != pool))
1330 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1336 di->keyname = keyname;
1337 di->keynames[0] = keyname;
1338 dataiterator_set_search(di, repo, p);
1343 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
1348 if (di->dupstr == di->kv.str)
1349 di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
1356 memset(&di->matcher, 0, sizeof(di->matcher));
1357 if (from->matcher.match)
1358 datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
1363 for (i = 1; i < di->nparents; i++)
1364 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1365 di->kv.parent = &di->parents[di->nparents - 1].kv;
1370 dataiterator_set_match(Dataiterator *di, const char *match, int flags)
1372 di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
1373 datamatcher_free(&di->matcher);
1374 memset(&di->matcher, 0, sizeof(di->matcher));
1378 if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
1388 dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
1392 di->flags &= ~SEARCH_THISSOLVID;
1396 if (!di->pool->urepos)
1404 di->repo = di->pool->repos[di->repoid];
1406 di->state = di_enterrepo;
1408 dataiterator_jump_to_solvid(di, p);
1412 dataiterator_set_keyname(Dataiterator *di, Id keyname)
1415 di->keyname = keyname;
1416 di->keynames[0] = keyname;
1420 dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
1424 if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
1426 di->state = di_bye; /* sorry */
1429 for (i = di->nkeynames + 1; i > 0; i--)
1430 di->keynames[i] = di->keynames[i - 1];
1431 di->keynames[0] = di->keyname = keyname;
1436 dataiterator_free(Dataiterator *di)
1438 if (di->matcher.match)
1439 datamatcher_free(&di->matcher);
1441 solv_free(di->dupstr);
1444 static unsigned char *
1445 dataiterator_find_keyname(Dataiterator *di, Id keyname)
1448 Repokey *keys = di->data->keys, *key;
1451 for (keyp = di->keyp; *keyp; keyp++)
1452 if (keys[*keyp].name == keyname)
1457 if (key->type == REPOKEY_TYPE_DELETED)
1459 if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
1460 return 0; /* get_data will not work, no need to forward */
1461 dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
1469 is_filelist_extension(Repodata *data)
1472 if (!repodata_precheck_keyname(data, SOLVABLE_FILELIST))
1474 for (j = 1; j < data->nkeys; j++)
1475 if (data->keys[j].name == SOLVABLE_FILELIST)
1477 if (j == data->nkeys)
1479 if (data->state != REPODATA_AVAILABLE)
1481 for (j = 1; j < data->nkeys; j++)
1482 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1488 dataiterator_filelistcheck(Dataiterator *di)
1491 int needcomplete = 0;
1492 Repodata *data = di->data;
1494 if ((di->flags & SEARCH_COMPLETE_FILELIST) != 0)
1495 if (!di->matcher.match
1496 || ((di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING
1497 && (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_GLOB)
1498 || !repodata_filelistfilter_matches(data, di->matcher.match))
1500 if (data->state != REPODATA_AVAILABLE)
1501 return needcomplete ? 1 : 0;
1504 /* we don't need the complete filelist, so ignore all stubs */
1505 if (data->repo->nrepodata == 2)
1507 for (j = 1; j < data->nkeys; j++)
1508 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1514 /* we need the complete filelist. check if we habe a filtered filelist and there's
1515 * a extension with the complete filelist later on */
1516 for (j = 1; j < data->nkeys; j++)
1517 if (data->keys[j].name == SOLVABLE_FILELIST)
1519 if (j == data->nkeys)
1520 return 0; /* does not have filelist */
1521 for (j = 1; j < data->nkeys; j++)
1522 if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
1524 if (j == data->nkeys)
1525 return 1; /* this is the externsion */
1526 while (data - data->repo->repodata + 1 < data->repo->nrepodata)
1529 if (is_filelist_extension(data))
1537 dataiterator_step(Dataiterator *di)
1541 if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate) {
1542 unsigned int ddpoff = di->ddp - di->vert_ddp;
1543 di->vert_off += ddpoff;
1544 di->vert_len -= ddpoff;
1545 di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len);
1546 di->vert_storestate = di->data->storestate;
1548 di->state = di_nextkey;
1554 case di_enterrepo: di_enterrepo:
1555 if (!di->repo || (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS)))
1557 if (!(di->flags & SEARCH_THISSOLVID))
1559 di->solvid = di->repo->start - 1; /* reset solvid iterator */
1560 goto di_nextsolvable;
1564 case di_entersolvable: di_entersolvable:
1567 di->repodataid = 1; /* reset repodata iterator */
1568 if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
1570 extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1];
1572 di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
1574 goto di_entersolvablekey;
1579 case di_enterrepodata: di_enterrepodata:
1582 if (di->repodataid >= di->repo->nrepodata)
1583 goto di_nextsolvable;
1584 di->data = di->repo->repodata + di->repodataid;
1586 if (di->repodataid && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
1587 goto di_nextrepodata;
1588 if (!maybe_load_repodata(di->data, di->keyname))
1589 goto di_nextrepodata;
1590 di->dp = solvid2data(di->data, di->solvid, &schema);
1592 goto di_nextrepodata;
1593 if (di->solvid == SOLVID_POS)
1594 di->solvid = di->pool->pos.solvid;
1595 /* reset key iterator */
1596 di->keyp = di->data->schemadata + di->data->schemata[schema];
1599 case di_enterschema: di_enterschema:
1601 di->dp = dataiterator_find_keyname(di, di->keyname);
1602 if (!di->dp || !*di->keyp)
1606 goto di_nextrepodata;
1610 case di_enterkey: di_enterkey:
1612 di->key = di->data->keys + *di->keyp;
1615 /* this is get_data() modified to store vert_ data */
1616 if (di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
1619 di->dp = data_read_id(di->dp, &off);
1620 di->dp = data_read_id(di->dp, &len);
1621 di->vert_ddp = di->ddp = get_vertical_data(di->data, di->key, off, len);
1624 di->vert_storestate = di->data->storestate;
1626 else if (di->key->storage == KEY_STORAGE_INCORE)
1629 if (di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0))
1630 di->dp = data_skip_key(di->data, di->dp, di->key);
1636 if (di->key->type == REPOKEY_TYPE_DELETED)
1638 if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
1640 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1646 di->ddp = data_fetch(di->ddp, &di->kv, di->key);
1648 di->state = di_nextkey;
1650 di->state = di_nextattr;
1653 case di_nextkey: di_nextkey:
1654 if (!di->keyname && *++di->keyp)
1660 case di_nextrepodata: di_nextrepodata:
1661 if (di->repodataid && ++di->repodataid < di->repo->nrepodata)
1662 goto di_enterrepodata;
1665 case di_nextsolvable: di_nextsolvable:
1666 if (!(di->flags & SEARCH_THISSOLVID))
1669 di->solvid = di->repo->start;
1672 for (; di->solvid < di->repo->end; di->solvid++)
1674 if (di->pool->solvables[di->solvid].repo == di->repo)
1675 goto di_entersolvable;
1680 case di_nextrepo: di_nextrepo:
1685 if (di->repoid < di->pool->nrepos)
1687 di->repo = di->pool->repos[di->repoid];
1693 case di_bye: di_bye:
1697 case di_enterarray: di_enterarray:
1698 if (di->key->name == REPOSITORY_SOLVABLES)
1700 di->ddp = data_read_id(di->ddp, (Id *)&di->kv.num);
1705 case di_nextarrayelement: di_nextarrayelement:
1708 di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
1709 if (di->kv.entry == di->kv.num)
1711 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1713 if (!(di->flags & SEARCH_ARRAYSENTINEL))
1715 di->kv.str = (char *)di->ddp;
1717 di->state = di_nextkey;
1720 if (di->kv.entry == di->kv.num - 1)
1722 if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
1723 di->ddp = data_read_id(di->ddp, &di->kv.id);
1724 di->kv.str = (char *)di->ddp;
1725 if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
1727 if ((di->flags & SEARCH_SUB) != 0)
1728 di->state = di_entersub;
1730 di->state = di_nextarrayelement;
1733 case di_entersub: di_entersub:
1734 if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
1735 goto di_nextarrayelement; /* sorry, full */
1736 di->parents[di->nparents].kv = di->kv;
1737 di->parents[di->nparents].dp = di->dp;
1738 di->parents[di->nparents].keyp = di->keyp;
1739 di->dp = (unsigned char *)di->kv.str;
1740 di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
1741 memset(&di->kv, 0, sizeof(di->kv));
1742 di->kv.parent = &di->parents[di->nparents].kv;
1744 di->keyname = di->keynames[di->nparents - di->rootlevel];
1745 goto di_enterschema;
1747 case di_leavesub: di_leavesub:
1748 if (di->nparents - 1 < di->rootlevel)
1751 di->dp = di->parents[di->nparents].dp;
1752 di->kv = di->parents[di->nparents].kv;
1753 di->keyp = di->parents[di->nparents].keyp;
1754 di->key = di->data->keys + *di->keyp;
1755 di->ddp = (unsigned char *)di->kv.str;
1756 di->keyname = di->keynames[di->nparents - di->rootlevel];
1757 goto di_nextarrayelement;
1759 /* special solvable attr handling follows */
1761 case di_nextsolvablekey: di_nextsolvablekey:
1762 if (di->keyname || di->key->name == RPM_RPMDBID)
1763 goto di_enterrepodata;
1767 case di_entersolvablekey: di_entersolvablekey:
1768 di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
1769 if (!di->idp || !*di->idp)
1770 goto di_nextsolvablekey;
1774 di->kv.id = *di->idp;
1775 di->kv.num = *di->idp; /* for rpmdbid */
1776 di->kv.num2 = 0; /* for rpmdbid */
1778 di->state = di_nextsolvablekey;
1784 case di_nextsolvableattr:
1785 di->state = di_nextsolvableattr;
1786 di->kv.id = *di->idp++;
1791 di->state = di_nextsolvablekey;
1797 if (di->matcher.match)
1800 /* simple pre-check so that we don't need to stringify */
1801 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0)
1802 if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
1804 if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
1806 if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
1810 if (!datamatcher_match(&di->matcher, str))
1815 if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->flags & SEARCH_FILES) != 0)
1816 repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags);
1818 /* found something! */
1824 dataiterator_entersub(Dataiterator *di)
1826 if (di->state == di_nextarrayelement)
1827 di->state = di_entersub;
1831 dataiterator_setpos(Dataiterator *di)
1833 if (di->kv.eof == 2)
1835 pool_clear_pos(di->pool);
1838 di->pool->pos.solvid = di->solvid;
1839 di->pool->pos.repo = di->repo;
1840 di->pool->pos.repodataid = di->data - di->repo->repodata;
1841 di->pool->pos.schema = di->kv.id;
1842 di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
1846 dataiterator_setpos_parent(Dataiterator *di)
1848 if (!di->kv.parent || di->kv.parent->eof == 2)
1850 pool_clear_pos(di->pool);
1853 di->pool->pos.solvid = di->solvid;
1854 di->pool->pos.repo = di->repo;
1855 di->pool->pos.repodataid = di->data - di->repo->repodata;
1856 di->pool->pos.schema = di->kv.parent->id;
1857 di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
1860 /* clones just the position, not the search keys/matcher */
1862 dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
1864 di->state = from->state;
1865 di->flags &= ~SEARCH_THISSOLVID;
1866 di->flags |= (from->flags & SEARCH_THISSOLVID);
1867 di->repo = from->repo;
1868 di->data = from->data;
1870 di->ddp = from->ddp;
1871 di->idp = from->idp;
1872 di->keyp = from->keyp;
1873 di->key = from->key;
1875 di->repodataid = from->repodataid;
1876 di->solvid = from->solvid;
1877 di->repoid = from->repoid;
1878 di->rootlevel = from->rootlevel;
1879 memcpy(di->parents, from->parents, sizeof(from->parents));
1880 di->nparents = from->nparents;
1884 for (i = 1; i < di->nparents; i++)
1885 di->parents[i].kv.parent = &di->parents[i - 1].kv;
1886 di->kv.parent = &di->parents[di->nparents - 1].kv;
1890 if (from->dupstr && from->dupstr == from->kv.str)
1892 di->dupstrn = from->dupstrn;
1893 di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
1898 dataiterator_seek(Dataiterator *di, int whence)
1900 if ((whence & DI_SEEK_STAY) != 0)
1901 di->rootlevel = di->nparents;
1902 switch (whence & ~DI_SEEK_STAY)
1905 if (di->state != di_nextarrayelement)
1907 if ((whence & DI_SEEK_STAY) != 0)
1908 di->rootlevel = di->nparents + 1; /* XXX: dangerous! */
1909 di->state = di_entersub;
1911 case DI_SEEK_PARENT:
1918 if (di->rootlevel > di->nparents)
1919 di->rootlevel = di->nparents;
1920 di->dp = di->parents[di->nparents].dp;
1921 di->kv = di->parents[di->nparents].kv;
1922 di->keyp = di->parents[di->nparents].keyp;
1923 di->key = di->data->keys + *di->keyp;
1924 di->ddp = (unsigned char *)di->kv.str;
1925 di->keyname = di->keynames[di->nparents - di->rootlevel];
1926 di->state = di_nextarrayelement;
1928 case DI_SEEK_REWIND:
1934 di->dp = (unsigned char *)di->kv.parent->str;
1935 di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
1936 di->state = di_enterschema;
1944 dataiterator_skip_attribute(Dataiterator *di)
1946 if (di->state == di_nextsolvableattr)
1947 di->state = di_nextsolvablekey;
1949 di->state = di_nextkey;
1953 dataiterator_skip_solvable(Dataiterator *di)
1958 di->keyname = di->keynames[0];
1959 di->state = di_nextsolvable;
1963 dataiterator_skip_repo(Dataiterator *di)
1968 di->keyname = di->keynames[0];
1969 di->state = di_nextrepo;
1973 dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
1978 di->keyname = di->keynames[0];
1979 if (solvid == SOLVID_POS)
1981 di->repo = di->pool->pos.repo;
1988 if (!di->pool->pos.repodataid && di->pool->pos.solvid == SOLVID_META) {
1989 solvid = SOLVID_META; /* META pos hack */
1991 di->data = di->repo->repodata + di->pool->pos.repodataid;
1995 else if (solvid > 0)
1997 di->repo = di->pool->solvables[solvid].repo;
2002 if (!di->pool->urepos)
2008 di->repo = di->pool->repos[di->repoid];
2010 if (solvid != SOLVID_POS)
2012 di->solvid = solvid;
2014 di->flags |= SEARCH_THISSOLVID;
2015 di->state = di_enterrepo;
2019 dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
2025 di->repoid = 0; /* 0 means stay at repo */
2028 di->flags &= ~SEARCH_THISSOLVID;
2029 di->state = di_enterrepo;
2033 dataiterator_match(Dataiterator *di, Datamatcher *ma)
2036 if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
2038 return ma ? datamatcher_match(ma, str) : 1;
2042 dataiterator_strdup(Dataiterator *di)
2046 if (!di->kv.str || di->kv.str == di->dupstr)
2048 switch (di->key->type)
2051 case REPOKEY_TYPE_DIRSTRARRAY:
2052 if (di->kv.num) /* was it stringified into tmp space? */
2053 l = strlen(di->kv.str) + 1;
2058 if (l < 0 && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
2060 switch (di->key->type)
2062 case REPOKEY_TYPE_STR:
2063 case REPOKEY_TYPE_DIRSTRARRAY:
2064 l = strlen(di->kv.str) + 1;
2067 l = solv_chksum_len(di->key->type);
2069 case REPOKEY_TYPE_BINARY:
2076 if (!di->dupstrn || di->dupstrn < l)
2078 di->dupstrn = l + 16;
2079 di->dupstr = solv_realloc(di->dupstr, di->dupstrn);
2082 memcpy(di->dupstr, di->kv.str, l);
2083 di->kv.str = di->dupstr;
2087 /************************************************************************
2088 * data modify functions
2091 /* extend repodata so that it includes solvables p */
2093 repodata_extend(Repodata *data, Id p)
2095 if (data->start == data->end)
2096 data->start = data->end = p;
2099 int old = data->end - data->start;
2100 int new = p - data->end + 1;
2103 data->attrs = solv_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
2104 memset(data->attrs + old, 0, new * sizeof(Id *));
2106 data->incoreoffset = solv_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
2107 memset(data->incoreoffset + old, 0, new * sizeof(Id));
2110 if (p < data->start)
2112 int old = data->end - data->start;
2113 int new = data->start - p;
2116 data->attrs = solv_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
2117 memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
2118 memset(data->attrs, 0, new * sizeof(Id *));
2120 data->incoreoffset = solv_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
2121 memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
2122 memset(data->incoreoffset, 0, new * sizeof(Id));
2127 /* shrink end of repodata */
2129 repodata_shrink(Repodata *data, int end)
2133 if (data->end <= end)
2135 if (data->start >= end)
2139 for (i = 0; i < data->end - data->start; i++)
2140 solv_free(data->attrs[i]);
2141 data->attrs = solv_free(data->attrs);
2143 data->incoreoffset = solv_free(data->incoreoffset);
2144 data->start = data->end = 0;
2149 for (i = end; i < data->end; i++)
2150 solv_free(data->attrs[i - data->start]);
2151 data->attrs = solv_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
2153 if (data->incoreoffset)
2154 data->incoreoffset = solv_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
2158 /* extend repodata so that it includes solvables from start to start + num - 1 */
2160 repodata_extend_block(Repodata *data, Id start, Id num)
2164 if (!data->incoreoffset)
2166 /* this also means that data->attrs is NULL */
2167 data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
2168 data->start = start;
2169 data->end = start + num;
2172 repodata_extend(data, start);
2174 repodata_extend(data, start + num - 1);
2177 /**********************************************************************/
2180 #define REPODATA_ATTRS_BLOCK 31
2181 #define REPODATA_ATTRDATA_BLOCK 1023
2182 #define REPODATA_ATTRIDDATA_BLOCK 63
2183 #define REPODATA_ATTRNUM64DATA_BLOCK 15
2187 repodata_new_handle(Repodata *data)
2191 data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
2192 data->nxattrs = 2; /* -1: SOLVID_META */
2194 data->xattrs = solv_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
2195 data->xattrs[data->nxattrs] = 0;
2196 return -(data->nxattrs++);
2200 repodata_get_attrp(Repodata *data, Id handle)
2204 if (handle == SOLVID_META && !data->xattrs)
2206 data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
2209 return data->xattrs - handle;
2211 if (handle < data->start || handle >= data->end)
2212 repodata_extend(data, handle);
2214 data->attrs = solv_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
2215 return data->attrs + (handle - data->start);
2219 repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
2225 app = repodata_get_attrp(data, handle);
2230 /* Determine equality based on the name only, allows us to change
2231 type (when overwrite is set), and makes TYPE_CONSTANT work. */
2232 for (pp = ap; *pp; pp += 2)
2233 if (data->keys[*pp].name == data->keys[keyid].name)
2237 if (overwrite || data->keys[*pp].type == REPOKEY_TYPE_DELETED)
2246 ap = solv_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
2256 repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
2260 keyid = repodata_key2id(data, key, 1);
2261 repodata_insert_keyid(data, solvid, keyid, val, 1);
2265 repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
2269 key.type = REPOKEY_TYPE_ID;
2271 key.storage = KEY_STORAGE_INCORE;
2272 repodata_set(data, solvid, &key, id);
2276 repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned long long num)
2280 key.type = REPOKEY_TYPE_NUM;
2282 key.storage = KEY_STORAGE_INCORE;
2283 if (num >= 0x80000000)
2285 data->attrnum64data = solv_extend(data->attrnum64data, data->attrnum64datalen, 1, sizeof(unsigned long long), REPODATA_ATTRNUM64DATA_BLOCK);
2286 data->attrnum64data[data->attrnum64datalen] = num;
2287 num = 0x80000000 | data->attrnum64datalen++;
2289 repodata_set(data, solvid, &key, (Id)num);
2293 repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
2297 if (data->localpool)
2298 id = stringpool_str2id(&data->spool, str, 1);
2300 id = pool_str2id(data->repo->pool, str, 1);
2302 key.type = REPOKEY_TYPE_ID;
2304 key.storage = KEY_STORAGE_INCORE;
2305 repodata_set(data, solvid, &key, id);
2309 repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
2313 key.type = REPOKEY_TYPE_CONSTANT;
2314 key.size = constant;
2315 key.storage = KEY_STORAGE_INCORE;
2316 repodata_set(data, solvid, &key, 0);
2320 repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
2324 key.type = REPOKEY_TYPE_CONSTANTID;
2326 key.storage = KEY_STORAGE_INCORE;
2327 repodata_set(data, solvid, &key, 0);
2331 repodata_set_void(Repodata *data, Id solvid, Id keyname)
2335 key.type = REPOKEY_TYPE_VOID;
2337 key.storage = KEY_STORAGE_INCORE;
2338 repodata_set(data, solvid, &key, 0);
2342 repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
2347 l = strlen(str) + 1;
2349 key.type = REPOKEY_TYPE_STR;
2351 key.storage = KEY_STORAGE_INCORE;
2352 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2353 memcpy(data->attrdata + data->attrdatalen, str, l);
2354 repodata_set(data, solvid, &key, data->attrdatalen);
2355 data->attrdatalen += l;
2359 repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len)
2367 key.type = REPOKEY_TYPE_BINARY;
2369 key.storage = KEY_STORAGE_INCORE;
2370 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK);
2371 dp = data->attrdata + data->attrdatalen;
2372 if (len >= (1 << 14))
2374 if (len >= (1 << 28))
2375 *dp++ = (len >> 28) | 128;
2376 if (len >= (1 << 21))
2377 *dp++ = (len >> 21) | 128;
2378 *dp++ = (len >> 14) | 128;
2380 if (len >= (1 << 7))
2381 *dp++ = (len >> 7) | 128;
2384 memcpy(dp, buf, len);
2385 repodata_set(data, solvid, &key, data->attrdatalen);
2386 data->attrdatalen = dp + len - data->attrdata;
2389 /* add an array element consisting of entrysize Ids to the repodata. modifies attriddata
2390 * so that the caller can append entrysize new elements plus the termination zero there */
2392 repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
2395 Id *ida, *pp, **ppp;
2397 /* check if it is the same as last time, this speeds things up a lot */
2398 if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
2400 /* great! just append the new data */
2401 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2402 data->attriddatalen--; /* overwrite terminating 0 */
2403 data->lastdatalen += entrysize;
2407 ppp = repodata_get_attrp(data, handle);
2411 for (; *pp; pp += 2)
2412 if (data->keys[*pp].name == keyname)
2415 if (!pp || !*pp || data->keys[*pp].type != keytype)
2417 /* not found. allocate new key */
2423 key.storage = KEY_STORAGE_INCORE;
2424 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2425 keyid = repodata_key2id(data, &key, 1);
2426 repodata_insert_keyid(data, handle, keyid, data->attriddatalen, 1);
2427 data->lasthandle = handle;
2428 data->lastkey = keyid;
2429 data->lastdatalen = data->attriddatalen + entrysize + 1;
2433 for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
2434 oldsize += entrysize;
2435 if (ida + 1 == data->attriddata + data->attriddatalen)
2437 /* this was the last entry, just append it */
2438 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2439 data->attriddatalen--; /* overwrite terminating 0 */
2443 /* too bad. move to back. */
2444 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2445 memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
2446 pp[1] = data->attriddatalen;
2447 data->attriddatalen += oldsize;
2449 data->lasthandle = handle;
2450 data->lastkey = *pp;
2451 data->lastdatalen = data->attriddatalen + entrysize + 1;
2455 repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2456 const unsigned char *str)
2461 if (!(l = solv_chksum_len(type)))
2466 key.storage = KEY_STORAGE_INCORE;
2467 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2468 memcpy(data->attrdata + data->attrdatalen, str, l);
2469 repodata_set(data, solvid, &key, data->attrdatalen);
2470 data->attrdatalen += l;
2474 repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
2477 unsigned char buf[64];
2480 if (!(l = solv_chksum_len(type)))
2482 if (l > sizeof(buf) || solv_hex2bin(&str, buf, l) != l)
2484 repodata_set_bin_checksum(data, solvid, keyname, type, buf);
2488 repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
2492 if (!(l = solv_chksum_len(type)))
2494 return pool_bin2hex(data->repo->pool, buf, l);
2497 /* rpm filenames don't contain the epoch, so strip it */
2498 static inline const char *
2499 evrid2vrstr(Pool *pool, Id evrid)
2501 const char *p, *evr = pool_id2str(pool, evrid);
2504 for (p = evr; *p >= '0' && *p <= '9'; p++)
2506 return p != evr && *p == ':' && p[1] ? p + 1 : evr;
2510 repodata_set_poolstrn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
2513 if (data->localpool)
2514 id = stringpool_strn2id(&data->spool, str, l, 1);
2516 id = pool_strn2id(data->repo->pool, str, l, 1);
2517 repodata_set_id(data, solvid, keyname, id);
2521 repodata_set_strn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
2524 repodata_set_str(data, solvid, keyname, str);
2527 char *s = solv_strdup(str);
2529 repodata_set_str(data, solvid, keyname, s);
2535 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
2537 Pool *pool = data->repo->pool;
2539 const char *str, *fp;
2543 repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
2546 if ((dir = strrchr(file, '/')) != 0)
2557 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2562 if (l == 1 && dir[0] == '.')
2564 s = pool->solvables + solvid;
2567 str = pool_id2str(pool, s->arch);
2568 if (!strncmp(dir, str, l) && !str[l])
2569 repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
2571 repodata_set_strn(data, solvid, SOLVABLE_MEDIADIR, dir, l);
2574 str = pool_id2str(pool, s->name);
2576 if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
2579 str = evrid2vrstr(pool, s->evr);
2581 if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
2584 str = pool_id2str(pool, s->arch);
2586 if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
2588 repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
2593 repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
2596 /* XXX: medianr is currently not stored */
2598 repodata_set_deltalocation(Repodata *data, Id handle, int medianr, const char *dir, const char *file)
2601 const char *evr, *suf, *s;
2605 if ((dir = strrchr(file, '/')) != 0)
2616 if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
2621 if (l == 1 && dir[0] == '.')
2624 repodata_set_poolstrn(data, handle, DELTA_LOCATION_DIR, dir, l);
2625 evr = strchr(file, '-');
2628 for (s = evr - 1; s > file; s--)
2635 suf = strrchr(file, '.');
2638 for (s = suf - 1; s > file; s--)
2644 if (!strcmp(suf, ".delta.rpm") || !strcmp(suf, ".patch.rpm"))
2646 /* We accept one more item as suffix. */
2647 for (s = suf - 1; s > file; s--)
2657 if (suf && evr && suf < evr)
2659 repodata_set_poolstrn(data, handle, DELTA_LOCATION_NAME, file, evr ? evr - file : strlen(file));
2661 repodata_set_poolstrn(data, handle, DELTA_LOCATION_EVR, evr + 1, suf ? suf - evr - 1: strlen(evr + 1));
2663 repodata_set_poolstr(data, handle, DELTA_LOCATION_SUFFIX, suf + 1);
2667 repodata_set_sourcepkg(Repodata *data, Id solvid, const char *sourcepkg)
2669 Pool *pool = data->repo->pool;
2670 Solvable *s = pool->solvables + solvid;
2671 const char *p, *sevr, *sarch, *name, *evr;
2673 p = strrchr(sourcepkg, '.');
2674 if (!p || strcmp(p, ".rpm") != 0)
2677 repodata_set_str(data, solvid, SOLVABLE_SOURCENAME, sourcepkg);
2681 while (p > sourcepkg && *p != '.')
2683 if (*p != '.' || p == sourcepkg)
2686 while (p > sourcepkg && *p != '-')
2688 if (*p != '-' || p == sourcepkg)
2691 while (p > sourcepkg && *p != '-')
2693 if (*p != '-' || p == sourcepkg)
2696 pool = s->repo->pool;
2698 name = pool_id2str(pool, s->name);
2699 if (name && !strncmp(sourcepkg, name, sevr - sourcepkg - 1) && name[sevr - sourcepkg - 1] == 0)
2700 repodata_set_void(data, solvid, SOLVABLE_SOURCENAME);
2702 repodata_set_id(data, solvid, SOLVABLE_SOURCENAME, pool_strn2id(pool, sourcepkg, sevr - sourcepkg - 1, 1));
2704 evr = evrid2vrstr(pool, s->evr);
2705 if (evr && !strncmp(sevr, evr, sarch - sevr - 1) && evr[sarch - sevr - 1] == 0)
2706 repodata_set_void(data, solvid, SOLVABLE_SOURCEEVR);
2708 repodata_set_id(data, solvid, SOLVABLE_SOURCEEVR, pool_strn2id(pool, sevr, sarch - sevr - 1, 1));
2710 if (!strcmp(sarch, "src.rpm"))
2711 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_SRC);
2712 else if (!strcmp(sarch, "nosrc.rpm"))
2713 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_NOSRC);
2715 repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, pool_strn2id(pool, sarch, strlen(sarch) - 4, 1));
2719 repodata_set_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
2725 key.type = REPOKEY_TYPE_IDARRAY;
2727 key.storage = KEY_STORAGE_INCORE;
2728 repodata_set(data, solvid, &key, data->attriddatalen);
2729 data->attriddata = solv_extend(data->attriddata, data->attriddatalen, q->count + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
2730 for (i = 0; i < q->count; i++)
2731 data->attriddata[data->attriddatalen++] = q->elements[i];
2732 data->attriddata[data->attriddatalen++] = 0;
2736 repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
2740 fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
2742 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
2743 data->attriddata[data->attriddatalen++] = dir;
2744 data->attriddata[data->attriddatalen++] = num;
2745 data->attriddata[data->attriddatalen++] = num2;
2746 data->attriddata[data->attriddatalen++] = 0;
2750 repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
2756 l = strlen(str) + 1;
2757 data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
2758 memcpy(data->attrdata + data->attrdatalen, str, l);
2759 stroff = data->attrdatalen;
2760 data->attrdatalen += l;
2763 fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str, data->attriddatalen);
2765 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
2766 data->attriddata[data->attriddatalen++] = dir;
2767 data->attriddata[data->attriddatalen++] = stroff;
2768 data->attriddata[data->attriddatalen++] = 0;
2772 repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
2775 fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
2777 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
2778 data->attriddata[data->attriddatalen++] = id;
2779 data->attriddata[data->attriddatalen++] = 0;
2783 repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
2787 if (data->localpool)
2788 id = stringpool_str2id(&data->spool, str, 1);
2790 id = pool_str2id(data->repo->pool, str, 1);
2791 repodata_add_idarray(data, solvid, keyname, id);
2795 repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2797 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
2798 data->attriddata[data->attriddatalen++] = ghandle;
2799 data->attriddata[data->attriddatalen++] = 0;
2803 repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
2805 repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
2806 data->attriddata[data->attriddatalen++] = ghandle;
2807 data->attriddata[data->attriddatalen++] = 0;
2811 repodata_set_kv(Repodata *data, Id solvid, Id keyname, Id keytype, KeyValue *kv)
2815 case REPOKEY_TYPE_ID:
2816 repodata_set_id(data, solvid, keyname, kv->id);
2818 case REPOKEY_TYPE_CONSTANTID:
2819 repodata_set_constantid(data, solvid, keyname, kv->id);
2821 case REPOKEY_TYPE_IDARRAY:
2822 repodata_add_idarray(data, solvid, keyname, kv->id);
2824 case REPOKEY_TYPE_STR:
2825 repodata_set_str(data, solvid, keyname, kv->str);
2827 case REPOKEY_TYPE_VOID:
2828 repodata_set_void(data, solvid, keyname);
2830 case REPOKEY_TYPE_NUM:
2831 repodata_set_num(data, solvid, keyname, SOLV_KV_NUM64(kv));
2833 case REPOKEY_TYPE_CONSTANT:
2834 repodata_set_constant(data, solvid, keyname, kv->num);
2836 case REPOKEY_TYPE_DIRNUMNUMARRAY:
2838 repodata_add_dirnumnum(data, solvid, keyname, kv->id, kv->num, kv->num2);
2840 case REPOKEY_TYPE_DIRSTRARRAY:
2841 repodata_add_dirstr(data, solvid, keyname, kv->id, kv->str);
2844 repodata_set_bin_checksum(data, solvid, keyname, keytype, (const unsigned char *)kv->str);
2852 repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname)
2855 app = repodata_get_attrp(data, solvid);
2859 for (; *ap; ap += 2)
2860 if (data->keys[*ap].name == keyname)
2866 for (; *ap; ap += 2)
2868 if (data->keys[*ap].name == keyname)
2876 /* XXX: does not work correctly, needs fix in iterators! */
2878 repodata_unset(Repodata *data, Id solvid, Id keyname)
2882 key.type = REPOKEY_TYPE_DELETED;
2884 key.storage = KEY_STORAGE_INCORE;
2885 repodata_set(data, solvid, &key, 0);
2888 /* add all (uninternalized) attrs from src to dest */
2890 repodata_merge_attrs(Repodata *data, Id dest, Id src)
2893 if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
2895 for (; *keyp; keyp += 2)
2896 repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
2899 /* add some (uninternalized) attrs from src to dest */
2901 repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
2904 if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
2906 for (; *keyp; keyp += 2)
2907 if (!keyidmap || MAPTST(keyidmap, keyp[0]))
2908 repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
2911 /* swap (uninternalized) attrs from src and dest */
2913 repodata_swap_attrs(Repodata *data, Id dest, Id src)
2916 if (!data->attrs || dest == src)
2918 if (dest < data->start || dest >= data->end)
2919 repodata_extend(data, dest);
2920 if (src < data->start || src >= data->end)
2921 repodata_extend(data, src);
2922 tmpattrs = data->attrs[dest - data->start];
2923 data->attrs[dest - data->start] = data->attrs[src - data->start];
2924 data->attrs[src - data->start] = tmpattrs;
2928 /**********************************************************************/
2930 /* TODO: unify with repo_write and repo_solv! */
2932 #define EXTDATA_BLOCK 1023
2940 data_addid(struct extdata *xd, Id sx)
2942 unsigned int x = (unsigned int)sx;
2945 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2946 dp = xd->buf + xd->len;
2951 *dp++ = (x >> 28) | 128;
2953 *dp++ = (x >> 21) | 128;
2954 *dp++ = (x >> 14) | 128;
2957 *dp++ = (x >> 7) | 128;
2959 xd->len = dp - xd->buf;
2963 data_addid64(struct extdata *xd, unsigned long long x)
2965 if (x >= 0x100000000)
2969 data_addid(xd, (Id)(x >> 35));
2970 xd->buf[xd->len - 1] |= 128;
2972 data_addid(xd, (Id)((unsigned int)x | 0x80000000));
2973 xd->buf[xd->len - 5] = (x >> 28) | 128;
2976 data_addid(xd, (Id)x);
2980 data_addideof(struct extdata *xd, Id sx, int eof)
2982 unsigned int x = (unsigned int)sx;
2985 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
2986 dp = xd->buf + xd->len;
2991 *dp++ = (x >> 27) | 128;
2993 *dp++ = (x >> 20) | 128;
2994 *dp++ = (x >> 13) | 128;
2997 *dp++ = (x >> 6) | 128;
2998 *dp++ = eof ? (x & 63) : (x & 63) | 64;
2999 xd->len = dp - xd->buf;
3003 data_addblob(struct extdata *xd, unsigned char *blob, int len)
3005 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
3006 memcpy(xd->buf + xd->len, blob, len);
3010 /*********************************/
3012 /* this is to reduct memory usage when internalizing oversized repos */
3014 compact_attrdata(Repodata *data, int entry, int nentry)
3017 unsigned int attrdatastart = data->attrdatalen;
3018 unsigned int attriddatastart = data->attriddatalen;
3019 if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
3021 for (i = entry; i < nentry; i++)
3023 Id v, *attrs = data->attrs[i];
3026 for (; *attrs; attrs += 2)
3028 switch (data->keys[*attrs].type)
3030 case REPOKEY_TYPE_STR:
3031 case REPOKEY_TYPE_BINARY:
3033 if ((unsigned int)attrs[1] < attrdatastart)
3034 attrdatastart = attrs[1];
3036 case REPOKEY_TYPE_DIRSTRARRAY:
3037 for (v = attrs[1]; data->attriddata[v] ; v += 2)
3038 if (data->attriddata[v + 1] < attrdatastart)
3039 attrdatastart = data->attriddata[v + 1];
3041 case REPOKEY_TYPE_IDARRAY:
3042 case REPOKEY_TYPE_DIRNUMNUMARRAY:
3043 if ((unsigned int)attrs[1] < attriddatastart)
3044 attriddatastart = attrs[1];
3046 case REPOKEY_TYPE_FIXARRAY:
3047 case REPOKEY_TYPE_FLEXARRAY:
3055 printf("compact_attrdata %d %d\n", entry, nentry);
3056 printf("attrdatastart: %d\n", attrdatastart);
3057 printf("attriddatastart: %d\n", attriddatastart);
3059 if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
3061 for (i = entry; i < nentry; i++)
3063 Id v, *attrs = data->attrs[i];
3066 for (; *attrs; attrs += 2)
3068 switch (data->keys[*attrs].type)
3070 case REPOKEY_TYPE_STR:
3071 case REPOKEY_TYPE_BINARY:
3073 attrs[1] -= attrdatastart;
3075 case REPOKEY_TYPE_DIRSTRARRAY:
3076 for (v = attrs[1]; data->attriddata[v] ; v += 2)
3077 data->attriddata[v + 1] -= attrdatastart;
3079 case REPOKEY_TYPE_IDARRAY:
3080 case REPOKEY_TYPE_DIRNUMNUMARRAY:
3081 attrs[1] -= attriddatastart;
3090 data->attrdatalen -= attrdatastart;
3091 memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
3092 data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
3094 if (attriddatastart)
3096 data->attriddatalen -= attriddatastart;
3097 memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
3098 data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
3102 /* internalalize some key into incore/vincore data */
3105 repodata_serialize_key(Repodata *data, struct extdata *newincore,
3106 struct extdata *newvincore,
3108 Repokey *key, Id val)
3112 unsigned int oldvincorelen = 0;
3116 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3119 oldvincorelen = xd->len;
3123 case REPOKEY_TYPE_VOID:
3124 case REPOKEY_TYPE_CONSTANT:
3125 case REPOKEY_TYPE_CONSTANTID:
3126 case REPOKEY_TYPE_DELETED:
3128 case REPOKEY_TYPE_STR:
3129 data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
3131 case REPOKEY_TYPE_MD5:
3132 data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
3134 case REPOKEY_TYPE_SHA1:
3135 data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
3137 case REPOKEY_TYPE_SHA224:
3138 data_addblob(xd, data->attrdata + val, SIZEOF_SHA224);
3140 case REPOKEY_TYPE_SHA256:
3141 data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
3143 case REPOKEY_TYPE_SHA384:
3144 data_addblob(xd, data->attrdata + val, SIZEOF_SHA384);
3146 case REPOKEY_TYPE_SHA512:
3147 data_addblob(xd, data->attrdata + val, SIZEOF_SHA512);
3149 case REPOKEY_TYPE_NUM:
3150 if (val & 0x80000000)
3152 data_addid64(xd, data->attrnum64data[val ^ 0x80000000]);
3156 case REPOKEY_TYPE_ID:
3157 case REPOKEY_TYPE_DIR:
3158 data_addid(xd, val);
3160 case REPOKEY_TYPE_BINARY:
3163 unsigned char *dp = data_read_id(data->attrdata + val, &len);
3164 dp += (unsigned int)len;
3165 data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
3168 case REPOKEY_TYPE_IDARRAY:
3169 for (ida = data->attriddata + val; *ida; ida++)
3170 data_addideof(xd, ida[0], ida[1] ? 0 : 1);
3172 case REPOKEY_TYPE_DIRNUMNUMARRAY:
3173 for (ida = data->attriddata + val; *ida; ida += 3)
3175 data_addid(xd, ida[0]);
3176 data_addid(xd, ida[1]);
3177 data_addideof(xd, ida[2], ida[3] ? 0 : 1);
3180 case REPOKEY_TYPE_DIRSTRARRAY:
3181 for (ida = data->attriddata + val; *ida; ida += 2)
3183 data_addideof(xd, ida[0], ida[2] ? 0 : 1);
3184 data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
3187 case REPOKEY_TYPE_FIXARRAY:
3191 for (ida = data->attriddata + val; *ida; ida++)
3195 kp = data->xattrs[-*ida];
3197 continue; /* ignore empty elements */
3199 for (; *kp; kp += 2)
3203 schemaid = repodata_schema2id(data, schema, 1);
3204 else if (schemaid != repodata_schema2id(data, schema, 0))
3206 pool_debug(data->repo->pool, SOLV_ERROR, "repodata_serialize_key: fixarray substructs with different schemas\n");
3211 data_addid(xd, num);
3214 data_addid(xd, schemaid);
3215 for (ida = data->attriddata + val; *ida; ida++)
3217 Id *kp = data->xattrs[-*ida];
3220 for (; *kp; kp += 2)
3221 repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3225 case REPOKEY_TYPE_FLEXARRAY:
3228 for (ida = data->attriddata + val; *ida; ida++)
3230 data_addid(xd, num);
3231 for (ida = data->attriddata + val; *ida; ida++)
3233 Id *kp = data->xattrs[-*ida];
3236 data_addid(xd, 0); /* XXX */
3243 schemaid = repodata_schema2id(data, schema, 1);
3244 data_addid(xd, schemaid);
3245 kp = data->xattrs[-*ida];
3247 repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
3252 pool_debug(data->repo->pool, SOLV_FATAL, "repodata_serialize_key: don't know how to handle type %d\n", key->type);
3255 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3257 /* put offset/len in incore */
3258 data_addid(newincore, data->lastverticaloffset + oldvincorelen);
3259 oldvincorelen = xd->len - oldvincorelen;
3260 data_addid(newincore, oldvincorelen);
3264 /* create a circular linked list of all keys that share
3265 * the same keyname */
3267 calculate_keylink(Repodata *data)
3271 Id maxkeyname = 0, *keytable = 0;
3272 link = solv_calloc(data->nkeys, sizeof(Id));
3273 if (data->nkeys <= 2)
3275 for (i = 1; i < data->nkeys; i++)
3277 Id n = data->keys[i].name;
3278 if (n >= maxkeyname)
3280 keytable = solv_realloc2(keytable, n + 128, sizeof(Id));
3281 memset(keytable + maxkeyname, 0, (n + 128 - maxkeyname) * sizeof(Id));
3282 maxkeyname = n + 128;
3292 /* remove links that just point to themselfs */
3293 for (i = 1; i < data->nkeys; i++)
3296 solv_free(keytable);
3301 repodata_internalize(Repodata *data)
3303 Repokey *key, solvkey;
3305 Id schemaid, keyid, *schema, *sp, oldschemaid, *keyp, *seen;
3306 Offset *oldincoreoffs = 0;
3308 unsigned char *dp, *ndp;
3310 struct extdata newincore;
3311 struct extdata newvincore;
3316 if (!data->attrs && !data->xattrs)
3320 printf("repodata_internalize %d\n", data->repodataid);
3321 printf(" attr data: %d K\n", data->attrdatalen / 1024);
3322 printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3324 newvincore.buf = data->vincore;
3325 newvincore.len = data->vincorelen;
3327 /* find the solvables key, create if needed */
3328 memset(&solvkey, 0, sizeof(solvkey));
3329 solvkey.name = REPOSITORY_SOLVABLES;
3330 solvkey.type = REPOKEY_TYPE_FLEXARRAY;
3332 solvkey.storage = KEY_STORAGE_INCORE;
3333 solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);
3335 schema = solv_malloc2(data->nkeys, sizeof(Id));
3336 seen = solv_malloc2(data->nkeys, sizeof(Id));
3338 /* Merge the data already existing (in data->schemata, ->incoredata and
3339 friends) with the new attributes in data->attrs[]. */
3340 nentry = data->end - data->start;
3341 memset(&newincore, 0, sizeof(newincore));
3342 data_addid(&newincore, 0); /* start data at offset 1 */
3344 data->mainschema = 0;
3345 data->mainschemaoffsets = solv_free(data->mainschemaoffsets);
3347 keylink = calculate_keylink(data);
3348 /* join entry data */
3349 /* we start with the meta data, entry -1 */
3350 for (entry = -1; entry < nentry; entry++)
3353 dp = data->incoredata;
3356 dp += entry >= 0 ? data->incoreoffset[entry] : 1;
3357 dp = data_read_id(dp, &oldschemaid);
3359 memset(seen, 0, data->nkeys * sizeof(Id));
3361 fprintf(stderr, "oldschemaid %d\n", oldschemaid);
3362 fprintf(stderr, "schemata %d\n", data->schemata[oldschemaid]);
3363 fprintf(stderr, "schemadata %p\n", data->schemadata);
3366 /* seen: -1: old data, 0: skipped, >0: id + 1 */
3370 for (keyp = data->schemadata + data->schemata[oldschemaid]; *keyp; keyp++)
3374 /* oops, should not happen */
3378 seen[*keyp] = -1; /* use old marker */
3381 haveoldkl = 1; /* potential keylink conflict */
3384 /* strip solvables key */
3385 if (entry < 0 && solvkeyid && seen[solvkeyid])
3388 for (sp = keyp = schema; *sp; sp++)
3389 if (*sp != solvkeyid)
3392 seen[solvkeyid] = 0;
3396 /* add new entries */
3398 keyp = data->attrs ? data->attrs[entry] : 0;
3400 keyp = data->xattrs ? data->xattrs[1] : 0;
3402 for (; *keyp; keyp += 2)
3408 if (haveoldkl && keylink[*keyp]) /* this should be pretty rare */
3411 for (kl = keylink[*keyp]; kl != *keyp; kl = keylink[kl])
3414 /* replacing old key kl, remove from schema and seen */
3416 for (osp = schema; osp < sp; osp++)
3419 memmove(osp, osp + 1, (sp - osp) * sizeof(Id));
3427 seen[*keyp] = keyp[1] + 1;
3430 /* add solvables key if needed */
3431 if (entry < 0 && data->end != data->start)
3433 *sp++ = solvkeyid; /* always last in schema */
3440 /* Ideally we'd like to sort the new schema here, to ensure
3441 schema equality independend of the ordering. */
3442 schemaid = repodata_schema2id(data, schema, 1);
3444 schemaid = oldschemaid;
3448 data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
3449 data->mainschema = schemaid;
3452 /* find offsets in old incore data */
3456 for (sp = data->schemadata + data->schemata[oldschemaid]; *sp; sp++)
3457 if (seen[*sp] == -1)
3458 lastneeded = sp + 1;
3462 oldincoreoffs = solv_malloc2(data->nkeys, 2 * sizeof(Offset));
3463 for (sp = data->schemadata + data->schemata[oldschemaid]; sp != lastneeded; sp++)
3465 /* Skip the data associated with this old key. */
3466 key = data->keys + *sp;
3468 if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
3470 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
3471 ndp = data_skip(ndp, REPOKEY_TYPE_ID);
3473 else if (key->storage == KEY_STORAGE_INCORE)
3474 ndp = data_skip_key(data, ndp, key);
3475 oldincoreoffs[*sp * 2] = dp - data->incoredata;
3476 oldincoreoffs[*sp * 2 + 1] = ndp - dp;
3482 /* just copy over the complete old entry (including the schemaid) if there was no new data */
3483 if (entry >= 0 && !neednewschema && oldschemaid && (!data->attrs || !data->attrs[entry]) && dp)
3485 ndp = data->incoredata + data->incoreoffset[entry];
3486 data->incoreoffset[entry] = newincore.len;
3487 data_addblob(&newincore, ndp, dp - ndp);
3491 /* Now create data blob. We walk through the (possibly new) schema
3492 and either copy over old data, or insert the new. */
3494 data->incoreoffset[entry] = newincore.len;
3495 data_addid(&newincore, schemaid);
3497 /* we don't use a pointer to the schemadata here as repodata_serialize_key
3498 * may call repodata_schema2id() which might realloc our schemadata */
3499 for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
3503 data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
3504 if (keyid == solvkeyid)
3506 /* add flexarray entry count */
3507 data_addid(&newincore, data->end - data->start);
3508 break; /* always the last entry */
3511 if (seen[keyid] == -1)
3513 if (oldincoreoffs[keyid * 2 + 1])
3514 data_addblob(&newincore, data->incoredata + oldincoreoffs[keyid * 2], oldincoreoffs[keyid * 2 + 1]);
3516 else if (seen[keyid])
3517 repodata_serialize_key(data, &newincore, &newvincore, schema, data->keys + keyid, seen[keyid] - 1);
3522 if (entry >= 0 && data->attrs)
3524 if (data->attrs[entry])
3525 data->attrs[entry] = solv_free(data->attrs[entry]);
3526 if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
3528 compact_attrdata(data, entry + 1, nentry); /* try to free some memory */
3530 printf(" attr data: %d K\n", data->attrdatalen / 1024);
3531 printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
3532 printf(" incore data: %d K\n", newincore.len / 1024);
3533 printf(" sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
3534 /* malloc_stats(); */
3539 /* free all xattrs */
3540 for (entry = 0; entry < data->nxattrs; entry++)
3541 if (data->xattrs[entry])
3542 solv_free(data->xattrs[entry]);
3543 data->xattrs = solv_free(data->xattrs);
3546 data->lasthandle = 0;
3548 data->lastdatalen = 0;
3552 solv_free(oldincoreoffs);
3553 repodata_free_schemahash(data);
3555 solv_free(data->incoredata);
3556 data->incoredata = newincore.buf;
3557 data->incoredatalen = newincore.len;
3558 data->incoredatafree = 0;
3560 data->vincore = newvincore.buf;
3561 data->vincorelen = newvincore.len;
3563 data->attrs = solv_free(data->attrs);
3564 data->attrdata = solv_free(data->attrdata);
3565 data->attriddata = solv_free(data->attriddata);
3566 data->attrnum64data = solv_free(data->attrnum64data);
3567 data->attrdatalen = 0;
3568 data->attriddatalen = 0;
3569 data->attrnum64datalen = 0;
3571 printf("repodata_internalize %d done\n", data->repodataid);
3572 printf(" incore data: %d K\n", data->incoredatalen / 1024);
3577 repodata_disable_paging(Repodata *data)
3579 if (maybe_load_repodata(data, 0))
3581 repopagestore_disable_paging(&data->store);
3587 repodata_load_stub(Repodata *data)
3589 Repo *repo = data->repo;
3590 Pool *pool = repo->pool;
3592 struct _Pool_tmpspace oldtmpspace;
3595 if (!pool->loadcallback)
3597 data->state = REPODATA_ERROR;
3600 data->state = REPODATA_LOADING;
3602 /* save tmp space and pos */
3603 oldtmpspace = pool->tmpspace;
3604 memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
3607 r = pool->loadcallback(pool, data, pool->loadcallbackdata);
3609 /* restore tmp space and pos */
3610 for (i = 0; i < POOL_TMPSPACEBUF; i++)
3611 solv_free(pool->tmpspace.buf[i]);
3612 pool->tmpspace = oldtmpspace;
3613 if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
3614 memset(&oldpos, 0, sizeof(oldpos));
3617 data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR;
3621 repodata_add_stubkey(Repodata *data, Id keyname, Id keytype)
3625 xkey.name = keyname;
3626 xkey.type = keytype;
3627 xkey.storage = KEY_STORAGE_INCORE;
3629 repodata_key2id(data, &xkey, 1);
3633 repodata_add_stub(Repodata **datap)
3635 Repodata *data = *datap;
3636 Repo *repo = data->repo;
3637 Id repodataid = data - repo->repodata;
3638 Repodata *sdata = repo_add_repodata(repo, 0);
3639 data = repo->repodata + repodataid;
3640 if (data->end > data->start)
3641 repodata_extend_block(sdata, data->start, data->end - data->start);
3642 sdata->state = REPODATA_STUB;
3643 sdata->loadcallback = repodata_load_stub;
3649 repodata_create_stubs(Repodata *data)
3651 Repo *repo = data->repo;
3652 Pool *pool = repo->pool;
3659 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
3660 while (dataiterator_step(&di))
3661 if (di.data == data)
3663 dataiterator_free(&di);
3666 stubdataids = solv_calloc(cnt, sizeof(*stubdataids));
3667 for (i = 0; i < cnt; i++)
3669 sdata = repodata_add_stub(&data);
3670 stubdataids[i] = sdata - repo->repodata;
3673 dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
3675 while (dataiterator_step(&di))
3677 if (di.data != data)
3679 if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
3681 dataiterator_entersub(&di);
3682 sdata = repo->repodata + stubdataids[i++];
3686 repodata_set_kv(sdata, SOLVID_META, di.key->name, di.key->type, &di.kv);
3687 if (di.key->name == REPOSITORY_KEYS && di.key->type == REPOKEY_TYPE_IDARRAY)
3692 xkeyname = di.kv.id;
3696 repodata_add_stubkey(sdata, xkeyname, di.kv.id);
3701 dataiterator_free(&di);
3702 for (i = 0; i < cnt; i++)
3703 repodata_internalize(repo->repodata + stubdataids[i]);
3704 solv_free(stubdataids);
3709 repodata_memused(Repodata *data)
3711 return data->incoredatalen + data->vincorelen;