X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=src%2Frepodata.c;h=4e75b6d61a9c4f9019a61e64c1f746501ebefdf4;hb=e0e41666c5a48c53ab30c943f2b0964a3600eadd;hp=c150e6fad443683d7350b30eb9eb2b8d868f394f;hpb=772a525a41907396f60f6642975a5fff193bd1c1;p=platform%2Fupstream%2Flibsolv.git diff --git a/src/repodata.c b/src/repodata.c index c150e6f..4e75b6d 100644 --- a/src/repodata.c +++ b/src/repodata.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, Novell Inc. + * Copyright (c) 2018, SUSE LLC. * * This program is licensed under the BSD license, read LICENSE.BSD * for further information @@ -34,6 +34,10 @@ #include "repopack.h" #include "repopage.h" +#ifdef _WIN32 + #include "strfncs.h" +#endif + #define REPODATA_BLOCK 255 static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key); @@ -94,6 +98,8 @@ repodata_freedata(Repodata *data) solv_free(data->attrnum64data); solv_free(data->dircache); + + repodata_free_filelistfilter(data); } void @@ -188,7 +194,7 @@ repodata_schema2id(Repodata *data, Id *schema, int create) data->schematahash = schematahash = solv_calloc(256, sizeof(Id)); for (i = 1; i < data->nschemata; i++) { - for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++) + for (sp = data->schemadata + data->schemata[i], h = 0; *sp;) h = h * 7 + *sp++; h &= 255; schematahash[h] = i; @@ -205,11 +211,13 @@ repodata_schema2id(Repodata *data, Id *schema, int create) cid = schematahash[h]; if (cid) { - if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id))) + if ((data->schemata[cid] + len <= data->schemadatalen) && + !memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id))) return cid; /* cache conflict, do a slow search */ for (cid = 1; cid < data->nschemata; cid++) - if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id))) + if ((data->schemata[cid] + len <= data->schemadatalen) && + !memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id))) return cid; } /* a new one */ @@ -268,17 +276,13 @@ repodata_str2dir(Repodata *data, const char *dir, int create) #endif const char *dire; - parent = 0; if (!*dir) - return 0; + return data->dirpool.ndirs ? 0 : dirpool_add_dir(&data->dirpool, 0, 0, create); while (*dir == '/' && dir[1] == '/') dir++; if (*dir == '/' && !dir[1]) - { - if (data->dirpool.ndirs) - return 1; - return dirpool_add_dir(&data->dirpool, 0, 1, create); - } + return data->dirpool.ndirs ? 1 : dirpool_add_dir(&data->dirpool, 0, 1, create); + parent = 0; #ifdef DIRCACHE_SIZE dirs = dir; if (data->dircache) @@ -355,6 +359,8 @@ repodata_dir2str(Repodata *data, Id did, const char *suf) if (!did) return suf ? suf : ""; + if (did == 1 && !suf) + return "/"; parent = did; while (parent) { @@ -479,11 +485,11 @@ get_vertical_data(Repodata *data, Repokey *key, Id off, Id len) if (off >= data->lastverticaloffset) { off -= data->lastverticaloffset; - if (off + len > data->vincorelen) + if ((unsigned int)off + len > data->vincorelen) return 0; return data->vincore + off; } - if (off + len > key->size) + if ((unsigned int)off + len > key->size) return 0; /* we now have the offset, go into vertical */ off += data->verticaloffset[key - data->keys]; @@ -520,17 +526,36 @@ get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance) return 0; } -static int -load_repodata(Repodata *data) +void +repodata_load(Repodata *data) { + if (data->state != REPODATA_STUB) + return; if (data->loadcallback) + data->loadcallback(data); + else + data->state = REPODATA_ERROR; +} + +static int +maybe_load_repodata_stub(Repodata *data, Id keyname) +{ + if (data->state != REPODATA_STUB) { - data->loadcallback(data); - if (data->state == REPODATA_AVAILABLE) - return 1; + data->state = REPODATA_ERROR; + return 0; } - data->state = REPODATA_ERROR; - return 0; + if (keyname) + { + int i; + for (i = 1; i < data->nkeys; i++) + if (keyname == data->keys[i].name) + break; + if (i == data->nkeys) + return 0; + } + repodata_load(data); + return data->state == REPODATA_AVAILABLE ? 1 : 0; } static inline int @@ -538,28 +563,11 @@ maybe_load_repodata(Repodata *data, Id keyname) { if (keyname && !repodata_precheck_keyname(data, keyname)) return 0; /* do not bother... */ - switch(data->state) - { - case REPODATA_STUB: - if (keyname) - { - int i; - for (i = 1; i < data->nkeys; i++) - if (keyname == data->keys[i].name) - break; - if (i == data->nkeys) - return 0; - } - return load_repodata(data); - case REPODATA_ERROR: - return 0; - case REPODATA_AVAILABLE: - case REPODATA_LOADING: - return 1; - default: - data->state = REPODATA_ERROR; - return 0; - } + if (data->state == REPODATA_AVAILABLE || data->state == REPODATA_LOADING) + return 1; + if (data->state == REPODATA_ERROR) + return 0; + return maybe_load_repodata_stub(data, keyname); } static inline unsigned char * @@ -568,17 +576,21 @@ solvid2data(Repodata *data, Id solvid, Id *schemap) unsigned char *dp = data->incoredata; if (!dp) return 0; - if (solvid == SOLVID_META) /* META */ - dp += 1; - else if (solvid == SOLVID_POS) /* META */ + if (solvid == SOLVID_META) + dp += 1; /* offset of "meta" solvable */ + else if (solvid == SOLVID_POS) { Pool *pool = data->repo->pool; if (data->repo != pool->pos.repo) return 0; if (data != data->repo->repodata + pool->pos.repodataid) return 0; - *schemap = pool->pos.schema; - return data->incoredata + pool->pos.dp; + dp += pool->pos.dp; + if (pool->pos.dp != 1) + { + *schemap = pool->pos.schema; + return dp; + } } else { @@ -593,7 +605,7 @@ solvid2data(Repodata *data, Id solvid, Id *schemap) * data lookup */ -static inline unsigned char * +static unsigned char * find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp) { unsigned char *dp; @@ -616,12 +628,61 @@ find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp) return 0; if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) return dp; /* no need to forward... */ + if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET) + return 0; /* get_data will not work, no need to forward */ dp = forward_to_key(data, *kp, keyp, dp); if (!dp) return 0; return get_data(data, key, &dp, 0); } +static const Id * +repodata_lookup_schemakeys(Repodata *data, Id solvid) +{ + Id schema; + if (!maybe_load_repodata(data, 0)) + return 0; + if (!solvid2data(data, solvid, &schema)) + return 0; + return data->schemadata + data->schemata[schema]; +} + +static Id * +alloc_keyskip() +{ + Id *keyskip = solv_calloc(3 + 256, sizeof(Id)); + keyskip[0] = 256; + keyskip[1] = keyskip[2] = 1; + return keyskip; +} + +Id * +repodata_fill_keyskip(Repodata *data, Id solvid, Id *keyskip) +{ + const Id *keyp; + Id maxkeyname, value; + keyp = repodata_lookup_schemakeys(data, solvid); + if (!keyp) + return keyskip; /* no keys for this solvid */ + if (!keyskip) + keyskip = alloc_keyskip(); + maxkeyname = keyskip[0]; + value = keyskip[1] + data->repodataid; + for (; *keyp; keyp++) + { + Id keyname = data->keys[*keyp].name; + if (keyname >= maxkeyname) + { + int newmax = (keyname | 255) + 1; + keyskip = solv_realloc2(keyskip, 3 + newmax, sizeof(Id)); + memset(keyskip + (3 + maxkeyname), 0, (newmax - maxkeyname) * sizeof(Id)); + keyskip[0] = maxkeyname = newmax; + } + keyskip[3 + keyname] = value; + } + return keyskip; +} + Id repodata_lookup_type(Repodata *data, Id solvid, Id keyname) { @@ -678,52 +739,32 @@ repodata_lookup_str(Repodata *data, Id solvid, Id keyname) return pool_id2str(data->repo->pool, id); } -int -repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long *value) +unsigned long long +repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long notfound) { unsigned char *dp; Repokey *key; unsigned int high, low; - *value = 0; dp = find_key_data(data, solvid, keyname, &key); if (!dp) - return 0; + return notfound; switch (key->type) { case REPOKEY_TYPE_NUM: data_read_num64(dp, &low, &high); - *value = (unsigned long long)high << 32 | low; - return 1; - case REPOKEY_TYPE_U32: - data_read_u32(dp, &low); - *value = low; - return 1; + return (unsigned long long)high << 32 | low; case REPOKEY_TYPE_CONSTANT: - *value = key->size; - return 1; + return key->size; default: - return 0; + return notfound; } } int repodata_lookup_void(Repodata *data, Id solvid, Id keyname) { - Id schema; - Id *keyp; - unsigned char *dp; - - if (!maybe_load_repodata(data, keyname)) - return 0; - dp = solvid2data(data, solvid, &schema); - if (!dp) - return 0; - /* can't use find_key_data as we need to test the type */ - for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++) - if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID) - return 1; - return 0; + return repodata_lookup_type(data, solvid, keyname) == REPOKEY_TYPE_VOID ? 1 : 0; } const unsigned char * @@ -735,8 +776,13 @@ repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep) dp = find_key_data(data, solvid, keyname, &key); if (!dp) return 0; - if (!(key->type == REPOKEY_TYPE_MD5 || key->type == REPOKEY_TYPE_SHA1 || key->type == REPOKEY_TYPE_SHA256)) - return 0; + switch (key->type) + { + case_CHKSUM_TYPES: + break; + default: + return 0; + } *typep = key->type; return dp; } @@ -753,18 +799,141 @@ repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q) dp = find_key_data(data, solvid, keyname, &key); if (!dp) return 0; - if (key->type != REPOKEY_TYPE_IDARRAY && key->type != REPOKEY_TYPE_REL_IDARRAY) - return 0; - for (;;) + switch (key->type) { - dp = data_read_ideof(dp, &id, &eof); + case REPOKEY_TYPE_CONSTANTID: + queue_push(q, key->size); + break; + case REPOKEY_TYPE_ID: + dp = data_read_id(dp, &id); queue_push(q, id); - if (eof) + break; + case REPOKEY_TYPE_IDARRAY: + for (;;) + { + dp = data_read_ideof(dp, &id, &eof); + queue_push(q, id); + if (eof) + break; + } + break; + default: + return 0; + } + return 1; +} + +const void * +repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp) +{ + unsigned char *dp; + Repokey *key; + Id len; + + dp = find_key_data(data, solvid, keyname, &key); + if (!dp || key->type != REPOKEY_TYPE_BINARY) + { + *lenp = 0; + return 0; + } + dp = data_read_id(dp, &len); + *lenp = len; + return dp; +} + +unsigned int +repodata_lookup_count(Repodata *data, Id solvid, Id keyname) +{ + unsigned char *dp; + Repokey *key; + unsigned int cnt = 0; + + dp = find_key_data(data, solvid, keyname, &key); + if (!dp) + return 0; + switch (key->type) + { + case REPOKEY_TYPE_IDARRAY: + case REPOKEY_TYPE_REL_IDARRAY: + for (cnt = 1; (*dp & 0xc0) != 0; dp++) + if ((*dp & 0xc0) == 0x40) + cnt++; + return cnt; + case REPOKEY_TYPE_FIXARRAY: + case REPOKEY_TYPE_FLEXARRAY: + data_read_id(dp, (int *)&cnt); + return cnt; + case REPOKEY_TYPE_DIRSTRARRAY: + for (;;) + { + cnt++; + while (*dp & 0x80) + dp++; + if (!(*dp++ & 0x40)) + return cnt; + dp += strlen((const char *)dp) + 1; + } + case REPOKEY_TYPE_DIRNUMNUMARRAY: + for (;;) + { + cnt++; + while (*dp++ & 0x80) + ; + while (*dp++ & 0x80) + ; + while (*dp & 0x80) + dp++; + if (!(*dp++ & 0x40)) + return cnt; + } + default: break; } return 1; } +/* highly specialized function to speed up fileprovides adding. + * - repodata must be available + * - solvid must be >= data->start and < data->end + * - returns NULL is not found, a "" entry if wrong type + * - also returns wrong type for REPOKEY_TYPE_DELETED + */ +const unsigned char * +repodata_lookup_packed_dirstrarray(Repodata *data, Id solvid, Id keyname) +{ + static unsigned char wrongtype[2] = { 0x00 /* dir id 0 */, 0 /* "" */ }; + unsigned char *dp; + Id schema, *keyp, *kp; + Repokey *key; + + if (!data->incoredata || !data->incoreoffset[solvid - data->start]) + return 0; + dp = data->incoredata + data->incoreoffset[solvid - data->start]; + dp = data_read_id(dp, &schema); + keyp = data->schemadata + data->schemata[schema]; + for (kp = keyp; *kp; kp++) + if (data->keys[*kp].name == keyname) + break; + if (!*kp) + return 0; + key = data->keys + *kp; + if (key->type != REPOKEY_TYPE_DIRSTRARRAY) + return wrongtype; + dp = forward_to_key(data, *kp, keyp, dp); + if (key->storage == KEY_STORAGE_INCORE) + return dp; + if (key->storage == KEY_STORAGE_VERTICAL_OFFSET && dp) + { + Id off, len; + dp = data_read_id(dp, &off); + data_read_id(dp, &len); + return get_vertical_data(data, key, off, len); + } + return 0; +} + +/* id translation functions */ + Id repodata_globalize_id(Repodata *data, Id id, int create) { @@ -784,49 +953,180 @@ repodata_localize_id(Repodata *data, Id id, int create) Id repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create) { + const char *s; if (!id || !data || !fromdata) return id; - if (!data->localpool || !fromdata->localpool) + if (data == fromdata || (!data->localpool && !fromdata->localpool)) + return id; + if (fromdata->localpool) + s = stringpool_id2str(&fromdata->spool, id); + else + s = pool_id2str(data->repo->pool, id); + if (data->localpool) + return stringpool_str2id(&data->spool, s, create); + else + return pool_str2id(data->repo->pool, s, create); +} + +Id +repodata_translate_dir_slow(Repodata *data, Repodata *fromdata, Id dir, int create, Id *cache) +{ + Id parent, compid; + if (!dir) { - if (fromdata->localpool) - id = repodata_globalize_id(fromdata, id, create); - if (data->localpool) - id = repodata_localize_id(data, id, create); - return id; + /* make sure that the dirpool has an entry */ + if (create && !data->dirpool.ndirs) + dirpool_add_dir(&data->dirpool, 0, 0, create); + return 0; } - /* localpool is set in both data and fromdata */ - return stringpool_str2id(&data->spool, stringpool_id2str(&fromdata->spool, id), create); + parent = dirpool_parent(&fromdata->dirpool, dir); + if (parent) + { + if (!(parent = repodata_translate_dir(data, fromdata, parent, create, cache))) + return 0; + } + compid = dirpool_compid(&fromdata->dirpool, dir); + if (compid > 1 && (data->localpool || fromdata->localpool)) + { + if (!(compid = repodata_translate_id(data, fromdata, compid, create))) + return 0; + } + if (!(compid = dirpool_add_dir(&data->dirpool, parent, compid, create))) + return 0; + if (cache) + { + cache[(dir & 255) * 2] = dir; + cache[(dir & 255) * 2 + 1] = compid; + } + return compid; } -Id -repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id voidid) +/************************************************************************ + * uninternalized lookup / search + */ + +static void +data_fetch_uninternalized(Repodata *data, Repokey *key, Id value, KeyValue *kv) +{ + Id *array; + kv->eof = 1; + switch (key->type) + { + case REPOKEY_TYPE_STR: + kv->str = (const char *)data->attrdata + value; + return; + case REPOKEY_TYPE_CONSTANT: + kv->num2 = 0; + kv->num = key->size; + return; + case REPOKEY_TYPE_CONSTANTID: + kv->id = key->size; + return; + case REPOKEY_TYPE_NUM: + kv->num2 = 0; + kv->num = value; + if (value & 0x80000000) + { + kv->num = (unsigned int)data->attrnum64data[value ^ 0x80000000]; + kv->num2 = (unsigned int)(data->attrnum64data[value ^ 0x80000000] >> 32); + } + return; + case_CHKSUM_TYPES: + kv->num = 0; /* not stringified */ + kv->str = (const char *)data->attrdata + value; + return; + case REPOKEY_TYPE_BINARY: + kv->str = (const char *)data_read_id(data->attrdata + value, (Id *)&kv->num); + return; + case REPOKEY_TYPE_IDARRAY: + array = data->attriddata + (value + kv->entry); + kv->id = array[0]; + kv->eof = array[1] ? 0 : 1; + return; + case REPOKEY_TYPE_DIRSTRARRAY: + kv->num = 0; /* not stringified */ + array = data->attriddata + (value + kv->entry * 2); + kv->id = array[0]; + kv->str = (const char *)data->attrdata + array[1]; + kv->eof = array[2] ? 0 : 1; + return; + case REPOKEY_TYPE_DIRNUMNUMARRAY: + array = data->attriddata + (value + kv->entry * 3); + kv->id = array[0]; + kv->num = array[1]; + kv->num2 = array[2]; + kv->eof = array[3] ? 0 : 1; + return; + case REPOKEY_TYPE_FIXARRAY: + case REPOKEY_TYPE_FLEXARRAY: + array = data->attriddata + (value + kv->entry); + kv->id = array[0]; /* the handle */ + kv->eof = array[1] ? 0 : 1; + return; + default: + kv->id = value; + return; + } +} + +Repokey * +repodata_lookup_kv_uninternalized(Repodata *data, Id solvid, Id keyname, KeyValue *kv) { Id *ap; - if (!data->attrs) + if (!data->attrs || solvid < data->start || solvid >= data->end) return 0; ap = data->attrs[solvid - data->start]; if (!ap) return 0; for (; *ap; ap += 2) { - if (data->keys[*ap].name != keyname) + Repokey *key = data->keys + *ap; + if (key->name != keyname) continue; - if (data->keys[*ap].type == REPOKEY_TYPE_VOID) - return voidid; - if (data->keys[*ap].type == REPOKEY_TYPE_ID) - return ap[1]; - return 0; + data_fetch_uninternalized(data, key, ap[1], kv); + return key; } return 0; } +void +repodata_search_uninternalized(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata) +{ + Id *ap; + int stop; + Solvable *s; + KeyValue kv; + + if (!data->attrs || solvid < data->start || solvid >= data->end) + return; + ap = data->attrs[solvid - data->start]; + if (!ap) + return; + for (; *ap; ap += 2) + { + Repokey *key = data->keys + *ap; + if (keyname && key->name != keyname) + continue; + s = solvid > 0 ? data->repo->pool->solvables + solvid : 0; + kv.entry = 0; + do + { + data_fetch_uninternalized(data, key, ap[1], &kv); + stop = callback(cbdata, s, data, key, &kv); + kv.entry++; + } + while (!kv.eof && !stop); + if (keyname || stop > SEARCH_NEXT_KEY) + return; + } +} /************************************************************************ * data search */ -int +const char * repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags) { switch (key->type) @@ -838,7 +1138,7 @@ repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int f kv->str = stringpool_id2str(&data->spool, kv->id); else kv->str = pool_id2str(pool, kv->id); - if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE) + if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE && (key->name == SOLVABLE_NAME || key->type == REPOKEY_TYPE_IDARRAY)) { const char *s; for (s = kv->str; *s >= 'a' && *s <= 'z'; s++) @@ -846,43 +1146,85 @@ repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int f if (*s == ':' && s > kv->str) kv->str = s + 1; } - return 1; + return kv->str; case REPOKEY_TYPE_STR: - return 1; + return kv->str; case REPOKEY_TYPE_DIRSTRARRAY: if (!(flags & SEARCH_FILES)) - return 1; /* match just the basename */ + return kv->str; /* match just the basename */ if (kv->num) - return 1; /* already stringified */ + return kv->str; /* already stringified */ /* Put the full filename into kv->str. */ kv->str = repodata_dir2str(data, kv->id, kv->str); kv->num = 1; /* mark stringification */ - return 1; - case REPOKEY_TYPE_MD5: - case REPOKEY_TYPE_SHA1: - case REPOKEY_TYPE_SHA256: + return kv->str; + case_CHKSUM_TYPES: if (!(flags & SEARCH_CHECKSUMS)) return 0; /* skip em */ if (kv->num) - return 1; /* already stringified */ + return kv->str; /* already stringified */ kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str); kv->num = 1; /* mark stringification */ - return 1; + return kv->str; default: return 0; } } +/* this is an internal hack to pass the parent kv to repodata_search_keyskip */ struct subschema_data { - Solvable *s; void *cbdata; + Id solvid; KeyValue *parent; }; +void +repodata_search_arrayelement(Repodata *data, Id solvid, Id keyname, int flags, KeyValue *kv, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata) +{ + repodata_search_keyskip(data, solvid, keyname, flags | SEARCH_SUBSCHEMA, (Id *)kv, callback, cbdata); +} + +static int +repodata_search_array(Repodata *data, Id solvid, Id keyname, int flags, Repokey *key, KeyValue *kv, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata) +{ + Solvable *s = solvid > 0 ? data->repo->pool->solvables + solvid : 0; + unsigned char *dp = (unsigned char *)kv->str; + int stop; + Id schema = 0; + + if (!dp || kv->entry != -1) + return 0; + while (++kv->entry < (int)kv->num) + { + if (kv->entry) + dp = data_skip_schema(data, dp, schema); + if (kv->entry == 0 || key->type == REPOKEY_TYPE_FLEXARRAY) + dp = data_read_id(dp, &schema); + kv->id = schema; + kv->str = (const char *)dp; + kv->eof = kv->entry == kv->num - 1 ? 1 : 0; + stop = callback(cbdata, s, data, key, kv); + if (stop && stop != SEARCH_ENTERSUB) + return stop; + if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB) + repodata_search_keyskip(data, solvid, keyname, flags | SEARCH_SUBSCHEMA, (Id *)kv, callback, cbdata); + } + if ((flags & SEARCH_ARRAYSENTINEL) != 0) + { + if (kv->entry) + dp = data_skip_schema(data, dp, schema); + kv->id = 0; + kv->str = (const char *)dp; + kv->eof = 2; + return callback(cbdata, s, data, key, kv); + } + return 0; +} + /* search a specific repodata */ void -repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata) +repodata_search_keyskip(Repodata *data, Id solvid, Id keyname, int flags, Id *keyskip, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata) { Id schema; Repokey *key; @@ -895,14 +1237,13 @@ repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback if (!maybe_load_repodata(data, keyname)) return; - if (solvid == SOLVID_SUBSCHEMA) + if ((flags & SEARCH_SUBSCHEMA) != 0) { - struct subschema_data *subd = cbdata; - cbdata = subd->cbdata; - s = subd->s; - schema = subd->parent->id; - dp = (unsigned char *)subd->parent->str; - kv.parent = subd->parent; + flags ^= SEARCH_SUBSCHEMA; + kv.parent = (KeyValue *)keyskip; + keyskip = 0; + schema = kv.parent->id; + dp = (unsigned char *)kv.parent->str; } else { @@ -910,9 +1251,9 @@ repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback dp = solvid2data(data, solvid, &schema); if (!dp) return; - s = data->repo->pool->solvables + solvid; kv.parent = 0; } + s = solvid > 0 ? data->repo->pool->solvables + solvid : 0; keyp = data->schemadata + data->schemata[schema]; if (keyname) { @@ -932,54 +1273,30 @@ repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback { stop = 0; key = data->keys + keyid; - ddp = get_data(data, key, &dp, *keyp ? 1 : 0); + ddp = get_data(data, key, &dp, *keyp && !onekey ? 1 : 0); - if (key->type == REPOKEY_TYPE_DELETED) - continue; - if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY) + if (keyskip && (key->name >= keyskip[0] || keyskip[3 + key->name] != keyskip[1] + data->repodataid)) + { + if (onekey) + return; + continue; + } + if (key->type == REPOKEY_TYPE_DELETED && !(flags & SEARCH_KEEP_TYPE_DELETED)) { - struct subschema_data subd; - int nentries; - Id schema = 0; - - subd.cbdata = cbdata; - subd.s = s; - subd.parent = &kv; - ddp = data_read_id(ddp, &nentries); - kv.num = nentries; - kv.entry = 0; - kv.eof = 0; - while (ddp && nentries > 0) - { - if (!--nentries) - kv.eof = 1; - if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry) - ddp = data_read_id(ddp, &schema); - kv.id = schema; - kv.str = (char *)ddp; - stop = callback(cbdata, s, data, key, &kv); - if (stop > SEARCH_NEXT_KEY) - return; - if (stop && stop != SEARCH_ENTERSUB) - break; - if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB) - repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd); - ddp = data_skip_schema(data, ddp, schema); - kv.entry++; - } - if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0) - { - /* sentinel */ - kv.eof = 2; - kv.str = (char *)ddp; - stop = callback(cbdata, s, data, key, &kv); - if (stop > SEARCH_NEXT_KEY) - return; - } if (onekey) return; continue; } + if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY) + { + kv.entry = -1; + ddp = data_read_id(ddp, (Id *)&kv.num); + kv.str = (const char *)ddp; + stop = repodata_search_array(data, solvid, 0, flags, key, &kv, callback, cbdata); + if (onekey || stop > SEARCH_NEXT_KEY) + return; + continue; + } kv.entry = 0; do { @@ -996,6 +1313,12 @@ repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback } void +repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata) +{ + repodata_search_keyskip(data, solvid, keyname, flags, 0, callback, cbdata); +} + +void repodata_setpos_kv(Repodata *data, KeyValue *kv) { Pool *pool = data->repo->pool; @@ -1067,6 +1390,7 @@ solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname) int datamatcher_init(Datamatcher *ma, const char *match, int flags) { + match = match ? solv_strdup(match) : 0; ma->match = match; ma->flags = flags; ma->error = 0; @@ -1104,6 +1428,8 @@ datamatcher_init(Datamatcher *ma, const char *match, int flags) void datamatcher_free(Datamatcher *ma) { + if (ma->match) + ma->match = solv_free((char *)ma->match); if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata) { regfree(ma->matchdata); @@ -1183,21 +1509,6 @@ datamatcher_checkbasename(Datamatcher *ma, const char *basename) return !strcmp(match, basename); } -int -repodata_filelistfilter_matches(Repodata *data, const char *str) -{ - /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */ - /* for now hardcoded */ - if (strstr(str, "bin/")) - return 1; - if (!strncmp(str, "/etc/", 5)) - return 1; - if (!strcmp(str, "/usr/lib/sendmail")) - return 1; - return 0; -} - - enum { di_bye, @@ -1258,10 +1569,7 @@ dataiterator_init_clone(Dataiterator *di, Dataiterator *from) if (di->dupstr) { if (di->dupstr == di->kv.str) - { - di->dupstr = solv_malloc(di->dupstrn); - memcpy(di->dupstr, from->dupstr, di->dupstrn); - } + di->dupstr = solv_memdup(di->dupstr, di->dupstrn); else { di->dupstr = 0; @@ -1279,6 +1587,10 @@ dataiterator_init_clone(Dataiterator *di, Dataiterator *from) di->parents[i].kv.parent = &di->parents[i - 1].kv; di->kv.parent = &di->parents[di->nparents - 1].kv; } + if (di->oldkeyskip) + di->oldkeyskip = solv_memdup2(di->oldkeyskip, 3 + di->oldkeyskip[0], sizeof(Id)); + if (di->keyskip) + di->keyskip = di->oldkeyskip; } int @@ -1354,13 +1666,15 @@ dataiterator_free(Dataiterator *di) datamatcher_free(&di->matcher); if (di->dupstr) solv_free(di->dupstr); + if (di->oldkeyskip) + solv_free(di->oldkeyskip); } -static inline unsigned char * +static unsigned char * dataiterator_find_keyname(Dataiterator *di, Id keyname) { - Id *keyp = di->keyp; - Repokey *keys = di->data->keys; + Id *keyp; + Repokey *keys = di->data->keys, *key; unsigned char *dp; for (keyp = di->keyp; *keyp; keyp++) @@ -1368,6 +1682,11 @@ dataiterator_find_keyname(Dataiterator *di, Id keyname) break; if (!*keyp) return 0; + key = keys + *keyp; + if (key->type == REPOKEY_TYPE_DELETED) + return 0; + if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET) + return 0; /* get_data will not work, no need to forward */ dp = forward_to_key(di->data, *keyp, di->keyp, di->dp); if (!dp) return 0; @@ -1375,41 +1694,21 @@ dataiterator_find_keyname(Dataiterator *di, Id keyname) return dp; } -static int -dataiterator_filelistcheck(Dataiterator *di) -{ - int j; - int needcomplete = 0; - Repodata *data = di->data; - - if ((di->matcher.flags & SEARCH_COMPLETE_FILELIST) != 0) - if (!di->matcher.match - || ((di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING - && (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_GLOB) - || !repodata_filelistfilter_matches(di->data, di->matcher.match)) - needcomplete = 1; - if (data->state != REPODATA_AVAILABLE) - return needcomplete ? 1 : 0; - for (j = 1; j < data->nkeys; j++) - if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST) - break; - return j == data->nkeys && !needcomplete ? 0 : 1; -} - int dataiterator_step(Dataiterator *di) { Id schema; - if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate) { - unsigned int ddpoff = di->ddp - di->vert_ddp; - di->vert_off += ddpoff; - di->vert_len -= ddpoff; - di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len); - di->vert_storestate = di->data->storestate; - if (!di->ddp) - di->state = di_nextkey; - } + if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate) + { + unsigned int ddpoff = di->ddp - di->vert_ddp; + di->vert_off += ddpoff; + di->vert_len -= ddpoff; + di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len); + di->vert_storestate = di->data->storestate; + if (!di->ddp) + di->state = di_nextkey; + } for (;;) { switch (di->state) @@ -1425,18 +1724,28 @@ dataiterator_step(Dataiterator *di) /* FALLTHROUGH */ case di_entersolvable: di_entersolvable: - if (di->repodataid) + if (!di->repodataid) + goto di_enterrepodata; /* POS case, repodata is set */ + if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames) { - di->repodataid = 1; /* reset repodata iterator */ - if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames) - { - extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1]; + extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1]; + di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0); + di->data = 0; + goto di_entersolvablekey; + } - di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0); - di->data = 0; - goto di_entersolvablekey; - } + if (di->keyname) + { + di->data = di->keyname == SOLVABLE_FILELIST ? repo_lookup_filelist_repodata(di->repo, di->solvid, &di->matcher) : repo_lookup_repodata_opt(di->repo, di->solvid, di->keyname); + if (!di->data) + goto di_nextsolvable; + di->repodataid = di->data - di->repo->repodata; + di->keyskip = 0; + goto di_enterrepodata; } + di_leavesolvablekey: + di->repodataid = 1; /* reset repodata iterator */ + di->keyskip = repo_create_keyskip(di->repo, di->solvid, &di->oldkeyskip); /* FALLTHROUGH */ case di_enterrepodata: di_enterrepodata: @@ -1446,8 +1755,6 @@ dataiterator_step(Dataiterator *di) goto di_nextsolvable; di->data = di->repo->repodata + di->repodataid; } - if (di->repodataid && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di)) - goto di_nextrepodata; if (!maybe_load_repodata(di->data, di->keyname)) goto di_nextrepodata; di->dp = solvid2data(di->data, di->solvid, &schema); @@ -1488,15 +1795,17 @@ dataiterator_step(Dataiterator *di) } else if (di->key->storage == KEY_STORAGE_INCORE) { - di->ddp = di->dp; + di->ddp = di->dp; /* start of data */ if (di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0)) - di->dp = data_skip_key(di->data, di->dp, di->key); + di->dp = data_skip_key(di->data, di->dp, di->key); /* advance to next key */ } else di->ddp = 0; if (!di->ddp) goto di_nextkey; - if (di->key->type == REPOKEY_TYPE_DELETED) + if (di->keyskip && (di->key->name >= di->keyskip[0] || di->keyskip[3 + di->key->name] != di->keyskip[1] + di->data->repodataid)) + goto di_nextkey; + if (di->key->type == REPOKEY_TYPE_DELETED && !(di->flags & SEARCH_KEEP_TYPE_DELETED)) goto di_nextkey; if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY) goto di_enterarray; @@ -1507,10 +1816,7 @@ dataiterator_step(Dataiterator *di) case di_nextattr: di->kv.entry++; di->ddp = data_fetch(di->ddp, &di->kv, di->key); - if (di->kv.eof) - di->state = di_nextkey; - else - di->state = di_nextattr; + di->state = di->kv.eof ? di_nextkey : di_nextattr; break; case di_nextkey: di_nextkey: @@ -1521,7 +1827,7 @@ dataiterator_step(Dataiterator *di) /* FALLTHROUGH */ case di_nextrepodata: di_nextrepodata: - if (di->repodataid && ++di->repodataid < di->repo->nrepodata) + if (!di->keyname && di->repodataid && ++di->repodataid < di->repo->nrepodata) goto di_enterrepodata; /* FALLTHROUGH */ @@ -1622,8 +1928,10 @@ dataiterator_step(Dataiterator *di) /* special solvable attr handling follows */ case di_nextsolvablekey: di_nextsolvablekey: - if (di->keyname || di->key->name == RPM_RPMDBID) - goto di_enterrepodata; + if (di->keyname) + goto di_nextsolvable; + if (di->key->name == RPM_RPMDBID) /* reached end of list? */ + goto di_leavesolvablekey; di->key++; /* FALLTHROUGH */ @@ -1657,23 +1965,27 @@ dataiterator_step(Dataiterator *di) } + /* we have a potential match */ if (di->matcher.match) { + const char *str; /* simple pre-check so that we don't need to stringify */ if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0) if (!datamatcher_checkbasename(&di->matcher, di->kv.str)) continue; - if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)) + /* now stringify so that we can do the matching */ + if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))) { if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)) return 1; continue; } - if (!datamatcher_match(&di->matcher, di->kv.str)) + if (!datamatcher_match(&di->matcher, str)) continue; } else { + /* stringify filelist if requested */ if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->flags & SEARCH_FILES) != 0) repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags); } @@ -1752,8 +2064,7 @@ dataiterator_clonepos(Dataiterator *di, Dataiterator *from) if (from->dupstr && from->dupstr == from->kv.str) { di->dupstrn = from->dupstrn; - di->dupstr = solv_malloc(from->dupstrn); - memcpy(di->dupstr, from->dupstr, di->dupstrn); + di->dupstr = solv_memdup(from->dupstr, from->dupstrn); } } @@ -1848,19 +2159,19 @@ dataiterator_jump_to_solvid(Dataiterator *di, Id solvid) return; } di->repoid = 0; - di->data = di->repo->repodata + di->pool->pos.repodataid; - di->repodataid = 0; - di->solvid = solvid; - di->state = di_enterrepo; - di->flags |= SEARCH_THISSOLVID; - return; + if (!di->pool->pos.repodataid && di->pool->pos.solvid == SOLVID_META) { + solvid = SOLVID_META; /* META pos hack */ + } else { + di->data = di->repo->repodata + di->pool->pos.repodataid; + di->repodataid = 0; + } } - if (solvid > 0) + else if (solvid > 0) { di->repo = di->pool->solvables[solvid].repo; di->repoid = 0; } - else if (di->repoid > 0) + if (di->repoid > 0) { if (!di->pool->urepos) { @@ -1870,7 +2181,8 @@ dataiterator_jump_to_solvid(Dataiterator *di, Id solvid) di->repoid = 1; di->repo = di->pool->repos[di->repoid]; } - di->repodataid = 1; + if (solvid != SOLVID_POS) + di->repodataid = 1; di->solvid = solvid; if (solvid) di->flags |= SEARCH_THISSOLVID; @@ -1894,11 +2206,10 @@ dataiterator_jump_to_repo(Dataiterator *di, Repo *repo) int dataiterator_match(Dataiterator *di, Datamatcher *ma) { - if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)) + const char *str; + if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))) return 0; - if (!ma) - return 1; - return datamatcher_match(ma, di->kv.str); + return ma ? datamatcher_match(ma, str) : 1; } void @@ -1910,9 +2221,7 @@ dataiterator_strdup(Dataiterator *di) return; switch (di->key->type) { - case REPOKEY_TYPE_MD5: - case REPOKEY_TYPE_SHA1: - case REPOKEY_TYPE_SHA256: + case_CHKSUM_TYPES: case REPOKEY_TYPE_DIRSTRARRAY: if (di->kv.num) /* was it stringified into tmp space? */ l = strlen(di->kv.str) + 1; @@ -1928,14 +2237,8 @@ dataiterator_strdup(Dataiterator *di) case REPOKEY_TYPE_DIRSTRARRAY: l = strlen(di->kv.str) + 1; break; - case REPOKEY_TYPE_MD5: - l = SIZEOF_MD5; - break; - case REPOKEY_TYPE_SHA1: - l = SIZEOF_SHA1; - break; - case REPOKEY_TYPE_SHA256: - l = SIZEOF_SHA256; + case_CHKSUM_TYPES: + l = solv_chksum_len(di->key->type); break; case REPOKEY_TYPE_BINARY: l = di->kv.num; @@ -2034,6 +2337,7 @@ repodata_extend_block(Repodata *data, Id start, Id num) return; if (!data->incoreoffset) { + /* this also means that data->attrs is NULL */ data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK); data->start = start; data->end = start + num; @@ -2662,22 +2966,63 @@ repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname, } void -repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle) +repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id handle) { repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1); - data->attriddata[data->attriddatalen++] = ghandle; + data->attriddata[data->attriddatalen++] = handle; data->attriddata[data->attriddatalen++] = 0; } void -repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle) +repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id handle) { repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1); - data->attriddata[data->attriddatalen++] = ghandle; + data->attriddata[data->attriddatalen++] = handle; data->attriddata[data->attriddatalen++] = 0; } void +repodata_set_kv(Repodata *data, Id solvid, Id keyname, Id keytype, KeyValue *kv) +{ + switch (keytype) + { + case REPOKEY_TYPE_ID: + repodata_set_id(data, solvid, keyname, kv->id); + break; + case REPOKEY_TYPE_CONSTANTID: + repodata_set_constantid(data, solvid, keyname, kv->id); + break; + case REPOKEY_TYPE_IDARRAY: + repodata_add_idarray(data, solvid, keyname, kv->id); + break; + case REPOKEY_TYPE_STR: + repodata_set_str(data, solvid, keyname, kv->str); + break; + case REPOKEY_TYPE_VOID: + repodata_set_void(data, solvid, keyname); + break; + case REPOKEY_TYPE_NUM: + repodata_set_num(data, solvid, keyname, SOLV_KV_NUM64(kv)); + break; + case REPOKEY_TYPE_CONSTANT: + repodata_set_constant(data, solvid, keyname, kv->num); + break; + case REPOKEY_TYPE_DIRNUMNUMARRAY: + if (kv->id) + repodata_add_dirnumnum(data, solvid, keyname, kv->id, kv->num, kv->num2); + break; + case REPOKEY_TYPE_DIRSTRARRAY: + repodata_add_dirstr(data, solvid, keyname, kv->id, kv->str); + break; + case_CHKSUM_TYPES: + repodata_set_bin_checksum(data, solvid, keyname, keytype, (const unsigned char *)kv->str); + break; + default: + break; + } +} + +void repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname) { Id *pp, *ap, **app; @@ -2685,6 +3030,11 @@ repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname) ap = *app; if (!ap) return; + if (!keyname) + { + *app = 0; /* delete all attributes */ + return; + } for (; *ap; ap += 2) if (data->keys[*ap].name == keyname) break; @@ -2702,7 +3052,6 @@ repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname) *pp = 0; } -/* XXX: does not work correctly, needs fix in iterators! */ void repodata_unset(Repodata *data, Id solvid, Id keyname) { @@ -2744,9 +3093,15 @@ repodata_swap_attrs(Repodata *data, Id dest, Id src) Id *tmpattrs; if (!data->attrs || dest == src) return; + if (dest < data->start || dest >= data->end) + repodata_extend(data, dest); + if (src < data->start || src >= data->end) + repodata_extend(data, src); tmpattrs = data->attrs[dest - data->start]; data->attrs[dest - data->start] = data->attrs[src - data->start]; data->attrs[src - data->start] = tmpattrs; + if (data->lasthandle == src || data->lasthandle == dest) + data->lasthandle = 0; } @@ -2834,6 +3189,96 @@ data_addblob(struct extdata *xd, unsigned char *blob, int len) /*********************************/ +/* this is to reduct memory usage when internalizing oversized repos */ +static void +compact_attrdata(Repodata *data, int entry, int nentry) +{ + int i; + unsigned int attrdatastart = data->attrdatalen; + unsigned int attriddatastart = data->attriddatalen; + if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024) + return; + for (i = entry; i < nentry; i++) + { + Id v, *attrs = data->attrs[i]; + if (!attrs) + continue; + for (; *attrs; attrs += 2) + { + switch (data->keys[*attrs].type) + { + case REPOKEY_TYPE_STR: + case REPOKEY_TYPE_BINARY: + case_CHKSUM_TYPES: + if ((unsigned int)attrs[1] < attrdatastart) + attrdatastart = attrs[1]; + break; + case REPOKEY_TYPE_DIRSTRARRAY: + for (v = attrs[1]; data->attriddata[v] ; v += 2) + if ((unsigned int)data->attriddata[v + 1] < attrdatastart) + attrdatastart = data->attriddata[v + 1]; + /* FALLTHROUGH */ + case REPOKEY_TYPE_IDARRAY: + case REPOKEY_TYPE_DIRNUMNUMARRAY: + if ((unsigned int)attrs[1] < attriddatastart) + attriddatastart = attrs[1]; + break; + case REPOKEY_TYPE_FIXARRAY: + case REPOKEY_TYPE_FLEXARRAY: + return; + default: + break; + } + } + } +#if 0 + printf("compact_attrdata %d %d\n", entry, nentry); + printf("attrdatastart: %d\n", attrdatastart); + printf("attriddatastart: %d\n", attriddatastart); +#endif + if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024) + return; + for (i = entry; i < nentry; i++) + { + Id v, *attrs = data->attrs[i]; + if (!attrs) + continue; + for (; *attrs; attrs += 2) + { + switch (data->keys[*attrs].type) + { + case REPOKEY_TYPE_STR: + case REPOKEY_TYPE_BINARY: + case_CHKSUM_TYPES: + attrs[1] -= attrdatastart; + break; + case REPOKEY_TYPE_DIRSTRARRAY: + for (v = attrs[1]; data->attriddata[v] ; v += 2) + data->attriddata[v + 1] -= attrdatastart; + /* FALLTHROUGH */ + case REPOKEY_TYPE_IDARRAY: + case REPOKEY_TYPE_DIRNUMNUMARRAY: + attrs[1] -= attriddatastart; + break; + default: + break; + } + } + } + if (attrdatastart) + { + data->attrdatalen -= attrdatastart; + memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen); + data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK); + } + if (attriddatastart) + { + data->attriddatalen -= attriddatastart; + memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id)); + data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK); + } +} + /* internalalize some key into incore/vincore data */ static void @@ -2858,6 +3303,7 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, case REPOKEY_TYPE_VOID: case REPOKEY_TYPE_CONSTANT: case REPOKEY_TYPE_CONSTANTID: + case REPOKEY_TYPE_DELETED: break; case REPOKEY_TYPE_STR: data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1); @@ -2868,9 +3314,18 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, case REPOKEY_TYPE_SHA1: data_addblob(xd, data->attrdata + val, SIZEOF_SHA1); break; + case REPOKEY_TYPE_SHA224: + data_addblob(xd, data->attrdata + val, SIZEOF_SHA224); + break; case REPOKEY_TYPE_SHA256: data_addblob(xd, data->attrdata + val, SIZEOF_SHA256); break; + case REPOKEY_TYPE_SHA384: + data_addblob(xd, data->attrdata + val, SIZEOF_SHA384); + break; + case REPOKEY_TYPE_SHA512: + data_addblob(xd, data->attrdata + val, SIZEOF_SHA512); + break; case REPOKEY_TYPE_NUM: if (val & 0x80000000) { @@ -2919,29 +3374,30 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, sp = schema; kp = data->xattrs[-*ida]; if (!kp) - continue; + continue; /* ignore empty elements */ num++; - for (;*kp; kp += 2) + for (; *kp; kp += 2) *sp++ = *kp; *sp = 0; if (!schemaid) schemaid = repodata_schema2id(data, schema, 1); else if (schemaid != repodata_schema2id(data, schema, 0)) { - pool_debug(data->repo->pool, SOLV_FATAL, "fixarray substructs with different schemas\n"); - exit(1); + pool_debug(data->repo->pool, SOLV_ERROR, "repodata_serialize_key: fixarray substructs with different schemas\n"); + num = 0; + break; } } + data_addid(xd, num); if (!num) break; - data_addid(xd, num); data_addid(xd, schemaid); for (ida = data->attriddata + val; *ida; ida++) { Id *kp = data->xattrs[-*ida]; if (!kp) continue; - for (;*kp; kp += 2) + for (; *kp; kp += 2) repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]); } break; @@ -2973,7 +3429,7 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, break; } default: - pool_debug(data->repo->pool, SOLV_FATAL, "don't know how to handle type %d\n", key->type); + pool_debug(data->repo->pool, SOLV_FATAL, "repodata_serialize_key: don't know how to handle type %d\n", key->type); exit(1); } if (key->storage == KEY_STORAGE_VERTICAL_OFFSET) @@ -2985,21 +3441,66 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore, } } +/* create a circular linked list of all keys that share + * the same keyname */ +static Id * +calculate_keylink(Repodata *data) +{ + int i, j; + Id *link; + Id maxkeyname = 0, *keytable = 0; + link = solv_calloc(data->nkeys, sizeof(Id)); + if (data->nkeys <= 2) + return link; + for (i = 1; i < data->nkeys; i++) + { + Id n = data->keys[i].name; + if (n >= maxkeyname) + { + keytable = solv_realloc2(keytable, n + 128, sizeof(Id)); + memset(keytable + maxkeyname, 0, (n + 128 - maxkeyname) * sizeof(Id)); + maxkeyname = n + 128; + } + j = keytable[n]; + if (j) + link[i] = link[j]; + else + j = i; + link[j] = i; + keytable[n] = i; + } + /* remove links that just point to themselfs */ + for (i = 1; i < data->nkeys; i++) + if (link[i] == i) + link[i] = 0; + solv_free(keytable); + return link; +} + void repodata_internalize(Repodata *data) { Repokey *key, solvkey; Id entry, nentry; - Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen; + Id schemaid, keyid, *schema, *sp, oldschemaid, *keyp, *seen; + Offset *oldincoreoffs = 0; + int schemaidx; unsigned char *dp, *ndp; - int newschema, oldcount; + int neednewschema; struct extdata newincore; struct extdata newvincore; Id solvkeyid; + Id *keylink; + int haveoldkl; if (!data->attrs && !data->xattrs) return; +#if 0 + printf("repodata_internalize %d\n", data->repodataid); + printf(" attr data: %d K\n", data->attrdatalen / 1024); + printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4)); +#endif newvincore.buf = data->vincore; newvincore.len = data->vincorelen; @@ -3023,143 +3524,197 @@ repodata_internalize(Repodata *data) data->mainschema = 0; data->mainschemaoffsets = solv_free(data->mainschemaoffsets); + keylink = calculate_keylink(data); /* join entry data */ /* we start with the meta data, entry -1 */ for (entry = -1; entry < nentry; entry++) { - memset(seen, 0, data->nkeys * sizeof(Id)); - oldschema = 0; + oldschemaid = 0; dp = data->incoredata; if (dp) { dp += entry >= 0 ? data->incoreoffset[entry] : 1; - dp = data_read_id(dp, &oldschema); + dp = data_read_id(dp, &oldschemaid); } + memset(seen, 0, data->nkeys * sizeof(Id)); #if 0 -fprintf(stderr, "oldschema %d\n", oldschema); -fprintf(stderr, "schemata %d\n", data->schemata[oldschema]); +fprintf(stderr, "oldschemaid %d\n", oldschemaid); +fprintf(stderr, "schemata %d\n", data->schemata[oldschemaid]); fprintf(stderr, "schemadata %p\n", data->schemadata); #endif - /* seen: -1: old data 0: skipped >0: id + 1 */ - newschema = 0; - oldcount = 0; + + /* seen: -1: old data, 0: skipped, >0: id + 1 */ + neednewschema = 0; sp = schema; - for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++) + haveoldkl = 0; + for (keyp = data->schemadata + data->schemata[oldschemaid]; *keyp; keyp++) { if (seen[*keyp]) { - pool_debug(data->repo->pool, SOLV_FATAL, "Inconsistent old data (key occured twice).\n"); - exit(1); + /* oops, should not happen */ + neednewschema = 1; + continue; } - seen[*keyp] = -1; + seen[*keyp] = -1; /* use old marker */ *sp++ = *keyp; - oldcount++; + if (keylink[*keyp]) + haveoldkl = 1; /* potential keylink conflict */ } - if (entry >= 0) - keyp = data->attrs ? data->attrs[entry] : 0; - else + + /* strip solvables key */ + if (entry < 0 && solvkeyid && seen[solvkeyid]) { - /* strip solvables key */ *sp = 0; for (sp = keyp = schema; *sp; sp++) if (*sp != solvkeyid) *keyp++ = *sp; - else - oldcount--; sp = keyp; seen[solvkeyid] = 0; - keyp = data->xattrs ? data->xattrs[1] : 0; + neednewschema = 1; } + + /* add new entries */ + if (entry >= 0) + keyp = data->attrs ? data->attrs[entry] : 0; + else + keyp = data->xattrs ? data->xattrs[1] : 0; if (keyp) for (; *keyp; keyp += 2) { if (!seen[*keyp]) { - newschema = 1; + neednewschema = 1; *sp++ = *keyp; + if (haveoldkl && keylink[*keyp]) /* this should be pretty rare */ + { + Id kl; + for (kl = keylink[*keyp]; kl != *keyp; kl = keylink[kl]) + if (seen[kl] == -1) + { + /* replacing old key kl, remove from schema and seen */ + Id *osp; + for (osp = schema; osp < sp; osp++) + if (*osp == kl) + { + memmove(osp, osp + 1, (sp - osp) * sizeof(Id)); + sp--; + seen[kl] = 0; + break; + } + } + } } seen[*keyp] = keyp[1] + 1; } + + /* add solvables key if needed */ if (entry < 0 && data->end != data->start) { - *sp++ = solvkeyid; - newschema = 1; + *sp++ = solvkeyid; /* always last in schema */ + neednewschema = 1; } + + /* commit schema */ *sp = 0; - if (newschema) + if (neednewschema) /* Ideally we'd like to sort the new schema here, to ensure - schema equality independend of the ordering. We can't do that - yet. For once see below (old ids need to come before new ids). - An additional difficulty is that we also need to move - the values with the keys. */ + schema equality independend of the ordering. */ schemaid = repodata_schema2id(data, schema, 1); else - schemaid = oldschema; + schemaid = oldschemaid; + if (entry < 0) + { + data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id)); + data->mainschema = schemaid; + } + + /* find offsets in old incore data */ + if (oldschemaid) + { + Id *lastneeded = 0; + for (sp = data->schemadata + data->schemata[oldschemaid]; *sp; sp++) + if (seen[*sp] == -1) + lastneeded = sp + 1; + if (lastneeded) + { + if (!oldincoreoffs) + oldincoreoffs = solv_malloc2(data->nkeys, 2 * sizeof(Offset)); + for (sp = data->schemadata + data->schemata[oldschemaid]; sp != lastneeded; sp++) + { + /* Skip the data associated with this old key. */ + key = data->keys + *sp; + ndp = dp; + if (key->storage == KEY_STORAGE_VERTICAL_OFFSET) + { + ndp = data_skip(ndp, REPOKEY_TYPE_ID); + ndp = data_skip(ndp, REPOKEY_TYPE_ID); + } + else if (key->storage == KEY_STORAGE_INCORE) + ndp = data_skip_key(data, ndp, key); + oldincoreoffs[*sp * 2] = dp - data->incoredata; + oldincoreoffs[*sp * 2 + 1] = ndp - dp; + dp = ndp; + } + } + } + + /* just copy over the complete old entry (including the schemaid) if there was no new data */ + if (entry >= 0 && !neednewschema && oldschemaid && (!data->attrs || !data->attrs[entry]) && dp) + { + ndp = data->incoredata + data->incoreoffset[entry]; + data->incoreoffset[entry] = newincore.len; + data_addblob(&newincore, ndp, dp - ndp); + goto entrydone; + } /* Now create data blob. We walk through the (possibly new) schema and either copy over old data, or insert the new. */ - /* XXX Here we rely on the fact that the (new) schema has the form - o1 o2 o3 o4 ... | n1 n2 n3 ... - (oX being the old keyids (possibly overwritten), and nX being - the new keyids). This rules out sorting the keyids in order - to ensure a small schema count. */ if (entry >= 0) data->incoreoffset[entry] = newincore.len; data_addid(&newincore, schemaid); - if (entry == -1) - { - data->mainschema = schemaid; - data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id)); - } - keypstart = data->schemadata + data->schemata[schemaid]; - for (keyp = keypstart; *keyp; keyp++) + + /* we don't use a pointer to the schemadata here as repodata_serialize_key + * may call repodata_schema2id() which might realloc our schemadata */ + for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++) { - if (entry == -1) - data->mainschemaoffsets[keyp - keypstart] = newincore.len; - if (*keyp == solvkeyid) - { - /* add flexarray entry count */ - data_addid(&newincore, data->end - data->start); - break; - } - key = data->keys + *keyp; -#if 0 - fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, pool_id2str(data->repo->pool, key->name), pool_id2str(data->repo->pool, key->type)); -#endif - ndp = dp; - if (oldcount) + if (entry < 0) { - /* Skip the data associated with this old key. */ - if (key->storage == KEY_STORAGE_VERTICAL_OFFSET) + data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len; + if (keyid == solvkeyid) { - ndp = data_skip(dp, REPOKEY_TYPE_ID); - ndp = data_skip(ndp, REPOKEY_TYPE_ID); + /* add flexarray entry count */ + data_addid(&newincore, data->end - data->start); + break; /* always the last entry */ } - else if (key->storage == KEY_STORAGE_INCORE) - ndp = data_skip_key(data, dp, key); - oldcount--; } - if (seen[*keyp] == -1) + if (seen[keyid] == -1) { - /* If this key was an old one _and_ was not overwritten with - a different value copy over the old value (we skipped it - above). */ - if (dp != ndp) - data_addblob(&newincore, dp, ndp - dp); - seen[*keyp] = 0; + if (oldincoreoffs[keyid * 2 + 1]) + data_addblob(&newincore, data->incoredata + oldincoreoffs[keyid * 2], oldincoreoffs[keyid * 2 + 1]); } - else if (seen[*keyp]) + else if (seen[keyid]) + repodata_serialize_key(data, &newincore, &newvincore, schema, data->keys + keyid, seen[keyid] - 1); + } + +entrydone: + /* free memory */ + if (entry >= 0 && data->attrs) + { + if (data->attrs[entry]) + data->attrs[entry] = solv_free(data->attrs[entry]); + if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry) { - /* Otherwise we have a new value. Parse it into the internal - form. */ - repodata_serialize_key(data, &newincore, &newvincore, - schema, key, seen[*keyp] - 1); + compact_attrdata(data, entry + 1, nentry); /* try to free some memory */ +#if 0 + printf(" attr data: %d K\n", data->attrdatalen / 1024); + printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4)); + printf(" incore data: %d K\n", newincore.len / 1024); + printf(" sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024); + /* malloc_stats(); */ +#endif } - dp = ndp; } - if (entry >= 0 && data->attrs && data->attrs[entry]) - data->attrs[entry] = solv_free(data->attrs[entry]); } /* free all xattrs */ for (entry = 0; entry < data->nxattrs; entry++) @@ -3173,6 +3728,8 @@ fprintf(stderr, "schemadata %p\n", data->schemadata); data->lastdatalen = 0; solv_free(schema); solv_free(seen); + solv_free(keylink); + solv_free(oldincoreoffs); repodata_free_schemahash(data); solv_free(data->incoredata); @@ -3180,7 +3737,6 @@ fprintf(stderr, "schemadata %p\n", data->schemadata); data->incoredatalen = newincore.len; data->incoredatafree = 0; - solv_free(data->vincore); data->vincore = newvincore.buf; data->vincorelen = newvincore.len; @@ -3191,6 +3747,10 @@ fprintf(stderr, "schemadata %p\n", data->schemadata); data->attrdatalen = 0; data->attriddatalen = 0; data->attrnum64datalen = 0; +#if 0 + printf("repodata_internalize %d done\n", data->repodataid); + printf(" incore data: %d K\n", data->incoredatalen / 1024); +#endif } void @@ -3203,13 +3763,15 @@ repodata_disable_paging(Repodata *data) } } +/* call the pool's loadcallback to load a stub repodata */ static void -repodata_load_stub(Repodata *data) +repodata_stub_loader(Repodata *data) { Repo *repo = data->repo; Pool *pool = repo->pool; int r, i; - struct _Pool_tmpspace oldtmpspace; + struct s_Pool_tmpspace oldtmpspace; + Datapos oldpos; if (!pool->loadcallback) { @@ -3218,21 +3780,53 @@ repodata_load_stub(Repodata *data) } data->state = REPODATA_LOADING; - /* save tmp space */ + /* save tmp space and pos */ oldtmpspace = pool->tmpspace; memset(&pool->tmpspace, 0, sizeof(pool->tmpspace)); + oldpos = pool->pos; r = pool->loadcallback(pool, data, pool->loadcallbackdata); - /* restore tmp space */ + /* restore tmp space and pos */ for (i = 0; i < POOL_TMPSPACEBUF; i++) solv_free(pool->tmpspace.buf[i]); pool->tmpspace = oldtmpspace; + if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid) + memset(&oldpos, 0, sizeof(oldpos)); + pool->pos = oldpos; data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR; } -void +static inline void +repodata_add_stubkey(Repodata *data, Id keyname, Id keytype) +{ + Repokey xkey; + + xkey.name = keyname; + xkey.type = keytype; + xkey.storage = KEY_STORAGE_INCORE; + xkey.size = 0; + repodata_key2id(data, &xkey, 1); +} + +static Repodata * +repodata_add_stub(Repodata **datap) +{ + Repodata *data = *datap; + Repo *repo = data->repo; + Id repodataid = data - repo->repodata; + Repodata *sdata = repo_add_repodata(repo, 0); + data = repo->repodata + repodataid; + if (data->end > data->start) + repodata_extend_block(sdata, data->start, data->end - data->start); + sdata->state = REPODATA_STUB; + sdata->loadcallback = repodata_stub_loader; + *datap = data; + return sdata; +} + +Repodata * repodata_create_stubs(Repodata *data) { Repo *repo = data->repo; @@ -3242,38 +3836,26 @@ repodata_create_stubs(Repodata *data) Dataiterator di; Id xkeyname = 0; int i, cnt = 0; - int repodataid; - int datastart, dataend; - repodataid = data - repo->repodata; - datastart = data->start; - dataend = data->end; dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0); while (dataiterator_step(&di)) - { - if (di.data - repo->repodata != repodataid) - continue; + if (di.data == data) cnt++; - } dataiterator_free(&di); if (!cnt) - return; + return data; stubdataids = solv_calloc(cnt, sizeof(*stubdataids)); for (i = 0; i < cnt; i++) { - sdata = repo_add_repodata(repo, 0); - if (dataend > datastart) - repodata_extend_block(sdata, datastart, dataend - datastart); + sdata = repodata_add_stub(&data); stubdataids[i] = sdata - repo->repodata; - sdata->state = REPODATA_STUB; - sdata->loadcallback = repodata_load_stub; } i = 0; dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0); sdata = 0; while (dataiterator_step(&di)) { - if (di.data - repo->repodata != repodataid) + if (di.data != data) continue; if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents) { @@ -3282,55 +3864,34 @@ repodata_create_stubs(Repodata *data) xkeyname = 0; continue; } - switch (di.key->type) + repodata_set_kv(sdata, SOLVID_META, di.key->name, di.key->type, &di.kv); + if (di.key->name == REPOSITORY_KEYS && di.key->type == REPOKEY_TYPE_IDARRAY) { - case REPOKEY_TYPE_ID: - repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id); - break; - case REPOKEY_TYPE_CONSTANTID: - repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id); - break; - case REPOKEY_TYPE_STR: - repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str); - break; - case REPOKEY_TYPE_VOID: - repodata_set_void(sdata, SOLVID_META, di.key->name); - break; - case REPOKEY_TYPE_NUM: - repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv)); - break; - case REPOKEY_TYPE_MD5: - case REPOKEY_TYPE_SHA1: - case REPOKEY_TYPE_SHA256: - repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str); - break; - case REPOKEY_TYPE_IDARRAY: - repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id); - if (di.key->name == REPOSITORY_KEYS) + if (!xkeyname) { - Repokey xkey; - - if (!xkeyname) - { - if (!di.kv.eof) - xkeyname = di.kv.id; - continue; - } - xkey.name = xkeyname; - xkey.type = di.kv.id; - xkey.storage = KEY_STORAGE_INCORE; - xkey.size = 0; - repodata_key2id(sdata, &xkey, 1); - xkeyname = 0; + if (!di.kv.eof) + xkeyname = di.kv.id; + } + else + { + repodata_add_stubkey(sdata, xkeyname, di.kv.id); + if (xkeyname == SOLVABLE_FILELIST) + repodata_set_filelisttype(sdata, REPODATA_FILELIST_EXTENSION); + xkeyname = 0; } - default: - break; } } dataiterator_free(&di); for (i = 0; i < cnt; i++) repodata_internalize(repo->repodata + stubdataids[i]); solv_free(stubdataids); + return data; +} + +void +repodata_set_filelisttype(Repodata *data, int type) +{ + data->filelisttype = type; } unsigned int @@ -3339,6 +3900,3 @@ repodata_memused(Repodata *data) return data->incoredatalen + data->vincorelen; } -/* -vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4: -*/