/*
- * Copyright (c) 2007, Novell Inc.
+ * Copyright (c) 2018, SUSE LLC.
*
* This program is licensed under the BSD license, read LICENSE.BSD
* for further information
#include "repopack.h"
#include "repopage.h"
+#ifdef _WIN32
+ #include "strfncs.h"
+#endif
+
#define REPODATA_BLOCK 255
static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);
solv_free(data->attrnum64data);
solv_free(data->dircache);
+
+ repodata_free_filelistfilter(data);
}
void
cid = schematahash[h];
if (cid)
{
- if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
+ if ((data->schemata[cid] + len <= data->schemadatalen) &&
+ !memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
return cid;
/* cache conflict, do a slow search */
for (cid = 1; cid < data->nschemata; cid++)
- if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
+ if ((data->schemata[cid] + len <= data->schemadatalen) &&
+ !memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
return cid;
}
/* a new one */
#endif
const char *dire;
- parent = 0;
if (!*dir)
- return 0;
+ return data->dirpool.ndirs ? 0 : dirpool_add_dir(&data->dirpool, 0, 0, create);
while (*dir == '/' && dir[1] == '/')
dir++;
if (*dir == '/' && !dir[1])
- {
- if (data->dirpool.ndirs)
- return 1;
- return dirpool_add_dir(&data->dirpool, 0, 1, create);
- }
+ return data->dirpool.ndirs ? 1 : dirpool_add_dir(&data->dirpool, 0, 1, create);
+ parent = 0;
#ifdef DIRCACHE_SIZE
dirs = dir;
if (data->dircache)
if (!did)
return suf ? suf : "";
+ if (did == 1 && !suf)
+ return "/";
parent = did;
while (parent)
{
if (off >= data->lastverticaloffset)
{
off -= data->lastverticaloffset;
- if (off + len > data->vincorelen)
+ if ((unsigned int)off + len > data->vincorelen)
return 0;
return data->vincore + off;
}
- if (off + len > key->size)
+ if ((unsigned int)off + len > key->size)
return 0;
/* we now have the offset, go into vertical */
off += data->verticaloffset[key - data->keys];
return 0;
}
-static int
-load_repodata(Repodata *data)
+void
+repodata_load(Repodata *data)
{
+ if (data->state != REPODATA_STUB)
+ return;
if (data->loadcallback)
+ data->loadcallback(data);
+ else
+ data->state = REPODATA_ERROR;
+}
+
+static int
+maybe_load_repodata_stub(Repodata *data, Id keyname)
+{
+ if (data->state != REPODATA_STUB)
{
- data->loadcallback(data);
- if (data->state == REPODATA_AVAILABLE)
- return 1;
+ data->state = REPODATA_ERROR;
+ return 0;
}
- data->state = REPODATA_ERROR;
- return 0;
+ if (keyname)
+ {
+ int i;
+ for (i = 1; i < data->nkeys; i++)
+ if (keyname == data->keys[i].name)
+ break;
+ if (i == data->nkeys)
+ return 0;
+ }
+ repodata_load(data);
+ return data->state == REPODATA_AVAILABLE ? 1 : 0;
}
static inline int
{
if (keyname && !repodata_precheck_keyname(data, keyname))
return 0; /* do not bother... */
- switch(data->state)
- {
- case REPODATA_STUB:
- if (keyname)
- {
- int i;
- for (i = 1; i < data->nkeys; i++)
- if (keyname == data->keys[i].name)
- break;
- if (i == data->nkeys)
- return 0;
- }
- return load_repodata(data);
- case REPODATA_ERROR:
- return 0;
- case REPODATA_AVAILABLE:
- case REPODATA_LOADING:
- return 1;
- default:
- data->state = REPODATA_ERROR;
- return 0;
- }
+ if (data->state == REPODATA_AVAILABLE || data->state == REPODATA_LOADING)
+ return 1;
+ if (data->state == REPODATA_ERROR)
+ return 0;
+ return maybe_load_repodata_stub(data, keyname);
}
static inline unsigned char *
* data lookup
*/
-static inline unsigned char *
+static unsigned char *
find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
{
unsigned char *dp;
return 0;
if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
return dp; /* no need to forward... */
+ if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
+ return 0; /* get_data will not work, no need to forward */
dp = forward_to_key(data, *kp, keyp, dp);
if (!dp)
return 0;
return get_data(data, key, &dp, 0);
}
+static const Id *
+repodata_lookup_schemakeys(Repodata *data, Id solvid)
+{
+ Id schema;
+ if (!maybe_load_repodata(data, 0))
+ return 0;
+ if (!solvid2data(data, solvid, &schema))
+ return 0;
+ return data->schemadata + data->schemata[schema];
+}
+
+static Id *
+alloc_keyskip()
+{
+ Id *keyskip = solv_calloc(3 + 256, sizeof(Id));
+ keyskip[0] = 256;
+ keyskip[1] = keyskip[2] = 1;
+ return keyskip;
+}
+
+Id *
+repodata_fill_keyskip(Repodata *data, Id solvid, Id *keyskip)
+{
+ const Id *keyp;
+ Id maxkeyname, value;
+ keyp = repodata_lookup_schemakeys(data, solvid);
+ if (!keyp)
+ return keyskip; /* no keys for this solvid */
+ if (!keyskip)
+ keyskip = alloc_keyskip();
+ maxkeyname = keyskip[0];
+ value = keyskip[1] + data->repodataid;
+ for (; *keyp; keyp++)
+ {
+ Id keyname = data->keys[*keyp].name;
+ if (keyname >= maxkeyname)
+ {
+ int newmax = (keyname | 255) + 1;
+ keyskip = solv_realloc2(keyskip, 3 + newmax, sizeof(Id));
+ memset(keyskip + (3 + maxkeyname), 0, (newmax - maxkeyname) * sizeof(Id));
+ keyskip[0] = maxkeyname = newmax;
+ }
+ keyskip[3 + keyname] = value;
+ }
+ return keyskip;
+}
+
Id
repodata_lookup_type(Repodata *data, Id solvid, Id keyname)
{
return pool_id2str(data->repo->pool, id);
}
-int
-repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long *value)
+unsigned long long
+repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long notfound)
{
unsigned char *dp;
Repokey *key;
unsigned int high, low;
- *value = 0;
dp = find_key_data(data, solvid, keyname, &key);
if (!dp)
- return 0;
+ return notfound;
switch (key->type)
{
case REPOKEY_TYPE_NUM:
data_read_num64(dp, &low, &high);
- *value = (unsigned long long)high << 32 | low;
- return 1;
- case REPOKEY_TYPE_U32:
- data_read_u32(dp, &low);
- *value = low;
- return 1;
+ return (unsigned long long)high << 32 | low;
case REPOKEY_TYPE_CONSTANT:
- *value = key->size;
- return 1;
+ return key->size;
default:
- return 0;
+ return notfound;
}
}
int
repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
{
- Id schema;
- Id *keyp;
- unsigned char *dp;
-
- if (!maybe_load_repodata(data, keyname))
- return 0;
- dp = solvid2data(data, solvid, &schema);
- if (!dp)
- return 0;
- /* can't use find_key_data as we need to test the type */
- for (keyp = data->schemadata + data->schemata[schema]; *keyp; keyp++)
- if (data->keys[*keyp].name == keyname && data->keys[*keyp].type == REPOKEY_TYPE_VOID)
- return 1;
- return 0;
+ return repodata_lookup_type(data, solvid, keyname) == REPOKEY_TYPE_VOID ? 1 : 0;
}
const unsigned char *
dp = find_key_data(data, solvid, keyname, &key);
if (!dp)
return 0;
- if (!(key->type == REPOKEY_TYPE_MD5 || key->type == REPOKEY_TYPE_SHA1 || key->type == REPOKEY_TYPE_SHA256))
- return 0;
+ switch (key->type)
+ {
+ case_CHKSUM_TYPES:
+ break;
+ default:
+ return 0;
+ }
*typep = key->type;
return dp;
}
dp = find_key_data(data, solvid, keyname, &key);
if (!dp)
return 0;
- if (key->type != REPOKEY_TYPE_IDARRAY)
- return 0;
- for (;;)
+ switch (key->type)
{
- dp = data_read_ideof(dp, &id, &eof);
+ case REPOKEY_TYPE_CONSTANTID:
+ queue_push(q, key->size);
+ break;
+ case REPOKEY_TYPE_ID:
+ dp = data_read_id(dp, &id);
queue_push(q, id);
- if (eof)
+ break;
+ case REPOKEY_TYPE_IDARRAY:
+ for (;;)
+ {
+ dp = data_read_ideof(dp, &id, &eof);
+ queue_push(q, id);
+ if (eof)
+ break;
+ }
+ break;
+ default:
+ return 0;
+ }
+ return 1;
+}
+
+const void *
+repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp)
+{
+ unsigned char *dp;
+ Repokey *key;
+ Id len;
+
+ dp = find_key_data(data, solvid, keyname, &key);
+ if (!dp || key->type != REPOKEY_TYPE_BINARY)
+ {
+ *lenp = 0;
+ return 0;
+ }
+ dp = data_read_id(dp, &len);
+ *lenp = len;
+ return dp;
+}
+
+unsigned int
+repodata_lookup_count(Repodata *data, Id solvid, Id keyname)
+{
+ unsigned char *dp;
+ Repokey *key;
+ unsigned int cnt = 0;
+
+ dp = find_key_data(data, solvid, keyname, &key);
+ if (!dp)
+ return 0;
+ switch (key->type)
+ {
+ case REPOKEY_TYPE_IDARRAY:
+ case REPOKEY_TYPE_REL_IDARRAY:
+ for (cnt = 1; (*dp & 0xc0) != 0; dp++)
+ if ((*dp & 0xc0) == 0x40)
+ cnt++;
+ return cnt;
+ case REPOKEY_TYPE_FIXARRAY:
+ case REPOKEY_TYPE_FLEXARRAY:
+ data_read_id(dp, (int *)&cnt);
+ return cnt;
+ case REPOKEY_TYPE_DIRSTRARRAY:
+ for (;;)
+ {
+ cnt++;
+ while (*dp & 0x80)
+ dp++;
+ if (!(*dp++ & 0x40))
+ return cnt;
+ dp += strlen((const char *)dp) + 1;
+ }
+ case REPOKEY_TYPE_DIRNUMNUMARRAY:
+ for (;;)
+ {
+ cnt++;
+ while (*dp++ & 0x80)
+ ;
+ while (*dp++ & 0x80)
+ ;
+ while (*dp & 0x80)
+ dp++;
+ if (!(*dp++ & 0x40))
+ return cnt;
+ }
+ default:
break;
}
return 1;
}
+/* highly specialized function to speed up fileprovides adding.
+ * - repodata must be available
+ * - solvid must be >= data->start and < data->end
+ * - returns NULL is not found, a "" entry if wrong type
+ * - also returns wrong type for REPOKEY_TYPE_DELETED
+ */
+const unsigned char *
+repodata_lookup_packed_dirstrarray(Repodata *data, Id solvid, Id keyname)
+{
+ static unsigned char wrongtype[2] = { 0x00 /* dir id 0 */, 0 /* "" */ };
+ unsigned char *dp;
+ Id schema, *keyp, *kp;
+ Repokey *key;
+
+ if (!data->incoredata || !data->incoreoffset[solvid - data->start])
+ return 0;
+ dp = data->incoredata + data->incoreoffset[solvid - data->start];
+ dp = data_read_id(dp, &schema);
+ keyp = data->schemadata + data->schemata[schema];
+ for (kp = keyp; *kp; kp++)
+ if (data->keys[*kp].name == keyname)
+ break;
+ if (!*kp)
+ return 0;
+ key = data->keys + *kp;
+ if (key->type != REPOKEY_TYPE_DIRSTRARRAY)
+ return wrongtype;
+ dp = forward_to_key(data, *kp, keyp, dp);
+ if (key->storage == KEY_STORAGE_INCORE)
+ return dp;
+ if (key->storage == KEY_STORAGE_VERTICAL_OFFSET && dp)
+ {
+ Id off, len;
+ dp = data_read_id(dp, &off);
+ data_read_id(dp, &len);
+ return get_vertical_data(data, key, off, len);
+ }
+ return 0;
+}
+
+/* id translation functions */
+
Id
repodata_globalize_id(Repodata *data, Id id, int create)
{
Id
repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create)
{
+ const char *s;
if (!id || !data || !fromdata)
return id;
- if (!data->localpool || !fromdata->localpool)
+ if (data == fromdata || (!data->localpool && !fromdata->localpool))
+ return id;
+ if (fromdata->localpool)
+ s = stringpool_id2str(&fromdata->spool, id);
+ else
+ s = pool_id2str(data->repo->pool, id);
+ if (data->localpool)
+ return stringpool_str2id(&data->spool, s, create);
+ else
+ return pool_str2id(data->repo->pool, s, create);
+}
+
+Id
+repodata_translate_dir_slow(Repodata *data, Repodata *fromdata, Id dir, int create, Id *cache)
+{
+ Id parent, compid;
+ if (!dir)
{
- if (fromdata->localpool)
- id = repodata_globalize_id(fromdata, id, create);
- if (data->localpool)
- id = repodata_localize_id(data, id, create);
- return id;
+ /* make sure that the dirpool has an entry */
+ if (create && !data->dirpool.ndirs)
+ dirpool_add_dir(&data->dirpool, 0, 0, create);
+ return 0;
+ }
+ parent = dirpool_parent(&fromdata->dirpool, dir);
+ if (parent)
+ {
+ if (!(parent = repodata_translate_dir(data, fromdata, parent, create, cache)))
+ return 0;
+ }
+ compid = dirpool_compid(&fromdata->dirpool, dir);
+ if (compid > 1 && (data->localpool || fromdata->localpool))
+ {
+ if (!(compid = repodata_translate_id(data, fromdata, compid, create)))
+ return 0;
}
- /* localpool is set in both data and fromdata */
- return stringpool_str2id(&data->spool, stringpool_id2str(&fromdata->spool, id), create);
+ if (!(compid = dirpool_add_dir(&data->dirpool, parent, compid, create)))
+ return 0;
+ if (cache)
+ {
+ cache[(dir & 255) * 2] = dir;
+ cache[(dir & 255) * 2 + 1] = compid;
+ }
+ return compid;
}
-Id
-repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id voidid)
+/************************************************************************
+ * uninternalized lookup / search
+ */
+
+static void
+data_fetch_uninternalized(Repodata *data, Repokey *key, Id value, KeyValue *kv)
+{
+ Id *array;
+ kv->eof = 1;
+ switch (key->type)
+ {
+ case REPOKEY_TYPE_STR:
+ kv->str = (const char *)data->attrdata + value;
+ return;
+ case REPOKEY_TYPE_CONSTANT:
+ kv->num2 = 0;
+ kv->num = key->size;
+ return;
+ case REPOKEY_TYPE_CONSTANTID:
+ kv->id = key->size;
+ return;
+ case REPOKEY_TYPE_NUM:
+ kv->num2 = 0;
+ kv->num = value;
+ if (value & 0x80000000)
+ {
+ kv->num = (unsigned int)data->attrnum64data[value ^ 0x80000000];
+ kv->num2 = (unsigned int)(data->attrnum64data[value ^ 0x80000000] >> 32);
+ }
+ return;
+ case_CHKSUM_TYPES:
+ kv->num = 0; /* not stringified */
+ kv->str = (const char *)data->attrdata + value;
+ return;
+ case REPOKEY_TYPE_BINARY:
+ kv->str = (const char *)data_read_id(data->attrdata + value, (Id *)&kv->num);
+ return;
+ case REPOKEY_TYPE_IDARRAY:
+ array = data->attriddata + (value + kv->entry);
+ kv->id = array[0];
+ kv->eof = array[1] ? 0 : 1;
+ return;
+ case REPOKEY_TYPE_DIRSTRARRAY:
+ kv->num = 0; /* not stringified */
+ array = data->attriddata + (value + kv->entry * 2);
+ kv->id = array[0];
+ kv->str = (const char *)data->attrdata + array[1];
+ kv->eof = array[2] ? 0 : 1;
+ return;
+ case REPOKEY_TYPE_DIRNUMNUMARRAY:
+ array = data->attriddata + (value + kv->entry * 3);
+ kv->id = array[0];
+ kv->num = array[1];
+ kv->num2 = array[2];
+ kv->eof = array[3] ? 0 : 1;
+ return;
+ case REPOKEY_TYPE_FIXARRAY:
+ case REPOKEY_TYPE_FLEXARRAY:
+ array = data->attriddata + (value + kv->entry);
+ kv->id = array[0]; /* the handle */
+ kv->eof = array[1] ? 0 : 1;
+ return;
+ default:
+ kv->id = value;
+ return;
+ }
+}
+
+Repokey *
+repodata_lookup_kv_uninternalized(Repodata *data, Id solvid, Id keyname, KeyValue *kv)
{
Id *ap;
- if (!data->attrs)
+ if (!data->attrs || solvid < data->start || solvid >= data->end)
return 0;
ap = data->attrs[solvid - data->start];
if (!ap)
return 0;
for (; *ap; ap += 2)
{
- if (data->keys[*ap].name != keyname)
+ Repokey *key = data->keys + *ap;
+ if (key->name != keyname)
continue;
- if (data->keys[*ap].type == REPOKEY_TYPE_VOID)
- return voidid;
- if (data->keys[*ap].type == REPOKEY_TYPE_ID)
- return ap[1];
- return 0;
+ data_fetch_uninternalized(data, key, ap[1], kv);
+ return key;
}
return 0;
}
+void
+repodata_search_uninternalized(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
+{
+ Id *ap;
+ int stop;
+ Solvable *s;
+ KeyValue kv;
+
+ if (!data->attrs || solvid < data->start || solvid >= data->end)
+ return;
+ ap = data->attrs[solvid - data->start];
+ if (!ap)
+ return;
+ for (; *ap; ap += 2)
+ {
+ Repokey *key = data->keys + *ap;
+ if (keyname && key->name != keyname)
+ continue;
+ s = solvid > 0 ? data->repo->pool->solvables + solvid : 0;
+ kv.entry = 0;
+ do
+ {
+ data_fetch_uninternalized(data, key, ap[1], &kv);
+ stop = callback(cbdata, s, data, key, &kv);
+ kv.entry++;
+ }
+ while (!kv.eof && !stop);
+ if (keyname || stop > SEARCH_NEXT_KEY)
+ return;
+ }
+}
/************************************************************************
* data search
*/
-int
+const char *
repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
{
switch (key->type)
kv->str = stringpool_id2str(&data->spool, kv->id);
else
kv->str = pool_id2str(pool, kv->id);
- if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE)
+ if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE && (key->name == SOLVABLE_NAME || key->type == REPOKEY_TYPE_IDARRAY))
{
const char *s;
for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
if (*s == ':' && s > kv->str)
kv->str = s + 1;
}
- return 1;
+ return kv->str;
case REPOKEY_TYPE_STR:
- return 1;
+ return kv->str;
case REPOKEY_TYPE_DIRSTRARRAY:
if (!(flags & SEARCH_FILES))
- return 1; /* match just the basename */
+ return kv->str; /* match just the basename */
if (kv->num)
- return 1; /* already stringified */
+ return kv->str; /* already stringified */
/* Put the full filename into kv->str. */
kv->str = repodata_dir2str(data, kv->id, kv->str);
kv->num = 1; /* mark stringification */
- return 1;
- case REPOKEY_TYPE_MD5:
- case REPOKEY_TYPE_SHA1:
- case REPOKEY_TYPE_SHA256:
+ return kv->str;
+ case_CHKSUM_TYPES:
if (!(flags & SEARCH_CHECKSUMS))
return 0; /* skip em */
if (kv->num)
- return 1; /* already stringified */
+ return kv->str; /* already stringified */
kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
kv->num = 1; /* mark stringification */
- return 1;
+ return kv->str;
default:
return 0;
}
}
+/* this is an internal hack to pass the parent kv to repodata_search_keyskip */
struct subschema_data {
- Solvable *s;
void *cbdata;
+ Id solvid;
KeyValue *parent;
};
+void
+repodata_search_arrayelement(Repodata *data, Id solvid, Id keyname, int flags, KeyValue *kv, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
+{
+ repodata_search_keyskip(data, solvid, keyname, flags | SEARCH_SUBSCHEMA, (Id *)kv, callback, cbdata);
+}
+
+static int
+repodata_search_array(Repodata *data, Id solvid, Id keyname, int flags, Repokey *key, KeyValue *kv, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
+{
+ Solvable *s = solvid > 0 ? data->repo->pool->solvables + solvid : 0;
+ unsigned char *dp = (unsigned char *)kv->str;
+ int stop;
+ Id schema = 0;
+
+ if (!dp || kv->entry != -1)
+ return 0;
+ while (++kv->entry < (int)kv->num)
+ {
+ if (kv->entry)
+ dp = data_skip_schema(data, dp, schema);
+ if (kv->entry == 0 || key->type == REPOKEY_TYPE_FLEXARRAY)
+ dp = data_read_id(dp, &schema);
+ kv->id = schema;
+ kv->str = (const char *)dp;
+ kv->eof = kv->entry == kv->num - 1 ? 1 : 0;
+ stop = callback(cbdata, s, data, key, kv);
+ if (stop && stop != SEARCH_ENTERSUB)
+ return stop;
+ if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
+ repodata_search_keyskip(data, solvid, keyname, flags | SEARCH_SUBSCHEMA, (Id *)kv, callback, cbdata);
+ }
+ if ((flags & SEARCH_ARRAYSENTINEL) != 0)
+ {
+ if (kv->entry)
+ dp = data_skip_schema(data, dp, schema);
+ kv->id = 0;
+ kv->str = (const char *)dp;
+ kv->eof = 2;
+ return callback(cbdata, s, data, key, kv);
+ }
+ return 0;
+}
+
/* search a specific repodata */
void
-repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
+repodata_search_keyskip(Repodata *data, Id solvid, Id keyname, int flags, Id *keyskip, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
{
Id schema;
Repokey *key;
if (!maybe_load_repodata(data, keyname))
return;
- if (solvid == SOLVID_SUBSCHEMA)
+ if ((flags & SEARCH_SUBSCHEMA) != 0)
{
- struct subschema_data *subd = cbdata;
- cbdata = subd->cbdata;
- s = subd->s;
- schema = subd->parent->id;
- dp = (unsigned char *)subd->parent->str;
- kv.parent = subd->parent;
+ flags ^= SEARCH_SUBSCHEMA;
+ kv.parent = (KeyValue *)keyskip;
+ keyskip = 0;
+ schema = kv.parent->id;
+ dp = (unsigned char *)kv.parent->str;
}
else
{
dp = solvid2data(data, solvid, &schema);
if (!dp)
return;
- s = data->repo->pool->solvables + solvid;
kv.parent = 0;
}
+ s = solvid > 0 ? data->repo->pool->solvables + solvid : 0;
keyp = data->schemadata + data->schemata[schema];
if (keyname)
{
{
stop = 0;
key = data->keys + keyid;
- ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
+ ddp = get_data(data, key, &dp, *keyp && !onekey ? 1 : 0);
- if (key->type == REPOKEY_TYPE_DELETED)
- continue;
- if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
+ if (keyskip && (key->name >= keyskip[0] || keyskip[3 + key->name] != keyskip[1] + data->repodataid))
+ {
+ if (onekey)
+ return;
+ continue;
+ }
+ if (key->type == REPOKEY_TYPE_DELETED && !(flags & SEARCH_KEEP_TYPE_DELETED))
{
- struct subschema_data subd;
- int nentries;
- Id schema = 0;
-
- subd.cbdata = cbdata;
- subd.s = s;
- subd.parent = &kv;
- ddp = data_read_id(ddp, &nentries);
- kv.num = nentries;
- kv.entry = 0;
- kv.eof = 0;
- while (ddp && nentries > 0)
- {
- if (!--nentries)
- kv.eof = 1;
- if (key->type == REPOKEY_TYPE_FLEXARRAY || !kv.entry)
- ddp = data_read_id(ddp, &schema);
- kv.id = schema;
- kv.str = (char *)ddp;
- stop = callback(cbdata, s, data, key, &kv);
- if (stop > SEARCH_NEXT_KEY)
- return;
- if (stop && stop != SEARCH_ENTERSUB)
- break;
- if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
- repodata_search(data, SOLVID_SUBSCHEMA, 0, flags, callback, &subd);
- ddp = data_skip_schema(data, ddp, schema);
- kv.entry++;
- }
- if (!nentries && (flags & SEARCH_ARRAYSENTINEL) != 0)
- {
- /* sentinel */
- kv.eof = 2;
- kv.str = (char *)ddp;
- stop = callback(cbdata, s, data, key, &kv);
- if (stop > SEARCH_NEXT_KEY)
- return;
- }
if (onekey)
return;
continue;
}
+ if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
+ {
+ kv.entry = -1;
+ ddp = data_read_id(ddp, (Id *)&kv.num);
+ kv.str = (const char *)ddp;
+ stop = repodata_search_array(data, solvid, 0, flags, key, &kv, callback, cbdata);
+ if (onekey || stop > SEARCH_NEXT_KEY)
+ return;
+ continue;
+ }
kv.entry = 0;
do
{
}
void
+repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
+{
+ repodata_search_keyskip(data, solvid, keyname, flags, 0, callback, cbdata);
+}
+
+void
repodata_setpos_kv(Repodata *data, KeyValue *kv)
{
Pool *pool = data->repo->pool;
int
datamatcher_init(Datamatcher *ma, const char *match, int flags)
{
- ma->match = match ? solv_strdup(match) : 0;
+ match = match ? solv_strdup(match) : 0;
+ ma->match = match;
ma->flags = flags;
ma->error = 0;
ma->matchdata = 0;
return !strcmp(match, basename);
}
-int
-repodata_filelistfilter_matches(Repodata *data, const char *str)
-{
- /* '.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$' */
- /* for now hardcoded */
- if (strstr(str, "bin/"))
- return 1;
- if (!strncmp(str, "/etc/", 5))
- return 1;
- if (!strcmp(str, "/usr/lib/sendmail"))
- return 1;
- return 0;
-}
-
-
enum {
di_bye,
if (di->dupstr)
{
if (di->dupstr == di->kv.str)
- {
- di->dupstr = solv_malloc(di->dupstrn);
- memcpy(di->dupstr, from->dupstr, di->dupstrn);
- }
+ di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
else
{
di->dupstr = 0;
di->parents[i].kv.parent = &di->parents[i - 1].kv;
di->kv.parent = &di->parents[di->nparents - 1].kv;
}
+ if (di->oldkeyskip)
+ di->oldkeyskip = solv_memdup2(di->oldkeyskip, 3 + di->oldkeyskip[0], sizeof(Id));
+ if (di->keyskip)
+ di->keyskip = di->oldkeyskip;
}
int
datamatcher_free(&di->matcher);
if (di->dupstr)
solv_free(di->dupstr);
+ if (di->oldkeyskip)
+ solv_free(di->oldkeyskip);
}
-static inline unsigned char *
+static unsigned char *
dataiterator_find_keyname(Dataiterator *di, Id keyname)
{
- Id *keyp = di->keyp;
- Repokey *keys = di->data->keys;
+ Id *keyp;
+ Repokey *keys = di->data->keys, *key;
unsigned char *dp;
for (keyp = di->keyp; *keyp; keyp++)
break;
if (!*keyp)
return 0;
+ key = keys + *keyp;
+ if (key->type == REPOKEY_TYPE_DELETED)
+ return 0;
+ if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
+ return 0; /* get_data will not work, no need to forward */
dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
if (!dp)
return 0;
return dp;
}
-static inline int
-is_filelist_extension(Repodata *data)
-{
- int j;
- if (!repodata_precheck_keyname(data, SOLVABLE_FILELIST))
- return 0;
- for (j = 1; j < data->nkeys; j++)
- if (data->keys[j].name == SOLVABLE_FILELIST)
- break;
- if (j == data->nkeys)
- return 0;
- if (data->state != REPODATA_AVAILABLE)
- return 1;
- for (j = 1; j < data->nkeys; j++)
- if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
- return 0;
- return 1;
-}
-
-static int
-dataiterator_filelistcheck(Dataiterator *di)
-{
- int j;
- int needcomplete = 0;
- Repodata *data = di->data;
-
- if ((di->flags & SEARCH_COMPLETE_FILELIST) != 0)
- if (!di->matcher.match
- || ((di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING
- && (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_GLOB)
- || !repodata_filelistfilter_matches(data, di->matcher.match))
- needcomplete = 1;
- if (data->state != REPODATA_AVAILABLE)
- return needcomplete ? 1 : 0;
- if (!needcomplete)
- {
- /* we don't need the complete filelist, so ignore all stubs */
- for (j = 1; j < data->nkeys; j++)
- if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
- return 1;
- return 0;
- }
- else
- {
- /* we need the complete filelist. check if we habe a filtered filelist and there's
- * a extension with the complete filelist later on */
- for (j = 1; j < data->nkeys; j++)
- if (data->keys[j].name == SOLVABLE_FILELIST)
- break;
- if (j == data->nkeys)
- return 0; /* does not have filelist */
- for (j = 1; j < data->nkeys; j++)
- if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
- break;
- if (j == data->nkeys)
- return 1; /* this is the externsion */
- while (data - data->repo->repodata + 1 < data->repo->nrepodata)
- {
- data++;
- if (is_filelist_extension(data))
- return 0;
- }
- return 1;
- }
-}
-
int
dataiterator_step(Dataiterator *di)
{
Id schema;
- if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate) {
- unsigned int ddpoff = di->ddp - di->vert_ddp;
- di->vert_off += ddpoff;
- di->vert_len -= ddpoff;
- di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len);
- di->vert_storestate = di->data->storestate;
- if (!di->ddp)
- di->state = di_nextkey;
- }
+ if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate)
+ {
+ unsigned int ddpoff = di->ddp - di->vert_ddp;
+ di->vert_off += ddpoff;
+ di->vert_len -= ddpoff;
+ di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len);
+ di->vert_storestate = di->data->storestate;
+ if (!di->ddp)
+ di->state = di_nextkey;
+ }
for (;;)
{
switch (di->state)
/* FALLTHROUGH */
case di_entersolvable: di_entersolvable:
- if (di->repodataid)
+ if (!di->repodataid)
+ goto di_enterrepodata; /* POS case, repodata is set */
+ if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
{
- di->repodataid = 1; /* reset repodata iterator */
- if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
- {
- extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1];
+ extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1];
+ di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
+ di->data = 0;
+ goto di_entersolvablekey;
+ }
- di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
- di->data = 0;
- goto di_entersolvablekey;
- }
+ if (di->keyname)
+ {
+ di->data = di->keyname == SOLVABLE_FILELIST ? repo_lookup_filelist_repodata(di->repo, di->solvid, &di->matcher) : repo_lookup_repodata_opt(di->repo, di->solvid, di->keyname);
+ if (!di->data)
+ goto di_nextsolvable;
+ di->repodataid = di->data - di->repo->repodata;
+ di->keyskip = 0;
+ goto di_enterrepodata;
}
+ di_leavesolvablekey:
+ di->repodataid = 1; /* reset repodata iterator */
+ di->keyskip = repo_create_keyskip(di->repo, di->solvid, &di->oldkeyskip);
/* FALLTHROUGH */
case di_enterrepodata: di_enterrepodata:
goto di_nextsolvable;
di->data = di->repo->repodata + di->repodataid;
}
- if (di->repodataid && di->keyname == SOLVABLE_FILELIST && !dataiterator_filelistcheck(di))
- goto di_nextrepodata;
if (!maybe_load_repodata(di->data, di->keyname))
goto di_nextrepodata;
di->dp = solvid2data(di->data, di->solvid, &schema);
}
else if (di->key->storage == KEY_STORAGE_INCORE)
{
- di->ddp = di->dp;
+ di->ddp = di->dp; /* start of data */
if (di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0))
- di->dp = data_skip_key(di->data, di->dp, di->key);
+ di->dp = data_skip_key(di->data, di->dp, di->key); /* advance to next key */
}
else
di->ddp = 0;
if (!di->ddp)
goto di_nextkey;
- if (di->key->type == REPOKEY_TYPE_DELETED)
+ if (di->keyskip && (di->key->name >= di->keyskip[0] || di->keyskip[3 + di->key->name] != di->keyskip[1] + di->data->repodataid))
+ goto di_nextkey;
+ if (di->key->type == REPOKEY_TYPE_DELETED && !(di->flags & SEARCH_KEEP_TYPE_DELETED))
goto di_nextkey;
if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
goto di_enterarray;
case di_nextattr:
di->kv.entry++;
di->ddp = data_fetch(di->ddp, &di->kv, di->key);
- if (di->kv.eof)
- di->state = di_nextkey;
- else
- di->state = di_nextattr;
+ di->state = di->kv.eof ? di_nextkey : di_nextattr;
break;
case di_nextkey: di_nextkey:
/* FALLTHROUGH */
case di_nextrepodata: di_nextrepodata:
- if (di->repodataid && ++di->repodataid < di->repo->nrepodata)
+ if (!di->keyname && di->repodataid && ++di->repodataid < di->repo->nrepodata)
goto di_enterrepodata;
/* FALLTHROUGH */
/* special solvable attr handling follows */
case di_nextsolvablekey: di_nextsolvablekey:
- if (di->keyname || di->key->name == RPM_RPMDBID)
- goto di_enterrepodata;
+ if (di->keyname)
+ goto di_nextsolvable;
+ if (di->key->name == RPM_RPMDBID) /* reached end of list? */
+ goto di_leavesolvablekey;
di->key++;
/* FALLTHROUGH */
}
+ /* we have a potential match */
if (di->matcher.match)
{
+ const char *str;
/* simple pre-check so that we don't need to stringify */
if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0)
if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
continue;
- if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
+ /* now stringify so that we can do the matching */
+ if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
{
if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
return 1;
continue;
}
- if (!datamatcher_match(&di->matcher, di->kv.str))
+ if (!datamatcher_match(&di->matcher, str))
continue;
}
else
{
+ /* stringify filelist if requested */
if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->flags & SEARCH_FILES) != 0)
repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags);
}
if (from->dupstr && from->dupstr == from->kv.str)
{
di->dupstrn = from->dupstrn;
- di->dupstr = solv_malloc(from->dupstrn);
- memcpy(di->dupstr, from->dupstr, di->dupstrn);
+ di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
}
}
return;
}
di->repoid = 0;
- di->data = di->repo->repodata + di->pool->pos.repodataid;
- di->repodataid = 0;
- di->solvid = solvid;
- di->state = di_enterrepo;
- di->flags |= SEARCH_THISSOLVID;
- return;
+ if (!di->pool->pos.repodataid && di->pool->pos.solvid == SOLVID_META) {
+ solvid = SOLVID_META; /* META pos hack */
+ } else {
+ di->data = di->repo->repodata + di->pool->pos.repodataid;
+ di->repodataid = 0;
+ }
}
- if (solvid > 0)
+ else if (solvid > 0)
{
di->repo = di->pool->solvables[solvid].repo;
di->repoid = 0;
}
- else if (di->repoid > 0)
+ if (di->repoid > 0)
{
if (!di->pool->urepos)
{
di->repoid = 1;
di->repo = di->pool->repos[di->repoid];
}
- di->repodataid = 1;
+ if (solvid != SOLVID_POS)
+ di->repodataid = 1;
di->solvid = solvid;
if (solvid)
di->flags |= SEARCH_THISSOLVID;
int
dataiterator_match(Dataiterator *di, Datamatcher *ma)
{
- if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
+ const char *str;
+ if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
return 0;
- if (!ma)
- return 1;
- return datamatcher_match(ma, di->kv.str);
+ return ma ? datamatcher_match(ma, str) : 1;
}
void
return;
switch (di->key->type)
{
- case REPOKEY_TYPE_MD5:
- case REPOKEY_TYPE_SHA1:
- case REPOKEY_TYPE_SHA256:
+ case_CHKSUM_TYPES:
case REPOKEY_TYPE_DIRSTRARRAY:
if (di->kv.num) /* was it stringified into tmp space? */
l = strlen(di->kv.str) + 1;
case REPOKEY_TYPE_DIRSTRARRAY:
l = strlen(di->kv.str) + 1;
break;
- case REPOKEY_TYPE_MD5:
- l = SIZEOF_MD5;
- break;
- case REPOKEY_TYPE_SHA1:
- l = SIZEOF_SHA1;
- break;
- case REPOKEY_TYPE_SHA256:
- l = SIZEOF_SHA256;
+ case_CHKSUM_TYPES:
+ l = solv_chksum_len(di->key->type);
break;
case REPOKEY_TYPE_BINARY:
l = di->kv.num;
return;
if (!data->incoreoffset)
{
+ /* this also means that data->attrs is NULL */
data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
data->start = start;
data->end = start + num;
}
void
-repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
+repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id handle)
{
repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
- data->attriddata[data->attriddatalen++] = ghandle;
+ data->attriddata[data->attriddatalen++] = handle;
data->attriddata[data->attriddatalen++] = 0;
}
void
-repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
+repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id handle)
{
repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
- data->attriddata[data->attriddatalen++] = ghandle;
+ data->attriddata[data->attriddatalen++] = handle;
data->attriddata[data->attriddatalen++] = 0;
}
void
+repodata_set_kv(Repodata *data, Id solvid, Id keyname, Id keytype, KeyValue *kv)
+{
+ switch (keytype)
+ {
+ case REPOKEY_TYPE_ID:
+ repodata_set_id(data, solvid, keyname, kv->id);
+ break;
+ case REPOKEY_TYPE_CONSTANTID:
+ repodata_set_constantid(data, solvid, keyname, kv->id);
+ break;
+ case REPOKEY_TYPE_IDARRAY:
+ repodata_add_idarray(data, solvid, keyname, kv->id);
+ break;
+ case REPOKEY_TYPE_STR:
+ repodata_set_str(data, solvid, keyname, kv->str);
+ break;
+ case REPOKEY_TYPE_VOID:
+ repodata_set_void(data, solvid, keyname);
+ break;
+ case REPOKEY_TYPE_NUM:
+ repodata_set_num(data, solvid, keyname, SOLV_KV_NUM64(kv));
+ break;
+ case REPOKEY_TYPE_CONSTANT:
+ repodata_set_constant(data, solvid, keyname, kv->num);
+ break;
+ case REPOKEY_TYPE_DIRNUMNUMARRAY:
+ if (kv->id)
+ repodata_add_dirnumnum(data, solvid, keyname, kv->id, kv->num, kv->num2);
+ break;
+ case REPOKEY_TYPE_DIRSTRARRAY:
+ repodata_add_dirstr(data, solvid, keyname, kv->id, kv->str);
+ break;
+ case_CHKSUM_TYPES:
+ repodata_set_bin_checksum(data, solvid, keyname, keytype, (const unsigned char *)kv->str);
+ break;
+ default:
+ break;
+ }
+}
+
+void
repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname)
{
Id *pp, *ap, **app;
ap = *app;
if (!ap)
return;
+ if (!keyname)
+ {
+ *app = 0; /* delete all attributes */
+ return;
+ }
for (; *ap; ap += 2)
if (data->keys[*ap].name == keyname)
break;
*pp = 0;
}
-/* XXX: does not work correctly, needs fix in iterators! */
void
repodata_unset(Repodata *data, Id solvid, Id keyname)
{
Id *tmpattrs;
if (!data->attrs || dest == src)
return;
+ if (dest < data->start || dest >= data->end)
+ repodata_extend(data, dest);
+ if (src < data->start || src >= data->end)
+ repodata_extend(data, src);
tmpattrs = data->attrs[dest - data->start];
data->attrs[dest - data->start] = data->attrs[src - data->start];
data->attrs[src - data->start] = tmpattrs;
+ if (data->lasthandle == src || data->lasthandle == dest)
+ data->lasthandle = 0;
}
/*********************************/
+/* this is to reduct memory usage when internalizing oversized repos */
+static void
+compact_attrdata(Repodata *data, int entry, int nentry)
+{
+ int i;
+ unsigned int attrdatastart = data->attrdatalen;
+ unsigned int attriddatastart = data->attriddatalen;
+ if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
+ return;
+ for (i = entry; i < nentry; i++)
+ {
+ Id v, *attrs = data->attrs[i];
+ if (!attrs)
+ continue;
+ for (; *attrs; attrs += 2)
+ {
+ switch (data->keys[*attrs].type)
+ {
+ case REPOKEY_TYPE_STR:
+ case REPOKEY_TYPE_BINARY:
+ case_CHKSUM_TYPES:
+ if ((unsigned int)attrs[1] < attrdatastart)
+ attrdatastart = attrs[1];
+ break;
+ case REPOKEY_TYPE_DIRSTRARRAY:
+ for (v = attrs[1]; data->attriddata[v] ; v += 2)
+ if ((unsigned int)data->attriddata[v + 1] < attrdatastart)
+ attrdatastart = data->attriddata[v + 1];
+ /* FALLTHROUGH */
+ case REPOKEY_TYPE_IDARRAY:
+ case REPOKEY_TYPE_DIRNUMNUMARRAY:
+ if ((unsigned int)attrs[1] < attriddatastart)
+ attriddatastart = attrs[1];
+ break;
+ case REPOKEY_TYPE_FIXARRAY:
+ case REPOKEY_TYPE_FLEXARRAY:
+ return;
+ default:
+ break;
+ }
+ }
+ }
+#if 0
+ printf("compact_attrdata %d %d\n", entry, nentry);
+ printf("attrdatastart: %d\n", attrdatastart);
+ printf("attriddatastart: %d\n", attriddatastart);
+#endif
+ if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
+ return;
+ for (i = entry; i < nentry; i++)
+ {
+ Id v, *attrs = data->attrs[i];
+ if (!attrs)
+ continue;
+ for (; *attrs; attrs += 2)
+ {
+ switch (data->keys[*attrs].type)
+ {
+ case REPOKEY_TYPE_STR:
+ case REPOKEY_TYPE_BINARY:
+ case_CHKSUM_TYPES:
+ attrs[1] -= attrdatastart;
+ break;
+ case REPOKEY_TYPE_DIRSTRARRAY:
+ for (v = attrs[1]; data->attriddata[v] ; v += 2)
+ data->attriddata[v + 1] -= attrdatastart;
+ /* FALLTHROUGH */
+ case REPOKEY_TYPE_IDARRAY:
+ case REPOKEY_TYPE_DIRNUMNUMARRAY:
+ attrs[1] -= attriddatastart;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ if (attrdatastart)
+ {
+ data->attrdatalen -= attrdatastart;
+ memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
+ data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
+ }
+ if (attriddatastart)
+ {
+ data->attriddatalen -= attriddatastart;
+ memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
+ data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
+ }
+}
+
/* internalalize some key into incore/vincore data */
static void
case REPOKEY_TYPE_VOID:
case REPOKEY_TYPE_CONSTANT:
case REPOKEY_TYPE_CONSTANTID:
+ case REPOKEY_TYPE_DELETED:
break;
case REPOKEY_TYPE_STR:
data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
case REPOKEY_TYPE_SHA1:
data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
break;
+ case REPOKEY_TYPE_SHA224:
+ data_addblob(xd, data->attrdata + val, SIZEOF_SHA224);
+ break;
case REPOKEY_TYPE_SHA256:
data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
break;
+ case REPOKEY_TYPE_SHA384:
+ data_addblob(xd, data->attrdata + val, SIZEOF_SHA384);
+ break;
+ case REPOKEY_TYPE_SHA512:
+ data_addblob(xd, data->attrdata + val, SIZEOF_SHA512);
+ break;
case REPOKEY_TYPE_NUM:
if (val & 0x80000000)
{
sp = schema;
kp = data->xattrs[-*ida];
if (!kp)
- continue;
+ continue; /* ignore empty elements */
num++;
- for (;*kp; kp += 2)
+ for (; *kp; kp += 2)
*sp++ = *kp;
*sp = 0;
if (!schemaid)
schemaid = repodata_schema2id(data, schema, 1);
else if (schemaid != repodata_schema2id(data, schema, 0))
{
- pool_debug(data->repo->pool, SOLV_FATAL, "fixarray substructs with different schemas\n");
- exit(1);
+ pool_debug(data->repo->pool, SOLV_ERROR, "repodata_serialize_key: fixarray substructs with different schemas\n");
+ num = 0;
+ break;
}
}
+ data_addid(xd, num);
if (!num)
break;
- data_addid(xd, num);
data_addid(xd, schemaid);
for (ida = data->attriddata + val; *ida; ida++)
{
Id *kp = data->xattrs[-*ida];
if (!kp)
continue;
- for (;*kp; kp += 2)
+ for (; *kp; kp += 2)
repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
}
break;
break;
}
default:
- pool_debug(data->repo->pool, SOLV_FATAL, "don't know how to handle type %d\n", key->type);
+ pool_debug(data->repo->pool, SOLV_FATAL, "repodata_serialize_key: don't know how to handle type %d\n", key->type);
exit(1);
}
if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
}
}
+/* create a circular linked list of all keys that share
+ * the same keyname */
+static Id *
+calculate_keylink(Repodata *data)
+{
+ int i, j;
+ Id *link;
+ Id maxkeyname = 0, *keytable = 0;
+ link = solv_calloc(data->nkeys, sizeof(Id));
+ if (data->nkeys <= 2)
+ return link;
+ for (i = 1; i < data->nkeys; i++)
+ {
+ Id n = data->keys[i].name;
+ if (n >= maxkeyname)
+ {
+ keytable = solv_realloc2(keytable, n + 128, sizeof(Id));
+ memset(keytable + maxkeyname, 0, (n + 128 - maxkeyname) * sizeof(Id));
+ maxkeyname = n + 128;
+ }
+ j = keytable[n];
+ if (j)
+ link[i] = link[j];
+ else
+ j = i;
+ link[j] = i;
+ keytable[n] = i;
+ }
+ /* remove links that just point to themselfs */
+ for (i = 1; i < data->nkeys; i++)
+ if (link[i] == i)
+ link[i] = 0;
+ solv_free(keytable);
+ return link;
+}
+
void
repodata_internalize(Repodata *data)
{
Repokey *key, solvkey;
Id entry, nentry;
- Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen;
+ Id schemaid, keyid, *schema, *sp, oldschemaid, *keyp, *seen;
+ Offset *oldincoreoffs = 0;
+ int schemaidx;
unsigned char *dp, *ndp;
- int newschema, oldcount;
+ int neednewschema;
struct extdata newincore;
struct extdata newvincore;
Id solvkeyid;
+ Id *keylink;
+ int haveoldkl;
if (!data->attrs && !data->xattrs)
return;
+#if 0
+ printf("repodata_internalize %d\n", data->repodataid);
+ printf(" attr data: %d K\n", data->attrdatalen / 1024);
+ printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
+#endif
newvincore.buf = data->vincore;
newvincore.len = data->vincorelen;
data->mainschema = 0;
data->mainschemaoffsets = solv_free(data->mainschemaoffsets);
+ keylink = calculate_keylink(data);
/* join entry data */
/* we start with the meta data, entry -1 */
for (entry = -1; entry < nentry; entry++)
{
- memset(seen, 0, data->nkeys * sizeof(Id));
- oldschema = 0;
+ oldschemaid = 0;
dp = data->incoredata;
if (dp)
{
dp += entry >= 0 ? data->incoreoffset[entry] : 1;
- dp = data_read_id(dp, &oldschema);
+ dp = data_read_id(dp, &oldschemaid);
}
+ memset(seen, 0, data->nkeys * sizeof(Id));
#if 0
-fprintf(stderr, "oldschema %d\n", oldschema);
-fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
+fprintf(stderr, "oldschemaid %d\n", oldschemaid);
+fprintf(stderr, "schemata %d\n", data->schemata[oldschemaid]);
fprintf(stderr, "schemadata %p\n", data->schemadata);
#endif
- /* seen: -1: old data 0: skipped >0: id + 1 */
- newschema = 0;
- oldcount = 0;
+
+ /* seen: -1: old data, 0: skipped, >0: id + 1 */
+ neednewschema = 0;
sp = schema;
- for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
+ haveoldkl = 0;
+ for (keyp = data->schemadata + data->schemata[oldschemaid]; *keyp; keyp++)
{
if (seen[*keyp])
{
- pool_debug(data->repo->pool, SOLV_FATAL, "Inconsistent old data (key occured twice).\n");
- exit(1);
+ /* oops, should not happen */
+ neednewschema = 1;
+ continue;
}
- seen[*keyp] = -1;
+ seen[*keyp] = -1; /* use old marker */
*sp++ = *keyp;
- oldcount++;
+ if (keylink[*keyp])
+ haveoldkl = 1; /* potential keylink conflict */
}
- if (entry >= 0)
- keyp = data->attrs ? data->attrs[entry] : 0;
- else
+
+ /* strip solvables key */
+ if (entry < 0 && solvkeyid && seen[solvkeyid])
{
- /* strip solvables key */
*sp = 0;
for (sp = keyp = schema; *sp; sp++)
if (*sp != solvkeyid)
*keyp++ = *sp;
- else
- oldcount--;
sp = keyp;
seen[solvkeyid] = 0;
- keyp = data->xattrs ? data->xattrs[1] : 0;
+ neednewschema = 1;
}
+
+ /* add new entries */
+ if (entry >= 0)
+ keyp = data->attrs ? data->attrs[entry] : 0;
+ else
+ keyp = data->xattrs ? data->xattrs[1] : 0;
if (keyp)
for (; *keyp; keyp += 2)
{
if (!seen[*keyp])
{
- newschema = 1;
+ neednewschema = 1;
*sp++ = *keyp;
+ if (haveoldkl && keylink[*keyp]) /* this should be pretty rare */
+ {
+ Id kl;
+ for (kl = keylink[*keyp]; kl != *keyp; kl = keylink[kl])
+ if (seen[kl] == -1)
+ {
+ /* replacing old key kl, remove from schema and seen */
+ Id *osp;
+ for (osp = schema; osp < sp; osp++)
+ if (*osp == kl)
+ {
+ memmove(osp, osp + 1, (sp - osp) * sizeof(Id));
+ sp--;
+ seen[kl] = 0;
+ break;
+ }
+ }
+ }
}
seen[*keyp] = keyp[1] + 1;
}
+
+ /* add solvables key if needed */
if (entry < 0 && data->end != data->start)
{
- *sp++ = solvkeyid;
- newschema = 1;
+ *sp++ = solvkeyid; /* always last in schema */
+ neednewschema = 1;
}
+
+ /* commit schema */
*sp = 0;
- if (newschema)
+ if (neednewschema)
/* Ideally we'd like to sort the new schema here, to ensure
- schema equality independend of the ordering. We can't do that
- yet. For once see below (old ids need to come before new ids).
- An additional difficulty is that we also need to move
- the values with the keys. */
+ schema equality independend of the ordering. */
schemaid = repodata_schema2id(data, schema, 1);
else
- schemaid = oldschema;
+ schemaid = oldschemaid;
+ if (entry < 0)
+ {
+ data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
+ data->mainschema = schemaid;
+ }
+
+ /* find offsets in old incore data */
+ if (oldschemaid)
+ {
+ Id *lastneeded = 0;
+ for (sp = data->schemadata + data->schemata[oldschemaid]; *sp; sp++)
+ if (seen[*sp] == -1)
+ lastneeded = sp + 1;
+ if (lastneeded)
+ {
+ if (!oldincoreoffs)
+ oldincoreoffs = solv_malloc2(data->nkeys, 2 * sizeof(Offset));
+ for (sp = data->schemadata + data->schemata[oldschemaid]; sp != lastneeded; sp++)
+ {
+ /* Skip the data associated with this old key. */
+ key = data->keys + *sp;
+ ndp = dp;
+ if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
+ {
+ ndp = data_skip(ndp, REPOKEY_TYPE_ID);
+ ndp = data_skip(ndp, REPOKEY_TYPE_ID);
+ }
+ else if (key->storage == KEY_STORAGE_INCORE)
+ ndp = data_skip_key(data, ndp, key);
+ oldincoreoffs[*sp * 2] = dp - data->incoredata;
+ oldincoreoffs[*sp * 2 + 1] = ndp - dp;
+ dp = ndp;
+ }
+ }
+ }
+
+ /* just copy over the complete old entry (including the schemaid) if there was no new data */
+ if (entry >= 0 && !neednewschema && oldschemaid && (!data->attrs || !data->attrs[entry]) && dp)
+ {
+ ndp = data->incoredata + data->incoreoffset[entry];
+ data->incoreoffset[entry] = newincore.len;
+ data_addblob(&newincore, ndp, dp - ndp);
+ goto entrydone;
+ }
/* Now create data blob. We walk through the (possibly new) schema
and either copy over old data, or insert the new. */
- /* XXX Here we rely on the fact that the (new) schema has the form
- o1 o2 o3 o4 ... | n1 n2 n3 ...
- (oX being the old keyids (possibly overwritten), and nX being
- the new keyids). This rules out sorting the keyids in order
- to ensure a small schema count. */
if (entry >= 0)
data->incoreoffset[entry] = newincore.len;
data_addid(&newincore, schemaid);
- if (entry == -1)
- {
- data->mainschema = schemaid;
- data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
- }
- keypstart = data->schemadata + data->schemata[schemaid];
- for (keyp = keypstart; *keyp; keyp++)
+
+ /* we don't use a pointer to the schemadata here as repodata_serialize_key
+ * may call repodata_schema2id() which might realloc our schemadata */
+ for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
{
- if (entry == -1)
- data->mainschemaoffsets[keyp - keypstart] = newincore.len;
- if (*keyp == solvkeyid)
- {
- /* add flexarray entry count */
- data_addid(&newincore, data->end - data->start);
- break;
- }
- key = data->keys + *keyp;
-#if 0
- fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, pool_id2str(data->repo->pool, key->name), pool_id2str(data->repo->pool, key->type));
-#endif
- ndp = dp;
- if (oldcount)
+ if (entry < 0)
{
- /* Skip the data associated with this old key. */
- if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
+ data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
+ if (keyid == solvkeyid)
{
- ndp = data_skip(dp, REPOKEY_TYPE_ID);
- ndp = data_skip(ndp, REPOKEY_TYPE_ID);
+ /* add flexarray entry count */
+ data_addid(&newincore, data->end - data->start);
+ break; /* always the last entry */
}
- else if (key->storage == KEY_STORAGE_INCORE)
- ndp = data_skip_key(data, dp, key);
- oldcount--;
}
- if (seen[*keyp] == -1)
+ if (seen[keyid] == -1)
{
- /* If this key was an old one _and_ was not overwritten with
- a different value copy over the old value (we skipped it
- above). */
- if (dp != ndp)
- data_addblob(&newincore, dp, ndp - dp);
- seen[*keyp] = 0;
+ if (oldincoreoffs[keyid * 2 + 1])
+ data_addblob(&newincore, data->incoredata + oldincoreoffs[keyid * 2], oldincoreoffs[keyid * 2 + 1]);
}
- else if (seen[*keyp])
+ else if (seen[keyid])
+ repodata_serialize_key(data, &newincore, &newvincore, schema, data->keys + keyid, seen[keyid] - 1);
+ }
+
+entrydone:
+ /* free memory */
+ if (entry >= 0 && data->attrs)
+ {
+ if (data->attrs[entry])
+ data->attrs[entry] = solv_free(data->attrs[entry]);
+ if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
{
- /* Otherwise we have a new value. Parse it into the internal
- form. */
- repodata_serialize_key(data, &newincore, &newvincore,
- schema, key, seen[*keyp] - 1);
+ compact_attrdata(data, entry + 1, nentry); /* try to free some memory */
+#if 0
+ printf(" attr data: %d K\n", data->attrdatalen / 1024);
+ printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
+ printf(" incore data: %d K\n", newincore.len / 1024);
+ printf(" sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
+ /* malloc_stats(); */
+#endif
}
- dp = ndp;
}
- if (entry >= 0 && data->attrs && data->attrs[entry])
- data->attrs[entry] = solv_free(data->attrs[entry]);
}
/* free all xattrs */
for (entry = 0; entry < data->nxattrs; entry++)
data->lastdatalen = 0;
solv_free(schema);
solv_free(seen);
+ solv_free(keylink);
+ solv_free(oldincoreoffs);
repodata_free_schemahash(data);
solv_free(data->incoredata);
data->incoredatalen = newincore.len;
data->incoredatafree = 0;
- solv_free(data->vincore);
data->vincore = newvincore.buf;
data->vincorelen = newvincore.len;
data->attrdatalen = 0;
data->attriddatalen = 0;
data->attrnum64datalen = 0;
+#if 0
+ printf("repodata_internalize %d done\n", data->repodataid);
+ printf(" incore data: %d K\n", data->incoredatalen / 1024);
+#endif
}
void
}
}
+/* call the pool's loadcallback to load a stub repodata */
static void
-repodata_load_stub(Repodata *data)
+repodata_stub_loader(Repodata *data)
{
Repo *repo = data->repo;
Pool *pool = repo->pool;
int r, i;
- struct _Pool_tmpspace oldtmpspace;
+ struct s_Pool_tmpspace oldtmpspace;
+ Datapos oldpos;
if (!pool->loadcallback)
{
}
data->state = REPODATA_LOADING;
- /* save tmp space */
+ /* save tmp space and pos */
oldtmpspace = pool->tmpspace;
memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
+ oldpos = pool->pos;
r = pool->loadcallback(pool, data, pool->loadcallbackdata);
- /* restore tmp space */
+ /* restore tmp space and pos */
for (i = 0; i < POOL_TMPSPACEBUF; i++)
solv_free(pool->tmpspace.buf[i]);
pool->tmpspace = oldtmpspace;
+ if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
+ memset(&oldpos, 0, sizeof(oldpos));
+ pool->pos = oldpos;
data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR;
}
if (data->end > data->start)
repodata_extend_block(sdata, data->start, data->end - data->start);
sdata->state = REPODATA_STUB;
- sdata->loadcallback = repodata_load_stub;
+ sdata->loadcallback = repodata_stub_loader;
*datap = data;
return sdata;
}
xkeyname = 0;
continue;
}
- switch (di.key->type)
+ repodata_set_kv(sdata, SOLVID_META, di.key->name, di.key->type, &di.kv);
+ if (di.key->name == REPOSITORY_KEYS && di.key->type == REPOKEY_TYPE_IDARRAY)
{
- case REPOKEY_TYPE_ID:
- repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
- break;
- case REPOKEY_TYPE_CONSTANTID:
- repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
- break;
- case REPOKEY_TYPE_STR:
- repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
- break;
- case REPOKEY_TYPE_VOID:
- repodata_set_void(sdata, SOLVID_META, di.key->name);
- break;
- case REPOKEY_TYPE_NUM:
- repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv));
- break;
- case REPOKEY_TYPE_MD5:
- case REPOKEY_TYPE_SHA1:
- case REPOKEY_TYPE_SHA256:
- repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
- break;
- case REPOKEY_TYPE_IDARRAY:
- repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
- if (di.key->name == REPOSITORY_KEYS)
+ if (!xkeyname)
{
- if (!xkeyname)
- {
- if (!di.kv.eof)
- xkeyname = di.kv.id;
- }
- else
- {
- repodata_add_stubkey(sdata, xkeyname, di.kv.id);
- xkeyname = 0;
- }
+ if (!di.kv.eof)
+ xkeyname = di.kv.id;
+ }
+ else
+ {
+ repodata_add_stubkey(sdata, xkeyname, di.kv.id);
+ if (xkeyname == SOLVABLE_FILELIST)
+ repodata_set_filelisttype(sdata, REPODATA_FILELIST_EXTENSION);
+ xkeyname = 0;
}
- break;
- default:
- break;
}
}
dataiterator_free(&di);
return data;
}
+void
+repodata_set_filelisttype(Repodata *data, int type)
+{
+ data->filelisttype = type;
+}
+
unsigned int
repodata_memused(Repodata *data)
{
return data->incoredatalen + data->vincorelen;
}
-/*
-vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4:
-*/
projects / platform / upstream / libsolv.git / blobdiff