if (off >= data->lastverticaloffset)
{
off -= data->lastverticaloffset;
- if (off + len > data->vincorelen)
+ if ((unsigned int)off + len > data->vincorelen)
return 0;
return data->vincore + off;
}
- if (off + len > key->size)
+ if ((unsigned int)off + len > key->size)
return 0;
/* we now have the offset, go into vertical */
off += data->verticaloffset[key - data->keys];
* data lookup
*/
-static inline unsigned char *
+static unsigned char *
find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
{
unsigned char *dp;
return 0;
if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
return dp; /* no need to forward... */
+ if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
+ return 0; /* get_data will not work, no need to forward */
dp = forward_to_key(data, *kp, keyp, dp);
if (!dp)
return 0;
dp = find_key_data(data, solvid, keyname, &key);
if (!dp)
return 0;
- if (!(key->type == REPOKEY_TYPE_MD5 || key->type == REPOKEY_TYPE_SHA1 || key->type == REPOKEY_TYPE_SHA256))
- return 0;
+ switch (key->type)
+ {
+ case_CHKSUM_TYPES:
+ break;
+ default:
+ return 0;
+ }
*typep = key->type;
return dp;
}
return 1;
}
+const void *
+repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp)
+{
+ unsigned char *dp;
+ Repokey *key;
+ Id len;
+
+ dp = find_key_data(data, solvid, keyname, &key);
+ if (!dp || key->type != REPOKEY_TYPE_BINARY)
+ {
+ *lenp = 0;
+ return 0;
+ }
+ dp = data_read_id(dp, &len);
+ *lenp = len;
+ return dp;
+}
+
Id
repodata_globalize_id(Repodata *data, Id id, int create)
{
*/
-int
+const char *
repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
{
switch (key->type)
if (*s == ':' && s > kv->str)
kv->str = s + 1;
}
- return 1;
+ return kv->str;
case REPOKEY_TYPE_STR:
- return 1;
+ return kv->str;
case REPOKEY_TYPE_DIRSTRARRAY:
if (!(flags & SEARCH_FILES))
- return 1; /* match just the basename */
+ return kv->str; /* match just the basename */
if (kv->num)
- return 1; /* already stringified */
+ return kv->str; /* already stringified */
/* Put the full filename into kv->str. */
kv->str = repodata_dir2str(data, kv->id, kv->str);
kv->num = 1; /* mark stringification */
- return 1;
- case REPOKEY_TYPE_MD5:
- case REPOKEY_TYPE_SHA1:
- case REPOKEY_TYPE_SHA256:
+ return kv->str;
+ case_CHKSUM_TYPES:
if (!(flags & SEARCH_CHECKSUMS))
return 0; /* skip em */
if (kv->num)
- return 1; /* already stringified */
+ return kv->str; /* already stringified */
kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
kv->num = 1; /* mark stringification */
- return 1;
+ return kv->str;
default:
return 0;
}
int
datamatcher_init(Datamatcher *ma, const char *match, int flags)
{
- ma->match = match ? solv_strdup(match) : 0;
+ match = match ? solv_strdup(match) : 0;
+ ma->match = match;
ma->flags = flags;
ma->error = 0;
ma->matchdata = 0;
if (di->dupstr)
{
if (di->dupstr == di->kv.str)
- {
- di->dupstr = solv_malloc(di->dupstrn);
- memcpy(di->dupstr, from->dupstr, di->dupstrn);
- }
+ di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
else
{
di->dupstr = 0;
solv_free(di->dupstr);
}
-static inline unsigned char *
+static unsigned char *
dataiterator_find_keyname(Dataiterator *di, Id keyname)
{
- Id *keyp = di->keyp;
- Repokey *keys = di->data->keys;
+ Id *keyp;
+ Repokey *keys = di->data->keys, *key;
unsigned char *dp;
for (keyp = di->keyp; *keyp; keyp++)
break;
if (!*keyp)
return 0;
+ key = keys + *keyp;
+ if (key->type == REPOKEY_TYPE_DELETED)
+ return 0;
+ if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
+ return 0; /* get_data will not work, no need to forward */
dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
if (!dp)
return 0;
if (!needcomplete)
{
/* we don't need the complete filelist, so ignore all stubs */
+ if (data->repo->nrepodata == 2)
+ return 1;
for (j = 1; j < data->nkeys; j++)
if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
return 1;
if (di->matcher.match)
{
+ const char *str;
/* simple pre-check so that we don't need to stringify */
if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0)
if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
continue;
- if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
+ if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
{
if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
return 1;
continue;
}
- if (!datamatcher_match(&di->matcher, di->kv.str))
+ if (!datamatcher_match(&di->matcher, str))
continue;
}
else
if (from->dupstr && from->dupstr == from->kv.str)
{
di->dupstrn = from->dupstrn;
- di->dupstr = solv_malloc(from->dupstrn);
- memcpy(di->dupstr, from->dupstr, di->dupstrn);
+ di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
}
}
int
dataiterator_match(Dataiterator *di, Datamatcher *ma)
{
- if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
+ const char *str;
+ if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
return 0;
- if (!ma)
- return 1;
- return datamatcher_match(ma, di->kv.str);
+ return ma ? datamatcher_match(ma, str) : 1;
}
void
return;
switch (di->key->type)
{
- case REPOKEY_TYPE_MD5:
- case REPOKEY_TYPE_SHA1:
- case REPOKEY_TYPE_SHA256:
+ case_CHKSUM_TYPES:
case REPOKEY_TYPE_DIRSTRARRAY:
if (di->kv.num) /* was it stringified into tmp space? */
l = strlen(di->kv.str) + 1;
case REPOKEY_TYPE_DIRSTRARRAY:
l = strlen(di->kv.str) + 1;
break;
- case REPOKEY_TYPE_MD5:
- l = SIZEOF_MD5;
- break;
- case REPOKEY_TYPE_SHA1:
- l = SIZEOF_SHA1;
- break;
- case REPOKEY_TYPE_SHA256:
- l = SIZEOF_SHA256;
+ case_CHKSUM_TYPES:
+ l = solv_chksum_len(di->key->type);
break;
case REPOKEY_TYPE_BINARY:
l = di->kv.num;
return;
if (!data->incoreoffset)
{
+ /* this also means that data->attrs is NULL */
data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
data->start = start;
data->end = start + num;
Id *tmpattrs;
if (!data->attrs || dest == src)
return;
+ if (dest < data->start || dest >= data->end)
+ repodata_extend(data, dest);
+ if (src < data->start || src >= data->end)
+ repodata_extend(data, src);
tmpattrs = data->attrs[dest - data->start];
data->attrs[dest - data->start] = data->attrs[src - data->start];
data->attrs[src - data->start] = tmpattrs;
/*********************************/
+/* this is to reduct memory usage when internalizing oversized repos */
+static void
+compact_attrdata(Repodata *data, int entry, int nentry)
+{
+ int i;
+ unsigned int attrdatastart = data->attrdatalen;
+ unsigned int attriddatastart = data->attriddatalen;
+ if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
+ return;
+ for (i = entry; i < nentry; i++)
+ {
+ Id v, *attrs = data->attrs[i];
+ if (!attrs)
+ continue;
+ for (; *attrs; attrs += 2)
+ {
+ switch (data->keys[*attrs].type)
+ {
+ case REPOKEY_TYPE_STR:
+ case REPOKEY_TYPE_BINARY:
+ case_CHKSUM_TYPES:
+ if ((unsigned int)attrs[1] < attrdatastart)
+ attrdatastart = attrs[1];
+ break;
+ case REPOKEY_TYPE_DIRSTRARRAY:
+ for (v = attrs[1]; data->attriddata[v] ; v += 2)
+ if (data->attriddata[v + 1] < attrdatastart)
+ attrdatastart = data->attriddata[v + 1];
+ /* FALLTHROUGH */
+ case REPOKEY_TYPE_IDARRAY:
+ case REPOKEY_TYPE_DIRNUMNUMARRAY:
+ if ((unsigned int)attrs[1] < attriddatastart)
+ attriddatastart = attrs[1];
+ break;
+ case REPOKEY_TYPE_FIXARRAY:
+ case REPOKEY_TYPE_FLEXARRAY:
+ return;
+ default:
+ break;
+ }
+ }
+ }
+#if 0
+ printf("compact_attrdata %d %d\n", entry, nentry);
+ printf("attrdatastart: %d\n", attrdatastart);
+ printf("attriddatastart: %d\n", attriddatastart);
+#endif
+ if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
+ return;
+ for (i = entry; i < nentry; i++)
+ {
+ Id v, *attrs = data->attrs[i];
+ if (!attrs)
+ continue;
+ for (; *attrs; attrs += 2)
+ {
+ switch (data->keys[*attrs].type)
+ {
+ case REPOKEY_TYPE_STR:
+ case REPOKEY_TYPE_BINARY:
+ case_CHKSUM_TYPES:
+ attrs[1] -= attrdatastart;
+ break;
+ case REPOKEY_TYPE_DIRSTRARRAY:
+ for (v = attrs[1]; data->attriddata[v] ; v += 2)
+ data->attriddata[v + 1] -= attrdatastart;
+ /* FALLTHROUGH */
+ case REPOKEY_TYPE_IDARRAY:
+ case REPOKEY_TYPE_DIRNUMNUMARRAY:
+ attrs[1] -= attriddatastart;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ if (attrdatastart)
+ {
+ data->attrdatalen -= attrdatastart;
+ memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
+ data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
+ }
+ if (attriddatastart)
+ {
+ data->attriddatalen -= attriddatastart;
+ memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
+ data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
+ }
+}
+
/* internalalize some key into incore/vincore data */
static void
case REPOKEY_TYPE_SHA1:
data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
break;
+ case REPOKEY_TYPE_SHA224:
+ data_addblob(xd, data->attrdata + val, SIZEOF_SHA224);
+ break;
case REPOKEY_TYPE_SHA256:
data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
break;
+ case REPOKEY_TYPE_SHA384:
+ data_addblob(xd, data->attrdata + val, SIZEOF_SHA384);
+ break;
+ case REPOKEY_TYPE_SHA512:
+ data_addblob(xd, data->attrdata + val, SIZEOF_SHA512);
+ break;
case REPOKEY_TYPE_NUM:
if (val & 0x80000000)
{
{
Repokey *key, solvkey;
Id entry, nentry;
- Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen;
+ Id schemaid, keyid, *schema, *sp, oldschema, *keyp, *seen;
+ int schemaidx;
unsigned char *dp, *ndp;
int newschema, oldcount;
struct extdata newincore;
if (!data->attrs && !data->xattrs)
return;
+#if 0
+ printf("repodata_internalize %d\n", data->repodataid);
+ printf(" attr data: %d K\n", data->attrdatalen / 1024);
+ printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
+#endif
newvincore.buf = data->vincore;
newvincore.len = data->vincorelen;
data->mainschema = schemaid;
data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
}
- keypstart = data->schemadata + data->schemata[schemaid];
- for (keyp = keypstart; *keyp; keyp++)
+ /* we don't use a pointer to the schemadata here as repodata_serialize_key
+ * may call repodata_schema2id() which might realloc our schemadata */
+ for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
{
if (entry == -1)
- data->mainschemaoffsets[keyp - keypstart] = newincore.len;
- if (*keyp == solvkeyid)
+ data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
+ if (keyid == solvkeyid)
{
/* add flexarray entry count */
data_addid(&newincore, data->end - data->start);
break;
}
- key = data->keys + *keyp;
+ key = data->keys + keyid;
#if 0
fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, pool_id2str(data->repo->pool, key->name), pool_id2str(data->repo->pool, key->type));
#endif
ndp = data_skip_key(data, dp, key);
oldcount--;
}
- if (seen[*keyp] == -1)
+ if (seen[keyid] == -1)
{
/* If this key was an old one _and_ was not overwritten with
a different value copy over the old value (we skipped it
above). */
if (dp != ndp)
data_addblob(&newincore, dp, ndp - dp);
- seen[*keyp] = 0;
+ seen[keyid] = 0;
}
- else if (seen[*keyp])
+ else if (seen[keyid])
{
- /* Otherwise we have a new value. Parse it into the internal
- form. */
- repodata_serialize_key(data, &newincore, &newvincore,
- schema, key, seen[*keyp] - 1);
+ /* Otherwise we have a new value. Parse it into the internal form. */
+ repodata_serialize_key(data, &newincore, &newvincore, schema, key, seen[keyid] - 1);
}
dp = ndp;
}
- if (entry >= 0 && data->attrs && data->attrs[entry])
- data->attrs[entry] = solv_free(data->attrs[entry]);
+ if (entry >= 0 && data->attrs)
+ {
+ if (data->attrs[entry])
+ data->attrs[entry] = solv_free(data->attrs[entry]);
+ if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
+ {
+ compact_attrdata(data, entry + 1, nentry); /* try to free some memory */
+#if 0
+ printf(" attr data: %d K\n", data->attrdatalen / 1024);
+ printf(" attrid data: %d K\n", data->attriddatalen / (1024 / 4));
+ printf(" incore data: %d K\n", newincore.len / 1024);
+ printf(" sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
+ /* malloc_stats(); */
+#endif
+ }
+ }
}
/* free all xattrs */
for (entry = 0; entry < data->nxattrs; entry++)
data->attrdatalen = 0;
data->attriddatalen = 0;
data->attrnum64datalen = 0;
+#if 0
+ printf("repodata_internalize %d done\n", data->repodataid);
+ printf(" incore data: %d K\n", data->incoredatalen / 1024);
+#endif
}
void
Pool *pool = repo->pool;
int r, i;
struct _Pool_tmpspace oldtmpspace;
+ Datapos oldpos;
if (!pool->loadcallback)
{
}
data->state = REPODATA_LOADING;
- /* save tmp space */
+ /* save tmp space and pos */
oldtmpspace = pool->tmpspace;
memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
+ oldpos = pool->pos;
r = pool->loadcallback(pool, data, pool->loadcallbackdata);
- /* restore tmp space */
+ /* restore tmp space and pos */
for (i = 0; i < POOL_TMPSPACEBUF; i++)
solv_free(pool->tmpspace.buf[i]);
pool->tmpspace = oldtmpspace;
+ if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
+ memset(&oldpos, 0, sizeof(oldpos));
+ pool->pos = oldpos;
data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR;
}
case REPOKEY_TYPE_NUM:
repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv));
break;
- case REPOKEY_TYPE_MD5:
- case REPOKEY_TYPE_SHA1:
- case REPOKEY_TYPE_SHA256:
+ case_CHKSUM_TYPES:
repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
break;
case REPOKEY_TYPE_IDARRAY:
return data->incoredatalen + data->vincorelen;
}
-/*
-vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4:
-*/
projects / platform / upstream / libsolv.git / blobdiff