#include "repo_write.h"
#include "repopage.h"
+#undef USE_IDARRAYBLOCK
+#define USE_REL_IDARRAY
+
/*------------------------------------------------------------------*/
/* Id map optimizations */
}
}
+static void
+write_compressed_blob(Repodata *data, void *blob, int len)
+{
+ unsigned char cpage[65536];
+ if (data->error)
+ return;
+ while (len > 0)
+ {
+ int chunk = len > sizeof(cpage) ? sizeof(cpage) : len;
+ int flag = (chunk == len ? 0x80 : 0x00);
+ int clen = repopagestore_compress_page(blob, chunk, cpage, sizeof(cpage) - 1);
+ if (!clen)
+ {
+ write_u8(data, flag);
+ write_u8(data, chunk >> 8);
+ write_u8(data, chunk);
+ write_blob(data, blob, chunk);
+ }
+ else
+ {
+ write_u8(data, flag | 0x40);
+ write_u8(data, clen >> 8);
+ write_u8(data, clen);
+ write_blob(data, cpage, clen);
+ }
+ blob += chunk;
+ len -= chunk;
+ }
+}
+
/*
* Id
*/
}
static inline void
-write_id_eof(Repodata *data, Id x, int eof)
-{
- if (x >= 64)
- x = (x & 63) | ((x & ~63) << 1);
- write_id(data, x | (eof ? 0 : 64));
-}
-
-
-
-static inline void
write_str(Repodata *data, const char *str)
{
if (data->error)
int len;
};
+#define DIRIDCACHE_SIZE 1024
+
struct cbdata {
Pool *pool;
Repo *repo;
Id lastdirid; /* last dir id seen in this repodata */
Id lastdirid_own; /* last dir id put in own pool */
+
+ Id diridcache[3 * DIRIDCACHE_SIZE];
};
#define NEEDID_BLOCK 1023
data_addid(xd, (Id)x);
}
-#define USE_REL_IDARRAY
#ifdef USE_REL_IDARRAY
static int
Id lids[64], *sids;
Id id, old;
- if (!ids)
- return;
- if (!*ids)
+ if (!ids || !*ids)
{
- data_addid(xd, 0);
+ data_addideof(xd, 0, 1);
return;
}
for (len = 0; len < 64 && ids[len]; len++)
#else
+#ifdef USE_IDARRAYBLOCK
+
+static void
+data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
+{
+ Id id;
+ Id last = 0, tmp;
+ if (!ids || !*ids)
+ {
+ data_addideof(xd, 0, 1);
+ return;
+ }
+ while ((id = *ids++) != 0)
+ {
+ if (needid)
+ id = needid[NEEDIDOFF(id)].need;
+ tmp = id;
+ if (id < last)
+ id = (last - id) * 2 - 1; /* [1, 2 * last - 1] odd */
+ else if (id < 2 * last)
+ id = (id - last) * 2; /* [0, 2 * last - 2] even */
+ last = tmp;
+ data_addideof(xd, id, *ids ? 0 : 1);
+ }
+}
+
+#else
+
static void
data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
{
Id id;
if (!ids || !*ids)
{
- data_addid(xd, 0);
+ data_addideof(xd, 0, 1);
return;
}
while ((id = *ids++) != 0)
#endif
+#endif
+
static inline void
data_addblob(struct extdata *xd, unsigned char *blob, int len)
{
static Id
putinowndirpool_slow(struct cbdata *cbdata, Repodata *data, Dirpool *dp, Id dir)
{
- Id compid, parent;
+ Id compid, parent, id;
+ Id *cacheent;
parent = dirpool_parent(dp, dir);
if (parent)
- parent = putinowndirpool_slow(cbdata, data, dp, parent);
+ {
+ /* put parent in own pool first */
+ cacheent = cbdata->diridcache + (parent & (DIRIDCACHE_SIZE - 1));
+ if (cacheent[0] == parent && cacheent[DIRIDCACHE_SIZE] == data->repodataid)
+ parent = cacheent[2 * DIRIDCACHE_SIZE];
+ else
+ parent = putinowndirpool_slow(cbdata, data, dp, parent);
+ }
compid = dirpool_compid(dp, dir);
if (cbdata->ownspool && compid > 1 && (!cbdata->clonepool || data->localpool))
compid = putinownpool(cbdata, data, compid);
- return dirpool_add_dir(cbdata->owndirpool, parent, compid, 1);
+ id = dirpool_add_dir(cbdata->owndirpool, parent, compid, 1);
+ /* cache result */
+ cacheent = cbdata->diridcache + (dir & (DIRIDCACHE_SIZE - 1));
+ cacheent[0] = dir;
+ cacheent[DIRIDCACHE_SIZE] = data->repodataid;
+ cacheent[2 * DIRIDCACHE_SIZE] = id;
+ return id;
}
static inline Id
putinowndirpool(struct cbdata *cbdata, Repodata *data, Id dir)
{
+ Id *cacheent;
if (dir && dir == cbdata->lastdirid)
return cbdata->lastdirid_own;
+ cacheent = cbdata->diridcache + (dir & (DIRIDCACHE_SIZE - 1));
+ if (dir && cacheent[0] == dir && cacheent[DIRIDCACHE_SIZE] == data->repodataid)
+ return cacheent[2 * DIRIDCACHE_SIZE];
cbdata->lastdirid = dir;
cbdata->lastdirid_own = putinowndirpool_slow(cbdata, data, &data->dirpool, dir);
return cbdata->lastdirid_own;
break;
case REPOKEY_TYPE_FIXARRAY:
case REPOKEY_TYPE_FLEXARRAY:
- if (kv->entry == 0)
- {
- if (kv->eof != 2)
- *cbdata->sp++ = 0; /* mark start */
- }
- else
+ if (kv->entry)
{
- /* just finished a schema, rewind to start */
+ /* finish schema, rewind to start */
Id *sp = cbdata->sp - 1;
*sp = 0;
while (sp[-1])
sp--;
- if (kv->entry == 1 || key->type == REPOKEY_TYPE_FLEXARRAY)
+ if (sp[-2] >= 0)
+ cbdata->subschemata[sp[-2]] = repodata_schema2id(cbdata->target, sp, 1);
+ cbdata->sp = sp - 2;
+ }
+ if (kv->eof != 2)
+ {
+ /* start new schema */
+ if (kv->entry == 0 || key->type == REPOKEY_TYPE_FLEXARRAY)
{
cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
- cbdata->subschemata[cbdata->nsubschemata++] = repodata_schema2id(cbdata->target, sp, 1);
+ *cbdata->sp++ = cbdata->nsubschemata++;
}
- cbdata->sp = kv->eof == 2 ? sp - 1: sp;
- }
+ else
+ *cbdata->sp++ = -1;
+ *cbdata->sp++ = 0;
+ }
break;
default:
break;
Repo *repo = s->repo;
Pool *pool = repo->pool;
struct extdata *xd = cbdata->extdata;
+#ifdef USE_IDARRAYBLOCK
+ struct extdata *xda = xd + cbdata->target->nkeys; /* idarray block */
+#else
+ struct extdata *xda = xd;
+#endif
+
NeedId *needid = cbdata->needid;
Id *idarraydata = repo->idarraydata;
if (s->vendor && keymap[SOLVABLE_VENDOR])
data_addid(xd, needid[s->vendor].need);
if (s->provides && keymap[SOLVABLE_PROVIDES])
- data_adddepids(xd, pool, needid, idarraydata + s->provides, SOLVABLE_FILEMARKER);
+ data_adddepids(xda, pool, needid, idarraydata + s->provides, SOLVABLE_FILEMARKER);
if (s->obsoletes && keymap[SOLVABLE_OBSOLETES])
- data_adddepids(xd, pool, needid, idarraydata + s->obsoletes, 0);
+ data_adddepids(xda, pool, needid, idarraydata + s->obsoletes, 0);
if (s->conflicts && keymap[SOLVABLE_CONFLICTS])
- data_adddepids(xd, pool, needid, idarraydata + s->conflicts, 0);
+ data_adddepids(xda, pool, needid, idarraydata + s->conflicts, 0);
if (s->requires && keymap[SOLVABLE_REQUIRES])
- data_adddepids(xd, pool, needid, idarraydata + s->requires, SOLVABLE_PREREQMARKER);
+ data_adddepids(xda, pool, needid, idarraydata + s->requires, SOLVABLE_PREREQMARKER);
if (s->recommends && keymap[SOLVABLE_RECOMMENDS])
- data_adddepids(xd, pool, needid, idarraydata + s->recommends, 0);
+ data_adddepids(xda, pool, needid, idarraydata + s->recommends, 0);
if (s->suggests && keymap[SOLVABLE_SUGGESTS])
- data_adddepids(xd, pool, needid, idarraydata + s->suggests, 0);
+ data_adddepids(xda, pool, needid, idarraydata + s->suggests, 0);
if (s->supplements && keymap[SOLVABLE_SUPPLEMENTS])
- data_adddepids(xd, pool, needid, idarraydata + s->supplements, 0);
+ data_adddepids(xda, pool, needid, idarraydata + s->supplements, 0);
if (s->enhances && keymap[SOLVABLE_ENHANCES])
- data_adddepids(xd, pool, needid, idarraydata + s->enhances, 0);
+ data_adddepids(xda, pool, needid, idarraydata + s->enhances, 0);
if (repo->rpmdbid && keymap[RPM_RPMDBID])
data_addid(xd, repo->rpmdbid[(s - pool->solvables) - repo->start]);
}
SOLVABLE_LEADSIGID,
SOLVABLE_CHANGELOG_AUTHOR,
SOLVABLE_CHANGELOG_TEXT,
+ SOLVABLE_SIGNATUREDATA,
0
};
Repowriter *
repowriter_free(Repowriter *writer)
{
+ solv_free(writer->userdata);
return solv_free(writer);
}
writer->solvableend = solvableend;
}
+void
+repowriter_set_userdata(Repowriter *writer, const void *data, int len)
+{
+ writer->userdata = solv_free(writer->userdata);
+ writer->userdatalen = 0;
+ if (len <= 0)
+ return;
+ writer->userdata = solv_memdup(data, len);
+ writer->userdatalen = len;
+}
+
/*
* the code works the following way:
*
Id type_constantid = 0;
+ /* sanity checks */
+ if (writer->userdatalen < 0 || writer->userdatalen >= 65536)
+ return pool_error(pool, -1, "illegal userdata length: %d", writer->userdatalen);
memset(&cbdata, 0, sizeof(cbdata));
cbdata.pool = pool;
if (i < SOLVABLE_PROVIDES)
keyd.type = REPOKEY_TYPE_ID;
else if (i < RPM_RPMDBID)
-#ifdef USE_REL_IDARRAY
- keyd.type = REPOKEY_TYPE_REL_IDARRAY;
-#else
keyd.type = REPOKEY_TYPE_IDARRAY;
-#endif
else
keyd.type = REPOKEY_TYPE_NUM;
+#ifdef USE_REL_IDARRAY
+ if (keyd.type == REPOKEY_TYPE_IDARRAY)
+ keyd.type = REPOKEY_TYPE_REL_IDARRAY;
+#endif
keyd.size = 0;
keyd.storage = KEY_STORAGE_SOLVABLE;
if (writer->keyfilter)
continue;
keyd.storage = KEY_STORAGE_SOLVABLE;
}
+#ifdef USE_IDARRAYBLOCK
+ if (keyd.type == REPOKEY_TYPE_IDARRAY)
+ keyd.storage = KEY_STORAGE_IDARRAYBLOCK;
+#endif
poolusage = 1;
clonepool = 1;
keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
keymap[n] = 0;
continue;
}
+ if (keyd.storage != KEY_STORAGE_VERTICAL_OFFSET)
+ keyd.storage = KEY_STORAGE_INCORE; /* do not mess with us */
}
if (data->state != REPODATA_STUB)
id = repodata_key2id(&target, &keyd, 1);
reloff += 3 * target.nkeys;
}
- needid = calloc(reloff + pool->nrels, sizeof(*needid));
+ needid = solv_calloc(reloff + pool->nrels, sizeof(*needid));
needid[0].map = reloff; /* remember size in case we need to grow */
cbdata.needid = needid;
/* collect all data
* we use extdata[0] for incore data and extdata[keyid] for vertical data
+ * we use extdata[nkeys] for the idarray_block data
*
* this must match the code above that creates the schema data!
*/
- cbdata.extdata = solv_calloc(target.nkeys, sizeof(struct extdata));
+ cbdata.extdata = solv_calloc(target.nkeys + 1, sizeof(struct extdata));
xd = cbdata.extdata;
cbdata.current_sub = 0;
target.fp = fp;
/* write header */
+ solv_flags = 0;
+ solv_flags |= SOLV_FLAG_PREFIX_POOL;
+ solv_flags |= SOLV_FLAG_SIZE_BYTES;
+ if (writer->userdatalen)
+ solv_flags |= SOLV_FLAG_USERDATA;
+ if (cbdata.extdata[target.nkeys].len)
+ solv_flags |= SOLV_FLAG_IDARRAYBLOCK;
/* write file header */
write_u32(&target, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
- write_u32(&target, SOLV_VERSION_8);
-
+ if ((solv_flags & (SOLV_FLAG_USERDATA | SOLV_FLAG_IDARRAYBLOCK)) != 0)
+ write_u32(&target, SOLV_VERSION_9);
+ else
+ write_u32(&target, SOLV_VERSION_8);
/* write counts */
write_u32(&target, nstrings);
write_u32(&target, anysolvableused ? nsolvables : 0);
write_u32(&target, target.nkeys);
write_u32(&target, target.nschemata);
- solv_flags = 0;
- solv_flags |= SOLV_FLAG_PREFIX_POOL;
- solv_flags |= SOLV_FLAG_SIZE_BYTES;
write_u32(&target, solv_flags);
+ /* write userdata */
+ if ((solv_flags & SOLV_FLAG_USERDATA) != 0)
+ {
+ write_u32(&target, writer->userdatalen);
+ write_blob(&target, writer->userdata, writer->userdatalen);
+ }
+
if (nstrings)
{
/*
}
else
{
- write_u32(&target, 0);
- write_u32(&target, 0);
+ write_u32(&target, 0); /* unpacked size */
+ write_u32(&target, 0); /* compressed size */
}
/*
for (i = 1; i < target.nschemata; i++)
write_idarray(&target, pool, 0, repodata_id2schema(&target, i));
+ /* write idarray_block data if not empty */
+ if (cbdata.extdata[target.nkeys].len)
+ {
+ unsigned int cnt = 0;
+ unsigned char *b;
+ unsigned int l;
+
+ xd = cbdata.extdata + target.nkeys;
+ /* calculate number of entries */
+ for (l = xd->len, b = xd->buf; l--;)
+ {
+ unsigned char x = *b++;
+ if ((x & 0x80) == 0)
+ cnt += (x & 0x40) ? 1 : 2;
+ }
+ write_id(&target, cnt);
+ if (cnt)
+ write_compressed_blob(&target, xd->buf, xd->len);
+ solv_free(xd->buf);
+ }
+
/*
* write incore data
*/
+ xd = cbdata.extdata;
write_id(&target, cbdata.maxdata);
- write_id(&target, cbdata.extdata[0].len);
- if (cbdata.extdata[0].len)
- write_blob(&target, cbdata.extdata[0].buf, cbdata.extdata[0].len);
- solv_free(cbdata.extdata[0].buf);
+ write_id(&target, xd->len);
+ if (xd->len)
+ write_blob(&target, xd->buf, xd->len);
+ solv_free(xd->buf);
/*
* write vertical data if we have any
projects / platform / upstream / libsolv.git / blobdiff