Id lastlen;
int doingsolvables; /* working on solvables data */
+ int filelistmode;
};
#define NEEDED_BLOCK 1023
xd->len = dp - xd->buf;
}
+static inline int
+data_addideof_len(Id sx)
+{
+ unsigned int x = (unsigned int)sx;
+ if (x >= (1 << 13))
+ {
+ if (x >= (1 << 27))
+ return 5;
+ return x >= (1 << 20) ? 4 : 3;
+ }
+ return x >= (1 << 6) ? 2 : 1;
+}
+
static void
data_addid64(struct extdata *xd, unsigned int x, unsigned int hx)
{
case REPOKEY_TYPE_SHA1:
data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA1);
break;
+ case REPOKEY_TYPE_SHA224:
+ data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA224);
+ break;
case REPOKEY_TYPE_SHA256:
data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA256);
break;
+ case REPOKEY_TYPE_SHA384:
+ data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA384);
+ break;
+ case REPOKEY_TYPE_SHA512:
+ data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA512);
+ break;
case REPOKEY_TYPE_U32:
u32 = kv->num;
v[0] = u32 >> 24;
if (cbdata->owndirpool)
id = putinowndirpool(cbdata, data, &data->dirpool, id);
id = cbdata->dirused[id];
+ if (cbdata->filelistmode > 0)
+ {
+ xd->len += data_addideof_len(id) + strlen(kv->str) + 1;
+ break;
+ }
data_addideof(xd, id, kv->eof);
data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
+ if (cbdata->filelistmode < 0)
+ return 0;
break;
case REPOKEY_TYPE_FIXARRAY:
if (kv->eof == 0)
SOLVABLE_PKGID,
SOLVABLE_HDRID,
SOLVABLE_LEADSIGID,
+ SOLVABLE_CHANGELOG_AUTHOR,
+ SOLVABLE_CHANGELOG_TEXT,
0
};
}
/*
+ * return true if the repodata contains the filelist (and just
+ * the filelist). The same code is used in the dataiterator. The way
+ * it is used is completely wrong, of course, as having the filelist
+ * key does not mean it is used for a specific solvable. Nevertheless
+ * it is better to have it than to write broken solv files.
+ */
+static inline int
+is_filelist_extension(Repodata *data)
+{
+ int j;
+ for (j = 1; j < data->nkeys; j++)
+ if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
+ return 0;
+ return 1;
+}
+
+
+static int
+write_compressed_extdata(Repodata *target, struct extdata *xd, unsigned char *vpage, int lpage)
+{
+ unsigned char *dp = xd->buf;
+ int l = xd->len;
+ while (l)
+ {
+ int ll = REPOPAGE_BLOBSIZE - lpage;
+ if (l < ll)
+ ll = l;
+ memcpy(vpage + lpage, dp, ll);
+ dp += ll;
+ lpage += ll;
+ l -= ll;
+ if (lpage == REPOPAGE_BLOBSIZE)
+ {
+ write_compressed_page(target, vpage, lpage);
+ lpage = 0;
+ }
+ }
+ return lpage;
+}
+
+/*
* Repo
*/
repo_write_filtered(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
{
Pool *pool = repo->pool;
- int i, j, n;
+ int i, j, n, lastfilelistn;
Solvable *s;
NeedId *needid;
int nstrings, nrels;
Id type_constantid = REPOKEY_TYPE_CONSTANTID;
- unsigned char *prefixcomp;
- unsigned int compsum;
- char *old_str;
-
memset(&cbdata, 0, sizeof(cbdata));
cbdata.repo = repo;
dirpool = 0;
dirpooldata = 0;
n = ID_NUM_INTERNAL;
+ lastfilelistn = 0;
FOR_REPODATAS(repo, i, data)
{
cbdata.keymapstart[i] = n;
idused = 1; /* dirs also use ids */
dirused = 1;
}
+ if (key->type == REPOKEY_TYPE_DIRSTRARRAY && key->name == SOLVABLE_FILELIST)
+ {
+ /* is this a file list extension */
+ if (is_filelist_extension(data))
+ {
+ /* hmm, we have a file list extension. Kill filelist of other repodata.
+ * XXX: this is wrong, as the extension does not need to cover all
+ * solvables of the other repodata */
+ if (lastfilelistn)
+ cbdata.keymap[lastfilelistn] = 0;
+ }
+ else
+ lastfilelistn = n;
+ }
}
if (idused)
{
{
data_addid(xd, repo->nsolvables); /* FLEXARRAY nentries */
cbdata.doingsolvables = 1;
+
+ /* check if we can do the special filelist memory optimization */
+ if (anyrepodataused)
+ {
+ for (i = 1; i < target.nkeys; i++)
+ if (target.keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
+ cbdata.filelistmode |= cbdata.filelistmode == 0 && target.keys[i].type == REPOKEY_TYPE_DIRSTRARRAY ? 1 : 2;
+ else if (target.keys[i].type == REPOKEY_TYPE_DIRSTRARRAY)
+ cbdata.filelistmode = 2;
+ if (cbdata.filelistmode != 1)
+ cbdata.filelistmode = 0;
+ }
+
for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
{
if (s->repo != repo)
solv_flags |= SOLV_FLAG_SIZE_BYTES;
write_u32(&target, solv_flags);
- /*
- * calculate prefix encoding of the strings
- */
- prefixcomp = solv_malloc(nstrings);
- compsum = 0;
- old_str = "";
-
- prefixcomp[0] = 0;
- for (i = 1; i < nstrings; i++)
+ if (nstrings)
{
- char *str = spool->stringspace + spool->strings[needid[i].map];
- int same;
- for (same = 0; same < 255; same++)
- if (!old_str[same] || old_str[same] != str[same])
- break;
- prefixcomp[i] = same;
- compsum += same;
- old_str = str;
- }
+ /*
+ * calculate prefix encoding of the strings
+ */
+ unsigned char *prefixcomp = solv_malloc(nstrings);
+ unsigned int compsum = 0;
+ char *old_str = "";
+
+ prefixcomp[0] = 0;
+ for (i = 1; i < nstrings; i++)
+ {
+ char *str = spool->stringspace + spool->strings[needid[i].map];
+ int same;
+ for (same = 0; same < 255; same++)
+ if (!old_str[same] || old_str[same] != str[same])
+ break;
+ prefixcomp[i] = same;
+ compsum += same;
+ old_str = str;
+ }
- /*
- * write strings
- */
- write_u32(&target, sizeid);
- /* we save compsum bytes but need 1 extra byte for every string */
- write_u32(&target, sizeid + (nstrings ? nstrings - 1 : 0) - compsum);
- if (sizeid + (nstrings ? nstrings - 1 : 0) != compsum)
- {
+ /*
+ * write strings
+ */
+ write_u32(&target, sizeid);
+ /* we save compsum bytes but need 1 extra byte for every string */
+ write_u32(&target, sizeid + nstrings - 1 - compsum);
for (i = 1; i < nstrings; i++)
{
char *str = spool->stringspace + spool->strings[needid[i].map];
write_u8(&target, prefixcomp[i]);
write_str(&target, str + prefixcomp[i]);
}
+ solv_free(prefixcomp);
}
- solv_free(prefixcomp);
-
-#if 0
- /* Build the prefix-encoding of the string pool. We need to know
- the size of that before writing it to the file, so we have to
- build a separate buffer for that. As it's temporarily possible
- that this actually is an expansion we can't easily reuse the
- stringspace for this. The max expansion per string is 1 byte,
- so it will fit into sizeid+nstrings bytes. */
- char *prefix = solv_malloc(sizeid + nstrings);
- char *pp = prefix;
- char *old_str = "";
- for (i = 1; i < nstrings; i++)
+ else
{
- char *str = spool->stringspace + spool->strings[needid[i].map];
- int same;
- size_t len;
- for (same = 0; same < 255; same++)
- if (!old_str[same] || !str[same] || old_str[same] != str[same])
- break;
- *pp++ = same;
- len = strlen(str + same) + 1;
- memcpy(pp, str + same, len);
- pp += len;
- old_str = str;
+ write_u32(&target, 0);
+ write_u32(&target, 0);
}
/*
- * write strings
- */
- write_u32(&target, sizeid);
- write_u32(&target, pp - prefix);
- write_blob(&target, prefix, pp - prefix);
- solv_free(prefix);
-#endif
-
- /*
* write RelDeps
*/
for (i = 0; i < nrels; i++)
if (i < target.nkeys)
{
/* yes, write it in pages */
- unsigned char *dp, vpage[REPOPAGE_BLOBSIZE];
- int l, ll, lpage = 0;
+ unsigned char vpage[REPOPAGE_BLOBSIZE];
+ int lpage = 0;
write_u32(&target, REPOPAGE_BLOBSIZE);
for (i = 1; i < target.nkeys; i++)
+ if (cbdata.extdata[i].len)
+ {
+ if (cbdata.filelistmode)
+ break;
+ lpage = write_compressed_extdata(&target, cbdata.extdata + i, vpage, lpage);
+ }
+ if (cbdata.filelistmode && i < target.nkeys)
{
- if (!cbdata.extdata[i].len)
- continue;
- l = cbdata.extdata[i].len;
- dp = cbdata.extdata[i].buf;
- while (l)
+ /* ok, just this single extdata, which is a filelist */
+ xd = cbdata.extdata + i;
+ xd->len = 0;
+ cbdata.filelistmode = -1;
+ for (j = 0; j < cbdata.nkeymap; j++)
+ if (cbdata.keymap[j] != i)
+ cbdata.keymap[j] = 0;
+ for (i = repo->start, s = pool->solvables + i; i < repo->end; i++, s++)
{
- ll = REPOPAGE_BLOBSIZE - lpage;
- if (l < ll)
- ll = l;
- memcpy(vpage + lpage, dp, ll);
- dp += ll;
- lpage += ll;
- l -= ll;
- if (lpage == REPOPAGE_BLOBSIZE)
+ if (s->repo != repo)
+ continue;
+ FOR_REPODATAS(repo, j, data)
+ {
+ if (!repodataused[j])
+ continue;
+ if (i < data->start || i >= data->end)
+ continue;
+ repodata_search(data, i, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_adddata, &cbdata);
+ }
+ if (xd->len > 1024 * 1024)
{
- write_compressed_page(&target, vpage, lpage);
- lpage = 0;
+ lpage = write_compressed_extdata(&target, xd, vpage, lpage);
+ xd->len = 0;
}
}
+ if (xd->len)
+ lpage = write_compressed_extdata(&target, xd, vpage, lpage);
}
if (lpage)
write_compressed_page(&target, vpage, lpage);
projects / platform / upstream / libsolv.git / blobdiff