use less memory when just writing the filelist

diff --git a/src/repo_write.c b/src/repo_write.c
index 2900b44..11002b4 100644 (file)
--- a/src/repo_write.c
+++ b/src/repo_write.c
@@ -397,6 +397,7 @@ struct cbdata {
   Id lastlen;
 
   int doingsolvables;  /* working on solvables data */
+  int filelistmode;
 };
 
 #define NEEDED_BLOCK 1023
@@ -450,6 +451,19 @@ data_addideof(struct extdata *xd, Id sx, int eof)
   xd->len = dp - xd->buf;
 }
 
+static inline int
+data_addideof_len(Id sx)
+{
+  unsigned int x = (unsigned int)sx;
+  if (x >= (1 << 13))
+    {
+      if (x >= (1 << 27))
+       return 5;
+      return x >= (1 << 20) ? 4 : 3;
+    }
+  return x >= (1 << 6) ? 2 : 1;
+}
+
 static void
 data_addid64(struct extdata *xd, unsigned int x, unsigned int hx)
 {
@@ -861,8 +875,15 @@ repo_write_adddata(struct cbdata *cbdata, Repodata *data, Repokey *key, KeyValue
        if (cbdata->owndirpool)
          id = putinowndirpool(cbdata, data, &data->dirpool, id);
        id = cbdata->dirused[id];
+       if (cbdata->filelistmode > 0)
+         {
+           xd->len += data_addideof_len(id) + strlen(kv->str) + 1;
+           break;
+         }
        data_addideof(xd, id, kv->eof);
        data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
+       if (cbdata->filelistmode < 0)
+         return 0;
        break;
       case REPOKEY_TYPE_FIXARRAY:
        if (kv->eof == 0)
@@ -1032,6 +1053,30 @@ is_filelist_extension(Repodata *data)
   return 1;
 }
 
+
+static int
+write_compressed_extdata(Repodata *target, struct extdata *xd, unsigned char *vpage, int lpage)
+{
+  unsigned char *dp = xd->buf;
+  int l = xd->len;
+  while (l)
+    {
+      int ll = REPOPAGE_BLOBSIZE - lpage;
+      if (l < ll)
+       ll = l;
+      memcpy(vpage + lpage, dp, ll);
+      dp += ll;
+      lpage += ll;
+      l -= ll;
+      if (lpage == REPOPAGE_BLOBSIZE)
+       {
+         write_compressed_page(target, vpage, lpage);
+         lpage = 0;
+       }
+    }
+  return lpage;
+}
+
 /*
  * Repo
  */
@@ -1731,6 +1776,19 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
     {
       data_addid(xd, repo->nsolvables);        /* FLEXARRAY nentries */
       cbdata.doingsolvables = 1;
+
+      /* check if we can do the special filelist memory optimization */
+      if (anyrepodataused)
+       {
+         for (i = 1; i < target.nkeys; i++)
+           if (target.keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
+             cbdata.filelistmode |= cbdata.filelistmode == 0 && target.keys[i].type == REPOKEY_TYPE_DIRSTRARRAY ? 1 : 2;
+           else if (target.keys[i].type == REPOKEY_TYPE_DIRSTRARRAY)
+             cbdata.filelistmode = 2;
+         if (cbdata.filelistmode != 1)
+           cbdata.filelistmode = 0;
+       }
+
       for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
        {
          if (s->repo != repo)
@@ -1918,31 +1976,46 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
   if (i < target.nkeys)
     {
       /* yes, write it in pages */
-      unsigned char *dp, vpage[REPOPAGE_BLOBSIZE];
-      int l, ll, lpage = 0;
+      unsigned char vpage[REPOPAGE_BLOBSIZE];
+      int lpage = 0;
 
       write_u32(&target, REPOPAGE_BLOBSIZE);
       for (i = 1; i < target.nkeys; i++)
+       if (cbdata.extdata[i].len)
+         {
+           if (cbdata.filelistmode)
+             break;
+           lpage = write_compressed_extdata(&target, cbdata.extdata + i, vpage, lpage);
+         }
+      if (cbdata.filelistmode && i < target.nkeys)
        {
-         if (!cbdata.extdata[i].len)
-           continue;
-         l = cbdata.extdata[i].len;
-         dp = cbdata.extdata[i].buf;
-         while (l)
+         /* ok, just this single extdata, which is a filelist */
+         xd = cbdata.extdata + i;
+         xd->len = 0;
+         cbdata.filelistmode = -1;
+         for (j = 0; j < cbdata.nkeymap; j++)
+           if (cbdata.keymap[j] != i)
+             cbdata.keymap[j] = 0;
+         for (i = repo->start, s = pool->solvables + i; i < repo->end; i++, s++)
            {
-             ll = REPOPAGE_BLOBSIZE - lpage;
-             if (l < ll)
-               ll = l;
-             memcpy(vpage + lpage, dp, ll);
-             dp += ll;
-             lpage += ll;
-             l -= ll;
-             if (lpage == REPOPAGE_BLOBSIZE)
+             if (s->repo != repo)
+               continue;
+             FOR_REPODATAS(repo, j, data)
+               {
+                 if (!repodataused[j])
+                   continue;
+                 if (i < data->start || i >= data->end)
+                   continue;
+                 repodata_search(data, i, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_adddata, &cbdata);
+               }
+             if (xd->len > 1024 * 1024)
                {
-                 write_compressed_page(&target, vpage, lpage);
-                 lpage = 0;
+                 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
+                 xd->len = 0;
                }
            }
+         if (xd->len)
+           lpage = write_compressed_extdata(&target, xd, vpage, lpage);
        }
       if (lpage)
        write_compressed_page(&target, vpage, lpage);