- bring down memory usage by

  * reducing REPODATA_ATTRS_BLOCK from 63 to 31
  * freeing the string/rel hashes in tool_write
  * not creating an in-core prefix compressed string area

diff --git a/ext/repo_write.c b/ext/repo_write.c
index 33cdfd205bc16ac3089bf90d715eb005fce214ed..9cb4f5732ded9eac32b15818f441afd0e7d7a1e5 100644 (file)
--- a/ext/repo_write.c
+++ b/ext/repo_write.c
@@ -207,17 +207,15 @@ write_id_eof(FILE *fp, Id x, int eof)
 
 
 
-#if 0
-static void
+static inline void
 write_str(FILE *fp, const char *str)
 {
-  if (fputs (str, fp) == EOF || putc (0, fp) == EOF)
+  if (fputs(str, fp) == EOF || putc(0, fp) == EOF)
     {
-      perror("write error");
+      perror("write error str");
       exit(1);
     }
 }
-#endif
 
 /*
  * Array of Ids
@@ -251,7 +249,7 @@ write_idarray(FILE *fp, Pool *pool, NeedId *needid, Id *ids)
 }
 
 static int
-cmp_ids (const void *pa, const void *pb, void *dp)
+cmp_ids(const void *pa, const void *pb, void *dp)
 {
   Id a = *(Id *)pa;
   Id b = *(Id *)pb;
@@ -303,7 +301,7 @@ write_idarray_sort(FILE *fp, Pool *pool, NeedId *needid, Id *ids, Id marker)
     if (sids[i] == marker)
       break;
   if (i > 1)
-    sat_sort(sids, i, sizeof (Id), cmp_ids, 0);
+    sat_sort(sids, i, sizeof(Id), cmp_ids, 0);
   if ((len - i) > 2)
     sat_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
 
@@ -478,7 +476,7 @@ data_addidarray_sort(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id
     if (sids[i] == marker)
       break;
   if (i > 1)
-    sat_sort(sids, i, sizeof (Id), cmp_ids, 0);
+    sat_sort(sids, i, sizeof(Id), cmp_ids, 0);
   if ((len - i) > 2)
     sat_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
 
@@ -1746,6 +1744,44 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
   solv_flags |= SOLV_FLAG_PREFIX_POOL;
   write_u32(fp, solv_flags);
 
+  /*
+   * calculate prefix encoding of the strings
+   */
+  unsigned char *prefixcomp = sat_malloc(nstrings);
+  unsigned int compsum = 0;
+  char *old_str = "";
+  
+  prefixcomp[0] = 0;
+  for (i = 1; i < nstrings; i++)
+    {
+      char *str = spool->stringspace + spool->strings[needid[i].map];
+      int same;
+      for (same = 0; same < 255; same++)
+       if (!old_str[same] || old_str[same] != str[same])
+         break;
+      prefixcomp[i] = same;
+      compsum += same;
+      old_str = str;
+    }
+
+  /*
+   * write strings
+   */
+  write_u32(fp, sizeid);
+  /* we save compsum bytes but need 1 extra byte for every string */
+  write_u32(fp, sizeid + (nstrings ? nstrings - 1 : 0) - compsum);
+  if (sizeid + (nstrings ? nstrings - 1 : 0) != compsum)
+    {
+      for (i = 1; i < nstrings; i++)
+       {
+         char *str = spool->stringspace + spool->strings[needid[i].map];
+         write_u8(fp, prefixcomp[i]);
+         write_str(fp, str + prefixcomp[i]);
+       }
+    }
+  sat_free(prefixcomp);
+
+#if 0
   /* Build the prefix-encoding of the string pool.  We need to know
      the size of that before writing it to the file, so we have to
      build a separate buffer for that.  As it's temporarily possible
@@ -1765,7 +1801,7 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
          break;
       *pp++ = same;
       len = strlen(str + same) + 1;
-      memcpy (pp, str + same, len);
+      memcpy(pp, str + same, len);
       pp += len;
       old_str = str;
     }
@@ -1784,6 +1820,7 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
        }
     }
   sat_free(prefix);
+#endif
 
   /*
    * write RelDeps

diff --git a/src/repodata.c b/src/repodata.c
index 8844ea7d93b41fe4895aec7045d8148dc28a2440..1b657f7b9e533259a30084b62ffafb95132604fa 100644 (file)
--- a/src/repodata.c
+++ b/src/repodata.c
@@ -1768,7 +1768,7 @@ repodata_extend_block(Repodata *data, Id start, Id num)
 /**********************************************************************/
 
 
-#define REPODATA_ATTRS_BLOCK 63
+#define REPODATA_ATTRS_BLOCK 31
 #define REPODATA_ATTRDATA_BLOCK 1023
 #define REPODATA_ATTRIDDATA_BLOCK 63
 

diff --git a/tools/common_write.c b/tools/common_write.c
index bb4bd197668d17e17cce2d6e028b94d66cf0a97d..73067bdba50df95de06b54c5e9d83f77565c48be 100644 (file)
--- a/tools/common_write.c
+++ b/tools/common_write.c
@@ -92,22 +92,21 @@ keyfilter_attr(Repo *data, Repokey *key, void *kfdata)
 static int
 keyfilter_language(Repo *repo, Repokey *key, void *kfdata)
 {
+  Pool *pool = repo->pool;
   const char *name, *p;
-  char *lang = kfdata, *bname;
+  char *lang = kfdata;
   int i;
-  Id id;
 
   name = id2str(repo->pool, key->name);
   p = strrchr(name, ':');
   if (!p || strcmp(p + 1, lang) != 0)
     return KEY_STORAGE_DROPPED;
-  /* find base name id */
-  bname = strdup(name);
-  bname[p - name] = 0;
-  id = str2id(repo->pool, bname, 1);
   for (i = 0; verticals[i]; i++)
-    if (id == verticals[i])
-      return KEY_STORAGE_VERTICAL_OFFSET;
+    {
+      const char *vname = id2str(pool, verticals[i]);
+      if (!strncmp(name, vname, p - name) && vname[p - name] == 0)
+       return KEY_STORAGE_VERTICAL_OFFSET;
+    }
   return KEY_STORAGE_INCORE;
 }
 
@@ -213,6 +212,8 @@ tool_write(Repo *repo, const char *basename, const char *attrname)
     }
   sat_free(addedfileprovides);
 
+  pool_freeidhashes(repo->pool);       /* free some mem */
+
   if (basename)
     {
       char fn[4096];
@@ -302,7 +303,6 @@ tool_write(Repo *repo, const char *basename, const char *attrname)
        free(languages[i]);
       sat_free(languages);
       repodata_free(info);
-      repo->nrepodata--;
       return 0;
     }
   if (attrname)
@@ -316,6 +316,5 @@ tool_write(Repo *repo, const char *basename, const char *attrname)
   repodata_internalize(info);
   repo_write(repo, stdout, keyfilter_solv, &kd, 0);
   repodata_free(info);
-  repo->nrepodata--;
   return 0;
 }