- cleanup array eof handling, improve data_readid/readnum

diff --git a/src/repo_solv.c b/src/repo_solv.c
index ea6be4c..858fa75 100644 (file)
--- a/src/repo_solv.c
+++ b/src/repo_solv.c
@@ -328,6 +328,33 @@ incore_add_id(Repodata *data, Id sx)
 }
 
 static void
+incore_add_ideof(Repodata *data, Id sx, int eof)
+{
+  unsigned int x = (unsigned int)sx;
+  unsigned char *dp;
+  /* make sure we have at least 5 bytes free */
+  if (data->incoredatafree < 5)
+    {
+      data->incoredata = solv_realloc(data->incoredata, data->incoredatalen + INCORE_ADD_CHUNK);
+      data->incoredatafree = INCORE_ADD_CHUNK;
+    }
+  dp = data->incoredata + data->incoredatalen;
+  if (x >= (1 << 13))
+    {
+      if (x >= (1 << 27))
+       *dp++ = (x >> 27) | 128;
+      if (x >= (1 << 20))
+       *dp++ = (x >> 20) | 128;
+      *dp++ = (x >> 13) | 128;
+    }
+  if (x >= (1 << 6))
+    *dp++ = (x >> 6) | 128;
+  *dp++ = eof ? (x & 63) : (x & 63) | 64;
+  data->incoredatafree -= dp - (data->incoredata + data->incoredatalen);
+  data->incoredatalen = dp - data->incoredata;
+}
+
+static void
 incore_add_blob(Repodata *data, unsigned char *buf, int len)
 {
   if (data->incoredatafree < len)
@@ -358,11 +385,7 @@ incore_map_idarray(Repodata *data, unsigned char *dp, Id *map, Id max)
          break;
        }
       id = map[id];
-      if (id < 0)
-       abort();        /* for now */
-      if (id >= 64)
-       id = (id & 63) | ((id & ~63) << 1);
-      incore_add_id(data, eof ? id : id | 64);
+      incore_add_ideof(data, id, eof);
       if (eof)
        break;
     }
@@ -1086,7 +1109,7 @@ printf("=> %s %s %p\n", pool_id2str(pool, keys[key].name), pool_id2str(pool, key
              if (keys[key].storage != KEY_STORAGE_INCORE)
                break;
              if (idmap)
-               incore_map_idarray(&data, dps, idmap, numid);
+               incore_map_idarray(&data, dps, idmap, numid + numrel);
              else
                incore_add_blob(&data, dps, dp - dps);
              break;

diff --git a/src/repo_write.c b/src/repo_write.c
index b666d3b..2a03f28 100644 (file)
--- a/src/repo_write.c
+++ b/src/repo_write.c
@@ -7,12 +7,12 @@
 
 /*
  * repo_write.c
- * 
+ *
  * Write Repo data out to a file in solv format
- * 
- * See doc/README.format for a description 
+ *
+ * See doc/README.format for a description
  * of the binary file format
- * 
+ *
  */
 
 #include <sys/types.h>
@@ -43,9 +43,9 @@ typedef struct needid {
  * increment need Id
  * idarray: array of Ids, ID_NULL terminated
  * needid: array of Id->NeedId
- * 
+ *
  * return size of array (including trailing zero)
- * 
+ *
  */
 
 static void
@@ -92,7 +92,7 @@ incneedidarray(Pool *pool, Id *idarray, NeedId *needid)
 
 
 /*
- * 
+ *
  */
 
 static int
@@ -401,6 +401,7 @@ data_addid(struct extdata *xd, Id sx)
 {
   unsigned int x = (unsigned int)sx;
   unsigned char *dp;
+
   xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
   dp = xd->buf + xd->len;
 
@@ -419,28 +420,43 @@ data_addid(struct extdata *xd, Id sx)
 }
 
 static inline void
-data_addideof(struct extdata *xd, Id x, int eof)
+data_addideof(struct extdata *xd, Id sx, int eof)
 {
-  if (x >= 64)
-    x = (x & 63) | ((x & ~63) << 1);
-  data_addid(xd, (eof ? x: x | 64));
+  unsigned int x = (unsigned int)sx;
+  unsigned char *dp;
+
+  xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
+  dp = xd->buf + xd->len;
+
+  if (x >= (1 << 13))
+    {
+      if (x >= (1 << 27))
+        *dp++ = (x >> 27) | 128;
+      if (x >= (1 << 20))
+        *dp++ = (x >> 20) | 128;
+      *dp++ = (x >> 13) | 128;
+    }
+  if (x >= (1 << 6))
+    *dp++ = (x >> 6) | 128;
+  *dp++ = eof ? (x & 63) : (x & 63) | 64;
+  xd->len = dp - xd->buf;
 }
 
-static void 
+static void
 data_addid64(struct extdata *xd, unsigned int x, unsigned int hx)
 {
   if (hx)
-    {    
+    {
       if (hx > 7)
         {
           data_addid(xd, (Id)(hx >> 3));
-          xd->buf[xd->len - 1] |= 128; 
+          xd->buf[xd->len - 1] |= 128;
          hx &= 7;
         }
       data_addid(xd, (Id)(x | 0x80000000));
-      xd->buf[xd->len - 5] = (x >> 28) | (hx << 4) | 128; 
+      xd->buf[xd->len - 5] = (x >> 28) | (hx << 4) | 128;
     }
-  else 
+  else
     data_addid(xd, (Id)x);
 }
 
@@ -518,18 +534,14 @@ data_addidarray_sort(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id
        }
       /* XXX If difference is zero we have multiple equal elements,
         we might want to skip writing them out.  */
-      if (id >= 64)
-       id = (id & 63) | ((id & ~63) << 1);
-      data_addid(xd, id | 64);
+      data_addideof(xd, id, 0);
     }
   id = sids[i];
   if (id == marker)
     id = 0;
   else
     id = id - old + 1;
-  if (id >= 64)
-    id = (id & 63) | ((id & ~63) << 1);
-  data_addid(xd, id);
+  data_addideof(xd, id, 1);
   if (sids != lids)
     solv_free(sids);
 }
@@ -752,7 +764,7 @@ repo_write_adddata(struct cbdata *cbdata, Repodata *data, Repokey *key, KeyValue
   rm = cbdata->keymap[cbdata->keymapstart[data->repodataid] + (key - data->keys)];
   if (!rm)
     return SEARCH_NEXT_KEY;    /* we do not want this one */
-  
+
   if (cbdata->target->keys[rm].storage == KEY_STORAGE_VERTICAL_OFFSET)
     {
       xd = cbdata->extdata + rm;       /* vertical buffer */
@@ -1014,7 +1026,7 @@ repo_write_filtered(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *
   unsigned char *repodataused;
   int anyrepodataused = 0;
   int anysolvableused = 0;
-  
+
   struct cbdata cbdata;
   int clonepool;
   Repokey *key;
@@ -1750,7 +1762,7 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
   prefixcomp = solv_malloc(nstrings);
   compsum = 0;
   old_str = "";
-  
+
   prefixcomp[0] = 0;
   for (i = 1; i < nstrings; i++)
     {
@@ -1785,7 +1797,7 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
   /* Build the prefix-encoding of the string pool.  We need to know
      the size of that before writing it to the file, so we have to
      build a separate buffer for that.  As it's temporarily possible
-     that this actually is an expansion we can't easily reuse the 
+     that this actually is an expansion we can't easily reuse the
      stringspace for this.  The max expansion per string is 1 byte,
      so it will fit into sizeid+nstrings bytes.  */
   char *prefix = solv_malloc(sizeid + nstrings);

diff --git a/src/repodata.c b/src/repodata.c
index 1e87aba..8f1771b 100644 (file)
--- a/src/repodata.c
+++ b/src/repodata.c
@@ -1546,6 +1546,7 @@ dataiterator_step(Dataiterator *di)
              /* not an array */
              di->kv.id = *di->idp;
              di->kv.num = *di->idp;    /* for rpmdbid */
+             di->kv.num2 = 0;          /* for rpmdbid */
              di->kv.entry = 0;
              di->state = di_nextsolvablekey;
              break;
@@ -2469,8 +2470,8 @@ struct extdata {
 static void
 data_addid(struct extdata *xd, Id sx)
 {
-  unsigned char *dp;
   unsigned int x = (unsigned int)sx;
+  unsigned char *dp;
 
   xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
   dp = xd->buf + xd->len;
@@ -2507,13 +2508,26 @@ data_addid64(struct extdata *xd, unsigned long long x)
 }
 
 static void
-data_addideof(struct extdata *xd, Id x, int eof)
+data_addideof(struct extdata *xd, Id sx, int eof)
 {
-  if (x < 0)
-    abort();   /* XXX: not yet */
-  if (x >= 64)
-    x = (x & 63) | ((x & ~63) << 1);
-  data_addid(xd, (eof ? x : x | 64));
+  unsigned int x = (unsigned int)sx;
+  unsigned char *dp;
+
+  xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
+  dp = xd->buf + xd->len;
+
+  if (x >= (1 << 13))
+    {
+      if (x >= (1 << 27))
+        *dp++ = (x >> 27) | 128;
+      if (x >= (1 << 20))
+        *dp++ = (x >> 20) | 128;
+      *dp++ = (x >> 13) | 128;
+    }
+  if (x >= (1 << 6))
+    *dp++ = (x >> 6) | 128;
+  *dp++ = eof ? (x & 63) : (x & 63) | 64;
+  xd->len = dp - xd->buf;
 }
 
 static void
@@ -3008,7 +3022,7 @@ repodata_create_stubs(Repodata *data)
              xkey.name = xkeyname;
               xkey.type = di.kv.id;
               xkey.storage = KEY_STORAGE_INCORE;
-              xkey.size = 0; 
+              xkey.size = 0;
               repodata_key2id(sdata, &xkey, 1);
               xkeyname = 0;
            }

diff --git a/src/repopack.h b/src/repopack.h
index 6b76665..c551022 100644 (file)
--- a/src/repopack.h
+++ b/src/repopack.h
@@ -13,8 +13,35 @@
 static inline unsigned char *
 data_read_id(unsigned char *dp, Id *idp)
 {
-  Id x = 0;
+  Id x;
   unsigned char c;
+  if (!(dp[0] & 0x80))
+    {
+      *idp = dp[0];
+      return dp + 1;
+    }
+  if (!(dp[1] & 0x80))
+    {
+      *idp = dp[0] << 7 ^ dp[1] ^ 0x4000;
+      return dp + 2;
+    }
+  if (!(dp[2] & 0x80))
+    {
+      *idp = dp[0] << 14 ^ dp[1] << 7 ^ dp[2] ^ 0x204000;
+      return dp + 3;
+    }
+  if (!(dp[3] & 0x80))
+    {
+      *idp = dp[0] << 21 ^ dp[1] << 14 ^ dp[2] << 7 ^ dp[3] ^ 0x10204000;
+      return dp + 4;
+    }
+  x = dp[0] << 28 ^ dp[1] << 21 ^ dp[2] << 14 ^ dp[3] << 7 ^ dp[4] ^ 0x10204000;
+  if (!(dp[4] & 0x80))
+    {
+      *idp = x;
+      return dp + 5;
+    }
+  dp += 5;
   for (;;)
     {
       c = *dp++;
@@ -30,34 +57,49 @@ data_read_id(unsigned char *dp, Id *idp)
 static inline unsigned char *
 data_read_num64(unsigned char *dp, unsigned int *low, unsigned int *high)
 {
-  unsigned int x = 0;
-  unsigned long long int xx;
+  unsigned long long int x;
   unsigned char c;
-  int n;
 
   *high = 0;
-  for (n = 4; n-- > 0;)
+  if (!(dp[0] & 0x80))
     {
-      c = *dp++;
-      if (!(c & 0x80))
-       {
-         *low = (x << 7) ^ c;
-         return dp;
-       }
-      x = (x << 7) ^ (c ^ 128);
+      *low = dp[0];
+      return dp + 1;
+    }
+  if (!(dp[1] & 0x80))
+    {
+      *low = dp[0] << 7 ^ dp[1] ^ 0x4000;
+      return dp + 2;
     }
-  xx = x;
+  if (!(dp[2] & 0x80))
+    {
+      *low = dp[0] << 14 ^ dp[1] << 7 ^ dp[2] ^ 0x204000;
+      return dp + 3;
+    }
+  if (!(dp[3] & 0x80))
+    {
+      *low = dp[0] << 21 ^ dp[1] << 14 ^ dp[2] << 7 ^ dp[3] ^ 0x10204000;
+      return dp + 4;
+    }
+  if (!(dp[4] & 0x80))
+    {
+      *low = dp[0] << 28 ^ dp[1] << 21 ^ dp[2] << 14 ^ dp[3] << 7 ^ dp[4] ^ 0x10204000;
+      *high = (dp[0] ^ 0x80) >> 4;
+      return dp + 5;
+    }
+  x = (unsigned long long)(dp[0] ^ 0x80) << 28 ^ (unsigned int)(dp[1] << 21 ^ dp[2] << 14 ^ dp[3] << 7 ^ dp[4] ^ 0x10204000);
+  dp += 5;
   for (;;)
     {
       c = *dp++;
       if (!(c & 0x80))
        {
-         xx = (xx << 7) ^ c;
-         *low = xx;
-         *high = xx >> 32;
+         x = (x << 7) ^ c;
+         *low = x;
+         *high = x >> 32;
          return dp;
        }
-      xx = (xx << 7) ^ (c ^ 128);
+      x = (x << 7) ^ (c ^ 128);
     }
 }