From: Michael Schroeder <mls@suse.de>
Date: Tue, 4 Dec 2007 17:44:31 +0000 (+0000)
Subject: - switch to new solv format using schemata
X-Git-Tag: BASE-SuSE-Code-12_1-Branch~1082
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=709c648013b2898308b51d5b08161db0fb65bd4c;p=platform%2Fupstream%2Flibsolv.git

- switch to new solv format using schemata
  please regenerate all solv files...
- split whatprovides array into whatprovides and whatprovides_rel so
  that we can add new ids while keeping the whatprovides information
- start to implement some search/lookup functions in repo.c
  needs quite a bit more love...
---

diff --git a/src/pool.c b/src/pool.c
index bf71d93..6c997c4 100644
--- a/src/pool.c
+++ b/src/pool.c
@@ -28,17 +28,6 @@
 
 #define SOLVABLE_BLOCK	255
 
-// reset all whatprovides
-// 
-void
-pool_freewhatprovides(Pool *pool)
-{
-  pool->whatprovides = xfree(pool->whatprovides);
-  pool->whatprovidesdata = xfree(pool->whatprovidesdata);
-  pool->whatprovidesdataoff = 0;
-  pool->whatprovidesdataleft = 0;
-}
-
 
 // list of string constants, so we can do pointer/Id instead of string comparison
 // index into array matches ID_xxx constants in pool.h
@@ -71,8 +60,7 @@ static const char *initpool_data[] = {
   0
 };
 
-// create pool
-// 
+/* create pool */
 Pool *
 pool_create(void)
 {
@@ -101,8 +89,7 @@ pool_create(void)
 }
 
 
-// empty the pool
-// 
+/* free all the resources of our pool */
 void
 pool_free(Pool *pool)
 {
@@ -204,6 +191,12 @@ pool_shrink_whatprovides_sortcmp(const void *ap, const void *bp)
   return *(Id *)ap - *(Id *)bp;
 }
 
+/*
+ * pool_shrink_whatprovides  - unify whatprovides data
+ *
+ * whatprovides_rel must be empty for this to work!
+ *
+ */
 static void
 pool_shrink_whatprovides(Pool *pool)
 {
@@ -287,10 +280,9 @@ pool_shrink_whatprovides(Pool *pool)
 /*
  * pool_createwhatprovides()
  * 
- * create hashes over complete pool to ease lookups
+ * create hashes over pool of solvables to ease provide lookups
  * 
  */
-
 void
 pool_createwhatprovides(Pool *pool)
 {
@@ -305,10 +297,11 @@ pool_createwhatprovides(Pool *pool)
   POOL_DEBUG(SAT_DEBUG_STATS, "number of solvables: %d\n", pool->nsolvables);
   POOL_DEBUG(SAT_DEBUG_STATS, "number of ids: %d + %d\n", pool->ss.nstrings, pool->nrels);
 
-  pool_freeidhashes(pool);
+  pool_freeidhashes(pool);	/* XXX: should not be here! */
   pool_freewhatprovides(pool);
-  num = pool->ss.nstrings + pool->nrels;
-  whatprovides = (Offset *)xcalloc(num, sizeof(Offset));
+  num = pool->ss.nstrings;
+  pool->whatprovides = whatprovides = (Offset *)xcalloc((num + WHATPROVIDES_BLOCK) & ~WHATPROVIDES_BLOCK, sizeof(Offset));
+  pool->whatprovides_rel = (Offset *)xcalloc((pool->nrels + WHATPROVIDES_BLOCK) & ~WHATPROVIDES_BLOCK, sizeof(Offset));
 
   /* count providers for each name */
   for (i = 1; i < pool->nsolvables; i++)
@@ -322,7 +315,7 @@ pool_createwhatprovides(Pool *pool)
       pp = s->repo->idarraydata + s->provides;
       while ((id = *pp++) != ID_NULL)
 	{
-	  if (ISRELDEP(id))
+	  while (ISRELDEP(id))
 	    {
 	      Reldep *rd = GETRELDEP(pool, id);
 	      id = rd->name;
@@ -345,8 +338,9 @@ pool_createwhatprovides(Pool *pool)
     }
 
   POOL_DEBUG(SAT_DEBUG_STATS, "provide ids: %d\n", np);
-  extra = 2 * pool->nrels;
 
+  /* reserve some space for relation data */
+  extra = 2 * pool->nrels;
   if (extra < 256)
     extra = 256;
 
@@ -369,7 +363,7 @@ pool_createwhatprovides(Pool *pool)
       pp = s->repo->idarraydata + s->provides;
       while ((id = *pp++) != 0)
 	{
-	  if (ISRELDEP(id))
+	  while (ISRELDEP(id))
 	    {
 	      Reldep *rd = GETRELDEP(pool, id);
 	      id = rd->name;
@@ -386,25 +380,38 @@ pool_createwhatprovides(Pool *pool)
 	  *d = i;		       /* put solvable Id into data */
 	}
     }
-  pool->whatprovides = whatprovides;
   pool->whatprovidesdata = whatprovidesdata;
   pool->whatprovidesdataoff = off;
   pool->whatprovidesdataleft = extra;
   pool_shrink_whatprovides(pool);
 }
 
+/*
+ * free all of our whatprovides data
+ * be careful, everything internalized with pool_queuetowhatprovides is gone, too
+ */
+void
+pool_freewhatprovides(Pool *pool)
+{
+  pool->whatprovides = xfree(pool->whatprovides);
+  pool->whatprovides_rel = xfree(pool->whatprovides_rel);
+  pool->whatprovidesdata = xfree(pool->whatprovidesdata);
+  pool->whatprovidesdataoff = 0;
+  pool->whatprovidesdataleft = 0;
+}
+
 
 /******************************************************************************/
 
 /*
- * pool_queuetowhatprovides
+ * pool_queuetowhatprovides  - add queue contents to whatprovidesdata
  * 
  * on-demand filling of provider information
  * move queue data into whatprovidesdata
  * q: queue of Ids
  * returns: Offset into whatprovides
+ *
  */
-
 Id
 pool_queuetowhatprovides(Pool *pool, Queue *q)
 {
@@ -458,7 +465,7 @@ pool_addrelproviders(Pool *pool, Id d)
   Id pid, *pidp;
   Id p, *pp, *pp2, *pp3;
 
-  d = GETRELID(pool, d);
+  d = GETRELID(d);
   queue_init_buffer(&plist, buf, sizeof(buf)/sizeof(*buf));
   switch (flags)
     {
@@ -491,7 +498,7 @@ pool_addrelproviders(Pool *pool, Id d)
 	  if (p > 1)
 	    {
 	      queue_free(&plist);
-	      pool->whatprovides[d] = p;
+	      pool->whatprovides_rel[d] = p;
 	      return pool->whatprovidesdata + p;
 	    }
 	  if (p == 1)
@@ -560,10 +567,10 @@ pool_addrelproviders(Pool *pool, Id d)
 #if 0
   POOL_DEBUG(DEBUG_1, "addrelproviders: adding %d packages to %d\n", plist.count, d);
 #endif
-  pool->whatprovides[d] = pool_queuetowhatprovides(pool, &plist);
+  pool->whatprovides_rel[d] = pool_queuetowhatprovides(pool, &plist);
   queue_free(&plist);
 
-  return pool->whatprovidesdata + pool->whatprovides[d];
+  return pool->whatprovidesdata + pool->whatprovides_rel[d];
 }
 
 /*************************************************************************/
diff --git a/src/pool.h b/src/pool.h
index 32159ae..602a971 100644
--- a/src/pool.h
+++ b/src/pool.h
@@ -87,6 +87,8 @@ struct _Pool {
    * whatprovidesdata[Offset] -> ID_NULL-terminated list of solvables providing Id
    */
   Offset *whatprovides;		/* Offset to providers of a specific name, Id -> Offset  */
+  Offset *whatprovides_rel;	/* Offset to providers of a specific relation, Id -> Offset  */
+
   Id *whatprovidesdata;		/* Ids of solvable providing Id */
   Offset whatprovidesdataoff;	/* next free slot within whatprovidesdata */
   int whatprovidesdataleft;	/* number of 'free slots' within whatprovidesdata */
@@ -133,7 +135,7 @@ struct _Pool {
 
 #define MAKERELDEP(id) ((id) | 0x80000000)
 #define ISRELDEP(id) (((id) & 0x80000000) != 0)
-#define GETRELID(pool, id) ((pool)->ss.nstrings + ((id) ^ 0x80000000))     /* returns Id  */
+#define GETRELID(id) ((id) ^ 0x80000000)				/* returns Id */
 #define GETRELDEP(pool, id) ((pool)->rels + ((id) ^ 0x80000000))	/* returns Reldep* */
 
 #define REL_GT		1
@@ -145,6 +147,10 @@ struct _Pool {
 #define REL_WITH	18
 #define REL_NAMESPACE	19
 
+#if !defined(__GNUC__) && !defined(__attribute__)
+# define __attribute__(x)
+#endif
+
 /**
  * Creates a new pool
  */
@@ -154,11 +160,7 @@ extern Pool *pool_create(void);
  */
 extern void pool_free(Pool *pool);
 
-extern void pool_debug(Pool *pool, int type, const char *format, ...)
-#ifdef __GNUC__
- __attribute__((format(printf, 3, 4)))
-#endif
-;
+extern void pool_debug(Pool *pool, int type, const char *format, ...) __attribute__((format(printf, 3, 4)));
 
 /**
  * Solvable management
@@ -197,9 +199,9 @@ static inline Id *pool_whatprovides(Pool *pool, Id d)
   Id v;
   if (!ISRELDEP(d))
     return pool->whatprovidesdata + pool->whatprovides[d];
-  v = GETRELID(pool, d);
-  if (pool->whatprovides[v])
-    return pool->whatprovidesdata + pool->whatprovides[v];
+  v = GETRELID(d);
+  if (pool->whatprovides_rel[v])
+    return pool->whatprovidesdata + pool->whatprovides_rel[v];
   return pool_addrelproviders(pool, d);
 }
 
diff --git a/src/poolid.c b/src/poolid.c
index 7165b8a..57dd5c4 100644
--- a/src/poolid.c
+++ b/src/poolid.c
@@ -21,18 +21,19 @@
 #include "util.h"
 
 
-// intern string into pool
-// return Id
+/* intern string into pool, return id */
 
 Id
 str2id(Pool *pool, const char *str, int create)
 {
-  int old_nstrings = pool->ss.nstrings;
+  int oldnstrings = pool->ss.nstrings;
   Id id = stringpool_str2id (&pool->ss, str, create);
-  /* If we changed the ID->string relations we need to get rid of an
-     existing provides lookup cache.  */
-  if (old_nstrings != pool->ss.nstrings)
-    pool_freewhatprovides(pool);
+  if (create && oldnstrings != pool->ss.nstrings && (id & WHATPROVIDES_BLOCK) == 0)
+    {
+      /* grow whatprovides array */
+      pool->whatprovides = xrealloc(pool->whatprovides, (id + (WHATPROVIDES_BLOCK + 1)) * sizeof(Offset));
+      memset(pool->whatprovides + id, 0, (WHATPROVIDES_BLOCK + 1) * sizeof(Offset));
+    }
   return id;
 }
 
@@ -51,7 +52,7 @@ rel2id(Pool *pool, Id name, Id evr, int flags, int create)
   hashtbl = pool->relhashtbl;
   ran = pool->rels;
   
-  // extend hashtable if needed
+  /* extend hashtable if needed */
   if (pool->nrels * 2 > hashmask)
     {
       xfree(pool->relhashtbl);
@@ -68,8 +69,7 @@ rel2id(Pool *pool, Id name, Id evr, int flags, int create)
 	}
     }
   
-  // compute hash and check for match
-
+  /* compute hash and check for match */
   h = relhash(name, evr, flags) & hashmask;
   hh = HASHCHAIN_START;
   while ((id = hashtbl[h]) != 0)
@@ -84,10 +84,8 @@ rel2id(Pool *pool, Id name, Id evr, int flags, int create)
   if (!create)
     return ID_NULL;
 
-  pool_freewhatprovides(pool);
-
   id = pool->nrels++;
-  // extend rel space if needed
+  /* extend rel space if needed */
   if ((id & REL_BLOCK) == 0)
     pool->rels = xrealloc(pool->rels, ((pool->nrels + REL_BLOCK) & ~REL_BLOCK) * sizeof(Reldep));
   hashtbl[h] = id;
@@ -95,6 +93,13 @@ rel2id(Pool *pool, Id name, Id evr, int flags, int create)
   ran->name = name;
   ran->evr = evr;
   ran->flags = flags;
+
+  /* extend whatprovides_rel if needed */
+  if (pool->whatprovides_rel && (id & WHATPROVIDES_BLOCK) == 0)
+    {
+      pool->whatprovides_rel = xrealloc(pool->whatprovides_rel, (id + (WHATPROVIDES_BLOCK + 1)) * sizeof(Offset));
+      memset(pool->whatprovides_rel + id, 0, (WHATPROVIDES_BLOCK + 1) * sizeof(Offset));
+    }
   return MAKERELDEP(id);
 }
 
@@ -236,7 +241,7 @@ dep2str(Pool *pool, Id id)
 void
 pool_shrink_strings(Pool *pool)
 {
-  stringpool_shrink (&pool->ss);
+  stringpool_shrink(&pool->ss);
 }
 
 void
diff --git a/src/poolid_private.h b/src/poolid_private.h
index 4a2ed8b..f861830 100644
--- a/src/poolid_private.h
+++ b/src/poolid_private.h
@@ -13,10 +13,11 @@
 #ifndef SATSOLVER_POOLID_PRIVATE_H
 #define SATSOLVER_POOLID_PRIVATE_H
 
-// the size of all buffers is incremented in blocks
-// these are the block values (increment values) for the
-// rel hashtable
-// 
-#define REL_BLOCK         1023          // hashtable for relations
+/* the size of all buffers is incremented in blocks
+ * these are the block values (increment values) for the
+ * rel hashtable
+ */
+#define REL_BLOCK		1023	/* hashtable for relations */
+#define WHATPROVIDES_BLOCK	1023
 
 #endif /* SATSOLVER_POOLID_PRIVATE_H */
diff --git a/src/pooltypes.h b/src/pooltypes.h
index 08a1df4..df6145f 100644
--- a/src/pooltypes.h
+++ b/src/pooltypes.h
@@ -14,7 +14,10 @@
 #define SATSOLVER_POOLTYPES_H
 
 /* version number for .solv files */
-#define SOLV_VERSION 0
+#define SOLV_VERSION_0 0
+#define SOLV_VERSION_1 1
+#define SOLV_FLAG_PACKEDSIZES 1
+#define SOLV_FLAG_VERTICAL    2
 
 struct _Stringpool;
 typedef struct _Stringpool Stringpool;
diff --git a/src/repo.c b/src/repo.c
index 5c7828f..05fed66 100644
--- a/src/repo.c
+++ b/src/repo.c
@@ -444,4 +444,103 @@ repo_fix_legacy(Repo *repo, Offset provides, Offset supplements)
   return supplements;
 }
 
+#if 0
+void
+repodata_search(Repodata *data, Id key)
+{
+}
+
+const char *
+repodata_lookup_id(Repodata *data, Id num, Id key)
+{
+  Id id, k, *kp, *keyp;
+
+  fseek(data->fp, data->itemoffsets[num] , SEEK_SET);
+  Id *keyp = data->schemadata + data->schemata[read_id(data->fp, data->numschemata)];
+  /* make sure our schema contains the key */
+  for (kp = keyp; (k = *kp++) != 0)
+    if (k == key)
+      break;
+  if (k == 0)
+    return 0;
+  /* get it */
+  while ((k = *keyp++) != 0)
+    {
+      if (k == key)
+	break;
+      switch (keys[key].type)
+	{
+	case TYPE_ID:
+	  while ((read_u8(data->fp) & 0x80) != 0)
+	    ;
+	  break;
+	case TYPE_U32:
+	  read_u32(data->fp);
+	  break;
+	case TYPE_STR:
+	  while(read_u8(data->fp) != 0)
+	    ;
+	  break;
+	case TYPE_IDARRAY:
+	  while ((read_u8(data->fp) & 0xc0) != 0)
+	    ;
+	  break;
+	}
+    }
+  id = read_id(data->fp, 0);
+  return data->ss.stringspace + data->ss.strings[id];
+}
+
+Id
+repo_lookup_id(Solvable *s, Id key)
+{
+  Solvable *rs;
+  Repo *repo = s->repo;
+  Repodata *data;
+  int i, j, n;
+
+  switch(key)
+    {
+    case SOLVABLE_NAME:
+      return s->name;
+    case SOLVABLE_ARCH:
+      return s->arch;
+    case SOLVABLE_EVR:
+      return s->evr;
+    case SOLVABLE_VENDOR:
+      return s->vendor;
+    }
+  /* convert solvable id into repo item count */
+  if (repo->end - repo->start + 1 == repo->nsolvables)
+    {
+      n = (s - pool->solvables);
+      if (n < repo->start || n > repo->end)
+	return 0;
+      n -= repo->start;
+    }
+  else
+    {
+      for (i = repo->start, rs = pool->solvables + i, n = 0; i < repo->end; i++, rs++)
+	{
+	  if (rs->repo != repo)
+	    continue;
+	  if (rs == s)
+	    break;
+	  n++;
+	}
+      if (i == repo->end)
+	return 0;
+    }
+  for (i = 0, data = repo->repodata; i < repo->nrepodata; i++, data++)
+    {
+      for (j = 0; j < data->nkeys; j++)
+	{
+	  if (data->keys[j].name == key && data->keys[j].type == TYPE_ID)
+	    return repodata_lookup_id(data, n, j);
+	}
+    }
+  return 0;
+}
+#endif
+
 // EOF
diff --git a/src/repo_solv.c b/src/repo_solv.c
index b50e21e..57b4f9c 100644
--- a/src/repo_solv.c
+++ b/src/repo_solv.c
@@ -97,7 +97,7 @@ read_id(FILE *fp, Id max)
       if (!(c & 128))
 	{
 	  x = (x << 7) | c;
-	  if (x >= max)
+	  if (max && x >= max)
 	    {
               pool_debug(mypool, SAT_FATAL, "read_id: id too large (%u/%u)\n", x, max);
 	      exit(1);
@@ -141,7 +141,9 @@ read_idarray(FILE *fp, Id max, Id *map, Id *store, Id *end)
 	      pool_debug(mypool, SAT_FATAL, "read_idarray: array overflow\n");
 	      exit(1);
 	    }
-	  *store++ = map[x];
+	  if (map)
+	    x = map[x];
+	  *store++ = x;
 	  if ((c & 64) == 0)
 	    {
 	      if (store == end)
@@ -162,11 +164,11 @@ read_idarray(FILE *fp, Id max, Id *map, Id *store, Id *end)
 
 /*-----------------------------------------------------------------*/
 
-typedef struct solvdata {
-  int type;
-  Id id;
-  unsigned int size;
-} SolvData;
+struct key {
+  Id name;
+  Id type;
+  Id size;
+};
 
 
 // ----------------------------------------------
@@ -180,9 +182,11 @@ void
 repo_add_solv(Repo *repo, FILE *fp)
 {
   Pool *pool = repo->pool;
-  int i, j, l;
-  unsigned int numid, numrel, numsolv, numsrcdata, numsolvdata;
-  int numsolvdatabits, type;
+  int i, l;
+  unsigned int numid, numrel, numsolv;
+  unsigned int numkeys, numschemata, numinfo;
+
+  int type;
   Offset sizeid;
   Offset *str;			       /* map Id -> Offset into string space */
   char *strsp;			       /* repo string space */
@@ -195,12 +199,14 @@ repo_add_solv(Repo *repo, FILE *fp)
   Id name, evr, did;
   int flags;
   Reldep *ran;
-  SolvData *solvdata;
-  unsigned int size, size_str, size_idarray;
+  unsigned int size_idarray;
   Id *idarraydatap, *idarraydataend;
   Offset ido;
-  unsigned int databits;
   Solvable *s;
+  unsigned int solvflags;
+  struct key *keys;
+  Id *schemadata, *schemadatap, *schemadataend;
+  Id *schemata, key;
 
   mypool = pool;
 
@@ -209,7 +215,7 @@ repo_add_solv(Repo *repo, FILE *fp)
       pool_debug(pool, SAT_FATAL, "not a SOLV file\n");
       exit(1);
     }
-  if (read_u32(fp) != SOLV_VERSION)
+  if (read_u32(fp) != SOLV_VERSION_1)
     {
       pool_debug(pool, SAT_FATAL, "unsupported SOLV version\n");
       exit(1);
@@ -219,7 +225,13 @@ repo_add_solv(Repo *repo, FILE *fp)
 
   numid = read_u32(fp);
   numrel = read_u32(fp);
-  numsolv= read_u32(fp);
+  numsolv = read_u32(fp);
+  numkeys = read_u32(fp);
+  numschemata = read_u32(fp);
+  numinfo = read_u32(fp);
+  solvflags = read_u32(fp);
+
+  /*******  Part 1: string IDs  *****************************************/
 
   sizeid = read_u32(fp);	       /* size of string+Id space */
 
@@ -342,6 +354,8 @@ repo_add_solv(Repo *repo, FILE *fp)
   pool_shrink_strings(pool);	       /* vacuum */
 
   
+  /*******  Part 2: Relation IDs  ***************************************/
+
   /*
    * read RelDeps
    * 
@@ -413,82 +427,81 @@ repo_add_solv(Repo *repo, FILE *fp)
       pool_shrink_rels(pool);		/* vacuum */
     }
 
-  /*
-   * read (but dont store yet) repo data
-   */
 
-#if 0
-  POOL_DEBUG(SAT_DEBUG_STATS, "read repo data\n");
-#endif
-  numsrcdata = read_u32(fp);
-  for (i = 0; i < numsrcdata; i++)
+  /*******  Part 3: Keys  ***********************************************/
+
+  keys = xcalloc(numkeys, sizeof(*keys));
+  /* keys start at 1 */
+  for (i = 1; i < numkeys; i++)
     {
-      type = read_u8(fp);
-      id = idmap[read_id(fp, numid)];
-      switch(type)
-	{
-	case TYPE_ID:
-          read_id(fp, numid + numrel);   /* just check Id */
-	  break;
-	case TYPE_U32:
-          read_u32(fp);
-	  break;
-	case TYPE_STR:
-	  while(read_u8(fp) != 0)
-	    ;
-	  break;
-	default:
-          pool_debug(pool, SAT_FATAL, "unknown type %d\n", type);
-	  exit(0);
-	}
+      keys[i].name = idmap[read_id(fp, numid)];
+      keys[i].type = read_id(fp, 0);
+      keys[i].size = read_id(fp, 0);
     }
 
-
-  /*
-   * read solvables
-   */
+  /*******  Part 4: Schemata ********************************************/
   
-#if 0
-  POOL_DEBUG(SAT_DEBUG_STATS, "read solvable data info\n");
-#endif
-  numsolvdata = read_u32(fp);
-  numsolvdatabits = 0;
-  solvdata = (SolvData *)xmalloc(numsolvdata * sizeof(SolvData));
-  size_idarray = 0;
-  size_str = 0;
+  id = read_id(fp, 0);
+  schemadata = xcalloc(id, sizeof(Id));
+  schemadatap = schemadata;
+  schemadataend = schemadata + id;
+  schemata = xcalloc(numschemata, sizeof(Id));
+  for (i = 0; i < numschemata; i++)
+    {
+      schemata[i] = schemadatap - schemadata;
+      schemadatap = read_idarray(fp, numid, 0, schemadatap, schemadataend);
+    }
 
-  for (i = 0; i < numsolvdata; i++)
+  /*******  Part 5: Info  ***********************************************/
+  /* we skip the info for now... */
+  for (i = 0; i < numinfo; i++)
     {
-      type = read_u8(fp);
-      solvdata[i].type = type;
-      if ((type & TYPE_BITMAP) != 0)
+      Id *schema = schemadata + schemata[read_id(fp, numschemata)];
+      while ((key = *schema++) != 0)
 	{
-	  type ^= TYPE_BITMAP;
-	  numsolvdatabits++;
-	}
-      id = idmap[read_id(fp, numid)];
-#if 0
-      POOL_DEBUG(SAT_DEBUG_STATS, "#%d: %s\n", i, id2str(pool, id));
-#endif
-      solvdata[i].id = id;
-      size = read_u32(fp);
-      solvdata[i].size = size;
-      if (id >= INTERESTED_START && id <= INTERESTED_END)
-	{
-	  if (type == TYPE_STR)
-	    size_str += size;
-	  if (type == TYPE_IDARRAY)
-	    size_idarray += size;
+	  type = keys[key].type;
+	  switch (type)
+	    {
+	      case TYPE_ID:
+	        read_id(fp, numid + numrel);   /* just check Id */
+	        break;
+	      case TYPE_U32:
+	        read_u32(fp);
+	        break;
+	      case TYPE_STR:
+	        while(read_u8(fp) != 0)
+		  ;
+	        break;
+	      case TYPE_IDARRAY:
+		while ((read_u8(fp) & 0xc0) != 0)
+		  ;
+		break;
+	      default:
+	        pool_debug(pool, SAT_FATAL, "unknown type %d\n", type);
+	        exit(0);
+	    }
 	}
     }
 
-  if (numsolvdatabits >= 32)
+  /*******  Part 6: packed sizes (optional)  ****************************/
+  if ((solvflags & SOLV_FLAG_PACKEDSIZES) != 0)
     {
-      pool_debug(pool, SAT_FATAL, "too many data map bits\n");
-      exit(1);
+      for (i = 0; i < numsolv; i++)
+	read_id(fp, 0);
+    }
+
+  /*******  Part 7: item data *******************************************/
+
+  /* calculate idarray size */
+  size_idarray = 0;
+  for (i = 1; i < numkeys; i++)
+    {
+      id = keys[i].name;
+      if (keys[i].type == TYPE_IDARRAY && id >= INTERESTED_START && id <= INTERESTED_END)
+	size_idarray += keys[i].size;
     }
 
-  /* make room for our idarrays */
+  /* allocate needed space in repo */
   if (size_idarray)
     {
       repo_reserve_ids(repo, 0, size_idarray);
@@ -503,37 +516,111 @@ repo_add_solv(Repo *repo, FILE *fp)
       idarraydataend = 0;
     }
 
-  /*
-   * read solvables
-   */
-  
-#if 0
-  POOL_DEBUG(SAT_DEBUG_STATS, "read solvables\n");
-#endif
+  /* read solvables */
   s = pool_id2solvable(pool, repo_add_solvable_block(repo, numsolv));
-  for (i = 0; i < numsolv; i++, s++)
+
+  if ((solvflags & SOLV_FLAG_VERTICAL) != 0)
     {
-      databits = 0;
-      if (numsolvdatabits)
+      Id *solvschema = xcalloc(numsolv, sizeof(Id));
+      unsigned char *used = xmalloc(numschemata);
+      Solvable *sstart = s;
+      Id type;
+
+      for (i = 0; i < numsolv; i++)
+	solvschema[i] = read_id(fp, numschemata);
+      for (key = 1; key < numkeys; key++)
 	{
-	  for (j = 0; j < (numsolvdatabits + 7) >> 3; j++)
-	    databits = (databits << 8) | read_u8(fp);
-	}
-      for (j = 0; j < numsolvdata; j++)
-	{
-	  type = solvdata[j].type;
-	  if ((type & TYPE_BITMAP) != 0)
+	  id = keys[key].name;
+	  type = keys[key].type;
+	  memset(used, 0, numschemata);
+	  for (i = 0; i < numschemata; i++)
 	    {
-	      if (!(databits & 1))
+	      Id *keyp = schemadata + schemata[i];
+	      while (*keyp)
+		if (*keyp++ == key)
+		  {
+		    used[i] = 1;
+		    break;
+		  }
+	    }
+	  for (i = 0, s = sstart; i < numsolv; i++, s++)
+	     {
+	      if (!used[solvschema[i]])
+		continue;
+	      switch (type)
 		{
-		  databits >>= 1;
-		  continue;
+		case TYPE_ID:
+		  did = idmap[read_id(fp, numid + numrel)];
+		  if (id == SOLVABLE_NAME)
+		    s->name = did;
+		  else if (id == SOLVABLE_ARCH)
+		    s->arch = did;
+		  else if (id == SOLVABLE_EVR)
+		    s->evr = did;
+		  else if (id == SOLVABLE_VENDOR)
+		    s->vendor = did;
+		  break;
+		case TYPE_U32:
+		  h = read_u32(fp);
+		  if (id == RPM_RPMDBID)
+		    {
+		      if (!repo->rpmdbid)
+			repo->rpmdbid = (Id *)xcalloc(numsolv, sizeof(Id));
+		      repo->rpmdbid[i] = h;
+		    }
+		  break;
+		case TYPE_STR:
+		  while(read_u8(fp) != 0)
+		    ;
+		  break;
+		case TYPE_IDARRAY:
+		  if (id < INTERESTED_START || id > INTERESTED_END)
+		    {
+		      /* not interested in array */
+		      while ((read_u8(fp) & 0xc0) != 0)
+			;
+		      break;
+		    }
+		  ido = idarraydatap - repo->idarraydata;
+		  idarraydatap = read_idarray(fp, numid + numrel, idmap, idarraydatap, idarraydataend);
+		  if (id == SOLVABLE_PROVIDES)
+		    s->provides = ido;
+		  else if (id == SOLVABLE_OBSOLETES)
+		    s->obsoletes = ido;
+		  else if (id == SOLVABLE_CONFLICTS)
+		    s->conflicts = ido;
+		  else if (id == SOLVABLE_REQUIRES)
+		    s->requires = ido;
+		  else if (id == SOLVABLE_RECOMMENDS)
+		    s->recommends= ido;
+		  else if (id == SOLVABLE_SUPPLEMENTS)
+		    s->supplements = ido;
+		  else if (id == SOLVABLE_SUGGESTS)
+		    s->suggests = ido;
+		  else if (id == SOLVABLE_ENHANCES)
+		    s->enhances = ido;
+		  else if (id == SOLVABLE_FRESHENS)
+		    s->freshens = ido;
+		  break;
 		}
-	      databits >>= 1;
-	      type ^= TYPE_BITMAP;
 	    }
-	  id = solvdata[j].id;
-	  switch (type)
+	}
+      xfree(used);
+      xfree(solvschema);
+      xfree(idmap);
+      xfree(schemata);
+      xfree(schemadata);
+      xfree(keys);
+      mypool = 0;
+      return;
+    }
+  for (i = 0; i < numsolv; i++, s++)
+    {
+      Id *keyp = schemadata + schemata[read_id(fp, numschemata)];
+      while ((key = *keyp++) != 0)
+	{
+	  id = keys[key].name;
+	  switch (keys[key].type)
 	    {
 	    case TYPE_ID:
 	      did = idmap[read_id(fp, numid + numrel)];
@@ -603,7 +690,9 @@ repo_add_solv(Repo *repo, FILE *fp)
 	}
     }
   xfree(idmap);
-  xfree(solvdata);
+  xfree(schemata);
+  xfree(schemadata);
+  xfree(keys);
   mypool = 0;
 }
 
diff --git a/src/solver.c b/src/solver.c
index 524a685..56017a1 100644
--- a/src/solver.c
+++ b/src/solver.c
@@ -3260,11 +3260,15 @@ solver_solve(Solver *solv, Queue *job)
   Repo *installed = solv->installed;
   int i;
   int oldnrules;
-  Map addedmap;			       /* '1' == have rule for solvable */
+  Map addedmap;		       /* '1' == have rpm-rules for solvable */
   Id how, what, p, *pp, d;
   Queue q;
   Solvable *s;
 
+  /* create whatprovides if not already there */
+  if (pool->whatprovides)
+    pool_createwhatprovides(pool);
+
   /* create obsolete index if needed */
   if (solv->noupdateprovide)
     create_obsolete_index(solv);
diff --git a/tools/dumpsolv.c b/tools/dumpsolv.c
index 4fe151b..19231ce 100644
--- a/tools/dumpsolv.c
+++ b/tools/dumpsolv.c
@@ -30,7 +30,7 @@ int main(int argc, char **argv)
 {
   Repo *repo;
   Pool *pool;
-  int i;
+  int i, n;
   Solvable *s;
 
   if (argc != 1)
@@ -45,13 +45,13 @@ int main(int argc, char **argv)
   repo = repo_create(pool, argc != 1 ? argv[1] : "<stdin>");
   repo_add_solv(repo, stdin);
   printf("repo contains %d solvables\n", repo->nsolvables);
-  for (i = repo->start; i < repo->end; i++)
+  for (i = repo->start, n = 1; i < repo->end; i++)
     {
       s = pool->solvables + i;
       if (s->repo != repo)
 	continue;
       printf("\n");
-      printf("solvable %d:\n", i);
+      printf("solvable %d:\n", n++);
       printf("name: %s %s %s\n", id2str(pool, s->name), id2str(pool, s->evr), id2str(pool, s->arch));
       if (s->vendor)
         printf("vendor: %s\n", id2str(pool, s->vendor));
@@ -65,5 +65,6 @@ int main(int argc, char **argv)
       printids(repo, "enhances", s->enhances);
       printids(repo, "freshens", s->freshens);
     }
+  pool_free(pool);
   exit(0);
 }
diff --git a/tools/repo_write.c b/tools/repo_write.c
index 2d7bd0d..85f6b23 100644
--- a/tools/repo_write.c
+++ b/tools/repo_write.c
@@ -23,6 +23,7 @@
 #include <string.h>
 
 #include "pool.h"
+#include "util.h"
 #include "repo_write.h"
 
 /*------------------------------------------------------------------*/
@@ -33,6 +34,9 @@ typedef struct needid {
   Id map;
 } NeedId;
 
+
+#define RELOFF(id) (pool->ss.nstrings + GETRELID(id))
+
 /*
  * increment need Id
  * idarray: array of Ids, ID_NULL terminated
@@ -57,7 +61,7 @@ incneedid(Pool *pool, Id *idarray, NeedId *needid)
       while (ISRELDEP(id))
 	{
 	  Reldep *rd = GETRELDEP(pool, id);
-	  needid[GETRELID(pool, id)].need++;
+	  needid[RELOFF(id)].need++;
 	  if (ISRELDEP(rd->evr))
 	    {
 	      Id ida[2];
@@ -165,13 +169,14 @@ write_idarray(FILE *fp, Pool *pool, NeedId *needid, Id *ids)
     return;
   if (!*ids)
     {
-      write_u8(fp, ID_NULL);
+      write_u8(fp, 0);
       return;
     }
   for (;;)
     {
       id = *ids++;
-      id = needid[ISRELDEP(id) ? GETRELID(pool, id) : id].need;
+      if (needid)
+        id = needid[ISRELDEP(id) ? RELOFF(id) : id].need;
       if (id >= 64)
 	id = (id & 63) | ((id & ~63) << 1);
       if (!*ids)
@@ -192,22 +197,29 @@ void
 repo_write(Repo *repo, FILE *fp)
 {
   Pool *pool = repo->pool;
-  int i, numsolvdata;
+  int i, n;
   Solvable *s;
   NeedId *needid;
-  int nstrings, nrels;
+  int nstrings, nrels, nkeys, nschemata;
   unsigned int sizeid;
   Reldep *ran;
   Id *idarraydata;
 
   int idsizes[RPM_RPMDBID + 1];
-  int bits, bitmaps;
+  int id2key[RPM_RPMDBID + 1];
   int nsolvables;
 
+  Id *schemadata, *schemadatap, *schema, *sp;
+  Id schemaid;
+  int schemadatalen;
+  Id *solvschema;	/* schema of our solvables */
+  Id lastschema[256];
+  Id lastschemakey[256];
+
   nsolvables = 0;
   idarraydata = repo->idarraydata;
 
-  needid = (NeedId *)calloc(pool->ss.nstrings + pool->nrels, sizeof(*needid));
+  needid = (NeedId *)xcalloc(pool->ss.nstrings + pool->nrels, sizeof(*needid));
 
   memset(idsizes, 0, sizeof(idsizes));
 
@@ -293,19 +305,120 @@ repo_write(Repo *repo, FILE *fp)
       needid[needid[pool->ss.nstrings + i].map].need = nstrings + i;
     }
 
+  /* find the keys we need */
+  nkeys = 1;
+  memset(id2key, 0, sizeof(id2key));
+  for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
+    if (idsizes[i])
+      id2key[i] = nkeys++;
+
+  /* find the schemata we need */
+  solvschema = xcalloc(repo->nsolvables, sizeof(Id));
+
+  memset(lastschema, 0, sizeof(lastschema));
+  memset(lastschemakey, 0, sizeof(lastschemakey));
+  schemadata = xmalloc(256 * sizeof(Id));
+  schemadatalen = 256;
+  schemadatap = schemadata;
+  *schemadatap++ = 0;
+  schemadatalen--;
+  nschemata = 0;
+  for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
+    {
+      unsigned int h;
+      Id *sp;
+
+      if (s->repo != repo)
+	continue;
+      if (schemadatalen < 32)
+	{
+	  int l = schemadatap - schemadata;
+          fprintf(stderr, "growing schemadata\n");
+	  schemadata = xrealloc(schemadata, (schemadatap - schemadata + 256) * sizeof(Id));
+	  schemadatalen = 256;
+	  schemadatap = schemadata + l;
+	}
+      schema = schemadatap;
+      *schema++ = SOLVABLE_NAME;
+      *schema++ = SOLVABLE_ARCH;
+      *schema++ = SOLVABLE_EVR;
+      if (s->vendor)
+        *schema++ = SOLVABLE_VENDOR;
+      if (s->provides)
+        *schema++ = SOLVABLE_PROVIDES;
+      if (s->obsoletes)
+        *schema++ = SOLVABLE_OBSOLETES;
+      if (s->conflicts)
+        *schema++ = SOLVABLE_CONFLICTS;
+      if (s->requires)
+        *schema++ = SOLVABLE_REQUIRES;
+      if (s->recommends)
+        *schema++ = SOLVABLE_RECOMMENDS;
+      if (s->suggests)
+        *schema++ = SOLVABLE_SUGGESTS;
+      if (s->supplements)
+        *schema++ = SOLVABLE_SUPPLEMENTS;
+      if (s->enhances)
+        *schema++ = SOLVABLE_ENHANCES;
+      if (s->freshens)
+        *schema++ = SOLVABLE_FRESHENS;
+      if (repo->rpmdbid)
+        *schema++ = RPM_RPMDBID;
+      *schema++ = 0;
+      for (sp = schemadatap, h = 0; *sp; )
+	h = h * 7 + *sp++;
+      h &= 255;
+      if (lastschema[h] && !memcmp(schemadata + lastschema[h], schemadatap, (schema - schemadatap) * sizeof(Id)))
+	{
+	  solvschema[n++] = lastschemakey[h];
+	  continue;
+	}
+      schemaid = 0;
+      for (sp = schemadata + 1; sp < schemadatap; )
+	{
+	  if (!memcmp(sp, schemadatap, (schema - schemadatap) * sizeof(Id)))
+	    break;
+	  while (*sp++)
+	    ;
+	  schemaid++;
+	}
+      if (sp >= schemadatap)
+	{
+	  if (schemaid != nschemata)
+	    abort();
+	  lastschema[h] = schemadatap - schemadata;
+	  lastschemakey[h] = schemaid;
+	  schemadatalen -= schema - schemadatap;
+	  schemadatap = schema;
+	  nschemata++;
+	}
+      solvschema[n++] = schemaid;
+    }
+  /* convert all schemas to keys */
+  for (sp = schemadata; sp < schemadatap; sp++)
+    *sp = id2key[*sp];
+
   /* write file header */
   write_u32(fp, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
-  write_u32(fp, SOLV_VERSION);
+  write_u32(fp, SOLV_VERSION_1);
 
   /* write counts */
   write_u32(fp, nstrings);
   write_u32(fp, nrels);
   write_u32(fp, nsolvables);
-  write_u32(fp, sizeid);
+  write_u32(fp, nkeys);
+  write_u32(fp, nschemata);
+  write_u32(fp, 0);	/* no info block */
+#if 0
+  write_u32(fp, SOLV_FLAG_VERTICAL);	/* no flags */
+#else
+  write_u32(fp, 0);	/* no flags */
+#endif
 
   /*
    * write strings
    */
+  write_u32(fp, sizeid);
   for (i = 1; i < nstrings; i++)
     {
       char *str = pool->ss.stringspace + pool->ss.strings[needid[i].map];
@@ -322,90 +435,134 @@ repo_write(Repo *repo, FILE *fp)
   for (i = 0; i < nrels; i++)
     {
       ran = pool->rels + (needid[pool->ss.nstrings + i].map - pool->ss.nstrings);
-      write_id(fp, needid[ISRELDEP(ran->name) ? GETRELID(pool, ran->name) : ran->name].need);
-      write_id(fp, needid[ISRELDEP(ran->evr) ? GETRELID(pool, ran->evr) : ran->evr].need);
+      write_id(fp, needid[ISRELDEP(ran->name) ? RELOFF(ran->name) : ran->name].need);
+      write_id(fp, needid[ISRELDEP(ran->evr) ? RELOFF(ran->evr) : ran->evr].need);
       write_u8( fp, ran->flags);
     }
 
-  write_u32(fp, 0);	/* no repo data */
-
   /*
-   * write Solvables
+   * write keys
    */
-  
-  numsolvdata = 0;
   for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
     {
-      if (idsizes[i])
-        numsolvdata++;
-    }
-  write_u32(fp, (unsigned int)numsolvdata);
-
-  bitmaps = 0;
-  for (i = SOLVABLE_NAME; i <= SOLVABLE_FRESHENS; i++)
-    {
       if (!idsizes[i])
 	continue;
-      if (i >= SOLVABLE_PROVIDES && i <= SOLVABLE_FRESHENS)
-	{
-	  write_u8(fp, TYPE_IDARRAY|TYPE_BITMAP);
-	  bitmaps++;
-	}
-      else
-	write_u8(fp, TYPE_ID);
       write_id(fp, needid[i].need);
       if (i >= SOLVABLE_PROVIDES && i <= SOLVABLE_FRESHENS)
-	write_u32(fp, idsizes[i]);
+	write_id(fp, TYPE_IDARRAY);
+      else if (i == RPM_RPMDBID)
+        write_id(fp, TYPE_U32);
       else
-	write_u32(fp, 0);
+        write_id(fp, TYPE_ID);
+      write_id(fp, idsizes[i]);
     }
 
-  if (repo->rpmdbid)
+  /*
+   * write schemata
+   */
+  write_id(fp, schemadatap - schemadata - 1);
+  for (sp = schemadata + 1; sp < schemadatap; )
     {
-      write_u8(fp, TYPE_U32);
-      write_id(fp, needid[RPM_RPMDBID].need);
-      write_u32(fp, 0);
+      write_idarray(fp, pool, 0, sp);
+      while (*sp++)
+	;
     }
+  
+#if 0
+  if (1)
+    {
+      Id id, key;
 
-  for (i = repo->start, s = pool->solvables + i; i < repo->end; i++, s++)
+      for (i = 0; i < nsolvables; i++)
+	write_id(fp, solvschema[i]);
+      unsigned char *used = xmalloc(nschemata);
+      for (id = SOLVABLE_NAME; id <= RPM_RPMDBID; id++)
+	{
+	  key = id2key[id];
+	  memset(used, 0, nschemata);
+	  for (sp = schemadata + 1, i = 0; sp < schemadatap; sp++)
+	    {
+	      if (*sp == 0)
+		i++;
+	      else if (*sp == key)
+		used[i] = 1;
+	    }
+	  for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
+	    {
+	      if (s->repo != repo)
+		continue;
+	      if (!used[solvschema[n++]])
+		continue;
+	      switch(id)
+		{
+		case SOLVABLE_NAME:
+		  write_id(fp, needid[s->name].need);
+		  break;
+		case SOLVABLE_ARCH:
+		  write_id(fp, needid[s->arch].need);
+		  break;
+		case SOLVABLE_EVR:
+		  write_id(fp, needid[s->evr].need);
+		  break;
+		case SOLVABLE_VENDOR:
+		  write_id(fp, needid[s->vendor].need);
+		  break;
+		case RPM_RPMDBID:
+		  write_u32(fp, repo->rpmdbid[i - repo->start]);
+		  break;
+		case SOLVABLE_PROVIDES:
+		  write_idarray(fp, pool, needid, idarraydata + s->provides);
+		  break;
+		case SOLVABLE_OBSOLETES:
+		  write_idarray(fp, pool, needid, idarraydata + s->obsoletes);
+		  break;
+		case SOLVABLE_CONFLICTS:
+		  write_idarray(fp, pool, needid, idarraydata + s->conflicts);
+		  break;
+		case SOLVABLE_REQUIRES:
+		  write_idarray(fp, pool, needid, idarraydata + s->requires);
+		  break;
+		case SOLVABLE_RECOMMENDS:
+		  write_idarray(fp, pool, needid, idarraydata + s->recommends);
+		  break;
+		case SOLVABLE_SUPPLEMENTS:
+		  write_idarray(fp, pool, needid, idarraydata + s->supplements);
+		  break;
+		case SOLVABLE_SUGGESTS:
+		  write_idarray(fp, pool, needid, idarraydata + s->suggests);
+		  break;
+		case SOLVABLE_ENHANCES:
+		  write_idarray(fp, pool, needid, idarraydata + s->enhances);
+		  break;
+		case SOLVABLE_FRESHENS:
+		  write_idarray(fp, pool, needid, idarraydata + s->freshens);
+		  break;
+		}
+	    }
+	}
+      xfree(used);
+      xfree(needid);
+      xfree(solvschema);
+      xfree(schemadata);
+      return;
+    }
+  
+#endif
+
+  /*
+   * write Solvables
+   */
+  for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
     {
       if (s->repo != repo)
 	continue;
-      bits = 0;
-      if (idsizes[SOLVABLE_FRESHENS])
-	bits = (bits << 1) | (s->freshens ? 1 : 0);
-      if (idsizes[SOLVABLE_ENHANCES])
-	bits = (bits << 1) | (s->enhances ? 1 : 0);
-      if (idsizes[SOLVABLE_SUPPLEMENTS])
-	bits = (bits << 1) | (s->supplements ? 1 : 0);
-      if (idsizes[SOLVABLE_SUGGESTS])
-	bits = (bits << 1) | (s->suggests ? 1 : 0);
-      if (idsizes[SOLVABLE_RECOMMENDS])
-	bits = (bits << 1) | (s->recommends ? 1 : 0);
-      if (idsizes[SOLVABLE_REQUIRES])
-	bits = (bits << 1) | (s->requires ? 1 : 0);
-      if (idsizes[SOLVABLE_CONFLICTS])
-	bits = (bits << 1) | (s->conflicts ? 1 : 0);
-      if (idsizes[SOLVABLE_OBSOLETES])
-	bits = (bits << 1) | (s->obsoletes ? 1 : 0);
-      if (idsizes[SOLVABLE_PROVIDES])
-	bits = (bits << 1) | (s->provides ? 1 : 0);
-
-      if (bitmaps > 24)
-	write_u8(fp, bits >> 24);
-      if (bitmaps > 16)
-	write_u8(fp, bits >> 16);
-      if (bitmaps > 8)
-	write_u8(fp, bits >> 8);
-      if (bitmaps)
-	write_u8(fp, bits);
-
+      /* keep in sync with schema generation! */
+      write_id(fp, solvschema[n++]);
       write_id(fp, needid[s->name].need);
       write_id(fp, needid[s->arch].need);
       write_id(fp, needid[s->evr].need);
-      if (idsizes[SOLVABLE_VENDOR])
+      if (s->vendor)
         write_id(fp, needid[s->vendor].need);
-
       if (s->provides)
         write_idarray(fp, pool, needid, idarraydata + s->provides);
       if (s->obsoletes)
@@ -428,7 +585,9 @@ repo_write(Repo *repo, FILE *fp)
         write_u32(fp, repo->rpmdbid[i - repo->start]);
     }
 
-  free(needid);
+  xfree(needid);
+  xfree(solvschema);
+  xfree(schemadata);
 }
 
 // EOF