X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=src%2Frepodata.c;h=4ab5d188bf39d9ce1135e92d46aeac478f83611a;hb=e679b515eddb3dd340fb25620de0160211f40fdc;hp=62ec955639db56a90c3fafddb17f7a8c4ec0d5f5;hpb=c8e2acdc6c28d39193d24c9ca0a934ee43225648;p=platform%2Fupstream%2Flibsolv.git

diff --git a/src/repodata.c b/src/repodata.c
index 62ec955..4ab5d18 100644
--- a/src/repodata.c
+++ b/src/repodata.c
@@ -268,17 +268,13 @@ repodata_str2dir(Repodata *data, const char *dir, int create)
 #endif
   const char *dire;
 
-  parent = 0;
   if (!*dir)
-    return 0;
+    return data->dirpool.ndirs ? 0 : dirpool_add_dir(&data->dirpool, 0, 0, create);
   while (*dir == '/' && dir[1] == '/')
     dir++;
   if (*dir == '/' && !dir[1])
-    {
-      if (data->dirpool.ndirs)
-        return 1;
-      return dirpool_add_dir(&data->dirpool, 0, 1, create);
-    }
+    return data->dirpool.ndirs ? 1 : dirpool_add_dir(&data->dirpool, 0, 1, create);
+  parent = 0;
 #ifdef DIRCACHE_SIZE
   dirs = dir;
   if (data->dircache)
@@ -355,6 +351,8 @@ repodata_dir2str(Repodata *data, Id did, const char *suf)
 
   if (!did)
     return suf ? suf : "";
+  if (did == 1 && !suf)
+    return "/";
   parent = did;
   while (parent)
     {
@@ -479,11 +477,11 @@ get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
   if (off >= data->lastverticaloffset)
     {
       off -= data->lastverticaloffset;
-      if (off + len > data->vincorelen)
+      if ((unsigned int)off + len > data->vincorelen)
 	return 0;
       return data->vincore + off;
     }
-  if (off + len > key->size)
+  if ((unsigned int)off + len > key->size)
     return 0;
   /* we now have the offset, go into vertical */
   off += data->verticaloffset[key - data->keys];
@@ -849,6 +847,62 @@ repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id void
   return 0;
 }
 
+const char *
+repodata_lookup_dirstrarray_uninternalized(Repodata *data, Id solvid, Id keyname, Id *didp, Id *iterp)
+{
+  Id *ap, did;
+  Id iter = *iterp;
+  if (iter == 0)	/* find key data */
+    {
+      if (!data->attrs)
+	return 0;
+      ap = data->attrs[solvid - data->start];
+      if (!ap)
+	return 0;
+      for (; *ap; ap += 2)
+	if (data->keys[*ap].name == keyname && data->keys[*ap].type == REPOKEY_TYPE_DIRSTRARRAY)
+	  break;
+      if (!*ap)
+	return 0;
+      iter = ap[1];
+    }
+  did = *didp;
+  for (ap = data->attriddata + iter; *ap; ap += 2)
+    {
+      if (did && ap[0] != did)
+	continue;
+      *didp = ap[0];
+      *iterp = ap - data->attriddata + 2;
+      return (const char *)data->attrdata + ap[1];
+    }
+  *iterp = 0;
+  return 0;
+}
+
+const unsigned char *
+repodata_lookup_bin_checksum_uninternalized(Repodata *data, Id solvid, Id keyname, Id *typep)
+{
+  Id *ap;
+  if (!data->attrs)
+    return 0;
+  ap = data->attrs[solvid - data->start];
+  if (!ap)
+    return 0;
+  for (; *ap; ap += 2)
+    {
+      if (data->keys[*ap].name != keyname)
+	continue;
+      switch (data->keys[*ap].type)
+	{
+	  case_CHKSUM_TYPES:
+	    *typep = data->keys[*ap].type;
+	    return (const unsigned char *)data->attrdata + ap[1];
+	  default:
+	    break;
+	}
+    }
+  return 0;
+}
 
 /************************************************************************
  * data search
@@ -962,7 +1016,11 @@ repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback
       ddp = get_data(data, key, &dp, *keyp ? 1 : 0);
 
       if (key->type == REPOKEY_TYPE_DELETED)
-	continue;
+	{
+	  if (onekey)
+	    return;
+	  continue;
+	}
       if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
 	{
 	  struct subschema_data subd;
@@ -1444,6 +1502,8 @@ dataiterator_filelistcheck(Dataiterator *di)
   if (!needcomplete)
     {
       /* we don't need the complete filelist, so ignore all stubs */
+      if (data->repo->nrepodata == 2)
+	return 1;
       for (j = 1; j < data->nkeys; j++)
 	if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
 	  return 1;
@@ -1925,19 +1985,19 @@ dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
 	  return;
 	}
       di->repoid = 0;
-      di->data = di->repo->repodata + di->pool->pos.repodataid;
-      di->repodataid = 0;
-      di->solvid = solvid;
-      di->state = di_enterrepo;
-      di->flags |= SEARCH_THISSOLVID;
-      return;
+      if (!di->pool->pos.repodataid && di->pool->pos.solvid == SOLVID_META) {
+	solvid = SOLVID_META;		/* META pos hack */
+      } else {
+        di->data = di->repo->repodata + di->pool->pos.repodataid;
+        di->repodataid = 0;
+      }
     }
-  if (solvid > 0)
+  else if (solvid > 0)
     {
       di->repo = di->pool->solvables[solvid].repo;
       di->repoid = 0;
     }
-  else if (di->repoid > 0)
+  if (di->repoid > 0)
     {
       if (!di->pool->urepos)
 	{
@@ -1947,7 +2007,8 @@ dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
       di->repoid = 1;
       di->repo = di->pool->repos[di->repoid];
     }
-  di->repodataid = 1;
+  if (solvid != SOLVID_POS)
+    di->repodataid = 1;
   di->solvid = solvid;
   if (solvid)
     di->flags |= SEARCH_THISSOLVID;
@@ -2102,6 +2163,7 @@ repodata_extend_block(Repodata *data, Id start, Id num)
     return;
   if (!data->incoreoffset)
     {
+      /* this also means that data->attrs is NULL */
       data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
       data->start = start;
       data->end = start + num;
@@ -2746,6 +2808,47 @@ repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
 }
 
 void
+repodata_set_kv(Repodata *data, Id solvid, Id keyname, Id keytype, KeyValue *kv)
+{
+  switch (keytype)
+    {
+    case REPOKEY_TYPE_ID:
+      repodata_set_id(data, solvid, keyname, kv->id);
+      break;
+    case REPOKEY_TYPE_CONSTANTID:
+      repodata_set_constantid(data, solvid, keyname, kv->id);
+      break;
+    case REPOKEY_TYPE_IDARRAY:
+      repodata_add_idarray(data, solvid, keyname, kv->id);
+      break;
+    case REPOKEY_TYPE_STR:
+      repodata_set_str(data, solvid, keyname, kv->str);
+      break;
+    case REPOKEY_TYPE_VOID:
+      repodata_set_void(data, solvid, keyname);
+      break;
+    case REPOKEY_TYPE_NUM:
+      repodata_set_num(data, solvid, keyname, SOLV_KV_NUM64(kv));
+      break;
+    case REPOKEY_TYPE_CONSTANT:
+      repodata_set_constant(data, solvid, keyname, kv->num);
+      break;
+    case REPOKEY_TYPE_DIRNUMNUMARRAY:
+      if (kv->id)
+        repodata_add_dirnumnum(data, solvid, keyname, kv->id, kv->num, kv->num2);
+      break;
+    case REPOKEY_TYPE_DIRSTRARRAY:
+      repodata_add_dirstr(data, solvid, keyname, kv->id, kv->str);
+      break;
+    case_CHKSUM_TYPES:
+      repodata_set_bin_checksum(data, solvid, keyname, keytype, (const unsigned char *)kv->str);
+      break;
+    default:
+      break;
+    }
+}
+
+void
 repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname)
 {
   Id *pp, *ap, **app;
@@ -2812,6 +2915,10 @@ repodata_swap_attrs(Repodata *data, Id dest, Id src)
   Id *tmpattrs;
   if (!data->attrs || dest == src)
     return;
+  if (dest < data->start || dest >= data->end)
+    repodata_extend(data, dest);
+  if (src < data->start || src >= data->end)
+    repodata_extend(data, src);
   tmpattrs = data->attrs[dest - data->start];
   data->attrs[dest - data->start] = data->attrs[src - data->start];
   data->attrs[src - data->start] = tmpattrs;
@@ -2902,6 +3009,96 @@ data_addblob(struct extdata *xd, unsigned char *blob, int len)
 
 /*********************************/
 
+/* this is to reduct memory usage when internalizing oversized repos */
+static void
+compact_attrdata(Repodata *data, int entry, int nentry)
+{
+  int i;
+  unsigned int attrdatastart = data->attrdatalen;
+  unsigned int attriddatastart = data->attriddatalen;
+  if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
+    return;
+  for (i = entry; i < nentry; i++)
+    {
+      Id v, *attrs = data->attrs[i];
+      if (!attrs)
+	continue;
+      for (; *attrs; attrs += 2)
+	{
+	  switch (data->keys[*attrs].type)
+	    {
+	    case REPOKEY_TYPE_STR:
+	    case REPOKEY_TYPE_BINARY:
+	    case_CHKSUM_TYPES:
+	      if ((unsigned int)attrs[1] < attrdatastart)
+		 attrdatastart = attrs[1];
+	      break;
+	    case REPOKEY_TYPE_DIRSTRARRAY:
+	      for (v = attrs[1]; data->attriddata[v] ; v += 2)
+		if (data->attriddata[v + 1] < attrdatastart)
+		  attrdatastart = data->attriddata[v + 1];
+	      /* FALLTHROUGH */
+	    case REPOKEY_TYPE_IDARRAY:
+	    case REPOKEY_TYPE_DIRNUMNUMARRAY:
+	      if ((unsigned int)attrs[1] < attriddatastart)
+		attriddatastart = attrs[1];
+	      break;
+	    case REPOKEY_TYPE_FIXARRAY:
+	    case REPOKEY_TYPE_FLEXARRAY:
+	      return;
+	    default:
+	      break;
+	    }
+	}
+    }
+#if 0
+  printf("compact_attrdata %d %d\n", entry, nentry);
+  printf("attrdatastart: %d\n", attrdatastart);
+  printf("attriddatastart: %d\n", attriddatastart);
+#endif
+  if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
+    return;
+  for (i = entry; i < nentry; i++)
+    {
+      Id v, *attrs = data->attrs[i];
+      if (!attrs)
+	continue;
+      for (; *attrs; attrs += 2)
+	{
+	  switch (data->keys[*attrs].type)
+	    {
+	    case REPOKEY_TYPE_STR:
+	    case REPOKEY_TYPE_BINARY:
+	    case_CHKSUM_TYPES:
+	      attrs[1] -= attrdatastart;
+	      break;
+	    case REPOKEY_TYPE_DIRSTRARRAY:
+	      for (v = attrs[1]; data->attriddata[v] ; v += 2)
+		data->attriddata[v + 1] -= attrdatastart;
+	      /* FALLTHROUGH */
+	    case REPOKEY_TYPE_IDARRAY:
+	    case REPOKEY_TYPE_DIRNUMNUMARRAY:
+	      attrs[1] -= attriddatastart;
+	      break;
+	    default:
+	      break;
+	    }
+	}
+    }
+  if (attrdatastart)
+    {
+      data->attrdatalen -= attrdatastart;
+      memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
+      data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
+    }
+  if (attriddatastart)
+    {
+      data->attriddatalen -= attriddatastart;
+      memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
+      data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
+    }
+}
+
 /* internalalize some key into incore/vincore data */
 
 static void
@@ -2926,6 +3123,7 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore,
     case REPOKEY_TYPE_VOID:
     case REPOKEY_TYPE_CONSTANT:
     case REPOKEY_TYPE_CONSTANTID:
+    case REPOKEY_TYPE_DELETED:
       break;
     case REPOKEY_TYPE_STR:
       data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
@@ -2996,29 +3194,30 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore,
 	    sp = schema;
 	    kp = data->xattrs[-*ida];
 	    if (!kp)
-	      continue;
+	      continue;		/* ignore empty elements */
 	    num++;
-	    for (;*kp; kp += 2)
+	    for (; *kp; kp += 2)
 	      *sp++ = *kp;
 	    *sp = 0;
 	    if (!schemaid)
 	      schemaid = repodata_schema2id(data, schema, 1);
 	    else if (schemaid != repodata_schema2id(data, schema, 0))
 	      {
-	 	pool_debug(data->repo->pool, SOLV_FATAL, "fixarray substructs with different schemas\n");
-		exit(1);
+	 	pool_debug(data->repo->pool, SOLV_ERROR, "repodata_serialize_key: fixarray substructs with different schemas\n");
+		num = 0;
+		break;
 	      }
 	  }
+	data_addid(xd, num);
 	if (!num)
 	  break;
-	data_addid(xd, num);
 	data_addid(xd, schemaid);
 	for (ida = data->attriddata + val; *ida; ida++)
 	  {
 	    Id *kp = data->xattrs[-*ida];
 	    if (!kp)
 	      continue;
-	    for (;*kp; kp += 2)
+	    for (; *kp; kp += 2)
 	      repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
 	  }
 	break;
@@ -3050,7 +3249,7 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore,
 	break;
       }
     default:
-      pool_debug(data->repo->pool, SOLV_FATAL, "don't know how to handle type %d\n", key->type);
+      pool_debug(data->repo->pool, SOLV_FATAL, "repodata_serialize_key: don't know how to handle type %d\n", key->type);
       exit(1);
     }
   if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
@@ -3062,22 +3261,66 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore,
     }
 }
 
+/* create a circular linked list of all keys that share
+ * the same keyname */
+static Id *
+calculate_keylink(Repodata *data)
+{
+  int i, j;
+  Id *link;
+  Id maxkeyname = 0, *keytable = 0;
+  link = solv_calloc(data->nkeys, sizeof(Id));
+  if (data->nkeys <= 2)
+    return link;
+  for (i = 1; i < data->nkeys; i++)
+    {
+      Id n = data->keys[i].name;
+      if (n >= maxkeyname)
+	{
+	  keytable = solv_realloc2(keytable, n + 128, sizeof(Id));
+	  memset(keytable + maxkeyname, 0, (n + 128 - maxkeyname) * sizeof(Id));
+	  maxkeyname = n + 128;
+	}
+      j = keytable[n];
+      if (j)
+	link[i] = link[j];
+      else
+	j = i;
+      link[j] = i;
+      keytable[n] = i;
+    }
+  /* remove links that just point to themselfs */
+  for (i = 1; i < data->nkeys; i++)
+    if (link[i] == i)
+      link[i] = 0;
+  solv_free(keytable);
+  return link;
+}
+
 void
 repodata_internalize(Repodata *data)
 {
   Repokey *key, solvkey;
   Id entry, nentry;
-  Id schemaid, keyid, *schema, *sp, oldschema, *keyp, *seen;
+  Id schemaid, keyid, *schema, *sp, oldschemaid, *keyp, *seen;
+  Offset *oldincoreoffs = 0;
   int schemaidx;
   unsigned char *dp, *ndp;
-  int newschema, oldcount;
+  int neednewschema;
   struct extdata newincore;
   struct extdata newvincore;
   Id solvkeyid;
+  Id *keylink;
+  int haveoldkl;
 
   if (!data->attrs && !data->xattrs)
     return;
 
+#if 0
+  printf("repodata_internalize %d\n", data->repodataid);
+  printf("  attr data: %d K\n", data->attrdatalen / 1024);
+  printf("  attrid data: %d K\n", data->attriddatalen / (1024 / 4));
+#endif
   newvincore.buf = data->vincore;
   newvincore.len = data->vincorelen;
 
@@ -3101,142 +3344,197 @@ repodata_internalize(Repodata *data)
   data->mainschema = 0;
   data->mainschemaoffsets = solv_free(data->mainschemaoffsets);
 
+  keylink = calculate_keylink(data);
   /* join entry data */
   /* we start with the meta data, entry -1 */
   for (entry = -1; entry < nentry; entry++)
     {
-      memset(seen, 0, data->nkeys * sizeof(Id));
-      oldschema = 0;
+      oldschemaid = 0;
       dp = data->incoredata;
       if (dp)
 	{
 	  dp += entry >= 0 ? data->incoreoffset[entry] : 1;
-          dp = data_read_id(dp, &oldschema);
+          dp = data_read_id(dp, &oldschemaid);
 	}
+      memset(seen, 0, data->nkeys * sizeof(Id));
 #if 0
-fprintf(stderr, "oldschema %d\n", oldschema);
-fprintf(stderr, "schemata %d\n", data->schemata[oldschema]);
+fprintf(stderr, "oldschemaid %d\n", oldschemaid);
+fprintf(stderr, "schemata %d\n", data->schemata[oldschemaid]);
 fprintf(stderr, "schemadata %p\n", data->schemadata);
 #endif
-      /* seen: -1: old data  0: skipped  >0: id + 1 */
-      newschema = 0;
-      oldcount = 0;
+
+      /* seen: -1: old data,  0: skipped,  >0: id + 1 */
+      neednewschema = 0;
       sp = schema;
-      for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++)
+      haveoldkl = 0;
+      for (keyp = data->schemadata + data->schemata[oldschemaid]; *keyp; keyp++)
 	{
 	  if (seen[*keyp])
 	    {
-	      pool_debug(data->repo->pool, SOLV_FATAL, "Inconsistent old data (key occured twice).\n");
-	      exit(1);
+	      /* oops, should not happen */
+	      neednewschema = 1;
+	      continue;
 	    }
-	  seen[*keyp] = -1;
+	  seen[*keyp] = -1;	/* use old marker */
 	  *sp++ = *keyp;
-	  oldcount++;
+	  if (keylink[*keyp])
+	    haveoldkl = 1;	/* potential keylink conflict */
 	}
-      if (entry >= 0)
-	keyp = data->attrs ? data->attrs[entry] : 0;
-      else
+
+      /* strip solvables key */
+      if (entry < 0 && solvkeyid && seen[solvkeyid])
 	{
-	  /* strip solvables key */
 	  *sp = 0;
 	  for (sp = keyp = schema; *sp; sp++)
 	    if (*sp != solvkeyid)
 	      *keyp++ = *sp;
-	    else
-	      oldcount--;
 	  sp = keyp;
 	  seen[solvkeyid] = 0;
-	  keyp = data->xattrs ? data->xattrs[1] : 0;
+	  neednewschema = 1;
 	}
+
+      /* add new entries */
+      if (entry >= 0)
+	keyp = data->attrs ? data->attrs[entry] : 0;
+      else
+        keyp = data->xattrs ? data->xattrs[1] : 0;
       if (keyp)
         for (; *keyp; keyp += 2)
 	  {
 	    if (!seen[*keyp])
 	      {
-	        newschema = 1;
+	        neednewschema = 1;
 	        *sp++ = *keyp;
+		if (haveoldkl && keylink[*keyp])		/* this should be pretty rare */
+		  {
+		    Id kl;
+		    for (kl = keylink[*keyp]; kl != *keyp; kl = keylink[kl])
+		      if (seen[kl] == -1)
+		        {
+			  /* replacing old key kl, remove from schema and seen */
+			  Id *osp;
+			  for (osp = schema; osp < sp; osp++)
+			    if (*osp == kl)
+			      {
+			        memmove(osp, osp + 1, (sp - osp) * sizeof(Id));
+			        sp--;
+			        seen[kl] = 0;
+				break;
+			      }
+		        }
+		  }
 	      }
 	    seen[*keyp] = keyp[1] + 1;
 	  }
+
+      /* add solvables key if needed */
       if (entry < 0 && data->end != data->start)
 	{
-	  *sp++ = solvkeyid;
-	  newschema = 1;
+	  *sp++ = solvkeyid;	/* always last in schema */
+	  neednewschema = 1;
 	}
+
+      /* commit schema */
       *sp = 0;
-      if (newschema)
+      if (neednewschema)
         /* Ideally we'd like to sort the new schema here, to ensure
-	   schema equality independend of the ordering.  We can't do that
-	   yet.  For once see below (old ids need to come before new ids).
-	   An additional difficulty is that we also need to move
-	   the values with the keys.  */
+	   schema equality independend of the ordering. */
 	schemaid = repodata_schema2id(data, schema, 1);
       else
-	schemaid = oldschema;
+	schemaid = oldschemaid;
+
+      if (entry < 0)
+	{
+	  data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
+	  data->mainschema = schemaid;
+	}
+
+      /* find offsets in old incore data */
+      if (oldschemaid)
+	{
+	  Id *lastneeded = 0;
+	  for (sp = data->schemadata + data->schemata[oldschemaid]; *sp; sp++)
+	    if (seen[*sp] == -1)
+	      lastneeded = sp + 1;
+	  if (lastneeded)
+	    {
+	      if (!oldincoreoffs)
+	        oldincoreoffs = solv_malloc2(data->nkeys, 2 * sizeof(Offset));
+	      for (sp = data->schemadata + data->schemata[oldschemaid]; sp != lastneeded; sp++)
+		{
+		  /* Skip the data associated with this old key.  */
+		  key = data->keys + *sp;
+		  ndp = dp;
+		  if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
+		    {
+		      ndp = data_skip(ndp, REPOKEY_TYPE_ID);
+		      ndp = data_skip(ndp, REPOKEY_TYPE_ID);
+		    }
+		  else if (key->storage == KEY_STORAGE_INCORE)
+		    ndp = data_skip_key(data, ndp, key);
+		  oldincoreoffs[*sp * 2] = dp - data->incoredata;
+		  oldincoreoffs[*sp * 2 + 1] = ndp - dp;
+		  dp = ndp;
+		}
+	    }
+	}
 
+      /* just copy over the complete old entry (including the schemaid) if there was no new data */
+      if (entry >= 0 && !neednewschema && oldschemaid && (!data->attrs || !data->attrs[entry]) && dp)
+	{
+	  ndp = data->incoredata + data->incoreoffset[entry];
+	  data->incoreoffset[entry] = newincore.len;
+	  data_addblob(&newincore, ndp, dp - ndp);
+	  goto entrydone;
+	}
 
       /* Now create data blob.  We walk through the (possibly new) schema
 	 and either copy over old data, or insert the new.  */
-      /* XXX Here we rely on the fact that the (new) schema has the form
-	 o1 o2 o3 o4 ... | n1 n2 n3 ...
-	 (oX being the old keyids (possibly overwritten), and nX being
-	  the new keyids).  This rules out sorting the keyids in order
-	 to ensure a small schema count.  */
       if (entry >= 0)
         data->incoreoffset[entry] = newincore.len;
       data_addid(&newincore, schemaid);
-      if (entry == -1)
-	{
-	  data->mainschema = schemaid;
-	  data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
-	}
+
       /* we don't use a pointer to the schemadata here as repodata_serialize_key
        * may call repodata_schema2id() which might realloc our schemadata */
       for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
 	{
-	  if (entry == -1)
-	    data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
-	  if (keyid == solvkeyid)
-	    {
-	      /* add flexarray entry count */
-	      data_addid(&newincore, data->end - data->start);
-	      break;
-	    }
-	  key = data->keys + keyid;
-#if 0
-	  fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, pool_id2str(data->repo->pool, key->name), pool_id2str(data->repo->pool, key->type));
-#endif
-	  ndp = dp;
-	  if (oldcount)
+	  if (entry < 0)
 	    {
-	      /* Skip the data associated with this old key.  */
-	      if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
+	      data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
+	      if (keyid == solvkeyid)
 		{
-		  ndp = data_skip(dp, REPOKEY_TYPE_ID);
-		  ndp = data_skip(ndp, REPOKEY_TYPE_ID);
+		  /* add flexarray entry count */
+		  data_addid(&newincore, data->end - data->start);
+		  break;	/* always the last entry */
 		}
-	      else if (key->storage == KEY_STORAGE_INCORE)
-		ndp = data_skip_key(data, dp, key);
-	      oldcount--;
 	    }
 	  if (seen[keyid] == -1)
 	    {
-	      /* If this key was an old one _and_ was not overwritten with
-		 a different value copy over the old value (we skipped it
-		 above).  */
-	      if (dp != ndp)
-		data_addblob(&newincore, dp, ndp - dp);
-	      seen[keyid] = 0;
+	      if (oldincoreoffs[keyid * 2 + 1])
+		data_addblob(&newincore, data->incoredata + oldincoreoffs[keyid * 2], oldincoreoffs[keyid * 2 + 1]);
 	    }
 	  else if (seen[keyid])
+	    repodata_serialize_key(data, &newincore, &newvincore, schema, data->keys + keyid, seen[keyid] - 1);
+	}
+
+entrydone:
+      /* free memory */
+      if (entry >= 0 && data->attrs)
+	{
+	  if (data->attrs[entry])
+	    data->attrs[entry] = solv_free(data->attrs[entry]);
+	  if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
 	    {
-	      /* Otherwise we have a new value.  Parse it into the internal form.  */
-	      repodata_serialize_key(data, &newincore, &newvincore, schema, key, seen[keyid] - 1);
+	      compact_attrdata(data, entry + 1, nentry);	/* try to free some memory */
+#if 0
+	      printf("  attr data: %d K\n", data->attrdatalen / 1024);
+	      printf("  attrid data: %d K\n", data->attriddatalen / (1024 / 4));
+	      printf("  incore data: %d K\n", newincore.len / 1024);
+	      printf("  sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
+	      /* malloc_stats(); */
+#endif
 	    }
-	  dp = ndp;
 	}
-      if (entry >= 0 && data->attrs && data->attrs[entry])
-	data->attrs[entry] = solv_free(data->attrs[entry]);
     }
   /* free all xattrs */
   for (entry = 0; entry < data->nxattrs; entry++)
@@ -3250,6 +3548,8 @@ fprintf(stderr, "schemadata %p\n", data->schemadata);
   data->lastdatalen = 0;
   solv_free(schema);
   solv_free(seen);
+  solv_free(keylink);
+  solv_free(oldincoreoffs);
   repodata_free_schemahash(data);
 
   solv_free(data->incoredata);
@@ -3257,7 +3557,6 @@ fprintf(stderr, "schemadata %p\n", data->schemadata);
   data->incoredatalen = newincore.len;
   data->incoredatafree = 0;
 
-  solv_free(data->vincore);
   data->vincore = newvincore.buf;
   data->vincorelen = newvincore.len;
 
@@ -3268,6 +3567,10 @@ fprintf(stderr, "schemadata %p\n", data->schemadata);
   data->attrdatalen = 0;
   data->attriddatalen = 0;
   data->attrnum64datalen = 0;
+#if 0
+  printf("repodata_internalize %d done\n", data->repodataid);
+  printf("  incore data: %d K\n", data->incoredatalen / 1024);
+#endif
 }
 
 void
@@ -3380,44 +3683,19 @@ repodata_create_stubs(Repodata *data)
 	  xkeyname = 0;
 	  continue;
 	}
-      switch (di.key->type)
+      repodata_set_kv(sdata, SOLVID_META, di.key->name, di.key->type, &di.kv);
+      if (di.key->name == REPOSITORY_KEYS && di.key->type == REPOKEY_TYPE_IDARRAY)
 	{
-        case REPOKEY_TYPE_ID:
-	  repodata_set_id(sdata, SOLVID_META, di.key->name, di.kv.id);
-	  break;
-	case REPOKEY_TYPE_CONSTANTID:
-	  repodata_set_constantid(sdata, SOLVID_META, di.key->name, di.kv.id);
-	  break;
-	case REPOKEY_TYPE_STR:
-	  repodata_set_str(sdata, SOLVID_META, di.key->name, di.kv.str);
-	  break;
-	case REPOKEY_TYPE_VOID:
-	  repodata_set_void(sdata, SOLVID_META, di.key->name);
-	  break;
-	case REPOKEY_TYPE_NUM:
-	  repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv));
-	  break;
-	case_CHKSUM_TYPES:
-	  repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
-	  break;
-	case REPOKEY_TYPE_IDARRAY:
-	  repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
-	  if (di.key->name == REPOSITORY_KEYS)
+	  if (!xkeyname)
 	    {
-	      if (!xkeyname)
-		{
-		  if (!di.kv.eof)
-		    xkeyname = di.kv.id;
-		}
-	      else
-		{
-		  repodata_add_stubkey(sdata, xkeyname, di.kv.id);
-		  xkeyname = 0;
-		}
+	      if (!di.kv.eof)
+		xkeyname = di.kv.id;
+	    }
+	  else
+	    {
+	      repodata_add_stubkey(sdata, xkeyname, di.kv.id);
+	      xkeyname = 0;
 	    }
-	  break;
-	default:
-	  break;
 	}
     }
   dataiterator_free(&di);