X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=src%2Frepodata.c;h=d527a421cc9a910fe4e0a356c35f6bc2b6d3548b;hb=f78f5de33f216051d91b9ff143ace8f4579afecf;hp=a812ba84dda0d13202164aaf4a5994c937bf0e40;hpb=7b272a2e7df10744df1b7224b1abebeba64f88d6;p=platform%2Fupstream%2Flibsolv.git

diff --git a/src/repodata.c b/src/repodata.c
index a812ba8..d527a42 100644
--- a/src/repodata.c
+++ b/src/repodata.c
@@ -188,7 +188,7 @@ repodata_schema2id(Repodata *data, Id *schema, int create)
       data->schematahash = schematahash = solv_calloc(256, sizeof(Id));
       for (i = 1; i < data->nschemata; i++)
 	{
-	  for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
+	  for (sp = data->schemadata + data->schemata[i], h = 0; *sp;)
 	    h = h * 7 + *sp++;
 	  h &= 255;
 	  schematahash[h] = i;
@@ -474,7 +474,7 @@ static unsigned char *
 get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
 {
   unsigned char *dp;
-  if (!len)
+  if (len <= 0)
     return 0;
   if (off >= data->lastverticaloffset)
     {
@@ -489,6 +489,7 @@ get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
   off += data->verticaloffset[key - data->keys];
   /* fprintf(stderr, "key %d page %d\n", key->name, off / REPOPAGE_BLOBSIZE); */
   dp = repopagestore_load_page_range(&data->store, off / REPOPAGE_BLOBSIZE, (off + len - 1) / REPOPAGE_BLOBSIZE);
+  data->storestate++;
   if (dp)
     dp += off % REPOPAGE_BLOBSIZE;
   return dp;
@@ -567,17 +568,21 @@ solvid2data(Repodata *data, Id solvid, Id *schemap)
   unsigned char *dp = data->incoredata;
   if (!dp)
     return 0;
-  if (solvid == SOLVID_META)	/* META */
-    dp += 1;
-  else if (solvid == SOLVID_POS)	/* META */
+  if (solvid == SOLVID_META)
+    dp += 1;	/* offset of "meta" solvable */
+  else if (solvid == SOLVID_POS)
     {
       Pool *pool = data->repo->pool;
       if (data->repo != pool->pos.repo)
 	return 0;
       if (data != data->repo->repodata + pool->pos.repodataid)
 	return 0;
-      *schemap = pool->pos.schema;
-      return data->incoredata + pool->pos.dp;
+      dp += pool->pos.dp;
+      if (pool->pos.dp != 1)
+        {
+          *schemap = pool->pos.schema;
+          return dp;
+	}
     }
   else
     {
@@ -592,7 +597,7 @@ solvid2data(Repodata *data, Id solvid, Id *schemap)
  * data lookup
  */
 
-static inline unsigned char *
+static unsigned char *
 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
 {
   unsigned char *dp;
@@ -615,6 +620,8 @@ find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
     return 0;
   if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
     return dp;	/* no need to forward... */
+  if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
+    return 0;	/* get_data will not work, no need to forward */
   dp = forward_to_key(data, *kp, keyp, dp);
   if (!dp)
     return 0;
@@ -734,8 +741,13 @@ repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
   dp = find_key_data(data, solvid, keyname, &key);
   if (!dp)
     return 0;
-  if (!(key->type == REPOKEY_TYPE_MD5 || key->type == REPOKEY_TYPE_SHA1 || key->type == REPOKEY_TYPE_SHA256))
-    return 0;
+  switch (key->type)
+    {
+    case_CHKSUM_TYPES:
+      break;
+    default:
+      return 0;
+    }
   *typep = key->type;
   return dp;
 }
@@ -752,7 +764,7 @@ repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
   dp = find_key_data(data, solvid, keyname, &key);
   if (!dp)
     return 0;
-  if (key->type != REPOKEY_TYPE_IDARRAY && key->type != REPOKEY_TYPE_REL_IDARRAY)
+  if (key->type != REPOKEY_TYPE_IDARRAY)
     return 0;
   for (;;)
     {
@@ -764,6 +776,24 @@ repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
   return 1;
 }
 
+const void *
+repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp)
+{
+  unsigned char *dp;
+  Repokey *key;
+  Id len;
+
+  dp = find_key_data(data, solvid, keyname, &key);
+  if (!dp || key->type != REPOKEY_TYPE_BINARY)
+    {
+      *lenp = 0;
+      return 0;
+    }
+  dp = data_read_id(dp, &len);
+  *lenp = len;
+  return dp;
+}
+
 Id
 repodata_globalize_id(Repodata *data, Id id, int create)
 {
@@ -781,6 +811,23 @@ repodata_localize_id(Repodata *data, Id id, int create)
 }
 
 Id
+repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create)
+{
+  if (!id || !data || !fromdata)
+    return id;
+  if (!data->localpool || !fromdata->localpool)
+    {
+      if (fromdata->localpool)
+	id = repodata_globalize_id(fromdata, id, create);
+      if (data->localpool)
+	id = repodata_localize_id(data, id, create);
+      return id;
+    }
+  /* localpool is set in both data and fromdata */
+  return stringpool_str2id(&data->spool, stringpool_id2str(&fromdata->spool, id), create);
+}
+
+Id
 repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id voidid)
 {
   Id *ap;
@@ -808,7 +855,7 @@ repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id void
  */
 
 
-int
+const char *
 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
 {
   switch (key->type)
@@ -828,26 +875,26 @@ repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int f
 	  if (*s == ':' && s > kv->str)
 	    kv->str = s + 1;
 	}
-      return 1;
+      return kv->str;
     case REPOKEY_TYPE_STR:
-      return 1;
+      return kv->str;
     case REPOKEY_TYPE_DIRSTRARRAY:
       if (!(flags & SEARCH_FILES))
-	return 1;	/* match just the basename */
+	return kv->str;	/* match just the basename */
+      if (kv->num)
+	return kv->str;	/* already stringified */
       /* Put the full filename into kv->str.  */
       kv->str = repodata_dir2str(data, kv->id, kv->str);
-      /* And to compensate for that put the "empty" directory into
-	 kv->id, so that later calls to repodata_dir2str on this data
-	 come up with the same filename again.  */
-      kv->id = 0;
-      return 1;
-    case REPOKEY_TYPE_MD5:
-    case REPOKEY_TYPE_SHA1:
-    case REPOKEY_TYPE_SHA256:
+      kv->num = 1;	/* mark stringification */
+      return kv->str;
+    case_CHKSUM_TYPES:
       if (!(flags & SEARCH_CHECKSUMS))
 	return 0;	/* skip em */
+      if (kv->num)
+	return kv->str;	/* already stringified */
       kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
-      return 1;
+      kv->num = 1;	/* mark stringification */
+      return kv->str;
     default:
       return 0;
     }
@@ -1047,6 +1094,7 @@ solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
 int
 datamatcher_init(Datamatcher *ma, const char *match, int flags)
 {
+  match = match ? solv_strdup(match) : 0;
   ma->match = match;
   ma->flags = flags;
   ma->error = 0;
@@ -1061,17 +1109,37 @@ datamatcher_init(Datamatcher *ma, const char *match, int flags)
 	  ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
 	}
     }
+  if ((flags & SEARCH_FILES) != 0 && match)
+    {
+      /* prepare basename check */
+      if ((flags & SEARCH_STRINGMASK) == SEARCH_STRING || (flags & SEARCH_STRINGMASK) == SEARCH_STRINGEND)
+	{
+	  const char *p = strrchr(match, '/');
+	  ma->matchdata = (void *)(p ? p + 1 : match);
+	}
+      else if ((flags & SEARCH_STRINGMASK) == SEARCH_GLOB)
+	{
+	  const char *p;
+	  for (p = match + strlen(match) - 1; p >= match; p--)
+	    if (*p == '[' || *p == ']' || *p == '*' || *p == '?' || *p == '/')
+	      break;
+	  ma->matchdata = (void *)(p + 1);
+	}
+    }
   return ma->error;
 }
 
 void
 datamatcher_free(Datamatcher *ma)
 {
+  if (ma->match)
+    ma->match = solv_free((char *)ma->match);
   if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
     {
       regfree(ma->matchdata);
-      ma->matchdata = solv_free(ma->matchdata);
+      solv_free(ma->matchdata);
     }
+  ma->matchdata = 0;
 }
 
 int
@@ -1082,67 +1150,67 @@ datamatcher_match(Datamatcher *ma, const char *str)
     {
     case SEARCH_SUBSTRING:
       if (ma->flags & SEARCH_NOCASE)
-	{
-	  if (!strcasestr(str, ma->match))
-	    return 0;
-	}
+	return strcasestr(str, ma->match) != 0;
       else
-	{
-	  if (!strstr(str, ma->match))
-	    return 0;
-	}
-      break;
+	return strstr(str, ma->match) != 0;
     case SEARCH_STRING:
       if (ma->flags & SEARCH_NOCASE)
-	{
-	  if (strcasecmp(ma->match, str))
-	    return 0;
-	}
+	return !strcasecmp(ma->match, str);
       else
-	{
-	  if (strcmp(ma->match, str))
-	    return 0;
-	}
-      break;
+	return !strcmp(ma->match, str);
     case SEARCH_STRINGSTART:
       if (ma->flags & SEARCH_NOCASE)
-	{
-	  if (strncasecmp(ma->match, str, strlen(ma->match)))
-	    return 0;
-	}
+        return !strncasecmp(ma->match, str, strlen(ma->match));
       else
-	{
-	  if (strncmp(ma->match, str, strlen(ma->match)))
-	    return 0;
-	}
-      break;
+        return !strncmp(ma->match, str, strlen(ma->match));
     case SEARCH_STRINGEND:
       l = strlen(str) - strlen(ma->match);
       if (l < 0)
 	return 0;
       if (ma->flags & SEARCH_NOCASE)
-	{
-	  if (strcasecmp(ma->match, str + l))
-	    return 0;
-	}
+	return !strcasecmp(ma->match, str + l);
       else
-	{
-	  if (strcmp(ma->match, str + l))
-	    return 0;
-	}
-      break;
+	return !strcmp(ma->match, str + l);
     case SEARCH_GLOB:
-      if (fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0))
-	return 0;
-      break;
+      return !fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0);
     case SEARCH_REGEX:
-      if (regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0))
+      return !regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0);
+    default:
+      return 0;
+    }
+}
+
+/* check if the matcher can match the provides basename */
+
+int
+datamatcher_checkbasename(Datamatcher *ma, const char *basename)
+{
+  int l;
+  const char *match = ma->matchdata;
+  if (!match)
+    return 1;
+  switch (ma->flags & SEARCH_STRINGMASK)
+    {
+    case SEARCH_STRING:
+      break;
+    case SEARCH_STRINGEND:
+      if (match != ma->match)
+	break;		/* had slash, do exact match on basename */
+      /* FALLTHROUGH */
+    case SEARCH_GLOB:
+      /* check if the basename ends with match */
+      l = strlen(basename) - strlen(match);
+      if (l < 0)
 	return 0;
+      basename += l;
       break;
     default:
-      return 0;
+      return 1;	/* maybe matches */
     }
-  return 1;
+  if ((ma->flags & SEARCH_NOCASE) != 0)
+    return !strcasecmp(match, basename);
+  else
+    return !strcmp(match, basename);
 }
 
 int
@@ -1217,6 +1285,16 @@ void
 dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
 {
   *di = *from;
+  if (di->dupstr)
+    {
+      if (di->dupstr == di->kv.str)
+        di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
+      else
+	{
+	  di->dupstr = 0;
+	  di->dupstrn = 0;
+	}
+    }
   memset(&di->matcher, 0, sizeof(di->matcher));
   if (from->matcher.match)
     datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
@@ -1301,13 +1379,15 @@ dataiterator_free(Dataiterator *di)
 {
   if (di->matcher.match)
     datamatcher_free(&di->matcher);
+  if (di->dupstr)
+    solv_free(di->dupstr);
 }
 
-static inline unsigned char *
+static unsigned char *
 dataiterator_find_keyname(Dataiterator *di, Id keyname)
 {
-  Id *keyp = di->keyp;
-  Repokey *keys = di->data->keys;
+  Id *keyp;
+  Repokey *keys = di->data->keys, *key;
   unsigned char *dp;
 
   for (keyp = di->keyp; *keyp; keyp++)
@@ -1315,6 +1395,11 @@ dataiterator_find_keyname(Dataiterator *di, Id keyname)
       break;
   if (!*keyp)
     return 0;
+  key = keys + *keyp;
+  if (key->type == REPOKEY_TYPE_DELETED)
+    return 0;
+  if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
+    return 0;		/* get_data will not work, no need to forward */
   dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
   if (!dp)
     return 0;
@@ -1322,6 +1407,25 @@ dataiterator_find_keyname(Dataiterator *di, Id keyname)
   return dp;
 }
 
+static inline int
+is_filelist_extension(Repodata *data)
+{
+  int j;
+  if (!repodata_precheck_keyname(data, SOLVABLE_FILELIST))
+    return 0;
+  for (j = 1; j < data->nkeys; j++)
+    if (data->keys[j].name == SOLVABLE_FILELIST)
+      break;
+  if (j == data->nkeys)
+    return 0;
+  if (data->state != REPODATA_AVAILABLE)
+    return 1;
+  for (j = 1; j < data->nkeys; j++)
+    if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
+      return 0;
+  return 1;
+}
+
 static int
 dataiterator_filelistcheck(Dataiterator *di)
 {
@@ -1329,18 +1433,44 @@ dataiterator_filelistcheck(Dataiterator *di)
   int needcomplete = 0;
   Repodata *data = di->data;
 
-  if ((di->matcher.flags & SEARCH_COMPLETE_FILELIST) != 0)
+  if ((di->flags & SEARCH_COMPLETE_FILELIST) != 0)
     if (!di->matcher.match
        || ((di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_STRING
            && (di->matcher.flags & (SEARCH_STRINGMASK|SEARCH_NOCASE)) != SEARCH_GLOB)
-       || !repodata_filelistfilter_matches(di->data, di->matcher.match))
+       || !repodata_filelistfilter_matches(data, di->matcher.match))
       needcomplete = 1;
   if (data->state != REPODATA_AVAILABLE)
     return needcomplete ? 1 : 0;
-  for (j = 1; j < data->nkeys; j++)
-    if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
-      break;
-  return j == data->nkeys && !needcomplete ? 0 : 1;
+  if (!needcomplete)
+    {
+      /* we don't need the complete filelist, so ignore all stubs */
+      for (j = 1; j < data->nkeys; j++)
+	if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
+	  return 1;
+      return 0;
+    }
+  else
+    {
+      /* we need the complete filelist. check if we habe a filtered filelist and there's
+       * a extension with the complete filelist later on */
+      for (j = 1; j < data->nkeys; j++)
+	if (data->keys[j].name == SOLVABLE_FILELIST)
+	  break;
+      if (j == data->nkeys)
+	return 0;	/* does not have filelist */
+      for (j = 1; j < data->nkeys; j++)
+	if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
+	  break;
+      if (j == data->nkeys)
+	return 1;	/* this is the externsion */
+      while (data - data->repo->repodata + 1 < data->repo->nrepodata)
+	{
+	  data++;
+	  if (is_filelist_extension(data))
+	    return 0;
+	}
+      return 1;
+    }
 }
 
 int
@@ -1348,6 +1478,15 @@ dataiterator_step(Dataiterator *di)
 {
   Id schema;
 
+  if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate) {
+    unsigned int ddpoff = di->ddp - di->vert_ddp;
+    di->vert_off += ddpoff;
+    di->vert_len -= ddpoff;
+    di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len);
+    di->vert_storestate = di->data->storestate;
+    if (!di->ddp)
+      di->state = di_nextkey;
+  }
   for (;;)
     {
       switch (di->state)
@@ -1411,7 +1550,27 @@ dataiterator_step(Dataiterator *di)
 	case di_enterkey: di_enterkey:
 	  di->kv.entry = -1;
 	  di->key = di->data->keys + *di->keyp;
-	  di->ddp = get_data(di->data, di->key, &di->dp, di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0) ? 1 : 0);
+	  if (!di->dp)
+	    goto di_nextkey;
+	  /* this is get_data() modified to store vert_ data */
+	  if (di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
+	    {
+	      Id off, len;
+	      di->dp = data_read_id(di->dp, &off);
+	      di->dp = data_read_id(di->dp, &len);
+	      di->vert_ddp = di->ddp = get_vertical_data(di->data, di->key, off, len);
+	      di->vert_off = off;
+	      di->vert_len = len;
+	      di->vert_storestate = di->data->storestate;
+	    }
+	  else if (di->key->storage == KEY_STORAGE_INCORE)
+	    {
+	      di->ddp = di->dp;
+	      if (di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0))
+		di->dp = data_skip_key(di->data, di->dp, di->key);
+	    }
+	  else
+	    di->ddp = 0;
 	  if (!di->ddp)
 	    goto di_nextkey;
           if (di->key->type == REPOKEY_TYPE_DELETED)
@@ -1577,20 +1736,18 @@ dataiterator_step(Dataiterator *di)
 
       if (di->matcher.match)
 	{
+	  const char *str;
 	  /* simple pre-check so that we don't need to stringify */
-	  if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && di->matcher.match && (di->matcher.flags & (SEARCH_FILES|SEARCH_NOCASE|SEARCH_STRINGMASK)) == (SEARCH_FILES|SEARCH_STRING))
-	    {
-	      int l = strlen(di->matcher.match) - strlen(di->kv.str);
-	      if (l < 0 || strcmp(di->matcher.match + l, di->kv.str))
-		continue;
-	    }
-	  if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
+	  if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0)
+	    if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
+	      continue;
+	  if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
 	    {
 	      if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
 		return 1;
 	      continue;
 	    }
-	  if (!datamatcher_match(&di->matcher, di->kv.str))
+	  if (!datamatcher_match(&di->matcher, str))
 	    continue;
 	}
       else
@@ -1668,6 +1825,13 @@ dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
 	di->parents[i].kv.parent = &di->parents[i - 1].kv;
       di->kv.parent = &di->parents[di->nparents - 1].kv;
     }
+  di->dupstr = 0;
+  di->dupstrn = 0;
+  if (from->dupstr && from->dupstr == from->kv.str)
+    {
+      di->dupstrn = from->dupstrn;
+      di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
+    }
 }
 
 void
@@ -1807,11 +1971,56 @@ dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
 int
 dataiterator_match(Dataiterator *di, Datamatcher *ma)
 {
-  if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
+  const char *str;
+  if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
     return 0;
-  if (!ma)
-    return 1;
-  return datamatcher_match(ma, di->kv.str);
+  return ma ? datamatcher_match(ma, str) : 1;
+}
+
+void
+dataiterator_strdup(Dataiterator *di)
+{
+  int l = -1;
+
+  if (!di->kv.str || di->kv.str == di->dupstr)
+    return;
+  switch (di->key->type)
+    {
+    case_CHKSUM_TYPES:
+    case REPOKEY_TYPE_DIRSTRARRAY:
+      if (di->kv.num)	/* was it stringified into tmp space? */
+        l = strlen(di->kv.str) + 1;
+      break;
+    default:
+      break;
+    }
+  if (l < 0 && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
+    {
+      switch (di->key->type)
+	{
+	case REPOKEY_TYPE_STR:
+	case REPOKEY_TYPE_DIRSTRARRAY:
+	  l = strlen(di->kv.str) + 1;
+	  break;
+	case_CHKSUM_TYPES:
+	  l = solv_chksum_len(di->key->type);
+	  break;
+	case REPOKEY_TYPE_BINARY:
+	  l = di->kv.num;
+	  break;
+	}
+    }
+  if (l >= 0)
+    {
+      if (!di->dupstrn || di->dupstrn < l)
+	{
+	  di->dupstrn = l + 16;
+	  di->dupstr = solv_realloc(di->dupstr, di->dupstrn);
+	}
+      if (l)
+        memcpy(di->dupstr, di->kv.str, l);
+      di->kv.str = di->dupstr;
+    }
 }
 
 /************************************************************************
@@ -1893,6 +2102,7 @@ repodata_extend_block(Repodata *data, Id start, Id num)
     return;
   if (!data->incoreoffset)
     {
+      /* this also means that data->attrs is NULL */
       data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
       data->start = start;
       data->end = start + num;
@@ -2235,6 +2445,31 @@ evrid2vrstr(Pool *pool, Id evrid)
   return p != evr && *p == ':' && p[1] ? p + 1 : evr;
 }
 
+static inline void
+repodata_set_poolstrn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
+{
+  Id id;
+  if (data->localpool)
+    id = stringpool_strn2id(&data->spool, str, l, 1);
+  else
+    id = pool_strn2id(data->repo->pool, str, l, 1);
+  repodata_set_id(data, solvid, keyname, id);
+}
+
+static inline void
+repodata_set_strn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
+{
+  if (!str[l])
+    repodata_set_str(data, solvid, keyname, str);
+  else
+    {
+      char *s = solv_strdup(str);
+      s[l] = 0;
+      repodata_set_str(data, solvid, keyname, s);
+      free(s);
+    }
+}
+
 void
 repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
 {
@@ -2271,15 +2506,8 @@ repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, c
       str = pool_id2str(pool, s->arch);
       if (!strncmp(dir, str, l) && !str[l])
 	repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
-      else if (!dir[l])
-	repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir);
       else
-	{
-	  char *dir2 = solv_strdup(dir);
-	  dir2[l] = 0;
-	  repodata_set_str(data, solvid, SOLVABLE_MEDIADIR, dir2);
-	  free(dir2);
-	}
+	repodata_set_strn(data, solvid, SOLVABLE_MEDIADIR, dir, l);
     }
   fp = file;
   str = pool_id2str(pool, s->name);
@@ -2304,16 +2532,6 @@ repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, c
   repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
 }
 
-static inline void repodata_set_poolstrn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
-{
-  Id id;
-  if (data->localpool)
-    id = stringpool_strn2id(&data->spool, str, l, 1);
-  else
-    id = pool_strn2id(data->repo->pool, str, l, 1);
-  repodata_set_id(data, solvid, keyname, id);
-}
-
 /* XXX: medianr is currently not stored */
 void
 repodata_set_deltalocation(Repodata *data, Id handle, int medianr, const char *dir, const char *file)
@@ -2529,7 +2747,7 @@ repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle)
 }
 
 void
-repodata_set_deleted_uninternalized(Repodata *data, Id solvid, Id keyname)
+repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname)
 {
   Id *pp, *ap, **app;
   app = repodata_get_attrp(data, solvid);
@@ -2555,7 +2773,7 @@ repodata_set_deleted_uninternalized(Repodata *data, Id solvid, Id keyname)
 
 /* XXX: does not work correctly, needs fix in iterators! */
 void
-repodata_set_deleted(Repodata *data, Id solvid, Id keyname)
+repodata_unset(Repodata *data, Id solvid, Id keyname)
 {
   Repokey key;
   key.name = keyname;
@@ -2595,6 +2813,10 @@ repodata_swap_attrs(Repodata *data, Id dest, Id src)
   Id *tmpattrs;
   if (!data->attrs || dest == src)
     return;
+  if (dest < data->start || dest >= data->end)
+    repodata_extend(data, dest);
+  if (src < data->start || src >= data->end)
+    repodata_extend(data, src);
   tmpattrs = data->attrs[dest - data->start];
   data->attrs[dest - data->start] = data->attrs[src - data->start];
   data->attrs[src - data->start] = tmpattrs;
@@ -2685,6 +2907,96 @@ data_addblob(struct extdata *xd, unsigned char *blob, int len)
 
 /*********************************/
 
+/* this is to reduct memory usage when internalizing oversized repos */
+static void
+compact_attrdata(Repodata *data, int entry, int nentry)
+{
+  int i;
+  unsigned int attrdatastart = data->attrdatalen;
+  unsigned int attriddatastart = data->attriddatalen;
+  if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
+    return;
+  for (i = entry; i < nentry; i++)
+    {
+      Id v, *attrs = data->attrs[i];
+      if (!attrs)
+	continue;
+      for (; *attrs; attrs += 2)
+	{
+	  switch (data->keys[*attrs].type)
+	    {
+	    case REPOKEY_TYPE_STR:
+	    case REPOKEY_TYPE_BINARY:
+	    case_CHKSUM_TYPES:
+	      if (attrs[1] < attrdatastart)
+		 attrdatastart = attrs[1];
+	      break;
+	    case REPOKEY_TYPE_DIRSTRARRAY:
+	      for (v = attrs[1]; data->attriddata[v] ; v += 2)
+		if (data->attriddata[v + 1] < attrdatastart)
+		  attrdatastart = data->attriddata[v + 1];
+	      /* FALLTHROUGH */
+	    case REPOKEY_TYPE_IDARRAY:
+	    case REPOKEY_TYPE_DIRNUMNUMARRAY:
+	      if (attrs[1] < attriddatastart)
+		attriddatastart = attrs[1];
+	      break;
+	    case REPOKEY_TYPE_FIXARRAY:
+	    case REPOKEY_TYPE_FLEXARRAY:
+	      return;
+	    default:
+	      break;
+	    }
+	}
+    }
+#if 0
+  printf("compact_attrdata %d %d\n", entry, nentry);
+  printf("attrdatastart: %d\n", attrdatastart);
+  printf("attriddatastart: %d\n", attriddatastart);
+#endif
+  if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
+    return;
+  for (i = entry; i < nentry; i++)
+    {
+      Id v, *attrs = data->attrs[i];
+      if (!attrs)
+	continue;
+      for (; *attrs; attrs += 2)
+	{
+	  switch (data->keys[*attrs].type)
+	    {
+	    case REPOKEY_TYPE_STR:
+	    case REPOKEY_TYPE_BINARY:
+	    case_CHKSUM_TYPES:
+	      attrs[1] -= attrdatastart;
+	      break;
+	    case REPOKEY_TYPE_DIRSTRARRAY:
+	      for (v = attrs[1]; data->attriddata[v] ; v += 2)
+		data->attriddata[v + 1] -= attrdatastart;
+	      /* FALLTHROUGH */
+	    case REPOKEY_TYPE_IDARRAY:
+	    case REPOKEY_TYPE_DIRNUMNUMARRAY:
+	      attrs[1] -= attriddatastart;
+	      break;
+	    default:
+	      break;
+	    }
+	}
+    }
+  if (attrdatastart)
+    {
+      data->attrdatalen -= attrdatastart;
+      memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
+      data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
+    }
+  if (attriddatastart)
+    {
+      data->attriddatalen -= attriddatastart;
+      memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
+      data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
+    }
+}
+
 /* internalalize some key into incore/vincore data */
 
 static void
@@ -2719,9 +3031,18 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore,
     case REPOKEY_TYPE_SHA1:
       data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
       break;
+    case REPOKEY_TYPE_SHA224:
+      data_addblob(xd, data->attrdata + val, SIZEOF_SHA224);
+      break;
     case REPOKEY_TYPE_SHA256:
       data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
       break;
+    case REPOKEY_TYPE_SHA384:
+      data_addblob(xd, data->attrdata + val, SIZEOF_SHA384);
+      break;
+    case REPOKEY_TYPE_SHA512:
+      data_addblob(xd, data->attrdata + val, SIZEOF_SHA512);
+      break;
     case REPOKEY_TYPE_NUM:
       if (val & 0x80000000)
 	{
@@ -2841,7 +3162,8 @@ repodata_internalize(Repodata *data)
 {
   Repokey *key, solvkey;
   Id entry, nentry;
-  Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen;
+  Id schemaid, keyid, *schema, *sp, oldschema, *keyp, *seen;
+  int schemaidx;
   unsigned char *dp, *ndp;
   int newschema, oldcount;
   struct extdata newincore;
@@ -2851,6 +3173,11 @@ repodata_internalize(Repodata *data)
   if (!data->attrs && !data->xattrs)
     return;
 
+#if 0
+  printf("repodata_internalize %d\n", data->repodataid);
+  printf("  attr data: %d K\n", data->attrdatalen / 1024);
+  printf("  attrid data: %d K\n", data->attriddatalen / (1024 / 4));
+#endif
   newvincore.buf = data->vincore;
   newvincore.len = data->vincorelen;
 
@@ -2963,18 +3290,19 @@ fprintf(stderr, "schemadata %p\n", data->schemadata);
 	  data->mainschema = schemaid;
 	  data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
 	}
-      keypstart = data->schemadata + data->schemata[schemaid];
-      for (keyp = keypstart; *keyp; keyp++)
+      /* we don't use a pointer to the schemadata here as repodata_serialize_key
+       * may call repodata_schema2id() which might realloc our schemadata */
+      for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
 	{
 	  if (entry == -1)
-	    data->mainschemaoffsets[keyp - keypstart] = newincore.len;
-	  if (*keyp == solvkeyid)
+	    data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
+	  if (keyid == solvkeyid)
 	    {
 	      /* add flexarray entry count */
 	      data_addid(&newincore, data->end - data->start);
 	      break;
 	    }
-	  key = data->keys + *keyp;
+	  key = data->keys + keyid;
 #if 0
 	  fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, pool_id2str(data->repo->pool, key->name), pool_id2str(data->repo->pool, key->type));
 #endif
@@ -2991,26 +3319,38 @@ fprintf(stderr, "schemadata %p\n", data->schemadata);
 		ndp = data_skip_key(data, dp, key);
 	      oldcount--;
 	    }
-	  if (seen[*keyp] == -1)
+	  if (seen[keyid] == -1)
 	    {
 	      /* If this key was an old one _and_ was not overwritten with
 		 a different value copy over the old value (we skipped it
 		 above).  */
 	      if (dp != ndp)
 		data_addblob(&newincore, dp, ndp - dp);
-	      seen[*keyp] = 0;
+	      seen[keyid] = 0;
 	    }
-	  else if (seen[*keyp])
+	  else if (seen[keyid])
 	    {
-	      /* Otherwise we have a new value.  Parse it into the internal
-		 form.  */
-	      repodata_serialize_key(data, &newincore, &newvincore,
-				     schema, key, seen[*keyp] - 1);
+	      /* Otherwise we have a new value.  Parse it into the internal form.  */
+	      repodata_serialize_key(data, &newincore, &newvincore, schema, key, seen[keyid] - 1);
 	    }
 	  dp = ndp;
 	}
-      if (entry >= 0 && data->attrs && data->attrs[entry])
-	data->attrs[entry] = solv_free(data->attrs[entry]);
+      if (entry >= 0 && data->attrs)
+	{
+	  if (data->attrs[entry])
+	    data->attrs[entry] = solv_free(data->attrs[entry]);
+	  if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
+	    {
+	      compact_attrdata(data, entry + 1, nentry);	/* try to free some memory */
+#if 0
+	      printf("  attr data: %d K\n", data->attrdatalen / 1024);
+	      printf("  attrid data: %d K\n", data->attriddatalen / (1024 / 4));
+	      printf("  incore data: %d K\n", newincore.len / 1024);
+	      printf("  sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
+	      /* malloc_stats(); */
+#endif
+	    }
+	}
     }
   /* free all xattrs */
   for (entry = 0; entry < data->nxattrs; entry++)
@@ -3042,13 +3382,20 @@ fprintf(stderr, "schemadata %p\n", data->schemadata);
   data->attrdatalen = 0;
   data->attriddatalen = 0;
   data->attrnum64datalen = 0;
+#if 0
+  printf("repodata_internalize %d done\n", data->repodataid);
+  printf("  incore data: %d K\n", data->incoredatalen / 1024);
+#endif
 }
 
 void
 repodata_disable_paging(Repodata *data)
 {
   if (maybe_load_repodata(data, 0))
-    repopagestore_disable_paging(&data->store);
+    {
+      repopagestore_disable_paging(&data->store);
+      data->storestate++;
+    }
 }
 
 static void
@@ -3058,6 +3405,7 @@ repodata_load_stub(Repodata *data)
   Pool *pool = repo->pool;
   int r, i;
   struct _Pool_tmpspace oldtmpspace;
+  Datapos oldpos;
 
   if (!pool->loadcallback)
     {
@@ -3066,21 +3414,53 @@ repodata_load_stub(Repodata *data)
     }
   data->state = REPODATA_LOADING;
 
-  /* save tmp space */
+  /* save tmp space and pos */
   oldtmpspace = pool->tmpspace;
   memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
+  oldpos = pool->pos;
 
   r = pool->loadcallback(pool, data, pool->loadcallbackdata);
 
-  /* restore tmp space */
+  /* restore tmp space and pos */
   for (i = 0; i < POOL_TMPSPACEBUF; i++)
     solv_free(pool->tmpspace.buf[i]);
   pool->tmpspace = oldtmpspace;
+  if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
+    memset(&oldpos, 0, sizeof(oldpos));
+  pool->pos = oldpos;
 
   data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR;
 }
 
-void
+static inline void
+repodata_add_stubkey(Repodata *data, Id keyname, Id keytype)
+{
+  Repokey xkey;
+
+  xkey.name = keyname;
+  xkey.type = keytype;
+  xkey.storage = KEY_STORAGE_INCORE;
+  xkey.size = 0;
+  repodata_key2id(data, &xkey, 1);
+}
+
+static Repodata *
+repodata_add_stub(Repodata **datap)
+{
+  Repodata *data = *datap;
+  Repo *repo = data->repo;
+  Id repodataid = data - repo->repodata;
+  Repodata *sdata = repo_add_repodata(repo, 0);
+  data = repo->repodata + repodataid;
+  if (data->end > data->start)
+    repodata_extend_block(sdata, data->start, data->end - data->start);
+  sdata->state = REPODATA_STUB;
+  sdata->loadcallback = repodata_load_stub;
+  *datap = data;
+  return sdata;
+}
+
+Repodata *
 repodata_create_stubs(Repodata *data)
 {
   Repo *repo = data->repo;
@@ -3090,38 +3470,26 @@ repodata_create_stubs(Repodata *data)
   Dataiterator di;
   Id xkeyname = 0;
   int i, cnt = 0;
-  int repodataid;
-  int datastart, dataend;
 
-  repodataid = data - repo->repodata;
-  datastart = data->start;
-  dataend = data->end;
   dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
   while (dataiterator_step(&di))
-    {
-      if (di.data - repo->repodata != repodataid)
-	continue;
+    if (di.data == data)
       cnt++;
-    }
   dataiterator_free(&di);
   if (!cnt)
-    return;
+    return data;
   stubdataids = solv_calloc(cnt, sizeof(*stubdataids));
   for (i = 0; i < cnt; i++)
     {
-      sdata = repo_add_repodata(repo, 0);
-      if (dataend > datastart)
-        repodata_extend_block(sdata, datastart, dataend - datastart);
+      sdata = repodata_add_stub(&data);
       stubdataids[i] = sdata - repo->repodata;
-      sdata->state = REPODATA_STUB;
-      sdata->loadcallback = repodata_load_stub;
     }
   i = 0;
   dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
   sdata = 0;
   while (dataiterator_step(&di))
     {
-      if (di.data - repo->repodata != repodataid)
+      if (di.data != data)
 	continue;
       if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
 	{
@@ -3147,30 +3515,25 @@ repodata_create_stubs(Repodata *data)
 	case REPOKEY_TYPE_NUM:
 	  repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv));
 	  break;
-	case REPOKEY_TYPE_MD5:
-	case REPOKEY_TYPE_SHA1:
-	case REPOKEY_TYPE_SHA256:
+	case_CHKSUM_TYPES:
 	  repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
 	  break;
 	case REPOKEY_TYPE_IDARRAY:
 	  repodata_add_idarray(sdata, SOLVID_META, di.key->name, di.kv.id);
 	  if (di.key->name == REPOSITORY_KEYS)
 	    {
-	      Repokey xkey;
-
 	      if (!xkeyname)
 		{
 		  if (!di.kv.eof)
 		    xkeyname = di.kv.id;
-		  continue;
 		}
-	      xkey.name = xkeyname;
-              xkey.type = di.kv.id;
-              xkey.storage = KEY_STORAGE_INCORE;
-              xkey.size = 0;
-              repodata_key2id(sdata, &xkey, 1);
-              xkeyname = 0;
+	      else
+		{
+		  repodata_add_stubkey(sdata, xkeyname, di.kv.id);
+		  xkeyname = 0;
+		}
 	    }
+	  break;
 	default:
 	  break;
 	}
@@ -3179,6 +3542,7 @@ repodata_create_stubs(Repodata *data)
   for (i = 0; i < cnt; i++)
     repodata_internalize(repo->repodata + stubdataids[i]);
   solv_free(stubdataids);
+  return data;
 }
 
 unsigned int
@@ -3187,6 +3551,3 @@ repodata_memused(Repodata *data)
   return data->incoredatalen + data->vincorelen;
 }
 
-/*
-vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4:
-*/