X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=src%2Frepodata.c;h=4ba1345b42b6f1cd3a9c7f4a54103cae6b36f24b;hb=40d5592805eaa650df84c50b9fed3cbdc4823fc5;hp=a61b80be6856c0e1d5e3b7e1c8029837884d6859;hpb=228d41221788430a09b85c488d62363c006c0993;p=platform%2Fupstream%2Flibsolv.git

diff --git a/src/repodata.c b/src/repodata.c
index a61b80b..4ba1345 100644
--- a/src/repodata.c
+++ b/src/repodata.c
@@ -188,7 +188,7 @@ repodata_schema2id(Repodata *data, Id *schema, int create)
       data->schematahash = schematahash = solv_calloc(256, sizeof(Id));
       for (i = 1; i < data->nschemata; i++)
 	{
-	  for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++)
+	  for (sp = data->schemadata + data->schemata[i], h = 0; *sp;)
 	    h = h * 7 + *sp++;
 	  h &= 255;
 	  schematahash[h] = i;
@@ -479,11 +479,11 @@ get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
   if (off >= data->lastverticaloffset)
     {
       off -= data->lastverticaloffset;
-      if (off + len > data->vincorelen)
+      if ((unsigned int)off + len > data->vincorelen)
 	return 0;
       return data->vincore + off;
     }
-  if (off + len > key->size)
+  if ((unsigned int)off + len > key->size)
     return 0;
   /* we now have the offset, go into vertical */
   off += data->verticaloffset[key - data->keys];
@@ -597,7 +597,7 @@ solvid2data(Repodata *data, Id solvid, Id *schemap)
  * data lookup
  */
 
-static inline unsigned char *
+static unsigned char *
 find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
 {
   unsigned char *dp;
@@ -620,6 +620,8 @@ find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
     return 0;
   if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
     return dp;	/* no need to forward... */
+  if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
+    return 0;	/* get_data will not work, no need to forward */
   dp = forward_to_key(data, *kp, keyp, dp);
   if (!dp)
     return 0;
@@ -739,8 +741,13 @@ repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
   dp = find_key_data(data, solvid, keyname, &key);
   if (!dp)
     return 0;
-  if (!(key->type == REPOKEY_TYPE_MD5 || key->type == REPOKEY_TYPE_SHA1 || key->type == REPOKEY_TYPE_SHA256))
-    return 0;
+  switch (key->type)
+    {
+    case_CHKSUM_TYPES:
+      break;
+    default:
+      return 0;
+    }
   *typep = key->type;
   return dp;
 }
@@ -769,6 +776,24 @@ repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
   return 1;
 }
 
+const void *
+repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp)
+{
+  unsigned char *dp;
+  Repokey *key;
+  Id len;
+
+  dp = find_key_data(data, solvid, keyname, &key);
+  if (!dp || key->type != REPOKEY_TYPE_BINARY)
+    {
+      *lenp = 0;
+      return 0;
+    }
+  dp = data_read_id(dp, &len);
+  *lenp = len;
+  return dp;
+}
+
 Id
 repodata_globalize_id(Repodata *data, Id id, int create)
 {
@@ -830,7 +855,7 @@ repodata_lookup_id_uninternalized(Repodata *data, Id solvid, Id keyname, Id void
  */
 
 
-int
+const char *
 repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
 {
   switch (key->type)
@@ -850,28 +875,26 @@ repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int f
 	  if (*s == ':' && s > kv->str)
 	    kv->str = s + 1;
 	}
-      return 1;
+      return kv->str;
     case REPOKEY_TYPE_STR:
-      return 1;
+      return kv->str;
     case REPOKEY_TYPE_DIRSTRARRAY:
       if (!(flags & SEARCH_FILES))
-	return 1;	/* match just the basename */
+	return kv->str;	/* match just the basename */
       if (kv->num)
-	return 1;	/* already stringified */
+	return kv->str;	/* already stringified */
       /* Put the full filename into kv->str.  */
       kv->str = repodata_dir2str(data, kv->id, kv->str);
       kv->num = 1;	/* mark stringification */
-      return 1;
-    case REPOKEY_TYPE_MD5:
-    case REPOKEY_TYPE_SHA1:
-    case REPOKEY_TYPE_SHA256:
+      return kv->str;
+    case_CHKSUM_TYPES:
       if (!(flags & SEARCH_CHECKSUMS))
 	return 0;	/* skip em */
       if (kv->num)
-	return 1;	/* already stringified */
+	return kv->str;	/* already stringified */
       kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
       kv->num = 1;	/* mark stringification */
-      return 1;
+      return kv->str;
     default:
       return 0;
     }
@@ -1071,7 +1094,8 @@ solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
 int
 datamatcher_init(Datamatcher *ma, const char *match, int flags)
 {
-  ma->match = match ? solv_strdup(match) : 0;
+  match = match ? solv_strdup(match) : 0;
+  ma->match = match;
   ma->flags = flags;
   ma->error = 0;
   ma->matchdata = 0;
@@ -1264,10 +1288,7 @@ dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
   if (di->dupstr)
     {
       if (di->dupstr == di->kv.str)
-	{
-	  di->dupstr = solv_malloc(di->dupstrn);
-	  memcpy(di->dupstr, from->dupstr, di->dupstrn);
-	}
+        di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
       else
 	{
 	  di->dupstr = 0;
@@ -1362,11 +1383,11 @@ dataiterator_free(Dataiterator *di)
     solv_free(di->dupstr);
 }
 
-static inline unsigned char *
+static unsigned char *
 dataiterator_find_keyname(Dataiterator *di, Id keyname)
 {
-  Id *keyp = di->keyp;
-  Repokey *keys = di->data->keys;
+  Id *keyp;
+  Repokey *keys = di->data->keys, *key;
   unsigned char *dp;
 
   for (keyp = di->keyp; *keyp; keyp++)
@@ -1374,6 +1395,11 @@ dataiterator_find_keyname(Dataiterator *di, Id keyname)
       break;
   if (!*keyp)
     return 0;
+  key = keys + *keyp;
+  if (key->type == REPOKEY_TYPE_DELETED)
+    return 0;
+  if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
+    return 0;		/* get_data will not work, no need to forward */
   dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
   if (!dp)
     return 0;
@@ -1418,6 +1444,8 @@ dataiterator_filelistcheck(Dataiterator *di)
   if (!needcomplete)
     {
       /* we don't need the complete filelist, so ignore all stubs */
+      if (data->repo->nrepodata == 2)
+	return 1;
       for (j = 1; j < data->nkeys; j++)
 	if (data->keys[j].name != REPOSITORY_SOLVABLES && data->keys[j].name != SOLVABLE_FILELIST)
 	  return 1;
@@ -1710,17 +1738,18 @@ dataiterator_step(Dataiterator *di)
 
       if (di->matcher.match)
 	{
+	  const char *str;
 	  /* simple pre-check so that we don't need to stringify */
 	  if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0)
 	    if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
 	      continue;
-	  if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
+	  if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
 	    {
 	      if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
 		return 1;
 	      continue;
 	    }
-	  if (!datamatcher_match(&di->matcher, di->kv.str))
+	  if (!datamatcher_match(&di->matcher, str))
 	    continue;
 	}
       else
@@ -1803,8 +1832,7 @@ dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
   if (from->dupstr && from->dupstr == from->kv.str)
     {
       di->dupstrn = from->dupstrn;
-      di->dupstr = solv_malloc(from->dupstrn);
-      memcpy(di->dupstr, from->dupstr, di->dupstrn);
+      di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
     }
 }
 
@@ -1945,11 +1973,10 @@ dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
 int
 dataiterator_match(Dataiterator *di, Datamatcher *ma)
 {
-  if (!repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags))
+  const char *str;
+  if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
     return 0;
-  if (!ma)
-    return 1;
-  return datamatcher_match(ma, di->kv.str);
+  return ma ? datamatcher_match(ma, str) : 1;
 }
 
 void
@@ -1961,9 +1988,7 @@ dataiterator_strdup(Dataiterator *di)
     return;
   switch (di->key->type)
     {
-    case REPOKEY_TYPE_MD5:
-    case REPOKEY_TYPE_SHA1:
-    case REPOKEY_TYPE_SHA256:
+    case_CHKSUM_TYPES:
     case REPOKEY_TYPE_DIRSTRARRAY:
       if (di->kv.num)	/* was it stringified into tmp space? */
         l = strlen(di->kv.str) + 1;
@@ -1979,14 +2004,8 @@ dataiterator_strdup(Dataiterator *di)
 	case REPOKEY_TYPE_DIRSTRARRAY:
 	  l = strlen(di->kv.str) + 1;
 	  break;
-	case REPOKEY_TYPE_MD5:
-	  l = SIZEOF_MD5;
-	  break;
-	case REPOKEY_TYPE_SHA1:
-	  l = SIZEOF_SHA1;
-	  break;
-	case REPOKEY_TYPE_SHA256:
-	  l = SIZEOF_SHA256;
+	case_CHKSUM_TYPES:
+	  l = solv_chksum_len(di->key->type);
 	  break;
 	case REPOKEY_TYPE_BINARY:
 	  l = di->kv.num;
@@ -2085,6 +2104,7 @@ repodata_extend_block(Repodata *data, Id start, Id num)
     return;
   if (!data->incoreoffset)
     {
+      /* this also means that data->attrs is NULL */
       data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
       data->start = start;
       data->end = start + num;
@@ -2795,6 +2815,10 @@ repodata_swap_attrs(Repodata *data, Id dest, Id src)
   Id *tmpattrs;
   if (!data->attrs || dest == src)
     return;
+  if (dest < data->start || dest >= data->end)
+    repodata_extend(data, dest);
+  if (src < data->start || src >= data->end)
+    repodata_extend(data, src);
   tmpattrs = data->attrs[dest - data->start];
   data->attrs[dest - data->start] = data->attrs[src - data->start];
   data->attrs[src - data->start] = tmpattrs;
@@ -2885,6 +2909,96 @@ data_addblob(struct extdata *xd, unsigned char *blob, int len)
 
 /*********************************/
 
+/* this is to reduct memory usage when internalizing oversized repos */
+static void
+compact_attrdata(Repodata *data, int entry, int nentry)
+{
+  int i;
+  unsigned int attrdatastart = data->attrdatalen;
+  unsigned int attriddatastart = data->attriddatalen;
+  if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
+    return;
+  for (i = entry; i < nentry; i++)
+    {
+      Id v, *attrs = data->attrs[i];
+      if (!attrs)
+	continue;
+      for (; *attrs; attrs += 2)
+	{
+	  switch (data->keys[*attrs].type)
+	    {
+	    case REPOKEY_TYPE_STR:
+	    case REPOKEY_TYPE_BINARY:
+	    case_CHKSUM_TYPES:
+	      if ((unsigned int)attrs[1] < attrdatastart)
+		 attrdatastart = attrs[1];
+	      break;
+	    case REPOKEY_TYPE_DIRSTRARRAY:
+	      for (v = attrs[1]; data->attriddata[v] ; v += 2)
+		if (data->attriddata[v + 1] < attrdatastart)
+		  attrdatastart = data->attriddata[v + 1];
+	      /* FALLTHROUGH */
+	    case REPOKEY_TYPE_IDARRAY:
+	    case REPOKEY_TYPE_DIRNUMNUMARRAY:
+	      if ((unsigned int)attrs[1] < attriddatastart)
+		attriddatastart = attrs[1];
+	      break;
+	    case REPOKEY_TYPE_FIXARRAY:
+	    case REPOKEY_TYPE_FLEXARRAY:
+	      return;
+	    default:
+	      break;
+	    }
+	}
+    }
+#if 0
+  printf("compact_attrdata %d %d\n", entry, nentry);
+  printf("attrdatastart: %d\n", attrdatastart);
+  printf("attriddatastart: %d\n", attriddatastart);
+#endif
+  if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
+    return;
+  for (i = entry; i < nentry; i++)
+    {
+      Id v, *attrs = data->attrs[i];
+      if (!attrs)
+	continue;
+      for (; *attrs; attrs += 2)
+	{
+	  switch (data->keys[*attrs].type)
+	    {
+	    case REPOKEY_TYPE_STR:
+	    case REPOKEY_TYPE_BINARY:
+	    case_CHKSUM_TYPES:
+	      attrs[1] -= attrdatastart;
+	      break;
+	    case REPOKEY_TYPE_DIRSTRARRAY:
+	      for (v = attrs[1]; data->attriddata[v] ; v += 2)
+		data->attriddata[v + 1] -= attrdatastart;
+	      /* FALLTHROUGH */
+	    case REPOKEY_TYPE_IDARRAY:
+	    case REPOKEY_TYPE_DIRNUMNUMARRAY:
+	      attrs[1] -= attriddatastart;
+	      break;
+	    default:
+	      break;
+	    }
+	}
+    }
+  if (attrdatastart)
+    {
+      data->attrdatalen -= attrdatastart;
+      memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
+      data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
+    }
+  if (attriddatastart)
+    {
+      data->attriddatalen -= attriddatastart;
+      memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
+      data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
+    }
+}
+
 /* internalalize some key into incore/vincore data */
 
 static void
@@ -2919,9 +3033,18 @@ repodata_serialize_key(Repodata *data, struct extdata *newincore,
     case REPOKEY_TYPE_SHA1:
       data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
       break;
+    case REPOKEY_TYPE_SHA224:
+      data_addblob(xd, data->attrdata + val, SIZEOF_SHA224);
+      break;
     case REPOKEY_TYPE_SHA256:
       data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
       break;
+    case REPOKEY_TYPE_SHA384:
+      data_addblob(xd, data->attrdata + val, SIZEOF_SHA384);
+      break;
+    case REPOKEY_TYPE_SHA512:
+      data_addblob(xd, data->attrdata + val, SIZEOF_SHA512);
+      break;
     case REPOKEY_TYPE_NUM:
       if (val & 0x80000000)
 	{
@@ -3041,7 +3164,8 @@ repodata_internalize(Repodata *data)
 {
   Repokey *key, solvkey;
   Id entry, nentry;
-  Id schemaid, *schema, *sp, oldschema, *keyp, *keypstart, *seen;
+  Id schemaid, keyid, *schema, *sp, oldschema, *keyp, *seen;
+  int schemaidx;
   unsigned char *dp, *ndp;
   int newschema, oldcount;
   struct extdata newincore;
@@ -3051,6 +3175,11 @@ repodata_internalize(Repodata *data)
   if (!data->attrs && !data->xattrs)
     return;
 
+#if 0
+  printf("repodata_internalize %d\n", data->repodataid);
+  printf("  attr data: %d K\n", data->attrdatalen / 1024);
+  printf("  attrid data: %d K\n", data->attriddatalen / (1024 / 4));
+#endif
   newvincore.buf = data->vincore;
   newvincore.len = data->vincorelen;
 
@@ -3163,18 +3292,19 @@ fprintf(stderr, "schemadata %p\n", data->schemadata);
 	  data->mainschema = schemaid;
 	  data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
 	}
-      keypstart = data->schemadata + data->schemata[schemaid];
-      for (keyp = keypstart; *keyp; keyp++)
+      /* we don't use a pointer to the schemadata here as repodata_serialize_key
+       * may call repodata_schema2id() which might realloc our schemadata */
+      for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
 	{
 	  if (entry == -1)
-	    data->mainschemaoffsets[keyp - keypstart] = newincore.len;
-	  if (*keyp == solvkeyid)
+	    data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
+	  if (keyid == solvkeyid)
 	    {
 	      /* add flexarray entry count */
 	      data_addid(&newincore, data->end - data->start);
 	      break;
 	    }
-	  key = data->keys + *keyp;
+	  key = data->keys + keyid;
 #if 0
 	  fprintf(stderr, "internalize %d(%d):%s:%s\n", entry, entry + data->start, pool_id2str(data->repo->pool, key->name), pool_id2str(data->repo->pool, key->type));
 #endif
@@ -3191,26 +3321,38 @@ fprintf(stderr, "schemadata %p\n", data->schemadata);
 		ndp = data_skip_key(data, dp, key);
 	      oldcount--;
 	    }
-	  if (seen[*keyp] == -1)
+	  if (seen[keyid] == -1)
 	    {
 	      /* If this key was an old one _and_ was not overwritten with
 		 a different value copy over the old value (we skipped it
 		 above).  */
 	      if (dp != ndp)
 		data_addblob(&newincore, dp, ndp - dp);
-	      seen[*keyp] = 0;
+	      seen[keyid] = 0;
 	    }
-	  else if (seen[*keyp])
+	  else if (seen[keyid])
 	    {
-	      /* Otherwise we have a new value.  Parse it into the internal
-		 form.  */
-	      repodata_serialize_key(data, &newincore, &newvincore,
-				     schema, key, seen[*keyp] - 1);
+	      /* Otherwise we have a new value.  Parse it into the internal form.  */
+	      repodata_serialize_key(data, &newincore, &newvincore, schema, key, seen[keyid] - 1);
 	    }
 	  dp = ndp;
 	}
-      if (entry >= 0 && data->attrs && data->attrs[entry])
-	data->attrs[entry] = solv_free(data->attrs[entry]);
+      if (entry >= 0 && data->attrs)
+	{
+	  if (data->attrs[entry])
+	    data->attrs[entry] = solv_free(data->attrs[entry]);
+	  if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
+	    {
+	      compact_attrdata(data, entry + 1, nentry);	/* try to free some memory */
+#if 0
+	      printf("  attr data: %d K\n", data->attrdatalen / 1024);
+	      printf("  attrid data: %d K\n", data->attriddatalen / (1024 / 4));
+	      printf("  incore data: %d K\n", newincore.len / 1024);
+	      printf("  sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
+	      /* malloc_stats(); */
+#endif
+	    }
+	}
     }
   /* free all xattrs */
   for (entry = 0; entry < data->nxattrs; entry++)
@@ -3242,6 +3384,10 @@ fprintf(stderr, "schemadata %p\n", data->schemadata);
   data->attrdatalen = 0;
   data->attriddatalen = 0;
   data->attrnum64datalen = 0;
+#if 0
+  printf("repodata_internalize %d done\n", data->repodataid);
+  printf("  incore data: %d K\n", data->incoredatalen / 1024);
+#endif
 }
 
 void
@@ -3261,6 +3407,7 @@ repodata_load_stub(Repodata *data)
   Pool *pool = repo->pool;
   int r, i;
   struct _Pool_tmpspace oldtmpspace;
+  Datapos oldpos;
 
   if (!pool->loadcallback)
     {
@@ -3269,16 +3416,20 @@ repodata_load_stub(Repodata *data)
     }
   data->state = REPODATA_LOADING;
 
-  /* save tmp space */
+  /* save tmp space and pos */
   oldtmpspace = pool->tmpspace;
   memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
+  oldpos = pool->pos;
 
   r = pool->loadcallback(pool, data, pool->loadcallbackdata);
 
-  /* restore tmp space */
+  /* restore tmp space and pos */
   for (i = 0; i < POOL_TMPSPACEBUF; i++)
     solv_free(pool->tmpspace.buf[i]);
   pool->tmpspace = oldtmpspace;
+  if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
+    memset(&oldpos, 0, sizeof(oldpos));
+  pool->pos = oldpos;
 
   data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR;
 }
@@ -3366,9 +3517,7 @@ repodata_create_stubs(Repodata *data)
 	case REPOKEY_TYPE_NUM:
 	  repodata_set_num(sdata, SOLVID_META, di.key->name, SOLV_KV_NUM64(&di.kv));
 	  break;
-	case REPOKEY_TYPE_MD5:
-	case REPOKEY_TYPE_SHA1:
-	case REPOKEY_TYPE_SHA256:
+	case_CHKSUM_TYPES:
 	  repodata_set_bin_checksum(sdata, SOLVID_META, di.key->name, di.key->type, (const unsigned char *)di.kv.str);
 	  break;
 	case REPOKEY_TYPE_IDARRAY:
@@ -3404,6 +3553,3 @@ repodata_memused(Repodata *data)
   return data->incoredatalen + data->vincorelen;
 }
 
-/*
-vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4:
-*/