- add repodata_join function to join a solvable block

- make repo_add_repodata smarter
- add repodata_create
- fix bug in repo_free_solvable block, the repodata was
  not shrunk
- add repodata_shrink function

diff --git a/ext/repo_content.c b/ext/repo_content.c
index 5d88ca4..5fe8b9b 100644 (file)
--- a/ext/repo_content.c
+++ b/ext/repo_content.c
@@ -219,25 +219,16 @@ repo_add_content(Repo *repo, FILE *fp, int flags)
   unsigned int numotherarchs = 0;
   Id *otherarchs = 0;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
-
   memset(&pd, 0, sizeof(pd));
   line = sat_malloc(1024);
   aline = 1024;
 
-  if (repo->nrepodata)
-    /* use last repodata */
-    data = repo->repodata + repo->nrepodata - 1;
-  else
-    data = repo_add_repodata(repo, 0);
-
   pd.repo = repo;
   linep = line;
   s = 0;
 
+  data = repo_add_repodata(repo, flags);
+
   for (;;)
     {
       char *key, *value;

diff --git a/ext/repo_deb.c b/ext/repo_deb.c
index 13cd107..9aaa3c1 100644 (file)
--- a/ext/repo_deb.c
+++ b/ext/repo_deb.c
@@ -283,10 +283,7 @@ repo_add_debpackages(Repo *repo, FILE *fp, int flags)
   int bufl, l, ll;
   Solvable *s;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
   buf = sat_malloc(4096);
   bufl = 4096;
   l = 0;
@@ -356,10 +353,7 @@ repo_add_debs(Repo *repo, const char **debs, int ndebs, int flags)
   Solvable *s;
   struct stat stb;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
   for (i = 0; i < ndebs; i++)
     {
       if ((fp = fopen(debs[i], "r")) == 0)

diff --git a/ext/repo_deltainfoxml.c b/ext/repo_deltainfoxml.c
index 6bc0cc9..222d398 100644 (file)
--- a/ext/repo_deltainfoxml.c
+++ b/ext/repo_deltainfoxml.c
@@ -553,10 +553,7 @@ repo_add_deltainfoxml(Repo *repo, FILE *fp, int flags)
   struct stateswitch *sw;
   Repodata *data;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
 
   memset(&pd, 0, sizeof(pd));
   for (i = 0, sw = stateswitches; sw->from != NUMSTATES; i++, sw++)

diff --git a/ext/repo_helix.c b/ext/repo_helix.c
index e62b810..94a286c 100644 (file)
--- a/ext/repo_helix.c
+++ b/ext/repo_helix.c
@@ -817,10 +817,7 @@ repo_add_helix(Repo *repo, FILE *fp, int flags)
   unsigned int now;
 
   now = sat_timems(0);
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
   
   /* prepare parsedata */
   memset(&pd, 0, sizeof(pd));

diff --git a/ext/repo_products.c b/ext/repo_products.c
index e751d84..5766ebe 100644 (file)
--- a/ext/repo_products.c
+++ b/ext/repo_products.c
@@ -437,10 +437,7 @@ repo_add_code11_products(Repo *repo, const char *dirpath, int flags)
   DIR *dir;
   int i;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
 
   memset(&pd, 0, sizeof(pd));
   pd.repo = repo;

diff --git a/ext/repo_repomdxml.c b/ext/repo_repomdxml.c
index 9b27804..915bf1e 100644 (file)
--- a/ext/repo_repomdxml.c
+++ b/ext/repo_repomdxml.c
@@ -459,10 +459,7 @@ repo_add_repomdxml(Repo *repo, FILE *fp, int flags)
   int i, l;
   struct stateswitch *sw;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
 
   memset(&pd, 0, sizeof(pd));
   pd.timestamp = 0;

diff --git a/ext/repo_rpmdb.c b/ext/repo_rpmdb.c
index b2be292..b1b07ba 100644 (file)
--- a/ext/repo_rpmdb.c
+++ b/ext/repo_rpmdb.c
@@ -1306,10 +1306,7 @@ repo_add_rpmdb(Repo *repo, Repo *ref, const char *rootdir, int flags)
   if (!rootdir)
     rootdir = "";
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
 
   if (ref && !(ref->nsolvables && ref->rpmdbid))
     ref = 0;
@@ -1706,10 +1703,7 @@ repo_add_rpms(Repo *repo, const char **rpms, int nrpms, int flags)
   Id chksumtype = 0;
   void *chksumh = 0;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
 
   if ((flags & RPM_ADD_WITH_SHA256SUM) != 0)
     chksumtype = REPOKEY_TYPE_SHA256;
@@ -2856,10 +2850,7 @@ repo_add_rpmdb_pubkeys(Repo *repo, const char *rootdir, int flags)
   Repodata *data;
   Solvable *s;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
 
   memset(&state, 0, sizeof(state));
   if (!(state.dbenv = opendbenv(rootdir)))
@@ -2897,10 +2888,7 @@ repo_add_pubkeys(Repo *repo, const char **keys, int nkeys, int flags)
   int i, bufl, l, ll;
   FILE *fp;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
   buf = 0;
   bufl = 0;
   for (i = 0; i < nkeys; i++)
@@ -2927,4 +2915,6 @@ repo_add_pubkeys(Repo *repo, const char **keys, int nkeys, int flags)
       pubkey2solvable(s, data, buf);
     }
   sat_free(buf);
+  if (!(flags & REPO_NO_INTERNALIZE))
+    repodata_internalize(data);
 }

diff --git a/ext/repo_rpmmd.c b/ext/repo_rpmmd.c
index 12d63bc..10dbf4e 100644 (file)
--- a/ext/repo_rpmmd.c
+++ b/ext/repo_rpmmd.c
@@ -1099,10 +1099,7 @@ repo_add_rpmmd(Repo *repo, FILE *fp, const char *language, int flags)
   unsigned int now;
 
   now = sat_timems(0);
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
 
   memset(&pd, 0, sizeof(pd));
   for (i = 0, sw = stateswitches; sw->from != NUMSTATES; i++, sw++)

diff --git a/ext/repo_susetags.c b/ext/repo_susetags.c
index b6f7b33..a7b0a51 100644 (file)
--- a/ext/repo_susetags.c
+++ b/ext/repo_susetags.c
@@ -424,10 +424,7 @@ repo_add_susetags(Repo *repo, FILE *fp, Id defvendor, const char *language, int
   if ((flags & SUSETAGS_EXTEND) && repo->nrepodata)
     indesc = 1;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
 
   memset(&pd, 0, sizeof(pd));
   line = malloc(1024);

diff --git a/ext/repo_updateinfoxml.c b/ext/repo_updateinfoxml.c
index 5e4b1f8..c2bf69e 100644 (file)
--- a/ext/repo_updateinfoxml.c
+++ b/ext/repo_updateinfoxml.c
@@ -571,10 +571,7 @@ repo_add_updateinfoxml(Repo *repo, FILE *fp, int flags)
   struct stateswitch *sw;
   Repodata *data;
 
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
+  data = repo_add_repodata(repo, flags);
 
   memset(&pd, 0, sizeof(pd));
   for (i = 0, sw = stateswitches; sw->from != NUMSTATES; i++, sw++)

diff --git a/ext/repo_zyppdb.c b/ext/repo_zyppdb.c
index cb0b1c9..f69c7bf 100644 (file)
--- a/ext/repo_zyppdb.c
+++ b/ext/repo_zyppdb.c
@@ -346,11 +346,7 @@ repo_add_zyppdb_products(Repo *repo, const char *dirpath, int flags)
   FILE *fp;
   Repodata *data;
   
-  if (!(flags & REPO_REUSE_REPODATA))
-    data = repo_add_repodata(repo, 0);
-  else
-    data = repo_last_repodata(repo);
-
+  data = repo_add_repodata(repo, flags);
   memset(&pd, 0, sizeof(pd));
   pd.repo = repo;
   pd.pool = repo->pool;

diff --git a/src/repo.c b/src/repo.c
index 2fe502d..943a1c1 100644 (file)
--- a/src/repo.c
+++ b/src/repo.c
@@ -128,6 +128,22 @@ repo_freeallrepos(Pool *pool, int reuseids)
   pool_free_solvable_block(pool, 2, pool->nsolvables - 2, reuseids);
 }
 
+void repo_free_solvable_block(Repo *repo, Id start, int count, int reuseids)
+{
+  Solvable *s; 
+  Repodata *data;
+  int i;
+  if (start + count == repo->end)
+    repo->end -= count;
+  repo->nsolvables -= count;
+  for (s = repo->pool->solvables + start, i = count; i--; s++)
+    s->repo = 0;
+  pool_free_solvable_block(repo->pool, start, count, reuseids);
+  for (i = 0, data = repo->repodata; i < repo->nrepodata; i++, data++)
+    if (data->end > repo->end)
+      repodata_shrink(data, repo->end);
+}
+
 
 /* repository sidedata is solvable data allocated on demand.
  * It is used for data that is normally not present
@@ -967,15 +983,16 @@ repo_lookup_void(Repo *repo, Id entry, Id keyname)
 /***********************************************************************/
 
 Repodata *
-repo_add_repodata(Repo *repo, int localpool)
+repo_add_repodata(Repo *repo, int flags)
 {
-  Repodata *data;
-
-  repo->nrepodata++;
-  repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata, sizeof(*data));
-  data = repo->repodata + repo->nrepodata - 1;
-  repodata_initdata(data, repo, localpool);
-  return data;
+  if ((flags & REPO_REUSE_REPODATA) != 0)
+    {
+      int i;
+      for (i = repo->nrepodata - 1; i >= 0; i--)
+       if (repo->repodata[i].state != REPODATA_STUB)
+         return repo->repodata + i;
+    }
+  return repodata_create(repo, (flags & REPO_LOCALPOOL) ? 1 : 0);
 }
 
 Repodata *

diff --git a/src/repo.h b/src/repo.h
index ee8f881..65114b7 100644 (file)
--- a/src/repo.h
+++ b/src/repo.h
@@ -47,6 +47,7 @@ typedef struct _Repo {
 extern Repo *repo_create(Pool *pool, const char *name);
 extern void repo_free(Repo *repo, int reuseids);
 extern void repo_freeallrepos(Pool *pool, int reuseids);
+extern void repo_free_solvable_block(Repo *repo, Id start, int count, int reuseids);
 extern void *repo_sidedata_create(Repo *repo, size_t size);
 extern void *repo_sidedata_extend(Repo *repo, void *b, size_t size, Id p, int count);
 
@@ -102,18 +103,6 @@ static inline Id repo_add_solvable_block(Repo *repo, int count)
   return p;
 }
 
-static inline void repo_free_solvable_block(Repo *repo, Id start, int count, int reuseids)
-{
-  extern void pool_free_solvable_block(Pool *pool, Id start, int count, int reuseids);
-  Solvable *s;
-  int i;
-  if (start + count == repo->end)
-    repo->end -= count;
-  repo->nsolvables -= count;
-  for (s = repo->pool->solvables + start, i = count; i--; s++)
-    s->repo = 0;
-  pool_free_solvable_block(repo->pool, start, count, reuseids);
-}
 
 #define FOR_REPO_SOLVABLES(r, p, s)                                            \
   for (p = (r)->start, s = (r)->pool->solvables + p; p < (r)->end; p++, s = (r)->pool->solvables + p)  \
@@ -193,8 +182,9 @@ typedef struct _KeyValue {
 /* standard flags used in the repo_add functions */
 #define REPO_REUSE_REPODATA            (1 << 0)
 #define REPO_NO_INTERNALIZE            (1 << 1)
+#define REPO_LOCALPOOL                 (1 << 2)
 
-Repodata *repo_add_repodata(Repo *repo, int localpool);
+Repodata *repo_add_repodata(Repo *repo, int flags);
 Repodata *repo_last_repodata(Repo *repo);
 
 void repo_search(Repo *repo, Id p, Id key, const char *match, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata);

diff --git a/src/repodata.c b/src/repodata.c
index 6df3717..13b4922 100644 (file)
--- a/src/repodata.c
+++ b/src/repodata.c
@@ -28,6 +28,8 @@
 #include "pool.h"
 #include "poolid_private.h"
 #include "util.h"
+#include "hash.h"
+#include "chksum.h"
 
 #include "repopack.h"
 #include "repopage.h"
@@ -95,6 +97,18 @@ repodata_freedata(Repodata *data)
   sat_free(data->attriddata);
 }
 
+Repodata *
+repodata_create(Repo *repo, int localpool)
+{
+  Repodata *data;
+
+  repo->nrepodata++;
+  repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata, sizeof(*data));
+  data = repo->repodata + repo->nrepodata - 1;
+  repodata_initdata(data, repo, localpool);
+  return data;
+}
+
 void
 repodata_free(Repodata *data)
 {
@@ -1598,6 +1612,36 @@ repodata_extend(Repodata *data, Id p)
     }
 }
 
+void
+repodata_shrink(Repodata *data, int end)
+{
+  int i;
+
+  if (data->end <= end)
+    return;
+  if (data->start >= end)
+    {
+      if (data->attrs)
+       {
+         for (i = 0; i < data->end - data->start; i++)
+           sat_free(data->attrs[i]);
+          data->attrs = sat_free(data->attrs);
+       }
+      data->incoreoffset = sat_free(data->incoreoffset);
+      data->start = data->end = 0;
+      return;
+    }
+  if (data->attrs)
+    {
+      for (i = end; i < data->end; i++)
+       sat_free(data->attrs[i - data->start]);
+      data->attrs = sat_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
+    }
+  if (data->incoreoffset)
+    data->incoreoffset = sat_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
+  data->end = end;
+}
+
 /* extend repodata so that it includes solvables from start to start + num - 1 */
 void
 repodata_extend_block(Repodata *data, Id start, Id num)
@@ -2529,6 +2573,98 @@ repodata_disable_paging(Repodata *data)
     repopagestore_disable_paging(&data->store);
 }
 
+static inline Hashval
+repodata_join_hash(Solvable *s, Id joinkey)
+{
+  if (joinkey == SOLVABLE_NAME)
+    return s->name;
+  if (joinkey == SOLVABLE_CHECKSUM)
+    {
+      Id type;
+      const unsigned char *chk = solvable_lookup_bin_checksum(s, joinkey, &type);
+      if (chk)
+        return 1 << 31 | chk[0] << 24 | chk[1] << 16 | chk[2] << 7 | chk[3];
+    }
+  return 0;
+}
+
+static inline int
+repodata_join_match(Solvable *s1, Solvable *s2, Id joinkey)
+{
+  if (joinkey == SOLVABLE_NAME)
+    return s1->name == s2->name && s1->evr == s2->evr && s1->arch == s2->arch ? 1 : 0;
+  if (joinkey == SOLVABLE_CHECKSUM)
+    {
+      const unsigned char *chk1, *chk2;
+      Id type1, type2;
+      chk1 = solvable_lookup_bin_checksum(s1, joinkey, &type1);
+      if (!chk1)
+       return 0;
+      chk2 = solvable_lookup_bin_checksum(s2, joinkey, &type2);
+      if (!chk2 || type1 != type2)
+       return 0;
+      return memcmp(chk1, chk2, sat_chksum_len(type1)) ? 0 : 1;
+    }
+  return 0;
+}
+
+void
+repodata_join(Repodata *data, Id joinkey)
+{
+  Repo *repo = data->repo;
+  Pool *pool = repo->pool;
+  Hashmask hm = mkmask(repo->nsolvables);
+  Hashtable ht = sat_calloc(hm + 1, sizeof(*ht));
+  Hashval h, hh;
+  int i, datastart, dataend;
+  Solvable *s;
+
+  datastart = data->start;
+  dataend = data->end;
+  if (datastart == dataend || repo->start == repo->end)
+    return;
+  FOR_REPO_SOLVABLES(repo, i, s)
+    {
+      if (i >= datastart && i < dataend)
+       continue;
+      h = repodata_join_hash(s, joinkey);
+      if (!h)
+       continue;
+      h &= hm;
+      hh = HASHCHAIN_START;
+      while (ht[h])
+        h = HASHCHAIN_NEXT(h, hh, hm);
+      ht[h] = i;
+    }
+  for (i = datastart; i < dataend; i++)
+    {
+      Solvable *s = pool->solvables + i;
+      if (s->repo != data->repo)
+       continue;
+      if (!data->attrs[i - data->start])
+       continue;
+      h = repodata_join_hash(s, joinkey);
+      if (!h)
+       continue;
+      h &= hm;
+      hh = HASHCHAIN_START;
+      while (ht[h])
+       {
+         Solvable *s2 = pool->solvables + ht[h];
+         if (repodata_join_match(s, s2, joinkey))
+           {
+             /* a match! move data over */
+             repodata_extend(data, ht[h]);
+             repodata_merge_attrs(data, ht[h], i);
+           }
+          h = HASHCHAIN_NEXT(h, hh, hm);
+       }
+    }
+  sat_free(ht);
+  /* all done! now free solvables */
+  repo_free_solvable_block(repo, datastart, dataend - datastart, 1);
+}
+
 /*
 vim:cinoptions={.5s,g0,p5,t0,(0,^-0.5s,n-0.5s:tw=78:cindent:sw=4:
 */

diff --git a/src/repodata.h b/src/repodata.h
index e0f2022..ed3c474 100644 (file)
--- a/src/repodata.h
+++ b/src/repodata.h
@@ -115,6 +115,8 @@ typedef struct _Repodata {
  */
 void repodata_initdata(Repodata *data, struct _Repo *repo, int localpool);
 void repodata_freedata(Repodata *data);
+
+Repodata *repodata_create(struct _Repo *repo, int localpool);
 void repodata_free(Repodata *data);
 
 
@@ -179,6 +181,7 @@ const unsigned char *repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id
  */
 void repodata_extend(Repodata *data, Id p);
 void repodata_extend_block(Repodata *data, Id p, int num);
+void repodata_shrink(Repodata *data, int end);
 
 /* internalize freshly set data, so that it is found by the search
  * functions and written out */
@@ -225,6 +228,8 @@ void repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id ghandle);
 */
 void repodata_merge_attrs(Repodata *data, Id dest, Id src);
 
+void repodata_join(Repodata *data, Id joinkey);
+
 /*
  * load all paged data, used to speed up copying in repo_rpmdb
  */