- make file list conversion twice as fast by adding a 1k dir cache

diff --git a/ext/repo_rpmmd.c b/ext/repo_rpmmd.c
index c2b723f..2599632 100644 (file)
--- a/ext/repo_rpmmd.c
+++ b/ext/repo_rpmmd.c
@@ -1222,6 +1222,7 @@ repo_add_rpmmd(Repo *repo, FILE *fp, const char *language, int flags)
   join_freemem(&pd.jd);
   stringpool_free(&pd.cspool);
   solv_free(pd.cscache);
+  repodata_free_dircache(data);
 
   if (!(flags & REPO_NO_INTERNALIZE))
     repodata_internalize(data);

diff --git a/ext/repo_susetags.c b/ext/repo_susetags.c
index 06173cc..9f6a7aa 100644 (file)
--- a/ext/repo_susetags.c
+++ b/ext/repo_susetags.c
@@ -1194,6 +1194,7 @@ repo_add_susetags(Repo *repo, FILE *fp, Id defvendor, const char *language, int
     }
 
   solv_free(joinhash);
+  repodata_free_dircache(data);
   if (!(flags & REPO_NO_INTERNALIZE))
     repodata_internalize(data);
 

diff --git a/ext/testcase.c b/ext/testcase.c
index b1ef07d..9afa230 100644 (file)
--- a/ext/testcase.c
+++ b/ext/testcase.c
@@ -1029,6 +1029,7 @@ testcase_add_susetags(Repo *repo, FILE *fp, int flags)
     finish_solvable(pool, data, s, filelist, nfilelist);
   solv_free(line);
   solv_free(filelist);
+  repodata_free_dircache(data);
   if (!(flags & REPO_NO_INTERNALIZE))
     repodata_internalize(data);
   return 0;

diff --git a/src/libsolv.ver b/src/libsolv.ver
index 49d52d8..a9b95a9 100644 (file)
--- a/src/libsolv.ver
+++ b/src/libsolv.ver
@@ -165,6 +165,7 @@ SOLV_1.0 {
                repodata_extend;
                repodata_extend_block;
                repodata_free;
+               repodata_free_dircache;
                repodata_free_schemahash;
                repodata_freedata;
                repodata_globalize_id;

diff --git a/src/pool.c b/src/pool.c
index a633767..89c6c1a 100644 (file)
--- a/src/pool.c
+++ b/src/pool.c
@@ -936,6 +936,7 @@ addfileprovides_cb(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyV
          if (did)
            MAPSET(&cbd->useddirs, did);
        }
+      repodata_free_dircache(data);
     }
   if (value->id >= data->dirpool.ndirs || !MAPTST(&cbd->useddirs, value->id))
     return 0;

diff --git a/src/repodata.c b/src/repodata.c
index 106b983..eb77915 100644 (file)
--- a/src/repodata.c
+++ b/src/repodata.c
@@ -91,6 +91,8 @@ repodata_freedata(Repodata *data)
 
   solv_free(data->attrdata);
   solv_free(data->attriddata);
+
+  solv_free(data->dircache);
 }
 
 void
@@ -247,13 +249,27 @@ static inline const char *strchrnul(const char *str, char x)
 }
 #endif
 
+#define DIRCACHE_SIZE 41       /* < 1k */
+
+#ifdef DIRCACHE_SIZE
+struct dircache {
+  Id ids[DIRCACHE_SIZE];
+  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
+};
+#endif
+
 Id
 repodata_str2dir(Repodata *data, const char *dir, int create)
 {
   Id id, parent;
+#ifdef DIRCACHE_SIZE
+  const char *dirs;
+#endif
   const char *dire;
 
   parent = 0;
+  if (!*dir)
+    return 0;
   while (*dir == '/' && dir[1] == '/')
     dir++;
   if (*dir == '/' && !dir[1])
@@ -262,6 +278,31 @@ repodata_str2dir(Repodata *data, const char *dir, int create)
         return 1;
       return dirpool_add_dir(&data->dirpool, 0, 1, create);
     }
+#ifdef DIRCACHE_SIZE
+  dirs = dir;
+  if (data->dircache)
+    {
+      int l;
+      struct dircache *dircache = data->dircache;
+      l = strlen(dir);
+      while (l > 0)
+       {
+         if (l < DIRCACHE_SIZE && dircache->ids[l] && !memcmp(dircache->str + l * (l - 1) / 2, dir, l))
+           {
+             parent = dircache->ids[l];
+             dir += l;
+             if (!*dir)
+               return parent;
+             while (*dir == '/')
+               dir++;
+             break;
+           }
+         while (--l)
+           if (dir[l] == '/')
+             break;
+       }
+    }
+#endif
   while (*dir)
     {
       dire = strchrnul(dir, '/');
@@ -274,6 +315,19 @@ repodata_str2dir(Repodata *data, const char *dir, int create)
       parent = dirpool_add_dir(&data->dirpool, parent, id, create);
       if (!parent)
        return 0;
+#ifdef DIRCACHE_SIZE
+      if (!data->dircache)
+       data->dircache = solv_calloc(1, sizeof(struct dircache));
+      if (data->dircache)
+       {
+         int l = dire - dirs;
+         if (l < DIRCACHE_SIZE)
+           {
+             data->dircache->ids[l] = parent;
+             memcpy(data->dircache->str + l * (l - 1) / 2, dirs, l);
+           }
+       }
+#endif
       if (!*dire)
        break;
       dir = dire + 1;
@@ -283,6 +337,12 @@ repodata_str2dir(Repodata *data, const char *dir, int create)
   return parent;
 }
 
+void
+repodata_free_dircache(Repodata *data)
+{
+  data->dircache = solv_free(data->dircache);
+}
+
 const char *
 repodata_dir2str(Repodata *data, Id did, const char *suf)
 {

diff --git a/src/repodata.h b/src/repodata.h
index bf0d2dc..5a92e75 100644 (file)
--- a/src/repodata.h
+++ b/src/repodata.h
@@ -39,6 +39,9 @@ typedef struct _Repokey {
 #define KEY_STORAGE_INCORE              2
 #define KEY_STORAGE_VERTICAL_OFFSET     3
 
+#ifdef LIBSOLV_INTERNAL
+struct dircache;
+#endif
 
 typedef struct _Repodata {
   Id repodataid;               /* our id */
@@ -107,6 +110,9 @@ typedef struct _Repodata {
   Id lasthandle;
   Id lastkey;
   Id lastdatalen;
+
+  /* directory cache to speed up repodata_str2dir */
+  struct dircache *dircache;
 #endif
 
 } Repodata;
@@ -242,6 +248,7 @@ void repodata_set_idarray(Repodata *data, Id solvid, Id keyname, Queue *q);
 /* directory (for package file list) */
 void repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2);
 void repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str);
+void repodata_free_dircache(Repodata *data);
 
 
 /* Arrays */