Generate missing database indexes automatically

- Now that we know whether backing files were created on open, we can
  easily detect the need for index generation too. As index generation
  requires expensive plowing through the entire Packages database, we
  to create all missing indexes at once instead of risking several
  loops over Packages. So if we spot a missing index, we open all
  our indexes and generate all the missing bits in one go.
- Berkeley DB does support on-demand index generation for secondary
  indexes created with db->associate(), but as rpm's indexes are in
  a slightly different format (they store tag index besides the
  header number), using it would require various other
  changes too. Also the BDB indexing callback recieves the data
  in serialized format, which in rpm's case would mean either
  headerLoad() for every single indexed bit, or passing the in-memory
  header through some other indirect means.

diff --git a/lib/backend/db3.c b/lib/backend/db3.c
index 66ba160..bcdebea 100644 (file)
--- a/lib/backend/db3.c
+++ b/lib/backend/db3.c
@@ -578,6 +578,12 @@ int dbiOpen(rpmdb rpmdb, rpmTag rpmtag, dbiIndex * dbip)
      */
     if (oflags & DB_RDONLY)
        dbi->dbi_verify_on_close = 0;
+    
+    /* Any indexes created here mean we'll need an index rebuild */
+    if (dbiType(dbi) == DBI_SECONDARY && (oflags & DB_CREATE)) {
+       rpmlog(RPMLOG_DEBUG, "index %s needs creating\n", dbi->dbi_file);
+       rpmdb->db_buildindex++;
+    }
 
     dbi->dbi_db = db;
     dbi->dbi_oflags = oflags;

diff --git a/lib/backend/dbi.h b/lib/backend/dbi.h
index dd1ada6..6cb20d8 100644 (file)
--- a/lib/backend/dbi.h
+++ b/lib/backend/dbi.h
@@ -22,6 +22,7 @@ struct rpmdb_s {
     void *     db_dbenv;       /*!< Berkeley DB_ENV handle. */
     int                db_ndbi;        /*!< No. of tag indices. */
     dbiIndex * _dbi;           /*!< Tag indices. */
+    int                db_buildindex;  /*!< Index rebuild indicator */
 
     struct rpmop_s db_getops;
     struct rpmop_s db_putops;

diff --git a/lib/rpmdb.c b/lib/rpmdb.c
index d8f9780..6710779 100644 (file)
--- a/lib/rpmdb.c
+++ b/lib/rpmdb.c
@@ -74,6 +74,8 @@ typedef struct _dbiIndexSet {
     size_t alloced;                    /*!< alloced size */
 } * dbiIndexSet;
 
+static int doOpenAll(rpmdb db);
+static int addToIndex(dbiIndex dbi, rpmTag rpmtag, unsigned int hdrNum, Header h);
 static unsigned int pkgInstance(dbiIndex dbi, int alloc);
 static rpmdb rpmdbUnlink(rpmdb db);
 
@@ -115,6 +117,40 @@ static inline pbm_set * PBM_REALLOC(pbm_set ** sp, int * odp, int nd)
     return *sp;
 }
 
+static int buildIndexes(rpmdb db)
+{
+    int rc = 0;
+    Header h;
+    rpmdbMatchIterator mi;
+
+    rc += doOpenAll(db);
+
+    /* If the main db was just created, this is expected - dont whine */
+    if (!(db->_dbi[0]->dbi_oflags & DB_CREATE)) {
+       rpmlog(RPMLOG_WARNING,
+              _("Generating %d missing index(es), please wait...\n"),
+              db->db_buildindex);
+    }
+
+    /* Don't call us again */
+    db->db_buildindex = 0;
+
+    mi = rpmdbInitIterator(db, RPMDBI_PACKAGES, NULL, 0);
+    while ((h = rpmdbNextIterator(mi))) {
+       unsigned int hdrNum = headerGetInstance(h);
+       /* Build all secondary indexes which were created on open */
+       for (int dbix = 1; dbix < dbiTagsMax; dbix++) {
+           dbiIndex dbi = db->_dbi[dbix];
+           if (dbi && (dbi->dbi_oflags & DB_CREATE)) {
+               rc += addToIndex(dbi, dbiTags[dbix], hdrNum, h);
+           }
+       }
+    }
+    rpmdbFreeIterator(mi);
+    
+    return rc;
+}
+
 /** \ingroup dbi
  * Return handle for an index database.
  * @param db           rpm database
@@ -578,16 +614,28 @@ const char *rpmdbHome(rpmdb db)
     return dbdir;
 }
 
+static int doOpenAll(rpmdb db)
+{
+    int rc = 0;
+    for (int dbix = 0; dbix < dbiTagsMax; dbix++) {
+       dbiIndex dbi = db->_dbi[dbix];
+       if (dbi == NULL) {
+           rc += (rpmdbOpenIndex(db, dbiTags[dbix], db->db_flags) == NULL);
+       }
+    }
+    return rc;
+}
+
 int rpmdbOpenAll(rpmdb db)
 {
     int rc = 0;
 
     if (db == NULL) return -2;
 
-    for (int dbix = 0; dbix < dbiTagsMax; dbix++) {
-       if (db->_dbi[dbix] != NULL)
-           continue;
-       (void) rpmdbOpenIndex(db, dbiTags[dbix], db->db_flags);
+    rc += doOpenAll(db);
+
+    if (db->db_buildindex) {
+       rc += buildIndexes(db);
     }
     return rc;
 }
@@ -1964,6 +2012,9 @@ rpmdbMatchIterator rpmdbInitIterator(rpmdb db, rpmTag rpmtag,
     if (dbi == NULL)
        return NULL;
 
+    if (db->db_buildindex)
+       buildIndexes(db);
+
     /*
      * Handle label and file name special cases.
      * Otherwise, retrieve join keys for secondary lookup.