From 0fae1010f0c9a563ea79a01d82eea80e7e8f7ece Mon Sep 17 00:00:00 2001 From: jbj Date: Mon, 26 Aug 2002 18:56:11 +0000 Subject: [PATCH] - set cachesize without a dbenv, the default is far too small. - db: don't return EACCES on db->close w/o environment. - unify cachesize configuration, with (or without) a dbenv. - comments regarding unsupported (yet) db-4.1.17 functionality. CVS patchset: 5674 CVS date: 2002/08/26 18:56:11 --- CHANGES | 6 + configure.ac | 48 +- db/env/db_salloc.c | 34 +- db/mp/mp_sync.c | 909 +++++++++++++++++------------------- macros.in | 8 +- popt/{configure.in => configure.ac} | 2 +- rpm.spec.in | 323 +------------ rpmdb/db3.c | 78 +++- rpmdb/dbconfig.c | 12 +- rpmdb/rpmdb.h | 5 +- 10 files changed, 562 insertions(+), 863 deletions(-) rename popt/{configure.in => configure.ac} (98%) diff --git a/CHANGES b/CHANGES index a2fc07a..4474679 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +4.1 -> 4.2: + - set cachesize without a dbenv, the default is far too small. + - db: don't return EACCES on db->close w/o environment. + - unify cachesize configuration, with (or without) a dbenv. + - comments regarding unsupported (yet) db-4.1.17 functionality. + 4.0.4 -> 4.1: - loosely wire beecrypt library into rpm. - drop rpmio/base64.[ch] in favor of beecrypt. diff --git a/configure.ac b/configure.ac index e059ca2..6ea9ef7 100644 --- a/configure.ac +++ b/configure.ac @@ -8,7 +8,7 @@ AC_INIT(rpmqv.c) AC_CANONICAL_SYSTEM AC_PREREQ(2.12) dnl Minimum Autoconf version required. AC_CONFIG_HEADERS -AM_INIT_AUTOMAKE(rpm, 4.1) +AM_INIT_AUTOMAKE(rpm, 4.2) AM_CONFIG_HEADER(config.h) dnl XXX AM_MAINTAINER_MODE @@ -16,26 +16,32 @@ dnl XXX AM_MAINTAINER_MODE dnl Set of available languages. ALL_LINGUAS="cs da de fi fr gl is ja ko no pl pt pt_BR ro ru sk sl sr sv tr" -# echo " -# **************************************************************************** -# * * -# * *** WARNING WARNING WARNING *** * -# * * -# * This is source code from the development branch of rpm-4.1. * -# * * -# * If you want the "production" rpm-4.0.4 code, then you should either use * -# * an rpm-4.0.4 src.rpm, or, if using a CVS checkout, do the following: * -# * * -# * cvs -d :pserver:anonymous@cvs.rpm.org:/cvs/devel login * -# * (no password, just carriage return) * -# * cvs -d :pserver:anonymous@cvs.rpm.org:/cvs/devel get rpm * -# * cd rpm * -# * cvs up -r rpm-4_0 * -# * * -# **************************************************************************** -# -# " -# sleep 10 +echo " +**************************************************************************** +* * +* *** WARNING WARNING WARNING *** * +* * +* This is source code from the development branch of rpm-4.2. * +* * +* If you want the "production" rpm-4.0.4 or rpm-4.1 code, then you should * +* either use an rpm-4.0.4 or rpm-4.1 src.rpm. Alternatively, if using a * +* CVS checkout, do the following: * +* * +* cvs -d :pserver:anonymous@cvs.rpm.org:/cvs/devel login * +* (no password, just carriage return) * +* cvs -d :pserver:anonymous@cvs.rpm.org:/cvs/devel get rpm * +* cd rpm * +* * +* Here's the rpm-4_1 branch: * +* cvs up -r rpm-4_1 * +* * +* Here's the rpm-4_0 branch: * +* cvs up -r rpm-4_1 * +* * +**************************************************************************** + +" +sleep 10 dnl Checks for programs. AC_PROG_CC diff --git a/db/env/db_salloc.c b/db/env/db_salloc.c index 4780107..881c63b 100644 --- a/db/env/db_salloc.c +++ b/db/env/db_salloc.c @@ -1,14 +1,14 @@ /*- * See the file LICENSE for redistribution information. * - * Copyright (c) 1996, 1997, 1998, 1999, 2000 + * Copyright (c) 1996-2002 * Sleepycat Software. All rights reserved. */ #include "db_config.h" #ifndef lint -static const char revid[] = "$Id: db_salloc.c,v 11.10 2000/12/06 19:55:44 ubell Exp $"; +static const char revid[] = "Id: db_salloc.c,v 11.15 2002/02/22 01:55:53 mjc Exp "; #endif /* not lint */ #ifndef NO_SYSTEM_INCLUDES @@ -59,8 +59,8 @@ __db_shalloc_init(area, size) } /* - * __db_shalloc -- - * Allocate some space from the shared region. + * __db_shalloc_size -- + * Return size of the shared region, including alignment. * * PUBLIC: int __db_shalloc_size __P((size_t, size_t)); */ @@ -81,7 +81,7 @@ __db_shalloc_size(len, align) if (align <= sizeof(db_align_t)) align = sizeof(db_align_t); - return (ALIGN(len, align) + sizeof (struct __data)); + return ((int)(ALIGN(len, align) + sizeof (struct __data))); } /* @@ -284,28 +284,6 @@ __db_shalloc_free(regionp, ptr) } /* - * __db_shalloc_count -- - * Return the amount of memory on the free list. - * - * PUBLIC: size_t __db_shalloc_count __P((void *)); - */ -size_t -__db_shalloc_count(addr) - void *addr; -{ - struct __data *elp; - size_t count; - - count = 0; - for (elp = SH_LIST_FIRST((struct __head *)addr, __data); - elp != NULL; - elp = SH_LIST_NEXT(elp, links, __data)) - count += elp->len; - - return (count); -} - -/* * __db_shsizeof -- * Return the size of a shalloc'd piece of memory. * @@ -355,6 +333,6 @@ __db_shalloc_dump(addr, fp) for (elp = SH_LIST_FIRST((struct __head *)addr, __data); elp != NULL; elp = SH_LIST_NEXT(elp, links, __data)) - fprintf(fp, "%#lx: %lu\t", (u_long)elp, (u_long)elp->len); + fprintf(fp, "%#lx: %lu\t", P_TO_ULONG(elp), (u_long)elp->len); fprintf(fp, "\n"); } diff --git a/db/mp/mp_sync.c b/db/mp/mp_sync.c index 1b0751d..8c176a3 100644 --- a/db/mp/mp_sync.c +++ b/db/mp/mp_sync.c @@ -1,13 +1,13 @@ /*- * See the file LICENSE for redistribution information. * - * Copyright (c) 1996, 1997, 1998, 1999, 2000 + * Copyright (c) 1996-2002 * Sleepycat Software. All rights reserved. */ #include "db_config.h" #ifndef lint -static const char revid[] = "$Id: mp_sync.c,v 11.29 2001/01/11 18:19:53 bostic Exp $"; +static const char revid[] = "Id: mp_sync.c,v 11.63 2002/08/13 18:30:07 bostic Exp "; #endif /* not lint */ #ifndef NO_SYSTEM_INCLUDES @@ -16,339 +16,92 @@ static const char revid[] = "$Id: mp_sync.c,v 11.29 2001/01/11 18:19:53 bostic E #include #endif -#ifdef HAVE_RPC -#include "db_server.h" -#endif - #include "db_int.h" -#include "db_shash.h" -#include "mp.h" +#include "dbinc/db_shash.h" +#include "dbinc/mp.h" -#ifdef HAVE_RPC -#include "gen_client_ext.h" -#include "rpc_client_ext.h" -#endif +typedef struct { + DB_MPOOL_HASH *track_hp; /* Hash bucket. */ + + roff_t track_off; /* Page file offset. */ + db_pgno_t track_pgno; /* Page number. */ +} BH_TRACK; static int __bhcmp __P((const void *, const void *)); -static int __memp_fsync __P((DB_MPOOLFILE *)); -static int __memp_sballoc __P((DB_ENV *, BH ***, u_int32_t *)); +static int __memp_close_flush_files __P((DB_ENV *, DB_MPOOL *)); +static int __memp_sync_files __P((DB_ENV *, DB_MPOOL *)); /* - * memp_sync -- + * __memp_sync -- * Mpool sync function. + * + * PUBLIC: int __memp_sync __P((DB_ENV *, DB_LSN *)); */ int -memp_sync(dbenv, lsnp) +__memp_sync(dbenv, lsnp) DB_ENV *dbenv; DB_LSN *lsnp; { - BH *bhp, **bharray; DB_MPOOL *dbmp; - DB_LSN tlsn; - MPOOL *c_mp, *mp; - MPOOLFILE *mfp; - u_int32_t ar_cnt, i, ndirty; - int ret, retry_done, retry_need, wrote; - -#ifdef HAVE_RPC - if (F_ISSET(dbenv, DB_ENV_RPCCLIENT)) - return (__dbcl_memp_sync(dbenv, lsnp)); -#endif + MPOOL *mp; + int ret; PANIC_CHECK(dbenv); - ENV_REQUIRES_CONFIG(dbenv, dbenv->mp_handle, DB_INIT_MPOOL); - - dbmp = dbenv->mp_handle; - mp = dbmp->reginfo[0].primary; + ENV_REQUIRES_CONFIG(dbenv, + dbenv->mp_handle, "memp_sync", DB_INIT_MPOOL); /* - * If no LSN is provided, flush the entire cache. - * - * !!! - * Our current behavior is to flush the entire cache, so there's - * nothing special we have to do here other than deal with NULL - * pointers. + * If no LSN is provided, flush the entire cache (reasonable usage + * even if there's no log subsystem configured). */ - if (lsnp == NULL) { - ZERO_LSN(tlsn); - lsnp = &tlsn; - F_SET(mp, MP_LSN_RETRY); - } else if (!LOGGING_ON(dbenv)) { - __db_err(dbenv, "memp_sync: requires logging"); - return (EINVAL); - } + if (lsnp != NULL) + ENV_REQUIRES_CONFIG(dbenv, + dbenv->lg_handle, "memp_sync", DB_INIT_LOG); - /* - * Sync calls are single-threaded so that we don't have multiple - * threads, with different checkpoint LSNs, walking the caches - * and updating the checkpoint LSNs and how many buffers remain - * to be written for the checkpoint. This shouldn't be a problem, - * any application that has multiple checkpoint threads isn't what - * I'd call trustworthy. - */ - MUTEX_LOCK(dbenv, &mp->sync_mutex, dbenv->lockfhp); + dbmp = dbenv->mp_handle; + mp = dbmp->reginfo[0].primary; - /* - * If the application is asking about a previous call to memp_sync(), - * and we haven't found any buffers that the application holding the - * pin couldn't write, return yes or no based on the current count. - * Note, if the application is asking about a LSN *smaller* than one - * we've already handled or are currently handling, then we return a - * result based on the count for the larger LSN. - */ - R_LOCK(dbenv, dbmp->reginfo); - if (!IS_ZERO_LSN(*lsnp) && - !F_ISSET(mp, MP_LSN_RETRY) && log_compare(lsnp, &mp->lsn) <= 0) { - if (mp->lsn_cnt == 0) { + /* If we've flushed to the requested LSN, return that information. */ + if (lsnp != NULL) { + R_LOCK(dbenv, dbmp->reginfo); + if (log_compare(lsnp, &mp->lsn) <= 0) { *lsnp = mp->lsn; - ret = 0; - } else - ret = DB_INCOMPLETE; + R_UNLOCK(dbenv, dbmp->reginfo); + return (0); + } R_UNLOCK(dbenv, dbmp->reginfo); - MUTEX_UNLOCK(dbenv, &mp->sync_mutex); - return (ret); } - /* - * Allocate room for a list of buffers, and decide how many buffers - * we can pin down. - * - * !!! - * Note: __memp_sballoc has released the region lock if we're not - * continuing forward. - */ - if ((ret = - __memp_sballoc(dbenv, &bharray, &ndirty)) != 0 || ndirty == 0) { - MUTEX_UNLOCK(dbenv, &mp->sync_mutex); + if ((ret = __memp_sync_int(dbenv, NULL, 0, DB_SYNC_CACHE, NULL)) != 0) return (ret); - } - retry_done = 0; -retry: retry_need = 0; - /* - * Start a new checkpoint. - * - * Save the LSN. We know that it's a new LSN, a retry, or larger than - * the one for which we were already doing a checkpoint. (BTW, I don't - * expect to see multiple LSN's from the same or multiple processes, - * but You Just Never Know. Responding as if they all called with the - * largest of the LSNs specified makes everything work.) - * - * We don't currently use the LSN we save. We could potentially save - * the last-written LSN in each buffer header and use it to determine - * what buffers need to be written. The problem with this is that it's - * sizeof(LSN) more bytes of buffer header. We currently write all the - * dirty buffers instead, but with a sufficiently large cache that's - * going to be a problem. - */ - mp->lsn = *lsnp; - - /* - * Clear the global count of buffers waiting to be written, walk the - * list of files clearing the count of buffers waiting to be written. - * - * Clear the retry flag. - */ - mp->lsn_cnt = 0; - for (mfp = SH_TAILQ_FIRST(&mp->mpfq, __mpoolfile); - mfp != NULL; mfp = SH_TAILQ_NEXT(mfp, q, __mpoolfile)) - mfp->lsn_cnt = 0; - F_CLR(mp, MP_LSN_RETRY); - - /* - * Walk each cache's list of buffers and mark all dirty buffers to be - * written and all pinned buffers to be potentially written (we can't - * know if they'll need to be written until the holder returns them to - * the cache). We do this in one pass while holding the region locked - * so that processes can't make new buffers dirty, causing us to never - * finish. Since the application may have restarted the sync using a - * different LSN value, clear any BH_SYNC | BH_SYNC_LOGFLSH flags that - * appear leftover from previous calls. - * - * Keep a count of the total number of buffers we need to write in - * MPOOL->lsn_cnt, and for each file, in MPOOLFILE->lsn_count. - */ - for (ar_cnt = 0, i = 0; i < mp->nreg; ++i) { - c_mp = dbmp->reginfo[i].primary; - for (bhp = SH_TAILQ_FIRST(&c_mp->bhq, __bh); - bhp != NULL; bhp = SH_TAILQ_NEXT(bhp, q, __bh)) { - if (F_ISSET(bhp, BH_DIRTY) || bhp->ref != 0) { - F_SET(bhp, BH_SYNC); - - ++mp->lsn_cnt; - - mfp = R_ADDR(dbmp->reginfo, bhp->mf_offset); - ++mfp->lsn_cnt; - - /* - * If the buffer isn't being used, we can write - * it immediately, so increment its reference - * count to lock it down, and save a reference - * to it. - * - * If we've run out space to store buffer refs, - * we're screwed. We don't want to realloc the - * array while holding a region lock, so we set - * a flag and deal with it later. - */ - if (bhp->ref == 0) { - ++bhp->ref; - bharray[ar_cnt] = bhp; - - if (++ar_cnt >= ndirty) { - retry_need = 1; - break; - } - } - } else - if (F_ISSET(bhp, BH_SYNC)) - F_CLR(bhp, BH_SYNC | BH_SYNC_LOGFLSH); - } - if (ar_cnt >= ndirty) - break; - } - - /* If there no buffers we can write immediately, we're done. */ - if (ar_cnt == 0) { - ret = mp->lsn_cnt ? DB_INCOMPLETE : 0; - goto done; - } - - R_UNLOCK(dbenv, dbmp->reginfo); - - /* - * Sort the buffers we're going to write immediately. - * - * We try and write the buffers in file/page order: it should reduce - * seeks by the underlying filesystem and possibly reduce the actual - * number of writes. - */ - if (ar_cnt > 1) - qsort(bharray, ar_cnt, sizeof(BH *), __bhcmp); - - /* - * Flush the log. We have to ensure the log records reflecting the - * changes on the database pages we're writing have already made it - * to disk. We usually do that as we write each page, but if we - * are going to write a large number of pages, repeatedly acquiring - * the log region lock is going to be expensive. Flush the entire - * log now, so that sync doesn't require any more log flushes. - */ - if (LOGGING_ON(dbenv) && (ret = log_flush(dbenv, NULL)) != 0) - goto done; - - R_LOCK(dbenv, dbmp->reginfo); - - /* Walk the array, writing buffers. */ - for (i = 0; i < ar_cnt; ++i) { - /* - * It's possible for a thread to have gotten the buffer since - * we listed it for writing. If the reference count is still - * 1, we're the only ones using the buffer, go ahead and write. - * If it's >1, then skip the buffer and assume that it will be - * written when it's returned to the cache. - */ - if (bharray[i]->ref > 1) { - --bharray[i]->ref; - continue; - } - - /* Write the buffer. */ - mfp = R_ADDR(dbmp->reginfo, bharray[i]->mf_offset); - ret = __memp_bhwrite(dbmp, mfp, bharray[i], NULL, &wrote); - - /* Release the buffer. */ - --bharray[i]->ref; - - if (ret == 0 && wrote) - continue; - - /* - * Any process syncing the shared memory buffer pool had best - * be able to write to any underlying file. Be understanding, - * but firm, on this point. - */ - if (ret == 0) { - __db_err(dbenv, "%s: unable to flush page: %lu", - __memp_fns(dbmp, mfp), (u_long)bharray[i]->pgno); - ret = EPERM; - } - - /* - * On error, clear MPOOL->lsn and set MP_LSN_RETRY so that no - * future checkpoint return can depend on this failure. Clear - * the buffer's BH_SYNC flag, because it's used to determine - * if lsn_cnt values are incremented/decremented. Don't bother - * to reset/clear: - * - * MPOOL->lsn_cnt - * MPOOLFILE->lsn_cnt - * - * they don't make any difference. - */ - ZERO_LSN(mp->lsn); - F_SET(mp, MP_LSN_RETRY); - - /* Release any buffers we're still pinning down. */ - while (++i < ar_cnt) { - bhp = bharray[i]; - --bhp->ref; - F_CLR(bhp, BH_SYNC | BH_SYNC_LOGFLSH); - } - - goto done; - } - - ret = mp->lsn_cnt != 0 ? DB_INCOMPLETE : 0; - - /* - * If there were too many buffers and we're not returning an error, we - * re-try the checkpoint once -- since we allocated 80% of the total - * buffer count, once should be enough. If it still doesn't work, some - * other thread of control is dirtying buffers as fast as we're writing - * them, and we might as well give up for now. In the latter case, set - * the global retry flag, we'll have to start from scratch on the next - * checkpoint. - */ - if (retry_need) { - if (retry_done) { - ret = DB_INCOMPLETE; - F_SET(mp, MP_LSN_RETRY); - } else { - retry_done = 1; - goto retry; - } + if (lsnp != NULL) { + R_LOCK(dbenv, dbmp->reginfo); + if (log_compare(lsnp, &mp->lsn) > 0) + mp->lsn = *lsnp; + R_UNLOCK(dbenv, dbmp->reginfo); } -done: R_UNLOCK(dbenv, dbmp->reginfo); - MUTEX_UNLOCK(dbenv, &mp->sync_mutex); - - __os_free(bharray, ndirty * sizeof(BH *)); - - return (ret); + return (0); } /* - * memp_fsync -- + * __memp_fsync -- * Mpool file sync function. + * + * PUBLIC: int __memp_fsync __P((DB_MPOOLFILE *)); */ int -memp_fsync(dbmfp) +__memp_fsync(dbmfp) DB_MPOOLFILE *dbmfp; { DB_ENV *dbenv; DB_MPOOL *dbmp; - int is_tmp; dbmp = dbmfp->dbmp; dbenv = dbmp->dbenv; -#ifdef HAVE_RPC - if (F_ISSET(dbenv, DB_ENV_RPCCLIENT)) - return (__dbcl_memp_fsync(dbmfp)); -#endif - PANIC_CHECK(dbenv); /* @@ -359,13 +112,10 @@ memp_fsync(dbmfp) if (F_ISSET(dbmfp, MP_READONLY)) return (0); - R_LOCK(dbenv, dbmp->reginfo); - is_tmp = F_ISSET(dbmfp->mfp, MP_TEMP); - R_UNLOCK(dbenv, dbmp->reginfo); - if (is_tmp) + if (F_ISSET(dbmfp->mfp, MP_TEMP)) return (0); - return (__memp_fsync(dbmfp)); + return (__memp_sync_int(dbenv, dbmfp, 0, DB_SYNC_FILE, NULL)); } /* @@ -379,6 +129,7 @@ __mp_xxx_fh(dbmfp, fhp) DB_MPOOLFILE *dbmfp; DB_FH **fhp; { + DB_ENV *dbenv; /* * This is a truly spectacular layering violation, intended ONLY to * support compatibility for the DB 1.85 DB->fd call. @@ -393,239 +144,457 @@ __mp_xxx_fh(dbmfp, fhp) * because we want to write to the backing file regardless so that * we get a file descriptor to return. */ - *fhp = &dbmfp->fh; - return (F_ISSET(&dbmfp->fh, DB_FH_VALID) ? 0 : __memp_fsync(dbmfp)); + *fhp = dbmfp->fhp; + if (F_ISSET(dbmfp->fhp, DB_FH_VALID)) + return (0); + dbenv = dbmfp->dbmp->dbenv; + + return (__memp_sync_int(dbenv, dbmfp, 0, DB_SYNC_FILE, NULL)); } /* - * __memp_fsync -- - * Mpool file internal sync function. + * __memp_sync_int -- + * Mpool sync internal function. + * + * PUBLIC: int __memp_sync_int + * PUBLIC: __P((DB_ENV *, DB_MPOOLFILE *, int, db_sync_op, int *)); */ -static int -__memp_fsync(dbmfp) +int +__memp_sync_int(dbenv, dbmfp, ar_max, op, wrotep) + DB_ENV *dbenv; DB_MPOOLFILE *dbmfp; + int ar_max, *wrotep; + db_sync_op op; { - BH *bhp, **bharray; - DB_ENV *dbenv; + BH *bhp; + BH_TRACK *bharray; DB_MPOOL *dbmp; + DB_MPOOL_HASH *hp; + DB_MUTEX *mutexp; MPOOL *c_mp, *mp; - size_t mf_offset; - u_int32_t ar_cnt, i, ndirty; - int incomplete, ret, retry_done, retry_need, wrote; + MPOOLFILE *mfp; + u_int32_t n_cache; + int ar_cnt, hb_lock, i, pass, remaining, ret, t_ret, wait_cnt, wrote; - dbmp = dbmfp->dbmp; - dbenv = dbmp->dbenv; + dbmp = dbenv->mp_handle; mp = dbmp->reginfo[0].primary; - - R_LOCK(dbenv, dbmp->reginfo); + pass = wrote = 0; /* - * Allocate room for a list of buffers, and decide how many buffers - * we can pin down. - * - * !!! - * Note: __memp_sballoc has released our region lock if we're not - * continuing forward. + * If the caller does not specify how many pages assume one + * per bucket. */ + if (ar_max == 0) + ar_max = mp->nreg * mp->htab_buckets; + if ((ret = - __memp_sballoc(dbenv, &bharray, &ndirty)) != 0 || ndirty == 0) + __os_malloc(dbenv, ar_max * sizeof(BH_TRACK), &bharray)) != 0) return (ret); - retry_done = 0; -retry: retry_need = 0; /* * Walk each cache's list of buffers and mark all dirty buffers to be - * written and all pinned buffers to be potentially written (we can't - * know if they'll need to be written until the holder returns them to - * the cache). We do this in one pass while holding the region locked - * so that processes can't make new buffers dirty, causing us to never - * finish. + * written and all pinned buffers to be potentially written, depending + * on our flags. */ - mf_offset = R_OFFSET(dbmp->reginfo, dbmfp->mfp); - for (ar_cnt = 0, incomplete = 0, i = 0; i < mp->nreg; ++i) { - c_mp = dbmp->reginfo[i].primary; - for (bhp = SH_TAILQ_FIRST(&c_mp->bhq, __bh); - bhp != NULL; bhp = SH_TAILQ_NEXT(bhp, q, __bh)) { - if (!F_ISSET(bhp, BH_DIRTY) || - bhp->mf_offset != mf_offset) - continue; - if (bhp->ref != 0 || F_ISSET(bhp, BH_LOCKED)) { - incomplete = 1; - continue; - } + for (ar_cnt = 0, n_cache = 0; n_cache < mp->nreg; ++n_cache) { + c_mp = dbmp->reginfo[n_cache].primary; + hp = R_ADDR(&dbmp->reginfo[n_cache], c_mp->htab); + for (i = 0; i < c_mp->htab_buckets; i++, hp++) { /* - * If the buffer isn't being used, we can write - * it immediately, so increment its reference - * count to lock it down, and save a reference - * to it. - * - * If we've run out space to store buffer refs, - * we're screwed. We don't want to realloc the - * array while holding a region lock, so we set - * a flag and deal with it later. + * We can check for empty buckets before locking as we + * only care if the pointer is zero or non-zero. We + * can ignore empty buckets because we only need write + * buffers that were dirty before we started. */ - ++bhp->ref; - bharray[ar_cnt] = bhp; - if (++ar_cnt >= ndirty) { - retry_need = 1; - break; + if (SH_TAILQ_FIRST(&hp->hash_bucket, __bh) == NULL) + continue; + + MUTEX_LOCK(dbenv, &hp->hash_mutex); + for (bhp = SH_TAILQ_FIRST(&hp->hash_bucket, __bh); + bhp != NULL; bhp = SH_TAILQ_NEXT(bhp, hq, __bh)) { + /* Always ignore unreferenced, clean pages. */ + if (bhp->ref == 0 && !F_ISSET(bhp, BH_DIRTY)) + continue; + + /* + * Checkpoints have to wait on all pinned pages, + * as pages may be marked dirty when returned to + * the cache. + * + * File syncs only wait on pages both pinned and + * dirty. (We don't care if pages are marked + * dirty when returned to the cache, that means + * there's another writing thread and flushing + * the cache for this handle is meaningless.) + */ + if (op == DB_SYNC_FILE && + !F_ISSET(bhp, BH_DIRTY)) + continue; + + mfp = R_ADDR(dbmp->reginfo, bhp->mf_offset); + + /* + * Ignore temporary files -- this means you + * can't even flush temporary files by handle. + * (Checkpoint doesn't require temporary files + * be flushed and the underlying buffer write + * write routine may not be able to write it + * anyway.) + */ + if (F_ISSET(mfp, MP_TEMP)) + continue; + + /* + * If we're flushing a specific file, see if + * this page is from that file. + */ + if (dbmfp != NULL && mfp != dbmfp->mfp) + continue; + + /* + * Ignore files that aren't involved in DB's + * transactional operations during checkpoints. + */ + if (dbmfp == NULL && mfp->lsn_off == -1) + continue; + + /* Track the buffer, we want it. */ + bharray[ar_cnt].track_hp = hp; + bharray[ar_cnt].track_pgno = bhp->pgno; + bharray[ar_cnt].track_off = bhp->mf_offset; + ar_cnt++; + + if (ar_cnt >= ar_max) { + if ((ret = __os_realloc(dbenv, + (ar_max * 2) * sizeof(BH_TRACK), + &bharray)) != 0) + break; + ar_max *= 2; + } } + MUTEX_UNLOCK(dbenv, &hp->hash_mutex); + + if (ret != 0) + goto err; } - if (ar_cnt >= ndirty) - break; } - /* If there no buffers we can write immediately, we're done. */ - if (ar_cnt == 0) { - ret = 0; + /* If there no buffers to write, we're done. */ + if (ar_cnt == 0) goto done; - } - R_UNLOCK(dbenv, dbmp->reginfo); - - /* Sort the buffers we're going to write. */ + /* + * Write the buffers in file/page order, trying to reduce seeks by the + * filesystem and, when pages are smaller than filesystem block sizes, + * reduce the actual number of writes. + */ if (ar_cnt > 1) - qsort(bharray, ar_cnt, sizeof(BH *), __bhcmp); + qsort(bharray, ar_cnt, sizeof(BH_TRACK), __bhcmp); - R_LOCK(dbenv, dbmp->reginfo); + /* + * If we're trickling buffers, only write enough to reach the correct + * percentage for this region. We may not write enough if the dirty + * buffers have an unbalanced distribution among the regions, but that + * seems unlikely. + */ + if (op == DB_SYNC_TRICKLE && ar_cnt > ar_max / (int)mp->nreg) + ar_cnt = ar_max / (int)mp->nreg; + + /* + * Flush the log. We have to ensure the log records reflecting the + * changes on the database pages we're writing have already made it + * to disk. We still have to check the log each time we write a page + * (because pages we are about to write may be modified after we have + * flushed the log), but in general this will at least avoid any I/O + * on the log's part. + */ + if (LOGGING_ON(dbenv) && (ret = dbenv->log_flush(dbenv, NULL)) != 0) + goto err; + + /* + * Walk the array, writing buffers. When we write a buffer, we NULL + * out its hash bucket pointer so we don't process a slot more than + * once. + */ + for (remaining = ar_cnt, i = pass = 0; remaining > 0; ++i) { + if (i >= ar_cnt) { + i = 0; + ++pass; + __os_sleep(dbenv, 1, 0); + } + if ((hp = bharray[i].track_hp) == NULL) + continue; + + /* Lock the hash bucket and find the buffer. */ + mutexp = &hp->hash_mutex; + MUTEX_LOCK(dbenv, mutexp); + for (bhp = SH_TAILQ_FIRST(&hp->hash_bucket, __bh); + bhp != NULL; bhp = SH_TAILQ_NEXT(bhp, hq, __bh)) + if (bhp->pgno == bharray[i].track_pgno && + bhp->mf_offset == bharray[i].track_off) + break; - /* Walk the array, writing buffers. */ - for (i = 0; i < ar_cnt;) { /* - * It's possible for a thread to have gotten the buffer since - * we listed it for writing. If the reference count is still - * 1, we're the only ones using the buffer, go ahead and write. - * If it's >1, then skip the buffer and assume that it will be - * written when it's returned to the cache. + * If we can't find the buffer we're done, somebody else had + * to have written it. + * + * If the buffer isn't pinned or dirty, we're done, there's + * no work needed. */ - if (bharray[i]->ref > 1) { - incomplete = 1; - --bharray[i++]->ref; + if (bhp == NULL || (bhp->ref == 0 && !F_ISSET(bhp, BH_DIRTY))) { + MUTEX_UNLOCK(dbenv, mutexp); + --remaining; + bharray[i].track_hp = NULL; continue; } - /* Write the buffer. */ - ret = __memp_pgwrite(dbmp, dbmfp, bharray[i], NULL, &wrote); + /* + * If the buffer is locked by another thread, ignore it, we'll + * come back to it. + * + * If the buffer is pinned and it's only the first or second + * time we have looked at it, ignore it, we'll come back to + * it. + * + * In either case, skip the buffer if we're not required to + * write it. + */ + if (F_ISSET(bhp, BH_LOCKED) || (bhp->ref != 0 && pass < 2)) { + MUTEX_UNLOCK(dbenv, mutexp); + if (op != DB_SYNC_CACHE && op != DB_SYNC_FILE) { + --remaining; + bharray[i].track_hp = NULL; + } + continue; + } + + /* + * The buffer is either pinned or dirty. + * + * Set the sync wait-for count, used to count down outstanding + * references to this buffer as they are returned to the cache. + */ + bhp->ref_sync = bhp->ref; - /* Release the buffer. */ - --bharray[i++]->ref; + /* Pin the buffer into memory and lock it. */ + ++bhp->ref; + F_SET(bhp, BH_LOCKED); + MUTEX_LOCK(dbenv, &bhp->mutex); - if (ret == 0) { - if (!wrote) - incomplete = 1; - continue; + /* + * Unlock the hash bucket and wait for the wait-for count to + * go to 0. No new thread can acquire the buffer because we + * have it locked. + * + * If a thread attempts to re-pin a page, the wait-for count + * will never go to 0 (the thread spins on our buffer lock, + * while we spin on the thread's ref count). Give up if we + * don't get the buffer in 3 seconds, we can try again later. + * + * If, when the wait-for count goes to 0, the buffer is found + * to be dirty, write it. + */ + MUTEX_UNLOCK(dbenv, mutexp); + for (wait_cnt = 1; + bhp->ref_sync != 0 && wait_cnt < 4; ++wait_cnt) + __os_sleep(dbenv, 1, 0); + MUTEX_LOCK(dbenv, mutexp); + hb_lock = 1; + + /* + * If the ref_sync count has gone to 0, we're going to be done + * with this buffer no matter what happens. + */ + if (bhp->ref_sync == 0) { + --remaining; + bharray[i].track_hp = NULL; } /* - * On error: + * If the ref_sync count has gone to 0 and the buffer is still + * dirty, we write it. We only try to write the buffer once. + * Any process checkpointing or trickle-flushing the pool + * must be able to write any underlying file -- if the write + * fails, error out. It would be very strange if file sync + * failed to write, but we don't care if it happens. + */ + if (bhp->ref_sync == 0 && F_ISSET(bhp, BH_DIRTY)) { + hb_lock = 0; + MUTEX_UNLOCK(dbenv, mutexp); + + mfp = R_ADDR(dbmp->reginfo, bhp->mf_offset); + if ((ret = __memp_bhwrite(dbmp, hp, mfp, bhp, 1)) == 0) + ++wrote; + else if (op == DB_SYNC_CACHE || op == DB_SYNC_TRICKLE) + __db_err(dbenv, "%s: unable to flush page: %lu", + __memp_fns(dbmp, mfp), (u_long)bhp->pgno); + else + ret = 0; + } + + /* + * If ref_sync count never went to 0, the buffer was written + * by another thread, or the write failed, we still have the + * buffer locked. + * + * We may or may not currently hold the hash bucket mutex. If + * the __memp_bhwrite -> __memp_pgwrite call was successful, + * then __memp_pgwrite will have swapped the buffer lock for + * the hash lock. All other call paths will leave us without + * the hash bucket lock. * - * Release any buffers we're still pinning down. + * The order of mutexes above was to acquire the buffer lock + * while holding the hash bucket lock. Don't deadlock here, + * release the buffer lock and then acquire the hash bucket + * lock. */ - while (i < ar_cnt) - --bharray[i++]->ref; - break; - } + if (F_ISSET(bhp, BH_LOCKED)) { + F_CLR(bhp, BH_LOCKED); + MUTEX_UNLOCK(dbenv, &bhp->mutex); - /* - * If there were too many buffers and we're not returning an error, we - * re-try the flush once -- since we allocated 80% of the total - * buffer count, once should be enough. If it still doesn't work, some - * other thread of control is dirtying buffers as fast as we're writing - * them, and we might as well give up. - */ - if (retry_need) { - if (retry_done) - incomplete = 1; - else { - retry_done = 1; - goto retry; + if (!hb_lock) + MUTEX_LOCK(dbenv, mutexp); } - } -done: R_UNLOCK(dbenv, dbmp->reginfo); + /* + * Reset the ref_sync count regardless of our success, we're + * done with this buffer for now. + */ + bhp->ref_sync = 0; + + /* Discard our reference and unlock the bucket. */ + --bhp->ref; + MUTEX_UNLOCK(dbenv, mutexp); - __os_free(bharray, ndirty * sizeof(BH *)); + if (ret != 0) + break; + } + +done: /* If we've opened files to flush pages, close them. */ + if ((t_ret = __memp_close_flush_files(dbenv, dbmp)) != 0 && ret == 0) + ret = t_ret; /* - * Sync the underlying file as the last thing we do, so that the OS - * has a maximal opportunity to flush buffers before we request it. - * - * !!!: - * Don't lock the region around the sync, fsync(2) has no atomicity - * issues. + * If doing a checkpoint or flushing a file for the application, we + * have to force the pages to disk. We don't do this as we go along + * because we want to give the OS as much time as possible to lazily + * flush, and because we have to flush files that might not even have + * had dirty buffers in the cache, so we have to walk the files list. */ - if (ret == 0) - ret = incomplete ? - DB_INCOMPLETE : __os_fsync(dbenv, &dbmfp->fh); + if (ret == 0 && (op == DB_SYNC_CACHE || op == DB_SYNC_FILE)) { + if (dbmfp == NULL) + ret = __memp_sync_files(dbenv, dbmp); + else + ret = __os_fsync(dbenv, dbmfp->fhp); + } + +err: __os_free(dbenv, bharray); + if (wrotep != NULL) + *wrotep = wrote; return (ret); } /* - * __memp_sballoc -- - * Allocate room for a list of buffers. + * __memp_sync_files -- + * Sync all the files in the environment, open or not. */ -static int -__memp_sballoc(dbenv, bharrayp, ndirtyp) +static +int __memp_sync_files(dbenv, dbmp) DB_ENV *dbenv; - BH ***bharrayp; - u_int32_t *ndirtyp; -{ DB_MPOOL *dbmp; - MPOOL *c_mp, *mp; - u_int32_t i, nclean, ndirty, maxpin; - int ret; +{ + DB_MPOOLFILE *dbmfp; + MPOOL *mp; + MPOOLFILE *mfp; + int ret, t_ret; - dbmp = dbenv->mp_handle; + ret = 0; mp = dbmp->reginfo[0].primary; - /* - * We don't want to hold the region lock while we write the buffers, - * so only lock it while we create a list. - * - * Walk through the list of caches, figuring out how many buffers - * we're going to need. - * - * Make a point of not holding the region lock across the library - * allocation call. - */ - for (nclean = ndirty = 0, i = 0; i < mp->nreg; ++i) { - c_mp = dbmp->reginfo[i].primary; - ndirty += c_mp->stat.st_page_dirty; - nclean += c_mp->stat.st_page_clean; + R_LOCK(dbenv, dbmp->reginfo); + for (mfp = SH_TAILQ_FIRST(&mp->mpfq, __mpoolfile); + mfp != NULL; mfp = SH_TAILQ_NEXT(mfp, q, __mpoolfile)) { + if (mfp->stat.st_page_out == 0 || + F_ISSET(mfp, MP_DEADFILE | MP_TEMP)) + continue; + + /* Look for an already open handle. */ + ret = 0; + MUTEX_THREAD_LOCK(dbenv, dbmp->mutexp); + for (dbmfp = TAILQ_FIRST(&dbmp->dbmfq); + dbmfp != NULL; dbmfp = TAILQ_NEXT(dbmfp, q)) + if (dbmfp->mfp == mfp) { + ret = __os_fsync(dbenv, dbmfp->fhp); + break; + } + MUTEX_THREAD_UNLOCK(dbenv, dbmp->mutexp); + if (ret != 0) + goto err; + + /* If we don't find one, open one. */ + if (dbmfp == NULL) { + if ((ret = dbenv->memp_fcreate(dbenv, &dbmfp, 0)) != 0) + goto err; + ret = __memp_fopen_int( + dbmfp, mfp, R_ADDR(dbmp->reginfo, mfp->path_off), + 0, 0, mfp->stat.st_pagesize); + if (ret == 0) + ret = __os_fsync(dbenv, dbmfp->fhp); + if ((t_ret = + __memp_fclose_int(dbmfp, 0)) != 0 && ret == 0) + ret = t_ret; + if (ret != 0) + goto err; + } } - R_UNLOCK(dbenv, dbmp->reginfo); - if (ndirty == 0) { - *ndirtyp = 0; - return (0); + + if (0) { +err: __db_err(dbenv, "%s: cannot sync: %s", + R_ADDR(dbmp->reginfo, mfp->path_off), db_strerror(ret)); } + R_UNLOCK(dbenv, dbmp->reginfo); - /* - * We don't want to pin down the entire buffer cache, otherwise we'll - * starve threads needing new pages. Don't pin down more than 80% of - * the cache, making sure that we don't screw up just because only a - * few pages have been created. - */ - maxpin = ((ndirty + nclean) * 8) / 10; - if (maxpin < 10) - maxpin = 10; + return (ret); +} + +/* + * __memp_close_flush_files -- + * Close files opened only to flush buffers. + */ +static int +__memp_close_flush_files(dbenv, dbmp) + DB_ENV *dbenv; + DB_MPOOL *dbmp; +{ + DB_MPOOLFILE *dbmfp; + int ret; /* - * Get a good-sized block of memory to hold buffer pointers, we don't - * want to run out, but correct if we want to allocate more than we - * would be allowed to store, regardless. + * The routine exists because we must close files opened by sync to + * flush buffers. There are two cases: first, extent files have to + * be closed so they may be removed when empty. Second, regular + * files have to be closed so we don't run out of descriptors (for + * example, and application partitioning its data into databases + * based on timestamps, so there's a continually increasing set of + * files). + * + * We mark files opened in the __memp_bhwrite() function with the + * MP_FLUSH flag. Here we walk through our file descriptor list, + * and, if a file was opened by __memp_bhwrite(), we close it. */ - ndirty += ndirty / 2 + 10; - if (ndirty > maxpin) - ndirty = maxpin; - if ((ret = - __os_malloc(dbenv, ndirty * sizeof(BH *), NULL, bharrayp)) != 0) - return (ret); - - *ndirtyp = ndirty; - - R_LOCK(dbenv, dbmp->reginfo); +retry: MUTEX_THREAD_LOCK(dbenv, dbmp->mutexp); + for (dbmfp = TAILQ_FIRST(&dbmp->dbmfq); + dbmfp != NULL; dbmfp = TAILQ_NEXT(dbmfp, q)) + if (F_ISSET(dbmfp, MP_FLUSH)) { + F_CLR(dbmfp, MP_FLUSH); + MUTEX_THREAD_UNLOCK(dbenv, dbmp->mutexp); + if ((ret = __memp_fclose_int(dbmfp, 0)) != 0) + return (ret); + goto retry; + } + MUTEX_THREAD_UNLOCK(dbenv, dbmp->mutexp); return (0); } @@ -634,15 +603,15 @@ static int __bhcmp(p1, p2) const void *p1, *p2; { - BH *bhp1, *bhp2; + BH_TRACK *bhp1, *bhp2; - bhp1 = *(BH * const *)p1; - bhp2 = *(BH * const *)p2; + bhp1 = (BH_TRACK *)p1; + bhp2 = (BH_TRACK *)p2; /* Sort by file (shared memory pool offset). */ - if (bhp1->mf_offset < bhp2->mf_offset) + if (bhp1->track_off < bhp2->track_off) return (-1); - if (bhp1->mf_offset > bhp2->mf_offset) + if (bhp1->track_off > bhp2->track_off) return (1); /* @@ -650,9 +619,9 @@ __bhcmp(p1, p2) * Defend against badly written quicksort code calling the comparison * function with two identical pointers (e.g., WATCOM C++ (Power++)). */ - if (bhp1->pgno < bhp2->pgno) + if (bhp1->track_pgno < bhp2->track_pgno) return (-1); - if (bhp1->pgno > bhp2->pgno) + if (bhp1->track_pgno > bhp2->track_pgno) return (1); return (0); } diff --git a/macros.in b/macros.in index 5c07fca..565c40a 100644 --- a/macros.in +++ b/macros.in @@ -1,7 +1,7 @@ #/*! \page config_macros Default configuration: @RPMCONFIGDIR@/macros # \verbatim # -# $Id: macros.in,v 1.121 2002/08/22 19:21:14 jbj Exp $ +# $Id: macros.in,v 1.122 2002/08/26 18:56:12 jbj Exp $ # # This is a global RPM configuration file. All changes made here will # be lost when the rpm package is upgraded. Any per-system configuration @@ -366,8 +366,8 @@ package or when debugging this package. \ # token works? Berkeley db flag or value #================================================== #---------------------- DBENV->open parameters and tunable values: -# mp_mmapsize=8Mb DBENV->set_mp_mmapsize -# mp_size=512Kb DBENV->set_cachesize +# mmapsize=16Mb DBENV->set_mp_mmapsize +# cachesize=1Mb DBENV->set_cachesize, DB->set_cachesize #---------------------- DBENV->open and DB->open common bits: # create DB_CREATE # thread ??? DB_THREAD (useless w/o posix mutexes on linux) @@ -386,7 +386,6 @@ package or when debugging this package. \ # lockdown ??? DB_LOCKDOWN # shared +++ DB_SYSTEM_MEM #---------------------- DB->open parameters and tunable values: -# cachesize=512Kb +++ DB->set_cachesize (meaningless if mp_size is used) # pagesize=512 +++ DB->set_pagesize #---------------------- DB->open bits: # excl ??? DB_EXCL @@ -418,7 +417,6 @@ package or when debugging this package. \ # Use a CDB database model for concurrent access. %__dbi_cdb create cdb mpool mp_mmapsize=16Mb mp_size=1Mb -# XXX The "traditional" rpmdb shared/exclusive fcntl(2) lock on Packages model: %__dbi_other %{?_tmppath:tmpdir=%{_tmppath}} %{?__dbi_cdb} # Note: adding nofsync here speeds up --rebuilddb a lot. diff --git a/popt/configure.in b/popt/configure.ac similarity index 98% rename from popt/configure.in rename to popt/configure.ac index ce0c80d..7b20ce6 100755 --- a/popt/configure.in +++ b/popt/configure.ac @@ -2,7 +2,7 @@ AC_INIT(popt.h) AC_CANONICAL_SYSTEM AC_PREREQ(2.12) AC_CONFIG_HEADERS -AM_INIT_AUTOMAKE(popt, 1.7) +AM_INIT_AUTOMAKE(popt, 1.8) AM_CONFIG_HEADER(config.h) ALL_LINGUAS="cs da de es eu_ES fi fr gl hu id is it ja ko no pl pt pt_BR ro ru sk sl sr sv tr uk wa zh zh_CN.GB2312" diff --git a/rpm.spec.in b/rpm.spec.in index 1f2e40d..b53b8a1 100644 --- a/rpm.spec.in +++ b/rpm.spec.in @@ -17,7 +17,7 @@ Name: rpm %define version @VERSION@ Version: %{version} %{expand: %%define rpm_version %{version}} -Release: 0.89 +Release: 0.1 Group: System Environment/Base Source: ftp://ftp.rpm.org/pub/rpm/dist/rpm-4.0.x/rpm-%{rpm_version}.tar.gz Copyright: GPL @@ -25,7 +25,7 @@ Conflicts: patch < 2.5 %ifos linux Prereq: gawk fileutils textutils mktemp shadow-utils %endif -Requires: popt = 1.7 +Requires: popt = 1.8 Obsoletes: rpm-perl < %{version} %if !%{with_internal_db} @@ -62,7 +62,7 @@ the package like its version, a description, etc. %package devel Summary: Development files for manipulating RPM packages. Group: Development/Libraries -Requires: rpm = %{rpm_version}, popt = 1.7 +Requires: rpm = %{rpm_version}, popt = 1.8 %description devel This package contains the RPM C library and header files. These @@ -91,7 +91,7 @@ Summary: Python bindings for apps which will manipulate RPM packages. Group: Development/Libraries Requires: rpm = %{rpm_version} Requires: python >= %{with_python_version} -Requires: popt = 1.7 +Requires: popt = 1.8 %description python The rpm-python package contains a module that permits applications @@ -111,7 +111,7 @@ Provides: perl(RPM::Database) = %{rpm_version} Provides: perl(RPM::Header) = %{rpm_version} Requires: rpm = %{rpm_version} Requires: perl >= 0:5.00503 -Requires: popt = 1.7 +Requires: popt = 1.8 Obsoletes: perl-Perl-RPM %description perl @@ -137,7 +137,7 @@ also available. %package -n popt Summary: A C library for parsing command line parameters. Group: Development/Libraries -Version: 1.7 +Version: 1.8 %description -n popt Popt is a C library for parsing command line parameters. Popt was @@ -522,309 +522,8 @@ fi %{__prefix}/include/popt.h %changelog -* Mon Aug 26 2002 Jeff Johnson 4.1-0.89 -- python: don't remap NOKEY/UNTRUSTED to OK. -- beecrypt: change local absolute symbols to defines for prelinking. -- update ru.po. - -* Fri Aug 22 2002 Jeff Johnson 4.1-0.88 -- merge signature returns into rpmRC. -- python: exceptions on NOKEY/NOTTRUSTED. - -* Thu Aug 21 2002 Jeff Johnson 4.1-0.87 -- fix: don't stop if db1 database is currently in /var/lib/rpm (#72224). -- add a macro to create a sub-package with debugging symbols. - -* Wed Aug 21 2002 Jeff Johnson 4.1-0.86 -- fix: region trailer offset sanity check wrong (#71996). - -* Tue Aug 20 2002 Jeff Johnson 4.1-0.85 -- python: stupid typo broke ts.check(). -- fix: add epoch to "already installed" check. -- check for interrupt during iteration. -- python: add ts.setProbFilter() method, remove ts.run() argument. - -* Mon Aug 19 2002 Jeff Johnson 4.1-0.82 -- add --with-efence to configure, check install/upgrade with efence. -- beecrypt: short hex string conversion overflows target buffer. -- mark "successors only" packages in transaction. -- reap scriptlets with SIGCHLD handler. -- rename PSM_t to rpmpsm, add methods and refcounts to manage. -- remove %%configure/%%makeinstall from arch-os/macros, default is OK. -- don't export MALLOC_CHECK_ to scriptlets. -- squeaky clean memory leak checking. -- always malloc rpmfi structure, refcounts are correct in rpmtsRun(). -- skip redundant /sbin/ldconfig scripts on upgrade (if possible). - -* Thu Aug 15 2002 Jeff Johnson 4.1-0.81 -- rollback: re-create empty transaction set for multiple rollbacks. -- fix: %%basename typo (Dmitry V. Levin). -- fix: queryformat segfaults (Dmitry V. Levin). - -* Wed Aug 14 2002 Jeff Johnson 4.1-0.80 -- fix: rebuilddb stat'ed target, not source, for rename sanity, take 2. -- python: explicit method to set transFlags. -- python: stuff package name into a string for repackage/erase callbacks. - -* Tue Aug 13 2002 Jeff Johnson 4.1-0.79 -- supply transitive closure for CLI packages from rpmdb-redhat database. -- fix: rebuilddb stat'ed target, not source, for rename sanity. -- create /var/lib/rpm if non-existent in, say, a chroot. -- erased packages are now repackaged into /var/spool/repackage. - -* Fri Aug 9 2002 Jeff Johnson 4.1-0.77 -- python: add return codes for rollbacks and fooDB methods. -- avoid generating fingerprints for locale/zoneinfo sub-directories. -- python: add (optional) ts.check() callback. -- python: include instance in IDTXload, filename in IDTXglob, return -- python: argument to ts.addErase (if integer) deletes that instance. -- python: rpmmi methods to return this instance, and number of members. - -* Wed Aug 7 2002 Jeff Johnson 4.1-0.75 -- fix: src.rpm installs need fd pos at payload. - -* Tue Aug 6 2002 Jeff Johnson 4.1-0.73 -- python: bindings to import pubkeys and display openpgp packets. - -* Mon Aug 5 2002 Jeff Johnson 4.1-0.72 -- python: bare bones rollback bindings. -- python: enable threads on callbacks and longish rpmlib calls. -- python: expose RPMTAG_SOURCEPACKAGE to identify source headers. -- python: eliminate rpm.headerFromPackage() tuple return, deprecated. -- python: add ts.hdrFromFdno(fdno) method. -- fix: check for lead magic, better error message on failure (#69751). -- python: the death of rpmdb-py.[ch], use ts.fooDB() methods instead. -- python: the death of rpm.headerFromPackage(), use ts.hdrFromFdno(). -- python: permit direct ts.dbMatch() python iterations. -- python: the death of rpm.checksig(), use ts.hdrFromFdno() instead. -- add bitmask for precise control of signature/digest verification. - -* Sun Aug 4 2002 Jeff Johnson 4.1-0.68 -- resurrect --rollback. -- renumber the callback types to be an orthogonal bit mask. -- provide repackage progress callbacks, compression is slow. -- fix: don't repackage %%ghost files. -- add --predefine to define macros before reading macro configuration. - -* Fri Aug 2 2002 Jeff Johnson 4.1-0.67 -- fix: identify athlon with 3DNOWEXT as "athlon", not "i786" (#70539). -- fix: repair --root with --verify (#70527). -- fix: signed pubkeys were imported incorrectly (#68291). -- include tgpg script to verify signatures using only gpg. -- check header blobs on export (i.e. rpmdbAdd()).. -- enable iterator header blob checks for install/erase modes. -- python: _vsflags_up2date macro to configure verify signature flags. - -* Thu Aug 1 2002 Jeff Johnson 4.1-0.63 -- add check-files to rpm-build manifest. -- python: methods to disable all new features for benchmarking. -- preserve rpmdb file attributes across --rebuilddb (#70367). - -* Wed Jul 31 2002 Jeff Johnson 4.1-0.62 -- make --querytags a common option, fix errant regex (#70135). -- db3: increase mpool and cachesize, compile w/o --enable-debug. -- configurable (default off) build failure if missing %%doc files (PLD). -- configurable (default off) build failure iff unpackaged files (PLD). -- change from default off to default on. - -* Tue Jul 30 2002 Jeff Johnson 4.1-0.59 -- prevent stale locks in __db files by closing databases on signals. - -* Mon Jul 29 2002 Jeff Johnson 4.1-0.58 -- popt: display sub-table options only once on --usage. -- wire --nosignatures et al as common options, rework CLI options. -- python: don't segfault in ts.GetKeys() on erased packages. -- update trpm. -- factor all mode-specific options into mode-specific tables. -- treat an unspecified epoch as Epoch: 0 everywhere. - -* Thu Jul 25 2002 Jeff Johnson 4.1-0.57 -- python: remove the old initdb/rebuilddb methods, use ts.fooDB(). -- python: 1st crack at backport to 1.5.2. -- popt: fix --usage (#62234). -- fix: --repackage repaired (#67217). -- fix: rpm2cpio disables signature checks (i.e. same behavior). - -* Wed Jul 24 2002 Jeff Johnson 4.1-0.55 -- imbue %ghost with missingok attribute with --verify (#68933). -- fix: segfault if given a text file as 2nd arg to -Uvh (#69508). -- python: add ts.hdrCheck(), ts.rebuildDB() methods. -- python: iterating on an ts object returns transaction elements now. --add yellowdog as vendor. - -* Tue Jul 23 2002 Jeff Johnson 4.1-0.53 -- verify signatures/digests retrieved through rpmdbNextIterator(). - -* Sun Jul 21 2002 Jeff Johnson 4.1-0.52 -- add methods to make signature handling opaque wrto rpmts. - -* Sat Jul 20 2002 Jeff Johnson 4.1-0.51 -- popt: parse file into string of options (#56860). -- version added to *.la dependency libraries (#69063). -- expose digests in rpmio API, but hide internal beecrypt API (#68999). - -* Mon Jul 15 2002 Jeff Johnson 4.1-0.50 -- have rpmdb-redhat autoconfigure rpm to use %%_solve_dbpath. - -* Sun Jul 14 2002 Jeff Johnson 4.1-0.49 -- python: sanity check fixes on rpmts/rpmte methods. - -* Sat Jul 13 2002 Jeff Johnson 4.1-0.48 -- install rpmal.h and rpmhash.h, implicit rpmps.hinclude, for now. -- revert headerFree/headerLink/headerUnlink debugging. -- popt: mingw32 portability configure check (#67911). -- teach gendiff about unreadable files (#42666). - -* Fri Jul 12 2002 Jeff Johnson 4.1-0.47 -- display signature details using rpm -qi. -- skip signing packages already signed with same key (#62118). - -* Thu Jul 11 2002 Jeff Johnson 4.1-0.45 -- placeholders for manifest constants for SuSE patch packages. -- fix: repair 2ndary match criteria with rpmdb iterators. -- update for sv.po. - -* Sat Jul 6 2002 Jeff Johnson 4.1-0.43 -- fix: permit deletions of added packages (#67108). - -* Fri Jul 5 2002 Jeff Johnson 4,1-0.42 -- fix: do lazy open of database in rpmtsInitDB() (#67994). -- update rpmcache.8. - -* Thu Jul 4 2002 Jeff Johnson 4.1-0.40 -- fix: forgot lazy open of database on --freshen path (#67907). -- proof-of-concept rpmcache functionality. - -* Wed Jul 3 2002 Jeff Johnson 4.1-0.39 -- use rpmfi in showQueryPackage(), eliminating headerGetEntry(). - -* Sun Jun 30 2002 Jeff Johnson 4.1-0.38 -- turn off annoying gpg secmem warning. -- warn only once for each NOKEY/UNTRUSTED key id. -- factor common options into table, add rpmcliInit() and rpmcliFini(). -- add preliminary rpmgraph(8) and rpmcache(8) executables to rpm-devel. - -* Fri Jun 28 2002 Jeff Johnson 4.1-0.36 -- fix: multiple relocations (#67596). -- add --build/--program-prefix, delete libtoolize, from %%configure. -- find-lang.sh: make sure that mo pattern matches leading '/' (#67368). -- disambiguate added/installed dependency failures (#50388). -- rescusitate remote ftp globs (#67678). -- open rpmdb early RDONLY, reopen later RDWR, avoid signed srpm noise. - -* Thu Jun 27 2002 Jeff Johnson 4.1-0.33 -- use rpmdb-redhat for suggestions if/when configured (#67430). -- disambiguate failures to import (with error messages, duh). - -* Mon Jun 24 2002 Jeff Johnson 4.1-0.32 -- python: link internal libelf (if used) directly into rpmmodule.so. - -* Fri Jun 21 2002 Jeff Johnson 4.1-0.31 -- fix: reading macro files can corrupt memory if BUFSIZ is teensy. -- fix: assertion failure iff incomplete package install (#66837). - -* Thu Jun 20 2002 Jeff Johnson 4.1-0.29 -- add translated man pages from PLD. -- resurrect libelf with Elf64, prelink verify should work everywhere. - -* Wed Jun 19 2002 Jeff Johnson 4.1-0.28 -- ix86: make sure that rpm can verify prelinked shared libraries. -- don't install /usr/lib/rpm/redhat per-vendor configuration anymore. - -* Sat Jun 15 2002 Jeff Johnson 4.1-0.27 -- beecrypt: merge changes from beecrypt-2.3.0. -- beecrypt: merge doxygen markup with rpmapi doco. -- beecrypt: revert cpu/arch compile option mixup (#66752). - -* Wed Jun 12 2002 Jeff Johnson 4.1-0.26 -- fix: separate existence and number checks for problems found (#66552). - -* Mon Jun 10 2002 Jeff Johnson 4.1-0.25 -- document digital signature verification in rpm.8. - -* Sun Jun 9 2002 Jeff Johnson 4.1-0.24 -- add /usr/lib/rpm/redhat/* per-vendor configuration. -- remove build mode compatibility aliases, documented and gone. - -* Wed Jun 5 2002 Jeff Johnson 4.1-0.23 -- python: portability fiddles (#54350). -- check inodes (but not blocks) on FAT file systems (#64256). - -* Tue Jun 4 2002 Jeff Johnson 4.1-0.22 -- use /etc/rpm/platform (if it exists), rather than uname(2), for arch. - -* Mon Jun 3 2002 Jeff Johnson 4.1-0.21 -- python: include rpmdb module, renamed from bsddb3. - -* Sat Jun 1 2002 Jeff Johnson 4.1-0.20 -- fix: use getgrnam, not getpwnam, to convert gid -> group. -- fix: avoid sign extension, use only 16 bits, when verifying rdev. -- python: separate {add,del}Macro methods, prepare for macro dictionary. -- i18n: copy current production PO files to top-of-stack. - -* Tue May 28 2002 Jeff Johnson 4.1-0.19 -- python: add rpmal/rpmte/rpmfd methods to bindings. -- perl: drop cpanflute and cpanflute2, will be in Chip's CPAN package. -- python: eliminate legacy db methods, add ts.dbMatch method. - -* Sun May 26 2002 Jeff Johnson 4.1-0.17 -- fix: un-resurrect "()(64bit)" markings using objdump on i386. - -* Fri May 24 2002 Jeff Johnson 4.1-0.16 -- python: add rpmds/rpmfi/rpmts methods. -- python: re-enable rpm-python sub-package. -- fix: resurrect "()(64bit)" markings using objdump. - -* Sun May 19 2002 Jeff Johnson 4.1-0.15 -- Grand Renaming of rpm data types. -- fix: synthesize unpacking progress callbacks for packages w/o files. - -* Thu May 16 2002 Jeff Johnson 4.1-0.14 -- opaque (well mostly) rpmTransactionSet using methods. - -* Mon May 6 2002 Jeff Johnson 4.1-0.13 -- rework most of rpmdb.c prepatory to implementing duplicates. -- fix: 2 memory leaks in headerSprintf. -- fix: db mire's access out-of-bounds memory. -- plug most install mode leaks. - -* Fri May 3 2002 Jeff Johnson 4.1-0.12 -- use DBT_DB_MALLOC to eliminate re-malloc'ing header blobs. - -* Thu May 2 2002 Jeff Johnson 4.1-0.11 -- eliminate db1 support. -- enable CDB by default. - -* Wed May 1 2002 Jeff Johnson 4.1-0.10 -- attempt to make peace with automake-1.6.1, autoconf-2.53. -- rip out two layers of dbN gook, internal Berkeley db is here to stay. - -* Fri Apr 19 2002 Jeff Johnson 4.1-0.09 -- fix: queries that evaluated to "" incorrectly returned NULL. -- fix: packages produced by rpm-4.0 dinna merge signature tags. - -* Sun Apr 14 2002 Jeff Johnson 4.1-0.08 -- use build time to choose one of multiple alternative suggestions. -- add --nosuggests to disable suggested resolutions. - -* Sat Apr 13 2002 Jeff Johnson 4.1-0.07 -- merge conflicts into problems, handle as transaction set variable. - -* Fri Apr 12 2002 Jeff Johnson 4.1-0.06 -- use rpmdb-redhat to suggest dependency resolution(s). - -* Thu Apr 11 2002 Jeff Johnson 4.1-0.05 -- rescusitate --rebuild. - -* Wed Apr 10 2002 Jeff Johnsion 4.1-0.04 -- beecrypt: add types.h, eliminate need for config.gnu.h. - -* Sat Mar 16 2002 Jeff Johnson -- *really* dump signature header immutable region. - -* Sun Mar 10 2002 Jeff Johnson -- make --addsign and --resign behave exactly the same. -- splint annotations, signature cleanup. -- drill ts/fi through verify mode, add methods to keep fi abstract. -- use mmap when calculating file digests on verify, ~20% faster. -- permit --dbpath and --root with signature (i.e. --import) modes. +* Mon Aug 26 2002 Jeff Johnson 4.2-0.1 +- set cachesize without a dbenv, the default is far too small. +- db: don't return EACCES on db->close w/o environment. +- unify cachesize configuration, with (or without) a dbenv. +- comments regarding unsupported (yet) db-4.1.17 functionality. diff --git a/rpmdb/db3.c b/rpmdb/db3.c index c2167d0..608e7fe 100644 --- a/rpmdb/db3.c +++ b/rpmdb/db3.c @@ -266,11 +266,22 @@ static int db_init(dbiIndex dbi, const char * dbhome, { int xx; /*@-noeffectuncon@*/ /* FIX: annotate db3 methods */ + + /* 4.1: dbenv->set_app_dispatch(???) */ + /* 4.1: dbenv->set_alloc(???) */ + /* 4.1: dbenv->set_data_dir(???) */ + /* 4.1: dbenv->set_encrypt(???) */ + dbenv->set_errcall(dbenv, rpmdb->db_errcall); dbenv->set_errfile(dbenv, rpmdb->db_errfile); dbenv->set_errpfx(dbenv, rpmdb->db_errpfx); /*@=noeffectuncon@*/ + + /* 4.1: dbenv->set_feedback(???) */ + /* 4.1: dbenv->set_flags(???) */ + /* dbenv->set_paniccall(???) */ + xx = dbenv->set_verbose(dbenv, DB_VERB_CHKPOINT, (dbi->dbi_verbose & DB_VERB_CHKPOINT)); xx = dbenv->set_verbose(dbenv, DB_VERB_DEADLOCK, @@ -279,16 +290,35 @@ static int db_init(dbiIndex dbi, const char * dbhome, (dbi->dbi_verbose & DB_VERB_RECOVERY)); xx = dbenv->set_verbose(dbenv, DB_VERB_WAITSFOR, (dbi->dbi_verbose & DB_VERB_WAITSFOR)); - /* dbenv->set_lg_max(???) */ + /* dbenv->set_lk_conflicts(???) */ /* dbenv->set_lk_detect(???) */ - /* dbenv->set_lk_max(???) */ - xx = dbenv->set_mp_mmapsize(dbenv, dbi->dbi_mp_mmapsize); - xx = cvtdberr(dbi, "dbenv->set_mp_mmapsize", xx, _debug); - xx = dbenv->set_cachesize(dbenv, 0, dbi->dbi_mp_size, 0); - xx = cvtdberr(dbi, "dbenv->set_cachesize", xx, _debug); + /* 4.1: dbenv->set_lk_max_lockers(???) */ + /* 4.1: dbenv->set_lk_max_locks(???) */ + /* 4.1: dbenv->set_lk_max_objects(???) */ + + /* 4.1: dbenv->set_lg_bsize(???) */ + /* 4.1: dbenv->set_lg_dir(???) */ + /* 4.1: dbenv->set_lg_max(???) */ + /* 4.1: dbenv->set_lg_regionmax(???) */ + + if (dbi->dbi_mmapsize) { + xx = dbenv->set_mp_mmapsize(dbenv, dbi->dbi_mmapsize); + xx = cvtdberr(dbi, "dbenv->set_mp_mmapsize", xx, _debug); + } + if (dbi->dbi_cachesize) { + xx = dbenv->set_cachesize(dbenv, 0, dbi->dbi_cachesize, 0); + xx = cvtdberr(dbi, "dbenv->set_cachesize", xx, _debug); + } + + /* 4.1 dbenv->set_timeout(???) */ /* dbenv->set_tx_max(???) */ + /* 4.1: dbenv->set_tx_timestamp(???) */ /* dbenv->set_tx_recover(???) */ + + /* dbenv->set_rep_transport(???) */ + /* dbenv->set_rep_limit(???) */ + if (dbi->dbi_no_fsync) { #if (DB_VERSION_MAJOR == 3 && DB_VERSION_MINOR != 0) || (DB_VERSION_MAJOR == 4) xx = db_env_set_func_fsync(db3_fsync_disable); @@ -995,18 +1025,7 @@ static int db3open(rpmdb rpmdb, rpmTag rpmtag, dbiIndex * dbip) /*@=moduncon@*/ rc = cvtdberr(dbi, "db_create", rc, _debug); if (rc == 0 && db != NULL) { - if (rc == 0 && dbi->dbi_lorder) { - rc = db->set_lorder(db, dbi->dbi_lorder); - rc = cvtdberr(dbi, "db->set_lorder", rc, _debug); - } - if (rc == 0 && dbi->dbi_cachesize) { - rc = db->set_cachesize(db, 0, dbi->dbi_cachesize, 0); - rc = cvtdberr(dbi, "db->set_cachesize", rc, _debug); - } - if (rc == 0 && dbi->dbi_pagesize) { - rc = db->set_pagesize(db, dbi->dbi_pagesize); - rc = cvtdberr(dbi, "db->set_pagesize", rc, _debug); - } + /* XXX 3.3.4 change. */ #if (DB_VERSION_MAJOR == 3 && DB_VERSION_MINOR == 3) || (DB_VERSION_MAJOR == 4) if (rc == 0 && @@ -1022,6 +1041,27 @@ static int db3open(rpmdb rpmdb, rpmTag rpmtag, dbiIndex * dbip) rc = cvtdberr(dbi, "db->set_malloc", rc, _debug); } #endif + +/* 4.1: db->set_cache_priority(???) */ + if (rc == 0 && !dbi->dbi_use_dbenv && dbi->dbi_cachesize) { + rc = db->set_cachesize(db, 0, dbi->dbi_cachesize, 0); + rc = cvtdberr(dbi, "db->set_cachesize", rc, _debug); + } +/* 4.1: db->set_encrypt(???) */ +/* 4.1: db->set_errcall(dbenv, rpmdb->db_errcall); */ +/* 4.1: db->set_errfile(dbenv, rpmdb->db_errfile); */ +/* 4.1: db->set_errpfx(dbenv, rpmdb->db_errpfx); */ + /* 4.1: db->set_feedback(???) */ + + if (rc == 0 && dbi->dbi_lorder) { + rc = db->set_lorder(db, dbi->dbi_lorder); + rc = cvtdberr(dbi, "db->set_lorder", rc, _debug); + } + if (rc == 0 && dbi->dbi_pagesize) { + rc = db->set_pagesize(db, dbi->dbi_pagesize); + rc = cvtdberr(dbi, "db->set_pagesize", rc, _debug); + } + /* 4.1: db->set_paniccall(???) */ if (rc == 0 && oflags & DB_CREATE) { switch(dbi->dbi_type) { default: @@ -1056,6 +1096,7 @@ static int db3open(rpmdb rpmdb, rpmTag rpmtag, dbiIndex * dbip) #endif break; case DB_BTREE: +/* 4.1: db->set_append_recno(???) */ if (dbi->dbi_bt_flags) { rc = db->set_flags(db, dbi->dbi_bt_flags); rc = cvtdberr(dbi, "db->set_bt_flags", rc, _debug); @@ -1087,6 +1128,7 @@ static int db3open(rpmdb rpmdb, rpmTag rpmtag, dbiIndex * dbip) break; case DB_RECNO: if (dbi->dbi_re_delim) { +/* 4.1: db->set_append_recno(???) */ rc = db->set_re_delim(db, dbi->dbi_re_delim); rc = cvtdberr(dbi, "db->set_re_selim", rc, _debug); if (rc) break; diff --git a/rpmdb/dbconfig.c b/rpmdb/dbconfig.c index ede58d8..8692236 100644 --- a/rpmdb/dbconfig.c +++ b/rpmdb/dbconfig.c @@ -190,9 +190,11 @@ struct poptOption rdbOptions[] = { { "lorder", 0,POPT_ARG_INT, &db3dbi.dbi_lorder, 0, NULL, NULL }, - { "mp_mmapsize", 0,POPT_ARG_INT, &db3dbi.dbi_mp_mmapsize, 0, + { "mmapsize", 0,POPT_ARG_INT, &db3dbi.dbi_mmapsize, 0, NULL, NULL }, - { "mp_size", 0,POPT_ARG_INT, &db3dbi.dbi_mp_size, 0, + { "mp_mmapsize", 0,POPT_ARG_INT, &db3dbi.dbi_mmapsize, 0, + NULL, NULL }, + { "mp_size", 0,POPT_ARG_INT, &db3dbi.dbi_cachesize, 0, NULL, NULL }, { "pagesize", 0,POPT_ARG_INT, &db3dbi.dbi_pagesize, 0, NULL, NULL }, @@ -259,7 +261,7 @@ dbiIndex db3Free(dbiIndex dbi) /** @todo Set a reasonable "last gasp" default db config. */ /*@observer@*/ /*@unchecked@*/ static const char *db3_config_default = - "db3:hash:mpool:cdb:usecursors:verbose:mp_mmapsize=8Mb:mp_size=512Kb:pagesize=512:perms=0644"; + "db3:hash:mpool:cdb:usecursors:verbose:mp_mmapsize=8Mb:cachesize=512Kb:pagesize=512:perms=0644"; /*@-bounds@*/ dbiIndex db3New(rpmdb rpmdb, rpmTag rpmtag) @@ -432,8 +434,8 @@ dbiIndex db3New(rpmdb rpmdb, rpmTag rpmtag) if (!dbi->dbi_use_dbenv) { /* db3 dbenv is always used now. */ dbi->dbi_use_dbenv = 1; dbi->dbi_eflags |= (DB_INIT_MPOOL|DB_JOINENV); - dbi->dbi_mp_mmapsize = 16 * 1024 * 1024; - dbi->dbi_mp_size = 1 * 1024 * 1024; + dbi->dbi_mmapsize = 16 * 1024 * 1024; + dbi->dbi_cachesize = 1 * 1024 * 1024; } if ((dbi->dbi_bt_flags | dbi->dbi_h_flags) & DB_DUP) diff --git a/rpmdb/rpmdb.h b/rpmdb/rpmdb.h index e868469..7314b17 100644 --- a/rpmdb/rpmdb.h +++ b/rpmdb/rpmdb.h @@ -321,8 +321,8 @@ struct _dbiIndex { int dbi_region_init; int dbi_tas_spins; /* mpool sub-system parameters */ - int dbi_mp_mmapsize; /*!< (10Mb) */ - int dbi_mp_size; /*!< (128Kb) */ + int dbi_mmapsize; /*!< (10Mb) */ + int dbi_cachesize; /*!< (128Kb) */ /* lock sub-system parameters */ unsigned int dbi_lk_max; unsigned int dbi_lk_detect; @@ -340,7 +340,6 @@ struct _dbiIndex { /*@modifies fileSystem @*/; #endif /* dbinfo parameters */ - int dbi_cachesize; /*!< */ int dbi_pagesize; /*!< (fs blksize) */ /*@unused@*/ /*@null@*/ void * (*dbi_malloc) (size_t nbytes) -- 2.7.4