camel/camel-block-file.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
   2 /*
   3  * Copyright (C) 2001-2003 Ximian Inc.
   4  *
   5  * Authors: Michael Zucchi <notzed@ximian.com>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of version 2 of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this program; if not, write to the
  18  * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  * Boston, MA 02110-1301, USA.
  20  */
  21
  22 #ifdef HAVE_CONFIG_H
  23 #include <config.h>
  24 #endif
  25
  26 #include <errno.h>
  27 #include <fcntl.h>
  28 #include <pthread.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <unistd.h>
  32 #include <sys/stat.h>
  33 #include <sys/types.h>
  34
  35 #include <glib.h>
  36 #include <glib/gstdio.h>
  37
  38 #include <libedataserver/e-msgport.h>
  39
  40 #include "camel-block-file.h"
  41 #include "camel-file-utils.h"
  42 #include "camel-private.h"
  43
  44 #define d(x) /*(printf("%s(%d):%s: ",  __FILE__, __LINE__, __PRETTY_FUNCTION__),(x))*/
  45
  46 /* Locks must be obtained in the order defined */
  47
  48 struct _CamelBlockFilePrivate {
  49         /* We use the private structure to form our lru list from */
  50         struct _CamelBlockFilePrivate *next;
  51         struct _CamelBlockFilePrivate *prev;
  52
  53         struct _CamelBlockFile *base;
  54
  55         pthread_mutex_t root_lock; /* for modifying the root block */
  56         pthread_mutex_t cache_lock; /* for refcounting, flag manip, cache manip */
  57         pthread_mutex_t io_lock; /* for all io ops */
  58
  59         unsigned int deleted:1;
  60 };
  61
  62
  63 #define CAMEL_BLOCK_FILE_LOCK(kf, lock) (pthread_mutex_lock(&(kf)->priv->lock))
  64 #define CAMEL_BLOCK_FILE_TRYLOCK(kf, lock) (pthread_mutex_trylock(&(kf)->priv->lock))
  65 #define CAMEL_BLOCK_FILE_UNLOCK(kf, lock) (pthread_mutex_unlock(&(kf)->priv->lock))
  66
  67 #define LOCK(x) pthread_mutex_lock(&x)
  68 #define UNLOCK(x) pthread_mutex_unlock(&x)
  69
  70 static pthread_mutex_t block_file_lock = PTHREAD_MUTEX_INITIALIZER;
  71
  72 /* lru cache of block files */
  73 static EDList block_file_list = E_DLIST_INITIALISER(block_file_list);
  74 /* list to store block files that are actually intialised */
  75 static EDList block_file_active_list = E_DLIST_INITIALISER(block_file_active_list);
  76 static int block_file_count = 0;
  77 static int block_file_threshhold = 10;
  78
  79 #define CBF_CLASS(o) ((CamelBlockFileClass *)(((CamelObject *)o)->klass))
  80
  81 static int sync_nolock(CamelBlockFile *bs);
  82 static int sync_block_nolock(CamelBlockFile *bs, CamelBlock *bl);
  83
  84 static int
  85 block_file_validate_root(CamelBlockFile *bs)
  86 {
  87         CamelBlockRoot *br;
  88         struct stat st;
  89         int retval;
  90
  91         br = bs->root;
  92
  93         retval = fstat (bs->fd, &st);
  94
  95         d(printf("Validate root: '%s'\n", bs->path));
  96         d(printf("version: %.8s (%.8s)\n", bs->root->version, bs->version));
  97         d(printf("block size: %d (%d)%s\n", br->block_size, bs->block_size,
  98                 br->block_size != bs->block_size ? " BAD":" OK"));
  99         d(printf("free: %ld (%d add size < %ld)%s\n", (long)br->free, br->free / bs->block_size * bs->block_size, (long)st.st_size,
 100                 (br->free > st.st_size) || (br->free % bs->block_size) != 0 ? " BAD":" OK"));
 101         d(printf("last: %ld (%d and size: %ld)%s\n", (long)br->last, br->last / bs->block_size * bs->block_size, (long)st.st_size,
 102                 (br->last != st.st_size) || ((br->last % bs->block_size) != 0) ? " BAD": " OK"));
 103         d(printf("flags: %s\n", (br->flags & CAMEL_BLOCK_FILE_SYNC)?"SYNC":"unSYNC"));
 104
 105         if (br->last == 0
 106             || memcmp(bs->root->version, bs->version, 8) != 0
 107             || br->block_size != bs->block_size
 108             || (br->free % bs->block_size) != 0
 109             || (br->last % bs->block_size) != 0
 110             || retval == -1
 111             || st.st_size != br->last
 112             || br->free > st.st_size
 113             || (br->flags & CAMEL_BLOCK_FILE_SYNC) == 0) {
 114 #if 0
 115                 if (retval != -1 && st.st_size > 0) {
 116                         g_warning("Invalid root: '%s'", bs->path);
 117                         g_warning("version: %.8s (%.8s)", bs->root->version, bs->version);
 118                         g_warning("block size: %d (%d)%s", br->block_size, bs->block_size,
 119                                   br->block_size != bs->block_size ? " BAD":" OK");
 120                         g_warning("free: %ld (%d add size < %ld)%s", (long)br->free, br->free / bs->block_size * bs->block_size, (long)st.st_size,
 121                                   (br->free > st.st_size) || (br->free % bs->block_size) != 0 ? " BAD":" OK");
 122                         g_warning("last: %ld (%d and size: %ld)%s", (long)br->last, br->last / bs->block_size * bs->block_size, (long)st.st_size,
 123                                   (br->last != st.st_size) || ((br->last % bs->block_size) != 0) ? " BAD": " OK");
 124                         g_warning("flags: %s", (br->flags & CAMEL_BLOCK_FILE_SYNC)?"SYNC":"unSYNC");
 125                 }
 126 #endif
 127                 return -1;
 128         }
 129
 130         return 0;
 131 }
 132
 133 static int
 134 block_file_init_root(CamelBlockFile *bs)
 135 {
 136         CamelBlockRoot *br = bs->root;
 137
 138         memset(br, 0, bs->block_size);
 139         memcpy(br->version, bs->version, 8);
 140         br->last = bs->block_size;
 141         br->flags = CAMEL_BLOCK_FILE_SYNC;
 142         br->free = 0;
 143         br->block_size = bs->block_size;
 144
 145         return 0;
 146 }
 147
 148 static void
 149 camel_block_file_class_init(CamelBlockFileClass *klass)
 150 {
 151         klass->validate_root = block_file_validate_root;
 152         klass->init_root = block_file_init_root;
 153 }
 154
 155 static guint
 156 block_hash_func(const void *v)
 157 {
 158         return ((camel_block_t) GPOINTER_TO_UINT(v)) >> CAMEL_BLOCK_SIZE_BITS;
 159 }
 160
 161 static void
 162 camel_block_file_init(CamelBlockFile *bs)
 163 {
 164         struct _CamelBlockFilePrivate *p;
 165
 166         bs->fd = -1;
 167         bs->block_size = CAMEL_BLOCK_SIZE;
 168         e_dlist_init(&bs->block_cache);
 169         bs->blocks = g_hash_table_new((GHashFunc)block_hash_func, NULL);
 170         /* this cache size and the text index size have been tuned for about the best
 171            with moderate memory usage.  Doubling the memory usage barely affects performance. */
 172         bs->block_cache_limit = 256;
 173
 174         p = bs->priv = g_malloc0(sizeof(*bs->priv));
 175         p->base = bs;
 176
 177         pthread_mutex_init(&p->root_lock, NULL);
 178         pthread_mutex_init(&p->cache_lock, NULL);
 179         pthread_mutex_init(&p->io_lock, NULL);
 180
 181         /* link into lru list */
 182         LOCK(block_file_lock);
 183         e_dlist_addhead(&block_file_list, (EDListNode *)p);
 184
 185 #if 0
 186         {
 187                 printf("dumping block list\n");
 188                 printf(" head = %p p = %p\n", block_file_list.head, p);
 189                 p = block_file_list.head;
 190                 while (p->next) {
 191                         printf(" '%s'\n", p->base->path);
 192                         p = p->next;
 193                 }
 194         }
 195 #endif
 196
 197         UNLOCK(block_file_lock);
 198 }
 199
 200 static void
 201 camel_block_file_finalise(CamelBlockFile *bs)
 202 {
 203         CamelBlock *bl, *bn;
 204         struct _CamelBlockFilePrivate *p;
 205
 206         p = bs->priv;
 207
 208         if (bs->root_block)
 209                 camel_block_file_sync(bs);
 210
 211         /* remove from lru list */
 212         LOCK(block_file_lock);
 213         if (bs->fd != -1)
 214                 block_file_count--;
 215         e_dlist_remove((EDListNode *)p);
 216         UNLOCK(block_file_lock);
 217
 218         bl = (CamelBlock *)bs->block_cache.head;
 219         bn = bl->next;
 220         while (bn) {
 221                 if (bl->refcount != 0)
 222                         g_warning("Block '%u' still referenced", bl->id);
 223                 g_free(bl);
 224                 bl = bn;
 225                 bn = bn->next;
 226         }
 227
 228         g_hash_table_destroy (bs->blocks);
 229
 230         if (bs->root_block)
 231                 camel_block_file_unref_block(bs, bs->root_block);
 232         g_free(bs->path);
 233         if (bs->fd != -1)
 234                 close(bs->fd);
 235
 236         pthread_mutex_destroy(&p->io_lock);
 237         pthread_mutex_destroy(&p->cache_lock);
 238         pthread_mutex_destroy(&p->root_lock);
 239
 240         g_free(p);
 241 }
 242
 243 CamelType
 244 camel_block_file_get_type(void)
 245 {
 246         static CamelType type = CAMEL_INVALID_TYPE;
 247
 248         if (type == CAMEL_INVALID_TYPE) {
 249                 type = camel_type_register(camel_object_get_type(), "CamelBlockFile",
 250                                            sizeof (CamelBlockFile),
 251                                            sizeof (CamelBlockFileClass),
 252                                            (CamelObjectClassInitFunc) camel_block_file_class_init,
 253                                            NULL,
 254                                            (CamelObjectInitFunc) camel_block_file_init,
 255                                            (CamelObjectFinalizeFunc) camel_block_file_finalise);
 256         }
 257
 258         return type;
 259 }
 260
 261 /* 'use' a block file for io */
 262 static int
 263 block_file_use(CamelBlockFile *bs)
 264 {
 265         struct _CamelBlockFilePrivate *nw, *nn, *p = bs->priv;
 266         CamelBlockFile *bf;
 267         int err;
 268
 269         /* We want to:
 270             remove file from active list
 271             lock it
 272
 273            Then when done:
 274             unlock it
 275             add it back to end of active list
 276         */
 277
 278         CAMEL_BLOCK_FILE_LOCK(bs, io_lock);
 279
 280         if (bs->fd != -1)
 281                 return 0;
 282         else if (p->deleted) {
 283                 CAMEL_BLOCK_FILE_UNLOCK(bs, io_lock);
 284                 errno = ENOENT;
 285                 return -1;
 286         } else
 287                 d(printf("Turning block file online: %s\n", bs->path));
 288
 289         if ((bs->fd = g_open(bs->path, bs->flags|O_BINARY, 0600)) == -1) {
 290                 err = errno;
 291                 CAMEL_BLOCK_FILE_UNLOCK(bs, io_lock);
 292                 errno = err;
 293                 return -1;
 294         }
 295
 296         LOCK(block_file_lock);
 297         e_dlist_remove((EDListNode *)p);
 298         e_dlist_addtail(&block_file_active_list, (EDListNode *)p);
 299
 300         block_file_count++;
 301
 302         nw = (struct _CamelBlockFilePrivate *)block_file_list.head;
 303         nn = nw->next;
 304         while (block_file_count > block_file_threshhold && nn) {
 305                 /* We never hit the current blockfile here, as its removed from the list first */
 306                 bf = nw->base;
 307                 if (bf->fd != -1) {
 308                         /* Need to trylock, as any of these lock levels might be trying
 309                            to lock the block_file_lock, so we need to check and abort if so */
 310                         if (CAMEL_BLOCK_FILE_TRYLOCK(bf, root_lock) == 0) {
 311                                 if (CAMEL_BLOCK_FILE_TRYLOCK(bf, cache_lock) == 0) {
 312                                         if (CAMEL_BLOCK_FILE_TRYLOCK(bf, io_lock) == 0) {
 313                                                 d(printf("[%d] Turning block file offline: %s\n", block_file_count-1, bf->path));
 314                                                 sync_nolock(bf);
 315                                                 close(bf->fd);
 316                                                 bf->fd = -1;
 317                                                 block_file_count--;
 318                                                 CAMEL_BLOCK_FILE_UNLOCK(bf, io_lock);
 319                                         }
 320                                         CAMEL_BLOCK_FILE_UNLOCK(bf, cache_lock);
 321                                 }
 322                                 CAMEL_BLOCK_FILE_UNLOCK(bf, root_lock);
 323                         }
 324                 }
 325                 nw = nn;
 326                 nn = nw->next;
 327         }
 328
 329         UNLOCK(block_file_lock);
 330
 331         return 0;
 332 }
 333
 334 static void
 335 block_file_unuse(CamelBlockFile *bs)
 336 {
 337         LOCK(block_file_lock);
 338         e_dlist_remove((EDListNode *)bs->priv);
 339         e_dlist_addtail(&block_file_list, (EDListNode *)bs->priv);
 340         UNLOCK(block_file_lock);
 341
 342         CAMEL_BLOCK_FILE_UNLOCK(bs, io_lock);
 343 }
 344
 345 /*
 346 o = camel_cache_get(c, key);
 347 camel_cache_unref(c, key);
 348 camel_cache_add(c, key, o);
 349 camel_cache_remove(c, key);
 350 */
 351
 352 /**
 353  * camel_block_file_new:
 354  * @path:
 355  * @:
 356  * @block_size:
 357  *
 358  * Allocate a new block file, stored at @path.  @version contains an 8 character
 359  * version string which must match the head of the file, or the file will be
 360  * intitialised.
 361  *
 362  * @block_size is currently ignored and is set to CAMEL_BLOCK_SIZE.
 363  *
 364  * Return value: The new block file, or NULL if it could not be created.
 365  **/
 366 CamelBlockFile *camel_block_file_new(const char *path, int flags, const char version[8], size_t block_size)
 367 {
 368         CamelBlockFile *bs;
 369
 370         bs = (CamelBlockFile *)camel_object_new(camel_block_file_get_type());
 371         memcpy(bs->version, version, 8);
 372         bs->path = g_strdup(path);
 373         bs->flags = flags;
 374
 375         bs->root_block = camel_block_file_get_block(bs, 0);
 376         if (bs->root_block == NULL) {
 377                 camel_object_unref((CamelObject *)bs);
 378                 return NULL;
 379         }
 380         camel_block_file_detach_block(bs, bs->root_block);
 381         bs->root = (CamelBlockRoot *)&bs->root_block->data;
 382
 383         /* we only need these flags on first open */
 384         bs->flags &= ~(O_CREAT|O_EXCL|O_TRUNC);
 385
 386         /* Do we need to init the root block? */
 387         if (CBF_CLASS(bs)->validate_root(bs) == -1) {
 388                 d(printf("Initialise root block: %.8s\n", version));
 389
 390                 CBF_CLASS(bs)->init_root(bs);
 391                 camel_block_file_touch_block(bs, bs->root_block);
 392                 if (block_file_use(bs) == -1) {
 393                         camel_object_unref((CamelObject *)bs);
 394                         return NULL;
 395                 }
 396                 if (sync_block_nolock(bs, bs->root_block) == -1
 397                     || ftruncate(bs->fd, bs->root->last) == -1) {
 398                         block_file_unuse(bs);
 399                         camel_object_unref((CamelObject *)bs);
 400                         return NULL;
 401                 }
 402                 block_file_unuse(bs);
 403         }
 404
 405         return bs;
 406 }
 407
 408 int
 409 camel_block_file_rename(CamelBlockFile *bs, const char *path)
 410 {
 411         int ret;
 412         struct stat st;
 413         int err;
 414
 415         CAMEL_BLOCK_FILE_LOCK(bs, io_lock);
 416
 417         ret = g_rename(bs->path, path);
 418         if (ret == -1) {
 419                 /* Maybe the rename actually worked */
 420                 err = errno;
 421                 if (g_stat(path, &st) == 0
 422                     && g_stat(bs->path, &st) == -1
 423                     && errno == ENOENT)
 424                         ret = 0;
 425                 errno = err;
 426         }
 427
 428         if (ret != -1) {
 429                 g_free(bs->path);
 430                 bs->path = g_strdup(path);
 431         }
 432
 433         CAMEL_BLOCK_FILE_UNLOCK(bs, io_lock);
 434
 435         return ret;
 436 }
 437
 438 int
 439 camel_block_file_delete(CamelBlockFile *bs)
 440 {
 441         int ret;
 442         struct _CamelBlockFilePrivate *p = bs->priv;
 443
 444         CAMEL_BLOCK_FILE_LOCK(bs, io_lock);
 445
 446         if (bs->fd != -1) {
 447                 LOCK(block_file_lock);
 448                 block_file_count--;
 449                 UNLOCK(block_file_lock);
 450                 close(bs->fd);
 451                 bs->fd = -1;
 452         }
 453
 454         p->deleted = TRUE;
 455         ret = g_unlink(bs->path);
 456
 457         CAMEL_BLOCK_FILE_UNLOCK(bs, io_lock);
 458
 459         return ret;
 460
 461 }
 462
 463 /**
 464  * camel_block_file_new_block:
 465  * @bs:
 466  *
 467  * Allocate a new block, return a pointer to it.  Old blocks
 468  * may be flushed to disk during this call.
 469  *
 470  * Return value: The block, or NULL if an error occured.
 471  **/
 472 CamelBlock *camel_block_file_new_block(CamelBlockFile *bs)
 473 {
 474         CamelBlock *bl;
 475
 476         CAMEL_BLOCK_FILE_LOCK(bs, root_lock);
 477
 478         if (bs->root->free) {
 479                 bl = camel_block_file_get_block(bs, bs->root->free);
 480                 if (bl == NULL)
 481                         goto fail;
 482                 bs->root->free = ((camel_block_t *)bl->data)[0];
 483         } else {
 484                 bl = camel_block_file_get_block(bs, bs->root->last);
 485                 if (bl == NULL)
 486                         goto fail;
 487                 bs->root->last += CAMEL_BLOCK_SIZE;
 488         }
 489
 490         bs->root_block->flags |= CAMEL_BLOCK_DIRTY;
 491
 492         bl->flags |= CAMEL_BLOCK_DIRTY;
 493         memset(bl->data, 0, CAMEL_BLOCK_SIZE);
 494 fail:
 495         CAMEL_BLOCK_FILE_UNLOCK(bs, root_lock);
 496
 497         return bl;
 498 }
 499
 500 /**
 501  * camel_block_file_free_block:
 502  * @bs:
 503  * @id:
 504  *
 505  *
 506  **/
 507 int camel_block_file_free_block(CamelBlockFile *bs, camel_block_t id)
 508 {
 509         CamelBlock *bl;
 510
 511         bl = camel_block_file_get_block(bs, id);
 512         if (bl == NULL)
 513                 return -1;
 514
 515         CAMEL_BLOCK_FILE_LOCK(bs, root_lock);
 516
 517         ((camel_block_t *)bl->data)[0] = bs->root->free;
 518         bs->root->free = bl->id;
 519         bs->root_block->flags |= CAMEL_BLOCK_DIRTY;
 520         bl->flags |= CAMEL_BLOCK_DIRTY;
 521         camel_block_file_unref_block(bs, bl);
 522
 523         CAMEL_BLOCK_FILE_UNLOCK(bs, root_lock);
 524
 525         return 0;
 526 }
 527
 528 /**
 529  * camel_block_file_get_block:
 530  * @bs:
 531  * @id:
 532  *
 533  * Retreive a block @id.
 534  *
 535  * Return value: The block, or NULL if blockid is invalid or a file error
 536  * occured.
 537  **/
 538 CamelBlock *camel_block_file_get_block(CamelBlockFile *bs, camel_block_t id)
 539 {
 540         CamelBlock *bl, *flush, *prev;
 541
 542         /* Sanity check: Dont allow reading of root block (except before its been read)
 543            or blocks with invalid block id's */
 544         if ((bs->root == NULL && id != 0)
 545             || (bs->root != NULL && (id > bs->root->last || id == 0))
 546             || (id % bs->block_size) != 0) {
 547                 errno = EINVAL;
 548                 return NULL;
 549         }
 550
 551         CAMEL_BLOCK_FILE_LOCK(bs, cache_lock);
 552
 553         bl = g_hash_table_lookup(bs->blocks, GUINT_TO_POINTER(id));
 554
 555         d(printf("Get  block %08x: %s\n", id, bl?"cached":"must read"));
 556
 557         if (bl == NULL) {
 558                 /* LOCK io_lock */
 559                 if (block_file_use(bs) == -1) {
 560                         CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock);
 561                         return NULL;
 562                 }
 563
 564                 bl = g_malloc0(sizeof(*bl));
 565                 bl->id = id;
 566                 if (lseek(bs->fd, id, SEEK_SET) == -1 ||
 567                     camel_read (bs->fd, bl->data, CAMEL_BLOCK_SIZE) == -1) {
 568                         block_file_unuse(bs);
 569                         CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock);
 570                         g_free(bl);
 571                         return NULL;
 572                 }
 573
 574                 bs->block_cache_count++;
 575                 g_hash_table_insert(bs->blocks, GUINT_TO_POINTER(bl->id), bl);
 576
 577                 /* flush old blocks */
 578                 flush = (CamelBlock *)bs->block_cache.tailpred;
 579                 prev = flush->prev;
 580                 while (bs->block_cache_count > bs->block_cache_limit && prev) {
 581                         if (flush->refcount == 0) {
 582                                 if (sync_block_nolock(bs, flush) != -1) {
 583                                         g_hash_table_remove(bs->blocks, GUINT_TO_POINTER(flush->id));
 584                                         e_dlist_remove((EDListNode *)flush);
 585                                         g_free(flush);
 586                                         bs->block_cache_count--;
 587                                 }
 588                         }
 589                         flush = prev;
 590                         prev = prev->prev;
 591                 }
 592                 /* UNLOCK io_lock */
 593                 block_file_unuse(bs);
 594         } else {
 595                 e_dlist_remove((EDListNode *)bl);
 596         }
 597
 598         e_dlist_addhead(&bs->block_cache, (EDListNode *)bl);
 599         bl->refcount++;
 600
 601         CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock);
 602
 603         d(printf("Got  block %08x\n", id));
 604
 605         return bl;
 606 }
 607
 608 /**
 609  * camel_block_file_detach_block:
 610  * @bs:
 611  * @bl:
 612  *
 613  * Detatch a block from the block file's cache.  The block should
 614  * be unref'd or attached when finished with.  The block file will
 615  * perform no writes of this block or flushing of it if the cache
 616  * fills.
 617  **/
 618 void camel_block_file_detach_block(CamelBlockFile *bs, CamelBlock *bl)
 619 {
 620         CAMEL_BLOCK_FILE_LOCK(bs, cache_lock);
 621
 622         g_hash_table_remove(bs->blocks, GUINT_TO_POINTER(bl->id));
 623         e_dlist_remove((EDListNode *)bl);
 624         bl->flags |= CAMEL_BLOCK_DETACHED;
 625
 626         CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock);
 627 }
 628
 629 /**
 630  * camel_block_file_attach_block:
 631  * @bs:
 632  * @bl:
 633  *
 634  * Reattach a block that has been detached.
 635  **/
 636 void camel_block_file_attach_block(CamelBlockFile *bs, CamelBlock *bl)
 637 {
 638         CAMEL_BLOCK_FILE_LOCK(bs, cache_lock);
 639
 640         g_hash_table_insert(bs->blocks, GUINT_TO_POINTER(bl->id), bl);
 641         e_dlist_addtail(&bs->block_cache, (EDListNode *)bl);
 642         bl->flags &= ~CAMEL_BLOCK_DETACHED;
 643
 644         CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock);
 645 }
 646
 647 /**
 648  * camel_block_file_touch_block:
 649  * @bs:
 650  * @bl:
 651  *
 652  * Mark a block as dirty.  The block will be written to disk if
 653  * it ever expires from the cache.
 654  **/
 655 void camel_block_file_touch_block(CamelBlockFile *bs, CamelBlock *bl)
 656 {
 657         CAMEL_BLOCK_FILE_LOCK(bs, root_lock);
 658         CAMEL_BLOCK_FILE_LOCK(bs, cache_lock);
 659
 660         bl->flags |= CAMEL_BLOCK_DIRTY;
 661
 662         if ((bs->root->flags & CAMEL_BLOCK_FILE_SYNC) && bl != bs->root_block) {
 663                 d(printf("turning off sync flag\n"));
 664                 bs->root->flags &= ~CAMEL_BLOCK_FILE_SYNC;
 665                 bs->root_block->flags |= CAMEL_BLOCK_DIRTY;
 666                 camel_block_file_sync_block(bs, bs->root_block);
 667         }
 668
 669         CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock);
 670         CAMEL_BLOCK_FILE_UNLOCK(bs, root_lock);
 671 }
 672
 673 /**
 674  * camel_block_file_unref_block:
 675  * @bs:
 676  * @bl:
 677  *
 678  * Mark a block as unused.  If a block is used it will not be
 679  * written to disk, or flushed from memory.
 680  *
 681  * If a block is detatched and this is the last reference, the
 682  * block will be freed.
 683  **/
 684 void camel_block_file_unref_block(CamelBlockFile *bs, CamelBlock *bl)
 685 {
 686         CAMEL_BLOCK_FILE_LOCK(bs, cache_lock);
 687
 688         if (bl->refcount == 1 && (bl->flags & CAMEL_BLOCK_DETACHED))
 689                 g_free(bl);
 690         else
 691                 bl->refcount--;
 692
 693         CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock);
 694 }
 695
 696 static int
 697 sync_block_nolock(CamelBlockFile *bs, CamelBlock *bl)
 698 {
 699         d(printf("Sync block %08x: %s\n", bl->id, (bl->flags & CAMEL_BLOCK_DIRTY)?"dirty":"clean"));
 700
 701         if (bl->flags & CAMEL_BLOCK_DIRTY) {
 702                 if (lseek(bs->fd, bl->id, SEEK_SET) == -1
 703                     || write(bs->fd, bl->data, CAMEL_BLOCK_SIZE) != CAMEL_BLOCK_SIZE) {
 704                         return -1;
 705                 }
 706                 bl->flags &= ~CAMEL_BLOCK_DIRTY;
 707         }
 708
 709         return 0;
 710 }
 711
 712 static int
 713 sync_nolock(CamelBlockFile *bs)
 714 {
 715         CamelBlock *bl, *bn;
 716         int work = FALSE;
 717
 718         bl = (CamelBlock *)bs->block_cache.head;
 719         bn = bl->next;
 720         while (bn) {
 721                 if (bl->flags & CAMEL_BLOCK_DIRTY) {
 722                         work = TRUE;
 723                         if (sync_block_nolock(bs, bl) == -1)
 724                                 return -1;
 725                 }
 726                 bl = bn;
 727                 bn = bn->next;
 728         }
 729
 730         if (!work
 731             && (bs->root_block->flags & CAMEL_BLOCK_DIRTY) == 0
 732             && (bs->root->flags & CAMEL_BLOCK_FILE_SYNC) != 0)
 733                 return 0;
 734
 735         d(printf("turning on sync flag\n"));
 736
 737         bs->root->flags |= CAMEL_BLOCK_FILE_SYNC;
 738         bs->root_block->flags |= CAMEL_BLOCK_DIRTY;
 739
 740         return sync_block_nolock(bs, bs->root_block);
 741 }
 742
 743 /**
 744  * camel_block_file_sync_block:
 745  * @bs:
 746  * @bl:
 747  *
 748  * Flush a block to disk immediately.  The block will only
 749  * be flushed to disk if it is marked as dirty (touched).
 750  *
 751  * Return value: -1 on io error.
 752  **/
 753 int camel_block_file_sync_block(CamelBlockFile *bs, CamelBlock *bl)
 754 {
 755         int ret;
 756
 757         /* LOCK io_lock */
 758         if (block_file_use(bs) == -1)
 759                 return -1;
 760
 761         ret = sync_block_nolock(bs, bl);
 762
 763         block_file_unuse(bs);
 764
 765         return ret;
 766 }
 767
 768 /**
 769  * camel_block_file_sync:
 770  * @bs:
 771  *
 772  * Sync all dirty blocks to disk, including the root block.
 773  *
 774  * Return value: -1 on io error.
 775  **/
 776 int camel_block_file_sync(CamelBlockFile *bs)
 777 {
 778         int ret;
 779
 780         CAMEL_BLOCK_FILE_LOCK(bs, root_lock);
 781         CAMEL_BLOCK_FILE_LOCK(bs, cache_lock);
 782
 783         /* LOCK io_lock */
 784         if (block_file_use(bs) == -1)
 785                 ret = -1;
 786         else {
 787                 ret = sync_nolock(bs);
 788                 block_file_unuse(bs);
 789         }
 790
 791         CAMEL_BLOCK_FILE_UNLOCK(bs, cache_lock);
 792         CAMEL_BLOCK_FILE_UNLOCK(bs, root_lock);
 793
 794         return ret;
 795 }
 796
 797 /* ********************************************************************** */
 798
 799 struct _CamelKeyFilePrivate {
 800         struct _CamelKeyFilePrivate *next;
 801         struct _CamelKeyFilePrivate *prev;
 802
 803         struct _CamelKeyFile *base;
 804         pthread_mutex_t lock;
 805         unsigned int deleted:1;
 806 };
 807
 808 #define CAMEL_KEY_FILE_LOCK(kf, lock) (pthread_mutex_lock(&(kf)->priv->lock))
 809 #define CAMEL_KEY_FILE_TRYLOCK(kf, lock) (pthread_mutex_trylock(&(kf)->priv->lock))
 810 #define CAMEL_KEY_FILE_UNLOCK(kf, lock) (pthread_mutex_unlock(&(kf)->priv->lock))
 811
 812 static pthread_mutex_t key_file_lock = PTHREAD_MUTEX_INITIALIZER;
 813
 814 /* lru cache of block files */
 815 static EDList key_file_list = E_DLIST_INITIALISER(key_file_list);
 816 static EDList key_file_active_list = E_DLIST_INITIALISER(key_file_active_list);
 817 static int key_file_count = 0;
 818 static const int key_file_threshhold = 10;
 819
 820 static void
 821 camel_key_file_class_init(CamelKeyFileClass *klass)
 822 {
 823 }
 824
 825 static void
 826 camel_key_file_init(CamelKeyFile *bs)
 827 {
 828         struct _CamelKeyFilePrivate *p;
 829
 830         p = bs->priv = g_malloc0(sizeof(*bs->priv));
 831         p->base = bs;
 832
 833         pthread_mutex_init(&p->lock, NULL);
 834
 835         LOCK(key_file_lock);
 836         e_dlist_addhead(&key_file_list, (EDListNode *)p);
 837         UNLOCK(key_file_lock);
 838 }
 839
 840 static void
 841 camel_key_file_finalise(CamelKeyFile *bs)
 842 {
 843         struct _CamelKeyFilePrivate *p = bs->priv;
 844
 845         LOCK(key_file_lock);
 846         e_dlist_remove((EDListNode *)p);
 847
 848         if (bs-> fp) {
 849                 key_file_count--;
 850                 fclose(bs->fp);
 851         }
 852
 853         UNLOCK(key_file_lock);
 854
 855         g_free(bs->path);
 856
 857         pthread_mutex_destroy(&p->lock);
 858
 859         g_free(p);
 860 }
 861
 862 CamelType
 863 camel_key_file_get_type(void)
 864 {
 865         static CamelType type = CAMEL_INVALID_TYPE;
 866
 867         if (type == CAMEL_INVALID_TYPE) {
 868                 type = camel_type_register(camel_object_get_type(), "CamelKeyFile",
 869                                            sizeof (CamelKeyFile),
 870                                            sizeof (CamelKeyFileClass),
 871                                            (CamelObjectClassInitFunc) camel_key_file_class_init,
 872                                            NULL,
 873                                            (CamelObjectInitFunc) camel_key_file_init,
 874                                            (CamelObjectFinalizeFunc) camel_key_file_finalise);
 875         }
 876
 877         return type;
 878 }
 879
 880 /* 'use' a key file for io */
 881 static int
 882 key_file_use(CamelKeyFile *bs)
 883 {
 884         struct _CamelKeyFilePrivate *nw, *nn, *p = bs->priv;
 885         CamelKeyFile *bf;
 886         int err, fd;
 887         char *flag;
 888
 889         /* We want to:
 890             remove file from active list
 891             lock it
 892
 893            Then when done:
 894             unlock it
 895             add it back to end of active list
 896         */
 897
 898         /* TODO: Check header on reset? */
 899
 900         CAMEL_KEY_FILE_LOCK(bs, lock);
 901
 902         if (bs->fp != NULL)
 903                 return 0;
 904         else if (p->deleted) {
 905                 CAMEL_KEY_FILE_UNLOCK(bs, lock);
 906                 errno = ENOENT;
 907                 return -1;
 908         } else
 909                 d(printf("Turning key file online: '%s'\n", bs->path));
 910
 911         if ((bs->flags & O_ACCMODE) == O_RDONLY)
 912                 flag = "rb";
 913         else
 914                 flag = "a+b";
 915
 916         if ((fd = g_open(bs->path, bs->flags|O_BINARY, 0600)) == -1
 917             || (bs->fp = fdopen(fd, flag)) == NULL) {
 918                 err = errno;
 919                 if (fd != -1)
 920                         close(fd);
 921                 CAMEL_KEY_FILE_UNLOCK(bs, lock);
 922                 errno = err;
 923                 return -1;
 924         }
 925
 926         LOCK(key_file_lock);
 927         e_dlist_remove((EDListNode *)p);
 928         e_dlist_addtail(&key_file_active_list, (EDListNode *)p);
 929
 930         key_file_count++;
 931
 932         nw = (struct _CamelKeyFilePrivate *)key_file_list.head;
 933         nn = nw->next;
 934         while (key_file_count > key_file_threshhold && nn) {
 935                 /* We never hit the current keyfile here, as its removed from the list first */
 936                 bf = nw->base;
 937                 if (bf->fp != NULL) {
 938                         /* Need to trylock, as any of these lock levels might be trying
 939                            to lock the key_file_lock, so we need to check and abort if so */
 940                         if (CAMEL_BLOCK_FILE_TRYLOCK(bf, lock) == 0) {
 941                                 d(printf("Turning key file offline: %s\n", bf->path));
 942                                 fclose(bf->fp);
 943                                 bf->fp = NULL;
 944                                 key_file_count--;
 945                                 CAMEL_BLOCK_FILE_UNLOCK(bf, lock);
 946                         }
 947                 }
 948                 nw = nn;
 949                 nn = nw->next;
 950         }
 951
 952         UNLOCK(key_file_lock);
 953
 954         return 0;
 955 }
 956
 957 static void
 958 key_file_unuse(CamelKeyFile *bs)
 959 {
 960         LOCK(key_file_lock);
 961         e_dlist_remove((EDListNode *)bs->priv);
 962         e_dlist_addtail(&key_file_list, (EDListNode *)bs->priv);
 963         UNLOCK(key_file_lock);
 964
 965         CAMEL_KEY_FILE_UNLOCK(bs, lock);
 966 }
 967
 968 /**
 969  * camel_key_file_new:
 970  * @path:
 971  * @flags: open flags
 972  * @version: Version string (header) of file.  Currently
 973  * written but not checked.
 974  *
 975  * Create a new key file.  A linked list of record blocks.
 976  *
 977  * Return value: A new key file, or NULL if the file could not
 978  * be opened/created/initialised.
 979  **/
 980 CamelKeyFile *
 981 camel_key_file_new(const char *path, int flags, const char version[8])
 982 {
 983         CamelKeyFile *kf;
 984         off_t last;
 985         int err;
 986
 987         d(printf("New key file '%s'\n", path));
 988
 989         kf = (CamelKeyFile *)camel_object_new(camel_key_file_get_type());
 990         kf->path = g_strdup(path);
 991         kf->fp = NULL;
 992         kf->flags = flags;
 993         kf->last = 8;
 994
 995         if (key_file_use(kf) == -1) {
 996                 camel_object_unref((CamelObject *)kf);
 997                 kf = NULL;
 998         } else {
 999                 fseek(kf->fp, 0, SEEK_END);
1000                 last = ftell(kf->fp);
1001                 if (last == 0) {
1002                         fwrite(version, 8, 1, kf->fp);
1003                         last += 8;
1004                 }
1005                 kf->last = last;
1006
1007                 err = ferror(kf->fp);
1008                 key_file_unuse(kf);
1009
1010                 /* we only need these flags on first open */
1011                 kf->flags &= ~(O_CREAT|O_EXCL|O_TRUNC);
1012
1013                 if (err) {
1014                         camel_object_unref((CamelObject *)kf);
1015                         kf = NULL;
1016                 }
1017         }
1018
1019         return kf;
1020 }
1021
1022 int
1023 camel_key_file_rename(CamelKeyFile *kf, const char *path)
1024 {
1025         int ret;
1026         struct stat st;
1027         int err;
1028
1029         CAMEL_KEY_FILE_LOCK(kf, lock);
1030
1031         ret = g_rename(kf->path, path);
1032         if (ret == -1) {
1033                 /* Maybe the rename actually worked */
1034                 err = errno;
1035                 if (g_stat(path, &st) == 0
1036                     && g_stat(kf->path, &st) == -1
1037                     && errno == ENOENT)
1038                         ret = 0;
1039                 errno = err;
1040         }
1041
1042         if (ret != -1) {
1043                 g_free(kf->path);
1044                 kf->path = g_strdup(path);
1045         }
1046
1047         CAMEL_KEY_FILE_UNLOCK(kf, lock);
1048
1049         return ret;
1050 }
1051
1052 int
1053 camel_key_file_delete(CamelKeyFile *kf)
1054 {
1055         int ret;
1056         struct _CamelKeyFilePrivate *p = kf->priv;
1057
1058         CAMEL_KEY_FILE_LOCK(kf, lock);
1059
1060         if (kf->fp) {
1061                 LOCK(key_file_lock);
1062                 key_file_count--;
1063                 UNLOCK(key_file_lock);
1064                 fclose(kf->fp);
1065                 kf->fp = NULL;
1066         }
1067
1068         p->deleted = TRUE;
1069         ret = g_unlink(kf->path);
1070
1071         CAMEL_KEY_FILE_UNLOCK(kf, lock);
1072
1073         return ret;
1074
1075 }
1076
1077 /**
1078  * camel_key_file_write:
1079  * @kf:
1080  * @parent:
1081  * @len:
1082  * @records:
1083  *
1084  * Write a new list of records to the key file.
1085  *
1086  * Return value: -1 on io error.  The key file will remain unchanged.
1087  **/
1088 int
1089 camel_key_file_write(CamelKeyFile *kf, camel_block_t *parent, size_t len, camel_key_t *records)
1090 {
1091         camel_block_t next;
1092         guint32 size;
1093         int ret = -1;
1094
1095         d(printf("write key %08x len = %d\n", *parent, len));
1096
1097         if (len == 0) {
1098                 d(printf(" new parent = %08x\n", *parent));
1099                 return 0;
1100         }
1101
1102         /* LOCK */
1103         if (key_file_use(kf) == -1)
1104                 return -1;
1105
1106         size = len;
1107
1108         /* FIXME: Use io util functions? */
1109         next = kf->last;
1110         fseek(kf->fp, kf->last, SEEK_SET);
1111         fwrite(parent, sizeof(*parent), 1, kf->fp);
1112         fwrite(&size, sizeof(size), 1, kf->fp);
1113         fwrite(records, sizeof(records[0]), len, kf->fp);
1114
1115         if (ferror(kf->fp)) {
1116                 clearerr(kf->fp);
1117         } else {
1118                 kf->last = ftell(kf->fp);
1119                 *parent = next;
1120                 ret = len;
1121         }
1122
1123         /* UNLOCK */
1124         key_file_unuse(kf);
1125
1126         d(printf(" new parent = %08x\n", *parent));
1127
1128         return ret;
1129 }
1130
1131 /**
1132  * camel_key_file_read:
1133  * @kf:
1134  * @start: The record pointer.  This will be set to the next record pointer on success.
1135  * @len: Number of records read, if != NULL.
1136  * @records: Records, allocated, must be freed with g_free, if != NULL.
1137  *
1138  * Read the next block of data from the key file.  Returns the number of
1139  * records.
1140  *
1141  * Return value: -1 on io error.
1142  **/
1143 int
1144 camel_key_file_read(CamelKeyFile *kf, camel_block_t *start, size_t *len, camel_key_t **records)
1145 {
1146         guint32 size;
1147         long pos = *start;
1148         camel_block_t next;
1149         int ret = -1;
1150
1151         if (pos == 0)
1152                 return 0;
1153
1154         /* LOCK */
1155         if (key_file_use(kf) == -1)
1156                 return -1;
1157
1158         if (fseek(kf->fp, pos, SEEK_SET) == -1
1159             || fread(&next, sizeof(next), 1, kf->fp) != 1
1160             || fread(&size, sizeof(size), 1, kf->fp) != 1
1161             || size > 1024) {
1162                 clearerr(kf->fp);
1163                 goto fail;
1164         }
1165
1166         if (len)
1167                 *len = size;
1168
1169         if (records) {
1170                 camel_key_t *keys = g_malloc(size * sizeof(camel_key_t));
1171
1172                 if (fread(keys, sizeof(camel_key_t), size, kf->fp) != size) {
1173                         g_free(keys);
1174                         goto fail;
1175                 }
1176                 *records = keys;
1177         }
1178
1179         *start = next;
1180
1181         ret = 0;
1182 fail:
1183         /* UNLOCK */
1184         key_file_unuse(kf);
1185
1186         return ret;
1187 }