drivers/md/dm-cache-metadata.c

   1 /*
   2  * Copyright (C) 2012 Red Hat, Inc.
   3  *
   4  * This file is released under the GPL.
   5  */
   6
   7 #include "dm-cache-metadata.h"
   8
   9 #include "persistent-data/dm-array.h"
  10 #include "persistent-data/dm-bitset.h"
  11 #include "persistent-data/dm-space-map.h"
  12 #include "persistent-data/dm-space-map-disk.h"
  13 #include "persistent-data/dm-transaction-manager.h"
  14
  15 #include <linux/device-mapper.h>
  16
  17 /*----------------------------------------------------------------*/
  18
  19 #define DM_MSG_PREFIX   "cache metadata"
  20
  21 #define CACHE_SUPERBLOCK_MAGIC 06142003
  22 #define CACHE_SUPERBLOCK_LOCATION 0
  23 #define CACHE_VERSION 1
  24 #define CACHE_METADATA_CACHE_SIZE 64
  25
  26 /*
  27  *  3 for btree insert +
  28  *  2 for btree lookup used within space map
  29  */
  30 #define CACHE_MAX_CONCURRENT_LOCKS 5
  31 #define SPACE_MAP_ROOT_SIZE 128
  32
  33 enum superblock_flag_bits {
  34         /* for spotting crashes that would invalidate the dirty bitset */
  35         CLEAN_SHUTDOWN,
  36 };
  37
  38 /*
  39  * Each mapping from cache block -> origin block carries a set of flags.
  40  */
  41 enum mapping_bits {
  42         /*
  43          * A valid mapping.  Because we're using an array we clear this
  44          * flag for an non existant mapping.
  45          */
  46         M_VALID = 1,
  47
  48         /*
  49          * The data on the cache is different from that on the origin.
  50          */
  51         M_DIRTY = 2
  52 };
  53
  54 struct cache_disk_superblock {
  55         __le32 csum;
  56         __le32 flags;
  57         __le64 blocknr;
  58
  59         __u8 uuid[16];
  60         __le64 magic;
  61         __le32 version;
  62
  63         __u8 policy_name[CACHE_POLICY_NAME_SIZE];
  64         __le32 policy_hint_size;
  65
  66         __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  67         __le64 mapping_root;
  68         __le64 hint_root;
  69
  70         __le64 discard_root;
  71         __le64 discard_block_size;
  72         __le64 discard_nr_blocks;
  73
  74         __le32 data_block_size;
  75         __le32 metadata_block_size;
  76         __le32 cache_blocks;
  77
  78         __le32 compat_flags;
  79         __le32 compat_ro_flags;
  80         __le32 incompat_flags;
  81
  82         __le32 read_hits;
  83         __le32 read_misses;
  84         __le32 write_hits;
  85         __le32 write_misses;
  86 } __packed;
  87
  88 struct dm_cache_metadata {
  89         struct block_device *bdev;
  90         struct dm_block_manager *bm;
  91         struct dm_space_map *metadata_sm;
  92         struct dm_transaction_manager *tm;
  93
  94         struct dm_array_info info;
  95         struct dm_array_info hint_info;
  96         struct dm_disk_bitset discard_info;
  97
  98         struct rw_semaphore root_lock;
  99         dm_block_t root;
 100         dm_block_t hint_root;
 101         dm_block_t discard_root;
 102
 103         sector_t discard_block_size;
 104         dm_dblock_t discard_nr_blocks;
 105
 106         sector_t data_block_size;
 107         dm_cblock_t cache_blocks;
 108         bool changed:1;
 109         bool clean_when_opened:1;
 110
 111         char policy_name[CACHE_POLICY_NAME_SIZE];
 112         size_t policy_hint_size;
 113         struct dm_cache_statistics stats;
 114 };
 115
 116 /*-------------------------------------------------------------------
 117  * superblock validator
 118  *-----------------------------------------------------------------*/
 119
 120 #define SUPERBLOCK_CSUM_XOR 9031977
 121
 122 static void sb_prepare_for_write(struct dm_block_validator *v,
 123                                  struct dm_block *b,
 124                                  size_t sb_block_size)
 125 {
 126         struct cache_disk_superblock *disk_super = dm_block_data(b);
 127
 128         disk_super->blocknr = cpu_to_le64(dm_block_location(b));
 129         disk_super->csum = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
 130                                                       sb_block_size - sizeof(__le32),
 131                                                       SUPERBLOCK_CSUM_XOR));
 132 }
 133
 134 static int sb_check(struct dm_block_validator *v,
 135                     struct dm_block *b,
 136                     size_t sb_block_size)
 137 {
 138         struct cache_disk_superblock *disk_super = dm_block_data(b);
 139         __le32 csum_le;
 140
 141         if (dm_block_location(b) != le64_to_cpu(disk_super->blocknr)) {
 142                 DMERR("sb_check failed: blocknr %llu: wanted %llu",
 143                       le64_to_cpu(disk_super->blocknr),
 144                       (unsigned long long)dm_block_location(b));
 145                 return -ENOTBLK;
 146         }
 147
 148         if (le64_to_cpu(disk_super->magic) != CACHE_SUPERBLOCK_MAGIC) {
 149                 DMERR("sb_check failed: magic %llu: wanted %llu",
 150                       le64_to_cpu(disk_super->magic),
 151                       (unsigned long long)CACHE_SUPERBLOCK_MAGIC);
 152                 return -EILSEQ;
 153         }
 154
 155         csum_le = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
 156                                              sb_block_size - sizeof(__le32),
 157                                              SUPERBLOCK_CSUM_XOR));
 158         if (csum_le != disk_super->csum) {
 159                 DMERR("sb_check failed: csum %u: wanted %u",
 160                       le32_to_cpu(csum_le), le32_to_cpu(disk_super->csum));
 161                 return -EILSEQ;
 162         }
 163
 164         return 0;
 165 }
 166
 167 static struct dm_block_validator sb_validator = {
 168         .name = "superblock",
 169         .prepare_for_write = sb_prepare_for_write,
 170         .check = sb_check
 171 };
 172
 173 /*----------------------------------------------------------------*/
 174
 175 static int superblock_read_lock(struct dm_cache_metadata *cmd,
 176                                 struct dm_block **sblock)
 177 {
 178         return dm_bm_read_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 179                                &sb_validator, sblock);
 180 }
 181
 182 static int superblock_lock_zero(struct dm_cache_metadata *cmd,
 183                                 struct dm_block **sblock)
 184 {
 185         return dm_bm_write_lock_zero(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 186                                      &sb_validator, sblock);
 187 }
 188
 189 static int superblock_lock(struct dm_cache_metadata *cmd,
 190                            struct dm_block **sblock)
 191 {
 192         return dm_bm_write_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 193                                 &sb_validator, sblock);
 194 }
 195
 196 /*----------------------------------------------------------------*/
 197
 198 static int __superblock_all_zeroes(struct dm_block_manager *bm, int *result)
 199 {
 200         int r;
 201         unsigned i;
 202         struct dm_block *b;
 203         __le64 *data_le, zero = cpu_to_le64(0);
 204         unsigned sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);
 205
 206         /*
 207          * We can't use a validator here - it may be all zeroes.
 208          */
 209         r = dm_bm_read_lock(bm, CACHE_SUPERBLOCK_LOCATION, NULL, &b);
 210         if (r)
 211                 return r;
 212
 213         data_le = dm_block_data(b);
 214         *result = 1;
 215         for (i = 0; i < sb_block_size; i++) {
 216                 if (data_le[i] != zero) {
 217                         *result = 0;
 218                         break;
 219                 }
 220         }
 221
 222         return dm_bm_unlock(b);
 223 }
 224
 225 static void __setup_mapping_info(struct dm_cache_metadata *cmd)
 226 {
 227         struct dm_btree_value_type vt;
 228
 229         vt.context = NULL;
 230         vt.size = sizeof(__le64);
 231         vt.inc = NULL;
 232         vt.dec = NULL;
 233         vt.equal = NULL;
 234         dm_array_info_init(&cmd->info, cmd->tm, &vt);
 235
 236         if (cmd->policy_hint_size) {
 237                 vt.size = sizeof(__le32);
 238                 dm_array_info_init(&cmd->hint_info, cmd->tm, &vt);
 239         }
 240 }
 241
 242 static int __write_initial_superblock(struct dm_cache_metadata *cmd)
 243 {
 244         int r;
 245         struct dm_block *sblock;
 246         size_t metadata_len;
 247         struct cache_disk_superblock *disk_super;
 248         sector_t bdev_size = i_size_read(cmd->bdev->bd_inode) >> SECTOR_SHIFT;
 249
 250         /* FIXME: see if we can lose the max sectors limit */
 251         if (bdev_size > DM_CACHE_METADATA_MAX_SECTORS)
 252                 bdev_size = DM_CACHE_METADATA_MAX_SECTORS;
 253
 254         r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);
 255         if (r < 0)
 256                 return r;
 257
 258         r = dm_tm_pre_commit(cmd->tm);
 259         if (r < 0)
 260                 return r;
 261
 262         r = superblock_lock_zero(cmd, &sblock);
 263         if (r)
 264                 return r;
 265
 266         disk_super = dm_block_data(sblock);
 267         disk_super->flags = 0;
 268         memset(disk_super->uuid, 0, sizeof(disk_super->uuid));
 269         disk_super->magic = cpu_to_le64(CACHE_SUPERBLOCK_MAGIC);
 270         disk_super->version = cpu_to_le32(CACHE_VERSION);
 271         memset(disk_super->policy_name, 0, CACHE_POLICY_NAME_SIZE);
 272         disk_super->policy_hint_size = 0;
 273
 274         r = dm_sm_copy_root(cmd->metadata_sm, &disk_super->metadata_space_map_root,
 275                             metadata_len);
 276         if (r < 0)
 277                 goto bad_locked;
 278
 279         disk_super->mapping_root = cpu_to_le64(cmd->root);
 280         disk_super->hint_root = cpu_to_le64(cmd->hint_root);
 281         disk_super->discard_root = cpu_to_le64(cmd->discard_root);
 282         disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
 283         disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
 284         disk_super->metadata_block_size = cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE >> SECTOR_SHIFT);
 285         disk_super->data_block_size = cpu_to_le32(cmd->data_block_size);
 286         disk_super->cache_blocks = cpu_to_le32(0);
 287         memset(disk_super->policy_name, 0, sizeof(disk_super->policy_name));
 288
 289         disk_super->read_hits = cpu_to_le32(0);
 290         disk_super->read_misses = cpu_to_le32(0);
 291         disk_super->write_hits = cpu_to_le32(0);
 292         disk_super->write_misses = cpu_to_le32(0);
 293
 294         return dm_tm_commit(cmd->tm, sblock);
 295
 296 bad_locked:
 297         dm_bm_unlock(sblock);
 298         return r;
 299 }
 300
 301 static int __format_metadata(struct dm_cache_metadata *cmd)
 302 {
 303         int r;
 304
 305         r = dm_tm_create_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 306                                  &cmd->tm, &cmd->metadata_sm);
 307         if (r < 0) {
 308                 DMERR("tm_create_with_sm failed");
 309                 return r;
 310         }
 311
 312         __setup_mapping_info(cmd);
 313
 314         r = dm_array_empty(&cmd->info, &cmd->root);
 315         if (r < 0)
 316                 goto bad;
 317
 318         dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
 319
 320         r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root);
 321         if (r < 0)
 322                 goto bad;
 323
 324         cmd->discard_block_size = 0;
 325         cmd->discard_nr_blocks = 0;
 326
 327         r = __write_initial_superblock(cmd);
 328         if (r)
 329                 goto bad;
 330
 331         cmd->clean_when_opened = true;
 332         return 0;
 333
 334 bad:
 335         dm_tm_destroy(cmd->tm);
 336         dm_sm_destroy(cmd->metadata_sm);
 337
 338         return r;
 339 }
 340
 341 static int __check_incompat_features(struct cache_disk_superblock *disk_super,
 342                                      struct dm_cache_metadata *cmd)
 343 {
 344         uint32_t features;
 345
 346         features = le32_to_cpu(disk_super->incompat_flags) & ~DM_CACHE_FEATURE_INCOMPAT_SUPP;
 347         if (features) {
 348                 DMERR("could not access metadata due to unsupported optional features (%lx).",
 349                       (unsigned long)features);
 350                 return -EINVAL;
 351         }
 352
 353         /*
 354          * Check for read-only metadata to skip the following RDWR checks.
 355          */
 356         if (get_disk_ro(cmd->bdev->bd_disk))
 357                 return 0;
 358
 359         features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;
 360         if (features) {
 361                 DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",
 362                       (unsigned long)features);
 363                 return -EINVAL;
 364         }
 365
 366         return 0;
 367 }
 368
 369 static int __open_metadata(struct dm_cache_metadata *cmd)
 370 {
 371         int r;
 372         struct dm_block *sblock;
 373         struct cache_disk_superblock *disk_super;
 374         unsigned long sb_flags;
 375
 376         r = superblock_read_lock(cmd, &sblock);
 377         if (r < 0) {
 378                 DMERR("couldn't read lock superblock");
 379                 return r;
 380         }
 381
 382         disk_super = dm_block_data(sblock);
 383
 384         r = __check_incompat_features(disk_super, cmd);
 385         if (r < 0)
 386                 goto bad;
 387
 388         r = dm_tm_open_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 389                                disk_super->metadata_space_map_root,
 390                                sizeof(disk_super->metadata_space_map_root),
 391                                &cmd->tm, &cmd->metadata_sm);
 392         if (r < 0) {
 393                 DMERR("tm_open_with_sm failed");
 394                 goto bad;
 395         }
 396
 397         __setup_mapping_info(cmd);
 398         dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
 399         sb_flags = le32_to_cpu(disk_super->flags);
 400         cmd->clean_when_opened = test_bit(CLEAN_SHUTDOWN, &sb_flags);
 401         return dm_bm_unlock(sblock);
 402
 403 bad:
 404         dm_bm_unlock(sblock);
 405         return r;
 406 }
 407
 408 static int __open_or_format_metadata(struct dm_cache_metadata *cmd,
 409                                      bool format_device)
 410 {
 411         int r, unformatted;
 412
 413         r = __superblock_all_zeroes(cmd->bm, &unformatted);
 414         if (r)
 415                 return r;
 416
 417         if (unformatted)
 418                 return format_device ? __format_metadata(cmd) : -EPERM;
 419
 420         return __open_metadata(cmd);
 421 }
 422
 423 static int __create_persistent_data_objects(struct dm_cache_metadata *cmd,
 424                                             bool may_format_device)
 425 {
 426         int r;
 427         cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE,
 428                                           CACHE_METADATA_CACHE_SIZE,
 429                                           CACHE_MAX_CONCURRENT_LOCKS);
 430         if (IS_ERR(cmd->bm)) {
 431                 DMERR("could not create block manager");
 432                 return PTR_ERR(cmd->bm);
 433         }
 434
 435         r = __open_or_format_metadata(cmd, may_format_device);
 436         if (r)
 437                 dm_block_manager_destroy(cmd->bm);
 438
 439         return r;
 440 }
 441
 442 static void __destroy_persistent_data_objects(struct dm_cache_metadata *cmd)
 443 {
 444         dm_sm_destroy(cmd->metadata_sm);
 445         dm_tm_destroy(cmd->tm);
 446         dm_block_manager_destroy(cmd->bm);
 447 }
 448
 449 typedef unsigned long (*flags_mutator)(unsigned long);
 450
 451 static void update_flags(struct cache_disk_superblock *disk_super,
 452                          flags_mutator mutator)
 453 {
 454         uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags));
 455         disk_super->flags = cpu_to_le32(sb_flags);
 456 }
 457
 458 static unsigned long set_clean_shutdown(unsigned long flags)
 459 {
 460         set_bit(CLEAN_SHUTDOWN, &flags);
 461         return flags;
 462 }
 463
 464 static unsigned long clear_clean_shutdown(unsigned long flags)
 465 {
 466         clear_bit(CLEAN_SHUTDOWN, &flags);
 467         return flags;
 468 }
 469
 470 static void read_superblock_fields(struct dm_cache_metadata *cmd,
 471                                    struct cache_disk_superblock *disk_super)
 472 {
 473         cmd->root = le64_to_cpu(disk_super->mapping_root);
 474         cmd->hint_root = le64_to_cpu(disk_super->hint_root);
 475         cmd->discard_root = le64_to_cpu(disk_super->discard_root);
 476         cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size);
 477         cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks));
 478         cmd->data_block_size = le32_to_cpu(disk_super->data_block_size);
 479         cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks));
 480         strncpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name));
 481         cmd->policy_hint_size = le32_to_cpu(disk_super->policy_hint_size);
 482
 483         cmd->stats.read_hits = le32_to_cpu(disk_super->read_hits);
 484         cmd->stats.read_misses = le32_to_cpu(disk_super->read_misses);
 485         cmd->stats.write_hits = le32_to_cpu(disk_super->write_hits);
 486         cmd->stats.write_misses = le32_to_cpu(disk_super->write_misses);
 487
 488         cmd->changed = false;
 489 }
 490
 491 /*
 492  * The mutator updates the superblock flags.
 493  */
 494 static int __begin_transaction_flags(struct dm_cache_metadata *cmd,
 495                                      flags_mutator mutator)
 496 {
 497         int r;
 498         struct cache_disk_superblock *disk_super;
 499         struct dm_block *sblock;
 500
 501         r = superblock_lock(cmd, &sblock);
 502         if (r)
 503                 return r;
 504
 505         disk_super = dm_block_data(sblock);
 506         update_flags(disk_super, mutator);
 507         read_superblock_fields(cmd, disk_super);
 508
 509         return dm_bm_flush_and_unlock(cmd->bm, sblock);
 510 }
 511
 512 static int __begin_transaction(struct dm_cache_metadata *cmd)
 513 {
 514         int r;
 515         struct cache_disk_superblock *disk_super;
 516         struct dm_block *sblock;
 517
 518         /*
 519          * We re-read the superblock every time.  Shouldn't need to do this
 520          * really.
 521          */
 522         r = superblock_read_lock(cmd, &sblock);
 523         if (r)
 524                 return r;
 525
 526         disk_super = dm_block_data(sblock);
 527         read_superblock_fields(cmd, disk_super);
 528         dm_bm_unlock(sblock);
 529
 530         return 0;
 531 }
 532
 533 static int __commit_transaction(struct dm_cache_metadata *cmd,
 534                                 flags_mutator mutator)
 535 {
 536         int r;
 537         size_t metadata_len;
 538         struct cache_disk_superblock *disk_super;
 539         struct dm_block *sblock;
 540
 541         /*
 542          * We need to know if the cache_disk_superblock exceeds a 512-byte sector.
 543          */
 544         BUILD_BUG_ON(sizeof(struct cache_disk_superblock) > 512);
 545
 546         r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root,
 547                             &cmd->discard_root);
 548         if (r)
 549                 return r;
 550
 551         r = dm_tm_pre_commit(cmd->tm);
 552         if (r < 0)
 553                 return r;
 554
 555         r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);
 556         if (r < 0)
 557                 return r;
 558
 559         r = superblock_lock(cmd, &sblock);
 560         if (r)
 561                 return r;
 562
 563         disk_super = dm_block_data(sblock);
 564
 565         if (mutator)
 566                 update_flags(disk_super, mutator);
 567
 568         disk_super->mapping_root = cpu_to_le64(cmd->root);
 569         disk_super->hint_root = cpu_to_le64(cmd->hint_root);
 570         disk_super->discard_root = cpu_to_le64(cmd->discard_root);
 571         disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
 572         disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
 573         disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks));
 574         strncpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name));
 575
 576         disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits);
 577         disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses);
 578         disk_super->write_hits = cpu_to_le32(cmd->stats.write_hits);
 579         disk_super->write_misses = cpu_to_le32(cmd->stats.write_misses);
 580
 581         r = dm_sm_copy_root(cmd->metadata_sm, &disk_super->metadata_space_map_root,
 582                             metadata_len);
 583         if (r < 0) {
 584                 dm_bm_unlock(sblock);
 585                 return r;
 586         }
 587
 588         return dm_tm_commit(cmd->tm, sblock);
 589 }
 590
 591 /*----------------------------------------------------------------*/
 592
 593 /*
 594  * The mappings are held in a dm-array that has 64-bit values stored in
 595  * little-endian format.  The index is the cblock, the high 48bits of the
 596  * value are the oblock and the low 16 bit the flags.
 597  */
 598 #define FLAGS_MASK ((1 << 16) - 1)
 599
 600 static __le64 pack_value(dm_oblock_t block, unsigned flags)
 601 {
 602         uint64_t value = from_oblock(block);
 603         value <<= 16;
 604         value = value | (flags & FLAGS_MASK);
 605         return cpu_to_le64(value);
 606 }
 607
 608 static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned *flags)
 609 {
 610         uint64_t value = le64_to_cpu(value_le);
 611         uint64_t b = value >> 16;
 612         *block = to_oblock(b);
 613         *flags = value & FLAGS_MASK;
 614 }
 615
 616 /*----------------------------------------------------------------*/
 617
 618 struct dm_cache_metadata *dm_cache_metadata_open(struct block_device *bdev,
 619                                                  sector_t data_block_size,
 620                                                  bool may_format_device,
 621                                                  size_t policy_hint_size)
 622 {
 623         int r;
 624         struct dm_cache_metadata *cmd;
 625
 626         cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 627         if (!cmd) {
 628                 DMERR("could not allocate metadata struct");
 629                 return NULL;
 630         }
 631
 632         init_rwsem(&cmd->root_lock);
 633         cmd->bdev = bdev;
 634         cmd->data_block_size = data_block_size;
 635         cmd->cache_blocks = 0;
 636         cmd->policy_hint_size = policy_hint_size;
 637         cmd->changed = true;
 638
 639         r = __create_persistent_data_objects(cmd, may_format_device);
 640         if (r) {
 641                 kfree(cmd);
 642                 return ERR_PTR(r);
 643         }
 644
 645         r = __begin_transaction_flags(cmd, clear_clean_shutdown);
 646         if (r < 0) {
 647                 dm_cache_metadata_close(cmd);
 648                 return ERR_PTR(r);
 649         }
 650
 651         return cmd;
 652 }
 653
 654 void dm_cache_metadata_close(struct dm_cache_metadata *cmd)
 655 {
 656         __destroy_persistent_data_objects(cmd);
 657         kfree(cmd);
 658 }
 659
 660 int dm_cache_resize(struct dm_cache_metadata *cmd, dm_cblock_t new_cache_size)
 661 {
 662         int r;
 663         __le64 null_mapping = pack_value(0, 0);
 664
 665         down_write(&cmd->root_lock);
 666         __dm_bless_for_disk(&null_mapping);
 667         r = dm_array_resize(&cmd->info, cmd->root, from_cblock(cmd->cache_blocks),
 668                             from_cblock(new_cache_size),
 669                             &null_mapping, &cmd->root);
 670         if (!r)
 671                 cmd->cache_blocks = new_cache_size;
 672         cmd->changed = true;
 673         up_write(&cmd->root_lock);
 674
 675         return r;
 676 }
 677
 678 int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,
 679                                    sector_t discard_block_size,
 680                                    dm_dblock_t new_nr_entries)
 681 {
 682         int r;
 683
 684         down_write(&cmd->root_lock);
 685         r = dm_bitset_resize(&cmd->discard_info,
 686                              cmd->discard_root,
 687                              from_dblock(cmd->discard_nr_blocks),
 688                              from_dblock(new_nr_entries),
 689                              false, &cmd->discard_root);
 690         if (!r) {
 691                 cmd->discard_block_size = discard_block_size;
 692                 cmd->discard_nr_blocks = new_nr_entries;
 693         }
 694
 695         cmd->changed = true;
 696         up_write(&cmd->root_lock);
 697
 698         return r;
 699 }
 700
 701 static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
 702 {
 703         return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root,
 704                                  from_dblock(b), &cmd->discard_root);
 705 }
 706
 707 static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
 708 {
 709         return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root,
 710                                    from_dblock(b), &cmd->discard_root);
 711 }
 712
 713 static int __is_discarded(struct dm_cache_metadata *cmd, dm_dblock_t b,
 714                           bool *is_discarded)
 715 {
 716         return dm_bitset_test_bit(&cmd->discard_info, cmd->discard_root,
 717                                   from_dblock(b), &cmd->discard_root,
 718                                   is_discarded);
 719 }
 720
 721 static int __discard(struct dm_cache_metadata *cmd,
 722                      dm_dblock_t dblock, bool discard)
 723 {
 724         int r;
 725
 726         r = (discard ? __set_discard : __clear_discard)(cmd, dblock);
 727         if (r)
 728                 return r;
 729
 730         cmd->changed = true;
 731         return 0;
 732 }
 733
 734 int dm_cache_set_discard(struct dm_cache_metadata *cmd,
 735                          dm_dblock_t dblock, bool discard)
 736 {
 737         int r;
 738
 739         down_write(&cmd->root_lock);
 740         r = __discard(cmd, dblock, discard);
 741         up_write(&cmd->root_lock);
 742
 743         return r;
 744 }
 745
 746 static int __load_discards(struct dm_cache_metadata *cmd,
 747                            load_discard_fn fn, void *context)
 748 {
 749         int r = 0;
 750         dm_block_t b;
 751         bool discard;
 752
 753         for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) {
 754                 dm_dblock_t dblock = to_dblock(b);
 755
 756                 if (cmd->clean_when_opened) {
 757                         r = __is_discarded(cmd, dblock, &discard);
 758                         if (r)
 759                                 return r;
 760                 } else
 761                         discard = false;
 762
 763                 r = fn(context, cmd->discard_block_size, dblock, discard);
 764                 if (r)
 765                         break;
 766         }
 767
 768         return r;
 769 }
 770
 771 int dm_cache_load_discards(struct dm_cache_metadata *cmd,
 772                            load_discard_fn fn, void *context)
 773 {
 774         int r;
 775
 776         down_read(&cmd->root_lock);
 777         r = __load_discards(cmd, fn, context);
 778         up_read(&cmd->root_lock);
 779
 780         return r;
 781 }
 782
 783 dm_cblock_t dm_cache_size(struct dm_cache_metadata *cmd)
 784 {
 785         dm_cblock_t r;
 786
 787         down_read(&cmd->root_lock);
 788         r = cmd->cache_blocks;
 789         up_read(&cmd->root_lock);
 790
 791         return r;
 792 }
 793
 794 static int __remove(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
 795 {
 796         int r;
 797         __le64 value = pack_value(0, 0);
 798
 799         __dm_bless_for_disk(&value);
 800         r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
 801                                &value, &cmd->root);
 802         if (r)
 803                 return r;
 804
 805         cmd->changed = true;
 806         return 0;
 807 }
 808
 809 int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
 810 {
 811         int r;
 812
 813         down_write(&cmd->root_lock);
 814         r = __remove(cmd, cblock);
 815         up_write(&cmd->root_lock);
 816
 817         return r;
 818 }
 819
 820 static int __insert(struct dm_cache_metadata *cmd,
 821                     dm_cblock_t cblock, dm_oblock_t oblock)
 822 {
 823         int r;
 824         __le64 value = pack_value(oblock, M_VALID);
 825         __dm_bless_for_disk(&value);
 826
 827         r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
 828                                &value, &cmd->root);
 829         if (r)
 830                 return r;
 831
 832         cmd->changed = true;
 833         return 0;
 834 }
 835
 836 int dm_cache_insert_mapping(struct dm_cache_metadata *cmd,
 837                             dm_cblock_t cblock, dm_oblock_t oblock)
 838 {
 839         int r;
 840
 841         down_write(&cmd->root_lock);
 842         r = __insert(cmd, cblock, oblock);
 843         up_write(&cmd->root_lock);
 844
 845         return r;
 846 }
 847
 848 struct thunk {
 849         load_mapping_fn fn;
 850         void *context;
 851
 852         struct dm_cache_metadata *cmd;
 853         bool respect_dirty_flags;
 854         bool hints_valid;
 855 };
 856
 857 static bool hints_array_initialized(struct dm_cache_metadata *cmd)
 858 {
 859         return cmd->hint_root && cmd->policy_hint_size;
 860 }
 861
 862 static bool hints_array_available(struct dm_cache_metadata *cmd,
 863                                   const char *policy_name)
 864 {
 865         bool policy_names_match = !strncmp(cmd->policy_name, policy_name,
 866                                            sizeof(cmd->policy_name));
 867
 868         return cmd->clean_when_opened && policy_names_match &&
 869                 hints_array_initialized(cmd);
 870 }
 871
 872 static int __load_mapping(void *context, uint64_t cblock, void *leaf)
 873 {
 874         int r = 0;
 875         bool dirty;
 876         __le64 value;
 877         __le32 hint_value = 0;
 878         dm_oblock_t oblock;
 879         unsigned flags;
 880         struct thunk *thunk = context;
 881         struct dm_cache_metadata *cmd = thunk->cmd;
 882
 883         memcpy(&value, leaf, sizeof(value));
 884         unpack_value(value, &oblock, &flags);
 885
 886         if (flags & M_VALID) {
 887                 if (thunk->hints_valid) {
 888                         r = dm_array_get_value(&cmd->hint_info, cmd->hint_root,
 889                                                cblock, &hint_value);
 890                         if (r && r != -ENODATA)
 891                                 return r;
 892                 }
 893
 894                 dirty = thunk->respect_dirty_flags ? (flags & M_DIRTY) : true;
 895                 r = thunk->fn(thunk->context, oblock, to_cblock(cblock),
 896                               dirty, le32_to_cpu(hint_value), thunk->hints_valid);
 897         }
 898
 899         return r;
 900 }
 901
 902 static int __load_mappings(struct dm_cache_metadata *cmd, const char *policy_name,
 903                            load_mapping_fn fn, void *context)
 904 {
 905         struct thunk thunk;
 906
 907         thunk.fn = fn;
 908         thunk.context = context;
 909
 910         thunk.cmd = cmd;
 911         thunk.respect_dirty_flags = cmd->clean_when_opened;
 912         thunk.hints_valid = hints_array_available(cmd, policy_name);
 913
 914         return dm_array_walk(&cmd->info, cmd->root, __load_mapping, &thunk);
 915 }
 916
 917 int dm_cache_load_mappings(struct dm_cache_metadata *cmd, const char *policy_name,
 918                            load_mapping_fn fn, void *context)
 919 {
 920         int r;
 921
 922         down_read(&cmd->root_lock);
 923         r = __load_mappings(cmd, policy_name, fn, context);
 924         up_read(&cmd->root_lock);
 925
 926         return r;
 927 }
 928
 929 static int __dump_mapping(void *context, uint64_t cblock, void *leaf)
 930 {
 931         int r = 0;
 932         __le64 value;
 933         dm_oblock_t oblock;
 934         unsigned flags;
 935
 936         memcpy(&value, leaf, sizeof(value));
 937         unpack_value(value, &oblock, &flags);
 938
 939         return r;
 940 }
 941
 942 static int __dump_mappings(struct dm_cache_metadata *cmd)
 943 {
 944         return dm_array_walk(&cmd->info, cmd->root, __dump_mapping, NULL);
 945 }
 946
 947 void dm_cache_dump(struct dm_cache_metadata *cmd)
 948 {
 949         down_read(&cmd->root_lock);
 950         __dump_mappings(cmd);
 951         up_read(&cmd->root_lock);
 952 }
 953
 954 int dm_cache_changed_this_transaction(struct dm_cache_metadata *cmd)
 955 {
 956         int r;
 957
 958         down_read(&cmd->root_lock);
 959         r = cmd->changed;
 960         up_read(&cmd->root_lock);
 961
 962         return r;
 963 }
 964
 965 static int __dirty(struct dm_cache_metadata *cmd, dm_cblock_t cblock, bool dirty)
 966 {
 967         int r;
 968         unsigned flags;
 969         dm_oblock_t oblock;
 970         __le64 value;
 971
 972         r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(cblock), &value);
 973         if (r)
 974                 return r;
 975
 976         unpack_value(value, &oblock, &flags);
 977
 978         if (((flags & M_DIRTY) && dirty) || (!(flags & M_DIRTY) && !dirty))
 979                 /* nothing to be done */
 980                 return 0;
 981
 982         value = pack_value(oblock, (flags & ~M_DIRTY) | (dirty ? M_DIRTY : 0));
 983         __dm_bless_for_disk(&value);
 984
 985         r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
 986                                &value, &cmd->root);
 987         if (r)
 988                 return r;
 989
 990         cmd->changed = true;
 991         return 0;
 992
 993 }
 994
 995 int dm_cache_set_dirty(struct dm_cache_metadata *cmd,
 996                        dm_cblock_t cblock, bool dirty)
 997 {
 998         int r;
 999
1000         down_write(&cmd->root_lock);
1001         r = __dirty(cmd, cblock, dirty);
1002         up_write(&cmd->root_lock);
1003
1004         return r;
1005 }
1006
1007 void dm_cache_metadata_get_stats(struct dm_cache_metadata *cmd,
1008                                  struct dm_cache_statistics *stats)
1009 {
1010         down_read(&cmd->root_lock);
1011         memcpy(stats, &cmd->stats, sizeof(*stats));
1012         up_read(&cmd->root_lock);
1013 }
1014
1015 void dm_cache_metadata_set_stats(struct dm_cache_metadata *cmd,
1016                                  struct dm_cache_statistics *stats)
1017 {
1018         down_write(&cmd->root_lock);
1019         memcpy(&cmd->stats, stats, sizeof(*stats));
1020         up_write(&cmd->root_lock);
1021 }
1022
1023 int dm_cache_commit(struct dm_cache_metadata *cmd, bool clean_shutdown)
1024 {
1025         int r;
1026         flags_mutator mutator = (clean_shutdown ? set_clean_shutdown :
1027                                  clear_clean_shutdown);
1028
1029         down_write(&cmd->root_lock);
1030         r = __commit_transaction(cmd, mutator);
1031         if (r)
1032                 goto out;
1033
1034         r = __begin_transaction(cmd);
1035
1036 out:
1037         up_write(&cmd->root_lock);
1038         return r;
1039 }
1040
1041 int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata *cmd,
1042                                            dm_block_t *result)
1043 {
1044         int r = -EINVAL;
1045
1046         down_read(&cmd->root_lock);
1047         r = dm_sm_get_nr_free(cmd->metadata_sm, result);
1048         up_read(&cmd->root_lock);
1049
1050         return r;
1051 }
1052
1053 int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,
1054                                    dm_block_t *result)
1055 {
1056         int r = -EINVAL;
1057
1058         down_read(&cmd->root_lock);
1059         r = dm_sm_get_nr_blocks(cmd->metadata_sm, result);
1060         up_read(&cmd->root_lock);
1061
1062         return r;
1063 }
1064
1065 /*----------------------------------------------------------------*/
1066
1067 static int begin_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1068 {
1069         int r;
1070         __le32 value;
1071         size_t hint_size;
1072         const char *policy_name = dm_cache_policy_get_name(policy);
1073
1074         if (!policy_name[0] ||
1075             (strlen(policy_name) > sizeof(cmd->policy_name) - 1))
1076                 return -EINVAL;
1077
1078         if (strcmp(cmd->policy_name, policy_name)) {
1079                 strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));
1080
1081                 hint_size = dm_cache_policy_get_hint_size(policy);
1082                 if (!hint_size)
1083                         return 0; /* short-circuit hints initialization */
1084                 cmd->policy_hint_size = hint_size;
1085
1086                 if (cmd->hint_root) {
1087                         r = dm_array_del(&cmd->hint_info, cmd->hint_root);
1088                         if (r)
1089                                 return r;
1090                 }
1091
1092                 r = dm_array_empty(&cmd->hint_info, &cmd->hint_root);
1093                 if (r)
1094                         return r;
1095
1096                 value = cpu_to_le32(0);
1097                 __dm_bless_for_disk(&value);
1098                 r = dm_array_resize(&cmd->hint_info, cmd->hint_root, 0,
1099                                     from_cblock(cmd->cache_blocks),
1100                                     &value, &cmd->hint_root);
1101                 if (r)
1102                         return r;
1103         }
1104
1105         return 0;
1106 }
1107
1108 int dm_cache_begin_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1109 {
1110         int r;
1111
1112         down_write(&cmd->root_lock);
1113         r = begin_hints(cmd, policy);
1114         up_write(&cmd->root_lock);
1115
1116         return r;
1117 }
1118
1119 static int save_hint(struct dm_cache_metadata *cmd, dm_cblock_t cblock,
1120                      uint32_t hint)
1121 {
1122         int r;
1123         __le32 value = cpu_to_le32(hint);
1124         __dm_bless_for_disk(&value);
1125
1126         r = dm_array_set_value(&cmd->hint_info, cmd->hint_root,
1127                                from_cblock(cblock), &value, &cmd->hint_root);
1128         cmd->changed = true;
1129
1130         return r;
1131 }
1132
1133 int dm_cache_save_hint(struct dm_cache_metadata *cmd, dm_cblock_t cblock,
1134                        uint32_t hint)
1135 {
1136         int r;
1137
1138         if (!hints_array_initialized(cmd))
1139                 return 0;
1140
1141         down_write(&cmd->root_lock);
1142         r = save_hint(cmd, cblock, hint);
1143         up_write(&cmd->root_lock);
1144
1145         return r;
1146 }