drivers/md/dm-cache-target.c

   1 /*
   2  * Copyright (C) 2012 Red Hat. All rights reserved.
   3  *
   4  * This file is released under the GPL.
   5  */
   6
   7 #include "dm.h"
   8 #include "dm-bio-prison.h"
   9 #include "dm-bio-record.h"
  10 #include "dm-cache-metadata.h"
  11
  12 #include <linux/dm-io.h>
  13 #include <linux/dm-kcopyd.h>
  14 #include <linux/init.h>
  15 #include <linux/mempool.h>
  16 #include <linux/module.h>
  17 #include <linux/slab.h>
  18 #include <linux/vmalloc.h>
  19
  20 #define DM_MSG_PREFIX "cache"
  21
  22 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle,
  23         "A percentage of time allocated for copying to and/or from cache");
  24
  25 /*----------------------------------------------------------------*/
  26
  27 /*
  28  * Glossary:
  29  *
  30  * oblock: index of an origin block
  31  * cblock: index of a cache block
  32  * promotion: movement of a block from origin to cache
  33  * demotion: movement of a block from cache to origin
  34  * migration: movement of a block between the origin and cache device,
  35  *            either direction
  36  */
  37
  38 /*----------------------------------------------------------------*/
  39
  40 static size_t bitset_size_in_bytes(unsigned nr_entries)
  41 {
  42         return sizeof(unsigned long) * dm_div_up(nr_entries, BITS_PER_LONG);
  43 }
  44
  45 static unsigned long *alloc_bitset(unsigned nr_entries)
  46 {
  47         size_t s = bitset_size_in_bytes(nr_entries);
  48         return vzalloc(s);
  49 }
  50
  51 static void clear_bitset(void *bitset, unsigned nr_entries)
  52 {
  53         size_t s = bitset_size_in_bytes(nr_entries);
  54         memset(bitset, 0, s);
  55 }
  56
  57 static void free_bitset(unsigned long *bits)
  58 {
  59         vfree(bits);
  60 }
  61
  62 /*----------------------------------------------------------------*/
  63
  64 #define PRISON_CELLS 1024
  65 #define MIGRATION_POOL_SIZE 128
  66 #define COMMIT_PERIOD HZ
  67 #define MIGRATION_COUNT_WINDOW 10
  68
  69 /*
  70  * The block size of the device holding cache data must be
  71  * between 32KB and 1GB.
  72  */
  73 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT)
  74 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
  75
  76 /*
  77  * FIXME: the cache is read/write for the time being.
  78  */
  79 enum cache_mode {
  80         CM_WRITE,               /* metadata may be changed */
  81         CM_READ_ONLY,           /* metadata may not be changed */
  82 };
  83
  84 struct cache_features {
  85         enum cache_mode mode;
  86         bool write_through:1;
  87 };
  88
  89 struct cache_stats {
  90         atomic_t read_hit;
  91         atomic_t read_miss;
  92         atomic_t write_hit;
  93         atomic_t write_miss;
  94         atomic_t demotion;
  95         atomic_t promotion;
  96         atomic_t copies_avoided;
  97         atomic_t cache_cell_clash;
  98         atomic_t commit_count;
  99         atomic_t discard_count;
 100 };
 101
 102 struct cache {
 103         struct dm_target *ti;
 104         struct dm_target_callbacks callbacks;
 105
 106         struct dm_cache_metadata *cmd;
 107
 108         /*
 109          * Metadata is written to this device.
 110          */
 111         struct dm_dev *metadata_dev;
 112
 113         /*
 114          * The slower of the two data devices.  Typically a spindle.
 115          */
 116         struct dm_dev *origin_dev;
 117
 118         /*
 119          * The faster of the two data devices.  Typically an SSD.
 120          */
 121         struct dm_dev *cache_dev;
 122
 123         /*
 124          * Size of the origin device in _complete_ blocks and native sectors.
 125          */
 126         dm_oblock_t origin_blocks;
 127         sector_t origin_sectors;
 128
 129         /*
 130          * Size of the cache device in blocks.
 131          */
 132         dm_cblock_t cache_size;
 133
 134         /*
 135          * Fields for converting from sectors to blocks.
 136          */
 137         uint32_t sectors_per_block;
 138         int sectors_per_block_shift;
 139
 140         spinlock_t lock;
 141         struct bio_list deferred_bios;
 142         struct bio_list deferred_flush_bios;
 143         struct bio_list deferred_writethrough_bios;
 144         struct list_head quiesced_migrations;
 145         struct list_head completed_migrations;
 146         struct list_head need_commit_migrations;
 147         sector_t migration_threshold;
 148         wait_queue_head_t migration_wait;
 149         atomic_t nr_migrations;
 150
 151         wait_queue_head_t quiescing_wait;
 152         atomic_t quiescing;
 153         atomic_t quiescing_ack;
 154
 155         /*
 156          * cache_size entries, dirty if set
 157          */
 158         dm_cblock_t nr_dirty;
 159         unsigned long *dirty_bitset;
 160
 161         /*
 162          * origin_blocks entries, discarded if set.
 163          */
 164         dm_dblock_t discard_nr_blocks;
 165         unsigned long *discard_bitset;
 166         uint32_t discard_block_size; /* a power of 2 times sectors per block */
 167
 168         /*
 169          * Rather than reconstructing the table line for the status we just
 170          * save it and regurgitate.
 171          */
 172         unsigned nr_ctr_args;
 173         const char **ctr_args;
 174
 175         struct dm_kcopyd_client *copier;
 176         struct workqueue_struct *wq;
 177         struct work_struct worker;
 178
 179         struct delayed_work waker;
 180         unsigned long last_commit_jiffies;
 181
 182         struct dm_bio_prison *prison;
 183         struct dm_deferred_set *all_io_ds;
 184
 185         mempool_t *migration_pool;
 186         struct dm_cache_migration *next_migration;
 187
 188         struct dm_cache_policy *policy;
 189         unsigned policy_nr_args;
 190
 191         bool need_tick_bio:1;
 192         bool sized:1;
 193         bool commit_requested:1;
 194         bool loaded_mappings:1;
 195         bool loaded_discards:1;
 196
 197         /*
 198          * Cache features such as write-through.
 199          */
 200         struct cache_features features;
 201
 202         struct cache_stats stats;
 203 };
 204
 205 struct per_bio_data {
 206         bool tick:1;
 207         unsigned req_nr:2;
 208         struct dm_deferred_entry *all_io_entry;
 209
 210         /*
 211          * writethrough fields.  These MUST remain at the end of this
 212          * structure and the 'cache' member must be the first as it
 213          * is used to determine the offset of the writethrough fields.
 214          */
 215         struct cache *cache;
 216         dm_cblock_t cblock;
 217         bio_end_io_t *saved_bi_end_io;
 218         struct dm_bio_details bio_details;
 219 };
 220
 221 struct dm_cache_migration {
 222         struct list_head list;
 223         struct cache *cache;
 224
 225         unsigned long start_jiffies;
 226         dm_oblock_t old_oblock;
 227         dm_oblock_t new_oblock;
 228         dm_cblock_t cblock;
 229
 230         bool err:1;
 231         bool writeback:1;
 232         bool demote:1;
 233         bool promote:1;
 234
 235         struct dm_bio_prison_cell *old_ocell;
 236         struct dm_bio_prison_cell *new_ocell;
 237 };
 238
 239 /*
 240  * Processing a bio in the worker thread may require these memory
 241  * allocations.  We prealloc to avoid deadlocks (the same worker thread
 242  * frees them back to the mempool).
 243  */
 244 struct prealloc {
 245         struct dm_cache_migration *mg;
 246         struct dm_bio_prison_cell *cell1;
 247         struct dm_bio_prison_cell *cell2;
 248 };
 249
 250 static void wake_worker(struct cache *cache)
 251 {
 252         queue_work(cache->wq, &cache->worker);
 253 }
 254
 255 /*----------------------------------------------------------------*/
 256
 257 static struct dm_bio_prison_cell *alloc_prison_cell(struct cache *cache)
 258 {
 259         /* FIXME: change to use a local slab. */
 260         return dm_bio_prison_alloc_cell(cache->prison, GFP_NOWAIT);
 261 }
 262
 263 static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell *cell)
 264 {
 265         dm_bio_prison_free_cell(cache->prison, cell);
 266 }
 267
 268 static int prealloc_data_structs(struct cache *cache, struct prealloc *p)
 269 {
 270         if (!p->mg) {
 271                 p->mg = mempool_alloc(cache->migration_pool, GFP_NOWAIT);
 272                 if (!p->mg)
 273                         return -ENOMEM;
 274         }
 275
 276         if (!p->cell1) {
 277                 p->cell1 = alloc_prison_cell(cache);
 278                 if (!p->cell1)
 279                         return -ENOMEM;
 280         }
 281
 282         if (!p->cell2) {
 283                 p->cell2 = alloc_prison_cell(cache);
 284                 if (!p->cell2)
 285                         return -ENOMEM;
 286         }
 287
 288         return 0;
 289 }
 290
 291 static void prealloc_free_structs(struct cache *cache, struct prealloc *p)
 292 {
 293         if (p->cell2)
 294                 free_prison_cell(cache, p->cell2);
 295
 296         if (p->cell1)
 297                 free_prison_cell(cache, p->cell1);
 298
 299         if (p->mg)
 300                 mempool_free(p->mg, cache->migration_pool);
 301 }
 302
 303 static struct dm_cache_migration *prealloc_get_migration(struct prealloc *p)
 304 {
 305         struct dm_cache_migration *mg = p->mg;
 306
 307         BUG_ON(!mg);
 308         p->mg = NULL;
 309
 310         return mg;
 311 }
 312
 313 /*
 314  * You must have a cell within the prealloc struct to return.  If not this
 315  * function will BUG() rather than returning NULL.
 316  */
 317 static struct dm_bio_prison_cell *prealloc_get_cell(struct prealloc *p)
 318 {
 319         struct dm_bio_prison_cell *r = NULL;
 320
 321         if (p->cell1) {
 322                 r = p->cell1;
 323                 p->cell1 = NULL;
 324
 325         } else if (p->cell2) {
 326                 r = p->cell2;
 327                 p->cell2 = NULL;
 328         } else
 329                 BUG();
 330
 331         return r;
 332 }
 333
 334 /*
 335  * You can't have more than two cells in a prealloc struct.  BUG() will be
 336  * called if you try and overfill.
 337  */
 338 static void prealloc_put_cell(struct prealloc *p, struct dm_bio_prison_cell *cell)
 339 {
 340         if (!p->cell2)
 341                 p->cell2 = cell;
 342
 343         else if (!p->cell1)
 344                 p->cell1 = cell;
 345
 346         else
 347                 BUG();
 348 }
 349
 350 /*----------------------------------------------------------------*/
 351
 352 static void build_key(dm_oblock_t oblock, struct dm_cell_key *key)
 353 {
 354         key->virtual = 0;
 355         key->dev = 0;
 356         key->block = from_oblock(oblock);
 357 }
 358
 359 /*
 360  * The caller hands in a preallocated cell, and a free function for it.
 361  * The cell will be freed if there's an error, or if it wasn't used because
 362  * a cell with that key already exists.
 363  */
 364 typedef void (*cell_free_fn)(void *context, struct dm_bio_prison_cell *cell);
 365
 366 static int bio_detain(struct cache *cache, dm_oblock_t oblock,
 367                       struct bio *bio, struct dm_bio_prison_cell *cell_prealloc,
 368                       cell_free_fn free_fn, void *free_context,
 369                       struct dm_bio_prison_cell **cell_result)
 370 {
 371         int r;
 372         struct dm_cell_key key;
 373
 374         build_key(oblock, &key);
 375         r = dm_bio_detain(cache->prison, &key, bio, cell_prealloc, cell_result);
 376         if (r)
 377                 free_fn(free_context, cell_prealloc);
 378
 379         return r;
 380 }
 381
 382 static int get_cell(struct cache *cache,
 383                     dm_oblock_t oblock,
 384                     struct prealloc *structs,
 385                     struct dm_bio_prison_cell **cell_result)
 386 {
 387         int r;
 388         struct dm_cell_key key;
 389         struct dm_bio_prison_cell *cell_prealloc;
 390
 391         cell_prealloc = prealloc_get_cell(structs);
 392
 393         build_key(oblock, &key);
 394         r = dm_get_cell(cache->prison, &key, cell_prealloc, cell_result);
 395         if (r)
 396                 prealloc_put_cell(structs, cell_prealloc);
 397
 398         return r;
 399 }
 400
 401 /*----------------------------------------------------------------*/
 402
 403 static bool is_dirty(struct cache *cache, dm_cblock_t b)
 404 {
 405         return test_bit(from_cblock(b), cache->dirty_bitset);
 406 }
 407
 408 static void set_dirty(struct cache *cache, dm_oblock_t oblock, dm_cblock_t cblock)
 409 {
 410         if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) {
 411                 cache->nr_dirty = to_cblock(from_cblock(cache->nr_dirty) + 1);
 412                 policy_set_dirty(cache->policy, oblock);
 413         }
 414 }
 415
 416 static void clear_dirty(struct cache *cache, dm_oblock_t oblock, dm_cblock_t cblock)
 417 {
 418         if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) {
 419                 policy_clear_dirty(cache->policy, oblock);
 420                 cache->nr_dirty = to_cblock(from_cblock(cache->nr_dirty) - 1);
 421                 if (!from_cblock(cache->nr_dirty))
 422                         dm_table_event(cache->ti->table);
 423         }
 424 }
 425
 426 /*----------------------------------------------------------------*/
 427
 428 static bool block_size_is_power_of_two(struct cache *cache)
 429 {
 430         return cache->sectors_per_block_shift >= 0;
 431 }
 432
 433 /* gcc on ARM generates spurious references to __udivdi3 and __umoddi3 */
 434 #if defined(CONFIG_ARM) && __GNUC__ == 4 && __GNUC_MINOR__ <= 6
 435 __always_inline
 436 #endif
 437 static dm_block_t block_div(dm_block_t b, uint32_t n)
 438 {
 439         do_div(b, n);
 440
 441         return b;
 442 }
 443
 444 static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock)
 445 {
 446         uint32_t discard_blocks = cache->discard_block_size;
 447         dm_block_t b = from_oblock(oblock);
 448
 449         if (!block_size_is_power_of_two(cache))
 450                 discard_blocks = discard_blocks / cache->sectors_per_block;
 451         else
 452                 discard_blocks >>= cache->sectors_per_block_shift;
 453
 454         b = block_div(b, discard_blocks);
 455
 456         return to_dblock(b);
 457 }
 458
 459 static void set_discard(struct cache *cache, dm_dblock_t b)
 460 {
 461         unsigned long flags;
 462
 463         atomic_inc(&cache->stats.discard_count);
 464
 465         spin_lock_irqsave(&cache->lock, flags);
 466         set_bit(from_dblock(b), cache->discard_bitset);
 467         spin_unlock_irqrestore(&cache->lock, flags);
 468 }
 469
 470 static void clear_discard(struct cache *cache, dm_dblock_t b)
 471 {
 472         unsigned long flags;
 473
 474         spin_lock_irqsave(&cache->lock, flags);
 475         clear_bit(from_dblock(b), cache->discard_bitset);
 476         spin_unlock_irqrestore(&cache->lock, flags);
 477 }
 478
 479 static bool is_discarded(struct cache *cache, dm_dblock_t b)
 480 {
 481         int r;
 482         unsigned long flags;
 483
 484         spin_lock_irqsave(&cache->lock, flags);
 485         r = test_bit(from_dblock(b), cache->discard_bitset);
 486         spin_unlock_irqrestore(&cache->lock, flags);
 487
 488         return r;
 489 }
 490
 491 static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
 492 {
 493         int r;
 494         unsigned long flags;
 495
 496         spin_lock_irqsave(&cache->lock, flags);
 497         r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
 498                      cache->discard_bitset);
 499         spin_unlock_irqrestore(&cache->lock, flags);
 500
 501         return r;
 502 }
 503
 504 /*----------------------------------------------------------------*/
 505
 506 static void load_stats(struct cache *cache)
 507 {
 508         struct dm_cache_statistics stats;
 509
 510         dm_cache_metadata_get_stats(cache->cmd, &stats);
 511         atomic_set(&cache->stats.read_hit, stats.read_hits);
 512         atomic_set(&cache->stats.read_miss, stats.read_misses);
 513         atomic_set(&cache->stats.write_hit, stats.write_hits);
 514         atomic_set(&cache->stats.write_miss, stats.write_misses);
 515 }
 516
 517 static void save_stats(struct cache *cache)
 518 {
 519         struct dm_cache_statistics stats;
 520
 521         stats.read_hits = atomic_read(&cache->stats.read_hit);
 522         stats.read_misses = atomic_read(&cache->stats.read_miss);
 523         stats.write_hits = atomic_read(&cache->stats.write_hit);
 524         stats.write_misses = atomic_read(&cache->stats.write_miss);
 525
 526         dm_cache_metadata_set_stats(cache->cmd, &stats);
 527 }
 528
 529 /*----------------------------------------------------------------
 530  * Per bio data
 531  *--------------------------------------------------------------*/
 532
 533 /*
 534  * If using writeback, leave out struct per_bio_data's writethrough fields.
 535  */
 536 #define PB_DATA_SIZE_WB (offsetof(struct per_bio_data, cache))
 537 #define PB_DATA_SIZE_WT (sizeof(struct per_bio_data))
 538
 539 static size_t get_per_bio_data_size(struct cache *cache)
 540 {
 541         return cache->features.write_through ? PB_DATA_SIZE_WT : PB_DATA_SIZE_WB;
 542 }
 543
 544 static struct per_bio_data *get_per_bio_data(struct bio *bio, size_t data_size)
 545 {
 546         struct per_bio_data *pb = dm_per_bio_data(bio, data_size);
 547         BUG_ON(!pb);
 548         return pb;
 549 }
 550
 551 static struct per_bio_data *init_per_bio_data(struct bio *bio, size_t data_size)
 552 {
 553         struct per_bio_data *pb = get_per_bio_data(bio, data_size);
 554
 555         pb->tick = false;
 556         pb->req_nr = dm_bio_get_target_bio_nr(bio);
 557         pb->all_io_entry = NULL;
 558
 559         return pb;
 560 }
 561
 562 /*----------------------------------------------------------------
 563  * Remapping
 564  *--------------------------------------------------------------*/
 565 static void remap_to_origin(struct cache *cache, struct bio *bio)
 566 {
 567         bio->bi_bdev = cache->origin_dev->bdev;
 568 }
 569
 570 static void remap_to_cache(struct cache *cache, struct bio *bio,
 571                            dm_cblock_t cblock)
 572 {
 573         sector_t bi_sector = bio->bi_sector;
 574
 575         bio->bi_bdev = cache->cache_dev->bdev;
 576         if (!block_size_is_power_of_two(cache))
 577                 bio->bi_sector = (from_cblock(cblock) * cache->sectors_per_block) +
 578                                 sector_div(bi_sector, cache->sectors_per_block);
 579         else
 580                 bio->bi_sector = (from_cblock(cblock) << cache->sectors_per_block_shift) |
 581                                 (bi_sector & (cache->sectors_per_block - 1));
 582 }
 583
 584 static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio)
 585 {
 586         unsigned long flags;
 587         size_t pb_data_size = get_per_bio_data_size(cache);
 588         struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
 589
 590         spin_lock_irqsave(&cache->lock, flags);
 591         if (cache->need_tick_bio &&
 592             !(bio->bi_rw & (REQ_FUA | REQ_FLUSH | REQ_DISCARD))) {
 593                 pb->tick = true;
 594                 cache->need_tick_bio = false;
 595         }
 596         spin_unlock_irqrestore(&cache->lock, flags);
 597 }
 598
 599 static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
 600                                   dm_oblock_t oblock)
 601 {
 602         check_if_tick_bio_needed(cache, bio);
 603         remap_to_origin(cache, bio);
 604         if (bio_data_dir(bio) == WRITE)
 605                 clear_discard(cache, oblock_to_dblock(cache, oblock));
 606 }
 607
 608 static void remap_to_cache_dirty(struct cache *cache, struct bio *bio,
 609                                  dm_oblock_t oblock, dm_cblock_t cblock)
 610 {
 611         check_if_tick_bio_needed(cache, bio);
 612         remap_to_cache(cache, bio, cblock);
 613         if (bio_data_dir(bio) == WRITE) {
 614                 set_dirty(cache, oblock, cblock);
 615                 clear_discard(cache, oblock_to_dblock(cache, oblock));
 616         }
 617 }
 618
 619 static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio)
 620 {
 621         sector_t block_nr = bio->bi_sector;
 622
 623         if (!block_size_is_power_of_two(cache))
 624                 (void) sector_div(block_nr, cache->sectors_per_block);
 625         else
 626                 block_nr >>= cache->sectors_per_block_shift;
 627
 628         return to_oblock(block_nr);
 629 }
 630
 631 static int bio_triggers_commit(struct cache *cache, struct bio *bio)
 632 {
 633         return bio->bi_rw & (REQ_FLUSH | REQ_FUA);
 634 }
 635
 636 static void issue(struct cache *cache, struct bio *bio)
 637 {
 638         unsigned long flags;
 639
 640         if (!bio_triggers_commit(cache, bio)) {
 641                 generic_make_request(bio);
 642                 return;
 643         }
 644
 645         /*
 646          * Batch together any bios that trigger commits and then issue a
 647          * single commit for them in do_worker().
 648          */
 649         spin_lock_irqsave(&cache->lock, flags);
 650         cache->commit_requested = true;
 651         bio_list_add(&cache->deferred_flush_bios, bio);
 652         spin_unlock_irqrestore(&cache->lock, flags);
 653 }
 654
 655 static void defer_writethrough_bio(struct cache *cache, struct bio *bio)
 656 {
 657         unsigned long flags;
 658
 659         spin_lock_irqsave(&cache->lock, flags);
 660         bio_list_add(&cache->deferred_writethrough_bios, bio);
 661         spin_unlock_irqrestore(&cache->lock, flags);
 662
 663         wake_worker(cache);
 664 }
 665
 666 static void writethrough_endio(struct bio *bio, int err)
 667 {
 668         struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT);
 669         bio->bi_end_io = pb->saved_bi_end_io;
 670
 671         if (err) {
 672                 bio_endio(bio, err);
 673                 return;
 674         }
 675
 676         dm_bio_restore(&pb->bio_details, bio);
 677         remap_to_cache(pb->cache, bio, pb->cblock);
 678
 679         /*
 680          * We can't issue this bio directly, since we're in interrupt
 681          * context.  So it gets put on a bio list for processing by the
 682          * worker thread.
 683          */
 684         defer_writethrough_bio(pb->cache, bio);
 685 }
 686
 687 /*
 688  * When running in writethrough mode we need to send writes to clean blocks
 689  * to both the cache and origin devices.  In future we'd like to clone the
 690  * bio and send them in parallel, but for now we're doing them in
 691  * series as this is easier.
 692  */
 693 static void remap_to_origin_then_cache(struct cache *cache, struct bio *bio,
 694                                        dm_oblock_t oblock, dm_cblock_t cblock)
 695 {
 696         struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT);
 697
 698         pb->cache = cache;
 699         pb->cblock = cblock;
 700         pb->saved_bi_end_io = bio->bi_end_io;
 701         dm_bio_record(&pb->bio_details, bio);
 702         bio->bi_end_io = writethrough_endio;
 703
 704         remap_to_origin_clear_discard(pb->cache, bio, oblock);
 705 }
 706
 707 /*----------------------------------------------------------------
 708  * Migration processing
 709  *
 710  * Migration covers moving data from the origin device to the cache, or
 711  * vice versa.
 712  *--------------------------------------------------------------*/
 713 static void free_migration(struct dm_cache_migration *mg)
 714 {
 715         mempool_free(mg, mg->cache->migration_pool);
 716 }
 717
 718 static void inc_nr_migrations(struct cache *cache)
 719 {
 720         atomic_inc(&cache->nr_migrations);
 721 }
 722
 723 static void dec_nr_migrations(struct cache *cache)
 724 {
 725         atomic_dec(&cache->nr_migrations);
 726
 727         /*
 728          * Wake the worker in case we're suspending the target.
 729          */
 730         wake_up(&cache->migration_wait);
 731 }
 732
 733 static void __cell_defer(struct cache *cache, struct dm_bio_prison_cell *cell,
 734                          bool holder)
 735 {
 736         (holder ? dm_cell_release : dm_cell_release_no_holder)
 737                 (cache->prison, cell, &cache->deferred_bios);
 738         free_prison_cell(cache, cell);
 739 }
 740
 741 static void cell_defer(struct cache *cache, struct dm_bio_prison_cell *cell,
 742                        bool holder)
 743 {
 744         unsigned long flags;
 745
 746         spin_lock_irqsave(&cache->lock, flags);
 747         __cell_defer(cache, cell, holder);
 748         spin_unlock_irqrestore(&cache->lock, flags);
 749
 750         wake_worker(cache);
 751 }
 752
 753 static void cleanup_migration(struct dm_cache_migration *mg)
 754 {
 755         struct cache *cache = mg->cache;
 756         free_migration(mg);
 757         dec_nr_migrations(cache);
 758 }
 759
 760 static void migration_failure(struct dm_cache_migration *mg)
 761 {
 762         struct cache *cache = mg->cache;
 763
 764         if (mg->writeback) {
 765                 DMWARN_LIMIT("writeback failed; couldn't copy block");
 766                 set_dirty(cache, mg->old_oblock, mg->cblock);
 767                 cell_defer(cache, mg->old_ocell, false);
 768
 769         } else if (mg->demote) {
 770                 DMWARN_LIMIT("demotion failed; couldn't copy block");
 771                 policy_force_mapping(cache->policy, mg->new_oblock, mg->old_oblock);
 772
 773                 cell_defer(cache, mg->old_ocell, mg->promote ? 0 : 1);
 774                 if (mg->promote)
 775                         cell_defer(cache, mg->new_ocell, 1);
 776         } else {
 777                 DMWARN_LIMIT("promotion failed; couldn't copy block");
 778                 policy_remove_mapping(cache->policy, mg->new_oblock);
 779                 cell_defer(cache, mg->new_ocell, 1);
 780         }
 781
 782         cleanup_migration(mg);
 783 }
 784
 785 static void migration_success_pre_commit(struct dm_cache_migration *mg)
 786 {
 787         unsigned long flags;
 788         struct cache *cache = mg->cache;
 789
 790         if (mg->writeback) {
 791                 cell_defer(cache, mg->old_ocell, false);
 792                 clear_dirty(cache, mg->old_oblock, mg->cblock);
 793                 cleanup_migration(mg);
 794                 return;
 795
 796         } else if (mg->demote) {
 797                 if (dm_cache_remove_mapping(cache->cmd, mg->cblock)) {
 798                         DMWARN_LIMIT("demotion failed; couldn't update on disk metadata");
 799                         policy_force_mapping(cache->policy, mg->new_oblock,
 800                                              mg->old_oblock);
 801                         if (mg->promote)
 802                                 cell_defer(cache, mg->new_ocell, true);
 803                         cleanup_migration(mg);
 804                         return;
 805                 }
 806         } else {
 807                 if (dm_cache_insert_mapping(cache->cmd, mg->cblock, mg->new_oblock)) {
 808                         DMWARN_LIMIT("promotion failed; couldn't update on disk metadata");
 809                         policy_remove_mapping(cache->policy, mg->new_oblock);
 810                         cleanup_migration(mg);
 811                         return;
 812                 }
 813         }
 814
 815         spin_lock_irqsave(&cache->lock, flags);
 816         list_add_tail(&mg->list, &cache->need_commit_migrations);
 817         cache->commit_requested = true;
 818         spin_unlock_irqrestore(&cache->lock, flags);
 819 }
 820
 821 static void migration_success_post_commit(struct dm_cache_migration *mg)
 822 {
 823         unsigned long flags;
 824         struct cache *cache = mg->cache;
 825
 826         if (mg->writeback) {
 827                 DMWARN("writeback unexpectedly triggered commit");
 828                 return;
 829
 830         } else if (mg->demote) {
 831                 cell_defer(cache, mg->old_ocell, mg->promote ? 0 : 1);
 832
 833                 if (mg->promote) {
 834                         mg->demote = false;
 835
 836                         spin_lock_irqsave(&cache->lock, flags);
 837                         list_add_tail(&mg->list, &cache->quiesced_migrations);
 838                         spin_unlock_irqrestore(&cache->lock, flags);
 839
 840                 } else
 841                         cleanup_migration(mg);
 842
 843         } else {
 844                 cell_defer(cache, mg->new_ocell, true);
 845                 clear_dirty(cache, mg->new_oblock, mg->cblock);
 846                 cleanup_migration(mg);
 847         }
 848 }
 849
 850 static void copy_complete(int read_err, unsigned long write_err, void *context)
 851 {
 852         unsigned long flags;
 853         struct dm_cache_migration *mg = (struct dm_cache_migration *) context;
 854         struct cache *cache = mg->cache;
 855
 856         if (read_err || write_err)
 857                 mg->err = true;
 858
 859         spin_lock_irqsave(&cache->lock, flags);
 860         list_add_tail(&mg->list, &cache->completed_migrations);
 861         spin_unlock_irqrestore(&cache->lock, flags);
 862
 863         wake_worker(cache);
 864 }
 865
 866 static void issue_copy_real(struct dm_cache_migration *mg)
 867 {
 868         int r;
 869         struct dm_io_region o_region, c_region;
 870         struct cache *cache = mg->cache;
 871
 872         o_region.bdev = cache->origin_dev->bdev;
 873         o_region.count = cache->sectors_per_block;
 874
 875         c_region.bdev = cache->cache_dev->bdev;
 876         c_region.sector = from_cblock(mg->cblock) * cache->sectors_per_block;
 877         c_region.count = cache->sectors_per_block;
 878
 879         if (mg->writeback || mg->demote) {
 880                 /* demote */
 881                 o_region.sector = from_oblock(mg->old_oblock) * cache->sectors_per_block;
 882                 r = dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, mg);
 883         } else {
 884                 /* promote */
 885                 o_region.sector = from_oblock(mg->new_oblock) * cache->sectors_per_block;
 886                 r = dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, mg);
 887         }
 888
 889         if (r < 0)
 890                 migration_failure(mg);
 891 }
 892
 893 static void avoid_copy(struct dm_cache_migration *mg)
 894 {
 895         atomic_inc(&mg->cache->stats.copies_avoided);
 896         migration_success_pre_commit(mg);
 897 }
 898
 899 static void issue_copy(struct dm_cache_migration *mg)
 900 {
 901         bool avoid;
 902         struct cache *cache = mg->cache;
 903
 904         if (mg->writeback || mg->demote)
 905                 avoid = !is_dirty(cache, mg->cblock) ||
 906                         is_discarded_oblock(cache, mg->old_oblock);
 907         else
 908                 avoid = is_discarded_oblock(cache, mg->new_oblock);
 909
 910         avoid ? avoid_copy(mg) : issue_copy_real(mg);
 911 }
 912
 913 static void complete_migration(struct dm_cache_migration *mg)
 914 {
 915         if (mg->err)
 916                 migration_failure(mg);
 917         else
 918                 migration_success_pre_commit(mg);
 919 }
 920
 921 static void process_migrations(struct cache *cache, struct list_head *head,
 922                                void (*fn)(struct dm_cache_migration *))
 923 {
 924         unsigned long flags;
 925         struct list_head list;
 926         struct dm_cache_migration *mg, *tmp;
 927
 928         INIT_LIST_HEAD(&list);
 929         spin_lock_irqsave(&cache->lock, flags);
 930         list_splice_init(head, &list);
 931         spin_unlock_irqrestore(&cache->lock, flags);
 932
 933         list_for_each_entry_safe(mg, tmp, &list, list)
 934                 fn(mg);
 935 }
 936
 937 static void __queue_quiesced_migration(struct dm_cache_migration *mg)
 938 {
 939         list_add_tail(&mg->list, &mg->cache->quiesced_migrations);
 940 }
 941
 942 static void queue_quiesced_migration(struct dm_cache_migration *mg)
 943 {
 944         unsigned long flags;
 945         struct cache *cache = mg->cache;
 946
 947         spin_lock_irqsave(&cache->lock, flags);
 948         __queue_quiesced_migration(mg);
 949         spin_unlock_irqrestore(&cache->lock, flags);
 950
 951         wake_worker(cache);
 952 }
 953
 954 static void queue_quiesced_migrations(struct cache *cache, struct list_head *work)
 955 {
 956         unsigned long flags;
 957         struct dm_cache_migration *mg, *tmp;
 958
 959         spin_lock_irqsave(&cache->lock, flags);
 960         list_for_each_entry_safe(mg, tmp, work, list)
 961                 __queue_quiesced_migration(mg);
 962         spin_unlock_irqrestore(&cache->lock, flags);
 963
 964         wake_worker(cache);
 965 }
 966
 967 static void check_for_quiesced_migrations(struct cache *cache,
 968                                           struct per_bio_data *pb)
 969 {
 970         struct list_head work;
 971
 972         if (!pb->all_io_entry)
 973                 return;
 974
 975         INIT_LIST_HEAD(&work);
 976         if (pb->all_io_entry)
 977                 dm_deferred_entry_dec(pb->all_io_entry, &work);
 978
 979         if (!list_empty(&work))
 980                 queue_quiesced_migrations(cache, &work);
 981 }
 982
 983 static void quiesce_migration(struct dm_cache_migration *mg)
 984 {
 985         if (!dm_deferred_set_add_work(mg->cache->all_io_ds, &mg->list))
 986                 queue_quiesced_migration(mg);
 987 }
 988
 989 static void promote(struct cache *cache, struct prealloc *structs,
 990                     dm_oblock_t oblock, dm_cblock_t cblock,
 991                     struct dm_bio_prison_cell *cell)
 992 {
 993         struct dm_cache_migration *mg = prealloc_get_migration(structs);
 994
 995         mg->err = false;
 996         mg->writeback = false;
 997         mg->demote = false;
 998         mg->promote = true;
 999         mg->cache = cache;
1000         mg->new_oblock = oblock;
1001         mg->cblock = cblock;
1002         mg->old_ocell = NULL;
1003         mg->new_ocell = cell;
1004         mg->start_jiffies = jiffies;
1005
1006         inc_nr_migrations(cache);
1007         quiesce_migration(mg);
1008 }
1009
1010 static void writeback(struct cache *cache, struct prealloc *structs,
1011                       dm_oblock_t oblock, dm_cblock_t cblock,
1012                       struct dm_bio_prison_cell *cell)
1013 {
1014         struct dm_cache_migration *mg = prealloc_get_migration(structs);
1015
1016         mg->err = false;
1017         mg->writeback = true;
1018         mg->demote = false;
1019         mg->promote = false;
1020         mg->cache = cache;
1021         mg->old_oblock = oblock;
1022         mg->cblock = cblock;
1023         mg->old_ocell = cell;
1024         mg->new_ocell = NULL;
1025         mg->start_jiffies = jiffies;
1026
1027         inc_nr_migrations(cache);
1028         quiesce_migration(mg);
1029 }
1030
1031 static void demote_then_promote(struct cache *cache, struct prealloc *structs,
1032                                 dm_oblock_t old_oblock, dm_oblock_t new_oblock,
1033                                 dm_cblock_t cblock,
1034                                 struct dm_bio_prison_cell *old_ocell,
1035                                 struct dm_bio_prison_cell *new_ocell)
1036 {
1037         struct dm_cache_migration *mg = prealloc_get_migration(structs);
1038
1039         mg->err = false;
1040         mg->writeback = false;
1041         mg->demote = true;
1042         mg->promote = true;
1043         mg->cache = cache;
1044         mg->old_oblock = old_oblock;
1045         mg->new_oblock = new_oblock;
1046         mg->cblock = cblock;
1047         mg->old_ocell = old_ocell;
1048         mg->new_ocell = new_ocell;
1049         mg->start_jiffies = jiffies;
1050
1051         inc_nr_migrations(cache);
1052         quiesce_migration(mg);
1053 }
1054
1055 /*----------------------------------------------------------------
1056  * bio processing
1057  *--------------------------------------------------------------*/
1058 static void defer_bio(struct cache *cache, struct bio *bio)
1059 {
1060         unsigned long flags;
1061
1062         spin_lock_irqsave(&cache->lock, flags);
1063         bio_list_add(&cache->deferred_bios, bio);
1064         spin_unlock_irqrestore(&cache->lock, flags);
1065
1066         wake_worker(cache);
1067 }
1068
1069 static void process_flush_bio(struct cache *cache, struct bio *bio)
1070 {
1071         size_t pb_data_size = get_per_bio_data_size(cache);
1072         struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
1073
1074         BUG_ON(bio->bi_size);
1075         if (!pb->req_nr)
1076                 remap_to_origin(cache, bio);
1077         else
1078                 remap_to_cache(cache, bio, 0);
1079
1080         issue(cache, bio);
1081 }
1082
1083 /*
1084  * People generally discard large parts of a device, eg, the whole device
1085  * when formatting.  Splitting these large discards up into cache block
1086  * sized ios and then quiescing (always neccessary for discard) takes too
1087  * long.
1088  *
1089  * We keep it simple, and allow any size of discard to come in, and just
1090  * mark off blocks on the discard bitset.  No passdown occurs!
1091  *
1092  * To implement passdown we need to change the bio_prison such that a cell
1093  * can have a key that spans many blocks.
1094  */
1095 static void process_discard_bio(struct cache *cache, struct bio *bio)
1096 {
1097         dm_block_t start_block = dm_sector_div_up(bio->bi_sector,
1098                                                   cache->discard_block_size);
1099         dm_block_t end_block = bio->bi_sector + bio_sectors(bio);
1100         dm_block_t b;
1101
1102         end_block = block_div(end_block, cache->discard_block_size);
1103
1104         for (b = start_block; b < end_block; b++)
1105                 set_discard(cache, to_dblock(b));
1106
1107         bio_endio(bio, 0);
1108 }
1109
1110 static bool spare_migration_bandwidth(struct cache *cache)
1111 {
1112         sector_t current_volume = (atomic_read(&cache->nr_migrations) + 1) *
1113                 cache->sectors_per_block;
1114         return current_volume < cache->migration_threshold;
1115 }
1116
1117 static bool is_writethrough_io(struct cache *cache, struct bio *bio,
1118                                dm_cblock_t cblock)
1119 {
1120         return bio_data_dir(bio) == WRITE &&
1121                 cache->features.write_through && !is_dirty(cache, cblock);
1122 }
1123
1124 static void inc_hit_counter(struct cache *cache, struct bio *bio)
1125 {
1126         atomic_inc(bio_data_dir(bio) == READ ?
1127                    &cache->stats.read_hit : &cache->stats.write_hit);
1128 }
1129
1130 static void inc_miss_counter(struct cache *cache, struct bio *bio)
1131 {
1132         atomic_inc(bio_data_dir(bio) == READ ?
1133                    &cache->stats.read_miss : &cache->stats.write_miss);
1134 }
1135
1136 static void process_bio(struct cache *cache, struct prealloc *structs,
1137                         struct bio *bio)
1138 {
1139         int r;
1140         bool release_cell = true;
1141         dm_oblock_t block = get_bio_block(cache, bio);
1142         struct dm_bio_prison_cell *cell_prealloc, *old_ocell, *new_ocell;
1143         struct policy_result lookup_result;
1144         size_t pb_data_size = get_per_bio_data_size(cache);
1145         struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
1146         bool discarded_block = is_discarded_oblock(cache, block);
1147         bool can_migrate = discarded_block || spare_migration_bandwidth(cache);
1148
1149         /*
1150          * Check to see if that block is currently migrating.
1151          */
1152         cell_prealloc = prealloc_get_cell(structs);
1153         r = bio_detain(cache, block, bio, cell_prealloc,
1154                        (cell_free_fn) prealloc_put_cell,
1155                        structs, &new_ocell);
1156         if (r > 0)
1157                 return;
1158
1159         r = policy_map(cache->policy, block, true, can_migrate, discarded_block,
1160                        bio, &lookup_result);
1161
1162         if (r == -EWOULDBLOCK)
1163                 /* migration has been denied */
1164                 lookup_result.op = POLICY_MISS;
1165
1166         switch (lookup_result.op) {
1167         case POLICY_HIT:
1168                 inc_hit_counter(cache, bio);
1169                 pb->all_io_entry = dm_deferred_entry_inc(cache->all_io_ds);
1170
1171                 if (is_writethrough_io(cache, bio, lookup_result.cblock))
1172                         remap_to_origin_then_cache(cache, bio, block, lookup_result.cblock);
1173                 else
1174                         remap_to_cache_dirty(cache, bio, block, lookup_result.cblock);
1175
1176                 issue(cache, bio);
1177                 break;
1178
1179         case POLICY_MISS:
1180                 inc_miss_counter(cache, bio);
1181                 pb->all_io_entry = dm_deferred_entry_inc(cache->all_io_ds);
1182                 remap_to_origin_clear_discard(cache, bio, block);
1183                 issue(cache, bio);
1184                 break;
1185
1186         case POLICY_NEW:
1187                 atomic_inc(&cache->stats.promotion);
1188                 promote(cache, structs, block, lookup_result.cblock, new_ocell);
1189                 release_cell = false;
1190                 break;
1191
1192         case POLICY_REPLACE:
1193                 cell_prealloc = prealloc_get_cell(structs);
1194                 r = bio_detain(cache, lookup_result.old_oblock, bio, cell_prealloc,
1195                                (cell_free_fn) prealloc_put_cell,
1196                                structs, &old_ocell);
1197                 if (r > 0) {
1198                         /*
1199                          * We have to be careful to avoid lock inversion of
1200                          * the cells.  So we back off, and wait for the
1201                          * old_ocell to become free.
1202                          */
1203                         policy_force_mapping(cache->policy, block,
1204                                              lookup_result.old_oblock);
1205                         atomic_inc(&cache->stats.cache_cell_clash);
1206                         break;
1207                 }
1208                 atomic_inc(&cache->stats.demotion);
1209                 atomic_inc(&cache->stats.promotion);
1210
1211                 demote_then_promote(cache, structs, lookup_result.old_oblock,
1212                                     block, lookup_result.cblock,
1213                                     old_ocell, new_ocell);
1214                 release_cell = false;
1215                 break;
1216
1217         default:
1218                 DMERR_LIMIT("%s: erroring bio, unknown policy op: %u", __func__,
1219                             (unsigned) lookup_result.op);
1220                 bio_io_error(bio);
1221         }
1222
1223         if (release_cell)
1224                 cell_defer(cache, new_ocell, false);
1225 }
1226
1227 static int need_commit_due_to_time(struct cache *cache)
1228 {
1229         return jiffies < cache->last_commit_jiffies ||
1230                jiffies > cache->last_commit_jiffies + COMMIT_PERIOD;
1231 }
1232
1233 static int commit_if_needed(struct cache *cache)
1234 {
1235         if (dm_cache_changed_this_transaction(cache->cmd) &&
1236             (cache->commit_requested || need_commit_due_to_time(cache))) {
1237                 atomic_inc(&cache->stats.commit_count);
1238                 cache->last_commit_jiffies = jiffies;
1239                 cache->commit_requested = false;
1240                 return dm_cache_commit(cache->cmd, false);
1241         }
1242
1243         return 0;
1244 }
1245
1246 static void process_deferred_bios(struct cache *cache)
1247 {
1248         unsigned long flags;
1249         struct bio_list bios;
1250         struct bio *bio;
1251         struct prealloc structs;
1252
1253         memset(&structs, 0, sizeof(structs));
1254         bio_list_init(&bios);
1255
1256         spin_lock_irqsave(&cache->lock, flags);
1257         bio_list_merge(&bios, &cache->deferred_bios);
1258         bio_list_init(&cache->deferred_bios);
1259         spin_unlock_irqrestore(&cache->lock, flags);
1260
1261         while (!bio_list_empty(&bios)) {
1262                 /*
1263                  * If we've got no free migration structs, and processing
1264                  * this bio might require one, we pause until there are some
1265                  * prepared mappings to process.
1266                  */
1267                 if (prealloc_data_structs(cache, &structs)) {
1268                         spin_lock_irqsave(&cache->lock, flags);
1269                         bio_list_merge(&cache->deferred_bios, &bios);
1270                         spin_unlock_irqrestore(&cache->lock, flags);
1271                         break;
1272                 }
1273
1274                 bio = bio_list_pop(&bios);
1275
1276                 if (bio->bi_rw & REQ_FLUSH)
1277                         process_flush_bio(cache, bio);
1278                 else if (bio->bi_rw & REQ_DISCARD)
1279                         process_discard_bio(cache, bio);
1280                 else
1281                         process_bio(cache, &structs, bio);
1282         }
1283
1284         prealloc_free_structs(cache, &structs);
1285 }
1286
1287 static void process_deferred_flush_bios(struct cache *cache, bool submit_bios)
1288 {
1289         unsigned long flags;
1290         struct bio_list bios;
1291         struct bio *bio;
1292
1293         bio_list_init(&bios);
1294
1295         spin_lock_irqsave(&cache->lock, flags);
1296         bio_list_merge(&bios, &cache->deferred_flush_bios);
1297         bio_list_init(&cache->deferred_flush_bios);
1298         spin_unlock_irqrestore(&cache->lock, flags);
1299
1300         while ((bio = bio_list_pop(&bios)))
1301                 submit_bios ? generic_make_request(bio) : bio_io_error(bio);
1302 }
1303
1304 static void process_deferred_writethrough_bios(struct cache *cache)
1305 {
1306         unsigned long flags;
1307         struct bio_list bios;
1308         struct bio *bio;
1309
1310         bio_list_init(&bios);
1311
1312         spin_lock_irqsave(&cache->lock, flags);
1313         bio_list_merge(&bios, &cache->deferred_writethrough_bios);
1314         bio_list_init(&cache->deferred_writethrough_bios);
1315         spin_unlock_irqrestore(&cache->lock, flags);
1316
1317         while ((bio = bio_list_pop(&bios)))
1318                 generic_make_request(bio);
1319 }
1320
1321 static void writeback_some_dirty_blocks(struct cache *cache)
1322 {
1323         int r = 0;
1324         dm_oblock_t oblock;
1325         dm_cblock_t cblock;
1326         struct prealloc structs;
1327         struct dm_bio_prison_cell *old_ocell;
1328
1329         memset(&structs, 0, sizeof(structs));
1330
1331         while (spare_migration_bandwidth(cache)) {
1332                 if (prealloc_data_structs(cache, &structs))
1333                         break;
1334
1335                 r = policy_writeback_work(cache->policy, &oblock, &cblock);
1336                 if (r)
1337                         break;
1338
1339                 r = get_cell(cache, oblock, &structs, &old_ocell);
1340                 if (r) {
1341                         policy_set_dirty(cache->policy, oblock);
1342                         break;
1343                 }
1344
1345                 writeback(cache, &structs, oblock, cblock, old_ocell);
1346         }
1347
1348         prealloc_free_structs(cache, &structs);
1349 }
1350
1351 /*----------------------------------------------------------------
1352  * Main worker loop
1353  *--------------------------------------------------------------*/
1354 static bool is_quiescing(struct cache *cache)
1355 {
1356         return atomic_read(&cache->quiescing);
1357 }
1358
1359 static void ack_quiescing(struct cache *cache)
1360 {
1361         if (is_quiescing(cache)) {
1362                 atomic_inc(&cache->quiescing_ack);
1363                 wake_up(&cache->quiescing_wait);
1364         }
1365 }
1366
1367 static void wait_for_quiescing_ack(struct cache *cache)
1368 {
1369         wait_event(cache->quiescing_wait, atomic_read(&cache->quiescing_ack));
1370 }
1371
1372 static void start_quiescing(struct cache *cache)
1373 {
1374         atomic_inc(&cache->quiescing);
1375         wait_for_quiescing_ack(cache);
1376 }
1377
1378 static void stop_quiescing(struct cache *cache)
1379 {
1380         atomic_set(&cache->quiescing, 0);
1381         atomic_set(&cache->quiescing_ack, 0);
1382 }
1383
1384 static void wait_for_migrations(struct cache *cache)
1385 {
1386         wait_event(cache->migration_wait, !atomic_read(&cache->nr_migrations));
1387 }
1388
1389 static void stop_worker(struct cache *cache)
1390 {
1391         cancel_delayed_work(&cache->waker);
1392         flush_workqueue(cache->wq);
1393 }
1394
1395 static void requeue_deferred_io(struct cache *cache)
1396 {
1397         struct bio *bio;
1398         struct bio_list bios;
1399
1400         bio_list_init(&bios);
1401         bio_list_merge(&bios, &cache->deferred_bios);
1402         bio_list_init(&cache->deferred_bios);
1403
1404         while ((bio = bio_list_pop(&bios)))
1405                 bio_endio(bio, DM_ENDIO_REQUEUE);
1406 }
1407
1408 static int more_work(struct cache *cache)
1409 {
1410         if (is_quiescing(cache))
1411                 return !list_empty(&cache->quiesced_migrations) ||
1412                         !list_empty(&cache->completed_migrations) ||
1413                         !list_empty(&cache->need_commit_migrations);
1414         else
1415                 return !bio_list_empty(&cache->deferred_bios) ||
1416                         !bio_list_empty(&cache->deferred_flush_bios) ||
1417                         !bio_list_empty(&cache->deferred_writethrough_bios) ||
1418                         !list_empty(&cache->quiesced_migrations) ||
1419                         !list_empty(&cache->completed_migrations) ||
1420                         !list_empty(&cache->need_commit_migrations);
1421 }
1422
1423 static void do_worker(struct work_struct *ws)
1424 {
1425         struct cache *cache = container_of(ws, struct cache, worker);
1426
1427         do {
1428                 if (!is_quiescing(cache)) {
1429                         writeback_some_dirty_blocks(cache);
1430                         process_deferred_writethrough_bios(cache);
1431                         process_deferred_bios(cache);
1432                 }
1433
1434                 process_migrations(cache, &cache->quiesced_migrations, issue_copy);
1435                 process_migrations(cache, &cache->completed_migrations, complete_migration);
1436
1437                 if (commit_if_needed(cache)) {
1438                         process_deferred_flush_bios(cache, false);
1439
1440                         /*
1441                          * FIXME: rollback metadata or just go into a
1442                          * failure mode and error everything
1443                          */
1444                 } else {
1445                         process_deferred_flush_bios(cache, true);
1446                         process_migrations(cache, &cache->need_commit_migrations,
1447                                            migration_success_post_commit);
1448                 }
1449
1450                 ack_quiescing(cache);
1451
1452         } while (more_work(cache));
1453 }
1454
1455 /*
1456  * We want to commit periodically so that not too much
1457  * unwritten metadata builds up.
1458  */
1459 static void do_waker(struct work_struct *ws)
1460 {
1461         struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker);
1462         policy_tick(cache->policy);
1463         wake_worker(cache);
1464         queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD);
1465 }
1466
1467 /*----------------------------------------------------------------*/
1468
1469 static int is_congested(struct dm_dev *dev, int bdi_bits)
1470 {
1471         struct request_queue *q = bdev_get_queue(dev->bdev);
1472         return bdi_congested(&q->backing_dev_info, bdi_bits);
1473 }
1474
1475 static int cache_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
1476 {
1477         struct cache *cache = container_of(cb, struct cache, callbacks);
1478
1479         return is_congested(cache->origin_dev, bdi_bits) ||
1480                 is_congested(cache->cache_dev, bdi_bits);
1481 }
1482
1483 /*----------------------------------------------------------------
1484  * Target methods
1485  *--------------------------------------------------------------*/
1486
1487 /*
1488  * This function gets called on the error paths of the constructor, so we
1489  * have to cope with a partially initialised struct.
1490  */
1491 static void destroy(struct cache *cache)
1492 {
1493         unsigned i;
1494
1495         if (cache->next_migration)
1496                 mempool_free(cache->next_migration, cache->migration_pool);
1497
1498         if (cache->migration_pool)
1499                 mempool_destroy(cache->migration_pool);
1500
1501         if (cache->all_io_ds)
1502                 dm_deferred_set_destroy(cache->all_io_ds);
1503
1504         if (cache->prison)
1505                 dm_bio_prison_destroy(cache->prison);
1506
1507         if (cache->wq)
1508                 destroy_workqueue(cache->wq);
1509
1510         if (cache->dirty_bitset)
1511                 free_bitset(cache->dirty_bitset);
1512
1513         if (cache->discard_bitset)
1514                 free_bitset(cache->discard_bitset);
1515
1516         if (cache->copier)
1517                 dm_kcopyd_client_destroy(cache->copier);
1518
1519         if (cache->cmd)
1520                 dm_cache_metadata_close(cache->cmd);
1521
1522         if (cache->metadata_dev)
1523                 dm_put_device(cache->ti, cache->metadata_dev);
1524
1525         if (cache->origin_dev)
1526                 dm_put_device(cache->ti, cache->origin_dev);
1527
1528         if (cache->cache_dev)
1529                 dm_put_device(cache->ti, cache->cache_dev);
1530
1531         if (cache->policy)
1532                 dm_cache_policy_destroy(cache->policy);
1533
1534         for (i = 0; i < cache->nr_ctr_args ; i++)
1535                 kfree(cache->ctr_args[i]);
1536         kfree(cache->ctr_args);
1537
1538         kfree(cache);
1539 }
1540
1541 static void cache_dtr(struct dm_target *ti)
1542 {
1543         struct cache *cache = ti->private;
1544
1545         destroy(cache);
1546 }
1547
1548 static sector_t get_dev_size(struct dm_dev *dev)
1549 {
1550         return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
1551 }
1552
1553 /*----------------------------------------------------------------*/
1554
1555 /*
1556  * Construct a cache device mapping.
1557  *
1558  * cache <metadata dev> <cache dev> <origin dev> <block size>
1559  *       <#feature args> [<feature arg>]*
1560  *       <policy> <#policy args> [<policy arg>]*
1561  *
1562  * metadata dev    : fast device holding the persistent metadata
1563  * cache dev       : fast device holding cached data blocks
1564  * origin dev      : slow device holding original data blocks
1565  * block size      : cache unit size in sectors
1566  *
1567  * #feature args   : number of feature arguments passed
1568  * feature args    : writethrough.  (The default is writeback.)
1569  *
1570  * policy          : the replacement policy to use
1571  * #policy args    : an even number of policy arguments corresponding
1572  *                   to key/value pairs passed to the policy
1573  * policy args     : key/value pairs passed to the policy
1574  *                   E.g. 'sequential_threshold 1024'
1575  *                   See cache-policies.txt for details.
1576  *
1577  * Optional feature arguments are:
1578  *   writethrough  : write through caching that prohibits cache block
1579  *                   content from being different from origin block content.
1580  *                   Without this argument, the default behaviour is to write
1581  *                   back cache block contents later for performance reasons,
1582  *                   so they may differ from the corresponding origin blocks.
1583  */
1584 struct cache_args {
1585         struct dm_target *ti;
1586
1587         struct dm_dev *metadata_dev;
1588
1589         struct dm_dev *cache_dev;
1590         sector_t cache_sectors;
1591
1592         struct dm_dev *origin_dev;
1593         sector_t origin_sectors;
1594
1595         uint32_t block_size;
1596
1597         const char *policy_name;
1598         int policy_argc;
1599         const char **policy_argv;
1600
1601         struct cache_features features;
1602 };
1603
1604 static void destroy_cache_args(struct cache_args *ca)
1605 {
1606         if (ca->metadata_dev)
1607                 dm_put_device(ca->ti, ca->metadata_dev);
1608
1609         if (ca->cache_dev)
1610                 dm_put_device(ca->ti, ca->cache_dev);
1611
1612         if (ca->origin_dev)
1613                 dm_put_device(ca->ti, ca->origin_dev);
1614
1615         kfree(ca);
1616 }
1617
1618 static bool at_least_one_arg(struct dm_arg_set *as, char **error)
1619 {
1620         if (!as->argc) {
1621                 *error = "Insufficient args";
1622                 return false;
1623         }
1624
1625         return true;
1626 }
1627
1628 static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as,
1629                               char **error)
1630 {
1631         int r;
1632         sector_t metadata_dev_size;
1633         char b[BDEVNAME_SIZE];
1634
1635         if (!at_least_one_arg(as, error))
1636                 return -EINVAL;
1637
1638         r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
1639                           &ca->metadata_dev);
1640         if (r) {
1641                 *error = "Error opening metadata device";
1642                 return r;
1643         }
1644
1645         metadata_dev_size = get_dev_size(ca->metadata_dev);
1646         if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING)
1647                 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
1648                        bdevname(ca->metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS);
1649
1650         return 0;
1651 }
1652
1653 static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as,
1654                            char **error)
1655 {
1656         int r;
1657
1658         if (!at_least_one_arg(as, error))
1659                 return -EINVAL;
1660
1661         r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
1662                           &ca->cache_dev);
1663         if (r) {
1664                 *error = "Error opening cache device";
1665                 return r;
1666         }
1667         ca->cache_sectors = get_dev_size(ca->cache_dev);
1668
1669         return 0;
1670 }
1671
1672 static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as,
1673                             char **error)
1674 {
1675         int r;
1676
1677         if (!at_least_one_arg(as, error))
1678                 return -EINVAL;
1679
1680         r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
1681                           &ca->origin_dev);
1682         if (r) {
1683                 *error = "Error opening origin device";
1684                 return r;
1685         }
1686
1687         ca->origin_sectors = get_dev_size(ca->origin_dev);
1688         if (ca->ti->len > ca->origin_sectors) {
1689                 *error = "Device size larger than cached device";
1690                 return -EINVAL;
1691         }
1692
1693         return 0;
1694 }
1695
1696 static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as,
1697                             char **error)
1698 {
1699         unsigned long block_size;
1700
1701         if (!at_least_one_arg(as, error))
1702                 return -EINVAL;
1703
1704         if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size ||
1705             block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
1706             block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
1707             block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
1708                 *error = "Invalid data block size";
1709                 return -EINVAL;
1710         }
1711
1712         if (block_size > ca->cache_sectors) {
1713                 *error = "Data block size is larger than the cache device";
1714                 return -EINVAL;
1715         }
1716
1717         ca->block_size = block_size;
1718
1719         return 0;
1720 }
1721
1722 static void init_features(struct cache_features *cf)
1723 {
1724         cf->mode = CM_WRITE;
1725         cf->write_through = false;
1726 }
1727
1728 static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
1729                           char **error)
1730 {
1731         static struct dm_arg _args[] = {
1732                 {0, 1, "Invalid number of cache feature arguments"},
1733         };
1734
1735         int r;
1736         unsigned argc;
1737         const char *arg;
1738         struct cache_features *cf = &ca->features;
1739
1740         init_features(cf);
1741
1742         r = dm_read_arg_group(_args, as, &argc, error);
1743         if (r)
1744                 return -EINVAL;
1745
1746         while (argc--) {
1747                 arg = dm_shift_arg(as);
1748
1749                 if (!strcasecmp(arg, "writeback"))
1750                         cf->write_through = false;
1751
1752                 else if (!strcasecmp(arg, "writethrough"))
1753                         cf->write_through = true;
1754
1755                 else {
1756                         *error = "Unrecognised cache feature requested";
1757                         return -EINVAL;
1758                 }
1759         }
1760
1761         return 0;
1762 }
1763
1764 static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
1765                         char **error)
1766 {
1767         static struct dm_arg _args[] = {
1768                 {0, 1024, "Invalid number of policy arguments"},
1769         };
1770
1771         int r;
1772
1773         if (!at_least_one_arg(as, error))
1774                 return -EINVAL;
1775
1776         ca->policy_name = dm_shift_arg(as);
1777
1778         r = dm_read_arg_group(_args, as, &ca->policy_argc, error);
1779         if (r)
1780                 return -EINVAL;
1781
1782         ca->policy_argv = (const char **)as->argv;
1783         dm_consume_args(as, ca->policy_argc);
1784
1785         return 0;
1786 }
1787
1788 static int parse_cache_args(struct cache_args *ca, int argc, char **argv,
1789                             char **error)
1790 {
1791         int r;
1792         struct dm_arg_set as;
1793
1794         as.argc = argc;
1795         as.argv = argv;
1796
1797         r = parse_metadata_dev(ca, &as, error);
1798         if (r)
1799                 return r;
1800
1801         r = parse_cache_dev(ca, &as, error);
1802         if (r)
1803                 return r;
1804
1805         r = parse_origin_dev(ca, &as, error);
1806         if (r)
1807                 return r;
1808
1809         r = parse_block_size(ca, &as, error);
1810         if (r)
1811                 return r;
1812
1813         r = parse_features(ca, &as, error);
1814         if (r)
1815                 return r;
1816
1817         r = parse_policy(ca, &as, error);
1818         if (r)
1819                 return r;
1820
1821         return 0;
1822 }
1823
1824 /*----------------------------------------------------------------*/
1825
1826 static struct kmem_cache *migration_cache;
1827
1828 #define NOT_CORE_OPTION 1
1829
1830 static int process_config_option(struct cache *cache, const char *key, const char *value)
1831 {
1832         unsigned long tmp;
1833
1834         if (!strcasecmp(key, "migration_threshold")) {
1835                 if (kstrtoul(value, 10, &tmp))
1836                         return -EINVAL;
1837
1838                 cache->migration_threshold = tmp;
1839                 return 0;
1840         }
1841
1842         return NOT_CORE_OPTION;
1843 }
1844
1845 static int set_config_value(struct cache *cache, const char *key, const char *value)
1846 {
1847         int r = process_config_option(cache, key, value);
1848
1849         if (r == NOT_CORE_OPTION)
1850                 r = policy_set_config_value(cache->policy, key, value);
1851
1852         if (r)
1853                 DMWARN("bad config value for %s: %s", key, value);
1854
1855         return r;
1856 }
1857
1858 static int set_config_values(struct cache *cache, int argc, const char **argv)
1859 {
1860         int r = 0;
1861
1862         if (argc & 1) {
1863                 DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs.");
1864                 return -EINVAL;
1865         }
1866
1867         while (argc) {
1868                 r = set_config_value(cache, argv[0], argv[1]);
1869                 if (r)
1870                         break;
1871
1872                 argc -= 2;
1873                 argv += 2;
1874         }
1875
1876         return r;
1877 }
1878
1879 static int create_cache_policy(struct cache *cache, struct cache_args *ca,
1880                                char **error)
1881 {
1882         cache->policy = dm_cache_policy_create(ca->policy_name,
1883                                                cache->cache_size,
1884                                                cache->origin_sectors,
1885                                                cache->sectors_per_block);
1886         if (!cache->policy) {
1887                 *error = "Error creating cache's policy";
1888                 return -ENOMEM;
1889         }
1890
1891         return 0;
1892 }
1893
1894 /*
1895  * We want the discard block size to be a power of two, at least the size
1896  * of the cache block size, and have no more than 2^14 discard blocks
1897  * across the origin.
1898  */
1899 #define MAX_DISCARD_BLOCKS (1 << 14)
1900
1901 static bool too_many_discard_blocks(sector_t discard_block_size,
1902                                     sector_t origin_size)
1903 {
1904         (void) sector_div(origin_size, discard_block_size);
1905
1906         return origin_size > MAX_DISCARD_BLOCKS;
1907 }
1908
1909 static sector_t calculate_discard_block_size(sector_t cache_block_size,
1910                                              sector_t origin_size)
1911 {
1912         sector_t discard_block_size;
1913
1914         discard_block_size = roundup_pow_of_two(cache_block_size);
1915
1916         if (origin_size)
1917                 while (too_many_discard_blocks(discard_block_size, origin_size))
1918                         discard_block_size *= 2;
1919
1920         return discard_block_size;
1921 }
1922
1923 #define DEFAULT_MIGRATION_THRESHOLD 2048
1924
1925 static int cache_create(struct cache_args *ca, struct cache **result)
1926 {
1927         int r = 0;
1928         char **error = &ca->ti->error;
1929         struct cache *cache;
1930         struct dm_target *ti = ca->ti;
1931         dm_block_t origin_blocks;
1932         struct dm_cache_metadata *cmd;
1933         bool may_format = ca->features.mode == CM_WRITE;
1934
1935         cache = kzalloc(sizeof(*cache), GFP_KERNEL);
1936         if (!cache)
1937                 return -ENOMEM;
1938
1939         cache->ti = ca->ti;
1940         ti->private = cache;
1941         ti->num_flush_bios = 2;
1942         ti->flush_supported = true;
1943
1944         ti->num_discard_bios = 1;
1945         ti->discards_supported = true;
1946         ti->discard_zeroes_data_unsupported = true;
1947
1948         cache->features = ca->features;
1949         ti->per_bio_data_size = get_per_bio_data_size(cache);
1950
1951         cache->callbacks.congested_fn = cache_is_congested;
1952         dm_table_add_target_callbacks(ti->table, &cache->callbacks);
1953
1954         cache->metadata_dev = ca->metadata_dev;
1955         cache->origin_dev = ca->origin_dev;
1956         cache->cache_dev = ca->cache_dev;
1957
1958         ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL;
1959
1960         /* FIXME: factor out this whole section */
1961         origin_blocks = cache->origin_sectors = ca->origin_sectors;
1962         origin_blocks = block_div(origin_blocks, ca->block_size);
1963         cache->origin_blocks = to_oblock(origin_blocks);
1964
1965         cache->sectors_per_block = ca->block_size;
1966         if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) {
1967                 r = -EINVAL;
1968                 goto bad;
1969         }
1970
1971         if (ca->block_size & (ca->block_size - 1)) {
1972                 dm_block_t cache_size = ca->cache_sectors;
1973
1974                 cache->sectors_per_block_shift = -1;
1975                 cache_size = block_div(cache_size, ca->block_size);
1976                 cache->cache_size = to_cblock(cache_size);
1977         } else {
1978                 cache->sectors_per_block_shift = __ffs(ca->block_size);
1979                 cache->cache_size = to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift);
1980         }
1981
1982         r = create_cache_policy(cache, ca, error);
1983         if (r)
1984                 goto bad;
1985
1986         cache->policy_nr_args = ca->policy_argc;
1987         cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD;
1988
1989         r = set_config_values(cache, ca->policy_argc, ca->policy_argv);
1990         if (r) {
1991                 *error = "Error setting cache policy's config values";
1992                 goto bad;
1993         }
1994
1995         cmd = dm_cache_metadata_open(cache->metadata_dev->bdev,
1996                                      ca->block_size, may_format,
1997                                      dm_cache_policy_get_hint_size(cache->policy));
1998         if (IS_ERR(cmd)) {
1999                 *error = "Error creating metadata object";
2000                 r = PTR_ERR(cmd);
2001                 goto bad;
2002         }
2003         cache->cmd = cmd;
2004
2005         spin_lock_init(&cache->lock);
2006         bio_list_init(&cache->deferred_bios);
2007         bio_list_init(&cache->deferred_flush_bios);
2008         bio_list_init(&cache->deferred_writethrough_bios);
2009         INIT_LIST_HEAD(&cache->quiesced_migrations);
2010         INIT_LIST_HEAD(&cache->completed_migrations);
2011         INIT_LIST_HEAD(&cache->need_commit_migrations);
2012         atomic_set(&cache->nr_migrations, 0);
2013         init_waitqueue_head(&cache->migration_wait);
2014
2015         init_waitqueue_head(&cache->quiescing_wait);
2016         atomic_set(&cache->quiescing, 0);
2017         atomic_set(&cache->quiescing_ack, 0);
2018
2019         r = -ENOMEM;
2020         cache->nr_dirty = 0;
2021         cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size));
2022         if (!cache->dirty_bitset) {
2023                 *error = "could not allocate dirty bitset";
2024                 goto bad;
2025         }
2026         clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size));
2027
2028         cache->discard_block_size =
2029                 calculate_discard_block_size(cache->sectors_per_block,
2030                                              cache->origin_sectors);
2031         cache->discard_nr_blocks = oblock_to_dblock(cache, cache->origin_blocks);
2032         cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks));
2033         if (!cache->discard_bitset) {
2034                 *error = "could not allocate discard bitset";
2035                 goto bad;
2036         }
2037         clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
2038
2039         cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2040         if (IS_ERR(cache->copier)) {
2041                 *error = "could not create kcopyd client";
2042                 r = PTR_ERR(cache->copier);
2043                 goto bad;
2044         }
2045
2046         cache->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
2047         if (!cache->wq) {
2048                 *error = "could not create workqueue for metadata object";
2049                 goto bad;
2050         }
2051         INIT_WORK(&cache->worker, do_worker);
2052         INIT_DELAYED_WORK(&cache->waker, do_waker);
2053         cache->last_commit_jiffies = jiffies;
2054
2055         cache->prison = dm_bio_prison_create(PRISON_CELLS);
2056         if (!cache->prison) {
2057                 *error = "could not create bio prison";
2058                 goto bad;
2059         }
2060
2061         cache->all_io_ds = dm_deferred_set_create();
2062         if (!cache->all_io_ds) {
2063                 *error = "could not create all_io deferred set";
2064                 goto bad;
2065         }
2066
2067         cache->migration_pool = mempool_create_slab_pool(MIGRATION_POOL_SIZE,
2068                                                          migration_cache);
2069         if (!cache->migration_pool) {
2070                 *error = "Error creating cache's migration mempool";
2071                 goto bad;
2072         }
2073
2074         cache->next_migration = NULL;
2075
2076         cache->need_tick_bio = true;
2077         cache->sized = false;
2078         cache->commit_requested = false;
2079         cache->loaded_mappings = false;
2080         cache->loaded_discards = false;
2081
2082         load_stats(cache);
2083
2084         atomic_set(&cache->stats.demotion, 0);
2085         atomic_set(&cache->stats.promotion, 0);
2086         atomic_set(&cache->stats.copies_avoided, 0);
2087         atomic_set(&cache->stats.cache_cell_clash, 0);
2088         atomic_set(&cache->stats.commit_count, 0);
2089         atomic_set(&cache->stats.discard_count, 0);
2090
2091         *result = cache;
2092         return 0;
2093
2094 bad:
2095         destroy(cache);
2096         return r;
2097 }
2098
2099 static int copy_ctr_args(struct cache *cache, int argc, const char **argv)
2100 {
2101         unsigned i;
2102         const char **copy;
2103
2104         copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
2105         if (!copy)
2106                 return -ENOMEM;
2107         for (i = 0; i < argc; i++) {
2108                 copy[i] = kstrdup(argv[i], GFP_KERNEL);
2109                 if (!copy[i]) {
2110                         while (i--)
2111                                 kfree(copy[i]);
2112                         kfree(copy);
2113                         return -ENOMEM;
2114                 }
2115         }
2116
2117         cache->nr_ctr_args = argc;
2118         cache->ctr_args = copy;
2119
2120         return 0;
2121 }
2122
2123 static int cache_ctr(struct dm_target *ti, unsigned argc, char **argv)
2124 {
2125         int r = -EINVAL;
2126         struct cache_args *ca;
2127         struct cache *cache = NULL;
2128
2129         ca = kzalloc(sizeof(*ca), GFP_KERNEL);
2130         if (!ca) {
2131                 ti->error = "Error allocating memory for cache";
2132                 return -ENOMEM;
2133         }
2134         ca->ti = ti;
2135
2136         r = parse_cache_args(ca, argc, argv, &ti->error);
2137         if (r)
2138                 goto out;
2139
2140         r = cache_create(ca, &cache);
2141         if (r)
2142                 goto out;
2143
2144         r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3);
2145         if (r) {
2146                 destroy(cache);
2147                 goto out;
2148         }
2149
2150         ti->private = cache;
2151
2152 out:
2153         destroy_cache_args(ca);
2154         return r;
2155 }
2156
2157 static int cache_map(struct dm_target *ti, struct bio *bio)
2158 {
2159         struct cache *cache = ti->private;
2160
2161         int r;
2162         dm_oblock_t block = get_bio_block(cache, bio);
2163         size_t pb_data_size = get_per_bio_data_size(cache);
2164         bool can_migrate = false;
2165         bool discarded_block;
2166         struct dm_bio_prison_cell *cell;
2167         struct policy_result lookup_result;
2168         struct per_bio_data *pb;
2169
2170         if (from_oblock(block) > from_oblock(cache->origin_blocks)) {
2171                 /*
2172                  * This can only occur if the io goes to a partial block at
2173                  * the end of the origin device.  We don't cache these.
2174                  * Just remap to the origin and carry on.
2175                  */
2176                 remap_to_origin_clear_discard(cache, bio, block);
2177                 return DM_MAPIO_REMAPPED;
2178         }
2179
2180         pb = init_per_bio_data(bio, pb_data_size);
2181
2182         if (bio->bi_rw & (REQ_FLUSH | REQ_FUA | REQ_DISCARD)) {
2183                 defer_bio(cache, bio);
2184                 return DM_MAPIO_SUBMITTED;
2185         }
2186
2187         /*
2188          * Check to see if that block is currently migrating.
2189          */
2190         cell = alloc_prison_cell(cache);
2191         if (!cell) {
2192                 defer_bio(cache, bio);
2193                 return DM_MAPIO_SUBMITTED;
2194         }
2195
2196         r = bio_detain(cache, block, bio, cell,
2197                        (cell_free_fn) free_prison_cell,
2198                        cache, &cell);
2199         if (r) {
2200                 if (r < 0)
2201                         defer_bio(cache, bio);
2202
2203                 return DM_MAPIO_SUBMITTED;
2204         }
2205
2206         discarded_block = is_discarded_oblock(cache, block);
2207
2208         r = policy_map(cache->policy, block, false, can_migrate, discarded_block,
2209                        bio, &lookup_result);
2210         if (r == -EWOULDBLOCK) {
2211                 cell_defer(cache, cell, true);
2212                 return DM_MAPIO_SUBMITTED;
2213
2214         } else if (r) {
2215                 DMERR_LIMIT("Unexpected return from cache replacement policy: %d", r);
2216                 bio_io_error(bio);
2217                 return DM_MAPIO_SUBMITTED;
2218         }
2219
2220         switch (lookup_result.op) {
2221         case POLICY_HIT:
2222                 inc_hit_counter(cache, bio);
2223                 pb->all_io_entry = dm_deferred_entry_inc(cache->all_io_ds);
2224
2225                 if (is_writethrough_io(cache, bio, lookup_result.cblock))
2226                         remap_to_origin_then_cache(cache, bio, block, lookup_result.cblock);
2227                 else
2228                         remap_to_cache_dirty(cache, bio, block, lookup_result.cblock);
2229
2230                 cell_defer(cache, cell, false);
2231                 break;
2232
2233         case POLICY_MISS:
2234                 inc_miss_counter(cache, bio);
2235                 pb->all_io_entry = dm_deferred_entry_inc(cache->all_io_ds);
2236
2237                 if (pb->req_nr != 0) {
2238                         /*
2239                          * This is a duplicate writethrough io that is no
2240                          * longer needed because the block has been demoted.
2241                          */
2242                         bio_endio(bio, 0);
2243                         cell_defer(cache, cell, false);
2244                         return DM_MAPIO_SUBMITTED;
2245                 } else {
2246                         remap_to_origin_clear_discard(cache, bio, block);
2247                         cell_defer(cache, cell, false);
2248                 }
2249                 break;
2250
2251         default:
2252                 DMERR_LIMIT("%s: erroring bio: unknown policy op: %u", __func__,
2253                             (unsigned) lookup_result.op);
2254                 bio_io_error(bio);
2255                 return DM_MAPIO_SUBMITTED;
2256         }
2257
2258         return DM_MAPIO_REMAPPED;
2259 }
2260
2261 static int cache_end_io(struct dm_target *ti, struct bio *bio, int error)
2262 {
2263         struct cache *cache = ti->private;
2264         unsigned long flags;
2265         size_t pb_data_size = get_per_bio_data_size(cache);
2266         struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
2267
2268         if (pb->tick) {
2269                 policy_tick(cache->policy);
2270
2271                 spin_lock_irqsave(&cache->lock, flags);
2272                 cache->need_tick_bio = true;
2273                 spin_unlock_irqrestore(&cache->lock, flags);
2274         }
2275
2276         check_for_quiesced_migrations(cache, pb);
2277
2278         return 0;
2279 }
2280
2281 static int write_dirty_bitset(struct cache *cache)
2282 {
2283         unsigned i, r;
2284
2285         for (i = 0; i < from_cblock(cache->cache_size); i++) {
2286                 r = dm_cache_set_dirty(cache->cmd, to_cblock(i),
2287                                        is_dirty(cache, to_cblock(i)));
2288                 if (r)
2289                         return r;
2290         }
2291
2292         return 0;
2293 }
2294
2295 static int write_discard_bitset(struct cache *cache)
2296 {
2297         unsigned i, r;
2298
2299         r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size,
2300                                            cache->discard_nr_blocks);
2301         if (r) {
2302                 DMERR("could not resize on-disk discard bitset");
2303                 return r;
2304         }
2305
2306         for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) {
2307                 r = dm_cache_set_discard(cache->cmd, to_dblock(i),
2308                                          is_discarded(cache, to_dblock(i)));
2309                 if (r)
2310                         return r;
2311         }
2312
2313         return 0;
2314 }
2315
2316 static int save_hint(void *context, dm_cblock_t cblock, dm_oblock_t oblock,
2317                      uint32_t hint)
2318 {
2319         struct cache *cache = context;
2320         return dm_cache_save_hint(cache->cmd, cblock, hint);
2321 }
2322
2323 static int write_hints(struct cache *cache)
2324 {
2325         int r;
2326
2327         r = dm_cache_begin_hints(cache->cmd, cache->policy);
2328         if (r) {
2329                 DMERR("dm_cache_begin_hints failed");
2330                 return r;
2331         }
2332
2333         r = policy_walk_mappings(cache->policy, save_hint, cache);
2334         if (r)
2335                 DMERR("policy_walk_mappings failed");
2336
2337         return r;
2338 }
2339
2340 /*
2341  * returns true on success
2342  */
2343 static bool sync_metadata(struct cache *cache)
2344 {
2345         int r1, r2, r3, r4;
2346
2347         r1 = write_dirty_bitset(cache);
2348         if (r1)
2349                 DMERR("could not write dirty bitset");
2350
2351         r2 = write_discard_bitset(cache);
2352         if (r2)
2353                 DMERR("could not write discard bitset");
2354
2355         save_stats(cache);
2356
2357         r3 = write_hints(cache);
2358         if (r3)
2359                 DMERR("could not write hints");
2360
2361         /*
2362          * If writing the above metadata failed, we still commit, but don't
2363          * set the clean shutdown flag.  This will effectively force every
2364          * dirty bit to be set on reload.
2365          */
2366         r4 = dm_cache_commit(cache->cmd, !r1 && !r2 && !r3);
2367         if (r4)
2368                 DMERR("could not write cache metadata.  Data loss may occur.");
2369
2370         return !r1 && !r2 && !r3 && !r4;
2371 }
2372
2373 static void cache_postsuspend(struct dm_target *ti)
2374 {
2375         struct cache *cache = ti->private;
2376
2377         start_quiescing(cache);
2378         wait_for_migrations(cache);
2379         stop_worker(cache);
2380         requeue_deferred_io(cache);
2381         stop_quiescing(cache);
2382
2383         (void) sync_metadata(cache);
2384 }
2385
2386 static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
2387                         bool dirty, uint32_t hint, bool hint_valid)
2388 {
2389         int r;
2390         struct cache *cache = context;
2391
2392         r = policy_load_mapping(cache->policy, oblock, cblock, hint, hint_valid);
2393         if (r)
2394                 return r;
2395
2396         if (dirty)
2397                 set_dirty(cache, oblock, cblock);
2398         else
2399                 clear_dirty(cache, oblock, cblock);
2400
2401         return 0;
2402 }
2403
2404 static int load_discard(void *context, sector_t discard_block_size,
2405                         dm_dblock_t dblock, bool discard)
2406 {
2407         struct cache *cache = context;
2408
2409         /* FIXME: handle mis-matched block size */
2410
2411         if (discard)
2412                 set_discard(cache, dblock);
2413         else
2414                 clear_discard(cache, dblock);
2415
2416         return 0;
2417 }
2418
2419 static int cache_preresume(struct dm_target *ti)
2420 {
2421         int r = 0;
2422         struct cache *cache = ti->private;
2423         sector_t actual_cache_size = get_dev_size(cache->cache_dev);
2424         (void) sector_div(actual_cache_size, cache->sectors_per_block);
2425
2426         /*
2427          * Check to see if the cache has resized.
2428          */
2429         if (from_cblock(cache->cache_size) != actual_cache_size || !cache->sized) {
2430                 cache->cache_size = to_cblock(actual_cache_size);
2431
2432                 r = dm_cache_resize(cache->cmd, cache->cache_size);
2433                 if (r) {
2434                         DMERR("could not resize cache metadata");
2435                         return r;
2436                 }
2437
2438                 cache->sized = true;
2439         }
2440
2441         if (!cache->loaded_mappings) {
2442                 r = dm_cache_load_mappings(cache->cmd, cache->policy,
2443                                            load_mapping, cache);
2444                 if (r) {
2445                         DMERR("could not load cache mappings");
2446                         return r;
2447                 }
2448
2449                 cache->loaded_mappings = true;
2450         }
2451
2452         if (!cache->loaded_discards) {
2453                 r = dm_cache_load_discards(cache->cmd, load_discard, cache);
2454                 if (r) {
2455                         DMERR("could not load origin discards");
2456                         return r;
2457                 }
2458
2459                 cache->loaded_discards = true;
2460         }
2461
2462         return r;
2463 }
2464
2465 static void cache_resume(struct dm_target *ti)
2466 {
2467         struct cache *cache = ti->private;
2468
2469         cache->need_tick_bio = true;
2470         do_waker(&cache->waker.work);
2471 }
2472
2473 /*
2474  * Status format:
2475  *
2476  * <#used metadata blocks>/<#total metadata blocks>
2477  * <#read hits> <#read misses> <#write hits> <#write misses>
2478  * <#demotions> <#promotions> <#blocks in cache> <#dirty>
2479  * <#features> <features>*
2480  * <#core args> <core args>
2481  * <#policy args> <policy args>*
2482  */
2483 static void cache_status(struct dm_target *ti, status_type_t type,
2484                          unsigned status_flags, char *result, unsigned maxlen)
2485 {
2486         int r = 0;
2487         unsigned i;
2488         ssize_t sz = 0;
2489         dm_block_t nr_free_blocks_metadata = 0;
2490         dm_block_t nr_blocks_metadata = 0;
2491         char buf[BDEVNAME_SIZE];
2492         struct cache *cache = ti->private;
2493         dm_cblock_t residency;
2494
2495         switch (type) {
2496         case STATUSTYPE_INFO:
2497                 /* Commit to ensure statistics aren't out-of-date */
2498                 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) {
2499                         r = dm_cache_commit(cache->cmd, false);
2500                         if (r)
2501                                 DMERR("could not commit metadata for accurate status");
2502                 }
2503
2504                 r = dm_cache_get_free_metadata_block_count(cache->cmd,
2505                                                            &nr_free_blocks_metadata);
2506                 if (r) {
2507                         DMERR("could not get metadata free block count");
2508                         goto err;
2509                 }
2510
2511                 r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata);
2512                 if (r) {
2513                         DMERR("could not get metadata device size");
2514                         goto err;
2515                 }
2516
2517                 residency = policy_residency(cache->policy);
2518
2519                 DMEMIT("%llu/%llu %u %u %u %u %u %u %llu %u ",
2520                        (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
2521                        (unsigned long long)nr_blocks_metadata,
2522                        (unsigned) atomic_read(&cache->stats.read_hit),
2523                        (unsigned) atomic_read(&cache->stats.read_miss),
2524                        (unsigned) atomic_read(&cache->stats.write_hit),
2525                        (unsigned) atomic_read(&cache->stats.write_miss),
2526                        (unsigned) atomic_read(&cache->stats.demotion),
2527                        (unsigned) atomic_read(&cache->stats.promotion),
2528                        (unsigned long long) from_cblock(residency),
2529                        cache->nr_dirty);
2530
2531                 if (cache->features.write_through)
2532                         DMEMIT("1 writethrough ");
2533                 else
2534                         DMEMIT("0 ");
2535
2536                 DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold);
2537                 if (sz < maxlen) {
2538                         r = policy_emit_config_values(cache->policy, result + sz, maxlen - sz);
2539                         if (r)
2540                                 DMERR("policy_emit_config_values returned %d", r);
2541                 }
2542
2543                 break;
2544
2545         case STATUSTYPE_TABLE:
2546                 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
2547                 DMEMIT("%s ", buf);
2548                 format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
2549                 DMEMIT("%s ", buf);
2550                 format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
2551                 DMEMIT("%s", buf);
2552
2553                 for (i = 0; i < cache->nr_ctr_args - 1; i++)
2554                         DMEMIT(" %s", cache->ctr_args[i]);
2555                 if (cache->nr_ctr_args)
2556                         DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]);
2557         }
2558
2559         return;
2560
2561 err:
2562         DMEMIT("Error");
2563 }
2564
2565 /*
2566  * Supports <key> <value>.
2567  *
2568  * The key migration_threshold is supported by the cache target core.
2569  */
2570 static int cache_message(struct dm_target *ti, unsigned argc, char **argv)
2571 {
2572         struct cache *cache = ti->private;
2573
2574         if (argc != 2)
2575                 return -EINVAL;
2576
2577         return set_config_value(cache, argv[0], argv[1]);
2578 }
2579
2580 static int cache_iterate_devices(struct dm_target *ti,
2581                                  iterate_devices_callout_fn fn, void *data)
2582 {
2583         int r = 0;
2584         struct cache *cache = ti->private;
2585
2586         r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data);
2587         if (!r)
2588                 r = fn(ti, cache->origin_dev, 0, ti->len, data);
2589
2590         return r;
2591 }
2592
2593 /*
2594  * We assume I/O is going to the origin (which is the volume
2595  * more likely to have restrictions e.g. by being striped).
2596  * (Looking up the exact location of the data would be expensive
2597  * and could always be out of date by the time the bio is submitted.)
2598  */
2599 static int cache_bvec_merge(struct dm_target *ti,
2600                             struct bvec_merge_data *bvm,
2601                             struct bio_vec *biovec, int max_size)
2602 {
2603         struct cache *cache = ti->private;
2604         struct request_queue *q = bdev_get_queue(cache->origin_dev->bdev);
2605
2606         if (!q->merge_bvec_fn)
2607                 return max_size;
2608
2609         bvm->bi_bdev = cache->origin_dev->bdev;
2610         return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2611 }
2612
2613 static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
2614 {
2615         /*
2616          * FIXME: these limits may be incompatible with the cache device
2617          */
2618         limits->max_discard_sectors = cache->discard_block_size * 1024;
2619         limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
2620 }
2621
2622 static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
2623 {
2624         struct cache *cache = ti->private;
2625         uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
2626
2627         /*
2628          * If the system-determined stacked limits are compatible with the
2629          * cache's blocksize (io_opt is a factor) do not override them.
2630          */
2631         if (io_opt_sectors < cache->sectors_per_block ||
2632             do_div(io_opt_sectors, cache->sectors_per_block)) {
2633                 blk_limits_io_min(limits, 0);
2634                 blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT);
2635         }
2636         set_discard_limits(cache, limits);
2637 }
2638
2639 /*----------------------------------------------------------------*/
2640
2641 static struct target_type cache_target = {
2642         .name = "cache",
2643         .version = {1, 1, 1},
2644         .module = THIS_MODULE,
2645         .ctr = cache_ctr,
2646         .dtr = cache_dtr,
2647         .map = cache_map,
2648         .end_io = cache_end_io,
2649         .postsuspend = cache_postsuspend,
2650         .preresume = cache_preresume,
2651         .resume = cache_resume,
2652         .status = cache_status,
2653         .message = cache_message,
2654         .iterate_devices = cache_iterate_devices,
2655         .merge = cache_bvec_merge,
2656         .io_hints = cache_io_hints,
2657 };
2658
2659 static int __init dm_cache_init(void)
2660 {
2661         int r;
2662
2663         r = dm_register_target(&cache_target);
2664         if (r) {
2665                 DMERR("cache target registration failed: %d", r);
2666                 return r;
2667         }
2668
2669         migration_cache = KMEM_CACHE(dm_cache_migration, 0);
2670         if (!migration_cache) {
2671                 dm_unregister_target(&cache_target);
2672                 return -ENOMEM;
2673         }
2674
2675         return 0;
2676 }
2677
2678 static void __exit dm_cache_exit(void)
2679 {
2680         dm_unregister_target(&cache_target);
2681         kmem_cache_destroy(migration_cache);
2682 }
2683
2684 module_init(dm_cache_init);
2685 module_exit(dm_cache_exit);
2686
2687 MODULE_DESCRIPTION(DM_NAME " cache target");
2688 MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
2689 MODULE_LICENSE("GPL");