1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2009-2011 Red Hat, Inc.
5 * Author: Mikulas Patocka <mpatocka@redhat.com>
7 * This file is released under the GPL.
10 #include <linux/dm-bufio.h>
12 #include <linux/device-mapper.h>
13 #include <linux/dm-io.h>
14 #include <linux/slab.h>
15 #include <linux/sched/mm.h>
16 #include <linux/jiffies.h>
17 #include <linux/vmalloc.h>
18 #include <linux/shrinker.h>
19 #include <linux/module.h>
20 #include <linux/rbtree.h>
21 #include <linux/stacktrace.h>
22 #include <linux/jump_label.h>
26 #define DM_MSG_PREFIX "bufio"
29 * Memory management policy:
30 * Limit the number of buffers to DM_BUFIO_MEMORY_PERCENT of main memory
31 * or DM_BUFIO_VMALLOC_PERCENT of vmalloc memory (whichever is lower).
32 * Always allocate at least DM_BUFIO_MIN_BUFFERS buffers.
33 * Start background writeback when there are DM_BUFIO_WRITEBACK_PERCENT
36 #define DM_BUFIO_MIN_BUFFERS 8
38 #define DM_BUFIO_MEMORY_PERCENT 2
39 #define DM_BUFIO_VMALLOC_PERCENT 25
40 #define DM_BUFIO_WRITEBACK_RATIO 3
41 #define DM_BUFIO_LOW_WATERMARK_RATIO 16
44 * Check buffer ages in this interval (seconds)
46 #define DM_BUFIO_WORK_TIMER_SECS 30
49 * Free buffers when they are older than this (seconds)
51 #define DM_BUFIO_DEFAULT_AGE_SECS 300
54 * The nr of bytes of cached data to keep around.
56 #define DM_BUFIO_DEFAULT_RETAIN_BYTES (256 * 1024)
59 * Align buffer writes to this boundary.
60 * Tests show that SSDs have the highest IOPS when using 4k writes.
62 #define DM_BUFIO_WRITE_ALIGN 4096
65 * dm_buffer->list_mode
71 /*--------------------------------------------------------------*/
74 * Rather than use an LRU list, we use a clock algorithm where entries
75 * are held in a circular list. When an entry is 'hit' a reference bit
76 * is set. The least recently used entry is approximated by running a
77 * cursor around the list selecting unreferenced entries. Referenced
78 * entries have their reference bit cleared as the cursor passes them.
81 struct list_head list;
87 struct list_head list;
88 struct lru_entry *stop;
93 struct list_head *cursor;
96 struct list_head iterators;
101 static void lru_init(struct lru *lru)
105 INIT_LIST_HEAD(&lru->iterators);
108 static void lru_destroy(struct lru *lru)
110 WARN_ON_ONCE(lru->cursor);
111 WARN_ON_ONCE(!list_empty(&lru->iterators));
115 * Insert a new entry into the lru.
117 static void lru_insert(struct lru *lru, struct lru_entry *le)
120 * Don't be tempted to set to 1, makes the lru aspect
123 atomic_set(&le->referenced, 0);
126 list_add_tail(&le->list, lru->cursor);
128 INIT_LIST_HEAD(&le->list);
129 lru->cursor = &le->list;
137 * Convert a list_head pointer to an lru_entry pointer.
139 static inline struct lru_entry *to_le(struct list_head *l)
141 return container_of(l, struct lru_entry, list);
145 * Initialize an lru_iter and add it to the list of cursors in the lru.
147 static void lru_iter_begin(struct lru *lru, struct lru_iter *it)
150 it->stop = lru->cursor ? to_le(lru->cursor->prev) : NULL;
151 it->e = lru->cursor ? to_le(lru->cursor) : NULL;
152 list_add(&it->list, &lru->iterators);
156 * Remove an lru_iter from the list of cursors in the lru.
158 static inline void lru_iter_end(struct lru_iter *it)
163 /* Predicate function type to be used with lru_iter_next */
164 typedef bool (*iter_predicate)(struct lru_entry *le, void *context);
167 * Advance the cursor to the next entry that passes the
168 * predicate, and return that entry. Returns NULL if the
169 * iteration is complete.
171 static struct lru_entry *lru_iter_next(struct lru_iter *it,
172 iter_predicate pred, void *context)
179 /* advance the cursor */
180 if (it->e == it->stop)
183 it->e = to_le(it->e->list.next);
185 if (pred(e, context))
193 * Invalidate a specific lru_entry and update all cursors in
194 * the lru accordingly.
196 static void lru_iter_invalidate(struct lru *lru, struct lru_entry *e)
200 list_for_each_entry(it, &lru->iterators, list) {
201 /* Move c->e forwards if necc. */
203 it->e = to_le(it->e->list.next);
208 /* Move it->stop backwards if necc. */
210 it->stop = to_le(it->stop->list.prev);
220 * Remove a specific entry from the lru.
222 static void lru_remove(struct lru *lru, struct lru_entry *le)
224 lru_iter_invalidate(lru, le);
225 if (lru->count == 1) {
228 if (lru->cursor == &le->list)
229 lru->cursor = lru->cursor->next;
236 * Mark as referenced.
238 static inline void lru_reference(struct lru_entry *le)
240 atomic_set(&le->referenced, 1);
246 * Remove the least recently used entry (approx), that passes the predicate.
247 * Returns NULL on failure.
252 ER_STOP, /* stop looking for something to evict */
255 typedef enum evict_result (*le_predicate)(struct lru_entry *le, void *context);
257 static struct lru_entry *lru_evict(struct lru *lru, le_predicate pred, void *context)
259 unsigned long tested = 0;
260 struct list_head *h = lru->cursor;
261 struct lru_entry *le;
266 * In the worst case we have to loop around twice. Once to clear
267 * the reference flags, and then again to discover the predicate
268 * fails for all entries.
270 while (tested < lru->count) {
271 le = container_of(h, struct lru_entry, list);
273 if (atomic_read(&le->referenced)) {
274 atomic_set(&le->referenced, 0);
277 switch (pred(le, context)) {
280 * Adjust the cursor, so we start the next
283 lru->cursor = le->list.next;
291 lru->cursor = le->list.next;
304 /*--------------------------------------------------------------*/
314 * Describes how the block was allocated:
315 * kmem_cache_alloc(), __get_free_pages() or vmalloc().
316 * See the comment at alloc_buffer_data.
320 DATA_MODE_GET_FREE_PAGES = 1,
321 DATA_MODE_VMALLOC = 2,
326 /* protected by the locks in dm_buffer_cache */
329 /* immutable, so don't need protecting */
332 unsigned char data_mode; /* DATA_MODE_* */
335 * These two fields are used in isolation, so do not need
336 * a surrounding lock.
339 unsigned long last_accessed;
342 * Everything else is protected by the mutex in
346 struct lru_entry lru;
347 unsigned char list_mode; /* LIST_* */
348 blk_status_t read_error;
349 blk_status_t write_error;
350 unsigned int dirty_start;
351 unsigned int dirty_end;
352 unsigned int write_start;
353 unsigned int write_end;
354 struct list_head write_list;
355 struct dm_bufio_client *c;
356 void (*end_io)(struct dm_buffer *b, blk_status_t bs);
357 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
359 unsigned int stack_len;
360 unsigned long stack_entries[MAX_STACK];
364 /*--------------------------------------------------------------*/
367 * The buffer cache manages buffers, particularly:
368 * - inc/dec of holder count
369 * - setting the last_accessed field
370 * - maintains clean/dirty state along with lru
371 * - selecting buffers that match predicates
373 * It does *not* handle:
374 * - allocation/freeing of buffers.
376 * - Eviction or cache sizing.
378 * cache_get() and cache_put() are threadsafe, you do not need to
379 * protect these calls with a surrounding mutex. All the other
380 * methods are not threadsafe; they do use locking primitives, but
381 * only enough to ensure get/put are threadsafe.
385 struct rw_semaphore lock;
387 } ____cacheline_aligned_in_smp;
389 struct dm_buffer_cache {
390 struct lru lru[LIST_SIZE];
392 * We spread entries across multiple trees to reduce contention
395 unsigned int num_locks;
396 struct buffer_tree trees[];
399 static inline unsigned int cache_index(sector_t block, unsigned int num_locks)
401 return dm_hash_locks_index(block, num_locks);
404 static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block)
406 down_read(&bc->trees[cache_index(block, bc->num_locks)].lock);
409 static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t block)
411 up_read(&bc->trees[cache_index(block, bc->num_locks)].lock);
414 static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t block)
416 down_write(&bc->trees[cache_index(block, bc->num_locks)].lock);
419 static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t block)
421 up_write(&bc->trees[cache_index(block, bc->num_locks)].lock);
425 * Sometimes we want to repeatedly get and drop locks as part of an iteration.
426 * This struct helps avoid redundant drop and gets of the same lock.
428 struct lock_history {
429 struct dm_buffer_cache *cache;
431 unsigned int previous;
432 unsigned int no_previous;
435 static void lh_init(struct lock_history *lh, struct dm_buffer_cache *cache, bool write)
439 lh->no_previous = cache->num_locks;
440 lh->previous = lh->no_previous;
443 static void __lh_lock(struct lock_history *lh, unsigned int index)
446 down_write(&lh->cache->trees[index].lock);
448 down_read(&lh->cache->trees[index].lock);
451 static void __lh_unlock(struct lock_history *lh, unsigned int index)
454 up_write(&lh->cache->trees[index].lock);
456 up_read(&lh->cache->trees[index].lock);
460 * Make sure you call this since it will unlock the final lock.
462 static void lh_exit(struct lock_history *lh)
464 if (lh->previous != lh->no_previous) {
465 __lh_unlock(lh, lh->previous);
466 lh->previous = lh->no_previous;
471 * Named 'next' because there is no corresponding
472 * 'up/unlock' call since it's done automatically.
474 static void lh_next(struct lock_history *lh, sector_t b)
476 unsigned int index = cache_index(b, lh->no_previous); /* no_previous is num_locks */
478 if (lh->previous != lh->no_previous) {
479 if (lh->previous != index) {
480 __lh_unlock(lh, lh->previous);
481 __lh_lock(lh, index);
482 lh->previous = index;
485 __lh_lock(lh, index);
486 lh->previous = index;
490 static inline struct dm_buffer *le_to_buffer(struct lru_entry *le)
492 return container_of(le, struct dm_buffer, lru);
495 static struct dm_buffer *list_to_buffer(struct list_head *l)
497 struct lru_entry *le = list_entry(l, struct lru_entry, list);
502 return le_to_buffer(le);
505 static void cache_init(struct dm_buffer_cache *bc, unsigned int num_locks)
509 bc->num_locks = num_locks;
511 for (i = 0; i < bc->num_locks; i++) {
512 init_rwsem(&bc->trees[i].lock);
513 bc->trees[i].root = RB_ROOT;
516 lru_init(&bc->lru[LIST_CLEAN]);
517 lru_init(&bc->lru[LIST_DIRTY]);
520 static void cache_destroy(struct dm_buffer_cache *bc)
524 for (i = 0; i < bc->num_locks; i++)
525 WARN_ON_ONCE(!RB_EMPTY_ROOT(&bc->trees[i].root));
527 lru_destroy(&bc->lru[LIST_CLEAN]);
528 lru_destroy(&bc->lru[LIST_DIRTY]);
534 * not threadsafe, or racey depending how you look at it
536 static inline unsigned long cache_count(struct dm_buffer_cache *bc, int list_mode)
538 return bc->lru[list_mode].count;
541 static inline unsigned long cache_total(struct dm_buffer_cache *bc)
543 return cache_count(bc, LIST_CLEAN) + cache_count(bc, LIST_DIRTY);
549 * Gets a specific buffer, indexed by block.
550 * If the buffer is found then its holder count will be incremented and
551 * lru_reference will be called.
555 static struct dm_buffer *__cache_get(const struct rb_root *root, sector_t block)
557 struct rb_node *n = root->rb_node;
561 b = container_of(n, struct dm_buffer, node);
563 if (b->block == block)
566 n = block < b->block ? n->rb_left : n->rb_right;
572 static void __cache_inc_buffer(struct dm_buffer *b)
574 atomic_inc(&b->hold_count);
575 WRITE_ONCE(b->last_accessed, jiffies);
578 static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, sector_t block)
582 cache_read_lock(bc, block);
583 b = __cache_get(&bc->trees[cache_index(block, bc->num_locks)].root, block);
585 lru_reference(&b->lru);
586 __cache_inc_buffer(b);
588 cache_read_unlock(bc, block);
596 * Returns true if the hold count hits zero.
599 static bool cache_put(struct dm_buffer_cache *bc, struct dm_buffer *b)
603 cache_read_lock(bc, b->block);
604 BUG_ON(!atomic_read(&b->hold_count));
605 r = atomic_dec_and_test(&b->hold_count);
606 cache_read_unlock(bc, b->block);
613 typedef enum evict_result (*b_predicate)(struct dm_buffer *, void *);
616 * Evicts a buffer based on a predicate. The oldest buffer that
617 * matches the predicate will be selected. In addition to the
618 * predicate the hold_count of the selected buffer will be zero.
620 struct evict_wrapper {
621 struct lock_history *lh;
627 * Wraps the buffer predicate turning it into an lru predicate. Adds
628 * extra test for hold_count.
630 static enum evict_result __evict_pred(struct lru_entry *le, void *context)
632 struct evict_wrapper *w = context;
633 struct dm_buffer *b = le_to_buffer(le);
635 lh_next(w->lh, b->block);
637 if (atomic_read(&b->hold_count))
638 return ER_DONT_EVICT;
640 return w->pred(b, w->context);
643 static struct dm_buffer *__cache_evict(struct dm_buffer_cache *bc, int list_mode,
644 b_predicate pred, void *context,
645 struct lock_history *lh)
647 struct evict_wrapper w = {.lh = lh, .pred = pred, .context = context};
648 struct lru_entry *le;
651 le = lru_evict(&bc->lru[list_mode], __evict_pred, &w);
655 b = le_to_buffer(le);
656 /* __evict_pred will have locked the appropriate tree. */
657 rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root);
662 static struct dm_buffer *cache_evict(struct dm_buffer_cache *bc, int list_mode,
663 b_predicate pred, void *context)
666 struct lock_history lh;
668 lh_init(&lh, bc, true);
669 b = __cache_evict(bc, list_mode, pred, context, &lh);
678 * Mark a buffer as clean or dirty. Not threadsafe.
680 static void cache_mark(struct dm_buffer_cache *bc, struct dm_buffer *b, int list_mode)
682 cache_write_lock(bc, b->block);
683 if (list_mode != b->list_mode) {
684 lru_remove(&bc->lru[b->list_mode], &b->lru);
685 b->list_mode = list_mode;
686 lru_insert(&bc->lru[b->list_mode], &b->lru);
688 cache_write_unlock(bc, b->block);
694 * Runs through the lru associated with 'old_mode', if the predicate matches then
695 * it moves them to 'new_mode'. Not threadsafe.
697 static void __cache_mark_many(struct dm_buffer_cache *bc, int old_mode, int new_mode,
698 b_predicate pred, void *context, struct lock_history *lh)
700 struct lru_entry *le;
702 struct evict_wrapper w = {.lh = lh, .pred = pred, .context = context};
705 le = lru_evict(&bc->lru[old_mode], __evict_pred, &w);
709 b = le_to_buffer(le);
710 b->list_mode = new_mode;
711 lru_insert(&bc->lru[b->list_mode], &b->lru);
715 static void cache_mark_many(struct dm_buffer_cache *bc, int old_mode, int new_mode,
716 b_predicate pred, void *context)
718 struct lock_history lh;
720 lh_init(&lh, bc, true);
721 __cache_mark_many(bc, old_mode, new_mode, pred, context, &lh);
728 * Iterates through all clean or dirty entries calling a function for each
729 * entry. The callback may terminate the iteration early. Not threadsafe.
733 * Iterator functions should return one of these actions to indicate
734 * how the iteration should proceed.
741 typedef enum it_action (*iter_fn)(struct dm_buffer *b, void *context);
743 static void __cache_iterate(struct dm_buffer_cache *bc, int list_mode,
744 iter_fn fn, void *context, struct lock_history *lh)
746 struct lru *lru = &bc->lru[list_mode];
747 struct lru_entry *le, *first;
752 first = le = to_le(lru->cursor);
754 struct dm_buffer *b = le_to_buffer(le);
756 lh_next(lh, b->block);
758 switch (fn(b, context)) {
767 le = to_le(le->list.next);
768 } while (le != first);
771 static void cache_iterate(struct dm_buffer_cache *bc, int list_mode,
772 iter_fn fn, void *context)
774 struct lock_history lh;
776 lh_init(&lh, bc, false);
777 __cache_iterate(bc, list_mode, fn, context, &lh);
784 * Passes ownership of the buffer to the cache. Returns false if the
785 * buffer was already present (in which case ownership does not pass).
786 * eg, a race with another thread.
788 * Holder count should be 1 on insertion.
792 static bool __cache_insert(struct rb_root *root, struct dm_buffer *b)
794 struct rb_node **new = &root->rb_node, *parent = NULL;
795 struct dm_buffer *found;
798 found = container_of(*new, struct dm_buffer, node);
800 if (found->block == b->block)
804 new = b->block < found->block ?
805 &found->node.rb_left : &found->node.rb_right;
808 rb_link_node(&b->node, parent, new);
809 rb_insert_color(&b->node, root);
814 static bool cache_insert(struct dm_buffer_cache *bc, struct dm_buffer *b)
818 if (WARN_ON_ONCE(b->list_mode >= LIST_SIZE))
821 cache_write_lock(bc, b->block);
822 BUG_ON(atomic_read(&b->hold_count) != 1);
823 r = __cache_insert(&bc->trees[cache_index(b->block, bc->num_locks)].root, b);
825 lru_insert(&bc->lru[b->list_mode], &b->lru);
826 cache_write_unlock(bc, b->block);
834 * Removes buffer from cache, ownership of the buffer passes back to the caller.
835 * Fails if the hold_count is not one (ie. the caller holds the only reference).
839 static bool cache_remove(struct dm_buffer_cache *bc, struct dm_buffer *b)
843 cache_write_lock(bc, b->block);
845 if (atomic_read(&b->hold_count) != 1) {
849 rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root);
850 lru_remove(&bc->lru[b->list_mode], &b->lru);
853 cache_write_unlock(bc, b->block);
860 typedef void (*b_release)(struct dm_buffer *);
862 static struct dm_buffer *__find_next(struct rb_root *root, sector_t block)
864 struct rb_node *n = root->rb_node;
866 struct dm_buffer *best = NULL;
869 b = container_of(n, struct dm_buffer, node);
871 if (b->block == block)
874 if (block <= b->block) {
885 static void __remove_range(struct dm_buffer_cache *bc,
886 struct rb_root *root,
887 sector_t begin, sector_t end,
888 b_predicate pred, b_release release)
895 b = __find_next(root, begin);
896 if (!b || (b->block >= end))
899 begin = b->block + 1;
901 if (atomic_read(&b->hold_count))
904 if (pred(b, NULL) == ER_EVICT) {
905 rb_erase(&b->node, root);
906 lru_remove(&bc->lru[b->list_mode], &b->lru);
912 static void cache_remove_range(struct dm_buffer_cache *bc,
913 sector_t begin, sector_t end,
914 b_predicate pred, b_release release)
918 for (i = 0; i < bc->num_locks; i++) {
919 down_write(&bc->trees[i].lock);
920 __remove_range(bc, &bc->trees[i].root, begin, end, pred, release);
921 up_write(&bc->trees[i].lock);
925 /*----------------------------------------------------------------*/
928 * Linking of buffers:
929 * All buffers are linked to buffer_cache with their node field.
931 * Clean buffers that are not being written (B_WRITING not set)
932 * are linked to lru[LIST_CLEAN] with their lru_list field.
934 * Dirty and clean buffers that are being written are linked to
935 * lru[LIST_DIRTY] with their lru_list field. When the write
936 * finishes, the buffer cannot be relinked immediately (because we
937 * are in an interrupt context and relinking requires process
938 * context), so some clean-not-writing buffers can be held on
939 * dirty_lru too. They are later added to lru in the process
942 struct dm_bufio_client {
943 struct block_device *bdev;
944 unsigned int block_size;
945 s8 sectors_per_block_bits;
951 int async_write_error;
953 void (*alloc_callback)(struct dm_buffer *buf);
954 void (*write_callback)(struct dm_buffer *buf);
955 struct kmem_cache *slab_buffer;
956 struct kmem_cache *slab_cache;
957 struct dm_io_client *dm_io;
959 struct list_head reserved_buffers;
960 unsigned int need_reserved_buffers;
962 unsigned int minimum_buffers;
966 struct shrinker shrinker;
967 struct work_struct shrink_work;
968 atomic_long_t need_shrink;
970 wait_queue_head_t free_buffer_wait;
972 struct list_head client_list;
975 * Used by global_cleanup to sort the clients list.
977 unsigned long oldest_buffer;
979 struct dm_buffer_cache cache; /* must be last member */
982 static DEFINE_STATIC_KEY_FALSE(no_sleep_enabled);
984 /*----------------------------------------------------------------*/
986 #define dm_bufio_in_request() (!!current->bio_list)
988 static void dm_bufio_lock(struct dm_bufio_client *c)
990 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
991 spin_lock_bh(&c->spinlock);
993 mutex_lock_nested(&c->lock, dm_bufio_in_request());
996 static void dm_bufio_unlock(struct dm_bufio_client *c)
998 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
999 spin_unlock_bh(&c->spinlock);
1001 mutex_unlock(&c->lock);
1004 /*----------------------------------------------------------------*/
1007 * Default cache size: available memory divided by the ratio.
1009 static unsigned long dm_bufio_default_cache_size;
1012 * Total cache size set by the user.
1014 static unsigned long dm_bufio_cache_size;
1017 * A copy of dm_bufio_cache_size because dm_bufio_cache_size can change
1018 * at any time. If it disagrees, the user has changed cache size.
1020 static unsigned long dm_bufio_cache_size_latch;
1022 static DEFINE_SPINLOCK(global_spinlock);
1025 * Buffers are freed after this timeout
1027 static unsigned int dm_bufio_max_age = DM_BUFIO_DEFAULT_AGE_SECS;
1028 static unsigned long dm_bufio_retain_bytes = DM_BUFIO_DEFAULT_RETAIN_BYTES;
1030 static unsigned long dm_bufio_peak_allocated;
1031 static unsigned long dm_bufio_allocated_kmem_cache;
1032 static unsigned long dm_bufio_allocated_get_free_pages;
1033 static unsigned long dm_bufio_allocated_vmalloc;
1034 static unsigned long dm_bufio_current_allocated;
1036 /*----------------------------------------------------------------*/
1039 * The current number of clients.
1041 static int dm_bufio_client_count;
1044 * The list of all clients.
1046 static LIST_HEAD(dm_bufio_all_clients);
1049 * This mutex protects dm_bufio_cache_size_latch and dm_bufio_client_count
1051 static DEFINE_MUTEX(dm_bufio_clients_lock);
1053 static struct workqueue_struct *dm_bufio_wq;
1054 static struct delayed_work dm_bufio_cleanup_old_work;
1055 static struct work_struct dm_bufio_replacement_work;
1058 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
1059 static void buffer_record_stack(struct dm_buffer *b)
1061 b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2);
1065 /*----------------------------------------------------------------*/
1067 static void adjust_total_allocated(struct dm_buffer *b, bool unlink)
1069 unsigned char data_mode;
1072 static unsigned long * const class_ptr[DATA_MODE_LIMIT] = {
1073 &dm_bufio_allocated_kmem_cache,
1074 &dm_bufio_allocated_get_free_pages,
1075 &dm_bufio_allocated_vmalloc,
1078 data_mode = b->data_mode;
1079 diff = (long)b->c->block_size;
1083 spin_lock(&global_spinlock);
1085 *class_ptr[data_mode] += diff;
1087 dm_bufio_current_allocated += diff;
1089 if (dm_bufio_current_allocated > dm_bufio_peak_allocated)
1090 dm_bufio_peak_allocated = dm_bufio_current_allocated;
1093 if (dm_bufio_current_allocated > dm_bufio_cache_size)
1094 queue_work(dm_bufio_wq, &dm_bufio_replacement_work);
1097 spin_unlock(&global_spinlock);
1101 * Change the number of clients and recalculate per-client limit.
1103 static void __cache_size_refresh(void)
1105 if (WARN_ON(!mutex_is_locked(&dm_bufio_clients_lock)))
1107 if (WARN_ON(dm_bufio_client_count < 0))
1110 dm_bufio_cache_size_latch = READ_ONCE(dm_bufio_cache_size);
1113 * Use default if set to 0 and report the actual cache size used.
1115 if (!dm_bufio_cache_size_latch) {
1116 (void)cmpxchg(&dm_bufio_cache_size, 0,
1117 dm_bufio_default_cache_size);
1118 dm_bufio_cache_size_latch = dm_bufio_default_cache_size;
1123 * Allocating buffer data.
1125 * Small buffers are allocated with kmem_cache, to use space optimally.
1127 * For large buffers, we choose between get_free_pages and vmalloc.
1128 * Each has advantages and disadvantages.
1130 * __get_free_pages can randomly fail if the memory is fragmented.
1131 * __vmalloc won't randomly fail, but vmalloc space is limited (it may be
1132 * as low as 128M) so using it for caching is not appropriate.
1134 * If the allocation may fail we use __get_free_pages. Memory fragmentation
1135 * won't have a fatal effect here, but it just causes flushes of some other
1136 * buffers and more I/O will be performed. Don't use __get_free_pages if it
1137 * always fails (i.e. order > MAX_ORDER).
1139 * If the allocation shouldn't fail we use __vmalloc. This is only for the
1140 * initial reserve allocation, so there's no risk of wasting all vmalloc
1143 static void *alloc_buffer_data(struct dm_bufio_client *c, gfp_t gfp_mask,
1144 unsigned char *data_mode)
1146 if (unlikely(c->slab_cache != NULL)) {
1147 *data_mode = DATA_MODE_SLAB;
1148 return kmem_cache_alloc(c->slab_cache, gfp_mask);
1151 if (c->block_size <= KMALLOC_MAX_SIZE &&
1152 gfp_mask & __GFP_NORETRY) {
1153 *data_mode = DATA_MODE_GET_FREE_PAGES;
1154 return (void *)__get_free_pages(gfp_mask,
1155 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT));
1158 *data_mode = DATA_MODE_VMALLOC;
1160 return __vmalloc(c->block_size, gfp_mask);
1164 * Free buffer's data.
1166 static void free_buffer_data(struct dm_bufio_client *c,
1167 void *data, unsigned char data_mode)
1169 switch (data_mode) {
1170 case DATA_MODE_SLAB:
1171 kmem_cache_free(c->slab_cache, data);
1174 case DATA_MODE_GET_FREE_PAGES:
1175 free_pages((unsigned long)data,
1176 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT));
1179 case DATA_MODE_VMALLOC:
1184 DMCRIT("dm_bufio_free_buffer_data: bad data mode: %d",
1191 * Allocate buffer and its data.
1193 static struct dm_buffer *alloc_buffer(struct dm_bufio_client *c, gfp_t gfp_mask)
1195 struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask);
1202 b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode);
1204 kmem_cache_free(c->slab_buffer, b);
1207 adjust_total_allocated(b, false);
1209 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
1216 * Free buffer and its data.
1218 static void free_buffer(struct dm_buffer *b)
1220 struct dm_bufio_client *c = b->c;
1222 adjust_total_allocated(b, true);
1223 free_buffer_data(c, b->data, b->data_mode);
1224 kmem_cache_free(c->slab_buffer, b);
1228 *--------------------------------------------------------------------------
1229 * Submit I/O on the buffer.
1231 * Bio interface is faster but it has some problems:
1232 * the vector list is limited (increasing this limit increases
1233 * memory-consumption per buffer, so it is not viable);
1235 * the memory must be direct-mapped, not vmalloced;
1237 * If the buffer is small enough (up to DM_BUFIO_INLINE_VECS pages) and
1238 * it is not vmalloced, try using the bio interface.
1240 * If the buffer is big, if it is vmalloced or if the underlying device
1241 * rejects the bio because it is too large, use dm-io layer to do the I/O.
1242 * The dm-io layer splits the I/O into multiple requests, avoiding the above
1244 *--------------------------------------------------------------------------
1248 * dm-io completion routine. It just calls b->bio.bi_end_io, pretending
1249 * that the request was handled directly with bio interface.
1251 static void dmio_complete(unsigned long error, void *context)
1253 struct dm_buffer *b = context;
1255 b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0);
1258 static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector,
1259 unsigned int n_sectors, unsigned int offset)
1262 struct dm_io_request io_req = {
1264 .notify.fn = dmio_complete,
1265 .notify.context = b,
1266 .client = b->c->dm_io,
1268 struct dm_io_region region = {
1274 if (b->data_mode != DATA_MODE_VMALLOC) {
1275 io_req.mem.type = DM_IO_KMEM;
1276 io_req.mem.ptr.addr = (char *)b->data + offset;
1278 io_req.mem.type = DM_IO_VMA;
1279 io_req.mem.ptr.vma = (char *)b->data + offset;
1282 r = dm_io(&io_req, 1, ®ion, NULL);
1284 b->end_io(b, errno_to_blk_status(r));
1287 static void bio_complete(struct bio *bio)
1289 struct dm_buffer *b = bio->bi_private;
1290 blk_status_t status = bio->bi_status;
1294 b->end_io(b, status);
1297 static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector,
1298 unsigned int n_sectors, unsigned int offset)
1304 bio = bio_kmalloc(1, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN);
1306 use_dmio(b, op, sector, n_sectors, offset);
1309 bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op);
1310 bio->bi_iter.bi_sector = sector;
1311 bio->bi_end_io = bio_complete;
1312 bio->bi_private = b;
1314 ptr = (char *)b->data + offset;
1315 len = n_sectors << SECTOR_SHIFT;
1317 __bio_add_page(bio, virt_to_page(ptr), len, offset_in_page(ptr));
1322 static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block)
1326 if (likely(c->sectors_per_block_bits >= 0))
1327 sector = block << c->sectors_per_block_bits;
1329 sector = block * (c->block_size >> SECTOR_SHIFT);
1335 static void submit_io(struct dm_buffer *b, enum req_op op,
1336 void (*end_io)(struct dm_buffer *, blk_status_t))
1338 unsigned int n_sectors;
1340 unsigned int offset, end;
1344 sector = block_to_sector(b->c, b->block);
1346 if (op != REQ_OP_WRITE) {
1347 n_sectors = b->c->block_size >> SECTOR_SHIFT;
1350 if (b->c->write_callback)
1351 b->c->write_callback(b);
1352 offset = b->write_start;
1354 offset &= -DM_BUFIO_WRITE_ALIGN;
1355 end += DM_BUFIO_WRITE_ALIGN - 1;
1356 end &= -DM_BUFIO_WRITE_ALIGN;
1357 if (unlikely(end > b->c->block_size))
1358 end = b->c->block_size;
1360 sector += offset >> SECTOR_SHIFT;
1361 n_sectors = (end - offset) >> SECTOR_SHIFT;
1364 if (b->data_mode != DATA_MODE_VMALLOC)
1365 use_bio(b, op, sector, n_sectors, offset);
1367 use_dmio(b, op, sector, n_sectors, offset);
1371 *--------------------------------------------------------------
1372 * Writing dirty buffers
1373 *--------------------------------------------------------------
1377 * The endio routine for write.
1379 * Set the error, clear B_WRITING bit and wake anyone who was waiting on
1382 static void write_endio(struct dm_buffer *b, blk_status_t status)
1384 b->write_error = status;
1385 if (unlikely(status)) {
1386 struct dm_bufio_client *c = b->c;
1388 (void)cmpxchg(&c->async_write_error, 0,
1389 blk_status_to_errno(status));
1392 BUG_ON(!test_bit(B_WRITING, &b->state));
1394 smp_mb__before_atomic();
1395 clear_bit(B_WRITING, &b->state);
1396 smp_mb__after_atomic();
1398 wake_up_bit(&b->state, B_WRITING);
1402 * Initiate a write on a dirty buffer, but don't wait for it.
1404 * - If the buffer is not dirty, exit.
1405 * - If there some previous write going on, wait for it to finish (we can't
1406 * have two writes on the same buffer simultaneously).
1407 * - Submit our write and don't wait on it. We set B_WRITING indicating
1408 * that there is a write in progress.
1410 static void __write_dirty_buffer(struct dm_buffer *b,
1411 struct list_head *write_list)
1413 if (!test_bit(B_DIRTY, &b->state))
1416 clear_bit(B_DIRTY, &b->state);
1417 wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
1419 b->write_start = b->dirty_start;
1420 b->write_end = b->dirty_end;
1423 submit_io(b, REQ_OP_WRITE, write_endio);
1425 list_add_tail(&b->write_list, write_list);
1428 static void __flush_write_list(struct list_head *write_list)
1430 struct blk_plug plug;
1432 blk_start_plug(&plug);
1433 while (!list_empty(write_list)) {
1434 struct dm_buffer *b =
1435 list_entry(write_list->next, struct dm_buffer, write_list);
1436 list_del(&b->write_list);
1437 submit_io(b, REQ_OP_WRITE, write_endio);
1440 blk_finish_plug(&plug);
1444 * Wait until any activity on the buffer finishes. Possibly write the
1445 * buffer if it is dirty. When this function finishes, there is no I/O
1446 * running on the buffer and the buffer is not dirty.
1448 static void __make_buffer_clean(struct dm_buffer *b)
1450 BUG_ON(atomic_read(&b->hold_count));
1452 /* smp_load_acquire() pairs with read_endio()'s smp_mb__before_atomic() */
1453 if (!smp_load_acquire(&b->state)) /* fast case */
1456 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
1457 __write_dirty_buffer(b, NULL);
1458 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
1461 static enum evict_result is_clean(struct dm_buffer *b, void *context)
1463 struct dm_bufio_client *c = context;
1465 /* These should never happen */
1466 if (WARN_ON_ONCE(test_bit(B_WRITING, &b->state)))
1467 return ER_DONT_EVICT;
1468 if (WARN_ON_ONCE(test_bit(B_DIRTY, &b->state)))
1469 return ER_DONT_EVICT;
1470 if (WARN_ON_ONCE(b->list_mode != LIST_CLEAN))
1471 return ER_DONT_EVICT;
1473 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep &&
1474 unlikely(test_bit(B_READING, &b->state)))
1475 return ER_DONT_EVICT;
1480 static enum evict_result is_dirty(struct dm_buffer *b, void *context)
1482 /* These should never happen */
1483 if (WARN_ON_ONCE(test_bit(B_READING, &b->state)))
1484 return ER_DONT_EVICT;
1485 if (WARN_ON_ONCE(b->list_mode != LIST_DIRTY))
1486 return ER_DONT_EVICT;
1492 * Find some buffer that is not held by anybody, clean it, unlink it and
1495 static struct dm_buffer *__get_unclaimed_buffer(struct dm_bufio_client *c)
1497 struct dm_buffer *b;
1499 b = cache_evict(&c->cache, LIST_CLEAN, is_clean, c);
1501 /* this also waits for pending reads */
1502 __make_buffer_clean(b);
1506 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
1509 b = cache_evict(&c->cache, LIST_DIRTY, is_dirty, NULL);
1511 __make_buffer_clean(b);
1519 * Wait until some other threads free some buffer or release hold count on
1522 * This function is entered with c->lock held, drops it and regains it
1525 static void __wait_for_free_buffer(struct dm_bufio_client *c)
1527 DECLARE_WAITQUEUE(wait, current);
1529 add_wait_queue(&c->free_buffer_wait, &wait);
1530 set_current_state(TASK_UNINTERRUPTIBLE);
1534 * It's possible to miss a wake up event since we don't always
1535 * hold c->lock when wake_up is called. So we have a timeout here,
1538 io_schedule_timeout(5 * HZ);
1540 remove_wait_queue(&c->free_buffer_wait, &wait);
1553 * Allocate a new buffer. If the allocation is not possible, wait until
1554 * some other thread frees a buffer.
1556 * May drop the lock and regain it.
1558 static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
1560 struct dm_buffer *b;
1561 bool tried_noio_alloc = false;
1564 * dm-bufio is resistant to allocation failures (it just keeps
1565 * one buffer reserved in cases all the allocations fail).
1566 * So set flags to not try too hard:
1567 * GFP_NOWAIT: don't wait; if we need to sleep we'll release our
1568 * mutex and wait ourselves.
1569 * __GFP_NORETRY: don't retry and rather return failure
1570 * __GFP_NOMEMALLOC: don't use emergency reserves
1571 * __GFP_NOWARN: don't print a warning in case of failure
1573 * For debugging, if we set the cache size to 1, no new buffers will
1577 if (dm_bufio_cache_size_latch != 1) {
1578 b = alloc_buffer(c, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
1583 if (nf == NF_PREFETCH)
1586 if (dm_bufio_cache_size_latch != 1 && !tried_noio_alloc) {
1588 b = alloc_buffer(c, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
1592 tried_noio_alloc = true;
1595 if (!list_empty(&c->reserved_buffers)) {
1596 b = list_to_buffer(c->reserved_buffers.next);
1597 list_del(&b->lru.list);
1598 c->need_reserved_buffers++;
1603 b = __get_unclaimed_buffer(c);
1607 __wait_for_free_buffer(c);
1611 static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf)
1613 struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
1618 if (c->alloc_callback)
1619 c->alloc_callback(b);
1625 * Free a buffer and wake other threads waiting for free buffers.
1627 static void __free_buffer_wake(struct dm_buffer *b)
1629 struct dm_bufio_client *c = b->c;
1632 if (!c->need_reserved_buffers)
1635 list_add(&b->lru.list, &c->reserved_buffers);
1636 c->need_reserved_buffers--;
1640 * We hold the bufio lock here, so no one can add entries to the
1641 * wait queue anyway.
1643 if (unlikely(waitqueue_active(&c->free_buffer_wait)))
1644 wake_up(&c->free_buffer_wait);
1647 static enum evict_result cleaned(struct dm_buffer *b, void *context)
1649 if (WARN_ON_ONCE(test_bit(B_READING, &b->state)))
1650 return ER_DONT_EVICT; /* should never happen */
1652 if (test_bit(B_DIRTY, &b->state) || test_bit(B_WRITING, &b->state))
1653 return ER_DONT_EVICT;
1658 static void __move_clean_buffers(struct dm_bufio_client *c)
1660 cache_mark_many(&c->cache, LIST_DIRTY, LIST_CLEAN, cleaned, NULL);
1663 struct write_context {
1665 struct list_head *write_list;
1668 static enum it_action write_one(struct dm_buffer *b, void *context)
1670 struct write_context *wc = context;
1672 if (wc->no_wait && test_bit(B_WRITING, &b->state))
1675 __write_dirty_buffer(b, wc->write_list);
1679 static void __write_dirty_buffers_async(struct dm_bufio_client *c, int no_wait,
1680 struct list_head *write_list)
1682 struct write_context wc = {.no_wait = no_wait, .write_list = write_list};
1684 __move_clean_buffers(c);
1685 cache_iterate(&c->cache, LIST_DIRTY, write_one, &wc);
1689 * Check if we're over watermark.
1690 * If we are over threshold_buffers, start freeing buffers.
1691 * If we're over "limit_buffers", block until we get under the limit.
1693 static void __check_watermark(struct dm_bufio_client *c,
1694 struct list_head *write_list)
1696 if (cache_count(&c->cache, LIST_DIRTY) >
1697 cache_count(&c->cache, LIST_CLEAN) * DM_BUFIO_WRITEBACK_RATIO)
1698 __write_dirty_buffers_async(c, 1, write_list);
1702 *--------------------------------------------------------------
1704 *--------------------------------------------------------------
1707 static void cache_put_and_wake(struct dm_bufio_client *c, struct dm_buffer *b)
1710 * Relying on waitqueue_active() is racey, but we sleep
1711 * with schedule_timeout anyway.
1713 if (cache_put(&c->cache, b) &&
1714 unlikely(waitqueue_active(&c->free_buffer_wait)))
1715 wake_up(&c->free_buffer_wait);
1719 * This assumes you have already checked the cache to see if the buffer
1720 * is already present (it will recheck after dropping the lock for allocation).
1722 static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
1723 enum new_flag nf, int *need_submit,
1724 struct list_head *write_list)
1726 struct dm_buffer *b, *new_b = NULL;
1730 /* This can't be called with NF_GET */
1731 if (WARN_ON_ONCE(nf == NF_GET))
1734 new_b = __alloc_buffer_wait(c, nf);
1739 * We've had a period where the mutex was unlocked, so need to
1740 * recheck the buffer tree.
1742 b = cache_get(&c->cache, block);
1744 __free_buffer_wake(new_b);
1748 __check_watermark(c, write_list);
1751 atomic_set(&b->hold_count, 1);
1752 WRITE_ONCE(b->last_accessed, jiffies);
1756 b->list_mode = LIST_CLEAN;
1761 b->state = 1 << B_READING;
1766 * We mustn't insert into the cache until the B_READING state
1767 * is set. Otherwise another thread could get it and use
1768 * it before it had been read.
1770 cache_insert(&c->cache, b);
1775 if (nf == NF_PREFETCH) {
1776 cache_put_and_wake(c, b);
1781 * Note: it is essential that we don't wait for the buffer to be
1782 * read if dm_bufio_get function is used. Both dm_bufio_get and
1783 * dm_bufio_prefetch can be used in the driver request routine.
1784 * If the user called both dm_bufio_prefetch and dm_bufio_get on
1785 * the same buffer, it would deadlock if we waited.
1787 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) {
1788 cache_put_and_wake(c, b);
1796 * The endio routine for reading: set the error, clear the bit and wake up
1797 * anyone waiting on the buffer.
1799 static void read_endio(struct dm_buffer *b, blk_status_t status)
1801 b->read_error = status;
1803 BUG_ON(!test_bit(B_READING, &b->state));
1805 smp_mb__before_atomic();
1806 clear_bit(B_READING, &b->state);
1807 smp_mb__after_atomic();
1809 wake_up_bit(&b->state, B_READING);
1813 * A common routine for dm_bufio_new and dm_bufio_read. Operation of these
1814 * functions is similar except that dm_bufio_new doesn't read the
1815 * buffer from the disk (assuming that the caller overwrites all the data
1816 * and uses dm_bufio_mark_buffer_dirty to write new data back).
1818 static void *new_read(struct dm_bufio_client *c, sector_t block,
1819 enum new_flag nf, struct dm_buffer **bp)
1821 int need_submit = 0;
1822 struct dm_buffer *b;
1824 LIST_HEAD(write_list);
1829 * Fast path, hopefully the block is already in the cache. No need
1830 * to get the client lock for this.
1832 b = cache_get(&c->cache, block);
1834 if (nf == NF_PREFETCH) {
1835 cache_put_and_wake(c, b);
1840 * Note: it is essential that we don't wait for the buffer to be
1841 * read if dm_bufio_get function is used. Both dm_bufio_get and
1842 * dm_bufio_prefetch can be used in the driver request routine.
1843 * If the user called both dm_bufio_prefetch and dm_bufio_get on
1844 * the same buffer, it would deadlock if we waited.
1846 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) {
1847 cache_put_and_wake(c, b);
1857 b = __bufio_new(c, block, nf, &need_submit, &write_list);
1861 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
1862 if (b && (atomic_read(&b->hold_count) == 1))
1863 buffer_record_stack(b);
1866 __flush_write_list(&write_list);
1872 submit_io(b, REQ_OP_READ, read_endio);
1874 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
1876 if (b->read_error) {
1877 int error = blk_status_to_errno(b->read_error);
1879 dm_bufio_release(b);
1881 return ERR_PTR(error);
1889 void *dm_bufio_get(struct dm_bufio_client *c, sector_t block,
1890 struct dm_buffer **bp)
1892 return new_read(c, block, NF_GET, bp);
1894 EXPORT_SYMBOL_GPL(dm_bufio_get);
1896 void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
1897 struct dm_buffer **bp)
1899 if (WARN_ON_ONCE(dm_bufio_in_request()))
1900 return ERR_PTR(-EINVAL);
1902 return new_read(c, block, NF_READ, bp);
1904 EXPORT_SYMBOL_GPL(dm_bufio_read);
1906 void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
1907 struct dm_buffer **bp)
1909 if (WARN_ON_ONCE(dm_bufio_in_request()))
1910 return ERR_PTR(-EINVAL);
1912 return new_read(c, block, NF_FRESH, bp);
1914 EXPORT_SYMBOL_GPL(dm_bufio_new);
1916 void dm_bufio_prefetch(struct dm_bufio_client *c,
1917 sector_t block, unsigned int n_blocks)
1919 struct blk_plug plug;
1921 LIST_HEAD(write_list);
1923 if (WARN_ON_ONCE(dm_bufio_in_request()))
1924 return; /* should never happen */
1926 blk_start_plug(&plug);
1928 for (; n_blocks--; block++) {
1930 struct dm_buffer *b;
1932 b = cache_get(&c->cache, block);
1934 /* already in cache */
1935 cache_put_and_wake(c, b);
1940 b = __bufio_new(c, block, NF_PREFETCH, &need_submit,
1942 if (unlikely(!list_empty(&write_list))) {
1944 blk_finish_plug(&plug);
1945 __flush_write_list(&write_list);
1946 blk_start_plug(&plug);
1949 if (unlikely(b != NULL)) {
1953 submit_io(b, REQ_OP_READ, read_endio);
1954 dm_bufio_release(b);
1966 blk_finish_plug(&plug);
1968 EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
1970 void dm_bufio_release(struct dm_buffer *b)
1972 struct dm_bufio_client *c = b->c;
1975 * If there were errors on the buffer, and the buffer is not
1976 * to be written, free the buffer. There is no point in caching
1979 if ((b->read_error || b->write_error) &&
1980 !test_bit_acquire(B_READING, &b->state) &&
1981 !test_bit(B_WRITING, &b->state) &&
1982 !test_bit(B_DIRTY, &b->state)) {
1985 /* cache remove can fail if there are other holders */
1986 if (cache_remove(&c->cache, b)) {
1987 __free_buffer_wake(b);
1995 cache_put_and_wake(c, b);
1997 EXPORT_SYMBOL_GPL(dm_bufio_release);
1999 void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b,
2000 unsigned int start, unsigned int end)
2002 struct dm_bufio_client *c = b->c;
2004 BUG_ON(start >= end);
2005 BUG_ON(end > b->c->block_size);
2009 BUG_ON(test_bit(B_READING, &b->state));
2011 if (!test_and_set_bit(B_DIRTY, &b->state)) {
2012 b->dirty_start = start;
2014 cache_mark(&c->cache, b, LIST_DIRTY);
2016 if (start < b->dirty_start)
2017 b->dirty_start = start;
2018 if (end > b->dirty_end)
2024 EXPORT_SYMBOL_GPL(dm_bufio_mark_partial_buffer_dirty);
2026 void dm_bufio_mark_buffer_dirty(struct dm_buffer *b)
2028 dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size);
2030 EXPORT_SYMBOL_GPL(dm_bufio_mark_buffer_dirty);
2032 void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c)
2034 LIST_HEAD(write_list);
2036 if (WARN_ON_ONCE(dm_bufio_in_request()))
2037 return; /* should never happen */
2040 __write_dirty_buffers_async(c, 0, &write_list);
2042 __flush_write_list(&write_list);
2044 EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers_async);
2047 * For performance, it is essential that the buffers are written asynchronously
2048 * and simultaneously (so that the block layer can merge the writes) and then
2051 * Finally, we flush hardware disk cache.
2053 static bool is_writing(struct lru_entry *e, void *context)
2055 struct dm_buffer *b = le_to_buffer(e);
2057 return test_bit(B_WRITING, &b->state);
2060 int dm_bufio_write_dirty_buffers(struct dm_bufio_client *c)
2063 unsigned long nr_buffers;
2064 struct lru_entry *e;
2067 LIST_HEAD(write_list);
2070 __write_dirty_buffers_async(c, 0, &write_list);
2072 __flush_write_list(&write_list);
2075 nr_buffers = cache_count(&c->cache, LIST_DIRTY);
2076 lru_iter_begin(&c->cache.lru[LIST_DIRTY], &it);
2077 while ((e = lru_iter_next(&it, is_writing, c))) {
2078 struct dm_buffer *b = le_to_buffer(e);
2079 __cache_inc_buffer(b);
2081 BUG_ON(test_bit(B_READING, &b->state));
2086 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
2089 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
2092 if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state))
2093 cache_mark(&c->cache, b, LIST_CLEAN);
2095 cache_put_and_wake(c, b);
2101 wake_up(&c->free_buffer_wait);
2104 a = xchg(&c->async_write_error, 0);
2105 f = dm_bufio_issue_flush(c);
2111 EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers);
2114 * Use dm-io to send an empty barrier to flush the device.
2116 int dm_bufio_issue_flush(struct dm_bufio_client *c)
2118 struct dm_io_request io_req = {
2119 .bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC,
2120 .mem.type = DM_IO_KMEM,
2121 .mem.ptr.addr = NULL,
2124 struct dm_io_region io_reg = {
2130 if (WARN_ON_ONCE(dm_bufio_in_request()))
2133 return dm_io(&io_req, 1, &io_reg, NULL);
2135 EXPORT_SYMBOL_GPL(dm_bufio_issue_flush);
2138 * Use dm-io to send a discard request to flush the device.
2140 int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t count)
2142 struct dm_io_request io_req = {
2143 .bi_opf = REQ_OP_DISCARD | REQ_SYNC,
2144 .mem.type = DM_IO_KMEM,
2145 .mem.ptr.addr = NULL,
2148 struct dm_io_region io_reg = {
2150 .sector = block_to_sector(c, block),
2151 .count = block_to_sector(c, count),
2154 if (WARN_ON_ONCE(dm_bufio_in_request()))
2155 return -EINVAL; /* discards are optional */
2157 return dm_io(&io_req, 1, &io_reg, NULL);
2159 EXPORT_SYMBOL_GPL(dm_bufio_issue_discard);
2161 static bool forget_buffer(struct dm_bufio_client *c, sector_t block)
2163 struct dm_buffer *b;
2165 b = cache_get(&c->cache, block);
2167 if (likely(!smp_load_acquire(&b->state))) {
2168 if (cache_remove(&c->cache, b))
2169 __free_buffer_wake(b);
2171 cache_put_and_wake(c, b);
2173 cache_put_and_wake(c, b);
2177 return b ? true : false;
2181 * Free the given buffer.
2183 * This is just a hint, if the buffer is in use or dirty, this function
2186 void dm_bufio_forget(struct dm_bufio_client *c, sector_t block)
2189 forget_buffer(c, block);
2192 EXPORT_SYMBOL_GPL(dm_bufio_forget);
2194 static enum evict_result idle(struct dm_buffer *b, void *context)
2196 return b->state ? ER_DONT_EVICT : ER_EVICT;
2199 void dm_bufio_forget_buffers(struct dm_bufio_client *c, sector_t block, sector_t n_blocks)
2202 cache_remove_range(&c->cache, block, block + n_blocks, idle, __free_buffer_wake);
2205 EXPORT_SYMBOL_GPL(dm_bufio_forget_buffers);
2207 void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned int n)
2209 c->minimum_buffers = n;
2211 EXPORT_SYMBOL_GPL(dm_bufio_set_minimum_buffers);
2213 unsigned int dm_bufio_get_block_size(struct dm_bufio_client *c)
2215 return c->block_size;
2217 EXPORT_SYMBOL_GPL(dm_bufio_get_block_size);
2219 sector_t dm_bufio_get_device_size(struct dm_bufio_client *c)
2221 sector_t s = bdev_nr_sectors(c->bdev);
2227 if (likely(c->sectors_per_block_bits >= 0))
2228 s >>= c->sectors_per_block_bits;
2230 sector_div(s, c->block_size >> SECTOR_SHIFT);
2233 EXPORT_SYMBOL_GPL(dm_bufio_get_device_size);
2235 struct dm_io_client *dm_bufio_get_dm_io_client(struct dm_bufio_client *c)
2239 EXPORT_SYMBOL_GPL(dm_bufio_get_dm_io_client);
2241 sector_t dm_bufio_get_block_number(struct dm_buffer *b)
2245 EXPORT_SYMBOL_GPL(dm_bufio_get_block_number);
2247 void *dm_bufio_get_block_data(struct dm_buffer *b)
2251 EXPORT_SYMBOL_GPL(dm_bufio_get_block_data);
2253 void *dm_bufio_get_aux_data(struct dm_buffer *b)
2257 EXPORT_SYMBOL_GPL(dm_bufio_get_aux_data);
2259 struct dm_bufio_client *dm_bufio_get_client(struct dm_buffer *b)
2263 EXPORT_SYMBOL_GPL(dm_bufio_get_client);
2265 static enum it_action warn_leak(struct dm_buffer *b, void *context)
2267 bool *warned = context;
2269 WARN_ON(!(*warned));
2271 DMERR("leaked buffer %llx, hold count %u, list %d",
2272 (unsigned long long)b->block, atomic_read(&b->hold_count), b->list_mode);
2273 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
2274 stack_trace_print(b->stack_entries, b->stack_len, 1);
2275 /* mark unclaimed to avoid WARN_ON at end of drop_buffers() */
2276 atomic_set(&b->hold_count, 0);
2281 static void drop_buffers(struct dm_bufio_client *c)
2284 struct dm_buffer *b;
2286 if (WARN_ON(dm_bufio_in_request()))
2287 return; /* should never happen */
2290 * An optimization so that the buffers are not written one-by-one.
2292 dm_bufio_write_dirty_buffers_async(c);
2296 while ((b = __get_unclaimed_buffer(c)))
2297 __free_buffer_wake(b);
2299 for (i = 0; i < LIST_SIZE; i++) {
2300 bool warned = false;
2302 cache_iterate(&c->cache, i, warn_leak, &warned);
2305 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
2306 while ((b = __get_unclaimed_buffer(c)))
2307 __free_buffer_wake(b);
2310 for (i = 0; i < LIST_SIZE; i++)
2311 WARN_ON(cache_count(&c->cache, i));
2316 static unsigned long get_retain_buffers(struct dm_bufio_client *c)
2318 unsigned long retain_bytes = READ_ONCE(dm_bufio_retain_bytes);
2320 if (likely(c->sectors_per_block_bits >= 0))
2321 retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT;
2323 retain_bytes /= c->block_size;
2325 return retain_bytes;
2328 static void __scan(struct dm_bufio_client *c)
2331 struct dm_buffer *b;
2332 unsigned long freed = 0;
2333 unsigned long retain_target = get_retain_buffers(c);
2334 unsigned long count = cache_total(&c->cache);
2336 for (l = 0; l < LIST_SIZE; l++) {
2338 if (count - freed <= retain_target)
2339 atomic_long_set(&c->need_shrink, 0);
2340 if (!atomic_long_read(&c->need_shrink))
2343 b = cache_evict(&c->cache, l,
2344 l == LIST_CLEAN ? is_clean : is_dirty, c);
2348 __make_buffer_clean(b);
2349 __free_buffer_wake(b);
2351 atomic_long_dec(&c->need_shrink);
2358 static void shrink_work(struct work_struct *w)
2360 struct dm_bufio_client *c = container_of(w, struct dm_bufio_client, shrink_work);
2367 static unsigned long dm_bufio_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
2369 struct dm_bufio_client *c;
2371 c = container_of(shrink, struct dm_bufio_client, shrinker);
2372 atomic_long_add(sc->nr_to_scan, &c->need_shrink);
2373 queue_work(dm_bufio_wq, &c->shrink_work);
2375 return sc->nr_to_scan;
2378 static unsigned long dm_bufio_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
2380 struct dm_bufio_client *c = container_of(shrink, struct dm_bufio_client, shrinker);
2381 unsigned long count = cache_total(&c->cache);
2382 unsigned long retain_target = get_retain_buffers(c);
2383 unsigned long queued_for_cleanup = atomic_long_read(&c->need_shrink);
2385 if (unlikely(count < retain_target))
2388 count -= retain_target;
2390 if (unlikely(count < queued_for_cleanup))
2393 count -= queued_for_cleanup;
2399 * Create the buffering interface
2401 struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsigned int block_size,
2402 unsigned int reserved_buffers, unsigned int aux_size,
2403 void (*alloc_callback)(struct dm_buffer *),
2404 void (*write_callback)(struct dm_buffer *),
2408 unsigned int num_locks;
2409 struct dm_bufio_client *c;
2412 if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) {
2413 DMERR("%s: block size not specified or is not multiple of 512b", __func__);
2418 num_locks = dm_num_hash_locks();
2419 c = kzalloc(sizeof(*c) + (num_locks * sizeof(struct buffer_tree)), GFP_KERNEL);
2424 cache_init(&c->cache, num_locks);
2427 c->block_size = block_size;
2428 if (is_power_of_2(block_size))
2429 c->sectors_per_block_bits = __ffs(block_size) - SECTOR_SHIFT;
2431 c->sectors_per_block_bits = -1;
2433 c->alloc_callback = alloc_callback;
2434 c->write_callback = write_callback;
2436 if (flags & DM_BUFIO_CLIENT_NO_SLEEP) {
2438 static_branch_inc(&no_sleep_enabled);
2441 mutex_init(&c->lock);
2442 spin_lock_init(&c->spinlock);
2443 INIT_LIST_HEAD(&c->reserved_buffers);
2444 c->need_reserved_buffers = reserved_buffers;
2446 dm_bufio_set_minimum_buffers(c, DM_BUFIO_MIN_BUFFERS);
2448 init_waitqueue_head(&c->free_buffer_wait);
2449 c->async_write_error = 0;
2451 c->dm_io = dm_io_client_create();
2452 if (IS_ERR(c->dm_io)) {
2453 r = PTR_ERR(c->dm_io);
2457 if (block_size <= KMALLOC_MAX_SIZE &&
2458 (block_size < PAGE_SIZE || !is_power_of_2(block_size))) {
2459 unsigned int align = min(1U << __ffs(block_size), (unsigned int)PAGE_SIZE);
2461 snprintf(slab_name, sizeof(slab_name), "dm_bufio_cache-%u", block_size);
2462 c->slab_cache = kmem_cache_create(slab_name, block_size, align,
2463 SLAB_RECLAIM_ACCOUNT, NULL);
2464 if (!c->slab_cache) {
2470 snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer-%u", aux_size);
2472 snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer");
2473 c->slab_buffer = kmem_cache_create(slab_name, sizeof(struct dm_buffer) + aux_size,
2474 0, SLAB_RECLAIM_ACCOUNT, NULL);
2475 if (!c->slab_buffer) {
2480 while (c->need_reserved_buffers) {
2481 struct dm_buffer *b = alloc_buffer(c, GFP_KERNEL);
2487 __free_buffer_wake(b);
2490 INIT_WORK(&c->shrink_work, shrink_work);
2491 atomic_long_set(&c->need_shrink, 0);
2493 c->shrinker.count_objects = dm_bufio_shrink_count;
2494 c->shrinker.scan_objects = dm_bufio_shrink_scan;
2495 c->shrinker.seeks = 1;
2496 c->shrinker.batch = 0;
2497 r = register_shrinker(&c->shrinker, "dm-bufio:(%u:%u)",
2498 MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev));
2502 mutex_lock(&dm_bufio_clients_lock);
2503 dm_bufio_client_count++;
2504 list_add(&c->client_list, &dm_bufio_all_clients);
2505 __cache_size_refresh();
2506 mutex_unlock(&dm_bufio_clients_lock);
2511 while (!list_empty(&c->reserved_buffers)) {
2512 struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next);
2514 list_del(&b->lru.list);
2517 kmem_cache_destroy(c->slab_cache);
2518 kmem_cache_destroy(c->slab_buffer);
2519 dm_io_client_destroy(c->dm_io);
2521 mutex_destroy(&c->lock);
2523 static_branch_dec(&no_sleep_enabled);
2528 EXPORT_SYMBOL_GPL(dm_bufio_client_create);
2531 * Free the buffering interface.
2532 * It is required that there are no references on any buffers.
2534 void dm_bufio_client_destroy(struct dm_bufio_client *c)
2540 unregister_shrinker(&c->shrinker);
2541 flush_work(&c->shrink_work);
2543 mutex_lock(&dm_bufio_clients_lock);
2545 list_del(&c->client_list);
2546 dm_bufio_client_count--;
2547 __cache_size_refresh();
2549 mutex_unlock(&dm_bufio_clients_lock);
2551 WARN_ON(c->need_reserved_buffers);
2553 while (!list_empty(&c->reserved_buffers)) {
2554 struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next);
2556 list_del(&b->lru.list);
2560 for (i = 0; i < LIST_SIZE; i++)
2561 if (cache_count(&c->cache, i))
2562 DMERR("leaked buffer count %d: %lu", i, cache_count(&c->cache, i));
2564 for (i = 0; i < LIST_SIZE; i++)
2565 WARN_ON(cache_count(&c->cache, i));
2567 cache_destroy(&c->cache);
2568 kmem_cache_destroy(c->slab_cache);
2569 kmem_cache_destroy(c->slab_buffer);
2570 dm_io_client_destroy(c->dm_io);
2571 mutex_destroy(&c->lock);
2573 static_branch_dec(&no_sleep_enabled);
2576 EXPORT_SYMBOL_GPL(dm_bufio_client_destroy);
2578 void dm_bufio_client_reset(struct dm_bufio_client *c)
2581 flush_work(&c->shrink_work);
2583 EXPORT_SYMBOL_GPL(dm_bufio_client_reset);
2585 void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start)
2589 EXPORT_SYMBOL_GPL(dm_bufio_set_sector_offset);
2591 /*--------------------------------------------------------------*/
2593 static unsigned int get_max_age_hz(void)
2595 unsigned int max_age = READ_ONCE(dm_bufio_max_age);
2597 if (max_age > UINT_MAX / HZ)
2598 max_age = UINT_MAX / HZ;
2600 return max_age * HZ;
2603 static bool older_than(struct dm_buffer *b, unsigned long age_hz)
2605 return time_after_eq(jiffies, READ_ONCE(b->last_accessed) + age_hz);
2608 struct evict_params {
2610 unsigned long age_hz;
2613 * This gets updated with the largest last_accessed (ie. most
2614 * recently used) of the evicted buffers. It will not be reinitialised
2615 * by __evict_many(), so you can use it across multiple invocations.
2617 unsigned long last_accessed;
2621 * We may not be able to evict this buffer if IO pending or the client
2622 * is still using it.
2624 * And if GFP_NOFS is used, we must not do any I/O because we hold
2625 * dm_bufio_clients_lock and we would risk deadlock if the I/O gets
2626 * rerouted to different bufio client.
2628 static enum evict_result select_for_evict(struct dm_buffer *b, void *context)
2630 struct evict_params *params = context;
2632 if (!(params->gfp & __GFP_FS) ||
2633 (static_branch_unlikely(&no_sleep_enabled) && b->c->no_sleep)) {
2634 if (test_bit_acquire(B_READING, &b->state) ||
2635 test_bit(B_WRITING, &b->state) ||
2636 test_bit(B_DIRTY, &b->state))
2637 return ER_DONT_EVICT;
2640 return older_than(b, params->age_hz) ? ER_EVICT : ER_STOP;
2643 static unsigned long __evict_many(struct dm_bufio_client *c,
2644 struct evict_params *params,
2645 int list_mode, unsigned long max_count)
2647 unsigned long count;
2648 unsigned long last_accessed;
2649 struct dm_buffer *b;
2651 for (count = 0; count < max_count; count++) {
2652 b = cache_evict(&c->cache, list_mode, select_for_evict, params);
2656 last_accessed = READ_ONCE(b->last_accessed);
2657 if (time_after_eq(params->last_accessed, last_accessed))
2658 params->last_accessed = last_accessed;
2660 __make_buffer_clean(b);
2661 __free_buffer_wake(b);
2669 static void evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
2671 struct evict_params params = {.gfp = 0, .age_hz = age_hz, .last_accessed = 0};
2672 unsigned long retain = get_retain_buffers(c);
2673 unsigned long count;
2674 LIST_HEAD(write_list);
2678 __check_watermark(c, &write_list);
2679 if (unlikely(!list_empty(&write_list))) {
2681 __flush_write_list(&write_list);
2685 count = cache_total(&c->cache);
2687 __evict_many(c, ¶ms, LIST_CLEAN, count - retain);
2692 static void cleanup_old_buffers(void)
2694 unsigned long max_age_hz = get_max_age_hz();
2695 struct dm_bufio_client *c;
2697 mutex_lock(&dm_bufio_clients_lock);
2699 __cache_size_refresh();
2701 list_for_each_entry(c, &dm_bufio_all_clients, client_list)
2702 evict_old_buffers(c, max_age_hz);
2704 mutex_unlock(&dm_bufio_clients_lock);
2707 static void work_fn(struct work_struct *w)
2709 cleanup_old_buffers();
2711 queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
2712 DM_BUFIO_WORK_TIMER_SECS * HZ);
2715 /*--------------------------------------------------------------*/
2718 * Global cleanup tries to evict the oldest buffers from across _all_
2719 * the clients. It does this by repeatedly evicting a few buffers from
2720 * the client that holds the oldest buffer. It's approximate, but hopefully
2723 static struct dm_bufio_client *__pop_client(void)
2725 struct list_head *h;
2727 if (list_empty(&dm_bufio_all_clients))
2730 h = dm_bufio_all_clients.next;
2732 return container_of(h, struct dm_bufio_client, client_list);
2736 * Inserts the client in the global client list based on its
2737 * 'oldest_buffer' field.
2739 static void __insert_client(struct dm_bufio_client *new_client)
2741 struct dm_bufio_client *c;
2742 struct list_head *h = dm_bufio_all_clients.next;
2744 while (h != &dm_bufio_all_clients) {
2745 c = container_of(h, struct dm_bufio_client, client_list);
2746 if (time_after_eq(c->oldest_buffer, new_client->oldest_buffer))
2751 list_add_tail(&new_client->client_list, h);
2754 static unsigned long __evict_a_few(unsigned long nr_buffers)
2756 unsigned long count;
2757 struct dm_bufio_client *c;
2758 struct evict_params params = {
2761 /* set to jiffies in case there are no buffers in this client */
2762 .last_accessed = jiffies
2770 count = __evict_many(c, ¶ms, LIST_CLEAN, nr_buffers);
2774 c->oldest_buffer = params.last_accessed;
2780 static void check_watermarks(void)
2782 LIST_HEAD(write_list);
2783 struct dm_bufio_client *c;
2785 mutex_lock(&dm_bufio_clients_lock);
2786 list_for_each_entry(c, &dm_bufio_all_clients, client_list) {
2788 __check_watermark(c, &write_list);
2791 mutex_unlock(&dm_bufio_clients_lock);
2793 __flush_write_list(&write_list);
2796 static void evict_old(void)
2798 unsigned long threshold = dm_bufio_cache_size -
2799 dm_bufio_cache_size / DM_BUFIO_LOW_WATERMARK_RATIO;
2801 mutex_lock(&dm_bufio_clients_lock);
2802 while (dm_bufio_current_allocated > threshold) {
2803 if (!__evict_a_few(64))
2807 mutex_unlock(&dm_bufio_clients_lock);
2810 static void do_global_cleanup(struct work_struct *w)
2817 *--------------------------------------------------------------
2819 *--------------------------------------------------------------
2823 * This is called only once for the whole dm_bufio module.
2824 * It initializes memory limit.
2826 static int __init dm_bufio_init(void)
2830 dm_bufio_allocated_kmem_cache = 0;
2831 dm_bufio_allocated_get_free_pages = 0;
2832 dm_bufio_allocated_vmalloc = 0;
2833 dm_bufio_current_allocated = 0;
2835 mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(),
2836 DM_BUFIO_MEMORY_PERCENT, 100) << PAGE_SHIFT;
2838 if (mem > ULONG_MAX)
2842 if (mem > mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100))
2843 mem = mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100);
2846 dm_bufio_default_cache_size = mem;
2848 mutex_lock(&dm_bufio_clients_lock);
2849 __cache_size_refresh();
2850 mutex_unlock(&dm_bufio_clients_lock);
2852 dm_bufio_wq = alloc_workqueue("dm_bufio_cache", WQ_MEM_RECLAIM, 0);
2856 INIT_DELAYED_WORK(&dm_bufio_cleanup_old_work, work_fn);
2857 INIT_WORK(&dm_bufio_replacement_work, do_global_cleanup);
2858 queue_delayed_work(dm_bufio_wq, &dm_bufio_cleanup_old_work,
2859 DM_BUFIO_WORK_TIMER_SECS * HZ);
2865 * This is called once when unloading the dm_bufio module.
2867 static void __exit dm_bufio_exit(void)
2871 cancel_delayed_work_sync(&dm_bufio_cleanup_old_work);
2872 destroy_workqueue(dm_bufio_wq);
2874 if (dm_bufio_client_count) {
2875 DMCRIT("%s: dm_bufio_client_count leaked: %d",
2876 __func__, dm_bufio_client_count);
2880 if (dm_bufio_current_allocated) {
2881 DMCRIT("%s: dm_bufio_current_allocated leaked: %lu",
2882 __func__, dm_bufio_current_allocated);
2886 if (dm_bufio_allocated_get_free_pages) {
2887 DMCRIT("%s: dm_bufio_allocated_get_free_pages leaked: %lu",
2888 __func__, dm_bufio_allocated_get_free_pages);
2892 if (dm_bufio_allocated_vmalloc) {
2893 DMCRIT("%s: dm_bufio_vmalloc leaked: %lu",
2894 __func__, dm_bufio_allocated_vmalloc);
2898 WARN_ON(bug); /* leaks are not worth crashing the system */
2901 module_init(dm_bufio_init)
2902 module_exit(dm_bufio_exit)
2904 module_param_named(max_cache_size_bytes, dm_bufio_cache_size, ulong, 0644);
2905 MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache");
2907 module_param_named(max_age_seconds, dm_bufio_max_age, uint, 0644);
2908 MODULE_PARM_DESC(max_age_seconds, "Max age of a buffer in seconds");
2910 module_param_named(retain_bytes, dm_bufio_retain_bytes, ulong, 0644);
2911 MODULE_PARM_DESC(retain_bytes, "Try to keep at least this many bytes cached in memory");
2913 module_param_named(peak_allocated_bytes, dm_bufio_peak_allocated, ulong, 0644);
2914 MODULE_PARM_DESC(peak_allocated_bytes, "Tracks the maximum allocated memory");
2916 module_param_named(allocated_kmem_cache_bytes, dm_bufio_allocated_kmem_cache, ulong, 0444);
2917 MODULE_PARM_DESC(allocated_kmem_cache_bytes, "Memory allocated with kmem_cache_alloc");
2919 module_param_named(allocated_get_free_pages_bytes, dm_bufio_allocated_get_free_pages, ulong, 0444);
2920 MODULE_PARM_DESC(allocated_get_free_pages_bytes, "Memory allocated with get_free_pages");
2922 module_param_named(allocated_vmalloc_bytes, dm_bufio_allocated_vmalloc, ulong, 0444);
2923 MODULE_PARM_DESC(allocated_vmalloc_bytes, "Memory allocated with vmalloc");
2925 module_param_named(current_allocated_bytes, dm_bufio_current_allocated, ulong, 0444);
2926 MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache");
2928 MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");
2929 MODULE_DESCRIPTION(DM_NAME " buffered I/O library");
2930 MODULE_LICENSE("GPL");