Just reorganizing things a bit.
Signed-off-by: Kent Overstreet <kmo@daterainc.com>
return ret;
}
+/* Sector allocator */
+
+struct open_bucket {
+ struct list_head list;
+ unsigned last_write_point;
+ unsigned sectors_free;
+ BKEY_PADDED(key);
+};
+
+/*
+ * We keep multiple buckets open for writes, and try to segregate different
+ * write streams for better cache utilization: first we look for a bucket where
+ * the last write to it was sequential with the current write, and failing that
+ * we look for a bucket that was last used by the same task.
+ *
+ * The ideas is if you've got multiple tasks pulling data into the cache at the
+ * same time, you'll get better cache utilization if you try to segregate their
+ * data and preserve locality.
+ *
+ * For example, say you've starting Firefox at the same time you're copying a
+ * bunch of files. Firefox will likely end up being fairly hot and stay in the
+ * cache awhile, but the data you copied might not be; if you wrote all that
+ * data to the same buckets it'd get invalidated at the same time.
+ *
+ * Both of those tasks will be doing fairly random IO so we can't rely on
+ * detecting sequential IO to segregate their data, but going off of the task
+ * should be a sane heuristic.
+ */
+static struct open_bucket *pick_data_bucket(struct cache_set *c,
+ const struct bkey *search,
+ unsigned write_point,
+ struct bkey *alloc)
+{
+ struct open_bucket *ret, *ret_task = NULL;
+
+ list_for_each_entry_reverse(ret, &c->data_buckets, list)
+ if (!bkey_cmp(&ret->key, search))
+ goto found;
+ else if (ret->last_write_point == write_point)
+ ret_task = ret;
+
+ ret = ret_task ?: list_first_entry(&c->data_buckets,
+ struct open_bucket, list);
+found:
+ if (!ret->sectors_free && KEY_PTRS(alloc)) {
+ ret->sectors_free = c->sb.bucket_size;
+ bkey_copy(&ret->key, alloc);
+ bkey_init(alloc);
+ }
+
+ if (!ret->sectors_free)
+ ret = NULL;
+
+ return ret;
+}
+
+/*
+ * Allocates some space in the cache to write to, and k to point to the newly
+ * allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
+ * end of the newly allocated space).
+ *
+ * May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
+ * sectors were actually allocated.
+ *
+ * If s->writeback is true, will not fail.
+ */
+bool bch_alloc_sectors(struct cache_set *c, struct bkey *k, unsigned sectors,
+ unsigned write_point, unsigned write_prio, bool wait)
+{
+ struct open_bucket *b;
+ BKEY_PADDED(key) alloc;
+ unsigned i;
+
+ /*
+ * We might have to allocate a new bucket, which we can't do with a
+ * spinlock held. So if we have to allocate, we drop the lock, allocate
+ * and then retry. KEY_PTRS() indicates whether alloc points to
+ * allocated bucket(s).
+ */
+
+ bkey_init(&alloc.key);
+ spin_lock(&c->data_bucket_lock);
+
+ while (!(b = pick_data_bucket(c, k, write_point, &alloc.key))) {
+ unsigned watermark = write_prio
+ ? WATERMARK_MOVINGGC
+ : WATERMARK_NONE;
+
+ spin_unlock(&c->data_bucket_lock);
+
+ if (bch_bucket_alloc_set(c, watermark, &alloc.key, 1, wait))
+ return false;
+
+ spin_lock(&c->data_bucket_lock);
+ }
+
+ /*
+ * If we had to allocate, we might race and not need to allocate the
+ * second time we call find_data_bucket(). If we allocated a bucket but
+ * didn't use it, drop the refcount bch_bucket_alloc_set() took:
+ */
+ if (KEY_PTRS(&alloc.key))
+ __bkey_put(c, &alloc.key);
+
+ for (i = 0; i < KEY_PTRS(&b->key); i++)
+ EBUG_ON(ptr_stale(c, &b->key, i));
+
+ /* Set up the pointer to the space we're allocating: */
+
+ for (i = 0; i < KEY_PTRS(&b->key); i++)
+ k->ptr[i] = b->key.ptr[i];
+
+ sectors = min(sectors, b->sectors_free);
+
+ SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors);
+ SET_KEY_SIZE(k, sectors);
+ SET_KEY_PTRS(k, KEY_PTRS(&b->key));
+
+ /*
+ * Move b to the end of the lru, and keep track of what this bucket was
+ * last used for:
+ */
+ list_move_tail(&b->list, &c->data_buckets);
+ bkey_copy_key(&b->key, k);
+ b->last_write_point = write_point;
+
+ b->sectors_free -= sectors;
+
+ for (i = 0; i < KEY_PTRS(&b->key); i++) {
+ SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors);
+
+ atomic_long_add(sectors,
+ &PTR_CACHE(c, &b->key, i)->sectors_written);
+ }
+
+ if (b->sectors_free < c->sb.block_size)
+ b->sectors_free = 0;
+
+ /*
+ * k takes refcounts on the buckets it points to until it's inserted
+ * into the btree, but if we're done with this bucket we just transfer
+ * get_data_bucket()'s refcount.
+ */
+ if (b->sectors_free)
+ for (i = 0; i < KEY_PTRS(&b->key); i++)
+ atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin);
+
+ spin_unlock(&c->data_bucket_lock);
+ return true;
+}
+
/* Init */
+void bch_open_buckets_free(struct cache_set *c)
+{
+ struct open_bucket *b;
+
+ while (!list_empty(&c->data_buckets)) {
+ b = list_first_entry(&c->data_buckets,
+ struct open_bucket, list);
+ list_del(&b->list);
+ kfree(b);
+ }
+}
+
+int bch_open_buckets_alloc(struct cache_set *c)
+{
+ int i;
+
+ spin_lock_init(&c->data_bucket_lock);
+
+ for (i = 0; i < 6; i++) {
+ struct open_bucket *b = kzalloc(sizeof(*b), GFP_KERNEL);
+ if (!b)
+ return -ENOMEM;
+
+ list_add(&b->list, &c->data_buckets);
+ }
+
+ return 0;
+}
+
int bch_cache_allocator_start(struct cache *ca)
{
struct task_struct *k = kthread_run(bch_allocator_thread,
struct bkey *, int, bool);
int bch_bucket_alloc_set(struct cache_set *, unsigned,
struct bkey *, int, bool);
+bool bch_alloc_sectors(struct cache_set *, struct bkey *, unsigned,
+ unsigned, unsigned, bool);
__printf(2, 3)
bool bch_cache_set_error(struct cache_set *, const char *, ...);
void bch_btree_cache_free(struct cache_set *);
int bch_btree_cache_alloc(struct cache_set *);
void bch_moving_init_cache_set(struct cache_set *);
+int bch_open_buckets_alloc(struct cache_set *);
+void bch_open_buckets_free(struct cache_set *);
int bch_cache_allocator_start(struct cache *ca);
int bch_cache_allocator_init(struct cache *ca);
closure_return(cl);
}
-struct open_bucket {
- struct list_head list;
- struct task_struct *last;
- unsigned sectors_free;
- BKEY_PADDED(key);
-};
-
-void bch_open_buckets_free(struct cache_set *c)
-{
- struct open_bucket *b;
-
- while (!list_empty(&c->data_buckets)) {
- b = list_first_entry(&c->data_buckets,
- struct open_bucket, list);
- list_del(&b->list);
- kfree(b);
- }
-}
-
-int bch_open_buckets_alloc(struct cache_set *c)
-{
- int i;
-
- spin_lock_init(&c->data_bucket_lock);
-
- for (i = 0; i < 6; i++) {
- struct open_bucket *b = kzalloc(sizeof(*b), GFP_KERNEL);
- if (!b)
- return -ENOMEM;
-
- list_add(&b->list, &c->data_buckets);
- }
-
- return 0;
-}
-
-/*
- * We keep multiple buckets open for writes, and try to segregate different
- * write streams for better cache utilization: first we look for a bucket where
- * the last write to it was sequential with the current write, and failing that
- * we look for a bucket that was last used by the same task.
- *
- * The ideas is if you've got multiple tasks pulling data into the cache at the
- * same time, you'll get better cache utilization if you try to segregate their
- * data and preserve locality.
- *
- * For example, say you've starting Firefox at the same time you're copying a
- * bunch of files. Firefox will likely end up being fairly hot and stay in the
- * cache awhile, but the data you copied might not be; if you wrote all that
- * data to the same buckets it'd get invalidated at the same time.
- *
- * Both of those tasks will be doing fairly random IO so we can't rely on
- * detecting sequential IO to segregate their data, but going off of the task
- * should be a sane heuristic.
- */
-static struct open_bucket *pick_data_bucket(struct cache_set *c,
- const struct bkey *search,
- struct task_struct *task,
- struct bkey *alloc)
-{
- struct open_bucket *ret, *ret_task = NULL;
-
- list_for_each_entry_reverse(ret, &c->data_buckets, list)
- if (!bkey_cmp(&ret->key, search))
- goto found;
- else if (ret->last == task)
- ret_task = ret;
-
- ret = ret_task ?: list_first_entry(&c->data_buckets,
- struct open_bucket, list);
-found:
- if (!ret->sectors_free && KEY_PTRS(alloc)) {
- ret->sectors_free = c->sb.bucket_size;
- bkey_copy(&ret->key, alloc);
- bkey_init(alloc);
- }
-
- if (!ret->sectors_free)
- ret = NULL;
-
- return ret;
-}
-
-/*
- * Allocates some space in the cache to write to, and k to point to the newly
- * allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
- * end of the newly allocated space).
- *
- * May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
- * sectors were actually allocated.
- *
- * If s->writeback is true, will not fail.
- */
-static bool bch_alloc_sectors(struct data_insert_op *op,
- struct bkey *k, unsigned sectors)
-{
- struct cache_set *c = op->c;
- struct open_bucket *b;
- BKEY_PADDED(key) alloc;
- unsigned i;
-
- /*
- * We might have to allocate a new bucket, which we can't do with a
- * spinlock held. So if we have to allocate, we drop the lock, allocate
- * and then retry. KEY_PTRS() indicates whether alloc points to
- * allocated bucket(s).
- */
-
- bkey_init(&alloc.key);
- spin_lock(&c->data_bucket_lock);
-
- while (!(b = pick_data_bucket(c, k, op->task, &alloc.key))) {
- unsigned watermark = op->write_prio
- ? WATERMARK_MOVINGGC
- : WATERMARK_NONE;
-
- spin_unlock(&c->data_bucket_lock);
-
- if (bch_bucket_alloc_set(c, watermark, &alloc.key,
- 1, op->writeback))
- return false;
-
- spin_lock(&c->data_bucket_lock);
- }
-
- /*
- * If we had to allocate, we might race and not need to allocate the
- * second time we call find_data_bucket(). If we allocated a bucket but
- * didn't use it, drop the refcount bch_bucket_alloc_set() took:
- */
- if (KEY_PTRS(&alloc.key))
- __bkey_put(c, &alloc.key);
-
- for (i = 0; i < KEY_PTRS(&b->key); i++)
- EBUG_ON(ptr_stale(c, &b->key, i));
-
- /* Set up the pointer to the space we're allocating: */
-
- for (i = 0; i < KEY_PTRS(&b->key); i++)
- k->ptr[i] = b->key.ptr[i];
-
- sectors = min(sectors, b->sectors_free);
-
- SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors);
- SET_KEY_SIZE(k, sectors);
- SET_KEY_PTRS(k, KEY_PTRS(&b->key));
-
- /*
- * Move b to the end of the lru, and keep track of what this bucket was
- * last used for:
- */
- list_move_tail(&b->list, &c->data_buckets);
- bkey_copy_key(&b->key, k);
- b->last = op->task;
-
- b->sectors_free -= sectors;
-
- for (i = 0; i < KEY_PTRS(&b->key); i++) {
- SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors);
-
- atomic_long_add(sectors,
- &PTR_CACHE(c, &b->key, i)->sectors_written);
- }
-
- if (b->sectors_free < c->sb.block_size)
- b->sectors_free = 0;
-
- /*
- * k takes refcounts on the buckets it points to until it's inserted
- * into the btree, but if we're done with this bucket we just transfer
- * get_data_bucket()'s refcount.
- */
- if (b->sectors_free)
- for (i = 0; i < KEY_PTRS(&b->key); i++)
- atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin);
-
- spin_unlock(&c->data_bucket_lock);
- return true;
-}
-
static void bch_data_invalidate(struct closure *cl)
{
struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
SET_KEY_INODE(k, op->inode);
SET_KEY_OFFSET(k, bio->bi_sector);
- if (!bch_alloc_sectors(op, k, bio_sectors(bio)))
+ if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),
+ op->write_point, op->write_prio,
+ op->writeback))
goto err;
n = bch_bio_split(bio, KEY_SIZE(k), GFP_NOIO, split);
s->iop.c = d->c;
s->d = d;
s->op.lock = -1;
- s->iop.task = current;
+ s->iop.write_point = hash_long((unsigned long) current, 16);
s->orig_bio = bio;
s->write = (bio->bi_rw & REQ_WRITE) != 0;
s->iop.flush_journal = (bio->bi_rw & (REQ_FLUSH|REQ_FUA)) != 0;
struct data_insert_op {
struct closure cl;
struct cache_set *c;
- struct task_struct *task;
struct bio *bio;
unsigned inode;
+ uint16_t write_point;
uint16_t write_prio;
short error;
unsigned bch_get_congested(struct cache_set *);
void bch_data_insert(struct closure *cl);
-void bch_open_buckets_free(struct cache_set *);
-int bch_open_buckets_alloc(struct cache_set *);
-
void bch_cached_dev_request_init(struct cached_dev *dc);
void bch_flash_dev_request_init(struct bcache_device *d);
projects / platform / kernel / linux-stable.git / commitdiff