#include "bcache.h"
#include "btree.h"
#include "debug.h"
-#include "request.h"
#include "writeback.h"
#include <linux/slab.h>
closure_sync(&cl);
}
-/* Cache lookup */
-
-static int submit_partial_cache_miss(struct btree *b, struct btree_op *op,
- struct bkey *k)
-{
- struct search *s = container_of(op, struct search, op);
- struct bio *bio = &s->bio.bio;
- int ret = 0;
-
- while (!ret &&
- !op->lookup_done) {
- unsigned sectors = INT_MAX;
-
- if (KEY_INODE(k) == op->inode) {
- if (KEY_START(k) <= bio->bi_sector)
- break;
-
- sectors = min_t(uint64_t, sectors,
- KEY_START(k) - bio->bi_sector);
- }
-
- ret = s->d->cache_miss(b, s, bio, sectors);
- }
-
- return ret;
-}
-
-/*
- * Read from a single key, handling the initial cache miss if the key starts in
- * the middle of the bio
- */
-static int submit_partial_cache_hit(struct btree *b, struct btree_op *op,
- struct bkey *k)
-{
- struct search *s = container_of(op, struct search, op);
- struct bio *bio = &s->bio.bio;
- unsigned ptr;
- struct bio *n;
-
- int ret = submit_partial_cache_miss(b, op, k);
- if (ret || op->lookup_done)
- return ret;
-
- /* XXX: figure out best pointer - for multiple cache devices */
- ptr = 0;
-
- PTR_BUCKET(b->c, k, ptr)->prio = INITIAL_PRIO;
-
- while (!op->lookup_done &&
- KEY_INODE(k) == op->inode &&
- bio->bi_sector < KEY_OFFSET(k)) {
- struct bkey *bio_key;
- sector_t sector = PTR_OFFSET(k, ptr) +
- (bio->bi_sector - KEY_START(k));
- unsigned sectors = min_t(uint64_t, INT_MAX,
- KEY_OFFSET(k) - bio->bi_sector);
-
- n = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split);
- if (n == bio)
- op->lookup_done = true;
-
- bio_key = &container_of(n, struct bbio, bio)->key;
-
- /*
- * The bucket we're reading from might be reused while our bio
- * is in flight, and we could then end up reading the wrong
- * data.
- *
- * We guard against this by checking (in cache_read_endio()) if
- * the pointer is stale again; if so, we treat it as an error
- * and reread from the backing device (but we don't pass that
- * error up anywhere).
- */
-
- bch_bkey_copy_single_ptr(bio_key, k, ptr);
- SET_PTR_OFFSET(bio_key, 0, sector);
-
- n->bi_end_io = bch_cache_read_endio;
- n->bi_private = &s->cl;
-
- __bch_submit_bbio(n, b->c);
- }
-
- return 0;
-}
-
-static int bch_btree_search_recurse(struct btree *b, struct btree_op *op)
-{
- struct search *s = container_of(op, struct search, op);
- struct bio *bio = &s->bio.bio;
-
- int ret = 0;
- struct bkey *k;
- struct btree_iter iter;
- bch_btree_iter_init(b, &iter, &KEY(op->inode, bio->bi_sector, 0));
-
- do {
- k = bch_btree_iter_next_filter(&iter, b, bch_ptr_bad);
- if (!k) {
- /*
- * b->key would be exactly what we want, except that
- * pointers to btree nodes have nonzero size - we
- * wouldn't go far enough
- */
-
- ret = submit_partial_cache_miss(b, op,
- &KEY(KEY_INODE(&b->key),
- KEY_OFFSET(&b->key), 0));
- break;
- }
-
- ret = b->level
- ? btree(search_recurse, k, b, op)
- : submit_partial_cache_hit(b, op, k);
- } while (!ret &&
- !op->lookup_done);
-
- return ret;
-}
-
-void bch_btree_search_async(struct closure *cl)
-{
- struct btree_op *op = container_of(cl, struct btree_op, cl);
-
- int ret = btree_root(search_recurse, op->c, op);
-
- if (ret == -EAGAIN)
- continue_at(cl, bch_btree_search_async, bcache_wq);
-
- closure_return(cl);
-}
-
/* Map across nodes or keys */
static int bch_btree_map_nodes_recurse(struct btree *b, struct btree_op *op,
bch_data_insert_start(cl);
}
-/* Common code for the make_request functions */
+/* Cache lookup */
-static void request_endio(struct bio *bio, int error)
-{
- struct closure *cl = bio->bi_private;
-
- if (error) {
- struct search *s = container_of(cl, struct search, cl);
- s->error = error;
- /* Only cache read errors are recoverable */
- s->recoverable = false;
- }
-
- bio_put(bio);
- closure_put(cl);
-}
-
-void bch_cache_read_endio(struct bio *bio, int error)
+static void bch_cache_read_endio(struct bio *bio, int error)
{
struct bbio *b = container_of(bio, struct bbio, bio);
struct closure *cl = bio->bi_private;
bch_bbio_endio(s->op.c, bio, error, "reading from cache");
}
+static int submit_partial_cache_miss(struct btree *b, struct search *s,
+ struct bkey *k)
+{
+ struct bio *bio = &s->bio.bio;
+ int ret = MAP_CONTINUE;
+
+ do {
+ unsigned sectors = INT_MAX;
+
+ if (KEY_INODE(k) == s->op.inode) {
+ if (KEY_START(k) <= bio->bi_sector)
+ break;
+
+ sectors = min_t(uint64_t, sectors,
+ KEY_START(k) - bio->bi_sector);
+ }
+
+ ret = s->d->cache_miss(b, s, bio, sectors);
+ } while (ret == MAP_CONTINUE);
+
+ return ret;
+}
+
+/*
+ * Read from a single key, handling the initial cache miss if the key starts in
+ * the middle of the bio
+ */
+static int submit_partial_cache_hit(struct btree_op *op, struct btree *b,
+ struct bkey *k)
+{
+ struct search *s = container_of(op, struct search, op);
+ struct bio *bio = &s->bio.bio;
+ unsigned ptr;
+ struct bio *n;
+
+ int ret = submit_partial_cache_miss(b, s, k);
+ if (ret != MAP_CONTINUE || !KEY_SIZE(k))
+ return ret;
+
+ /* XXX: figure out best pointer - for multiple cache devices */
+ ptr = 0;
+
+ PTR_BUCKET(b->c, k, ptr)->prio = INITIAL_PRIO;
+
+ while (ret == MAP_CONTINUE &&
+ KEY_INODE(k) == op->inode &&
+ bio->bi_sector < KEY_OFFSET(k)) {
+ struct bkey *bio_key;
+ sector_t sector = PTR_OFFSET(k, ptr) +
+ (bio->bi_sector - KEY_START(k));
+ unsigned sectors = min_t(uint64_t, INT_MAX,
+ KEY_OFFSET(k) - bio->bi_sector);
+
+ n = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split);
+ if (n == bio)
+ ret = MAP_DONE;
+
+ bio_key = &container_of(n, struct bbio, bio)->key;
+
+ /*
+ * The bucket we're reading from might be reused while our bio
+ * is in flight, and we could then end up reading the wrong
+ * data.
+ *
+ * We guard against this by checking (in cache_read_endio()) if
+ * the pointer is stale again; if so, we treat it as an error
+ * and reread from the backing device (but we don't pass that
+ * error up anywhere).
+ */
+
+ bch_bkey_copy_single_ptr(bio_key, k, ptr);
+ SET_PTR_OFFSET(bio_key, 0, sector);
+
+ n->bi_end_io = bch_cache_read_endio;
+ n->bi_private = &s->cl;
+
+ __bch_submit_bbio(n, b->c);
+ }
+
+ return ret;
+}
+
+static void cache_lookup(struct closure *cl)
+{
+ struct btree_op *op = container_of(cl, struct btree_op, cl);
+ struct search *s = container_of(op, struct search, op);
+ struct bio *bio = &s->bio.bio;
+
+ int ret = bch_btree_map_keys(op, op->c,
+ &KEY(op->inode, bio->bi_sector, 0),
+ submit_partial_cache_hit, 1);
+ if (ret == -EAGAIN)
+ continue_at(cl, cache_lookup, bcache_wq);
+
+ closure_return(cl);
+}
+
+/* Common code for the make_request functions */
+
+static void request_endio(struct bio *bio, int error)
+{
+ struct closure *cl = bio->bi_private;
+
+ if (error) {
+ struct search *s = container_of(cl, struct search, cl);
+ s->error = error;
+ /* Only cache read errors are recoverable */
+ s->recoverable = false;
+ }
+
+ bio_put(bio);
+ closure_put(cl);
+}
+
static void bio_complete(struct search *s)
{
if (s->orig_bio) {
static int cached_dev_cache_miss(struct btree *b, struct search *s,
struct bio *bio, unsigned sectors)
{
- int ret = 0;
+ int ret = MAP_CONTINUE;
unsigned reada = 0;
struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
struct bio *miss, *cache_bio;
if (s->cache_miss || s->op.bypass) {
miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split);
- if (miss == bio)
- s->op.lookup_done = true;
+ ret = miss == bio ? MAP_DONE : MAP_CONTINUE;
goto out_submit;
}
return ret;
miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split);
- if (miss == bio)
- s->op.lookup_done = true;
- else
- /* btree_search_recurse()'s btree iterator is no good anymore */
- ret = -EINTR;
+
+ /* btree_search_recurse()'s btree iterator is no good anymore */
+ ret = miss == bio ? MAP_DONE : -EINTR;
cache_bio = bio_alloc_bioset(GFP_NOWAIT,
DIV_ROUND_UP(s->cache_bio_sectors, PAGE_SECTORS),
{
struct closure *cl = &s->cl;
- closure_call(&s->op.cl, bch_btree_search_async, NULL, cl);
+ closure_call(&s->op.cl, cache_lookup, NULL, cl);
continue_at(cl, cached_dev_read_done_bh, NULL);
}
bio_advance(bio, min(sectors << 9, bio->bi_size));
if (!bio->bi_size)
- s->op.lookup_done = true;
+ return MAP_DONE;
- return 0;
+ return MAP_CONTINUE;
}
static void flash_dev_nodata(struct closure *cl)
closure_call(&s->op.cl, bch_data_insert, NULL, cl);
} else {
- closure_call(&s->op.cl, bch_btree_search_async, NULL, cl);
+ closure_call(&s->op.cl, cache_lookup, NULL, cl);
}
continue_at(cl, search_free, NULL);
projects / platform / kernel / linux-arm64.git / commitdiff