};
/*
- * For bio-based dm.
- * One of these is allocated per target within a bio. Hopefully
- * this will be simplified out one day.
- */
-struct dm_target_io {
- struct dm_io *io;
- struct dm_target *ti;
- union map_info info;
-};
-
-/*
* For request-based dm.
* One of these is allocated per request.
*/
#define MIN_IOS 256
static struct kmem_cache *_io_cache;
-static struct kmem_cache *_tio_cache;
static struct kmem_cache *_rq_tio_cache;
/*
if (!_io_cache)
return r;
- /* allocate a slab for the target ios */
- _tio_cache = KMEM_CACHE(dm_target_io, 0);
- if (!_tio_cache)
- goto out_free_io_cache;
-
_rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
if (!_rq_tio_cache)
- goto out_free_tio_cache;
+ goto out_free_io_cache;
_rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0);
if (!_rq_bio_info_cache)
kmem_cache_destroy(_rq_bio_info_cache);
out_free_rq_tio_cache:
kmem_cache_destroy(_rq_tio_cache);
-out_free_tio_cache:
- kmem_cache_destroy(_tio_cache);
out_free_io_cache:
kmem_cache_destroy(_io_cache);
{
kmem_cache_destroy(_rq_bio_info_cache);
kmem_cache_destroy(_rq_tio_cache);
- kmem_cache_destroy(_tio_cache);
kmem_cache_destroy(_io_cache);
unregister_blkdev(_major, _name);
dm_uevent_exit();
static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
{
- mempool_free(tio, md->tio_pool);
+ bio_put(&tio->clone);
}
static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md,
error = -EIO;
if (endio) {
- r = endio(tio->ti, bio, error, &tio->info);
+ r = endio(tio->ti, bio, error);
if (r < 0 || r == DM_ENDIO_REQUEUE)
/*
* error and requeue request are handled
}
free_tio(md, tio);
- bio_put(bio);
dec_pending(io, error);
}
if (!md_in_flight(md))
wake_up(&md->wait);
+ /*
+ * Run this off this callpath, as drivers could invoke end_io while
+ * inside their request_fn (and holding the queue lock). Calling
+ * back into ->request_fn() could deadlock attempting to grab the
+ * queue lock again.
+ */
if (run_queue)
- blk_run_queue(md->queue);
+ blk_run_queue_async(md->queue);
/*
* dm_put() must be at the end of this function. See the comment above
}
EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
-static void __map_bio(struct dm_target *ti, struct bio *clone,
- struct dm_target_io *tio)
+static void __map_bio(struct dm_target *ti, struct dm_target_io *tio)
{
int r;
sector_t sector;
struct mapped_device *md;
+ struct bio *clone = &tio->clone;
clone->bi_end_io = clone_endio;
clone->bi_private = tio;
*/
atomic_inc(&tio->io->io_count);
sector = clone->bi_sector;
- r = ti->type->map(ti, clone, &tio->info);
+ r = ti->type->map(ti, clone);
if (r == DM_MAPIO_REMAPPED) {
/* the bio has been remapped so dispatch it */
/* error the io and bail out, or requeue it if needed */
md = tio->io->md;
dec_pending(tio->io, r);
- bio_put(clone);
free_tio(md, tio);
} else if (r) {
DMWARN("unimplemented target map return value: %d", r);
/*
* Creates a little bio that just does part of a bvec.
*/
-static struct bio *split_bvec(struct bio *bio, sector_t sector,
- unsigned short idx, unsigned int offset,
- unsigned int len, struct bio_set *bs)
+static void split_bvec(struct dm_target_io *tio, struct bio *bio,
+ sector_t sector, unsigned short idx, unsigned int offset,
+ unsigned int len, struct bio_set *bs)
{
- struct bio *clone;
+ struct bio *clone = &tio->clone;
struct bio_vec *bv = bio->bi_io_vec + idx;
- clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
*clone->bi_io_vec = *bv;
clone->bi_sector = sector;
bio_integrity_trim(clone,
bio_sector_offset(bio, idx, offset), len);
}
-
- return clone;
}
/*
* Creates a bio that consists of range of complete bvecs.
*/
-static struct bio *clone_bio(struct bio *bio, sector_t sector,
- unsigned short idx, unsigned short bv_count,
- unsigned int len, struct bio_set *bs)
+static void clone_bio(struct dm_target_io *tio, struct bio *bio,
+ sector_t sector, unsigned short idx,
+ unsigned short bv_count, unsigned int len,
+ struct bio_set *bs)
{
- struct bio *clone;
+ struct bio *clone = &tio->clone;
- clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
__bio_clone(clone, bio);
clone->bi_sector = sector;
clone->bi_idx = idx;
bio_integrity_trim(clone,
bio_sector_offset(bio, idx, 0), len);
}
-
- return clone;
}
static struct dm_target_io *alloc_tio(struct clone_info *ci,
- struct dm_target *ti)
+ struct dm_target *ti, int nr_iovecs)
{
- struct dm_target_io *tio = mempool_alloc(ci->md->tio_pool, GFP_NOIO);
+ struct dm_target_io *tio;
+ struct bio *clone;
+
+ clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, ci->md->bs);
+ tio = container_of(clone, struct dm_target_io, clone);
tio->io = ci->io;
tio->ti = ti;
memset(&tio->info, 0, sizeof(tio->info));
+ tio->target_request_nr = 0;
return tio;
}
static void __issue_target_request(struct clone_info *ci, struct dm_target *ti,
unsigned request_nr, sector_t len)
{
- struct dm_target_io *tio = alloc_tio(ci, ti);
- struct bio *clone;
+ struct dm_target_io *tio = alloc_tio(ci, ti, ci->bio->bi_max_vecs);
+ struct bio *clone = &tio->clone;
- tio->info.target_request_nr = request_nr;
+ tio->target_request_nr = request_nr;
/*
* Discard requests require the bio's inline iovecs be initialized.
* ci->bio->bi_max_vecs is BIO_INLINE_VECS anyway, for both flush
* and discard, so no need for concern about wasted bvec allocations.
*/
- clone = bio_clone_bioset(ci->bio, GFP_NOIO, ci->md->bs);
+ __bio_clone(clone, ci->bio);
if (len) {
clone->bi_sector = ci->sector;
clone->bi_size = to_bytes(len);
}
- __map_bio(ti, clone, tio);
+ __map_bio(ti, tio);
}
static void __issue_target_requests(struct clone_info *ci, struct dm_target *ti,
*/
static void __clone_and_map_simple(struct clone_info *ci, struct dm_target *ti)
{
- struct bio *clone, *bio = ci->bio;
+ struct bio *bio = ci->bio;
struct dm_target_io *tio;
- tio = alloc_tio(ci, ti);
- clone = clone_bio(bio, ci->sector, ci->idx,
- bio->bi_vcnt - ci->idx, ci->sector_count,
- ci->md->bs);
- __map_bio(ti, clone, tio);
+ tio = alloc_tio(ci, ti, bio->bi_max_vecs);
+ clone_bio(tio, bio, ci->sector, ci->idx, bio->bi_vcnt - ci->idx,
+ ci->sector_count, ci->md->bs);
+ __map_bio(ti, tio);
ci->sector_count = 0;
}
-static int __clone_and_map_discard(struct clone_info *ci)
+typedef unsigned (*get_num_requests_fn)(struct dm_target *ti);
+
+static unsigned get_num_discard_requests(struct dm_target *ti)
+{
+ return ti->num_discard_requests;
+}
+
+static unsigned get_num_write_same_requests(struct dm_target *ti)
+{
+ return ti->num_write_same_requests;
+}
+
+typedef bool (*is_split_required_fn)(struct dm_target *ti);
+
+static bool is_split_required_for_discard(struct dm_target *ti)
+{
+ return ti->split_discard_requests;
+}
+
+static int __clone_and_map_changing_extent_only(struct clone_info *ci,
+ get_num_requests_fn get_num_requests,
+ is_split_required_fn is_split_required)
{
struct dm_target *ti;
sector_t len;
return -EIO;
/*
- * Even though the device advertised discard support,
- * that does not mean every target supports it, and
+ * Even though the device advertised support for this type of
+ * request, that does not mean every target supports it, and
* reconfiguration might also have changed that since the
* check was performed.
*/
- if (!ti->num_discard_requests)
+ if (!get_num_requests || !get_num_requests(ti))
return -EOPNOTSUPP;
- if (!ti->split_discard_requests)
+ if (is_split_required && !is_split_required(ti))
len = min(ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
else
len = min(ci->sector_count, max_io_len(ci->sector, ti));
return 0;
}
+static int __clone_and_map_discard(struct clone_info *ci)
+{
+ return __clone_and_map_changing_extent_only(ci, get_num_discard_requests,
+ is_split_required_for_discard);
+}
+
+static int __clone_and_map_write_same(struct clone_info *ci)
+{
+ return __clone_and_map_changing_extent_only(ci, get_num_write_same_requests, NULL);
+}
+
static int __clone_and_map(struct clone_info *ci)
{
- struct bio *clone, *bio = ci->bio;
+ struct bio *bio = ci->bio;
struct dm_target *ti;
sector_t len = 0, max;
struct dm_target_io *tio;
if (unlikely(bio->bi_rw & REQ_DISCARD))
return __clone_and_map_discard(ci);
+ else if (unlikely(bio->bi_rw & REQ_WRITE_SAME))
+ return __clone_and_map_write_same(ci);
ti = dm_table_find_target(ci->map, ci->sector);
if (!dm_target_is_valid(ti))
len += bv_len;
}
- tio = alloc_tio(ci, ti);
- clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
- ci->md->bs);
- __map_bio(ti, clone, tio);
+ tio = alloc_tio(ci, ti, bio->bi_max_vecs);
+ clone_bio(tio, bio, ci->sector, ci->idx, i - ci->idx, len,
+ ci->md->bs);
+ __map_bio(ti, tio);
ci->sector += len;
ci->sector_count -= len;
len = min(remaining, max);
- tio = alloc_tio(ci, ti);
- clone = split_bvec(bio, ci->sector, ci->idx,
- bv->bv_offset + offset, len,
- ci->md->bs);
+ tio = alloc_tio(ci, ti, 1);
+ split_bvec(tio, bio, ci->sector, ci->idx,
+ bv->bv_offset + offset, len, ci->md->bs);
- __map_bio(ti, clone, tio);
+ __map_bio(ti, tio);
ci->sector += len;
ci->sector_count -= len;
static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
{
- struct dm_md_mempools *p;
+ struct dm_md_mempools *p = dm_table_get_md_mempools(t);
- if (md->io_pool && md->tio_pool && md->bs)
- /* the md already has necessary mempools */
+ if (md->io_pool && (md->tio_pool || dm_table_get_type(t) == DM_TYPE_BIO_BASED) && md->bs) {
+ /*
+ * The md already has necessary mempools. Reload just the
+ * bioset because front_pad may have changed because
+ * a different table was loaded.
+ */
+ bioset_free(md->bs);
+ md->bs = p->bs;
+ p->bs = NULL;
goto out;
+ }
- p = dm_table_get_md_mempools(t);
BUG_ON(!p || md->io_pool || md->tio_pool || md->bs);
md->io_pool = p->io_pool;
}
EXPORT_SYMBOL_GPL(dm_noflush_suspending);
-struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity)
+struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, unsigned per_bio_data_size)
{
struct dm_md_mempools *pools = kmalloc(sizeof(*pools), GFP_KERNEL);
unsigned int pool_size = (type == DM_TYPE_BIO_BASED) ? 16 : MIN_IOS;
if (!pools)
return NULL;
+ per_bio_data_size = roundup(per_bio_data_size, __alignof__(struct dm_target_io));
+
pools->io_pool = (type == DM_TYPE_BIO_BASED) ?
mempool_create_slab_pool(MIN_IOS, _io_cache) :
mempool_create_slab_pool(MIN_IOS, _rq_bio_info_cache);
if (!pools->io_pool)
goto free_pools_and_out;
- pools->tio_pool = (type == DM_TYPE_BIO_BASED) ?
- mempool_create_slab_pool(MIN_IOS, _tio_cache) :
- mempool_create_slab_pool(MIN_IOS, _rq_tio_cache);
- if (!pools->tio_pool)
- goto free_io_pool_and_out;
+ pools->tio_pool = NULL;
+ if (type == DM_TYPE_REQUEST_BASED) {
+ pools->tio_pool = mempool_create_slab_pool(MIN_IOS, _rq_tio_cache);
+ if (!pools->tio_pool)
+ goto free_io_pool_and_out;
+ }
pools->bs = (type == DM_TYPE_BIO_BASED) ?
- bioset_create(pool_size, 0) :
+ bioset_create(pool_size,
+ per_bio_data_size + offsetof(struct dm_target_io, clone)) :
bioset_create(pool_size,
offsetof(struct dm_rq_clone_bio_info, clone));
if (!pools->bs)
bioset_free(pools->bs);
free_tio_pool_and_out:
- mempool_destroy(pools->tio_pool);
+ if (pools->tio_pool)
+ mempool_destroy(pools->tio_pool);
free_io_pool_and_out:
mempool_destroy(pools->io_pool);
projects / platform / adaptation / renesas_rcar / renesas_kernel.git / blobdiff