From e27363472f9bc96db831ceb2c31cf8b9a7c5b6f3 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Wed, 9 Mar 2022 16:07:06 -0500 Subject: [PATCH] dm: factor out dm_io_complete Optimizes dm_io_dec_pending() slightly by avoiding local variables. Signed-off-by: Mike Snitzer --- drivers/md/dm.c | 149 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 77 insertions(+), 72 deletions(-) diff --git a/drivers/md/dm.c b/drivers/md/dm.c index c470f54..f8df6e3 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -841,89 +841,94 @@ static int __noflush_suspending(struct mapped_device *md) return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); } +static void dm_io_complete(struct dm_io *io) +{ + blk_status_t io_error; + struct mapped_device *md = io->md; + struct bio *bio = io->orig_bio; + + if (io->status == BLK_STS_DM_REQUEUE) { + unsigned long flags; + /* + * Target requested pushing back the I/O. + */ + spin_lock_irqsave(&md->deferred_lock, flags); + if (__noflush_suspending(md) && + !WARN_ON_ONCE(dm_is_zone_write(md, bio))) { + /* NOTE early return due to BLK_STS_DM_REQUEUE below */ + bio_list_add_head(&md->deferred, bio); + } else { + /* + * noflush suspend was interrupted or this is + * a write to a zoned target. + */ + io->status = BLK_STS_IOERR; + } + spin_unlock_irqrestore(&md->deferred_lock, flags); + } + + io_error = io->status; + if (io->was_accounted) + dm_end_io_acct(io, bio); + else if (!io_error) { + /* + * Must handle target that DM_MAPIO_SUBMITTED only to + * then bio_endio() rather than dm_submit_bio_remap() + */ + __dm_start_io_acct(io, bio); + dm_end_io_acct(io, bio); + } + free_io(io); + smp_wmb(); + this_cpu_dec(*md->pending_io); + + /* nudge anyone waiting on suspend queue */ + if (unlikely(wq_has_sleeper(&md->wait))) + wake_up(&md->wait); + + if (io_error == BLK_STS_DM_REQUEUE) { + /* + * Upper layer won't help us poll split bio, io->orig_bio + * may only reflect a subset of the pre-split original, + * so clear REQ_POLLED in case of requeue + */ + bio->bi_opf &= ~REQ_POLLED; + return; + } + + if (bio_is_flush_with_data(bio)) { + /* + * Preflush done for flush with data, reissue + * without REQ_PREFLUSH. + */ + bio->bi_opf &= ~REQ_PREFLUSH; + queue_io(md, bio); + } else { + /* done with normal IO or empty flush */ + if (io_error) + bio->bi_status = io_error; + bio_endio(bio); + } +} + /* * Decrements the number of outstanding ios that a bio has been * cloned into, completing the original io if necc. */ void dm_io_dec_pending(struct dm_io *io, blk_status_t error) { - unsigned long flags; - blk_status_t io_error; - struct bio *bio; - struct mapped_device *md = io->md; - /* Push-back supersedes any I/O errors */ if (unlikely(error)) { + unsigned long flags; spin_lock_irqsave(&io->endio_lock, flags); - if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md))) + if (!(io->status == BLK_STS_DM_REQUEUE && + __noflush_suspending(io->md))) io->status = error; spin_unlock_irqrestore(&io->endio_lock, flags); } - if (atomic_dec_and_test(&io->io_count)) { - bio = io->orig_bio; - if (io->status == BLK_STS_DM_REQUEUE) { - /* - * Target requested pushing back the I/O. - */ - spin_lock_irqsave(&md->deferred_lock, flags); - if (__noflush_suspending(md) && - !WARN_ON_ONCE(dm_is_zone_write(md, bio))) { - /* NOTE early return due to BLK_STS_DM_REQUEUE below */ - bio_list_add_head(&md->deferred, bio); - } else { - /* - * noflush suspend was interrupted or this is - * a write to a zoned target. - */ - io->status = BLK_STS_IOERR; - } - spin_unlock_irqrestore(&md->deferred_lock, flags); - } - - io_error = io->status; - if (io->was_accounted) - dm_end_io_acct(io, bio); - else if (!io_error) { - /* - * Must handle target that DM_MAPIO_SUBMITTED only to - * then bio_endio() rather than dm_submit_bio_remap() - */ - __dm_start_io_acct(io, bio); - dm_end_io_acct(io, bio); - } - free_io(io); - smp_wmb(); - this_cpu_dec(*md->pending_io); - - /* nudge anyone waiting on suspend queue */ - if (unlikely(wq_has_sleeper(&md->wait))) - wake_up(&md->wait); - - if (io_error == BLK_STS_DM_REQUEUE) { - /* - * Upper layer won't help us poll split bio, io->orig_bio - * may only reflect a subset of the pre-split original, - * so clear REQ_POLLED in case of requeue - */ - bio->bi_opf &= ~REQ_POLLED; - return; - } - - if (bio_is_flush_with_data(bio)) { - /* - * Preflush done for flush with data, reissue - * without REQ_PREFLUSH. - */ - bio->bi_opf &= ~REQ_PREFLUSH; - queue_io(md, bio); - } else { - /* done with normal IO or empty flush */ - if (io_error) - bio->bi_status = io_error; - bio_endio(bio); - } - } + if (atomic_dec_and_test(&io->io_count)) + dm_io_complete(io); } void disable_discard(struct mapped_device *md) @@ -1562,7 +1567,7 @@ static void dm_split_and_process_bio(struct mapped_device *md, if (bio->bi_opf & REQ_PREFLUSH) { error = __send_empty_flush(&ci); - /* dm_io_dec_pending submits any data associated with flush */ + /* dm_io_complete submits any data associated with flush */ goto out; } @@ -1575,7 +1580,7 @@ static void dm_split_and_process_bio(struct mapped_device *md, * Remainder must be passed to submit_bio_noacct() so it gets handled * *after* bios already submitted have been completely processed. * We take a clone of the original to store in ci.io->orig_bio to be - * used by dm_end_io_acct() and for dm_io_dec_pending() to use for + * used by dm_end_io_acct() and for dm_io_complete() to use for * completion handling. */ orig_bio = bio_split(bio, bio_sectors(bio) - ci.sector_count, -- 2.7.4