void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw,
sector_t bi_sector, unsigned bi_sectors, bool end,
- unsigned long duration_jiffies,
+ unsigned long start_time,
struct dm_stats_aux *stats_aux)
{
struct dm_stat *s;
sector_t end_sector;
struct dm_stats_last_position *last;
bool got_precise_time;
+ unsigned long duration_jiffies = 0;
if (unlikely(!bi_sectors))
return;
));
WRITE_ONCE(last->last_sector, end_sector);
WRITE_ONCE(last->last_rw, bi_rw);
- }
+ } else
+ duration_jiffies = jiffies - start_time;
rcu_read_lock();
}
EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
-static void start_io_acct(struct dm_io *io)
+static bool bio_is_flush_with_data(struct bio *bio)
{
- struct mapped_device *md = io->md;
- struct bio *bio = io->orig_bio;
+ return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size);
+}
+
+static void dm_io_acct(bool end, struct mapped_device *md, struct bio *bio,
+ unsigned long start_time, struct dm_stats_aux *stats_aux)
+{
+ bool is_flush_with_data;
+ unsigned int bi_size;
+
+ /* If REQ_PREFLUSH set save any payload but do not account it */
+ is_flush_with_data = bio_is_flush_with_data(bio);
+ if (is_flush_with_data) {
+ bi_size = bio->bi_iter.bi_size;
+ bio->bi_iter.bi_size = 0;
+ }
+
+ if (!end)
+ bio_start_io_acct_time(bio, start_time);
+ else
+ bio_end_io_acct(bio, start_time);
- bio_start_io_acct_time(bio, io->start_time);
if (unlikely(dm_stats_used(&md->stats)))
dm_stats_account_io(&md->stats, bio_data_dir(bio),
bio->bi_iter.bi_sector, bio_sectors(bio),
- false, 0, &io->stats_aux);
+ end, start_time, stats_aux);
+
+ /* Restore bio's payload so it does get accounted upon requeue */
+ if (is_flush_with_data)
+ bio->bi_iter.bi_size = bi_size;
+}
+
+static void start_io_acct(struct dm_io *io)
+{
+ dm_io_acct(false, io->md, io->orig_bio, io->start_time, &io->stats_aux);
}
static void end_io_acct(struct mapped_device *md, struct bio *bio,
unsigned long start_time, struct dm_stats_aux *stats_aux)
{
- unsigned long duration = jiffies - start_time;
-
- bio_end_io_acct(bio, start_time);
-
- if (unlikely(dm_stats_used(&md->stats)))
- dm_stats_account_io(&md->stats, bio_data_dir(bio),
- bio->bi_iter.bi_sector, bio_sectors(bio),
- true, duration, stats_aux);
+ dm_io_acct(true, md, bio, start_time, stats_aux);
}
static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
if (io_error == BLK_STS_DM_REQUEUE)
return;
- if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
+ if (bio_is_flush_with_data(bio)) {
/*
* Preflush done for flush with data, reissue
* without REQ_PREFLUSH.