dm: put all polled dm_io instances into a single list

Now that bio_split() isn't used by DM's bio splitting, it is a bit
overkill to link dm_io into an hlist given there is only single dm_io
in the list.

Convert to using a single list for holding all dm_io instances
associated with this bio.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Mike Snitzer <snitzer@kernel.org>

diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h
index d2d188c..f3cfc7a 100644 (file)
--- a/drivers/md/dm-core.h
+++ b/drivers/md/dm-core.h
@@ -259,7 +259,7 @@ struct dm_io {
        spinlock_t lock;
        unsigned long start_time;
        void *data;
-       struct hlist_node node;
+       struct dm_io *next;
        struct task_struct *map_task;
        struct dm_stats_aux stats_aux;
 

diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 6bc2dc8..cd084a7 100644 (file)
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1559,7 +1559,7 @@ static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
 }
 
 /*
- * Reuse ->bi_private as hlist head for storing all dm_io instances
+ * Reuse ->bi_private as dm_io list head for storing all dm_io instances
  * associated with this bio, and this bio's bi_private needs to be
  * stored in dm_io->data before the reuse.
  *
@@ -1567,36 +1567,37 @@ static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
  * touch it after splitting. Meantime it won't be changed by anyone after
  * bio is submitted. So this reuse is safe.
  */
-static inline struct hlist_head *dm_get_bio_hlist_head(struct bio *bio)
+static inline struct dm_io **dm_poll_list_head(struct bio *bio)
 {
-       return (struct hlist_head *)&bio->bi_private;
+       return (struct dm_io **)&bio->bi_private;
 }
 
 static void dm_queue_poll_io(struct bio *bio, struct dm_io *io)
 {
-       struct hlist_head *head = dm_get_bio_hlist_head(bio);
+       struct dm_io **head = dm_poll_list_head(bio);
 
        if (!(bio->bi_opf & REQ_DM_POLL_LIST)) {
                bio->bi_opf |= REQ_DM_POLL_LIST;
                /*
                 * Save .bi_private into dm_io, so that we can reuse
-                * .bi_private as hlist head for storing dm_io list
+                * .bi_private as dm_io list head for storing dm_io list
                 */
                io->data = bio->bi_private;
 
-               INIT_HLIST_HEAD(head);
-
                /* tell block layer to poll for completion */
                bio->bi_cookie = ~BLK_QC_T_NONE;
+
+               io->next = NULL;
        } else {
                /*
                 * bio recursed due to split, reuse original poll list,
                 * and save bio->bi_private too.
                 */
-               io->data = hlist_entry(head->first, struct dm_io, node)->data;
+               io->data = (*head)->data;
+               io->next = *head;
        }
 
-       hlist_add_head(&io->node, head);
+       *head = io;
 }
 
 /*
@@ -1685,8 +1686,8 @@ out:
         * Drop the extra reference count for non-POLLED bio, and hold one
         * reference for POLLED bio, which will be released in dm_poll_bio
         *
-        * Add every dm_io instance into the hlist_head which is stored in
-        * bio->bi_private, so that dm_poll_bio can poll them all.
+        * Add every dm_io instance into the dm_io list head which is stored
+        * in bio->bi_private, so that dm_poll_bio can poll them all.
         */
        if (error || !ci.submit_as_polled) {
                /*
@@ -1748,18 +1749,16 @@ static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob,
 static int dm_poll_bio(struct bio *bio, struct io_comp_batch *iob,
                       unsigned int flags)
 {
-       struct hlist_head *head = dm_get_bio_hlist_head(bio);
-       struct hlist_head tmp = HLIST_HEAD_INIT;
-       struct hlist_node *next;
-       struct dm_io *io;
+       struct dm_io **head = dm_poll_list_head(bio);
+       struct dm_io *list = *head;
+       struct dm_io *tmp = NULL;
+       struct dm_io *curr, *next;
 
        /* Only poll normal bio which was marked as REQ_DM_POLL_LIST */
        if (!(bio->bi_opf & REQ_DM_POLL_LIST))
                return 0;
 
-       WARN_ON_ONCE(hlist_empty(head));
-
-       hlist_move_list(head, &tmp);
+       WARN_ON_ONCE(!list);
 
        /*
         * Restore .bi_private before possibly completing dm_io.
@@ -1770,24 +1769,27 @@ static int dm_poll_bio(struct bio *bio, struct io_comp_batch *iob,
         * clearing REQ_DM_POLL_LIST here.
         */
        bio->bi_opf &= ~REQ_DM_POLL_LIST;
-       bio->bi_private = hlist_entry(tmp.first, struct dm_io, node)->data;
+       bio->bi_private = list->data;
 
-       hlist_for_each_entry_safe(io, next, &tmp, node) {
-               if (dm_poll_dm_io(io, iob, flags)) {
-                       hlist_del_init(&io->node);
+       for (curr = list, next = curr->next; curr; curr = next, next =
+                       curr ? curr->next : NULL) {
+               if (dm_poll_dm_io(curr, iob, flags)) {
                        /*
                         * clone_endio() has already occurred, so no
                         * error handling is needed here.
                         */
-                       __dm_io_dec_pending(io);
+                       __dm_io_dec_pending(curr);
+               } else {
+                       curr->next = tmp;
+                       tmp = curr;
                }
        }
 
        /* Not done? */
-       if (!hlist_empty(&tmp)) {
+       if (tmp) {
                bio->bi_opf |= REQ_DM_POLL_LIST;
                /* Reset bio->bi_private to dm_io list head */
-               hlist_move_list(&tmp, head);
+               *head = tmp;
                return 0;
        }
        return 1;