io_uring: fix counter inc/dec mismatch in async_list

We could queue a work for each req in defer and link list without
increasing async_list->cnt, so we shouldn't decrease it while exiting
from workqueue as well if we didn't process the req in async list.

Thanks to Jens Axboe <axboe@kernel.dk> for his guidance.

Fixes: 31b515106428 ("io_uring: allow workqueue item to handle multiple buffered requests")
Signed-off-by: Zhengyuan Liu <liuzhengyuan@kylinos.cn>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 708d133..5ec06e5 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -333,7 +333,8 @@ struct io_kiocb {
 #define REQ_F_IO_DRAIN         16      /* drain existing IO first */
 #define REQ_F_IO_DRAINED       32      /* drain done */
 #define REQ_F_LINK             64      /* linked sqes */
-#define REQ_F_FAIL_LINK                128     /* fail rest of links */
+#define REQ_F_LINK_DONE                128     /* linked sqes done */
+#define REQ_F_FAIL_LINK                256     /* fail rest of links */
        u64                     user_data;
        u32                     result;
        u32                     sequence;
@@ -632,6 +633,7 @@ static void io_req_link_next(struct io_kiocb *req)
                        nxt->flags |= REQ_F_LINK;
                }
 
+               nxt->flags |= REQ_F_LINK_DONE;
                INIT_WORK(&nxt->work, io_sq_wq_submit_work);
                queue_work(req->ctx->sqo_wq, &nxt->work);
        }
@@ -1844,6 +1846,10 @@ restart:
                /* async context always use a copy of the sqe */
                kfree(sqe);
 
+               /* req from defer and link list needn't decrease async cnt */
+               if (req->flags & (REQ_F_IO_DRAINED | REQ_F_LINK_DONE))
+                       goto out;
+
                if (!async_list)
                        break;
                if (!list_empty(&req_list)) {
@@ -1891,6 +1897,7 @@ restart:
                }
        }
 
+out:
        if (cur_mm) {
                set_fs(old_fs);
                unuse_mm(cur_mm);