io_uring: complete all requests in task context

This patch adds ctx->task_complete flag. If set, we'll complete all
requests in the context of the original task. Note, this extends to
completion CQE posting only but not io_kiocb cleanup / free, e.g. io-wq
may free the requests in the free calllback. This flag will be used
later for optimisations purposes.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/21ece72953f76bb2e77659a72a14326227ab6460.1670384893.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h
index 29e519752da46daa82af84aea3b0f5914b749528..934e5dd4ccc08d14f8d8731a41fd73c3c73d4d96 100644 (file)
--- a/include/linux/io_uring.h
+++ b/include/linux/io_uring.h
@@ -11,6 +11,8 @@ enum io_uring_cmd_flags {
        IO_URING_F_UNLOCKED             = 2,
        /* the request is executed from poll, it should not be freed */
        IO_URING_F_MULTISHOT            = 4,
+       /* executed by io-wq */
+       IO_URING_F_IOWQ                 = 8,
        /* int's last bit, sign checks are usually faster than a bit test */
        IO_URING_F_NONBLOCK             = INT_MIN,
 

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index accdfecee9534e9b66d3b72692140f760102cfa0..6be1e1359c898576a631defe189becd684fdfdc8 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -208,6 +208,8 @@ struct io_ring_ctx {
                unsigned int            drain_disabled: 1;
                unsigned int            has_evfd: 1;
                unsigned int            syscall_iopoll: 1;
+               /* all CQEs should be posted only by the submitter task */
+               unsigned int            task_complete: 1;
        } ____cacheline_aligned_in_smp;
 
        /* submission data */

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index ba5ef1e675c1c186ad8aa37ea35b92eb4a462a39..13d56472dd49faeefcaf4cb168da9bddc034f1af 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -921,8 +921,11 @@ static void __io_req_complete_post(struct io_kiocb *req)
 
 void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 {
-       if (!(issue_flags & IO_URING_F_UNLOCKED) ||
-           !(req->ctx->flags & IORING_SETUP_IOPOLL)) {
+       if (req->ctx->task_complete && (issue_flags & IO_URING_F_IOWQ)) {
+               req->io_task_work.func = io_req_task_complete;
+               io_req_task_work_add(req);
+       } else if (!(issue_flags & IO_URING_F_UNLOCKED) ||
+                  !(req->ctx->flags & IORING_SETUP_IOPOLL)) {
                __io_req_complete_post(req);
        } else {
                struct io_ring_ctx *ctx = req->ctx;
@@ -1830,7 +1833,7 @@ void io_wq_submit_work(struct io_wq_work *work)
 {
        struct io_kiocb *req = container_of(work, struct io_kiocb, work);
        const struct io_op_def *def = &io_op_defs[req->opcode];
-       unsigned int issue_flags = IO_URING_F_UNLOCKED;
+       unsigned int issue_flags = IO_URING_F_UNLOCKED | IO_URING_F_IOWQ;
        bool needs_poll = false;
        int ret = 0, err = -ECANCELED;
 
@@ -3490,6 +3493,11 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
        if (!ctx)
                return -ENOMEM;
 
+       if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
+           !(ctx->flags & IORING_SETUP_IOPOLL) &&
+           !(ctx->flags & IORING_SETUP_SQPOLL))
+               ctx->task_complete = true;
+
        /*
         * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
         * space applications don't need to do io completion events