io_uring: set task_work notify method at init time

While doing so, switch SQPOLL to TWA_SIGNAL_NO_IPI as well, as that
just does a task wakeup and then we can remove the special wakeup we
have in task_work_add.

Reviewed-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/20220426014904.60384-5-axboe@kernel.dk
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index e92029bd3936a5c8a572d0a06a95915e8cdb3004..3c669d8f5e57fb393c50c8ad7e817d25fc9db81e 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -366,6 +366,7 @@ struct io_ring_ctx {
 
                struct io_rings         *rings;
                unsigned int            flags;
+               enum task_work_notify_mode      notify_method;
                unsigned int            compat: 1;
                unsigned int            drain_next: 1;
                unsigned int            restricted: 1;
@@ -2621,8 +2622,8 @@ static void tctx_task_work(struct callback_head *cb)
 static void io_req_task_work_add(struct io_kiocb *req, bool priority)
 {
        struct task_struct *tsk = req->task;
+       struct io_ring_ctx *ctx = req->ctx;
        struct io_uring_task *tctx = tsk->io_uring;
-       enum task_work_notify_mode notify;
        struct io_wq_work_node *node;
        unsigned long flags;
        bool running;
@@ -2645,18 +2646,8 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority)
        if (running)
                return;
 
-       /*
-        * SQPOLL kernel thread doesn't need notification, just a wakeup. For
-        * all other cases, use TWA_SIGNAL unconditionally to ensure we're
-        * processing task_work. There's no reliable way to tell if TWA_RESUME
-        * will do the job.
-        */
-       notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
-       if (likely(!task_work_add(tsk, &tctx->task_work, notify))) {
-               if (notify == TWA_NONE)
-                       wake_up_process(tsk);
+       if (likely(!task_work_add(tsk, &tctx->task_work, ctx->notify_method)))
                return;
-       }
 
        spin_lock_irqsave(&tctx->task_lock, flags);
        tctx->task_running = false;
@@ -11335,6 +11326,14 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
        if (!capable(CAP_IPC_LOCK))
                ctx->user = get_uid(current_user());
 
+       /*
+        * For SQPOLL, we just need a wakeup, always.
+        */
+       if (ctx->flags & IORING_SETUP_SQPOLL)
+               ctx->notify_method = TWA_SIGNAL_NO_IPI;
+       else
+               ctx->notify_method = TWA_SIGNAL;
+
        /*
         * This is just grabbed for accounting purposes. When a process exits,
         * the mm is exited and dropped before the files, hence we need to hang