io_uring: remove the need for relying on an io-wq fallback worker

We hit this case when the task is exiting, and we need somewhere to
do background cleanup of requests. Instead of relying on the io-wq
task manager to do this work for us, just stuff it somewhere where
we can safely run it ourselves directly.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index c36bbcd..800b299 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -16,7 +16,6 @@
 #include <linux/kthread.h>
 #include <linux/rculist_nulls.h>
 #include <linux/fs_struct.h>
-#include <linux/task_work.h>
 #include <linux/blk-cgroup.h>
 #include <linux/audit.h>
 #include <linux/cpu.h>
@@ -775,9 +774,6 @@ static int io_wq_manager(void *data)
        complete(&wq->done);
 
        while (!kthread_should_stop()) {
-               if (current->task_works)
-                       task_work_run();
-
                for_each_node(node) {
                        struct io_wqe *wqe = wq->wqes[node];
                        bool fork_worker[2] = { false, false };
@@ -800,9 +796,6 @@ static int io_wq_manager(void *data)
                schedule_timeout(HZ);
        }
 
-       if (current->task_works)
-               task_work_run();
-
 out:
        if (refcount_dec_and_test(&wq->refs)) {
                complete(&wq->done);
@@ -1160,11 +1153,6 @@ void io_wq_destroy(struct io_wq *wq)
                __io_wq_destroy(wq);
 }
 
-struct task_struct *io_wq_get_task(struct io_wq *wq)
-{
-       return wq->manager;
-}
-
 static bool io_wq_worker_affinity(struct io_worker *worker, void *data)
 {
        struct task_struct *task = worker->task;

diff --git a/fs/io-wq.h b/fs/io-wq.h
index 096f102..a161070 100644 (file)
--- a/fs/io-wq.h
+++ b/fs/io-wq.h
@@ -124,8 +124,6 @@ typedef bool (work_cancel_fn)(struct io_wq_work *, void *);
 enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
                                        void *data, bool cancel_all);
 
-struct task_struct *io_wq_get_task(struct io_wq *wq);
-
 #if defined(CONFIG_IO_WQ)
 extern void io_wq_worker_sleeping(struct task_struct *);
 extern void io_wq_worker_running(struct task_struct *);

diff --git a/fs/io_uring.c b/fs/io_uring.c
index ace7494..e3eb373 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -456,6 +456,9 @@ struct io_ring_ctx {
 
        struct io_restriction           restrictions;
 
+       /* exit task_work */
+       struct callback_head            *exit_task_work;
+
        /* Keep this last, we don't need it for the fast path */
        struct work_struct              exit_work;
 };
@@ -2328,11 +2331,14 @@ static int io_req_task_work_add(struct io_kiocb *req)
 static void io_req_task_work_add_fallback(struct io_kiocb *req,
                                          task_work_func_t cb)
 {
-       struct task_struct *tsk = io_wq_get_task(req->ctx->io_wq);
+       struct io_ring_ctx *ctx = req->ctx;
+       struct callback_head *head;
 
        init_task_work(&req->task_work, cb);
-       task_work_add(tsk, &req->task_work, TWA_NONE);
-       wake_up_process(tsk);
+       do {
+               head = READ_ONCE(ctx->exit_task_work);
+               req->task_work.next = head;
+       } while (cmpxchg(&ctx->exit_task_work, head, &req->task_work) != head);
 }
 
 static void __io_req_task_cancel(struct io_kiocb *req, int error)
@@ -8835,6 +8841,28 @@ static int io_remove_personalities(int id, void *p, void *data)
        return 0;
 }
 
+static void io_run_ctx_fallback(struct io_ring_ctx *ctx)
+{
+       struct callback_head *work, *head, *next;
+
+       do {
+               do {
+                       head = NULL;
+                       work = READ_ONCE(ctx->exit_task_work);
+               } while (cmpxchg(&ctx->exit_task_work, work, head) != work);
+
+               if (!work)
+                       break;
+
+               do {
+                       next = work->next;
+                       work->func(work);
+                       work = next;
+                       cond_resched();
+               } while (work);
+       } while (1);
+}
+
 static void io_ring_exit_work(struct work_struct *work)
 {
        struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
@@ -8848,6 +8876,7 @@ static void io_ring_exit_work(struct work_struct *work)
         */
        do {
                io_uring_try_cancel_requests(ctx, NULL, NULL);
+               io_run_ctx_fallback(ctx);
        } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
        io_ring_ctx_free(ctx);
 }
@@ -9243,6 +9272,8 @@ static int io_uring_flush(struct file *file, void *data)
                io_req_caches_free(ctx, current);
        }
 
+       io_run_ctx_fallback(ctx);
+
        if (!tctx)
                return 0;