io_uring: deduplicate core cancellations sequence

Files and task cancellations go over same steps trying to cancel
requests in io-wq, poll, etc. Deduplicate it with a helper.

note: new io_uring_try_cancel_requests() is former
__io_uring_cancel_task_requests() with files passed as an agrument and
flushing overflowed requests.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 24ad36d..a750c50 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -1003,9 +1003,9 @@ enum io_mem_account {
        ACCT_PINNED,
 };
 
-static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx,
-                                           struct task_struct *task);
-
+static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
+                                        struct task_struct *task,
+                                        struct files_struct *files);
 static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node);
 static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node(
                        struct io_ring_ctx *ctx);
@@ -8817,7 +8817,7 @@ static void io_ring_exit_work(struct work_struct *work)
         * as nobody else will be looking for them.
         */
        do {
-               __io_uring_cancel_task_requests(ctx, NULL);
+               io_uring_try_cancel_requests(ctx, NULL, NULL);
        } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
        io_ring_ctx_free(ctx);
 }
@@ -8931,6 +8931,40 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx,
        }
 }
 
+static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
+                                        struct task_struct *task,
+                                        struct files_struct *files)
+{
+       struct io_task_cancel cancel = { .task = task, .files = files, };
+
+       while (1) {
+               enum io_wq_cancel cret;
+               bool ret = false;
+
+               if (ctx->io_wq) {
+                       cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb,
+                                              &cancel, true);
+                       ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
+               }
+
+               /* SQPOLL thread does its own polling */
+               if (!(ctx->flags & IORING_SETUP_SQPOLL) && !files) {
+                       while (!list_empty_careful(&ctx->iopoll_list)) {
+                               io_iopoll_try_reap_events(ctx);
+                               ret = true;
+                       }
+               }
+
+               ret |= io_poll_remove_all(ctx, task, files);
+               ret |= io_kill_timeouts(ctx, task, files);
+               ret |= io_run_task_work();
+               io_cqring_overflow_flush(ctx, true, task, files);
+               if (!ret)
+                       break;
+               cond_resched();
+       }
+}
+
 static int io_uring_count_inflight(struct io_ring_ctx *ctx,
                                   struct task_struct *task,
                                   struct files_struct *files)
@@ -8950,7 +8984,6 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx,
                                  struct files_struct *files)
 {
        while (!list_empty_careful(&ctx->inflight_list)) {
-               struct io_task_cancel cancel = { .task = task, .files = files };
                DEFINE_WAIT(wait);
                int inflight;
 
@@ -8958,13 +8991,7 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx,
                if (!inflight)
                        break;
 
-               io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true);
-               io_poll_remove_all(ctx, task, files);
-               io_kill_timeouts(ctx, task, files);
-               io_cqring_overflow_flush(ctx, true, task, files);
-               /* cancellations _may_ trigger task work */
-               io_run_task_work();
-
+               io_uring_try_cancel_requests(ctx, task, files);
                prepare_to_wait(&task->io_uring->wait, &wait,
                                TASK_UNINTERRUPTIBLE);
                if (inflight == io_uring_count_inflight(ctx, task, files))
@@ -8973,37 +9000,6 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx,
        }
 }
 
-static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx,
-                                           struct task_struct *task)
-{
-       while (1) {
-               struct io_task_cancel cancel = { .task = task, .files = NULL, };
-               enum io_wq_cancel cret;
-               bool ret = false;
-
-               if (ctx->io_wq) {
-                       cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb,
-                                              &cancel, true);
-                       ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
-               }
-
-               /* SQPOLL thread does its own polling */
-               if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
-                       while (!list_empty_careful(&ctx->iopoll_list)) {
-                               io_iopoll_try_reap_events(ctx);
-                               ret = true;
-                       }
-               }
-
-               ret |= io_poll_remove_all(ctx, task, NULL);
-               ret |= io_kill_timeouts(ctx, task, NULL);
-               ret |= io_run_task_work();
-               if (!ret)
-                       break;
-               cond_resched();
-       }
-}
-
 static void io_disable_sqo_submit(struct io_ring_ctx *ctx)
 {
        mutex_lock(&ctx->uring_lock);
@@ -9033,11 +9029,10 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx,
        }
 
        io_cancel_defer_files(ctx, task, files);
-       io_cqring_overflow_flush(ctx, true, task, files);
 
        io_uring_cancel_files(ctx, task, files);
        if (!files)
-               __io_uring_cancel_task_requests(ctx, task);
+               io_uring_try_cancel_requests(ctx, task, NULL);
 
        if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) {
                atomic_dec(&task->io_uring->in_idle);