io_uring: don't reinstall quiesce node for each tw

There is no need to reinit data and install a new rsrc node every time
we get a task_work, it's detrimental, just execute it and conitnue
waiting.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/3895d3344164cd9b3a0bbb24a6e357e20a13434b.1669821213.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index b36d3253416557c9caba8965e6f8e968995d32ba..d25309400a4591f3f45f9ae54376baa897fd3105 100644 (file)
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -309,22 +309,27 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
        /* As we may drop ->uring_lock, other task may have started quiesce */
        if (data->quiesce)
                return -ENXIO;
+       ret = io_rsrc_node_switch_start(ctx);
+       if (ret)
+               return ret;
+       io_rsrc_node_switch(ctx, data);
+
+       /* kill initial ref, already quiesced if zero */
+       if (atomic_dec_and_test(&data->refs))
+               return 0;
 
        data->quiesce = true;
+       mutex_unlock(&ctx->uring_lock);
        do {
-               ret = io_rsrc_node_switch_start(ctx);
-               if (ret)
-                       break;
-               io_rsrc_node_switch(ctx, data);
-
-               /* kill initial ref, already quiesced if zero */
-               if (atomic_dec_and_test(&data->refs))
-                       break;
-               mutex_unlock(&ctx->uring_lock);
-
                ret = io_run_task_work_sig(ctx);
-               if (ret < 0)
-                       goto reinit;
+               if (ret < 0) {
+                       atomic_inc(&data->refs);
+                       /* wait for all works potentially completing data->done */
+                       flush_delayed_work(&ctx->rsrc_put_work);
+                       reinit_completion(&data->done);
+                       mutex_lock(&ctx->uring_lock);
+                       break;
+               }
 
                flush_delayed_work(&ctx->rsrc_put_work);
                ret = wait_for_completion_interruptible(&data->done);
@@ -338,14 +343,7 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
                         */
                        mutex_unlock(&ctx->uring_lock);
                }
-reinit:
-               atomic_inc(&data->refs);
-               /* wait for all works potentially completing data->done */
-               flush_delayed_work(&ctx->rsrc_put_work);
-               reinit_completion(&data->done);
-
-               mutex_lock(&ctx->uring_lock);
-       } while (ret >= 0);
+       } while (1);
        data->quiesce = false;
 
        return ret;