io_uring: inline llist_add()

We'll need to grab some information from the previous request in the tw
list, inline llist_add(), it'll be used in the following patch.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/f0165493af7b379943c792114b972f331e7d7d10.1680782017.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 6f175fe..786ecfa 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1303,8 +1303,15 @@ static __cold void io_fallback_tw(struct io_uring_task *tctx)
 static void io_req_local_work_add(struct io_kiocb *req)
 {
        struct io_ring_ctx *ctx = req->ctx;
+       struct llist_node *first;
 
-       if (!llist_add(&req->io_task_work.node, &ctx->work_llist))
+       first = READ_ONCE(ctx->work_llist.first);
+       do {
+               req->io_task_work.node.next = first;
+       } while (!try_cmpxchg(&ctx->work_llist.first, &first,
+                             &req->io_task_work.node));
+
+       if (first)
                return;
 
        /* needed for the following wake up */