io_uring: don't expose io_fill_cqe_aux()

Deduplicate some code and add a helper for filling an aux CQE, locking
and notification.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/b7c6557c8f9dc5c4cfb01292116c682a0ff61081.1655455613.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index eeda167..8c1b0e0 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -676,8 +676,8 @@ bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, s32 res,
        return true;
 }
 
-bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
-                    u32 cflags)
+static bool io_fill_cqe_aux(struct io_ring_ctx *ctx,
+                           u64 user_data, s32 res, u32 cflags)
 {
        struct io_uring_cqe *cqe;
 
@@ -704,6 +704,20 @@ bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
        return io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
 }
 
+bool io_post_aux_cqe(struct io_ring_ctx *ctx,
+                    u64 user_data, s32 res, u32 cflags)
+{
+       bool filled;
+
+       spin_lock(&ctx->completion_lock);
+       filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
+       io_commit_cqring(ctx);
+       spin_unlock(&ctx->completion_lock);
+       if (filled)
+               io_cqring_ev_posted(ctx);
+       return filled;
+}
+
 static void __io_req_complete_put(struct io_kiocb *req)
 {
        /*

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 3f06fba..18754fb 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -239,8 +239,7 @@ void io_req_complete_failed(struct io_kiocb *req, s32 res);
 void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
 void io_req_complete_post(struct io_kiocb *req);
 void __io_req_complete_post(struct io_kiocb *req);
-bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
-                    u32 cflags);
+bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
 void io_cqring_ev_posted(struct io_ring_ctx *ctx);
 void __io_commit_cqring_flush(struct io_ring_ctx *ctx);
 

diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c
index 3b89f9a..7c3c5f3 100644 (file)
--- a/io_uring/msg_ring.c
+++ b/io_uring/msg_ring.c
@@ -34,7 +34,6 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)
 {
        struct io_msg *msg = io_kiocb_to_cmd(req);
        struct io_ring_ctx *target_ctx;
-       bool filled;
        int ret;
 
        ret = -EBADFD;
@@ -43,16 +42,8 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)
 
        ret = -EOVERFLOW;
        target_ctx = req->file->private_data;
-
-       spin_lock(&target_ctx->completion_lock);
-       filled = io_fill_cqe_aux(target_ctx, msg->user_data, msg->len, 0);
-       io_commit_cqring(target_ctx);
-       spin_unlock(&target_ctx->completion_lock);
-
-       if (filled) {
-               io_cqring_ev_posted(target_ctx);
+       if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))
                ret = 0;
-       }
 
 done:
        if (ret < 0)

diff --git a/io_uring/net.c b/io_uring/net.c
index fe1fe92..35d0183 100644 (file)
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -644,22 +644,12 @@ retry:
                io_req_set_res(req, ret, 0);
                return IOU_OK;
        }
-       if (ret >= 0) {
-               bool filled;
-
-               spin_lock(&ctx->completion_lock);
-               filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret,
-                                        IORING_CQE_F_MORE);
-               io_commit_cqring(ctx);
-               spin_unlock(&ctx->completion_lock);
-               if (filled) {
-                       io_cqring_ev_posted(ctx);
-                       goto retry;
-               }
-               ret = -ECANCELED;
-       }
 
-       return ret;
+       if (ret < 0)
+               return ret;
+       if (io_post_aux_cqe(ctx, req->cqe.user_data, ret, IORING_CQE_F_MORE))
+               goto retry;
+       return -ECANCELED;
 }
 
 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

diff --git a/io_uring/poll.c b/io_uring/poll.c
index 9ae2982..e0c181f 100644 (file)
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -214,23 +214,15 @@ static int io_poll_check_events(struct io_kiocb *req, bool *locked)
                if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
                        __poll_t mask = mangle_poll(req->cqe.res &
                                                    req->apoll_events);
-                       bool filled;
-
-                       spin_lock(&ctx->completion_lock);
-                       filled = io_fill_cqe_aux(ctx, req->cqe.user_data,
-                                                mask, IORING_CQE_F_MORE);
-                       io_commit_cqring(ctx);
-                       spin_unlock(&ctx->completion_lock);
-                       if (filled) {
-                               io_cqring_ev_posted(ctx);
-                               continue;
-                       }
-                       return -ECANCELED;
-               }
 
-               ret = io_poll_issue(req, locked);
-               if (ret)
-                       return ret;
+                       if (!io_post_aux_cqe(ctx, req->cqe.user_data,
+                                            mask, IORING_CQE_F_MORE))
+                               return -ECANCELED;
+               } else {
+                       ret = io_poll_issue(req, locked);
+                       if (ret)
+                               return ret;
+               }
 
                /*
                 * Release all references, retry if someone tried to restart

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 214ff0d..7fed310 100644 (file)
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -174,17 +174,13 @@ static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
                list_del(&prsrc->list);
 
                if (prsrc->tag) {
-                       if (ctx->flags & IORING_SETUP_IOPOLL)
+                       if (ctx->flags & IORING_SETUP_IOPOLL) {
                                mutex_lock(&ctx->uring_lock);
-
-                       spin_lock(&ctx->completion_lock);
-                       io_fill_cqe_aux(ctx, prsrc->tag, 0, 0);
-                       io_commit_cqring(ctx);
-                       spin_unlock(&ctx->completion_lock);
-                       io_cqring_ev_posted(ctx);
-
-                       if (ctx->flags & IORING_SETUP_IOPOLL)
+                               io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
                                mutex_unlock(&ctx->uring_lock);
+                       } else {
+                               io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
+                       }
                }
 
                rsrc_data->do_put(ctx, prsrc);