io_uring/rsrc: protect node refs with uring_lock

Currently, for nodes we have an atomic counter and some cached
(non-atomic) refs protected by uring_lock. Let's put all ref
manipulations under uring_lock and get rid of the atomic part.
It's free as in all cases we care about we already hold the lock.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/25b142feed7d831008257d90c8b17c0115d4fc15.1680576071.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 98320f4..36a76c7 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -967,7 +967,7 @@ bool io_aux_cqe(struct io_ring_ctx *ctx, bool defer, u64 user_data, s32 res, u32
        return true;
 }
 
-static void __io_req_complete_post(struct io_kiocb *req)
+static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 {
        struct io_ring_ctx *ctx = req->ctx;
        struct io_rsrc_node *rsrc_node = NULL;
@@ -1003,7 +1003,11 @@ static void __io_req_complete_post(struct io_kiocb *req)
        }
        io_cq_unlock_post(ctx);
 
-       io_put_rsrc_node(rsrc_node);
+       if (rsrc_node) {
+               io_ring_submit_lock(ctx, issue_flags);
+               io_put_rsrc_node(rsrc_node);
+               io_ring_submit_unlock(ctx, issue_flags);
+       }
 }
 
 void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
@@ -1013,12 +1017,12 @@ void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
                io_req_task_work_add(req);
        } else if (!(issue_flags & IO_URING_F_UNLOCKED) ||
                   !(req->ctx->flags & IORING_SETUP_IOPOLL)) {
-               __io_req_complete_post(req);
+               __io_req_complete_post(req, issue_flags);
        } else {
                struct io_ring_ctx *ctx = req->ctx;
 
                mutex_lock(&ctx->uring_lock);
-               __io_req_complete_post(req);
+               __io_req_complete_post(req, issue_flags & ~IO_URING_F_UNLOCKED);
                mutex_unlock(&ctx->uring_lock);
        }
 }
@@ -1120,7 +1124,10 @@ static __cold void io_free_req_tw(struct io_kiocb *req, struct io_tw_state *ts)
 {
        struct io_ring_ctx *ctx = req->ctx;
 
-       io_put_rsrc_node(req->rsrc_node);
+       if (req->rsrc_node) {
+               io_tw_lock(ctx, ts);
+               io_put_rsrc_node(req->rsrc_node);
+       }
        io_dismantle_req(req);
        io_put_task_remote(req->task, 1);
 
@@ -2790,7 +2797,6 @@ static void io_req_caches_free(struct io_ring_ctx *ctx)
 static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
 {
        io_sq_thread_finish(ctx);
-       io_rsrc_refs_drop(ctx);
        /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */
        io_wait_rsrc_data(ctx->buf_data);
        io_wait_rsrc_data(ctx->file_data);

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 1e7c960..1237fc7 100644 (file)
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -27,23 +27,10 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
                                  struct io_mapped_ubuf **pimu,
                                  struct page **last_hpage);
 
-#define IO_RSRC_REF_BATCH      100
-
 /* only define max */
 #define IORING_MAX_FIXED_FILES (1U << 20)
 #define IORING_MAX_REG_BUFFERS (1U << 14)
 
-void io_rsrc_refs_drop(struct io_ring_ctx *ctx)
-       __must_hold(&ctx->uring_lock)
-{
-       struct io_rsrc_node *node = ctx->rsrc_node;
-
-       if (node && node->cached_refs) {
-               io_rsrc_put_node(node, node->cached_refs);
-               node->cached_refs = 0;
-       }
-}
-
 int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
 {
        unsigned long page_limit, cur_pages, new_pages;
@@ -153,13 +140,6 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slo
        *slot = NULL;
 }
 
-void io_rsrc_refs_refill(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
-       __must_hold(&ctx->uring_lock)
-{
-       node->cached_refs += IO_RSRC_REF_BATCH;
-       refcount_add(IO_RSRC_REF_BATCH, &node->refs);
-}
-
 static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
 {
        struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
@@ -225,7 +205,8 @@ void io_rsrc_node_destroy(struct io_rsrc_node *ref_node)
        kfree(ref_node);
 }
 
-__cold void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
+void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
+       __must_hold(&node->rsrc_data->ctx->uring_lock)
 {
        struct io_ring_ctx *ctx = node->rsrc_data->ctx;
        unsigned long flags;
@@ -269,7 +250,7 @@ static struct io_rsrc_node *io_rsrc_node_alloc(void)
        if (!ref_node)
                return NULL;
 
-       refcount_set(&ref_node->refs, 1);
+       ref_node->refs = 1;
        INIT_LIST_HEAD(&ref_node->node);
        INIT_LIST_HEAD(&ref_node->rsrc_list);
        ref_node->done = false;
@@ -283,8 +264,6 @@ void io_rsrc_node_switch(struct io_ring_ctx *ctx,
        WARN_ON_ONCE(!ctx->rsrc_backup_node);
        WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
 
-       io_rsrc_refs_drop(ctx);
-
        if (data_to_kill) {
                struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
 
@@ -295,14 +274,13 @@ void io_rsrc_node_switch(struct io_ring_ctx *ctx,
 
                atomic_inc(&data_to_kill->refs);
                /* put master ref */
-               io_rsrc_put_node(rsrc_node, 1);
+               io_put_rsrc_node(rsrc_node);
                ctx->rsrc_node = NULL;
        }
 
        if (!ctx->rsrc_node) {
                ctx->rsrc_node = ctx->rsrc_backup_node;
                ctx->rsrc_backup_node = NULL;
-               ctx->rsrc_node->cached_refs = 0;
        }
 }
 

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 8164777..a961030 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -37,13 +37,12 @@ struct io_rsrc_data {
 };
 
 struct io_rsrc_node {
-       refcount_t                      refs;
        struct list_head                node;
        struct list_head                rsrc_list;
        struct io_rsrc_data             *rsrc_data;
        struct llist_node               llist;
+       int                             refs;
        bool                            done;
-       int                             cached_refs;
 };
 
 struct io_mapped_ubuf {
@@ -57,10 +56,8 @@ struct io_mapped_ubuf {
 void io_rsrc_put_tw(struct callback_head *cb);
 void io_rsrc_node_ref_zero(struct io_rsrc_node *node);
 void io_rsrc_put_work(struct work_struct *work);
-void io_rsrc_refs_refill(struct io_ring_ctx *ctx, struct io_rsrc_node *node);
 void io_wait_rsrc_data(struct io_rsrc_data *data);
 void io_rsrc_node_destroy(struct io_rsrc_node *ref_node);
-void io_rsrc_refs_drop(struct io_ring_ctx *ctx);
 int io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
 int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
                          struct io_rsrc_node *node, void *rsrc);
@@ -109,38 +106,22 @@ int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
 int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
                        unsigned int size, unsigned int type);
 
-static inline void io_rsrc_put_node(struct io_rsrc_node *node, int nr)
-{
-       if (refcount_sub_and_test(nr, &node->refs))
-               io_rsrc_node_ref_zero(node);
-}
-
 static inline void io_put_rsrc_node(struct io_rsrc_node *node)
 {
-       if (node)
-               io_rsrc_put_node(node, 1);
+       if (node && !--node->refs)
+               io_rsrc_node_ref_zero(node);
 }
 
 static inline void io_req_put_rsrc_locked(struct io_kiocb *req,
                                          struct io_ring_ctx *ctx)
-       __must_hold(&ctx->uring_lock)
 {
-       struct io_rsrc_node *node = req->rsrc_node;
-
-       if (node) {
-               if (node == ctx->rsrc_node)
-                       node->cached_refs++;
-               else
-                       io_rsrc_put_node(node, 1);
-       }
+       io_put_rsrc_node(req->rsrc_node);
 }
 
 static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx,
                                       struct io_rsrc_node *node)
 {
-       node->cached_refs--;
-       if (unlikely(node->cached_refs < 0))
-               io_rsrc_refs_refill(ctx, node);
+       node->refs++;
 }
 
 static inline void io_req_set_rsrc_node(struct io_kiocb *req,