io_uring: export io_put_task()

Make io_put_task() available to non-core parts of io_uring, we'll need
it for notification infrastructure.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/3686807d4c03b72e389947b0e8692d4d44334ef0.1657643355.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index d54b8b7..368c34d 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -4,6 +4,7 @@
 #include <linux/blkdev.h>
 #include <linux/task_work.h>
 #include <linux/bitmap.h>
+#include <linux/llist.h>
 #include <uapi/linux/io_uring.h>
 
 struct io_wq_work_node {
@@ -43,6 +44,30 @@ struct io_hash_table {
        unsigned                hash_bits;
 };
 
+/*
+ * Arbitrary limit, can be raised if need be
+ */
+#define IO_RINGFD_REG_MAX 16
+
+struct io_uring_task {
+       /* submission side */
+       int                             cached_refs;
+       const struct io_ring_ctx        *last;
+       struct io_wq                    *io_wq;
+       struct file                     *registered_rings[IO_RINGFD_REG_MAX];
+
+       struct xarray                   xa;
+       struct wait_queue_head          wait;
+       atomic_t                        in_idle;
+       atomic_t                        inflight_tracked;
+       struct percpu_counter           inflight;
+
+       struct { /* task_work */
+               struct llist_head       task_list;
+               struct callback_head    task_work;
+       } ____cacheline_aligned_in_smp;
+};
+
 struct io_uring {
        u32 head ____cacheline_aligned_in_smp;
        u32 tail ____cacheline_aligned_in_smp;

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 4b3fd64..7795cfe 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -608,7 +608,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)
        return ret;
 }
 
-static void __io_put_task(struct task_struct *task, int nr)
+void __io_put_task(struct task_struct *task, int nr)
 {
        struct io_uring_task *tctx = task->io_uring;
 
@@ -618,15 +618,6 @@ static void __io_put_task(struct task_struct *task, int nr)
        put_task_struct_many(task, nr);
 }
 
-/* must to be called somewhat shortly after putting a request */
-static inline void io_put_task(struct task_struct *task, int nr)
-{
-       if (likely(task == current))
-               task->io_uring->cached_refs += nr;
-       else
-               __io_put_task(task, nr);
-}
-
 static void io_task_refs_refill(struct io_uring_task *tctx)
 {
        unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 5db0a60..b1c0c0a 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -71,6 +71,7 @@ void io_wq_submit_work(struct io_wq_work *work);
 
 void io_free_req(struct io_kiocb *req);
 void io_queue_next(struct io_kiocb *req);
+void __io_put_task(struct task_struct *task, int nr);
 
 bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
                        bool cancel_all);
@@ -258,4 +259,13 @@ static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx)
                __io_commit_cqring_flush(ctx);
 }
 
+/* must to be called somewhat shortly after putting a request */
+static inline void io_put_task(struct task_struct *task, int nr)
+{
+       if (likely(task == current))
+               task->io_uring->cached_refs += nr;
+       else
+               __io_put_task(task, nr);
+}
+
 #endif

diff --git a/io_uring/tctx.h b/io_uring/tctx.h
index 8a33ff6..25974be 100644 (file)
--- a/io_uring/tctx.h
+++ b/io_uring/tctx.h
@@ -1,31 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
 
-#include <linux/llist.h>
-
-/*
- * Arbitrary limit, can be raised if need be
- */
-#define IO_RINGFD_REG_MAX 16
-
-struct io_uring_task {
-       /* submission side */
-       int                             cached_refs;
-       const struct io_ring_ctx        *last;
-       struct io_wq                    *io_wq;
-       struct file                     *registered_rings[IO_RINGFD_REG_MAX];
-
-       struct xarray                   xa;
-       struct wait_queue_head          wait;
-       atomic_t                        in_idle;
-       atomic_t                        inflight_tracked;
-       struct percpu_counter           inflight;
-
-       struct { /* task_work */
-               struct llist_head       task_list;
-               struct callback_head    task_work;
-       } ____cacheline_aligned_in_smp;
-};
-
 struct io_tctx_node {
        struct list_head        ctx_node;
        struct task_struct      *task;