io_uring: add IORING_SETUP_TASKRUN_FLAG

If IORING_SETUP_COOP_TASKRUN is set to use cooperative scheduling for
running task_work, then IORING_SETUP_TASKRUN_FLAG can be set so the
application can tell if task_work is pending in the kernel for this
ring. This allows use cases like io_uring_peek_cqe() to still function
appropriately, or for the task to know when it would be useful to
call io_uring_wait_cqe() to run pending events.

Reviewed-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/20220426014904.60384-7-axboe@kernel.dk
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 0b9ae36159118e8fcce54e6d0acb674a849126f3..72cb2d50125c60f2e802cba6464cbf38e558e620 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2506,6 +2506,8 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked)
 {
        if (!ctx)
                return;
+       if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
+               atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
        if (*locked) {
                io_submit_flush_completions(ctx);
                mutex_unlock(&ctx->uring_lock);
@@ -2646,6 +2648,9 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority)
        if (running)
                return;
 
+       if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
+               atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
+
        if (likely(!task_work_add(tsk, &tctx->task_work, ctx->notify_method)))
                return;
 
@@ -11333,12 +11338,15 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
        ret = -EINVAL;
        if (ctx->flags & IORING_SETUP_SQPOLL) {
                /* IPI related flags don't make sense with SQPOLL */
-               if (ctx->flags & IORING_SETUP_COOP_TASKRUN)
+               if (ctx->flags & (IORING_SETUP_COOP_TASKRUN |
+                                 IORING_SETUP_TASKRUN_FLAG))
                        goto err;
                ctx->notify_method = TWA_SIGNAL_NO_IPI;
        } else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) {
                ctx->notify_method = TWA_SIGNAL_NO_IPI;
        } else {
+               if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
+                       goto err;
                ctx->notify_method = TWA_SIGNAL;
        }
 
@@ -11440,10 +11448,10 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
                        IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
                        IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
                        IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |
-                       IORING_SETUP_COOP_TASKRUN))
+                       IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG))
                return -EINVAL;
 
-       return  io_uring_create(entries, &p, params);
+       return io_uring_create(entries, &p, params);
 }
 
 SYSCALL_DEFINE2(io_uring_setup, u32, entries,

diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index a84f29d657c3360bc4b9a21dbad6dc2f87d640ae..fad63564678acbd29b122863d34012f123ef9e21 100644 (file)
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -110,6 +110,12 @@ enum {
  * a task running in userspace, and saves an IPI.
  */
 #define IORING_SETUP_COOP_TASKRUN      (1U << 8)
+/*
+ * If COOP_TASKRUN is set, get notified if task work is available for
+ * running and a kernel transition would be needed to run it. This sets
+ * IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN.
+ */
+#define IORING_SETUP_TASKRUN_FLAG      (1U << 9)
 
 enum {
        IORING_OP_NOP,
@@ -256,6 +262,7 @@ struct io_sqring_offsets {
  */
 #define IORING_SQ_NEED_WAKEUP  (1U << 0) /* needs io_uring_enter wakeup */
 #define IORING_SQ_CQ_OVERFLOW  (1U << 1) /* CQ ring is overflown */
+#define IORING_SQ_TASKRUN      (1U << 2) /* task should enter the kernel */
 
 struct io_cqring_offsets {
        __u32 head;