Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

[platform/kernel/linux-starfive.git] / io_uring / io_uring.h

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index ab4b2a1..2711865 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -3,6 +3,8 @@
 
 #include <linux/errno.h>
 #include <linux/lockdep.h>
+#include <linux/resume_user_mode.h>
+#include <linux/kasan.h>
 #include <linux/io_uring_types.h>
 #include <uapi/linux/eventpoll.h>
 #include "io-wq.h"
@@ -28,8 +30,6 @@ enum {
 struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);
 bool io_req_cqe_overflow(struct io_kiocb *req);
 int io_run_task_work_sig(struct io_ring_ctx *ctx);
-int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked);
-int io_run_local_work(struct io_ring_ctx *ctx);
 void io_req_defer_failed(struct io_kiocb *req, s32 res);
 void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags);
 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
@@ -72,7 +72,6 @@ void io_wq_submit_work(struct io_wq_work *work);
 
 void io_free_req(struct io_kiocb *req);
 void io_queue_next(struct io_kiocb *req);
-void __io_put_task(struct task_struct *task, int nr);
 void io_task_refs_refill(struct io_uring_task *tctx);
 bool __io_alloc_req_refill(struct io_ring_ctx *ctx);
 
@@ -222,6 +221,13 @@ static inline void io_commit_cqring(struct io_ring_ctx *ctx)
        smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
 }
 
+static inline void io_poll_wq_wake(struct io_ring_ctx *ctx)
+{
+       if (wq_has_sleeper(&ctx->poll_wq))
+               __wake_up(&ctx->poll_wq, TASK_NORMAL, 0,
+                               poll_to_key(EPOLL_URING_WAKE | EPOLLIN));
+}
+
 /* requires smb_mb() prior, see wq_has_sleeper() */
 static inline void __io_cqring_wake(struct io_ring_ctx *ctx)
 {
@@ -270,6 +276,15 @@ static inline int io_run_task_work(void)
         */
        if (test_thread_flag(TIF_NOTIFY_SIGNAL))
                clear_notify_signal();
+       /*
+        * PF_IO_WORKER never returns to userspace, so check here if we have
+        * notify work that needs processing.
+        */
+       if (current->flags & PF_IO_WORKER &&
+           test_thread_flag(TIF_NOTIFY_RESUME)) {
+               __set_current_state(TASK_RUNNING);
+               resume_user_mode_work(NULL);
+       }
        if (task_work_pending(current)) {
                __set_current_state(TASK_RUNNING);
                task_work_run();
@@ -284,42 +299,6 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
        return task_work_pending(current) || !wq_list_empty(&ctx->work_llist);
 }
 
-static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
-{
-       int ret = 0;
-       int ret2;
-
-       if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
-               ret = io_run_local_work(ctx);
-
-       /* want to run this after in case more is added */
-       ret2 = io_run_task_work();
-
-       /* Try propagate error in favour of if tasks were run,
-        * but still make sure to run them if requested
-        */
-       if (ret >= 0)
-               ret += ret2;
-
-       return ret;
-}
-
-static inline int io_run_local_work_locked(struct io_ring_ctx *ctx)
-{
-       bool locked;
-       int ret;
-
-       if (llist_empty(&ctx->work_llist))
-               return 0;
-
-       locked = true;
-       ret = __io_run_local_work(ctx, &locked);
-       /* shouldn't happen! */
-       if (WARN_ON_ONCE(!locked))
-               mutex_lock(&ctx->uring_lock);
-       return ret;
-}
-
 static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)
 {
        if (!*locked) {
@@ -345,19 +324,11 @@ static inline void io_req_complete_defer(struct io_kiocb *req)
 
 static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx)
 {
-       if (unlikely(ctx->off_timeout_used || ctx->drain_active || ctx->has_evfd))
+       if (unlikely(ctx->off_timeout_used || ctx->drain_active ||
+                    ctx->has_evfd || ctx->poll_activated))
                __io_commit_cqring_flush(ctx);
 }
 
-/* must to be called somewhat shortly after putting a request */
-static inline void io_put_task(struct task_struct *task, int nr)
-{
-       if (likely(task == current))
-               task->io_uring->cached_refs += nr;
-       else
-               __io_put_task(task, nr);
-}
-
 static inline void io_get_task_refs(int nr)
 {
        struct io_uring_task *tctx = current->io_uring;
@@ -372,19 +343,31 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx)
        return !ctx->submit_state.free_list.next;
 }
 
-static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx)
+extern struct kmem_cache *req_cachep;
+
+static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx)
 {
-       if (unlikely(io_req_cache_empty(ctx)))
-               return __io_alloc_req_refill(ctx);
-       return true;
+       struct io_kiocb *req;
+
+       req = container_of(ctx->submit_state.free_list.next, struct io_kiocb, comp_list);
+       kasan_unpoison_object_data(req_cachep, req);
+       wq_stack_extract(&ctx->submit_state.free_list);
+       return req;
 }
 
-static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
+static inline bool io_alloc_req(struct io_ring_ctx *ctx, struct io_kiocb **req)
 {
-       struct io_wq_work_node *node;
+       if (unlikely(io_req_cache_empty(ctx))) {
+               if (!__io_alloc_req_refill(ctx))
+                       return false;
+       }
+       *req = io_extract_req(ctx);
+       return true;
+}
 
-       node = wq_stack_extract(&ctx->submit_state.free_list);
-       return container_of(node, struct io_kiocb, comp_list);
+static inline bool io_allowed_defer_tw_run(struct io_ring_ctx *ctx)
+{
+       return likely(ctx->submitter_task == current);
 }
 
 static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx)