Merge tag 'v6.1-rockchip-dtsfixes1' of git://git.kernel.org/pub/scm/linux/kernel...

[platform/kernel/linux-starfive.git] / io_uring / io_uring.c

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 6cc16e3..8840cf3 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -176,6 +176,11 @@ static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
        return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
 }
 
+static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx)
+{
+       return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);
+}
+
 static bool io_match_linked(struct io_kiocb *head)
 {
        struct io_kiocb *req;
@@ -1173,7 +1178,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
        }
 }
 
-int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
+int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked)
 {
        struct llist_node *node;
        struct llist_node fake;
@@ -1192,7 +1197,7 @@ again:
                struct io_kiocb *req = container_of(node, struct io_kiocb,
                                                    io_task_work.node);
                prefetch(container_of(next, struct io_kiocb, io_task_work.node));
-               req->io_task_work.func(req, &locked);
+               req->io_task_work.func(req, locked);
                ret++;
                node = next;
        }
@@ -1208,7 +1213,7 @@ again:
                goto again;
        }
 
-       if (locked)
+       if (*locked)
                io_submit_flush_completions(ctx);
        trace_io_uring_local_work_run(ctx, ret, loops);
        return ret;
@@ -1225,7 +1230,7 @@ int io_run_local_work(struct io_ring_ctx *ctx)
 
        __set_current_state(TASK_RUNNING);
        locked = mutex_trylock(&ctx->uring_lock);
-       ret = __io_run_local_work(ctx, locked);
+       ret = __io_run_local_work(ctx, &locked);
        if (locked)
                mutex_unlock(&ctx->uring_lock);
 
@@ -1446,8 +1451,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
                    io_task_work_pending(ctx)) {
                        u32 tail = ctx->cached_cq_tail;
 
-                       if (!llist_empty(&ctx->work_llist))
-                               __io_run_local_work(ctx, true);
+                       (void) io_run_local_work_locked(ctx);
 
                        if (task_work_pending(current) ||
                            wq_list_empty(&ctx->iopoll_list)) {
@@ -1764,7 +1768,7 @@ int io_poll_issue(struct io_kiocb *req, bool *locked)
        io_tw_lock(req->ctx, locked);
        if (unlikely(req->task->flags & PF_EXITING))
                return -EFAULT;
-       return io_issue_sqe(req, IO_URING_F_NONBLOCK);
+       return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT);
 }
 
 struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
@@ -2316,7 +2320,7 @@ static inline bool io_has_work(struct io_ring_ctx *ctx)
 static inline bool io_should_wake(struct io_wait_queue *iowq)
 {
        struct io_ring_ctx *ctx = iowq->ctx;
-       int dist = ctx->cached_cq_tail - (int) iowq->cq_tail;
+       int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail;
 
        /*
         * Wake up if we have enough events, or if a timeout occurred since we
@@ -2400,7 +2404,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
                        return ret;
                io_cqring_overflow_flush(ctx);
 
-               if (io_cqring_events(ctx) >= min_events)
+               /* if user messes with these they will just get an early return */
+               if (__io_cqring_events_user(ctx) >= min_events)
                        return 0;
        } while (ret > 0);