io_uring: change ->cqe_cached invariant for CQE32

With IORING_SETUP_CQE32 ->cqe_cached doesn't store a real address but
rather an implicit offset into cqes. Store the real cqe pointer and
increment it accordingly if CQE32.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/1ee1838cba16bed96381a006950b36ba640d998c.1655455613.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 7acb94c..0dbf6a7 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -701,11 +701,8 @@ struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)
 {
        struct io_rings *rings = ctx->rings;
        unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
-       unsigned int shift = 0;
        unsigned int free, queued, len;
 
-       if (ctx->flags & IORING_SETUP_CQE32)
-               shift = 1;
 
        /* userspace may cheat modifying the tail, be safe and do min */
        queued = min(__io_cqring_events(ctx), ctx->cq_entries);
@@ -715,11 +712,19 @@ struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)
        if (!len)
                return NULL;
 
-       ctx->cached_cq_tail++;
+       if (ctx->flags & IORING_SETUP_CQE32) {
+               off <<= 1;
+               len <<= 1;
+       }
+
        ctx->cqe_cached = &rings->cqes[off];
        ctx->cqe_sentinel = ctx->cqe_cached + len;
+
+       ctx->cached_cq_tail++;
        ctx->cqe_cached++;
-       return &rings->cqes[off << shift];
+       if (ctx->flags & IORING_SETUP_CQE32)
+               ctx->cqe_cached++;
+       return &rings->cqes[off];
 }
 
 static bool io_fill_cqe_aux(struct io_ring_ctx *ctx,

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index dfb490e..558a860 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -22,14 +22,10 @@ static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
        if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {
                struct io_uring_cqe *cqe = ctx->cqe_cached;
 
-               if (ctx->flags & IORING_SETUP_CQE32) {
-                       unsigned int off = ctx->cqe_cached - ctx->rings->cqes;
-
-                       cqe += off;
-               }
-
                ctx->cached_cq_tail++;
                ctx->cqe_cached++;
+               if (ctx->flags & IORING_SETUP_CQE32)
+                       ctx->cqe_cached++;
                return cqe;
        }