io_uring: simplify big_cqe handling

Don't keep big_cqe bits of req in a union with hash_node, find a
separate space for it. It's bit safer, but also if we keep it always
initialised, we can get rid of ugly REQ_F_CQE32_INIT handling.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/447aa1b2968978c99e655ba88db536e903df0fe9.1692916914.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index f04ce51..9795eda 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -409,7 +409,6 @@ enum {
        REQ_F_SINGLE_POLL_BIT,
        REQ_F_DOUBLE_POLL_BIT,
        REQ_F_PARTIAL_IO_BIT,
-       REQ_F_CQE32_INIT_BIT,
        REQ_F_APOLL_MULTISHOT_BIT,
        REQ_F_CLEAR_POLLIN_BIT,
        REQ_F_HASH_LOCKED_BIT,
@@ -479,8 +478,6 @@ enum {
        REQ_F_PARTIAL_IO        = BIT(REQ_F_PARTIAL_IO_BIT),
        /* fast poll multishot mode */
        REQ_F_APOLL_MULTISHOT   = BIT(REQ_F_APOLL_MULTISHOT_BIT),
-       /* ->extra1 and ->extra2 are initialised */
-       REQ_F_CQE32_INIT        = BIT(REQ_F_CQE32_INIT_BIT),
        /* recvmsg special flag, clear EPOLLIN */
        REQ_F_CLEAR_POLLIN      = BIT(REQ_F_CLEAR_POLLIN_BIT),
        /* hashed into ->cancel_hash_locked, protected by ->uring_lock */
@@ -579,13 +576,7 @@ struct io_kiocb {
        struct io_task_work             io_task_work;
        unsigned                        nr_tw;
        /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
-       union {
-               struct hlist_node       hash_node;
-               struct {
-                       u64             extra1;
-                       u64             extra2;
-               };
-       };
+       struct hlist_node               hash_node;
        /* internal polling, see IORING_FEAT_FAST_POLL */
        struct async_poll               *apoll;
        /* opcode allocated if it needs to store data for async defer */
@@ -595,6 +586,11 @@ struct io_kiocb {
        /* custom credentials, valid IFF REQ_F_CREDS is set */
        const struct cred               *creds;
        struct io_wq_work               work;
+
+       struct {
+               u64                     extra1;
+               u64                     extra2;
+       } big_cqe;
 };
 
 struct io_overflow_cqe {

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 3e0fe1e..0aeb332 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -807,13 +807,10 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
 
 void io_req_cqe_overflow(struct io_kiocb *req)
 {
-       if (!(req->flags & REQ_F_CQE32_INIT)) {
-               req->extra1 = 0;
-               req->extra2 = 0;
-       }
        io_cqring_event_overflow(req->ctx, req->cqe.user_data,
                                req->cqe.res, req->cqe.flags,
-                               req->extra1, req->extra2);
+                               req->big_cqe.extra1, req->big_cqe.extra2);
+       memset(&req->big_cqe, 0, sizeof(req->big_cqe));
 }
 
 /*
@@ -1057,6 +1054,7 @@ static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
        req->async_data = NULL;
        /* not necessary, but safer to zero */
        memset(&req->cqe, 0, sizeof(req->cqe));
+       memset(&req->big_cqe, 0, sizeof(req->big_cqe));
 }
 
 static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 4655982..9b5dfb6 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -148,21 +148,12 @@ static inline bool io_fill_cqe_req(struct io_ring_ctx *ctx, struct io_kiocb *req
        if (trace_io_uring_complete_enabled())
                trace_io_uring_complete(req->ctx, req, req->cqe.user_data,
                                        req->cqe.res, req->cqe.flags,
-                                       (req->flags & REQ_F_CQE32_INIT) ? req->extra1 : 0,
-                                       (req->flags & REQ_F_CQE32_INIT) ? req->extra2 : 0);
+                                       req->big_cqe.extra1, req->big_cqe.extra2);
 
        memcpy(cqe, &req->cqe, sizeof(*cqe));
-
        if (ctx->flags & IORING_SETUP_CQE32) {
-               u64 extra1 = 0, extra2 = 0;
-
-               if (req->flags & REQ_F_CQE32_INIT) {
-                       extra1 = req->extra1;
-                       extra2 = req->extra2;
-               }
-
-               WRITE_ONCE(cqe->big_cqe[0], extra1);
-               WRITE_ONCE(cqe->big_cqe[1], extra2);
+               memcpy(cqe->big_cqe, &req->big_cqe, sizeof(*cqe));
+               memset(&req->big_cqe, 0, sizeof(req->big_cqe));
        }
        return true;
 }

diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 8e7a03c..537795f 100644 (file)
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -43,9 +43,8 @@ EXPORT_SYMBOL_GPL(io_uring_cmd_do_in_task_lazy);
 static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
                                          u64 extra1, u64 extra2)
 {
-       req->extra1 = extra1;
-       req->extra2 = extra2;
-       req->flags |= REQ_F_CQE32_INIT;
+       req->big_cqe.extra1 = extra1;
+       req->big_cqe.extra2 = extra2;
 }
 
 /*