io_uring: move provided and fixed buffers into the same io_kiocb area

These are mutually exclusive - if you use provided buffers, then you
cannot use fixed buffers and vice versa. Move them into the same spot
in the io_kiocb, which is also advantageous for provided buffers as
they get near the submit side hot cacheline.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 3d5a4e64d6fe8fd7ae4d4732c872809746fadc6d..a260a0fd7b177690f5cc2a438544e927c159687d 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -958,8 +958,14 @@ struct io_kiocb {
        struct task_struct              *task;
 
        struct io_rsrc_node             *rsrc_node;
-       /* store used ubuf, so we can prevent reloading */
-       struct io_mapped_ubuf           *imu;
+
+       union {
+               /* store used ubuf, so we can prevent reloading */
+               struct io_mapped_ubuf   *imu;
+
+               /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
+               struct io_buffer        *kbuf;
+       };
 
        union {
                /* used by request caches, completion batching and iopoll */
@@ -976,8 +982,6 @@ struct io_kiocb {
        struct async_poll               *apoll;
        /* opcode allocated if it needs to store data for async defer */
        void                            *async_data;
-       /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
-       struct io_buffer                *kbuf;
        /* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */
        struct io_kiocb                 *link;
        /* custom credentials, valid IFF REQ_F_CREDS is set */