io_uring: avoid unnecessary io_wq_work copy for fast poll feature

Basically IORING_OP_POLL_ADD command and async armed poll handlers
for regular commands don't touch io_wq_work, so only REQ_F_WORK_INITIALIZED
is set, can we do io_wq_work copy and restore.

Signed-off-by: Xiaoguang Wang <xiaoguang.wang@linux.alibaba.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 5a2c004..d72b2a9 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -4262,7 +4262,8 @@ static void io_async_task_func(struct callback_head *cb)
        spin_unlock_irq(&ctx->completion_lock);
 
        /* restore ->work in case we need to retry again */
-       memcpy(&req->work, &apoll->work, sizeof(req->work));
+       if (req->flags & REQ_F_WORK_INITIALIZED)
+               memcpy(&req->work, &apoll->work, sizeof(req->work));
        kfree(apoll);
 
        if (!canceled) {
@@ -4359,7 +4360,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
                return false;
 
        req->flags |= REQ_F_POLLED;
-       memcpy(&apoll->work, &req->work, sizeof(req->work));
+       if (req->flags & REQ_F_WORK_INITIALIZED)
+               memcpy(&apoll->work, &req->work, sizeof(req->work));
        had_io = req->io != NULL;
 
        get_task_struct(current);
@@ -4384,7 +4386,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
                if (!had_io)
                        io_poll_remove_double(req);
                spin_unlock_irq(&ctx->completion_lock);
-               memcpy(&req->work, &apoll->work, sizeof(req->work));
+               if (req->flags & REQ_F_WORK_INITIALIZED)
+                       memcpy(&req->work, &apoll->work, sizeof(req->work));
                kfree(apoll);
                return false;
        }
@@ -4429,7 +4432,9 @@ static bool io_poll_remove_one(struct io_kiocb *req)
                         * io_req_work_drop_env below when dropping the
                         * final reference.
                         */
-                       memcpy(&req->work, &apoll->work, sizeof(req->work));
+                       if (req->flags & REQ_F_WORK_INITIALIZED)
+                               memcpy(&req->work, &apoll->work,
+                                      sizeof(req->work));
                        kfree(apoll);
                }
        }