io_uring: fail links more in io_submit_sqe()

Instead of marking a link with REQ_F_FAIL_LINK on an error and delaying
its failing to the caller, do it eagerly right when after getting an
error in io_submit_sqe(). This renders FAIL_LINK checks in
io_queue_link_head() useless and we can skip it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 45f78fd..2fdfe5f 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -6645,15 +6645,6 @@ fail_req:
        }
 }
 
-static inline void io_queue_link_head(struct io_kiocb *req)
-{
-       if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
-               io_put_req(req);
-               io_req_complete(req, -ECANCELED);
-       } else
-               io_queue_sqe(req);
-}
-
 /*
  * Check SQE restrictions (opcode and flags).
  *
@@ -6768,9 +6759,13 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 fail_req:
                io_put_req(req);
                io_req_complete(req, ret);
-               /* fail even hard links since we don't submit */
-               if (link->head)
+               if (link->head) {
+                       /* fail even hard links since we don't submit */
                        link->head->flags |= REQ_F_FAIL_LINK;
+                       io_put_req(link->head);
+                       io_req_complete(link->head, -ECANCELED);
+                       link->head = NULL;
+               }
                return ret;
        }
        ret = io_req_prep(req, sqe);
@@ -6811,7 +6806,7 @@ fail_req:
 
                /* last request of a link, enqueue the link */
                if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
-                       io_queue_link_head(head);
+                       io_queue_sqe(head);
                        link->head = NULL;
                }
        } else {
@@ -6837,7 +6832,7 @@ static void io_submit_state_end(struct io_submit_state *state,
                                struct io_ring_ctx *ctx)
 {
        if (state->link.head)
-               io_queue_link_head(state->link.head);
+               io_queue_sqe(state->link.head);
        if (state->comp.nr)
                io_submit_flush_completions(&state->comp, ctx);
        if (state->plug_started)