io_uring: fail cancellation for EXITING tasks

[platform/kernel/linux-rpi.git] / fs / io_uring.c

diff --git a/fs/io_uring.c b/fs/io_uring.c
index e68d278..edd30ba 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -456,6 +456,8 @@ struct io_ring_ctx {
                struct work_struct              exit_work;
                struct list_head                tctx_list;
                struct completion               ref_comp;
+               u32                             iowq_limits[2];
+               bool                            iowq_limits_set;
        };
 };
 
@@ -1368,11 +1370,6 @@ static void io_req_track_inflight(struct io_kiocb *req)
        }
 }
 
-static inline void io_unprep_linked_timeout(struct io_kiocb *req)
-{
-       req->flags &= ~REQ_F_LINK_TIMEOUT;
-}
-
 static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)
 {
        if (WARN_ON_ONCE(!req->link))
@@ -1433,10 +1430,10 @@ static void io_prep_async_link(struct io_kiocb *req)
        if (req->flags & REQ_F_LINK_TIMEOUT) {
                struct io_ring_ctx *ctx = req->ctx;
 
-               spin_lock(&ctx->completion_lock);
+               spin_lock_irq(&ctx->timeout_lock);
                io_for_each_link(cur, req)
                        io_prep_async_work(cur);
-               spin_unlock(&ctx->completion_lock);
+               spin_unlock_irq(&ctx->timeout_lock);
        } else {
                io_for_each_link(cur, req)
                        io_prep_async_work(cur);
@@ -5700,6 +5697,7 @@ static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
        int posted = 0, i;
 
        spin_lock(&ctx->completion_lock);
+       spin_lock_irq(&ctx->timeout_lock);
        for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
                struct hlist_head *list;
 
@@ -5709,6 +5707,7 @@ static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
                                posted += io_poll_remove_one(req);
                }
        }
+       spin_unlock_irq(&ctx->timeout_lock);
        spin_unlock(&ctx->completion_lock);
 
        if (posted)
@@ -6887,10 +6886,11 @@ static inline struct file *io_file_get(struct io_ring_ctx *ctx,
 static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)
 {
        struct io_kiocb *prev = req->timeout.prev;
-       int ret;
+       int ret = -ENOENT;
 
        if (prev) {
-               ret = io_try_cancel_userdata(req, prev->user_data);
+               if (!(req->task->flags & PF_EXITING))
+                       ret = io_try_cancel_userdata(req, prev->user_data);
                io_req_complete_post(req, ret ?: -ETIME, 0);
                io_put_req(prev);
        } else {
@@ -6983,7 +6983,7 @@ issue_sqe:
                switch (io_arm_poll_handler(req)) {
                case IO_APOLL_READY:
                        if (linked_timeout)
-                               io_unprep_linked_timeout(req);
+                               io_queue_linked_timeout(linked_timeout);
                        goto issue_sqe;
                case IO_APOLL_ABORTED:
                        /*
@@ -9526,9 +9526,9 @@ static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
                struct io_ring_ctx *ctx = req->ctx;
 
                /* protect against races with linked timeouts */
-               spin_lock(&ctx->completion_lock);
+               spin_lock_irq(&ctx->timeout_lock);
                ret = io_match_task(req, cancel->task, cancel->all);
-               spin_unlock(&ctx->completion_lock);
+               spin_unlock_irq(&ctx->timeout_lock);
        } else {
                ret = io_match_task(req, cancel->task, cancel->all);
        }
@@ -9542,12 +9542,14 @@ static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
        LIST_HEAD(list);
 
        spin_lock(&ctx->completion_lock);
+       spin_lock_irq(&ctx->timeout_lock);
        list_for_each_entry_reverse(de, &ctx->defer_list, list) {
                if (io_match_task(de->req, task, cancel_all)) {
                        list_cut_position(&list, &ctx->defer_list, &de->list);
                        break;
                }
        }
+       spin_unlock_irq(&ctx->timeout_lock);
        spin_unlock(&ctx->completion_lock);
        if (list_empty(&list))
                return false;
@@ -9638,7 +9640,16 @@ static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
                ret = io_uring_alloc_task_context(current, ctx);
                if (unlikely(ret))
                        return ret;
+
                tctx = current->io_uring;
+               if (ctx->iowq_limits_set) {
+                       unsigned int limits[2] = { ctx->iowq_limits[0],
+                                                  ctx->iowq_limits[1], };
+
+                       ret = io_wq_max_workers(tctx->io_wq, limits);
+                       if (ret)
+                               return ret;
+               }
        }
        if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
                node = kmalloc(sizeof(*node), GFP_KERNEL);
@@ -10643,7 +10654,9 @@ static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
 
 static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
                                        void __user *arg)
+       __must_hold(&ctx->uring_lock)
 {
+       struct io_tctx_node *node;
        struct io_uring_task *tctx = NULL;
        struct io_sq_data *sqd = NULL;
        __u32 new_count[2];
@@ -10674,13 +10687,21 @@ static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
                tctx = current->io_uring;
        }
 
-       ret = -EINVAL;
-       if (!tctx || !tctx->io_wq)
-               goto err;
+       BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));
 
-       ret = io_wq_max_workers(tctx->io_wq, new_count);
-       if (ret)
-               goto err;
+       for (i = 0; i < ARRAY_SIZE(new_count); i++)
+               if (new_count[i])
+                       ctx->iowq_limits[i] = new_count[i];
+       ctx->iowq_limits_set = true;
+
+       ret = -EINVAL;
+       if (tctx && tctx->io_wq) {
+               ret = io_wq_max_workers(tctx->io_wq, new_count);
+               if (ret)
+                       goto err;
+       } else {
+               memset(new_count, 0, sizeof(new_count));
+       }
 
        if (sqd) {
                mutex_unlock(&sqd->lock);
@@ -10690,6 +10711,22 @@ static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
        if (copy_to_user(arg, new_count, sizeof(new_count)))
                return -EFAULT;
 
+       /* that's it for SQPOLL, only the SQPOLL task creates requests */
+       if (sqd)
+               return 0;
+
+       /* now propagate the restriction to all registered users */
+       list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
+               struct io_uring_task *tctx = node->task->io_uring;
+
+               if (WARN_ON_ONCE(!tctx->io_wq))
+                       continue;
+
+               for (i = 0; i < ARRAY_SIZE(new_count); i++)
+                       new_count[i] = ctx->iowq_limits[i];
+               /* ignore errors, it always returns zero anyway */
+               (void)io_wq_max_workers(tctx->io_wq, new_count);
+       }
        return 0;
 err:
        if (sqd) {