io-wq: reorder cancellation pending -> running

[platform/kernel/linux-starfive.git] / fs / io-wq.c

diff --git a/fs/io-wq.c b/fs/io-wq.c
index 4023c98..03c7e37 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -10,7 +10,6 @@
 #include <linux/errno.h>
 #include <linux/sched/signal.h>
 #include <linux/mm.h>
-#include <linux/mmu_context.h>
 #include <linux/sched/mm.h>
 #include <linux/percpu.h>
 #include <linux/slab.h>
@@ -112,6 +111,7 @@ struct io_wq {
        unsigned long state;
 
        free_work_fn *free_work;
+       io_wq_work_fn *do_work;
 
        struct task_struct *manager;
        struct user_struct *user;
@@ -170,8 +170,7 @@ static bool __io_worker_unuse(struct io_wqe *wqe, struct io_worker *worker)
                        dropped_lock = true;
                }
                __set_current_state(TASK_RUNNING);
-               set_fs(KERNEL_DS);
-               unuse_mm(worker->mm);
+               kthread_unuse_mm(worker->mm);
                mmput(worker->mm);
                worker->mm = NULL;
        }
@@ -418,18 +417,15 @@ static struct io_wq_work *io_get_next_work(struct io_wqe *wqe)
 static void io_wq_switch_mm(struct io_worker *worker, struct io_wq_work *work)
 {
        if (worker->mm) {
-               unuse_mm(worker->mm);
+               kthread_unuse_mm(worker->mm);
                mmput(worker->mm);
                worker->mm = NULL;
        }
-       if (!work->mm) {
-               set_fs(KERNEL_DS);
+       if (!work->mm)
                return;
-       }
+
        if (mmget_not_zero(work->mm)) {
-               use_mm(work->mm);
-               if (!worker->mm)
-                       set_fs(USER_DS);
+               kthread_use_mm(work->mm);
                worker->mm = work->mm;
                /* hang on to this mm */
                work->mm = NULL;
@@ -528,7 +524,7 @@ get_next:
 
                        hash = io_get_work_hash(work);
                        linked = old_work = work;
-                       linked->func(&linked);
+                       wq->do_work(&linked);
                        linked = (old_work == linked) ? NULL : linked;
 
                        work = next_hashed;
@@ -785,7 +781,7 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe)
                struct io_wq_work *old_work = work;
 
                work->flags |= IO_WQ_WORK_CANCEL;
-               work->func(&work);
+               wq->do_work(&work);
                work = (work == old_work) ? NULL : work;
                wq->free_work(old_work);
        } while (work);
@@ -931,19 +927,14 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
        return ret;
 }
 
-static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe,
-                                           struct io_cb_cancel_data *match)
+static bool io_wqe_cancel_pending_work(struct io_wqe *wqe,
+                                      struct io_cb_cancel_data *match)
 {
        struct io_wq_work_node *node, *prev;
        struct io_wq_work *work;
        unsigned long flags;
        bool found = false;
 
-       /*
-        * First check pending list, if we're lucky we can just remove it
-        * from there. CANCEL_OK means that the work is returned as-new,
-        * no completion will be posted for it.
-        */
        spin_lock_irqsave(&wqe->lock, flags);
        wq_list_for_each(node, prev, &wqe->work_list) {
                work = container_of(node, struct io_wq_work, list);
@@ -956,21 +947,20 @@ static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe,
        }
        spin_unlock_irqrestore(&wqe->lock, flags);
 
-       if (found) {
+       if (found)
                io_run_cancel(work, wqe);
-               return IO_WQ_CANCEL_OK;
-       }
+       return found;
+}
+
+static bool io_wqe_cancel_running_work(struct io_wqe *wqe,
+                                      struct io_cb_cancel_data *match)
+{
+       bool found;
 
-       /*
-        * Now check if a free (going busy) or busy worker has the work
-        * currently running. If we find it there, we'll return CANCEL_RUNNING
-        * as an indication that we attempt to signal cancellation. The
-        * completion will run normally in this case.
-        */
        rcu_read_lock();
        found = io_wq_for_each_worker(wqe, io_wq_worker_cancel, match);
        rcu_read_unlock();
-       return found ? IO_WQ_CANCEL_RUNNING : IO_WQ_CANCEL_NOTFOUND;
+       return found;
 }
 
 enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
@@ -980,18 +970,34 @@ enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
                .fn     = cancel,
                .data   = data,
        };
-       enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;
        int node;
 
+       /*
+        * First check pending list, if we're lucky we can just remove it
+        * from there. CANCEL_OK means that the work is returned as-new,
+        * no completion will be posted for it.
+        */
        for_each_node(node) {
                struct io_wqe *wqe = wq->wqes[node];
 
-               ret = io_wqe_cancel_work(wqe, &match);
-               if (ret != IO_WQ_CANCEL_NOTFOUND)
-                       break;
+               if (io_wqe_cancel_pending_work(wqe, &match))
+                       return IO_WQ_CANCEL_OK;
        }
 
-       return ret;
+       /*
+        * Now check if a free (going busy) or busy worker has the work
+        * currently running. If we find it there, we'll return CANCEL_RUNNING
+        * as an indication that we attempt to signal cancellation. The
+        * completion will run normally in this case.
+        */
+       for_each_node(node) {
+               struct io_wqe *wqe = wq->wqes[node];
+
+               if (io_wqe_cancel_running_work(wqe, &match))
+                       return IO_WQ_CANCEL_RUNNING;
+       }
+
+       return IO_WQ_CANCEL_NOTFOUND;
 }
 
 static bool io_wq_io_cb_cancel_data(struct io_wq_work *work, void *data)
@@ -1023,7 +1029,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
        int ret = -ENOMEM, node;
        struct io_wq *wq;
 
-       if (WARN_ON_ONCE(!data->free_work))
+       if (WARN_ON_ONCE(!data->free_work || !data->do_work))
                return ERR_PTR(-EINVAL);
 
        wq = kzalloc(sizeof(*wq), GFP_KERNEL);
@@ -1037,6 +1043,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
        }
 
        wq->free_work = data->free_work;
+       wq->do_work = data->do_work;
 
        /* caller must already hold a reference to this */
        wq->user = data->user;
@@ -1093,7 +1100,7 @@ err:
 
 bool io_wq_get(struct io_wq *wq, struct io_wq_data *data)
 {
-       if (data->free_work != wq->free_work)
+       if (data->free_work != wq->free_work || data->do_work != wq->do_work)
                return false;
 
        return refcount_inc_not_zero(&wq->use_refs);