*
* F: wq->flush_mutex protected.
*
- * W: workqueue_lock protected.
+ * WQ: wq_mutex protected.
+ *
+ * WR: wq_mutex protected for writes. Sched-RCU protected for reads.
*
- * R: workqueue_lock protected for writes. Sched-RCU protected for reads.
+ * W: workqueue_lock protected.
*
* FR: wq->flush_mutex and workqueue_lock protected for writes. Sched-RCU
* protected for reads.
struct ida worker_ida; /* L: for worker IDs */
struct workqueue_attrs *attrs; /* I: worker attributes */
- struct hlist_node hash_node; /* W: unbound_pool_hash node */
- int refcnt; /* W: refcnt for unbound pools */
+ struct hlist_node hash_node; /* WQ: unbound_pool_hash node */
+ int refcnt; /* WQ: refcnt for unbound pools */
/*
* The current concurrency level. As it's likely to be accessed
* the appropriate worker_pool through its pool_workqueues.
*/
struct workqueue_struct {
- unsigned int flags; /* W: WQ_* flags */
+ unsigned int flags; /* WQ: WQ_* flags */
struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwq's */
struct list_head pwqs; /* FR: all pwqs of this wq */
- struct list_head list; /* W: list of all workqueues */
+ struct list_head list; /* WQ: list of all workqueues */
struct mutex flush_mutex; /* protects wq flushing */
int work_color; /* F: current work color */
struct list_head maydays; /* W: pwqs requesting rescue */
struct worker *rescuer; /* I: rescue worker */
- int nr_drainers; /* W: drain in progress */
+ int nr_drainers; /* WQ: drain in progress */
int saved_max_active; /* W: saved pwq max_active */
#ifdef CONFIG_SYSFS
static struct kmem_cache *pwq_cache;
-/* Serializes the accesses to the list of workqueues. */
+static DEFINE_MUTEX(wq_mutex); /* protects workqueues and pools */
static DEFINE_SPINLOCK(workqueue_lock);
-static LIST_HEAD(workqueues);
-static bool workqueue_freezing; /* W: have wqs started freezing? */
+
+static LIST_HEAD(workqueues); /* WQ: list of all workqueues */
+static bool workqueue_freezing; /* WQ: have wqs started freezing? */
/* the per-cpu worker pools */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
cpu_worker_pools);
-/*
- * R: idr of all pools. Modifications are protected by workqueue_lock.
- * Read accesses are protected by sched-RCU protected.
- */
-static DEFINE_IDR(worker_pool_idr);
+static DEFINE_IDR(worker_pool_idr); /* WR: idr of all pools */
-/* W: hash of all unbound pools keyed by pool->attrs */
+/* WQ: hash of all unbound pools keyed by pool->attrs */
static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
/* I: attributes used when instantiating standard unbound pools on demand */
#define CREATE_TRACE_POINTS
#include <trace/events/workqueue.h>
+#define assert_rcu_or_wq_mutex() \
+ rcu_lockdep_assert(rcu_read_lock_sched_held() || \
+ lockdep_is_held(&wq_mutex), \
+ "sched RCU or wq_mutex should be held")
+
#define assert_rcu_or_wq_lock() \
rcu_lockdep_assert(rcu_read_lock_sched_held() || \
lockdep_is_held(&workqueue_lock), \
* @pool: iteration cursor
* @pi: integer used for iteration
*
- * This must be called either with workqueue_lock held or sched RCU read
- * locked. If the pool needs to be used beyond the locking in effect, the
- * caller is responsible for guaranteeing that the pool stays online.
+ * This must be called either with wq_mutex held or sched RCU read locked.
+ * If the pool needs to be used beyond the locking in effect, the caller is
+ * responsible for guaranteeing that the pool stays online.
*
* The if/else clause exists only for the lockdep assertion and can be
* ignored.
*/
#define for_each_pool(pool, pi) \
idr_for_each_entry(&worker_pool_idr, pool, pi) \
- if (({ assert_rcu_or_wq_lock(); false; })) { } \
+ if (({ assert_rcu_or_wq_mutex(); false; })) { } \
else
/**
{
int ret;
+ lockdep_assert_held(&wq_mutex);
+
do {
if (!idr_pre_get(&worker_pool_idr, GFP_KERNEL))
return -ENOMEM;
-
- spin_lock_irq(&workqueue_lock);
ret = idr_get_new(&worker_pool_idr, pool, &pool->id);
- spin_unlock_irq(&workqueue_lock);
} while (ret == -EAGAIN);
return ret;
*
* Return the worker_pool @work was last associated with. %NULL if none.
*
- * Pools are created and destroyed under workqueue_lock, and allows read
- * access under sched-RCU read lock. As such, this function should be
- * called under workqueue_lock or with preemption disabled.
+ * Pools are created and destroyed under wq_mutex, and allows read access
+ * under sched-RCU read lock. As such, this function should be called
+ * under wq_mutex or with preemption disabled.
*
* All fields of the returned pool are accessible as long as the above
* mentioned locking is in effect. If the returned pool needs to be used
unsigned long data = atomic_long_read(&work->data);
int pool_id;
- assert_rcu_or_wq_lock();
+ assert_rcu_or_wq_mutex();
if (data & WORK_STRUCT_PWQ)
return ((struct pool_workqueue *)
* hotter than drain_workqueue() and already looks at @wq->flags.
* Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
*/
- spin_lock_irq(&workqueue_lock);
+ mutex_lock(&wq_mutex);
if (!wq->nr_drainers++)
wq->flags |= __WQ_DRAINING;
- spin_unlock_irq(&workqueue_lock);
+ mutex_unlock(&wq_mutex);
reflush:
flush_workqueue(wq);
goto reflush;
}
- spin_lock(&workqueue_lock);
+ local_irq_enable();
+
+ mutex_lock(&wq_mutex);
if (!--wq->nr_drainers)
wq->flags &= ~__WQ_DRAINING;
- spin_unlock(&workqueue_lock);
-
- local_irq_enable();
+ mutex_unlock(&wq_mutex);
}
EXPORT_SYMBOL_GPL(drain_workqueue);
{
struct worker *worker;
- spin_lock_irq(&workqueue_lock);
+ mutex_lock(&wq_mutex);
if (--pool->refcnt) {
- spin_unlock_irq(&workqueue_lock);
+ mutex_unlock(&wq_mutex);
return;
}
/* sanity checks */
if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) ||
WARN_ON(!list_empty(&pool->worklist))) {
- spin_unlock_irq(&workqueue_lock);
+ mutex_unlock(&wq_mutex);
return;
}
idr_remove(&worker_pool_idr, pool->id);
hash_del(&pool->hash_node);
- spin_unlock_irq(&workqueue_lock);
+ mutex_unlock(&wq_mutex);
/*
* Become the manager and destroy all workers. Grabbing
*/
static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
{
- static DEFINE_MUTEX(create_mutex);
u32 hash = wqattrs_hash(attrs);
struct worker_pool *pool;
- mutex_lock(&create_mutex);
+ mutex_lock(&wq_mutex);
/* do we already have a matching pool? */
- spin_lock_irq(&workqueue_lock);
hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
if (wqattrs_equal(pool->attrs, attrs)) {
pool->refcnt++;
goto out_unlock;
}
}
- spin_unlock_irq(&workqueue_lock);
/* nope, create a new one */
pool = kzalloc(sizeof(*pool), GFP_KERNEL);
goto fail;
/* install */
- spin_lock_irq(&workqueue_lock);
hash_add(unbound_pool_hash, &pool->hash_node, hash);
out_unlock:
- spin_unlock_irq(&workqueue_lock);
- mutex_unlock(&create_mutex);
+ mutex_unlock(&wq_mutex);
return pool;
fail:
- mutex_unlock(&create_mutex);
+ mutex_unlock(&wq_mutex);
if (pool)
put_unbound_pool(pool);
return NULL;
goto err_destroy;
/*
- * workqueue_lock protects global freeze state and workqueues list.
- * Grab it, adjust max_active and add the new workqueue to
- * workqueues list.
+ * wq_mutex protects global freeze state and workqueues list. Grab
+ * it, adjust max_active and add the new @wq to workqueues list.
*/
- spin_lock_irq(&workqueue_lock);
+ mutex_lock(&wq_mutex);
+ spin_lock_irq(&workqueue_lock);
for_each_pwq(pwq, wq)
pwq_adjust_max_active(pwq);
+ spin_unlock_irq(&workqueue_lock);
list_add(&wq->list, &workqueues);
- spin_unlock_irq(&workqueue_lock);
+ mutex_unlock(&wq_mutex);
return wq;
/* drain it before proceeding with destruction */
drain_workqueue(wq);
- spin_lock_irq(&workqueue_lock);
-
/* sanity checks */
+ spin_lock_irq(&workqueue_lock);
for_each_pwq(pwq, wq) {
int i;
return;
}
}
+ spin_unlock_irq(&workqueue_lock);
/*
* wq list is used to freeze wq, remove from list after
* flushing is complete in case freeze races us.
*/
+ mutex_lock(&wq_mutex);
list_del_init(&wq->list);
-
- spin_unlock_irq(&workqueue_lock);
+ mutex_unlock(&wq_mutex);
workqueue_sysfs_unregister(wq);
* pool->worklist.
*
* CONTEXT:
- * Grabs and releases workqueue_lock and pool->lock's.
+ * Grabs and releases wq_mutex, workqueue_lock and pool->lock's.
*/
void freeze_workqueues_begin(void)
{
struct pool_workqueue *pwq;
int pi;
- spin_lock_irq(&workqueue_lock);
+ mutex_lock(&wq_mutex);
WARN_ON_ONCE(workqueue_freezing);
workqueue_freezing = true;
/* set FREEZING */
for_each_pool(pool, pi) {
- spin_lock(&pool->lock);
+ spin_lock_irq(&pool->lock);
WARN_ON_ONCE(pool->flags & POOL_FREEZING);
pool->flags |= POOL_FREEZING;
- spin_unlock(&pool->lock);
+ spin_unlock_irq(&pool->lock);
}
/* suppress further executions by setting max_active to zero */
+ spin_lock_irq(&workqueue_lock);
list_for_each_entry(wq, &workqueues, list) {
for_each_pwq(pwq, wq)
pwq_adjust_max_active(pwq);
}
-
spin_unlock_irq(&workqueue_lock);
+
+ mutex_unlock(&wq_mutex);
}
/**
* between freeze_workqueues_begin() and thaw_workqueues().
*
* CONTEXT:
- * Grabs and releases workqueue_lock.
+ * Grabs and releases wq_mutex.
*
* RETURNS:
* %true if some freezable workqueues are still busy. %false if freezing
struct workqueue_struct *wq;
struct pool_workqueue *pwq;
- spin_lock_irq(&workqueue_lock);
+ mutex_lock(&wq_mutex);
WARN_ON_ONCE(!workqueue_freezing);
* nr_active is monotonically decreasing. It's safe
* to peek without lock.
*/
+ preempt_disable();
for_each_pwq(pwq, wq) {
WARN_ON_ONCE(pwq->nr_active < 0);
if (pwq->nr_active) {
busy = true;
+ preempt_enable();
goto out_unlock;
}
}
+ preempt_enable();
}
out_unlock:
- spin_unlock_irq(&workqueue_lock);
+ mutex_unlock(&wq_mutex);
return busy;
}
* frozen works are transferred to their respective pool worklists.
*
* CONTEXT:
- * Grabs and releases workqueue_lock and pool->lock's.
+ * Grabs and releases wq_mutex, workqueue_lock and pool->lock's.
*/
void thaw_workqueues(void)
{
struct worker_pool *pool;
int pi;
- spin_lock_irq(&workqueue_lock);
+ mutex_lock(&wq_mutex);
if (!workqueue_freezing)
goto out_unlock;
/* clear FREEZING */
for_each_pool(pool, pi) {
- spin_lock(&pool->lock);
+ spin_lock_irq(&pool->lock);
WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
pool->flags &= ~POOL_FREEZING;
- spin_unlock(&pool->lock);
+ spin_unlock_irq(&pool->lock);
}
/* restore max_active and repopulate worklist */
+ spin_lock_irq(&workqueue_lock);
list_for_each_entry(wq, &workqueues, list) {
for_each_pwq(pwq, wq)
pwq_adjust_max_active(pwq);
}
+ spin_unlock_irq(&workqueue_lock);
/* kick workers */
for_each_pool(pool, pi) {
- spin_lock(&pool->lock);
+ spin_lock_irq(&pool->lock);
wake_up_worker(pool);
- spin_unlock(&pool->lock);
+ spin_unlock_irq(&pool->lock);
}
workqueue_freezing = false;
out_unlock:
- spin_unlock_irq(&workqueue_lock);
+ mutex_unlock(&wq_mutex);
}
#endif /* CONFIG_FREEZER */
pool->attrs->nice = std_nice[i++];
/* alloc pool ID */
+ mutex_lock(&wq_mutex);
BUG_ON(worker_pool_assign_id(pool));
+ mutex_unlock(&wq_mutex);
}
}
projects / kernel / kernel-generic.git / commitdiff