srcu: Convert ->srcu_lock_count and ->srcu_unlock_count to atomic

NMI-safe variants of srcu_read_lock() and srcu_read_unlock() are needed
by printk(), which on many architectures entails read-modify-write
atomic operations.  This commit prepares Tree SRCU for this change by
making both ->srcu_lock_count and ->srcu_unlock_count by atomic_long_t.

[ paulmck: Apply feedback from John Ogness. ]

Link: https://lore.kernel.org/all/20220910221947.171557773@linutronix.de/
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: John Ogness <john.ogness@linutronix.de>
Cc: Petr Mladek <pmladek@suse.com>

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index e3014319d1ade73060534db1865429416dea581f..0c4eca07d78d5c6a6ccd0b525fc1a065989bb442 100644 (file)
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -23,8 +23,8 @@ struct srcu_struct;
  */
 struct srcu_data {
        /* Read-side state. */
-       unsigned long srcu_lock_count[2];       /* Locks per CPU. */
-       unsigned long srcu_unlock_count[2];     /* Unlocks per CPU. */
+       atomic_long_t srcu_lock_count[2];       /* Locks per CPU. */
+       atomic_long_t srcu_unlock_count[2];     /* Unlocks per CPU. */
 
        /* Update-side state. */
        spinlock_t __private lock ____cacheline_internodealigned_in_smp;

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 1c304fec89c02df9121b8cfa60087a843bf5f49c..25e9458da6a26449bdd50915eab7ca2c9e88a766 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -417,7 +417,7 @@ static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx)
        for_each_possible_cpu(cpu) {
                struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);
 
-               sum += READ_ONCE(cpuc->srcu_lock_count[idx]);
+               sum += atomic_long_read(&cpuc->srcu_lock_count[idx]);
        }
        return sum;
 }
@@ -434,7 +434,7 @@ static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx)
        for_each_possible_cpu(cpu) {
                struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);
 
-               sum += READ_ONCE(cpuc->srcu_unlock_count[idx]);
+               sum += atomic_long_read(&cpuc->srcu_unlock_count[idx]);
        }
        return sum;
 }
@@ -503,10 +503,10 @@ static bool srcu_readers_active(struct srcu_struct *ssp)
        for_each_possible_cpu(cpu) {
                struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);
 
-               sum += READ_ONCE(cpuc->srcu_lock_count[0]);
-               sum += READ_ONCE(cpuc->srcu_lock_count[1]);
-               sum -= READ_ONCE(cpuc->srcu_unlock_count[0]);
-               sum -= READ_ONCE(cpuc->srcu_unlock_count[1]);
+               sum += atomic_long_read(&cpuc->srcu_lock_count[0]);
+               sum += atomic_long_read(&cpuc->srcu_lock_count[1]);
+               sum -= atomic_long_read(&cpuc->srcu_unlock_count[0]);
+               sum -= atomic_long_read(&cpuc->srcu_unlock_count[1]);
        }
        return sum;
 }
@@ -636,7 +636,7 @@ int __srcu_read_lock(struct srcu_struct *ssp)
        int idx;
 
        idx = READ_ONCE(ssp->srcu_idx) & 0x1;
-       this_cpu_inc(ssp->sda->srcu_lock_count[idx]);
+       this_cpu_inc(ssp->sda->srcu_lock_count[idx].counter);
        smp_mb(); /* B */  /* Avoid leaking the critical section. */
        return idx;
 }
@@ -650,7 +650,7 @@ EXPORT_SYMBOL_GPL(__srcu_read_lock);
 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)
 {
        smp_mb(); /* C */  /* Avoid leaking the critical section. */
-       this_cpu_inc(ssp->sda->srcu_unlock_count[idx]);
+       this_cpu_inc(ssp->sda->srcu_unlock_count[idx].counter);
 }
 EXPORT_SYMBOL_GPL(__srcu_read_unlock);
 
@@ -1687,8 +1687,8 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)
                        struct srcu_data *sdp;
 
                        sdp = per_cpu_ptr(ssp->sda, cpu);
-                       u0 = data_race(sdp->srcu_unlock_count[!idx]);
-                       u1 = data_race(sdp->srcu_unlock_count[idx]);
+                       u0 = data_race(atomic_long_read(&sdp->srcu_unlock_count[!idx]));
+                       u1 = data_race(atomic_long_read(&sdp->srcu_unlock_count[idx]));
 
                        /*
                         * Make sure that a lock is always counted if the corresponding
@@ -1696,8 +1696,8 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)
                         */
                        smp_rmb();
 
-                       l0 = data_race(sdp->srcu_lock_count[!idx]);
-                       l1 = data_race(sdp->srcu_lock_count[idx]);
+                       l0 = data_race(atomic_long_read(&sdp->srcu_lock_count[!idx]));
+                       l1 = data_race(atomic_long_read(&sdp->srcu_lock_count[idx]));
 
                        c0 = l0 - u0;
                        c1 = l1 - u1;