rcu/nocb: Allow lockless use of rcu_segcblist_restempty()

Currently, rcu_segcblist_restempty() assumes that the callback list
is not being changed by other CPUs, but upcoming changes will require
it to operate locklessly.  This commit therefore adds the needed
READ_ONCE() calls, along with the WRITE_ONCE() calls when updating
the callback list.

Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>

diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c
index 700779f..0e7fe67 100644 (file)
--- a/kernel/rcu/rcu_segcblist.c
+++ b/kernel/rcu/rcu_segcblist.c
@@ -147,8 +147,8 @@ void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
                rsclp->len_lazy++;
        smp_mb(); /* Ensure counts are updated before callback is enqueued. */
        rhp->next = NULL;
-       *rsclp->tails[RCU_NEXT_TAIL] = rhp;
-       rsclp->tails[RCU_NEXT_TAIL] = &rhp->next;
+       WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp);
+       WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next);
 }
 
 /*
@@ -176,9 +176,9 @@ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
        for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--)
                if (rsclp->tails[i] != rsclp->tails[i - 1])
                        break;
-       *rsclp->tails[i] = rhp;
+       WRITE_ONCE(*rsclp->tails[i], rhp);
        for (; i <= RCU_NEXT_TAIL; i++)
-               rsclp->tails[i] = &rhp->next;
+               WRITE_ONCE(rsclp->tails[i], &rhp->next);
        return true;
 }
 
@@ -214,11 +214,11 @@ void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
                return; /* Nothing to do. */
        *rclp->tail = rsclp->head;
        rsclp->head = *rsclp->tails[RCU_DONE_TAIL];
-       *rsclp->tails[RCU_DONE_TAIL] = NULL;
+       WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
        rclp->tail = rsclp->tails[RCU_DONE_TAIL];
        for (i = RCU_CBLIST_NSEGS - 1; i >= RCU_DONE_TAIL; i--)
                if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL])
-                       rsclp->tails[i] = &rsclp->head;
+                       WRITE_ONCE(rsclp->tails[i], &rsclp->head);
 }
 
 /*
@@ -237,9 +237,9 @@ void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
                return; /* Nothing to do. */
        *rclp->tail = *rsclp->tails[RCU_DONE_TAIL];
        rclp->tail = rsclp->tails[RCU_NEXT_TAIL];
-       *rsclp->tails[RCU_DONE_TAIL] = NULL;
+       WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
        for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++)
-               rsclp->tails[i] = rsclp->tails[RCU_DONE_TAIL];
+               WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]);
 }
 
 /*
@@ -271,7 +271,7 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
        rsclp->head = rclp->head;
        for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++)
                if (&rsclp->head == rsclp->tails[i])
-                       rsclp->tails[i] = rclp->tail;
+                       WRITE_ONCE(rsclp->tails[i], rclp->tail);
                else
                        break;
        rclp->head = NULL;
@@ -287,8 +287,8 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
 {
        if (!rclp->head)
                return; /* Nothing to do. */
-       *rsclp->tails[RCU_NEXT_TAIL] = rclp->head;
-       rsclp->tails[RCU_NEXT_TAIL] = rclp->tail;
+       WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);
+       WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);
        rclp->head = NULL;
        rclp->tail = &rclp->head;
 }
@@ -312,7 +312,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)
        for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) {
                if (ULONG_CMP_LT(seq, rsclp->gp_seq[i]))
                        break;
-               rsclp->tails[RCU_DONE_TAIL] = rsclp->tails[i];
+               WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]);
        }
 
        /* If no callbacks moved, nothing more need be done. */
@@ -321,7 +321,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)
 
        /* Clean up tail pointers that might have been misordered above. */
        for (j = RCU_WAIT_TAIL; j < i; j++)
-               rsclp->tails[j] = rsclp->tails[RCU_DONE_TAIL];
+               WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]);
 
        /*
         * Callbacks moved, so clean up the misordered ->tails[] pointers
@@ -332,7 +332,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)
        for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) {
                if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL])
                        break;  /* No more callbacks. */
-               rsclp->tails[j] = rsclp->tails[i];
+               WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]);
                rsclp->gp_seq[j] = rsclp->gp_seq[i];
        }
 }
@@ -397,7 +397,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq)
         * structure other than in the RCU_NEXT_TAIL segment.
         */
        for (; i < RCU_NEXT_TAIL; i++) {
-               rsclp->tails[i] = rsclp->tails[RCU_NEXT_TAIL];
+               WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]);
                rsclp->gp_seq[i] = seq;
        }
        return true;

diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h
index 8f37833..f74960f 100644 (file)
--- a/kernel/rcu/rcu_segcblist.h
+++ b/kernel/rcu/rcu_segcblist.h
@@ -79,7 +79,7 @@ static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
  */
 static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
 {
-       return !*rsclp->tails[seg];
+       return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
 }
 
 void rcu_segcblist_init(struct rcu_segcblist *rsclp);