rcu/kvfree: Add debug check for GP complete for kfree_rcu_cpu list

Under low-memory conditions, kvfree_rcu() will use each object's
rcu_head structure to queue objects in a singly linked list headed by
the kfree_rcu_cpu structure's ->head field.  This list is passed to
call_rcu() as a unit, but there is no indication of which grace period
this list needs to wait for.  This in turn prevents adding debug checks
in the kfree_rcu_work() as was done for the two page-of-pointers channels
in the kfree_rcu_cpu structure.

This commit therefore adds a ->head_free_gp_snap field to the
kfree_rcu_cpu_work structure to record this grace-period number.  It also
adds a WARN_ON_ONCE() to kfree_rcu_monitor() that checks to make sure
that the required grace period has in fact elapsed.

[ paulmck: Fix kerneldoc issue raised by Stephen Rothwell. ]

Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 91d75fd..7452ba9 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2773,6 +2773,7 @@ struct kvfree_rcu_bulk_data {
  * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
  * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
  * @head_free: List of kfree_rcu() objects waiting for a grace period
+ * @head_free_gp_snap: Grace-period snapshot to check for attempted premature frees.
  * @bulk_head_free: Bulk-List of kvfree_rcu() objects waiting for a grace period
  * @krcp: Pointer to @kfree_rcu_cpu structure
  */
@@ -2780,6 +2781,7 @@ struct kvfree_rcu_bulk_data {
 struct kfree_rcu_cpu_work {
        struct rcu_work rcu_work;
        struct rcu_head *head_free;
+       struct rcu_gp_oldstate head_free_gp_snap;
        struct list_head bulk_head_free[FREE_N_CHANNELS];
        struct kfree_rcu_cpu *krcp;
 };
@@ -2985,6 +2987,7 @@ static void kfree_rcu_work(struct work_struct *work)
        struct rcu_head *head;
        struct kfree_rcu_cpu *krcp;
        struct kfree_rcu_cpu_work *krwp;
+       struct rcu_gp_oldstate head_gp_snap;
        int i;
 
        krwp = container_of(to_rcu_work(work),
@@ -2999,6 +3002,7 @@ static void kfree_rcu_work(struct work_struct *work)
        // Channel 3.
        head = krwp->head_free;
        krwp->head_free = NULL;
+       head_gp_snap = krwp->head_free_gp_snap;
        raw_spin_unlock_irqrestore(&krcp->lock, flags);
 
        // Handle the first two channels.
@@ -3015,7 +3019,8 @@ static void kfree_rcu_work(struct work_struct *work)
         * queued on a linked list through their rcu_head structures.
         * This list is named "Channel 3".
         */
-       kvfree_rcu_list(head);
+       if (head && !WARN_ON_ONCE(!poll_state_synchronize_rcu_full(&head_gp_snap)))
+               kvfree_rcu_list(head);
 }
 
 static bool
@@ -3147,6 +3152,7 @@ static void kfree_rcu_monitor(struct work_struct *work)
                        // objects queued on the linked list.
                        if (!krwp->head_free) {
                                krwp->head_free = krcp->head;
+                               get_state_synchronize_rcu_full(&krwp->head_free_gp_snap);
                                atomic_set(&krcp->head_count, 0);
                                WRITE_ONCE(krcp->head, NULL);
                        }