rcu/tree: Make debug_objects logic independent of rcu_head

kfree_rcu()'s debug_objects logic uses the address of the object's
embedded rcu_head to queue/unqueue. Instead of this, make use of the
object's address itself as preparation for future headless kfree_rcu()
support.

Reviewed-by: Uladzislau Rezki <urezki@gmail.com>
Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
Signed-off-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 5151fe4e142984e8514a5e81ae160870f8523d81..143c1e9265b645b8811d6420d813c79e92e11ae1 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2970,13 +2970,11 @@ EXPORT_SYMBOL_GPL(call_rcu);
  * @nr_records: Number of active pointers in the array
  * @records: Array of the kfree_rcu() pointers
  * @next: Next bulk object in the block chain
- * @head_free_debug: For debug, when CONFIG_DEBUG_OBJECTS_RCU_HEAD is set
  */
 struct kfree_rcu_bulk_data {
        unsigned long nr_records;
        void *records[KFREE_BULK_MAX_ENTR];
        struct kfree_rcu_bulk_data *next;
-       struct rcu_head *head_free_debug;
 };
 
 /**
@@ -3026,11 +3024,13 @@ struct kfree_rcu_cpu {
 static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc);
 
 static __always_inline void
-debug_rcu_head_unqueue_bulk(struct rcu_head *head)
+debug_rcu_bhead_unqueue(struct kfree_rcu_bulk_data *bhead)
 {
 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
-       for (; head; head = head->next)
-               debug_rcu_head_unqueue(head);
+       int i;
+
+       for (i = 0; i < bhead->nr_records; i++)
+               debug_rcu_head_unqueue((struct rcu_head *)(bhead->records[i]));
 #endif
 }
 
@@ -3060,7 +3060,7 @@ static void kfree_rcu_work(struct work_struct *work)
        for (; bhead; bhead = bnext) {
                bnext = bhead->next;
 
-               debug_rcu_head_unqueue_bulk(bhead->head_free_debug);
+               debug_rcu_bhead_unqueue(bhead);
 
                rcu_lock_acquire(&rcu_callback_map);
                trace_rcu_invoke_kfree_bulk_callback(rcu_state.name,
@@ -3082,14 +3082,15 @@ static void kfree_rcu_work(struct work_struct *work)
         */
        for (; head; head = next) {
                unsigned long offset = (unsigned long)head->func;
+               void *ptr = (void *)head - offset;
 
                next = head->next;
-               debug_rcu_head_unqueue(head);
+               debug_rcu_head_unqueue((struct rcu_head *)ptr);
                rcu_lock_acquire(&rcu_callback_map);
                trace_rcu_invoke_kfree_callback(rcu_state.name, head, offset);
 
                if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset)))
-                       kfree((void *)head - offset);
+                       kfree(ptr);
 
                rcu_lock_release(&rcu_callback_map);
                cond_resched_tasks_rcu_qs();
@@ -3228,18 +3229,11 @@ kfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp,
                /* Initialize the new block. */
                bnode->nr_records = 0;
                bnode->next = krcp->bhead;
-               bnode->head_free_debug = NULL;
 
                /* Attach it to the head. */
                krcp->bhead = bnode;
        }
 
-#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
-       head->func = func;
-       head->next = krcp->bhead->head_free_debug;
-       krcp->bhead->head_free_debug = head;
-#endif
-
        /* Finally insert. */
        krcp->bhead->records[krcp->bhead->nr_records++] =
                (void *) head - (unsigned long) func;
@@ -3263,14 +3257,17 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 {
        unsigned long flags;
        struct kfree_rcu_cpu *krcp;
+       void *ptr;
 
        local_irq_save(flags);  // For safely calling this_cpu_ptr().
        krcp = this_cpu_ptr(&krc);
        if (krcp->initialized)
                raw_spin_lock(&krcp->lock);
 
+       ptr = (void *)head - (unsigned long)func;
+
        // Queue the object but don't yet schedule the batch.
-       if (debug_rcu_head_queue(head)) {
+       if (debug_rcu_head_queue(ptr)) {
                // Probable double kfree_rcu(), just leak.
                WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
                          __func__, head);