page_pool: unlink from napi during destroy

[platform/kernel/linux-starfive.git] / net / core / page_pool.c

diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index 97f20f7..e212e9d 100644 (file)
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -839,6 +839,21 @@ void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
        pool->xdp_mem_id = mem->id;
 }
 
+void page_pool_unlink_napi(struct page_pool *pool)
+{
+       if (!pool->p.napi)
+               return;
+
+       /* To avoid races with recycling and additional barriers make sure
+        * pool and NAPI are unlinked when NAPI is disabled.
+        */
+       WARN_ON(!test_bit(NAPI_STATE_SCHED, &pool->p.napi->state) ||
+               READ_ONCE(pool->p.napi->list_owner) != -1);
+
+       WRITE_ONCE(pool->p.napi, NULL);
+}
+EXPORT_SYMBOL(page_pool_unlink_napi);
+
 void page_pool_destroy(struct page_pool *pool)
 {
        if (!pool)
@@ -847,6 +862,7 @@ void page_pool_destroy(struct page_pool *pool)
        if (!page_pool_put(pool))
                return;
 
+       page_pool_unlink_napi(pool);
        page_pool_free_frag(pool);
 
        if (!page_pool_release(pool))
@@ -900,7 +916,7 @@ bool page_pool_return_skb_page(struct page *page, bool napi_safe)
         * in the same context as the consumer would run, so there's
         * no possible race.
         */
-       napi = pp->p.napi;
+       napi = READ_ONCE(pp->p.napi);
        allow_direct = napi_safe && napi &&
                READ_ONCE(napi->list_owner) == smp_processor_id();