intel: Remove rcu_read_lock() around XDP program invocation

The Intel drivers all have rcu_read_lock()/rcu_read_unlock() pairs around
XDP program invocations. However, the actual lifetime of the objects
referred by the XDP program invocation is longer, all the way through to
the call to xdp_do_flush(), making the scope of the rcu_read_lock() too
small. This turns out to be harmless because it all happens in a single
NAPI poll cycle (and thus under local_bh_disable()), but it makes the
rcu_read_lock() misleading.

Rather than extend the scope of the rcu_read_lock(), just get rid of it
entirely. With the addition of RCU annotations to the XDP_REDIRECT map
types that take bh execution into account, lockdep even understands this to
be safe, so there's really no reason to keep it around.

Signed-off-by: Toke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Tested-by: Jesper Dangaard Brouer <brouer@redhat.com> # i40e
Cc: Jesse Brandeburg <jesse.brandeburg@intel.com>
Cc: Tony Nguyen <anthony.l.nguyen@intel.com>
Cc: intel-wired-lan@lists.osuosl.org
Link: https://lore.kernel.org/bpf/20210624160609.292325-12-toke@redhat.com

diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index b883ab8..38eb815 100644 (file)
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -2298,7 +2298,6 @@ static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
        struct bpf_prog *xdp_prog;
        u32 act;
 
-       rcu_read_lock();
        xdp_prog = READ_ONCE(rx_ring->xdp_prog);
 
        if (!xdp_prog)
@@ -2334,7 +2333,6 @@ out_failure:
                break;
        }
 xdp_out:
-       rcu_read_unlock();
        return result;
 }
 

diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
index 68f177a..e7e778c 100644 (file)
--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
@@ -153,7 +153,6 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
        struct bpf_prog *xdp_prog;
        u32 act;
 
-       rcu_read_lock();
        /* NB! xdp_prog will always be !NULL, due to the fact that
         * this path is enabled by setting an XDP program.
         */
@@ -164,7 +163,6 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
                err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
                if (err)
                        goto out_failure;
-               rcu_read_unlock();
                return I40E_XDP_REDIR;
        }
 
@@ -188,7 +186,6 @@ out_failure:
                result = I40E_XDP_CONSUMED;
                break;
        }
-       rcu_read_unlock();
        return result;
 }
 

diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c
index 917eba7..dd791ca 100644 (file)
--- a/drivers/net/ethernet/intel/ice/ice_txrx.c
+++ b/drivers/net/ethernet/intel/ice/ice_txrx.c
@@ -1135,15 +1135,11 @@ int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget)
                xdp.frame_sz = ice_rx_frame_truesize(rx_ring, size);
 #endif
 
-               rcu_read_lock();
                xdp_prog = READ_ONCE(rx_ring->xdp_prog);
-               if (!xdp_prog) {
-                       rcu_read_unlock();
+               if (!xdp_prog)
                        goto construct_skb;
-               }
 
                xdp_res = ice_run_xdp(rx_ring, &xdp, xdp_prog);
-               rcu_read_unlock();
                if (!xdp_res)
                        goto construct_skb;
                if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR)) {

diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c
index 239b9bf..8a09336 100644 (file)
--- a/drivers/net/ethernet/intel/ice/ice_xsk.c
+++ b/drivers/net/ethernet/intel/ice/ice_xsk.c
@@ -466,7 +466,6 @@ ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp)
        struct ice_ring *xdp_ring;
        u32 act;
 
-       rcu_read_lock();
        /* ZC patch is enabled only when XDP program is set,
         * so here it can not be NULL
         */
@@ -478,7 +477,6 @@ ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp)
                err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
                if (err)
                        goto out_failure;
-               rcu_read_unlock();
                return ICE_XDP_REDIR;
        }
 
@@ -503,7 +501,6 @@ out_failure:
                break;
        }
 
-       rcu_read_unlock();
        return result;
 }
 

diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 5db303d..7e6435d 100644 (file)
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -8381,7 +8381,6 @@ static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter,
        struct bpf_prog *xdp_prog;
        u32 act;
 
-       rcu_read_lock();
        xdp_prog = READ_ONCE(rx_ring->xdp_prog);
 
        if (!xdp_prog)
@@ -8416,7 +8415,6 @@ out_failure:
                break;
        }
 xdp_out:
-       rcu_read_unlock();
        return ERR_PTR(-result);
 }
 

diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c
index 3f6b6d4..9532309 100644 (file)
--- a/drivers/net/ethernet/intel/igc/igc_main.c
+++ b/drivers/net/ethernet/intel/igc/igc_main.c
@@ -2240,18 +2240,15 @@ static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter,
        struct bpf_prog *prog;
        int res;
 
-       rcu_read_lock();
-
        prog = READ_ONCE(adapter->xdp_prog);
        if (!prog) {
                res = IGC_XDP_PASS;
-               goto unlock;
+               goto out;
        }
 
        res = __igc_xdp_run_prog(adapter, prog, xdp);
 
-unlock:
-       rcu_read_unlock();
+out:
        return ERR_PTR(-res);
 }
 

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 2ac5b82..ffff69e 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -2199,7 +2199,6 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
        struct xdp_frame *xdpf;
        u32 act;
 
-       rcu_read_lock();
        xdp_prog = READ_ONCE(rx_ring->xdp_prog);
 
        if (!xdp_prog)
@@ -2237,7 +2236,6 @@ out_failure:
                break;
        }
 xdp_out:
-       rcu_read_unlock();
        return ERR_PTR(-result);
 }
 

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
index f72d297..96dd1a4 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
@@ -100,7 +100,6 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter,
        struct xdp_frame *xdpf;
        u32 act;
 
-       rcu_read_lock();
        xdp_prog = READ_ONCE(rx_ring->xdp_prog);
        act = bpf_prog_run_xdp(xdp_prog, xdp);
 
@@ -108,7 +107,6 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter,
                err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
                if (err)
                        goto out_failure;
-               rcu_read_unlock();
                return IXGBE_XDP_REDIR;
        }
 
@@ -134,7 +132,6 @@ out_failure:
                result = IXGBE_XDP_CONSUMED;
                break;
        }
-       rcu_read_unlock();
        return result;
 }
 

diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index dc56931..c714e1e 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -1054,7 +1054,6 @@ static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter,
        struct bpf_prog *xdp_prog;
        u32 act;
 
-       rcu_read_lock();
        xdp_prog = READ_ONCE(rx_ring->xdp_prog);
 
        if (!xdp_prog)
@@ -1082,7 +1081,6 @@ out_failure:
                break;
        }
 xdp_out:
-       rcu_read_unlock();
        return ERR_PTR(-result);
 }