net: adjust net_device layout for cacheline usage

The current layout of net_device is not optimal for cacheline usage.

The member adj_list.lower linked list is split between cacheline 2 and 3.
The ifindex is placed together with stats (struct net_device_stats),
although most modern drivers don't update this stats member.

The members netdev_ops, mtu and hard_header_len are placed on three
different cachelines. These members are accessed for XDP redirect into
devmap, which were noticeably with perf tool. When not using the map
redirect variant (like TC-BPF does), then ifindex is also used, which is
placed on a separate fourth cacheline. These members are also accessed
during forwarding with regular network stack. The members priv_flags and
flags are on fast-path for network stack transmit path in __dev_queue_xmit
(currently located together with mtu cacheline).

This patch creates a read mostly cacheline, with the purpose of keeping the
above mentioned members on the same cacheline.

Some netdev_features_t members also becomes part of this cacheline, which is
on purpose, as function netif_skb_features() is on fast-path via
validate_xmit_skb().

Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Link: https://lore.kernel.org/r/161168277983.410784.12401225493601624417.stgit@firesoul
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 9e85725..e9e7ada 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1858,7 +1858,6 @@ struct net_device {
        unsigned long           mem_end;
        unsigned long           mem_start;
        unsigned long           base_addr;
-       int                     irq;
 
        /*
         *      Some hardware also needs these fields (state,dev_list,
@@ -1880,6 +1879,23 @@ struct net_device {
                struct list_head lower;
        } adj_list;
 
+       /* Read-mostly cache-line for fast-path access */
+       unsigned int            flags;
+       unsigned int            priv_flags;
+       const struct net_device_ops *netdev_ops;
+       int                     ifindex;
+       unsigned short          gflags;
+       unsigned short          hard_header_len;
+
+       /* Note : dev->mtu is often read without holding a lock.
+        * Writers usually hold RTNL.
+        * It is recommended to use READ_ONCE() to annotate the reads,
+        * and to use WRITE_ONCE() to annotate the writes.
+        */
+       unsigned int            mtu;
+       unsigned short          needed_headroom;
+       unsigned short          needed_tailroom;
+
        netdev_features_t       features;
        netdev_features_t       hw_features;
        netdev_features_t       wanted_features;
@@ -1888,10 +1904,15 @@ struct net_device {
        netdev_features_t       mpls_features;
        netdev_features_t       gso_partial_features;
 
-       int                     ifindex;
+       unsigned int            min_mtu;
+       unsigned int            max_mtu;
+       unsigned short          type;
+       unsigned char           min_header_len;
+       unsigned char           name_assign_type;
+
        int                     group;
 
-       struct net_device_stats stats;
+       struct net_device_stats stats; /* not used by modern drivers */
 
        atomic_long_t           rx_dropped;
        atomic_long_t           tx_dropped;
@@ -1905,7 +1926,6 @@ struct net_device {
        const struct iw_handler_def *wireless_handlers;
        struct iw_public_data   *wireless_data;
 #endif
-       const struct net_device_ops *netdev_ops;
        const struct ethtool_ops *ethtool_ops;
 #ifdef CONFIG_NET_L3_MASTER_DEV
        const struct l3mdev_ops *l3mdev_ops;
@@ -1924,34 +1944,12 @@ struct net_device {
 
        const struct header_ops *header_ops;
 
-       unsigned int            flags;
-       unsigned int            priv_flags;
-
-       unsigned short          gflags;
-       unsigned short          padded;
-
        unsigned char           operstate;
        unsigned char           link_mode;
 
        unsigned char           if_port;
        unsigned char           dma;
 
-       /* Note : dev->mtu is often read without holding a lock.
-        * Writers usually hold RTNL.
-        * It is recommended to use READ_ONCE() to annotate the reads,
-        * and to use WRITE_ONCE() to annotate the writes.
-        */
-       unsigned int            mtu;
-       unsigned int            min_mtu;
-       unsigned int            max_mtu;
-       unsigned short          type;
-       unsigned short          hard_header_len;
-       unsigned char           min_header_len;
-       unsigned char           name_assign_type;
-
-       unsigned short          needed_headroom;
-       unsigned short          needed_tailroom;
-
        /* Interface address info. */
        unsigned char           perm_addr[MAX_ADDR_LEN];
        unsigned char           addr_assign_type;
@@ -1962,7 +1960,10 @@ struct net_device {
        unsigned short          neigh_priv_len;
        unsigned short          dev_id;
        unsigned short          dev_port;
+       unsigned short          padded;
+
        spinlock_t              addr_list_lock;
+       int                     irq;
 
        struct netdev_hw_addr_list      uc;
        struct netdev_hw_addr_list      mc;