tcp: move inet->rx_dst_ifindex to sk->sk_rx_dst_ifindex

[ Upstream commit 0c0a5ef809f9150e9229e7b13e43183b681b7a39 ]

Increase cache locality by moving rx_dst_ifindex next to sk->sk_rx_dst

This is part of an effort to reduce cache line misses in TCP fast path.

This removes one cache line miss in early demux.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h
index 89163ef..9e1111f 100644 (file)
--- a/include/net/inet_sock.h
+++ b/include/net/inet_sock.h
@@ -207,11 +207,10 @@ struct inet_sock {
        __be32                  inet_saddr;
        __s16                   uc_ttl;
        __u16                   cmsg_flags;
+       struct ip_options_rcu __rcu     *inet_opt;
        __be16                  inet_sport;
        __u16                   inet_id;
 
-       struct ip_options_rcu __rcu     *inet_opt;
-       int                     rx_dst_ifindex;
        __u8                    tos;
        __u8                    min_ttl;
        __u8                    mc_ttl;

diff --git a/include/net/sock.h b/include/net/sock.h
index 7ac5075..7526012 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -259,6 +259,7 @@ struct bpf_local_storage;
   *    @sk_rcvbuf: size of receive buffer in bytes
   *    @sk_wq: sock wait queue and async head
   *    @sk_rx_dst: receive input route used by early demux
+  *    @sk_rx_dst_ifindex: ifindex for @sk_rx_dst
   *    @sk_dst_cache: destination cache
   *    @sk_dst_pending_confirm: need to confirm neighbour
   *    @sk_policy: flow policy
@@ -431,6 +432,8 @@ struct sock {
        struct xfrm_policy __rcu *sk_policy[2];
 #endif
        struct dst_entry        *sk_rx_dst;
+       int                     sk_rx_dst_ifindex;
+
        struct dst_entry __rcu  *sk_dst_cache;
        atomic_t                sk_omem_alloc;
        int                     sk_sndbuf;

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 5b8ce65..f6838ee 100644 (file)
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -1703,7 +1703,7 @@ int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
                sock_rps_save_rxhash(sk, skb);
                sk_mark_napi_id(sk, skb);
                if (dst) {
-                       if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
+                       if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
                            !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,
                                             dst, 0)) {
                                dst_release(dst);
@@ -1788,7 +1788,7 @@ int tcp_v4_early_demux(struct sk_buff *skb)
                        if (dst)
                                dst = dst_check(dst, 0);
                        if (dst &&
-                           inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
+                           sk->sk_rx_dst_ifindex == skb->skb_iif)
                                skb_dst_set_noref(skb, dst);
                }
        }
@@ -2201,7 +2201,7 @@ void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
 
        if (dst && dst_hold_safe(dst)) {
                sk->sk_rx_dst = dst;
-               inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
+               sk->sk_rx_dst_ifindex = skb->skb_iif;
        }
 }
 EXPORT_SYMBOL(inet_sk_rx_dst_set);

diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index b03dd02..c72586e 100644 (file)
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -108,7 +108,7 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
                const struct rt6_info *rt = (const struct rt6_info *)dst;
 
                sk->sk_rx_dst = dst;
-               inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
+               sk->sk_rx_dst_ifindex = skb->skb_iif;
                tcp_inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);
        }
 }
@@ -1509,7 +1509,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
                sock_rps_save_rxhash(sk, skb);
                sk_mark_napi_id(sk, skb);
                if (dst) {
-                       if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
+                       if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
                            INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
                                            dst, np->rx_dst_cookie) == NULL) {
                                dst_release(dst);
@@ -1880,7 +1880,7 @@ INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb)
                        if (dst)
                                dst = dst_check(dst, tcp_inet6_sk(sk)->rx_dst_cookie);
                        if (dst &&
-                           inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
+                           sk->sk_rx_dst_ifindex == skb->skb_iif)
                                skb_dst_set_noref(skb, dst);
                }
        }