udp: track the forward memory release threshold in an hot cacheline

When the receiver process and the BH runs on different cores,
udp_rmem_release() experience a cache miss while accessing sk_rcvbuf,
as the latter shares the same cacheline with sk_forward_alloc, written
by the BH.

With this patch, UDP tracks the rcvbuf value and its update via custom
SOL_SOCKET socket options, and copies the forward memory threshold value
used by udp_rmem_release() in a different cacheline, already accessed by
the above function and uncontended.

Since the UDP socket init operation grown a bit, factor out the common
code between v4 and v6 in a shared helper.

Overall the above give a 10% peek throughput increase under UDP flood.

Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Acked-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/udp.h b/include/linux/udp.h
index e96da4157d04daa97cbf064f9b2c013e1ecdb9b9..5cdba00a904a4f96fa0c7636c70bccf722102e2a 100644 (file)
--- a/include/linux/udp.h
+++ b/include/linux/udp.h
@@ -87,6 +87,9 @@ struct udp_sock {
 
        /* This field is dirtied by udp_recvmsg() */
        int             forward_deficit;
+
+       /* This fields follows rcvbuf value, and is touched by udp_recvmsg */
+       int             forward_threshold;
 };
 
 #define UDP_MAX_SEGMENTS       (1 << 6UL)

diff --git a/include/net/udp.h b/include/net/udp.h
index fee053bcd17c6ddc7cea6c055e587f4321e800e0..de4b528522bb91037646d1ae996f64f2ec8358ea 100644 (file)
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -174,6 +174,15 @@ INDIRECT_CALLABLE_DECLARE(int udpv6_rcv(struct sk_buff *));
 struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
                                  netdev_features_t features, bool is_ipv6);
 
+static inline void udp_lib_init_sock(struct sock *sk)
+{
+       struct udp_sock *up = udp_sk(sk);
+
+       skb_queue_head_init(&up->reader_queue);
+       up->forward_threshold = sk->sk_rcvbuf >> 2;
+       set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags);
+}
+
 /* hash routines shared between UDPv4/6 and UDP-Litev4/6 */
 static inline int udp_lib_hash(struct sock *sk)
 {

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index c83e5271030b4731222d95cd02cf05dbc93628fb..e77c8f0e90878c5eac6226cfbcda4ee91f53bd7e 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1448,7 +1448,7 @@ static void udp_rmem_release(struct sock *sk, int size, int partial,
        if (likely(partial)) {
                up->forward_deficit += size;
                size = up->forward_deficit;
-               if (size < (sk->sk_rcvbuf >> 2) &&
+               if (size < READ_ONCE(up->forward_threshold) &&
                    !skb_queue_empty(&up->reader_queue))
                        return;
        } else {
@@ -1622,7 +1622,7 @@ static void udp_destruct_sock(struct sock *sk)
 
 int udp_init_sock(struct sock *sk)
 {
-       skb_queue_head_init(&udp_sk(sk)->reader_queue);
+       udp_lib_init_sock(sk);
        sk->sk_destruct = udp_destruct_sock;
        return 0;
 }
@@ -2671,6 +2671,18 @@ int udp_lib_setsockopt(struct sock *sk, int level, int optname,
        int err = 0;
        int is_udplite = IS_UDPLITE(sk);
 
+       if (level == SOL_SOCKET) {
+               err = sk_setsockopt(sk, level, optname, optval, optlen);
+
+               if (optname == SO_RCVBUF || optname == SO_RCVBUFFORCE) {
+                       sockopt_lock_sock(sk);
+                       /* paired with READ_ONCE in udp_rmem_release() */
+                       WRITE_ONCE(up->forward_threshold, sk->sk_rcvbuf >> 2);
+                       sockopt_release_sock(sk);
+               }
+               return err;
+       }
+
        if (optlen < sizeof(int))
                return -EINVAL;
 
@@ -2784,7 +2796,7 @@ EXPORT_SYMBOL(udp_lib_setsockopt);
 int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
                   unsigned int optlen)
 {
-       if (level == SOL_UDP  ||  level == SOL_UDPLITE)
+       if (level == SOL_UDP  ||  level == SOL_UDPLITE || level == SOL_SOCKET)
                return udp_lib_setsockopt(sk, level, optname,
                                          optval, optlen,
                                          udp_push_pending_frames);

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 2260406740d3b701909c1e0342f010356596d5a2..297f7cc06044420482a61e580cf0b3210b450ab4 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -64,7 +64,7 @@ static void udpv6_destruct_sock(struct sock *sk)
 
 int udpv6_init_sock(struct sock *sk)
 {
-       skb_queue_head_init(&udp_sk(sk)->reader_queue);
+       udp_lib_init_sock(sk);
        sk->sk_destruct = udpv6_destruct_sock;
        return 0;
 }
@@ -1669,7 +1669,7 @@ void udpv6_destroy_sock(struct sock *sk)
 int udpv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
                     unsigned int optlen)
 {
-       if (level == SOL_UDP  ||  level == SOL_UDPLITE)
+       if (level == SOL_UDP  ||  level == SOL_UDPLITE || level == SOL_SOCKET)
                return udp_lib_setsockopt(sk, level, optname,
                                          optval, optlen,
                                          udp_v6_push_pending_frames);