ipv6: factorize sk_wmem_alloc updates done by __ip6_append_data()

While testing my inet defrag changes, I found that the senders
could spend ~20% of cpu cycles in skb_set_owner_w() updating
sk->sk_wmem_alloc for every fragment they cook, competing
with TX completion of prior skbs possibly happening on another cpus.

The solution to this problem is to use alloc_skb() instead
of sock_wmalloc() and manually perform a single sk_wmem_alloc change.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 2c7f09c3c39ed8a1e85a967e105ff3cc30dce5b9..323d7a354ffb6f75e2a948dea63a8018ed0e057f 100644 (file)
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -1259,6 +1259,7 @@ static int __ip6_append_data(struct sock *sk,
        struct ipv6_txoptions *opt = v6_cork->opt;
        int csummode = CHECKSUM_NONE;
        unsigned int maxnonfragsize, headersize;
+       unsigned int wmem_alloc_delta = 0;
 
        skb = skb_peek_tail(queue);
        if (!skb) {
@@ -1411,11 +1412,10 @@ alloc_new_skb:
                                                (flags & MSG_DONTWAIT), &err);
                        } else {
                                skb = NULL;
-                               if (refcount_read(&sk->sk_wmem_alloc) <=
+                               if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <=
                                    2 * sk->sk_sndbuf)
-                                       skb = sock_wmalloc(sk,
-                                                          alloclen + hh_len, 1,
-                                                          sk->sk_allocation);
+                                       skb = alloc_skb(alloclen + hh_len,
+                                                       sk->sk_allocation);
                                if (unlikely(!skb))
                                        err = -ENOBUFS;
                        }
@@ -1474,6 +1474,11 @@ alloc_new_skb:
                        /*
                         * Put the packet on the pending queue
                         */
+                       if (!skb->destructor) {
+                               skb->destructor = sock_wfree;
+                               skb->sk = sk;
+                               wmem_alloc_delta += skb->truesize;
+                       }
                        __skb_queue_tail(queue, skb);
                        continue;
                }
@@ -1520,12 +1525,13 @@ alloc_new_skb:
                        skb->len += copy;
                        skb->data_len += copy;
                        skb->truesize += copy;
-                       refcount_add(copy, &sk->sk_wmem_alloc);
+                       wmem_alloc_delta += copy;
                }
                offset += copy;
                length -= copy;
        }
 
+       refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
        return 0;
 
 error_efault:
@@ -1533,6 +1539,7 @@ error_efault:
 error:
        cork->length -= length;
        IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
+       refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
        return err;
 }