net/tcp: optimise io_uring zc ubuf refcounting

io_uring keeps a reference to ubuf_info during submission, so if
tcp_sendmsg_locked() sees msghdr::msg_ubuf in can be sure the buffer
will be kept alive and doesn't need to additionally pin it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 40f591f7fce13bbafe36ea0be08aa5ecafdca989..3d18e295bb2f534c35dc8a6cf671f28e8a345180 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -1231,7 +1231,6 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
        if ((flags & MSG_ZEROCOPY) && size) {
                if (msg->msg_ubuf) {
                        uarg = msg->msg_ubuf;
-                       net_zcopy_get(uarg);
                        zc = sk->sk_route_caps & NETIF_F_SG;
                } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
                        skb = tcp_write_queue_tail(sk);
@@ -1458,7 +1457,9 @@ out:
                tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
        }
 out_nopush:
-       net_zcopy_put(uarg);
+       /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
+       if (uarg && !msg->msg_ubuf)
+               net_zcopy_put(uarg);
        return copied + copied_syn;
 
 do_error:
@@ -1467,7 +1468,9 @@ do_error:
        if (copied + copied_syn)
                goto out;
 out_err:
-       net_zcopy_put_abort(uarg, true);
+       /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
+       if (uarg && !msg->msg_ubuf)
+               net_zcopy_put_abort(uarg, true);
        err = sk_stream_error(sk, flags, err);
        /* make sure we wake any epoll edge trigger waiter */
        if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {