net: Use sendmsg(MSG_SPLICE_PAGES) not sendpage in skb_send_sock()

Use sendmsg() with MSG_SPLICE_PAGES rather than sendpage in
skb_send_sock().  This causes pages to be spliced from the source iterator
if possible.

This allows ->sendpage() to be replaced by something that can handle
multiple multipage folios in a single transaction.

Note that this could perhaps be improved to fill out a bvec array with all
the frags and then make a single sendmsg call, possibly sticking the header
on the front also.

Signed-off-by: David Howells <dhowells@redhat.com>
cc: Jens Axboe <axboe@kernel.dk>
cc: Matthew Wilcox <willy@infradead.org>
Link: https://lore.kernel.org/r/20230623225513.2732256-3-dhowells@redhat.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index fee2b1c..6c5915e 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -2989,32 +2989,32 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
 }
 EXPORT_SYMBOL_GPL(skb_splice_bits);
 
-static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg,
-                           struct kvec *vec, size_t num, size_t size)
+static int sendmsg_locked(struct sock *sk, struct msghdr *msg)
 {
        struct socket *sock = sk->sk_socket;
+       size_t size = msg_data_left(msg);
 
        if (!sock)
                return -EINVAL;
-       return kernel_sendmsg(sock, msg, vec, num, size);
+
+       if (!sock->ops->sendmsg_locked)
+               return sock_no_sendmsg_locked(sk, msg, size);
+
+       return sock->ops->sendmsg_locked(sk, msg, size);
 }
 
-static int sendpage_unlocked(struct sock *sk, struct page *page, int offset,
-                            size_t size, int flags)
+static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg)
 {
        struct socket *sock = sk->sk_socket;
 
        if (!sock)
                return -EINVAL;
-       return kernel_sendpage(sock, page, offset, size, flags);
+       return sock_sendmsg(sock, msg);
 }
 
-typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg,
-                           struct kvec *vec, size_t num, size_t size);
-typedef int (*sendpage_func)(struct sock *sk, struct page *page, int offset,
-                            size_t size, int flags);
+typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg);
 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset,
-                          int len, sendmsg_func sendmsg, sendpage_func sendpage)
+                          int len, sendmsg_func sendmsg)
 {
        unsigned int orig_len = len;
        struct sk_buff *head = skb;
@@ -3034,8 +3034,9 @@ do_frag_list:
                memset(&msg, 0, sizeof(msg));
                msg.msg_flags = MSG_DONTWAIT;
 
-               ret = INDIRECT_CALL_2(sendmsg, kernel_sendmsg_locked,
-                                     sendmsg_unlocked, sk, &msg, &kv, 1, slen);
+               iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen);
+               ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
+                                     sendmsg_unlocked, sk, &msg);
                if (ret <= 0)
                        goto error;
 
@@ -3066,11 +3067,18 @@ do_frag_list:
                slen = min_t(size_t, len, skb_frag_size(frag) - offset);
 
                while (slen) {
-                       ret = INDIRECT_CALL_2(sendpage, kernel_sendpage_locked,
-                                             sendpage_unlocked, sk,
-                                             skb_frag_page(frag),
-                                             skb_frag_off(frag) + offset,
-                                             slen, MSG_DONTWAIT);
+                       struct bio_vec bvec;
+                       struct msghdr msg = {
+                               .msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT,
+                       };
+
+                       bvec_set_page(&bvec, skb_frag_page(frag), slen,
+                                     skb_frag_off(frag) + offset);
+                       iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1,
+                                     slen);
+
+                       ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
+                                             sendmsg_unlocked, sk, &msg);
                        if (ret <= 0)
                                goto error;
 
@@ -3107,16 +3115,14 @@ error:
 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
                         int len)
 {
-       return __skb_send_sock(sk, skb, offset, len, kernel_sendmsg_locked,
-                              kernel_sendpage_locked);
+       return __skb_send_sock(sk, skb, offset, len, sendmsg_locked);
 }
 EXPORT_SYMBOL_GPL(skb_send_sock_locked);
 
 /* Send skb data on a socket. Socket must be unlocked. */
 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
 {
-       return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked,
-                              sendpage_unlocked);
+       return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked);
 }
 
 /**