kcm: Support MSG_SPLICE_PAGES

Make AF_KCM sendmsg() support MSG_SPLICE_PAGES.  This causes pages to be
spliced from the source iterator if possible.

This allows ->sendpage() to be replaced by something that can handle
multiple multipage folios in a single transaction.

Signed-off-by: David Howells <dhowells@redhat.com>
cc: Tom Herbert <tom@herbertland.com>
cc: Tom Herbert <tom@quantonium.net>
cc: Cong Wang <cong.wang@bytedance.com>
cc: Jens Axboe <axboe@kernel.dk>
cc: Matthew Wilcox <willy@infradead.org>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c
index cfe828b..8555ede 100644 (file)
--- a/net/kcm/kcmsock.c
+++ b/net/kcm/kcmsock.c
@@ -989,29 +989,52 @@ start:
                        merge = false;
                }
 
-               copy = min_t(int, msg_data_left(msg),
-                            pfrag->size - pfrag->offset);
+               if (msg->msg_flags & MSG_SPLICE_PAGES) {
+                       copy = msg_data_left(msg);
+                       if (!sk_wmem_schedule(sk, copy))
+                               goto wait_for_memory;
 
-               if (!sk_wmem_schedule(sk, copy))
-                       goto wait_for_memory;
+                       err = skb_splice_from_iter(skb, &msg->msg_iter, copy,
+                                                  sk->sk_allocation);
+                       if (err < 0) {
+                               if (err == -EMSGSIZE)
+                                       goto wait_for_memory;
+                               goto out_error;
+                       }
 
-               err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
-                                              pfrag->page,
-                                              pfrag->offset,
-                                              copy);
-               if (err)
-                       goto out_error;
+                       copy = err;
+                       skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
+                       sk_wmem_queued_add(sk, copy);
+                       sk_mem_charge(sk, copy);
 
-               /* Update the skb. */
-               if (merge) {
-                       skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
+                       if (head != skb)
+                               head->truesize += copy;
                } else {
-                       skb_fill_page_desc(skb, i, pfrag->page,
-                                          pfrag->offset, copy);
-                       get_page(pfrag->page);
+                       copy = min_t(int, msg_data_left(msg),
+                                    pfrag->size - pfrag->offset);
+                       if (!sk_wmem_schedule(sk, copy))
+                               goto wait_for_memory;
+
+                       err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
+                                                      pfrag->page,
+                                                      pfrag->offset,
+                                                      copy);
+                       if (err)
+                               goto out_error;
+
+                       /* Update the skb. */
+                       if (merge) {
+                               skb_frag_size_add(
+                                       &skb_shinfo(skb)->frags[i - 1], copy);
+                       } else {
+                               skb_fill_page_desc(skb, i, pfrag->page,
+                                                  pfrag->offset, copy);
+                               get_page(pfrag->page);
+                       }
+
+                       pfrag->offset += copy;
                }
 
-               pfrag->offset += copy;
                copied += copy;
                if (head != skb) {
                        head->len += copy;