tcp_bpf: Make tcp_bpf_sendpage() go through tcp_bpf_sendmsg(MSG_SPLICE_PAGES)

Make tcp_bpf_sendpage() a wrapper around tcp_bpf_sendmsg(MSG_SPLICE_PAGES)
rather than a loop calling tcp_sendpage().  sendpage() will be removed in
the future.

Signed-off-by: David Howells <dhowells@redhat.com>
cc: John Fastabend <john.fastabend@gmail.com>
cc: Jakub Sitnicki <jakub@cloudflare.com>
cc: Jens Axboe <axboe@kernel.dk>
cc: Matthew Wilcox <willy@infradead.org>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c
index e75023e..5a84053 100644 (file)
--- a/net/ipv4/tcp_bpf.c
+++ b/net/ipv4/tcp_bpf.c
@@ -568,49 +568,18 @@ out_err:
 static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset,
                            size_t size, int flags)
 {
-       struct sk_msg tmp, *msg = NULL;
-       int err = 0, copied = 0;
-       struct sk_psock *psock;
-       bool enospc = false;
-
-       psock = sk_psock_get(sk);
-       if (unlikely(!psock))
-               return tcp_sendpage(sk, page, offset, size, flags);
+       struct bio_vec bvec;
+       struct msghdr msg = {
+               .msg_flags = flags | MSG_SPLICE_PAGES,
+       };
 
-       lock_sock(sk);
-       if (psock->cork) {
-               msg = psock->cork;
-       } else {
-               msg = &tmp;
-               sk_msg_init(msg);
-       }
+       bvec_set_page(&bvec, page, size, offset);
+       iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size);
 
-       /* Catch case where ring is full and sendpage is stalled. */
-       if (unlikely(sk_msg_full(msg)))
-               goto out_err;
-
-       sk_msg_page_add(msg, page, size, offset);
-       sk_mem_charge(sk, size);
-       copied = size;
-       if (sk_msg_full(msg))
-               enospc = true;
-       if (psock->cork_bytes) {
-               if (size > psock->cork_bytes)
-                       psock->cork_bytes = 0;
-               else
-                       psock->cork_bytes -= size;
-               if (psock->cork_bytes && !enospc)
-                       goto out_err;
-               /* All cork bytes are accounted, rerun the prog. */
-               psock->eval = __SK_NONE;
-               psock->cork_bytes = 0;
-       }
+       if (flags & MSG_SENDPAGE_NOTLAST)
+               msg.msg_flags |= MSG_MORE;
 
-       err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags);
-out_err:
-       release_sock(sk);
-       sk_psock_put(sk, psock);
-       return copied ? copied : err;
+       return tcp_bpf_sendmsg(sk, &msg, size);
 }
 
 enum {