tcp_bpf: Inline do_tcp_sendpages as it's now a wrapper around tcp_sendmsg

do_tcp_sendpages() is now just a small wrapper around tcp_sendmsg_locked(),
so inline it.  This is part of replacing ->sendpage() with a call to
sendmsg() with MSG_SPLICE_PAGES set.

Signed-off-by: David Howells <dhowells@redhat.com>
cc: John Fastabend <john.fastabend@gmail.com>
cc: Jakub Sitnicki <jakub@cloudflare.com>
cc: David Ahern <dsahern@kernel.org>
cc: Jens Axboe <axboe@kernel.dk>
cc: Matthew Wilcox <willy@infradead.org>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c
index 2e95474..0291d15 100644 (file)
--- a/net/ipv4/tcp_bpf.c
+++ b/net/ipv4/tcp_bpf.c
@@ -72,11 +72,13 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,
 {
        bool apply = apply_bytes;
        struct scatterlist *sge;
+       struct msghdr msghdr = { .msg_flags = flags | MSG_SPLICE_PAGES, };
        struct page *page;
        int size, ret = 0;
        u32 off;
 
        while (1) {
+               struct bio_vec bvec;
                bool has_tx_ulp;
 
                sge = sk_msg_elem(msg, msg->sg.start);
@@ -88,16 +90,18 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,
                tcp_rate_check_app_limited(sk);
 retry:
                has_tx_ulp = tls_sw_has_ctx_tx(sk);
-               if (has_tx_ulp) {
-                       flags |= MSG_SENDPAGE_NOPOLICY;
-                       ret = kernel_sendpage_locked(sk,
-                                                    page, off, size, flags);
-               } else {
-                       ret = do_tcp_sendpages(sk, page, off, size, flags);
-               }
+               if (has_tx_ulp)
+                       msghdr.msg_flags |= MSG_SENDPAGE_NOPOLICY;
 
+               if (flags & MSG_SENDPAGE_NOTLAST)
+                       msghdr.msg_flags |= MSG_MORE;
+
+               bvec_set_page(&bvec, page, size, off);
+               iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, size);
+               ret = tcp_sendmsg_locked(sk, &msghdr, size);
                if (ret <= 0)
                        return ret;
+
                if (apply)
                        apply_bytes -= ret;
                msg->sg.size -= ret;
@@ -404,7 +408,7 @@ static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
        long timeo;
        int flags;
 
-       /* Don't let internal do_tcp_sendpages() flags through */
+       /* Don't let internal sendpage flags through */
        flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED);
        flags |= MSG_NO_SHARED_FRAGS;