1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
7 * Implementation of the Transmission Control Protocol(TCP).
9 * IPv4 specific functions
13 * linux/ipv4/tcp_input.c
14 * linux/ipv4/tcp_output.c
16 * See tcp.c for author information
21 * David S. Miller : New socket lookup architecture.
22 * This code is dedicated to John Dyson.
23 * David S. Miller : Change semantics of established hash,
24 * half is devoted to TIME_WAIT sockets
25 * and the rest go in the other half.
26 * Andi Kleen : Add support for syncookies and fixed
27 * some bugs: ip options weren't passed to
28 * the TCP layer, missed a check for an
30 * Andi Kleen : Implemented fast path mtu discovery.
31 * Fixed many serious bugs in the
32 * request_sock handling and moved
33 * most of it into the af independent code.
34 * Added tail drop and some other bugfixes.
35 * Added new listen semantics.
36 * Mike McLagan : Routing by source
37 * Juan Jose Ciarlante: ip_dynaddr bits
38 * Andi Kleen: various fixes.
39 * Vitaly E. Lavrov : Transparent proxy revived after year
41 * Andi Kleen : Fix new listen.
42 * Andi Kleen : Fix accept error reporting.
43 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
44 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
45 * a single port at the same time.
48 #define pr_fmt(fmt) "TCP: " fmt
50 #include <linux/bottom_half.h>
51 #include <linux/types.h>
52 #include <linux/fcntl.h>
53 #include <linux/module.h>
54 #include <linux/random.h>
55 #include <linux/cache.h>
56 #include <linux/jhash.h>
57 #include <linux/init.h>
58 #include <linux/times.h>
59 #include <linux/slab.h>
61 #include <net/net_namespace.h>
63 #include <net/inet_hashtables.h>
65 #include <net/transp_v6.h>
67 #include <net/inet_common.h>
68 #include <net/timewait_sock.h>
70 #include <net/secure_seq.h>
71 #include <net/busy_poll.h>
73 #include <linux/inet.h>
74 #include <linux/ipv6.h>
75 #include <linux/stddef.h>
76 #include <linux/proc_fs.h>
77 #include <linux/seq_file.h>
78 #include <linux/inetdevice.h>
79 #include <linux/btf_ids.h>
81 #include <crypto/hash.h>
82 #include <linux/scatterlist.h>
84 #include <trace/events/tcp.h>
86 #ifdef CONFIG_TCP_MD5SIG
87 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
88 __be32 daddr, __be32 saddr, const struct tcphdr *th);
91 struct inet_hashinfo tcp_hashinfo;
92 EXPORT_SYMBOL(tcp_hashinfo);
94 static DEFINE_PER_CPU(struct sock *, ipv4_tcp_sk);
96 static u32 tcp_v4_init_seq(const struct sk_buff *skb)
98 return secure_tcp_seq(ip_hdr(skb)->daddr,
101 tcp_hdr(skb)->source);
104 static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb)
106 return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr);
109 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
111 int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse);
112 const struct inet_timewait_sock *tw = inet_twsk(sktw);
113 const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
114 struct tcp_sock *tp = tcp_sk(sk);
117 /* Still does not detect *everything* that goes through
118 * lo, since we require a loopback src or dst address
119 * or direct binding to 'lo' interface.
121 bool loopback = false;
122 if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
124 #if IS_ENABLED(CONFIG_IPV6)
125 if (tw->tw_family == AF_INET6) {
126 if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
127 ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
128 ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
129 ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
134 if (ipv4_is_loopback(tw->tw_daddr) ||
135 ipv4_is_loopback(tw->tw_rcv_saddr))
142 /* With PAWS, it is safe from the viewpoint
143 of data integrity. Even without PAWS it is safe provided sequence
144 spaces do not overlap i.e. at data rates <= 80Mbit/sec.
146 Actually, the idea is close to VJ's one, only timestamp cache is
147 held not per host, but per port pair and TW bucket is used as state
150 If TW bucket has been already destroyed we fall back to VJ's scheme
151 and use initial timestamp retrieved from peer table.
153 if (tcptw->tw_ts_recent_stamp &&
154 (!twp || (reuse && time_after32(ktime_get_seconds(),
155 tcptw->tw_ts_recent_stamp)))) {
156 /* In case of repair and re-using TIME-WAIT sockets we still
157 * want to be sure that it is safe as above but honor the
158 * sequence numbers and time stamps set as part of the repair
161 * Without this check re-using a TIME-WAIT socket with TCP
162 * repair would accumulate a -1 on the repair assigned
163 * sequence number. The first time it is reused the sequence
164 * is -1, the second time -2, etc. This fixes that issue
165 * without appearing to create any others.
167 if (likely(!tp->repair)) {
168 u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
172 WRITE_ONCE(tp->write_seq, seq);
173 tp->rx_opt.ts_recent = tcptw->tw_ts_recent;
174 tp->rx_opt.ts_recent_stamp = tcptw->tw_ts_recent_stamp;
182 EXPORT_SYMBOL_GPL(tcp_twsk_unique);
184 static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr,
187 /* This check is replicated from tcp_v4_connect() and intended to
188 * prevent BPF program called below from accessing bytes that are out
189 * of the bound specified by user in addr_len.
191 if (addr_len < sizeof(struct sockaddr_in))
194 sock_owned_by_me(sk);
196 return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr);
199 /* This will initiate an outgoing connection. */
200 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
202 struct inet_bind_hashbucket *prev_addr_hashbucket = NULL;
203 struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
204 struct inet_timewait_death_row *tcp_death_row;
205 __be32 daddr, nexthop, prev_sk_rcv_saddr;
206 struct inet_sock *inet = inet_sk(sk);
207 struct tcp_sock *tp = tcp_sk(sk);
208 struct ip_options_rcu *inet_opt;
209 struct net *net = sock_net(sk);
210 __be16 orig_sport, orig_dport;
215 if (addr_len < sizeof(struct sockaddr_in))
218 if (usin->sin_family != AF_INET)
219 return -EAFNOSUPPORT;
221 nexthop = daddr = usin->sin_addr.s_addr;
222 inet_opt = rcu_dereference_protected(inet->inet_opt,
223 lockdep_sock_is_held(sk));
224 if (inet_opt && inet_opt->opt.srr) {
227 nexthop = inet_opt->opt.faddr;
230 orig_sport = inet->inet_sport;
231 orig_dport = usin->sin_port;
232 fl4 = &inet->cork.fl.u.ip4;
233 rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
234 sk->sk_bound_dev_if, IPPROTO_TCP, orig_sport,
238 if (err == -ENETUNREACH)
239 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
243 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
248 if (!inet_opt || !inet_opt->opt.srr)
251 if (!inet->inet_saddr) {
252 if (inet_csk(sk)->icsk_bind2_hash) {
253 prev_addr_hashbucket = inet_bhashfn_portaddr(&tcp_hashinfo,
254 sk, net, inet->inet_num);
255 prev_sk_rcv_saddr = sk->sk_rcv_saddr;
257 inet->inet_saddr = fl4->saddr;
260 sk_rcv_saddr_set(sk, inet->inet_saddr);
262 if (prev_addr_hashbucket) {
263 err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk);
265 inet->inet_saddr = 0;
266 sk_rcv_saddr_set(sk, prev_sk_rcv_saddr);
272 if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
273 /* Reset inherited state */
274 tp->rx_opt.ts_recent = 0;
275 tp->rx_opt.ts_recent_stamp = 0;
276 if (likely(!tp->repair))
277 WRITE_ONCE(tp->write_seq, 0);
280 inet->inet_dport = usin->sin_port;
281 sk_daddr_set(sk, daddr);
283 inet_csk(sk)->icsk_ext_hdr_len = 0;
285 inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
287 tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
289 /* Socket identity is still unknown (sport may be zero).
290 * However we set state to SYN-SENT and not releasing socket
291 * lock select source port, enter ourselves into the hash tables and
292 * complete initialization after this.
294 tcp_set_state(sk, TCP_SYN_SENT);
295 tcp_death_row = &net->ipv4.tcp_death_row;
296 err = inet_hash_connect(tcp_death_row, sk);
302 rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
303 inet->inet_sport, inet->inet_dport, sk);
309 /* OK, now commit destination to socket. */
310 sk->sk_gso_type = SKB_GSO_TCPV4;
311 sk_setup_caps(sk, &rt->dst);
314 if (likely(!tp->repair)) {
316 WRITE_ONCE(tp->write_seq,
317 secure_tcp_seq(inet->inet_saddr,
321 tp->tsoffset = secure_tcp_ts_off(net, inet->inet_saddr,
325 inet->inet_id = prandom_u32();
327 if (tcp_fastopen_defer_connect(sk, &err))
332 err = tcp_connect(sk);
341 * This unhashes the socket and releases the local port,
344 tcp_set_state(sk, TCP_CLOSE);
346 sk->sk_route_caps = 0;
347 inet->inet_dport = 0;
350 EXPORT_SYMBOL(tcp_v4_connect);
353 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
354 * It can be called through tcp_release_cb() if socket was owned by user
355 * at the time tcp_v4_err() was called to handle ICMP message.
357 void tcp_v4_mtu_reduced(struct sock *sk)
359 struct inet_sock *inet = inet_sk(sk);
360 struct dst_entry *dst;
363 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
365 mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
366 dst = inet_csk_update_pmtu(sk, mtu);
370 /* Something is about to be wrong... Remember soft error
371 * for the case, if this connection will not able to recover.
373 if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
374 sk->sk_err_soft = EMSGSIZE;
378 if (inet->pmtudisc != IP_PMTUDISC_DONT &&
379 ip_sk_accept_pmtu(sk) &&
380 inet_csk(sk)->icsk_pmtu_cookie > mtu) {
381 tcp_sync_mss(sk, mtu);
383 /* Resend the TCP packet because it's
384 * clear that the old packet has been
385 * dropped. This is the new "fast" path mtu
388 tcp_simple_retransmit(sk);
389 } /* else let the usual retransmit timer handle it */
391 EXPORT_SYMBOL(tcp_v4_mtu_reduced);
393 static void do_redirect(struct sk_buff *skb, struct sock *sk)
395 struct dst_entry *dst = __sk_dst_check(sk, 0);
398 dst->ops->redirect(dst, sk, skb);
402 /* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
403 void tcp_req_err(struct sock *sk, u32 seq, bool abort)
405 struct request_sock *req = inet_reqsk(sk);
406 struct net *net = sock_net(sk);
408 /* ICMPs are not backlogged, hence we cannot get
409 * an established socket here.
411 if (seq != tcp_rsk(req)->snt_isn) {
412 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
415 * Still in SYN_RECV, just remove it silently.
416 * There is no good way to pass the error to the newly
417 * created socket, and POSIX does not want network
418 * errors returned from accept().
420 inet_csk_reqsk_queue_drop(req->rsk_listener, req);
421 tcp_listendrop(req->rsk_listener);
425 EXPORT_SYMBOL(tcp_req_err);
427 /* TCP-LD (RFC 6069) logic */
428 void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
430 struct inet_connection_sock *icsk = inet_csk(sk);
431 struct tcp_sock *tp = tcp_sk(sk);
436 if (sock_owned_by_user(sk))
439 if (seq != tp->snd_una || !icsk->icsk_retransmits ||
443 skb = tcp_rtx_queue_head(sk);
444 if (WARN_ON_ONCE(!skb))
447 icsk->icsk_backoff--;
448 icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
449 icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
451 tcp_mstamp_refresh(tp);
452 delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
453 remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
456 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
457 remaining, TCP_RTO_MAX);
459 /* RTO revert clocked out retransmission.
460 * Will retransmit now.
462 tcp_retransmit_timer(sk);
465 EXPORT_SYMBOL(tcp_ld_RTO_revert);
468 * This routine is called by the ICMP module when it gets some
469 * sort of error condition. If err < 0 then the socket should
470 * be closed and the error returned to the user. If err > 0
471 * it's just the icmp type << 8 | icmp code. After adjustment
472 * header points to the first 8 bytes of the tcp header. We need
473 * to find the appropriate port.
475 * The locking strategy used here is very "optimistic". When
476 * someone else accesses the socket the ICMP is just dropped
477 * and for some paths there is no check at all.
478 * A more general error queue to queue errors for later handling
479 * is probably better.
483 int tcp_v4_err(struct sk_buff *skb, u32 info)
485 const struct iphdr *iph = (const struct iphdr *)skb->data;
486 struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
488 struct inet_sock *inet;
489 const int type = icmp_hdr(skb)->type;
490 const int code = icmp_hdr(skb)->code;
492 struct request_sock *fastopen;
495 struct net *net = dev_net(skb->dev);
497 sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr,
498 th->dest, iph->saddr, ntohs(th->source),
501 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
504 if (sk->sk_state == TCP_TIME_WAIT) {
505 inet_twsk_put(inet_twsk(sk));
508 seq = ntohl(th->seq);
509 if (sk->sk_state == TCP_NEW_SYN_RECV) {
510 tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
511 type == ICMP_TIME_EXCEEDED ||
512 (type == ICMP_DEST_UNREACH &&
513 (code == ICMP_NET_UNREACH ||
514 code == ICMP_HOST_UNREACH)));
519 /* If too many ICMPs get dropped on busy
520 * servers this needs to be solved differently.
521 * We do take care of PMTU discovery (RFC1191) special case :
522 * we can receive locally generated ICMP messages while socket is held.
524 if (sock_owned_by_user(sk)) {
525 if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
526 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
528 if (sk->sk_state == TCP_CLOSE)
531 if (static_branch_unlikely(&ip4_min_ttl)) {
532 /* min_ttl can be changed concurrently from do_ip_setsockopt() */
533 if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
534 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
540 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
541 fastopen = rcu_dereference(tp->fastopen_rsk);
542 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
543 if (sk->sk_state != TCP_LISTEN &&
544 !between(seq, snd_una, tp->snd_nxt)) {
545 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
551 if (!sock_owned_by_user(sk))
552 do_redirect(skb, sk);
554 case ICMP_SOURCE_QUENCH:
555 /* Just silently ignore these. */
557 case ICMP_PARAMETERPROB:
560 case ICMP_DEST_UNREACH:
561 if (code > NR_ICMP_UNREACH)
564 if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
565 /* We are not interested in TCP_LISTEN and open_requests
566 * (SYN-ACKs send out by Linux are always <576bytes so
567 * they should go through unfragmented).
569 if (sk->sk_state == TCP_LISTEN)
572 WRITE_ONCE(tp->mtu_info, info);
573 if (!sock_owned_by_user(sk)) {
574 tcp_v4_mtu_reduced(sk);
576 if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
582 err = icmp_err_convert[code].errno;
583 /* check if this ICMP message allows revert of backoff.
587 (code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH))
588 tcp_ld_RTO_revert(sk, seq);
590 case ICMP_TIME_EXCEEDED:
597 switch (sk->sk_state) {
600 /* Only in fast or simultaneous open. If a fast open socket is
601 * already accepted it is treated as a connected one below.
603 if (fastopen && !fastopen->sk)
606 ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
608 if (!sock_owned_by_user(sk)) {
615 sk->sk_err_soft = err;
620 /* If we've already connected we will keep trying
621 * until we time out, or the user gives up.
623 * rfc1122 4.2.3.9 allows to consider as hard errors
624 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
625 * but it is obsoleted by pmtu discovery).
627 * Note, that in modern internet, where routing is unreliable
628 * and in each dark corner broken firewalls sit, sending random
629 * errors ordered by their masters even this two messages finally lose
630 * their original sense (even Linux sends invalid PORT_UNREACHs)
632 * Now we are in compliance with RFCs.
637 if (!sock_owned_by_user(sk) && inet->recverr) {
640 } else { /* Only an error on timeout */
641 sk->sk_err_soft = err;
650 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
652 struct tcphdr *th = tcp_hdr(skb);
654 th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
655 skb->csum_start = skb_transport_header(skb) - skb->head;
656 skb->csum_offset = offsetof(struct tcphdr, check);
659 /* This routine computes an IPv4 TCP checksum. */
660 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
662 const struct inet_sock *inet = inet_sk(sk);
664 __tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
666 EXPORT_SYMBOL(tcp_v4_send_check);
669 * This routine will send an RST to the other tcp.
671 * Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
673 * Answer: if a packet caused RST, it is not for a socket
674 * existing in our system, if it is matched to a socket,
675 * it is just duplicate segment or bug in other side's TCP.
676 * So that we build reply only basing on parameters
677 * arrived with segment.
678 * Exception: precedence violation. We do not implement it in any case.
681 #ifdef CONFIG_TCP_MD5SIG
682 #define OPTION_BYTES TCPOLEN_MD5SIG_ALIGNED
684 #define OPTION_BYTES sizeof(__be32)
687 static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
689 const struct tcphdr *th = tcp_hdr(skb);
692 __be32 opt[OPTION_BYTES / sizeof(__be32)];
694 struct ip_reply_arg arg;
695 #ifdef CONFIG_TCP_MD5SIG
696 struct tcp_md5sig_key *key = NULL;
697 const __u8 *hash_location = NULL;
698 unsigned char newhash[16];
700 struct sock *sk1 = NULL;
702 u64 transmit_time = 0;
706 /* Never send a reset in response to a reset. */
710 /* If sk not NULL, it means we did a successful lookup and incoming
711 * route had to be correct. prequeue might have dropped our dst.
713 if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
716 /* Swap the send and the receive. */
717 memset(&rep, 0, sizeof(rep));
718 rep.th.dest = th->source;
719 rep.th.source = th->dest;
720 rep.th.doff = sizeof(struct tcphdr) / 4;
724 rep.th.seq = th->ack_seq;
727 rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
728 skb->len - (th->doff << 2));
731 memset(&arg, 0, sizeof(arg));
732 arg.iov[0].iov_base = (unsigned char *)&rep;
733 arg.iov[0].iov_len = sizeof(rep.th);
735 net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
736 #ifdef CONFIG_TCP_MD5SIG
738 hash_location = tcp_parse_md5sig_option(th);
739 if (sk && sk_fullsock(sk)) {
740 const union tcp_md5_addr *addr;
743 /* sdif set, means packet ingressed via a device
744 * in an L3 domain and inet_iif is set to it.
746 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
747 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
748 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
749 } else if (hash_location) {
750 const union tcp_md5_addr *addr;
751 int sdif = tcp_v4_sdif(skb);
752 int dif = inet_iif(skb);
756 * active side is lost. Try to find listening socket through
757 * source port, and then find md5 key through listening socket.
758 * we are not loose security here:
759 * Incoming packet is checked with md5 hash with finding key,
760 * no RST generated if md5 hash doesn't match.
762 sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0,
764 th->source, ip_hdr(skb)->daddr,
765 ntohs(th->source), dif, sdif);
766 /* don't send rst if it can't find key */
770 /* sdif set, means packet ingressed via a device
771 * in an L3 domain and dif is set to it.
773 l3index = sdif ? dif : 0;
774 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
775 key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
780 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
781 if (genhash || memcmp(hash_location, newhash, 16) != 0)
787 rep.opt[0] = htonl((TCPOPT_NOP << 24) |
789 (TCPOPT_MD5SIG << 8) |
791 /* Update length and the length the header thinks exists */
792 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
793 rep.th.doff = arg.iov[0].iov_len / 4;
795 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
796 key, ip_hdr(skb)->saddr,
797 ip_hdr(skb)->daddr, &rep.th);
800 /* Can't co-exist with TCPMD5, hence check rep.opt[0] */
801 if (rep.opt[0] == 0) {
802 __be32 mrst = mptcp_reset_option(skb);
806 arg.iov[0].iov_len += sizeof(mrst);
807 rep.th.doff = arg.iov[0].iov_len / 4;
811 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
812 ip_hdr(skb)->saddr, /* XXX */
813 arg.iov[0].iov_len, IPPROTO_TCP, 0);
814 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
815 arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
817 /* When socket is gone, all binding information is lost.
818 * routing might fail in this case. No choice here, if we choose to force
819 * input interface, we will misroute in case of asymmetric route.
822 arg.bound_dev_if = sk->sk_bound_dev_if;
824 trace_tcp_send_reset(sk, skb);
827 BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
828 offsetof(struct inet_timewait_sock, tw_bound_dev_if));
830 arg.tos = ip_hdr(skb)->tos;
831 arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
833 ctl_sk = this_cpu_read(ipv4_tcp_sk);
834 sock_net_set(ctl_sk, net);
836 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
837 inet_twsk(sk)->tw_mark : sk->sk_mark;
838 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
839 inet_twsk(sk)->tw_priority : sk->sk_priority;
840 transmit_time = tcp_transmit_time(sk);
841 xfrm_sk_clone_policy(ctl_sk, sk);
843 ip_send_unicast_reply(ctl_sk,
844 skb, &TCP_SKB_CB(skb)->header.h4.opt,
845 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
846 &arg, arg.iov[0].iov_len,
850 xfrm_sk_free_policy(ctl_sk);
851 sock_net_set(ctl_sk, &init_net);
852 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
853 __TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
856 #ifdef CONFIG_TCP_MD5SIG
862 /* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
863 outside socket context is ugly, certainly. What can I do?
866 static void tcp_v4_send_ack(const struct sock *sk,
867 struct sk_buff *skb, u32 seq, u32 ack,
868 u32 win, u32 tsval, u32 tsecr, int oif,
869 struct tcp_md5sig_key *key,
870 int reply_flags, u8 tos)
872 const struct tcphdr *th = tcp_hdr(skb);
875 __be32 opt[(TCPOLEN_TSTAMP_ALIGNED >> 2)
876 #ifdef CONFIG_TCP_MD5SIG
877 + (TCPOLEN_MD5SIG_ALIGNED >> 2)
881 struct net *net = sock_net(sk);
882 struct ip_reply_arg arg;
886 memset(&rep.th, 0, sizeof(struct tcphdr));
887 memset(&arg, 0, sizeof(arg));
889 arg.iov[0].iov_base = (unsigned char *)&rep;
890 arg.iov[0].iov_len = sizeof(rep.th);
892 rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
893 (TCPOPT_TIMESTAMP << 8) |
895 rep.opt[1] = htonl(tsval);
896 rep.opt[2] = htonl(tsecr);
897 arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
900 /* Swap the send and the receive. */
901 rep.th.dest = th->source;
902 rep.th.source = th->dest;
903 rep.th.doff = arg.iov[0].iov_len / 4;
904 rep.th.seq = htonl(seq);
905 rep.th.ack_seq = htonl(ack);
907 rep.th.window = htons(win);
909 #ifdef CONFIG_TCP_MD5SIG
911 int offset = (tsecr) ? 3 : 0;
913 rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
915 (TCPOPT_MD5SIG << 8) |
917 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
918 rep.th.doff = arg.iov[0].iov_len/4;
920 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
921 key, ip_hdr(skb)->saddr,
922 ip_hdr(skb)->daddr, &rep.th);
925 arg.flags = reply_flags;
926 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
927 ip_hdr(skb)->saddr, /* XXX */
928 arg.iov[0].iov_len, IPPROTO_TCP, 0);
929 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
931 arg.bound_dev_if = oif;
933 arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
935 ctl_sk = this_cpu_read(ipv4_tcp_sk);
936 sock_net_set(ctl_sk, net);
937 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
938 inet_twsk(sk)->tw_mark : sk->sk_mark;
939 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
940 inet_twsk(sk)->tw_priority : sk->sk_priority;
941 transmit_time = tcp_transmit_time(sk);
942 ip_send_unicast_reply(ctl_sk,
943 skb, &TCP_SKB_CB(skb)->header.h4.opt,
944 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
945 &arg, arg.iov[0].iov_len,
949 sock_net_set(ctl_sk, &init_net);
950 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
954 static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
956 struct inet_timewait_sock *tw = inet_twsk(sk);
957 struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
959 tcp_v4_send_ack(sk, skb,
960 tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
961 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
962 tcp_time_stamp_raw() + tcptw->tw_ts_offset,
965 tcp_twsk_md5_key(tcptw),
966 tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
973 static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
974 struct request_sock *req)
976 const union tcp_md5_addr *addr;
979 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
980 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
982 u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
986 * The window field (SEG.WND) of every outgoing segment, with the
987 * exception of <SYN> segments, MUST be right-shifted by
988 * Rcv.Wind.Shift bits:
990 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
991 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
992 tcp_v4_send_ack(sk, skb, seq,
993 tcp_rsk(req)->rcv_nxt,
994 req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
995 tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
998 tcp_md5_do_lookup(sk, l3index, addr, AF_INET),
999 inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
1004 * Send a SYN-ACK after having received a SYN.
1005 * This still operates on a request_sock only, not on a big
1008 static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
1010 struct request_sock *req,
1011 struct tcp_fastopen_cookie *foc,
1012 enum tcp_synack_type synack_type,
1013 struct sk_buff *syn_skb)
1015 const struct inet_request_sock *ireq = inet_rsk(req);
1018 struct sk_buff *skb;
1021 /* First, grab a route. */
1022 if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
1025 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
1028 __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
1030 tos = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
1031 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
1032 (inet_sk(sk)->tos & INET_ECN_MASK) :
1035 if (!INET_ECN_is_capable(tos) &&
1036 tcp_bpf_ca_needs_ecn((struct sock *)req))
1037 tos |= INET_ECN_ECT_0;
1040 err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
1042 rcu_dereference(ireq->ireq_opt),
1045 err = net_xmit_eval(err);
1052 * IPv4 request_sock destructor.
1054 static void tcp_v4_reqsk_destructor(struct request_sock *req)
1056 kfree(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1));
1059 #ifdef CONFIG_TCP_MD5SIG
1061 * RFC2385 MD5 checksumming requires a mapping of
1062 * IP address->MD5 Key.
1063 * We need to maintain these in the sk structure.
1066 DEFINE_STATIC_KEY_FALSE(tcp_md5_needed);
1067 EXPORT_SYMBOL(tcp_md5_needed);
1069 static bool better_md5_match(struct tcp_md5sig_key *old, struct tcp_md5sig_key *new)
1074 /* l3index always overrides non-l3index */
1075 if (old->l3index && new->l3index == 0)
1077 if (old->l3index == 0 && new->l3index)
1080 return old->prefixlen < new->prefixlen;
1083 /* Find the Key structure for an address. */
1084 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1085 const union tcp_md5_addr *addr,
1088 const struct tcp_sock *tp = tcp_sk(sk);
1089 struct tcp_md5sig_key *key;
1090 const struct tcp_md5sig_info *md5sig;
1092 struct tcp_md5sig_key *best_match = NULL;
1095 /* caller either holds rcu_read_lock() or socket lock */
1096 md5sig = rcu_dereference_check(tp->md5sig_info,
1097 lockdep_sock_is_held(sk));
1101 hlist_for_each_entry_rcu(key, &md5sig->head, node,
1102 lockdep_sock_is_held(sk)) {
1103 if (key->family != family)
1105 if (key->flags & TCP_MD5SIG_FLAG_IFINDEX && key->l3index != l3index)
1107 if (family == AF_INET) {
1108 mask = inet_make_mask(key->prefixlen);
1109 match = (key->addr.a4.s_addr & mask) ==
1110 (addr->a4.s_addr & mask);
1111 #if IS_ENABLED(CONFIG_IPV6)
1112 } else if (family == AF_INET6) {
1113 match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
1120 if (match && better_md5_match(best_match, key))
1125 EXPORT_SYMBOL(__tcp_md5_do_lookup);
1127 static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
1128 const union tcp_md5_addr *addr,
1129 int family, u8 prefixlen,
1130 int l3index, u8 flags)
1132 const struct tcp_sock *tp = tcp_sk(sk);
1133 struct tcp_md5sig_key *key;
1134 unsigned int size = sizeof(struct in_addr);
1135 const struct tcp_md5sig_info *md5sig;
1137 /* caller either holds rcu_read_lock() or socket lock */
1138 md5sig = rcu_dereference_check(tp->md5sig_info,
1139 lockdep_sock_is_held(sk));
1142 #if IS_ENABLED(CONFIG_IPV6)
1143 if (family == AF_INET6)
1144 size = sizeof(struct in6_addr);
1146 hlist_for_each_entry_rcu(key, &md5sig->head, node,
1147 lockdep_sock_is_held(sk)) {
1148 if (key->family != family)
1150 if ((key->flags & TCP_MD5SIG_FLAG_IFINDEX) != (flags & TCP_MD5SIG_FLAG_IFINDEX))
1152 if (key->l3index != l3index)
1154 if (!memcmp(&key->addr, addr, size) &&
1155 key->prefixlen == prefixlen)
1161 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1162 const struct sock *addr_sk)
1164 const union tcp_md5_addr *addr;
1167 l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
1168 addr_sk->sk_bound_dev_if);
1169 addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
1170 return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1172 EXPORT_SYMBOL(tcp_v4_md5_lookup);
1174 /* This can be called on a newly created socket, from other files */
1175 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1176 int family, u8 prefixlen, int l3index, u8 flags,
1177 const u8 *newkey, u8 newkeylen, gfp_t gfp)
1179 /* Add Key to the list */
1180 struct tcp_md5sig_key *key;
1181 struct tcp_sock *tp = tcp_sk(sk);
1182 struct tcp_md5sig_info *md5sig;
1184 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
1186 /* Pre-existing entry - just update that one.
1187 * Note that the key might be used concurrently.
1188 * data_race() is telling kcsan that we do not care of
1189 * key mismatches, since changing MD5 key on live flows
1190 * can lead to packet drops.
1192 data_race(memcpy(key->key, newkey, newkeylen));
1194 /* Pairs with READ_ONCE() in tcp_md5_hash_key().
1195 * Also note that a reader could catch new key->keylen value
1196 * but old key->key[], this is the reason we use __GFP_ZERO
1197 * at sock_kmalloc() time below these lines.
1199 WRITE_ONCE(key->keylen, newkeylen);
1204 md5sig = rcu_dereference_protected(tp->md5sig_info,
1205 lockdep_sock_is_held(sk));
1207 md5sig = kmalloc(sizeof(*md5sig), gfp);
1212 INIT_HLIST_HEAD(&md5sig->head);
1213 rcu_assign_pointer(tp->md5sig_info, md5sig);
1216 key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO);
1219 if (!tcp_alloc_md5sig_pool()) {
1220 sock_kfree_s(sk, key, sizeof(*key));
1224 memcpy(key->key, newkey, newkeylen);
1225 key->keylen = newkeylen;
1226 key->family = family;
1227 key->prefixlen = prefixlen;
1228 key->l3index = l3index;
1230 memcpy(&key->addr, addr,
1231 (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6) ? sizeof(struct in6_addr) :
1232 sizeof(struct in_addr));
1233 hlist_add_head_rcu(&key->node, &md5sig->head);
1236 EXPORT_SYMBOL(tcp_md5_do_add);
1238 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
1239 u8 prefixlen, int l3index, u8 flags)
1241 struct tcp_md5sig_key *key;
1243 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
1246 hlist_del_rcu(&key->node);
1247 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1248 kfree_rcu(key, rcu);
1251 EXPORT_SYMBOL(tcp_md5_do_del);
1253 static void tcp_clear_md5_list(struct sock *sk)
1255 struct tcp_sock *tp = tcp_sk(sk);
1256 struct tcp_md5sig_key *key;
1257 struct hlist_node *n;
1258 struct tcp_md5sig_info *md5sig;
1260 md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
1262 hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
1263 hlist_del_rcu(&key->node);
1264 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1265 kfree_rcu(key, rcu);
1269 static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
1270 sockptr_t optval, int optlen)
1272 struct tcp_md5sig cmd;
1273 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
1274 const union tcp_md5_addr *addr;
1279 if (optlen < sizeof(cmd))
1282 if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
1285 if (sin->sin_family != AF_INET)
1288 flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
1290 if (optname == TCP_MD5SIG_EXT &&
1291 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
1292 prefixlen = cmd.tcpm_prefixlen;
1297 if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
1298 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
1299 struct net_device *dev;
1302 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
1303 if (dev && netif_is_l3_master(dev))
1304 l3index = dev->ifindex;
1308 /* ok to reference set/not set outside of rcu;
1309 * right now device MUST be an L3 master
1311 if (!dev || !l3index)
1315 addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;
1317 if (!cmd.tcpm_keylen)
1318 return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index, flags);
1320 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1323 return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index, flags,
1324 cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL);
1327 static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp,
1328 __be32 daddr, __be32 saddr,
1329 const struct tcphdr *th, int nbytes)
1331 struct tcp4_pseudohdr *bp;
1332 struct scatterlist sg;
1339 bp->protocol = IPPROTO_TCP;
1340 bp->len = cpu_to_be16(nbytes);
1342 _th = (struct tcphdr *)(bp + 1);
1343 memcpy(_th, th, sizeof(*th));
1346 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
1347 ahash_request_set_crypt(hp->md5_req, &sg, NULL,
1348 sizeof(*bp) + sizeof(*th));
1349 return crypto_ahash_update(hp->md5_req);
1352 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
1353 __be32 daddr, __be32 saddr, const struct tcphdr *th)
1355 struct tcp_md5sig_pool *hp;
1356 struct ahash_request *req;
1358 hp = tcp_get_md5sig_pool();
1360 goto clear_hash_noput;
1363 if (crypto_ahash_init(req))
1365 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
1367 if (tcp_md5_hash_key(hp, key))
1369 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1370 if (crypto_ahash_final(req))
1373 tcp_put_md5sig_pool();
1377 tcp_put_md5sig_pool();
1379 memset(md5_hash, 0, 16);
1383 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1384 const struct sock *sk,
1385 const struct sk_buff *skb)
1387 struct tcp_md5sig_pool *hp;
1388 struct ahash_request *req;
1389 const struct tcphdr *th = tcp_hdr(skb);
1390 __be32 saddr, daddr;
1392 if (sk) { /* valid for establish/request sockets */
1393 saddr = sk->sk_rcv_saddr;
1394 daddr = sk->sk_daddr;
1396 const struct iphdr *iph = ip_hdr(skb);
1401 hp = tcp_get_md5sig_pool();
1403 goto clear_hash_noput;
1406 if (crypto_ahash_init(req))
1409 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len))
1411 if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
1413 if (tcp_md5_hash_key(hp, key))
1415 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1416 if (crypto_ahash_final(req))
1419 tcp_put_md5sig_pool();
1423 tcp_put_md5sig_pool();
1425 memset(md5_hash, 0, 16);
1428 EXPORT_SYMBOL(tcp_v4_md5_hash_skb);
1432 static void tcp_v4_init_req(struct request_sock *req,
1433 const struct sock *sk_listener,
1434 struct sk_buff *skb)
1436 struct inet_request_sock *ireq = inet_rsk(req);
1437 struct net *net = sock_net(sk_listener);
1439 sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
1440 sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
1441 RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
1444 static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
1445 struct sk_buff *skb,
1447 struct request_sock *req)
1449 tcp_v4_init_req(req, sk, skb);
1451 if (security_inet_conn_request(sk, skb, req))
1454 return inet_csk_route_req(sk, &fl->u.ip4, req);
1457 struct request_sock_ops tcp_request_sock_ops __read_mostly = {
1459 .obj_size = sizeof(struct tcp_request_sock),
1460 .rtx_syn_ack = tcp_rtx_synack,
1461 .send_ack = tcp_v4_reqsk_send_ack,
1462 .destructor = tcp_v4_reqsk_destructor,
1463 .send_reset = tcp_v4_send_reset,
1464 .syn_ack_timeout = tcp_syn_ack_timeout,
1467 const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1468 .mss_clamp = TCP_MSS_DEFAULT,
1469 #ifdef CONFIG_TCP_MD5SIG
1470 .req_md5_lookup = tcp_v4_md5_lookup,
1471 .calc_md5_hash = tcp_v4_md5_hash_skb,
1473 #ifdef CONFIG_SYN_COOKIES
1474 .cookie_init_seq = cookie_v4_init_sequence,
1476 .route_req = tcp_v4_route_req,
1477 .init_seq = tcp_v4_init_seq,
1478 .init_ts_off = tcp_v4_init_ts_off,
1479 .send_synack = tcp_v4_send_synack,
1482 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
1484 /* Never answer to SYNs send to broadcast or multicast */
1485 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
1488 return tcp_conn_request(&tcp_request_sock_ops,
1489 &tcp_request_sock_ipv4_ops, sk, skb);
1495 EXPORT_SYMBOL(tcp_v4_conn_request);
1499 * The three way handshake has completed - we got a valid synack -
1500 * now create the new socket.
1502 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1503 struct request_sock *req,
1504 struct dst_entry *dst,
1505 struct request_sock *req_unhash,
1508 struct inet_request_sock *ireq;
1509 bool found_dup_sk = false;
1510 struct inet_sock *newinet;
1511 struct tcp_sock *newtp;
1513 #ifdef CONFIG_TCP_MD5SIG
1514 const union tcp_md5_addr *addr;
1515 struct tcp_md5sig_key *key;
1518 struct ip_options_rcu *inet_opt;
1520 if (sk_acceptq_is_full(sk))
1523 newsk = tcp_create_openreq_child(sk, req, skb);
1527 newsk->sk_gso_type = SKB_GSO_TCPV4;
1528 inet_sk_rx_dst_set(newsk, skb);
1530 newtp = tcp_sk(newsk);
1531 newinet = inet_sk(newsk);
1532 ireq = inet_rsk(req);
1533 sk_daddr_set(newsk, ireq->ir_rmt_addr);
1534 sk_rcv_saddr_set(newsk, ireq->ir_loc_addr);
1535 newsk->sk_bound_dev_if = ireq->ir_iif;
1536 newinet->inet_saddr = ireq->ir_loc_addr;
1537 inet_opt = rcu_dereference(ireq->ireq_opt);
1538 RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
1539 newinet->mc_index = inet_iif(skb);
1540 newinet->mc_ttl = ip_hdr(skb)->ttl;
1541 newinet->rcv_tos = ip_hdr(skb)->tos;
1542 inet_csk(newsk)->icsk_ext_hdr_len = 0;
1544 inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1545 newinet->inet_id = prandom_u32();
1547 /* Set ToS of the new socket based upon the value of incoming SYN.
1548 * ECT bits are set later in tcp_init_transfer().
1550 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1551 newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1554 dst = inet_csk_route_child_sock(sk, newsk, req);
1558 /* syncookie case : see end of cookie_v4_check() */
1560 sk_setup_caps(newsk, dst);
1562 tcp_ca_openreq_child(newsk, dst);
1564 tcp_sync_mss(newsk, dst_mtu(dst));
1565 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1567 tcp_initialize_rcv_mss(newsk);
1569 #ifdef CONFIG_TCP_MD5SIG
1570 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1571 /* Copy over the MD5 key from the original socket */
1572 addr = (union tcp_md5_addr *)&newinet->inet_daddr;
1573 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1576 * We're using one, so create a matching key
1577 * on the newsk structure. If we fail to get
1578 * memory, then we end up not copying the key
1581 tcp_md5_do_add(newsk, addr, AF_INET, 32, l3index, key->flags,
1582 key->key, key->keylen, GFP_ATOMIC);
1583 sk_gso_disable(newsk);
1587 if (__inet_inherit_port(sk, newsk) < 0)
1589 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1591 if (likely(*own_req)) {
1592 tcp_move_syn(newtp, req);
1593 ireq->ireq_opt = NULL;
1595 newinet->inet_opt = NULL;
1597 if (!req_unhash && found_dup_sk) {
1598 /* This code path should only be executed in the
1599 * syncookie case only
1601 bh_unlock_sock(newsk);
1609 NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1616 newinet->inet_opt = NULL;
1617 inet_csk_prepare_forced_close(newsk);
1621 EXPORT_SYMBOL(tcp_v4_syn_recv_sock);
1623 static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
1625 #ifdef CONFIG_SYN_COOKIES
1626 const struct tcphdr *th = tcp_hdr(skb);
1629 sk = cookie_v4_check(sk, skb);
1634 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
1635 struct tcphdr *th, u32 *cookie)
1638 #ifdef CONFIG_SYN_COOKIES
1639 mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
1640 &tcp_request_sock_ipv4_ops, sk, th);
1642 *cookie = __cookie_v4_init_sequence(iph, th, &mss);
1643 tcp_synq_overflow(sk);
1649 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1651 /* The socket must have it's spinlock held when we get
1652 * here, unless it is a TCP_LISTEN socket.
1654 * We have a potential double-lock case here, so even when
1655 * doing backlog processing we use the BH locking scheme.
1656 * This is because we cannot sleep with the original spinlock
1659 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
1661 enum skb_drop_reason reason;
1664 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1665 struct dst_entry *dst;
1667 dst = rcu_dereference_protected(sk->sk_rx_dst,
1668 lockdep_sock_is_held(sk));
1670 sock_rps_save_rxhash(sk, skb);
1671 sk_mark_napi_id(sk, skb);
1673 if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1674 !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,
1676 RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1680 tcp_rcv_established(sk, skb);
1684 reason = SKB_DROP_REASON_NOT_SPECIFIED;
1685 if (tcp_checksum_complete(skb))
1688 if (sk->sk_state == TCP_LISTEN) {
1689 struct sock *nsk = tcp_v4_cookie_check(sk, skb);
1694 if (tcp_child_process(sk, nsk, skb)) {
1701 sock_rps_save_rxhash(sk, skb);
1703 if (tcp_rcv_state_process(sk, skb)) {
1710 tcp_v4_send_reset(rsk, skb);
1712 kfree_skb_reason(skb, reason);
1713 /* Be careful here. If this function gets more complicated and
1714 * gcc suffers from register pressure on the x86, sk (in %ebx)
1715 * might be destroyed here. This current version compiles correctly,
1716 * but you have been warned.
1721 reason = SKB_DROP_REASON_TCP_CSUM;
1722 trace_tcp_bad_csum(skb);
1723 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1724 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1727 EXPORT_SYMBOL(tcp_v4_do_rcv);
1729 int tcp_v4_early_demux(struct sk_buff *skb)
1731 const struct iphdr *iph;
1732 const struct tcphdr *th;
1735 if (skb->pkt_type != PACKET_HOST)
1738 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1744 if (th->doff < sizeof(struct tcphdr) / 4)
1747 sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
1748 iph->saddr, th->source,
1749 iph->daddr, ntohs(th->dest),
1750 skb->skb_iif, inet_sdif(skb));
1753 skb->destructor = sock_edemux;
1754 if (sk_fullsock(sk)) {
1755 struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
1758 dst = dst_check(dst, 0);
1760 sk->sk_rx_dst_ifindex == skb->skb_iif)
1761 skb_dst_set_noref(skb, dst);
1767 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
1768 enum skb_drop_reason *reason)
1770 u32 limit, tail_gso_size, tail_gso_segs;
1771 struct skb_shared_info *shinfo;
1772 const struct tcphdr *th;
1773 struct tcphdr *thtail;
1774 struct sk_buff *tail;
1775 unsigned int hdrlen;
1781 /* In case all data was pulled from skb frags (in __pskb_pull_tail()),
1782 * we can fix skb->truesize to its real value to avoid future drops.
1783 * This is valid because skb is not yet charged to the socket.
1784 * It has been noticed pure SACK packets were sometimes dropped
1785 * (if cooked by drivers without copybreak feature).
1791 if (unlikely(tcp_checksum_complete(skb))) {
1793 trace_tcp_bad_csum(skb);
1794 *reason = SKB_DROP_REASON_TCP_CSUM;
1795 __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1796 __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1800 /* Attempt coalescing to last skb in backlog, even if we are
1802 * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
1804 th = (const struct tcphdr *)skb->data;
1805 hdrlen = th->doff * 4;
1807 tail = sk->sk_backlog.tail;
1810 thtail = (struct tcphdr *)tail->data;
1812 if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
1813 TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
1814 ((TCP_SKB_CB(tail)->tcp_flags |
1815 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
1816 !((TCP_SKB_CB(tail)->tcp_flags &
1817 TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
1818 ((TCP_SKB_CB(tail)->tcp_flags ^
1819 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE | TCPHDR_CWR)) ||
1820 #ifdef CONFIG_TLS_DEVICE
1821 tail->decrypted != skb->decrypted ||
1823 thtail->doff != th->doff ||
1824 memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
1827 __skb_pull(skb, hdrlen);
1829 shinfo = skb_shinfo(skb);
1830 gso_size = shinfo->gso_size ?: skb->len;
1831 gso_segs = shinfo->gso_segs ?: 1;
1833 shinfo = skb_shinfo(tail);
1834 tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
1835 tail_gso_segs = shinfo->gso_segs ?: 1;
1837 if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
1838 TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
1840 if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
1841 TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
1842 thtail->window = th->window;
1845 /* We have to update both TCP_SKB_CB(tail)->tcp_flags and
1846 * thtail->fin, so that the fast path in tcp_rcv_established()
1847 * is not entered if we append a packet with a FIN.
1848 * SYN, RST, URG are not present.
1849 * ACK is set on both packets.
1850 * PSH : we do not really care in TCP stack,
1851 * at least for 'GRO' packets.
1853 thtail->fin |= th->fin;
1854 TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1856 if (TCP_SKB_CB(skb)->has_rxtstamp) {
1857 TCP_SKB_CB(tail)->has_rxtstamp = true;
1858 tail->tstamp = skb->tstamp;
1859 skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
1862 /* Not as strict as GRO. We only need to carry mss max value */
1863 shinfo->gso_size = max(gso_size, tail_gso_size);
1864 shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);
1866 sk->sk_backlog.len += delta;
1867 __NET_INC_STATS(sock_net(sk),
1868 LINUX_MIB_TCPBACKLOGCOALESCE);
1869 kfree_skb_partial(skb, fragstolen);
1872 __skb_push(skb, hdrlen);
1875 /* Only socket owner can try to collapse/prune rx queues
1876 * to reduce memory overhead, so add a little headroom here.
1877 * Few sockets backlog are possibly concurrently non empty.
1879 limit = READ_ONCE(sk->sk_rcvbuf) + READ_ONCE(sk->sk_sndbuf) + 64*1024;
1881 if (unlikely(sk_add_backlog(sk, skb, limit))) {
1883 *reason = SKB_DROP_REASON_SOCKET_BACKLOG;
1884 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
1889 EXPORT_SYMBOL(tcp_add_backlog);
1891 int tcp_filter(struct sock *sk, struct sk_buff *skb)
1893 struct tcphdr *th = (struct tcphdr *)skb->data;
1895 return sk_filter_trim_cap(sk, skb, th->doff * 4);
1897 EXPORT_SYMBOL(tcp_filter);
1899 static void tcp_v4_restore_cb(struct sk_buff *skb)
1901 memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
1902 sizeof(struct inet_skb_parm));
1905 static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
1906 const struct tcphdr *th)
1908 /* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
1909 * barrier() makes sure compiler wont play fool^Waliasing games.
1911 memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
1912 sizeof(struct inet_skb_parm));
1915 TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1916 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1917 skb->len - th->doff * 4);
1918 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1919 TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1920 TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1921 TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
1922 TCP_SKB_CB(skb)->sacked = 0;
1923 TCP_SKB_CB(skb)->has_rxtstamp =
1924 skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1931 int tcp_v4_rcv(struct sk_buff *skb)
1933 struct net *net = dev_net(skb->dev);
1934 enum skb_drop_reason drop_reason;
1935 int sdif = inet_sdif(skb);
1936 int dif = inet_iif(skb);
1937 const struct iphdr *iph;
1938 const struct tcphdr *th;
1943 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1944 if (skb->pkt_type != PACKET_HOST)
1947 /* Count it even if it's bad */
1948 __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1950 if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1953 th = (const struct tcphdr *)skb->data;
1955 if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1956 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1959 if (!pskb_may_pull(skb, th->doff * 4))
1962 /* An explanation is required here, I think.
1963 * Packet length and doff are validated by header prediction,
1964 * provided case of th->doff==0 is eliminated.
1965 * So, we defer the checks. */
1967 if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
1970 th = (const struct tcphdr *)skb->data;
1973 sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source,
1974 th->dest, sdif, &refcounted);
1979 if (sk->sk_state == TCP_TIME_WAIT)
1982 if (sk->sk_state == TCP_NEW_SYN_RECV) {
1983 struct request_sock *req = inet_reqsk(sk);
1984 bool req_stolen = false;
1987 sk = req->rsk_listener;
1988 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1989 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1991 drop_reason = tcp_inbound_md5_hash(sk, skb,
1992 &iph->saddr, &iph->daddr,
1993 AF_INET, dif, sdif);
1994 if (unlikely(drop_reason)) {
1995 sk_drops_add(sk, skb);
1999 if (tcp_checksum_complete(skb)) {
2003 if (unlikely(sk->sk_state != TCP_LISTEN)) {
2004 nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
2006 inet_csk_reqsk_queue_drop_and_put(sk, req);
2010 /* reuseport_migrate_sock() has already held one sk_refcnt
2014 /* We own a reference on the listener, increase it again
2015 * as we might lose it too soon.
2021 if (!tcp_filter(sk, skb)) {
2022 th = (const struct tcphdr *)skb->data;
2024 tcp_v4_fill_cb(skb, iph, th);
2025 nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
2027 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
2032 /* Another cpu got exclusive access to req
2033 * and created a full blown socket.
2034 * Try to feed this packet to this socket
2035 * instead of discarding it.
2037 tcp_v4_restore_cb(skb);
2041 goto discard_and_relse;
2046 tcp_v4_restore_cb(skb);
2047 } else if (tcp_child_process(sk, nsk, skb)) {
2048 tcp_v4_send_reset(nsk, skb);
2049 goto discard_and_relse;
2056 if (static_branch_unlikely(&ip4_min_ttl)) {
2057 /* min_ttl can be changed concurrently from do_ip_setsockopt() */
2058 if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
2059 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
2060 goto discard_and_relse;
2064 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
2065 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2066 goto discard_and_relse;
2069 drop_reason = tcp_inbound_md5_hash(sk, skb, &iph->saddr,
2070 &iph->daddr, AF_INET, dif, sdif);
2072 goto discard_and_relse;
2076 if (tcp_filter(sk, skb)) {
2077 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
2078 goto discard_and_relse;
2080 th = (const struct tcphdr *)skb->data;
2082 tcp_v4_fill_cb(skb, iph, th);
2086 if (sk->sk_state == TCP_LISTEN) {
2087 ret = tcp_v4_do_rcv(sk, skb);
2088 goto put_and_return;
2091 sk_incoming_cpu_update(sk);
2093 bh_lock_sock_nested(sk);
2094 tcp_segs_in(tcp_sk(sk), skb);
2096 if (!sock_owned_by_user(sk)) {
2097 ret = tcp_v4_do_rcv(sk, skb);
2099 if (tcp_add_backlog(sk, skb, &drop_reason))
2100 goto discard_and_relse;
2111 drop_reason = SKB_DROP_REASON_NO_SOCKET;
2112 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2115 tcp_v4_fill_cb(skb, iph, th);
2117 if (tcp_checksum_complete(skb)) {
2119 drop_reason = SKB_DROP_REASON_TCP_CSUM;
2120 trace_tcp_bad_csum(skb);
2121 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
2123 __TCP_INC_STATS(net, TCP_MIB_INERRS);
2125 tcp_v4_send_reset(NULL, skb);
2129 SKB_DR_OR(drop_reason, NOT_SPECIFIED);
2130 /* Discard frame. */
2131 kfree_skb_reason(skb, drop_reason);
2135 sk_drops_add(sk, skb);
2141 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
2142 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2143 inet_twsk_put(inet_twsk(sk));
2147 tcp_v4_fill_cb(skb, iph, th);
2149 if (tcp_checksum_complete(skb)) {
2150 inet_twsk_put(inet_twsk(sk));
2153 switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
2155 struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev),
2158 iph->saddr, th->source,
2159 iph->daddr, th->dest,
2163 inet_twsk_deschedule_put(inet_twsk(sk));
2165 tcp_v4_restore_cb(skb);
2173 tcp_v4_timewait_ack(sk, skb);
2176 tcp_v4_send_reset(sk, skb);
2177 inet_twsk_deschedule_put(inet_twsk(sk));
2179 case TCP_TW_SUCCESS:;
2184 static struct timewait_sock_ops tcp_timewait_sock_ops = {
2185 .twsk_obj_size = sizeof(struct tcp_timewait_sock),
2186 .twsk_unique = tcp_twsk_unique,
2187 .twsk_destructor= tcp_twsk_destructor,
2190 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
2192 struct dst_entry *dst = skb_dst(skb);
2194 if (dst && dst_hold_safe(dst)) {
2195 rcu_assign_pointer(sk->sk_rx_dst, dst);
2196 sk->sk_rx_dst_ifindex = skb->skb_iif;
2199 EXPORT_SYMBOL(inet_sk_rx_dst_set);
2201 const struct inet_connection_sock_af_ops ipv4_specific = {
2202 .queue_xmit = ip_queue_xmit,
2203 .send_check = tcp_v4_send_check,
2204 .rebuild_header = inet_sk_rebuild_header,
2205 .sk_rx_dst_set = inet_sk_rx_dst_set,
2206 .conn_request = tcp_v4_conn_request,
2207 .syn_recv_sock = tcp_v4_syn_recv_sock,
2208 .net_header_len = sizeof(struct iphdr),
2209 .setsockopt = ip_setsockopt,
2210 .getsockopt = ip_getsockopt,
2211 .addr2sockaddr = inet_csk_addr2sockaddr,
2212 .sockaddr_len = sizeof(struct sockaddr_in),
2213 .mtu_reduced = tcp_v4_mtu_reduced,
2215 EXPORT_SYMBOL(ipv4_specific);
2217 #ifdef CONFIG_TCP_MD5SIG
2218 static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
2219 .md5_lookup = tcp_v4_md5_lookup,
2220 .calc_md5_hash = tcp_v4_md5_hash_skb,
2221 .md5_parse = tcp_v4_parse_md5_keys,
2225 /* NOTE: A lot of things set to zero explicitly by call to
2226 * sk_alloc() so need not be done here.
2228 static int tcp_v4_init_sock(struct sock *sk)
2230 struct inet_connection_sock *icsk = inet_csk(sk);
2234 icsk->icsk_af_ops = &ipv4_specific;
2236 #ifdef CONFIG_TCP_MD5SIG
2237 tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
2243 void tcp_v4_destroy_sock(struct sock *sk)
2245 struct tcp_sock *tp = tcp_sk(sk);
2247 trace_tcp_destroy_sock(sk);
2249 tcp_clear_xmit_timers(sk);
2251 tcp_cleanup_congestion_control(sk);
2253 tcp_cleanup_ulp(sk);
2255 /* Cleanup up the write buffer. */
2256 tcp_write_queue_purge(sk);
2258 /* Check if we want to disable active TFO */
2259 tcp_fastopen_active_disable_ofo_check(sk);
2261 /* Cleans up our, hopefully empty, out_of_order_queue. */
2262 skb_rbtree_purge(&tp->out_of_order_queue);
2264 #ifdef CONFIG_TCP_MD5SIG
2265 /* Clean up the MD5 key list, if any */
2266 if (tp->md5sig_info) {
2267 tcp_clear_md5_list(sk);
2268 kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu);
2269 tp->md5sig_info = NULL;
2273 /* Clean up a referenced TCP bind bucket. */
2274 if (inet_csk(sk)->icsk_bind_hash)
2277 BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
2279 /* If socket is aborted during connect operation */
2280 tcp_free_fastopen_req(tp);
2281 tcp_fastopen_destroy_cipher(sk);
2282 tcp_saved_syn_free(tp);
2284 sk_sockets_allocated_dec(sk);
2286 EXPORT_SYMBOL(tcp_v4_destroy_sock);
2288 #ifdef CONFIG_PROC_FS
2289 /* Proc filesystem TCP sock list dumping. */
2291 static unsigned short seq_file_family(const struct seq_file *seq);
2293 static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
2295 unsigned short family = seq_file_family(seq);
2297 /* AF_UNSPEC is used as a match all */
2298 return ((family == AF_UNSPEC || family == sk->sk_family) &&
2299 net_eq(sock_net(sk), seq_file_net(seq)));
2302 /* Find a non empty bucket (starting from st->bucket)
2303 * and return the first sk from it.
2305 static void *listening_get_first(struct seq_file *seq)
2307 struct tcp_iter_state *st = seq->private;
2310 for (; st->bucket <= tcp_hashinfo.lhash2_mask; st->bucket++) {
2311 struct inet_listen_hashbucket *ilb2;
2312 struct hlist_nulls_node *node;
2315 ilb2 = &tcp_hashinfo.lhash2[st->bucket];
2316 if (hlist_nulls_empty(&ilb2->nulls_head))
2319 spin_lock(&ilb2->lock);
2320 sk_nulls_for_each(sk, node, &ilb2->nulls_head) {
2321 if (seq_sk_match(seq, sk))
2324 spin_unlock(&ilb2->lock);
2330 /* Find the next sk of "cur" within the same bucket (i.e. st->bucket).
2331 * If "cur" is the last one in the st->bucket,
2332 * call listening_get_first() to return the first sk of the next
2335 static void *listening_get_next(struct seq_file *seq, void *cur)
2337 struct tcp_iter_state *st = seq->private;
2338 struct inet_listen_hashbucket *ilb2;
2339 struct hlist_nulls_node *node;
2340 struct sock *sk = cur;
2345 sk = sk_nulls_next(sk);
2346 sk_nulls_for_each_from(sk, node) {
2347 if (seq_sk_match(seq, sk))
2351 ilb2 = &tcp_hashinfo.lhash2[st->bucket];
2352 spin_unlock(&ilb2->lock);
2354 return listening_get_first(seq);
2357 static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
2359 struct tcp_iter_state *st = seq->private;
2364 rc = listening_get_first(seq);
2366 while (rc && *pos) {
2367 rc = listening_get_next(seq, rc);
2373 static inline bool empty_bucket(const struct tcp_iter_state *st)
2375 return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain);
2379 * Get first established socket starting from bucket given in st->bucket.
2380 * If st->bucket is zero, the very first socket in the hash is returned.
2382 static void *established_get_first(struct seq_file *seq)
2384 struct tcp_iter_state *st = seq->private;
2387 for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
2389 struct hlist_nulls_node *node;
2390 spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket);
2392 /* Lockless fast path for the common case of empty buckets */
2393 if (empty_bucket(st))
2397 sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
2398 if (seq_sk_match(seq, sk))
2401 spin_unlock_bh(lock);
2407 static void *established_get_next(struct seq_file *seq, void *cur)
2409 struct tcp_iter_state *st = seq->private;
2410 struct hlist_nulls_node *node;
2411 struct sock *sk = cur;
2416 sk = sk_nulls_next(sk);
2418 sk_nulls_for_each_from(sk, node) {
2419 if (seq_sk_match(seq, sk))
2423 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2425 return established_get_first(seq);
2428 static void *established_get_idx(struct seq_file *seq, loff_t pos)
2430 struct tcp_iter_state *st = seq->private;
2434 rc = established_get_first(seq);
2437 rc = established_get_next(seq, rc);
2443 static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
2446 struct tcp_iter_state *st = seq->private;
2448 st->state = TCP_SEQ_STATE_LISTENING;
2449 rc = listening_get_idx(seq, &pos);
2452 st->state = TCP_SEQ_STATE_ESTABLISHED;
2453 rc = established_get_idx(seq, pos);
2459 static void *tcp_seek_last_pos(struct seq_file *seq)
2461 struct tcp_iter_state *st = seq->private;
2462 int bucket = st->bucket;
2463 int offset = st->offset;
2464 int orig_num = st->num;
2467 switch (st->state) {
2468 case TCP_SEQ_STATE_LISTENING:
2469 if (st->bucket > tcp_hashinfo.lhash2_mask)
2471 st->state = TCP_SEQ_STATE_LISTENING;
2472 rc = listening_get_first(seq);
2473 while (offset-- && rc && bucket == st->bucket)
2474 rc = listening_get_next(seq, rc);
2478 st->state = TCP_SEQ_STATE_ESTABLISHED;
2480 case TCP_SEQ_STATE_ESTABLISHED:
2481 if (st->bucket > tcp_hashinfo.ehash_mask)
2483 rc = established_get_first(seq);
2484 while (offset-- && rc && bucket == st->bucket)
2485 rc = established_get_next(seq, rc);
2493 void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
2495 struct tcp_iter_state *st = seq->private;
2498 if (*pos && *pos == st->last_pos) {
2499 rc = tcp_seek_last_pos(seq);
2504 st->state = TCP_SEQ_STATE_LISTENING;
2508 rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
2511 st->last_pos = *pos;
2514 EXPORT_SYMBOL(tcp_seq_start);
2516 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2518 struct tcp_iter_state *st = seq->private;
2521 if (v == SEQ_START_TOKEN) {
2522 rc = tcp_get_idx(seq, 0);
2526 switch (st->state) {
2527 case TCP_SEQ_STATE_LISTENING:
2528 rc = listening_get_next(seq, v);
2530 st->state = TCP_SEQ_STATE_ESTABLISHED;
2533 rc = established_get_first(seq);
2536 case TCP_SEQ_STATE_ESTABLISHED:
2537 rc = established_get_next(seq, v);
2542 st->last_pos = *pos;
2545 EXPORT_SYMBOL(tcp_seq_next);
2547 void tcp_seq_stop(struct seq_file *seq, void *v)
2549 struct tcp_iter_state *st = seq->private;
2551 switch (st->state) {
2552 case TCP_SEQ_STATE_LISTENING:
2553 if (v != SEQ_START_TOKEN)
2554 spin_unlock(&tcp_hashinfo.lhash2[st->bucket].lock);
2556 case TCP_SEQ_STATE_ESTABLISHED:
2558 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2562 EXPORT_SYMBOL(tcp_seq_stop);
2564 static void get_openreq4(const struct request_sock *req,
2565 struct seq_file *f, int i)
2567 const struct inet_request_sock *ireq = inet_rsk(req);
2568 long delta = req->rsk_timer.expires - jiffies;
2570 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2571 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
2576 ntohs(ireq->ir_rmt_port),
2578 0, 0, /* could print option size, but that is af dependent. */
2579 1, /* timers active (only the expire timer) */
2580 jiffies_delta_to_clock_t(delta),
2582 from_kuid_munged(seq_user_ns(f),
2583 sock_i_uid(req->rsk_listener)),
2584 0, /* non standard timer */
2585 0, /* open_requests have no inode */
2590 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
2593 unsigned long timer_expires;
2594 const struct tcp_sock *tp = tcp_sk(sk);
2595 const struct inet_connection_sock *icsk = inet_csk(sk);
2596 const struct inet_sock *inet = inet_sk(sk);
2597 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2598 __be32 dest = inet->inet_daddr;
2599 __be32 src = inet->inet_rcv_saddr;
2600 __u16 destp = ntohs(inet->inet_dport);
2601 __u16 srcp = ntohs(inet->inet_sport);
2605 if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2606 icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2607 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2609 timer_expires = icsk->icsk_timeout;
2610 } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2612 timer_expires = icsk->icsk_timeout;
2613 } else if (timer_pending(&sk->sk_timer)) {
2615 timer_expires = sk->sk_timer.expires;
2618 timer_expires = jiffies;
2621 state = inet_sk_state_load(sk);
2622 if (state == TCP_LISTEN)
2623 rx_queue = READ_ONCE(sk->sk_ack_backlog);
2625 /* Because we don't lock the socket,
2626 * we might find a transient negative value.
2628 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2629 READ_ONCE(tp->copied_seq), 0);
2631 seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2632 "%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
2633 i, src, srcp, dest, destp, state,
2634 READ_ONCE(tp->write_seq) - tp->snd_una,
2637 jiffies_delta_to_clock_t(timer_expires - jiffies),
2638 icsk->icsk_retransmits,
2639 from_kuid_munged(seq_user_ns(f), sock_i_uid(sk)),
2640 icsk->icsk_probes_out,
2642 refcount_read(&sk->sk_refcnt), sk,
2643 jiffies_to_clock_t(icsk->icsk_rto),
2644 jiffies_to_clock_t(icsk->icsk_ack.ato),
2645 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
2647 state == TCP_LISTEN ?
2648 fastopenq->max_qlen :
2649 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
2652 static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2653 struct seq_file *f, int i)
2655 long delta = tw->tw_timer.expires - jiffies;
2659 dest = tw->tw_daddr;
2660 src = tw->tw_rcv_saddr;
2661 destp = ntohs(tw->tw_dport);
2662 srcp = ntohs(tw->tw_sport);
2664 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2665 " %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
2666 i, src, srcp, dest, destp, tw->tw_substate, 0, 0,
2667 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2668 refcount_read(&tw->tw_refcnt), tw);
2673 static int tcp4_seq_show(struct seq_file *seq, void *v)
2675 struct tcp_iter_state *st;
2676 struct sock *sk = v;
2678 seq_setwidth(seq, TMPSZ - 1);
2679 if (v == SEQ_START_TOKEN) {
2680 seq_puts(seq, " sl local_address rem_address st tx_queue "
2681 "rx_queue tr tm->when retrnsmt uid timeout "
2687 if (sk->sk_state == TCP_TIME_WAIT)
2688 get_timewait4_sock(v, seq, st->num);
2689 else if (sk->sk_state == TCP_NEW_SYN_RECV)
2690 get_openreq4(v, seq, st->num);
2692 get_tcp4_sock(v, seq, st->num);
2698 #ifdef CONFIG_BPF_SYSCALL
2699 struct bpf_tcp_iter_state {
2700 struct tcp_iter_state state;
2701 unsigned int cur_sk;
2702 unsigned int end_sk;
2703 unsigned int max_sk;
2704 struct sock **batch;
2705 bool st_bucket_done;
2708 struct bpf_iter__tcp {
2709 __bpf_md_ptr(struct bpf_iter_meta *, meta);
2710 __bpf_md_ptr(struct sock_common *, sk_common);
2711 uid_t uid __aligned(8);
2714 static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
2715 struct sock_common *sk_common, uid_t uid)
2717 struct bpf_iter__tcp ctx;
2719 meta->seq_num--; /* skip SEQ_START_TOKEN */
2721 ctx.sk_common = sk_common;
2723 return bpf_iter_run_prog(prog, &ctx);
2726 static void bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state *iter)
2728 while (iter->cur_sk < iter->end_sk)
2729 sock_put(iter->batch[iter->cur_sk++]);
2732 static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
2733 unsigned int new_batch_sz)
2735 struct sock **new_batch;
2737 new_batch = kvmalloc(sizeof(*new_batch) * new_batch_sz,
2738 GFP_USER | __GFP_NOWARN);
2742 bpf_iter_tcp_put_batch(iter);
2743 kvfree(iter->batch);
2744 iter->batch = new_batch;
2745 iter->max_sk = new_batch_sz;
2750 static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
2751 struct sock *start_sk)
2753 struct bpf_tcp_iter_state *iter = seq->private;
2754 struct tcp_iter_state *st = &iter->state;
2755 struct hlist_nulls_node *node;
2756 unsigned int expected = 1;
2759 sock_hold(start_sk);
2760 iter->batch[iter->end_sk++] = start_sk;
2762 sk = sk_nulls_next(start_sk);
2763 sk_nulls_for_each_from(sk, node) {
2764 if (seq_sk_match(seq, sk)) {
2765 if (iter->end_sk < iter->max_sk) {
2767 iter->batch[iter->end_sk++] = sk;
2772 spin_unlock(&tcp_hashinfo.lhash2[st->bucket].lock);
2777 static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
2778 struct sock *start_sk)
2780 struct bpf_tcp_iter_state *iter = seq->private;
2781 struct tcp_iter_state *st = &iter->state;
2782 struct hlist_nulls_node *node;
2783 unsigned int expected = 1;
2786 sock_hold(start_sk);
2787 iter->batch[iter->end_sk++] = start_sk;
2789 sk = sk_nulls_next(start_sk);
2790 sk_nulls_for_each_from(sk, node) {
2791 if (seq_sk_match(seq, sk)) {
2792 if (iter->end_sk < iter->max_sk) {
2794 iter->batch[iter->end_sk++] = sk;
2799 spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
2804 static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
2806 struct bpf_tcp_iter_state *iter = seq->private;
2807 struct tcp_iter_state *st = &iter->state;
2808 unsigned int expected;
2809 bool resized = false;
2812 /* The st->bucket is done. Directly advance to the next
2813 * bucket instead of having the tcp_seek_last_pos() to skip
2814 * one by one in the current bucket and eventually find out
2815 * it has to advance to the next bucket.
2817 if (iter->st_bucket_done) {
2820 if (st->state == TCP_SEQ_STATE_LISTENING &&
2821 st->bucket > tcp_hashinfo.lhash2_mask) {
2822 st->state = TCP_SEQ_STATE_ESTABLISHED;
2828 /* Get a new batch */
2831 iter->st_bucket_done = false;
2833 sk = tcp_seek_last_pos(seq);
2835 return NULL; /* Done */
2837 if (st->state == TCP_SEQ_STATE_LISTENING)
2838 expected = bpf_iter_tcp_listening_batch(seq, sk);
2840 expected = bpf_iter_tcp_established_batch(seq, sk);
2842 if (iter->end_sk == expected) {
2843 iter->st_bucket_done = true;
2847 if (!resized && !bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2)) {
2855 static void *bpf_iter_tcp_seq_start(struct seq_file *seq, loff_t *pos)
2857 /* bpf iter does not support lseek, so it always
2858 * continue from where it was stop()-ped.
2861 return bpf_iter_tcp_batch(seq);
2863 return SEQ_START_TOKEN;
2866 static void *bpf_iter_tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2868 struct bpf_tcp_iter_state *iter = seq->private;
2869 struct tcp_iter_state *st = &iter->state;
2872 /* Whenever seq_next() is called, the iter->cur_sk is
2873 * done with seq_show(), so advance to the next sk in
2876 if (iter->cur_sk < iter->end_sk) {
2877 /* Keeping st->num consistent in tcp_iter_state.
2878 * bpf_iter_tcp does not use st->num.
2879 * meta.seq_num is used instead.
2882 /* Move st->offset to the next sk in the bucket such that
2883 * the future start() will resume at st->offset in
2884 * st->bucket. See tcp_seek_last_pos().
2887 sock_put(iter->batch[iter->cur_sk++]);
2890 if (iter->cur_sk < iter->end_sk)
2891 sk = iter->batch[iter->cur_sk];
2893 sk = bpf_iter_tcp_batch(seq);
2896 /* Keeping st->last_pos consistent in tcp_iter_state.
2897 * bpf iter does not do lseek, so st->last_pos always equals to *pos.
2899 st->last_pos = *pos;
2903 static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
2905 struct bpf_iter_meta meta;
2906 struct bpf_prog *prog;
2907 struct sock *sk = v;
2912 if (v == SEQ_START_TOKEN)
2915 if (sk_fullsock(sk))
2916 slow = lock_sock_fast(sk);
2918 if (unlikely(sk_unhashed(sk))) {
2923 if (sk->sk_state == TCP_TIME_WAIT) {
2925 } else if (sk->sk_state == TCP_NEW_SYN_RECV) {
2926 const struct request_sock *req = v;
2928 uid = from_kuid_munged(seq_user_ns(seq),
2929 sock_i_uid(req->rsk_listener));
2931 uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
2935 prog = bpf_iter_get_info(&meta, false);
2936 ret = tcp_prog_seq_show(prog, &meta, v, uid);
2939 if (sk_fullsock(sk))
2940 unlock_sock_fast(sk, slow);
2945 static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
2947 struct bpf_tcp_iter_state *iter = seq->private;
2948 struct bpf_iter_meta meta;
2949 struct bpf_prog *prog;
2953 prog = bpf_iter_get_info(&meta, true);
2955 (void)tcp_prog_seq_show(prog, &meta, v, 0);
2958 if (iter->cur_sk < iter->end_sk) {
2959 bpf_iter_tcp_put_batch(iter);
2960 iter->st_bucket_done = false;
2964 static const struct seq_operations bpf_iter_tcp_seq_ops = {
2965 .show = bpf_iter_tcp_seq_show,
2966 .start = bpf_iter_tcp_seq_start,
2967 .next = bpf_iter_tcp_seq_next,
2968 .stop = bpf_iter_tcp_seq_stop,
2971 static unsigned short seq_file_family(const struct seq_file *seq)
2973 const struct tcp_seq_afinfo *afinfo;
2975 #ifdef CONFIG_BPF_SYSCALL
2976 /* Iterated from bpf_iter. Let the bpf prog to filter instead. */
2977 if (seq->op == &bpf_iter_tcp_seq_ops)
2981 /* Iterated from proc fs */
2982 afinfo = pde_data(file_inode(seq->file));
2983 return afinfo->family;
2986 static const struct seq_operations tcp4_seq_ops = {
2987 .show = tcp4_seq_show,
2988 .start = tcp_seq_start,
2989 .next = tcp_seq_next,
2990 .stop = tcp_seq_stop,
2993 static struct tcp_seq_afinfo tcp4_seq_afinfo = {
2997 static int __net_init tcp4_proc_init_net(struct net *net)
2999 if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
3000 sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
3005 static void __net_exit tcp4_proc_exit_net(struct net *net)
3007 remove_proc_entry("tcp", net->proc_net);
3010 static struct pernet_operations tcp4_net_ops = {
3011 .init = tcp4_proc_init_net,
3012 .exit = tcp4_proc_exit_net,
3015 int __init tcp4_proc_init(void)
3017 return register_pernet_subsys(&tcp4_net_ops);
3020 void tcp4_proc_exit(void)
3022 unregister_pernet_subsys(&tcp4_net_ops);
3024 #endif /* CONFIG_PROC_FS */
3026 /* @wake is one when sk_stream_write_space() calls us.
3027 * This sends EPOLLOUT only if notsent_bytes is half the limit.
3028 * This mimics the strategy used in sock_def_write_space().
3030 bool tcp_stream_memory_free(const struct sock *sk, int wake)
3032 const struct tcp_sock *tp = tcp_sk(sk);
3033 u32 notsent_bytes = READ_ONCE(tp->write_seq) -
3034 READ_ONCE(tp->snd_nxt);
3036 return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
3038 EXPORT_SYMBOL(tcp_stream_memory_free);
3040 struct proto tcp_prot = {
3042 .owner = THIS_MODULE,
3044 .pre_connect = tcp_v4_pre_connect,
3045 .connect = tcp_v4_connect,
3046 .disconnect = tcp_disconnect,
3047 .accept = inet_csk_accept,
3049 .init = tcp_v4_init_sock,
3050 .destroy = tcp_v4_destroy_sock,
3051 .shutdown = tcp_shutdown,
3052 .setsockopt = tcp_setsockopt,
3053 .getsockopt = tcp_getsockopt,
3054 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
3055 .keepalive = tcp_set_keepalive,
3056 .recvmsg = tcp_recvmsg,
3057 .sendmsg = tcp_sendmsg,
3058 .sendpage = tcp_sendpage,
3059 .backlog_rcv = tcp_v4_do_rcv,
3060 .release_cb = tcp_release_cb,
3062 .unhash = inet_unhash,
3063 .get_port = inet_csk_get_port,
3064 .put_port = inet_put_port,
3065 #ifdef CONFIG_BPF_SYSCALL
3066 .psock_update_sk_prot = tcp_bpf_update_proto,
3068 .enter_memory_pressure = tcp_enter_memory_pressure,
3069 .leave_memory_pressure = tcp_leave_memory_pressure,
3070 .stream_memory_free = tcp_stream_memory_free,
3071 .sockets_allocated = &tcp_sockets_allocated,
3072 .orphan_count = &tcp_orphan_count,
3074 .memory_allocated = &tcp_memory_allocated,
3075 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc,
3077 .memory_pressure = &tcp_memory_pressure,
3078 .sysctl_mem = sysctl_tcp_mem,
3079 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
3080 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
3081 .max_header = MAX_TCP_HEADER,
3082 .obj_size = sizeof(struct tcp_sock),
3083 .slab_flags = SLAB_TYPESAFE_BY_RCU,
3084 .twsk_prot = &tcp_timewait_sock_ops,
3085 .rsk_prot = &tcp_request_sock_ops,
3087 .no_autobind = true,
3088 .diag_destroy = tcp_abort,
3090 EXPORT_SYMBOL(tcp_prot);
3092 static void __net_exit tcp_sk_exit(struct net *net)
3094 if (net->ipv4.tcp_congestion_control)
3095 bpf_module_put(net->ipv4.tcp_congestion_control,
3096 net->ipv4.tcp_congestion_control->owner);
3099 static int __net_init tcp_sk_init(struct net *net)
3103 net->ipv4.sysctl_tcp_ecn = 2;
3104 net->ipv4.sysctl_tcp_ecn_fallback = 1;
3106 net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
3107 net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
3108 net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
3109 net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
3110 net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
3112 net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
3113 net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
3114 net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
3116 net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
3117 net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
3118 net->ipv4.sysctl_tcp_syncookies = 1;
3119 net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
3120 net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
3121 net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
3122 net->ipv4.sysctl_tcp_orphan_retries = 0;
3123 net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
3124 net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
3125 net->ipv4.sysctl_tcp_tw_reuse = 2;
3126 net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
3128 refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1);
3129 cnt = tcp_hashinfo.ehash_mask + 1;
3130 net->ipv4.tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
3131 net->ipv4.tcp_death_row.hashinfo = &tcp_hashinfo;
3133 net->ipv4.sysctl_max_syn_backlog = max(128, cnt / 128);
3134 net->ipv4.sysctl_tcp_sack = 1;
3135 net->ipv4.sysctl_tcp_window_scaling = 1;
3136 net->ipv4.sysctl_tcp_timestamps = 1;
3137 net->ipv4.sysctl_tcp_early_retrans = 3;
3138 net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
3139 net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior. */
3140 net->ipv4.sysctl_tcp_retrans_collapse = 1;
3141 net->ipv4.sysctl_tcp_max_reordering = 300;
3142 net->ipv4.sysctl_tcp_dsack = 1;
3143 net->ipv4.sysctl_tcp_app_win = 31;
3144 net->ipv4.sysctl_tcp_adv_win_scale = 1;
3145 net->ipv4.sysctl_tcp_frto = 2;
3146 net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
3147 /* This limits the percentage of the congestion window which we
3148 * will allow a single TSO frame to consume. Building TSO frames
3149 * which are too large can cause TCP streams to be bursty.
3151 net->ipv4.sysctl_tcp_tso_win_divisor = 3;
3152 /* Default TSQ limit of 16 TSO segments */
3153 net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536;
3155 /* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */
3156 net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;
3158 net->ipv4.sysctl_tcp_min_tso_segs = 2;
3159 net->ipv4.sysctl_tcp_tso_rtt_log = 9; /* 2^9 = 512 usec */
3160 net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
3161 net->ipv4.sysctl_tcp_autocorking = 1;
3162 net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
3163 net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
3164 net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
3165 if (net != &init_net) {
3166 memcpy(net->ipv4.sysctl_tcp_rmem,
3167 init_net.ipv4.sysctl_tcp_rmem,
3168 sizeof(init_net.ipv4.sysctl_tcp_rmem));
3169 memcpy(net->ipv4.sysctl_tcp_wmem,
3170 init_net.ipv4.sysctl_tcp_wmem,
3171 sizeof(init_net.ipv4.sysctl_tcp_wmem));
3173 net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
3174 net->ipv4.sysctl_tcp_comp_sack_slack_ns = 100 * NSEC_PER_USEC;
3175 net->ipv4.sysctl_tcp_comp_sack_nr = 44;
3176 net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
3177 net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0;
3178 atomic_set(&net->ipv4.tfo_active_disable_times, 0);
3180 /* Reno is always built in */
3181 if (!net_eq(net, &init_net) &&
3182 bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
3183 init_net.ipv4.tcp_congestion_control->owner))
3184 net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
3186 net->ipv4.tcp_congestion_control = &tcp_reno;
3191 static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
3195 inet_twsk_purge(&tcp_hashinfo, AF_INET);
3197 list_for_each_entry(net, net_exit_list, exit_list) {
3198 WARN_ON_ONCE(!refcount_dec_and_test(&net->ipv4.tcp_death_row.tw_refcount));
3199 tcp_fastopen_ctx_destroy(net);
3203 static struct pernet_operations __net_initdata tcp_sk_ops = {
3204 .init = tcp_sk_init,
3205 .exit = tcp_sk_exit,
3206 .exit_batch = tcp_sk_exit_batch,
3209 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3210 DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
3211 struct sock_common *sk_common, uid_t uid)
3213 #define INIT_BATCH_SZ 16
3215 static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
3217 struct bpf_tcp_iter_state *iter = priv_data;
3220 err = bpf_iter_init_seq_net(priv_data, aux);
3224 err = bpf_iter_tcp_realloc_batch(iter, INIT_BATCH_SZ);
3226 bpf_iter_fini_seq_net(priv_data);
3233 static void bpf_iter_fini_tcp(void *priv_data)
3235 struct bpf_tcp_iter_state *iter = priv_data;
3237 bpf_iter_fini_seq_net(priv_data);
3238 kvfree(iter->batch);
3241 static const struct bpf_iter_seq_info tcp_seq_info = {
3242 .seq_ops = &bpf_iter_tcp_seq_ops,
3243 .init_seq_private = bpf_iter_init_tcp,
3244 .fini_seq_private = bpf_iter_fini_tcp,
3245 .seq_priv_size = sizeof(struct bpf_tcp_iter_state),
3248 static const struct bpf_func_proto *
3249 bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id,
3250 const struct bpf_prog *prog)
3253 case BPF_FUNC_setsockopt:
3254 return &bpf_sk_setsockopt_proto;
3255 case BPF_FUNC_getsockopt:
3256 return &bpf_sk_getsockopt_proto;
3262 static struct bpf_iter_reg tcp_reg_info = {
3264 .ctx_arg_info_size = 1,
3266 { offsetof(struct bpf_iter__tcp, sk_common),
3267 PTR_TO_BTF_ID_OR_NULL },
3269 .get_func_proto = bpf_iter_tcp_get_func_proto,
3270 .seq_info = &tcp_seq_info,
3273 static void __init bpf_iter_register(void)
3275 tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
3276 if (bpf_iter_reg_target(&tcp_reg_info))
3277 pr_warn("Warning: could not register bpf iterator tcp\n");
3282 void __init tcp_v4_init(void)
3286 for_each_possible_cpu(cpu) {
3289 res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
3290 IPPROTO_TCP, &init_net);
3292 panic("Failed to create the TCP control socket.\n");
3293 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
3295 /* Please enforce IP_DF and IPID==0 for RST and
3296 * ACK sent in SYN-RECV and TIME-WAIT state.
3298 inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
3300 per_cpu(ipv4_tcp_sk, cpu) = sk;
3302 if (register_pernet_subsys(&tcp_sk_ops))
3303 panic("Failed to create the TCP control socket.\n");
3305 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3306 bpf_iter_register();